From 82ae8a1440ddbef4bb413dd1ca591b5fd1986f75 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Tue, 9 Apr 2024 19:25:20 +0200 Subject: [PATCH 001/379] tracing model via computational graph - DynapcnnNetworkGraph constructor (WIP) --- sinabs/backend/dynapcnn/__init__.py | 3 + .../dynapcnn/dynapcnn_network_graph.py | 93 ++++ sinabs/backend/dynapcnn/graph_tracer.py | 153 ++++++ .../graph_tracer_tester.ipynb | 452 ++++++++++++++++++ .../jit_based_tracer_sinabs.ipynb | 256 ++++++++++ 5 files changed, 957 insertions(+) create mode 100644 sinabs/backend/dynapcnn/dynapcnn_network_graph.py create mode 100644 sinabs/backend/dynapcnn/graph_tracer.py create mode 100644 tests/test_nonsequential/graph_tracer_tester.ipynb create mode 100644 tests/test_nonsequential/jit_based_tracer_sinabs.ipynb diff --git a/sinabs/backend/dynapcnn/__init__.py b/sinabs/backend/dynapcnn/__init__.py index 62e5e7ff..b783baca 100644 --- a/sinabs/backend/dynapcnn/__init__.py +++ b/sinabs/backend/dynapcnn/__init__.py @@ -2,4 +2,7 @@ DynapcnnCompatibleNetwork, DynapcnnNetwork, ) +from .dynapcnn_network_graph import ( + DynapcnnNetworkGraph, +) from .dynapcnn_visualizer import DynapcnnVisualizer diff --git a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py new file mode 100644 index 00000000..6cdc64ad --- /dev/null +++ b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py @@ -0,0 +1,93 @@ +import time +from subprocess import CalledProcessError +from typing import List, Optional, Sequence, Tuple, Union + +import samna +import torch +import torch.nn as nn + +import sinabs + +from .chip_factory import ChipFactory +from .dvs_layer import DVSLayer +from .dynapcnn_layer import DynapcnnLayer +from .io import disable_timestamps, enable_timestamps, open_device, reset_timestamps +from .utils import ( + DEFAULT_IGNORED_LAYER_TYPES, + build_from_list, + convert_model_to_layer_list, + infer_input_shape, + parse_device_id, +) + +from .graph_tracer import GraphTracer + +class DynapcnnNetworkGraph(nn.Module): + """Given a sinabs spiking network, prepare a dynapcnn-compatible network. This can be used to + test the network will be equivalent once on DYNAPCNN. This class also provides utilities to + make the dynapcnn configuration and upload it to DYNAPCNN. + """ + + def __init__( + self, + snn: Union[nn.Sequential, sinabs.Network], + input_shape: Optional[Tuple[int, int, int]] = None, + dvs_input: bool = False, + discretize: bool = True + ): + """ + DynapcnnNetworkGraph: a class turning sinabs networks into dynapcnn + compatible networks, and making dynapcnn configurations. + + Parameters + ---------- + snn: sinabs.Network + SNN that determines the structure of the `DynapcnnNetwork` + input_shape: None or tuple of ints + Shape of the input, convention: (features, height, width) + If None, `snn` needs an InputLayer + dvs_input: bool + Does dynapcnn receive input from its DVS camera? + discretize: bool + If True, discretize the parameters and thresholds. + This is needed for uploading weights to dynapcnn. Set to False only for + testing purposes. + """ + super().__init__() + + # Computational graph from original PyTorch module. + self.graph_tracer = GraphTracer( + snn.analog_model, + torch.randn((1, *input_shape)) # torch.jit needs the batch dimension. + ) + + # This attribute stores the location/core-id of each of the DynapcnnLayers upon placement on chip + self.chip_layers_ordering = [] + + self.input_shape = input_shape # Convert models to sequential + layers = convert_model_to_layer_list( + model=snn.spiking_model, ignore=DEFAULT_IGNORED_LAYER_TYPES + ) + + for i, l in enumerate(layers): + print(i, l) + + # Check if dvs input is expected + if dvs_input: + self.dvs_input = True + else: + self.dvs_input = False + + input_shape = infer_input_shape(layers, input_shape=input_shape) + assert len(input_shape) == 3, "infer_input_shape did not return 3-tuple" + + # Build model from layers + self.sequence = build_from_list( + layers, + in_shape=input_shape, + discretize=discretize, + dvs_input=self.dvs_input, + ) + + # Fix graph + self.sinabs_edges = self.graph_tracer.remove_ignored_nodes(DEFAULT_IGNORED_LAYER_TYPES) \ No newline at end of file diff --git a/sinabs/backend/dynapcnn/graph_tracer.py b/sinabs/backend/dynapcnn/graph_tracer.py new file mode 100644 index 00000000..5f7e8add --- /dev/null +++ b/sinabs/backend/dynapcnn/graph_tracer.py @@ -0,0 +1,153 @@ +import torch +import torch.nn as nn +import re, copy +import numpy as np +import networkx as nx +from typing import Union + +import matplotlib.pyplot as plt + +class GraphTracer(): + def __init__(self, model: Union[nn.Sequential, nn.Module], dummy_input: np.array) -> None: + """ .""" + + trace = torch.jit.trace(model, dummy_input) + _ = trace(dummy_input) + __ = copy.deepcopy(trace) + + self.graph = __.graph + + self.modules_map, self.name_2_indx_map = self.get_named_modules(model) + self.forward_edges = self.get_foward_edges() + self.ATens = self.get_ATen_operations() + self.edges_list = self.get_graph_edges() + + def from_name_2_indx(self, name): + if name in self.name_2_indx_map: + return self.name_2_indx_map[name] + else: + last_indx = None + for _name, indx in self.name_2_indx_map.items(): + last_indx = indx + self.name_2_indx_map[name] = last_indx+1 + return self.name_2_indx_map[name] + + def get_named_modules(self, module: nn.Module): + """ .""" + modules_map = {} + name_2_indx_map = {} + indx = 0 + for name, mod in module.named_modules(): + if name: + modules_map[indx] = mod + name_2_indx_map[name] = indx + indx += 1 + return modules_map, name_2_indx_map + + def get_foward_edges(self): + """ .""" + forward_edges = {} + for node in self.graph.nodes(): + node = str(node) + regex = re.compile(r'%(.*?) :.*prim::CallMethod\[name="forward"\]\(%(.*?), %(.*?)\)') + match = regex.search(node) + if match: + source = match.group(3).replace('_', '') + target = match.group(2).replace('_', '') + result = match.group(1).replace('_', '') + forward_edges[self.from_name_2_indx(result)] = (self.from_name_2_indx(source), self.from_name_2_indx(target)) + + return forward_edges + + def get_graph_edges(self): + """ .""" + edges = [] + last_result = None + + for result_node, forward_edge in self.forward_edges.items(): + src = forward_edge[0] + trg = forward_edge[1] + + if not last_result: + last_result = result_node + edges.append(('input', trg)) + elif src == last_result: + edges.append((edges[-1][1], trg)) + last_result = result_node + else: + scr1, scr2 = self.get_ATen_operands(src) + edges.append((scr1, trg)) + edges.append((scr2, trg)) + last_result = result_node + + edges.append((edges[-1][1], 'output')) + + return edges[1:-1] + + def get_ATen_operands(self, node): + """ .""" + if node in self.ATens: + src1 = self.ATens[node]['args'][1] + src2 = self.ATens[node]['args'][0] + return self.forward_edges[src1][1], self.forward_edges[src2][1] + else: + # throw error + return None, None + + def get_ATen_operations(self): + """ ATen is PyTorch's tensor library backend, which provides a set of operations that operate on + tensors directly. These include arithmetic operations (add, mul, etc.), mathematical + functions (sin, cos, etc.), and tensor manipulation operations (view, reshape, etc.).""" + ATens = {} + for node in self.graph.nodes(): + node = str(node) + regex = re.compile(r'%(.*?) :.*aten::(.*?)\(%(.*?), %(.*?), %(.*?)\)') + + match = regex.search(node) + + if match: + result_node = match.group(1) + operation = match.group(2) + operator1 = self.from_name_2_indx(match.group(3)) + operator2 = self.from_name_2_indx(match.group(4)) + const_operator = match.group(5) + ATens[result_node] = {'op': operation, 'args': (operator1, operator2, const_operator)} + return ATens + + def remove_ignored_nodes(self, default_ignored_nodes): + """ Recreates the edges list based on layers that 'DynapcnnNetwork' will ignored. This + is done by setting the source (target) node of an edge where the source (target) node + will be dropped as the node that originally targeted this node to be dropped. + """ + edges = copy.deepcopy(self.edges_list) + new_edges = [] + + for edge_idx in range(len(edges)): + _src = edges[edge_idx][0] + _trg = edges[edge_idx][1] + + if isinstance(self.modules_map[_src], default_ignored_nodes): + # all edges where node '_src' is target change it to node '_trg' as their target. + for edge in edges: + if edge[1] == _src: + new_edge = (edge[0], _trg) + elif isinstance(self.modules_map[_trg], default_ignored_nodes): + # all edges where node '_trg' is source change it to node '_src' as their source. + for edge in edges: + if edge[0] == _trg: + new_edge = (_src, edge[1]) + else: + new_edge = (_src, _trg) + + if new_edge not in new_edges: + new_edges.append(new_edge) + + return new_edges + + def plot_graph(self): + """ .""" + G = nx.DiGraph(self.edges_list) + layout = nx.spring_layout(G) + nx.draw(G, pos = layout, with_labels=True, node_size=800) + plt.title('GraphTracer (new)') + plt.show() diff --git a/tests/test_nonsequential/graph_tracer_tester.ipynb b/tests/test_nonsequential/graph_tracer_tester.ipynb new file mode 100644 index 00000000..93005a5a --- /dev/null +++ b/tests/test_nonsequential/graph_tracer_tester.ipynb @@ -0,0 +1,452 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 37, + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "import torch.nn as nn\n", + "from typing import Union\n", + "import re, copy\n", + "import numpy as np" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 38, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "torch.manual_seed(0)" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "metadata": {}, + "outputs": [], + "source": [ + "channels = 1\n", + "height = 28\n", + "width = 28\n", + "\n", + "input_dummy = torch.randn((1, channels, height, width))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Graph Tracer" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "metadata": {}, + "outputs": [], + "source": [ + "class GraphTracer():\n", + " def __init__(self, model: Union[nn.Sequential, nn.Module], dummy_input: np.array) -> None:\n", + " \"\"\" .\"\"\"\n", + "\n", + " trace = torch.jit.trace(model, dummy_input)\n", + " _ = trace(dummy_input)\n", + " __ = copy.deepcopy(trace)\n", + "\n", + " self.graph = __.graph\n", + "\n", + " self.modules_map, self.name_2_indx_map = self.get_named_modules(model)\n", + " self.forward_edges = self.get_foward_edges()\n", + " self.ATens = self.get_ATen_operations()\n", + " self.edges_list = self.get_graph_edges()\n", + "\n", + " def from_name_2_indx(self, name):\n", + " if name in self.name_2_indx_map:\n", + " return self.name_2_indx_map[name]\n", + " else:\n", + " last_indx = None\n", + " for _name, indx in self.name_2_indx_map.items():\n", + " last_indx = indx\n", + " self.name_2_indx_map[name] = last_indx+1\n", + " return self.name_2_indx_map[name]\n", + "\n", + " def get_named_modules(self, module: nn.Module):\n", + " \"\"\" .\"\"\"\n", + " modules_map = {}\n", + " name_2_indx_map = {}\n", + " indx = 0\n", + " for name, mod in module.named_modules():\n", + " if name:\n", + " modules_map[indx] = mod\n", + " name_2_indx_map[name] = indx\n", + " indx += 1\n", + " return modules_map, name_2_indx_map\n", + " \n", + " def get_foward_edges(self):\n", + " \"\"\" .\"\"\"\n", + " forward_edges = {}\n", + " for node in self.graph.nodes():\n", + " node = str(node)\n", + " regex = re.compile(r'%(.*?) :.*prim::CallMethod\\[name=\"forward\"\\]\\(%(.*?), %(.*?)\\)')\n", + " match = regex.search(node)\n", + " if match:\n", + " source = match.group(3).replace('_', '')\n", + " target = match.group(2).replace('_', '')\n", + " result = match.group(1).replace('_', '')\n", + " forward_edges[self.from_name_2_indx(result)] = (self.from_name_2_indx(source), self.from_name_2_indx(target))\n", + " \n", + " return forward_edges\n", + "\n", + " def get_graph_edges(self):\n", + " \"\"\" .\"\"\"\n", + " edges = []\n", + " last_result = None\n", + "\n", + " for result_node, forward_edge in self.forward_edges.items():\n", + " src = forward_edge[0]\n", + " trg = forward_edge[1]\n", + "\n", + " if not last_result:\n", + " last_result = result_node\n", + " edges.append(('input', trg))\n", + " elif src == last_result:\n", + " edges.append((edges[-1][1], trg))\n", + " last_result = result_node\n", + " else:\n", + " scr1, scr2 = self.get_ATen_operands(src)\n", + " edges.append((scr1, trg))\n", + " edges.append((scr2, trg))\n", + " last_result = result_node\n", + " \n", + " edges.append((edges[-1][1], 'output'))\n", + "\n", + " return edges[1:-1]\n", + " \n", + " def get_ATen_operands(self, node):\n", + " \"\"\" .\"\"\"\n", + " if node in self.ATens:\n", + " src1 = self.ATens[node]['args'][1]\n", + " src2 = self.ATens[node]['args'][0]\n", + " return self.forward_edges[src1][1], self.forward_edges[src2][1]\n", + " else:\n", + " # throw error\n", + " return None, None\n", + " \n", + " def get_ATen_operations(self):\n", + " \"\"\" ATen is PyTorch's tensor library backend, which provides a set of operations that operate on \n", + " tensors directly. These include arithmetic operations (add, mul, etc.), mathematical \n", + " functions (sin, cos, etc.), and tensor manipulation operations (view, reshape, etc.).\"\"\"\n", + " ATens = {}\n", + " for node in self.graph.nodes():\n", + " node = str(node)\n", + " regex = re.compile(r'%(.*?) :.*aten::(.*?)\\(%(.*?), %(.*?), %(.*?)\\)')\n", + "\n", + " match = regex.search(node)\n", + "\n", + " if match:\n", + " result_node = match.group(1)\n", + " operation = match.group(2)\n", + " operator1 = self.from_name_2_indx(match.group(3))\n", + " operator2 = self.from_name_2_indx(match.group(4))\n", + " const_operator = match.group(5)\n", + " ATens[result_node] = {'op': operation, 'args': (operator1, operator2, const_operator)}\n", + " return ATens\n", + " \n", + " def remove_ignored_nodes(self, default_ignored_nodes):\n", + " \"\"\" Recreates the edges list based on layers that 'DynapcnnNetwork' will ignored. This\n", + " is done by setting the source (target) node of an edge where the source (target) node\n", + " will be dropped as the node that originally targeted this node to be dropped.\n", + " \"\"\"\n", + " edges = copy.deepcopy(self.edges_list[1:-1])\n", + " new_edges = []\n", + "\n", + " for edge_idx in range(len(edges)):\n", + " _src = edges[edge_idx][0]\n", + " _trg = edges[edge_idx][1]\n", + "\n", + " if isinstance(self.modules_map[_src], default_ignored_nodes):\n", + " # all edges where node '_src' is target change it to node '_trg' as their target.\n", + " for edge in edges:\n", + " if edge[1] == _src:\n", + " new_edge = (edge[0], _trg)\n", + " elif isinstance(self.modules_map[_trg], default_ignored_nodes):\n", + " # all edges where node '_trg' is source change it to node '_src' as their source.\n", + " for edge in edges:\n", + " if edge[0] == _trg:\n", + " new_edge = (_src, edge[1])\n", + " else:\n", + " new_edge = (_src, _trg)\n", + " \n", + " if new_edge not in new_edges:\n", + " new_edges.append(new_edge)\n", + "\n", + " return new_edges\n", + " \n", + " def plot_graph(self):\n", + " \"\"\" .\"\"\"\n", + " G = nx.DiGraph(self.edges_list)\n", + " layout = nx.spring_layout(G)\n", + " nx.draw(G, pos = layout, with_labels=True, node_size=800)\n", + " plt.title('GraphTracer (new)')\n", + " plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Tracing 1" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "metadata": {}, + "outputs": [], + "source": [ + "ann1 = nn.Sequential(\n", + " nn.Conv2d(1, 20, 5, 1, bias=False),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(2,2),\n", + " nn.Conv2d(20, 32, 5, 1, bias=False),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(2,2),\n", + " nn.Conv2d(32, 128, 3, 1, bias=False),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(2,2),\n", + " nn.Flatten(),\n", + " nn.Linear(128, 500, bias=False),\n", + " nn.ReLU(),\n", + " nn.Linear(500, 10, bias=False),\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "metadata": {}, + "outputs": [], + "source": [ + "gtracer1 = GraphTracer(ann1, input_dummy)" + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0 Conv2d(1, 20, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", + "1 ReLU()\n", + "2 AvgPool2d(kernel_size=2, stride=2, padding=0)\n", + "3 Conv2d(20, 32, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", + "4 ReLU()\n", + "5 AvgPool2d(kernel_size=2, stride=2, padding=0)\n", + "6 Conv2d(32, 128, kernel_size=(3, 3), stride=(1, 1), bias=False)\n", + "7 ReLU()\n", + "8 AvgPool2d(kernel_size=2, stride=2, padding=0)\n", + "9 Flatten(start_dim=1, end_dim=-1)\n", + "10 Linear(in_features=128, out_features=500, bias=False)\n", + "11 ReLU()\n", + "12 Linear(in_features=500, out_features=10, bias=False)\n" + ] + } + ], + "source": [ + "for name, mod in gtracer1.modules_map.items():\n", + " print(name, mod)" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(0, 1)\n", + "(1, 2)\n", + "(2, 3)\n", + "(3, 4)\n", + "(4, 5)\n", + "(5, 6)\n", + "(6, 7)\n", + "(7, 8)\n", + "(8, 9)\n", + "(9, 10)\n", + "(10, 11)\n", + "(11, 12)\n", + "12\n" + ] + } + ], + "source": [ + "for edge in gtracer1.edges_list:\n", + " print(edge)\n", + "print(len(gtracer1.edges_list))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Tracing 2" + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "metadata": {}, + "outputs": [], + "source": [ + "class ANN(nn.Module):\n", + " def __init__(self) -> None:\n", + " super().__init__()\n", + "\n", + " self.con1 = nn.Conv2d(1, 20, 5, 1, bias=False)\n", + " self.rel1 = nn.ReLU()\n", + " self.pool1 = nn.AvgPool2d(2,2)\n", + " self.conv2 = nn.Conv2d(20, 32, 5, 1, bias=False)\n", + " self.rel2 = nn.ReLU()\n", + " self.pool2 = nn.AvgPool2d(2,2)\n", + " self.conv3 = nn.Conv2d(32, 128, 3, 1, bias=False)\n", + " self.rel3 = nn.ReLU()\n", + " self.pool3 = nn.AvgPool2d(2,2)\n", + " self.flat = nn.Flatten()\n", + " self.fc1 = nn.Linear(128, 500, bias=False)\n", + " self.rel4 = nn.ReLU()\n", + " self.fc2 = nn.Linear(500, 10, bias=False)\n", + "\n", + " def forward(self, x):\n", + " \n", + " con1_out = self.con1(x)\n", + " rel1_out = self.rel1(con1_out)\n", + " pool1_out = self.pool1(rel1_out)\n", + " conv2_out = self.conv2(pool1_out)\n", + " rel2_out = self.rel2(conv2_out)\n", + " pool2_out = self.pool2(rel2_out)\n", + " conv3_out = self.conv3(pool2_out)\n", + " rel3_out = self.rel3(conv3_out)\n", + " pool3_out = self.pool3(rel3_out)\n", + " flat_out = self.flat(pool3_out)\n", + " fc1_out = self.fc1(flat_out)\n", + " rel4_out = self.rel4(fc1_out)\n", + " fc2_out = self.fc2(rel4_out)\n", + "\n", + " return fc2_out\n", + "\n", + "ann2 = ANN()" + ] + }, + { + "cell_type": "code", + "execution_count": 46, + "metadata": {}, + "outputs": [], + "source": [ + "gtracer2 = GraphTracer(ann2, input_dummy)" + ] + }, + { + "cell_type": "code", + "execution_count": 47, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0 Conv2d(1, 20, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", + "1 ReLU()\n", + "2 AvgPool2d(kernel_size=2, stride=2, padding=0)\n", + "3 Conv2d(20, 32, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", + "4 ReLU()\n", + "5 AvgPool2d(kernel_size=2, stride=2, padding=0)\n", + "6 Conv2d(32, 128, kernel_size=(3, 3), stride=(1, 1), bias=False)\n", + "7 ReLU()\n", + "8 AvgPool2d(kernel_size=2, stride=2, padding=0)\n", + "9 Flatten(start_dim=1, end_dim=-1)\n", + "10 Linear(in_features=128, out_features=500, bias=False)\n", + "11 ReLU()\n", + "12 Linear(in_features=500, out_features=10, bias=False)\n" + ] + } + ], + "source": [ + "for name, mod in gtracer2.modules_map.items():\n", + " print(name, mod)" + ] + }, + { + "cell_type": "code", + "execution_count": 48, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(0, 1)\n", + "(1, 2)\n", + "(2, 3)\n", + "(3, 4)\n", + "(4, 5)\n", + "(5, 6)\n", + "(6, 7)\n", + "(7, 8)\n", + "(8, 9)\n", + "(9, 10)\n", + "(10, 11)\n", + "(11, 12)\n", + "12\n" + ] + } + ], + "source": [ + "for edge in gtracer2.edges_list:\n", + " print(edge)\n", + "print(len(gtracer2.edges_list))" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "speck-rescnn", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tests/test_nonsequential/jit_based_tracer_sinabs.ipynb b/tests/test_nonsequential/jit_based_tracer_sinabs.ipynb new file mode 100644 index 00000000..9b313aed --- /dev/null +++ b/tests/test_nonsequential/jit_based_tracer_sinabs.ipynb @@ -0,0 +1,256 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "import torch.nn as nn\n", + "import networkx as nx\n", + "import matplotlib.pyplot as plt\n", + "from sinabs.from_torch import from_model\n", + "from sinabs.backend.dynapcnn import DynapcnnNetwork, DynapcnnNetworkGraph" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "torch.manual_seed(0)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "channels = 1\n", + "height = 28\n", + "width = 28\n", + "\n", + "input_shape = (channels, height, width)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Network Module (pure Pytorch)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "ann = nn.Sequential(\n", + " nn.Conv2d(1, 20, 5, 1, bias=False),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(2,2),\n", + " nn.Conv2d(20, 32, 5, 1, bias=False),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(2,2),\n", + " nn.Conv2d(32, 128, 3, 1, bias=False),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(2,2),\n", + " nn.Flatten(),\n", + " nn.Linear(128, 500, bias=False),\n", + " nn.ReLU(),\n", + " nn.Flatten(),\n", + " nn.Linear(500, 10, bias=False),\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Sinabs Model" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0 Conv2d(1, 20, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", + "1 IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-1.), batch_size=1, num_timesteps=-1)\n", + "2 AvgPool2d(kernel_size=2, stride=2, padding=0)\n", + "3 Conv2d(20, 32, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", + "4 IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-1.), batch_size=1, num_timesteps=-1)\n", + "5 AvgPool2d(kernel_size=2, stride=2, padding=0)\n", + "6 Conv2d(32, 128, kernel_size=(3, 3), stride=(1, 1), bias=False)\n", + "7 IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-1.), batch_size=1, num_timesteps=-1)\n", + "8 AvgPool2d(kernel_size=2, stride=2, padding=0)\n", + "9 Flatten(start_dim=1, end_dim=-1)\n", + "10 Linear(in_features=128, out_features=500, bias=False)\n", + "11 IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-1.), batch_size=1, num_timesteps=-1)\n", + "12 Flatten(start_dim=1, end_dim=-1)\n", + "13 Linear(in_features=500, out_features=10, bias=False)\n", + "14 IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-1.), batch_size=1, num_timesteps=-1)\n" + ] + } + ], + "source": [ + "sinabs_model = from_model(ann, add_spiking_output=True, batch_size=1)\n", + "count = 0\n", + "for l in sinabs_model.spiking_model:\n", + " print(count, l)\n", + " count += 1" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0 Conv2d(1, 20, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", + "1 IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-1.), batch_size=1, num_timesteps=-1)\n", + "2 AvgPool2d(kernel_size=2, stride=2, padding=0)\n", + "3 Conv2d(20, 32, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", + "4 IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-1.), batch_size=1, num_timesteps=-1)\n", + "5 AvgPool2d(kernel_size=2, stride=2, padding=0)\n", + "6 Conv2d(32, 128, kernel_size=(3, 3), stride=(1, 1), bias=False)\n", + "7 IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-1.), batch_size=1, num_timesteps=-1)\n", + "8 AvgPool2d(kernel_size=2, stride=2, padding=0)\n", + "9 Linear(in_features=128, out_features=500, bias=False)\n", + "10 IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-1.), batch_size=1, num_timesteps=-1)\n", + "11 Linear(in_features=500, out_features=10, bias=False)\n", + "12 IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-1.), batch_size=1, num_timesteps=-1)\n" + ] + } + ], + "source": [ + "hw_model = DynapcnnNetworkGraph(\n", + " sinabs_model,\n", + " discretize=True,\n", + " input_shape=input_shape\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(0, 1)\n", + "(1, 2)\n", + "(2, 3)\n", + "(3, 4)\n", + "(4, 5)\n", + "(5, 6)\n", + "(6, 7)\n", + "(7, 8)\n", + "(8, 9)\n", + "(9, 10)\n", + "(10, 11)\n", + "(11, 12)\n", + "(12, 13)\n" + ] + } + ], + "source": [ + "for edge in hw_model.graph_tracer.edges_list:\n", + " print(edge)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(0, 1)\n", + "(1, 2)\n", + "(2, 3)\n", + "(3, 4)\n", + "(4, 5)\n", + "(5, 6)\n", + "(6, 7)\n", + "(7, 8)\n", + "(8, 10)\n", + "(10, 11)\n", + "(11, 13)\n" + ] + } + ], + "source": [ + "for edge in hw_model.sinabs_edges:\n", + " print(edge)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "speck-rescnn", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 12629d56d38b543c3be68b3f02b0f0f521d9334d Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 9 Apr 2024 19:44:56 +0200 Subject: [PATCH 002/379] removing ignored layers and remapping remaining nodes within the original model comp. graph edges --- .../dynapcnn/dynapcnn_network_graph.py | 2 +- sinabs/backend/dynapcnn/graph_tracer.py | 26 ++++++++++++++++--- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py index 6cdc64ad..1c2a49f6 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py @@ -90,4 +90,4 @@ def __init__( ) # Fix graph - self.sinabs_edges = self.graph_tracer.remove_ignored_nodes(DEFAULT_IGNORED_LAYER_TYPES) \ No newline at end of file + self.sinabs_edges, _ = self.graph_tracer.remove_ignored_nodes(DEFAULT_IGNORED_LAYER_TYPES) \ No newline at end of file diff --git a/sinabs/backend/dynapcnn/graph_tracer.py b/sinabs/backend/dynapcnn/graph_tracer.py index 5f7e8add..95014f16 100644 --- a/sinabs/backend/dynapcnn/graph_tracer.py +++ b/sinabs/backend/dynapcnn/graph_tracer.py @@ -120,18 +120,22 @@ def remove_ignored_nodes(self, default_ignored_nodes): will be dropped as the node that originally targeted this node to be dropped. """ edges = copy.deepcopy(self.edges_list) - new_edges = [] + parsed_edges = [] + removed_nodes = [] + # removing ignored nodes from edges. for edge_idx in range(len(edges)): _src = edges[edge_idx][0] _trg = edges[edge_idx][1] if isinstance(self.modules_map[_src], default_ignored_nodes): + removed_nodes.append(_src) # all edges where node '_src' is target change it to node '_trg' as their target. for edge in edges: if edge[1] == _src: new_edge = (edge[0], _trg) elif isinstance(self.modules_map[_trg], default_ignored_nodes): + removed_nodes.append(_trg) # all edges where node '_trg' is source change it to node '_src' as their source. for edge in edges: if edge[0] == _trg: @@ -139,10 +143,24 @@ def remove_ignored_nodes(self, default_ignored_nodes): else: new_edge = (_src, _trg) - if new_edge not in new_edges: - new_edges.append(new_edge) + if new_edge not in parsed_edges: + parsed_edges.append(new_edge) + + # remapping nodes indexes. + remapped_nodes = {} + for node_indx, __ in self.modules_map.items(): + _ = [x for x in removed_nodes if node_indx > x] + remapped_nodes[node_indx] = node_indx - len(_) + + for x in removed_nodes: + del remapped_nodes[x] + + # remapping nodes names in parsed edges. + remapped_edges = [] + for edge in parsed_edges: + remapped_edges.append((remapped_nodes[edge[0]], remapped_nodes[edge[1]])) - return new_edges + return remapped_edges, parsed_edges def plot_graph(self): """ .""" From 9c7c31cd8704435deda7d1e698a795b1574050c9 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Wed, 10 Apr 2024 15:04:52 +0200 Subject: [PATCH 003/379] DynapcnnNetworkGraph constructor - converting sinabs_model.analog_model's graph into the equivalent sinabs_model.spiking_model grpah --- .../dynapcnn/dynapcnn_network_graph.py | 54 ++++++- sinabs/backend/dynapcnn/graph_tracer.py | 12 +- .../jit_based_tracer_sinabs.ipynb | 142 +++++++++++++----- 3 files changed, 155 insertions(+), 53 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py index 1c2a49f6..12094fd8 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py @@ -3,6 +3,7 @@ from typing import List, Optional, Sequence, Tuple, Union import samna +import sinabs.layers import torch import torch.nn as nn @@ -65,29 +66,66 @@ def __init__( self.chip_layers_ordering = [] self.input_shape = input_shape # Convert models to sequential - layers = convert_model_to_layer_list( + self.layers = convert_model_to_layer_list( model=snn.spiking_model, ignore=DEFAULT_IGNORED_LAYER_TYPES ) - for i, l in enumerate(layers): - print(i, l) - # Check if dvs input is expected if dvs_input: self.dvs_input = True else: self.dvs_input = False - input_shape = infer_input_shape(layers, input_shape=input_shape) + input_shape = infer_input_shape(self.layers, input_shape=input_shape) assert len(input_shape) == 3, "infer_input_shape did not return 3-tuple" # Build model from layers self.sequence = build_from_list( - layers, + self.layers, in_shape=input_shape, discretize=discretize, dvs_input=self.dvs_input, ) - # Fix graph - self.sinabs_edges, _ = self.graph_tracer.remove_ignored_nodes(DEFAULT_IGNORED_LAYER_TYPES) \ No newline at end of file + # Get sinabs graph + self.sinabs_edges = self.get_sinabs_edges(snn) + + def get_sinabs_edges(self, sinabs_model): + """ Converts the computational graph extracted from 'sinabs_model.analog_model' into its equivalent + representation for the 'sinabs_model.spiking_model'. + """ + # parse original graph to ammend edges containing nodes dropped in 'convert_model_to_layer_list()'. + sinabs_edges = self.graph_tracer.remove_ignored_nodes(DEFAULT_IGNORED_LAYER_TYPES) + + if DynapcnnNetworkGraph.was_spiking_output_added(sinabs_model): + # spiking output layer has been added: create new edge. + last_edge = sinabs_edges[-1] + new_edge = (last_edge[1], last_edge[1]+1) + sinabs_edges.append(new_edge) + else: + pass + + return sinabs_edges + + @staticmethod + def was_spiking_output_added(sinabs_model): + """ Compares the models outputed by 'sinabs.from_torch.from_model()' to check if + a spiking output was added to the spiking version of the analog model. + """ + analog_modules = [] + spiking_modules = [] + + for mod in sinabs_model.analog_model: + analog_modules.append(mod) + + for mod in sinabs_model.spiking_model: + spiking_modules.append(mod) + + if len(analog_modules) != len(spiking_modules): + if isinstance(spiking_modules[-1], sinabs.layers.iaf.IAFSqueeze): + return True + else: + # throw error + return False + else: + return False \ No newline at end of file diff --git a/sinabs/backend/dynapcnn/graph_tracer.py b/sinabs/backend/dynapcnn/graph_tracer.py index 95014f16..a454b086 100644 --- a/sinabs/backend/dynapcnn/graph_tracer.py +++ b/sinabs/backend/dynapcnn/graph_tracer.py @@ -146,6 +146,8 @@ def remove_ignored_nodes(self, default_ignored_nodes): if new_edge not in parsed_edges: parsed_edges.append(new_edge) + removed_nodes = list(set(removed_nodes)) + # remapping nodes indexes. remapped_nodes = {} for node_indx, __ in self.modules_map.items(): @@ -160,12 +162,12 @@ def remove_ignored_nodes(self, default_ignored_nodes): for edge in parsed_edges: remapped_edges.append((remapped_nodes[edge[0]], remapped_nodes[edge[1]])) - return remapped_edges, parsed_edges + return remapped_edges - def plot_graph(self): + @staticmethod + def plot_graph(edges_list): """ .""" - G = nx.DiGraph(self.edges_list) + G = nx.DiGraph(edges_list) layout = nx.spring_layout(G) nx.draw(G, pos = layout, with_labels=True, node_size=800) - plt.title('GraphTracer (new)') - plt.show() + plt.show() \ No newline at end of file diff --git a/tests/test_nonsequential/jit_based_tracer_sinabs.ipynb b/tests/test_nonsequential/jit_based_tracer_sinabs.ipynb index 9b313aed..6b8dea76 100644 --- a/tests/test_nonsequential/jit_based_tracer_sinabs.ipynb +++ b/tests/test_nonsequential/jit_based_tracer_sinabs.ipynb @@ -22,7 +22,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 2, @@ -78,6 +78,39 @@ ")" ] }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0 Conv2d(1, 20, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", + "1 ReLU()\n", + "2 AvgPool2d(kernel_size=2, stride=2, padding=0)\n", + "3 Conv2d(20, 32, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", + "4 ReLU()\n", + "5 AvgPool2d(kernel_size=2, stride=2, padding=0)\n", + "6 Conv2d(32, 128, kernel_size=(3, 3), stride=(1, 1), bias=False)\n", + "7 ReLU()\n", + "8 AvgPool2d(kernel_size=2, stride=2, padding=0)\n", + "9 Flatten(start_dim=1, end_dim=-1)\n", + "10 Linear(in_features=128, out_features=500, bias=False)\n", + "11 ReLU()\n", + "12 Flatten(start_dim=1, end_dim=-1)\n", + "13 Linear(in_features=500, out_features=10, bias=False)\n" + ] + } + ], + "source": [ + "count = 0\n", + "for l in ann:\n", + " print(count, l)\n", + " count += 1" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -87,7 +120,16 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "sinabs_model = from_model(ann, add_spiking_output=True, batch_size=1)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, "metadata": {}, "outputs": [ { @@ -123,7 +165,6 @@ } ], "source": [ - "sinabs_model = from_model(ann, add_spiking_output=True, batch_size=1)\n", "count = 0\n", "for l in sinabs_model.spiking_model:\n", " print(count, l)\n", @@ -132,7 +173,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 8, "metadata": {}, "outputs": [ { @@ -140,31 +181,41 @@ "output_type": "stream", "text": [ "0 Conv2d(1, 20, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", - "1 IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-1.), batch_size=1, num_timesteps=-1)\n", + "1 ReLU()\n", "2 AvgPool2d(kernel_size=2, stride=2, padding=0)\n", "3 Conv2d(20, 32, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", - "4 IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-1.), batch_size=1, num_timesteps=-1)\n", + "4 ReLU()\n", "5 AvgPool2d(kernel_size=2, stride=2, padding=0)\n", "6 Conv2d(32, 128, kernel_size=(3, 3), stride=(1, 1), bias=False)\n", - "7 IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-1.), batch_size=1, num_timesteps=-1)\n", + "7 ReLU()\n", "8 AvgPool2d(kernel_size=2, stride=2, padding=0)\n", - "9 Linear(in_features=128, out_features=500, bias=False)\n", - "10 IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-1.), batch_size=1, num_timesteps=-1)\n", - "11 Linear(in_features=500, out_features=10, bias=False)\n", - "12 IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-1.), batch_size=1, num_timesteps=-1)\n" + "9 Flatten(start_dim=1, end_dim=-1)\n", + "10 Linear(in_features=128, out_features=500, bias=False)\n", + "11 ReLU()\n", + "12 Flatten(start_dim=1, end_dim=-1)\n", + "13 Linear(in_features=500, out_features=10, bias=False)\n" ] } ], + "source": [ + "count = 0\n", + "for l in sinabs_model.analog_model:\n", + " print(count, l)\n", + " count += 1" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## DynapCNN Model" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], "source": [ "hw_model = DynapcnnNetworkGraph(\n", " sinabs_model,\n", @@ -175,37 +226,47 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 10, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "(0, 1)\n", - "(1, 2)\n", - "(2, 3)\n", - "(3, 4)\n", - "(4, 5)\n", - "(5, 6)\n", - "(6, 7)\n", - "(7, 8)\n", - "(8, 9)\n", - "(9, 10)\n", - "(10, 11)\n", - "(11, 12)\n", - "(12, 13)\n" + "0 Conv2d(1, 20, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", + "1 IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-1.), batch_size=1, num_timesteps=-1)\n", + "2 AvgPool2d(kernel_size=2, stride=2, padding=0)\n", + "3 Conv2d(20, 32, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", + "4 IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-1.), batch_size=1, num_timesteps=-1)\n", + "5 AvgPool2d(kernel_size=2, stride=2, padding=0)\n", + "6 Conv2d(32, 128, kernel_size=(3, 3), stride=(1, 1), bias=False)\n", + "7 IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-1.), batch_size=1, num_timesteps=-1)\n", + "8 AvgPool2d(kernel_size=2, stride=2, padding=0)\n", + "9 Linear(in_features=128, out_features=500, bias=False)\n", + "10 IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-1.), batch_size=1, num_timesteps=-1)\n", + "11 Linear(in_features=500, out_features=10, bias=False)\n", + "12 IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-1.), batch_size=1, num_timesteps=-1)\n" ] } ], "source": [ - "for edge in hw_model.graph_tracer.edges_list:\n", - " print(edge)" + "for i, l in enumerate(hw_model.layers):\n", + " print(i, l)" ] }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 11, "metadata": {}, "outputs": [ { @@ -220,9 +281,10 @@ "(5, 6)\n", "(6, 7)\n", "(7, 8)\n", - "(8, 10)\n", + "(8, 9)\n", + "(9, 10)\n", "(10, 11)\n", - "(11, 13)\n" + "(11, 12)\n" ] } ], From b26f7cb636750c08bbc09e3ad509bba7b164e009 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Wed, 10 Apr 2024 16:47:18 +0200 Subject: [PATCH 004/379] (WIP) parsing sinabs edges to create DynapcnnLayer objects --- .../dynapcnn/dynapcnn_network_graph.py | 25 +++++-- sinabs/backend/dynapcnn/utils.py | 26 ++++++++ sinabs/backend/dynapcnn/utils_graph.py | 25 +++++++ .../jit_based_tracer_sinabs.ipynb | 66 ++++++++++++++++++- 4 files changed, 134 insertions(+), 8 deletions(-) create mode 100644 sinabs/backend/dynapcnn/utils_graph.py diff --git a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py index 12094fd8..290e9e36 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py @@ -16,6 +16,7 @@ from .utils import ( DEFAULT_IGNORED_LAYER_TYPES, build_from_list, + build_from_graph, convert_model_to_layer_list, infer_input_shape, parse_device_id, @@ -53,24 +54,28 @@ def __init__( If True, discretize the parameters and thresholds. This is needed for uploading weights to dynapcnn. Set to False only for testing purposes. + + 1. @TODO | 'self.sinabs_edges' has to be validated to be abiding to the device contraints before being passed to 'build_from_graph()'. """ super().__init__() + dvs_input = False # @TODO for now the graph part is not taking into consideration this. + # Computational graph from original PyTorch module. self.graph_tracer = GraphTracer( snn.analog_model, - torch.randn((1, *input_shape)) # torch.jit needs the batch dimension. + torch.randn((1, *input_shape)) # torch.jit needs the batch dimension. ) - # This attribute stores the location/core-id of each of the DynapcnnLayers upon placement on chip + # This attribute stores the location/core-id of each of the DynapcnnLayers upon placement on chip. self.chip_layers_ordering = [] - self.input_shape = input_shape # Convert models to sequential + self.input_shape = input_shape # convert models to sequential. self.layers = convert_model_to_layer_list( model=snn.spiking_model, ignore=DEFAULT_IGNORED_LAYER_TYPES ) - # Check if dvs input is expected + # Check if dvs input is expected. if dvs_input: self.dvs_input = True else: @@ -79,7 +84,7 @@ def __init__( input_shape = infer_input_shape(self.layers, input_shape=input_shape) assert len(input_shape) == 3, "infer_input_shape did not return 3-tuple" - # Build model from layers + # Build model from layers. self.sequence = build_from_list( self.layers, in_shape=input_shape, @@ -87,9 +92,17 @@ def __init__( dvs_input=self.dvs_input, ) - # Get sinabs graph + # Get sinabs graph. self.sinabs_edges = self.get_sinabs_edges(snn) + _ = build_from_graph( + layers=self.layers, + in_shape=input_shape, + edges=self.sinabs_edges) + + for i, mod in enumerate(self.sequence): + print(i, mod) + def get_sinabs_edges(self, sinabs_model): """ Converts the computational graph extracted from 'sinabs_model.analog_model' into its equivalent representation for the 'sinabs_model.spiking_model'. diff --git a/sinabs/backend/dynapcnn/utils.py b/sinabs/backend/dynapcnn/utils.py index 2719fc65..5fdcf956 100644 --- a/sinabs/backend/dynapcnn/utils.py +++ b/sinabs/backend/dynapcnn/utils.py @@ -13,6 +13,8 @@ from .exceptions import InputConfigurationError, MissingLayer, UnexpectedLayer from .flipdims import FlipDims +from .utils_graph import process_edge + if TYPE_CHECKING: from sinabs.backend.dynapcnn.dynapcnn_network import DynapcnnNetwork @@ -401,6 +403,7 @@ def build_from_list( dvs_layer, lyr_indx_next, rescale_factor = construct_dvs_layer( layers, input_shape=in_shape, idx_start=lyr_indx_next, dvs_input=dvs_input ) + if dvs_layer is not None: compatible_layers.append(dvs_layer) in_shape = dvs_layer.get_output_shape() @@ -537,3 +540,26 @@ def extend_readout_layer(model: "DynapcnnNetwork") -> "DynapcnnNetwork": torch.zeros(size=(1, *input_shape)) ) # run a forward pass to initialize the new weights and last IAF return model + +def build_from_graph(layers: list, in_shape, edges: List[Tuple[int, int]]): + """ .""" + print('\n [ ENTERED build_from_graph() ]\n') + + # @TODO the graph extraction is not yet considering DVS input. + dvs_layer, lyr_indx_next, rescale_factor = construct_dvs_layer( + layers, + input_shape=in_shape, + idx_start=0, + dvs_input=False) + + layers_to_cores_map = {} + + if dvs_layer is not None: + # @TODO the graph extraction is not yet considering DVS input. + pass + else: + # looping though graph edges. + for edge in edges: + process_edge(layers, edge, layers_to_cores_map) + + return None \ No newline at end of file diff --git a/sinabs/backend/dynapcnn/utils_graph.py b/sinabs/backend/dynapcnn/utils_graph.py new file mode 100644 index 00000000..d9a12239 --- /dev/null +++ b/sinabs/backend/dynapcnn/utils_graph.py @@ -0,0 +1,25 @@ +from typing import Tuple +import torch.nn as nn +import sinabs.layers as sl + +VALID_SINABS_EDGES = [ + (nn.Conv2d, sl.iaf.IAFSqueeze), + (sl.iaf.IAFSqueeze, nn.AvgPool2d), + (sl.iaf.IAFSqueeze, nn.Conv2d), + (sl.iaf.IAFSqueeze, nn.Linear), + (nn.AvgPool2d, nn.Conv2d), + (nn.AvgPool2d, nn.Linear), + (nn.Linear, sl.iaf.IAFSqueeze), +] + +VALID_SINABS_NODE_FAN_IN = [] +VALID_SINABS_NODE_FAN_OUT = [] + +def process_edge(layers: list, edge: Tuple[int, int], mapper: dict): + """ .""" + edge_layers = (type(layers[edge[0]]), type(layers[edge[1]])) + + if edge_layers in VALID_SINABS_EDGES: + print(edge) + else: + ... \ No newline at end of file diff --git a/tests/test_nonsequential/jit_based_tracer_sinabs.ipynb b/tests/test_nonsequential/jit_based_tracer_sinabs.ipynb index 6b8dea76..2fe29056 100644 --- a/tests/test_nonsequential/jit_based_tracer_sinabs.ipynb +++ b/tests/test_nonsequential/jit_based_tracer_sinabs.ipynb @@ -22,7 +22,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 2, @@ -215,7 +215,62 @@ "cell_type": "code", "execution_count": 9, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + " [ ENTERED build_from_graph() ]\n", + "\n", + "(0, 1)\n", + "(1, 2)\n", + "(2, 3)\n", + "(3, 4)\n", + "(4, 5)\n", + "(5, 6)\n", + "(6, 7)\n", + "(7, 8)\n", + "(8, 9)\n", + "(9, 10)\n", + "(10, 11)\n", + "(11, 12)\n", + "0 DynapcnnLayer(\n", + " (conv_layer): Conv2d(1, 20, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", + " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(635.), min_v_mem=Parameter containing:\n", + " tensor(-635.), batch_size=1, num_timesteps=-1)\n", + " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", + ")\n", + "1 DynapcnnLayer(\n", + " (conv_layer): Conv2d(20, 32, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", + " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(11361.), min_v_mem=Parameter containing:\n", + " tensor(-11361.), batch_size=1, num_timesteps=-1)\n", + " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", + ")\n", + "2 DynapcnnLayer(\n", + " (conv_layer): Conv2d(32, 128, kernel_size=(3, 3), stride=(1, 1), bias=False)\n", + " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(8621.), min_v_mem=Parameter containing:\n", + " tensor(-8621.), batch_size=1, num_timesteps=-1)\n", + " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", + ")\n", + "3 DynapcnnLayer(\n", + " (conv_layer): Conv2d(128, 500, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(5747.), min_v_mem=Parameter containing:\n", + " tensor(-5747.), batch_size=1, num_timesteps=-1)\n", + ")\n", + "4 DynapcnnLayer(\n", + " (conv_layer): Conv2d(500, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(2841.), min_v_mem=Parameter containing:\n", + " tensor(-2841.), batch_size=1, num_timesteps=-1)\n", + ")\n" + ] + } + ], "source": [ "hw_model = DynapcnnNetworkGraph(\n", " sinabs_model,\n", @@ -292,6 +347,13 @@ "for edge in hw_model.sinabs_edges:\n", " print(edge)" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## DynapCNN Configuration" + ] } ], "metadata": { From cc0f18e9aedeae56828c798d0d4517b744866cd9 Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 10 Apr 2024 21:08:04 +0200 Subject: [PATCH 005/379] (WIP) combining nodes within edges into layer blocks for DynapcnnLayer objects --- sinabs/backend/dynapcnn/utils_graph.py | 73 +++++++++++++++++++++----- 1 file changed, 61 insertions(+), 12 deletions(-) diff --git a/sinabs/backend/dynapcnn/utils_graph.py b/sinabs/backend/dynapcnn/utils_graph.py index d9a12239..97c2bade 100644 --- a/sinabs/backend/dynapcnn/utils_graph.py +++ b/sinabs/backend/dynapcnn/utils_graph.py @@ -2,15 +2,16 @@ import torch.nn as nn import sinabs.layers as sl -VALID_SINABS_EDGES = [ - (nn.Conv2d, sl.iaf.IAFSqueeze), - (sl.iaf.IAFSqueeze, nn.AvgPool2d), - (sl.iaf.IAFSqueeze, nn.Conv2d), - (sl.iaf.IAFSqueeze, nn.Linear), - (nn.AvgPool2d, nn.Conv2d), - (nn.AvgPool2d, nn.Linear), - (nn.Linear, sl.iaf.IAFSqueeze), -] +# @TODO this constraints are ideally device-dependent. +VALID_SINABS_EDGES = { + 0: (nn.Conv2d, sl.iaf.IAFSqueeze), + 1: (sl.iaf.IAFSqueeze, nn.AvgPool2d), + 2: (sl.iaf.IAFSqueeze, nn.Conv2d), + 3: (sl.iaf.IAFSqueeze, nn.Linear), + 4: (nn.AvgPool2d, nn.Conv2d), + 5: (nn.AvgPool2d, nn.Linear), + 6: (nn.Linear, sl.iaf.IAFSqueeze), +} VALID_SINABS_NODE_FAN_IN = [] VALID_SINABS_NODE_FAN_OUT = [] @@ -18,8 +19,56 @@ def process_edge(layers: list, edge: Tuple[int, int], mapper: dict): """ .""" edge_layers = (type(layers[edge[0]]), type(layers[edge[1]])) + edge_type = is_valid_edge(edge_layers) - if edge_layers in VALID_SINABS_EDGES: - print(edge) + if edge_type: + # incorporate modules within the edge to one DynapcnnLayer. + update_dynapcnnlayer_mapper(edge_type, edge, mapper, layers) else: - ... \ No newline at end of file + raise TypeError(f'Invalid graph edge: {edge_layers}') + +def is_valid_edge(edge): + """. """ + for key, edge_type in VALID_SINABS_EDGES.items(): + if edge == edge_type: + return key + return None + +def update_dynapcnnlayer_mapper(edge_type: int, edge: Tuple[int, int], mapper: dict, layers: list): + """ .""" + if edge_type == 0: # (conv, iaf): has to be a new DynapcnnLayer -> @TODO not necessarily! See 'edge_type == 2'. + new_key = 0 + for indx, layers_set in mapper.items(): # @TODO have to check if node for conv exists on 'mapper' (see 'edge_type == 2'). + new_key += 1 + mapper[new_key] = {edge[0]: layers[edge[0]], edge[1]: layers[edge[1]]} + elif edge_type == 1: # (iaf, pool): pool has to be part of a previously initialized DynapcnnLayer. + matched = False + for indx, layers_set in mapper.items(): + if layers[edge[0]] == layers_set[edge[1]]: + mapper[indx][edge[1]] = layers[edge[1]] + matched = True + break + if not matched: + raise TypeError(f'Edge {edge} can not be matched to already mapped layers.') + elif edge_type == 2: # (iaf, conv): must be an edge between an existing DynapcnnLayers and a new one. + matched = False + for indx, layers_set in mapper.items(): + for node_indx, mod in layers_set.items(): + if node_indx == edge[0]: + mapper[indx+1] = {edge[1]: layers[edge[1]]} + matched = True + break + if matched: + break + if not matched: + raise TypeError(f'Edge {edge} can not be matched to already mapped layers.') + elif edge_type == 3: # (, ): ... + ... + elif edge_type == 4: # (, ): ... + ... + elif edge_type == 5: # (, ): ... + ... + elif edge_type == 6: # (, ): ... + ... + else: + raise TypeError(f'Invalid graph edge type: {edge_type}') From 79928cdef7b8ad59e50ee72320ef82b3567f1d6e Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Thu, 11 Apr 2024 15:38:20 +0200 Subject: [PATCH 006/379] combining nodes within edges into layer blocks for DynapcnnLayer objects --- .../dynapcnn/dynapcnn_network_graph.py | 3 - .../backend/dynapcnn/sinabs_edges_handler.py | 116 ++++++++++++++++++ sinabs/backend/dynapcnn/sinabs_edges_utils.py | 42 +++++++ sinabs/backend/dynapcnn/utils.py | 19 ++- .../jit_based_tracer_sinabs.ipynb | 80 ++++++++---- 5 files changed, 231 insertions(+), 29 deletions(-) create mode 100644 sinabs/backend/dynapcnn/sinabs_edges_handler.py create mode 100644 sinabs/backend/dynapcnn/sinabs_edges_utils.py diff --git a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py index 290e9e36..6137e4c7 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py @@ -100,9 +100,6 @@ def __init__( in_shape=input_shape, edges=self.sinabs_edges) - for i, mod in enumerate(self.sequence): - print(i, mod) - def get_sinabs_edges(self, sinabs_model): """ Converts the computational graph extracted from 'sinabs_model.analog_model' into its equivalent representation for the 'sinabs_model.spiking_model'. diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py new file mode 100644 index 00000000..d6b7ab35 --- /dev/null +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -0,0 +1,116 @@ +# functionality : functions implementing the pre-processing of edges into blocks of nodes (modules) for future creation of DynapcnnLayer objects. +# author : Willian Soares Girao +# contact : williansoaresgirao@gmail.com + +from typing import Tuple, List +import torch.nn as nn +from .sinabs_edges_utils import * + +def process_edge(layers: List[nn.Module], edge: Tuple[int, int], mapper: dict) -> None: + """ Read in an edge describing the connection between two layers (nodes in the computational graph). If 'edge' + is a valid connection between two layers, update 'mapper' to incorporate these layers into a new or existing dictonary + containing the modules comprising a future DynacnnLayer object. + + Parameters + ---------- + layers : list of modules returned by 'utils.convert_model_to_layer_list()'. + edge : tuple representing the connection between two nodes in computational graph of 'DynapcnnNetworkGraph.snn.spiking_model'. + mapper : dictionary where each 'key' is the index of a future 'DynapcnnLayer' and 'value' is a dictionary ('key': node, 'value': module). + """ + edge_type = is_valid_edge(edge, layers, VALID_SINABS_EDGES) + + if isinstance(edge_type, int): # incorporate modules within the edge to a dict representing a future DynapcnnLayer. + update_dynapcnnlayer_mapper(edge_type, edge, mapper, layers) + else: + raise InvalidEdge(edge, type(layers[edge[0]]), type(layers[edge[1]])) + +def is_valid_edge(edge: Tuple[int, int], layers: List[nn.Module], valid_edges_map: dict) -> int: + """ Checks if the modules each node in 'edge' represent are a valid connection between a sinabs network to be + loaded on Speck. + + Parameters + ---------- + valid_edges_map: dictionary where each 'key' is the type (index) of a pre-defined valid edge. + + Returns + ---------- + edge_type: the edge type specified in 'valid_edges_map' ('None' if edge is not valid). + """ + edge_layers = (layers[edge[0]], layers[edge[1]]) + for edge_type, sinabs_edge in valid_edges_map.items(): + if (type(edge_layers[0]) == sinabs_edge[0]) and (type(edge_layers[1]) == sinabs_edge[1]): + return edge_type + return None + +def update_dynapcnnlayer_mapper(edge_type: int, edge: Tuple[int, int], mapper: dict, layers: List[nn.Module]) -> None: + """ Parses the nodes within an edge and incorporate them either into a **new** or an **already existing** DynapcnnLayer represented + in 'mapper'. + """ + + if edge_type in [0, 6]: + init_xor_complete_new_dynapcnnlayer_blk(mapper, edge, layers) + + elif edge_type == 1: + add_pool_to_dynapcnnlayer_blk(mapper, edge, layers) + + elif edge_type in [2, 3, 4, 5]: + connect_dynapcnnlayer_blks(mapper, edge, layers) + + else: + raise InvalidEdgeType(edge, edge_type) + +def init_xor_complete_new_dynapcnnlayer_blk(mapper: dict, edge: Tuple[int, int], layers: List[nn.Module]) -> None: + """ Incorporates nodes from either a '(conv, neuron)' or a '(linear, neuron)' edge. These are either initiating a (new) DynapcnnLayer + or completing a conv->neuron sequence (in the case the node for 'conv' as already been incorporated somewhere in 'mapper'). 'nn.Linear' layers + are converted into 'nn.Conv2d' by DynapcnnLayer. + """ + matched = False + dynapcnnlayer_indx = 0 + for indx, dynapcnnlayer in mapper.items(): # see if 'edge[0]' exists in a DynapcnnLayer block. + for node, _ in dynapcnnlayer.items(): + if node == edge[0]: + dynapcnnlayer_indx = indx + matched = True + break + if matched: # 'edge[0]' found: 'edge[1]' belongs to its DynapcnnLayer block. + mapper[dynapcnnlayer_indx][edge[1]] = layers[edge[1]] + break + + if not matched: # 'edge[0]' not found: start new DynapcnnLayer block. + dynapcnnlayer_indx = 0 + for indx, _ in mapper.items(): + dynapcnnlayer_indx += 1 + mapper[dynapcnnlayer_indx] = {edge[0]: layers[edge[0]], edge[1]: layers[edge[1]]} + +def connect_dynapcnnlayer_blks(mapper: dict, edge: Tuple[int, int], layers: List[nn.Module]) -> None: + """ Incorporates nodes from either a '(neuron, conv)/(neuron, lin)' or '(pool, conv)/(pool, lin)' edge. These represent connections between an existing + DynapcnnLayer in 'mapper' and a new one yet to be represented in 'mapper'. 'nn.Linear' layers are converted into 'nn.Conv2d' by DynapcnnLayer. + """ + dynapcnnlayer_indx = 0 + matched = False + for indx, dynapcnnlayer in mapper.items(): + for node, _ in dynapcnnlayer.items(): + if node == edge[0]: # 'edge[0]' is ending DynapcnnLayer block 'indx'. + dynapcnnlayer_indx = indx+1 + matched = True + break + if matched: + break + if matched: + mapper[dynapcnnlayer_indx] = {edge[1]: layers[edge[1]]} # 'edge[1]' starts new DynapcnnLayer block as 'indx+1'. + else: + raise UnmatchedNode(edge, node) + +def add_pool_to_dynapcnnlayer_blk(mapper: dict, edge: Tuple[int, int], layers: List[nn.Module]) -> None: + """ Incorporating a '(neuron, pool)' edge. Node 'pool' has to be part of an already existing DynapcnnLayer in 'mapper'.""" + matched = False + for indx, dynapcnnlayer in mapper.items(): + for node, _ in dynapcnnlayer.items(): + if node == edge[0]: + dynapcnnlayer[edge[1]] = layers[edge[1]] # 'edge[0]' is a neuron layer inputing into pooling layer 'edge[1]'. + matched = True + break + if matched: + break + if not matched: + raise UnmatchedNode(edge, node) diff --git a/sinabs/backend/dynapcnn/sinabs_edges_utils.py b/sinabs/backend/dynapcnn/sinabs_edges_utils.py new file mode 100644 index 00000000..4153d0cf --- /dev/null +++ b/sinabs/backend/dynapcnn/sinabs_edges_utils.py @@ -0,0 +1,42 @@ +import sinabs.layers as sl +import torch.nn as nn +from typing import Tuple + +# Constraints. @TODO this constraints are ideally device-dependent. + +VALID_SINABS_EDGES = { + 0: (nn.Conv2d, sl.iaf.IAFSqueeze), # 'nn.Conv2d' is always followed by a 'sl.iaf'. + 1: (sl.iaf.IAFSqueeze, nn.AvgPool2d), + 2: (sl.iaf.IAFSqueeze, nn.Conv2d), + 3: (sl.iaf.IAFSqueeze, nn.Linear), # 'nn.Linear' layers are converted into 'nn.Conv2d' by 'DynapcnnLayer'. + 4: (nn.AvgPool2d, nn.Conv2d), # 'nn.Pool2d' is always "ending" a DynapcnnLayer sequence of modules (comes after a 'sl.iaf'). + 5: (nn.AvgPool2d, nn.Linear), # 'nn.Linear' layers are converted into 'nn.Conv2d' by 'DynapcnnLayer'. + 6: (nn.Linear, sl.iaf.IAFSqueeze), +} + +VALID_SINABS_NODE_FAN_IN = [] +VALID_SINABS_NODE_FAN_OUT = [] + +# Edge exceptions. + +class InvalidEdge(Exception): + edge: Tuple[int, int] + source: type + target: type + + def __init__(self, edge, source, target): + super().__init__(f"Invalid edge {edge}: {source} can not target {target}.") + +class InvalidEdgeType(Exception): + edge: Tuple[int, int] + type: int + + def __init__(self, edge, type): + super().__init__(f"Invalid edge type {type} for edge {edge}.") + +class UnmatchedNode(Exception): + edge: Tuple[int, int] + node: int + + def __init__(self, edge, node): + super().__init__(f"Node {node} in edge {edge} can not found in previously processed edges.") \ No newline at end of file diff --git a/sinabs/backend/dynapcnn/utils.py b/sinabs/backend/dynapcnn/utils.py index 5fdcf956..2877f59a 100644 --- a/sinabs/backend/dynapcnn/utils.py +++ b/sinabs/backend/dynapcnn/utils.py @@ -13,7 +13,7 @@ from .exceptions import InputConfigurationError, MissingLayer, UnexpectedLayer from .flipdims import FlipDims -from .utils_graph import process_edge +from .sinabs_edges_handler import process_edge if TYPE_CHECKING: from sinabs.backend.dynapcnn.dynapcnn_network import DynapcnnNetwork @@ -552,14 +552,23 @@ def build_from_graph(layers: list, in_shape, edges: List[Tuple[int, int]]): idx_start=0, dvs_input=False) + ''' + holds 'blocks' of modules that populate a single DynapcnnLayer object. + key (int): DynapcnnLayer index. + value (dict): node index (key) and its type (module). + ''' layers_to_cores_map = {} - if dvs_layer is not None: - # @TODO the graph extraction is not yet considering DVS input. + if dvs_layer is not None: # @TODO the graph extraction is not yet considering DVS input. pass - else: - # looping though graph edges. + else: # looping though graph edges. for edge in edges: process_edge(layers, edge, layers_to_cores_map) + + for key, val in layers_to_cores_map.items(): + print(key) + for k, v in val.items(): + print(' ', k, type(v)) + print('\n') return None \ No newline at end of file diff --git a/tests/test_nonsequential/jit_based_tracer_sinabs.ipynb b/tests/test_nonsequential/jit_based_tracer_sinabs.ipynb index 2fe29056..f1140fa3 100644 --- a/tests/test_nonsequential/jit_based_tracer_sinabs.ipynb +++ b/tests/test_nonsequential/jit_based_tracer_sinabs.ipynb @@ -22,7 +22,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 2, @@ -221,48 +221,35 @@ "output_type": "stream", "text": [ "\n", - " [ ENTERED build_from_graph() ]\n", "\n", - "(0, 1)\n", - "(1, 2)\n", - "(2, 3)\n", - "(3, 4)\n", - "(4, 5)\n", - "(5, 6)\n", - "(6, 7)\n", - "(7, 8)\n", - "(8, 9)\n", - "(9, 10)\n", - "(10, 11)\n", - "(11, 12)\n", - "0 DynapcnnLayer(\n", + "DynapcnnLayer(\n", " (conv_layer): Conv2d(1, 20, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", " tensor(635.), min_v_mem=Parameter containing:\n", " tensor(-635.), batch_size=1, num_timesteps=-1)\n", " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", ")\n", - "1 DynapcnnLayer(\n", + "DynapcnnLayer(\n", " (conv_layer): Conv2d(20, 32, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", " tensor(11361.), min_v_mem=Parameter containing:\n", " tensor(-11361.), batch_size=1, num_timesteps=-1)\n", " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", ")\n", - "2 DynapcnnLayer(\n", + "DynapcnnLayer(\n", " (conv_layer): Conv2d(32, 128, kernel_size=(3, 3), stride=(1, 1), bias=False)\n", " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", " tensor(8621.), min_v_mem=Parameter containing:\n", " tensor(-8621.), batch_size=1, num_timesteps=-1)\n", " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", ")\n", - "3 DynapcnnLayer(\n", + "DynapcnnLayer(\n", " (conv_layer): Conv2d(128, 500, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", " tensor(5747.), min_v_mem=Parameter containing:\n", " tensor(-5747.), batch_size=1, num_timesteps=-1)\n", ")\n", - "4 DynapcnnLayer(\n", + "DynapcnnLayer(\n", " (conv_layer): Conv2d(500, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", " tensor(2841.), min_v_mem=Parameter containing:\n", @@ -271,6 +258,57 @@ ] } ], + "source": [ + "hw_model_old = DynapcnnNetwork(\n", + " sinabs_model.spiking_model,\n", + " discretize=True,\n", + " input_shape=input_shape\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + " [ ENTERED build_from_graph() ]\n", + "\n", + "0\n", + " 0 \n", + " 1 \n", + " 2 \n", + "\n", + "\n", + "1\n", + " 3 \n", + " 4 \n", + " 5 \n", + "\n", + "\n", + "2\n", + " 6 \n", + " 7 \n", + " 8 \n", + "\n", + "\n", + "3\n", + " 9 \n", + " 10 \n", + "\n", + "\n", + "4\n", + " 11 \n", + " 12 \n", + "\n", + "\n" + ] + } + ], "source": [ "hw_model = DynapcnnNetworkGraph(\n", " sinabs_model,\n", @@ -281,7 +319,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 11, "metadata": {}, "outputs": [ { @@ -321,7 +359,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 12, "metadata": {}, "outputs": [ { From 1b6af5e3761c39a535b1944f387a183439c22e16 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Thu, 11 Apr 2024 15:41:07 +0200 Subject: [PATCH 007/379] untracking renamed file --- sinabs/backend/dynapcnn/utils_graph.py | 74 -------------------------- 1 file changed, 74 deletions(-) delete mode 100644 sinabs/backend/dynapcnn/utils_graph.py diff --git a/sinabs/backend/dynapcnn/utils_graph.py b/sinabs/backend/dynapcnn/utils_graph.py deleted file mode 100644 index 97c2bade..00000000 --- a/sinabs/backend/dynapcnn/utils_graph.py +++ /dev/null @@ -1,74 +0,0 @@ -from typing import Tuple -import torch.nn as nn -import sinabs.layers as sl - -# @TODO this constraints are ideally device-dependent. -VALID_SINABS_EDGES = { - 0: (nn.Conv2d, sl.iaf.IAFSqueeze), - 1: (sl.iaf.IAFSqueeze, nn.AvgPool2d), - 2: (sl.iaf.IAFSqueeze, nn.Conv2d), - 3: (sl.iaf.IAFSqueeze, nn.Linear), - 4: (nn.AvgPool2d, nn.Conv2d), - 5: (nn.AvgPool2d, nn.Linear), - 6: (nn.Linear, sl.iaf.IAFSqueeze), -} - -VALID_SINABS_NODE_FAN_IN = [] -VALID_SINABS_NODE_FAN_OUT = [] - -def process_edge(layers: list, edge: Tuple[int, int], mapper: dict): - """ .""" - edge_layers = (type(layers[edge[0]]), type(layers[edge[1]])) - edge_type = is_valid_edge(edge_layers) - - if edge_type: - # incorporate modules within the edge to one DynapcnnLayer. - update_dynapcnnlayer_mapper(edge_type, edge, mapper, layers) - else: - raise TypeError(f'Invalid graph edge: {edge_layers}') - -def is_valid_edge(edge): - """. """ - for key, edge_type in VALID_SINABS_EDGES.items(): - if edge == edge_type: - return key - return None - -def update_dynapcnnlayer_mapper(edge_type: int, edge: Tuple[int, int], mapper: dict, layers: list): - """ .""" - if edge_type == 0: # (conv, iaf): has to be a new DynapcnnLayer -> @TODO not necessarily! See 'edge_type == 2'. - new_key = 0 - for indx, layers_set in mapper.items(): # @TODO have to check if node for conv exists on 'mapper' (see 'edge_type == 2'). - new_key += 1 - mapper[new_key] = {edge[0]: layers[edge[0]], edge[1]: layers[edge[1]]} - elif edge_type == 1: # (iaf, pool): pool has to be part of a previously initialized DynapcnnLayer. - matched = False - for indx, layers_set in mapper.items(): - if layers[edge[0]] == layers_set[edge[1]]: - mapper[indx][edge[1]] = layers[edge[1]] - matched = True - break - if not matched: - raise TypeError(f'Edge {edge} can not be matched to already mapped layers.') - elif edge_type == 2: # (iaf, conv): must be an edge between an existing DynapcnnLayers and a new one. - matched = False - for indx, layers_set in mapper.items(): - for node_indx, mod in layers_set.items(): - if node_indx == edge[0]: - mapper[indx+1] = {edge[1]: layers[edge[1]]} - matched = True - break - if matched: - break - if not matched: - raise TypeError(f'Edge {edge} can not be matched to already mapped layers.') - elif edge_type == 3: # (, ): ... - ... - elif edge_type == 4: # (, ): ... - ... - elif edge_type == 5: # (, ): ... - ... - elif edge_type == 6: # (, ): ... - ... - else: - raise TypeError(f'Invalid graph edge type: {edge_type}') From 5bbaf255e2a7b0e4e12d2e3de2a37d236be4fd5f Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Thu, 11 Apr 2024 19:53:32 +0200 Subject: [PATCH 008/379] creating mapper from sets of nodes to DynapcnnLayer objects + creating mapper for DynapcnnLayer destinations --- .../dynapcnn/dynapcnn_network_graph.py | 8 +- .../backend/dynapcnn/sinabs_edges_handler.py | 102 ++++- sinabs/backend/dynapcnn/sinabs_edges_utils.py | 47 +- sinabs/backend/dynapcnn/utils.py | 31 +- .../build_from_graph_tester.ipynb | 416 ++++++++++++++++++ .../jit_based_tracer_sinabs.ipynb | 64 ++- 6 files changed, 593 insertions(+), 75 deletions(-) create mode 100644 tests/test_nonsequential/build_from_graph_tester.ipynb diff --git a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py index 6137e4c7..ba3b49ca 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py @@ -54,8 +54,6 @@ def __init__( If True, discretize the parameters and thresholds. This is needed for uploading weights to dynapcnn. Set to False only for testing purposes. - - 1. @TODO | 'self.sinabs_edges' has to be validated to be abiding to the device contraints before being passed to 'build_from_graph()'. """ super().__init__() @@ -95,10 +93,14 @@ def __init__( # Get sinabs graph. self.sinabs_edges = self.get_sinabs_edges(snn) - _ = build_from_graph( + self.layers_to_cores_map, self.core_to_core_map = build_from_graph( layers=self.layers, in_shape=input_shape, edges=self.sinabs_edges) + + @staticmethod + def build_from_graph_(): # @TODO used for debug only (remove when class is complete). + return build_from_graph def get_sinabs_edges(self, sinabs_model): """ Converts the computational graph extracted from 'sinabs_model.analog_model' into its equivalent diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index d6b7ab35..4809c6a4 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -19,7 +19,7 @@ def process_edge(layers: List[nn.Module], edge: Tuple[int, int], mapper: dict) - """ edge_type = is_valid_edge(edge, layers, VALID_SINABS_EDGES) - if isinstance(edge_type, int): # incorporate modules within the edge to a dict representing a future DynapcnnLayer. + if isinstance(edge_type, int): # incorporate modules within the edge to a dict representing a future DynapcnnLayer. update_dynapcnnlayer_mapper(edge_type, edge, mapper, layers) else: raise InvalidEdge(edge, type(layers[edge[0]]), type(layers[edge[1]])) @@ -66,17 +66,17 @@ def init_xor_complete_new_dynapcnnlayer_blk(mapper: dict, edge: Tuple[int, int], """ matched = False dynapcnnlayer_indx = 0 - for indx, dynapcnnlayer in mapper.items(): # see if 'edge[0]' exists in a DynapcnnLayer block. + for indx, dynapcnnlayer in mapper.items(): # see if 'edge[0]' exists in a DynapcnnLayer block. for node, _ in dynapcnnlayer.items(): if node == edge[0]: dynapcnnlayer_indx = indx matched = True break - if matched: # 'edge[0]' found: 'edge[1]' belongs to its DynapcnnLayer block. + if matched: # 'edge[0]' found: 'edge[1]' belongs to its DynapcnnLayer block. mapper[dynapcnnlayer_indx][edge[1]] = layers[edge[1]] break - if not matched: # 'edge[0]' not found: start new DynapcnnLayer block. + if not matched: # 'edge[0]' not found: start new DynapcnnLayer block. dynapcnnlayer_indx = 0 for indx, _ in mapper.items(): dynapcnnlayer_indx += 1 @@ -86,31 +86,97 @@ def connect_dynapcnnlayer_blks(mapper: dict, edge: Tuple[int, int], layers: List """ Incorporates nodes from either a '(neuron, conv)/(neuron, lin)' or '(pool, conv)/(pool, lin)' edge. These represent connections between an existing DynapcnnLayer in 'mapper' and a new one yet to be represented in 'mapper'. 'nn.Linear' layers are converted into 'nn.Conv2d' by DynapcnnLayer. """ - dynapcnnlayer_indx = 0 - matched = False - for indx, dynapcnnlayer in mapper.items(): - for node, _ in dynapcnnlayer.items(): - if node == edge[0]: # 'edge[0]' is ending DynapcnnLayer block 'indx'. - dynapcnnlayer_indx = indx+1 - matched = True + if not is_initialized_node(edge[1], mapper): + dynapcnnlayer_indx = 0 + matched = False + for indx, dynapcnnlayer in mapper.items(): + for node, _ in dynapcnnlayer.items(): + if node == edge[0]: # 'edge[0]' is ending DynapcnnLayer block 'indx'. + dynapcnnlayer_indx = indx+1 + matched = True + break + if matched: break if matched: - break - if matched: - mapper[dynapcnnlayer_indx] = {edge[1]: layers[edge[1]]} # 'edge[1]' starts new DynapcnnLayer block as 'indx+1'. - else: - raise UnmatchedNode(edge, node) + mapper[dynapcnnlayer_indx] = {edge[1]: layers[edge[1]]} # 'edge[1]' starts new DynapcnnLayer block as 'indx+1'. + else: + raise UnmatchedNode(edge, node) def add_pool_to_dynapcnnlayer_blk(mapper: dict, edge: Tuple[int, int], layers: List[nn.Module]) -> None: - """ Incorporating a '(neuron, pool)' edge. Node 'pool' has to be part of an already existing DynapcnnLayer in 'mapper'.""" + """ Incorporating a '(neuron, pool)' edge. Node 'pool' has to be part of an already existing DynapcnnLayer in 'mapper'. """ matched = False for indx, dynapcnnlayer in mapper.items(): for node, _ in dynapcnnlayer.items(): if node == edge[0]: - dynapcnnlayer[edge[1]] = layers[edge[1]] # 'edge[0]' is a neuron layer inputing into pooling layer 'edge[1]'. + dynapcnnlayer[edge[1]] = layers[edge[1]] # 'edge[0]' is a neuron layer inputing into pooling layer 'edge[1]'. matched = True break if matched: break if not matched: raise UnmatchedNode(edge, node) + +def is_initialized_node(node: int, mapper: dict) -> bool: + """ Finds if 'node' existis within 'mapper'. """ + for _, dynapcnnlayer in mapper.items(): + for _node, __ in dynapcnnlayer.items(): + if _node == node: + return True + return False + +def get_dynapcnnlayers_destinations(layers: List[nn.Module], edges: List[Tuple[int, int]], mapper: dict) -> dict: + """ Loops over the edges list describing the computational graph. It will access each node in the graph and find to which + DynapcnnLayer they belong to. If source and target belong to different DynapcnnLayers (described as a dictionary in 'mapper') + the destination of the 'DynapcnnLayer.source' is set to be 'DynapcnnLayer.target'. + + Parameters + ---------- + layers : list of modules returned by 'utils.convert_model_to_layer_list()'. + edges : list of tuples representing the connection between nodes in computational graph of 'DynapcnnNetworkGraph.snn.spiking_model'. + mapper : dictionary where each 'key' is the index of a future 'DynapcnnLayer' and 'value' its modules (output of 'process_edge(mapper)'). + + Returns + ---------- + dynapcnnlayers_destinations_map: dictionary where each 'key' is the index of a future 'DynapcnnLayer' and 'value' is its list of destinations (DynapcnnLayers). + """ + dynapcnnlayers_destinations_map = {} + used_layer_edges = [] + + for edge in edges: + source_layer = get_dynapcnnlayer_index(edge[0], mapper) + destination_layer = get_dynapcnnlayer_index(edge[1], mapper) + + if source_layer not in dynapcnnlayers_destinations_map: + dynapcnnlayers_destinations_map[source_layer] = [] + + if source_layer != destination_layer and is_valid_dynapcnnlayer_pairing(layers, edge, VALID_DYNAPCNNLAYER_EDGES): + # valid connection between modules in two different DynapcnnLayer. + + if len(dynapcnnlayers_destinations_map[source_layer]) > 2: + # DynapcnnLayers can not have more than two destinations. + raise MaxDestinationsReached(source_layer) + else: + if (destination_layer, source_layer) not in used_layer_edges and destination_layer not in dynapcnnlayers_destinations_map[source_layer]: + # edge does not create a loop between layers. + dynapcnnlayers_destinations_map[source_layer].append(destination_layer) + used_layer_edges.append((source_layer, destination_layer)) + else: + raise InvalidLayerLoop(source_layer, destination_layer) + + del used_layer_edges + + return dynapcnnlayers_destinations_map + +def get_dynapcnnlayer_index(node: int, mapper: dict) -> int: + """ Returns the DynapcnnLayer index to which 'node' belongs to. """ + for indx, dynapcnnlayer in mapper.items(): + if node in dynapcnnlayer: + return indx + raise UnknownNode(node) + +def is_valid_dynapcnnlayer_pairing(layers: List[nn.Module], edge: Tuple[int, int], valid_dynapcnnlayer_edges: List[Tuple[nn.Module, nn.Module]]) -> bool: + """ Checks if the module in 'DynapcnnLayer.source' is targetting a valid module in 'DynapcnnLayer.target'. """ + if (type(layers[edge[0]]), type(layers[edge[1]])) in valid_dynapcnnlayer_edges: + return True + else: + raise InvalidLayerDestination(type(layers[edge[0]]), type(layers[edge[1]])) \ No newline at end of file diff --git a/sinabs/backend/dynapcnn/sinabs_edges_utils.py b/sinabs/backend/dynapcnn/sinabs_edges_utils.py index 4153d0cf..b1b89848 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_utils.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_utils.py @@ -2,18 +2,25 @@ import torch.nn as nn from typing import Tuple -# Constraints. @TODO this constraints are ideally device-dependent. +# Constraints. # @TODO constraints are ideally device-dependent. VALID_SINABS_EDGES = { - 0: (nn.Conv2d, sl.iaf.IAFSqueeze), # 'nn.Conv2d' is always followed by a 'sl.iaf'. + 0: (nn.Conv2d, sl.iaf.IAFSqueeze), # 'nn.Conv2d' is always followed by a 'sl.iaf'. 1: (sl.iaf.IAFSqueeze, nn.AvgPool2d), 2: (sl.iaf.IAFSqueeze, nn.Conv2d), - 3: (sl.iaf.IAFSqueeze, nn.Linear), # 'nn.Linear' layers are converted into 'nn.Conv2d' by 'DynapcnnLayer'. - 4: (nn.AvgPool2d, nn.Conv2d), # 'nn.Pool2d' is always "ending" a DynapcnnLayer sequence of modules (comes after a 'sl.iaf'). - 5: (nn.AvgPool2d, nn.Linear), # 'nn.Linear' layers are converted into 'nn.Conv2d' by 'DynapcnnLayer'. - 6: (nn.Linear, sl.iaf.IAFSqueeze), + 3: (sl.iaf.IAFSqueeze, nn.Linear), # same case as '2' since 'nn.Linear' layers are converted into 'nn.Conv2d' by 'DynapcnnLayer'. + 4: (nn.AvgPool2d, nn.Conv2d), # 'nn.Pool2d' is always "ending" a DynapcnnLayer sequence of modules (comes after a 'sl.iaf'). + 5: (nn.AvgPool2d, nn.Linear), # same as case '4' since 'nn.Linear' layers are converted into 'nn.Conv2d' by 'DynapcnnLayer'. + 6: (nn.Linear, sl.iaf.IAFSqueeze), # same as case '0' since 'nn.Linear' layers are converted into 'nn.Conv2d' by 'DynapcnnLayer'. } +VALID_DYNAPCNNLAYER_EDGES = [ + (sl.iaf.IAFSqueeze, nn.Conv2d), + (sl.iaf.IAFSqueeze, nn.Linear), # 'nn.Linear' layers are converted into 'nn.Conv2d' by 'DynapcnnLayer'. + (nn.AvgPool2d, nn.Conv2d), + (nn.AvgPool2d, nn.Linear), # 'nn.Linear' layers are converted into 'nn.Conv2d' by 'DynapcnnLayer'. +] + VALID_SINABS_NODE_FAN_IN = [] VALID_SINABS_NODE_FAN_OUT = [] @@ -39,4 +46,30 @@ class UnmatchedNode(Exception): node: int def __init__(self, edge, node): - super().__init__(f"Node {node} in edge {edge} can not found in previously processed edges.") \ No newline at end of file + super().__init__(f"Node {node} in edge {edge} can not found in previously processed edges.") + +class UnknownNode(Exception): + node: int + + def __init__(self, node): + super().__init__(f"Node {node} can not be found within any DynapcnnLayer mapper.") + +class MaxDestinationsReached(Exception): + dynapcnnlayer_index: int + + def __init__(self, dynapcnnlayer_index): + super().__init__(f"DynapcnnLayer with index {dynapcnnlayer_index} has more than 2 destinations.") + +class InvalidLayerLoop(Exception): + dynapcnnlayerA_index: int + dynapcnnlayerB_index: int + + def __init__(self, dynapcnnlayerA_index, dynapcnnlayerB_index): + super().__init__(f"DynapcnnLayer {dynapcnnlayerA_index} can not connect to {dynapcnnlayerB_index} since reverse edge already exists.") + +class InvalidLayerDestination(Exception): + dynapcnnlayerA: type + dynapcnnlayerB: type + + def __init__(self, dynapcnnlayerA, dynapcnnlayerB): + super().__init__(f"DynapcnnLayer {dynapcnnlayerA} in one core can not connect to {dynapcnnlayerB} in another core.") \ No newline at end of file diff --git a/sinabs/backend/dynapcnn/utils.py b/sinabs/backend/dynapcnn/utils.py index 2877f59a..98a1b17b 100644 --- a/sinabs/backend/dynapcnn/utils.py +++ b/sinabs/backend/dynapcnn/utils.py @@ -13,7 +13,7 @@ from .exceptions import InputConfigurationError, MissingLayer, UnexpectedLayer from .flipdims import FlipDims -from .sinabs_edges_handler import process_edge +from .sinabs_edges_handler import process_edge, get_dynapcnnlayers_destinations if TYPE_CHECKING: from sinabs.backend.dynapcnn.dynapcnn_network import DynapcnnNetwork @@ -541,16 +541,17 @@ def extend_readout_layer(model: "DynapcnnNetwork") -> "DynapcnnNetwork": ) # run a forward pass to initialize the new weights and last IAF return model -def build_from_graph(layers: list, in_shape, edges: List[Tuple[int, int]]): +def build_from_graph(layers: List[nn.Module], in_shape, edges: List[Tuple[int, int]]): """ .""" print('\n [ ENTERED build_from_graph() ]\n') # @TODO the graph extraction is not yet considering DVS input. - dvs_layer, lyr_indx_next, rescale_factor = construct_dvs_layer( - layers, - input_shape=in_shape, - idx_start=0, - dvs_input=False) + # dvs_layer, lyr_indx_next, rescale_factor = construct_dvs_layer( + # layers, + # input_shape=in_shape, + # idx_start=0, + # dvs_input=False) + dvs_layer = None ''' holds 'blocks' of modules that populate a single DynapcnnLayer object. @@ -565,10 +566,18 @@ def build_from_graph(layers: list, in_shape, edges: List[Tuple[int, int]]): for edge in edges: process_edge(layers, edge, layers_to_cores_map) + print('node to DynapcnnLayer mapping:') + for key, val in layers_to_cores_map.items(): - print(key) + print('DynapcnnLayer index: ', key) for k, v in val.items(): - print(' ', k, type(v)) - print('\n') + print(f' [{k} {type(v)}]') + + core_to_core_map = get_dynapcnnlayers_destinations(layers, edges, layers_to_cores_map) + + print('\nDynapcnnLayer to DynapcnnLayer mapping:') + + for key, val in core_to_core_map.items(): + print(f'DynapcnnLayer {key} destinations: {val}') - return None \ No newline at end of file + return layers_to_cores_map, core_to_core_map \ No newline at end of file diff --git a/tests/test_nonsequential/build_from_graph_tester.ipynb b/tests/test_nonsequential/build_from_graph_tester.ipynb new file mode 100644 index 00000000..11d2682b --- /dev/null +++ b/tests/test_nonsequential/build_from_graph_tester.ipynb @@ -0,0 +1,416 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "import torch.nn as nn\n", + "import networkx as nx\n", + "import matplotlib.pyplot as plt\n", + "from sinabs.from_torch import from_model\n", + "from sinabs.backend.dynapcnn import DynapcnnNetwork, DynapcnnNetworkGraph\n", + "import sinabs as snb" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "torch.manual_seed(0)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Dummy Initialization" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "channels = 1\n", + "height = 28\n", + "width = 28\n", + "\n", + "input_shape = (channels, height, width)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "ann = nn.Sequential(\n", + " nn.Conv2d(1, 20, 5, 1, bias=False),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(2,2),\n", + " nn.Conv2d(20, 32, 5, 1, bias=False),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(2,2),\n", + " nn.Conv2d(32, 128, 3, 1, bias=False),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(2,2),\n", + " nn.Flatten(),\n", + " nn.Linear(128, 500, bias=False),\n", + " nn.ReLU(),\n", + " nn.Flatten(),\n", + " nn.Linear(500, 10, bias=False),\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "sinabs_model = from_model(ann, add_spiking_output=True, batch_size=1)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + " [ ENTERED build_from_graph() ]\n", + "\n", + "node to DynapcnnLayer mapping:\n", + "DynapcnnLayer index: 0\n", + " [0 ]\n", + " [1 ]\n", + " [2 ]\n", + "DynapcnnLayer index: 1\n", + " [3 ]\n", + " [4 ]\n", + " [5 ]\n", + "DynapcnnLayer index: 2\n", + " [6 ]\n", + " [7 ]\n", + " [8 ]\n", + "DynapcnnLayer index: 3\n", + " [9 ]\n", + " [10 ]\n", + "DynapcnnLayer index: 4\n", + " [11 ]\n", + " [12 ]\n", + "\n", + "DynapcnnLayer to DynapcnnLayer mapping:\n", + "DynapcnnLayer 0 destinations: [1]\n", + "DynapcnnLayer 1 destinations: [2]\n", + "DynapcnnLayer 2 destinations: [3]\n", + "DynapcnnLayer 3 destinations: [4]\n", + "DynapcnnLayer 4 destinations: []\n" + ] + } + ], + "source": [ + "hw_model = DynapcnnNetworkGraph(\n", + " sinabs_model,\n", + " discretize=True,\n", + " input_shape=input_shape\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Valide Non-sequential Edges" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Test 1" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "dummy_layer_list = [\n", + " torch.nn.Conv2d(1, 20, kernel_size=(5, 5), stride=(1, 1), bias=False),\n", + " snb.layers.iaf.IAFSqueeze(batch_size=1),\n", + " torch.nn.AvgPool2d(kernel_size=2, stride=2, padding=0),\n", + " torch.nn.Conv2d(1, 20, kernel_size=(5, 5), stride=(1, 1), bias=False),\n", + " snb.layers.iaf.IAFSqueeze(batch_size=1),\n", + " torch.nn.AvgPool2d(kernel_size=2, stride=2, padding=0),\n", + " torch.nn.Conv2d(1, 20, kernel_size=(5, 5), stride=(1, 1), bias=False),\n", + " snb.layers.iaf.IAFSqueeze(batch_size=1),\n", + " torch.nn.AvgPool2d(kernel_size=2, stride=2, padding=0),\n", + " torch.nn.Linear(in_features=128, out_features=500, bias=False),\n", + " snb.layers.iaf.IAFSqueeze(batch_size=1),\n", + " torch.nn.Linear(in_features=128, out_features=500, bias=False),\n", + " snb.layers.iaf.IAFSqueeze(batch_size=1),\n", + " torch.nn.Linear(in_features=128, out_features=500, bias=False),\n", + " snb.layers.iaf.IAFSqueeze(batch_size=1),]" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "dummy_edges = [\n", + " (0, 1),\n", + " (1, 2),\n", + " (2, 3),\n", + " (3, 4),\n", + " (4, 5),\n", + " (5, 6),\n", + " (6, 7),\n", + " (7, 8),\n", + " (8, 9),\n", + " (9, 10),\n", + " (10, 11),\n", + " (10, 13),\n", + " (11, 12),\n", + " (12, 13),\n", + " (13, 14),]" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "build_from_graph = DynapcnnNetworkGraph.build_from_graph_()" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + " [ ENTERED build_from_graph() ]\n", + "\n", + "node to DynapcnnLayer mapping:\n", + "DynapcnnLayer index: 0\n", + " [0 ]\n", + " [1 ]\n", + " [2 ]\n", + "DynapcnnLayer index: 1\n", + " [3 ]\n", + " [4 ]\n", + " [5 ]\n", + "DynapcnnLayer index: 2\n", + " [6 ]\n", + " [7 ]\n", + " [8 ]\n", + "DynapcnnLayer index: 3\n", + " [9 ]\n", + " [10 ]\n", + "DynapcnnLayer index: 4\n", + " [13 ]\n", + " [14 ]\n", + "DynapcnnLayer index: 5\n", + " [11 ]\n", + " [12 ]\n", + "\n", + "DynapcnnLayer to DynapcnnLayer mapping:\n", + "DynapcnnLayer 0 destinations: [1]\n", + "DynapcnnLayer 1 destinations: [2]\n", + "DynapcnnLayer 2 destinations: [3]\n", + "DynapcnnLayer 3 destinations: [5, 4]\n", + "DynapcnnLayer 5 destinations: [4]\n", + "DynapcnnLayer 4 destinations: []\n" + ] + }, + { + "data": { + "text/plain": [ + "(None, None)" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "build_from_graph(dummy_layer_list, None, dummy_edges)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Test 2" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "dummy_layer_list = [\n", + " torch.nn.Conv2d(1, 20, kernel_size=(5, 5), stride=(1, 1), bias=False),\n", + " snb.layers.iaf.IAFSqueeze(batch_size=1),\n", + " torch.nn.AvgPool2d(kernel_size=2, stride=2, padding=0),\n", + " torch.nn.Conv2d(1, 20, kernel_size=(5, 5), stride=(1, 1), bias=False),\n", + " snb.layers.iaf.IAFSqueeze(batch_size=1),\n", + " torch.nn.AvgPool2d(kernel_size=2, stride=2, padding=0),\n", + " torch.nn.Conv2d(1, 20, kernel_size=(5, 5), stride=(1, 1), bias=False),\n", + " snb.layers.iaf.IAFSqueeze(batch_size=1),\n", + " torch.nn.AvgPool2d(kernel_size=2, stride=2, padding=0),\n", + " torch.nn.Linear(in_features=128, out_features=500, bias=False),\n", + " snb.layers.iaf.IAFSqueeze(batch_size=1),\n", + " torch.nn.Linear(in_features=128, out_features=500, bias=False),\n", + " snb.layers.iaf.IAFSqueeze(batch_size=1),\n", + " torch.nn.Linear(in_features=128, out_features=500, bias=False),\n", + " snb.layers.iaf.IAFSqueeze(batch_size=1),]" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "dummy_edges = [\n", + " (0, 1),\n", + " (1, 2),\n", + " (1, 6),\n", + " (2, 3),\n", + " (3, 4),\n", + " (4, 5),\n", + " (5, 6),\n", + " (6, 7),\n", + " (7, 8),\n", + " (8, 9),\n", + " (9, 10),\n", + " (10, 11),\n", + " (10, 13),\n", + " (11, 12),\n", + " (12, 13),\n", + " (13, 14),]" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "build_from_graph = DynapcnnNetworkGraph.build_from_graph_()" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + " [ ENTERED build_from_graph() ]\n", + "\n", + "node to DynapcnnLayer mapping:\n", + "DynapcnnLayer index: 0\n", + " [0 ]\n", + " [1 ]\n", + " [2 ]\n", + "DynapcnnLayer index: 1\n", + " [3 ]\n", + " [4 ]\n", + " [5 ]\n", + "DynapcnnLayer index: 2\n", + " [6 ]\n", + " [7 ]\n", + " [8 ]\n", + "DynapcnnLayer index: 3\n", + " [9 ]\n", + " [10 ]\n", + "DynapcnnLayer index: 4\n", + " [13 ]\n", + " [14 ]\n", + "DynapcnnLayer index: 5\n", + " [11 ]\n", + " [12 ]\n", + "\n", + "DynapcnnLayer to DynapcnnLayer mapping:\n", + "DynapcnnLayer 0 destinations: [2, 1]\n", + "DynapcnnLayer 1 destinations: [2]\n", + "DynapcnnLayer 2 destinations: [3]\n", + "DynapcnnLayer 3 destinations: [5, 4]\n", + "DynapcnnLayer 5 destinations: [4]\n", + "DynapcnnLayer 4 destinations: []\n" + ] + }, + { + "data": { + "text/plain": [ + "(None, None)" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "build_from_graph(dummy_layer_list, None, dummy_edges)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "speck-rescnn", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tests/test_nonsequential/jit_based_tracer_sinabs.ipynb b/tests/test_nonsequential/jit_based_tracer_sinabs.ipynb index f1140fa3..65cf8de4 100644 --- a/tests/test_nonsequential/jit_based_tracer_sinabs.ipynb +++ b/tests/test_nonsequential/jit_based_tracer_sinabs.ipynb @@ -11,7 +11,8 @@ "import networkx as nx\n", "import matplotlib.pyplot as plt\n", "from sinabs.from_torch import from_model\n", - "from sinabs.backend.dynapcnn import DynapcnnNetwork, DynapcnnNetworkGraph" + "from sinabs.backend.dynapcnn import DynapcnnNetwork, DynapcnnNetworkGraph\n", + "import sinabs as snb" ] }, { @@ -22,7 +23,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 2, @@ -278,34 +279,32 @@ "\n", " [ ENTERED build_from_graph() ]\n", "\n", - "0\n", - " 0 \n", - " 1 \n", - " 2 \n", + "node to DynapcnnLayer mapping:\n", + "DynapcnnLayer index: 0\n", + " [0 ]\n", + " [1 ]\n", + " [2 ]\n", + "DynapcnnLayer index: 1\n", + " [3 ]\n", + " [4 ]\n", + " [5 ]\n", + "DynapcnnLayer index: 2\n", + " [6 ]\n", + " [7 ]\n", + " [8 ]\n", + "DynapcnnLayer index: 3\n", + " [9 ]\n", + " [10 ]\n", + "DynapcnnLayer index: 4\n", + " [11 ]\n", + " [12 ]\n", "\n", - "\n", - "1\n", - " 3 \n", - " 4 \n", - " 5 \n", - "\n", - "\n", - "2\n", - " 6 \n", - " 7 \n", - " 8 \n", - "\n", - "\n", - "3\n", - " 9 \n", - " 10 \n", - "\n", - "\n", - "4\n", - " 11 \n", - " 12 \n", - "\n", - "\n" + "DynapcnnLayer to DynapcnnLayer mapping:\n", + "DynapcnnLayer 0 destinations: [1]\n", + "DynapcnnLayer 1 destinations: [2]\n", + "DynapcnnLayer 2 destinations: [3]\n", + "DynapcnnLayer 3 destinations: [4]\n", + "DynapcnnLayer 4 destinations: []\n" ] } ], @@ -385,13 +384,6 @@ "for edge in hw_model.sinabs_edges:\n", " print(edge)" ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## DynapCNN Configuration" - ] } ], "metadata": { From 284e98be371eb930910d73318c1718e72e94353d Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Fri, 12 Apr 2024 19:18:42 +0200 Subject: [PATCH 009/379] building DynapcnnLayer objects and their respective destinations from mappers --- .../dynapcnn/dynapcnn_network_graph.py | 58 +++--- sinabs/backend/dynapcnn/exceptions.py | 13 ++ sinabs/backend/dynapcnn/utils.py | 174 +++++++++++++++--- .../jit_based_tracer_sinabs.ipynb | 97 ++++++---- tests/test_nonsequential/test_speck.ipynb | 156 ++++++++++++++++ 5 files changed, 408 insertions(+), 90 deletions(-) create mode 100644 tests/test_nonsequential/test_speck.ipynb diff --git a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py index ba3b49ca..27a386a6 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py @@ -82,43 +82,28 @@ def __init__( input_shape = infer_input_shape(self.layers, input_shape=input_shape) assert len(input_shape) == 3, "infer_input_shape did not return 3-tuple" - # Build model from layers. - self.sequence = build_from_list( - self.layers, - in_shape=input_shape, - discretize=discretize, - dvs_input=self.dvs_input, - ) - # Get sinabs graph. self.sinabs_edges = self.get_sinabs_edges(snn) - self.layers_to_cores_map, self.core_to_core_map = build_from_graph( + # Build model from layers. + self.dynapcnn_layers, self.nodes_to_dcnnl_map, self.dcnnl_to_dcnnl_map = build_from_graph( + discretize=discretize, layers=self.layers, in_shape=input_shape, edges=self.sinabs_edges) + + def __str__(self): + pretty_print = '' + for idx, layer_dest in self.dynapcnn_layers.items(): + layer = layer_dest['layer'] + dest = layer_dest['destinations'] + pretty_print += f'\nlayer index: {idx}\nlayer modules: {layer}\nlayer destinations: {dest}\n' + return pretty_print @staticmethod - def build_from_graph_(): # @TODO used for debug only (remove when class is complete). + def build_from_graph_(): # @TODO used for debug only (remove when class is complete). return build_from_graph - def get_sinabs_edges(self, sinabs_model): - """ Converts the computational graph extracted from 'sinabs_model.analog_model' into its equivalent - representation for the 'sinabs_model.spiking_model'. - """ - # parse original graph to ammend edges containing nodes dropped in 'convert_model_to_layer_list()'. - sinabs_edges = self.graph_tracer.remove_ignored_nodes(DEFAULT_IGNORED_LAYER_TYPES) - - if DynapcnnNetworkGraph.was_spiking_output_added(sinabs_model): - # spiking output layer has been added: create new edge. - last_edge = sinabs_edges[-1] - new_edge = (last_edge[1], last_edge[1]+1) - sinabs_edges.append(new_edge) - else: - pass - - return sinabs_edges - @staticmethod def was_spiking_output_added(sinabs_model): """ Compares the models outputed by 'sinabs.from_torch.from_model()' to check if @@ -140,4 +125,21 @@ def was_spiking_output_added(sinabs_model): # throw error return False else: - return False \ No newline at end of file + return False + + def get_sinabs_edges(self, sinabs_model): + """ Converts the computational graph extracted from 'sinabs_model.analog_model' into its equivalent + representation for the 'sinabs_model.spiking_model'. + """ + # parse original graph to ammend edges containing nodes dropped in 'convert_model_to_layer_list()'. + sinabs_edges = self.graph_tracer.remove_ignored_nodes(DEFAULT_IGNORED_LAYER_TYPES) + + if DynapcnnNetworkGraph.was_spiking_output_added(sinabs_model): + # spiking output layer has been added: create new edge. + last_edge = sinabs_edges[-1] + new_edge = (last_edge[1], last_edge[1]+1) + sinabs_edges.append(new_edge) + else: + pass + + return sinabs_edges \ No newline at end of file diff --git a/sinabs/backend/dynapcnn/exceptions.py b/sinabs/backend/dynapcnn/exceptions.py index cd5c63aa..832467c3 100644 --- a/sinabs/backend/dynapcnn/exceptions.py +++ b/sinabs/backend/dynapcnn/exceptions.py @@ -17,3 +17,16 @@ class InputConfigurationError(Exception): """Is raised when input to dynapcnn is not configured correctly.""" pass + +class WrongModuleCount(Exception): + dynapcnnlayer_indx: type + modules_count: type + + def __init__(self, dynapcnnlayer_indx, modules_count): + super().__init__(f"A DynapcnnLayer {dynapcnnlayer_indx} should have 2 or 3 modules but found {modules_count}.") + +class WrongPoolingModule(Exception): + pooling_module: type + + def __init__(self, pooling_module,): + super().__init__(f"The function 'utils.build_SumPool2d(mod)' expects 'mod = nn.AvgPool2d' but got 'mod = {pooling_module}'.") diff --git a/sinabs/backend/dynapcnn/utils.py b/sinabs/backend/dynapcnn/utils.py index 98a1b17b..07dfdb9b 100644 --- a/sinabs/backend/dynapcnn/utils.py +++ b/sinabs/backend/dynapcnn/utils.py @@ -1,5 +1,5 @@ from copy import deepcopy -from typing import TYPE_CHECKING, List, Optional, Tuple, Type, Union +from typing import TYPE_CHECKING, List, Optional, Tuple, Type, Union, Dict import torch import torch.nn as nn @@ -10,7 +10,7 @@ from .crop2d import Crop2d from .dvs_layer import DVSLayer, expand_to_pair from .dynapcnn_layer import DynapcnnLayer -from .exceptions import InputConfigurationError, MissingLayer, UnexpectedLayer +from .exceptions import InputConfigurationError, MissingLayer, UnexpectedLayer, WrongModuleCount, WrongPoolingModule from .flipdims import FlipDims from .sinabs_edges_handler import process_edge, get_dynapcnnlayers_destinations @@ -242,7 +242,6 @@ def construct_next_pooling_layer( rescale_factor = 1 cumulative_pooling = expand_to_pair(1) - idx_next = idx_start # Figure out pooling dims while idx_next < len(layers): @@ -270,6 +269,7 @@ def construct_next_pooling_layer( cumulative_pooling[0] * pooling[0], cumulative_pooling[1] * pooling[1], ) + # Update rescaling factor if isinstance(lyr, nn.AvgPool2d): rescale_factor *= pooling[0] * pooling[1] @@ -541,43 +541,161 @@ def extend_readout_layer(model: "DynapcnnNetwork") -> "DynapcnnNetwork": ) # run a forward pass to initialize the new weights and last IAF return model -def build_from_graph(layers: List[nn.Module], in_shape, edges: List[Tuple[int, int]]): - """ .""" - print('\n [ ENTERED build_from_graph() ]\n') +def build_from_graph( + discretize: bool, + layers: List[nn.Module], + in_shape: Tuple[int, int, int], + edges: List[Tuple[int, int]]) -> Tuple[List[DynapcnnLayer], Dict[int, Dict[int, nn.Module]], Dict[int, List[int]]]: + """ Parses each edge of a 'sinabs_mode.spiking_model' computational graph. Each node (layer) is assigned to a + DynapcnnLayer object. The target destination of each DynapcnnLayer is computed via edges connecting nodes in + different DynapcnnLayer objects. + + Parameters + ---------- + discretize: If True, discretize the parameters and thresholds. This is needed for uploading weights to dynapcnn. + Set to False only for testing purposes. + layers : List of modules returned by 'utils.convert_model_to_layer_list()'. + in_shape : Tuple describing the input to the very first layer (batch_size, hight, width). + edges : List of edges returned by 'DynapcnnNetworkGraph.get_sinabs_edges()'. + + Returns + ---------- + dynapcnn_layers : A dictionary containing DynapcnnLayer objects and a list with their destinations. + nodes_to_dcnnl_map: Sets of layers that comprise a DynapcnnLayer. + key [int]: DynapcnnLayer index. + value [dict]: node index as 'key' and its module as 'value'. + dcnnl_to_dcnnl_map: List of destinations for each DynapcnnLayer in 'dynapcnn_layers'. + key [int]: index of a DynapcnnLayer. + value [list]: indexes of DynapcnnLayer a layer targets. + """ # @TODO the graph extraction is not yet considering DVS input. + # dvs_layer, lyr_indx_next, rescale_factor = construct_dvs_layer( # layers, # input_shape=in_shape, # idx_start=0, # dvs_input=False) dvs_layer = None - - ''' - holds 'blocks' of modules that populate a single DynapcnnLayer object. - key (int): DynapcnnLayer index. - value (dict): node index (key) and its type (module). - ''' - layers_to_cores_map = {} - - if dvs_layer is not None: # @TODO the graph extraction is not yet considering DVS input. - pass - else: # looping though graph edges. + rescale_factor = 1 + + nodes_to_dcnnl_map = {} # mapper from nodes to sets of layers that populate a DynapcnnLayer. + + if dvs_layer is not None: + pass # @TODO the graph extraction is not yet considering DVS input. + else: for edge in edges: - process_edge(layers, edge, layers_to_cores_map) + process_edge( # figure out to which (future) DynapcnnLayer each node will belong to. + layers, edge, nodes_to_dcnnl_map) + + # look for edges between connecting nodes in different (future) DynapcnnLayer. + dcnnl_to_dcnnl_map = get_dynapcnnlayers_destinations(layers, edges, nodes_to_dcnnl_map) - print('node to DynapcnnLayer mapping:') + # turn sets of layers into DynapcnnLayer objects. + dynapcnn_layers = construct_dynapcnnlayers_from_mapper( + discretize, nodes_to_dcnnl_map, dcnnl_to_dcnnl_map, in_shape, rescale_factor) + + return dynapcnn_layers, nodes_to_dcnnl_map, dcnnl_to_dcnnl_map - for key, val in layers_to_cores_map.items(): - print('DynapcnnLayer index: ', key) - for k, v in val.items(): - print(f' [{k} {type(v)}]') +def construct_dynapcnnlayers_from_mapper( + discretize: bool, + nodes_to_dcnnl_map: dict, + dcnnl_to_dcnnl_map: dict, + input_shape: Union[Tuple[int, int], Tuple[int, int, int]], + rescale_factor: int) -> Dict[int, Dict[DynapcnnLayer, List]]: + """ Consumes a dictionaries containing sets of layers to be used to populate a DynapcnnLayer object. + + Parameters + ---------- + rescale_factor: Rescaling factor needed when turning AvgPool to SumPool. May differ from the pooling kernel in + certain cases. + Returns + ---------- + dynapcnn_layers: A dictionary containing DynapcnnLayer objects and a list with their destinations. + """ - core_to_core_map = get_dynapcnnlayers_destinations(layers, edges, layers_to_cores_map) + dynapcnn_layers = {} + + for dynapcnnl_indx, layer_modules in nodes_to_dcnnl_map.items(): + dynapcnnlayer, input_shape, rescale_factor = construct_dynapcnnlayer( + discretize, layer_modules, dynapcnnl_indx, input_shape, rescale_factor) + + dynapcnn_layers[dynapcnnl_indx] = { + 'layer': dynapcnnlayer, + 'destinations': dcnnl_to_dcnnl_map[dynapcnnl_indx] + } + + return dynapcnn_layers + +def construct_dynapcnnlayer( + discretize: bool, + layer_modules: dict, + layer_index: int, + input_shape: Union[Tuple[int, int], Tuple[int, int, int]], + rescale_factor: int) -> Tuple[DynapcnnLayer, Union[Tuple[int, int], Tuple[int, int, int]], int]: + """ Extract the modules (layers) in a dictionary and uses them to instantiate a DynapcnnLayer object. """ + lyr_conv = None + lyr_spk = None + lyr_pool = None + rescale_factor_after_pooling = 1 + + iterator = iter(layer_modules.items()) # 'next(iterator)' returns the node id in the computational graph and the layer (nn.Module) associated with it. + + if len(layer_modules) == 3: # there's a pooling layer. + _, lyr_conv = next(iterator) + _, lyr_spk = next(iterator) + _, _pool = next(iterator) + + lyr_pool, rescale_factor_after_pooling = build_SumPool2d(_pool) + + elif len(layer_modules) == 2: # there's only a conv layer folowed by a neuron layer. + _, lyr_conv = next(iterator) + _, lyr_spk = next(iterator) - print('\nDynapcnnLayer to DynapcnnLayer mapping:') + else: + raise WrongModuleCount(layer_index, len(layer_modules)) + + dynapcnnlayer = DynapcnnLayer( + conv = lyr_conv, + spk = lyr_spk, + pool = lyr_pool, + in_shape = input_shape, + discretize = discretize, + rescale_weights = rescale_factor, + ) + + return dynapcnnlayer, dynapcnnlayer.get_output_shape(), rescale_factor_after_pooling - for key, val in core_to_core_map.items(): - print(f'DynapcnnLayer {key} destinations: {val}') +def build_SumPool2d(module: nn.AvgPool2d) -> Tuple[sl.SumPool2d, int]: + """ Converts a 'nn.AvgPool2d' into a 'sl.SumPool2d' layer. """ + + if isinstance(module, nn.AvgPool2d): + if module.padding != 0: + raise ValueError("Padding is not supported for the pooling layers.") + elif isinstance(module, sl.SumPool2d): + pass + else: + raise WrongPoolingModule(type(module)) - return layers_to_cores_map, core_to_core_map \ No newline at end of file + rescale_factor = 1 + cumulative_pooling = expand_to_pair(1) + pooling = expand_to_pair(module.kernel_size) + + if module.stride is not None: + stride = expand_to_pair(module.stride) + if pooling != stride: + raise ValueError( + f"Stride length {module.stride} should be the same as pooling kernel size {module.kernel_size}" + ) + + cumulative_pooling = ( # compute cumulative pooling. + cumulative_pooling[0] * pooling[0], + cumulative_pooling[1] * pooling[1], + ) + + if isinstance(module, nn.AvgPool2d): # update rescaling factor. + rescale_factor *= pooling[0] * pooling[1] + + lyr_pool = sl.SumPool2d(cumulative_pooling) + + return lyr_pool, rescale_factor \ No newline at end of file diff --git a/tests/test_nonsequential/jit_based_tracer_sinabs.ipynb b/tests/test_nonsequential/jit_based_tracer_sinabs.ipynb index 65cf8de4..49e3f409 100644 --- a/tests/test_nonsequential/jit_based_tracer_sinabs.ipynb +++ b/tests/test_nonsequential/jit_based_tracer_sinabs.ipynb @@ -23,7 +23,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 2, @@ -271,54 +271,83 @@ "cell_type": "code", "execution_count": 10, "metadata": {}, + "outputs": [], + "source": [ + "hw_model = DynapcnnNetworkGraph(\n", + " sinabs_model,\n", + " discretize=True,\n", + " input_shape=input_shape\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", - " [ ENTERED build_from_graph() ]\n", + "layer index: 0\n", + "layer modules: DynapcnnLayer(\n", + " (conv_layer): Conv2d(1, 20, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", + " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(635.), min_v_mem=Parameter containing:\n", + " tensor(-635.), batch_size=1, num_timesteps=-1)\n", + " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", + ")\n", + "layer destinations: [1]\n", "\n", - "node to DynapcnnLayer mapping:\n", - "DynapcnnLayer index: 0\n", - " [0 ]\n", - " [1 ]\n", - " [2 ]\n", - "DynapcnnLayer index: 1\n", - " [3 ]\n", - " [4 ]\n", - " [5 ]\n", - "DynapcnnLayer index: 2\n", - " [6 ]\n", - " [7 ]\n", - " [8 ]\n", - "DynapcnnLayer index: 3\n", - " [9 ]\n", - " [10 ]\n", - "DynapcnnLayer index: 4\n", - " [11 ]\n", - " [12 ]\n", + "layer index: 1\n", + "layer modules: DynapcnnLayer(\n", + " (conv_layer): Conv2d(20, 32, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", + " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(11361.), min_v_mem=Parameter containing:\n", + " tensor(-11361.), batch_size=1, num_timesteps=-1)\n", + " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", + ")\n", + "layer destinations: [2]\n", "\n", - "DynapcnnLayer to DynapcnnLayer mapping:\n", - "DynapcnnLayer 0 destinations: [1]\n", - "DynapcnnLayer 1 destinations: [2]\n", - "DynapcnnLayer 2 destinations: [3]\n", - "DynapcnnLayer 3 destinations: [4]\n", - "DynapcnnLayer 4 destinations: []\n" + "layer index: 2\n", + "layer modules: DynapcnnLayer(\n", + " (conv_layer): Conv2d(32, 128, kernel_size=(3, 3), stride=(1, 1), bias=False)\n", + " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(8621.), min_v_mem=Parameter containing:\n", + " tensor(-8621.), batch_size=1, num_timesteps=-1)\n", + " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", + ")\n", + "layer destinations: [3]\n", + "\n", + "layer index: 3\n", + "layer modules: DynapcnnLayer(\n", + " (conv_layer): Conv2d(128, 500, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(5747.), min_v_mem=Parameter containing:\n", + " tensor(-5747.), batch_size=1, num_timesteps=-1)\n", + ")\n", + "layer destinations: [4]\n", + "\n", + "layer index: 4\n", + "layer modules: DynapcnnLayer(\n", + " (conv_layer): Conv2d(500, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(2841.), min_v_mem=Parameter containing:\n", + " tensor(-2841.), batch_size=1, num_timesteps=-1)\n", + ")\n", + "layer destinations: []\n", + "\n" ] } ], "source": [ - "hw_model = DynapcnnNetworkGraph(\n", - " sinabs_model,\n", - " discretize=True,\n", - " input_shape=input_shape\n", - ")" + "print(hw_model)" ] }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 12, "metadata": {}, "outputs": [ { @@ -358,7 +387,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 13, "metadata": {}, "outputs": [ { diff --git a/tests/test_nonsequential/test_speck.ipynb b/tests/test_nonsequential/test_speck.ipynb new file mode 100644 index 00000000..e0d0b307 --- /dev/null +++ b/tests/test_nonsequential/test_speck.ipynb @@ -0,0 +1,156 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "DynapcnnLayer(\n", + " (conv_layer): Conv2d(1, 20, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", + " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(637.), min_v_mem=Parameter containing:\n", + " tensor(-637.), batch_size=1, num_timesteps=-1)\n", + " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", + ")\n", + "DynapcnnLayer(\n", + " (conv_layer): Conv2d(20, 32, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", + " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(11360.), min_v_mem=Parameter containing:\n", + " tensor(-11360.), batch_size=1, num_timesteps=-1)\n", + " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", + ")\n", + "DynapcnnLayer(\n", + " (conv_layer): Conv2d(32, 128, kernel_size=(3, 3), stride=(1, 1), bias=False)\n", + " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(8621.), min_v_mem=Parameter containing:\n", + " tensor(-8621.), batch_size=1, num_timesteps=-1)\n", + " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", + ")\n", + "DynapcnnLayer(\n", + " (conv_layer): Conv2d(128, 500, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(5748.), min_v_mem=Parameter containing:\n", + " tensor(-5748.), batch_size=1, num_timesteps=-1)\n", + ")\n", + "DynapcnnLayer(\n", + " (conv_layer): Conv2d(500, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(2840.), min_v_mem=Parameter containing:\n", + " tensor(-2840.), batch_size=1, num_timesteps=-1)\n", + ")\n", + "Network is valid\n" + ] + }, + { + "data": { + "text/plain": [ + "DynapcnnNetwork(\n", + " (sequence): Sequential(\n", + " (0): DynapcnnLayer(\n", + " (conv_layer): Conv2d(1, 20, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", + " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(637.), min_v_mem=Parameter containing:\n", + " tensor(-637.), batch_size=1, num_timesteps=-1)\n", + " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", + " )\n", + " (1): DynapcnnLayer(\n", + " (conv_layer): Conv2d(20, 32, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", + " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(11360.), min_v_mem=Parameter containing:\n", + " tensor(-11360.), batch_size=1, num_timesteps=-1)\n", + " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", + " )\n", + " (2): DynapcnnLayer(\n", + " (conv_layer): Conv2d(32, 128, kernel_size=(3, 3), stride=(1, 1), bias=False)\n", + " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(8621.), min_v_mem=Parameter containing:\n", + " tensor(-8621.), batch_size=1, num_timesteps=-1)\n", + " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", + " )\n", + " (3): DynapcnnLayer(\n", + " (conv_layer): Conv2d(128, 500, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(5748.), min_v_mem=Parameter containing:\n", + " tensor(-5748.), batch_size=1, num_timesteps=-1)\n", + " )\n", + " (4): DynapcnnLayer(\n", + " (conv_layer): Conv2d(500, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(2840.), min_v_mem=Parameter containing:\n", + " tensor(-2840.), batch_size=1, num_timesteps=-1)\n", + " )\n", + " )\n", + ")" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import torch\n", + "import torch.nn as nn\n", + "from typing import List\n", + "from sinabs.from_torch import from_model\n", + "from sinabs.backend.dynapcnn import DynapcnnNetwork\n", + "\n", + "ann = nn.Sequential(\n", + " nn.Conv2d(1, 20, 5, 1, bias=False),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(2,2),\n", + " nn.Conv2d(20, 32, 5, 1, bias=False),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(2,2),\n", + " nn.Conv2d(32, 128, 3, 1, bias=False),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(2,2),\n", + " nn.Flatten(),\n", + " nn.Linear(128, 500, bias=False),\n", + " nn.ReLU(),\n", + " nn.Linear(500, 10, bias=False),\n", + ")\n", + "\n", + "# Convert your model to SNN\n", + "sinabs_model = from_model(ann, add_spiking_output=True, batch_size=1) # Your sinabs SNN model\n", + "\n", + "# Convert your SNN to `DynapcnnNetwork`\n", + "hw_model = DynapcnnNetwork(\n", + " sinabs_model.spiking_model,\n", + " discretize=True,\n", + " input_shape=(1, 28, 28)\n", + ")\n", + "\n", + "# Deploy model to a dev-kit\n", + "hw_model.to(device=\"speck2edevkit:0\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "speck-rescnn", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 2ad147ef3a55619e4a8bc1ed233c5b1be6bc2b86 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Fri, 12 Apr 2024 20:49:59 +0200 Subject: [PATCH 010/379] (WIP) starting .to() method functionality --- .../dynapcnn/dynapcnn_network_graph.py | 177 ++++++++++++++---- 1 file changed, 145 insertions(+), 32 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py index 27a386a6..27ac054d 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py @@ -1,3 +1,7 @@ +# functionality : ... +# author : Willian Soares Girao +# contact : williansoaresgirao@gmail.com + import time from subprocess import CalledProcessError from typing import List, Optional, Sequence, Tuple, Union @@ -59,34 +63,26 @@ def __init__( dvs_input = False # @TODO for now the graph part is not taking into consideration this. - # Computational graph from original PyTorch module. - self.graph_tracer = GraphTracer( + self.graph_tracer = GraphTracer( # computational graph from original PyTorch module. snn.analog_model, torch.randn((1, *input_shape)) # torch.jit needs the batch dimension. ) - # This attribute stores the location/core-id of each of the DynapcnnLayers upon placement on chip. - self.chip_layers_ordering = [] - self.input_shape = input_shape # convert models to sequential. self.layers = convert_model_to_layer_list( model=snn.spiking_model, ignore=DEFAULT_IGNORED_LAYER_TYPES ) - # Check if dvs input is expected. - if dvs_input: - self.dvs_input = True - else: - self.dvs_input = False + self.dvs_input = dvs_input # check if dvs input is expected. input_shape = infer_input_shape(self.layers, input_shape=input_shape) assert len(input_shape) == 3, "infer_input_shape did not return 3-tuple" - # Get sinabs graph. - self.sinabs_edges = self.get_sinabs_edges(snn) + self.sinabs_edges = self.get_sinabs_edges(snn) # get sinabs graph. - # Build model from layers. - self.dynapcnn_layers, self.nodes_to_dcnnl_map, self.dcnnl_to_dcnnl_map = build_from_graph( + self.dynapcnn_layers, \ + self.nodes_to_dcnnl_map, \ + self.dcnnl_to_dcnnl_map = build_from_graph( # build model from graph edges. discretize=discretize, layers=self.layers, in_shape=input_shape, @@ -103,6 +99,140 @@ def __str__(self): @staticmethod def build_from_graph_(): # @TODO used for debug only (remove when class is complete). return build_from_graph + + def to( + self, + device="cpu", + chip_layers_ordering="auto", + monitor_layers: Optional[Union[List, str]] = None, + config_modifier=None, + slow_clk_frequency: int = None, + ): + """ .""" + self.device = device + + if isinstance(device, torch.device): + return super().to(device) + + elif isinstance(device, str): + device_name, _ = parse_device_id(device) + + if device_name in ChipFactory.supported_devices: # pragma: no cover + + config = self.make_config( # generate config. + chip_layers_ordering=chip_layers_ordering, + device=device, + monitor_layers=monitor_layers, + config_modifier=config_modifier, + ) + + def make_config( + self, + chip_layers_ordering: Union[Sequence[int], str] = "auto", + device="dynapcnndevkit:0", + monitor_layers: Optional[Union[List, str]] = None, + config_modifier=None, + ): + """Prepare and output the `samna` DYNAPCNN configuration for this network. + + Parameters + ---------- + + chip_layers_ordering: sequence of integers or `auto` + The order in which the dynapcnn layers will be used. If `auto`, + an automated procedure will be used to find a valid ordering. + A list of layers on the device where you want each of the model's DynapcnnLayers to be placed. + Note: This list should be the same length as the number of dynapcnn layers in your model. + + device: String + dynapcnndevkit, speck2b or speck2devkit + + monitor_layers: None/List/Str + A list of all layers in the module that you want to monitor. Indexing starts with the first non-dvs layer. + If you want to monitor the dvs layer for eg. + :: + + monitor_layers = ["dvs"] # If you want to monitor the output of the pre-processing layer + monitor_layers = ["dvs", 8] # If you want to monitor preprocessing and layer 8 + monitor_layers = "all" # If you want to monitor all the layers + + If this value is left as None, by default the last layer of the model is monitored. + + config_modifier: + A user configuration modifier method. + This function can be used to make any custom changes you want to make to the configuration object. + + Returns + ------- + Configuration object + Object defining the configuration for the device + + Raises + ------ + ImportError + If samna is not available. + ValueError + If the generated configuration is not valid for the specified device. + """ + config, is_compatible = self._make_config( + chip_layers_ordering=chip_layers_ordering, + device=device, + monitor_layers=monitor_layers, + config_modifier=config_modifier, + ) + + if is_compatible: # validate config. + print("Network is valid") + return config + else: + raise ValueError(f"Generated config is not valid for {device}") + + def _make_config( + self, + chip_layers_ordering: Union[Sequence[int], str] = "auto", + device="dynapcnndevkit:0", + monitor_layers: Optional[Union[List, str]] = None, + config_modifier=None, + ) -> Tuple["SamnaConfiguration", bool]: + """ Prepare and output the `samna` configuration for this network. """ + + config_builder = ChipFactory(device).get_config_builder() + + has_dvs_layer = isinstance(self.dynapcnn_layers[0]['layer'], DVSLayer) + + if chip_layers_ordering == "auto": # figure out mapping of each DynapcnnLayer into one core. + chip_layers_ordering = config_builder.get_valid_mapping(self) + + else: # mapping from each DynapcnnLayer into cores has been provided. + if has_dvs_layer: # @TODO maybe this has to be modified given the new representation of layers in a dictionary (instead of a list). + chip_layers_ordering = chip_layers_ordering[: len(self.sequence) - 1] + + chip_layers_ordering = chip_layers_ordering[: len(self.sequence)] + + def get_sinabs_edges(self, sinabs_model): + """ Converts the computational graph extracted from 'sinabs_model.analog_model' into its equivalent + representation for the 'sinabs_model.spiking_model'. + + Parameters + ---------- + sinabs_model: ... + + Returns + sinabs_edges: ... + ---------- + """ + # parse original graph to ammend edges containing nodes dropped in 'convert_model_to_layer_list()'. + sinabs_edges = self.graph_tracer.remove_ignored_nodes(DEFAULT_IGNORED_LAYER_TYPES) + + if DynapcnnNetworkGraph.was_spiking_output_added(sinabs_model): + # spiking output layer has been added: create new edge. + last_edge = sinabs_edges[-1] + new_edge = (last_edge[1], last_edge[1]+1) + sinabs_edges.append(new_edge) + else: + pass + + return sinabs_edges @staticmethod def was_spiking_output_added(sinabs_model): @@ -125,21 +255,4 @@ def was_spiking_output_added(sinabs_model): # throw error return False else: - return False - - def get_sinabs_edges(self, sinabs_model): - """ Converts the computational graph extracted from 'sinabs_model.analog_model' into its equivalent - representation for the 'sinabs_model.spiking_model'. - """ - # parse original graph to ammend edges containing nodes dropped in 'convert_model_to_layer_list()'. - sinabs_edges = self.graph_tracer.remove_ignored_nodes(DEFAULT_IGNORED_LAYER_TYPES) - - if DynapcnnNetworkGraph.was_spiking_output_added(sinabs_model): - # spiking output layer has been added: create new edge. - last_edge = sinabs_edges[-1] - new_edge = (last_edge[1], last_edge[1]+1) - sinabs_edges.append(new_edge) - else: - pass - - return sinabs_edges \ No newline at end of file + return False \ No newline at end of file From 62d0c0f33b4b37f91cba910b97ad2bf57651a752 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Mon, 15 Apr 2024 14:14:33 +0200 Subject: [PATCH 011/379] (WIP) DynapcnnNetworkGraph.to() - from DynapcnnLayer to CNNLayerConfig --- sinabs/backend/dynapcnn/chips/dynapcnn.py | 119 +++++++++++++----- sinabs/backend/dynapcnn/config_builder.py | 39 ++++-- .../dynapcnn/dynapcnn_network_graph.py | 50 ++++++-- sinabs/backend/dynapcnn/exceptions.py | 6 + sinabs/backend/dynapcnn/mapping.py | 43 +++++-- sinabs/backend/dynapcnn/utils.py | 10 +- 6 files changed, 200 insertions(+), 67 deletions(-) diff --git a/sinabs/backend/dynapcnn/chips/dynapcnn.py b/sinabs/backend/dynapcnn/chips/dynapcnn.py index 849932e9..14209e45 100644 --- a/sinabs/backend/dynapcnn/chips/dynapcnn.py +++ b/sinabs/backend/dynapcnn/chips/dynapcnn.py @@ -1,5 +1,5 @@ import copy -from typing import List +from typing import List, Union, Dict from warnings import warn import samna @@ -12,6 +12,7 @@ from sinabs.backend.dynapcnn.dynapcnn_layer import DynapcnnLayer from sinabs.backend.dynapcnn.mapping import LayerConstraints +import sinabs class DynapcnnConfigBuilder(ConfigBuilder): @classmethod @@ -190,6 +191,7 @@ def write_dynapcnn_layer_config( # Update configuration of the DYNAPCNN layer chip_layer.dimensions = config_dict["dimensions"] config_dict.pop("dimensions") + for i in range(len(config_dict["destinations"])): if "pooling" in config_dict["destinations"][i]: chip_layer.destinations[i].pooling = config_dict["destinations"][i][ @@ -201,38 +203,97 @@ def write_dynapcnn_layer_config( setattr(chip_layer, param, value) except TypeError as e: raise TypeError(f"Unexpected parameter {param} or value. {e}") - + @classmethod - def build_config(cls, model: "DynapcnnNetwork", chip_layers: List[int]): - layers = model.sequence - config = cls.get_default_config() - - has_dvs_layer = False - i_cnn_layer = 0 # Instantiate an iterator for the cnn cores - for i, chip_equivalent_layer in enumerate(layers): - if isinstance(chip_equivalent_layer, DVSLayer): - chip_layer = config.dvs_layer - cls.write_dvs_layer_config(chip_equivalent_layer, chip_layer) - has_dvs_layer = True - elif isinstance(chip_equivalent_layer, DynapcnnLayer): - chip_layer = config.cnn_layers[chip_layers[i_cnn_layer]] - cls.write_dynapcnn_layer_config(chip_equivalent_layer, chip_layer) - i_cnn_layer += 1 - else: - # in our generated network there is a spurious layer... - # should never happen - raise TypeError("Unexpected layer in the model") + def write_dynapcnn_layer_config_graph( + cls, dcnnl_data: dict, chip_layer: "CNNLayerConfig" + ): + """ .""" + + config_dict = cls.get_dynapcnn_layer_config_dict( # extracting from a DynapcnnLayer the config. variables for its CNNLayerConfig. + layer=dcnnl_data['layer']) + + chip_layer.dimensions = config_dict["dimensions"] + config_dict.pop("dimensions") + + pooling = None + if "pooling" in config_dict["destinations"][0]: + pooling = config_dict["destinations"][0]["pooling"] # TODO make pooling be destination-dependent. + config_dict.pop("destinations") + + for dest_idx in range(len(dcnnl_data['destinations'])): # configuring the destinations for this DynapcnnLayer. + chip_layer.destinations[dest_idx].enable = True + chip_layer.destinations[dest_idx].layer = dcnnl_data['destinations'][dest_idx] - if i == len(layers) - 1: - # last layer - chip_layer.destinations[0].enable = False + if isinstance(pooling, int): + chip_layer.destinations[dest_idx].pooling = pooling + + if len(dcnnl_data['destinations']) == 0: # this is the output layer. + chip_layer.destinations[0].enable = False + chip_layer.destinations[1].enable = False + + for param, value in config_dict.items(): + try: + setattr(chip_layer, param, value) + except TypeError as e: + raise TypeError(f"Unexpected parameter {param} or value. {e}") + + @classmethod + def build_config(cls, model: Union["DynapcnnNetwork", "DynapcnnNetworkGraph"], chip_layers: List[int]): + + if type(model) == sinabs.backend.dynapcnn.dynapcnn_network.DynapcnnNetwork: + layers = model.sequence + config = cls.get_default_config() + + has_dvs_layer = False + i_cnn_layer = 0 # Instantiate an iterator for the cnn cores + for i, chip_equivalent_layer in enumerate(layers): + if isinstance(chip_equivalent_layer, DVSLayer): + chip_layer = config.dvs_layer + cls.write_dvs_layer_config(chip_equivalent_layer, chip_layer) + has_dvs_layer = True + elif isinstance(chip_equivalent_layer, DynapcnnLayer): + chip_layer = config.cnn_layers[chip_layers[i_cnn_layer]] + cls.write_dynapcnn_layer_config(chip_equivalent_layer, chip_layer) + i_cnn_layer += 1 + else: + # in our generated network there is a spurious layer... + # should never happen + raise TypeError("Unexpected layer in the model") + + if i == len(layers) - 1: + # last layer + chip_layer.destinations[0].enable = False + else: + # Set destination layer + chip_layer.destinations[0].layer = chip_layers[i_cnn_layer] + chip_layer.destinations[0].enable = True + + if not has_dvs_layer: + config.dvs_layer.pass_sensor_events = False + + elif type(model) == sinabs.backend.dynapcnn.dynapcnn_network_graph.DynapcnnNetworkGraph: + config = cls.get_default_config() + has_dvs_layer = False + + for _, layer_data in model.dynapcnn_layers.items(): + if isinstance(layer_data['layer'], DVSLayer): + pass # TODO DVSLayer not supported yet. + + elif isinstance(layer_data['layer'], DynapcnnLayer): + chip_layer = config.cnn_layers[layer_data['core_idx']] + cls.write_dynapcnn_layer_config_graph(layer_data, chip_layer) + + else: + raise TypeError("Unexpected layer in the model.") # shouldn't happen since type checks are made previously. + + if not has_dvs_layer: # TODO DVSLayer not supported yet. + config.dvs_layer.pass_sensor_events = False else: - # Set destination layer - chip_layer.destinations[0].layer = chip_layers[i_cnn_layer] - chip_layer.destinations[0].enable = True + config.dvs_layer.pass_sensor_events = False - if not has_dvs_layer: - config.dvs_layer.pass_sensor_events = False + else: + raise TypeError(f"Unexpected model {type(model)}.") return config diff --git a/sinabs/backend/dynapcnn/config_builder.py b/sinabs/backend/dynapcnn/config_builder.py index a96e04c6..17bf86c3 100644 --- a/sinabs/backend/dynapcnn/config_builder.py +++ b/sinabs/backend/dynapcnn/config_builder.py @@ -3,10 +3,13 @@ from typing import List import samna +import sinabs.backend +import sinabs.backend.dynapcnn from .dvs_layer import DVSLayer from .mapping import LayerConstraints, get_valid_mapping - +import sinabs +from .exceptions import InvalidModel class ConfigBuilder(ABC): @classmethod @@ -75,15 +78,31 @@ def get_valid_mapping(cls, model: "DynapcnnNetwork") -> List[int]: The index of the core on chip to which the i-th layer in the model is mapped is the value of the i-th entry in the list. """ - mapping = get_valid_mapping(model, cls.get_constraints()) - # turn the mapping into a dict - mapping = {m[0]: m[1] for m in mapping} - # Check if there is a dvs layer in the model - num_dynapcnn_cores = len(model.sequence) - if isinstance(model.sequence[0], DVSLayer): - num_dynapcnn_cores -= 1 - # apply the mapping - chip_layers_ordering = [mapping[i] for i in range(num_dynapcnn_cores)] + + chip_layers_ordering = [] + + if type(model) == sinabs.backend.dynapcnn.dynapcnn_network.DynapcnnNetwork: + mapping = get_valid_mapping(model, cls.get_constraints()) + # turn the mapping into a dict + mapping = {m[0]: m[1] for m in mapping} + # Check if there is a dvs layer in the model + num_dynapcnn_cores = len(model.sequence) + if isinstance(model.sequence[0], DVSLayer): + num_dynapcnn_cores -= 1 + # apply the mapping + chip_layers_ordering = [mapping[i] for i in range(num_dynapcnn_cores)] + + elif type(model) == sinabs.backend.dynapcnn.dynapcnn_network_graph.DynapcnnNetworkGraph: + mapping = get_valid_mapping(model, cls.get_constraints()) + + if isinstance(model.dynapcnn_layers[0]['layer'], DVSLayer): + pass # TODO not handling DVSLayer yet. + + for (dcnnl, core_idx) in mapping: + model.dynapcnn_layers[dcnnl]['core_idx'] = core_idx + else: + raise InvalidModel(model) + return chip_layers_ordering @classmethod diff --git a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py index 27ac054d..b1503373 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py @@ -90,15 +90,12 @@ def __init__( def __str__(self): pretty_print = '' - for idx, layer_dest in self.dynapcnn_layers.items(): - layer = layer_dest['layer'] - dest = layer_dest['destinations'] - pretty_print += f'\nlayer index: {idx}\nlayer modules: {layer}\nlayer destinations: {dest}\n' + for idx, layer_data in self.dynapcnn_layers.items(): + layer = layer_data['layer'] + dest = layer_data['destinations'] + core = layer_data['core_idx'] + pretty_print += f'\nlayer index: {idx}\nlayer modules: {layer}\nlayer destinations: {dest}\nassigned core: {core}\n' return pretty_print - - @staticmethod - def build_from_graph_(): # @TODO used for debug only (remove when class is complete). - return build_from_graph def to( self, @@ -204,10 +201,35 @@ def _make_config( chip_layers_ordering = config_builder.get_valid_mapping(self) else: # mapping from each DynapcnnLayer into cores has been provided. - if has_dvs_layer: # @TODO maybe this has to be modified given the new representation of layers in a dictionary (instead of a list). - chip_layers_ordering = chip_layers_ordering[: len(self.sequence) - 1] + if has_dvs_layer: + pass # TODO not handling DVSLayer yet. + + config = config_builder.build_config(self, []) # update config. + + if self.input_shape and self.input_shape[0] == 1: # ??? + config.dvs_layer.merge = True + + monitor_chip_layers = [] # TODO all this monitoring part needs validation still. + if monitor_layers is None: # check if any monitoring is enabled (if not, enable monitoring for the last layer). + for _, dcnnl_data in self.dynapcnn_layers.items(): + if len(dcnnl_data['destinations']) == 0: + monitor_chip_layers.append(dcnnl_data['core_idx']) + break + elif monitor_layers == "all": + for _, dcnnl_data in self.dynapcnn_layers.items(): # monitor each chip core (if not a DVSLayer). + if not isinstance(dcnnl_data['layer'], DVSLayer): + monitor_chip_layers.append(dcnnl_data['core_idx']) + + if monitor_layers: + if "dvs" in monitor_layers: + monitor_chip_layers.append("dvs") + + config_builder.monitor_layers(config, monitor_chip_layers) # enable monitors on the specified layers. - chip_layers_ordering = chip_layers_ordering[: len(self.sequence)] + if config_modifier is not None: # apply user config modifier. + config = config_modifier(config) + + return config, config_builder.validate_configuration(config) # validate config. def get_sinabs_edges(self, sinabs_model): """ Converts the computational graph extracted from 'sinabs_model.analog_model' into its equivalent @@ -255,4 +277,8 @@ def was_spiking_output_added(sinabs_model): # throw error return False else: - return False \ No newline at end of file + return False + + @staticmethod + def build_from_graph_(): # @TODO used for debug only (remove when class is complete). + return build_from_graph \ No newline at end of file diff --git a/sinabs/backend/dynapcnn/exceptions.py b/sinabs/backend/dynapcnn/exceptions.py index 832467c3..71f33bdb 100644 --- a/sinabs/backend/dynapcnn/exceptions.py +++ b/sinabs/backend/dynapcnn/exceptions.py @@ -30,3 +30,9 @@ class WrongPoolingModule(Exception): def __init__(self, pooling_module,): super().__init__(f"The function 'utils.build_SumPool2d(mod)' expects 'mod = nn.AvgPool2d' but got 'mod = {pooling_module}'.") + +class InvalidModel(Exception): + model: type + + def __init__(self, model,): + super().__init__(f"'model' accepts either a DynapcnnNetwork or a DynapcnnNetworkGraph but {model} was given.") diff --git a/sinabs/backend/dynapcnn/mapping.py b/sinabs/backend/dynapcnn/mapping.py index 5fefaa0b..19f51d6f 100644 --- a/sinabs/backend/dynapcnn/mapping.py +++ b/sinabs/backend/dynapcnn/mapping.py @@ -1,11 +1,14 @@ from collections import deque from copy import deepcopy from dataclasses import dataclass -from typing import List, Optional, Tuple +from typing import List, Optional, Tuple, Union from .dvs_layer import DVSLayer from .dynapcnn_layer import DynapcnnLayer +import sinabs +from .exceptions import InvalidModel + @dataclass class LayerConstraints: @@ -44,33 +47,47 @@ def find_chip_layers( def get_valid_mapping( - model: "DynapcnnNetwork", constraints: List[LayerConstraints] + model: Union["DynapcnnNetwork", "DynapcnnNetworkGraph"], constraints: List[LayerConstraints] ) -> List[Tuple[int, int]]: """Given a model, find a valid layer ordering for its placement within the constraints provided. Parameters ---------- - model: - DynapcnnNetwork - constraints: - A list of all the layer's constraints + model: an instance of a DynapcnnNetwork or a DynapcnnNetworkGraph. + constraints: a list of all the layer's constraints. Returns + netmap: a list of tuples with (dynapcnnlayer index, core index). ------- """ layer_mapping = [] - for layer in model.sequence: - if isinstance(layer, DynapcnnLayer): - layer_mapping.append(find_chip_layers(layer, constraints)) + if type(model) == sinabs.backend.dynapcnn.dynapcnn_network.DynapcnnNetwork: + for layer in model.sequence: + if isinstance(layer, DynapcnnLayer): + layer_mapping.append(find_chip_layers(layer, constraints)) + + graph = make_flow_graph(layer_mapping, len(constraints)) + + new_graph = edmonds(graph, 0, len(graph) - 1) # use graph algorithm to find suitable cores for each DynapcnnLayer. + + netmap = recover_mapping(new_graph, layer_mapping) + + elif type(model) == sinabs.backend.dynapcnn.dynapcnn_network_graph.DynapcnnNetworkGraph: + for _, layer_data in model.dynapcnn_layers.items(): + if isinstance(layer_data['layer'], DynapcnnLayer): + layer_mapping.append(find_chip_layers(layer_data['layer'], constraints)) + + graph = make_flow_graph(layer_mapping, len(constraints)) + + new_graph = edmonds(graph, 0, len(graph) - 1) # use graph algorithm to find suitable cores for each DynapcnnLayer. - graph = make_flow_graph(layer_mapping, len(constraints)) + netmap = recover_mapping(new_graph, layer_mapping) - # Call mapping - new_graph = edmonds(graph, 0, len(graph) - 1) + else: + raise InvalidModel(model) - netmap = recover_mapping(new_graph, layer_mapping) return netmap diff --git a/sinabs/backend/dynapcnn/utils.py b/sinabs/backend/dynapcnn/utils.py index 07dfdb9b..a4314127 100644 --- a/sinabs/backend/dynapcnn/utils.py +++ b/sinabs/backend/dynapcnn/utils.py @@ -579,13 +579,13 @@ def build_from_graph( dvs_layer = None rescale_factor = 1 - nodes_to_dcnnl_map = {} # mapper from nodes to sets of layers that populate a DynapcnnLayer. + nodes_to_dcnnl_map = {} # mapper from nodes to sets of layers that populate a DynapcnnLayer. if dvs_layer is not None: - pass # @TODO the graph extraction is not yet considering DVS input. + pass # @TODO the graph extraction is not yet considering DVS input. else: for edge in edges: - process_edge( # figure out to which (future) DynapcnnLayer each node will belong to. + process_edge( # figure out to which (future) DynapcnnLayer each node will belong to. layers, edge, nodes_to_dcnnl_map) # look for edges between connecting nodes in different (future) DynapcnnLayer. @@ -595,6 +595,10 @@ def build_from_graph( dynapcnn_layers = construct_dynapcnnlayers_from_mapper( discretize, nodes_to_dcnnl_map, dcnnl_to_dcnnl_map, in_shape, rescale_factor) + for idx, layer_data in dynapcnn_layers.items(): + if 'core_idx' not in layer_data: + layer_data['core_idx'] = -1 # a DynapcnnLayer gets assigned a core index when 'DynapcnnNetworkGraph.to()' is called. + return dynapcnn_layers, nodes_to_dcnnl_map, dcnnl_to_dcnnl_map def construct_dynapcnnlayers_from_mapper( From 599810ee6730b02dca1d4658707ea863d7d6d741 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Tue, 16 Apr 2024 11:12:06 +0200 Subject: [PATCH 012/379] from model's DynapcnnLayers to CNNLayerConfig config. sucessfull --- sinabs/backend/dynapcnn/chips/dynapcnn.py | 20 +- sinabs/backend/dynapcnn/config_builder.py | 1 + .../jit_based_tracer_sinabs.ipynb | 264 +++++++++++------- 3 files changed, 169 insertions(+), 116 deletions(-) diff --git a/sinabs/backend/dynapcnn/chips/dynapcnn.py b/sinabs/backend/dynapcnn/chips/dynapcnn.py index 14209e45..5f47076e 100644 --- a/sinabs/backend/dynapcnn/chips/dynapcnn.py +++ b/sinabs/backend/dynapcnn/chips/dynapcnn.py @@ -188,6 +188,7 @@ def write_dynapcnn_layer_config( configuration is written. """ config_dict = cls.get_dynapcnn_layer_config_dict(layer=layer) + # Update configuration of the DYNAPCNN layer chip_layer.dimensions = config_dict["dimensions"] config_dict.pop("dimensions") @@ -205,12 +206,10 @@ def write_dynapcnn_layer_config( raise TypeError(f"Unexpected parameter {param} or value. {e}") @classmethod - def write_dynapcnn_layer_config_graph( - cls, dcnnl_data: dict, chip_layer: "CNNLayerConfig" - ): + def write_dynapcnn_layer_config_graph(cls, dcnnl_data: dict, chip_layer: "CNNLayerConfig", dynapcnn_layers: dict): """ .""" - config_dict = cls.get_dynapcnn_layer_config_dict( # extracting from a DynapcnnLayer the config. variables for its CNNLayerConfig. + config_dict = cls.get_dynapcnn_layer_config_dict( # extracting from a DynapcnnLayer the config. variables for its CNNLayerConfig. layer=dcnnl_data['layer']) chip_layer.dimensions = config_dict["dimensions"] @@ -218,17 +217,19 @@ def write_dynapcnn_layer_config_graph( pooling = None if "pooling" in config_dict["destinations"][0]: - pooling = config_dict["destinations"][0]["pooling"] # TODO make pooling be destination-dependent. + pooling = config_dict["destinations"][0]["pooling"] # TODO make pooling be destination-dependent. config_dict.pop("destinations") - for dest_idx in range(len(dcnnl_data['destinations'])): # configuring the destinations for this DynapcnnLayer. + for dest_idx in range(len(dcnnl_data['destinations'])): # configuring the destinations for this DynapcnnLayer. chip_layer.destinations[dest_idx].enable = True - chip_layer.destinations[dest_idx].layer = dcnnl_data['destinations'][dest_idx] + + destination_core_idx = dynapcnn_layers[dcnnl_data['destinations'][dest_idx]]['core_idx'] # retrive the core to wich a destination DynapcnnLayer has been assigned to. + chip_layer.destinations[dest_idx].layer = destination_core_idx if isinstance(pooling, int): chip_layer.destinations[dest_idx].pooling = pooling - if len(dcnnl_data['destinations']) == 0: # this is the output layer. + if len(dcnnl_data['destinations']) == 0: # this is the output layer. chip_layer.destinations[0].enable = False chip_layer.destinations[1].enable = False @@ -247,6 +248,7 @@ def build_config(cls, model: Union["DynapcnnNetwork", "DynapcnnNetworkGraph"], c has_dvs_layer = False i_cnn_layer = 0 # Instantiate an iterator for the cnn cores + _prev_idx = 0 for i, chip_equivalent_layer in enumerate(layers): if isinstance(chip_equivalent_layer, DVSLayer): chip_layer = config.dvs_layer @@ -282,7 +284,7 @@ def build_config(cls, model: Union["DynapcnnNetwork", "DynapcnnNetworkGraph"], c elif isinstance(layer_data['layer'], DynapcnnLayer): chip_layer = config.cnn_layers[layer_data['core_idx']] - cls.write_dynapcnn_layer_config_graph(layer_data, chip_layer) + cls.write_dynapcnn_layer_config_graph(layer_data, chip_layer, model.dynapcnn_layers) else: raise TypeError("Unexpected layer in the model.") # shouldn't happen since type checks are made previously. diff --git a/sinabs/backend/dynapcnn/config_builder.py b/sinabs/backend/dynapcnn/config_builder.py index 17bf86c3..a42d1994 100644 --- a/sinabs/backend/dynapcnn/config_builder.py +++ b/sinabs/backend/dynapcnn/config_builder.py @@ -100,6 +100,7 @@ def get_valid_mapping(cls, model: "DynapcnnNetwork") -> List[int]: for (dcnnl, core_idx) in mapping: model.dynapcnn_layers[dcnnl]['core_idx'] = core_idx + else: raise InvalidModel(model) diff --git a/tests/test_nonsequential/jit_based_tracer_sinabs.ipynb b/tests/test_nonsequential/jit_based_tracer_sinabs.ipynb index 49e3f409..3b38715c 100644 --- a/tests/test_nonsequential/jit_based_tracer_sinabs.ipynb +++ b/tests/test_nonsequential/jit_based_tracer_sinabs.ipynb @@ -23,7 +23,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 2, @@ -216,49 +216,7 @@ "cell_type": "code", "execution_count": 9, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "\n", - "DynapcnnLayer(\n", - " (conv_layer): Conv2d(1, 20, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", - " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", - " tensor(635.), min_v_mem=Parameter containing:\n", - " tensor(-635.), batch_size=1, num_timesteps=-1)\n", - " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", - ")\n", - "DynapcnnLayer(\n", - " (conv_layer): Conv2d(20, 32, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", - " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", - " tensor(11361.), min_v_mem=Parameter containing:\n", - " tensor(-11361.), batch_size=1, num_timesteps=-1)\n", - " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", - ")\n", - "DynapcnnLayer(\n", - " (conv_layer): Conv2d(32, 128, kernel_size=(3, 3), stride=(1, 1), bias=False)\n", - " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", - " tensor(8621.), min_v_mem=Parameter containing:\n", - " tensor(-8621.), batch_size=1, num_timesteps=-1)\n", - " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", - ")\n", - "DynapcnnLayer(\n", - " (conv_layer): Conv2d(128, 500, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", - " tensor(5747.), min_v_mem=Parameter containing:\n", - " tensor(-5747.), batch_size=1, num_timesteps=-1)\n", - ")\n", - "DynapcnnLayer(\n", - " (conv_layer): Conv2d(500, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", - " tensor(2841.), min_v_mem=Parameter containing:\n", - " tensor(-2841.), batch_size=1, num_timesteps=-1)\n", - ")\n" - ] - } - ], + "outputs": [], "source": [ "hw_model_old = DynapcnnNetwork(\n", " sinabs_model.spiking_model,\n", @@ -271,78 +229,76 @@ "cell_type": "code", "execution_count": 10, "metadata": {}, - "outputs": [], - "source": [ - "hw_model = DynapcnnNetworkGraph(\n", - " sinabs_model,\n", - " discretize=True,\n", - " input_shape=input_shape\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "\n", - "layer index: 0\n", - "layer modules: DynapcnnLayer(\n", - " (conv_layer): Conv2d(1, 20, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", - " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", - " tensor(635.), min_v_mem=Parameter containing:\n", - " tensor(-635.), batch_size=1, num_timesteps=-1)\n", - " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", - ")\n", - "layer destinations: [1]\n", - "\n", - "layer index: 1\n", - "layer modules: DynapcnnLayer(\n", - " (conv_layer): Conv2d(20, 32, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", - " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", - " tensor(11361.), min_v_mem=Parameter containing:\n", - " tensor(-11361.), batch_size=1, num_timesteps=-1)\n", - " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", - ")\n", - "layer destinations: [2]\n", - "\n", - "layer index: 2\n", - "layer modules: DynapcnnLayer(\n", - " (conv_layer): Conv2d(32, 128, kernel_size=(3, 3), stride=(1, 1), bias=False)\n", - " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", - " tensor(8621.), min_v_mem=Parameter containing:\n", - " tensor(-8621.), batch_size=1, num_timesteps=-1)\n", - " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", - ")\n", - "layer destinations: [3]\n", - "\n", - "layer index: 3\n", - "layer modules: DynapcnnLayer(\n", - " (conv_layer): Conv2d(128, 500, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", - " tensor(5747.), min_v_mem=Parameter containing:\n", - " tensor(-5747.), batch_size=1, num_timesteps=-1)\n", - ")\n", - "layer destinations: [4]\n", - "\n", - "layer index: 4\n", - "layer modules: DynapcnnLayer(\n", - " (conv_layer): Conv2d(500, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", - " tensor(2841.), min_v_mem=Parameter containing:\n", - " tensor(-2841.), batch_size=1, num_timesteps=-1)\n", - ")\n", - "layer destinations: []\n", - "\n" + "Network is valid\n" ] + }, + { + "data": { + "text/plain": [ + "DynapcnnNetwork(\n", + " (sequence): Sequential(\n", + " (0): DynapcnnLayer(\n", + " (conv_layer): Conv2d(1, 20, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", + " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(635.), min_v_mem=Parameter containing:\n", + " tensor(-635.), batch_size=1, num_timesteps=-1)\n", + " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", + " )\n", + " (1): DynapcnnLayer(\n", + " (conv_layer): Conv2d(20, 32, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", + " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(11361.), min_v_mem=Parameter containing:\n", + " tensor(-11361.), batch_size=1, num_timesteps=-1)\n", + " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", + " )\n", + " (2): DynapcnnLayer(\n", + " (conv_layer): Conv2d(32, 128, kernel_size=(3, 3), stride=(1, 1), bias=False)\n", + " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(8621.), min_v_mem=Parameter containing:\n", + " tensor(-8621.), batch_size=1, num_timesteps=-1)\n", + " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", + " )\n", + " (3): DynapcnnLayer(\n", + " (conv_layer): Conv2d(128, 500, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(5747.), min_v_mem=Parameter containing:\n", + " tensor(-5747.), batch_size=1, num_timesteps=-1)\n", + " )\n", + " (4): DynapcnnLayer(\n", + " (conv_layer): Conv2d(500, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(2841.), min_v_mem=Parameter containing:\n", + " tensor(-2841.), batch_size=1, num_timesteps=-1)\n", + " )\n", + " )\n", + ")" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ - "print(hw_model)" + "hw_model_old.to(device=\"speck2edevkit:0\")" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "hw_model = DynapcnnNetworkGraph(\n", + " sinabs_model,\n", + " discretize=True,\n", + " input_shape=input_shape\n", + ")" ] }, { @@ -413,6 +369,100 @@ "for edge in hw_model.sinabs_edges:\n", " print(edge)" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Deploying Model" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Network is valid\n" + ] + } + ], + "source": [ + "hw_model.to(device=\"speck2edevkit:0\")" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "layer index: 0\n", + "layer modules: DynapcnnLayer(\n", + " (conv_layer): Conv2d(1, 20, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", + " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(635.), min_v_mem=Parameter containing:\n", + " tensor(-635.), batch_size=1, num_timesteps=-1)\n", + " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", + ")\n", + "layer destinations: [1]\n", + "assigned core: 0\n", + "\n", + "layer index: 1\n", + "layer modules: DynapcnnLayer(\n", + " (conv_layer): Conv2d(20, 32, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", + " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(11361.), min_v_mem=Parameter containing:\n", + " tensor(-11361.), batch_size=1, num_timesteps=-1)\n", + " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", + ")\n", + "layer destinations: [2]\n", + "assigned core: 3\n", + "\n", + "layer index: 2\n", + "layer modules: DynapcnnLayer(\n", + " (conv_layer): Conv2d(32, 128, kernel_size=(3, 3), stride=(1, 1), bias=False)\n", + " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(8621.), min_v_mem=Parameter containing:\n", + " tensor(-8621.), batch_size=1, num_timesteps=-1)\n", + " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", + ")\n", + "layer destinations: [3]\n", + "assigned core: 5\n", + "\n", + "layer index: 3\n", + "layer modules: DynapcnnLayer(\n", + " (conv_layer): Conv2d(128, 500, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(5747.), min_v_mem=Parameter containing:\n", + " tensor(-5747.), batch_size=1, num_timesteps=-1)\n", + ")\n", + "layer destinations: [4]\n", + "assigned core: 6\n", + "\n", + "layer index: 4\n", + "layer modules: DynapcnnLayer(\n", + " (conv_layer): Conv2d(500, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(2841.), min_v_mem=Parameter containing:\n", + " tensor(-2841.), batch_size=1, num_timesteps=-1)\n", + ")\n", + "layer destinations: []\n", + "assigned core: 1\n", + "\n" + ] + } + ], + "source": [ + "print(hw_model)" + ] } ], "metadata": { From e1991d95fbf7f2aada74f316b04a5e5fa90ab9cf Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Tue, 16 Apr 2024 11:22:25 +0200 Subject: [PATCH 013/379] .to() method ready --- .../dynapcnn/dynapcnn_network_graph.py | 46 ++++++++++++++++++- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py index b1503373..4e198fab 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py @@ -114,15 +114,57 @@ def to( elif isinstance(device, str): device_name, _ = parse_device_id(device) - if device_name in ChipFactory.supported_devices: # pragma: no cover + if device_name in ChipFactory.supported_devices: # pragma: no cover - config = self.make_config( # generate config. + config = self.make_config( # generate config. chip_layers_ordering=chip_layers_ordering, device=device, monitor_layers=monitor_layers, config_modifier=config_modifier, ) + self.samna_device = open_device(device) # apply configuration to device. + self.samna_device.get_model().apply_configuration(config) + time.sleep(1) + + if slow_clk_frequency is not None: # set external slow-clock if needed. + dk_io = self.samna_device.get_io_module() + dk_io.set_slow_clk(True) + dk_io.set_slow_clk_rate(slow_clk_frequency) # Hz + + builder = ChipFactory(device).get_config_builder() + + self.samna_input_buffer = builder.get_input_buffer() # create input source node. + self.samna_output_buffer = builder.get_output_buffer() # create output sink node node. + + self.device_input_graph = samna.graph.EventFilterGraph() # connect source node to device sink. + self.device_input_graph.sequential( + [ + self.samna_input_buffer, + self.samna_device.get_model().get_sink_node(), + ] + ) + + self.device_output_graph = samna.graph.EventFilterGraph() # connect sink node to device. + self.device_output_graph.sequential( + [ + self.samna_device.get_model().get_source_node(), + self.samna_output_buffer, + ] + ) + + self.device_input_graph.start() + self.device_output_graph.start() + self.samna_config = config + + return self + + else: + return super().to(device) + + else: + raise Exception("Unknown device description.") + def make_config( self, chip_layers_ordering: Union[Sequence[int], str] = "auto", From 443821cbda633b4368169907d19312f5688146f5 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Tue, 16 Apr 2024 13:24:48 +0200 Subject: [PATCH 014/379] methods/functions documentation --- sinabs/backend/dynapcnn/dynapcnn_network.py | 1 + .../dynapcnn/dynapcnn_network_graph.py | 89 +++++++++++++++++-- sinabs/backend/dynapcnn/graph_tracer.py | 2 +- 3 files changed, 83 insertions(+), 9 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index 4a1bb525..9d59236f 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -276,6 +276,7 @@ def _make_config( ] if "dvs" in monitor_layers: monitor_chip_layers.append("dvs") + config_builder.monitor_layers(config, monitor_chip_layers) # Fix default factory setting to not return input events (UGLY!! Ideally this should happen in samna) diff --git a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py index 4e198fab..7bcc7e3c 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py @@ -105,7 +105,41 @@ def to( config_modifier=None, slow_clk_frequency: int = None, ): - """ .""" + """Note that the model parameters are only ever transferred to the device on the `to` call, + so changing a threshold or weight of a model that is deployed will have no effect on the + model on chip until `to` is called again. + + Parameters + ---------- + + device: String + cpu:0, cuda:0, dynapcnndevkit, speck2devkit + + chip_layers_ordering: sequence of integers or `auto` + The order in which the dynapcnn layers will be used. If `auto`, + an automated procedure will be used to find a valid ordering. + A list of layers on the device where you want each of the model's DynapcnnLayers to be placed. + The index of the core on chip to which the i-th layer in the model is mapped is the value of the i-th entry in the list. + Note: This list should be the same length as the number of dynapcnn layers in your model. + + monitor_layers: None/List + A list of all layers in the module that you want to monitor. Indexing starts with the first non-dvs layer. + If you want to monitor the dvs layer for eg. + :: + + monitor_layers = ["dvs"] # If you want to monitor the output of the pre-processing layer + monitor_layers = ["dvs", 8] # If you want to monitor preprocessing and layer 8 + monitor_layers = "all" # If you want to monitor all the layers + + config_modifier: + A user configuration modifier method. + This function can be used to make any custom changes you want to make to the configuration object. + + Note + ---- + chip_layers_ordering and monitor_layers are used only when using synsense devices. + For GPU or CPU usage these options are ignored. + """ self.device = device if isinstance(device, torch.device): @@ -233,8 +267,48 @@ def _make_config( monitor_layers: Optional[Union[List, str]] = None, config_modifier=None, ) -> Tuple["SamnaConfiguration", bool]: - """ Prepare and output the `samna` configuration for this network. """ + """Prepare and output the `samna` configuration for this network. + + Parameters + ---------- + + chip_layers_ordering: sequence of integers or `auto` + The order in which the dynapcnn layers will be used. If `auto`, + an automated procedure will be used to find a valid ordering. + A list of layers on the device where you want each of the model's DynapcnnLayers to be placed. + The index of the core on chip to which the i-th layer in the model is mapped is the value of the i-th entry in the list. + Note: This list should be the same length as the number of dynapcnn layers in your model. + + device: String + dynapcnndevkit, speck2b or speck2devkit + + monitor_layers: None/List/Str + A list of all layers in the module that you want to monitor. Indexing starts with the first non-dvs layer. + If you want to monitor the dvs layer for eg. + :: + + monitor_layers = ["dvs"] # If you want to monitor the output of the pre-processing layer + monitor_layers = ["dvs", 8] # If you want to monitor preprocessing and layer 8 + monitor_layers = "all" # If you want to monitor all the layers + If this value is left as None, by default the last layer of the model is monitored. + + config_modifier: + A user configuration modifier method. + This function can be used to make any custom changes you want to make to the configuration object. + + Returns + ------- + Configuration object + Object defining the configuration for the device + Bool + True if the configuration is valid for the given device. + + Raises + ------ + ImportError + If samna is not available. + """ config_builder = ChipFactory(device).get_config_builder() has_dvs_layer = isinstance(self.dynapcnn_layers[0]['layer'], DVSLayer) @@ -246,7 +320,7 @@ def _make_config( if has_dvs_layer: pass # TODO not handling DVSLayer yet. - config = config_builder.build_config(self, []) # update config. + config = config_builder.build_config(self, None) # update config. if self.input_shape and self.input_shape[0] == 1: # ??? config.dvs_layer.merge = True @@ -273,25 +347,24 @@ def _make_config( return config, config_builder.validate_configuration(config) # validate config. - def get_sinabs_edges(self, sinabs_model): + def get_sinabs_edges(self, sinabs_model: sinabs.network.Network) -> List[Tuple[int, int]]: """ Converts the computational graph extracted from 'sinabs_model.analog_model' into its equivalent representation for the 'sinabs_model.spiking_model'. Parameters ---------- - sinabs_model: ... + sinabs_model: a sinabs network object created from a PyTorch model. Returns - sinabs_edges: ... + sinabs_edges: a list of tuples representing the edges between the layers of a sinabs model. ---------- """ # parse original graph to ammend edges containing nodes dropped in 'convert_model_to_layer_list()'. sinabs_edges = self.graph_tracer.remove_ignored_nodes(DEFAULT_IGNORED_LAYER_TYPES) if DynapcnnNetworkGraph.was_spiking_output_added(sinabs_model): - # spiking output layer has been added: create new edge. last_edge = sinabs_edges[-1] - new_edge = (last_edge[1], last_edge[1]+1) + new_edge = (last_edge[1], last_edge[1]+1) # spiking output layer has been added: create new edge. sinabs_edges.append(new_edge) else: pass diff --git a/sinabs/backend/dynapcnn/graph_tracer.py b/sinabs/backend/dynapcnn/graph_tracer.py index a454b086..ddc8fe01 100644 --- a/sinabs/backend/dynapcnn/graph_tracer.py +++ b/sinabs/backend/dynapcnn/graph_tracer.py @@ -115,7 +115,7 @@ def get_ATen_operations(self): return ATens def remove_ignored_nodes(self, default_ignored_nodes): - """ Recreates the edges list based on layers that 'DynapcnnNetwork' will ignored. This + """ Recreates the edges list based on layers that 'DynapcnnNetwork' will ignore. This is done by setting the source (target) node of an edge where the source (target) node will be dropped as the node that originally targeted this node to be dropped. """ From 4148973f7f21f1a490ee03b2a68b1004545f6af6 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Tue, 16 Apr 2024 15:34:38 +0200 Subject: [PATCH 015/379] 'from_model()' modified to work with both nn.Sequential and class inheriting from nn.Module --- sinabs/from_torch.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/sinabs/from_torch.py b/sinabs/from_torch.py index beebf96b..42b5ae4e 100644 --- a/sinabs/from_torch.py +++ b/sinabs/from_torch.py @@ -101,10 +101,28 @@ def mapper_fn(module): **kwargs_backend, ).to(device), ) + + elif isinstance(model, nn.Module): + layers = [layer for _, layer in model.named_children()] + + if not isinstance(layers[-1], (nn.ReLU, sl.NeuromorphicReLU)): + snn.add_module( + "spike_output", + spike_layer_class( + spike_threshold=spike_threshold, + spike_fn=spike_fn, + reset_fn=reset_fn, + surrogate_grad_fn=surrogate_grad_fn, + min_v_mem=min_v_mem, + **kwargs_backend, + ).to(device), + ) + + else: + warn("Spiking output can only be added to sequential models that do not end in a ReLU. No layer has been added.") + else: - warn( - "Spiking output can only be added to sequential models that do not end in a ReLU. No layer has been added." - ) + warn("Spiking output can only be added to sequential models that do not end in a ReLU. No layer has been added.") for module in snn.modules(): if bias_rescaling != 1.0 and isinstance(module, (nn.Linear, nn.Conv2d)): From 16790a0cb0afe36d885eb0cc3a139d286d2fa5af Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Tue, 16 Apr 2024 15:37:09 +0200 Subject: [PATCH 016/379] 'convert_model_to_layer()' modified to work with both nn.Sequential and class inheriting from nn.Module --- sinabs/backend/dynapcnn/utils.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/sinabs/backend/dynapcnn/utils.py b/sinabs/backend/dynapcnn/utils.py index a4314127..301de6de 100644 --- a/sinabs/backend/dynapcnn/utils.py +++ b/sinabs/backend/dynapcnn/utils.py @@ -427,26 +427,32 @@ def build_from_list( def convert_model_to_layer_list( - model: Union[nn.Sequential, sinabs.Network], + model: Union[nn.Sequential, sinabs.Network, nn.Module], ignore: Union[Type, Tuple[Type, ...]] = (), ) -> List[nn.Module]: """Convert a model to a list of layers. Parameters ---------- - model: nn.Sequential or sinabs.Network - ignore: type or tuple of types of modules to be ignored + model: nn.Sequential, nn.Module or sinabs.Network. + ignore: type or tuple of types of modules to be ignored. Returns ------- - List[nn.Module] + List[nn.Module] """ if isinstance(model, sinabs.Network): return convert_model_to_layer_list(model.spiking_model) + elif isinstance(model, nn.Sequential): layers = [layer for layer in model if not isinstance(layer, ignore)] + + elif isinstance(model, nn.Module): + layers = [layer for _, layer in model.named_children() if not isinstance(layer, ignore)] + else: raise TypeError("Expected torch.nn.Sequential or sinabs.Network") + return layers From 20035477c02023888124155f846f105a116c066d Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Tue, 16 Apr 2024 15:40:06 +0200 Subject: [PATCH 017/379] accepting 'snn.analog/spiking_model' as nn.Sequential or class inheriting from nn.Module --- .../dynapcnn/dynapcnn_network_graph.py | 47 +++++++++++++++---- sinabs/backend/dynapcnn/exceptions.py | 6 +++ 2 files changed, 43 insertions(+), 10 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py index 7bcc7e3c..da7d8ecc 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py @@ -27,6 +27,8 @@ ) from .graph_tracer import GraphTracer +from .exceptions import InvalidTorchModel +from warnings import warn class DynapcnnNetworkGraph(nn.Module): """Given a sinabs spiking network, prepare a dynapcnn-compatible network. This can be used to @@ -36,7 +38,7 @@ class DynapcnnNetworkGraph(nn.Module): def __init__( self, - snn: Union[nn.Sequential, sinabs.Network], + snn: Union[nn.Sequential, sinabs.Network, nn.Module], input_shape: Optional[Tuple[int, int, int]] = None, dvs_input: bool = False, discretize: bool = True @@ -61,15 +63,16 @@ def __init__( """ super().__init__() - dvs_input = False # @TODO for now the graph part is not taking into consideration this. + dvs_input = False # TODO for now the graph part is not taking into consideration this. self.graph_tracer = GraphTracer( # computational graph from original PyTorch module. snn.analog_model, torch.randn((1, *input_shape)) # torch.jit needs the batch dimension. ) - self.input_shape = input_shape # convert models to sequential. - self.layers = convert_model_to_layer_list( + self.input_shape = input_shape + + self.layers = convert_model_to_layer_list( # convert models to sequential. model=snn.spiking_model, ignore=DEFAULT_IGNORED_LAYER_TYPES ) @@ -372,25 +375,49 @@ def get_sinabs_edges(self, sinabs_model: sinabs.network.Network) -> List[Tuple[i return sinabs_edges @staticmethod - def was_spiking_output_added(sinabs_model): + def was_spiking_output_added(sinabs_model: sinabs.Network) -> bool: """ Compares the models outputed by 'sinabs.from_torch.from_model()' to check if a spiking output was added to the spiking version of the analog model. + + Parameters + ---------- + sinabs_model: a sinabs network. `sinabs_model.analog_model`\`sinabs_model.spiking_model` need to be either a nn.Module or a nn.Sequential. + + Returns + ---------- + bool: wheter or not a neuron layers has been added to the `sinabs_model.spiking_model`. """ analog_modules = [] spiking_modules = [] - for mod in sinabs_model.analog_model: - analog_modules.append(mod) + if isinstance(sinabs_model.analog_model, nn.Sequential): + for mod in sinabs_model.analog_model: + analog_modules.append(mod) - for mod in sinabs_model.spiking_model: - spiking_modules.append(mod) + elif isinstance(sinabs_model.analog_model, nn.Module): + analog_modules = [layer for _, layer in sinabs_model.analog_model.named_children()] + + else: + raise InvalidTorchModel('sinabs_model.analog_model') + + if isinstance(sinabs_model.spiking_model, nn.Sequential): + for mod in sinabs_model.spiking_model: + spiking_modules.append(mod) + + elif isinstance(sinabs_model.spiking_model, nn.Module): + spiking_modules = [layer for _, layer in sinabs_model.spiking_model.named_children()] + + else: + raise InvalidTorchModel('sinabs_model.spiking_model') if len(analog_modules) != len(spiking_modules): if isinstance(spiking_modules[-1], sinabs.layers.iaf.IAFSqueeze): return True + else: - # throw error + warn(f'sinabs.spiking_model has a {type(spiking_modules[-1])} as last layer.') return False + else: return False diff --git a/sinabs/backend/dynapcnn/exceptions.py b/sinabs/backend/dynapcnn/exceptions.py index 71f33bdb..08aeb923 100644 --- a/sinabs/backend/dynapcnn/exceptions.py +++ b/sinabs/backend/dynapcnn/exceptions.py @@ -36,3 +36,9 @@ class InvalidModel(Exception): def __init__(self, model,): super().__init__(f"'model' accepts either a DynapcnnNetwork or a DynapcnnNetworkGraph but {model} was given.") + +class InvalidTorchModel(Exception): + network_type: str + + def __init__(self, network_type): + super().__init__(f"A {network_type} needs to be either of type nn.Sequential or nn.Module.") From 6ebff882df81504acf69a8a7e5d8d0431f131776 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Tue, 16 Apr 2024 16:57:48 +0200 Subject: [PATCH 018/379] (WIP) tester scripts --- .../graph_tracer_tester.ipynb | 245 +++++----- .../jit_based_tracer_sinabs.ipynb | 90 +++- ...est_DynapcnnNetworkGraph_1_skip_conn.ipynb | 435 ++++++++++++++++++ 3 files changed, 637 insertions(+), 133 deletions(-) create mode 100644 tests/test_nonsequential/test_DynapcnnNetworkGraph_1_skip_conn.ipynb diff --git a/tests/test_nonsequential/graph_tracer_tester.ipynb b/tests/test_nonsequential/graph_tracer_tester.ipynb index 93005a5a..c3259683 100644 --- a/tests/test_nonsequential/graph_tracer_tester.ipynb +++ b/tests/test_nonsequential/graph_tracer_tester.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 37, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -10,21 +10,23 @@ "import torch.nn as nn\n", "from typing import Union\n", "import re, copy\n", - "import numpy as np" + "import numpy as np\n", + "import networkx as nx\n", + "import matplotlib.pyplot as plt" ] }, { "cell_type": "code", - "execution_count": 38, + "execution_count": 2, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 38, + "execution_count": 2, "metadata": {}, "output_type": "execute_result" } @@ -35,7 +37,7 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -55,7 +57,7 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -167,23 +169,27 @@ " return ATens\n", " \n", " def remove_ignored_nodes(self, default_ignored_nodes):\n", - " \"\"\" Recreates the edges list based on layers that 'DynapcnnNetwork' will ignored. This\n", + " \"\"\" Recreates the edges list based on layers that 'DynapcnnNetwork' will ignore. This\n", " is done by setting the source (target) node of an edge where the source (target) node\n", " will be dropped as the node that originally targeted this node to be dropped.\n", " \"\"\"\n", - " edges = copy.deepcopy(self.edges_list[1:-1])\n", - " new_edges = []\n", + " edges = copy.deepcopy(self.edges_list)\n", + " parsed_edges = []\n", + " removed_nodes = []\n", "\n", + " # removing ignored nodes from edges.\n", " for edge_idx in range(len(edges)):\n", " _src = edges[edge_idx][0]\n", " _trg = edges[edge_idx][1]\n", "\n", " if isinstance(self.modules_map[_src], default_ignored_nodes):\n", + " removed_nodes.append(_src)\n", " # all edges where node '_src' is target change it to node '_trg' as their target.\n", " for edge in edges:\n", " if edge[1] == _src:\n", " new_edge = (edge[0], _trg)\n", " elif isinstance(self.modules_map[_trg], default_ignored_nodes):\n", + " removed_nodes.append(_trg)\n", " # all edges where node '_trg' is source change it to node '_src' as their source.\n", " for edge in edges:\n", " if edge[0] == _trg:\n", @@ -191,120 +197,36 @@ " else:\n", " new_edge = (_src, _trg)\n", " \n", - " if new_edge not in new_edges:\n", - " new_edges.append(new_edge)\n", + " if new_edge not in parsed_edges:\n", + " parsed_edges.append(new_edge)\n", "\n", - " return new_edges\n", + " removed_nodes = list(set(removed_nodes))\n", + "\n", + " # remapping nodes indexes.\n", + " remapped_nodes = {}\n", + " for node_indx, __ in self.modules_map.items():\n", + " _ = [x for x in removed_nodes if node_indx > x]\n", + " remapped_nodes[node_indx] = node_indx - len(_)\n", + " \n", + " for x in removed_nodes:\n", + " del remapped_nodes[x]\n", + "\n", + " # remapping nodes names in parsed edges.\n", + " remapped_edges = []\n", + " for edge in parsed_edges:\n", + " remapped_edges.append((remapped_nodes[edge[0]], remapped_nodes[edge[1]]))\n", + "\n", + " return remapped_edges\n", " \n", - " def plot_graph(self):\n", + " @staticmethod\n", + " def plot_graph(edges_list):\n", " \"\"\" .\"\"\"\n", - " G = nx.DiGraph(self.edges_list)\n", + " G = nx.DiGraph(edges_list)\n", " layout = nx.spring_layout(G)\n", " nx.draw(G, pos = layout, with_labels=True, node_size=800)\n", - " plt.title('GraphTracer (new)')\n", " plt.show()" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Tracing 1" - ] - }, - { - "cell_type": "code", - "execution_count": 41, - "metadata": {}, - "outputs": [], - "source": [ - "ann1 = nn.Sequential(\n", - " nn.Conv2d(1, 20, 5, 1, bias=False),\n", - " nn.ReLU(),\n", - " nn.AvgPool2d(2,2),\n", - " nn.Conv2d(20, 32, 5, 1, bias=False),\n", - " nn.ReLU(),\n", - " nn.AvgPool2d(2,2),\n", - " nn.Conv2d(32, 128, 3, 1, bias=False),\n", - " nn.ReLU(),\n", - " nn.AvgPool2d(2,2),\n", - " nn.Flatten(),\n", - " nn.Linear(128, 500, bias=False),\n", - " nn.ReLU(),\n", - " nn.Linear(500, 10, bias=False),\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 42, - "metadata": {}, - "outputs": [], - "source": [ - "gtracer1 = GraphTracer(ann1, input_dummy)" - ] - }, - { - "cell_type": "code", - "execution_count": 43, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "0 Conv2d(1, 20, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", - "1 ReLU()\n", - "2 AvgPool2d(kernel_size=2, stride=2, padding=0)\n", - "3 Conv2d(20, 32, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", - "4 ReLU()\n", - "5 AvgPool2d(kernel_size=2, stride=2, padding=0)\n", - "6 Conv2d(32, 128, kernel_size=(3, 3), stride=(1, 1), bias=False)\n", - "7 ReLU()\n", - "8 AvgPool2d(kernel_size=2, stride=2, padding=0)\n", - "9 Flatten(start_dim=1, end_dim=-1)\n", - "10 Linear(in_features=128, out_features=500, bias=False)\n", - "11 ReLU()\n", - "12 Linear(in_features=500, out_features=10, bias=False)\n" - ] - } - ], - "source": [ - "for name, mod in gtracer1.modules_map.items():\n", - " print(name, mod)" - ] - }, - { - "cell_type": "code", - "execution_count": 44, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(0, 1)\n", - "(1, 2)\n", - "(2, 3)\n", - "(3, 4)\n", - "(4, 5)\n", - "(5, 6)\n", - "(6, 7)\n", - "(7, 8)\n", - "(8, 9)\n", - "(9, 10)\n", - "(10, 11)\n", - "(11, 12)\n", - "12\n" - ] - } - ], - "source": [ - "for edge in gtracer1.edges_list:\n", - " print(edge)\n", - "print(len(gtracer1.edges_list))" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -314,7 +236,7 @@ }, { "cell_type": "code", - "execution_count": 45, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -325,29 +247,43 @@ " self.con1 = nn.Conv2d(1, 20, 5, 1, bias=False)\n", " self.rel1 = nn.ReLU()\n", " self.pool1 = nn.AvgPool2d(2,2)\n", + "\n", " self.conv2 = nn.Conv2d(20, 32, 5, 1, bias=False)\n", " self.rel2 = nn.ReLU()\n", " self.pool2 = nn.AvgPool2d(2,2)\n", + "\n", " self.conv3 = nn.Conv2d(32, 128, 3, 1, bias=False)\n", " self.rel3 = nn.ReLU()\n", " self.pool3 = nn.AvgPool2d(2,2)\n", + "\n", " self.flat = nn.Flatten()\n", + "\n", " self.fc1 = nn.Linear(128, 500, bias=False)\n", " self.rel4 = nn.ReLU()\n", " self.fc2 = nn.Linear(500, 10, bias=False)\n", "\n", + " self.residual_projection = nn.Conv2d(20, 32, 1, 6, bias=False) # from self.con1 to self.con3.\n", + " self.residual_projection.weight.requires_grad = False # no training of parameters.\n", + " self.residual_projection.weight.data.fill_(1) # compute the identity.\n", + "\n", " def forward(self, x):\n", " \n", " con1_out = self.con1(x)\n", " rel1_out = self.rel1(con1_out)\n", " pool1_out = self.pool1(rel1_out)\n", + "\n", + " residual = self.residual_projection(rel1_out)\n", + "\n", " conv2_out = self.conv2(pool1_out)\n", " rel2_out = self.rel2(conv2_out)\n", " pool2_out = self.pool2(rel2_out)\n", - " conv3_out = self.conv3(pool2_out)\n", + "\n", + " conv3_out = self.conv3(pool2_out + residual)\n", " rel3_out = self.rel3(conv3_out)\n", " pool3_out = self.pool3(rel3_out)\n", + "\n", " flat_out = self.flat(pool3_out)\n", + " \n", " fc1_out = self.fc1(flat_out)\n", " rel4_out = self.rel4(fc1_out)\n", " fc2_out = self.fc2(rel4_out)\n", @@ -359,7 +295,57 @@ }, { "cell_type": "code", - "execution_count": 46, + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "con1_out = ann2.rel1(ann2.con1(input_dummy))\n", + "pool1 = ann2.pool1(con1_out)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "torch.Size([1, 32, 4, 4])\n" + ] + } + ], + "source": [ + "\n", + "con2_out = ann2.rel2(ann2.conv2(pool1))\n", + "pool2 = ann2.pool2(con2_out)\n", + "\n", + "print(pool2.shape)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "torch.Size([1, 32, 4, 4])\n" + ] + } + ], + "source": [ + "residual = ann2.residual_projection(con1_out)\n", + "\n", + "print(residual.shape)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -368,7 +354,7 @@ }, { "cell_type": "code", - "execution_count": 47, + "execution_count": 10, "metadata": {}, "outputs": [ { @@ -387,7 +373,8 @@ "9 Flatten(start_dim=1, end_dim=-1)\n", "10 Linear(in_features=128, out_features=500, bias=False)\n", "11 ReLU()\n", - "12 Linear(in_features=500, out_features=10, bias=False)\n" + "12 Linear(in_features=500, out_features=10, bias=False)\n", + "13 Conv2d(20, 32, kernel_size=(1, 1), stride=(6, 6), bias=False)\n" ] } ], @@ -398,7 +385,7 @@ }, { "cell_type": "code", - "execution_count": 48, + "execution_count": 11, "metadata": {}, "outputs": [ { @@ -407,24 +394,26 @@ "text": [ "(0, 1)\n", "(1, 2)\n", - "(2, 3)\n", + "(None, 18)\n", + "(None, 18)\n", + "(None, 3)\n", + "(None, 3)\n", "(3, 4)\n", "(4, 5)\n", - "(5, 6)\n", + "(None, 6)\n", + "(None, 6)\n", "(6, 7)\n", "(7, 8)\n", "(8, 9)\n", "(9, 10)\n", "(10, 11)\n", - "(11, 12)\n", - "12\n" + "(11, 12)\n" ] } ], "source": [ "for edge in gtracer2.edges_list:\n", - " print(edge)\n", - "print(len(gtracer2.edges_list))" + " print(edge)" ] } ], diff --git a/tests/test_nonsequential/jit_based_tracer_sinabs.ipynb b/tests/test_nonsequential/jit_based_tracer_sinabs.ipynb index 3b38715c..b6c845e2 100644 --- a/tests/test_nonsequential/jit_based_tracer_sinabs.ipynb +++ b/tests/test_nonsequential/jit_based_tracer_sinabs.ipynb @@ -23,7 +23,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 2, @@ -214,15 +214,50 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 21, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "--------------------------------------------------------\n", + "0 Conv2d(1, 20, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", + "1 IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-1.), batch_size=1, num_timesteps=-1)\n", + "2 AvgPool2d(kernel_size=2, stride=2, padding=0)\n", + "3 Conv2d(20, 32, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", + "4 IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-1.), batch_size=1, num_timesteps=-1)\n", + "5 AvgPool2d(kernel_size=2, stride=2, padding=0)\n", + "6 Conv2d(32, 128, kernel_size=(3, 3), stride=(1, 1), bias=False)\n", + "7 IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-1.), batch_size=1, num_timesteps=-1)\n", + "8 AvgPool2d(kernel_size=2, stride=2, padding=0)\n", + "9 Linear(in_features=128, out_features=500, bias=False)\n", + "10 IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-1.), batch_size=1, num_timesteps=-1)\n", + "11 Linear(in_features=500, out_features=10, bias=False)\n", + "12 IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-1.), batch_size=1, num_timesteps=-1)\n", + "--------------------------------------------------------\n", + "True\n" + ] + } + ], "source": [ "hw_model_old = DynapcnnNetwork(\n", " sinabs_model.spiking_model,\n", " discretize=True,\n", " input_shape=input_shape\n", - ")" + ")\n", + "\n", + "print(isinstance(sinabs_model.spiking_model, nn.Sequential))" ] }, { @@ -292,7 +327,42 @@ "cell_type": "code", "execution_count": 11, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "--------------------------------------------------------\n", + "0 Conv2d(1, 20, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", + "1 IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-1.), batch_size=1, num_timesteps=-1)\n", + "2 AvgPool2d(kernel_size=2, stride=2, padding=0)\n", + "3 Conv2d(20, 32, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", + "4 IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-1.), batch_size=1, num_timesteps=-1)\n", + "5 AvgPool2d(kernel_size=2, stride=2, padding=0)\n", + "6 Conv2d(32, 128, kernel_size=(3, 3), stride=(1, 1), bias=False)\n", + "7 IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-1.), batch_size=1, num_timesteps=-1)\n", + "8 AvgPool2d(kernel_size=2, stride=2, padding=0)\n", + "9 Linear(in_features=128, out_features=500, bias=False)\n", + "10 IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-1.), batch_size=1, num_timesteps=-1)\n", + "11 Linear(in_features=500, out_features=10, bias=False)\n", + "12 IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-1.), batch_size=1, num_timesteps=-1)\n", + "--------------------------------------------------------\n", + "True\n", + "True\n", + " True\n" + ] + } + ], "source": [ "hw_model = DynapcnnNetworkGraph(\n", " sinabs_model,\n", @@ -388,6 +458,16 @@ "text": [ "Network is valid\n" ] + }, + { + "data": { + "text/plain": [ + "DynapcnnNetworkGraph()" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ diff --git a/tests/test_nonsequential/test_DynapcnnNetworkGraph_1_skip_conn.ipynb b/tests/test_nonsequential/test_DynapcnnNetworkGraph_1_skip_conn.ipynb new file mode 100644 index 00000000..8847f91f --- /dev/null +++ b/tests/test_nonsequential/test_DynapcnnNetworkGraph_1_skip_conn.ipynb @@ -0,0 +1,435 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "import torch.nn as nn\n", + "import networkx as nx\n", + "import matplotlib.pyplot as plt\n", + "from sinabs.from_torch import from_model\n", + "from sinabs.backend.dynapcnn import DynapcnnNetwork, DynapcnnNetworkGraph\n", + "import sinabs as snb" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "torch.manual_seed(0)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "channels = 1\n", + "height = 28\n", + "width = 28\n", + "\n", + "input_shape = (channels, height, width)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Network Module (pure Pytorch)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "class ResCNN(nn.Module):\n", + " def __init__(self):\n", + " super().__init__()\n", + "\n", + " self.con1 = nn.Conv2d(1, 20, 5, 1, bias=False)\n", + " self.rel1 = nn.ReLU()\n", + " self.pool1 = nn.AvgPool2d(2,2)\n", + "\n", + " self.conv2 = nn.Conv2d(20, 32, 5, 1, bias=False)\n", + " self.rel2 = nn.ReLU()\n", + " self.pool2 = nn.AvgPool2d(2,2)\n", + "\n", + " self.conv3 = nn.Conv2d(32, 128, 3, 1, bias=False)\n", + " self.rel3 = nn.ReLU()\n", + " self.pool3 = nn.AvgPool2d(2,2)\n", + "\n", + " self.flat = nn.Flatten()\n", + "\n", + " self.fc1 = nn.Linear(128, 500, bias=False)\n", + " self.rel4 = nn.ReLU()\n", + " self.fc2 = nn.Linear(500, 10, bias=False)\n", + "\n", + " def forward(self, x):\n", + " \n", + " con1_out = self.con1(x)\n", + " rel1_out = self.rel1(con1_out)\n", + " pool1_out = self.pool1(rel1_out)\n", + "\n", + " conv2_out = self.conv2(pool1_out)\n", + " rel2_out = self.rel2(conv2_out)\n", + " pool2_out = self.pool2(rel2_out)\n", + "\n", + " conv3_out = self.conv3(pool2_out + rel1_out)\n", + " rel3_out = self.rel3(conv3_out)\n", + " pool3_out = self.pool3(rel3_out)\n", + "\n", + " flat_out = self.flat(pool3_out)\n", + " \n", + " fc1_out = self.fc1(flat_out)\n", + " rel4_out = self.rel4(fc1_out)\n", + " fc2_out = self.fc2(rel4_out)\n", + "\n", + " return fc2_out\n", + "\n", + "rescnn = ResCNN()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Sinabs Model" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "sinabs_model = from_model(rescnn, add_spiking_output=True, batch_size=1)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "ResCNN(\n", + " (con1): Conv2d(1, 20, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", + " (rel1): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(1.), min_v_mem=Parameter containing:\n", + " tensor(-1.), batch_size=1, num_timesteps=-1)\n", + " (pool1): AvgPool2d(kernel_size=2, stride=2, padding=0)\n", + " (conv2): Conv2d(20, 32, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", + " (rel2): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(1.), min_v_mem=Parameter containing:\n", + " tensor(-1.), batch_size=1, num_timesteps=-1)\n", + " (pool2): AvgPool2d(kernel_size=2, stride=2, padding=0)\n", + " (conv3): Conv2d(32, 128, kernel_size=(3, 3), stride=(1, 1), bias=False)\n", + " (rel3): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(1.), min_v_mem=Parameter containing:\n", + " tensor(-1.), batch_size=1, num_timesteps=-1)\n", + " (pool3): AvgPool2d(kernel_size=2, stride=2, padding=0)\n", + " (flat): Flatten(start_dim=1, end_dim=-1)\n", + " (fc1): Linear(in_features=128, out_features=500, bias=False)\n", + " (rel4): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(1.), min_v_mem=Parameter containing:\n", + " tensor(-1.), batch_size=1, num_timesteps=-1)\n", + " (fc2): Linear(in_features=500, out_features=10, bias=False)\n", + " (spike_output): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(1.), min_v_mem=Parameter containing:\n", + " tensor(-1.), batch_size=1, num_timesteps=-1)\n", + ")\n" + ] + } + ], + "source": [ + "print(sinabs_model.spiking_model)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "ResCNN(\n", + " (con1): Conv2d(1, 20, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", + " (rel1): ReLU()\n", + " (pool1): AvgPool2d(kernel_size=2, stride=2, padding=0)\n", + " (conv2): Conv2d(20, 32, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", + " (rel2): ReLU()\n", + " (pool2): AvgPool2d(kernel_size=2, stride=2, padding=0)\n", + " (conv3): Conv2d(32, 128, kernel_size=(3, 3), stride=(1, 1), bias=False)\n", + " (rel3): ReLU()\n", + " (pool3): AvgPool2d(kernel_size=2, stride=2, padding=0)\n", + " (flat): Flatten(start_dim=1, end_dim=-1)\n", + " (fc1): Linear(in_features=128, out_features=500, bias=False)\n", + " (rel4): ReLU()\n", + " (fc2): Linear(in_features=500, out_features=10, bias=False)\n", + ")\n" + ] + } + ], + "source": [ + "print(sinabs_model.analog_model)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## DynapCNN Model" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "ename": "RuntimeError", + "evalue": "The size of tensor a (4) must match the size of tensor b (24) at non-singleton dimension 3", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[8], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m hw_model \u001b[38;5;241m=\u001b[39m DynapcnnNetworkGraph(\n\u001b[1;32m 2\u001b[0m sinabs_model,\n\u001b[1;32m 3\u001b[0m discretize\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m 4\u001b[0m input_shape\u001b[38;5;241m=\u001b[39minput_shape\n\u001b[1;32m 5\u001b[0m )\n", + "File \u001b[0;32m~/Documents/github/sinabs/sinabs/backend/dynapcnn/dynapcnn_network_graph.py:68\u001b[0m, in \u001b[0;36mDynapcnnNetworkGraph.__init__\u001b[0;34m(self, snn, input_shape, dvs_input, discretize)\u001b[0m\n\u001b[1;32m 64\u001b[0m \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39m\u001b[38;5;21m__init__\u001b[39m()\n\u001b[1;32m 66\u001b[0m dvs_input \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m \u001b[38;5;66;03m# TODO for now the graph part is not taking into consideration this.\u001b[39;00m\n\u001b[0;32m---> 68\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgraph_tracer \u001b[38;5;241m=\u001b[39m GraphTracer( \u001b[38;5;66;03m# computational graph from original PyTorch module.\u001b[39;00m\n\u001b[1;32m 69\u001b[0m snn\u001b[38;5;241m.\u001b[39manalog_model, \n\u001b[1;32m 70\u001b[0m torch\u001b[38;5;241m.\u001b[39mrandn((\u001b[38;5;241m1\u001b[39m, \u001b[38;5;241m*\u001b[39minput_shape)) \u001b[38;5;66;03m# torch.jit needs the batch dimension.\u001b[39;00m\n\u001b[1;32m 71\u001b[0m )\n\u001b[1;32m 73\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39minput_shape \u001b[38;5;241m=\u001b[39m input_shape\n\u001b[1;32m 75\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlayers \u001b[38;5;241m=\u001b[39m convert_model_to_layer_list( \u001b[38;5;66;03m# convert models to sequential.\u001b[39;00m\n\u001b[1;32m 76\u001b[0m model\u001b[38;5;241m=\u001b[39msnn\u001b[38;5;241m.\u001b[39mspiking_model, ignore\u001b[38;5;241m=\u001b[39mDEFAULT_IGNORED_LAYER_TYPES\n\u001b[1;32m 77\u001b[0m )\n", + "File \u001b[0;32m~/Documents/github/sinabs/sinabs/backend/dynapcnn/graph_tracer.py:14\u001b[0m, in \u001b[0;36mGraphTracer.__init__\u001b[0;34m(self, model, dummy_input)\u001b[0m\n\u001b[1;32m 11\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m, model: Union[nn\u001b[38;5;241m.\u001b[39mSequential, nn\u001b[38;5;241m.\u001b[39mModule], dummy_input: np\u001b[38;5;241m.\u001b[39marray) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 12\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\" .\"\"\"\u001b[39;00m\n\u001b[0;32m---> 14\u001b[0m trace \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mjit\u001b[38;5;241m.\u001b[39mtrace(model, dummy_input)\n\u001b[1;32m 15\u001b[0m _ \u001b[38;5;241m=\u001b[39m trace(dummy_input)\n\u001b[1;32m 16\u001b[0m __ \u001b[38;5;241m=\u001b[39m copy\u001b[38;5;241m.\u001b[39mdeepcopy(trace)\n", + "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/jit/_trace.py:806\u001b[0m, in \u001b[0;36mtrace\u001b[0;34m(func, example_inputs, optimize, check_trace, check_inputs, check_tolerance, strict, _force_outplace, _module_class, _compilation_unit, example_kwarg_inputs, _store_inputs)\u001b[0m\n\u001b[1;32m 804\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 805\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mexample_kwarg_inputs should be a dict\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m--> 806\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m trace_module(\n\u001b[1;32m 807\u001b[0m func,\n\u001b[1;32m 808\u001b[0m {\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mforward\u001b[39m\u001b[38;5;124m\"\u001b[39m: example_inputs},\n\u001b[1;32m 809\u001b[0m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m 810\u001b[0m check_trace,\n\u001b[1;32m 811\u001b[0m wrap_check_inputs(check_inputs),\n\u001b[1;32m 812\u001b[0m check_tolerance,\n\u001b[1;32m 813\u001b[0m strict,\n\u001b[1;32m 814\u001b[0m _force_outplace,\n\u001b[1;32m 815\u001b[0m _module_class,\n\u001b[1;32m 816\u001b[0m example_inputs_is_kwarg\u001b[38;5;241m=\u001b[39m\u001b[38;5;28misinstance\u001b[39m(example_kwarg_inputs, \u001b[38;5;28mdict\u001b[39m),\n\u001b[1;32m 817\u001b[0m _store_inputs\u001b[38;5;241m=\u001b[39m_store_inputs,\n\u001b[1;32m 818\u001b[0m )\n\u001b[1;32m 819\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m (\n\u001b[1;32m 820\u001b[0m \u001b[38;5;28mhasattr\u001b[39m(func, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m__self__\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 821\u001b[0m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(func\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__self__\u001b[39m, torch\u001b[38;5;241m.\u001b[39mnn\u001b[38;5;241m.\u001b[39mModule)\n\u001b[1;32m 822\u001b[0m \u001b[38;5;129;01mand\u001b[39;00m func\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mforward\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 823\u001b[0m ):\n\u001b[1;32m 824\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m example_inputs \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n", + "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/jit/_trace.py:1074\u001b[0m, in \u001b[0;36mtrace_module\u001b[0;34m(mod, inputs, optimize, check_trace, check_inputs, check_tolerance, strict, _force_outplace, _module_class, _compilation_unit, example_inputs_is_kwarg, _store_inputs)\u001b[0m\n\u001b[1;32m 1072\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 1073\u001b[0m example_inputs \u001b[38;5;241m=\u001b[39m make_tuple(example_inputs)\n\u001b[0;32m-> 1074\u001b[0m module\u001b[38;5;241m.\u001b[39m_c\u001b[38;5;241m.\u001b[39m_create_method_from_trace(\n\u001b[1;32m 1075\u001b[0m method_name,\n\u001b[1;32m 1076\u001b[0m func,\n\u001b[1;32m 1077\u001b[0m example_inputs,\n\u001b[1;32m 1078\u001b[0m var_lookup_fn,\n\u001b[1;32m 1079\u001b[0m strict,\n\u001b[1;32m 1080\u001b[0m _force_outplace,\n\u001b[1;32m 1081\u001b[0m argument_names,\n\u001b[1;32m 1082\u001b[0m _store_inputs,\n\u001b[1;32m 1083\u001b[0m )\n\u001b[1;32m 1085\u001b[0m check_trace_method \u001b[38;5;241m=\u001b[39m module\u001b[38;5;241m.\u001b[39m_c\u001b[38;5;241m.\u001b[39m_get_method(method_name)\n\u001b[1;32m 1087\u001b[0m \u001b[38;5;66;03m# Check the trace against new traces created from user-specified inputs\u001b[39;00m\n", + "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/nn/modules/module.py:1511\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1509\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs) \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m 1510\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1511\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n", + "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/nn/modules/module.py:1520\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1515\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1516\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1517\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m 1518\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1519\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1520\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m forward_call(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m 1522\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 1523\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n", + "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/nn/modules/module.py:1501\u001b[0m, in \u001b[0;36mModule._slow_forward\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m 1499\u001b[0m recording_scopes \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[1;32m 1500\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m-> 1501\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mforward(\u001b[38;5;241m*\u001b[39m\u001b[38;5;28minput\u001b[39m, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m 1502\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[1;32m 1503\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m recording_scopes:\n", + "Cell \u001b[0;32mIn[4], line 33\u001b[0m, in \u001b[0;36mResCNN.forward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m 30\u001b[0m rel2_out \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mrel2(conv2_out)\n\u001b[1;32m 31\u001b[0m pool2_out \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpool2(rel2_out)\n\u001b[0;32m---> 33\u001b[0m conv3_out \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconv3(pool2_out \u001b[38;5;241m+\u001b[39m rel1_out)\n\u001b[1;32m 34\u001b[0m rel3_out \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mrel3(conv3_out)\n\u001b[1;32m 35\u001b[0m pool3_out \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpool3(rel3_out)\n", + "\u001b[0;31mRuntimeError\u001b[0m: The size of tensor a (4) must match the size of tensor b (24) at non-singleton dimension 3" + ] + } + ], + "source": [ + "hw_model = DynapcnnNetworkGraph(\n", + " sinabs_model,\n", + " discretize=True,\n", + " input_shape=input_shape\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0 Conv2d(1, 20, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", + "1 IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-1.), batch_size=1, num_timesteps=-1)\n", + "2 AvgPool2d(kernel_size=2, stride=2, padding=0)\n", + "3 Conv2d(20, 32, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", + "4 IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-1.), batch_size=1, num_timesteps=-1)\n", + "5 AvgPool2d(kernel_size=2, stride=2, padding=0)\n", + "6 Conv2d(32, 128, kernel_size=(3, 3), stride=(1, 1), bias=False)\n", + "7 IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-1.), batch_size=1, num_timesteps=-1)\n", + "8 AvgPool2d(kernel_size=2, stride=2, padding=0)\n", + "9 Linear(in_features=128, out_features=500, bias=False)\n", + "10 IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-1.), batch_size=1, num_timesteps=-1)\n", + "11 Linear(in_features=500, out_features=10, bias=False)\n", + "12 IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-1.), batch_size=1, num_timesteps=-1)\n" + ] + } + ], + "source": [ + "for i, l in enumerate(hw_model.layers):\n", + " print(i, l)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(0, 1)\n", + "(1, 2)\n", + "(2, 3)\n", + "(3, 4)\n", + "(4, 5)\n", + "(5, 6)\n", + "(6, 7)\n", + "(7, 8)\n", + "(8, 9)\n", + "(9, 10)\n", + "(10, 11)\n", + "(11, 12)\n" + ] + } + ], + "source": [ + "for edge in hw_model.sinabs_edges:\n", + " print(edge)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Deploying Model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Network is valid\n" + ] + }, + { + "data": { + "text/plain": [ + "DynapcnnNetworkGraph()" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "hw_model.to(device=\"speck2edevkit:0\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "layer index: 0\n", + "layer modules: DynapcnnLayer(\n", + " (conv_layer): Conv2d(1, 20, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", + " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(635.), min_v_mem=Parameter containing:\n", + " tensor(-635.), batch_size=1, num_timesteps=-1)\n", + " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", + ")\n", + "layer destinations: [1]\n", + "assigned core: 0\n", + "\n", + "layer index: 1\n", + "layer modules: DynapcnnLayer(\n", + " (conv_layer): Conv2d(20, 32, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", + " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(11361.), min_v_mem=Parameter containing:\n", + " tensor(-11361.), batch_size=1, num_timesteps=-1)\n", + " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", + ")\n", + "layer destinations: [2]\n", + "assigned core: 3\n", + "\n", + "layer index: 2\n", + "layer modules: DynapcnnLayer(\n", + " (conv_layer): Conv2d(32, 128, kernel_size=(3, 3), stride=(1, 1), bias=False)\n", + " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(8621.), min_v_mem=Parameter containing:\n", + " tensor(-8621.), batch_size=1, num_timesteps=-1)\n", + " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", + ")\n", + "layer destinations: [3]\n", + "assigned core: 5\n", + "\n", + "layer index: 3\n", + "layer modules: DynapcnnLayer(\n", + " (conv_layer): Conv2d(128, 500, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(5747.), min_v_mem=Parameter containing:\n", + " tensor(-5747.), batch_size=1, num_timesteps=-1)\n", + ")\n", + "layer destinations: [4]\n", + "assigned core: 6\n", + "\n", + "layer index: 4\n", + "layer modules: DynapcnnLayer(\n", + " (conv_layer): Conv2d(500, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(2841.), min_v_mem=Parameter containing:\n", + " tensor(-2841.), batch_size=1, num_timesteps=-1)\n", + ")\n", + "layer destinations: []\n", + "assigned core: 1\n", + "\n" + ] + } + ], + "source": [ + "print(hw_model)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "speck-rescnn", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From cf90769850eda1eca73c04c32075cc85b77faca0 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Wed, 17 Apr 2024 15:40:51 +0200 Subject: [PATCH 019/379] converting NIR graph into DynapcnnNetworkGraph edges --- sinabs/backend/dynapcnn/NIRGraphExtractor.py | 107 ++++++++++++++++++ .../dynapcnn/dynapcnn_network_graph.py | 25 ++-- 2 files changed, 122 insertions(+), 10 deletions(-) create mode 100644 sinabs/backend/dynapcnn/NIRGraphExtractor.py diff --git a/sinabs/backend/dynapcnn/NIRGraphExtractor.py b/sinabs/backend/dynapcnn/NIRGraphExtractor.py new file mode 100644 index 00000000..c20f4217 --- /dev/null +++ b/sinabs/backend/dynapcnn/NIRGraphExtractor.py @@ -0,0 +1,107 @@ +import torch +import torch.nn as nn +import nirtorch +import copy + +class NIRtoDynapcnnNetworkGraph(): + def __init__(self, analog_model, dummy_input) -> None: + """ . + + TODO + [ ] test it with nn.Sequential and a version defined as class inheriting from nn.Module. + """ + + nir_graph = nirtorch.extract_torch_graph(analog_model, dummy_input, model_name=None).ignore_tensors() + + self.edges_list, self.name_2_indx_map = self.get_edges_from_nir(nir_graph, analog_model) + + self.modules_map = self.get_named_modules(analog_model) + + def get_edges_from_nir(self, nir_graph, analog_model): + """ .""" + edges_list = [] + name_2_indx_map = {} + idx_counter = 0 + + for src_node in nir_graph.node_list: # source node. + if src_node.name not in name_2_indx_map: + name_2_indx_map[src_node.name] = idx_counter + idx_counter += 1 + + for trg_node in src_node.outgoing_nodes: # target node. + if trg_node.name not in name_2_indx_map: + name_2_indx_map[trg_node.name] = idx_counter + idx_counter += 1 + + edges_list.append((name_2_indx_map[src_node.name], name_2_indx_map[trg_node.name])) + + return edges_list, name_2_indx_map + + def get_named_modules(self, analog_model): + """ .""" + modules_map = {} + + if isinstance(analog_model, nn.Sequential): # access modules via `.named_modules()`. + for name, module in analog_model.named_modules(): + if name != '': # skip the module itself. + modules_map[self.name_2_indx_map[name]] = module + + elif isinstance(analog_model, nn.Module): # access modules via `.named_children()`. + for name, module in analog_model.named_children(): + modules_map[self.name_2_indx_map[name]] = module + + else: + # TODO raise error + pass + + return modules_map + + def remove_ignored_nodes(self, default_ignored_nodes): + """ Recreates the edges list based on layers that 'DynapcnnNetwork' will ignore. This + is done by setting the source (target) node of an edge where the source (target) node + will be dropped as the node that originally targeted this node to be dropped. + """ + edges = copy.deepcopy(self.edges_list) + parsed_edges = [] + removed_nodes = [] + + # removing ignored nodes from edges. + for edge_idx in range(len(edges)): + _src = edges[edge_idx][0] + _trg = edges[edge_idx][1] + + if isinstance(self.modules_map[_src], default_ignored_nodes): + removed_nodes.append(_src) + # all edges where node '_src' is target change it to node '_trg' as their target. + for edge in edges: + if edge[1] == _src: + new_edge = (edge[0], _trg) + elif isinstance(self.modules_map[_trg], default_ignored_nodes): + removed_nodes.append(_trg) + # all edges where node '_trg' is source change it to node '_src' as their source. + for edge in edges: + if edge[0] == _trg: + new_edge = (_src, edge[1]) + else: + new_edge = (_src, _trg) + + if new_edge not in parsed_edges: + parsed_edges.append(new_edge) + + removed_nodes = list(set(removed_nodes)) + + # remapping nodes indexes. + remapped_nodes = {} + for node_indx, __ in self.modules_map.items(): + _ = [x for x in removed_nodes if node_indx > x] + remapped_nodes[node_indx] = node_indx - len(_) + + for x in removed_nodes: + del remapped_nodes[x] + + # remapping nodes names in parsed edges. + remapped_edges = [] + for edge in parsed_edges: + remapped_edges.append((remapped_nodes[edge[0]], remapped_nodes[edge[1]])) + + return remapped_edges \ No newline at end of file diff --git a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py index da7d8ecc..e24c1c4b 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py @@ -30,6 +30,8 @@ from .exceptions import InvalidTorchModel from warnings import warn +from .NIRGraphExtractor import NIRtoDynapcnnNetworkGraph + class DynapcnnNetworkGraph(nn.Module): """Given a sinabs spiking network, prepare a dynapcnn-compatible network. This can be used to test the network will be equivalent once on DYNAPCNN. This class also provides utilities to @@ -41,7 +43,8 @@ def __init__( snn: Union[nn.Sequential, sinabs.Network, nn.Module], input_shape: Optional[Tuple[int, int, int]] = None, dvs_input: bool = False, - discretize: bool = True + discretize: bool = True, + use_jit_tracer: bool = True ): """ DynapcnnNetworkGraph: a class turning sinabs networks into dynapcnn @@ -65,10 +68,16 @@ def __init__( dvs_input = False # TODO for now the graph part is not taking into consideration this. - self.graph_tracer = GraphTracer( # computational graph from original PyTorch module. - snn.analog_model, - torch.randn((1, *input_shape)) # torch.jit needs the batch dimension. - ) + if use_jit_tracer: # TODO this is deprecated now: we want to use the graph from NIR (remove it). + self.graph_tracer = GraphTracer( # computational graph from original PyTorch module. + snn.analog_model, + torch.randn((1, *input_shape)) # torch.jit needs the batch dimension. + ) + + else: + self.graph_tracer = NIRtoDynapcnnNetworkGraph( # computational graph from original PyTorch module. + snn.analog_model, + torch.randn((1, *input_shape))) # needs the batch dimension. self.input_shape = input_shape @@ -419,8 +428,4 @@ def was_spiking_output_added(sinabs_model: sinabs.Network) -> bool: return False else: - return False - - @staticmethod - def build_from_graph_(): # @TODO used for debug only (remove when class is complete). - return build_from_graph \ No newline at end of file + return False \ No newline at end of file From 6fdc56c8d872505cd0d97f551ca86ac00e20ca15 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Wed, 17 Apr 2024 17:33:52 +0200 Subject: [PATCH 020/379] refactor: extracting graph from sinabs_mode.spiking_model (instead of .analog_model) using NIR + 'build_from_graph()' using dict mapping sinabs_edges to their respective modules --- sinabs/backend/dynapcnn/NIRGraphExtractor.py | 2 +- .../dynapcnn/dynapcnn_network_graph.py | 108 ++++-------------- .../backend/dynapcnn/sinabs_edges_handler.py | 22 ++-- sinabs/backend/dynapcnn/utils.py | 4 +- 4 files changed, 35 insertions(+), 101 deletions(-) diff --git a/sinabs/backend/dynapcnn/NIRGraphExtractor.py b/sinabs/backend/dynapcnn/NIRGraphExtractor.py index c20f4217..88726c54 100644 --- a/sinabs/backend/dynapcnn/NIRGraphExtractor.py +++ b/sinabs/backend/dynapcnn/NIRGraphExtractor.py @@ -104,4 +104,4 @@ def remove_ignored_nodes(self, default_ignored_nodes): for edge in parsed_edges: remapped_edges.append((remapped_nodes[edge[0]], remapped_nodes[edge[1]])) - return remapped_edges \ No newline at end of file + return remapped_edges, remapped_nodes \ No newline at end of file diff --git a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py index e24c1c4b..9baeeae6 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py @@ -4,7 +4,7 @@ import time from subprocess import CalledProcessError -from typing import List, Optional, Sequence, Tuple, Union +from typing import List, Optional, Sequence, Tuple, Union, Dict import samna import sinabs.layers @@ -19,10 +19,7 @@ from .io import disable_timestamps, enable_timestamps, open_device, reset_timestamps from .utils import ( DEFAULT_IGNORED_LAYER_TYPES, - build_from_list, build_from_graph, - convert_model_to_layer_list, - infer_input_shape, parse_device_id, ) @@ -43,8 +40,7 @@ def __init__( snn: Union[nn.Sequential, sinabs.Network, nn.Module], input_shape: Optional[Tuple[int, int, int]] = None, dvs_input: bool = False, - discretize: bool = True, - use_jit_tracer: bool = True + discretize: bool = True ): """ DynapcnnNetworkGraph: a class turning sinabs networks into dynapcnn @@ -67,37 +63,23 @@ def __init__( super().__init__() dvs_input = False # TODO for now the graph part is not taking into consideration this. - - if use_jit_tracer: # TODO this is deprecated now: we want to use the graph from NIR (remove it). - self.graph_tracer = GraphTracer( # computational graph from original PyTorch module. - snn.analog_model, - torch.randn((1, *input_shape)) # torch.jit needs the batch dimension. - ) - - else: - self.graph_tracer = NIRtoDynapcnnNetworkGraph( # computational graph from original PyTorch module. - snn.analog_model, - torch.randn((1, *input_shape))) # needs the batch dimension. - + self.dvs_input = dvs_input # check if dvs input is expected. self.input_shape = input_shape - self.layers = convert_model_to_layer_list( # convert models to sequential. - model=snn.spiking_model, ignore=DEFAULT_IGNORED_LAYER_TYPES - ) - - self.dvs_input = dvs_input # check if dvs input is expected. + assert len(self.input_shape) == 3, "infer_input_shape did not return 3-tuple" - input_shape = infer_input_shape(self.layers, input_shape=input_shape) - assert len(input_shape) == 3, "infer_input_shape did not return 3-tuple" + self.graph_tracer = NIRtoDynapcnnNetworkGraph( # computational graph from original PyTorch module. + snn.spiking_model, + torch.randn((1, *self.input_shape))) # needs the batch dimension. - self.sinabs_edges = self.get_sinabs_edges(snn) # get sinabs graph. + self.sinabs_edges, self.sinabs_modules_map = self.get_sinabs_edges_and_modules(snn) self.dynapcnn_layers, \ self.nodes_to_dcnnl_map, \ self.dcnnl_to_dcnnl_map = build_from_graph( # build model from graph edges. discretize=discretize, - layers=self.layers, - in_shape=input_shape, + layers=self.sinabs_modules_map, + in_shape=self.input_shape, edges=self.sinabs_edges) def __str__(self): @@ -359,9 +341,10 @@ def _make_config( return config, config_builder.validate_configuration(config) # validate config. - def get_sinabs_edges(self, sinabs_model: sinabs.network.Network) -> List[Tuple[int, int]]: - """ Converts the computational graph extracted from 'sinabs_model.analog_model' into its equivalent - representation for the 'sinabs_model.spiking_model'. + def get_sinabs_edges_and_modules(self, sinabs_model: sinabs.network.Network) -> Tuple[List[Tuple[int, int]], Dict[int, nn.Module]]: + """ The computational graph extracted from 'sinabs_model.spiking_model' contains layers that are ignored (e.g. a nn.Flatten() will be + ignored when creating a `DynapcnnLayer` instance). Thus the list of edges from such model need to be rebuilt such that if there are + edges `(A, X)` and `(X, B)`, and `X` is an ignored layer, an edge `(A, B)` is created. Parameters ---------- @@ -369,63 +352,14 @@ def get_sinabs_edges(self, sinabs_model: sinabs.network.Network) -> List[Tuple[i Returns sinabs_edges: a list of tuples representing the edges between the layers of a sinabs model. + sinabs_modules_map: a dictionary containing the nodes of the graph as `key` and their associated module as `value`. ---------- """ - # parse original graph to ammend edges containing nodes dropped in 'convert_model_to_layer_list()'. - sinabs_edges = self.graph_tracer.remove_ignored_nodes(DEFAULT_IGNORED_LAYER_TYPES) - - if DynapcnnNetworkGraph.was_spiking_output_added(sinabs_model): - last_edge = sinabs_edges[-1] - new_edge = (last_edge[1], last_edge[1]+1) # spiking output layer has been added: create new edge. - sinabs_edges.append(new_edge) - else: - pass - - return sinabs_edges + sinabs_edges, remapped_nodes = self.graph_tracer.remove_ignored_nodes( # remap `(A, X)` and `(X, B)` into `(A, B)`. + DEFAULT_IGNORED_LAYER_TYPES) - @staticmethod - def was_spiking_output_added(sinabs_model: sinabs.Network) -> bool: - """ Compares the models outputed by 'sinabs.from_torch.from_model()' to check if - a spiking output was added to the spiking version of the analog model. + sinabs_modules_map = {} # nodes (layers' "names") need remapping in case some layers have been removed (e.g. nn.Flattern()). + for orig_name, new_name in remapped_nodes.items(): + sinabs_modules_map[new_name] = self.graph_tracer.modules_map[orig_name] - Parameters - ---------- - sinabs_model: a sinabs network. `sinabs_model.analog_model`\`sinabs_model.spiking_model` need to be either a nn.Module or a nn.Sequential. - - Returns - ---------- - bool: wheter or not a neuron layers has been added to the `sinabs_model.spiking_model`. - """ - analog_modules = [] - spiking_modules = [] - - if isinstance(sinabs_model.analog_model, nn.Sequential): - for mod in sinabs_model.analog_model: - analog_modules.append(mod) - - elif isinstance(sinabs_model.analog_model, nn.Module): - analog_modules = [layer for _, layer in sinabs_model.analog_model.named_children()] - - else: - raise InvalidTorchModel('sinabs_model.analog_model') - - if isinstance(sinabs_model.spiking_model, nn.Sequential): - for mod in sinabs_model.spiking_model: - spiking_modules.append(mod) - - elif isinstance(sinabs_model.spiking_model, nn.Module): - spiking_modules = [layer for _, layer in sinabs_model.spiking_model.named_children()] - - else: - raise InvalidTorchModel('sinabs_model.spiking_model') - - if len(analog_modules) != len(spiking_modules): - if isinstance(spiking_modules[-1], sinabs.layers.iaf.IAFSqueeze): - return True - - else: - warn(f'sinabs.spiking_model has a {type(spiking_modules[-1])} as last layer.') - return False - - else: - return False \ No newline at end of file + return sinabs_edges, sinabs_modules_map \ No newline at end of file diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index 4809c6a4..64827f56 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -2,18 +2,18 @@ # author : Willian Soares Girao # contact : williansoaresgirao@gmail.com -from typing import Tuple, List +from typing import Tuple, List, Dict import torch.nn as nn from .sinabs_edges_utils import * -def process_edge(layers: List[nn.Module], edge: Tuple[int, int], mapper: dict) -> None: +def process_edge(layers: Dict[int, nn.Module], edge: Tuple[int, int], mapper: dict) -> None: """ Read in an edge describing the connection between two layers (nodes in the computational graph). If 'edge' is a valid connection between two layers, update 'mapper' to incorporate these layers into a new or existing dictonary containing the modules comprising a future DynacnnLayer object. Parameters ---------- - layers : list of modules returned by 'utils.convert_model_to_layer_list()'. + layers : a dictionary containing the nodes of the graph as `key` and their associated module as `value`. edge : tuple representing the connection between two nodes in computational graph of 'DynapcnnNetworkGraph.snn.spiking_model'. mapper : dictionary where each 'key' is the index of a future 'DynapcnnLayer' and 'value' is a dictionary ('key': node, 'value': module). """ @@ -24,7 +24,7 @@ def process_edge(layers: List[nn.Module], edge: Tuple[int, int], mapper: dict) - else: raise InvalidEdge(edge, type(layers[edge[0]]), type(layers[edge[1]])) -def is_valid_edge(edge: Tuple[int, int], layers: List[nn.Module], valid_edges_map: dict) -> int: +def is_valid_edge(edge: Tuple[int, int], layers: Dict[int, nn.Module], valid_edges_map: dict) -> int: """ Checks if the modules each node in 'edge' represent are a valid connection between a sinabs network to be loaded on Speck. @@ -42,7 +42,7 @@ def is_valid_edge(edge: Tuple[int, int], layers: List[nn.Module], valid_edges_ma return edge_type return None -def update_dynapcnnlayer_mapper(edge_type: int, edge: Tuple[int, int], mapper: dict, layers: List[nn.Module]) -> None: +def update_dynapcnnlayer_mapper(edge_type: int, edge: Tuple[int, int], mapper: dict, layers: Dict[int, nn.Module]) -> None: """ Parses the nodes within an edge and incorporate them either into a **new** or an **already existing** DynapcnnLayer represented in 'mapper'. """ @@ -59,7 +59,7 @@ def update_dynapcnnlayer_mapper(edge_type: int, edge: Tuple[int, int], mapper: d else: raise InvalidEdgeType(edge, edge_type) -def init_xor_complete_new_dynapcnnlayer_blk(mapper: dict, edge: Tuple[int, int], layers: List[nn.Module]) -> None: +def init_xor_complete_new_dynapcnnlayer_blk(mapper: dict, edge: Tuple[int, int], layers: Dict[int, nn.Module]) -> None: """ Incorporates nodes from either a '(conv, neuron)' or a '(linear, neuron)' edge. These are either initiating a (new) DynapcnnLayer or completing a conv->neuron sequence (in the case the node for 'conv' as already been incorporated somewhere in 'mapper'). 'nn.Linear' layers are converted into 'nn.Conv2d' by DynapcnnLayer. @@ -82,7 +82,7 @@ def init_xor_complete_new_dynapcnnlayer_blk(mapper: dict, edge: Tuple[int, int], dynapcnnlayer_indx += 1 mapper[dynapcnnlayer_indx] = {edge[0]: layers[edge[0]], edge[1]: layers[edge[1]]} -def connect_dynapcnnlayer_blks(mapper: dict, edge: Tuple[int, int], layers: List[nn.Module]) -> None: +def connect_dynapcnnlayer_blks(mapper: dict, edge: Tuple[int, int], layers: Dict[int, nn.Module]) -> None: """ Incorporates nodes from either a '(neuron, conv)/(neuron, lin)' or '(pool, conv)/(pool, lin)' edge. These represent connections between an existing DynapcnnLayer in 'mapper' and a new one yet to be represented in 'mapper'. 'nn.Linear' layers are converted into 'nn.Conv2d' by DynapcnnLayer. """ @@ -102,7 +102,7 @@ def connect_dynapcnnlayer_blks(mapper: dict, edge: Tuple[int, int], layers: List else: raise UnmatchedNode(edge, node) -def add_pool_to_dynapcnnlayer_blk(mapper: dict, edge: Tuple[int, int], layers: List[nn.Module]) -> None: +def add_pool_to_dynapcnnlayer_blk(mapper: dict, edge: Tuple[int, int], layers: Dict[int, nn.Module]) -> None: """ Incorporating a '(neuron, pool)' edge. Node 'pool' has to be part of an already existing DynapcnnLayer in 'mapper'. """ matched = False for indx, dynapcnnlayer in mapper.items(): @@ -124,14 +124,14 @@ def is_initialized_node(node: int, mapper: dict) -> bool: return True return False -def get_dynapcnnlayers_destinations(layers: List[nn.Module], edges: List[Tuple[int, int]], mapper: dict) -> dict: +def get_dynapcnnlayers_destinations(layers: Dict[int, nn.Module], edges: List[Tuple[int, int]], mapper: dict) -> dict: """ Loops over the edges list describing the computational graph. It will access each node in the graph and find to which DynapcnnLayer they belong to. If source and target belong to different DynapcnnLayers (described as a dictionary in 'mapper') the destination of the 'DynapcnnLayer.source' is set to be 'DynapcnnLayer.target'. Parameters ---------- - layers : list of modules returned by 'utils.convert_model_to_layer_list()'. + layers : a dictionary containing the nodes of the graph as `key` and their associated module as `value`. edges : list of tuples representing the connection between nodes in computational graph of 'DynapcnnNetworkGraph.snn.spiking_model'. mapper : dictionary where each 'key' is the index of a future 'DynapcnnLayer' and 'value' its modules (output of 'process_edge(mapper)'). @@ -174,7 +174,7 @@ def get_dynapcnnlayer_index(node: int, mapper: dict) -> int: return indx raise UnknownNode(node) -def is_valid_dynapcnnlayer_pairing(layers: List[nn.Module], edge: Tuple[int, int], valid_dynapcnnlayer_edges: List[Tuple[nn.Module, nn.Module]]) -> bool: +def is_valid_dynapcnnlayer_pairing(layers: Dict[int, nn.Module], edge: Tuple[int, int], valid_dynapcnnlayer_edges: List[Tuple[nn.Module, nn.Module]]) -> bool: """ Checks if the module in 'DynapcnnLayer.source' is targetting a valid module in 'DynapcnnLayer.target'. """ if (type(layers[edge[0]]), type(layers[edge[1]])) in valid_dynapcnnlayer_edges: return True diff --git a/sinabs/backend/dynapcnn/utils.py b/sinabs/backend/dynapcnn/utils.py index 301de6de..d1971749 100644 --- a/sinabs/backend/dynapcnn/utils.py +++ b/sinabs/backend/dynapcnn/utils.py @@ -549,7 +549,7 @@ def extend_readout_layer(model: "DynapcnnNetwork") -> "DynapcnnNetwork": def build_from_graph( discretize: bool, - layers: List[nn.Module], + layers: dict, in_shape: Tuple[int, int, int], edges: List[Tuple[int, int]]) -> Tuple[List[DynapcnnLayer], Dict[int, Dict[int, nn.Module]], Dict[int, List[int]]]: """ Parses each edge of a 'sinabs_mode.spiking_model' computational graph. Each node (layer) is assigned to a @@ -560,7 +560,7 @@ def build_from_graph( ---------- discretize: If True, discretize the parameters and thresholds. This is needed for uploading weights to dynapcnn. Set to False only for testing purposes. - layers : List of modules returned by 'utils.convert_model_to_layer_list()'. + layers : ... in_shape : Tuple describing the input to the very first layer (batch_size, hight, width). edges : List of edges returned by 'DynapcnnNetworkGraph.get_sinabs_edges()'. From c8a32248a1e55707c3e39dceda837661c6eff5bd Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Wed, 17 Apr 2024 20:19:29 +0200 Subject: [PATCH 021/379] (WIP) computing in/output_shape when creating DynapcnnNetwork instances for model with Merge layers --- .../dynapcnn/dynapcnn_network_graph.py | 10 +++- .../backend/dynapcnn/sinabs_edges_handler.py | 54 ++++++++++++++++++- sinabs/backend/dynapcnn/utils.py | 11 +++- 3 files changed, 70 insertions(+), 5 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py index 9baeeae6..b127945f 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py @@ -28,6 +28,7 @@ from warnings import warn from .NIRGraphExtractor import NIRtoDynapcnnNetworkGraph +from .sinabs_edges_handler import merge_handler class DynapcnnNetworkGraph(nn.Module): """Given a sinabs spiking network, prepare a dynapcnn-compatible network. This can be used to @@ -74,6 +75,9 @@ def __init__( self.sinabs_edges, self.sinabs_modules_map = self.get_sinabs_edges_and_modules(snn) + # for edge in self.sinabs_edges: + # print(edge) + self.dynapcnn_layers, \ self.nodes_to_dcnnl_map, \ self.dcnnl_to_dcnnl_map = build_from_graph( # build model from graph edges. @@ -351,9 +355,9 @@ def get_sinabs_edges_and_modules(self, sinabs_model: sinabs.network.Network) -> sinabs_model: a sinabs network object created from a PyTorch model. Returns + ---------- sinabs_edges: a list of tuples representing the edges between the layers of a sinabs model. sinabs_modules_map: a dictionary containing the nodes of the graph as `key` and their associated module as `value`. - ---------- """ sinabs_edges, remapped_nodes = self.graph_tracer.remove_ignored_nodes( # remap `(A, X)` and `(X, B)` into `(A, B)`. DEFAULT_IGNORED_LAYER_TYPES) @@ -362,4 +366,6 @@ def get_sinabs_edges_and_modules(self, sinabs_model: sinabs.network.Network) -> for orig_name, new_name in remapped_nodes.items(): sinabs_modules_map[new_name] = self.graph_tracer.modules_map[orig_name] - return sinabs_edges, sinabs_modules_map \ No newline at end of file + edges_without_merge = merge_handler(sinabs_edges, sinabs_modules_map) # bypass merging layers to connect the nodes involved in them directly to the node where the merge happens. + + return edges_without_merge, sinabs_modules_map \ No newline at end of file diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index 64827f56..e3a72b67 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -3,8 +3,10 @@ # contact : williansoaresgirao@gmail.com from typing import Tuple, List, Dict +import sinabs.layers import torch.nn as nn from .sinabs_edges_utils import * +import sinabs, copy def process_edge(layers: Dict[int, nn.Module], edge: Tuple[int, int], mapper: dict) -> None: """ Read in an edge describing the connection between two layers (nodes in the computational graph). If 'edge' @@ -179,4 +181,54 @@ def is_valid_dynapcnnlayer_pairing(layers: Dict[int, nn.Module], edge: Tuple[int if (type(layers[edge[0]]), type(layers[edge[1]])) in valid_dynapcnnlayer_edges: return True else: - raise InvalidLayerDestination(type(layers[edge[0]]), type(layers[edge[1]])) \ No newline at end of file + raise InvalidLayerDestination(type(layers[edge[0]]), type(layers[edge[1]])) + +def merge_handler(sinabs_edges: List[Tuple[int, int]], sinabs_modules_map: Dict[int, nn.Module]) -> List[Tuple[int, int]]: + """ Handles connections between nodes made via a `sinabs.layers.Merge` layer. If `X` is a merge layer then edges `(X, C)` are removed + from the edges list since they don't affect the creationg of DynapcnnLayers. Edges `(Y, X)` are turned into a edge `(Y, C)` pointing + directly to the node receiving the merged inputs such that the DynapcnnLayer containing `Y` can have the DynapcnnLayer containing `C` + as one of its destinations. + + Parameters + ---------- + sinabs_edges: ... + sinabs_modules_map: ... + + Reurns + edges_without_merge: ... + ---------- + """ + edges = copy.deepcopy(sinabs_edges) + edges_without_merge = [] + merge_nodes = {} + + for edge in edges: # finding the nodes representing Merge layers. + src = edge[0] + trg = edge[1] + + if isinstance(sinabs_modules_map[src], sinabs.layers.Merge): + if src not in merge_nodes: # found node receiving merged inputs from two previous layers. + merge_nodes[src] = {'sources': [], 'merge_into': trg} + + for _edge in edges: + if _edge[1] == src: # found node used as argument for a Merge layer. + merge_nodes[src]['sources'].append(_edge[0]) + if len(merge_nodes[src]['sources']) > 2: + raise ValueError("A Merge layer can not have more than two inputs.") + + for edge in edges: # removing edges connection from/to merging layers from the computational graph. + src = edge[0] + trg = edge[1] + + if src in merge_nodes: # edge (`Merge`, trg) is not necessary for later DynapcnnLayer creation. + pass + + elif trg in merge_nodes: # point `src` directly to the node it was previously targeting via a Merge layer. + new_edge = (src, merge_nodes[trg]['merge_into']) + edges_without_merge.append(new_edge) + + else: # edge not involved in merging. + edges_without_merge.append(edge) + + return edges_without_merge + diff --git a/sinabs/backend/dynapcnn/utils.py b/sinabs/backend/dynapcnn/utils.py index d1971749..e3cbd4d8 100644 --- a/sinabs/backend/dynapcnn/utils.py +++ b/sinabs/backend/dynapcnn/utils.py @@ -549,7 +549,7 @@ def extend_readout_layer(model: "DynapcnnNetwork") -> "DynapcnnNetwork": def build_from_graph( discretize: bool, - layers: dict, + layers: Dict[int, nn.Module], in_shape: Tuple[int, int, int], edges: List[Tuple[int, int]]) -> Tuple[List[DynapcnnLayer], Dict[int, Dict[int, nn.Module]], Dict[int, List[int]]]: """ Parses each edge of a 'sinabs_mode.spiking_model' computational graph. Each node (layer) is assigned to a @@ -560,7 +560,7 @@ def build_from_graph( ---------- discretize: If True, discretize the parameters and thresholds. This is needed for uploading weights to dynapcnn. Set to False only for testing purposes. - layers : ... + layers : a dictionary containing the nodes of the graph as `key` and their associated module as `value`. in_shape : Tuple describing the input to the very first layer (batch_size, hight, width). edges : List of edges returned by 'DynapcnnNetworkGraph.get_sinabs_edges()'. @@ -665,6 +665,11 @@ def construct_dynapcnnlayer( else: raise WrongModuleCount(layer_index, len(layer_modules)) + print('input shape: ', input_shape) + print(lyr_conv) + print(lyr_spk) + print(lyr_pool) + dynapcnnlayer = DynapcnnLayer( conv = lyr_conv, spk = lyr_spk, @@ -673,6 +678,8 @@ def construct_dynapcnnlayer( discretize = discretize, rescale_weights = rescale_factor, ) + print('output shape: ', dynapcnnlayer.get_output_shape()) + print('------------------------------------------') return dynapcnnlayer, dynapcnnlayer.get_output_shape(), rescale_factor_after_pooling From 309a9b73a4a04dfc4c679fa3fe49dabb93176ce4 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Thu, 18 Apr 2024 13:18:07 +0200 Subject: [PATCH 022/379] creating mapping with skip connections between DynapcnnLayers from SNN as a nn.Module --- sinabs/backend/dynapcnn/NIRGraphExtractor.py | 29 +- .../dynapcnn/dynapcnn_network_graph.py | 31 +- .../backend/dynapcnn/sinabs_edges_handler.py | 15 +- sinabs/backend/dynapcnn/sinabs_edges_utils.py | 7 + sinabs/backend/dynapcnn/utils.py | 53 +++- ...DynapcnnNetworkGraph_from_NIRgraph_3.ipynb | 288 ++++++++++++++++++ 6 files changed, 374 insertions(+), 49 deletions(-) create mode 100644 tests/test_nonsequential/DynapcnnNetworkGraph_from_NIRgraph_3.ipynb diff --git a/sinabs/backend/dynapcnn/NIRGraphExtractor.py b/sinabs/backend/dynapcnn/NIRGraphExtractor.py index 88726c54..8c636c24 100644 --- a/sinabs/backend/dynapcnn/NIRGraphExtractor.py +++ b/sinabs/backend/dynapcnn/NIRGraphExtractor.py @@ -4,20 +4,16 @@ import copy class NIRtoDynapcnnNetworkGraph(): - def __init__(self, analog_model, dummy_input) -> None: - """ . - - TODO - [ ] test it with nn.Sequential and a version defined as class inheriting from nn.Module. - """ + def __init__(self, spiking_model, dummy_input) -> None: + """ .""" - nir_graph = nirtorch.extract_torch_graph(analog_model, dummy_input, model_name=None).ignore_tensors() + nir_graph = nirtorch.extract_torch_graph(spiking_model, dummy_input, model_name=None).ignore_tensors() - self.edges_list, self.name_2_indx_map = self.get_edges_from_nir(nir_graph, analog_model) + self.edges_list, self.name_2_indx_map = self.get_edges_from_nir(nir_graph) - self.modules_map = self.get_named_modules(analog_model) + self.modules_map = self.get_named_modules(spiking_model) - def get_edges_from_nir(self, nir_graph, analog_model): + def get_edges_from_nir(self, nir_graph): """ .""" edges_list = [] name_2_indx_map = {} @@ -37,22 +33,21 @@ def get_edges_from_nir(self, nir_graph, analog_model): return edges_list, name_2_indx_map - def get_named_modules(self, analog_model): + def get_named_modules(self, model): """ .""" modules_map = {} - if isinstance(analog_model, nn.Sequential): # access modules via `.named_modules()`. - for name, module in analog_model.named_modules(): + if isinstance(model, nn.Sequential): # access modules via `.named_modules()`. + for name, module in model.named_modules(): if name != '': # skip the module itself. modules_map[self.name_2_indx_map[name]] = module - elif isinstance(analog_model, nn.Module): # access modules via `.named_children()`. - for name, module in analog_model.named_children(): + elif isinstance(model, nn.Module): # access modules via `.named_children()`. + for name, module in model.named_children(): modules_map[self.name_2_indx_map[name]] = module else: - # TODO raise error - pass + raise ValueError('Either a nn.Sequential or a nn.Module is required.') return modules_map diff --git a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py index b127945f..0a4cf626 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py @@ -49,8 +49,7 @@ def __init__( Parameters ---------- - snn: sinabs.Network - SNN that determines the structure of the `DynapcnnNetwork` + snn: ... input_shape: None or tuple of ints Shape of the input, convention: (features, height, width) If None, `snn` needs an InputLayer @@ -70,13 +69,10 @@ def __init__( assert len(self.input_shape) == 3, "infer_input_shape did not return 3-tuple" self.graph_tracer = NIRtoDynapcnnNetworkGraph( # computational graph from original PyTorch module. - snn.spiking_model, + snn, torch.randn((1, *self.input_shape))) # needs the batch dimension. - self.sinabs_edges, self.sinabs_modules_map = self.get_sinabs_edges_and_modules(snn) - - # for edge in self.sinabs_edges: - # print(edge) + self.sinabs_edges, self.sinabs_modules_map, self.merge_nodes = self.get_sinabs_edges_and_modules() self.dynapcnn_layers, \ self.nodes_to_dcnnl_map, \ @@ -84,7 +80,8 @@ def __init__( discretize=discretize, layers=self.sinabs_modules_map, in_shape=self.input_shape, - edges=self.sinabs_edges) + edges=self.sinabs_edges, + merge_nodes=self.merge_nodes) def __str__(self): pretty_print = '' @@ -345,14 +342,10 @@ def _make_config( return config, config_builder.validate_configuration(config) # validate config. - def get_sinabs_edges_and_modules(self, sinabs_model: sinabs.network.Network) -> Tuple[List[Tuple[int, int]], Dict[int, nn.Module]]: - """ The computational graph extracted from 'sinabs_model.spiking_model' contains layers that are ignored (e.g. a nn.Flatten() will be + def get_sinabs_edges_and_modules(self) -> Tuple[List[Tuple[int, int]], Dict[int, nn.Module]]: + """ The computational graph extracted from `snn` might contain layers that are ignored (e.g. a `nn.Flatten` will be ignored when creating a `DynapcnnLayer` instance). Thus the list of edges from such model need to be rebuilt such that if there are edges `(A, X)` and `(X, B)`, and `X` is an ignored layer, an edge `(A, B)` is created. - - Parameters - ---------- - sinabs_model: a sinabs network object created from a PyTorch model. Returns ---------- @@ -366,6 +359,12 @@ def get_sinabs_edges_and_modules(self, sinabs_model: sinabs.network.Network) -> for orig_name, new_name in remapped_nodes.items(): sinabs_modules_map[new_name] = self.graph_tracer.modules_map[orig_name] - edges_without_merge = merge_handler(sinabs_edges, sinabs_modules_map) # bypass merging layers to connect the nodes involved in them directly to the node where the merge happens. + edges_without_merge, merge_nodes = merge_handler(sinabs_edges, sinabs_modules_map) # bypass merging layers to connect the nodes involved in them directly to the node where the merge happens. + + merge_data = {} + for key, val in merge_nodes.items(): + merge_data[val['merge_into']] = {} + for src in val['sources']: + merge_data[val['merge_into']][src] = None - return edges_without_merge, sinabs_modules_map \ No newline at end of file + return edges_without_merge, sinabs_modules_map, merge_data \ No newline at end of file diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index e3a72b67..721ab773 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -1,6 +1,9 @@ -# functionality : functions implementing the pre-processing of edges into blocks of nodes (modules) for future creation of DynapcnnLayer objects. -# author : Willian Soares Girao -# contact : williansoaresgirao@gmail.com +""" +functionality : functions implementing the pre-processing of edges into blocks of nodes (modules) for future + creation of DynapcnnLayer objects. +author : Willian Soares Girao +contact : williansoaresgirao@gmail.com +""" from typing import Tuple, List, Dict import sinabs.layers @@ -39,9 +42,11 @@ def is_valid_edge(edge: Tuple[int, int], layers: Dict[int, nn.Module], valid_edg edge_type: the edge type specified in 'valid_edges_map' ('None' if edge is not valid). """ edge_layers = (layers[edge[0]], layers[edge[1]]) + for edge_type, sinabs_edge in valid_edges_map.items(): if (type(edge_layers[0]) == sinabs_edge[0]) and (type(edge_layers[1]) == sinabs_edge[1]): return edge_type + return None def update_dynapcnnlayer_mapper(edge_type: int, edge: Tuple[int, int], mapper: dict, layers: Dict[int, nn.Module]) -> None: @@ -219,7 +224,7 @@ def merge_handler(sinabs_edges: List[Tuple[int, int]], sinabs_modules_map: Dict[ for edge in edges: # removing edges connection from/to merging layers from the computational graph. src = edge[0] trg = edge[1] - + # print('> ', edge) if src in merge_nodes: # edge (`Merge`, trg) is not necessary for later DynapcnnLayer creation. pass @@ -230,5 +235,5 @@ def merge_handler(sinabs_edges: List[Tuple[int, int]], sinabs_modules_map: Dict[ else: # edge not involved in merging. edges_without_merge.append(edge) - return edges_without_merge + return edges_without_merge, merge_nodes diff --git a/sinabs/backend/dynapcnn/sinabs_edges_utils.py b/sinabs/backend/dynapcnn/sinabs_edges_utils.py index b1b89848..29566153 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_utils.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_utils.py @@ -1,3 +1,10 @@ +""" +functionality : implementation of exceptions and constraints regarding the processing of edges from a network + computational graph. +author : Willian Soares Girao +contact : williansoaresgirao@gmail.com +""" + import sinabs.layers as sl import torch.nn as nn from typing import Tuple diff --git a/sinabs/backend/dynapcnn/utils.py b/sinabs/backend/dynapcnn/utils.py index e3cbd4d8..0583c6fa 100644 --- a/sinabs/backend/dynapcnn/utils.py +++ b/sinabs/backend/dynapcnn/utils.py @@ -551,7 +551,8 @@ def build_from_graph( discretize: bool, layers: Dict[int, nn.Module], in_shape: Tuple[int, int, int], - edges: List[Tuple[int, int]]) -> Tuple[List[DynapcnnLayer], Dict[int, Dict[int, nn.Module]], Dict[int, List[int]]]: + edges: List[Tuple[int, int]], + merge_nodes: dict) -> Tuple[List[DynapcnnLayer], Dict[int, Dict[int, nn.Module]], Dict[int, List[int]]]: """ Parses each edge of a 'sinabs_mode.spiking_model' computational graph. Each node (layer) is assigned to a DynapcnnLayer object. The target destination of each DynapcnnLayer is computed via edges connecting nodes in different DynapcnnLayer objects. @@ -599,7 +600,7 @@ def build_from_graph( # turn sets of layers into DynapcnnLayer objects. dynapcnn_layers = construct_dynapcnnlayers_from_mapper( - discretize, nodes_to_dcnnl_map, dcnnl_to_dcnnl_map, in_shape, rescale_factor) + discretize, nodes_to_dcnnl_map, dcnnl_to_dcnnl_map, in_shape, rescale_factor, merge_nodes) for idx, layer_data in dynapcnn_layers.items(): if 'core_idx' not in layer_data: @@ -612,7 +613,8 @@ def construct_dynapcnnlayers_from_mapper( nodes_to_dcnnl_map: dict, dcnnl_to_dcnnl_map: dict, input_shape: Union[Tuple[int, int], Tuple[int, int, int]], - rescale_factor: int) -> Dict[int, Dict[DynapcnnLayer, List]]: + rescale_factor: int, + merge_nodes: dict = None) -> Dict[int, Dict[DynapcnnLayer, List]]: """ Consumes a dictionaries containing sets of layers to be used to populate a DynapcnnLayer object. Parameters @@ -628,7 +630,7 @@ def construct_dynapcnnlayers_from_mapper( for dynapcnnl_indx, layer_modules in nodes_to_dcnnl_map.items(): dynapcnnlayer, input_shape, rescale_factor = construct_dynapcnnlayer( - discretize, layer_modules, dynapcnnl_indx, input_shape, rescale_factor) + discretize, layer_modules, dynapcnnl_indx, input_shape, rescale_factor, merge_nodes) dynapcnn_layers[dynapcnnl_indx] = { 'layer': dynapcnnlayer, @@ -642,7 +644,8 @@ def construct_dynapcnnlayer( layer_modules: dict, layer_index: int, input_shape: Union[Tuple[int, int], Tuple[int, int, int]], - rescale_factor: int) -> Tuple[DynapcnnLayer, Union[Tuple[int, int], Tuple[int, int, int]], int]: + rescale_factor: int, + merge_nodes: dict = None) -> Tuple[DynapcnnLayer, Union[Tuple[int, int], Tuple[int, int, int]], int]: """ Extract the modules (layers) in a dictionary and uses them to instantiate a DynapcnnLayer object. """ lyr_conv = None lyr_spk = None @@ -665,10 +668,19 @@ def construct_dynapcnnlayer( else: raise WrongModuleCount(layer_index, len(layer_modules)) - print('input shape: ', input_shape) - print(lyr_conv) - print(lyr_spk) - print(lyr_pool) + ################ TODO HACKY STUFF MAKE IT BETTER ################## + for node_idx, _ in layer_modules.items(): + if node_idx in merge_nodes: + # print('>>>> merge_nodes: ', merge_nodes[node_idx], get_input_shape_from_merge(merge_nodes[node_idx])) + input_shape = get_input_shape_from_merge(merge_nodes[node_idx]) + break + + ############################################################################### + + # print('input shape: ', input_shape) + # print(lyr_conv) + # print(lyr_spk) + # print(lyr_pool) dynapcnnlayer = DynapcnnLayer( conv = lyr_conv, @@ -678,11 +690,30 @@ def construct_dynapcnnlayer( discretize = discretize, rescale_weights = rescale_factor, ) - print('output shape: ', dynapcnnlayer.get_output_shape()) - print('------------------------------------------') + # print('output shape: ', dynapcnnlayer.get_output_shape()) + + ################ TODO HACKY STUFF MAKE IT BETTER ################## + if len(merge_nodes) != 0: + for merge_target, merge_data in merge_nodes.items(): + for node_idx, _ in layer_modules.items(): + if node_idx in merge_data and merge_data[node_idx] == None: + if isinstance(layer_modules[node_idx], sinabs.layers.iaf.IAFSqueeze): + merge_data[node_idx] = dynapcnnlayer.get_neuron_shape() + else: + merge_data[node_idx] = dynapcnnlayer.get_output_shape() + ################################################################### return dynapcnnlayer, dynapcnnlayer.get_output_shape(), rescale_factor_after_pooling +def get_input_shape_from_merge(merge_data): + ################ TODO HACKY STUFF MAKE IT BETTER ################## + input_shape = [0, 0, 0] + for key, val in merge_data.items(): + for i in range(len(val)): + input_shape[i] = val[i] if val[i] > input_shape[i] else input_shape[i] + return (input_shape[0], input_shape[1], input_shape[2]) + + def build_SumPool2d(module: nn.AvgPool2d) -> Tuple[sl.SumPool2d, int]: """ Converts a 'nn.AvgPool2d' into a 'sl.SumPool2d' layer. """ diff --git a/tests/test_nonsequential/DynapcnnNetworkGraph_from_NIRgraph_3.ipynb b/tests/test_nonsequential/DynapcnnNetworkGraph_from_NIRgraph_3.ipynb new file mode 100644 index 00000000..04cc5f88 --- /dev/null +++ b/tests/test_nonsequential/DynapcnnNetworkGraph_from_NIRgraph_3.ipynb @@ -0,0 +1,288 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "import torch.nn as nn\n", + "from sinabs.from_torch import from_model\n", + "from sinabs.backend.dynapcnn import DynapcnnNetworkGraph\n", + "from sinabs.layers import Merge, IAFSqueeze" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "torch.manual_seed(0)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "channels = 1\n", + "height = 28\n", + "width = 28\n", + "\n", + "input_shape = (channels, height, width)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Network Module (pure Pytorch)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "class SNN(nn.Module):\n", + " def __init__(self) -> None:\n", + " super().__init__()\n", + "\n", + " self.conv1 = nn.Conv2d(1, 20, 5, 1, bias=False)\n", + " self.iaf1 = IAFSqueeze(batch_size=1)\n", + " self.pool1 = nn.AvgPool2d(2,2)\n", + "\n", + " self.conv2 = nn.Conv2d(20, 32, 5, 1, bias=False)\n", + " self.iaf2 = IAFSqueeze(batch_size=1)\n", + " self.pool2 = nn.AvgPool2d(2,2)\n", + "\n", + " self.conv3 = nn.Conv2d(32, 2, 3, 1, bias=False)\n", + " self.iaf3 = IAFSqueeze(batch_size=1)\n", + " self.pool3 = nn.AvgPool2d(2,2)\n", + "\n", + " self.flat = nn.Flatten()\n", + "\n", + " self.fc1 = nn.Linear(242, 500, bias=False)\n", + " self.iaf4 = IAFSqueeze(batch_size=1)\n", + " self.fc2 = nn.Linear(500, 10, bias=False)\n", + " self.iaf5 = IAFSqueeze(batch_size=1)\n", + "\n", + " self.adder = Merge()\n", + "\n", + " def forward(self, x):\n", + " \n", + " con1_out = self.conv1(x)\n", + " iaf1_out = self.iaf1(con1_out)\n", + " pool1_out = self.pool1(iaf1_out)\n", + "\n", + " conv2_out = self.conv2(pool1_out)\n", + " iaf2_out = self.iaf2(conv2_out)\n", + " pool2_out = self.pool2(iaf2_out)\n", + "\n", + " conv3_out = self.conv3(self.adder(iaf1_out, pool2_out))\n", + " iaf3_out = self.iaf3(conv3_out)\n", + " pool3_out = self.pool3(iaf3_out)\n", + "\n", + " flat_out = self.flat(pool3_out)\n", + " \n", + " fc1_out = self.fc1(flat_out)\n", + " iaf4_out = self.iaf4(fc1_out)\n", + " fc2_out = self.fc2(iaf4_out)\n", + " iaf5_out = self.iaf5(fc2_out)\n", + "\n", + " return iaf5_out" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "snn = SNN()" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "iaf1_out: torch.Size([1, 20, 24, 24])\n", + "pool2_out: torch.Size([1, 32, 4, 4])\n", + "added: torch.Size([1, 32, 24, 24])\n", + "pool3_out: torch.Size([1, 2, 11, 11])\n", + "flat_out: torch.Size([1, 242])\n" + ] + } + ], + "source": [ + "x = torch.randn((1, *input_shape))\n", + "\n", + "con1_out = snn.conv1(x)\n", + "iaf1_out = snn.iaf1(con1_out)\n", + "print('iaf1_out: ', iaf1_out.shape)\n", + "pool1_out = snn.pool1(iaf1_out)\n", + "\n", + "conv2_out = snn.conv2(pool1_out)\n", + "iaf2_out = snn.iaf2(conv2_out)\n", + "pool2_out = snn.pool2(iaf2_out)\n", + "print('pool2_out: ', pool2_out.shape)\n", + "\n", + "added = snn.adder(iaf1_out, pool2_out)\n", + "print('added: ', added.shape)\n", + "\n", + "conv3_out = snn.conv3(added)\n", + "iaf3_out = snn.iaf3(conv3_out)\n", + "pool3_out = snn.pool3(iaf3_out)\n", + "print('pool3_out: ', pool3_out.shape)\n", + "\n", + "flat_out = snn.flat(pool3_out)\n", + "print('flat_out: ', flat_out.shape)\n", + "\n", + "fc1_out = snn.fc1(flat_out)\n", + "iaf4_out = snn.iaf4(fc1_out)\n", + "fc2_out = snn.fc2(iaf4_out)\n", + "iaf5_out = snn.iaf5(fc2_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## DynapCNN Model" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "hw_model = DynapcnnNetworkGraph(\n", + " snn,\n", + " discretize=True,\n", + " input_shape=input_shape\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "layer index: 0\n", + "layer modules: DynapcnnLayer(\n", + " (conv_layer): Conv2d(1, 20, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", + " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(1.), min_v_mem=Parameter containing:\n", + " tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", + ")\n", + "layer destinations: [2, 1]\n", + "assigned core: -1\n", + "\n", + "layer index: 1\n", + "layer modules: DynapcnnLayer(\n", + " (conv_layer): Conv2d(20, 32, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", + " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(1.), min_v_mem=Parameter containing:\n", + " tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", + ")\n", + "layer destinations: [2]\n", + "assigned core: -1\n", + "\n", + "layer index: 2\n", + "layer modules: DynapcnnLayer(\n", + " (conv_layer): Conv2d(32, 2, kernel_size=(3, 3), stride=(1, 1), bias=False)\n", + " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(1.), min_v_mem=Parameter containing:\n", + " tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", + ")\n", + "layer destinations: [3]\n", + "assigned core: -1\n", + "\n", + "layer index: 3\n", + "layer modules: DynapcnnLayer(\n", + " (conv_layer): Conv2d(2, 500, kernel_size=(11, 11), stride=(1, 1), bias=False)\n", + " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(1.), min_v_mem=Parameter containing:\n", + " tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + ")\n", + "layer destinations: [4]\n", + "assigned core: -1\n", + "\n", + "layer index: 4\n", + "layer modules: DynapcnnLayer(\n", + " (conv_layer): Conv2d(500, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(1.), min_v_mem=Parameter containing:\n", + " tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + ")\n", + "layer destinations: []\n", + "assigned core: -1\n", + "\n" + ] + } + ], + "source": [ + "print(hw_model)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "hw_model.to(device=\"speck2edevkit:0\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "speck-rescnn", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 70ecddbb16d1af858b2f2f08038c2e63cc2003fd Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Thu, 18 Apr 2024 13:46:48 +0200 Subject: [PATCH 023/379] more descriptive exception msg for --- sinabs/backend/dynapcnn/mapping.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sinabs/backend/dynapcnn/mapping.py b/sinabs/backend/dynapcnn/mapping.py index 19f51d6f..bad403b3 100644 --- a/sinabs/backend/dynapcnn/mapping.py +++ b/sinabs/backend/dynapcnn/mapping.py @@ -205,7 +205,7 @@ def recover_mapping(graph, layer_mapping) -> List[Tuple[int, int]]: if edge.flow == 1: mapping.append((i, edge.t - len(layer_mapping) - 1)) if len(mapping) != len(layer_mapping): - raise ValueError("No valid mapping found") + raise ValueError("One of the DynapcnnLayers could not be mapped to any core.") return mapping From 7bbe228d7d1dd51148b17043a1bae261efc2df82 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Thu, 18 Apr 2024 19:13:40 +0200 Subject: [PATCH 024/379] (WIP) new DynapcnnLayer to handle multiple pooling --- sinabs/backend/dynapcnn/NIRGraphExtractor.py | 33 ++ .../backend/dynapcnn/sinabs_edges_handler.py | 2 +- sinabs/backend/dynapcnn/utils.py | 8 +- ...DynapcnnNetworkGraph_from_NIRgraph_1.ipynb | 291 ++++++++++++++++++ ...DynapcnnNetworkGraph_from_NIRgraph_2.ipynb | 256 +++++++++++++++ ...DynapcnnNetworkGraph_from_NIRgraph_3.ipynb | 182 ++++++----- ...DynapcnnNetworkGraph_from_NIRgraph_4.ipynb | 228 ++++++++++++++ 7 files changed, 908 insertions(+), 92 deletions(-) create mode 100644 tests/test_nonsequential/DynapcnnNetworkGraph_from_NIRgraph_1.ipynb create mode 100644 tests/test_nonsequential/DynapcnnNetworkGraph_from_NIRgraph_2.ipynb create mode 100644 tests/test_nonsequential/DynapcnnNetworkGraph_from_NIRgraph_4.ipynb diff --git a/sinabs/backend/dynapcnn/NIRGraphExtractor.py b/sinabs/backend/dynapcnn/NIRGraphExtractor.py index 8c636c24..cf3962f5 100644 --- a/sinabs/backend/dynapcnn/NIRGraphExtractor.py +++ b/sinabs/backend/dynapcnn/NIRGraphExtractor.py @@ -13,6 +13,8 @@ def __init__(self, spiking_model, dummy_input) -> None: self.modules_map = self.get_named_modules(spiking_model) + # self.get_modules_io() + def get_edges_from_nir(self, nir_graph): """ .""" edges_list = [] @@ -51,6 +53,37 @@ def get_named_modules(self, model): return modules_map + def get_modules_io(self, input_dummy): + """ .""" + modules_io_map = {} + + for edge in self.edges_list: + src = edge[0] + trg = edge[1] + + # pass input through source. + if src not in modules_io_map: + modules_io_map[src] = {'input': None, 'output': None} + + if src == 0: + _input = input_dummy + else: + inp_node = self.find_my_input_node(src) + _input = modules_io_map[inp_node]['output'] + + _output = self.modules_map[src](_input) + + # pass input through target. + if trg not in modules_io_map: + modules_io_map[trg] = {'input': None, 'output': None} + + def find_my_input_node(self, node_idx): + for edge in self.edges_list: + if edge[1] == node_idx: + return edge[0] + return -1 + + def remove_ignored_nodes(self, default_ignored_nodes): """ Recreates the edges list based on layers that 'DynapcnnNetwork' will ignore. This is done by setting the source (target) node of an edge where the source (target) node diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index 721ab773..3e24e9a0 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -236,4 +236,4 @@ def merge_handler(sinabs_edges: List[Tuple[int, int]], sinabs_modules_map: Dict[ edges_without_merge.append(edge) return edges_without_merge, merge_nodes - + \ No newline at end of file diff --git a/sinabs/backend/dynapcnn/utils.py b/sinabs/backend/dynapcnn/utils.py index 0583c6fa..e1527887 100644 --- a/sinabs/backend/dynapcnn/utils.py +++ b/sinabs/backend/dynapcnn/utils.py @@ -583,6 +583,7 @@ def build_from_graph( # input_shape=in_shape, # idx_start=0, # dvs_input=False) + dvs_layer = None rescale_factor = 1 @@ -671,17 +672,11 @@ def construct_dynapcnnlayer( ################ TODO HACKY STUFF MAKE IT BETTER ################## for node_idx, _ in layer_modules.items(): if node_idx in merge_nodes: - # print('>>>> merge_nodes: ', merge_nodes[node_idx], get_input_shape_from_merge(merge_nodes[node_idx])) input_shape = get_input_shape_from_merge(merge_nodes[node_idx]) break ############################################################################### - # print('input shape: ', input_shape) - # print(lyr_conv) - # print(lyr_spk) - # print(lyr_pool) - dynapcnnlayer = DynapcnnLayer( conv = lyr_conv, spk = lyr_spk, @@ -690,7 +685,6 @@ def construct_dynapcnnlayer( discretize = discretize, rescale_weights = rescale_factor, ) - # print('output shape: ', dynapcnnlayer.get_output_shape()) ################ TODO HACKY STUFF MAKE IT BETTER ################## if len(merge_nodes) != 0: diff --git a/tests/test_nonsequential/DynapcnnNetworkGraph_from_NIRgraph_1.ipynb b/tests/test_nonsequential/DynapcnnNetworkGraph_from_NIRgraph_1.ipynb new file mode 100644 index 00000000..237e9068 --- /dev/null +++ b/tests/test_nonsequential/DynapcnnNetworkGraph_from_NIRgraph_1.ipynb @@ -0,0 +1,291 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "import torch.nn as nn\n", + "import networkx as nx\n", + "import matplotlib.pyplot as plt\n", + "from sinabs.from_torch import from_model\n", + "from sinabs.backend.dynapcnn import DynapcnnNetwork, DynapcnnNetworkGraph\n", + "import sinabs as snb" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "torch.manual_seed(0)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "channels = 1\n", + "height = 28\n", + "width = 28\n", + "\n", + "input_shape = (channels, height, width)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Network Module (pure Pytorch)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "ann = nn.Sequential(\n", + " nn.Conv2d(1, 20, 5, 1, bias=False),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(2,2),\n", + " nn.Conv2d(20, 32, 5, 1, bias=False),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(2,2),\n", + " nn.Conv2d(32, 128, 3, 1, bias=False),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(2,2),\n", + " nn.Flatten(),\n", + " nn.Linear(128, 500, bias=False),\n", + " nn.ReLU(),\n", + " nn.Linear(500, 10, bias=False),\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Sinabs Model" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "sinabs_model = from_model(ann, add_spiking_output=True, batch_size=1)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## DynapCNN Model" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + ">>>> 0\n", + ">>>> 1\n", + ">>>> 2\n", + ">>>> 3\n", + ">>>> 4\n", + ">>>> 5\n", + ">>>> 6\n", + ">>>> 7\n", + ">>>> 8\n", + ">>>> 9\n", + ">>>> 10\n", + ">>>> 11\n", + ">>>> 12\n", + ">>>> spike_output\n", + "```mermaid\n", + "graph TD;\n", + "0 --> 1;\n", + "1 --> 2;\n", + "2 --> 3;\n", + "3 --> 4;\n", + "4 --> 5;\n", + "5 --> 6;\n", + "6 --> 7;\n", + "7 --> 8;\n", + "8 --> 9;\n", + "9 --> 10;\n", + "10 --> 11;\n", + "11 --> 12;\n", + "12 --> spike_output;\n", + "spike_output;\n", + "\n", + "```\n", + "\n" + ] + }, + { + "ename": "KeyError", + "evalue": "13", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mKeyError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[6], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m hw_model \u001b[38;5;241m=\u001b[39m DynapcnnNetworkGraph(\n\u001b[1;32m 2\u001b[0m sinabs_model,\n\u001b[1;32m 3\u001b[0m discretize\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m 4\u001b[0m input_shape\u001b[38;5;241m=\u001b[39minput_shape\n\u001b[1;32m 5\u001b[0m )\n", + "File \u001b[0;32m~/Documents/github/sinabs/sinabs/backend/dynapcnn/dynapcnn_network_graph.py:80\u001b[0m, in \u001b[0;36mDynapcnnNetworkGraph.__init__\u001b[0;34m(self, snn, input_shape, dvs_input, discretize)\u001b[0m\n\u001b[1;32m 72\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgraph_tracer \u001b[38;5;241m=\u001b[39m NIRtoDynapcnnNetworkGraph( \u001b[38;5;66;03m# computational graph from original PyTorch module.\u001b[39;00m\n\u001b[1;32m 73\u001b[0m snn\u001b[38;5;241m.\u001b[39mspiking_model,\n\u001b[1;32m 74\u001b[0m torch\u001b[38;5;241m.\u001b[39mrandn((\u001b[38;5;241m1\u001b[39m, \u001b[38;5;241m*\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39minput_shape))) \u001b[38;5;66;03m# needs the batch dimension. \u001b[39;00m\n\u001b[1;32m 76\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msinabs_edges, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msinabs_modules_map, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmerge_nodes \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mget_sinabs_edges_and_modules(snn)\n\u001b[1;32m 78\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdynapcnn_layers, \\\n\u001b[1;32m 79\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnodes_to_dcnnl_map, \\\n\u001b[0;32m---> 80\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdcnnl_to_dcnnl_map \u001b[38;5;241m=\u001b[39m build_from_graph( \u001b[38;5;66;03m# build model from graph edges.\u001b[39;00m\n\u001b[1;32m 81\u001b[0m discretize\u001b[38;5;241m=\u001b[39mdiscretize,\n\u001b[1;32m 82\u001b[0m layers\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msinabs_modules_map, \n\u001b[1;32m 83\u001b[0m in_shape\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39minput_shape,\n\u001b[1;32m 84\u001b[0m edges\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msinabs_edges,\n\u001b[1;32m 85\u001b[0m merge_nodes\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmerge_nodes)\n", + "File \u001b[0;32m~/Documents/github/sinabs/sinabs/backend/dynapcnn/utils.py:595\u001b[0m, in \u001b[0;36mbuild_from_graph\u001b[0;34m(discretize, layers, in_shape, edges, merge_nodes)\u001b[0m\n\u001b[1;32m 593\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 594\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m edge \u001b[38;5;129;01min\u001b[39;00m edges:\n\u001b[0;32m--> 595\u001b[0m process_edge( \u001b[38;5;66;03m# figure out to which (future) DynapcnnLayer each node will belong to.\u001b[39;00m\n\u001b[1;32m 596\u001b[0m layers, edge, nodes_to_dcnnl_map)\n\u001b[1;32m 598\u001b[0m \u001b[38;5;66;03m# look for edges between connecting nodes in different (future) DynapcnnLayer.\u001b[39;00m\n\u001b[1;32m 599\u001b[0m dcnnl_to_dcnnl_map \u001b[38;5;241m=\u001b[39m get_dynapcnnlayers_destinations(layers, edges, nodes_to_dcnnl_map)\n", + "File \u001b[0;32m~/Documents/github/sinabs/sinabs/backend/dynapcnn/sinabs_edges_handler.py:22\u001b[0m, in \u001b[0;36mprocess_edge\u001b[0;34m(layers, edge, mapper)\u001b[0m\n\u001b[1;32m 11\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mprocess_edge\u001b[39m(layers: Dict[\u001b[38;5;28mint\u001b[39m, nn\u001b[38;5;241m.\u001b[39mModule], edge: Tuple[\u001b[38;5;28mint\u001b[39m, \u001b[38;5;28mint\u001b[39m], mapper: \u001b[38;5;28mdict\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 12\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\" Read in an edge describing the connection between two layers (nodes in the computational graph). If 'edge'\u001b[39;00m\n\u001b[1;32m 13\u001b[0m \u001b[38;5;124;03m is a valid connection between two layers, update 'mapper' to incorporate these layers into a new or existing dictonary\u001b[39;00m\n\u001b[1;32m 14\u001b[0m \u001b[38;5;124;03m containing the modules comprising a future DynacnnLayer object.\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 20\u001b[0m \u001b[38;5;124;03m mapper : dictionary where each 'key' is the index of a future 'DynapcnnLayer' and 'value' is a dictionary ('key': node, 'value': module).\u001b[39;00m\n\u001b[1;32m 21\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[0;32m---> 22\u001b[0m edge_type \u001b[38;5;241m=\u001b[39m is_valid_edge(edge, layers, VALID_SINABS_EDGES)\n\u001b[1;32m 24\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(edge_type, \u001b[38;5;28mint\u001b[39m): \u001b[38;5;66;03m# incorporate modules within the edge to a dict representing a future DynapcnnLayer.\u001b[39;00m\n\u001b[1;32m 25\u001b[0m update_dynapcnnlayer_mapper(edge_type, edge, mapper, layers)\n", + "File \u001b[0;32m~/Documents/github/sinabs/sinabs/backend/dynapcnn/sinabs_edges_handler.py:41\u001b[0m, in \u001b[0;36mis_valid_edge\u001b[0;34m(edge, layers, valid_edges_map)\u001b[0m\n\u001b[1;32m 29\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mis_valid_edge\u001b[39m(edge: Tuple[\u001b[38;5;28mint\u001b[39m, \u001b[38;5;28mint\u001b[39m], layers: Dict[\u001b[38;5;28mint\u001b[39m, nn\u001b[38;5;241m.\u001b[39mModule], valid_edges_map: \u001b[38;5;28mdict\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28mint\u001b[39m:\n\u001b[1;32m 30\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\" Checks if the modules each node in 'edge' represent are a valid connection between a sinabs network to be \u001b[39;00m\n\u001b[1;32m 31\u001b[0m \u001b[38;5;124;03m loaded on Speck.\u001b[39;00m\n\u001b[1;32m 32\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 39\u001b[0m \u001b[38;5;124;03m edge_type: the edge type specified in 'valid_edges_map' ('None' if edge is not valid).\u001b[39;00m\n\u001b[1;32m 40\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[0;32m---> 41\u001b[0m edge_layers \u001b[38;5;241m=\u001b[39m (layers[edge[\u001b[38;5;241m0\u001b[39m]], layers[edge[\u001b[38;5;241m1\u001b[39m]])\n\u001b[1;32m 42\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m edge_type, sinabs_edge \u001b[38;5;129;01min\u001b[39;00m valid_edges_map\u001b[38;5;241m.\u001b[39mitems():\n\u001b[1;32m 43\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m (\u001b[38;5;28mtype\u001b[39m(edge_layers[\u001b[38;5;241m0\u001b[39m]) \u001b[38;5;241m==\u001b[39m sinabs_edge[\u001b[38;5;241m0\u001b[39m]) \u001b[38;5;129;01mand\u001b[39;00m (\u001b[38;5;28mtype\u001b[39m(edge_layers[\u001b[38;5;241m1\u001b[39m]) \u001b[38;5;241m==\u001b[39m sinabs_edge[\u001b[38;5;241m1\u001b[39m]):\n", + "\u001b[0;31mKeyError\u001b[0m: 13" + ] + } + ], + "source": [ + "hw_model = DynapcnnNetworkGraph(\n", + " sinabs_model,\n", + " discretize=True,\n", + " input_shape=input_shape\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Network is valid\n" + ] + }, + { + "data": { + "text/plain": [ + "DynapcnnNetworkGraph()" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "hw_model.to(device=\"speck2edevkit:0\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "layer index: 0\n", + "layer modules: DynapcnnLayer(\n", + " (conv_layer): Conv2d(1, 20, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", + " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(635.), min_v_mem=Parameter containing:\n", + " tensor(-635.), batch_size=1, num_timesteps=-1)\n", + " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", + ")\n", + "layer destinations: [1]\n", + "assigned core: 0\n", + "\n", + "layer index: 1\n", + "layer modules: DynapcnnLayer(\n", + " (conv_layer): Conv2d(20, 32, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", + " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(11361.), min_v_mem=Parameter containing:\n", + " tensor(-11361.), batch_size=1, num_timesteps=-1)\n", + " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", + ")\n", + "layer destinations: [2]\n", + "assigned core: 3\n", + "\n", + "layer index: 2\n", + "layer modules: DynapcnnLayer(\n", + " (conv_layer): Conv2d(32, 128, kernel_size=(3, 3), stride=(1, 1), bias=False)\n", + " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(8621.), min_v_mem=Parameter containing:\n", + " tensor(-8621.), batch_size=1, num_timesteps=-1)\n", + " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", + ")\n", + "layer destinations: [3]\n", + "assigned core: 5\n", + "\n", + "layer index: 3\n", + "layer modules: DynapcnnLayer(\n", + " (conv_layer): Conv2d(128, 500, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(5747.), min_v_mem=Parameter containing:\n", + " tensor(-5747.), batch_size=1, num_timesteps=-1)\n", + ")\n", + "layer destinations: [4]\n", + "assigned core: 6\n", + "\n", + "layer index: 4\n", + "layer modules: DynapcnnLayer(\n", + " (conv_layer): Conv2d(500, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(2841.), min_v_mem=Parameter containing:\n", + " tensor(-2841.), batch_size=1, num_timesteps=-1)\n", + ")\n", + "layer destinations: []\n", + "assigned core: 1\n", + "\n" + ] + } + ], + "source": [ + "print(hw_model)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "speck-rescnn", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tests/test_nonsequential/DynapcnnNetworkGraph_from_NIRgraph_2.ipynb b/tests/test_nonsequential/DynapcnnNetworkGraph_from_NIRgraph_2.ipynb new file mode 100644 index 00000000..dddb75a3 --- /dev/null +++ b/tests/test_nonsequential/DynapcnnNetworkGraph_from_NIRgraph_2.ipynb @@ -0,0 +1,256 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "import torch.nn as nn\n", + "import networkx as nx\n", + "import matplotlib.pyplot as plt\n", + "from sinabs.from_torch import from_model\n", + "from sinabs.backend.dynapcnn import DynapcnnNetwork, DynapcnnNetworkGraph\n", + "import sinabs as snb" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "torch.manual_seed(0)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "channels = 1\n", + "height = 28\n", + "width = 28\n", + "\n", + "input_shape = (channels, height, width)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Network Module (pure Pytorch)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "class ANN(nn.Module):\n", + " def __init__(self) -> None:\n", + " super().__init__()\n", + "\n", + " self.con1 = nn.Conv2d(1, 20, 5, 1, bias=False)\n", + " self.rel1 = nn.ReLU()\n", + " self.pool1 = nn.AvgPool2d(2,2)\n", + "\n", + " self.conv2 = nn.Conv2d(20, 32, 5, 1, bias=False)\n", + " self.rel2 = nn.ReLU()\n", + " self.pool2 = nn.AvgPool2d(2,2)\n", + "\n", + " self.conv3 = nn.Conv2d(32, 128, 3, 1, bias=False)\n", + " self.rel3 = nn.ReLU()\n", + " self.pool3 = nn.AvgPool2d(2,2)\n", + "\n", + " self.flat = nn.Flatten()\n", + "\n", + " self.fc1 = nn.Linear(128, 500, bias=False)\n", + " self.rel4 = nn.ReLU()\n", + " self.fc2 = nn.Linear(500, 10, bias=False)\n", + "\n", + " def forward(self, x):\n", + " \n", + " con1_out = self.con1(x)\n", + " rel1_out = self.rel1(con1_out)\n", + " pool1_out = self.pool1(rel1_out)\n", + "\n", + " conv2_out = self.conv2(pool1_out)\n", + " rel2_out = self.rel2(conv2_out)\n", + " pool2_out = self.pool2(rel2_out)\n", + "\n", + " conv3_out = self.conv3(pool2_out)\n", + " rel3_out = self.rel3(conv3_out)\n", + " pool3_out = self.pool3(rel3_out)\n", + "\n", + " flat_out = self.flat(pool3_out)\n", + " \n", + " fc1_out = self.fc1(flat_out)\n", + " rel4_out = self.rel4(fc1_out)\n", + " fc2_out = self.fc2(rel4_out)\n", + "\n", + " return fc2_out" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "ann = ANN()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Sinabs Model" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "sinabs_model = from_model(ann, add_spiking_output=True, batch_size=1)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## DynapCNN Model" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "hw_model = DynapcnnNetworkGraph(\n", + " sinabs_model,\n", + " discretize=True,\n", + " input_shape=input_shape,\n", + " use_jit_tracer=False\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "# hw_model.to(device=\"speck2edevkit:0\")" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "layer index: 0\n", + "layer modules: DynapcnnLayer(\n", + " (conv_layer): Conv2d(1, 20, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", + " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(635.), min_v_mem=Parameter containing:\n", + " tensor(-635.), batch_size=1, num_timesteps=-1)\n", + " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", + ")\n", + "layer destinations: [1]\n", + "assigned core: -1\n", + "\n", + "layer index: 1\n", + "layer modules: DynapcnnLayer(\n", + " (conv_layer): Conv2d(20, 32, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", + " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(11361.), min_v_mem=Parameter containing:\n", + " tensor(-11361.), batch_size=1, num_timesteps=-1)\n", + " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", + ")\n", + "layer destinations: [2]\n", + "assigned core: -1\n", + "\n", + "layer index: 2\n", + "layer modules: DynapcnnLayer(\n", + " (conv_layer): Conv2d(32, 128, kernel_size=(3, 3), stride=(1, 1), bias=False)\n", + " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(8621.), min_v_mem=Parameter containing:\n", + " tensor(-8621.), batch_size=1, num_timesteps=-1)\n", + " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", + ")\n", + "layer destinations: [3]\n", + "assigned core: -1\n", + "\n", + "layer index: 3\n", + "layer modules: DynapcnnLayer(\n", + " (conv_layer): Conv2d(128, 500, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(5747.), min_v_mem=Parameter containing:\n", + " tensor(-5747.), batch_size=1, num_timesteps=-1)\n", + ")\n", + "layer destinations: [4]\n", + "assigned core: -1\n", + "\n", + "layer index: 4\n", + "layer modules: DynapcnnLayer(\n", + " (conv_layer): Conv2d(500, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(2841.), min_v_mem=Parameter containing:\n", + " tensor(-2841.), batch_size=1, num_timesteps=-1)\n", + ")\n", + "layer destinations: []\n", + "assigned core: -1\n", + "\n" + ] + } + ], + "source": [ + "print(hw_model)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "speck-rescnn", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tests/test_nonsequential/DynapcnnNetworkGraph_from_NIRgraph_3.ipynb b/tests/test_nonsequential/DynapcnnNetworkGraph_from_NIRgraph_3.ipynb index 04cc5f88..af555728 100644 --- a/tests/test_nonsequential/DynapcnnNetworkGraph_from_NIRgraph_3.ipynb +++ b/tests/test_nonsequential/DynapcnnNetworkGraph_from_NIRgraph_3.ipynb @@ -21,7 +21,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 2, @@ -50,7 +50,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Network Module (pure Pytorch)" + "## Network Module (Model #1)\n", + "\n", + "This one won't pass the `config_builder.validate_configuration` because the nodes inputing into the `Merge` layer don't have the same output dimension." ] }, { @@ -63,21 +65,21 @@ " def __init__(self) -> None:\n", " super().__init__()\n", "\n", - " self.conv1 = nn.Conv2d(1, 20, 5, 1, bias=False)\n", + " self.conv1 = nn.Conv2d(1, 10, 5, 1, bias=False)\n", " self.iaf1 = IAFSqueeze(batch_size=1)\n", " self.pool1 = nn.AvgPool2d(2,2)\n", "\n", - " self.conv2 = nn.Conv2d(20, 32, 5, 1, bias=False)\n", + " self.conv2 = nn.Conv2d(10, 20, 5, 1, bias=False)\n", " self.iaf2 = IAFSqueeze(batch_size=1)\n", " self.pool2 = nn.AvgPool2d(2,2)\n", "\n", - " self.conv3 = nn.Conv2d(32, 2, 3, 1, bias=False)\n", + " self.conv3 = nn.Conv2d(20, 1, 3, 1, bias=False)\n", " self.iaf3 = IAFSqueeze(batch_size=1)\n", " self.pool3 = nn.AvgPool2d(2,2)\n", "\n", " self.flat = nn.Flatten()\n", "\n", - " self.fc1 = nn.Linear(242, 500, bias=False)\n", + " self.fc1 = nn.Linear(121, 500, bias=False)\n", " self.iaf4 = IAFSqueeze(batch_size=1)\n", " self.fc2 = nn.Linear(500, 10, bias=False)\n", " self.iaf5 = IAFSqueeze(batch_size=1)\n", @@ -121,19 +123,7 @@ "cell_type": "code", "execution_count": 6, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "iaf1_out: torch.Size([1, 20, 24, 24])\n", - "pool2_out: torch.Size([1, 32, 4, 4])\n", - "added: torch.Size([1, 32, 24, 24])\n", - "pool3_out: torch.Size([1, 2, 11, 11])\n", - "flat_out: torch.Size([1, 242])\n" - ] - } - ], + "outputs": [], "source": [ "x = torch.randn((1, *input_shape))\n", "\n", @@ -164,6 +154,52 @@ "iaf5_out = snn.iaf5(fc2_out)" ] }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "print(f'DynapcnnLayer 0 [core 0]:')\n", + "print(f' input: {x.shape}')\n", + "con1_out = snn.conv1(x)\n", + "print(f' conv1: {con1_out.shape}')\n", + "iaf1_out = snn.iaf1(con1_out)\n", + "print(f' iaf1: {iaf1_out.shape}')\n", + "pool1_out = snn.pool1(iaf1_out)\n", + "print(f' pool1: {pool1_out.shape}\\n')\n", + "\n", + "print(f'DynapcnnLayer 1 [core 1]:')\n", + "print(f' input: {pool1_out.shape}')\n", + "conv2_out = snn.conv2(pool1_out)\n", + "print(f' conv2: {conv2_out.shape}')\n", + "iaf2_out = snn.iaf2(conv2_out)\n", + "print(f' iaf2: {iaf2_out.shape}')\n", + "pool2_out = snn.pool2(iaf2_out)\n", + "print(f' pool2: {pool2_out.shape}\\n')\n", + "\n", + "added = snn.adder(iaf1_out, pool2_out)\n", + "\n", + "print(f'DynapcnnLayer 2 [core 2]:')\n", + "print(f' input: {added.shape}')\n", + "conv3_out = snn.conv3(added)\n", + "print(f' conv3: {conv3_out.shape}')\n", + "iaf3_out = snn.iaf3(conv3_out)\n", + "print(f' iaf3: {iaf3_out.shape}')\n", + "pool3_out = snn.pool3(iaf3_out)\n", + "print(f' pool3: {pool3_out.shape}')\n", + "\n", + "flat_out = snn.flat(pool3_out)\n", + "\n", + "fc1_out = snn.fc1(flat_out)\n", + "iaf4_out = snn.iaf4(fc1_out)\n", + "print(f'DynapcnnLayer 3: {iaf4_out.shape}')\n", + "\n", + "fc2_out = snn.fc2(iaf4_out)\n", + "iaf5_out = snn.iaf5(fc2_out)\n", + "print(f'DynapcnnLayer 4: {iaf5_out.shape}')" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -173,9 +209,32 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 8, "metadata": {}, - "outputs": [], + "outputs": [ + { + "ename": "RuntimeError", + "evalue": "mat1 and mat2 shapes cannot be multiplied (1x625 and 16x500)", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[8], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m hw_model \u001b[38;5;241m=\u001b[39m DynapcnnNetworkGraph(\n\u001b[1;32m 2\u001b[0m snn,\n\u001b[1;32m 3\u001b[0m discretize\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m 4\u001b[0m input_shape\u001b[38;5;241m=\u001b[39minput_shape\n\u001b[1;32m 5\u001b[0m )\n", + "File \u001b[0;32m~/Documents/github/sinabs/sinabs/backend/dynapcnn/dynapcnn_network_graph.py:71\u001b[0m, in \u001b[0;36mDynapcnnNetworkGraph.__init__\u001b[0;34m(self, snn, input_shape, dvs_input, discretize)\u001b[0m\n\u001b[1;32m 67\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39minput_shape \u001b[38;5;241m=\u001b[39m input_shape\n\u001b[1;32m 69\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39minput_shape) \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m3\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124minfer_input_shape did not return 3-tuple\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m---> 71\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgraph_tracer \u001b[38;5;241m=\u001b[39m NIRtoDynapcnnNetworkGraph( \u001b[38;5;66;03m# computational graph from original PyTorch module.\u001b[39;00m\n\u001b[1;32m 72\u001b[0m snn,\n\u001b[1;32m 73\u001b[0m torch\u001b[38;5;241m.\u001b[39mrandn((\u001b[38;5;241m1\u001b[39m, \u001b[38;5;241m*\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39minput_shape))) \u001b[38;5;66;03m# needs the batch dimension. \u001b[39;00m\n\u001b[1;32m 75\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msinabs_edges, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msinabs_modules_map, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmerge_nodes \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mget_sinabs_edges_and_modules()\n\u001b[1;32m 77\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdynapcnn_layers, \\\n\u001b[1;32m 78\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnodes_to_dcnnl_map, \\\n\u001b[1;32m 79\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdcnnl_to_dcnnl_map \u001b[38;5;241m=\u001b[39m build_from_graph( \u001b[38;5;66;03m# build model from graph edges.\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 83\u001b[0m edges\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msinabs_edges,\n\u001b[1;32m 84\u001b[0m merge_nodes\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmerge_nodes)\n", + "File \u001b[0;32m~/Documents/github/sinabs/sinabs/backend/dynapcnn/NIRGraphExtractor.py:10\u001b[0m, in \u001b[0;36mNIRtoDynapcnnNetworkGraph.__init__\u001b[0;34m(self, spiking_model, dummy_input)\u001b[0m\n\u001b[1;32m 7\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m, spiking_model, dummy_input) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 8\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\" .\"\"\"\u001b[39;00m\n\u001b[0;32m---> 10\u001b[0m nir_graph \u001b[38;5;241m=\u001b[39m nirtorch\u001b[38;5;241m.\u001b[39mextract_torch_graph(spiking_model, dummy_input, model_name\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m)\u001b[38;5;241m.\u001b[39mignore_tensors()\n\u001b[1;32m 12\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39medges_list, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mname_2_indx_map \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mget_edges_from_nir(nir_graph)\n\u001b[1;32m 14\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmodules_map \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mget_named_modules(spiking_model)\n", + "File \u001b[0;32m~/.local/lib/python3.11/site-packages/nirtorch/graph.py:433\u001b[0m, in \u001b[0;36mextract_torch_graph\u001b[0;34m(model, sample_data, model_name, model_args)\u001b[0m\n\u001b[1;32m 412\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Extract computational graph between various modules in the model\u001b[39;00m\n\u001b[1;32m 413\u001b[0m \u001b[38;5;124;03mNOTE: This method is not capable of any compute happening outside of module\u001b[39;00m\n\u001b[1;32m 414\u001b[0m \u001b[38;5;124;03mdefinitions.\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 428\u001b[0m \u001b[38;5;124;03m Graph: A graph object representing the computational graph of the given model\u001b[39;00m\n\u001b[1;32m 429\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 430\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m GraphTracer(\n\u001b[1;32m 431\u001b[0m named_modules_map(model, model_name\u001b[38;5;241m=\u001b[39mmodel_name)\n\u001b[1;32m 432\u001b[0m ) \u001b[38;5;28;01mas\u001b[39;00m tracer, torch\u001b[38;5;241m.\u001b[39mno_grad():\n\u001b[0;32m--> 433\u001b[0m _ \u001b[38;5;241m=\u001b[39m model(sample_data, \u001b[38;5;241m*\u001b[39mmodel_args)\n\u001b[1;32m 435\u001b[0m \u001b[38;5;66;03m# HACK: The current graph is using copy-constructors, that detaches\u001b[39;00m\n\u001b[1;32m 436\u001b[0m \u001b[38;5;66;03m# the traced output_types from the original graph.\u001b[39;00m\n\u001b[1;32m 437\u001b[0m \u001b[38;5;66;03m# In the future, find a way to synchronize the two representations\u001b[39;00m\n\u001b[1;32m 438\u001b[0m tracer\u001b[38;5;241m.\u001b[39mgraph\u001b[38;5;241m.\u001b[39mmodule_output_types \u001b[38;5;241m=\u001b[39m tracer\u001b[38;5;241m.\u001b[39moutput_types\n", + "File \u001b[0;32m~/.local/lib/python3.11/site-packages/nirtorch/graph.py:344\u001b[0m, in \u001b[0;36mmodule_forward_wrapper..my_forward\u001b[0;34m(mod, *args, **kwargs)\u001b[0m\n\u001b[1;32m 343\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mmy_forward\u001b[39m(mod: nn\u001b[38;5;241m.\u001b[39mModule, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Any:\n\u001b[0;32m--> 344\u001b[0m out \u001b[38;5;241m=\u001b[39m _torch_module_call(mod, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m 346\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(out, \u001b[38;5;28mtuple\u001b[39m):\n\u001b[1;32m 347\u001b[0m out_tuple \u001b[38;5;241m=\u001b[39m (out[\u001b[38;5;241m0\u001b[39m],)\n", + "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/nn/modules/module.py:1511\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1509\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs) \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m 1510\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1511\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n", + "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/nn/modules/module.py:1520\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1515\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1516\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1517\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m 1518\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1519\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1520\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m forward_call(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m 1522\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 1523\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n", + "Cell \u001b[0;32mIn[4], line 89\u001b[0m, in \u001b[0;36mSNN.forward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m 85\u001b[0m iaf3_out \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39miaf3(conv3_out)\n\u001b[1;32m 87\u001b[0m flat_out \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mflat(iaf3_out)\n\u001b[0;32m---> 89\u001b[0m fc1_out \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfc1(flat_out)\n\u001b[1;32m 90\u001b[0m iaf4_out \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39miaf4(fc1_out)\n\u001b[1;32m 91\u001b[0m fc2_out \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfc2(iaf4_out)\n", + "File \u001b[0;32m~/.local/lib/python3.11/site-packages/nirtorch/graph.py:344\u001b[0m, in \u001b[0;36mmodule_forward_wrapper..my_forward\u001b[0;34m(mod, *args, **kwargs)\u001b[0m\n\u001b[1;32m 343\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mmy_forward\u001b[39m(mod: nn\u001b[38;5;241m.\u001b[39mModule, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Any:\n\u001b[0;32m--> 344\u001b[0m out \u001b[38;5;241m=\u001b[39m _torch_module_call(mod, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m 346\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(out, \u001b[38;5;28mtuple\u001b[39m):\n\u001b[1;32m 347\u001b[0m out_tuple \u001b[38;5;241m=\u001b[39m (out[\u001b[38;5;241m0\u001b[39m],)\n", + "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/nn/modules/module.py:1511\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1509\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs) \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m 1510\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1511\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n", + "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/nn/modules/module.py:1520\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1515\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1516\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1517\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m 1518\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1519\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1520\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m forward_call(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m 1522\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 1523\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n", + "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/nn/modules/linear.py:116\u001b[0m, in \u001b[0;36mLinear.forward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m 115\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;28minput\u001b[39m: Tensor) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tensor:\n\u001b[0;32m--> 116\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m F\u001b[38;5;241m.\u001b[39mlinear(\u001b[38;5;28minput\u001b[39m, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mweight, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbias)\n", + "\u001b[0;31mRuntimeError\u001b[0m: mat1 and mat2 shapes cannot be multiplied (1x625 and 16x500)" + ] + } + ], "source": [ "hw_model = DynapcnnNetworkGraph(\n", " snn,\n", @@ -186,79 +245,34 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "\n", - "layer index: 0\n", - "layer modules: DynapcnnLayer(\n", - " (conv_layer): Conv2d(1, 20, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", - " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", - " tensor(1.), min_v_mem=Parameter containing:\n", - " tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", - ")\n", - "layer destinations: [2, 1]\n", - "assigned core: -1\n", - "\n", - "layer index: 1\n", - "layer modules: DynapcnnLayer(\n", - " (conv_layer): Conv2d(20, 32, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", - " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", - " tensor(1.), min_v_mem=Parameter containing:\n", - " tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", - ")\n", - "layer destinations: [2]\n", - "assigned core: -1\n", - "\n", - "layer index: 2\n", - "layer modules: DynapcnnLayer(\n", - " (conv_layer): Conv2d(32, 2, kernel_size=(3, 3), stride=(1, 1), bias=False)\n", - " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", - " tensor(1.), min_v_mem=Parameter containing:\n", - " tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", - ")\n", - "layer destinations: [3]\n", - "assigned core: -1\n", - "\n", - "layer index: 3\n", - "layer modules: DynapcnnLayer(\n", - " (conv_layer): Conv2d(2, 500, kernel_size=(11, 11), stride=(1, 1), bias=False)\n", - " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", - " tensor(1.), min_v_mem=Parameter containing:\n", - " tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - ")\n", - "layer destinations: [4]\n", - "assigned core: -1\n", - "\n", - "layer index: 4\n", - "layer modules: DynapcnnLayer(\n", - " (conv_layer): Conv2d(500, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", - " tensor(1.), min_v_mem=Parameter containing:\n", - " tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - ")\n", - "layer destinations: []\n", - "assigned core: -1\n", + "Layer[2] input size x: 24 different than Layer[0] output size x: 24 pooling: 2\n", + "Layer[2] input size y: 24 different than Layer[0] output size y: 24 pooling: 2\n", + "Layer[2] input size x: 24 different than Layer[1] output size x: 8 pooling: 2\n", + "Layer[2] input size y: 24 different than Layer[1] output size y: 8 pooling: 2\n", "\n" ] + }, + { + "ename": "ValueError", + "evalue": "Generated config is not valid for speck2edevkit:0", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[9], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m hw_model\u001b[38;5;241m.\u001b[39mto(device\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mspeck2edevkit:0\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", + "File \u001b[0;32m~/Documents/github/sinabs/sinabs/backend/dynapcnn/dynapcnn_network_graph.py:148\u001b[0m, in \u001b[0;36mDynapcnnNetworkGraph.to\u001b[0;34m(self, device, chip_layers_ordering, monitor_layers, config_modifier, slow_clk_frequency)\u001b[0m\n\u001b[1;32m 144\u001b[0m device_name, _ \u001b[38;5;241m=\u001b[39m parse_device_id(device)\n\u001b[1;32m 146\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m device_name \u001b[38;5;129;01min\u001b[39;00m ChipFactory\u001b[38;5;241m.\u001b[39msupported_devices: \u001b[38;5;66;03m# pragma: no cover\u001b[39;00m\n\u001b[0;32m--> 148\u001b[0m config \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmake_config( \u001b[38;5;66;03m# generate config.\u001b[39;00m\n\u001b[1;32m 149\u001b[0m chip_layers_ordering\u001b[38;5;241m=\u001b[39mchip_layers_ordering,\n\u001b[1;32m 150\u001b[0m device\u001b[38;5;241m=\u001b[39mdevice,\n\u001b[1;32m 151\u001b[0m monitor_layers\u001b[38;5;241m=\u001b[39mmonitor_layers,\n\u001b[1;32m 152\u001b[0m config_modifier\u001b[38;5;241m=\u001b[39mconfig_modifier,\n\u001b[1;32m 153\u001b[0m )\n\u001b[1;32m 155\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msamna_device \u001b[38;5;241m=\u001b[39m open_device(device) \u001b[38;5;66;03m# apply configuration to device.\u001b[39;00m\n\u001b[1;32m 156\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msamna_device\u001b[38;5;241m.\u001b[39mget_model()\u001b[38;5;241m.\u001b[39mapply_configuration(config)\n", + "File \u001b[0;32m~/Documents/github/sinabs/sinabs/backend/dynapcnn/dynapcnn_network_graph.py:256\u001b[0m, in \u001b[0;36mDynapcnnNetworkGraph.make_config\u001b[0;34m(self, chip_layers_ordering, device, monitor_layers, config_modifier)\u001b[0m\n\u001b[1;32m 254\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m config\n\u001b[1;32m 255\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 256\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mGenerated config is not valid for \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mdevice\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n", + "\u001b[0;31mValueError\u001b[0m: Generated config is not valid for speck2edevkit:0" + ] } ], - "source": [ - "print(hw_model)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], "source": [ "hw_model.to(device=\"speck2edevkit:0\")" ] diff --git a/tests/test_nonsequential/DynapcnnNetworkGraph_from_NIRgraph_4.ipynb b/tests/test_nonsequential/DynapcnnNetworkGraph_from_NIRgraph_4.ipynb new file mode 100644 index 00000000..34a1ff86 --- /dev/null +++ b/tests/test_nonsequential/DynapcnnNetworkGraph_from_NIRgraph_4.ipynb @@ -0,0 +1,228 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "import torch.nn as nn\n", + "from sinabs.from_torch import from_model\n", + "from sinabs.backend.dynapcnn import DynapcnnNetworkGraph\n", + "from sinabs.layers import Merge, IAFSqueeze" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "torch.manual_seed(0)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "channels = 1\n", + "height = 28\n", + "width = 28\n", + "\n", + "input_shape = (channels, height, width)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Network Module (pure Pytorch)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "class SNN(nn.Module):\n", + " def __init__(self) -> None:\n", + " super().__init__()\n", + "\n", + " self.conv1 = nn.Conv2d(1, 30, 2, 1, bias=False)\n", + " self.iaf1 = IAFSqueeze(batch_size=1)\n", + " self.pool1 = nn.AvgPool2d(2,2)\n", + " self.pool1a = nn.AvgPool2d(4,4)\n", + "\n", + " self.conv2 = nn.Conv2d(30, 30, 2, 1, bias=False)\n", + " self.iaf2 = IAFSqueeze(batch_size=1)\n", + " self.pool2 = nn.AvgPool2d(2,2)\n", + "\n", + " self.conv3 = nn.Conv2d(30, 1, 3, 1, bias=False)\n", + " self.iaf3 = IAFSqueeze(batch_size=1)\n", + "\n", + " self.flat = nn.Flatten()\n", + "\n", + " self.fc1 = nn.Linear(16, 500, bias=False)\n", + " self.iaf4 = IAFSqueeze(batch_size=1)\n", + " self.fc2 = nn.Linear(500, 10, bias=False)\n", + " self.iaf5 = IAFSqueeze(batch_size=1)\n", + "\n", + " self.adder = Merge()\n", + "\n", + " def forward(self, x):\n", + " \n", + " con1_out = self.conv1(x)\n", + " iaf1_out = self.iaf1(con1_out)\n", + " pool1_out = self.pool1(iaf1_out)\n", + "\n", + " conv2_out = self.conv2(pool1_out)\n", + " iaf2_out = self.iaf2(conv2_out)\n", + " pool2_out = self.pool2(iaf2_out)\n", + "\n", + " conv3_out = self.conv3(self.adder(iaf1_out, pool2_out))\n", + " iaf3_out = self.iaf3(conv3_out)\n", + "\n", + " flat_out = self.flat(iaf3_out)\n", + " \n", + " fc1_out = self.fc1(flat_out)\n", + " iaf4_out = self.iaf4(fc1_out)\n", + " fc2_out = self.fc2(iaf4_out)\n", + " iaf5_out = self.iaf5(fc2_out)\n", + "\n", + " return iaf5_out" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "snn = SNN()" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "DynapcnnLayer 0 [core 0]: input: torch.Size([1, 1, 28, 28])\n", + " conv1: torch.Size([1, 30, 27, 27])\n", + " iaf1: torch.Size([1, 30, 27, 27])\n", + " pool1: torch.Size([1, 30, 13, 13])\n", + " pool1a: torch.Size([1, 30, 6, 6])\n", + "\n", + "DynapcnnLayer 1 [core 1]: input: torch.Size([1, 30, 13, 13])\n", + " conv2: torch.Size([1, 30, 12, 12])\n", + " iaf2: torch.Size([1, 30, 12, 12])\n", + " pool2: torch.Size([1, 30, 6, 6])\n", + "\n", + "DynapcnnLayer 2 [core 2]: input: torch.Size([1, 30, 6, 6]) [ Merge(pool1a, pool2) ]\n", + " conv3: torch.Size([1, 1, 4, 4])\n", + " iaf3: torch.Size([1, 1, 4, 4])\n", + "\n", + "DynapcnnLayer 3 [core ]: input: torch.Size([1, 16])\n", + " fc1: torch.Size([1, 500])\n", + " iaf4: torch.Size([1, 500])\n", + "\n", + "DynapcnnLayer 4 [core ]: input: torch.Size([1, 500])\n", + " fc2: torch.Size([1, 10])\n", + " iaf5: torch.Size([1, 10])\n", + "\n" + ] + } + ], + "source": [ + "x = torch.randn((1, *input_shape))\n", + "\n", + "print(f'DynapcnnLayer 0 [core 0]: input: {x.shape}')\n", + "con1_out = snn.conv1(x)\n", + "print(f' conv1: {con1_out.shape}')\n", + "iaf1_out = snn.iaf1(con1_out)\n", + "print(f' iaf1: {iaf1_out.shape}')\n", + "pool1_out = snn.pool1(iaf1_out)\n", + "print(f' pool1: {pool1_out.shape}')\n", + "pool1a_out = snn.pool1a(iaf1_out)\n", + "print(f' pool1a: {pool1a_out.shape}\\n')\n", + "\n", + "print(f'DynapcnnLayer 1 [core 1]: input: {pool1_out.shape}')\n", + "conv2_out = snn.conv2(pool1_out)\n", + "print(f' conv2: {conv2_out.shape}')\n", + "iaf2_out = snn.iaf2(conv2_out)\n", + "print(f' iaf2: {iaf2_out.shape}')\n", + "pool2_out = snn.pool2(iaf2_out)\n", + "print(f' pool2: {pool2_out.shape}\\n')\n", + "\n", + "added = snn.adder(pool1a_out, pool2_out)\n", + "\n", + "print(f'DynapcnnLayer 2 [core 2]: input: {added.shape} [ Merge(pool1a, pool2) ]')\n", + "conv3_out = snn.conv3(added)\n", + "print(f' conv3: {conv3_out.shape}')\n", + "iaf3_out = snn.iaf3(conv3_out)\n", + "print(f' iaf3: {iaf3_out.shape}\\n')\n", + "\n", + "flat_out = snn.flat(iaf3_out)\n", + "\n", + "print(f'DynapcnnLayer 3 [core ]: input: {flat_out.shape}')\n", + "fc1_out = snn.fc1(flat_out)\n", + "print(f' fc1: {fc1_out.shape}')\n", + "iaf4_out = snn.iaf4(fc1_out)\n", + "print(f' iaf4: {iaf4_out.shape}\\n')\n", + "\n", + "\n", + "print(f'DynapcnnLayer 4 [core ]: input: {iaf4_out.shape}')\n", + "fc2_out = snn.fc2(iaf4_out)\n", + "print(f' fc2: {fc2_out.shape}')\n", + "iaf5_out = snn.iaf5(fc2_out)\n", + "print(f' iaf5: {iaf5_out.shape}\\n')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## DynapCNN Model" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "speck-rescnn", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 55d7202804eefd141b719670318655e33eb4ab02 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Fri, 19 Apr 2024 12:45:39 +0200 Subject: [PATCH 025/379] mapping nodes I/O shapes --- sinabs/backend/dynapcnn/NIRGraphExtractor.py | 132 ++++++++++++++++--- 1 file changed, 114 insertions(+), 18 deletions(-) diff --git a/sinabs/backend/dynapcnn/NIRGraphExtractor.py b/sinabs/backend/dynapcnn/NIRGraphExtractor.py index cf3962f5..db61913c 100644 --- a/sinabs/backend/dynapcnn/NIRGraphExtractor.py +++ b/sinabs/backend/dynapcnn/NIRGraphExtractor.py @@ -2,6 +2,8 @@ import torch.nn as nn import nirtorch import copy +import sinabs +from typing import Tuple, Dict class NIRtoDynapcnnNetworkGraph(): def __init__(self, spiking_model, dummy_input) -> None: @@ -13,7 +15,7 @@ def __init__(self, spiking_model, dummy_input) -> None: self.modules_map = self.get_named_modules(spiking_model) - # self.get_modules_io() + self.nodes_io_shapes = self.get_nodes_io_shapes(dummy_input) def get_edges_from_nir(self, nir_graph): """ .""" @@ -53,36 +55,126 @@ def get_named_modules(self, model): return modules_map - def get_modules_io(self, input_dummy): + def get_nodes_io_shapes(self, input_dummy) -> Dict[int, Dict[str, torch.Size]]: """ .""" - modules_io_map = {} + nodes_io_map = {} + flagged_merge_nodes = {} for edge in self.edges_list: src = edge[0] trg = edge[1] - # pass input through source. - if src not in modules_io_map: - modules_io_map[src] = {'input': None, 'output': None} + if isinstance(self.modules_map[src], sinabs.layers.merge.Merge): + # At this point the output of Merge has to have been calculated. + + # pass input through target. + if trg not in nodes_io_map: + nodes_io_map[trg] = {'input': None, 'output': None} + + inp_node = self.find_input_to_node(trg) # find node generating the input to be used. + _input = nodes_io_map[inp_node]['output'] + + _output = self.modules_map[trg](_input) # forward input through the node. + + nodes_io_map[trg] = {'input': _input, 'output': _output} # save node's input/output. + + elif isinstance(self.modules_map[trg], sinabs.layers.merge.Merge): + # Merge requires two inputs: need to check if both of its inputs have been calculated. + if trg not in flagged_merge_nodes: + flagged_merge_nodes[trg] = {} + + args = self.find_merge_arguments(trg) + + for arg in args: + if arg in nodes_io_map: # one input to Merge has been computed. + flagged_merge_nodes[trg][arg] = nodes_io_map[arg] + + if len(flagged_merge_nodes[trg]) == 2: # both arguments to Merge have been computed. + if trg not in nodes_io_map: + nodes_io_map[trg] = {'input': None, 'output': None} + + _output = self.modules_map[trg]( + nodes_io_map[args[0]]['output'], + nodes_io_map[args[1]]['output']) + + _input = torch.max(torch.stack([ # Merge expands each input dim. into the max of that dim. between input tensors. + nodes_io_map[args[0]]['output'], + nodes_io_map[args[1]]['output']]), dim=0) + + nodes_io_map[trg]['input'] = _input.values + nodes_io_map[trg]['output'] = _output + + # pass input through source. + if src not in nodes_io_map: + nodes_io_map[src] = {'input': None, 'output': None} + + if src == 0: + _input = input_dummy # first node in the graph. + else: + inp_node = self.find_input_to_node(src) # find node generating the input to be used. + _input = nodes_io_map[inp_node]['output'] + + _output = self.modules_map[src](_input) # forward input through the node. + + nodes_io_map[src] = {'input': _input, 'output': _output} # save node's input/output. - if src == 0: - _input = input_dummy else: - inp_node = self.find_my_input_node(src) - _input = modules_io_map[inp_node]['output'] - - _output = self.modules_map[src](_input) - # pass input through target. - if trg not in modules_io_map: - modules_io_map[trg] = {'input': None, 'output': None} + # pass input through source. + if src not in nodes_io_map: + nodes_io_map[src] = {'input': None, 'output': None} + + if src == 0: + _input = input_dummy # first node in the graph. + else: + inp_node = self.find_input_to_node(src) # find node generating the input to be used. + _input = nodes_io_map[inp_node]['output'] + + _output = self.modules_map[src](_input) # forward input through the node. + + nodes_io_map[src] = {'input': _input, 'output': _output} # save node's input/output. - def find_my_input_node(self, node_idx): + # pass input through target. + if trg not in nodes_io_map: + nodes_io_map[trg] = {'input': None, 'output': None} + + inp_node = self.find_input_to_node(trg) # find node generating the input to be used. + _input = nodes_io_map[inp_node]['output'] + + _output = self.modules_map[trg](_input) # forward input through the node. + + nodes_io_map[trg] = {'input': _input, 'output': _output} # save node's input/output. + + for node, io in nodes_io_map.items(): + nodes_io_map[node]['input'] = io['input'].shape + nodes_io_map[node]['output'] = io['output'].shape + + return nodes_io_map + + + def find_input_to_node(self, node): + """ .""" for edge in self.edges_list: - if edge[1] == node_idx: + if edge[1] == node: return edge[0] return -1 + + def find_node_variable_name(self, node): + """ .""" + for key, val in self.name_2_indx_map.items(): + if val == node: + return key + return None + def find_merge_arguments(self, merge_node): + """ .""" + args = [] + for edge in self.edges_list: + if edge[1] == merge_node: + args.append(edge[0]) + if len(args) == 2: + break + return args def remove_ignored_nodes(self, default_ignored_nodes): """ Recreates the edges list based on layers that 'DynapcnnNetwork' will ignore. This @@ -132,4 +224,8 @@ def remove_ignored_nodes(self, default_ignored_nodes): for edge in parsed_edges: remapped_edges.append((remapped_nodes[edge[0]], remapped_nodes[edge[1]])) - return remapped_edges, remapped_nodes \ No newline at end of file + return remapped_edges, remapped_nodes + + def get_node_io_shapes(self, node) -> Tuple[torch.Size, torch.Size]: + """ .""" + return self.nodes_io_shapes[node]['input'], self.nodes_io_shapes[node]['output'] From cdffebab4da8b193fa7600f4c09b2632423ab0a4 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Fri, 19 Apr 2024 15:41:25 +0200 Subject: [PATCH 026/379] (WIP) refactoring nodes to DynapcnnLayer mapper: I/O shapes for each node added to the mapper --- sinabs/backend/dynapcnn/NIRGraphExtractor.py | 1 - .../dynapcnn/dynapcnn_network_graph.py | 43 ++++++++++++++++--- .../backend/dynapcnn/sinabs_edges_handler.py | 32 +++++++++----- 3 files changed, 57 insertions(+), 19 deletions(-) diff --git a/sinabs/backend/dynapcnn/NIRGraphExtractor.py b/sinabs/backend/dynapcnn/NIRGraphExtractor.py index db61913c..a331bbff 100644 --- a/sinabs/backend/dynapcnn/NIRGraphExtractor.py +++ b/sinabs/backend/dynapcnn/NIRGraphExtractor.py @@ -151,7 +151,6 @@ def get_nodes_io_shapes(self, input_dummy) -> Dict[int, Dict[str, torch.Size]]: return nodes_io_map - def find_input_to_node(self, node): """ .""" for edge in self.edges_list: diff --git a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py index 0a4cf626..2bd09558 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py @@ -62,26 +62,31 @@ def __init__( """ super().__init__() - dvs_input = False # TODO for now the graph part is not taking into consideration this. - self.dvs_input = dvs_input # check if dvs input is expected. + dvs_input = False # TODO for now the graph part is not taking into consideration this. + self.dvs_input = dvs_input # check if dvs input is expected. self.input_shape = input_shape assert len(self.input_shape) == 3, "infer_input_shape did not return 3-tuple" - self.graph_tracer = NIRtoDynapcnnNetworkGraph( # computational graph from original PyTorch module. + self.graph_tracer = NIRtoDynapcnnNetworkGraph( # computational graph from original PyTorch module. snn, - torch.randn((1, *self.input_shape))) # needs the batch dimension. + torch.randn((1, *self.input_shape))) # needs the batch dimension. - self.sinabs_edges, self.sinabs_modules_map, self.merge_nodes = self.get_sinabs_edges_and_modules() + self.sinabs_edges, \ + self.sinabs_modules_map, \ + self.merge_nodes, \ + self.nodes_name_remap = self.get_sinabs_edges_and_modules() self.dynapcnn_layers, \ self.nodes_to_dcnnl_map, \ - self.dcnnl_to_dcnnl_map = build_from_graph( # build model from graph edges. + self.dcnnl_to_dcnnl_map = build_from_graph( # build model from graph edges. discretize=discretize, layers=self.sinabs_modules_map, in_shape=self.input_shape, edges=self.sinabs_edges, merge_nodes=self.merge_nodes) + + self.populate_nodes_io() # update the I/O shapes for each layer in 'self.nodes_to_dcnnl_map'. def __str__(self): pretty_print = '' @@ -367,4 +372,28 @@ def get_sinabs_edges_and_modules(self) -> Tuple[List[Tuple[int, int]], Dict[int, for src in val['sources']: merge_data[val['merge_into']][src] = None - return edges_without_merge, sinabs_modules_map, merge_data \ No newline at end of file + return edges_without_merge, sinabs_modules_map, merge_data, remapped_nodes + + def populate_nodes_io(self): + """ .""" + + def find_original_node_name(name_mapper, node): + for orig_name, new_name in name_mapper.items(): + if new_name == node: + return orig_name + raise ValueError(f'Node {node} could not be found within the name remapping done by self.get_sinabs_edges_and_modules().') + + for dcnnl_idx, dcnnl_data in self.nodes_to_dcnnl_map.items(): + for node, node_data in dcnnl_data.items(): + if isinstance(node, int): # node dictionary with layer data. + orig_name = find_original_node_name(self.nodes_name_remap, node) + _in, _out = self.graph_tracer.get_node_io_shapes(orig_name) + node_data['input_shape'] = _in + node_data['output_shape'] = _out + + for dcnnl_idx, dcnnl_data in self.nodes_to_dcnnl_map.items(): + print(f'DynapcnnLayer-index {dcnnl_idx}') + for key, val in dcnnl_data.items(): + print(key, val['input_shape'], val['output_shape']) + print('\n') + \ No newline at end of file diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index 3e24e9a0..cc9ea4a5 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -67,12 +67,13 @@ def update_dynapcnnlayer_mapper(edge_type: int, edge: Tuple[int, int], mapper: d raise InvalidEdgeType(edge, edge_type) def init_xor_complete_new_dynapcnnlayer_blk(mapper: dict, edge: Tuple[int, int], layers: Dict[int, nn.Module]) -> None: - """ Incorporates nodes from either a '(conv, neuron)' or a '(linear, neuron)' edge. These are either initiating a (new) DynapcnnLayer - or completing a conv->neuron sequence (in the case the node for 'conv' as already been incorporated somewhere in 'mapper'). 'nn.Linear' layers - are converted into 'nn.Conv2d' by DynapcnnLayer. + """ Incorporates nodes from either a `(conv, neuron)` or a `(linear, neuron)` edge. These are either initiating a new `dict` mapping + into a future `DynapcnnLayer` or completing a `conv->neuron` sequence (in the case the node for `conv` as already been incorporated + somewhere in `mapper`). Obs.: `nn.Linear` layers are converted into `nn.Conv2d` by `DynapcnnLayer`. """ matched = False dynapcnnlayer_indx = 0 + for indx, dynapcnnlayer in mapper.items(): # see if 'edge[0]' exists in a DynapcnnLayer block. for node, _ in dynapcnnlayer.items(): if node == edge[0]: @@ -80,18 +81,22 @@ def init_xor_complete_new_dynapcnnlayer_blk(mapper: dict, edge: Tuple[int, int], matched = True break if matched: # 'edge[0]' found: 'edge[1]' belongs to its DynapcnnLayer block. - mapper[dynapcnnlayer_indx][edge[1]] = layers[edge[1]] + mapper[dynapcnnlayer_indx][edge[1]] = {'layer': layers[edge[1]], 'input_shape': None, 'output_shape': None} break if not matched: # 'edge[0]' not found: start new DynapcnnLayer block. dynapcnnlayer_indx = 0 for indx, _ in mapper.items(): dynapcnnlayer_indx += 1 - mapper[dynapcnnlayer_indx] = {edge[0]: layers[edge[0]], edge[1]: layers[edge[1]]} + mapper[dynapcnnlayer_indx] = { + edge[0]: {'layer': layers[edge[0]], 'input_shape': None, 'output_shape': None}, + edge[1]: {'layer': layers[edge[1]], 'input_shape': None, 'output_shape': None} + } def connect_dynapcnnlayer_blks(mapper: dict, edge: Tuple[int, int], layers: Dict[int, nn.Module]) -> None: - """ Incorporates nodes from either a '(neuron, conv)/(neuron, lin)' or '(pool, conv)/(pool, lin)' edge. These represent connections between an existing - DynapcnnLayer in 'mapper' and a new one yet to be represented in 'mapper'. 'nn.Linear' layers are converted into 'nn.Conv2d' by DynapcnnLayer. + """ Incorporates nodes from either a `(neuron, conv)/(neuron, lin)` or `(pool, conv)/(pool, lin)` edge. These represent connections between an existing + `dict` in `mapper` that will be mapped into a `DynapcnnLayer` and a new one yet to be represented in `mapper`. Obs.: `nn.Linear` layers are converted + into `nn.Conv2d` by `DynapcnnLayer`. """ if not is_initialized_node(edge[1], mapper): dynapcnnlayer_indx = 0 @@ -105,17 +110,22 @@ def connect_dynapcnnlayer_blks(mapper: dict, edge: Tuple[int, int], layers: Dict if matched: break if matched: - mapper[dynapcnnlayer_indx] = {edge[1]: layers[edge[1]]} # 'edge[1]' starts new DynapcnnLayer block as 'indx+1'. + while (dynapcnnlayer_indx in mapper): + dynapcnnlayer_indx += 1 + mapper[dynapcnnlayer_indx] = { # 'edge[1]' starts new DynapcnnLayer block. + edge[1]: {'layer': layers[edge[1]], 'input_shape': None, 'output_shape': None}} else: raise UnmatchedNode(edge, node) def add_pool_to_dynapcnnlayer_blk(mapper: dict, edge: Tuple[int, int], layers: Dict[int, nn.Module]) -> None: - """ Incorporating a '(neuron, pool)' edge. Node 'pool' has to be part of an already existing DynapcnnLayer in 'mapper'. """ + """ Incorporating a `(neuron, pool)` edge. Node `pool` has to be part of an already existing `dict` mapping into a `DynapcnnLaye` in `mapper`. + """ matched = False for indx, dynapcnnlayer in mapper.items(): for node, _ in dynapcnnlayer.items(): if node == edge[0]: - dynapcnnlayer[edge[1]] = layers[edge[1]] # 'edge[0]' is a neuron layer inputing into pooling layer 'edge[1]'. + dynapcnnlayer[edge[1]] = { # 'edge[0]' is a neuron layer inputing into pooling layer 'edge[1]'. + 'layer': layers[edge[1]], 'input_shape': None, 'output_shape': None} matched = True break if matched: @@ -224,7 +234,7 @@ def merge_handler(sinabs_edges: List[Tuple[int, int]], sinabs_modules_map: Dict[ for edge in edges: # removing edges connection from/to merging layers from the computational graph. src = edge[0] trg = edge[1] - # print('> ', edge) + if src in merge_nodes: # edge (`Merge`, trg) is not necessary for later DynapcnnLayer creation. pass From 29bdc3ce9ce2694b252ea768ae0027c7a41857ef Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Fri, 19 Apr 2024 19:29:36 +0200 Subject: [PATCH 027/379] (WIP) writting new DynapcnnLayer class --- sinabs/backend/dynapcnn/dynapcnn_layer_new.py | 208 +++++++++++++++++ .../dynapcnn/dynapcnn_network_graph.py | 43 ++-- .../backend/dynapcnn/sinabs_edges_handler.py | 9 +- sinabs/backend/dynapcnn/utils.py | 211 +++++++++--------- ...DynapcnnNetworkGraph_from_NIRgraph_4.ipynb | 147 +++++++----- 5 files changed, 434 insertions(+), 184 deletions(-) create mode 100644 sinabs/backend/dynapcnn/dynapcnn_layer_new.py diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer_new.py b/sinabs/backend/dynapcnn/dynapcnn_layer_new.py new file mode 100644 index 00000000..71993b53 --- /dev/null +++ b/sinabs/backend/dynapcnn/dynapcnn_layer_new.py @@ -0,0 +1,208 @@ +from copy import deepcopy +from typing import Dict, Optional, Tuple, Union +from warnings import warn + +import numpy as np +import torch +from torch import nn + +import sinabs.activation +import sinabs.layers as sl + +from .discretize import discretize_conv_spike_ +from .dvs_layer import expand_to_pair + + +class DynapcnnLayer(nn.Module): + """Create a DynapcnnLayer object representing a dynapcnn layer. + + Requires a convolutional layer, a sinabs spiking layer and an optional + pooling value. The layers are used in the order conv -> spike -> pool. + + Parameters + ---------- + conv: torch.nn.Conv2d or torch.nn.Linear + Convolutional or linear layer (linear will be converted to convolutional) + spk: sinabs.layers.IAFSqueeze + Sinabs IAF layer + in_shape: tuple of int + The input shape, needed to create dynapcnn configs if the network does not + contain an input layer. Convention: (features, height, width) + pool: int or None + Integer representing the sum pooling kernel and stride. If `None`, no + pooling will be applied. + discretize: bool + Whether to discretize parameters. + rescale_weights: int + Layer weights will be divided by this value. + """ + + def __init__( + self, + dcnnl_data: dict, + discretize: bool, + rescale_weights: int = 1, # TODO remove. + ): + super().__init__() + """ + ... + + TODO + 1) need to figure out how to apply 'rescale_weights' since there are more than two poolings. + 2) currently there's no way the forward would work since there are more than two poolings. + """ + + #### NEW WAY ####################################################################### + + conv = None + conv_node_id = None + + spk = None + spk_node_id = None + + pool = [] + pool_node_id = [] + + for key, value in dcnnl_data.items(): + if isinstance(key, int): + # value has data pertaining a node (torch/sinabs layer). + if isinstance(value['layer'], sl.IAFSqueeze): + spk = value['layer'] + spk_node_id = key + elif isinstance(value['layer'], nn.Linear) or isinstance(value['layer'], nn.Conv2d): + conv = value['layer'] + conv_node_id = key + elif isinstance(value['layer'], sl.SumPool2d): + pool.append(value['layer']) + pool_node_id.append(key) + else: + raise ValueError(f'Node {key} has not valid layer associated with it.') + + if not conv: + raise ValueError(f'Convolution layer not present.') + + if not spk: + raise ValueError(f'Spiking layer not present.') + + spk = deepcopy(spk) + if spk.is_state_initialised(): + spk.v_mem = spk.v_mem.data.unsqueeze(-1).unsqueeze(-1) # expand dims. + + if isinstance(conv, nn.Linear): + conv = self._convert_linear_to_conv(conv, dcnnl_data[conv_node_id]) + else: + conv = deepcopy(conv) + + # TODO have to consider that two poolings might be projecting to this conv (these lines of code are deprecated). + # this weight rescale comes from the node projecting into this 'conv' node. + if rescale_weights != 1: + # this has to be done after copying but before discretizing + conv.weight.data = (conv.weight / rescale_weights).clone().detach() + + # int conversion is done while writing the config. + if discretize: + conv, spk = discretize_conv_spike_(conv, spk, to_int=False) + + # consolidate layers. + self.conv_layer = conv + self.spk_layer = spk + self.pool_layer = [] + if len(pool) != 0: + for plyr in pool: + if plyr.kernel_size[0] != plyr.kernel_size[1]: + raise ValueError("Only square kernels are supported") + self.pool_layer.append(deepcopy(plyr)) + + def _convert_linear_to_conv(self, lin: nn.Linear, layer_data: dict) -> nn.Conv2d: + """Convert Linear layer to Conv2d. + + Parameters + ---------- + lin: nn.Linear + Linear layer to be converted + + Returns + ------- + nn.Conv2d + Convolutional layer equivalent to `lin`. + """ + + # TODO linear layers are convered to conv so the input shapes in the original + # mapper has to be updated accordingly. Another problem is that the input shape + # after a flatten layer won't match the fact that there's a convolution output before + # the flatten. + input_shape = tuple(list(layer_data['input_shape'])[1:-1]) # removing the batch dimension. + + in_chan, in_h, in_w = input_shape + + if lin.in_features != in_chan * in_h * in_w: + raise ValueError("Shapes don't match.") + + layer = nn.Conv2d( + in_channels=in_chan, + kernel_size=(in_h, in_w), + out_channels=lin.out_features, + padding=0, + bias=lin.bias is not None, + ) + + if lin.bias is not None: + layer.bias.data = lin.bias.data.clone().detach() + + layer.weight.data = ( + lin.weight.data.clone() + .detach() + .reshape((lin.out_features, in_chan, in_h, in_w)) + ) + + return layer + + def zero_grad(self, set_to_none: bool = False) -> None: + return self.spk_layer.zero_grad(set_to_none) + + ######################################################################################## + + def summary(self) -> dict: # TODO deprecated. + return { + "pool": ( + None if self.pool_layer is None else list(self.pool_layer.kernel_size) + ), + "kernel": list(self.conv_layer.weight.data.shape), + "neuron": self.get_neuron_shape(), + } + + def memory_summary(self): # TODO deprecated. + """Computes the amount of memory required for each of the components. Note that this is not + necessarily the same as the number of parameters due to some architecture design + constraints. + + .. math:: + + K_{MT} = c \\cdot 2^{\\lceil \\log_2\\left(k_xk_y\\right) \\rceil + \\lceil \\log_2\\left(f\\right) \\rceil} + + .. math:: + + N_{MT} = f \\cdot 2^{ \\lceil \\log_2\\left(f_y\\right) \\rceil + \\lceil \\log_2\\left(f_x\\right) \\rceil } + + Returns + ------- + A dictionary with keys kernel, neuron and bias and the corresponding memory sizes + """ + summary = self.summary() + f, c, h, w = summary["kernel"] + f, neuron_height, neuron_width = self.get_neuron_shape() + + return { + "kernel": c * pow(2, np.ceil(np.log2(h * w)) + np.ceil(np.log2(f))), + "neuron": f + * pow(2, np.ceil(np.log2(neuron_height)) + np.ceil(np.log2(neuron_width))), + "bias": 0 if self.conv_layer.bias is None else len(self.conv_layer.bias), + } + + def forward(self, x): # TODO deprecated. + """Torch forward pass.""" + x = self.conv_layer(x) + x = self.spk_layer(x) + if self.pool_layer is not None: + x = self.pool_layer(x) + return x diff --git a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py index 2bd09558..009ca58b 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py @@ -20,6 +20,7 @@ from .utils import ( DEFAULT_IGNORED_LAYER_TYPES, build_from_graph, + build_nodes_to_dcnnl_map, parse_device_id, ) @@ -74,19 +75,28 @@ def __init__( self.sinabs_edges, \ self.sinabs_modules_map, \ - self.merge_nodes, \ self.nodes_name_remap = self.get_sinabs_edges_and_modules() + + self.nodes_to_dcnnl_map = build_nodes_to_dcnnl_map( + layers=self.sinabs_modules_map, + edges=self.sinabs_edges) + + # update the I/O shapes for each layer in 'self.nodes_to_dcnnl_map'. + self.populate_nodes_io() - self.dynapcnn_layers, \ - self.nodes_to_dcnnl_map, \ - self.dcnnl_to_dcnnl_map = build_from_graph( # build model from graph edges. + self.dynapcnn_layers = build_from_graph( # build model from graph edges. discretize=discretize, - layers=self.sinabs_modules_map, - in_shape=self.input_shape, edges=self.sinabs_edges, - merge_nodes=self.merge_nodes) + nodes_to_dcnnl_map=self.nodes_to_dcnnl_map) - self.populate_nodes_io() # update the I/O shapes for each layer in 'self.nodes_to_dcnnl_map'. + for dcnnl_idx, dcnnl_data in self.nodes_to_dcnnl_map.items(): + print(f'DynapcnnLayer-index {dcnnl_idx}') + for key, val in dcnnl_data.items(): + if isinstance(key, int): + print(key, val['layer'], val['input_shape']) + else: + print(key, val) + print('\n') def __str__(self): pretty_print = '' @@ -366,13 +376,7 @@ def get_sinabs_edges_and_modules(self) -> Tuple[List[Tuple[int, int]], Dict[int, edges_without_merge, merge_nodes = merge_handler(sinabs_edges, sinabs_modules_map) # bypass merging layers to connect the nodes involved in them directly to the node where the merge happens. - merge_data = {} - for key, val in merge_nodes.items(): - merge_data[val['merge_into']] = {} - for src in val['sources']: - merge_data[val['merge_into']][src] = None - - return edges_without_merge, sinabs_modules_map, merge_data, remapped_nodes + return edges_without_merge, sinabs_modules_map, remapped_nodes def populate_nodes_io(self): """ .""" @@ -389,11 +393,4 @@ def find_original_node_name(name_mapper, node): orig_name = find_original_node_name(self.nodes_name_remap, node) _in, _out = self.graph_tracer.get_node_io_shapes(orig_name) node_data['input_shape'] = _in - node_data['output_shape'] = _out - - for dcnnl_idx, dcnnl_data in self.nodes_to_dcnnl_map.items(): - print(f'DynapcnnLayer-index {dcnnl_idx}') - for key, val in dcnnl_data.items(): - print(key, val['input_shape'], val['output_shape']) - print('\n') - \ No newline at end of file + node_data['output_shape'] = _out \ No newline at end of file diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index cc9ea4a5..b84e24fe 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -180,9 +180,12 @@ def get_dynapcnnlayers_destinations(layers: Dict[int, nn.Module], edges: List[Tu else: raise InvalidLayerLoop(source_layer, destination_layer) - del used_layer_edges - - return dynapcnnlayers_destinations_map + for dcnnl_idx, destinations in dynapcnnlayers_destinations_map.items(): # TODO document the 'rescale_factor' better. + mapper[dcnnl_idx]['destinations'] = destinations + mapper[dcnnl_idx]['destinations_rescale_factor'] = {} # needed for when SumPool is built. + + for dest in destinations: + mapper[dcnnl_idx]['destinations_rescale_factor'][dest] = 1 def get_dynapcnnlayer_index(node: int, mapper: dict) -> int: """ Returns the DynapcnnLayer index to which 'node' belongs to. """ diff --git a/sinabs/backend/dynapcnn/utils.py b/sinabs/backend/dynapcnn/utils.py index e1527887..b04cff75 100644 --- a/sinabs/backend/dynapcnn/utils.py +++ b/sinabs/backend/dynapcnn/utils.py @@ -9,7 +9,8 @@ from .crop2d import Crop2d from .dvs_layer import DVSLayer, expand_to_pair -from .dynapcnn_layer import DynapcnnLayer +#from .dynapcnn_layer import DynapcnnLayer +from .dynapcnn_layer_new import DynapcnnLayer from .exceptions import InputConfigurationError, MissingLayer, UnexpectedLayer, WrongModuleCount, WrongPoolingModule from .flipdims import FlipDims @@ -547,35 +548,8 @@ def extend_readout_layer(model: "DynapcnnNetwork") -> "DynapcnnNetwork": ) # run a forward pass to initialize the new weights and last IAF return model -def build_from_graph( - discretize: bool, - layers: Dict[int, nn.Module], - in_shape: Tuple[int, int, int], - edges: List[Tuple[int, int]], - merge_nodes: dict) -> Tuple[List[DynapcnnLayer], Dict[int, Dict[int, nn.Module]], Dict[int, List[int]]]: - """ Parses each edge of a 'sinabs_mode.spiking_model' computational graph. Each node (layer) is assigned to a - DynapcnnLayer object. The target destination of each DynapcnnLayer is computed via edges connecting nodes in - different DynapcnnLayer objects. - - Parameters - ---------- - discretize: If True, discretize the parameters and thresholds. This is needed for uploading weights to dynapcnn. - Set to False only for testing purposes. - layers : a dictionary containing the nodes of the graph as `key` and their associated module as `value`. - in_shape : Tuple describing the input to the very first layer (batch_size, hight, width). - edges : List of edges returned by 'DynapcnnNetworkGraph.get_sinabs_edges()'. - - Returns - ---------- - dynapcnn_layers : A dictionary containing DynapcnnLayer objects and a list with their destinations. - nodes_to_dcnnl_map: Sets of layers that comprise a DynapcnnLayer. - key [int]: DynapcnnLayer index. - value [dict]: node index as 'key' and its module as 'value'. - dcnnl_to_dcnnl_map: List of destinations for each DynapcnnLayer in 'dynapcnn_layers'. - key [int]: index of a DynapcnnLayer. - value [list]: indexes of DynapcnnLayer a layer targets. - """ - +def build_nodes_to_dcnnl_map(layers: Dict[int, nn.Module], edges: List[Tuple[int, int]]): + """ .""" # @TODO the graph extraction is not yet considering DVS input. # dvs_layer, lyr_indx_next, rescale_factor = construct_dvs_layer( @@ -585,37 +559,52 @@ def build_from_graph( # dvs_input=False) dvs_layer = None - rescale_factor = 1 nodes_to_dcnnl_map = {} # mapper from nodes to sets of layers that populate a DynapcnnLayer. if dvs_layer is not None: - pass # @TODO the graph extraction is not yet considering DVS input. + pass # TODO the graph extraction is not yet considering DVS input. else: for edge in edges: process_edge( # figure out to which (future) DynapcnnLayer each node will belong to. layers, edge, nodes_to_dcnnl_map) - # look for edges between connecting nodes in different (future) DynapcnnLayer. - dcnnl_to_dcnnl_map = get_dynapcnnlayers_destinations(layers, edges, nodes_to_dcnnl_map) + get_dynapcnnlayers_destinations( # look for edges between connecting nodes in different (future) DynapcnnLayer. + layers, edges, nodes_to_dcnnl_map) + + return nodes_to_dcnnl_map + +def build_from_graph( + discretize: bool, + edges: List[Tuple[int, int]], + nodes_to_dcnnl_map: dict) -> dict: + """ Parses each edge of a 'sinabs_mode.spiking_model' computational graph. Each node (layer) is assigned to a + DynapcnnLayer object. The target destination of each DynapcnnLayer is computed via edges connecting nodes in + different DynapcnnLayer objects. - # turn sets of layers into DynapcnnLayer objects. - dynapcnn_layers = construct_dynapcnnlayers_from_mapper( - discretize, nodes_to_dcnnl_map, dcnnl_to_dcnnl_map, in_shape, rescale_factor, merge_nodes) + Parameters + ---------- + ... + + Returns + ---------- + ... + """ + + + dynapcnn_layers = construct_dynapcnnlayers_from_mapper( # turn sets of layers into DynapcnnLayer objects. + discretize, nodes_to_dcnnl_map, edges) for idx, layer_data in dynapcnn_layers.items(): if 'core_idx' not in layer_data: layer_data['core_idx'] = -1 # a DynapcnnLayer gets assigned a core index when 'DynapcnnNetworkGraph.to()' is called. - return dynapcnn_layers, nodes_to_dcnnl_map, dcnnl_to_dcnnl_map + return None def construct_dynapcnnlayers_from_mapper( discretize: bool, nodes_to_dcnnl_map: dict, - dcnnl_to_dcnnl_map: dict, - input_shape: Union[Tuple[int, int], Tuple[int, int, int]], - rescale_factor: int, - merge_nodes: dict = None) -> Dict[int, Dict[DynapcnnLayer, List]]: + edges: List[Tuple[int, int]]) -> Dict[int, Dict[DynapcnnLayer, List]]: """ Consumes a dictionaries containing sets of layers to be used to populate a DynapcnnLayer object. Parameters @@ -629,84 +618,94 @@ def construct_dynapcnnlayers_from_mapper( dynapcnn_layers = {} - for dynapcnnl_indx, layer_modules in nodes_to_dcnnl_map.items(): - dynapcnnlayer, input_shape, rescale_factor = construct_dynapcnnlayer( - discretize, layer_modules, dynapcnnl_indx, input_shape, rescale_factor, merge_nodes) + for dpcnnl_idx, dcnnl_data in nodes_to_dcnnl_map.items(): + dynapcnnlayer = construct_dynapcnnlayer( + discretize, dcnnl_data, edges, nodes_to_dcnnl_map) - dynapcnn_layers[dynapcnnl_indx] = { + print('> ', dpcnnl_idx) + + dynapcnn_layers[dpcnnl_idx] = { 'layer': dynapcnnlayer, - 'destinations': dcnnl_to_dcnnl_map[dynapcnnl_indx] + 'destinations': nodes_to_dcnnl_map[dpcnnl_idx]['destinations'] } return dynapcnn_layers def construct_dynapcnnlayer( discretize: bool, - layer_modules: dict, - layer_index: int, - input_shape: Union[Tuple[int, int], Tuple[int, int, int]], - rescale_factor: int, - merge_nodes: dict = None) -> Tuple[DynapcnnLayer, Union[Tuple[int, int], Tuple[int, int, int]], int]: - """ Extract the modules (layers) in a dictionary and uses them to instantiate a DynapcnnLayer object. """ - lyr_conv = None - lyr_spk = None - lyr_pool = None - rescale_factor_after_pooling = 1 - - iterator = iter(layer_modules.items()) # 'next(iterator)' returns the node id in the computational graph and the layer (nn.Module) associated with it. - - if len(layer_modules) == 3: # there's a pooling layer. - _, lyr_conv = next(iterator) - _, lyr_spk = next(iterator) - _, _pool = next(iterator) - - lyr_pool, rescale_factor_after_pooling = build_SumPool2d(_pool) - - elif len(layer_modules) == 2: # there's only a conv layer folowed by a neuron layer. - _, lyr_conv = next(iterator) - _, lyr_spk = next(iterator) + dcnnl_data: dict, + edges: List[Tuple[int, int]], + nodes_to_dcnnl_map: dict, + layer_index = None, # TODO remove. + ) -> DynapcnnLayer: + """ Extract the modules (layers) in a dictionary and uses them to instantiate a DynapcnnLayer object. - else: - raise WrongModuleCount(layer_index, len(layer_modules)) - - ################ TODO HACKY STUFF MAKE IT BETTER ################## - for node_idx, _ in layer_modules.items(): - if node_idx in merge_nodes: - input_shape = get_input_shape_from_merge(merge_nodes[node_idx]) - break + Parameters + ---------- + dcnnl_data: contains the nodes to be merged into a DynapcnnLayer, their I/O shapes and the index of the other DynapcnnLayers to + be set as destinations. + """ - ############################################################################### - + # convert all AvgPool2d in 'dcnnl_data' into SumPool2d. + convert_Avg_to_Sum_pooling(dcnnl_data, edges, nodes_to_dcnnl_map) + + # TODO 'rescale_weight' information is inside 'dcnnl_data' but it is not yet being used. + # TODO the input shapes work fine when processing conv layers but once we reach linear layers + # their input shapes has to come from the conv layers projecting to the node. Currently also a + # linear layer after a flatten has the input shape coming out of the flatten but it needs to be + # the conv output shape before the flatten. A linear layer coming after a previous linear layer + # converted into conv needs to have its input shape updated based on this conversion. + + # instantiate a DynapcnnLayer from the data in 'dcnnl_data'. dynapcnnlayer = DynapcnnLayer( - conv = lyr_conv, - spk = lyr_spk, - pool = lyr_pool, - in_shape = input_shape, - discretize = discretize, - rescale_weights = rescale_factor, - ) + dcnnl_data = dcnnl_data, + discretize = discretize + ) - ################ TODO HACKY STUFF MAKE IT BETTER ################## - if len(merge_nodes) != 0: - for merge_target, merge_data in merge_nodes.items(): - for node_idx, _ in layer_modules.items(): - if node_idx in merge_data and merge_data[node_idx] == None: - if isinstance(layer_modules[node_idx], sinabs.layers.iaf.IAFSqueeze): - merge_data[node_idx] = dynapcnnlayer.get_neuron_shape() - else: - merge_data[node_idx] = dynapcnnlayer.get_output_shape() - ################################################################### - - return dynapcnnlayer, dynapcnnlayer.get_output_shape(), rescale_factor_after_pooling + return dynapcnnlayer -def get_input_shape_from_merge(merge_data): - ################ TODO HACKY STUFF MAKE IT BETTER ################## - input_shape = [0, 0, 0] - for key, val in merge_data.items(): - for i in range(len(val)): - input_shape[i] = val[i] if val[i] > input_shape[i] else input_shape[i] - return (input_shape[0], input_shape[1], input_shape[2]) +def convert_Avg_to_Sum_pooling(dcnnl_data: dict, edges: list, nodes_to_dcnnl_map: dict): + """ Converts every `AvgPool2d` node within `dcnnl_data` into a `SumPool2d` and update their respective `rescale_factor` (to + be used when creating the `DynapcnnLayer` instance for this layer's destinations). + Parameters + ---------- + dcnnl_data: ... + edges: ... + nodes_to_dcnnl_map: ... + """ + for key, value in dcnnl_data.items(): + if isinstance(key, int): + # accessing the node 'key's dictionary. + + if isinstance(value['layer'], nn.AvgPool2d): + # convert AvgPool2d into SumPool2d. + lyr_pool, rescale_factor = build_SumPool2d(value['layer']) + + # turn avg into sum pool. + value['layer'] = lyr_pool + + # find which node 'key' will target. + for edge in edges: + if edge[0] == key: + # target within DynapcnnLayer index 'trg_dcnnl_idx'. + trg_dcnnl_idx = find_nodes_dcnnl_idx(edge[1], nodes_to_dcnnl_map) + + # update the rescale factor for the target of node 'key'. + dcnnl_data['destinations_rescale_factor'][trg_dcnnl_idx] = rescale_factor + +def find_nodes_dcnnl_idx(node, nodes_to_dcnnl_map): + """ .""" + for dcnnl_idx, dcnnl_data in nodes_to_dcnnl_map.items(): + for key, value in dcnnl_data.items(): + if isinstance(key, int): + # 'key' is a node. + if key == node: + # node belongs to DynapcnnLayer index 'dcnnl_idx'. + return dcnnl_idx + + # this exception should never happen. + raise ValueError(f'Node {node} is not part of any dictionary mapping into a DynapcnnLayer.') def build_SumPool2d(module: nn.AvgPool2d) -> Tuple[sl.SumPool2d, int]: """ Converts a 'nn.AvgPool2d' into a 'sl.SumPool2d' layer. """ diff --git a/tests/test_nonsequential/DynapcnnNetworkGraph_from_NIRgraph_4.ipynb b/tests/test_nonsequential/DynapcnnNetworkGraph_from_NIRgraph_4.ipynb index 34a1ff86..da71104d 100644 --- a/tests/test_nonsequential/DynapcnnNetworkGraph_from_NIRgraph_4.ipynb +++ b/tests/test_nonsequential/DynapcnnNetworkGraph_from_NIRgraph_4.ipynb @@ -21,7 +21,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 2, @@ -63,24 +63,25 @@ " def __init__(self) -> None:\n", " super().__init__()\n", "\n", - " self.conv1 = nn.Conv2d(1, 30, 2, 1, bias=False)\n", - " self.iaf1 = IAFSqueeze(batch_size=1)\n", - " self.pool1 = nn.AvgPool2d(2,2)\n", - " self.pool1a = nn.AvgPool2d(4,4)\n", + " self.conv1 = nn.Conv2d(1, 30, 2, 1, bias=False) # node 0\n", + " self.iaf1 = IAFSqueeze(batch_size=1) # node 1\n", + " self.pool1 = nn.AvgPool2d(2,2) # node 2\n", + " self.pool1a = nn.AvgPool2d(4,4) # node 3\n", "\n", - " self.conv2 = nn.Conv2d(30, 30, 2, 1, bias=False)\n", - " self.iaf2 = IAFSqueeze(batch_size=1)\n", - " self.pool2 = nn.AvgPool2d(2,2)\n", + " self.conv2 = nn.Conv2d(30, 30, 2, 1, bias=False)# node 4\n", + " self.iaf2 = IAFSqueeze(batch_size=1) # node 6\n", + " self.pool2 = nn.AvgPool2d(2,2) # node 7\n", "\n", - " self.conv3 = nn.Conv2d(30, 1, 3, 1, bias=False)\n", - " self.iaf3 = IAFSqueeze(batch_size=1)\n", + " self.conv3 = nn.Conv2d(30, 1, 3, 1, bias=False) # node 8\n", + " self.iaf3 = IAFSqueeze(batch_size=1) # node 9\n", "\n", " self.flat = nn.Flatten()\n", "\n", - " self.fc1 = nn.Linear(16, 500, bias=False)\n", - " self.iaf4 = IAFSqueeze(batch_size=1)\n", - " self.fc2 = nn.Linear(500, 10, bias=False)\n", - " self.iaf5 = IAFSqueeze(batch_size=1)\n", + " self.fc1 = nn.Linear(16, 500, bias=False) # node 10\n", + " self.iaf4 = IAFSqueeze(batch_size=1) # node 11\n", + " \n", + " self.fc2 = nn.Linear(500, 10, bias=False) # node 12\n", + " self.iaf5 = IAFSqueeze(batch_size=1) # node 13\n", "\n", " self.adder = Merge()\n", "\n", @@ -89,12 +90,13 @@ " con1_out = self.conv1(x)\n", " iaf1_out = self.iaf1(con1_out)\n", " pool1_out = self.pool1(iaf1_out)\n", + " pool1a_out = self.pool1a(iaf1_out)\n", "\n", " conv2_out = self.conv2(pool1_out)\n", " iaf2_out = self.iaf2(conv2_out)\n", " pool2_out = self.pool2(iaf2_out)\n", "\n", - " conv3_out = self.conv3(self.adder(iaf1_out, pool2_out))\n", + " conv3_out = self.conv3(self.adder(pool1a_out, pool2_out))\n", " iaf3_out = self.iaf3(conv3_out)\n", "\n", " flat_out = self.flat(iaf3_out)\n", @@ -125,28 +127,28 @@ "name": "stdout", "output_type": "stream", "text": [ - "DynapcnnLayer 0 [core 0]: input: torch.Size([1, 1, 28, 28])\n", - " conv1: torch.Size([1, 30, 27, 27])\n", - " iaf1: torch.Size([1, 30, 27, 27])\n", - " pool1: torch.Size([1, 30, 13, 13])\n", - " pool1a: torch.Size([1, 30, 6, 6])\n", + "DynapcnnLayer 0 [core 0]: ... torch.Size([1, 1, 28, 28])\n", + " conv1: [1, 30, 27, 27]\n", + " iaf1: [1, 30, 27, 27]\n", + " pool1: [1, 30, 13, 13]\n", + " pool1a: [1, 30, 6, 6]\n", "\n", - "DynapcnnLayer 1 [core 1]: input: torch.Size([1, 30, 13, 13])\n", - " conv2: torch.Size([1, 30, 12, 12])\n", - " iaf2: torch.Size([1, 30, 12, 12])\n", - " pool2: torch.Size([1, 30, 6, 6])\n", + "DynapcnnLayer 1 [core 1]: ... [1, 30, 13, 13]\n", + " conv2: [1, 30, 12, 12]\n", + " iaf2: [1, 30, 12, 12]\n", + " pool2: [1, 30, 6, 6]\n", "\n", - "DynapcnnLayer 2 [core 2]: input: torch.Size([1, 30, 6, 6]) [ Merge(pool1a, pool2) ]\n", - " conv3: torch.Size([1, 1, 4, 4])\n", - " iaf3: torch.Size([1, 1, 4, 4])\n", + "DynapcnnLayer 2 [core 2]: ... [1, 30, 6, 6] [ Merge(pool1a, pool2) ]\n", + " conv3: [1, 1, 4, 4]\n", + " iaf3: [1, 1, 4, 4]\n", "\n", - "DynapcnnLayer 3 [core ]: input: torch.Size([1, 16])\n", - " fc1: torch.Size([1, 500])\n", - " iaf4: torch.Size([1, 500])\n", + "DynapcnnLayer 3 [core ]: ... [1, 16]\n", + " fc1: [1, 500]\n", + " iaf4: [1, 500]\n", "\n", - "DynapcnnLayer 4 [core ]: input: torch.Size([1, 500])\n", - " fc2: torch.Size([1, 10])\n", - " iaf5: torch.Size([1, 10])\n", + "DynapcnnLayer 4 [core ]: ... [1, 500]\n", + " fc2: [1, 10]\n", + " iaf5: [1, 10]\n", "\n" ] } @@ -154,53 +156,94 @@ "source": [ "x = torch.randn((1, *input_shape))\n", "\n", - "print(f'DynapcnnLayer 0 [core 0]: input: {x.shape}')\n", + "print(f'DynapcnnLayer 0 [core 0]: ... {(x.shape)}')\n", "con1_out = snn.conv1(x)\n", - "print(f' conv1: {con1_out.shape}')\n", + "print(f' conv1: {list(con1_out.shape)}')\n", "iaf1_out = snn.iaf1(con1_out)\n", - "print(f' iaf1: {iaf1_out.shape}')\n", + "print(f' iaf1: {list(iaf1_out.shape)}')\n", "pool1_out = snn.pool1(iaf1_out)\n", - "print(f' pool1: {pool1_out.shape}')\n", + "print(f' pool1: {list(pool1_out.shape)}')\n", "pool1a_out = snn.pool1a(iaf1_out)\n", - "print(f' pool1a: {pool1a_out.shape}\\n')\n", + "print(f' pool1a: {list(pool1a_out.shape)}\\n')\n", "\n", - "print(f'DynapcnnLayer 1 [core 1]: input: {pool1_out.shape}')\n", + "print(f'DynapcnnLayer 1 [core 1]: ... {list(pool1_out.shape)}')\n", "conv2_out = snn.conv2(pool1_out)\n", - "print(f' conv2: {conv2_out.shape}')\n", + "print(f' conv2: {list(conv2_out.shape)}')\n", "iaf2_out = snn.iaf2(conv2_out)\n", - "print(f' iaf2: {iaf2_out.shape}')\n", + "print(f' iaf2: {list(iaf2_out.shape)}')\n", "pool2_out = snn.pool2(iaf2_out)\n", - "print(f' pool2: {pool2_out.shape}\\n')\n", + "print(f' pool2: {list(pool2_out.shape)}\\n')\n", "\n", "added = snn.adder(pool1a_out, pool2_out)\n", "\n", - "print(f'DynapcnnLayer 2 [core 2]: input: {added.shape} [ Merge(pool1a, pool2) ]')\n", + "print(f'DynapcnnLayer 2 [core 2]: ... {list(added.shape)} [ Merge(pool1a, pool2) ]')\n", "conv3_out = snn.conv3(added)\n", - "print(f' conv3: {conv3_out.shape}')\n", + "print(f' conv3: {list(conv3_out.shape)}')\n", "iaf3_out = snn.iaf3(conv3_out)\n", - "print(f' iaf3: {iaf3_out.shape}\\n')\n", + "print(f' iaf3: {list(iaf3_out.shape)}\\n')\n", "\n", "flat_out = snn.flat(iaf3_out)\n", "\n", - "print(f'DynapcnnLayer 3 [core ]: input: {flat_out.shape}')\n", + "print(f'DynapcnnLayer 3 [core ]: ... {list(flat_out.shape)}')\n", "fc1_out = snn.fc1(flat_out)\n", - "print(f' fc1: {fc1_out.shape}')\n", + "print(f' fc1: {list(fc1_out.shape)}')\n", "iaf4_out = snn.iaf4(fc1_out)\n", - "print(f' iaf4: {iaf4_out.shape}\\n')\n", + "print(f' iaf4: {list(iaf4_out.shape)}\\n')\n", "\n", "\n", - "print(f'DynapcnnLayer 4 [core ]: input: {iaf4_out.shape}')\n", + "print(f'DynapcnnLayer 4 [core ]: ... {list(iaf4_out.shape)}')\n", "fc2_out = snn.fc2(iaf4_out)\n", - "print(f' fc2: {fc2_out.shape}')\n", + "print(f' fc2: {list(fc2_out.shape)}')\n", "iaf5_out = snn.iaf5(fc2_out)\n", - "print(f' iaf5: {iaf5_out.shape}\\n')" + "print(f' iaf5: {list(iaf5_out.shape)}\\n')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## DynapCNN Model" + "## DynapcnnNetwork" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "> 0\n", + "> 1\n", + "> 2\n", + "torch.Size([1, 16]) 16\n" + ] + }, + { + "ename": "ValueError", + "evalue": "not enough values to unpack (expected 3, got 0)", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[7], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m hw_model \u001b[38;5;241m=\u001b[39m \u001b[43mDynapcnnNetworkGraph\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 2\u001b[0m \u001b[43m \u001b[49m\u001b[43msnn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 3\u001b[0m \u001b[43m \u001b[49m\u001b[43mdiscretize\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 4\u001b[0m \u001b[43m \u001b[49m\u001b[43minput_shape\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43minput_shape\u001b[49m\n\u001b[1;32m 5\u001b[0m \u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/Github/sinabs/sinabs/backend/dynapcnn/dynapcnn_network_graph.py:87\u001b[0m, in \u001b[0;36mDynapcnnNetworkGraph.__init__\u001b[0;34m(self, snn, input_shape, dvs_input, discretize)\u001b[0m\n\u001b[1;32m 84\u001b[0m \u001b[38;5;66;03m# update the I/O shapes for each layer in 'self.nodes_to_dcnnl_map'.\u001b[39;00m\n\u001b[1;32m 85\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpopulate_nodes_io()\n\u001b[0;32m---> 87\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdynapcnn_layers \u001b[38;5;241m=\u001b[39m \u001b[43mbuild_from_graph\u001b[49m\u001b[43m(\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;66;43;03m# build model from graph edges.\u001b[39;49;00m\n\u001b[1;32m 88\u001b[0m \u001b[43m \u001b[49m\u001b[43mdiscretize\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdiscretize\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 89\u001b[0m \u001b[43m \u001b[49m\u001b[43medges\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msinabs_edges\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 90\u001b[0m \u001b[43m \u001b[49m\u001b[43mnodes_to_dcnnl_map\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mnodes_to_dcnnl_map\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 92\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m dcnnl_idx, dcnnl_data \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnodes_to_dcnnl_map\u001b[38;5;241m.\u001b[39mitems():\n\u001b[1;32m 93\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mDynapcnnLayer-index \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mdcnnl_idx\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m)\n", + "File \u001b[0;32m~/Github/sinabs/sinabs/backend/dynapcnn/utils.py:595\u001b[0m, in \u001b[0;36mbuild_from_graph\u001b[0;34m(discretize, edges, nodes_to_dcnnl_map)\u001b[0m\n\u001b[1;32m 577\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mbuild_from_graph\u001b[39m(\n\u001b[1;32m 578\u001b[0m discretize: \u001b[38;5;28mbool\u001b[39m,\n\u001b[1;32m 579\u001b[0m edges: List[Tuple[\u001b[38;5;28mint\u001b[39m, \u001b[38;5;28mint\u001b[39m]],\n\u001b[1;32m 580\u001b[0m nodes_to_dcnnl_map: \u001b[38;5;28mdict\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28mdict\u001b[39m:\n\u001b[1;32m 581\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\" Parses each edge of a 'sinabs_mode.spiking_model' computational graph. Each node (layer) is assigned to a \u001b[39;00m\n\u001b[1;32m 582\u001b[0m \u001b[38;5;124;03m DynapcnnLayer object. The target destination of each DynapcnnLayer is computed via edges connecting nodes in \u001b[39;00m\n\u001b[1;32m 583\u001b[0m \u001b[38;5;124;03m different DynapcnnLayer objects.\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 591\u001b[0m \u001b[38;5;124;03m ...\u001b[39;00m\n\u001b[1;32m 592\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[0;32m--> 595\u001b[0m dynapcnn_layers \u001b[38;5;241m=\u001b[39m \u001b[43mconstruct_dynapcnnlayers_from_mapper\u001b[49m\u001b[43m(\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;66;43;03m# turn sets of layers into DynapcnnLayer objects.\u001b[39;49;00m\n\u001b[1;32m 596\u001b[0m \u001b[43m \u001b[49m\u001b[43mdiscretize\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnodes_to_dcnnl_map\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43medges\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 598\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m idx, layer_data \u001b[38;5;129;01min\u001b[39;00m dynapcnn_layers\u001b[38;5;241m.\u001b[39mitems():\n\u001b[1;32m 599\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mcore_idx\u001b[39m\u001b[38;5;124m'\u001b[39m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m layer_data:\n", + "File \u001b[0;32m~/Github/sinabs/sinabs/backend/dynapcnn/utils.py:622\u001b[0m, in \u001b[0;36mconstruct_dynapcnnlayers_from_mapper\u001b[0;34m(discretize, nodes_to_dcnnl_map, edges)\u001b[0m\n\u001b[1;32m 619\u001b[0m dynapcnn_layers \u001b[38;5;241m=\u001b[39m {}\n\u001b[1;32m 621\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m dpcnnl_idx, dcnnl_data \u001b[38;5;129;01min\u001b[39;00m nodes_to_dcnnl_map\u001b[38;5;241m.\u001b[39mitems():\n\u001b[0;32m--> 622\u001b[0m dynapcnnlayer \u001b[38;5;241m=\u001b[39m \u001b[43mconstruct_dynapcnnlayer\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 623\u001b[0m \u001b[43m \u001b[49m\u001b[43mdiscretize\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdcnnl_data\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43medges\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnodes_to_dcnnl_map\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 625\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m> \u001b[39m\u001b[38;5;124m'\u001b[39m, dpcnnl_idx)\n\u001b[1;32m 627\u001b[0m dynapcnn_layers[dpcnnl_idx] \u001b[38;5;241m=\u001b[39m {\n\u001b[1;32m 628\u001b[0m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mlayer\u001b[39m\u001b[38;5;124m'\u001b[39m: dynapcnnlayer, \n\u001b[1;32m 629\u001b[0m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mdestinations\u001b[39m\u001b[38;5;124m'\u001b[39m: nodes_to_dcnnl_map[dpcnnl_idx][\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mdestinations\u001b[39m\u001b[38;5;124m'\u001b[39m]\n\u001b[1;32m 630\u001b[0m }\n", + "File \u001b[0;32m~/Github/sinabs/sinabs/backend/dynapcnn/utils.py:653\u001b[0m, in \u001b[0;36mconstruct_dynapcnnlayer\u001b[0;34m(discretize, dcnnl_data, edges, nodes_to_dcnnl_map, layer_index)\u001b[0m\n\u001b[1;32m 650\u001b[0m convert_Avg_to_Sum_pooling(dcnnl_data, edges, nodes_to_dcnnl_map)\n\u001b[1;32m 652\u001b[0m \u001b[38;5;66;03m# instantiate a DynapcnnLayer from the data in 'dcnnl_data'.\u001b[39;00m\n\u001b[0;32m--> 653\u001b[0m dynapcnnlayer \u001b[38;5;241m=\u001b[39m \u001b[43mDynapcnnLayer\u001b[49m\u001b[43m(\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;66;43;03m# TODO 'rescale_weight' information is inside 'dcnnl_data' but it is not yet being used.\u001b[39;49;00m\n\u001b[1;32m 654\u001b[0m \u001b[43m \u001b[49m\u001b[43mdcnnl_data\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mdcnnl_data\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 655\u001b[0m \u001b[43m \u001b[49m\u001b[43mdiscretize\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mdiscretize\u001b[49m\n\u001b[1;32m 656\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 658\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m dynapcnnlayer\n", + "File \u001b[0;32m~/Github/sinabs/sinabs/backend/dynapcnn/dynapcnn_layer_new.py:92\u001b[0m, in \u001b[0;36mDynapcnnLayer.__init__\u001b[0;34m(self, dcnnl_data, discretize, rescale_weights)\u001b[0m\n\u001b[1;32m 89\u001b[0m spk\u001b[38;5;241m.\u001b[39mv_mem \u001b[38;5;241m=\u001b[39m spk\u001b[38;5;241m.\u001b[39mv_mem\u001b[38;5;241m.\u001b[39mdata\u001b[38;5;241m.\u001b[39munsqueeze(\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m)\u001b[38;5;241m.\u001b[39munsqueeze(\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m) \u001b[38;5;66;03m# expand dims.\u001b[39;00m\n\u001b[1;32m 91\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(conv, nn\u001b[38;5;241m.\u001b[39mLinear):\n\u001b[0;32m---> 92\u001b[0m conv \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_convert_linear_to_conv\u001b[49m\u001b[43m(\u001b[49m\u001b[43mconv\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdcnnl_data\u001b[49m\u001b[43m[\u001b[49m\u001b[43mconv_node_id\u001b[49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 93\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 94\u001b[0m conv \u001b[38;5;241m=\u001b[39m deepcopy(conv)\n", + "File \u001b[0;32m~/Github/sinabs/sinabs/backend/dynapcnn/dynapcnn_layer_new.py:134\u001b[0m, in \u001b[0;36mDynapcnnLayer._convert_linear_to_conv\u001b[0;34m(self, lin, layer_data)\u001b[0m\n\u001b[1;32m 130\u001b[0m \u001b[38;5;28mprint\u001b[39m(layer_data[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124minput_shape\u001b[39m\u001b[38;5;124m'\u001b[39m], lin\u001b[38;5;241m.\u001b[39min_features)\n\u001b[1;32m 132\u001b[0m input_shape \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mtuple\u001b[39m(\u001b[38;5;28mlist\u001b[39m(layer_data[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124minput_shape\u001b[39m\u001b[38;5;124m'\u001b[39m])[\u001b[38;5;241m1\u001b[39m:\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m]) \u001b[38;5;66;03m# removing the batch dimension.\u001b[39;00m\n\u001b[0;32m--> 134\u001b[0m in_chan, in_h, in_w \u001b[38;5;241m=\u001b[39m input_shape\n\u001b[1;32m 136\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m lin\u001b[38;5;241m.\u001b[39min_features \u001b[38;5;241m!=\u001b[39m in_chan \u001b[38;5;241m*\u001b[39m in_h \u001b[38;5;241m*\u001b[39m in_w:\n\u001b[1;32m 137\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mShapes don\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mt match.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", + "\u001b[0;31mValueError\u001b[0m: not enough values to unpack (expected 3, got 0)" + ] + } + ], + "source": [ + "hw_model = DynapcnnNetworkGraph(\n", + " snn,\n", + " discretize=True,\n", + " input_shape=input_shape\n", + ")" ] } ], From d23e3b45fe3dc021dd69f33e11ea1ec779f7997f Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Sat, 20 Apr 2024 14:40:48 +0200 Subject: [PATCH 028/379] need some sort of 'Diverge' layer to allow a DynapcnnLayer to have multiple poolings --- .../test_nonsequential/CNN_branching_2.ipynb | 248 ++++++++++++++++++ 1 file changed, 248 insertions(+) create mode 100644 tests/test_nonsequential/CNN_branching_2.ipynb diff --git a/tests/test_nonsequential/CNN_branching_2.ipynb b/tests/test_nonsequential/CNN_branching_2.ipynb new file mode 100644 index 00000000..4adb2f5e --- /dev/null +++ b/tests/test_nonsequential/CNN_branching_2.ipynb @@ -0,0 +1,248 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "import torch.nn as nn" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "torch.manual_seed(0)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "channels = 1\n", + "height = 28\n", + "width = 28\n", + "batch = 1\n", + "\n", + "input_shape = (batch, channels, height, width)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "x = torch.randn(input_shape)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Network Module" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "class Diverge(nn.Module):\n", + " def __init__(self, pre_pooling_block, pooling) -> None:\n", + " super().__init__()\n", + "\n", + " self.pre_pooling_block = pre_pooling_block\n", + " self.pooling = pooling\n", + "\n", + " def forward(self, x):\n", + " x = self.pre_pooling_block(x)\n", + " x = self.pooling(x)\n", + " return x" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "class CNNLayerBlock(nn.Module):\n", + " def __init__(self, pre_activation, activation, pooling = None) -> None:\n", + " super().__init__()\n", + "\n", + " self.pre_activation = pre_activation\n", + " self.activation = activation\n", + " self.pooling = pooling\n", + "\n", + " def forward(self, x):\n", + " x = self.pre_activation(x)\n", + " x = self.activation(x)\n", + " if self.pooling:\n", + " x = self.pooling(x)\n", + " return x" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "class CNN(nn.Module):\n", + " def __init__(self) -> None:\n", + " super().__init__()\n", + "\n", + " # CNNLayerBlock 1 - this block wants to branch out to target two different layers.\n", + "\n", + " self.cnnlblk_1 = CNNLayerBlock(\n", + " pre_activation = nn.Conv2d(1, 30, 2, 1, bias=False), \n", + " activation = nn.ReLU()\n", + " )\n", + " \n", + " self.diverge_1a = Diverge( # pooling 1a.\n", + " pre_pooling_block = self.cnnlblk_1,\n", + " pooling = nn.AvgPool2d(2,2)\n", + " )\n", + " \n", + " self.diverge_1b = Diverge( # pooling 1b.\n", + " pre_pooling_block = self.cnnlblk_1, \n", + " pooling = nn.AvgPool2d(4,4)\n", + " )\n", + "\n", + " # CNNLayerBlock 2.\n", + "\n", + " self.cnnlblk_2 = CNNLayerBlock(\n", + " pre_activation = nn.Conv2d(30, 30, 2, 1, bias=False),\n", + " activation = nn.ReLU(),\n", + " pooling = nn.AvgPool2d(2,2)\n", + " )\n", + "\n", + " # CNNLayerBlock 3.\n", + "\n", + " self.cnnlblk_3 = CNNLayerBlock(\n", + " pre_activation = nn.Conv2d(30, 1, 3, 1, bias=False), \n", + " activation = nn.ReLU()\n", + " )\n", + "\n", + " # DynapcnnLayer 4. \n", + "\n", + " self.cnnlblk_4 = CNNLayerBlock(\n", + " pre_activation = nn.Linear(16, 500, bias=False), \n", + " activation = nn.ReLU()\n", + " )\n", + " \n", + " # DynapcnnLayer 5.\n", + "\n", + " self.cnnlblk_5 = CNNLayerBlock(\n", + " pre_activation = nn.Linear(500, 10, bias=False), \n", + " activation = nn.ReLU()\n", + " )\n", + "\n", + " # 'support' layers\n", + "\n", + " self.flat = nn.Flatten()\n", + "\n", + "\n", + " def forward(self, x):\n", + " # CNNLayerBlock 1.\n", + " div1a_out = self.diverge_1a(x)\n", + " div1b_out = self.diverge_1b(x)\n", + " \n", + " # CNNLayerBlock 2.\n", + " blk2_out = self.cnnlblk_2(div1a_out)\n", + "\n", + " # CNNLayerBlock 3.\n", + " blk3_out = self.cnnlblk_3(blk2_out + div1b_out)\n", + "\n", + " # CNNLayerBlock 4.\n", + " blk4_out = self.cnnlblk_4(self.flat(blk3_out))\n", + "\n", + " # CNNLayerBlock 5.\n", + " blk5_out = self.cnnlblk_5(blk4_out)\n", + "\n", + " return blk5_out" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "cnn = CNN()" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "torch.Size([1, 30, 6, 6]) torch.Size([1, 30, 6, 6])\n" + ] + } + ], + "source": [ + "# CNNLayerBlock 1.\n", + "div1a_out = cnn.diverge_1a(x)\n", + "div1b_out = cnn.diverge_1b(x)\n", + "\n", + "# CNNLayerBlock 2.\n", + "blk2_out = cnn.cnnlblk_2(div1a_out)\n", + "\n", + "# CNNLayerBlock 3.\n", + "print(blk2_out.shape, div1b_out.shape)\n", + "blk3_out = cnn.cnnlblk_3(blk2_out + div1b_out)\n", + "\n", + "# CNNLayerBlock 4.\n", + "blk4_out = cnn.cnnlblk_4(cnn.flat(blk3_out))\n", + "\n", + "# CNNLayerBlock 5.\n", + "blk5_out = cnn.cnnlblk_5(blk4_out)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "speck-rescnn", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From cd72fc471045ac03c9d093194dc27ee78b51e6ba Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Mon, 22 Apr 2024 11:29:06 +0200 Subject: [PATCH 029/379] populating nodes' I/O shapes from the list of final edges (self.sinabs_edges) --- .../dynapcnn/dynapcnn_network_graph.py | 78 ++++++++++++++----- 1 file changed, 59 insertions(+), 19 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py index 009ca58b..583530b3 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py @@ -63,13 +63,16 @@ def __init__( """ super().__init__() - dvs_input = False # TODO for now the graph part is not taking into consideration this. - self.dvs_input = dvs_input # check if dvs input is expected. + # TODO for now the graph part is not taking into consideration this. + # check if dvs input is expected. + dvs_input = False + self.dvs_input = dvs_input self.input_shape = input_shape assert len(self.input_shape) == 3, "infer_input_shape did not return 3-tuple" - self.graph_tracer = NIRtoDynapcnnNetworkGraph( # computational graph from original PyTorch module. + # computational graph from original PyTorch module. + self.graph_tracer = NIRtoDynapcnnNetworkGraph( snn, torch.randn((1, *self.input_shape))) # needs the batch dimension. @@ -84,19 +87,20 @@ def __init__( # update the I/O shapes for each layer in 'self.nodes_to_dcnnl_map'. self.populate_nodes_io() - self.dynapcnn_layers = build_from_graph( # build model from graph edges. - discretize=discretize, - edges=self.sinabs_edges, - nodes_to_dcnnl_map=self.nodes_to_dcnnl_map) + # # build model from graph edges. + # self.dynapcnn_layers = build_from_graph( + # discretize=discretize, + # edges=self.sinabs_edges, + # nodes_to_dcnnl_map=self.nodes_to_dcnnl_map) - for dcnnl_idx, dcnnl_data in self.nodes_to_dcnnl_map.items(): - print(f'DynapcnnLayer-index {dcnnl_idx}') - for key, val in dcnnl_data.items(): - if isinstance(key, int): - print(key, val['layer'], val['input_shape']) - else: - print(key, val) - print('\n') + # for dcnnl_idx, dcnnl_data in self.nodes_to_dcnnl_map.items(): + # print(f'DynapcnnLayer-index {dcnnl_idx}') + # for key, val in dcnnl_data.items(): + # if isinstance(key, int): + # print(key, val['layer'], val['input_shape']) + # else: + # print(key, val) + # print('\n') def __str__(self): pretty_print = '' @@ -381,16 +385,52 @@ def get_sinabs_edges_and_modules(self) -> Tuple[List[Tuple[int, int]], Dict[int, def populate_nodes_io(self): """ .""" + for edge in self.sinabs_edges: + print(edge) + print('\n') + def find_original_node_name(name_mapper, node): for orig_name, new_name in name_mapper.items(): if new_name == node: return orig_name raise ValueError(f'Node {node} could not be found within the name remapping done by self.get_sinabs_edges_and_modules().') - + + def find_my_input(edges_list, node): + for edge in edges_list: + if edge[1] == node: + # TODO nodes originally receiving input from merge will appear twice in the list of edges, one + # edge per input to the merge layer. For now both inputs to a `Merge` have the same dimensions + # necessarily so this works for now but later will have to be revised. + return edge[0] + raise ValueError(f'Node {node} is not receiving input from any other node in the graph.') + + # access the I/O shapes for each node in `self.sinabs_edges` from the original graph in `self.graph_tracer`. for dcnnl_idx, dcnnl_data in self.nodes_to_dcnnl_map.items(): for node, node_data in dcnnl_data.items(): - if isinstance(node, int): # node dictionary with layer data. + # node dictionary with layer data. + if isinstance(node, int): + # some nodes might have been renamed (e.g. after droppping a `nn.Flatten`), so find how node was originally named. orig_name = find_original_node_name(self.nodes_name_remap, node) _in, _out = self.graph_tracer.get_node_io_shapes(orig_name) - node_data['input_shape'] = _in - node_data['output_shape'] = _out \ No newline at end of file + + # update node I/O shape in the mapper (drop batch dimension). + if node != 0: + # Find node outputing into the current node being processed (this will be the input shape). This is + # necessary cuz if a node originally receives input from a `nn.Flatten` for instance, when mapped into + # a `DynapcnnLayer` it will be receiving the input from a privious `sl.SumPool2d`. + input_node = find_my_input(self.sinabs_edges, node) + input_node_orig_name = find_original_node_name(self.nodes_name_remap, input_node) + _, _input_source_shape = self.graph_tracer.get_node_io_shapes(input_node_orig_name) + node_data['input_shape'] = tuple(list(_input_source_shape)[1:]) + else: + # first node does not have an input source within the graph. + node_data['input_shape'] = tuple(list(_in)[1:]) + + node_data['output_shape'] = tuple(list(_out)[1:]) + + for dcnnl_idx, dcnnl_data in self.nodes_to_dcnnl_map.items(): + print(f'DynapcnnLayer-index {dcnnl_idx}') + for key, val in dcnnl_data.items(): + if isinstance(key, int): + print(key, val['input_shape'], val['output_shape']) + print('\n') \ No newline at end of file From 5669c8118935c4b5a724ced7b61928be9084309c Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Mon, 22 Apr 2024 14:12:04 +0200 Subject: [PATCH 030/379] DynapcnnLayer instances created from edges + dynamically updating nodes' I/O shape from modificaitons to layers done by a DynapcnnLayer instantiation --- sinabs/backend/dynapcnn/dynapcnn_layer_new.py | 76 ++++++++++++++++--- .../dynapcnn/dynapcnn_network_graph.py | 32 ++------ sinabs/backend/dynapcnn/utils.py | 29 +++++-- 3 files changed, 92 insertions(+), 45 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer_new.py b/sinabs/backend/dynapcnn/dynapcnn_layer_new.py index 71993b53..c1b3811c 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer_new.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer_new.py @@ -41,6 +41,7 @@ def __init__( self, dcnnl_data: dict, discretize: bool, + nodes_mapper: dict, rescale_weights: int = 1, # TODO remove. ): super().__init__() @@ -51,14 +52,13 @@ def __init__( 1) need to figure out how to apply 'rescale_weights' since there are more than two poolings. 2) currently there's no way the forward would work since there are more than two poolings. """ - - #### NEW WAY ####################################################################### + self.lin_to_conv_conversion = False conv = None conv_node_id = None spk = None - spk_node_id = None + self.spk_node_id = None pool = [] pool_node_id = [] @@ -68,7 +68,7 @@ def __init__( # value has data pertaining a node (torch/sinabs layer). if isinstance(value['layer'], sl.IAFSqueeze): spk = value['layer'] - spk_node_id = key + self.spk_node_id = key elif isinstance(value['layer'], nn.Linear) or isinstance(value['layer'], nn.Conv2d): conv = value['layer'] conv_node_id = key @@ -89,7 +89,15 @@ def __init__( spk.v_mem = spk.v_mem.data.unsqueeze(-1).unsqueeze(-1) # expand dims. if isinstance(conv, nn.Linear): - conv = self._convert_linear_to_conv(conv, dcnnl_data[conv_node_id]) + conv, conv_in_shape = self._convert_linear_to_conv(conv, dcnnl_data[conv_node_id]) + + # the original `nn.Linear` output shape becomes the equivalent `nn.Conv2d` shape. + conv_out_shape = self._update_conv_node_output_shape( + conv_layer=conv, layer_data=dcnnl_data[conv_node_id], input_shape=conv_in_shape) + + # the I/O shapes for neuron layer following the new conv need also to be updated. + self._update_neuron_node_output_shape(layer_data=dcnnl_data[self.spk_node_id], input_shape=conv_out_shape) + else: conv = deepcopy(conv) @@ -113,6 +121,17 @@ def __init__( raise ValueError("Only square kernels are supported") self.pool_layer.append(deepcopy(plyr)) + def __str__(self): + pretty_print = '' + + pretty_print += f'(con_layer): {self.conv_layer}\n' + pretty_print += f'(spk_layer): {self.spk_layer}\n' + if len(self.pool_layer) != 0: + for idx, lyr in enumerate(self.pool_layer): + pretty_print += f'(pool_layer {idx}): {lyr}\n' + + return pretty_print + def _convert_linear_to_conv(self, lin: nn.Linear, layer_data: dict) -> nn.Conv2d: """Convert Linear layer to Conv2d. @@ -126,12 +145,9 @@ def _convert_linear_to_conv(self, lin: nn.Linear, layer_data: dict) -> nn.Conv2d nn.Conv2d Convolutional layer equivalent to `lin`. """ + self.lin_to_conv_conversion = True - # TODO linear layers are convered to conv so the input shapes in the original - # mapper has to be updated accordingly. Another problem is that the input shape - # after a flatten layer won't match the fact that there's a convolution output before - # the flatten. - input_shape = tuple(list(layer_data['input_shape'])[1:-1]) # removing the batch dimension. + input_shape = layer_data['input_shape'] in_chan, in_h, in_w = input_shape @@ -155,12 +171,48 @@ def _convert_linear_to_conv(self, lin: nn.Linear, layer_data: dict) -> nn.Conv2d .reshape((lin.out_features, in_chan, in_h, in_w)) ) - return layer + return layer, input_shape + + def _update_conv_node_output_shape(self, conv_layer: nn.Conv2d, layer_data: dict, input_shape: tuple) -> Tuple: + """ The input shapes to nodes are extracted using a list of edges by finding the output shape of the 1st element + in the edge and setting it as the input shape to the 2nd element in the edge. If a node used to be a `nn.Linear` + and it became a `nn.Conv2d`, output shape in the mapper needs to be updated, otherwise there will be a mismatch + between its output and the input it provides to another node. + """ + layer_data['output_shape'] = self.get_conv_output_shape(conv_layer, input_shape) + + return layer_data['output_shape'] + + def _update_neuron_node_output_shape(self, layer_data: dict, input_shape: tuple) -> None: + """ Following the conversion of a `nn.Linear` into a `nn.Conv2d` the neuron layer in the + sequence also needs its I/O shapes uodated. + """ + layer_data['input_shape'] = input_shape + layer_data['output_shape'] = layer_data['input_shape'] + + def get_modified_node_it(self, dcnnl_data: dict) -> Union[Tuple[int, tuple], Tuple[None, None]]: + """ .""" + if self.lin_to_conv_conversion: + return self.spk_node_id, dcnnl_data[self.spk_node_id]['output_shape'] + return None, None def zero_grad(self, set_to_none: bool = False) -> None: return self.spk_layer.zero_grad(set_to_none) - ######################################################################################## + def get_conv_output_shape(self, conv_layer: nn.Conv2d, input_shape: tuple): + # get the layer's parameters. + out_channels = conv_layer.out_channels + kernel_size = conv_layer.kernel_size + stride = conv_layer.stride + padding = conv_layer.padding + dilation = conv_layer.dilation + + # compute the output height and width. + out_height = ((input_shape[1] + 2 * padding[0] - dilation[0] * (kernel_size[0] - 1) - 1) // stride[0]) + 1 + out_width = ((input_shape[2] + 2 * padding[1] - dilation[1] * (kernel_size[1] - 1) - 1) // stride[1]) + 1 + + return (out_channels, out_height, out_width) + def summary(self) -> dict: # TODO deprecated. return { diff --git a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py index 583530b3..a45f1da5 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py @@ -87,20 +87,11 @@ def __init__( # update the I/O shapes for each layer in 'self.nodes_to_dcnnl_map'. self.populate_nodes_io() - # # build model from graph edges. - # self.dynapcnn_layers = build_from_graph( - # discretize=discretize, - # edges=self.sinabs_edges, - # nodes_to_dcnnl_map=self.nodes_to_dcnnl_map) - - # for dcnnl_idx, dcnnl_data in self.nodes_to_dcnnl_map.items(): - # print(f'DynapcnnLayer-index {dcnnl_idx}') - # for key, val in dcnnl_data.items(): - # if isinstance(key, int): - # print(key, val['layer'], val['input_shape']) - # else: - # print(key, val) - # print('\n') + # build model from graph edges. + self.dynapcnn_layers = build_from_graph( + discretize=discretize, + edges=self.sinabs_edges, + nodes_to_dcnnl_map=self.nodes_to_dcnnl_map) def __str__(self): pretty_print = '' @@ -385,10 +376,6 @@ def get_sinabs_edges_and_modules(self) -> Tuple[List[Tuple[int, int]], Dict[int, def populate_nodes_io(self): """ .""" - for edge in self.sinabs_edges: - print(edge) - print('\n') - def find_original_node_name(name_mapper, node): for orig_name, new_name in name_mapper.items(): if new_name == node: @@ -426,11 +413,4 @@ def find_my_input(edges_list, node): # first node does not have an input source within the graph. node_data['input_shape'] = tuple(list(_in)[1:]) - node_data['output_shape'] = tuple(list(_out)[1:]) - - for dcnnl_idx, dcnnl_data in self.nodes_to_dcnnl_map.items(): - print(f'DynapcnnLayer-index {dcnnl_idx}') - for key, val in dcnnl_data.items(): - if isinstance(key, int): - print(key, val['input_shape'], val['output_shape']) - print('\n') \ No newline at end of file + node_data['output_shape'] = tuple(list(_out)[1:]) \ No newline at end of file diff --git a/sinabs/backend/dynapcnn/utils.py b/sinabs/backend/dynapcnn/utils.py index b04cff75..eadbcc18 100644 --- a/sinabs/backend/dynapcnn/utils.py +++ b/sinabs/backend/dynapcnn/utils.py @@ -599,7 +599,7 @@ def build_from_graph( if 'core_idx' not in layer_data: layer_data['core_idx'] = -1 # a DynapcnnLayer gets assigned a core index when 'DynapcnnNetworkGraph.to()' is called. - return None + return dynapcnn_layers def construct_dynapcnnlayers_from_mapper( discretize: bool, @@ -622,22 +622,36 @@ def construct_dynapcnnlayers_from_mapper( dynapcnnlayer = construct_dynapcnnlayer( discretize, dcnnl_data, edges, nodes_to_dcnnl_map) - print('> ', dpcnnl_idx) - dynapcnn_layers[dpcnnl_idx] = { 'layer': dynapcnnlayer, 'destinations': nodes_to_dcnnl_map[dpcnnl_idx]['destinations'] } + node, output_shape = dynapcnnlayer.get_modified_node_it(dcnnl_data) + + if isinstance(node, int) and isinstance(output_shape, tuple): + update_nodes_io(node, output_shape, nodes_to_dcnnl_map, edges) + return dynapcnn_layers +def update_nodes_io(updated_node: int, output_shape: tuple, nodes_to_dcnnl_map: dict, edges: List[Tuple[int, int]]) -> None: + """ .""" + for edge in edges: + if edge[0] == updated_node: + # found source node where output shape has been modified. + for dcnnl_idx, dcnnl_data in nodes_to_dcnnl_map.items(): + for key, val in dcnnl_data.items(): + if isinstance(key, int): + # accessing node data (layer, input_shape, output_shape). + if key == edge[1]: + # accessing node targeted by `updated_node` (its input shape becomes `updated_node.output_shape`). + val['input_shape'] = output_shape + def construct_dynapcnnlayer( discretize: bool, dcnnl_data: dict, edges: List[Tuple[int, int]], - nodes_to_dcnnl_map: dict, - layer_index = None, # TODO remove. - ) -> DynapcnnLayer: + nodes_to_dcnnl_map: dict) -> DynapcnnLayer: """ Extract the modules (layers) in a dictionary and uses them to instantiate a DynapcnnLayer object. Parameters @@ -659,7 +673,8 @@ def construct_dynapcnnlayer( # instantiate a DynapcnnLayer from the data in 'dcnnl_data'. dynapcnnlayer = DynapcnnLayer( dcnnl_data = dcnnl_data, - discretize = discretize + discretize = discretize, + nodes_mapper = nodes_to_dcnnl_map ) return dynapcnnlayer From 7a4d594b8dbca53decc9a15800b8603fc158b739 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Mon, 22 Apr 2024 15:13:02 +0200 Subject: [PATCH 031/379] if two DynapcnnLayers target a DynapcnnLayer X, X.conv rescaling factor is the highest amongst the branches merging into it --- sinabs/backend/dynapcnn/dynapcnn_layer_new.py | 55 ++++++------------- .../dynapcnn/dynapcnn_network_graph.py | 14 ++++- sinabs/backend/dynapcnn/utils.py | 13 +++-- 3 files changed, 39 insertions(+), 43 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer_new.py b/sinabs/backend/dynapcnn/dynapcnn_layer_new.py index c1b3811c..92388c13 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer_new.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer_new.py @@ -14,54 +14,34 @@ class DynapcnnLayer(nn.Module): - """Create a DynapcnnLayer object representing a dynapcnn layer. - - Requires a convolutional layer, a sinabs spiking layer and an optional - pooling value. The layers are used in the order conv -> spike -> pool. - - Parameters - ---------- - conv: torch.nn.Conv2d or torch.nn.Linear - Convolutional or linear layer (linear will be converted to convolutional) - spk: sinabs.layers.IAFSqueeze - Sinabs IAF layer - in_shape: tuple of int - The input shape, needed to create dynapcnn configs if the network does not - contain an input layer. Convention: (features, height, width) - pool: int or None - Integer representing the sum pooling kernel and stride. If `None`, no - pooling will be applied. - discretize: bool - Whether to discretize parameters. - rescale_weights: int - Layer weights will be divided by this value. - """ + """Create a DynapcnnLayer object representing a dynapcnn layer. """ def __init__( self, dcnnl_data: dict, discretize: bool, - nodes_mapper: dict, rescale_weights: int = 1, # TODO remove. ): super().__init__() """ ... + Parameters + ---------- + TODO - 1) need to figure out how to apply 'rescale_weights' since there are more than two poolings. - 2) currently there's no way the forward would work since there are more than two poolings. + 1) currently there's no way the forward would work since there are more than two poolings. """ self.lin_to_conv_conversion = False conv = None - conv_node_id = None + self.conv_node_id = None spk = None self.spk_node_id = None pool = [] - pool_node_id = [] + self.pool_node_id = [] for key, value in dcnnl_data.items(): if isinstance(key, int): @@ -71,10 +51,10 @@ def __init__( self.spk_node_id = key elif isinstance(value['layer'], nn.Linear) or isinstance(value['layer'], nn.Conv2d): conv = value['layer'] - conv_node_id = key + self.conv_node_id = key elif isinstance(value['layer'], sl.SumPool2d): pool.append(value['layer']) - pool_node_id.append(key) + self.pool_node_id.append(key) else: raise ValueError(f'Node {key} has not valid layer associated with it.') @@ -89,11 +69,11 @@ def __init__( spk.v_mem = spk.v_mem.data.unsqueeze(-1).unsqueeze(-1) # expand dims. if isinstance(conv, nn.Linear): - conv, conv_in_shape = self._convert_linear_to_conv(conv, dcnnl_data[conv_node_id]) + conv, conv_in_shape = self._convert_linear_to_conv(conv, dcnnl_data[self.conv_node_id]) # the original `nn.Linear` output shape becomes the equivalent `nn.Conv2d` shape. conv_out_shape = self._update_conv_node_output_shape( - conv_layer=conv, layer_data=dcnnl_data[conv_node_id], input_shape=conv_in_shape) + conv_layer=conv, layer_data=dcnnl_data[self.conv_node_id], input_shape=conv_in_shape) # the I/O shapes for neuron layer following the new conv need also to be updated. self._update_neuron_node_output_shape(layer_data=dcnnl_data[self.spk_node_id], input_shape=conv_out_shape) @@ -101,11 +81,10 @@ def __init__( else: conv = deepcopy(conv) - # TODO have to consider that two poolings might be projecting to this conv (these lines of code are deprecated). # this weight rescale comes from the node projecting into this 'conv' node. - if rescale_weights != 1: + if dcnnl_data['conv_rescale_factor'] != 1: # this has to be done after copying but before discretizing - conv.weight.data = (conv.weight / rescale_weights).clone().detach() + conv.weight.data = (conv.weight / dcnnl_data['conv_rescale_factor']).clone().detach() # int conversion is done while writing the config. if discretize: @@ -122,13 +101,13 @@ def __init__( self.pool_layer.append(deepcopy(plyr)) def __str__(self): - pretty_print = '' + pretty_print = '\n' - pretty_print += f'(con_layer): {self.conv_layer}\n' - pretty_print += f'(spk_layer): {self.spk_layer}\n' + pretty_print += f'(node {self.conv_node_id}): {self.conv_layer}\n' + pretty_print += f'(node {self.spk_node_id}): {self.spk_layer}' if len(self.pool_layer) != 0: for idx, lyr in enumerate(self.pool_layer): - pretty_print += f'(pool_layer {idx}): {lyr}\n' + pretty_print += f'\n(node {self.pool_node_id[idx]}): {lyr}' return pretty_print diff --git a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py index a45f1da5..5670317f 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py @@ -92,14 +92,26 @@ def __init__( discretize=discretize, edges=self.sinabs_edges, nodes_to_dcnnl_map=self.nodes_to_dcnnl_map) + + print('------------------------------------------------------') + for dcnnl_idx, dcnnl_data in self.nodes_to_dcnnl_map.items(): + print(f'DynapcnnLayer-index {dcnnl_idx}') + for key, val in dcnnl_data.items(): + if isinstance(key, int): + print(key, val['input_shape'], val['output_shape']) + else: + print(key, val) + print('\n') + print('------------------------------------------------------') def __str__(self): pretty_print = '' for idx, layer_data in self.dynapcnn_layers.items(): + pretty_print += f'---- DynapcnnLayer {idx} ----------------------------------------------------------' layer = layer_data['layer'] dest = layer_data['destinations'] core = layer_data['core_idx'] - pretty_print += f'\nlayer index: {idx}\nlayer modules: {layer}\nlayer destinations: {dest}\nassigned core: {core}\n' + pretty_print += f'\n> layer modules: {layer}\n> layer destinations: {dest}\n> assigned core: {core}\n\n' return pretty_print def to( diff --git a/sinabs/backend/dynapcnn/utils.py b/sinabs/backend/dynapcnn/utils.py index eadbcc18..e2a8dc40 100644 --- a/sinabs/backend/dynapcnn/utils.py +++ b/sinabs/backend/dynapcnn/utils.py @@ -619,6 +619,7 @@ def construct_dynapcnnlayers_from_mapper( dynapcnn_layers = {} for dpcnnl_idx, dcnnl_data in nodes_to_dcnnl_map.items(): + # create a `DynapcnnLayer` from the set of layers in `dcnnl_data`. dynapcnnlayer = construct_dynapcnnlayer( discretize, dcnnl_data, edges, nodes_to_dcnnl_map) @@ -627,8 +628,10 @@ def construct_dynapcnnlayers_from_mapper( 'destinations': nodes_to_dcnnl_map[dpcnnl_idx]['destinations'] } + # check if any node (layer) in `dynapcnnlayer` has been modified (e.g. `nn.Linear` turned `nn.Conv2d`). node, output_shape = dynapcnnlayer.get_modified_node_it(dcnnl_data) + # one of the layers in `dynapcnnlayer` had its type modified (update input shape of nodes receiving from it). if isinstance(node, int) and isinstance(output_shape, tuple): update_nodes_io(node, output_shape, nodes_to_dcnnl_map, edges) @@ -673,8 +676,7 @@ def construct_dynapcnnlayer( # instantiate a DynapcnnLayer from the data in 'dcnnl_data'. dynapcnnlayer = DynapcnnLayer( dcnnl_data = dcnnl_data, - discretize = discretize, - nodes_mapper = nodes_to_dcnnl_map + discretize = discretize ) return dynapcnnlayer @@ -703,11 +705,14 @@ def convert_Avg_to_Sum_pooling(dcnnl_data: dict, edges: list, nodes_to_dcnnl_map # find which node 'key' will target. for edge in edges: if edge[0] == key: - # target within DynapcnnLayer index 'trg_dcnnl_idx'. + # find index of DynapcnnLayer where the target of `edge[0]` is. trg_dcnnl_idx = find_nodes_dcnnl_idx(edge[1], nodes_to_dcnnl_map) # update the rescale factor for the target of node 'key'. - dcnnl_data['destinations_rescale_factor'][trg_dcnnl_idx] = rescale_factor + if rescale_factor > nodes_to_dcnnl_map[trg_dcnnl_idx]['conv_rescale_factor']: + # If more than one DynapcnnLayers target `trg_dcnnl_idx` with different rescale + # factors, the highest amongst them is used. + nodes_to_dcnnl_map[trg_dcnnl_idx]['conv_rescale_factor'] = rescale_factor def find_nodes_dcnnl_idx(node, nodes_to_dcnnl_map): """ .""" From c421154d48445df57e271e33ecd4d21cf3be989e Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Mon, 22 Apr 2024 15:22:46 +0200 Subject: [PATCH 032/379] updated in-line documentation --- sinabs/backend/dynapcnn/dynapcnn_layer_new.py | 8 +++++--- sinabs/backend/dynapcnn/utils.py | 17 +++++------------ 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer_new.py b/sinabs/backend/dynapcnn/dynapcnn_layer_new.py index 92388c13..67965e98 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer_new.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer_new.py @@ -19,8 +19,7 @@ class DynapcnnLayer(nn.Module): def __init__( self, dcnnl_data: dict, - discretize: bool, - rescale_weights: int = 1, # TODO remove. + discretize: bool ): super().__init__() """ @@ -94,7 +93,9 @@ def __init__( self.conv_layer = conv self.spk_layer = spk self.pool_layer = [] + if len(pool) != 0: + # the 1st pooling targets the 1st destination in `dcnnl_data['destinations']`, the 2nd pooling targets the 2nd destination... for plyr in pool: if plyr.kernel_size[0] != plyr.kernel_size[1]: raise ValueError("Only square kernels are supported") @@ -179,6 +180,7 @@ def zero_grad(self, set_to_none: bool = False) -> None: return self.spk_layer.zero_grad(set_to_none) def get_conv_output_shape(self, conv_layer: nn.Conv2d, input_shape: tuple): + """ .""" # get the layer's parameters. out_channels = conv_layer.out_channels kernel_size = conv_layer.kernel_size @@ -236,4 +238,4 @@ def forward(self, x): # TODO deprecate x = self.spk_layer(x) if self.pool_layer is not None: x = self.pool_layer(x) - return x + return x \ No newline at end of file diff --git a/sinabs/backend/dynapcnn/utils.py b/sinabs/backend/dynapcnn/utils.py index e2a8dc40..2ee14b99 100644 --- a/sinabs/backend/dynapcnn/utils.py +++ b/sinabs/backend/dynapcnn/utils.py @@ -666,13 +666,6 @@ def construct_dynapcnnlayer( # convert all AvgPool2d in 'dcnnl_data' into SumPool2d. convert_Avg_to_Sum_pooling(dcnnl_data, edges, nodes_to_dcnnl_map) - # TODO 'rescale_weight' information is inside 'dcnnl_data' but it is not yet being used. - # TODO the input shapes work fine when processing conv layers but once we reach linear layers - # their input shapes has to come from the conv layers projecting to the node. Currently also a - # linear layer after a flatten has the input shape coming out of the flatten but it needs to be - # the conv output shape before the flatten. A linear layer coming after a previous linear layer - # converted into conv needs to have its input shape updated based on this conversion. - # instantiate a DynapcnnLayer from the data in 'dcnnl_data'. dynapcnnlayer = DynapcnnLayer( dcnnl_data = dcnnl_data, @@ -693,7 +686,7 @@ def convert_Avg_to_Sum_pooling(dcnnl_data: dict, edges: list, nodes_to_dcnnl_map """ for key, value in dcnnl_data.items(): if isinstance(key, int): - # accessing the node 'key's dictionary. + # accessing the node `key` dictionary. if isinstance(value['layer'], nn.AvgPool2d): # convert AvgPool2d into SumPool2d. @@ -702,13 +695,13 @@ def convert_Avg_to_Sum_pooling(dcnnl_data: dict, edges: list, nodes_to_dcnnl_map # turn avg into sum pool. value['layer'] = lyr_pool - # find which node 'key' will target. + # find which node `key` will target. for edge in edges: if edge[0] == key: # find index of DynapcnnLayer where the target of `edge[0]` is. trg_dcnnl_idx = find_nodes_dcnnl_idx(edge[1], nodes_to_dcnnl_map) - # update the rescale factor for the target of node 'key'. + # update the rescale factor for the target of node `key`. if rescale_factor > nodes_to_dcnnl_map[trg_dcnnl_idx]['conv_rescale_factor']: # If more than one DynapcnnLayers target `trg_dcnnl_idx` with different rescale # factors, the highest amongst them is used. @@ -719,9 +712,9 @@ def find_nodes_dcnnl_idx(node, nodes_to_dcnnl_map): for dcnnl_idx, dcnnl_data in nodes_to_dcnnl_map.items(): for key, value in dcnnl_data.items(): if isinstance(key, int): - # 'key' is a node. + # `key` is a node. if key == node: - # node belongs to DynapcnnLayer index 'dcnnl_idx'. + # node belongs to DynapcnnLayer index `dcnnl_idx`. return dcnnl_idx # this exception should never happen. From 3f59859a76f408cb15f669bff5419baec6b9ff63 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Mon, 22 Apr 2024 18:06:24 +0200 Subject: [PATCH 033/379] (WIP) retrieving configuration dict from DynapcnnLayer to set CNNLayerConfig --- sinabs/backend/dynapcnn/chips/dynapcnn.py | 70 ++++++++-- sinabs/backend/dynapcnn/dynapcnn_layer_new.py | 126 ++++++++++++++++-- .../dynapcnn/dynapcnn_network_graph.py | 60 ++++++--- 3 files changed, 215 insertions(+), 41 deletions(-) diff --git a/sinabs/backend/dynapcnn/chips/dynapcnn.py b/sinabs/backend/dynapcnn/chips/dynapcnn.py index 5f47076e..c7292b30 100644 --- a/sinabs/backend/dynapcnn/chips/dynapcnn.py +++ b/sinabs/backend/dynapcnn/chips/dynapcnn.py @@ -9,7 +9,8 @@ import sinabs from sinabs.backend.dynapcnn.config_builder import ConfigBuilder from sinabs.backend.dynapcnn.dvs_layer import DVSLayer, expand_to_pair -from sinabs.backend.dynapcnn.dynapcnn_layer import DynapcnnLayer +# from sinabs.backend.dynapcnn.dynapcnn_layer import DynapcnnLayer +from sinabs.backend.dynapcnn.dynapcnn_layer_new import DynapcnnLayer from sinabs.backend.dynapcnn.mapping import LayerConstraints import sinabs @@ -207,10 +208,35 @@ def write_dynapcnn_layer_config( @classmethod def write_dynapcnn_layer_config_graph(cls, dcnnl_data: dict, chip_layer: "CNNLayerConfig", dynapcnn_layers: dict): - """ .""" - - config_dict = cls.get_dynapcnn_layer_config_dict( # extracting from a DynapcnnLayer the config. variables for its CNNLayerConfig. - layer=dcnnl_data['layer']) + """ Uses the data in `dcnnl_data` to configure a `CNNLayerConfig` to be deployed on chip. + + Parameters + ---------- + dcnnl_data: + contains the DynapcnnLayer (`dcnnl_data['layer']`), is list of destination DynapcnnLayer indexes + (`dcnnl_data['destinations']`), and the core ID it is to be mapped to (`dcnnl_data['core_idx']`). + chip_layer: + a `CNNLayerConfig` (indexed by `dcnnl_data['core_idx']`) used to represent the DynapcnnLayer + in `dcnnl_data['layer']`. + dynapcnn_layers: + a dictionary with keys being the ID of each DynapcnnLayer and values being the dictionary with the + `dcnnl_data` structure described above. This is used to retrieve the `core_idx` for each of the + layers in `dcnnl_data['destinations']` such that `chip_layer.destinations` can be configured. + """ + + # extracting from a DynapcnnLayer the config. variables for its CNNLayerConfig. + config_dict = dcnnl_data['layer'].get_layer_config_dict() + + # use core indexing instead of DynapcnnLayer indexing for destinations. + for dest_config in config_dict['destinations']: + dcnnl_idx = dest_config['layer'] + dcnnl_core_idx = dynapcnn_layers[dcnnl_idx]['core_idx'] # get the core the destination DynapcnnLayer is using. + dest_config['layer'] = dcnnl_core_idx + + for key, val in config_dict.items(): + print(key, val) + + input('...') chip_layer.dimensions = config_dict["dimensions"] config_dict.pop("dimensions") @@ -223,7 +249,7 @@ def write_dynapcnn_layer_config_graph(cls, dcnnl_data: dict, chip_layer: "CNNLay for dest_idx in range(len(dcnnl_data['destinations'])): # configuring the destinations for this DynapcnnLayer. chip_layer.destinations[dest_idx].enable = True - destination_core_idx = dynapcnn_layers[dcnnl_data['destinations'][dest_idx]]['core_idx'] # retrive the core to wich a destination DynapcnnLayer has been assigned to. + destination_core_idx = dynapcnn_layers[dcnnl_data['destinations'][dest_idx]]['core_idx'] # retrive the core to wich the destination DynapcnnLayer has been assigned to. chip_layer.destinations[dest_idx].layer = destination_core_idx if isinstance(pooling, int): @@ -233,18 +259,36 @@ def write_dynapcnn_layer_config_graph(cls, dcnnl_data: dict, chip_layer: "CNNLay chip_layer.destinations[0].enable = False chip_layer.destinations[1].enable = False - for param, value in config_dict.items(): + for param, value in config_dict.items(): # set remaining attributes. try: setattr(chip_layer, param, value) except TypeError as e: raise TypeError(f"Unexpected parameter {param} or value. {e}") @classmethod - def build_config(cls, model: Union["DynapcnnNetwork", "DynapcnnNetworkGraph"], chip_layers: List[int]): + def build_config(cls, model: Union["DynapcnnNetwork", "DynapcnnNetworkGraph"], chip_layers: Union[List[int], None]) -> DynapcnnConfiguration: + """ Uses `DynapcnnLayer` objects to configure their equivalent chip core via a `CNNLayerConfig` object that is built + using using the `DynapcnnLayer` properties. + + Parameters + ---------- + model: + either a `DynapcnnNetwork` or a `DynapcnnNetworkGraph` instance where the model (DynapcnnLayer) layers can be found. + chip_layers: + a list containing the core indexes where each `DynapcnnLayer` will be mapped to (if `model` is an instance of `DynapcnnNetwork`, otherwise `None`). + + Returns + ---------- + config: + an instance of a `DynapcnnConfiguration`. + """ + config = cls.get_default_config() if type(model) == sinabs.backend.dynapcnn.dynapcnn_network.DynapcnnNetwork: + """ loops through `DynapcnnNetwork.sequence`, sequentially using the core IDs in `chip_layers` to configure their + respective `CNNLayerConfig`. + """ layers = model.sequence - config = cls.get_default_config() has_dvs_layer = False i_cnn_layer = 0 # Instantiate an iterator for the cnn cores @@ -275,8 +319,11 @@ def build_config(cls, model: Union["DynapcnnNetwork", "DynapcnnNetworkGraph"], c config.dvs_layer.pass_sensor_events = False elif type(model) == sinabs.backend.dynapcnn.dynapcnn_network_graph.DynapcnnNetworkGraph: - config = cls.get_default_config() - has_dvs_layer = False + """ loops through `DynapcnnNetworkGraph.dynapcnn_layers`, where each represented layer representation constains their + core ID to be loaded onto and their target destinations. Each `layer_data` has all the info. necessary to config. + their respective `CNNLayerConfig` object. + """ + has_dvs_layer = False # TODO DVSLayer not supported yet. for _, layer_data in model.dynapcnn_layers.items(): if isinstance(layer_data['layer'], DVSLayer): @@ -287,6 +334,7 @@ def build_config(cls, model: Union["DynapcnnNetwork", "DynapcnnNetworkGraph"], c cls.write_dynapcnn_layer_config_graph(layer_data, chip_layer, model.dynapcnn_layers) else: + print('[error] ', layer_data['layer']) raise TypeError("Unexpected layer in the model.") # shouldn't happen since type checks are made previously. if not has_dvs_layer: # TODO DVSLayer not supported yet. diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer_new.py b/sinabs/backend/dynapcnn/dynapcnn_layer_new.py index 67965e98..8b455c67 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer_new.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer_new.py @@ -35,12 +35,16 @@ def __init__( conv = None self.conv_node_id = None + self.conv_in_shape = None + self.conv_out_shape = None spk = None self.spk_node_id = None pool = [] self.pool_node_id = [] + + self.dynapcnnlayer_destination = dcnnl_data['destinations'] for key, value in dcnnl_data.items(): if isinstance(key, int): @@ -71,15 +75,23 @@ def __init__( conv, conv_in_shape = self._convert_linear_to_conv(conv, dcnnl_data[self.conv_node_id]) # the original `nn.Linear` output shape becomes the equivalent `nn.Conv2d` shape. - conv_out_shape = self._update_conv_node_output_shape( + self.conv_out_shape = self._update_conv_node_output_shape( conv_layer=conv, layer_data=dcnnl_data[self.conv_node_id], input_shape=conv_in_shape) # the I/O shapes for neuron layer following the new conv need also to be updated. - self._update_neuron_node_output_shape(layer_data=dcnnl_data[self.spk_node_id], input_shape=conv_out_shape) + self._update_neuron_node_output_shape(layer_data=dcnnl_data[self.spk_node_id], input_shape=self.conv_out_shape) else: + self.conv_out_shape = dcnnl_data[self.conv_node_id]['output_shape'] conv = deepcopy(conv) + # check if convolution kernel is a square. + if conv.kernel_size[0] != conv.kernel_size[1]: + raise ValueError('The kernel of a `nn.Conv2d` must have the same height and width.') + + # input shape of conv layer. + self.conv_in_shape = dcnnl_data[self.conv_node_id]['input_shape'] + # this weight rescale comes from the node projecting into this 'conv' node. if dcnnl_data['conv_rescale_factor'] != 1: # this has to be done after copying but before discretizing @@ -194,17 +206,113 @@ def get_conv_output_shape(self, conv_layer: nn.Conv2d, input_shape: tuple): return (out_channels, out_height, out_width) - - def summary(self) -> dict: # TODO deprecated. + def summary(self) -> dict: + # TODO I can't see pooling being used in checking memory constraints by the builder so I'm ignoring for now the fact that multiple pooling could exist. return { - "pool": ( - None if self.pool_layer is None else list(self.pool_layer.kernel_size) + "pool": ( # ignoring for now that there could be multiple poolings (just use the first one). + None if len(self.pool_layer) == 0 else list(self.pool_layer[0].kernel_size) ), "kernel": list(self.conv_layer.weight.data.shape), - "neuron": self.get_neuron_shape(), + "neuron": self.conv_out_shape, # neuron layer output has the same shape as the convolution layer ouput. } - def memory_summary(self): # TODO deprecated. + def get_layer_config_dict(self) -> dict: + """ Returns a dict containing the properties required to configure a `CNNLayerConfig` instance.""" + config_dict = {} + + # configures `CNNLayerConfig.dimensions` (instance of `CNNLayerDimensions`). + dimensions = {} + + # input shape of convolution. + dimensions['input_shape'] = { + 'size': {'x': self.conv_in_shape[2], 'y': self.conv_in_shape[1]}, + 'feature_count': self.conv_in_shape[0] + } + + # ouput shape of convolution. + dimensions['output_shape'] = { + 'size': {'x': self.conv_out_shape[2], 'y': self.conv_out_shape[1]}, + 'feature_count': self.conv_out_shape[0] + } + + # convolution padding, stride and kernel sizes. + dimensions['padding'] = {'x': self.conv_layer.padding[1], 'y': self.conv_layer.padding[0]} + dimensions['stride'] = {'x': self.conv_layer.stride[1], 'y': self.conv_layer.stride[0]} + dimensions['kernel_size'] = self.conv_layer.kernel_size[0] + + config_dict.update(dimensions) # update config dict. + + # update parameters from convolution. + if self.conv_layer.bias is not None: + (weights, biases) = self.conv_layer.parameters() + else: + (weights,) = self.conv_layer.parameters() + biases = torch.zeros(self.conv_layer.out_channels) + + # parameters of the convolution in the DynapcnnLayer. + + weights = weights.transpose(2, 3) # need this to match samna convention. + config_dict['weights'] = weights.int().tolist() # 4-D list of lists representing kernel parameters. + config_dict['biases'] = biases.int().tolist() + config_dict['leak_enable'] = biases.bool().any() + + # parameters of the neurons in the DynapcnnLayer. + + # set neuron states. # TODO coppied from the old implementation. + if not self.spk_layer.is_state_initialised(): + # then we assign no initial neuron state to DYNAP-CNN. + f, h, w = self.conv_out_shape # same as the convolution layer. + neurons_state = torch.zeros(f, w, h) + + elif self.spk_layer.v_mem.dim() == 4: + # 4-D states should be the norm when there is a batch dim. + neurons_state = self.spk_layer.v_mem.transpose(2, 3)[0] + + else: + raise ValueError(f"Current v_mem (shape: {self.spk_layer.v_mem.shape}) of spiking layer not understood.") + # TODO error here: find where `self.spk_layer.v_mem` is being initialized. + + # resetting vs returning to 0. # TODO coppied from the old implementation. + if isinstance(self.spk_layer.reset_fn, sinabs.activation.MembraneReset): + return_to_zero = True # neurons in this layer will return to 0 when firing. + elif isinstance(self.spk_layer.reset_fn, sinabs.activation.MembraneSubtract): + return_to_zero = False # threshold will be subtracted from the value their membrane potential reached before firing. + else: + raise Exception("Unknown reset mechanism. Only MembraneReset and MembraneSubtract are currently understood.") + + if self.spk_layer.min_v_mem is None: + min_v_mem = -(2**15) + else: + min_v_mem = int(self.spk_layer.min_v_mem) + + # set neuron configuration for this DynapcnnLayer. + config_dict.update( + { + "return_to_zero": return_to_zero, + "threshold_high": int(self.spk_layer.spike_threshold), + "threshold_low": min_v_mem, + "monitor_enable": False, + "neurons_initial_value": neurons_state.int().tolist() + } + ) + + # set pooling configuration for each destinaition. This configures a `CNNLayerConfig.destinations` (instance of `CNNLayerDimensions`). + config_dict['destinations'] = [] + if len(self.pool_layer) != 0: + for i in range(len(self.pool_layer)): + dest_config = { + 'layer': self.dynapcnnlayer_destination[i],# TODO this destination index is not the core index yet, just the index of the DynapcnnLayers themselves. + 'enable': True, + 'pooling': self.pool_layer[i].kernel_size + } + + config_dict['destinations'].append(dest_config) + + # setting of the kill bits need to be done outside this method. + + return config_dict + + def memory_summary(self): """Computes the amount of memory required for each of the components. Note that this is not necessarily the same as the number of parameters due to some architecture design constraints. @@ -223,7 +331,7 @@ def memory_summary(self): # TODO deprecate """ summary = self.summary() f, c, h, w = summary["kernel"] - f, neuron_height, neuron_width = self.get_neuron_shape() + f, neuron_height, neuron_width = self.conv_out_shape # neuron layer output has the same shape as the convolution layer ouput. return { "kernel": c * pow(2, np.ceil(np.log2(h * w)) + np.ceil(np.log2(f))), diff --git a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py index 5670317f..055ffab3 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py @@ -92,17 +92,6 @@ def __init__( discretize=discretize, edges=self.sinabs_edges, nodes_to_dcnnl_map=self.nodes_to_dcnnl_map) - - print('------------------------------------------------------') - for dcnnl_idx, dcnnl_data in self.nodes_to_dcnnl_map.items(): - print(f'DynapcnnLayer-index {dcnnl_idx}') - for key, val in dcnnl_data.items(): - if isinstance(key, int): - print(key, val['input_shape'], val['output_shape']) - else: - print(key, val) - print('\n') - print('------------------------------------------------------') def __str__(self): pretty_print = '' @@ -111,7 +100,12 @@ def __str__(self): layer = layer_data['layer'] dest = layer_data['destinations'] core = layer_data['core_idx'] - pretty_print += f'\n> layer modules: {layer}\n> layer destinations: {dest}\n> assigned core: {core}\n\n' + + if 'core_destinations' in layer_data: + core_dest = layer_data['destinations'] + pretty_print += f'\n> layer modules: {layer}\n> layer destinations: {dest}\n> core destinations: {core_dest}\n> assigned core: {core}\n\n' + else: + pretty_print += f'\n> layer modules: {layer}\n> layer destinations: {dest}\n> assigned core: {core}\n\n' return pretty_print def to( @@ -330,26 +324,35 @@ def _make_config( has_dvs_layer = isinstance(self.dynapcnn_layers[0]['layer'], DVSLayer) - if chip_layers_ordering == "auto": # figure out mapping of each DynapcnnLayer into one core. + if chip_layers_ordering == "auto": + # figure out mapping of each DynapcnnLayer into one core. chip_layers_ordering = config_builder.get_valid_mapping(self) - else: # mapping from each DynapcnnLayer into cores has been provided. + else: + # mapping from each DynapcnnLayer into cores has been provided. if has_dvs_layer: pass # TODO not handling DVSLayer yet. - config = config_builder.build_config(self, None) # update config. + # we now know what core has been assigned to each DynapcnnLayer: add their core destinations info. + self.add_core_destinations_to_mapper() # TODO remove this. + + # update config. + config = config_builder.build_config(self, None) if self.input_shape and self.input_shape[0] == 1: # ??? config.dvs_layer.merge = True - monitor_chip_layers = [] # TODO all this monitoring part needs validation still. - if monitor_layers is None: # check if any monitoring is enabled (if not, enable monitoring for the last layer). + # TODO all this monitoring part needs validation still. + monitor_chip_layers = [] + if monitor_layers is None: + # check if any monitoring is enabled (if not, enable monitoring for the last layer). for _, dcnnl_data in self.dynapcnn_layers.items(): if len(dcnnl_data['destinations']) == 0: monitor_chip_layers.append(dcnnl_data['core_idx']) break elif monitor_layers == "all": - for _, dcnnl_data in self.dynapcnn_layers.items(): # monitor each chip core (if not a DVSLayer). + for _, dcnnl_data in self.dynapcnn_layers.items(): + # monitor each chip core (if not a DVSLayer). if not isinstance(dcnnl_data['layer'], DVSLayer): monitor_chip_layers.append(dcnnl_data['core_idx']) @@ -357,12 +360,27 @@ def _make_config( if "dvs" in monitor_layers: monitor_chip_layers.append("dvs") - config_builder.monitor_layers(config, monitor_chip_layers) # enable monitors on the specified layers. + # enable monitors on the specified layers. + config_builder.monitor_layers(config, monitor_chip_layers) - if config_modifier is not None: # apply user config modifier. + if config_modifier is not None: + # apply user config modifier. config = config_modifier(config) - return config, config_builder.validate_configuration(config) # validate config. + return config, config_builder.validate_configuration(config) + + def add_core_destinations_to_mapper(self): + """ .""" + + for idx, layer_data in self.dynapcnn_layers.items(): + destinations = layer_data['destinations'] + core_destinations = [] + + for lyr_dest in destinations: + core_dest = self.dynapcnn_layers[lyr_dest]['core_idx'] + core_destinations.append(core_destinations) + + layer_data['core_destinations'] = core_destinations def get_sinabs_edges_and_modules(self) -> Tuple[List[Tuple[int, int]], Dict[int, nn.Module]]: """ The computational graph extracted from `snn` might contain layers that are ignored (e.g. a `nn.Flatten` will be From e481afc078f54aee5d1afad17de53929126c9bd3 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Mon, 22 Apr 2024 19:01:23 +0200 Subject: [PATCH 034/379] successfully configuring model with single skip connection to the device --- sinabs/backend/dynapcnn/chips/dynapcnn.py | 28 +---- sinabs/backend/dynapcnn/dynapcnn_layer_new.py | 9 +- .../dynapcnn/dynapcnn_network_graph.py | 16 --- sinabs/backend/dynapcnn/mapping.py | 3 +- ...DynapcnnNetworkGraph_from_NIRgraph_4.ipynb | 114 ++++++++++++++---- 5 files changed, 102 insertions(+), 68 deletions(-) diff --git a/sinabs/backend/dynapcnn/chips/dynapcnn.py b/sinabs/backend/dynapcnn/chips/dynapcnn.py index c7292b30..179337a5 100644 --- a/sinabs/backend/dynapcnn/chips/dynapcnn.py +++ b/sinabs/backend/dynapcnn/chips/dynapcnn.py @@ -233,32 +233,12 @@ def write_dynapcnn_layer_config_graph(cls, dcnnl_data: dict, chip_layer: "CNNLay dcnnl_core_idx = dynapcnn_layers[dcnnl_idx]['core_idx'] # get the core the destination DynapcnnLayer is using. dest_config['layer'] = dcnnl_core_idx - for key, val in config_dict.items(): - print(key, val) - - input('...') - - chip_layer.dimensions = config_dict["dimensions"] - config_dict.pop("dimensions") - - pooling = None - if "pooling" in config_dict["destinations"][0]: - pooling = config_dict["destinations"][0]["pooling"] # TODO make pooling be destination-dependent. + # set the destinations configuration. + for i in range(len(config_dict['destinations'])): + chip_layer.destinations[i] = config_dict['destinations'][i] config_dict.pop("destinations") - for dest_idx in range(len(dcnnl_data['destinations'])): # configuring the destinations for this DynapcnnLayer. - chip_layer.destinations[dest_idx].enable = True - - destination_core_idx = dynapcnn_layers[dcnnl_data['destinations'][dest_idx]]['core_idx'] # retrive the core to wich the destination DynapcnnLayer has been assigned to. - chip_layer.destinations[dest_idx].layer = destination_core_idx - - if isinstance(pooling, int): - chip_layer.destinations[dest_idx].pooling = pooling - - if len(dcnnl_data['destinations']) == 0: # this is the output layer. - chip_layer.destinations[0].enable = False - chip_layer.destinations[1].enable = False - + # set remaining configuration. for param, value in config_dict.items(): # set remaining attributes. try: setattr(chip_layer, param, value) diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer_new.py b/sinabs/backend/dynapcnn/dynapcnn_layer_new.py index 8b455c67..3fc377fa 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer_new.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer_new.py @@ -69,6 +69,11 @@ def __init__( spk = deepcopy(spk) if spk.is_state_initialised(): + # TODO this line bellow is causing an exception on `.v_men.shape` to be raised in `.get_layer_config_dict()`. Find out why. + # spk.v_mem = spk.v_mem.data.unsqueeze(-1).unsqueeze(-1) # expand dims. + + # TODO hacky stuff: make it better (THIS SEEMS TO BE FIXING THE PROBLEM ABOVE THO). + if len(list(spk.v_mem.shape)) != 4: spk.v_mem = spk.v_mem.data.unsqueeze(-1).unsqueeze(-1) # expand dims. if isinstance(conv, nn.Linear): @@ -240,7 +245,7 @@ def get_layer_config_dict(self) -> dict: dimensions['stride'] = {'x': self.conv_layer.stride[1], 'y': self.conv_layer.stride[0]} dimensions['kernel_size'] = self.conv_layer.kernel_size[0] - config_dict.update(dimensions) # update config dict. + config_dict['dimensions'] = dimensions # update config dict. # update parameters from convolution. if self.conv_layer.bias is not None: @@ -303,7 +308,7 @@ def get_layer_config_dict(self) -> dict: dest_config = { 'layer': self.dynapcnnlayer_destination[i],# TODO this destination index is not the core index yet, just the index of the DynapcnnLayers themselves. 'enable': True, - 'pooling': self.pool_layer[i].kernel_size + 'pooling': self.pool_layer[i].kernel_size[0] # TODO make sure the kernel is a square. } config_dict['destinations'].append(dest_config) diff --git a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py index 055ffab3..172f4dd7 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py @@ -333,9 +333,6 @@ def _make_config( if has_dvs_layer: pass # TODO not handling DVSLayer yet. - # we now know what core has been assigned to each DynapcnnLayer: add their core destinations info. - self.add_core_destinations_to_mapper() # TODO remove this. - # update config. config = config_builder.build_config(self, None) @@ -369,19 +366,6 @@ def _make_config( return config, config_builder.validate_configuration(config) - def add_core_destinations_to_mapper(self): - """ .""" - - for idx, layer_data in self.dynapcnn_layers.items(): - destinations = layer_data['destinations'] - core_destinations = [] - - for lyr_dest in destinations: - core_dest = self.dynapcnn_layers[lyr_dest]['core_idx'] - core_destinations.append(core_destinations) - - layer_data['core_destinations'] = core_destinations - def get_sinabs_edges_and_modules(self) -> Tuple[List[Tuple[int, int]], Dict[int, nn.Module]]: """ The computational graph extracted from `snn` might contain layers that are ignored (e.g. a `nn.Flatten` will be ignored when creating a `DynapcnnLayer` instance). Thus the list of edges from such model need to be rebuilt such that if there are diff --git a/sinabs/backend/dynapcnn/mapping.py b/sinabs/backend/dynapcnn/mapping.py index bad403b3..1015e80c 100644 --- a/sinabs/backend/dynapcnn/mapping.py +++ b/sinabs/backend/dynapcnn/mapping.py @@ -4,7 +4,8 @@ from typing import List, Optional, Tuple, Union from .dvs_layer import DVSLayer -from .dynapcnn_layer import DynapcnnLayer +# from .dynapcnn_layer import DynapcnnLayer +from .dynapcnn_layer_new import DynapcnnLayer import sinabs from .exceptions import InvalidModel diff --git a/tests/test_nonsequential/DynapcnnNetworkGraph_from_NIRgraph_4.ipynb b/tests/test_nonsequential/DynapcnnNetworkGraph_from_NIRgraph_4.ipynb index da71104d..48d8ed01 100644 --- a/tests/test_nonsequential/DynapcnnNetworkGraph_from_NIRgraph_4.ipynb +++ b/tests/test_nonsequential/DynapcnnNetworkGraph_from_NIRgraph_4.ipynb @@ -21,7 +21,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 2, @@ -209,41 +209,105 @@ "cell_type": "code", "execution_count": 7, "metadata": {}, + "outputs": [], + "source": [ + "hw_model = DynapcnnNetworkGraph(\n", + " snn,\n", + " discretize=True,\n", + " input_shape=input_shape\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "> 0\n", - "> 1\n", - "> 2\n", - "torch.Size([1, 16]) 16\n" + "---- DynapcnnLayer 0 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 0): Conv2d(1, 30, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + "(node 1): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "(node 2): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", + "(node 3): SumPool2d(norm_type=1, kernel_size=(4, 4), stride=None, ceil_mode=False)\n", + "> layer destinations: [1, 2]\n", + "> assigned core: -1\n", + "\n", + "---- DynapcnnLayer 1 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 4): Conv2d(30, 30, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + "(node 6): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "(node 7): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", + "> layer destinations: [2]\n", + "> assigned core: -1\n", + "\n", + "---- DynapcnnLayer 2 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 8): Conv2d(30, 1, kernel_size=(3, 3), stride=(1, 1), bias=False)\n", + "(node 9): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [3]\n", + "> assigned core: -1\n", + "\n", + "---- DynapcnnLayer 3 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 10): Conv2d(1, 500, kernel_size=(4, 4), stride=(1, 1), bias=False)\n", + "(node 11): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [4]\n", + "> assigned core: -1\n", + "\n", + "---- DynapcnnLayer 4 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 12): Conv2d(500, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + "(node 13): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: []\n", + "> assigned core: -1\n", + "\n", + "\n" ] - }, + } + ], + "source": [ + "print(hw_model)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ { - "ename": "ValueError", - "evalue": "not enough values to unpack (expected 3, got 0)", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[7], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m hw_model \u001b[38;5;241m=\u001b[39m \u001b[43mDynapcnnNetworkGraph\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 2\u001b[0m \u001b[43m \u001b[49m\u001b[43msnn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 3\u001b[0m \u001b[43m \u001b[49m\u001b[43mdiscretize\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 4\u001b[0m \u001b[43m \u001b[49m\u001b[43minput_shape\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43minput_shape\u001b[49m\n\u001b[1;32m 5\u001b[0m \u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/Github/sinabs/sinabs/backend/dynapcnn/dynapcnn_network_graph.py:87\u001b[0m, in \u001b[0;36mDynapcnnNetworkGraph.__init__\u001b[0;34m(self, snn, input_shape, dvs_input, discretize)\u001b[0m\n\u001b[1;32m 84\u001b[0m \u001b[38;5;66;03m# update the I/O shapes for each layer in 'self.nodes_to_dcnnl_map'.\u001b[39;00m\n\u001b[1;32m 85\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpopulate_nodes_io()\n\u001b[0;32m---> 87\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdynapcnn_layers \u001b[38;5;241m=\u001b[39m \u001b[43mbuild_from_graph\u001b[49m\u001b[43m(\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;66;43;03m# build model from graph edges.\u001b[39;49;00m\n\u001b[1;32m 88\u001b[0m \u001b[43m \u001b[49m\u001b[43mdiscretize\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdiscretize\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 89\u001b[0m \u001b[43m \u001b[49m\u001b[43medges\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msinabs_edges\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 90\u001b[0m \u001b[43m \u001b[49m\u001b[43mnodes_to_dcnnl_map\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mnodes_to_dcnnl_map\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 92\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m dcnnl_idx, dcnnl_data \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnodes_to_dcnnl_map\u001b[38;5;241m.\u001b[39mitems():\n\u001b[1;32m 93\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mDynapcnnLayer-index \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mdcnnl_idx\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m)\n", - "File \u001b[0;32m~/Github/sinabs/sinabs/backend/dynapcnn/utils.py:595\u001b[0m, in \u001b[0;36mbuild_from_graph\u001b[0;34m(discretize, edges, nodes_to_dcnnl_map)\u001b[0m\n\u001b[1;32m 577\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mbuild_from_graph\u001b[39m(\n\u001b[1;32m 578\u001b[0m discretize: \u001b[38;5;28mbool\u001b[39m,\n\u001b[1;32m 579\u001b[0m edges: List[Tuple[\u001b[38;5;28mint\u001b[39m, \u001b[38;5;28mint\u001b[39m]],\n\u001b[1;32m 580\u001b[0m nodes_to_dcnnl_map: \u001b[38;5;28mdict\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28mdict\u001b[39m:\n\u001b[1;32m 581\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\" Parses each edge of a 'sinabs_mode.spiking_model' computational graph. Each node (layer) is assigned to a \u001b[39;00m\n\u001b[1;32m 582\u001b[0m \u001b[38;5;124;03m DynapcnnLayer object. The target destination of each DynapcnnLayer is computed via edges connecting nodes in \u001b[39;00m\n\u001b[1;32m 583\u001b[0m \u001b[38;5;124;03m different DynapcnnLayer objects.\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 591\u001b[0m \u001b[38;5;124;03m ...\u001b[39;00m\n\u001b[1;32m 592\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[0;32m--> 595\u001b[0m dynapcnn_layers \u001b[38;5;241m=\u001b[39m \u001b[43mconstruct_dynapcnnlayers_from_mapper\u001b[49m\u001b[43m(\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;66;43;03m# turn sets of layers into DynapcnnLayer objects.\u001b[39;49;00m\n\u001b[1;32m 596\u001b[0m \u001b[43m \u001b[49m\u001b[43mdiscretize\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnodes_to_dcnnl_map\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43medges\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 598\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m idx, layer_data \u001b[38;5;129;01min\u001b[39;00m dynapcnn_layers\u001b[38;5;241m.\u001b[39mitems():\n\u001b[1;32m 599\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mcore_idx\u001b[39m\u001b[38;5;124m'\u001b[39m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m layer_data:\n", - "File \u001b[0;32m~/Github/sinabs/sinabs/backend/dynapcnn/utils.py:622\u001b[0m, in \u001b[0;36mconstruct_dynapcnnlayers_from_mapper\u001b[0;34m(discretize, nodes_to_dcnnl_map, edges)\u001b[0m\n\u001b[1;32m 619\u001b[0m dynapcnn_layers \u001b[38;5;241m=\u001b[39m {}\n\u001b[1;32m 621\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m dpcnnl_idx, dcnnl_data \u001b[38;5;129;01min\u001b[39;00m nodes_to_dcnnl_map\u001b[38;5;241m.\u001b[39mitems():\n\u001b[0;32m--> 622\u001b[0m dynapcnnlayer \u001b[38;5;241m=\u001b[39m \u001b[43mconstruct_dynapcnnlayer\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 623\u001b[0m \u001b[43m \u001b[49m\u001b[43mdiscretize\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdcnnl_data\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43medges\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnodes_to_dcnnl_map\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 625\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m> \u001b[39m\u001b[38;5;124m'\u001b[39m, dpcnnl_idx)\n\u001b[1;32m 627\u001b[0m dynapcnn_layers[dpcnnl_idx] \u001b[38;5;241m=\u001b[39m {\n\u001b[1;32m 628\u001b[0m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mlayer\u001b[39m\u001b[38;5;124m'\u001b[39m: dynapcnnlayer, \n\u001b[1;32m 629\u001b[0m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mdestinations\u001b[39m\u001b[38;5;124m'\u001b[39m: nodes_to_dcnnl_map[dpcnnl_idx][\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mdestinations\u001b[39m\u001b[38;5;124m'\u001b[39m]\n\u001b[1;32m 630\u001b[0m }\n", - "File \u001b[0;32m~/Github/sinabs/sinabs/backend/dynapcnn/utils.py:653\u001b[0m, in \u001b[0;36mconstruct_dynapcnnlayer\u001b[0;34m(discretize, dcnnl_data, edges, nodes_to_dcnnl_map, layer_index)\u001b[0m\n\u001b[1;32m 650\u001b[0m convert_Avg_to_Sum_pooling(dcnnl_data, edges, nodes_to_dcnnl_map)\n\u001b[1;32m 652\u001b[0m \u001b[38;5;66;03m# instantiate a DynapcnnLayer from the data in 'dcnnl_data'.\u001b[39;00m\n\u001b[0;32m--> 653\u001b[0m dynapcnnlayer \u001b[38;5;241m=\u001b[39m \u001b[43mDynapcnnLayer\u001b[49m\u001b[43m(\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;66;43;03m# TODO 'rescale_weight' information is inside 'dcnnl_data' but it is not yet being used.\u001b[39;49;00m\n\u001b[1;32m 654\u001b[0m \u001b[43m \u001b[49m\u001b[43mdcnnl_data\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mdcnnl_data\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 655\u001b[0m \u001b[43m \u001b[49m\u001b[43mdiscretize\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mdiscretize\u001b[49m\n\u001b[1;32m 656\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 658\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m dynapcnnlayer\n", - "File \u001b[0;32m~/Github/sinabs/sinabs/backend/dynapcnn/dynapcnn_layer_new.py:92\u001b[0m, in \u001b[0;36mDynapcnnLayer.__init__\u001b[0;34m(self, dcnnl_data, discretize, rescale_weights)\u001b[0m\n\u001b[1;32m 89\u001b[0m spk\u001b[38;5;241m.\u001b[39mv_mem \u001b[38;5;241m=\u001b[39m spk\u001b[38;5;241m.\u001b[39mv_mem\u001b[38;5;241m.\u001b[39mdata\u001b[38;5;241m.\u001b[39munsqueeze(\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m)\u001b[38;5;241m.\u001b[39munsqueeze(\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m) \u001b[38;5;66;03m# expand dims.\u001b[39;00m\n\u001b[1;32m 91\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(conv, nn\u001b[38;5;241m.\u001b[39mLinear):\n\u001b[0;32m---> 92\u001b[0m conv \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_convert_linear_to_conv\u001b[49m\u001b[43m(\u001b[49m\u001b[43mconv\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdcnnl_data\u001b[49m\u001b[43m[\u001b[49m\u001b[43mconv_node_id\u001b[49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 93\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 94\u001b[0m conv \u001b[38;5;241m=\u001b[39m deepcopy(conv)\n", - "File \u001b[0;32m~/Github/sinabs/sinabs/backend/dynapcnn/dynapcnn_layer_new.py:134\u001b[0m, in \u001b[0;36mDynapcnnLayer._convert_linear_to_conv\u001b[0;34m(self, lin, layer_data)\u001b[0m\n\u001b[1;32m 130\u001b[0m \u001b[38;5;28mprint\u001b[39m(layer_data[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124minput_shape\u001b[39m\u001b[38;5;124m'\u001b[39m], lin\u001b[38;5;241m.\u001b[39min_features)\n\u001b[1;32m 132\u001b[0m input_shape \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mtuple\u001b[39m(\u001b[38;5;28mlist\u001b[39m(layer_data[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124minput_shape\u001b[39m\u001b[38;5;124m'\u001b[39m])[\u001b[38;5;241m1\u001b[39m:\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m]) \u001b[38;5;66;03m# removing the batch dimension.\u001b[39;00m\n\u001b[0;32m--> 134\u001b[0m in_chan, in_h, in_w \u001b[38;5;241m=\u001b[39m input_shape\n\u001b[1;32m 136\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m lin\u001b[38;5;241m.\u001b[39min_features \u001b[38;5;241m!=\u001b[39m in_chan \u001b[38;5;241m*\u001b[39m in_h \u001b[38;5;241m*\u001b[39m in_w:\n\u001b[1;32m 137\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mShapes don\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mt match.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", - "\u001b[0;31mValueError\u001b[0m: not enough values to unpack (expected 3, got 0)" + "name": "stdout", + "output_type": "stream", + "text": [ + "Network is valid\n" ] + }, + { + "data": { + "text/plain": [ + "DynapcnnNetworkGraph()" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ - "hw_model = DynapcnnNetworkGraph(\n", - " snn,\n", - " discretize=True,\n", - " input_shape=input_shape\n", - ")" + "hw_model.to(device=\"speck2edevkit:0\")" ] } ], From c9e11546aaad3fcfc2a432905360bbe9b27f500d Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Tue, 23 Apr 2024 13:11:41 +0200 Subject: [PATCH 035/379] in-line code documentation updated --- sinabs/backend/dynapcnn/NIRGraphExtractor.py | 274 +++++++++++------- .../dynapcnn/dynapcnn_network_graph.py | 58 ++-- .../backend/dynapcnn/sinabs_edges_handler.py | 82 ++++-- sinabs/backend/dynapcnn/utils.py | 58 ++-- 4 files changed, 292 insertions(+), 180 deletions(-) diff --git a/sinabs/backend/dynapcnn/NIRGraphExtractor.py b/sinabs/backend/dynapcnn/NIRGraphExtractor.py index a331bbff..b2af6b41 100644 --- a/sinabs/backend/dynapcnn/NIRGraphExtractor.py +++ b/sinabs/backend/dynapcnn/NIRGraphExtractor.py @@ -3,25 +3,109 @@ import nirtorch import copy import sinabs -from typing import Tuple, Dict +from typing import Tuple, Dict, List, Union class NIRtoDynapcnnNetworkGraph(): - def __init__(self, spiking_model, dummy_input) -> None: - """ .""" + def __init__(self, spiking_model: nn.Module, dummy_input: torch.tensor): + """ Class implementing the extraction of the computational graph from `spiking_model`, where + each node represents a layer in the model and the list of edges represents how the data flow between + the layers. + + Parameters + ---------- + spiking_model (nn.Module): a sinabs-compatible spiking network. + dummy_input (torch.tensor): a random input sample to be fed through the model to acquire both + the computational graph (via `nirtorch`) and the I/O shapes of each node. Its a 4-D shape + with `(batch, channels, heigh, width)`. + """ + # extract computational graph. nir_graph = nirtorch.extract_torch_graph(spiking_model, dummy_input, model_name=None).ignore_tensors() - self.edges_list, self.name_2_indx_map = self.get_edges_from_nir(nir_graph) + # converts the NIR representation into a list of edges with nodes represented as integers. + self._edges_list, self._name_2_indx_map = self._get_edges_from_nir(nir_graph) - self.modules_map = self.get_named_modules(spiking_model) + # recovers the associated `nn.Module` (layer) of each node. + self.modules_map = self._get_named_modules(spiking_model) + + # retrieves what the I/O shape for each node's module is. + self._nodes_io_shapes = self._get_nodes_io_shapes(dummy_input) + + ### Publich Methods ### + + def remove_ignored_nodes(self, default_ignored_nodes): + """ Recreates the edges list based on layers that 'DynapcnnNetwork' will ignore. This + is done by setting the source (target) node of an edge where the source (target) node + will be dropped as the node that originally targeted this node to be dropped. + """ + edges = copy.deepcopy(self._edges_list) + parsed_edges = [] + removed_nodes = [] + + # removing ignored nodes from edges. + for edge_idx in range(len(edges)): + _src = edges[edge_idx][0] + _trg = edges[edge_idx][1] + + if isinstance(self.modules_map[_src], default_ignored_nodes): + removed_nodes.append(_src) + # all edges where node '_src' is target change it to node '_trg' as their target. + for edge in edges: + if edge[1] == _src: + new_edge = (edge[0], _trg) + elif isinstance(self.modules_map[_trg], default_ignored_nodes): + removed_nodes.append(_trg) + # all edges where node '_trg' is source change it to node '_src' as their source. + for edge in edges: + if edge[0] == _trg: + new_edge = (_src, edge[1]) + else: + new_edge = (_src, _trg) + + if new_edge not in parsed_edges: + parsed_edges.append(new_edge) + + removed_nodes = list(set(removed_nodes)) - self.nodes_io_shapes = self.get_nodes_io_shapes(dummy_input) + # remapping nodes indexes. + remapped_nodes = {} + for node_indx, __ in self.modules_map.items(): + _ = [x for x in removed_nodes if node_indx > x] + remapped_nodes[node_indx] = node_indx - len(_) + + for x in removed_nodes: + del remapped_nodes[x] - def get_edges_from_nir(self, nir_graph): - """ .""" + # remapping nodes names in parsed edges. + remapped_edges = [] + for edge in parsed_edges: + remapped_edges.append((remapped_nodes[edge[0]], remapped_nodes[edge[1]])) + + return remapped_edges, remapped_nodes + + def get_node_io_shapes(self, node: int) -> Tuple[torch.Size, torch.Size]: + """ Returns the I/O tensors' shapes of `node`. """ + return self._nodes_io_shapes[node]['input'], self._nodes_io_shapes[node]['output'] + + ### Pivate Methods ### + + def _get_edges_from_nir(self, nir_graph: nirtorch.graph.Graph) -> Union[List[Tuple[int, int]], Dict[str, int]]: + """ Standardize the representation of `nirtorch.graph.Graph` into a list of edges (`Tuple[int, int]`) where + each node in `nir_graph` is represented by an interger (with the source node starting as `0`). + + Parameters + ---------- + nir_graph (nirtorch.graph.Graph): a NIR graph representation of `spiking_model`. + + Returns + ---------- + edges_list (list): tuples describing the connections between layers in `spiking_model`. + name_2_indx_map (dict): `key` is the original variable name for a layer in `spiking_model` and `value + is an integer representing the layer in a standard format. + """ edges_list = [] name_2_indx_map = {} - idx_counter = 0 + idx_counter = 0 # TODO maybe make sure the input node from nir always gets assined `0`. for src_node in nir_graph.node_list: # source node. if src_node.name not in name_2_indx_map: @@ -37,30 +121,45 @@ def get_edges_from_nir(self, nir_graph): return edges_list, name_2_indx_map - def get_named_modules(self, model): - """ .""" + def _get_named_modules(self, model: nn.Module) -> Dict[int, nn.Module]: + """ Find for each node in the graph what its associated layer in `model` is. + + Parameters + ---------- + model (nn.Module): the `spiking_model` used as argument to the class instance. + + Returns + ---------- + modules_map (dict): the mapping between a node (`key` as an `int`) and its module (`value` as a `nn.Module`). + """ modules_map = {} - if isinstance(model, nn.Sequential): # access modules via `.named_modules()`. + if isinstance(model, nn.Sequential): # TODO shouldn't accept `nn.Sequential` any longer. + # access modules via `.named_modules()`. for name, module in model.named_modules(): - if name != '': # skip the module itself. - modules_map[self.name_2_indx_map[name]] = module + if name != '': + # skip the module itself. + modules_map[self._name_2_indx_map[name]] = module - elif isinstance(model, nn.Module): # access modules via `.named_children()`. + elif isinstance(model, nn.Module): + # access modules via `.named_children()`. for name, module in model.named_children(): - modules_map[self.name_2_indx_map[name]] = module + modules_map[self._name_2_indx_map[name]] = module else: raise ValueError('Either a nn.Sequential or a nn.Module is required.') return modules_map - def get_nodes_io_shapes(self, input_dummy) -> Dict[int, Dict[str, torch.Size]]: - """ .""" + def _get_nodes_io_shapes(self, input_dummy: torch.tensor) -> Dict[int, Dict[str, torch.Size]]: + """ Loops through the graph represented in `self._edges_list` and propagates the inputs through the nodes, starting from + `node 0` fed `input_dummy`. + """ nodes_io_map = {} flagged_merge_nodes = {} - for edge in self.edges_list: + # propagate inputs through the nodes. + for edge in self._edges_list: src = edge[0] trg = edge[1] @@ -71,25 +170,30 @@ def get_nodes_io_shapes(self, input_dummy) -> Dict[int, Dict[str, torch.Size]]: if trg not in nodes_io_map: nodes_io_map[trg] = {'input': None, 'output': None} - inp_node = self.find_input_to_node(trg) # find node generating the input to be used. + # find node generating the input to be used. + inp_node = self._find_input_to_node(trg) _input = nodes_io_map[inp_node]['output'] - _output = self.modules_map[trg](_input) # forward input through the node. + # forward input through the node. + _output = self.modules_map[trg](_input) - nodes_io_map[trg] = {'input': _input, 'output': _output} # save node's input/output. + # save node's input/output. + nodes_io_map[trg] = {'input': _input, 'output': _output} elif isinstance(self.modules_map[trg], sinabs.layers.merge.Merge): # Merge requires two inputs: need to check if both of its inputs have been calculated. if trg not in flagged_merge_nodes: flagged_merge_nodes[trg] = {} - args = self.find_merge_arguments(trg) + args = self._find_merge_arguments(trg) for arg in args: - if arg in nodes_io_map: # one input to Merge has been computed. + if arg in nodes_io_map: + # one input to Merge has been computed. flagged_merge_nodes[trg][arg] = nodes_io_map[arg] - if len(flagged_merge_nodes[trg]) == 2: # both arguments to Merge have been computed. + if len(flagged_merge_nodes[trg]) == 2: + # both arguments to Merge have been computed. if trg not in nodes_io_map: nodes_io_map[trg] = {'input': None, 'output': None} @@ -97,7 +201,8 @@ def get_nodes_io_shapes(self, input_dummy) -> Dict[int, Dict[str, torch.Size]]: nodes_io_map[args[0]]['output'], nodes_io_map[args[1]]['output']) - _input = torch.max(torch.stack([ # Merge expands each input dim. into the max of that dim. between input tensors. + # Merge expands each input dim. into the max of that dim. between input tensors. + _input = torch.max(torch.stack([ nodes_io_map[args[0]]['output'], nodes_io_map[args[1]]['output']]), dim=0) @@ -109,122 +214,73 @@ def get_nodes_io_shapes(self, input_dummy) -> Dict[int, Dict[str, torch.Size]]: nodes_io_map[src] = {'input': None, 'output': None} if src == 0: - _input = input_dummy # first node in the graph. + # first node in the graph. + _input = input_dummy + else: - inp_node = self.find_input_to_node(src) # find node generating the input to be used. + # find node generating the input to be used. + inp_node = self._find_input_to_node(src) _input = nodes_io_map[inp_node]['output'] - _output = self.modules_map[src](_input) # forward input through the node. + # forward input through the node. + _output = self.modules_map[src](_input) - nodes_io_map[src] = {'input': _input, 'output': _output} # save node's input/output. + # save node's input/output. + nodes_io_map[src] = {'input': _input, 'output': _output} else: - # pass input through source. if src not in nodes_io_map: nodes_io_map[src] = {'input': None, 'output': None} if src == 0: - _input = input_dummy # first node in the graph. + # first node in the graph. + _input = input_dummy else: - inp_node = self.find_input_to_node(src) # find node generating the input to be used. + # find node generating the input to be used. + inp_node = self._find_input_to_node(src) _input = nodes_io_map[inp_node]['output'] - _output = self.modules_map[src](_input) # forward input through the node. + # forward input through the node. + _output = self.modules_map[src](_input) - nodes_io_map[src] = {'input': _input, 'output': _output} # save node's input/output. + # save node's input/output. + nodes_io_map[src] = {'input': _input, 'output': _output} # pass input through target. if trg not in nodes_io_map: nodes_io_map[trg] = {'input': None, 'output': None} - inp_node = self.find_input_to_node(trg) # find node generating the input to be used. + # find node generating the input to be used. + inp_node = self._find_input_to_node(trg) _input = nodes_io_map[inp_node]['output'] - _output = self.modules_map[trg](_input) # forward input through the node. - - nodes_io_map[trg] = {'input': _input, 'output': _output} # save node's input/output. + # forward input through the node. + _output = self.modules_map[trg](_input) + + # save node's input/output. + nodes_io_map[trg] = {'input': _input, 'output': _output} + # replace the I/O tensor information by its shape information. for node, io in nodes_io_map.items(): nodes_io_map[node]['input'] = io['input'].shape nodes_io_map[node]['output'] = io['output'].shape return nodes_io_map - def find_input_to_node(self, node): - """ .""" - for edge in self.edges_list: + def _find_input_to_node(self, node: int) -> int: + """ Finds the first edge `(X, node)` returns `X`. """ + for edge in self._edges_list: if edge[1] == node: return edge[0] - return -1 - - def find_node_variable_name(self, node): - """ .""" - for key, val in self.name_2_indx_map.items(): - if val == node: - return key - return None - - def find_merge_arguments(self, merge_node): - """ .""" + raise ValueError(f'Node {node} is not the target node of any edge in the graph.') + + def _find_merge_arguments(self, merge_node: int) -> list: + """ A `Merge` layer receives two inputs. Return the two inputs to `merge_node` representing a `Merge` layer. """ args = [] - for edge in self.edges_list: + for edge in self._edges_list: if edge[1] == merge_node: args.append(edge[0]) if len(args) == 2: break - return args - - def remove_ignored_nodes(self, default_ignored_nodes): - """ Recreates the edges list based on layers that 'DynapcnnNetwork' will ignore. This - is done by setting the source (target) node of an edge where the source (target) node - will be dropped as the node that originally targeted this node to be dropped. - """ - edges = copy.deepcopy(self.edges_list) - parsed_edges = [] - removed_nodes = [] - - # removing ignored nodes from edges. - for edge_idx in range(len(edges)): - _src = edges[edge_idx][0] - _trg = edges[edge_idx][1] - - if isinstance(self.modules_map[_src], default_ignored_nodes): - removed_nodes.append(_src) - # all edges where node '_src' is target change it to node '_trg' as their target. - for edge in edges: - if edge[1] == _src: - new_edge = (edge[0], _trg) - elif isinstance(self.modules_map[_trg], default_ignored_nodes): - removed_nodes.append(_trg) - # all edges where node '_trg' is source change it to node '_src' as their source. - for edge in edges: - if edge[0] == _trg: - new_edge = (_src, edge[1]) - else: - new_edge = (_src, _trg) - - if new_edge not in parsed_edges: - parsed_edges.append(new_edge) - - removed_nodes = list(set(removed_nodes)) - - # remapping nodes indexes. - remapped_nodes = {} - for node_indx, __ in self.modules_map.items(): - _ = [x for x in removed_nodes if node_indx > x] - remapped_nodes[node_indx] = node_indx - len(_) - - for x in removed_nodes: - del remapped_nodes[x] - - # remapping nodes names in parsed edges. - remapped_edges = [] - for edge in parsed_edges: - remapped_edges.append((remapped_nodes[edge[0]], remapped_nodes[edge[1]])) - - return remapped_edges, remapped_nodes - - def get_node_io_shapes(self, node) -> Tuple[torch.Size, torch.Size]: - """ .""" - return self.nodes_io_shapes[node]['input'], self.nodes_io_shapes[node]['output'] + return args \ No newline at end of file diff --git a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py index 172f4dd7..f1510692 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py @@ -39,31 +39,23 @@ class DynapcnnNetworkGraph(nn.Module): def __init__( self, - snn: Union[nn.Sequential, sinabs.Network, nn.Module], - input_shape: Optional[Tuple[int, int, int]] = None, + snn: nn.Module, + input_shape: Tuple[int, int, int], dvs_input: bool = False, discretize: bool = True ): """ - DynapcnnNetworkGraph: a class turning sinabs networks into dynapcnn - compatible networks, and making dynapcnn configurations. - Parameters ---------- - snn: ... - input_shape: None or tuple of ints - Shape of the input, convention: (features, height, width) - If None, `snn` needs an InputLayer - dvs_input: bool - Does dynapcnn receive input from its DVS camera? - discretize: bool - If True, discretize the parameters and thresholds. - This is needed for uploading weights to dynapcnn. Set to False only for - testing purposes. + snn : a `nn.Module` implementing a spiking network. + input_shape: a description of the input dimensions (features, height, width). + dvs_input: wether or not dynapcnn receive input from its DVS camera. + discretize: If `True`, discretize the parameters and thresholds. This is needed for uploading + weights to dynapcnn. Set to `False` only for testing purposes. """ super().__init__() - # TODO for now the graph part is not taking into consideration this. + # TODO for now the graph part is not taking into consideration DVS inputs. # check if dvs input is expected. dvs_input = False self.dvs_input = dvs_input @@ -80,11 +72,12 @@ def __init__( self.sinabs_modules_map, \ self.nodes_name_remap = self.get_sinabs_edges_and_modules() + # create a dict holding the data necessary to instantiate a `DynapcnnLayer`. self.nodes_to_dcnnl_map = build_nodes_to_dcnnl_map( layers=self.sinabs_modules_map, edges=self.sinabs_edges) - # update the I/O shapes for each layer in 'self.nodes_to_dcnnl_map'. + # updates 'self.nodes_to_dcnnl_map' to include the I/O shape for each node. self.populate_nodes_io() # build model from graph edges. @@ -336,7 +329,7 @@ def _make_config( # update config. config = config_builder.build_config(self, None) - if self.input_shape and self.input_shape[0] == 1: # ??? + if self.input_shape and self.input_shape[0] == 1: # TODO not handling DVSLayer yet (this is from the old implementation, should be revised). config.dvs_layer.merge = True # TODO all this monitoring part needs validation still. @@ -366,37 +359,48 @@ def _make_config( return config, config_builder.validate_configuration(config) - def get_sinabs_edges_and_modules(self) -> Tuple[List[Tuple[int, int]], Dict[int, nn.Module]]: + def get_sinabs_edges_and_modules(self) -> Tuple[List[Tuple[int, int]], Dict[int, nn.Module], Dict[int, int]]: """ The computational graph extracted from `snn` might contain layers that are ignored (e.g. a `nn.Flatten` will be ignored when creating a `DynapcnnLayer` instance). Thus the list of edges from such model need to be rebuilt such that if there are edges `(A, X)` and `(X, B)`, and `X` is an ignored layer, an edge `(A, B)` is created. Returns ---------- - sinabs_edges: a list of tuples representing the edges between the layers of a sinabs model. - sinabs_modules_map: a dictionary containing the nodes of the graph as `key` and their associated module as `value`. + edges_without_merge (list): a list of edges based on `sinabs_edges` but where edges involving a `Merge` layer have been + remapped to connect the nodes involved in the merging directly. + sinabs_modules_map (dict): a dict containing the nodes of the graph (described now by `edges_without_merge`) as `key` and + their associated module as `value`. + remapped_nodes (dict): a dict where `key` is the original node name (as extracted by `self.graph_tracer`) and `value` is + the new node name (after ignored layers have been dropped and `Merge` layers have be processed before being removed). """ - sinabs_edges, remapped_nodes = self.graph_tracer.remove_ignored_nodes( # remap `(A, X)` and `(X, B)` into `(A, B)`. + + # remap `(A, X)` and `(X, B)` into `(A, B)` if `X` is a layer in the original `snn` to be ignored. + sinabs_edges, remapped_nodes = self.graph_tracer.remove_ignored_nodes( DEFAULT_IGNORED_LAYER_TYPES) - sinabs_modules_map = {} # nodes (layers' "names") need remapping in case some layers have been removed (e.g. nn.Flattern()). + # nodes (layers' "names") need remapping in case some layers have been removed (e.g. a `nn.Flattern` is ignored). + sinabs_modules_map = {} for orig_name, new_name in remapped_nodes.items(): sinabs_modules_map[new_name] = self.graph_tracer.modules_map[orig_name] - edges_without_merge, merge_nodes = merge_handler(sinabs_edges, sinabs_modules_map) # bypass merging layers to connect the nodes involved in them directly to the node where the merge happens. + # bypass merging layers to connect the nodes involved in them directly to the node where the merge happens. + edges_without_merge = merge_handler(sinabs_edges, sinabs_modules_map) return edges_without_merge, sinabs_modules_map, remapped_nodes def populate_nodes_io(self): - """ .""" + """ Loops through the nodes in the original graph to retrieve their I/O tensor shapes and add them to their respective + representations in `self.nodes_to_dcnnl_map`.""" - def find_original_node_name(name_mapper, node): + def find_original_node_name(name_mapper: dict, node: int): + """ Find what a node is originally named when built in `self.graph_tracer`. """ for orig_name, new_name in name_mapper.items(): if new_name == node: return orig_name raise ValueError(f'Node {node} could not be found within the name remapping done by self.get_sinabs_edges_and_modules().') - def find_my_input(edges_list, node): + def find_my_input(edges_list: list, node: int): + """ Returns the node `X` in the first edge `(X, node)`.""" for edge in edges_list: if edge[1] == node: # TODO nodes originally receiving input from merge will appear twice in the list of edges, one diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index b84e24fe..a57439ca 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -12,15 +12,25 @@ import sinabs, copy def process_edge(layers: Dict[int, nn.Module], edge: Tuple[int, int], mapper: dict) -> None: - """ Read in an edge describing the connection between two layers (nodes in the computational graph). If 'edge' - is a valid connection between two layers, update 'mapper' to incorporate these layers into a new or existing dictonary - containing the modules comprising a future DynacnnLayer object. + """ Read in an edge describing the connection between two layers (nodes in the computational graph). If `edge` + is a valid connection between two layers, update `mapper` to incorporate these layers into a new or existing dictonary + containing the modules comprising a future `DynacnnLayer` object. + + After of call of this function `mapper` is updated to incorporate a set of nodes into the data required to create a + `DynapcnnLayer` instance. For example, after processing the 1st edge `(0, 1)`, an entry `0` for a future `DynapcnnLayer` is + created and its set of nodes will include node `0` and node `1`: + + mapper[0] = { + 0: {'layer': Conv2d, 'input_shape': None, 'output_shape': None}, + 1: {'layer': IAFSqueeze, 'input_shape': None, 'output_shape': None}, + ... + } Parameters ---------- - layers : a dictionary containing the nodes of the graph as `key` and their associated module as `value`. - edge : tuple representing the connection between two nodes in computational graph of 'DynapcnnNetworkGraph.snn.spiking_model'. - mapper : dictionary where each 'key' is the index of a future 'DynapcnnLayer' and 'value' is a dictionary ('key': node, 'value': module). + layers (dict): a dictionary containing the nodes of the graph as `key` and their associated module as `value`. + edge (tuple): tuple representing the connection between two nodes in computational graph of 'DynapcnnNetworkGraph.snn.spiking_model'. + mapper (dict): dictionary where each 'key' is the index of a future 'DynapcnnLayer' and 'value' is a dictionary ('key': node, 'value': module). """ edge_type = is_valid_edge(edge, layers, VALID_SINABS_EDGES) @@ -146,11 +156,23 @@ def get_dynapcnnlayers_destinations(layers: Dict[int, nn.Module], edges: List[Tu DynapcnnLayer they belong to. If source and target belong to different DynapcnnLayers (described as a dictionary in 'mapper') the destination of the 'DynapcnnLayer.source' is set to be 'DynapcnnLayer.target'. + After one call of this function an attribute `destination` is added to an entry in `mapper` to save the indexes (a different `key` + in `mapper`) of `DynapcnnLayer`s targeted by another `DynapcnnLayer`. For example, if in an edge `(1, 4)` the node `1` belongs to + `mapper[0]` and node `4` belongs to `mapper[2]`, the former is updated to tager the latter, like the following: + + mapper[0] = { + 0: {'layer': Conv2d, ...}, + 1: {'layer': IAFSqueeze, ...}, # node `1` in edge `(1, 4)` belongs to `mapper[0]`... + ... + 'destinations': [2], # ... so DynacnnLayer built from `mapper[2]` is destination of DynapcnnLayer built from `mapper[0]`. + ... + } + Parameters ---------- - layers : a dictionary containing the nodes of the graph as `key` and their associated module as `value`. - edges : list of tuples representing the connection between nodes in computational graph of 'DynapcnnNetworkGraph.snn.spiking_model'. - mapper : dictionary where each 'key' is the index of a future 'DynapcnnLayer' and 'value' its modules (output of 'process_edge(mapper)'). + layers (dict): contains the nodes of the graph as `key` and their associated module as `value`. + edges (list): tuples representing the connection between nodes in computational graph spiking network. + mapper (dict): each 'key' is the index of a future `DynapcnnLayer` and `value` the data necessary to instantiate it. Returns ---------- @@ -182,10 +204,7 @@ def get_dynapcnnlayers_destinations(layers: Dict[int, nn.Module], edges: List[Tu for dcnnl_idx, destinations in dynapcnnlayers_destinations_map.items(): # TODO document the 'rescale_factor' better. mapper[dcnnl_idx]['destinations'] = destinations - mapper[dcnnl_idx]['destinations_rescale_factor'] = {} # needed for when SumPool is built. - - for dest in destinations: - mapper[dcnnl_idx]['destinations_rescale_factor'][dest] = 1 + mapper[dcnnl_idx]['conv_rescale_factor'] = 1 def get_dynapcnnlayer_index(node: int, mapper: dict) -> int: """ Returns the DynapcnnLayer index to which 'node' belongs to. """ @@ -203,50 +222,59 @@ def is_valid_dynapcnnlayer_pairing(layers: Dict[int, nn.Module], edge: Tuple[int def merge_handler(sinabs_edges: List[Tuple[int, int]], sinabs_modules_map: Dict[int, nn.Module]) -> List[Tuple[int, int]]: """ Handles connections between nodes made via a `sinabs.layers.Merge` layer. If `X` is a merge layer then edges `(X, C)` are removed - from the edges list since they don't affect the creationg of DynapcnnLayers. Edges `(Y, X)` are turned into a edge `(Y, C)` pointing - directly to the node receiving the merged inputs such that the DynapcnnLayer containing `Y` can have the DynapcnnLayer containing `C` + from the edges list since they don't affect the creationg of `DynapcnnLayer`s. Edges `(Y, X)` are turned into a edge `(Y, C)` pointing + directly to the node receiving the merged inputs such that the `DynapcnnLayer` containing `Y` can have the `DynapcnnLayer` containing `C` as one of its destinations. Parameters ---------- - sinabs_edges: ... - sinabs_modules_map: ... + sinabs_edges (list): edges extracted from the computational graph of the network provided to `DynapcnnNetworkGraph` where edges involving + layers that are ignored (e.g. `nn.Flatten`) have been removed (the nodes previously linked via dropped nodes are linked to each other directly). + sinabs_modules_map (dict): mapping where the `key` represents a node in the graph and the `value` represents the node's layer. - Reurns - edges_without_merge: ... + Returns ---------- + edges_without_merge (list): edges based on `sinabs_edges` but where edges involving a `Merge` layer have been remapped to connect the nodes + involved in the merging directly. """ edges = copy.deepcopy(sinabs_edges) edges_without_merge = [] merge_nodes = {} - for edge in edges: # finding the nodes representing Merge layers. + # finding the nodes representing Merge layers. + for edge in edges: src = edge[0] trg = edge[1] if isinstance(sinabs_modules_map[src], sinabs.layers.Merge): - if src not in merge_nodes: # found node receiving merged inputs from two previous layers. + if src not in merge_nodes: + # found node receiving merged inputs from two previous layers. merge_nodes[src] = {'sources': [], 'merge_into': trg} for _edge in edges: - if _edge[1] == src: # found node used as argument for a Merge layer. + if _edge[1] == src: + # found node used as argument for a Merge layer. merge_nodes[src]['sources'].append(_edge[0]) if len(merge_nodes[src]['sources']) > 2: raise ValueError("A Merge layer can not have more than two inputs.") - for edge in edges: # removing edges connection from/to merging layers from the computational graph. + for edge in edges: + # removing edges connection from/to merging layers from the computational graph. src = edge[0] trg = edge[1] - if src in merge_nodes: # edge (`Merge`, trg) is not necessary for later DynapcnnLayer creation. + if src in merge_nodes: + # edge (`Merge`, trg) is not necessary for later DynapcnnLayer creation. pass - elif trg in merge_nodes: # point `src` directly to the node it was previously targeting via a Merge layer. + elif trg in merge_nodes: + # point `src` directly to the node it was previously targeting via a Merge layer. new_edge = (src, merge_nodes[trg]['merge_into']) edges_without_merge.append(new_edge) - else: # edge not involved in merging. + else: + # edge not involved in merging. edges_without_merge.append(edge) - return edges_without_merge, merge_nodes + return edges_without_merge \ No newline at end of file diff --git a/sinabs/backend/dynapcnn/utils.py b/sinabs/backend/dynapcnn/utils.py index 2ee14b99..9768d6da 100644 --- a/sinabs/backend/dynapcnn/utils.py +++ b/sinabs/backend/dynapcnn/utils.py @@ -548,8 +548,23 @@ def extend_readout_layer(model: "DynapcnnNetwork") -> "DynapcnnNetwork": ) # run a forward pass to initialize the new weights and last IAF return model -def build_nodes_to_dcnnl_map(layers: Dict[int, nn.Module], edges: List[Tuple[int, int]]): - """ .""" +def build_nodes_to_dcnnl_map(layers: Dict[int, nn.Module], edges: List[Tuple[int, int]]) -> dict: + """ Initializes and populates a `dict` that will map data into a future `DynapcnnLayer` instance. The call + to `process_edge()` initializes a `key` (the index of a `DynapcnnLayer`) and assigns to it a dict containing the + nodes (layers in a `nn.Module`) that should belong to the same `DynapcnnLayer`. The call to `get_dynapcnnlayers_destinations()` + further incorporates to each "DynapcnnLayer dictionary" a `destinations` attribute, which is a list of integers indicating the + the target destinations of a `DynapcnnLayer` instance. + + Parameters + --------- + layers (dict): constains the nodes of a graph as `key` and their associated module as `value`. + edges (list): edges describing how nodes connect to each other. + + Returns + --------- + nodes_to_dcnnl_map (dict): each entry represents the gathered data necessary to instantiate a `DynapcnnLayer` object (e.g. nodes, + their I/O shapes, the list of `DynapcnnLayer` that are to be targeted, etc). + """ # @TODO the graph extraction is not yet considering DVS input. # dvs_layer, lyr_indx_next, rescale_factor = construct_dvs_layer( @@ -560,17 +575,19 @@ def build_nodes_to_dcnnl_map(layers: Dict[int, nn.Module], edges: List[Tuple[int dvs_layer = None - nodes_to_dcnnl_map = {} # mapper from nodes to sets of layers that populate a DynapcnnLayer. + # mapper from nodes to sets of layers that populate a DynapcnnLayer. + nodes_to_dcnnl_map = {} if dvs_layer is not None: - pass # TODO the graph extraction is not yet considering DVS input. + # TODO the graph extraction is not yet considering DVS input. + pass else: for edge in edges: - process_edge( # figure out to which (future) DynapcnnLayer each node will belong to. - layers, edge, nodes_to_dcnnl_map) + # Figure out to which (future) DynapcnnLayer each node will belong to. + process_edge(layers, edge, nodes_to_dcnnl_map) - get_dynapcnnlayers_destinations( # look for edges between connecting nodes in different (future) DynapcnnLayer. - layers, edges, nodes_to_dcnnl_map) + # look for edges between connecting nodes in different (future) DynapcnnLayer. + get_dynapcnnlayers_destinations(layers, edges, nodes_to_dcnnl_map) return nodes_to_dcnnl_map @@ -584,20 +601,24 @@ def build_from_graph( Parameters ---------- - ... + discretize (bool): ... + edges (list): edges describing how nodes connect to each other. + nodes_to_dcnnl_map (dict): each entry represents the gathered data necessary to instantiate a `DynapcnnLayer` object (e.g. nodes, + their I/O shapes, the list of `DynapcnnLayer` that are to be targeted, etc). Returns ---------- - ... + dynapcnn_layers (dict): `DynapcnnLayer` instances, each created from an entry in `nodes_to_dcnnl_map`. """ - - dynapcnn_layers = construct_dynapcnnlayers_from_mapper( # turn sets of layers into DynapcnnLayer objects. - discretize, nodes_to_dcnnl_map, edges) + # turn each entry in `nodes_to_dcnnl_map` into a `DynapcnnLayer` instance. + dynapcnn_layers = construct_dynapcnnlayers_from_mapper(discretize, nodes_to_dcnnl_map, edges) + # initialize attribute holding to which core a `DynapcnnLayer` instance in `dynapcnn_layers` will be mapped to. for idx, layer_data in dynapcnn_layers.items(): if 'core_idx' not in layer_data: - layer_data['core_idx'] = -1 # a DynapcnnLayer gets assigned a core index when 'DynapcnnNetworkGraph.to()' is called. + # a `DynapcnnLayer` gets assigned a core index when `DynapcnnNetworkGraph.to()`` is called. + layer_data['core_idx'] = -1 return dynapcnn_layers @@ -609,11 +630,14 @@ def construct_dynapcnnlayers_from_mapper( Parameters ---------- - rescale_factor: Rescaling factor needed when turning AvgPool to SumPool. May differ from the pooling kernel in - certain cases. + discretize (bool): ... + nodes_to_dcnnl_map (dict): each entry represents the gathered data necessary to instantiate a `DynapcnnLayer` object (e.g. nodes, + their I/O shapes, the list of `DynapcnnLayer` that are to be targeted, etc). + edges (list): edges describing how nodes connect to each other. + Returns ---------- - dynapcnn_layers: A dictionary containing DynapcnnLayer objects and a list with their destinations. + dynapcnn_layers (dict): `DynapcnnLayer` instances, each created from an entry in `nodes_to_dcnnl_map`. """ dynapcnn_layers = {} From 14e0ba14523302e35fb2429641bf7e1fcc35a4ba Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Tue, 23 Apr 2024 16:58:36 +0200 Subject: [PATCH 036/379] (WIP) creating nn.Module with custom forward from DynapcnnLayer instances and their connection graph --- sinabs/backend/dynapcnn/dynapcnn_layer_new.py | 28 +- .../dynapcnn/dynapcnn_network_graph.py | 32 +- .../dynapcnn/dynapcnnnetwork_module.py | 88 ++++ ...DynapcnnNetworkGraph_from_NIRgraph_1.ipynb | 291 ------------ ...DynapcnnNetworkGraph_from_NIRgraph_2.ipynb | 256 ----------- ...DynapcnnNetworkGraph_from_NIRgraph_3.ipynb | 302 ------------ ...DynapcnnNetworkGraph_from_NIRgraph_4.ipynb | 98 +++- ...est_DynapcnnNetworkGraph_1_skip_conn.ipynb | 435 ------------------ 8 files changed, 226 insertions(+), 1304 deletions(-) create mode 100644 sinabs/backend/dynapcnn/dynapcnnnetwork_module.py delete mode 100644 tests/test_nonsequential/DynapcnnNetworkGraph_from_NIRgraph_1.ipynb delete mode 100644 tests/test_nonsequential/DynapcnnNetworkGraph_from_NIRgraph_2.ipynb delete mode 100644 tests/test_nonsequential/DynapcnnNetworkGraph_from_NIRgraph_3.ipynb delete mode 100644 tests/test_nonsequential/test_DynapcnnNetworkGraph_1_skip_conn.ipynb diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer_new.py b/sinabs/backend/dynapcnn/dynapcnn_layer_new.py index 3fc377fa..59c4c142 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer_new.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer_new.py @@ -12,7 +12,6 @@ from .discretize import discretize_conv_spike_ from .dvs_layer import expand_to_pair - class DynapcnnLayer(nn.Module): """Create a DynapcnnLayer object representing a dynapcnn layer. """ @@ -27,9 +26,8 @@ def __init__( Parameters ---------- - - TODO - 1) currently there's no way the forward would work since there are more than two poolings. + dcnnl_data (dict): ... + discretize (bool): ... """ self.lin_to_conv_conversion = False @@ -128,6 +126,18 @@ def __str__(self): pretty_print += f'\n(node {self.pool_node_id[idx]}): {lyr}' return pretty_print + + def forward(self, x): + """Torch forward pass.""" + + x = self.conv_layer(x) + x = self.spk_layer(x) + + if len(self.pool_layer) == 1: + # single pooling layer (not a divergent node). + x = self.pool_layer[0](x) + + return x def _convert_linear_to_conv(self, lin: nn.Linear, layer_data: dict) -> nn.Conv2d: """Convert Linear layer to Conv2d. @@ -343,12 +353,4 @@ def memory_summary(self): "neuron": f * pow(2, np.ceil(np.log2(neuron_height)) + np.ceil(np.log2(neuron_width))), "bias": 0 if self.conv_layer.bias is None else len(self.conv_layer.bias), - } - - def forward(self, x): # TODO deprecated. - """Torch forward pass.""" - x = self.conv_layer(x) - x = self.spk_layer(x) - if self.pool_layer is not None: - x = self.pool_layer(x) - return x \ No newline at end of file + } \ No newline at end of file diff --git a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py index f1510692..87a29da3 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py @@ -31,6 +31,8 @@ from .NIRGraphExtractor import NIRtoDynapcnnNetworkGraph from .sinabs_edges_handler import merge_handler +from .dynapcnnnetwork_module import DynapcnnNetworkModule + class DynapcnnNetworkGraph(nn.Module): """Given a sinabs spiking network, prepare a dynapcnn-compatible network. This can be used to test the network will be equivalent once on DYNAPCNN. This class also provides utilities to @@ -70,7 +72,7 @@ def __init__( self.sinabs_edges, \ self.sinabs_modules_map, \ - self.nodes_name_remap = self.get_sinabs_edges_and_modules() + self.nodes_name_remap = self._get_sinabs_edges_and_modules() # create a dict holding the data necessary to instantiate a `DynapcnnLayer`. self.nodes_to_dcnnl_map = build_nodes_to_dcnnl_map( @@ -78,9 +80,9 @@ def __init__( edges=self.sinabs_edges) # updates 'self.nodes_to_dcnnl_map' to include the I/O shape for each node. - self.populate_nodes_io() + self._populate_nodes_io() - # build model from graph edges. + # build `DynapcnnLayer` instances from graph edges and mapper. self.dynapcnn_layers = build_from_graph( discretize=discretize, edges=self.sinabs_edges, @@ -101,6 +103,24 @@ def __str__(self): pretty_print += f'\n> layer modules: {layer}\n> layer destinations: {dest}\n> assigned core: {core}\n\n' return pretty_print + def get_network_module(self): + """ .""" + + # get connections between `DynapcnnLayer`s. + dcnnl_edges = self._get_dynapcnnlayers_edges() + + network_module = DynapcnnNetworkModule(dcnnl_edges, self.dynapcnn_layers) + + def _get_dynapcnnlayers_edges(self) -> List[Tuple[int, int]]: + """ Create edges representing connections between `DynapcnnLayer` instances. """ + dcnnl_edges = [] + + for dcnnl_idx, layer_data in self.dynapcnn_layers.items(): + for dest in layer_data['destinations']: + dcnnl_edges.append((dcnnl_idx, dest)) + + return dcnnl_edges + def to( self, device="cpu", @@ -359,7 +379,7 @@ def _make_config( return config, config_builder.validate_configuration(config) - def get_sinabs_edges_and_modules(self) -> Tuple[List[Tuple[int, int]], Dict[int, nn.Module], Dict[int, int]]: + def _get_sinabs_edges_and_modules(self) -> Tuple[List[Tuple[int, int]], Dict[int, nn.Module], Dict[int, int]]: """ The computational graph extracted from `snn` might contain layers that are ignored (e.g. a `nn.Flatten` will be ignored when creating a `DynapcnnLayer` instance). Thus the list of edges from such model need to be rebuilt such that if there are edges `(A, X)` and `(X, B)`, and `X` is an ignored layer, an edge `(A, B)` is created. @@ -388,7 +408,7 @@ def get_sinabs_edges_and_modules(self) -> Tuple[List[Tuple[int, int]], Dict[int, return edges_without_merge, sinabs_modules_map, remapped_nodes - def populate_nodes_io(self): + def _populate_nodes_io(self): """ Loops through the nodes in the original graph to retrieve their I/O tensor shapes and add them to their respective representations in `self.nodes_to_dcnnl_map`.""" @@ -397,7 +417,7 @@ def find_original_node_name(name_mapper: dict, node: int): for orig_name, new_name in name_mapper.items(): if new_name == node: return orig_name - raise ValueError(f'Node {node} could not be found within the name remapping done by self.get_sinabs_edges_and_modules().') + raise ValueError(f'Node {node} could not be found within the name remapping done by self._get_sinabs_edges_and_modules().') def find_my_input(edges_list: list, node: int): """ Returns the node `X` in the first edge `(X, node)`.""" diff --git a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py new file mode 100644 index 00000000..6013ef3e --- /dev/null +++ b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py @@ -0,0 +1,88 @@ +# functionality : ... +# author : Willian Soares Girao +# contact : williansoaresgirao@gmail.com + +import torch.nn as nn +from typing import List, Tuple, Dict + +class DynapcnnNetworkModule(nn.Module): + """ .""" + def __init__(self, dcnnl_edges: List[Tuple[int, int]], dynapcnn_layers: Dict) -> nn.Module: + super().__init__() + + self.model_forward = self._build_module_forward_from_graph(dcnnl_edges, dynapcnn_layers) + + def _build_module_forward_from_graph(self, dcnnl_edges, dynapcnn_layers): + """ + ... + + TODO the `Merge` layer has to be recreated here if a node appears as the targert in more than one edge. + """ + forward_map = {} + new_edges_set = [] + divergent_nodes = [] + + for edge in dcnnl_edges: + source_dcnnl = edge[0] + target_dcnnl = edge[1] + + new_edge_2_append = [] + + # processing the source `DynapcnnLayer`. + + if source_dcnnl not in forward_map: + forward_map[source_dcnnl] = dynapcnn_layers[source_dcnnl]['layer'] + + if len(forward_map[source_dcnnl].pool_layer) > 1: + # this `DynapcnnLayer` is a divergent point in the graph. + divergent_nodes.append(source_dcnnl) + for i in range(len(forward_map[source_dcnnl].pool_layer)): + + # create edge representing forward through the i-th pooling layer. + pool_name = f'{source_dcnnl}_pool{i}' + new_edges_set.append((source_dcnnl, pool_name)) + + # create forward 'node' for the i-th pooling layer. + if pool_name not in forward_map: + forward_map[pool_name] = forward_map[source_dcnnl].pool_layer[i] + + # create edge from i-th pooling to its target `DynapcnnLayer`. + new_edge_2_append.append((pool_name, dynapcnn_layers[source_dcnnl]['destinations'][i])) + + # processing the target `DynapcnnLayer`. + + if target_dcnnl not in forward_map: + forward_map[target_dcnnl] = dynapcnn_layers[target_dcnnl]['layer'] + + if len(forward_map[target_dcnnl].pool_layer) > 1: + # this `DynapcnnLayer` is a divergent point in the graph. + divergent_nodes.append(target_dcnnl) + for i in range(len(forward_map[target_dcnnl].pool_layer)): + + # create edge representing forward through the i-th pooling layer. + pool_name = f'{target_dcnnl}_pool{i}' + new_edges_set.append((target_dcnnl, pool_name)) + + # create forward 'node' for the i-th pooling layer. + if pool_name not in forward_map: + forward_map[pool_name] = forward_map[target_dcnnl].pool_layer[i] + + # create edge from i-th pooling to its target `DynapcnnLayer`. + new_edge_2_append.append((pool_name, dynapcnn_layers[target_dcnnl]['destinations'][i])) + + if source_dcnnl not in divergent_nodes and target_dcnnl not in divergent_nodes: + # save original edge. + new_edges_set.append(edge) + + if len(new_edge_2_append) != 0: + new_edges_set.extend(new_edge_2_append) + + print('original edges: ') + for edge in dcnnl_edges: + print(edge) + + print('\nforward edges: ') + for edge in new_edges_set: + print(edge) + + return forward_map diff --git a/tests/test_nonsequential/DynapcnnNetworkGraph_from_NIRgraph_1.ipynb b/tests/test_nonsequential/DynapcnnNetworkGraph_from_NIRgraph_1.ipynb deleted file mode 100644 index 237e9068..00000000 --- a/tests/test_nonsequential/DynapcnnNetworkGraph_from_NIRgraph_1.ipynb +++ /dev/null @@ -1,291 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import torch\n", - "import torch.nn as nn\n", - "import networkx as nx\n", - "import matplotlib.pyplot as plt\n", - "from sinabs.from_torch import from_model\n", - "from sinabs.backend.dynapcnn import DynapcnnNetwork, DynapcnnNetworkGraph\n", - "import sinabs as snb" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "torch.manual_seed(0)" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "channels = 1\n", - "height = 28\n", - "width = 28\n", - "\n", - "input_shape = (channels, height, width)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Network Module (pure Pytorch)" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "ann = nn.Sequential(\n", - " nn.Conv2d(1, 20, 5, 1, bias=False),\n", - " nn.ReLU(),\n", - " nn.AvgPool2d(2,2),\n", - " nn.Conv2d(20, 32, 5, 1, bias=False),\n", - " nn.ReLU(),\n", - " nn.AvgPool2d(2,2),\n", - " nn.Conv2d(32, 128, 3, 1, bias=False),\n", - " nn.ReLU(),\n", - " nn.AvgPool2d(2,2),\n", - " nn.Flatten(),\n", - " nn.Linear(128, 500, bias=False),\n", - " nn.ReLU(),\n", - " nn.Linear(500, 10, bias=False),\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Sinabs Model" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "sinabs_model = from_model(ann, add_spiking_output=True, batch_size=1)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## DynapCNN Model" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - ">>>> 0\n", - ">>>> 1\n", - ">>>> 2\n", - ">>>> 3\n", - ">>>> 4\n", - ">>>> 5\n", - ">>>> 6\n", - ">>>> 7\n", - ">>>> 8\n", - ">>>> 9\n", - ">>>> 10\n", - ">>>> 11\n", - ">>>> 12\n", - ">>>> spike_output\n", - "```mermaid\n", - "graph TD;\n", - "0 --> 1;\n", - "1 --> 2;\n", - "2 --> 3;\n", - "3 --> 4;\n", - "4 --> 5;\n", - "5 --> 6;\n", - "6 --> 7;\n", - "7 --> 8;\n", - "8 --> 9;\n", - "9 --> 10;\n", - "10 --> 11;\n", - "11 --> 12;\n", - "12 --> spike_output;\n", - "spike_output;\n", - "\n", - "```\n", - "\n" - ] - }, - { - "ename": "KeyError", - "evalue": "13", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mKeyError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[6], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m hw_model \u001b[38;5;241m=\u001b[39m DynapcnnNetworkGraph(\n\u001b[1;32m 2\u001b[0m sinabs_model,\n\u001b[1;32m 3\u001b[0m discretize\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m 4\u001b[0m input_shape\u001b[38;5;241m=\u001b[39minput_shape\n\u001b[1;32m 5\u001b[0m )\n", - "File \u001b[0;32m~/Documents/github/sinabs/sinabs/backend/dynapcnn/dynapcnn_network_graph.py:80\u001b[0m, in \u001b[0;36mDynapcnnNetworkGraph.__init__\u001b[0;34m(self, snn, input_shape, dvs_input, discretize)\u001b[0m\n\u001b[1;32m 72\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgraph_tracer \u001b[38;5;241m=\u001b[39m NIRtoDynapcnnNetworkGraph( \u001b[38;5;66;03m# computational graph from original PyTorch module.\u001b[39;00m\n\u001b[1;32m 73\u001b[0m snn\u001b[38;5;241m.\u001b[39mspiking_model,\n\u001b[1;32m 74\u001b[0m torch\u001b[38;5;241m.\u001b[39mrandn((\u001b[38;5;241m1\u001b[39m, \u001b[38;5;241m*\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39minput_shape))) \u001b[38;5;66;03m# needs the batch dimension. \u001b[39;00m\n\u001b[1;32m 76\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msinabs_edges, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msinabs_modules_map, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmerge_nodes \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mget_sinabs_edges_and_modules(snn)\n\u001b[1;32m 78\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdynapcnn_layers, \\\n\u001b[1;32m 79\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnodes_to_dcnnl_map, \\\n\u001b[0;32m---> 80\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdcnnl_to_dcnnl_map \u001b[38;5;241m=\u001b[39m build_from_graph( \u001b[38;5;66;03m# build model from graph edges.\u001b[39;00m\n\u001b[1;32m 81\u001b[0m discretize\u001b[38;5;241m=\u001b[39mdiscretize,\n\u001b[1;32m 82\u001b[0m layers\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msinabs_modules_map, \n\u001b[1;32m 83\u001b[0m in_shape\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39minput_shape,\n\u001b[1;32m 84\u001b[0m edges\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msinabs_edges,\n\u001b[1;32m 85\u001b[0m merge_nodes\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmerge_nodes)\n", - "File \u001b[0;32m~/Documents/github/sinabs/sinabs/backend/dynapcnn/utils.py:595\u001b[0m, in \u001b[0;36mbuild_from_graph\u001b[0;34m(discretize, layers, in_shape, edges, merge_nodes)\u001b[0m\n\u001b[1;32m 593\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 594\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m edge \u001b[38;5;129;01min\u001b[39;00m edges:\n\u001b[0;32m--> 595\u001b[0m process_edge( \u001b[38;5;66;03m# figure out to which (future) DynapcnnLayer each node will belong to.\u001b[39;00m\n\u001b[1;32m 596\u001b[0m layers, edge, nodes_to_dcnnl_map)\n\u001b[1;32m 598\u001b[0m \u001b[38;5;66;03m# look for edges between connecting nodes in different (future) DynapcnnLayer.\u001b[39;00m\n\u001b[1;32m 599\u001b[0m dcnnl_to_dcnnl_map \u001b[38;5;241m=\u001b[39m get_dynapcnnlayers_destinations(layers, edges, nodes_to_dcnnl_map)\n", - "File \u001b[0;32m~/Documents/github/sinabs/sinabs/backend/dynapcnn/sinabs_edges_handler.py:22\u001b[0m, in \u001b[0;36mprocess_edge\u001b[0;34m(layers, edge, mapper)\u001b[0m\n\u001b[1;32m 11\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mprocess_edge\u001b[39m(layers: Dict[\u001b[38;5;28mint\u001b[39m, nn\u001b[38;5;241m.\u001b[39mModule], edge: Tuple[\u001b[38;5;28mint\u001b[39m, \u001b[38;5;28mint\u001b[39m], mapper: \u001b[38;5;28mdict\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 12\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\" Read in an edge describing the connection between two layers (nodes in the computational graph). If 'edge'\u001b[39;00m\n\u001b[1;32m 13\u001b[0m \u001b[38;5;124;03m is a valid connection between two layers, update 'mapper' to incorporate these layers into a new or existing dictonary\u001b[39;00m\n\u001b[1;32m 14\u001b[0m \u001b[38;5;124;03m containing the modules comprising a future DynacnnLayer object.\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 20\u001b[0m \u001b[38;5;124;03m mapper : dictionary where each 'key' is the index of a future 'DynapcnnLayer' and 'value' is a dictionary ('key': node, 'value': module).\u001b[39;00m\n\u001b[1;32m 21\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[0;32m---> 22\u001b[0m edge_type \u001b[38;5;241m=\u001b[39m is_valid_edge(edge, layers, VALID_SINABS_EDGES)\n\u001b[1;32m 24\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(edge_type, \u001b[38;5;28mint\u001b[39m): \u001b[38;5;66;03m# incorporate modules within the edge to a dict representing a future DynapcnnLayer.\u001b[39;00m\n\u001b[1;32m 25\u001b[0m update_dynapcnnlayer_mapper(edge_type, edge, mapper, layers)\n", - "File \u001b[0;32m~/Documents/github/sinabs/sinabs/backend/dynapcnn/sinabs_edges_handler.py:41\u001b[0m, in \u001b[0;36mis_valid_edge\u001b[0;34m(edge, layers, valid_edges_map)\u001b[0m\n\u001b[1;32m 29\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mis_valid_edge\u001b[39m(edge: Tuple[\u001b[38;5;28mint\u001b[39m, \u001b[38;5;28mint\u001b[39m], layers: Dict[\u001b[38;5;28mint\u001b[39m, nn\u001b[38;5;241m.\u001b[39mModule], valid_edges_map: \u001b[38;5;28mdict\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28mint\u001b[39m:\n\u001b[1;32m 30\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\" Checks if the modules each node in 'edge' represent are a valid connection between a sinabs network to be \u001b[39;00m\n\u001b[1;32m 31\u001b[0m \u001b[38;5;124;03m loaded on Speck.\u001b[39;00m\n\u001b[1;32m 32\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 39\u001b[0m \u001b[38;5;124;03m edge_type: the edge type specified in 'valid_edges_map' ('None' if edge is not valid).\u001b[39;00m\n\u001b[1;32m 40\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[0;32m---> 41\u001b[0m edge_layers \u001b[38;5;241m=\u001b[39m (layers[edge[\u001b[38;5;241m0\u001b[39m]], layers[edge[\u001b[38;5;241m1\u001b[39m]])\n\u001b[1;32m 42\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m edge_type, sinabs_edge \u001b[38;5;129;01min\u001b[39;00m valid_edges_map\u001b[38;5;241m.\u001b[39mitems():\n\u001b[1;32m 43\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m (\u001b[38;5;28mtype\u001b[39m(edge_layers[\u001b[38;5;241m0\u001b[39m]) \u001b[38;5;241m==\u001b[39m sinabs_edge[\u001b[38;5;241m0\u001b[39m]) \u001b[38;5;129;01mand\u001b[39;00m (\u001b[38;5;28mtype\u001b[39m(edge_layers[\u001b[38;5;241m1\u001b[39m]) \u001b[38;5;241m==\u001b[39m sinabs_edge[\u001b[38;5;241m1\u001b[39m]):\n", - "\u001b[0;31mKeyError\u001b[0m: 13" - ] - } - ], - "source": [ - "hw_model = DynapcnnNetworkGraph(\n", - " sinabs_model,\n", - " discretize=True,\n", - " input_shape=input_shape\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Network is valid\n" - ] - }, - { - "data": { - "text/plain": [ - "DynapcnnNetworkGraph()" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "hw_model.to(device=\"speck2edevkit:0\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "layer index: 0\n", - "layer modules: DynapcnnLayer(\n", - " (conv_layer): Conv2d(1, 20, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", - " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", - " tensor(635.), min_v_mem=Parameter containing:\n", - " tensor(-635.), batch_size=1, num_timesteps=-1)\n", - " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", - ")\n", - "layer destinations: [1]\n", - "assigned core: 0\n", - "\n", - "layer index: 1\n", - "layer modules: DynapcnnLayer(\n", - " (conv_layer): Conv2d(20, 32, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", - " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", - " tensor(11361.), min_v_mem=Parameter containing:\n", - " tensor(-11361.), batch_size=1, num_timesteps=-1)\n", - " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", - ")\n", - "layer destinations: [2]\n", - "assigned core: 3\n", - "\n", - "layer index: 2\n", - "layer modules: DynapcnnLayer(\n", - " (conv_layer): Conv2d(32, 128, kernel_size=(3, 3), stride=(1, 1), bias=False)\n", - " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", - " tensor(8621.), min_v_mem=Parameter containing:\n", - " tensor(-8621.), batch_size=1, num_timesteps=-1)\n", - " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", - ")\n", - "layer destinations: [3]\n", - "assigned core: 5\n", - "\n", - "layer index: 3\n", - "layer modules: DynapcnnLayer(\n", - " (conv_layer): Conv2d(128, 500, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", - " tensor(5747.), min_v_mem=Parameter containing:\n", - " tensor(-5747.), batch_size=1, num_timesteps=-1)\n", - ")\n", - "layer destinations: [4]\n", - "assigned core: 6\n", - "\n", - "layer index: 4\n", - "layer modules: DynapcnnLayer(\n", - " (conv_layer): Conv2d(500, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", - " tensor(2841.), min_v_mem=Parameter containing:\n", - " tensor(-2841.), batch_size=1, num_timesteps=-1)\n", - ")\n", - "layer destinations: []\n", - "assigned core: 1\n", - "\n" - ] - } - ], - "source": [ - "print(hw_model)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "speck-rescnn", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/tests/test_nonsequential/DynapcnnNetworkGraph_from_NIRgraph_2.ipynb b/tests/test_nonsequential/DynapcnnNetworkGraph_from_NIRgraph_2.ipynb deleted file mode 100644 index dddb75a3..00000000 --- a/tests/test_nonsequential/DynapcnnNetworkGraph_from_NIRgraph_2.ipynb +++ /dev/null @@ -1,256 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import torch\n", - "import torch.nn as nn\n", - "import networkx as nx\n", - "import matplotlib.pyplot as plt\n", - "from sinabs.from_torch import from_model\n", - "from sinabs.backend.dynapcnn import DynapcnnNetwork, DynapcnnNetworkGraph\n", - "import sinabs as snb" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "torch.manual_seed(0)" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "channels = 1\n", - "height = 28\n", - "width = 28\n", - "\n", - "input_shape = (channels, height, width)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Network Module (pure Pytorch)" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "class ANN(nn.Module):\n", - " def __init__(self) -> None:\n", - " super().__init__()\n", - "\n", - " self.con1 = nn.Conv2d(1, 20, 5, 1, bias=False)\n", - " self.rel1 = nn.ReLU()\n", - " self.pool1 = nn.AvgPool2d(2,2)\n", - "\n", - " self.conv2 = nn.Conv2d(20, 32, 5, 1, bias=False)\n", - " self.rel2 = nn.ReLU()\n", - " self.pool2 = nn.AvgPool2d(2,2)\n", - "\n", - " self.conv3 = nn.Conv2d(32, 128, 3, 1, bias=False)\n", - " self.rel3 = nn.ReLU()\n", - " self.pool3 = nn.AvgPool2d(2,2)\n", - "\n", - " self.flat = nn.Flatten()\n", - "\n", - " self.fc1 = nn.Linear(128, 500, bias=False)\n", - " self.rel4 = nn.ReLU()\n", - " self.fc2 = nn.Linear(500, 10, bias=False)\n", - "\n", - " def forward(self, x):\n", - " \n", - " con1_out = self.con1(x)\n", - " rel1_out = self.rel1(con1_out)\n", - " pool1_out = self.pool1(rel1_out)\n", - "\n", - " conv2_out = self.conv2(pool1_out)\n", - " rel2_out = self.rel2(conv2_out)\n", - " pool2_out = self.pool2(rel2_out)\n", - "\n", - " conv3_out = self.conv3(pool2_out)\n", - " rel3_out = self.rel3(conv3_out)\n", - " pool3_out = self.pool3(rel3_out)\n", - "\n", - " flat_out = self.flat(pool3_out)\n", - " \n", - " fc1_out = self.fc1(flat_out)\n", - " rel4_out = self.rel4(fc1_out)\n", - " fc2_out = self.fc2(rel4_out)\n", - "\n", - " return fc2_out" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "ann = ANN()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Sinabs Model" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "sinabs_model = from_model(ann, add_spiking_output=True, batch_size=1)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## DynapCNN Model" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "hw_model = DynapcnnNetworkGraph(\n", - " sinabs_model,\n", - " discretize=True,\n", - " input_shape=input_shape,\n", - " use_jit_tracer=False\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "# hw_model.to(device=\"speck2edevkit:0\")" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "layer index: 0\n", - "layer modules: DynapcnnLayer(\n", - " (conv_layer): Conv2d(1, 20, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", - " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", - " tensor(635.), min_v_mem=Parameter containing:\n", - " tensor(-635.), batch_size=1, num_timesteps=-1)\n", - " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", - ")\n", - "layer destinations: [1]\n", - "assigned core: -1\n", - "\n", - "layer index: 1\n", - "layer modules: DynapcnnLayer(\n", - " (conv_layer): Conv2d(20, 32, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", - " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", - " tensor(11361.), min_v_mem=Parameter containing:\n", - " tensor(-11361.), batch_size=1, num_timesteps=-1)\n", - " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", - ")\n", - "layer destinations: [2]\n", - "assigned core: -1\n", - "\n", - "layer index: 2\n", - "layer modules: DynapcnnLayer(\n", - " (conv_layer): Conv2d(32, 128, kernel_size=(3, 3), stride=(1, 1), bias=False)\n", - " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", - " tensor(8621.), min_v_mem=Parameter containing:\n", - " tensor(-8621.), batch_size=1, num_timesteps=-1)\n", - " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", - ")\n", - "layer destinations: [3]\n", - "assigned core: -1\n", - "\n", - "layer index: 3\n", - "layer modules: DynapcnnLayer(\n", - " (conv_layer): Conv2d(128, 500, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", - " tensor(5747.), min_v_mem=Parameter containing:\n", - " tensor(-5747.), batch_size=1, num_timesteps=-1)\n", - ")\n", - "layer destinations: [4]\n", - "assigned core: -1\n", - "\n", - "layer index: 4\n", - "layer modules: DynapcnnLayer(\n", - " (conv_layer): Conv2d(500, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", - " tensor(2841.), min_v_mem=Parameter containing:\n", - " tensor(-2841.), batch_size=1, num_timesteps=-1)\n", - ")\n", - "layer destinations: []\n", - "assigned core: -1\n", - "\n" - ] - } - ], - "source": [ - "print(hw_model)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "speck-rescnn", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/tests/test_nonsequential/DynapcnnNetworkGraph_from_NIRgraph_3.ipynb b/tests/test_nonsequential/DynapcnnNetworkGraph_from_NIRgraph_3.ipynb deleted file mode 100644 index af555728..00000000 --- a/tests/test_nonsequential/DynapcnnNetworkGraph_from_NIRgraph_3.ipynb +++ /dev/null @@ -1,302 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import torch\n", - "import torch.nn as nn\n", - "from sinabs.from_torch import from_model\n", - "from sinabs.backend.dynapcnn import DynapcnnNetworkGraph\n", - "from sinabs.layers import Merge, IAFSqueeze" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "torch.manual_seed(0)" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "channels = 1\n", - "height = 28\n", - "width = 28\n", - "\n", - "input_shape = (channels, height, width)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Network Module (Model #1)\n", - "\n", - "This one won't pass the `config_builder.validate_configuration` because the nodes inputing into the `Merge` layer don't have the same output dimension." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "class SNN(nn.Module):\n", - " def __init__(self) -> None:\n", - " super().__init__()\n", - "\n", - " self.conv1 = nn.Conv2d(1, 10, 5, 1, bias=False)\n", - " self.iaf1 = IAFSqueeze(batch_size=1)\n", - " self.pool1 = nn.AvgPool2d(2,2)\n", - "\n", - " self.conv2 = nn.Conv2d(10, 20, 5, 1, bias=False)\n", - " self.iaf2 = IAFSqueeze(batch_size=1)\n", - " self.pool2 = nn.AvgPool2d(2,2)\n", - "\n", - " self.conv3 = nn.Conv2d(20, 1, 3, 1, bias=False)\n", - " self.iaf3 = IAFSqueeze(batch_size=1)\n", - " self.pool3 = nn.AvgPool2d(2,2)\n", - "\n", - " self.flat = nn.Flatten()\n", - "\n", - " self.fc1 = nn.Linear(121, 500, bias=False)\n", - " self.iaf4 = IAFSqueeze(batch_size=1)\n", - " self.fc2 = nn.Linear(500, 10, bias=False)\n", - " self.iaf5 = IAFSqueeze(batch_size=1)\n", - "\n", - " self.adder = Merge()\n", - "\n", - " def forward(self, x):\n", - " \n", - " con1_out = self.conv1(x)\n", - " iaf1_out = self.iaf1(con1_out)\n", - " pool1_out = self.pool1(iaf1_out)\n", - "\n", - " conv2_out = self.conv2(pool1_out)\n", - " iaf2_out = self.iaf2(conv2_out)\n", - " pool2_out = self.pool2(iaf2_out)\n", - "\n", - " conv3_out = self.conv3(self.adder(iaf1_out, pool2_out))\n", - " iaf3_out = self.iaf3(conv3_out)\n", - " pool3_out = self.pool3(iaf3_out)\n", - "\n", - " flat_out = self.flat(pool3_out)\n", - " \n", - " fc1_out = self.fc1(flat_out)\n", - " iaf4_out = self.iaf4(fc1_out)\n", - " fc2_out = self.fc2(iaf4_out)\n", - " iaf5_out = self.iaf5(fc2_out)\n", - "\n", - " return iaf5_out" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "snn = SNN()" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "x = torch.randn((1, *input_shape))\n", - "\n", - "con1_out = snn.conv1(x)\n", - "iaf1_out = snn.iaf1(con1_out)\n", - "print('iaf1_out: ', iaf1_out.shape)\n", - "pool1_out = snn.pool1(iaf1_out)\n", - "\n", - "conv2_out = snn.conv2(pool1_out)\n", - "iaf2_out = snn.iaf2(conv2_out)\n", - "pool2_out = snn.pool2(iaf2_out)\n", - "print('pool2_out: ', pool2_out.shape)\n", - "\n", - "added = snn.adder(iaf1_out, pool2_out)\n", - "print('added: ', added.shape)\n", - "\n", - "conv3_out = snn.conv3(added)\n", - "iaf3_out = snn.iaf3(conv3_out)\n", - "pool3_out = snn.pool3(iaf3_out)\n", - "print('pool3_out: ', pool3_out.shape)\n", - "\n", - "flat_out = snn.flat(pool3_out)\n", - "print('flat_out: ', flat_out.shape)\n", - "\n", - "fc1_out = snn.fc1(flat_out)\n", - "iaf4_out = snn.iaf4(fc1_out)\n", - "fc2_out = snn.fc2(iaf4_out)\n", - "iaf5_out = snn.iaf5(fc2_out)" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "print(f'DynapcnnLayer 0 [core 0]:')\n", - "print(f' input: {x.shape}')\n", - "con1_out = snn.conv1(x)\n", - "print(f' conv1: {con1_out.shape}')\n", - "iaf1_out = snn.iaf1(con1_out)\n", - "print(f' iaf1: {iaf1_out.shape}')\n", - "pool1_out = snn.pool1(iaf1_out)\n", - "print(f' pool1: {pool1_out.shape}\\n')\n", - "\n", - "print(f'DynapcnnLayer 1 [core 1]:')\n", - "print(f' input: {pool1_out.shape}')\n", - "conv2_out = snn.conv2(pool1_out)\n", - "print(f' conv2: {conv2_out.shape}')\n", - "iaf2_out = snn.iaf2(conv2_out)\n", - "print(f' iaf2: {iaf2_out.shape}')\n", - "pool2_out = snn.pool2(iaf2_out)\n", - "print(f' pool2: {pool2_out.shape}\\n')\n", - "\n", - "added = snn.adder(iaf1_out, pool2_out)\n", - "\n", - "print(f'DynapcnnLayer 2 [core 2]:')\n", - "print(f' input: {added.shape}')\n", - "conv3_out = snn.conv3(added)\n", - "print(f' conv3: {conv3_out.shape}')\n", - "iaf3_out = snn.iaf3(conv3_out)\n", - "print(f' iaf3: {iaf3_out.shape}')\n", - "pool3_out = snn.pool3(iaf3_out)\n", - "print(f' pool3: {pool3_out.shape}')\n", - "\n", - "flat_out = snn.flat(pool3_out)\n", - "\n", - "fc1_out = snn.fc1(flat_out)\n", - "iaf4_out = snn.iaf4(fc1_out)\n", - "print(f'DynapcnnLayer 3: {iaf4_out.shape}')\n", - "\n", - "fc2_out = snn.fc2(iaf4_out)\n", - "iaf5_out = snn.iaf5(fc2_out)\n", - "print(f'DynapcnnLayer 4: {iaf5_out.shape}')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## DynapCNN Model" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "ename": "RuntimeError", - "evalue": "mat1 and mat2 shapes cannot be multiplied (1x625 and 16x500)", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[8], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m hw_model \u001b[38;5;241m=\u001b[39m DynapcnnNetworkGraph(\n\u001b[1;32m 2\u001b[0m snn,\n\u001b[1;32m 3\u001b[0m discretize\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m 4\u001b[0m input_shape\u001b[38;5;241m=\u001b[39minput_shape\n\u001b[1;32m 5\u001b[0m )\n", - "File \u001b[0;32m~/Documents/github/sinabs/sinabs/backend/dynapcnn/dynapcnn_network_graph.py:71\u001b[0m, in \u001b[0;36mDynapcnnNetworkGraph.__init__\u001b[0;34m(self, snn, input_shape, dvs_input, discretize)\u001b[0m\n\u001b[1;32m 67\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39minput_shape \u001b[38;5;241m=\u001b[39m input_shape\n\u001b[1;32m 69\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39minput_shape) \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m3\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124minfer_input_shape did not return 3-tuple\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m---> 71\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgraph_tracer \u001b[38;5;241m=\u001b[39m NIRtoDynapcnnNetworkGraph( \u001b[38;5;66;03m# computational graph from original PyTorch module.\u001b[39;00m\n\u001b[1;32m 72\u001b[0m snn,\n\u001b[1;32m 73\u001b[0m torch\u001b[38;5;241m.\u001b[39mrandn((\u001b[38;5;241m1\u001b[39m, \u001b[38;5;241m*\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39minput_shape))) \u001b[38;5;66;03m# needs the batch dimension. \u001b[39;00m\n\u001b[1;32m 75\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msinabs_edges, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msinabs_modules_map, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmerge_nodes \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mget_sinabs_edges_and_modules()\n\u001b[1;32m 77\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdynapcnn_layers, \\\n\u001b[1;32m 78\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnodes_to_dcnnl_map, \\\n\u001b[1;32m 79\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdcnnl_to_dcnnl_map \u001b[38;5;241m=\u001b[39m build_from_graph( \u001b[38;5;66;03m# build model from graph edges.\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 83\u001b[0m edges\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msinabs_edges,\n\u001b[1;32m 84\u001b[0m merge_nodes\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmerge_nodes)\n", - "File \u001b[0;32m~/Documents/github/sinabs/sinabs/backend/dynapcnn/NIRGraphExtractor.py:10\u001b[0m, in \u001b[0;36mNIRtoDynapcnnNetworkGraph.__init__\u001b[0;34m(self, spiking_model, dummy_input)\u001b[0m\n\u001b[1;32m 7\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m, spiking_model, dummy_input) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 8\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\" .\"\"\"\u001b[39;00m\n\u001b[0;32m---> 10\u001b[0m nir_graph \u001b[38;5;241m=\u001b[39m nirtorch\u001b[38;5;241m.\u001b[39mextract_torch_graph(spiking_model, dummy_input, model_name\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m)\u001b[38;5;241m.\u001b[39mignore_tensors()\n\u001b[1;32m 12\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39medges_list, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mname_2_indx_map \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mget_edges_from_nir(nir_graph)\n\u001b[1;32m 14\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmodules_map \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mget_named_modules(spiking_model)\n", - "File \u001b[0;32m~/.local/lib/python3.11/site-packages/nirtorch/graph.py:433\u001b[0m, in \u001b[0;36mextract_torch_graph\u001b[0;34m(model, sample_data, model_name, model_args)\u001b[0m\n\u001b[1;32m 412\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Extract computational graph between various modules in the model\u001b[39;00m\n\u001b[1;32m 413\u001b[0m \u001b[38;5;124;03mNOTE: This method is not capable of any compute happening outside of module\u001b[39;00m\n\u001b[1;32m 414\u001b[0m \u001b[38;5;124;03mdefinitions.\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 428\u001b[0m \u001b[38;5;124;03m Graph: A graph object representing the computational graph of the given model\u001b[39;00m\n\u001b[1;32m 429\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 430\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m GraphTracer(\n\u001b[1;32m 431\u001b[0m named_modules_map(model, model_name\u001b[38;5;241m=\u001b[39mmodel_name)\n\u001b[1;32m 432\u001b[0m ) \u001b[38;5;28;01mas\u001b[39;00m tracer, torch\u001b[38;5;241m.\u001b[39mno_grad():\n\u001b[0;32m--> 433\u001b[0m _ \u001b[38;5;241m=\u001b[39m model(sample_data, \u001b[38;5;241m*\u001b[39mmodel_args)\n\u001b[1;32m 435\u001b[0m \u001b[38;5;66;03m# HACK: The current graph is using copy-constructors, that detaches\u001b[39;00m\n\u001b[1;32m 436\u001b[0m \u001b[38;5;66;03m# the traced output_types from the original graph.\u001b[39;00m\n\u001b[1;32m 437\u001b[0m \u001b[38;5;66;03m# In the future, find a way to synchronize the two representations\u001b[39;00m\n\u001b[1;32m 438\u001b[0m tracer\u001b[38;5;241m.\u001b[39mgraph\u001b[38;5;241m.\u001b[39mmodule_output_types \u001b[38;5;241m=\u001b[39m tracer\u001b[38;5;241m.\u001b[39moutput_types\n", - "File \u001b[0;32m~/.local/lib/python3.11/site-packages/nirtorch/graph.py:344\u001b[0m, in \u001b[0;36mmodule_forward_wrapper..my_forward\u001b[0;34m(mod, *args, **kwargs)\u001b[0m\n\u001b[1;32m 343\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mmy_forward\u001b[39m(mod: nn\u001b[38;5;241m.\u001b[39mModule, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Any:\n\u001b[0;32m--> 344\u001b[0m out \u001b[38;5;241m=\u001b[39m _torch_module_call(mod, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m 346\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(out, \u001b[38;5;28mtuple\u001b[39m):\n\u001b[1;32m 347\u001b[0m out_tuple \u001b[38;5;241m=\u001b[39m (out[\u001b[38;5;241m0\u001b[39m],)\n", - "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/nn/modules/module.py:1511\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1509\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs) \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m 1510\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1511\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n", - "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/nn/modules/module.py:1520\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1515\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1516\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1517\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m 1518\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1519\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1520\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m forward_call(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m 1522\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 1523\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n", - "Cell \u001b[0;32mIn[4], line 89\u001b[0m, in \u001b[0;36mSNN.forward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m 85\u001b[0m iaf3_out \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39miaf3(conv3_out)\n\u001b[1;32m 87\u001b[0m flat_out \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mflat(iaf3_out)\n\u001b[0;32m---> 89\u001b[0m fc1_out \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfc1(flat_out)\n\u001b[1;32m 90\u001b[0m iaf4_out \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39miaf4(fc1_out)\n\u001b[1;32m 91\u001b[0m fc2_out \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfc2(iaf4_out)\n", - "File \u001b[0;32m~/.local/lib/python3.11/site-packages/nirtorch/graph.py:344\u001b[0m, in \u001b[0;36mmodule_forward_wrapper..my_forward\u001b[0;34m(mod, *args, **kwargs)\u001b[0m\n\u001b[1;32m 343\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mmy_forward\u001b[39m(mod: nn\u001b[38;5;241m.\u001b[39mModule, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Any:\n\u001b[0;32m--> 344\u001b[0m out \u001b[38;5;241m=\u001b[39m _torch_module_call(mod, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m 346\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(out, \u001b[38;5;28mtuple\u001b[39m):\n\u001b[1;32m 347\u001b[0m out_tuple \u001b[38;5;241m=\u001b[39m (out[\u001b[38;5;241m0\u001b[39m],)\n", - "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/nn/modules/module.py:1511\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1509\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs) \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m 1510\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1511\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n", - "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/nn/modules/module.py:1520\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1515\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1516\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1517\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m 1518\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1519\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1520\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m forward_call(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m 1522\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 1523\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n", - "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/nn/modules/linear.py:116\u001b[0m, in \u001b[0;36mLinear.forward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m 115\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;28minput\u001b[39m: Tensor) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tensor:\n\u001b[0;32m--> 116\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m F\u001b[38;5;241m.\u001b[39mlinear(\u001b[38;5;28minput\u001b[39m, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mweight, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbias)\n", - "\u001b[0;31mRuntimeError\u001b[0m: mat1 and mat2 shapes cannot be multiplied (1x625 and 16x500)" - ] - } - ], - "source": [ - "hw_model = DynapcnnNetworkGraph(\n", - " snn,\n", - " discretize=True,\n", - " input_shape=input_shape\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Layer[2] input size x: 24 different than Layer[0] output size x: 24 pooling: 2\n", - "Layer[2] input size y: 24 different than Layer[0] output size y: 24 pooling: 2\n", - "Layer[2] input size x: 24 different than Layer[1] output size x: 8 pooling: 2\n", - "Layer[2] input size y: 24 different than Layer[1] output size y: 8 pooling: 2\n", - "\n" - ] - }, - { - "ename": "ValueError", - "evalue": "Generated config is not valid for speck2edevkit:0", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[9], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m hw_model\u001b[38;5;241m.\u001b[39mto(device\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mspeck2edevkit:0\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", - "File \u001b[0;32m~/Documents/github/sinabs/sinabs/backend/dynapcnn/dynapcnn_network_graph.py:148\u001b[0m, in \u001b[0;36mDynapcnnNetworkGraph.to\u001b[0;34m(self, device, chip_layers_ordering, monitor_layers, config_modifier, slow_clk_frequency)\u001b[0m\n\u001b[1;32m 144\u001b[0m device_name, _ \u001b[38;5;241m=\u001b[39m parse_device_id(device)\n\u001b[1;32m 146\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m device_name \u001b[38;5;129;01min\u001b[39;00m ChipFactory\u001b[38;5;241m.\u001b[39msupported_devices: \u001b[38;5;66;03m# pragma: no cover\u001b[39;00m\n\u001b[0;32m--> 148\u001b[0m config \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmake_config( \u001b[38;5;66;03m# generate config.\u001b[39;00m\n\u001b[1;32m 149\u001b[0m chip_layers_ordering\u001b[38;5;241m=\u001b[39mchip_layers_ordering,\n\u001b[1;32m 150\u001b[0m device\u001b[38;5;241m=\u001b[39mdevice,\n\u001b[1;32m 151\u001b[0m monitor_layers\u001b[38;5;241m=\u001b[39mmonitor_layers,\n\u001b[1;32m 152\u001b[0m config_modifier\u001b[38;5;241m=\u001b[39mconfig_modifier,\n\u001b[1;32m 153\u001b[0m )\n\u001b[1;32m 155\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msamna_device \u001b[38;5;241m=\u001b[39m open_device(device) \u001b[38;5;66;03m# apply configuration to device.\u001b[39;00m\n\u001b[1;32m 156\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msamna_device\u001b[38;5;241m.\u001b[39mget_model()\u001b[38;5;241m.\u001b[39mapply_configuration(config)\n", - "File \u001b[0;32m~/Documents/github/sinabs/sinabs/backend/dynapcnn/dynapcnn_network_graph.py:256\u001b[0m, in \u001b[0;36mDynapcnnNetworkGraph.make_config\u001b[0;34m(self, chip_layers_ordering, device, monitor_layers, config_modifier)\u001b[0m\n\u001b[1;32m 254\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m config\n\u001b[1;32m 255\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 256\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mGenerated config is not valid for \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mdevice\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n", - "\u001b[0;31mValueError\u001b[0m: Generated config is not valid for speck2edevkit:0" - ] - } - ], - "source": [ - "hw_model.to(device=\"speck2edevkit:0\")" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "speck-rescnn", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/tests/test_nonsequential/DynapcnnNetworkGraph_from_NIRgraph_4.ipynb b/tests/test_nonsequential/DynapcnnNetworkGraph_from_NIRgraph_4.ipynb index 48d8ed01..6d15f9aa 100644 --- a/tests/test_nonsequential/DynapcnnNetworkGraph_from_NIRgraph_4.ipynb +++ b/tests/test_nonsequential/DynapcnnNetworkGraph_from_NIRgraph_4.ipynb @@ -21,7 +21,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 2, @@ -309,6 +309,102 @@ "source": [ "hw_model.to(device=\"speck2edevkit:0\")" ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "---- DynapcnnLayer 0 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 0): Conv2d(1, 30, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + "(node 1): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "(node 2): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", + "(node 3): SumPool2d(norm_type=1, kernel_size=(4, 4), stride=None, ceil_mode=False)\n", + "> layer destinations: [1, 2]\n", + "> assigned core: 0\n", + "\n", + "---- DynapcnnLayer 1 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 4): Conv2d(30, 30, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + "(node 6): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "(node 7): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", + "> layer destinations: [2]\n", + "> assigned core: 1\n", + "\n", + "---- DynapcnnLayer 2 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 8): Conv2d(30, 1, kernel_size=(3, 3), stride=(1, 1), bias=False)\n", + "(node 9): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [3]\n", + "> assigned core: 2\n", + "\n", + "---- DynapcnnLayer 3 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 10): Conv2d(1, 500, kernel_size=(4, 4), stride=(1, 1), bias=False)\n", + "(node 11): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [4]\n", + "> assigned core: 3\n", + "\n", + "---- DynapcnnLayer 4 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 12): Conv2d(500, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + "(node 13): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: []\n", + "> assigned core: 4\n", + "\n", + "\n" + ] + } + ], + "source": [ + "print(hw_model)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "original edges: \n", + "(0, 1)\n", + "(0, 2)\n", + "(1, 2)\n", + "(2, 3)\n", + "(3, 4)\n", + "\n", + "forward edges: \n", + "(0, '0_pool0')\n", + "(0, '0_pool1')\n", + "('0_pool0', 1)\n", + "('0_pool1', 2)\n", + "(1, 2)\n", + "(2, 3)\n", + "(3, 4)\n" + ] + } + ], + "source": [ + "hw_model.get_network_module()" + ] } ], "metadata": { diff --git a/tests/test_nonsequential/test_DynapcnnNetworkGraph_1_skip_conn.ipynb b/tests/test_nonsequential/test_DynapcnnNetworkGraph_1_skip_conn.ipynb deleted file mode 100644 index 8847f91f..00000000 --- a/tests/test_nonsequential/test_DynapcnnNetworkGraph_1_skip_conn.ipynb +++ /dev/null @@ -1,435 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import torch\n", - "import torch.nn as nn\n", - "import networkx as nx\n", - "import matplotlib.pyplot as plt\n", - "from sinabs.from_torch import from_model\n", - "from sinabs.backend.dynapcnn import DynapcnnNetwork, DynapcnnNetworkGraph\n", - "import sinabs as snb" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "torch.manual_seed(0)" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "channels = 1\n", - "height = 28\n", - "width = 28\n", - "\n", - "input_shape = (channels, height, width)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Network Module (pure Pytorch)" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "class ResCNN(nn.Module):\n", - " def __init__(self):\n", - " super().__init__()\n", - "\n", - " self.con1 = nn.Conv2d(1, 20, 5, 1, bias=False)\n", - " self.rel1 = nn.ReLU()\n", - " self.pool1 = nn.AvgPool2d(2,2)\n", - "\n", - " self.conv2 = nn.Conv2d(20, 32, 5, 1, bias=False)\n", - " self.rel2 = nn.ReLU()\n", - " self.pool2 = nn.AvgPool2d(2,2)\n", - "\n", - " self.conv3 = nn.Conv2d(32, 128, 3, 1, bias=False)\n", - " self.rel3 = nn.ReLU()\n", - " self.pool3 = nn.AvgPool2d(2,2)\n", - "\n", - " self.flat = nn.Flatten()\n", - "\n", - " self.fc1 = nn.Linear(128, 500, bias=False)\n", - " self.rel4 = nn.ReLU()\n", - " self.fc2 = nn.Linear(500, 10, bias=False)\n", - "\n", - " def forward(self, x):\n", - " \n", - " con1_out = self.con1(x)\n", - " rel1_out = self.rel1(con1_out)\n", - " pool1_out = self.pool1(rel1_out)\n", - "\n", - " conv2_out = self.conv2(pool1_out)\n", - " rel2_out = self.rel2(conv2_out)\n", - " pool2_out = self.pool2(rel2_out)\n", - "\n", - " conv3_out = self.conv3(pool2_out + rel1_out)\n", - " rel3_out = self.rel3(conv3_out)\n", - " pool3_out = self.pool3(rel3_out)\n", - "\n", - " flat_out = self.flat(pool3_out)\n", - " \n", - " fc1_out = self.fc1(flat_out)\n", - " rel4_out = self.rel4(fc1_out)\n", - " fc2_out = self.fc2(rel4_out)\n", - "\n", - " return fc2_out\n", - "\n", - "rescnn = ResCNN()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Sinabs Model" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "sinabs_model = from_model(rescnn, add_spiking_output=True, batch_size=1)" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "ResCNN(\n", - " (con1): Conv2d(1, 20, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", - " (rel1): IAFSqueeze(spike_threshold=Parameter containing:\n", - " tensor(1.), min_v_mem=Parameter containing:\n", - " tensor(-1.), batch_size=1, num_timesteps=-1)\n", - " (pool1): AvgPool2d(kernel_size=2, stride=2, padding=0)\n", - " (conv2): Conv2d(20, 32, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", - " (rel2): IAFSqueeze(spike_threshold=Parameter containing:\n", - " tensor(1.), min_v_mem=Parameter containing:\n", - " tensor(-1.), batch_size=1, num_timesteps=-1)\n", - " (pool2): AvgPool2d(kernel_size=2, stride=2, padding=0)\n", - " (conv3): Conv2d(32, 128, kernel_size=(3, 3), stride=(1, 1), bias=False)\n", - " (rel3): IAFSqueeze(spike_threshold=Parameter containing:\n", - " tensor(1.), min_v_mem=Parameter containing:\n", - " tensor(-1.), batch_size=1, num_timesteps=-1)\n", - " (pool3): AvgPool2d(kernel_size=2, stride=2, padding=0)\n", - " (flat): Flatten(start_dim=1, end_dim=-1)\n", - " (fc1): Linear(in_features=128, out_features=500, bias=False)\n", - " (rel4): IAFSqueeze(spike_threshold=Parameter containing:\n", - " tensor(1.), min_v_mem=Parameter containing:\n", - " tensor(-1.), batch_size=1, num_timesteps=-1)\n", - " (fc2): Linear(in_features=500, out_features=10, bias=False)\n", - " (spike_output): IAFSqueeze(spike_threshold=Parameter containing:\n", - " tensor(1.), min_v_mem=Parameter containing:\n", - " tensor(-1.), batch_size=1, num_timesteps=-1)\n", - ")\n" - ] - } - ], - "source": [ - "print(sinabs_model.spiking_model)" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "ResCNN(\n", - " (con1): Conv2d(1, 20, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", - " (rel1): ReLU()\n", - " (pool1): AvgPool2d(kernel_size=2, stride=2, padding=0)\n", - " (conv2): Conv2d(20, 32, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", - " (rel2): ReLU()\n", - " (pool2): AvgPool2d(kernel_size=2, stride=2, padding=0)\n", - " (conv3): Conv2d(32, 128, kernel_size=(3, 3), stride=(1, 1), bias=False)\n", - " (rel3): ReLU()\n", - " (pool3): AvgPool2d(kernel_size=2, stride=2, padding=0)\n", - " (flat): Flatten(start_dim=1, end_dim=-1)\n", - " (fc1): Linear(in_features=128, out_features=500, bias=False)\n", - " (rel4): ReLU()\n", - " (fc2): Linear(in_features=500, out_features=10, bias=False)\n", - ")\n" - ] - } - ], - "source": [ - "print(sinabs_model.analog_model)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## DynapCNN Model" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "ename": "RuntimeError", - "evalue": "The size of tensor a (4) must match the size of tensor b (24) at non-singleton dimension 3", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[8], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m hw_model \u001b[38;5;241m=\u001b[39m DynapcnnNetworkGraph(\n\u001b[1;32m 2\u001b[0m sinabs_model,\n\u001b[1;32m 3\u001b[0m discretize\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m 4\u001b[0m input_shape\u001b[38;5;241m=\u001b[39minput_shape\n\u001b[1;32m 5\u001b[0m )\n", - "File \u001b[0;32m~/Documents/github/sinabs/sinabs/backend/dynapcnn/dynapcnn_network_graph.py:68\u001b[0m, in \u001b[0;36mDynapcnnNetworkGraph.__init__\u001b[0;34m(self, snn, input_shape, dvs_input, discretize)\u001b[0m\n\u001b[1;32m 64\u001b[0m \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39m\u001b[38;5;21m__init__\u001b[39m()\n\u001b[1;32m 66\u001b[0m dvs_input \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m \u001b[38;5;66;03m# TODO for now the graph part is not taking into consideration this.\u001b[39;00m\n\u001b[0;32m---> 68\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgraph_tracer \u001b[38;5;241m=\u001b[39m GraphTracer( \u001b[38;5;66;03m# computational graph from original PyTorch module.\u001b[39;00m\n\u001b[1;32m 69\u001b[0m snn\u001b[38;5;241m.\u001b[39manalog_model, \n\u001b[1;32m 70\u001b[0m torch\u001b[38;5;241m.\u001b[39mrandn((\u001b[38;5;241m1\u001b[39m, \u001b[38;5;241m*\u001b[39minput_shape)) \u001b[38;5;66;03m# torch.jit needs the batch dimension.\u001b[39;00m\n\u001b[1;32m 71\u001b[0m )\n\u001b[1;32m 73\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39minput_shape \u001b[38;5;241m=\u001b[39m input_shape\n\u001b[1;32m 75\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlayers \u001b[38;5;241m=\u001b[39m convert_model_to_layer_list( \u001b[38;5;66;03m# convert models to sequential.\u001b[39;00m\n\u001b[1;32m 76\u001b[0m model\u001b[38;5;241m=\u001b[39msnn\u001b[38;5;241m.\u001b[39mspiking_model, ignore\u001b[38;5;241m=\u001b[39mDEFAULT_IGNORED_LAYER_TYPES\n\u001b[1;32m 77\u001b[0m )\n", - "File \u001b[0;32m~/Documents/github/sinabs/sinabs/backend/dynapcnn/graph_tracer.py:14\u001b[0m, in \u001b[0;36mGraphTracer.__init__\u001b[0;34m(self, model, dummy_input)\u001b[0m\n\u001b[1;32m 11\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m, model: Union[nn\u001b[38;5;241m.\u001b[39mSequential, nn\u001b[38;5;241m.\u001b[39mModule], dummy_input: np\u001b[38;5;241m.\u001b[39marray) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 12\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\" .\"\"\"\u001b[39;00m\n\u001b[0;32m---> 14\u001b[0m trace \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mjit\u001b[38;5;241m.\u001b[39mtrace(model, dummy_input)\n\u001b[1;32m 15\u001b[0m _ \u001b[38;5;241m=\u001b[39m trace(dummy_input)\n\u001b[1;32m 16\u001b[0m __ \u001b[38;5;241m=\u001b[39m copy\u001b[38;5;241m.\u001b[39mdeepcopy(trace)\n", - "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/jit/_trace.py:806\u001b[0m, in \u001b[0;36mtrace\u001b[0;34m(func, example_inputs, optimize, check_trace, check_inputs, check_tolerance, strict, _force_outplace, _module_class, _compilation_unit, example_kwarg_inputs, _store_inputs)\u001b[0m\n\u001b[1;32m 804\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 805\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mexample_kwarg_inputs should be a dict\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m--> 806\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m trace_module(\n\u001b[1;32m 807\u001b[0m func,\n\u001b[1;32m 808\u001b[0m {\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mforward\u001b[39m\u001b[38;5;124m\"\u001b[39m: example_inputs},\n\u001b[1;32m 809\u001b[0m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m 810\u001b[0m check_trace,\n\u001b[1;32m 811\u001b[0m wrap_check_inputs(check_inputs),\n\u001b[1;32m 812\u001b[0m check_tolerance,\n\u001b[1;32m 813\u001b[0m strict,\n\u001b[1;32m 814\u001b[0m _force_outplace,\n\u001b[1;32m 815\u001b[0m _module_class,\n\u001b[1;32m 816\u001b[0m example_inputs_is_kwarg\u001b[38;5;241m=\u001b[39m\u001b[38;5;28misinstance\u001b[39m(example_kwarg_inputs, \u001b[38;5;28mdict\u001b[39m),\n\u001b[1;32m 817\u001b[0m _store_inputs\u001b[38;5;241m=\u001b[39m_store_inputs,\n\u001b[1;32m 818\u001b[0m )\n\u001b[1;32m 819\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m (\n\u001b[1;32m 820\u001b[0m \u001b[38;5;28mhasattr\u001b[39m(func, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m__self__\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 821\u001b[0m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(func\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__self__\u001b[39m, torch\u001b[38;5;241m.\u001b[39mnn\u001b[38;5;241m.\u001b[39mModule)\n\u001b[1;32m 822\u001b[0m \u001b[38;5;129;01mand\u001b[39;00m func\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mforward\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 823\u001b[0m ):\n\u001b[1;32m 824\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m example_inputs \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n", - "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/jit/_trace.py:1074\u001b[0m, in \u001b[0;36mtrace_module\u001b[0;34m(mod, inputs, optimize, check_trace, check_inputs, check_tolerance, strict, _force_outplace, _module_class, _compilation_unit, example_inputs_is_kwarg, _store_inputs)\u001b[0m\n\u001b[1;32m 1072\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 1073\u001b[0m example_inputs \u001b[38;5;241m=\u001b[39m make_tuple(example_inputs)\n\u001b[0;32m-> 1074\u001b[0m module\u001b[38;5;241m.\u001b[39m_c\u001b[38;5;241m.\u001b[39m_create_method_from_trace(\n\u001b[1;32m 1075\u001b[0m method_name,\n\u001b[1;32m 1076\u001b[0m func,\n\u001b[1;32m 1077\u001b[0m example_inputs,\n\u001b[1;32m 1078\u001b[0m var_lookup_fn,\n\u001b[1;32m 1079\u001b[0m strict,\n\u001b[1;32m 1080\u001b[0m _force_outplace,\n\u001b[1;32m 1081\u001b[0m argument_names,\n\u001b[1;32m 1082\u001b[0m _store_inputs,\n\u001b[1;32m 1083\u001b[0m )\n\u001b[1;32m 1085\u001b[0m check_trace_method \u001b[38;5;241m=\u001b[39m module\u001b[38;5;241m.\u001b[39m_c\u001b[38;5;241m.\u001b[39m_get_method(method_name)\n\u001b[1;32m 1087\u001b[0m \u001b[38;5;66;03m# Check the trace against new traces created from user-specified inputs\u001b[39;00m\n", - "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/nn/modules/module.py:1511\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1509\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs) \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m 1510\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1511\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n", - "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/nn/modules/module.py:1520\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1515\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1516\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1517\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m 1518\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1519\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1520\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m forward_call(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m 1522\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 1523\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n", - "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/nn/modules/module.py:1501\u001b[0m, in \u001b[0;36mModule._slow_forward\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m 1499\u001b[0m recording_scopes \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[1;32m 1500\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m-> 1501\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mforward(\u001b[38;5;241m*\u001b[39m\u001b[38;5;28minput\u001b[39m, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m 1502\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[1;32m 1503\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m recording_scopes:\n", - "Cell \u001b[0;32mIn[4], line 33\u001b[0m, in \u001b[0;36mResCNN.forward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m 30\u001b[0m rel2_out \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mrel2(conv2_out)\n\u001b[1;32m 31\u001b[0m pool2_out \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpool2(rel2_out)\n\u001b[0;32m---> 33\u001b[0m conv3_out \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconv3(pool2_out \u001b[38;5;241m+\u001b[39m rel1_out)\n\u001b[1;32m 34\u001b[0m rel3_out \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mrel3(conv3_out)\n\u001b[1;32m 35\u001b[0m pool3_out \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpool3(rel3_out)\n", - "\u001b[0;31mRuntimeError\u001b[0m: The size of tensor a (4) must match the size of tensor b (24) at non-singleton dimension 3" - ] - } - ], - "source": [ - "hw_model = DynapcnnNetworkGraph(\n", - " sinabs_model,\n", - " discretize=True,\n", - " input_shape=input_shape\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "0 Conv2d(1, 20, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", - "1 IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-1.), batch_size=1, num_timesteps=-1)\n", - "2 AvgPool2d(kernel_size=2, stride=2, padding=0)\n", - "3 Conv2d(20, 32, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", - "4 IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-1.), batch_size=1, num_timesteps=-1)\n", - "5 AvgPool2d(kernel_size=2, stride=2, padding=0)\n", - "6 Conv2d(32, 128, kernel_size=(3, 3), stride=(1, 1), bias=False)\n", - "7 IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-1.), batch_size=1, num_timesteps=-1)\n", - "8 AvgPool2d(kernel_size=2, stride=2, padding=0)\n", - "9 Linear(in_features=128, out_features=500, bias=False)\n", - "10 IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-1.), batch_size=1, num_timesteps=-1)\n", - "11 Linear(in_features=500, out_features=10, bias=False)\n", - "12 IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-1.), batch_size=1, num_timesteps=-1)\n" - ] - } - ], - "source": [ - "for i, l in enumerate(hw_model.layers):\n", - " print(i, l)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(0, 1)\n", - "(1, 2)\n", - "(2, 3)\n", - "(3, 4)\n", - "(4, 5)\n", - "(5, 6)\n", - "(6, 7)\n", - "(7, 8)\n", - "(8, 9)\n", - "(9, 10)\n", - "(10, 11)\n", - "(11, 12)\n" - ] - } - ], - "source": [ - "for edge in hw_model.sinabs_edges:\n", - " print(edge)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Deploying Model" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Network is valid\n" - ] - }, - { - "data": { - "text/plain": [ - "DynapcnnNetworkGraph()" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "hw_model.to(device=\"speck2edevkit:0\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "layer index: 0\n", - "layer modules: DynapcnnLayer(\n", - " (conv_layer): Conv2d(1, 20, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", - " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", - " tensor(635.), min_v_mem=Parameter containing:\n", - " tensor(-635.), batch_size=1, num_timesteps=-1)\n", - " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", - ")\n", - "layer destinations: [1]\n", - "assigned core: 0\n", - "\n", - "layer index: 1\n", - "layer modules: DynapcnnLayer(\n", - " (conv_layer): Conv2d(20, 32, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", - " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", - " tensor(11361.), min_v_mem=Parameter containing:\n", - " tensor(-11361.), batch_size=1, num_timesteps=-1)\n", - " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", - ")\n", - "layer destinations: [2]\n", - "assigned core: 3\n", - "\n", - "layer index: 2\n", - "layer modules: DynapcnnLayer(\n", - " (conv_layer): Conv2d(32, 128, kernel_size=(3, 3), stride=(1, 1), bias=False)\n", - " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", - " tensor(8621.), min_v_mem=Parameter containing:\n", - " tensor(-8621.), batch_size=1, num_timesteps=-1)\n", - " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", - ")\n", - "layer destinations: [3]\n", - "assigned core: 5\n", - "\n", - "layer index: 3\n", - "layer modules: DynapcnnLayer(\n", - " (conv_layer): Conv2d(128, 500, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", - " tensor(5747.), min_v_mem=Parameter containing:\n", - " tensor(-5747.), batch_size=1, num_timesteps=-1)\n", - ")\n", - "layer destinations: [4]\n", - "assigned core: 6\n", - "\n", - "layer index: 4\n", - "layer modules: DynapcnnLayer(\n", - " (conv_layer): Conv2d(500, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", - " tensor(2841.), min_v_mem=Parameter containing:\n", - " tensor(-2841.), batch_size=1, num_timesteps=-1)\n", - ")\n", - "layer destinations: []\n", - "assigned core: 1\n", - "\n" - ] - } - ], - "source": [ - "print(hw_model)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "speck-rescnn", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} From f12a2b464788e6f76e6514ec5588c1a2d6ac15f9 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Wed, 24 Apr 2024 14:02:50 +0200 Subject: [PATCH 037/379] model forward computed from DynapcnnLayer instances and their connectivity --- .../dynapcnn/dynapcnn_network_graph.py | 57 +++++---- .../dynapcnn/dynapcnnnetwork_module.py | 108 +++++++++++++++--- ...DynapcnnNetworkGraph_from_NIRgraph_4.ipynb | 37 +----- 3 files changed, 135 insertions(+), 67 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py index 87a29da3..3fa118ec 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py @@ -37,6 +37,9 @@ class DynapcnnNetworkGraph(nn.Module): """Given a sinabs spiking network, prepare a dynapcnn-compatible network. This can be used to test the network will be equivalent once on DYNAPCNN. This class also provides utilities to make the dynapcnn configuration and upload it to DYNAPCNN. + + TODO turn what is now the `forward` in `self.network = DynapcnnNetworkModule` into a forward method for this class. + TODO `make_config` and `_make_config` should be merged into a single method. """ def __init__( @@ -87,6 +90,11 @@ def __init__( discretize=discretize, edges=self.sinabs_edges, nodes_to_dcnnl_map=self.nodes_to_dcnnl_map) + + # the trainable network instance: set at the end of the `.make_config()` call if configuration is valid. + self.network = None + + ### Public Methods ### def __str__(self): pretty_print = '' @@ -103,24 +111,6 @@ def __str__(self): pretty_print += f'\n> layer modules: {layer}\n> layer destinations: {dest}\n> assigned core: {core}\n\n' return pretty_print - def get_network_module(self): - """ .""" - - # get connections between `DynapcnnLayer`s. - dcnnl_edges = self._get_dynapcnnlayers_edges() - - network_module = DynapcnnNetworkModule(dcnnl_edges, self.dynapcnn_layers) - - def _get_dynapcnnlayers_edges(self) -> List[Tuple[int, int]]: - """ Create edges representing connections between `DynapcnnLayer` instances. """ - dcnnl_edges = [] - - for dcnnl_idx, layer_data in self.dynapcnn_layers.items(): - for dest in layer_data['destinations']: - dcnnl_edges.append((dcnnl_idx, dest)) - - return dcnnl_edges - def to( self, device="cpu", @@ -222,7 +212,7 @@ def to( else: raise Exception("Unknown device description.") - + def make_config( self, chip_layers_ordering: Union[Sequence[int], str] = "auto", @@ -278,12 +268,39 @@ def make_config( config_modifier=config_modifier, ) - if is_compatible: # validate config. + if is_compatible: + # validate config. print("Network is valid") + + # constructs a `nn.Module` class combining the `DynapcnnLayer` uploaded to the chip. + self.network = self._get_network_module() + return config else: raise ValueError(f"Generated config is not valid for {device}") + ### Private Methods ### + + def _get_network_module(self) -> nn.Module: + """ Uses the `DynapcnnLayer` instances in `self.dynapcnn_layers` and the connectivity between the cores + to craete a `nn.Module` with a forward method that incorporates each `DynapcnnLayer` into a trainable network. + """ + + # get connections between `DynapcnnLayer`s. + dcnnl_edges = self._get_dynapcnnlayers_edges() + + return DynapcnnNetworkModule(dcnnl_edges, self.dynapcnn_layers) + + def _get_dynapcnnlayers_edges(self) -> List[Tuple[int, int]]: + """ Create edges representing connections between `DynapcnnLayer` instances. """ + dcnnl_edges = [] + + for dcnnl_idx, layer_data in self.dynapcnn_layers.items(): + for dest in layer_data['destinations']: + dcnnl_edges.append((dcnnl_idx, dest)) + + return dcnnl_edges + def _make_config( self, chip_layers_ordering: Union[Sequence[int], str] = "auto", diff --git a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py index 6013ef3e..c1bdd1c6 100644 --- a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py +++ b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py @@ -3,20 +3,31 @@ # contact : williansoaresgirao@gmail.com import torch.nn as nn -from typing import List, Tuple, Dict +from sinabs.layers import Merge +from typing import List, Tuple, Dict, Union +import copy class DynapcnnNetworkModule(nn.Module): - """ .""" + """ + Uses the set of `DynapcnnLayer` instances that have been configured to the chip and how they address each other + to define what the `forward` method of the model should do. + + Parameters + ---------- + dcnnl_edges (list): tuples representing the output->input mapping between `DynapcnnLayer` instances + that have been used as configuration for each core `CNNLayerConifg`. + dynapcnn_layers (dict): the `DynapcnnLayer` instances along with their supporting metadata (e.g. assigned core, + destination layers, etc.). + """ + def __init__(self, dcnnl_edges: List[Tuple[int, int]], dynapcnn_layers: Dict) -> nn.Module: super().__init__() - self.model_forward = self._build_module_forward_from_graph(dcnnl_edges, dynapcnn_layers) + self._forward_edges, self._forward_map = self._build_module_forward_from_graph(dcnnl_edges, dynapcnn_layers) - def _build_module_forward_from_graph(self, dcnnl_edges, dynapcnn_layers): + def _build_module_forward_from_graph(self, dcnnl_edges: list, dynapcnn_layers: dict) -> Union[list, dict]: """ - ... - - TODO the `Merge` layer has to be recreated here if a node appears as the targert in more than one edge. + TODO use copy.deepcopy for create the `forward_map`. """ forward_map = {} new_edges_set = [] @@ -77,12 +88,81 @@ def _build_module_forward_from_graph(self, dcnnl_edges, dynapcnn_layers): if len(new_edge_2_append) != 0: new_edges_set.extend(new_edge_2_append) - print('original edges: ') - for edge in dcnnl_edges: - print(edge) + forward_edges = self._find_merging_nodes(new_edges_set, forward_map) + + return forward_edges, forward_map + + def _find_merging_nodes(self, edges_list: list, forward_map: dict) -> list: + """ Loops through the edges and see if a node appeards in more than one edge. If so, this is a node + that requires a `Merge` layer. For instance, edges `(A, X)` and `(B, X)` will be replace by two new + edges `((A, B), Merge_X)` and `(Merge_X, X)`, where `A` and `B` are the inputs to a `Merge` feeding into `X`. + """ + merge_mapping = {} + + for edge in edges_list: + src = edge[0] + trg = edge[1] + + if trg in merge_mapping: + # node needs to receive input from a `Merge` layer. + merge_arguments = ( + merge_mapping[trg]['src'], # merge_arguments[0] = source (from 1st edge containing `trg`). + src) # merge_arguments[1] = `src` (the source of the 2nd edge containing `trg`). + + merge_mapping[trg] = {'src': merge_arguments} + + else: + merge_mapping[trg] = {'src': src} + + final_edges = [] + merge_idx = 0 + + # create edges `((A, B), Merge_X)` and `(Merge_X, X)`. + for trg, src in merge_mapping.items(): + _ = src['src'] + + if isinstance(_, tuple): + # `trg` receives from a `Merge` layer. + merge_node = f'merge_{merge_idx}' + forward_map[merge_node] = Merge() + + new_edge = (_, merge_node) + final_edges.append(new_edge) + + new_edge = (merge_node, trg) + final_edges.append(new_edge) + + merge_idx += 1 + + else: + final_edges.append((_, trg)) - print('\nforward edges: ') - for edge in new_edges_set: - print(edge) + return final_edges + + def forward(self, x): + """ The torch forward uses `self._forward_edges` to feed data throguh the + layers in `self._forward_map`. + """ + + layers_outputs = {} + + # input node has to be `0`. + layers_outputs[0] = self._forward_map[0](x) + + for edge in self._forward_edges: + src = edge[0] + trg = edge[1] + + # gets the input to the target node (must have been computed already). + if isinstance(src, tuple): + # `trg` is a Merge layer. + arg1 = layers_outputs[src[0]] + arg2 = layers_outputs[src[1]] + + layers_outputs[trg] = self._forward_map[trg](arg1, arg2) + + else: + x = layers_outputs[src] + layers_outputs[trg] = self._forward_map[trg](x) - return forward_map + return layers_outputs[trg] \ No newline at end of file diff --git a/tests/test_nonsequential/DynapcnnNetworkGraph_from_NIRgraph_4.ipynb b/tests/test_nonsequential/DynapcnnNetworkGraph_from_NIRgraph_4.ipynb index 6d15f9aa..25830f00 100644 --- a/tests/test_nonsequential/DynapcnnNetworkGraph_from_NIRgraph_4.ipynb +++ b/tests/test_nonsequential/DynapcnnNetworkGraph_from_NIRgraph_4.ipynb @@ -21,7 +21,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 2, @@ -298,7 +298,9 @@ { "data": { "text/plain": [ - "DynapcnnNetworkGraph()" + "DynapcnnNetworkGraph(\n", + " (network): DynapcnnNetworkModule()\n", + ")" ] }, "execution_count": 9, @@ -374,37 +376,6 @@ "source": [ "print(hw_model)" ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "original edges: \n", - "(0, 1)\n", - "(0, 2)\n", - "(1, 2)\n", - "(2, 3)\n", - "(3, 4)\n", - "\n", - "forward edges: \n", - "(0, '0_pool0')\n", - "(0, '0_pool1')\n", - "('0_pool0', 1)\n", - "('0_pool1', 2)\n", - "(1, 2)\n", - "(2, 3)\n", - "(3, 4)\n" - ] - } - ], - "source": [ - "hw_model.get_network_module()" - ] } ], "metadata": { From b0b8e12c44a6c507e612026230217524e134c595 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Wed, 24 Apr 2024 17:03:13 +0200 Subject: [PATCH 038/379] (WIP) training DynapcnnNetwork model --- .../dynapcnn/dynapcnnnetwork_module.py | 27 ++ ...DynapcnnNetworkGraph_from_NIRgraph_4.ipynb | 447 +++++++++++++----- 2 files changed, 364 insertions(+), 110 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py index c1bdd1c6..61b5e3c3 100644 --- a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py +++ b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py @@ -6,6 +6,8 @@ from sinabs.layers import Merge from typing import List, Tuple, Dict, Union import copy +import sinabs +import sinabs.layers as sl class DynapcnnNetworkModule(nn.Module): """ @@ -139,6 +141,31 @@ def _find_merging_nodes(self, edges_list: list, forward_map: dict) -> list: return final_edges + def parameters(self) -> list: + """ .""" + parameters = [] + + for module in self._forward_map.values(): + if isinstance(module, sinabs.backend.dynapcnn.dynapcnn_layer_new.DynapcnnLayer): + parameters.extend(module.conv_layer.parameters()) + + return parameters + + def init_weights(self): + """ .""" + for node, module in self._forward_map.items(): + if isinstance(module, nn.Conv2d): + nn.init.xavier_normal_(module.weight.data) + + def detach_neuron_states(self) -> None: + """ Detach the neuron states and activations from current computation graph (necessary). """ + + for module in self._forward_map.values(): + if isinstance(module, sinabs.backend.dynapcnn.dynapcnn_layer_new.DynapcnnLayer): + if isinstance(module.spk_layer, sl.StatefulLayer): + for name, buffer in module.spk_layer.named_buffers(): + buffer.detach_() + def forward(self, x): """ The torch forward uses `self._forward_edges` to feed data throguh the layers in `self._forward_map`. diff --git a/tests/test_nonsequential/DynapcnnNetworkGraph_from_NIRgraph_4.ipynb b/tests/test_nonsequential/DynapcnnNetworkGraph_from_NIRgraph_4.ipynb index 25830f00..2bedf5be 100644 --- a/tests/test_nonsequential/DynapcnnNetworkGraph_from_NIRgraph_4.ipynb +++ b/tests/test_nonsequential/DynapcnnNetworkGraph_from_NIRgraph_4.ipynb @@ -21,7 +21,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 2, @@ -39,9 +39,9 @@ "metadata": {}, "outputs": [], "source": [ - "channels = 1\n", - "height = 28\n", - "width = 28\n", + "channels = 2\n", + "height = 34\n", + "width = 34\n", "\n", "input_shape = (channels, height, width)" ] @@ -50,7 +50,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Network Module (pure Pytorch)" + "## Network Module\n", + "\n", + "We need to define a `nn.Module` implementing the network we want the chip to reproduce." ] }, { @@ -63,21 +65,21 @@ " def __init__(self) -> None:\n", " super().__init__()\n", "\n", - " self.conv1 = nn.Conv2d(1, 30, 2, 1, bias=False) # node 0\n", + " self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False) # node 0\n", " self.iaf1 = IAFSqueeze(batch_size=1) # node 1\n", - " self.pool1 = nn.AvgPool2d(2,2) # node 2\n", + " self.pool1 = nn.AvgPool2d(3,3) # node 2\n", " self.pool1a = nn.AvgPool2d(4,4) # node 3\n", "\n", - " self.conv2 = nn.Conv2d(30, 30, 2, 1, bias=False)# node 4\n", + " self.conv2 = nn.Conv2d(10, 10, 4, 1, bias=False)# node 4\n", " self.iaf2 = IAFSqueeze(batch_size=1) # node 6\n", - " self.pool2 = nn.AvgPool2d(2,2) # node 7\n", + " # self.pool2 = nn.AvgPool2d(3,3) # node 7\n", "\n", - " self.conv3 = nn.Conv2d(30, 1, 3, 1, bias=False) # node 8\n", + " self.conv3 = nn.Conv2d(10, 1, 2, 1, bias=False) # node 8\n", " self.iaf3 = IAFSqueeze(batch_size=1) # node 9\n", "\n", " self.flat = nn.Flatten()\n", "\n", - " self.fc1 = nn.Linear(16, 500, bias=False) # node 10\n", + " self.fc1 = nn.Linear(49, 500, bias=False) # node 10\n", " self.iaf4 = IAFSqueeze(batch_size=1) # node 11\n", " \n", " self.fc2 = nn.Linear(500, 10, bias=False) # node 12\n", @@ -94,9 +96,9 @@ "\n", " conv2_out = self.conv2(pool1_out)\n", " iaf2_out = self.iaf2(conv2_out)\n", - " pool2_out = self.pool2(iaf2_out)\n", + " # pool2_out = self.pool2(iaf2_out)\n", "\n", - " conv3_out = self.conv3(self.adder(pool1a_out, pool2_out))\n", + " conv3_out = self.conv3(self.adder(pool1a_out, iaf2_out))\n", " iaf3_out = self.iaf3(conv3_out)\n", "\n", " flat_out = self.flat(iaf3_out)\n", @@ -118,6 +120,13 @@ "snn = SNN()" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The following exemplifies how each of the layers in the SNN should be grouped together to form a `DynapcnnLayer` instance. Let's check the shapes of the tensors each of the (original) layers in the model inputs and outputs by feeding a fake input through the model:" + ] + }, { "cell_type": "code", "execution_count": 6, @@ -127,26 +136,25 @@ "name": "stdout", "output_type": "stream", "text": [ - "DynapcnnLayer 0 [core 0]: ... torch.Size([1, 1, 28, 28])\n", - " conv1: [1, 30, 27, 27]\n", - " iaf1: [1, 30, 27, 27]\n", - " pool1: [1, 30, 13, 13]\n", - " pool1a: [1, 30, 6, 6]\n", + "DynapcnnLayer 0: ... [1, 2, 34, 34]\n", + " conv1: [1, 10, 33, 33]\n", + " iaf1: [1, 10, 33, 33]\n", + " pool1: [1, 10, 11, 11]\n", + " pool1a: [1, 10, 8, 8]\n", "\n", - "DynapcnnLayer 1 [core 1]: ... [1, 30, 13, 13]\n", - " conv2: [1, 30, 12, 12]\n", - " iaf2: [1, 30, 12, 12]\n", - " pool2: [1, 30, 6, 6]\n", + "DynapcnnLayer 1: ... [1, 10, 11, 11]\n", + " conv2: [1, 10, 8, 8]\n", + " iaf2: [1, 10, 8, 8]\n", "\n", - "DynapcnnLayer 2 [core 2]: ... [1, 30, 6, 6] [ Merge(pool1a, pool2) ]\n", - " conv3: [1, 1, 4, 4]\n", - " iaf3: [1, 1, 4, 4]\n", + "DynapcnnLayer 2: ... [1, 10, 8, 8] [ Merge(pool1a, iaf2_out) ]\n", + " conv3: [1, 1, 7, 7]\n", + " iaf3: [1, 1, 7, 7]\n", "\n", - "DynapcnnLayer 3 [core ]: ... [1, 16]\n", + "DynapcnnLayer 3: ... [1, 49]\n", " fc1: [1, 500]\n", " iaf4: [1, 500]\n", "\n", - "DynapcnnLayer 4 [core ]: ... [1, 500]\n", + "DynapcnnLayer 4: ... [1, 500]\n", " fc2: [1, 10]\n", " iaf5: [1, 10]\n", "\n" @@ -156,7 +164,7 @@ "source": [ "x = torch.randn((1, *input_shape))\n", "\n", - "print(f'DynapcnnLayer 0 [core 0]: ... {(x.shape)}')\n", + "print(f'DynapcnnLayer 0: ... {list(x.shape)}')\n", "con1_out = snn.conv1(x)\n", "print(f' conv1: {list(con1_out.shape)}')\n", "iaf1_out = snn.iaf1(con1_out)\n", @@ -166,17 +174,17 @@ "pool1a_out = snn.pool1a(iaf1_out)\n", "print(f' pool1a: {list(pool1a_out.shape)}\\n')\n", "\n", - "print(f'DynapcnnLayer 1 [core 1]: ... {list(pool1_out.shape)}')\n", + "print(f'DynapcnnLayer 1: ... {list(pool1_out.shape)}')\n", "conv2_out = snn.conv2(pool1_out)\n", "print(f' conv2: {list(conv2_out.shape)}')\n", "iaf2_out = snn.iaf2(conv2_out)\n", - "print(f' iaf2: {list(iaf2_out.shape)}')\n", - "pool2_out = snn.pool2(iaf2_out)\n", - "print(f' pool2: {list(pool2_out.shape)}\\n')\n", + "print(f' iaf2: {list(iaf2_out.shape)}\\n')\n", + "# pool2_out = snn.pool2(iaf2_out)\n", + "# print(f' pool2: {list(pool2_out.shape)}\\n')\n", "\n", - "added = snn.adder(pool1a_out, pool2_out)\n", + "added = snn.adder(pool1a_out, iaf2_out)\n", "\n", - "print(f'DynapcnnLayer 2 [core 2]: ... {list(added.shape)} [ Merge(pool1a, pool2) ]')\n", + "print(f'DynapcnnLayer 2: ... {list(added.shape)} [ Merge(pool1a, iaf2_out) ]')\n", "conv3_out = snn.conv3(added)\n", "print(f' conv3: {list(conv3_out.shape)}')\n", "iaf3_out = snn.iaf3(conv3_out)\n", @@ -184,14 +192,14 @@ "\n", "flat_out = snn.flat(iaf3_out)\n", "\n", - "print(f'DynapcnnLayer 3 [core ]: ... {list(flat_out.shape)}')\n", + "print(f'DynapcnnLayer 3: ... {list(flat_out.shape)}')\n", "fc1_out = snn.fc1(flat_out)\n", "print(f' fc1: {list(fc1_out.shape)}')\n", "iaf4_out = snn.iaf4(fc1_out)\n", "print(f' iaf4: {list(iaf4_out.shape)}\\n')\n", "\n", "\n", - "print(f'DynapcnnLayer 4 [core ]: ... {list(iaf4_out.shape)}')\n", + "print(f'DynapcnnLayer 4: ... {list(iaf4_out.shape)}')\n", "fc2_out = snn.fc2(iaf4_out)\n", "print(f' fc2: {list(fc2_out.shape)}')\n", "iaf5_out = snn.iaf5(fc2_out)\n", @@ -202,7 +210,13 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## DynapcnnNetwork" + "## DynapcnnNetwork Class\n", + "\n", + "In the constructor of `DynapcnnNetworkGraph` the SNN passed as argument (defined as a `nn.Module`) will be parsed such that each layer is represented in a computational graph (using `nirtorch.extract_torch_graph`). \n", + "\n", + "The layers are the `nodes` of the graph, while their connectivity (how the outputs from a layer are sent to other layers) is represented as `edges`, represented in a `list` of `tuples`.\n", + "\n", + "Once the constructor finishes its initialization, the `hw_model.dynapcnn_layers` property is a dictionary where each entry represents the ID of a `DynapcnnLayer` instance (an `int` from `0` to `L`), with this entry containing a `DynapcnnLayer` instance where a subset of the layers in the original SNN has been incorporated into, the core such instance has been assigned to, and the list of `DynapcnnLayer` instances (their IDs) the layer targets." ] }, { @@ -219,73 +233,17 @@ ] }, { - "cell_type": "code", - "execution_count": 8, + "cell_type": "markdown", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "---- DynapcnnLayer 0 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 0): Conv2d(1, 30, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", - "(node 1): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "(node 2): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", - "(node 3): SumPool2d(norm_type=1, kernel_size=(4, 4), stride=None, ceil_mode=False)\n", - "> layer destinations: [1, 2]\n", - "> assigned core: -1\n", - "\n", - "---- DynapcnnLayer 1 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 4): Conv2d(30, 30, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", - "(node 6): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "(node 7): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", - "> layer destinations: [2]\n", - "> assigned core: -1\n", - "\n", - "---- DynapcnnLayer 2 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 8): Conv2d(30, 1, kernel_size=(3, 3), stride=(1, 1), bias=False)\n", - "(node 9): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [3]\n", - "> assigned core: -1\n", - "\n", - "---- DynapcnnLayer 3 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 10): Conv2d(1, 500, kernel_size=(4, 4), stride=(1, 1), bias=False)\n", - "(node 11): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [4]\n", - "> assigned core: -1\n", - "\n", - "---- DynapcnnLayer 4 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 12): Conv2d(500, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - "(node 13): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: []\n", - "> assigned core: -1\n", - "\n", - "\n" - ] - } - ], "source": [ - "print(hw_model)" + "The `hw_model.to()` call will figure out into which core eac `DynapcnnLayer` instance will be assigned to. Once this assingment is made the instance itself is used to configure the `CNNLayerConfig` instance representing the core's configuration.\n", + "\n", + "If the cores' configuration is valid, each `DynapcnnLayer` instance and their respective destinations will be used to create a computational graph that encodes how the `forward` method of `hw_model.network` (a `nn.Module` using the `DynapcnnLayer` instances) propagates that through the network." ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 8, "metadata": {}, "outputs": [ { @@ -303,7 +261,7 @@ ")" ] }, - "execution_count": 9, + "execution_count": 8, "metadata": {}, "output_type": "execute_result" } @@ -312,9 +270,16 @@ "hw_model.to(device=\"speck2edevkit:0\")" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The layers comprising our `hw_model` and their respective metadata can be inspected by calling `print` on a `DynapcnnNetworkGraph` instance." + ] + }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 9, "metadata": {}, "outputs": [ { @@ -323,29 +288,28 @@ "text": [ "---- DynapcnnLayer 0 ----------------------------------------------------------\n", "> layer modules: \n", - "(node 0): Conv2d(1, 30, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + "(node 0): Conv2d(2, 10, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", "(node 1): IAFSqueeze(spike_threshold=Parameter containing:\n", "tensor(1.), min_v_mem=Parameter containing:\n", "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "(node 2): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", + "(node 2): SumPool2d(norm_type=1, kernel_size=(3, 3), stride=None, ceil_mode=False)\n", "(node 3): SumPool2d(norm_type=1, kernel_size=(4, 4), stride=None, ceil_mode=False)\n", "> layer destinations: [1, 2]\n", "> assigned core: 0\n", "\n", "---- DynapcnnLayer 1 ----------------------------------------------------------\n", "> layer modules: \n", - "(node 4): Conv2d(30, 30, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + "(node 4): Conv2d(10, 10, kernel_size=(4, 4), stride=(1, 1), bias=False)\n", "(node 6): IAFSqueeze(spike_threshold=Parameter containing:\n", "tensor(1.), min_v_mem=Parameter containing:\n", "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "(node 7): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", "> layer destinations: [2]\n", "> assigned core: 1\n", "\n", "---- DynapcnnLayer 2 ----------------------------------------------------------\n", "> layer modules: \n", - "(node 8): Conv2d(30, 1, kernel_size=(3, 3), stride=(1, 1), bias=False)\n", - "(node 9): IAFSqueeze(spike_threshold=Parameter containing:\n", + "(node 7): Conv2d(10, 1, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + "(node 8): IAFSqueeze(spike_threshold=Parameter containing:\n", "tensor(1.), min_v_mem=Parameter containing:\n", "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", "> layer destinations: [3]\n", @@ -353,8 +317,8 @@ "\n", "---- DynapcnnLayer 3 ----------------------------------------------------------\n", "> layer modules: \n", - "(node 10): Conv2d(1, 500, kernel_size=(4, 4), stride=(1, 1), bias=False)\n", - "(node 11): IAFSqueeze(spike_threshold=Parameter containing:\n", + "(node 9): Conv2d(1, 500, kernel_size=(7, 7), stride=(1, 1), bias=False)\n", + "(node 10): IAFSqueeze(spike_threshold=Parameter containing:\n", "tensor(1.), min_v_mem=Parameter containing:\n", "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", "> layer destinations: [4]\n", @@ -362,8 +326,8 @@ "\n", "---- DynapcnnLayer 4 ----------------------------------------------------------\n", "> layer modules: \n", - "(node 12): Conv2d(500, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - "(node 13): IAFSqueeze(spike_threshold=Parameter containing:\n", + "(node 11): Conv2d(500, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + "(node 12): IAFSqueeze(spike_threshold=Parameter containing:\n", "tensor(1.), min_v_mem=Parameter containing:\n", "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", "> layer destinations: []\n", @@ -376,6 +340,269 @@ "source": [ "print(hw_model)" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Training our DynapcnnNetwork\n", + "\n", + "Preparing the data..." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "# https://synsense.gitlab.io/sinabs-dynapcnn/getting_started/notebooks/nmnist_quick_start.html\n", + "\n", + "from tonic.datasets.nmnist import NMNIST\n", + "from tonic.transforms import ToFrame\n", + "from torch.utils.data import DataLoader\n", + "from torch.nn import CrossEntropyLoss\n", + "from torch.optim import SGD\n", + "from tqdm.notebook import tqdm\n", + " \n", + "# download dataset\n", + "root_dir = \"./NMNIST\"\n", + "_ = NMNIST(save_to=root_dir, train=True)\n", + "_ = NMNIST(save_to=root_dir, train=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "type of data is: \n", + "(4686,)\n", + "time length of sample data is: 300760 micro seconds\n", + "there are 4686 events in the sample data\n", + "the label of the sample data is: 5\n" + ] + } + ], + "source": [ + "sample_data, label = NMNIST(save_to=root_dir, train=False)[0]\n", + "\n", + "print(f\"type of data is: {type(sample_data)}\")\n", + "print(sample_data.shape)\n", + "print(f\"time length of sample data is: {sample_data['t'][-1] - sample_data['t'][0]} micro seconds\")\n", + "print(f\"there are {len(sample_data)} events in the sample data\")\n", + "print(f\"the label of the sample data is: {label}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The transformed array is in shape [Time-Step, Channel, Height, Width] --> (100, 2, 34, 34)\n" + ] + } + ], + "source": [ + "n_time_steps = 100\n", + "to_raster = ToFrame(sensor_size=NMNIST.sensor_size, n_time_bins=n_time_steps)\n", + "\n", + "snn_train_dataset = NMNIST(save_to=root_dir, train=True, transform=to_raster)\n", + "snn_test_dataset = NMNIST(save_to=root_dir, train=False, transform=to_raster)\n", + "\n", + "# check the transformed data\n", + "sample_data, label = snn_train_dataset[0]\n", + "print(f\"The transformed array is in shape [Time-Step, Channel, Height, Width] --> {sample_data.shape}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Setting the training hyperparameters..." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "# if torch.cuda.is_available():\n", + "# device = torch.device('cuda:0')\n", + "# print('device: ', torch.cuda.get_device_name(0))\n", + "# else:\n", + "device = torch.device('cpu')" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "epochs = 1\n", + "lr = 1e-3\n", + "batch_size = 4\n", + "num_workers = 4\n", + "device = \"cuda:0\"\n", + "shuffle = True" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "snn_train_dataloader = DataLoader(snn_train_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True)\n", + "snn_test_dataloader = DataLoader(snn_test_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=False)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Initializing our weights..." + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "hw_model.network.init_weights()" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "DynapcnnNetworkModule()" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "hw_model.network.to(device=device)" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "optimizer = SGD(params=hw_model.network.parameters(), lr=lr)\n", + "criterion = CrossEntropyLoss()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The training loop..." + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "77410e6f9aab4c7a9515e31cec5c52de", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/15000 [00:00 7\u001b[0m data \u001b[38;5;241m=\u001b[39m data\u001b[38;5;241m.\u001b[39mreshape(\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m, \u001b[38;5;241m2\u001b[39m, \u001b[38;5;241m34\u001b[39m, \u001b[38;5;241m34\u001b[39m)\u001b[38;5;241m.\u001b[39mto(dtype\u001b[38;5;241m=\u001b[39mtorch\u001b[38;5;241m.\u001b[39mfloat, device\u001b[38;5;241m=\u001b[39mdevice)\n\u001b[1;32m 8\u001b[0m label \u001b[38;5;241m=\u001b[39m label\u001b[38;5;241m.\u001b[39mto(dtype\u001b[38;5;241m=\u001b[39mtorch\u001b[38;5;241m.\u001b[39mlong, device\u001b[38;5;241m=\u001b[39mdevice)\n\u001b[1;32m 9\u001b[0m \u001b[38;5;66;03m# forward\u001b[39;00m\n", + "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/cuda/__init__.py:302\u001b[0m, in \u001b[0;36m_lazy_init\u001b[0;34m()\u001b[0m\n\u001b[1;32m 300\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCUDA_MODULE_LOADING\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m os\u001b[38;5;241m.\u001b[39menviron:\n\u001b[1;32m 301\u001b[0m os\u001b[38;5;241m.\u001b[39menviron[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCUDA_MODULE_LOADING\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mLAZY\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m--> 302\u001b[0m torch\u001b[38;5;241m.\u001b[39m_C\u001b[38;5;241m.\u001b[39m_cuda_init()\n\u001b[1;32m 303\u001b[0m \u001b[38;5;66;03m# Some of the queued calls may reentrantly call _lazy_init();\u001b[39;00m\n\u001b[1;32m 304\u001b[0m \u001b[38;5;66;03m# we need to just return without initializing in that case.\u001b[39;00m\n\u001b[1;32m 305\u001b[0m \u001b[38;5;66;03m# However, we must not let any *other* threads in!\u001b[39;00m\n\u001b[1;32m 306\u001b[0m _tls\u001b[38;5;241m.\u001b[39mis_initializing \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n", + "\u001b[0;31mRuntimeError\u001b[0m: The NVIDIA driver on your system is too old (found version 11040). Please update your GPU driver by downloading and installing a new version from the URL: http://www.nvidia.com/Download/index.aspx Alternatively, go to: https://pytorch.org to install a PyTorch version that has been compiled with your version of the CUDA driver." + ] + } + ], + "source": [ + "for e in range(epochs):\n", + "\n", + " # train\n", + " train_p_bar = tqdm(snn_train_dataloader)\n", + " for data, label in train_p_bar:\n", + " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", + " data = data.reshape(-1, 2, 34, 34).to(dtype=torch.float, device=device)\n", + " label = label.to(dtype=torch.long, device=device)\n", + " # forward\n", + " optimizer.zero_grad()\n", + " output = hw_model.network(data)\n", + " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", + " output = output.reshape(batch_size, n_time_steps, -1)\n", + " # accumulate all time-steps output for final prediction\n", + " output = output.sum(dim=1)\n", + " loss = criterion(output, label)\n", + " # backward\n", + " loss.backward()\n", + " optimizer.step()\n", + " \n", + " # detach the neuron states and activations from current computation graph(necessary)\n", + " hw_model.network.detach_neuron_states()\n", + " \n", + " # set progressing bar\n", + " train_p_bar.set_description(f\"Epoch {e} - BPTT Training Loss: {round(loss.item(), 4)}\")\n", + "\n", + " # validate\n", + " correct_predictions = []\n", + " with torch.no_grad():\n", + " test_p_bar = tqdm(snn_test_dataloader)\n", + " for data, label in test_p_bar:\n", + " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", + " data = data.reshape(-1, 2, 34, 34).to(dtype=torch.float, device=device)\n", + " label = label.to(dtype=torch.long, device=device)\n", + " # forward\n", + " output = hw_model.network(data)\n", + " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", + " output = output.reshape(batch_size, n_time_steps, -1)\n", + " # accumulate all time-steps output for final prediction\n", + " output = output.sum(dim=1)\n", + " # calculate accuracy\n", + " pred = output.argmax(dim=1, keepdim=True)\n", + " # compute the total correct predictions\n", + " correct_predictions.append(pred.eq(label.view_as(pred)))\n", + " # set progressing bar\n", + " test_p_bar.set_description(f\"Epoch {e} - BPTT Testing Model...\")\n", + " \n", + " correct_predictions = torch.cat(correct_predictions)\n", + " print(f\"Epoch {e} - BPTT accuracy: {correct_predictions.sum().item()/(len(correct_predictions))*100}%\")" + ] } ], "metadata": { From cfc847638cc0a63d55a6254556a8f138312d5320 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Thu, 25 Apr 2024 19:00:06 +0200 Subject: [PATCH 039/379] DynapcnnNetwork model is trainable --- sinabs/backend/dynapcnn/dynapcnn_layer_new.py | 2 +- .../dynapcnn/dynapcnn_network_graph.py | 6 +- .../dynapcnn/dynapcnnnetwork_module.py | 34 +++-- ...DynapcnnNetworkGraph_from_NIRgraph_4.ipynb | 116 +++++++++++------- 4 files changed, 99 insertions(+), 59 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer_new.py b/sinabs/backend/dynapcnn/dynapcnn_layer_new.py index 59c4c142..1a07827d 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer_new.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer_new.py @@ -138,7 +138,7 @@ def forward(self, x): x = self.pool_layer[0](x) return x - + def _convert_linear_to_conv(self, lin: nn.Linear, layer_data: dict) -> nn.Conv2d: """Convert Linear layer to Conv2d. diff --git a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py index 3fa118ec..ed1bdff2 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py @@ -33,12 +33,11 @@ from .dynapcnnnetwork_module import DynapcnnNetworkModule -class DynapcnnNetworkGraph(nn.Module): +class DynapcnnNetworkGraph(): """Given a sinabs spiking network, prepare a dynapcnn-compatible network. This can be used to test the network will be equivalent once on DYNAPCNN. This class also provides utilities to make the dynapcnn configuration and upload it to DYNAPCNN. - TODO turn what is now the `forward` in `self.network = DynapcnnNetworkModule` into a forward method for this class. TODO `make_config` and `_make_config` should be merged into a single method. """ @@ -58,7 +57,6 @@ def __init__( discretize: If `True`, discretize the parameters and thresholds. This is needed for uploading weights to dynapcnn. Set to `False` only for testing purposes. """ - super().__init__() # TODO for now the graph part is not taking into consideration DVS inputs. # check if dvs input is expected. @@ -91,7 +89,7 @@ def __init__( edges=self.sinabs_edges, nodes_to_dcnnl_map=self.nodes_to_dcnnl_map) - # the trainable network instance: set at the end of the `.make_config()` call if configuration is valid. + # the trainable network (a `nn.Module`) instance: set at the end of the `.make_config()` call if configuration is valid. self.network = None ### Public Methods ### diff --git a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py index 61b5e3c3..869bf7eb 100644 --- a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py +++ b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py @@ -44,7 +44,7 @@ def _build_module_forward_from_graph(self, dcnnl_edges: list, dynapcnn_layers: d # processing the source `DynapcnnLayer`. if source_dcnnl not in forward_map: - forward_map[source_dcnnl] = dynapcnn_layers[source_dcnnl]['layer'] + forward_map[source_dcnnl] = copy.deepcopy(dynapcnn_layers[source_dcnnl]['layer']) if len(forward_map[source_dcnnl].pool_layer) > 1: # this `DynapcnnLayer` is a divergent point in the graph. @@ -57,7 +57,7 @@ def _build_module_forward_from_graph(self, dcnnl_edges: list, dynapcnn_layers: d # create forward 'node' for the i-th pooling layer. if pool_name not in forward_map: - forward_map[pool_name] = forward_map[source_dcnnl].pool_layer[i] + forward_map[pool_name] = copy.deepcopy(forward_map[source_dcnnl].pool_layer[i]) # create edge from i-th pooling to its target `DynapcnnLayer`. new_edge_2_append.append((pool_name, dynapcnn_layers[source_dcnnl]['destinations'][i])) @@ -65,7 +65,7 @@ def _build_module_forward_from_graph(self, dcnnl_edges: list, dynapcnn_layers: d # processing the target `DynapcnnLayer`. if target_dcnnl not in forward_map: - forward_map[target_dcnnl] = dynapcnn_layers[target_dcnnl]['layer'] + forward_map[target_dcnnl] = copy.deepcopy(dynapcnn_layers[target_dcnnl]['layer']) if len(forward_map[target_dcnnl].pool_layer) > 1: # this `DynapcnnLayer` is a divergent point in the graph. @@ -78,7 +78,7 @@ def _build_module_forward_from_graph(self, dcnnl_edges: list, dynapcnn_layers: d # create forward 'node' for the i-th pooling layer. if pool_name not in forward_map: - forward_map[pool_name] = forward_map[target_dcnnl].pool_layer[i] + forward_map[pool_name] = copy.deepcopy(forward_map[target_dcnnl].pool_layer[i]) # create edge from i-th pooling to its target `DynapcnnLayer`. new_edge_2_append.append((pool_name, dynapcnn_layers[target_dcnnl]['destinations'][i])) @@ -145,17 +145,31 @@ def parameters(self) -> list: """ .""" parameters = [] - for module in self._forward_map.values(): - if isinstance(module, sinabs.backend.dynapcnn.dynapcnn_layer_new.DynapcnnLayer): - parameters.extend(module.conv_layer.parameters()) + for layer in self._forward_map.values(): + if isinstance(layer, sinabs.backend.dynapcnn.dynapcnn_layer_new.DynapcnnLayer): + parameters.extend(layer.conv_layer.parameters()) return parameters def init_weights(self): """ .""" - for node, module in self._forward_map.items(): - if isinstance(module, nn.Conv2d): - nn.init.xavier_normal_(module.weight.data) + for layer in self._forward_map.values(): + if isinstance(layer, sinabs.backend.dynapcnn.dynapcnn_layer_new.DynapcnnLayer): + nn.init.xavier_normal_(layer.conv_layer.weight.data) + + def to(self, device): + """ .""" + for layer in self._forward_map.values(): + if isinstance(layer, sinabs.backend.dynapcnn.dynapcnn_layer_new.DynapcnnLayer): + layer.conv_layer.to(device) + layer.spk_layer.to(device) + + # if there's more than one pooling each of them becomes a node that is catched by the `else` statement. + if len(layer.pool_layer) == 1: + layer.pool_layer[0].to(device) + else: + # this nodes are created from `DynapcnnLayer`s that have multiple poolings (each pooling becomes a new node). + layer.to(device) def detach_neuron_states(self) -> None: """ Detach the neuron states and activations from current computation graph (necessary). """ diff --git a/tests/test_nonsequential/DynapcnnNetworkGraph_from_NIRgraph_4.ipynb b/tests/test_nonsequential/DynapcnnNetworkGraph_from_NIRgraph_4.ipynb index 2bedf5be..2be0c3c9 100644 --- a/tests/test_nonsequential/DynapcnnNetworkGraph_from_NIRgraph_4.ipynb +++ b/tests/test_nonsequential/DynapcnnNetworkGraph_from_NIRgraph_4.ipynb @@ -10,7 +10,8 @@ "import torch.nn as nn\n", "from sinabs.from_torch import from_model\n", "from sinabs.backend.dynapcnn import DynapcnnNetworkGraph\n", - "from sinabs.layers import Merge, IAFSqueeze" + "from sinabs.layers import Merge, IAFSqueeze\n", + "import copy" ] }, { @@ -21,7 +22,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 2, @@ -256,9 +257,7 @@ { "data": { "text/plain": [ - "DynapcnnNetworkGraph(\n", - " (network): DynapcnnNetworkModule()\n", - ")" + "" ] }, "execution_count": 8, @@ -267,7 +266,7 @@ } ], "source": [ - "hw_model.to(device=\"speck2edevkit:0\")" + "hw_model.to(device=\"speck2fmodule:0\")" ] }, { @@ -407,12 +406,12 @@ "name": "stdout", "output_type": "stream", "text": [ - "The transformed array is in shape [Time-Step, Channel, Height, Width] --> (100, 2, 34, 34)\n" + "The transformed array is in shape [Time-Step, Channel, Height, Width] --> (50, 2, 34, 34)\n" ] } ], "source": [ - "n_time_steps = 100\n", + "n_time_steps = 50\n", "to_raster = ToFrame(sensor_size=NMNIST.sensor_size, n_time_bins=n_time_steps)\n", "\n", "snn_train_dataset = NMNIST(save_to=root_dir, train=True, transform=to_raster)\n", @@ -434,13 +433,21 @@ "cell_type": "code", "execution_count": 13, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "device: NVIDIA GeForce RTX 3070 Ti\n" + ] + } + ], "source": [ - "# if torch.cuda.is_available():\n", - "# device = torch.device('cuda:0')\n", - "# print('device: ', torch.cuda.get_device_name(0))\n", - "# else:\n", - "device = torch.device('cpu')" + "if torch.cuda.is_available():\n", + " device = torch.device('cuda:0')\n", + " print('device: ', torch.cuda.get_device_name(0))\n", + "else:\n", + " device = torch.device('cpu')" ] }, { @@ -450,10 +457,9 @@ "outputs": [], "source": [ "epochs = 1\n", - "lr = 1e-3\n", - "batch_size = 4\n", + "lr = 1e-4\n", + "batch_size = 64\n", "num_workers = 4\n", - "device = \"cuda:0\"\n", "shuffle = True" ] }, @@ -480,27 +486,16 @@ "metadata": {}, "outputs": [], "source": [ - "hw_model.network.init_weights()" + "hw_cnn = hw_model.network" ] }, { "cell_type": "code", "execution_count": 17, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "DynapcnnNetworkModule()" - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "hw_model.network.to(device=device)" + "hw_cnn.to(device)" ] }, { @@ -509,7 +504,16 @@ "metadata": {}, "outputs": [], "source": [ - "optimizer = SGD(params=hw_model.network.parameters(), lr=lr)\n", + "hw_cnn.init_weights()" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [], + "source": [ + "optimizer = SGD(params=hw_cnn.parameters(), lr=lr)\n", "criterion = CrossEntropyLoss()" ] }, @@ -522,33 +526,53 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "losses = []\n", + "batches = []\n", + "batch_count = 0" + ] + }, + { + "cell_type": "code", + "execution_count": 20, "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "77410e6f9aab4c7a9515e31cec5c52de", + "model_id": "d73e71745e264aceb8dc7cf7320d6d6a", "version_major": 2, "version_minor": 0 }, "text/plain": [ - " 0%| | 0/15000 [00:00 7\u001b[0m data \u001b[38;5;241m=\u001b[39m data\u001b[38;5;241m.\u001b[39mreshape(\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m, \u001b[38;5;241m2\u001b[39m, \u001b[38;5;241m34\u001b[39m, \u001b[38;5;241m34\u001b[39m)\u001b[38;5;241m.\u001b[39mto(dtype\u001b[38;5;241m=\u001b[39mtorch\u001b[38;5;241m.\u001b[39mfloat, device\u001b[38;5;241m=\u001b[39mdevice)\n\u001b[1;32m 8\u001b[0m label \u001b[38;5;241m=\u001b[39m label\u001b[38;5;241m.\u001b[39mto(dtype\u001b[38;5;241m=\u001b[39mtorch\u001b[38;5;241m.\u001b[39mlong, device\u001b[38;5;241m=\u001b[39mdevice)\n\u001b[1;32m 9\u001b[0m \u001b[38;5;66;03m# forward\u001b[39;00m\n", - "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/cuda/__init__.py:302\u001b[0m, in \u001b[0;36m_lazy_init\u001b[0;34m()\u001b[0m\n\u001b[1;32m 300\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCUDA_MODULE_LOADING\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m os\u001b[38;5;241m.\u001b[39menviron:\n\u001b[1;32m 301\u001b[0m os\u001b[38;5;241m.\u001b[39menviron[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCUDA_MODULE_LOADING\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mLAZY\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m--> 302\u001b[0m torch\u001b[38;5;241m.\u001b[39m_C\u001b[38;5;241m.\u001b[39m_cuda_init()\n\u001b[1;32m 303\u001b[0m \u001b[38;5;66;03m# Some of the queued calls may reentrantly call _lazy_init();\u001b[39;00m\n\u001b[1;32m 304\u001b[0m \u001b[38;5;66;03m# we need to just return without initializing in that case.\u001b[39;00m\n\u001b[1;32m 305\u001b[0m \u001b[38;5;66;03m# However, we must not let any *other* threads in!\u001b[39;00m\n\u001b[1;32m 306\u001b[0m _tls\u001b[38;5;241m.\u001b[39mis_initializing \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n", - "\u001b[0;31mRuntimeError\u001b[0m: The NVIDIA driver on your system is too old (found version 11040). Please update your GPU driver by downloading and installing a new version from the URL: http://www.nvidia.com/Download/index.aspx Alternatively, go to: https://pytorch.org to install a PyTorch version that has been compiled with your version of the CUDA driver." + "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[20], line 18\u001b[0m\n\u001b[1;32m 16\u001b[0m loss \u001b[38;5;241m=\u001b[39m criterion(output, label)\n\u001b[1;32m 17\u001b[0m \u001b[38;5;66;03m# backward\u001b[39;00m\n\u001b[0;32m---> 18\u001b[0m loss\u001b[38;5;241m.\u001b[39mbackward()\n\u001b[1;32m 19\u001b[0m optimizer\u001b[38;5;241m.\u001b[39mstep()\n\u001b[1;32m 21\u001b[0m \u001b[38;5;66;03m# detach the neuron states and activations from current computation graph(necessary)\u001b[39;00m\n", + "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/_tensor.py:522\u001b[0m, in \u001b[0;36mTensor.backward\u001b[0;34m(self, gradient, retain_graph, create_graph, inputs)\u001b[0m\n\u001b[1;32m 512\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m has_torch_function_unary(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m 513\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m handle_torch_function(\n\u001b[1;32m 514\u001b[0m Tensor\u001b[38;5;241m.\u001b[39mbackward,\n\u001b[1;32m 515\u001b[0m (\u001b[38;5;28mself\u001b[39m,),\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 520\u001b[0m inputs\u001b[38;5;241m=\u001b[39minputs,\n\u001b[1;32m 521\u001b[0m )\n\u001b[0;32m--> 522\u001b[0m torch\u001b[38;5;241m.\u001b[39mautograd\u001b[38;5;241m.\u001b[39mbackward(\n\u001b[1;32m 523\u001b[0m \u001b[38;5;28mself\u001b[39m, gradient, retain_graph, create_graph, inputs\u001b[38;5;241m=\u001b[39minputs\n\u001b[1;32m 524\u001b[0m )\n", + "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/autograd/__init__.py:266\u001b[0m, in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[0m\n\u001b[1;32m 261\u001b[0m retain_graph \u001b[38;5;241m=\u001b[39m create_graph\n\u001b[1;32m 263\u001b[0m \u001b[38;5;66;03m# The reason we repeat the same comment below is that\u001b[39;00m\n\u001b[1;32m 264\u001b[0m \u001b[38;5;66;03m# some Python versions print out the first line of a multi-line function\u001b[39;00m\n\u001b[1;32m 265\u001b[0m \u001b[38;5;66;03m# calls in the traceback and some print out the last line\u001b[39;00m\n\u001b[0;32m--> 266\u001b[0m Variable\u001b[38;5;241m.\u001b[39m_execution_engine\u001b[38;5;241m.\u001b[39mrun_backward( \u001b[38;5;66;03m# Calls into the C++ engine to run the backward pass\u001b[39;00m\n\u001b[1;32m 267\u001b[0m tensors,\n\u001b[1;32m 268\u001b[0m grad_tensors_,\n\u001b[1;32m 269\u001b[0m retain_graph,\n\u001b[1;32m 270\u001b[0m create_graph,\n\u001b[1;32m 271\u001b[0m inputs,\n\u001b[1;32m 272\u001b[0m allow_unreachable\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m 273\u001b[0m accumulate_grad\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m 274\u001b[0m )\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m: " ] } ], @@ -563,7 +587,7 @@ " label = label.to(dtype=torch.long, device=device)\n", " # forward\n", " optimizer.zero_grad()\n", - " output = hw_model.network(data)\n", + " output = hw_cnn(data)\n", " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", " output = output.reshape(batch_size, n_time_steps, -1)\n", " # accumulate all time-steps output for final prediction\n", @@ -574,11 +598,15 @@ " optimizer.step()\n", " \n", " # detach the neuron states and activations from current computation graph(necessary)\n", - " hw_model.network.detach_neuron_states()\n", + " hw_cnn.detach_neuron_states()\n", " \n", " # set progressing bar\n", " train_p_bar.set_description(f\"Epoch {e} - BPTT Training Loss: {round(loss.item(), 4)}\")\n", "\n", + " batch_count += 1\n", + " losses.append(loss.item())\n", + " batches.append(batch_count)\n", + "\n", " # validate\n", " correct_predictions = []\n", " with torch.no_grad():\n", @@ -588,7 +616,7 @@ " data = data.reshape(-1, 2, 34, 34).to(dtype=torch.float, device=device)\n", " label = label.to(dtype=torch.long, device=device)\n", " # forward\n", - " output = hw_model.network(data)\n", + " output = hw_cnn(data)\n", " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", " output = output.reshape(batch_size, n_time_steps, -1)\n", " # accumulate all time-steps output for final prediction\n", From 16dab381180aeea9ad8d9ed0257c92430be80659 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Fri, 26 Apr 2024 11:50:11 +0200 Subject: [PATCH 040/379] 1st trained DynapcnnNetwork: can't use nn.AvgPool2d cuz of the 'different rescale_factor' issue --- .../DynapcnnNetwork-example_1.ipynb | 679 ++++++++++++++++++ 1 file changed, 679 insertions(+) create mode 100644 tests/test_nonsequential/DynapcnnNetwork-example_1.ipynb diff --git a/tests/test_nonsequential/DynapcnnNetwork-example_1.ipynb b/tests/test_nonsequential/DynapcnnNetwork-example_1.ipynb new file mode 100644 index 00000000..a294325f --- /dev/null +++ b/tests/test_nonsequential/DynapcnnNetwork-example_1.ipynb @@ -0,0 +1,679 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "import torch.nn as nn\n", + "from sinabs.backend.dynapcnn import DynapcnnNetworkGraph\n", + "from sinabs.layers import Merge, IAFSqueeze\n", + "import matplotlib.pyplot as plt" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "torch.manual_seed(0)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "channels = 2\n", + "height = 34\n", + "width = 34\n", + "\n", + "input_shape = (channels, height, width)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Network Module\n", + "\n", + "We need to define a `nn.Module` implementing the network we want the chip to reproduce." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "class SNN(nn.Module):\n", + " def __init__(self) -> None:\n", + " super().__init__()\n", + "\n", + " self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False) # node 0\n", + " self.iaf1 = IAFSqueeze(batch_size=1) # node 1\n", + " self.pool1 = nn.AvgPool2d(3,3) # node 2\n", + " self.pool1a = nn.AvgPool2d(4,4) # node 3\n", + "\n", + " self.conv2 = nn.Conv2d(10, 10, 4, 1, bias=False)# node 4\n", + " self.iaf2 = IAFSqueeze(batch_size=1) # node 6\n", + " # self.pool2 = nn.AvgPool2d(3,3) # node 7\n", + "\n", + " self.conv3 = nn.Conv2d(10, 1, 2, 1, bias=False) # node 8\n", + " self.iaf3 = IAFSqueeze(batch_size=1) # node 9\n", + "\n", + " self.flat = nn.Flatten()\n", + "\n", + " self.fc1 = nn.Linear(49, 500, bias=False) # node 10\n", + " self.iaf4 = IAFSqueeze(batch_size=1) # node 11\n", + " \n", + " self.fc2 = nn.Linear(500, 10, bias=False) # node 12\n", + " self.iaf5 = IAFSqueeze(batch_size=1) # node 13\n", + "\n", + " self.adder = Merge()\n", + "\n", + " def forward(self, x):\n", + " \n", + " con1_out = self.conv1(x)\n", + " iaf1_out = self.iaf1(con1_out)\n", + " pool1_out = self.pool1(iaf1_out)\n", + " pool1a_out = self.pool1a(iaf1_out)\n", + "\n", + " conv2_out = self.conv2(pool1_out)\n", + " iaf2_out = self.iaf2(conv2_out)\n", + " # pool2_out = self.pool2(iaf2_out)\n", + "\n", + " conv3_out = self.conv3(self.adder(pool1a_out, iaf2_out))\n", + " iaf3_out = self.iaf3(conv3_out)\n", + "\n", + " flat_out = self.flat(iaf3_out)\n", + " \n", + " fc1_out = self.fc1(flat_out)\n", + " iaf4_out = self.iaf4(fc1_out)\n", + " fc2_out = self.fc2(iaf4_out)\n", + " iaf5_out = self.iaf5(fc2_out)\n", + "\n", + " return iaf5_out" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "snn = SNN()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The following exemplifies how each of the layers in the SNN should be grouped together to form a `DynapcnnLayer` instance. Let's check the shapes of the tensors each of the (original) layers in the model inputs and outputs by feeding a fake input through the model:" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "DynapcnnLayer 0: ... [1, 2, 34, 34]\n", + " conv1: [1, 10, 33, 33]\n", + " iaf1: [1, 10, 33, 33]\n", + " pool1: [1, 10, 11, 11]\n", + " pool1a: [1, 10, 8, 8]\n", + "\n", + "DynapcnnLayer 1: ... [1, 10, 11, 11]\n", + " conv2: [1, 10, 8, 8]\n", + " iaf2: [1, 10, 8, 8]\n", + "\n", + "DynapcnnLayer 2: ... [1, 10, 8, 8] [ Merge(pool1a, iaf2_out) ]\n", + " conv3: [1, 1, 7, 7]\n", + " iaf3: [1, 1, 7, 7]\n", + "\n", + "DynapcnnLayer 3: ... [1, 49]\n", + " fc1: [1, 500]\n", + " iaf4: [1, 500]\n", + "\n", + "DynapcnnLayer 4: ... [1, 500]\n", + " fc2: [1, 10]\n", + " iaf5: [1, 10]\n", + "\n" + ] + } + ], + "source": [ + "x = torch.randn((1, *input_shape))\n", + "\n", + "print(f'DynapcnnLayer 0: ... {list(x.shape)}')\n", + "con1_out = snn.conv1(x)\n", + "print(f' conv1: {list(con1_out.shape)}')\n", + "iaf1_out = snn.iaf1(con1_out)\n", + "print(f' iaf1: {list(iaf1_out.shape)}')\n", + "pool1_out = snn.pool1(iaf1_out)\n", + "print(f' pool1: {list(pool1_out.shape)}')\n", + "pool1a_out = snn.pool1a(iaf1_out)\n", + "print(f' pool1a: {list(pool1a_out.shape)}\\n')\n", + "\n", + "print(f'DynapcnnLayer 1: ... {list(pool1_out.shape)}')\n", + "conv2_out = snn.conv2(pool1_out)\n", + "print(f' conv2: {list(conv2_out.shape)}')\n", + "iaf2_out = snn.iaf2(conv2_out)\n", + "print(f' iaf2: {list(iaf2_out.shape)}\\n')\n", + "# pool2_out = snn.pool2(iaf2_out)\n", + "# print(f' pool2: {list(pool2_out.shape)}\\n')\n", + "\n", + "added = snn.adder(pool1a_out, iaf2_out)\n", + "\n", + "print(f'DynapcnnLayer 2: ... {list(added.shape)} [ Merge(pool1a, iaf2_out) ]')\n", + "conv3_out = snn.conv3(added)\n", + "print(f' conv3: {list(conv3_out.shape)}')\n", + "iaf3_out = snn.iaf3(conv3_out)\n", + "print(f' iaf3: {list(iaf3_out.shape)}\\n')\n", + "\n", + "flat_out = snn.flat(iaf3_out)\n", + "\n", + "print(f'DynapcnnLayer 3: ... {list(flat_out.shape)}')\n", + "fc1_out = snn.fc1(flat_out)\n", + "print(f' fc1: {list(fc1_out.shape)}')\n", + "iaf4_out = snn.iaf4(fc1_out)\n", + "print(f' iaf4: {list(iaf4_out.shape)}\\n')\n", + "\n", + "\n", + "print(f'DynapcnnLayer 4: ... {list(iaf4_out.shape)}')\n", + "fc2_out = snn.fc2(iaf4_out)\n", + "print(f' fc2: {list(fc2_out.shape)}')\n", + "iaf5_out = snn.iaf5(fc2_out)\n", + "print(f' iaf5: {list(iaf5_out.shape)}\\n')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## DynapcnnNetwork Class\n", + "\n", + "In the constructor of `DynapcnnNetworkGraph` the SNN passed as argument (defined as a `nn.Module`) will be parsed such that each layer is represented in a computational graph (using `nirtorch.extract_torch_graph`). \n", + "\n", + "The layers are the `nodes` of the graph, while their connectivity (how the outputs from a layer are sent to other layers) is represented as `edges`, represented in a `list` of `tuples`.\n", + "\n", + "Once the constructor finishes its initialization, the `hw_model.dynapcnn_layers` property is a dictionary where each entry represents the ID of a `DynapcnnLayer` instance (an `int` from `0` to `L`), with this entry containing a `DynapcnnLayer` instance where a subset of the layers in the original SNN has been incorporated into, the core such instance has been assigned to, and the list of `DynapcnnLayer` instances (their IDs) the layer targets." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "hw_model = DynapcnnNetworkGraph(\n", + " snn,\n", + " discretize=True,\n", + " input_shape=input_shape\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The `hw_model.to()` call will figure out into which core eac `DynapcnnLayer` instance will be assigned to. Once this assingment is made the instance itself is used to configure the `CNNLayerConfig` instance representing the core's configuration.\n", + "\n", + "If the cores' configuration is valid, each `DynapcnnLayer` instance and their respective destinations will be used to create a computational graph that encodes how the `forward` method of `hw_model.network` (a `nn.Module` using the `DynapcnnLayer` instances) propagates that through the network." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Network is valid\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "hw_model.to(device=\"speck2fmodule:0\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The layers comprising our `hw_model` and their respective metadata can be inspected by calling `print` on a `DynapcnnNetworkGraph` instance." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "---- DynapcnnLayer 0 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 0): Conv2d(2, 10, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + "(node 1): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "(node 2): SumPool2d(norm_type=1, kernel_size=(3, 3), stride=None, ceil_mode=False)\n", + "(node 3): SumPool2d(norm_type=1, kernel_size=(4, 4), stride=None, ceil_mode=False)\n", + "> layer destinations: [1, 2]\n", + "> assigned core: 0\n", + "\n", + "---- DynapcnnLayer 1 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 4): Conv2d(10, 10, kernel_size=(4, 4), stride=(1, 1), bias=False)\n", + "(node 6): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [2]\n", + "> assigned core: 1\n", + "\n", + "---- DynapcnnLayer 2 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 7): Conv2d(10, 1, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + "(node 8): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [3]\n", + "> assigned core: 2\n", + "\n", + "---- DynapcnnLayer 3 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 9): Conv2d(1, 500, kernel_size=(7, 7), stride=(1, 1), bias=False)\n", + "(node 10): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [4]\n", + "> assigned core: 3\n", + "\n", + "---- DynapcnnLayer 4 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 11): Conv2d(500, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + "(node 12): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: []\n", + "> assigned core: 4\n", + "\n", + "\n" + ] + } + ], + "source": [ + "print(hw_model)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Training our DynapcnnNetwork\n", + "\n", + "Preparing the data..." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "# https://synsense.gitlab.io/sinabs-dynapcnn/getting_started/notebooks/nmnist_quick_start.html\n", + "\n", + "from tonic.datasets.nmnist import NMNIST\n", + "from tonic.transforms import ToFrame\n", + "from torch.utils.data import DataLoader\n", + "from torch.nn import CrossEntropyLoss\n", + "from torch.optim import SGD\n", + "from tqdm.notebook import tqdm\n", + " \n", + "# download dataset\n", + "root_dir = \"./NMNIST\"\n", + "_ = NMNIST(save_to=root_dir, train=True)\n", + "_ = NMNIST(save_to=root_dir, train=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "type of data is: \n", + "(4686,)\n", + "time length of sample data is: 300760 micro seconds\n", + "there are 4686 events in the sample data\n", + "the label of the sample data is: 5\n" + ] + } + ], + "source": [ + "sample_data, label = NMNIST(save_to=root_dir, train=False)[0]\n", + "\n", + "print(f\"type of data is: {type(sample_data)}\")\n", + "print(sample_data.shape)\n", + "print(f\"time length of sample data is: {sample_data['t'][-1] - sample_data['t'][0]} micro seconds\")\n", + "print(f\"there are {len(sample_data)} events in the sample data\")\n", + "print(f\"the label of the sample data is: {label}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The transformed array is in shape [Time-Step, Channel, Height, Width] --> (50, 2, 34, 34)\n" + ] + } + ], + "source": [ + "n_time_steps = 50\n", + "to_raster = ToFrame(sensor_size=NMNIST.sensor_size, n_time_bins=n_time_steps)\n", + "\n", + "snn_train_dataset = NMNIST(save_to=root_dir, train=True, transform=to_raster)\n", + "snn_test_dataset = NMNIST(save_to=root_dir, train=False, transform=to_raster)\n", + "\n", + "# check the transformed data\n", + "sample_data, label = snn_train_dataset[0]\n", + "print(f\"The transformed array is in shape [Time-Step, Channel, Height, Width] --> {sample_data.shape}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Setting the training hyperparameters..." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "device: NVIDIA GeForce RTX 3070 Ti\n" + ] + } + ], + "source": [ + "if torch.cuda.is_available():\n", + " device = torch.device('cuda:0')\n", + " print('device: ', torch.cuda.get_device_name(0))\n", + "else:\n", + " device = torch.device('cpu')" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "epochs = 1\n", + "lr = 1e-4\n", + "batch_size = 64\n", + "num_workers = 4\n", + "shuffle = True" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "snn_train_dataloader = DataLoader(snn_train_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True)\n", + "snn_test_dataloader = DataLoader(snn_test_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=False)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Initializing our weights..." + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "hw_cnn = hw_model.network" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "hw_cnn.to(device)" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "hw_cnn.init_weights()" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [], + "source": [ + "optimizer = SGD(params=hw_cnn.parameters(), lr=lr)\n", + "criterion = CrossEntropyLoss()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The training loop..." + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [], + "source": [ + "losses = []\n", + "batches = []\n", + "batch_count = 0" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "0a2474dc5d01479ead00063a63022136", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/937 [00:00" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plt.plot(batches, losses)\n", + "plt.ylabel('loss')\n", + "plt.xlabel('batches')\n", + "plt.show()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "speck-rescnn", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 8441535183d7a2f6a28111ff8e5aa39713ead47c Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Fri, 26 Apr 2024 11:54:33 +0200 Subject: [PATCH 041/379] cleanup: removed old test jupyter notebooks --- ...DynapcnnNetworkGraph_from_NIRgraph_4.ipynb | 657 ------------------ .../build_from_graph_tester.ipynb | 416 ----------- .../graph_tracer_tester.ipynb | 441 ------------ .../jit_based_tracer_sinabs.ipynb | 569 --------------- tests/test_nonsequential/test_speck.ipynb | 156 ----- 5 files changed, 2239 deletions(-) delete mode 100644 tests/test_nonsequential/DynapcnnNetworkGraph_from_NIRgraph_4.ipynb delete mode 100644 tests/test_nonsequential/build_from_graph_tester.ipynb delete mode 100644 tests/test_nonsequential/graph_tracer_tester.ipynb delete mode 100644 tests/test_nonsequential/jit_based_tracer_sinabs.ipynb delete mode 100644 tests/test_nonsequential/test_speck.ipynb diff --git a/tests/test_nonsequential/DynapcnnNetworkGraph_from_NIRgraph_4.ipynb b/tests/test_nonsequential/DynapcnnNetworkGraph_from_NIRgraph_4.ipynb deleted file mode 100644 index 2be0c3c9..00000000 --- a/tests/test_nonsequential/DynapcnnNetworkGraph_from_NIRgraph_4.ipynb +++ /dev/null @@ -1,657 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import torch\n", - "import torch.nn as nn\n", - "from sinabs.from_torch import from_model\n", - "from sinabs.backend.dynapcnn import DynapcnnNetworkGraph\n", - "from sinabs.layers import Merge, IAFSqueeze\n", - "import copy" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "torch.manual_seed(0)" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "channels = 2\n", - "height = 34\n", - "width = 34\n", - "\n", - "input_shape = (channels, height, width)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Network Module\n", - "\n", - "We need to define a `nn.Module` implementing the network we want the chip to reproduce." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "class SNN(nn.Module):\n", - " def __init__(self) -> None:\n", - " super().__init__()\n", - "\n", - " self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False) # node 0\n", - " self.iaf1 = IAFSqueeze(batch_size=1) # node 1\n", - " self.pool1 = nn.AvgPool2d(3,3) # node 2\n", - " self.pool1a = nn.AvgPool2d(4,4) # node 3\n", - "\n", - " self.conv2 = nn.Conv2d(10, 10, 4, 1, bias=False)# node 4\n", - " self.iaf2 = IAFSqueeze(batch_size=1) # node 6\n", - " # self.pool2 = nn.AvgPool2d(3,3) # node 7\n", - "\n", - " self.conv3 = nn.Conv2d(10, 1, 2, 1, bias=False) # node 8\n", - " self.iaf3 = IAFSqueeze(batch_size=1) # node 9\n", - "\n", - " self.flat = nn.Flatten()\n", - "\n", - " self.fc1 = nn.Linear(49, 500, bias=False) # node 10\n", - " self.iaf4 = IAFSqueeze(batch_size=1) # node 11\n", - " \n", - " self.fc2 = nn.Linear(500, 10, bias=False) # node 12\n", - " self.iaf5 = IAFSqueeze(batch_size=1) # node 13\n", - "\n", - " self.adder = Merge()\n", - "\n", - " def forward(self, x):\n", - " \n", - " con1_out = self.conv1(x)\n", - " iaf1_out = self.iaf1(con1_out)\n", - " pool1_out = self.pool1(iaf1_out)\n", - " pool1a_out = self.pool1a(iaf1_out)\n", - "\n", - " conv2_out = self.conv2(pool1_out)\n", - " iaf2_out = self.iaf2(conv2_out)\n", - " # pool2_out = self.pool2(iaf2_out)\n", - "\n", - " conv3_out = self.conv3(self.adder(pool1a_out, iaf2_out))\n", - " iaf3_out = self.iaf3(conv3_out)\n", - "\n", - " flat_out = self.flat(iaf3_out)\n", - " \n", - " fc1_out = self.fc1(flat_out)\n", - " iaf4_out = self.iaf4(fc1_out)\n", - " fc2_out = self.fc2(iaf4_out)\n", - " iaf5_out = self.iaf5(fc2_out)\n", - "\n", - " return iaf5_out" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "snn = SNN()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The following exemplifies how each of the layers in the SNN should be grouped together to form a `DynapcnnLayer` instance. Let's check the shapes of the tensors each of the (original) layers in the model inputs and outputs by feeding a fake input through the model:" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "DynapcnnLayer 0: ... [1, 2, 34, 34]\n", - " conv1: [1, 10, 33, 33]\n", - " iaf1: [1, 10, 33, 33]\n", - " pool1: [1, 10, 11, 11]\n", - " pool1a: [1, 10, 8, 8]\n", - "\n", - "DynapcnnLayer 1: ... [1, 10, 11, 11]\n", - " conv2: [1, 10, 8, 8]\n", - " iaf2: [1, 10, 8, 8]\n", - "\n", - "DynapcnnLayer 2: ... [1, 10, 8, 8] [ Merge(pool1a, iaf2_out) ]\n", - " conv3: [1, 1, 7, 7]\n", - " iaf3: [1, 1, 7, 7]\n", - "\n", - "DynapcnnLayer 3: ... [1, 49]\n", - " fc1: [1, 500]\n", - " iaf4: [1, 500]\n", - "\n", - "DynapcnnLayer 4: ... [1, 500]\n", - " fc2: [1, 10]\n", - " iaf5: [1, 10]\n", - "\n" - ] - } - ], - "source": [ - "x = torch.randn((1, *input_shape))\n", - "\n", - "print(f'DynapcnnLayer 0: ... {list(x.shape)}')\n", - "con1_out = snn.conv1(x)\n", - "print(f' conv1: {list(con1_out.shape)}')\n", - "iaf1_out = snn.iaf1(con1_out)\n", - "print(f' iaf1: {list(iaf1_out.shape)}')\n", - "pool1_out = snn.pool1(iaf1_out)\n", - "print(f' pool1: {list(pool1_out.shape)}')\n", - "pool1a_out = snn.pool1a(iaf1_out)\n", - "print(f' pool1a: {list(pool1a_out.shape)}\\n')\n", - "\n", - "print(f'DynapcnnLayer 1: ... {list(pool1_out.shape)}')\n", - "conv2_out = snn.conv2(pool1_out)\n", - "print(f' conv2: {list(conv2_out.shape)}')\n", - "iaf2_out = snn.iaf2(conv2_out)\n", - "print(f' iaf2: {list(iaf2_out.shape)}\\n')\n", - "# pool2_out = snn.pool2(iaf2_out)\n", - "# print(f' pool2: {list(pool2_out.shape)}\\n')\n", - "\n", - "added = snn.adder(pool1a_out, iaf2_out)\n", - "\n", - "print(f'DynapcnnLayer 2: ... {list(added.shape)} [ Merge(pool1a, iaf2_out) ]')\n", - "conv3_out = snn.conv3(added)\n", - "print(f' conv3: {list(conv3_out.shape)}')\n", - "iaf3_out = snn.iaf3(conv3_out)\n", - "print(f' iaf3: {list(iaf3_out.shape)}\\n')\n", - "\n", - "flat_out = snn.flat(iaf3_out)\n", - "\n", - "print(f'DynapcnnLayer 3: ... {list(flat_out.shape)}')\n", - "fc1_out = snn.fc1(flat_out)\n", - "print(f' fc1: {list(fc1_out.shape)}')\n", - "iaf4_out = snn.iaf4(fc1_out)\n", - "print(f' iaf4: {list(iaf4_out.shape)}\\n')\n", - "\n", - "\n", - "print(f'DynapcnnLayer 4: ... {list(iaf4_out.shape)}')\n", - "fc2_out = snn.fc2(iaf4_out)\n", - "print(f' fc2: {list(fc2_out.shape)}')\n", - "iaf5_out = snn.iaf5(fc2_out)\n", - "print(f' iaf5: {list(iaf5_out.shape)}\\n')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## DynapcnnNetwork Class\n", - "\n", - "In the constructor of `DynapcnnNetworkGraph` the SNN passed as argument (defined as a `nn.Module`) will be parsed such that each layer is represented in a computational graph (using `nirtorch.extract_torch_graph`). \n", - "\n", - "The layers are the `nodes` of the graph, while their connectivity (how the outputs from a layer are sent to other layers) is represented as `edges`, represented in a `list` of `tuples`.\n", - "\n", - "Once the constructor finishes its initialization, the `hw_model.dynapcnn_layers` property is a dictionary where each entry represents the ID of a `DynapcnnLayer` instance (an `int` from `0` to `L`), with this entry containing a `DynapcnnLayer` instance where a subset of the layers in the original SNN has been incorporated into, the core such instance has been assigned to, and the list of `DynapcnnLayer` instances (their IDs) the layer targets." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "hw_model = DynapcnnNetworkGraph(\n", - " snn,\n", - " discretize=True,\n", - " input_shape=input_shape\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The `hw_model.to()` call will figure out into which core eac `DynapcnnLayer` instance will be assigned to. Once this assingment is made the instance itself is used to configure the `CNNLayerConfig` instance representing the core's configuration.\n", - "\n", - "If the cores' configuration is valid, each `DynapcnnLayer` instance and their respective destinations will be used to create a computational graph that encodes how the `forward` method of `hw_model.network` (a `nn.Module` using the `DynapcnnLayer` instances) propagates that through the network." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Network is valid\n" - ] - }, - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "hw_model.to(device=\"speck2fmodule:0\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The layers comprising our `hw_model` and their respective metadata can be inspected by calling `print` on a `DynapcnnNetworkGraph` instance." - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "---- DynapcnnLayer 0 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 0): Conv2d(2, 10, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", - "(node 1): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "(node 2): SumPool2d(norm_type=1, kernel_size=(3, 3), stride=None, ceil_mode=False)\n", - "(node 3): SumPool2d(norm_type=1, kernel_size=(4, 4), stride=None, ceil_mode=False)\n", - "> layer destinations: [1, 2]\n", - "> assigned core: 0\n", - "\n", - "---- DynapcnnLayer 1 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 4): Conv2d(10, 10, kernel_size=(4, 4), stride=(1, 1), bias=False)\n", - "(node 6): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [2]\n", - "> assigned core: 1\n", - "\n", - "---- DynapcnnLayer 2 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 7): Conv2d(10, 1, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", - "(node 8): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [3]\n", - "> assigned core: 2\n", - "\n", - "---- DynapcnnLayer 3 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 9): Conv2d(1, 500, kernel_size=(7, 7), stride=(1, 1), bias=False)\n", - "(node 10): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [4]\n", - "> assigned core: 3\n", - "\n", - "---- DynapcnnLayer 4 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 11): Conv2d(500, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - "(node 12): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: []\n", - "> assigned core: 4\n", - "\n", - "\n" - ] - } - ], - "source": [ - "print(hw_model)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Training our DynapcnnNetwork\n", - "\n", - "Preparing the data..." - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "# https://synsense.gitlab.io/sinabs-dynapcnn/getting_started/notebooks/nmnist_quick_start.html\n", - "\n", - "from tonic.datasets.nmnist import NMNIST\n", - "from tonic.transforms import ToFrame\n", - "from torch.utils.data import DataLoader\n", - "from torch.nn import CrossEntropyLoss\n", - "from torch.optim import SGD\n", - "from tqdm.notebook import tqdm\n", - " \n", - "# download dataset\n", - "root_dir = \"./NMNIST\"\n", - "_ = NMNIST(save_to=root_dir, train=True)\n", - "_ = NMNIST(save_to=root_dir, train=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "type of data is: \n", - "(4686,)\n", - "time length of sample data is: 300760 micro seconds\n", - "there are 4686 events in the sample data\n", - "the label of the sample data is: 5\n" - ] - } - ], - "source": [ - "sample_data, label = NMNIST(save_to=root_dir, train=False)[0]\n", - "\n", - "print(f\"type of data is: {type(sample_data)}\")\n", - "print(sample_data.shape)\n", - "print(f\"time length of sample data is: {sample_data['t'][-1] - sample_data['t'][0]} micro seconds\")\n", - "print(f\"there are {len(sample_data)} events in the sample data\")\n", - "print(f\"the label of the sample data is: {label}\")" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The transformed array is in shape [Time-Step, Channel, Height, Width] --> (50, 2, 34, 34)\n" - ] - } - ], - "source": [ - "n_time_steps = 50\n", - "to_raster = ToFrame(sensor_size=NMNIST.sensor_size, n_time_bins=n_time_steps)\n", - "\n", - "snn_train_dataset = NMNIST(save_to=root_dir, train=True, transform=to_raster)\n", - "snn_test_dataset = NMNIST(save_to=root_dir, train=False, transform=to_raster)\n", - "\n", - "# check the transformed data\n", - "sample_data, label = snn_train_dataset[0]\n", - "print(f\"The transformed array is in shape [Time-Step, Channel, Height, Width] --> {sample_data.shape}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Setting the training hyperparameters..." - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "device: NVIDIA GeForce RTX 3070 Ti\n" - ] - } - ], - "source": [ - "if torch.cuda.is_available():\n", - " device = torch.device('cuda:0')\n", - " print('device: ', torch.cuda.get_device_name(0))\n", - "else:\n", - " device = torch.device('cpu')" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [], - "source": [ - "epochs = 1\n", - "lr = 1e-4\n", - "batch_size = 64\n", - "num_workers = 4\n", - "shuffle = True" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [], - "source": [ - "snn_train_dataloader = DataLoader(snn_train_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True)\n", - "snn_test_dataloader = DataLoader(snn_test_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=False)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Initializing our weights..." - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [], - "source": [ - "hw_cnn = hw_model.network" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [], - "source": [ - "hw_cnn.to(device)" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [], - "source": [ - "hw_cnn.init_weights()" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [], - "source": [ - "optimizer = SGD(params=hw_cnn.parameters(), lr=lr)\n", - "criterion = CrossEntropyLoss()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The training loop..." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "losses = []\n", - "batches = []\n", - "batch_count = 0" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "d73e71745e264aceb8dc7cf7320d6d6a", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - " 0%| | 0/937 [00:00 18\u001b[0m loss\u001b[38;5;241m.\u001b[39mbackward()\n\u001b[1;32m 19\u001b[0m optimizer\u001b[38;5;241m.\u001b[39mstep()\n\u001b[1;32m 21\u001b[0m \u001b[38;5;66;03m# detach the neuron states and activations from current computation graph(necessary)\u001b[39;00m\n", - "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/_tensor.py:522\u001b[0m, in \u001b[0;36mTensor.backward\u001b[0;34m(self, gradient, retain_graph, create_graph, inputs)\u001b[0m\n\u001b[1;32m 512\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m has_torch_function_unary(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m 513\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m handle_torch_function(\n\u001b[1;32m 514\u001b[0m Tensor\u001b[38;5;241m.\u001b[39mbackward,\n\u001b[1;32m 515\u001b[0m (\u001b[38;5;28mself\u001b[39m,),\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 520\u001b[0m inputs\u001b[38;5;241m=\u001b[39minputs,\n\u001b[1;32m 521\u001b[0m )\n\u001b[0;32m--> 522\u001b[0m torch\u001b[38;5;241m.\u001b[39mautograd\u001b[38;5;241m.\u001b[39mbackward(\n\u001b[1;32m 523\u001b[0m \u001b[38;5;28mself\u001b[39m, gradient, retain_graph, create_graph, inputs\u001b[38;5;241m=\u001b[39minputs\n\u001b[1;32m 524\u001b[0m )\n", - "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/autograd/__init__.py:266\u001b[0m, in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[0m\n\u001b[1;32m 261\u001b[0m retain_graph \u001b[38;5;241m=\u001b[39m create_graph\n\u001b[1;32m 263\u001b[0m \u001b[38;5;66;03m# The reason we repeat the same comment below is that\u001b[39;00m\n\u001b[1;32m 264\u001b[0m \u001b[38;5;66;03m# some Python versions print out the first line of a multi-line function\u001b[39;00m\n\u001b[1;32m 265\u001b[0m \u001b[38;5;66;03m# calls in the traceback and some print out the last line\u001b[39;00m\n\u001b[0;32m--> 266\u001b[0m Variable\u001b[38;5;241m.\u001b[39m_execution_engine\u001b[38;5;241m.\u001b[39mrun_backward( \u001b[38;5;66;03m# Calls into the C++ engine to run the backward pass\u001b[39;00m\n\u001b[1;32m 267\u001b[0m tensors,\n\u001b[1;32m 268\u001b[0m grad_tensors_,\n\u001b[1;32m 269\u001b[0m retain_graph,\n\u001b[1;32m 270\u001b[0m create_graph,\n\u001b[1;32m 271\u001b[0m inputs,\n\u001b[1;32m 272\u001b[0m allow_unreachable\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m 273\u001b[0m accumulate_grad\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m 274\u001b[0m )\n", - "\u001b[0;31mKeyboardInterrupt\u001b[0m: " - ] - } - ], - "source": [ - "for e in range(epochs):\n", - "\n", - " # train\n", - " train_p_bar = tqdm(snn_train_dataloader)\n", - " for data, label in train_p_bar:\n", - " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", - " data = data.reshape(-1, 2, 34, 34).to(dtype=torch.float, device=device)\n", - " label = label.to(dtype=torch.long, device=device)\n", - " # forward\n", - " optimizer.zero_grad()\n", - " output = hw_cnn(data)\n", - " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", - " output = output.reshape(batch_size, n_time_steps, -1)\n", - " # accumulate all time-steps output for final prediction\n", - " output = output.sum(dim=1)\n", - " loss = criterion(output, label)\n", - " # backward\n", - " loss.backward()\n", - " optimizer.step()\n", - " \n", - " # detach the neuron states and activations from current computation graph(necessary)\n", - " hw_cnn.detach_neuron_states()\n", - " \n", - " # set progressing bar\n", - " train_p_bar.set_description(f\"Epoch {e} - BPTT Training Loss: {round(loss.item(), 4)}\")\n", - "\n", - " batch_count += 1\n", - " losses.append(loss.item())\n", - " batches.append(batch_count)\n", - "\n", - " # validate\n", - " correct_predictions = []\n", - " with torch.no_grad():\n", - " test_p_bar = tqdm(snn_test_dataloader)\n", - " for data, label in test_p_bar:\n", - " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", - " data = data.reshape(-1, 2, 34, 34).to(dtype=torch.float, device=device)\n", - " label = label.to(dtype=torch.long, device=device)\n", - " # forward\n", - " output = hw_cnn(data)\n", - " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", - " output = output.reshape(batch_size, n_time_steps, -1)\n", - " # accumulate all time-steps output for final prediction\n", - " output = output.sum(dim=1)\n", - " # calculate accuracy\n", - " pred = output.argmax(dim=1, keepdim=True)\n", - " # compute the total correct predictions\n", - " correct_predictions.append(pred.eq(label.view_as(pred)))\n", - " # set progressing bar\n", - " test_p_bar.set_description(f\"Epoch {e} - BPTT Testing Model...\")\n", - " \n", - " correct_predictions = torch.cat(correct_predictions)\n", - " print(f\"Epoch {e} - BPTT accuracy: {correct_predictions.sum().item()/(len(correct_predictions))*100}%\")" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "speck-rescnn", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/tests/test_nonsequential/build_from_graph_tester.ipynb b/tests/test_nonsequential/build_from_graph_tester.ipynb deleted file mode 100644 index 11d2682b..00000000 --- a/tests/test_nonsequential/build_from_graph_tester.ipynb +++ /dev/null @@ -1,416 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import torch\n", - "import torch.nn as nn\n", - "import networkx as nx\n", - "import matplotlib.pyplot as plt\n", - "from sinabs.from_torch import from_model\n", - "from sinabs.backend.dynapcnn import DynapcnnNetwork, DynapcnnNetworkGraph\n", - "import sinabs as snb" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "torch.manual_seed(0)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Dummy Initialization" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "channels = 1\n", - "height = 28\n", - "width = 28\n", - "\n", - "input_shape = (channels, height, width)" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "ann = nn.Sequential(\n", - " nn.Conv2d(1, 20, 5, 1, bias=False),\n", - " nn.ReLU(),\n", - " nn.AvgPool2d(2,2),\n", - " nn.Conv2d(20, 32, 5, 1, bias=False),\n", - " nn.ReLU(),\n", - " nn.AvgPool2d(2,2),\n", - " nn.Conv2d(32, 128, 3, 1, bias=False),\n", - " nn.ReLU(),\n", - " nn.AvgPool2d(2,2),\n", - " nn.Flatten(),\n", - " nn.Linear(128, 500, bias=False),\n", - " nn.ReLU(),\n", - " nn.Flatten(),\n", - " nn.Linear(500, 10, bias=False),\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "sinabs_model = from_model(ann, add_spiking_output=True, batch_size=1)" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - " [ ENTERED build_from_graph() ]\n", - "\n", - "node to DynapcnnLayer mapping:\n", - "DynapcnnLayer index: 0\n", - " [0 ]\n", - " [1 ]\n", - " [2 ]\n", - "DynapcnnLayer index: 1\n", - " [3 ]\n", - " [4 ]\n", - " [5 ]\n", - "DynapcnnLayer index: 2\n", - " [6 ]\n", - " [7 ]\n", - " [8 ]\n", - "DynapcnnLayer index: 3\n", - " [9 ]\n", - " [10 ]\n", - "DynapcnnLayer index: 4\n", - " [11 ]\n", - " [12 ]\n", - "\n", - "DynapcnnLayer to DynapcnnLayer mapping:\n", - "DynapcnnLayer 0 destinations: [1]\n", - "DynapcnnLayer 1 destinations: [2]\n", - "DynapcnnLayer 2 destinations: [3]\n", - "DynapcnnLayer 3 destinations: [4]\n", - "DynapcnnLayer 4 destinations: []\n" - ] - } - ], - "source": [ - "hw_model = DynapcnnNetworkGraph(\n", - " sinabs_model,\n", - " discretize=True,\n", - " input_shape=input_shape\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Valide Non-sequential Edges" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Test 1" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "dummy_layer_list = [\n", - " torch.nn.Conv2d(1, 20, kernel_size=(5, 5), stride=(1, 1), bias=False),\n", - " snb.layers.iaf.IAFSqueeze(batch_size=1),\n", - " torch.nn.AvgPool2d(kernel_size=2, stride=2, padding=0),\n", - " torch.nn.Conv2d(1, 20, kernel_size=(5, 5), stride=(1, 1), bias=False),\n", - " snb.layers.iaf.IAFSqueeze(batch_size=1),\n", - " torch.nn.AvgPool2d(kernel_size=2, stride=2, padding=0),\n", - " torch.nn.Conv2d(1, 20, kernel_size=(5, 5), stride=(1, 1), bias=False),\n", - " snb.layers.iaf.IAFSqueeze(batch_size=1),\n", - " torch.nn.AvgPool2d(kernel_size=2, stride=2, padding=0),\n", - " torch.nn.Linear(in_features=128, out_features=500, bias=False),\n", - " snb.layers.iaf.IAFSqueeze(batch_size=1),\n", - " torch.nn.Linear(in_features=128, out_features=500, bias=False),\n", - " snb.layers.iaf.IAFSqueeze(batch_size=1),\n", - " torch.nn.Linear(in_features=128, out_features=500, bias=False),\n", - " snb.layers.iaf.IAFSqueeze(batch_size=1),]" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "dummy_edges = [\n", - " (0, 1),\n", - " (1, 2),\n", - " (2, 3),\n", - " (3, 4),\n", - " (4, 5),\n", - " (5, 6),\n", - " (6, 7),\n", - " (7, 8),\n", - " (8, 9),\n", - " (9, 10),\n", - " (10, 11),\n", - " (10, 13),\n", - " (11, 12),\n", - " (12, 13),\n", - " (13, 14),]" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "build_from_graph = DynapcnnNetworkGraph.build_from_graph_()" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - " [ ENTERED build_from_graph() ]\n", - "\n", - "node to DynapcnnLayer mapping:\n", - "DynapcnnLayer index: 0\n", - " [0 ]\n", - " [1 ]\n", - " [2 ]\n", - "DynapcnnLayer index: 1\n", - " [3 ]\n", - " [4 ]\n", - " [5 ]\n", - "DynapcnnLayer index: 2\n", - " [6 ]\n", - " [7 ]\n", - " [8 ]\n", - "DynapcnnLayer index: 3\n", - " [9 ]\n", - " [10 ]\n", - "DynapcnnLayer index: 4\n", - " [13 ]\n", - " [14 ]\n", - "DynapcnnLayer index: 5\n", - " [11 ]\n", - " [12 ]\n", - "\n", - "DynapcnnLayer to DynapcnnLayer mapping:\n", - "DynapcnnLayer 0 destinations: [1]\n", - "DynapcnnLayer 1 destinations: [2]\n", - "DynapcnnLayer 2 destinations: [3]\n", - "DynapcnnLayer 3 destinations: [5, 4]\n", - "DynapcnnLayer 5 destinations: [4]\n", - "DynapcnnLayer 4 destinations: []\n" - ] - }, - { - "data": { - "text/plain": [ - "(None, None)" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "build_from_graph(dummy_layer_list, None, dummy_edges)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Test 2" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "dummy_layer_list = [\n", - " torch.nn.Conv2d(1, 20, kernel_size=(5, 5), stride=(1, 1), bias=False),\n", - " snb.layers.iaf.IAFSqueeze(batch_size=1),\n", - " torch.nn.AvgPool2d(kernel_size=2, stride=2, padding=0),\n", - " torch.nn.Conv2d(1, 20, kernel_size=(5, 5), stride=(1, 1), bias=False),\n", - " snb.layers.iaf.IAFSqueeze(batch_size=1),\n", - " torch.nn.AvgPool2d(kernel_size=2, stride=2, padding=0),\n", - " torch.nn.Conv2d(1, 20, kernel_size=(5, 5), stride=(1, 1), bias=False),\n", - " snb.layers.iaf.IAFSqueeze(batch_size=1),\n", - " torch.nn.AvgPool2d(kernel_size=2, stride=2, padding=0),\n", - " torch.nn.Linear(in_features=128, out_features=500, bias=False),\n", - " snb.layers.iaf.IAFSqueeze(batch_size=1),\n", - " torch.nn.Linear(in_features=128, out_features=500, bias=False),\n", - " snb.layers.iaf.IAFSqueeze(batch_size=1),\n", - " torch.nn.Linear(in_features=128, out_features=500, bias=False),\n", - " snb.layers.iaf.IAFSqueeze(batch_size=1),]" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "dummy_edges = [\n", - " (0, 1),\n", - " (1, 2),\n", - " (1, 6),\n", - " (2, 3),\n", - " (3, 4),\n", - " (4, 5),\n", - " (5, 6),\n", - " (6, 7),\n", - " (7, 8),\n", - " (8, 9),\n", - " (9, 10),\n", - " (10, 11),\n", - " (10, 13),\n", - " (11, 12),\n", - " (12, 13),\n", - " (13, 14),]" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "build_from_graph = DynapcnnNetworkGraph.build_from_graph_()" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - " [ ENTERED build_from_graph() ]\n", - "\n", - "node to DynapcnnLayer mapping:\n", - "DynapcnnLayer index: 0\n", - " [0 ]\n", - " [1 ]\n", - " [2 ]\n", - "DynapcnnLayer index: 1\n", - " [3 ]\n", - " [4 ]\n", - " [5 ]\n", - "DynapcnnLayer index: 2\n", - " [6 ]\n", - " [7 ]\n", - " [8 ]\n", - "DynapcnnLayer index: 3\n", - " [9 ]\n", - " [10 ]\n", - "DynapcnnLayer index: 4\n", - " [13 ]\n", - " [14 ]\n", - "DynapcnnLayer index: 5\n", - " [11 ]\n", - " [12 ]\n", - "\n", - "DynapcnnLayer to DynapcnnLayer mapping:\n", - "DynapcnnLayer 0 destinations: [2, 1]\n", - "DynapcnnLayer 1 destinations: [2]\n", - "DynapcnnLayer 2 destinations: [3]\n", - "DynapcnnLayer 3 destinations: [5, 4]\n", - "DynapcnnLayer 5 destinations: [4]\n", - "DynapcnnLayer 4 destinations: []\n" - ] - }, - { - "data": { - "text/plain": [ - "(None, None)" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "build_from_graph(dummy_layer_list, None, dummy_edges)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "speck-rescnn", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/tests/test_nonsequential/graph_tracer_tester.ipynb b/tests/test_nonsequential/graph_tracer_tester.ipynb deleted file mode 100644 index c3259683..00000000 --- a/tests/test_nonsequential/graph_tracer_tester.ipynb +++ /dev/null @@ -1,441 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import torch\n", - "import torch.nn as nn\n", - "from typing import Union\n", - "import re, copy\n", - "import numpy as np\n", - "import networkx as nx\n", - "import matplotlib.pyplot as plt" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "torch.manual_seed(0)" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "channels = 1\n", - "height = 28\n", - "width = 28\n", - "\n", - "input_dummy = torch.randn((1, channels, height, width))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Graph Tracer" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "class GraphTracer():\n", - " def __init__(self, model: Union[nn.Sequential, nn.Module], dummy_input: np.array) -> None:\n", - " \"\"\" .\"\"\"\n", - "\n", - " trace = torch.jit.trace(model, dummy_input)\n", - " _ = trace(dummy_input)\n", - " __ = copy.deepcopy(trace)\n", - "\n", - " self.graph = __.graph\n", - "\n", - " self.modules_map, self.name_2_indx_map = self.get_named_modules(model)\n", - " self.forward_edges = self.get_foward_edges()\n", - " self.ATens = self.get_ATen_operations()\n", - " self.edges_list = self.get_graph_edges()\n", - "\n", - " def from_name_2_indx(self, name):\n", - " if name in self.name_2_indx_map:\n", - " return self.name_2_indx_map[name]\n", - " else:\n", - " last_indx = None\n", - " for _name, indx in self.name_2_indx_map.items():\n", - " last_indx = indx\n", - " self.name_2_indx_map[name] = last_indx+1\n", - " return self.name_2_indx_map[name]\n", - "\n", - " def get_named_modules(self, module: nn.Module):\n", - " \"\"\" .\"\"\"\n", - " modules_map = {}\n", - " name_2_indx_map = {}\n", - " indx = 0\n", - " for name, mod in module.named_modules():\n", - " if name:\n", - " modules_map[indx] = mod\n", - " name_2_indx_map[name] = indx\n", - " indx += 1\n", - " return modules_map, name_2_indx_map\n", - " \n", - " def get_foward_edges(self):\n", - " \"\"\" .\"\"\"\n", - " forward_edges = {}\n", - " for node in self.graph.nodes():\n", - " node = str(node)\n", - " regex = re.compile(r'%(.*?) :.*prim::CallMethod\\[name=\"forward\"\\]\\(%(.*?), %(.*?)\\)')\n", - " match = regex.search(node)\n", - " if match:\n", - " source = match.group(3).replace('_', '')\n", - " target = match.group(2).replace('_', '')\n", - " result = match.group(1).replace('_', '')\n", - " forward_edges[self.from_name_2_indx(result)] = (self.from_name_2_indx(source), self.from_name_2_indx(target))\n", - " \n", - " return forward_edges\n", - "\n", - " def get_graph_edges(self):\n", - " \"\"\" .\"\"\"\n", - " edges = []\n", - " last_result = None\n", - "\n", - " for result_node, forward_edge in self.forward_edges.items():\n", - " src = forward_edge[0]\n", - " trg = forward_edge[1]\n", - "\n", - " if not last_result:\n", - " last_result = result_node\n", - " edges.append(('input', trg))\n", - " elif src == last_result:\n", - " edges.append((edges[-1][1], trg))\n", - " last_result = result_node\n", - " else:\n", - " scr1, scr2 = self.get_ATen_operands(src)\n", - " edges.append((scr1, trg))\n", - " edges.append((scr2, trg))\n", - " last_result = result_node\n", - " \n", - " edges.append((edges[-1][1], 'output'))\n", - "\n", - " return edges[1:-1]\n", - " \n", - " def get_ATen_operands(self, node):\n", - " \"\"\" .\"\"\"\n", - " if node in self.ATens:\n", - " src1 = self.ATens[node]['args'][1]\n", - " src2 = self.ATens[node]['args'][0]\n", - " return self.forward_edges[src1][1], self.forward_edges[src2][1]\n", - " else:\n", - " # throw error\n", - " return None, None\n", - " \n", - " def get_ATen_operations(self):\n", - " \"\"\" ATen is PyTorch's tensor library backend, which provides a set of operations that operate on \n", - " tensors directly. These include arithmetic operations (add, mul, etc.), mathematical \n", - " functions (sin, cos, etc.), and tensor manipulation operations (view, reshape, etc.).\"\"\"\n", - " ATens = {}\n", - " for node in self.graph.nodes():\n", - " node = str(node)\n", - " regex = re.compile(r'%(.*?) :.*aten::(.*?)\\(%(.*?), %(.*?), %(.*?)\\)')\n", - "\n", - " match = regex.search(node)\n", - "\n", - " if match:\n", - " result_node = match.group(1)\n", - " operation = match.group(2)\n", - " operator1 = self.from_name_2_indx(match.group(3))\n", - " operator2 = self.from_name_2_indx(match.group(4))\n", - " const_operator = match.group(5)\n", - " ATens[result_node] = {'op': operation, 'args': (operator1, operator2, const_operator)}\n", - " return ATens\n", - " \n", - " def remove_ignored_nodes(self, default_ignored_nodes):\n", - " \"\"\" Recreates the edges list based on layers that 'DynapcnnNetwork' will ignore. This\n", - " is done by setting the source (target) node of an edge where the source (target) node\n", - " will be dropped as the node that originally targeted this node to be dropped.\n", - " \"\"\"\n", - " edges = copy.deepcopy(self.edges_list)\n", - " parsed_edges = []\n", - " removed_nodes = []\n", - "\n", - " # removing ignored nodes from edges.\n", - " for edge_idx in range(len(edges)):\n", - " _src = edges[edge_idx][0]\n", - " _trg = edges[edge_idx][1]\n", - "\n", - " if isinstance(self.modules_map[_src], default_ignored_nodes):\n", - " removed_nodes.append(_src)\n", - " # all edges where node '_src' is target change it to node '_trg' as their target.\n", - " for edge in edges:\n", - " if edge[1] == _src:\n", - " new_edge = (edge[0], _trg)\n", - " elif isinstance(self.modules_map[_trg], default_ignored_nodes):\n", - " removed_nodes.append(_trg)\n", - " # all edges where node '_trg' is source change it to node '_src' as their source.\n", - " for edge in edges:\n", - " if edge[0] == _trg:\n", - " new_edge = (_src, edge[1])\n", - " else:\n", - " new_edge = (_src, _trg)\n", - " \n", - " if new_edge not in parsed_edges:\n", - " parsed_edges.append(new_edge)\n", - "\n", - " removed_nodes = list(set(removed_nodes))\n", - "\n", - " # remapping nodes indexes.\n", - " remapped_nodes = {}\n", - " for node_indx, __ in self.modules_map.items():\n", - " _ = [x for x in removed_nodes if node_indx > x]\n", - " remapped_nodes[node_indx] = node_indx - len(_)\n", - " \n", - " for x in removed_nodes:\n", - " del remapped_nodes[x]\n", - "\n", - " # remapping nodes names in parsed edges.\n", - " remapped_edges = []\n", - " for edge in parsed_edges:\n", - " remapped_edges.append((remapped_nodes[edge[0]], remapped_nodes[edge[1]]))\n", - "\n", - " return remapped_edges\n", - " \n", - " @staticmethod\n", - " def plot_graph(edges_list):\n", - " \"\"\" .\"\"\"\n", - " G = nx.DiGraph(edges_list)\n", - " layout = nx.spring_layout(G)\n", - " nx.draw(G, pos = layout, with_labels=True, node_size=800)\n", - " plt.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Tracing 2" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "class ANN(nn.Module):\n", - " def __init__(self) -> None:\n", - " super().__init__()\n", - "\n", - " self.con1 = nn.Conv2d(1, 20, 5, 1, bias=False)\n", - " self.rel1 = nn.ReLU()\n", - " self.pool1 = nn.AvgPool2d(2,2)\n", - "\n", - " self.conv2 = nn.Conv2d(20, 32, 5, 1, bias=False)\n", - " self.rel2 = nn.ReLU()\n", - " self.pool2 = nn.AvgPool2d(2,2)\n", - "\n", - " self.conv3 = nn.Conv2d(32, 128, 3, 1, bias=False)\n", - " self.rel3 = nn.ReLU()\n", - " self.pool3 = nn.AvgPool2d(2,2)\n", - "\n", - " self.flat = nn.Flatten()\n", - "\n", - " self.fc1 = nn.Linear(128, 500, bias=False)\n", - " self.rel4 = nn.ReLU()\n", - " self.fc2 = nn.Linear(500, 10, bias=False)\n", - "\n", - " self.residual_projection = nn.Conv2d(20, 32, 1, 6, bias=False) # from self.con1 to self.con3.\n", - " self.residual_projection.weight.requires_grad = False # no training of parameters.\n", - " self.residual_projection.weight.data.fill_(1) # compute the identity.\n", - "\n", - " def forward(self, x):\n", - " \n", - " con1_out = self.con1(x)\n", - " rel1_out = self.rel1(con1_out)\n", - " pool1_out = self.pool1(rel1_out)\n", - "\n", - " residual = self.residual_projection(rel1_out)\n", - "\n", - " conv2_out = self.conv2(pool1_out)\n", - " rel2_out = self.rel2(conv2_out)\n", - " pool2_out = self.pool2(rel2_out)\n", - "\n", - " conv3_out = self.conv3(pool2_out + residual)\n", - " rel3_out = self.rel3(conv3_out)\n", - " pool3_out = self.pool3(rel3_out)\n", - "\n", - " flat_out = self.flat(pool3_out)\n", - " \n", - " fc1_out = self.fc1(flat_out)\n", - " rel4_out = self.rel4(fc1_out)\n", - " fc2_out = self.fc2(rel4_out)\n", - "\n", - " return fc2_out\n", - "\n", - "ann2 = ANN()" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "con1_out = ann2.rel1(ann2.con1(input_dummy))\n", - "pool1 = ann2.pool1(con1_out)" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "torch.Size([1, 32, 4, 4])\n" - ] - } - ], - "source": [ - "\n", - "con2_out = ann2.rel2(ann2.conv2(pool1))\n", - "pool2 = ann2.pool2(con2_out)\n", - "\n", - "print(pool2.shape)" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "torch.Size([1, 32, 4, 4])\n" - ] - } - ], - "source": [ - "residual = ann2.residual_projection(con1_out)\n", - "\n", - "print(residual.shape)" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "gtracer2 = GraphTracer(ann2, input_dummy)" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "0 Conv2d(1, 20, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", - "1 ReLU()\n", - "2 AvgPool2d(kernel_size=2, stride=2, padding=0)\n", - "3 Conv2d(20, 32, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", - "4 ReLU()\n", - "5 AvgPool2d(kernel_size=2, stride=2, padding=0)\n", - "6 Conv2d(32, 128, kernel_size=(3, 3), stride=(1, 1), bias=False)\n", - "7 ReLU()\n", - "8 AvgPool2d(kernel_size=2, stride=2, padding=0)\n", - "9 Flatten(start_dim=1, end_dim=-1)\n", - "10 Linear(in_features=128, out_features=500, bias=False)\n", - "11 ReLU()\n", - "12 Linear(in_features=500, out_features=10, bias=False)\n", - "13 Conv2d(20, 32, kernel_size=(1, 1), stride=(6, 6), bias=False)\n" - ] - } - ], - "source": [ - "for name, mod in gtracer2.modules_map.items():\n", - " print(name, mod)" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(0, 1)\n", - "(1, 2)\n", - "(None, 18)\n", - "(None, 18)\n", - "(None, 3)\n", - "(None, 3)\n", - "(3, 4)\n", - "(4, 5)\n", - "(None, 6)\n", - "(None, 6)\n", - "(6, 7)\n", - "(7, 8)\n", - "(8, 9)\n", - "(9, 10)\n", - "(10, 11)\n", - "(11, 12)\n" - ] - } - ], - "source": [ - "for edge in gtracer2.edges_list:\n", - " print(edge)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "speck-rescnn", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/tests/test_nonsequential/jit_based_tracer_sinabs.ipynb b/tests/test_nonsequential/jit_based_tracer_sinabs.ipynb deleted file mode 100644 index b6c845e2..00000000 --- a/tests/test_nonsequential/jit_based_tracer_sinabs.ipynb +++ /dev/null @@ -1,569 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import torch\n", - "import torch.nn as nn\n", - "import networkx as nx\n", - "import matplotlib.pyplot as plt\n", - "from sinabs.from_torch import from_model\n", - "from sinabs.backend.dynapcnn import DynapcnnNetwork, DynapcnnNetworkGraph\n", - "import sinabs as snb" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "torch.manual_seed(0)" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "channels = 1\n", - "height = 28\n", - "width = 28\n", - "\n", - "input_shape = (channels, height, width)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Network Module (pure Pytorch)" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "ann = nn.Sequential(\n", - " nn.Conv2d(1, 20, 5, 1, bias=False),\n", - " nn.ReLU(),\n", - " nn.AvgPool2d(2,2),\n", - " nn.Conv2d(20, 32, 5, 1, bias=False),\n", - " nn.ReLU(),\n", - " nn.AvgPool2d(2,2),\n", - " nn.Conv2d(32, 128, 3, 1, bias=False),\n", - " nn.ReLU(),\n", - " nn.AvgPool2d(2,2),\n", - " nn.Flatten(),\n", - " nn.Linear(128, 500, bias=False),\n", - " nn.ReLU(),\n", - " nn.Flatten(),\n", - " nn.Linear(500, 10, bias=False),\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "0 Conv2d(1, 20, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", - "1 ReLU()\n", - "2 AvgPool2d(kernel_size=2, stride=2, padding=0)\n", - "3 Conv2d(20, 32, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", - "4 ReLU()\n", - "5 AvgPool2d(kernel_size=2, stride=2, padding=0)\n", - "6 Conv2d(32, 128, kernel_size=(3, 3), stride=(1, 1), bias=False)\n", - "7 ReLU()\n", - "8 AvgPool2d(kernel_size=2, stride=2, padding=0)\n", - "9 Flatten(start_dim=1, end_dim=-1)\n", - "10 Linear(in_features=128, out_features=500, bias=False)\n", - "11 ReLU()\n", - "12 Flatten(start_dim=1, end_dim=-1)\n", - "13 Linear(in_features=500, out_features=10, bias=False)\n" - ] - } - ], - "source": [ - "count = 0\n", - "for l in ann:\n", - " print(count, l)\n", - " count += 1" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Sinabs Model" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "sinabs_model = from_model(ann, add_spiking_output=True, batch_size=1)" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "0 Conv2d(1, 20, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", - "1 IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-1.), batch_size=1, num_timesteps=-1)\n", - "2 AvgPool2d(kernel_size=2, stride=2, padding=0)\n", - "3 Conv2d(20, 32, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", - "4 IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-1.), batch_size=1, num_timesteps=-1)\n", - "5 AvgPool2d(kernel_size=2, stride=2, padding=0)\n", - "6 Conv2d(32, 128, kernel_size=(3, 3), stride=(1, 1), bias=False)\n", - "7 IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-1.), batch_size=1, num_timesteps=-1)\n", - "8 AvgPool2d(kernel_size=2, stride=2, padding=0)\n", - "9 Flatten(start_dim=1, end_dim=-1)\n", - "10 Linear(in_features=128, out_features=500, bias=False)\n", - "11 IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-1.), batch_size=1, num_timesteps=-1)\n", - "12 Flatten(start_dim=1, end_dim=-1)\n", - "13 Linear(in_features=500, out_features=10, bias=False)\n", - "14 IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-1.), batch_size=1, num_timesteps=-1)\n" - ] - } - ], - "source": [ - "count = 0\n", - "for l in sinabs_model.spiking_model:\n", - " print(count, l)\n", - " count += 1" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "0 Conv2d(1, 20, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", - "1 ReLU()\n", - "2 AvgPool2d(kernel_size=2, stride=2, padding=0)\n", - "3 Conv2d(20, 32, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", - "4 ReLU()\n", - "5 AvgPool2d(kernel_size=2, stride=2, padding=0)\n", - "6 Conv2d(32, 128, kernel_size=(3, 3), stride=(1, 1), bias=False)\n", - "7 ReLU()\n", - "8 AvgPool2d(kernel_size=2, stride=2, padding=0)\n", - "9 Flatten(start_dim=1, end_dim=-1)\n", - "10 Linear(in_features=128, out_features=500, bias=False)\n", - "11 ReLU()\n", - "12 Flatten(start_dim=1, end_dim=-1)\n", - "13 Linear(in_features=500, out_features=10, bias=False)\n" - ] - } - ], - "source": [ - "count = 0\n", - "for l in sinabs_model.analog_model:\n", - " print(count, l)\n", - " count += 1" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## DynapCNN Model" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "--------------------------------------------------------\n", - "0 Conv2d(1, 20, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", - "1 IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-1.), batch_size=1, num_timesteps=-1)\n", - "2 AvgPool2d(kernel_size=2, stride=2, padding=0)\n", - "3 Conv2d(20, 32, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", - "4 IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-1.), batch_size=1, num_timesteps=-1)\n", - "5 AvgPool2d(kernel_size=2, stride=2, padding=0)\n", - "6 Conv2d(32, 128, kernel_size=(3, 3), stride=(1, 1), bias=False)\n", - "7 IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-1.), batch_size=1, num_timesteps=-1)\n", - "8 AvgPool2d(kernel_size=2, stride=2, padding=0)\n", - "9 Linear(in_features=128, out_features=500, bias=False)\n", - "10 IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-1.), batch_size=1, num_timesteps=-1)\n", - "11 Linear(in_features=500, out_features=10, bias=False)\n", - "12 IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-1.), batch_size=1, num_timesteps=-1)\n", - "--------------------------------------------------------\n", - "True\n" - ] - } - ], - "source": [ - "hw_model_old = DynapcnnNetwork(\n", - " sinabs_model.spiking_model,\n", - " discretize=True,\n", - " input_shape=input_shape\n", - ")\n", - "\n", - "print(isinstance(sinabs_model.spiking_model, nn.Sequential))" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Network is valid\n" - ] - }, - { - "data": { - "text/plain": [ - "DynapcnnNetwork(\n", - " (sequence): Sequential(\n", - " (0): DynapcnnLayer(\n", - " (conv_layer): Conv2d(1, 20, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", - " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", - " tensor(635.), min_v_mem=Parameter containing:\n", - " tensor(-635.), batch_size=1, num_timesteps=-1)\n", - " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", - " )\n", - " (1): DynapcnnLayer(\n", - " (conv_layer): Conv2d(20, 32, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", - " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", - " tensor(11361.), min_v_mem=Parameter containing:\n", - " tensor(-11361.), batch_size=1, num_timesteps=-1)\n", - " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", - " )\n", - " (2): DynapcnnLayer(\n", - " (conv_layer): Conv2d(32, 128, kernel_size=(3, 3), stride=(1, 1), bias=False)\n", - " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", - " tensor(8621.), min_v_mem=Parameter containing:\n", - " tensor(-8621.), batch_size=1, num_timesteps=-1)\n", - " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", - " )\n", - " (3): DynapcnnLayer(\n", - " (conv_layer): Conv2d(128, 500, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", - " tensor(5747.), min_v_mem=Parameter containing:\n", - " tensor(-5747.), batch_size=1, num_timesteps=-1)\n", - " )\n", - " (4): DynapcnnLayer(\n", - " (conv_layer): Conv2d(500, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", - " tensor(2841.), min_v_mem=Parameter containing:\n", - " tensor(-2841.), batch_size=1, num_timesteps=-1)\n", - " )\n", - " )\n", - ")" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "hw_model_old.to(device=\"speck2edevkit:0\")" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "--------------------------------------------------------\n", - "0 Conv2d(1, 20, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", - "1 IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-1.), batch_size=1, num_timesteps=-1)\n", - "2 AvgPool2d(kernel_size=2, stride=2, padding=0)\n", - "3 Conv2d(20, 32, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", - "4 IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-1.), batch_size=1, num_timesteps=-1)\n", - "5 AvgPool2d(kernel_size=2, stride=2, padding=0)\n", - "6 Conv2d(32, 128, kernel_size=(3, 3), stride=(1, 1), bias=False)\n", - "7 IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-1.), batch_size=1, num_timesteps=-1)\n", - "8 AvgPool2d(kernel_size=2, stride=2, padding=0)\n", - "9 Linear(in_features=128, out_features=500, bias=False)\n", - "10 IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-1.), batch_size=1, num_timesteps=-1)\n", - "11 Linear(in_features=500, out_features=10, bias=False)\n", - "12 IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-1.), batch_size=1, num_timesteps=-1)\n", - "--------------------------------------------------------\n", - "True\n", - "True\n", - " True\n" - ] - } - ], - "source": [ - "hw_model = DynapcnnNetworkGraph(\n", - " sinabs_model,\n", - " discretize=True,\n", - " input_shape=input_shape\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "0 Conv2d(1, 20, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", - "1 IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-1.), batch_size=1, num_timesteps=-1)\n", - "2 AvgPool2d(kernel_size=2, stride=2, padding=0)\n", - "3 Conv2d(20, 32, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", - "4 IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-1.), batch_size=1, num_timesteps=-1)\n", - "5 AvgPool2d(kernel_size=2, stride=2, padding=0)\n", - "6 Conv2d(32, 128, kernel_size=(3, 3), stride=(1, 1), bias=False)\n", - "7 IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-1.), batch_size=1, num_timesteps=-1)\n", - "8 AvgPool2d(kernel_size=2, stride=2, padding=0)\n", - "9 Linear(in_features=128, out_features=500, bias=False)\n", - "10 IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-1.), batch_size=1, num_timesteps=-1)\n", - "11 Linear(in_features=500, out_features=10, bias=False)\n", - "12 IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-1.), batch_size=1, num_timesteps=-1)\n" - ] - } - ], - "source": [ - "for i, l in enumerate(hw_model.layers):\n", - " print(i, l)" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(0, 1)\n", - "(1, 2)\n", - "(2, 3)\n", - "(3, 4)\n", - "(4, 5)\n", - "(5, 6)\n", - "(6, 7)\n", - "(7, 8)\n", - "(8, 9)\n", - "(9, 10)\n", - "(10, 11)\n", - "(11, 12)\n" - ] - } - ], - "source": [ - "for edge in hw_model.sinabs_edges:\n", - " print(edge)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Deploying Model" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Network is valid\n" - ] - }, - { - "data": { - "text/plain": [ - "DynapcnnNetworkGraph()" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "hw_model.to(device=\"speck2edevkit:0\")" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "layer index: 0\n", - "layer modules: DynapcnnLayer(\n", - " (conv_layer): Conv2d(1, 20, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", - " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", - " tensor(635.), min_v_mem=Parameter containing:\n", - " tensor(-635.), batch_size=1, num_timesteps=-1)\n", - " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", - ")\n", - "layer destinations: [1]\n", - "assigned core: 0\n", - "\n", - "layer index: 1\n", - "layer modules: DynapcnnLayer(\n", - " (conv_layer): Conv2d(20, 32, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", - " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", - " tensor(11361.), min_v_mem=Parameter containing:\n", - " tensor(-11361.), batch_size=1, num_timesteps=-1)\n", - " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", - ")\n", - "layer destinations: [2]\n", - "assigned core: 3\n", - "\n", - "layer index: 2\n", - "layer modules: DynapcnnLayer(\n", - " (conv_layer): Conv2d(32, 128, kernel_size=(3, 3), stride=(1, 1), bias=False)\n", - " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", - " tensor(8621.), min_v_mem=Parameter containing:\n", - " tensor(-8621.), batch_size=1, num_timesteps=-1)\n", - " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", - ")\n", - "layer destinations: [3]\n", - "assigned core: 5\n", - "\n", - "layer index: 3\n", - "layer modules: DynapcnnLayer(\n", - " (conv_layer): Conv2d(128, 500, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", - " tensor(5747.), min_v_mem=Parameter containing:\n", - " tensor(-5747.), batch_size=1, num_timesteps=-1)\n", - ")\n", - "layer destinations: [4]\n", - "assigned core: 6\n", - "\n", - "layer index: 4\n", - "layer modules: DynapcnnLayer(\n", - " (conv_layer): Conv2d(500, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", - " tensor(2841.), min_v_mem=Parameter containing:\n", - " tensor(-2841.), batch_size=1, num_timesteps=-1)\n", - ")\n", - "layer destinations: []\n", - "assigned core: 1\n", - "\n" - ] - } - ], - "source": [ - "print(hw_model)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "speck-rescnn", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/tests/test_nonsequential/test_speck.ipynb b/tests/test_nonsequential/test_speck.ipynb deleted file mode 100644 index e0d0b307..00000000 --- a/tests/test_nonsequential/test_speck.ipynb +++ /dev/null @@ -1,156 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "\n", - "DynapcnnLayer(\n", - " (conv_layer): Conv2d(1, 20, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", - " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", - " tensor(637.), min_v_mem=Parameter containing:\n", - " tensor(-637.), batch_size=1, num_timesteps=-1)\n", - " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", - ")\n", - "DynapcnnLayer(\n", - " (conv_layer): Conv2d(20, 32, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", - " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", - " tensor(11360.), min_v_mem=Parameter containing:\n", - " tensor(-11360.), batch_size=1, num_timesteps=-1)\n", - " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", - ")\n", - "DynapcnnLayer(\n", - " (conv_layer): Conv2d(32, 128, kernel_size=(3, 3), stride=(1, 1), bias=False)\n", - " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", - " tensor(8621.), min_v_mem=Parameter containing:\n", - " tensor(-8621.), batch_size=1, num_timesteps=-1)\n", - " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", - ")\n", - "DynapcnnLayer(\n", - " (conv_layer): Conv2d(128, 500, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", - " tensor(5748.), min_v_mem=Parameter containing:\n", - " tensor(-5748.), batch_size=1, num_timesteps=-1)\n", - ")\n", - "DynapcnnLayer(\n", - " (conv_layer): Conv2d(500, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", - " tensor(2840.), min_v_mem=Parameter containing:\n", - " tensor(-2840.), batch_size=1, num_timesteps=-1)\n", - ")\n", - "Network is valid\n" - ] - }, - { - "data": { - "text/plain": [ - "DynapcnnNetwork(\n", - " (sequence): Sequential(\n", - " (0): DynapcnnLayer(\n", - " (conv_layer): Conv2d(1, 20, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", - " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", - " tensor(637.), min_v_mem=Parameter containing:\n", - " tensor(-637.), batch_size=1, num_timesteps=-1)\n", - " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", - " )\n", - " (1): DynapcnnLayer(\n", - " (conv_layer): Conv2d(20, 32, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", - " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", - " tensor(11360.), min_v_mem=Parameter containing:\n", - " tensor(-11360.), batch_size=1, num_timesteps=-1)\n", - " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", - " )\n", - " (2): DynapcnnLayer(\n", - " (conv_layer): Conv2d(32, 128, kernel_size=(3, 3), stride=(1, 1), bias=False)\n", - " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", - " tensor(8621.), min_v_mem=Parameter containing:\n", - " tensor(-8621.), batch_size=1, num_timesteps=-1)\n", - " (pool_layer): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", - " )\n", - " (3): DynapcnnLayer(\n", - " (conv_layer): Conv2d(128, 500, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", - " tensor(5748.), min_v_mem=Parameter containing:\n", - " tensor(-5748.), batch_size=1, num_timesteps=-1)\n", - " )\n", - " (4): DynapcnnLayer(\n", - " (conv_layer): Conv2d(500, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (spk_layer): IAFSqueeze(spike_threshold=Parameter containing:\n", - " tensor(2840.), min_v_mem=Parameter containing:\n", - " tensor(-2840.), batch_size=1, num_timesteps=-1)\n", - " )\n", - " )\n", - ")" - ] - }, - "execution_count": 1, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import torch\n", - "import torch.nn as nn\n", - "from typing import List\n", - "from sinabs.from_torch import from_model\n", - "from sinabs.backend.dynapcnn import DynapcnnNetwork\n", - "\n", - "ann = nn.Sequential(\n", - " nn.Conv2d(1, 20, 5, 1, bias=False),\n", - " nn.ReLU(),\n", - " nn.AvgPool2d(2,2),\n", - " nn.Conv2d(20, 32, 5, 1, bias=False),\n", - " nn.ReLU(),\n", - " nn.AvgPool2d(2,2),\n", - " nn.Conv2d(32, 128, 3, 1, bias=False),\n", - " nn.ReLU(),\n", - " nn.AvgPool2d(2,2),\n", - " nn.Flatten(),\n", - " nn.Linear(128, 500, bias=False),\n", - " nn.ReLU(),\n", - " nn.Linear(500, 10, bias=False),\n", - ")\n", - "\n", - "# Convert your model to SNN\n", - "sinabs_model = from_model(ann, add_spiking_output=True, batch_size=1) # Your sinabs SNN model\n", - "\n", - "# Convert your SNN to `DynapcnnNetwork`\n", - "hw_model = DynapcnnNetwork(\n", - " sinabs_model.spiking_model,\n", - " discretize=True,\n", - " input_shape=(1, 28, 28)\n", - ")\n", - "\n", - "# Deploy model to a dev-kit\n", - "hw_model.to(device=\"speck2edevkit:0\")" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "speck-rescnn", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} From 1fc3702cffe3b75b3e68e637945484bf5c35a409 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Fri, 26 Apr 2024 17:02:07 +0200 Subject: [PATCH 042/379] baseline sequential model - NMNIST --- .../test_nonsequential/CNN_branching_2.ipynb | 248 ------------- .../baseline-SCNN-example_1-NNI.ipynb | 351 ++++++++++++++++++ 2 files changed, 351 insertions(+), 248 deletions(-) delete mode 100644 tests/test_nonsequential/CNN_branching_2.ipynb create mode 100644 tests/test_nonsequential/baseline-SCNN-example_1-NNI.ipynb diff --git a/tests/test_nonsequential/CNN_branching_2.ipynb b/tests/test_nonsequential/CNN_branching_2.ipynb deleted file mode 100644 index 4adb2f5e..00000000 --- a/tests/test_nonsequential/CNN_branching_2.ipynb +++ /dev/null @@ -1,248 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import torch\n", - "import torch.nn as nn" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "torch.manual_seed(0)" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "channels = 1\n", - "height = 28\n", - "width = 28\n", - "batch = 1\n", - "\n", - "input_shape = (batch, channels, height, width)" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "x = torch.randn(input_shape)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Network Module" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "class Diverge(nn.Module):\n", - " def __init__(self, pre_pooling_block, pooling) -> None:\n", - " super().__init__()\n", - "\n", - " self.pre_pooling_block = pre_pooling_block\n", - " self.pooling = pooling\n", - "\n", - " def forward(self, x):\n", - " x = self.pre_pooling_block(x)\n", - " x = self.pooling(x)\n", - " return x" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "class CNNLayerBlock(nn.Module):\n", - " def __init__(self, pre_activation, activation, pooling = None) -> None:\n", - " super().__init__()\n", - "\n", - " self.pre_activation = pre_activation\n", - " self.activation = activation\n", - " self.pooling = pooling\n", - "\n", - " def forward(self, x):\n", - " x = self.pre_activation(x)\n", - " x = self.activation(x)\n", - " if self.pooling:\n", - " x = self.pooling(x)\n", - " return x" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "class CNN(nn.Module):\n", - " def __init__(self) -> None:\n", - " super().__init__()\n", - "\n", - " # CNNLayerBlock 1 - this block wants to branch out to target two different layers.\n", - "\n", - " self.cnnlblk_1 = CNNLayerBlock(\n", - " pre_activation = nn.Conv2d(1, 30, 2, 1, bias=False), \n", - " activation = nn.ReLU()\n", - " )\n", - " \n", - " self.diverge_1a = Diverge( # pooling 1a.\n", - " pre_pooling_block = self.cnnlblk_1,\n", - " pooling = nn.AvgPool2d(2,2)\n", - " )\n", - " \n", - " self.diverge_1b = Diverge( # pooling 1b.\n", - " pre_pooling_block = self.cnnlblk_1, \n", - " pooling = nn.AvgPool2d(4,4)\n", - " )\n", - "\n", - " # CNNLayerBlock 2.\n", - "\n", - " self.cnnlblk_2 = CNNLayerBlock(\n", - " pre_activation = nn.Conv2d(30, 30, 2, 1, bias=False),\n", - " activation = nn.ReLU(),\n", - " pooling = nn.AvgPool2d(2,2)\n", - " )\n", - "\n", - " # CNNLayerBlock 3.\n", - "\n", - " self.cnnlblk_3 = CNNLayerBlock(\n", - " pre_activation = nn.Conv2d(30, 1, 3, 1, bias=False), \n", - " activation = nn.ReLU()\n", - " )\n", - "\n", - " # DynapcnnLayer 4. \n", - "\n", - " self.cnnlblk_4 = CNNLayerBlock(\n", - " pre_activation = nn.Linear(16, 500, bias=False), \n", - " activation = nn.ReLU()\n", - " )\n", - " \n", - " # DynapcnnLayer 5.\n", - "\n", - " self.cnnlblk_5 = CNNLayerBlock(\n", - " pre_activation = nn.Linear(500, 10, bias=False), \n", - " activation = nn.ReLU()\n", - " )\n", - "\n", - " # 'support' layers\n", - "\n", - " self.flat = nn.Flatten()\n", - "\n", - "\n", - " def forward(self, x):\n", - " # CNNLayerBlock 1.\n", - " div1a_out = self.diverge_1a(x)\n", - " div1b_out = self.diverge_1b(x)\n", - " \n", - " # CNNLayerBlock 2.\n", - " blk2_out = self.cnnlblk_2(div1a_out)\n", - "\n", - " # CNNLayerBlock 3.\n", - " blk3_out = self.cnnlblk_3(blk2_out + div1b_out)\n", - "\n", - " # CNNLayerBlock 4.\n", - " blk4_out = self.cnnlblk_4(self.flat(blk3_out))\n", - "\n", - " # CNNLayerBlock 5.\n", - " blk5_out = self.cnnlblk_5(blk4_out)\n", - "\n", - " return blk5_out" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "cnn = CNN()" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "torch.Size([1, 30, 6, 6]) torch.Size([1, 30, 6, 6])\n" - ] - } - ], - "source": [ - "# CNNLayerBlock 1.\n", - "div1a_out = cnn.diverge_1a(x)\n", - "div1b_out = cnn.diverge_1b(x)\n", - "\n", - "# CNNLayerBlock 2.\n", - "blk2_out = cnn.cnnlblk_2(div1a_out)\n", - "\n", - "# CNNLayerBlock 3.\n", - "print(blk2_out.shape, div1b_out.shape)\n", - "blk3_out = cnn.cnnlblk_3(blk2_out + div1b_out)\n", - "\n", - "# CNNLayerBlock 4.\n", - "blk4_out = cnn.cnnlblk_4(cnn.flat(blk3_out))\n", - "\n", - "# CNNLayerBlock 5.\n", - "blk5_out = cnn.cnnlblk_5(blk4_out)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "speck-rescnn", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/tests/test_nonsequential/baseline-SCNN-example_1-NNI.ipynb b/tests/test_nonsequential/baseline-SCNN-example_1-NNI.ipynb new file mode 100644 index 00000000..863d0c40 --- /dev/null +++ b/tests/test_nonsequential/baseline-SCNN-example_1-NNI.ipynb @@ -0,0 +1,351 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "import torch.nn as nn\n", + "import sinabs.layers as sl\n", + "import tqdm\n", + "\n", + "from tonic.datasets.nmnist import NMNIST\n", + "from tonic.transforms import ToFrame\n", + "from torch.utils.data import DataLoader\n", + "from torch.nn import CrossEntropyLoss\n", + "from torch.optim import SGD" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "torch.manual_seed(0)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Loading Data" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "batch_size = 64\n", + "num_workers = 4\n", + "epochs = 1" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "root_dir = \"./NMNIST\"\n", + "_ = NMNIST(save_to=root_dir, train=True)\n", + "_ = NMNIST(save_to=root_dir, train=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "n_time_steps = 50\n", + "to_raster = ToFrame(sensor_size=NMNIST.sensor_size, n_time_bins=n_time_steps)\n", + "\n", + "snn_train_dataset = NMNIST(save_to=root_dir, train=True, transform=to_raster)\n", + "snn_test_dataset = NMNIST(save_to=root_dir, train=False, transform=to_raster)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "snn_train_dataloader = DataLoader(snn_train_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True)\n", + "snn_test_dataloader = DataLoader(snn_test_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=False)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Network Module\n", + "\n", + "We need to define a `nn.Module` implementing the network we want the chip to reproduce." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "device: NVIDIA GeForce RTX 3070 Ti\n" + ] + } + ], + "source": [ + "if torch.cuda.is_available():\n", + " device = torch.device('cuda:0')\n", + " print('device: ', torch.cuda.get_device_name(0))\n", + "else:\n", + " device = torch.device('cpu')" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "class SNN(nn.Module):\n", + " def __init__(self) -> None:\n", + " super().__init__()\n", + "\n", + " self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False) \n", + " self.iaf1 = sl.IAFSqueeze(batch_size=1) \n", + " self.pool1 = sl.SumPool2d(3,3) \n", + " self.pool1a = sl.SumPool2d(4,4) \n", + "\n", + " self.conv2 = nn.Conv2d(10, 10, 4, 1, bias=False)\n", + " self.iaf2 = sl.IAFSqueeze(batch_size=1) \n", + "\n", + " self.conv3 = nn.Conv2d(10, 1, 2, 1, bias=False) \n", + " self.iaf3 = sl.IAFSqueeze(batch_size=1) \n", + "\n", + " self.flat = nn.Flatten()\n", + "\n", + " self.fc1 = nn.Linear(49, 100, bias=False) \n", + " self.iaf4 = sl.IAFSqueeze(batch_size=1) \n", + " \n", + " self.fc2 = nn.Linear(100, 10, bias=False) \n", + " self.iaf5 = sl.IAFSqueeze(batch_size=1) \n", + "\n", + "\n", + " def detach_neuron_states(self):\n", + " for name, layer in self.named_modules():\n", + " if name != '':\n", + " if isinstance(layer, sl.StatefulLayer):\n", + " for name, buffer in layer.named_buffers():\n", + " buffer.detach_()\n", + "\n", + " def init_weights(self):\n", + " for name, layer in self.named_modules():\n", + " if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear):\n", + " nn.init.xavier_normal_(layer.weight.data)\n", + "\n", + " def forward(self, x):\n", + " \n", + " con1_out = self.conv1(x)\n", + " iaf1_out = self.iaf1(con1_out)\n", + " pool1_out = self.pool1(iaf1_out)\n", + "\n", + " conv2_out = self.conv2(pool1_out)\n", + " iaf2_out = self.iaf2(conv2_out)\n", + "\n", + " conv3_out = self.conv3(iaf2_out)\n", + " iaf3_out = self.iaf3(conv3_out)\n", + "\n", + " flat_out = self.flat(iaf3_out)\n", + " \n", + " fc1_out = self.fc1(flat_out)\n", + " iaf4_out = self.iaf4(fc1_out)\n", + " fc2_out = self.fc2(iaf4_out)\n", + " iaf5_out = self.iaf5(fc2_out)\n", + "\n", + " return iaf5_out" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "snn = SNN().to(device)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "snn.init_weights()" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "optimizer = SGD(snn.parameters(), lr=0.001)\n", + "loss_fn = CrossEntropyLoss()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Define train and test" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "def train(dataloader, model, loss_fn, optimizer, epochs, test_func, dataloader_test):\n", + " epochs_y = []\n", + " epochs_x = []\n", + " model.train()\n", + "\n", + " for e in range(epochs):\n", + " losses = []\n", + " batches = []\n", + " batch_count = 0\n", + " train_p_bar = tqdm(snn_train_dataloader)\n", + "\n", + " for X, y in train_p_bar:\n", + " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", + " X = X.reshape(-1, 2, 34, 34).to(dtype=torch.float, device=device)\n", + " y = y.to(dtype=torch.long, device=device)\n", + "\n", + " # forward\n", + " pred = model(X)\n", + "\n", + " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", + " pred = pred.reshape(batch_size, n_time_steps, -1)\n", + "\n", + " # accumulate all time-steps output for final prediction\n", + " pred = pred.sum(dim = 1)\n", + " loss = loss_fn(pred, y)\n", + "\n", + " # gradient update\n", + " optimizer.zero_grad()\n", + " loss.backward()\n", + " optimizer.step()\n", + "\n", + " # detach the neuron states and activations from current computation graph(necessary)\n", + " model.detach_neuron_states()\n", + "\n", + " train_p_bar.set_description(f\"Epoch {e} - BPTT Training Loss: {round(loss.item(), 4)}\")\n", + "\n", + " batch_count += 1\n", + " losses.append(loss.item())\n", + " batches.append(batch_count)\n", + "\n", + " epochs_y.append(losses)\n", + " epochs_x.append(batches)\n", + "\n", + " acc = test_func(dataloader_test, model)\n", + " print(f'Epoch {e} accuracy: {acc}')\n", + "\n", + " return epochs_x, epochs_y\n" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "def test(dataloader, model):\n", + " correct_predictions = []\n", + " with torch.no_grad():\n", + " test_p_bar = tqdm(dataloader)\n", + " for X, y in test_p_bar:\n", + " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", + " X = X.reshape(-1, 2, 34, 34).to(dtype=torch.float, device=device)\n", + " y = y.to(dtype=torch.long, device=device)\n", + "\n", + " # forward\n", + " output = model(X)\n", + "\n", + " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", + " output = output.reshape(batch_size, n_time_steps, -1)\n", + "\n", + " # accumulate all time-steps output for final prediction\n", + " output = output.sum(dim=1)\n", + "\n", + " # calculate accuracy\n", + " pred = output.argmax(dim=1, keepdim=True)\n", + "\n", + " # compute the total correct predictions\n", + " correct_predictions.append(pred.eq(y.view_as(pred)))\n", + "\n", + " test_p_bar.set_description(f\"Testing Model...\")\n", + " \n", + " correct_predictions = torch.cat(correct_predictions)\n", + " return correct_predictions.sum().item()/(len(correct_predictions))*100" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Training loop (HPO)" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "train(snn_train_dataloader, snn, loss_fn, optimizer, test, snn_test_dataloader)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "speck-rescnn", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 805313704e692c2a050e78e3d2f7ed5d185e0b45 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Fri, 26 Apr 2024 17:26:47 +0200 Subject: [PATCH 043/379] trying NNI --- tests/test_nonsequential/NNI-test/main.ipynb | 136 ++++++++++++++ tests/test_nonsequential/NNI-test/model.py | 175 +++++++++++++++++++ 2 files changed, 311 insertions(+) create mode 100644 tests/test_nonsequential/NNI-test/main.ipynb create mode 100644 tests/test_nonsequential/NNI-test/model.py diff --git a/tests/test_nonsequential/NNI-test/main.ipynb b/tests/test_nonsequential/NNI-test/main.ipynb new file mode 100644 index 00000000..553c1170 --- /dev/null +++ b/tests/test_nonsequential/NNI-test/main.ipynb @@ -0,0 +1,136 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from nni.experiment import Experiment\n", + "experiment = Experiment('local')" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "search_space = {\n", + " 'lr': {'_type': 'loguniform', '_value': [0.001, 0.0001]},\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "experiment.config.trial_command = 'python model.py'\n", + "experiment.config.trial_code_directory = '.'" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "experiment.config.search_space = search_space" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "experiment.config.tuner.name = 'TPE'\n", + "experiment.config.tuner.class_args['optimize_mode'] = 'maximize'" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "experiment.config.max_trial_number = 5\n", + "experiment.config.trial_concurrency = 2" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[2024-04-26 17:05:41] \u001b[32mCreating experiment, Experiment ID: \u001b[36mda0h79xv\u001b[0m\n", + "[2024-04-26 17:05:41] \u001b[32mStarting web server...\u001b[0m\n", + "[2024-04-26 17:05:42] \u001b[32mSetting up...\u001b[0m\n", + "[2024-04-26 17:05:42] \u001b[32mWeb portal URLs: \u001b[36mhttp://127.0.0.1:8080 http://192.168.11.68:8080 http://172.17.0.1:8080\u001b[0m\n" + ] + }, + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "experiment.run(8080)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[2024-04-26 17:05:52] \u001b[32mStopping experiment, please wait...\u001b[0m\n", + "[2024-04-26 17:05:52] \u001b[32mSaving experiment checkpoint...\u001b[0m\n", + "[2024-04-26 17:05:52] \u001b[32mStopping NNI manager, if any...\u001b[0m\n", + "[2024-04-26 17:05:54] \u001b[32mExperiment stopped.\u001b[0m\n" + ] + } + ], + "source": [ + "#input('Press enter to quit')\n", + "experiment.stop()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "speck-rescnn", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tests/test_nonsequential/NNI-test/model.py b/tests/test_nonsequential/NNI-test/model.py new file mode 100644 index 00000000..2ba97930 --- /dev/null +++ b/tests/test_nonsequential/NNI-test/model.py @@ -0,0 +1,175 @@ +import torch +import torch.nn as nn +import sinabs.layers as sl +import nni + +from tonic.datasets.nmnist import NMNIST +from tonic.transforms import ToFrame +from torch.utils.data import DataLoader +from torch.nn import CrossEntropyLoss +from torch.optim import SGD + +params = { + 'lr': 0.001, +} + +optimized_params = nni.get_next_parameter() +params.update(optimized_params) +print(params) + +###### Loading Data ###### + +batch_size = 32 +num_workers = 4 +epochs = 1 + +root_dir = "./NMNIST" +_ = NMNIST(save_to=root_dir, train=True) +_ = NMNIST(save_to=root_dir, train=False) + +n_time_steps = 50 +to_raster = ToFrame(sensor_size=NMNIST.sensor_size, n_time_bins=n_time_steps) + +snn_train_dataset = NMNIST(save_to=root_dir, train=True, transform=to_raster) +snn_test_dataset = NMNIST(save_to=root_dir, train=False, transform=to_raster) + +snn_train_dataloader = DataLoader(snn_train_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True) +snn_test_dataloader = DataLoader(snn_test_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=False) + +###### Defining the Model ###### + +if torch.cuda.is_available(): + device = torch.device('cuda:0') + print('device: ', torch.cuda.get_device_name(0)) +else: + device = torch.device('cpu') + +class SNN(nn.Module): + def __init__(self) -> None: + super().__init__() + + self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False) + self.iaf1 = sl.IAFSqueeze(batch_size=1) + self.pool1 = sl.SumPool2d(3,3) + self.pool1a = sl.SumPool2d(4,4) + + self.conv2 = nn.Conv2d(10, 10, 4, 1, bias=False) + self.iaf2 = sl.IAFSqueeze(batch_size=1) + + self.conv3 = nn.Conv2d(10, 1, 2, 1, bias=False) + self.iaf3 = sl.IAFSqueeze(batch_size=1) + + self.flat = nn.Flatten() + + self.fc1 = nn.Linear(49, 100, bias=False) + self.iaf4 = sl.IAFSqueeze(batch_size=1) + + self.fc2 = nn.Linear(100, 10, bias=False) + self.iaf5 = sl.IAFSqueeze(batch_size=1) + + + def detach_neuron_states(self): + for name, layer in self.named_modules(): + if name != '': + if isinstance(layer, sl.StatefulLayer): + for name, buffer in layer.named_buffers(): + buffer.detach_() + + def init_weights(self): + for name, layer in self.named_modules(): + if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear): + nn.init.xavier_normal_(layer.weight.data) + + def forward(self, x): + + con1_out = self.conv1(x) + iaf1_out = self.iaf1(con1_out) + pool1_out = self.pool1(iaf1_out) + + conv2_out = self.conv2(pool1_out) + iaf2_out = self.iaf2(conv2_out) + + conv3_out = self.conv3(iaf2_out) + iaf3_out = self.iaf3(conv3_out) + + flat_out = self.flat(iaf3_out) + + fc1_out = self.fc1(flat_out) + iaf4_out = self.iaf4(fc1_out) + fc2_out = self.fc2(iaf4_out) + iaf5_out = self.iaf5(fc2_out) + + return iaf5_out + +snn = SNN().to(device) + +snn.init_weights() + +optimizer = SGD(snn.parameters(), lr=params['lr']) +loss_fn = CrossEntropyLoss() + +###### Defining Train/Test ###### + +def train(dataloader, model, loss_fn, optimizer): + size = len(dataloader.dataset) + model.train() + for batch, (X, y) in enumerate(dataloader): + # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width] + X = X.reshape(-1, 2, 34, 34).to(dtype=torch.float, device=device) + y = y.to(dtype=torch.long, device=device) + + # forward + pred = model(X) + + # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes] + pred = pred.reshape(batch_size, n_time_steps, -1) + + # accumulate all time-steps output for final prediction + pred = pred.sum(dim = 1) + loss = loss_fn(pred, y) + + # gradient update + optimizer.zero_grad() + loss.backward() + optimizer.step() + + # detach the neuron states and activations from current computation graph(necessary) + model.detach_neuron_states() + + break + +def test(dataloader, model): + correct_predictions = [] + with torch.no_grad(): + for X, y in dataloader: + # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width] + X = X.reshape(-1, 2, 34, 34).to(dtype=torch.float, device=device) + y = y.to(dtype=torch.long, device=device) + + # forward + output = model(X) + + # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes] + output = output.reshape(batch_size, n_time_steps, -1) + + # accumulate all time-steps output for final prediction + output = output.sum(dim=1) + + # calculate accuracy + pred = output.argmax(dim=1, keepdim=True) + + # compute the total correct predictions + correct_predictions.append(pred.eq(y.view_as(pred))) + + break + + return correct_predictions.sum().item()/(len(correct_predictions))*100 + +###### Training loop (HPO) ###### + +for t in range(epochs): + print(f"Epoch {t+1}\n-------------------------------") + train(snn_train_dataloader, snn, loss_fn, optimizer) + accuracy = test(snn_test_dataloader, snn) + nni.report_intermediate_result(accuracy) +nni.report_final_result(accuracy) \ No newline at end of file From ecb5f35107994e0cefddc2275cb91b1c171917d2 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Fri, 26 Apr 2024 19:41:16 +0200 Subject: [PATCH 044/379] baseline accuracy - example 1 --- .../baseline-SCNN-example_1-NNI.ipynb | 138 +++++++++++++----- 1 file changed, 100 insertions(+), 38 deletions(-) diff --git a/tests/test_nonsequential/baseline-SCNN-example_1-NNI.ipynb b/tests/test_nonsequential/baseline-SCNN-example_1-NNI.ipynb index 863d0c40..159942e2 100644 --- a/tests/test_nonsequential/baseline-SCNN-example_1-NNI.ipynb +++ b/tests/test_nonsequential/baseline-SCNN-example_1-NNI.ipynb @@ -2,20 +2,24 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": 16, "metadata": {}, "outputs": [], "source": [ "import torch\n", "import torch.nn as nn\n", "import sinabs.layers as sl\n", - "import tqdm\n", + "from tqdm.notebook import tqdm\n", "\n", "from tonic.datasets.nmnist import NMNIST\n", "from tonic.transforms import ToFrame\n", "from torch.utils.data import DataLoader\n", "from torch.nn import CrossEntropyLoss\n", - "from torch.optim import SGD" + "from torch.optim import Adam\n", + "\n", + "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential\n", + "\n", + "import matplotlib.pyplot as plt" ] }, { @@ -26,7 +30,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 2, @@ -47,18 +51,19 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "batch_size = 64\n", "num_workers = 4\n", - "epochs = 1" + "epochs = 1\n", + "lr = 1e-3" ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -69,7 +74,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -82,7 +87,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -101,7 +106,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 7, "metadata": {}, "outputs": [ { @@ -122,7 +127,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ @@ -130,24 +135,22 @@ " def __init__(self) -> None:\n", " super().__init__()\n", "\n", - " self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False) \n", - " self.iaf1 = sl.IAFSqueeze(batch_size=1) \n", - " self.pool1 = sl.SumPool2d(3,3) \n", - " self.pool1a = sl.SumPool2d(4,4) \n", + " self.conv1 = nn.Conv2d(2, 8, 3, 1, bias=False)\n", + " self.iaf1 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + " self.pool1 = nn.AvgPool2d(2,2)\n", "\n", - " self.conv2 = nn.Conv2d(10, 10, 4, 1, bias=False)\n", - " self.iaf2 = sl.IAFSqueeze(batch_size=1) \n", + " self.conv2 = nn.Conv2d(8, 16, 3, 1, bias=False)\n", + " self.iaf2 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + " self.pool2 = nn.AvgPool2d(2,2)\n", "\n", - " self.conv3 = nn.Conv2d(10, 1, 2, 1, bias=False) \n", - " self.iaf3 = sl.IAFSqueeze(batch_size=1) \n", + " self.conv3 = nn.Conv2d(16, 16, 3, 1, bias=False)\n", + " self.iaf3 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + " self.pool3 = nn.AvgPool2d(2,2)\n", "\n", " self.flat = nn.Flatten()\n", "\n", - " self.fc1 = nn.Linear(49, 100, bias=False) \n", - " self.iaf4 = sl.IAFSqueeze(batch_size=1) \n", - " \n", - " self.fc2 = nn.Linear(100, 10, bias=False) \n", - " self.iaf5 = sl.IAFSqueeze(batch_size=1) \n", + " self.fc1 = nn.Linear(64, 10, bias=False)\n", + " self.iaf4 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", "\n", "\n", " def detach_neuron_states(self):\n", @@ -170,23 +173,23 @@ "\n", " conv2_out = self.conv2(pool1_out)\n", " iaf2_out = self.iaf2(conv2_out)\n", + " pool2_out = self.pool2(iaf2_out)\n", "\n", - " conv3_out = self.conv3(iaf2_out)\n", + " conv3_out = self.conv3(pool2_out)\n", " iaf3_out = self.iaf3(conv3_out)\n", + " pool3_out = self.pool3(iaf3_out)\n", "\n", - " flat_out = self.flat(iaf3_out)\n", + " flat_out = self.flat(pool3_out)\n", " \n", " fc1_out = self.fc1(flat_out)\n", " iaf4_out = self.iaf4(fc1_out)\n", - " fc2_out = self.fc2(iaf4_out)\n", - " iaf5_out = self.iaf5(fc2_out)\n", "\n", - " return iaf5_out" + " return iaf4_out" ] }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -195,7 +198,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ @@ -204,11 +207,11 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 11, "metadata": {}, "outputs": [], "source": [ - "optimizer = SGD(snn.parameters(), lr=0.001)\n", + "optimizer = Adam(snn.parameters(), lr=lr, betas=(0.9, 0.999), eps=1e-8)\n", "loss_fn = CrossEntropyLoss()" ] }, @@ -221,7 +224,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 12, "metadata": {}, "outputs": [], "source": [ @@ -276,7 +279,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 13, "metadata": {}, "outputs": [], "source": [ @@ -319,11 +322,70 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 14, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "3417439bf42847f98f43aad80ffb8e53", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/937 [00:00" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ - "train(snn_train_dataloader, snn, loss_fn, optimizer, test, snn_test_dataloader)" + "plt.plot(epochs_x[0], epochs_y[0])\n", + "plt.xlabel('batches')\n", + "plt.ylabel('loss')\n", + "plt.show()" ] } ], From 6d890ff7a86066e2c1b725ecf44b33efe86d8800 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Fri, 26 Apr 2024 20:10:30 +0200 Subject: [PATCH 045/379] non-sequential version --- .../non-sequential-SCNN-example_1.ipynb | 445 ++++++++++++++++++ 1 file changed, 445 insertions(+) create mode 100644 tests/test_nonsequential/non-sequential-SCNN-example_1.ipynb diff --git a/tests/test_nonsequential/non-sequential-SCNN-example_1.ipynb b/tests/test_nonsequential/non-sequential-SCNN-example_1.ipynb new file mode 100644 index 00000000..becc5d50 --- /dev/null +++ b/tests/test_nonsequential/non-sequential-SCNN-example_1.ipynb @@ -0,0 +1,445 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "import torch.nn as nn\n", + "import sinabs.layers as sl\n", + "from tqdm.notebook import tqdm\n", + "\n", + "from tonic.datasets.nmnist import NMNIST\n", + "from tonic.transforms import ToFrame\n", + "from torch.utils.data import DataLoader\n", + "from torch.nn import CrossEntropyLoss\n", + "from torch.optim import Adam\n", + "\n", + "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential\n", + "\n", + "import matplotlib.pyplot as plt" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "torch.manual_seed(0)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Loading Data" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "batch_size = 64\n", + "num_workers = 4\n", + "epochs = 5\n", + "lr = 1e-3" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "root_dir = \"./NMNIST\"\n", + "_ = NMNIST(save_to=root_dir, train=True)\n", + "_ = NMNIST(save_to=root_dir, train=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "n_time_steps = 50\n", + "to_raster = ToFrame(sensor_size=NMNIST.sensor_size, n_time_bins=n_time_steps)\n", + "\n", + "snn_train_dataset = NMNIST(save_to=root_dir, train=True, transform=to_raster)\n", + "snn_test_dataset = NMNIST(save_to=root_dir, train=False, transform=to_raster)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "snn_train_dataloader = DataLoader(snn_train_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True)\n", + "snn_test_dataloader = DataLoader(snn_test_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=False)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Network Module\n", + "\n", + "We need to define a `nn.Module` implementing the network we want the chip to reproduce." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "device: NVIDIA GeForce RTX 3070 Ti\n" + ] + } + ], + "source": [ + "if torch.cuda.is_available():\n", + " device = torch.device('cuda:0')\n", + " print('device: ', torch.cuda.get_device_name(0))\n", + "else:\n", + " device = torch.device('cpu')" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "class SNN(nn.Module):\n", + " def __init__(self) -> None:\n", + " super().__init__()\n", + "\n", + " self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False)\n", + " self.iaf1 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + " self.pool1 = nn.AvgPool2d(2,2)\n", + " self.pool1a = nn.AvgPool2d(6,6)\n", + "\n", + " self.conv2 = nn.Conv2d(10, 10, 2, 1, bias=False)\n", + " self.iaf2 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + " self.pool2 = nn.AvgPool2d(3,3)\n", + "\n", + " self.conv3 = nn.Conv2d(10, 10, 3, 1, bias=False)\n", + " self.iaf3 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + " self.pool3 = nn.AvgPool2d(2,2)\n", + "\n", + " self.flat = nn.Flatten()\n", + "\n", + " self.fc1 = nn.Linear(10, 10, bias=False)\n", + " self.iaf4 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + "\n", + " self.merge = sl.Merge()\n", + "\n", + "\n", + " def detach_neuron_states(self):\n", + " for name, layer in self.named_modules():\n", + " if name != '':\n", + " if isinstance(layer, sl.StatefulLayer):\n", + " for name, buffer in layer.named_buffers():\n", + " buffer.detach_()\n", + "\n", + " def init_weights(self):\n", + " for name, layer in self.named_modules():\n", + " if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear):\n", + " nn.init.xavier_normal_(layer.weight.data)\n", + "\n", + " def forward(self, x):\n", + " \n", + " con1_out = self.conv1(x)\n", + " iaf1_out = self.iaf1(con1_out)\n", + " pool1_out = self.pool1(iaf1_out)\n", + " pool1a_out = self.pool1a(iaf1_out)\n", + "\n", + " conv2_out = self.conv2(pool1_out)\n", + " iaf2_out = self.iaf2(conv2_out)\n", + " pool2_out = self.pool2(iaf2_out)\n", + "\n", + " merged = self.merge(pool1a_out, pool2_out)\n", + "\n", + " conv3_out = self.conv3(merged)\n", + " iaf3_out = self.iaf3(conv3_out)\n", + " pool3_out = self.pool3(iaf3_out)\n", + "\n", + " flat_out = self.flat(pool3_out)\n", + " \n", + " fc1_out = self.fc1(flat_out)\n", + " iaf4_out = self.iaf4(fc1_out)\n", + "\n", + " return iaf4_out" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "snn = SNN().to(device)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "# x = torch.randn((batch_size, 2, 34, 34)).to(device)\n", + "\n", + "# con1_out = snn.conv1(x)\n", + "# iaf1_out = snn.iaf1(con1_out)\n", + "# pool1_out = snn.pool1(iaf1_out)\n", + "# pool1a_out = snn.pool1a(iaf1_out)\n", + "# print(pool1a_out.shape)\n", + "\n", + "# conv2_out = snn.conv2(pool1_out)\n", + "# iaf2_out = snn.iaf2(conv2_out)\n", + "# pool2_out = snn.pool2(iaf2_out)\n", + "# print(pool2_out.shape)\n", + "\n", + "# conv3_out = snn.conv3(pool2_out)\n", + "# iaf3_out = snn.iaf3(conv3_out)\n", + "# pool3_out = snn.pool3(iaf3_out)\n", + "\n", + "# flat_out = snn.flat(pool3_out)\n", + "# print(flat_out.shape)\n", + "\n", + "# fc1_out = snn.fc1(flat_out)\n", + "# iaf4_out = snn.iaf4(fc1_out)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "snn.init_weights()" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "optimizer = Adam(snn.parameters(), lr=lr, betas=(0.9, 0.999), eps=1e-8)\n", + "loss_fn = CrossEntropyLoss()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Define train and test" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "def train(dataloader, model, loss_fn, optimizer, epochs, test_func, dataloader_test):\n", + " epochs_y = []\n", + " epochs_x = []\n", + " epochs_acc = []\n", + " model.train()\n", + "\n", + " for e in range(epochs):\n", + " losses = []\n", + " batches = []\n", + " batch_count = 0\n", + " train_p_bar = tqdm(snn_train_dataloader)\n", + "\n", + " for X, y in train_p_bar:\n", + " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", + " X = X.reshape(-1, 2, 34, 34).to(dtype=torch.float, device=device)\n", + " y = y.to(dtype=torch.long, device=device)\n", + "\n", + " # forward\n", + " pred = model(X)\n", + "\n", + " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", + " pred = pred.reshape(batch_size, n_time_steps, -1)\n", + "\n", + " # accumulate all time-steps output for final prediction\n", + " pred = pred.sum(dim = 1)\n", + " loss = loss_fn(pred, y)\n", + "\n", + " # gradient update\n", + " optimizer.zero_grad()\n", + " loss.backward()\n", + " optimizer.step()\n", + "\n", + " # detach the neuron states and activations from current computation graph(necessary)\n", + " model.detach_neuron_states()\n", + "\n", + " train_p_bar.set_description(f\"Epoch {e} - BPTT Training Loss: {round(loss.item(), 4)}\")\n", + "\n", + " batch_count += 1\n", + " losses.append(loss.item())\n", + " batches.append(batch_count)\n", + "\n", + " epochs_y.append(losses)\n", + " epochs_x.append(batches)\n", + "\n", + " acc = test_func(dataloader_test, model)\n", + " print(f'Epoch {e} accuracy: {acc}')\n", + " epochs_acc.append(acc)\n", + "\n", + " return epochs_x, epochs_y, epochs_acc\n" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "def test(dataloader, model):\n", + " correct_predictions = []\n", + " with torch.no_grad():\n", + " test_p_bar = tqdm(dataloader)\n", + " for X, y in test_p_bar:\n", + " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", + " X = X.reshape(-1, 2, 34, 34).to(dtype=torch.float, device=device)\n", + " y = y.to(dtype=torch.long, device=device)\n", + "\n", + " # forward\n", + " output = model(X)\n", + "\n", + " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", + " output = output.reshape(batch_size, n_time_steps, -1)\n", + "\n", + " # accumulate all time-steps output for final prediction\n", + " output = output.sum(dim=1)\n", + "\n", + " # calculate accuracy\n", + " pred = output.argmax(dim=1, keepdim=True)\n", + "\n", + " # compute the total correct predictions\n", + " correct_predictions.append(pred.eq(y.view_as(pred)))\n", + "\n", + " test_p_bar.set_description(f\"Testing Model...\")\n", + " \n", + " correct_predictions = torch.cat(correct_predictions)\n", + " return correct_predictions.sum().item()/(len(correct_predictions))*100" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Training loop (HPO)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "546940b28031476fb3bee7a239fd1529", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/937 [00:00 1\u001b[0m epochs_x, epochs_y \u001b[38;5;241m=\u001b[39m train(snn_train_dataloader, snn, loss_fn, optimizer, epochs, test, snn_test_dataloader)\n", + "Cell \u001b[0;32mIn[13], line 29\u001b[0m, in \u001b[0;36mtrain\u001b[0;34m(dataloader, model, loss_fn, optimizer, epochs, test_func, dataloader_test)\u001b[0m\n\u001b[1;32m 27\u001b[0m \u001b[38;5;66;03m# gradient update\u001b[39;00m\n\u001b[1;32m 28\u001b[0m optimizer\u001b[38;5;241m.\u001b[39mzero_grad()\n\u001b[0;32m---> 29\u001b[0m loss\u001b[38;5;241m.\u001b[39mbackward()\n\u001b[1;32m 30\u001b[0m optimizer\u001b[38;5;241m.\u001b[39mstep()\n\u001b[1;32m 32\u001b[0m \u001b[38;5;66;03m# detach the neuron states and activations from current computation graph(necessary)\u001b[39;00m\n", + "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/_tensor.py:522\u001b[0m, in \u001b[0;36mTensor.backward\u001b[0;34m(self, gradient, retain_graph, create_graph, inputs)\u001b[0m\n\u001b[1;32m 512\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m has_torch_function_unary(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m 513\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m handle_torch_function(\n\u001b[1;32m 514\u001b[0m Tensor\u001b[38;5;241m.\u001b[39mbackward,\n\u001b[1;32m 515\u001b[0m (\u001b[38;5;28mself\u001b[39m,),\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 520\u001b[0m inputs\u001b[38;5;241m=\u001b[39minputs,\n\u001b[1;32m 521\u001b[0m )\n\u001b[0;32m--> 522\u001b[0m torch\u001b[38;5;241m.\u001b[39mautograd\u001b[38;5;241m.\u001b[39mbackward(\n\u001b[1;32m 523\u001b[0m \u001b[38;5;28mself\u001b[39m, gradient, retain_graph, create_graph, inputs\u001b[38;5;241m=\u001b[39minputs\n\u001b[1;32m 524\u001b[0m )\n", + "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/autograd/__init__.py:266\u001b[0m, in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[0m\n\u001b[1;32m 261\u001b[0m retain_graph \u001b[38;5;241m=\u001b[39m create_graph\n\u001b[1;32m 263\u001b[0m \u001b[38;5;66;03m# The reason we repeat the same comment below is that\u001b[39;00m\n\u001b[1;32m 264\u001b[0m \u001b[38;5;66;03m# some Python versions print out the first line of a multi-line function\u001b[39;00m\n\u001b[1;32m 265\u001b[0m \u001b[38;5;66;03m# calls in the traceback and some print out the last line\u001b[39;00m\n\u001b[0;32m--> 266\u001b[0m Variable\u001b[38;5;241m.\u001b[39m_execution_engine\u001b[38;5;241m.\u001b[39mrun_backward( \u001b[38;5;66;03m# Calls into the C++ engine to run the backward pass\u001b[39;00m\n\u001b[1;32m 267\u001b[0m tensors,\n\u001b[1;32m 268\u001b[0m grad_tensors_,\n\u001b[1;32m 269\u001b[0m retain_graph,\n\u001b[1;32m 270\u001b[0m create_graph,\n\u001b[1;32m 271\u001b[0m inputs,\n\u001b[1;32m 272\u001b[0m allow_unreachable\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m 273\u001b[0m accumulate_grad\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m 274\u001b[0m )\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m: " + ] + } + ], + "source": [ + "epochs_x, epochs_y, epochs_acc = train(snn_train_dataloader, snn, loss_fn, optimizer, epochs, test, snn_test_dataloader)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAGwCAYAAABVdURTAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAABpgUlEQVR4nO3dd3hUVfoH8O+UZNI7KUACoRiatAQlSBUMAhbWuhZQdy2oCBhZFOtaw6q/FbGAKIosIu5uUFlBBJQqHRJ6EwIJISEESK8zc39/hJncO3OnZDIlmXw/z5PHmXvPvXNmgszLe95zjkIQBAFEREREXkLp6Q4QERERORODGyIiIvIqDG6IiIjIqzC4ISIiIq/C4IaIiIi8CoMbIiIi8ioMboiIiMirqD3dAXfT6/U4f/48goODoVAoPN0dIiIisoMgCCgvL0f79u2hVFrPzbS54Ob8+fOIj4/3dDeIiIjIAXl5eejYsaPVNm0uuAkODgbQ8OGEhIR4uDdERERkj7KyMsTHxxu/x61pc8GNYSgqJCSEwQ0REVErY09JCQuKiYiIyKswuCEiIiKvwuCGiIiIvAqDGyIiIvIqDG6IiIjIqzC4ISIiIq/C4IaIiIi8CoMbIiIi8ioMboiIiMirMLghIiIir8LghoiIiLwKgxsnqtPqcdvHWzFjeZanu0JERNRmMbhxoqzcKzhwrhQ/ZJ9HvU7v6e4QERG1SQxunEirF4yPL5bXerAnREREbReDGye6XFlnfFxQWuPBnhAREbVdDG6c6FJFY7amoLTagz0hIiJquxjcOJE4c1PMYSkiIiKPYHDjRJdEwY1OsNKQiIiIXIbBjRPVahtnSAkCoxsiIiJPYHDjRHpRQKNncENEROQRDG6cSBzP6BnbEBEReQSDGydi5oaIiMjzGNw4kThbw9iGiIjIMxjcOJE4W8OCYiIiIs9gcONEgmRYyoMdISIiasMY3DiRXrRXJmtuiIiIPIPBjRMJYOaGiIjI0xjcOJEkoGHmhoiIyCMY3DgRa26IiIg8j8GNE+kli/gxuiEiIvIEBjdOpGfmhoiIyOMY3DiRdBE/RjdERESewODGicQBDUMbIiIiz2Bw40SSYSmOSxEREXkEgxsnki7i57l+EBERtWUMbpyIu4ITERF5HoMbJxJYUExERORxDG6cSM+CYiIiIo9jcONEHJYiIiLyPI8GN/Pnz0ffvn0REhKCkJAQpKam4ueff7bYfuPGjVAoFGY/x44dc2OvLZOuUOy5fhAREbVlak++eMeOHTFnzhx069YNAPD111/j9ttvR1ZWFnr37m3xuuPHjyMkJMT4vF27di7vqz0k69wwc0NEROQRHg1ubr31Vsnzt99+G/Pnz8eOHTusBjfR0dEICwuz6zVqa2tRW1trfF5WVuZQX+0hydzoLbcjIiIi12kxNTc6nQ7Lly9HZWUlUlNTrbYdMGAA4uLiMHr0aGzYsMFq24yMDISGhhp/4uPjndltCWlBMTM3REREnuDx4ObgwYMICgqCRqPBlClT8P3336NXr16ybePi4rBw4UJkZmZixYoVSEpKwujRo7F582aL9589ezZKS0uNP3l5ea56K6y5ISIiagE8OiwFAElJScjOzkZJSQkyMzPx0EMPYdOmTbIBTlJSEpKSkozPU1NTkZeXh/fffx/Dhw+Xvb9Go4FGo3FZ/8UEzpYiIiLyOI9nbnx9fdGtWzekpKQgIyMD/fr1w4cffmj39YMHD8bJkydd2EP7SRfx81w/iIiI2jKPBzemBEGQFADbkpWVhbi4OBf2yH56zpYiIiLyOI8OS7344osYN24c4uPjUV5ejuXLl2Pjxo1Ys2YNgIZ6mfz8fCxZsgQAMHfuXHTu3Bm9e/dGXV0dli5diszMTGRmZnrybRhJF/HzYEeIiIjaMI8GNxcuXMCkSZNQUFCA0NBQ9O3bF2vWrMFNN90EACgoKEBubq6xfV1dHWbOnIn8/Hz4+/ujd+/eWLVqFcaPH++ptyAhSAqKGd0QERF5gkJoY+MnZWVlCA0NRWlpqWQhQGcY+d4GnLlUBQCYcG0cPnlgoFPvT0RE1FY15fu7xdXctGZ6Zm6IiIg8jsGNE0kLij3YESIiojaMwY0TseaGiIjI8xjcOBFnSxEREXkegxsn4jo3REREnsfgxolYUExEROR5DG6cSJDsCk5ERESewODGibgrOBERkecxuHEi1twQERF5HoMbJ9LrxbOlGNwQERF5AoMbJ5Ksc6P3XD+IiIjaMgY3TiRIHjNzQ0RE5AkMbpxIPBS14/Rl6FhVTERE5HYMbpzItM7my605HuoJERFR28XgxolMEzXf7cnzTEeIiIjaMAY3TmQ6/buqVuuhnhAREbVdDG6cyDRzU1mn80xHiIiI2jAGN05kWnNTVcfMDRERkbsxuHESQRBgum5fvY6zpYiIiNyNwY2TcEFiIiKiloHBjZNwuwUiIqKWgcGNk3C9PiIiopaBwY2TMHNDRETUMjC4cRLGNkRERC0DgxsnYeaGiIioZWBw4yQMboiIiFoGBjdOwoJiIiKiloHBjZOY7itFREREnsHgxkmYuSEiImoZGNw4iaXMjZ5RDxERkVsxuHESSzGMlsENERGRWzG4cRKLmRvW4hAREbkVgxsniQzSYMfs0XhrYh/JcWZuiIiI3IvBjZOolArEhvohOlgjOa7TMbghIiJyJwY3TqYzydRo9XoP9YSIiKht8mhwM3/+fPTt2xchISEICQlBamoqfv75Z6vXbNq0CcnJyfDz80OXLl2wYMECN/XWPu1MMzesuSEiInIrjwY3HTt2xJw5c7Bnzx7s2bMHN954I26//XYcPnxYtn1OTg7Gjx+PYcOGISsrCy+++CKmTZuGzMxMN/fcspTOEXhxfA/jc9NMDhEREbmWQmhhS+tGRETgvffew1//+lezc88//zxWrlyJo0ePGo9NmTIF+/fvx/bt2+26f1lZGUJDQ1FaWoqQkBCn9dtUj1d+Rk29HltmjUJ8RIDLXoeIiKgtaMr3d4upudHpdFi+fDkqKyuRmpoq22b79u1IS0uTHBs7diz27NmD+vp62Wtqa2tRVlYm+XEHtbLho+VUcCIiIvfyeHBz8OBBBAUFQaPRYMqUKfj+++/Rq1cv2baFhYWIiYmRHIuJiYFWq0VxcbHsNRkZGQgNDTX+xMfHO/09yFEpFQA4FZyIiMjdPB7cJCUlITs7Gzt27MCTTz6Jhx56CEeOHLHYXqFQSJ4bRtVMjxvMnj0bpaWlxp+8vDzndd4KQ3DDmhsiIiL3Unu6A76+vujWrRsAICUlBbt378aHH36Izz77zKxtbGwsCgsLJceKioqgVqsRGRkpe3+NRgONRiN7zpWMmRuuc0NERORWHs/cmBIEAbW1tbLnUlNTsW7dOsmxtWvXIiUlBT4+Pu7ont3UV4Mb1twQERG5l0eDmxdffBFbtmzBmTNncPDgQbz00kvYuHEjHnjgAQANQ0qTJ082tp8yZQrOnj2L9PR0HD16FF9++SUWLVqEmTNneuotWMSaGyIiIs/w6LDUhQsXMGnSJBQUFCA0NBR9+/bFmjVrcNNNNwEACgoKkJuba2yfmJiI1atX49lnn8Unn3yC9u3bY968ebjzzjs99RYsaqy54QrFRERE7uTR4GbRokVWzy9evNjs2IgRI7Bv3z4X9ch5GoMbD3eEiIiojWlxNTfeQm0clmJ0Q0RE5E4MblxEqeBUcCIiIk9gcOMiahULiomIiDyBwY2LqAzbLzC4ISIicisGNy6i5lRwIiIij2Bw4yIq1twQERF5BIMbF+EifkRERJ7B4MZFDAXFrLkhIiJyLwY3LmKYCs7MDRERkXsxuHERNbdfICIi8ggGNy7C7ReIiIg8g8GNixhqbpi5ISIici8GNy7CmhsiIiLPYHDjIo01NwxuiIiI3InBjYsYtl9gcENEROReDG5cxJC5yfj5GGq1Og/3hoiIqO1gcOMiyqvBDQDM33jKgz0hIiJqWxjcuIhaFNwcOFfqwZ4QERG1LQxuXEQlCm4qarUe7AkREVHbwuDGRcTBTVUdgxsiIiJ3YXDjIuJhqapaFhQTERG5C4MbFxFnbk4XV6K0qt6DvSEiImo7GNy4iDhzAwBLtp/xTEeIiIjaGAY3LqI0CW72nyvxTEeIiIjaGAY3LmKauSkorfFQT4iIiNoWBjcuYth+waC8hjOmiIiI3IHBjYuoTD5ZrnVDRETkHgxuXMQ8c1MPQeAmmkRERK7G4MZFVNKSG9TrBNRq9Z7pDBERURvC4MaNWHdDRETkegxuXEQ8AGVY0K+8hgv5ERERuRqDGxfRi6Kb8AAfACwqJiIicgcGNy4iLh5uF+wHADhfwrVuiIiIXI3BjRv0jw8FAGTlXfFwT4iIiLwfgxsXEc/67h8fBgA4cr7MM50hIiJqQzwa3GRkZGDQoEEIDg5GdHQ0Jk6ciOPHj1u9ZuPGjVAoFGY/x44dc1Ov7aMXRTcxIQ3DUpcr65CdV4L1Ry54qltERERez6PBzaZNm/D0009jx44dWLduHbRaLdLS0lBZWWnz2uPHj6OgoMD40717dzf02H7i2VJhAb4AgJKqekz85Hc8umQPTl4o90zHiIiIvJzaky++Zs0ayfOvvvoK0dHR2Lt3L4YPH2712ujoaISFhbmwd80jHpYK82+YLVVSVWc8dqSgDN1jgt3dLfJi/9pxFvHh/hiZFO3prhAReVSLqrkpLS0FAERERNhsO2DAAMTFxWH06NHYsGGDxXa1tbUoKyuT/LiDeFgq/GrmprJOZzxWUsU1b8h59ueV4JUfDuHhr3Z7uitERB7XYoIbQRCQnp6OoUOHok+fPhbbxcXFYeHChcjMzMSKFSuQlJSE0aNHY/PmzbLtMzIyEBoaavyJj4931VuwKNhPDYXJdgxXRFkcouY6X1Lt6S4QEbUYHh2WEps6dSoOHDiArVu3Wm2XlJSEpKQk4/PU1FTk5eXh/ffflx3Kmj17NtLT043Py8rK3BLgGFYlBgClUoFQfx9JtoaZG3ImbslKRNSoRWRunnnmGaxcuRIbNmxAx44dm3z94MGDcfLkSdlzGo0GISEhkh93uO+6BCRGBeKJEV0AAIlRgZLzZdX1KKmqw/JdudyWgYiIyIk8mrkRBAHPPPMMvv/+e2zcuBGJiYkO3ScrKwtxcXFO7l3zhPr7YMPMkcbnt/Rtj6zcEuPz8lotZq84iJ8PFeLXY0X4fHKK+ztJXkNcwK7XC1AqFZYbExF5OY8GN08//TSWLVuGH3/8EcHBwSgsLAQAhIaGwt/fH0DDsFJ+fj6WLFkCAJg7dy46d+6M3r17o66uDkuXLkVmZiYyMzM99j7sEROikTwvr6nHjtOXAQDrjlxAdl6JcbE/oqYSRANT9Xo9NEqVB3tDRORZHh2Wmj9/PkpLSzFy5EjExcUZf7777jtjm4KCAuTm5hqf19XVYebMmejbty+GDRuGrVu3YtWqVbjjjjs88RbsplFLv2wuVUgLiid+8rs7u0NeRpy50eqcW4FTVFaDd9ccw7krVU69LxGRq3h8WMqWxYsXS57PmjULs2bNclGPXEejlsaRJ4sqzNrU6/TwUbWIMihqZcT/Jzk7uHli6V5k5Zbgl8OF+PW5kU69NxGRK/Cb1E181bY/6vIarRt6Qt5I/A+Fer3eqfc21Iqdumh75XAiopaAwY2bmGZu5JRV2zdrSqd3zcTfOT8fw6RFO1Gvk3457j17GTOWZ6GorMbq9fqr/Xp3zTF8seU0zhRX4ostp1EtWryQXKNelK1xduaGiKi1aTHr3Hg705obOZcq69DZZMq4qR+z8/F85gF8dN9A3NQrxlndAwAs2HQKALD1j2KMEi3hf+f87QAavkA/eWCg5Bq9XkB1vQ6PLN6N4vJazLtvAD7d2HCft1YdBQBo9QKmjOjq1L6SlDggNQ1OiYjaGmZu3ETjY/ujvnP+Nkz7Ngt6vYBlO3PxR1Hj5prVdTr8kJWP6cuzUVOvx/OZB5zav1ptY3bFUi3UkYKGrStKq+qN2aOHvtqFfq+vxa6cyzhdXIlPNvxhft1592x50RIJgoDPN5/G/I2nMH/jKdTU6yAIArLzSlBR67xhSLngpk6rx4JNp3CssO1+/kTUNjFz4ya+dhYKr9x/HnpBwE8HCgAAZ+ZMAADM+fkovt5+1tjO36cxE1RVp4WfWiVZ2+TI+TJMW56FGWO645a+7Y3HD5wrQYifD44WlOHfe/Lwf/f0R0SgL0pFKyYrRXtFiAMdpQLIKa5E2gebMDIpGp9PTsGWk8WS/v98qNDsPXUM97frvZvS6QUs352L6zpHtLpNRnOKK6FSKFBUXoO3Vx81HtcLArq2C8SUpfswMCEMK566ocn3rq7Twd9Xmgms0zYGN9qrgefnW07jvV+OY87Px4x/jprDdAsRIqKWipkbN7Enc2NgCGwMSqvqJYENAFyqrEVRWQ3mbzyFXq/+gie/2YuV+89jxvIs1NTr8Pr/DuOPogpMXZZlvCYr9wpu+/h33Pf5Djz5zT5sOH4R7/1yHABwRRTc5BRXYvaKA8gprsSlysYp6wqFAr8cLkS9TsC6IxcwY3njva2pqW/44i2vqZfNCp27UoV31xwz1vQYMg+Ze8/hpe8P4aYP5PcNa6rNJy7ihjm/YcvJi8ZjK/adw6RFO3GxvNasvSAIds3oM7Q1qKrTYtT7GzH8vQ04e0k6fXp/Xgne/Kkh2NknWtTRlE4v4O8rD2PR1hyMfG8D5q4/AQDYdqoYPV9dg7nrT+BQfinuW7gD2Xklkpqb4opa7Mu9gj1nLpv1ccq/9uK+hTuM9VFNYU/dmKOuVNbZ/VkTEdnCzI2b2FNzI+dyZR0GvrnO7HhNvR7XvfOr8fkvhy/gl8MXAAD94sMkM68EQUCtVo/7P98JACgobSwM/vlQAa5LDMeH6xu3r3j9f0cAAN/uysPMtGuMx/8oqpC0+yH7vF3vobJWi5MXyjF+3hb4+agwuEskqut0eHx4Fwy/ph0eW7IXRwvKsPfsFYzqEY2560/gTwM64ttdubL3e+7f+1FRW487B3aEr1qJxKhAdIoMhFanh1qUIfvnuhM4eK4En01Kga9aiXdWH0V+STUmLdqF9ekjUKfVI/3f+wEAX287g6k3dkNpdT1iQvwAAA8u2olLFXVYOXUofNVK1NTrcP/nO9AhPADv3dUXfj4qXKmsw8j3N6K0uh6v3doL91+fgMx9+cY+XK6Urme09sgFuz6zad9mYdXBxiB37vqT6BEbjClL9xmfL9l+Fpcr63DPgu2YemM3Y9sZy7NRJBOslddqseZwQ2btfGk1OoYH2NUXA0f/DNvyy+FCPPGvvfjLDYl49dZeLnkNImpbGNy4iT3/6u0eHWS2/o1cYGNLYWmNsT4GaKjluVxZh+p681lLJVX1ePa7/Rbv9f7aE5Lncvew5bs9eTh2oRz1OgH1Oi3WXf2C3/pHMXq3D8HRq33dmXMZO3Masg2mgc3nm0/jr0MTUV6rRea+cwBgDOYAYM4d1+K9X46je0wQHh6SiMFdIjDv14ZA7IesfAzsFIZgv8Y/7mP+uUly/2W7cvF9Vj6Kymvwa/pIRIdo8PsflwAAe85eRmqXSDy2ZA/25ZZgX24JhnWLwj2D4vG/A+dRenWW25s/HcHZS1VYvO2M8b6ni61Pn67V6qBRq1BWUw+9XkBYgC/OFFdKAhsDQ2BjYAic6nR6Sc2NXGADABdEQa0jSRJ7ljNwxDtXh+2+/D2HwQ0ROQWDGzexFdwE+qqQFBssu7hfU322+bTkubXhD3sFadTNKoDdnyffh8N2Fhu/vfooQgN80Lu9/ManL6w4CAC4dPoydpy+jHbBjdtdzLKj+FqcYVm09TR6iV7neGE5Qvx8JPVF+8+V4J5B8ZJhpwBftSSwAcyDNFOj/28T3ry9D2Z8l43S6nr83939EBHka7O/pursmCElztjVaps+o8reurGm0nM4ioicjDU3bqKwUY05eUhnPDqsi+y5od2i8PCQzk7px0vje6KLjenmcr7+yyCr558RDYu4yrojF5B/pdqutnI1NPb6evtZPJ950Ph8/sZTOFZYLmlzpKAMfxRVYNHWHOMxR4K/c1eq8cji3cbsz3P/2Y/cS03f5uDHLOtDhCcvlCNPtH2CYXbc7jOXMe3bLBSVW1/DCGha3Zgler2ATzb8gR2nLxmPMbYhImdjcNMCfPjn/nh2zDW4tkOo2bm0XjFY+uj1+PttvWWvndA3Dr89N0L2nGlA1C5Yg3sGxePulPgm97Fbu2Dc0C3S+Hxcn1jj4xA/NZ5LS8LLE3piXJ9YfHL/QLlbNNupogrkl9gX3DRFapdIq+eLymvxY3a+5FhWbonZ0JazvLbycJOvKbSxwOJNH2zGS98fMj43zK66e8F2rNx/Hq/+IP+a4uEuPyfU3Kw6WID3fjmOPy/cYTzG4IaInI3BTQtwU68Y+KqVUCnNszvBfj7GxyF+5qOISoUCXdoFoW9HaWAUH+GPaaO7S46tmT4Mof4+uCelI8IDfHBL3ziLfXp2zDXY9dJoAECHMH+E+KvRt2OY8XynyMbsT9nV4uVHh3XB/AeTMf7aWKhl3ktznS6uxCrRTLLYED/sfzUNPWKbN0186o3dkNolEs+OucZiG8OQ1OsWgkxLw2UAEKxR45/39GtWH53NdFjqRFG5bLsrouG6IJk/f0119hK3cCAi12Nw40b/vKcfHhuWiBVPDUHXdoG4rnMEnr+5BwJ8G780/v1EKh4dmmh8HuLfeO6bRwfjusQIrHhqiPFYTnFDjc4XD6VIVixe+fRQRAT6Gr90Hx7SGZFBDXUokUEa7HxxDD66bwA6RcrPmOnTIQTRwX7Y9eJorJo2FAqFAgMTwo3nA3xVFtc9USgU2D57NN7+Ux+8dmsvDOrceF1Kp3D5iwB8/9QQPH9zD+Pz12/rjeNv3Sw5tufsFQDAWxP7YF36cIQG+OCDe/tbvKc9esWF4NvHB2P6mO5W2ykUDRmrqCCN5PjDQzpbXYFZQMPnJTbhWsuBpcFjwxJttnGUaXBTW6+XLBppcLGicXjPGVO1VUrzv3JYc0NEzsbgxo3uGNgRL03ohYEJ4fj1uZH495RUPDlS+qV4XWIEXr6lF+5K7ogAX5WkDufajqH49xOpGJgQjrG9GwKZyYM7AwCig/2wcFIyMp8cgv2vpiE8sKEodfnjg/HqLb3MamJ81UooFApjIJUQESAJmnrGNQRF0SF+CAtouFeXdo3ZGn8fFUZc087ie20XrMED13fCIzckIrVrlPG44V5AQ1BxV3JH+KgUSL/pGgxICEePuMYsTHKncGjUKjw5sis2zBwpuf+gzhHGrFbPuBBMH90dGrUSmU82vofb+rXH8scHY9W0oRb72dCnxuxYRKDlYl4fpRLRIX749IGBkiG/EH8fRFq57sHBncx2e7enAHjENdE22zjqx+x8pGY0LiWQX1KNMf/cjA3HigAANfU6LN1xFgfOlRrbaJ2wp5lcTTKDGyJyNs6WaqHeu6sv3prYB34+8nUOc+8dgCMFpRgQ35gJUSgUSDbJjAT7+eAvQy1nAB4c3AmhAb5I7hSODmH+WPHUEJRW1aN9mPmqwu1DG4/V6/V4966+eOWHQ5h0NcCypE6SJWj8Ils9fRgA4J0/XWucZhwV2JgV6RXXONSTGBWIR4cm4outOVApFegeHSR5jRljumP66O6SVZqjgzUYfLWeZvEjgzDju2zc3q+9cUHE3u1DcH1ipKTY+62JffDUN41Trq9PjECdTo+s3BLcf30CgIYA9LrECESHaPC//QV4eEhn2YLcyamdcH1iJEb3jDYLbuzZTLRfvHSoMdhP7bSd41fsy5c9vnTHWYzqEY35G0/hw19PSs45svCfKaVMus80tvlw/UmoVQo8Pcr1RepE5J0Y3LRQCoXCYmADAP6+KiR3inDK69zWr3F7BvHQk9xrGpRU1SM62A+fTUqx+Rri7RfkvpzF66f06RCCf9x5LbpFB0kCFQB45sbuUCiAewclmJ1TKBTGYbLHh3fByuzzeHxEY9ZrZFI0sl65CTX1evy4/zziQv2xatows76M6xOL354bgchADb7POofx18ZBrVJixb5zuO+6BEnbp0Z2w1MjG76AtTKZmFFJ0RjVozH7cvj1sXj2u2zc3CcWK/fbXgBRXG8FNMyak9vewpl+PVaEBZtOYWfOJbNz+8+Vol6nNwvUmkKurkwcMxVX1OKDq6sx/+WGRLNtJoiI7MHghhzibyXwMnVPSjxOX6zEiKR2qKjRYmfOZYxMkh/SUigUuHdQguy50AAfvDTB9iJvL47vidnjephNv1coFPD3VWH7C6Nlv2QNbbq0a8gKPXxDY8bL0jR9A/Fwm8FAkyxaoEaNhZMbgsFrO4TieGG5ZO0ZW8ZdG2cMbv5+ay/8/epK0s425+djktlwYu+sPorXbpUvqraHuNBcrxeuBqmN0Y04y6fjcBUROYg1N9QkH903AGN6xlgd6jLlq1bi1Vt7YcQ17TD+2lj8PH0YFjyY7MJeWl9XyN9X5fTVdsX3y3wyFZv+NhKh/j4W23ePCcb22aMxObVTw/PoIHx8/wCzdoZanntT4iVFyKEBPtjz8hiz9rf1a4/dL5kfbypxHZLYV7+fadZ9xRk3Q92RpRiGtThE5ChmbqhJbu3XHreKhrGaSqFQGIuVvc2WWaNQWl2PPjLrFVky6+Ye6BIViJv7xCE21A9nL1XhvV+O4727+gIAfp4xDIfPl2FE93aSwEAQYDZr650/XWusC9r8t1HIPleCvh1C8ZfFu21uA2EqxEpg1hzimps6nR5+PiqLQYxOx+CGiBzD4IbISeIjAtDU5RGDNGrJ8NdTI7viruSOxs07o4P9EJ3kZ9e97ruu8dUTIgOQcHWa/28zR6LnK2sc2hfM2cSjgYYhKHEIIw50nDE7i4jaJg5LEbUgCoXCGNhY0y8+DAAw997+8FUpseDBZKtDcaYzr2wR175Yqk9yhDheMQY3omN6vbgtgxsicgwzN0StyM4XR+NieS26Xi16njigA8ZfG2ezhuif9/THnJ+P4WJ5LbafNp8JZcqw7cL00d1xU68Y3PLRVuO5owVlDg8tirMxhteQZmv0sm2JiJqCmRuiViQmxM+spsee4uj2Yf6Yd98AjBGtYm2NIaviq1ZCrZJmbsZ9uEV26rs9xGvlGLND4syNKNCxp+Ym73IVTlyQ3zqCiNouBjdEbUh8uPnijHKKru6q7qtSyu4T1tQCZQBYuf+8cfsMoHELCHEII87WiLM4lgx7dwPSPtiMSxWO7wJPRN6HwQ1RGxIf0biXWIcwf/xpQAfZdhuPXwTQkLmRW1X4oGhbBnscOFeCad9m4X+ixQvr5IalRNmaptTc5F6ualJ/iMi7seaGqA0RBzfr00fA31eF77Pkt2IArg5LyWx2WVhm/+KDAJAjk+mRKyg+I9o1nDU3ROQoBjdEbUiQRo1lj14PAbBrawMflRIqlXnmpk7rWM2N3D3EGZqpy7KMj7U2am6csdcVEXknBjdEbcyQblG2G13VkLkxD27qZQqKBUFAvU6we/VnucyNmM5G8CLenoFhDhGJseaGiAA0bvUg5qtSyNbcyGVupi7LwuCMX1FSVWc8lne5CgWl1bIBjHH7BQuhSZ1Oj0e+2oV/rj0ue57r4BCRJczcEBEA+SnlljI3daLMTWl1PWb+Zz/WHbkAAPjlcCHuHZSA6jodhr27AQAwpGuk+T1sZG7WH7mADccvYsPxi0hPSzI7b8dkKiJqo5i5ISIAMFvPBrBccyMelvpg3QljYGO4BgCKRdOzt50yXzhQbraUWEWt1mp/uWs4EVniUHDz9ddfY9WqVcbns2bNQlhYGIYMGYKzZ886rXNE5D4+MrOiaurl0yO1omGpUxcrJOcMGRlbe1nJ7S0lZqug2FZNDhG1XQ4FN++88w78/RsWA9u+fTs+/vhjvPvuu4iKisKzzz7r1A4SkXvIZW4qa7UI9DUfva7T6pF3uQr/3XsOJVX1knMl1Q3Py2vqza4zvQdgeViq3sa4k8DMDRFZ4FDNTV5eHrp16wYA+OGHH3DXXXfh8ccfxw033ICRI0c6s39E5CYqmcxN//gwqJQKfHz/AMk07XqdHmPnbkZVnXl25sfs87j/+gSU11gfVqqzsYVDUzI3jHOISMyhzE1QUBAuXWoYQ1+7di3GjBkDAPDz80N1dbXzekdEbuNjkrm5tV97dI4KBACE+vtIztVp9bKBDdCwseZfvtptO7ixsVaOre0XJFPBbUQ3F8tr7VoXJ6e4Ekt3nJWd6k5ErYdDwc1NN92ERx99FI8++ihOnDiBCRMmAAAOHz6Mzp07O7N/ROQmKqUCg7tEGJ8nxQRJzonZyrrsOXul2cFNvc568LLmUKHxsbX6m105lzHo7fV4/F97rL4eAIx6fyNe/uEQFv9+xmZbImq5HApuPvnkE6SmpuLixYvIzMxEZGTDNM+9e/fivvvus/s+GRkZGDRoEIKDgxEdHY2JEyfi+HH5NS3ENm3ahOTkZPj5+aFLly5YsGCBI2+DiAAkxQQDAO4Y0AFL/3q98Xhyp8ZAx3QLBntWKLZZc2NzWKrxvFzw8uqPhxvPW8ncLNp6GgCw/miR1dcT23Xmst1tiajlcajmJiwsDB9//LHZ8ddff71J99m0aROefvppDBo0CFqtFi+99BLS0tJw5MgRBAYGyl6Tk5OD8ePH47HHHsPSpUvx+++/46mnnkK7du1w5513OvJ2iNq0fz+RigP5JbihaxSUSgU2zhyJ08UVSBWtTWOauSmtth64ALanctsKkM5daRzi1gmC1b+srI1gye2NZQtreIhaN4eCmzVr1iAoKAhDhw4F0JDJ+fzzz9GrVy988sknCA8Pt/s+Yl999RWio6Oxd+9eDB8+XPaaBQsWICEhAXPnzgUA9OzZE3v27MH7778vG9zU1taitrZxvY2ysjK7+kbUVoQG+GBY93bG552jAo21NgamC/ldKKuFLX8UVVg9f/ZSJWavOGDx/EnR9abBi2n9jLXVipUyixDaxuiGqDVzaFjqb3/7mzFIOHjwIJ577jmMHz8ep0+fRnp6usOdKS0tBQBERERYbLN9+3akpaVJjo0dOxZ79uxBfb35vyYzMjIQGhpq/ImPj3e4f0RtlT2Zm3fv6it5/usx68NAG45fxLe78ux6/eMXyvGv7WeMQ1WmQ1qGYan1Ry5g71npkJLMDHebmLkhat0cytzk5OSgV69eAIDMzEzccssteOedd7Bv3z6MHz/eoY4IgoD09HQMHToUffr0sdiusLAQMTExkmMxMTHQarUoLi5GXFyc5Nzs2bMlAVdZWRkDHKImklsDx5TcjCoAiArSSFYrdsTET34H0FBk/Jehiag1WVxQrxeQd7kKjy5pKBo+M2eC8ZwjmRvGNkStm0OZG19fX1RVVQEA1q9fb8ykREREODzsM3XqVBw4cADffvutzbYKk438DDMpTI8DgEajQUhIiOSHiJpGbn8pU6bBDQBEBPoiKsh8Q05HbT/dsARFrVY6DV2nF5BfIr8MhUrm7wWxilotamyspkxErYtDwc3QoUORnp6ON998E7t27TJOBT9x4gQ6duzY5Ps988wzWLlyJTZs2GDz+tjYWBQWFkqOFRUVQa1WG2dtEZFzyS3wZyoswDy4iQ/3l92Q01GGIKTWpBhZLwiwFMKYDqkBQGlVPf614yzyLlehz2u/YMw/N0nOt4TVj6vqtPjt2AUGXkQOcOhvnY8//hhqtRr//e9/MX/+fHTo0AEA8PPPP+Pmm2+2+z6CIGDq1KlYsWIFfvvtNyQmJtq8JjU1FevWrZMcW7t2LVJSUuDjY/6XKxE1nz2ZmzB/8wxNx4gA40aazmAYjjLP3Egzt+KCY7ngZsZ3WXjlh0PGXcvPXamWLNzn+dAGmPZtNv6yeA/e+OmIp7tC1Oo49LdOQkICfvrpJ+zfvx9//etfjcc/+OADzJs3z+77PP3001i6dCmWLVuG4OBgFBYWorCwULLK8ezZszF58mTj8ylTpuDs2bNIT0/H0aNH8eWXX2LRokWYOXOmI2+FiOwgHtl5eUJP2fNBfuYlfPHhAfB1ZnBzNagx3dDTdJ0brY3gZsPxi2bHBr/zq/GxrcWMdXoBf128G3N+Pmazz45af7Rhp/VlO3Nd9hpE3srhv3V0Oh0yMzPx1ltv4e2338aKFSug0zUtfTp//nyUlpZi5MiRiIuLM/589913xjYFBQXIzW38nzsxMRGrV6/Gxo0b0b9/f7z55puYN28e17ghciFx7HDvIPOCfB+lEv4+KrPj8RH+8HHqsJQhc2NeUCwOwMSL/ilt1NwYXKqss7sfu89cxq/HirBg0ym7ryEi93FottQff/yB8ePHIz8/H0lJSRAEASdOnEB8fDxWrVqFrl272nUfe8a1Fy9ebHZsxIgR2LdvX1O7TURO4KNS4m9jk/DeL42riQsQZDMkDZkbR9aZkVejNdTcSP8hZVpz07AvVUOwJe7X2UuV6BQpv0CoWFNqbvR6wWxG1u4zl3HuShX+NKDpNYim5D5XIrLOoX9STZs2DV27dkVeXh727duHrKws5ObmIjExEdOmTXN2H4nIwzqG+2NY9yiM7R0DPx8Vnh7VDcffaqyvszSM0y5Yg52nzbcyCPA1z/LYw1JBcWFZDT7fctr4XGdhWGrEexuRe6nKodcW8xNlqaplCn7vXrAdz363H4fyS83OWZvZJYehDVHTOZS52bRpE3bs2CFZbC8yMhJz5szBDTfc4LTOEVHLoFAo8C/RvlMAoFE3fsEbVgh+cXwPHMovQ0SgLy5X1qFHbDDKZbZh6BDmL1mB2F6GYSnTrRveXSPdk65eJ+DzzafRp0Oo2bDUjpxLTX5dU+Id1CvrtAjUyP9Ver6kGn06hEqOPfPtPqw+WIgFDybj5j6xNl/L3mE1ImrkUHCj0WhQXl5udryiogK+vs5b04KIWgfDKM7jw82HpO9Nicd3e6QrEcsVHxtMTu2E/Xkl2H9OPusBmGduTO09ewVvrz4KAHhwcILk3Kz/Wt7ywcDwfs5eqkT7MH+zGV/iUauqWh0QLH8fuZliqw82LGXx2eZT9gU3zitZImozHPrf5pZbbsHjjz+OnTt3QhAECIKAHTt2YMqUKbjtttuc3UciasVeu62X2bFgP8vLNky4Ng5/GtBB9ly9To9jhWWY9m2W1dcsKq8xPl66o+mzjfSCgF8OF2LEexvx16/3mJ0XD3tV1kkzU+LdzK1Ng7c3H8PMDVHTORTczJs3D127dkVqair8/Pzg5+eHIUOGoFu3bsYNLYmIACDAV43OkQGSY4FWam7UKgUS2wXJntPqBdw9f7vN1yytsr1ruTXbTl3CpxsbZkJtPmE+bVw89byqTlpzI973Sq4exx7VontW1elwpQkzuYjIweAmLCwMP/74I06cOIH//ve/+M9//oMTJ07g+++/R1hYmJO7SEQtXWKU9RlI/31yiOS52kpGQ6VUoouF++n0gmwNj6krzQxuAGB/XonFc+JFAh/5ajf+teOs8bl436vHluzBh+tPyt5DvOhgdZ0OhaWN2aYXvz8oaTvzP/vt7jcRNaHmxtZu3xs3bjQ+/uc//+lwh4io9UnpFG71fFSQBgkRAci93DBTaUzPaPxv/3nZtmqlAu3D/JvVn5Jq12U6Fmw6hQPnSozPK2q1eOWHQ5g0uBMA83qgD9afwPQx3a3e87Ele7D1j2JkPpmK5E4R+D4rX3J+6x/Fzuk8URthd3CTlWV9jNtAbvNKIvJO793VF5n7zuHF8earFpt6YVwPPPXNPlyXGIHb+rVHkEYtW8+iUiqavbZLiRMyN3LyLlfZXJXYdA0eS8Tv0BC8vPT9IayZMdysrc7WkslEJGF3cLNhwwZX9oOIWqG7U+Jxd4r5isVyxl8bhy2zRiE80BcKhQKje8bItrNnHytbrlS5JnNjz/o0tmZyWWNanGygZXBD1CScZEhEbhMfEYAg0ZowcnGMIWvz8/RhDr9OcwuKLblsR2Fvbb19wY1ckpszo4icg8ENEXmMXGGx+urCLj3jQhy+72UnZ25+OtBQH3SxvNZmW3uHpeSoGNwQOQWDGyLyGB+Z1I3Kjr2ofntuBPx8LP/15eyam6nLsnD4fCkulNXYbGvvsFR1vc5sDyvTPaqIyDEMbojIY+QKh+UCHlP+vioo3Lzr0umLlWZr2siRy9zU6/TIyr0iWeDvUH4ZHluy12QHc+f0laitc2j7BSIiZ5BbwbddsMbmdf4+KtmaFbGoIF8UVzhveKpep5cs0GeJXM3N26uOYvG2M3h6lHR7ivVHL6BedE/W3BA5BzM3ROQxapMhqISIANnlJDqE+SNZtJZOQ+bGukGdI2y0aJryGi0u2xEsyQ1LLd52BgDwyYZTZufEAVNzp8ATUQNmbojIY9SiXSHfuL03bu4tv5GkWqWQ7MTtq1LaXFNrUOcIJMUGY66FFYKb6rWVh+1qZ7pjuS31WmZuiJyNmRsi8hhx5mZyamdEh/jJtlMpFZIhLIXCdsVNdIgGt/Zr74xu2uWez7Zj8e85svtJRQX5Wr3OQC+4dz2bgtJqZKw+inNXqtz6ukSuxuCGiDzG3mEYtUlwA8ivEyMW4ucDH6X7/orblXMZf//fEZRWm8/UahcsH7QBwKmLlcbH7l6J+PEle/HZ5tOYtGiXW1+XyNUY3BCRx9gbfKiUSsmwFGB7q5cQfx+7ppU7m9w+UOEBPnZdqxcESYGxqx3MLwUA5BRX2mhJ1LowuCEijzEtKDZlmF306i29zBb8s525Uds1rdzZ8i6bD/FodfZlZLR6AVW1lqebi3cj/2DdCdzx6e+otjA9fcOxIhw+X2rX6xJ5GwY3ROQxY67uLxUVJD/9+29je+DoGzcjtWskAn1VTbp3iL+PZNjrwcEJeGpkVytXOEehzEJ/tXZmY/R6ARUW9pe6UlmHwRm/4pUfDgEAPvz1JPbllmDl/nyzticulOORxbsxYd7WJvScyHswuCEij3l6VDe8f3c//O+ZGyy28b8a1Dx70zVoH+qHmWnXAGgoQLYm2E8tyfZ0axckW+zrbHI1wbV2vq5OEFBVKx/cLNuVi6LyWvxrx1msO3LBeFyuTOd4YbnZsaKyGuw8fcmufhC1dgxuiMhjfNVK3JXcEXGh/jbbxoX6Y9vs0Zh6Y3cAwLQbuyGtl/zO4gCgUaskdToCgDsHdmx2nx1hz+J/AKDTCai0MMwkHoZ7bMke42O5omyt3vz1Bmf8insX7sC2U+Y1QUTehsENEbVKapUSw65pZ7WN+ItfLwB9OoRizQzHdxt3lKW6GFPnS2uwaGuO7DlLk99n/fcAKkyyPfUyNT6GDM+WkwxuyPsxuCGiVstSwfDCSclXzzf+FWfYpFLtgSLjihr5oSY5/9t/Xva4TiYbY7D2cCH0esE4lVw8pdx0c043L6VD5BEMboio1ZLbm+qB6xOQdnWlY6Ukc2MIbtz/156lIuGmqLMy40qrE3Dngm0Y9+Fm6PSCZINOrd40uGl4bm82iag14vYLRNRqyU0lD7Awq8rwHW9r+rkrOJIt8VUrJVs5aK3U7ZRU1yErtwQAcL6kWhLQaHUCfEQfiYCGaeKPLN7d9E4RtRLM3BBRq+Urk7lRWhh2MmRu5LI9jjJdWNCZQvyk//Y0zcCIiXc/1+oFybo69SbDWXq9gOczDzipl0QtE4MbImq1TBf2AwCVhdX9DNkTZ9bcdG0X5LR7mQrSSIMbaysXF4nW1qms1ZplbsQE2P8ZXCirwTaZFZeJWjoGN0TUasllTiztV2VY3de05qZ9qOV9n2xx5UaXAb7S4Oar389YbFtUXmt8nHe5CsUVjc9Nh7P0gmAxu2Xq3s+24/4vdkrW1SFqDVhzQ0StltwQk8XgxkLNzZK/Xo8/iiqQue9ck7/EXbnPpX8TVmS+KApunvxmn+RcvWgWFdCQwbJ3w9Izlxq2kvhyaw5usrKmEFFLw8wNEbVacsMrFoelcDVzYxLchPr74OY+sRYLka0xnWbdLdr6MFWXqEC77+3vY39/xJkaU1qd3mxIy97gxuDAuZImtSfyNAY3RNRq+ajN/wrT+Mj/tWZIXpjuRG6IhZr6hQ+Yz4JKTgjHnpfH4N6UeNn2SbHBdt+7KZmbK1X1Fs9p9QJqRbOuFm87g9MXm7YLuDu2rSByJgY3RNRqmQYqAODvKz/absiyKJUKLHooBQAwuEsEIgN9Ld7LFrmam6ggDVQWZlGFX30tezQlc2ONVidYLUa2B9f9o9aGNTdE1Gr5qM2DCEu7h2tEWZ7RPWOQkzEeAKC4mrpxZP0bSzU3lpJA4QE+dt/bkWEyOfUyw1JNxVWNqbXxaOZm8+bNuPXWW9G+fXsoFAr88MMPVttv3LgRCoXC7OfYsWPu6TARtShyqw2bBgUvT+iJ5E7heGhIZ8lxw98fjfdyYFjKQk7DUt1PeID9mRs/Z2Vu9ALqtc6NTgRBwNPL9uGFZqyXo9cLyPj5KNYcKnRiz4gaeDRzU1lZiX79+uGRRx7BnXfeafd1x48fR0hIiPF5u3bWN88jIu8kt4if6bDUo8O64NFhXWzeS27NHFtMt3syBDsKC8FN9xj7a26clbk5d6XKbM2c5soprsSqAwUAgLcm9nHos1tzuBCfbToNADgzZ4JT+0fk0eBm3LhxGDduXJOvi46ORlhYmPM7REStitxQkqVhKVscWbnYdLaUgaXi5OHdo5DSKRx7zl6xeW9n1dxMXZaFF8f3cMq9gIZp5++sPmp8rhMEh75ICktrbDciclCrLCgeMGAA4uLiMHr0aGzYsMFq29raWpSVlUl+iMg7yAUkTZllJL1X04eldBaCG0sjXAqFAs/edI1d97Y068teof6N9T2fbjzVrHuJPfXNXqw/WmR8bmWzcqtcuQAiUasKbuLi4rBw4UJkZmZixYoVSEpKwujRo7F582aL12RkZCA0NNT4Ex8vP0WTiFofuYDEdGVfezkyFdxiQbGVe9n7Khp1Y5BmaWq5JX07hiI+wr/Jr2mP3WekWSdLAV5TVDlh13QisVYV3CQlJeGxxx7DwIEDkZqaik8//RQTJkzA+++/b/Ga2bNno7S01PiTl5fnxh4TkSvJZW7cOywl/1xpoeYGACpq7fsi9xXN7uoY7m+lpTkflVJSbG2pBkjsUH4pZv5nP4orapsUbLz10xEs25nbpP4B0s/uxvc3Nfl6ImtaVXAjZ/DgwTh58qTF8xqNBiEhIZIfIvIOctkWR4elNDILAlry7JiGoaW3JvaR75eVYKK8xr7AQdyfjhFNDW4UkqyWPUmpWz7aiv/uPYeUt9aj79/XIjuvxK7XWr47Dy9+f7BJ/QOkM80Ky1h/Q87V6oObrKwsxMXFebobROQBGrUSXdpJtzRwdFjqz9cl4JqYIEwZ0dVm2+ljuuPg39Nwc59Y2fPWgok40Uadu14cbbGdOJMUF9r0zI048CuuqGvS9Vq9gImf/N6ka+5esA3zfrX8D01TrtyXi8ijs6UqKirwxx9/GJ/n5OQgOzsbERERSEhIwOzZs5Gfn48lS5YAAObOnYvOnTujd+/eqKurw9KlS5GZmYnMzExPvQUi8iCFQoF1z46AXhAw79eTCPHzcah2BgCCNGqsfXYEAGDBJtsFuMF+lhfkk6u5+fSBgQCA1K6ReOP23ugZF2L1HuKF9wKbGLD5qpSotHP4y1l2n7mC3WeuYNro7na1b0pBcb1OD6VC4fDvtrUqq6lHdZ0OMSGO71zfVnk0uNmzZw9GjRplfJ6eng4AeOihh7B48WIUFBQgN7dxLLeurg4zZ85Efn4+/P390bt3b6xatQrjx493e9+JqGVQKRVQQYHn0pI83RXjQIv4e/vrv1yH6xMjjIvyKRQKTE7tfLWd/Bd8iJ8anUWbbPo1ceaUj0qJ6vrmrUps6t+783DPIOdNyLA3tqnT6nHDP35DVJAGP08f5rTXbw36/n0tAGDfKzchoglbd5CHg5uRI0da/J8bABYvXix5PmvWLMyaNcvFvSIiap7S6saNLId2i7KYcZAr9N0wcyRiQ/xw7kqV8Zh45tSgzuFmM5Y+n5yCgtJqvPrjYQANG4pa+7vVEbMyD6BTZIBT72mPExfKcbG8FhfLLe987u2OFZRhSLcoT3ejVWn1NTdERM52e//2zbr+SlVjjYu9QylPDO+CDTNHIjEqEP6+KnSLDsK9KfF4amRXyR5a4kDH4KZeMQgTbe3go1Lg7T/JFzuLJUYF2mwjdvZSlc02h/JLJcGdJfYGX+LhK30bLdSxtrQAyWNwQ0RkYu69/XHszZsR4udYcvtyZdMKeAGgW3SQJNhQKBT4x119MevmHpJtJiwtNugj+gL0VSmR3CkCr9zSy+prPpd2De4Y2KHJfbXmlo+2YuwHltceM7A3ThG3c8aaOq2FOPhra7VGzsBdwYmITCgUCvj5qBxa+wYASqpsZy4MPpuUjO2nLuFPAywHGT6iaeGW1tBRSwKghsem+1OF+vtIsioqhQKRTajl+M9e+9YJs2dqt/kaQYLsMJ04c6PTC3DSrhQtXr2u8X1bWzeJ5DFzQ0RkgT3/Yv7wz/2NjyODGgKFlM7hAKRbIFgytncs/n5bb6ubT4ozN5YW5BPvs2UIbnQm6ZG+HUMlz1VKRZN2Hzet9WkO09lS4i9zMXEz0/fjzbSifS0Mfw6zcq/gyaV7kXfZ9vBgW8fMDRGRBfZkbm7v3wFqpRIr9+dj6qhuAIDn0pLQMTwAY3vHOL0flv4RrxYFYoYaHdNgwDSQUSkVTVq8sLmW7cyFRq3EnckdYRqm1Ov0klWZl+/KxZWqegy6GigCbWtYShzsGRaF/NOn2wAA565U43/PDPVIv1oLBjdERBbYW+swoW8cJvRtXEw0SKPGX4cmuqQflnok3m7BkOnR2hXcuHacx7DeTq1Wb1zJ+JZ+cWYFxVqTzM0LKxravntXX+OxtlRQrBWtc2Qa0J4prnRzb1ofDksREVmgboGFnKb1F8FXi57FhcaGLSh0Jlt2+5pkotRKZbN3H7dGpxcwft4WDH93g6TIuqZOb1ZzU6eTX5enVFS/ZBqsOVt2XgkeX7IHOU4KHipqtXhn9VHst3MrCzFr77XthHiOY3BDRGSBOGMy4pp2AIA+HTy7P504tukeHYRfZgwHIC0oNqxonBgVJLnWdKaVUtm0PbWaavupSzh7qQqXKuuQsfqo8XhVvVaytxQgrTERZ3XEQY+rMzcTP/kda49cwJNL9zrlfv9cewILN5/G7U3cygKQBjdyxddkHYMbIiIL7r8+AQCQ0ikcc+/tj5cn9MSXDw/yaJ/EmZs7kzuifVjDvlPiLJMhczOmZzQGd4kwHpcbZmtKQXFTrcg6Z3z867Ei4+PqOp3ZF/aS7WeNj8UxTJ22MbhxV83NmUvOydwcLShz+FrxsJRp8TVDG9sY3BARWfBQamf8+4lUfP2X6xAe6ItHh3VBdLCH9/kRxSfi7zzxbClD5kahUODJkd2Mx00LpAXBdZkbrU6PrSeLZc/9UVRhFqjM39i4n5feQuZmytJ9KCitdnJPzTlr6rVpdqopxAXFZsENoxubGNwQEVmgVCpwXWIEAjUtZ+6F+GtX/OUpLigO0DRmY8TBi2nmRi8ILisortXqUVOvkz33+L/2Wh1ikgQ3oszN/rwSzPrvgSb3Je9yFb7dlSu5lzWOhDYHzpVg9oqDuFTRuE1Ec0bRxMN0pvdpTtDkSi1puIzBDRFRKyL+/pBkbkSBS4BoqEk8vdq0QFrvwszNl1tzUFZjeWfyapnA56vfcwBI31e9SaGxtS0gjhaU4Z4F27Er57Lk+Mj3N2L2ioP4fMtpe7pucS0ha277+Hd8uyvXOCMMQLPGj8Szx0yDBsPTkqo6yfCVJ60+WIBBb6/HztOXPN0VAAxuiIhaFUsL2UmGpUSZJvEMKZVSgU1/G2l8rhcEaFxUc/N/605YPV9abR74vP6/I8Z+GZgGN9bijslf7sKuM5dxz2fbJccNn9kOO794mzMqtT+v1Pi4ecNSVjI3AnD2UiX6v7EOd87f5vBrONNT3+xDcUUdHlm829NdAcDghoioVRHXqoj/RS+upxFvu6Axydx0imzcv0oQBLcu4idWUmV5/y3xl3mtyVCSadzx+x/FWLDpFARBcNrO4c2puKmqawzamjcsJd12QkyAgJXZ5wEA+8+VoiUxDUY9peUMJBMRkU16C1OEVTKzpQDpLuIqpTSQaRfkBz8XrnNjTZmVncPFmZsV+/Il50yHjB74YicA4JoY6bR3OfYON1nbhbu8ph7LduZiQt84dAwPMDsvHm5rTg2KOEiQG5ZqqTuFt5SyG2ZuiIhaEUvTocWHxUNRkpqbq0NXXz6cgtdv641rO4a6fIViSyzV4wiCAMHKP/5ziiux92xDTU1ReeMGneeu2J5FZW84YK3d26uOIuPnY7jtY/m1a6SznKTnLpbX4nyJfbO9xNka84LilruZZguJbZi5ISJqTaKCNMbH4i+SiEBfdIlqGHIKD2jc6dtXZtjpxh6Ne155aliq1ELmpqJWa3M9mzvnb8cPT9+AiaLF8ez5srfUxHRHcmv32nm1WFm84rIlpu9i0NvrAQAH/56GYD/rm6qKC4rnrDmK649HmvTX5st7REuZMcXMDRFRK/D55BSM6RmNF8b1MB4zHZZa++xwrEsfIRmyEAc3cl884YG+ZsfcwVLNTWl1Pe5buMPm9ct2npU8tyu4kTm27VQx+r+xDiv3n29sJ9Owuk6HTzb8gXw7My8AJL8g8aym8yU1cq0lxMNSh/LLsGhrTuNtAbTQUakWg5kbIqJW4KZeMbipl3SXcdPZOGqZXczFmRm5Wk97dj53BUvFtmXVWhy/UG7zetN+i5+abjNhsOH4Rew4fQmDuzRmQR79eg+q6nSY9m2WqKX59R/9dhKfihYatIf4PYqHq+wJTKzuLSVwWMoWZm6IiFope0YAxGvbmG6kabB62jDcndzRWd1qluIK+2Y8fbMzV/Jc/GVvrY7ozyZZIbnPsLii1izLdcDGrCS5TVbFwad4AUF7CpttzTpqscFNC4luGNwQEbVS9nyPiL9ILdWy9Gofgvfu7uekXjXPo0v2OHSdNLix/6vN0s7vy3ZJgye5fbnExK9pWJlZHEta2vXc1M7Tl7B8V64k0yOHw1LWMbghImplrk9s2AzzjgEdmnSdI0uQdAz3b/pFJh4cnGB3W3u3SDAlDj78fFT4146z+CErH59tMh9KMkynFwRBsvih2AfrThofbz5xUbaAWJzdEQdXo/9vU8N5UVtxcGNpIcYD50pw78IdeGHFQezPK5FtYyAOWt1RxFtTr8OZYudsKOoOrLkhImpllj02GBU1WoQGWJ9xY8p0A0ZLEiIC0CM2GDf1isEnG/6w+/7RwRoUySykl9YrFkt35Mpc4TziUZpLlbV45YdDFtuWVtcjv6QaD3+1G1eq5GdtafV6lFbX4901x8yGwAyq63UIMG5S2ng8v6QaB8+VSoKOelHQZjrkdOR8GX47dgFxoY2B5KVK68Nz4sxNvU6Ar7ppqZwrlXV4PvMA7k6JN6vlkjPxk99xrLAc/34iFdclRths72kMboiIWhmVUtHkwAawnDEw5eejxMLJKQCAzzbbtx8TAPz5ugSUVddj8bYzkuOWsiPOJF7nxlelRE295QzQpcpafPjrSav1PTqdgPsW7sCRgjKLbcprtMbgxnRRvVs/3ip5bi1zM37eFgBA58jGRQGr6+Q3HTUQv55Wr4evjYEYnV5AQWm1ceHBD389ibVHLmDtkQs4M2eC1WsB4FhhQ5H3j9n5NoOb7acuIbVrpNU2rsZhKSKiNsLe4EbMUk2KpbbirR8MfN0wI+tQfmPBr60ZYJcq6hBiY52Zer3eamADALWiAMrWp3RFNKyltVDYfUa0KWhFreVNRwHpMJit+hwAmPFdNob+YwN+OtAw5d3a9hf2vq4l931ueyq/qzG4ISJqI+wdllKIvqqbknVRKRWyU7zFU9RdVQh7QjR93HQ/KlNlNVq0C9ZYbWNPICjOxtj60r9XNEtLa0cwYmuRQPHnqNMLOHupEhPmbcGP2fmy7f93dR2fTzc01CAFaBwbuFEpFajVWs8qtQQMboiI2oh4mb2QbFEr7f+aUCkVqK4zzziI152xtTKvo8S7jIv3d5JTp9UbZzRZYm2dmcY2DcFNUVkNLtmxYnFT7m2pFshAHKdqdXo8+vUeHD5fhunLs61fd/W/gTIZNkuWi2aOHcovRdLLa/DeL8fsvt4TGNwQEXm5ZY9ej0eHJmLykE52tRevz2JpQTw5Wp0eVTK1IuJhIksZk4hAX1zX2fFC1bKaxmDAVtalVquT7N4tx54kV722odHdn2233VjEvuDGerAkntZfrxdwsqjCrtc2FDkbaoXExyx5YcVB4+M9Z68AAD7Z0LQFDd2NwQ0RkZcb0i0KL9/Sy6FNMu+/3v5p3OW1WvSLDzM7Lg5uYkP8zM6PuKYd9rw0Bp0im55ZMmjKFPJarXwQ1uTXvDosdVZUK2MPrU6PC2U1WLL9jMXaGlvBlXh3ePHWDmEmheZLtp/B91nnGq8zBjeNfxYqr34W6d9l49Gv9zhlarkj9V3OxNlSREQkIa65mdi/A347dtFYs2FNeY0Wfx4Uj6o6Ld5Z3ThsIS5Kjg4xz9woFA2zf/x83LNDeZ1Wb1dAEh7gY3V4SOvIwkFoyNzc//kOnLpYiT/szLiYsrS1Q6AoI5NfUo1Xfzwse5044CyrroePSoEVWQ31Oi//cAg940Lw4GD7Mn1y6nV6qJSe2XEeYOaGiIisUCgU6C+TjZFTVl0PtUqJx4d3RY/YYONxW5kbAz8f93wlbT5xEQdFs6sssZV8uFhRa3NWkxytTsCpiw0L4v16tKjJ1wPSzEilqA/iz7BSpm9/FFVgxvIsya7s5TVaSRH2Nztz8fIPh5qVwWHmhoiIWhTTDTntLbspr5H/ohfX7cRYCW5szXJyll+P2RdQ2BrqmrosC8EOzDoSTwUP9nPsa/iNn44YH4tnVvmKhh4tTeP/IVuahauo1coWWP90oADtwxxbodqeGWGuxMwNEREBAN64vTeC/dR432SfKbndxgHgxFvjJM/HWFjpVnx9RKCv2XnDV7ClIaBecSGWuuxS9uwHVe5A5kY8o8nR4EZMHNyIa3HsTZ5odXrJmj0Gz3ybhTvnb3OoT/UW1vJxF2ZuiIgIADA5tTMevL6T2Wq74k0h5903ACOT2kEQAF+TDSrvv06++FicQejSLtDs/LUdQgFYXljO1swhV3HH0Ip41pKjxMGNOKiwtFigKa1ecPraNczcEBFRi2Ea2ACARlToq1IoEOLng1B/8/VqLO2cHeCrQlqvGIxKaifJwoy4ph3+NjYJT43qBgAYkBAue73cflWuMGVEV7e8jpgjNTum3lt73PhYHFTYG2DU6fRWt6twhOn+We7m0eBm8+bNuPXWW9G+fXsoFAr88MMPNq/ZtGkTkpOT4efnhy5dumDBggWu7ygRURvmJ8rQWApgTIl3rVYoFFg4OQVfPXIdFAoF1j07HI/c0Bkf3z8AT4/qZpwl9aSF4MJVGZSoIOnMrRu6uX8/JFsrEdujzsKmnPYGGA0Fzo7N2rJ4Tw8XFHs0uKmsrES/fv3w8ccf29U+JycH48ePx7Bhw5CVlYUXX3wR06ZNQ2Zmpot7SkTUdomnaDdlrylLuscE47Vbe5utVuzvq7I5YyraxrYJpkyHzsSu7dCYRboruSPCA8zrgVztkpXNOx1RrxMgCAL0esHuAGPVgfM2Vza2pLC0Bhmrj5odd3SavLN4tOZm3LhxGDdunO2GVy1YsAAJCQmYO3cuAKBnz57Ys2cP3n//fdx5550u6iURUdsmDm5UFqZOmcY8joZApjt6T07thD7tQ/HqykOY/0AyRia1Q+Ls1bLXvntXX8z67wHj886RAbhUUWdx1pN41CalU7js3lu+KqVdhcWOKrMww8xRWr0ek7/chYLSGrx2ay+7rjGdPdUUT/xrD/afM59Wb89mnq7UqgqKt2/fjrS0NMmxsWPHYtGiRaivr4ePj/kYcG1tLWprGyPjsjLru7wSEZGUOJtiKXNj73CVLaaZltdu7Q2VUoE7kzvafI3OkdJi5YpaHaztZzmuTyw2n7gIoKHWSC7R4at2bXDjbHVaPbacLAYAHMp37fedXi/IBjaA/cXMrtKqCooLCwsREyOdahgTEwOtVovi4mLZazIyMhAaGmr8iY+Pd0dXiYi8hp9JQbEchY1dse3lazLt3BDQ2BM8hZtsPVBZq7XYrykjukraqxQK+cyNlWGtlki8rYS98WakzPR8ezy4aKfFc2265sYRpn9QDSsoWvoDPHv2bJSWlhp/8vLyXN5HIiJv4idaGM5SkOGkxA3eueNah681reFRKiBZiVcs0FcFlWjHc6VSukaMgWmwZa+wAB/8dWii5NikZmxn4AidnSsM29pF3ZJtpy5ZPMep4E0QGxuLwsJCybGioiKo1WpERspXuWs0GoSEhEh+iIjIfhof27OlTDM6jiZyRiZFO3YhzPv2+eQUJEaZr6sDAAEaNcRxi1KhMAuOAMczN50iA/HKLdKaF3UTdlh3BnundztjE1FTni4oblXBTWpqKtatWyc5tnbtWqSkpMjW2xARUfOJMzeW/j2udNKwFNC0+p2RSe2Mj8X1QPER/hjSLQqfPjBQdouEAJPMjUqpQFJsMJ4dcw1mpl1jPK5xMLgxdEVcryTeY+vVW2wX+/o0Mxia9+vJZl3fHPVteViqoqIC2dnZyM7OBtAw1Ts7Oxu5ubkAGoaUJk+ebGw/ZcoUnD17Funp6Th69Ci+/PJLLFq0CDNnzvRE94mI2gRx5sbS2ilOjG3sdnv/9pKARrwAoSHY6hkXglk3J5ldq1IqpNdebT99THdMGtxZ0s4RhkyWv4/8kF6Ar+0ds921S7ortOnMzZ49ezBgwAAMGDAAAJCeno4BAwbg1VdfBQAUFBQYAx0ASExMxOrVq7Fx40b0798fb775JubNm8dp4ERELiTOXlia4mu6svHzN/cAADyU2vQ6k6aEE+IhFblgBZDfY6m0ql7SRvxYPBTlaKG04fMQByji+h17Ahf/VhzctOmp4CNHjrS6pfrixYvNjo0YMQL79u1zYa+IiEhM/AWvszDF13RYavg17bD/tTSEOGFjSGuGdW+HbacuITzAR5IZEcdaciscJ0YFSmpgxNeKgxtHC6UN14kDFH9RtsbWYoWmfWptOBWciIhajcSoINnjct/Dof4+DmU+7L1EAeCvQxPxz3v64efpwy2uwSOe4r1q2lDMueNajO4ZLQkexMXF4uMKBbD5b6PM7pncKRxfPTzIYt8M9zDslB4Z6CsZitKobWdlKpy8wJ87eXq2VKtaxI+IiDxjffpwFJXXWpx95Kx1bgBAAQUsly43EtCQZbljYEezc+JhEfEAQe/2oejdvmEXcpXJ/leW+pIQGWB2/MkRXWWPGxgyWek3XYOO4f4YlRSN7acbp05r7MjcBGrUKHfCxpqe4OmNMxncEBGRTd2ig9EtOtji+ZY2glKrbazFkVucD5BmaCwtTmjpfamUCqszxAzn/HxUmJzaGQCQnVdiPG8tc7PgwYG4VFmHjccvovBIjcV2APDEiC4I9feBVifgs02nUOmCad2OcNVmp/ZicENERM1mKThwiBNuJd5PytLXrKWaG2lfLB23PlVb7nbi+htLU8zvuy4eN/eJAwCsP3LB4v0NogI1eGx4FwDA51tO22zvSrf1a486rR5rDhe27angRETkHZw5LOUM4uDGx8Iqw9JhKfn7WMtIdQwPwMNDOqNHrHlGSy5YEtfc+KiUsm3Edbj2zDgSz1Kz5zcwfXR3O1o5JsBXZQwY2/RUcCIi8g5KJ36b2BsmWdtdQLzZ5Z8HxaNffBieu+kaSRv7hqUs9Obqa//9tt6Ymda4jk7Q1QUDR/UwX2nZTxTcqFUK2eBGvGWCPRt2ipNH9gSYAxLCbLaxJUhmUUSgYTaYIZBkQTEREbV6Th2WslOElQ0fxVmPQI0aPz59g1kbtdL2thKGo+/f3Q+/HbuA1QcbtgASRINdwaLp7uvTRyAr9wrSesea3Us8LOWjVMp+Zr3iGrcIsqcoV6m0nX2y1AdH9W4fgp05l82OB/iqjDO86j08FZzBDRERNZszt1+wdauP7huAFfvOYcaY5g2xiLNNFmdLXT18V3JH3NovDqsPrjFrc11iBB4cnICu7YIQG+qHcdfGyd5LEtyopZmbjDuuRVl1PSaJFj20K7hp4ufub8fKyJYEadS4Y2AH5F6ukj0f5u+Ly5UNG5Uyc0NERK2Wn48SNfV6XN9FfvNiV7i1X3vc2q99s+9jX+bG9tCVQqHAWxNt72YuDizUSqWknqdvx8Yp6gb1WtsBgsqOmps/D4rH8t15AJq3pcO+V26Cr1qJx5fskT0f6u9jLLJmzQ0REbVav8wYjhfG9cBLE3o67Z4KZ0yXsoM9NTfhgY2bMovbW6v3sURc2CwIAtQq+U01DezL3DQ+lss+HX/rZtzQLcr43NdCcbU9DCs3W5paHxrgYwwYOVuKiIharU6RgZgyoqvFIlN3W/zIIEQF+eLLh1NstjVdiVjso/sGYGBCGF67tbeoTfOCG/FWFGEBvpIhJbnMkT0FxeJ7yIVnGrVK8t58TaagvzyhJ54Y0QWjRLury/nu8cHGx1oLgUtYC8rctIw/jURERFc1p3xnZFI0dr80xq6ZQ9YyMc4a+hJTq5TY/2oa9IIAX7VSsuWDj8x0M3syN9YCNDmmGaKaeh1mj2vIunV+YZVZ+7uTO6J/Qphk2NFSPU2If+P+Xm1640wiIiJTzR2UsnfNHfFeVIId2z2IOfrVHRrQOMwlqfmRWRDQUoCQEBFgLOoN9vMRnbFdN2SauRlspVZKrVTgvbv7mR23tClmkEZtHGrz9ArFHJYiIqI2qbk1NM2llGRurE8LF5t6Yzfj49gQP5uvIxmWEmVu+nUMRUrnCIvXvTCuh+xxS5mbyCBf4/vw9K7gzNwQEVGL4q7VjiXBTROvFZwQDdmqufnnPf3w8YY/MDKpHf6yuHGGknjGU0yoxvjY0sfWtV3jTu7izM2AhHBJuyeGd8Fnm0/jwcEJeHRoF3SysDGoabHwyqkNawgF+DZmbjgsRURE5AEqSYFw076M+3QItd3IBvFLqmVmMUWH+OGN2/sgz2RdmSrRTuFRgaLgxsLrJMUG44vJKYgN9bO8hxaAWTf3wC1926NnXLBsfwx0JlmZvh3DjI9ZUExERCTDXWsdKx3I3Ox5eQxKq+vRPsy/2a8vrvNRWwk6TDMyHcIbX9veFYrH9Ioxf32TgE6lVODajraDNvFlH/65v+Sc4X14eio4gxsiImrz7N0+IipIg6ggje2GdpBmbiy/vrRoGLg+MRJvTexjtmGnuHB4+DXtmr2CsyUZd1yLBz7fiefSrsHt/TtIzqmNe0sxc0NERNTIjdtU3X99AvKvVONaJwwzNZUkuLGy82iovw+mj+6OD389CaAhw/Lg4E5m7cTx2ZK/XGf79e3vqkTfjmHY/1qaJGtk0DgsxcwNERGRR7zzJ9vbJriKeFjIWi0MANzYI9oY3Nho6hZygQ0ArlBMREQk57qr05PFu217o6Z8/YuzMhY3+Wzq67sg/lCzoJiIiMjcu3f1xZe/5+Du5HhPd8WlmhJchJjU3chx1xR6awyZG0tbNLitHx59dSIiIhORQRr8baz8AnLexNIGlHI6RwViZto1CA/0dWGPmo+ZGyIiojasqbmNqTdan/1kb+JmTM9orD9aJFuU3FzGgmJmboiIiNoeZ2c37A1uFk5KQXmtFqH+toe6mspYUOzh2VIsKCYiIvKAkup6p95PYWdJsVKpcElgA7ScYSkGN0RERB7gic06Xc1H1TIKihncEBEReYGYEOesnNwcxu0XmLkhIiJquwJ9VbYb2eH/7u6P1C6RWPzIIKfczxHGzA1XKCYiImq7BiSEO+U+CZEB+PbxwU65l6MMKy3rPDzmphCaus97K1dWVobQ0FCUlpYiJCTE090hIqI2au/Zy/h621m8PKEnokP8PN0dpxAEAXrB9nYSjmjK9zczN0RERB6Q3CkCyZ0iPN0Np1IoFLCywbnbsOaGiIiIvAqDGyIiIvIqDG6IiIjIqzC4ISIiIq/i8eDm008/RWJiIvz8/JCcnIwtW7ZYbLtx40YoFAqzn2PHjrmxx0RERNSSeTS4+e677zBjxgy89NJLyMrKwrBhwzBu3Djk5uZave748eMoKCgw/nTvbn2nVCIiImo7PLrOzfXXX4+BAwdi/vz5xmM9e/bExIkTkZGRYdZ+48aNGDVqFK5cuYKwsDC7XqO2tha1tbXG52VlZYiPj+c6N0RERK1IU9a58Vjmpq6uDnv37kVaWprkeFpaGrZt22b12gEDBiAuLg6jR4/Ghg0brLbNyMhAaGio8Sc+Pr7ZfSciIqKWy2PBTXFxMXQ6HWJiYiTHY2JiUFhYKHtNXFwcFi5ciMzMTKxYsQJJSUkYPXo0Nm/ebPF1Zs+ejdLSUuNPXl6eU98HERERtSweX6FYoZAuZSgIgtkxg6SkJCQlJRmfp6amIi8vD++//z6GDx8ue41Go4FG4/mdUomIiMg9PJa5iYqKgkqlMsvSFBUVmWVzrBk8eDBOnjzp7O4RERFRK+Wx4MbX1xfJyclYt26d5Pi6deswZMgQu++TlZWFuLg4Z3ePiIiIWimPDkulp6dj0qRJSElJQWpqKhYuXIjc3FxMmTIFQEO9TH5+PpYsWQIAmDt3Ljp37ozevXujrq4OS5cuRWZmJjIzMz35NoiIiKgF8Whwc++99+LSpUt44403UFBQgD59+mD16tXo1KkTAKCgoECy5k1dXR1mzpyJ/Px8+Pv7o3fv3li1ahXGjx/vqbdARERELYxH17nxhKbMkyciIqKWoVWsc0NERETkCgxuiIiIyKswuCEiIiKvwuCGiIiIvAqDGyIiIvIqDG6IiIjIqzC4ISIiIq/C4IaIiIi8CoMbIiIi8ioMboiIiMirMLghIiIir8LghoiIiLwKgxsiIiLyKgxuiIiIyKswuCEiIiKvwuCGiIiIvAqDGyIiIvIqDG6IiIjIqzC4ISIiIq/C4IaIiIi8CoMbIiIi8ioMboiIiMirMLghIiIir8LghoiIiLwKgxsiIiLyKgxuiIiIyKswuCEiIiKvwuCGiIiIvAqDGyIiIvIqDG6IiIjIqzC4ISIiIq/C4IaIiIi8CoMbIiIi8ioMboiIiMirMLghIiIir+Lx4ObTTz9FYmIi/Pz8kJycjC1btlhtv2nTJiQnJ8PPzw9dunTBggUL3NRTIiIiag08Gtx89913mDFjBl566SVkZWVh2LBhGDduHHJzc2Xb5+TkYPz48Rg2bBiysrLw4osvYtq0acjMzHRzz4mIiKilUgiCIHjqxa+//noMHDgQ8+fPNx7r2bMnJk6ciIyMDLP2zz//PFauXImjR48aj02ZMgX79+/H9u3b7XrNsrIyhIaGorS0FCEhIc1/E0RERORyTfn+VrupT2bq6uqwd+9evPDCC5LjaWlp2LZtm+w127dvR1pamuTY2LFjsWjRItTX18PHx8fsmtraWtTW1hqfl5aWAmj4kIiIiKh1MHxv25OT8VhwU1xcDJ1Oh5iYGMnxmJgYFBYWyl5TWFgo216r1aK4uBhxcXFm12RkZOD11183Ox4fH9+M3hMREZEnlJeXIzQ01GobjwU3BgqFQvJcEASzY7bayx03mD17NtLT043P9Xo9Ll++jMjISKuv01RlZWWIj49HXl4eh7s8hL8Dz+Ln73n8HXgWP3/XEgQB5eXlaN++vc22HgtuoqKioFKpzLI0RUVFZtkZg9jYWNn2arUakZGRstdoNBpoNBrJsbCwMMc7bkNISAj/UHsYfweexc/f8/g78Cx+/q5jK2Nj4LHZUr6+vkhOTsa6deskx9etW4chQ4bIXpOammrWfu3atUhJSZGttyEiIqK2x6NTwdPT0/HFF1/gyy+/xNGjR/Hss88iNzcXU6ZMAdAwpDR58mRj+ylTpuDs2bNIT0/H0aNH8eWXX2LRokWYOXOmp94CERERtTAerbm59957cenSJbzxxhsoKChAnz59sHr1anTq1AkAUFBQIFnzJjExEatXr8azzz6LTz75BO3bt8e8efNw5513euotGGk0Grz22mtmQ2DkPvwdeBY/f8/j78Cz+Pm3HB5d54aIiIjI2Ty+/QIRERGRMzG4ISIiIq/C4IaIiIi8CoMbIiIi8ioMbpzk008/RWJiIvz8/JCcnIwtW7Z4ukteISMjA4MGDUJwcDCio6MxceJEHD9+XNJGEAT8/e9/R/v27eHv74+RI0fi8OHDkja1tbV45plnEBUVhcDAQNx22204d+6cO9+KV8jIyIBCocCMGTOMx/j5u1Z+fj4efPBBREZGIiAgAP3798fevXuN5/n5u5ZWq8XLL7+MxMRE+Pv7o0uXLnjjjTeg1+uNbfg7aIEEarbly5cLPj4+wueffy4cOXJEmD59uhAYGCicPXvW011r9caOHSt89dVXwqFDh4Ts7GxhwoQJQkJCglBRUWFsM2fOHCE4OFjIzMwUDh48KNx7771CXFycUFZWZmwzZcoUoUOHDsK6deuEffv2CaNGjRL69esnaLVaT7ytVmnXrl1C586dhb59+wrTp083Hufn7zqXL18WOnXqJDz88MPCzp07hZycHGH9+vXCH3/8YWzDz9+13nrrLSEyMlL46aefhJycHOE///mPEBQUJMydO9fYhr+DlofBjRNcd911wpQpUyTHevToIbzwwgse6pH3KioqEgAImzZtEgRBEPR6vRAbGyvMmTPH2KampkYIDQ0VFixYIAiCIJSUlAg+Pj7C8uXLjW3y8/MFpVIprFmzxr1voJUqLy8XunfvLqxbt04YMWKEMbjh5+9azz//vDB06FCL5/n5u96ECROEv/zlL5Jjd9xxh/Dggw8KgsDfQUvFYalmqqurw969e5GWliY5npaWhm3btnmoV96rtLQUABAREQEAyMnJQWFhoeTz12g0GDFihPHz37t3L+rr6yVt2rdvjz59+vB3ZKenn34aEyZMwJgxYyTH+fm71sqVK5GSkoK7774b0dHRGDBgAD7//HPjeX7+rjd06FD8+uuvOHHiBABg//792Lp1K8aPHw+Av4OWyuO7grd2xcXF0Ol0Zpt9xsTEmG3ySc0jCALS09MxdOhQ9OnTBwCMn7Hc53/27FljG19fX4SHh5u14e/ItuXLl2Pfvn3YvXu32Tl+/q51+vRpzJ8/H+np6XjxxRexa9cuTJs2DRqNBpMnT+bn7wbPP/88SktL0aNHD6hUKuh0Orz99tu47777APD/gZaKwY2TKBQKyXNBEMyOUfNMnToVBw4cwNatW83OOfL583dkW15eHqZPn461a9fCz8/PYjt+/q6h1+uRkpKCd955BwAwYMAAHD58GPPnz5fsu8fP33W+++47LF26FMuWLUPv3r2RnZ2NGTNmoH379njooYeM7fg7aFk4LNVMUVFRUKlUZtF3UVGRWSRPjnvmmWewcuVKbNiwAR07djQej42NBQCrn39sbCzq6upw5coVi21I3t69e1FUVITk5GSo1Wqo1Wps2rQJ8+bNg1qtNn5+/PxdIy4uDr169ZIc69mzp3HPPf75d72//e1veOGFF/DnP/8Z1157LSZNmoRnn30WGRkZAPg7aKkY3DSTr68vkpOTsW7dOsnxdevWYciQIR7qlfcQBAFTp07FihUr8NtvvyExMVFyPjExEbGxsZLPv66uDps2bTJ+/snJyfDx8ZG0KSgowKFDh/g7smH06NE4ePAgsrOzjT8pKSl44IEHkJ2djS5duvDzd6EbbrjBbOmDEydOGDcX5p9/16uqqoJSKf2qVKlUxqng/B20UB4qZPYqhqngixYtEo4cOSLMmDFDCAwMFM6cOePprrV6Tz75pBAaGips3LhRKCgoMP5UVVUZ28yZM0cIDQ0VVqxYIRw8eFC47777ZKdhduzYUVi/fr2wb98+4cYbb+Q0TAeJZ0sJAj9/V9q1a5egVquFt99+Wzh58qTwzTffCAEBAcLSpUuNbfj5u9ZDDz0kdOjQwTgVfMWKFUJUVJQwa9YsYxv+DloeBjdO8sknnwidOnUSfH19hYEDBxqnKlPzAJD9+eqrr4xt9Hq98NprrwmxsbGCRqMRhg8fLhw8eFByn+rqamHq1KlCRESE4O/vL9xyyy1Cbm6um9+NdzANbvj5u9b//vc/oU+fPoJGoxF69OghLFy4UHKen79rlZWVCdOnTxcSEhIEPz8/oUuXLsJLL70k1NbWGtvwd9DyKARBEDyZOSIiIiJyJtbcEBERkVdhcENERERehcENEREReRUGN0RERORVGNwQERGRV2FwQ0RERF6FwQ0RERF5FQY3RERE5FUY3BCRW4wcORIzZsxw62ueOXMGCoUC2dnZbn1dIvIsBjdE1Cps3LgRCoUCJSUlnu4KEbVwDG6IiIjIqzC4ISK30Wq1mDp1KsLCwhAZGYmXX34Zhu3tli5dipSUFAQHByM2Nhb3338/ioqKADQML40aNQoAEB4eDoVCgYcffhgAoNfr8Y9//APdunWDRqNBQkIC3n77bcnrnj59GqNGjUJAQAD69euH7du3S85v27YNw4cPh7+/P+Lj4zFt2jRUVlYaz3/66afo3r07/Pz8EBMTg7vuustVHxEROQGDGyJym6+//hpqtRo7d+7EvHnz8MEHH+CLL74AANTV1eHNN9/E/v378cMPPyAnJ8cYwMTHxyMzMxMAcPz4cRQUFODDDz8EAMyePRv/+Mc/8Morr+DIkSNYtmwZYmJiJK/70ksvYebMmcjOzsY111yD++67D1qtFgBw8OBBjB07FnfccQcOHDiA7777Dlu3bsXUqVMBAHv27MG0adPwxhtv4Pjx41izZg2GDx/ujo+LiBzl4V3JiaiNGDFihNCzZ09Br9cbjz3//PNCz549Zdvv2rVLACCUl5cLgiAIGzZsEAAIV65cMbYpKysTNBqN8Pnnn8veIycnRwAgfPHFF8Zjhw8fFgAIR48eFQRBECZNmiQ8/vjjkuu2bNkiKJVKobq6WsjMzBRCQkKEsrIyh943EbkfMzdE5DaDBw+GQqEwPk9NTcXJkyeh0+mQlZWF22+/HZ06dUJwcDBGjhwJAMjNzbV4v6NHj6K2thajR4+2+rp9+/Y1Po6LiwMA45DX3r17sXjxYgQFBRl/xo4dC71ej5ycHNx0003o1KkTunTpgkmTJuGbb75BVVWVox8BEbkBgxsi8riamhqkpaUhKCgIS5cuxe7du/H9998DaBiussTf39+u+/v4+BgfG4IrvV5v/O8TTzyB7Oxs48/+/ftx8uRJdO3aFcHBwdi3bx++/fZbxMXF4dVXX0W/fv04a4uoBWNwQ0Rus2PHDrPn3bt3x7Fjx1BcXIw5c+Zg2LBh6NGjhzGzYuDr6wsA0Ol0xmPdu3eHv78/fv31V4f7NHDgQBw+fBjdunUz+zG8plqtxpgxY/Duu+/iwIEDOHPmDH777TeHX5OIXIvBDRG5TV5eHtLT03H8+HF8++23+OijjzB9+nQkJCTA19cXH330EU6fPo2VK1fizTfflFzbqVMnKBQK/PTTT7h48SIqKirg5+eH559/HrNmzcKSJUtw6tQp7NixA4sWLbK7T88//zy2b9+Op59+GtnZ2Th58iRWrlyJZ555BgDw008/Yd68ecjOzsbZs2exZMkS6PV6JCUlOfWzISLnYXBDRG4zefJkVFdX47rrrsPTTz+NZ555Bo8//jjatWuHxYsX4z//+Q969eqFOXPm4P3335dc26FDB7z++ut44YUXEBMTY5zN9Morr+C5557Dq6++ip49e+Lee+81y/pY07dvX2zatAknT57EsGHDMGDAALzyyivG2pywsDCsWLECN954I3r27IkFCxbg22+/Re/evZ33wRCRUykE4eoiE0RERERegJkbIiIi8ioMboiIiMirMLghIiIir8LghoiIiLwKgxsiIiLyKgxuiIiIyKswuCEiIiKvwuCGiIiIvAqDGyIiIvIqDG6IiIjIqzC4ISIiIq/y/2N+pnOFG341AAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plt.plot(epochs_x[-1], epochs_y[-1])\n", + "plt.xlabel('batches')\n", + "plt.ylabel('loss')\n", + "plt.ylim(0,)\n", + "plt.show()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "speck-rescnn", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 858e21c2dbc6cbb535a8e77e278b0c373c34719a Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Fri, 26 Apr 2024 20:38:08 +0200 Subject: [PATCH 046/379] baseline accuracy | model 1 | NMNIST --- .../baseline-SCNN-example_1-NNI.ipynb | 234 ++++++++++++++++-- 1 file changed, 217 insertions(+), 17 deletions(-) diff --git a/tests/test_nonsequential/baseline-SCNN-example_1-NNI.ipynb b/tests/test_nonsequential/baseline-SCNN-example_1-NNI.ipynb index 159942e2..90889ea7 100644 --- a/tests/test_nonsequential/baseline-SCNN-example_1-NNI.ipynb +++ b/tests/test_nonsequential/baseline-SCNN-example_1-NNI.ipynb @@ -19,7 +19,8 @@ "\n", "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential\n", "\n", - "import matplotlib.pyplot as plt" + "import matplotlib.pyplot as plt\n", + "import numpy as np" ] }, { @@ -30,7 +31,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 2, @@ -57,7 +58,7 @@ "source": [ "batch_size = 64\n", "num_workers = 4\n", - "epochs = 1\n", + "epochs = 5\n", "lr = 1e-3" ] }, @@ -135,24 +136,24 @@ " def __init__(self) -> None:\n", " super().__init__()\n", "\n", - " self.conv1 = nn.Conv2d(2, 8, 3, 1, bias=False)\n", + " self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False)\n", " self.iaf1 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", " self.pool1 = nn.AvgPool2d(2,2)\n", + " self.pool1a = nn.AvgPool2d(6,6)\n", "\n", - " self.conv2 = nn.Conv2d(8, 16, 3, 1, bias=False)\n", + " self.conv2 = nn.Conv2d(10, 10, 2, 1, bias=False)\n", " self.iaf2 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - " self.pool2 = nn.AvgPool2d(2,2)\n", + " self.pool2 = nn.AvgPool2d(3,3)\n", "\n", - " self.conv3 = nn.Conv2d(16, 16, 3, 1, bias=False)\n", + " self.conv3 = nn.Conv2d(10, 10, 3, 1, bias=False)\n", " self.iaf3 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", " self.pool3 = nn.AvgPool2d(2,2)\n", "\n", " self.flat = nn.Flatten()\n", "\n", - " self.fc1 = nn.Linear(64, 10, bias=False)\n", + " self.fc1 = nn.Linear(10, 10, bias=False)\n", " self.iaf4 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", "\n", - "\n", " def detach_neuron_states(self):\n", " for name, layer in self.named_modules():\n", " if name != '':\n", @@ -231,6 +232,7 @@ "def train(dataloader, model, loss_fn, optimizer, epochs, test_func, dataloader_test):\n", " epochs_y = []\n", " epochs_x = []\n", + " epochs_acc = []\n", " model.train()\n", "\n", " for e in range(epochs):\n", @@ -273,8 +275,9 @@ "\n", " acc = test_func(dataloader_test, model)\n", " print(f'Epoch {e} accuracy: {acc}')\n", + " epochs_acc.append(acc)\n", "\n", - " return epochs_x, epochs_y\n" + " return epochs_x, epochs_y, epochs_acc\n" ] }, { @@ -328,7 +331,42 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "3417439bf42847f98f43aad80ffb8e53", + "model_id": "0cd81220753e46039b5cac482961c5c5", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/937 [00:00" ] @@ -382,9 +525,66 @@ } ], "source": [ - "plt.plot(epochs_x[0], epochs_y[0])\n", + "plt.plot(epochs_x[-1], epochs_y[-1])\n", "plt.xlabel('batches')\n", "plt.ylabel('loss')\n", + "plt.ylim(0,)\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAGwCAYAAABVdURTAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAABTz0lEQVR4nO3de1xUZf4H8M9cgOGOgCAoV/GOIIIXUPJOqy6b1W+lbNVKS9Ty1mVRy9Ta2NwurluY5qV1K6PS3Eq2pJSLYhkG3vMCCAiDCMhwkwFmzu8PZBS5yChwmOHzfr3mJZx5zpnv49mczz7nnOeRCIIggIiIiMhISMUugIiIiKg9MdwQERGRUWG4ISIiIqPCcENERERGheGGiIiIjArDDRERERkVhhsiIiIyKnKxC+hsWq0W+fn5sLa2hkQiEbscIiIiagNBEFBeXg5XV1dIpa2PzXS7cJOfnw83NzexyyAiIqJ7kJubiz59+rTaptuFG2trawD1fzk2NjYiV0NERERtUVZWBjc3N933eGu6XbhpuBRlY2PDcENERGRg2nJLCW8obkFSUhLCw8Ph6uoKiUSCffv2tXnfI0eOQC6XY9iwYU3eKy0txeLFi+Hi4gKFQoFBgwYhLi6u/QonIiLq5rrdyE1bVVZWwt/fH0899RQeffTRNu+nUqkwZ84cTJo0CVevXm30Xk1NDaZMmQInJyd89dVX6NOnD3Jzc9s0xEZERERtw3DTgqlTp2Lq1Kl677dgwQLMmjULMpmsyWjPjh07UFJSgpSUFJiYmAAAPDw82qNcIiIiuomXpdrRzp07kZGRgddee63Z97/55hsEBwdj8eLFcHZ2hq+vL958801oNJpOrpSIiMh4ceSmnVy8eBFRUVFITk6GXN78X2tmZiYOHjyIJ554AnFxcbh48SIWL16Muro6rFmzppMrJiIiMk4MN+1Ao9Fg1qxZWLduHfr3799iO61WCycnJ2zduhUymQyBgYHIz8/HP/7xD4YbIiKidsJw0w7Ky8uRmpqKtLQ0PPfccwDqg4wgCJDL5Thw4AAmTpwIFxcXmJiYQCaT6fYdNGgQCgoKUFNTA1NTU7G6QEREZDQYbtqBjY0NTp061WhbTEwMDh48iK+++gpeXl4AgDFjxuCzzz6DVqvVTR194cIFuLi4MNgQERG1E4abFlRUVODSpUu637OyspCeng57e3u4u7tj5cqVyMvLw65duyCVSuHr69tofycnJygUikbbFy5ciH/9619YunQpnn/+eVy8eBFvvvkmlixZ0mn9IiIiMnYMNy1ITU3FhAkTdL+vWLECADB37lx8/PHHUCqVyMnJabSPUnUDWUWV8HK0bPaYbm5uOHDgAJYvXw4/Pz/07t0bS5cuxV//+teO6wgREVE3IxEEQRC7iM5UVlYGW1tbqFSqdl1+IfbXHKzcewpaAZBKgOhHhiJihHu7HZ+IiKg70+f7m/PctAOl6gaibgYbANAKwKq9p6FU3RC3MCIiom6I4aYdZBVV4s7xL40g4HJRlTgFERERdWMMN+3Ay9ES0jsWKZVJAE9HC3EKIiIi6sYYbtqBi605oh8Z2ijgzBvrDRdbc/GKIiIi6qYYbtpJxAh3HImaiMmDnAAA5wrKRK6IiIioe2K4aUcutuZ4LXwIZFIJki8W4dQVldglERERdTsMN+3Mzd4Cf/J3BQBsTrx0l9ZERETU3hhuOsDC8X0BAP87XYCMaxUiV0NERNS9MNx0gP7O1pg8yBmCAHyYkCF2OURERN0Kw00HWTShfvTm67Q85JdyMj8iIqLOwnDTQYa798Bob3vUaQV8lJwpdjlERETdBsNNB1o03gcA8PmxXJRU1ohcDRERUffAcNOBQvs5YmhvW9yo1eDjI1lil0NERNQtiBpukpKSEB4eDldXV0gkEuzbt6/N+x45cgRyuRzDhg3rsPrul0QiwaKbT059nHIZFeo6kSsiIiIyfqKGm8rKSvj7++P999/Xaz+VSoU5c+Zg0qRJHVRZ+3lwSC9497REWXUdPv05W+xyiIiIjJ6o4Wbq1Kl444038Mgjj+i134IFCzBr1iwEBwffta1arUZZWVmjV2eSSiWIHFc/erPtcBaqazWd+vlERETdjcHdc7Nz505kZGTgtddea1P76Oho2Nra6l5ubm4dXGFTM4b1houtAtfK1djz25VO/3wiIqLuxKDCzcWLFxEVFYVPP/0Ucrm8TfusXLkSKpVK98rNze3gKpsylUvxTKg3AGBLYibqNNpOr4GIiKi7MJhwo9FoMGvWLKxbtw79+/dv835mZmawsbFp9BLDYyPd0MPCBDklVdh/SilKDURERN2BwYSb8vJypKam4rnnnoNcLodcLsf69etx4sQJyOVyHDx4UOwSW2VhKsfTY7wAAJsTMiAIgsgVERERGSeDCTc2NjY4deoU0tPTda/IyEgMGDAA6enpGDVqlNgl3tWcYE9Ymsrwe0E5Dv5eKHY5RERERqltN650kIqKCly6dEn3e1ZWFtLT02Fvbw93d3esXLkSeXl52LVrF6RSKXx9fRvt7+TkBIVC0WR7V2VrYYK/jPbAlqRMxCRkYOJAJ0gkErHLIiIiMiqijtykpqYiICAAAQEBAIAVK1YgICAAa9asAQAolUrk5OSIWWK7mzfWC6ZyKY5nX8exrBKxyyEiIjI6EqGb3fxRVlYGW1tbqFQq0W4uXvX1KXz2Sw7G9e+Jfz89UpQaiIiIDIk+398Gc8+NMVnwgDekEiDxwjWczlOJXQ4REZFRYbgRgYeDJcL9XQEAmxMzRK6GiIjIuDDciGThzQU1404pkXmtQuRqiIiIjAfDjUgG9rLBpIFOEIT6WYuJiIiofTDciGjRhPrRm71pV6BU3RC5GiIiIuPAcCOiQA97jPSyR61GwLbkLLHLISIiMgoMNyJbdPPem93HcnC9skbkaoiIiAwfw43IxvXviSGuNqiq0eDjlMtil0NERGTwGG5EJpFIsGi8DwDg45TLqFDXiVwRERGRYWO46QL+4NsL3o6WUN2oxe5fjGu5CSIios7GcNMFyKQSLBjnDQDYdjgT6jqNyBUREREZLoabLuLhgD7oZaPA1TI19v6WJ3Y5REREBovhposwlUsxP9QLALAlMQMabbdaz5SIiKjdMNx0IY+PdIedhQkuF1ch7pRS7HKIiIgMEsNNF2JpJsdTIfWjNzEJGRAEjt4QERHpi+Gmi5kb4gFLUxnOKcuQcP6a2OUQEREZHIabLsbOwhSzRrkDAGISLolcDRERkeFhuOmC5od6w1Qmxa+Xr+PXyyVil0NERGRQGG66IGcbBR4N7A0AiDnE0RsiIiJ9MNx0UQse6AupBDh0/hrO5peJXQ4REZHBYLjpojwdLTHdzxUAsDkxQ+RqiIiIDAfDTRe2cFxfAMD+k/m4XFQpcjVERESGgeGmCxvsaoMJA3pCKwBbkjh6Q0RE1BYMN13cogk+AIA9x/Nwtaxa5GqIiIi6PoabLm6Epz1GePZAjUaLbcmZYpdDRETU5THcGIBF4+tHbz79JQelVTUiV0NERNS1MdwYgPEDemKQiw2qajT4d0q22OUQERF1aQw3BkAikWDR+Ponp3amZKFSXSdyRURERF0Xw42BmDbUBZ4OFiitqsXuYzlil0NERNRlMdwYCJlUggU3573ZlpwFdZ1G5IqIiIi6JoYbA/LI8N5wtjFDQVk19qXliV0OERFRl8RwY0DM5DLMH+sNAPgwMRMarSByRURERF0Pw42BeXyUO2zNTZBVVInvTxeIXQ4REVGXw3BjYKzM5HgyxBMAEJNwCYLA0RsiIqLbMdwYoCdDPGFhKsOZ/DIkXSwSuxwiIqIuheHGAPWwNMXjI90BAB8cuiRyNURERF0Lw42Bmh/qBROZBMeySnA8u0TscoiIiLoMUcNNUlISwsPD4erqColEgn379rXafu/evZgyZQp69uwJGxsbBAcH44cffuicYrsYF1tzPBLQBwAQcyhD5GqIiIi6DlHDTWVlJfz9/fH++++3qX1SUhKmTJmCuLg4HD9+HBMmTEB4eDjS0tI6uNKuacE4b0gkwE+/F+L3gjKxyyEiIuoSJEIXedxGIpHg66+/xowZM/Tab8iQIYiIiMCaNWva1L6srAy2trZQqVSwsbG5h0q7lsWf/Yb9J5V4aJgr/vlYgNjlEBERdQh9vr8N+p4brVaL8vJy2Nvbt9hGrVajrKys0cuYLLy5JMO3J/KRU1wlcjVERETiM+hw884776CyshIzZ85ssU10dDRsbW11Lzc3t06ssOP59rbFuP49oRWAD5N47w0REZHBhpvdu3dj7dq1iI2NhZOTU4vtVq5cCZVKpXvl5uZ2YpWdY9H4+tGbr1KvoLCsWuRqiIiIxGWQ4SY2Nhbz5s3DF198gcmTJ7fa1szMDDY2No1exmaklz0CPXqgRqPF9sNZYpdDREQkKoMLN7t378aTTz6Jzz77DNOnTxe7nC5BIpHoRm8++TkbqqpakSsiIiISj6jhpqKiAunp6UhPTwcAZGVlIT09HTk5OQDqLynNmTNH13737t2YM2cO3nnnHYwePRoFBQUoKCiASqUSo/wuZeJAJwzsZY3KGg12Hb0sdjlERESiETXcpKamIiAgAAEB9Y8wr1ixAgEBAbrHupVKpS7oAMCWLVtQV1eHxYsXw8XFRfdaunSpKPV3JRKJBAtvjt7sTLmMqpo6kSsiIiISR5eZ56azGNs8N7er02gx8Z1E5JRUYc0fB+PpsV5il0RERNQuus08N9SYXCbFgnHeAICPkjNRU6cVuSIiIqLOx3BjZB4d3gc9rc2gVFVjX3qe2OUQERF1OoYbI6MwkWH+zctRHyZmQKPtVlcdiYiIGG6M0ROjPWCjkCPzWiUOnCkQuxwiIqJOxXBjhKzM5HgyxBMAEJOQgW52zzgREXVzDDdG6skxXjA3keFUngqHLxWJXQ4REVGnYbgxUvaWpnhsZP0ioR8cuiRyNURERJ2H4caIPRPqDROZBD9nluC3nOtil0NERNQpGG6MmKudOWYM6w0AiDmUIXI1REREnYPhxshFju8LiQT48dxVnC8oF7scIiKiDsdwY+T69rTCVN9eAOrnvSEiIjJ2DDfdwKLxPgCAb07kI7ekSuRqiIiIOhbDTTfg29sWof0codEK2JLE0RsiIjJuDDfdRMPozRepV1BYXi1yNURERB2H4aabGO1tjwB3O9TUabHj8GWxyyEiIuowDDfdhEQi0Y3efPJzNlQ3akWuiIiIqGMw3HQjkwY6YYCzNSrUdfjk52yxyyEiIuoQDDfdiFQqwcLxfQEAOw5n4UaNRuSKiIiI2h/DTTfzRz8XuNmbo7iyBrG/5ohdDhERUbtjuOlm5DIpnn2gfvTmo+Qs1Gq0IldERETUvhhuuqE/B/aBo5UZ8kpv4L/p+WKXQ0RE1K4YbrohhYkM88Z6AahfkkGrFUSuiIiIqP0w3HRTfxntDmuFHJcKK3Dg7FWxyyEiImo3DDfdlLXCBHODPQEAmxMuQRA4ekNERMaB4aYbe2qMJxQmUpy4okJKRrHY5RAREbULhptuzMHKDI+NcAcAfHDoksjVEBERtQ+Gm27umQe8IZdKkJJRjPTcUrHLISIium8MN91cbztzPDSsNwAghqM3RERkBBhuCAvHe0MiAQ6cvYqLV8vFLoeIiOi+MNwQfJys8eDgXgCAzYkZIldDRER0fxhuCACwaEL9kgz/Tc9HbkmVyNUQERHdO4YbAgD49bHDWB9HaLQCPkrOFLscIiKie8ZwQzqLxteP3sT+motr5WqRqyEiIro3DDekE9zXAf5udlDXabHzSJbY5RAREd0ThhvSkUgkutGb/xzNRll1rcgVERER6Y/hhhqZMsgZ/ZysUK6uwyc/Z4tdDhERkd4YbqgRqVSChTdHb3YczkJ1rUbkioiIiPTDcENNhPu7oredOYoqavBFaq7Y5RAREelF1HCTlJSE8PBwuLq6QiKRYN++fXfdJzExEYGBgVAoFPD29saHH37Y8YV2MyYyKRaM8wYAbEnMRK1GK3JFREREbSdquKmsrIS/vz/ef//9NrXPysrCtGnTEBoairS0NKxatQpLlizBnj17OrjS7mdmkBscrUyRV3oD357IF7scIiKiNpOL+eFTp07F1KlT29z+ww8/hLu7OzZu3AgAGDRoEFJTU/H222/j0UcfbXYftVoNtfrWnC1lZWX3VXN3oTCR4akxXvjHD+exOSEDM4b1hlQqEbssIiKiuzKoe26OHj2KsLCwRtsefPBBpKamora2+ceWo6OjYWtrq3u5ubl1RqlGYXawB6zN5LhYWIEfz10VuxwiIqI2MahwU1BQAGdn50bbnJ2dUVdXh6Kiomb3WblyJVQqle6Vm8sbZNvKRmGC2cEeAICYhAwIgiByRURERHdnUOEGqJ9o7nYNX7h3bm9gZmYGGxubRi9qu6fHesFMLkV6bimOZhaLXQ4REdFdGVS46dWrFwoKChptKywshFwuh4ODg0hVGTdHKzNEjKi/lBdzKEPkaoiIiO7OoMJNcHAw4uPjG207cOAAgoKCYGJiIlJVxu+ZUG/IpBIcvlSEk1dKxS6HiIioVaKGm4qKCqSnpyM9PR1A/aPe6enpyMnJAVB/v8ycOXN07SMjI5GdnY0VK1bg3Llz2LFjB7Zv344XX3xRjPK7DTd7Czzk7wqAozdERNT1iRpuUlNTERAQgICAAADAihUrEBAQgDVr1gAAlEqlLugAgJeXF+Li4pCQkIBhw4bh9ddfx6ZNm1p8DJzaT+TNJRl+OFuAS4UVIldDRETUMonQzR6BKSsrg62tLVQqFW8u1tOzu1Jx4OxV/F9gH7z9Z3+xyyEiom5En+9vg7rnhsS1aIIPAGBfWh7ySm+IXA0REVHzGG6ozYa52SGkrwPqtAI+SsoUuxwiIqJmMdyQXhaNrx+9+fzXHBRXqO/SmoiIqPMx3JBexvg4wK+PLaprtdh55LLY5RARETXBcEN6kUgkWHTzyal/H72M8urm1/QiIiISC8MN6S1scC/07WmJ8uo6fPpLzt13ICIi6kQMN6Q3qVSChTfvvdmWnIXqWo3IFREREd3CcEP35KFhruhtZ46iCjW+PH5F7HKIiIh09A43N27cQFVVle737OxsbNy4EQcOHGjXwqhrM5FJ8UyoFwBga1IG6jRakSsiIiKqp3e4eeihh7Br1y4AQGlpKUaNGoV33nkHDz30EDZv3tzuBVLXFTHCHQ6WpsgtuYHvTirFLoeIiAjAPYSb3377DaGhoQCAr776Cs7OzsjOzsauXbuwadOmdi+Qui5zUxmeGuMJANickAGttlut5EFERF2U3uGmqqoK1tbWAIADBw7gkUcegVQqxejRo5Gdnd3uBVLXNjvYE1Zmcpy/Wo6DvxeKXQ4REZH+4cbHxwf79u1Dbm4ufvjhB4SFhQEACgsLuRBlN2RrboK/jPYAAMQkXEI3W4eViIi6IL3DzZo1a/Diiy/C09MTo0aNQnBwMID6UZyAgIB2L5C6vqfHesJULsVvOaX4JatE7HKIiKib0zvc/N///R9ycnKQmpqK77//Xrd90qRJeO+999q1ODIMTtYKzAzqAwD44NAlkashIqLu7p7muenVqxcCAgIglUpRVlaGffv2wdraGgMHDmzv+shALHigL2RSCZIvFuHUFZXY5RARUTemd7iZOXMm3n//fQD1c94EBQVh5syZ8PPzw549e9q9QDIMbvYWCPdzAQBsTuToDRERiUfvcJOUlKR7FPzrr7+GIAgoLS3Fpk2b8MYbb7R7gWQ4GpZk+N/pAmRcqxC5GiIi6q70DjcqlQr29vYAgO+//x6PPvooLCwsMH36dFy8eLHdCyTDMaCXNSYPcoYgAFsSM8Quh4iIuim9w42bmxuOHj2KyspKfP/997pHwa9fvw6FQtHuBZJhWTShLwDg67Q85JfeELkaIiLqjvQON8uWLcMTTzyBPn36wNXVFePHjwdQf7lq6NCh7V0fGZjh7j0w2tsetRoBHyVnil0OERF1Q3qHm0WLFuHo0aPYsWMHDh8+DKm0/hDe3t6854YAAItu3nvz+bFclFTWiFwNERF1NxLhPqaUbdhVIpG0W0EdraysDLa2tlCpVJxRuYMIgoDw9w/jdF4Zlkz0wYqwAWKXREREBk6f7+97mudm165dGDp0KMzNzWFubg4/Pz/85z//uadiyfhIJBLd6M3HKZdRoa4TuSIiIupO9A437777LhYuXIhp06bhiy++QGxsLP7whz8gMjKSMxSTzoNDesG7pyXKquvw2S9cUJWIiDqP3pelvLy8sG7dOsyZM6fR9n//+99Yu3YtsrKy2rXA9sbLUp3ni9RcvPzVSThZmyHp5QlQmMjELomIiAxUh16WUiqVCAkJabI9JCQESqVS38OREZsxrDdcbBUoLFdjz29XxC6HiIi6Cb3DjY+PD7744osm22NjY9GvX792KYqMg6lcimdCvQEAWxIzUafRilwRERF1B3J9d1i3bh0iIiKQlJSEMWPGQCKR4PDhw/jpp5+aDT3UvT020g3/OngROSVV2H9KiYeG9Ra7JCIiMnJ6j9w8+uij+OWXX+Do6Ih9+/Zh7969cHR0xLFjx/Dwww93RI1kwCxM5XhqjBcAYHNCBu5j5gEiIqI2ua95bgwRbyjufKqqWoT8/SdU1miw48kgTBzoLHZJRERkYPT5/m7TZamysrI2fzgDA93J1sIEfxntgS1JmYg5lMFwQ0REHapN4cbOzu6usxALggCJRAKNRtMuhZFxmTfWCztTLiM1+zqOZZVgpJe92CUREZGRalO4OXToUEfXQUbOyUaB/wvsg89+ycEHhy5hpNdIsUsiIiIj1aZwM27cuI6ug7qBBQ944/NjOUi8cA2n81Tw7W0rdklERGSE7mltKaJ74eFgiT/6uQIANidmiFwNEREZK4Yb6lQLx/cFAPzvlBJZRZUiV0NERMZI9HATExMDLy8vKBQKBAYGIjk5udX2n376Kfz9/WFhYQEXFxc89dRTKC4u7qRq6X4NcrHBpIFO0ArAFo7eEBFRBxA13MTGxmLZsmVYvXo10tLSEBoaiqlTpyInJ6fZ9ocPH8acOXMwb948nDlzBl9++SV+/fVXzJ8/v5Mrp/uxaEL96M2e366gQFUtcjVERGRs7inc1NXV4ccff8SWLVtQXl4OAMjPz0dFRYVex3n33Xcxb948zJ8/H4MGDcLGjRvh5uaGzZs3N9v+559/hqenJ5YsWQIvLy+MHTsWCxYsQGpq6r10g0QS6GGPkV72qNUI+Cg5U+xyiIjIyOgdbrKzszF06FA89NBDWLx4Ma5duwYA2LBhA1588cU2H6empgbHjx9HWFhYo+1hYWFISUlpdp+QkBBcuXIFcXFxEAQBV69exVdffYXp06e3+DlqtRplZWWNXiS+RTfvvdl9LAfXK2tEroaIiIyJ3uFm6dKlCAoKwvXr12Fubq7b/vDDD+Onn35q83GKioqg0Wjg7Nx4tlpnZ2cUFBQ0u09ISAg+/fRTREREwNTUFL169YKdnR3+9a9/tfg50dHRsLW11b3c3NzaXCN1nHH9e2KIqw2qajT4OOWy2OUQEZER0TvcHD58GK+88gpMTU0bbffw8EBeXp7eBdw583HDTMfNOXv2LJYsWYI1a9bg+PHj+P7775GVlYXIyMgWj79y5UqoVCrdKzc3V+8aqf1JJBLdk1Mfp1xGpbpO5IqIiMhYtGkSv9tptdpml1i4cuUKrK2t23wcR0dHyGSyJqM0hYWFTUZzGkRHR2PMmDF46aWXAAB+fn6wtLREaGgo3njjDbi4uDTZx8zMDGZmZm2uizrPVF8XeDleQFZRJXYfy8H8UG+xSyIiIiOg98jNlClTsHHjRt3vEokEFRUVeO211zBt2rQ2H8fU1BSBgYGIj49vtD0+Ph4hISHN7lNVVQWptHHJMpkMQP2IDxkWmVSCyHH1geaj5Eyo67guGRER3T+9w817772HxMREDB48GNXV1Zg1axY8PT2Rl5eHt956S69jrVixAtu2bcOOHTtw7tw5LF++HDk5ObrLTCtXrsScOXN07cPDw7F3715s3rwZmZmZOHLkCJYsWYKRI0fC1dVV365QF/BwQB/0slHgapkae3/T/7ImERHRnfS+LOXq6or09HTs3r0bv/32G7RaLebNm4cnnnii0Q3GbREREYHi4mKsX78eSqUSvr6+iIuLg4eHBwBAqVQ2mvPmySefRHl5Od5//3288MILsLOzw8SJE/UOVdR1mMqlmB/qhTf2n8OWxAzMDHKDTNr6CvREREStkQjd7HpOWVkZbG1toVKpYGNjI3Y5BKBSXYcxbx1EaVUt/vV4AML9OQpHRESN6fP9rffIzTfffNPsdolEAoVCAR8fH3h5eel7WOrGLM3keDLEExt/vIiYhAz80c+lxSfmiIiI7kbvcDNjxgxIJJImN/A2bJNIJBg7diz27duHHj16tFuhZNyeDPHE1qRMnFOWIeHCNUwY4CR2SUREZKD0vqE4Pj4eI0aMQHx8vG7umPj4eIwcORLfffcdkpKSUFxcrNdsxUR2FqZ4YpQ7AGDzIS6oSURE907vkZulS5di69atjR7XnjRpEhQKBZ599lmcOXMGGzduxNNPP92uhZLxmx/qjX+nZOPY5RL8erkEIzztxS6JiIgMkN4jNxkZGc3eyGNjY4PMzPpFEPv164eioqL7r466FWcbBR4N7A0AiDl0SeRqiIjIUOkdbgIDA/HSSy/pFswEgGvXruHll1/GiBEjAAAXL15Enz592q9K6jYWPNAXUglw6Pw1nM3nIqdERKQ/vcPN9u3bkZWVhT59+sDHxwf9+vVDnz59cPnyZWzbtg0AUFFRgVdffbXdiyXj5+loiWlD65fR2JzIe2+IiEh/9zTPjSAI+OGHH3DhwgUIgoCBAwdiypQpTZZG6Io4z03XdyZfhembDkMqAQ6+MB6ejpZil0RERCLT5/ubk/hRl/TUzmM4dP4aHh/pjuhHhopdDhERiaxDJ/EDgMrKSiQmJiInJwc1NTWN3luyZMm9HJKokUUTfHDo/DXsOX4Fyyb3g7ONQuySiIjIQOgdbtLS0jBt2jRUVVWhsrIS9vb2KCoqgoWFBZycnBhuqF2M8LTHCM8e+PXydWxLzsTq6YPFLomIiAyE3jfJLF++HOHh4SgpKYG5uTl+/vlnZGdnIzAwEG+//XZH1Ejd1KLxPgCAT3/JQWlVzV1aExER1dM73KSnp+OFF16ATCaDTCaDWq2Gm5sbNmzYgFWrVnVEjdRNjR/QE4NcbFBVo8G/U7LFLoeIiAyE3uHGxMREt6ihs7MzcnJyAAC2tra6n4nag0QiwcLxfQEAH6dkoaqmTuSKiIjIEOgdbgICApCamgoAmDBhAtasWYNPP/0Uy5Ytw9ChfKqF2tf0oS7wdLDA9apa7D6WK3Y5RERkAPQON2+++SZcXOonWXv99dfh4OCAhQsXorCwEFu3bm33Aql7k0klWDCufvTmo6RM1NRpRa6IiIi6Or2elhIEAT179sSQIUMAAD179kRcXFyHFEbU4JHhvbHxxwsoKKvG12lXEDHCXeySiIioC9Nr5EYQBPTr1w9XrlzpqHqImjCTyzB/rDcA4MPETGi03WreSSIi0pNe4UYqlaJfv34oLi7uqHqImvX4KHfYmpsgq6gS358uELscIiLqwvS+52bDhg146aWXcPr06Y6oh6hZVmZyzA3xBADEJFxCN1s1hIiI9KD32lI9evRAVVUV6urqYGpqCnNz80bvl5SUtGuB7Y1rSxmu65U1GPPWwfp5b54eiXH9e4pdEhERdZIOXVtq48aN91oX0X3pYWmKx0e6Y/vhLMQcusRwQ0REzdI73MydO7cj6iBqk/mhXth19DJ+ySrB8ewSBHrYi10SERF1MXrfcwMAGRkZeOWVV/D444+jsLAQAPD999/jzJkz7Voc0Z1cbM3xSEAfAEDMoQyRqyEioq5I73CTmJiIoUOH4pdffsHevXtRUVEBADh58iRee+21di+Q6E4LxnlDIgF++r0QvxeUiV0OERF1MXqHm6ioKLzxxhuIj4+HqampbvuECRNw9OjRdi2OqDnePa0wzbd+luzNCRy9ISKixvQON6dOncLDDz/cZHvPnj05/w11moYFNb89kY+c4iqRqyEioq5E73BjZ2cHpVLZZHtaWhp69+7dLkUR3Y1vb1uM698TWgHYksTRGyIiukXvcDNr1iz89a9/RUFBASQSCbRaLY4cOYIXX3wRc+bM6YgaiZq16ObozZfHr6CwrFrkaoiIqKvQO9z87W9/g7u7O3r37o2KigoMHjwYDzzwAEJCQvDKK690RI1EzRrpZY9Ajx6oqdNi++EsscshIqIuQu8ZihtkZGQgLS0NWq0WAQEB6NevX3vX1iE4Q7Fx+encVcz7dyosTWVIiZoEWwsTsUsiIqIO0KEzFCcmJmLcuHHo27cv+vbte89FErWHiQOdMLCXNX4vKMeuo5fx/CTDCNlERNRx9L4sNWXKFLi7uyMqKoqLZ5LoJBKJ7smpnSmXcaNGI3JFREQkNr3DTX5+Pl5++WUkJyfDz88Pfn5+2LBhA65cudIR9RHd1fShLnC3t0BJZQ0+/zVH7HKIiEhkeocbR0dHPPfcczhy5AgyMjIQERGBXbt2wdPTExMnTuyIGolaJZdJsWCcNwDgo6RM1NRpRa6IiIjEdE9rSzXw8vJCVFQU/v73v2Po0KFITExsr7qI9PLo8D7oaW2GfFU19qXniV0OERGJ6J7DzZEjR7Bo0SK4uLhg1qxZGDJkCL777rv2rI2ozRQmMswf6wUA+DAxAxrtPT0ESERERkDvcLNq1Sp4eXlh4sSJyM7OxsaNG1FQUIBPPvkEU6dO7YgaidrkidEesFHIkXmtEgfOFIhdDhERiUTvcJOQkIAXX3wReXl52L9/P2bNmgULC4t7LiAmJgZeXl5QKBQIDAxEcnJyq+3VajVWr14NDw8PmJmZoW/fvtixY8c9fz4ZDyszOeaGeAIAYhIycI9TOBERkYHTe56blJSUdvvw2NhYLFu2DDExMRgzZgy2bNmCqVOn4uzZs3B3d292n5kzZ+Lq1avYvn07fHx8UFhYiLq6unariQzbU2O8sC05C6fyVDh8qQih/XqKXRIREXWye56h+OzZs8jJyUFNTU2j7X/605/afIxRo0Zh+PDh2Lx5s27boEGDMGPGDERHRzdp//333+Oxxx5DZmYm7O3t2/QZarUaarVa93tZWRnc3Nw4Q7ERW/ftGew8chnB3g7Y/exoscshIqJ20KEzFGdmZuLhhx/GqVOnIJFIdEP/EokEAKDRtG0StZqaGhw/fhxRUVGNtoeFhbU4OvTNN98gKCgIGzZswH/+8x9YWlriT3/6E15//XWYm5s3u090dDTWrVvX1u6REXgm1Buf/JyNo5nF+C3nOoa79xC7JCIi6kR633OzdOlSeHl54erVq7CwsMCZM2eQlJSEoKAgJCQktPk4RUVF0Gg0cHZ2brTd2dkZBQXN3wyamZmJw4cP4/Tp0/j666+xceNGfPXVV1i8eHGLn7Ny5UqoVCrdKzc3t801kmFytTPHjGG9AQAxhzJEroaIiDqb3uHm6NGjWL9+PXr27AmpVAqpVIqxY8ciOjoaS5Ys0buAhhGfBoIgNNnWQKvVQiKR4NNPP8XIkSMxbdo0vPvuu/j4449x48aNZvcxMzODjY1NoxcZv8jxfSGRAD+eu4rzBeVil0NERJ1I73Cj0WhgZWUFoH624vz8fACAh4cHzp8/3+bjODo6QiaTNRmlKSwsbDKa08DFxQW9e/eGra2tbtugQYMgCAKXf6BG+va0wh+G9AJQP+8NERF1H3qHG19fX5w8eRJA/Q3BGzZswJEjR7B+/Xp4e3u3+TimpqYIDAxEfHx8o+3x8fEICQlpdp8xY8YgPz8fFRUVum0XLlyAVCpFnz599O0KGblF430AAN+cyEduSZXI1RARUWfRO9y88sor0Grr1+554403kJ2djdDQUMTFxWHTpk16HWvFihXYtm0bduzYgXPnzmH58uXIyclBZGQkgPr7ZebMmaNrP2vWLDg4OOCpp57C2bNnkZSUhJdeeglPP/10izcUU/c1tI8tQvs5QqMVsDUpU+xyiIiok+j9tNSDDz6o+9nb2xtnz55FSUkJevTo0eK9Mi2JiIhAcXEx1q9fD6VSCV9fX8TFxcHDwwMAoFQqkZNza5VnKysrxMfH4/nnn0dQUBAcHBwwc+ZMvPHGG/p2g7qJReN9kHyxCLGpuXh+kg+crBVil0RERB3snue5MVT6PCdPhk8QBDyyOQVpOaWIHNcXUVMHil0SERHdA32+v+9rVXCirk4ikejuvfnk52yobtSKXBEREXU0hhsyepMGOqG/sxUq1HX45OdsscshIqIOxnBDRk8qlWDh+L4AgB2Hs3Cjpm2zaBMRkWFiuKFuIdzPFX16mKO4sgZfpHKWaiIiY8ZwQ92CXCbFgnH1ozdbkzJRq9GKXBEREXUUhhvqNv4c2AeOVmbIK72Bb9LzxS6HiIg6CMMNdRsKExnmjfUCAGxOzIBW261mQSAi6jYYbqhb+ctod1gr5LhUWIEDZ6+KXQ4REXUAhhvqVqwVJpgTXD8D9uaES+hmc1gSEXULDDdk1GJiYuDl5QWFQoHAwEAkJyfjqTFeUJhIceKKCikZxc3ud+TIEcjlcgwbNqzJexs3bsSAAQNgbm4ONzc3LF++HNXV1R3cEyIiaiuGGzJasbGxWLZsGVavXo20tDSEhoZi6tSpqCq5isdGuAMAYhIuNdlPpVJhzpw5mDRpUpP3Pv30U0RFReG1117DuXPnsH37dsTGxmLlypUd3h8iImobhhsyWu+++y7mzZuH+fPnY9CgQdi4cSPc3NywefNmPPOAN+RSCY5cKkZ6bmmj/RYsWIBZs2YhODi4yTGPHj2KMWPGYNasWfD09ERYWBgef/xxpKamdlKviIjobhhuyCjV1NTg+PHjCAsLa7Q9LCwMKSkp6G1njoeG9QYAxBy6NXqzc+dOZGRk4LXXXmv2uGPHjsXx48dx7NgxAEBmZibi4uIwffr0DuoJERHpSy52AUQdoaioCBqNBs7Ozo22Ozs7o6CgAACwcLw39qZdwYGzV3HxajlQVoCoqCgkJydDLm/+P43HHnsM165dw9ixYyEIAurq6rBw4UJERUV1eJ+IiKhtOHJDRk0ikTT6XRAE3TYfJ2uEDa4PPx8cuoBZs2Zh3bp16N+/f4vHS0hIwN/+9jfExMTgt99+w969e/Hdd9/h9ddf77hOEBGRXjhyQ0bJ0dERMplMN0rToLCwsNFozqLxPvjhzFX891gGLqemIi0tDc899xwAQKvVQhAEyOVyHDhwABMnTsSrr76K2bNnY/78+QCAoUOHorKyEs8++yxWr14NqZT/f4GISGz8l5iMkqmpKQIDAxEfH99oe3x8PEJCQnS/+7vZYayPI7Qm5oj8516kp6frXpGRkRgwYADS09MxatQoAEBVVVWTACOTySAIAufMISLqIjhyQ0ZrxYoVmD17NoKCghAcHIytW7ciJycHkZGRAICVK1ciLy8Pi9a8i8OXinCwUIHXPfvB0coMAODk5ASFQgFfX1/dMcPDw/Huu+8iICAAo0aNwqVLl/Dqq6/iT3/6E2QymSj9JCKixhhuyGhFRESguLgY69evh1KphK+vL+Li4uDhUT9DsVKpRE5ODoL7OsDfzQ4nckux6ceL+MPQXvBytGz2mK+88gokEgleeeUV5OXloWfPnggPD8ff/va3zuwaERG1QiJ0s7H0srIy2NraQqVSwcbGRuxyqIv44UwBFvznuO53qQSIfmQoIm5O9kdEROLS5/ub99wQAfB1bfwfilYAVu09BaXqhkgVERHRvWK4IQKQXVLVZJtGAN6M+x2n81S8WZiIyIDwnhsiAF6OlpBK6kdsbvftiXx8eyIf3o6W+KO/K8L9XNDP2VqcIomIqE14zw3RTbG/5mDV3tPQCAKkEuDxke4oqazBwd8Loa7T6toN7GWNcH9X/NHPBR4Ozd94TERE7Uuf72+GG6LbKFU3cLmoCp6OFnCxNQcAVKjr8OPZq/j2RD6SLl5DrebWfzJ+fWwR7ueK6X4ucLUzF6tsIiKjx3DTCoYbuh+qqlr8cKYA357MR0pGMTS3XccK8uiBcH9XTB3aC07WChGrJCIyPgw3rWC4ofZSVKHG/04X4NsT+fj1cgka/kuSSoDR3g4I93fFH4b0Qg9LU3ELJSIyAgw3rWC4oY5QoKrG/lNKfHsiH+m5pbrtcqkEY/s5ItzPFVOGOMNGYSJekUREBozhphUMN9TRckuq8N1JJb47mY8z+WW67aYyKcYP6Ilwf1dMGuQEC1M+rEhE1FYMN61guKHOlHGtAt+dUOLbk/m4VFih225uIsOkQU4I93fFuP49oTDhulRERK1huGkFww2JQRAEnL9ajm9P5OO7k0pkF9+aNNDaTI4pQ5wR7ueKsf0cYSLj3JpERHdiuGkFww2JTRAEnMpT6YKOUlWte8/OwgRTfXsh3M8Vo7wdIJNKRKyUiKjrYLhpBcMNdSVarYDfcq7j2xP52H+qAEUVat17jlZmmD60F/7o74pA9x6QMugQUTfGcNMKhhvqqjRaAb9kFuPbk/n43+kClFbV6t5zsVXgj34uCPd3xdDetpBIGHSIqHthuGkFww0ZglqNFocvFeHbE/mIP3MV5eo63Xvu9hYI93fBH/1cMbCXNYMOEXULDDetYLghQ1Ndq0HihWv49kQ+fjpXiBu1Gt17Pk5WCPdzxR/9XdC3p5WIVRIRdSyGm1Yw3JAhq6qpw0/nCvHtiXwkXLiGmtsW9BzsYqNb0NPN3kLEKomI2p8+39+iP3MaExMDLy8vKBQKBAYGIjk5uU37HTlyBHK5HMOGDevYAom6EAtTOcL9XbF1ThBSX5mMd/7sj/EDekIuleCssgxvff87QjccwowPjmD74SwU3PYkFhFRdyHqyE1sbCxmz56NmJgYjBkzBlu2bMG2bdtw9uxZuLu7t7ifSqXC8OHD4ePjg6tXryI9Pb3Nn8mRGzJG1ytr8P2Z+nWufs4sRsN6nhIJMMLTvn5BT99ecLQyE7dQIqJ7ZDCXpUaNGoXhw4dj8+bNum2DBg3CjBkzEB0d3eJ+jz32GPr16weZTIZ9+/Yx3BDdprC8Gv87VR90UrOv67bLpBKE9HVAuJ8rHhzSC7YWXOeKiAyHPt/foi1uU1NTg+PHjyMqKqrR9rCwMKSkpLS4386dO5GRkYFPPvkEb7zxxl0/R61WQ62+NXdIWVlZK62JDJ+TtQJzQzwxN8QT+aU3sP9k/fIPJ6+okHyxCMkXi7B63yk80K9+navJg51hZcZ1rojIeIj2L1pRURE0Gg2cnZ0bbXd2dkZBQUGz+1y8eBFRUVFITk6GXN620qOjo7Fu3br7rpfIELnameOZB7zxzAPeyC6uxHcn61cu/72gHD/9Xoiffi+EmVyKiQPr17maMMAJ5qZc54qIDJvo/3ftzjk6BEFodt4OjUaDWbNmYd26dejfv3+bj79y5UqsWLFC93tZWRnc3NzuvWAiA+XhYInFE3yweIIPLl4tx7cnlfjuRD4yiyrxv9MF+N/pAliYyjBlcP06V6H9HWEmZ9AhIsMj2j03NTU1sLCwwJdffomHH35Yt33p0qVIT09HYmJio/alpaXo0aMHZLJb/9hqtVoIggCZTIYDBw5g4sSJd/1c3nNDdIsgCDirLMO3J+pHdPJKb+jes1bI8YchvRDu74qQvg6Qc0FPIhKRQd1QHBgYiJiYGN22wYMH46GHHmpyQ7FWq8XZs2cbbYuJicHBgwfx1VdfwcvLC5aWlnf9TIYbouYJgoD03FJ8e0KJ/afycbXs1r1q9pam9Qt6+rtihKc9F/Qkok5nEDcUA8CKFSswe/ZsBAUFITg4GFu3bkVOTg4iIyMB1F9SysvLw65duyCVSuHr69tofycnJygUiibbiUh/EokEAe49EODeA69MH4RfL5fg25P5iDtVgJLKGnz6Sw4+/SUHTtZmmH5znasANzsu/0BEXY6o4SYiIgLFxcVYv349lEolfH19ERcXBw8PDwCAUqlETk6OmCUSdUtSqQSjvB0wytsBa8OH4GhmMb49kY/vTxegsFyNnUcuY+eRy+htZ44/+rsg3M8VQ1xtGHSIqEvg8gtE1GY1dVokX6xf5yr+7FVU1txa58rL0RLhN0d0+jlbi1glERkjg1p+gYgMh6lcikmDnLHxsQAcf3UKNj8xHNOG9oKZXIqsokpsOngJU95LwoPvJeH9gxdxuajyvj9TnyVaDh8+jDFjxsDBwQHm5uYYOHAg3nvvvUZtPvroI4SGhqJHjx7o0aMHJk+ejGPHjt13nUTUdXDkhojuW4W6Dj+du4pvT+Qj8cI11Gpu/bMytLctwv1dMN3PFb3tzPU6rr5LtKSlpeH333+Hn58fLC0tcfjwYSxYsADvvfcenn32WQDAE088gTFjxiAkJAQKhQIbNmzA3r17cebMGfTu3fv+/iKIqMMYzNNSYmC4IepYqqpa/HC2fvmHlIxiaLS3/okJ9OiBcD8XTPNzgZO14q7HutclWm73yCOPwNLSEv/5z3+afV+j0aBHjx54//33MWfOnDYdk4g6n8E8LUVExsfWwgQzg9wwM8gNxRVq/O90fdA5drkEx7Ov43j2daz/7ixGeTkg3N8Vf/DtBXtL0ybHudclWm6XlpaGlJSUVpdqqaqqQm1tLezt7fXrKBF1WQw3RNRhHKzM8JfRHvjLaA9cLavWrXOVllOKo5nFOJpZjDX/PY0xPo4I93dF2BBn2CjqF/S8lyVaGvTp0wfXrl1DXV0d1q5di/nz57fYNioqCr1798bkyZPvv8NE1CUw3BBRp3C2UeDpsV54eqwXckuqsP9U/azIZ/LLkHjhGhIvXIPpXinGDahf0HOIbR2Ati/Rcrvk5GRUVFTg559/RlRUFHx8fPD44483abdhwwbs3r0bCQkJUCjufpmMiAwDww0RdTo3ewtEjuuLyHF9kXmtAt+dVOKbE/m4VFiB+LNXEX/2KhRSLSRSGeKOnUNA0EgoTOqXXiksLGwymnMnLy8vAMDQoUNx9epVrF27tkm4efvtt/Hmm2/ixx9/hJ+fX8d0lIhEwXBDRKLy7mmFJZP64fmJPjh/tRzfnai/dJVdXAUT577Y+O89+LqkN8IGOyPc3xUH4uMx46GH2nx8QRCgVqsbbfvHP/6BN954Az/88AOCgoLau0tEJDKGGyLqEiQSCQb2ssHAXjZ4Iaw/TuWp8CYW4Iu3/4qCXj6IzRuEbW9/j8qMLJR6jMORS0X4ZtvbUObnY9euXQCADz74AO7u7hg4cCCA+nlv3n77bTz//PO6z9mwYQNeffVVfPbZZ/D09NTdv2NlZQUrK6vO7zgRtTs+Ck5EXdoHH3yAv0W/hcKrBTBz8oT1+HlQuNWvJ1dx4J+wqrmOL7/7AYHuPfDm2+9hx7aPUJCXAxO5HH379sUzzzyDBQsWQCqtn7PU09MT2dnZTT7ntddew9q1azuza0SkB85z0wqGGyLDpdEK+CWrGN+eUOJ/p5UorarVvWdrLkfZjToIAKQS4M2Hh+KxkU0n+iMiw8Rw0wqGGyLjUKvR4vClInx3QonvTysbrXPVwMPeHO4OlnC1NYeLnUL3p4utOVztFLAw5ZV5IkPBcNMKhhsi45NwvhBP7vxV7/1szU3gYquAq515oz8bwk8vWwXM5LIOqJiI9MUZiomoWxnQyxpSCXDbSg+QSoD3IoZBXaeFsrQaStUN5KuqoSy9AaWqGhXqOqhu1EJ1oxa/F5S3eGxHKzO42ikahZ7b/3SyNoNcxjWIWxMTE4N//OMfUCqVGDJkCDZu3IjQ0NBm2+7duxebN29Geno61Go1hgwZgrVr1+LBBx/UtRk/fjwSExOb7Dtt2jTs37+/w/pBhoPhhogMnoutOaIfGYpVe09DIwiQSSR48xFfPDSs5YUwy6proSytRr7qxq3wc/NPpaoa+aU3oK7ToqhCjaIKNU5eUTV7HJlUAidrs/rwY2cOV10IuhWAHCxNIZW2PvGgsYqNjcWyZcsaLX46derUFhc/TUpKwpQpU/Dmm2/Czs4OO3fuRHh4OH755RcEBAQAqA9ANTU1un2Ki4vh7++PP//5z53WL+raeFmKiIyGUnUDl4uq4OloARdb/VYgv5MgCCiprNEFHaWqaRC6WlaNOu3d/wk1lUnRy1bR+NJXoyCkgK25yV1nXjZE7bH46ZAhQxAREYE1a9Y0+/7GjRuxZs0aKJVKWFpatkvd1PXwshQRdUsutub3HWoaSCQSOFiZwcHKDL69bZtto9EKKKpQ3wo/N/9Uqm4gr7T+Eti1CjVqNFrklFQhp6Sqxc8zN5Hpbnq+89JXw5+WZob1T3Z7LH6q1WpRXl7e6sKm27dvx2OPPcZgQzqG9V8KEVEXIpNK4GyjgLONAgEttKmp0+JqWbUu9OQ3cwmspLIGN2o1yLxWicxrlS1+no1C3uzIT0Mo6mWr0C1T0RXcz+KnDd555x1UVlZi5syZzb5/7NgxnD59Gtu3b7/vesl4MNwQEXUgU7kUbvYWcLO3aLFNda2mPvyU1t/0XD8CdFsAKq1GuboOZdV1KCsov8sN0KY3R7CauQRmZw5nEW6AvpfFTwFg9+7dWLt2Lf773//Cycmp2Tbbt2+Hr68vRo4c2S61knFguCEiEpnCRAYvR0t4ObZ8WaW8urbxpa+bQagh/OSrbqC6VouiihoUVdTgVF7zN0BLJYCTteLWvD+3hR9Xu/pRIEdLs3a5AdrR0REymazJKE1bFj+NjY3FvHnz8OWXX2Ly5MnNtqmqqsLnn3+O9evX33etZFwYboiIDIC1wgTWChP0d7Zu9n1BEFBaVYu82+77uX3kJ191A1fLqlGrEVBQVo2CsmqkobTZY5nIJDdvgL414nPnJTA7i7vfAG1qaorAwEDEx8fj4Ycf1m2Pj4/HQ60sfrp79248/fTT2L17N6ZPn95iuy+++AJqtRp/+ctfWq2Duh+GGyIiIyCRSNDD0hQ9LE1bvAFa23AD9O0jP3c8CXa1vD4A5ZbcQG7JjRY/z9xEdnPU57ZH3++4BGZlJseKFSswe/ZsBAUFITg4GFu3bkVOTg4iIyMBACtXrkReXp5u8dPdu3djzpw5+Oc//4nRo0frRn3Mzc1ha9u4X9u3b8eMGTPg4ODQHn+FZET4KDgREenUam7dAN3SJbDiypq7HwiAtUIOV1tzqH77Dud++BSVpdfg3ncgXnz1DUwNmwQXWwUin5mHy5cvIyEhAQAQMvYBHD2S3ORYc+fOxccff6z7/cKFCxgwYAAOHDiAKVOmtEfXqQXtPQnj3r178eabb+LSpUuora1Fv3798MILL2D27Nmt1sHlF1rBcENEdH+qazUouHPen4YwdPMSWHl1XZuOZW9pqnvUvbK6FkczSyAAkEiA5yf64IlRHuhhYQpTOWeBFkNsbCxmz57daBLGbdu2tTgJ47Jly+Dq6ooJEyboJmF8++23G03CmJCQgOvXr2PgwIEwNTXFd999hxdeeAH79+9vFILuxHDTCoYbIqKOV6Gua3Tpq7lLYDdqmy522hIbhbx+3iFLU9hbmup+drCq/93Rykz3s72FKZfEaCedMQkjAAwfPhzTp0/H66+/3mIbTuJHRESisjKTo5+zNfq1cgO06kat7qbnwxeLsDPlcpN2EgACUP8YfHUdsopangfodnYWJvXhx/JW6Gk2EFmaws7CFLJuujxGazpjEkZBEHDw4EGcP38eb7311n3X3IDhhoiIOp1EIoGdRX2wGOxqg8GuNvj30cuNFj+VSSRIenk8LEzlKK5Uo7iiBsWVN18VapRU1tzcduu961U1EASgtKoWpVW1yGhlUsRbtQD2Fg0BqIVAdNt7tuYm3WKtsI6chFGlUqF3795Qq9WQyWSIiYlp13unGG6IiEh0LS1+2rtH/eSHPSxN4dP8PH6NaLQCrlfVoKSyBkWNAlDjQFRUWf9zaVUtBAG60HSx8O6fIZNK0MPCFI7NBCD728JRw8iRjbncoNcN64hJGK2trZGeno6Kigr89NNPWLFiBby9vTF+/Ph2qZnhhoiIuoSIEe54oH/P+1r8VCaVwNHKDI5WZi3OCXS7Wo0W16vqA0+rgejmz2XVdbo1xYoq1G2qyUQmqb8XyNLsViC6LQDdecnMyqxrhKGOnIRRKpXCx8cHADBs2DCcO3cO0dHRDDdERGR82nPx07YwkUnhZK2Ak7WiTe1r6rQ3w4665UBUeev3CnUdajUCrpapcbWsbWHIVC697d4gMzi2chO1vaUpLExlHRKGOnoSxtsJggC1um1/P23BcENERNRGpnIpetkq0Mu2bWGoulbT5N6gksqbl8XuuIeouKJ+AdWaOu3NWaar2/QZChPpHSNBZk0vmd0WiPRZXLUjJmGMjo5GUFAQ+vbti5qaGsTFxWHXrl2Nnsi6Xww3REREHURhIqufvdmubaNRVTV1ugB0+43Sd44QNfyurtOiulaLvNIbyCtteUbp21mYyhqNCul+biYQzXj0/7CxuBjr16+HUqmEr68v4uLi4OHhAQBQKpXIycnRHXvLli2oq6vD4sWLsXjxYt322ydhrKysxKJFi3DlyhWYm5tj4MCB+OSTTxAREdHGv9W74zw3REREBkgQBFTWaFBScftIkPrmSFBzl8zUqNXo/5VvbSa/eaP0nSNBTe8h+uncVbyy7zS0Qv0irdGPDEXEiKaT/d0LznNDRERk5CQSCazM5LAyk8PdweKu7QVBQLm6YWRIjaKGEaKK5gNRSWUN6rT1+5Sr65BdXKVXfVoBWLX3NB7o37NT76MCGG6IiIi6BYlEAhuFCWwUJvBytLxre61WQFl17W3Bp+VA1HAJ7c5xIY0g4HJRFcMNERERiU8qvTXRYt+ed2+fd70KoRsONZmI0dPx7qNK7Y2LbxAREdF9693DAtGPDIXs5mPpDRMxdvaoDcCRGyIiImon7TERY3sQfeQmJiYGXl5eUCgUCAwMRHJycott9+7diylTpqBnz56wsbFBcHAwfvjhh06sloiIiFrjYmuO4L4OogUbQORwExsbi2XLlmH16tVIS0tDaGgopk6d2uiZ+dslJSVhypQpiIuLw/HjxzFhwgSEh4cjLS2tkysnIiKirkrUeW5GjRqF4cOHN5qVcNCgQZgxYwaio6PbdIwhQ4YgIiICa9asafZ9tVrdaErnsrIyuLm5cZ4bIiIiA6LPPDeijdzU1NTg+PHjCAsLa7Q9LCwMKSkpbTqGVqtFeXk57O3tW2wTHR0NW1tb3cvNze2+6iYiIqKuTbRwU1RUBI1G02RlUWdn5yYrkLbknXfeQWVlJWbOnNlim5UrV0KlUuleubm591U3ERERdW2iPy1150qmgiC0aXXT3bt3Y+3atfjvf/8LJyenFtuZmZnBzMzsvuskIiIiwyBauHF0dIRMJmsySlNYWNhkNOdOsbGxmDdvHr788ktMnjy5I8skIiIiAyPaZSlTU1MEBgYiPj6+0fb4+HiEhIS0uN/u3bvx5JNP4rPPPsP06dM7ukwiIiIyMKJellqxYgVmz56NoKAgBAcHY+vWrcjJyUFkZCSA+vtl8vLysGvXLgD1wWbOnDn45z//idGjR+tGfczNzWFraytaP4iIiKjrEDXcREREoLi4GOvXr4dSqYSvry/i4uLg4eEBAFAqlY3mvNmyZQvq6uqwePFiLF68WLd97ty5+Pjjjzu7fCIiIuqCRJ3nRgz6PCdPREREXYNBzHNDRERE1BEYboiIiMioMNwQERGRUWG4ISIiIqPCcENERERGheGGiIiIjArDDRERERkVhhsiIiIyKgw3REREZFQYboiIiMioMNwQERGRUWG4ISIiIqPCcENERERGheGGiIiIjArDDRERERkVhhsiIiIyKgw3REREZFQYboiIiMioMNwQERGRUWG4ISIiIqPCcENERERGheGGiIiIjArDDRERERkVhhsiIiIyKgw3REREZFQYboiIiMioMNwQERGRUWG4ISIiIqPCcENERERGheGGiIiIjArDDRERERkVhhsiIiIyKgw3REREZFQYboiIiMioMNwQERGRUWG4ISIiIqPCcENERERGRfRwExMTAy8vLygUCgQGBiI5ObnV9omJiQgMDIRCoYC3tzc+/PDDTqqUiIiIDIGo4SY2NhbLli3D6tWrkZaWhtDQUEydOhU5OTnNts/KysK0adMQGhqKtLQ0rFq1CkuWLMGePXs6uXIiIiLqqiSCIAhiffioUaMwfPhwbN68Wbdt0KBBmDFjBqKjo5u0/+tf/4pvvvkG586d022LjIzEiRMncPTo0TZ9ZllZGWxtbaFSqWBjY3P/nSAiIqIOp8/3t7yTamqipqYGx48fR1RUVKPtYWFhSElJaXafo0ePIiwsrNG2Bx98ENu3b0dtbS1MTEya7KNWq6FWq3W/q1QqAPV/SURERGQYGr632zImI1q4KSoqgkajgbOzc6Ptzs7OKCgoaHafgoKCZtvX1dWhqKgILi4uTfaJjo7GunXrmmx3c3O7j+qJiIhIDOXl5bC1tW21jWjhpoFEImn0uyAITbbdrX1z2xusXLkSK1as0P2u1WpRUlICBweHVj/nXpSVlcHNzQ25ublGecnL2PsHGH8f2T/DZ+x9ZP8MX0f1URAElJeXw9XV9a5tRQs3jo6OkMlkTUZpCgsLm4zONOjVq1ez7eVyORwcHJrdx8zMDGZmZo222dnZ3XvhbWBjY2O0/6MFjL9/gPH3kf0zfMbeR/bP8HVEH+82YtNAtKelTE1NERgYiPj4+Ebb4+PjERIS0uw+wcHBTdofOHAAQUFBzd5vQ0RERN2PqI+Cr1ixAtu2bcOOHTtw7tw5LF++HDk5OYiMjARQf0lpzpw5uvaRkZHIzs7GihUrcO7cOezYsQPbt2/Hiy++KFYXiIiIqIsR9Z6biIgIFBcXY/369VAqlfD19UVcXBw8PDwAAEqlstGcN15eXoiLi8Py5cvxwQcfwNXVFZs2bcKjjz4qVhcaMTMzw2uvvdbkMpixMPb+AcbfR/bP8Bl7H9k/w9cV+ijqPDdERERE7U305ReIiIiI2hPDDRERERkVhhsiIiIyKgw3REREZFQYbvQUExMDLy8vKBQKBAYGIjk5udX2iYmJCAwMhEKhgLe3Nz788MNOqvTe6NO/hIQESCSSJq/ff/+9Eytuu6SkJISHh8PV1RUSiQT79u276z6Gdv707aMhncPo6GiMGDEC1tbWcHJywowZM3D+/Pm77mdI5/Be+mhI53Dz5s3w8/PTTe4WHByM//3vf63uY0jnT9/+GdK5a050dDQkEgmWLVvWajsxziHDjR5iY2OxbNkyrF69GmlpaQgNDcXUqVMbPa5+u6ysLEybNg2hoaFIS0vDqlWrsGTJEuzZs6eTK28bffvX4Pz581AqlbpXv379Oqli/VRWVsLf3x/vv/9+m9ob2vkD9O9jA0M4h4mJiVi8eDF+/vlnxMfHo66uDmFhYaisrGxxH0M7h/fSxwaGcA779OmDv//970hNTUVqaiomTpyIhx56CGfOnGm2vaGdP33718AQzt2dfv31V2zduhV+fn6tthPtHArUZiNHjhQiIyMbbRs4cKAQFRXVbPuXX35ZGDhwYKNtCxYsEEaPHt1hNd4Pfft36NAhAYBw/fr1TqiufQEQvv7661bbGNr5u1Nb+mjI57CwsFAAICQmJrbYxtDPYVv6aMjnUBAEoUePHsK2bduafc/Qz58gtN4/Qz135eXlQr9+/YT4+Hhh3LhxwtKlS1tsK9Y55MhNG9XU1OD48eMICwtrtD0sLAwpKSnN7nP06NEm7R988EGkpqaitra2w2q9F/fSvwYBAQFwcXHBpEmTcOjQoY4ss1MZ0vm7X4Z4DlUqFQDA3t6+xTaGfg7b0scGhnYONRoNPv/8c1RWViI4OLjZNoZ8/trSvwaGdu4WL16M6dOnY/LkyXdtK9Y5ZLhpo6KiImg0miaLejo7OzdZzLNBQUFBs+3r6upQVFTUYbXei3vpn4uLC7Zu3Yo9e/Zg7969GDBgACZNmoSkpKTOKLnDGdL5u1eGeg4FQcCKFSswduxY+Pr6ttjOkM9hW/toaOfw1KlTsLKygpmZGSIjI/H1119j8ODBzbY1xPOnT/8M7dwBwOeff47ffvsN0dHRbWov1jkUdfkFQySRSBr9LghCk213a9/c9q5Cn/4NGDAAAwYM0P0eHByM3NxcvP3223jggQc6tM7OYmjnT1+Geg6fe+45nDx5EocPH75rW0M9h23to6GdwwEDBiA9PR2lpaXYs2cP5s6di8TExBYDgKGdP336Z2jnLjc3F0uXLsWBAwegUCjavJ8Y55AjN23k6OgImUzWZBSjsLCwSSpt0KtXr2bby+VyODg4dFit9+Je+tec0aNH4+LFi+1dnigM6fy1p65+Dp9//nl88803OHToEPr06dNqW0M9h/r0sTld+RyamprCx8cHQUFBiI6Ohr+/P/75z38229YQz58+/WtOVz53x48fR2FhIQIDAyGXyyGXy5GYmIhNmzZBLpdDo9E02Uesc8hw00ampqYIDAxEfHx8o+3x8fEICQlpdp/g4OAm7Q8cOICgoCCYmJh0WK334l7615y0tDS4uLi0d3miMKTz15666jkUBAHPPfcc9u7di4MHD8LLy+uu+xjaObyXPjanq57D5giCALVa3ex7hnb+mtNa/5rTlc/dpEmTcOrUKaSnp+teQUFBeOKJJ5Ceng6ZTNZkH9HOYYfermxkPv/8c8HExETYvn27cPbsWWHZsmWCpaWlcPnyZUEQBCEqKkqYPXu2rn1mZqZgYWEhLF++XDh79qywfft2wcTERPjqq6/E6kKr9O3fe++9J3z99dfChQsXhNOnTwtRUVECAGHPnj1idaFV5eXlQlpampCWliYAEN59910hLS1NyM7OFgTB8M+fIOjfR0M6hwsXLhRsbW2FhIQEQalU6l5VVVW6NoZ+Du+lj4Z0DleuXCkkJSUJWVlZwsmTJ4VVq1YJUqlUOHDggCAIhn/+9O2fIZ27ltz5tFRXOYcMN3r64IMPBA8PD8HU1FQYPnx4o0c0586dK4wbN65R+4SEBCEgIEAwNTUVPD09hc2bN3dyxfrRp39vvfWW0LdvX0GhUAg9evQQxo4dK+zfv1+Eqtum4bHLO19z584VBME4zp++fTSkc9hcvwAIO3fu1LUx9HN4L300pHP49NNP6/596dmzpzBp0iTdF78gGP7507d/hnTuWnJnuOkq51AiCDfv7CEiIiIyArznhoiIiIwKww0REREZFYYbIiIiMioMN0RERGRUGG6IiIjIqDDcEBERkVFhuCEiIiKjwnBDRERERoXhhoi6vYSEBEgkEpSWlopdChG1A4YbIiIiMioMN0RERGRUGG6ISHSCIGDDhg3w9vaGubk5/P398dVXXwG4dclo//798Pf3h0KhwKhRo3Dq1KlGx9izZw+GDBkCMzMzeHp64p133mn0vlqtxssvvww3NzeYmZmhX79+2L59e6M2x48fR1BQECwsLBASEoLz5893bMeJqEMw3BCR6F555RXs3LkTmzdvxpkzZ7B8+XL85S9/QWJioq7NSy+9hLfffhu//vornJyc8Kc//Qm1tbUA6kPJzJkz8dhjj+HUqVNYu3YtXn31VXz88ce6/efMmYPPP/8cmzZtwrlz5/Dhhx/CysqqUR2rV6/GO++8g9TUVMjlcjz99NOd0n8ial9cFZyIRFVZWQlHR0ccPHgQwcHBuu3z589HVVUVnn32WUyYMAGff/45IiIiAAAlJSXo06cPPv74Y8ycORNPPPEErl27hgMHDuj2f/nll7F//36cOXMGFy5cwIABAxAfH4/Jkyc3qSEhIQETJkzAjz/+iEmTJgEA4uLiMH36dNy4cQMKhaKD/xaIqD1x5IaIRHX27FlUV1djypQpsLKy0r127dqFjIwMXbvbg4+9vT0GDBiAc+fOAQDOnTuHMWPGNDrumDFjcPHiRWg0GqSnp0Mmk2HcuHGt1uLn56f72cXFBQBQWFh4330kos4lF7sAIuretFotAGD//v3o3bt3o/fMzMwaBZw7SSQSAPX37DT83OD2QWlzc/M21WJiYtLk2A31EZHh4MgNEYlq8ODBMDMzQ05ODnx8fBq93NzcdO1+/vln3c/Xr1/HhQsXMHDgQN0xDh8+3Oi4KSkp6N+/P2QyGYYOHQqtVtvoHh4iMl4cuSEiUVlbW+PFF1/E8uXLodVqMXbsWJSVlSElJQVWVlbw8PAAAKxfvx4ODg5wdnbG6tWr4ejoiBkzZgAAXnjhBYwYMQKvv/46IiIicPToUbz//vuIiYkBAHh6emLu3Ll4+umnsWnTJvj7+yM7OxuFhYWYOXOmWF0nog7CcENEonv99dfh5OSE6OhoZGZmws7ODsOHD8eqVat0l4X+/ve/Y+nSpbh48SL8/f3xzTffwNTUFAAwfPhwfPHFF1izZg1ef/11uLi4YP369XjyySd1n7F582asWrUKixYtQnFxMdzd3bFq1SoxuktEHYxPSxFRl9bwJNP169dhZ2cndjlEZAB4zw0REREZFYYbIiIiMiq8LEVERERGhSM3REREZFQYboiIiMioMNwQERGRUWG4ISIiIqPCcENERERGheGGiIiIjArDDRERERkVhhsiIiIyKv8PfJNtMHVpF1wAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "y_avg = []\n", + "for y in epochs_y:\n", + " y_avg.append(np.mean(y))\n", + "\n", + "plt.plot(np.arange(len(epochs_x)), y_avg, marker = '.')\n", + "plt.xlabel('epoch')\n", + "plt.ylabel('average loss')\n", + "plt.ylim(0,)\n", + "for i, txt in enumerate(y_avg):\n", + " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom')\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkkAAAG2CAYAAABrrBJlAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAABe1ElEQVR4nO3deVhTV/4/8HcIEBYhiigJiCyKIIuIO6617tW64T6jVmtrRzuV2rqg0nFH7NTWpdVpf221OlW/U9dxKWKtWqpj3VAQRa0KLiAukLAGSO7vD2s0JSDBQBJ4v54nz8jNuSef0+uYN+eee69IEAQBRERERKTDytQFEBEREZkjhiQiIiIiPRiSiIiIiPRgSCIiIiLSgyGJiIiISA+GJCIiIiI9GJKIiIiI9GBIIiIiItKDIYmIiIhID4YkIiIiIj1MGpKOHz+O119/He7u7hCJRNi9e7fO+4IgYOHChXB3d4e9vT1eeeUVXLp0SaeNSqXC3//+d7i6usLR0RGDBw/GnTt3XvjZX3zxBXx8fGBnZ4e2bdvil19+MebQiIiIyMKZNCTl5+cjNDQU69at0/v+ypUrsWrVKqxbtw6nT5+GTCZDnz59kJubq20TGRmJXbt2Ydu2bUhISEBeXh4GDRoEtVpd7udu374dkZGRmD9/Ps6fP49u3bphwIABSE9PN/oYiYiIyDKJzOUBtyKRCLt27cLQoUMBPJlFcnd3R2RkJObMmQPgyayRm5sbYmNjMXXqVCgUCjRq1AibN2/G6NGjAQD37t2Dp6cnDhw4gH79+un9rI4dO6JNmzZYv369dlvLli0xdOhQxMTEVO9AiYiIyCJYm7qA8ty8eROZmZno27evdptEIkGPHj1w4sQJTJ06FWfPnkVJSYlOG3d3dwQHB+PEiRN6Q1JxcTHOnj2LuXPn6mzv27cvTpw4UW49KpUKKpVK+7NGo8Hjx4/RsGFDiESilxkqERER1RBBEJCbmwt3d3dYWVV8Qs1sQ1JmZiYAwM3NTWe7m5sb0tLStG1sbW3RoEGDMm2e7v9nDx8+hFqt1ttvefsAQExMDBYtWmTwOIiIiMj83L59G02aNKmwjdmGpKf+PEsjCMILZ24q08bQfqOiojBz5kztzwqFAk2bNsXt27fh7Oxc4WcRERGReVAqlfD09ISTk9ML25ptSJLJZACezBbJ5XLt9qysLO0skEwmQ3FxMbKzs3Vmk7KystC5c2e9/bq6ukIsFpeZNXq+X30kEgkkEkmZ7c7OzgxJREREFqYyS2XM9j5JPj4+kMlkiI+P124rLi7GsWPHtAGobdu2sLGx0WmTkZGB5OTkckOSra0t2rZtq7MPAMTHx5e7DxEREdU9Jp1JysvLw/Xr17U/37x5E4mJiXBxcUHTpk0RGRmJ5cuXw8/PD35+fli+fDkcHBwwbtw4AIBUKsWbb76JDz74AA0bNoSLiws+/PBDhISEoHfv3tp+e/XqhWHDhuHdd98FAMycORPjx49Hu3btEB4eji+//BLp6el45513avY/ABEREZktk4akM2fOoGfPntqfn675mThxIjZu3IjZs2ejsLAQ06ZNQ3Z2Njp27IhDhw7pnEf89NNPYW1tjVGjRqGwsBC9evXCxo0bIRaLtW1+//13PHz4UPvz6NGj8ejRIyxevBgZGRkIDg7GgQMH4OXlVQOjJiIiIktgNvdJsjRKpRJSqRQKhYJrkoiIiCyEId/fZrsmiYiIiMiUGJKIiIiI9GBIIiIiItKDIYmIiIhID4YkIiIiIj0YkoiIiIj0YEgiIiIi0oMhiYiIiEgPhiQiIiIiPRiSiIiIiPRgSCIiIiLSgyGJiIiISA+GJCIiIiI9GJKIiIiI9GBIIiIiItKDIYmIiIhID4YkIiIiIj0YkoiIiIj0YEgiIiIi0oMhiYiIiEgPhiQiIiIiPRiSiIiIiPRgSCIiIiLSgyGJiIiISA+GJCIiIiI9GJKIiIiI9GBIIiIiItKDIYmIiIhID4YkIiIiIj0YkoiIiIj0YEgiIiIi0oMhiYiIiEgPhiQiIiIiPRiSiIiIiPRgSCIiIiLSw+xDUm5uLiIjI+Hl5QV7e3t07twZp0+f1r4vEon0vj7++ONy+9y4caPefYqKimpiSERERGQBrE1dwItMmTIFycnJ2Lx5M9zd3bFlyxb07t0bKSkp8PDwQEZGhk77gwcP4s0330RERESF/To7OyM1NVVnm52dndHrJyIiIstk1iGpsLAQO3bswJ49e9C9e3cAwMKFC7F7926sX78eS5cuhUwm09lnz5496NmzJ3x9fSvsWyQSldmXiIiI6CmzPt1WWloKtVpdZobH3t4eCQkJZdrfv38f+/fvx5tvvvnCvvPy8uDl5YUmTZpg0KBBOH/+fIXtVSoVlEqlzouIiIhqL7MOSU5OTggPD8eSJUtw7949qNVqbNmyBadOnSpzmg0ANm3aBCcnJwwfPrzCfgMCArBx40bs3bsXW7duhZ2dHbp06YJr166Vu09MTAykUqn25enp+dLjIyIiIvMlEgRBMHURFfn9998xefJkHD9+HGKxGG3atEGLFi1w7tw5pKSk6LQNCAhAnz59sHbtWoM+Q6PRoE2bNujevTvWrFmjt41KpYJKpdL+rFQq4enpCYVCAWdnZ8MHRkRERDVOqVRCKpVW6vvbrNckAUCzZs1w7Ngx5OfnQ6lUQi6XY/To0fDx8dFp98svvyA1NRXbt283+DOsrKzQvn37CmeSJBIJJBKJwX0TERGRZTLr023Pc3R0hFwuR3Z2NuLi4jBkyBCd97/++mu0bdsWoaGhBvctCAISExMhl8uNVS4RERFZOLOfSYqLi4MgCPD398f169cxa9Ys+Pv7Y9KkSdo2SqUS//nPf/DJJ5/o7WPChAnw8PBATEwMAGDRokXo1KkT/Pz8oFQqsWbNGiQmJuLzzz+vkTERERGR+TP7kKRQKBAVFYU7d+7AxcUFERERWLZsGWxsbLRttm3bBkEQMHbsWL19pKenw8rq2aRZTk4O3n77bWRmZkIqlSIsLAzHjx9Hhw4dqn08REREZBnMfuG2uTJk4RcRERGZB0O+vy1mTRIRERFRTWJIIiIiItKDIYmIiIhID4YkIiIiIj0YkoiIiIj0YEgiIiIi0oMhiYiIiEgPhiQiIiIiPRiSiIiIiPRgSCIiIiLSgyGJiIiISA+GJCIiIiI9GJKIiIiI9GBIIiIiItKDIYmIiIhID4YkIiIiIj0YkoiIiIj0YEgiIiIi0oMhiYiIiEgPhiQiIiIiPRiSiIiIiPRgSCIiIiLSgyGJiIiISA+GJCIiIiI9GJKIiIiI9GBIIiIiItKDIYmIiIhID4YkIiIiIj0YkoiIiMiocnNzERkZCS8vL9jb26Nz5844ffq03rZTp06FSCTCZ599VmGfJSUlWLx4MZo1awY7OzuEhobixx9/LNPu7t27+Otf/4qGDRvCwcEBrVu3xtmzZ6s0Dusq7UVERERUjilTpiA5ORmbN2+Gu7s7tmzZgt69eyMlJQUeHh7adrt378apU6fg7u7+wj4XLFiALVu24KuvvkJAQADi4uIwbNgwnDhxAmFhYQCA7OxsdOnSBT179sTBgwfRuHFj/P7776hfv36VxiESBEGo0p51nFKphFQqhUKhgLOzs6nLISIiMguFhYVwcnLCnj17MHDgQO321q1bY9CgQVi6dCmAJzM+HTt2RFxcHAYOHIjIyEhERkaW26+7uzvmz5+P6dOna7cNHToU9erVw5YtWwAAc+fOxa+//opffvml3H4M+f7m6TYiIiIymtLSUqjVatjZ2elst7e3R0JCAgBAo9Fg/PjxmDVrFoKCgirVr0qlqrBPANi7dy/atWuHkSNHonHjxggLC8NXX31V5bEwJBEREZHRODk5ITw8HEuWLMG9e/egVquxZcsWnDp1ChkZGQCA2NhYWFtb47333qt0v/369cOqVatw7do1aDQaxMfHY8+ePdo+AeDGjRtYv349/Pz8EBcXh3feeQfvvfcevvvuuyqNhSGJiIiIjGrz5s0QBAEeHh6QSCRYs2YNxo0bB7FYjLNnz2L16tXYuHEjRCJRpftcvXo1/Pz8EBAQAFtbW7z77ruYNGkSxGKxto1Go0GbNm2wfPlyhIWFYerUqXjrrbewfv36Ko2DIYmIiIiMqlmzZjh27Bjy8vJw+/Zt/PbbbygpKYGPjw9++eUXZGVloWnTprC2toa1tTXS0tLwwQcfwNvbu9w+GzVqhN27dyM/Px9paWm4cuUK6tWrBx8fH20buVyOwMBAnf1atmyJ9PT0Ko3D7EPSiy4jfOONNyASiXRenTp1emG/O3bsQGBgICQSCQIDA7Fr167qHAYREVGd4+joCLlcjuzsbMTFxWHIkCEYP348Ll68iMTERO3L3d0ds2bNQlxc3Av7tLOzg4eHB0pLS7Fjxw4MGTJE+16XLl2Qmpqq0/7q1avw8vKqUv1mH5KmTJmC+Ph4bN68GUlJSejbty969+6Nu3fvatv0798fGRkZ2teBAwcq7PPkyZMYPXo0xo8fjwsXLmD8+PEYNWoUTp06Vd3DISKiF6iOe+w8PbXz51dRUZFOuy+++AI+Pj6ws7ND27ZtK7xKisoXFxeHH3/8ETdv3kR8fDx69uwJf39/TJo0CQ0bNkRwcLDOy8bGBjKZDP7+/to+JkyYgKioKO3Pp06dws6dO3Hjxg388ssv6N+/PzQaDWbPnq1t8/777+N///sfli9fjuvXr+P777/Hl19+qXNFnEEEM1ZQUCCIxWJh3759OttDQ0OF+fPnC4IgCBMnThSGDBliUL+jRo0S+vfvr7OtX79+wpgxYyrdh0KhEAAICoXCoM8mIqKKjRo1SggMDBSOHTsmXLt2TfjHP/4hODs7C3fu3NFpt2vXLiE0NFRwd3cXPv300wr7/PbbbwVnZ2chIyND5/W8bdu2CTY2NsJXX30lpKSkCDNmzBAcHR2FtLQ0Yw+x1tu+fbvg6+sr2NraCjKZTJg+fbqQk5NTbnsvL68yxzC8SzdhwPDRwr2cAkEQBOHo0aNCy5YtBYlEIjRs2FAYP368cPfu3TJ9/fe//xWCg4MFiUQiBAQECF9++aXO+4Z8f5t1SFIqlQIA4fDhwzrbO3XqJPTo0UMQhCchSSqVCo0aNRL8/PyEKVOmCPfv36+wX09PT2HVqlU621atWiU0bdq00rUxJBERGV9lfjkWBEG4c+eO4OHhISQnJ+v9gv2zb7/9VpBKpRW26dChg/DOO+/obAsICBDmzp1r0Bjo5W37LU3wmbtP8JqzT/CZu0/Y9pvxgqoh399mfcft5y8jbNmyJdzc3LB161acOnUKfn5+AIABAwZg5MiR8PLyws2bNxEdHY1XX30VZ8+ehUQi0dtvZmYm3NzcdLa5ubkhMzOz3FpUKhVUKpX2Z6VSaYQREhHR86rrHjsAkJeXBy8vL6jVarRu3RpLlizR3qm5uLgYZ8+exdy5c3X26du3L06cOPGSo6KnBEFArqoUOfkleFxQjOyCYuQUFONxfgly/vj5Xk4RjlzJ0u6jEYB5O5PRvUUjyKX2NVqvWYck4MllhJMnT4aHhwfEYjHatGmDcePG4dy5cwCA0aNHa9sGBwejXbt28PLywv79+zF8+PBy+/3zZYeCIFR4KWJMTAwWLVr0kqMhIqKKVOaX46rcYycgIAAbN25ESEgIlEolVq9ejS5duuDChQvw8/PDw4cPoVarDf4Fui5TawQoC5+EnadBR1/oyf5je3bBk22lGsMf9KEWBNx6WMCQ9GdPLyPMz8+HUqmEXC7H6NGjdS75e55cLoeXlxeuXbtWbp8ymazMX/qsrKwy/+d4XlRUFGbOnKn9WalUwtPT08DREBHRi1T0y/HTe+ycO3fOoHvsdOrUSefK5y5duqBNmzZYu3Yt1qxZo91u6C/QtUWJWvNHwCnB4/ziPwKO7p+z84uftSkohqKwBFV9sJm9jRgNHGxQ38EWLo62qO9ggwYOtmjgaAuxCPjs8DU837VYJIK3q4NRxmoIsw9JTzk6OsLR0VF7GeHKlSv1tnv06BFu374NuVxebl/h4eGIj4/H+++/r9126NAhdO7cudx9JBJJuafviIjIeCr65fj5e+w8pVar8cEHH+Czzz7DrVu3KvUZVlZWaN++vfYXaldXV4jFYoN/gTZHRSVqPH4+0JQXep6e7sovQa6qtMqf5ySxRgNH23JDT4Onf3awRQPHJ3+2sxFX2KdMaod5O5OhFgSIRSIsHx5c47NIgAWEpLi4OAiCAH9/f1y/fh2zZs3SXkaYl5eHhQsXIiIiAnK5HLdu3cK8efPg6uqKYcOGafuYMGECPDw8EBMTAwCYMWMGunfvjtjYWAwZMgR79uzB4cOHdZ7/QkREpqXvl+OIiAj07t1bp12/fv0wfvx4TJo0qdJ9C4KAxMREhISEAABsbW3Rtm1bxMfH63x/xMfH69yHpyYJgoA8Vak26OgNPX+c3np2SqsYRSWaKn2eSATUt9cNN2VCj8MfocfxyZ/rO9jARmz8uwmNbt8U3Vs0wq2HBfB2dTBJQAIsICQpFApERUXhzp07cHFxQUREBJYtWwYbGxuUlpYiKSkJ3333HXJyciCXy9GzZ09s374dTk5O2j7S09NhZfXsIHbu3Bnbtm3DggULEB0djWbNmmH79u3o2LGjKYZIRETPqeiXYxsbGzRs2FCnfXn32Hn+l+NFixahU6dO8PPzg1KpxJo1a5CYmIjPP/9cu8/MmTMxfvx4tGvXDuHh4fjyyy+Rnp6Od95556XHpNEIUBQ+W5vz51NXT4OO9s9/rN8pUVftfJa1lUg36Pwxi/P0z/pmepztbSC2Mp9Ti3KpvcnC0VNmH5JGjRqFUaNG6X3P3t6+UnfnPHr0aJltI0aMwIgRI162PCIiMrKKfjmurD//cpyTk4O3334bmZmZkEqlCAsLw/Hjx9GhQwdtm9GjR+PRo0dYvHgxMjIyEBwcjAMHDpS5W3OJWoOcp6er8v906qqcdTyKwhJUYb0yAMDOxkrndJU29OiZ6Xn653oS6zqxlqq6iQShqsuu6jalUgmpVAqFQgFnZ2dTl0NERM/JUBTi5sN8+Lg6VjgbUVSi1l6BlVNQ/Mdl6SXIyS/+Y1anbOh52fU79R11T109DTrlnd6yt614/Q4ZxpDvb7OfSSIiIjLEt7/exOJ9KRAEQATg1ZaN4VHf/o+wozsDVFiirtJniESA1N5G76mr8kJPfXtb2Fqb/dPA6DkMSUREZJHUGgG3HuXjcobyj1cuku8qkJX77Ma/AoCfLmeV3wmerN95EmaehJoGDjZ/BBv9V2aZ4/odqh4MSUREZPbyVKW48kcYSsnIxeUMJVIzcys9EzQ8zB3BHvV1gs7T4MP1O1QehiQiIjIbgiDgTnYhUrSzQ09miNIfF+htb2djBX83JwS6O6Ol3BmNnSSY9u9zOoukxSIRZvUPMPmVUmR5GJKIiMgkikrUSM3M/WN26EkgupKRW+7CaJmzHVrKndBS7qx9+bg6ljntFTM8xCxuREiWjyGJiIiqlSAIuK9U6YShyxlK3HyYr/eyeBuxCM0bOyFQ7oyW8if/GyB3houjbaU+z1xuREiWjyGJiIiMprhUg+tZeX86XaZEdkGJ3vYNHW3/mBV6NkPUrFG9l74KzBxuREiWjyGJiIiq5FGeCpf/WET9dJbo9wd5eu8SbSUCmjWq99ypsiczRI2cJFw0TWaLIYmIiCpUqtbg5sP8P2aHnoWi5y+1f56TnTVayp21p8tayp3Rws3phQ81JTI3DElERKSlKCzRXmp/OSMXlzOfXGqvKtX/0FTvhg46C6lbyp3gUd+es0NUKzAkEVGtkpubi+joaOzatQtZWVkICwvD6tWr0b59ewDAwoULsW3bNty+fVv75Pdly5ZV+IDrnTt3Yvny5bh+/TpKSkrg5+eHDz74AOPHj9e2KS0txcKFC/Hvf/8bmZmZkMvleOONN7BgwQKdZ4iZC41GQPrjgudOlT2ZIbqbU6i3vYOtGP6yZ+uGAuVO8Jc5o56EXyNUe/FvNxHVKlOmTEFycjI2b94Md3d3bNmyBb1790ZKSgo8PDzQokULrFu3Dr6+vigsLMSnn36Kvn374vr162jUqJHePl1cXDB//nwEBATA1tYW+/btw6RJk9C4cWP069cPABAbG4sNGzZg06ZNCAoKwpkzZzBp0iRIpVLMmDGjJv8TlFFQXIormc+tHbr3ZHYov1j/jRg96tuXudTey8UBVrzDNNUxfMBtFfEBt0Tmp7CwEE5OTtizZw8GDhyo3d66dWsMGjQIS5cuLbPP0/8vHz58GL169ar0Z7Vp0wYDBw7EkiVLAACDBg2Cm5sbvv76a22biIgIODg4YPPmzS8xqsoTBAH3FEW4fO+P02WZT06Z3XqUD33/0ttaW6GFWz20lDlrb8bYUuYMqYNNjdRLZAp8wC0R1UmlpaVQq9Wws7PT2W5vb4+EhIQy7YuLi/Hll19CKpUiNDS0Up8hCAKOHDmC1NRUxMbGard37doVGzZswNWrV9GiRQtcuHABCQkJ+Oyzz15qTOUpKlE/udT+3nM3YszMhaJQ/6X2jZwkOleVtZQ7w9fVEdZi8zsVSGQuGJKIqNZwcnJCeHg4lixZgpYtW8LNzQ1bt27FqVOn4Ofnp223b98+jBkzBgUFBZDL5YiPj4erq2uFfSsUCnh4eEClUkEsFuOLL75Anz59tO/PmTMHCoUCAQEBEIvFUKvVWLZsGcaOHfvS48rKLdK5quxyhhK/P8iHWs+dGK2tRH9cav/kdNnTGSLXepKXroOormFIIqJaZfPmzZg8eTI8PDwgFovRpk0bjBs3DufOndO26dmzJxITE/Hw4UN89dVXGDVqFE6dOoXGjRuX26+TkxMSExORl5eHn376CTNnzoSvry9eeeUVAMD27duxZcsWfP/99wgKCkJiYiIiIyPh7u6OiRMnVqr2ErUGvz/Ie3Zl2R+B6GFesd729R1s0FLmrHMzRj+3epBY81J7ImPgmqQq4pokIvOWn58PpVIJuVyO0aNHIy8vD/v379fb1s/PD5MnT0ZUVFSl+58yZQpu376NuLg4AICnpyfmzp2L6dOna9ssXboUW7ZswZUrV8rsn1NQXOa+Q9fu56FYXfZSe5EI8Gno+NzM0JNAJHO246X2RAbimiQiqvMcHR3h6OiI7OxsxMXFYeXKleW2FQQBKpX+GyNWdp+CgoIyl/qLxWJoNM/PDj0LRRmKIr391pNYI0DmpHPfIX+ZExxs+c81UU3j/+uIqFaJi4uDIAjw9/fH9evXMWvWLPj7+2PSpEnIz8/HsmXLMHjwYMjlcjx69AhffPEF7ty5g5EjR2r7mDBhAjw8PBATEwMAiImJQbt27dCsWTMUFxfjwIED+O6777B+/XrtPq+//jqWLlsGlaQBNPWbIOHUGez/YiXqhfRGr0+O6a3V08X+udNlT+5Q3aSBPS+1JzITDElEVKsoFApERUXhzp07cHFxQUREBJYtWwYbGxuo1WpcuXIFmzZtwsOHD9GwYUO0b98ev/zyC4KCgrR9XL9xCw/zi5GhKIRcao/8/HxMmzYNd+7cgb29PQICAvDZhq/h2akvPjt8FZczlLjabCRyk7Px4fvvQVOggLieCxxC+qFelzGws7GCv5vufYcC5E5wtuOl9kTmjGuSqohrkohqp+2n0xG1Mwka4clDWRcNDkZIE6nOlWVXMnKRqyrVu7/M2a7MjRh9XB0h5uwQkVngmiQioirIUBRqAxIAaAQgek+y3rY2YhGaN3bSue9QS7kzXBxta7BiIqpODElEVOfdyynEj8mZ2H46HXpuPQRnO2u0alJfZ4aoWaN6sLXmjRiJajOGJCKqk24/LsDB5AwcSMpE4u2ccttZiYAfI7vBvb5DzRVHRGaBIYmI6owbD/JwMDkTB5MzkHxXqd0uEgHtvBpgQLAcJWoNVv6YCrUgQCwSYfnwYAYkojqKIYmIai1BEHAtKw8HkjLwY3ImrmTmat+zEgGdfBtiQLAM/YJkaOz87Hlvg1u749bDAni7OkAutTdF6URkBhiSiKhWEQQBKRlKHEx6MmP0+4N87XvWViJ0bu6KAcEy9A10Q8Nynmcml9ozHBERQxIRWT5BEHDxjgIHkp/MGKU9KtC+Zyu2Qjc/V/QPlqFPoBvqO/DqMyKqHIYkIrJIGo2A87ezcSApEz8mZ+JuTqH2PYm1FV7xb4QBwXK82rIxb9pIRFXCkEREFkOtEXD61mMcTMrAj5cycV/57NlpDrZi9AxojAHBMvT0bwxHCf95I6KXw39FiMislao1+N+NxziQnIFDlzLxMK9Y+149iTV6t2yMASFy9GjRCHY2YhNWSkS1DUMSEZmd4lINfv39IQ4mZSA+5T6yC0q070ntbdAn0A2vhcjQpbkrJNYMRkRUPRiSiMgsFJWo8cu1P4LR5fvILXr2bDQXR1v0C3LDgGA5wps1hI2Yd7omourHkEREJlNQXIpjqQ9wIDkTRy7fR36xWvteIycJ+gfJMCBEhg7eLrBmMCKiGsaQREQ1Kk9ViiNXsnAwKQM/p2ahqESjfU8utUP/YBleC5GjTdMGEFuJTFgpEdV1Zh+ScnNzER0djV27diErKwthYWFYvXo12rdvj5KSEixYsAAHDhzAjRs3IJVK0bt3b6xYsQLu7u7l9rlx40ZMmjSpzPbCwkLY2dnp2YOIXoaisAQ/Xb6PA0mZOH7tAYpLnwUjTxd7DAiWY0CwDKFN6sOKwYiIzITZh6QpU6YgOTkZmzdvhru7O7Zs2YLevXsjJSUF9erVw7lz5xAdHY3Q0FBkZ2cjMjISgwcPxpkzZyrs19nZGampqTrbGJCIjCc7vxjxKfdxIDkDv15/iBK1oH3Px9URA/6YMQpyd4ZIxGBEROZHJAiC8OJmplFYWAgnJyfs2bMHAwcO1G5v3bo1Bg0ahKVLl5bZ5/Tp0+jQoQPS0tLQtGlTvf1u3LgRkZGRyMnJqXJtSqUSUqkUCoUCzs7OVe6HqDZ5kKvCoZRMHEzKxMkbj6DWPPvnpYVbPfQPluO1EBn83ZwYjIjIJAz5/jbrmaTS0lKo1eoyMzz29vZISEjQu49CoYBIJEL9+vUr7DsvLw9eXl5Qq9Vo3bo1lixZgrCwsHLbq1QqqFTPblynVCrLbUtUl2QqihB3KRMHkjJw+tZjPJeLECh3xmshMvQPlqN543qmK5KIqAoMDklHjx7FK6+8Ug2llOXk5ITw8HAsWbIELVu2hJubG7Zu3YpTp07Bz8+vTPuioiLMnTsX48aNqzAdBgQEYOPGjQgJCYFSqcTq1avRpUsXXLhwQW+/ABATE4NFixYZbWxEluxOdgF+TM7EweRMnE3L1nkvtIkUA0KerDHyauhoogqJiF6ewafb7Ozs4OHhgUmTJmHixInw9PSsrtoAAL///jsmT56M48ePQywWo02bNmjRogXOnTuHlJQUbbuSkhKMHDkS6enpOHr0qEGnwDQaDdq0aYPu3btjzZo1etvom0ny9PTk6TaqM9Ie5eNgciYOJmXgwh2FznttvRpgQLAM/YNlaNLAwUQVEhG9WLWebrt37x62bNmCjRs3YuHChejVqxfefPNNDB06FLa2xn+6drNmzXDs2DHk5+dDqVRCLpdj9OjR8PHx0bYpKSnBqFGjcPPmTRw5csTg0GJlZYX27dvj2rVr5baRSCSQSCRVHgeRJbqelYcfkzNwICkTKRnPTjGLREAHbxe8FiJHvyAZZFJe9EBEtc9LLdxOTEzEN998g61bt0Kj0eAvf/kL3nzzTYSGhhqzRh3Z2dnw8fHBypUr8fbbb2sD0rVr1/Dzzz+jUaNGBvcpCAI6dOiAkJAQfPPNN5Xahwu3qTYSBAGp93NxMCkTB5MzcPV+nvY9sZUI4b4NMSBEhr6BMjRy4i8NRGR5DPn+fumr2+7du4cvv/wSK1asgLW1NYqKihAeHo4NGzYgKCjoZboGAMTFxUEQBPj7++P69euYNWsWJBIJEhISIBKJEBERgXPnzmHfvn1wc3PT7ufi4qKd2ZowYQI8PDwQExMDAFi0aBE6deoEPz8/KJVKrFmzBps3b8avv/6KDh06VKouhiSqLQRBwKV7ShxIysCPyZm48TBf+56NWIQuzV3xWrAcfQLd0MDR+LPFREQ1qdqvbispKcGePXvwzTffID4+Hu3atcO6deswduxYPH78GHPmzMHIkSN11gxVlUKhQFRUFO7cuQMXFxdERERg2bJlsLGxwa1bt7B3714AT24L8Lyff/5Zu8A8PT0dVlbPHmmQk5ODt99+G5mZmZBKpQgLC8Px48crHZCILJ0gCEi8nfNkjVFyBm4/LtS+Z2tthe5+jfBaiAy9WrpBam9jwkqJiEzH4Jmkv//979i6dSsA4K9//SumTJmC4OBgnTbp6enw9vaGRqPR10WtwJkksjQajYCz6dk4kJSBuORM3FMUad+zs7FCT//GGBAix6sBjVFPYtZ3ByEiqrJqnUlKSUnB2rVrERERUe5CbXd3d/z888+Gdk1ERlaq1uC3W49xMCkTcZcykZX77ApNR1sxXm3phteCZejh3wgOtgxGRETPM/ix2j/99BPGjh1b4ZVs1tbW6NGjx0sVRmQKubm5iIyMhJeXF+zt7dG5c2ecPn1a+/7OnTvRr18/uLq6QiQSITEx8YV97ty5E+3atUP9+vXh6OiI1q1bY/PmzQZ9riFK1Bocv/oAUTsvouPynzDuq1PY/L80ZOWq4GRnjeFtPPDVhHY4G90Ha8eGYUCInAGJiEgPg/9ljImJgZubGyZPnqyz/ZtvvsGDBw8wZ84coxVHVNMqelagh4cH8vPz0aVLF4wcORJvvfVWpfp0cXHB/PnzERAQAFtbW+zbtw+TJk1C48aN0a9fv0p97ouoStX49fpDHEjKRHzKfSgKS7Tv1XewQb9AGfqHyNClmStsrQ3+3YiIqE4yeE2St7c3vv/+e3Tu3Fln+6lTpzBmzBjcvHnTqAWaK65Jqn0MeVbgrVu34OPjg/Pnz5e5aKAy2rRpg4EDB2LJkiVVekYhABSVqHHs6gMcTMrAT5ezkKsq1b7nWs8W/YJkGBAsR0dfF9iIGYyIiIBqXpOUmZkJuVxeZnujRo2QkZFhaHdEZqMqzwo0lCAIOHLkCFJTUxEbG2vw5+arSnE09QEOJGfg5ytZKChWa99zc5ZgQLAc/YNlaO/tArEVHyBLRPQyDA5Jnp6e+PXXX3XueA0Av/76K9zd3Y1WGFFNM/RZgYZQKBTw8PCASqWCWCzGF198gT59+lTqc3OLSnDkShYOJGXgaOoDqEqfXTXqUd8eA4JlGBAiQ5hnA1gxGBERGY3BIWnKlCmIjIxESUkJXn31VQBPFnPPnj0bH3zwgdELJKpJmzdvxuTJk+Hh4aF9VuC4ceNw7ty5l+rXyckJiYmJyMvLw08//YSZM2fC19dXey+vP39uaOswdO0/FEkXEtF2yWEUq58FI6+GDhgQ/OQBsq2aSCESMRgREVUHg0PS7Nmz8fjxY0ybNg3FxcUAnjz0ds6cOYiKijJ6gUQ1qTLPCqwKKysrNG/eHMCTtUaXL19GTEyMNiQ1a9YMO/cfwn/P3sS+szeQ+BC4smsFBLuGKFZr0KyRI14LkWNAsBwt5U4MRkRENcDgkCQSiRAbG4vo6GhcvnwZ9vb28PPz48NfqVZxdHSEo6MjsrOzERcXh5UrVxq1f0EQoFKpkJVbhLhL93EwKQP/u/EImj8uo1AX5aH41nkMf2c2lr7fHX5uTkb9fCIierEq3xylXr16aN++vTFrITI5fc8K9Pf3x6RJkwAAjx8/Rnp6Ou7duwcASE1NBQDIZDLIZDIAZZ8VGBMTg3bt2qFZs2YoLi7Gth17sGnTd2g95kN0XP4TBAEovHEWABAcGICWjgU4smkV3FsFYsvKubCx4WNBiIhMoUoh6fTp0/jPf/6D9PR07Sm3p3bu3GmUwohMoaJnBQLA3r17tYEJAMaMGQMA+Mc//oGFCxcCAK7fuIWH+cXIUBRCLrVHfn4+3n7nb7h75w5gbQur+h5oMHAmHjbpCghAWNP6aGRXH3GbPkPCnrtI0fO5RERU8wy+T9K2bdswYcIE9O3bF/Hx8ejbty+uXbuGzMxMDBs2DN9++2111WpWeJ8k0mf76XRE7UyCRgCsRECfQDfcyylC0l2Fto1IBLT3ckH/YBn6B8vgXt/ehBUTEdUt1XqfpOXLl+PTTz/F9OnT4eTkhNWrV8PHxwdTp07Ve/8koroiQ1GoDUgAoBGAuEv3ATwJTJ18G2JAsAz9gmRo7GxXQU9ERGQODA5Jv//+u/auwBKJBPn5+RCJRHj//ffx6quvYtGiRUYvksgS3HyYrw1Iz3urmw/e6dEMDevx4gYiIkti8LMKXFxckJubCwDw8PBAcnIyACAnJwcFBQXGrY7Igvi4OpbZJhaJMLmrDwMSEZEFMngmqVu3boiPj0dISAhGjRqFGTNm4MiRI4iPj0evXr2qo0Yii5D2SPeXBLFIhOXDgyGXcs0REZElMjgkrVu3DkVFRQCAqKgo2NjYICEhAcOHD0d0dLTRCySyBEUlaszdcREAMLS1O0a3bwpvVwcGJCIiC2bQ1W2lpaX497//jX79+mnvCVNX8eo2el7Mwcv417EbkDnb4dDM7nC246X7RETmyJDvb4PWJFlbW+Nvf/sbVCrVSxVIVJtcvJODr47fAAAsHRrMgEREVEsYvHC7Y8eOOH/+fHXUQmRxStQazP7hIjQCMDjUHb0D3UxdEhERGYnBa5KmTZuGDz74AHfu3EHbtm3h6Kh7RU+rVq2MVhyRufvXsd9xJTMXDRxs8I/XA01dDhERGZHBd9y2sio7+SQSiSAIAkQiEdRqtdGKM2dck0TXs3Lx2uoEFKs1WD2mNYa09jB1SURE9ALVesftmzdvVrkwotpCrREw+4eLKFZr0NO/EQaHupu6JCIiMjKDQ5KXl1d11EFkUTafvIVz6TlwtBVj2bAQiEQiU5dERERGZnBI+u677yp8f8KECVUuhsgS3MkuwMq4VADA3Nda8gG1RES1lMEhacaMGTo/l5SUoKCgALa2tnBwcGBIolpNEATM25WMgmI1Oni74C8dmpq6JCIiqiYG3wIgOztb55WXl4fU1FR07doVW7durY4aiczGznN3cfzqA9haW2FFRAisrHiajYiotjI4JOnj5+eHFStWlJllIqpNHuSqsHhfCgDg/d4t4NuonokrIiKi6mSUkAQAYrEY9+7dM1Z3RGZn4d5LUBSWINjDGW918zF1OUREVM0MXpO0d+9enZ8FQUBGRgbWrVuHLl26GK0wInPyY3Im9idlQGwlQmxEK1iLjfb7BRERmSmDQ9LQoUN1fhaJRGjUqBFeffVVfPLJJ8aqi8hsKApL8NGeZADA1O6+CHKXmrgiIiKqCQaHJI1GUx11EJmtmAOXkZWrgq+rI97r5WfqcoiIqIbwnAFRBX69/hDbTt8GAMSOaAU7G7GJKyIioppicEgaMWIEVqxYUWb7xx9/jJEjRxqlKCJzUFBciqidSQCACeFeaO/tYuKKiIioJhkcko4dO4aBAweW2d6/f38cP37cKEURmYNVh64i/XEB3KV2mN0/wNTlEBFRDTM4JOXl5cHW1rbMdhsbGyiVSqMU9bzc3FxERkbCy8sL9vb26Ny5M06fPq19XxAELFy4EO7u7rC3t8crr7yCS5cuvbDfHTt2IDAwEBKJBIGBgdi1a5fRayfLdT49G9/8+uRhzsuGh6CexODle0REZOEMDknBwcHYvn17me3btm1DYGCgUYp63pQpUxAfH4/NmzcjKSkJffv2Re/evXH37l0AwMqVK7Fq1SqsW7cOp0+fhkwmQ58+fZCbm1tunydPnsTo0aMxfvx4XLhwAePHj8eoUaNw6tQpo9dPlqe4VIM5Oy5CIwDDwjzQ07+xqUsiIiITEAmCIBiyw969exEREYFx48bh1VdfBQD89NNP2Lp1K/7zn/+UuUXAyygsLISTkxP27Nmjc4qvdevWGDRoEJYsWQJ3d3dERkZizpw5AACVSgU3NzfExsZi6tSpevsdPXo0lEolDh48qN3Wv39/NGjQoNKPVlEqlZBKpVAoFHB2dn6JUZK5+ezwVXx2+BoaOtoifmYPuDiWnTklIiLLZMj3t8EzSYMHD8bu3btx/fp1TJs2DR988AHu3LmDw4cPGzUgAUBpaSnUajXs7Ox0ttvb2yMhIQE3b95EZmYm+vbtq31PIpGgR48eOHHiRLn9njx5UmcfAOjXr1+F+6hUKiiVSp0X1T5X7+fi85+vAwAWDg5iQCIiqsOqtNBi4MCBehdvG5uTkxPCw8OxZMkStGzZEm5ubti6dStOnToFPz8/ZGZmAgDc3Nx09nNzc0NaWlq5/WZmZurd52l/+sTExGDRokUvMRoyd2qNgNk/XESJWkDvlm4Y1Epu6pKIiMiEDJ5JOn36tN61O6dOncKZM2eMUtTzNm/eDEEQ4OHhAYlEgjVr1mDcuHEQi5/dr0Yk0n0SuyAIZbb9maH7REVFQaFQaF+3b9+uwmjInG08cQuJt3PgJLHG0qHBL/w7REREtZvBIWn69Ol6A8Ldu3cxffp0oxT1vGbNmuHYsWPIy8vD7du38dtvv6GkpAQ+Pj6QyWQAUGYGKCsrq8xM0fNkMpnB+0gkEjg7O+u8qPZIf1SAf8alAgDmDWwJmdTuBXsQEVFtZ3BISklJQZs2bcpsDwsLQ0pKilGK0sfR0RFyuRzZ2dmIi4vDkCFDtEEpPj5e2664uBjHjh1D586dy+0rPDxcZx8AOHToUIX7UO0lCAKidl1EYYka4b4NMaa9p6lLIiIiM2DwmiSJRIL79+/D19dXZ3tGRgasrY1/L5m4uDgIggB/f39cv34ds2bNgr+/PyZNmgSRSITIyEgsX74cfn5+8PPzw/Lly+Hg4IBx48Zp+5gwYQI8PDwQExMDAJgxYwa6d++O2NhYDBkyBHv27MHhw4eRkJBg9PrJ/P3nzB38ev0RJNZWiBkewtNsREQEoAohqU+fPoiKisKePXsglT55GnpOTg7mzZuHPn36GL1AhUKBqKgo3LlzBy4uLoiIiMCyZctgY2MDAJg9ezYKCwsxbdo0ZGdno2PHjjh06BCcnJy0faSnp8PK6tmkWefOnbFt2zYsWLAA0dHRaNasGbZv346OHTsavX4yb1nKIizZ/2QG9IO+LeDt6mjiioiIyFwYfJ+ku3fvonv37nj06BHCwsIAAImJiXBzc0N8fDw8PevGqQreJ6l2eGfzWfx4KROtmkix82+dYS3mM5+JiGozQ76/DZ5J8vDwwMWLF/Hvf/8bFy5cgL29PSZNmoSxY8dqZ3eILMHBpAz8eCkT1lYixEa0YkAiIiIdVVpE5OjoiLffftvYtRDVmJyCYkTvefKMv2mvNENLOWcDiYhIV5VXWqekpCA9PR3FxcU62wcPHvzSRRFVt6X7L+NhngrNG9fD9Febm7ocIiIyQwaHpBs3bmDYsGFISkqCSCTC0yVNT68IUqvVxq2QyMiOX32AH87egUgExEa0gsRa/OKdiIiozjF4EcaMGTPg4+OD+/fvw8HBAZcuXcLx48fRrl07HD16tBpKJDKefFUponYmAQAmhnujrVcDE1dERETmyuCZpJMnT+LIkSNo1KgRrKysYGVlha5duyImJgbvvfcezp8/Xx11EhnFx3GpuJtTCI/69pjVz9/U5RARkRkzeCZJrVajXr16AABXV1fcu3cPAODl5YXU1FTjVkdkRGfTsrHp5C0AQMzwEDhKjH/zUyIiqj0M/pYIDg7GxYsX4evri44dO2LlypWwtbXFl19+WeYu3ETmQlWqxpwdFyEIwIi2TdC9RSNTl0RERGbO4JC0YMEC5OfnAwCWLl2KQYMGoVu3bmjYsCG2b99u9AKJjOHzI9dxPSsPrvUkWDCwpanLISIiC2BwSOrXr5/2z76+vkhJScHjx4/RoEEDPvOKzNLlDCW+OPo7AGDJkCDUd7A1cUVERGQJjLIow8XFxRjdEBldqVqDOTsuolQjoH+QDANC5KYuiYiILASfw0C12je/3sTFOwo421lj8ZAgU5dDREQWhCGJaq1bD/PxyaGrAIAFAwPR2NnOxBUREZElYUiiWkkQBMzdeRGqUg26NG+Ike2amLokIiKyMAaHpOPHj6O0tLTM9tLSUhw/ftwoRRG9rG2nb+N/Nx7D3kaMmGGteFEBEREZzOCQ1LNnTzx+/LjMdoVCgZ49exqlKKKXkakowvL9lwEAH/bzR9OGDiauiIiILJHBIUkQBL2/lT969AiOjo5GKYqoqgRBwILdSchVlaK1Z3280dnb1CUREZGFqvQtAIYPHw4AEIlEeOONNyCRSLTvqdVqXLx4EZ07dzZ+hUQG2HcxA4cvZ8FGLMLKEa0gtuJpNiIiqppKhySpVArgyW/qTk5OsLe3175na2uLTp064a233jJ+hUSV9Di/GAv3XgIATO/ZHC3cnExcERERWbJKh6Rvv/0WAODt7Y0PP/yQp9bI7CzZl4JH+cXwd3PCtFeam7ocIiKycAavSZo9e7bOmqS0tDR89tlnOHTokFELIzLEz6lZ2HX+LqxEQOyIVrC15t0tiIjo5Rj8TTJkyBB89913AICcnBx06NABn3zyCYYMGYL169cbvUCiF8lTlWL+ziQAwOQuPmjtWd+0BRERUa1gcEg6d+4cunXrBgD44YcfIJPJkJaWhu+++w5r1qwxeoFEL7Lyxyu4pyhCUxcHzOzbwtTlEBFRLWFwSCooKICT05MFsYcOHcLw4cNhZWWFTp06IS0tzegFElXkt5uP8d3JJ3/vVgwPgYOtUZ7ZTEREZHhIat68OXbv3o3bt28jLi4Offv2BQBkZWXB2dnZ6AUSlaeoRI25Oy4CAMa090Tn5q4mroiIiGoTg0PSRx99hA8//BDe3t7o0KEDwsPDATyZVQoLCzN6gUTlWfPTNdx4mI/GThJEvdbS1OUQEVEtY/C5iREjRqBr167IyMhAaGiodnuvXr0wbNgwoxZHVJ7kuwr86/gNAMCSocGQ2tuYuCIiIqptqnSdtEwmg5OTE+Lj41FYWAgAaN++PQICAoxaHJE+pWoN5uy4CLVGwMAQOfoFyUxdEhER1UIGh6RHjx6hV69eaNGiBV577TVkZGQAAKZMmYIPPvjA6AUS/dlXv9zEpXtKSO1tsHBwkKnLISKiWsrgkPT+++/DxsYG6enpcHB49nT10aNH48cffzRqcUR/duNBHj49fBUA8NGgQDRykrxgDyIioqoxeE3SoUOHEBcXhyZNmuhs9/Pz4y0AqFppNALm7khCcakG3Vs0wvA2HqYuiYiIajGDZ5Ly8/N1ZpCeevjwISQS/lZP1effv6Xjt1uP4WArxvJhwTqPxyEiIjI2g0NS9+7dtY8lAQCRSASNRoOPP/4YPXv2NGpxRE/dzSnEigOXAQCz+/mjSYOyQZ2IiMiYDD7d9vHHH+OVV17BmTNnUFxcjNmzZ+PSpUt4/Pgxfv311+qokeo4QRAwf1cS8ovVaOvVAOPDvU1dEhER1QEGzyQFBgbi4sWL6NChA/r06YP8/HwMHz4c58+fR7NmzaqjRqrj9iTew9HUB7AVWyE2IgRiK55mIyKi6mdwSEpPT4ebmxsWLVqEffv24cCBA1i6dCnkcjnS09ONWlxpaSkWLFgAHx8f2Nvbw9fXF4sXL4ZGo9G2EYlEel8ff/xxuf1u3LhR7z5FRUVGrZ9e3qM8FRb99xIA4L1ezdG8sZOJKyIiorrC4NNtPj4+yMjIQOPGjXW2P3r0CD4+PlCr1UYrLjY2Fhs2bMCmTZsQFBSEM2fOYNKkSZBKpZgxYwYAaO/T9NTBgwfx5ptvIiIiosK+nZ2dkZqaqrPNzs7OaLWTcSz6bwqyC0oQIHPC1B6cqSQioppjcEgSBEHvVUV5eXlGDxknT57EkCFDMHDgQACAt7c3tm7dijNnzmjbyGS6d1ves2cPevbsCV9f3wr7FolEZfYl83I45T72XrgHKxHw8YhQ2IirdIN4IiKiKql0SJo5cyaAJ+EiOjpa5zYAarUap06dQuvWrY1aXNeuXbFhwwZcvXoVLVq0wIULF5CQkIDPPvtMb/v79+9j//792LRp0wv7zsvLg5eXF9RqNVq3bo0lS5ZU+IBelUoFlUql/VmpVBo8Hqo8ZVEJFuxOBgC81c0XIU2kJq6IiIjqmkqHpPPnzwN4MpOUlJQEW1tb7Xu2trYIDQ3Fhx9+aNTi5syZA4VCgYCAAIjFYqjVaixbtgxjx47V237Tpk1wcnLC8OHDK+w3ICAAGzduREhICJRKJVavXo0uXbrgwoUL8PPz07tPTEwMFi1a9NJjospZcfAKMpVF8G7ogMjeLUxdDhER1UEiQRAEQ3aYNGkSVq9eDWdn5+qqSWvbtm2YNWsWPv74YwQFBSExMRGRkZFYtWoVJk6cWKZ9QEAA+vTpg7Vr1xr0ORqNBm3atEH37t2xZs0avW30zSR5enpCoVDUyH+LuuTk748w9qv/AQC2vtUJ4c0amrgiIiKqLZRKJaRSaaW+vw1ek/Ttt99WuTBDzZo1C3PnzsWYMWMAACEhIUhLS0NMTEyZkPTLL78gNTUV27dvN/hzrKys0L59e1y7dq3cNhKJhHcUrwFFJWpE7bwIABjXsSkDEhERmYxZr4QtKCiAlZVuiWKxWOcWAE99/fXXaNu2LUJDQw3+HEEQkJiYCLlcXuVayTg+PXwVtx4VQOZsh7kDAkxdDhER1WEGzyTVpNdffx3Lli1D06ZNERQUhPPnz2PVqlWYPHmyTjulUon//Oc/+OSTT/T2M2HCBHh4eCAmJgYAsGjRInTq1Al+fn5QKpVYs2YNEhMT8fnnn1f7mKh8F+/k4KvjNwAAS4cGw9nOxsQVERFRXWbWIWnt2rWIjo7GtGnTkJWVBXd3d0ydOhUfffSRTrtt27ZBEIRyF3Snp6frzEjl5OTg7bffRmZmJqRSKcLCwnD8+HF06NChWsdD5StRazD7h4vQCMDgUHf0DnQzdUlERFTHGbxwm54wZOEXvdi6I9fwz0NX0cDBBodn9kDDelz/RURExmfI97dZr0miuuF6Vi7W/HQdAPCP14MYkIiIyCwwJJFJqTUCZv9wEcVqDXr6N8KQ1u6mLomIiAgAQxKZ2OaTt3AuPQeOtmIsGxai95E3REREpsCQRCZzJ7sAK+OePGR47mst4V7f3sQVERERPcOQRCYhCALm7UpGQbEaHbxd8JcOTU1dEhERkQ6GJDKJnefu4vjVB7C1tsKKiBBYWfE0GxERmReGJKpxD3JVWLwvBQAQ2dsPvo3qmbgiIiKishiSqMYt3HsJisISBLk7461uvqYuh4iISC+GJKpRPyZnYn9SBsRWIsRGtIKNmH8FiYjIPPEbimqMoqAE0XuSAQBTu/si2ENq4oqIiIjKx5BENWb5gct4kKuCr6sj3uvlZ+pyiIiIKsSQRDXi1+sPsf3MbQBA7IhWsLMRm7giIiKiijEkUbUrKC5F1M4kAMCEcC+093YxcUVEREQvxpBE1W7VoatIf1wAd6kdZvcPMHU5RERElcKQRNXqfHo2vvn1JgBg2bAQ1JNYm7giIiKiymFIompTXKrBnB0XoRGAYWEe6BnQ2NQlERERVRpDElWbL45ex9X7eWjoaIvoQYGmLoeIiMggDElULa7ez8XnP18HACwcHAQXR1sTV0RERGQYhiQyOrVGwOwfLqJELaB3SzcMaiU3dUlEREQGY0gio9t44hYSb+fASWKNpUODIRKJTF0SERGRwRiSyKjSHxXgn3GpAICo11pCJrUzcUVERERVw5BERiMIAqJ2XURhiRqdfF0wpr2nqUsiIiKqMoYkMpr/nLmDX68/gsTaCiuGt4KVFU+zERGR5WJIIqO4ryzCkv0pAIAP+raAt6ujiSsiIiJ6OQxJZBQf7UlGblEpWjWRYnIXH1OXQ0RE9NIYkuilHUzKQNyl+7C2EiE2ohWsxfxrRURElo/fZvRScgqKEb3nEgBg2ivN0FLubOKKiIiIjIMhiV7K0v2X8TBPheaN62H6q81NXQ4REZHRMCRRlR2/+gA/nL0DkQiIjQiBxFps6pKIiIiMhiGJqiRfVYqonUkAgInh3mjr5WLiioiIiIyLIYmq5OO4VNzNKYRHfXvM6udv6nKIiIiMjiGJDHY2LRubTt4CAMQMD4GjxNq0BREREVUDhiQyiKpUjTk7LkIQgBFtm6B7i0amLomIiKhaMCSRQT4/ch3Xs/LgWk+CBQNbmrocIiKiasOQRJV2OUOJL47+DgBYPCQI9R1sTVwRERFR9THrkFRaWooFCxbAx8cH9vb28PX1xeLFi6HRaLRt3njjDYhEIp1Xp06dXtj3jh07EBgYCIlEgsDAQOzatas6h2LxStUazNlxEaUaAf2C3DAgWGbqkoiIiKqVWa+4jY2NxYYNG7Bp0yYEBQXhzJkzmDRpEqRSKWbMmKFt179/f3z77bfan21tK57hOHnyJEaPHo0lS5Zg2LBh2LVrF0aNGoWEhAR07Nix2sZjyb759SYu3lHAyc4aS4YEQyQSmbokIiKiaiUSBEEwdRHlGTRoENzc3PD1119rt0VERMDBwQGbN28G8GQmKScnB7t37650v6NHj4ZSqcTBgwe12/r3748GDRpg69atlepDqVRCKpVCoVDA2bl2P4rj1sN89PvsOFSlGqyMaIVR7T1NXRIREVGVGPL9bdan27p27YqffvoJV69eBQBcuHABCQkJeO2113TaHT16FI0bN0aLFi3w1ltvISsrq8J+T548ib59++ps69evH06cOFHuPiqVCkqlUudVF2g0AubsuAhVqQZdmjfEyHZNTF0SERFRjTDr021z5syBQqFAQEAAxGIx1Go1li1bhrFjx2rbDBgwACNHjoSXlxdu3ryJ6OhovPrqqzh79iwkEonefjMzM+Hm5qazzc3NDZmZmeXWEhMTg0WLFhlnYBZk2+nbOHXzMextxIgZ1oqn2YiIqM4w65C0fft2bNmyBd9//z2CgoKQmJiIyMhIuLu7Y+LEiQCenDp7Kjg4GO3atYOXlxf279+P4cOHl9v3n7/sBUGoMABERUVh5syZ2p+VSiU8PWv3aadMRRFiDlwGAHzYzx9NGzqYuCIiIqKaY9YhadasWZg7dy7GjBkDAAgJCUFaWhpiYmK0IenP5HI5vLy8cO3atXL7lclkZWaNsrKyyswuPU8ikZQ7M1UbCYKABbuTkKsqRWvP+nijs7epSyIiIqpRZr0mqaCgAFZWuiWKxWKdWwD82aNHj3D79m3I5fJy24SHhyM+Pl5n26FDh9C5c+eXK7gW2XcxA4cvZ8FGLEJsRCuIrXiajYiI6haznkl6/fXXsWzZMjRt2hRBQUE4f/48Vq1ahcmTJwMA8vLysHDhQkREREAul+PWrVuYN28eXF1dMWzYMG0/EyZMgIeHB2JiYgAAM2bMQPfu3REbG4shQ4Zgz549OHz4MBISEkwyTnPzOL8YC/deAgBM79kc/jInE1dERERU88w6JK1duxbR0dGYNm0asrKy4O7ujqlTp+Kjjz4C8GRWKSkpCd999x1ycnIgl8vRs2dPbN++HU5Oz77Y09PTdWakOnfujG3btmHBggWIjo5Gs2bNsH37dt4j6Q9L9qXgUX4x/N2cMO2V5qYuh4iIyCTM+j5J5qy23ifp5ytZmLTxNKxEwM5pXdDas76pSyIiIjKaWnOfJKpZeapSzN+VBACY3MWHAYmIiOo0hiTSWvnjFdxTFKGpiwNm9m1h6nKIiIhMiiGJAAC/3XyM706mAQBihofAwdasl6sRERFVO4YkQlGJGnN3XAQAjG7niS7NXU1cERERkekxJBHW/HQNNx7mo7GTBPMGtjR1OURERGaBIamOS76rwL+O3wAALBkaDKm9jYkrIiIiMg8MSXVYiVqD2T9chFojYGCIHP2CZKYuiYiIyGwwJNVhX/1yAykZSkjtbbBwcJCpyyEiIjIrDEl11I0Hefjs8JOHAH80KBCNnOrOw3uJiIgqgyGpDtJoBMzdkYTiUg26+blieBsPU5dERERkdhiS6qB//5aO3249hoOtGMuHhUAkEpm6JCIiIrPDkFTH3M0pxIoDlwEAs/v5w9PFwcQVERERmSeGpDpEEATM35WE/GI12no1wPhwb1OXREREZLYYkuqQPYn3cDT1AWzFVoiNCIHYiqfZiIiIysOQVEc8ylNh0X8vAQDe69UczRs7mbgiIiIi88aQVEcs+m8KsgtKECBzwtQezUxdDhERkdljSKoDDqfcx94L92AlAlaOaAUbMQ87ERHRi/DbspZTFpVgwe5kAMBb3XzRqkl90xZERERkIRiSarkVB68gU1kE74YOiOzdwtTlEBERWQyGpFrs5O+P8P2pdABAzPBWsLcVm7giIiIiy8GQVEsVFqsRtfMiAGBcx6YIb9bQxBURERFZFoakWuqzw1dx61EBZM52mDsgwNTlEBERWRyGpFro4p0cfPXLDQDA0qHBcLazMXFFRERElochqZYpUWsw+4eL0AjA66Hu6B3oZuqSiIiILBJDUi3zr2O/40pmLho42OAfrweauhwiIiKLxZBUi1zPysWan64DAP7xehBc60lMXBEREZHlYkiqJdQaAbN/uIhitQY9/RthSGt3U5dERERk0RiSaonNJ2/hXHoOHG3FWDYsBCKRyNQlERERWTSGpFrg9uMCrIxLBQDMfa0l3Ovbm7giIiIiy8eQZOEEQcC8XUkoKFajg7cL/tKhqalLIiIiqhUYkizcznN38cu1h7C1tsKKiBBYWfE0GxERkTEwJFmwB7kqLN6XAgCI7O0H30b1TFwRERFR7cGQZMEW7r0ERWEJgtyd8VY3X1OXQ0REVKswJFmoH5MzsT8pA2IrEWIjWsFGzENJRERkTPxmtUCKghJE70kGAEzt7otgD6mJKyIiIqp9zDoklZaWYsGCBfDx8YG9vT18fX2xePFiaDQaAEBJSQnmzJmDkJAQODo6wt3dHRMmTMC9e/cq7Hfjxo0QiURlXkVFRTUxrJe2/MBlPMhVwdfVEe/18jN1OURERLWStakLqEhsbCw2bNiATZs2ISgoCGfOnMGkSZMglUoxY8YMFBQU4Ny5c4iOjkZoaCiys7MRGRmJwYMH48yZMxX27ezsjNTUVJ1tdnZ21Tkco/j1+kNsP3MbABA7ohXsbMQmroiIiKh2MuuZpJMnT2LIkCEYOHAgvL29MWLECPTt21cbgKRSKeLj4zFq1Cj4+/ujU6dOWLt2Lc6ePYv09PQK+xaJRJDJZDqvl/WimS8A2LlzJ/r16wdXV1eIRCIkJia+sN+SkhIsXrwYvr7N0K2lO+598y46295Ge28XbRtvb2+9s2PTp09/6XERERHVRWYdkrp27YqffvoJV69eBQBcuHABCQkJeO2118rdR6FQQCQSoX79+hX2nZeXBy8vLzRp0gSDBg3C+fPnX7repzNf69atw+XLl7Fy5Up8/PHHWLt2rbZNfn4+unTpghUrVlS63wULFuBf//oXuk6YBfmb69EkfDB2ffy+Ts2nT59GRkaG9hUfHw8AGDly5EuPi4iIqC4y69Ntc+bMgUKhQEBAAMRiMdRqNZYtW4axY8fqbV9UVIS5c+di3LhxcHZ2LrffgIAAbNy4ESEhIVAqlVi9ejW6dOmCCxcuwM9P/xoflUoFlUql/VmpVJZp8/zMF/Bkdmfr1q06p/7Gjx8PALh169YLx//U5s2bMeFv7+P/CjxhUx/4OvJ1rBal4ZNPPsGWLVsAAI0aNdLZZ8WKFWjWrBl69OhR6c8hIiKiZ8x6Jmn79u3YsmULvv/+e5w7dw6bNm3CP//5T2zatKlM25KSEowZMwYajQZffPFFhf126tQJf/3rXxEaGopu3brh//7v/9CiRQudGZ8/i4mJgVQq1b48PT3LtKnKzFdlqFQq/Df5ATQCMCzMAz0DGsPe3h4JCQl62xcXF2PLli2YPHkyH3RLRERURWY9kzRr1izMnTsXY8aMAQCEhIQgLS0NMTExmDhxorZdSUkJRo0ahZs3b+LIkSMVziLpY2Vlhfbt2+PatWvltomKisLMmTO1PyuVyjJBydCZr8ryDu2MpJ+2w++vgZj/Wi/Ex8djz549UKvVetvv3r0bOTk5eOONN17qc4mIiOoys55JKigogJWVbolisVhnIfTTgHTt2jUcPnwYDRs2NPhzBEFAYmIi5HJ5uW0kEgmcnZ11Xn9myMxXZaVm5iK71ThYu7jjyto3IWtQD++++y4mTZoEsVj/lW1ff/01BgwYAHd39yp/LhERUV1n1jNJr7/+OpYtW4amTZsiKCgI58+fx6pVqzB58mQAT64mGzFiBM6dO4d9+/ZBrVYjMzMTAODi4gJbW1sAwIQJE+Dh4YGYmBgAwKJFi9CpUyf4+flBqVRizZo1SExMxOeff/5S9VZ25quy1BoBc3ZchMbOGePmr8XaUUF4/Pgx3N3dMXfuXPj4+JTZJy0tDYcPH8bOnTtfaixERER1nVmHpLVr1yI6OhrTpk1DVlYW3N3dMXXqVHz00UcAgDt37mDv3r0AgNatW+vs+/PPP+OVV14BAKSnp+vMSOXk5ODtt99GZmYmpFIpwsLCcPz4cXTo0OGl6q3MzJchNp64hcTbOXCSWGPp0GDY29vBw8MDJSUl2LFjB0aNGlVmn2+//RaNGzfWLh4nIiKiqhEJgiCYughLpFQqIZVKoVAotKfe3njjDRw+fBj/+te/tDNfb7/9NiZPnozY2FgAwOPHj5Geno579+5h4MCB2LZtG/z9/XXu1TRhwgQ4uTTGEafeKCxRY7JfKYIbqNG6dWvcvXsXCxcuxM2bN3Hu3DmdWx1oNBr4+Phg7NixBt1igIiIqK7Q9/1dHrNek2Rp1q5dixEjRmDatGlo2bIlPvzwQ0ydOhVLlizRttm7dy/CwsK0Mz1jxoxBWFgYNmzYoG1z/cYt7Dl5CYUlanTydUE3XykWLFiAwMBADBs2DB4eHkhISChzL6jDhw8jPT1dezqSiIiIqo4zSVVkSBI1xPbT6Zi7IwlPD8qsfv6Y3rO50fonIiKqyziTZKEyFIWI2vksIAHAqkNXkaEoNFlNREREdRVDkhm5+TAfmj/N66kFAbceFpimICIiojqMIcmM+Lg6wupPN8gWi0TwdnUwTUFERER1GEOSGZFL7REzPATiPx4lIhaJsHx4MORSexNXRkREVPeY9X2S6qLR7Zuie4tGuPWwAN6uDgxIREREJsKQZIbkUnuGIyIiIhPj6TYiIiIiPRiSiIiIiPRgSCIiIiLSgyGJiIiISA+GJCIiIiI9GJKIiIiI9GBIIiIiItKDIYmIiIhID4YkIiIiIj0YkoiIiIj0YEgiIiIi0oMhiYiIiEgPhiQiIiIiPRiSiIiIiPRgSCIiIiLSgyGJiIiISA+GJCIiIiI9GJKIiIiI9GBIIiIiItKDIYmIiIhID4YkIiIiIj0YkoiIiIj0YEgiIiIi0oMhiYiIiEgPhiQiIiIiPRiSiIiIiPRgSCIiIiLSgyGJiIiISA+zDkmlpaVYsGABfHx8YG9vD19fXyxevBgajUbbRhAELFy4EO7u7rC3t8crr7yCS5cuvbDvHTt2IDAwEBKJBIGBgdi1a1d1DoWIiIgsjFmHpNjYWGzYsAHr1q3D5cuXsXLlSnz88cdYu3atts3KlSuxatUqrFu3DqdPn4ZMJkOfPn2Qm5tbbr8nT57E6NGjMX78eFy4cAHjx4/HqFGjcOrUqZoYFhEREVkAkSAIgqmLKM+gQYPg5uaGr7/+WrstIiICDg4O2Lx5MwRBgLu7OyIjIzFnzhwAgEqlgpubG2JjYzF16lS9/Y4ePRpKpRIHDx7Ubuvfvz8aNGiArVu3Vqo2pVIJqVQKhUIBZ2fnlxglERER1RRDvr+ta6imKunatSs2bNiAq1evokWLFrhw4QISEhLw2WefAQBu3ryJzMxM9O3bV7uPRCJBjx49cOLEiXJD0smTJ/H+++/rbOvXr5+2X31UKhVUKpX2Z4VCAeDJf2wiIiKyDE+/tyszR2TWIWnOnDlQKBQICAiAWCyGWq3GsmXLMHbsWABAZmYmAMDNzU1nPzc3N6SlpZXbb2Zmpt59nvanT0xMDBYtWlRmu6enZ6XHQ0REROYhNzcXUqm0wjZmHZK2b9+OLVu24Pvvv0dQUBASExMRGRkJd3d3TJw4UdtOJBLp7CcIQpltf2boPlFRUZg5c6b2Z41Gg8ePH6Nhw4Yv/CxDKZVKeHp64vbt27XyVB7HZ/lq+xhr+/iA2j9Gjs/yVdcYBUFAbm4u3N3dX9jWrEPSrFmzMHfuXIwZMwYAEBISgrS0NMTExGDixImQyWQAnswMyeVy7X5ZWVllZoqeJ5PJyswavWgfiUQCiUSis61+/fqGDskgzs7OtfYvP8Dx1Qa1fYy1fXxA7R8jx2f5qmOML5pBesqsr24rKCiAlZVuiWKxWHsLAB8fH8hkMsTHx2vfLy4uxrFjx9C5c+dy+w0PD9fZBwAOHTpU4T5ERERUt5j1TNLrr7+OZcuWoWnTpggKCsL58+exatUqTJ48GcCTU2aRkZFYvnw5/Pz84Ofnh+XLl8PBwQHjxo3T9jNhwgR4eHggJiYGADBjxgx0794dsbGxGDJkCPbs2YPDhw8jISHBJOMkIiIi82PWIWnt2rWIjo7GtGnTkJWVBXd3d0ydOhUfffSRts3s2bNRWFiIadOmITs7Gx07dsShQ4fg5OSkbZOenq4zI9W5c2ds27YNCxYsQHR0NJo1a4bt27ejY8eONTq+8kgkEvzjH/8oc3qvtuD4LF9tH2NtHx9Q+8fI8Vk+cxijWd8niYiIiMhUzHpNEhEREZGpMCQRERER6cGQRERERKQHQxIRERGRHgxJJvLFF1/Ax8cHdnZ2aNu2LX755ZcK2x87dgxt27aFnZ0dfH19sWHDhhqqtGoMGd/Ro0chEonKvK5cuVKDFVfe8ePH8frrr8Pd3R0ikQi7d+9+4T6WdPwMHZ+lHb+YmBi0b98eTk5OaNy4MYYOHYrU1NQX7mcpx7Aq47O0Y7h+/Xq0atVKe5PB8PBwnQeW62Mpxw8wfHyWdvz+LCYmRntLn4qY4hgyJJnA9u3bERkZifnz5+P8+fPo1q0bBgwYgPT0dL3tb968iddeew3dunXD+fPnMW/ePLz33nvYsWNHDVdeOYaO76nU1FRkZGRoX35+fjVUsWHy8/MRGhqKdevWVaq9pR0/Q8f3lKUcv2PHjmH69On43//+h/j4eJSWlqJv377Iz88vdx9LOoZVGd9TlnIMmzRpghUrVuDMmTM4c+YMXn31VQwZMgSXLl3S296Sjh9g+PiespTj97zTp0/jyy+/RKtWrSpsZ7JjKFCN69Chg/DOO+/obAsICBDmzp2rt/3s2bOFgIAAnW1Tp04VOnXqVG01vgxDx/fzzz8LAITs7OwaqM64AAi7du2qsI2lHb/nVWZ8lnz8BEEQsrKyBADCsWPHym1jycewMuOz9GMoCILQoEED4f/9v/+n9z1LPn5PVTQ+Sz1+ubm5gp+fnxAfHy/06NFDmDFjRrltTXUMOZNUw4qLi3H27Fn07dtXZ3vfvn1x4sQJvfucPHmyTPt+/frhzJkzKCkpqbZaq6Iq43sqLCwMcrkcvXr1ws8//1ydZdYoSzp+L8NSj59CoQAAuLi4lNvGko9hZcb3lCUeQ7VajW3btiE/Px/h4eF621jy8avM+J6ytOM3ffp0DBw4EL17935hW1MdQ4akGvbw4UOo1eoyD9N1c3Mr89DdpzIzM/W2Ly0txcOHD6ut1qqoyvjkcjm+/PJL7NixAzt37oS/vz969eqF48eP10TJ1c6Sjl9VWPLxEwQBM2fORNeuXREcHFxuO0s9hpUdnyUew6SkJNSrVw8SiQTvvPMOdu3ahcDAQL1tLfH4GTI+Szx+27Ztw7lz57SPC3sRUx1Ds34sSW0mEol0fhYEocy2F7XXt91cGDI+f39/+Pv7a38ODw/H7du38c9//hPdu3ev1jpriqUdP0NY8vF79913cfHixUo9t9ESj2Flx2eJx9Df3x+JiYnIycnBjh07MHHiRBw7dqzcIGFpx8+Q8Vna8bt9+zZmzJiBQ4cOwc7OrtL7meIYciaphrm6ukIsFpeZVcnKyiqTkp+SyWR621tbW6Nhw4bVVmtVVGV8+nTq1AnXrl0zdnkmYUnHz1gs4fj9/e9/x969e/Hzzz+jSZMmFba1xGNoyPj0MfdjaGtri+bNm6Ndu3aIiYlBaGgoVq9erbetJR4/Q8anjzkfv7NnzyIrKwtt27aFtbU1rK2tcezYMaxZswbW1tZQq9Vl9jHVMWRIqmG2trZo27Yt4uPjdbbHx8ejc+fOevcJDw8v0/7QoUNo164dbGxsqq3WqqjK+PQ5f/485HK5scszCUs6fsZizsdPEAS8++672LlzJ44cOQIfH58X7mNJx7Aq49PHnI+hPoIgQKVS6X3Pko5feSoanz7mfPx69eqFpKQkJCYmal/t2rXDX/7yFyQmJkIsFpfZx2THsFqXhZNe27ZtE2xsbISvv/5aSElJESIjIwVHR0fh1q1bgiAIwty5c4Xx48dr29+4cUNwcHAQ3n//fSElJUX4+uuvBRsbG+GHH34w1RAqZOj4Pv30U2HXrl3C1atXheTkZGHu3LkCAGHHjh2mGkKFcnNzhfPnzwvnz58XAAirVq0Szp8/L6SlpQmCYPnHz9DxWdrx+9vf/iZIpVLh6NGjQkZGhvZVUFCgbWPJx7Aq47O0YxgVFSUcP35cuHnzpnDx4kVh3rx5gpWVlXDo0CFBECz7+AmC4eOztOOnz5+vbjOXY8iQZCKff/654OXlJdja2gpt2rTRuTx34sSJQo8ePXTaHz16VAgLCxNsbW0Fb29vYf369TVcsWEMGV9sbKzQrFkzwc7OTmjQoIHQtWtXYf/+/SaounKeXm7759fEiRMFQbD842fo+Czt+OkbGwDh22+/1bax5GNYlfFZ2jGcPHmy9t+XRo0aCb169dIGCEGw7OMnCIaPz9KOnz5/DknmcgxFgvDHyiciIiIi0uKaJCIiIiI9GJKIiIiI9GBIIiIiItKDIYmIiIhID4YkIiIiIj0YkoiIiIj0YEgiIiIi0oMhiYjISI4ePQqRSIScnBxTl0JERsCQRERERKQHQxIRERGRHgxJRFRrCIKAlStXwtfXF/b29ggNDcUPP/wA4NmpsP379yM0NBR2dnbo2LEjkpKSdPrYsWMHgoKCIJFI4O3tjU8++UTnfZVKhdmzZ8PT0xMSiQR+fn74+uuvddqcPXsW7dq1g4ODAzp37ozU1NTqHTgRVQuGJCKqNRYsWIBvv/0W69evx6VLl/D+++/jr3/9K44dO6ZtM2vWLPzzn//E6dOn0bhxYwwePBglJSUAnoSbUaNGYcyYMUhKSsLChQsRHR2NjRs3avefMGECtm3bhjVr1uDy5cvYsGED6tWrp1PH/Pnz8cknn+DMmTOwtrbG5MmTa2T8RGRcfMAtEdUK+fn5cHV1xZEjRxAeHq7dPmXKFBQUFODtt99Gz549sW3bNowePRoA8PjxYzRp0gQbN27EqFGj8Je//AUPHjzAoUOHtPvPnj0b+/fvx6VLl3D16lX4+/sjPj4evXv3LlPD0aNH0bNnTxw+fBi9evUCABw4cAADBw5EYWEh7Ozsqvm/AhEZE2eSiKhWSElJQVFREfr06YN69eppX9999x1+//13bbvnA5SLiwv8/f1x+fJlAMDly5fRpUsXnX67dOmCa9euQa1WIzExEWKxGD169KiwllatWmn/LJfLAQBZWVkvPUYiqlnWpi6AiMgYNBoNAGD//v3w8PDQeU8ikegEpT8TiUQAnqxpevrnp56fbLe3t69ULTY2NmX6flofEVkOziQRUa0QGBgIiUSC9PR0NG/eXOfl6empbfe///1P++fs7GxcvXoVAQEB2j4SEhJ0+j1x4gRatGgBsViMkJAQaDQanTVORFR7cSaJiGoFJycnfPjhh3j//feh0WjQtWtXKJVKnDhxAvXq1YOXlxcAYPHixWjYsCHc3Nwwf/58uLq6YujQoQCADz74AO3bt8eSJUswevRonDx5EuvWrcMXX3wBAPD29sbEiRMxefJkrFmzBqGhoUhLS0NWVhZGjRplqqETUTVhSCKiWmPJkiVo3LgxYmJicOPGDdSvXx9t2rTBvHnztKe7VqxYgRkzZuDatWsIDQ3F3r17YWtrCwBo06YN/u///g8fffQRlixZArlcjsWLF+ONN97Qfsb69esxb948TJs2DY8ePULTpk0xb948UwyXiKoZr24jojrh6ZVn2dnZqF+/vqnLISILwDVJRERERHowJBERERHpwdNtRERERHpwJomIiIhID4YkIiIiIj0YkoiIiIj0YEgiIiIi0oMhiYiIiEgPhiQiIiIiPRiSiIiIiPRgSCIiIiLSgyGJiIiISI//D674pTXVsk6xAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plt.plot(np.arange(len(epochs_x)), epochs_acc, marker = '.')\n", + "plt.xlabel('epoch')\n", + "plt.ylabel('test accuracy')\n", + "plt.ylim(80, 100)\n", + "for i, txt in enumerate(epochs_acc):\n", + " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom')\n", "plt.show()" ] } From cddc499c86985a331c0461069a940f46608552d3 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Fri, 26 Apr 2024 20:54:51 +0200 Subject: [PATCH 047/379] 2nd non-sequential example --- .../non-sequential-SCNN-example_2.ipynb | 430 ++++++++++++++++++ 1 file changed, 430 insertions(+) create mode 100644 tests/test_nonsequential/non-sequential-SCNN-example_2.ipynb diff --git a/tests/test_nonsequential/non-sequential-SCNN-example_2.ipynb b/tests/test_nonsequential/non-sequential-SCNN-example_2.ipynb new file mode 100644 index 00000000..754d2065 --- /dev/null +++ b/tests/test_nonsequential/non-sequential-SCNN-example_2.ipynb @@ -0,0 +1,430 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "import torch.nn as nn\n", + "import sinabs.layers as sl\n", + "from tqdm.notebook import tqdm\n", + "\n", + "from tonic.datasets.nmnist import NMNIST\n", + "from tonic.transforms import ToFrame\n", + "from torch.utils.data import DataLoader\n", + "from torch.nn import CrossEntropyLoss\n", + "from torch.optim import Adam\n", + "\n", + "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential\n", + "\n", + "import matplotlib.pyplot as plt" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "torch.manual_seed(0)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Loading Data" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "batch_size = 64\n", + "num_workers = 4\n", + "epochs = 5\n", + "lr = 1e-3" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "root_dir = \"./NMNIST\"\n", + "_ = NMNIST(save_to=root_dir, train=True)\n", + "_ = NMNIST(save_to=root_dir, train=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "n_time_steps = 50\n", + "to_raster = ToFrame(sensor_size=NMNIST.sensor_size, n_time_bins=n_time_steps)\n", + "\n", + "snn_train_dataset = NMNIST(save_to=root_dir, train=True, transform=to_raster)\n", + "snn_test_dataset = NMNIST(save_to=root_dir, train=False, transform=to_raster)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "snn_train_dataloader = DataLoader(snn_train_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True)\n", + "snn_test_dataloader = DataLoader(snn_test_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=False)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Network Module\n", + "\n", + "We need to define a `nn.Module` implementing the network we want the chip to reproduce." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "device: NVIDIA GeForce RTX 3070 Ti\n" + ] + } + ], + "source": [ + "if torch.cuda.is_available():\n", + " device = torch.device('cuda:0')\n", + " print('device: ', torch.cuda.get_device_name(0))\n", + "else:\n", + " device = torch.device('cpu')" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "class SNN(nn.Module):\n", + " def __init__(self) -> None:\n", + " super().__init__()\n", + "\n", + " self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False)\n", + " self.iaf1 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + " self.pool1 = nn.AvgPool2d(2,2)\n", + "\n", + " self.conv2 = nn.Conv2d(10, 10, 2, 1, bias=False)\n", + " self.iaf2 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + " self.pool2 = nn.AvgPool2d(3,3)\n", + "\n", + " self.conv3 = nn.Conv2d(10, 10, 3, 1, bias=False)\n", + " self.iaf3 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + " self.pool3 = nn.AvgPool2d(2,2)\n", + "\n", + " self.flat = nn.Flatten()\n", + "\n", + " self.fc1 = nn.Linear(10, 100, bias=False)\n", + " self.iaf4 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + "\n", + " self.fc2 = nn.Linear(100, 100, bias=False)\n", + " self.iaf5 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + "\n", + " self.fc3 = nn.Linear(100, 100, bias=False)\n", + " self.iaf6 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + "\n", + " self.fc4 = nn.Linear(100, 10, bias=False)\n", + " self.iaf7 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + "\n", + " self.merge = sl.Merge()\n", + "\n", + " def detach_neuron_states(self):\n", + " for name, layer in self.named_modules():\n", + " if name != '':\n", + " if isinstance(layer, sl.StatefulLayer):\n", + " for name, buffer in layer.named_buffers():\n", + " buffer.detach_()\n", + "\n", + " def init_weights(self):\n", + " for name, layer in self.named_modules():\n", + " if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear):\n", + " nn.init.xavier_normal_(layer.weight.data)\n", + "\n", + " def forward(self, x):\n", + " \n", + " con1_out = self.conv1(x)\n", + " iaf1_out = self.iaf1(con1_out)\n", + " pool1_out = self.pool1(iaf1_out)\n", + "\n", + " conv2_out = self.conv2(pool1_out)\n", + " iaf2_out = self.iaf2(conv2_out)\n", + " pool2_out = self.pool2(iaf2_out)\n", + "\n", + " conv3_out = self.conv3(pool2_out)\n", + " iaf3_out = self.iaf3(conv3_out)\n", + " pool3_out = self.pool3(iaf3_out)\n", + "\n", + " flat_out = self.flat(pool3_out)\n", + " \n", + " fc1_out = self.fc1(flat_out)\n", + " iaf4_out = self.iaf4(fc1_out)\n", + "\n", + " fc2_out = self.fc2(iaf4_out)\n", + " iaf5_out = self.iaf5(fc2_out)\n", + "\n", + " fc3_out = self.fc3(iaf5_out)\n", + " iaf6_out = self.iaf6(fc3_out)\n", + "\n", + " merge_out = self.merge(iaf4_out, iaf6_out)\n", + "\n", + " fc4_out = self.fc4(merge_out)\n", + " iaf7_out = self.iaf7(fc4_out)\n", + "\n", + " return iaf7_out" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "snn = SNN().to(device)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "snn.init_weights()" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "optimizer = Adam(snn.parameters(), lr=lr, betas=(0.9, 0.999), eps=1e-8)\n", + "loss_fn = CrossEntropyLoss()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Define train and test" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "def train(dataloader, model, loss_fn, optimizer, epochs, test_func, dataloader_test):\n", + " epochs_y = []\n", + " epochs_x = []\n", + " epochs_acc = []\n", + " model.train()\n", + "\n", + " for e in range(epochs):\n", + " losses = []\n", + " batches = []\n", + " batch_count = 0\n", + " train_p_bar = tqdm(snn_train_dataloader)\n", + "\n", + " for X, y in train_p_bar:\n", + " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", + " X = X.reshape(-1, 2, 34, 34).to(dtype=torch.float, device=device)\n", + " y = y.to(dtype=torch.long, device=device)\n", + "\n", + " # forward\n", + " pred = model(X)\n", + "\n", + " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", + " pred = pred.reshape(batch_size, n_time_steps, -1)\n", + "\n", + " # accumulate all time-steps output for final prediction\n", + " pred = pred.sum(dim = 1)\n", + " loss = loss_fn(pred, y)\n", + "\n", + " # gradient update\n", + " optimizer.zero_grad()\n", + " loss.backward()\n", + " optimizer.step()\n", + "\n", + " # detach the neuron states and activations from current computation graph(necessary)\n", + " model.detach_neuron_states()\n", + "\n", + " train_p_bar.set_description(f\"Epoch {e} - BPTT Training Loss: {round(loss.item(), 4)}\")\n", + "\n", + " batch_count += 1\n", + " losses.append(loss.item())\n", + " batches.append(batch_count)\n", + "\n", + " epochs_y.append(losses)\n", + " epochs_x.append(batches)\n", + "\n", + " acc = test_func(dataloader_test, model)\n", + " print(f'Epoch {e} accuracy: {acc}')\n", + " epochs_acc.append(acc)\n", + "\n", + " return epochs_x, epochs_y, epochs_acc\n" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "def test(dataloader, model):\n", + " correct_predictions = []\n", + " with torch.no_grad():\n", + " test_p_bar = tqdm(dataloader)\n", + " for X, y in test_p_bar:\n", + " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", + " X = X.reshape(-1, 2, 34, 34).to(dtype=torch.float, device=device)\n", + " y = y.to(dtype=torch.long, device=device)\n", + "\n", + " # forward\n", + " output = model(X)\n", + "\n", + " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", + " output = output.reshape(batch_size, n_time_steps, -1)\n", + "\n", + " # accumulate all time-steps output for final prediction\n", + " output = output.sum(dim=1)\n", + "\n", + " # calculate accuracy\n", + " pred = output.argmax(dim=1, keepdim=True)\n", + "\n", + " # compute the total correct predictions\n", + " correct_predictions.append(pred.eq(y.view_as(pred)))\n", + "\n", + " test_p_bar.set_description(f\"Testing Model...\")\n", + " \n", + " correct_predictions = torch.cat(correct_predictions)\n", + " return correct_predictions.sum().item()/(len(correct_predictions))*100" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Training loop (HPO)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "546940b28031476fb3bee7a239fd1529", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/937 [00:00 1\u001b[0m epochs_x, epochs_y \u001b[38;5;241m=\u001b[39m train(snn_train_dataloader, snn, loss_fn, optimizer, epochs, test, snn_test_dataloader)\n", + "Cell \u001b[0;32mIn[13], line 29\u001b[0m, in \u001b[0;36mtrain\u001b[0;34m(dataloader, model, loss_fn, optimizer, epochs, test_func, dataloader_test)\u001b[0m\n\u001b[1;32m 27\u001b[0m \u001b[38;5;66;03m# gradient update\u001b[39;00m\n\u001b[1;32m 28\u001b[0m optimizer\u001b[38;5;241m.\u001b[39mzero_grad()\n\u001b[0;32m---> 29\u001b[0m loss\u001b[38;5;241m.\u001b[39mbackward()\n\u001b[1;32m 30\u001b[0m optimizer\u001b[38;5;241m.\u001b[39mstep()\n\u001b[1;32m 32\u001b[0m \u001b[38;5;66;03m# detach the neuron states and activations from current computation graph(necessary)\u001b[39;00m\n", + "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/_tensor.py:522\u001b[0m, in \u001b[0;36mTensor.backward\u001b[0;34m(self, gradient, retain_graph, create_graph, inputs)\u001b[0m\n\u001b[1;32m 512\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m has_torch_function_unary(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m 513\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m handle_torch_function(\n\u001b[1;32m 514\u001b[0m Tensor\u001b[38;5;241m.\u001b[39mbackward,\n\u001b[1;32m 515\u001b[0m (\u001b[38;5;28mself\u001b[39m,),\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 520\u001b[0m inputs\u001b[38;5;241m=\u001b[39minputs,\n\u001b[1;32m 521\u001b[0m )\n\u001b[0;32m--> 522\u001b[0m torch\u001b[38;5;241m.\u001b[39mautograd\u001b[38;5;241m.\u001b[39mbackward(\n\u001b[1;32m 523\u001b[0m \u001b[38;5;28mself\u001b[39m, gradient, retain_graph, create_graph, inputs\u001b[38;5;241m=\u001b[39minputs\n\u001b[1;32m 524\u001b[0m )\n", + "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/autograd/__init__.py:266\u001b[0m, in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[0m\n\u001b[1;32m 261\u001b[0m retain_graph \u001b[38;5;241m=\u001b[39m create_graph\n\u001b[1;32m 263\u001b[0m \u001b[38;5;66;03m# The reason we repeat the same comment below is that\u001b[39;00m\n\u001b[1;32m 264\u001b[0m \u001b[38;5;66;03m# some Python versions print out the first line of a multi-line function\u001b[39;00m\n\u001b[1;32m 265\u001b[0m \u001b[38;5;66;03m# calls in the traceback and some print out the last line\u001b[39;00m\n\u001b[0;32m--> 266\u001b[0m Variable\u001b[38;5;241m.\u001b[39m_execution_engine\u001b[38;5;241m.\u001b[39mrun_backward( \u001b[38;5;66;03m# Calls into the C++ engine to run the backward pass\u001b[39;00m\n\u001b[1;32m 267\u001b[0m tensors,\n\u001b[1;32m 268\u001b[0m grad_tensors_,\n\u001b[1;32m 269\u001b[0m retain_graph,\n\u001b[1;32m 270\u001b[0m create_graph,\n\u001b[1;32m 271\u001b[0m inputs,\n\u001b[1;32m 272\u001b[0m allow_unreachable\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m 273\u001b[0m accumulate_grad\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m 274\u001b[0m )\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m: " + ] + } + ], + "source": [ + "epochs_x, epochs_y, epochs_acc = train(snn_train_dataloader, snn, loss_fn, optimizer, epochs, test, snn_test_dataloader)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAGwCAYAAABVdURTAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAABpgUlEQVR4nO3dd3hUVfoH8O+UZNI7KUACoRiatAQlSBUMAhbWuhZQdy2oCBhZFOtaw6q/FbGAKIosIu5uUFlBBJQqHRJ6EwIJISEESK8zc39/hJncO3OnZDIlmXw/z5PHmXvPvXNmgszLe95zjkIQBAFEREREXkLp6Q4QERERORODGyIiIvIqDG6IiIjIqzC4ISIiIq/C4IaIiIi8CoMbIiIi8ioMboiIiMirqD3dAXfT6/U4f/48goODoVAoPN0dIiIisoMgCCgvL0f79u2hVFrPzbS54Ob8+fOIj4/3dDeIiIjIAXl5eejYsaPVNm0uuAkODgbQ8OGEhIR4uDdERERkj7KyMsTHxxu/x61pc8GNYSgqJCSEwQ0REVErY09JCQuKiYiIyKswuCEiIiKvwuCGiIiIvAqDGyIiIvIqDG6IiIjIqzC4ISIiIq/C4IaIiIi8CoMbIiIi8ioMboiIiMirMLghIiIir8LghoiIiLwKgxsnqtPqcdvHWzFjeZanu0JERNRmMbhxoqzcKzhwrhQ/ZJ9HvU7v6e4QERG1SQxunEirF4yPL5bXerAnREREbReDGye6XFlnfFxQWuPBnhAREbVdDG6c6FJFY7amoLTagz0hIiJquxjcOJE4c1PMYSkiIiKPYHDjRJdEwY1OsNKQiIiIXIbBjRPVahtnSAkCoxsiIiJPYHDjRHpRQKNncENEROQRDG6cSBzP6BnbEBEReQSDGydi5oaIiMjzGNw4kThbw9iGiIjIMxjcOJE4W8OCYiIiIs9gcONEgmRYyoMdISIiasMY3DiRXrRXJmtuiIiIPIPBjRMJYOaGiIjI0xjcOJEkoGHmhoiIyCMY3DgRa26IiIg8j8GNE+kli/gxuiEiIvIEBjdOpGfmhoiIyOMY3DiRdBE/RjdERESewODGicQBDUMbIiIiz2Bw40SSYSmOSxEREXkEgxsnki7i57l+EBERtWUMbpyIu4ITERF5HoMbJxJYUExERORxDG6cSM+CYiIiIo9jcONEHJYiIiLyPI8GN/Pnz0ffvn0REhKCkJAQpKam4ueff7bYfuPGjVAoFGY/x44dc2OvLZOuUOy5fhAREbVlak++eMeOHTFnzhx069YNAPD111/j9ttvR1ZWFnr37m3xuuPHjyMkJMT4vF27di7vqz0k69wwc0NEROQRHg1ubr31Vsnzt99+G/Pnz8eOHTusBjfR0dEICwuz6zVqa2tRW1trfF5WVuZQX+0hydzoLbcjIiIi12kxNTc6nQ7Lly9HZWUlUlNTrbYdMGAA4uLiMHr0aGzYsMFq24yMDISGhhp/4uPjndltCWlBMTM3REREnuDx4ObgwYMICgqCRqPBlClT8P3336NXr16ybePi4rBw4UJkZmZixYoVSEpKwujRo7F582aL9589ezZKS0uNP3l5ea56K6y5ISIiagE8OiwFAElJScjOzkZJSQkyMzPx0EMPYdOmTbIBTlJSEpKSkozPU1NTkZeXh/fffx/Dhw+Xvb9Go4FGo3FZ/8UEzpYiIiLyOI9nbnx9fdGtWzekpKQgIyMD/fr1w4cffmj39YMHD8bJkydd2EP7SRfx81w/iIiI2jKPBzemBEGQFADbkpWVhbi4OBf2yH56zpYiIiLyOI8OS7344osYN24c4uPjUV5ejuXLl2Pjxo1Ys2YNgIZ6mfz8fCxZsgQAMHfuXHTu3Bm9e/dGXV0dli5diszMTGRmZnrybRhJF/HzYEeIiIjaMI8GNxcuXMCkSZNQUFCA0NBQ9O3bF2vWrMFNN90EACgoKEBubq6xfV1dHWbOnIn8/Hz4+/ujd+/eWLVqFcaPH++ptyAhSAqKGd0QERF5gkJoY+MnZWVlCA0NRWlpqWQhQGcY+d4GnLlUBQCYcG0cPnlgoFPvT0RE1FY15fu7xdXctGZ6Zm6IiIg8jsGNE0kLij3YESIiojaMwY0TseaGiIjI8xjcOBFnSxEREXkegxsn4jo3REREnsfgxolYUExEROR5DG6cSJDsCk5ERESewODGibgrOBERkecxuHEi1twQERF5HoMbJ9LrxbOlGNwQERF5AoMbJ5Ksc6P3XD+IiIjaMgY3TiRIHjNzQ0RE5AkMbpxIPBS14/Rl6FhVTERE5HYMbpzItM7my605HuoJERFR28XgxolMEzXf7cnzTEeIiIjaMAY3TmQ6/buqVuuhnhAREbVdDG6cyDRzU1mn80xHiIiI2jAGN05kWnNTVcfMDRERkbsxuHESQRBgum5fvY6zpYiIiNyNwY2TcEFiIiKiloHBjZNwuwUiIqKWgcGNk3C9PiIiopaBwY2TMHNDRETUMjC4cRLGNkRERC0DgxsnYeaGiIioZWBw4yQMboiIiFoGBjdOwoJiIiKiloHBjZOY7itFREREnsHgxkmYuSEiImoZGNw4iaXMjZ5RDxERkVsxuHESSzGMlsENERGRWzG4cRKLmRvW4hAREbkVgxsniQzSYMfs0XhrYh/JcWZuiIiI3IvBjZOolArEhvohOlgjOa7TMbghIiJyJwY3TqYzydRo9XoP9YSIiKht8mhwM3/+fPTt2xchISEICQlBamoqfv75Z6vXbNq0CcnJyfDz80OXLl2wYMECN/XWPu1MMzesuSEiInIrjwY3HTt2xJw5c7Bnzx7s2bMHN954I26//XYcPnxYtn1OTg7Gjx+PYcOGISsrCy+++CKmTZuGzMxMN/fcspTOEXhxfA/jc9NMDhEREbmWQmhhS+tGRETgvffew1//+lezc88//zxWrlyJo0ePGo9NmTIF+/fvx/bt2+26f1lZGUJDQ1FaWoqQkBCn9dtUj1d+Rk29HltmjUJ8RIDLXoeIiKgtaMr3d4upudHpdFi+fDkqKyuRmpoq22b79u1IS0uTHBs7diz27NmD+vp62Wtqa2tRVlYm+XEHtbLho+VUcCIiIvfyeHBz8OBBBAUFQaPRYMqUKfj+++/Rq1cv2baFhYWIiYmRHIuJiYFWq0VxcbHsNRkZGQgNDTX+xMfHO/09yFEpFQA4FZyIiMjdPB7cJCUlITs7Gzt27MCTTz6Jhx56CEeOHLHYXqFQSJ4bRtVMjxvMnj0bpaWlxp+8vDzndd4KQ3DDmhsiIiL3Unu6A76+vujWrRsAICUlBbt378aHH36Izz77zKxtbGwsCgsLJceKioqgVqsRGRkpe3+NRgONRiN7zpWMmRuuc0NERORWHs/cmBIEAbW1tbLnUlNTsW7dOsmxtWvXIiUlBT4+Pu7ont3UV4Mb1twQERG5l0eDmxdffBFbtmzBmTNncPDgQbz00kvYuHEjHnjgAQANQ0qTJ082tp8yZQrOnj2L9PR0HD16FF9++SUWLVqEmTNneuotWMSaGyIiIs/w6LDUhQsXMGnSJBQUFCA0NBR9+/bFmjVrcNNNNwEACgoKkJuba2yfmJiI1atX49lnn8Unn3yC9u3bY968ebjzzjs99RYsaqy54QrFRERE7uTR4GbRokVWzy9evNjs2IgRI7Bv3z4X9ch5GoMbD3eEiIiojWlxNTfeQm0clmJ0Q0RE5E4MblxEqeBUcCIiIk9gcOMiahULiomIiDyBwY2LqAzbLzC4ISIicisGNy6i5lRwIiIij2Bw4yIq1twQERF5BIMbF+EifkRERJ7B4MZFDAXFrLkhIiJyLwY3LmKYCs7MDRERkXsxuHERNbdfICIi8ggGNy7C7ReIiIg8g8GNixhqbpi5ISIici8GNy7CmhsiIiLPYHDjIo01NwxuiIiI3InBjYsYtl9gcENEROReDG5cxJC5yfj5GGq1Og/3hoiIqO1gcOMiyqvBDQDM33jKgz0hIiJqWxjcuIhaFNwcOFfqwZ4QERG1LQxuXEQlCm4qarUe7AkREVHbwuDGRcTBTVUdgxsiIiJ3YXDjIuJhqapaFhQTERG5C4MbFxFnbk4XV6K0qt6DvSEiImo7GNy4iDhzAwBLtp/xTEeIiIjaGAY3LqI0CW72nyvxTEeIiIjaGAY3LmKauSkorfFQT4iIiNoWBjcuYth+waC8hjOmiIiI3IHBjYuoTD5ZrnVDRETkHgxuXMQ8c1MPQeAmmkRERK7G4MZFVNKSG9TrBNRq9Z7pDBERURvC4MaNWHdDRETkegxuXEQ8AGVY0K+8hgv5ERERuRqDGxfRi6Kb8AAfACwqJiIicgcGNy4iLh5uF+wHADhfwrVuiIiIXI3BjRv0jw8FAGTlXfFwT4iIiLwfgxsXEc/67h8fBgA4cr7MM50hIiJqQzwa3GRkZGDQoEEIDg5GdHQ0Jk6ciOPHj1u9ZuPGjVAoFGY/x44dc1Ov7aMXRTcxIQ3DUpcr65CdV4L1Ry54qltERERez6PBzaZNm/D0009jx44dWLduHbRaLdLS0lBZWWnz2uPHj6OgoMD40717dzf02H7i2VJhAb4AgJKqekz85Hc8umQPTl4o90zHiIiIvJzaky++Zs0ayfOvvvoK0dHR2Lt3L4YPH2712ujoaISFhbmwd80jHpYK82+YLVVSVWc8dqSgDN1jgt3dLfJi/9pxFvHh/hiZFO3prhAReVSLqrkpLS0FAERERNhsO2DAAMTFxWH06NHYsGGDxXa1tbUoKyuT/LiDeFgq/GrmprJOZzxWUsU1b8h59ueV4JUfDuHhr3Z7uitERB7XYoIbQRCQnp6OoUOHok+fPhbbxcXFYeHChcjMzMSKFSuQlJSE0aNHY/PmzbLtMzIyEBoaavyJj4931VuwKNhPDYXJdgxXRFkcouY6X1Lt6S4QEbUYHh2WEps6dSoOHDiArVu3Wm2XlJSEpKQk4/PU1FTk5eXh/ffflx3Kmj17NtLT043Py8rK3BLgGFYlBgClUoFQfx9JtoaZG3ImbslKRNSoRWRunnnmGaxcuRIbNmxAx44dm3z94MGDcfLkSdlzGo0GISEhkh93uO+6BCRGBeKJEV0AAIlRgZLzZdX1KKmqw/JdudyWgYiIyIk8mrkRBAHPPPMMvv/+e2zcuBGJiYkO3ScrKwtxcXFO7l3zhPr7YMPMkcbnt/Rtj6zcEuPz8lotZq84iJ8PFeLXY0X4fHKK+ztJXkNcwK7XC1AqFZYbExF5OY8GN08//TSWLVuGH3/8EcHBwSgsLAQAhIaGwt/fH0DDsFJ+fj6WLFkCAJg7dy46d+6M3r17o66uDkuXLkVmZiYyMzM99j7sEROikTwvr6nHjtOXAQDrjlxAdl6JcbE/oqYSRANT9Xo9NEqVB3tDRORZHh2Wmj9/PkpLSzFy5EjExcUZf7777jtjm4KCAuTm5hqf19XVYebMmejbty+GDRuGrVu3YtWqVbjjjjs88RbsplFLv2wuVUgLiid+8rs7u0NeRpy50eqcW4FTVFaDd9ccw7krVU69LxGRq3h8WMqWxYsXS57PmjULs2bNclGPXEejlsaRJ4sqzNrU6/TwUbWIMihqZcT/Jzk7uHli6V5k5Zbgl8OF+PW5kU69NxGRK/Cb1E181bY/6vIarRt6Qt5I/A+Fer3eqfc21Iqdumh75XAiopaAwY2bmGZu5JRV2zdrSqd3zcTfOT8fw6RFO1Gvk3457j17GTOWZ6GorMbq9fqr/Xp3zTF8seU0zhRX4ostp1EtWryQXKNelK1xduaGiKi1aTHr3Hg705obOZcq69DZZMq4qR+z8/F85gF8dN9A3NQrxlndAwAs2HQKALD1j2KMEi3hf+f87QAavkA/eWCg5Bq9XkB1vQ6PLN6N4vJazLtvAD7d2HCft1YdBQBo9QKmjOjq1L6SlDggNQ1OiYjaGmZu3ETjY/ujvnP+Nkz7Ngt6vYBlO3PxR1Hj5prVdTr8kJWP6cuzUVOvx/OZB5zav1ptY3bFUi3UkYKGrStKq+qN2aOHvtqFfq+vxa6cyzhdXIlPNvxhft1592x50RIJgoDPN5/G/I2nMH/jKdTU6yAIArLzSlBR67xhSLngpk6rx4JNp3CssO1+/kTUNjFz4ya+dhYKr9x/HnpBwE8HCgAAZ+ZMAADM+fkovt5+1tjO36cxE1RVp4WfWiVZ2+TI+TJMW56FGWO645a+7Y3HD5wrQYifD44WlOHfe/Lwf/f0R0SgL0pFKyYrRXtFiAMdpQLIKa5E2gebMDIpGp9PTsGWk8WS/v98qNDsPXUM97frvZvS6QUs352L6zpHtLpNRnOKK6FSKFBUXoO3Vx81HtcLArq2C8SUpfswMCEMK566ocn3rq7Twd9Xmgms0zYGN9qrgefnW07jvV+OY87Px4x/jprDdAsRIqKWipkbN7Enc2NgCGwMSqvqJYENAFyqrEVRWQ3mbzyFXq/+gie/2YuV+89jxvIs1NTr8Pr/DuOPogpMXZZlvCYr9wpu+/h33Pf5Djz5zT5sOH4R7/1yHABwRRTc5BRXYvaKA8gprsSlysYp6wqFAr8cLkS9TsC6IxcwY3njva2pqW/44i2vqZfNCp27UoV31xwz1vQYMg+Ze8/hpe8P4aYP5PcNa6rNJy7ihjm/YcvJi8ZjK/adw6RFO3GxvNasvSAIds3oM7Q1qKrTYtT7GzH8vQ04e0k6fXp/Xgne/Kkh2NknWtTRlE4v4O8rD2PR1hyMfG8D5q4/AQDYdqoYPV9dg7nrT+BQfinuW7gD2Xklkpqb4opa7Mu9gj1nLpv1ccq/9uK+hTuM9VFNYU/dmKOuVNbZ/VkTEdnCzI2b2FNzI+dyZR0GvrnO7HhNvR7XvfOr8fkvhy/gl8MXAAD94sMkM68EQUCtVo/7P98JACgobSwM/vlQAa5LDMeH6xu3r3j9f0cAAN/uysPMtGuMx/8oqpC0+yH7vF3vobJWi5MXyjF+3hb4+agwuEskqut0eHx4Fwy/ph0eW7IXRwvKsPfsFYzqEY2560/gTwM64ttdubL3e+7f+1FRW487B3aEr1qJxKhAdIoMhFanh1qUIfvnuhM4eK4En01Kga9aiXdWH0V+STUmLdqF9ekjUKfVI/3f+wEAX287g6k3dkNpdT1iQvwAAA8u2olLFXVYOXUofNVK1NTrcP/nO9AhPADv3dUXfj4qXKmsw8j3N6K0uh6v3doL91+fgMx9+cY+XK6Urme09sgFuz6zad9mYdXBxiB37vqT6BEbjClL9xmfL9l+Fpcr63DPgu2YemM3Y9sZy7NRJBOslddqseZwQ2btfGk1OoYH2NUXA0f/DNvyy+FCPPGvvfjLDYl49dZeLnkNImpbGNy4iT3/6u0eHWS2/o1cYGNLYWmNsT4GaKjluVxZh+p681lLJVX1ePa7/Rbv9f7aE5Lncvew5bs9eTh2oRz1OgH1Oi3WXf2C3/pHMXq3D8HRq33dmXMZO3Masg2mgc3nm0/jr0MTUV6rRea+cwBgDOYAYM4d1+K9X46je0wQHh6SiMFdIjDv14ZA7IesfAzsFIZgv8Y/7mP+uUly/2W7cvF9Vj6Kymvwa/pIRIdo8PsflwAAe85eRmqXSDy2ZA/25ZZgX24JhnWLwj2D4vG/A+dRenWW25s/HcHZS1VYvO2M8b6ni61Pn67V6qBRq1BWUw+9XkBYgC/OFFdKAhsDQ2BjYAic6nR6Sc2NXGADABdEQa0jSRJ7ljNwxDtXh+2+/D2HwQ0ROQWDGzexFdwE+qqQFBssu7hfU322+bTkubXhD3sFadTNKoDdnyffh8N2Fhu/vfooQgN80Lu9/ManL6w4CAC4dPoydpy+jHbBjdtdzLKj+FqcYVm09TR6iV7neGE5Qvx8JPVF+8+V4J5B8ZJhpwBftSSwAcyDNFOj/28T3ry9D2Z8l43S6nr83939EBHka7O/pursmCElztjVaps+o8reurGm0nM4ioicjDU3bqKwUY05eUhnPDqsi+y5od2i8PCQzk7px0vje6KLjenmcr7+yyCr558RDYu4yrojF5B/pdqutnI1NPb6evtZPJ950Ph8/sZTOFZYLmlzpKAMfxRVYNHWHOMxR4K/c1eq8cji3cbsz3P/2Y/cS03f5uDHLOtDhCcvlCNPtH2CYXbc7jOXMe3bLBSVW1/DCGha3Zgler2ATzb8gR2nLxmPMbYhImdjcNMCfPjn/nh2zDW4tkOo2bm0XjFY+uj1+PttvWWvndA3Dr89N0L2nGlA1C5Yg3sGxePulPgm97Fbu2Dc0C3S+Hxcn1jj4xA/NZ5LS8LLE3piXJ9YfHL/QLlbNNupogrkl9gX3DRFapdIq+eLymvxY3a+5FhWbonZ0JazvLbycJOvKbSxwOJNH2zGS98fMj43zK66e8F2rNx/Hq/+IP+a4uEuPyfU3Kw6WID3fjmOPy/cYTzG4IaInI3BTQtwU68Y+KqVUCnNszvBfj7GxyF+5qOISoUCXdoFoW9HaWAUH+GPaaO7S46tmT4Mof4+uCelI8IDfHBL3ziLfXp2zDXY9dJoAECHMH+E+KvRt2OY8XynyMbsT9nV4uVHh3XB/AeTMf7aWKhl3ktznS6uxCrRTLLYED/sfzUNPWKbN0186o3dkNolEs+OucZiG8OQ1OsWgkxLw2UAEKxR45/39GtWH53NdFjqRFG5bLsrouG6IJk/f0119hK3cCAi12Nw40b/vKcfHhuWiBVPDUHXdoG4rnMEnr+5BwJ8G780/v1EKh4dmmh8HuLfeO6bRwfjusQIrHhqiPFYTnFDjc4XD6VIVixe+fRQRAT6Gr90Hx7SGZFBDXUokUEa7HxxDD66bwA6RcrPmOnTIQTRwX7Y9eJorJo2FAqFAgMTwo3nA3xVFtc9USgU2D57NN7+Ux+8dmsvDOrceF1Kp3D5iwB8/9QQPH9zD+Pz12/rjeNv3Sw5tufsFQDAWxP7YF36cIQG+OCDe/tbvKc9esWF4NvHB2P6mO5W2ykUDRmrqCCN5PjDQzpbXYFZQMPnJTbhWsuBpcFjwxJttnGUaXBTW6+XLBppcLGicXjPGVO1VUrzv3JYc0NEzsbgxo3uGNgRL03ohYEJ4fj1uZH495RUPDlS+qV4XWIEXr6lF+5K7ogAX5WkDufajqH49xOpGJgQjrG9GwKZyYM7AwCig/2wcFIyMp8cgv2vpiE8sKEodfnjg/HqLb3MamJ81UooFApjIJUQESAJmnrGNQRF0SF+CAtouFeXdo3ZGn8fFUZc087ie20XrMED13fCIzckIrVrlPG44V5AQ1BxV3JH+KgUSL/pGgxICEePuMYsTHKncGjUKjw5sis2zBwpuf+gzhHGrFbPuBBMH90dGrUSmU82vofb+rXH8scHY9W0oRb72dCnxuxYRKDlYl4fpRLRIX749IGBkiG/EH8fRFq57sHBncx2e7enAHjENdE22zjqx+x8pGY0LiWQX1KNMf/cjA3HigAANfU6LN1xFgfOlRrbaJ2wp5lcTTKDGyJyNs6WaqHeu6sv3prYB34+8nUOc+8dgCMFpRgQ35gJUSgUSDbJjAT7+eAvQy1nAB4c3AmhAb5I7hSODmH+WPHUEJRW1aN9mPmqwu1DG4/V6/V4966+eOWHQ5h0NcCypE6SJWj8Ils9fRgA4J0/XWucZhwV2JgV6RXXONSTGBWIR4cm4outOVApFegeHSR5jRljumP66O6SVZqjgzUYfLWeZvEjgzDju2zc3q+9cUHE3u1DcH1ipKTY+62JffDUN41Trq9PjECdTo+s3BLcf30CgIYA9LrECESHaPC//QV4eEhn2YLcyamdcH1iJEb3jDYLbuzZTLRfvHSoMdhP7bSd41fsy5c9vnTHWYzqEY35G0/hw19PSs45svCfKaVMus80tvlw/UmoVQo8Pcr1RepE5J0Y3LRQCoXCYmADAP6+KiR3inDK69zWr3F7BvHQk9xrGpRU1SM62A+fTUqx+Rri7RfkvpzF66f06RCCf9x5LbpFB0kCFQB45sbuUCiAewclmJ1TKBTGYbLHh3fByuzzeHxEY9ZrZFI0sl65CTX1evy4/zziQv2xatows76M6xOL354bgchADb7POofx18ZBrVJixb5zuO+6BEnbp0Z2w1MjG76AtTKZmFFJ0RjVozH7cvj1sXj2u2zc3CcWK/fbXgBRXG8FNMyak9vewpl+PVaEBZtOYWfOJbNz+8+Vol6nNwvUmkKurkwcMxVX1OKDq6sx/+WGRLNtJoiI7MHghhzibyXwMnVPSjxOX6zEiKR2qKjRYmfOZYxMkh/SUigUuHdQguy50AAfvDTB9iJvL47vidnjephNv1coFPD3VWH7C6Nlv2QNbbq0a8gKPXxDY8bL0jR9A/Fwm8FAkyxaoEaNhZMbgsFrO4TieGG5ZO0ZW8ZdG2cMbv5+ay/8/epK0s425+djktlwYu+sPorXbpUvqraHuNBcrxeuBqmN0Y04y6fjcBUROYg1N9QkH903AGN6xlgd6jLlq1bi1Vt7YcQ17TD+2lj8PH0YFjyY7MJeWl9XyN9X5fTVdsX3y3wyFZv+NhKh/j4W23ePCcb22aMxObVTw/PoIHx8/wCzdoZanntT4iVFyKEBPtjz8hiz9rf1a4/dL5kfbypxHZLYV7+fadZ9xRk3Q92RpRiGtThE5ChmbqhJbu3XHreKhrGaSqFQGIuVvc2WWaNQWl2PPjLrFVky6+Ye6BIViJv7xCE21A9nL1XhvV+O4727+gIAfp4xDIfPl2FE93aSwEAQYDZr650/XWusC9r8t1HIPleCvh1C8ZfFu21uA2EqxEpg1hzimps6nR5+PiqLQYxOx+CGiBzD4IbISeIjAtDU5RGDNGrJ8NdTI7viruSOxs07o4P9EJ3kZ9e97ruu8dUTIgOQcHWa/28zR6LnK2sc2hfM2cSjgYYhKHEIIw50nDE7i4jaJg5LEbUgCoXCGNhY0y8+DAAw997+8FUpseDBZKtDcaYzr2wR175Yqk9yhDheMQY3omN6vbgtgxsicgwzN0StyM4XR+NieS26Xi16njigA8ZfG2ezhuif9/THnJ+P4WJ5LbafNp8JZcqw7cL00d1xU68Y3PLRVuO5owVlDg8tirMxhteQZmv0sm2JiJqCmRuiViQmxM+spsee4uj2Yf6Yd98AjBGtYm2NIaviq1ZCrZJmbsZ9uEV26rs9xGvlGLND4syNKNCxp+Ym73IVTlyQ3zqCiNouBjdEbUh8uPnijHKKru6q7qtSyu4T1tQCZQBYuf+8cfsMoHELCHEII87WiLM4lgx7dwPSPtiMSxWO7wJPRN6HwQ1RGxIf0biXWIcwf/xpQAfZdhuPXwTQkLmRW1X4oGhbBnscOFeCad9m4X+ixQvr5IalRNmaptTc5F6ualJ/iMi7seaGqA0RBzfr00fA31eF77Pkt2IArg5LyWx2WVhm/+KDAJAjk+mRKyg+I9o1nDU3ROQoBjdEbUiQRo1lj14PAbBrawMflRIqlXnmpk7rWM2N3D3EGZqpy7KMj7U2am6csdcVEXknBjdEbcyQblG2G13VkLkxD27qZQqKBUFAvU6we/VnucyNmM5G8CLenoFhDhGJseaGiAA0bvUg5qtSyNbcyGVupi7LwuCMX1FSVWc8lne5CgWl1bIBjHH7BQuhSZ1Oj0e+2oV/rj0ue57r4BCRJczcEBEA+SnlljI3daLMTWl1PWb+Zz/WHbkAAPjlcCHuHZSA6jodhr27AQAwpGuk+T1sZG7WH7mADccvYsPxi0hPSzI7b8dkKiJqo5i5ISIAMFvPBrBccyMelvpg3QljYGO4BgCKRdOzt50yXzhQbraUWEWt1mp/uWs4EVniUHDz9ddfY9WqVcbns2bNQlhYGIYMGYKzZ886rXNE5D4+MrOiaurl0yO1omGpUxcrJOcMGRlbe1nJ7S0lZqug2FZNDhG1XQ4FN++88w78/RsWA9u+fTs+/vhjvPvuu4iKisKzzz7r1A4SkXvIZW4qa7UI9DUfva7T6pF3uQr/3XsOJVX1knMl1Q3Py2vqza4zvQdgeViq3sa4k8DMDRFZ4FDNTV5eHrp16wYA+OGHH3DXXXfh8ccfxw033ICRI0c6s39E5CYqmcxN//gwqJQKfHz/AMk07XqdHmPnbkZVnXl25sfs87j/+gSU11gfVqqzsYVDUzI3jHOISMyhzE1QUBAuXWoYQ1+7di3GjBkDAPDz80N1dbXzekdEbuNjkrm5tV97dI4KBACE+vtIztVp9bKBDdCwseZfvtptO7ixsVaOre0XJFPBbUQ3F8tr7VoXJ6e4Ekt3nJWd6k5ErYdDwc1NN92ERx99FI8++ihOnDiBCRMmAAAOHz6Mzp07O7N/ROQmKqUCg7tEGJ8nxQRJzonZyrrsOXul2cFNvc568LLmUKHxsbX6m105lzHo7fV4/F97rL4eAIx6fyNe/uEQFv9+xmZbImq5HApuPvnkE6SmpuLixYvIzMxEZGTDNM+9e/fivvvus/s+GRkZGDRoEIKDgxEdHY2JEyfi+HH5NS3ENm3ahOTkZPj5+aFLly5YsGCBI2+DiAAkxQQDAO4Y0AFL/3q98Xhyp8ZAx3QLBntWKLZZc2NzWKrxvFzw8uqPhxvPW8ncLNp6GgCw/miR1dcT23Xmst1tiajlcajmJiwsDB9//LHZ8ddff71J99m0aROefvppDBo0CFqtFi+99BLS0tJw5MgRBAYGyl6Tk5OD8ePH47HHHsPSpUvx+++/46mnnkK7du1w5513OvJ2iNq0fz+RigP5JbihaxSUSgU2zhyJ08UVSBWtTWOauSmtth64ALanctsKkM5daRzi1gmC1b+srI1gye2NZQtreIhaN4eCmzVr1iAoKAhDhw4F0JDJ+fzzz9GrVy988sknCA8Pt/s+Yl999RWio6Oxd+9eDB8+XPaaBQsWICEhAXPnzgUA9OzZE3v27MH7778vG9zU1taitrZxvY2ysjK7+kbUVoQG+GBY93bG552jAo21NgamC/ldKKuFLX8UVVg9f/ZSJWavOGDx/EnR9abBi2n9jLXVipUyixDaxuiGqDVzaFjqb3/7mzFIOHjwIJ577jmMHz8ep0+fRnp6usOdKS0tBQBERERYbLN9+3akpaVJjo0dOxZ79uxBfb35vyYzMjIQGhpq/ImPj3e4f0RtlT2Zm3fv6it5/usx68NAG45fxLe78ux6/eMXyvGv7WeMQ1WmQ1qGYan1Ry5g71npkJLMDHebmLkhat0cytzk5OSgV69eAIDMzEzccssteOedd7Bv3z6MHz/eoY4IgoD09HQMHToUffr0sdiusLAQMTExkmMxMTHQarUoLi5GXFyc5Nzs2bMlAVdZWRkDHKImklsDx5TcjCoAiArSSFYrdsTET34H0FBk/Jehiag1WVxQrxeQd7kKjy5pKBo+M2eC8ZwjmRvGNkStm0OZG19fX1RVVQEA1q9fb8ykREREODzsM3XqVBw4cADffvutzbYKk438DDMpTI8DgEajQUhIiOSHiJpGbn8pU6bBDQBEBPoiKsh8Q05HbT/dsARFrVY6DV2nF5BfIr8MhUrm7wWxilotamyspkxErYtDwc3QoUORnp6ON998E7t27TJOBT9x4gQ6duzY5Ps988wzWLlyJTZs2GDz+tjYWBQWFkqOFRUVQa1WG2dtEZFzyS3wZyoswDy4iQ/3l92Q01GGIKTWpBhZLwiwFMKYDqkBQGlVPf614yzyLlehz2u/YMw/N0nOt4TVj6vqtPjt2AUGXkQOcOhvnY8//hhqtRr//e9/MX/+fHTo0AEA8PPPP+Pmm2+2+z6CIGDq1KlYsWIFfvvtNyQmJtq8JjU1FevWrZMcW7t2LVJSUuDjY/6XKxE1nz2ZmzB/8wxNx4gA40aazmAYjjLP3Egzt+KCY7ngZsZ3WXjlh0PGXcvPXamWLNzn+dAGmPZtNv6yeA/e+OmIp7tC1Oo49LdOQkICfvrpJ+zfvx9//etfjcc/+OADzJs3z+77PP3001i6dCmWLVuG4OBgFBYWorCwULLK8ezZszF58mTj8ylTpuDs2bNIT0/H0aNH8eWXX2LRokWYOXOmI2+FiOwgHtl5eUJP2fNBfuYlfPHhAfB1ZnBzNagx3dDTdJ0brY3gZsPxi2bHBr/zq/GxrcWMdXoBf128G3N+Pmazz45af7Rhp/VlO3Nd9hpE3srhv3V0Oh0yMzPx1ltv4e2338aKFSug0zUtfTp//nyUlpZi5MiRiIuLM/589913xjYFBQXIzW38nzsxMRGrV6/Gxo0b0b9/f7z55puYN28e17ghciFx7HDvIPOCfB+lEv4+KrPj8RH+8HHqsJQhc2NeUCwOwMSL/ilt1NwYXKqss7sfu89cxq/HirBg0ym7ryEi93FottQff/yB8ePHIz8/H0lJSRAEASdOnEB8fDxWrVqFrl272nUfe8a1Fy9ebHZsxIgR2LdvX1O7TURO4KNS4m9jk/DeL42riQsQZDMkDZkbR9aZkVejNdTcSP8hZVpz07AvVUOwJe7X2UuV6BQpv0CoWFNqbvR6wWxG1u4zl3HuShX+NKDpNYim5D5XIrLOoX9STZs2DV27dkVeXh727duHrKws5ObmIjExEdOmTXN2H4nIwzqG+2NY9yiM7R0DPx8Vnh7VDcffaqyvszSM0y5Yg52nzbcyCPA1z/LYw1JBcWFZDT7fctr4XGdhWGrEexuRe6nKodcW8xNlqaplCn7vXrAdz363H4fyS83OWZvZJYehDVHTOZS52bRpE3bs2CFZbC8yMhJz5szBDTfc4LTOEVHLoFAo8C/RvlMAoFE3fsEbVgh+cXwPHMovQ0SgLy5X1qFHbDDKZbZh6BDmL1mB2F6GYSnTrRveXSPdk65eJ+DzzafRp0Oo2bDUjpxLTX5dU+Id1CvrtAjUyP9Ver6kGn06hEqOPfPtPqw+WIgFDybj5j6xNl/L3mE1ImrkUHCj0WhQXl5udryiogK+vs5b04KIWgfDKM7jw82HpO9Nicd3e6QrEcsVHxtMTu2E/Xkl2H9OPusBmGduTO09ewVvrz4KAHhwcILk3Kz/Wt7ywcDwfs5eqkT7MH+zGV/iUauqWh0QLH8fuZliqw82LGXx2eZT9gU3zitZImozHPrf5pZbbsHjjz+OnTt3QhAECIKAHTt2YMqUKbjtttuc3UciasVeu62X2bFgP8vLNky4Ng5/GtBB9ly9To9jhWWY9m2W1dcsKq8xPl66o+mzjfSCgF8OF2LEexvx16/3mJ0XD3tV1kkzU+LdzK1Ng7c3H8PMDVHTORTczJs3D127dkVqair8/Pzg5+eHIUOGoFu3bsYNLYmIACDAV43OkQGSY4FWam7UKgUS2wXJntPqBdw9f7vN1yytsr1ruTXbTl3CpxsbZkJtPmE+bVw89byqTlpzI973Sq4exx7VontW1elwpQkzuYjIweAmLCwMP/74I06cOIH//ve/+M9//oMTJ07g+++/R1hYmJO7SEQtXWKU9RlI/31yiOS52kpGQ6VUoouF++n0gmwNj6krzQxuAGB/XonFc+JFAh/5ajf+teOs8bl436vHluzBh+tPyt5DvOhgdZ0OhaWN2aYXvz8oaTvzP/vt7jcRNaHmxtZu3xs3bjQ+/uc//+lwh4io9UnpFG71fFSQBgkRAci93DBTaUzPaPxv/3nZtmqlAu3D/JvVn5Jq12U6Fmw6hQPnSozPK2q1eOWHQ5g0uBMA83qgD9afwPQx3a3e87Ele7D1j2JkPpmK5E4R+D4rX3J+6x/Fzuk8URthd3CTlWV9jNtAbvNKIvJO793VF5n7zuHF8earFpt6YVwPPPXNPlyXGIHb+rVHkEYtW8+iUiqavbZLiRMyN3LyLlfZXJXYdA0eS8Tv0BC8vPT9IayZMdysrc7WkslEJGF3cLNhwwZX9oOIWqG7U+Jxd4r5isVyxl8bhy2zRiE80BcKhQKje8bItrNnHytbrlS5JnNjz/o0tmZyWWNanGygZXBD1CScZEhEbhMfEYAg0ZowcnGMIWvz8/RhDr9OcwuKLblsR2Fvbb19wY1ckpszo4icg8ENEXmMXGGx+urCLj3jQhy+72UnZ25+OtBQH3SxvNZmW3uHpeSoGNwQOQWDGyLyGB+Z1I3Kjr2ofntuBPx8LP/15eyam6nLsnD4fCkulNXYbGvvsFR1vc5sDyvTPaqIyDEMbojIY+QKh+UCHlP+vioo3Lzr0umLlWZr2siRy9zU6/TIyr0iWeDvUH4ZHluy12QHc+f0laitc2j7BSIiZ5BbwbddsMbmdf4+KtmaFbGoIF8UVzhveKpep5cs0GeJXM3N26uOYvG2M3h6lHR7ivVHL6BedE/W3BA5BzM3ROQxapMhqISIANnlJDqE+SNZtJZOQ+bGukGdI2y0aJryGi0u2xEsyQ1LLd52BgDwyYZTZufEAVNzp8ATUQNmbojIY9SiXSHfuL03bu4tv5GkWqWQ7MTtq1LaXFNrUOcIJMUGY66FFYKb6rWVh+1qZ7pjuS31WmZuiJyNmRsi8hhx5mZyamdEh/jJtlMpFZIhLIXCdsVNdIgGt/Zr74xu2uWez7Zj8e85svtJRQX5Wr3OQC+4dz2bgtJqZKw+inNXqtz6ukSuxuCGiDzG3mEYtUlwA8ivEyMW4ucDH6X7/orblXMZf//fEZRWm8/UahcsH7QBwKmLlcbH7l6J+PEle/HZ5tOYtGiXW1+XyNUY3BCRx9gbfKiUSsmwFGB7q5cQfx+7ppU7m9w+UOEBPnZdqxcESYGxqx3MLwUA5BRX2mhJ1LowuCEijzEtKDZlmF306i29zBb8s525Uds1rdzZ8i6bD/FodfZlZLR6AVW1lqebi3cj/2DdCdzx6e+otjA9fcOxIhw+X2rX6xJ5GwY3ROQxY67uLxUVJD/9+29je+DoGzcjtWskAn1VTbp3iL+PZNjrwcEJeGpkVytXOEehzEJ/tXZmY/R6ARUW9pe6UlmHwRm/4pUfDgEAPvz1JPbllmDl/nyzticulOORxbsxYd7WJvScyHswuCEij3l6VDe8f3c//O+ZGyy28b8a1Dx70zVoH+qHmWnXAGgoQLYm2E8tyfZ0axckW+zrbHI1wbV2vq5OEFBVKx/cLNuVi6LyWvxrx1msO3LBeFyuTOd4YbnZsaKyGuw8fcmufhC1dgxuiMhjfNVK3JXcEXGh/jbbxoX6Y9vs0Zh6Y3cAwLQbuyGtl/zO4gCgUaskdToCgDsHdmx2nx1hz+J/AKDTCai0MMwkHoZ7bMke42O5omyt3vz1Bmf8insX7sC2U+Y1QUTehsENEbVKapUSw65pZ7WN+ItfLwB9OoRizQzHdxt3lKW6GFPnS2uwaGuO7DlLk99n/fcAKkyyPfUyNT6GDM+WkwxuyPsxuCGiVstSwfDCSclXzzf+FWfYpFLtgSLjihr5oSY5/9t/Xva4TiYbY7D2cCH0esE4lVw8pdx0c043L6VD5BEMboio1ZLbm+qB6xOQdnWlY6Ukc2MIbtz/156lIuGmqLMy40qrE3Dngm0Y9+Fm6PSCZINOrd40uGl4bm82iag14vYLRNRqyU0lD7Awq8rwHW9r+rkrOJIt8VUrJVs5aK3U7ZRU1yErtwQAcL6kWhLQaHUCfEQfiYCGaeKPLN7d9E4RtRLM3BBRq+Urk7lRWhh2MmRu5LI9jjJdWNCZQvyk//Y0zcCIiXc/1+oFybo69SbDWXq9gOczDzipl0QtE4MbImq1TBf2AwCVhdX9DNkTZ9bcdG0X5LR7mQrSSIMbaysXF4nW1qms1ZplbsQE2P8ZXCirwTaZFZeJWjoGN0TUasllTiztV2VY3de05qZ9qOV9n2xx5UaXAb7S4Oar389YbFtUXmt8nHe5CsUVjc9Nh7P0gmAxu2Xq3s+24/4vdkrW1SFqDVhzQ0StltwQk8XgxkLNzZK/Xo8/iiqQue9ck7/EXbnPpX8TVmS+KApunvxmn+RcvWgWFdCQwbJ3w9Izlxq2kvhyaw5usrKmEFFLw8wNEbVacsMrFoelcDVzYxLchPr74OY+sRYLka0xnWbdLdr6MFWXqEC77+3vY39/xJkaU1qd3mxIy97gxuDAuZImtSfyNAY3RNRq+ajN/wrT+Mj/tWZIXpjuRG6IhZr6hQ+Yz4JKTgjHnpfH4N6UeNn2SbHBdt+7KZmbK1X1Fs9p9QJqRbOuFm87g9MXm7YLuDu2rSByJgY3RNRqmQYqAODvKz/absiyKJUKLHooBQAwuEsEIgN9Ld7LFrmam6ggDVQWZlGFX30tezQlc2ONVidYLUa2B9f9o9aGNTdE1Gr5qM2DCEu7h2tEWZ7RPWOQkzEeAKC4mrpxZP0bSzU3lpJA4QE+dt/bkWEyOfUyw1JNxVWNqbXxaOZm8+bNuPXWW9G+fXsoFAr88MMPVttv3LgRCoXC7OfYsWPu6TARtShyqw2bBgUvT+iJ5E7heGhIZ8lxw98fjfdyYFjKQk7DUt1PeID9mRs/Z2Vu9ALqtc6NTgRBwNPL9uGFZqyXo9cLyPj5KNYcKnRiz4gaeDRzU1lZiX79+uGRRx7BnXfeafd1x48fR0hIiPF5u3bWN88jIu8kt4if6bDUo8O64NFhXWzeS27NHFtMt3syBDsKC8FN9xj7a26clbk5d6XKbM2c5soprsSqAwUAgLcm9nHos1tzuBCfbToNADgzZ4JT+0fk0eBm3LhxGDduXJOvi46ORlhYmPM7REStitxQkqVhKVscWbnYdLaUgaXi5OHdo5DSKRx7zl6xeW9n1dxMXZaFF8f3cMq9gIZp5++sPmp8rhMEh75ICktrbDciclCrLCgeMGAA4uLiMHr0aGzYsMFq29raWpSVlUl+iMg7yAUkTZllJL1X04eldBaCG0sjXAqFAs/edI1d97Y068teof6N9T2fbjzVrHuJPfXNXqw/WmR8bmWzcqtcuQAiUasKbuLi4rBw4UJkZmZixYoVSEpKwujRo7F582aL12RkZCA0NNT4Ex8vP0WTiFofuYDEdGVfezkyFdxiQbGVe9n7Khp1Y5BmaWq5JX07hiI+wr/Jr2mP3WekWSdLAV5TVDlh13QisVYV3CQlJeGxxx7DwIEDkZqaik8//RQTJkzA+++/b/Ga2bNno7S01PiTl5fnxh4TkSvJZW7cOywl/1xpoeYGACpq7fsi9xXN7uoY7m+lpTkflVJSbG2pBkjsUH4pZv5nP4orapsUbLz10xEs25nbpP4B0s/uxvc3Nfl6ImtaVXAjZ/DgwTh58qTF8xqNBiEhIZIfIvIOctkWR4elNDILAlry7JiGoaW3JvaR75eVYKK8xr7AQdyfjhFNDW4UkqyWPUmpWz7aiv/uPYeUt9aj79/XIjuvxK7XWr47Dy9+f7BJ/QOkM80Ky1h/Q87V6oObrKwsxMXFebobROQBGrUSXdpJtzRwdFjqz9cl4JqYIEwZ0dVm2+ljuuPg39Nwc59Y2fPWgok40Uadu14cbbGdOJMUF9r0zI048CuuqGvS9Vq9gImf/N6ka+5esA3zfrX8D01TrtyXi8ijs6UqKirwxx9/GJ/n5OQgOzsbERERSEhIwOzZs5Gfn48lS5YAAObOnYvOnTujd+/eqKurw9KlS5GZmYnMzExPvQUi8iCFQoF1z46AXhAw79eTCPHzcah2BgCCNGqsfXYEAGDBJtsFuMF+lhfkk6u5+fSBgQCA1K6ReOP23ugZF2L1HuKF9wKbGLD5qpSotHP4y1l2n7mC3WeuYNro7na1b0pBcb1OD6VC4fDvtrUqq6lHdZ0OMSGO71zfVnk0uNmzZw9GjRplfJ6eng4AeOihh7B48WIUFBQgN7dxLLeurg4zZ85Efn4+/P390bt3b6xatQrjx493e9+JqGVQKRVQQYHn0pI83RXjQIv4e/vrv1yH6xMjjIvyKRQKTE7tfLWd/Bd8iJ8anUWbbPo1ceaUj0qJ6vrmrUps6t+783DPIOdNyLA3tqnT6nHDP35DVJAGP08f5rTXbw36/n0tAGDfKzchoglbd5CHg5uRI0da/J8bABYvXix5PmvWLMyaNcvFvSIiap7S6saNLId2i7KYcZAr9N0wcyRiQ/xw7kqV8Zh45tSgzuFmM5Y+n5yCgtJqvPrjYQANG4pa+7vVEbMyD6BTZIBT72mPExfKcbG8FhfLLe987u2OFZRhSLcoT3ejVWn1NTdERM52e//2zbr+SlVjjYu9QylPDO+CDTNHIjEqEP6+KnSLDsK9KfF4amRXyR5a4kDH4KZeMQgTbe3go1Lg7T/JFzuLJUYF2mwjdvZSlc02h/JLJcGdJfYGX+LhK30bLdSxtrQAyWNwQ0RkYu69/XHszZsR4udYcvtyZdMKeAGgW3SQJNhQKBT4x119MevmHpJtJiwtNugj+gL0VSmR3CkCr9zSy+prPpd2De4Y2KHJfbXmlo+2YuwHltceM7A3ThG3c8aaOq2FOPhra7VGzsBdwYmITCgUCvj5qBxa+wYASqpsZy4MPpuUjO2nLuFPAywHGT6iaeGW1tBRSwKghsem+1OF+vtIsioqhQKRTajl+M9e+9YJs2dqt/kaQYLsMJ04c6PTC3DSrhQtXr2u8X1bWzeJ5DFzQ0RkgT3/Yv7wz/2NjyODGgKFlM7hAKRbIFgytncs/n5bb6ubT4ozN5YW5BPvs2UIbnQm6ZG+HUMlz1VKRZN2Hzet9WkO09lS4i9zMXEz0/fjzbSifS0Mfw6zcq/gyaV7kXfZ9vBgW8fMDRGRBfZkbm7v3wFqpRIr9+dj6qhuAIDn0pLQMTwAY3vHOL0flv4RrxYFYoYaHdNgwDSQUSkVTVq8sLmW7cyFRq3EnckdYRqm1Ov0klWZl+/KxZWqegy6GigCbWtYShzsGRaF/NOn2wAA565U43/PDPVIv1oLBjdERBbYW+swoW8cJvRtXEw0SKPGX4cmuqQflnok3m7BkOnR2hXcuHacx7DeTq1Wb1zJ+JZ+cWYFxVqTzM0LKxravntXX+OxtlRQrBWtc2Qa0J4prnRzb1ofDksREVmgboGFnKb1F8FXi57FhcaGLSh0Jlt2+5pkotRKZbN3H7dGpxcwft4WDH93g6TIuqZOb1ZzU6eTX5enVFS/ZBqsOVt2XgkeX7IHOU4KHipqtXhn9VHst3MrCzFr77XthHiOY3BDRGSBOGMy4pp2AIA+HTy7P504tukeHYRfZgwHIC0oNqxonBgVJLnWdKaVUtm0PbWaavupSzh7qQqXKuuQsfqo8XhVvVaytxQgrTERZ3XEQY+rMzcTP/kda49cwJNL9zrlfv9cewILN5/G7U3cygKQBjdyxddkHYMbIiIL7r8+AQCQ0ikcc+/tj5cn9MSXDw/yaJ/EmZs7kzuifVjDvlPiLJMhczOmZzQGd4kwHpcbZmtKQXFTrcg6Z3z867Ei4+PqOp3ZF/aS7WeNj8UxTJ22MbhxV83NmUvOydwcLShz+FrxsJRp8TVDG9sY3BARWfBQamf8+4lUfP2X6xAe6ItHh3VBdLCH9/kRxSfi7zzxbClD5kahUODJkd2Mx00LpAXBdZkbrU6PrSeLZc/9UVRhFqjM39i4n5feQuZmytJ9KCitdnJPzTlr6rVpdqopxAXFZsENoxubGNwQEVmgVCpwXWIEAjUtZ+6F+GtX/OUpLigO0DRmY8TBi2nmRi8ILisortXqUVOvkz33+L/2Wh1ikgQ3oszN/rwSzPrvgSb3Je9yFb7dlSu5lzWOhDYHzpVg9oqDuFTRuE1Ec0bRxMN0pvdpTtDkSi1puIzBDRFRKyL+/pBkbkSBS4BoqEk8vdq0QFrvwszNl1tzUFZjeWfyapnA56vfcwBI31e9SaGxtS0gjhaU4Z4F27Er57Lk+Mj3N2L2ioP4fMtpe7pucS0ha277+Hd8uyvXOCMMQLPGj8Szx0yDBsPTkqo6yfCVJ60+WIBBb6/HztOXPN0VAAxuiIhaFUsL2UmGpUSZJvEMKZVSgU1/G2l8rhcEaFxUc/N/605YPV9abR74vP6/I8Z+GZgGN9bijslf7sKuM5dxz2fbJccNn9kOO794mzMqtT+v1Pi4ecNSVjI3AnD2UiX6v7EOd87f5vBrONNT3+xDcUUdHlm829NdAcDghoioVRHXqoj/RS+upxFvu6Axydx0imzcv0oQBLcu4idWUmV5/y3xl3mtyVCSadzx+x/FWLDpFARBcNrO4c2puKmqawzamjcsJd12QkyAgJXZ5wEA+8+VoiUxDUY9peUMJBMRkU16C1OEVTKzpQDpLuIqpTSQaRfkBz8XrnNjTZmVncPFmZsV+/Il50yHjB74YicA4JoY6bR3OfYON1nbhbu8ph7LduZiQt84dAwPMDsvHm5rTg2KOEiQG5ZqqTuFt5SyG2ZuiIhaEUvTocWHxUNRkpqbq0NXXz6cgtdv641rO4a6fIViSyzV4wiCAMHKP/5ziiux92xDTU1ReeMGneeu2J5FZW84YK3d26uOIuPnY7jtY/m1a6SznKTnLpbX4nyJfbO9xNka84LilruZZguJbZi5ISJqTaKCNMbH4i+SiEBfdIlqGHIKD2jc6dtXZtjpxh6Ne155aliq1ELmpqJWa3M9mzvnb8cPT9+AiaLF8ez5srfUxHRHcmv32nm1WFm84rIlpu9i0NvrAQAH/56GYD/rm6qKC4rnrDmK649HmvTX5st7REuZMcXMDRFRK/D55BSM6RmNF8b1MB4zHZZa++xwrEsfIRmyEAc3cl884YG+ZsfcwVLNTWl1Pe5buMPm9ct2npU8tyu4kTm27VQx+r+xDiv3n29sJ9Owuk6HTzb8gXw7My8AJL8g8aym8yU1cq0lxMNSh/LLsGhrTuNtAbTQUakWg5kbIqJW4KZeMbipl3SXcdPZOGqZXczFmRm5Wk97dj53BUvFtmXVWhy/UG7zetN+i5+abjNhsOH4Rew4fQmDuzRmQR79eg+q6nSY9m2WqKX59R/9dhKfihYatIf4PYqHq+wJTKzuLSVwWMoWZm6IiFope0YAxGvbmG6kabB62jDcndzRWd1qluIK+2Y8fbMzV/Jc/GVvrY7ozyZZIbnPsLii1izLdcDGrCS5TVbFwad4AUF7CpttzTpqscFNC4luGNwQEbVS9nyPiL9ILdWy9Gofgvfu7uekXjXPo0v2OHSdNLix/6vN0s7vy3ZJgye5fbnExK9pWJlZHEta2vXc1M7Tl7B8V64k0yOHw1LWMbghImplrk9s2AzzjgEdmnSdI0uQdAz3b/pFJh4cnGB3W3u3SDAlDj78fFT4146z+CErH59tMh9KMkynFwRBsvih2AfrThofbz5xUbaAWJzdEQdXo/9vU8N5UVtxcGNpIcYD50pw78IdeGHFQezPK5FtYyAOWt1RxFtTr8OZYudsKOoOrLkhImpllj02GBU1WoQGWJ9xY8p0A0ZLEiIC0CM2GDf1isEnG/6w+/7RwRoUySykl9YrFkt35Mpc4TziUZpLlbV45YdDFtuWVtcjv6QaD3+1G1eq5GdtafV6lFbX4901x8yGwAyq63UIMG5S2ng8v6QaB8+VSoKOelHQZjrkdOR8GX47dgFxoY2B5KVK68Nz4sxNvU6Ar7ppqZwrlXV4PvMA7k6JN6vlkjPxk99xrLAc/34iFdclRths72kMboiIWhmVUtHkwAawnDEw5eejxMLJKQCAzzbbtx8TAPz5ugSUVddj8bYzkuOWsiPOJF7nxlelRE295QzQpcpafPjrSav1PTqdgPsW7sCRgjKLbcprtMbgxnRRvVs/3ip5bi1zM37eFgBA58jGRQGr6+Q3HTUQv55Wr4evjYEYnV5AQWm1ceHBD389ibVHLmDtkQs4M2eC1WsB4FhhQ5H3j9n5NoOb7acuIbVrpNU2rsZhKSKiNsLe4EbMUk2KpbbirR8MfN0wI+tQfmPBr60ZYJcq6hBiY52Zer3eamADALWiAMrWp3RFNKyltVDYfUa0KWhFreVNRwHpMJit+hwAmPFdNob+YwN+OtAw5d3a9hf2vq4l931ueyq/qzG4ISJqI+wdllKIvqqbknVRKRWyU7zFU9RdVQh7QjR93HQ/KlNlNVq0C9ZYbWNPICjOxtj60r9XNEtLa0cwYmuRQPHnqNMLOHupEhPmbcGP2fmy7f93dR2fTzc01CAFaBwbuFEpFajVWs8qtQQMboiI2oh4mb2QbFEr7f+aUCkVqK4zzziI152xtTKvo8S7jIv3d5JTp9UbZzRZYm2dmcY2DcFNUVkNLtmxYnFT7m2pFshAHKdqdXo8+vUeHD5fhunLs61fd/W/gTIZNkuWi2aOHcovRdLLa/DeL8fsvt4TGNwQEXm5ZY9ej0eHJmLykE52tRevz2JpQTw5Wp0eVTK1IuJhIksZk4hAX1zX2fFC1bKaxmDAVtalVquT7N4tx54kV722odHdn2233VjEvuDGerAkntZfrxdwsqjCrtc2FDkbaoXExyx5YcVB4+M9Z68AAD7Z0LQFDd2NwQ0RkZcb0i0KL9/Sy6FNMu+/3v5p3OW1WvSLDzM7Lg5uYkP8zM6PuKYd9rw0Bp0im55ZMmjKFPJarXwQ1uTXvDosdVZUK2MPrU6PC2U1WLL9jMXaGlvBlXh3ePHWDmEmheZLtp/B91nnGq8zBjeNfxYqr34W6d9l49Gv9zhlarkj9V3OxNlSREQkIa65mdi/A347dtFYs2FNeY0Wfx4Uj6o6Ld5Z3ThsIS5Kjg4xz9woFA2zf/x83LNDeZ1Wb1dAEh7gY3V4SOvIwkFoyNzc//kOnLpYiT/szLiYsrS1Q6AoI5NfUo1Xfzwse5044CyrroePSoEVWQ31Oi//cAg940Lw4GD7Mn1y6nV6qJSe2XEeYOaGiIisUCgU6C+TjZFTVl0PtUqJx4d3RY/YYONxW5kbAz8f93wlbT5xEQdFs6sssZV8uFhRa3NWkxytTsCpiw0L4v16tKjJ1wPSzEilqA/iz7BSpm9/FFVgxvIsya7s5TVaSRH2Nztz8fIPh5qVwWHmhoiIWhTTDTntLbspr5H/ohfX7cRYCW5szXJyll+P2RdQ2BrqmrosC8EOzDoSTwUP9nPsa/iNn44YH4tnVvmKhh4tTeP/IVuahauo1coWWP90oADtwxxbodqeGWGuxMwNEREBAN64vTeC/dR432SfKbndxgHgxFvjJM/HWFjpVnx9RKCv2XnDV7ClIaBecSGWuuxS9uwHVe5A5kY8o8nR4EZMHNyIa3HsTZ5odXrJmj0Gz3ybhTvnb3OoT/UW1vJxF2ZuiIgIADA5tTMevL6T2Wq74k0h5903ACOT2kEQAF+TDSrvv06++FicQejSLtDs/LUdQgFYXljO1swhV3HH0Ip41pKjxMGNOKiwtFigKa1ecPraNczcEBFRi2Ea2ACARlToq1IoEOLng1B/8/VqLO2cHeCrQlqvGIxKaifJwoy4ph3+NjYJT43qBgAYkBAue73cflWuMGVEV7e8jpgjNTum3lt73PhYHFTYG2DU6fRWt6twhOn+We7m0eBm8+bNuPXWW9G+fXsoFAr88MMPNq/ZtGkTkpOT4efnhy5dumDBggWu7ygRURvmJ8rQWApgTIl3rVYoFFg4OQVfPXIdFAoF1j07HI/c0Bkf3z8AT4/qZpwl9aSF4MJVGZSoIOnMrRu6uX8/JFsrEdujzsKmnPYGGA0Fzo7N2rJ4Tw8XFHs0uKmsrES/fv3w8ccf29U+JycH48ePx7Bhw5CVlYUXX3wR06ZNQ2Zmpot7SkTUdomnaDdlrylLuscE47Vbe5utVuzvq7I5YyraxrYJpkyHzsSu7dCYRboruSPCA8zrgVztkpXNOx1RrxMgCAL0esHuAGPVgfM2Vza2pLC0Bhmrj5odd3SavLN4tOZm3LhxGDdunO2GVy1YsAAJCQmYO3cuAKBnz57Ys2cP3n//fdx5550u6iURUdsmDm5UFqZOmcY8joZApjt6T07thD7tQ/HqykOY/0AyRia1Q+Ls1bLXvntXX8z67wHj886RAbhUUWdx1pN41CalU7js3lu+KqVdhcWOKrMww8xRWr0ek7/chYLSGrx2ay+7rjGdPdUUT/xrD/afM59Wb89mnq7UqgqKt2/fjrS0NMmxsWPHYtGiRaivr4ePj/kYcG1tLWprGyPjsjLru7wSEZGUOJtiKXNj73CVLaaZltdu7Q2VUoE7kzvafI3OkdJi5YpaHaztZzmuTyw2n7gIoKHWSC7R4at2bXDjbHVaPbacLAYAHMp37fedXi/IBjaA/cXMrtKqCooLCwsREyOdahgTEwOtVovi4mLZazIyMhAaGmr8iY+Pd0dXiYi8hp9JQbEchY1dse3lazLt3BDQ2BM8hZtsPVBZq7XYrykjukraqxQK+cyNlWGtlki8rYS98WakzPR8ezy4aKfFc2265sYRpn9QDSsoWvoDPHv2bJSWlhp/8vLyXN5HIiJv4idaGM5SkOGkxA3eueNah681reFRKiBZiVcs0FcFlWjHc6VSukaMgWmwZa+wAB/8dWii5NikZmxn4AidnSsM29pF3ZJtpy5ZPMep4E0QGxuLwsJCybGioiKo1WpERspXuWs0GoSEhEh+iIjIfhof27OlTDM6jiZyRiZFO3YhzPv2+eQUJEaZr6sDAAEaNcRxi1KhMAuOAMczN50iA/HKLdKaF3UTdlh3BnundztjE1FTni4oblXBTWpqKtatWyc5tnbtWqSkpMjW2xARUfOJMzeW/j2udNKwFNC0+p2RSe2Mj8X1QPER/hjSLQqfPjBQdouEAJPMjUqpQFJsMJ4dcw1mpl1jPK5xMLgxdEVcryTeY+vVW2wX+/o0Mxia9+vJZl3fHPVteViqoqIC2dnZyM7OBtAw1Ts7Oxu5ubkAGoaUJk+ebGw/ZcoUnD17Funp6Th69Ci+/PJLLFq0CDNnzvRE94mI2gRx5sbS2ilOjG3sdnv/9pKARrwAoSHY6hkXglk3J5ldq1IqpNdebT99THdMGtxZ0s4RhkyWv4/8kF6Ar+0ds921S7ortOnMzZ49ezBgwAAMGDAAAJCeno4BAwbg1VdfBQAUFBQYAx0ASExMxOrVq7Fx40b0798fb775JubNm8dp4ERELiTOXlia4mu6svHzN/cAADyU2vQ6k6aEE+IhFblgBZDfY6m0ql7SRvxYPBTlaKG04fMQByji+h17Ahf/VhzctOmp4CNHjrS6pfrixYvNjo0YMQL79u1zYa+IiEhM/AWvszDF13RYavg17bD/tTSEOGFjSGuGdW+HbacuITzAR5IZEcdaciscJ0YFSmpgxNeKgxtHC6UN14kDFH9RtsbWYoWmfWptOBWciIhajcSoINnjct/Dof4+DmU+7L1EAeCvQxPxz3v64efpwy2uwSOe4r1q2lDMueNajO4ZLQkexMXF4uMKBbD5b6PM7pncKRxfPTzIYt8M9zDslB4Z6CsZitKobWdlKpy8wJ87eXq2VKtaxI+IiDxjffpwFJXXWpx95Kx1bgBAAQUsly43EtCQZbljYEezc+JhEfEAQe/2oejdvmEXcpXJ/leW+pIQGWB2/MkRXWWPGxgyWek3XYOO4f4YlRSN7acbp05r7MjcBGrUKHfCxpqe4OmNMxncEBGRTd2ig9EtOtji+ZY2glKrbazFkVucD5BmaCwtTmjpfamUCqszxAzn/HxUmJzaGQCQnVdiPG8tc7PgwYG4VFmHjccvovBIjcV2APDEiC4I9feBVifgs02nUOmCad2OcNVmp/ZicENERM1mKThwiBNuJd5PytLXrKWaG2lfLB23PlVb7nbi+htLU8zvuy4eN/eJAwCsP3LB4v0NogI1eGx4FwDA51tO22zvSrf1a486rR5rDhe27angRETkHZw5LOUM4uDGx8Iqw9JhKfn7WMtIdQwPwMNDOqNHrHlGSy5YEtfc+KiUsm3Edbj2zDgSz1Kz5zcwfXR3O1o5JsBXZQwY2/RUcCIi8g5KJ36b2BsmWdtdQLzZ5Z8HxaNffBieu+kaSRv7hqUs9Obqa//9tt6Ymda4jk7Q1QUDR/UwX2nZTxTcqFUK2eBGvGWCPRt2ipNH9gSYAxLCbLaxJUhmUUSgYTaYIZBkQTEREbV6Th2WslOElQ0fxVmPQI0aPz59g1kbtdL2thKGo+/f3Q+/HbuA1QcbtgASRINdwaLp7uvTRyAr9wrSesea3Us8LOWjVMp+Zr3iGrcIsqcoV6m0nX2y1AdH9W4fgp05l82OB/iqjDO86j08FZzBDRERNZszt1+wdauP7huAFfvOYcaY5g2xiLNNFmdLXT18V3JH3NovDqsPrjFrc11iBB4cnICu7YIQG+qHcdfGyd5LEtyopZmbjDuuRVl1PSaJFj20K7hp4ufub8fKyJYEadS4Y2AH5F6ukj0f5u+Ly5UNG5Uyc0NERK2Wn48SNfV6XN9FfvNiV7i1X3vc2q99s+9jX+bG9tCVQqHAWxNt72YuDizUSqWknqdvx8Yp6gb1WtsBgsqOmps/D4rH8t15AJq3pcO+V26Cr1qJx5fskT0f6u9jLLJmzQ0REbVav8wYjhfG9cBLE3o67Z4KZ0yXsoM9NTfhgY2bMovbW6v3sURc2CwIAtQq+U01DezL3DQ+lss+HX/rZtzQLcr43NdCcbU9DCs3W5paHxrgYwwYOVuKiIharU6RgZgyoqvFIlN3W/zIIEQF+eLLh1NstjVdiVjso/sGYGBCGF67tbeoTfOCG/FWFGEBvpIhJbnMkT0FxeJ7yIVnGrVK8t58TaagvzyhJ54Y0QWjRLury/nu8cHGx1oLgUtYC8rctIw/jURERFc1p3xnZFI0dr80xq6ZQ9YyMc4a+hJTq5TY/2oa9IIAX7VSsuWDj8x0M3syN9YCNDmmGaKaeh1mj2vIunV+YZVZ+7uTO6J/Qphk2NFSPU2If+P+Xm1640wiIiJTzR2UsnfNHfFeVIId2z2IOfrVHRrQOMwlqfmRWRDQUoCQEBFgLOoN9vMRnbFdN2SauRlspVZKrVTgvbv7mR23tClmkEZtHGrz9ArFHJYiIqI2qbk1NM2llGRurE8LF5t6Yzfj49gQP5uvIxmWEmVu+nUMRUrnCIvXvTCuh+xxS5mbyCBf4/vw9K7gzNwQEVGL4q7VjiXBTROvFZwQDdmqufnnPf3w8YY/MDKpHf6yuHGGknjGU0yoxvjY0sfWtV3jTu7izM2AhHBJuyeGd8Fnm0/jwcEJeHRoF3SysDGoabHwyqkNawgF+DZmbjgsRURE5AEqSYFw076M+3QItd3IBvFLqmVmMUWH+OGN2/sgz2RdmSrRTuFRgaLgxsLrJMUG44vJKYgN9bO8hxaAWTf3wC1926NnXLBsfwx0JlmZvh3DjI9ZUExERCTDXWsdKx3I3Ox5eQxKq+vRPsy/2a8vrvNRWwk6TDMyHcIbX9veFYrH9Ioxf32TgE6lVODajraDNvFlH/65v+Sc4X14eio4gxsiImrz7N0+IipIg6ggje2GdpBmbiy/vrRoGLg+MRJvTexjtmGnuHB4+DXtmr2CsyUZd1yLBz7fiefSrsHt/TtIzqmNe0sxc0NERNTIjdtU3X99AvKvVONaJwwzNZUkuLGy82iovw+mj+6OD389CaAhw/Lg4E5m7cTx2ZK/XGf79e3vqkTfjmHY/1qaJGtk0DgsxcwNERGRR7zzJ9vbJriKeFjIWi0MANzYI9oY3Nho6hZygQ0ArlBMREQk57qr05PFu217o6Z8/YuzMhY3+Wzq67sg/lCzoJiIiMjcu3f1xZe/5+Du5HhPd8WlmhJchJjU3chx1xR6awyZG0tbNLitHx59dSIiIhORQRr8baz8AnLexNIGlHI6RwViZto1CA/0dWGPmo+ZGyIiojasqbmNqTdan/1kb+JmTM9orD9aJFuU3FzGgmJmboiIiNoeZ2c37A1uFk5KQXmtFqH+toe6mspYUOzh2VIsKCYiIvKAkup6p95PYWdJsVKpcElgA7ScYSkGN0RERB7gic06Xc1H1TIKihncEBEReYGYEOesnNwcxu0XmLkhIiJquwJ9VbYb2eH/7u6P1C6RWPzIIKfczxHGzA1XKCYiImq7BiSEO+U+CZEB+PbxwU65l6MMKy3rPDzmphCaus97K1dWVobQ0FCUlpYiJCTE090hIqI2au/Zy/h621m8PKEnokP8PN0dpxAEAXrB9nYSjmjK9zczN0RERB6Q3CkCyZ0iPN0Np1IoFLCywbnbsOaGiIiIvAqDGyIiIvIqDG6IiIjIqzC4ISIiIq/i8eDm008/RWJiIvz8/JCcnIwtW7ZYbLtx40YoFAqzn2PHjrmxx0RERNSSeTS4+e677zBjxgy89NJLyMrKwrBhwzBu3Djk5uZave748eMoKCgw/nTvbn2nVCIiImo7PLrOzfXXX4+BAwdi/vz5xmM9e/bExIkTkZGRYdZ+48aNGDVqFK5cuYKwsDC7XqO2tha1tbXG52VlZYiPj+c6N0RERK1IU9a58Vjmpq6uDnv37kVaWprkeFpaGrZt22b12gEDBiAuLg6jR4/Ghg0brLbNyMhAaGio8Sc+Pr7ZfSciIqKWy2PBTXFxMXQ6HWJiYiTHY2JiUFhYKHtNXFwcFi5ciMzMTKxYsQJJSUkYPXo0Nm/ebPF1Zs+ejdLSUuNPXl6eU98HERERtSweX6FYoZAuZSgIgtkxg6SkJCQlJRmfp6amIi8vD++//z6GDx8ue41Go4FG4/mdUomIiMg9PJa5iYqKgkqlMsvSFBUVmWVzrBk8eDBOnjzp7O4RERFRK+Wx4MbX1xfJyclYt26d5Pi6deswZMgQu++TlZWFuLg4Z3ePiIiIWimPDkulp6dj0qRJSElJQWpqKhYuXIjc3FxMmTIFQEO9TH5+PpYsWQIAmDt3Ljp37ozevXujrq4OS5cuRWZmJjIzMz35NoiIiKgF8Whwc++99+LSpUt44403UFBQgD59+mD16tXo1KkTAKCgoECy5k1dXR1mzpyJ/Px8+Pv7o3fv3li1ahXGjx/vqbdARERELYxH17nxhKbMkyciIqKWoVWsc0NERETkCgxuiIiIyKswuCEiIiKvwuCGiIiIvAqDGyIiIvIqDG6IiIjIqzC4ISIiIq/C4IaIiIi8CoMbIiIi8ioMboiIiMirMLghIiIir8LghoiIiLwKgxsiIiLyKgxuiIiIyKswuCEiIiKvwuCGiIiIvAqDGyIiIvIqDG6IiIjIqzC4ISIiIq/C4IaIiIi8CoMbIiIi8ioMboiIiMirMLghIiIir8LghoiIiLwKgxsiIiLyKgxuiIiIyKswuCEiIiKvwuCGiIiIvAqDGyIiIvIqDG6IiIjIqzC4ISIiIq/C4IaIiIi8CoMbIiIi8ioMboiIiMirMLghIiIir+Lx4ObTTz9FYmIi/Pz8kJycjC1btlhtv2nTJiQnJ8PPzw9dunTBggUL3NRTIiIiag08Gtx89913mDFjBl566SVkZWVh2LBhGDduHHJzc2Xb5+TkYPz48Rg2bBiysrLw4osvYtq0acjMzHRzz4mIiKilUgiCIHjqxa+//noMHDgQ8+fPNx7r2bMnJk6ciIyMDLP2zz//PFauXImjR48aj02ZMgX79+/H9u3b7XrNsrIyhIaGorS0FCEhIc1/E0RERORyTfn+VrupT2bq6uqwd+9evPDCC5LjaWlp2LZtm+w127dvR1pamuTY2LFjsWjRItTX18PHx8fsmtraWtTW1hqfl5aWAmj4kIiIiKh1MHxv25OT8VhwU1xcDJ1Oh5iYGMnxmJgYFBYWyl5TWFgo216r1aK4uBhxcXFm12RkZOD11183Ox4fH9+M3hMREZEnlJeXIzQ01GobjwU3BgqFQvJcEASzY7bayx03mD17NtLT043P9Xo9Ll++jMjISKuv01RlZWWIj49HXl4eh7s8hL8Dz+Ln73n8HXgWP3/XEgQB5eXlaN++vc22HgtuoqKioFKpzLI0RUVFZtkZg9jYWNn2arUakZGRstdoNBpoNBrJsbCwMMc7bkNISAj/UHsYfweexc/f8/g78Cx+/q5jK2Nj4LHZUr6+vkhOTsa6deskx9etW4chQ4bIXpOammrWfu3atUhJSZGttyEiIqK2x6NTwdPT0/HFF1/gyy+/xNGjR/Hss88iNzcXU6ZMAdAwpDR58mRj+ylTpuDs2bNIT0/H0aNH8eWXX2LRokWYOXOmp94CERERtTAerbm59957cenSJbzxxhsoKChAnz59sHr1anTq1AkAUFBQIFnzJjExEatXr8azzz6LTz75BO3bt8e8efNw5513euotGGk0Grz22mtmQ2DkPvwdeBY/f8/j78Cz+Pm3HB5d54aIiIjI2Ty+/QIRERGRMzG4ISIiIq/C4IaIiIi8CoMbIiIi8ioMbpzk008/RWJiIvz8/JCcnIwtW7Z4ukteISMjA4MGDUJwcDCio6MxceJEHD9+XNJGEAT8/e9/R/v27eHv74+RI0fi8OHDkja1tbV45plnEBUVhcDAQNx22204d+6cO9+KV8jIyIBCocCMGTOMx/j5u1Z+fj4efPBBREZGIiAgAP3798fevXuN5/n5u5ZWq8XLL7+MxMRE+Pv7o0uXLnjjjTeg1+uNbfg7aIEEarbly5cLPj4+wueffy4cOXJEmD59uhAYGCicPXvW011r9caOHSt89dVXwqFDh4Ts7GxhwoQJQkJCglBRUWFsM2fOHCE4OFjIzMwUDh48KNx7771CXFycUFZWZmwzZcoUoUOHDsK6deuEffv2CaNGjRL69esnaLVaT7ytVmnXrl1C586dhb59+wrTp083Hufn7zqXL18WOnXqJDz88MPCzp07hZycHGH9+vXCH3/8YWzDz9+13nrrLSEyMlL46aefhJycHOE///mPEBQUJMydO9fYhr+DlofBjRNcd911wpQpUyTHevToIbzwwgse6pH3KioqEgAImzZtEgRBEPR6vRAbGyvMmTPH2KampkYIDQ0VFixYIAiCIJSUlAg+Pj7C8uXLjW3y8/MFpVIprFmzxr1voJUqLy8XunfvLqxbt04YMWKEMbjh5+9azz//vDB06FCL5/n5u96ECROEv/zlL5Jjd9xxh/Dggw8KgsDfQUvFYalmqqurw969e5GWliY5npaWhm3btnmoV96rtLQUABAREQEAyMnJQWFhoeTz12g0GDFihPHz37t3L+rr6yVt2rdvjz59+vB3ZKenn34aEyZMwJgxYyTH+fm71sqVK5GSkoK7774b0dHRGDBgAD7//HPjeX7+rjd06FD8+uuvOHHiBABg//792Lp1K8aPHw+Av4OWyuO7grd2xcXF0Ol0Zpt9xsTEmG3ySc0jCALS09MxdOhQ9OnTBwCMn7Hc53/27FljG19fX4SHh5u14e/ItuXLl2Pfvn3YvXu32Tl+/q51+vRpzJ8/H+np6XjxxRexa9cuTJs2DRqNBpMnT+bn7wbPP/88SktL0aNHD6hUKuh0Orz99tu47777APD/gZaKwY2TKBQKyXNBEMyOUfNMnToVBw4cwNatW83OOfL583dkW15eHqZPn461a9fCz8/PYjt+/q6h1+uRkpKCd955BwAwYMAAHD58GPPnz5fsu8fP33W+++47LF26FMuWLUPv3r2RnZ2NGTNmoH379njooYeM7fg7aFk4LNVMUVFRUKlUZtF3UVGRWSRPjnvmmWewcuVKbNiwAR07djQej42NBQCrn39sbCzq6upw5coVi21I3t69e1FUVITk5GSo1Wqo1Wps2rQJ8+bNg1qtNn5+/PxdIy4uDr169ZIc69mzp3HPPf75d72//e1veOGFF/DnP/8Z1157LSZNmoRnn30WGRkZAPg7aKkY3DSTr68vkpOTsW7dOsnxdevWYciQIR7qlfcQBAFTp07FihUr8NtvvyExMVFyPjExEbGxsZLPv66uDps2bTJ+/snJyfDx8ZG0KSgowKFDh/g7smH06NE4ePAgsrOzjT8pKSl44IEHkJ2djS5duvDzd6EbbrjBbOmDEydOGDcX5p9/16uqqoJSKf2qVKlUxqng/B20UB4qZPYqhqngixYtEo4cOSLMmDFDCAwMFM6cOePprrV6Tz75pBAaGips3LhRKCgoMP5UVVUZ28yZM0cIDQ0VVqxYIRw8eFC47777ZKdhduzYUVi/fr2wb98+4cYbb+Q0TAeJZ0sJAj9/V9q1a5egVquFt99+Wzh58qTwzTffCAEBAcLSpUuNbfj5u9ZDDz0kdOjQwTgVfMWKFUJUVJQwa9YsYxv+DloeBjdO8sknnwidOnUSfH19hYEDBxqnKlPzAJD9+eqrr4xt9Hq98NprrwmxsbGCRqMRhg8fLhw8eFByn+rqamHq1KlCRESE4O/vL9xyyy1Cbm6um9+NdzANbvj5u9b//vc/oU+fPoJGoxF69OghLFy4UHKen79rlZWVCdOnTxcSEhIEPz8/oUuXLsJLL70k1NbWGtvwd9DyKARBEDyZOSIiIiJyJtbcEBERkVdhcENERERehcENEREReRUGN0RERORVGNwQERGRV2FwQ0RERF6FwQ0RERF5FQY3RERE5FUY3BCRW4wcORIzZsxw62ueOXMGCoUC2dnZbn1dIvIsBjdE1Cps3LgRCoUCJSUlnu4KEbVwDG6IiIjIqzC4ISK30Wq1mDp1KsLCwhAZGYmXX34Zhu3tli5dipSUFAQHByM2Nhb3338/ioqKADQML40aNQoAEB4eDoVCgYcffhgAoNfr8Y9//APdunWDRqNBQkIC3n77bcnrnj59GqNGjUJAQAD69euH7du3S85v27YNw4cPh7+/P+Lj4zFt2jRUVlYaz3/66afo3r07/Pz8EBMTg7vuustVHxEROQGDGyJym6+//hpqtRo7d+7EvHnz8MEHH+CLL74AANTV1eHNN9/E/v378cMPPyAnJ8cYwMTHxyMzMxMAcPz4cRQUFODDDz8EAMyePRv/+Mc/8Morr+DIkSNYtmwZYmJiJK/70ksvYebMmcjOzsY111yD++67D1qtFgBw8OBBjB07FnfccQcOHDiA7777Dlu3bsXUqVMBAHv27MG0adPwxhtv4Pjx41izZg2GDx/ujo+LiBzl4V3JiaiNGDFihNCzZ09Br9cbjz3//PNCz549Zdvv2rVLACCUl5cLgiAIGzZsEAAIV65cMbYpKysTNBqN8Pnnn8veIycnRwAgfPHFF8Zjhw8fFgAIR48eFQRBECZNmiQ8/vjjkuu2bNkiKJVKobq6WsjMzBRCQkKEsrIyh943EbkfMzdE5DaDBw+GQqEwPk9NTcXJkyeh0+mQlZWF22+/HZ06dUJwcDBGjhwJAMjNzbV4v6NHj6K2thajR4+2+rp9+/Y1Po6LiwMA45DX3r17sXjxYgQFBRl/xo4dC71ej5ycHNx0003o1KkTunTpgkmTJuGbb75BVVWVox8BEbkBgxsi8riamhqkpaUhKCgIS5cuxe7du/H9998DaBiussTf39+u+/v4+BgfG4IrvV5v/O8TTzyB7Oxs48/+/ftx8uRJdO3aFcHBwdi3bx++/fZbxMXF4dVXX0W/fv04a4uoBWNwQ0Rus2PHDrPn3bt3x7Fjx1BcXIw5c+Zg2LBh6NGjhzGzYuDr6wsA0Ol0xmPdu3eHv78/fv31V4f7NHDgQBw+fBjdunUz+zG8plqtxpgxY/Duu+/iwIEDOHPmDH777TeHX5OIXIvBDRG5TV5eHtLT03H8+HF8++23+OijjzB9+nQkJCTA19cXH330EU6fPo2VK1fizTfflFzbqVMnKBQK/PTTT7h48SIqKirg5+eH559/HrNmzcKSJUtw6tQp7NixA4sWLbK7T88//zy2b9+Op59+GtnZ2Th58iRWrlyJZ555BgDw008/Yd68ecjOzsbZs2exZMkS6PV6JCUlOfWzISLnYXBDRG4zefJkVFdX47rrrsPTTz+NZ555Bo8//jjatWuHxYsX4z//+Q969eqFOXPm4P3335dc26FDB7z++ut44YUXEBMTY5zN9Morr+C5557Dq6++ip49e+Lee+81y/pY07dvX2zatAknT57EsGHDMGDAALzyyivG2pywsDCsWLECN954I3r27IkFCxbg22+/Re/evZ33wRCRUykE4eoiE0RERERegJkbIiIi8ioMboiIiMirMLghIiIir8LghoiIiLwKgxsiIiLyKgxuiIiIyKswuCEiIiKvwuCGiIiIvAqDGyIiIvIqDG6IiIjIqzC4ISIiIq/y/2N+pnOFG341AAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plt.plot(epochs_x[-1], epochs_y[-1])\n", + "plt.xlabel('batches')\n", + "plt.ylabel('loss')\n", + "plt.ylim(0,)\n", + "plt.show()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "speck-rescnn", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From c2ac1ced8b0243ae14933459cb429238d4d3981d Mon Sep 17 00:00:00 2001 From: WillianSG Date: Fri, 26 Apr 2024 20:57:57 +0200 Subject: [PATCH 048/379] non-sequential example 1 performance --- .../non-sequential-SCNN-example_1.ipynb | 240 ++++++++++++++++-- 1 file changed, 222 insertions(+), 18 deletions(-) diff --git a/tests/test_nonsequential/non-sequential-SCNN-example_1.ipynb b/tests/test_nonsequential/non-sequential-SCNN-example_1.ipynb index becc5d50..16c36d57 100644 --- a/tests/test_nonsequential/non-sequential-SCNN-example_1.ipynb +++ b/tests/test_nonsequential/non-sequential-SCNN-example_1.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": 18, "metadata": {}, "outputs": [], "source": [ @@ -19,7 +19,8 @@ "\n", "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential\n", "\n", - "import matplotlib.pyplot as plt" + "import matplotlib.pyplot as plt\n", + "import numpy as np" ] }, { @@ -30,7 +31,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 2, @@ -113,7 +114,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "device: NVIDIA GeForce RTX 3070 Ti\n" + "device: NVIDIA RTX A4000\n" ] } ], @@ -366,7 +367,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "546940b28031476fb3bee7a239fd1529", + "model_id": "fbe5ba8d578d481b97c8e81bebb4d2c7", "version_major": 2, "version_minor": 0 }, @@ -378,17 +379,164 @@ "output_type": "display_data" }, { - "ename": "KeyboardInterrupt", - "evalue": "", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[15], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m epochs_x, epochs_y \u001b[38;5;241m=\u001b[39m train(snn_train_dataloader, snn, loss_fn, optimizer, epochs, test, snn_test_dataloader)\n", - "Cell \u001b[0;32mIn[13], line 29\u001b[0m, in \u001b[0;36mtrain\u001b[0;34m(dataloader, model, loss_fn, optimizer, epochs, test_func, dataloader_test)\u001b[0m\n\u001b[1;32m 27\u001b[0m \u001b[38;5;66;03m# gradient update\u001b[39;00m\n\u001b[1;32m 28\u001b[0m optimizer\u001b[38;5;241m.\u001b[39mzero_grad()\n\u001b[0;32m---> 29\u001b[0m loss\u001b[38;5;241m.\u001b[39mbackward()\n\u001b[1;32m 30\u001b[0m optimizer\u001b[38;5;241m.\u001b[39mstep()\n\u001b[1;32m 32\u001b[0m \u001b[38;5;66;03m# detach the neuron states and activations from current computation graph(necessary)\u001b[39;00m\n", - "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/_tensor.py:522\u001b[0m, in \u001b[0;36mTensor.backward\u001b[0;34m(self, gradient, retain_graph, create_graph, inputs)\u001b[0m\n\u001b[1;32m 512\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m has_torch_function_unary(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m 513\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m handle_torch_function(\n\u001b[1;32m 514\u001b[0m Tensor\u001b[38;5;241m.\u001b[39mbackward,\n\u001b[1;32m 515\u001b[0m (\u001b[38;5;28mself\u001b[39m,),\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 520\u001b[0m inputs\u001b[38;5;241m=\u001b[39minputs,\n\u001b[1;32m 521\u001b[0m )\n\u001b[0;32m--> 522\u001b[0m torch\u001b[38;5;241m.\u001b[39mautograd\u001b[38;5;241m.\u001b[39mbackward(\n\u001b[1;32m 523\u001b[0m \u001b[38;5;28mself\u001b[39m, gradient, retain_graph, create_graph, inputs\u001b[38;5;241m=\u001b[39minputs\n\u001b[1;32m 524\u001b[0m )\n", - "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/autograd/__init__.py:266\u001b[0m, in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[0m\n\u001b[1;32m 261\u001b[0m retain_graph \u001b[38;5;241m=\u001b[39m create_graph\n\u001b[1;32m 263\u001b[0m \u001b[38;5;66;03m# The reason we repeat the same comment below is that\u001b[39;00m\n\u001b[1;32m 264\u001b[0m \u001b[38;5;66;03m# some Python versions print out the first line of a multi-line function\u001b[39;00m\n\u001b[1;32m 265\u001b[0m \u001b[38;5;66;03m# calls in the traceback and some print out the last line\u001b[39;00m\n\u001b[0;32m--> 266\u001b[0m Variable\u001b[38;5;241m.\u001b[39m_execution_engine\u001b[38;5;241m.\u001b[39mrun_backward( \u001b[38;5;66;03m# Calls into the C++ engine to run the backward pass\u001b[39;00m\n\u001b[1;32m 267\u001b[0m tensors,\n\u001b[1;32m 268\u001b[0m grad_tensors_,\n\u001b[1;32m 269\u001b[0m retain_graph,\n\u001b[1;32m 270\u001b[0m create_graph,\n\u001b[1;32m 271\u001b[0m inputs,\n\u001b[1;32m 272\u001b[0m allow_unreachable\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m 273\u001b[0m accumulate_grad\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m 274\u001b[0m )\n", - "\u001b[0;31mKeyboardInterrupt\u001b[0m: " + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "df009ff9b4ff4e3f99d67171a79422dc", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/156 [00:00" ] @@ -419,6 +567,62 @@ "plt.ylim(0,)\n", "plt.show()" ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAGwCAYAAABVdURTAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABVm0lEQVR4nO3de1xUZf4H8M/MADOgMIjcEQVFvIsoSl7SVJTUtbTd1dQNV2vL1NJQS800q40ualZeK0vbtryUWr90RcQLqZiJUl5RLooXriozMMAAM+f3Bzo5cpHBGQ4zfN6v17w2zjznzPfp7MSH5zzPORJBEAQQERER2Qip2AUQERERmRPDDREREdkUhhsiIiKyKQw3REREZFMYboiIiMimMNwQERGRTWG4ISIiIptiJ3YBDU2v1+PGjRtwdnaGRCIRuxwiIiKqA0EQUFhYCF9fX0iltY/NNLlwc+PGDfj7+4tdBhEREdXD1atX0apVq1rbNLlw4+zsDKDyX46Li4vI1RAREVFdqNVq+Pv7G36P16bJhZu7l6JcXFxqDTcJCQn48MMPkZSUhKysLOzYsQNjxoyp9dharRZvvfUWvvnmG2RnZ8PHxweLFy/G1KlTAQCPPfYYDh06VGW/kSNHYteuXfXvFBERURNRlyklTS7c1JVGo0FISAimTp2Kp556qk77jBs3Djk5OdiwYQOCgoKQlZUFvV5veH/79u0oKysz/Hzz5k2EhITg73//u9nrJyIiaqoYbmowYsQIjBgxos7t9+zZg0OHDiE9PR1ubm4AgICAAKM2d7fftXnzZjg5OTHcEBERmRGXgpvJTz/9hLCwMHzwwQfw8/NDcHAw5s6di5KSkhr32bBhA55++mk0a9asASslIiKybRy5MZP09HQcPnwYCoUCO3bsQH5+PqZPn46bN2/iq6++qtL++PHjOHPmDDZs2CBCtURERLaL4cZM9Ho9JBIJ/vvf/0KpVAIAVqxYgb/97W9Ys2YNHB0djdpv2LAB3bp1Q58+fcQol4iIyGbxspSZ+Pj4wM/PzxBsAKBTp04QBAHXrl0zaqvRaLB582Y8++yzDV0mERGRzWO4MZP+/fvjxo0bKCoqMmy7ePEipFJplZsNbdu2DVqtFv/4xz8aukwiIiKbx3BTg6KiIiQnJyM5ORkAkJGRgeTkZGRmZgIAFixYgKioKEP7iRMnomXLlpgyZQrOnTuHhIQEzJs3D1OnTq32ktSYMWPQsmXLBusPERFRU8E5NzU4ceIEBg8ebPg5OjoaADB58mRs3LgRWVlZhqADAM2bN8d/t/8f5rwyG73CwuDesiXGjRuHd955x+i4KSkpOHz4MPbu3dswHSEiImpiJIIgCGIX0ZDUajWUSiVUKpVZH7+w5bdMLNh+GnoBkEqAmKe6YXzv1mY7PhERUVNmyu9vXpYygyxViSHYAIBeABZuP4MsVc33uCEiIiLLYLgxg4x8jSHY3KUTBFzOLxanICIioiaM4cYMAt2bQXrfc7xkEiDA3UmcgoiIiJowhhsz8FE6IuapbkYBZ0Kf1vBROta8ExEREVkEw42ZjO/dGkfmD8HYUF8AwK8Zt6C7/1oVERERWRzDjRn5KB3x5hNd4aKww6XcIvz0+3WxSyIiImpyGG7MTOlojxcGtQMAfBR3CeU6vcgVERERNS0MNxYwpX8A3Js7IPNWMbaduPbgHYiIiMhsGG4swMnBDtMfCwIAfLr/EkrLdSJXRERE1HQw3FjIxPDW8FEqkKUqxX9/zXzwDkRERGQWDDcWorCX4eWh7QEAaw6kQqOtELkiIiKipoHhxoL+1qsV2rR0wk1NGTYevSx2OURERE2CqOEmISEBo0ePhq+vLyQSCXbu3PnAfbRaLV5//XW0adMGcrkcAQEB+PLLLy1fbD3Yy6SYHVE5erP+UBpUJeUiV0RERGT7RA03Go0GISEhWL16dZ33GTduHOLj47FhwwakpKTgu+++Q4cOHSxY5cN5IsQP7T2bQ11agS9+SRe7HCIiIptnJ+aHjxgxAiNGjKhz+z179uDQoUNIT0+Hm5sbACAgIKDWfbRaLbRareFntVpdr1rrSyaVYM7wYEz75iS+PJyBf/YLQMvm8gatgYiIqCmxqjk3P/30E8LCwvDBBx/Az88PwcHBmDt3LkpKSmrcJyYmBkql0vDy9/dvwIorRXbxRjc/JTRlOqw9mNbgn09ERNSUWFW4SU9Px+HDh3HmzBns2LEDK1euxPfff4/p06fXuM+CBQugUqkMr6tXrzZgxZUkksrRGwD4+tgVZKtKG7wGIiKipsKqwo1er4dEIsF///tf9OnTByNHjsSKFSuwadOmGkdv5HI5XFxcjF5iGBTsgd4BLVBWocen+y+JUgMREVFTYFXhxsfHB35+flAqlYZtnTp1giAIuHatcT/mQCKRYO7wyonPW367isybxSJXREREZJusKtz0798fN27cQFFRkWHbxYsXIZVK0apVKxErq5vwti3xaHt3VOgFrIy/KHY5RERENknUcFNUVITk5GQkJycDADIyMpCcnIzMzMrHFSxYsABRUVGG9hMnTkTLli0xZcoUnDt3DgkJCZg3bx6mTp0KR0dHMbpgsrujNztPXUdqbqHI1RAREdkeUcPNiRMnEBoaitDQUABAdHQ0QkNDsXjxYgBAVlaWIegAQPPmzREXF4eCggKEhYVh0qRJGD16ND755BNR6q+PEH9XDOvsBb0ArIjj6A0REZG5SQRBEMQuoiGp1WoolUqoVCrRJhdfyFZjxMe/QBCAn18agK5+ygfvRERE1ISZ8vvbqubc2IqO3i4Y3d0XAEdviIiIzI3hRiSvDAuGTCrB/gu5SLpyW+xyiIiIbAbDjUgC3Zvhbz0rV3gti00RuRoiIiLbwXAjopcj2sNBJkVi+k0cSc0XuxwiIiKbwHAjIj9XR0wMbw0A+DA2BU1sbjcREZFFMNyIbPrgdlDYS5F8tQDx53PFLoeIiMjqMdyIzNNZgcn9AgAAy/amQK/n6A0REdHDYLhpBKYNbAdnuR0uZBdi95ksscshIiKyagw3jUCLZg549tFAAJX3vanQ6UWuiIiIyHox3DQSzw4IRAsne6TnabDj1HWxyyEiIrJaDDeNhLPCHtMGtQMAfBx/CWUVHL0hIiKqD4abRiSqbwA8nOW4drsEW37LfPAOREREVAXDTSPi6CDDS0OCAACf7k9FSZlO5IqIiIisD8NNI/N079bwc3VEbqEW/zl2WexyiIiIrA7DTSPjYCfFrIj2AIC1B9NQWFouckVERETWheGmEXoq1A9t3ZvhdnE5vjx8WexyiIiIrArDTSNkJ5Ni9rBgAMAXv6SjoLhM5IqIiIisB8NNI/WXbj7o6O2MQm0F1ieki10OERGR1WC4aaSkUgnmDO8AANh45DJyC0tFroiIiMg6MNw0YhGdPBHi74qSch3WHEgTuxwiIiKrwHDTiEkkEsy7M3rz7a+ZuF5QInJFREREjR/DTSPXP6glHmnrhjKdHp/GXxK7HCIiokaP4aaRk0gkmBdZOXqzLekaMvI1IldERETUuDHcWIFebdwwuIMHdHoBK/ddFLscIiKiRo3hxkrcXTn10+83kJJdKHI1REREjRfDjZXo6qfEiK7eEARgRVyK2OUQERE1Wgw3ViR6WDAkEiD2bA7+uFYgdjlERESNEsONFWnv5YyxPfwAAMv2cu4NERFRdRhurMzsiGDYSSVIuJiH4xm3xC6HiIio0WG4sTKtWzphXG9/AMCy2BQIgiByRURERI0Lw40VemlIEBzspDh++RYSLuWLXQ4REVGjImq4SUhIwOjRo+Hr6wuJRIKdO3fWed8jR47Azs4OPXr0sFh9jZWP0hHPPNIGALB8L0dviIiI7iVquNFoNAgJCcHq1atN2q+goABRUVEYOnSohSpr/F58rB2cHGT445oKsWdzxC6HiIio0RA13IwYMQLvvPMOxo4da9J+06ZNw8SJE9G3b18LVdb4uTeXY0r/AACV973R6Tl6Q0REBFjhnJuvvvoK6enpWLJkSZ3aa7VaqNVqo5eteP7RdnBW2OFiThF+/uOG2OUQERE1ClYVbi5duoT58+fjm2++gZ2dXZ32iYmJgVKpNLz8/f0tXGXDUTrZ44WBbQEAH8VdRLlOL3JFRERE4rOacKPT6TBx4kQsXboUwcHBdd5vwYIFUKlUhtfVq1ctWGXDm9I/EC2bOeDyzWL8kHRN7HKIiIhEZzXhprCwECdOnMDMmTNhZ2cHOzs7vPXWW/j9999hZ2eH/fv3V7ufXC6Hi4uL0cuWNJPb4cXH2gEAPom/BG2FTuSKiIiIxGU14cbFxQWnT59GcnKy4TVt2jR06NABycnJCA8PF7tE0fzjkTbwdlHghqoU3/6aKXY5REREoqrbxBULKSoqQmpqquHnjIwMJCcnw83NDa1bt8aCBQtw/fp1fP3115BKpejatavR/p6enlAoFFW2NzUKexleGhqE13ecweoDqRjf2x9ODqKeWiIiItGIOnJz4sQJhIaGIjQ0FAAQHR2N0NBQLF68GACQlZWFzEyORNTFuDB/tHZzQn5RGTYevSx2OURERKKRCE3s9rZqtRpKpRIqlcrm5t/8kHQNc7b9DqWjPRJeHQylo73YJREREZmFKb+/rWbODT3YmFA/BHk2h6qkHBsOZ4hdDhERkSgYbmyITCpB9LDKZfIbfknHLU2ZyBURERE1PIYbG/N4F2908XWBpkyHdYfSxC6HiIiowTHc2BipVIK5wzsAADYdvYwcdanIFRERETUshhsb9FgHD/Rq0wLaCj1W7U998A5EREQ2hOHGBkkkf47ebP4tE1dvFYtcERERUcNhuLFRfdu1xIAgd5TrBHwcf0nscoiIiBoMw40NmzO8cuXU9pPXkJpbJHI1REREDYPhxoaFtm6BiE6e0AvAR/suil0OERFRg2C4sXHRwyrn3uz6IwvnbqhFroaIiMjyGG5sXGdfF/yluw8AYEVcisjVEBERWR7DTRPwyrBgSCXAvvO5OJl5W+xyiIiILIrhpglo59Ecf+3ZCgCwfC9Hb4iIyLYx3DQRLw9tD3uZBEdSb+JoWr7Y5RAREVkMw00T4e/mhAl9WgMAlsWmQBAEkSsiIiKyDIabJmTm4CDI7aQ4mVmAAym5YpdDRERkEQw3TYiniwKT+wUAAJbFXoRez9EbIiKyPQw3Tcy0Qe3QXG6Hc1lq/O9MttjlEBERmR3DTRPj1swBUwcEAqi8742OozdERGRjGG6aoOceDYTS0R5peRrsPHVd7HKIiIjMiuGmCXJR2GPaoHYAgJXxF1FWoRe5IiIiIvNhuGmiJvdrA/fmcly9VYKtJ66KXQ4REZHZMNw0UU4Odpg5uHL05tP9l1BarhO5IiIiIvNguGnCJoS3hq9SgRy1Ft8cuyJ2OURERGbBcNOEye1keHloewDAmoNpKNJWiFwRERHRw2O4aeL+2qsVAlo64ZamDF8dzhC7HCIioofGcNPE2cukeGVYMADgs1/SoSouF7kiIiKih8NwQxjd3RcdvJxRWFqBz35JE7scIiKih8JwQ5BKJYgeXjl689WRy8gv0opcERERUf0x3BAAYHhnL4S0UqK4TIc1Bzh6Q0RE1ovhhgAAEokEc4Z3AAB88+sVZKlKRK6IiIiofhhuyODR9u7oE+iGsgo9PolPFbscIiKiehE13CQkJGD06NHw9fWFRCLBzp07a22/fft2DBs2DB4eHnBxcUHfvn0RGxvbMMU2ARKJBHPvjN5sO3EVV25qRK6IiIjIdKKGG41Gg5CQEKxevbpO7RMSEjBs2DDs3r0bSUlJGDx4MEaPHo1Tp05ZuNKmo0+gGwYGe6BCL2Dlvktil0NERGQyiSAIgthFAJWjBjt27MCYMWNM2q9Lly4YP348Fi9eXKf2arUaSqUSKpUKLi4u9ajU9v1xrQBPrDoCiQSInT0QwV7OYpdERERNnCm/v616zo1er0dhYSHc3NxqbKPVaqFWq41eVLvurVwR2cULggB8FHdR7HKIiIhMYtXhZtmyZSgqKsK4ceNqbBMTEwOlUml4+fv7N2CF1mvO8A6QSID/ncnGmesqscshIiKqM6sNN99++y2WLl2KrVu3wtPTs8Z2CxYsgEqlMryuXr3agFVar2AvZzwZ4gsAWLY3ReRqiIiI6s4qw83mzZvx3HPPYevWrYiIiKi1rVwuh4uLi9GL6mZ2RDBkUgkOpuThxOVbYpdDRERUJ1YXbr777jtMmTIF3333HUaNGiV2OTYtwL0ZxoW1AgB8GJuCRjL3nIiIqFaihpuioiIkJycjOTkZAJCRkYHk5GRkZmYCqLykFBUVZWj/7bffIioqCsuXL0d4eDiys7ORnZ0NlYpzQizlpSHt4SCT4teMWzicmi92OURERA8karg5ceIEQkNDERoaCgCIjo5GaGioYVl3VlaWIegAwGeffYaKigrMmDEDPj4+htesWbNEqb8p8HV1xMTw1gCAZRy9ISIiK9Bo7nPTUHifG9PlFWox8IMDKCnX4bNnemF4F2+xSyIioiamydznhhqGh7Mc/+wfAABYEXcRen2TysNERGRlGG6oTl4Y2BbOcjtcyC7Ez6ezxC6HiIioRgw3VCeuTg7418C2AICVcRdRodOLXBEREVH1GG6ozqYOCIRbMwek52uw/eR1scshIiKqFsMN1VlzuR1eHNQOAPBx/CVoK3QiV0RERFQVww2Z5Jm+beDlIsf1ghJsPs5HWRARUePDcEMmUdjLMHNIewDAqgOpKCnj6A0RETUuDDdksvFh/mjVwhF5hVpsSrwsdjlERERGGG7IZA52UswaWjl6s+5QGtSl5SJXRERE9CeGG6qXsaF+aOvRDAXF5fjycIbY5RARERkw3FC92MmkiB4WDAD44pcM3NaUiVwRERFRJYYbqreRXX3QyccFRdoKrEtIE7scIiIiAAw39BCkUgnmDq8cvdl09DJy1aUiV0RERMRwQw9pSEdPhLZ2RWm5HqsPpIpdDhEREcMNPRyJRIJ5wzsAAL49nolrt4tFroiIiJo6hht6aP2C3NGvXUuU6wR8En9J7HKIiKiJY7ghs5hzZ/Tmh5PXkZ5XJHI1RETUlDHckFn0atMCQzp6QqcX8NE+jt4QEZF4GG7IbObcWTn1f7/fwPkstcjVEBFRU8VwQ2bTxVeJUd18AAAr4i6KXA0RETVVDDdkVq8MC4ZUAsSdy0Hy1QKxyyEioiaI4YbMKsizOcaGtgIALN+bInI1RETUFJkcbkpKSlBc/Oe9TK5cuYKVK1di7969Zi2MrNfsiPawl0nwy6V8HEu/KXY5RETUxJgcbp588kl8/fXXAICCggKEh4dj+fLlePLJJ7F27VqzF0jWx9/NCeN7+wMAlsWmQBAEkSsiIqKmxORwc/LkSTz66KMAgO+//x5eXl64cuUKvv76a3zyySdmL5Cs08zB7SG3k+LElds4eDFP7HKIiKgJMTncFBcXw9nZGQCwd+9ePPXUU5BKpXjkkUdw5coVsxdI1slbqcAzj7QBUDn3hqM3RETUUEwON0FBQdi5cyeuXr2K2NhYDB8+HACQm5sLFxcXsxdI1uvFx9qhmYMMZ66rsedMttjlEBFRE2FyuFm8eDHmzp2LgIAAhIeHo2/fvgAqR3FCQ0PNXiBZr5bN5Zg6IBBA5X1vdHqO3hARkeVJhHpcL8jOzkZWVhZCQkIglVbmo+PHj8PFxQUdO3Y0e5HmpFaroVQqoVKpONLUAFQl5Xj0/f1Ql1bgo/EhhmXiREREpjDl93e97nPj7e2N0NBQSKVSqNVq7Ny5E87Ozo0+2FDDUzra44VB7QAAH8VdQrlOL3JFRERk60wON+PGjcOqVasAVN7zJiwsDOPGjUP37t3xww8/mL1Asn5T+gfAvbkDMm8VY9uJa2KXQ0RENs7kcJOQkGBYCr5jxw4IgoCCggJ88skneOedd8xeIFk/Jwc7TH8sCADw6f5LKC3XiVwRERHZMpPDjUqlgpubGwBgz549+Otf/wonJyeMGjUKly5dMulYCQkJGD16NHx9fSGRSLBz584H7nPw4EH07NkTcrkcQUFB2Lhxo6ldIBFMDG8NH6UCWapS/PfXTLHLISIiG2ZyuPH390diYiI0Gg327NljWAp++/ZtKBQKk46l0WgQEhKC1atX16l9RkYGRo0ahcGDByM5ORmzZ8/Gc889h9jYWFO7QQ1MYS/DS0PaAwDWHEiFRlshckVERGSr7EzdYfbs2Zg0aRKaN2+ONm3a4LHHHgNQOQrTrVs3k441YsQIjBgxos7t161bh8DAQCxfvhwA0KlTJxw+fBgfffQRIiMjq91Hq9VCq9Uaflar1SbVSObz97BWWJ+Qhis3i7Hx6GXMGBwkdklERGSDTB65mT59OhITE/Hll1/i8OHDhqXgbdu2tficm8TERERERBhti4yMRGJiYo37xMTEQKlUGl7+/v4WrZFqZi+TYnZE5ejN+kNpUJWUi1wRERHZonotBQ8LC8PYsWPRrFkzw231R40ahf79+5u1uPtlZ2fDy8vLaJuXlxfUajVKSkqq3WfBggVQqVSG19WrVy1aI9XuiRA/tPdsDnVpBb74JV3scoiIyAbVK9x8/fXX6NatGxwdHeHo6Iju3bvjP//5j7lrMwu5XA4XFxejF4lHJpVgzvBgAMCXhzNws0j7gD2IiIhMY3K4WbFiBV588UWMHDkSW7duxdatW/H4449j2rRp+OijjyxRo4G3tzdycnKMtuXk5MDFxQWOjo4W/Wwyn8gu3ujmp4SmTIe1B9PELoeIiGyMyeHm008/xdq1a/H+++/jiSeewBNPPIEPPvgAa9aswSeffGKJGg369u2L+Ph4o21xcXGG51uRdZBI/hy9+frYFWSrSkWuiIiIbInJ4SYrKwv9+vWrsr1fv37Iysoy6VhFRUVITk5GcnIygMql3snJycjMrLwPyoIFCxAVFWVoP23aNKSnp+PVV1/FhQsXsGbNGmzduhWvvPKKqd0gkQ0K9kDvgBYoq9Dj0/2m3R+JiIioNiaHm6CgIGzdurXK9i1btqB9+/YmHevEiRMIDQ01PE08OjoaoaGhWLx4MYDKIHU36ABAYGAgdu3ahbi4OISEhGD58uX44osvalwGTo1X5ehNBwDAlt+uIvNmscgVERGRrTD5qeA//PADxo8fj4iICMPqqCNHjiA+Ph5bt27F2LFjLVKoufCp4I3LMxt+xS+X8vFUTz+sGNdD7HKIiKiRsuhTwf/617/i119/hbu7O3bu3ImdO3fC3d0dx48fb/TBhhqfu6M3O09dR2puocjVEBGRLTB55MbaceSm8fnX1ycQdy4Ho7r5YPWknmKXQ0REjZApv7/r9PgFUx5ZwMBAppozPBj7zudg1+ksvHhdha5+SrFLIiIiK1ancOPq6gqJRFJrG0EQIJFIoNPpzFIYNR0dvV0wursvfvr9BlbEXcSX/+wtdklERGTF6hRuDhw4YOk6qIl7ZVgwdp3Owv4LuUi6chu92rQQuyQiIrJSdQo3gwYNsnQd1MQFujfD33q2wpYTV7EsNgXfPf+I2CUREZGVqtezpYgs4eWI9nCQSZGYfhNHUvPFLoeIiKwUww01Gn6ujpjQxx8A8GFsCprYQj4iIjIThhtqVGYMCYLCXorkqwWIP58rdjlERGSFGG6oUfF0VmByvwAAwLK9KdDrOXpDRESmqVe4qaiowL59+7B+/XoUFlbeVfbGjRsoKioya3HUNE0b2A7OcjtcyC7E7jOmPYyViIjI5HBz5coVdOvWDU8++SRmzJiBvLw8AMD777+PuXPnmr1AanpaNHPAs48GAgBWxF1EhU4vckVERGRNTA43s2bNQlhYGG7fvg1HR0fD9rFjxyI+Pt6sxVHT9eyAQLRwskd6ngY7Tl0XuxwiIrIiJoebX375BYsWLYKDg4PR9oCAAFy/zl9CZB7OCntMG9QOAPBx/CWUVXD0hoiI6sbkcKPX66t9xMK1a9fg7OxslqKIACCqbwA8nOW4drsEW37LFLscIiKyEiaHm+HDh2PlypWGnyUSCYqKirBkyRKMHDnSnLVRE+foIMPMwUEAgE/3p6KkjM8tIyKiBzM53CxfvhxHjhxB586dUVpaiokTJxouSb3//vuWqJGasKf7+MPP1RG5hVr859hlscshIiIrIBHqcRvYiooKbN68GX/88QeKiorQs2dPTJo0yWiCcWOlVquhVCqhUqng4uIidjlUB1t/u4pXf/gDLZzskfDqYDgr7MUuiYiIGpgpv7/rFW6sGcON9anQ6TH8owSk52vwSkQwZkW0F7skIiJqYKb8/q7TU8Hv9dNPP1W7XSKRQKFQICgoCIGBgaYelqhGdjIpZg8LxsvfncIXv6Rjcr82cHVyePCORETUJJkcbsaMGQOJRFLloYZ3t0kkEgwYMAA7d+5EixYtzFYoNW1/6eaDNQdScSG7EOsT0vHa4x3FLomIiBopkycUx8XFoXfv3oiLi4NKpYJKpUJcXBzCw8Px888/IyEhATdv3uTdismspFIJ5gzvAADYeOQycgtLRa6IiIgaK5NHbmbNmoXPPvsM/fr1M2wbOnQoFAoFnn/+eZw9exYrV67E1KlTzVooUUQnT4T4u+L3qwVYcyANbz7RReySiIioETJ55CYtLa3aiTwuLi5IT08HALRv3x75+fkPXx3RPSQSCebdGb359tdMXC8oEbkiIiJqjEwON7169cK8efMMD8wEgLy8PLz66qvo3bs3AODSpUvw9/c3X5VEd/QPaonwQDeU6fT4NP6S2OUQEVEjZHK42bBhAzIyMtCqVSsEBQUhKCgIrVq1wuXLl/HFF18AAIqKirBo0SKzF0skkUgwL7Jy9GZb0jVk5GtEroiIiBqbet3nRq/XY+/evbh48SIAoEOHDhg2bBikUpOzUoPjfW5swz+/Oo6DKXl4socvPn46VOxyiIjIwngTv1ow3NiGM9dV+MunhyGRAHtmDUQHbz60lYjIlln0Jn4AoNFocOjQIWRmZqKsrMzovZdffrk+hyQySVc/JUZ09cb/zmRjRVwK1j8TJnZJRETUSJgcbk6dOoWRI0eiuLgYGo0Gbm5uyM/Ph5OTEzw9PRluqMFEDwvGnrPZiD2bgz+uFaB7K1exSyIiokbA5Ekyr7zyCkaPHo3bt2/D0dERx44dw5UrV9CrVy8sW7bMEjUSVau9lzPG9vADACzbe1HkaoiIqLEwOdwkJydjzpw5kEqlkMlk0Gq18Pf3xwcffICFCxfWq4jVq1cjICAACoUC4eHhOH78eK3tV65ciQ4dOsDR0RH+/v545ZVXUFrKO9Y2RbMjgmEnlSDhYh6OZ9wSuxwiImoETA439vb2hlVRnp6eyMzMBAAolUpcvXrV5AK2bNmC6OhoLFmyBCdPnkRISAgiIyORm5tbbftvv/0W8+fPx5IlS3D+/Hls2LABW7ZsqXewIuvWuqUTxvWuvKfSstiUKs88IyKipsfkcBMaGorffvsNADBo0CAsXrwY//3vfzF79mx07drV5AJWrFiBf/3rX5gyZQo6d+6MdevWwcnJCV9++WW17Y8ePYr+/ftj4sSJCAgIwPDhwzFhwoQHjvaQ7XppSBAc7KQ4fvkWEi7xzthERE2dyeHm3XffhY+PDwDg3//+N1q0aIEXX3wReXl5+Oyzz0w6VllZGZKSkhAREfFnQVIpIiIikJiYWO0+/fr1Q1JSkiHMpKenY/fu3Rg5cmS17bVaLdRqtdGLbIuP0hH/CG8DAFi+l6M3RERNnUmrpQRBgKenp2GExtPTE3v27Kn3h+fn50On08HLy8tou5eXFy5cuFDtPhMnTkR+fj4GDBgAQRBQUVGBadOm1XhZKiYmBkuXLq13jWQdpg9uh82/ZeKPayrEns3B4129xS6JiIhEYtLIjSAICAoKqtfcGnM5ePAg3n33XaxZswYnT57E9u3bsWvXLrz99tvVtl+wYAFUKpXhJWbtZDnuzeWY0j8AALAiLgU6PUdviIiaKpPCjVQqRfv27XHz5k2zfLi7uztkMhlycnKMtufk5MDbu/q/vN944w0888wzeO6559CtWzeMHTsW7777LmJiYqDX66u0l8vlcHFxMXqRbXr+0XZwVtjhYk4Rfv7jhtjlEBGRSEyec/Pee+9h3rx5OHPmzEN/uIODA3r16oX4+HjDNr1ej/j4ePTt27fafYqLi6s8w0omkwEA51o0cUone7wwsC0A4KO4iyjXVQ27RERk+0y+Q3FUVBSKi4sREhICBwcHODo6Gr1/65Zp9xqJjo7G5MmTERYWhj59+mDlypXQaDSYMmWK4fP8/PwQExMDABg9ejRWrFiB0NBQhIeHIzU1FW+88QZGjx5tCDnUdE3pH4ivjlzG5ZvF+CHpGp7u01rskoiIqIGZHG5Wrlxp1gLGjx+PvLw8LF68GNnZ2ejRowf27NljmGScmZlpNFKzaNEiSCQSLFq0CNevX4eHhwdGjx6Nf//732ati6xTM7kdXnysHd7ZdR6fxF/C2J5+kNsx9BIRNSV8KjjZnNJyHQZ9eAA5ai2WjO6MKf0DxS6JiIgekim/v02ecwMAaWlpWLRoESZMmGC4k/D//vc/nD17tj6HIzIrhb0MLw1pDwBYfSAVxWUVIldEREQNyeRwc+jQIXTr1g2//vortm/fjqKiIgDA77//jiVLlpi9QKL6GBfmD383R+QXlWHj0ctil0NERA3I5HAzf/58vPPOO4iLi4ODg4Nh+5AhQ3Ds2DGzFkdUXw52UsweGgwAWH8oHaqScpErIiKihmJyuDl9+jTGjh1bZbunpyfy8/lcH2o8xoT6IcizOVQl5dhwOEPscoiIqIGYHG5cXV2RlZVVZfupU6fg5+dnlqKIzEEmlSB6WOXozYZf0nFLUyZyRURE1BBMDjdPP/00XnvtNWRnZ0MikUCv1+PIkSOYO3cuoqKiLFEjUb093sUbXXxdoCnTYd2hNLHLISKiBlCvp4J37NgR/v7+KCoqQufOnTFw4ED069cPixYtskSNRPUmlUowd3gHAMCmo5eRoy4VuSIiIrK0et/nJjMzE2fOnEFRURFCQ0PRvn17c9dmEbzPTdMjCAL+ti4RSVdu45lH2uDtMV3FLomIiExk0fvcHD58GADQunVrjBw5EuPGjbOaYENNk0QiwZzhlXNvNv+Wiau3ikWuiIiILMnkcDNkyBAEBgZi4cKFOHfunCVqIjK7fu3c0T+oJcp1Aj6OvyR2OUREZEEmh5sbN25gzpw5OHToELp27YoePXrgww8/xLVr1yxRH5HZ3J17s/3kNaTmFolcDRERWYrJ4cbd3R0zZ87EkSNHkJaWhr///e/YtGkTAgICMGTIEEvUSGQWoa1bIKKTJ/QCsHLfRbHLISIiC6nXs6XuCgwMxPz58/Hee++hW7duOHTokLnqIrKI6GGVozc//5GFczfUIldDRESWUO9wc+TIEUyfPh0+Pj6YOHEiunbtil27dpmzNqKHtnr1agQEBEChUCA8PBxF1y7gL919AAAr4lKqtC8oKMCMGTPg4+MDuVyO4OBg7N692/D+m2++CYlEYvTq2LFjg/WHiIgezM7UHRYsWIDNmzfjxo0bGDZsGD7++GM8+eSTcHJyskR9RPW2ZcsWREdHY926dQgPD8fKlSsRGRmJ2CMnsft0Fvadz8XJzNvo2boFAKCsrAzDhg2Dp6cnvv/+e/j5+eHKlStwdXU1Om6XLl2wb98+w892diZ/jYiIyIJM/q9yQkIC5s2bh3HjxsHd3d0SNRGZxYoVK/Cvf/0LU6ZMAQCsW7cOu3btwv6ftuCvPUdgW9I1LN+bgv8+9wgA4Msvv8StW7dw9OhR2NvbAwACAgKqHNfOzg7e3t4N1g8iIjKNyZel7l6OYrChxqysrAxJSUmIiIgwbJNKpYiIiEBiYiJeHtoe9jIJjqTexNG0yge+/vTTT+jbty9mzJgBLy8vdO3aFe+++y50Op3RsS9dugRfX1+0bdsWkyZNQmZmZoP2jYiIalfv8fRz584hMzMTZWXGDyN84oknHrooooeVn58PnU4HLy8vo+1eXl64cOEC/N2c8HTv1vjPsStYFpuCH15sifT0dOzfvx+TJk3C7t27kZqaiunTp6O8vBxLliwBAISHh2Pjxo3o0KEDsrKysHTpUjz66KM4c+YMnJ2dxegqERHdx+Rwk56ejrFjx+L06dOQSCS4+/QGiUQCAFX+yiVqrGYOCcLWE1dxMrMAB1Jyodfr4enpic8++wwymQy9evXC9evX8eGHHxrCzYgRIwz7d+/eHeHh4WjTpg22bt2KZ599VqyuEBHRPUy+LDVr1iwEBgYiNzcXTk5OOHv2LBISEhAWFoaDBw9aoEQi07m7u0MmkyEnJ8doe05OjmG+jJeLApP7BQAAlsVehI+PD4KDgyGTyQztO3XqhOzs7CojlHe5uroiODgYqamplukIERGZzORwk5iYiLfeegvu7u6QSqWQSqUYMGAAYmJi8PLLL1uiRiKTOTg4oFevXoiPjzds0+v1iI+PR9++fQ3bpg1qh+ZyO5zLUsO7fQhSU1Oh1+sN71+8WBl6HBwcqv2coqIipKWlwcfHx3KdISIik5gcbnQ6nWFugbu7O27cuAEAaNOmDVJSqt43hEgs0dHR+Pzzz7Fp0yacP38eL774IjQajWH1VFRUFD58ZwmmDggEAFz3eRS3bt3CrFmzcPHiRezatQvvvvsuZsyYYTjm3LlzcejQIVy+fBlHjx7F2LFjIZPJMGHCBFH6SEREVZk856Zr1674/fffERgYiPDwcHzwwQdwcHDAZ599hrZt21qiRqJ6GT9+PPLy8rB48WJkZ2ejR48e2LNnj2GScWZmJqRSKRY8GohNRy/jWokTZr7/JX5c/x4+//xz+Pn5YdasWXjttdcMx7x27RomTJiAmzdvwsPDAwMGDMCxY8fg4eEhVjeJiOg+EuHujOA6io2NhUajwVNPPYXU1FT85S9/wcWLF9GyZUts2bKl0T9fSq1WQ6lUQqVSwcXFRexyqJFYezAN7++5YPhZKgFinuqG8b1bi1gVERHdZcrvb5PDTXVu3bqFFi1aGFZMNWYMN1Sd9LwiDFlu/Gw0mQQ4PH8IfJSOIlVFRER3mfL7+6EenHmXm5ubVQQboppkq0urbNMJwLu7L+BU5m3o9Q/9NwARETUQPhSHCECgezNIJcD9Geb/fr+B//v9Brxc5BjW2QuRXbzxSNuWsJeZ5e8CIiKyALNclrImvCxFNdnyWyYWbj8DnSBAKgHG924NdWk5Dl7Ihabsz5tTuijsMLSTFyK7eGFgsAecHPg3AhGRpTX4nBtrwnBDtclSleByfjEC3J0Mc21Ky3VITLuJ2LPZiDuXg5uaP2/oJ7eTYmCwByK7eGNoR0+0aFb9/XCIiOjhMNzUguGGHoZOLyDpym3Ens1G7NlsXLtdYnhPJpWgT4AbIrt4YXgXb/i6ciIyEZG5MNzUguGGzEUQBJzLUmPv2RzEns3GhexCo/e7t1Iisos3Irt4IciTD9UkInoYDDe1YLghS7lyU2MIOkmZt3HvN6utRzMM71wZdEJauUIq5epCIiJTNPhS8Ie1evVqBAQEQKFQIDw8HMePH6+1fUFBAWbMmAEfHx/I5XIEBwdj9+7dDVQtUfXatGyGfw1si+9f7IdfFw7Fu2O7YVCwB+xlEqTnabDuUBrGrjmKfu/txxs7z+DwpXyU6/QPPjAREZlE9JGbLVu2ICoqCuvWrUN4eDhWrlyJbdu2ISUlBZ6enlXal5WVoX///vD09MTChQvh5+eHK1euwNXVFSEhIQ/8PI7cUEMrLC3HgZQ8xJ7NrrLySuloj6EdPTGcK6+IiGplVZelwsPD0bt3b6xatQpA5ZOb/f398dJLL2H+/PlV2q9btw4ffvghLly4AHt7e5M/j+GGxFRarsPRtHzEnsnBvvPGK68U9lI82r5y5VVEJ0+4OnHlFRHRXVYTbsrKyuDk5ITvv/8eY8aMMWyfPHkyCgoK8OOPP1bZZ+TIkXBzc4OTkxN+/PFHeHh4YOLEiXjttdcgk8mqtNdqtdBqtYaf1Wo1/P39GW5IdA9aeRUe6IbILt4Y3sWLj4AgoibPlHAj6hh4fn4+dDqd4SnNd3l5eeHChQvV7pOeno79+/dj0qRJ2L17N1JTUzF9+nSUl5djyZIlVdrHxMRg6dKlFqmf6GHIpBL0CXRDn0A3LBrVCeey1Ig9m4O9d1ZeHU27iaNpN7Hkp7NceUVEZAJRR25u3LgBPz8/HD16FH379jVsf/XVV3Ho0CH8+uuvVfYJDg5GaWkpMjIyDCM1K1aswIcffoisrKwq7TlyQ9boyk3NnRGdHJysZuVVZdDxRnc/JVdeEVGTYDUjN+7u7pDJZMjJyTHanpOTA29v72r38fHxgb29vdElqE6dOiE7OxtlZWVwcDCepyCXyyGXy81fPJEFtWnZDM8PbIfnB7ZDbmEp9p3LRezZbBxNy0d6ngZrD6Zh7cE0eLsoMLyLF4Z39kZ4Wzc+84qICCKHGwcHB/Tq1Qvx8fGGOTd6vR7x8fGYOXNmtfv0798f3377LfR6PaTSyv+QX7x4ET4+PlWCDZEt8HRWYGJ4a0wMr3zW1YELudh7NgcHU3KRrS7F14lX8HXilXtWXnljULAHHB2qzkEjImoKRF8ttWXLFkyePBnr169Hnz59sHLlSmzduhUXLlyAl5cXoqKi4Ofnh5iYGADA1atX0aVLF0yePBkvvfQSLl26hKlTp+Lll1/G66+//sDP42opshUPWnk18M7Kq6FceUVENsBqLksBwPjx45GXl4fFixcjOzsbPXr0wJ49ewyTjDMzMw0jNADg7++P2NhYvPLKK+jevTv8/Pwwa9YsvPbaa2J1gUgUCnsZhnT0wpCOXtDpBZy4fAuxd+6QfL2gBHvP5WDvuRzIpBI80tYNwztz5RURNQ2ij9w0NI7ckK0TBAFnb6ix92w29p7LqfLMq5BWSgy/MyE5yLO5SFUSEZnGau5zIwaGG2pqLudrsPdc9Suv2t278qqVEhIJV14RUePEcFMLhhtqynILSxF3LgexZ3OQmJaPct2fX/+7K68iu3ijTyBXXhFR48JwUwuGG6JK9668OpCSi+L7n3nVyRORXbwxsD1XXhGR+BhuasFwQ1RVabkOR1LzEXs2G/vO5+LWfSuvBgV7YHhnrrwiIvEw3NSC4YaodhU6PU5cuY2996y8uuvuyqvILt4Y3tkb3kqFiJUSUVPCcFMLhhuiurt35VXs2Ryk5Ny38srfFZF37pDMlVdEZEmm/P7mjEEiqpFEIkFXPyWih3dA7CsDcXDuY1gwoiN6tWkBiQT4/WoBPtiTgogVhzB0+UF8sOcCfr9aAHP+zbR69WoEBARAoVAgPDwcx48fr9N+mzdvhkQiMdz9/K6ioiLMnDkTrVq1gqOjIzp37ox169aZrV4iEh9HboioXnLVpYg7X/3KKx+lAsM7/7nyyq6eK6+2bNmCqKgorFu3DuHh4Vi5ciW2bduGlJQUeHp61rjf5cuXMWDAALRt2xZubm7YuXOn4b3nn38e+/fvxxdffIGAgADs3bsX06dPx/bt2/HEE0/Uq04isjxelqoFww2R+d1deRV7NhsHU/KMVl65OtljSMf6rbwKDw9H7969sWrVKgCVz57z9/fHSy+9hPnz51e7j06nw8CBAzF16lT88ssvKCgoMAo3Xbt2xfjx4/HGG28YtvXq1QsjRozAO++8Y2LPiaih8LIUETUoF4U9nuzhhzWTeuHkG8OwYXIYxoW1glszBxQUl2P7yet44T9J6Pl2HF74zwlsP3kNquLyWo9ZVlaGpKQkREREGLZJpVJEREQgMTGxxv3eeusteHp64tlnn632/X79+uGnn37C9evXIQgCDhw4gIsXL2L48OH16zwRNTqiP1uKiGyLwl6GoZ28MLSTl2HlVezZbOw9m4PrBSV3nn+VAzupBI+0bYnILl4YVs3Kq/z8fOh0OsNz5u7y8vLChQsXqv3sw4cPY8OGDUhOTq6xvk8//RTPP/88WrVqBTs7O0ilUnz++ecYOHDgQ/ediBoHhhsishg7mRSPtG2JR9q2xOK/dK6y8upwaj4Op+bjjR/PGlZeRXbxRjsP01deFRYW4plnnsHnn38Od3f3Gtt9+umnOHbsGH766Se0adMGCQkJmDFjBnx9fY1GiYjIenHODRGJIiNfcyfoZONkZoHRe0GezTE0uAXeGNMT27Ztw9ixYw3vTZ48GQUFBfjxxx+N9klOTkZoaChksj/n9Oj1egCVl7NSUlLg6+sLpVKJHTt2YNSoUYZ2zz33HK5du4Y9e/ZYoKdEZA6m/P7myA0RiSLQvRleGNQOLwxqh1x1Kfaeq7xpYGLaTaTmFiE1twgyz3aY/sEm/C4LxvDOXghr44r4+HjMnDmzyvE6duyI06dPG21btGgRCgsL8fHHH8Pf3x+lpaUoLy+HVGo83VAmkxmCEBFZP4YbIhKdp4sC/3ikDf7xSBuoSspxMKVy5dVPfZ/C9R+XY9X6tvjcJxilyf8HTYEagX1HobRch+efnQI/Pz/ExMRAoVCga9euRsd1dXUFAMN2BwcHDBo0CPPmzYOjoyPatGmDQ4cO4euvv8aKFSsauttEZCEMN0TUqCgdK1dePdnDDyvG9cDcN53x9WercOt2Puw9A+H21Jt47X+ZeHPfdah+O4uOBSVQFZdD6WSPLFUJMvI1CHRvBh+lY7XH37x5MxYsWIBJkybh1q1baNOmDf79739j2rRpDdtRIrIYzrkhIqtQ3cqru+ykEgS4N0NabhEEAFIJEPNUN4zv3Vq8gonIrHgTv1ow3BBZv7vPvIq9MyH5Yk5Rte1GdPVGVz8lAt2bIdC9GQJaNjPpJoJE1Hgw3NSC4YbI9mw/eQ3RW3+vU1tfpQKBHs3uBJ7maHsn+LRq4Vjvx0QQkeVxtRQRNSl927WEVALo7/lTTSoBnh/YFnmFZUjPL0J6ngaqknLcUJXihqoUR1JvGh3DXiaBv5sT2ro3Q1uP5obRnrbuzeDhLIdEImngXhFRfTHcEJHV81E6Iuapbli4/Qx0ggCZRIJ3n+paZc7NbU0Z0vM1yMjXICO/CBn5GqTnaXD5pgal5Xqk51X+jPO5Rvs1c5DdGe1pbgg8ge7NEOjRDC4K+4bsKhHVAS9LEZHNyFKV4HJ+MQLcnWpcLVUdvV5Atrq0Muzka5CRVxl+0vM1uHqr2GhE6H7uzR3uBJ7mhstdbd2boXVLJ8jtOL+HyFw456YWDDdEZIqyCj0ybxVXGe3JyNcgt1Bb435SCeDXwtFoXs/dl6+rI2RSXuYiMgXDTS0YbojIXIq0Fbh8z2hP+p3wk5GnQaG2osb9HOykCGjpVDnK42F8qcutmYPNze9ZvXo1PvzwQ2RnZyMkJASffvop+vTp88D9Nm/ejAkTJuDJJ5/Ezp07DdsFQcCSJUvw+eefo6CgAP3798fatWvRvn17C/aCxMZwUwuGGyKyNEEQkF9UZhjt+fNSlwZXbhajTFfzox5cFHYI9Kg62hPo3gzN5NY3TXLLli2IiorCunXrEB4ejpUrV2Lbtm1ISUmBp6dnjftdvnwZAwYMQNu2beHm5mYUbt5//33ExMRg06ZNCAwMxBtvvIHTp0/j3LlzUCgUNR6TrBvDTS0YbohITDq9gBsFJXcCT5Fhnk96ngY3VCWo7b/IXi5yw2jPveHH380J9o10GXt4eDh69+6NVatWAah8mKm/vz9eeuklzJ8/v9p9dDodBg4ciKlTp+KXX35BQUGBIdwIggBfX1/MmTMHc+fOBQCoVCp4eXlh48aNePrppxukX9TwuBSciKiRkkkrl5z7uzlhULCH0Xul5TpcuVlcZbQnI1+Dm5oy5Ki1yFFrcSz9VpVjtnZzMhrlaXtnNZe3i0K0y1xlZWVISkrCggULDNukUikiIiKQmJhY435vvfUWPD098eyzz+KXX34xei8jIwPZ2dmIiIgwbFMqlQgPD0diYiLDDQFguCEiajQU9jJ08HZGB2/nKu+pisuRcfPOpOY8DdLuCT8l5TpDCLqfo70MAXfCTluPe8NPcyidLLuMPT8/HzqdDl5eXkbbvby8cOHChWr3OXz4MDZs2IDk5ORq38/OzjYc4/5j3n2PiOGGiMgKKJ3s0cPJFT38XY22C4KAHLXWaDLz3aCTeasYJeU6nM9S43yWusox3Zo5VDvaE9CyGRT2Db+MvbCwEM888ww+//xzuLu7N/jnk+1guCEismISiQTeSgW8lQr0a2ccCMp1ely7XVJ5metO6Ln7v9nqUtzSlOGWpgxJV27fd0zAV+n4Z+jx+HO0x69F3Zexu7u7QyaTIScnx2h7Tk4OvL29q7RPS0vD5cuXMXr0aMM2vb5y8rWdnR1SUlIM++Xk5MDHx8fomD169KhTXWT7GG6IiGyUvUxqCChDOhq/p9FW4PJNjdFoT+XE5iKoSytwvaAE1wtKcDg132g/B5kUre8uY793NZdHM3g0N35MhYODA3r16oX4+HiMGTMGQGVYiY+Px8yZM6vU27FjR5w+fdpo26JFi1BYWIiPP/4Y/v7+sLe3h7e3N+Lj4w1hRq1W49dff8WLL7748P/SyCYw3BARNUHN5Hbo4qtEF1+l0XZBEHC7uLza0Z6MmxqUVeiRmluE1NyqT2JvLrerMtrzt8nT8Eb0iwgLC0OfPn2wcuVKaDQaTJkyBQAQFRUFPz8/xMTEQKFQoGvXrkbHdHV1BQCj7bNnz8Y777yD9u3bG5aC+/r6GgIUUaMIN+a+wRMREdWPRCKBWzMHuDVzQ682bkbv6fUCbqhKDHN60u+Z33PtdjGKtBU4fV2F09dV9+zlCadH/4nnZ72KCs1teAV0wAvvfoHf8wW0lRTi8pUrkEr/XMaedef4ge7NanyExquvvgqNRoPnn38eBQUFGDBgAPbs2cN73JCB6Pe5scQNnmrD+9wQEZmftkKHq7eKq4z2pOdrkF9U+2MqWrWovMxVrtMjMe0mBFTO+1n8l874Z78Am7tjM9WPVd3Ez9w3eLqfVquFVvvnF0utVsPf35/hhoiogahLy3G5mtGejHwNimp5TAUAOMgk8FY6wttFAS+lAt4ucni5KODlUjmJ2ttFAQ9nuSiru5oKU66ubN++He+++y5SU1NRXl6O9u3bY86cOXjmmWcMbf75z39i06ZNRvtFRkZiz549tdZhNTfxs8QNnu4XExODpUuXmq1mIiIyjYvCHt1buaJ7K1ej7YIgIK9Ii4w8Dfadz8Hnv2RU2bdMJyDzVjEybxXX+hktnOwNgcfL+W4QUsBbKYenc+V2NycHSPnAUpNs2bIF0dHRRldXIiMja7y64ubmhtdffx0dO3aEg4MDfv75Z0yZMgWenp6IjIw0tHv88cfx1VdfGX6Wy+VmrVvUcGOJGzzdb8GCBYiOjjb8fHfkhoiIxCWRSODprICnswKtWzphw+EM6O+5liCVAFtf6AsBQLaqFDnq0sr/LdQiR1WKbHXlq6xCj9vF5bhdXI4L2YU1fp69rPLzvFzklSHIpTIA3T8S5OjAUaC7VqxYgX/961+GCeDr1q3Drl278OWXX1Z7deWxxx4z+nnWrFnYtGkTDh8+bBRu5HJ5tbcDMJdGMaG4rupzgye5XG72REhEROblo3REzFPdsHD7GegEATKJBO8+1RVhAW617icIAlQl5ZVB504AylFrka0uNQSgHLUWNzValOsEwxL32jgr7O6M+twNPnJDCLq7zb25vM73+7FW9b26cpcgCNi/fz9SUlLw/vvvG7138OBBeHp6okWLFhgyZAjeeecdtGzZ0my1ixpuLHGDp3bt2lm2aCIisojxvVtjYLAHLucXI8DdqcbVUveSSCRwdXKAq5MDOnrXPA+jXKdHbqEW2apS5Kr/HPXJUZXeeWZX5c/FZToUllagsLQIl6pZ7n6XTCqBR3M5vJQKeDlXHQnyVlbODXJWWPYRF5ZUn6srQOWDTP38/KDVaiGTybBmzRoMGzbM8P7jjz+Op556CoGBgUhLS8PChQsxYsQIJCYmQiYzz6iZqOHGEjd4IiIi6+WjdKxTqDGVvUwKP1dH+LnWfGxBEFCoragMP6o7oz/qey6H3QlAeYVa6PSCISDVppmDzOiyl9FI0D0TohvrU93rw9nZGcnJySgqKkJ8fDyio6PRtm1bwyWrex9u2q1bN3Tv3h3t2rXDwYMHMXToULPUIPplqejoaEyePNmsN3giIiIylUQigYvCHi4KewR5Vn146V06vYD8ospRoGz1PSNBKu2fYUhdisLSCmjKdJV3fq7moaZ/fi7Qspm8crTn3snQLgp43pkf5O2igNLRvkGXxZt6deUuqVSKoKAgAECPHj1w/vx5xMTEVJmPc1fbtm3h7u6O1NRU2wk348ePR15eHhYvXozs7Gz06NEDe/bsMQyDZWZmGt3giYiISEwyqcQwGhNSS7visoo7Iz5/XvbKVpUit7DUaHvFnbCUX6TFGVR9wOldcjvpn5e+GmBZvKlXV2qi1+uNbslyv2vXruHmzZtGzwp7WKLf56ah8SZ+RETUWOj1Am4Vlxld9spRa++ZDF35ul1cXudjmnNZ/JYtWzB58mSsX7/ecHVl69atuHDhAry8vIyurgCVt18JCwtDu3btoNVqsXv3bsyfPx9r167Fc889h6KiIixduhR//etf4e3tjbS0NLz66qsoLCzE6dOna10AZDX3uSEiImrKpFIJ3JvL4d5cjq5+yhrblZbrkKuuOg+o8rKY1mLL4k29uqLRaDB9+nRcu3YNjo6O6NixI7755huMHz8eACCTyfDHH39g06ZNKCgogK+vL4YPH463337brCubOXJDRERkAwRBQEFxOXIK75kArdIip7DUaCQov6iszsesz7L4ujwfrD44ckNERNTESCQStGjmgBbNal8WX1ahR96dCdF/3hjR+L5A2apSlJSbviy+vEKP81lqCKi8CWPMU90wvndrC/S2dgw3RERETYiDXd2Xxd8beIwvhT14WbxeABZuP4OBwR4WWd5fG4YbIiIiMnLvsvj2XjUvi6/Q6ZFfVIYcdSkOXczDiriLRu/rBAGX84sbPNxwjTURERHVi51MCm+lAiH+rvh7WCvcvwBLJpEgwN2pwetiuCEiIqKHdvf5YLI7Nxq8+3ywhh61AXhZioiIiMykPs8HswSGGyIiIjIbSz0fzBS8LEVEREQ2heGGiIiIbArDDREREdkUhhsiIiKyKQw3REREZFMYboiIiMimMNwQERGRTWG4ISIiIpvCcENEREQ2heGGiIiIbArDDREREdkUhhsiIiKyKQw3REREZFMYboiIiMimMNwQERGRTWG4ISIiIpvCcENEREQ2heGGiIiIbArDDREREdkUhhsiIiKyKQw3REREZFMYboiIiMimNIpws3r1agQEBEChUCA8PBzHjx+vse3nn3+ORx99FC1atECLFi0QERFRa3siIiJqWkQPN1u2bEF0dDSWLFmCkydPIiQkBJGRkcjNza22/cGDBzFhwgQcOHAAiYmJ8Pf3x/Dhw3H9+vUGrpyIiIgaI4kgCIKYBYSHh6N3795YtWoVAECv18Pf3x8vvfQS5s+f/8D9dTodWrRogVWrViEqKuqB7dVqNZRKJVQqFVxcXB66fiIiIrI8U35/izpyU1ZWhqSkJERERBi2SaVSREREIDExsU7HKC4uRnl5Odzc3Kp9X6vVQq1WG72IiIjIdokabvLz86HT6eDl5WW03cvLC9nZ2XU6xmuvvQZfX1+jgHSvmJgYKJVKw8vf3/+h6yYiIqLGS/Q5Nw/jvffew+bNm7Fjxw4oFIpq2yxYsAAqlcrwunr1agNXSURERA3JTswPd3d3h0wmQ05OjtH2nJwceHt717rvsmXL8N5772Hfvn3o3r17je3kcjnkcrlZ6iUiIqLGT9SRGwcHB/Tq1Qvx8fGGbXq9HvHx8ejbt2+N+33wwQd4++23sWfPHoSFhTVEqURERGQlRB25AYDo6GhMnjwZYWFh6NOnD1auXAmNRoMpU6YAAKKiouDn54eYmBgAwPvvv4/Fixfj22+/RUBAgGFuTvPmzdG8eXPR+kFERESNg+jhZvz48cjLy8PixYuRnZ2NHj16YM+ePYZJxpmZmZBK/xxgWrt2LcrKyvC3v/3N6DhLlizBm2++2ZClExERUSMk+n1uGhrvc0NERGR9rOY+N0RERETmxnBDRERENoXhhoiIiGwKww0RERHZFIYbIiIisikMN0RERGRTGG6IiIjIpjDcEBERkU1huCEiIiKbwnBDRERENoXhhoiIiGwKww0RERHZFIYbIiIisikMN0RERGRTGG6IiIjIpjDcEBERkU1huCEiIiKbwnBDRERENoXhhoiIiGwKww0RERHZFIYbIiIisikMN0RERGRTGG6IiIjIpjDcEBERkU1huCEiIiKbwnBDRERENoXhhoiIiGwKww0RERHZFIYbIiIisikMN0RERGRTGG6IiIjIpjDcEBERkU1pFOFm9erVCAgIgEKhQHh4OI4fP15r+23btqFjx45QKBTo1q0bdu/e3UCVEhERUWMnerjZsmULoqOjsWTJEpw8eRIhISGIjIxEbm5ute2PHj2KCRMm4Nlnn8WpU6cwZswYjBkzBmfOnGngyomIiKgxkgiCIIhZQHh4OHr37o1Vq1YBAPR6Pfz9/fHSSy9h/vz5VdqPHz8eGo0GP//8s2HbI488gh49emDdunUP/Dy1Wg2lUgmVSgUXFxfzdYSIiIgsxpTf33YNVFO1ysrKkJSUhAULFhi2SaVSREREIDExsdp9EhMTER0dbbQtMjISO3furLa9VquFVqs1/KxSqQBU/ksiIiIi63D393ZdxmREDTf5+fnQ6XTw8vIy2u7l5YULFy5Uu092dna17bOzs6ttHxMTg6VLl1bZ7u/vX8+qiYiISCyFhYVQKpW1thE13DSEBQsWGI306PV63Lp1Cy1btoREIjHrZ6nVavj7++Pq1as2ecnL1vsH2H4f2T/rZ+t9ZP+sn6X6KAgCCgsL4evr+8C2ooYbd3d3yGQy5OTkGG3PycmBt7d3tft4e3ub1F4ul0Mulxttc3V1rX/RdeDi4mKz/6cFbL9/gO33kf2zfrbeR/bP+lmijw8asblL1NVSDg4O6NWrF+Lj4w3b9Ho94uPj0bdv32r36du3r1F7AIiLi6uxPRERETUtol+Wio6OxuTJkxEWFoY+ffpg5cqV0Gg0mDJlCgAgKioKfn5+iImJAQDMmjULgwYNwvLlyzFq1Chs3rwZJ06cwGeffSZmN4iIiKiRED3cjB8/Hnl5eVi8eDGys7PRo0cP7NmzxzBpODMzE1LpnwNM/fr1w7fffotFixZh4cKFaN++PXbu3ImuXbuK1QUDuVyOJUuWVLkMZitsvX+A7feR/bN+tt5H9s/6NYY+in6fGyIiIiJzEv0OxURERETmxHBDRERENoXhhoiIiGwKww0RERHZFIYbE61evRoBAQFQKBQIDw/H8ePHa22/bds2dOzYEQqFAt26dcPu3bsbqNL6MaV/GzduhEQiMXopFIoGrNY0CQkJGD16NHx9fSGRSGp8Htm9Dh48iJ49e0IulyMoKAgbN260eJ0Pw9Q+Hjx4sMo5lEgkNT7OREwxMTHo3bs3nJ2d4enpiTFjxiAlJeWB+1nTd7A+fbSm7+HatWvRvXt3w83d+vbti//973+17mNN5w8wvY/WdP6q895770EikWD27Nm1tmvo88hwY4ItW7YgOjoaS5YswcmTJxESEoLIyEjk5uZW2/7o0aOYMGECnn32WZw6dQpjxozBmDFjcObMmQauvG5M7R9QeQfKrKwsw+vKlSsNWLFpNBoNQkJCsHr16jq1z8jIwKhRozB48GAkJydj9uzZeO655xAbG2vhSuvP1D7elZKSYnQePT09LVRh/R06dAgzZszAsWPHEBcXh/LycgwfPhwajabGfaztO1ifPgLW8z1s1aoV3nvvPSQlJeHEiRMYMmQInnzySZw9e7ba9tZ2/gDT+whYz/m732+//Yb169eje/futbYT5TwKVGd9+vQRZsyYYfhZp9MJvr6+QkxMTLXtx40bJ4waNcpoW3h4uPDCCy9YtM76MrV/X331laBUKhuoOvMCIOzYsaPWNq+++qrQpUsXo23jx48XIiMjLViZ+dSljwcOHBAACLdv326QmswpNzdXACAcOnSoxjbW9h28X136aM3fQ0EQhBYtWghffPFFte9Z+/m7q7Y+Wuv5KywsFNq3by/ExcUJgwYNEmbNmlVjWzHOI0du6qisrAxJSUmIiIgwbJNKpYiIiEBiYmK1+yQmJhq1B4DIyMga24upPv0DgKKiIrRp0wb+/v4P/OvE2ljT+XtYPXr0gI+PD4YNG4YjR46IXU6dqFQqAICbm1uNbaz9HNalj4B1fg91Oh02b94MjUZT4+NzrP381aWPgHWevxkzZmDUqFFVzk91xDiPDDd1lJ+fD51OZ7hz8l1eXl41zk/Izs42qb2Y6tO/Dh064Msvv8SPP/6Ib775Bnq9Hv369cO1a9caomSLq+n8qdVqlJSUiFSVefn4+GDdunX44Ycf8MMPP8Df3x+PPfYYTp48KXZptdLr9Zg9ezb69+9f693Jrek7eL+69tHavoenT59G8+bNIZfLMW3aNOzYsQOdO3eutq21nj9T+mht5w8ANm/ejJMnTxoei/QgYpxH0R+/QNarb9++Rn+N9OvXD506dcL69evx9ttvi1gZ1VWHDh3QoUMHw8/9+vVDWloaPvroI/znP/8RsbLazZgxA2fOnMHhw4fFLsVi6tpHa/sedujQAcnJyVCpVPj+++8xefJkHDp0qMZf/tbIlD5a2/m7evUqZs2ahbi4uEY98Znhpo7c3d0hk8mQk5NjtD0nJwfe3t7V7uPt7W1SezHVp3/3s7e3R2hoKFJTUy1RYoOr6fy5uLjA0dFRpKosr0+fPo06NMycORM///wzEhIS0KpVq1rbWtN38F6m9PF+jf176ODggKCgIABAr1698Ntvv+Hjjz/G+vXrq7S11vNnSh/v19jPX1JSEnJzc9GzZ0/DNp1Oh4SEBKxatQparRYymcxoHzHOIy9L1ZGDgwN69eqF+Ph4wza9Xo/4+Pgar6X27dvXqD0AxMXF1XrtVSz16d/9dDodTp8+DR8fH0uV2aCs6fyZU3JycqM8h4IgYObMmdixYwf279+PwMDAB+5jbeewPn28n7V9D/V6PbRabbXvWdv5q0ltfbxfYz9/Q4cOxenTp5GcnGx4hYWFYdKkSUhOTq4SbACRzqPFpirboM2bNwtyuVzYuHGjcO7cOeH5558XXF1dhezsbEEQBOGZZ54R5s+fb2h/5MgRwc7OTli2bJlw/vx5YcmSJYK9vb1w+vRpsbpQK1P7t3TpUiE2NlZIS0sTkpKShKefflpQKBTC2bNnxepCrQoLC4VTp04Jp06dEgAIK1asEE6dOiVcuXJFEARBmD9/vvDMM88Y2qenpwtOTk7CvHnzhPPnzwurV68WZDKZsGfPHrG68ECm9vGjjz4Sdu7cKVy6dEk4ffq0MGvWLEEqlQr79u0Tqws1evHFFwWlUikcPHhQyMrKMryKi4sNbaz9O1ifPlrT93D+/PnCoUOHhIyMDOGPP/4Q5s+fL0gkEmHv3r2CIFj/+RME0/toTeevJvevlmoM55HhxkSffvqp0Lp1a8HBwUHo06ePcOzYMcN7gwYNEiZPnmzUfuvWrUJwcLDg4OAgdOnSRdi1a1cDV2waU/o3e/ZsQ1svLy9h5MiRwsmTJ0Woum7uLnu+/3W3T5MnTxYGDRpUZZ8ePXoIDg4OQtu2bYWvvvqqwes2hal9fP/994V27doJCoVCcHNzEx577DFh//794hT/ANX1C4DRObH272B9+mhN38OpU6cKbdq0ERwcHAQPDw9h6NChhl/6gmD9508QTO+jNZ2/mtwfbhrDeZQIgiBYblyIiIiIqGFxzg0RERHZFIYbIiIisikMN0RERGRTGG6IiIjIpjDcEBERkU1huCEiIiKbwnBDRERENoXhhoiIiGwKww0RNXkHDx6ERCJBQUGB2KUQkRkw3BAREZFNYbghIiIim8JwQ0Si0+v1iImJQWBgIBwdHRESEoLvv/8ewJ+XjHbt2oXu3btDoVDgkUcewZkzZ4yO8cMPP6BLly6Qy+UICAjA8uXLjd7XarV47bXX4O/vD7lcjqCgIGzYsMGoTVJSEsLCwuDk5IR+/fohJSXFsh0nIotguCEi0cXExODrr7/GunXrcPbsWbzyyiv4xz/+gUOHDhnazJs3D8uXL8dvv/0GDw8PjB49GuXl5QAqQ8m4cePw9NNP4/Tp03jzzTfxxhtvYOPGjYb9o6Ki8N133+GTTz7B+fPnsX79ejRv3tyojtdffx3Lly/HiRMnYGdnh6lTpzZI/4nIvPhUcCISlVarhZubG/bt24e+ffsatj/33HMoLi7G888/j8GDB2Pz5s0YP348AODWrVto1aoVNm7ciHHjxmHSpEnIy8vD3r17Dfu/+uqr2LVrF86ePYuLFy+iQ4cOiIuLQ0RERJUaDh48iMGDB2Pfvn0YOnQoAGD37t0YNWoUSkpKoFAoLPxvgYjMiSM3RCSq1NRUFBcXY9iwYWjevLnh9fXXXyMtLc3Q7t7g4+bmhg4dOuD8+fMAgPPnz6N///5Gx+3fvz8uXboEnU6H5ORkyGQyDBo0qNZaunfvbvhnHx8fAEBubu5D95GIGpad2AUQUdNWVFQEANi1axf8/PyM3pPL5UYBp74cHR3r1M7e3t7wzxKJBEDlfCAisi4cuSEiUXXu3BlyuRyZmZkICgoyevn7+xvaHTt2zPDPt2/fxsWLF9GpUycAQKdOnXDkyBGj4x45cgTBwcGQyWTo1q0b9Hq90RweIrJdHLkhIlE5Oztj7ty5eOWVV6DX6zFgwACoVCocOXIELi4uaNOmDQDgrbfeQsuWLeHl5YXXX38d7u7uGDNmDABgzpw56N27N95++22MHz8eiYmJWLVqFdasWQMACAgIwOTJkzF16lR88sknCAkJwZUrV5Cbm4tx48aJ1XUishCGGyIS3dtvvw0PDw/ExMQgPT0drq6u6NmzJxYuXGi4LPTee+9h1qxZuHTpEnr06IH/+7//g4ODAwCgZ8+e2Lp1KxYvXoy3334bPj4+eOutt/DPf/7T8Blr167FwoULMX36dNy8eROtW7fGwoULxeguEVkYV0sRUaN2dyXT7du34erqKnY5RGQFOOeGiIiIbArDDREREdkUXpYiIiIim8KRGyIiIrIpDDdERERkUxhuiIiIyKYw3BAREZFNYbghIiIim8JwQ0RERDaF4YaIiIhsCsMNERER2ZT/B8tt/1iJ6d/9AAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "y_avg = []\n", + "for y in epochs_y:\n", + " y_avg.append(np.mean(y))\n", + "\n", + "plt.plot(np.arange(len(epochs_x)), y_avg, marker = '.')\n", + "plt.xlabel('epoch')\n", + "plt.ylabel('average loss')\n", + "plt.ylim(0,)\n", + "for i, txt in enumerate(y_avg):\n", + " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom')\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAG2CAYAAACZEEfAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABSpklEQVR4nO3deVxU5f4H8M8wwLAIo+yMgiAuCO4LBJmmoGhmbr9cMsUtraxcSsUKy8xI780y66rdXFBcslLb7lUBU69FigpuKCoiuLCoyAz7MnN+f5CTE4uMAjNz/Lxfr3m9mnOeOX4f53L5+JznPI9EEAQBRERERCJlZugCiIiIiBoTww4RERGJGsMOERERiRrDDhEREYkaww4RERGJGsMOERERiRrDDhEREYkaww4RERGJGsMOERERiRrDDhEREYmaQcPO4cOHMWzYMCgUCkgkEuzZs0fnvCAIWLx4Mdzd3WFtbY3Q0FBcunRJp01eXh4mTJgAe3t7NG/eHNOmTUNhYWET9oKIiIiMmUHDTlFREbp27Yovv/yyxvMrVqzA559/jrVr1+Lo0aOwtbVFWFgYSktLtW0mTJiAc+fOITY2Fj///DMOHz6MGTNmNFUXiIiIyMhJjGUjUIlEgt27d2PEiBEAqkZ1FAoF3nzzTbz11lsAAKVSCVdXV2zatAnjxo3D+fPn4efnh8TERPTq1QsAsHfvXjzzzDO4fv06FAqFobpDRERERsLc0AXUJj09HdnZ2QgNDdUek8vlCAwMREJCAsaNG4eEhAQ0b95cG3QAIDQ0FGZmZjh69ChGjhxZ47XLyspQVlamfa/RaJCXlwdHR0dIJJLG6xQRERE1GEEQUFBQAIVCATOz2m9WGW3Yyc7OBgC4urrqHHd1ddWey87OhouLi855c3NzODg4aNvUJCoqCkuWLGngiomIiMgQrl27hlatWtV63mjDTmNatGgR5s2bp32vVCrh6emJa9euwd7e3oCVERERUX2pVCp4eHjAzs6uznZGG3bc3NwAADk5OXB3d9cez8nJQbdu3bRtcnNzdT5XWVmJvLw87edrIpPJIJPJqh23t7dn2CEiIjIxD5qCYrTr7Hh7e8PNzQ3x8fHaYyqVCkePHkVQUBAAICgoCPn5+Thx4oS2zYEDB6DRaBAYGNjkNRMREZHxMejITmFhIS5fvqx9n56ejuTkZDg4OMDT0xNz5szBhx9+iHbt2sHb2xuRkZFQKBTaJ7Y6duyIwYMH46WXXsLatWtRUVGB1157DePGjeOTWERERATAwGHn+PHj6N+/v/b9vXk04eHh2LRpExYsWICioiLMmDED+fn56NOnD/bu3QsrKyvtZ7Zu3YrXXnsNISEhMDMzw+jRo/H55583eV+IiIjIOBnNOjuGpFKpIJfLoVQqOWeHiIjIRNT397fRztkhIiIiaggMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkakYfdgoKCjBnzhy0bt0a1tbWCA4ORmJiovb85MmTIZFIdF6DBw82YMVERERkTMwNXcCDTJ8+HWfPnsWWLVugUCgQExOD0NBQpKSkoGXLlgCAwYMHY+PGjdrPyGQyQ5VLRERERsaoR3ZKSkrw/fffY8WKFejbty/atm2L999/H23btsWaNWu07WQyGdzc3LSvFi1aGLBqIiIiMiZGHXYqKyuhVqthZWWlc9za2hpHjhzRvj948CBcXFzQoUMHvPLKK7hz506d1y0rK4NKpdJ5ERERkTgZddixs7NDUFAQli5dips3b0KtViMmJgYJCQnIysoCUHULa/PmzYiPj8fy5ctx6NAhDBkyBGq1utbrRkVFQS6Xa18eHh5N1SUiIiJqYhJBEARDF1GXtLQ0TJ06FYcPH4ZUKkWPHj3Qvn17nDhxAufPn6/W/sqVK/Dx8UFcXBxCQkJqvGZZWRnKysq071UqFTw8PKBUKmFvb99ofSEiIqKGo1KpIJfLH/j726hHdgDAx8cHhw4dQmFhIa5du4Zjx46hoqICbdq0qbF9mzZt4OTkhMuXL9d6TZlMBnt7e50XERERiZPRh517bG1t4e7ujrt372Lfvn0YPnx4je2uX7+OO3fuwN3dvYkrJCIiImNk9I+e79u3D4IgoEOHDrh8+TLmz58PX19fTJkyBYWFhViyZAlGjx4NNzc3pKWlYcGCBWjbti3CwsIMXToREREZAaMf2VEqlZg1axZ8fX0xadIk9OnTB/v27YOFhQWkUilOnz6N5557Du3bt8e0adPQs2dP/O9//+NaO0RERATABCYoN4X6TnAiIiIi4yGaCcpEREREj4Jhh4iIiESNYYeIiIhEjWGHiIiIRI1hh4iIiESNYYeIiIhEjWGHiIiIRI1hh4iIiESNYYeIiIhEjWGHiIiIRI1hh4iIiESNYYeIiIhEjWGHiIiIRI1hh4iIiESNYYeIiIhEjWGHiIiIRI1hh4iIiGpUUFCAOXPmoHXr1rC2tkZwcDASExMBABUVFVi4cCE6d+4MW1tbKBQKTJo0CTdv3qzzmocPH8awYcOgUCggkUiwZ8+eam127dqFQYMGwdHRERKJBMnJyY/UD4YdIiIiqtH06dMRGxuLLVu24MyZMxg0aBBCQ0Nx48YNFBcX4+TJk4iMjMTJkyexa9cupKam4rnnnqvzmkVFRejatSu+/PLLOtv06dMHy5cvb5B+SARBEBrkSiZMpVJBLpdDqVTC3t7e0OUQEREZXElJCezs7PDDDz9g6NCh2uM9e/bEkCFD8OGHH1b7TGJiIgICApCRkQFPT88H/hkSiQS7d+/GiBEjajx/9epVeHt7IykpCd26dat2vr6/vzmyQ0RERNVUVlZCrVbDyspK57i1tTWOHDlS42eUSiUkEgmaN2/eBBXWH8MOERERVWNnZ4egoCAsXboUN2/ehFqtRkxMDBISEpCVlVWtfWlpKRYuXIjx48cb3V0Shh0iIiKq0ZYtWyAIAlq2bAmZTIbPP/8c48ePh5mZbnyoqKjAmDFjIAgC1qxZY6Bqa8ewQ0RERDXy8fHBoUOHUFhYiGvXruHYsWOoqKhAmzZttG3uBZ2MjAzExsYa3agOwLBDRERED2Brawt3d3fcvXsX+/btw/DhwwH8FXQuXbqEuLg4ODo6GrjSmpkbugAiIiIyTvv27YMgCOjQoQMuX76M+fPnw9fXF1OmTEFFRQX+7//+DydPnsTPP/8MtVqN7OxsAICDgwMsLS0BACEhIRg5ciRee+01AEBhYSEuX76s/TPS09ORnJwMBwcH7RNceXl5yMzM1K7Zk5qaCgBwc3ODm5ub3v3gyA4RERHVSKlUYtasWfD19cWkSZPQp08f7Nu3DxYWFrhx4wZ+/PFHXL9+Hd26dYO7u7v29fvvv2uvcfHSZSRdzESWsgQAcPz4cXTv3h3du3cHAMybNw/du3fH4sWLtZ/58ccf0b17d+0j7+PGjUP37t2xdu3ah+oH19kB19khIiJqDN8kZmLRrjPQCICZBIga1Rljez94/Z364jo7REREZDBnbygR8X1V0AEAjQC8veusdoSnKXHODhERET0SQRBw/W4JjqXnIfFqHo6l5+HK7aJq7dSCgKu3i+Eut27S+hh2iIiISC+CICDtViGOplcFm8T0PNxUlj7wc1KJBF5ONk1QoS6GHSIiIqpTpVqD81kFOHY1D8fS7yDx6l3kFZXrtDE3k6BTSzkCvR0Q4O2AXq0dsPdcFt7edRZqQYBUIsFHozo1+agOwLBDREREf1NWqcbp60oc+3Pk5kTGXRSWVeq0kZmboYdnC/T2dkCgtwO6ezaHjaVurBjb2xN92zvj6u1ieDnZGCToAAw7REREj73CskqczLiLxKt5OJqeh+Rr+Siv1Oi0sbMyR6/WLRDg7YgAbwd0bimHpfmDn3Nyl1sbLOTcw7BDRET0mLlbVK6dSJx4NQ9nb6qg1uiuROPUzBIB3g7o7VV1W8rXzR5SM4mBKn40DDtEREQil60s1c63OZaeh4s5hdXatGxurZ1v09vbAW2cbCGRmGa4+TuGHSIiIhERBAEZd4qr5tv8OXqTmVdcrV1bl2bo7VU136a3twNaNjfsrabGxLBDRERkwjQaARdzC3AsvWq+TWJ6HnILynTamEkAP4U9ArwcEeDdAr28HODUTGagipseww4REZEJqVBrcPaG8r45N3ehLKnQaWMpNUNXD7l2vk3P1i1gZ2VhoIoNj2GHiIjIiJVWqJGUma+dTHwi4y5KKtQ6bWwspejZugUCvKpuSXXzaA4rC6mBKjY+DDtERERGRFVagRMZd7Vr3Jy+no8Kte6TUs1tLNCrtYN2QrGfwh4WUm53WRuGHSIiIgO6XViGxPsmE5/PUuFvT4HD1V5Wtb6NV9U6N+1cmsHMRB8DNwSGHSIioiZ0/W6xdr7NsfQ8pN2qvmFma0cbBPw53ybA2wGeDjaieQzcEBh2iIiIGknVhplFOruB38gvqdbO181OO5k4wNsBrvZWBqhWvHiDj4iIGoVarUZkZCS8vb1hbW0NHx8fLF26FILw1z2anJwcTJ48GQqFAjY2Nhg8eDAuXbpU53UrKirwwQcfwMfHB1ZWVujatSv27t1brd2XX34JLy8vWFlZITAwEMeOHWvwPv6dWiPg7A0lNhxJx8tbTqDXh3EIXXkIb+8+g91JN3AjvwRSMwm6ejTHjL5t8PWkXkhePBB75/TF0hGdMKyrgkGnEXBkh4iIGsXy5cuxZs0aREdHw9/fH8ePH8eUKVMgl8vxxhtvQBAEjBgxAhYWFvjhhx9gb2+PlStXIjQ0FCkpKbC1ta3xuu+++y5iYmLw73//G76+vti3bx9GjhyJ33//Hd27dwcAfPPNN5g3bx7Wrl2LwMBAfPbZZwgLC0NqaipcXFwarI9llWqcua7Uzrc5cfUuCmrYMLO7Z/M/b0s5ortnc9jK+Ou3KUmE+yP2Y0qlUkEul0OpVMLe3t7Q5RARicKzzz4LV1dXrF+/Xnts9OjRsLa2RkxMDC5evIgOHTrg7Nmz8Pf3BwBoNBq4ubnho48+wvTp02u8rkKhwDvvvINZs2bVeF0ACAwMRO/evfHFF19or+vh4YHXX38dERERD92n4vJKnMzIr9p24WoekjLzUfa3DTObyczRy6tF1S0pLwd0biWHzJyPgTeG+v7+ZrQkIqJGERwcjK+++goXL15E+/btcerUKRw5cgQrV64EAJSVVa3ya2X1120bMzMzyGQyHDlypNawU1ZWpvMZALC2tsaRI0cAAOXl5Thx4gQWLVqkc93Q0FAkJCTo1Yf84nIcv3oXx/7cDfzcDSUq//aolKOtpc58m47uprthplgx7BARUaOIiIiASqWCr68vpFIp1Go1li1bhgkTJgAAfH194enpiUWLFmHdunWwtbXFp59+iuvXryMrK6vW64aFhWHlypXo27cvfHx8EB8fj127dkGtrlpo7/bt21Cr1XB1ddX5nKurKy5cuFBnzTmqUp3JxBeyC6q1adncWmc3cB9n8WyYKVYMO0RE1Ch27tyJrVu3Ytu2bfD390dycjLmzJkDhUKB8PBwWFhYYNeuXZg2bRocHBwglUoRGhqKIUOGoK4ZFqtWrcJLL70EX19fSCQS+Pj4YMqUKdiwYYNe9QmCgGt5JTj6507giVfzcPVO9Q0zfZxttaM2vb0c0KqFjd5/F2RYDDtERNQo5s+fj4iICIwbNw4A0LlzZ2RkZCAqKgrh4eEAgJ49eyI5ORlKpRLl5eVwdnZGYGAgevXqVet1nZ2dsWfPHpSWluLOnTtQKBSIiIhAmzZtAABOTk6QSqXIycnR+Vx2dg6atXDClj8y/lzj5g5yVLobZkokgJ+7vXa+TS8vBzjbPT4bZoqV0YedgoICREZGYvfu3cjNzUX37t2xatUq9O7dG0BVMn/vvffw73//G/n5+XjyySexZs0atGvXzsCVExE93oqLi2FmprvCiVQqhUajqdZWLpcDAC5duoTjx49j6dKlD7y+lZUVWrZsiYqKCnz//fcYM2YMAMDS0hI9e/ZEbFwcvHs+XbUb+JXb2PHDf2DbfSjS9pzVXsNCKkGXVs21Izc9W7eA/WO8YaZYGX3YmT59Os6ePYstW7ZAoVAgJiZG+1hiy5YtsWLFCnz++eeIjo6Gt7c3IiMjERYWhpSUlGoT2IiIqOkMGzYMy5Ytg6enJ/z9/ZGUlISVK1di6tSp2jbffvstnJ2d4enpiTNnzmD27NkYMWIEBg0apG0zadIktGzZElFRUQCAo0eP4saNG+jWrRtu3LiB999/HxqNBm/MfRN/XLmDxPQ8WHYbhjVrP8A36ZaQubeH6vgPUJeVwKl7GALbOmnn23T35IaZjwXBiBUXFwtSqVT4+eefdY736NFDeOeddwSNRiO4ubkJ//jHP7Tn8vPzBZlMJmzfvr3ef45SqRQACEqlssFqJyJ63KlUKmH27NmCp6enYGVlJbRp00Z45513hLKyMm2bVatWCa1atRIsLCwET09P4d1339U5LwiC0K9fPyE8PFz7/uDBg0LHjh0FmUwm2Dd3EHqEPCcMjdojtHv7P0LrhT9rXy1CZwoWchfBzNxC8OrYVYjes18or1Q3VfepCdT397dRr7NTUFAAe3t7xMXFISQkRHu8T58+MDc3x4YNG+Dj44OkpCR069ZNe75fv37o1q0bVq1aVeN1y8rKtI88AlXP6Xt4eHCdHSIiI5OlLEH67SJ4O9nCUmqGxKt/7gZ+9Q5SblbfMNPFTqa9JRXg7YD2LnbcMFPERLHOjp2dHYKCgrB06VJ07NgRrq6u2L59OxISEtC2bVtkZ2cDQI2PF947V5OoqCgsWbKkUWsnIqJHsyUhA4t/OIu6/kXu6WDzV7jxckBrR26YSdUZddgBgC1btmDq1Klo2bIlpFIpevTogfHjx+PEiRMPfc1FixZh3rx52vf3RnaIiMiw8orKEX8+Bz+euon/Xbpd7XwbJ1sEt3VEgLcjArwc4Cbn3Ex6MKMPOz4+Pjh06BCKioqgUqng7u6OsWPHok2bNnBzcwNQtZGcu7u79jM5OTk6t7X+TiaTQSbjo4RERMbgWl4x9p3LRmxKDhKv5lW7NXW/ZSM7I8jHsemKI1Ew+rBzj62tLWxtbXH37l3s27cPK1asgLe3N9zc3BAfH68NNyqVCkePHsUrr7xi2IKJiKhGgiDg3E0V9qfkYP+57GqrFPsr7BHUxhHrf0vH/bNKpRIJvJy4oB/pz+jDzr59+yAIAjp06IDLly9j/vz58PX1xZQpUyCRSDBnzhx8+OGHaNeunfbRc4VCgREjRhi6dCIi+lOlWoNjV/Ow/1wOYlNycCO/RHtOaiZBgJcDBvm7YqCfq3aF4nauzfD2rrNQCwKkEgk+GtUJ7nJrQ3WBTJjRhx2lUolFixbh+vXrcHBwwOjRo7Fs2TJYWFQt+rRgwQIUFRVhxowZyM/PR58+fbB3716usUNEZGDF5ZU4fPEW9p/LQfyFXChLKrTnrC2k6NveCYP83DDA1wUtbC2rfX5sb0/0be+Mq7eL4eVkw6BDD82oHz1vKvV9dI2IiOp2p7AM8edzsT8lG/+7dBtllX+tluxga4nQji4Y5OeGPu2cuJgfPTJRPHpORETGL+NOEWJTcrD/XA6OZ+hOMPZ0sMEgP1cM8ndDz9YtIOWaN2QADDtERKQXQRBw9oYK+1Oysf9cDlJzdCcYd24p1wac9q7NuO4NGRzDDhERPVCFWoOjV/IQm5KN/Sk5yFKWas9JzSR4oo0DBvm5IdTPFS2bc24NGReGHSIiqlFRWSUOXbyF/eeyceBCLlSlldpzNpZSPN3BGQP9XDGggyvkNtwpnIwXww4REWndKihD/Pkc7E/JwZHLt1F+3wRjp2aWCO3oikH+rgj24QRjMh0MO0REj7n020XYf67q9tTJzLs6C/l5OdogzN8NA/1c0d2TE4zJNDHsEBE9ZjQaAWduKLUTjC/lFuqc79pKjkH+bhjk54q2LpxgTKaPYYeI6DFQXqnBH1fuIDalagXjbNVfE4zNzSQI8nHEID9XhPq5cvE+Eh2GHSIikSoorfhzgnEOfr2Qi4KyvyYY21pK8bSvCwb5ueLpDi6QW3OCMYmX3mHn119/Rf/+/RujFiIiekS5qlLEnq8avfn98h2Uq++fYCzDQL97E4wdITPnBGN6POgddgYPHoxWrVphypQpCA8Ph4eHR2PURURE9ZR2qxD7z+Vgf0o2kjLzdc61cbLFoHsTjD2aw4wTjOkxpHfYuXHjBrZs2YLo6GgsWbIEAwYMwLRp0zBixAhYWlbfyI2IiBqWRiPg1PV87E/Jwf5z2Ui7VaRzvptHcwzyd8UgPze0dWlmoCqJjMcjbQR68uRJbNy4Edu3bwcAvPDCC5g2bRq6du3aYAU2BW4ESkTGrqxSjYS0O9ifkoO4lBzkFpRpz1lIJQj2ccJAP1cM9HOFq72VASslajr1/f39yLue37x5E1999RU+/vhjmJubo7S0FEFBQVi7di38/f0f5dJNhmGHiIyRqrQCB1OrVjA+mHoLhfdNMG4mM0f/PycY9+vgDHsrTjCmx0+j7npeUVGBH374ARs2bEBsbCx69eqFL774AuPHj8etW7fw7rvv4vnnn0dKSspDd4CI6HGUoyqt2kE8JQcJabdRof7r36MudvcmGLvhiTYOnGBMVE96j+y8/vrr2L59OwRBwMSJEzF9+nR06tRJp012djYUCgU0Gk0tVzEuHNkhIkMRBAFptwqx71xVwDl1LV/nfFuXZtodxLu0lHOCMdF9Gm1kJyUlBatXr8aoUaMgk8lqbOPk5IRff/1V30sTET0WNBoBSdfysT8lG7HncnDl9l8TjCUSoLtHc+0TVD7OnGBM9KjM9P1AfHw8xo8fX2vQAQBzc3P069fvkQojosebWq1GZGQkvL29YW1tDR8fHyxduhT3D0ZLJJIaX//4xz/qvPaXX34JLy8vWFlZITAwEMeOHdM5//TTT1e75ssvv/xI/SmtUOPXC7lYtOs0Aj6Kx+g1v2PdoSu4crsIllIz9O/gjKhRnXH07RDsevVJvNzPh0GHqIHoPbITFRUFV1dXTJ06Vef4hg0bcOvWLSxcuLDBiiOix9fy5cuxZs0aREdHw9/fH8ePH8eUKVMgl8vxxhtvAACysrJ0PvPf//4X06ZNw+jRo2u97jfffIN58+Zh7dq1CAwMxGeffYawsDCkpqbCxcVF2+6ll17CBx98oH1vY2Ojdx+UJRU4mJqL/edycDA1F0Xlau05OytzDPB1wSA/N/Tr4IxmMi5oT9RY9J6z4+XlhW3btiE4OFjn+NGjRzFu3Dikp6c3aIFNgXN2iIzPs88+C1dXV6xfv157bPTo0bC2tkZMTEyNnxkxYgQKCgoQHx9f63UDAwPRu3dvfPHFFwAAjUYDDw8PvP7664iIiABQNbLTrVs3fPbZZ3rXnaUsqZpgfC4Hf1y5g0rNX/8X62ZvhUH+VY+HB3o7wtJc78F1IrpPo83Zyc7Ohru7e7Xjzs7O1f6VRUT0sIKDg/HVV1/h4sWLaN++PU6dOoUjR45g5cqVNbbPycnBL7/8gujo6FqvWV5ejhMnTmDRokXaY2ZmZggNDUVCQoJO261btyImJgZubm4YNmwYIiMjaxzdEQQBl3ILsf9cNvan5OD0daXO+fauzTDIzw2D/F3RuaWcO4gTGYDeYcfDwwO//fYbvL29dY7/9ttvUCgUDVYYET3eIiIioFKp4OvrC6lUCrVajWXLlmHChAk1to+OjoadnR1GjRpV6zVv374NtVoNV1dXneOurq64cOGC9v0LL7yA1q1bQ6FQ4PTp01i4cCFSU1Oxa9cuAIBaI+Bk5t0/R3CycfVOsfazEgnQq3WLPxf4c4O3k+2j/DUQUQPQO+y89NJLmDNnDioqKjBgwAAAVZOWFyxYgDfffLPBCySix9POnTuxdetWbNu2Df7+/khOTsacOXOgUCgQHh5erf2GDRswYcIEWFk9+urBM2bM0P53586d4e7ujpCQEGzZdxRnC6wRdz4Hd4rKtW0szc3wVFsnDPJ3xQBfVzjb1f4ABxE1Pb3Dzvz583Hnzh28+uqrKC+v+mG3srLCwoULdYaGiYgexfz58xEREYFx48YBqAodGRkZiIqKqhZ2/ve//yE1NRXffPNNndd0cnKCVCpFTk6OzvGcnBy4ublVa68srsCB1Bz8cqVq37+31u+DdZueAAB7K3OEdHTFID9X9G3vDFtOMCYyWnr/dEokEixfvhyRkZE4f/48rK2t0a5duzofRSci0ldxcTHMzHQn8Eql0hoXK12/fj169uz5wH35LC0t0bNnT8THx2PEiBEAqiYox8fH47XXXgMA3MgvQeyf82+OpudBrRFQer1qNXg3N3eMDPbCID9X9PZ2gIWUE4yJTMFD/1OkWbNm6N27d0PWQkSkNWzYMCxbtgyenp7w9/dHUlISVq5cWW3ZC5VKhW+//RaffPJJjdcJCQnByJEjtWFm3rx5CA8PR69evRAQEIBPP/0UBYWFqGzbD8+u/h+SzqaiKOUgrH16Q2ptB5eKHGQeWIOegcFIXDWdE4yJTNBDhZ3jx49j586dyMzM1N7KuufeBD4iokexevVqREZG4tVXX0Vubi4UCgVmzpyJxYsX67TbsWMHBEHA+PHja7xOWloabt++rX0/duxY5OTmIuLtd3ErNwc27j6wfW4xvj6eBwCQmpvDIuccCk7/gsqyElh7eGDqi+Pw7rvvMugQmSi919nZsWMHJk2ahLCwMOzfvx+DBg3CxYsXkZOTg5EjR2Ljxo2NVWuj4To7ROKVpSxB+u0iKOTW2kfE4y/kIu++CcYyczM81c4Zg/xdEeLrAsdmvC1PZAoabZ2djz76CJ9++ilmzZoFOzs7rFq1Ct7e3pg5c2aN6+8QERlChVqDf/16GZ/FXUJN/6JrbmOBEN+qBf76tneCjSUnGBOJld4/3WlpaRg6dCiAqsl+RUVFkEgkmDt3LgYMGIAlS5Y0eJFERHVRlVbg/E0VzmepkPLnKzW7ABXq6jFnTK9WGNm9FXp7tYA5JxgTPRb0DjstWrRAQUEBAKBly5Y4e/YsOnfujPz8fBQXFz/g00RED08QBFy/W/JXqLmpwvlsFa7lldT7GiO7t0KQj2MjVklExkbvsNO3b1/Exsaic+fOeP755zF79mwcOHAAsbGxCAkJaYwaiegxVFapxqWcwr9CzZ8Bp6C0ssb2LZtbo6O7Pfzc7eCnsIdTMxnGrEvAfVtTQSqRwMtJ/w09ici06R12vvjiC5SWlgIA3nnnHVhYWOD333/H6NGj8e677zZ4gUQkfnlF5VVh5mZVoDmfpcLl3EKdTTTvsZBK0M6lKtBUhRt7dHS3Q3Mby2pto0Z1xtu7zkItCJBKJPhoVCe4y62boktEZET0ehqrsrIS27ZtQ1hYWLW9ZUwZn8YiahoajYCrd4pwPqsAKVnKP0dsCpCtKq2xfXMbiz/DTFWo8VPYw8e5mV67hWcpS3D1djG8nGwYdIhEplGexjI3N8fLL7+M8+fPP3KBRCRuxeWVSM0u0LkNdSG7AMXl6hrbeznaVI3WuNlrR23c5VaPvLaNu9yaIYfoMaf3bayAgAAkJyejdevWjVEPEZkYQRCQW1BWbW5N+u0i1DRuLDM3g++9uTV/jtZ0cLNHM+4tRUSNRO//d3n11Vcxb948XLt2DT179oStra3O+S5dujRYcURkXCrVGly5XaQztyblpkpnB/D7OdvJ/roNpai6FeXtZAupGVciJqKmo/cKyn/fmA+o2hxUEARIJBKo1TUPURszztkhqk5VWoELWQVIuan8M9gUIDWnAOWV1TfiNJMAPs7NdEJNR3d7ONtxJWIiajyNtoJyenr6IxVG1BTUajXef/99xMTEIDs7GwqFApMnT9bZ32jy5MmIjo7W+VxYWBj27t1b63XXrFmDNWvW4OrVqwAAf39/LF68GEOGDNG2efrpp3Ho0CGdz82cORNr165toN41LEEQcCO/RHe0Jqv2tWuayczR0d3uvieh7NHBzQ5WFtImrpyIqH70Djucq0OmYPny5VizZg2io6Ph7++P48ePY8qUKZDL5XjjjTe07QYPHqyzn5tMVvdIRKtWrfDxxx+jXbt2EAQB0dHRGD58OJKSkuDv769t99JLL+GDDz7QvrexMY61Xe5fu+b8fXNsVHWuXfPX3JqO7vbwaGEDM96GIiITonfY2bx5c53nJ02a9NDFEDWU33//HcOHD9dubeLl5YXt27fj2LFjOu1kMhnc3Nzqfd1hw4bpvF+2bBnWrFmDP/74Qyfs2NjY6HXdxnD/2jX3RmvqWrumrcv9oabqv2tau4aIyNToHXZmz56t876iogLFxcWwtLSEjY0Nww4ZheDgYHz11Ve4ePEi2rdvj1OnTuHIkSNYuXKlTruDBw/CxcUFLVq0wIABA/Dhhx/C0bF+Wwmo1Wp8++23KCoqQlBQkM65rVu3IiYmBm5ubhg2bBgiIyMbbXRHoxGQkVf8520oZdUaNjdVda5dc+/x7nu3odq66Ld2DRGRKdE77Ny9e7fasUuXLuGVV17B/PnzG6QookcVEREBlUoFX19fSKVSqNVqLFu2DBMmTNC2GTx4MEaNGgVvb2+kpaXh7bffxpAhQ5CQkACptPb5J2fOnEFQUBBKS0vRrFkz7N69G35+ftrzL7zwAlq3bg2FQoHTp09j4cKFSE1Nxa5dux65XyXlalzI1n0S6kFr19w/t8ZP0TBr1xARmRK9n8aqzfHjx/Hiiy/iwoULDXG5JsWnscRnx44dmD9/Pv7xj3/A398fycnJmDNnDlauXInw8PAaP3PlyhX4+PggLi6uzn3eysvLkZmZCaVSie+++w5ff/01Dh06pBN47nfgwAGEhITg8uXL8PHxqVf9giDgVkEZzumzdo2bnc5oja87164hInFrtKexar2QuTlu3rzZUJcjeiTz589HREQExo0bBwDo3LkzMjIyEBUVVWvYadOmDZycnHD58uU6w46lpSXatm0LAOjZsycSExOxatUqrFu3rsb2gYGBAFBr2Ll/7Zr7d/Oube0ap2ay+0KNHfwV9vBytIW5lLehiIhqonfY+fHHH3XeC4KArKwsfPHFF3jyyScbrDCiR1FcXFxtTSipVAqNpvoaMfdcv34dd+7cgbu7u15/lkajQVlZWa3nk5OTAQDu7u46a9dU7Q+lqnPtmjbOzXSehOrobgcXOyu96iMietzpHXZGjBih814ikcDZ2RkDBgzAJ5980lB1ET2SYcOGYdmyZfD09IS/vz+SkpKwcuVKTJ06FQBQWFiIJUuWYPTo0XBzc0NaWhoWLFiAtm3bIiwsTHudkJAQjBw5Eq+99hoAYNGiRRgyZAg8PT1RUFCAbdu24eDBg9i3bx8AIC0tDdu2bcOQIUNQYWGL/f87htUfRcK9Q3fM2nsb17btr7FeW0updk7NvTk2XLuGiKhh6B126vqXMZGxWL16NSIjI/Hqq68iNzcXCoUCM2fOxOLFiwFUjfKcPn0a0dHRyM/Ph0KhwKBBg7B06VKdtXbS0tJw+/Zt7fvc3FxMmjQJWVlZkMvl6NKlC376z3/g7heAncev4Y/T6YjZsgtLov4BdXkpzO2dYNMuCDbB47SL9CnkVjpza/wUXLuGiKgxNdgEZVPGCcpUmyxlCdJvF8HbyRbucmvt2jX3noSq79o1Hd3/mjzMtWuIiBpGo01QHj16NAICArBw4UKd4ytWrEBiYiK+/fZb/aslMkLfJGYiYtcZ7dNPcmtzKEtqXmlYbm2hM7fGj2vXEBEZDb1HdpydnXHgwAF07txZ5/iZM2cQGhqKnJycBi2wKXBkh/4uS1mC4KgDqOmHo7WjzV+3oLh2DRGRwdT397fe/+wsLCyEpWX1YXgLCwuoVCp9L1cntVqNyMhIeHt7w9raGj4+Pli6dCnuz2eTJ0+GRCLReQ0ePLhB66DHiyAIWP7fCzUGnQ3hvXBofn+sebEn3ghph1A/VyiaWzPoEBEZMb1vY3Xu3BnffPONdqLnPTt27Kh1UbWH1VibORLVRqMR8N6P57AnufqaUVKJBB0VHPkjIjI1eoedyMhIjBo1CmlpaRgwYAAAID4+Htu3b2/w+TqNtZkjUU0q1BrM//YU9iTfhEQCDO+qwE+nsqAWBEglEnw0qhPc5daGLpOIiPSkd9gZNmwY9uzZg48++gjfffcdrK2t0aVLF8TFxaFfv34NWlxjbeZYVlamswhcQ99+I9NTWqHGrK0nEX8hF+ZmEnwypiuGd2uJhUN8cfV2MbycbBh0iIhMlFE/eq7RaPD2229jxYoVOps5Llq0SNtmx44dsLGx0dnMsVmzZnVu5vj+++9jyZIl1Y5zgvLjqaC0AtOjj+Noeh5k5mZY82IPDPB1NXRZRET0APWdoKx32ElMTIRGo9Hu93PP0aNHIZVK0atXr4eruAaNtZljTSM7Hh4eDDuPoTuFZZi8MRFnbihhJzPH1+G9ENim9lFBIiIyHo32NNasWbNw7dq1asdv3LiBWbNm6Xu5Ot2/mWPnzp0xceJEzJ07F1FRUbV+5v7NHGsjk8lgb2+v86LHT5ayBGPWJeDMDSUcbC2xfcYTDDpERCKk95ydlJQU9OjRo9rx7t27IyUlpUGKuqcpN3Okx0v67SK8+PVR3MgvgUJuhc3TAtHWpZmhyyIiokag98iOTCarceHArKwsmJvrnZ3qdG8zx19++QVXr17F7t27sXLlSowcORJA1Zo/8+fPxx9//IGrV68iPj4ew4cPr7aZI9H9Um6q8Pza33EjvwRtnGzx7SvBDDpERCKm95yd8ePHIysrCz/88APkcjkAID8/HyNGjICLiwt27tzZYMUVFBQgMjISu3fv1m7mOH78eCxevBiWlpYoKSnBiBEjkJSUVG0zR1fX+k8w5QrKj4/jV/MwZVMiCkor4eduj83TAuDUjOsyERGZokaboHzjxg307dsXd+7cQffu3QEAycnJcHV1RWxsLDw8PB6tcgNg2Hk8HEzNxcsxJ1BaoUFvrxb4Orw35NYWhi6LiIgeUqNtBNqyZUucPn0aW7duxalTp2BtbY0pU6Zg/PjxsLDgLw4yTj+fvom53ySjQi3g6Q7OWDOhJ6wta16agIiIxOWhJtnY2tpixowZDV0LUaPYfiwTb++u2r382S7uWDmmG3cjJyJ6jDz0jOKUlBRkZmaivLxc5/hzzz33yEURNZR1h9IQ9d8LAIAXAj2xdHgnSM24aScR0eNE77Bz5coVjBw5EmfOnIFEItHuQH5v12e1Wt2wFRI9BEEQsGJfKtYcTAMAvPK0DxaEdeDu5EREjyG9x/Jnz54Nb29v5ObmwsbGBufOncPhw4fRq1cvHDx4sBFKJNKPWiPg3T1ntUFn4WBfLBzsy6BDRPSY0ntkJyEhAQcOHICTkxPMzMxgZmaGPn36ICoqCm+88QaSkpIao06ieqlQazBv5yn8dKpq5/JlIzrjhUBPQ5dFREQGpPfIjlqthp2dHQDAyckJN2/eBAC0bt0aqampDVsdkR5KytWYsfk4fjp1ExZSCT4f151Bh4iI9B/Z6dSpE06dOgVvb28EBgZixYoVsLS0xFdffYU2bdo0Ro1ED6QqrcD0Tcdx7GoerCzMsPbFnni6g4uhyyIiIiOgd9h59913UVRUBAD44IMP8Oyzz+Kpp56Co6MjvvnmmwYvkOhBbheWIXzDMZy7qYKdlTk2TO6N3l4Ohi6LiIiMhN4rKNckLy8PLVq0MNkJoFxB2XTdyC/BxK+P4srtIjjaWmLztAD4K+SGLouIiJpAo62gXBMHB/4rmppe2q1CTPz6KG4qS9GyuTW2TAtAG2du6ElERLoadptyoiZy9oYS4RuO4U5ROdo42yJmWiAUza0NXRYRERkhhh0yOcfS8zBtUyIKyirRqaU9oqcEwJE7lxMRUS0Ydsik/HqhaufyskoNArwd8HV4L9hbcQNaIiKqnd7r7Bw+fBiVlZXVjldWVuLw4cMNUhRRTX48dRMvbT6OskoNBvi6YPPUAAYdIiJ6IL3DTv/+/ZGXl1ftuFKpRP/+/RukKKK/23o0A7N3JKFSI2B4NwXWTewJKwupocsiIiIToPdtLEEQanzE/M6dO7C1tW2Qooju96+Dl7Fib9Xq3C8+4YkPnusEM+5cTkRE9VTvsDNq1CgAVbubT548GTLZXxNC1Wo1Tp8+jeDg4IavkB5bgiDg470XsO7QFQDArP4+eGsQdy4nIiL91DvsyOVVC7UJggA7OztYW//1mK+lpSWeeOIJvPTSSw1fIT2WqnYuP4Ptx64BAN5+xhcz+voYuCoiIjJF9Q47GzduBAB4eXnhrbfe4i0rajTllRrM3ZmMX05nwUwCRI3qjLG9uaEnERE9HL3n7CxYsAD37zCRkZGB3bt3w8/PD4MGDWrQ4ujxU1KuxssxJ3Do4i1YSCVYNa47nunsbuiyiIjIhOn9NNbw4cOxefNmAEB+fj4CAgLwySefYPjw4VizZk2DF0iPD2VJBSauP4pDF2/B2kKKr8N7M+gQEdEj0zvsnDx5Ek899RQA4LvvvoObmxsyMjKwefNmfP755w1eID0ebhWUYdxXf+B4xl3YW5kjZnoA+rV3NnRZREQkAnrfxiouLoadnR0AYP/+/Rg1ahTMzMzwxBNPICMjo8ELJPG7frcYE9cfQ/rtIjg1k2HLtAB0dOfu80RE1DD0Htlp27Yt9uzZg2vXrmHfvn3aeTq5ubl1bq9OVJPLuYV4fm0C0m8XoWVza3z7chCDDhERNSi9w87ixYvx1ltvwcvLCwEBAQgKCgJQNcrTvXv3Bi+QxOvMdSXGrEtAlrIUbV2a4ftXguHtxKf8iIioYUmE+x+tqqfs7GxkZWWha9euMDOrykvHjh2Dvb09fH19G7zIxqZSqSCXy6FUKjk61UT+uHIH06OPo7CsEl1aybFpSgAcbC0NXRYREZmQ+v7+1ntkBwDc3NxgZ2eH2NhYlJSUAAB69+5tkkGHml78+RyEbziGwrJKPNHGAVunBzLoEBFRo9E77Ny5cwchISFo3749nnnmGWRlZQEApk2bhjfffLPBCyRx+SH5BmZuOYGySg1CO7pg05QA2HHnciIiakR6h525c+fCwsICmZmZsLGx0R4fO3Ys9u7d26DFkbhsSbiKOd8ko1IjYGT3lljzIncuJyKixqf3o+f79+/Hvn370KpVK53j7dq146PnVCNBEPDlr5fxz/0XAQDhQa3x3jB/7lxORERNQu+wU1RUpDOic09eXp7OTuhEQFXQ+eg/5/Hv/6UDAN4Y0BZzB7bnzuVERNRk9L6N9dRTT2m3iwAAiUQCjUaDFStWoH///g1aHJk2tUZAxPdntEHn3aEdMW9QBwYdIiJqUnqP7KxYsQIhISE4fvw4ysvLsWDBApw7dw55eXn47bffGqNGMkFllWrM2ZGM/57NhpkE+HhUF4zp7WHosoiI6DGk98hOp06dcPHiRfTp0wfDhw9HUVERRo0ahaSkJPj4+DRGjWRiissrMT36OP57NhuWUjP8a0IPBh0iIjIYvRcVzMzMhIeHR423IjIzM+Hp6dlgxTUVLirYcJTFFZiy6RhOZubDxlKKdRN74ql23NCTiIgaXqMtKujt7Y1bt25VO37nzh14e3vrezkSkdyCUoz9KgEnM/P/3Lk8kEGHiIgMTu85O4Ig1DiqU1hYCCsrqwYpikzPtbxivLj+KDLuFMPZrmrncl83jpIREZHh1TvszJs3D0DV01eRkZE6j5+r1WocPXoU3bp1a/ACyfhdyinAi+uPIkdVBg8Ha8RMC0RrR27oSURExqHeYScpKQlA1cjOmTNnYGn5115GlpaW6Nq1K956662Gr5CM2qlr+Zi88RjuFlegnUszbJkWCDc5R/iIiMh41Dvs/PrrrwCAKVOmYNWqVZzIS0hIu4Pp0YkoKlej6587l7fghp5ERGRk9J6zs3Hjxsaog0xMbEoOZm07ifJKDYJ9HPHVpF5oJtP7f05ERESNjr+dSG+7k67jrW9PQ60RMNDPFavHd+eGnkREZLQYdkgv0b9fxXs/ngMAjOrREitGd4G5VO8VDIiIiJoMww7ViyAIWH3gMlbGVu1cPjnYC4uf9ePO5UREZPQYduiBNBoBy/5zHuuPVG3oOSe0HWaHtOOGnkREZBIYdqhOlWoNInadwXcnrgMAFj/rh6l9uFI2ERGZDoYdqlVZpRqztydj77lsSM0kWD66C/6vZytDl0VERKQXhh2qUVFZJWZuOYEjl2/DUmqG1S90R5i/m6HLIiIi0hvDDlWTX1yOyRsTkXytaufyf0/qhSfbOhm6LCIioodi1M8Mq9VqREZGwtvbG9bW1vDx8cHSpUshCIK2jSAIWLx4Mdzd3WFtbY3Q0FBcunTJgFWbtlxVKcau+wPJ1/LR3MYC2156gkGHiIhMmlGHneXLl2PNmjX44osvcP78eSxfvhwrVqzA6tWrtW1WrFiBzz//HGvXrsXRo0dha2uLsLAwlJaWGrBy05R5pxj/tzYBqTkFcLGT4ZsZQejm0dzQZRERET0SiXD/MImRefbZZ+Hq6or169drj40ePRrW1taIiYmBIAhQKBR48803tZuQKpVKuLq6YtOmTRg3bly9/hyVSgW5XA6lUvnY7vl1MacAL359FLkFZfB0sEHMtEB4Oto8+INEREQGUt/f30Y9shMcHIz4+HhcvFi1kN2pU6dw5MgRDBkyBACQnp6O7OxshIaGaj8jl8sRGBiIhIQEg9RsipIy72LMugTkFpShg6sdvns5iEGHiIhEw6gnKEdEREClUsHX1xdSqRRqtRrLli3DhAkTAADZ2dkAAFdXV53Pubq6as/VpKysDGVlZdr3KpWqEao3Db9dvo2XNh9Hcbka3TyaY9OU3mhuw53LiYhIPIx6ZGfnzp3YunUrtm3bhpMnTyI6Ohr//Oc/ER0d/UjXjYqKglwu1748PDwaqGLTsu9cNqZsTERxuRp92jph6/RABh0iIhIdow478+fPR0REBMaNG4fOnTtj4sSJmDt3LqKiogAAbm5V677k5OTofC4nJ0d7riaLFi2CUqnUvq5du9Z4nTBS3524jldiTqBcrcFgfzesn9wLtjKjHugjIiJ6KEYddoqLi2FmpluiVCqFRqMBAHh7e8PNzQ3x8fHa8yqVCkePHkVQUFCt15XJZLC3t9d5PU42HEnHW9+egkYA/q9nK3zxQnfIzKWGLouIiKhRGPU/5YcNG4Zly5bB09MT/v7+SEpKwsqVKzF16lQAgEQiwZw5c/Dhhx+iXbt28Pb2RmRkJBQKBUaMGGHY4o2QIAj4LO4SVsVXrUM0rY833nmmI3cuJyIiUTPqsLN69WpERkbi1VdfRW5uLhQKBWbOnInFixdr2yxYsABFRUWYMWMG8vPz0adPH+zduxdWVlYGrNz4aDQCPvg5BZt+vwoAeHNge7w2oC13LiciItEz6nV2morY19mpVGuw4PvT2HXyBgBgyXP+CA/2MmxRREREj6i+v7+NemSHHl1phRqvb09CbEoOpGYS/PP5LhjZnTuXExHR44NhR8QKyyoxY/Nx/J52B5bmZvjyhR4Y6Of64A8SERGJCMOOSN0tKsfkjcdw6roStpZS/Du8F4J9uKEnERE9fhh2RChbWYqJ64/iUm4hWthYIHpqALq0am7osoiIiAyCYUdkMu4UYcLXR3H9bgnc7K2wZVoA2rnaGbosIiIig2HYEZEL2SpMXH8MtwrK0NqxaudyDwdu6ElERI83hh2ROJl5F1M2JkJZUgFfNztsnhYAFzuuNURERMSwIwL/u3QLMzafQEmFGj08m2Pj5ADIbSwMXRYREZFRYNgxcXvPZuGN7ckoV2vwVDsnrJvYEzaW/FqJiIju4W9FE7bz+DVEfH8aGgF4prMbPh3bjRt6EhER/Q3Djon6+n9X8OEv5wEAY3t54KNRnSHlhp5ERETVMOyYGEEQsDL2IlYfuAwAmNG3DRYN8eWGnkRERLVg2DEhGo2AJT+dQ3RCBgBgflgHvPq0D4MOERFRHRh2TESFWoMF353G7qQbkEiAD57zx8QgL0OXRUREZPQYdkxAaYUar207ibjzuZCaSbByTFcM79bS0GURERGZBIYdI1dQWoHp0cdxND0PMnMz/GtCD4R05M7lRERE9cWwY8Ty/ty5/PR1JZrJzPF1eC880cbR0GURERGZFIYdI5WlLMHE9cdwObcQDraWiJ4SgM6t5IYui4iIyOQw7Bih9NtFePHro7iRXwJ3uRW2TAtEW5dmhi6LiIjIJDHsGJmUmypM2nAMtwvL4O1kiy3TAtCqBXcuJyIielhmhi7AGHl5eUEikVR7zZo1S9smISEBAwYMgK2tLezt7dG3b1+UlJTUek21Wo3IyEh4e3vD2toaPj4+WLp0KQRB0Lb5x7poBD7VH0nLRiFj+bOIDLJh0CEiInpEHNmpQWJiItRqtfb92bNnMXDgQDz//PMAqoLO4MGDsWjRIqxevRrm5uY4deoUzMxqz47Lly/HmjVrEB0dDX9/fxw/fhxTpkyBXC7HG2+8gUMXb2Hlf85AquiIbkEDcXLrcjjYWjZ6X4mIiMROItw/tPCYUqlUkMvlUCqVsLe3r3Z+zpw5+Pnnn3Hp0iVIJBI88cQTGDhwIJYuXVrvP+PZZ5+Fq6sr1q9frz02evRoWFtbY/yCf2DON0moUAvo194Zi/o4omOHtkhKSkK3bt0aootERESi86Df3/fwNtYDlJeXIyYmBlOnToVEIkFubi6OHj0KFxcXBAcHw9XVFf369cORI0fqvE5wcDDi4+Nx8eJFAMCpU6dw5MgRNG/fG69vP4kKtYChXdzx70m9YGXJncuJiIgaCm9jPcCePXuQn5+PyZMnAwCuXLkCAHj//ffxz3/+E926dcPmzZsREhKCs2fPol27djVeJyIiAiqVCr6+vpBKpVCr1Rg+fR5+Lm4LABgf4IEPR3DnciIioobGkZ0HWL9+PYYMGQKFQgEA0Gg0AICZM2diypQp6N69Oz799FN06NABGzZsqPU6O3fuxNatW7Ft2zacOHECY96Mwo9bvkLhmXjM7NcGH41k0CEiImoMDDt1yMjIQFxcHKZPn6495u7uDgDw8/PTaduxY0dkZmbWeq358+cjIiICY8aMxc4rEvwh7QS73sNhfuYHLBrSkTuXExERNRKGnTps3LgRLi4uGDp0qPaYl5cXFAoFUlNTddpevHgRrVu3rvVaxcXFEADM3ZmMmD8yIZEAYZ3cYW/F+TlERESNiXN2aqHRaLBx40aEh4fD3PyvvyaJRIL58+fjvffeQ9euXdGtWzdER0fjwoUL+O6777TtQkJCMHLkSLz22msAgGeGPouFkR/ANuQVWDu3xvh2Gnz99UZMnTpV+5m8vDxkZmbi5s2bAKANVG5ubnBzc2uKbhMREYkOw04t4uLikJmZqRNG7pkzZw5KS0sxd+5c5OXloWvXroiNjYWPj4+2TVpaGm7fvg0AUJVWoLDnREhT8nF3/7+gLCvAzpYKzJw5E4sXL9Z+5scff8SUKVO078eNGwcAeO+99/D+++83Uk+JiIjEjevsoP7P6esrS1mCU9fy8WncRaRmF8JOZo71k3sjwNuhwf4MIiKix1V9f39zZKeRfJOYiUW7zkDzZ5S0tZRi+4wn0Kkldy4nIiJqSpyg3AiylCU6QQcASirUcGzG7R+IiIiaGsNOI0i/XaQTdABAIwBXbxcbpiAiIqLHGMNOI/B2ssXf1weUSiTwcuIO5kRERE2NYacRuMutETWqM6R/LhQolUjw0ahOcJdbG7gyIiKixw8nKDeSsb090be9M67eLoaXkw2DDhERkYEw7DQid7k1Qw4REZGB8TYWERERiRrDDhEREYkaww4RERGJGsMOERERiRrDDhEREYkaww4RERGJGsMOERERiRrDDhEREYkaww4RERGJGsMOERERiRrDDhEREYkaww4RERGJGsMOERERiZrRhx0vLy9IJJJqr1mzZgEAnn766WrnXn75ZQNXTURERMbC3NAFPEhiYiLUarX2/dmzZzFw4EA8//zz2mMvvfQSPvjgA+17GxubJq2RiIiIjJfRhx1nZ2ed9x9//DF8fHzQr18/7TEbGxu4ubk1dWlERERkAoz+Ntb9ysvLERMTg6lTp0IikWiPb926FU5OTujUqRMWLVqE4uLiOq9TVlYGlUql8yIiIiJxMvqRnfvt2bMH+fn5mDx5svbYCy+8gNatW0OhUOD06dNYuHAhUlNTsWvXrlqvExUVhSVLljRBxURERGRoEkEQBEMXUV9hYWGwtLTETz/9VGubAwcOICQkBJcvX4aPj0+NbcrKylBWVqZ9r1Kp4OHhAaVSCXt7+wavm4iIiBqeSqWCXC5/4O9vkxnZycjIQFxcXJ0jNgAQGBgIAHWGHZlMBplM1uA1EhERkfExmTk7GzduhIuLC4YOHVpnu+TkZACAu7t7E1RFRERExs4kRnY0Gg02btyI8PBwmJv/VXJaWhq2bduGZ555Bo6Ojjh9+jTmzp2Lvn37okuXLgasmIiIiIyFSYSduLg4ZGZmYurUqTrHLS0tERcXh88++wxFRUXw8PDA6NGj8e677xqoUiIiIjI2JjVBubHUd4ITERERGY/6/v42mTk7RERERA+DYYeIiIhEjWGHiIiIRI1hh4iIiESNYYeIiIhEjWGHiIiIRI1hh4iIiESNYYeIiIhEjWGHiIiIRI1hh4iIiESNYYeIiIhEjWGHiIiIRI1hh4iIiESNYYeIiIhEjWGHiIiIRI1hh4iIiESNYYeIiIhEjWGHiIiIRI1hh4iIiESNYYeIiIhEjWGHiIiIRI1hh4iIiESNYYeIiIhEjWGHiIiIRI1hh4iIiESNYYeIiIhEjWGHiIiIRI1hh4iIiESNYYeIiIhEjWGHiIiIRI1hh4iIiESNYYeIiIhEjWGHiIiIRI1hh4iIiESNYYeIiIhEjWGHiIiIRI1hh4iIiESNYYeIiIhEjWGHiIiIRI1hh4iIiESNYYeIiIhEjWGHiIiIRI1hh4iIiESNYYeIiIhEjWGHiIiIRI1hh4iIiESNYYeIiIhEjWGHiIiIRI1hh4iIiESNYYeIiIhEjWGHiIiIRM3ow46XlxckEkm116xZswAApaWlmDVrFhwdHdGsWTOMHj0aOTk5Bq6aiIiIjIXRh53ExERkZWVpX7GxsQCA559/HgAwd+5c/PTTT/j2229x6NAh3Lx5E6NGjTJkyURERGREJIIgCIYuQh9z5szBzz//jEuXLkGlUsHZ2Rnbtm3D//3f/wEALly4gI4dOyIhIQFPPPFEva6pUqkgl8uhVCphb2/fmOUTERFRA6nv72/zJqzpkZWXlyMmJgbz5s2DRCLBiRMnUFFRgdDQUG0bX19feHp61hl2ysrKUFZWpn2vVCoBVP2lERERkWm493v7QeM2JhV29uzZg/z8fEyePBkAkJ2dDUtLSzRv3lynnaurK7Kzs2u9TlRUFJYsWVLtuIeHR0OWS0RERE2goKAAcrm81vMmFXbWr1+PIUOGQKFQPNJ1Fi1ahHnz5mnfazQa5OXlwdHRERKJ5FHL1FKpVPDw8MC1a9dEe3tM7H1k/0yf2PvI/pk+sfexMfsnCAIKCgoemAtMJuxkZGQgLi4Ou3bt0h5zc3NDeXk58vPzdUZ3cnJy4ObmVuu1ZDIZZDKZzrG/jw41JHt7e1H+D/h+Yu8j+2f6xN5H9s/0ib2PjdW/ukZ07jH6p7Hu2bhxI1xcXDB06FDtsZ49e8LCwgLx8fHaY6mpqcjMzERQUJAhyiQiIiIjYxIjOxqNBhs3bkR4eDjMzf8qWS6XY9q0aZg3bx4cHBxgb2+P119/HUFBQfV+EouIiIjEzSTCTlxcHDIzMzF16tRq5z799FOYmZlh9OjRKCsrQ1hYGP71r38ZoMrqZDIZ3nvvvWq3zMRE7H1k/0yf2PvI/pk+sffRGPpncuvsEBEREenDZObsEBERET0Mhh0iIiISNYYdIiIiEjWGHSIiIhI1hp1H9OWXX8LLywtWVlYIDAzEsWPH6mz/7bffwtfXF1ZWVujcuTP+85//NFGlD0ef/m3atAkSiUTnZWVl1YTV6ufw4cMYNmwYFAoFJBIJ9uzZ88DPHDx4ED169IBMJkPbtm2xadOmRq/zUejbx4MHD1b7DiUSSZ3brxhSVFQUevfuDTs7O7i4uGDEiBFITU194OdM5efwYfpnaj+Ha9asQZcuXbQLzgUFBeG///1vnZ8xle8P0L9/pvb9/d3HH38MiUSCOXPm1Nmuqb9Dhp1H8M0332DevHl47733cPLkSXTt2hVhYWHIzc2tsf3vv/+O8ePHY9q0aUhKSsKIESMwYsQInD17tokrrx99+wdUrZCZlZWlfWVkZDRhxfopKipC165d8eWXX9arfXp6OoYOHYr+/fsjOTkZc+bMwfTp07Fv375GrvTh6dvHe1JTU3W+RxcXl0aq8NEcOnQIs2bNwh9//IHY2FhUVFRg0KBBKCoqqvUzpvRz+DD9A0zr57BVq1b4+OOPceLECRw/fhwDBgzA8OHDce7cuRrbm9L3B+jfP8C0vr/7JSYmYt26dejSpUud7QzyHQr00AICAoRZs2Zp36vVakGhUAhRUVE1th8zZowwdOhQnWOBgYHCzJkzG7XOh6Vv/zZu3CjI5fImqq5hARB2795dZ5sFCxYI/v7+OsfGjh0rhIWFNWJlDac+ffz1118FAMLdu3ebpKaGlpubKwAQDh06VGsbU/s5vF99+mfKP4f3tGjRQvj6669rPGfK3989dfXPVL+/goICoV27dkJsbKzQr18/Yfbs2bW2NcR3yJGdh1ReXo4TJ04gNDRUe8zMzAyhoaFISEio8TMJCQk67QEgLCys1vaG9DD9A4DCwkK0bt0aHh4eD/zXi6kxpe/vUXXr1g3u7u4YOHAgfvvtN0OXU29KpRIA4ODgUGsbU/4e69M/wHR/DtVqNXbs2IGioqJat/wx5e+vPv0DTPP7mzVrFoYOHVrtu6mJIb5Dhp2HdPv2bajVari6uuocd3V1rXV+Q3Z2tl7tDelh+tehQwds2LABP/zwA2JiYqDRaBAcHIzr1683RcmNrrbvT6VSoaSkxEBVNSx3d3esXbsW33//Pb7//nt4eHjg6aefxsmTJw1d2gNpNBrMmTMHTz75JDp16lRrO1P6Obxffftnij+HZ86cQbNmzSCTyfDyyy9j9+7d8PPzq7GtKX5/+vTPFL+/HTt24OTJk4iKiqpXe0N8hyaxXQSZhqCgIJ1/rQQHB6Njx45Yt24dli5dasDKqL46dOiADh06aN8HBwcjLS0Nn376KbZs2WLAyh5s1qxZOHv2LI4cOWLoUhpFfftnij+HHTp0QHJyMpRKJb777juEh4fj0KFDtQYCU6NP/0zt+7t27Rpmz56N2NhYo55IzbDzkJycnCCVSpGTk6NzPCcnB25ubjV+xs3NTa/2hvQw/fs7CwsLdO/eHZcvX26MEptcbd+fvb09rK2tDVRV4wsICDD6APHaa6/h559/xuHDh9GqVas625rSz+E9+vTv70zh59DS0hJt27YFAPTs2ROJiYlYtWoV1q1bV62tKX5/+vTv74z9+ztx4gRyc3PRo0cP7TG1Wo3Dhw/jiy++QFlZGaRSqc5nDPEd8jbWQ7K0tETPnj0RHx+vPabRaBAfH1/rvdigoCCd9gAQGxtb571bQ3mY/v2dWq3GmTNn4O7u3lhlNilT+v4aUnJystF+h4Ig4LXXXsPu3btx4MABeHt7P/AzpvQ9Pkz//s4Ufw41Gg3KyspqPGdK319t6urf3xn79xcSEoIzZ84gOTlZ++rVqxcmTJiA5OTkakEHMNB32GhTnx8DO3bsEGQymbBp0yYhJSVFmDFjhtC8eXMhOztbEARBmDhxohAREaFt/9tvvwnm5ubCP//5T+H8+fPCe++9J1hYWAhnzpwxVBfqpG//lixZIuzbt09IS0sTTpw4IYwbN06wsrISzp07Z6gu1KmgoEBISkoSkpKSBADCypUrhaSkJCEjI0MQBEGIiIgQJk6cqG1/5coVwcbGRpg/f75w/vx54csvvxSkUqmwd+9eQ3XhgfTt46effirs2bNHuHTpknDmzBlh9uzZgpmZmRAXF2eoLtTplVdeEeRyuXDw4EEhKytL+youLta2MeWfw4fpn6n9HEZERAiHDh0S0tPThdOnTwsRERGCRCIR9u/fLwiCaX9/gqB//0zt+6vJ35/GMobvkGHnEa1evVrw9PQULC0thYCAAOGPP/7QnuvXr58QHh6u037nzp1C+/btBUtLS8Hf31/45Zdfmrhi/ejTvzlz5mjburq6Cs8884xw8uRJA1RdP/ces/77616fwsPDhX79+lX7TLdu3QRLS0uhTZs2wsaNG5u8bn3o28fly5cLPj4+gpWVleDg4CA8/fTTwoEDBwxTfD3U1DcAOt+LKf8cPkz/TO3ncOrUqULr1q0FS0tLwdnZWQgJCdEGAUEw7e9PEPTvn6l9fzX5e9gxhu9QIgiC0HjjRkRERESGxTk7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0REf3Pw4EFIJBLk5+cbuhQiagAMO0RERCRqDDtEREQkagw7RGR0NBoNoqKi4O3tDWtra3Tt2hXfffcdgL9uMf3yyy/o0qULrKys8MQTT+Ds2bM61/j+++/h7+8PmUwGLy8vfPLJJzrny8rKsHDhQnh4eEAmk6Ft27ZYv369TpsTJ06gV69esLGxQXBwMFJTUxu340TUKBh2iMjoREVFYfPmzVi7di3OnTuHuXPn4sUXX8ShQ4e0bebPn49PPvkEiYmJcHZ2xrBhw1BRUQGgKqSMGTMG48aNw5kzZ/D+++8jMjISmzZt0n5+0qRJ2L59Oz7//HOcP38e69atQ7NmzXTqeOedd/DJJ5/g+PHjMDc3x9SpU5uk/0TUsLgRKBEZlbKyMjg4OCAuLg5BQUHa49OnT0dxcTFmzJiB/v37Y8eOHRg7diwAIC8vD61atcKmTZswZswYTJgwAbdu3cL+/fu1n1+wYAF++eUXnDt3DhcvXkSHDh0QGxuL0NDQajUcPHgQ/fv3R1xcHEJCQgAA//nPfzB06FCUlJTAysqqkf8WiKghcWSHiIzK5cuXUVxcjIEDB6JZs2ba1+bNm5GWlqZtd38QcnBwQIcOHXD+/HkAwPnz5/Hkk0/qXPfJJ5/EpUuXoFarkZycDKlUin79+tVZS5cuXbT/7e7uDgDIzc195D4SUdMyN3QBRET3KywsBAD88ssvaNmypc45mUymE3gelrW1db3aWVhYaP9bIpEAqJpPRESmhSM7RGRU/Pz8IJPJkJmZibZt2+q8PDw8tO3++OMP7X/fvXsXFy9eRMeOHQEAHTt2xG+//aZz3d9++w3t27eHVCpF586dodFodOYAEZF4cWSHiIyKnZ0d3nrrLcydOxcajQZ9+vSBUqnEb7/9Bnt7e7Ru3RoA8MEHH8DR0RGurq5455134OTkhBEjRgAA3nzzTfTu3RtLly7F2LFjkZCQgC+++AL/+te/AABeXl4IDw/H1KlT8fnnn6Nr167IyMhAbm4uxowZY6iuE1EjYdghIqOzdOlSODs7IyoqCleuXEHz5s3Ro0cPvP3229rbSB9//DFmz56NS5cuoVu3bvjpp59gaWkJAOjRowd27tyJxYsXY+nSpXB3d8cHH3yAyZMna/+MNWvW4O2338arr76KO3fuwNPTE2+//bYhuktEjYxPYxGRSbn3pNTdu3fRvHlzQ5dDRCaAc3aIiIhI1Bh2iIiISNR4G4uIiIhEjSM7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkav8PYsojZWtCM4MAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plt.plot(np.arange(len(epochs_x)), epochs_acc, marker = '.')\n", + "plt.xlabel('epoch')\n", + "plt.ylabel('test accuracy')\n", + "plt.ylim(70, 100)\n", + "for i, txt in enumerate(epochs_acc):\n", + " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom')\n", + "plt.show()" + ] } ], "metadata": { From a7afda87029b0cf6f6c413e824c27fb498ee897a Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Fri, 26 Apr 2024 21:12:41 +0200 Subject: [PATCH 049/379] 2nd baseline sequential: 7 layer blocks --- .../baseline-SCNN-example_2.ipynb | 630 ++++++++++++++++++ 1 file changed, 630 insertions(+) create mode 100644 tests/test_nonsequential/baseline-SCNN-example_2.ipynb diff --git a/tests/test_nonsequential/baseline-SCNN-example_2.ipynb b/tests/test_nonsequential/baseline-SCNN-example_2.ipynb new file mode 100644 index 00000000..893b5e79 --- /dev/null +++ b/tests/test_nonsequential/baseline-SCNN-example_2.ipynb @@ -0,0 +1,630 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "import torch.nn as nn\n", + "import sinabs.layers as sl\n", + "from tqdm.notebook import tqdm\n", + "\n", + "from tonic.datasets.nmnist import NMNIST\n", + "from tonic.transforms import ToFrame\n", + "from torch.utils.data import DataLoader\n", + "from torch.nn import CrossEntropyLoss\n", + "from torch.optim import Adam\n", + "\n", + "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "torch.manual_seed(0)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Loading Data" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "batch_size = 64\n", + "num_workers = 4\n", + "epochs = 5\n", + "lr = 1e-3" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "root_dir = \"./NMNIST\"\n", + "_ = NMNIST(save_to=root_dir, train=True)\n", + "_ = NMNIST(save_to=root_dir, train=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "n_time_steps = 50\n", + "to_raster = ToFrame(sensor_size=NMNIST.sensor_size, n_time_bins=n_time_steps)\n", + "\n", + "snn_train_dataset = NMNIST(save_to=root_dir, train=True, transform=to_raster)\n", + "snn_test_dataset = NMNIST(save_to=root_dir, train=False, transform=to_raster)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "snn_train_dataloader = DataLoader(snn_train_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True)\n", + "snn_test_dataloader = DataLoader(snn_test_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=False)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Network Module\n", + "\n", + "We need to define a `nn.Module` implementing the network we want the chip to reproduce." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "device: NVIDIA GeForce RTX 3070 Ti\n" + ] + } + ], + "source": [ + "if torch.cuda.is_available():\n", + " device = torch.device('cuda:0')\n", + " print('device: ', torch.cuda.get_device_name(0))\n", + "else:\n", + " device = torch.device('cpu')" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "class SNN(nn.Module):\n", + " def __init__(self) -> None:\n", + " super().__init__()\n", + "\n", + " self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False)\n", + " self.iaf1 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + " self.pool1 = nn.AvgPool2d(2,2)\n", + "\n", + " self.conv2 = nn.Conv2d(10, 10, 2, 1, bias=False)\n", + " self.iaf2 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + " self.pool2 = nn.AvgPool2d(3,3)\n", + "\n", + " self.conv3 = nn.Conv2d(10, 10, 3, 1, bias=False)\n", + " self.iaf3 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + " self.pool3 = nn.AvgPool2d(2,2)\n", + "\n", + " self.flat = nn.Flatten()\n", + "\n", + " self.fc1 = nn.Linear(10, 100, bias=False)\n", + " self.iaf4 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + "\n", + " self.fc2 = nn.Linear(100, 100, bias=False)\n", + " self.iaf5 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + "\n", + " self.fc3 = nn.Linear(100, 100, bias=False)\n", + " self.iaf6 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + "\n", + " self.fc4 = nn.Linear(100, 10, bias=False)\n", + " self.iaf7 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + "\n", + " def detach_neuron_states(self):\n", + " for name, layer in self.named_modules():\n", + " if name != '':\n", + " if isinstance(layer, sl.StatefulLayer):\n", + " for name, buffer in layer.named_buffers():\n", + " buffer.detach_()\n", + "\n", + " def init_weights(self):\n", + " for name, layer in self.named_modules():\n", + " if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear):\n", + " nn.init.xavier_normal_(layer.weight.data)\n", + "\n", + " def forward(self, x):\n", + " \n", + " con1_out = self.conv1(x)\n", + " iaf1_out = self.iaf1(con1_out)\n", + " pool1_out = self.pool1(iaf1_out)\n", + "\n", + " conv2_out = self.conv2(pool1_out)\n", + " iaf2_out = self.iaf2(conv2_out)\n", + " pool2_out = self.pool2(iaf2_out)\n", + "\n", + " conv3_out = self.conv3(pool2_out)\n", + " iaf3_out = self.iaf3(conv3_out)\n", + " pool3_out = self.pool3(iaf3_out)\n", + "\n", + " flat_out = self.flat(pool3_out)\n", + " \n", + " fc1_out = self.fc1(flat_out)\n", + " iaf4_out = self.iaf4(fc1_out)\n", + "\n", + " fc2_out = self.fc2(iaf4_out)\n", + " iaf5_out = self.iaf5(fc2_out)\n", + "\n", + " fc3_out = self.fc3(iaf5_out)\n", + " iaf6_out = self.iaf6(fc3_out)\n", + "\n", + " fc4_out = self.fc4(iaf6_out)\n", + " iaf7_out = self.iaf7(fc4_out)\n", + "\n", + " return iaf7_out" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "snn = SNN().to(device)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "snn.init_weights()" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "optimizer = Adam(snn.parameters(), lr=lr, betas=(0.9, 0.999), eps=1e-8)\n", + "loss_fn = CrossEntropyLoss()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Define train and test" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "def train(dataloader, model, loss_fn, optimizer, epochs, test_func, dataloader_test):\n", + " epochs_y = []\n", + " epochs_x = []\n", + " epochs_acc = []\n", + " model.train()\n", + "\n", + " for e in range(epochs):\n", + " losses = []\n", + " batches = []\n", + " batch_count = 0\n", + " train_p_bar = tqdm(snn_train_dataloader)\n", + "\n", + " for X, y in train_p_bar:\n", + " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", + " X = X.reshape(-1, 2, 34, 34).to(dtype=torch.float, device=device)\n", + " y = y.to(dtype=torch.long, device=device)\n", + "\n", + " # forward\n", + " pred = model(X)\n", + "\n", + " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", + " pred = pred.reshape(batch_size, n_time_steps, -1)\n", + "\n", + " # accumulate all time-steps output for final prediction\n", + " pred = pred.sum(dim = 1)\n", + " loss = loss_fn(pred, y)\n", + "\n", + " # gradient update\n", + " optimizer.zero_grad()\n", + " loss.backward()\n", + " optimizer.step()\n", + "\n", + " # detach the neuron states and activations from current computation graph(necessary)\n", + " model.detach_neuron_states()\n", + "\n", + " train_p_bar.set_description(f\"Epoch {e} - BPTT Training Loss: {round(loss.item(), 4)}\")\n", + "\n", + " batch_count += 1\n", + " losses.append(loss.item())\n", + " batches.append(batch_count)\n", + "\n", + " epochs_y.append(losses)\n", + " epochs_x.append(batches)\n", + "\n", + " acc = test_func(dataloader_test, model)\n", + " print(f'Epoch {e} accuracy: {acc}')\n", + " epochs_acc.append(acc)\n", + "\n", + " return epochs_x, epochs_y, epochs_acc\n" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "def test(dataloader, model):\n", + " correct_predictions = []\n", + " with torch.no_grad():\n", + " test_p_bar = tqdm(dataloader)\n", + " for X, y in test_p_bar:\n", + " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", + " X = X.reshape(-1, 2, 34, 34).to(dtype=torch.float, device=device)\n", + " y = y.to(dtype=torch.long, device=device)\n", + "\n", + " # forward\n", + " output = model(X)\n", + "\n", + " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", + " output = output.reshape(batch_size, n_time_steps, -1)\n", + "\n", + " # accumulate all time-steps output for final prediction\n", + " output = output.sum(dim=1)\n", + "\n", + " # calculate accuracy\n", + " pred = output.argmax(dim=1, keepdim=True)\n", + "\n", + " # compute the total correct predictions\n", + " correct_predictions.append(pred.eq(y.view_as(pred)))\n", + "\n", + " test_p_bar.set_description(f\"Testing Model...\")\n", + " \n", + " correct_predictions = torch.cat(correct_predictions)\n", + " return correct_predictions.sum().item()/(len(correct_predictions))*100" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Training loop (HPO)" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "392406a27a8146319e1aab0a40fe2f59", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/937 [00:00" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plt.plot(epochs_x[-1], epochs_y[-1])\n", + "plt.xlabel('batches')\n", + "plt.ylabel('loss')\n", + "plt.ylim(0,)\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAGwCAYAAABVdURTAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAABTAElEQVR4nO3de1xUZf4H8M/MwDCgMIhc5C4o4g1NQREVlRRMXVe7rJSu1qa/1rL1QmqirWu2vygvZW6pWZo/dxMpRXNXNsVU0CAvBOUVSRAQQQRluAk4zPn9gU6NXGSQ4TDD5/16zevVPPOcM9/HU83H55zzHIkgCAKIiIiITIRU7AKIiIiIWhPDDREREZkUhhsiIiIyKQw3REREZFIYboiIiMikMNwQERGRSWG4ISIiIpNiJnYBbU2j0eDGjRuwtraGRCIRuxwiIiJqBkEQUFZWBhcXF0ilTc/NdLhwc+PGDbi7u4tdBhEREbVAbm4u3NzcmuzT4cKNtbU1gLo/HBsbG5GrISIiouYoLS2Fu7u79ne8KR0u3Dw4FWVjY8NwQ0REZGSac0kJLyhuRGJiIiZPngwXFxdIJBLs37+/yf6xsbEIDQ2Fg4MDbGxsEBQUhEOHDtXrV1JSgnnz5sHZ2RkKhQJ9+vRBXFycgUZBRETU8TDcNKKiogIDBw7Exx9/3Kz+iYmJCA0NRVxcHFJSUhASEoLJkycjNTVV26empgahoaG4du0a9uzZg/T0dHz22WdwdXU11DCIiIg6nA53Wqq5JkyYgAkTJjS7/4YNG3Tev/vuu/jmm2/w73//G4MGDQIAbN++Hbdv30ZSUhLMzc0BAJ6enq1WMxEREXHmxmA0Gg3KyspgZ2enbTtw4ACCgoIwb948ODk5oX///nj33XdRW1srYqVERESmhTM3BrJ+/XpUVFRg2rRp2rbMzEwcPXoUM2bMQFxcHDIyMjBv3jyo1WqsXLlSxGqJiIhMB8ONAURHR2PVqlX45ptv4OjoqG3XaDRwdHTE1q1bIZPJ4O/vjxs3bmDt2rUMN0RERK2E4aaVxcTEYPbs2fj6668xbtw4nc+cnZ1hbm4OmUymbevTpw8KCgpQU1MDuVze1uUSERGZHF5z04qio6Px0ksvYdeuXZg0aVK9z0eMGIFffvkFGo1G23blyhU4Ozsz2BAREbUShptGlJeXIy0tDWlpaQCArKwspKWlIScnBwAQGRmJWbNmaftHR0dj1qxZWL9+PYYNG4aCggIUFBRApVJp+7z66qsoLi7GggULcOXKFRw8eBDvvvsu5s2b16ZjIyIiMmUSQRAEsYtoS6WlpVAqlVCpVE2uUHz8+HGEhITUa3/xxRexY8cOvPTSS7h27RqOHz8OABgzZgwSEhIa7f9AcnIyFi1ahLS0NLi6umL27Nl48803dU5VERERka7m/n4DDDettt/dp3MQGXsOAgCpBIh6xg/hQzxabf9EREQdmT6/3zwt1QryVXcRua8u2ACARgCWx55HvuquqHURERF1RAw3rSCrqAIPz3/VCgKuFVWKUxAREVEHxnDTCrzsO0H60ENKpRKgu72VOAURERF1YAw3rcBZaYmoZ/wg+81j2Ae628JZaSliVURERB0Tw00rCR/igZPLQvDu1P4AgLTcEqQXlIlcFRERUcfDcNOKnJWWmD7MExP6d4MgAOsOp4tdEhERUYfDcGMAb4T1glQCxF+8iR9z7ohdDhERUYfCcGMAPR2t8exgNwDA2m/T0cGWEiIiIhIVw42BLBjnA7lMiuTMYnz/S7HY5RAREXUYDDcG4tbFCtMD61YoXnvoMmdviIiI2gjDjQG9/mRPWMll+Om6CocuFIhdDhERUYfAcGNA9p0tMHukFwBg3eErqNVw9oaIiMjQGG4M7H9GecPWyhy/FJYj9sfrYpdDRERk8hhuDMxGYY5XR/cAAGw4koFqda3IFREREZk2UcNNYmIiJk+eDBcXF0gkEuzfv7/J/rGxsQgNDYWDgwNsbGwQFBSEQ4cOtU2xj2FWUHc4Wlsgr+Quok/liF0OERGRSRM13FRUVGDgwIH4+OOPm9U/MTERoaGhiIuLQ0pKCkJCQjB58mSkpqYauNLHYymXYf5YHwDAx8d+QUW1WuSKiIiITJdEaCf3KEskEuzbtw9Tp07Va7t+/fohPDwcK1eubPDz6upqVFdXa9+XlpbC3d0dKpUKNjY2j1OyXu7VajDugwRkF1dicVgvvP6kT5t9NxERkbErLS2FUqls1u+3UV9zo9FoUFZWBjs7u0b7REVFQalUal/u7u5tWOGvzGVSRIT2AgB8mpiJksoaUeogIiIydUYdbtavX4+KigpMmzat0T6RkZFQqVTaV25ubhtWqGvyABf07maNsio1tiRkilYHERGRKTPacBMdHY1Vq1YhJiYGjo6OjfazsLCAjY2NzkssUqkEi8N8AQA7krJQWFolWi1ERESmyijDTUxMDGbPno2vvvoK48aNE7scvYzt44jBHraouqfBxqMZYpdDRERkcowu3ERHR+Oll17Crl27MGnSJLHL0ZtEIsHSp3oDAHafzkVOcaXIFREREZkWUcNNeXk50tLSkJaWBgDIyspCWloacnLq1oKJjIzErFmztP2jo6Mxa9YsrF+/HsOGDUNBQQEKCgqgUqnEKL/Fhnl3RbCPPdQaAR8euSJ2OURERCZF1HBz9uxZDBo0CIMGDQIAREREYNCgQdrbuvPz87VBBwA+/fRTqNVqzJs3D87OztrXggULRKn/cSwdXzd7sz8tD5cLSkWuhoiIyHS0m3Vu2oo+98kb2mtfpiDuXAHG9XHC5y8GiFoLERFRe9Zh1rkxdhGhvpBKgCOXbuLHnDtil0NERGQSGG5E1NOxM54d7AYAWPttOjrYJBoREZFBMNyIbGFoL8hlUiRnFuPkL0Vil0NERGT0GG5E5mpriRnDPAAAaw9x9oaIiOhxMdy0A/NCesJKLsPP11U4dKFA7HKIiIiMGsNNO2Df2QKzR3oBANYdvoJaDWdviIiIWorhpp34n1HesLUyxy+F5Yj98brY5RARERkthpt2wkZhjldH9wAAbDiSgWp1rcgVERERGSeGm3bkxeHd4WRjgbySu4g+lfPoDYiIiKgehpt2RGEuw1+e9AEAfHzsF1RUq0WuiIiIyPgw3LQz4UPc4dnVCkXlNfji+yyxyyEiIjI6DDftjLlMiojQXgCATxMzUVJZI3JFRERExoXhph2aPMAFvbtZo6xKjS0JmWKXQ0REZFQYbtohqVSCxWG+AIAdSVkoLK0SuSIiIiLjwXDTTo3t44jBHraouqfBxqMZYpdDRERkNBhu2imJRIKlT/UGAOw+nYvs4gqRKyIiIjIODDft2DDvrhjVywFqjYANRzh7Q0RE1BwMN+3ckvvX3uxPy8PlglKRqyEiImr/GG7aOT83JSb6dYMgAOsOXRG7HCIionaP4cYIRIT6QioBjly6iZTsO2KXQ0RE1K4x3BiBno6d8Zy/GwBg7aHLEARB5IqIiIjaL4YbI7FgXC/IZVL8kHkbJ38pErscIiKidovhxki42lpixjAPAMDaQ+mcvSEiImoEw40RmRfSE1ZyGX6+rsK35wvELoeIiKhdYrgxIvadLTBnpBcAYN3hdNRqOHtDRET0MIYbIzNnlDdsrcxx9VYFYn+8LnY5RERE7Q7DjZGxUZjj1dE9AAAbjmSgWl0rckVERETtC8ONEXpxeHc42Vggr+Qudp3KEbscIiKidoXhxggpzGWYP9YHAPDJsV9QUa0WuSIiIqL2g+HGSE0LcIdnVysUldfgi++zxC6HiIio3WC4MVLmMikiQnsBAD5NzERJZY3IFREREbUPDDdGbPIAF/TuZo2yKjU2J1wVuxwiIqJ2geHGiEmlEiwZ7wsA+L+ka7hZWiVyRUREROJjuDFyT/Z2hL9nF1Td0+AfRzPELoeIiEh0DDdGTiL5dfZm9+lcZBdXiFwRERGRuBhuTMAw764Y1csBao2AD+OviF0OERGRqBhuTMTS+7M33/x0A5cLSkWuhoiISDwMNyaiv6sSk/ycIQjAukOcvSEioo6L4caELArtBakEOHLpJlKy74hdDhERkSgYbkxIT8fOeM7fDQCw9tBlCIIgckVERERtj+HGxCwY1wtymRQ/ZN7GyV+KxC6HiIiozTHcmBhXW0v8cZgnAGDtoXTO3hARUYfDcGOCXgvpASu5DD9fV+Hb8wVil0NERNSmGG5MkH1nC8wZ6QUAWHc4HepajcgVERERtR2GGxM1Z5Q3bK3McfVWBfal5oldDhERUZsRNdwkJiZi8uTJcHFxgUQiwf79+x+5TUJCAvz9/aFQKODt7Y0tW7YYvlAjZKMwx2tjegAANhzJQLW6VuSKiIiI2oao4aaiogIDBw7Exx9/3Kz+WVlZmDhxIoKDg5Gamorly5dj/vz52Lt3r4ErNU6zgrrDycYCeSV3setUjtjlEBERtQmJ0E5up5FIJNi3bx+mTp3aaJ8333wTBw4cwKVLl7Rtc+fOxU8//YTk5ORmfU9paSmUSiVUKhVsbGwet+x278tT2Vix7zy6dpIjcWkIOlmYiV0SERGR3vT5/Taqa26Sk5MRFham0zZ+/HicPXsW9+7da3Cb6upqlJaW6rw6kmkB7uje1QrFFTX44vssscshIiIyOKMKNwUFBXByctJpc3JyglqtRlFRwwvWRUVFQalUal/u7u5tUWq7YS6TYlFoLwDAp4mZKKmsEbkiIiIiwzKqcAPUnb76rQdn1R5ufyAyMhIqlUr7ys3NNXiN7c3kAS7o3c0aZVVqbE64KnY5REREBmVU4aZbt24oKNBdlK6wsBBmZmbo2rVrg9tYWFjAxsZG59XRSKUSLBnvCwDY8f013CytErkiIiIiwzGqcBMUFIT4+HidtsOHDyMgIADm5uYiVWUcnuztCH/PLqhWa/CPoxlil0NERGQwooab8vJypKWlIS0tDUDdrd5paWnIyam7bTkyMhKzZs3S9p87dy6ys7MRERGBS5cuYfv27di2bRsWL14sRvlGRSKRYOn92Zvdp3ORXVwhckVERESGIWq4OXv2LAYNGoRBgwYBACIiIjBo0CCsXLkSAJCfn68NOgDg5eWFuLg4HD9+HE888QTeeecdbNy4Ec8++6wo9RubQO+uGN3LAWqNgA/jr4hdDhERkUG0m3Vu2kpHW+fmYefzVPjdP05CIgH+uyAYvbt1vD8DIiIyPia7zg09vv6uSkzyc4YgAOsOcfaGiIhMD8NNBxQR1gsyqQRHLt1ESvYdscshIiJqVQw3HVAPh854brAbAGDtocvoYGcmiYjIxDHcdFDzx/lALpPih8zbOJHR8OrORERExojhpoNytbXEH4d5AgDWHkrn7A0REZkMhpsObF5ID3SSy3AuT4Vvzxc8egMiIiIjwHDTgXXtbIHZwd4AgHWH06Gu1YhcERER0eNjuOng5gR7wdbKHFdvVSA2NU/scoiIiB4bw00HZ6Mwx2tjegAAPjqSgWp1rcgVERERPR6GG8KsoO7oZqNAXsld7DqV8+gNiIiI2jGGG4LCXIb5Y30AAB8f/QUV1WqRKyIiImo5hhsCAPwhwA3du1qhuKIG209miV0OERFRizHcEADAXCbFotBeAICtiZkoqawRuSIiIqKWYbghrckDXNDH2QZl1WpsTrgqdjlEREQtwnBDWlKpBEvG183e7Pj+Gm6WVolcERERkf4YbkhHiK8j/D27oFqtwcbvMsQuh4iISG8MN6RDIpFg6XhfAEDMmVxkF1eIXBEREZF+GG6onkDvrhjdywFqjYAP46+IXQ4REZFeGG6oQUvuz95889MNXC4oFbkaIiKi5mO4oQb1d1Vikp8zBAFYdyhd7HKIiIiajeGGGhUR1gsyqQRHLhUiJfuO2OUQERE1C8MNNaqHQ2c8N9gNALD20GUIgiByRURERI/GcENNWjDOB3IzKX7IvI0TGUVil0NERPRIDDfUJBdbS8wc5gkAWHsonbM3RETU7jHc0CO9NqYHOsllOJenwrfnC8Quh4iIqEkMN/RIXTtbYHawNwBg3eF0qGs1IldERETUOIYbapb/CfZCFytzXL1VgdjUPLHLISIiahTDDTWLtcIcr43pCQD46EgGqtW1IldERETUMIYbaraZQZ7oZqNAXsldfPlDjtjlEBERNYjhhppNYS7D/LE+AIBPjv2Cimq1yBURERHVx3BDevlDgBu6d7VCcUUNtp/MErscIiKiehhuSC/mMikiwuoeqrk1MRN3KmpEroiIiEgXww3p7Xd+zujjbIOyajW2JFwVuxwiIiIdDDekN6lUgiXjewEAdiRdw83SKpErIiIi+hXDDbVIiK8jAjy7oFqtwcbvMsQuh4iISIvhhlpEIpFg6VO9AQAxZ3KRXVwhckVERER1GG6oxYZ62WF0LweoNQI+iL8idjlEREQAGG7oMS0ZX3fn1IGfbuBSfqnI1RARETHc0GPq76rEpAHOEARg/eF0scshIiJiuKHH90ZoL8ikEhy5VIiU7Ntil0NERB0cww09Nm+HznhusBsAYM236RAEQeSKiIioI2O4oVaxYJwP5GZSnMq6jRMZRWKXQ0REHRjDDbUKF1tLzBzmCQBYe4izN0REJB6GG2o1r43pgU5yGc7lqfDf8wVil0NERB0Uww21mq6dLTA72BsAsO5wOtS1GpErIiKijojhhlrV/wR7oYuVOTJvVSA2NU/scoiIqAMSPdxs2rQJXl5eUCgU8Pf3x4kTJ5rs/+WXX2LgwIGwsrKCs7Mz/vSnP6G4uLiNqqVHsVaY47UxPQEAHx3JQLW6VuSKiIiooxE13MTExGDhwoVYsWIFUlNTERwcjAkTJiAnJ6fB/idPnsSsWbMwe/ZsXLhwAV9//TXOnDmDOXPmtHHl1JSZQZ7oZqNAXsldfPlDw8eSiIjIUPQON3fv3kVlZaX2fXZ2NjZs2IDDhw/r/eUffPABZs+ejTlz5qBPnz7YsGED3N3dsXnz5gb7//DDD+jevTvmz58PLy8vjBw5En/+859x9uzZRr+juroapaWlOi8yLIW5DPPH+gAAPjn2C8qr1SJXREREHYne4WbKlCnYuXMnAKCkpASBgYFYv349pkyZ0mgoaUhNTQ1SUlIQFham0x4WFoakpKQGtxk+fDiuX7+OuLg4CIKAmzdvYs+ePZg0aVKj3xMVFQWlUql9ubu7N7tGark/BLihe1crFFfU4IuTWWKXQ0REHYje4ebHH39EcHAwAGDPnj1wcnJCdnY2du7ciY0bNzZ7P0VFRaitrYWTk5NOu5OTEwoKGr6NePjw4fjyyy8RHh4OuVyObt26wdbWFv/4xz8a/Z7IyEioVCrtKzc3t9k1UsuZy6SICKt7qObWxEzcqagRuSIiIuoo9A43lZWVsLa2BgAcPnwYzzzzDKRSKYYNG4bs7Gy9C5BIJDrvBUGo1/bAxYsXMX/+fKxcuRIpKSn49ttvkZWVhblz5za6fwsLC9jY2Oi8qG38zs8ZfZ1tUFatxpaEq2KXQ0REHYTe4aZnz57Yv38/cnNzcejQIe1ppcLCQr2Cg729PWQyWb1ZmsLCwnqzOQ9ERUVhxIgRWLJkCQYMGIDx48dj06ZN2L59O/Lz8/UdChmYVCrBkvF1szc7kq6hQFUlckVERNQR6B1uVq5cicWLF6N79+4IDAxEUFAQgLpZnEGDBjV7P3K5HP7+/oiPj9dpj4+Px/DhwxvcprKyElKpbskymQwAuNx/OzXG1wEBnl1QrdbgH0czxC6HiIg6AL3DzXPPPYecnBycPXsW3377rbZ97Nix+PDDD/XaV0REBD7//HNs374dly5dwqJFi5CTk6M9zRQZGYlZs2Zp+0+ePBmxsbHYvHkzMjMz8f3332P+/PkYOnQoXFxc9B0KtQGJRIKlT/UGAMScyUV2cYXIFRERkakza8lG3bp1Q7du3QAApaWlOHr0KHx9fdG7d2+99hMeHo7i4mKsXr0a+fn56N+/P+Li4uDpWfcAxvz8fJ01b1566SWUlZXh448/xhtvvAFbW1s8+eSTeP/991syDGojQ73sMMbXAcfTb+GD+Cv46Pnmz/ARERHpSyLoeT5n2rRpGDVqFF5//XXcvXsXAwcOxLVr1yAIAnbv3o1nn33WULW2itLSUiiVSqhUKl5c3IbO56nwu3+chEQCxM0PRh9n/tkTEVHz6fP7rfdpqcTERO2t4Pv27YMgCCgpKcHGjRvx97//vWUVk8nr76rEpAHOEARg/eF0scshIiITpne4UalUsLOzAwB8++23ePbZZ2FlZYVJkyYhI4MXjFLj3gjtBZlUgiOXCpGSfVvscoiIyETpHW7c3d2RnJyMiooKfPvtt9pbwe/cuQOFQtHqBZLp8HbojD/4uwEA1nybzjvciIjIIPQONwsXLsSMGTPg5uYGFxcXjBkzBkDd6So/P7/Wro9MzPyxPpCbSXEq6zYSM4rELoeIiEyQ3uHmtddeQ3JyMrZv346TJ09q153x9vbmNTf0SC62lpg5rO5uuLWHLnP2hoiIWp3ed0v91oNNG3tcQnvEu6XEV1xejVFrjqGiphabZgzGRD9nsUsiIqJ2zqB3SwHAzp074efnB0tLS1haWmLAgAH45z//2aJiqePp2tkCc4K9AQDrDqdDXasRuSIiIjIleoebDz74AK+++iomTpyIr776CjExMXjqqacwd+5cvVcopo5rTrAXuliZI/NWBWJ/zBO7HCIiMiF6n5by8vLC22+/rfNYBAD4v//7P6xatQpZWVmtWmBr42mp9uOzxEz8b9wluCgVOLZkDCzMZGKXRERE7ZRBT0vl5+c3+GDL4cOH88ncpJeZQZ7oZqPADVUVvvwh59EbEBERNYPe4aZnz5746quv6rXHxMTAx8enVYqijkFhLsOCcXX/znxy7BeUV6tFroiIiEyB3g/OfPvttxEeHo7ExESMGDECEokEJ0+exHfffddg6CFqynP+btiamImsogpsP5mF+WMZkImI6PHoPXPz7LPP4tSpU7C3t8f+/fsRGxsLe3t7nD59Gk8//bQhaiQTZi6TYlFoLwB11+DcqagRuSIiIjJ2j7XOjTHiBcXtj0Yj4Hf/OImL+aX48yhvRE7sI3ZJRETUzrT6BcWlpaXNfhHpSyqVYMl4XwDAjqRrKFBViVwREREZs2Zdc2Nra/vIVYgFQYBEIkFtbW2rFEYdyxhfBwzp3gVnrt3BxqMZePdpPqeMiIhaplnh5tixY4augzo4iUSCJeN7Y9qnyfjqTC5eCfZGd/tOYpdFRERGqFnhZvTo0YaugwhDvewwxtcBx9Nv4cMjV/DR84PELomIiIxQi54tRWQoi8Pqrr058NMNXMrnNVxERKQ/hhtqV/q7KvG7Ac4QBGDdoXSxyyEiIiPEcEPtTkRoL8ikEnx3uRAp2bfFLoeIiIwMww21O94OnfEHfzcAwJpv09HBlmIiIqLH1KJwo1arceTIEXz66acoKysDANy4cQPl5eWtWhx1XAvG+UBuJsWprNtIzCgSuxwiIjIieoeb7Oxs+Pn5YcqUKZg3bx5u3boFAFizZg0WL17c6gVSx+SstMSsYZ4AgLWHLkOj4ewNERE1j97hZsGCBQgICMCdO3dgaWmpbX/66afx3XfftWpx1LG9FtITneQynM8rxbcXCsQuh4iIjITe4ebkyZN46623IJfLddo9PT2Rl5fXaoUR2XWSY06wNwBg3eF0qGs1IldERETGQO9wo9FoGnzEwvXr12Ftbd0qRRE9MCfYC12szJF5qwKxPzI8ExHRo+kdbkJDQ7Fhwwbte4lEgvLycvztb3/DxIkTW7M2IlgrzDEvpCcAYMORK6i6x2eXERFR0/QONx9++CESEhLQt29fVFVVYfr06ejevTvy8vLw/vvvG6JG6uD+OMwTzkoFbqiqsOtUjtjlEBFROycRWrCIyN27dxEdHY0ff/wRGo0GgwcPxowZM3QuMG6vSktLoVQqoVKpYGNjI3Y51EzRp3MQGXsOXTvJkbA0BJ0tmvVYNCIiMhH6/H63KNwYM4Yb46Su1SD0w0RkFVUgIrQX5o/1EbskIiJqQ/r8fuv9198DBw402C6RSKBQKNCzZ094eXnpu1uiJpnJpIgI7YW/RKfis8RMzBzmiS6d5I/ekIiIOhy9w83UqVMhkUjqLYn/oE0ikWDkyJHYv38/unTp0mqFEk3yc8bm41dxMb8UWxKuInJiH7FLIiKidkjvC4rj4+MxZMgQxMfHQ6VSQaVSIT4+HkOHDsV//vMfJCYmori4mKsVU6uTSiVYMt4XALAj6RoKVFUiV0RERO2R3jM3CxYswNatWzF8+HBt29ixY6FQKPDKK6/gwoUL2LBhA15++eVWLZQIAMb4OmBI9y44c+0ONh7NwLtP+4ldEhERtTN6z9xcvXq1wQt5bGxskJmZCQDw8fFBUREfdkitTyKRYOlTvQEAX53JxbWiCpErIiKi9kbvcOPv748lS5ZoH5gJALdu3cLSpUsxZMgQAEBGRgbc3Nxar0qi3xjS3Q4hvg5QawR8eOSK2OUQEVE7o3e42bZtG7KysuDm5oaePXvCx8cHbm5uuHbtGj7//HMAQHl5Of7617+2erFED7wRVnftzYGfbuBSfqnI1RARUXvSonVuBEHAoUOHcOXKFQiCgN69eyM0NBRSqd5Zqc1xnRvT8fquH/Gfn/Mxtrcjtr00ROxyiIjIgLiIXxMYbkxHVlEFxn2QgFqNgD1zgxDQ3U7skoiIyEAMuogfAFRUVCAhIQE5OTmoqanR+Wz+/Pkt2SWR3rzsO2FagBuiT+dizaF0xLwyDBKJROyyiIhIZHqHm9TUVEycOBGVlZWoqKiAnZ0dioqKYGVlBUdHR4YbalPzx/pg7495OJ11G4kZRRjdy0HskoiISGR6XySzaNEiTJ48Gbdv34alpSV++OEHZGdnw9/fH+vWrTNEjUSNclZaYtYwTwDA2kOXodF0qLOsRETUAL3DTVpaGt544w3IZDLIZDJUV1fD3d0da9aswfLlyw1RI1GTXgvpic4WZjifV4r/ni8QuxwiIhKZ3uHG3Nxce12Dk5MTcnJyAABKpVL7z0Rtya6THHOC6x7Wuj4+HepajcgVERGRmPQON4MGDcLZs2cBACEhIVi5ciW+/PJLLFy4EH5++i+Fv2nTJnh5eUGhUMDf3x8nTpxosn91dTVWrFgBT09PWFhYoEePHti+fbve30umZfZIL3SxMkfmrQrE/pgndjlERCQivcPNu+++C2dnZwDAO++8g65du+LVV19FYWEhtm7dqte+YmJisHDhQqxYsQKpqakIDg7GhAkTmpwBmjZtGr777jts27YN6enpiI6ORu/evfUdBpkYa4U55oX0BABsOHIFVfdqRa6IiIjEotc6N4IgICcnB46OjrC0tHzsLw8MDMTgwYOxefNmbVufPn0wdepUREVF1ev/7bff4vnnn0dmZibs7Fq2pgnXuTFdVfdqEbLuOPJVVfjr7/pi9kgvsUsiIqJWos/vt14zN4IgwMfHB9evX3+sAgGgpqYGKSkpCAsL02kPCwtDUlJSg9scOHAAAQEBWLNmDVxdXdGrVy8sXrwYd+/ebfR7qqurUVpaqvMi06Qwl2HBWB8AwKZjv6C8Wi1yRUREJAa9wo1UKoWPjw+Ki4sf+4uLiopQW1sLJycnnXYnJycUFDR8x0tmZiZOnjyJ8+fPY9++fdiwYQP27NmDefPmNfo9UVFRUCqV2pe7u/tj107t13P+bvCy74TiihpsP5kldjlERCQCva+5WbNmDZYsWYLz58+3SgEPrygrCEKjq8xqNBpIJBJ8+eWXGDp0KCZOnIgPPvgAO3bsaHT2JjIyEiqVSvvKzc1tlbqpfTKTSRER2gsA8FliJu5U1DxiCyIiMjV6h5s//vGPOH36NAYOHAhLS0vY2dnpvJrL3t4eMpms3ixNYWFhvdmcB5ydneHq6gqlUqlt69OnDwRBaPRUmYWFBWxsbHReZNom+Tmjr7MNyqrV2JxwVexyiIiojen9+IUNGza0yhfL5XL4+/sjPj4eTz/9tLY9Pj4eU6ZMaXCbESNG4Ouvv0Z5eTk6d+4MALhy5QqkUinc3NxapS4yflKpBEue8sWfvjiD/0u6hpdHeKGbUiF2WURE1EZEfSp4TEwMZs6ciS1btiAoKAhbt27FZ599hgsXLsDT0xORkZHIy8vDzp07AQDl5eXo06cPhg0bhrfffhtFRUWYM2cORo8ejc8++6xZ38m7pToGQRAQ/ukPOH3tNqYHeuDdp/Vfg4mIiNoPg90t9cDVq1fx1ltv4YUXXkBhYSGAutu0L1y4oNd+wsPDsWHDBqxevRpPPPEEEhMTERcXB0/PumcF5efn66x507lzZ8THx6OkpAQBAQGYMWMGJk+ejI0bN7ZkGGTCJJK62RsA+OpMLq4VVYhcERERtRW9Z24SEhIwYcIEjBgxAomJibh06RK8vb2xZs0anD59Gnv27DFUra2CMzcdy5++OI1j6bfw+4Eu2PjCILHLISKiFjLozM2yZcvw97//HfHx8ZDL5dr2kJAQJCcn618tkQEtHl83e3Pgpxu4eINrHBERdQR6h5tz587pXAD8gIODQ6usf0PUmvq5KDF5oAsAYP3hdJGrISKitqB3uLG1tUV+fn699tTUVLi6urZKUUStKSK0F2RSCb67XIiz126LXQ4RERmY3uFm+vTpePPNN1FQUACJRAKNRoPvv/8eixcvxqxZswxRI9Fj8bLvhGkBdUsFrDmUDhFvECQiojagd7j53//9X3h4eMDV1RXl5eXo27cvRo0aheHDh+Ott94yRI1Ej23+WB/IzaQ4nXUbiRlFYpdDREQG1OJ1bq5evYrU1FRoNBoMGjQIPj4+rV2bQfBuqY7rfw9exGcnstDf1QYH5o2EVNrwYz6IiKj90ef3W+8VihMSEjB69Gj06NEDPXr0aHGRRG3t1TE9EX06F+fzSvHf8wWYNMBZ7JKIiMgA9D4tFRoaCg8PDyxbtqzVHp5J1BbsOskxJ9gLALA+Ph3qWo3IFRERkSHoHW5u3LiBpUuX4sSJExgwYAAGDBiANWvWNPrgSqL2ZE6wN+w6yZF5qwKxP+aJXQ4RERmA3uHG3t4er7/+Or7//ntcvXoV4eHh2LlzJ7p3744nn3zSEDUStdimTZvg5eUFhUIBf39/pJ5Oxmtj6k6nbjhyBVX3arV9jx8/DolEUu91+fJlnX3u3bsXffv2hYWFBfr27Yt9+/a16ZiIiKhpLXq21ANeXl5YtmwZ3nvvPfj5+SEhIaG16iJ6bDExMVi4cCFWrFiB1NRUBAcHY8KECRjlIoGzUoEbqip8eSqn3nbp6enIz8/Xvn57sXxycjLCw8Mxc+ZM/PTTT5g5cyamTZuGU6dOteXQiIioCS2+W+r777/Hl19+iT179qCqqgq///3vMWPGDEyYMKG1a2xVvFuq4wgMDMTgwYOxefNmbVufPn0wdepUDHz6VSyLPQe7TnIkLg1BZwszHD9+HCEhIbhz5w5sbW0b3Gd4eDhKS0vx3//+V9v21FNPoUuXLoiOjjb0kIiIOiyDPltq+fLl8PLywpNPPons7Gxs2LABBQUF+Ne//tXugw11HDU1NUhJSUFYWJhOe1hYGJKSkvCcvxu87TvhdkUNtp/M0ukzaNAgODs7Y+zYsTh27JjOZ8nJyfX2OX78eCQlJRlmIEREpDe9w83x48exePFi5OXl4eDBg5g+fTqsrKwMURtRixUVFaG2thZOTk467U5OTigoKICZTIqIsF4AgM8SM3GnogbOzs7YunUr9u7di9jYWPj6+mLs2LFITEzUbl9QUNDoPomIqH3Qe50b/g2VjIlEortQnyAI2raJ/Z3R1/kqLuaXYnPCVSyf2Ae+vr7avkFBQcjNzcW6deswatSoZu2TiIjEp3e4eeDixYvIyclBTU2NTvvvf//7xy6K6HHZ29tDJpPVm1EpLCzUzrxIpRIsecoXf/riDP4v6RpeHuGFbkqFTv9hw4bhX//6l/Z9t27dmtwnERGJT+9wk5mZiaeffhrnzp2DRCLRPoTwwd9ca2trm9qcqE3I5XL4+/sjPj4eTz/9tLY9Pj4eU6ZM0b4f08sBQ7vb4fS12/jouwxEPeOns5/U1FQ4O/+6knFQUBDi4+OxaNEibdvhw4cxfPhwA46GiIj0oXe4WbBgAby8vHDkyBF4e3vj9OnTKC4uxhtvvIF169YZokaiFomIiMDMmTMREBCAoKAgbN26FTk5OZg7dy4AIDIyEnl5eViyegP+sCUZn236B7zKn0RIkD9qamrwr3/9C3v37sXevXu1+1ywYAFGjRqF999/H1OmTME333yDI0eO4OTJk2INk4iIHqJ3uElOTsbRo0fh4OAAqVQKqVSKkSNHIioqCvPnz0dqaqoh6iTSW3h4OIqLi7F69Wrk5+ejf//+iIuLg6enJwAgPz8fOTk5GNLdDiG+Dth/6h6WLFmCKlURrCwt0b9/Pxw8eBATJ07U7nP48OHYvXs33nrrLfz1r39Fjx49EBMTg8DAQLGGSURED9F7nZsuXbogJSUF3t7e6NGjBz7//HOEhITg6tWr8PPzQ2VlpaFqbRVc54YacuGGCpM2/jr7IpUAUc/4IXyIh4hVERHRAwZ9Knj//v3x888/w9vbG4GBgVizZg3kcjm2bt0Kb2/vFhdNJCa7TnKd9xoBiIw9h1G9HOCstBSpKiIiagm917l56623oNHUPU3573//O7KzsxEcHIy4uDhs3Lix1QskagtZRRX12jQCMP2zU4g+nYOKarUIVRERUUu0+PELv3X79m106dLFKNb64Gkpaki+6i5GvHcUmkb+a+hsYYapg1wwfagn+rrw3xsioramz+93q4QbY8JwQ42JOZOD5bHnUSsIkEkkWDGpD9QaDaJP5+rM7DzhbosZgR743QAXWMplIlZMRNRxMNw0geGGmpKvuotrRZXobm+lvdZGEAQkXy3Gl6dzcOh8AdT3p3esFWZ4drAbpgd6oJeTtZhlExGZPIabJjDc0OO4VVaNr1NyEX06B7m372rbh3TvgumBHpjQ3xkKc87mEBG1NoabJjDcUGvQaASc+KUIu05l48ilQtTen82xtTLHc4Pd8EKgB3o4dBa5SiIi08Fw0wSGG2ptN0urEHMmF7tP5+CGqkrbPszbDjMCPTG+XzfIzfS+MZGIiH6D4aYJDDdkKLUaAQlXCvHlDzk4ll6ovfOqayc5/hDgjheGusOzaydxiyQiMlIMN01guKG2kFdyFzFnchFzJgc3S6u17cE+9pgR6IGxfZxgLuNsDhFRczHcNIHhhtqSulaD7y4XYtepHCRm3MKD/9ocrC0QHuCO54e6w62LlbhFEhEZAYabJjDckFhyb1ci+nQOvjp7HUXldbM5EgkwppcDpgd6IsTXAWaczSEiahDDTRMYbkhsNWoN4i/exK7T2fj+l2Jtu7NSgfAh7ggf4s7nWRERPYThpgkMN9SeZBVVIPp0DvakXMftihoAdU8kf7K3E2YM88AoHwfIpO3/sSZERIbGcNMEhhtqj6rVtfj2fAF2ncrBqazb2nZXW0u8MNQd0wLc4WijELFCIiJxMdw0geGG2rtfCsuw61Qu9qTkorSq7mnkZlIJQvs6YXqgB0b0sIeUszlE1MEw3DSB4YaMRdW9Whz8OR+7TucgJfuOtt2zqxVeGOqB5/zdYN/ZQsQKiYjaDsNNExhuyBhdLijFrlM52PdjHsqq62ZzzGUSjO/XDTMCPTHM2w4SCWdziMh0Mdw0geGGjFlljRr/+SkfX57Kxk/XVdp2b4dOmH5/NsfWSi5ihUREhsFw0wSGGzIV5/NU2HU6B9+k5qGiphYAIDeTYpKfM6YHeiDAswtnc4jIZDDcNIHhhkxNebUa36TlYdepHFy4Uapt7+XUGdOHeuDpwW5QWpqLWCER0eNjuGkCww2ZKkEQ8NN1FXadysaBn26g6p4GAKAwl2LyABdMD/TAE+62nM0hIqPEcNMEhhvqCFR372F/at1sTvrNMm17H2cbTA/0wNQnXGCt4GwOERkPhpsmMNxQRyIIAn7MuYMvf8jBf87lo0ZdN5tjJZdhyhMumD7UE35uSpGrJCJ6NIabJjDcUEdVUlmDvT/mYdepbFy9VaFtH+CmxPShHvj9Ey6wkpuJWCERUeP0+f0W/RHEmzZtgpeXFxQKBfz9/XHixIlmbff999/DzMwMTzzxhGELJDIRtlZyzB7phSMRo7H7lWH4/UAXmMsk+Pm6CstizyHwf7/DX/efx6X80kfvjIioHRN15iYmJgYzZ87Epk2bMGLECHz66af4/PPPcfHiRXh4eDS6nUqlwuDBg9GzZ0/cvHkTaWlpzf5OztwQ/aq4vBp7Uq4j+nQOrhVXatsHedhiRqAnfjfAGQpzmYgVEhHVMZrTUoGBgRg8eDA2b96sbevTpw+mTp2KqKioRrd7/vnn4ePjA5lMhv379zPcED0mjUZA0tVi7DqdjcMXbkKtqfvfgo3CDM8MdsOMQA/4OFmLXCURdWRGcVqqpqYGKSkpCAsL02kPCwtDUlJSo9t98cUXuHr1Kv72t78163uqq6tRWlqq8yIiXVKpBCN97LFphj+SIp/EkvG+cOtiidIqNXYkXUPoh4mYtiUZ36TloVpdK3a5RERNEu3qwaKiItTW1sLJyUmn3cnJCQUFBQ1uk5GRgWXLluHEiRMwM2te6VFRUXj77bcfu16ijsLRWoF5IT3x6ugeSMy4hV2ncvDd5UKcvnYbp6/dRhcrczzn74YXhnrA26Gz2OUSEdUj+q0RDy8oJghCg4uM1dbWYvr06Xj77bfRq1evZu8/MjISERER2velpaVwd3dvecFEHYRUKsEYX0eM8XVEgaoKMWdysftMDvJVVfjsRBY+O5GF4T26YnqgB8L6doPcTPT7E4iIAIgYbuzt7SGTyerN0hQWFtabzQGAsrIynD17FqmpqXj99dcBABqNBoIgwMzMDIcPH8aTTz5ZbzsLCwtYWFgYZhBEHUQ3pQILxvlgXkgPHE+/hV2nc3AsvRBJV4uRdLUY9p3l+EOAO14Y4gGPrlZil0tEHZzoFxT7+/tj06ZN2ra+fftiypQp9S4o1mg0uHjxok7bpk2bcPToUezZswdeXl7o1KnTI7+TFxQTtY7rdyoRcyYXMWdyUVhWrW0P9rHHjEAPjO3jBHMZZ3OIqHXo8/st6mmpiIgIzJw5EwEBAQgKCsLWrVuRk5ODuXPnAqg7pZSXl4edO3dCKpWif//+Ots7OjpCoVDUayciw3PrYoU3wnwxf6wPvrtUiC9PZeNERpH25WhtgfAh7nh+qAdcbS3FLpeIOhBR/1oVHh6ODRs2YPXq1XjiiSeQmJiIuLg4eHp6AgDy8/ORk5MjZolE9AjmMime6t8N/5wdiMQlIXh1TA/Yd5ajsKwa/zj6C4LfP4qXd5zBkYs3UavRf6JYn4U+T548iREjRqBr166wtLRE79698eGHHzbaf/fu3ZBIJJg6daredRFR+8XHLxBRq6tRa3D4YgF2ncpB0tVibbuLUoHwIR4IH+KObkrFI/ej70KfqampuHz5MgYMGIBOnTrh5MmT+POf/4wPP/wQr7zyik7f7OxsjBgxAt7e3rCzs8P+/fsfe9xEZDhGs4ifGBhuiNpW5q1yRJ/OwZ6U67hTeQ8AIJNK8GRvR8wI9MAoHwdIpfXvkARavtDnbz3zzDPo1KkT/vnPf2rbamtrMXr0aPzpT3/CiRMnUFJSwnBD1M4ZxSJ+RNQxeDt0xopJfZEcORYbwp/A0O52qNUIiL94Ey99cQaj1h7DJ8d+QWFZlc52LV3o87dSU1ORlJSE0aNH67SvXr0aDg4OmD179uMNjojaJdHXuSGijkFhLsPUQa6YOsgVGTfLsOt0DvamXMf1O3ex9lA6Poy/grB+Tpg+1BPDe3Rt0UKfD7i5ueHWrVtQq9VYtWoV5syZo/3s+++/x7Zt2/R6bAsRGReGGyJqcz5O1vjb5H5YOr43Dp7Lx65T2fgxpwRx5woQd64A3btaYYJ33fpUzV3o87dOnDiB8vJy/PDDD1i2bBl69uyJF154AWVlZfjjH/+Izz77DPb29gYbHxGJi+GGiERjKZfhOX83POfvhkv5pdh1Kgf7UvNwrbgSmwpVgESKv3+dhL869UKglx0kEkmjC33+lpeXFwDAz88PN2/exKpVq/DCCy/g6tWruHbtGiZPnqztq9FoAABmZmZIT09Hjx49DDdgImoTDDdE1C70cbbBO1P7Y9mE3vj3Tzew63QOCrr1ROLxo3he3hs9HDrhhaEeOHT4MJ7W49ZtQRBQXV23yGDv3r1x7tw5nc/feustlJWV4aOPPuKjWYhMBMMNEbUrnSzM8PxQDzw/1ANrpcuwbMErsHbrhcuOvojYvQHlV6/hlmswzl67jT1b1uDGjRvYuXMnAOCTTz6Bh4cHevfuDaBu3Zt169bhL3/5CwA0uOinra0tAHAxUCITwnBDRO3Wktf+hE64i/ffX4Mb+fmwcuoOxz+swpHrAo5sSca9oz/BWn0HpVX3YKMwx52Kaqx/YykK8nJgbmaGHj164L333sOf//xnsYdCRG2I69wQkdEQBAFpuSXYdSoH//75Bqru1V0vozCXor+LEik5dyAIgFQCRD3jh/Ah9Rf6IyLjxEX8msBwQ2QaVHfvYd+P17HrdA6u3Cyv97lEAkT/zzAM7W7X6CKBRGQ8GG6awHBDZFoEQcCOpGt4+98XG/zc2sIM/V2VGOCmxAA3WwxwU8Kti+UjbycnovbFaJ4KTkT0uCQSCZ7q3w3v/OciHn4up1wmQVm1GsmZxUjO/PUZV12szOHnZosBvwk9TjYWDDxEJoIzN0RkEmLO5GB57HnUCgJkEgnefaY/nh3shozCcvx8vQQ/X1fh5+sqXC4oxb3a+v/bc7C2wEA3Jfxcbe8HHiW6drYQYSRE1BCelmoCww2R6cpX3cW1okp0t7eCs9KywT7V6lqkF5Thp+sqnLsfejIKy1H78LQPAFdbSwxwU8LPTYkBrrbwc1NCaWlu6GEQUQMYbprAcENED7tbU4uL+Sr8lKvCuTwVfr5egsyiCjT0f8fuXa201+74uSrR31WJThY8w09kaAw3TWC4IaLmKKu6h/N5pXWntPJUOHddhZzblfX6SSRAT4fOvwYeNyX6OttAYS4ToWoi08Vw0wSGGyJqqTsVNdqZnZ+v183y5Kuq6vUzk0rQy8kaA91/vYbHt5s1zGVSEaomMg0MN01guCGi1lRYWoVzeSqda3iKK2rq9ZObSdHH2eb+Rct1d2j1dOwMGdfgIWoWhpsmMNwQkSEJgoAbqiqcu15yP/DUzfSUVqnr9bU0l6G/q43ONTzdu3biooNEDWC4aQLDDRG1NUEQkF1cef/anbrQcz5Phcqa2np9rRVm2pmdB4GHiw4SMdw0ieGGiNqDWo2AzFvl2mt3frpegos3SlGt1tTra9dJDj9XZd0prfuhx8lGIULVROJhuGkCww0RtVf3ajW4crOs7lTW/QuXL+eXQd3AGjxONhbwc7W9H3jqZnrsOslFqPrRNm3ahLVr1yI/Px/9+vXDhg0bEBwc3GDf2NhYbN68GWlpaaiurka/fv2watUqjB8/XqdfSUkJVqxYgdjYWNy5cwdeXl5Yv349Jk6c2BZDIhHw8QtEREbIXCZFPxcl+rko8fz9tqp7tbhcUKa9WLlu0cEy3Cytxs3Smzhy6aZ2e7cu9xcdvB96+rspYaMQd9HBmJgYLFy4EJs2bcKIESPw6aefYsKECbh48SI8POo/tT0xMRGhoaF49913YWtriy+++AKTJ0/GqVOnMGjQIABATU0NQkND4ejoiD179sDNzQ25ubmwtrZu6+FRO8WZGyIiI1NZo8bFG6U6d2hlFlU02NfbvhP87l+7M9DdFv1cbGAlb7u/1wYGBmLw4MHYvHmztq1Pnz6YOnUqoqKimrWPfv36ITw8HCtXrgQAbNmyBWvXrsXly5dhbs4VozsKztwQEZkwK7kZArrbIaC7nbattOoezufVzeycu153Dc/1O3eRWVSBzKIKfJN2AwAglQA+jtbwc/v1Gp7e3awNsuhgTU0NUlJSsGzZMp32sLAwJCUlNWsfGo0GZWVlsLP7dawHDhxAUFAQ5s2bh2+++QYODg6YPn063nzzTchkXDyRGG6IiEyCjcIcw3vYY3gPe23b7Yoa/Hy9ROcanpul1Ui/WYb0m2XYk3IdAGAuk8C3m7XOQ0N7OT3+ooNFRUWora2Fk5OTTruTkxMKCgqatY/169ejoqIC06ZN07ZlZmbi6NGjmDFjBuLi4pCRkYF58+ZBrVZrZ3eoY2O4ISIyUXad5Bjj64gxvo7atpulVdq1d36+P9Nzu6IG5/NKcT6vFNGn6/pZmEnR18UGA1zrZncGuinh7dCyRQcfvo1dEIRm3doeHR2NVatW4ZtvvoGj469j0Gg0cHR0xNatWyGTyeDv748bN25g7dq1DDcEgOGGiKhDcbJRwKmvAuP61s2mCIKAvJK7909lqXAur+4anrIqNVJzSpCaUwIgGwDQSS5DP1fl/cCjxEA3W3h2tWo0qNjb20Mmk9WbpSksLKw3m/OwmJgYzJ49G19//TXGjRun85mzszPMzc11TkH16dMHBQUFqKmpgVzePu8ao7bDcENE1IFJJBK4dbGCWxcrTPBzBgBoNAKyb1f++gyt6yqcv6FCRU0tTmfdxums29rtbRRmGOBmW3c7uqsSA9xt4aJUQCKRQC6Xw9/fH/Hx8Xj66ae128THx2PKlCmN1hQdHY2XX34Z0dHRmDRpUr3PR4wYgV27dkGj0UAqrTt1duXKFTg7OzPYEADeLSV2OURERqFWI+Dq/UUHH4Sei/mlqGlg0cGuneT3n5Bui9s/H8OayL9gy5YtCAoKwtatW/HZZ5/hwoUL8PT0RGRkJPLy8rBz504AdcFm1qxZ+Oijj/DMM89o92lpaQmlUgkAyM3NRd++ffHSSy/hL3/5CzIyMvDyyy9j/vz5WLFiRdv8gXQgrb1OUWxsLN5991388ssvuHfvHnx8fPDGG29g5syZTdbBRfyawHBDRNQ67tVqkF5QpvOk9PSC+osOlv14EOVnYqEuvw1Xr15Y/Ld3MXPqU+jSSY6XXnoJ165dw/HjxwEAw0eOQvL3J+p914svvogdO3Zo3ycnJ2PRokVIS0uDq6srZs+ezbulDCAmJgYzZ87UWafo888/b3SdooULF8LFxQUhISHadYrWrVuns07R8ePHcefOHfTu3RtyuRz/+c9/8MYbb+DgwYP1Fmv8LYabJjDcEBEZTtW9WlzKL617pERu3TU8vxSWo4FFluFuZ4kB9+/Q8nNTIuNmGd7+90VohLpb1qOe8UP4kPo/oNR2DLFOUUMGDx6MSZMm4Z133mm0D9e5ISIiUSjMZRjk0QWDPLoAQXVtFdVqXLhR+us1PHkqZBVVIPf2XeTevouD5/Lr7UcjAMv2nkNWUQXculjB1socSkvdl7XCvEV3b1HzGGqdot8SBAFHjx5Feno63n///ceu+QGGGyIiMqhOFmYY6mWHoV6//sCp7v666ODP10twJus2iipqdLYTAGxJyGx0vxIJ0NnCTBt2fhuAbCzrhyGlpTlsLeX3g5EZpAxGTTLUOkUAoFKp4OrqiurqashkMmzatAmhoaGtVjvDDRERtTmlpTlG9LTHiJ51iw7mq+5ixHtHdU5fSQBM8OuGe7UCVHfvofTuPajuvypraiEIQFmVGmVValy/c1ev75dIAGsLMygbmBGy+U0IaujV0YJRa69TBADW1tZIS0tDeXk5vvvuO0RERMDb2xtjxoxplZoZboiISHTOSktEPeOH5bHnUSsIkEkkePeZ/o1ec1Oj1miDzsPBp96rUvf93Xt1wai0So3SKjVy0bJgZGslrxeKGpwtsvr1c2sL4wlGhlqnCACkUil69uwJAHjiiSdw6dIlREVFMdwQEZFpCR/igVG9HHCtqBLd7a3grLRstK/cTAoHaws4WFvo/T2NBaOSyhqo7qqbDE0PByN9SSWAtaKBENTADNLDwamtg5Gh1ilqiCAIqK6ufuyaH2C4ISKidsNZadlkqGkNjxOMqtW1Dc8UVd7TBqOSuzUNziRV3dNAI0D7Xl9SCerNDjU0W2T78GdWdcGoOaeSHhYREYGZM2ciICBAu05RTk4O5s6dCwBNrlM0bNgw7azPb9cpioqKQkBAAHr06IGamhrExcVh586dOndkPS6GGyIiomayMJPB0VoGR2uF3ts+HIxKKhs+jfbbYPSgT7W6LhiVVNa16auhYPSomSJbK3NMnPIMPvywCKtXr0Z+fj769++PuLg4eHp6AgDy8/ORk5Oj/Z5PP/0UarUa8+bNw7x587Ttv12nqKKiAq+99hquX78OS0tL9O7dG//6178QHh6u97gaw3VuiIiI2rmqe7VNXldUUtn4dUfVDawirQ+ZVAIbhVmjs0W/vUstJfsOPj+ZBcEAaxVxnRsiIiITojCXQWEug6ON/jNGD4JRSSMXWNebLfrNP9eoNajVCLhTeQ939Jwx0gjA8tjzGNXLweCnGh/GcENERGTCHjcYNXTnWUkDoej6nUpcuVmus32tIOBaUSXDDREREbUPD4KRUzOCUUNrFckkEnS3tzJghQ2Ttvk3EhERkcl5sFaR7P5dWQ/WKmrrWRuAMzdERETUSvRZq8iQGG6IiIio1bTFWkWPIvppqU2bNsHLywsKhQL+/v44ceJEo31jY2MRGhoKBwcH2NjYICgoCIcOHWrDaomIiKi9EzXcxMTEYOHChVixYgVSU1MRHByMCRMm6CwI9FuJiYkIDQ1FXFwcUlJSEBISgsmTJyM1NbWNKyciIqL2StRF/AIDAzF48GCdJZf79OmDqVOnIioqqln76NevH8LDw7Fy5coGP6+urtZ5XkVpaSnc3d25iB8REZER0WcRP9FmbmpqapCSkoKwsDCd9rCwMCQlJTVrHxqNBmVlZbCzs2u0T1RUFJRKpfbl7u7+WHUTERFR+yZauCkqKkJtbW29x6Y7OTnVe7x6Y9avX4+KigpMmzat0T6RkZFQqVTaV25u7mPVTURERO2b6HdLPfyUUkEQmvXk0ujoaKxatQrffPMNHB0dG+1nYWEBCwv9n/xKRERExkm0cGNvbw+ZTFZvlqawsLDebM7DYmJiMHv2bHz99dcYN26cIcskIiIiIyPaaSm5XA5/f3/Ex8frtMfHx2P48OGNbhcdHY2XXnoJu3btwqRJkwxdJhERERkZUU9LRUREYObMmQgICEBQUBC2bt2KnJwczJ07F0Dd9TJ5eXnYuXMngLpgM2vWLHz00UcYNmyYdtbH0tISSqVStHEQERFR+yFquAkPD0dxcTFWr16N/Px89O/fH3FxcfD09AQA5Ofn66x58+mnn0KtVmPevHmYN2+etv3FF1/Ejh072rp8IiIiaodEXedGDPrcJ09ERETtg1Gsc0NERERkCAw3REREZFIYboiIiMikMNwQERGRSWG4ISIiIpPCcENEREQmheGGiIiITArDDREREZkUhhsiIiIyKQw3REREZFIYboiIiMikMNwQERGRSWG4ISIiIpPCcENEREQmheGGiIiITArDDREREZkUhhsiIiIyKQw3REREZFIYboiIiMikMNwQERGRSWG4ISIiIpPCcENEREQmheGGiIiITArDDREREZkUhhsiIiIyKQw3REREZFIYboiIiMikMNwQERGRSWG4ISIiIpPCcENEREQmheGGiIiITArDDREREZkUhhsiIiIyKQw3REREZFIYboiIiMikMNwQERGRSWG4ISIiIpPCcENEREQmheGGiIiITArDDREREZkUhhsiIiIyKQw3REREZFIYboiIiMikMNwQERGRSWG4ISIiIpMierjZtGkTvLy8oFAo4O/vjxMnTjTZPyEhAf7+/lAoFPD29saWLVvaqFIiIiIyBqKGm5iYGCxcuBArVqxAamoqgoODMWHCBOTk5DTYPysrCxMnTkRwcDBSU1OxfPlyzJ8/H3v37m3jyomIiKi9kgiCIIj15YGBgRg8eDA2b96sbevTpw+mTp2KqKioev3ffPNNHDhwAJcuXdK2zZ07Fz/99BOSk5Ob9Z2lpaVQKpVQqVSwsbF5/EEQERGRwenz+23WRjXVU1NTg5SUFCxbtkynPSwsDElJSQ1uk5ycjLCwMJ228ePHY9u2bbh37x7Mzc3rbVNdXY3q6mrte5VKBaDuD4mIiIiMw4Pf7ebMyYgWboqKilBbWwsnJyeddicnJxQUFDS4TUFBQYP91Wo1ioqK4OzsXG+bqKgovP322/Xa3d3dH6N6IiIiEkNZWRmUSmWTfUQLNw9IJBKd94Ig1Gt7VP+G2h+IjIxERESE9r1Go8Ht27fRtWvXJr+nJUpLS+Hu7o7c3FyTPOVl6uMDTH+MHJ/xM/UxcnzGz1BjFAQBZWVlcHFxeWRf0cKNvb09ZDJZvVmawsLCerMzD3Tr1q3B/mZmZujatWuD21hYWMDCwkKnzdbWtuWFN4ONjY3J/ksLmP74ANMfI8dn/Ex9jByf8TPEGB81Y/OAaHdLyeVy+Pv7Iz4+Xqc9Pj4ew4cPb3CboKCgev0PHz6MgICABq+3ISIioo5H1FvBIyIi8Pnnn2P79u24dOkSFi1ahJycHMydOxdA3SmlWbNmafvPnTsX2dnZiIiIwKVLl7B9+3Zs27YNixcvFmsIRERE1M6Ies1NeHg4iouLsXr1auTn56N///6Ii4uDp6cnACA/P19nzRsvLy/ExcVh0aJF+OSTT+Di4oKNGzfi2WefFWsIOiwsLPC3v/2t3mkwU2Hq4wNMf4wcn/Ez9TFyfMavPYxR1HVuiIiIiFqb6I9fICIiImpNDDdERERkUhhuiIiIyKQw3BAREZFJYbjR06ZNm+Dl5QWFQgF/f3+cOHGiyf4JCQnw9/eHQqGAt7c3tmzZ0kaVtow+4zt+/DgkEkm91+XLl9uw4uZLTEzE5MmT4eLiAolEgv379z9yG2M7fvqO0ZiOYVRUFIYMGQJra2s4Ojpi6tSpSE9Pf+R2xnQMWzJGYzqGmzdvxoABA7SLuwUFBeG///1vk9sY0/HTd3zGdOwaEhUVBYlEgoULFzbZT4xjyHCjh5iYGCxcuBArVqxAamoqgoODMWHCBJ3b1X8rKysLEydORHBwMFJTU7F8+XLMnz8fe/fubePKm0ff8T2Qnp6O/Px87cvHx6eNKtZPRUUFBg4ciI8//rhZ/Y3t+AH6j/EBYziGCQkJmDdvHn744QfEx8dDrVYjLCwMFRUVjW5jbMewJWN8wBiOoZubG9577z2cPXsWZ8+exZNPPokpU6bgwoULDfY3tuOn7/geMIZj97AzZ85g69atGDBgQJP9RDuGAjXb0KFDhblz5+q09e7dW1i2bFmD/ZcuXSr07t1bp+3Pf/6zMGzYMIPV+Dj0Hd+xY8cEAMKdO3faoLrWBUDYt29fk32M7fg9rDljNOZjWFhYKAAQEhISGu1j7MewOWM05mMoCILQpUsX4fPPP2/wM2M/foLQ9PiM9diVlZUJPj4+Qnx8vDB69GhhwYIFjfYV6xhy5qaZampqkJKSgrCwMJ32sLAwJCUlNbhNcnJyvf7jx4/H2bNnce/ePYPV2hItGd8DgwYNgrOzM8aOHYtjx44Zssw2ZUzH73EZ4zFUqVQAADs7u0b7GPsxbM4YHzC2Y1hbW4vdu3ejoqICQUFBDfYx5uPXnPE9YGzHbt68eZg0aRLGjRv3yL5iHUOGm2YqKipCbW1tvYd6Ojk51XuY5wMFBQUN9ler1SgqKjJYrS3RkvE5Oztj69at2Lt3L2JjY+Hr64uxY8ciMTGxLUo2OGM6fi1lrMdQEARERERg5MiR6N+/f6P9jPkYNneMxnYMz507h86dO8PCwgJz587Fvn370Ldv3wb7GuPx02d8xnbsAGD37t348ccfERUV1az+Yh1DUR+/YIwkEonOe0EQ6rU9qn9D7e2FPuPz9fWFr6+v9n1QUBByc3Oxbt06jBo1yqB1thVjO376MtZj+Prrr+Pnn3/GyZMnH9nXWI9hc8dobMfQ19cXaWlpKCkpwd69e/Hiiy8iISGh0QBgbMdPn/EZ27HLzc3FggULcPjwYSgUimZvJ8Yx5MxNM9nb20Mmk9WbxSgsLKyXSh/o1q1bg/3NzMzQtWtXg9XaEi0ZX0OGDRuGjIyM1i5PFMZ0/FpTez+Gf/nLX3DgwAEcO3YMbm5uTfY11mOozxgb0p6PoVwuR8+ePREQEICoqCgMHDgQH330UYN9jfH46TO+hrTnY5eSkoLCwkL4+/vDzMwMZmZmSEhIwMaNG2FmZoba2tp624h1DBlumkkul8Pf3x/x8fE67fHx8Rg+fHiD2wQFBdXrf/jwYQQEBMDc3NxgtbZES8bXkNTUVDg7O7d2eaIwpuPXmtrrMRQEAa+//jpiY2Nx9OhReHl5PXIbYzuGLRljQ9rrMWyIIAiorq5u8DNjO34NaWp8DWnPx27s2LE4d+4c0tLStK+AgADMmDEDaWlpkMlk9bYR7Rga9HJlE7N7927B3Nxc2LZtm3Dx4kVh4cKFQqdOnYRr164JgiAIy5YtE2bOnKntn5mZKVhZWQmLFi0SLl68KGzbtk0wNzcX9uzZI9YQmqTv+D788ENh3759wpUrV4Tz588Ly5YtEwAIe/fuFWsITSorKxNSU1OF1NRUAYDwwQcfCKmpqUJ2drYgCMZ//ARB/zEa0zF89dVXBaVSKRw/flzIz8/XviorK7V9jP0YtmSMxnQMIyMjhcTERCErK0v4+eefheXLlwtSqVQ4fPiwIAjGf/z0HZ8xHbvGPHy3VHs5hgw3evrkk08ET09PQS6XC4MHD9a5RfPFF18URo8erdP/+PHjwqBBgwS5XC50795d2Lx5cxtXrB99xvf+++8LPXr0EBQKhdClSxdh5MiRwsGDB0Wounke3Hb58OvFF18UBME0jp++YzSmY9jQuAAIX3zxhbaPsR/DlozRmI7hyy+/rP3/i4ODgzB27FjtD78gGP/x03d8xnTsGvNwuGkvx1AiCPev7CEiIiIyAbzmhoiIiEwKww0RERGZFIYbIiIiMikMN0RERGRSGG6IiIjIpDDcEBERkUlhuCEiIiKTwnBDREREJoXhhog6vOPHj0MikaCkpETsUoioFTDcEBERkUlhuCEiIiKTwnBDRKITBAFr1qyBt7c3LC0tMXDgQOzZswfAr6eMDh48iIEDB0KhUCAwMBDnzp3T2cfevXvRr18/WFhYoHv37li/fr3O59XV1Vi6dCnc3d1hYWEBHx8fbNu2TadPSkoKAgICYGVlheHDhyM9Pd2wAycig2C4ISLRvfXWW/jiiy+wefNmXLhwAYsWLcIf//hHJCQkaPssWbIE69atw5kzZ+Do6Ijf//73uHfvHoC6UDJt2jQ8//zzOHfuHFatWoW//vWv2LFjh3b7WbNmYffu3di4cSMuXbqELVu2oHPnzjp1rFixAuvXr8fZs2dhZmaGl19+uU3GT0Sti08FJyJRVVRUwN7eHkePHkVQUJC2fc6cOaisrMQrr7yCkJAQ7N69G+Hh4QCA27dvw83NDTt27MC0adMwY8YM3Lp1C4cPH9Zuv3TpUhw8eBAXLlzAlStX4Ovri/j4eIwbN65eDcePH0dISAiOHDmCsWPHAgDi4uIwadIk3L17FwqFwsB/CkTUmjhzQ0SiunjxIqqqqhAaGorOnTtrXzt37sTVq1e1/X4bfOzs7ODr64tLly4BAC5duoQRI0bo7HfEiBHIyMhAbW0t0tLSIJPJMHr06CZrGTBggPafnZ2dAQCFhYWPPUYialtmYhdARB2bRqMBABw8eBCurq46n1lYWOgEnIdJJBIAddfsPPjnB347KW1padmsWszNzevt+0F9RGQ8OHNDRKLq27cvLCwskJOTg549e+q83N3dtf1++OEH7T/fuXMHV65cQe/evbX7OHnypM5+k5KS0KtXL8hkMvj5+UGj0ehcw0NEposzN0QkKmtrayxevBiLFi2CRqPByJEjUVpaiqSkJHTu3Bmenp4AgNWrV6Nr165wcnLCihUrYG9vj6lTpwIA3njjDQwZMgTvvPMOwsPDkZycjI8//hibNm0CAHTv3h0vvvgiXn75ZWzcuBEDBw5EdnY2CgsLMW3aNLGGTkQGwnBDRKJ755134OjoiKioKGRmZsLW1haDBw/G8uXLtaeF3nvvPSxYsAAZGRkYOHAgDhw4ALlcDgAYPHgwvvrqK6xcuRLvvPMOnJ2dsXr1arz00kva79i8eTOWL1+O1157DcXFxfDw8MDy5cvFGC4RGRjvliKidu3BnUx37tyBra2t2OUQkRHgNTdERERkUhhuiIiIyKTwtBQRERGZFM7cEBERkUlhuCEiIiKTwnBDREREJoXhhoiIiEwKww0RERGZFIYbIiIiMikMN0RERGRSGG6IiIjIpPw/eoVjEwZCx54AAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "y_avg = []\n", + "for y in epochs_y:\n", + " y_avg.append(np.mean(y))\n", + "\n", + "plt.plot(np.arange(len(epochs_x)), y_avg, marker = '.')\n", + "plt.xlabel('epoch')\n", + "plt.ylabel('average loss')\n", + "plt.ylim(0,)\n", + "for i, txt in enumerate(y_avg):\n", + " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom')\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkkAAAG2CAYAAABrrBJlAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAABcAklEQVR4nO3deVxU5f4H8M8wwLCjgAKDyKIIiog7omaaa+4rpKWmWd703iTLNbXUlNS0Ukuvv1tRWmqmqDf1CuYWSeYCuOCCioAKorIM6wAz5/cHOTkyIIMDMwOf9+vF68bxOc98n063+fScc55HJAiCACIiIiJSY6LvAoiIiIgMEUMSERERkQYMSUREREQaMCQRERERacCQRERERKQBQxIRERGRBgxJRERERBowJBERERFpwJBEREREpAFDEhEREZEGeg1JJ0+exLBhwyCVSiESibB37161PxcEAR999BGkUiksLS3Ru3dvXL58Wa2NXC7Hv/71Lzg5OcHa2hrDhw/HnTt3nvnZX331Fby8vGBhYYFOnTrht99+0+XQiIiIyMjpNSQVFBQgMDAQGzdu1Pjnq1evxrp167Bx40acOXMGLi4u6N+/P/Ly8lRtwsLCEBkZiR07diAmJgb5+fkYOnQoFApFpZ+7c+dOhIWF4YMPPkBcXBxeeOEFvPzyy0hNTdX5GImIiMg4iQxlg1uRSITIyEiMHDkSQPksklQqRVhYGObNmwegfNbI2dkZq1atwvTp05Gbm4smTZpg69atCA0NBQDcu3cP7u7uOHjwIAYOHKjxs4KCgtCxY0ds2rRJdax169YYOXIkwsPDa3egREREZBRM9V1AZZKTk5GRkYEBAwaojkkkErz44os4deoUpk+fjnPnzqG0tFStjVQqRdu2bXHq1CmNIamkpATnzp3D/Pnz1Y4PGDAAp06dqrQeuVwOuVyu+l2pVCIrKwuOjo4QiUTPM1QiIiKqI4IgIC8vD1KpFCYmVd9QM9iQlJGRAQBwdnZWO+7s7IyUlBRVG3NzczRu3LhCm8fnP+3hw4dQKBQa+63sHAAIDw/H0qVLtR4HERERGZ60tDQ0a9asyjYGG5Iee3qWRhCEZ87cVKeNtv0uWLAAs2fPVv2em5uL5s2bIy0tDXZ2dlV+FhERERkGmUwGd3d32NraPrOtwYYkFxcXAOWzRa6urqrjmZmZqlkgFxcXlJSUIDs7W202KTMzE927d9fYr5OTE8RicYVZoyf71UQikUAikVQ4bmdnx5BERERkZKrzqIzBrpPk5eUFFxcXREdHq46VlJTgxIkTqgDUqVMnmJmZqbVJT0/HpUuXKg1J5ubm6NSpk9o5ABAdHV3pOURERNTw6HUmKT8/Hzdu3FD9npycjPj4eDg4OKB58+YICwvDypUr4ePjAx8fH6xcuRJWVlaYMGECAMDe3h5vvPEG3nvvPTg6OsLBwQHvv/8+AgIC0K9fP1W/ffv2xahRo/DPf/4TADB79mxMnDgRnTt3RnBwMLZs2YLU1FT84x//qNu/AURERGSw9BqSzp49iz59+qh+f/zMz+TJkxEREYG5c+eiqKgIM2bMQHZ2NoKCghAVFaV2H/Gzzz6DqakpQkJCUFRUhL59+yIiIgJisVjV5ubNm3j48KHq99DQUDx69AjLli1Deno62rZti4MHD8LDw6MORk1ERETGwGDWSTI2MpkM9vb2yM3N5TNJRERERkKb72+DfSaJiIiISJ8YkoiIiIg0YEgiIiIi0oAhiYiIiEgDhiQiIiIiDRiSiIiIiDRgSCIiIiLSgCGJiIiISAOGJCIiIiINGJKIiIiINGBIIiIiItKAIYmIiIhIA4YkIiIiIg0YkoiIiIg0YEgiIiIi0oAhiYiIiEgDhiQiIiIiDRiSiIiIiDRgSCIiIiLSgCGJiIiISAOGJCIiIiINGJKIiIiINGBIIiIiItKAIYmIiIhIA4YkIiIiIg0YkoiIiIg0YEgiIiIi0oAhiYiIiEgDhiQiIiIiDRiSiIiIiDRgSCIiIiLSgCGJiIiISAOGJCIiIiINGJKIiIiINGBIIiIiItLA4ENSXl4ewsLC4OHhAUtLS3Tv3h1nzpxR/blIJNL4s2bNmkr7jIiI0HhOcXFxXQyJiIiIjICpvgt4lmnTpuHSpUvYunUrpFIptm3bhn79+iExMRFubm5IT09Xa3/o0CG88cYbGDNmTJX92tnZ4dq1a2rHLCwsdF4/ERERGSeDDklFRUXYvXs39u3bh169egEAPvroI+zduxebNm3Cxx9/DBcXF7Vz9u3bhz59+sDb27vKvkUiUYVziYiIiB4z6NttZWVlUCgUFWZ4LC0tERMTU6H9/fv3ceDAAbzxxhvP7Ds/Px8eHh5o1qwZhg4diri4uCrby+VyyGQytR8iIiKqvww6JNna2iI4OBjLly/HvXv3oFAosG3bNpw+fbrCbTYA+O6772Bra4vRo0dX2a+fnx8iIiKwf/9+bN++HRYWFujRoweSkpIqPSc8PBz29vaqH3d39+ceHxERERkukSAIgr6LqMrNmzcxdepUnDx5EmKxGB07dkSrVq1w/vx5JCYmqrX18/ND//79sWHDBq0+Q6lUomPHjujVqxfWr1+vsY1cLodcLlf9LpPJ4O7ujtzcXNjZ2Wk/MCIiIqpzMpkM9vb21fr+NuhnkgCgRYsWOHHiBAoKCiCTyeDq6orQ0FB4eXmptfvtt99w7do17Ny5U+vPMDExQZcuXaqcSZJIJJBIJFr3TURERMbJoG+3Pcna2hqurq7Izs7G4cOHMWLECLU///rrr9GpUycEBgZq3bcgCIiPj4erq6uuyiUiIiIjZ/AzSYcPH4YgCPD19cWNGzcwZ84c+Pr6YsqUKao2MpkMu3btwtq1azX2MWnSJLi5uSE8PBwAsHTpUnTr1g0+Pj6QyWRYv3494uPj8eWXX9bJmIiIiMjwGXxIys3NxYIFC3Dnzh04ODhgzJgxWLFiBczMzFRtduzYAUEQMH78eI19pKamwsTk70mznJwcvPXWW8jIyIC9vT06dOiAkydPomvXrrU+HiIiIjIOBv/gtqHS5sEvIiIiMgzafH8bzTNJRERERHWJIYmIiIhIA4YkIiIiIg0YkoiIiIg0YEgiIiIi0oAhiYiIiEgDhiQiIiIiDRiSiIiIiDRgSCIiIiLSgCGJiIiISAOGJCIiIiINGJKIiIiINGBIIiIiItKAIYmIiIhIA4YkIiIiIg0YkoiIiIg0YEgiIiIi0oAhiYiIiEgDhiQiIiIiDRiSiIiIiDRgSCIiIiLSgCGJiIiISAOGJCIiIiINGJKIiIiINGBIIiIiItKAIYmIiIhIA4YkIiIiIg0YkoiIiIg0YEgiIiIi0oAhiYiIiEgDhiQiIiIiDRiSiIiIiDRgSCIiIiLSgCGJiIiISAOGJCIiIiINDD4k5eXlISwsDB4eHrC0tET37t1x5swZ1Z+//vrrEIlEaj/dunV7Zr+7d+9GmzZtIJFI0KZNG0RGRtbmMIiIiMjIGHxImjZtGqKjo7F161ZcvHgRAwYMQL9+/XD37l1Vm0GDBiE9PV31c/DgwSr7jI2NRWhoKCZOnIiEhARMnDgRISEhOH36dG0Ph4iIiIyESBAEQd9FVKaoqAi2trbYt28fhgwZojrevn17DB06FB9//DFef/115OTkYO/evdXuNzQ0FDKZDIcOHVIdGzRoEBo3bozt27dXqw+ZTAZ7e3vk5ubCzs6u2p9NRERE+qPN97dBzySVlZVBoVDAwsJC7bilpSViYmJUvx8/fhxNmzZFq1at8OabbyIzM7PKfmNjYzFgwAC1YwMHDsSpU6d0VzwREREZNYMOSba2tggODsby5ctx7949KBQKbNu2DadPn0Z6ejoA4OWXX8YPP/yAo0ePYu3atThz5gxeeuklyOXySvvNyMiAs7Oz2jFnZ2dkZGRUeo5cLodMJlP7ISIiovrLoEMSAGzduhWCIMDNzQ0SiQTr16/HhAkTIBaLAZTfOhsyZAjatm2LYcOG4dChQ7h+/ToOHDhQZb8ikUjtd0EQKhx7Unh4OOzt7VU/7u7uzz84IiIiMlgGH5JatGiBEydOID8/H2lpafjzzz9RWloKLy8vje1dXV3h4eGBpKSkSvt0cXGpMGuUmZlZYXbpSQsWLEBubq7qJy0trWYDIiIiIqNg8CHpMWtra7i6uiI7OxuHDx/GiBEjNLZ79OgR0tLS4OrqWmlfwcHBiI6OVjsWFRWF7t27V3qORCKBnZ2d2g8RERHVX6b6LuBZDh8+DEEQ4Ovrixs3bmDOnDnw9fXFlClTkJ+fj48++ghjxoyBq6srbt++jYULF8LJyQmjRo1S9TFp0iS4ubkhPDwcADBr1iz06tULq1atwogRI7Bv3z4cOXJE7WFwIiIiatgMfiYpNzcXM2fOhJ+fHyZNmoSePXsiKioKZmZmEIvFuHjxIkaMGIFWrVph8uTJaNWqFWJjY2Fra6vqIzU1VfWgNwB0794dO3bswLfffot27dohIiICO3fuRFBQkD6GSERET3jWIsJPmj59OkQiET7//PNn9vusRYS1+VxqGAx+JikkJAQhISEa/8zS0hKHDx9+Zh/Hjx+vcGzs2LEYO3bs85ZHREQ6Nm3aNFy6dAlbt26FVCrFtm3b0K9fPyQmJsLNzU3Vbu/evTh9+jSkUukz+3y8iPDy5csxatQoREZGIiQkBDExMar/QK7u51LDYdCLSRoyLiZJRKR71VlEGADu3r2LoKAgHD58GEOGDEFYWBjCwsIq7fdZiwhX93PJ+NWbxSSJiKhhqc4iwkqlEhMnTsScOXPg7+9frX6ftYhwdRcvpoaFIYmIiAxGdRYRXrVqFUxNTfHOO+9Uu99nLSJcnc+lhochiYiIDEpViwifO3cOX3zxBSIiIqpcAFiTZy0i/KzFi6nhYUgiIiKDUtUiwr/99hsyMzPRvHlzmJqawtTUFCkpKXjvvffg6elZaZ/VWURY28WLqf5jSCIiIoOkaRHhiRMn4sKFC4iPj1f9SKVSzJkzp8q3nbVZRLi6ixdT/WfwSwAQEVHDUtUiwmZmZnB0dFRrb2ZmBhcXF/j6+qqO1WQR4ao+lxomziQREZFBqWoR4eq6ces2Eq4nIz23CED1FhHWxedS/cJ1kmqI6yQRERmmnWdSsWDPRSgFwEQEhI8OQGiX5vouiwwE10kiIqIGRxAEnLrxEPN3lwckAFAKwII9F3ElXabf4sgo8ZkkIiIySkqlgCsZMvyZnIXTt7Lw5+0sZBWUVGwnAC9/8Rua2krQ2tUOfq62aO1S/r8tmtjATMz5AtKMIYmIiIxCmUKJy/dkOJ38CKdvZeHM7SzIisvU2khMRZCXaX6KJDNPjsy8Bzhx/YHqmJlYhJZNbdHaxbY8PLnawc/FDk1sJbU6FjIODElERGSQSsqUuHAnB6eTs3A6OQvnbmehoESh1sbaXIxOng4I8ir/adesESLj7mDhnktQCALEIhFWjm6LIe2kuJaRh6sZMlxN//t/8+RluJIuK78dF/d3v0425vBzsYOfi61q9qllUxtITLmwZEPCB7driA9uExHpVnGpAnGpOaqZori0bBSXKtXa2FmYoquXA7p6OSDIyxH+UjuYarhdlp5bhNsPC+HpZAVXe0uNnycIAu5kF+FqRh6upstw5a/glPyoAJq+GcUmIrRoYq2abXp8287ZTqL16t+kP9p8fzMk1RBDEhHR8ymQl+FcSnb5M0XJj5CQlosShXoocrA2R1dPBwR5lwcjPxc7iE1qN5AUlShw/X4erqTLcDUjTzXT9PStvccaW5mphSY/V1u0craFhRlnnQwRQ1IdYEgiItJOblEpzqWUP2T9R3IWLt3NhUKp/hXU1FaCIG9HdPVyQDcvB7RsamMQszSCICBDVvxXYMpTzT7delhQYQxA+dIDXk7W8HO1Q2vVLTs7SO0tDGI8DRlDUh1gSCIiqlpWQQn+TM5SzRQlpssq3MZya2RZ/jyRtwO6ejnC09HKqEJEcakCNzLzK8w6ZReWamxva2GK1i52aO1qCz/X8meefF1sYWXOR4TrCkNSHWBIIiJSl5lXrHod/3TyI1y/n1+hjaejFYK8ymeKgrwd0KyxlR4qrV2CIOBBnhxXHj/r9FeAupGZjzINs04iEeDhYFXhWadmjS1hUsu3FhsihqQ6wJBERA3dvZwinE5+pApGtx4WVGjj09Tmr0DkiK6eDnCxt9BDpYahpEyJmw/UZ52uZuThQZ5cY3sbiSl8XWzh51I+69Tmr2edbC0Mf5uUvLw8LF68GJGRkcjMzESHDh3wxRdfoEuXLigtLcWiRYtw8OBB3Lp1C/b29ujXrx8++eQTSKXSSvssLS1FeHg4vvvuO9y9exe+vr5YtWoVBg0aVK3PfUyb72/O7xER0TMJgoDUrMLy1/H/mim6k12k1kYkAvxc7FSv43fxcoCTDdcbeszc1AStXe3Q2lX9i/lhvly1LMGV9PLwdCMzH/l/Pdh+LiVbrb27gyX8XMr7af1XgPJwsDKoWadp06bh0qVL2Lp1K6RSKbZt24Z+/fohMTERNjY2OH/+PBYvXozAwEBkZ2cjLCwMw4cPx9mzZyvtc9GiRdi2bRv+7//+D35+fjh8+DBGjRqFU6dOoUOHDs/8XDc3N63HwZmkGuJMEhHVZ4Ig4OaDArWZogxZsVobsYkIbaV2qtfxu3g6wN7K8Gc5jEGpQonkhwXqs07peRWuwWOWZmL4utiWP+v01/pOfi52erkeRUVFsLW1xb59+zBkyBDV8fbt22Po0KH4+OOPK5xz5swZdO3aFSkpKWjeXPM+e1KpFB988AFmzpypOjZy5EjY2Nhg27Zt1f5cziQREZFWlEoB1+7nqR6y/jM5Cw/z1bf4MBOL0K5ZIwT9tU5RZ08H2Ej4NVIbzMQmaOVcfnttxBPHswtKnrhVVx6grmXkoahUgfi0HMSn5aj149bI8q/bdbaq2SdPRyuNa0vpSllZGRQKBSws1G+tWlpaIiYmRuM5ubm5EIlEaNSoUaX9yuXyKvusyec+C2eSaogzSURkzBRKAYmPt/hILt/iI+epN7LMTU3Qwb0Rgrwd0c3LAR2aN4alOdf+MTQKpYDkhwVqq4lfSc/D3Zwije0lpuUB7MnVxFu72KGxtbnOaurevTvMzc3x448/wtnZGdu3b8ekSZPg4+ODa9euqbUtLi5Gz5494efnh23btlXa54QJE5CQkIC9e/eiRYsW+PXXXzFixAgoFArI5fJqfy4f3K4DDElEZExKFUpcuJOrmik6dzsbeXL1xREtzcTo7Nn4r8UbHRHobs9tOIxYblGpaiuWx886PZ510sTZTvL3s05/zTx5N7Gu0QbAN2/exNSpU3Hy5EmIxWJ07NgRrVq1wvnz55GYmKhqV1painHjxiE1NRXHjx+v8vv0wYMHePPNN/Hf//4XIpEILVq0QL9+/fDtt9+isLCw2p/LkFQHGJKIyJAVlyqQkFa+79mfyVk4l5Jd4cvRVmKKzp6NEeTtiCAvB7R1s6/RFyIZD6Wy/AH8qxkyJKaXL1FwNSMPqVmFGtubi03QsqmN2mrirV3tqv1AfkFBAWQyGVxdXREaGor8/HwcOHAAQHlACgkJwa1bt3D06FE4OjpWq8/i4mI8evQIUqkU8+fPxy+//ILLly9X+3P5TBIRNVi18eoxAOTk5OCDDz7Anj17kJ2dDS8vL6xduxaDBw9Wtfnqq6+wZs0apKenw9/fH59//jleeOGF2h4yAKCwpKx837Nbj/BHchbi03JQUqa+xUcjKzN09Sx/nqibtyNau9b+Fh9kWExMRPB0soankzUGtXVVHc+Xl+FaxuPVxB/ftstDvrwMiekyJKbLANxVtXeykfw122SrWt+pRVPrCjOP1tbWsLa2RnZ2Ng4fPozVq1cD+DsgJSUl4dixY9UOSABgYWEBNzc3lJaWYvfu3QgJCanQprLP1RZnkmqIM0lEhik0NBSXLl3Cpk2bVK8Af/bZZ6pXj8eOHYs333xT7dXjsrKyKl89LikpQY8ePdC0aVMsXLgQzZo1Q1paGmxtbREYGAgA2LlzJyZOnIivvvoKPXr0wL///W/85z//QWJiYqVv6zyPvOJSnE3JxulbWfgz+REu3MmtsFChk41EtZp1kJcjfJraGNRr4mTYHm8A/PgNu8e37W5XsgGwqYkILZqUzzoJafFo1tgSvbu2R05GKubOnQuJRIKYmBiIRCKMGTMG58+fxy+//AJnZ2dVHw4ODjA3L382atKkSXBzc0N4eDgA4PTp07h79y7at2+Pu3fv4qOPPkJycjLOnz+veuD78OHDEAQBvr6+uHHjBubMmaP6XDOz8jf9eLutDjAkERme2nr1ePPmzVizZg2uXr2q+hft04KCgtCxY0ds2rRJdax169YYOXKk6l/yzyOnsARnbmfj9K3yB60v38vF04s3u9pb/PXmmSOCvB3g7WRtVFt8kHEoLCnD9fv5qtXEH68s/uQGwAVXfkPOye9QlvcQppZ28OrcB2Onz0H7llLYlWbj5e6BGvs+duwYevfuDQDo3bs3PD09ERERAQA4ceIE3n77bdy6dQs2NjYYPHhwhVngn376CQsWLMCdO3fg4OCAMWPGYMWKFbC3t1e1YUiqAwxJRIYnLy8PdnZ2OHLkCPr27as6HhwcDIlEguPHj1c458iRIxgwYABycnIq/f/y4MGD4eDgACsrK+zbtw9NmjTBhAkTMG/ePIjFYpSUlMDKygq7du3CqFGjVOfNmjUL8fHxOHHihNZjeZgv/2t9ovJQdO1+XoX/em/uYPXXGkXlM0XuDpYMRaQXgiAgPbe4wmritx7kVwjzQPkGwN5NbJ64XVf+v65PbACcnluE5IcF8HKyhqu9pc5q5TNJRNQg2draIjg4GMuXL0fr1q1VrwCfPn0aPj4+FdoXFxdj/vz5mDBhQpX/snz8YOmrr76KgwcPIikpCTNnzkRZWRmWLFmChw8fQqFQqN02AABnZ2dkZGRUq/aM3GLV6/inbz3CzQcVt/jwbmKNIC9H1TpF0ka6++Igeh4ikQjSRpaQNrJE39Z///+guFSBpPv5uPLXc07lM08y5BSW4kZmPm5k5uOXC+mq9nYWpvBztYOpCIi9lQUB5YEqfHQAQrvo/rb1szAkEVG9snXrVkydOhVubm6qV4AnTJiA8+fPq7UrLS3FK6+8AqVSia+++qrKPpVKJZo2bYotW7ZALBajU6dOuHfvHtasWYMlS5ao2j09iyMIQqUzO2l/bfHx51/BKOVRxbeL/Fxs/17N2qsxmto23H3PyDhZmIkR0MweAc3+vt0lCAIy8+QVVhO/+SAfsuIy/JmcpdaHUgAW7rmEXq2a6HRGqToYkoioXmnRogVOnDhR4RVgLy8vVZvHb9YkJyfj6NGjz5xyd3V1hZmZGcTiv9/cad26NTIyMlBSUgInJyeIxeIKs0aZmZlwdnaGIJQv9le+RlH5K/lPL/RnIgLaSO0Q5OWIrl4O6OrpoNPF/YgMhUgkgrOdBZztLNDbt6nquLxMgZuZBfjvhXvYdPym2jkKQcDth4UMSUREuqDLV4979OiBH3/8EUqlEiYm5esIXb9+Ha6urqo3cTp16oTo6GiMGjUKSqWAGw/y8fP+g3AJ6ImuK3+tsNO72ESEADd7BHk7oJuXIzp5NoadEezuTlRbJKZitJHaobG1Gf594qbas0xikQieTlZ1XhNDEhHVK5peAfb19cWUKVNQVlaGsWPHql49VigUqtmfql49fvvtt7FhwwbMmjUL//rXv5CUlISVK1finXfeAVC+LcS41/+B+e9Mx+USJzyw8sCd2P8i/+4diAb1gmmeHOZiE7R3b4Qg7/LniTo2bwxr7ntGVIGrvSXCRwdg4Z5LUAgCxCIRVo5uW+ezSABDEhHVM7m5uRpfATYzM8Pt27exf/9+AOXLAjzpyVePU1NTVTNGAODu7o6oqCi8++67aNeuHdzc3BDy+ltwCB6HNyLO4M/bWcgrbgr7PtPw+65/Q1GQBUkTT7w0ax2GDeyJIG8HtHdvBAszbvFBVB2hXZqjV6smuP2wEJ5OVnoJSACXAKgxLgFAVH89/eqxvEyh2vfsj1uPcC4lG4Ul6lt8WJuL0dnz8cKNDghwawRzU27xQWRo6tUSALWxxUBERASmTJlS4XhRUREsLPj2CFFDtvNMKhbsuQilAIgAeDlZ425OEeRPbfFhZ2GqevMsyNsBbVztYMp9z4jqFYMPSdOmTcOlS5ewdetW1RYD/fr1U20xcP78eSxevFhti4Hhw4dXucUAANjZ2eHatWtqxxiQiBq2s7ezMH/3RTyeXhcA3HpYvl6Ro7W5auHGrl6O8HOx5RYfRPWcQd9uq60tBiIiIhAWFoacnJwa18bbbUT1g0Ip4Pi1TGz9IwXHrz3Q2ObTce0wpmMzrmZNVA/Um9ttZWVlUCgUFWZ4LC0tERMTo/Gc3NxciEQi1WZ3lcnPz4eHhwcUCgXat2+P5cuXo0OHDpW2l8vlkMv/foVXJpNVfyBEZHAe5svx09k0/PBHaoU1i54kFonQo6UTAxJRA6T1DXRNex/Vlie3GLh37x4UCgW2bduG06dPIz09vUL76m4x4Ofnh4iICOzfvx/bt2+HhYUFevTogaSkpErPCQ8Ph729verH3d1dJ2MkorojCALOpWQhbEccuocfxer/XcPdnCLYW5rhzRe8cPz93lg1JgDivwKRPl89JiL90/p2m4WFBdzc3DBlyhRMnjy51sPCzZs3MXXqVJw8eVK1xUCrVq1w/vx5JCYmqtqVlpZi3LhxSE1NxfHjx7W6BaZUKtGxY0f06tUL69ev19hG00ySu7s7b7cRGYECeRn2xt/F1tgUXM3IUx0PdG+E14KaY1igVO31/PTcIr2/ekxEtaNWb7fdu3cP27ZtQ0REBD766CP07dsXb7zxBkaOHKlaiE2XamOLgaeZmJigS5cuVc4kSSQSSCSSGo+DiOpe0v08bPsjBbvP30W+vAwAIDE1wYj2UrzWzQPtmjXSeJ6rvSXDERE934Pb8fHx+Oabb7B9+3YolUq8+uqreOONNxAYGKjLGtVkZ2fDy8sLq1evxltvvVVhi4EmTZpo3acgCOjatSsCAgLwzTffVOscPrhNZJhKFUpEXb6PrX/cxh+3/t4o08vJGq8GNcfYTs3QyIp7ohE1VNp8fz/322337t3Dli1b8Mknn8DU1BTFxcUIDg7G5s2b4e/v/zxdA9C8xYBEIkFMTAxEIhHGjBmj2mLA2dlZdV5VWwwsXboU3bp1g4+PD2QyGdavX4+tW7fi999/R9euXatVF0MSkWFJzy3C9j/TsOPPVGT+tU+aiQjo19oZE4M90KOFE1/ZJ6Laf7uttLQU+/btwzfffIPo6Gh07twZGzduxPjx45GVlYV58+Zh3Lhxas8M1VRtbDGQk5ODt956CxkZGbC3t0eHDh1w8uTJagckIjIMgiDg1M1H2Bqbgugr96H4a0dMJxsJxnd1x/iuzSFtxNtmRFQzWs8k/etf/8L27dsBAK+99hqmTZuGtm3bqrVJTU2Fp6cnlEqlpi7qBc4kEelPblEpdp+7g22nU3DrQYHqeJCXAyYGe2BAGxduCUJEGtXqTFJiYiI2bNiAMWPGVPqgtlQqxbFjx7TtmoioSpfu5mJrbAr2JdxFcWn5f4TZSEwxuqMbXuvmgVbOtnqukIjqE4NecduQcSaJqG4Ulypw4EI6tv6Rgvi0HNVxPxdbvNbNAyM7uMFGYtDr4hKRAanVmaTw8HA4Oztj6tSpase/+eYbPHjwAPPmzdO2SyKiClIfFeKH0yn46WwasgtLAQBmYhFebuuKicEe6OzRmKtgE1Gt0jok/fvf/8aPP/5Y4bi/vz9eeeUVhiQiqrEn91E7cf0BHs9zuzWyxISg5gjp7I4mtlyvjIjqhtYhKSMjA66urhWON2nSRONWIUREz1LZPmovtmqCid080MevKcR8fZ+I6pjWIcnd3R2///672orXAPD7779DKpXqrDAiqt/K91HLxtY/UnDwYjpKFeXTRo2szBDS2R0TujaHp5O1nqskooZM65A0bdo0hIWFobS0FC+99BIA4Ndff8XcuXPx3nvv6bxAIqpfqtpHbWI3Dwxt56q2jxoRkb5oHZLmzp2LrKwszJgxAyUlJQDKN72dN28eFixYoPMCiah+qOk+akRE+lLjJQDy8/Nx5coVWFpawsfHp8Ft/solAIie7Vn7qI3r5A57KzM9VkhEDU2tb0sCADY2NujSpUtNTyeieoz7qBFRfVCjkHTmzBns2rULqampqltuj+3Zs0cnhRGRcREEAb/feIRtf1TcR21CV3e8wn3UiMjIaB2SduzYgUmTJmHAgAGIjo7GgAEDkJSUhIyMDIwaNao2aiQiA5ZbVIqfz93BD3+k4NZD7qNGRPWH1iFp5cqV+OyzzzBz5kzY2triiy++gJeXF6ZPn65x/SQiqp+4jxoR1Xdah6SbN29iyJAhAACJRIKCggKIRCK8++67eOmll7B06VKdF0lEhoH7qBFRQ6L1v80cHByQl1e+tombmxsuXbqEgIAA5OTkoLCwUOcFEpH+VbaP2uAAV7zWjfuoEVH9pHVIeuGFFxAdHY2AgACEhIRg1qxZOHr0KKKjo9G3b9/aqJGI9ID7qBFRQ6d1SNq4cSOKi4sBAAsWLICZmRliYmIwevRoLF68WOcFElHd4j5qRETltFpMsqysDD/88AMGDhwIFxeX2qzL4HExSapPuI8aETUUtbaYpKmpKd5++21cuXLluQokIsPAfdSIiCqn9e22oKAgxMXFwcPDozbqIaI6UNU+ahO7eSKgmb2eKyQi0j+tQ9KMGTPw3nvv4c6dO+jUqROsrdWn4Nu1a6ez4ohId6raR+21bh4Y27EZ91EjInqC1hvcmphUXDlXJBJBEASIRCIoFAqdFWfI+EwSGYvK9lHr38YZE7t5onsLR+6jRkQNRq1ucJucnFzjwoiobnAfNSKi56d1SOKzSESGi/uoERHpjtYh6fvvv6/yzydNmlTjYoioZriPGhGR7mn9TFLjxo3Vfi8tLUVhYSHMzc1hZWWFrKysSs6sX/hMUv2Ul5eHxYsXIzIyEpmZmejQoQO++OILdOnSBUD5baylS5diy5YtyM7ORlBQEL788kv4+/tX2e/u3buxePFi3Lx5Ey1atMCKFSswatQo1Z97enoiJSWlwnkzZszAl19+qbHPqvZRmxjsgRHtuY8aEdHTavWZpOzs7ArHkpKS8Pbbb2POnDnadkdkUKZNm4ZLly5h69atkEql2LZtG/r164fExES4ublh9erVWLduHSIiItCqVSt8/PHH6N+/P65duwZbW82zNbGxsQgNDcXy5csxatQoREZGIiQkBDExMQgKCgIAnDlzRu2lh0uXLqF///4YN25chf6q2kdtYjcPdOI+akREOqH1TFJlzp49i9deew1Xr17VRXcGjzNJ9U9RURFsbW2xb98+DBkyRHW8ffv2GDp0KJYvXw6pVIqwsDDMmzcPACCXy+Hs7IxVq1Zh+vTpGvsNDQ2FTCbDoUOHVMcGDRqExo0bY/v27RrPCQsLwy+//IKkpKTyt0a5jxoRkU7U6kxSZcRiMe7du6er7ojqXFlZGRQKBSwsLNSOW1paIiYmBsnJycjIyMCAAQNUfyaRSPDiiy/i1KlTlYak2NhYvPvuu2rHBg4ciM8//1xj+5KSEmzbtg2zZ8/Go4IS7qNGRKQnWoek/fv3q/0uCALS09OxceNG9OjRQ2eFEdU1W1tbBAcHY/ny5WjdujWcnZ2xfft2nD59Gj4+PsjIyAAAODs7q53n7Oys8XmixzIyMjSe87i/p0VGRiInJwc3G3VGcPivFfZRezWoOTwcuY8aEVFt0zokjRw5Uu13kUiEJk2a4KWXXsLatWt1VReRXmzduhVTp06Fm5sbxGIxOnbsiAkTJuD8+fOqNk8/7/N4IdWqVOecx/uozfpwLcw9O+LX1PLnjdr/tY/aEO6jRkRUp7QOSUqlsjbqIDIILVq0wIkTJ1BQUACZTAZXV1eEhobCy8sLLi4uAMpnhlxdXVXnZGZmVpgpepKLi0uFWaMnz3lyH7WczHt4dP0cpGM/QGhnd7zWzYP7qBER6QlXlSPSwNraGq6ursjOzsbhw4cxYsQIVVCKjo5WtSspKcGJEyfQvXv3SvsKDg5WOwcADh8+DM82HfDKllj0/+wkvotNQb68DKY3T8C+sSMu/GcBVo1tx4BERKRHWoeksWPH4pNPPqlwfM2aNRpfVyYyJocPH8b//vc/JCcnIzo6Gn369IGvry+mTJkCkUiEsLAwrFy5EpGRkbh06RJef/11WFlZYcKECao+Jk2ahAULFqh+nzVrFqKiorBq1SqcPBOPodPeR1T0EVxv0gt/3MqCiQgY6O+M76d0gXDtGP7x5lQ42nHLECIifdP6dtuJEyfw4YcfVjg+aNAgfPrppzopikhfcnNzsWDBAty5cwcODg4YM2YMVqxYATMzMwDA3LlzUVRUhBkzZqgWk4yKilJbIyk1NVVtI+jg4GB8+NkWfBK+DDkLF8G0kQuchs9DM992GN/l733UoqKikJqaiqlTp9b5uImIqCKt10mytLREfHw8fH191Y5fvXoVHTp0QFFRUSVn1oy+VkB+Fq6TRJVJzy1C8sMCOFlL8NuNhxX2Uevm7YDXunEfNSIifajVdZLatm2LnTt3YsmSJWrHd+zYgTZt2mjb3TPpawVkoprYeSYVC/ZchPKp//SwkZhiTEc3vMp91IiIjIbWM0n79+/HmDFjMGHCBLz00ksAgF9//RXbt2/Hrl27KiwR8DwMaQXkp3EmiZ6WnluEHp8crRCQ5g7yxeRgT1hzHzUiIr3T5vtb67n+4cOHY+/evbhx4wZmzJiB9957D3fu3MGRI0d0GpCA518BuTKxsbFq5wDlKyBXdY5cLodMJlP7IXpS8sOCCgEJADq4N2ZAIiIyQjX6N/eQIUPUZnZqi6GsgAwA4eHhWLp06XOMhuq7krKKa4iJRSJ4OlnpoRoiInpeWs8knTlzBqdPn65w/PTp0zh79qxOinrS1q1bIQgC3NzcIJFIsH79ekyYMAFi8d8rD9fWCshPWrBgAXJzc1U/aWlpNRgN1VfyMgXCD6pv7iwWibBydFu42vN1fiIiY6R1SJo5c6bGgHD37l3MnDlTJ0U96fEKyPn5+UhLS8Off/6J0tLSCisgP+l5V0DWRCKRwM7OTu2H6LF10ddx7X4enGzMcfCdntj+ZjfEzO+D0C7N9V0aERHVkNYhKTExER07dqxwvEOHDkhMTNRJUZrU9grIUVFRVZ5DVJlzKVnYcvIWACB8dDu0kdojuIUjZ5CIiIyc1s8kSSQS3L9/H97e3mrH09PTYWqq+4dTDx8+DEEQ4Ovrixs3bmDOnDkaV0D28fGBj48PVq5cqXEFZDc3N4SHhwMoXwG5V69eWLVqFUaMGIF9+/bhyJEjiImJ0Xn9VL8VlpRh9k8JEARgTMdm6N+m8tlIIiIyLlqnmv79+2PBggXYt28f7O3L95XKycnBwoUL0b9/f50XWBsrIHfv3h07duzAokWLsHjxYrRo0QI7d+7kGkmktfCDV5HyqBBSewt8OFz364QREZH+aL1O0t27d9GrVy88evQIHTp0AADEx8fD2dkZ0dHRcHd3r5VCDQ3XSaLfkh5g4td/AgB+mBaEHi2d9FwRERE9S62uuO3m5oYLFy7ghx9+QEJCAiwtLTFlyhSMHz9eNbtDVN/lFpVizq4LAIDJwR4MSERE9VCNHiKytrbGW2+9petaiIzG0v9eRoasGF5O1pj/cmt9l0NERLWgxk9aJyYmIjU1FSUlJWrHhw8f/txFERmyw5czsOf8XZiIgE/HBcLSXPzsk4iIyOhoHZJu3bqFUaNG4eLFixCJRHj8SNPjhRgVCoVuKyQyIA/z5Vi45yIAYPqLLdDJo7GeKyIiotqi9TpJs2bNgpeXF+7fvw8rKytcvnwZJ0+eROfOnXH8+PFaKJHIMAiCgA8iL+JRQQn8XGwR1s9H3yUREVEt0nomKTY2FkePHkWTJk1gYmICExMT9OzZE+Hh4XjnnXcQFxdXG3US6d3e+Ls4fPk+zMQirAtpD4kpb7MREdVnWs8kKRQK2NjYAACcnJxw7949AICHhweuXbum2+qIDER6bhGW7LsMAJjV1wdtpFz2gYiovtN6Jqlt27a4cOECvL29ERQUhNWrV8Pc3BxbtmypsAo3UX0gCALm/nwBecVlaO/eCP94sYW+SyIiojqgdUhatGgRCgoKAAAff/wxhg4dihdeeAGOjo7YuXOnzgsk0rdtp1PxW9JDWJiZYG1IIEzFWk/AEhGREdI6JA0cOFD1197e3khMTERWVhYaN26sesONqL64/bAAKw9cAQDMG+SHFk1s9FwRERHVFZ3sSOvg4KCLbogMikIp4P1dCSgqVSDY2xGTgz31XRIREdUh3jcgqsR/fruFsynZsJGYYs24djAx4UwpEVFDwpBEpMG1jDysjboOAFgytA2aNbbSc0VERFTXGJKInlJSpsTsn+JRolCir19TjOvcTN8lERGRHmgdkk6ePImysrIKx8vKynDy5EmdFEWkTxuPJuHyPRkaWZkhfEwAX0ggImqgtA5Jffr0QVZWVoXjubm56NOnj06KItKXhLQcfHn8JgDg45Ft0dTWQs8VERGRvmgdkgRB0Phf1o8ePYK1tbVOiiLSh+JSBWb/FA+FUsCwQCmGtpPquyQiItKjai8BMHr0aACASCTC66+/DolEovozhUKBCxcuoHv37rqvkKiOrDl8DTcfFKCprQTLR/jruxwiItKzaocke3t7AOUzSba2trC0tFT9mbm5Obp164Y333xT9xUS1YE/bj3CN78nAwBWjWmHRlbmeq6IiIj0rdoh6dtvvwUAeHp64v333+etNao38uVleH9XAgQBGN/VHX38muq7JCIiMgBaP5M0d+5ctWeSUlJS8PnnnyMqKkqnhRHVlRUHEnEnuwjNGlvigyFt9F0OEREZCK1D0ogRI/D9998DAHJyctC1a1esXbsWI0aMwKZNm3ReIFFtOnY1E9v/TINIBHw6LhA2Ep3s1ENERPWA1iHp/PnzeOGFFwAAP//8M1xcXJCSkoLvv/8e69ev13mBRLUlp7AE83ZfAABM7eGFbt6Oeq6IiIgMidYhqbCwELa2tgCAqKgojB49GiYmJujWrRtSUlJ0XiBRbVm87zIy8+Ro2dQGcwb66rscIiIyMFqHpJYtW2Lv3r1IS0vD4cOHMWDAAABAZmYm7OzsdF4gUW345cI9/DfhHsQmIqwdFwgLM7G+SyIiIgOjdUhasmQJ3n//fXh6eqJr164IDg4GUD6r1KFDB50XSKRrmXnFWLT3EgBgZu8WCHRvpN+CiIjIIGn9lOrYsWPRs2dPpKenIzAwUHW8b9++GDVqlE6LI9I1QRCwYPdF5BSWwl9qh3++5KPvkoiIyEBpPZMEAC4uLrC1tUV0dDSKiooAAF26dIGfn59OiyPStV1n7+DXq5kwF5tgXUh7mJvW6P8CRETUAGj9DfHo0SP07dsXrVq1wuDBg5Geng4AmDZtGt577z2dF0ikK2lZhVj2SyIA4L0BreDrYqvnioiIyJBpHZLeffddmJmZITU1FVZWVqrjoaGh+N///qfT4oh0RakUMPfnC8iXl6GzR2NMe8Fb3yUREZGB0/qZpKioKBw+fBjNmjVTO+7j48MlAMhgfRd7G7G3HsHSTIxPxwVCbCJ69klERNSgaT2TVFBQoDaD9NjDhw8hkUh0UhSRLt18kI9PDl0FACwc0hqeTtx3kIiInk3rkNSrVy/VtiQAIBKJoFQqsWbNGvTp00enxRE9rzKFErN/SoC8TIkXfJzwWlBzfZdERERGQuvbbWvWrEHv3r1x9uxZlJSUYO7cubh8+TKysrLw+++/10aNRDW2+cRNJKTlwNbCFKvHtlPbnJmIiKgqWs8ktWnTBhcuXEDXrl3Rv39/FBQUYPTo0YiLi0OLFi1qo0aiGrl8Lxdf/JoEAFg63B+u9pZ6roiIiIyJ1iEpNTUVzs7OWLp0KX755RccPHgQH3/8MVxdXZGamqrT4srKyrBo0SJ4eXnB0tIS3t7eWLZsGZRKpaqNSCTS+LNmzZpK+42IiNB4TnFxsU7rJ/2Rlynw3k8JKFUIGOjvjFEd3PRdEhERGRmtb7d5eXkhPT0dTZs2VTv+6NEjeHl5QaFQ6Ky4VatWYfPmzfjuu+/g7++Ps2fPYsqUKbC3t8esWbMAQLVO02OHDh3CG2+8gTFjxlTZt52dHa5du6Z2zMLCQme1k359fiQJVzPy4GhtjpWjAnibjYiItKZ1SBIEQeMXTn5+vs5DRmxsLEaMGIEhQ4YAADw9PbF9+3acPXtW1cbFxUXtnH379qFPnz7w9q56HRyRSFThXKofzqVk4d8nbgIAVo4OgKMN37okIiLtVTskzZ49G0B5uFi8eLHaMgAKhQKnT59G+/btdVpcz549sXnzZly/fh2tWrVCQkICYmJi8Pnnn2tsf//+fRw4cADffffdM/vOz8+Hh4cHFAoF2rdvj+XLl1e5Qa9cLodcLlf9LpPJtB4P1b7CkjK891MClAIwuoMbBvozCBMRUc1UOyTFxcUBKJ9JunjxIszNzVV/Zm5ujsDAQLz//vs6LW7evHnIzc2Fn58fxGIxFAoFVqxYgfHjx2ts/91338HW1hajR4+usl8/Pz9EREQgICAAMpkMX3zxBXr06IGEhAT4+Gje8DQ8PBxLly597jFR7Vp16CpuPyqEq70FPhzur+9yiIjIiIkEQRC0OWHKlCn44osvYGdnV1s1qezYsQNz5szBmjVr4O/vj/j4eISFhWHdunWYPHlyhfZ+fn7o378/NmzYoNXnKJVKdOzYEb169cL69es1ttE0k+Tu7o7c3Nw6+XtBz/b7jYd49T+nAQBb3+iKF3ya6LkiIiIyNDKZDPb29tX6/tb6maRvv/22xoVpa86cOZg/fz5eeeUVAEBAQABSUlIQHh5eIST99ttvuHbtGnbu3Kn155iYmKBLly5ISkqqtI1EIuGK4gZMVlyKObsSAAATu3kwIBER0XPTegmAulRYWAgTE/USxWKx2hIAj3399dfo1KkTAgMDtf4cQRAQHx8PV1fXGtdK+rV0fyLu5RbDw9EKCwb76bscIiKqB7SeSapLw4YNw4oVK9C8eXP4+/sjLi4O69atw9SpU9XayWQy7Nq1C2vXrtXYz6RJk+Dm5obw8HAAwNKlS9GtWzf4+PhAJpNh/fr1iI+Px5dfflnrYyLdi7qcgd3n70AkAtaOC4SVuUH/Y01EREbCoL9NNmzYgMWLF2PGjBnIzMyEVCrF9OnTsWTJErV2O3bsgCAIlT7QnZqaqjYjlZOTg7feegsZGRmwt7dHhw4dcPLkSXTt2rVWx0O69yhfjoWRFwEAb/XyRmdPBz1XRERE9YXWD25TOW0e/KLaIQgCZvxwHocuZcDX2Rb7/9UDElOxvssiIiIDps33t0E/k0RUlX3x93DoUgZMTURYGxLIgERERDrFkERGKSO3GEv2XQIAvNPXB23d7PVcERER1TcMSWR0BEHA3N0XICsuQ2Aze8zo3ULfJRERUT3EkERG58c/U3Hy+gNITE2wNqQ9TMX8x5iIiHSP3y5kVFIeFWDFgSsAgLmD/NCyqY2eKyIiovqKIYmMhkIp4P1dCSgsUSDIywFTunvquyQiIqrHGJLIaHwdcwtnbmfD2lyMT8cFwsREpO+SiIioHmNIIqNw/X4ePj18HQCweGgbuDtY6bkiIiKq7xiSyOCVKpSY/VM8ShRKvOTXFKFd3PVdEhERNQAMSWTwNh69gUt3ZbC3NMMnowMgEvE2GxER1T6GJDJoF+7kYOOxGwCA5SPboqmdhZ4rIiKihoIhiQxWcakCs39KgEIpYEg7VwwPlOq7JCIiakAYkshgrY26hhuZ+WhiK8HHI9rquxwiImpgGJLIIJ2+9Qj/iUkGAKwaE4DG1uZ6roiIiBoahiQyOPnyMrz/cwIEAQjt7I6X/Jz1XRIRETVADElkcFYcuIK0rCK4NbLEoqGt9V0OERE1UAxJZFCOXcvE9j9TAQBrxrWDrYWZnisiIqKGiiGJDEZOYQnm/XwBADClhye6t3DSc0VERNSQMSSRwfhw/2Vk5snh3cQa8wb56bscIiJq4BiSyCAcvJiOffH3YCIC1oW0h4WZWN8lERFRA8eQRHqXmVeMDyIvAgBm9G6J9u6N9FsQERERGJJIzwRBwMI9l5BdWIo2rnZ4p6+PvksiIiICwJBEevbzuTs4cuU+zMUmWBcaCHNT/iNJRESGgd9IpDd3c4qw7L+JAIB3+7eCn4udnisiIiL6G0MS6YVSKWDOrgTkycvQsXkjvNXLW98lERERqWFIIr34PvY2Tt18BEszMdaGtIfYRKTvkoiIiNQwJFGdu/UgH5/87yoAYMFgP3g5Weu5IiIioooYkqhOlSmUeG9XAopLlejZ0gmvBXnouyQiIiKNGJKoTv375C3EpebAVmKK1WPbwYS32YiIyEAxJFGdSbwnw+dHrgMAPhzuD2kjSz1XREREVDmGJKoT8jIFZv8Uj1KFgP5tnDGmo5u+SyIiIqoSQxLVifW/JuFqRh4crM0RPjoAIhFvsxERkWFjSKJadz41G5uO3wQArBzVFk42Ej1XRERE9GwMSVSrikoUeP+nBCgFYGR7KQa1ddV3SURERNXCkES1atX/ruLWwwK42Flg6fC2+i6HiIio2gw6JJWVlWHRokXw8vKCpaUlvL29sWzZMiiVSlWb119/HSKRSO2nW7duz+x79+7daNOmDSQSCdq0aYPIyMjaHEqD9PuNh4g4dRsAsGpsO9hbmem3ICIiIi2Y6ruAqqxatQqbN2/Gd999B39/f5w9exZTpkyBvb09Zs2apWo3aNAgfPvtt6rfzc3Nq+w3NjYWoaGhWL58OUaNGoXIyEiEhIQgJiYGQUFBtTaehkRWXIq5P18AALwa1Bwvtmqi54qIiIi0IxIEQdB3EZUZOnQonJ2d8fXXX6uOjRkzBlZWVti6dSuA8pmknJwc7N27t9r9hoaGQiaT4dChQ6pjgwYNQuPGjbF9+/Zq9SGTyWBvb4/c3FzY2XH3+qfN2ZWAXefuoLmDFQ7NegHWEoPO40RE1EBo8/1t0LfbevbsiV9//RXXr5cvQJiQkICYmBgMHjxYrd3x48fRtGlTtGrVCm+++SYyMzOr7Dc2NhYDBgxQOzZw4ECcOnWq0nPkcjlkMpnaD2l2JPE+dp27A5EI+HRcIAMSEREZJYP+9po3bx5yc3Ph5+cHsVgMhUKBFStWYPz48ao2L7/8MsaNGwcPDw8kJydj8eLFeOmll3Du3DlIJJpfNc/IyICzs7PaMWdnZ2RkZFRaS3h4OJYuXaqbgdVjWQUlmL/nIgDgzRe80dXLQc8VERER1YxBh6SdO3di27Zt+PHHH+Hv74/4+HiEhYVBKpVi8uTJAMpvnT3Wtm1bdO7cGR4eHjhw4ABGjx5dad9PL2YoCEKVCxwuWLAAs2fPVv0uk8ng7u5e06HVS4IgYNHei3iYL0crZxvM7t9K3yURERHVmEGHpDlz5mD+/Pl45ZVXAAABAQFISUlBeHi4KiQ9zdXVFR4eHkhKSqq0XxcXlwqzRpmZmRVml54kkUgqnZmicvsT7uHgxQyYmoiwdlx7WJiJ9V0SERFRjRn0M0mFhYUwMVEvUSwWqy0B8LRHjx4hLS0Nrq6VL1oYHByM6OhotWNRUVHo3r378xXcgN2XFWPJvssAgH++1BIBzez1XBEREdHzMeiZpGHDhmHFihVo3rw5/P39ERcXh3Xr1mHq1KkAgPz8fHz00UcYM2YMXF1dcfv2bSxcuBBOTk4YNWqUqp9JkybBzc0N4eHhAIBZs2ahV69eWLVqFUaMGIF9+/bhyJEjiImJ0cs4jZ0gCJj78wXkFpUiwM0eM/u01HdJREREz82gQ9KGDRuwePFizJgxA5mZmZBKpZg+fTqWLFkCoHxW6eLFi/j++++Rk5MDV1dX9OnTBzt37oStra2qn9TUVLUZqe7du2PHjh1YtGgRFi9ejBYtWmDnzp1cI6mGtv+ZhhPXH8Dc1ATrQgJhJjboCUoiIqJqMeh1kgwZ10kql/qoEIO+OInCEgUWDWmNaS9467skIiKiStWbdZLIsCmVAt7/OQGFJQp09XLA1B5e+i6JiIhIZxiSqMa++T0ZfyZnwcpcjE/HBsLEpPIlFIiIiIwNQxLVSNL9PKw+fA0AsGhIGzR3tNJzRURERLrFkERaK1UoMfunBJSUKdHbtwnGd+WimkREVP8wJJHWvjp2Exfv5sLe0gyrxrSrcqVyIiIiY8WQRFq5eCcXG46Wr2a+bIQ/nO0s9FwRERFR7WBIomorLlVg9k/xKFMKGBzgguGBUn2XREREVGsYkqja1kVfR1JmPpxsJPh4ZABvsxERUb3GkETV8mdyFv7vt1sAgE9GB8DB2lzPFREREdUuhiR6pgJ5Gd7flQBBAMZ1aoZ+bZz1XRIREVGtY0iiZ1p58ApSswrh1sgSi4e10Xc5REREdYIhiap04voD/HA6FQCwZmw72FmY6bkiIiKiusGQRJXKLSzF3J8TAACvd/dE95ZOeq6IiIio7jAkUaU++u9l3JfJ4eVkjXmD/PRdDhERUZ1iSCKN/ncpHZFxd2EiAtaGBMLSXKzvkoiIiOoUQxJV8CBPjoWRlwAA/3ixBTo2b6znioiIiOoeQxKpEQQBCyMvIqugBH4utpjVz0ffJREREekFQxKp2X3+LqIT78NMLMJnoe0hMeVtNiIiapgYkkjlXk4Rlu6/DAAI69cKrV3t9FwRERGR/jAkEQBAqRQw9+cLyJOXoUPzRpjey1vfJREREekVQxIBALadTkHMjYewMDPB2nGBMBXzHw0iImrY+E1ISH5YgJUHrwAA5g/yg3cTGz1XREREpH8MSQ2cQingvZ/iUVyqRPcWjpgU7KnvkoiIiAwCQ1IDt+XkLZxPzYGNxBRrxgXCxESk75KIiIgMAkNSA3Y1Q4bPoq8DAJYMawO3RpZ6roiIiMhwMCQ1UCVlSry7MwElCiX6tW6KcZ2a6bskIiIig8KQ1ECt/zUJV9JlaGxlhpWjAyAS8TYbERHRkxiSGqC41Gx8dfwGAGDFqAA0tbXQc0VERESGhyGpgSkqUeC9nxKgFIDhgVIMDnDVd0lEREQGiSGpgVl9+CpuPSxAU1sJlo3w13c5REREBoshqQE5dfMhvv39NgBg1dh2aGRlrt+CiIiIDBhDUgORV1yKObsuAADGd22OPr5N9VwRERGRYWNIaiCW/5KIuzlFcHewxAdDWuu7HCIiIoPHkNQA/HrlPn46ewciEfDp2EDYSEz1XRIREZHBY0iq57ILSjB/z0UAwBs9vBDk7ajnioiIiIyDQYeksrIyLFq0CF5eXrC0tIS3tzeWLVsGpVIJACgtLcW8efMQEBAAa2trSKVSTJo0Cffu3auy34iICIhEogo/xcXFdTGsOrVo3yU8yJOjZVMbvD/QV9/lEBERGQ2Dvu+yatUqbN68Gd999x38/f1x9uxZTJkyBfb29pg1axYKCwtx/vx5LF68GIGBgcjOzkZYWBiGDx+Os2fPVtm3nZ0drl27pnbMwqJ+Laq4P+EeDlxIh9hEhHUhgbAwE+u7JCIiIqNh0CEpNjYWI0aMwJAhQwAAnp6e2L59uyoA2dvbIzo6Wu2cDRs2oGvXrkhNTUXz5s0r7VskEsHFxaX2itezTFkxFu+9BACY2acl2jVrpN+CiIiIjIxB327r2bMnfv31V1y/Xr5TfUJCAmJiYjB48OBKz8nNzYVIJEKjRo2q7Ds/Px8eHh5o1qwZhg4diri4OF2WrleCIGDe7gvILSpFWzc7/OullvouiYiIyOgY9EzSvHnzkJubCz8/P4jFYigUCqxYsQLjx4/X2L64uBjz58/HhAkTYGdnV2m/fn5+iIiIQEBAAGQyGb744gv06NEDCQkJ8PHx0XiOXC6HXC5X/S6TyZ5vcLVo55k0HLv2AOamJlgX0h5mYoPOwkRERAbJoEPSzp07sW3bNvz444/w9/dHfHw8wsLCIJVKMXnyZLW2paWleOWVV6BUKvHVV19V2W+3bt3QrVs31e89evRAx44dsWHDBqxfv17jOeHh4Vi6dOnzD6qWpWUVYvkviQCA9we0QitnWz1XREREZJxEgiAI+i6iMu7u7pg/fz5mzpypOvbxxx9j27ZtuHr1qupYaWkpQkJCcOvWLRw9ehSOjtq/5v7mm2/izp07OHTokMY/1zST5O7ujtzc3CpnreqSUilg/P/9gdPJWeji2Rg73gqG2ESk77KIiIgMhkwmg729fbW+vw16JqmwsBAmJuq3isRisWoJAODvgJSUlIRjx47VKCAJgoD4+HgEBARU2kYikUAikWjdd1369tRtnE7OgpW5GJ+OC2RAIiIieg4GHZKGDRuGFStWoHnz5vD390dcXBzWrVuHqVOnAihfR2ns2LE4f/48fvnlFygUCmRkZAAAHBwcYG5evoHrpEmT4ObmhvDwcADA0qVL0a1bN/j4+EAmk2H9+vWIj4/Hl19+qZ+B6sCNzHys/l/57NrCwa3h4Wit54qIiIiMm0GHpA0bNmDx4sWYMWMGMjMzIZVKMX36dCxZsgQAcOfOHezfvx8A0L59e7Vzjx07ht69ewMAUlNT1WakcnJy8NZbbyEjIwP29vbo0KEDTp48ia5du9bJuHStTKHEez/FQ16mRK9WTfBqUOVLHxAREVH1GPQzSYZMm3uatW39r0lYF30ddhamiHr3RbjY169FMYmIiHRFm+9vvhtu5C7dzcX6X5MAAEtH+DMgERER6QhDkhGTlykw+6d4lCkFDPJ3wcj2bvouiYiIqN5gSDJi66Kv4/r9fDjZmGPFqLYQifg2GxERka4wJBmps7ezsOXkLQDAylEBcLQx7OUJiIiIjA1DkhEqkJfhvV0JEARgTMdmGOBffzfqJSIi0heGJCMUfugKUh4VwtXeAkuGtdF3OURERPUSQ5IOlZWVYdGiRfDy8oKlpSW8vb2xbNkytRXC9+zZg4EDB8LJyQkikQjx8fHP7Pfy5csYM2YMPD09IRKJ8NWGDQCANWMDYW9pVqF9eHg4RCIRwsLCdDU0IiKiBochSYdWrVqFzZs3Y+PGjbhy5QpWr16NNWvWYMNfoQYACgoK0KNHD3zyySfV7rewsBDe3t5YsuxjmNk4AAAmBXugp49ThbZnzpzBli1b0K5du+cfEBERUQNm0CtuG5vY2FiMGDECQ4YMAQB4enpi+/btOHv2rKrNxIkTAQC3b9+udr9dunRBly5dMHtnPJQmpnCwNsP8l/0qtMvPz8err76K//u//8PHH3/8fIMhIiJq4DiTpEM9e/bEr7/+iuvXrwMAEhISEBMTg8GDBz933/+7lIE9cXchAjCivRuszCvm25kzZ2LIkCHo16/fc38eERFRQ8eZJB2aN28ecnNz4efnB7FYDIVCgRUrVmD8+PHP1e/DfDk+iLwIALCxMIW7g1WFNjt27MD58+dx5syZ5/osIiIiKseQpEM7d+7Etm3b8OOPP8Lf3x/x8fEICwuDVCrF5MmTa9SnIAj4IPIiHhWUwM/FFlc0PKidlpaGWbNmISoqChYW3JaEiIhIFxiSdGjOnDmYP38+XnnlFQBAQEAAUlJSEB4eXuOQFBl3F4cv34eZWIS1IYEY8nnFNufOnUNmZiY6deqkOqZQKHDy5Els3LgRcrkcYrG4Rp9PRETUUDEk6VBhYSFMTNQf8xKLxWpLAGjjXk4RPtx/GQAwq68P/KX2Gtv17dsXFy9eVDs2ZcoU+Pn5Yd68eQxIRERENcCQpEPDhg3DihUr0Lx5c/j7+yMuLg7r1q3D1KlTVW2ysrKQmpqKe/fuAQCuXbsGAHBxcYGLS/nK2ZMmTYJUKsVdn1HIKy5DgKs1utnnIT4+HiUlJbh79y7i4+NhY2ODli1bwtbWFm3btlWrxdraGo6OjhWOExERUfWIBEEQ9F2EMZLJZLC3t0dubi7s7OwAAHl5eVi8eDEiIyORmZkJqVSK8ePHY8mSJTA3NwcAREREYMqUKRX6+/DDD/HRRx8BAHr37g2FtRPSAqZAYmqCf49sjj5dKoadF198EcePH9dYX+/evdG+fXt8/vnnOhkvERFRfaDp+7syDEk1pM3fZG39mfwIr/3nT5QolFgytA2m9vTSaf9EREQNlTbf37zdZmC2/5mKBXv+fr7I0pzPExEREekDF5M0IOm5RVi4R/0B7EWRl5CeW6SnioiIiBouhiQDkvywAE/f+1QIAm4/LNRLPURERA0ZQ5IB8XKyholI/ZhYJIKnU8UVtomIiKh2MSQZEFd7S4SPDoBYVJ6UxCIRVo5uC1d7Sz1XRkRE1PDwwW0DE9qlOXq1aoLbDwvh6WTFgERERKQnDEkGyNXekuGIiIhIz3i7jYiIiEgDhiQiIiIiDRiSiIiIiDRgSCIiIiLSgCGJiIiISAOGJCIiIiINGJKIiIiINGBIIiIiItKAIYmIiIhIA4YkIiIiIg0YkoiIiIg0YEgiIiIi0sCgQ1JZWRkWLVoELy8vWFpawtvbG8uWLYNSqVS1EQQBH330EaRSKSwtLdG7d29cvnz5mX3v3r0bbdq0gUQiQZs2bRAZGVmbQyEiIiIjY9AhadWqVdi8eTM2btyIK1euYPXq1VizZg02bNigarN69WqsW7cOGzduxJkzZ+Di4oL+/fsjLy+v0n5jY2MRGhqKiRMnIiEhARMnTkRISAhOnz5dF8MiIiIiIyASBEHQdxGVGTp0KJydnfH111+rjo0ZMwZWVlbYunUrBEGAVCpFWFgY5s2bBwCQy+VwdnbGqlWrMH36dI39hoaGQiaT4dChQ6pjgwYNQuPGjbF9+/Zq1SaTyWBvb4/c3FzY2dk9xyiJiIiormjz/W1aRzXVSM+ePbF582Zcv34drVq1QkJCAmJiYvD5558DAJKTk5GRkYEBAwaozpFIJHjxxRdx6tSpSkNSbGws3n33XbVjAwcOVPWriVwuh1wuV/2em5sLoPxvNhERERmHx9/b1ZkjMuiQNG/ePOTm5sLPzw9isRgKhQIrVqzA+PHjAQAZGRkAAGdnZ7XznJ2dkZKSUmm/GRkZGs953J8m4eHhWLp0aYXj7u7u1R4PERERGYa8vDzY29tX2cagQ9LOnTuxbds2/Pjjj/D390d8fDzCwsIglUoxefJkVTuRSKR2niAIFY49TdtzFixYgNmzZ6t+VyqVyMrKgqOj4zM/S1symQzu7u5IS0url7fyOD7jV9/HWN/HB9T/MXJ8xq+2xigIAvLy8iCVSp/Z1qBD0pw5czB//ny88sorAICAgACkpKQgPDwckydPhouLC4DymSFXV1fVeZmZmRVmip7k4uJSYdboWedIJBJIJBK1Y40aNdJ2SFqxs7Ort//wAxxffVDfx1jfxwfU/zFyfMavNsb4rBmkxwz67bbCwkKYmKiXKBaLVUsAeHl5wcXFBdHR0ao/LykpwYkTJ9C9e/dK+w0ODlY7BwCioqKqPIeIiIgaFoOeSRo2bBhWrFiB5s2bw9/fH3FxcVi3bh2mTp0KoPyWWVhYGFauXAkfHx/4+Phg5cqVsLKywoQJE1T9TJo0CW5ubggPDwcAzJo1C7169cKqVaswYsQI7Nu3D0eOHEFMTIxexklERESGx6BD0oYNG7B48WLMmDEDmZmZkEqlmD59OpYsWaJqM3fuXBQVFWHGjBnIzs5GUFAQoqKiYGtrq2qTmpqqNiPVvXt37NixA4sWLcLixYvRokUL7Ny5E0FBQXU6vspIJBJ8+OGHFW7v1Rccn/Gr72Os7+MD6v8YOT7jZwhjNOh1koiIiIj0xaCfSSIiIiLSF4YkIiIiIg0YkoiIiIg0YEgiIiIi0oAhSU+++uoreHl5wcLCAp06dcJvv/1WZfsTJ06gU6dOsLCwgLe3NzZv3lxHldaMNuM7fvw4RCJRhZ+rV6/WYcXVd/LkSQwbNgxSqRQikQh79+595jnGdP20HZ+xXb/w8HB06dIFtra2aNq0KUaOHIlr16498zxjuYY1GZ+xXcNNmzahXbt2qkUGg4OD1TYs18RYrh+g/fiM7fo9LTw8XLWkT1X0cQ0ZkvRg586dCAsLwwcffIC4uDi88MILePnll5GamqqxfXJyMgYPHowXXngBcXFxWLhwId555x3s3r27jiuvHm3H99i1a9eQnp6u+vHx8amjirVTUFCAwMBAbNy4sVrtje36aTu+x4zl+p04cQIzZ87EH3/8gejoaJSVlWHAgAEoKCio9BxjuoY1Gd9jxnINmzVrhk8++QRnz57F2bNn8dJLL2HEiBG4fPmyxvbGdP0A7cf3mLFcvyedOXMGW7ZsQbt27apsp7drKFCd69q1q/CPf/xD7Zifn58wf/58je3nzp0r+Pn5qR2bPn260K1bt1qr8XloO75jx44JAITs7Ow6qE63AAiRkZFVtjG26/ek6ozPmK+fIAhCZmamAEA4ceJEpW2M+RpWZ3zGfg0FQRAaN24s/Oc//9H4Z8Z8/R6ranzGev3y8vIEHx8fITo6WnjxxReFWbNmVdpWX9eQM0l1rKSkBOfOncOAAQPUjg8YMACnTp3SeE5sbGyF9gMHDsTZs2dRWlpaa7XWRE3G91iHDh3g6uqKvn374tixY7VZZp0ypuv3PIz1+uXm5gIAHBwcKm1jzNewOuN7zBivoUKhwI4dO1BQUIDg4GCNbYz5+lVnfI8Z2/WbOXMmhgwZgn79+j2zrb6uIUNSHXv48CEUCkWFzXSdnZ0rbLr7WEZGhsb2ZWVlePjwYa3VWhM1GZ+rqyu2bNmC3bt3Y8+ePfD19UXfvn1x8uTJuii51hnT9asJY75+giBg9uzZ6NmzJ9q2bVtpO2O9htUdnzFew4sXL8LGxgYSiQT/+Mc/EBkZiTZt2mhsa4zXT5vxGeP127FjB86fP6/aLuxZ9HUNDXpbkvpMJBKp/S4IQoVjz2qv6bih0GZ8vr6+8PX1Vf0eHByMtLQ0fPrpp+jVq1et1llXjO36acOYr98///lPXLhwoVr7NhrjNazu+IzxGvr6+iI+Ph45OTnYvXs3Jk+ejBMnTlQaJIzt+mkzPmO7fmlpaZg1axaioqJgYWFR7fP0cQ05k1THnJycIBaLK8yqZGZmVkjJj7m4uGhsb2pqCkdHx1qrtSZqMj5NunXrhqSkJF2XpxfGdP10xRiu37/+9S/s378fx44dQ7Nmzapsa4zXUJvxaWLo19Dc3BwtW7ZE586dER4ejsDAQHzxxRca2xrj9dNmfJoY8vU7d+4cMjMz0alTJ5iamsLU1BQnTpzA+vXrYWpqCoVCUeEcfV1DhqQ6Zm5ujk6dOiE6OlrteHR0NLp3767xnODg4Arto6Ki0LlzZ5iZmdVarTVRk/FpEhcXB1dXV12XpxfGdP10xZCvnyAI+Oc//4k9e/bg6NGj8PLyeuY5xnQNazI+TQz5GmoiCALkcrnGPzOm61eZqsaniSFfv759++LixYuIj49X/XTu3Bmvvvoq4uPjIRaLK5yjt2tYq4+Fk0Y7duwQzMzMhK+//lpITEwUwsLCBGtra+H27duCIAjC/PnzhYkTJ6ra37p1S7CyshLeffddITExUfj6668FMzMz4eeff9bXEKqk7fg+++wzITIyUrh+/bpw6dIlYf78+QIAYffu3foaQpXy8vKEuLg4IS4uTgAgrFu3ToiLixNSUlIEQTD+66ft+Izt+r399tuCvb29cPz4cSE9PV31U1hYqGpjzNewJuMztmu4YMEC4eTJk0JycrJw4cIFYeHChYKJiYkQFRUlCIJxXz9B0H58xnb9NHn67TZDuYYMSXry5ZdfCh4eHoK5ubnQsWNHtddzJ0+eLLz44otq7Y8fPy506NBBMDc3Fzw9PYVNmzbVccXa0WZ8q1atElq0aCFYWFgIjRs3Fnr27CkcOHBAD1VXz+PXbZ/+mTx5siAIxn/9tB2fsV0/TWMDIHz77beqNsZ8DWsyPmO7hlOnTlX9+6VJkyZC3759VQFCEIz7+gmC9uMztuunydMhyVCuoUgQ/nryiYiIiIhU+EwSERERkQYMSUREREQaMCQRERERacCQRERERKQBQxIRERGRBgxJRERERBowJBERERFpwJBERKQjx48fh0gkQk5Ojr5LISIdYEgiIiIi0oAhiYiIiEgDhiQiqjcEQcDq1avh7e0NS0tLBAYG4ueffwbw962wAwcOIDAwEBYWFggKCsLFixfV+ti9ezf8/f0hkUjg6emJtWvXqv25XC7H3Llz4e7uDolEAh8fH3z99ddqbc6dO4fOnTvDysoK3bt3x7Vr12p34ERUKxiSiKjeWLRoEb799lts2rQJly9fxrvvvovXXnsNJ06cULWZM2cOPv30U5w5cwZNmzbF8OHDUVpaCqA83ISEhOCVV17BxYsX8dFHH2Hx4sWIiIhQnT9p0iTs2LED69evx5UrV7B582bY2Nio1fHBBx9g7dq1OHv2LExNTTF16tQ6GT8R6RY3uCWieqGgoABOTk44evQogoODVcenTZuGwsJCvPXWW+jTpw927NiB0NBQAEBWVhaaNWuGiIgIhISE4NVXX8WDBw8QFRWlOn/u3Lk4cOAALl++jOvXr8PX1xfR0dHo169fhRqOHz+OPn364MiRI+jbty8A4ODBgxgyZAiKiopgYWFRy38XiEiXOJNERPVCYmIiiouL0b9/f9jY2Kh+vv/+e9y8eVPV7skA5eDgAF9fX1y5cgUAcOXKFfTo0UOt3x49eiApKQkKhQLx8fEQi8V48cUXq6ylXbt2qr92dXUFAGRmZj73GImobpnquwAiIl1QKpUAgAMHDsDNzU3tzyQSiVpQeppIJAJQ/kzT479+7MnJdktLy2rVYmZmVqHvx/URkfHgTBIR1Qtt2rSBRCJBamoqWrZsqfbj7u6uavfHH3+o/jo7OxvXr1+Hn5+fqo+YmBi1fk+dOoVWrVpBLBYjICAASqVS7RknIqq/OJNERPWCra0t3n//fbz77rtQKpXo2bMnZDIZTp06BRsbG3h4eAAAli1bBkdHRzg7O+ODDz6Ak5MTRo4cCQB477330KVLFyxfvhyhoaGIjY3Fxo0b8dVXXwEAPD09MXnyZEydOhXr169HYGAgUlJSkJmZiZCQEH0NnYhqCUMSEdUby5cvR9OmTREeHo5bt26hUaNG6NixIxYuXKi63fXJJ59g1qxZSEpKQmBgIPbv3w9zc3MAQMeOHfHTTz9hyZIlWL58OVxdXbFs2TK8/vrrqs/YtGkTFi5ciBkzZuDRo0do3rw5Fi5cqI/hElEt49ttRNQgPH7zLDs7G40aNdJ3OURkBPhMEhEREZEGDElEREREGvB2GxEREZEGnEkiIiIi0oAhiYiIiEgDhiQiIiIiDRiSiIiIiDRgSCIiIiLSgCGJiIiISAOGJCIiIiINGJKIiIiINGBIIiIiItLg/wE7w+URUhR3kQAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plt.plot(np.arange(len(epochs_x)), epochs_acc, marker = '.')\n", + "plt.xlabel('epoch')\n", + "plt.ylabel('test accuracy')\n", + "plt.ylim(80, 100)\n", + "for i, txt in enumerate(epochs_acc):\n", + " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom')\n", + "plt.show()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "speck-rescnn", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From decf3875055fedaf293b082a44db29a8be431048 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Fri, 26 Apr 2024 21:48:23 +0200 Subject: [PATCH 050/379] non-sequential 2nd example --- .../non-sequential-SCNN-example_2.ipynb | 263 ++++++++++++++++-- 1 file changed, 236 insertions(+), 27 deletions(-) diff --git a/tests/test_nonsequential/non-sequential-SCNN-example_2.ipynb b/tests/test_nonsequential/non-sequential-SCNN-example_2.ipynb index 754d2065..c6420691 100644 --- a/tests/test_nonsequential/non-sequential-SCNN-example_2.ipynb +++ b/tests/test_nonsequential/non-sequential-SCNN-example_2.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": 17, "metadata": {}, "outputs": [], "source": [ @@ -19,7 +19,8 @@ "\n", "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential\n", "\n", - "import matplotlib.pyplot as plt" + "import matplotlib.pyplot as plt\n", + "import numpy as np" ] }, { @@ -30,7 +31,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 2, @@ -113,7 +114,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "device: NVIDIA GeForce RTX 3070 Ti\n" + "device: NVIDIA RTX A4000\n" ] } ], @@ -138,6 +139,7 @@ " self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False)\n", " self.iaf1 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", " self.pool1 = nn.AvgPool2d(2,2)\n", + " self.pool1a = nn.AvgPool2d(6,6)\n", "\n", " self.conv2 = nn.Conv2d(10, 10, 2, 1, bias=False)\n", " self.iaf2 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", @@ -161,7 +163,8 @@ " self.fc4 = nn.Linear(100, 10, bias=False)\n", " self.iaf7 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", "\n", - " self.merge = sl.Merge()\n", + " self.merge_fc = sl.Merge()\n", + " self.merge_conv = sl.Merge()\n", "\n", " def detach_neuron_states(self):\n", " for name, layer in self.named_modules():\n", @@ -180,12 +183,15 @@ " con1_out = self.conv1(x)\n", " iaf1_out = self.iaf1(con1_out)\n", " pool1_out = self.pool1(iaf1_out)\n", + " pool1a_out = self.pool1a(iaf1_out)\n", "\n", " conv2_out = self.conv2(pool1_out)\n", " iaf2_out = self.iaf2(conv2_out)\n", " pool2_out = self.pool2(iaf2_out)\n", "\n", - " conv3_out = self.conv3(pool2_out)\n", + " merged_conv_out = self.merge_conv(pool1a_out, pool2_out)\n", + "\n", + " conv3_out = self.conv3(merged_conv_out)\n", " iaf3_out = self.iaf3(conv3_out)\n", " pool3_out = self.pool3(iaf3_out)\n", "\n", @@ -200,9 +206,9 @@ " fc3_out = self.fc3(iaf5_out)\n", " iaf6_out = self.iaf6(fc3_out)\n", "\n", - " merge_out = self.merge(iaf4_out, iaf6_out)\n", + " merge_fc_out = self.merge_fc(iaf4_out, iaf6_out)\n", "\n", - " fc4_out = self.fc4(merge_out)\n", + " fc4_out = self.fc4(merge_fc_out)\n", " iaf7_out = self.iaf7(fc4_out)\n", "\n", " return iaf7_out" @@ -219,7 +225,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ @@ -228,7 +234,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 11, "metadata": {}, "outputs": [], "source": [ @@ -245,7 +251,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 12, "metadata": {}, "outputs": [], "source": [ @@ -302,7 +308,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 13, "metadata": {}, "outputs": [], "source": [ @@ -345,13 +351,118 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 14, "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "546940b28031476fb3bee7a239fd1529", + "model_id": "1b6ac71c89994fada6d3e4a0ef50b939", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/937 [00:00 1\u001b[0m epochs_x, epochs_y \u001b[38;5;241m=\u001b[39m train(snn_train_dataloader, snn, loss_fn, optimizer, epochs, test, snn_test_dataloader)\n", - "Cell \u001b[0;32mIn[13], line 29\u001b[0m, in \u001b[0;36mtrain\u001b[0;34m(dataloader, model, loss_fn, optimizer, epochs, test_func, dataloader_test)\u001b[0m\n\u001b[1;32m 27\u001b[0m \u001b[38;5;66;03m# gradient update\u001b[39;00m\n\u001b[1;32m 28\u001b[0m optimizer\u001b[38;5;241m.\u001b[39mzero_grad()\n\u001b[0;32m---> 29\u001b[0m loss\u001b[38;5;241m.\u001b[39mbackward()\n\u001b[1;32m 30\u001b[0m optimizer\u001b[38;5;241m.\u001b[39mstep()\n\u001b[1;32m 32\u001b[0m \u001b[38;5;66;03m# detach the neuron states and activations from current computation graph(necessary)\u001b[39;00m\n", - "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/_tensor.py:522\u001b[0m, in \u001b[0;36mTensor.backward\u001b[0;34m(self, gradient, retain_graph, create_graph, inputs)\u001b[0m\n\u001b[1;32m 512\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m has_torch_function_unary(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m 513\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m handle_torch_function(\n\u001b[1;32m 514\u001b[0m Tensor\u001b[38;5;241m.\u001b[39mbackward,\n\u001b[1;32m 515\u001b[0m (\u001b[38;5;28mself\u001b[39m,),\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 520\u001b[0m inputs\u001b[38;5;241m=\u001b[39minputs,\n\u001b[1;32m 521\u001b[0m )\n\u001b[0;32m--> 522\u001b[0m torch\u001b[38;5;241m.\u001b[39mautograd\u001b[38;5;241m.\u001b[39mbackward(\n\u001b[1;32m 523\u001b[0m \u001b[38;5;28mself\u001b[39m, gradient, retain_graph, create_graph, inputs\u001b[38;5;241m=\u001b[39minputs\n\u001b[1;32m 524\u001b[0m )\n", - "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/autograd/__init__.py:266\u001b[0m, in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[0m\n\u001b[1;32m 261\u001b[0m retain_graph \u001b[38;5;241m=\u001b[39m create_graph\n\u001b[1;32m 263\u001b[0m \u001b[38;5;66;03m# The reason we repeat the same comment below is that\u001b[39;00m\n\u001b[1;32m 264\u001b[0m \u001b[38;5;66;03m# some Python versions print out the first line of a multi-line function\u001b[39;00m\n\u001b[1;32m 265\u001b[0m \u001b[38;5;66;03m# calls in the traceback and some print out the last line\u001b[39;00m\n\u001b[0;32m--> 266\u001b[0m Variable\u001b[38;5;241m.\u001b[39m_execution_engine\u001b[38;5;241m.\u001b[39mrun_backward( \u001b[38;5;66;03m# Calls into the C++ engine to run the backward pass\u001b[39;00m\n\u001b[1;32m 267\u001b[0m tensors,\n\u001b[1;32m 268\u001b[0m grad_tensors_,\n\u001b[1;32m 269\u001b[0m retain_graph,\n\u001b[1;32m 270\u001b[0m create_graph,\n\u001b[1;32m 271\u001b[0m inputs,\n\u001b[1;32m 272\u001b[0m allow_unreachable\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m 273\u001b[0m accumulate_grad\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m 274\u001b[0m )\n", - "\u001b[0;31mKeyboardInterrupt\u001b[0m: " + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "524e1f2fe9674b508c3b41e54ce5becb", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/156 [00:00" ] @@ -404,6 +557,62 @@ "plt.ylim(0,)\n", "plt.show()" ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAGxCAYAAACeKZf2AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABSe0lEQVR4nO3deVxU5f4H8M/MwMyAMAPIvigqijsgKqmZWiil10S7Vypv+LPtVloaLS6ZtoqWlpWm7WbdUiuXbhguKFqmmSyliLgLAsOiMuzbzPn9gU5NAjLIcGaGz/v1mteNw3POfB/PHefjOc95HokgCAKIiIiIbIRU7AKIiIiI2hLDDREREdkUhhsiIiKyKQw3REREZFMYboiIiMimMNwQERGRTWG4ISIiIpvCcENEREQ2xU7sAtqbXq9HXl4enJ2dIZFIxC6HiIiIWkAQBJSVlcHX1xdSafPXZjpcuMnLy0NAQIDYZRAREVEr5OTkwN/fv9k2HS7cODs7A2j4w1GpVE22O3DgAN59912kp6dDo9Hgv//9L/7xj3802f7777/HJ598gqNHj6K2tha9e/fGvHnzEBkZ2epjEhERUYPS0lIEBAQYvseb0+HCzbVbUSqVqtlwAwDh4eF49NFHMWXKFDg6Ojbb/siRI7jrrrvwxhtvwMXFBZ999hnuvfde/PrrrwgLC2vVMYmIiMhYS4aUSDrawpmlpaVQq9XQarUtDhYSiQRbtmxBdHS0Se/Vr18/xMTEYNGiRW12TCIioo7IlO9vPi1lJnq9HmVlZXBzcxO7FCIiog6F4cZMli9fjvLyckydOlXsUoiIiDqUDjfmpj189dVXePnll7Ft2zZ4enqKXQ4REVGHwnDTxjZs2ICHH34Y33zzjdGTUkRERNQ+eFuqDX399deYMWMGvv76a0yYMEHscoiIiDokXrlpQnl5OU6fPm34+dy5c0hPT4ebmxu6dOmC+fPnIzc3F+vXrwfQcCtq+vTpeOeddxAREQGNRgMAcHBwgFqtbtExiYiI6ObxUfAmJCcnY8yYMddtnz59OtatW4f/+7//w/nz55GcnAwAGD16NPbt29dk+5Yck4iIiBpnyqPgDDdtKF9bhXPFFejm3gk+aoc2PTYREVFHZsr3N29LtZGNv2Vj/uaj0AuAVALETxmAmCG81URERNTeRB1QvH//fkycOBG+vr6QSCTYunVrs+03b96MsWPHwsPDAyqVCsOGDcOOHTvap9hm5GurMO9qsAEAvQAs2HwM+doqcQsjIiLqgEQNNxUVFQgJCcHq1atb1H7//v0YO3Ystm/fjpSUFIwZMwYTJ05EWlqamStt3rniCvz95p5OEHC+uFKcgoiIiDowUW9L3XXXXbjrrrta3H7lypVGPy9ZsgTbtm3D//73P6PFKdtbN/dOkEpguHIDNNyaCnR3FK0mIiKijsqq57lpyfpNNTU1KC0tNXq1NR+1A+KnDID0LwuV3h3iy0HFREREIrDqcNOS9Zvi4+OhVqsNr4CAALPUEjOkCw7Mux0zRgQCAH46VYyy6jqzvBcRERE1zWrDzbX1mzZt2tTs+k3z58+HVqs1vHJycsxWk4/aAQvG90F3j064VFGLNclnzPZeRERE1DirDDfX1m/atGnTDddvUigUUKlURi9zspdJMf+uPgCAT34+h9wSPjFFRETUnqwu3FjD+k2RfTwR0c0NNfV6LN+RJXY5REREHYqo4aa8vBzp6elIT08H8OdaS9nZ2QAabinFxsYa2n/11VeIjY3FihUrDOs3aTQaaLVaMcpvkkQiwcIJfQEAW9Jy8cfFEnELIiIi6kBEDTdHjhxBWFiY4THuuLg4hIWFYdGiRQCA/Px8Q9ABgA8//BD19fWYOXMmfHx8DK/Zs2eLUn9zBvirMSXMDwDwekImOtgqF0RERKLh2lJmlFdShTHLk1FTr8eHD4RjXD9vs74fERGRrTLl+9vqxtxYE18XBzw8shsAYOmPJ1Cn04tcERERke1juDGzx0b1gLuTHGeLK/DVr9k33oGIiIhuCsONmTkr7TEnshcAYOXuk9BWcWI/IiIic2K4aQf3DglAkKcTrlTW4f3k02KXQ0REZNMYbtqBnUyKBeN7AwA++/k8ci5ztXAiIiJzYbhpJ2OCPTEiqDNqdXq8yYn9iIiIzIbhpp1IJBIsGN8HEgnw/e95SM8pEbskIiIim8Rw0476+apxzyB/AMDrCcc5sR8REZEZMNy0s2fHBUNpL8Vv569gR4ZG7HKIiIhsDsNNO/NWK/HoyO4AGib2q63nxH5ERERtieFGBI+O6gF3JwXOX6rEl4cuiF0OERGRTWG4EYGTwg7PjGuY2O/dPaegreTEfkRERG2F4UYkUwcHINjLGSWVdVi195TY5RAREdkMhhuRyKQSzL86sd/nv1xA9iVO7EdERNQWGG5ENDrYEyN7uqNWp8eyHSfELoeIiMgmMNyI7NrEfgl/5CPlwhWxyyEiIrJ6DDci6+OjwtTwAADAa5zYj4iI6KYx3FiAuHG94GAvQ1p2CbYf5cR+REREN4PhxgJ4qZT4z6irE/slZqKmXidyRURERNaL4cZCPHpbd3g6K5BzuQpfHOTEfkRERK3FcGMhHOV2eHZcMADg3aRTuFJRK3JFRERE1onhxoLcE+6P3t7OKK2ux3t7TotdDhERkVViuLEgMqkEL0zoAwD44tB5nC+uELkiIiIi68NwY2FG9vTA6GAP1OkELP2RE/sRERGZiuHGAi0Y3wdSCZCYocFv5y+LXQ4REZFVYbixQL28nBEzpAsA4LWETOj1nNiPiIiopRhuLNTTY3uik1yG33NK8MPRfLHLISIishoMNxbK01mJx0b1AAAs+/EEqus4sR8REVFLMNxYsIdHdoe3Sonckip8/st5scshIiKyCgw3FsxBLsOzUQ0T+63aexqXObEfERHRDTHcWLgpYX7o66NCWXU93tl9UuxyiIiILB7DjYWTSiVYeHViv//+mo0zReUiV0RERGTZGG6swPAgd9zR2xP1ek7sR0REdCMMN1Zi/vjekEkl2HW8AIfOXhK7HCIiIovFcGMlgjydcd/QAADA65zYj4iIqEkMN1ZkTmQvOCnscDRXi+9/zxO7HCIiIovEcGNF3J0UeHx0w8R+byRyYj8iIqLGMNxYmYdu7QZftRJ52mp88vM5scshIiKyOAw3VkZpL8NzdzZM7Lcm+QyKy2tEroiIiMiyMNxYoUkhfhjgp0Z5TT1WcmI/IiIiIww3VkgqleCFqxP7fX04B6cLy0SuiIiIyHIw3FipW7p3xti+XtDpBcRv58R+RERE1zDcWLH5d/WGnVSCpBOF+OV0sdjlEBERWQSGGyvW3cMJ0yK6AABe386J/YiIiACGG6s3O7IXnBV2yMgrxea0XLHLISIiEh3DjZVz6yTHzNuDAADLd2ShqpYT+xERUcfGcGMD/m94IPxcHKAprcbHP50VuxwiIiJRMdzYAKW9DM9fm9hv3xkUllWLXBEREZF4RA03+/fvx8SJE+Hr6wuJRIKtW7fecJ/k5GQMGjQICoUCQUFBWLdundnrtAZ3h/giJMAFlbU6vL3rlNjlEBERiUbUcFNRUYGQkBCsXr26Re3PnTuHCRMmYMyYMUhPT8ecOXPw8MMPY8eOHWau1PJJJBIsvDqx38bfsnGygBP7ERFRxyQRBMEinh+WSCTYsmULoqOjm2wzd+5cJCQk4NixY4Zt9957L0pKSpCYmNii9yktLYVarYZWq4VKpbrZsi3OY1+kIDFDg9HBHlg3Y6jY5RAREbUJU76/rWrMzcGDBxEZGWm0LSoqCgcPHmxyn5qaGpSWlhq9bNm8qxP7JWcV4adTRWKXQ0RE1O6sKtxoNBp4eXkZbfPy8kJpaSmqqqoa3Sc+Ph5qtdrwCggIaI9SRRPo3gkPDOsKAHg9IRM6TuxHREQdjFWFm9aYP38+tFqt4ZWTkyN2SWb31O09oVLa4YSmDN+lXBS7HCIionZlVeHG29sbBQUFRtsKCgqgUqng4ODQ6D4KhQIqlcroZetcO8nx5O09AQDLd2ahsrZe5IqIiIjaj1WFm2HDhiEpKclo265duzBs2DCRKrJcscO7ooubIwrLavDhfk7sR0REHYeo4aa8vBzp6elIT08H0PCod3p6OrKzswE03FKKjY01tH/sscdw9uxZPP/88zhx4gTef/99bNq0CU8//bQY5Vs0hZ0Mc+/sDQD4YN9ZFJZyYj8iIuoYRA03R44cQVhYGMLCwgAAcXFxCAsLw6JFiwAA+fn5hqADAN26dUNCQgJ27dqFkJAQrFixAh9//DGioqJEqd/SjR/gjUFdXFBVp8OKnSfFLoeIiKhdWMw8N+3F1ue5+buUC1dwz5pfIJEA258aiT4+tt9nIiKyPTY7zw2ZLryrKyYM8IEgAEu2Z4pdDhERkdkx3HQAc+/sDXuZBD+dKkZyVqHY5RAREZkVw00H0KWzI6YPCwTQcPWmXqcXtyAiIiIzYrjpIJ68vSdcHO1xsqAc33BiPyIismEMNx2E2tEeT12d2G/FzpOoqOHEfkREZJsYbjqQf9/SFYGdHVFcXoMP9p0RuxwiIiKzYLjpQOR2Usy7q2Fivw9/Oot8beOLjRIREVkzhpsOJqqfN4YEuqK6Ts+J/YiIyCYx3HQwEokEC8b3AQB8l3oRGXlakSsiIiJqWww3HVBYF1dMDPGFIACvJ2Sig01STURENo7hpoN6PioYcjspfjlzCXs5sR8REdkQhpsOKsDNETNGBAIAlmw/wYn9iIjIZjDcdGBPjA6Cq6M9TheWY8NvOWKXQ0RE1CYYbjowtYM95kT2AgC8veskyqrrRK6IiIjo5jHcdHD3R3RBd/dOuFRRi7Wc2I+IiGwAw00HZy/7c2K/j386h7wSTuxHRETWjeGGMLavF4Z2c0NNvR7Ld2SJXQ4REdFNYbghSCQSLJzQMLHf5rRcHL3Iif2IiMh6MdwQAGCgvwsmh/kBAF7ffpwT+xERkdViuCGDZ6OCobCT4tDZy9idyYn9iIjIOjHckIGfiwMeurUbACB+eybqOLEfERFZIYYbMvL46B7o3EmOs8UV+PpwttjlEBERmYzhhow4K+0xZ2zDxH4rd59CKSf2IyIiK8NwQ9e5b0gAenh0wuWKWry/lxP7ERGRdWG4oevYyaRYML7h0fBPD5xDzuVKkSsiIiJqOYYbatTtvT0xvEdn1NbrsXwnJ/YjIiLrwXBDjZJIJFgwvg8kEmBbeh7Sc0rELomIiKhFGG6oSf391JgS5g8AeD2BE/sREZF1YLihZj0b1QtKeyl+O38FOzIKxC6HiIjohhhuqFk+agc8MrI7AGDpj5morefEfkREZNkYbuiG/jOqB9ydFDh/qRL//fWC2OUQERE1i+GGbshJYYe4qxP7vZN0CtpKTuxHRESWi+GGWmTqYH/08nJCSWUdViefFrscIiKiJjHcUIvYyaSYf3Viv3UHznNiPyIislgMN9Rio3t5YGRPd9Tq9FiaeELscoiIiBrFcEMtJpFIMP+uhon9Ev7IR8qFK2KXREREdB2GGzJJX18V/hXOif2IiMhyMdyQyZ4ZFwwHexlSs0vw4zGN2OUQEREZYbghk3mplHj0tmsT+51ATb1O5IqIiIj+xHBDrfKfUd3h6axA9uVKfHGQE/sREZHlYLihVnGU2+GZcQ0T+7235zRKKmtFroiIiKgBww212j/DA9Db2xnaqjq8m8SJ/YiIyDIw3FCryaQSLLg6sd8Xh87jfHGFyBUREREx3NBNuq2XB0b18kCdTsAyTuxHREQWgOGGbtqC8X0glQA/HtPgyPnLYpdDREQdHMMN3bRgb2fEDAkAALyWkMmJ/YiISFQMN9Qmnh7bC45yGdJzSvDDH/lil0NERB2Y6OFm9erVCAwMhFKpREREBA4fPtxs+5UrVyI4OBgODg4ICAjA008/jerq6naqlpri6azEY6N6AACWJZ5AdR0n9iMiInGIGm42btyIuLg4LF68GKmpqQgJCUFUVBQKCwsbbf/VV19h3rx5WLx4MTIzM/HJJ59g48aNWLBgQTtXTo15ZGR3eKkUuHilCp//cl7scoiIqIMSNdy89dZbeOSRRzBjxgz07dsXa9euhaOjIz799NNG2//yyy8YMWIE7r//fgQGBmLcuHG47777bni1h9qHg1yGZ8cFAwBW7T2NyxWc2I+IiNqfaOGmtrYWKSkpiIyM/LMYqRSRkZE4ePBgo/sMHz4cKSkphjBz9uxZbN++HePHj2+XmunGpgzyR18fFcqq6/Fu0imxyyEiog5ItHBTXFwMnU4HLy8vo+1eXl7QaBpfafr+++/HK6+8gltvvRX29vbo0aMHRo8e3extqZqaGpSWlhq9yHxkUglemNAwsd+Xhy7gbFG5yBUREVFHI/qAYlMkJydjyZIleP/995GamorNmzcjISEBr776apP7xMfHQ61WG14BAQHtWHHHNCLIHbf39kS9XsDSHzmxHxERtS/Rwo27uztkMhkKCgqMthcUFMDb27vRfV588UU88MADePjhhzFgwABMnjwZS5YsQXx8PPR6faP7zJ8/H1qt1vDKyclp877Q9RaM7w2ZVIKdxwvw69lLYpdDREQdiGjhRi6XIzw8HElJSYZter0eSUlJGDZsWKP7VFZWQio1LlkmkwFAkxPHKRQKqFQqoxeZX5CnM+69OrHf69szoddzYj8iImofot6WiouLw0cffYTPP/8cmZmZePzxx1FRUYEZM2YAAGJjYzF//nxD+4kTJ2LNmjXYsGEDzp07h127duHFF1/ExIkTDSGHLMfTY3vBSWGHPy5q8f3veWKXQ0REHYSdmG8eExODoqIiLFq0CBqNBqGhoUhMTDQMMs7Ozja6UrNw4UJIJBIsXLgQubm58PDwwMSJE/H666+L1QVqhruTAo+P7oE3d2ThzR1ZuLO/N5T2DKFERGReEqGDLQRUWloKtVoNrVbLW1TtoLpOhzHLk5GvrcbzdwbjidFBYpdERERWyJTvb6t6Woqsj9JehueiGib2e3/vGVwqrxG5IiIisnUmh5uqqipUVlYafr5w4QJWrlyJnTt3tmlhZDuiQ/0wwE+N8pp6rNzNif2IiMi8TA43kyZNwvr16wEAJSUliIiIwIoVKzBp0iSsWbOmzQsk6yeVSrBgfMPEfl8dzsbpQk7sR0RE5mNyuElNTcXIkSMBAN9++y28vLxw4cIFrF+/Hu+++26bF0i2YViPzojs4wWdXsDSHzPFLoeIiGyYyeGmsrISzs7OAICdO3diypQpkEqluOWWW3DhwoU2L5Bsx/yrE/vtzizEL2eKxS6HiIhslMnhJigoCFu3bkVOTg527NiBcePGAQAKCwv59BE1q4eHE6ZFdAEAvJ7Aif2IiMg8TA43ixYtwrPPPovAwEBEREQYZhPeuXMnwsLC2rxAsi2z7+gJZ4UdMvJKsSUtV+xyiIjIBrVqnhuNRoP8/HyEhIQYJtk7fPgwVCoVevfu3eZFtiXOcyO+NclnsCzxBHzUSux5ZjQc5JzYj4iImmf2eW68vb0RFhYGqVSK0tJSbN26Fc7OzhYfbMgyzBgRCD8XB+Rrq/HJz2fFLoeIiGyMyeFm6tSpWLVqFYCGOW8GDx6MqVOnYuDAgfjuu+/avECyPUp7GZ6/s2FivzXJZ1BUxon9iIio7Zgcbvbv3294FHzLli0QBAElJSV499138dprr7V5gWSbJg70RYi/GhW1Ory9+6TY5RARkQ0xOdxotVq4ubkBABITE3HPPffA0dEREyZMwKlTnH2WWkYqleCFCX0BABsOZ+NUQZnIFRERka0wOdwEBATg4MGDqKioQGJiouFR8CtXrkCpVLZ5gWS7hnZzQ1Q/L+gFYMl2TuxHRERtw+RwM2fOHEybNg3+/v7w9fXF6NGjATTcrhowYEBb10c2bt5dfWAnlWBvVhF+PsWJ/YiI6OaZHG6eeOIJHDx4EJ9++il+/vlnw6Pg3bt355gbMlk390749y1dAQCvJRyHjhP7ERHRTWrVPDfXXNtVIpG0WUHmxnluLM+VilqMenMvSqvr8cY/B2Lq4ACxSyIiIgtj9nlu1q9fjwEDBsDBwQEODg4YOHAgvvjii1YVS+TaSY4nb+8JAFixMwuVtfUiV0RERNbM5HDz1ltv4fHHH8f48eOxadMmbNq0CXfeeScee+wxvP322+aokTqA2OFdEeDmgILSGny0/5zY5RARkRUz+bZUt27d8PLLLyM2NtZo++eff46XXnoJ585Z9hcTb0tZrh/+yMOsr9LgKJch+dnR8FTx6TsiImpg1ttS+fn5GD58+HXbhw8fjvz8fFMPR2QwYYAPwrq4oLJWh7d2cWI/IiJqHZPDTVBQEDZt2nTd9o0bN6Jnz55tUhR1TBKJBAsn9AEAbDqSgxOaUpErIiIia2Rn6g4vv/wyYmJisH//fowYMQIAcODAASQlJTUaeohMEd7VDeMHeGP7UQ2WbD+B9Q8OFbskIiKyMiZfubnnnnvw66+/wt3dHVu3bsXWrVvh7u6Ow4cPY/LkyeaokTqYuXf2hr1Mgv0ni7DvZJHY5RARkZW5qXlurBEHFFuH1344jo9/PodgL2dsnz0SMqn1zKVERERtz5Tv7xbdliotbfnYBwYGaguzbg/CNykXkVVQhm+O5ODeoV3ELomIiKxEi8KNi4vLDWchFgQBEokEOp2uTQqjjs3FUY6n7uiJV384jhW7TmJiiC86KUweIkZERB1Qi74t9u7da+46iK7zwC1dsf7geVy4VIkP9p9F3NheYpdERERWgGNuyKL9eDQfj/83FUp7KZKfHQNvNSf2IyLqiMy+thRRe7mzvzcGd3VFdZ0eK3ZmiV0OERFZAYYbsmgSiQQvXJ3Y79vUi8jI04pcERERWTqGG7J4YV1cMTHEF4IALNmeiQ52J5WIiEzEcENW4fmoYMhlUhw4fQnJWZzYj4iImtaqcFNfX4/du3fjgw8+QFlZGQAgLy8P5eXlbVoc0TUBbo6YMSIQQMPVm3qdXtyCiIjIYpkcbi5cuIABAwZg0qRJmDlzJoqKGv4VvWzZMjz77LNtXiDRNU+MCYKroz1OFZZj45EcscshIiILZXK4mT17NgYPHowrV67AwcHBsH3y5MlISkpq0+KI/krtYI/ZdzSsPP/2rpMoq64TuSIiIrJEJoebn376CQsXLoRcLjfaHhgYiNzc3DYrjKgx90d0RTf3Tigur8UH+86KXQ4REVkgk8ONXq9vdImFixcvwtnZuU2KImqK3E6KeXf1BgB89NNZ5JVUiVwRERFZGpPDzbhx47By5UrDzxKJBOXl5Vi8eDHGjx/flrURNWpcXy8M7eaGmno9lnNiPyIi+huTw82KFStw4MAB9O3bF9XV1bj//vsNt6SWLVtmjhqJjEgkEiy8OrHf5tRcHMvlxH5ERPSnVq0tVV9fjw0bNuCPP/5AeXk5Bg0ahGnTphkNMLZUXFvKdszZkIat6Xm4pbsbvn7klhuuXE9ERNbLlO/vFq0Kft1Odnb497//3ariiNrKs1HB2H5Mg0NnLyMpsxCRfb3ELomIiCyAyeHm+++/b3S7RCKBUqlEUFAQunXrdtOFEd2Iv6sjHrq1G9Ykn8GSHzMxKtgD9jJOuk1E1NGZHG6io6MhkUiuW9/n2jaJRIJbb70VW7duhaura5sVStSYx0f3wMbfcnC2qAIbDmfjgWGBYpdEREQiM/mfubt27cKQIUOwa9cuaLVaaLVa7Nq1CxEREfjhhx+wf/9+XLp0ibMVU7tQKe3xdOTVif12n0IpJ/YjIurwTB5Q3L9/f3z44YcYPny40fYDBw7g0UcfRUZGBnbv3o0HH3wQ2dnZbVpsW+CAYttTp9PjzpX7caaoAo+P7oG5d/YWuyQiImpjpnx/m3zl5syZM40eVKVS4ezZhhlje/bsieLiYlMPTdQq9jIp5t/V8Gj4Jz+fw8UrlSJXREREYjI53ISHh+O5554zLJgJAEVFRXj++ecxZMgQAMCpU6cQEBDQdlUS3cAdfTwxrHtn1NbrsXwHJ/YjIurITA43n3zyCc6dOwd/f38EBQUhKCgI/v7+OH/+PD7++GMAQHl5ORYuXNjmxRI1RSKR4IUJfSCRAFvT8/B7TonYJRERkUhMDjfBwcE4fvw4tm3bhqeeegpPPfUUvv/+e2RkZKBXr14AGp6oeuCBB1p0vNWrVyMwMBBKpRIRERE4fPhws+1LSkowc+ZM+Pj4QKFQoFevXti+fbup3SAb1N9PjclhfgCA1xMyr3uij4iIOoZWTeInlUpx55134s4777ypN9+4cSPi4uKwdu1aREREYOXKlYiKikJWVhY8PT2va19bW4uxY8fC09MT3377Lfz8/HDhwgW4uLjcVB1kO54dF4yEP/Jx+Pxl7DxegKh+3mKXRERE7axVyy9UVFRg3759yM7ORm1trdHvnnrqqRYfJyIiAkOGDMGqVasANKw4HhAQgCeffBLz5s27rv3atWvx5ptv4sSJE7C3tze1bAB8WqojWL4jC6v2nkY3907YMec2yO04sR8RkbUz5fvb5HCTlpaG8ePHo7KyEhUVFXBzc0NxcTEcHR3h6elpeGLqRmpra+Ho6Ihvv/0W0dHRhu3Tp09HSUkJtm3bdt0+48ePh5ubGxwdHbFt2zZ4eHjg/vvvx9y5cyGTyVr0vgw3tq+8ph6j39yL4vJavDSxL/5vBGfMJiKydmZ9FPzpp5/GxIkTceXKFTg4OODQoUO4cOECwsPDsXz58hYfp7i4GDqdDl5exusBeXl5QaPRNLrP2bNn8e2330Kn02H79u148cUXsWLFCrz22mtNvk9NTQ1KS0uNXmTbnBR2eHpsw/ivd5JOQVvFif2IiDoSk8NNeno6nnnmGUilUshkMtTU1CAgIABvvPEGFixYYI4aDfR6PTw9PfHhhx8iPDwcMTExeOGFF7B27dom94mPj4darTa8+Ih6xxAzOAA9PZ1wpbIO7+89LXY5RETUjkwON/b29pBKG3bz9PQ0zEKsVquRk5PT4uO4u7tDJpOhoKDAaHtBQQG8vRsfBOrj44NevXoZ3YLq06cPNBrNdWN/rpk/f75hmQitVmtSjWS97GRSLBjfMLHfZwfOI+cyJ/YjIuooTA43YWFh+O233wAAo0aNwqJFi/Df//4Xc+bMQf/+/Vt8HLlcjvDwcCQlJRm26fV6JCUlYdiwYY3uM2LECJw+fRp6vd6w7eTJk/Dx8YFcLm90H4VCAZVKZfSijmF0sAduDXJHrU6PZYknxC6HiIjaicnhZsmSJfDx8QEAvP7663B1dcXjjz+OoqIifPjhhyYdKy4uDh999BE+//xzZGZm4vHHH0dFRQVmzJgBAIiNjcX8+fMN7R9//HFcvnwZs2fPxsmTJ5GQkIAlS5Zg5syZpnaDOgCJRIIF4xsm9vvhj3ykZl8RuyQiImoHJs1zIwgCPD09DVdoPD09kZiY2Oo3j4mJQVFRERYtWgSNRoPQ0FAkJiYaBhlnZ2cbboEBQEBAAHbs2IGnn34aAwcOhJ+fH2bPno25c+e2ugaybX19VfjnIH98k3IRrydk4tvHhkEikYhdFhERmZFJj4Lr9XoolUpkZGSgZ8+e5qzLbPgoeMej0VZjzPJkVNXpsGbaINw1wEfskoiIyERmexRcKpWiZ8+euHTp0k0VSNSevNVKPHJbdwDA0sQTqK3X32APIiKyZiaPuVm6dCmee+45HDt2zBz1EJnFf27rDg9nBS5cqsQXhy6IXQ4REZmRyTMUu7q6orKyEvX19ZDL5XBwcDD6/eXLl9u0wLbG21Id14bD2Zi3+SjUDvbY99xouDg2/oQdERFZHlO+v01eOHPlypWtrYuo3a1evRpvvvkmNBoNBoaEwHvYg9DAH+/tOY0X/9HXqO26desMT+pdo1AoUF1dDQCoq6vDwoULsX37dpw9exZqtRqRkZFYunQpfH19261PRETUPJPDzfTp081RB1Gba2zV+a8/mwuX2NVYf/A8Yod1RdfOnYz2UalUyMrKMvz81yerKisrkZqaihdffBEhISG4cuUKZs+ejbvvvhtHjhxpt34REVHzWrUq+JkzZ/DZZ5/hzJkzeOedd+Dp6Ykff/wRXbp0Qb9+/cxRZ5vhbamOo6lV572GReNy0HiMH+CN96eFG9qvW7cOc+bMQUlJSYvf47fffsPQoUNx4cIFdOnSpa27QEREV5l14cx9+/ZhwIAB+PXXX7F582aUl5cDAH7//XcsXry4dRUTtbHa2lqkpKQgMjLSsE0qlSIyMhKu5echlQDbj2qQcsF4jFh5eTm6du2KgIAATJo0CRkZGc2+j1arhUQigYuLizm6QURErWByuJk3bx5ee+017Nq1y2jJg9tvvx2HDh1q0+KIWqu5VefLrxRj6uCGBVRfS8jEtYuXwcHB+PTTT7Ft2zZ8+eWX0Ov1GD58OC5evNjoe1RXV2Pu3Lm47777eBWQiMiCmBxujh49ismTJ1+33dPTE8XFxW1SFJG5xY3tBUe5DGnZJUg4mg8AGDZsGGJjYxEaGopRo0Zh8+bN8PDwwAcffHDd/nV1dZg6dSoEQcCaNWvau3wiImqGyeHGxcUF+fn5121PS0uDn59fmxRFdLNutOq8p0qJ/9zWAwCwLPEEaup11x3D3t4eYWFhOH36tNH2a8HmwoUL2LVrF6/aEBFZGJPDzb333ou5c+dCo9FAIpFAr9fjwIEDePbZZxEbG2uOGolM1pJV5x+5rRu8VArkXK7C57+cv+4YOp0OR48eNSwUC/wZbE6dOoXdu3ejc+fOZu8LERGZxuRHwa+twh0QEACdToe+fftCp9Ph/vvvx8KFC81RI1GrxMXFYfr06Rg8eDCGDh2KlStXGq06/9jDD8JH6owC7/F4b89pnNu5HmNuG4GgoCCUlJTgzTffxIULF/Dwww8DaAg2//znP5GamooffvgBOp0OGo0GAODm5mY0Bo2IiMTTqkfBgYYVu48dO4by8nKEhYVZzUKafBS8Y1m1apVhEr/Q0FC8++67iIiIAACMHj0aXbsGonDQQ8jML4X38Y3I/30fNBoNXF1dER4ejtdeew1hYWEAgPPnz6Nbt26Nvs/evXsxevTo9uoWEVGHY8r3t8nh5ueff8att956UwWKieGG/u7nU8X49ye/wk4qwVePRKBeL6Cbeyf4qB1uvDMREbULs4YbuVwOPz8/3Hffffj3v/+Nvn373ngnC8JwQ42Z8dlh7M0qMvwslQDxUwYgZggn5iMisgRmncQvLy8PzzzzDPbt24f+/fsjNDQUb775ZpNzgRBZg0dGdjf6WS8ACzYfQ762SqSKiIiotUwON+7u7pg1axYOHDiAM2fO4F//+hc+//xzBAYG4vbbbzdHjUTmJ7l+k04Q8H16HnT6Vg1LIyIikbR6QPE1Op0OP/74I1588UX88ccf0Omuny/EkvC2FDUmX1uFEUv3oLEc4+mswN0hvpg8yA99fVRGi2kSEVH7MOttqWsOHDiAJ554Aj4+Prj//vvRv39/JCQktPZwRKLyUTsgfsoAyK4GF6kEuKW7G1wc7VFYVoOPfz6HCe/+jDtX/oS1+87wdhURkQUz+crN/PnzsWHDBuTl5WHs2LGYNm0aJk2aBEdHR3PV2KZ45Yaak6+twvniSgS6O8JH7YDaej2SswqxJS0XSZmFqNXpAQASCTCse2dMDvPDXQN84KQwecooIiIygVmflhoxYgSmTZuGqVOnwt3d/aYKFQPDDbWWtrIO24/lY0tqLg6f/3M1caW9FGP7emNKmB9G9nSHnazVF0SJiKgJZg031o7hhtpCzuVKbEvPxea0XJwtqjBsd3eS4x8DfTFlkB8G+Kk5PoeIqI20S7g5fvw4srOzUVtba7T97rvvbs3h2g3DDbUlQRDwx0UttqTl4n+/5+FSxZ+fhx4enTBlkD8mhfrC39U6btsSEVkqs4abs2fPYvLkyTh69CgkEgmu7X7tX6h8Woo6qjqdHj+dKsLm1FzsOl6Amnq94XdDu7lhytXxOWoHexGrJCKyTmYNNxMnToRMJsPHH3+Mbt264fDhw7h06RKeeeYZLF++HCNHjryp4s2N4YbaQ2l1HRKPabAlNReHzl3CtU+Z3E6KyD6emBzmj1G9PCC34/gcIqKWMGu4cXd3x549ezBw4ECo1WocPnwYwcHB2LNnD5555hmkpaXdVPHmxnBD7S2vpArb0vOwJe0iThaUG7a7OtpjYogvosP8EBbgwvE5RETNMOX72+TnV3U6HZydnQE0BJ28vDwEBweja9euyMrKal3FRDbM18UBj4/ugcdGdUdGXim2puVi2+95KCqrwfqDF7D+4AV0c++E6FA/TA7zQ5fOHJ9DRHQzTA43/fv3x++//45u3bohIiICb7zxBuRyOT788EN07979xgcg6qAkEgn6+6nR30+NeXf1xoEzl7Al9SJ2ZBTgXHEF3t59Em/vPonwrq6YHOaHfwz0gYujXOyyiYisjsm3pXbs2IGKigpMmTIFp0+fxj/+8Q+cPHkSnTt3xsaNGy1+fSneliJLU1FTjx0ZGmxJy8WB08WGJSDsZRKMCfbElEF+GNPbEwo7mbiFEhGJqN3nubl8+TJcXV2tYswAww1ZsoLSanyfnofNabnIzC81bFc72GPCQB9MDvPD4K7W8VkjImpLnMSvGQw3ZC1OaEqxJS0X29LyoCmtNmwPcHPA5FA/RIf5obuHk4gVEhG1H4abZjDckLXR6QUcOnsJm1NzkXgsHxW1f84lFRLggilhfpgY4gu3ThyfQ0S2i+GmGQw3ZM2qanXYebxhfM5Pp4qhuzpAx04qwehgD0SH+SGyjxeU9hyfQ0S2heGmGQw3ZCuKymrwv9/zsCUtF0dztYbtzgo7jB/gg8mD/DA00A1SKcfnEJH1Y7hpBsMN2aLThWXYkpaLrWl5yC2pMmz3c3HApNCGhTyDPJ1FrJCI6OYw3DSD4YZsmV4v4PD5y9iSmovtR/NRVlNv+N0APzWiw/xwd4gvPJwVIlZJRGQ6hptmMNxQR1Fdp0NSZiG2pF1EclYR6q+Oz5FJJRjZ0x2Tw/wwrq83HOQcn0NElo/hphkMN9QRXSqvQcLRfGxOzUV6Tolheye5DHf298GUQX64pXtnyDg+h4gsFMNNMxhuqKM7W1SOrVcX8sy5/Of4HG+VEpNCfTF5kB96e/OzQUSWheGmGQw3RA0EQUDKhSvYnJaLhD/yoa2qM/yuj48Kk8N8MSnUD14qpYhVEhE1YLhpBsMN0fVq6nXYe6IIW9IuYs+JQtTpGv5akEqAEUHuiA71w539vdFJYfJau0REbYLhphkMN0TNK6msRcLRfGxJzcWRC1cM2x3sZYjq54XJg/wxokdn2MmkIlZJRB0Nw00zGG6IWi77UiW2pudiS1ouzhVXGLZ7OCtwd4gvJof5oZ+vigt5EpHZMdw0g+GGyHSCICA9pwRb0nLxv9/zcKXyz/E5vbycEB3mh+hQP/i6OIhYJRHZMlO+v3ldmYhuSCKRIKyLK16Z1B+/LojEx7GDMWGAD+R2UpwsKMcbiVkYsWwP7vvwEDYdyUFZdd2ND9pCq1evRmBgIJRKJSIiInD48OEm227evBmDBw+Gi4sLOnXqhNDQUHzxxRdGbcrLyzFr1iz4+/vDwcEBffv2xdq1a9usXiISH6/cEFGraavqkHisYf6cX89dNmxX2Ekxtq8Xpgzyw8ieHrBv5ficjRs3IjY2FmvXrkVERARWrlyJb775BllZWfD09LyufXJyMq5cuYLevXtDLpfjhx9+wDPPPIOEhARERUUBAB599FHs2bMHH3/8MQIDA7Fz50488cQT2Lx5M+6+++7W/UEQkdnxtlQzGG6IzOPilUpsS8/D5tSLOFP05/iczp3kmHh1fM5Af7VJ43MiIiIwZMgQrFq1CgCg1+sREBCAJ598EvPmzWvRMQYNGoQJEybg1VdfBQD0798fMTExePHFFw1twsPDcdddd+G1115rcW1E1L54W4qI2p2/qyNmjgnC7rhR+N+sWzFjRCDcneS4VFGLdb+cx6TVB3DHW/uwas8p5FyuvOHxamtrkZKSgsjISMM2qVSKyMhIHDx48Ib7C4KApKQkZGVl4bbbbjNsHz58OL7//nvk5uZCEATs3bsXJ0+exLhx41rXcSKyOBYRbky5p/5XGzZsgEQiQXR0tHkLJKIWk0gkGOCvxuKJ/XBo/h34bMYQ3B3iC6W9FGeLKrB850mMfGMvpq49iK8PZxtNHvhXxcXF0Ol08PLyMtru5eUFjUbT5PtrtVo4OTlBLpdjwoQJeO+99zB27FjD79977z307dsX/v7+kMvluPPOO7F69WqjAERE1k30Gbk2btyIuLg4o3vqUVFRTd5Tv+b8+fN49tlnMXLkyHaslohMYSeTYkywJ8YEe6Ksug47MgqwJe0ifjlzCYfPX8bh85exeFsG7ujjiclhfhgd7Am53c39m8vZ2Rnp6ekoLy9HUlIS4uLi0L17d4wePRpAQ7g5dOgQvv/+e3Tt2hX79+/HzJkz4evra3SViIisl+hjblpzT12n0+G2227Dgw8+iJ9++gklJSXYunVri96PY26IxJevrcK29DxsSc1FVkGZYbuLoz3+MdAHk8P80d/bEZ06dcK3335rdHV2+vTpKCkpwbZt21r0Xg8//DBycnKwY8cOVFVVQa1WY8uWLZgwYYJRm4sXLyIxMbHN+khEbctqxty09p76K6+8Ak9PTzz00EM3fI+amhqUlpYavYhIXD5qBzw2qgd2PH0btj81Eo+M7AZPZwVKKuvw5aFs3LPmF0S9+wv8evbD1oQ/A4der0dSUhKGDRvW4vfS6/WoqakBANTV1aGurg5SqfFffTKZDHq9vm06R0SiE/W2VHP31E+cONHoPj///DM++eQTpKent+g94uPj8fLLL99sqURkJn19Vejr2xfz7uqDX84UY0tqLhIzNDh/qRKVve7E55+9jaM17vjX+DE4sfNrVFRUYMaMGQCA2NhY+Pn5IT4+HkDD533w4MHo0aMHampqsH37dnzxxRdYs2YNAEClUmHUqFF47rnn4ODggK5du2Lfvn1Yv3493nrrLdH+DIiobYk+5sYUZWVleOCBB/DRRx/B3d29RfvMnz8fcXFxhp9LS0sREBBgrhKJqJVkUglG9vTAyJ4eeK22HjszCrA5zQPbq7T4fduHSP1qGRRe3RH11NtILdTj9s46ZGdnG12FqaiowBNPPIGLFy/CwcEBvXv3xpdffomYmBhDmw0bNmD+/PmYNm0aLl++jK5du+L111/HY489Jka3icgMRB1zU1tbC0dHxxbfU09PT0dYWBhkMplh27VLyVKpFFlZWejRo0ez78kxN0TWpbC0Gt//noctabnIyPvztrJKaYcJAxvmzxnc1RVSqQT52iqcK65AN/dO8FFzKQgiW2JVk/hFRERg6NCheO+99wA0hJUuXbpg1qxZ1w0orq6uxunTp422LVy4EGVlZXjnnXfQq1cvyOXyZt+P4YbIep0sKMPm1FxsS89FvrbasN3f1QG9vJyRnFUIvQBIJUD8lAGIGdJFxGqJqC2Z8v0t+m2puLg4TJ8+HYMHD8bQoUOxcuXKJu+pK5VK9O/f32h/FxcXALhuOxHZnl5ezph3V288HxWMQ+cuYUtqLn48psHFK1W4eKXK0E4vAPM2H4WnsxLDgzpDYSdr5qhEZGtEDzcxMTEoKirCokWLoNFoEBoaisTERMMg47/fUycikkolGN7DHcN7uOOVSf3xfvJpvLfH+KquIAAz1v0GO6kEPb2c0c9Xhf6+KvTzU6OPjwpOCtH/+iMiMxH9tlR7420pItuTr63CiKV7oP/b32ZqpR201fXXtZdIgG6dO6Gvrwr9/dTo56tCP1813Do1f1ubiMRjVWNu2hvDDZFt2vhbNhZsPgadIEAmkWDJlP6YOjgA+dpqHMvVIiOvFBl5Df/71/E6f+WrVqLfX8JOfz8VvFVKkxb7JCLzYLhpBsMNke3K11bhfHElAt0dm31a6lJ5zdWwU4pjeVoczyvFueKKRtu6dZIbwk6/q1d6uro5Qipl4CFqTww3zWC4IaLGlFXXITO/DBl5WhzLbbjKc6qwHLq/3+sC4KSwQ18fldFtrSBPJ9jLOD6QyFwYbprBcENELVVdp8PJgjJD2MnIK0Vmfilq6q9fqkFuJ0Vvb2ejqzx9fFRQ2vNJLaK2wHDTDIYbIroZ9To9zhZXXDeOp6yRgcsyqQQ9PDoZwk4/XzX6+qqgdrAXoXIi68Zw0wyGGyJqa4IgIOdyFY7laY1uaxWX1zbavoubI/r7qYxCj4ezop2rJrIuDDfNYLghovYgCAIKy2qMwk5GXqnRZIN/5emsMHosvZ+vCv6uDnxSi+gqhptmMNwQkZhKKmtx/OpTWtee2DpTVI7G/iZWO9hfDTt/Dlzu5u4EmZU9qbV69Wq8+eab0Gg0CAkJwXvvvYehQ4c22nbz5s1YsmQJTp8+jbq6OvTs2RPPPPMMHnjgAaM2a9euRUpKCi5fvoy0tDSEhoa2U29ILFa1/AIRUUfi4ijH8CB3DA9yN2yrrK03PKmVkdsQfE4WlEFbVYdfzlzCL2cuGdo62MvQx8fZ6CpPTy8ni11iYuPGjYiLi8PatWsRERGBlStXIioqCllZWfD09LyuvZubG1544QX07t0bcrkcP/zwA2bMmAFPT09ERUUBaFj9/dZbb8XUqVPxyCOPtHeXyArwyg0RkQWqrdfjVGEZMq7e0jp29UmtylrddW3tZRL09HQ2GsfTx0eFThawxERERASGDBmCVatWAWhYHDkgIABPPvnkdYsjN2XQoEGYMGECXn31VaPt58+fR7du3XjlpoPglRsiIisnt5NeDSpqAAEAAJ1ewLniCmRcnXjw2q2tkso6HM8vxfH8UgAXAVxdYsK9E/r/ZfLBfr4quDi23xITtbW1SElJwfz58w3bpFIpIiMjcfDgwRvuLwgC9uzZg6ysLCxbtsycpZKNYbghIrISMqkEQZ5OCPJ0wqRQPwANASC3pKph/M7Vx9OP5WlRUFqDs0UVOFtUge9/zzMcw8/FwWh5iX6+anipFGYZuFxcXAydTmdYCPkaLy8vnDhxosn9tFot/Pz8UFNTA5lMhvfffx9jx45t8/rIdjHcEBFZMYlEAn9XR/i7OiKqn7dhe/HVJSaO5f55lefCpUrkllQht6QKO48XGNq6O8nR11fdsGr61Ss9XURcYsLZ2Rnp6ekoLy9HUlIS4uLi0L17d4wePVqUesj6MNwQEdkgdycFRvXywKheHoZtpdV1OH71Ca1rg5dPF5WjuLwW+08WYf/JIkNbZ4Ud+viqjG5r9fDoBDsTlphwd3eHTCZDQUGB0faCggJ4e3s3sVfDraugoCAAQGhoKDIzMxEfH89wQy3GcENE1EGolPa4pXtn3NK9s2FbdZ0OJzR/rql1PE+LTE0ZymrqcfjcZRw+d9nQVnFtiYmr43f6+6oR7O3c5BITcrkc4eHhSEpKQnR0NICGAcVJSUmYNWtWi+vW6/WoqalpXaepQ2K4ISLqwJT2MoQGuCA0wMWwrU6nx5micsNj6Rl5pcjMK0VZTT1+v6jF7xe1hrYyqQQ9PZ3Q99o4Ht+GBUWdlQ1LTMTFxWH69OkYPHgwhg4dipUrV6KiogIzZswAAMTGxsLPzw/x8fEAgPj4eAwePBg9evRATU0Ntm/fji+++AJr1qwxvOfly5eRnZ2NvLyGsURZWVkAAG9v72avCFHHwXBDRERG7GVS9PZWobe3CveE+wMA9HoB2ZcrDQOWrw1gvlRRixOaMpzQlGFzaq7hGIGdHRvG7/iF4z/Pv4SFL76IwoIChIaGIjEx0TDIODs7G1Lpn7e6Kioq8MQTT+DixYtwcHBA79698eWXXyImJsbQ5vvvvzeEIwC49957AQCLFy/GSy+9ZM4/GrISnOeGiIhaRRAEFJRev8REbknjS0x4q5QNT2oZJiBUwc/FeImJfG0VzhVXoJt7J/ioHdqrK2QFuPxCMxhuiIjM60pFrWHQ8rGr/3uuuKLRJSZcHO0N43e0VXXYdCQHegGQSoD4KQMQM6RL+3eALBLDTTMYboiI2l9FTT0y80sNj6dn5JXiZEEZ6vXNfwXd0dsT3T06wVvtAB+1Et5qJXzVDvBwVljdGlvWypS1wT766COsX78ex44dAwCEh4djyZIlRu2bmlPpjTfewHPPPddkHQw3zWC4ISKyDDX1OpwqKEdGnha7jxdiV2bBjXe6SiaVwNNZAW+1Ej5qJXz+En4a/tcBXs4Kkx5dp+tt3LgRsbGxRmuDffPNN02uDTZt2jSMGDECw4cPh1KpxLJly7BlyxZkZGTAz69h4kmNRmO0z48//oiHHnoIp0+fRvfu3ZusheGmGQw3RESWJ19bhRFL9+CvF3KkEuDJ23uisrYe+dpqaLTVyNdWo6C0+oZXfK7t7+GsaLjqo/oz+Pi4XA1CKiW8VErI7RiAmnKza4PpdDq4urpi1apViI2NbbRNdHQ0ysrKkJSU1OyxuLYUERFZFR+1A+KnDMCCzcegEwTIJBIsmdK/0TE3Or2AS+U1yL8adjTaqr/8dzXyS6ug0VajTtcw4LmgtAa/N/Pe7k4K+Lo0hJ1rV318/nJFyEutsNhV183pZtcGA4DKykrU1dXBzc2t0d8XFBQgISEBn3/+eZvUfA3DDRERWYSYIV1wWy8PnC+uRKC7Y5NPS8mkEniqlPBUKRES0Pix9HoBlypqr17tqYKmtCH85Jc0BKFrP9fW61FcXoPi8hr8AW3jBwPQuZPc6BbYn7e//rwl1tRkhtaqtWuD/dXcuXPh6+uLyMjIRn//+eefw9nZGVOmTLnpev+K4YaIiCxGQ1C4+UfApVIJPJwV8HBWYIC/utE2giDgSmUd8kqqrl7x+ctVoJJrAagK1XV6XKqoxaWrT4E1xdXR/m8Dn5VGP/uolXCUd5yv3aVLl2LDhg1ITk6GUqlstM2nn36KadOmNfn71uo4f8pERER/IZFI4NZJDrdOcvT3azoAaavqrt72qjIa+6PRViNPW4X8kmpU1elwpbIOVyrrkJnfdABSO9gbD3xWOcDHRWm4DeatdoCTwjK+mlu7NhgALF++HEuXLsXu3bsxcODARtv89NNPyMrKwsaNG9us5mss40+QiIjIAkkkErg4yuHiKEcfn8YHsQqCgNLqesMtsL+PBboWhspr6qGtqoO2qg4nNGVNvqezwq4h/LgYD4T2Vivh69JwS8xZYdfkI9VtpbVrg73xxht4/fXXsWPHDgwePLjJdp988gnCw8MREhLS1qUz3BAREd0MiUQCtYM91A72CPZ2brJdWXWd0VWf668GVaG0uh5lNfUoKyzHqcLyJo/VSS4zGu/j87dbYL5qB6gcbj4Ambo22LJly7Bo0SJ89dVXCAwMNDz27eTkBCcnJ8NxS0tL8c0332DFihU3VV9TGG6IiIjagbPSHs5Ke/T0ajoAVdTUG4Wda2OB/joQuqSyDhW1OpwpqsCZooomj+VgLzOEncbnA3KAq6N9swEoJiYGRUVFWLRoETQazQ3XBluzZg1qa2vxz3/+0+g4f1/3a8OGDRAEAffdd9+N/thahfPcEBERWZGqWl3DYGejJ78aglDe1YHQlytqW3QshZ3UKOw0FoLcHOWQmjAbtLnWB+M8N0RERDbKQS5DN/dO6Obeqck21XU6FJQ2duvrz6tCxeW1qKnX4/ylSpy/VNnkseQyKbzUCuOrPqqG22C+Lg0/u3dSQCqVYONv2Zi/+ajo64Pxyg0REVEHVFOvQ2FpTaNjf64FoaLymkYXPP07O6kE7k4KaEqrjbbLJBL8PG9Mm1zB4ZUbIiIiapbCToYAN0cEuDk22aa2Xo/Csusff//rz4VlDcth/D3YAIBOEHC+uLJNb0+1BMMNERERNUpuJ4W/qyP8XZsOQPU6PQrLanAsV4v/fJGCv17okUkkCHRvel9z4WphRERE1Gp2Mil8XRwwrp83lt4zALKrT19dWx+sva/aALxyQ0RERG2kpeuDmRvDDREREbWZtlof7GbwthQRERHZFIYbIiIisikMN0RERGRTGG6IiIjIpjDcEBERkU1huCEiIiKbwnBDRERENoXhhoiIiGwKww0RERHZFIYbIiIisikWEW5Wr16NwMBAKJVKRERE4PDhw022/eijjzBy5Ei4urrC1dUVkZGRzbYnIiKijkX0cLNx40bExcVh8eLFSE1NRUhICKKiolBYWNho++TkZNx3333Yu3cvDh48iICAAIwbNw65ubntXDkRERFZIokgCIKYBURERGDIkCFYtWoVAECv1yMgIABPPvkk5s2bd8P9dTodXF1dsWrVKsTGxt6wfWlpKdRqNbRaLVQq1U3XT0REROZnyve3qFduamtrkZKSgsjISMM2qVSKyMhIHDx4sEXHqKysRF1dHdzc3MxVJhEREVkROzHfvLi4GDqdDl5eXkbbvby8cOLEiRYdY+7cufD19TUKSH9VU1ODmpoaw8+lpaWtL5iIiIgsnuhjbm7G0qVLsWHDBmzZsgVKpbLRNvHx8VCr1YZXQEBAO1dJRERE7UnUcOPu7g6ZTIaCggKj7QUFBfD29m523+XLl2Pp0qXYuXMnBg4c2GS7+fPnQ6vVGl45OTltUjsRERFZJlHDjVwuR3h4OJKSkgzb9Ho9kpKSMGzYsCb3e+ONN/Dqq68iMTERgwcPbvY9FAoFVCqV0YuIiIhsl6hjbgAgLi4O06dPx+DBgzF06FCsXLkSFRUVmDFjBgAgNjYWfn5+iI+PBwAsW7YMixYtwldffYXAwEBoNBoAgJOTE5ycnETrBxEREVkG0cNNTEwMioqKsGjRImg0GoSGhiIxMdEwyDg7OxtS6Z8XmNasWYPa2lr885//NDrO4sWL8dJLL7Vn6URERGSBRJ/npr1xnhsiIiLrYzXz3BARERG1NYYbIiIisikMN0RERGRTGG6IiIjIpjDcEBERkU1huCEiIiKbwnBDRERENoXhhoiIiGwKww0RERHZFIYbIiIisikMN0RERGRTGG6IiIjIpjDcEBERkU1huCEiIiKbwnBDRERENoXhhoiIiGwKww0RERHZFIYbIiIisikMN0RERGRTGG6IiIjIpjDcEBERkU1huCEiIiKbwnBDRERENoXhhoiIiGwKww0RERHZFIYbIiIisikMN0RERGRTGG6IiIjIpjDcEBERkU1huCEiIiKbwnBDRERENoXhhoiIiGwKww0RERHZFIYbIiIisikMN0RERGRTGG6IiIjIpjDcEBERkU1huCEiIiKbwnBDRERENoXhhoiIiGwKww0RERHZFIYbIiIisikMN0RERGRTGG6IiIjIpjDcEBERkU1huCEiIiKbwnBDRERENoXhhoiIiGyKRYSb1atXIzAwEEqlEhERETh8+HCz7b/55hv07t0bSqUSAwYMwPbt29upUiIiIrJ0ooebjRs3Ii4uDosXL0ZqaipCQkIQFRWFwsLCRtv/8ssvuO+++/DQQw8hLS0N0dHRiI6OxrFjx9q5ciIiIrJEEkEQBDELiIiIwJAhQ7Bq1SoAgF6vR0BAAJ588knMmzfvuvYxMTGoqKjADz/8YNh2yy23IDQ0FGvXrr3h+5WWlkKtVkOr1UKlUrVdR4iIiMhsTPn+FvXKTW1tLVJSUhAZGWnYJpVKERkZiYMHDza6z8GDB43aA0BUVFST7YmIiKhjsRPzzYuLi6HT6eDl5WW03cvLCydOnGh0H41G02h7jUbTaPuamhrU1NQYftZqtQAaEiARERFZh2vf2y254SRquGkP8fHxePnll6/bHhAQIEI1REREdDPKysqgVqubbSNquHF3d4dMJkNBQYHR9oKCAnh7eze6j7e3t0nt58+fj7i4OMPPer0ely9fRufOnSGRSG6yB8ZKS0sREBCAnJwcmxzPY+v9A2y/j+yf9bP1PrJ/1s9cfRQEAWVlZfD19b1hW1HDjVwuR3h4OJKSkhAdHQ2gIXwkJSVh1qxZje4zbNgwJCUlYc6cOYZtu3btwrBhwxptr1AooFAojLa5uLi0RflNUqlUNvt/WsD2+wfYfh/ZP+tn631k/6yfOfp4oys214h+WyouLg7Tp0/H4MGDMXToUKxcuRIVFRWYMWMGACA2NhZ+fn6Ij48HAMyePRujRo3CihUrMGHCBGzYsAFHjhzBhx9+KGY3iIiIyEKIHm5iYmJQVFSERYsWQaPRIDQ0FImJiYZBw9nZ2ZBK/3yoa/jw4fjqq6+wcOFCLFiwAD179sTWrVvRv39/sbpAREREFkT0cAMAs2bNavI2VHJy8nXb/vWvf+Ff//qXmasynUKhwOLFi6+7DWYrbL1/gO33kf2zfrbeR/bP+llCH0WfxI+IiIioLYm+/AIRERFRW2K4ISIiIpvCcENEREQ2heHGRKtXr0ZgYCCUSiUiIiJw+PDhZtt/88036N27N5RKJQYMGIDt27e3U6WtY0r/1q1bB4lEYvRSKpXtWK1p9u/fj4kTJ8LX1xcSiQRbt2694T7JyckYNGgQFAoFgoKCsG7dOrPX2Vqm9i85Ofm68yeRSJpcykRs8fHxGDJkCJydneHp6Yno6GhkZWXdcD9r+gy2po/W9Dlcs2YNBg4caJj/ZNiwYfjxxx+b3ceazp+p/bOmc9eYpUuXQiKRGM071xgxziHDjQk2btyIuLg4LF68GKmpqQgJCUFUVBQKCwsbbf/LL7/gvvvuw0MPPYS0tDRER0cjOjoax44da+fKW8bU/gENkzTl5+cbXhcuXGjHik1TUVGBkJAQrF69ukXtz507hwkTJmDMmDFIT0/HnDlz8PDDD2PHjh1mrrR1TO3fNVlZWUbn0NPT00wV3px9+/Zh5syZOHToEHbt2oW6ujqMGzcOFRUVTe5jbZ/B1vQRsJ7Pob+/P5YuXYqUlBQcOXIEt99+OyZNmoSMjIxG21vb+TO1f4D1nLu/++233/DBBx9g4MCBzbYT7RwK1GJDhw4VZs6cafhZp9MJvr6+Qnx8fKPtp06dKkyYMMFoW0REhPCf//zHrHW2lqn9++yzzwS1Wt1O1bUtAMKWLVuabfP8888L/fr1M9oWExMjREVFmbGyttGS/u3du1cAIFy5cqVdamprhYWFAgBh3759Tbaxts/g37Wkj9b8ORQEQXB1dRU+/vjjRn9n7edPEJrvn7Weu7KyMqFnz57Crl27hFGjRgmzZ89usq1Y55BXblqotrYWKSkpiIyMNGyTSqWIjIzEwYMHG93n4MGDRu0BICoqqsn2YmpN/wCgvLwcXbt2RUBAwA3/hWJtrOn83YzQ0FD4+Phg7NixOHDggNjltJhWqwUAuLm5NdnG2s9hS/oIWOfnUKfTYcOGDaioqGhy+RxrPn8t6R9gnedu5syZmDBhwnXnpjFinUOGmxYqLi6GTqczzJx8jZeXV5NjFDQajUntxdSa/gUHB+PTTz/Ftm3b8OWXX0Kv12P48OG4ePFie5Rsdk2dv9LSUlRVVYlUVdvx8fHB2rVr8d133+G7775DQEAARo8ejdTUVLFLuyG9Xo85c+ZgxIgRzc5Obk2fwb9raR+t7XN49OhRODk5QaFQ4LHHHsOWLVvQt2/fRtta4/kzpX/Wdu4AYMOGDUhNTTUsiXQjYp1Di5ihmKzTsGHDjP5FMnz4cPTp0wcffPABXn31VREro5YIDg5GcHCw4efhw4fjzJkzePvtt/HFF1+IWNmNzZw5E8eOHcPPP/8sdilm09I+WtvnMDg4GOnp6dBqtfj2228xffp07Nu3r8kAYG1M6Z+1nbucnBzMnj0bu3btsviBzww3LeTu7g6ZTIaCggKj7QUFBfD29m50H29vb5Pai6k1/fs7e3t7hIWF4fTp0+Yosd01df5UKhUcHBxEqsq8hg4davGBYdasWfjhhx+wf/9++Pv7N9vWmj6Df2VKH//O0j+HcrkcQUFBAIDw8HD89ttveOedd/DBBx9c19Yaz58p/fs7Sz93KSkpKCwsxKBBgwzbdDod9u/fj1WrVqGmpgYymcxoH7HOIW9LtZBcLkd4eDiSkpIM2/R6PZKSkpq8nzps2DCj9gCwa9euZu+/iqU1/fs7nU6Ho0ePwsfHx1xltitrOn9tJT093WLPnyAImDVrFrZs2YI9e/agW7duN9zH2s5ha/r4d9b2OdTr9aipqWn0d9Z2/hrTXP/+ztLP3R133IGjR48iPT3d8Bo8eDCmTZuG9PT064INIOI5NOtwZRuzYcMGQaFQCOvWrROOHz8uPProo4KLi4ug0WgEQRCEBx54QJg3b56h/YEDBwQ7Ozth+fLlQmZmprB48WLB3t5eOHr0qFhdaJap/Xv55ZeFHTt2CGfOnBFSUlKEe++9V1AqlUJGRoZYXWhWWVmZkJaWJqSlpQkAhLfeektIS0sTLly4IAiCIMybN0944IEHDO3Pnj0rODo6Cs8995yQmZkprF69WpDJZEJiYqJYXWiWqf17++23ha1btwqnTp0Sjh49KsyePVuQSqXC7t27xepCsx5//HFBrVYLycnJQn5+vuFVWVlpaGPtn8HW9NGaPofz5s0T9u3bJ5w7d074448/hHnz5gkSiUTYuXOnIAjWf/5M7Z81nbum/P1pKUs5hww3JnrvvfeELl26CHK5XBg6dKhw6NAhw+9GjRolTJ8+3aj9pk2bhF69eglyuVzo16+fkJCQ0M4Vm8aU/s2ZM8fQ1svLSxg/fryQmpoqQtUtc+3R57+/rvVp+vTpwqhRo67bJzQ0VJDL5UL37t2Fzz77rN3rbilT+7ds2TKhR48eglKpFNzc3ITRo0cLe/bsEaf4FmisbwCMzom1fwZb00dr+hw++OCDQteuXQW5XC54eHgId9xxh+GLXxCs//yZ2j9rOndN+Xu4sZRzyFXBiYiIyKZwzA0RERHZFIYbIiIisikMN0RERGRTGG6IiIjIpjDcEBERkU1huCEiIiKbwnBDRERENoXhhoiIiGwKww0RdXjJycmQSCQoKSkRuxQiagMMN0RERGRTGG6IiIjIpjDcEJHo9Ho94uPj0a1bNzg4OCAkJATffvstgD9vGSUkJGDgwIFQKpW45ZZbcOzYMaNjfPfdd+jXrx8UCgUCAwOxYsUKo9/X1NRg7ty5CAgIgEKhQFBQED755BOjNikpKRg8eDAcHR0xfPhwZGVlmbfjRGQWDDdEJLr4+HisX78ea9euRUZGBp5++mn8+9//xr59+wxtnnvuOaxYsQK//fYbPDw8MHHiRNTV1QFoCCVTp07Fvffei6NHj+Kll17Ciy++iHXr1hn2j42Nxddff413330XmZmZ+OCDD+Dk5GRUxwsvvIAVK1bgyJEjsLOzw4MPPtgu/SeitsVVwYlIVDU1NXBzc8Pu3bsxbNgww/aHH34YlZWVePTRRzFmzBhs2LABMTExAIDLly/D398f69atw9SpUzFt2jQUFRVh586dhv2ff/55JCQkICMjAydPnkRwcDB27dqFyMjI62pITk7GmDFjsHv3btxxxx0AgO3bt2PChAmoqqqCUqk0858CEbUlXrkhIlGdPn0alZWVGDt2LJycnAyv9evX48yZM4Z2fw0+bm5uCA4ORmZmJgAgMzMTI0aMMDruiBEjcOrUKeh0OqSnp0Mmk2HUqFHN1jJw4EDDf/v4+AAACgsLb7qPRNS+7MQugIg6tvLycgBAQkIC/Pz8jH6nUCiMAk5rOTg4tKidvb294b8lEgmAhvFARGRdeOWGiETVt29fKBQKZGdnIygoyOgVEBBgaHfo0CHDf1+5cgUnT55Enz59AAB9+vTBgQMHjI574MAB9OrVCzKZDAMGDIBerzcaw0NEtotXbohIVM7Oznj22Wfx9NNPQ6/X49Zbb4VWq8WBAwegUqnQtWtXAMArr7yCzp07w8vLCy+88ALc3d0RHR0NAHjmmWcwZMgQvPrqq4iJicHBgwexatUqvP/++wCAwMBATJ8+HQ8++CDeffddhISE4MKFCygsLMTUqVPF6joRmQnDDRGJ7tVXX4WHhwfi4+Nx9uxZuLi4YNCgQViwYIHhttDSpUsxe/ZsnDp1CqGhofjf//4HuVwOABg0aBA2bdqERYsW4dVXX4WPjw9eeeUV/N///Z/hPdasWYMFCxbgiSeewKVLl9ClSxcsWLBAjO4SkZnxaSkismjXnmS6cuUKXFxcxC6HiKwAx9wQERGRTWG4ISIiIpvC21JERERkU3jlhoiIiGwKww0RERHZFIYbIiIisikMN0RERGRTGG6IiIjIpjDcEBERkU1huCEiIiKbwnBDRERENoXhhoiIiGzK/wOA1bRvpRx+WAAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "y_avg = []\n", + "for y in epochs_y:\n", + " y_avg.append(np.mean(y))\n", + "\n", + "plt.plot(np.arange(len(epochs_x)), y_avg, marker = '.')\n", + "plt.xlabel('epoch')\n", + "plt.ylabel('average loss')\n", + "plt.ylim(0,)\n", + "for i, txt in enumerate(y_avg):\n", + " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom')\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAG2CAYAAACZEEfAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABPmklEQVR4nO3deVxU5f4H8M+wDQMCKsgyiIKg4q64EGqLQeFyTdPrltcNt1v2S9Q0tchMDfVeW8xyKcV9K5cWbxJaahouoJhbKqiAypIgM6wDzJzfH+jkKCCDM8zM8fN+veaVc+Y5h+/TkebTOc95HokgCAKIiIiIRMrK1AUQERERGRPDDhEREYkaww4RERGJGsMOERERiRrDDhEREYkaww4RERGJGsMOERERiRrDDhEREYkaww4RERGJGsMOERERiZpJw86RI0fQv39/yOVySCQS7N27V+dzQRDw/vvvw8vLCzKZDGFhYbh69apOm9zcXIwcORLOzs6oX78+xo8fj4KCgjrsBREREZkzk4adwsJCdOjQAV988UWlny9duhTLly/HqlWrcOLECTg6OiI8PBwlJSXaNiNHjsSFCxcQFxeHH3/8EUeOHMGkSZPqqgtERERk5iTmshCoRCLBnj17MHDgQAAVV3XkcjlmzJiBt99+GwCgUCjg4eGB9evXY/jw4bh06RJat26NU6dOoUuXLgCA/fv3o2/fvrh58ybkcrmpukNERERmwsbUBVTl+vXryMzMRFhYmHabi4sLgoODER8fj+HDhyM+Ph7169fXBh0ACAsLg5WVFU6cOIFXX3210mOrVCqoVCrte41Gg9zcXLi6ukIikRivU0RERGQwgiAgPz8fcrkcVlZV36wy27CTmZkJAPDw8NDZ7uHhof0sMzMT7u7uOp/b2NigYcOG2jaViY6Oxvz58w1cMREREZlCeno6GjduXOXnZht2jGnOnDmYPn269r1CoUCTJk2Qnp4OZ2dnE1ZGRERENaVUKuHj4wMnJ6dq25lt2PH09AQAZGVlwcvLS7s9KysLHTt21LbJzs7W2a+8vBy5ubna/SsjlUohlUof2e7s7MywQ0REZGEeNwTFbOfZ8fPzg6enJw4ePKjdplQqceLECYSEhAAAQkJCkJeXh8TERG2bX375BRqNBsHBwXVeMxEREZkfk17ZKSgoQHJysvb99evXkZSUhIYNG6JJkyaIjIzEwoUL0bx5c/j5+SEqKgpyuVz7xFarVq3Qu3dvTJw4EatWrUJZWRnefPNNDB8+nE9iEREREQATh52EhAT06tVL+/7+OJoxY8Zg/fr1mDVrFgoLCzFp0iTk5eWhZ8+e2L9/P+zt7bX7bNmyBW+++SZCQ0NhZWWFwYMHY/ny5XXeFyIiIjJPZjPPjikplUq4uLhAoVBwzA4REZGFqOn3t9mO2SEiIiIyBIYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1sw87+fn5iIyMRNOmTSGTydC9e3ecOnVK+/nYsWMhkUh0Xr179zZhxURERGRObExdwONMmDAB58+fx6ZNmyCXy7F582aEhYXh4sWL8Pb2BgD07t0bMTEx2n2kUqmpyiUiIiIzY9ZXdoqLi7Fr1y4sXboUzz33HAICAvDBBx8gICAAK1eu1LaTSqXw9PTUvho0aGDCqomIiMicmHXYKS8vh1qthr29vc52mUyGo0ePat8fOnQI7u7uaNmyJV5//XXk5ORUe1yVSgWlUqnzIiIiInEy67Dj5OSEkJAQLFiwALdv34ZarcbmzZsRHx+PjIwMABW3sDZu3IiDBw9iyZIlOHz4MPr06QO1Wl3lcaOjo+Hi4qJ9+fj41FWXiIiIqI5JBEEQTF1EdVJSUhAREYEjR47A2toaQUFBaNGiBRITE3Hp0qVH2l+7dg3+/v44cOAAQkNDKz2mSqWCSqXSvlcqlfDx8YFCoYCzs7PR+kJERESGo1Qq4eLi8tjvb7O+sgMA/v7+OHz4MAoKCpCeno6TJ0+irKwMzZo1q7R9s2bN4ObmhuTk5CqPKZVK4ezsrPMiIiIicTL7sHOfo6MjvLy8cPfuXcTGxmLAgAGVtrt58yZycnLg5eVVxxUSERGROTL7R89jY2MhCAJatmyJ5ORkzJw5E4GBgRg3bhwKCgowf/58DB48GJ6enkhJScGsWbMQEBCA8PBwU5dOREREZsDsr+woFApMmTIFgYGBGD16NHr27InY2FjY2trC2toaf/zxB1555RW0aNEC48ePR+fOnfHbb79xrh0iIiICYAEDlOtCTQc4ERERkfkQzQBlIiIioifBsENERESixrBDREREosawQ0RERKLGsENERESixrBDREREosawQ0RERKLGsENERESixrBDREREosawQ0RERKLGsENERESVys/PR2RkJJo2bQqZTIbu3bvj1KlTAICysjK88847aNeuHRwdHSGXyzF69Gjcvn37sce9desW/vWvf8HV1RUymQzt2rVDQkKC9vPdu3fj5ZdfhqurKyQSCZKSkp6oHww7REREVKkJEyYgLi4OmzZtwrlz5/Dyyy8jLCwMt27dQlFREU6fPo2oqCicPn0au3fvxuXLl/HKK69Ue8y7d++iR48esLW1xU8//YSLFy9i2bJlaNCggbZNYWEhevbsiSVLlhikH1wIFFwIlIiI6GHFxcVwcnLCd999h379+mm3d+7cGX369MHChQsf2efUqVPo1q0bUlNT0aRJk0qPO3v2bBw7dgy//fbbY2u4ceMG/Pz8cObMGXTs2PGRz7kQKBEREdVaeXk51Go17O3tdbbLZDIcPXq00n0UCgUkEgnq169f5XG///57dOnSBUOGDIG7uzs6deqEr776ypClP4Jhh4iIiB7h5OSEkJAQLFiwALdv34ZarcbmzZsRHx+PjIyMR9qXlJTgnXfewYgRI6q9ynLt2jWsXLkSzZs3R2xsLF5//XW89dZb2LBhg9H6wrBDREREldq0aRMEQYC3tzekUimWL1+OESNGwMpKNz6UlZVh6NChEAQBK1eurPaYGo0GQUFB+Oijj9CpUydMmjQJEydOxKpVq4zWD4YdIiIiqpS/vz8OHz6MgoICpKen4+TJkygrK0OzZs20be4HndTUVMTFxT127KuXlxdat26ts61Vq1ZIS0szSh8Ahh0iIiJ6DEdHR3h5eeHu3buIjY3FgAEDAPwddK5evYoDBw7A1dX1scfq0aMHLl++rLPtypUraNq0qVFqBxh2iIjISKqbowWo3VwqZWVl+PDDD+Hv7w97e3t06NAB+/fvr7L94sWLIZFIEBkZaYAePX1iY2Oxf/9+XL9+HXFxcejVqxcCAwMxbtw4lJWV4Z///CcSEhKwZcsWqNVqZGZmIjMzE6WlpdpjhIaGYsWKFdr306ZNw/Hjx/HRRx8hOTkZW7duxZo1azBlyhRtm9zcXCQlJeHixYsAgMuXLyMpKQmZmZm164hAgkKhEAAICoXC1KUQEYnG0KFDhdatWwuHDx8Wrl69KsybN09wdnYWbt68KQiCIGzcuFGYP3++8NVXXwkAhDNnzjz2mLNmzRLkcrmwb98+ISUlRfjyyy8Fe3t74fTp04+0PXnypODr6yu0b99emDp1qoF793TYsWOH0KxZM8HOzk7w9PQUpkyZIuTl5QmCIAjXr18XAFT6+vXXX7XHaOzTRIj4v5nC7bwi7bYffvhBaNu2rSCVSoXAwEBhzZo1Oj83Jiam0uPOmzdPp11Nv785zw44zw4RkaHpM0fL4+ZSeZBcLse7776rcxVg8ODBkMlk2Lx5s3ZbQUEBgoKC8OWXX2LhwoXo2LEjPv30U4P1j2pmx6k0zNl9DhoBsJIA0YPaYVjXyuffqY2afn/bGOwnEhER3VObOVpqQqVS1eiYU6ZMQb9+/RAWFlbp5Hf0ZMrVGiiKy5BXXIa8olLcLSzD3aJS5BWVIa+4FHeLypChKMavf/6l3UcjAHN3n8dzLRrBy0VWp/Uy7BARkcE9OEdLq1at4OHhgW3btiE+Ph4BAQG1Pm54eDg+/vhjPPfcc/D398fBgwexe/duqNVqbZvt27fj9OnTOuODqHKCIKBAVV4RUooqAos2tBTdDzAV4SWvqBR5xWW4W1gKZUl5rX6eWhBw404Rww4REYnDpk2bEBERAW9vb1hbWyMoKAgjRoxAYmJirY/52WefYeLEiQgMDIREIoG/vz/GjRuHdevWAQDS09MxdepUxMXFPXIFSOxU5eoqQsq9qy8PhpaiMtwtKoOiuBRl6tqPZnGyt0EDBzvUd7BFfQc7NHCwRQMHO7jIbGElAT49cBUPHt1aIoGvm8OTd1ZPDDtERGQU9+doKSwshFKphJeXF4YNG6YzR4u+GjVqhL1796KkpAQ5OTmQy+WYPXu29piJiYnIzs5GUFCQdh+1Wo0jR45gxYoVUKlUsLa2fuK+GZNaI0B57xaRNrTcu02kKC57JLTcDzdFperHH7wKUhurB0KL7b0/V4SXv4PM/ff32slsYWNd/UPdni72mLv7PNSCAGuJBB8NalvnV3UAhh0iIjIyR0dHODo6audoWbp06RMf097eHt7e3igrK8OuXbswdOhQABWPOZ87d06n7bhx4xAYGIh33nmnToOOIAgoKlVXelvo/pWVB6+43A8yiuIy1PbRISsJtGGkgYMd6sseuOLiWHHF5cHQ0sDRFvVldpDZGeffy7CuTfBci0a4cacIvm4OJgk6AMMOEREZSWxsLARBQMuWLZGcnIyZM2dq52gBKuZSSUtLw+3btwFAO9Gcp6cnPD09AQCjR4+Gt7c3oqOjAQAnTpzArVu30LFjR9y6dQsffPABNBoNZs2aBaBirFDbtm116nB0dISrq+sj2/VRWq5BXnEpFPdCSnW3iRRFfw/WLVVrav0z60ltHrjKYvvAlZYqrrjI7OBkbwMrK0mtf6YxeLnITBZy7mPYISIio1AoFJgzZw5u3ryJhg0bYvDgwVi0aBFsbW0BVKx+fT/4AMDw4cMBAPPmzcMHH3wAAEi+dgN3CkuRoSiGl4sMJSUleO+993Dt2jXUq1cPffv2xaZNm6pdZftBGo2A/JLyijDy0G0i7ZNFD4SX+1dkClS1G5ALAHbWVg+FFt3bRA0c7ODioHvFxUVmCzsbzvtrKJxnB5xnh4jIHD1ujpZinVtEpTpXXB6+TVQRZCrea2r5rSeRQHsbqLLbRPUd/77CUv/ebaMGDraQ2VpDIjGvqy1iUdPvb4YdMOwQEZmTAlU5Em7kYlzMKTz8BeXfyBGFqoqQoyqv/S0iRztr3bEtNRiY62xva3a3iJ52nFSQiIjMlqK4DKk5hbiRU4TUO/f+ee/9nQJVlful/FWo897GSqJzO6iqMS71H7hN5OJgC6mNeT+RRYbFsENEREaRV1SK63cKkZpThBs5uv/MLSytdt/6MlvkFZfpbJNIgE+HdUQzt3ra20SOdrxFRI/HsENERLUiCAJyC0txI6cIN+4U/n2l5t4/FQ+FlYc1cpLC19UBvq6O8HVzRNN7f27i6gBne1vsOJX2yBwtAzp611HvSEwYdoiIqEqCIOCvAhVSc4ruXaX5O9Ck3ilC/mOeUvJ0tkdTVwf4uTmiqasjfF0d0NS1Itg4Sqv/CjKXOVrI8jHsEJHZys/PR1RUFPbs2YPs7Gx06tQJn332Gbp27Qqg4ot43rx5+Oqrr5CXl4cePXpg5cqVaN68ebXHvXXrFt555x389NNPKCoqQkBAAGJiYtClSxcAwNixY7FhwwadfcLDw7F//37jdNTENBoB2fkq3MgpxI2Hxs+k5hRWOzOvRALIXWRoei/E+Lk53As1jmjS0OGJJ6szhzlayPIx7BCR2ZowYQLOnz+PTZs2QS6XY/PmzQgLC8PFixfh7e2NpUuXYvny5diwYQP8/PwQFRWF8PBwXLx4scp1ke7evYsePXqgV69e+Omnn9CoUSNcvXoVDRo00GnXu3dvxMTEaN9LpVKj9tXYNBoBGcoSpN4pxPX742fujadJzS1ESVnVTzZZSQDvBjL4uv59q6ni1pMDGjdwgL0tB/uSeeOj5+Cj50TmqLi4GE5OTvjuu+/Qr18/7fbOnTujT58+WLBgAeRyOWbMmIG3334bQMUkdh4eHli/fr12grqHzZ49G8eOHcNvv/1W5c8eO3Ys8vLysHfvXoP2ydjK1RpkKEoqrtA8NI4mLbcIpdU8qm1tJYFPA5nOrSa/e+NoGjdw4AR3ZJb46DkRWbTy8nKo1epHrtDIZDIcPXoU169fR2ZmJsLCwrSfubi4IDg4GPHx8VWGne+//x7h4eEYMmQIDh8+DG9vb7zxxhuYOHGiTrtDhw7B3d0dDRo0wIsvvoiFCxfC1dXV8B3VU5lag1t3i7VPNd0fR5OaU4T0u0XVrmBtay2BT0MH3Ss0bhXhRl5fBtvHLOpIZKkYdojILDk5OSEkJAQLFixAq1at4OHhgW3btiE+Ph4BAQHIzMwEAHh4eOjs5+Hhof2sMteuXcPKlSsxffp0zJ07F6dOncJbb70FOzs7jBkzBkDFLaxBgwbBz88PKSkpmDt3Lvr06YP4+Pg6WUiytFyD9Lv3xs3cKdJeqUnNKcTNu8VQVzMFsJ2NFZo2dPj7Cs29MOPr6gh5fRmsOSkePYUYdojIbG3atAkRERHw9vaGtbU1goKCMGLECCQmJtb6mBqNBl26dMFHH30EAOjUqRPOnz+PVatWacPOg1eF2rVrh/bt28Pf3x+HDh1CaGjok3XqnpIyNdJzi7Qh5sH5aG7nFVe7pIG9rZXO1ZkHg42Xsz1n+SV6CMMOEZktf39/HD58GIWFhVAqlfDy8sKwYcPQrFkz7arYWVlZ8PLy0u6TlZWFjh07VnlMLy8vtG7dWmdbq1atsGvXrir3adasGdzc3JCcnKxX2CkuVSM1t+LqTOpD42gylCWobsSkg521dhDwg+NofF0d4eEs5UR6RHpg2CEis+fo6AhHR0fcvXsXsbGxWLp0Kfz8/ODp6YmDBw9qw41SqcSJEyfw+uuvV3msHj164PLlyzrbrly5gqZNm1a5z82bN5GTk6MTqu4rUJVrx8zcuDf3TMXTToXIUla97AEAOEltdCbTa+rqoH3fqB4DDZGhMOwQkdmKjY2FIAho2bIlkpOTMXPmTAQGBmLcuHGQSCSIjIzEwoUL0bx5c+2j53K5HAMHDtQeIzQ0FK+++irefPNNAMC0adPQvXt3fPTRRxg6dChOnjyJNWvWYM2aNQCAgoICzJ8/H4MHD4anpydSUlIwY+ZMNPFthnKvdljxy1XtFZrHreMEVKySfX8QsO4VGgc0dLRjoCGqAww7RGS2FAoF5syZg5s3b6Jhw4YYPHgwFi1aBFtbWwDArFmzUFhYiEmTJiEvLw89e/bE/v37dZ7gunI1GWeupCFDUQwvFxm6du2KPXv2YM6cOfjwww/h5+eHTz/9FP1eHYKk9DxcuXUHP/x6HF+sWYuSwnzYOrnCrmlH1O83HdN3Xay0zoaOdhWzBN8fP/PAraf6DnZ18u+KiKrGeXbAeXaIxGrHqTTM2X0OGqFiYrx3+7ZCxyYNdNdwulOzdZzc6t1bx8lNd/xME1cHuMhs66hHRPQgzrNDRE+1tNxCzN59TjsIWCMAC/ZdqnYfD2dpxWR6ro5o6vb3OJqmro6o95h1nIjIfPG3l4hE5WpWPr5JvIntJ9MrfdrJ1dEWLTyc4ev2wGPbbg5o0tABDnb8TyKRGPE3m4gsnrKkDD+ezcDOhHQkpedV2c5KAvz41rNcWJLoKcOwQ0QWSaMRcPx6Dr5JuImfzmdoF7K0tpKgV0t3DO3SGHcKVIjaewFqQYC1RIKPBrVl0CF6CjHsEJFFuXm3CLsSb+GbxHTcvFus3d7cvR6GdGmMgZ284e7099NYvQLdceNOEXzdHBh0iJ5SZh928vPzERUVhT179iA7OxudOnXCZ599hq5duwIABEHAvHnz8NVXXyEvLw89evTAypUr0bx5cxNXTkSGUlKmRuyFTHyTcBPHUu5ox+I4SW3Qv6McQzo3Rkef+pXOWePlImPIIXrKmX3YmTBhAs6fP49NmzZBLpdj8+bNCAsLw8WLF+Ht7Y2lS5di+fLl2LBhg3ZSsfDwcFy8ePGR1ZKJyHIIgoA/biqwMyEd35+9jfyScu1n3f1dMbSLD8LbeEJmZ/yFOYnIspn1PDvFxcVwcnLCd999h379+mm3d+7cGX369MGCBQsgl8sxY8YMvP322wAqJiHz8PDA+vXrdRbzqw7n2SEyH3cKVNh75hZ2JqTjSlaBdrt3fRn+2bkx/tm5MXwaOpiwQiIyF6KYZ6e8vBxqtfqRKzQymQxHjx7F9evXkZmZibCwMO1nLi4uCA4ORnx8fJVhR6VSQaX6e4p3pVJpnA4QUY2UqTU4dPkvfJOQjl/+zEb5vSW/pTZW6NPWE0O6+CCkmStX8yaiWjHrsOPk5ISQkBAsWLAArVq1goeHB7Zt24b4+HgEBAQgMzMTAODh4aGzn4eHh/azykRHR2P+/PlGrZ2IHu/+nDi7T9/SWWOqg099DO3SGP9oL+fsxET0xMw67ADApk2bEBERAW9vb1hbWyMoKAgjRoxAYmJirY85Z84cTJ8+XfteqVTCx8fHEOUS0WNUNSeOWz07vNrJG0O6+KCFh5PpCiQi0TH7sOPv74/Dhw+jsLAQSqUSXl5eGDZsGJo1awZPT08AQFZWFry8vLT7ZGVloWPHjlUeUyqVQiqVGrt0IrqnJnPi9Ap0h621lYkrJSIxMvuwc5+joyMcHR1x9+5dxMbGYunSpfDz84OnpycOHjyoDTdKpRInTpzA66+/btqCiQg37xbh28Sb+DbxZo3mxCEiMgazDzuxsbEQBAEtW7ZEcnIyZs6cicDAQIwbNw4SiQSRkZFYuHAhmjdvrn30XC6XY+DAgaYuneipdH9OnJ0J6fg9JUevOXGIiIzB7MOOQqHAnDlzcPPmTTRs2BCDBw/GokWLYGtbMWhx1qxZKCwsxKRJk5CXl4eePXti//79nGOHqA4JgoCzNxX4hnPiEJEZMut5duoK59khqp2/8ivmxPkmkXPiEFHdE8U8O0Rkfu7PibMzIR2/ck4cIrIADDtEVCOcE4eILBXDDhFViXPiEJEYMOwQkQ6NRsDxazn4JpFz4hCRODDsEBEAzolDROLFsEP0FOOcOET0NGDYIXrKVDcnTo8AVwzpzDlxiEhcGHaInhLVzYkzpEtjDA7inDhEJE4MO0Qi9rg5cYZ28cEznBOHiERO77Dz66+/olevXsaohYgMhHPiEBH9Te+w07t3bzRu3Bjjxo3DmDFj4OPjY4y6iEhPnBOHiKhyeoedW7duYdOmTdiwYQPmz5+PF198EePHj8fAgQNhZ2dnjBqJqArVzYnzYqA7hnTmnDhERE+0EOjp06cRExODbdu2AQBee+01jB8/Hh06dDBYgXWBC4GSpaluTpyhXXwwsJM3GjlJTVghEZHx1fT7+4lXPb99+zbWrFmDxYsXw8bGBiUlJQgJCcGqVavQpk2bJzl0nWHYIUvwuDlxhnbxQYfGLpwTh4ieGkZd9bysrAzfffcd1q1bh7i4OHTp0gUrVqzAiBEj8Ndff+G9997DkCFDcPHixVp3gIg4Jw4RkSHofWXn//7v/7Bt2zYIgoBRo0ZhwoQJaNu2rU6bzMxMyOVyaDQagxZrLLyyQ+aGc+IQET1eTb+/9R61ePHiRXz++ee4ffs2Pv3000eCDgC4ubnh119/1ffQRAajVqsRFRUFPz8/yGQy+Pv7Y8GCBXgw20skkkpf//nPf6o87pEjR9C/f3/I5XJIJBLs3bu30naXLl3CK6+8AhcXFzg6OqJr165IS0urtuYytQZxF7MwcWMCQqIPYtH/LuFKVgGkNlYY2FGOrROC8dusXogMa8GgQ0SkB71vYx08ePDxB7WxwfPPP1+rgogMYcmSJVi5ciU2bNiANm3aICEhAePGjYOLiwveeustAEBGRobOPj/99BPGjx+PwYMHV3ncwsJCdOjQARERERg0aFClbVJSUtCzZ0+MHz8e8+fPh7OzMy5cuAB7+8oX0eScOERExqX3bazo6Gh4eHggIiJCZ/u6devw119/4Z133jFogXWBt7HE5x//+Ac8PDywdu1a7bbBgwdDJpNh8+bNle4zcOBA5Ofn1yjQAxVXhvbs2YOBAwfqbB8+fDhsbW2xadOmKvflnDhERE/OaLexVq9ejcDAwEe2t2nTBqtWrdL3cERG0b17dxw8eBBXrlwBAJw9exZHjx5Fnz59Km2flZWFffv2Yfz48U/0czUaDfbt24cWLVogPDwc7u7uCA4Oxt69e6HRCPg9+Q6m7UhCt0UHMHfPOSSl58HaSoKXWntgzajOiJ8Tinf7tWbQISIyIL1vY2VmZsLLy+uR7Y0aNXrktgCRqcyePRtKpRKBgYGwtraGWq3GokWLMHLkyErbb9iwAU5OTlXemqqp7OxsFBQUYPHixVi4cCGWLFmCHXu+x6BBg9Bm4jLkN2ihbcs5cYiI6obeYcfHxwfHjh2Dn5+fzvZjx45BLpcbrDCiJ7Fz505s2bIFW7duRZs2bZCUlITIyEjI5XKMGTPmkfbr1q3DyJEjqxxXU1P3n0D8R/9X0KzXUPwnIR2/l3aGvX9XXPttL5oNmcs5cYiI6pjeYWfixImIjIxEWVkZXnzxRQAVg5ZnzZqFGTNmGLxAotqYOXMmZs+ejeHDhwMA2rVrh9TUVERHRz8Sdn777TdcvnwZO3bseKKfKQgCbhXbwMraBoey7XBie5L2M1//FhAy/8TJd8M4Jw4RUR3TO+zMnDkTOTk5eOONN1BaWgoAsLe3xzvvvIM5c+YYvECi2igqKoKVle6QNGtr60rnflq7di06d+5c62VOHp4Tx9YjAIXZ6Qh4YE6ct8Z/DVnr5gw6REQmoHfYkUgkWLJkCaKionDp0iXIZDI0b94cUinHHJD56N+/PxYtWoQmTZqgTZs2OHPmDD7++ONHniJUKpX45ptvsGzZskqPExoaildffRVvvvkmAKCgoADJyckoU1eEpsU7j+DWD+kQpPVg4+wOqY0VwoZPQOznczC6QTJCfZvgu63r8MMPP+DQoUNG7TMREVVBIEGhUAgABIVCYepSyECUSqUwdepUoUmTJoK9vb3QrFkz4d133xVUKpVOu9WrVwsymUzIy8ur9DiNfZoIEf83U7idVyQIgiBs2rVPAPDIyye4j7D5+A0hr6hUEARBWLt2rRAQECDY29sLHTp0EPbu3WvcDhMRPYVq+v1dq4VAExISsHPnTqSlpWlvZd23e/fuJ09gdYzz7FBldpxKw5zd56ARAAmAxg1kSH9ghXHOiUNEZFpGWwh0+/btGD16NMLDw/Hzzz/j5ZdfxpUrV5CVlYVXX331iYomMhcZimJt0AEqLt+k3y2GlQQIbeWBIZ0bo1egO2yt9Z6qioiI6pjeYeejjz7CJ598gilTpsDJyQmfffYZ/Pz8MHny5Ern3yGyRH9mKLVB50FfjgxC77b8e05EZEn0/t/SlJQU9OvXDwBgZ2eHwsJCSCQSTJs2DWvWrDF4gUR1Lb+kDJ8dSH5ku7VEgg4+9eu+ICIieiJ6h50GDRogPz8fAODt7Y3z588DAPLy8lBUVGTY6ojqmKKoDP9aexJJN/MgtbGC1b05/6wlEnw0qC28XGSmLZCIiPSm922s5557DnFxcWjXrh2GDBmCqVOn4pdffkFcXBxCQ0ONUSNRncgpUGHU2pO4mKFEfQdbbIoIhpuTHW7cKYKvmwODDhGRhdL7aazc3FyUlJRALpdDo9Fg6dKl+P3339G8eXO89957aNCggbFqNRo+jUXZyhKM/PoErmYXwK2eFFsmBKOlJ5+wIiIyZ0Z5Gqu8vBw//vgjwsPDAQBWVlaYPXv2k1VKZGK38oox8qvjuJFTBE9ne2yZGAz/RvVMXRYRERmIXmN2bGxs8O9//xslJSXGqoeoTqXlFGHoqnjcyClC4wYy7JwcwqBDRCQyeg9Q7tatG5KSkoxQClHdSs4uwJDVv+NWXjH83Byxc3IImrg6mLosIiIyML0HKL/xxhuYPn060tPT0blzZzg6Oup83r59e4MVR2Qsf2Yq8a+vT+BOQSlaeNTD5gnBcHeyN3VZRERkBHoPUH54JWmgYnFQQRAgkUigVqsNVlxd4QDlp8u5mwqMWncCeUVlaO3ljM0TgtHQ0c7UZRERkZ6MtlzE9evXn6gwIlNKTM3F2HWnkK8qR0ef+tgwrhtcHGxNXRYRERmR3mGnadOmxqiDyOh+T7mDCRsSUFSqRje/hlg3tivqSfX+FSAiIguj93/pN27cWO3no0ePrnUxRMZy6HI2Jm9KhKpcg2ebu2HNqC6Q2VmbuiwiIqoDeo/ZeXjSwLKyMhQVFcHOzg4ODg7Izc01aIF1gWN2xC32Qibe3HoaZWoBoYHu+GJkEOxtGXSIiCxdTb+/9X70/O7duzqvgoICXL58GT179sS2bdueqGgiQ/vh7G28saUi6PRt54mV/+rMoENE9JTRO+xUpnnz5li8eDGmTp1qiMMRGcQ3CemYuv0M1BoBr3byxvLhnWBnY5C/8kREZEEMNjrTxsYGt2/fNtThiJ7IpuOpiNp7HgAwopsPFg1sB6v7S5gTEdFTRe+w8/333+u8FwQBGRkZWLFiBXr06GGwwohq6+vfrmHhvksAgLHdfTGvf2tIJAw6RERPK73DzsCBA3XeSyQSNGrUCC+++CKWLVtmqLqIamXFL1fx35+vAAD+/bw/3undkkGHiOgpp3fY0Wg0xqiD6IkIgoD//nwZX/yaAgCYFtYCb4UGMOgQEZHhxuwQmYogCFi47xLWHq2Y3Xtu30BMes7fxFUREZG50PvRlMGDB2PJkiWPbF+6dCmGDBlikKKIakqjEfDe3vPaoPPhgDYMOkREpEPvsHPkyBH07dv3ke19+vTBkSNHDFIUUU2oNQJmfvsHtpxIg0QCLB3cHqNDfE1dFhERmRm9w05BQQHs7B5dIdrW1hZKpdIgRd2nVqsRFRUFPz8/yGQy+Pv7Y8GCBXhw0uexY8dCIpHovHr37m3QOsj8lKk1mLr9DHadvglrKwk+HdYRQ7v6mLosIiIyQ3qP2WnXrh127NiB999/X2f79u3b0bp1a4MVBgBLlizBypUrsWHDBrRp0wYJCQkYN24cXFxc8NZbb2nb9e7dGzExMdr3UqnUoHWQeVGVq/Hm1jOIu5gFW2sJPh/RCb3bepm6LCIiMlN6h52oqCgMGjQIKSkpePHFFwEABw8exLZt2/DNN98YtLjff/8dAwYMQL9+/QAAvr6+2LZtG06ePKnTTiqVwtPT06A/m8xTcakakzcn4siVv2BnY4XV/+qMXoHupi6LiIjMmN63sfr374+9e/ciOTkZb7zxBmbMmIGbN2/iwIEDj8zB86S6d++OgwcP4sqVinlTzp49i6NHj6JPnz467Q4dOgR3d3e0bNkSr7/+OnJycqo9rkqlglKp1HmR+StUlWPc+pM4cuUvyGytETO2K4MOERE9lt6rntcljUaDuXPnYunSpbC2toZarcaiRYswZ84cbZvt27fDwcEBfn5+SElJwdy5c1GvXj3Ex8fD2rryBR8/+OADzJ8//5HtXPXcfClLyjB23UmcTstDPakNYsZ1RVffhqYui4iITKimq57rHXZOnToFjUaD4OBgne0nTpyAtbU1unTpUruKK7F9+3bMnDkT//nPf9CmTRskJSUhMjISH3/8McaMGVPpPteuXYO/vz8OHDiA0NDQStuoVCqoVCrte6VSCR8fH4YdM3W3sBSj153EuVsKuMhssTGiGzr41Dd1WUREZGI1DTt638aaMmUK0tPTH9l+69YtTJkyRd/DVWvmzJmYPXs2hg8fjnbt2mHUqFGYNm0aoqOjq9ynWbNmcHNzQ3JycpVtpFIpnJ2ddV5knv7KV2H4muM4d0uBho522DbxGQYdIiLSi94DlC9evIigoKBHtnfq1AkXL140SFH3FRUVwcpKN49ZW1tXu2TFzZs3kZOTAy8vPp1j6TIUxRj51Qlcu1MIdycptkwIRnMPJ1OXRUREFkbvKztSqRRZWVmPbM/IyICNjWFXn+jfvz8WLVqEffv24caNG9izZw8+/vhjvPrqqwAq5vyZOXMmjh8/jhs3buDgwYMYMGAAAgICEB4ebtBaqG6l5xZh6Op4XLtTCO/6MuycHMKgQ0REtaL3mJ0RI0YgIyMD3333HVxcXAAAeXl5GDhwINzd3bFz506DFZefn4+oqCjs2bMH2dnZkMvlGDFiBN5//33Y2dmhuLgYAwcOxJkzZ5CXlwe5XI6XX34ZCxYsgIeHR41/Tk3v+VHduH6nEK99dRwZihI0aeiArROD0biBg6nLIiIiM2O0Acq3bt3Cc889h5ycHHTq1AkAkJSUBA8PD8TFxcHHx/JmsWXYMR9XsvIx8usT+CtfBf9Gjtgy4Rl4utibuiwiIjJDNf3+1vu+k7e3N/744w9s2bIFZ8+ehUwmw7hx4zBixAjY2to+UdH0dDt/S4HR604it7AUgZ5O2DwhGG71OBs2ERE9mVoNsnF0dMSkSZMMXQs9xc6k3cWYdSehLClH+8Yu2BjRDfUdHl2DjYiISF+1HlF88eJFpKWlobS0VGf7K6+88sRF0dPlxLUcRKw/hcJSNbo0bYB147rC2Z5XCYmIyDD0DjvXrl3Dq6++inPnzkEikWhXIJdIJAAqVionqqmjV+9gwsZTKCnToLu/K74a3QWOUsM+1UdERE83vR89nzp1Kvz8/JCdnQ0HBwdcuHABR44cQZcuXXDo0CEjlEhidfBSFiI2VASdF1o2wrqxXRl0iIjI4PT+ZomPj8cvv/wCNzc3WFlZwcrKCj179kR0dDTeeustnDlzxhh1ksj871wG3tp2BuUaAeFtPLB8RCdIbSpfy4yIiOhJ6H1lR61Ww8mpYnI3Nzc33L59GwDQtGlTXL582bDVkSjtOXMTb249jXKNgFc6yLHitSAGHSIiMhq9r+y0bdsWZ8+ehZ+fH4KDg7F06VLY2dlhzZo1aNasmTFqJBHZfjINc/acgyAAQzo3xuLB7WFtJTF1WUREJGJ6h5333nsPhYWFAIAPP/wQ//jHP/Dss8/C1dUVO3bsMHiBJB7rj13HBz9UrJ826pmmmP9KG1gx6BARkZHpPYNyZXJzc9GgQQPtE1mWhjMoG9+qwylY/NOfAICJz/phbt9WFvv3hYiIzIPRZlCuTMOGDQ1xGBIhQRDw6YGr+OzgVQDAWy8GYNpLLRh0iIiozvA5XzIaQRCweP+fWH34GgBgZnhLTOkVYOKqiIjoacOwQ0ah0QiY/8MFbIhPBQC8/4/WiOjpZ+KqiIjoacSwQwan1giYu/scdiSkQyIBFg1sh9eCm5i6LCIiekrpPc/OkSNHUF5e/sj28vJyHDlyxCBFkeUqV2swfWcSdiSkw0oC/PefHRh0iIjIpPQOO7169UJubu4j2xUKBXr16mWQosgylZZr8ObWM/gu6TZsrCT4fEQQBndubOqyiIjoKaf3bSxBECp9kiYnJweOjo4GKYosT0mZGq9vTsSvl/+CnbUVvhgZhJdae5i6LCIiopqHnUGDBgGoWN187NixkEql2s/UajX++OMPdO/e3fAVktkrKi3HxI0JOJacA3tbK6wZ1QXPtWhk6rKIiIgA6BF2XFxcAFRc2XFycoJMJtN+Zmdnh2eeeQYTJ040fIVk1vJLyhCx/hRO3bgLBztrrBvbFc80czV1WURERFo1DjsxMTEAAF9fX7z99tu8ZUXIKyrFmHUncfamAk72NtgQ0Q1BTRqYuiwiIiIdeg9QnjVrls6YndTUVHz66af4+eefDVoYmbecAhVGfHUCZ28q0MDBFtsmPsOgQ0REZknvsDNgwABs3LgRAJCXl4du3bph2bJlGDBgAFauXGnwAsn8ZCtLMGzNcVzKUMKtnhTbJ4WgrbeLqcsiIiKqlN5h5/Tp03j22WcBAN9++y08PT2RmpqKjRs3Yvny5QYvkMzLrbxiDF0dj+TsAni52GPn5GfQ0tPJ1GURERFVSe9Hz4uKiuDkVPHl9vPPP2PQoEGwsrLCM888g9TUVIMXSOYjNacQr311ArfyitG4gQzbJj4Dn4YOpi6LiIioWnpf2QkICMDevXuRnp6O2NhYvPzyywCA7OzsapdXJ8uWnF2AoavjcSuvGM3cHPHNv0MYdIiIyCLoHXbef/99vP322/D19UW3bt0QEhICoOIqT6dOnQxeIJnepQwlhq2OR5ZShRYe9bB98jPwcpE9fkciIiIzIBEEQdB3p8zMTGRkZKBDhw6wsqrISydPnoSzszMCAwMNXqSxKZVKuLi4QKFQ8OrUQ/64mYfR604ir6gMbeTO2DQ+GA0d7UxdFhERUY2/v/W+sgMAnp6ecHJyQlxcHIqLiwEAXbt2tcigQ1VLTM3FyK9OIK+oDJ2a1MfWic8w6BARkcXRO+zk5OQgNDQULVq0QN++fZGRkQEAGD9+PGbMmGHwAsk0fk+5g1FrTyJfVY5ufg2xaXwwXGS2pi6LiIhIb3qHnWnTpsHW1hZpaWlwcPh7gOqwYcOwf/9+gxZHpnHocjbGxZxCUakazzZ3w4Zx3VBPqveDe0RERGZB72+wn3/+GbGxsWjcuLHO9ubNm/PRcxGIvZCJN7eeRplaQFgrd6x4LQj2ttamLouIiKjW9A47hYWFOld07svNzdVZCZ0sz/dnb2PajiSoNQL6tfPCJ8M6ws6mVsO6iIiIzIbe32TPPvusdrkIAJBIJNBoNFi6dCl69epl0OKo7uxMSMfU7Weg1ggY1Mkbnw1n0CEiInHQ+8rO0qVLERoaioSEBJSWlmLWrFm4cOECcnNzcezYMWPUSEa2Kf4Gor67AAAY0a0JFg1sCysryWP2IiIisgx6/69727ZtceXKFfTs2RMDBgxAYWEhBg0ahDNnzsDf398YNZIRff3bNW3QGdvdFx+9yqBDRETiovekgmlpafDx8YFE8ugXYlpaGpo0aWKw4urK0zqp4OcHr2JZ3BUAwOsv+GNWeMtKzysREZE5Mtqkgn5+fvjrr78e2Z6TkwM/Pz99D0cmIAgC/hP7pzbozHipBYMOERGJlt5jdgRBqPRLsaCgAPb29gYpioxHEAQs+PES1h27DgB4t28rTHyumYmrIiIiMp4ah53p06cDqHj6KioqSufxc7VajRMnTqBjx44GL5AMR6MR8N5357H1RBoAYMGANhgV4mvaooiIiIysxmHnzJkzACquDJw7dw52dn+vkWRnZ4cOHTrg7bffNnyFZBDlag1m7foDu0/fgkQCLBncHkO7+Ji6LCIiIqOrcdj59ddfAQDjxo3DZ5999lQN5LV0ZWoNInckYd8fGbC2kuDjoR0woKO3qcsiIiKqE3qP2YmJiTFGHWQkqnI1pmw5gwOXsmBrLcHnI4LQu62nqcsiIiKqM1zdUcSKS9WYtCkBv129A6mNFVaN6oxeLd1NXRYREVGdYtgRqQJVOSZsOIXj13Ihs7XG2jFd0D3AzdRlERER1TmGHRFSFJdhXMxJnE7LQz2pDdaP64ouvg1NXRYREZFJMOyIzN3CUoxadwLnbynhIrPFxohu6OBT39RlERERmQzDjoj8la/Cv74+gctZ+XB1tMOm8cFoLedTc0RE9HRj2BGJDEUxRn51AtfuFMLdSYqtE4MR4O5k6rKIiIhMjmFHBNJzi/Da18eRnlsM7/oybJkQDF83R1OXRUREZBYYdizctb8KMPLrE8hQlKCpqwO2TAhG4wYOj9+RiIjoKaH3qud1Sa1WIyoqCn5+fpDJZPD398eCBQsgCIK2jSAIeP/99+Hl5QWZTIawsDBcvXrVhFXXncuZ+Ri6+jgyFCXwb+SInZNDGHSIiIgeYtZhZ8mSJVi5ciVWrFiBS5cuYcmSJVi6dCk+//xzbZulS5di+fLlWLVqFU6cOAFHR0eEh4ejpKTEhJUb3/lbCgxfE487BSoEejphx+QQeDhz1XkiIqKHSYQHL5OYmX/84x/w8PDA2rVrtdsGDx4MmUyGzZs3QxAEyOVyzJgxQ7sIqUKhgIeHB9avX4/hw4fX6OcolUq4uLhAoVBYxJpfp9PuYsy6k8gvKUeHxi7YENEN9R3sHr8jERGRiNT0+9usr+x0794dBw8exJUrVwAAZ8+exdGjR9GnTx8AwPXr15GZmYmwsDDtPi4uLggODkZ8fLxJaja2E9dyMOrrE8gvKUdX3wbYPCGYQYeIiKgaZj1Aefbs2VAqlQgMDIS1tTXUajUWLVqEkSNHAgAyMzMBAB4eHjr7eXh4aD+rjEqlgkql0r5XKpVGqN7wfrv6FyZuTEBJmQbd/V3x9ZgucLAz61NIRERkcmZ9ZWfnzp3YsmULtm7ditOnT2PDhg3473//iw0bNjzRcaOjo+Hi4qJ9+fj4GKhi4zlwMQvj11cEnV4tG2Hd2K4MOkRERDVg1mFn5syZmD17NoYPH4527dph1KhRmDZtGqKjowEAnp6eAICsrCyd/bKysrSfVWbOnDlQKBTaV3p6uvE6YQD7/sjAvzcnolStQe82nlg9qgvsba1NXRYREZFFMOuwU1RUBCsr3RKtra2h0WgAAH5+fvD09MTBgwe1nyuVSpw4cQIhISFVHlcqlcLZ2VnnZa72nLmJ/9t2GuUaAQM6yrHitU6wszHr00ZERGRWzPo+SP/+/bFo0SI0adIEbdq0wZkzZ/Dxxx8jIiICACCRSBAZGYmFCxeiefPm8PPzQ1RUFORyOQYOHGja4g1g28k0zN1zDoIADO3SGNGD2sPaSmLqsoiIiCyKWYedzz//HFFRUXjjjTeQnZ0NuVyOyZMn4/3339e2mTVrFgoLCzFp0iTk5eWhZ8+e2L9/P+ztLXvOmZhj1zH/h4sAgNEhTfFB/zawYtAhIiLSm1nPs1NXzG2enZWHUrBk/58AgEnPNcOcPoGQSBh0iIiIHlTT72+zvrLztBEEAZ8cuIrlByuWu3grtDmmhTVn0CEiInoCHOlaBV9fX0gkkkdeU6ZMAVAxx8+oUaPg6ekJR0dHBAUFYdeuXdUec+XKlWjfvr12UHRISAh++uknABVBZ/FPf2L5watQ3boE6c8LETWwE1xcXPDcc8+huLjY6H0mIiISI17ZqcKpU6egVqu178+fP4+XXnoJQ4YMAQCMHj0aeXl5+P777+Hm5oatW7di6NChSEhIQKdOnSo9ZuPGjbF48WI0b94cgiBgw4YNGDBgABITT+Oba8DG+FSobl2CYs98TH3vXfTv/zVsbGxw9uzZR55KIyIioprhmB3U7J5fZGQkfvzxR1y9ehUSiQT16tXDypUrMWrUKG0bV1dXLFmyBBMmTKjxz27YsCGCh72FSy5dIZEA1j9EYcSr/bBgwYIn7hcREZGYiWJtLHNRWlqKzZs3IyIiQjt+pnv37tixYwdyc3Oh0Wiwfft2lJSU4IUXXqjRMdVqNbZs3QpFfgHOlnnASgK8H+qNlAtn4O7uju7du8PDwwPPP/88jh49asTeERERiRvDTg3s3bsXeXl5GDt2rHbbzp07UVZWBldXV0ilUkyePBl79uxBQEBAtcc6d+4c6tWrB6lUioiJk+E2cC4c3Jvi8xFBaFWvBADwwQcfYOLEidi/fz+CgoIQGhqKq1evGrOLREREosWwUwNr165Fnz59IJfLtduioqKQl5eHAwcOICEhAdOnT8fQoUNx7ty5ao/VsmVLnDiViD7vroV9+97I2fcJ5jxTD/3ae2lnhp48eTLGjRuHTp064ZNPPkHLli2xbt06o/aRiIhIrDhA+TFSU1Nx4MAB7N69W7stJSUFK1aswPnz59GmTRsAQIcOHfDbb7/hiy++wKpVq6o8XjmssOjoXZxTucErLAL1JFk4uW8Lxr/yHLy8vAAArVu31tmnVatWSEtLM0LviIiIxI9Xdh4jJiYG7u7u6Nevn3ZbUVERAFS7bldl8kvKMHrtSfyekgNHO2tsGNcNzvY2UKlUACoed5fL5bh8+bLOfleuXEHTpk0N1SUiIqKnCsNONTQaDWJiYjBmzBjY2Px9ESwwMBABAQGYPHkyTp48iZSUFCxbtgxxcXE6a3KFhoZixYoVAIC8olIEvTIOR4/+BvuSO5jX0wl7v/ovDh06hJEjRwKoWOtr5syZWL58Ob799lskJycjKioKf/75J8aPH1+nfSciIhIL3saqxoEDB5CWlqZdePQ+W1tb/O9//8Ps2bPRv39/FBQUICAgABs2bEDfvn217VJSUnDnzh3cKVDhX1+fQEZmNkpPfYLcort4c6sL2rdvj9jYWLz00kvafSIjI1FSUoJp06YhNzcXHTp0QFxcHPz9/eus30RERGLCeXZgvLWxMhTFOJ16F/+JvYwbOUVwqyfF1onBaOHhZLCfQURE9LTi2lgmtuNUGubsPgfNvSjpIrPBzsnPoFmjeqYtjIiI6CnDMTtGkKEo1gk6AJBfUg6ZnbXpiiIiInpKMewYwfU7hTpBBwA0AnDjTpFpCiIiInqKMewYgZ+bI6wkutusJRL4ujmYpiAiIqKnGMOOEXi5yBA9qB2s762jZS2R4KNBbeHlIjNxZURERE8fDlA2kmFdm+C5Fo1w404RfN0cGHSIiIhMhGHHiLxcZAw5REREJsbbWERERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkamYfdnx9fSGRSB55TZkyBQDwwgsvPPLZv//9bxNXTURERObCxtQFPM6pU6egVqu178+fP4+XXnoJQ4YM0W6bOHEiPvzwQ+17BweHOq2RiIiIzJfZh51GjRrpvF+8eDH8/f3x/PPPa7c5ODjA09OzrksjIiIiC2D2t7EeVFpais2bNyMiIgISiUS7fcuWLXBzc0Pbtm0xZ84cFBUVVXsclUoFpVKp8yIiIiJxMvsrOw/au3cv8vLyMHbsWO221157DU2bNoVcLscff/yBd955B5cvX8bu3burPE50dDTmz59fBxUTERGRqUkEQRBMXURNhYeHw87ODj/88EOVbX755ReEhoYiOTkZ/v7+lbZRqVRQqVTa90qlEj4+PlAoFHB2djZ43URERGR4SqUSLi4uj/3+tpgrO6mpqThw4EC1V2wAIDg4GACqDTtSqRRSqdTgNRIREZH5sZgxOzExMXB3d0e/fv2qbZeUlAQA8PLyqoOqiIiIyNxZxJUdjUaDmJgYjBkzBjY2f5eckpKCrVu3om/fvnB1dcUff/yBadOm4bnnnkP79u1NWDERERGZC4sIOwcOHEBaWhoiIiJ0ttvZ2eHAgQP49NNPUVhYCB8fHwwePBjvvfeeiSolIiIic2NRA5SNpaYDnIiIiMh81PT722LG7BARERHVBsMOERERiRrDDhEREYkaww4RERGJGsMOERERiRrDDhEREYkaww4RERGJGsMOERERiRrDDhEREYkaww4RERGJGsMOERERiRrDDhEREYkaww4RERGJGsMOERERiRrDDhEREYkaww4RERGJGsMOERERiRrDDhEREYkaww4RERGJGsMOERERiRrDDhEREYkaww4RERGJGsMOERERiRrDDhEREYkaww4RERGJGsMOERERiRrDDhEREYkaww4RERGJGsMOERERiRrDDhEREYkaww4RERGJGsMOERERiRrDDhEREYkaww4RERGJGsMOERERiRrDDhEREYkaww4RERGJGsMOERERiRrDDhEREYkaww4RERGJGsMOERERiRrDDhEREYkaww4RERGJGsMOERERiRrDDhEREYkaww4RERGJGsMOERERiRrDDhEREYkaww4RERGJGsMOERERiRrDDhEREYma2YcdX19fSCSSR15TpkwBAJSUlGDKlClwdXVFvXr1MHjwYGRlZZm4aiIiIjIXZh92Tp06hYyMDO0rLi4OADBkyBAAwLRp0/DDDz/gm2++weHDh3H79m0MGjTIlCUTERGRGZEIgiCYugh9REZG4scff8TVq1ehVCrRqFEjbN26Ff/85z8BAH/++SdatWqF+Ph4PPPMMzU6plKphIuLCxQKBZydnY1ZPhERERlITb+/beqwpidWWlqKzZs3Y/r06ZBIJEhMTERZWRnCwsK0bQIDA9GkSZNqw45KpYJKpdK+VygUACr+pREREZFluP+9/bjrNhYVdvbu3Yu8vDyMHTsWAJCZmQk7OzvUr19fp52HhwcyMzOrPE50dDTmz5//yHYfHx9DlktERER1ID8/Hy4uLlV+blFhZ+3atejTpw/kcvkTHWfOnDmYPn269r1Go0Fubi5cXV0hkUietEwtpVIJHx8fpKeni/b2mNj7yP5ZPrH3kf2zfGLvozH7JwgC8vPzH5sLLCbspKam4sCBA9i9e7d2m6enJ0pLS5GXl6dzdScrKwuenp5VHksqlUIqlepse/jqkCE5OzuL8i/wg8TeR/bP8om9j+yf5RN7H43Vv+qu6Nxn9k9j3RcTEwN3d3f069dPu61z586wtbXFwYMHtdsuX76MtLQ0hISEmKJMIiIiMjMWcWVHo9EgJiYGY8aMgY3N3yW7uLhg/PjxmD59Oho2bAhnZ2f83//9H0JCQmr8JBYRERGJm0WEnQMHDiAtLQ0RERGPfPbJJ5/AysoKgwcPhkqlQnh4OL788ksTVPkoqVSKefPmPXLLTEzE3kf2z/KJvY/sn+UTex/NoX8WN88OERERkT4sZswOERERUW0w7BAREZGoMewQERGRqDHsEBERkagx7DyhL774Ar6+vrC3t0dwcDBOnjxZbftvvvkGgYGBsLe3R7t27fC///2vjiqtHX36t379ekgkEp2Xvb19HVarnyNHjqB///6Qy+WQSCTYu3fvY/c5dOgQgoKCIJVKERAQgPXr1xu9ziehbx8PHTr0yDmUSCTVLr9iStHR0ejatSucnJzg7u6OgQMH4vLly4/dz1J+D2vTP0v7PVy5ciXat2+vnXAuJCQEP/30U7X7WMr5A/Tvn6Wdv4ctXrwYEokEkZGR1bar63PIsPMEduzYgenTp2PevHk4ffo0OnTogPDwcGRnZ1fa/vfff8eIESMwfvx4nDlzBgMHDsTAgQNx/vz5Oq68ZvTtH1AxQ2ZGRob2lZqaWocV66ewsBAdOnTAF198UaP2169fR79+/dCrVy8kJSUhMjISEyZMQGxsrJErrT19+3jf5cuXdc6ju7u7kSp8MocPH8aUKVNw/PhxxMXFoaysDC+//DIKCwur3MeSfg9r0z/Asn4PGzdujMWLFyMxMREJCQl48cUXMWDAAFy4cKHS9pZ0/gD9+wdY1vl70KlTp7B69Wq0b9++2nYmOYcC1Vq3bt2EKVOmaN+r1WpBLpcL0dHRlbYfOnSo0K9fP51twcHBwuTJk41aZ23p27+YmBjBxcWljqozLADCnj17qm0za9YsoU2bNjrbhg0bJoSHhxuxMsOpSR9//fVXAYBw9+7dOqnJ0LKzswUAwuHDh6tsY2m/hw+qSf8s+ffwvgYNGghff/11pZ9Z8vm7r7r+Wer5y8/PF5o3by7ExcUJzz//vDB16tQq25riHPLKTi2VlpYiMTERYWFh2m1WVlYICwtDfHx8pfvEx8frtAeA8PDwKtubUm36BwAFBQVo2rQpfHx8Hvt/L5bGks7fk+rYsSO8vLzw0ksv4dixY6Yup8YUCgUAoGHDhlW2seTzWJP+AZb7e6hWq7F9+3YUFhZWueSPJZ+/mvQPsMzzN2XKFPTr1++Rc1MZU5xDhp1aunPnDtRqNTw8PHS2e3h4VDm+ITMzU6/2plSb/rVs2RLr1q3Dd999h82bN0Oj0aB79+64efNmXZRsdFWdP6VSieLiYhNVZVheXl5YtWoVdu3ahV27dsHHxwcvvPACTp8+berSHkuj0SAyMhI9evRA27Ztq2xnSb+HD6pp/yzx9/DcuXOoV68epFIp/v3vf2PPnj1o3bp1pW0t8fzp0z9LPH/bt2/H6dOnER0dXaP2pjiHFrFcBFmGkJAQnf9b6d69O1q1aoXVq1djwYIFJqyMaqply5Zo2bKl9n337t2RkpKCTz75BJs2bTJhZY83ZcoUnD9/HkePHjV1KUZR0/5Z4u9hy5YtkZSUBIVCgW+//RZjxozB4cOHqwwElkaf/lna+UtPT8fUqVMRFxdn1gOpGXZqyc3NDdbW1sjKytLZnpWVBU9Pz0r38fT01Ku9KdWmfw+ztbVFp06dkJycbIwS61xV58/Z2RkymcxEVRlft27dzD5AvPnmm/jxxx9x5MgRNG7cuNq2lvR7eJ8+/XuYJfwe2tnZISAgAADQuXNnnDp1Cp999hlWr179SFtLPH/69O9h5n7+EhMTkZ2djaCgIO02tVqNI0eOYMWKFVCpVLC2ttbZxxTnkLexasnOzg6dO3fGwYMHtds0Gg0OHjxY5b3YkJAQnfYAEBcXV+29W1OpTf8eplarce7cOXh5eRmrzDplSefPkJKSksz2HAqCgDfffBN79uzBL7/8Aj8/v8fuY0nnsTb9e5gl/h5qNBqoVKpKP7Ok81eV6vr3MHM/f6GhoTh37hySkpK0ry5dumDkyJFISkp6JOgAJjqHRhv6/BTYvn27IJVKhfXr1wsXL14UJk2aJNSvX1/IzMwUBEEQRo0aJcyePVvb/tixY4KNjY3w3//+V7h06ZIwb948wdbWVjh37pypulAtffs3f/58ITY2VkhJSRESExOF4cOHC/b29sKFCxdM1YVq5efnC2fOnBHOnDkjABA+/vhj4cyZM0JqaqogCIIwe/ZsYdSoUdr2165dExwcHISZM2cKly5dEr744gvB2tpa2L9/v6m68Fj69vGTTz4R9u7dK1y9elU4d+6cMHXqVMHKyko4cOCAqbpQrddff11wcXERDh06JGRkZGhfRUVF2jaW/HtYm/5Z2u/h7NmzhcOHDwvXr18X/vjjD2H27NmCRCIRfv75Z0EQLPv8CYL+/bO081eZh5/GModzyLDzhD7//HOhSZMmgp2dndCtWzfh+PHj2s+ef/55YcyYMTrtd+7cKbRo0UKws7MT2rRpI+zbt6+OK9aPPv2LjIzUtvXw8BD69u0rnD592gRV18z9x6wfft3v05gxY4Tnn3/+kX06duwo2NnZCc2aNRNiYmLqvG596NvHJUuWCP7+/oK9vb3QsGFD4YUXXhB++eUX0xRfA5X1DYDOebHk38Pa9M/Sfg8jIiKEpk2bCnZ2dkKjRo2E0NBQbRAQBMs+f4Kgf/8s7fxV5uGwYw7nUCIIgmC860ZEREREpsUxO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtERA85dOgQJBIJ8vLyTF0KERkAww4RERGJGsMOERERiRrDDhGZHY1Gg+joaPj5+UEmk6FDhw749ttvAfx9i2nfvn1o37497O3t8cwzz+D8+fM6x9i1axfatGkDqVQKX19fLFu2TOdzlUqFd955Bz4+PpBKpQgICMDatWt12iQmJqJLly5wcHBA9+7dcfnyZeN2nIiMgmGHiMxOdHQ0Nm7ciFWrVuHChQuYNm0a/vWvf+Hw4cPaNjNnzsSyZctw6tQpNGrUCP3790dZWRmAipAydOhQDB8+HOfOncMHH3yAqKgorF+/Xrv/6NGjsW3bNixfvhyXLl3C6tWrUa9ePZ063n33XSxbtgwJCQmwsbFBREREnfSfiAyLC4ESkVlRqVRo2LAhDhw4gJCQEO32CRMmoKioCJMmTUKvXr2wfft2DBs2DACQm5uLxo0bY/369Rg6dChGjhyJv/76Cz///LN2/1mzZmHfvn24cOECrly5gpYtWyIuLg5hYWGP1HDo0CH06tULBw4cQGhoKADgf//7H/r164fi4mLY29sb+d8CERkSr+wQkVlJTk5GUVERXnrpJdSrV0/72rhxI1JSUrTtHgxCDRs2RMuWLXHp0iUAwKVLl9CjRw+d4/bo0QNXr16FWq1GUlISrK2t8fzzz1dbS/v27bV/9vLyAgBkZ2c/cR+JqG7ZmLoAIqIHFRQUAAD27dsHb29vnc+kUqlO4KktmUxWo3a2trbaP0skEgAV44mIyLLwyg4RmZXWrVtDKpUiLS0NAQEBOi8fHx9tu+PHj2v/fPfuXVy5cgWtWrUCALRq1QrHjh3TOe6xY8fQokULWFtbo127dtBoNDpjgIhIvHhlh4jMipOTE95++21MmzYNGo0GPXv2hEKhwLFjx+Ds7IymTZsCAD788EO4urrCw8MD7777Ltzc3DBw4EAAwIwZM9C1a1csWLAAw4YNQ3x8PFasWIEvv/wSAODr64sxY8YgIiICy5cvR4cOHZCamors7GwMHTrUVF0nIiNh2CEis7NgwQI0atQI0dHRuHbtGurXr4+goCDMnTtXextp8eLFmDp1Kq5evYqOHTvihx9+gJ2dHQAgKCgIO3fuxPvvv48FCxbAy8sLH374IcaOHav9GStXrsTcuXPxxhtvICcnB02aNMHcuXNN0V0iMjI+jUVEFuX+k1J3795F/fr1TV0OEVkAjtkhIiIiUWPYISIiIlHjbSwiIiISNV7ZISIiIlFj2CEiIiJRY9ghIiIiUWPYISIiIlFj2CEiIiJRY9ghIiIiUWPYISIiIlFj2CEiIiJRY9ghIiIiUft/VTGNs0unyBkAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plt.plot(np.arange(len(epochs_x)), epochs_acc, marker = '.')\n", + "plt.xlabel('epoch')\n", + "plt.ylabel('test accuracy')\n", + "plt.ylim(70, 100)\n", + "for i, txt in enumerate(epochs_acc):\n", + " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom')\n", + "plt.show()" + ] } ], "metadata": { From 9777f9420fc29e8d814bf1a04b9c670f47c15376 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Fri, 26 Apr 2024 21:49:06 +0200 Subject: [PATCH 051/379] 3rd baseline example: DVS gesture --- .../baseline-SCNN-example_3.ipynb | 510 ++++++++++++++++++ 1 file changed, 510 insertions(+) create mode 100644 tests/test_nonsequential/baseline-SCNN-example_3.ipynb diff --git a/tests/test_nonsequential/baseline-SCNN-example_3.ipynb b/tests/test_nonsequential/baseline-SCNN-example_3.ipynb new file mode 100644 index 00000000..3b694e0d --- /dev/null +++ b/tests/test_nonsequential/baseline-SCNN-example_3.ipynb @@ -0,0 +1,510 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "import torch.nn as nn\n", + "import sinabs.layers as sl\n", + "from tqdm.notebook import tqdm\n", + "\n", + "from tonic.datasets.dvsgesture import DVSGesture\n", + "from tonic.transforms import ToFrame\n", + "from torch.utils.data import DataLoader\n", + "from torch.nn import CrossEntropyLoss\n", + "from torch.optim import Adam\n", + "\n", + "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "torch.manual_seed(0)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Loading Data" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "batch_size = 8\n", + "num_workers = 4\n", + "epochs = 5\n", + "lr = 1e-3" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "root_dir = \"./DVSGESTURE\"\n", + "_ = DVSGesture(save_to=root_dir, train=True)\n", + "_ = DVSGesture(save_to=root_dir, train=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "n_time_steps = 100\n", + "to_raster = ToFrame(sensor_size=DVSGesture.sensor_size, n_time_bins=n_time_steps)\n", + "\n", + "snn_train_dataset = DVSGesture(save_to=root_dir, train=True, transform=to_raster)\n", + "snn_test_dataset = DVSGesture(save_to=root_dir, train=False, transform=to_raster)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The transformed array is in shape [Time-Step, Channel, Height, Width] --> (100, 2, 128, 128)\n" + ] + } + ], + "source": [ + "sample_data, label = snn_train_dataset[0]\n", + "print(f\"The transformed array is in shape [Time-Step, Channel, Height, Width] --> {sample_data.shape}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "snn_train_dataloader = DataLoader(snn_train_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True)\n", + "snn_test_dataloader = DataLoader(snn_test_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=False)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Network Module\n", + "\n", + "We need to define a `nn.Module` implementing the network we want the chip to reproduce." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "device: NVIDIA GeForce RTX 3070 Ti\n" + ] + } + ], + "source": [ + "if torch.cuda.is_available():\n", + " device = torch.device('cuda:0')\n", + " print('device: ', torch.cuda.get_device_name(0))\n", + "else:\n", + " device = torch.device('cpu')" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "class SNN(nn.Module):\n", + " def __init__(self) -> None:\n", + " super().__init__()\n", + "\n", + " self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False)\n", + " self.iaf1 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + " self.pool1 = nn.AvgPool2d(2,2)\n", + "\n", + " self.conv2 = nn.Conv2d(10, 10, 2, 1, bias=False)\n", + " self.iaf2 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + " self.pool2 = nn.AvgPool2d(3,3)\n", + "\n", + " self.conv3 = nn.Conv2d(10, 10, 3, 1, bias=False)\n", + " self.iaf3 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + " self.pool3 = nn.AvgPool2d(2,2)\n", + "\n", + " self.flat = nn.Flatten()\n", + "\n", + " self.fc1 = nn.Linear(810, 100, bias=False)\n", + " self.iaf4 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + "\n", + " self.fc2 = nn.Linear(100, 100, bias=False)\n", + " self.iaf5 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + "\n", + " self.fc3 = nn.Linear(100, 100, bias=False)\n", + " self.iaf6 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + "\n", + " self.fc4 = nn.Linear(100, 10, bias=False)\n", + " self.iaf7 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + "\n", + " def detach_neuron_states(self):\n", + " for name, layer in self.named_modules():\n", + " if name != '':\n", + " if isinstance(layer, sl.StatefulLayer):\n", + " for name, buffer in layer.named_buffers():\n", + " buffer.detach_()\n", + "\n", + " def init_weights(self):\n", + " for name, layer in self.named_modules():\n", + " if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear):\n", + " nn.init.xavier_normal_(layer.weight.data)\n", + "\n", + " def forward(self, x):\n", + " \n", + " con1_out = self.conv1(x)\n", + " iaf1_out = self.iaf1(con1_out)\n", + " pool1_out = self.pool1(iaf1_out)\n", + "\n", + " conv2_out = self.conv2(pool1_out)\n", + " iaf2_out = self.iaf2(conv2_out)\n", + " pool2_out = self.pool2(iaf2_out)\n", + "\n", + " conv3_out = self.conv3(pool2_out)\n", + " iaf3_out = self.iaf3(conv3_out)\n", + " pool3_out = self.pool3(iaf3_out)\n", + "\n", + " flat_out = self.flat(pool3_out)\n", + " \n", + " fc1_out = self.fc1(flat_out)\n", + " iaf4_out = self.iaf4(fc1_out)\n", + "\n", + " fc2_out = self.fc2(iaf4_out)\n", + " iaf5_out = self.iaf5(fc2_out)\n", + "\n", + " fc3_out = self.fc3(iaf5_out)\n", + " iaf6_out = self.iaf6(fc3_out)\n", + "\n", + " fc4_out = self.fc4(iaf6_out)\n", + " iaf7_out = self.iaf7(fc4_out)\n", + "\n", + " return iaf7_out" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "snn = SNN().to(device)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "snn.init_weights()" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "optimizer = Adam(snn.parameters(), lr=lr, betas=(0.9, 0.999), eps=1e-8)\n", + "loss_fn = CrossEntropyLoss()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Define train and test" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "def train(dataloader, model, loss_fn, optimizer, epochs, test_func, dataloader_test):\n", + " epochs_y = []\n", + " epochs_x = []\n", + " epochs_acc = []\n", + " model.train()\n", + "\n", + " for e in range(epochs):\n", + " losses = []\n", + " batches = []\n", + " batch_count = 0\n", + " train_p_bar = tqdm(snn_train_dataloader)\n", + "\n", + " for X, y in train_p_bar:\n", + " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", + " X = X.reshape(-1, 2, 128, 128).to(dtype=torch.float, device=device)\n", + " y = y.to(dtype=torch.long, device=device)\n", + "\n", + " # forward\n", + " pred = model(X)\n", + "\n", + " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", + " pred = pred.reshape(batch_size, n_time_steps, -1)\n", + "\n", + " # accumulate all time-steps output for final prediction\n", + " pred = pred.sum(dim = 1)\n", + " loss = loss_fn(pred, y)\n", + "\n", + " # gradient update\n", + " optimizer.zero_grad()\n", + " loss.backward()\n", + " optimizer.step()\n", + "\n", + " # detach the neuron states and activations from current computation graph(necessary)\n", + " model.detach_neuron_states()\n", + "\n", + " train_p_bar.set_description(f\"Epoch {e} - BPTT Training Loss: {round(loss.item(), 4)}\")\n", + "\n", + " batch_count += 1\n", + " losses.append(loss.item())\n", + " batches.append(batch_count)\n", + "\n", + " epochs_y.append(losses)\n", + " epochs_x.append(batches)\n", + "\n", + " acc = test_func(dataloader_test, model)\n", + " print(f'Epoch {e} accuracy: {acc}')\n", + " epochs_acc.append(acc)\n", + "\n", + " return epochs_x, epochs_y, epochs_acc\n" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "def test(dataloader, model):\n", + " correct_predictions = []\n", + " with torch.no_grad():\n", + " test_p_bar = tqdm(dataloader)\n", + " for X, y in test_p_bar:\n", + " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", + " X = X.reshape(-1, 2, 128, 128).to(dtype=torch.float, device=device)\n", + " y = y.to(dtype=torch.long, device=device)\n", + "\n", + " # forward\n", + " output = model(X)\n", + "\n", + " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", + " output = output.reshape(batch_size, n_time_steps, -1)\n", + "\n", + " # accumulate all time-steps output for final prediction\n", + " output = output.sum(dim=1)\n", + "\n", + " # calculate accuracy\n", + " pred = output.argmax(dim=1, keepdim=True)\n", + "\n", + " # compute the total correct predictions\n", + " correct_predictions.append(pred.eq(y.view_as(pred)))\n", + "\n", + " test_p_bar.set_description(f\"Testing Model...\")\n", + " \n", + " correct_predictions = torch.cat(correct_predictions)\n", + " return correct_predictions.sum().item()/(len(correct_predictions))*100" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Training loop" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "5a519f3a02424e86843a42fee66ba952", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/134 [00:00= 0 && t < n_classes` failed.\n", + "../aten/src/ATen/native/cuda/Loss.cu:250: nll_loss_forward_reduce_cuda_kernel_2d: block: [0,0,0], thread: [1,0,0] Assertion `t >= 0 && t < n_classes` failed.\n" + ] + }, + { + "ename": "RuntimeError", + "evalue": "CUDA error: device-side assert triggered\nCUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.\nFor debugging consider passing CUDA_LAUNCH_BLOCKING=1.\nCompile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.\n", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[15], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m epochs_x, epochs_y, epochs_acc \u001b[38;5;241m=\u001b[39m train(snn_train_dataloader, snn, loss_fn, optimizer, epochs, test, snn_test_dataloader)\n", + "Cell \u001b[0;32mIn[13], line 30\u001b[0m, in \u001b[0;36mtrain\u001b[0;34m(dataloader, model, loss_fn, optimizer, epochs, test_func, dataloader_test)\u001b[0m\n\u001b[1;32m 28\u001b[0m \u001b[38;5;66;03m# gradient update\u001b[39;00m\n\u001b[1;32m 29\u001b[0m optimizer\u001b[38;5;241m.\u001b[39mzero_grad()\n\u001b[0;32m---> 30\u001b[0m loss\u001b[38;5;241m.\u001b[39mbackward()\n\u001b[1;32m 31\u001b[0m optimizer\u001b[38;5;241m.\u001b[39mstep()\n\u001b[1;32m 33\u001b[0m \u001b[38;5;66;03m# detach the neuron states and activations from current computation graph(necessary)\u001b[39;00m\n", + "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/_tensor.py:522\u001b[0m, in \u001b[0;36mTensor.backward\u001b[0;34m(self, gradient, retain_graph, create_graph, inputs)\u001b[0m\n\u001b[1;32m 512\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m has_torch_function_unary(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m 513\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m handle_torch_function(\n\u001b[1;32m 514\u001b[0m Tensor\u001b[38;5;241m.\u001b[39mbackward,\n\u001b[1;32m 515\u001b[0m (\u001b[38;5;28mself\u001b[39m,),\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 520\u001b[0m inputs\u001b[38;5;241m=\u001b[39minputs,\n\u001b[1;32m 521\u001b[0m )\n\u001b[0;32m--> 522\u001b[0m torch\u001b[38;5;241m.\u001b[39mautograd\u001b[38;5;241m.\u001b[39mbackward(\n\u001b[1;32m 523\u001b[0m \u001b[38;5;28mself\u001b[39m, gradient, retain_graph, create_graph, inputs\u001b[38;5;241m=\u001b[39minputs\n\u001b[1;32m 524\u001b[0m )\n", + "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/autograd/__init__.py:259\u001b[0m, in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[0m\n\u001b[1;32m 250\u001b[0m inputs \u001b[38;5;241m=\u001b[39m (\n\u001b[1;32m 251\u001b[0m (inputs,)\n\u001b[1;32m 252\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(inputs, (torch\u001b[38;5;241m.\u001b[39mTensor, graph\u001b[38;5;241m.\u001b[39mGradientEdge))\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 255\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28mtuple\u001b[39m()\n\u001b[1;32m 256\u001b[0m )\n\u001b[1;32m 258\u001b[0m grad_tensors_ \u001b[38;5;241m=\u001b[39m _tensor_or_tensors_to_tuple(grad_tensors, \u001b[38;5;28mlen\u001b[39m(tensors))\n\u001b[0;32m--> 259\u001b[0m grad_tensors_ \u001b[38;5;241m=\u001b[39m _make_grads(tensors, grad_tensors_, is_grads_batched\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m)\n\u001b[1;32m 260\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m retain_graph \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 261\u001b[0m retain_graph \u001b[38;5;241m=\u001b[39m create_graph\n", + "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/autograd/__init__.py:142\u001b[0m, in \u001b[0;36m_make_grads\u001b[0;34m(outputs, grads, is_grads_batched)\u001b[0m\n\u001b[1;32m 136\u001b[0m msg \u001b[38;5;241m=\u001b[39m (\n\u001b[1;32m 137\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mgrad can be implicitly created only for real scalar outputs\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 138\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m but got \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mout\u001b[38;5;241m.\u001b[39mdtype\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 139\u001b[0m )\n\u001b[1;32m 140\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(msg)\n\u001b[1;32m 141\u001b[0m new_grads\u001b[38;5;241m.\u001b[39mappend(\n\u001b[0;32m--> 142\u001b[0m torch\u001b[38;5;241m.\u001b[39mones_like(out, memory_format\u001b[38;5;241m=\u001b[39mtorch\u001b[38;5;241m.\u001b[39mpreserve_format)\n\u001b[1;32m 143\u001b[0m )\n\u001b[1;32m 144\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 145\u001b[0m new_grads\u001b[38;5;241m.\u001b[39mappend(\u001b[38;5;28;01mNone\u001b[39;00m)\n", + "\u001b[0;31mRuntimeError\u001b[0m: CUDA error: device-side assert triggered\nCUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.\nFor debugging consider passing CUDA_LAUNCH_BLOCKING=1.\nCompile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.\n" + ] + } + ], + "source": [ + "epochs_x, epochs_y, epochs_acc = train(snn_train_dataloader, snn, loss_fn, optimizer, epochs, test, snn_test_dataloader)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAG1CAYAAAAFuNXgAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAACzSklEQVR4nO2deZgUxfnHvz2z7C7nIiDLtSyIyCGKuHgA4i3eiVEjUYPRSJTghcSLEC9MxBijaCJE44HGi1+CV5So64UgnsshAnLIsYjLfSznLrvTvz+Gme2jqrqqj+me2ffzPDzsdFdXVXdXV731vm+9pem6roMgCIIgCCJHiIVdAYIgCIIgCD8h4YYgCIIgiJyChBuCIAiCIHIKEm4IgiAIgsgpSLghCIIgCCKnIOGGIAiCIIicgoQbgiAIgiByChJuCIIgCILIKUi4IQiCIAgipyDhhiAIgiCInCJ04Wby5Mno3r07CgsLUVZWhlmzZnHTXnnlldA0zfbv8MMPz2CNCYIgCIKIMqEKN9OmTcOYMWMwfvx4zJs3D0OHDsXZZ5+NyspKZvpHH30UVVVV6X9r165FmzZt8POf/zzDNScIgiAIIqpoYW6cedxxx+Hoo4/GlClT0sf69OmDCy64ABMnTnS8/vXXX8eFF16IVatWobS0VKrMRCKBH3/8ES1btoSmaa7rThAEQRBE5tB1HTt37kSnTp0Qi4l1M3kZqpON2tpaVFRU4I477jAdHzZsGObMmSOVx9NPP43TTz9dKNjU1NSgpqYm/XvdunXo27evu0oTBEEQBBEqa9euRZcuXYRpQhNuNm/ejPr6ehQXF5uOFxcXY/369Y7XV1VV4X//+x9eeuklYbqJEyfi3nvvtR1fu3YtWrVqpVZpgiAIgiBCobq6GiUlJWjZsqVj2tCEmxRW05Cu61LmoqlTp6J169a44IILhOnGjRuHsWPHpn+nHk6rVq1IuCEIgiCILENGRghNuGnXrh3i8bhNS7Nx40abNseKrut45plnMGLECOTn5wvTFhQUoKCgwHN9CYIgCILIDkJbLZWfn4+ysjKUl5ebjpeXl2Pw4MHCa2fOnIkVK1bg6quvDrKKBEEQBEFkIaGapcaOHYsRI0Zg4MCBGDRoEJ588klUVlZi1KhRAJImpXXr1uH55583Xff000/juOOOQ79+/cKoNkEQBEEQESZU4Wb48OHYsmULJkyYgKqqKvTr1w8zZsxIr36qqqqyxbzZsWMHpk+fjkcffTSMKhMEQRAEEXFCjXMTBtXV1SgqKsKOHTvIoZggCIIgsgSV8Tv07RcIgiAIgiD8hIQbgiAIgiByChJuCIIgCILIKUi4IQiCIAgipyDhhiAIgiCInIKEG4IgCIIgcgoSbgiCIAiCyClIuCEIgiAIIqcg4YbwjbH/Nx/XvTgXjSwuJEEQBBExSLghfGF3TR1enbsOby+sQtWOfWFXhyAIgmjEkHBD+IJRV1OfIM0NQRAEER4k3BAEQRAEkVOQcEMQBEEQRE5Bwg1BEARBEDkFCTcEQRAEQeQUJNwQBEEQBJFTkHBDEARBEEROQcINQRAEQRA5BQk3BEEQBEHkFCTcEL6ghV0BgiAIgjgACTdEKDw1ayXGvfoN7UNFEARB+E5e2BUgcgNVEeWPby8BAFx4dBcc062N/xUiCIIgGi2kuSFCZXdNXdhVIAiCIHIMEm4IgiAIgsgpSLghfMGt7wx53BAEQRB+Q8INQRAEQRA5BQk3hC+QBoYgCIKICiTcEARBEASRU5BwQwjRdR0T/rsYz3+22iGd2wJcXkcQBEEQHCjODSFkbuV2PPPpKgDAFYO6hVsZgiAIgpCANDeEkOp9++USutTA6KS6IYicYn99IuwqEAQJN4SYoLdHoN0XCCJ3WLt1D3rf+Q5+/9rCsKtCNHJIuCF8gTQwBEE8+clK1Cd0vPRFZdhVIRo5JNwQQkizQhCELDTJIaICCTeEkIRkX0VCUObRdR0jn/sa1zz/Ne2uTkQCaoZEVKDVUoQQ8rmJLpt21eD9JRsAANv37MdBzfNDrhHR2KHPmYgKpLkhhEhrboKtBsFCZ/5JEATR6CHhhnBAfdhU0cbQoEwQuQNpYomoQMINIUTe54Z6tTDRwq4AQQCg6QoRFUi4IYSQzEIQhCzUXxBRgYQbQggt7SQIQhYSboioQMINIUS2s3K9byb1hr5AT5EgCKKB0IWbyZMno3v37igsLERZWRlmzZolTF9TU4Px48ejtLQUBQUF6NGjB5555pkM1TZaVKzZhlv+vQCbd9UEVkYi6KXggeZOEIQKdfUJXPuvr/HEzO9dXU+aXiIqhBrnZtq0aRgzZgwmT56MIUOG4IknnsDZZ5+NxYsXo2vXrsxrLrnkEmzYsAFPP/00Dj30UGzcuBF1dXUZrnk0uGjKHADArn11+MeIslDropuWJVMHRxDZyDuL1uPdRRvw7qINuPakHsrXkyKWiAqhCjcPP/wwrr76aowcORIAMGnSJLz77ruYMmUKJk6caEv/zjvvYObMmVi5ciXatGkDAOjWrZuwjJqaGtTUNGg2qqur/buBiLBq8+7A8qbOiiAaD3tq6z1dT90FERVCM0vV1taioqICw4YNMx0fNmwY5syZw7zmzTffxMCBA/Hggw+ic+fOOOyww3DLLbdg79693HImTpyIoqKi9L+SkhJf7yPXkTVLGbU1JBARBEEQYRKa5mbz5s2or69HcXGx6XhxcTHWr1/PvGblypWYPXs2CgsL8dprr2Hz5s0YPXo0tm7dyvW7GTduHMaOHZv+XV1dTQKOAm4EFZVLSBDyAAW3ISIGfc9EVAh9bylNM/fQuq7bjqVIJBLQNA0vvvgiioqKACRNWxdffDEef/xxNG3a1HZNQUEBCgoK/K94I0HaoZg6NYJo9JC/HREVQjNLtWvXDvF43Kal2bhxo02bk6Jjx47o3LlzWrABgD59+kDXdfzwww+B1jfKBNmhuMlZbXk3dYYEQRCEv4Qm3OTn56OsrAzl5eWm4+Xl5Rg8eDDzmiFDhuDHH3/Erl270seWLVuGWCyGLl26BFrfRgspbgiCIIgsI9Q4N2PHjsVTTz2FZ555BkuWLMHNN9+MyspKjBo1CkDSX+aKK65Ip7/sssvQtm1bXHXVVVi8eDE++eQT3Hrrrfj1r3/NNEkR3nGjFSJBhyAaKfTxExEhVJ+b4cOHY8uWLZgwYQKqqqrQr18/zJgxA6WlpQCAqqoqVFZWptO3aNEC5eXluOGGGzBw4EC0bdsWl1xyCf74xz+GdQs5j/zGmcHWg/AXkW8bQbiFugEiKoTuUDx69GiMHj2aeW7q1Km2Y71797aZsojgcLVaSuEaEooyz7uL1uPWfy/Ao5cOwCm92oddHSKHoO1UiKgQ+vYLRLRxE+dGBeoK/UFlULn2XxWo3leHq579KsAaEY0R+p6JqEDCTQ4Q5GQpCJGFZncEQRBEkJBwQ4iR1dzo7L99yp5wgB4jEQXoeyaiAgk3hBBjX+WXxoU6QILITejTJqICCTeEkKC3XyD8gQRGIgqQyZmICiTcEEKMDsWifotiEocAPUiCIAgmJNwQQgJfCk4jtC/QcySiALVCIiqQcEMIMWluBOlU1NGkuvYHnfuDIEKC2iEREUi4yQGi1p+oaBFIziGICOHxeyQNIhEVSLghhJh9bvgdl5opivAD0/L78KpBEGloskJEBRJuCCFB+9wQBBEhaLsxIkcg4YYQonP+Fl7jkJCEH38wmgDomRJRgNohERVIuCGEuI08LJ2//1k2SsjXgYgC1A6JqEDCDSFEeuNMk/8HrZzKBPToCIIg2JBwQ0jj16yMZnf+Q4IOQRBEAyTcEEJkNSvk/5F56DETUYO+fSIqkHBDCEkE4HNDHaA/6JIBFgkiU1A7JKICCTeEEFlBhAQWgiCoHyCiAgk3hBBZh2Ii85hXstF7IgiCSEHCDSHEzZCpFK2YxmRfoOdIRANqiEQ0IOGGEKPLOQqbg/2JOzgaiAkiN6Fvm4gKJNzkAEGaJILuq2hZOEFECM8bZxJENCDhhhBi2jhT0HXpkhoep3wIeWiWTEQN8v0iogIJN4QQ6quyA3pPBEEQDZBwQwiRjXPjZoNNpzyDZnrFDzhr0ieo3LInvEp4wBQ4kbRhRASgVkhEBRJuCCHmAdSnPCPSA/7u3wvw3fqd+MMb34ZdFYLICaLybRMECTeEGBdB/FTs7rJJf9y+F1dP/Qqfrtgsnbcs+2rrfc8zEwS9YztBqELNkIgKJNwQQkzmJp9GUDe53D79G3zw3UZc/tQXvtTBhOZ/lgRBEER4kHBDCJEXaNyZr2TT/rh9r0KuamSrbOPk5/Tx0o24f8YS1NUnMlUlopFDq6WIqEDCTQ4QZHdiciiWvMZxKXjEOkAtW6UbB6589is8+clKTPt6LTfNvv3ZaZIjAsLwLbzyZSX2k2BMZCkk3BBCorJxphagBKJlqe7GHFuI/wJEWq97/7vI1zoRucMdry7Ec3NWK11D+50RUYGEG0JIQiE4XwMO2y+4r04g5ILmxu0znV6xztd6ELnFV6u3Cs/ruo6tu2s554KoEUHIQcIN4Qtu+7EozO6yVbgJ/8kRjZ2J//sOR99Xjtfn2YXkRAS+baLxQsINIcTUQflkonLT5wUpf2SrWcoIjSNEEDh9G09+shIA8Me3FwMIJi4WQbiBhBtCSNA+N1HoALNWc+PDw6PIxoQI1fZBsZeIqEDCDSHETYh/6tPCgJ46ET5mRS+1SSI8SLjJBQLsQwKZiUWszwtyJVawuHH2JjLNx0s34tg/vY9Plm0KuyrOWNqRvMnWno7aJBEmJNwQQuRD+LkcaCPQAWaraENkB1c++xU27qzBFc98GXZVlFE2S0XhgyYIkHBDOGCKpeLiGuZ5Fx1gkMqVbFXcuPD1JohASH1D5HNDRAUSbgghwTsUh98DZqlsQxCRw7wlSPjfNtF4IeGGEJKQjIJrxClV1GZ02epzY97UVJAuYs+byD1YXxC1OyJMQhduJk+ejO7du6OwsBBlZWWYNWsWN+3HH38MTdNs/7777rsM1rhxEZntFwLUr2SnaOMPNAARfqDb/iBTKREuoQo306ZNw5gxYzB+/HjMmzcPQ4cOxdlnn43KykrhdUuXLkVVVVX6X8+ePTNU48ZHIBtnuq5NMGSp4kZ62a3f97d8w05sqN7nb6ZETmBeWBC1L51oTIQq3Dz88MO4+uqrMXLkSPTp0weTJk1CSUkJpkyZIryuffv26NChQ/pfPB7PUI0bH/Kxbdx1ZNHo/7JUujHg1iylKvhU7diLMx75BMfd/4HahUROk2pG5ORORIXQhJva2lpUVFRg2LBhpuPDhg3DnDlzhNcOGDAAHTt2xGmnnYaPPvpImLampgbV1dWmf4QCLlY/OAk6UZvRZa3mxofhQ/VVfFe103OZRO4i6wdGEEETmnCzefNm1NfXo7i42HS8uLgY69evZ17TsWNHPPnkk5g+fTpeffVV9OrVC6eddho++eQTbjkTJ05EUVFR+l9JSYmv95HryG5+l9XbL4RdAR/wUztDEKow25ilTVbv24/pFT9g5779GakT0bjJC7sC1pUquq5zV6/06tULvXr1Sv8eNGgQ1q5di4ceeggnnngi85px48Zh7Nix6d/V1dUk4Cjgammngs+NrFAU5AAdy9LRPwxnb1rem+N4/BTMcbHMbeWGl+Zh5rJNOP3b9njqV8d4K4ggHAhNc9OuXTvE43Gblmbjxo02bY6I448/HsuXL+eeLygoQKtWrUz/co0ghxvdLN3IXRNITYIjS2WbUPbxIVMDwSK1mlE0cZl5YPuJ95dszFCtiMZMaMJNfn4+ysrKUF5ebjpeXl6OwYMHS+czb948dOzY0e/qEQcIeiwjTYA/kFmKiALkUExEhVDNUmPHjsWIESMwcOBADBo0CE8++SQqKysxatQoAEmT0rp16/D8888DACZNmoRu3brh8MMPR21tLV544QVMnz4d06dPD/M2chrZ7RdUwq5HLUR71pqlQhg+ovC+iOjB+oRk/fUIIghCFW6GDx+OLVu2YMKECaiqqkK/fv0wY8YMlJaWAgCqqqpMMW9qa2txyy23YN26dWjatCkOP/xwvP322zjnnHPCuoWcJ+j+KRLdX3bKNtL463MTDN+u24GbXpmHO87ugzP6ypuliWBR9eui1VJEVAjdoXj06NEYPXo089zUqVNNv2+77TbcdtttGagVkcLNbt+OS8HdrC8nbPjx6FSzCGoZ/zXPf40fd+zDb57/GqsfODeQMggJvL5egUMxQWSS0LdfIKKNXwIN/7rwyVazlJFs97nZVVMXdhUID6R3BTcejMLHTTRaQtfcNBb27a/Hjr37UdyqMOyqKOFmRY6jQBSxTi8Lxn4bH323EbNXbJZKKxR8FMsN6tVFrEkQPkDvlAgTEm4yxMl/+Rjrq/fh41tORrd2zcOujjS+CTQer8vWnbuD4qqpX5l+Z0pzRlZEggVz+wVqK0SIkFkqQ6w/sNHgx0uzK8ZDwkVnpaK4icJWDLEckJuyySy1b3992FUgAsLko0e6GyJESLghhLgRaFQElih0f7muFfJXfvSW2X1vLUbvO9/Bwh92+FQfIgqkvqHGrLkhoT1akHCTYYIYSIPVfsjFuVHK0UUHmKvix6crNmN6xQ+e88nUOOK1qT09exUA4OHypZaMveVLRIPGGsTv8Y9WoPed7+CDJRvCrgpxABJuMky2KQnMggi/u5IN9me7zkWd/CbMd3L5U1/gd/9egKXrg9ttO4ptLte1ZbmCsk+W8e9GpLr5y7tJYf326QtDrgmRgoQbQkjg2y9EoAPUIqAXSvlkuUUseHrK2pyXf1mZCf8VED5gmuSE/2lnnDiNqJGBXkWGybY+PCHZWanEt3DjaBjkRD/blAgsQSZbzFL8jAPKN0vYVVOHh8uXBarBc0OWfRqhkwsxs3IFEm4IIY1h9pVt3ZHqOxH1t2FpzrLtmQfNg+98h8c+WI4zJ30SdlVMeAkVkImmtXbrHpz44Ed4bs7q4AuTgISb6EDCTabJssYv2z+pBPtrzCsq/ID1yIRaNV/NUvTCgmCBz6vH3pi/DkMe+BDfrsvsqrRMLwW/763FqNy6B3e/uSjwsmSI0YgaGehVEEKCntlHYbDMttmWn+9E1bHXr6Kz7JFnHTe9Mh/rtu/FDS/PU7tQE/7kX5bafiHDE5fa+kTwhSgQp4YdGUi4yTBBNP1o+FvIOxJGbefgVH+0b389/vnJSqzYuCvcCjnAfmQuIxQrvgByuYkOO/ftx+rNu4VpausUB39d+FPp8sb4TrNtopTLkHBDCEm4kD5ULpFNmok+4/GPVuBPM5bg9IdnBl+YB6IgEHqHBgGvHPunD3DyQx9jxcbwnZAbNDfGSU5ONFQlSLaJDiTcEEJkfWmUBJqIdXop00zFmm2B5F9TV49Ln/wcj32w3Jf8WO8hYo+UyAB7D0TEnbWcv4Gq6FvbtrvW07do3Mk9lU2mNTdRa/fxXNjLJUcg4SbDZJtk76bzUAriF4HOKeh38vq8dfhs5RY8XL7Ml/xYzyxzpskIvLAsJirP780FP2LAfeWY+L/vXF2/b389+t39rv2EZNDPXIXMUtGBhBtCiK7gSyOdp8LKqhSZCLQXVL+0p5b2nLHSWMeAsybNUnfydcDNo7zvrcUAgCc/WemqzHXb95rrkDJLGY41QtmGIm9HCBJuMkwUouGqILtXjOuNMyPQAQatSa5P+HuTTM1Nhp6jb6ul/Mkm61i6YSf+u+BHX/MUvRLeYOv1PSY4bVq0DUtjsNg09gjFa7fuweZdNWFXAwCQF3YFiGgTBeEjaIIWOP1+hm6cvINA13WaqTZS6i1tMPUNiTQ3mqb5/jFE40tooDEvBd++pxZDH/wIALD6gXNDrg1pbjJOtrV9s1lKzqFYzecmnO7JWG7qnQQl5FgHAhYqJbOD+LlcCq6cPpi9gxqjf4ZfiAM4sk967Yd42kiRyTnLuj5XNGZh//tN4rAEmYaEmwyTbU0/V8cc430F/U54mhbXAkkO7C3ViMcAJtn2OBKW8DkNPjd84bcxvPPGYHrLFki4CZjvN+3Cq3N/CLsarpFd2qmyG7CbKKZ+d4wJk+Ym2B6J75/gLj8/BRkvd56jci8hAV9gZ/8NZJ+/oRsay1LwJ2Z+j2ue/xp1pgjR0eoRyOcmYE77qzkgXDbNXmrq6tUjnAJQaeRhfQ6qPr4/bt+LwiZxtGme71tZbu9d1aFYuHGmD2UT4eLGodgrPFOr0CwVQFWiZs5sLEvBUyEE/vftepzfv1PItWFDmpscIIjvu7YugYF/fB8LDRvvCW37CnlbVdcrNu7C7f/5Bmu37nFRU3cY6+DUH+3Ysx+DH/gQR99X7qosv81SqhKJvxtnGvN1n7FoFv+LJz/D4h+rXedNNBDU4M/TRhr574Iqpm9bLtNYhJsUqUCSUYSEmwyTLarZNVt2Y+e+OueEDNS2X9Bx4eRPMe3rtfj11K+46fx+amafG3Hu32/2ttcU1yzlMj9mhOIQdGBeSrSOAca8Pl+5FcOf+MxD7kTQ8B2KG47/Y+b3mLlsU/p3tvR9XqBdwaMDvQqCyW5m4Dm51VJOWO3y1QeEqOUZ3LAywZhRBjXp4pql3CpuQtTEZ8oMsLPGnWBN+AvvdduXgh9Ib0m3yKCBawxKjcamuYny3ZJwk2ki0BreXbQelz75Oap27OWm2eNhcImWFZyNymopzwHPeGYptzt5Sx/0n8YegZZIwnv3ohVSQXR9UWuDjU24iTIk3DRCrv1XBT5buQV3v7GIm2YXQ7gR+9y4cyIOz6E4c74AMs6XKoS5FNyvgmgM8A832jSvQoHVLJVyXLbHttFsaXKZRrJYKisg4aYRs21PLfccaz8k2f5QqeMMaepl7JuD7nT9vsWoTFbD8PPJFaK2ykcVNwJ7Yxj3G8tS8GyAhJsMky1Nf3etollKyefGsFpKrRT/UAri562Wfse5YZm5MuW/E1SE4sZGtj87a5vm+dyY0mRL5+eBxmaWMk4Mo9amSbhpxIhWL+xlaW4kG6+SiUo2qc+dRkLF6cYj3FmuW6EpVIdif/JpDCtnRERsHFCmTlJgN/ncBDDwR0172NiEmyhrIEm4yTDZYndm+dyIcOtoGlbnZCzVaaD1+v3KOl9K58c85i6z0JqjdSl4dPvIQMjU5qfKQRolr+DvdM/W6ACNRHNDIyqAaAg99CoIJmyfG38arBtByO9+kTW4BBbN1e84N4wLM2eWIvwgAn2/J2Q1N0YagWyTNZNXv4jy/ZJwk2ECWQ4ZwJCzR9XnxkA29NsqM2ev9yOzK7gKYari3ewL5oU532/GigzGP8oUUTOnpLC/U3Y96607ZwpTJ2kMJpt4I7hHGaIgvNPeUhkmUm1ftOeQonbAdRA/+cv8xVQHFR8hXXm2wlPRut8VnHHMVU7e2mPQA/SKjTtx2T+/AACsfuDcQMvKNJnq/IPqburqrc41yf+sbdrsc+N/PaIwiBppzKulIvYqSHOTaSIl3Ajw0lBVBu3wdgU3VuJAGf4W0VAWZ+9R12Yp1zVh5OVhtZQXrM+ale/S9bmnsUkh89xlvyO/9nxjp2d/FbKmVrM/W5Z0fh7Ilv49aKIg6JBwQzBR1twobp2pep3fMzQls5RHUwx/40z1vJLXsZaCZ8hB1Sez1Py12/H4RytQV8/fdT6qphs/cNox+4MlG9D/3vdQvnhDBmvFgv0OrD436aXggleWq0oN47L4xmB6M8ITXcmhmAiVoD7DuwSRj22E9A0wFDfK18nC9bnx0Qm4po4vJPiJX6/rh2178Zd3l+KVr9YCaHxLw5021b76ua9Rva8Ov3n+68xU6ACyYxJPKPXDLLVg7Xa8/U2VXOIIYPy+G7PPTfjijBkSbjJM9nTiDO2A5MaZO/buZ8bJYaUNKcyNabaV6pB5ZXidhfBlG/+6g2v/VZHx2ZIfpa3ctPtAXiyDSLZ8K+r4+a7ctSPpL495lLtaSpiT3Pv86eOf4rqX5mLhDzsc00ZAQWAy0dFS8CQReC0k3BDyRKEj8Qu3y6ndDEp+m6V4+e23OnkGTOpZ1NYl8MTM77HYsAO0LCKhNbfNUtHE/sx5q6XUtZGqE5TvNzn7XEWhjdQ3YrNUlCHhJsME0fbXbt2L3S528RYOLAHGPzGZhCQL8vuxqWwj4HkpuMIs9/GPVuCFz9eI68Op0N79fG1ZIqGjpo5/XhpG4c/NWY2J//sO5zw2Szm7xjoU6JmxIgaGzecmvXEmmMcBuXc9r3Jb+m9+oMBoYTJL5apjEQe+tjuz9WBBwk2GCeqlD37gQ+VrglT7ywpxmfgItuyqwQ/b9piOGfvN1J8ydXZlAOBNci0nKrfswV/eXYo/vP6tqzrUCISbi/8xB0fe855y5GlR2am/F/3obD7g0VgnuuFrHNgPXvZ75ArsggxkQij8bPKchjKiMEJK0NgcimUmpOG37wgIN5MnT0b37t1RWFiIsrIyzJolN/v79NNPkZeXh6OOOirYCvpMUC99x979vuanrLlxG7PF1VVqlP3xfZzw54+wbXfDLuhefB7q6hO4641v8dY3P0ql55qlLL9lBQ9e3UWam7mV21FTl8Dn32+RKoNftv1YzMNsNTUYZMk45htRvV/Zasn63HgZ6nkbzprKi8BzzBYNk19wLZIRewyhCjfTpk3DmDFjMH78eMybNw9Dhw7F2WefjcrKSuF1O3bswBVXXIHTTjstQzX1j0w3gLVb92D8awuxkmG/Vp1kuA3UJzonH+fG+4zIaMNPKNTBWt+KNdvw/GdrcP1L87B26x7+hQfgz3INf9vOCZy3Ocf37Q/e1mHa0f3An15WiIjea9Q6Sz/x89bCeE7W1VJSS8EVR5ts0dwY69kIFDdympsIvLpQhZuHH34YV199NUaOHIk+ffpg0qRJKCkpwZQpU4TXXXvttbjsssswaNCgDNXUPzL90n/z/Nd48YtKXPLE50rXqWqY7G6IySNrt+7B/321Fvt5S0fD2jiTtbeUzHXQTbPWRRJOtHzNjX3FVsNvQR1c+NwEwoF65MW9CDc+1SXLkI2zlHEXDsl68f3IrL44hr8V9TgympsoYAzSmcn+vbYugbHT5uP1eesyVyjME8OgI1B7ITThpra2FhUVFRg2bJjp+LBhwzBnzhzOVcCzzz6L77//HnfffbdUOTU1Naiurjb9C5NMf67frd8JANi8q0bpOnaIf/Xan/iXj3Db9G/wzOxVnvLxgx1792PaV5Wo3rff4jvCr89366sx6oUK0zHzs2n48e26HRj2yEx8sMQceE1mA2XdUg/xE2Kf3WcQbnizK68dECtXL34GEesPM4bsIJgX0bXFdofi5P/CjTMVXzbP9BU1MrXDu5X/+3otXp23DmOmzc9oubz+MgraGiOhfTmbN29GfX09iouLTceLi4uxfv165jXLly/HHXfcgRdffBF5eXLbYk2cOBFFRUXpfyUlJZ7rniv4KmlbGnaqoaf+/3wl29cjkx/Eb1+Yi9unL8TYafNNHZKoDpf84zOTP5Ou8zuz3zz/NZZt2IWrnzMHXpNaLWV7fgKzlITmJhPPNdXJeVkhErXZXqaQFfBlZBtfTVyW3+8v2YgRT39h20hX1o/My+YLMr4sXu597dY9uPypz/Hx0o0ecnG3+tMPthp8CDOJXMiM4OvhROjTAqvNnbcxYX19PS677DLce++9OOyww6TzHzduHHbs2JH+t3btWs919kIUwlK7RcnnxvLb+E4z+QiMz7v2gGns/SUbufs9WaneZ3fyNXdmDX/vZKQF5OLcWAc7Ub/OO2VcLRXUI2bdihfhpjGsLmGRTZqbWcs349lPV5uOcetvOW5aCq74roPWiPz+tYX4dMUWXPnsV57yyeY+3Q3Zcruh7Qrerl07xONxm5Zm48aNNm0OAOzcuRNff/015s2bh+uvvx4AkEgkoOs68vLy8N577+HUU0+1XVdQUICCgoJgbsIFfrWL9xaxtVtW8mKaK/Wu13paP3je+Bd0x6ASHVi28zVpfXyqgzWNbDRoIzI+N36apYJ2KDayb3897pj+DU7vW4zzjuzkuryoINvsvfrc8MtRMy1Y42hxd7oXtFvVWwnaLLVpp5qpnodoYUDllj34fvMunNKrvS9lRQFj/8fzo2rUS8Hz8/NRVlaG8vJy0/Hy8nIMHjzYlr5Vq1ZYuHAh5s+fn/43atQo9OrVC/Pnz8dxxx2Xqap7w6d3fs2/KpwTwf2smjVrElXdqTHzNDeyj8NtHy+lNVEUsLgdO+e41GopjlmPeR3X56ZBHcWvIz9fVVJZxb04FFvy4jF1zmq8Pv9HXP/SPNdlRQmb4y0nXV48fM0NC2uTTg1ysj43Mt9c0A7FQWgNrbd14l8+wlXPfoXPBCEY9tTWuVpOHpbOU8ZfMQrandA0NwAwduxYjBgxAgMHDsSgQYPw5JNPorKyEqNGjQKQNCmtW7cOzz//PGKxGPr162e6vn379igsLLQdjzKZlmhFwo1w9QLLoVihxYps7+Y8pbN0hd/9o67zhRJeUTL+CTrknwXPpGbczys4s5Q9Zy+aG9EAYyxpi6JDfNSxtkve+/Ia8VY93INcy+EOaoLyY5YJjlPdpDQ3Hhq6l1V+pipI1GHe2m0Y1KOt7fi23bUYcF85+nZshRk3DfWlPkETlgO1KqEKN8OHD8eWLVswYcIEVFVVoV+/fpgxYwZKS0sBAFVVVY4xb7KNTLcLt52jajWdNA+mjs1Fj+R2/JTzdzlQhkR+OnRLdGO+acn5uM78GxB3IFzNjcT2Cn5OVlN1JodidWSFCBnB0Smrddv3Yu6abTjniI6u35W1GvxAbnyNlDGPel1HzOGL81NzM2fFZvyn4gfcdX5ftG6WD8A/zY15laNanWcu2wQAWFwV7ipeFYyvmG+WCp9QhRsAGD16NEaPHs08N3XqVOG199xzD+655x7/KxUgmX7peW6FG0WzlBM8v8igNVlBeParmrFkBSyz2l5UPvt4fb2zoOUVVr6ZWAruRxDHKGF9jry786q50XXghD9/CF1POrxfdlxXcXrpfC1CTGopuOAa40Ao0z79DOJ32VNfAACaxGP488VHAvBvHyiRedkJVrOeV7kN3/ywA1cMKg2s3S9dvxOdWheiZWET5WtVzfJhEU2Dbg6Tec0N/xWLvhvVSZPtvjg2ecfrfEYmgJ4KyaXg5t9OyHTSNnOXi7oZywlKaGTF4jEK0KodnJetG7IZuwM5Gz8G4FRZn36/2XNe1jwdj3vQ2HLifnpi3fa96b+9mFONWM3LXvnZ5Dm4+81FeNdh0Yjb6n+9eivOnPQJTnnoY1fXy00Ywxd0SLjJMJn3uXF3HauWakvB2TM7az5BPw2WcBPTrHVI/pDf7FNNYJLbi0W3nBOYpTinZARSr5ulsmapRgHlgyUbMfXTVS4yZpXVcDAbRKB9++vx0LtLTTtb85DtB2Q0r372Kby2ZW03sn4XXF87iTrLlOHl3jO5yp77XAWdzvIN9i1z/OC9xckgo5t3uYuTI/dewoeEmwyTaYHWbZwM5RVEtuvNv3mmizAcimOa5vrj08Gfqcn41pjz4puRxHFunNXCQT1XVr2M2oWRz3+Ne/67GBVrnAd4QMGklQXSzT8/WYm/f7TCtLM1j0yZpVSRN0uxjsk3OpmkdfUSg6iHdu6fWaqhEomEjh8N2iEnsqBZ24iC4CIDCTcZRrVhbKzeh+temosvOBF+nfDXodh9s+aPYUH73LA0N5onQcAc3dh9ByzUYLnxuZFQ3aQcGN3Ccp5kqfc3VO+Tyk/UPE0B4LJgGFi2UX6mLdvkZL7fMCwArKbmdz2CXpXjn0NxA698tRaDH/hQOg5ZGK5kXos0x/niTbQ8FuIDJNxkGsW3Pu7VhXj7myoMf1Jt48sUbh2KVWUO28aPlvNeNTduBzem5sbS6lVuVdd1vpmJd42EqGj3wRCYpTjHZcxSU+esdk4kgPW+WAOwbLMTde5RsNuroFJf2YE745obTr2s70l2wQHv/Ur5qgUc5yYIh+IUT36yUupaL0J7aE72DNO05XAkIOEmw6g2gMqtezyV59ZhkxnET8XnhrOaQjUfr7B3//ZolvLpBnTBDEi8Wio8s5S5wOR/7D5Wrt3Jdu7GMtbv2IdfPvUFyhdv4F8QcWTfj8wAHMYYZ62+ZtGGpo97iGBbr+uoT+hY9OMOuf3ZJDCW65dDMasWsvUSCvfuKhM4xlchsylwWJBwk2Ey73MjCOInCqCmqrlx+O15tZTrODf2YzaHYmUtFedaCfOT6bggjTjODRsZdbFXjLFHUn+xqirtSuPivd795reYvWIzfvP8186JM4jKE69P6HikfBk+XSFeweSnWcqYk9d+SDWCuS2tROJEQsf9M5bg3Mdm44H/LVHIXQ7jxM/LJpQy/ke8CUk2LhY0+wo2/K1x0oQFCTchsX7Hvoyo3d373HjrvKwYq+El6JUqTJ+bmAa2W7DcQMI3M6nZn0UqXdFT4fvccNIL8lJB13VsZOzHw16RJquRkdXwNOB2lUeUeG3eOjz6wXJcfiD+ihFjm/VPu+ANay3YA7p8fjJJ6xI6np69CgDwz1mr5DOXxDjxO/q+clTv2+8qH2/fV/Y53SQ4/Vb44owZEm4yjK7rmPrpKhw/8QP8+Z2lgZcnE4MkkdAZG+PZ04lNJeLfkVst5aFc4/YHMsKZjA5G13VpbRL3PQZslvrre8vwr8/XSJUhK1On0qkIudEY7r2xevNu7jmjCSbTcYC479Ty/VoF2uR0gWGW4vrcOL9vVoTiL1Zuwc3T5vuyHYf12bpdes3sKyXSuMk3CuiiWVk6TWbqIoKEmwyjA7jnv4sBAP+Y+b1jepWJW22dfepu/IBrLVP71JkrnvkSh9/9rinAlVdfPr/j3Ljt4rlxbgy/lT5EXS7isPk4W40rMkW5cSg2+/D4z98/WsE8znoefq/wDmKTQ99ReOiiwI7Gc1HR3FiRnfzw49w4w3pGw5/8HK/NW4e731x0oEz3Ld36bN2vvWBouWVNhdF8vUKM9yazb15YuBJunnvuObz99tvp37fddhtat26NwYMHY82aNYIrGxfMFQUBvvXz/zbbdizfEMVv574623kAmH3A7v/G/HWGo6y6C40lwp8884Ps83D72GRMJl78izz561iOy+bLNUspCl1eSXXqbJ8bn81SWTgIiBCtBDK3kWBeHlejIlme13p5XS3ldqGF0ffParJXWX1UsWYbPlji3aE9jGbtayBPj3UJElfCzf3334+mTZsCAD777DP8/e9/x4MPPoh27drh5ptv9rWC2YyMutJPlm7YaTtm/IDXOnQIqnu/OLHREOvE7HOTOXgDr1snWB26p1VJIgFGNl++WYqTV0BPPFUEawyS7T5zTWiRxc9lzsJo1oZ372XpsPVK1s70St+UxO0bJyb5llDrfjQb6yRHRXNz0ZQ5uPq5r7F26x45sxQnH+Gijgz0lHtq6/Do+8uxdL197ODBM39/+N1Gw/HwxR5Xws3atWtx6KGHAgBef/11XHzxxbjmmmswceJEzJo1y9cKZjPs5dSZfenGTlRltsOqpaL1Buf/vUGT5HWm6LtZyuXgr+sC85PgGnZ6cx14u43LlxOsWYoH29fC/sbYS4VT5+TLywWByOazYrgpVpv9fOUWrNzkLRy/l4CTtnQMs7PSdySR1hihuEnc/5du3ZrGjUZjffU+9jOTfJDmFWxqX63b78B43SPly/DI+8tw5qRPpK83TdAO/Nqyq0Y6tk+mcCXctGjRAlu2JCPmvvfeezj99NMBAIWFhdi7Vz70dK7DaqrVe9155LvF2FFaNTfWj8P4WzU6KMuheEN1g9Mft+OwXPfSF5WYvdy/Df5ktl9QlTe5qwW49medmcbuQCyruWEf52oDAjNLHfif+Yztx5hRbX2qy7frdmDkc19hGUN7mSlUBnfr1gIird2KjTvxiyc/x6l/ncnMK4xgbrIRio3fvapm2NgHNcmzSiLJvLy0H2uf5OYxxjhCnWy9zH2uevle+XZdtfI1xveSqrN1KX34ehsgz81FZ5xxBkaOHIkBAwZg2bJlOPfccwEAixYtQrdu3fysX1bD+oAf+5DtlMnDq33U+MHsrLH73BhXSZlMR5KdFw/rB2/Om69h+P1rCwEAqx84V74wATIbZ6aQedI6rGpZNW0JT6iyaoTEcW5kzFLsMoNANnibFx80p3Q/+ftsJHRg0Y/V+GzcaXKZ+sS+/fW4edp8/O9buZD7gNih2Prul64Xa2xEM37V/oNvPnEuk3mth+7LKKznx2PYscf7xND47di1Z25yZJu43TgUJ3QdcZ9dA5ywCY0S6IxOzCqYRcAq5U5z8/jjj2PQoEHYtGkTpk+fjrZt2wIAKioqcOmll/pawWwmCoGMjB2EdWnlio27cPjd76Z/m2ZWiuU4pec7FEuqb92uZGAJMVaHYtU8fahDMh+zYMQTTmQrkOBINEGZQnVOxwawNTds7b2OH7btYQ72vGqzBuxUHap2yO1p5ScvfL5GSbAB7O/E2CalhGTJd+qXz429fPNvTVNrZzIpje2qSTyGy55q2ILGjzuxtlvZFXnGfpTnpyMbcdzYlkUTmtWbd+N7j2ZJFvkuzH2sCV3QW2W4wZXmpnXr1vj73/9uO37vvfd6rlAuEQXp1fjBWIO8/bDNbELUONqV9DFF3xRu3mD/HQTsIH7O96LrOtdvxE9tDS9f2fzMdVM7LsvOffuZnVcqX2b+hke3Y+9+fPb9ZpzQ82Bbsg++25gOjZCtbNujHlTQ+jxNbcrwnXqO7K2IbL5uIhTzzLMy5OfFsOhHdROKCJbmpj6hQ4M5hMbUT1fh3xU/4F9XH4c2zfNN1/EWJ0jjoC0HgLr6BE5+6GMAwOIJZ6JZvqthm1UkmlgdjyRg9WfWZxmFib0rzc0777yD2bMbnEUff/xxHHXUUbjsssuwbds23yqX7URBuDFpbgKskM3nxnKeFdlWBV83ztTMgW5SdZfd/8qslnW+RsZ5WUVzI+NzY1a/8/OSYcCEchw1oZxfH8Y9GWfBVz37JUa9MBf3z7CH0J/lo38VkD3OxsKl4AxnXRFuPmsZbaKpDnDWdqqYelWrbHUoTj0TL12a9R0kEsCpf/0YF0z+1PTN3vPfxVj0YzUe+2B58jrDOZ7PjSxmh2LzudTPfYb4ZUbfFq8uC4A74cbsc6PbjkUFV8LNrbfeiurqpBS9cOFC/O53v8M555yDlStXYuzYsb5WMJvJxP4+jmkNSZ1Uh04rNlRuxzoze/ubKnyxcsuBc8Z08nm6gXUfcatDsYJDoM7J0w1WIUnkD2CEdy6ICMW6rqPOod04LQWfW7kdADC94gd/KiUgjGB/bp619Zmavz2JMtWL9BVmHR0qpfrdG5PwBmEvz8H6Ha3avBtrtuzBNz/sYNYv5Z9oPMeLdm4XVDhCo1Ofi+DMyoBLzY2hOuur9+HuN761R3cOu4HCpVlq1apV6Nu3LwBg+vTpOO+883D//fdj7ty5OOecc3ytYDYTlBlS5Ixoq4OhEk7XiWYRgLi9yghyL31ZieMOaSudJ7csjsmIBbPD4DgUW6+LM51iraulnO+Aa5ayLAU3aYEk87PWTeZ6FaSaGsuhmGnS86FCgNDhIqiIvu98ux6bdtVgxPGltnNuvnNxED9nTZ8pvUK5r879AV+s3MoVWOUdvC3CGTTHSYLThEJEvgvHVydEDsWs2qX6T+O7428vIVcH4+W2vkryu5Klti5he475eer5Gev5xMzU8m9z8N4IyDbuNDf5+fnYsye5rPj999/HsGHDAABt2rRJa3SI4CRuFecto0DjpPGRNcvIIFZRGzUM6gWpXMJKa5vdH0gju1RVuc48E4BlJssLwidbPq9deNEgCoWsAydlHYozobqOBbShzKgXKnDn699ixUZ/nDrFmrmGv2VMnbJoAMb+3wJM+3otdiiGpLCvlmLVSZyH055EonuyahhYQ/KTn3yv9FysgQiNl7LeT6r/NPncQGN+I7K1cBKo/OKpWStx2B/+h9nLN5vK9Kq5iTKuuoITTjgBY8eOxX333Ycvv/wyvRR82bJl6NKli68VzGaCagMiM4H14zZpbpyEG2M+rFmYcMAXZp3MnzWbd77M0zV8s5Rxdswqgy8oeFGv847rljLdaG64S9Q9NESZ6LeyGyZmRLgJ2CxljecBuBMeZX1uZNpOGG5GLK0Hs64ceYb5zVkO7jGEqbBGKGZddP+M7/DFqq3YsXc/pn1V6bh03KrJNjs829On+l2jUBSLsb8R6zFe0zdNqKzCluX/ZHrD3wov/o9vJ/3dbv3PAtNxN8KNDFEQgFzd2d///nfk5eXhP//5D6ZMmYLOnTsDAP73v//hrLPO8rWC2Yy1sfpFnXXZkwFrn2laLeVkllK0+4sQaW54DrkqjH9tIf7w+kLHdEx/EI5ZyqlzY6ZzrIFd0Jj66Spc/tTn2FNbZzpuFpqEFWDCCy7oBZl2wE7DX67tBtOSZkG6oDeaZGbv4r5Ez8L6fTjdURjjCFtzwxjkOdKNjNn76zXb0n+3LJTzoNhQvQ9jXpmH26cvxHUvzRWmFQkgTM0Nw3mWp7mxlcU5LhM41XjYa/NOrgZryMQq3Kzdugd3vv6tcNf6KDoPs3Dlc9O1a1e89dZbtuOPPPKI5wrlEkE5FIs0N3WJBOKxePq3ilkqZlbd2HCzJYAJDx+m8aPetqcWL35RCQC4ZVgvtG6Wz72Ot3Gmafxgzrw4GereB+nU0ueipk3Sx5/5dBWuPfEQ5/LBfw8yjsaqCN+5bv7fb9x8P6klvPUJ3bYxoh/4JNugjrU5Uyo/h0HWmsZPeEK19b5ll4LzLFFszTD/pmQH9Zim4aOlmwA0bAjMq49tOb7h7z+8/i0e+nl/0/lUemOfypsoySL0uQkAaxnGODe6ruPKZ7/E95t244MlGzCHEwxTTlsdvgDkesF8fX09Xn/9dSxZsgSapqFPnz746U9/ing87nxxIyGotipSadfV6ygwvFVjH1rvVB9TIDFvlWd9qMyotR5V+k4rebhxbkxOm/b6ieqluiqJl2RXTX367xUbd+HW/3zjeI2oTF69PAk3MvfHTBRc5ybKOR7T8NSslfjLu0sx7dpBOKqkta9lswZZN/4vAtnGphn0aym4ykIEJ6xZaRL1cDKVii63Pi+eY62KQGuPqttw4D8VPzCEm9R11poqTI6U6sPN3jXWscOouZm1fDO+35TU2PwoCIYp059EQbnjSrhZsWIFzjnnHKxbtw69evWCrutYtmwZSkpK8Pbbb6NHjx5+1zMrcfN+99TWmYI0sb7h/QKzlHWwF0UotuK4WkqoSjefZNUxHZtCMk/WtYBzPY1w49w4INqmyTwbVfvQZU1PbvybeKu4gtOsmP83nfO5TNmYHjFNS/sXjHt1If5301B/K8LAzb2KJijGU1zNjYvepWa/s52c6xti88FnaW7E2hze305lA6LnYMZJtvGyf179AQnLKGhZzckN9eJJKtZ0gmsOYDWDNfytjrXZGbdfuOKZL6XyiIDcIoUrn5sbb7wRPXr0wNq1azF37lzMmzcPlZWV6N69O2688Ua/65i1qH48d73xLfre9S4q1mwVprNuumfEFpjKUAeRKtyKVxVpTR1DuGGk8zoIOkcath/TJDbOdLNaiZue46PjxvRkzUOmXp7MaBICrUrcJT/KFvrcGHq0NVt2YxdjPzX1ehgr4o9TvEiLIuMM7ua7qamrd07EwaopkY1zoyLYi7Wllvpw0qkslVYWbnT7dbpHHTfv+ZjSeMjfSiKhmwS8PBemW6nd5ZVz9R9Xws3MmTPx4IMPok2bNuljbdu2xQMPPICZM2f6VrlsR7UDev6zZKyAh8uXCdOJNDc/bt+Ly5/6HO8c2OtGtP2ClZjJLGVH5XZYwk06HwnNh/RWBI6aG3uCmAbmzcgsy7Q7/jpWkYuwMxeZLTiF1ptVN+D8UEI0AKRU2F7bip8Y2/Ce2noMeeBDz3kaHyvbLOUiT+GKR+PfMgOJXAX27XcWbpxy2l+fwJ7aOo5wIq6bkzZRRTPMQ2W1nH0LDHH61DszRQLnaW7ctAldx0rD/lHp1Yg+qkGdzPgyuDdVZxZXwk1BQQF27txpO75r1y7k5/OdOxsbQTlV7Rdobm759wJ8umILRr1QAUBt+wU/49zUsjQ3CgODWKshlw4QORSztSmO9YKuvCqJN1j5rbkxzyjZ5asiuvQ3z3/NzT+svs06uKnGc2FhNgvYcfOd2yIUG/NTbF8ijHntkzBLOXHSgx+h713v2jViXvdYckB2B28VPYTNx8Xhaaf6Uj+1LcYyv1y1Faf+1a4c8FMx6offVQT3yGTiSrg577zzcM011+CLL75AasO/zz//HKNGjcJPfvITv+uYtQT1sYs0N9+tNwudRuFGLc6NHRVpvJZRx9TA41WtaVULA8l7+2HbHns+jIxiGl/gEB1jlS8Db7AS5eNmFst7vd4cip2vZa6c8aHxmzVpYmE0RRArpJwdZdXztGpueO1CRvgX+SMZ85IxSzndS8rRdJk13D7YwoGKllPscyN3jUoQR9s7cKgfa7VUMu6Vc/vnZW1M9t8FP7LTcLRfbvDju5TqvyMgALkSbh577DH06NEDgwYNQmFhIQoLCzF48GAceuihmDRpks9VzF7CEG5EdVDR3LAqnzry1KyVOP9vs01BsqzJaxgqcKbmhlMX1Qiu1/7ra5zw549QvniDYz48u7zx8F/eXcrptKwWH/cv2e0MiFdkwqIu91oOEK7tnD+w82sVgGxj2wXaD4Q+N5yy+enlvhU/NDc8eKuleBpEdlrvEwoVs5RtJ2sn4SblY2YRPpkTQck6mN+15ZxuT+R1TEnteu4lv2zR3LhaLdW6dWu88cYbWLFiBZYsWQJd19G3b18ceuihftcvq3ErZTs1OJZWhIexE/1i1VaMfO4rqetEDTi1GuWfs1biljN7MdOwbbsHNDeGI7JmKfPWCA0nUx3N+0s2AgCemb0KZ/QtNpy3550XM6vQWVV48YtKnN+/E6duXjQhxh/8dEKtjsQ1sptwOiFzqWzME78Q1SkWsObGr9ytWlRevJPkn/IhFN5dtN7sO2fIS87nxkNbYR5k5+c1AjpPW6Xkc2PJ0+k7Sb0zKY2P5GM0m6l5z0o5Wy6qpjgWUYhhI4O0cOO02/fHH3+c/vvhhx92XaFcQkXCVRkwRaulUqQ2SDN2orV1ibQQwK6D4W/m0gfzz/0Gr1drepYJjN3vsO9F1mTjpJZmbr8Qs0QV5RS1nRG+XbeWz62lsY7sC2QC5Kmcm1u5Hde9OBd3n9/XcoFEJblluVNBB6mWFmUdxPYLZs0Noz4ublYUQE5F22pMv6umDtf+q4JbjsjJn1W2EZnH6vQcPMW5sVzw5eqtqKtP2L4hp/cvMkdb62TdoLchQrG13u6+ESsyZmVfHHVNwq/65TLXRMEsJS3czJs3TyqdX2rbXEC2Ie7bX4/z/zbbcJ04vYxZKjWBVZm1y2zYZ0S0jNCtcCJTvjFvm6OhZUbHege7ajirPayCEuf2vATxk9/iQXSOf/bthVWoqavHhJ/2Sx8LQ3MTJKLvKojtF5zuzs3dyy4FB6yO/skBl3X5fobwYkwno7nxgqNZivN3w/X8Z8JqY89/tkb6m2VhFTBte03p5mef9rmxmn8VBTVeOus1qd+8/tKPpu7GxBQFwUUGaeHmo48+CrIeOYlsI/h46UYsV9htWEa4SQ3yKo3XadC2Dqp5BjWJNT07QjErTzbWsszBt9h/W9MB7CXV8yq34435Dc57DUsuzelYzqm6rrvqEFgIBRhJ4ZDFmi17fFNlS2mmmMe8PySub5Eg6yDMUrwgain8mP0ac51XuZ2bLjXgsopk3btxwJbpN/wet3gDs+xS8hSsdz5z2SbbMbU4N+bfIm2a8bxMnBuZie3e2nqzJlfCLMXDqmWSxc3ExKsfWKYIZktQQglrn+PUMGolzFKpdu60QspUrkAjwqJJnP8xsfpRFZW+sdobqvdh1vLNhnMNJ633Z1VL8+7j7YVVhjoc+J+h4rZFZtWt2hc1lbSsycHsCFqPP729GJ+v3HKgnmJYM1C3yN2fh6mrC0RZ82QbXdfx7bodrrQXxg1w2U7x3m724ilzTJtE3ja9YRsOm9Mrq36p/xnvwazldF9HpwjRMnsssVa8bdlVg4+WbkQioTuYYhmTJUaVVGRbu++M+DdTuNHdfV/PfroKfe56B28aVkhxhXnOQgFzXeXLNj6iVDw0Gd5c8COuef5rXwJjZgLXe0sRzshKxaLOkSWNi3YFT18HNcEmWQ8xM5duwqm9G5x18+J8zU09Q2WipR2K1QbMe95cxD0nUuEny5LHeg9XTWU7Xyv73DA6dVZ51qv27a/HDS/Pw+Ifq7Fu+178c9YqrH7gXEeBIzlQqAmq/FpIpMn0JE1kluKMbq/NW4ex/7cAA7q2xmujhygV5/T8vN6/UbCx5W0rSwegSZlVARcCjc8vk+dnk/r7zEmzsHlXDSZeeATO6deRmw/rPrzq6Kzv1VFzw/C50cHuz2zvzXLk3gOb5742bx03DQtu0FPHK9ksXLdDOu2NL88DAKzbvtcx7YqNu1DatrnLWvkDaW4CRLafUJ2xSJmlNE1pyTjgPEN47kAE5RQinxtpzQ2vLoYTm3bWcM9ZtRRWzY2KZkW2g1DdboA723K45t8VP6B88QapzsRIQmcPJG6QEYwysVrKWMSCHxo65ERCx0LDb55D6StfrgVgNvlIl+3xvBd4mhuRBsd0LEOSZzLot7gdmPuj5JnNu5LfdvniDcLBXXZ3dJW7tfYdVoW4zdTO9LmRrJdExazzwXS/xLmWt4I0aLbsqnVMc/VzX2egJmJIcxMgss1NtVlKmaWgHmqbt/EiD6NwY00t9LmRGXgF2hHjKibbbsG2enDyNxWli+vCr5q6Q7FA62S9ppoTXdepTPsM1EPHJ3N/jDSZ6msnvb8Mj324Iv2bp7nx8gyC1tyIM5ffKJbVnvzS3sqtluIfe+fbKmwTxMWyBta0wtTcMCql8i7sAoj4u0n1p8Z0P2zbi6KmTex5u2hvvHYmFdRRuTT3ZHoBgVtIcxMg0mYpxcbCWhXBQsZ8ZSSh6/hk2SZMr/hBEBG04YTRLMXKywqzM5K43vp8LnniM2459g3+VJ6tnJaHt82B6BpWemFnLtw1Wlyq9bzCfqk2pO7Pffae+dtHK0y/eaulvPibOLeh4J6AXXPDF8RZtchUwDXrZrRWRr0w1/SbNfURazIFkyVTvvI3bBX8fthm1pDKaG5ueHke0wfFjUZJRnDhzgUV3rPXVVbZItyQ5iZA3JqlnK6T2t1bE+9BxcNp23vjh210KOY537HgaTKMyH5A1nTWibuU5kaXT2tMLw9HdS3o8nTwOyEZzY0xTThB/Lx3gMYcZJ8FL/y+X4EXM92vW4tjlp/2BfHhmXOySEYgdtJgsdqBXDlJzY2aQM9sEw6PgLW0O8XUOauFdWT53ADJzYpt1XDxKqzfDGvjTP6Kqsw1TFVtYFiQ5iZQJDU3ig7FMkJL0iylrrlxwlh2nmAjF14Qv2Ubdkp56MvMVgDnpeDyu4uz94ix10uX6mzMebP/Fu/8LaqDGJkVNrJIDZgsLYIP/Z/sJqNGeD43Xqpj1rwxBvAA+3oZs63wmE9ImaVYxyQfDm95ewret2IT/qRKO5DWSViz5JZShAelueBlK7NRr5LmxqMrdpbINuELN5MnT0b37t1RWFiIsrIyzJo1i5t29uzZGDJkCNq2bYumTZuid+/eeOSRRzJYWzXcagKcLmPtuG1F0zTsr1NrhTL1NW6+lxfn+9ywPjYNGoY98gn+9fka+0lbXeTq7nYpuJH3Fm/Asfd/gKoDmwKKSJqlpKrGrIPsBpDi6MXiCiR0SzkuOuOvVm/FS19USg0WQTkUu8mD63PjoUJOPg9ObSwVLdxr2cbyZVbouMFt1Gze+T++vQRnP2rv01lhF9QFenWfG+N5px2ybZqbAxKWzezrk7DDz6fh+KcrNuPCyZ9i2YadvgTxcwOZpSSYNm0axowZg8mTJ2PIkCF44okncPbZZ2Px4sXo2rWrLX3z5s1x/fXX48gjj0Tz5s0xe/ZsXHvttWjevDmuueaaEO5AjFuzlBNyq6XM2yPIINNot+xu8JQXam4Yeb290L7rrczMVMWJ0ibcSD6CTTtrbKuyuGUqDiP82Za7gcSpdKtZyk1f9PN/JP2a/nLxkY5pg+rqxr+2ULmMIDQ3qj5WQPLdprWuHgq3T3z4Qk3QY44o++q9+zF7uT2oHgAsqaq252XJLKZpygK915hDTpMU62lWhOJk3RjXcoRSpfqkBFnD8bveSIbFuPZfFbj8uIYxMpPyhupq0bAIVXPz8MMP4+qrr8bIkSPRp08fTJo0CSUlJZgyZQoz/YABA3DppZfi8MMPR7du3fDLX/4SZ555plDbEyayH5otlcNlMqugNMjtQWUqViL5aX+d2ZBeUFGWWWpDtV144OXx949WYM73mw+k4WMrx+Zz4++HqEN9m4oER68sFtoE6mOHMhMJ81N1egZ/eH0h99zqLbuF1/IiNvuxNNVoApXfFZpzwjefG4Zwwcj65Ic+TjuaetqMUjdHnk1rblxokOTKYx/XOLF1UizfuAv3HIjd4qqc5FpyLtIRzyUega4nY0g5DdLW+01wnr1fPQwvH1Y1t+42L8fOpM9Nlsg24Qk3tbW1qKiowLBhw0zHhw0bhjlz5kjlMW/ePMyZMwcnnXQSN01NTQ2qq6tN/4Jm+55afLR0o7RwoToQSJulFFdLqdbDlNxyqax0zyvy+c/W4LJ/fiFOBIbPjWT+XlDNk/eYxA7FOt+J1qEjs0UodqjfC59X8ushoeZXDaXvBtlnzjNL+VU2qxqsY2u27EkHaPPSBnWwdwyXrYeb8tjH/R0+mWYpQXr2UnCGhkSi7OtfnofD737XpIVm19GMkuZGoh72i9hX+bUxZwpaLRUwmzdvRn19PYqLi03Hi4uLsX692OG0S5cu2LRpE+rq6nDPPfdg5MiR3LQTJ07Evffe60udZbloyhx8v2k3LhzQWSq9qK2w2qHc3lLe4tzIoNoZMfPw+J04maX8ntFYHY+l8uckET0jJ62OiHpbhGJxehFO1yZ0cdh8v5AVlmXMUt+u24F+nYvky3bpc5OahHh5PCrO4UEGcksKsf7mZ8Q5zo1/E8W3v6lyTAPAtO0GwN5+AeCYCm3aHed62XcbZ+fFQuXVeH2P2SLchO5QbF0NJLMB2KxZs/D111/jH//4ByZNmoSXX36Zm3bcuHHYsWNH+t/atWt9qbeI7zclVfn//cbuY8JCdQCWbVxu4tyoIBrknZz1eNepYo9QbD4fhApVdVdwXnrVpa/p6yTKM2mIvJhkHErTeWl8fu6y2fGEG+PzPO9vs5XKNr8L+RtLfX9+LUM3/pY1j3ktL33ce9ZCeBGO0+WzNDc+bWLKLZPTp/GcvEXXyiATxC+F6qpQY7/otc+Vn7iGKwSFprlp164d4vG4TUuzceNGmzbHSvfu3QEARxxxBDZs2IB77rkHl156KTNtQUEBCgoK/Km0Im4dip0an1SYG0Ocm7yYJqXF8bMt+ul0pqIh0jQNMxZW4aPvNuKPP+sXiM+NbJaJhI5RL1Rgd23DCjP51VKCcxKaG5X0QiQ0N6z26L/GTC4dP86N+7Idfaw4eacj2rovmumPN7dyG/4rseGin+i6v++U6VCsqLkJYhNTU16WrBJp4Ub9WjflqeTllCQe05A4MB5kSnOT0AHB3sqBE5rmJj8/H2VlZSgvLzcdLy8vx+DBg6Xz0XUdNTVyq1yiimpbk/uAtXScmwLJpaiqkrZIgyGtufH6oVl6Gk0DRr84F/+u+AGvfLnWd82NrsvFnQCSGyK+t3iDMC/+SdEpB+HXYkIIcuNMvwc9HrL3wHPCdvYd0jH54xX48DvW+xILpLys02YpD4+HtSXAhZPn4NlPV5uOAT45FHPuJhnfyXP2/HKc4txwhBueZotbrsI9WJOmrrVPHhhaNEZ+qzbvxq+nfoWKNVsly+NXVtW30KjR9Poa5SftjVRzAwBjx47FiBEjMHDgQAwaNAhPPvkkKisrMWrUKABJk9K6devw/PPPAwAef/xxdO3aFb179waQjHvz0EMP4YYbbgjtHkTIOm6ptgGZATupuTkg3DSJm7QHPGQFkhS+2Mh9Lsf4EW/bU4s2zfOl6qGC7GDO8o2SFTp06NxQW3ICG1/wVME5Ki1HLe+7UCmZobsFZvh0xRY8+M5SAMDqB841nXPS3PDqphpEk4XM4N1gqvJcnHQ9/M4vqbnhFyL7KJ3qWatgqpeNni71bHTgty9U4Lv1O/Hhdxs9lZfC5L7hUAeTo32GhI6wV1WFKtwMHz4cW7ZswYQJE1BVVYV+/fphxowZKC0tBQBUVVWhsrJhJUcikcC4ceOwatUq5OXloUePHnjggQdw7bXXhnULQqTUibqu9LFs2lmD/1T84JivhgazVKGk5kbZoVigwZA2S0mpXPmJrLMo49iWnxcLICaDfAwZlnB7638WSF0r7MxVhVAPczWnohI6bym46yLZ9ZBMx18JLs7hxx38nddF+5wB/LrV1ctFvRYh41CsC84pI9IY+qq5MePkUMzeW0p+r7oUeyQmeU552YP4Ma7V7XVet43fxlj5GvOyYvNVdbjzuI+aG1nCdjwOfW+p0aNHY/To0cxzU6dONf2+4YYbIqulccul//wc5x7ZiXveOkD+eupX0nmnlqIXNIlLpVcVBESpZSdIXpc5Wqts1Nzkx2PBOxQL6s9ybv10xRapa0XVVtV2eXkGTpfqUqm8I22WcrnOVXSVW1PHhup9ePDdpa7qk6Iu4ewcLnIy9gtd93cxuLWuTlsCMNswy+fG4RnsqbVvcsmDl5V14sF+LjrmVm4zHYk5hCngblbMOKZsljKUnSmhI2TZJnzhJpeR6Wc/X7kVJxzaznRM1CYWrtshXbaqz43qhmiivX/qJfXIfji3GYUyo0Npfl5MKiaQCjwzDAveyp0U4r2lBIKPlLZLLi/HfNxqblyXyCtHLh13Y0GH60VCkVOEYl7Wr8+XWy0pYue+Ovzm+a8dywL8MQPwsghaExeLAZVb93DT84L42TebFKOkueHctC2OFKfQi6Z8ZqpXnoNww4tqLCOMOKUIwSoVuuYm9KXghHxjm/Lx99J5atDSZilp4UbZ3MHHT42Jk8+Nud5mzU0Q35fsUnAv8eSEmhvFm/KmuRFfzFK/J4/7++ClN0DlHLd2tHMrt+GcR2fh85VbOFcYy2b/rVo3P2CW75Nhasfe/XhvETvGmM4p2y3WvD77fgsuf+oLbnquVkMX/7ay1wezlEzgQHu9dMcAk6qCpUr3Yiy7sZilSLjxGaMnvOy7lR18/vzOd9L10LSGOBuyG/cp+6cYO31OTAiFLFyR0M0aJ2PHk58XC2YpuGRaJxOJ0KFYF0UodsY8IDtfsZ6zaaizSYatufEb6dfISLdpZw2Wb9xlOjb8ic+wuKoav3jyc8csncyQmezCRTGFvDb1K57+Aqu3sLUnuh7sffLKTbF5l31FrMaMahy8Wcpmcpf0w3IUbqwCkSC3Lbtr8fhHKwzXiu+btYVH0IRslSLhxm+Mqkjp1VIBNYP9B0YdWeFGXXPDTy+//YI3lWsiYXbINu5HlJ8XC8A8oktHOHXS3IgdKN2dS6cx3LnMqzh+4gf4YIl9GbTUailm+f4i2zRZbfJBxqRgv8K+a6ayM3GzIviyjedqLPiBb/JOLgX380a955XcP88saTibUeXz/83zXzOFfiknb4aJyUm44S0s4dXZuH2E020ZS87UPlTWCM+ZhoSbAHFykkshEzdBlYSuY/8Bf5P8uKzPjVoZInW9rP+OnBZCIETpuqks41LPJnH/NTdWwUYYi8JBunVyKFZxMBSXIwfL7Ol0bXL7BZZZSrJQSeTD79uPVe/b73id6E252RU8KESblAZtBvBVtPEhM2Og0nS+3rNNs3DdDvz+NfumsvbVUsFobpyOq6TRXPrcrN26B89+ukrJnJcibLMUORQHiKyEbI0ePLdyOxIJ3dG7XkRCNzoUZ361VKb2lqpPmIWb/QYHYs2H/K3MX7vd9NuLz43T3lI8AVFK26XzfvBhRbGWmQmz0/hvDpRKx0goO8ngwYpzo+s6Xp27Dh2LCjM2EwY4ZjGfzFLCcrnv2WV+PuVjjSVkreO/PluNg1sWus6/iqW5sXwnsn2ds8+N++99yAMf4tM7TsXBLdnR+E1B/BRe5FmTPsHu2nqs3Spexs4ibOGGNDcRgDWI/d/X3vbA0vWGWY20z42qWcrY6bvMy2vztwoBRs2NDn+3gQCS/huyOPur8M9ZV4GpYsxbNhv2DNTBLAV7nKYgkG9P7uoiUrKx9lB7de46/O7fC3D5019kdMkr26GYf863cv3Ozw/NDViaG/PvO99YhFEvVAjz4QkEqTKsWC2arO+UdX/G1VLMrSMs1zREnuZWL01tfQJPzVrJPW8ySyk8+1Tw1znfb5a/6ABhB/Ej4SZA3JqlAGDWCvXGZEQ3mGuayJqlfPS5kV5W7lDm2q17hEnqdfNqqZo6wz5Ouv8f2Lrt5hmMWHvlVLg7s5SqMCG/0ao9ndPz483o/R5opX1uWJobic9QKNww8k9tiptsY5nU3DCOpTQ3QWqQfPYo9st/xxoF3E22IoUKWwhxNkvZroFZe8IKE8E1S0k+eJEZXPMYxM9N/Kiwt18g4SZAZEN9q8aXkSGh62kzQ36eXMNU1RT85d2l3LrLdvgiJ0YAGPrgR8LzVp8bs23Yf63Czn3yqy2cHqfYLGVd4m4851y2aYNOL5obGeEmAyuIvGgCXcb1ayjbuBrvwP9GPzaV2Cle8Rr/yHW58Fd48iMnTdNsArmbfEXxqFin5CIU2w/mxY3CjXO+DZlxq2dCpKE3xv9y0ye6+YRCVtyQcBMF/NiDxoqOhkB68pobtTK279mP1+etO1Cg+eIVlqW3XhCuyrKYpfbtN5il9OBnD+JVTQ4mHYfzLGFz084aKW3U7/6vYZsH2c7MjZDNC+LnN540NxJdsyiN8f6uevZLfPPDdjQxDCSzlnvTsqrANkvp3HN+luurz41PZim7z03wA7fM4glWLYxbILA0ITaz1IHfst9XvmALbmP7dvPs3UwQwva5IYfiCCAaVNzOOhMJPb36SVa4cePjsXYbOz7Fd+t3KufFQ+ibktBNkX737TeYpRC83ddJ8BIhOm8PTpjkmD+9L1Uv4/OX7WPY7dBBQOMk8d0sFUBK6RwNN5PQgQse/xRn9evgezluyYRZyt/NFyC9tJy187cRlSX9LOIxVqwcQ/kM0cf6mTDvgyGoGB2K4yzhhudQLPnkWZqbVN2MmiJXmhtXwo36NX5Cwk0EYPk6eNSkmzQ3eXHNsZMA3DX6lEAUZDsW5Z00vzVIN3v3W31u1GtWkBdDjQ/bNnjR3Oi6f87Qsp1jva5j864ajH5xrqkeIqxxhlTLlMXLUnAZZH1uknVhf7OZQKwpDLZcX7WgupymMC+mcQUYlvCv+r07rWi0tosde/fLmaUYeeUZbENssxS7DrK3ZJ3E6rqOy/75xYH+v6FA1qpIJ9ysOPR/02I1yCwVAVgf+Zote/DOt1WuOyxdb2jEeTHNcZ8jXj0crwlZ9VgvCOLn1mQi86xSiFc8OVzrkK9fal3ZbOrqdTz07lJ8uaohyrYf+9r4QZjOiaxnEJbKnb0UPNwJhtv8ZPoOo0BgxU3oAitO37r1bP9738MPlt292b5q9mPGW2E7FOvM37Jtzaq52bizBp+t3IJZyzebfAUzJXSEPDSQcBMFWB/pwnU7MOqFuVj0Y7WrPI2OtnFNk9rnyJVwc0DBEVZD1nVz8EHjhp063GkQVK4RCyhOmhtxvqpBFXmo+Nzs2GsOeCej7XPjiKyKbH4s516vWiTWZxHEIgAZWM/huc/WQOe8BxkSCR2XOmxD4fNiKWkfnjyBH0k9Q6Oj+gjiMU18DUMIefsb84aorOe+m9EOjYIa21GZ/Vv2luyam4a/jf3/K1+phxnJRp8bEm4iwJsLvO8ebEU3qH3jsZjUUj43WphMzKhFZdRbzFLWfabcVM8v/27nZdT8BDyhwQ3SmpuE/Xk5XcoL4ud3q5B9Fp8xNsJUfZ8yS33dqPb9gFfqnO+3uBYoV27exXxu5nLlv6VD27dAt7bNHPOTNUvxYC3EUH0EqpobwO4M/PKXzsKCDnNQVlaAVvuGnGrSjXWDZGM1va4YdNO2QlbckHCTqxg1N3lxOc2Nm/46VUaQQo4oZ+tScGuH6UYF6zSQdm7dFFcMKj1QOR1bdtXgu/V2DZtTPsKzkj4JMsjm4m4puN97DrGxPoohh7b1LW/jBoQA454ZtxeaWYpTbjK4pLs61dZJXKfLa8BiGnByr/bi7CTNrnmCxRBss5TaM4hp4vtiCQVuAsfrOmBUQjHNUoxrAPm2Jlo44j1Kt70OnYrEkZ9Jc0MEgo6Gjz8u6XPjRhDIhM/Nyk27ued0Hag1OP/WmTQ37gQ2p4/StE8LgLI/vo+zJs1Szke4Kzj4cW5UkQ/il7B19E4DWoJjrvBb4LFmV9yyEK2bNfElb2u8Jpm9g8IySzlt2eEGmXtZs2UP80WPPKG77VhM06Q0BUbFC09DI9oXj+XUray5cZBUZDQ3ssQNZinW92Htf/V0Wrn8RbUKQnMjEjyT15BwQwSAMUJx0ucmGIfiTKyWElGf0E3BEs2bHDaYdrq3a47fDLV3xCycHoOmya1mc3omxpg8tmt1/zoHWeGGHYxMfM1H323EN4xAjDe9Mh9vzF8nVa4MtmehAc2ayO2ZJqNx2GsJIWCE9VwCCE0liUggdoeMEP3OovVYusEe3qF/SWvbsXhMc9QU6JZyeb41LQv5C3qZ8cEUHwJrSbYRliDjVk4wCnDM2DhWk7Cio7g1ncb5WxbztiN2RP5QQPhLwUm4yVGMwe3iMbmZlBtNQVi+BykSus7V3CQSDR/oeUd2xM8HlvhSZnJmmnygoke2Y4/zbtQ8rMEJvSD7WutZPjcOF/9pxhLuuZtemS9XsAT2jltDYb6ccCODMbK1jOYmiMCbMghX57lsL3WSnusffrfRdqwJY4CT6W+sDtA8IUM0gLI1N2rPQNPEDsVszY1SEWnicQfhxvpb0SwVpBM/qx9oIljJBpBZiggIo+Ym6XPj/EW60RSkGnBY7TiRMAs3pg8SDbMHTXLFmAwy2eyuqcOYafNdl5F0uHR9uTkvFeHGcmzL7lp/KuERa0epaUCzgIQbu4BnTx9SmBvx6jyXeYo0iKb8LQXkxTSmZkOqr4FZGONdI9Ks+LEUXDK+qQnXZinDdayJpH0puPl/J2xfrylwn1weRpwmu0bB87cn97CdD0+7mYSEmxwloTfMLpM+N87XuFsKHq50Xq/r3CBfxtlh8v79kW7Mm9Cxyxb5Cckg63Apl5dcPqzOLJNbC4hgWKXQVNIsJcOe/SLhxtk/IlOIXqXb5mKM6i0u21wAz4E1aZZyysw84PJ8XzRNw0sjj2Oey8hqKcZpt72IMS/WwG9fCp72upHKX+QI7ypAq9Es5eBzw3IuJs0NERhGn5u4hHTjZjaqGovBb/btT6C2nt05GzU3sk6OMhiz4X2/Xsv6cfvejK+WCltQFcHqKAtlfW4kbsukubE8MdZjCW8puMjnxl2d9tVJCjeW303ibCEmLvGtWR3meQJ4PKZh8KHtmOeYZinFRxDT1LdfUAnymSI50XJIY3PmTyLb1ASyjcvo88a8WGYp8Vrz8/42W7lMPyHhJodJOUnGOepjK66WTYc8IO6prTeZpUzoDZ1mTHPXKbHQtIZvOai7n/zx976tlqrasc+XfMLEprnRxCtpVNlTa4jgaimLGeY/hzQ3NZJmKesAydPcxGLOphtdNz9DXt1FczKmWUrxi3RwG2GqaVzv92dZ7GA/b/6tbJbimLVYecvgrLkxbASqnn3gkHDjI0/PXhV2FUx8uiIZmEs2zk02br+wp7YOtTyzlCHwmKZ5jfTQQDKvZG5BajvCWErpd5F+CQE2nxto0hoHGfYIHIpvfHmeLX1Y7T4Qs5Ss5saSf5M4OziojJbYanblaRZEExLWpEbZ58ZNED+1Ihow1I1ZT4451K1GTub5yl7Putoo3PqlFfcTEm585L63FoddBSbxWEwuzo2LDyATQfxEiDQ3CUMHqmKW+tmAzsLzxmy4WiMfCMdM5G+Z5/qkmmZpbvYyQtwzr5VIY8xrT41zvuHFuWGXqzkEoxMh7VBs+d0kjz1hiElMJHSYnyFPWBQJSkzhxqFcKzGn1VJMzY0LsxSchQXru1U2S1mFI8Pf7sJ8GPO2X29c2u7f1NE/SLhpBAQa5yYKmhuOgDHu1YXpvZKSsWmcn8HVJ3QXxtYAzIJSrV9LmhgElXUmZ1lLqtztjWbFOnBrGnsfKea1Mj43BqfaoQ9+6Jg+LOFmbuU27rngHYrNv0UOxU6fmtUHhfc4hZob1gei+BCcg/j597EY+0q5peBqE0ebz46DWckJJ2Es7rBXVtiIe3EiJwgyzk29ZSl2ptlTW4/9AingvcUbACRt9zLPIKY5dwTGIH68GCF+fOxBCY4a+DPckGVVLvaxQDMJJF4xbnS4v17Ht+t2oF/nIm76sISbu95YxD3ntkY1LldL5cdjzHYe1+SC+Jl9OjhmKYHwsd8XzY34POv+3GqpnZqMPb6SWv7Gy//1+RrMr9ye/u3Z5YBxeX4e+dwQIZMnuf2Cm292zootOOwP/8MTn6yUSn/l4G7qhQjYUyNwKDYga5aKaZqSkMdbhu4HQQ2gbuN0hIl1QNE0eY2DDNYB3mmlR9gaSyvLN+xCvcvAIjWSkxPrPfMC7DlpQwC7zw2vrYuC4O5nLQVXfC3JfpF/EetTcbNSTtedhSKeWclNEL87X/8W0+f+kP7tZuLqpLnJc/TGDhfS3DQC4tIbZ6p/ADtrkqtMkhv3OePXiqUUe/bXCTU3KZwikRrTOUVs1QyCkkzZbglSc8MjWkN2A6xH8dDP+2PE0186Xsva1NSK6rMOOzK3lb9/tAJtmue7ulZWuFm9eY/pdxOR5sbhM//TjMXo16lBM+bGLMXewkDRLOXoUGw/72bSkdwKRvGatFlKtgznvNTKF19v8rmJ4Hwp2qIX4QtB+tyo4leU4BR7auqlOmfZcmOa88CVNHElM+RpjfyQS4Iy94naQtib3fGwr5YChvY8GG9cN8Tx2h+27XVMoyqshB0CgcVWl9GkZTVgX67eavrtJYjf2q178b9v1zuWKTRLMbSmqq9FZmWXFTdaEBnNDesa4//O6fkJXUUoNi7VZ5w3LwWPnnRDwk0jIE/S5yYT45qMylqFPbX1Uk69TqsijOmchDzjs+RpbvzQugSmFYpeP+SI9ZWk3kFTn7ZgYAWEExF2CAQ/cStE58dj7CB3Pn7jqlm58blRXS3lTnPjwofmwN3IXpZKxxJyvC4WYT0jk3AbwT6FhJtGQDwmGaE4I5obf7+CvftlfW7klsrGNOeBzuguyVtG+/vXFjqW5URQpg9RUwhjyGZtvmjFWq/UG/BrHFVt+/VhbS4VADUuhegde/czBzVZ530ZVDUrymYpFw3IjdbO6mMkew2gcF06vVIxXBwjFBvj3PhTpK+QcNMIkN04MxNOkn6bpQBIrZpR8rlxcMw0PsrPVm5hpvl2nfcl0EGZpUQq5DAUEjKOiSyH4uT//jQo1V2+c0lzw1p1JMO67Wxzn8xqKVn8ngyp5s9qX+4cinXlbyudXlq2Sfno+NM2nTQ3cZPPTfTEGxJuGgHJlULhCDedWze11cVv9knEO3HaQ8aYztks5RzHww+CiqEThIDphTzJ1TVGNMv/XuFp63gDhWyMnWzAbTvbU1vHDuInaQaXQbW/cBOhWHQJq3Q3/aQbzU0qvWxwRt1nzU29g3BjDuIXPUi4aQTkxWKBbb/gRLP8OEaf3CP9OzzNjdyMRsahWENmHOiC8rkROhQHUqIY3pJiIzaHYi1llvJLc8MTbnzJPtK41RDyTLJOWxqokNIOnN2vAwDg4JYFwvSqUZqdquqfz40Lzc2B/1U3zvRrkmoOAmjPM+awWirsSRQJN42AuGScmyBcPBK6bmr4Qagv53zPNg0ZSfrcOBOLac4+N4q3cFhxCzSV3MHayNqtzqt8XBGxaVaexAaYvP7aL+GGN2BFLZ5NEHgRolnfc0ywWkp1wEu930eGH4Xpvx2EsWccJkyvrLlxjFBsx00/6cXnRn61lGKlDLA+I+N9srJ22BQ8cJOiEyTcNAJkN84MYnlrQjdrOdwsvfQDTdNQclAziXTO/hca1AQcXQ/eR6N7u+bSaUWdzifLNvlRHSWaSLQJ0Z5KfsAb4HNftPHftyseA/fFqH7/qeSFTeIoK23juO+b6vuKxzShRtevyZgOdeFjSVU1Tn3oY/x3wY+SZSQLcCOQN2H4vZmWgrN8bhyejd8rY1Uh4cYnnAK/hUmYPjetmjYx9XNhtfeYpiE/L4b/jBokTBeX8LmR2RjQSpTix0TN9y8us1qKsxTcrw401zU3R3Yp4p6TDeLHgvX044LvQ3U2bxWGCpvEcf/PjuCmV31dTv2iX5+KG83Nuu17sXLzbu6iBVYZgDvNEkvoNEcodjBLsUICkFkqNwhyA0WvJLdfcE4XhFnq4Uv6m5p9WF71xhmgOJ3muKWC6i24iXGhikqVwlYXW2HNGq0s3bDT9NvvpeC57nNTVnoQ95xbs9TUq45hHhcJnKptj5WX9dDPy7rg52VdAMhFozYiIVf7QjJCcbCNSRTnxgmWU795V3D7NcZ3yYtUHSYk3PhEDce5LgrI+tz4bTpp3awJehzcwtTywxpYU+U6L/10dhjUoLYaJKHrwccQUqhPEG+gdbMmrq+VcSi2ktbcBNyeckVzky/wa3K7P9oJh7ZT3n5BVRhlDbrWvI0bA78xX86E01AfuVWUXtldU4e5ho0sA0FPmaXUL2VpT513BXcwS5Fwkxt4Ue0GTZN4THLjzGA+c2PJEr6jwdQhbcYQp4tpmm0Wf/GBWaExLyXDVMTGxyD6nPOP7OT62riLDfj8XgrOI4K7LLhCNBC51dzEOPFsdPC/D1UzIqvfsuaddGB21xIy5Rfy7qINgZfhu+bGYSm4WXPjLIRmGhJufKKoaROce2THsKvBJC8up2kIbhfqhr/D1tw4dYIxDbbdlUcO7W767cYsFTRqVfL/HXhxFJeJUGzF7yB+PHJFcyN6P24dinmCQX1CF2huvPncAAzNjcRGnTyctl/IJlRXVxlhBdI0t32Gz41xtRQjz0bvUDx58mR0794dhYWFKCsrw6xZs7hpX331VZxxxhk4+OCD0apVKwwaNAjvvvtuBmvLp2l+HGcd3iHsajBpEo9JDT5+yzasjyw8nxs5H41YzK65saJqlooaQfQ5XoQbN9dqku/TK3pICtneHVr6mp/ou/OyWoqVrUhz4Ha1lLlM80GjWUoVa32m/3awpSx3+YaBnjZLqXfkbIdiY972a5x8bhq1WWratGkYM2YMxo8fj3nz5mHo0KE4++yzUVlZyUz/ySef4IwzzsCMGTNQUVGBU045Beeffz7mzZuX4ZqzCftl8siXNEsFhVFbEt5qqQN1kQi3LrOJosptZGKllIrQGERT8KS58WCWIp8bOUSvx+/FEMnwD7x6qGpu7G3Dei/JtueuHVi/m7LSg1Da1jlkRBRRDfpnhPX9Ou0KTqulBDz88MO4+uqrMXLkSPTp0weTJk1CSUkJpkyZwkw/adIk3HbbbTjmmGPQs2dP3H///ejZsyf++9//csuoqalBdXW16V9QhP0yeTiZpY7ozF8m6gcqZqmipu4dU2Xq4Ki50YBju7fxteyo+W0EIRB4EW7cOBSnyFXhxm8Np+g5eRFuWLlaA3ea66GWP8tHT+RQrEpcs8e5yVZ5Nm2WcmEId1wKzngoxs+W9fzD3m8qNOGmtrYWFRUVGDZsmOn4sGHDMGfOHKk8EokEdu7ciTZt+IPRxIkTUVRUlP5XUlLiqd4iwn6ZPPIcVktdeLQ4MBYALJlwlnK5ab8IwzEnO2zLwjzlcuTqkjJjOHv433V+X1x3Sg9hOpXe1E1no4pKywuilXpZ9ikTodhGqm0F3IOFJZj6PVESCZ9+D+YJnd8XqgrBrHZl/YZV407l5zU0GlZ1MvG9BkGDQ7H6tcwIxaZdwe049eWNdin45s2bUV9fj+LiYtPx4uJirF+/XiqPv/71r9i9ezcuueQSbppx48Zhx44d6X9r1671VG8Rbt/llMuPDtS2q2niODcys9+CPPdNRSWIX6vCYDQ3skvBY1qyDtef0pObRrXzy9aZoApenAdlNs600hDnJtgO1Old/8bibA4Az1w50HO5ft9WYI9J0efGjzg3VuIxtfszaodZ+esOviZRxYvPDU8D15C3/bzJ54ZxfdiWjGCmyQpYJXxd16U0IC+//DLuuecevPHGG2jfvj03XUFBAQoKxJut+YWbjvby47ri7COCX2UlqptMB+KlczS+T6dnFJTmpsHnRpwuVVdROl3gU+CVbm2bYcuuWuysqVO6TuX9BKFh9KS5ceVQfOB/16XK4TROFOTZg0K2be69v/F7Y9ZM+tyJBtcgNDdxzpJ0Hq0K87BpZ006L2ttjdXPRp8rd5obdbOUk0Nx2JaM0DQ37dq1Qzwet2lpNm7caNPmWJk2bRquvvpq/N///R9OP/30IKuphJtXmalOR9TQWB3ItScdIn09D7a0L76mVUA+N7LPWSXYnywqnc3vhvVyJUiqdO7rtvu/IacXv5kmLsxSUXEoZgax82HK6vdtBTWLZrW7+oS5/sZNY5U3zpQK4hdTel4tDNphVt9nHMhdxjcMhYbtF9QrzXovzkH8jL8YQmjIa7FDKz4/Px9lZWUoLy83HS8vL8fgwYM5VyU1NldeeSVeeuklnHvuuUFXUwk3LzNTG0mKirGOLaVtm2Hc2X18K1vFoTg4n5vk/7LffZh7YIUdH8INmXYobohz47pYR3Rdd/S5YQn9XgS9dL6eczATlBDIWwpuFHqMz0O1bbPalU1zE1N7XgWGDo/VZ5uXQGePdJMyobrS3DCeoCncF22/oMbYsWPx1FNP4ZlnnsGSJUtw8803o7KyEqNGjQKQ9Je54oor0ulffvllXHHFFfjrX/+K448/HuvXr8f69euxY8eOsG7BhBvtRqbev9AsZTnHS+m2rual4OJMgva56dq2GTq3bspNl9oZXVRPUQRWZp4KvY2muRuInHYyDxovHZmrODcZ8LnR9Yb2wK+HHVZANBFDe7az55vB1VJ+Y23vRs2cajthbr9g+R2LyW0MnM4zbumPLK/Y6GeVTWap+2d8h2v/9TVq6uqVr2U9vnoHzY2zz00jFm6GDx+OSZMmYcKECTjqqKPwySefYMaMGSgtLQUAVFVVmWLePPHEE6irq8N1112Hjh07pv/ddNNNYd2CCTcvM3VN0N+QqL+1CTec+/CjqcYcnP8yYZa66/y+3HSpAH7OvjnyZau825jmTmv0/abdzONHlbRWz8wFXrRNbuLcpMsNsP9M6Lrju2O1A1kfoh4HN8f7Y0/Eyb3sPoPZY5ayk1wt1fC7iVWYUIC5/QLL50YhW6OwxcrfKM8Gviecz7y7aAOemLnSMd34c5w180bBniXkGT9b1pgRtgY6dIfi0aNHY/To0cxzU6dONf3++OOPg6+QB2Tf5dUndMfTs1cB8EeFLYPQ58ZScb7mRnMlhVnNUhr4WxK0CtihGABaFvDLSG29YHxezfMz+ZmozUKjghun4PS1HsxSefEYbjqtJ95bvAFLqvyNYZXQJXxuGF+LrCaqIC+OQ9u3xOzlmxn5+ksmB5qErpvqb9Rkqe8tZT9m97lRcyg2aZIcVktlmWwDAHhzgfPmoVbzP6vP2W8M4uditdRNp/FXnGaC0LdfyCVkHSON6bwMCiqI1MG2DoeT1G1NjdfFNHHMHS8+N/f99HCpSrQQlGHceuH+nx2BO87ujZI2loiliqulVJaOu9Xc3HpmL+bxjJk9M74UvIGbzzgM9/5E8O5dktB1R+GGVXWWsCYSqOOMfsNvATcogZmVry7Q3Kj6jrOejd3nRlVzY/YVsb/hhiNrtrA1olFGZhNn61jFenz7ahvMW6w+zCgYsp7/+f3db6brByTc+Mgx3dpg0CFt0adjK2E6Y7tysyOyG4QOxUH73Fg0NyLhprlgEHCiqUDDYiyzmSCdUQ192XFdMeokezA/HeYIrG2a5wvrpWaWEj8fHqNP7oG5d55h03xlar6e+SB+VlOq6+K56LrzzF12tVTH1oWYfPnReHHkcfb0LNOLdC3lyKRzp8jnRnn7BYlnk9TcyOPkA2Sc4GyorlHIOXuwCuCs13Lb9G/SfztpbqxkaqGMiNDNUrlEPKbh5WuOx9zKbbhwMj/KsvGDapKx1VKihmj+zfe5ERmU+Nj2lhLcshdNVr4g0KDx/kVlOG2aycKpxipZunUo1jQNbZrnh2bndiOfpHBllnL47QdJnxunpeAMXwOOH8c5lnhWqWSsZ+e3M2ZgPjeMfBOW1VJG4V9Vg8R8NpZjMS8+Nw57KuUqNs2Nw/NzdCiWU/5nFNLcBIBTx2TU1sQj4HNjPcdN6YPmRtPEsywvmiyR0GI8JZpVyHZsmuDDtqNilvK247ht0M/QjN3LYOwliB/vtx8kpJaCM44x0ol3ymZtoCQuV5XMrpYyP5eORYXpv1W7O6ZDseXhxBVXS+XnGSdb9r2lGoNwY/3mHHVfDjHL/A466Qck3ASAU19tFJqD8LlhfedKZimffW6s9RD1Q540AIKbNEVJFmluJKJ2WccpXwUIy/O56OguSpfLLuv3E7faphSuNFUZ0N0kdGd/KfZuyGzNDQ/m5pCOtVND04Bbhh3mc67mepa0SYZY+PWQ7qY0HQ2hF9R3BXeWHuMOEyYrJgdnxoVutLdGfnl8V0/XZwKr5sZpGGL5nsXN0k3kIOEmAJzsjcbBVTUmhgysDkTFPspL697npuHCuMMmnsZzqvtZicwbxlsUCUH1Eg4y1hROj8WLz835/RtMGXedx1/Cnq5LCJ1MTNM8rfpzc2Um7lPXdTiFD2JPJNRMHezlzo7VUyKmabj+1J644+zevuZrrOcTvxyIL8efhkE92prSdGjVoLlRFRtkgvjFYppSI3JaLeVVc3PLsF74eZnapCTT2BeRqDc4Yx52jbGLSvkMCTcB4CSwGDUlQSwFZ+UojHMjqT1yq3o0XqU59EPGZyfj9Z/ipd8cJzRpGTtEkXBVLxkMT+XjVekqYxYtiNGPSMZJz2ZizEAnE9O8xSfyQ/MVjFlKZim43EHRgMl6r36r+VNlBNkc8uIa2rdMCjLGd9quRcNeW83y7XtxiWCbpSzlqi4Ft5qlLOe9CjfxmDdhPxPYzVJinH1uone/JNwEgJNpxdiZBeFVzvQDEGlubAOiP5ob3mZrorrEYsCx3doAAH5xTIlUOXkxDYN7tDMt8WSVm0L0zF05FDuZq1UiFFu66XxDY0qaf5yut+cXNJqm4aBm4hVj4uvFv5nXOPz2A5kgfmwtKTsvHszNIX3umVNF+D8GGQUF1tFkf/jWDSdg2jXHo7ViO2FpWVmaGyWHYpNZyv+WE4Q23m9sz9BFHxZxqxQJN0Hg5BRrbFheorPyYPsBiOpjvZ6Xr8v6GC50WraZF4vhXyOPxftjT8RZ/TpI5Z96nsYN+oR1EHRolx3rbC+37p3jJECoam6M6Y2aGw3OnXEYE6i4puGgZu41NzFNM5ncZAYcu0Dk/43LxLmRnUiwtAENAkfwmhvZDWG9YZzJG45qGvp1LsJxh7RVviuWVnl/vVm7qupzYzZLKVZIgmSXHsXhvgFb4FaHdsHcOFO4Wir8+yfhJgCcnISNDSGIpbvG/P9+2YBkOUJtiZwpw+0AYprFOawGiscaIrdKl3cgmSh+jXGQYsmTnVs3xaJ7z0Rp2+ZyRWrsv1koBXXWzHU1LW/X2P5Kxi0WbOczZJY6yCHWjwgNwHkG3yKZT0J6hZ8HdAez1LmWpd0plDU3EjtfeyUTG2fy3pvTBotGrhzcDf1F7RnAtj21pt9xhy1drBjNUpqm+b71TTZobkRCHWv8Ysa5MfnchC/MWIn+W8hCnExNxrNBr5Y678hklEgVh2K/V0tZVyoJzVIm3xi5/I/pdhAAseam1uC/w3o/8ZjmOoCgUzWLWxUIz19+XIO2KGbpbI2zTA3md3Prmb3wh3P74KlfDeTWxXpNEMQ0TRiB14mkH5Z5wHGTh984LQWfeNER0s77Yp8bV9VTItXkg/SN4IXj5/3NoqRNM7xx3ZD0b9az2brbLNwkt3SRv698i0OxSgRxGSIQv84Rq3XBWGfZCbdQaI3AMyDhJgCcNTfmwd5vZJen8s5xOwrFqqbu02aWEuRjnPUY68HbluHakw7BpOFJ7VRhPr851xpU2V5XpyR3BTdeK7548uVlwvO/OMYs3Jg0Nzafm4ay2jTPx8ihh5gcNlkOxUFHp9U0b4Om9Vo5zY3lt0PjPOmwg1Wrha9Wb+P6S53Wuz1aFTaRbjeqq6X8JtXP+F2S+Tsw/m3s49jHnfID2Cb+c4/saEnjZeNM+etk0Ry001HAHpVefVKZCaHcCxGvXnbiqLkxDvYBfAWsLIVxbgLS3KQGBuN11tVAVox9mbFavzimJB1Hw3h+3Nl9cHDL5ODuRXOjMsBY984RcUTnIhzavoUwTZ51rxvDOGhdDu/kxMd89wFPJVP533y6uzgqSeHIkJ+Mzw3k2iyQdFB/8OIjlet148vzuJobkbCgHucmA8INY6IRVBnWckwBLx3zSP5f2CTZ7o/sXGRL07GoKc7oW5z+3SQeU+qbjN+UVVPqF05b8ISNTWY0PEBZs5rKew0DEm4CwKlxGBtCIKulGMdEA5y0Q7HbntFkanKKUMwevZkb9Fl+FwqEG6M/DnO/Gg+vQfS699TWOV5v1PRZ34VxlqnrFjs3o84sjUbQmpvUoHbT6T0xtGc719fzfrNQuiXNfedbxwkNIDLzsOrGdCg+UCvW+/F7wE3XVyKtdRIhQqZP0Lg/+PnNvfMMzL3zDK4vl9GBvUk8ptQ3GfuJoLRml3JWev7jl2ItbqawjlHWCagMpnYrOYZkEhJuAkBlS4UghBtVs4vNLMVJ7Fq2MfztFCrdPNCLB3LrAMDblf3qE7qn/XIAtqCnemuyq6X27XeOm2PeONJsljJqdXRdd4wtwXr3XtvY3ef3lRZa3AzKGvjmDek8nK5x+Qgu++cXzOOpZ8p6tEyfG8GDYQvufvuBpDQ3zg/imNI20vmaBkVOUDezL46cVrtZfp5wQ1rjN5Ofp2YGSmmFgOC0ZnnxGPOb6duxFdp6cL73C/t+gsZzcs/EtCt4JMQZMyTcBICjk7ChJVkbWb4fhkzJDrehDnIqfrfN12pykBW0TAMe3KuP7zyvr6+OlCrV2Lu/3jGNVXNjvE/ju0noEmYp62/Nu1/BVUO6o9gQZdaK0cHTzaDsZssI230Krgpi/ErXmSlg2tMnFIP4ScaSlCZVTZlnMWJQKYYPlIsxZcSUt0VbyzjMRPY7NX4zSbOU/EsuMGlu3GwFzMdoLmOhacGbiWUQ+VnKCjdm0yP/XFiQcBMAxsYxsPQg5MdjuHJwt/QxsybD/ApEgehkkQ0s1lAHucFFVUDo1aElsyxRLqZ9XxxMMH6iGrvPLLDx0+2tNQs37VsW4I3rhuBsQwyfuEVbZVq2biioPqGuudE0f2anQZq2rA7JUqYOhc5Us4VG9I4o4i8zzo1wKbj9mN+am3QbkHi2TeIx/FnSR8lsmeBMTBwEclN+UqWa+4n8vJia5ibPINz4KGj89uQeeHKE2OwUiwVvJpbBapYy/nSnuYkeJNwEgLHxdm3TDIsmnIl7fnJ4+pjIobiJ4n5KLNgOxaKZrXVAZKdVbcCPXTrgwHXmwVuouTGurPBQtipe+hvRYFzatpnpdzymoX9Ja9Oyc+sMKMHV3OjOTnzWQR+aL8KNbOgO6xguG+rAmGrH3v0S6eW0jalzovNlpQc5lmeFFxSPV45I68j6Nv33ueELY7I4fSO8V63ybcmmNU4CVR2K8y0OxX7JkckYXuKaaMiMA7kT1u/ZpLmRfAnmIH7h35MV98EpCC7G2YAOuy+ISAXoR9wb2dlkCnmHYrV6dCxqarsuHpM3kZln8/53+EaUHAstFWFdWZAXw+l9i3HLsF7McoxmCqvwZ1x+bO1o8hy0WazB1p+9m+TysAk3cc3zLsvs+qilFSV3Uz/e6iM3Dqqswc7vJyawonHTSuXLiU9kNUXL5i37/IzPTNWUbxSM/NSiyGQV0/yZbHhF5Iogvdeg4L1GwQeHhJuAYcXJEDlv+TLL9mqW4qZ1VzfjVSqrpcz+JbIfnDshSOWx67AM9oxr+3ZqhccvO5pZP8ASMdlyvbH6xo6mPqGjRWEeUG3Oy5Q/o75BCcwyJNXfzg4kXscYUfvQIJ5Ry26WaiQ1nlpzVXnUDX4wjIsC09w4V5CXRoO4WlImKieHYsfaJTE7FMewX0FANcXS8nEMlskqpgUTuFUVkVAnWz8n/7+wIbNUwDh9c9ZJhx+huz2bpXidm8sWbBXmZDfxFM0MeHx2x2nCVRY8/F4SysstJbyZ2oVllrufsxt6QgdaGM1ZjFJU370ssnlYfUXcCOupmbVxhZsVld3PHTU39f5pbtzMWNlxcYJaLeWclpXm5F4HO67q4mloVHznZJuqsZ9oEo+hps7ZcT9FvmH7BV8jFEv5immRcCgW7S1VkCe3c3sUNFAiSLgJGNZnY2wSVodifxqMPQ9RvrZznKRua2Y1u4jga27k6FBUiJN7qUejVRUAdI7Tr1N+abOUzjFLxYCaep5wo5siNcsKMn60KdnHY1+e73yhrpufwd8uHYDfnXGYUkwQp1JE9Xdllko7FMsLWTwyYZbiaZpYWO/h9D7t8cyvjmGmNb5vrs+NIG972epCdJO4ZgrS6URewLuCp+DFQIqE5kawiMS4VF6EirkxDEi4CRhHs5SlVQQ1EIkan/xqKbcVavgzFtOEzqnc2AkBfz0qCjPrK2W9Ml51U8eNeVjV+dZdjxvK1bnbUPDQfLLxux0E3JTdoagpbjitJ9q24O/JZc1VrLkR10G0NQIPnlpf5X5TbYC5WorRb4w8obt03lZYW6Hc/7Mj2GkPPN2zDk+u6LvlzF5cbYNZAWn8du1l28+wypbD+Mry82Jqwo1B4PZTiyKTU0xjb35r5J7z+6J/F3tkZj+x3rfxp6zmxpxH9KQbEm4ChqVdts7Ujfgi3DCOiT4o6xleWieV+1VDujnmH3f4uOMcFbbSU3Hlc+Pvx8l7VqlB0ay5MVyn8X2GbGYpCY2RBn/ivLgNUSBjZtWhK/eN9lsXt29R2+VFIRaReqZWk4abp8RcLcVIlxePpfdVUp39p8owDlzWlXwpUtWZ8sujsfCeYejdgb+VgFEI08wb2BvKZh9nl62uImwSUxNu8i17S/llAZRzKDYLVywqt+71p0ICrO3HZJaS1twYrzefi4Imh4SbgHGy51o7fz9UlqoOxSr+CyJ4eyhZN9ETZc8VbjT2bJaFm77K08aPrDviZMc0SxnVu4JykmapJsK0rE6maxv7IPbI8P6CkuzkC0IUdCxqCPBnffZuAoJJpVe4XtMYFxgw+twc210uOi9vxu/XaimeMunvlw7A/LvOQP+S1kplpIowCsf8ScyB/zXNsb2ZnN9N7dj4Hcu1b2M9nTA+n1hMQ42S5oa9Oa9XbCZKThqnNnLJMV18qxMPUeBMWc0NT1MXFUi4CRjmeGxoCamO7aKjkw36ptN6ei5T1anUeornzOjUb8t07E6xIIzqfpWw7V5RWy2lc81K6WOca1NpjZ2ziireaXBi1eWBi+wB2X42wN6BntLrYK6pIj9u7/C6tmmG98eeiPduPrHhoAufGzfYBHJRWsj73MjWl2eWUpFtUmlZ75HlVK5pyftu3Sxf+WtIldGi0Nh+xPWSQZdox8ZynPoI2bKtfZRbs5QouKIqThoZIKndEk1gu7drntSUBaz6EK2Qlfe5MV4fPfGGhJuAYQkKrQwdTKqR/eXiI/HF70/D2Ud09Fwme0bPb3zWDoenpXdqvrwO3/gM4jEN1554CD8PgeYmSFTiXei6WSMnu3kiYFgtZYxzI3mfVp8bGSFWA1DcqhAnHOq8N9SzVx2Ly47ryjzH0txoGnBo+5am2b0V+VmgGnYNlUh4F4vGoq0ReKTeo83/yoXmlXXNzhp7IENjKqfm+qef9cO9hsChqXZhbD/8uqoIbhzHesOfBzVrWL3oJADITmKsb6yW46fGoolBW67rOu4+vy8AYNRJPQAko8q7QWZyF9PEq6XatcjMvlP2/q7ht+w3C8vrNsXgcl813yDhJmCMnd8fL+iHXxxTgmF9O+Coktbo2b5F2mQQi2nC/XsAoJ3BwVK0kSHbF4Ofr/UUbzbjGH2Tc9q0siimYfgxJXht9GBbusOKW5hNWBZ1doAx/Hz3uXG7WkrULdQndBTkxYQpeVd7Xe7KFG4kypGx31ubm8yb4HfN7LSitmvU3MhO5FMDlDW5K7MU45pd++y7yctm/cyVA3H5caVmLd+B19CyoEEQ5ZqleBodxlPmaTCrDVGmD27Z0G+JzJuisq241dzMuu0Uk3auPqFjxKBu+OL3p+H2s5IBN5+96hjcfPphchUxYPdjsaeJaeKJVPqdBBmxFA4OxZKaG7PpEXjrxhN8qZtfUBC/gDE20V8eX5r++9XfJgd3lZneu2OGYtGP1Ti2exu8u2g9Zi3fLH2tilnKzeoRURkm+/gBsxTLP+fV0UPM9RLUUYSsb44R1fBCTup4Xn1Tr5sX50ZEQneOGWL3n2JrGFQpkNwWxFqObPRYY72lqqrQIDRN/IjrDLN+2edkFBxMZUnXqgFW29vJEm5MPg7skkraNMWpvZObN5qdeu1mKd53onIPpmZsuHDTrpr034WGjSqd2oPbSNgywk37lgUoadPMdN+pvs44sWxZ2ASn9m6PR95fJlWXFNa+nD350ITaq0zGjilp0xRrDzgvm8xSstpWk+ZGMzmeR8FMRZqbgOF1IMkl0fIN4O+XDUDbFgU48bCDTZ0FO2/GMZHmxmqWculzw/swTWapA5nIaJdUN1NM4cqhWKFLT5qlDNeyhAxOfql3bnYoZudli1ys65ZZn7M5LPUzU8KN7TqHtgr4o5ET5yF+tybNjWRtWnGW5Kfa6a+HdEdZ6UFSvlysmTwr9o7qeGGcbLAcivfUsgPfKX1rlolLis07axmpJYQb6XLNz4cXG8qUt5b6v6EUXl/HE0AuPZa/W7rMYhBNE080MynclN98UvpvY3/lJs6N3xu9+gFpbgLGL+2itB0U7IFV7JNg/s3T3Dj1ebzz5pUN4jxE+QW6t5QHMV/WsdeYlrcU3IhVhZ/QrbuCO9clreUOwizFqIC1FGnNDedvmfSAOKJv0hGXn5cbTWXKz4gX8+iuA34cFWu24rb/fIO7zz8cPFwNaJxLeGaiVLswvsc9tXbtkCBrJrx23LoZ2w/LaWNgWbOe9ZXJaG5YTYT36nkTSNG7kpmsxjRNKARlUrgx3qPZoTgYP7lMQ8JNwPg1HquYWmQGPdE5rnDj0Jy5ZbA0NxL5ZzICpoqvhA7+IOJcTvJ/o9M2b6msVaC1maUY+Qf1mOTNUhafG8nrVN+v0qokiNuuG58bXjBFazsqK22DD353MrdegLxpWlUAFNULAJrl87RPEhVwKOOGUw/Ftt21uKjMvCrP2SwlLiNF59ZNTb9lHHFZ/Rqvr2vKGeBF7UgUO6bhmPh9+7mRpwrGYv34ZqMg+JBZKmD82iNGJRdWwxL1KSqzYBF8s5Q9DdtfxPzbzcaZbgnXobgBY4fI0twY36PcKq3kAeMrPcXF9hSyDsVWjM6b48/po1yukaKmxpgr5tJFTdZJc2PKR7IuvBVibibe0gOaRDqz0G00SzX8PWn4URh5QnfuogTRxpni8hr+bt0sHw8PPwpDLKv0nDQ3so/vl8eX4srB3fDsVcltIe7/2RHCRRbW+qXg9XVc4UZQQZn3mIxQLDgf0tYMxnfeRFrbGgURhg8JNwHjlylFJR+2maThmDWMuzU5T1XrZNrgNXVWwDrV3Yk1Td60EvSu4FaYGhQHs5TIZyclSBzbzRxQTtftK8js5XLMUoYC/87YrdwJVpwbFtZHb+wof3PiIfhy/Gn2a3Tn9tC1TTP89/qG1RhqmhtzYp6/TLIycnmmNDe2CMUuhGSrxqO4VQH+ftkAWzqzENzw99ucVSq8CLIXDOiMP5zXl1tXmVs4vFPSeVRnfNsiCnxyKM7Pi+GenxyOU3q1BwCUtGmGf119nMN3bD/J09wU5rPrKcpezizlsM9fBDQ30n2hKF0E5B4SbgLGPzcRhZxYWhHD3+1amvfssce5YZflVlBjZcf6hkVLKVW+FTfVVNsTSHeOc8O5NuXbYxoUTKtggBk3DsXok3vgTz/rJ6wn25HZuR7NOSt9RMguD7Vinak3YTg3yQitMc2pPfDzSO5Eb8jL8q6NcZekHYqbsn1u3IxNxkfSv0trfPH703HekZ1s6ayrU1Ic3qmIma95mxfvo42x/Od/fSwA9W+tSZ64HkGO7axHwPvuZX3FjMg5FDsEMQ0o6KUTbhZvGG834JXrriDhJmA6FYlj17hF6CCsmN4KL5S5UwPmu9zItfw8S4fCW0UUBDJ7IBlxjFDsYJaq5wTxA5LbWNx2Vm+0PhD87KbTeqJTUSFGn9zDNLNT8a1iDdoXl8mHeWd29oyirK/a6hvBG2SdndU15fZwy7DDUNyqAHec3ds80BsufvQXR+G2s3qnf+s68NJvjhPmO7D0IO537cYh1PhORRFzZbSdvFV4KhoBmaSpTU1VBzUnDaDXz1xUHeN9jT65B/p0bJWODG9Py2unGlfzJ7MUHBC/i7A0N0a6HNTUORHcaSkzCQk3AfHcr4/F+f074XZDx+kFlU5ERZOQTG/+XVOXXCL6m6FJ85W8rwS7FJZdW+a7sG6/YMymbfN8PP2rgczrVOPcHNyyAOPOcf+emMKk5Xdq0Ev5INx8RjJImHVpKeu53HzGYfj0jlPRvlWhJc4Ny/zIriPrkTzI2JaBh1PwNR6dWzfFkyPK0gKDWwWCBrFfFOv+Rp3UA5+POw0lbZpx1e5HdmltE0gG92iHQ9o155Y1+fKj+YMf9yo+xvJFbVdGuDMvzWb/7YRa2AW1b81pewu/fd94ed92Vm/876ahylpMTQO+HH8685zsvoCiZEGvlhrWtxh/vMCuETYW27wgD5+NO9Uxr4hbpWi1VFCcdNjBOOkwdcdNHipdCOv7MPYZ9uWr5gv27U9qbn5/Th9cMagbSg5EUXYSGnj9ksuYgMLO/Os/nO7LzOGIzkV48/ohvs9CrPl9fMvJ+GzlFvxsQGcAwNCeB2P+XWegqGkTk6aM68x5ID+T5oaRjr8UnJFWoSM1xkf5w7l98NB7S/HQz/vb0lnLicc0DDu8g+m3GzRNfa8xnglAZiNHUaA1UcBBpcE55X9muMbtt9JwPTs6pEr75qV0ilAsg18Rit3gR9YaNBQ2iaN9ywJs3FljOif77lVWrvrNI8OPYgp0muXvjkXO2hvTmOK9ar5Dwk2WoBKinr0ruLFDtjhBWtLu3Z/U3GialhZskteJ4dWJqbmR6GrEUZX96QTqE7pyXrquLuiVtGlmepYA0mYnFeKKmpvUc3YTtdlISZtmuPqE7mjaJI6RQw/BVUO6swUVSznWmTrrncpUTbOuMrFkw8qCl5znaGvMRxRXSjhAe2yWwng9xr8lJhKunEQFebOOq7Yqv5aC8xCvmvPeZ6SyeObKY3Drf77Bkqrq9DmZ7Rec6uHC1UcJmTrJ+9zwNY5RMFmRWSpLUFH/shwMRW1NNs6N2/GRdZ1M2zd35pr0E1Cppttl747+Rwp5GYWEVk3F8w2nODe2egg0N6rceV5f3HJmcv8dWQ1M3OLLpLqfUfo85DQu3Os5Wh/7kvLkk+LFsQHMS9KteJ15izQ3PIdiI8ZBhhc92Am1aN1qLcup3URhUBSRql2/zkX4301DMbhH2/Q52W8iTLOUzDL/iL8CaUhzE0GKWxVgQ7VZ5anSh9x1Xl+0bZGPC47qzDzvdoWHkyDA65hYHaA1Jcte7WUwkyUoL3/j3jpONInH8OxVx6C2LuGozVF1KE4LNxnSG1uLsb5XXuftNKAmzVLG31ahhH1NQ/6c4xzNDW/vKBuWgr06hIp2KJcZ+M0OyQ1/q5ggVW5BtVk51cPr85t44REY9+pC5rkgBm2Tdkxa4OenS72+oD5X7jNw8WyiLgSFrrmZPHkyunfvjsLCQpSVlWHWrFnctFVVVbjsssvQq1cvxGIxjBkzJnMVzSAvjjwe7VoU4MZTD3V1fVGzJvj9OX3Qt1PDRmZ+mHiMH1zvDi3t+XCuYy8Fb0jdPD+O168bwkjD/tsRiZ4hNXgNMsy85LPXLb8bOL1P+wP/FyvleUqv9jjT4JvCw6gIYQkEPIfNsGzi1o5cpv/nCW0mXxcJaU3jCIIyka+tQfpOPOxgXHR0F1tMGWstvHb4sppErlmKE23ZD4UAMwvFhuVUj44eV5deemxXVPyB7fDrhz8Lz+wLsCZo7PJEQpDXScgbjH5UBms4CtVrokiows20adMwZswYjB8/HvPmzcPQoUNx9tlno7Kykpm+pqYGBx98MMaPH4/+/e3OjLnCoe1b4Kvxp2HssF7pY147UT+kbOOH986YE6XLYPvcNPDcr49Fv852U5rZpuvuw39xJHtZ7/9uGoq7zuuL287qxTwvwrpxppEnRgzEK9ccj6stgRL9wklzw40umiHVjbUYmZD0yePOeYsGxp7tW+Dorq2553mCDk8DZDVLlbZphr9e0p8bU4aXnyqifa5ksjY+f2NOamYpedq1KHBOZMzboR5dDmomPC9DW06dVAW8R39xlO0YL0hmMn85n5tWnOjWQENfyavq8IH8jTsBoLStu+fXrmWDxli2qZji3LgqNVhCFW4efvhhXH311Rg5ciT69OmDSZMmoaSkBFOmTGGm79atGx599FFcccUVKCoSdzLZjr3T9dZ8/JGy3dVBpGoH+B+TH7NNa/j3FCVtmuHXJ3Tn7q/jBO91xGMajj+krfTmc6o4+dxYnV3TDsWB1MaOVavllw+BTXNjPR/TMP23g3HfT/kbVKZolm/YMNByLpWvVbjxw5/CCiup0CFWMc6NW58bXjEsweSILkX4w7l98MSIMi9Zp2maH8x3A6gLnj89qjMuOMocTFGUg2iFnZGDOJuKAuLvdOatJ+OCAWxXg4Y6qDlsP3bpAIw4vhTnm4JGyt1H1P2jQvO5qa2tRUVFBe644w7T8WHDhmHOnDm+lVNTU4Oamgb/h+rqakHq3MVvzY1KGbIRim1pwB/MRHjdATvKmAYpGc1Nhn1urMh0+Lqum26FVdeCJjFLRFRW7CRNqmEZTU625Afytfrc8IQbp7AKqsibpdjlJEyaG3ZAP8e8JZxOjYwcegjnjJ0OArNTz/YtpPNxg5s3Y3vO1p+G87Lv/qDmfL860esvyIs7vkenOELWd/uT/p3wk/6d8N169XFRVFIU5J7QNDebN29GfX09iovNvgnFxcVYv369b+VMnDgRRUVF6X8lJWK1XlSxLwVXaz3mwcPlCiHHMnj+HpyByHAlMz9b64yG0KIjPAHKtBSc8dzsmpskmaqvtWlZV0up8uDFR6Jrm2b4y8VH2syULGQ0J0atjHVAatDcmGfXQWhuWMiuluJRz9HcqGjQghyYOhY1xbNXHsM89+b17D2y/MKP+7J+c8ZfNhMsJ482gkUDqe/U2AyuGtINP+nfCcWtChx7fdb2JqY6cbXkzn5osnlFhdBXS7HML36qu8aNG4exY8emf1dXV2elgON1cDI+UqeonIWcfYRkhKJhfYvx3uIN+OXxXdPHXAfxUyy7Ia278lQw+TZkUM5x8rmxxhFJfUsZWy1lKcdpJpmC5xNzycASXHLAz2BvbX1DObx8JIR+o88DL3ULi1lKWiOj0Hex7kE6zg0vT6NwYzgelM+NG07p3Z55PEiTFODOjGK95GDLvnzG87IC5EHNBWYpxuu/+3xnU2sKpxVbvLMybct2jWCyEQW5JzThpl27dojH4zYtzcaNG23aHC8UFBSgoEDN6S2KeB2cjA3xwqO74N7/LmamK2wSw1s3sGdQjlXQgEd/MQBfrt6K4w9p2NHaeQk5+7i1Q+7aphk276p1qkVOY5yYsQYsruYmBEXTsd3aoE+HVo7pkruCO2O8Xd79yIxfJn8aS/pUvoWW58gLrW8Vur1qbkS+WjKrB3kTiaC2X8gmXJmlDFddXNbFNGmz5ikrQHZqzY/+6zSJ8/oZyzj0u3v/0dCqGwnNLJWfn4+ysjKUl5ebjpeXl2Pw4MEh1Sq6iNq8zLJjY3MtyIvhulN6MNPdemZvHNrevswbkHAMRnL2ddJhB5sivLoNeGf9xh79xQCcdXgHTP+tuH0EPZAndwUPB6ddwWU1JUFhfC7/N2qQL7tRpxBF2U4hU5rJ54ZzRYFFyJCOYeJSMBh/Th+c0utg/KS/fTfwFOoOxUafGx80Nz42rf+MGoTz+3fCpcd2dU7sE15ltod+3t8Wudr4XGUdijsWNeU6vqfemEpVrWE5TuAsohCRiwJtqKulxo4di6eeegrPPPMMlixZgptvvhmVlZUYNWoUgKRJ6YorrjBdM3/+fMyfPx+7du3Cpk2bMH/+fCxezNZC5BKipeBPjijDsL5iAUe28YpSOfrc8JwcHVdLsa+zzoRK2jTDP0aUoaz0IGF+sp2MW8KcozjtLWXddTkdxC/AOnlFh6SDuU9pjJobe8yi5G+reU9WRnO7KOk3Jx6CZ686Vri1g4MvOQB/zKWZGOcGdmuDv106IL0zveymk15w4+ztdIlIcyO6dsSgbpj+20G246pB/A5uWYDjurcxHXvu1/x25KdZSkQUhKVQfW6GDx+OLVu2YMKECaiqqkK/fv0wY8YMlJaWAkgG7bPGvBkwYED674qKCrz00ksoLS3F6tWrM1n1jCOKXxCLaWjbQhzZ1tR4Be1O2CZdamBEqwNkUemobz+rN+ZVbsdVQ7p5LpdLSMuPYk6amzw580lQBFmOjEOxnM9NQ7dXsz9hOpfKt6AJe0m9FfukIzOdukw5bvWLqqulvFBWehDevH6IL/FtnHDzbpxkLmOWqgJaWWkb2zHV7WAOataEuVlt2+b5qNqxz5ae9wh4Pm/ZTOgOxaNHj8bo0aOZ56ZOnWo7lqlOOir8e9QgrNy0C8d0s38IKphnffzW601zwz5+1ZBuWLB2O84+gh2Bt7gVL+iWu6+spE0zfHrHqa6ulUXn/B00ZrMHw+fG6lAsmW8GJs5cjuhcJDXwSAUNk7iPgiZxXDigM3bs3Y8uB7H9Hwry5DQ3QTpS/qR/J7y54Ecfc5REYgD0kyO7tA4kXysjji9VvsZZWG4470dMJ9W+hFc/npDE15I75ymCHIoJZY7p1sazYAPYl/pxZ2eCDsytYNksPw9PXjHQdvw/owahet9+dCxiDzCRnUGEKF87+dzYHIolV0v5EZpelZMOOxgXl3XByb0ONh2XaZu8tihzHxqAh4cfxTyX1tzYnqNjttLly/KXnx+Jo0paY8JbSbO78d2feXgxPvxuo231jpEom6XC4NJj/V8lK1ot5U5IUHtpmsZ+z6orVL0GeXVagRsG0asREQwWe72bDsz6vbRu1gTb9+w35KuW6UAHoS2MAVeGKwaVhrZqyzFCMSdCqZOJwvis82Ia6lyu31fpmw85uDnOZzjQyphT+GYpb6SyFTmOGmndzF08HBkK8uI4zrDqsKnByfnnZSXoUNQURzC2LUnhWrhxd1mkaRLXXGmeVHxu3Lz7G049FH/7cEX6t9M7s2967I8ZmrUSr3eHlvhu/U7hdX+8oB+qduxFn47OqyIzTegbZxLuUP2MzD43GlcVLPqYrd/LnDtONZl/Djm4uWKtxJjNENExR/5qcLfwgviZNHD2l2WNUJxKoRJdOqitI6x4sTBzNTcee7RUvlbNDY+f9O+EnxlC4vstjxsnDMY4MLGYhpMOOxhtRNFuJfJPbc7b1zA4edn/K6q41Uw4CURu4twY+d2wXlg84cz0b9VvQgO7b1TW3DCq/tJvjnfUdv3y+FLcemZvqfwyDQk3jQSrFuT0Pu3x6C+OwvtjT5LOw/oRNcvPQ+fWTfHRLSfjtdGDhfEb3BAFj3sr8Zi7GaBfmG3jdmyrJA4kcurrjO3Di3CjIvR58Z/zEsRPBrt5j50uLx7DIwYTl9KSa4mkXoROmed78xmH4f2xJ2HcOQ0DVPS+Oh9weVNO78goRFjDAMg2BeP+dqqTJl4Zxg1Yz+H4OprzsZu72zTPx0/6i/eyijIk3GQpquOrdRmppmn46VGdcahlPxdR58ybDXRv1xwDuh6kViFFouJHztSEZLByTvFW7JqblM+Nk1mq4e+m+e67BZVH4TZytagcme9CplirWUrWRKoyeT9SYFJK4Um4kUijaRoObd9CKvx+Ngs9QTnMG78r67fpZg7k9E1YhR++z03DwQuOchZQzMsU/Fg5FX5rIZ+bRoJsgxU2yRAFjIjINqFjnB2yngnPnKKkucnLjFmKu6IjxI4xVSVepGcnZISg924+ETMWVuE3EhtOGvNrqqpRU/hoeINbruDWf8/pKqOGxI9YPQ0RsuXfNes1q+4Iz0uTzS2BNDeNBO/htaPl9xIWYVvKjLNDlnBg09ykVU0OGZs0N5kSbtxfy7vU7SB2ycBkMLkxp/cEYPef8FNzc1hxS4w5/TCpFSbG7FSFG9dxbniam7AbvwdcCzcqZilf1EPJDCdedAS6t2uOh37eX5haZim4qvnTj7EiCpDmJmdwcHyTzUXBoTiTRMUslSIK2y+wngkvMqmK5qa4VSGAHS5qp/pc2KlL2zkHdOOZ2aQ6csaxP190JG45sxfatyx0nS8/d/doJqFTbS4atW8mTNyO0U5aLKMQ4WUpeN+OrbC4qhoXHp0Usnsc3AIf3XKyc/0kzFJS4RG4wo3jpY75hQUJN40FHxob9ZVGH5aGY2EF8WMN8NbVM5ogrRFjv3zfT/uhfPEGV/VTcRJOmIMDY8Fdw1BTV2/atZtfDvu4l1UxPMFGBb99O4wz5yB8bhoKMpapVExWEJTmxmiWcruvGAD857eDsHLTbhzeSW1J9aBD2qJ6X53tuFGjJCfwa8y/3ZKJ7TScILNU1qLWeGQbrChdmNGhI2MSC90s1fA3y6xjjbib6thUNDcdigqF8VP8wmpWK2rWBO1byQkY3I0zA3o/sup5P+PcAGYhTtUsNbhHW1fXZXL7hUzh9rWo+Nx42SS2WX4e+klG6k5x53l9cfMZh4H1datumspLUlev1u8++ouj0L5lAab8skzpuiAgzU0jwal9n3BoO3z6/Wac1Y+/bNCLj0SuYRxcM9npG2eHLJ8b6+yepWli4ZdtXaWJeGlOQQXx4+GHWdcNNXX16b9VfaFK2zbHrNtOUd7bje9zo5RNpHCvuRFfV5/pTtFQ3JWDu3GFaWO1ZGQu3vOps6pXHfjpUZ3xk/6dIuGrQ5qbRoLTx/2vq4/FkglnCYOCpeBFwQ2SqPgPpJ9iWGYpB4diKw2aG3Fa6yt1rSlTWgruf5ybFoX+zdf+adgyRH5XcH879X2GjT3drGIradMMLaQcl41midwjqMFWFMn7xMPaBVJmilSbdPK5kdHa81K4iVQeBcEGIOEma1FtP86722rSNn1rbJzGhKyZJ7jyxQ7FAPDOmKG2Y1HcW0p5abMB3v0M6dEOFx7dGePP6eM67xRn9C1O/y3bYfv9FI3CrBezhwpRGZz8xLVZSsHnxsrPy0ow5fKjbVt0eMFYmug9Pf2rgYhpwIMXHyl178asjBMbVbNUlCDhJkdwChfvR3913Sk9kBfT8OeLjvSeWZbCmgX9ekh3AMAplg0gg4an+ejdQX2fl0wKN/f/7Aj071J0wF/AX2IxDQ9fchR+c6JzDBkVZB+P38+xf5ciXHR0F9wyzP9nZcQa5JOTKtA6BIn7ODfi60SajVhMw9lHdEQHST8yt7C6gVN7F2PZH8/GJQNLMLBbGxzSrjlO692emwdv0lRX36A57N6uOfqXtPajyhmBfG5yhOtPPRSzV2zG8IHsvUBK2jgvr3Xi1jN744ZTe2Zs76Fs4YIBnXFElyKU+vCMVZDRGKd2XFfZW8oLMvO8y47risuO6+qxHPczSlcbKEqm89uhWNM0/PUScawTv8kFxc1RJa0xf+329O/gNDdqPimZJO+ArTk/L4b3x54kDt7Kcyg2dDIf3XIyRr9YgQVr/axlcJBwkyO0a1Eg3CeqfctCvHHdEM9b04cl2IS5UstI2ixlqU+PgzNvqhP5rDz9q4F459v1uEZSg2Gd2bp93Jl6TxlvDiGZpcIgFzbOfPpXAzFjYRV+2L4XT8xcifsvPMJVPk637MYnxW9kBH0nkybf58YsvEWkG5aChJssxU0/k00qxajSEDcm1GokEdThtD7FOK1Pg8+ISpybbCDTwq7s82npo0NzJpG5vWxqIm1bFGDEoG4AgJtPP8z1pMyLz002EeOYpfaTzw2RaQb1aAtNA/p1VvevyEYiIUxEhCGHtkW7Fvk4/pC20tc49cF++Ypk6jVluj04+V78+aIjcEy3g3DT6cH6xhDqeNE2O5kwo+Bw68e3wHMoHtozueLr4JYF3gvJMNk5zSDQsrAJlkw4K5Rl2Y2ZVGcXZpf2wtXHoT6hp23qfmDtw50c1HlkSujI9PN30twMP6Yrhh/jzY/IC7ef1Rt/fuc7/GpQaWh1aIzIaG78XH3G+r78+BZ4wnvHoqb4avzpaY1kNk0ySbjJYhqTY2/UvqkwP3JN05AXV+swnezyA7oeZPr94MVH4urnvsYNp/ZUrl8myPTz75phZ3FVRp10CM48vBjd2jZ3db3MANypdVNs3FnjKv9sxemxRMHnxg9MmhvLLWWj1gYg4YYglMgmvwMjPGGg/OYT8d8FP2KkxfH40PYtMfPWU9TLiZwYakfFv+jl3xyP5Rt3YlAPeRNgGGiahkMCdmr/26UDcNcb3+Lak3oEWk6UcDJHRmG1lN9mKRGDerTFO4vWey8wA5BwQ2QFkVGHSkb8jRqHHNycOevuWdwSY4f18q2czJml1Av61aBSfLFqK845oqP0NYN6tI28YJMpSto0w7NXHRt2NTLKYcVigTHTmpug+h3ZvQcvP64rWhbm4ZhubQKph5+QcENkBdkmTESNR4YfhQffWYpfDe4WaDlHdinCD9v2BloG4E6Iuven/fyvSI6QTcu8M8kFR3XG1t21KCs9iHk+Cqul/OgbYwKzlJG8eAwXHt3Fc3mZgIQbglAgUkvBFehY1BSPDD8q8HL+dMER6Ny6KS4uYweT9Er/ktZYsHY7fnJUp0Dyb6yEsf1GNhCLaRg5lB8rKnd8bnLv/ZNwQ2QFUREmcrET8JODmudj/Ll9A8t/+qhB2LmvTnmna0LMUSWtMbD0oMg7TkeNTGtumMFC/fC5MWUXkc7WIyTcEIQCJNuES148RoJNAMRjGv7z28FhVyPryLRw06l1U7xx3RAUNfVvM05AvFoqWyHhhsgKova9RWU7CIIgGhfWSPN+CPu5qJGmCHAEoUCqC7ioLOlUd0TnovAqQxBEo+fG03ri1N7t8dilA8KuSqQgzQ2RHURMU3Jkl9b4fNxpaNuCTCQEQfAJWidS1LQJnrnyGN/yi1ZP6x7S3BCEBA/9vD9aFubhiRED08c6FBWiCW1/QRCEAL/9Y4ImV0zupLkhsoKwP7eLy7rgwgGdEcu27bMJggiVP190JG58ZR6uPZG/pDxKlOTIijkSbghCEhJsCIJQpWvbZnj9uiFhV8ORT249Bbtq6tCuRXbuJWWFdOpEpBlyaDL0/UVZEhWTIAgiG+nathn6dmoVdjV8gzQ3RKT516+Pw+7aOrQszC67NUEQuc8jw/vj5mkLaKVSBCHhhog0sZhGgg1BEJHkZwO64NwjOiE/j4wgUYPeCEEQBEG4hASbaEJvhSAIgiCInIKEG4IgCIIgcgoSbgiCIAiCyClIuCEIgiAIIqcg4YYgCIIgiJyChBuCIAiCIHKK0IWbyZMno3v37igsLERZWRlmzZolTD9z5kyUlZWhsLAQhxxyCP7xj39kqKYEQRAEQWQDoQo306ZNw5gxYzB+/HjMmzcPQ4cOxdlnn43Kykpm+lWrVuGcc87B0KFDMW/ePPz+97/HjTfeiOnTp2e45gRBEARBRBVND3F/8+OOOw5HH300pkyZkj7Wp08fXHDBBZg4caIt/e23344333wTS5YsSR8bNWoUFixYgM8++0yqzOrqahQVFWHHjh1o1Sp39tEgCIIgiFxGZfwObfuF2tpaVFRU4I477jAdHzZsGObMmcO85rPPPsOwYcNMx84880w8/fTT2L9/P5o0sYfpr6mpQU1NTfr3jh07ACQfEkEQBEEQ2UFq3JbRyYQm3GzevBn19fUoLi42HS8uLsb69euZ16xfv56Zvq6uDps3b0bHjh1t10ycOBH33nuv7XhJSYmH2hMEQRAEEQY7d+5EUVGRME3oG2dqmmb6reu67ZhTetbxFOPGjcPYsWPTvxOJBLZu3Yq2bdsKy1GluroaJSUlWLt2LZm7QoLeQbjQ8w8fegfhQs8/WHRdx86dO9GpUyfHtKEJN+3atUM8HrdpaTZu3GjTzqTo0KEDM31eXh7atm3LvKagoAAFBQWmY61bt3ZfcQdatWpFjTpk6B2ECz3/8KF3EC70/IPDSWOTIrTVUvn5+SgrK0N5ebnpeHl5OQYPHsy8ZtCgQbb07733HgYOHMj0tyEIgiAIovER6lLwsWPH4qmnnsIzzzyDJUuW4Oabb0ZlZSVGjRoFIGlSuuKKK9LpR40ahTVr1mDs2LFYsmQJnnnmGTz99NO45ZZbwroFgiAIgiAiRqg+N8OHD8eWLVswYcIEVFVVoV+/fpgxYwZKS0sBAFVVVaaYN927d8eMGTNw88034/HHH0enTp3w2GOP4aKLLgrrFtIUFBTg7rvvtpnAiMxB7yBc6PmHD72DcKHnHx1CjXNDEARBEAThN6Fvv0AQBEEQBOEnJNwQBEEQBJFTkHBDEARBEEROQcINQRAEQRA5BQk3PjF58mR0794dhYWFKCsrw6xZs8KuUk4wceJEHHPMMWjZsiXat2+PCy64AEuXLjWl0XUd99xzDzp16oSmTZvi5JNPxqJFi0xpampqcMMNN6Bdu3Zo3rw5fvKTn+CHH37I5K3kBBMnToSmaRgzZkz6GD3/YFm3bh1++ctfom3btmjWrBmOOuooVFRUpM/T8w+Wuro6/OEPf0D37t3RtGlTHHLIIZgwYQISiUQ6Db2DCKITnnnllVf0Jk2a6P/85z/1xYsX6zfddJPevHlzfc2aNWFXLes588wz9WeffVb/9ttv9fnz5+vnnnuu3rVrV33Xrl3pNA888IDesmVLffr06frChQv14cOH6x07dtSrq6vTaUaNGqV37txZLy8v1+fOnaufcsopev/+/fW6urowbisr+fLLL/Vu3brpRx55pH7TTTelj9PzD46tW7fqpaWl+pVXXql/8cUX+qpVq/T3339fX7FiRToNPf9g+eMf/6i3bdtWf+utt/RVq1bp//73v/UWLVrokyZNSqehdxA9SLjxgWOPPVYfNWqU6Vjv3r31O+64I6Qa5S4bN27UAegzZ87UdV3XE4mE3qFDB/2BBx5Ip9m3b59eVFSk/+Mf/9B1Xde3b9+uN2nSRH/llVfSadatW6fHYjH9nXfeyewNZCk7d+7Ue/bsqZeXl+snnXRSWrih5x8st99+u37CCSdwz9PzD55zzz1X//Wvf206duGFF+q//OUvdV2ndxBVyCzlkdraWlRUVGDYsGGm48OGDcOcOXNCqlXusmPHDgBAmzZtAACrVq3C+vXrTc+/oKAAJ510Uvr5V1RUYP/+/aY0nTp1Qr9+/egdSXLdddfh3HPPxemnn246Ts8/WN58800MHDgQP//5z9G+fXsMGDAA//znP9Pn6fkHzwknnIAPPvgAy5YtAwAsWLAAs2fPxjnnnAOA3kFUCX1X8Gxn8+bNqK+vt232WVxcbNvkk/CGrusYO3YsTjjhBPTr1w8A0s+Y9fzXrFmTTpOfn4+DDjrIlobekTOvvPIK5s6di6+++sp2jp5/sKxcuRJTpkzB2LFj8fvf/x5ffvklbrzxRhQUFOCKK66g558Bbr/9duzYsQO9e/dGPB5HfX09/vSnP+HSSy8FQN9AVCHhxic0TTP91nXddozwxvXXX49vvvkGs2fPtp1z8/zpHTmzdu1a3HTTTXjvvfdQWFjITUfPPxgSiQQGDhyI+++/HwAwYMAALFq0CFOmTDHtu0fPPzimTZuGF154AS+99BIOP/xwzJ8/H2PGjEGnTp3wq1/9Kp2O3kG0ILOUR9q1a4d4PG6Tvjdu3GiT5An33HDDDXjzzTfx0UcfoUuXLunjHTp0AADh8+/QoQNqa2uxbds2bhqCTUVFBTZu3IiysjLk5eUhLy8PM2fOxGOPPYa8vLz086PnHwwdO3ZE3759Tcf69OmT3nOP2n/w3Hrrrbjjjjvwi1/8AkcccQRGjBiBm2++GRMnTgRA7yCqkHDjkfz8fJSVlaG8vNx0vLy8HIMHDw6pVrmDruu4/vrr8eqrr+LDDz9E9+7dTee7d++ODh06mJ5/bW0tZs6cmX7+ZWVlaNKkiSlNVVUVvv32W3pHDpx22mlYuHAh5s+fn/43cOBAXH755Zg/fz4OOeQQev4BMmTIEFvog2XLlqU3F6b2Hzx79uxBLGYeKuPxeHopOL2DiBKSI3NOkVoK/vTTT+uLFy/Wx4wZozdv3lxfvXp12FXLen7729/qRUVF+scff6xXVVWl/+3Zsyed5oEHHtCLior0V199VV+4cKF+6aWXMpdhdunSRX///ff1uXPn6qeeeiotw3SJcbWUrtPzD5Ivv/xSz8vL0//0pz/py5cv11988UW9WbNm+gsvvJBOQ88/WH71q1/pnTt3Ti8Ff/XVV/V27drpt912WzoNvYPoQcKNTzz++ON6aWmpnp+frx999NHppcqENwAw/z377LPpNIlEQr/77rv1Dh066AUFBfqJJ56oL1y40JTP3r179euvv15v06aN3rRpU/28887TKysrM3w3uYFVuKHnHyz//e9/9X79+ukFBQV679699SeffNJ0np5/sFRXV+s33XST3rVrV72wsFA/5JBD9PHjx+s1NTXpNPQOooem67oepuaIIAiCIAjCT8jnhiAIgiCInIKEG4IgCIIgcgoSbgiCIAiCyClIuCEIgiAIIqcg4YYgCIIgiJyChBuCIAiCIHIKEm4IgiAIgsgpSLghCIIgCCKnIOGGIIiMcPLJJ2PMmDEZLXP16tXQNA3z58/PaLkEQYQLCTcEQWQFH3/8MTRNw/bt28OuCkEQEYeEG4IgCIIgcgoSbgiCyBh1dXW4/vrr0bp1a7Rt2xZ/+MMfkNre7oUXXsDAgQPRsmVLdOjQAZdddhk2btwIIGleOuWUUwAABx10EDRNw5VXXgkASCQS+POf/4xDDz0UBQUF6Nq1K/70pz+Zyl25ciVOOeUUNGvWDP3798dnn31mOj9nzhyceOKJaNq0KUpKSnDjjTdi9+7d6fOTJ09Gz549UVhYiOLiYlx88cVBPSKCIHyAhBuCIDLGc889h7y8PHzxxRd47LHH8Mgjj+Cpp54CANTW1uK+++7DggUL8Prrr2PVqlVpAaakpATTp08HACxduhRVVVV49NFHAQDjxo3Dn//8Z9x5551YvHgxXnrpJRQXF5vKHT9+PG655RbMnz8fhx12GC699FLU1dUBABYuXIgzzzwTF154Ib755htMmzYNs2fPxvXXXw8A+Prrr3HjjTdiwoQJWLp0Kd555x2ceOKJmXhcBEG4JeRdyQmCaCScdNJJep8+ffREIpE+dvvtt+t9+vRhpv/yyy91APrOnTt1Xdf1jz76SAegb9u2LZ2murpaLygo0P/5z38y81i1apUOQH/qqafSxxYtWqQD0JcsWaLruq6PGDFCv+aaa0zXzZo1S4/FYvrevXv16dOn661atdKrq6td3TdBEJmHNDcEQWSM448/HpqmpX8PGjQIy5cvR319PebNm4ef/vSnKC0tRcuWLXHyyScDACorK7n5LVmyBDU1NTjttNOE5R555JHpvzt27AgAaZNXRUUFpk6dihYtWqT/nXnmmUgkEli1ahXOOOMMlJaW4pBDDsGIESPw4osvYs+ePW4fAUEQGYCEG4IgQmffvn0YNmwYWrRogRdeeAFfffUVXnvtNQBJcxWPpk2bSuXfpEmT9N8p4SqRSKT/v/baazF//vz0vwULFmD58uXo0aMHWrZsiblz5+Lll19Gx44dcdddd6F///60aosgIgwJNwRBZIzPP//c9rtnz5747rvvsHnzZjzwwAMYOnQoevfundaspMjPzwcA1NfXp4/17NkTTZs2xQcffOC6TkcffTQWLVqEQw891PYvVWZeXh5OP/10PPjgg/jmm2+wevVqfPjhh67LJAgiWEi4IQgiY6xduxZjx47F0qVL8fLLL+Nvf/sbbrrpJnTt2hX5+fn429/+hpUrV+LNN9/EfffdZ7q2tLQUmqbhrbfewqZNm7Br1y4UFhbi9ttvx2233Ybnn38e33//PT7//HM8/fTT0nW6/fbb8dlnn+G6667D/PnzsXz5crz55pu44YYbAABvvfUWHnvsMcyfPx9r1qzB888/j0QigV69evn6bAiC8A8SbgiCyBhXXHEF9u7di2OPPRbXXXcdbrjhBlxzzTU4+OCDMXXqVPz73/9G37598cADD+Chhx4yXdu5c2fce++9uOOOO1BcXJxezXTnnXfid7/7He666y706dMHw4cPt2l9RBx55JGYOXMmli9fjqFDh2LAgAG488470745rVu3xquvvopTTz0Vffr0wT/+8Q+8/PLLOPzww/17MARB+Iqm6weCTBAEQRAEQeQApLkhCIIgCCKnIOGGIAiCIIicgoQbgiAIgiByChJuCIIgCILIKUi4IQiCIAgipyDhhiAIgiCInIKEG4IgCIIgcgoSbgiCIAiCyClIuCEIgiAIIqcg4YYgCIIgiJyChBuCIAiCIHKK/wddWwgQsId6hgAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plt.plot(epochs_x[-1], epochs_y[-1])\n", + "plt.xlabel('batches')\n", + "plt.ylabel('loss')\n", + "plt.ylim(0,)\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAGwCAYAAABVdURTAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAABTAElEQVR4nO3de1xUZf4H8M/MwDCgMIhc5C4o4g1NQREVlRRMXVe7rJSu1qa/1rL1QmqirWu2vygvZW6pWZo/dxMpRXNXNsVU0CAvBOUVSRAQQQRluAk4zPn9gU6NXGSQ4TDD5/16zevVPPOcM9/HU83H55zzHIkgCAKIiIiITIRU7AKIiIiIWhPDDREREZkUhhsiIiIyKQw3REREZFIYboiIiMikMNwQERGRSWG4ISIiIpNiJnYBbU2j0eDGjRuwtraGRCIRuxwiIiJqBkEQUFZWBhcXF0ilTc/NdLhwc+PGDbi7u4tdBhEREbVAbm4u3NzcmuzT4cKNtbU1gLo/HBsbG5GrISIiouYoLS2Fu7u79ne8KR0u3Dw4FWVjY8NwQ0REZGSac0kJLyhuRGJiIiZPngwXFxdIJBLs37+/yf6xsbEIDQ2Fg4MDbGxsEBQUhEOHDtXrV1JSgnnz5sHZ2RkKhQJ9+vRBXFycgUZBRETU8TDcNKKiogIDBw7Exx9/3Kz+iYmJCA0NRVxcHFJSUhASEoLJkycjNTVV26empgahoaG4du0a9uzZg/T0dHz22WdwdXU11DCIiIg6nA53Wqq5JkyYgAkTJjS7/4YNG3Tev/vuu/jmm2/w73//G4MGDQIAbN++Hbdv30ZSUhLMzc0BAJ6enq1WMxEREXHmxmA0Gg3KyspgZ2enbTtw4ACCgoIwb948ODk5oX///nj33XdRW1srYqVERESmhTM3BrJ+/XpUVFRg2rRp2rbMzEwcPXoUM2bMQFxcHDIyMjBv3jyo1WqsXLlSxGqJiIhMB8ONAURHR2PVqlX45ptv4OjoqG3XaDRwdHTE1q1bIZPJ4O/vjxs3bmDt2rUMN0RERK2E4aaVxcTEYPbs2fj6668xbtw4nc+cnZ1hbm4OmUymbevTpw8KCgpQU1MDuVze1uUSERGZHF5z04qio6Px0ksvYdeuXZg0aVK9z0eMGIFffvkFGo1G23blyhU4Ozsz2BAREbUShptGlJeXIy0tDWlpaQCArKwspKWlIScnBwAQGRmJWbNmaftHR0dj1qxZWL9+PYYNG4aCggIUFBRApVJp+7z66qsoLi7GggULcOXKFRw8eBDvvvsu5s2b16ZjIyIiMmUSQRAEsYtoS6WlpVAqlVCpVE2uUHz8+HGEhITUa3/xxRexY8cOvPTSS7h27RqOHz8OABgzZgwSEhIa7f9AcnIyFi1ahLS0NLi6umL27Nl48803dU5VERERka7m/n4DDDettt/dp3MQGXsOAgCpBIh6xg/hQzxabf9EREQdmT6/3zwt1QryVXcRua8u2ACARgCWx55HvuquqHURERF1RAw3rSCrqAIPz3/VCgKuFVWKUxAREVEHxnDTCrzsO0H60ENKpRKgu72VOAURERF1YAw3rcBZaYmoZ/wg+81j2Ae628JZaSliVURERB0Tw00rCR/igZPLQvDu1P4AgLTcEqQXlIlcFRERUcfDcNOKnJWWmD7MExP6d4MgAOsOp4tdEhERUYfDcGMAb4T1glQCxF+8iR9z7ohdDhERUYfCcGMAPR2t8exgNwDA2m/T0cGWEiIiIhIVw42BLBjnA7lMiuTMYnz/S7HY5RAREXUYDDcG4tbFCtMD61YoXnvoMmdviIiI2gjDjQG9/mRPWMll+Om6CocuFIhdDhERUYfAcGNA9p0tMHukFwBg3eErqNVw9oaIiMjQGG4M7H9GecPWyhy/FJYj9sfrYpdDRERk8hhuDMxGYY5XR/cAAGw4koFqda3IFREREZk2UcNNYmIiJk+eDBcXF0gkEuzfv7/J/rGxsQgNDYWDgwNsbGwQFBSEQ4cOtU2xj2FWUHc4Wlsgr+Quok/liF0OERGRSRM13FRUVGDgwIH4+OOPm9U/MTERoaGhiIuLQ0pKCkJCQjB58mSkpqYauNLHYymXYf5YHwDAx8d+QUW1WuSKiIiITJdEaCf3KEskEuzbtw9Tp07Va7t+/fohPDwcK1eubPDz6upqVFdXa9+XlpbC3d0dKpUKNjY2j1OyXu7VajDugwRkF1dicVgvvP6kT5t9NxERkbErLS2FUqls1u+3UV9zo9FoUFZWBjs7u0b7REVFQalUal/u7u5tWOGvzGVSRIT2AgB8mpiJksoaUeogIiIydUYdbtavX4+KigpMmzat0T6RkZFQqVTaV25ubhtWqGvyABf07maNsio1tiRkilYHERGRKTPacBMdHY1Vq1YhJiYGjo6OjfazsLCAjY2NzkssUqkEi8N8AQA7krJQWFolWi1ERESmyijDTUxMDGbPno2vvvoK48aNE7scvYzt44jBHraouqfBxqMZYpdDRERkcowu3ERHR+Oll17Crl27MGnSJLHL0ZtEIsHSp3oDAHafzkVOcaXIFREREZkWUcNNeXk50tLSkJaWBgDIyspCWloacnLq1oKJjIzErFmztP2jo6Mxa9YsrF+/HsOGDUNBQQEKCgqgUqnEKL/Fhnl3RbCPPdQaAR8euSJ2OURERCZF1HBz9uxZDBo0CIMGDQIAREREYNCgQdrbuvPz87VBBwA+/fRTqNVqzJs3D87OztrXggULRKn/cSwdXzd7sz8tD5cLSkWuhoiIyHS0m3Vu2oo+98kb2mtfpiDuXAHG9XHC5y8GiFoLERFRe9Zh1rkxdhGhvpBKgCOXbuLHnDtil0NERGQSGG5E1NOxM54d7AYAWPttOjrYJBoREZFBMNyIbGFoL8hlUiRnFuPkL0Vil0NERGT0GG5E5mpriRnDPAAAaw9x9oaIiOhxMdy0A/NCesJKLsPP11U4dKFA7HKIiIiMGsNNO2Df2QKzR3oBANYdvoJaDWdviIiIWorhpp34n1HesLUyxy+F5Yj98brY5RARERkthpt2wkZhjldH9wAAbDiSgWp1rcgVERERGSeGm3bkxeHd4WRjgbySu4g+lfPoDYiIiKgehpt2RGEuw1+e9AEAfHzsF1RUq0WuiIiIyPgw3LQz4UPc4dnVCkXlNfji+yyxyyEiIjI6DDftjLlMiojQXgCATxMzUVJZI3JFRERExoXhph2aPMAFvbtZo6xKjS0JmWKXQ0REZFQYbtohqVSCxWG+AIAdSVkoLK0SuSIiIiLjwXDTTo3t44jBHraouqfBxqMZYpdDRERkNBhu2imJRIKlT/UGAOw+nYvs4gqRKyIiIjIODDft2DDvrhjVywFqjYANRzh7Q0RE1BwMN+3ckvvX3uxPy8PlglKRqyEiImr/GG7aOT83JSb6dYMgAOsOXRG7HCIionaP4cYIRIT6QioBjly6iZTsO2KXQ0RE1K4x3BiBno6d8Zy/GwBg7aHLEARB5IqIiIjaL4YbI7FgXC/IZVL8kHkbJ38pErscIiKidovhxki42lpixjAPAMDaQ+mcvSEiImoEw40RmRfSE1ZyGX6+rsK35wvELoeIiKhdYrgxIvadLTBnpBcAYN3hdNRqOHtDRET0MIYbIzNnlDdsrcxx9VYFYn+8LnY5RERE7Q7DjZGxUZjj1dE9AAAbjmSgWl0rckVERETtC8ONEXpxeHc42Vggr+Qudp3KEbscIiKidoXhxggpzGWYP9YHAPDJsV9QUa0WuSIiIqL2g+HGSE0LcIdnVysUldfgi++zxC6HiIio3WC4MVLmMikiQnsBAD5NzERJZY3IFREREbUPDDdGbPIAF/TuZo2yKjU2J1wVuxwiIqJ2geHGiEmlEiwZ7wsA+L+ka7hZWiVyRUREROJjuDFyT/Z2hL9nF1Td0+AfRzPELoeIiEh0DDdGTiL5dfZm9+lcZBdXiFwRERGRuBhuTMAw764Y1csBao2AD+OviF0OERGRqBhuTMTS+7M33/x0A5cLSkWuhoiISDwMNyaiv6sSk/ycIQjAukOcvSEioo6L4caELArtBakEOHLpJlKy74hdDhERkSgYbkxIT8fOeM7fDQCw9tBlCIIgckVERERtj+HGxCwY1wtymRQ/ZN7GyV+KxC6HiIiozTHcmBhXW0v8cZgnAGDtoXTO3hARUYfDcGOCXgvpASu5DD9fV+Hb8wVil0NERNSmGG5MkH1nC8wZ6QUAWHc4HepajcgVERERtR2GGxM1Z5Q3bK3McfVWBfal5oldDhERUZsRNdwkJiZi8uTJcHFxgUQiwf79+x+5TUJCAvz9/aFQKODt7Y0tW7YYvlAjZKMwx2tjegAANhzJQLW6VuSKiIiI2oao4aaiogIDBw7Exx9/3Kz+WVlZmDhxIoKDg5Gamorly5dj/vz52Lt3r4ErNU6zgrrDycYCeSV3setUjtjlEBERtQmJ0E5up5FIJNi3bx+mTp3aaJ8333wTBw4cwKVLl7Rtc+fOxU8//YTk5ORmfU9paSmUSiVUKhVsbGwet+x278tT2Vix7zy6dpIjcWkIOlmYiV0SERGR3vT5/Taqa26Sk5MRFham0zZ+/HicPXsW9+7da3Cb6upqlJaW6rw6kmkB7uje1QrFFTX44vssscshIiIyOKMKNwUFBXByctJpc3JyglqtRlFRwwvWRUVFQalUal/u7u5tUWq7YS6TYlFoLwDAp4mZKKmsEbkiIiIiwzKqcAPUnb76rQdn1R5ufyAyMhIqlUr7ys3NNXiN7c3kAS7o3c0aZVVqbE64KnY5REREBmVU4aZbt24oKNBdlK6wsBBmZmbo2rVrg9tYWFjAxsZG59XRSKUSLBnvCwDY8f013CytErkiIiIiwzGqcBMUFIT4+HidtsOHDyMgIADm5uYiVWUcnuztCH/PLqhWa/CPoxlil0NERGQwooab8vJypKWlIS0tDUDdrd5paWnIyam7bTkyMhKzZs3S9p87dy6ys7MRERGBS5cuYfv27di2bRsWL14sRvlGRSKRYOn92Zvdp3ORXVwhckVERESGIWq4OXv2LAYNGoRBgwYBACIiIjBo0CCsXLkSAJCfn68NOgDg5eWFuLg4HD9+HE888QTeeecdbNy4Ec8++6wo9RubQO+uGN3LAWqNgA/jr4hdDhERkUG0m3Vu2kpHW+fmYefzVPjdP05CIgH+uyAYvbt1vD8DIiIyPia7zg09vv6uSkzyc4YgAOsOcfaGiIhMD8NNBxQR1gsyqQRHLt1ESvYdscshIiJqVQw3HVAPh854brAbAGDtocvoYGcmiYjIxDHcdFDzx/lALpPih8zbOJHR8OrORERExojhpoNytbXEH4d5AgDWHkrn7A0REZkMhpsObF5ID3SSy3AuT4Vvzxc8egMiIiIjwHDTgXXtbIHZwd4AgHWH06Gu1YhcERER0eNjuOng5gR7wdbKHFdvVSA2NU/scoiIiB4bw00HZ6Mwx2tjegAAPjqSgWp1rcgVERERPR6GG8KsoO7oZqNAXsld7DqV8+gNiIiI2jGGG4LCXIb5Y30AAB8f/QUV1WqRKyIiImo5hhsCAPwhwA3du1qhuKIG209miV0OERFRizHcEADAXCbFotBeAICtiZkoqawRuSIiIqKWYbghrckDXNDH2QZl1WpsTrgqdjlEREQtwnBDWlKpBEvG183e7Pj+Gm6WVolcERERkf4YbkhHiK8j/D27oFqtwcbvMsQuh4iISG8MN6RDIpFg6XhfAEDMmVxkF1eIXBEREZF+GG6onkDvrhjdywFqjYAP46+IXQ4REZFeGG6oQUvuz95889MNXC4oFbkaIiKi5mO4oQb1d1Vikp8zBAFYdyhd7HKIiIiajeGGGhUR1gsyqQRHLhUiJfuO2OUQERE1C8MNNaqHQ2c8N9gNALD20GUIgiByRURERI/GcENNWjDOB3IzKX7IvI0TGUVil0NERPRIDDfUJBdbS8wc5gkAWHsonbM3RETU7jHc0CO9NqYHOsllOJenwrfnC8Quh4iIqEkMN/RIXTtbYHawNwBg3eF0qGs1IldERETUOIYbapb/CfZCFytzXL1VgdjUPLHLISIiahTDDTWLtcIcr43pCQD46EgGqtW1IldERETUMIYbaraZQZ7oZqNAXsldfPlDjtjlEBERNYjhhppNYS7D/LE+AIBPjv2Cimq1yBURERHVx3BDevlDgBu6d7VCcUUNtp/MErscIiKiehhuSC/mMikiwuoeqrk1MRN3KmpEroiIiEgXww3p7Xd+zujjbIOyajW2JFwVuxwiIiIdDDekN6lUgiXjewEAdiRdw83SKpErIiIi+hXDDbVIiK8jAjy7oFqtwcbvMsQuh4iISIvhhlpEIpFg6VO9AQAxZ3KRXVwhckVERER1GG6oxYZ62WF0LweoNQI+iL8idjlEREQAGG7oMS0ZX3fn1IGfbuBSfqnI1RARETHc0GPq76rEpAHOEARg/eF0scshIiJiuKHH90ZoL8ikEhy5VIiU7Ntil0NERB0cww09Nm+HznhusBsAYM236RAEQeSKiIioI2O4oVaxYJwP5GZSnMq6jRMZRWKXQ0REHRjDDbUKF1tLzBzmCQBYe4izN0REJB6GG2o1r43pgU5yGc7lqfDf8wVil0NERB0Uww21mq6dLTA72BsAsO5wOtS1GpErIiKijojhhlrV/wR7oYuVOTJvVSA2NU/scoiIqAMSPdxs2rQJXl5eUCgU8Pf3x4kTJ5rs/+WXX2LgwIGwsrKCs7Mz/vSnP6G4uLiNqqVHsVaY47UxPQEAHx3JQLW6VuSKiIiooxE13MTExGDhwoVYsWIFUlNTERwcjAkTJiAnJ6fB/idPnsSsWbMwe/ZsXLhwAV9//TXOnDmDOXPmtHHl1JSZQZ7oZqNAXsldfPlDw8eSiIjIUPQON3fv3kVlZaX2fXZ2NjZs2IDDhw/r/eUffPABZs+ejTlz5qBPnz7YsGED3N3dsXnz5gb7//DDD+jevTvmz58PLy8vjBw5En/+859x9uzZRr+juroapaWlOi8yLIW5DPPH+gAAPjn2C8qr1SJXREREHYne4WbKlCnYuXMnAKCkpASBgYFYv349pkyZ0mgoaUhNTQ1SUlIQFham0x4WFoakpKQGtxk+fDiuX7+OuLg4CIKAmzdvYs+ePZg0aVKj3xMVFQWlUql9ubu7N7tGark/BLihe1crFFfU4IuTWWKXQ0REHYje4ebHH39EcHAwAGDPnj1wcnJCdnY2du7ciY0bNzZ7P0VFRaitrYWTk5NOu5OTEwoKGr6NePjw4fjyyy8RHh4OuVyObt26wdbWFv/4xz8a/Z7IyEioVCrtKzc3t9k1UsuZy6SICKt7qObWxEzcqagRuSIiIuoo9A43lZWVsLa2BgAcPnwYzzzzDKRSKYYNG4bs7Gy9C5BIJDrvBUGo1/bAxYsXMX/+fKxcuRIpKSn49ttvkZWVhblz5za6fwsLC9jY2Oi8qG38zs8ZfZ1tUFatxpaEq2KXQ0REHYTe4aZnz57Yv38/cnNzcejQIe1ppcLCQr2Cg729PWQyWb1ZmsLCwnqzOQ9ERUVhxIgRWLJkCQYMGIDx48dj06ZN2L59O/Lz8/UdChmYVCrBkvF1szc7kq6hQFUlckVERNQR6B1uVq5cicWLF6N79+4IDAxEUFAQgLpZnEGDBjV7P3K5HP7+/oiPj9dpj4+Px/DhwxvcprKyElKpbskymQwAuNx/OzXG1wEBnl1QrdbgH0czxC6HiIg6AL3DzXPPPYecnBycPXsW3377rbZ97Nix+PDDD/XaV0REBD7//HNs374dly5dwqJFi5CTk6M9zRQZGYlZs2Zp+0+ePBmxsbHYvHkzMjMz8f3332P+/PkYOnQoXFxc9B0KtQGJRIKlT/UGAMScyUV2cYXIFRERkakza8lG3bp1Q7du3QAApaWlOHr0KHx9fdG7d2+99hMeHo7i4mKsXr0a+fn56N+/P+Li4uDpWfcAxvz8fJ01b1566SWUlZXh448/xhtvvAFbW1s8+eSTeP/991syDGojQ73sMMbXAcfTb+GD+Cv46Pnmz/ARERHpSyLoeT5n2rRpGDVqFF5//XXcvXsXAwcOxLVr1yAIAnbv3o1nn33WULW2itLSUiiVSqhUKl5c3IbO56nwu3+chEQCxM0PRh9n/tkTEVHz6fP7rfdpqcTERO2t4Pv27YMgCCgpKcHGjRvx97//vWUVk8nr76rEpAHOEARg/eF0scshIiITpne4UalUsLOzAwB8++23ePbZZ2FlZYVJkyYhI4MXjFLj3gjtBZlUgiOXCpGSfVvscoiIyETpHW7c3d2RnJyMiooKfPvtt9pbwe/cuQOFQtHqBZLp8HbojD/4uwEA1nybzjvciIjIIPQONwsXLsSMGTPg5uYGFxcXjBkzBkDd6So/P7/Wro9MzPyxPpCbSXEq6zYSM4rELoeIiEyQ3uHmtddeQ3JyMrZv346TJ09q153x9vbmNTf0SC62lpg5rO5uuLWHLnP2hoiIWp3ed0v91oNNG3tcQnvEu6XEV1xejVFrjqGiphabZgzGRD9nsUsiIqJ2zqB3SwHAzp074efnB0tLS1haWmLAgAH45z//2aJiqePp2tkCc4K9AQDrDqdDXasRuSIiIjIleoebDz74AK+++iomTpyIr776CjExMXjqqacwd+5cvVcopo5rTrAXuliZI/NWBWJ/zBO7HCIiMiF6n5by8vLC22+/rfNYBAD4v//7P6xatQpZWVmtWmBr42mp9uOzxEz8b9wluCgVOLZkDCzMZGKXRERE7ZRBT0vl5+c3+GDL4cOH88ncpJeZQZ7oZqPADVUVvvwh59EbEBERNYPe4aZnz5746quv6rXHxMTAx8enVYqijkFhLsOCcXX/znxy7BeUV6tFroiIiEyB3g/OfPvttxEeHo7ExESMGDECEokEJ0+exHfffddg6CFqynP+btiamImsogpsP5mF+WMZkImI6PHoPXPz7LPP4tSpU7C3t8f+/fsRGxsLe3t7nD59Gk8//bQhaiQTZi6TYlFoLwB11+DcqagRuSIiIjJ2j7XOjTHiBcXtj0Yj4Hf/OImL+aX48yhvRE7sI3ZJRETUzrT6BcWlpaXNfhHpSyqVYMl4XwDAjqRrKFBViVwREREZs2Zdc2Nra/vIVYgFQYBEIkFtbW2rFEYdyxhfBwzp3gVnrt3BxqMZePdpPqeMiIhaplnh5tixY4augzo4iUSCJeN7Y9qnyfjqTC5eCfZGd/tOYpdFRERGqFnhZvTo0YaugwhDvewwxtcBx9Nv4cMjV/DR84PELomIiIxQi54tRWQoi8Pqrr058NMNXMrnNVxERKQ/hhtqV/q7KvG7Ac4QBGDdoXSxyyEiIiPEcEPtTkRoL8ikEnx3uRAp2bfFLoeIiIwMww21O94OnfEHfzcAwJpv09HBlmIiIqLH1KJwo1arceTIEXz66acoKysDANy4cQPl5eWtWhx1XAvG+UBuJsWprNtIzCgSuxwiIjIieoeb7Oxs+Pn5YcqUKZg3bx5u3boFAFizZg0WL17c6gVSx+SstMSsYZ4AgLWHLkOj4ewNERE1j97hZsGCBQgICMCdO3dgaWmpbX/66afx3XfftWpx1LG9FtITneQynM8rxbcXCsQuh4iIjITe4ebkyZN46623IJfLddo9PT2Rl5fXaoUR2XWSY06wNwBg3eF0qGs1IldERETGQO9wo9FoGnzEwvXr12Ftbd0qRRE9MCfYC12szJF5qwKxPzI8ExHRo+kdbkJDQ7Fhwwbte4lEgvLycvztb3/DxIkTW7M2IlgrzDEvpCcAYMORK6i6x2eXERFR0/QONx9++CESEhLQt29fVFVVYfr06ejevTvy8vLw/vvvG6JG6uD+OMwTzkoFbqiqsOtUjtjlEBFROycRWrCIyN27dxEdHY0ff/wRGo0GgwcPxowZM3QuMG6vSktLoVQqoVKpYGNjI3Y51EzRp3MQGXsOXTvJkbA0BJ0tmvVYNCIiMhH6/H63KNwYM4Yb46Su1SD0w0RkFVUgIrQX5o/1EbskIiJqQ/r8fuv9198DBw402C6RSKBQKNCzZ094eXnpu1uiJpnJpIgI7YW/RKfis8RMzBzmiS6d5I/ekIiIOhy9w83UqVMhkUjqLYn/oE0ikWDkyJHYv38/unTp0mqFEk3yc8bm41dxMb8UWxKuInJiH7FLIiKidkjvC4rj4+MxZMgQxMfHQ6VSQaVSIT4+HkOHDsV//vMfJCYmori4mKsVU6uTSiVYMt4XALAj6RoKVFUiV0RERO2R3jM3CxYswNatWzF8+HBt29ixY6FQKPDKK6/gwoUL2LBhA15++eVWLZQIAMb4OmBI9y44c+0ONh7NwLtP+4ldEhERtTN6z9xcvXq1wQt5bGxskJmZCQDw8fFBUREfdkitTyKRYOlTvQEAX53JxbWiCpErIiKi9kbvcOPv748lS5ZoH5gJALdu3cLSpUsxZMgQAEBGRgbc3Nxar0qi3xjS3Q4hvg5QawR8eOSK2OUQEVE7o3e42bZtG7KysuDm5oaePXvCx8cHbm5uuHbtGj7//HMAQHl5Of7617+2erFED7wRVnftzYGfbuBSfqnI1RARUXvSonVuBEHAoUOHcOXKFQiCgN69eyM0NBRSqd5Zqc1xnRvT8fquH/Gfn/Mxtrcjtr00ROxyiIjIgLiIXxMYbkxHVlEFxn2QgFqNgD1zgxDQ3U7skoiIyEAMuogfAFRUVCAhIQE5OTmoqanR+Wz+/Pkt2SWR3rzsO2FagBuiT+dizaF0xLwyDBKJROyyiIhIZHqHm9TUVEycOBGVlZWoqKiAnZ0dioqKYGVlBUdHR4YbalPzx/pg7495OJ11G4kZRRjdy0HskoiISGR6XySzaNEiTJ48Gbdv34alpSV++OEHZGdnw9/fH+vWrTNEjUSNclZaYtYwTwDA2kOXodF0qLOsRETUAL3DTVpaGt544w3IZDLIZDJUV1fD3d0da9aswfLlyw1RI1GTXgvpic4WZjifV4r/ni8QuxwiIhKZ3uHG3Nxce12Dk5MTcnJyAABKpVL7z0Rtya6THHOC6x7Wuj4+HepajcgVERGRmPQON4MGDcLZs2cBACEhIVi5ciW+/PJLLFy4EH5++i+Fv2nTJnh5eUGhUMDf3x8nTpxosn91dTVWrFgBT09PWFhYoEePHti+fbve30umZfZIL3SxMkfmrQrE/pgndjlERCQivcPNu+++C2dnZwDAO++8g65du+LVV19FYWEhtm7dqte+YmJisHDhQqxYsQKpqakIDg7GhAkTmpwBmjZtGr777jts27YN6enpiI6ORu/evfUdBpkYa4U55oX0BABsOHIFVfdqRa6IiIjEotc6N4IgICcnB46OjrC0tHzsLw8MDMTgwYOxefNmbVufPn0wdepUREVF1ev/7bff4vnnn0dmZibs7Fq2pgnXuTFdVfdqEbLuOPJVVfjr7/pi9kgvsUsiIqJWos/vt14zN4IgwMfHB9evX3+sAgGgpqYGKSkpCAsL02kPCwtDUlJSg9scOHAAAQEBWLNmDVxdXdGrVy8sXrwYd+/ebfR7qqurUVpaqvMi06Qwl2HBWB8AwKZjv6C8Wi1yRUREJAa9wo1UKoWPjw+Ki4sf+4uLiopQW1sLJycnnXYnJycUFDR8x0tmZiZOnjyJ8+fPY9++fdiwYQP27NmDefPmNfo9UVFRUCqV2pe7u/tj107t13P+bvCy74TiihpsP5kldjlERCQCva+5WbNmDZYsWYLz58+3SgEPrygrCEKjq8xqNBpIJBJ8+eWXGDp0KCZOnIgPPvgAO3bsaHT2JjIyEiqVSvvKzc1tlbqpfTKTSRER2gsA8FliJu5U1DxiCyIiMjV6h5s//vGPOH36NAYOHAhLS0vY2dnpvJrL3t4eMpms3ixNYWFhvdmcB5ydneHq6gqlUqlt69OnDwRBaPRUmYWFBWxsbHReZNom+Tmjr7MNyqrV2JxwVexyiIiojen9+IUNGza0yhfL5XL4+/sjPj4eTz/9tLY9Pj4eU6ZMaXCbESNG4Ouvv0Z5eTk6d+4MALhy5QqkUinc3NxapS4yflKpBEue8sWfvjiD/0u6hpdHeKGbUiF2WURE1EZEfSp4TEwMZs6ciS1btiAoKAhbt27FZ599hgsXLsDT0xORkZHIy8vDzp07AQDl5eXo06cPhg0bhrfffhtFRUWYM2cORo8ejc8++6xZ38m7pToGQRAQ/ukPOH3tNqYHeuDdp/Vfg4mIiNoPg90t9cDVq1fx1ltv4YUXXkBhYSGAutu0L1y4oNd+wsPDsWHDBqxevRpPPPEEEhMTERcXB0/PumcF5efn66x507lzZ8THx6OkpAQBAQGYMWMGJk+ejI0bN7ZkGGTCJJK62RsA+OpMLq4VVYhcERERtRW9Z24SEhIwYcIEjBgxAomJibh06RK8vb2xZs0anD59Gnv27DFUra2CMzcdy5++OI1j6bfw+4Eu2PjCILHLISKiFjLozM2yZcvw97//HfHx8ZDL5dr2kJAQJCcn618tkQEtHl83e3Pgpxu4eINrHBERdQR6h5tz587pXAD8gIODQ6usf0PUmvq5KDF5oAsAYP3hdJGrISKitqB3uLG1tUV+fn699tTUVLi6urZKUUStKSK0F2RSCb67XIiz126LXQ4RERmY3uFm+vTpePPNN1FQUACJRAKNRoPvv/8eixcvxqxZswxRI9Fj8bLvhGkBdUsFrDmUDhFvECQiojagd7j53//9X3h4eMDV1RXl5eXo27cvRo0aheHDh+Ott94yRI1Ej23+WB/IzaQ4nXUbiRlFYpdDREQG1OJ1bq5evYrU1FRoNBoMGjQIPj4+rV2bQfBuqY7rfw9exGcnstDf1QYH5o2EVNrwYz6IiKj90ef3W+8VihMSEjB69Gj06NEDPXr0aHGRRG3t1TE9EX06F+fzSvHf8wWYNMBZ7JKIiMgA9D4tFRoaCg8PDyxbtqzVHp5J1BbsOskxJ9gLALA+Ph3qWo3IFRERkSHoHW5u3LiBpUuX4sSJExgwYAAGDBiANWvWNPrgSqL2ZE6wN+w6yZF5qwKxP+aJXQ4RERmA3uHG3t4er7/+Or7//ntcvXoV4eHh2LlzJ7p3744nn3zSEDUStdimTZvg5eUFhUIBf39/pJ5Oxmtj6k6nbjhyBVX3arV9jx8/DolEUu91+fJlnX3u3bsXffv2hYWFBfr27Yt9+/a16ZiIiKhpLXq21ANeXl5YtmwZ3nvvPfj5+SEhIaG16iJ6bDExMVi4cCFWrFiB1NRUBAcHY8KECRjlIoGzUoEbqip8eSqn3nbp6enIz8/Xvn57sXxycjLCw8Mxc+ZM/PTTT5g5cyamTZuGU6dOteXQiIioCS2+W+r777/Hl19+iT179qCqqgq///3vMWPGDEyYMKG1a2xVvFuq4wgMDMTgwYOxefNmbVufPn0wdepUDHz6VSyLPQe7TnIkLg1BZwszHD9+HCEhIbhz5w5sbW0b3Gd4eDhKS0vx3//+V9v21FNPoUuXLoiOjjb0kIiIOiyDPltq+fLl8PLywpNPPons7Gxs2LABBQUF+Ne//tXugw11HDU1NUhJSUFYWJhOe1hYGJKSkvCcvxu87TvhdkUNtp/M0ukzaNAgODs7Y+zYsTh27JjOZ8nJyfX2OX78eCQlJRlmIEREpDe9w83x48exePFi5OXl4eDBg5g+fTqsrKwMURtRixUVFaG2thZOTk467U5OTigoKICZTIqIsF4AgM8SM3GnogbOzs7YunUr9u7di9jYWPj6+mLs2LFITEzUbl9QUNDoPomIqH3Qe50b/g2VjIlEortQnyAI2raJ/Z3R1/kqLuaXYnPCVSyf2Ae+vr7avkFBQcjNzcW6deswatSoZu2TiIjEp3e4eeDixYvIyclBTU2NTvvvf//7xy6K6HHZ29tDJpPVm1EpLCzUzrxIpRIsecoXf/riDP4v6RpeHuGFbkqFTv9hw4bhX//6l/Z9t27dmtwnERGJT+9wk5mZiaeffhrnzp2DRCLRPoTwwd9ca2trm9qcqE3I5XL4+/sjPj4eTz/9tLY9Pj4eU6ZM0b4f08sBQ7vb4fS12/jouwxEPeOns5/U1FQ4O/+6knFQUBDi4+OxaNEibdvhw4cxfPhwA46GiIj0oXe4WbBgAby8vHDkyBF4e3vj9OnTKC4uxhtvvIF169YZokaiFomIiMDMmTMREBCAoKAgbN26FTk5OZg7dy4AIDIyEnl5eViyegP+sCUZn236B7zKn0RIkD9qamrwr3/9C3v37sXevXu1+1ywYAFGjRqF999/H1OmTME333yDI0eO4OTJk2INk4iIHqJ3uElOTsbRo0fh4OAAqVQKqVSKkSNHIioqCvPnz0dqaqoh6iTSW3h4OIqLi7F69Wrk5+ejf//+iIuLg6enJwAgPz8fOTk5GNLdDiG+Dth/6h6WLFmCKlURrCwt0b9/Pxw8eBATJ07U7nP48OHYvXs33nrrLfz1r39Fjx49EBMTg8DAQLGGSURED9F7nZsuXbogJSUF3t7e6NGjBz7//HOEhITg6tWr8PPzQ2VlpaFqbRVc54YacuGGCpM2/jr7IpUAUc/4IXyIh4hVERHRAwZ9Knj//v3x888/w9vbG4GBgVizZg3kcjm2bt0Kb2/vFhdNJCa7TnKd9xoBiIw9h1G9HOCstBSpKiIiagm917l56623oNHUPU3573//O7KzsxEcHIy4uDhs3Lix1QskagtZRRX12jQCMP2zU4g+nYOKarUIVRERUUu0+PELv3X79m106dLFKNb64Gkpaki+6i5GvHcUmkb+a+hsYYapg1wwfagn+rrw3xsioramz+93q4QbY8JwQ42JOZOD5bHnUSsIkEkkWDGpD9QaDaJP5+rM7DzhbosZgR743QAXWMplIlZMRNRxMNw0geGGmpKvuotrRZXobm+lvdZGEAQkXy3Gl6dzcOh8AdT3p3esFWZ4drAbpgd6oJeTtZhlExGZPIabJjDc0OO4VVaNr1NyEX06B7m372rbh3TvgumBHpjQ3xkKc87mEBG1NoabJjDcUGvQaASc+KUIu05l48ilQtTen82xtTLHc4Pd8EKgB3o4dBa5SiIi08Fw0wSGG2ptN0urEHMmF7tP5+CGqkrbPszbDjMCPTG+XzfIzfS+MZGIiH6D4aYJDDdkKLUaAQlXCvHlDzk4ll6ovfOqayc5/hDgjheGusOzaydxiyQiMlIMN01guKG2kFdyFzFnchFzJgc3S6u17cE+9pgR6IGxfZxgLuNsDhFRczHcNIHhhtqSulaD7y4XYtepHCRm3MKD/9ocrC0QHuCO54e6w62LlbhFEhEZAYabJjDckFhyb1ci+nQOvjp7HUXldbM5EgkwppcDpgd6IsTXAWaczSEiahDDTRMYbkhsNWoN4i/exK7T2fj+l2Jtu7NSgfAh7ggf4s7nWRERPYThpgkMN9SeZBVVIPp0DvakXMftihoAdU8kf7K3E2YM88AoHwfIpO3/sSZERIbGcNMEhhtqj6rVtfj2fAF2ncrBqazb2nZXW0u8MNQd0wLc4WijELFCIiJxMdw0geGG2rtfCsuw61Qu9qTkorSq7mnkZlIJQvs6YXqgB0b0sIeUszlE1MEw3DSB4YaMRdW9Whz8OR+7TucgJfuOtt2zqxVeGOqB5/zdYN/ZQsQKiYjaDsNNExhuyBhdLijFrlM52PdjHsqq62ZzzGUSjO/XDTMCPTHM2w4SCWdziMh0Mdw0geGGjFlljRr/+SkfX57Kxk/XVdp2b4dOmH5/NsfWSi5ihUREhsFw0wSGGzIV5/NU2HU6B9+k5qGiphYAIDeTYpKfM6YHeiDAswtnc4jIZDDcNIHhhkxNebUa36TlYdepHFy4Uapt7+XUGdOHeuDpwW5QWpqLWCER0eNjuGkCww2ZKkEQ8NN1FXadysaBn26g6p4GAKAwl2LyABdMD/TAE+62nM0hIqPEcNMEhhvqCFR372F/at1sTvrNMm17H2cbTA/0wNQnXGCt4GwOERkPhpsmMNxQRyIIAn7MuYMvf8jBf87lo0ZdN5tjJZdhyhMumD7UE35uSpGrJCJ6NIabJjDcUEdVUlmDvT/mYdepbFy9VaFtH+CmxPShHvj9Ey6wkpuJWCERUeP0+f0W/RHEmzZtgpeXFxQKBfz9/XHixIlmbff999/DzMwMTzzxhGELJDIRtlZyzB7phSMRo7H7lWH4/UAXmMsk+Pm6CstizyHwf7/DX/efx6X80kfvjIioHRN15iYmJgYzZ87Epk2bMGLECHz66af4/PPPcfHiRXh4eDS6nUqlwuDBg9GzZ0/cvHkTaWlpzf5OztwQ/aq4vBp7Uq4j+nQOrhVXatsHedhiRqAnfjfAGQpzmYgVEhHVMZrTUoGBgRg8eDA2b96sbevTpw+mTp2KqKioRrd7/vnn4ePjA5lMhv379zPcED0mjUZA0tVi7DqdjcMXbkKtqfvfgo3CDM8MdsOMQA/4OFmLXCURdWRGcVqqpqYGKSkpCAsL02kPCwtDUlJSo9t98cUXuHr1Kv72t78163uqq6tRWlqq8yIiXVKpBCN97LFphj+SIp/EkvG+cOtiidIqNXYkXUPoh4mYtiUZ36TloVpdK3a5RERNEu3qwaKiItTW1sLJyUmn3cnJCQUFBQ1uk5GRgWXLluHEiRMwM2te6VFRUXj77bcfu16ijsLRWoF5IT3x6ugeSMy4hV2ncvDd5UKcvnYbp6/dRhcrczzn74YXhnrA26Gz2OUSEdUj+q0RDy8oJghCg4uM1dbWYvr06Xj77bfRq1evZu8/MjISERER2velpaVwd3dvecFEHYRUKsEYX0eM8XVEgaoKMWdysftMDvJVVfjsRBY+O5GF4T26YnqgB8L6doPcTPT7E4iIAIgYbuzt7SGTyerN0hQWFtabzQGAsrIynD17FqmpqXj99dcBABqNBoIgwMzMDIcPH8aTTz5ZbzsLCwtYWFgYZhBEHUQ3pQILxvlgXkgPHE+/hV2nc3AsvRBJV4uRdLUY9p3l+EOAO14Y4gGPrlZil0tEHZzoFxT7+/tj06ZN2ra+fftiypQp9S4o1mg0uHjxok7bpk2bcPToUezZswdeXl7o1KnTI7+TFxQTtY7rdyoRcyYXMWdyUVhWrW0P9rHHjEAPjO3jBHMZZ3OIqHXo8/st6mmpiIgIzJw5EwEBAQgKCsLWrVuRk5ODuXPnAqg7pZSXl4edO3dCKpWif//+Ots7OjpCoVDUayciw3PrYoU3wnwxf6wPvrtUiC9PZeNERpH25WhtgfAh7nh+qAdcbS3FLpeIOhBR/1oVHh6ODRs2YPXq1XjiiSeQmJiIuLg4eHp6AgDy8/ORk5MjZolE9AjmMime6t8N/5wdiMQlIXh1TA/Yd5ajsKwa/zj6C4LfP4qXd5zBkYs3UavRf6JYn4U+T548iREjRqBr166wtLRE79698eGHHzbaf/fu3ZBIJJg6daredRFR+8XHLxBRq6tRa3D4YgF2ncpB0tVibbuLUoHwIR4IH+KObkrFI/ej70KfqampuHz5MgYMGIBOnTrh5MmT+POf/4wPP/wQr7zyik7f7OxsjBgxAt7e3rCzs8P+/fsfe9xEZDhGs4ifGBhuiNpW5q1yRJ/OwZ6U67hTeQ8AIJNK8GRvR8wI9MAoHwdIpfXvkARavtDnbz3zzDPo1KkT/vnPf2rbamtrMXr0aPzpT3/CiRMnUFJSwnBD1M4ZxSJ+RNQxeDt0xopJfZEcORYbwp/A0O52qNUIiL94Ey99cQaj1h7DJ8d+QWFZlc52LV3o87dSU1ORlJSE0aNH67SvXr0aDg4OmD179uMNjojaJdHXuSGijkFhLsPUQa6YOsgVGTfLsOt0DvamXMf1O3ex9lA6Poy/grB+Tpg+1BPDe3Rt0UKfD7i5ueHWrVtQq9VYtWoV5syZo/3s+++/x7Zt2/R6bAsRGReGGyJqcz5O1vjb5H5YOr43Dp7Lx65T2fgxpwRx5woQd64A3btaYYJ33fpUzV3o87dOnDiB8vJy/PDDD1i2bBl69uyJF154AWVlZfjjH/+Izz77DPb29gYbHxGJi+GGiERjKZfhOX83POfvhkv5pdh1Kgf7UvNwrbgSmwpVgESKv3+dhL869UKglx0kEkmjC33+lpeXFwDAz88PN2/exKpVq/DCCy/g6tWruHbtGiZPnqztq9FoAABmZmZIT09Hjx49DDdgImoTDDdE1C70cbbBO1P7Y9mE3vj3Tzew63QOCrr1ROLxo3he3hs9HDrhhaEeOHT4MJ7W49ZtQRBQXV23yGDv3r1x7tw5nc/feustlJWV4aOPPuKjWYhMBMMNEbUrnSzM8PxQDzw/1ANrpcuwbMErsHbrhcuOvojYvQHlV6/hlmswzl67jT1b1uDGjRvYuXMnAOCTTz6Bh4cHevfuDaBu3Zt169bhL3/5CwA0uOinra0tAHAxUCITwnBDRO3Wktf+hE64i/ffX4Mb+fmwcuoOxz+swpHrAo5sSca9oz/BWn0HpVX3YKMwx52Kaqx/YykK8nJgbmaGHj164L333sOf//xnsYdCRG2I69wQkdEQBAFpuSXYdSoH//75Bqru1V0vozCXor+LEik5dyAIgFQCRD3jh/Ah9Rf6IyLjxEX8msBwQ2QaVHfvYd+P17HrdA6u3Cyv97lEAkT/zzAM7W7X6CKBRGQ8GG6awHBDZFoEQcCOpGt4+98XG/zc2sIM/V2VGOCmxAA3WwxwU8Kti+UjbycnovbFaJ4KTkT0uCQSCZ7q3w3v/OciHn4up1wmQVm1GsmZxUjO/PUZV12szOHnZosBvwk9TjYWDDxEJoIzN0RkEmLO5GB57HnUCgJkEgnefaY/nh3shozCcvx8vQQ/X1fh5+sqXC4oxb3a+v/bc7C2wEA3Jfxcbe8HHiW6drYQYSRE1BCelmoCww2R6cpX3cW1okp0t7eCs9KywT7V6lqkF5Thp+sqnLsfejIKy1H78LQPAFdbSwxwU8LPTYkBrrbwc1NCaWlu6GEQUQMYbprAcENED7tbU4uL+Sr8lKvCuTwVfr5egsyiCjT0f8fuXa201+74uSrR31WJThY8w09kaAw3TWC4IaLmKKu6h/N5pXWntPJUOHddhZzblfX6SSRAT4fOvwYeNyX6OttAYS4ToWoi08Vw0wSGGyJqqTsVNdqZnZ+v183y5Kuq6vUzk0rQy8kaA91/vYbHt5s1zGVSEaomMg0MN01guCGi1lRYWoVzeSqda3iKK2rq9ZObSdHH2eb+Rct1d2j1dOwMGdfgIWoWhpsmMNwQkSEJgoAbqiqcu15yP/DUzfSUVqnr9bU0l6G/q43ONTzdu3biooNEDWC4aQLDDRG1NUEQkF1cef/anbrQcz5Phcqa2np9rRVm2pmdB4GHiw4SMdw0ieGGiNqDWo2AzFvl2mt3frpegos3SlGt1tTra9dJDj9XZd0prfuhx8lGIULVROJhuGkCww0RtVf3ajW4crOs7lTW/QuXL+eXQd3AGjxONhbwc7W9H3jqZnrsOslFqPrRNm3ahLVr1yI/Px/9+vXDhg0bEBwc3GDf2NhYbN68GWlpaaiurka/fv2watUqjB8/XqdfSUkJVqxYgdjYWNy5cwdeXl5Yv349Jk6c2BZDIhHw8QtEREbIXCZFPxcl+rko8fz9tqp7tbhcUKa9WLlu0cEy3Cytxs3Smzhy6aZ2e7cu9xcdvB96+rspYaMQd9HBmJgYLFy4EJs2bcKIESPw6aefYsKECbh48SI8POo/tT0xMRGhoaF49913YWtriy+++AKTJ0/GqVOnMGjQIABATU0NQkND4ejoiD179sDNzQ25ubmwtrZu6+FRO8WZGyIiI1NZo8bFG6U6d2hlFlU02NfbvhP87l+7M9DdFv1cbGAlb7u/1wYGBmLw4MHYvHmztq1Pnz6YOnUqoqKimrWPfv36ITw8HCtXrgQAbNmyBWvXrsXly5dhbs4VozsKztwQEZkwK7kZArrbIaC7nbattOoezufVzeycu153Dc/1O3eRWVSBzKIKfJN2AwAglQA+jtbwc/v1Gp7e3awNsuhgTU0NUlJSsGzZMp32sLAwJCUlNWsfGo0GZWVlsLP7dawHDhxAUFAQ5s2bh2+++QYODg6YPn063nzzTchkXDyRGG6IiEyCjcIcw3vYY3gPe23b7Yoa/Hy9ROcanpul1Ui/WYb0m2XYk3IdAGAuk8C3m7XOQ0N7OT3+ooNFRUWora2Fk5OTTruTkxMKCgqatY/169ejoqIC06ZN07ZlZmbi6NGjmDFjBuLi4pCRkYF58+ZBrVZrZ3eoY2O4ISIyUXad5Bjj64gxvo7atpulVdq1d36+P9Nzu6IG5/NKcT6vFNGn6/pZmEnR18UGA1zrZncGuinh7dCyRQcfvo1dEIRm3doeHR2NVatW4ZtvvoGj469j0Gg0cHR0xNatWyGTyeDv748bN25g7dq1DDcEgOGGiKhDcbJRwKmvAuP61s2mCIKAvJK7909lqXAur+4anrIqNVJzSpCaUwIgGwDQSS5DP1fl/cCjxEA3W3h2tWo0qNjb20Mmk9WbpSksLKw3m/OwmJgYzJ49G19//TXGjRun85mzszPMzc11TkH16dMHBQUFqKmpgVzePu8ao7bDcENE1IFJJBK4dbGCWxcrTPBzBgBoNAKyb1f++gyt6yqcv6FCRU0tTmfdxums29rtbRRmGOBmW3c7uqsSA9xt4aJUQCKRQC6Xw9/fH/Hx8Xj66ae128THx2PKlCmN1hQdHY2XX34Z0dHRmDRpUr3PR4wYgV27dkGj0UAqrTt1duXKFTg7OzPYEADeLSV2OURERqFWI+Dq/UUHH4Sei/mlqGlg0cGuneT3n5Bui9s/H8OayL9gy5YtCAoKwtatW/HZZ5/hwoUL8PT0RGRkJPLy8rBz504AdcFm1qxZ+Oijj/DMM89o92lpaQmlUgkAyM3NRd++ffHSSy/hL3/5CzIyMvDyyy9j/vz5WLFiRdv8gXQgrb1OUWxsLN5991388ssvuHfvHnx8fPDGG29g5syZTdbBRfyawHBDRNQ67tVqkF5QpvOk9PSC+osOlv14EOVnYqEuvw1Xr15Y/Ld3MXPqU+jSSY6XXnoJ165dw/HjxwEAw0eOQvL3J+p914svvogdO3Zo3ycnJ2PRokVIS0uDq6srZs+ezbulDCAmJgYzZ87UWafo888/b3SdooULF8LFxQUhISHadYrWrVuns07R8ePHcefOHfTu3RtyuRz/+c9/8MYbb+DgwYP1Fmv8LYabJjDcEBEZTtW9WlzKL617pERu3TU8vxSWo4FFluFuZ4kB9+/Q8nNTIuNmGd7+90VohLpb1qOe8UP4kPo/oNR2DLFOUUMGDx6MSZMm4Z133mm0D9e5ISIiUSjMZRjk0QWDPLoAQXVtFdVqXLhR+us1PHkqZBVVIPf2XeTevouD5/Lr7UcjAMv2nkNWUQXculjB1socSkvdl7XCvEV3b1HzGGqdot8SBAFHjx5Feno63n///ceu+QGGGyIiMqhOFmYY6mWHoV6//sCp7v666ODP10twJus2iipqdLYTAGxJyGx0vxIJ0NnCTBt2fhuAbCzrhyGlpTlsLeX3g5EZpAxGTTLUOkUAoFKp4OrqiurqashkMmzatAmhoaGtVjvDDRERtTmlpTlG9LTHiJ51iw7mq+5ixHtHdU5fSQBM8OuGe7UCVHfvofTuPajuvypraiEIQFmVGmVValy/c1ev75dIAGsLMygbmBGy+U0IaujV0YJRa69TBADW1tZIS0tDeXk5vvvuO0RERMDb2xtjxoxplZoZboiISHTOSktEPeOH5bHnUSsIkEkkePeZ/o1ec1Oj1miDzsPBp96rUvf93Xt1wai0So3SKjVy0bJgZGslrxeKGpwtsvr1c2sL4wlGhlqnCACkUil69uwJAHjiiSdw6dIlREVFMdwQEZFpCR/igVG9HHCtqBLd7a3grLRstK/cTAoHaws4WFvo/T2NBaOSyhqo7qqbDE0PByN9SSWAtaKBENTADNLDwamtg5Gh1ilqiCAIqK6ufuyaH2C4ISKidsNZadlkqGkNjxOMqtW1Dc8UVd7TBqOSuzUNziRV3dNAI0D7Xl9SCerNDjU0W2T78GdWdcGoOaeSHhYREYGZM2ciICBAu05RTk4O5s6dCwBNrlM0bNgw7azPb9cpioqKQkBAAHr06IGamhrExcVh586dOndkPS6GGyIiomayMJPB0VoGR2uF3ts+HIxKKhs+jfbbYPSgT7W6LhiVVNa16auhYPSomSJbK3NMnPIMPvywCKtXr0Z+fj769++PuLg4eHp6AgDy8/ORk5Oj/Z5PP/0UarUa8+bNw7x587Ttv12nqKKiAq+99hquX78OS0tL9O7dG//6178QHh6u97gaw3VuiIiI2rmqe7VNXldUUtn4dUfVDawirQ+ZVAIbhVmjs0W/vUstJfsOPj+ZBcEAaxVxnRsiIiITojCXQWEug6ON/jNGD4JRSSMXWNebLfrNP9eoNajVCLhTeQ939Jwx0gjA8tjzGNXLweCnGh/GcENERGTCHjcYNXTnWUkDoej6nUpcuVmus32tIOBaUSXDDREREbUPD4KRUzOCUUNrFckkEnS3tzJghQ2Ttvk3EhERkcl5sFaR7P5dWQ/WKmrrWRuAMzdERETUSvRZq8iQGG6IiIio1bTFWkWPIvppqU2bNsHLywsKhQL+/v44ceJEo31jY2MRGhoKBwcH2NjYICgoCIcOHWrDaomIiKi9EzXcxMTEYOHChVixYgVSU1MRHByMCRMm6CwI9FuJiYkIDQ1FXFwcUlJSEBISgsmTJyM1NbWNKyciIqL2StRF/AIDAzF48GCdJZf79OmDqVOnIioqqln76NevH8LDw7Fy5coGP6+urtZ5XkVpaSnc3d25iB8REZER0WcRP9FmbmpqapCSkoKwsDCd9rCwMCQlJTVrHxqNBmVlZbCzs2u0T1RUFJRKpfbl7u7+WHUTERFR+yZauCkqKkJtbW29x6Y7OTnVe7x6Y9avX4+KigpMmzat0T6RkZFQqVTaV25u7mPVTURERO2b6HdLPfyUUkEQmvXk0ujoaKxatQrffPMNHB0dG+1nYWEBCwv9n/xKRERExkm0cGNvbw+ZTFZvlqawsLDebM7DYmJiMHv2bHz99dcYN26cIcskIiIiIyPaaSm5XA5/f3/Ex8frtMfHx2P48OGNbhcdHY2XXnoJu3btwqRJkwxdJhERERkZUU9LRUREYObMmQgICEBQUBC2bt2KnJwczJ07F0Dd9TJ5eXnYuXMngLpgM2vWLHz00UcYNmyYdtbH0tISSqVStHEQERFR+yFquAkPD0dxcTFWr16N/Px89O/fH3FxcfD09AQA5Ofn66x58+mnn0KtVmPevHmYN2+etv3FF1/Ejh072rp8IiIiaodEXedGDPrcJ09ERETtg1Gsc0NERERkCAw3REREZFIYboiIiMikMNwQERGRSWG4ISIiIpPCcENEREQmheGGiIiITArDDREREZkUhhsiIiIyKQw3REREZFIYboiIiMikMNwQERGRSWG4ISIiIpPCcENEREQmheGGiIiITArDDREREZkUhhsiIiIyKQw3REREZFIYboiIiMikMNwQERGRSWG4ISIiIpPCcENEREQmheGGiIiITArDDREREZkUhhsiIiIyKQw3REREZFIYboiIiMikMNwQERGRSWG4ISIiIpPCcENEREQmheGGiIiITArDDREREZkUhhsiIiIyKQw3REREZFIYboiIiMikMNwQERGRSWG4ISIiIpPCcENEREQmheGGiIiITArDDREREZkUhhsiIiIyKQw3REREZFIYboiIiMikMNwQERGRSWG4ISIiIpMierjZtGkTvLy8oFAo4O/vjxMnTjTZPyEhAf7+/lAoFPD29saWLVvaqFIiIiIyBqKGm5iYGCxcuBArVqxAamoqgoODMWHCBOTk5DTYPysrCxMnTkRwcDBSU1OxfPlyzJ8/H3v37m3jyomIiKi9kgiCIIj15YGBgRg8eDA2b96sbevTpw+mTp2KqKioev3ffPNNHDhwAJcuXdK2zZ07Fz/99BOSk5Ob9Z2lpaVQKpVQqVSwsbF5/EEQERGRwenz+23WRjXVU1NTg5SUFCxbtkynPSwsDElJSQ1uk5ycjLCwMJ228ePHY9u2bbh37x7Mzc3rbVNdXY3q6mrte5VKBaDuD4mIiIiMw4Pf7ebMyYgWboqKilBbWwsnJyeddicnJxQUFDS4TUFBQYP91Wo1ioqK4OzsXG+bqKgovP322/Xa3d3dH6N6IiIiEkNZWRmUSmWTfUQLNw9IJBKd94Ig1Gt7VP+G2h+IjIxERESE9r1Go8Ht27fRtWvXJr+nJUpLS+Hu7o7c3FyTPOVl6uMDTH+MHJ/xM/UxcnzGz1BjFAQBZWVlcHFxeWRf0cKNvb09ZDJZvVmawsLCerMzD3Tr1q3B/mZmZujatWuD21hYWMDCwkKnzdbWtuWFN4ONjY3J/ksLmP74ANMfI8dn/Ex9jByf8TPEGB81Y/OAaHdLyeVy+Pv7Iz4+Xqc9Pj4ew4cPb3CboKCgev0PHz6MgICABq+3ISIioo5H1FvBIyIi8Pnnn2P79u24dOkSFi1ahJycHMydOxdA3SmlWbNmafvPnTsX2dnZiIiIwKVLl7B9+3Zs27YNixcvFmsIRERE1M6Ies1NeHg4iouLsXr1auTn56N///6Ii4uDp6cnACA/P19nzRsvLy/ExcVh0aJF+OSTT+Di4oKNGzfi2WefFWsIOiwsLPC3v/2t3mkwU2Hq4wNMf4wcn/Ez9TFyfMavPYxR1HVuiIiIiFqb6I9fICIiImpNDDdERERkUhhuiIiIyKQw3BAREZFJYbjR06ZNm+Dl5QWFQgF/f3+cOHGiyf4JCQnw9/eHQqGAt7c3tmzZ0kaVtow+4zt+/DgkEkm91+XLl9uw4uZLTEzE5MmT4eLiAolEgv379z9yG2M7fvqO0ZiOYVRUFIYMGQJra2s4Ojpi6tSpSE9Pf+R2xnQMWzJGYzqGmzdvxoABA7SLuwUFBeG///1vk9sY0/HTd3zGdOwaEhUVBYlEgoULFzbZT4xjyHCjh5iYGCxcuBArVqxAamoqgoODMWHCBJ3b1X8rKysLEydORHBwMFJTU7F8+XLMnz8fe/fubePKm0ff8T2Qnp6O/Px87cvHx6eNKtZPRUUFBg4ciI8//rhZ/Y3t+AH6j/EBYziGCQkJmDdvHn744QfEx8dDrVYjLCwMFRUVjW5jbMewJWN8wBiOoZubG9577z2cPXsWZ8+exZNPPokpU6bgwoULDfY3tuOn7/geMIZj97AzZ85g69atGDBgQJP9RDuGAjXb0KFDhblz5+q09e7dW1i2bFmD/ZcuXSr07t1bp+3Pf/6zMGzYMIPV+Dj0Hd+xY8cEAMKdO3faoLrWBUDYt29fk32M7fg9rDljNOZjWFhYKAAQEhISGu1j7MewOWM05mMoCILQpUsX4fPPP2/wM2M/foLQ9PiM9diVlZUJPj4+Qnx8vDB69GhhwYIFjfYV6xhy5qaZampqkJKSgrCwMJ32sLAwJCUlNbhNcnJyvf7jx4/H2bNnce/ePYPV2hItGd8DgwYNgrOzM8aOHYtjx44Zssw2ZUzH73EZ4zFUqVQAADs7u0b7GPsxbM4YHzC2Y1hbW4vdu3ejoqICQUFBDfYx5uPXnPE9YGzHbt68eZg0aRLGjRv3yL5iHUOGm2YqKipCbW1tvYd6Ojk51XuY5wMFBQUN9ler1SgqKjJYrS3RkvE5Oztj69at2Lt3L2JjY+Hr64uxY8ciMTGxLUo2OGM6fi1lrMdQEARERERg5MiR6N+/f6P9jPkYNneMxnYMz507h86dO8PCwgJz587Fvn370Ldv3wb7GuPx02d8xnbsAGD37t348ccfERUV1az+Yh1DUR+/YIwkEonOe0EQ6rU9qn9D7e2FPuPz9fWFr6+v9n1QUBByc3Oxbt06jBo1yqB1thVjO376MtZj+Prrr+Pnn3/GyZMnH9nXWI9hc8dobMfQ19cXaWlpKCkpwd69e/Hiiy8iISGh0QBgbMdPn/EZ27HLzc3FggULcPjwYSgUimZvJ8Yx5MxNM9nb20Mmk9WbxSgsLKyXSh/o1q1bg/3NzMzQtWtXg9XaEi0ZX0OGDRuGjIyM1i5PFMZ0/FpTez+Gf/nLX3DgwAEcO3YMbm5uTfY11mOozxgb0p6PoVwuR8+ePREQEICoqCgMHDgQH330UYN9jfH46TO+hrTnY5eSkoLCwkL4+/vDzMwMZmZmSEhIwMaNG2FmZoba2tp624h1DBlumkkul8Pf3x/x8fE67fHx8Rg+fHiD2wQFBdXrf/jwYQQEBMDc3NxgtbZES8bXkNTUVDg7O7d2eaIwpuPXmtrrMRQEAa+//jpiY2Nx9OhReHl5PXIbYzuGLRljQ9rrMWyIIAiorq5u8DNjO34NaWp8DWnPx27s2LE4d+4c0tLStK+AgADMmDEDaWlpkMlk9bYR7Rga9HJlE7N7927B3Nxc2LZtm3Dx4kVh4cKFQqdOnYRr164JgiAIy5YtE2bOnKntn5mZKVhZWQmLFi0SLl68KGzbtk0wNzcX9uzZI9YQmqTv+D788ENh3759wpUrV4Tz588Ly5YtEwAIe/fuFWsITSorKxNSU1OF1NRUAYDwwQcfCKmpqUJ2drYgCMZ//ARB/zEa0zF89dVXBaVSKRw/flzIz8/XviorK7V9jP0YtmSMxnQMIyMjhcTERCErK0v4+eefheXLlwtSqVQ4fPiwIAjGf/z0HZ8xHbvGPHy3VHs5hgw3evrkk08ET09PQS6XC4MHD9a5RfPFF18URo8erdP/+PHjwqBBgwS5XC50795d2Lx5cxtXrB99xvf+++8LPXr0EBQKhdClSxdh5MiRwsGDB0Wounke3Hb58OvFF18UBME0jp++YzSmY9jQuAAIX3zxhbaPsR/DlozRmI7hyy+/rP3/i4ODgzB27FjtD78gGP/x03d8xnTsGvNwuGkvx1AiCPev7CEiIiIyAbzmhoiIiEwKww0RERGZFIYbIiIiMikMN0RERGRSGG6IiIjIpDDcEBERkUlhuCEiIiKTwnBDREREJoXhhog6vOPHj0MikaCkpETsUoioFTDcEBERkUlhuCEiIiKTwnBDRKITBAFr1qyBt7c3LC0tMXDgQOzZswfAr6eMDh48iIEDB0KhUCAwMBDnzp3T2cfevXvRr18/WFhYoHv37li/fr3O59XV1Vi6dCnc3d1hYWEBHx8fbNu2TadPSkoKAgICYGVlheHDhyM9Pd2wAycig2C4ISLRvfXWW/jiiy+wefNmXLhwAYsWLcIf//hHJCQkaPssWbIE69atw5kzZ+Do6Ijf//73uHfvHoC6UDJt2jQ8//zzOHfuHFatWoW//vWv2LFjh3b7WbNmYffu3di4cSMuXbqELVu2oHPnzjp1rFixAuvXr8fZs2dhZmaGl19+uU3GT0Sti08FJyJRVVRUwN7eHkePHkVQUJC2fc6cOaisrMQrr7yCkJAQ7N69G+Hh4QCA27dvw83NDTt27MC0adMwY8YM3Lp1C4cPH9Zuv3TpUhw8eBAXLlzAlStX4Ovri/j4eIwbN65eDcePH0dISAiOHDmCsWPHAgDi4uIwadIk3L17FwqFwsB/CkTUmjhzQ0SiunjxIqqqqhAaGorOnTtrXzt37sTVq1e1/X4bfOzs7ODr64tLly4BAC5duoQRI0bo7HfEiBHIyMhAbW0t0tLSIJPJMHr06CZrGTBggPafnZ2dAQCFhYWPPUYialtmYhdARB2bRqMBABw8eBCurq46n1lYWOgEnIdJJBIAddfsPPjnB347KW1padmsWszNzevt+0F9RGQ8OHNDRKLq27cvLCwskJOTg549e+q83N3dtf1++OEH7T/fuXMHV65cQe/evbX7OHnypM5+k5KS0KtXL8hkMvj5+UGj0ehcw0NEposzN0QkKmtrayxevBiLFi2CRqPByJEjUVpaiqSkJHTu3Bmenp4AgNWrV6Nr165wcnLCihUrYG9vj6lTpwIA3njjDQwZMgTvvPMOwsPDkZycjI8//hibNm0CAHTv3h0vvvgiXn75ZWzcuBEDBw5EdnY2CgsLMW3aNLGGTkQGwnBDRKJ755134OjoiKioKGRmZsLW1haDBw/G8uXLtaeF3nvvPSxYsAAZGRkYOHAgDhw4ALlcDgAYPHgwvvrqK6xcuRLvvPMOnJ2dsXr1arz00kva79i8eTOWL1+O1157DcXFxfDw8MDy5cvFGC4RGRjvliKidu3BnUx37tyBra2t2OUQkRHgNTdERERkUhhuiIiIyKTwtBQRERGZFM7cEBERkUlhuCEiIiKTwnBDREREJoXhhoiIiEwKww0RERGZFIYbIiIiMikMN0RERGRSGG6IiIjIpPw/eoVjEwZCx54AAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "y_avg = []\n", + "for y in epochs_y:\n", + " y_avg.append(np.mean(y))\n", + "\n", + "plt.plot(np.arange(len(epochs_x)), y_avg, marker = '.')\n", + "plt.xlabel('epoch')\n", + "plt.ylabel('average loss')\n", + "plt.ylim(0,)\n", + "for i, txt in enumerate(y_avg):\n", + " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom')\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkkAAAG2CAYAAABrrBJlAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAABcAklEQVR4nO3deVxU5f4H8M8wwLCjgAKDyKIIiog7omaaa+4rpKWmWd703iTLNbXUlNS0Ukuvv1tRWmqmqDf1CuYWSeYCuOCCioAKorIM6wAz5/cHOTkyIIMDMwOf9+vF68bxOc98n063+fScc55HJAiCACIiIiJSY6LvAoiIiIgMEUMSERERkQYMSUREREQaMCQRERERacCQRERERKQBQxIRERGRBgxJRERERBowJBERERFpwJBEREREpAFDEhEREZEGeg1JJ0+exLBhwyCVSiESibB37161PxcEAR999BGkUiksLS3Ru3dvXL58Wa2NXC7Hv/71Lzg5OcHa2hrDhw/HnTt3nvnZX331Fby8vGBhYYFOnTrht99+0+XQiIiIyMjpNSQVFBQgMDAQGzdu1Pjnq1evxrp167Bx40acOXMGLi4u6N+/P/Ly8lRtwsLCEBkZiR07diAmJgb5+fkYOnQoFApFpZ+7c+dOhIWF4YMPPkBcXBxeeOEFvPzyy0hNTdX5GImIiMg4iQxlg1uRSITIyEiMHDkSQPksklQqRVhYGObNmwegfNbI2dkZq1atwvTp05Gbm4smTZpg69atCA0NBQDcu3cP7u7uOHjwIAYOHKjxs4KCgtCxY0ds2rRJdax169YYOXIkwsPDa3egREREZBRM9V1AZZKTk5GRkYEBAwaojkkkErz44os4deoUpk+fjnPnzqG0tFStjVQqRdu2bXHq1CmNIamkpATnzp3D/Pnz1Y4PGDAAp06dqrQeuVwOuVyu+l2pVCIrKwuOjo4QiUTPM1QiIiKqI4IgIC8vD1KpFCYmVd9QM9iQlJGRAQBwdnZWO+7s7IyUlBRVG3NzczRu3LhCm8fnP+3hw4dQKBQa+63sHAAIDw/H0qVLtR4HERERGZ60tDQ0a9asyjYGG5Iee3qWRhCEZ87cVKeNtv0uWLAAs2fPVv2em5uL5s2bIy0tDXZ2dlV+FhERERkGmUwGd3d32NraPrOtwYYkFxcXAOWzRa6urqrjmZmZqlkgFxcXlJSUIDs7W202KTMzE927d9fYr5OTE8RicYVZoyf71UQikUAikVQ4bmdnx5BERERkZKrzqIzBrpPk5eUFFxcXREdHq46VlJTgxIkTqgDUqVMnmJmZqbVJT0/HpUuXKg1J5ubm6NSpk9o5ABAdHV3pOURERNTw6HUmKT8/Hzdu3FD9npycjPj4eDg4OKB58+YICwvDypUr4ePjAx8fH6xcuRJWVlaYMGECAMDe3h5vvPEG3nvvPTg6OsLBwQHvv/8+AgIC0K9fP1W/ffv2xahRo/DPf/4TADB79mxMnDgRnTt3RnBwMLZs2YLU1FT84x//qNu/AURERGSw9BqSzp49iz59+qh+f/zMz+TJkxEREYG5c+eiqKgIM2bMQHZ2NoKCghAVFaV2H/Gzzz6DqakpQkJCUFRUhL59+yIiIgJisVjV5ubNm3j48KHq99DQUDx69AjLli1Deno62rZti4MHD8LDw6MORk1ERETGwGDWSTI2MpkM9vb2yM3N5TNJRERERkKb72+DfSaJiIiISJ8YkoiIiIg0YEgiIiIi0oAhiYiIiEgDhiQiIiIiDRiSiIiIiDRgSCIiIiLSgCGJiIiISAOGJCIiIiINGJKIiIiINGBIIiIiItKAIYmIiIhIA4YkIiIiIg0YkoiIiIg0YEgiIiIi0oAhiYiIiEgDhiQiIiIiDRiSiIiIiDRgSCIiIiLSgCGJiIiISAOGJCIiIiINGJKIiIiINGBIIiIiItKAIYmIiIhIA4YkIiIiIg0YkoiIiIg0YEgiIiIi0oAhiYiIiEgDhiQiIiIiDRiSiIiIiDRgSCIiIiLSgCGJiIiISAOGJCIiIiINGJKIiIiINGBIIiIiItLA4ENSXl4ewsLC4OHhAUtLS3Tv3h1nzpxR/blIJNL4s2bNmkr7jIiI0HhOcXFxXQyJiIiIjICpvgt4lmnTpuHSpUvYunUrpFIptm3bhn79+iExMRFubm5IT09Xa3/o0CG88cYbGDNmTJX92tnZ4dq1a2rHLCwsdF4/ERERGSeDDklFRUXYvXs39u3bh169egEAPvroI+zduxebNm3Cxx9/DBcXF7Vz9u3bhz59+sDb27vKvkUiUYVziYiIiB4z6NttZWVlUCgUFWZ4LC0tERMTU6H9/fv3ceDAAbzxxhvP7Ds/Px8eHh5o1qwZhg4diri4uCrby+VyyGQytR8iIiKqvww6JNna2iI4OBjLly/HvXv3oFAosG3bNpw+fbrCbTYA+O6772Bra4vRo0dX2a+fnx8iIiKwf/9+bN++HRYWFujRoweSkpIqPSc8PBz29vaqH3d39+ceHxERERkukSAIgr6LqMrNmzcxdepUnDx5EmKxGB07dkSrVq1w/vx5JCYmqrX18/ND//79sWHDBq0+Q6lUomPHjujVqxfWr1+vsY1cLodcLlf9LpPJ4O7ujtzcXNjZ2Wk/MCIiIqpzMpkM9vb21fr+NuhnkgCgRYsWOHHiBAoKCiCTyeDq6orQ0FB4eXmptfvtt99w7do17Ny5U+vPMDExQZcuXaqcSZJIJJBIJFr3TURERMbJoG+3Pcna2hqurq7Izs7G4cOHMWLECLU///rrr9GpUycEBgZq3bcgCIiPj4erq6uuyiUiIiIjZ/AzSYcPH4YgCPD19cWNGzcwZ84c+Pr6YsqUKao2MpkMu3btwtq1azX2MWnSJLi5uSE8PBwAsHTpUnTr1g0+Pj6QyWRYv3494uPj8eWXX9bJmIiIiMjwGXxIys3NxYIFC3Dnzh04ODhgzJgxWLFiBczMzFRtduzYAUEQMH78eI19pKamwsTk70mznJwcvPXWW8jIyIC9vT06dOiAkydPomvXrrU+HiIiIjIOBv/gtqHS5sEvIiIiMgzafH8bzTNJRERERHWJIYmIiIhIA4YkIiIiIg0YkoiIiIg0YEgiIiIi0oAhiYiIiEgDhiQiIiIiDRiSiIiIiDRgSCIiIiLSgCGJiIiISAOGJCIiIiINGJKIiIiINGBIIiIiItKAIYmIiIhIA4YkIiIiIg0YkoiIiIg0YEgiIiIi0oAhiYiIiEgDhiQiIiIiDRiSiIiIiDRgSCIiIiLSgCGJiIiISAOGJCIiIiINGJKIiIiINGBIIiIiItKAIYmIiIhIA4YkIiIiIg0YkoiIiIg0YEgiIiIi0oAhiYiIiEgDhiQiIiIiDRiSiIiIiDRgSCIiIiLSgCGJiIiISAOGJCIiIiINDD4k5eXlISwsDB4eHrC0tET37t1x5swZ1Z+//vrrEIlEaj/dunV7Zr+7d+9GmzZtIJFI0KZNG0RGRtbmMIiIiMjIGHxImjZtGqKjo7F161ZcvHgRAwYMQL9+/XD37l1Vm0GDBiE9PV31c/DgwSr7jI2NRWhoKCZOnIiEhARMnDgRISEhOH36dG0Ph4iIiIyESBAEQd9FVKaoqAi2trbYt28fhgwZojrevn17DB06FB9//DFef/115OTkYO/evdXuNzQ0FDKZDIcOHVIdGzRoEBo3bozt27dXqw+ZTAZ7e3vk5ubCzs6u2p9NRERE+qPN97dBzySVlZVBoVDAwsJC7bilpSViYmJUvx8/fhxNmzZFq1at8OabbyIzM7PKfmNjYzFgwAC1YwMHDsSpU6d0VzwREREZNYMOSba2tggODsby5ctx7949KBQKbNu2DadPn0Z6ejoA4OWXX8YPP/yAo0ePYu3atThz5gxeeuklyOXySvvNyMiAs7Oz2jFnZ2dkZGRUeo5cLodMJlP7ISIiovrLoEMSAGzduhWCIMDNzQ0SiQTr16/HhAkTIBaLAZTfOhsyZAjatm2LYcOG4dChQ7h+/ToOHDhQZb8ikUjtd0EQKhx7Unh4OOzt7VU/7u7uzz84IiIiMlgGH5JatGiBEydOID8/H2lpafjzzz9RWloKLy8vje1dXV3h4eGBpKSkSvt0cXGpMGuUmZlZYXbpSQsWLEBubq7qJy0trWYDIiIiIqNg8CHpMWtra7i6uiI7OxuHDx/GiBEjNLZ79OgR0tLS4OrqWmlfwcHBiI6OVjsWFRWF7t27V3qORCKBnZ2d2g8RERHVX6b6LuBZDh8+DEEQ4Ovrixs3bmDOnDnw9fXFlClTkJ+fj48++ghjxoyBq6srbt++jYULF8LJyQmjRo1S9TFp0iS4ubkhPDwcADBr1iz06tULq1atwogRI7Bv3z4cOXJE7WFwIiIiatgMfiYpNzcXM2fOhJ+fHyZNmoSePXsiKioKZmZmEIvFuHjxIkaMGIFWrVph8uTJaNWqFWJjY2Fra6vqIzU1VfWgNwB0794dO3bswLfffot27dohIiICO3fuRFBQkD6GSERET3jWIsJPmj59OkQiET7//PNn9vusRYS1+VxqGAx+JikkJAQhISEa/8zS0hKHDx9+Zh/Hjx+vcGzs2LEYO3bs85ZHREQ6Nm3aNFy6dAlbt26FVCrFtm3b0K9fPyQmJsLNzU3Vbu/evTh9+jSkUukz+3y8iPDy5csxatQoREZGIiQkBDExMar/QK7u51LDYdCLSRoyLiZJRKR71VlEGADu3r2LoKAgHD58GEOGDEFYWBjCwsIq7fdZiwhX93PJ+NWbxSSJiKhhqc4iwkqlEhMnTsScOXPg7+9frX6ftYhwdRcvpoaFIYmIiAxGdRYRXrVqFUxNTfHOO+9Uu99nLSJcnc+lhochiYiIDEpViwifO3cOX3zxBSIiIqpcAFiTZy0i/KzFi6nhYUgiIiKDUtUiwr/99hsyMzPRvHlzmJqawtTUFCkpKXjvvffg6elZaZ/VWURY28WLqf5jSCIiIoOkaRHhiRMn4sKFC4iPj1f9SKVSzJkzp8q3nbVZRLi6ixdT/WfwSwAQEVHDUtUiwmZmZnB0dFRrb2ZmBhcXF/j6+qqO1WQR4ao+lxomziQREZFBqWoR4eq6ces2Eq4nIz23CED1FhHWxedS/cJ1kmqI6yQRERmmnWdSsWDPRSgFwEQEhI8OQGiX5vouiwwE10kiIqIGRxAEnLrxEPN3lwckAFAKwII9F3ElXabf4sgo8ZkkIiIySkqlgCsZMvyZnIXTt7Lw5+0sZBWUVGwnAC9/8Rua2krQ2tUOfq62aO1S/r8tmtjATMz5AtKMIYmIiIxCmUKJy/dkOJ38CKdvZeHM7SzIisvU2khMRZCXaX6KJDNPjsy8Bzhx/YHqmJlYhJZNbdHaxbY8PLnawc/FDk1sJbU6FjIODElERGSQSsqUuHAnB6eTs3A6OQvnbmehoESh1sbaXIxOng4I8ir/adesESLj7mDhnktQCALEIhFWjm6LIe2kuJaRh6sZMlxN//t/8+RluJIuK78dF/d3v0425vBzsYOfi61q9qllUxtITLmwZEPCB7driA9uExHpVnGpAnGpOaqZori0bBSXKtXa2FmYoquXA7p6OSDIyxH+UjuYarhdlp5bhNsPC+HpZAVXe0uNnycIAu5kF+FqRh6upstw5a/glPyoAJq+GcUmIrRoYq2abXp8287ZTqL16t+kP9p8fzMk1RBDEhHR8ymQl+FcSnb5M0XJj5CQlosShXoocrA2R1dPBwR5lwcjPxc7iE1qN5AUlShw/X4erqTLcDUjTzXT9PStvccaW5mphSY/V1u0craFhRlnnQwRQ1IdYEgiItJOblEpzqWUP2T9R3IWLt3NhUKp/hXU1FaCIG9HdPVyQDcvB7RsamMQszSCICBDVvxXYMpTzT7delhQYQxA+dIDXk7W8HO1Q2vVLTs7SO0tDGI8DRlDUh1gSCIiqlpWQQn+TM5SzRQlpssq3MZya2RZ/jyRtwO6ejnC09HKqEJEcakCNzLzK8w6ZReWamxva2GK1i52aO1qCz/X8meefF1sYWXOR4TrCkNSHWBIIiJSl5lXrHod/3TyI1y/n1+hjaejFYK8ymeKgrwd0KyxlR4qrV2CIOBBnhxXHj/r9FeAupGZjzINs04iEeDhYFXhWadmjS1hUsu3FhsihqQ6wJBERA3dvZwinE5+pApGtx4WVGjj09Tmr0DkiK6eDnCxt9BDpYahpEyJmw/UZ52uZuThQZ5cY3sbiSl8XWzh51I+69Tmr2edbC0Mf5uUvLw8LF68GJGRkcjMzESHDh3wxRdfoEuXLigtLcWiRYtw8OBB3Lp1C/b29ujXrx8++eQTSKXSSvssLS1FeHg4vvvuO9y9exe+vr5YtWoVBg0aVK3PfUyb72/O7xER0TMJgoDUrMLy1/H/mim6k12k1kYkAvxc7FSv43fxcoCTDdcbeszc1AStXe3Q2lX9i/lhvly1LMGV9PLwdCMzH/l/Pdh+LiVbrb27gyX8XMr7af1XgPJwsDKoWadp06bh0qVL2Lp1K6RSKbZt24Z+/fohMTERNjY2OH/+PBYvXozAwEBkZ2cjLCwMw4cPx9mzZyvtc9GiRdi2bRv+7//+D35+fjh8+DBGjRqFU6dOoUOHDs/8XDc3N63HwZmkGuJMEhHVZ4Ig4OaDArWZogxZsVobsYkIbaV2qtfxu3g6wN7K8Gc5jEGpQonkhwXqs07peRWuwWOWZmL4utiWP+v01/pOfi52erkeRUVFsLW1xb59+zBkyBDV8fbt22Po0KH4+OOPK5xz5swZdO3aFSkpKWjeXPM+e1KpFB988AFmzpypOjZy5EjY2Nhg27Zt1f5cziQREZFWlEoB1+7nqR6y/jM5Cw/z1bf4MBOL0K5ZIwT9tU5RZ08H2Ej4NVIbzMQmaOVcfnttxBPHswtKnrhVVx6grmXkoahUgfi0HMSn5aj149bI8q/bdbaq2SdPRyuNa0vpSllZGRQKBSws1G+tWlpaIiYmRuM5ubm5EIlEaNSoUaX9yuXyKvusyec+C2eSaogzSURkzBRKAYmPt/hILt/iI+epN7LMTU3Qwb0Rgrwd0c3LAR2aN4alOdf+MTQKpYDkhwVqq4lfSc/D3Zwije0lpuUB7MnVxFu72KGxtbnOaurevTvMzc3x448/wtnZGdu3b8ekSZPg4+ODa9euqbUtLi5Gz5494efnh23btlXa54QJE5CQkIC9e/eiRYsW+PXXXzFixAgoFArI5fJqfy4f3K4DDElEZExKFUpcuJOrmik6dzsbeXL1xREtzcTo7Nn4r8UbHRHobs9tOIxYblGpaiuWx886PZ510sTZTvL3s05/zTx5N7Gu0QbAN2/exNSpU3Hy5EmIxWJ07NgRrVq1wvnz55GYmKhqV1painHjxiE1NRXHjx+v8vv0wYMHePPNN/Hf//4XIpEILVq0QL9+/fDtt9+isLCw2p/LkFQHGJKIyJAVlyqQkFa+79mfyVk4l5Jd4cvRVmKKzp6NEeTtiCAvB7R1s6/RFyIZD6Wy/AH8qxkyJKaXL1FwNSMPqVmFGtubi03QsqmN2mrirV3tqv1AfkFBAWQyGVxdXREaGor8/HwcOHAAQHlACgkJwa1bt3D06FE4OjpWq8/i4mI8evQIUqkU8+fPxy+//ILLly9X+3P5TBIRNVi18eoxAOTk5OCDDz7Anj17kJ2dDS8vL6xduxaDBw9Wtfnqq6+wZs0apKenw9/fH59//jleeOGF2h4yAKCwpKx837Nbj/BHchbi03JQUqa+xUcjKzN09Sx/nqibtyNau9b+Fh9kWExMRPB0soankzUGtXVVHc+Xl+FaxuPVxB/ftstDvrwMiekyJKbLANxVtXeykfw122SrWt+pRVPrCjOP1tbWsLa2RnZ2Ng4fPozVq1cD+DsgJSUl4dixY9UOSABgYWEBNzc3lJaWYvfu3QgJCanQprLP1RZnkmqIM0lEhik0NBSXLl3Cpk2bVK8Af/bZZ6pXj8eOHYs333xT7dXjsrKyKl89LikpQY8ePdC0aVMsXLgQzZo1Q1paGmxtbREYGAgA2LlzJyZOnIivvvoKPXr0wL///W/85z//QWJiYqVv6zyPvOJSnE3JxulbWfgz+REu3MmtsFChk41EtZp1kJcjfJraGNRr4mTYHm8A/PgNu8e37W5XsgGwqYkILZqUzzoJafFo1tgSvbu2R05GKubOnQuJRIKYmBiIRCKMGTMG58+fxy+//AJnZ2dVHw4ODjA3L382atKkSXBzc0N4eDgA4PTp07h79y7at2+Pu3fv4qOPPkJycjLOnz+veuD78OHDEAQBvr6+uHHjBubMmaP6XDOz8jf9eLutDjAkERme2nr1ePPmzVizZg2uXr2q+hft04KCgtCxY0ds2rRJdax169YYOXKk6l/yzyOnsARnbmfj9K3yB60v38vF04s3u9pb/PXmmSOCvB3g7WRtVFt8kHEoLCnD9fv5qtXEH68s/uQGwAVXfkPOye9QlvcQppZ28OrcB2Onz0H7llLYlWbj5e6BGvs+duwYevfuDQDo3bs3PD09ERERAQA4ceIE3n77bdy6dQs2NjYYPHhwhVngn376CQsWLMCdO3fg4OCAMWPGYMWKFbC3t1e1YUiqAwxJRIYnLy8PdnZ2OHLkCPr27as6HhwcDIlEguPHj1c458iRIxgwYABycnIq/f/y4MGD4eDgACsrK+zbtw9NmjTBhAkTMG/ePIjFYpSUlMDKygq7du3CqFGjVOfNmjUL8fHxOHHihNZjeZgv/2t9ovJQdO1+XoX/em/uYPXXGkXlM0XuDpYMRaQXgiAgPbe4wmritx7kVwjzQPkGwN5NbJ64XVf+v65PbACcnluE5IcF8HKyhqu9pc5q5TNJRNQg2draIjg4GMuXL0fr1q1VrwCfPn0aPj4+FdoXFxdj/vz5mDBhQpX/snz8YOmrr76KgwcPIikpCTNnzkRZWRmWLFmChw8fQqFQqN02AABnZ2dkZGRUq/aM3GLV6/inbz3CzQcVt/jwbmKNIC9H1TpF0ka6++Igeh4ikQjSRpaQNrJE39Z///+guFSBpPv5uPLXc07lM08y5BSW4kZmPm5k5uOXC+mq9nYWpvBztYOpCIi9lQUB5YEqfHQAQrvo/rb1szAkEVG9snXrVkydOhVubm6qV4AnTJiA8+fPq7UrLS3FK6+8AqVSia+++qrKPpVKJZo2bYotW7ZALBajU6dOuHfvHtasWYMlS5ao2j09iyMIQqUzO2l/bfHx51/BKOVRxbeL/Fxs/17N2qsxmto23H3PyDhZmIkR0MweAc3+vt0lCAIy8+QVVhO/+SAfsuIy/JmcpdaHUgAW7rmEXq2a6HRGqToYkoioXmnRogVOnDhR4RVgLy8vVZvHb9YkJyfj6NGjz5xyd3V1hZmZGcTiv9/cad26NTIyMlBSUgInJyeIxeIKs0aZmZlwdnaGIJQv9le+RlH5K/lPL/RnIgLaSO0Q5OWIrl4O6OrpoNPF/YgMhUgkgrOdBZztLNDbt6nquLxMgZuZBfjvhXvYdPym2jkKQcDth4UMSUREuqDLV4979OiBH3/8EUqlEiYm5esIXb9+Ha6urqo3cTp16oTo6GiMGjUKSqWAGw/y8fP+g3AJ6ImuK3+tsNO72ESEADd7BHk7oJuXIzp5NoadEezuTlRbJKZitJHaobG1Gf594qbas0xikQieTlZ1XhNDEhHVK5peAfb19cWUKVNQVlaGsWPHql49VigUqtmfql49fvvtt7FhwwbMmjUL//rXv5CUlISVK1finXfeAVC+LcS41/+B+e9Mx+USJzyw8sCd2P8i/+4diAb1gmmeHOZiE7R3b4Qg7/LniTo2bwxr7ntGVIGrvSXCRwdg4Z5LUAgCxCIRVo5uW+ezSABDEhHVM7m5uRpfATYzM8Pt27exf/9+AOXLAjzpyVePU1NTVTNGAODu7o6oqCi8++67aNeuHdzc3BDy+ltwCB6HNyLO4M/bWcgrbgr7PtPw+65/Q1GQBUkTT7w0ax2GDeyJIG8HtHdvBAszbvFBVB2hXZqjV6smuP2wEJ5OVnoJSACXAKgxLgFAVH89/eqxvEyh2vfsj1uPcC4lG4Ul6lt8WJuL0dnz8cKNDghwawRzU27xQWRo6tUSALWxxUBERASmTJlS4XhRUREsLPj2CFFDtvNMKhbsuQilAIgAeDlZ425OEeRPbfFhZ2GqevMsyNsBbVztYMp9z4jqFYMPSdOmTcOlS5ewdetW1RYD/fr1U20xcP78eSxevFhti4Hhw4dXucUAANjZ2eHatWtqxxiQiBq2s7ezMH/3RTyeXhcA3HpYvl6Ro7W5auHGrl6O8HOx5RYfRPWcQd9uq60tBiIiIhAWFoacnJwa18bbbUT1g0Ip4Pi1TGz9IwXHrz3Q2ObTce0wpmMzrmZNVA/Um9ttZWVlUCgUFWZ4LC0tERMTo/Gc3NxciEQi1WZ3lcnPz4eHhwcUCgXat2+P5cuXo0OHDpW2l8vlkMv/foVXJpNVfyBEZHAe5svx09k0/PBHaoU1i54kFonQo6UTAxJRA6T1DXRNex/Vlie3GLh37x4UCgW2bduG06dPIz09vUL76m4x4Ofnh4iICOzfvx/bt2+HhYUFevTogaSkpErPCQ8Ph729verH3d1dJ2MkorojCALOpWQhbEccuocfxer/XcPdnCLYW5rhzRe8cPz93lg1JgDivwKRPl89JiL90/p2m4WFBdzc3DBlyhRMnjy51sPCzZs3MXXqVJw8eVK1xUCrVq1w/vx5JCYmqtqVlpZi3LhxSE1NxfHjx7W6BaZUKtGxY0f06tUL69ev19hG00ySu7s7b7cRGYECeRn2xt/F1tgUXM3IUx0PdG+E14KaY1igVO31/PTcIr2/ekxEtaNWb7fdu3cP27ZtQ0REBD766CP07dsXb7zxBkaOHKlaiE2XamOLgaeZmJigS5cuVc4kSSQSSCSSGo+DiOpe0v08bPsjBbvP30W+vAwAIDE1wYj2UrzWzQPtmjXSeJ6rvSXDERE934Pb8fHx+Oabb7B9+3YolUq8+uqreOONNxAYGKjLGtVkZ2fDy8sLq1evxltvvVVhi4EmTZpo3acgCOjatSsCAgLwzTffVOscPrhNZJhKFUpEXb6PrX/cxh+3/t4o08vJGq8GNcfYTs3QyIp7ohE1VNp8fz/322337t3Dli1b8Mknn8DU1BTFxcUIDg7G5s2b4e/v/zxdA9C8xYBEIkFMTAxEIhHGjBmj2mLA2dlZdV5VWwwsXboU3bp1g4+PD2QyGdavX4+tW7fi999/R9euXatVF0MSkWFJzy3C9j/TsOPPVGT+tU+aiQjo19oZE4M90KOFE1/ZJ6Laf7uttLQU+/btwzfffIPo6Gh07twZGzduxPjx45GVlYV58+Zh3Lhxas8M1VRtbDGQk5ODt956CxkZGbC3t0eHDh1w8uTJagckIjIMgiDg1M1H2Bqbgugr96H4a0dMJxsJxnd1x/iuzSFtxNtmRFQzWs8k/etf/8L27dsBAK+99hqmTZuGtm3bqrVJTU2Fp6cnlEqlpi7qBc4kEelPblEpdp+7g22nU3DrQYHqeJCXAyYGe2BAGxduCUJEGtXqTFJiYiI2bNiAMWPGVPqgtlQqxbFjx7TtmoioSpfu5mJrbAr2JdxFcWn5f4TZSEwxuqMbXuvmgVbOtnqukIjqE4NecduQcSaJqG4Ulypw4EI6tv6Rgvi0HNVxPxdbvNbNAyM7uMFGYtDr4hKRAanVmaTw8HA4Oztj6tSpase/+eYbPHjwAPPmzdO2SyKiClIfFeKH0yn46WwasgtLAQBmYhFebuuKicEe6OzRmKtgE1Gt0jok/fvf/8aPP/5Y4bi/vz9eeeUVhiQiqrEn91E7cf0BHs9zuzWyxISg5gjp7I4mtlyvjIjqhtYhKSMjA66urhWON2nSRONWIUREz1LZPmovtmqCid080MevKcR8fZ+I6pjWIcnd3R2///672orXAPD7779DKpXqrDAiqt/K91HLxtY/UnDwYjpKFeXTRo2szBDS2R0TujaHp5O1nqskooZM65A0bdo0hIWFobS0FC+99BIA4Ndff8XcuXPx3nvv6bxAIqpfqtpHbWI3Dwxt56q2jxoRkb5oHZLmzp2LrKwszJgxAyUlJQDKN72dN28eFixYoPMCiah+qOk+akRE+lLjJQDy8/Nx5coVWFpawsfHp8Ft/solAIie7Vn7qI3r5A57KzM9VkhEDU2tb0sCADY2NujSpUtNTyeieoz7qBFRfVCjkHTmzBns2rULqampqltuj+3Zs0cnhRGRcREEAb/feIRtf1TcR21CV3e8wn3UiMjIaB2SduzYgUmTJmHAgAGIjo7GgAEDkJSUhIyMDIwaNao2aiQiA5ZbVIqfz93BD3+k4NZD7qNGRPWH1iFp5cqV+OyzzzBz5kzY2triiy++gJeXF6ZPn65x/SQiqp+4jxoR1Xdah6SbN29iyJAhAACJRIKCggKIRCK8++67eOmll7B06VKdF0lEhoH7qBFRQ6L1v80cHByQl1e+tombmxsuXbqEgIAA5OTkoLCwUOcFEpH+VbaP2uAAV7zWjfuoEVH9pHVIeuGFFxAdHY2AgACEhIRg1qxZOHr0KKKjo9G3b9/aqJGI9ID7qBFRQ6d1SNq4cSOKi4sBAAsWLICZmRliYmIwevRoLF68WOcFElHd4j5qRETltFpMsqysDD/88AMGDhwIFxeX2qzL4HExSapPuI8aETUUtbaYpKmpKd5++21cuXLluQokIsPAfdSIiCqn9e22oKAgxMXFwcPDozbqIaI6UNU+ahO7eSKgmb2eKyQi0j+tQ9KMGTPw3nvv4c6dO+jUqROsrdWn4Nu1a6ez4ohId6raR+21bh4Y27EZ91EjInqC1hvcmphUXDlXJBJBEASIRCIoFAqdFWfI+EwSGYvK9lHr38YZE7t5onsLR+6jRkQNRq1ucJucnFzjwoiobnAfNSKi56d1SOKzSESGi/uoERHpjtYh6fvvv6/yzydNmlTjYoioZriPGhGR7mn9TFLjxo3Vfi8tLUVhYSHMzc1hZWWFrKysSs6sX/hMUv2Ul5eHxYsXIzIyEpmZmejQoQO++OILdOnSBUD5baylS5diy5YtyM7ORlBQEL788kv4+/tX2e/u3buxePFi3Lx5Ey1atMCKFSswatQo1Z97enoiJSWlwnkzZszAl19+qbHPqvZRmxjsgRHtuY8aEdHTavWZpOzs7ArHkpKS8Pbbb2POnDnadkdkUKZNm4ZLly5h69atkEql2LZtG/r164fExES4ublh9erVWLduHSIiItCqVSt8/PHH6N+/P65duwZbW82zNbGxsQgNDcXy5csxatQoREZGIiQkBDExMQgKCgIAnDlzRu2lh0uXLqF///4YN25chf6q2kdtYjcPdOI+akREOqH1TFJlzp49i9deew1Xr17VRXcGjzNJ9U9RURFsbW2xb98+DBkyRHW8ffv2GDp0KJYvXw6pVIqwsDDMmzcPACCXy+Hs7IxVq1Zh+vTpGvsNDQ2FTCbDoUOHVMcGDRqExo0bY/v27RrPCQsLwy+//IKkpKTyt0a5jxoRkU7U6kxSZcRiMe7du6er7ojqXFlZGRQKBSwsLNSOW1paIiYmBsnJycjIyMCAAQNUfyaRSPDiiy/i1KlTlYak2NhYvPvuu2rHBg4ciM8//1xj+5KSEmzbtg2zZ8/Go4IS7qNGRKQnWoek/fv3q/0uCALS09OxceNG9OjRQ2eFEdU1W1tbBAcHY/ny5WjdujWcnZ2xfft2nD59Gj4+PsjIyAAAODs7q53n7Oys8XmixzIyMjSe87i/p0VGRiInJwc3G3VGcPivFfZRezWoOTwcuY8aEVFt0zokjRw5Uu13kUiEJk2a4KWXXsLatWt1VReRXmzduhVTp06Fm5sbxGIxOnbsiAkTJuD8+fOqNk8/7/N4IdWqVOecx/uozfpwLcw9O+LX1PLnjdr/tY/aEO6jRkRUp7QOSUqlsjbqIDIILVq0wIkTJ1BQUACZTAZXV1eEhobCy8sLLi4uAMpnhlxdXVXnZGZmVpgpepKLi0uFWaMnz3lyH7WczHt4dP0cpGM/QGhnd7zWzYP7qBER6QlXlSPSwNraGq6ursjOzsbhw4cxYsQIVVCKjo5WtSspKcGJEyfQvXv3SvsKDg5WOwcADh8+DM82HfDKllj0/+wkvotNQb68DKY3T8C+sSMu/GcBVo1tx4BERKRHWoeksWPH4pNPPqlwfM2aNRpfVyYyJocPH8b//vc/JCcnIzo6Gn369IGvry+mTJkCkUiEsLAwrFy5EpGRkbh06RJef/11WFlZYcKECao+Jk2ahAULFqh+nzVrFqKiorBq1SqcPBOPodPeR1T0EVxv0gt/3MqCiQgY6O+M76d0gXDtGP7x5lQ42nHLECIifdP6dtuJEyfw4YcfVjg+aNAgfPrppzopikhfcnNzsWDBAty5cwcODg4YM2YMVqxYATMzMwDA3LlzUVRUhBkzZqgWk4yKilJbIyk1NVVtI+jg4GB8+NkWfBK+DDkLF8G0kQuchs9DM992GN/l733UoqKikJqaiqlTp9b5uImIqCKt10mytLREfHw8fH191Y5fvXoVHTp0QFFRUSVn1oy+VkB+Fq6TRJVJzy1C8sMCOFlL8NuNhxX2Uevm7YDXunEfNSIifajVdZLatm2LnTt3YsmSJWrHd+zYgTZt2mjb3TPpawVkoprYeSYVC/ZchPKp//SwkZhiTEc3vMp91IiIjIbWM0n79+/HmDFjMGHCBLz00ksAgF9//RXbt2/Hrl27KiwR8DwMaQXkp3EmiZ6WnluEHp8crRCQ5g7yxeRgT1hzHzUiIr3T5vtb67n+4cOHY+/evbhx4wZmzJiB9957D3fu3MGRI0d0GpCA518BuTKxsbFq5wDlKyBXdY5cLodMJlP7IXpS8sOCCgEJADq4N2ZAIiIyQjX6N/eQIUPUZnZqi6GsgAwA4eHhWLp06XOMhuq7krKKa4iJRSJ4OlnpoRoiInpeWs8knTlzBqdPn65w/PTp0zh79qxOinrS1q1bIQgC3NzcIJFIsH79ekyYMAFi8d8rD9fWCshPWrBgAXJzc1U/aWlpNRgN1VfyMgXCD6pv7iwWibBydFu42vN1fiIiY6R1SJo5c6bGgHD37l3MnDlTJ0U96fEKyPn5+UhLS8Off/6J0tLSCisgP+l5V0DWRCKRwM7OTu2H6LF10ddx7X4enGzMcfCdntj+ZjfEzO+D0C7N9V0aERHVkNYhKTExER07dqxwvEOHDkhMTNRJUZrU9grIUVFRVZ5DVJlzKVnYcvIWACB8dDu0kdojuIUjZ5CIiIyc1s8kSSQS3L9/H97e3mrH09PTYWqq+4dTDx8+DEEQ4Ovrixs3bmDOnDkaV0D28fGBj48PVq5cqXEFZDc3N4SHhwMoXwG5V69eWLVqFUaMGIF9+/bhyJEjiImJ0Xn9VL8VlpRh9k8JEARgTMdm6N+m8tlIIiIyLlqnmv79+2PBggXYt28f7O3L95XKycnBwoUL0b9/f50XWBsrIHfv3h07duzAokWLsHjxYrRo0QI7d+7kGkmktfCDV5HyqBBSewt8OFz364QREZH+aL1O0t27d9GrVy88evQIHTp0AADEx8fD2dkZ0dHRcHd3r5VCDQ3XSaLfkh5g4td/AgB+mBaEHi2d9FwRERE9S62uuO3m5oYLFy7ghx9+QEJCAiwtLTFlyhSMHz9eNbtDVN/lFpVizq4LAIDJwR4MSERE9VCNHiKytrbGW2+9petaiIzG0v9eRoasGF5O1pj/cmt9l0NERLWgxk9aJyYmIjU1FSUlJWrHhw8f/txFERmyw5czsOf8XZiIgE/HBcLSXPzsk4iIyOhoHZJu3bqFUaNG4eLFixCJRHj8SNPjhRgVCoVuKyQyIA/z5Vi45yIAYPqLLdDJo7GeKyIiotqi9TpJs2bNgpeXF+7fvw8rKytcvnwZJ0+eROfOnXH8+PFaKJHIMAiCgA8iL+JRQQn8XGwR1s9H3yUREVEt0nomKTY2FkePHkWTJk1gYmICExMT9OzZE+Hh4XjnnXcQFxdXG3US6d3e+Ls4fPk+zMQirAtpD4kpb7MREdVnWs8kKRQK2NjYAACcnJxw7949AICHhweuXbum2+qIDER6bhGW7LsMAJjV1wdtpFz2gYiovtN6Jqlt27a4cOECvL29ERQUhNWrV8Pc3BxbtmypsAo3UX0gCALm/nwBecVlaO/eCP94sYW+SyIiojqgdUhatGgRCgoKAAAff/wxhg4dihdeeAGOjo7YuXOnzgsk0rdtp1PxW9JDWJiZYG1IIEzFWk/AEhGREdI6JA0cOFD1197e3khMTERWVhYaN26sesONqL64/bAAKw9cAQDMG+SHFk1s9FwRERHVFZ3sSOvg4KCLbogMikIp4P1dCSgqVSDY2xGTgz31XRIREdUh3jcgqsR/fruFsynZsJGYYs24djAx4UwpEVFDwpBEpMG1jDysjboOAFgytA2aNbbSc0VERFTXGJKInlJSpsTsn+JRolCir19TjOvcTN8lERGRHmgdkk6ePImysrIKx8vKynDy5EmdFEWkTxuPJuHyPRkaWZkhfEwAX0ggImqgtA5Jffr0QVZWVoXjubm56NOnj06KItKXhLQcfHn8JgDg45Ft0dTWQs8VERGRvmgdkgRB0Phf1o8ePYK1tbVOiiLSh+JSBWb/FA+FUsCwQCmGtpPquyQiItKjai8BMHr0aACASCTC66+/DolEovozhUKBCxcuoHv37rqvkKiOrDl8DTcfFKCprQTLR/jruxwiItKzaocke3t7AOUzSba2trC0tFT9mbm5Obp164Y333xT9xUS1YE/bj3CN78nAwBWjWmHRlbmeq6IiIj0rdoh6dtvvwUAeHp64v333+etNao38uVleH9XAgQBGN/VHX38muq7JCIiMgBaP5M0d+5ctWeSUlJS8PnnnyMqKkqnhRHVlRUHEnEnuwjNGlvigyFt9F0OEREZCK1D0ogRI/D9998DAHJyctC1a1esXbsWI0aMwKZNm3ReIFFtOnY1E9v/TINIBHw6LhA2Ep3s1ENERPWA1iHp/PnzeOGFFwAAP//8M1xcXJCSkoLvv/8e69ev13mBRLUlp7AE83ZfAABM7eGFbt6Oeq6IiIgMidYhqbCwELa2tgCAqKgojB49GiYmJujWrRtSUlJ0XiBRbVm87zIy8+Ro2dQGcwb66rscIiIyMFqHpJYtW2Lv3r1IS0vD4cOHMWDAAABAZmYm7OzsdF4gUW345cI9/DfhHsQmIqwdFwgLM7G+SyIiIgOjdUhasmQJ3n//fXh6eqJr164IDg4GUD6r1KFDB50XSKRrmXnFWLT3EgBgZu8WCHRvpN+CiIjIIGn9lOrYsWPRs2dPpKenIzAwUHW8b9++GDVqlE6LI9I1QRCwYPdF5BSWwl9qh3++5KPvkoiIyEBpPZMEAC4uLrC1tUV0dDSKiooAAF26dIGfn59OiyPStV1n7+DXq5kwF5tgXUh7mJvW6P8CRETUAGj9DfHo0SP07dsXrVq1wuDBg5Geng4AmDZtGt577z2dF0ikK2lZhVj2SyIA4L0BreDrYqvnioiIyJBpHZLeffddmJmZITU1FVZWVqrjoaGh+N///qfT4oh0RakUMPfnC8iXl6GzR2NMe8Fb3yUREZGB0/qZpKioKBw+fBjNmjVTO+7j48MlAMhgfRd7G7G3HsHSTIxPxwVCbCJ69klERNSgaT2TVFBQoDaD9NjDhw8hkUh0UhSRLt18kI9PDl0FACwc0hqeTtx3kIiInk3rkNSrVy/VtiQAIBKJoFQqsWbNGvTp00enxRE9rzKFErN/SoC8TIkXfJzwWlBzfZdERERGQuvbbWvWrEHv3r1x9uxZlJSUYO7cubh8+TKysrLw+++/10aNRDW2+cRNJKTlwNbCFKvHtlPbnJmIiKgqWs8ktWnTBhcuXEDXrl3Rv39/FBQUYPTo0YiLi0OLFi1qo0aiGrl8Lxdf/JoEAFg63B+u9pZ6roiIiIyJ1iEpNTUVzs7OWLp0KX755RccPHgQH3/8MVxdXZGamqrT4srKyrBo0SJ4eXnB0tIS3t7eWLZsGZRKpaqNSCTS+LNmzZpK+42IiNB4TnFxsU7rJ/2Rlynw3k8JKFUIGOjvjFEd3PRdEhERGRmtb7d5eXkhPT0dTZs2VTv+6NEjeHl5QaFQ6Ky4VatWYfPmzfjuu+/g7++Ps2fPYsqUKbC3t8esWbMAQLVO02OHDh3CG2+8gTFjxlTZt52dHa5du6Z2zMLCQme1k359fiQJVzPy4GhtjpWjAnibjYiItKZ1SBIEQeMXTn5+vs5DRmxsLEaMGIEhQ4YAADw9PbF9+3acPXtW1cbFxUXtnH379qFPnz7w9q56HRyRSFThXKofzqVk4d8nbgIAVo4OgKMN37okIiLtVTskzZ49G0B5uFi8eLHaMgAKhQKnT59G+/btdVpcz549sXnzZly/fh2tWrVCQkICYmJi8Pnnn2tsf//+fRw4cADffffdM/vOz8+Hh4cHFAoF2rdvj+XLl1e5Qa9cLodcLlf9LpPJtB4P1b7CkjK891MClAIwuoMbBvozCBMRUc1UOyTFxcUBKJ9JunjxIszNzVV/Zm5ujsDAQLz//vs6LW7evHnIzc2Fn58fxGIxFAoFVqxYgfHjx2ts/91338HW1hajR4+usl8/Pz9EREQgICAAMpkMX3zxBXr06IGEhAT4+Gje8DQ8PBxLly597jFR7Vp16CpuPyqEq70FPhzur+9yiIjIiIkEQRC0OWHKlCn44osvYGdnV1s1qezYsQNz5szBmjVr4O/vj/j4eISFhWHdunWYPHlyhfZ+fn7o378/NmzYoNXnKJVKdOzYEb169cL69es1ttE0k+Tu7o7c3Nw6+XtBz/b7jYd49T+nAQBb3+iKF3ya6LkiIiIyNDKZDPb29tX6/tb6maRvv/22xoVpa86cOZg/fz5eeeUVAEBAQABSUlIQHh5eIST99ttvuHbtGnbu3Kn155iYmKBLly5ISkqqtI1EIuGK4gZMVlyKObsSAAATu3kwIBER0XPTegmAulRYWAgTE/USxWKx2hIAj3399dfo1KkTAgMDtf4cQRAQHx8PV1fXGtdK+rV0fyLu5RbDw9EKCwb76bscIiKqB7SeSapLw4YNw4oVK9C8eXP4+/sjLi4O69atw9SpU9XayWQy7Nq1C2vXrtXYz6RJk+Dm5obw8HAAwNKlS9GtWzf4+PhAJpNh/fr1iI+Px5dfflnrYyLdi7qcgd3n70AkAtaOC4SVuUH/Y01EREbCoL9NNmzYgMWLF2PGjBnIzMyEVCrF9OnTsWTJErV2O3bsgCAIlT7QnZqaqjYjlZOTg7feegsZGRmwt7dHhw4dcPLkSXTt2rVWx0O69yhfjoWRFwEAb/XyRmdPBz1XRERE9YXWD25TOW0e/KLaIQgCZvxwHocuZcDX2Rb7/9UDElOxvssiIiIDps33t0E/k0RUlX3x93DoUgZMTURYGxLIgERERDrFkERGKSO3GEv2XQIAvNPXB23d7PVcERER1TcMSWR0BEHA3N0XICsuQ2Aze8zo3ULfJRERUT3EkERG58c/U3Hy+gNITE2wNqQ9TMX8x5iIiHSP3y5kVFIeFWDFgSsAgLmD/NCyqY2eKyIiovqKIYmMhkIp4P1dCSgsUSDIywFTunvquyQiIqrHGJLIaHwdcwtnbmfD2lyMT8cFwsREpO+SiIioHmNIIqNw/X4ePj18HQCweGgbuDtY6bkiIiKq7xiSyOCVKpSY/VM8ShRKvOTXFKFd3PVdEhERNQAMSWTwNh69gUt3ZbC3NMMnowMgEvE2GxER1T6GJDJoF+7kYOOxGwCA5SPboqmdhZ4rIiKihoIhiQxWcakCs39KgEIpYEg7VwwPlOq7JCIiakAYkshgrY26hhuZ+WhiK8HHI9rquxwiImpgGJLIIJ2+9Qj/iUkGAKwaE4DG1uZ6roiIiBoahiQyOPnyMrz/cwIEAQjt7I6X/Jz1XRIRETVADElkcFYcuIK0rCK4NbLEoqGt9V0OERE1UAxJZFCOXcvE9j9TAQBrxrWDrYWZnisiIqKGiiGJDEZOYQnm/XwBADClhye6t3DSc0VERNSQMSSRwfhw/2Vk5snh3cQa8wb56bscIiJq4BiSyCAcvJiOffH3YCIC1oW0h4WZWN8lERFRA8eQRHqXmVeMDyIvAgBm9G6J9u6N9FsQERERGJJIzwRBwMI9l5BdWIo2rnZ4p6+PvksiIiICwJBEevbzuTs4cuU+zMUmWBcaCHNT/iNJRESGgd9IpDd3c4qw7L+JAIB3+7eCn4udnisiIiL6G0MS6YVSKWDOrgTkycvQsXkjvNXLW98lERERqWFIIr34PvY2Tt18BEszMdaGtIfYRKTvkoiIiNQwJFGdu/UgH5/87yoAYMFgP3g5Weu5IiIioooYkqhOlSmUeG9XAopLlejZ0gmvBXnouyQiIiKNGJKoTv375C3EpebAVmKK1WPbwYS32YiIyEAxJFGdSbwnw+dHrgMAPhzuD2kjSz1XREREVDmGJKoT8jIFZv8Uj1KFgP5tnDGmo5u+SyIiIqoSQxLVifW/JuFqRh4crM0RPjoAIhFvsxERkWFjSKJadz41G5uO3wQArBzVFk42Ej1XRERE9GwMSVSrikoUeP+nBCgFYGR7KQa1ddV3SURERNXCkES1atX/ruLWwwK42Flg6fC2+i6HiIio2gw6JJWVlWHRokXw8vKCpaUlvL29sWzZMiiVSlWb119/HSKRSO2nW7duz+x79+7daNOmDSQSCdq0aYPIyMjaHEqD9PuNh4g4dRsAsGpsO9hbmem3ICIiIi2Y6ruAqqxatQqbN2/Gd999B39/f5w9exZTpkyBvb09Zs2apWo3aNAgfPvtt6rfzc3Nq+w3NjYWoaGhWL58OUaNGoXIyEiEhIQgJiYGQUFBtTaehkRWXIq5P18AALwa1Bwvtmqi54qIiIi0IxIEQdB3EZUZOnQonJ2d8fXXX6uOjRkzBlZWVti6dSuA8pmknJwc7N27t9r9hoaGQiaT4dChQ6pjgwYNQuPGjbF9+/Zq9SGTyWBvb4/c3FzY2XH3+qfN2ZWAXefuoLmDFQ7NegHWEoPO40RE1EBo8/1t0LfbevbsiV9//RXXr5cvQJiQkICYmBgMHjxYrd3x48fRtGlTtGrVCm+++SYyMzOr7Dc2NhYDBgxQOzZw4ECcOnWq0nPkcjlkMpnaD2l2JPE+dp27A5EI+HRcIAMSEREZJYP+9po3bx5yc3Ph5+cHsVgMhUKBFStWYPz48ao2L7/8MsaNGwcPDw8kJydj8eLFeOmll3Du3DlIJJpfNc/IyICzs7PaMWdnZ2RkZFRaS3h4OJYuXaqbgdVjWQUlmL/nIgDgzRe80dXLQc8VERER1YxBh6SdO3di27Zt+PHHH+Hv74/4+HiEhYVBKpVi8uTJAMpvnT3Wtm1bdO7cGR4eHjhw4ABGjx5dad9PL2YoCEKVCxwuWLAAs2fPVv0uk8ng7u5e06HVS4IgYNHei3iYL0crZxvM7t9K3yURERHVmEGHpDlz5mD+/Pl45ZVXAAABAQFISUlBeHi4KiQ9zdXVFR4eHkhKSqq0XxcXlwqzRpmZmRVml54kkUgqnZmicvsT7uHgxQyYmoiwdlx7WJiJ9V0SERFRjRn0M0mFhYUwMVEvUSwWqy0B8LRHjx4hLS0Nrq6VL1oYHByM6OhotWNRUVHo3r378xXcgN2XFWPJvssAgH++1BIBzez1XBEREdHzMeiZpGHDhmHFihVo3rw5/P39ERcXh3Xr1mHq1KkAgPz8fHz00UcYM2YMXF1dcfv2bSxcuBBOTk4YNWqUqp9JkybBzc0N4eHhAIBZs2ahV69eWLVqFUaMGIF9+/bhyJEjiImJ0cs4jZ0gCJj78wXkFpUiwM0eM/u01HdJREREz82gQ9KGDRuwePFizJgxA5mZmZBKpZg+fTqWLFkCoHxW6eLFi/j++++Rk5MDV1dX9OnTBzt37oStra2qn9TUVLUZqe7du2PHjh1YtGgRFi9ejBYtWmDnzp1cI6mGtv+ZhhPXH8Dc1ATrQgJhJjboCUoiIqJqMeh1kgwZ10kql/qoEIO+OInCEgUWDWmNaS9467skIiKiStWbdZLIsCmVAt7/OQGFJQp09XLA1B5e+i6JiIhIZxiSqMa++T0ZfyZnwcpcjE/HBsLEpPIlFIiIiIwNQxLVSNL9PKw+fA0AsGhIGzR3tNJzRURERLrFkERaK1UoMfunBJSUKdHbtwnGd+WimkREVP8wJJHWvjp2Exfv5sLe0gyrxrSrcqVyIiIiY8WQRFq5eCcXG46Wr2a+bIQ/nO0s9FwRERFR7WBIomorLlVg9k/xKFMKGBzgguGBUn2XREREVGsYkqja1kVfR1JmPpxsJPh4ZABvsxERUb3GkETV8mdyFv7vt1sAgE9GB8DB2lzPFREREdUuhiR6pgJ5Gd7flQBBAMZ1aoZ+bZz1XRIREVGtY0iiZ1p58ApSswrh1sgSi4e10Xc5REREdYIhiap04voD/HA6FQCwZmw72FmY6bkiIiKiusGQRJXKLSzF3J8TAACvd/dE95ZOeq6IiIio7jAkUaU++u9l3JfJ4eVkjXmD/PRdDhERUZ1iSCKN/ncpHZFxd2EiAtaGBMLSXKzvkoiIiOoUQxJV8CBPjoWRlwAA/3ixBTo2b6znioiIiOoeQxKpEQQBCyMvIqugBH4utpjVz0ffJREREekFQxKp2X3+LqIT78NMLMJnoe0hMeVtNiIiapgYkkjlXk4Rlu6/DAAI69cKrV3t9FwRERGR/jAkEQBAqRQw9+cLyJOXoUPzRpjey1vfJREREekVQxIBALadTkHMjYewMDPB2nGBMBXzHw0iImrY+E1ISH5YgJUHrwAA5g/yg3cTGz1XREREpH8MSQ2cQingvZ/iUVyqRPcWjpgU7KnvkoiIiAwCQ1IDt+XkLZxPzYGNxBRrxgXCxESk75KIiIgMAkNSA3Y1Q4bPoq8DAJYMawO3RpZ6roiIiMhwMCQ1UCVlSry7MwElCiX6tW6KcZ2a6bskIiIig8KQ1ECt/zUJV9JlaGxlhpWjAyAS8TYbERHRkxiSGqC41Gx8dfwGAGDFqAA0tbXQc0VERESGhyGpgSkqUeC9nxKgFIDhgVIMDnDVd0lEREQGiSGpgVl9+CpuPSxAU1sJlo3w13c5REREBoshqQE5dfMhvv39NgBg1dh2aGRlrt+CiIiIDBhDUgORV1yKObsuAADGd22OPr5N9VwRERGRYWNIaiCW/5KIuzlFcHewxAdDWuu7HCIiIoPHkNQA/HrlPn46ewciEfDp2EDYSEz1XRIREZHBY0iq57ILSjB/z0UAwBs9vBDk7ajnioiIiIyDQYeksrIyLFq0CF5eXrC0tIS3tzeWLVsGpVIJACgtLcW8efMQEBAAa2trSKVSTJo0Cffu3auy34iICIhEogo/xcXFdTGsOrVo3yU8yJOjZVMbvD/QV9/lEBERGQ2Dvu+yatUqbN68Gd999x38/f1x9uxZTJkyBfb29pg1axYKCwtx/vx5LF68GIGBgcjOzkZYWBiGDx+Os2fPVtm3nZ0drl27pnbMwqJ+Laq4P+EeDlxIh9hEhHUhgbAwE+u7JCIiIqNh0CEpNjYWI0aMwJAhQwAAnp6e2L59uyoA2dvbIzo6Wu2cDRs2oGvXrkhNTUXz5s0r7VskEsHFxaX2itezTFkxFu+9BACY2acl2jVrpN+CiIiIjIxB327r2bMnfv31V1y/Xr5TfUJCAmJiYjB48OBKz8nNzYVIJEKjRo2q7Ds/Px8eHh5o1qwZhg4diri4OF2WrleCIGDe7gvILSpFWzc7/OullvouiYiIyOgY9EzSvHnzkJubCz8/P4jFYigUCqxYsQLjx4/X2L64uBjz58/HhAkTYGdnV2m/fn5+iIiIQEBAAGQyGb744gv06NEDCQkJ8PHx0XiOXC6HXC5X/S6TyZ5vcLVo55k0HLv2AOamJlgX0h5mYoPOwkRERAbJoEPSzp07sW3bNvz444/w9/dHfHw8wsLCIJVKMXnyZLW2paWleOWVV6BUKvHVV19V2W+3bt3QrVs31e89evRAx44dsWHDBqxfv17jOeHh4Vi6dOnzD6qWpWUVYvkviQCA9we0QitnWz1XREREZJxEgiAI+i6iMu7u7pg/fz5mzpypOvbxxx9j27ZtuHr1qupYaWkpQkJCcOvWLRw9ehSOjtq/5v7mm2/izp07OHTokMY/1zST5O7ujtzc3CpnreqSUilg/P/9gdPJWeji2Rg73gqG2ESk77KIiIgMhkwmg729fbW+vw16JqmwsBAmJuq3isRisWoJAODvgJSUlIRjx47VKCAJgoD4+HgEBARU2kYikUAikWjdd1369tRtnE7OgpW5GJ+OC2RAIiIieg4GHZKGDRuGFStWoHnz5vD390dcXBzWrVuHqVOnAihfR2ns2LE4f/48fvnlFygUCmRkZAAAHBwcYG5evoHrpEmT4ObmhvDwcADA0qVL0a1bN/j4+EAmk2H9+vWIj4/Hl19+qZ+B6sCNzHys/l/57NrCwa3h4Wit54qIiIiMm0GHpA0bNmDx4sWYMWMGMjMzIZVKMX36dCxZsgQAcOfOHezfvx8A0L59e7Vzjx07ht69ewMAUlNT1WakcnJy8NZbbyEjIwP29vbo0KEDTp48ia5du9bJuHStTKHEez/FQ16mRK9WTfBqUOVLHxAREVH1GPQzSYZMm3uatW39r0lYF30ddhamiHr3RbjY169FMYmIiHRFm+9vvhtu5C7dzcX6X5MAAEtH+DMgERER6QhDkhGTlykw+6d4lCkFDPJ3wcj2bvouiYiIqN5gSDJi66Kv4/r9fDjZmGPFqLYQifg2GxERka4wJBmps7ezsOXkLQDAylEBcLQx7OUJiIiIjA1DkhEqkJfhvV0JEARgTMdmGOBffzfqJSIi0heGJCMUfugKUh4VwtXeAkuGtdF3OURERPUSQ5IOlZWVYdGiRfDy8oKlpSW8vb2xbNkytRXC9+zZg4EDB8LJyQkikQjx8fHP7Pfy5csYM2YMPD09IRKJ8NWGDQCANWMDYW9pVqF9eHg4RCIRwsLCdDU0IiKiBochSYdWrVqFzZs3Y+PGjbhy5QpWr16NNWvWYMNfoQYACgoK0KNHD3zyySfV7rewsBDe3t5YsuxjmNk4AAAmBXugp49ThbZnzpzBli1b0K5du+cfEBERUQNm0CtuG5vY2FiMGDECQ4YMAQB4enpi+/btOHv2rKrNxIkTAQC3b9+udr9dunRBly5dMHtnPJQmpnCwNsP8l/0qtMvPz8err76K//u//8PHH3/8fIMhIiJq4DiTpEM9e/bEr7/+iuvXrwMAEhISEBMTg8GDBz933/+7lIE9cXchAjCivRuszCvm25kzZ2LIkCHo16/fc38eERFRQ8eZJB2aN28ecnNz4efnB7FYDIVCgRUrVmD8+PHP1e/DfDk+iLwIALCxMIW7g1WFNjt27MD58+dx5syZ5/osIiIiKseQpEM7d+7Etm3b8OOPP8Lf3x/x8fEICwuDVCrF5MmTa9SnIAj4IPIiHhWUwM/FFlc0PKidlpaGWbNmISoqChYW3JaEiIhIFxiSdGjOnDmYP38+XnnlFQBAQEAAUlJSEB4eXuOQFBl3F4cv34eZWIS1IYEY8nnFNufOnUNmZiY6deqkOqZQKHDy5Els3LgRcrkcYrG4Rp9PRETUUDEk6VBhYSFMTNQf8xKLxWpLAGjjXk4RPtx/GQAwq68P/KX2Gtv17dsXFy9eVDs2ZcoU+Pn5Yd68eQxIRERENcCQpEPDhg3DihUr0Lx5c/j7+yMuLg7r1q3D1KlTVW2ysrKQmpqKe/fuAQCuXbsGAHBxcYGLS/nK2ZMmTYJUKsVdn1HIKy5DgKs1utnnIT4+HiUlJbh79y7i4+NhY2ODli1bwtbWFm3btlWrxdraGo6OjhWOExERUfWIBEEQ9F2EMZLJZLC3t0dubi7s7OwAAHl5eVi8eDEiIyORmZkJqVSK8ePHY8mSJTA3NwcAREREYMqUKRX6+/DDD/HRRx8BAHr37g2FtRPSAqZAYmqCf49sjj5dKoadF198EcePH9dYX+/evdG+fXt8/vnnOhkvERFRfaDp+7syDEk1pM3fZG39mfwIr/3nT5QolFgytA2m9vTSaf9EREQNlTbf37zdZmC2/5mKBXv+fr7I0pzPExEREekDF5M0IOm5RVi4R/0B7EWRl5CeW6SnioiIiBouhiQDkvywAE/f+1QIAm4/LNRLPURERA0ZQ5IB8XKyholI/ZhYJIKnU8UVtomIiKh2MSQZEFd7S4SPDoBYVJ6UxCIRVo5uC1d7Sz1XRkRE1PDwwW0DE9qlOXq1aoLbDwvh6WTFgERERKQnDEkGyNXekuGIiIhIz3i7jYiIiEgDhiQiIiIiDRiSiIiIiDRgSCIiIiLSgCGJiIiISAOGJCIiIiINGJKIiIiINGBIIiIiItKAIYmIiIhIA4YkIiIiIg0YkoiIiIg0YEgiIiIi0sCgQ1JZWRkWLVoELy8vWFpawtvbG8uWLYNSqVS1EQQBH330EaRSKSwtLdG7d29cvnz5mX3v3r0bbdq0gUQiQZs2bRAZGVmbQyEiIiIjY9AhadWqVdi8eTM2btyIK1euYPXq1VizZg02bNigarN69WqsW7cOGzduxJkzZ+Di4oL+/fsjLy+v0n5jY2MRGhqKiRMnIiEhARMnTkRISAhOnz5dF8MiIiIiIyASBEHQdxGVGTp0KJydnfH111+rjo0ZMwZWVlbYunUrBEGAVCpFWFgY5s2bBwCQy+VwdnbGqlWrMH36dI39hoaGQiaT4dChQ6pjgwYNQuPGjbF9+/Zq1SaTyWBvb4/c3FzY2dk9xyiJiIiormjz/W1aRzXVSM+ePbF582Zcv34drVq1QkJCAmJiYvD5558DAJKTk5GRkYEBAwaozpFIJHjxxRdx6tSpSkNSbGws3n33XbVjAwcOVPWriVwuh1wuV/2em5sLoPxvNhERERmHx9/b1ZkjMuiQNG/ePOTm5sLPzw9isRgKhQIrVqzA+PHjAQAZGRkAAGdnZ7XznJ2dkZKSUmm/GRkZGs953J8m4eHhWLp0aYXj7u7u1R4PERERGYa8vDzY29tX2cagQ9LOnTuxbds2/Pjjj/D390d8fDzCwsIglUoxefJkVTuRSKR2niAIFY49TdtzFixYgNmzZ6t+VyqVyMrKgqOj4zM/S1symQzu7u5IS0url7fyOD7jV9/HWN/HB9T/MXJ8xq+2xigIAvLy8iCVSp/Z1qBD0pw5czB//ny88sorAICAgACkpKQgPDwckydPhouLC4DymSFXV1fVeZmZmRVmip7k4uJSYdboWedIJBJIJBK1Y40aNdJ2SFqxs7Ort//wAxxffVDfx1jfxwfU/zFyfMavNsb4rBmkxwz67bbCwkKYmKiXKBaLVUsAeHl5wcXFBdHR0ao/LykpwYkTJ9C9e/dK+w0ODlY7BwCioqKqPIeIiIgaFoOeSRo2bBhWrFiB5s2bw9/fH3FxcVi3bh2mTp0KoPyWWVhYGFauXAkfHx/4+Phg5cqVsLKywoQJE1T9TJo0CW5ubggPDwcAzJo1C7169cKqVaswYsQI7Nu3D0eOHEFMTIxexklERESGx6BD0oYNG7B48WLMmDEDmZmZkEqlmD59OpYsWaJqM3fuXBQVFWHGjBnIzs5GUFAQoqKiYGtrq2qTmpqqNiPVvXt37NixA4sWLcLixYvRokUL7Ny5E0FBQXU6vspIJBJ8+OGHFW7v1Rccn/Gr72Os7+MD6v8YOT7jZwhjNOh1koiIiIj0xaCfSSIiIiLSF4YkIiIiIg0YkoiIiIg0YEgiIiIi0oAhSU+++uoreHl5wcLCAp06dcJvv/1WZfsTJ06gU6dOsLCwgLe3NzZv3lxHldaMNuM7fvw4RCJRhZ+rV6/WYcXVd/LkSQwbNgxSqRQikQh79+595jnGdP20HZ+xXb/w8HB06dIFtra2aNq0KUaOHIlr16498zxjuYY1GZ+xXcNNmzahXbt2qkUGg4OD1TYs18RYrh+g/fiM7fo9LTw8XLWkT1X0cQ0ZkvRg586dCAsLwwcffIC4uDi88MILePnll5GamqqxfXJyMgYPHowXXngBcXFxWLhwId555x3s3r27jiuvHm3H99i1a9eQnp6u+vHx8amjirVTUFCAwMBAbNy4sVrtje36aTu+x4zl+p04cQIzZ87EH3/8gejoaJSVlWHAgAEoKCio9BxjuoY1Gd9jxnINmzVrhk8++QRnz57F2bNn8dJLL2HEiBG4fPmyxvbGdP0A7cf3mLFcvyedOXMGW7ZsQbt27apsp7drKFCd69q1q/CPf/xD7Zifn58wf/58je3nzp0r+Pn5qR2bPn260K1bt1qr8XloO75jx44JAITs7Ow6qE63AAiRkZFVtjG26/ek6ozPmK+fIAhCZmamAEA4ceJEpW2M+RpWZ3zGfg0FQRAaN24s/Oc//9H4Z8Z8/R6ranzGev3y8vIEHx8fITo6WnjxxReFWbNmVdpWX9eQM0l1rKSkBOfOncOAAQPUjg8YMACnTp3SeE5sbGyF9gMHDsTZs2dRWlpaa7XWRE3G91iHDh3g6uqKvn374tixY7VZZp0ypuv3PIz1+uXm5gIAHBwcKm1jzNewOuN7zBivoUKhwI4dO1BQUIDg4GCNbYz5+lVnfI8Z2/WbOXMmhgwZgn79+j2zrb6uIUNSHXv48CEUCkWFzXSdnZ0rbLr7WEZGhsb2ZWVlePjwYa3VWhM1GZ+rqyu2bNmC3bt3Y8+ePfD19UXfvn1x8uTJuii51hnT9asJY75+giBg9uzZ6NmzJ9q2bVtpO2O9htUdnzFew4sXL8LGxgYSiQT/+Mc/EBkZiTZt2mhsa4zXT5vxGeP127FjB86fP6/aLuxZ9HUNDXpbkvpMJBKp/S4IQoVjz2qv6bih0GZ8vr6+8PX1Vf0eHByMtLQ0fPrpp+jVq1et1llXjO36acOYr98///lPXLhwoVr7NhrjNazu+IzxGvr6+iI+Ph45OTnYvXs3Jk+ejBMnTlQaJIzt+mkzPmO7fmlpaZg1axaioqJgYWFR7fP0cQ05k1THnJycIBaLK8yqZGZmVkjJj7m4uGhsb2pqCkdHx1qrtSZqMj5NunXrhqSkJF2XpxfGdP10xRiu37/+9S/s378fx44dQ7Nmzapsa4zXUJvxaWLo19Dc3BwtW7ZE586dER4ejsDAQHzxxRca2xrj9dNmfJoY8vU7d+4cMjMz0alTJ5iamsLU1BQnTpzA+vXrYWpqCoVCUeEcfV1DhqQ6Zm5ujk6dOiE6OlrteHR0NLp3767xnODg4Arto6Ki0LlzZ5iZmdVarTVRk/FpEhcXB1dXV12XpxfGdP10xZCvnyAI+Oc//4k9e/bg6NGj8PLyeuY5xnQNazI+TQz5GmoiCALkcrnGPzOm61eZqsaniSFfv759++LixYuIj49X/XTu3Bmvvvoq4uPjIRaLK5yjt2tYq4+Fk0Y7duwQzMzMhK+//lpITEwUwsLCBGtra+H27duCIAjC/PnzhYkTJ6ra37p1S7CyshLeffddITExUfj6668FMzMz4eeff9bXEKqk7fg+++wzITIyUrh+/bpw6dIlYf78+QIAYffu3foaQpXy8vKEuLg4IS4uTgAgrFu3ToiLixNSUlIEQTD+66ft+Izt+r399tuCvb29cPz4cSE9PV31U1hYqGpjzNewJuMztmu4YMEC4eTJk0JycrJw4cIFYeHChYKJiYkQFRUlCIJxXz9B0H58xnb9NHn67TZDuYYMSXry5ZdfCh4eHoK5ubnQsWNHtddzJ0+eLLz44otq7Y8fPy506NBBMDc3Fzw9PYVNmzbVccXa0WZ8q1atElq0aCFYWFgIjRs3Fnr27CkcOHBAD1VXz+PXbZ/+mTx5siAIxn/9tB2fsV0/TWMDIHz77beqNsZ8DWsyPmO7hlOnTlX9+6VJkyZC3759VQFCEIz7+gmC9uMztuunydMhyVCuoUgQ/nryiYiIiIhU+EwSERERkQYMSUREREQaMCQRERERacCQRERERKQBQxIRERGRBgxJRERERBowJBERERFpwJBERKQjx48fh0gkQk5Ojr5LISIdYEgiIiIi0oAhiYiIiEgDhiQiqjcEQcDq1avh7e0NS0tLBAYG4ueffwbw962wAwcOIDAwEBYWFggKCsLFixfV+ti9ezf8/f0hkUjg6emJtWvXqv25XC7H3Llz4e7uDolEAh8fH3z99ddqbc6dO4fOnTvDysoK3bt3x7Vr12p34ERUKxiSiKjeWLRoEb799lts2rQJly9fxrvvvovXXnsNJ06cULWZM2cOPv30U5w5cwZNmzbF8OHDUVpaCqA83ISEhOCVV17BxYsX8dFHH2Hx4sWIiIhQnT9p0iTs2LED69evx5UrV7B582bY2Nio1fHBBx9g7dq1OHv2LExNTTF16tQ6GT8R6RY3uCWieqGgoABOTk44evQogoODVcenTZuGwsJCvPXWW+jTpw927NiB0NBQAEBWVhaaNWuGiIgIhISE4NVXX8WDBw8QFRWlOn/u3Lk4cOAALl++jOvXr8PX1xfR0dHo169fhRqOHz+OPn364MiRI+jbty8A4ODBgxgyZAiKiopgYWFRy38XiEiXOJNERPVCYmIiiouL0b9/f9jY2Kh+vv/+e9y8eVPV7skA5eDgAF9fX1y5cgUAcOXKFfTo0UOt3x49eiApKQkKhQLx8fEQi8V48cUXq6ylXbt2qr92dXUFAGRmZj73GImobpnquwAiIl1QKpUAgAMHDsDNzU3tzyQSiVpQeppIJAJQ/kzT479+7MnJdktLy2rVYmZmVqHvx/URkfHgTBIR1Qtt2rSBRCJBamoqWrZsqfbj7u6uavfHH3+o/jo7OxvXr1+Hn5+fqo+YmBi1fk+dOoVWrVpBLBYjICAASqVS7RknIqq/OJNERPWCra0t3n//fbz77rtQKpXo2bMnZDIZTp06BRsbG3h4eAAAli1bBkdHRzg7O+ODDz6Ak5MTRo4cCQB477330KVLFyxfvhyhoaGIjY3Fxo0b8dVXXwEAPD09MXnyZEydOhXr169HYGAgUlJSkJmZiZCQEH0NnYhqCUMSEdUby5cvR9OmTREeHo5bt26hUaNG6NixIxYuXKi63fXJJ59g1qxZSEpKQmBgIPbv3w9zc3MAQMeOHfHTTz9hyZIlWL58OVxdXbFs2TK8/vrrqs/YtGkTFi5ciBkzZuDRo0do3rw5Fi5cqI/hElEt49ttRNQgPH7zLDs7G40aNdJ3OURkBPhMEhEREZEGDElEREREGvB2GxEREZEGnEkiIiIi0oAhiYiIiEgDhiQiIiIiDRiSiIiIiDRgSCIiIiLSgCGJiIiISAOGJCIiIiINGJKIiIiINGBIIiIiItLg/wE7w+URUhR3kQAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plt.plot(np.arange(len(epochs_x)), epochs_acc, marker = '.')\n", + "plt.xlabel('epoch')\n", + "plt.ylabel('test accuracy')\n", + "plt.ylim(80, 100)\n", + "for i, txt in enumerate(epochs_acc):\n", + " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom')\n", + "plt.show()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "speck-rescnn", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From eb4140d2f0feab8ff60ae8ea29922f272d33a101 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Sat, 27 Apr 2024 01:26:59 +0200 Subject: [PATCH 052/379] DVS gesture: sequential vs residual cnn --- .../baseline-SCNN-example_3.ipynb | 385 +++++++-- .../non-sequential-SCNN-example_3.ipynb | 808 ++++++++++++++++++ 2 files changed, 1145 insertions(+), 48 deletions(-) create mode 100644 tests/test_nonsequential/non-sequential-SCNN-example_3.ipynb diff --git a/tests/test_nonsequential/baseline-SCNN-example_3.ipynb b/tests/test_nonsequential/baseline-SCNN-example_3.ipynb index 3b694e0d..800ef861 100644 --- a/tests/test_nonsequential/baseline-SCNN-example_3.ipynb +++ b/tests/test_nonsequential/baseline-SCNN-example_3.ipynb @@ -31,7 +31,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 2, @@ -56,9 +56,9 @@ "metadata": {}, "outputs": [], "source": [ - "batch_size = 8\n", - "num_workers = 4\n", - "epochs = 5\n", + "batch_size = 3\n", + "num_workers = 1\n", + "epochs = 10\n", "lr = 1e-3" ] }, @@ -79,7 +79,7 @@ "metadata": {}, "outputs": [], "source": [ - "n_time_steps = 100\n", + "n_time_steps = 50\n", "to_raster = ToFrame(sensor_size=DVSGesture.sensor_size, n_time_bins=n_time_steps)\n", "\n", "snn_train_dataset = DVSGesture(save_to=root_dir, train=True, transform=to_raster)\n", @@ -95,7 +95,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "The transformed array is in shape [Time-Step, Channel, Height, Width] --> (100, 2, 128, 128)\n" + "The transformed array is in shape [Time-Step, Channel, Height, Width] --> (50, 2, 128, 128)\n" ] } ], @@ -132,7 +132,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "device: NVIDIA GeForce RTX 3070 Ti\n" + "device: NVIDIA RTX A4000\n" ] } ], @@ -177,7 +177,7 @@ " self.fc3 = nn.Linear(100, 100, bias=False)\n", " self.iaf6 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", "\n", - " self.fc4 = nn.Linear(100, 10, bias=False)\n", + " self.fc4 = nn.Linear(100, 11, bias=False)\n", " self.iaf7 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", "\n", " def detach_neuron_states(self):\n", @@ -366,77 +366,366 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "5a519f3a02424e86843a42fee66ba952", + "model_id": "d1e6254a097e41c198599735d8d1a5c2", "version_major": 2, "version_minor": 0 }, "text/plain": [ - " 0%| | 0/134 [00:00= 0 && t < n_classes` failed.\n", - "../aten/src/ATen/native/cuda/Loss.cu:250: nll_loss_forward_reduce_cuda_kernel_2d: block: [0,0,0], thread: [1,0,0] Assertion `t >= 0 && t < n_classes` failed.\n" + "Epoch 0 accuracy: 33.71212121212121\n" ] }, { - "ename": "RuntimeError", - "evalue": "CUDA error: device-side assert triggered\nCUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.\nFor debugging consider passing CUDA_LAUNCH_BLOCKING=1.\nCompile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.\n", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[15], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m epochs_x, epochs_y, epochs_acc \u001b[38;5;241m=\u001b[39m train(snn_train_dataloader, snn, loss_fn, optimizer, epochs, test, snn_test_dataloader)\n", - "Cell \u001b[0;32mIn[13], line 30\u001b[0m, in \u001b[0;36mtrain\u001b[0;34m(dataloader, model, loss_fn, optimizer, epochs, test_func, dataloader_test)\u001b[0m\n\u001b[1;32m 28\u001b[0m \u001b[38;5;66;03m# gradient update\u001b[39;00m\n\u001b[1;32m 29\u001b[0m optimizer\u001b[38;5;241m.\u001b[39mzero_grad()\n\u001b[0;32m---> 30\u001b[0m loss\u001b[38;5;241m.\u001b[39mbackward()\n\u001b[1;32m 31\u001b[0m optimizer\u001b[38;5;241m.\u001b[39mstep()\n\u001b[1;32m 33\u001b[0m \u001b[38;5;66;03m# detach the neuron states and activations from current computation graph(necessary)\u001b[39;00m\n", - "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/_tensor.py:522\u001b[0m, in \u001b[0;36mTensor.backward\u001b[0;34m(self, gradient, retain_graph, create_graph, inputs)\u001b[0m\n\u001b[1;32m 512\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m has_torch_function_unary(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m 513\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m handle_torch_function(\n\u001b[1;32m 514\u001b[0m Tensor\u001b[38;5;241m.\u001b[39mbackward,\n\u001b[1;32m 515\u001b[0m (\u001b[38;5;28mself\u001b[39m,),\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 520\u001b[0m inputs\u001b[38;5;241m=\u001b[39minputs,\n\u001b[1;32m 521\u001b[0m )\n\u001b[0;32m--> 522\u001b[0m torch\u001b[38;5;241m.\u001b[39mautograd\u001b[38;5;241m.\u001b[39mbackward(\n\u001b[1;32m 523\u001b[0m \u001b[38;5;28mself\u001b[39m, gradient, retain_graph, create_graph, inputs\u001b[38;5;241m=\u001b[39minputs\n\u001b[1;32m 524\u001b[0m )\n", - "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/autograd/__init__.py:259\u001b[0m, in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[0m\n\u001b[1;32m 250\u001b[0m inputs \u001b[38;5;241m=\u001b[39m (\n\u001b[1;32m 251\u001b[0m (inputs,)\n\u001b[1;32m 252\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(inputs, (torch\u001b[38;5;241m.\u001b[39mTensor, graph\u001b[38;5;241m.\u001b[39mGradientEdge))\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 255\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28mtuple\u001b[39m()\n\u001b[1;32m 256\u001b[0m )\n\u001b[1;32m 258\u001b[0m grad_tensors_ \u001b[38;5;241m=\u001b[39m _tensor_or_tensors_to_tuple(grad_tensors, \u001b[38;5;28mlen\u001b[39m(tensors))\n\u001b[0;32m--> 259\u001b[0m grad_tensors_ \u001b[38;5;241m=\u001b[39m _make_grads(tensors, grad_tensors_, is_grads_batched\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m)\n\u001b[1;32m 260\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m retain_graph \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 261\u001b[0m retain_graph \u001b[38;5;241m=\u001b[39m create_graph\n", - "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/autograd/__init__.py:142\u001b[0m, in \u001b[0;36m_make_grads\u001b[0;34m(outputs, grads, is_grads_batched)\u001b[0m\n\u001b[1;32m 136\u001b[0m msg \u001b[38;5;241m=\u001b[39m (\n\u001b[1;32m 137\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mgrad can be implicitly created only for real scalar outputs\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 138\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m but got \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mout\u001b[38;5;241m.\u001b[39mdtype\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 139\u001b[0m )\n\u001b[1;32m 140\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(msg)\n\u001b[1;32m 141\u001b[0m new_grads\u001b[38;5;241m.\u001b[39mappend(\n\u001b[0;32m--> 142\u001b[0m torch\u001b[38;5;241m.\u001b[39mones_like(out, memory_format\u001b[38;5;241m=\u001b[39mtorch\u001b[38;5;241m.\u001b[39mpreserve_format)\n\u001b[1;32m 143\u001b[0m )\n\u001b[1;32m 144\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 145\u001b[0m new_grads\u001b[38;5;241m.\u001b[39mappend(\u001b[38;5;28;01mNone\u001b[39;00m)\n", - "\u001b[0;31mRuntimeError\u001b[0m: CUDA error: device-side assert triggered\nCUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.\nFor debugging consider passing CUDA_LAUNCH_BLOCKING=1.\nCompile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.\n" + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "9130fb565ce7454980dcff9772b98f26", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/359 [00:00" + " 0%| | 0/88 [00:00" ] @@ -461,12 +750,12 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "metadata": {}, "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkkAAAG2CAYAAABrrBJlAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAABcAklEQVR4nO3deVxU5f4H8M8wwLCjgAKDyKIIiog7omaaa+4rpKWmWd703iTLNbXUlNS0Ukuvv1tRWmqmqDf1CuYWSeYCuOCCioAKorIM6wAz5/cHOTkyIIMDMwOf9+vF68bxOc98n063+fScc55HJAiCACIiIiJSY6LvAoiIiIgMEUMSERERkQYMSUREREQaMCQRERERacCQRERERKQBQxIRERGRBgxJRERERBowJBERERFpwJBEREREpAFDEhEREZEGeg1JJ0+exLBhwyCVSiESibB37161PxcEAR999BGkUiksLS3Ru3dvXL58Wa2NXC7Hv/71Lzg5OcHa2hrDhw/HnTt3nvnZX331Fby8vGBhYYFOnTrht99+0+XQiIiIyMjpNSQVFBQgMDAQGzdu1Pjnq1evxrp167Bx40acOXMGLi4u6N+/P/Ly8lRtwsLCEBkZiR07diAmJgb5+fkYOnQoFApFpZ+7c+dOhIWF4YMPPkBcXBxeeOEFvPzyy0hNTdX5GImIiMg4iQxlg1uRSITIyEiMHDkSQPksklQqRVhYGObNmwegfNbI2dkZq1atwvTp05Gbm4smTZpg69atCA0NBQDcu3cP7u7uOHjwIAYOHKjxs4KCgtCxY0ds2rRJdax169YYOXIkwsPDa3egREREZBRM9V1AZZKTk5GRkYEBAwaojkkkErz44os4deoUpk+fjnPnzqG0tFStjVQqRdu2bXHq1CmNIamkpATnzp3D/Pnz1Y4PGDAAp06dqrQeuVwOuVyu+l2pVCIrKwuOjo4QiUTPM1QiIiKqI4IgIC8vD1KpFCYmVd9QM9iQlJGRAQBwdnZWO+7s7IyUlBRVG3NzczRu3LhCm8fnP+3hw4dQKBQa+63sHAAIDw/H0qVLtR4HERERGZ60tDQ0a9asyjYGG5Iee3qWRhCEZ87cVKeNtv0uWLAAs2fPVv2em5uL5s2bIy0tDXZ2dlV+FhERERkGmUwGd3d32NraPrOtwYYkFxcXAOWzRa6urqrjmZmZqlkgFxcXlJSUIDs7W202KTMzE927d9fYr5OTE8RicYVZoyf71UQikUAikVQ4bmdnx5BERERkZKrzqIzBrpPk5eUFFxcXREdHq46VlJTgxIkTqgDUqVMnmJmZqbVJT0/HpUuXKg1J5ubm6NSpk9o5ABAdHV3pOURERNTw6HUmKT8/Hzdu3FD9npycjPj4eDg4OKB58+YICwvDypUr4ePjAx8fH6xcuRJWVlaYMGECAMDe3h5vvPEG3nvvPTg6OsLBwQHvv/8+AgIC0K9fP1W/ffv2xahRo/DPf/4TADB79mxMnDgRnTt3RnBwMLZs2YLU1FT84x//qNu/AURERGSw9BqSzp49iz59+qh+f/zMz+TJkxEREYG5c+eiqKgIM2bMQHZ2NoKCghAVFaV2H/Gzzz6DqakpQkJCUFRUhL59+yIiIgJisVjV5ubNm3j48KHq99DQUDx69AjLli1Deno62rZti4MHD8LDw6MORk1ERETGwGDWSTI2MpkM9vb2yM3N5TNJRERERkKb72+DfSaJiIiISJ8YkoiIiIg0YEgiIiIi0oAhiYiIiEgDhiQiIiIiDRiSiIiIiDRgSCIiIiLSgCGJiIiISAOGJCIiIiINGJKIiIiINGBIIiIiItKAIYmIiIhIA4YkIiIiIg0YkoiIiIg0YEgiIiIi0oAhiYiIiEgDhiQiIiIiDRiSiIiIiDRgSCIiIiLSgCGJiIiISAOGJCIiIiINGJKIiIiINGBIIiIiItKAIYmIiIhIA4YkIiIiIg0YkoiIiIg0YEgiIiIi0oAhiYiIiEgDhiQiIiIiDRiSiIiIiDRgSCIiIiLSgCGJiIiISAOGJCIiIiINGJKIiIiINGBIIiIiItLA4ENSXl4ewsLC4OHhAUtLS3Tv3h1nzpxR/blIJNL4s2bNmkr7jIiI0HhOcXFxXQyJiIiIjICpvgt4lmnTpuHSpUvYunUrpFIptm3bhn79+iExMRFubm5IT09Xa3/o0CG88cYbGDNmTJX92tnZ4dq1a2rHLCwsdF4/ERERGSeDDklFRUXYvXs39u3bh169egEAPvroI+zduxebNm3Cxx9/DBcXF7Vz9u3bhz59+sDb27vKvkUiUYVziYiIiB4z6NttZWVlUCgUFWZ4LC0tERMTU6H9/fv3ceDAAbzxxhvP7Ds/Px8eHh5o1qwZhg4diri4uCrby+VyyGQytR8iIiKqvww6JNna2iI4OBjLly/HvXv3oFAosG3bNpw+fbrCbTYA+O6772Bra4vRo0dX2a+fnx8iIiKwf/9+bN++HRYWFujRoweSkpIqPSc8PBz29vaqH3d39+ceHxERERkukSAIgr6LqMrNmzcxdepUnDx5EmKxGB07dkSrVq1w/vx5JCYmqrX18/ND//79sWHDBq0+Q6lUomPHjujVqxfWr1+vsY1cLodcLlf9LpPJ4O7ujtzcXNjZ2Wk/MCIiIqpzMpkM9vb21fr+NuhnkgCgRYsWOHHiBAoKCiCTyeDq6orQ0FB4eXmptfvtt99w7do17Ny5U+vPMDExQZcuXaqcSZJIJJBIJFr3TURERMbJoG+3Pcna2hqurq7Izs7G4cOHMWLECLU///rrr9GpUycEBgZq3bcgCIiPj4erq6uuyiUiIiIjZ/AzSYcPH4YgCPD19cWNGzcwZ84c+Pr6YsqUKao2MpkMu3btwtq1azX2MWnSJLi5uSE8PBwAsHTpUnTr1g0+Pj6QyWRYv3494uPj8eWXX9bJmIiIiMjwGXxIys3NxYIFC3Dnzh04ODhgzJgxWLFiBczMzFRtduzYAUEQMH78eI19pKamwsTk70mznJwcvPXWW8jIyIC9vT06dOiAkydPomvXrrU+HiIiIjIOBv/gtqHS5sEvIiIiMgzafH8bzTNJRERERHWJIYmIiIhIA4YkIiIiIg0YkoiIiIg0YEgiIiIi0oAhiYiIiEgDhiQiIiIiDRiSiIiIiDRgSCIiIiLSgCGJiIiISAOGJCIiIiINGJKIiIiINGBIIiIiItKAIYmIiIhIA4YkIiIiIg0YkoiIiIg0YEgiIiIi0oAhiYiIiEgDhiQiIiIiDRiSiIiIiDRgSCIiIiLSgCGJiIiISAOGJCIiIiINGJKIiIiINGBIIiIiItKAIYmIiIhIA4YkIiIiIg0YkoiIiIg0YEgiIiIi0oAhiYiIiEgDhiQiIiIiDRiSiIiIiDRgSCIiIiLSgCGJiIiISAOGJCIiIiINDD4k5eXlISwsDB4eHrC0tET37t1x5swZ1Z+//vrrEIlEaj/dunV7Zr+7d+9GmzZtIJFI0KZNG0RGRtbmMIiIiMjIGHxImjZtGqKjo7F161ZcvHgRAwYMQL9+/XD37l1Vm0GDBiE9PV31c/DgwSr7jI2NRWhoKCZOnIiEhARMnDgRISEhOH36dG0Ph4iIiIyESBAEQd9FVKaoqAi2trbYt28fhgwZojrevn17DB06FB9//DFef/115OTkYO/evdXuNzQ0FDKZDIcOHVIdGzRoEBo3bozt27dXqw+ZTAZ7e3vk5ubCzs6u2p9NRERE+qPN97dBzySVlZVBoVDAwsJC7bilpSViYmJUvx8/fhxNmzZFq1at8OabbyIzM7PKfmNjYzFgwAC1YwMHDsSpU6d0VzwREREZNYMOSba2tggODsby5ctx7949KBQKbNu2DadPn0Z6ejoA4OWXX8YPP/yAo0ePYu3atThz5gxeeuklyOXySvvNyMiAs7Oz2jFnZ2dkZGRUeo5cLodMJlP7ISIiovrLoEMSAGzduhWCIMDNzQ0SiQTr16/HhAkTIBaLAZTfOhsyZAjatm2LYcOG4dChQ7h+/ToOHDhQZb8ikUjtd0EQKhx7Unh4OOzt7VU/7u7uzz84IiIiMlgGH5JatGiBEydOID8/H2lpafjzzz9RWloKLy8vje1dXV3h4eGBpKSkSvt0cXGpMGuUmZlZYXbpSQsWLEBubq7qJy0trWYDIiIiIqNg8CHpMWtra7i6uiI7OxuHDx/GiBEjNLZ79OgR0tLS4OrqWmlfwcHBiI6OVjsWFRWF7t27V3qORCKBnZ2d2g8RERHVX6b6LuBZDh8+DEEQ4Ovrixs3bmDOnDnw9fXFlClTkJ+fj48++ghjxoyBq6srbt++jYULF8LJyQmjRo1S9TFp0iS4ubkhPDwcADBr1iz06tULq1atwogRI7Bv3z4cOXJE7WFwIiIiatgMfiYpNzcXM2fOhJ+fHyZNmoSePXsiKioKZmZmEIvFuHjxIkaMGIFWrVph8uTJaNWqFWJjY2Fra6vqIzU1VfWgNwB0794dO3bswLfffot27dohIiICO3fuRFBQkD6GSERET3jWIsJPmj59OkQiET7//PNn9vusRYS1+VxqGAx+JikkJAQhISEa/8zS0hKHDx9+Zh/Hjx+vcGzs2LEYO3bs85ZHREQ6Nm3aNFy6dAlbt26FVCrFtm3b0K9fPyQmJsLNzU3Vbu/evTh9+jSkUukz+3y8iPDy5csxatQoREZGIiQkBDExMar/QK7u51LDYdCLSRoyLiZJRKR71VlEGADu3r2LoKAgHD58GEOGDEFYWBjCwsIq7fdZiwhX93PJ+NWbxSSJiKhhqc4iwkqlEhMnTsScOXPg7+9frX6ftYhwdRcvpoaFIYmIiAxGdRYRXrVqFUxNTfHOO+9Uu99nLSJcnc+lhochiYiIDEpViwifO3cOX3zxBSIiIqpcAFiTZy0i/KzFi6nhYUgiIiKDUtUiwr/99hsyMzPRvHlzmJqawtTUFCkpKXjvvffg6elZaZ/VWURY28WLqf5jSCIiIoOkaRHhiRMn4sKFC4iPj1f9SKVSzJkzp8q3nbVZRLi6ixdT/WfwSwAQEVHDUtUiwmZmZnB0dFRrb2ZmBhcXF/j6+qqO1WQR4ao+lxomziQREZFBqWoR4eq6ces2Eq4nIz23CED1FhHWxedS/cJ1kmqI6yQRERmmnWdSsWDPRSgFwEQEhI8OQGiX5vouiwwE10kiIqIGRxAEnLrxEPN3lwckAFAKwII9F3ElXabf4sgo8ZkkIiIySkqlgCsZMvyZnIXTt7Lw5+0sZBWUVGwnAC9/8Rua2krQ2tUOfq62aO1S/r8tmtjATMz5AtKMIYmIiIxCmUKJy/dkOJ38CKdvZeHM7SzIisvU2khMRZCXaX6KJDNPjsy8Bzhx/YHqmJlYhJZNbdHaxbY8PLnawc/FDk1sJbU6FjIODElERGSQSsqUuHAnB6eTs3A6OQvnbmehoESh1sbaXIxOng4I8ir/adesESLj7mDhnktQCALEIhFWjm6LIe2kuJaRh6sZMlxN//t/8+RluJIuK78dF/d3v0425vBzsYOfi61q9qllUxtITLmwZEPCB7driA9uExHpVnGpAnGpOaqZori0bBSXKtXa2FmYoquXA7p6OSDIyxH+UjuYarhdlp5bhNsPC+HpZAVXe0uNnycIAu5kF+FqRh6upstw5a/glPyoAJq+GcUmIrRoYq2abXp8287ZTqL16t+kP9p8fzMk1RBDEhHR8ymQl+FcSnb5M0XJj5CQlosShXoocrA2R1dPBwR5lwcjPxc7iE1qN5AUlShw/X4erqTLcDUjTzXT9PStvccaW5mphSY/V1u0craFhRlnnQwRQ1IdYEgiItJOblEpzqWUP2T9R3IWLt3NhUKp/hXU1FaCIG9HdPVyQDcvB7RsamMQszSCICBDVvxXYMpTzT7delhQYQxA+dIDXk7W8HO1Q2vVLTs7SO0tDGI8DRlDUh1gSCIiqlpWQQn+TM5SzRQlpssq3MZya2RZ/jyRtwO6ejnC09HKqEJEcakCNzLzK8w6ZReWamxva2GK1i52aO1qCz/X8meefF1sYWXOR4TrCkNSHWBIIiJSl5lXrHod/3TyI1y/n1+hjaejFYK8ymeKgrwd0KyxlR4qrV2CIOBBnhxXHj/r9FeAupGZjzINs04iEeDhYFXhWadmjS1hUsu3FhsihqQ6wJBERA3dvZwinE5+pApGtx4WVGjj09Tmr0DkiK6eDnCxt9BDpYahpEyJmw/UZ52uZuThQZ5cY3sbiSl8XWzh51I+69Tmr2edbC0Mf5uUvLw8LF68GJGRkcjMzESHDh3wxRdfoEuXLigtLcWiRYtw8OBB3Lp1C/b29ujXrx8++eQTSKXSSvssLS1FeHg4vvvuO9y9exe+vr5YtWoVBg0aVK3PfUyb72/O7xER0TMJgoDUrMLy1/H/mim6k12k1kYkAvxc7FSv43fxcoCTDdcbeszc1AStXe3Q2lX9i/lhvly1LMGV9PLwdCMzH/l/Pdh+LiVbrb27gyX8XMr7af1XgPJwsDKoWadp06bh0qVL2Lp1K6RSKbZt24Z+/fohMTERNjY2OH/+PBYvXozAwEBkZ2cjLCwMw4cPx9mzZyvtc9GiRdi2bRv+7//+D35+fjh8+DBGjRqFU6dOoUOHDs/8XDc3N63HwZmkGuJMEhHVZ4Ig4OaDArWZogxZsVobsYkIbaV2qtfxu3g6wN7K8Gc5jEGpQonkhwXqs07peRWuwWOWZmL4utiWP+v01/pOfi52erkeRUVFsLW1xb59+zBkyBDV8fbt22Po0KH4+OOPK5xz5swZdO3aFSkpKWjeXPM+e1KpFB988AFmzpypOjZy5EjY2Nhg27Zt1f5cziQREZFWlEoB1+7nqR6y/jM5Cw/z1bf4MBOL0K5ZIwT9tU5RZ08H2Ej4NVIbzMQmaOVcfnttxBPHswtKnrhVVx6grmXkoahUgfi0HMSn5aj149bI8q/bdbaq2SdPRyuNa0vpSllZGRQKBSws1G+tWlpaIiYmRuM5ubm5EIlEaNSoUaX9yuXyKvusyec+C2eSaogzSURkzBRKAYmPt/hILt/iI+epN7LMTU3Qwb0Rgrwd0c3LAR2aN4alOdf+MTQKpYDkhwVqq4lfSc/D3Zwije0lpuUB7MnVxFu72KGxtbnOaurevTvMzc3x448/wtnZGdu3b8ekSZPg4+ODa9euqbUtLi5Gz5494efnh23btlXa54QJE5CQkIC9e/eiRYsW+PXXXzFixAgoFArI5fJqfy4f3K4DDElEZExKFUpcuJOrmik6dzsbeXL1xREtzcTo7Nn4r8UbHRHobs9tOIxYblGpaiuWx886PZ510sTZTvL3s05/zTx5N7Gu0QbAN2/exNSpU3Hy5EmIxWJ07NgRrVq1wvnz55GYmKhqV1painHjxiE1NRXHjx+v8vv0wYMHePPNN/Hf//4XIpEILVq0QL9+/fDtt9+isLCw2p/LkFQHGJKIyJAVlyqQkFa+79mfyVk4l5Jd4cvRVmKKzp6NEeTtiCAvB7R1s6/RFyIZD6Wy/AH8qxkyJKaXL1FwNSMPqVmFGtubi03QsqmN2mrirV3tqv1AfkFBAWQyGVxdXREaGor8/HwcOHAAQHlACgkJwa1bt3D06FE4OjpWq8/i4mI8evQIUqkU8+fPxy+//ILLly9X+3P5TBIRNVi18eoxAOTk5OCDDz7Anj17kJ2dDS8vL6xduxaDBw9Wtfnqq6+wZs0apKenw9/fH59//jleeOGF2h4yAKCwpKx837Nbj/BHchbi03JQUqa+xUcjKzN09Sx/nqibtyNau9b+Fh9kWExMRPB0soankzUGtXVVHc+Xl+FaxuPVxB/ftstDvrwMiekyJKbLANxVtXeykfw122SrWt+pRVPrCjOP1tbWsLa2RnZ2Ng4fPozVq1cD+DsgJSUl4dixY9UOSABgYWEBNzc3lJaWYvfu3QgJCanQprLP1RZnkmqIM0lEhik0NBSXLl3Cpk2bVK8Af/bZZ6pXj8eOHYs333xT7dXjsrKyKl89LikpQY8ePdC0aVMsXLgQzZo1Q1paGmxtbREYGAgA2LlzJyZOnIivvvoKPXr0wL///W/85z//QWJiYqVv6zyPvOJSnE3JxulbWfgz+REu3MmtsFChk41EtZp1kJcjfJraGNRr4mTYHm8A/PgNu8e37W5XsgGwqYkILZqUzzoJafFo1tgSvbu2R05GKubOnQuJRIKYmBiIRCKMGTMG58+fxy+//AJnZ2dVHw4ODjA3L382atKkSXBzc0N4eDgA4PTp07h79y7at2+Pu3fv4qOPPkJycjLOnz+veuD78OHDEAQBvr6+uHHjBubMmaP6XDOz8jf9eLutDjAkERme2nr1ePPmzVizZg2uXr2q+hft04KCgtCxY0ds2rRJdax169YYOXKk6l/yzyOnsARnbmfj9K3yB60v38vF04s3u9pb/PXmmSOCvB3g7WRtVFt8kHEoLCnD9fv5qtXEH68s/uQGwAVXfkPOye9QlvcQppZ28OrcB2Onz0H7llLYlWbj5e6BGvs+duwYevfuDQDo3bs3PD09ERERAQA4ceIE3n77bdy6dQs2NjYYPHhwhVngn376CQsWLMCdO3fg4OCAMWPGYMWKFbC3t1e1YUiqAwxJRIYnLy8PdnZ2OHLkCPr27as6HhwcDIlEguPHj1c458iRIxgwYABycnIq/f/y4MGD4eDgACsrK+zbtw9NmjTBhAkTMG/ePIjFYpSUlMDKygq7du3CqFGjVOfNmjUL8fHxOHHihNZjeZgv/2t9ovJQdO1+XoX/em/uYPXXGkXlM0XuDpYMRaQXgiAgPbe4wmritx7kVwjzQPkGwN5NbJ64XVf+v65PbACcnluE5IcF8HKyhqu9pc5q5TNJRNQg2draIjg4GMuXL0fr1q1VrwCfPn0aPj4+FdoXFxdj/vz5mDBhQpX/snz8YOmrr76KgwcPIikpCTNnzkRZWRmWLFmChw8fQqFQqN02AABnZ2dkZGRUq/aM3GLV6/inbz3CzQcVt/jwbmKNIC9H1TpF0ka6++Igeh4ikQjSRpaQNrJE39Z///+guFSBpPv5uPLXc07lM08y5BSW4kZmPm5k5uOXC+mq9nYWpvBztYOpCIi9lQUB5YEqfHQAQrvo/rb1szAkEVG9snXrVkydOhVubm6qV4AnTJiA8+fPq7UrLS3FK6+8AqVSia+++qrKPpVKJZo2bYotW7ZALBajU6dOuHfvHtasWYMlS5ao2j09iyMIQqUzO2l/bfHx51/BKOVRxbeL/Fxs/17N2qsxmto23H3PyDhZmIkR0MweAc3+vt0lCAIy8+QVVhO/+SAfsuIy/JmcpdaHUgAW7rmEXq2a6HRGqToYkoioXmnRogVOnDhR4RVgLy8vVZvHb9YkJyfj6NGjz5xyd3V1hZmZGcTiv9/cad26NTIyMlBSUgInJyeIxeIKs0aZmZlwdnaGIJQv9le+RlH5K/lPL/RnIgLaSO0Q5OWIrl4O6OrpoNPF/YgMhUgkgrOdBZztLNDbt6nquLxMgZuZBfjvhXvYdPym2jkKQcDth4UMSUREuqDLV4979OiBH3/8EUqlEiYm5esIXb9+Ha6urqo3cTp16oTo6GiMGjUKSqWAGw/y8fP+g3AJ6ImuK3+tsNO72ESEADd7BHk7oJuXIzp5NoadEezuTlRbJKZitJHaobG1Gf594qbas0xikQieTlZ1XhNDEhHVK5peAfb19cWUKVNQVlaGsWPHql49VigUqtmfql49fvvtt7FhwwbMmjUL//rXv5CUlISVK1finXfeAVC+LcS41/+B+e9Mx+USJzyw8sCd2P8i/+4diAb1gmmeHOZiE7R3b4Qg7/LniTo2bwxr7ntGVIGrvSXCRwdg4Z5LUAgCxCIRVo5uW+ezSABDEhHVM7m5uRpfATYzM8Pt27exf/9+AOXLAjzpyVePU1NTVTNGAODu7o6oqCi8++67aNeuHdzc3BDy+ltwCB6HNyLO4M/bWcgrbgr7PtPw+65/Q1GQBUkTT7w0ax2GDeyJIG8HtHdvBAszbvFBVB2hXZqjV6smuP2wEJ5OVnoJSACXAKgxLgFAVH89/eqxvEyh2vfsj1uPcC4lG4Ul6lt8WJuL0dnz8cKNDghwawRzU27xQWRo6tUSALWxxUBERASmTJlS4XhRUREsLPj2CFFDtvNMKhbsuQilAIgAeDlZ425OEeRPbfFhZ2GqevMsyNsBbVztYMp9z4jqFYMPSdOmTcOlS5ewdetW1RYD/fr1U20xcP78eSxevFhti4Hhw4dXucUAANjZ2eHatWtqxxiQiBq2s7ezMH/3RTyeXhcA3HpYvl6Ro7W5auHGrl6O8HOx5RYfRPWcQd9uq60tBiIiIhAWFoacnJwa18bbbUT1g0Ip4Pi1TGz9IwXHrz3Q2ObTce0wpmMzrmZNVA/Um9ttZWVlUCgUFWZ4LC0tERMTo/Gc3NxciEQi1WZ3lcnPz4eHhwcUCgXat2+P5cuXo0OHDpW2l8vlkMv/foVXJpNVfyBEZHAe5svx09k0/PBHaoU1i54kFonQo6UTAxJRA6T1DXRNex/Vlie3GLh37x4UCgW2bduG06dPIz09vUL76m4x4Ofnh4iICOzfvx/bt2+HhYUFevTogaSkpErPCQ8Ph729verH3d1dJ2MkorojCALOpWQhbEccuocfxer/XcPdnCLYW5rhzRe8cPz93lg1JgDivwKRPl89JiL90/p2m4WFBdzc3DBlyhRMnjy51sPCzZs3MXXqVJw8eVK1xUCrVq1w/vx5JCYmqtqVlpZi3LhxSE1NxfHjx7W6BaZUKtGxY0f06tUL69ev19hG00ySu7s7b7cRGYECeRn2xt/F1tgUXM3IUx0PdG+E14KaY1igVO31/PTcIr2/ekxEtaNWb7fdu3cP27ZtQ0REBD766CP07dsXb7zxBkaOHKlaiE2XamOLgaeZmJigS5cuVc4kSSQSSCSSGo+DiOpe0v08bPsjBbvP30W+vAwAIDE1wYj2UrzWzQPtmjXSeJ6rvSXDERE934Pb8fHx+Oabb7B9+3YolUq8+uqreOONNxAYGKjLGtVkZ2fDy8sLq1evxltvvVVhi4EmTZpo3acgCOjatSsCAgLwzTffVOscPrhNZJhKFUpEXb6PrX/cxh+3/t4o08vJGq8GNcfYTs3QyIp7ohE1VNp8fz/322337t3Dli1b8Mknn8DU1BTFxcUIDg7G5s2b4e/v/zxdA9C8xYBEIkFMTAxEIhHGjBmj2mLA2dlZdV5VWwwsXboU3bp1g4+PD2QyGdavX4+tW7fi999/R9euXatVF0MSkWFJzy3C9j/TsOPPVGT+tU+aiQjo19oZE4M90KOFE1/ZJ6Laf7uttLQU+/btwzfffIPo6Gh07twZGzduxPjx45GVlYV58+Zh3Lhxas8M1VRtbDGQk5ODt956CxkZGbC3t0eHDh1w8uTJagckIjIMgiDg1M1H2Bqbgugr96H4a0dMJxsJxnd1x/iuzSFtxNtmRFQzWs8k/etf/8L27dsBAK+99hqmTZuGtm3bqrVJTU2Fp6cnlEqlpi7qBc4kEelPblEpdp+7g22nU3DrQYHqeJCXAyYGe2BAGxduCUJEGtXqTFJiYiI2bNiAMWPGVPqgtlQqxbFjx7TtmoioSpfu5mJrbAr2JdxFcWn5f4TZSEwxuqMbXuvmgVbOtnqukIjqE4NecduQcSaJqG4Ulypw4EI6tv6Rgvi0HNVxPxdbvNbNAyM7uMFGYtDr4hKRAanVmaTw8HA4Oztj6tSpase/+eYbPHjwAPPmzdO2SyKiClIfFeKH0yn46WwasgtLAQBmYhFebuuKicEe6OzRmKtgE1Gt0jok/fvf/8aPP/5Y4bi/vz9eeeUVhiQiqrEn91E7cf0BHs9zuzWyxISg5gjp7I4mtlyvjIjqhtYhKSMjA66urhWON2nSRONWIUREz1LZPmovtmqCid080MevKcR8fZ+I6pjWIcnd3R2///672orXAPD7779DKpXqrDAiqt/K91HLxtY/UnDwYjpKFeXTRo2szBDS2R0TujaHp5O1nqskooZM65A0bdo0hIWFobS0FC+99BIA4Ndff8XcuXPx3nvv6bxAIqpfqtpHbWI3Dwxt56q2jxoRkb5oHZLmzp2LrKwszJgxAyUlJQDKN72dN28eFixYoPMCiah+qOk+akRE+lLjJQDy8/Nx5coVWFpawsfHp8Ft/solAIie7Vn7qI3r5A57KzM9VkhEDU2tb0sCADY2NujSpUtNTyeieoz7qBFRfVCjkHTmzBns2rULqampqltuj+3Zs0cnhRGRcREEAb/feIRtf1TcR21CV3e8wn3UiMjIaB2SduzYgUmTJmHAgAGIjo7GgAEDkJSUhIyMDIwaNao2aiQiA5ZbVIqfz93BD3+k4NZD7qNGRPWH1iFp5cqV+OyzzzBz5kzY2triiy++gJeXF6ZPn65x/SQiqp+4jxoR1Xdah6SbN29iyJAhAACJRIKCggKIRCK8++67eOmll7B06VKdF0lEhoH7qBFRQ6L1v80cHByQl1e+tombmxsuXbqEgIAA5OTkoLCwUOcFEpH+VbaP2uAAV7zWjfuoEVH9pHVIeuGFFxAdHY2AgACEhIRg1qxZOHr0KKKjo9G3b9/aqJGI9ID7qBFRQ6d1SNq4cSOKi4sBAAsWLICZmRliYmIwevRoLF68WOcFElHd4j5qRETltFpMsqysDD/88AMGDhwIFxeX2qzL4HExSapPuI8aETUUtbaYpKmpKd5++21cuXLluQokIsPAfdSIiCqn9e22oKAgxMXFwcPDozbqIaI6UNU+ahO7eSKgmb2eKyQi0j+tQ9KMGTPw3nvv4c6dO+jUqROsrdWn4Nu1a6ez4ohId6raR+21bh4Y27EZ91EjInqC1hvcmphUXDlXJBJBEASIRCIoFAqdFWfI+EwSGYvK9lHr38YZE7t5onsLR+6jRkQNRq1ucJucnFzjwoiobnAfNSKi56d1SOKzSESGi/uoERHpjtYh6fvvv6/yzydNmlTjYoioZriPGhGR7mn9TFLjxo3Vfi8tLUVhYSHMzc1hZWWFrKysSs6sX/hMUv2Ul5eHxYsXIzIyEpmZmejQoQO++OILdOnSBUD5baylS5diy5YtyM7ORlBQEL788kv4+/tX2e/u3buxePFi3Lx5Ey1atMCKFSswatQo1Z97enoiJSWlwnkzZszAl19+qbHPqvZRmxjsgRHtuY8aEdHTavWZpOzs7ArHkpKS8Pbbb2POnDnadkdkUKZNm4ZLly5h69atkEql2LZtG/r164fExES4ublh9erVWLduHSIiItCqVSt8/PHH6N+/P65duwZbW82zNbGxsQgNDcXy5csxatQoREZGIiQkBDExMQgKCgIAnDlzRu2lh0uXLqF///4YN25chf6q2kdtYjcPdOI+akREOqH1TFJlzp49i9deew1Xr17VRXcGjzNJ9U9RURFsbW2xb98+DBkyRHW8ffv2GDp0KJYvXw6pVIqwsDDMmzcPACCXy+Hs7IxVq1Zh+vTpGvsNDQ2FTCbDoUOHVMcGDRqExo0bY/v27RrPCQsLwy+//IKkpKTyt0a5jxoRkU7U6kxSZcRiMe7du6er7ojqXFlZGRQKBSwsLNSOW1paIiYmBsnJycjIyMCAAQNUfyaRSPDiiy/i1KlTlYak2NhYvPvuu2rHBg4ciM8//1xj+5KSEmzbtg2zZ8/Go4IS7qNGRKQnWoek/fv3q/0uCALS09OxceNG9OjRQ2eFEdU1W1tbBAcHY/ny5WjdujWcnZ2xfft2nD59Gj4+PsjIyAAAODs7q53n7Oys8XmixzIyMjSe87i/p0VGRiInJwc3G3VGcPivFfZRezWoOTwcuY8aEVFt0zokjRw5Uu13kUiEJk2a4KWXXsLatWt1VReRXmzduhVTp06Fm5sbxGIxOnbsiAkTJuD8+fOqNk8/7/N4IdWqVOecx/uozfpwLcw9O+LX1PLnjdr/tY/aEO6jRkRUp7QOSUqlsjbqIDIILVq0wIkTJ1BQUACZTAZXV1eEhobCy8sLLi4uAMpnhlxdXVXnZGZmVpgpepKLi0uFWaMnz3lyH7WczHt4dP0cpGM/QGhnd7zWzYP7qBER6QlXlSPSwNraGq6ursjOzsbhw4cxYsQIVVCKjo5WtSspKcGJEyfQvXv3SvsKDg5WOwcADh8+DM82HfDKllj0/+wkvotNQb68DKY3T8C+sSMu/GcBVo1tx4BERKRHWoeksWPH4pNPPqlwfM2aNRpfVyYyJocPH8b//vc/JCcnIzo6Gn369IGvry+mTJkCkUiEsLAwrFy5EpGRkbh06RJef/11WFlZYcKECao+Jk2ahAULFqh+nzVrFqKiorBq1SqcPBOPodPeR1T0EVxv0gt/3MqCiQgY6O+M76d0gXDtGP7x5lQ42nHLECIifdP6dtuJEyfw4YcfVjg+aNAgfPrppzopikhfcnNzsWDBAty5cwcODg4YM2YMVqxYATMzMwDA3LlzUVRUhBkzZqgWk4yKilJbIyk1NVVtI+jg4GB8+NkWfBK+DDkLF8G0kQuchs9DM992GN/l733UoqKikJqaiqlTp9b5uImIqCKt10mytLREfHw8fH191Y5fvXoVHTp0QFFRUSVn1oy+VkB+Fq6TRJVJzy1C8sMCOFlL8NuNhxX2Uevm7YDXunEfNSIifajVdZLatm2LnTt3YsmSJWrHd+zYgTZt2mjb3TPpawVkoprYeSYVC/ZchPKp//SwkZhiTEc3vMp91IiIjIbWM0n79+/HmDFjMGHCBLz00ksAgF9//RXbt2/Hrl27KiwR8DwMaQXkp3EmiZ6WnluEHp8crRCQ5g7yxeRgT1hzHzUiIr3T5vtb67n+4cOHY+/evbhx4wZmzJiB9957D3fu3MGRI0d0GpCA518BuTKxsbFq5wDlKyBXdY5cLodMJlP7IXpS8sOCCgEJADq4N2ZAIiIyQjX6N/eQIUPUZnZqi6GsgAwA4eHhWLp06XOMhuq7krKKa4iJRSJ4OlnpoRoiInpeWs8knTlzBqdPn65w/PTp0zh79qxOinrS1q1bIQgC3NzcIJFIsH79ekyYMAFi8d8rD9fWCshPWrBgAXJzc1U/aWlpNRgN1VfyMgXCD6pv7iwWibBydFu42vN1fiIiY6R1SJo5c6bGgHD37l3MnDlTJ0U96fEKyPn5+UhLS8Off/6J0tLSCisgP+l5V0DWRCKRwM7OTu2H6LF10ddx7X4enGzMcfCdntj+ZjfEzO+D0C7N9V0aERHVkNYhKTExER07dqxwvEOHDkhMTNRJUZrU9grIUVFRVZ5DVJlzKVnYcvIWACB8dDu0kdojuIUjZ5CIiIyc1s8kSSQS3L9/H97e3mrH09PTYWqq+4dTDx8+DEEQ4Ovrixs3bmDOnDkaV0D28fGBj48PVq5cqXEFZDc3N4SHhwMoXwG5V69eWLVqFUaMGIF9+/bhyJEjiImJ0Xn9VL8VlpRh9k8JEARgTMdm6N+m8tlIIiIyLlqnmv79+2PBggXYt28f7O3L95XKycnBwoUL0b9/f50XWBsrIHfv3h07duzAokWLsHjxYrRo0QI7d+7kGkmktfCDV5HyqBBSewt8OFz364QREZH+aL1O0t27d9GrVy88evQIHTp0AADEx8fD2dkZ0dHRcHd3r5VCDQ3XSaLfkh5g4td/AgB+mBaEHi2d9FwRERE9S62uuO3m5oYLFy7ghx9+QEJCAiwtLTFlyhSMHz9eNbtDVN/lFpVizq4LAIDJwR4MSERE9VCNHiKytrbGW2+9petaiIzG0v9eRoasGF5O1pj/cmt9l0NERLWgxk9aJyYmIjU1FSUlJWrHhw8f/txFERmyw5czsOf8XZiIgE/HBcLSXPzsk4iIyOhoHZJu3bqFUaNG4eLFixCJRHj8SNPjhRgVCoVuKyQyIA/z5Vi45yIAYPqLLdDJo7GeKyIiotqi9TpJs2bNgpeXF+7fvw8rKytcvnwZJ0+eROfOnXH8+PFaKJHIMAiCgA8iL+JRQQn8XGwR1s9H3yUREVEt0nomKTY2FkePHkWTJk1gYmICExMT9OzZE+Hh4XjnnXcQFxdXG3US6d3e+Ls4fPk+zMQirAtpD4kpb7MREdVnWs8kKRQK2NjYAACcnJxw7949AICHhweuXbum2+qIDER6bhGW7LsMAJjV1wdtpFz2gYiovtN6Jqlt27a4cOECvL29ERQUhNWrV8Pc3BxbtmypsAo3UX0gCALm/nwBecVlaO/eCP94sYW+SyIiojqgdUhatGgRCgoKAAAff/wxhg4dihdeeAGOjo7YuXOnzgsk0rdtp1PxW9JDWJiZYG1IIEzFWk/AEhGREdI6JA0cOFD1197e3khMTERWVhYaN26sesONqL64/bAAKw9cAQDMG+SHFk1s9FwRERHVFZ3sSOvg4KCLbogMikIp4P1dCSgqVSDY2xGTgz31XRIREdUh3jcgqsR/fruFsynZsJGYYs24djAx4UwpEVFDwpBEpMG1jDysjboOAFgytA2aNbbSc0VERFTXGJKInlJSpsTsn+JRolCir19TjOvcTN8lERGRHmgdkk6ePImysrIKx8vKynDy5EmdFEWkTxuPJuHyPRkaWZkhfEwAX0ggImqgtA5Jffr0QVZWVoXjubm56NOnj06KItKXhLQcfHn8JgDg45Ft0dTWQs8VERGRvmgdkgRB0Phf1o8ePYK1tbVOiiLSh+JSBWb/FA+FUsCwQCmGtpPquyQiItKjai8BMHr0aACASCTC66+/DolEovozhUKBCxcuoHv37rqvkKiOrDl8DTcfFKCprQTLR/jruxwiItKzaocke3t7AOUzSba2trC0tFT9mbm5Obp164Y333xT9xUS1YE/bj3CN78nAwBWjWmHRlbmeq6IiIj0rdoh6dtvvwUAeHp64v333+etNao38uVleH9XAgQBGN/VHX38muq7JCIiMgBaP5M0d+5ctWeSUlJS8PnnnyMqKkqnhRHVlRUHEnEnuwjNGlvigyFt9F0OEREZCK1D0ogRI/D9998DAHJyctC1a1esXbsWI0aMwKZNm3ReIFFtOnY1E9v/TINIBHw6LhA2Ep3s1ENERPWA1iHp/PnzeOGFFwAAP//8M1xcXJCSkoLvv/8e69ev13mBRLUlp7AE83ZfAABM7eGFbt6Oeq6IiIgMidYhqbCwELa2tgCAqKgojB49GiYmJujWrRtSUlJ0XiBRbVm87zIy8+Ro2dQGcwb66rscIiIyMFqHpJYtW2Lv3r1IS0vD4cOHMWDAAABAZmYm7OzsdF4gUW345cI9/DfhHsQmIqwdFwgLM7G+SyIiIgOjdUhasmQJ3n//fXh6eqJr164IDg4GUD6r1KFDB50XSKRrmXnFWLT3EgBgZu8WCHRvpN+CiIjIIGn9lOrYsWPRs2dPpKenIzAwUHW8b9++GDVqlE6LI9I1QRCwYPdF5BSWwl9qh3++5KPvkoiIyEBpPZMEAC4uLrC1tUV0dDSKiooAAF26dIGfn59OiyPStV1n7+DXq5kwF5tgXUh7mJvW6P8CRETUAGj9DfHo0SP07dsXrVq1wuDBg5Geng4AmDZtGt577z2dF0ikK2lZhVj2SyIA4L0BreDrYqvnioiIyJBpHZLeffddmJmZITU1FVZWVqrjoaGh+N///qfT4oh0RakUMPfnC8iXl6GzR2NMe8Fb3yUREZGB0/qZpKioKBw+fBjNmjVTO+7j48MlAMhgfRd7G7G3HsHSTIxPxwVCbCJ69klERNSgaT2TVFBQoDaD9NjDhw8hkUh0UhSRLt18kI9PDl0FACwc0hqeTtx3kIiInk3rkNSrVy/VtiQAIBKJoFQqsWbNGvTp00enxRE9rzKFErN/SoC8TIkXfJzwWlBzfZdERERGQuvbbWvWrEHv3r1x9uxZlJSUYO7cubh8+TKysrLw+++/10aNRDW2+cRNJKTlwNbCFKvHtlPbnJmIiKgqWs8ktWnTBhcuXEDXrl3Rv39/FBQUYPTo0YiLi0OLFi1qo0aiGrl8Lxdf/JoEAFg63B+u9pZ6roiIiIyJ1iEpNTUVzs7OWLp0KX755RccPHgQH3/8MVxdXZGamqrT4srKyrBo0SJ4eXnB0tIS3t7eWLZsGZRKpaqNSCTS+LNmzZpK+42IiNB4TnFxsU7rJ/2Rlynw3k8JKFUIGOjvjFEd3PRdEhERGRmtb7d5eXkhPT0dTZs2VTv+6NEjeHl5QaFQ6Ky4VatWYfPmzfjuu+/g7++Ps2fPYsqUKbC3t8esWbMAQLVO02OHDh3CG2+8gTFjxlTZt52dHa5du6Z2zMLCQme1k359fiQJVzPy4GhtjpWjAnibjYiItKZ1SBIEQeMXTn5+vs5DRmxsLEaMGIEhQ4YAADw9PbF9+3acPXtW1cbFxUXtnH379qFPnz7w9q56HRyRSFThXKofzqVk4d8nbgIAVo4OgKMN37okIiLtVTskzZ49G0B5uFi8eLHaMgAKhQKnT59G+/btdVpcz549sXnzZly/fh2tWrVCQkICYmJi8Pnnn2tsf//+fRw4cADffffdM/vOz8+Hh4cHFAoF2rdvj+XLl1e5Qa9cLodcLlf9LpPJtB4P1b7CkjK891MClAIwuoMbBvozCBMRUc1UOyTFxcUBKJ9JunjxIszNzVV/Zm5ujsDAQLz//vs6LW7evHnIzc2Fn58fxGIxFAoFVqxYgfHjx2ts/91338HW1hajR4+usl8/Pz9EREQgICAAMpkMX3zxBXr06IGEhAT4+Gje8DQ8PBxLly597jFR7Vp16CpuPyqEq70FPhzur+9yiIjIiIkEQRC0OWHKlCn44osvYGdnV1s1qezYsQNz5szBmjVr4O/vj/j4eISFhWHdunWYPHlyhfZ+fn7o378/NmzYoNXnKJVKdOzYEb169cL69es1ttE0k+Tu7o7c3Nw6+XtBz/b7jYd49T+nAQBb3+iKF3ya6LkiIiIyNDKZDPb29tX6/tb6maRvv/22xoVpa86cOZg/fz5eeeUVAEBAQABSUlIQHh5eIST99ttvuHbtGnbu3Kn155iYmKBLly5ISkqqtI1EIuGK4gZMVlyKObsSAAATu3kwIBER0XPTegmAulRYWAgTE/USxWKx2hIAj3399dfo1KkTAgMDtf4cQRAQHx8PV1fXGtdK+rV0fyLu5RbDw9EKCwb76bscIiKqB7SeSapLw4YNw4oVK9C8eXP4+/sjLi4O69atw9SpU9XayWQy7Nq1C2vXrtXYz6RJk+Dm5obw8HAAwNKlS9GtWzf4+PhAJpNh/fr1iI+Px5dfflnrYyLdi7qcgd3n70AkAtaOC4SVuUH/Y01EREbCoL9NNmzYgMWLF2PGjBnIzMyEVCrF9OnTsWTJErV2O3bsgCAIlT7QnZqaqjYjlZOTg7feegsZGRmwt7dHhw4dcPLkSXTt2rVWx0O69yhfjoWRFwEAb/XyRmdPBz1XRERE9YXWD25TOW0e/KLaIQgCZvxwHocuZcDX2Rb7/9UDElOxvssiIiIDps33t0E/k0RUlX3x93DoUgZMTURYGxLIgERERDrFkERGKSO3GEv2XQIAvNPXB23d7PVcERER1TcMSWR0BEHA3N0XICsuQ2Aze8zo3ULfJRERUT3EkERG58c/U3Hy+gNITE2wNqQ9TMX8x5iIiHSP3y5kVFIeFWDFgSsAgLmD/NCyqY2eKyIiovqKIYmMhkIp4P1dCSgsUSDIywFTunvquyQiIqrHGJLIaHwdcwtnbmfD2lyMT8cFwsREpO+SiIioHmNIIqNw/X4ePj18HQCweGgbuDtY6bkiIiKq7xiSyOCVKpSY/VM8ShRKvOTXFKFd3PVdEhERNQAMSWTwNh69gUt3ZbC3NMMnowMgEvE2GxER1T6GJDJoF+7kYOOxGwCA5SPboqmdhZ4rIiKihoIhiQxWcakCs39KgEIpYEg7VwwPlOq7JCIiakAYkshgrY26hhuZ+WhiK8HHI9rquxwiImpgGJLIIJ2+9Qj/iUkGAKwaE4DG1uZ6roiIiBoahiQyOPnyMrz/cwIEAQjt7I6X/Jz1XRIRETVADElkcFYcuIK0rCK4NbLEoqGt9V0OERE1UAxJZFCOXcvE9j9TAQBrxrWDrYWZnisiIqKGiiGJDEZOYQnm/XwBADClhye6t3DSc0VERNSQMSSRwfhw/2Vk5snh3cQa8wb56bscIiJq4BiSyCAcvJiOffH3YCIC1oW0h4WZWN8lERFRA8eQRHqXmVeMDyIvAgBm9G6J9u6N9FsQERERGJJIzwRBwMI9l5BdWIo2rnZ4p6+PvksiIiICwJBEevbzuTs4cuU+zMUmWBcaCHNT/iNJRESGgd9IpDd3c4qw7L+JAIB3+7eCn4udnisiIiL6G0MS6YVSKWDOrgTkycvQsXkjvNXLW98lERERqWFIIr34PvY2Tt18BEszMdaGtIfYRKTvkoiIiNQwJFGdu/UgH5/87yoAYMFgP3g5Weu5IiIioooYkqhOlSmUeG9XAopLlejZ0gmvBXnouyQiIiKNGJKoTv375C3EpebAVmKK1WPbwYS32YiIyEAxJFGdSbwnw+dHrgMAPhzuD2kjSz1XREREVDmGJKoT8jIFZv8Uj1KFgP5tnDGmo5u+SyIiIqoSQxLVifW/JuFqRh4crM0RPjoAIhFvsxERkWFjSKJadz41G5uO3wQArBzVFk42Ej1XRERE9GwMSVSrikoUeP+nBCgFYGR7KQa1ddV3SURERNXCkES1atX/ruLWwwK42Flg6fC2+i6HiIio2gw6JJWVlWHRokXw8vKCpaUlvL29sWzZMiiVSlWb119/HSKRSO2nW7duz+x79+7daNOmDSQSCdq0aYPIyMjaHEqD9PuNh4g4dRsAsGpsO9hbmem3ICIiIi2Y6ruAqqxatQqbN2/Gd999B39/f5w9exZTpkyBvb09Zs2apWo3aNAgfPvtt6rfzc3Nq+w3NjYWoaGhWL58OUaNGoXIyEiEhIQgJiYGQUFBtTaehkRWXIq5P18AALwa1Bwvtmqi54qIiIi0IxIEQdB3EZUZOnQonJ2d8fXXX6uOjRkzBlZWVti6dSuA8pmknJwc7N27t9r9hoaGQiaT4dChQ6pjgwYNQuPGjbF9+/Zq9SGTyWBvb4/c3FzY2XH3+qfN2ZWAXefuoLmDFQ7NegHWEoPO40RE1EBo8/1t0LfbevbsiV9//RXXr5cvQJiQkICYmBgMHjxYrd3x48fRtGlTtGrVCm+++SYyMzOr7Dc2NhYDBgxQOzZw4ECcOnWq0nPkcjlkMpnaD2l2JPE+dp27A5EI+HRcIAMSEREZJYP+9po3bx5yc3Ph5+cHsVgMhUKBFStWYPz48ao2L7/8MsaNGwcPDw8kJydj8eLFeOmll3Du3DlIJJpfNc/IyICzs7PaMWdnZ2RkZFRaS3h4OJYuXaqbgdVjWQUlmL/nIgDgzRe80dXLQc8VERER1YxBh6SdO3di27Zt+PHHH+Hv74/4+HiEhYVBKpVi8uTJAMpvnT3Wtm1bdO7cGR4eHjhw4ABGjx5dad9PL2YoCEKVCxwuWLAAs2fPVv0uk8ng7u5e06HVS4IgYNHei3iYL0crZxvM7t9K3yURERHVmEGHpDlz5mD+/Pl45ZVXAAABAQFISUlBeHi4KiQ9zdXVFR4eHkhKSqq0XxcXlwqzRpmZmRVml54kkUgqnZmicvsT7uHgxQyYmoiwdlx7WJiJ9V0SERFRjRn0M0mFhYUwMVEvUSwWqy0B8LRHjx4hLS0Nrq6VL1oYHByM6OhotWNRUVHo3r378xXcgN2XFWPJvssAgH++1BIBzez1XBEREdHzMeiZpGHDhmHFihVo3rw5/P39ERcXh3Xr1mHq1KkAgPz8fHz00UcYM2YMXF1dcfv2bSxcuBBOTk4YNWqUqp9JkybBzc0N4eHhAIBZs2ahV69eWLVqFUaMGIF9+/bhyJEjiImJ0cs4jZ0gCJj78wXkFpUiwM0eM/u01HdJREREz82gQ9KGDRuwePFizJgxA5mZmZBKpZg+fTqWLFkCoHxW6eLFi/j++++Rk5MDV1dX9OnTBzt37oStra2qn9TUVLUZqe7du2PHjh1YtGgRFi9ejBYtWmDnzp1cI6mGtv+ZhhPXH8Dc1ATrQgJhJjboCUoiIqJqMeh1kgwZ10kql/qoEIO+OInCEgUWDWmNaS9467skIiKiStWbdZLIsCmVAt7/OQGFJQp09XLA1B5e+i6JiIhIZxiSqMa++T0ZfyZnwcpcjE/HBsLEpPIlFIiIiIwNQxLVSNL9PKw+fA0AsGhIGzR3tNJzRURERLrFkERaK1UoMfunBJSUKdHbtwnGd+WimkREVP8wJJHWvjp2Exfv5sLe0gyrxrSrcqVyIiIiY8WQRFq5eCcXG46Wr2a+bIQ/nO0s9FwRERFR7WBIomorLlVg9k/xKFMKGBzgguGBUn2XREREVGsYkqja1kVfR1JmPpxsJPh4ZABvsxERUb3GkETV8mdyFv7vt1sAgE9GB8DB2lzPFREREdUuhiR6pgJ5Gd7flQBBAMZ1aoZ+bZz1XRIREVGtY0iiZ1p58ApSswrh1sgSi4e10Xc5REREdYIhiap04voD/HA6FQCwZmw72FmY6bkiIiKiusGQRJXKLSzF3J8TAACvd/dE95ZOeq6IiIio7jAkUaU++u9l3JfJ4eVkjXmD/PRdDhERUZ1iSCKN/ncpHZFxd2EiAtaGBMLSXKzvkoiIiOoUQxJV8CBPjoWRlwAA/3ixBTo2b6znioiIiOoeQxKpEQQBCyMvIqugBH4utpjVz0ffJREREekFQxKp2X3+LqIT78NMLMJnoe0hMeVtNiIiapgYkkjlXk4Rlu6/DAAI69cKrV3t9FwRERGR/jAkEQBAqRQw9+cLyJOXoUPzRpjey1vfJREREekVQxIBALadTkHMjYewMDPB2nGBMBXzHw0iImrY+E1ISH5YgJUHrwAA5g/yg3cTGz1XREREpH8MSQ2cQingvZ/iUVyqRPcWjpgU7KnvkoiIiAwCQ1IDt+XkLZxPzYGNxBRrxgXCxESk75KIiIgMAkNSA3Y1Q4bPoq8DAJYMawO3RpZ6roiIiMhwMCQ1UCVlSry7MwElCiX6tW6KcZ2a6bskIiIig8KQ1ECt/zUJV9JlaGxlhpWjAyAS8TYbERHRkxiSGqC41Gx8dfwGAGDFqAA0tbXQc0VERESGhyGpgSkqUeC9nxKgFIDhgVIMDnDVd0lEREQGiSGpgVl9+CpuPSxAU1sJlo3w13c5REREBoshqQE5dfMhvv39NgBg1dh2aGRlrt+CiIiIDBhDUgORV1yKObsuAADGd22OPr5N9VwRERGRYWNIaiCW/5KIuzlFcHewxAdDWuu7HCIiIoPHkNQA/HrlPn46ewciEfDp2EDYSEz1XRIREZHBY0iq57ILSjB/z0UAwBs9vBDk7ajnioiIiIyDQYeksrIyLFq0CF5eXrC0tIS3tzeWLVsGpVIJACgtLcW8efMQEBAAa2trSKVSTJo0Cffu3auy34iICIhEogo/xcXFdTGsOrVo3yU8yJOjZVMbvD/QV9/lEBERGQ2Dvu+yatUqbN68Gd999x38/f1x9uxZTJkyBfb29pg1axYKCwtx/vx5LF68GIGBgcjOzkZYWBiGDx+Os2fPVtm3nZ0drl27pnbMwqJ+Laq4P+EeDlxIh9hEhHUhgbAwE+u7JCIiIqNh0CEpNjYWI0aMwJAhQwAAnp6e2L59uyoA2dvbIzo6Wu2cDRs2oGvXrkhNTUXz5s0r7VskEsHFxaX2itezTFkxFu+9BACY2acl2jVrpN+CiIiIjIxB327r2bMnfv31V1y/Xr5TfUJCAmJiYjB48OBKz8nNzYVIJEKjRo2q7Ds/Px8eHh5o1qwZhg4diri4OF2WrleCIGDe7gvILSpFWzc7/OullvouiYiIyOgY9EzSvHnzkJubCz8/P4jFYigUCqxYsQLjx4/X2L64uBjz58/HhAkTYGdnV2m/fn5+iIiIQEBAAGQyGb744gv06NEDCQkJ8PHx0XiOXC6HXC5X/S6TyZ5vcLVo55k0HLv2AOamJlgX0h5mYoPOwkRERAbJoEPSzp07sW3bNvz444/w9/dHfHw8wsLCIJVKMXnyZLW2paWleOWVV6BUKvHVV19V2W+3bt3QrVs31e89evRAx44dsWHDBqxfv17jOeHh4Vi6dOnzD6qWpWUVYvkviQCA9we0QitnWz1XREREZJxEgiAI+i6iMu7u7pg/fz5mzpypOvbxxx9j27ZtuHr1qupYaWkpQkJCcOvWLRw9ehSOjtq/5v7mm2/izp07OHTokMY/1zST5O7ujtzc3CpnreqSUilg/P/9gdPJWeji2Rg73gqG2ESk77KIiIgMhkwmg729fbW+vw16JqmwsBAmJuq3isRisWoJAODvgJSUlIRjx47VKCAJgoD4+HgEBARU2kYikUAikWjdd1369tRtnE7OgpW5GJ+OC2RAIiIieg4GHZKGDRuGFStWoHnz5vD390dcXBzWrVuHqVOnAihfR2ns2LE4f/48fvnlFygUCmRkZAAAHBwcYG5evoHrpEmT4ObmhvDwcADA0qVL0a1bN/j4+EAmk2H9+vWIj4/Hl19+qZ+B6sCNzHys/l/57NrCwa3h4Wit54qIiIiMm0GHpA0bNmDx4sWYMWMGMjMzIZVKMX36dCxZsgQAcOfOHezfvx8A0L59e7Vzjx07ht69ewMAUlNT1WakcnJy8NZbbyEjIwP29vbo0KEDTp48ia5du9bJuHStTKHEez/FQ16mRK9WTfBqUOVLHxAREVH1GPQzSYZMm3uatW39r0lYF30ddhamiHr3RbjY169FMYmIiHRFm+9vvhtu5C7dzcX6X5MAAEtH+DMgERER6QhDkhGTlykw+6d4lCkFDPJ3wcj2bvouiYiIqN5gSDJi66Kv4/r9fDjZmGPFqLYQifg2GxERka4wJBmps7ezsOXkLQDAylEBcLQx7OUJiIiIjA1DkhEqkJfhvV0JEARgTMdmGOBffzfqJSIi0heGJCMUfugKUh4VwtXeAkuGtdF3OURERPUSQ5IOlZWVYdGiRfDy8oKlpSW8vb2xbNkytRXC9+zZg4EDB8LJyQkikQjx8fHP7Pfy5csYM2YMPD09IRKJ8NWGDQCANWMDYW9pVqF9eHg4RCIRwsLCdDU0IiKiBochSYdWrVqFzZs3Y+PGjbhy5QpWr16NNWvWYMNfoQYACgoK0KNHD3zyySfV7rewsBDe3t5YsuxjmNk4AAAmBXugp49ThbZnzpzBli1b0K5du+cfEBERUQNm0CtuG5vY2FiMGDECQ4YMAQB4enpi+/btOHv2rKrNxIkTAQC3b9+udr9dunRBly5dMHtnPJQmpnCwNsP8l/0qtMvPz8err76K//u//8PHH3/8fIMhIiJq4DiTpEM9e/bEr7/+iuvXrwMAEhISEBMTg8GDBz933/+7lIE9cXchAjCivRuszCvm25kzZ2LIkCHo16/fc38eERFRQ8eZJB2aN28ecnNz4efnB7FYDIVCgRUrVmD8+PHP1e/DfDk+iLwIALCxMIW7g1WFNjt27MD58+dx5syZ5/osIiIiKseQpEM7d+7Etm3b8OOPP8Lf3x/x8fEICwuDVCrF5MmTa9SnIAj4IPIiHhWUwM/FFlc0PKidlpaGWbNmISoqChYW3JaEiIhIFxiSdGjOnDmYP38+XnnlFQBAQEAAUlJSEB4eXuOQFBl3F4cv34eZWIS1IYEY8nnFNufOnUNmZiY6deqkOqZQKHDy5Els3LgRcrkcYrG4Rp9PRETUUDEk6VBhYSFMTNQf8xKLxWpLAGjjXk4RPtx/GQAwq68P/KX2Gtv17dsXFy9eVDs2ZcoU+Pn5Yd68eQxIRERENcCQpEPDhg3DihUr0Lx5c/j7+yMuLg7r1q3D1KlTVW2ysrKQmpqKe/fuAQCuXbsGAHBxcYGLS/nK2ZMmTYJUKsVdn1HIKy5DgKs1utnnIT4+HiUlJbh79y7i4+NhY2ODli1bwtbWFm3btlWrxdraGo6OjhWOExERUfWIBEEQ9F2EMZLJZLC3t0dubi7s7OwAAHl5eVi8eDEiIyORmZkJqVSK8ePHY8mSJTA3NwcAREREYMqUKRX6+/DDD/HRRx8BAHr37g2FtRPSAqZAYmqCf49sjj5dKoadF198EcePH9dYX+/evdG+fXt8/vnnOhkvERFRfaDp+7syDEk1pM3fZG39mfwIr/3nT5QolFgytA2m9vTSaf9EREQNlTbf37zdZmC2/5mKBXv+fr7I0pzPExEREekDF5M0IOm5RVi4R/0B7EWRl5CeW6SnioiIiBouhiQDkvywAE/f+1QIAm4/LNRLPURERA0ZQ5IB8XKyholI/ZhYJIKnU8UVtomIiKh2MSQZEFd7S4SPDoBYVJ6UxCIRVo5uC1d7Sz1XRkRE1PDwwW0DE9qlOXq1aoLbDwvh6WTFgERERKQnDEkGyNXekuGIiIhIz3i7jYiIiEgDhiQiIiIiDRiSiIiIiDRgSCIiIiLSgCGJiIiISAOGJCIiIiINGJKIiIiINGBIIiIiItKAIYmIiIhIA4YkIiIiIg0YkoiIiIg0YEgiIiIi0sCgQ1JZWRkWLVoELy8vWFpawtvbG8uWLYNSqVS1EQQBH330EaRSKSwtLdG7d29cvnz5mX3v3r0bbdq0gUQiQZs2bRAZGVmbQyEiIiIjY9AhadWqVdi8eTM2btyIK1euYPXq1VizZg02bNigarN69WqsW7cOGzduxJkzZ+Di4oL+/fsjLy+v0n5jY2MRGhqKiRMnIiEhARMnTkRISAhOnz5dF8MiIiIiIyASBEHQdxGVGTp0KJydnfH111+rjo0ZMwZWVlbYunUrBEGAVCpFWFgY5s2bBwCQy+VwdnbGqlWrMH36dI39hoaGQiaT4dChQ6pjgwYNQuPGjbF9+/Zq1SaTyWBvb4/c3FzY2dk9xyiJiIiormjz/W1aRzXVSM+ePbF582Zcv34drVq1QkJCAmJiYvD5558DAJKTk5GRkYEBAwaozpFIJHjxxRdx6tSpSkNSbGws3n33XbVjAwcOVPWriVwuh1wuV/2em5sLoPxvNhERERmHx9/b1ZkjMuiQNG/ePOTm5sLPzw9isRgKhQIrVqzA+PHjAQAZGRkAAGdnZ7XznJ2dkZKSUmm/GRkZGs953J8m4eHhWLp0aYXj7u7u1R4PERERGYa8vDzY29tX2cagQ9LOnTuxbds2/Pjjj/D390d8fDzCwsIglUoxefJkVTuRSKR2niAIFY49TdtzFixYgNmzZ6t+VyqVyMrKgqOj4zM/S1symQzu7u5IS0url7fyOD7jV9/HWN/HB9T/MXJ8xq+2xigIAvLy8iCVSp/Z1qBD0pw5czB//ny88sorAICAgACkpKQgPDwckydPhouLC4DymSFXV1fVeZmZmRVmip7k4uJSYdboWedIJBJIJBK1Y40aNdJ2SFqxs7Ort//wAxxffVDfx1jfxwfU/zFyfMavNsb4rBmkxwz67bbCwkKYmKiXKBaLVUsAeHl5wcXFBdHR0ao/LykpwYkTJ9C9e/dK+w0ODlY7BwCioqKqPIeIiIgaFoOeSRo2bBhWrFiB5s2bw9/fH3FxcVi3bh2mTp0KoPyWWVhYGFauXAkfHx/4+Phg5cqVsLKywoQJE1T9TJo0CW5ubggPDwcAzJo1C7169cKqVaswYsQI7Nu3D0eOHEFMTIxexklERESGx6BD0oYNG7B48WLMmDEDmZmZkEqlmD59OpYsWaJqM3fuXBQVFWHGjBnIzs5GUFAQoqKiYGtrq2qTmpqqNiPVvXt37NixA4sWLcLixYvRokUL7Ny5E0FBQXU6vspIJBJ8+OGHFW7v1Rccn/Gr72Os7+MD6v8YOT7jZwhjNOh1koiIiIj0xaCfSSIiIiLSF4YkIiIiIg0YkoiIiIg0YEgiIiIi0oAhSU+++uoreHl5wcLCAp06dcJvv/1WZfsTJ06gU6dOsLCwgLe3NzZv3lxHldaMNuM7fvw4RCJRhZ+rV6/WYcXVd/LkSQwbNgxSqRQikQh79+595jnGdP20HZ+xXb/w8HB06dIFtra2aNq0KUaOHIlr16498zxjuYY1GZ+xXcNNmzahXbt2qkUGg4OD1TYs18RYrh+g/fiM7fo9LTw8XLWkT1X0cQ0ZkvRg586dCAsLwwcffIC4uDi88MILePnll5GamqqxfXJyMgYPHowXXngBcXFxWLhwId555x3s3r27jiuvHm3H99i1a9eQnp6u+vHx8amjirVTUFCAwMBAbNy4sVrtje36aTu+x4zl+p04cQIzZ87EH3/8gejoaJSVlWHAgAEoKCio9BxjuoY1Gd9jxnINmzVrhk8++QRnz57F2bNn8dJLL2HEiBG4fPmyxvbGdP0A7cf3mLFcvyedOXMGW7ZsQbt27apsp7drKFCd69q1q/CPf/xD7Zifn58wf/58je3nzp0r+Pn5qR2bPn260K1bt1qr8XloO75jx44JAITs7Ow6qE63AAiRkZFVtjG26/ek6ozPmK+fIAhCZmamAEA4ceJEpW2M+RpWZ3zGfg0FQRAaN24s/Oc//9H4Z8Z8/R6ranzGev3y8vIEHx8fITo6WnjxxReFWbNmVdpWX9eQM0l1rKSkBOfOncOAAQPUjg8YMACnTp3SeE5sbGyF9gMHDsTZs2dRWlpaa7XWRE3G91iHDh3g6uqKvn374tixY7VZZp0ypuv3PIz1+uXm5gIAHBwcKm1jzNewOuN7zBivoUKhwI4dO1BQUIDg4GCNbYz5+lVnfI8Z2/WbOXMmhgwZgn79+j2zrb6uIUNSHXv48CEUCkWFzXSdnZ0rbLr7WEZGhsb2ZWVlePjwYa3VWhM1GZ+rqyu2bNmC3bt3Y8+ePfD19UXfvn1x8uTJuii51hnT9asJY75+giBg9uzZ6NmzJ9q2bVtpO2O9htUdnzFew4sXL8LGxgYSiQT/+Mc/EBkZiTZt2mhsa4zXT5vxGeP127FjB86fP6/aLuxZ9HUNDXpbkvpMJBKp/S4IQoVjz2qv6bih0GZ8vr6+8PX1Vf0eHByMtLQ0fPrpp+jVq1et1llXjO36acOYr98///lPXLhwoVr7NhrjNazu+IzxGvr6+iI+Ph45OTnYvXs3Jk+ejBMnTlQaJIzt+mkzPmO7fmlpaZg1axaioqJgYWFR7fP0cQ05k1THnJycIBaLK8yqZGZmVkjJj7m4uGhsb2pqCkdHx1qrtSZqMj5NunXrhqSkJF2XpxfGdP10xRiu37/+9S/s378fx44dQ7Nmzapsa4zXUJvxaWLo19Dc3BwtW7ZE586dER4ejsDAQHzxxRca2xrj9dNmfJoY8vU7d+4cMjMz0alTJ5iamsLU1BQnTpzA+vXrYWpqCoVCUeEcfV1DhqQ6Zm5ujk6dOiE6OlrteHR0NLp3767xnODg4Arto6Ki0LlzZ5iZmdVarTVRk/FpEhcXB1dXV12XpxfGdP10xZCvnyAI+Oc//4k9e/bg6NGj8PLyeuY5xnQNazI+TQz5GmoiCALkcrnGPzOm61eZqsaniSFfv759++LixYuIj49X/XTu3Bmvvvoq4uPjIRaLK5yjt2tYq4+Fk0Y7duwQzMzMhK+//lpITEwUwsLCBGtra+H27duCIAjC/PnzhYkTJ6ra37p1S7CyshLeffddITExUfj6668FMzMz4eeff9bXEKqk7fg+++wzITIyUrh+/bpw6dIlYf78+QIAYffu3foaQpXy8vKEuLg4IS4uTgAgrFu3ToiLixNSUlIEQTD+66ft+Izt+r399tuCvb29cPz4cSE9PV31U1hYqGpjzNewJuMztmu4YMEC4eTJk0JycrJw4cIFYeHChYKJiYkQFRUlCIJxXz9B0H58xnb9NHn67TZDuYYMSXry5ZdfCh4eHoK5ubnQsWNHtddzJ0+eLLz44otq7Y8fPy506NBBMDc3Fzw9PYVNmzbVccXa0WZ8q1atElq0aCFYWFgIjRs3Fnr27CkcOHBAD1VXz+PXbZ/+mTx5siAIxn/9tB2fsV0/TWMDIHz77beqNsZ8DWsyPmO7hlOnTlX9+6VJkyZC3759VQFCEIz7+gmC9uMztuunydMhyVCuoUgQ/nryiYiIiIhU+EwSERERkQYMSUREREQaMCQRERERacCQRERERKQBQxIRERGRBgxJRERERBowJBERERFpwJBERKQjx48fh0gkQk5Ojr5LISIdYEgiIiIi0oAhiYiIiEgDhiQiqjcEQcDq1avh7e0NS0tLBAYG4ueffwbw962wAwcOIDAwEBYWFggKCsLFixfV+ti9ezf8/f0hkUjg6emJtWvXqv25XC7H3Llz4e7uDolEAh8fH3z99ddqbc6dO4fOnTvDysoK3bt3x7Vr12p34ERUKxiSiKjeWLRoEb799lts2rQJly9fxrvvvovXXnsNJ06cULWZM2cOPv30U5w5cwZNmzbF8OHDUVpaCqA83ISEhOCVV17BxYsX8dFHH2Hx4sWIiIhQnT9p0iTs2LED69evx5UrV7B582bY2Nio1fHBBx9g7dq1OHv2LExNTTF16tQ6GT8R6RY3uCWieqGgoABOTk44evQogoODVcenTZuGwsJCvPXWW+jTpw927NiB0NBQAEBWVhaaNWuGiIgIhISE4NVXX8WDBw8QFRWlOn/u3Lk4cOAALl++jOvXr8PX1xfR0dHo169fhRqOHz+OPn364MiRI+jbty8A4ODBgxgyZAiKiopgYWFRy38XiEiXOJNERPVCYmIiiouL0b9/f9jY2Kh+vv/+e9y8eVPV7skA5eDgAF9fX1y5cgUAcOXKFfTo0UOt3x49eiApKQkKhQLx8fEQi8V48cUXq6ylXbt2qr92dXUFAGRmZj73GImobpnquwAiIl1QKpUAgAMHDsDNzU3tzyQSiVpQeppIJAJQ/kzT479+7MnJdktLy2rVYmZmVqHvx/URkfHgTBIR1Qtt2rSBRCJBamoqWrZsqfbj7u6uavfHH3+o/jo7OxvXr1+Hn5+fqo+YmBi1fk+dOoVWrVpBLBYjICAASqVS7RknIqq/OJNERPWCra0t3n//fbz77rtQKpXo2bMnZDIZTp06BRsbG3h4eAAAli1bBkdHRzg7O+ODDz6Ak5MTRo4cCQB477330KVLFyxfvhyhoaGIjY3Fxo0b8dVXXwEAPD09MXnyZEydOhXr169HYGAgUlJSkJmZiZCQEH0NnYhqCUMSEdUby5cvR9OmTREeHo5bt26hUaNG6NixIxYuXKi63fXJJ59g1qxZSEpKQmBgIPbv3w9zc3MAQMeOHfHTTz9hyZIlWL58OVxdXbFs2TK8/vrrqs/YtGkTFi5ciBkzZuDRo0do3rw5Fi5cqI/hElEt49ttRNQgPH7zLDs7G40aNdJ3OURkBPhMEhEREZEGDElEREREGvB2GxEREZEGnEkiIiIi0oAhiYiIiEgDhiQiIiIiDRiSiIiIiDRgSCIiIiLSgCGJiIiISAOGJCIiIiINGJKIiIiINGBIIiIiItLg/wE7w+URUhR3kQAAAABJRU5ErkJggg==", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAG2CAYAAACZEEfAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABX5UlEQVR4nO3dd1QU198G8GdZqohUaUpTFESkKIioMVGwxSS2xPIziS3GJNiNUYyoiQU0amyxvtbEnsQeC0HFhoggdrGggkpRyiIgxd15/0A32SAKCiwMz+ecPUfuzg7fEWWfvXOLRBAEAUREREQipaHuAoiIiIgqEsMOERERiRrDDhEREYkaww4RERGJGsMOERERiRrDDhEREYkaww4RERGJGsMOERERiRrDDhEREYkaww4RERGJmlrDzvHjx/Hhhx/C2toaEokEu3btUnleEARMnToVVlZW0NPTg7+/P27evKlyTHp6OgYMGIA6derAyMgIQ4cORXZ2diVeBREREVVlag07OTk5cHd3xy+//PLS5+fOnYvFixdjxYoViIyMhL6+Pjp37oy8vDzlMQMGDMCVK1cQGhqKffv24fjx4/jyyy8r6xKIiIioipNUlY1AJRIJdu7ciR49egAo6tWxtrbG+PHj8e233wIAZDIZLCwssH79evTr1w/Xrl2Di4sLoqKi4OXlBQA4ePAg3n//fdy/fx/W1tbquhwiIiKqIjTVXUBJ7ty5g+TkZPj7+yvbDA0N4ePjg4iICPTr1w8REREwMjJSBh0A8Pf3h4aGBiIjI9GzZ8+Xnjs/Px/5+fnKrxUKBdLT02FqagqJRFJxF0VERETlRhAEPHnyBNbW1tDQKPlmVZUNO8nJyQAACwsLlXYLCwvlc8nJyTA3N1d5XlNTEyYmJspjXiY4OBg//PBDOVdMRERE6pCYmIj69euX+HyVDTsVKTAwEOPGjVN+LZPJYGtri8TERNSpU0eNlREREVFpZWVlwcbGBgYGBq88rsqGHUtLSwBASkoKrKyslO0pKSnw8PBQHpOamqryumfPniE9PV35+pfR0dGBjo5OsfY6deow7BAREVUzrxuCUmXX2XFwcIClpSXCwsKUbVlZWYiMjISvry8AwNfXF5mZmYiOjlYec+TIESgUCvj4+FR6zURERFT1qLVnJzs7G7du3VJ+fefOHcTGxsLExAS2trYYM2YMZs6ciUaNGsHBwQFBQUGwtrZWzthq0qQJunTpgmHDhmHFihUoLCzEiBEj0K9fP87EIiIiIgBqDjvnzp1D+/btlV+/GEczcOBArF+/Ht999x1ycnLw5ZdfIjMzE23btsXBgwehq6urfM2mTZswYsQI+Pn5QUNDA71798bixYsr/VqIiIioaqoy6+yoU1ZWFgwNDSGTyThmh4iIqJoo7ft3lR2zQ0RERFQeGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNSqdNiRy+UICgqCg4MD9PT00LBhQ8yYMQOCICiPEQQBU6dOhZWVFfT09ODv74+bN2+qsWoiIiKqSqp02JkzZw6WL1+OpUuX4tq1a5gzZw7mzp2LJUuWKI+ZO3cuFi9ejBUrViAyMhL6+vro3Lkz8vLy1Fg5ERERVRUS4d/dJFXMBx98AAsLC6xZs0bZ1rt3b+jp6eG3336DIAiwtrbG+PHj8e233wIAZDIZLCwssH79evTr169U3ycrKwuGhoaQyWSoU6dOhVwLERERla/Svn9X6Z6d1q1bIywsDDdu3AAAXLhwASdPnkTXrl0BAHfu3EFycjL8/f2VrzE0NISPjw8iIiJKPG9+fj6ysrJUHkRERCROmuou4FUmTZqErKwsODs7QyqVQi6XY9asWRgwYAAAIDk5GQBgYWGh8joLCwvlcy8THByMH374oeIKJyIioiqjSvfsbN++HZs2bcLmzZsRExODDRs2YN68ediwYcNbnTcwMBAymUz5SExMLKeKiYiIqKqp0j07EyZMwKRJk5Rjb5o1a4Z79+4hODgYAwcOhKWlJQAgJSUFVlZWytelpKTAw8OjxPPq6OhAR0enQmsnIiKiqqFK9+zk5uZCQ0O1RKlUCoVCAQBwcHCApaUlwsLClM9nZWUhMjISvr6+lVorERERVU1Vumfnww8/xKxZs2Bra4umTZvi/PnzWLBgAYYMGQIAkEgkGDNmDGbOnIlGjRrBwcEBQUFBsLa2Ro8ePdRbPBEREVUJVTrsLFmyBEFBQfjmm2+QmpoKa2trDB8+HFOnTlUe89133yEnJwdffvklMjMz0bZtWxw8eBC6urpqrJyIiIiqiiq9zk5l4To7RERE1Y8o1tkhIiIielsMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERPRSDx48wKeffgpTU1Po6emhWbNmOHfunMox165dw0cffQRDQ0Po6+vD29sbCQkJJZ5z9erVeOedd2BsbAxjY2P4+/vj7NmzKsekpKRg0KBBsLa2Rq1atdClSxfcvHnzja+DYYeIiIiKycjIQJs2baClpYUDBw7g6tWrmD9/PoyNjZXH3L59G23btoWzszOOHTuGixcvIigoCLq6uiWe99ixY+jfvz+OHj2KiIgI2NjYoFOnTnjw4AEAQBAE9OjRA/Hx8di9ezfOnz8POzs7+Pv7Iycn542uRSIIgvBGrxSRrKwsGBoaQiaToU6dOuouh4iISO0mTZqEU6dO4cSJEyUe069fP2hpaeHXX3994+8jl8thbGyMpUuX4vPPP8eNGzfg5OSEy5cvo2nTpgAAhUIBS0tLzJ49G1988YXytaV9/2bPDhERERWzZ88eeHl54ZNPPoG5uTk8PT2xevVq5fMKhQL79+9H48aN0blzZ5ibm8PHxwe7du0q0/fJzc1FYWEhTExMAAD5+fkAoNI7pKGhAR0dHZw8efKNroVhh4iIiIqJj4/H8uXL0ahRIxw6dAhff/01Ro0ahQ0bNgAAUlNTkZ2djZCQEHTp0gWHDx9Gz5490atXL4SHh5f6+0ycOBHW1tbw9/cHADg7O8PW1haBgYHIyMhAQUEB5syZg/v37yMpKemNroW3scDbWERERP+lra0NLy8vnD59Wtk2atQoREVFISIiAg8fPkS9evXQv39/bN68WXnMRx99BH19fWzZsuW13yMkJARz587FsWPH4ObmpmyPjo7G0KFDceHCBUilUvj7+0NDQwOCIODAgQPK43gbi4iIiN6YlZUVXFxcVNqaNGminGllZmYGTU3NVx7zKvPmzUNISAgOHz6sEnQAoEWLFoiNjUVmZiaSkpJw8OBBpKWloUGDBm90LQw7REREVEybNm0QFxen0nbjxg3Y2dkBKOr58fb2fuUxJZk7dy5mzJiBgwcPwsvLq8TjDA0NUbduXdy8eRPnzp1D9+7d3+haGHaIqEYrzToiL3z11VeQSCRYuHDha8/7yy+/wN7eHrq6uvDx8VFZRyQ9PR0jR46Ek5MT9PT0YGtri1GjRkEmk5XXZRG9tbFjx+LMmTOYPXs2bt26hc2bN2PVqlUICAhQHjNhwgRs27YNq1evxq1bt7B06VLs3bsX33zzjfKYzz//HIGBgcqv58yZg6CgIKxduxb29vZITk5GcnIysrOzlcfs2LEDx44dU04/79ixI3r06IFOnTq92cUIJMhkMgGAIJPJ1F0KEVWi9PR0wc7OThg0aJAQGRkpxMfHC4cOHRJu3bpV7Ng///xTcHd3F6ytrYWff/75lefdunWroK2tLaxdu1a4cuWKMGzYMMHIyEhISUkRBEEQLl26JPTq1UvYs2ePcOvWLSEsLExo1KiR0Lt374q4TKI3tnfvXsHV1VXQ0dERnJ2dhVWrVhU7Zs2aNYKjo6Ogq6sruLu7C7t27VJ5/t133xUGDhyo/NrOzk4AUOwxbdo05TGLFi0S6tevL2hpaQm2trbClClThPz8/GLfu7Tv3xygDA5QJqqpSrOOCFDU++Pj44NDhw6hW7duGDNmDMaMGVPi8T4+PvD29sbSpUsBFE3RtbGxwciRIzFp0qSXvmbHjh349NNPkZOTA01NzTe+JqKqJkn2FHce58DBTB9Whnrlem4OUCYieo3XrSMCFAWVzz77DBMmTFAucPYqBQUFiI6OVk6jBYrWCPH390dERESJr3vxy5pBh8RkW1QC2oQcwf9WR6JNyBFsi3r9wOWKwLBDRDXW69YRAYrGF2hqamLUqFGlOufjx48hl8thYWGh0m5hYYHk5OQSXzNjxgx8+eWXb34xRFXMg4xcTPrjEhTP7x8pBGDyn5eRJHta6bXwIwQR1VgKhQJeXl6YPXs2AMDT0xOXL1/GihUrMHDgQERHR2PRokWIiYmBRCKpkBqysrLQrVs3uLi4YPr06RXyPYgqW7IsD1/9Fo3/jpORCwLuPs4t99tZr8OeHSKqsV63jsiJEyeQmpoKW1tbaGpqQlNTE/fu3cP48eNhb2//0nOamZlBKpUiJSVFpT0lJQWWlpYqbU+ePEGXLl1gYGCAnTt3QktLq/wujkhN9l18iM4Lj+PSg6xiz0klEtib1ar0mhh2iKjGet06Ip999hkuXryI2NhY5cPa2hoTJkzAoUOHXnpObW1ttGjRAmFhYco2hUKBsLAw+Pr6KtuysrLQqVMnaGtrY8+ePa/cJZqoOsjKK8TYbbEYsfk8ZE8L0ayeIcZ3bAzp815RqUSC2b1cK71XB+BtLCKqwcaOHYvWrVtj9uzZ6NOnD86ePYtVq1Zh1apVAABTU1OYmpqqvEZLSwuWlpZwcnJStvn5+aFnz54YMWIEAGDcuHEYOHAgvLy80LJlSyxcuBA5OTkYPHgwgH+CTm5uLn777TdkZWUhK6voU3DdunUhlUor4/KJys2Z+DSM334BDzKfQkMCBLR3xCi/RtCSauBjr/q4+zgX9ma11BJ0AIYdIqrBvL29sXPnTgQGBuLHH3+Eg4MDFi5ciAEDBpTpPLdv38bjx4+VX/ft2xePHj3C1KlTkZycDA8PDxw8eFA5aDkmJgaRkZEAAEdHR5Vz3blzp8RbZERVTf4zORYcvoFVJ+IhCICtSS383NcdLexMlMdYGeqpLeS8wHV2wHV2iOjtVOQ6IkRVVVzyE4zZFotrSUW9kv28bTDlAxfU1qm8fpTSvn+zZ4eI6C1si0pA4J9F02s1JEBwr2bo622r7rKIKoxCIWDtqTuYezAOBXIFTPS1EdKrGTo1tXz9i9WEA5SJiN7QnUc5xdYRCfzzEi7ez6z0Wl63x5cgCJg6dSqsrKygp6cHf39/3Lx585XnfPLkCcaMGQM7Ozvo6emhdevWiIqKUjkmOzsbI0aMQP369aGnpwcXFxesWLGixl+HWD3MfIpP10Ri5v5rKJAr0MHZHIfGtKvSQQdg2CGqdBXxy7w05/3zzz/RqVMnmJqaQiKRIDY2tiIur0bIK5Rjzck76LHsVLF1RBQC8NHSU2gTcgQjt5zHulN3cCExE4VyRYXVk5GRgTZt2kBLSwsHDhzA1atXMX/+fBgbGyuPmTt3LhYvXowVK1YgMjIS+vr66Ny5M/Ly8ko87xdffIHQ0FD8+uuvuHTpEjp16gR/f388ePBAecy4ceNw8OBB/Pbbb7h27RrGjBmDESNGYM+ePTX2OsRqd+wDdFl4HKdvp0FPS4pZPV2xZqAX6hroqLu01+KYHXDMDlWejIwMeHp6on379vj6669Rt25d3Lx5Ew0bNkTDhg0BFK3YGxwcjA0bNsDBwQFBQUG4dOkSrl69WuL05NKc99dff8WdO3dgbW2NYcOG4fz58/Dw8KisSxeFvEI5NkcmYHn4bTx6kl/icRKgWAjS1dKAWz0jeNoZobmtMZrbGpfbm8Tr9vgSBAHW1tYYP348vv32WwBF21NYWFhg/fr16NevX7HXPH36FAYGBti9eze6deumbG/RogW6du2KmTNnAgBcXV3Rt29fBAUFlXhMTbsOsZHlFiJo92XsufAQAOBuY4Sf+7ijQd3aaq6MY3aIqqQ5c+bAxsYG69atU7Y5ODgo/ywIAhYuXIgpU6age/fuAICNGzfCwsICu3bteukv89KcFyhaMwYA7t69W16XU2PkFcqx9WwClh27jdTnIaeekR5GdHCEQiFg6u4rkAuCch2Rbm7WuJCYiZh7GYhJyEBMQiZkTwtx9m46zt5NV57X1qQWmtsaobldUfhxtjSAprTsHe579uxB586d8cknnyA8PBz16tXDN998g2HDhgEomuGVnJyssl+XoaEhfHx8EBER8dJ/V8+ePYNcLi8WsPX09HDy5Enl161bt8aePXswZMgQWFtb49ixY7hx4wZ+/vnnGnsdYnL61mOM33EBSbI8SDUkGNnBESPaO77Rv1N1YtghqkQV8cu8NOelN5NXKMe2qEQsO3YLKVn/hJyA9o74uEV9aGsW/cLv0MS82DoibRzN0MbRDEDRgM74xzmIScjA+YQMxNzLxI3UJ0hIz0VCei52xRZ9Yq6lLYVbfUM0tzVGCztjeNoaw0Rf+7V1vtjja9y4cZg8eTKioqIwatQoaGtrY+DAgco9ucqyX5eBgQF8fX0xY8YMNGnSBBYWFtiyZQsiIiJUpssvWbIEX375JerXrw9NTU1oaGhg9erVaNeuXVn+qkV1HWKQVyjHT4fisObkHQCAg5k+FvRxh6et8WteWTUx7BBVoor4ZV6a81LZ5D+TY3tUIn45ehvJWUVjQawMdRHQ3hGfeNWHjqbqon+vW0dEQ0MCR/PacDSvjT5eNgCKVpuNTchU9vycT8jAk7xnOBOfjjPx//T+OJjpw9PWSBmAGlsYQKqhuk/X6/b4elO//vorhgwZgnr16kEqlaJ58+bo378/oqOjlccsWbIEZ86cwZ49e2BnZ4fjx48jICAA1tbWKqG9NMRyHdXd1YdZGLstFnEpTwAA//OxxZRuTVBLu/pGhupbOVE1VFG/zCvqvDVN/jM5dpy7j2VHb+Gh7J+Q8017R/R5Sch5G3V0tdCucV20a1wXQFHvz61H2cpbX9H3MnD7UQ7uPC56/BlTNJi2to4m3G2Ken+a2xmjuY1xiXt8/fHHHwCg3JMrJSUFVlZWymNSUlJeOW6rYcOGCA8PR05ODrKysmBlZYW+ffuiQYMGAIrGw0yePBk7d+5Ujodxc3NDbGws5s2bV+aQIJbrqK7kCgH/dyIe8w/fQIFcAbPa2pj7sRs6OFu8/sVVXJnDztGjR9G+ffuKqIVI9Crql/nrzkuvVvBMgR3RifjlyD8hx6KODgLaO6Kvt025hpySaGhI0NjCAI0tDNCvZdE6PZm5BTifmInz9/7p/cnOf4ZTt9Jw6laa8rX5po44eCoGW88moLmdMRzr1lbZ48vBwQGWlpYICwtT/jvKyspCZGQkvv7669fWpq+vD319fWRkZODQoUOYO3cuAKCwsBCFhYXQ0FAdvyGVSqFQlH322ev2Kqsu11Ed3c/IxfjtFxB5p6hXsaOLBUJ6NYNp7ao/06o0yhx2unTpgvr162Pw4MEYOHAgbGxsKqIuIlGqqF/mrzsvvVyhXIHfo+9j6ZFbeJD5FABgbqCDb95riH4tbaGrpd49qoxqaaO9kznaO5kDKPrkfSPlSdGtr3tFt8DuPM6BpNkHePDbBHw9IQi1nNtC4/FtJO1bgf7jZuD4jUfwsDXCmDFjMHPmTDRq1Eg5y8/a2ho9evRQfr//7vF16NAhCIIAJycn3Lp1CxMmTICzs7Nyj686derg3XffxYQJE6Cnpwc7OzuEh4dj48aNWLBgQZmv93V7lUkkkmpxHdWJIAjYef4Bpu2+gif5z1BLW4ppH7qgj5cNJBLJ609QXQhl9OjRI2HBggWCu7u7oKmpKXTq1EnYtm2bkJ+fX9ZTVRkymUwAIMhkMnWXQiJ39uxZQVNTU5g1a5Zw8+ZNYdOmTUKtWrWE3377TXlMSEiIYGRkJOzevVu4ePGi0L17d8HBwUF4+vSp8pgOHToIS5YsKdN509LShPPnzwv79+8XAAhbt24Vzp8/LyQlJVXOxVchBc/kwtaz94Q2IWGC3cR9gt3EfYLXzFBh7cl44WnBM3WXVyZp2fnC31eThUE/LBMMrRsIEqmWoGlSXzDpPEJ5bfaT9gn+848KrT/+UjA0qSto6+gIfn5+QlxcnMq56tvYCkNGThAeZuYKgiAI27ZtExo0aCBoa2sLlpaWQkBAgJCZmanymqSkJGHQoEGCtbW1oKurKzg5OQnz588XFArFG13P3r17BVdXV0FHR0dwdnYWVq1apfK8QqEQgoKCBAsLC0GnCl9HdZCRky9881u08t9Jz19OCncfZ6u7rDIp7fv3W62zExMTg3Xr1mHLli0AgP/9738YOnQo3N3dyyeJVRKus0OVad++fQgMDMTNmzfh4OCAcePGqcyaEgQB06ZNw6pVq5CZmYm2bdti2bJlaNy4sfIYe3t7DBo0CNOnTy/1edevX6/8JPtv06ZNUzmPmBXKFdgZ8wBLjt5EYnpRT45ZbR18/V5DDPBRf09OeXgmV+B68hOcfz7uJyYhEwnpucWOM9TTgqetEVo8H/tzKzUbP+y9Uu23veD2HaVz4uYjfLvjAlKy8qGpIcEY/0b46t2G1W5KeWnfv996UcGHDx9i1apVCAkJgaamJvLy8uDr64sVK1agadOmb3PqSsOwQ9UNN54sm2dyBXaef4AlR24p3/jNamvjq3cbYoCPHfS0q3/IeZVHT/KLwk9CBs7fy8SF+5nIf/b6sSg+DibQqUYBML9Qrhxz8oKGBDg1qQP/nzyXVyhHyIHrWH/6LgCgQV19LOzrAbf6Rmqt601VaNgpLCzE7t27sXbtWoSGhsLLywtDhw5F//798ejRI0yZMgUxMTG4evXqW10EULQE/sSJE3HgwAHk5ubC0dER69atg5eXF4B/PgWvXr0amZmZaNOmDZYvX45GjRqV+nsw7FB1sunMPUzZfRkCP7m+1jO5ArtiH2LJkZu4l1YUckz1i0LOp63EH3JKUihX4FpSlrLnJ+LWYzzOKVB3WRWmlYMJ+rW0RXsncxjW0lJ3OWpz+YEMY7bF4lZqNgDgc187BHZtUq3/H1RY2Bk5ciS2bNkCQRDw2Wef4YsvvoCrq6vKMcnJybC2tn7rUewVtbT+fzHsUHVxLSkLXRepLqUvATD9Ixd0dbWCeZ3S/ZsXu2dyBfZceIglR27hzuMcAICJvjaGt2uAz3ztqvV6IRUhSfYUbUKOKDc0BQCJBAjq1gRGtV6/qGFVkZlbgBn7rhXbquMFqYYELe1N0NHFAh1dLGBjUqtS61MXuULAivDbWPj3DRTKBdQ10MFPH7vhvecD36uzCgs7fn5++OKLL9CrVy/o6Lx8StqzZ89w6tQpvPvuu2Wr+j8qYp+Ul2HYoerg7uMc9Ft1RrnI3cs0qKuPVg1Mnz9MYG5QMeFn+vTp+OGHH1TanJyccP36ddy9e7fYVhUvbN++HZ988kmx9sLCQkyZMgV//fUX4uPjYWhoCH9/f4SEhMDa2lp5XHp6OkaOHIm9e/dCQ0MDvXv3xqJFi1C7dtEePXKFgD0XHmBJ2C3E/yvkfNmuAT5rZQd9HYackmyLSsDkPy+rbHtRHXsMVa8D+Pq9olWSQ6+mKBfJe8HZ0kAZfFytDaGhIaLZR88lpudi3PZYRN3NAAB0aWqJ2b2alWpl7uqg0sbsVCQXFxd07twZ9+/ff+kS+PHx8WjYsGGxDQ3fffddeHh4YNGiRS89b35+PvLz/9nELysrCzY2Ngw7VGVF30vHFxvOISO3sNhzEgCNLWvjRko2/vu/ueHz8OPb0BQ+DqbltvHk9OnT8fvvv+Pvv/9WtmlqasLMzAxyuRyPHj1SOX7VqlX46aefkJSUpAwm/yaTyfDxxx9j2LBhcHd3R0ZGBkaPHg25XK6yc3vXrl2RlJSElStXorCwEIMHD4a3tzd+/W0T9l18iEVhNxH/qCjkGNfSwrB2DTDQ154hp5SSZE+LbXtRHZV0HQlpuQi9loLQq8mIupsB+b+6sizq6MC/SVHw8W1o+lZrK73qw8ALERER+P777xEZGQmpVAoPDw8cOnQIenov/3uXy+WYPn06fvvtN+Xdk0GDBmHKlCmQSCQv/cDQyNMXyU69ka9tiNo6mpj+UVP0bl5PVFPKK2wj0ODgYFhYWGDIkCEq7WvXrsWjR48wceLEsldbgopaWj84OLjYP0SiqmrfxYcYt/0CCp4p4FbfEB+6WyPkr+vFPoHLnhbi7J10nIlPw5n4NFxNysLtRzm4/SgHmyITAACNzGur9Py8zYJhmpqaykUQ/00qlRZr37lzJ/r06fPSoAMU7f8VGhqq0rZ06VK0bNkSCQkJsLW1xbVr13Dw4EFERUUpx+wtXLQYH37QDQmOvXC/oOhNwqiWFoa90wADW9ujNkNOmbxu24vqoqTrsDWthaFtHTC0rQMycwtwNC4VoVdTEB73CClZ+dgUmYBNkQnQ15biXae66OhigfZO5m90K69p06bFPgy8EBERgS5duiAwMBBLliyBpqYmLly4UGxRw3+bM2cOli9fjg0bNqBp06Y4d+4cBg8eDENDQ4waNQq5ubmIiYlBUFAQ7Bo1QciuaPy1OhiIuYiPpm3Agj4eNea23cuU+TfBypUrsXnz5mLtTZs2Rb9+/co17FTUEviBgYEYN26c8usXPTtEVYkgCFh5PB4hB4o+DXZ0scCifh6opa2JD9ysin1yNdTTUnbJA0XjF87eSUdEfBrOxKfjWlIWbqZm42ZqNn49cw8A0NiiKPz4NjCFTwPTMnVt37x5E9bW1tDV1YWvry+Cg4Nha1v8tkd0dDRiY2Pxyy+/lOn6ZTIZJBIJjIyMABS9QRgZGcHLywsKhYD9l5Lw8xVtCJDgxuXzsHJ/F8PeccDA1vYw0K25g1CpdIxqaaOnZ3309KyP/GdyRNxOQ+jVFPx9LQUpWfn461Iy/rqUDKmGBN72xujoYomOTSxga1q6wFDShwGgaPHEUaNGYdKkSco2JyenV57v9OnT6N69u3I7C3t7e2zZsgVnz54F8M8HhqNxqRjx+0U8yq8L885f4/76sZjbpV6NDjrAG4Sd5ORklWXsX6hbty6SkpLKpagXKmppfR0dnRLHGxFVBc/kCkzdcwWbn/fIDG5jjyndXJQbQJbmE7hRLW10amqJTk2L/p9k5BQg8l89P9eTn+BGSjZupGRjY0RR+HG2NFD2+vg4mMK4hPDj4+OD9evXw8nJCUlJSfjhhx/wzjvv4PLlyzAwMFA5ds2aNWjSpAlat25d6uvPy8vDxIkT0b9/f2XXdHJyMszNzYtuV/19EzefzyiR6hngPRttrJnYniGH3oiOphTvOZnjPSdzzOjuiksPZPj7WgpCr6bgevIT5easM/ZdhZNF0TgffxcLuNUreZxPSR8GUlNTERkZiQEDBqB169a4ffs2nJ2dMWvWLLRt27bEGlu3bo1Vq1bhxo0baNy4MS5cuICTJ08qV3h+WiDH7L+uKT/INDKvjT4eDhi+QQJTk+q5U3l5KnPYsbGxwalTp4oNQDx16pTKQMLyUNH7pBBVRdn5zxCwKQbhNx49nxHjgiFtXz7gtyyM9bXRxdUSXVyLwk96TgHO3klDxO2inp+4lCe4nlz0eLEGh7OlAXwbFt328nEwUXbnd+3aVXleNzc3+Pj4wM7ODtu3b8fQoUOVzz19+hSbN29GUFBQqessLCxEnz59IAgCli9fDqBok8y45Ce4n/EUIzafBwAY6Grii7YNMHOtNto7mzPoULnQ0JDA3cYI7jZGGN/JCQlpucrgc/Zu0f+TuJQnWHr0FswNdOD/vDfVt4GpclHKV30YiI+PB1A0rmfevHnw8PDAxo0b4efnh8uXL5e4bMqkSZOQlZUFZ2dnSKVSyOVyzJo1CwMGDMDF+5kYsy1WOV5tcBt7jH7PHn7vtVP5wFCTlTnsDBs2DGPGjEFhYSE6dOgAAAgLC8N3332H8ePHl2tx5bVPClF1kSR7iiHrz+FaUhZ0tTSwuJ+nsmemvJnoa6OLqxW6uBb1iqZl56v0/NxIyVaGn3Wn7kIiAZpY1lEOeG5pb6Jcs8TIyAiNGzfGrVu3VL7H77//jtzcXHz++eelqulF0Ll37x6OHDmC2rUNcOBSEhaF3cS5W7nIy0qHga4mhrZ1wOA2DtDXkmB8enqJtwuI3pataS0MaeuAIc/H+RyLe4TQqyk4FpeK1Cf52ByZgM3Px/m0a/x8nE87P2Wv6H8/DDRp0gQAMHz4cOWK5p6enggLC8PatWsRHBz80jq2b9+OTZs2YfPmzWjatCliY2MxZswYXMnUwBlpMzxTCLCoo4N5n7ijlb0RevfurfKBoaYrc9iZMGEC0tLS8M0336CgoGgRKl1dXUycOBGBgYHlWpy3tzd27tyJwMBA/Pjjj3BwcMDChQsxYMAA5THfffcdcnJy8OWXXyqX1j948GCp19ghqiquPszCkPVRSM7Kg1ltHawZ6AV3G6NK+/6mtXXwfjMrvN+sKPw8zs5HZHw6IuIf40x8Om6lZuNqUhauJmVh7ak7kEgAF6s68G1gCncLXdy6fRufffaZyjnXrFmDjz76CHXr1n3t938RdG7evIkjR44gKqkQi347iWtJWQAAY3tXpOXnYGlHI7zbpmjrjMOHD0OhUMDHx6ec/zaIijOqpY0envXQw7Me8p/JcSY+HaFXk/H31VQkZ+XhwOVkHLhcNM7Hy85YOYbOzvSfDwMvOgleNkQjISGhxO89YcIETJo0SbmkSh3rBrDYfhzb/m8p6g1bgW5uVpjVwxX6WhKVDwzs1SnyxlPPs7Ozce3aNejp6aFRo0bVegwM19khdTsWl4qATTHIKZDD0bw21g3yrnIDClOf5CEyvqjn59dFM1Fg7QlNQ3M8e5IO2clNKEiNh9+U3/Cue0P4NjSFsTwdLZo1xV9//YUuXboUO5+zszOCg4PRs2dPFBYW4uOPP0ZMTAwmL1yP368WjScCAAMjI3zRrhGGtm2Afr0/QkpKClasWKGceu7l5fXSSRNElUUQhKJxPldTcPj5OJ9/a2CkgTOz+mPEt5MQPOU72NraYMiQIZgxY4byGE9PT3Tt2lU5Iee/TE1NMXPmTHz11VfYFpWIH/ddRVL4FuReDsO2vyPR3cMaz549U35gOHr0aKk+ZFR3olhnp7Iw7JA6bY5MQNDuy5ArBLRuaIrln7aAoV7VHn/Sr18/HAsPR1paOvQMjKBn0xSaPv+DlvE/EwUyj29AwfVwfL/xCFo71oW3g4nKVHCJRIKff1mJlp17QZrzCK3cXV72rbD3wGF80KUjgKJFBUeMGKGyqODixYtLnNJOpA7DR4yGsbMvrj7RQfS120g/XvRhwPqL5bCyMEed24dxZscKrFr9f2jp1RwbNmzAvHnzcPnyZeXuAH5+fujZsydGjBgBABg0aBAOh/6NZn2/xZWndVCQchtZocswZMgQLFs0X+UDw759+1SWZDExMYG2tjgWEfyvCg07586dw/bt25GQkKC8lfXCn3/+WfZq1Yxhh9RBoRAw91AcVoTfBgD0bl4fwb2aQVuzeu06/EKyLA+Rd4rG+0TcTsPdNNWdtqUaErjWM4Tv89le99Jylbts/5u+thQDW9tj2DsNSpwNRlSV9evXD8ePH0daWhrMzOqigWsL2HUZgthMXWTnPwMAyM7sQPb5/UBeNho4N8XP837C+x3bK89hb2+PQYMGYfr06QCAfdG3MXTkBKRdOQlFrgym5hb4ctBnmDZtGrS1tV+5cvnRo0fx3nvvVfRlq0WFhZ2tW7fi888/R+fOnXH48GF06tQJN27cQEpKCnr27Il169a9dfGVjWGHKlteoRzjd1zA/otFyzWM9W+MUX6OolrZNEn2tGjMz+00nLmTptyI81U+a2WHsR0bi2Ype6J/ezHO5+/n6/kkyf7Z+kVDAnjZm6Dj81Wc7c30kSR7iutJWdgd+xC7Yh8CAJwsDLCwnweaWPG9CqjAsOPm5obhw4cjICAABgYGuHDhAhwcHDB8+HBYWVlVy5WJGXaoMqXnFGDYxnOIvpcBLakEc3q7oVfz+uouq8I9zHyqnOl1NC4Vj54U32V7y7BW8G1oqobqiCqXIAi4/CDr+fYVKcqB+C+YG+jg0ZN8lU1Nh73jgPGdnJRT3KkCw46+vj6uXLkCe3t7mJqa4tixY2jWrBmuXbuGDh06lPvCgpWBYYcqy53HORi87izupuWijq4mVn7mVSPf3F+2y7ZUIsHJSe1FsV0BUVklphet5/P3tRScuZ0G+X/emTUkwKlJHfj/4z9K+/5d5sEBxsbGePKkaKR5vXr1cPnyZQBAZmYmcnNf301NVFOdu5uOXstO4W5aLuob6+HPb1rXyKADFK0AHdyrGaTPb9u92OOLv8ipprIxqYXBbRyw6YtWWPGpV7HnFQJw9zHfY99UmdfZadeuHUJDQ9GsWTN88sknGD16NI4cOYLQ0FD4+flVRI1E1d7eCw8xfkfRZp7u9Q3xfwO9y20H8uqqr7ct2jWuK4pdtonKk2v9OtCQoFjPp71Z1VqOojop822s9PR05OXlwdraGgqFAnPnzsXp06fRqFEjTJkyBcbG1W8PDt7GoooiCAJWhMdjzsGizTw7uVhgUT9P6GnznjsRlWxbVAIm/3kZckFQ9nz29S6+0W5NVyFjdp49e4bNmzejc+fOKnP4qzuGHaoIhXIFpu6+jC1nEwEAQ9o44PtuTZSbeRIRvUqS7Cl7Pl+jtO/fZbqNpampia+++grXrl176wKJxOxJXiECNp/H8RuPoCEBpn7ggkFt3n4zTyKqOawM9RhyykmZx+y0bNkSsbGxyp3HiUhVkuwpBq+LwvXkJ9DTkmJxf090dBFPTygRUXVT5rDzzTffYNy4cUhMTESLFi2gr6+v8rybm1u5FUdU3Vx5KMOQ9VFIycpHXQMdrB3ojWb1DdVdFhFRjVbmAcoaGsVnq0skEgiCAIlEArlcXm7FVRaO2aHycDQuFSOeb+bZ2KI21g7yRn1jzp4gIqooFTJmBwDu3LnzVoURidGmyHuYuvtKtdrMk4iopihz2OFYHaJ/KBQC5hy6jpXh8QCAj1vUx+ye1XczTyIiMSpz2Nm4ceMrn//888/fuBii6iSvUI7x2y9g/6WiLVLGdWyMkR3EtZknEZEYlHnMzn8XDSwsLERubi60tbVRq1YtpKenl2uBlYFjdqis0rLzMWzjOcQkZEJLKsHcj93Q01P8m3kSEVUlFTZmJyMjo1jbzZs38fXXX2PChAllPR1RtRP/KBuD10fh3vPNPFd97oVWDWrmHldERNVBmcPOyzRq1AghISH49NNPcf369fI4JVGVFHU3HcM2nkNmbiFsTPSwbpA3HM0N1F0WERG9QrmEHaBodeWHDx+W1+mIqpw9Fx7i2+0XUCBXwN3GCGsGesGsds3ezJOIqDooc9jZs2ePyteCICApKQlLly5FmzZtyq0woqpCEAQsO3YbPx2KAwB0bmqBhX25mScRUXVR5vmxPXr0UHn06tUL06dPh5ubG9auXVsRNRKpCAkJgUQiwZgxY5RtycnJ+Oyzz2BpaQl9fX00b94cf/zxxyvPI5fLERQUBAcHB+jp6aFhw4aYMWMG/j1mv1CuwLDFuzHhywFI+LkPHi78BDGLv8aj5AcVdXlERFTOytyzo1AoKqIOolKJiorCypUri21L8vnnnyMzMxN79uyBmZkZNm/ejD59+uDcuXPw9PR86bnmzJmD5cuXY8OGDWjatCnOnTuHwYMHw9DQEKNGjcKTvEJ89vMe7P1xMAzcOyLw+6n43zvOuHLlCnR1dSvjcomIqByU25gdooqWnZ2NAQMGYPXq1Zg5c6bKc6dPn8by5cvRsmVLAMCUKVPw888/Izo6usSwc/r0aXTv3h3dunUDANjb22PLli04e/YsHmY+xZD1UTixcRFqO3rjj3XL4NekaDPPhg0bVuBVEhFReSvzbazevXtjzpw5xdrnzp2LTz75pFyKInqZgIAAdOvWDf7+/sWea926NbZt24b09HQoFAps3boVeXl5eO+990o8X+vWrREWFoYbN24AAC5cuICTJ0/C1edd9Fx2CteSZMiLP4fPOrfC3DGfw9zcHD4+Pti1a1cFXSEREVWEMoed48eP4/333y/W3rVrVxw/frxciiL6r61btyImJgbBwcEvfX779u0oLCyEqakpdHR0MHz4cOzcuROOjo4lnnPSpEno168fnJ2doaWlBU9PT3w04AtsfGSDlKx82NcqhKLgKdYtX4guXbrg8OHD6NmzJ3r16oXw8PCKulQiIipnZb6NlZ2dDW1t7WLtWlpayMrKKpeiiP4tMTERo0ePRmhoaIljZYKCgpCZmYm///4bZmZm2LVrF/r06YMTJ06gWbNmL33N9u3bsWnTJmzevBlNmzbFL7//jdVzp8G4w1N06dkXQR0s4Twd6N69O8aOHQsA8PDwwOnTp7FixQq8++67FXXJRERUjsrcs9OsWTNs27atWPvWrVvh4uJSLkUR/Vt0dDRSU1PRvHlzaGpqQlNTE+Hh4Vi8eDE0NTVx+/ZtLF26FGvXroWfnx/c3d0xbdo0eHl54ZdffinxvBMmTMCkSZPQp09f7E3UxMH8xjDw7g4hdifWDfaGQ30raGpqFvt33aRJEyQkJFT0ZRMRUTkpc89OUFAQevXqhdu3b6NDhw4AgLCwMGzZsgU7duwo9wKJ/Pz8cOnSJZW2wYMHw9nZGRMnTkRubi4AQENDNbtLpdJXzh7Mzc2FXABGbInBX5eSAQDtGpvjerIWtKQagFQb3t7eiIuLU3ndjRs3YGdnVx6XRkRElaDMYefDDz/Erl27MHv2bPz+++/Q09ODm5sb/v77b3brU4UwMDCAq6urSpu+vj5MTU3h6uqKwsJCODo6Yvjw4Zg3bx5MTU2xa9cuhIaGYt++fcrX+Pn5oWfPnhgxYgQAoFOX9zEx6AcY+H8DfXN79G3wDGvWbMSQIUOUr5kwYQL69u2Ldu3aoX379jh48CD27t2LY8eOVcq1ExHR23ujqefdunVTTtclUjctLS389ddfmDRpEj788ENkZ2fD0dERGzZsUBlMf/v2bTx+/Ljoz4+ycc+5H7TispAZuhyyvCzsqGeN4cOHY+rUqcrX9OzZEytWrEBwcDBGjRoFJycn/PHHH2jbtm2lXycREb0ZifDv5WJLISoqCgqFAj4+PirtkZGRkEql8PLyKtcCK0Npt4in6i1J9hR3HudAlluISX9eguxpIWxNamHdYG80rFtb3eUREVEZlfb9u8wDlAMCApCYmFis/cGDBwgICCjr6YgqxbaoBLQJOYL/rY7E15tiIHtaCA8bI/z5TWsGHSIikSvzbayrV6+iefPmxdo9PT1x9erVcimKqDw9yMjFpD8v4b99mAv7enDXciKiGqDMPTs6OjpISUkp1p6UlARNTe4+QVVHalYefjl6C72Wny4WdAAgSZZX+UUREVGlK3M66dSpEwIDA7F7924YGhoCADIzMzF58mR07Nix3AskKotncgXCbzzC1qhEHLmeCrni5UPSpBIJ7M1qVXJ1RESkDmUOO/PmzUO7du1gZ2en3GAxNjYWFhYW+PXXX8u9QKLSSEjLxfZzidgRnYiUrHxlu5edMfp62yCvUI7pe65CLgiQSiSY3csVVoZ6aqyYiIgqS5lnYwFATk4ONm3ahAsXLijX2enfvz+0tLQqosYKx9lY1VP+MzkOXUnBtqgEnLqVpmw30ddGL8966NfSBo7mBsr2JNlT3H2cC3uzWgw6REQiUNr37zcKO2LDsFO9xCU/wbaoRPx5/j4ycwsBABIJ0NbRDP28beHvYg4dTamaqyQioopW2vfvNx5RfPXqVSQkJKCgoECl/aOPPnrTUxKVKCf/GfZdfIitUYk4n5CpbLcy1MUnXjb4pEV92JhwDA4RERVX5rATHx+Pnj174tKlS5BIJHjRMSSRSAAAcrm8fCukGksQBFy4L8PWswnYe+EhcgqK/m1pakjg38QCfVvaoF2jupBqSNRcKRERVWVlnno+evRoODg4IDU1FbVq1cKVK1dw/PhxeHl5cb+gKi4kJAQSiQRjxoxRtg0fPhwNGzaEnp4e6tati+7du+P69euvPM+gQYMgkUhUHl26dCl23P79++Hj4wM9PT0YGxujR48epaozM7cA607dQddFJ9Djl1PYGpWInAI5HMz0MamrM04HdsCKz1qgvZM5gw4REb1WmXt2IiIicOTIEZiZmUFDQwMaGhpo27atcu+g8+fPV0Sd9JaioqKwcuVKuLm5qbS3aNECAwYMgK2tLdLT0zF9+nR06tQJd+7cgVRa8riXLl26YN26dcqvdXRUF+f7448/MGzYMMyePRsdOnTAs2fPcPny5RLPp1AIOBOfhq1RiTh4JRkFz4p2K9fR1EC3Zlbo622Dlg4myh5EIiKi0ipz2JHL5TAwKJrhYmZmhocPH8LJyQl2dnaIi4sr9wLp7WVnZ2PAgAFYvXo1Zs6cqfLcl19+qfyzvb09Zs6cCXd3d9y9excNGzYs8Zw6OjqwtLR86XPPnj3D6NGj8dNPP2Ho0KHKdhcXl2LHpmTl4ffo+9h+LhH30nKV7U2s6qB/Sxt0d68Hw1rVc5YfERFVDWUOO66urrhw4QIcHBzg4+ODuXPnQltbG6tWrUKDBg0qokZ6SwEBAejWrRv8/f2LhZ1/y8nJwbp16+Dg4AAbG5tXnvPYsWMwNzeHsbExOnTogJkzZ8LU1BQAEBMTgwcPHkBDQwOenp5ITk6Gh4cHfvrpJ7i6uuKZXIFjcUUL/x2N+2fhv9o6mujuYY1+3rZwrVeHvThERFQuyhx2pkyZgpycHADAjz/+iA8++ADvvPMOTE1NsW3btnIvkN7O1q1bERMTg6ioqBKPWbZsGb777jvk5OTAyckJoaGh0NbWLvH4Ll26oFevXnBwcMDt27cxefJkdO3aFREREZBKpYiPjwcATJ8+HQsWLIC9vT3mz5+Pdu++i9Er92N/XDZSn/yz8J+3vTH6etvi/WaWqKXNLUeIiKh8lfmdpXPnzso/Ozo64vr160hPT4exsTE/iVcxiYmJGD16NEJDQ6Grq1vicQMGDEDHjh2RlJSEefPmoU+fPjh16lSJr+nXr5/yz82aNYObmxsaNmyIY8eOwc/PDwpF0Xib77//Ht0+6oHDV1OgeOcryHbtx+L/+w0GHl1hoq+N3s3roa+36sJ/RERE5a1cPkabmJiUx2monEVHRyM1NVVll3q5XI7jx49j6dKlyM/Ph1QqhaGhIQwNDdGoUSO0atUKxsbG2LlzJ/r371+q79OgQQOYmZnh1q1b8PPzg5WVFQDgTIYelgWHKRf+0zSyhLVmDuYMaA7/JhbQ1izzZEAiIqIy4z0DEfPz88OlS5dU2gYPHgxnZ2dMnDjxpbOtBEGAIAjIz88v9lxJ7t+/j7S0NBibmWNbVAJ+jSkEpFrYfewcDNw7wcpQF708LDF3bQZGfuSL95tZvfW1ERERlRbDjogZGBjA1dVVpU1fXx+mpqZwdXVFfHw8tm3bhk6dOqFu3bq4f/8+QkJCoKenh/fff1/5GmdnZwQHB6Nnz57Izs7GDz/8gN69e8PS0hK3bt3CyLHfwsjSBtOipXgaVRSuDD3fR37kVkz8uA0+aN0QC+bPg4ZEgk8++aRS/w6IiIgYdmowXV1dnDhxAgsXLkRGRgYsLCzQrl07nD59Gubm5srj4uLiIJPJAABSqRQXL17E+vUbkJGZCS0DU2jausOo97d4qpCigZk++nrb4MPvfsOC2dMxf/JIzHz6FD4+Pjhy5AiMjY3VdblERFRDlXkj0OPHj6N169bQ1FTNSc+ePcPp06fRrl27ci2wMnAj0FdLkj3Fncc5sDOphXtpudgSlYhDl5NRIFdd+K9fS1t423OgOhERVY4K2wi0ffv2SEpKUvnkDwAymQzt27fn3lgisy0qAYF/XoLiJZHY5fnCfx951IOhHhf+IyKiqqnMYUcQhJd+ck9LS4O+vn65FEVVQ5LsKSb9eQn/7fvr6WmNoW0bwLWeoXoKIyIiKoNSh51evXoBKNrdfNCgQSp7Icnlcly8eBGtW7cu/wpJLQqeKTBr/7ViQQcA+njZMugQEVG1UeqwY2hY9OYmCAIMDAygp6enfE5bWxutWrXCsGHDyr9CqnT30nIwast5XLgvK/acVCKBvVktNVRFRET0Zkoddl7scG1vb49vv/2Wt6xEatf5B5iy6zKy85+hjm7RXlWbIxMhFwRIJRLM7uUKK0O915+IiIioiijzbKynT59CEATUqlX06f7evXvYuXMnXFxc0KlTpwopsqJxNhaQnf8MU3dfxp8xDwAALe1N8HM/D9Qz0kOS7CnuPs6FvVktBh0iIqoyKmw2Vvfu3dGrVy989dVXyMzMRMuWLaGtrY3Hjx9jwYIF+Prrr9+qcKp8l+7LMHJLDO6m5UJDAozya4QR7R2hKS3azsHKUI8hh4iIqq0yb04UExODd955BwDw+++/w9LSEvfu3cPGjRuxePHici+QKo5CIWD18Xj0Wn4Kd9NyYW2oi61f+mKMf2Nl0CEiIqruytyzk5ubCwODol2qDx8+jF69ekFDQwOtWrXCvXv3yr1AqhiPnuRj/I4LOH7jEQCgS1NLhPRuBqNa2mqujIiIqHyV+eO7o6Mjdu3ahcTERBw6dEg5Tic1NbXGjnepbsJvPELXRcdx/MYj6GhqYFZPVyz/tDmDDhERiVKZe3amTp2K//3vfxg7diw6dOgAX19fAEW9PJ6enuVeIJWfgmcKzDsch1XH4wEAThYGWPI/TzS2MFBzZURERBWnzD07H3/8MRISEnDu3DkcOnRI2e7n54eff/65XIv7r5CQEEgkEowZM0bZlpeXh4CAAJiamqJ27dro3bs3UlJSKrSO6ujO4xz0Xn5aGXQ+a2WH3SPaMOgQEZHovdEoVEtLSxgYGCA0NBRPnz4FAHh7e8PZ2blci/u3qKgorFy5Em5ubirtY8eOxd69e7Fjxw6Eh4fj4cOHytWeqcifMffxweITuPRABkM9Laz8rAVm9HCFrpZU3aURERFVuDKHnbS0NPj5+aFx48Z4//33kZSUBAAYOnQoxo8fX+4FAkB2djYGDBiA1atXw9jYWNkuk8mwZs0aLFiwAB06dECLFi2wbt06nD59GmfOnKmQWqqTJ3mFGLstFuO2X0BOgRwtHUxwYPQ76NzUUt2lERERVZoyh52xY8dCS0sLCQkJyoUFAaBv3744ePBguRb3QkBAALp16wZ/f3+V9ujoaBQWFqq0Ozs7w9bWFhERESWeLz8/H1lZWSoPsbmQmIkPlpzEzvMPoCEBxnVsjC3DWsHaiOvlEBFRzVLmAcqHDx/GoUOHUL9+fZX2Ro0aVcjU861btyImJgZRUVHFnktOToa2tjaMjIxU2i0sLJCcnFziOYODg/HDDz+Ud6lVgkIhYPWJePx0KA7PFALqGelhUT8PeNmbqLs0IiIitShz2MnJyVHp0XkhPT1dZSf08pCYmIjRo0cjNDQUurq65XbewMBAjBs3Tvl1VlYWbGxsyu386pL6JA/jt1/AiZuPAQDvN7NEcE83GNbSUnNlRERE6lPm21jvvPMONm7cqPxaIpFAoVBg7ty5aN++fbkWFx0djdTUVDRv3hyamprQ1NREeHg4Fi9eDE1NTVhYWKCgoACZmZkqr0tJSYGlZcnjUnR0dFCnTh2VR3V3NC4VXReewImbj6GrpYHgXs3wy/+aM+gQEVGNV+aenblz58LPzw/nzp1DQUEBvvvuO1y5cgXp6ek4depUuRbn5+eHS5cuqbQNHjwYzs7OmDhxImxsbKClpYWwsDD07t0bABAXF4eEhATl+j9il/9MjrkH47Dm5B0AgLOlAZb090QjTiknIiIC8AZhx9XVFTdu3MDSpUthYGCA7Oxs9OrVCwEBAbCysirX4gwMDODq6qrSpq+vD1NTU2X70KFDMW7cOJiYmKBOnToYOXIkfH190apVq3KtpSqKf5SNUVvP4/KDogHWA33tEPh+E04pJyIi+pcyh52EhATY2Njg+++/f+lztra25VJYaf3888/Q0NBA7969kZ+fj86dO2PZsmWVWkNlEwQBv0ffx7Q9V5BbIIdRLS389LE7OrpYqLs0IiKiKkciCIJQlhdIpVIkJSXB3NxcpT0tLQ3m5uaQy+XlWmBlyMrKgqGhIWQyWZUfv/MkrxDf77yMPRceAgBaNTDBwr6esDQsvwHcRERE1UFp37/L3LMjCAIkEkmx9uzs7HKdMUXFnU/IwKit55GY/hRSDQnG+jfC1+85QqpR/OdBRERERUoddl5M1ZZIJAgKClKZfi6XyxEZGQkPD49yL5CK1s5Zcfw2Fhy+oVw7Z3F/D7Sw49o5REREr1PqsHP+/HkART07ly5dgra2tvI5bW1tuLu749tvvy3/Cmu41Kw8jN0ei1O30gAA3dysMLtnMxjqcUo5ERFRaZQ67Bw9ehRA0dTvRYsWVfmxLWJw5HoKvt1xEek5BdDTkmL6Ry7o42Xz0tuIRERE9HJlXlRw3bp1NSLoLF++HG5ubspFB319fXHgwAHl88OHD0fDhg2hp6eHunXronv37rh+/forzymRSF76+Omnn5THzJo1C76+vtDS0YO/R0Ok5xSgiVUd7B3ZBn29bRl0iIiIyqjMYaemqF+/PkJCQhAdHY1z586hQ4cO6N69O65cuQIAyh3Wr127hkOHDkEQBHTq1OmVs9GSkpJUHmvXroVEIlEuiAgAyRnZyLT0gp57FwDAoNb22PlNaziac5FAIiKiN1HmqediVNqpayYmJvjpp58wdOjQYs9dvHgR7u7uuHXrFho2bFiq79ujRw88efIEYWFhEAQBO84VrZ3ztFAORdxRpIetRnaW7I2vi4iISMwqbOp5TSSXy7Fjxw7k5OS8dBuKnJwcrFu3Dg4ODqXeUDQlJQX79+/Hhg0bkJVXiMl/XsK+i0kAgNYNTeFj54RpR3nLioiI6G3xNtYrXLp0CbVr14aOjg6++uor7Ny5Ey4uLsrnly1bhtq1a6N27do4cOAAQkNDVWapvcqGDRtgYGAAB6/2eH/RCey7mASphgTfdXHCr0N9ONuKiIionDDsvIKTkxNiY2MRGRmJr7/+GgMHDsTVq1eVzw8YMADnz59HeHg4GjdujD59+iAvL69U5167di2avfsBPl1/HvcznqK+sR52fOWLb7hIIBERUbnibaxX0NbWhqOjI4CiAclRUVFYtGgRVq5cCQAwNDSEoaEhGjVqhFatWsHY2Bg7d+5E//79X3neXQf+RlxcHLJaB0BbIeBDd2vM6umKOrrszSEiIipvDDtloFAokJ+f/9LnBEGAIAglPv/C31dTMGTyXGhbOsKwXiP82L0pPm5Rn1PKiYiIKgjDTgkCAwPRtWtX2Nra4smTJ9i8eTOOHTuGQ4cOIT4+Htu2bUOnTp1Qt25d3L9/HyEhIdDT08P777+vPIezszOCg4PRs2dP5BXKEXLgOtYevYrMK8fh0mMEdo9qi4Z1a6t834SEBKSnpyMhIQFyuRyxsbEAAEdHR9SurXosERERvR7DTglSU1Px+eefIykpCYaGhnBzc8OhQ4fQsWNHPHz4ECdOnMDChQuRkZEBCwsLtGvXDqdPn1bZDT4uLg7nbt6H5e3H+GHvVVxPfoKca8ehKZHgyPLJMDctHl6mTp2KDRs2KL/29PQEULSC9XvvvVfh101ERCQ2XGcHpZ+nXxbbohIQ+OclKP71t2uqr415n7ijvbN5yS8kIiKiUuE6O2qUJHtaLOgAwPrB3mhW30gtNREREdVUnHpeAe48zikWdAAgO7/krSSIiIioYjDsVAAHM338d6kcqUQCe7Na6imIiIioBmPYqQBWhnoI7tUM0ufTyaUSCWb3coWVoZ6aKyMiIqp5OGangvT1tkW7xnVx93Eu7M1qMegQERGpCcNOBbIy1GPIISIiUjPexiIiIiJRY9ghIiIiUWPYISIiIlFj2CEiIiJRY9ghIiIiUWPYISIiIlFj2CEiIiJRY9ghIiIiUWPYISIiIlFj2CEiIiJRY9ghIiIiUWPYISIiIlFj2CEiIiJRY9ghIiIiUWPYISIiIlFj2CEiIiJRY9ghIiIiUWPYISIiIlFj2CEiIiJRY9ghIiIiUWPYISIiIlFj2CEiIiJRY9ghIiIiUWPYISIiIlFj2CEiIiJRY9ghIiIiUWPYISIiIlFj2CEiIiJRY9ghIiIiUWPYISIiIlFj2CEiIiJRY9ghIiIiUWPYISIiIlFj2CEiIiJRY9ghIiIiUWPYISIiIlFj2CEiIiJRY9ghIiIiUWPYISIiIlFj2CEiIiJRY9ghIiIiUWPYISIiIlGr0mEnODgY3t7eMDAwgLm5OXr06IG4uDiVY/Ly8hAQEABTU1PUrl0bvXv3RkpKipoqJiIioqqmSoed8PBwBAQE4MyZMwgNDUVhYSE6deqEnJwc5TFjx47F3r17sWPHDoSHh+Phw4fo1auXGqsmIiKiqkQiCIKg7iJK69GjRzA3N0d4eDjatWsHmUyGunXrYvPmzfj4448BANevX0eTJk0QERGBVq1aleq8WVlZMDQ0hEwmQ506dSryEoiIiKiclPb9u0r37PyXTCYDAJiYmAAAoqOjUVhYCH9/f+Uxzs7OsLW1RURERInnyc/PR1ZWlsqDiIiIxKnahB2FQoExY8agTZs2cHV1BQAkJydDW1sbRkZGKsdaWFggOTm5xHMFBwfD0NBQ+bCxsanI0omIiEiNqk3YCQgIwOXLl7F169a3PldgYCBkMpnykZiYWA4VEhERUVWkqe4CSmPEiBHYt28fjh8/jvr16yvbLS0tUVBQgMzMTJXenZSUFFhaWpZ4Ph0dHejo6FRkyURERFRFVOmeHUEQMGLECOzcuRNHjhyBg4ODyvMtWrSAlpYWwsLClG1xcXFISEiAr69vZZdLREREVVCV7tkJCAjA5s2bsXv3bhgYGCjH4RgaGkJPTw+GhoYYOnQoxo0bBxMTE9SpUwcjR46Er69vqWdiERERkbhV6annEonkpe3r1q3DoEGDABQtKjh+/Hhs2bIF+fn56Ny5M5YtW/bK21j/xannRERE1U9p37+rdNipLAw7RERE1Y8o19khIiIiKiuGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNdGEnV9++QX29vbQ1dWFj48Pzp49q+6SiIiIqAoQRdjZtm0bxo0bh2nTpiEmJgbu7u7o3LkzUlNT1V0aERERqZkows6CBQswbNgwDB48GC4uLlixYgVq1aqFtWvXqrs0IiIiUjNNdRfwtgoKChAdHY3AwEBlm4aGBvz9/REREfHS1+Tn5yM/P1/5tUwmAwBkZWVVbLFERERUbl68bwuC8Mrjqn3Yefz4MeRyOSwsLFTaLSwscP369Ze+Jjg4GD/88EOxdhsbmwqpkYiIiCrOkydPYGhoWOLz1T7svInAwECMGzdO+bVCoUB6ejpMTU0hkUjK7ftkZWXBxsYGiYmJqFOnTrmdl94Mfx5VD38mVQt/HlULfx6vJwgCnjx5Amtr61ceV+3DjpmZGaRSKVJSUlTaU1JSYGlp+dLX6OjoQEdHR6XNyMiookpEnTp1+A+1CuHPo+rhz6Rq4c+jauHP49Ve1aPzQrUfoKytrY0WLVogLCxM2aZQKBAWFgZfX181VkZERERVQbXv2QGAcePGYeDAgfDy8kLLli2xcOFC5OTkYPDgweoujYiIiNRMFGGnb9++ePToEaZOnYrk5GR4eHjg4MGDxQYtVzYdHR1Mmzat2C0zUg/+PKoe/kyqFv48qhb+PMqPRHjdfC0iIiKiaqzaj9khIiIiehWGHSIiIhI1hh0iIiISNYYdIiIiEjWGnQr0yy+/wN7eHrq6uvDx8cHZs2fVXVKNFBwcDG9vbxgYGMDc3Bw9evRAXFycusui50JCQiCRSDBmzBh1l1JjPXjwAJ9++ilMTU2hp6eHZs2a4dy5c+ouq8aSy+UICgqCg4MD9PT00LBhQ8yYMeO1+z9RyRh2Ksi2bdswbtw4TJs2DTExMXB3d0fnzp2Rmpqq7tJqnPDwcAQEBODMmTMIDQ1FYWEhOnXqhJycHHWXVuNFRUVh5cqVcHNzU3cpNVZGRgbatGkDLS0tHDhwAFevXsX8+fNhbGys7tJqrDlz5mD58uVYunQprl27hjlz5mDu3LlYsmSJukurtjj1vIL4+PjA29sbS5cuBVC0qrONjQ1GjhyJSZMmqbm6mu3Ro0cwNzdHeHg42rVrp+5yaqzs7Gw0b94cy5Ytw8yZM+Hh4YGFCxequ6waZ9KkSTh16hROnDih7lLouQ8++AAWFhZYs2aNsq13797Q09PDb7/9psbKqi/27FSAgoICREdHw9/fX9mmoaEBf39/REREqLEyAgCZTAYAMDExUXMlNVtAQAC6deum8v+EKt+ePXvg5eWFTz75BObm5vD09MTq1avVXVaN1rp1a4SFheHGjRsAgAsXLuDkyZPo2rWrmiurvkSxgnJV8/jxY8jl8mIrOFtYWOD69etqqoqAoh62MWPGoE2bNnB1dVV3OTXW1q1bERMTg6ioKHWXUuPFx8dj+fLlGDduHCZPnoyoqCiMGjUK2traGDhwoLrLq5EmTZqErKwsODs7QyqVQi6XY9asWRgwYIC6S6u2GHaoRgkICMDly5dx8uRJdZdSYyUmJmL06NEIDQ2Frq6uusup8RQKBby8vDB79mwAgKenJy5fvowVK1Yw7KjJ9u3bsWnTJmzevBlNmzZFbGwsxowZA2tra/5M3hDDTgUwMzODVCpFSkqKSntKSgosLS3VVBWNGDEC+/btw/Hjx1G/fn11l1NjRUdHIzU1Fc2bN1e2yeVyHD9+HEuXLkV+fj6kUqkaK6xZrKys4OLiotLWpEkT/PHHH2qqiCZMmIBJkyahX79+AIBmzZrh3r17CA4OZth5QxyzUwG0tbXRokULhIWFKdsUCgXCwsLg6+urxspqJkEQMGLECOzcuRNHjhyBg4ODukuq0fz8/HDp0iXExsYqH15eXhgwYABiY2MZdCpZmzZtii3FcOPGDdjZ2ampIsrNzYWGhurbs1QqhUKhUFNF1R97dirIuHHjMHDgQHh5eaFly5ZYuHAhcnJyMHjwYHWXVuMEBARg8+bN2L17NwwMDJCcnAwAMDQ0hJ6enpqrq3kMDAyKjZfS19eHqakpx1GpwdixY9G6dWvMnj0bffr0wdmzZ7Fq1SqsWrVK3aXVWB9++CFmzZoFW1tbNG3aFOfPn8eCBQswZMgQdZdWbXHqeQVaunQpfvrpJyQnJ8PDwwOLFy+Gj4+PusuqcSQSyUvb161bh0GDBlVuMfRS7733Hqeeq9G+ffsQGBiImzdvwsHBAePGjcOwYcPUXVaN9eTJEwQFBWHnzp1ITU2FtbU1+vfvj6lTp0JbW1vd5VVLDDtEREQkahyzQ0RERKLGsENERESixrBDREREosawQ0RERKLGsENERESixrBDREREosawQ0RERKLGsENE9B/Hjh2DRCJBZmamukshonLAsENERESixrBDREREosawQ0RVjkKhQHBwMBwcHKCnpwd3d3f8/vvvAP65xbR//364ublBV1cXrVq1wuXLl1XO8ccff6Bp06bQ0dGBvb095s+fr/J8fn4+Jk6cCBsbG+jo6MDR0RFr1qxROSY6OhpeXl6oVasWWrduXWx3cCKqHhh2iKjKCQ4OxsaNG7FixQpcuXIFY8eOxaefforw8HDlMRMmTMD8+fMRFRWFunXr4sMPP0RhYSGAopDSp08f9OvXD5cuXcL06dMRFBSE9evXK1//+eefY8uWLVi8eDGuXbuGlStXonbt2ip1fP/995g/fz7OnTsHTU1N7jpNVE1xI1AiqlLy8/NhYmKCv//+G76+vsr2L774Arm5ufjyyy/Rvn17bN26FX379gUApKeno379+li/fj369OmDAQMG4NGjRzh8+LDy9d999x3279+PK1eu4MaNG3ByckJoaCj8/f2L1XDs2DG0b98ef//9N/z8/AAAf/31F7p164anT59CV1e3gv8WiKg8sWeHiKqUW7duITc3Fx07dkTt2rWVj40bN+L27dvK4/4dhExMTODk5IRr164BAK5du4Y2bdqonLdNmza4efMm5HI5YmNjIZVK8e67776yFjc3N+WfraysAACpqalvfY1EVLk01V0AEdG/ZWdnAwD279+PevXqqTyno6OjEnjelJ6eXqmO09LSUv5ZIpEAKBpPRETVC3t2iKhKcXFxgY6ODhISEuDo6KjysLGxUR535swZ5Z8zMjJw48YNNGnSBADQpEkTnDp1SuW8p06dQuPGjSGVStGsWTMoFAqVMUBEJF7s2SGiKsXAwADffvstxo4dC4VCgbZt20Imk+HUqVOoU6cO7OzsAAA//vgjTE1NYWFhge+//x5mZmbo0aMHAGD8+PHw9vbGjBkz0LdvX0RERGDp0qVYtmwZAMDe3h4DBw7EkCFDsHjxYri7u+PevXtITU1Fnz591HXpRFRBGHaIqMqZMWMG6tati+DgYMTHx8PIyAjNmzfH5MmTlbeRQkJCMHr0aNy8eRMeHh7Yu3cvtLW1AQDNmzfH9u3bMXXqVMyYMQNWVlb48ccfMWjQIOX3WL58OSZPnoxvvvkGaWlpsLW1xeTJk9VxuURUwTgbi4iqlRczpTIyMmBkZKTucoioGuCYHSIiIhI1hh0iIiISNd7GIiIiIlFjzw4RERGJGsMOERERiRrDDhEREYkaww4RERGJGsMOERERiRrDDhEREYkaww4RERGJGsMOERERiRrDDhEREYna/wObtA88RS/mpQAAAABJRU5ErkJggg==", "text/plain": [ "
" ] @@ -479,7 +768,7 @@ "plt.plot(np.arange(len(epochs_x)), epochs_acc, marker = '.')\n", "plt.xlabel('epoch')\n", "plt.ylabel('test accuracy')\n", - "plt.ylim(80, 100)\n", + "plt.ylim(0, 100)\n", "for i, txt in enumerate(epochs_acc):\n", " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom')\n", "plt.show()" diff --git a/tests/test_nonsequential/non-sequential-SCNN-example_3.ipynb b/tests/test_nonsequential/non-sequential-SCNN-example_3.ipynb new file mode 100644 index 00000000..f3ac5d89 --- /dev/null +++ b/tests/test_nonsequential/non-sequential-SCNN-example_3.ipynb @@ -0,0 +1,808 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "import torch.nn as nn\n", + "import sinabs.layers as sl\n", + "from tqdm.notebook import tqdm\n", + "\n", + "from tonic.datasets.dvsgesture import DVSGesture\n", + "from tonic.transforms import ToFrame\n", + "from torch.utils.data import DataLoader\n", + "from torch.nn import CrossEntropyLoss\n", + "from torch.optim import Adam\n", + "\n", + "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "torch.manual_seed(0)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Loading Data" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "batch_size = 3\n", + "num_workers = 1\n", + "epochs = 10\n", + "lr = 1e-3" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "root_dir = \"./DVSGESTURE\"\n", + "_ = DVSGesture(save_to=root_dir, train=True)\n", + "_ = DVSGesture(save_to=root_dir, train=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "n_time_steps = 50\n", + "to_raster = ToFrame(sensor_size=DVSGesture.sensor_size, n_time_bins=n_time_steps)\n", + "\n", + "snn_train_dataset = DVSGesture(save_to=root_dir, train=True, transform=to_raster)\n", + "snn_test_dataset = DVSGesture(save_to=root_dir, train=False, transform=to_raster)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The transformed array is in shape [Time-Step, Channel, Height, Width] --> (50, 2, 128, 128)\n" + ] + } + ], + "source": [ + "sample_data, label = snn_train_dataset[0]\n", + "print(f\"The transformed array is in shape [Time-Step, Channel, Height, Width] --> {sample_data.shape}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "snn_train_dataloader = DataLoader(snn_train_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True)\n", + "snn_test_dataloader = DataLoader(snn_test_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=False)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Network Module\n", + "\n", + "We need to define a `nn.Module` implementing the network we want the chip to reproduce." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "device: NVIDIA RTX A4000\n" + ] + } + ], + "source": [ + "if torch.cuda.is_available():\n", + " device = torch.device('cuda:0')\n", + " print('device: ', torch.cuda.get_device_name(0))\n", + "else:\n", + " device = torch.device('cpu')" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "class SNN(nn.Module):\n", + " def __init__(self) -> None:\n", + " super().__init__()\n", + "\n", + " self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False)\n", + " self.iaf1 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + " self.pool1 = nn.AvgPool2d(2,2)\n", + " self.pool1a = nn.AvgPool2d(6,6)\n", + "\n", + " self.conv2 = nn.Conv2d(10, 10, 2, 1, bias=False)\n", + " self.iaf2 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + " self.pool2 = nn.AvgPool2d(3,3)\n", + "\n", + " self.conv3 = nn.Conv2d(10, 10, 3, 1, bias=False)\n", + " self.iaf3 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + " self.pool3 = nn.AvgPool2d(2,2)\n", + "\n", + " self.flat = nn.Flatten()\n", + "\n", + " self.fc1 = nn.Linear(810, 100, bias=False)\n", + " self.iaf4 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + "\n", + " self.fc2 = nn.Linear(100, 100, bias=False)\n", + " self.iaf5 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + "\n", + " self.fc3 = nn.Linear(100, 100, bias=False)\n", + " self.iaf6 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + "\n", + " self.fc4 = nn.Linear(100, 11, bias=False)\n", + " self.iaf7 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + "\n", + " self.merge_fc = sl.Merge()\n", + " self.merge_conv = sl.Merge()\n", + "\n", + " def detach_neuron_states(self):\n", + " for name, layer in self.named_modules():\n", + " if name != '':\n", + " if isinstance(layer, sl.StatefulLayer):\n", + " for name, buffer in layer.named_buffers():\n", + " buffer.detach_()\n", + "\n", + " def init_weights(self):\n", + " for name, layer in self.named_modules():\n", + " if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear):\n", + " nn.init.xavier_normal_(layer.weight.data)\n", + "\n", + " def forward(self, x):\n", + " \n", + " con1_out = self.conv1(x)\n", + " iaf1_out = self.iaf1(con1_out)\n", + " pool1_out = self.pool1(iaf1_out)\n", + " pool1a_out = self.pool1a(iaf1_out)\n", + "\n", + " conv2_out = self.conv2(pool1_out)\n", + " iaf2_out = self.iaf2(conv2_out)\n", + " pool2_out = self.pool2(iaf2_out)\n", + "\n", + " merged_conv_out = self.merge_conv(pool1a_out, pool2_out)\n", + "\n", + " conv3_out = self.conv3(merged_conv_out)\n", + " iaf3_out = self.iaf3(conv3_out)\n", + " pool3_out = self.pool3(iaf3_out)\n", + "\n", + " flat_out = self.flat(pool3_out)\n", + " \n", + " fc1_out = self.fc1(flat_out)\n", + " iaf4_out = self.iaf4(fc1_out)\n", + "\n", + " fc2_out = self.fc2(iaf4_out)\n", + " iaf5_out = self.iaf5(fc2_out)\n", + "\n", + " fc3_out = self.fc3(iaf5_out)\n", + " iaf6_out = self.iaf6(fc3_out)\n", + "\n", + " merge_fc_out = self.merge_fc(iaf4_out, iaf6_out)\n", + "\n", + " fc4_out = self.fc4(merge_fc_out)\n", + " iaf7_out = self.iaf7(fc4_out)\n", + "\n", + " return iaf7_out" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "snn = SNN().to(device)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "snn.init_weights()" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "optimizer = Adam(snn.parameters(), lr=lr, betas=(0.9, 0.999), eps=1e-8)\n", + "loss_fn = CrossEntropyLoss()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Define train and test" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "def train(dataloader, model, loss_fn, optimizer, epochs, test_func, dataloader_test):\n", + " epochs_y = []\n", + " epochs_x = []\n", + " epochs_acc = []\n", + " model.train()\n", + "\n", + " for e in range(epochs):\n", + " losses = []\n", + " batches = []\n", + " batch_count = 0\n", + " train_p_bar = tqdm(snn_train_dataloader)\n", + "\n", + " for X, y in train_p_bar:\n", + " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", + " X = X.reshape(-1, 2, 128, 128).to(dtype=torch.float, device=device)\n", + " y = y.to(dtype=torch.long, device=device)\n", + "\n", + " # forward\n", + " pred = model(X)\n", + "\n", + " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", + " pred = pred.reshape(batch_size, n_time_steps, -1)\n", + "\n", + " # accumulate all time-steps output for final prediction\n", + " pred = pred.sum(dim = 1)\n", + " loss = loss_fn(pred, y)\n", + "\n", + " # gradient update\n", + " optimizer.zero_grad()\n", + " loss.backward()\n", + " optimizer.step()\n", + "\n", + " # detach the neuron states and activations from current computation graph(necessary)\n", + " model.detach_neuron_states()\n", + "\n", + " train_p_bar.set_description(f\"Epoch {e} - BPTT Training Loss: {round(loss.item(), 4)}\")\n", + "\n", + " batch_count += 1\n", + " losses.append(loss.item())\n", + " batches.append(batch_count)\n", + "\n", + " epochs_y.append(losses)\n", + " epochs_x.append(batches)\n", + "\n", + " acc = test_func(dataloader_test, model)\n", + " print(f'Epoch {e} accuracy: {acc}')\n", + " epochs_acc.append(acc)\n", + "\n", + " return epochs_x, epochs_y, epochs_acc\n" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "def test(dataloader, model):\n", + " correct_predictions = []\n", + " with torch.no_grad():\n", + " test_p_bar = tqdm(dataloader)\n", + " for X, y in test_p_bar:\n", + " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", + " X = X.reshape(-1, 2, 128, 128).to(dtype=torch.float, device=device)\n", + " y = y.to(dtype=torch.long, device=device)\n", + "\n", + " # forward\n", + " output = model(X)\n", + "\n", + " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", + " output = output.reshape(batch_size, n_time_steps, -1)\n", + "\n", + " # accumulate all time-steps output for final prediction\n", + " output = output.sum(dim=1)\n", + "\n", + " # calculate accuracy\n", + " pred = output.argmax(dim=1, keepdim=True)\n", + "\n", + " # compute the total correct predictions\n", + " correct_predictions.append(pred.eq(y.view_as(pred)))\n", + "\n", + " test_p_bar.set_description(f\"Testing Model...\")\n", + " \n", + " correct_predictions = torch.cat(correct_predictions)\n", + " return correct_predictions.sum().item()/(len(correct_predictions))*100" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Training loop (HPO)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "43cb492ef2204104a2f1048eb2ae3a85", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/359 [00:00" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "y_avg = []\n", + "for y in epochs_y:\n", + " y_avg.append(np.mean(y))\n", + "\n", + "plt.plot(np.arange(len(epochs_x)), y_avg, marker = '.')\n", + "plt.xlabel('epoch')\n", + "plt.ylabel('average loss')\n", + "plt.ylim(0,)\n", + "for i, txt in enumerate(y_avg):\n", + " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom')\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAG2CAYAAACZEEfAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABYNklEQVR4nO3deZyNdf/H8deZfTGLGWMWZrMv2WkspWSyJGWJuCVKVEiIUGkjgzZJkX5CdxQKdbeQpYjsS5FsY2cWZsyMmTGLOef3x3BymsEMM87Mmffz8TiPmfO9rvM9n8vpvs97vtf3ur4Gk8lkQkRERMRG2Vm7ABEREZHipLAjIiIiNk1hR0RERGyawo6IiIjYNIUdERERsWkKOyIiImLTFHZERETEpinsiIiIiE1T2BERERGbprAjIiIiNs2qYWf9+vV07tyZoKAgDAYDy5cvt9huMpl49dVXCQwMxNXVlcjISA4dOmSxT2JiIn369MHT0xNvb28GDBhAamrqbTwKERERKcmsGnbS0tJo0KABH330Ub7bp06dyvTp05k1axZbtmzB3d2d9u3bk5GRYd6nT58+/PXXX6xatYrvv/+e9evXM2jQoNt1CCIiIlLCGUrKQqAGg4Fly5bRpUsXIHdUJygoiBdeeIFRo0YBkJycjL+/P/PmzaNXr178/fff1KlTh23bttG0aVMAVqxYwQMPPMCpU6cICgqy1uGIiIhICeFg7QKu5ejRo8TGxhIZGWlu8/LyIiIigk2bNtGrVy82bdqEt7e3OegAREZGYmdnx5YtW+jatWu+fWdmZpKZmWl+bjQaSUxMxNfXF4PBUHwHJSIiIkXGZDJx4cIFgoKCsLO79smqEht2YmNjAfD397do9/f3N2+LjY2lYsWKFtsdHBzw8fEx75OfqKgo3njjjSKuWERERKzh5MmTVK5c+ZrbS2zYKU7jxo1j5MiR5ufJycmEhIRw8uRJPD09rViZiIiIFFRKSgrBwcF4eHhcd78SG3YCAgIAiIuLIzAw0NweFxdHw4YNzfvEx8dbvO7SpUskJiaaX58fZ2dnnJ2d87R7enoq7IiIiJQyN5qCUmLvsxMeHk5AQABr1qwxt6WkpLBlyxZatGgBQIsWLUhKSmLHjh3mfdauXYvRaCQiIuK21ywiIiIlj1VHdlJTUzl8+LD5+dGjR9m9ezc+Pj6EhIQwfPhwJk6cSPXq1QkPD2f8+PEEBQWZr9iqXbs2HTp0YODAgcyaNYvs7GyGDh1Kr169dCWWiIiIAFYOO9u3b6dNmzbm51fm0fTr14958+bx4osvkpaWxqBBg0hKSuKuu+5ixYoVuLi4mF+zYMEChg4dStu2bbGzs6N79+5Mnz79th+LiIiIlEwl5j471pSSkoKXlxfJycmasyMiIlJKFPT7u8TO2REREREpCgo7IiIiYtMUdkRERMSmKeyIiIiITVPYEREREZumsCMiIiI2TWFHREREbJrCjoiIiNg0hR0RERGxaQo7IiIiYtMUdkRERMSmKeyIiIiITVPYEREREZumsCMiIiI2TWFHREREbJrCjoiIiNg0hR0RERGxaQo7IiIiYtMUdkRERMSmKeyIiIiITVPYEREREZumsCMiIiI2TWFHREREbJrCjoiIiNg0hR0RERGxaQo7IiIiYtMUdkRERMSmKeyIiIiITVPYEREREZumsCMiIiI2TWFHREREbJrCjoiIiNg0hR0RERGxaQo7IiIiYtMUdkRERMSmKeyIiIiITVPYEREREZumsCMiIiI2TWFHREREbJrCjoiIiNg0hR0RERGxaQo7IiIiYtMUdkRERMSmKeyIiIiITVPYEREREZumsCMiIiI2TWFHREREbJrCjoiIiNg0hR0RERGxaQo7IiIiYtMUdkRERMSmKeyIiIiITVPYEREREZumsCMiIiI2TWFHREREbJrCjoiIiNg0hR0RERGxaQo7IiIiYtMUdkRERMSmKeyIiIiITVPYEREREZumsCMiIiI2TWFHREREbFqJDjs5OTmMHz+e8PBwXF1dqVq1KhMmTMBkMpn3MZlMvPrqqwQGBuLq6kpkZCSHDh2yYtUiIiJSkpTosDNlyhRmzpzJjBkz+Pvvv5kyZQpTp07lww8/NO8zdepUpk+fzqxZs9iyZQvu7u60b9+ejIwMK1YuIiIiJYXBdPUwSQnz4IMP4u/vz5w5c8xt3bt3x9XVlS+++AKTyURQUBAvvPACo0aNAiA5ORl/f3/mzZtHr169CvQ+KSkpeHl5kZycjKenZ7Eci4iIiBStgn5/l+iRnZYtW7JmzRoOHjwIwB9//MGGDRvo2LEjAEePHiU2NpbIyEjza7y8vIiIiGDTpk3X7DczM5OUlBSLh4iIiNgmB2sXcD1jx44lJSWFWrVqYW9vT05ODm+99RZ9+vQBIDY2FgB/f3+L1/n7+5u35ScqKoo33nij+AoXERGREqNEj+wsXryYBQsWsHDhQnbu3Mn8+fN55513mD9//i31O27cOJKTk82PkydPFlHFIiIiUtKU6LAzevRoxo4dS69evahXrx59+/ZlxIgRREVFARAQEABAXFycxevi4uLM2/Lj7OyMp6enxUNERET+ERYWhsFgyPMYMmSIeZ9NmzZx33334e7ujqenJ61bt+bixYvX7HP9+vV07tyZoKAgDAYDy5cvv24NzzzzDAaDgWnTpt3SsZTosJOeno6dnWWJ9vb2GI1GAMLDwwkICGDNmjXm7SkpKWzZsoUWLVrc1lpFRERsybZt24iJiTE/Vq1aBUCPHj2A3KDToUMH2rVrx9atW9m2bRtDhw7N8719tbS0NBo0aMBHH310w/dftmwZmzdvJigo6JaPpUTP2encuTNvvfUWISEh1K1bl127dvHee+/x5JNPAmAwGBg+fDgTJ06kevXqhIeHM378eIKCgujSpYt1ixcRESnF/Pz8LJ5PnjyZqlWrcs899wAwYsQIhg0bxtixY8371KxZ87p9duzY0XyR0fWcPn2a5557jpUrV9KpU6ebqN5SiR7Z+fDDD3nkkUcYPHgwtWvXZtSoUTz99NNMmDDBvM+LL77Ic889x6BBg2jWrBmpqamsWLECFxcXK1YuIiJiO7Kysvjiiy948sknMRgMxMfHs2XLFipWrEjLli3x9/fnnnvuYcOGDbf8Xkajkb59+zJ69Gjq1q1bBNWX8LDj4eHBtGnTOH78OBcvXiQ6OpqJEyfi5ORk3sdgMPDmm28SGxtLRkYGq1evpkaNGlasWkREyrIbzXW5995782x75plnrttnQVYLSExMpE+fPnh6euLt7c2AAQNITU0tkmNavnw5SUlJ9O/fH4AjR44A8PrrrzNw4EBWrFhB48aNadu27S2vYjBlyhQcHBwYNmzYrZZtVqLDjoiISGlzo7kuAAMHDrTYZ+rUqdftsyCrBfTp04e//vqLVatW8f3337N+/XoGDRpUJMc0Z84cOnbsaJ4/c2Xu7NNPP80TTzxBo0aNeP/996lZsyafffbZTb/Pjh07+OCDD5g3bx4Gg6FIaocSPmdHRESktLnRXBcANze36141fDWTycS0adN45ZVXePjhhwH4/PPP8ff3Z/ny5fTq1Yu///6bFStWsG3bNpo2bQrkTgV54IEHeOedd25pku/x48dZvXo1S5cuNbcFBgYCUKdOHYt9a9euzYkTJ276vX777Tfi4+MJCQkxt+Xk5PDCCy8wbdo0jh07dlP9amRHRESkmPx7rssVCxYsoEKFCtxxxx2MGzeO9PT0a/ZRkNUCNm3ahLe3tznoAERGRmJnZ8eWLVtu6Rjmzp1LxYoVLSYKh4WFERQUxIEDByz2PXjwIKGhoTf9Xn379uXPP/9k9+7d5kdQUBCjR49m5cqVN92vRnZERESKyb/nugD85z//ITQ0lKCgIP7880/GjBnDgQMHLEZOrlaQ1QJiY2OpWLGixXYHBwd8fHyuu6LAjRiNRubOnUu/fv1wcPgnMhgMBkaPHs1rr71GgwYNaNiwIfPnz2f//v18/fXX5v3atm1L165dGTp0KACpqakcPnzYvP3o0aPs3r0bHx8fQkJC8PX1xdfX16IGR0dHAgICbnil1/Uo7IiIiBSTf891ASzm0dSrV4/AwEDatm1LdHQ0VatWtUaZ17R69WpOnDhhvuXL1YYPH05GRgYjRowgMTGRBg0asGrVKotjiI6O5uipGH6PPkd4BXcO7NpOmzZtzNtHjhwJQL9+/Zg3b16xHYfCjoiISDHIb65LfiIiIgA4fPhwvmHn6tUCrsyVufK8YcOG5n3i4+MtXnfp0iUSExMLPDcoP+3atcNkMl1z+9ixYy3us/NvU5asZ9zSPXzz6RbsDBDVrd51+8vPzc7TuZrCjoiISDHIb65Lfnbv3g1gEWSudvVqAVfCzZXVAp599lkAWrRoQVJSEjt27KBJkyYArF27FqPRaA5TxeVSjpHz6dkkpGWSmJpFQloWCamZHE9IZ+7vx8z7GU3w0tK9tK7hR6CXa7HW9G8KOyIiIkXsWnNdoqOjWbhwIQ888AC+vr78+eefjBgxgtatW1O/fn3zfrVq1SIqKoquXbsWaLWA2rVr06FDBwYOHMisWbPIzs5m6NCh9OrVq9BXYuUYTSSlXwktWbkhJi2Lc6lZJKZlXm7LDTSJaVkkXcymoIM1OSYTx86lK+yIiIiUdtea6+Lk5MTq1auZNm0aaWlpBAcH0717d1555RWL/Q4cOEBycrL5+YsvvkhaWhqDBg0iKSmJu+66K89qAQsWLGDo0KG0bdsWOzs7unfvzvTp0zEaTSRdzCYxLfNyYMkNKlfCTG6QyQ0uiWlZJKZnFTi8XGEwgLerI77lnPFxd6JCOSecHexYvusMV3dlbzAQVsGtcJ0XAYOpsCfPbFBKSgpeXl4kJydrBXQRESkRYpIvcvRcGuEV3C1GQoxGEykZ2XmCS34h5sqojPEmvum93Rxzg4t7boDxLeeEr7vT5d+d8b3808fdifJujjjY572bzaJtJ3hp6V5yTCbsDQYmdbuDR5uF5PNuN6eg398a2RERESlBjEYT09ce4oPVh8yjItUquuNgZ2cONTk3kV48XRyocDmcWAYWp8ujMf+EmvJuTjjmE14K69FmIbSu4cexc+mEVXC77aevrlDYERERsSKTycTBuFQ2H0lg85EENkUnkHQx22Kfw/FpeV7n4exgDiqWwcWZClfa3Z3N4cXJwTr3EQ70crVayLlCYUdEREqEsLAwjh8/nqd98ODBfPTRR8yePZuFCxeyc+dOLly4wPnz5/H29r5un6+//jpvvPGGRVvNmjXZv3+/+Xl0dDSjRo1iw4YNZGZm0qFDBz788MM8N/ErKiaTicPxueFm05EEthxJJCEt64avG9exFq2qVTAHHGcH+2KpzxYp7IiISImwbds2cnJyzM/37t3L/fffb15AMz09nQ4dOtChQwfGjRtX4H7r1q3L6tWrzc+vvjoqLS2Ndu3a0aBBA9auXQvA+PHj6dy5M5s3b8bO7tZHQ0wmE9Fn09h0eeRmy5EEzqVahhsXRzuahvrQoqov1Su688wXOy3m2dgbDDzUMMjqIySllcKOiIiUCDdaQHP48OEA/Prrr4Xq18HB4Zo31tu4cSPHjh1j165d5gmu8+fPp3z58qxdu9ZiPaqCMplMHDmXZj4ltflIIudSMy32cXawo2lYeZqH+9K8qi8NKntbnGaK6lYvz8ReBZ2bp7AjIiIlzpUFNEeOHGmxgObNOHToEEFBQbi4uNCiRQuioqLMq2pnZmZiMBhwdnY27+/i4oKdnR0bNmwoUNgxmUwcS0i/HGxyH/EXLMONk4MdTULK06KqL82r+NIg2Ou6p6FKysReW6GwIyIiJU5+C2jejIiICObNm0fNmjWJiYnhjTfe4O6772bv3r14eHjQvHlz3N3dGTNmDJMmTcJkMjF27FhycnKIiYnJt0+TycTxhHTznJvNRxKIS8kbbhqHeNO8Sm64aRjsjYtj4ebYlISJvbZCYUdEREqc/BbQvBkdO3Y0/16/fn0iIiIIDQ1l8eLFDBgwAD8/P5YsWcKzzz7L9OnTsbOzo3fv3jRu3Ng8X8dkMnEy8SKbjpxj85FENh9JICY5w+J9nOztaBjiTYvL4aZRSOHDjRQfhR0RERtw+vRpxowZw08//UR6ejrVqlVj7ty5NG3aFMhdNHLMmDH8/PPPJCUl0bp1az788EOqV69+zT7vvfde1q1bl6f9gQce4IcffgByg8Brr73Gp59+SlJSEq1atWLmzJnX7fdGCrqA5s3w9vamRo0aHD582NzWrl07oqOjOXfuHA4ODnh7e1PR35/ad3Vg5KLdbD6SwJl/hRtHewONgsvTvIoPzav40ji0vMJNCaawIyJSyp0/f55WrVrRpk0bfvrpJ/z8/Dh06BDly5cHcgNJly5dcHR05Ntvv8XT05P33nuPyMhI9u3bh7u7e779Ll26lKysf64aSkhIoEGDBuarowCmTp3K9OnTmT9/vnnNpvbt27Nv3z6LpQwKo6ALaN6M1NRUoqOj6du3r0X7qfPpbDp2kc1HElm5ag5n48/yS3oIjrtOA7nhpkFlb/Ocm8Yh5XF1UrgpLRR2RERKuSlTphAcHMzcuXPNbeHh4ebfDx06xObNm9m7dy9169YFYObMmQQEBPDll1/y1FNP5duvj4+PxfOvvvoKNzc3c9gxmUxMmzaNV155hYcffhiAzz//HH9/f5YvX06vXr0KfSzXWkATIDY2ltjYWPOozJ49e/Dw8CAkJMRca9u2benatStDhw4FYNSoUXTu3JnQ0FDOnDnDa6+9hr29Pfc+0IVvdpxi05EEvl28gFQXf+zcvMg8s5/zq2fjdWcXmjeuZx65aRJaHjcnfWWWVvrkRERKue+++4727dvTo0cP1q1bR6VKlRg8eDADBw4Ecq84AixGWuzs7HB2dmbDhg3XDDv/NmfOHHr16mUeCTp69CixsbEWVyx5eXkRERHBpk2bbirsXGsBTYBZs2ZZ3CCwdevWQO5I0JWJzNHR0Rw9FcPv0ecIr+DOqVOn6N27NwkJCZTz9qFitQaEPfk+D/3fHnM/508eJW3vpxgzUvENqMQzw0cx5bVxlHNxLHT9UjJpIVC0EKiIlG5XQszIkSPp0aMH27Zt4/nnn2fWrFn069eP7OxsqlWrRkREBJ988gnu7u68//77jB07lnbt2rFy5cobvsfWrVuJiIhgy5Yt3HnnnQD8/vvvtGrVijNnzhAYGGjet2fPnhgMBhYtWlQ8B3wdi7adYNzSPRhNYACahpYnPjWT4wnpFvvZ2xmoV8nr8tVSPjQN86Gcs/7+L220EKiISBlhNBpp2rQpkyZNAqBRo0bs3bvXHHYcHR1ZunQpAwYMwMfHB3t7eyIjI+nYsSMF/Xt3zpw51KtXzxx0SoL0rEucSEzn2Ll0jieksS8mhW93nzFvNwHbjp8HwM5Abri5POemaWh5PDRyU2Yo7IiIlHKBgYHUqVPHoq127dp888035udNmjRh9+7dJCcnk5WVhZ+fHxEREearta4nLS2Nr776ijfffNOi/cpdiePi4ixGduLi4mjYsOEtHNE/ki9mcyIhnWMJaRxPSON4QjrHLz//9437ruXF9jXo2yJM4aYMU9gRESnlWrVqxYEDByzaDh48SGhoaJ59vby8gNxJy9u3b2fChAk37H/JkiVkZmby2GOPWbSHh4cTEBDAmjVrzOEmJSWFLVu28OyzzxaodpPJRGJaFscS0q8KM2nm5+fTs6/7em83R0J93Qn1ccPX3Yl5vx/j6rEqe4OBro0rK+iUcQo7IiKl3IgRI2jZsiWTJk2iZ8+ebN26ldmzZzN79mzzPkuWLMHPz4+QkBD27NnD888/T5cuXWjXrp15n8cff5xKlSoRFRVl0f+cOXPo0qULvr6+Fu0Gg4Hhw4czceJEqlevbr70PCgoiC5dupj3MxpNxF/INIeZY1dCTWIax8+lcyHz0nWPr0I5Z8J83Qj1dSfM140QXzfCfN0J9XXD283JYt9agR5aU0ryUNgRESnlmjVrxrJlyxg3bhxvvvkm4eHhTJs2jT59+pj3iYmJYeTIkeZTTo8//jjjx4+36OfEiRN5Vvk+cOAAGzZs4Oeff873vV988UXS0tIYNGgQSUlJ3NE4gmcnz+HdNUfMp5yOJ6aRkW287jEEebnkjtBcFWpCfd0J8XUr1MRhrSkl+dHVWOhqLBERgJjkixw9l0Z4Bfd8Q0LWJSOnzqfnOdV0PCGdk+fTyc659teJvZ2BSt6uhF41KnPlZ7CPm+4+LDdFV2OJiEiBXX3Jtp0BHosIJdjHzeKU0+nzFzFe589jJ3s7gn1cCbs8InN1qKlU3hVHe7trv1ikGCnsiIiUcZuPJDD2mz3mib1GE3y++Xi++7o62l8+1XQlzLibnwd6uWJvZ7h9hYsUkMKOiEgZlJCayf/+OMOyXaf541Ryvvs0r+JDk9Dyl+fQ5M6j8fNwxmBQoJHSRWFHRKSMyMjOYfXfcSzbeZp1B89y6fI5KTsDeU5P2RsMvP9oQ03wFZugsCMiYsOMRhNbjyWybOdpftwTY3GZd71KXnRtVInODYJYuz9Ol2yLzVLYERGxQYfjU1m26xTLd53hdNJFc3slb1cebhhEt8aVqFbRw9yuS7bFlinsiIjYiHNXzcP586p5OB7ODjxQL5AujSoREe6D3TUmEQd6uSrkiE1S2BERKcUysnNYtS+OZbty5+HkXJ5842Bn4J4afnRtXInI2v66j42UabrpgYiUaadPn+axxx7D19cXV1dX6tWrx/bt2/Pd95lnnsFgMDBt2rTr9hkWFobBYMjzGDJkCADHjh3Ld7vBYGDJkiU3rNloNLEpOoEXv/6DZhNX89yXu1i7P54co4n6lb14vXMdtrzUljn9m/Fg/SAFHSnzNLIjImXW+fPnadWqFW3atOGnn37Cz8+PQ4cOUb58+Tz7Llu2jM2bNxMUFHTDfrdt20ZOTo75+d69e7n//vvp0aMHAMHBwcTExFi8Zvbs2bz99tt07Njxmv0ejr/A0p2n+XZ33nk4XRoF0bVRZapVLHfD+kTKGoUdESmzpkyZQnBwMHPnzjW3hYeH59nv9OnTPPfcc6xcuZJOnTrdsF8/Pz+L55MnT6Zq1arcc889ANjb2xMQEGCxz7Jly+jZsyflylmGlXOpmXy3O3cezp7TV83DcXGgU71AujaqRLOwa8/DERGFHREpw7777jvat29Pjx49WLduHZUqVWLw4MEMHDjQvI/RaKRv376MHj2aunXrFvo9srKy+OKLLxg5cuQ1b8a3Y8cOdu/ezUcffQTkzsP5eV8cy3aeYv2hcxbzcO6t6UfXRpVpW7uiTk+JFJDCjoiUWUeOHGHmzJmMHDmSl156iW3btjFs2DCcnJzo168fkDv64+DgwLBhw27qPZYvX05SUhL9+/e/5j5z5syhdu3a4F+D0Uv+4Ke9saRedT+cBsHedGtUiQfrB+Jbzvmm6hApyxR2RKTMMhqNNG3alEmTJgHQqFEj9u7dy6xZs+jXrx87duzggw8+YOfOnTe9RMKcOXPo2LHjNef67DkWz9zPv8C/9X/4z6dbzO2Vy7vStVElujSqRFU/zcMRuRUKOyJSZgUGBlKnTh2Lttq1a/PNN98A8NtvvxEfH09ISIh5e05ODi+88ALTpk3j2LFj1+3/+PHjrF69mqVLl1q0n72QyXd/nGHZrlNsXrmcjIsXyanWGm8XBx6sH0jXRpVpGlpe83BEiojCjoiUWa1ateLAgQMWbQcPHiQ0NBSAvn37EhkZabG9ffv29O3blyeeeOKG/c+dO5eKFSvSqVMnLmbl8PO+WJbtOs1vV83DSduzirDGrfloYBvuq6V5OCLFQWFHRMqsESNG0LJlSyZNmkTPnj3ZunUrs2fPZvbs2QD4+vri6+tr8RpHR0cCAgKoWbOmua1t27Z07dqVoUOHmtuMRiNz584l8qEejF32Fyv+NQ+nYbA3rSpmM2bqXmZ+8iMd6gUW89GKlF0KOyJSZjVr1oxly5Yxbtw43nzzTcLDw5k2bRp9+vQpVD8HDx1m18ETxCRfJNDLlYNxF5j8f4s5ceIEa0134LjjFADBPq50bZg7D6eKXzleeuklKleuTLt27Yrj8ETkMoPJZDJZuwhrS0lJwcvLi+TkZDw9Pa1djoiUIou2nWDc0j0YTWAAAr1cOJOcYd7u6eJAp/q5C282DS1/0xOdRSSvgn5/a2RHROQmHT2bxthv9nDlL0YTcCY5Awc7aFPLn26NKtFG83BErE5hR0SkEHKMJjYfSWDpztP88OcZ8hsan9mnCffXDchni4hYg8KOiEgB7I9NYdnldaliUzKuuZ+9wcAdlb1uY2UiciMKOyIi1xCfksF3f5xh6c7T7ItJMbd7uTrSqX4g3RpV4nB8Ki8v20uOyYS9wcCkbncQ6OVqxapF5N8UdkRErpKedYmf/4pj6a7TbDh0lsu3w8HR3kCbmhXp1jh3Ho6zQ+48nKZhPtxT049j59IJq+CmoCNSAinsiEiZl2M08Xv0OZbtOs2KvbGkZ+WYtzUO8aZr48o8WC+Q8u5O+b4+0MtVIUekBFPYEZEy6++YFJbtOs23u08Tl5Jpbg/1daNLw0p0bVSJsAruVqxQRIqCnbULEJHS5/XXX8dgMFg8atWqZd4eHR1N165d8fPzw9PTk549exIXF3fdPnNychg/fjzh4eG4urpStWpVJkyYwNW3Auvfv3+e9+3QoUOhao9PyeDT9UfoMG09HT/4jdnrjxCXkomXqyN9IkL45tkW/DrqXkbcX0NBR8RGaGRHRG5K3bp1Wb16tfm5g0Pu/52kpaXRrl07GjRowNq1awEYP348nTt3ZvPmzdjZ5f831pQpU5g5cybz58+nbt26bN++nSeeeAIvLy+GDRtm3q9Dhw7MnTvX/NzZ2fmGtaZlXuLnfbEs3XmajYfPWczDua9WRbo2qkybWn7meTgiYlsUdkTkpjg4OBAQkPdeMhs3buTYsWPs2rXLfEfT+fPnU758edauXZtnYc0rfv/9dx5++GE6deoEQFhYGF9++SVbt2612M/Z2Tnf9/23HKOJjYdz5+Gs/MtyHk6T0PJ0bVSJB+sH4u2W/zwcEbEdhT6N9csvvxRHHSJSyhw6dIigoCCqVKlCnz59OHHiBACZmZkYDAaLERcXFxfs7OzYsGHDNftr2bIla9as4eDBgwD88ccfbNiwgY4dO1rs9+uvv1KxYkVq1qzJs88+S0JCgsX2fWdSeOuHfbSIWsPjn21l2a7TpGflEOrrxvDI6qwbfS/fPNuSx5qHKuiIlBGFHtnp0KEDlStX5oknnqBfv34EBwcXR10iUoJFREQwb948atasSUxMDG+88QZ33303e/fupXnz5ri7uzNmzBgmTZqEyWRi7Nix5OTkEBMTc80+x44dS0pKCrVq1cLe3p6cnBzeeusti0U5O3ToQLdu3QgPDyc6OpqXXnqJjh07snzlL/zvz1iW7TrN/tgL5v293Rx5sH4gXRtVpnGIt9alEimjCh12Tp8+zX//+1/mz5/PG2+8wX333ceAAQPo0qULTk76K0mkLLh6tKV+/fpEREQQGhrK4sWLGTBgAEuWLOHZZ59l+vTp2NnZ0bt3bxo3bnzN+ToAixcvZsGCBSxcuJC6deuye/duhg8fTlBQEP369QOgV69e5v2r1KjNGYMfQ7rcRePB03AJbQiAk71d7jycxpVoU7MiTg66DkOkrCt02KlQoQIjRoxgxIgR7Ny5k7lz5zJ48GAGDx7Mf/7zHwYMGECDBg2Ko1YRKaG8vb2pUaMGhw8fBqBdu3ZER0dz7tw5HBwc8Pb2JiAggCpVqlyzj9GjRzN27FhzoKlXrx7Hjx8nKirKHHZyjCY2HD7Hsp2nWPlXHBezc7Bz9ST7fAx3tW5D18aVeLBeEF5ujsV/0CJSatzSBOXGjRsTEBCAr68vkydP5rPPPuPjjz+mRYsWzJo1i7p16xZVnSJSgqWmphIdHU3fvn0t2itUqADA2rVriY+P56GHHrpmH+np6XlGfuzt7TEajfx1Jjl3Xao/znD2wj/3wwm0T+NExgWmPnYPAx5rWYRHJCK25KbGd7Ozs/n666954IEHCA0NZeXKlcyYMYO4uDgOHz5MaGgoPXr0KJICT58+zWOPPYavry+urq7Uq1eP7du3m7ebTCZeffVVAgMDcXV1JTIykkOHDhXJe4tI/kaNGsW6des4duwYv//+O127dsXe3p7evXsDMHfuXDZv3kx0dDRffPEFPXr0YMSIEdSsWdPcR9u2bZkxY4b5eefOnXnrrbf44YcfOHbsGJ99sYhJU94mu3JTOk3fwOy1+zj43UwcEw7TuYoDY+tnk7ViCtWqVeOxHtcOUSIihR7Zee655/jyyy8xmUz07duXqVOncscdd5i3u7u788477xAUFHTLxZ0/f55WrVrRpk0bfvrpJ/z8/Dh06BDly5c37zN16lSmT5/O/PnzCQ8PZ/z48bRv3559+/bh4uJyyzWISF6nTp2id+/eJCQk4Ofnx1133cXmzZvx8/MD4MCBA4wbN47ExETCwsJ4+eWXGTFihEUfV05zXfHhhx8y9qWX6f/U05xPOIvB3Qe3Ou0wNumJs70d99wRyK4tCZz49i0+mZ9EUFAQ7dq1Y8KECQW6146IlF0G09W3Jy2Atm3b8tRTT9GtW7dr/h/MpUuX2LhxI/fcc88tFTd27Fg2btzIb7/9lu92k8lEUFAQL7zwAqNGjQIgOTkZf39/5s2bZzGZ8XpSUlLw8vIiOTnZfF8QESl+MckXORyfytkLmaw7eJafL8/DuaJZWHm6NqpMp3qBmocjInkU9Pu70GHndqpTpw7t27fn1KlTrFu3jkqVKjF48GAGDhwIwJEjR6hatSq7du2iYcOG5tfdc889NGzYkA8++CDffjMzM8nM/Oe8f0pKCsHBwQo7IrfRrF8PM2XFAf79f0DhFdzp2ih3XapgHzer1CYipUNBw06h5+xERUXx2Wef5Wn/7LPPmDJlSmG7u64jR44wc+ZMqlevzsqVK3n22WcZNmwY8+fPByA2NhYAf39/i9f5+/ubt+UnKioKLy8v80P3ChK5fU4nXWT4V7uY/K+gYwA+fbwJa1+4h2FtqyvoiEiRKXTY+eSTTywW/Luibt26zJo1q0iKusJoNNK4cWMmTZpEo0aNGDRoEAMHDrzl9xk3bhzJycnmx8mTJ4uoYhG5ljNJF3ll+R7uffsXlu8+k2e7CSjn7Kgb/4lIkSv0BOXY2FgCAwPztPv5+V337qg3IzAwkDp16li01a5dm2+++QbAvD5OXFycRU1xcXEWp7X+zdnZWRMaRW6TmOSLfPxLNIu2nSQrxwhA4xBvdp1M4uqT6PYGA2EVNJojIkWv0CM7wcHBbNy4MU/7xo0bi+QKrKu1atWKAwcOWLQdPHiQ0NBQAMLDwwkICGDNmjXm7SkpKWzZsoUWLVoUaS0iUjixyRm89u1e7pn6K//dfJysHCMR4T58ObA5Swe3YnK3ethfHsWxNxiY1O0OAr1crVy1iNiiQo/sDBw4kOHDh5Odnc19990HwJo1a3jxxRd54YUXirS4ESNG0LJlSyZNmkTPnj3ZunUrs2fPZvbs2QAYDAaGDx/OxIkTqV69uvnS86CgILp06VKktYhIwcSlZDDz12gWbj1B1qXckZw7w3wYfn91WlatYN7v0WYhtK7hx7Fz6YRVcFPQEZFiU+iwM3r0aBISEhg8eDBZWVlA7orGY8aMYdy4cUVaXLNmzVi2bBnjxo3jzTffJDw8nGnTplksDPjiiy+SlpbGoEGDSEpK4q677mLFihW6x47IbRafksHMddEs3HKCzMshp1lYeUZE1qBFVd985+IEerkq5IhIsbvpS89TU1P5+++/cXV1pXr16qV6DozusyNy8+IvZDDr1yMs2HLcHHKahOaGnFbV8g85IiJFoaDf3ze9Nla5cuVo1qzZzb5cREq5sxcy+WRdNF9sOU5G9j8Tj0fcX4O7qlVQyBGREuOmws727dtZvHgxJ06cMJ/KumLp0qVFUpiIlEznUjOZvf4In286Zg45DYNzQ07r6go5IlLyFDrsfPXVVzz++OO0b9+en3/+mXbt2nHw4EHi4uLo2rVrcdQoIiVAgjnkHDcv6dAg2JsRkdW5p4afQo6IlFiFvvR80qRJvP/++/zvf//DycmJDz74gP3799OzZ09CQkKKo0YRXn/9dQwGg8Xj6ptbPv3001StWhVXV1f8/Px4+OGH2b9//3X7/Hd/Vx5vv/22eZ+dO3dy//334+3tja+vL4MGDSI1NbXYjrMkSkzLYvJP+7l76i98sv4IF7NzqF/Zi7n9m7F8cEvurVlRQUdESrRCh53o6Gg6deoEgJOTE2lpaRgMBkaMGGG+JFykONStW5eYmBjzY8OGDeZtTZo0Ye7cufz999+sXLkSk8lEu3btyMnJuWZ/V/cVExPDZ599hsFgoHv37gCcOXOGyMhIqlWrxpYtW1ixYgV//fUX/fv3L+5DLRHOp2UxZcV+7pqyllnroknPyqFeJS8+69+Ub4e0ok0thRwRKR0KfRqrfPnyXLhwAYBKlSqxd+9e6tWrR1JSEunp6UVeoMgVDg4O5rtm/9ugQYPMv4eFhTFx4kQaNGjAsWPHqFq1ar6v+Xdf3377LW3atKFKlSoAfP/99zg6OvLRRx9hZ5f7d8GsWbOoX78+hw8fplq1akVxWCVOUnoWn/52hHkbj5GWlRsW76jkyfC2NWhbWwFHREqfQo/stG7dmlWrVgHQo0cPnn/+eQYOHEjv3r1p27ZtkRcocsWhQ4cICgqiSpUq9OnThxMnTuS7X1paGnPnziU8PLzAi7zGxcXxww8/MGDAAHNbZmYmTk5O5qAD4Oqae0+Yq0eVCqM4TseZTCZeffVVAgMDcXV1JTIykkOHDlns89BDDxESEoKLiwuBgYH07duXM2cs16dKSs/inZUHuGvKL3z0SzRpWTnUDfLk08eb8r+hdxFZx19BR0RKpUKHnRkzZtCrVy8AXn75ZUaOHElcXBzdu3dnzpw5RV6gCEBERATz5s1jxYoVzJw5k6NHj3L33XebRxkBPv74Y8qVK0e5cuX46aefWLVqFU5OTgXqf/78+Xh4eNCtWzdz23333UdsbCxvv/02WVlZnD9/nrFjxwLc0jpwRX06burUqUyfPp1Zs2axZcsW3N3dad++PRkZGeZ92rRpw+LFizlw4ADffPMN0dHRPPLIIwAkp2fz3s8HuHvKL8z45TCpmZeoHejJJ32b8P1zd3G/Qo6IlHamQsjOzjbNnz/fFBsbW5iXlXjJyckmwJScnGztUqSAzp8/b/L09DT93//9n7ktKSnJdPDgQdO6detMnTt3NjVu3Nh08eLFAvVXs2ZN09ChQ/O0L1iwwOTv72+yt7c3OTk5mUaNGmXy9/c3TZ48+abqfu2110wNGjQo8P5//PGHCTAdPnw43+1Go9EUEBBgevvtt81tSUlJJmdnZ9OXX355zX6//fZbk8FgML39417THa+uMIWO+d4UOuZ7U/v315l+2hNjyskxFrhGERFrKej3d6FGdhwcHHjmmWcs/mIUsQZvb29q1KjB4cOHzW1eXl5Ur16d1q1b8/XXX7N//36WLVt2w75+++03Dhw4wFNPPZVn23/+8x9iY2M5ffo0CQkJvP7665w9e9Y8r+dmFOXpuKNHjxIbG0tkZKS5zcvLi4iICDZt2pTva46dieO192bhWrk2M9Yd40LmJWr6ezCzT2N+HHY3He4IwM5OIzkiYjsKfRrrzjvvZPfu3cVQihS1G80PmT17Nvfeey+enp4YDAaSkpJu2GdOTg7jx48nPDwcV1dXqlatyoQJEzBdtepIXFwc/fv3JygoCDc3Nzp06JBnDsmtSk1NJTo6msDAwHy3m0wmTCYTmZmZN+xrzpw5NGnShAYNGlxzH39/f8qVK8eiRYtwcXHh/vvvv6m6i/p0XGxsrLm+f9d7ZdsVI14YhbOrG+GVAth36Ag+XV+hhn85Pu7TmJ+ev5uO9QIVckTEJhX6aqzBgwczcuRITp48SZMmTXB3d7fYXr9+/SIrTm5d3bp1Wb16tfm5g8M/H3l6ejodOnSgQ4cOBV7EdcqUKcycOZP58+dTt25dtm/fzhNPPIGXlxfDhg3DZDLRpUsXHB0d+fbbb/H09OS9994jMjKSffv25fnvpaBGjRpF586dCQ0N5cyZM7z22mvY29vTu3dvjhw5wqJFi2jXrh1+fn6cOnWKyZMn4+rqygMPPGDuo1atWkRFRVnc/DIlJYUlS5bw7rvv5vu+M2bMoGXLlpQrV45Vq1YxevRoJk+ejLe3900dR8eOHc2/169fn4iICEJDQ1m8eLF5cnSfPn24//77iYmJ4Z133qFnz55s3Ljxphe3vZCRzbyNx/jZPoIKfWtwKTmezK2LCNj1f/z03irs7Qv9N4+ISKlS6LBzZXLysGHDzG0GgwGTyYTBYLjuREq5/a53ufbw4cMB+PXXXwvc3++//87DDz9svtdSWFgYX375JVu3bgVyT9Fs3ryZvXv3UrduXQBmzpxJQEAAX375Zb6nigri1KlT9O7dm4SEBPz8/LjrrrvYvHkzfn5+ZGdn89tvvzFt2jTOnz+Pv78/rVu35vfff6dixYrmPg4cOEBycrJFv1999RUmk4nevXvn+75bt27ltddeIzU1lVq1avHJJ5/Qt2/fmzqG/FzrdNyVU3LNmzenfPnyLFu2LN8ar3y2cXFxFqNccXFx1K1Xn49+Ocynvx0hKT0b7NyoXasiz7d9kPrl+xEWGsLWrVto0aJFkR2PiEhJVOiwc/To0eKoQ4rJlfkhLi4utGjRgqioqFu603XLli2ZPXs2Bw8epEaNGvzxxx9s2LCB9957D8B82ujqUQg7OzucnZ3ZsGHDTYedr7766prbgoKC+PHHH2/Yx5mkdI6eSyMm+SKBXrmXkA8aNMjiHj3/9vnnnxe+2EK4cjruWgHqRqfjwsPDCQgIYM2aNTRs2BCAM2cT+X3TZo5VbMWPKw8AUNXPnWFtq/Ng/SDs7QzmeUIFOc0nIlLaFTrshIaGFkcdUgyuzA+pWbMmMTExvPHGG9x9993s3bsXDw+Pm+pz7NixpKSkUKtWLezt7cnJyeGtt96iT58+QO6popCQEMaNG8cnn3yCu7s777//PqdOnbqly7Vv1aJtJxi3dA9GE9gZ4K2u9eh95+1f3qSoT8cZDAaGDx/OxIkTCQ6rwp/Jznz49kRwKw+hzaji507HihdwTtpDKN6cOnmJ6Ohoxo8fT9WqVTWqIyJlQqHDzo3+0n388cdvuhgpWgWZH1JYixcvZsGCBSxcuJC6deuye/duhg8fTlBQEP369cPR0ZGlS5cyYMAAfHx8sLe3JzIyko4dO1pMYi4uGdk5nEhM59i5tNyfCWkcjL3A1mPnzfsYTTBu6R7e/N9feLo64uHiSDlnBzxcHK766Ug5Fwc8L7eVc3Gw2O/Kvu5ODoWa1Fscp+OGDh/J+n2n6NN/AJcupuJSuQ6Nn36bsb2a8VCDSuz7ay/PP/8+b7z+OmlpaQQGBtKhQwdeeeUVnJ2di+YfXkSkBDOYCvkNVL58eYvn2dnZpKen4+TkhJubG4mJiUVa4O2QkpKCl5cXycnJeHp6WrucYtWsWTMiIyOJiooyt/3666+0adOG8+fP33DibXBwMGPHjmXIkCHmtokTJ/LFF1/kudNvcnIyWVlZ+Pn5ERERQdOmTfnoo49u+RguZGRzPCHdHGaOn8v9eSIxnZjk239bhKuD0pVQ5OF8VXC6vM3TxdH8+z+BKbfN3cm+UDfui0m+yP6YFHYcP8+XW0+SkJYFQJivG8PaVuehBkE4aOKxiNi4gn5/F3pk5/z583naDh06xLPPPsvo0aML253cRjeaH1IQ6enpFssnANjb22M0GvPs6+XlBeT+97F9+3YmTJhQ4PdJSs/iWEI6xxPSOHYu9+fxxNyf51KzrvtaD2cHwiq4E+rrRqivG16ujkT9tJ+rY72dARY93RxXRwcuZFwiNfMSFzKyL/+86vnl31MyLpl/v5CRzYWMS1wy5naYmpnbfivsDODu7ICH8+URpDyh6J+RpX0xKXyz4xRX/5US6uvGc/dVp0tDhRwRkX8rdNjJT/Xq1Zk8eTKPPfbYDdfxkdvnevNDIPceLbGxseYrgfbs2YOHhwchISH4+PgA0LZtW7p27crQoUMB6Ny5M2+99RYhISHUrVuXXbt28d577/Hkk0+a33fJkiX4+fkREhLCnj17eP755+nSpQvt2rUz72MymTibmsnxhKtPOV0JN2mkZFw/PPi6OxHi60aYb26oufIz1Ned8m6OeUZJvFwdeWnpXnJMJuwNBiZ1u4NmYb43/W9rMpnIvGT8JxxlXOJCZrb5d3Moyrz6+ZX9LINVjtGE0QQXMnKfU8jRKYMB/jvgTkJ8bu6yfhERW1ckYQdyL3H+98KCYl3Xmx8CuSt4v/HGG+b9W7duDcDcuXPp378/ANHR0Zw7d868z4cffsj48eMZPHgw8fHxBAUF8fTTT/Pqq6+a94mJiTGvmVbRP4DIh3vQtvezRP30t8Upp/Ss69+mwN/TmVBfd8Iuh5groSbE1w1PF8dC/Vs82iyE1jX8OHYunbAKbuarsW6WwWDAxdEeF0d7KpS7+XkvJpOJjGwjFzKzLULRtUaXjiWksSk64V99wOnzGQo7IiLXUOg5O999953Fc5PJRExMDDNmzCA4OJiffvqpSAu8HcrSnJ2bEZN8kaPn0giv4J4nJGTnGDmTdNHilNOJxDSOXZ5Tk3Up7+mtK+wMEOTtetWojNvlcONOiI8brk72xX1opU5M8kVaTV6L8ar/1dobDGwY2+aWA5yISGlTbHN2unTpYvHcYDDg5+fHfffdd8270ErpdfUl2wYDdKoXiI+7kzncnDp/kRzjtfOyo72B4PL/BJmrTzlVLu+Gk4PmlxRGoJcrUd3q5Tklp6AjInJthR7ZsUUa2clffqMI+XF2sDOfXgq7anQm1NeNQC8XTZgtBjHJF4vslJyISGlVbCM7UnYcPZeWb9B5uEEQLav5mkNNRQ9nLSB5mwV6uSrkiIgUUKH/5O7evTtTpkzJ0z516lR69OhRJEVJybDjeN7bDNgbDIx9oBaPNguheRVfArxcFHRERKREK3TYWb9+vcWt66/o2LEj69evL5KixPp+PRDP+6sOAnAlymh+iIiIlEaFPo2VmpqKk5NTnnZHR0dSUlKKpCixrv2xKQxduAujCXo0qcyI+6tzPOGi5oeIiEipVOiRnXr16rFo0aI87V999RV16tQpkqLEeuIvZDBg3nZSMy/RvIoPb3WtR5C3Gy2q+iroiIhIqVTokZ3x48fTrVs3oqOjue+++wBYs2YNX375JUuWLCnyAuX2uZiVw8D52zmddJEqFdyZ9VgTXRouIiKlXqHDTufOnVm+fDmTJk3i66+/xtXVlfr167N69Wruueee4qhRbgOj0cTIxbv541Qy3m6OfNa/Gd5ueU9XioiIlDY3del5p06d6NSpU1HXIlb09s8H+GlvLI72Bmb3bUpYBS09ICIitqHQ5yi2bdvGli1b8rRv2bKF7du3F0lRcnst3naSmb9GAzD1kfrcGe5j5YpERESKTqHDzpAhQzh58mSe9tOnTzNkyJAiKUpun9+jz/HSsj0ADGtbna6NKlu5IhERkaJV6LCzb98+GjdunKe9UaNG7Nu3r0iKktsj+mwqz/x3B5eMJh5qEMSIyOrWLklERKTIFTrsODs7ExcXl6c9JiYGBwetPlFaJKZl8eS8baRkXKJxiDdTH6mPwaA7IYuIiO0pdNhp164d48aNIzk52dyWlJTESy+9xP3331+kxUnxyLyUw9P/3c7xhHSCfVz59PGmuDjaW7ssERGRYlHooZh33nmH1q1bExoaSqNGjQDYvXs3/v7+/Pe//y3yAqVomUwmxnz9J9uOncfDxYHP+jXDt5yztcsSEREpNoUOO5UqVeLPP/9kwYIF/PHHH7i6uvLEE0/Qu3dvHB0di6NGKULT1xxm+e4z2NsZmNmnCdX9PaxdkoiISLG6qUk27u7uDBo0qKhrkWL27e7TvL86d3HPiV3u4K7qFaxckYiISPG76RnF+/bt48SJE2RlZVm0P/TQQ7dclBS97ccSGb3kTwAGta5C7ztDrFyRiIjI7VHosHPkyBG6du3Knj17MBgMmEwmAPOVPDk5OUVbodyyEwnpDPrvDrJyjLSr48/YDrWsXZKIiMhtU+irsZ5//nnCw8OJj4/Hzc2Nv/76i/Xr19O0aVN+/fXXYihRbkXyxWyemLeVxLQs6lXyYlqvhtjZ6RJzEREpOwo9srNp0ybWrl1LhQoVsLOzw87OjrvuuouoqCiGDRvGrl27iqNOuQnZOUYGL9hB9Nk0Ar1c+L9+TXFz0r2QRESkbCn0yE5OTg4eHrlX8FSoUIEzZ84AEBoayoEDB4q2OrlpJpOJ8cv3svFwAu5O9szp1wx/TxdrlyUiInLbFfrP/DvuuIM//viD8PBwIiIimDp1Kk5OTsyePZsqVaoUR41yE2avP8JX205iZ4AP/9OIOkGe1i5JRETEKgoddl555RXS0tIAePPNN3nwwQe5++678fX1ZdGiRUVeoBTeir0xTF6xH4DxD9bhvlr+Vq5IRETEegymK5dT3YLExETKly9fatdWSklJwcvLi+TkZDw9S/cIyJ+nkuj5ySYyso083iKUNx6qW2o/FxERkesp6Pd3kcxW9fHxKYpu5BadSbrIgPnbycg2cm9NP159sI6CjoiIlHmFnqAsJVNq5iWenLeNsxcyqRXgwYe9G+Fgr49XRERE34Y24FKOkecW7mR/7AUqlHNmTv9meLhonTIRERFQ2LEJE3/4m18OnMXF0Y45/ZpSydvV2iWJiIiUGIUOO+vXr+fSpUt52i9dusT69euLpCgpuHkbjzLv92MAvN+zIQ2Cva1aj4iISElT6LDTpk0bEhMT87QnJyfTpk2bIilKCuaX/fG8+f0+AMZ0qEXHeoFWrkhERKTkKXTYMZlM+V7hk5CQgLu7e5EUJTf2d0wKQxfuxGiCnk0r88w9uqGjiIhIfgp86Xm3bt2A3NXN+/fvj7Ozs3lbTk4Of/75Jy1btiz6CiWP+JQMBszbRlpWDi2q+DKxSz1dYi4iInINBQ47Xl5eQO7IjoeHB66u/0yCdXJyonnz5gwcOLDoKxQLF7NyeOrz7ZxJzqCKnzuzHmuCk4PmmYuIiFxLgcPO3LlzAQgLC2PUqFE6ZWUFRqOJEYt28+epZMq7OTK3fzO83HSJuYiIyPUUekjgxRdftDhlcvz4caZNm8bPP/9cpIVJXlNXHmDFX7E42dsx+/GmhPoqcIqIiNxIocPOww8/zOeffw5AUlISd955J++++y4PP/wwM2fOLPICJddXW08wa100AFMfqU+zMC3RISIiUhCFDjs7d+7k7rvvBuDrr78mICCA48eP8/nnnzN9+vQiL1Bg4+FzvLJ8LwDPt61Ol0aVrFyRiIhI6VHosJOeno6HhwcAP//8M926dcPOzo7mzZtz/PjxIi+wrDscf4FnvtjBJaOJhxoEMTyyurVLEhERKVUKHXaqVavG8uXLOXnyJCtXrqRdu3YAxMfHX3d5dSm8hNRMnpy3nQsZl2gSWp6pj9TXJeYiIiKFVOiw8+qrrzJq1CjCwsK48847adGiBZA7ytOoUaMiL7CsysjO4en/7uBEYjrBPq7M7tsEF0d7a5clIiJS6hQ67DzyyCOcOHGC7du3s3LlSnN727Ztef/994u0uH+bPHkyBoOB4cOHm9syMjIYMmQIvr6+lCtXju7duxMXF1esdRQ3k8nEmG/+ZPvx83i4ODC3fzN8yznf+IUiIiKSx03djS4gIAAPDw9WrVrFxYsXAWjWrBm1atUq0uKutm3bNj755BPq169v0T5ixAj+97//sWTJEtatW8eZM2fMd3surT5Yc4hvd5/Bwc7ArMeaUK2ih7VLEhERKbUKHXYSEhJo27YtNWrU4IEHHiAmJgaAAQMG8MILLxR5gQCpqan06dOHTz/9lPLly5vbk5OTmTNnDu+99x733XcfTZo0Ye7cufz+++9s3ry5WGopbst3nWba6kMATOxyB62qVbByRSIiIqVbocPOiBEjcHR05MSJE7i5uZnbH330UVasWFGkxV0xZMgQOnXqRGRkpEX7jh07yM7OtmivVasWISEhbNq06Zr9ZWZmkpKSYvEoCbYdS+TFr/8E4Ol7qtDrzhArVyQiIlL6FXi5iCt+/vlnVq5cSeXKlS3aq1evXiyXnn/11Vfs3LmTbdu25dkWGxuLk5MT3t7eFu3+/v7ExsZes8+oqCjeeOONoi71lhxPSGPQ59vJyjHSoW4AY9oX3ylBERGRsqTQIztpaWkWIzpXJCYmWqyEXhROnjzJ888/z4IFC3BxcSmyfseNG0dycrL5cfLkySLr+2Ykp2fzxLxtnE/Ppn5lL95/tCF2drrEXEREpCgUOuzcfffd5uUiAAwGA0ajkalTp9KmTZsiLW7Hjh3Ex8fTuHFjHBwccHBwYN26dUyfPh0HBwf8/f3JysoiKSnJ4nVxcXEEBARcs19nZ2c8PT0tHlebOXMm9evXN29r0aIFP/30k3n7vffei8FgsHg888wz1z2WuLg4+vfvT1BQEG5ubnTo0IFDhw6RdcnIM1/s4MjZNHzt0nHe8DHhIZVwd3encePGfPPNN4X/hxMRERGzQp/Gmjp1Km3btmX79u1kZWXx4osv8tdff5GYmMjGjRuLtLi2bduyZ88ei7YnnniCWrVqMWbMGIKDg3F0dGTNmjV0794dgAMHDnDixAnz/X9uRuXKlZk8eTLVq1fHZDIxf/58Hn74YXbt2kXdunUBGDhwIG+++ab5NfmNdl1hMpno0qULjo6OfPvtt3h6evLee+8RGRlJt7cWselIAu5O9rism8nxi6l89913VKhQgYULF9KzZ0+2b9+uexiJiIjcJIPJZDIV9kXJycnMmDGDP/74g9TUVBo3bsyQIUMIDAwsjhot3HvvvTRs2JBp06YB8Oyzz/Ljjz8yb948PD09ee655wD4/fffC9xnSkoKXl5eJCcnX/Mu0D4+Prz99tsMGDAgTw03cvDgQWrWrMnevXvNYcloNOLl64dT8z54NWzP//VrykNNqzJz5kz69u1rfq2vry9TpkzhqaeeKvDxiIiIlAUF+f6GmxjZOXHiBMHBwbz88sv5bgsJub1XEL3//vvY2dnRvXt3MjMzad++PR9//HGR9Z+Tk8OSJUtIS0uzGC1asGABX3zxBQEBAXTu3Jnx48dfc3QnMzMTwGLe0c/74riYY4fp1D5efWUE99Xyp2XLlixatIhOnTrh7e3N4sWLycjI4N577y2y4xERESlrCj2yY29vT0xMDBUrVrRoT0hIoGLFiuTk5BRpgbdDfslwz549tGjRgoyMDMqVK8fChQt54IEHAJg9ezahoaEEBQXx559/MmbMGO68806WLl2ab//Z2dlUq1aNiIgIPvnkE6ITs+k4aCzn1s6lasOWHN6Ve/ovKSmJRx99lJ9//hkHBwfc3NxYsmSJef0xERER+UexjeyYTKZ8F6NMTU0t0iumrK1mzZrs3r2b5ORkvv76a/r168e6deuoU6cOgwYNMu9Xr149AgMDadu2LdHR0VStWjVPX46OjixdupQBAwbg4+MDdna4hDakUr0WVPP/5+7I48ePJykpidWrV1OhQgWWL19Oz549+e2336hXr95tOW4RERFbU+CRnZEjRwLwwQcfMHDgQItTNjk5OWzZsgV7e/sin6R8OxQkGUZGRlK1alU++eSTPNvS0tIoV64cK1asoH379td8nwsZ2XR5fxUHY5KoW6UyCQtHcWezZnz00UdER0dTrVo1i3k9V963WrVqzJo169YPVERExIYU+cjOrl27gNyRnT179uDk5GTe5uTkRIMGDRg1atQtlFyyGY1G89ybf9u9ezfAdSdoX8ox8tyXu4hONhHgX5GX7y7PvS/s4K2JEwFIT08HwM7O8m4A9vb2GI3GIjgCERGRsqnAYeeXX34Bci/9/uCDD66boEq7cePG0bFjR0JCQrhw4QILFy7k119/ZeXKlURHR5vn7/j6+vLnn38yYsQIWrdubbFIaa1atYiKiqJr164A9B73Pr+dysLNx58+1V3o2/0punTpYp6PU6tWLapVq8bTTz/NO++8g6+vL8uXL2fVqlV8//33Vvl3EBERsQWFnrMzd+7c4qijRImPj+fxxx8nJiYGLy8v6tevz8qVK7n//vs5efIkq1evZtq0aaSlpREcHEz37t155ZVXLPo4cOAAycnJAMzbeJQ1Ow+SsnUphovJvB8UyOOPP8748ePN+zs6OvLjjz8yduxYOnfuTGpqKtWqVWP+/PnmidEiIiJSeDd1nx1bU9BzfoUVk3yRpTtP8c7Kg5iAsR1r8cw9eScwi4iISOEV29VYUjCLtp1g7NI9XImSzULL83TrKtYtSkREpAwq9NpYcmMxyRcZd1XQAdhx4jyxKRnWK0pERKSMUtgpBkfPpWH818lBowmOnUu3TkEiIiJlmMJOMQiv4I7dv+67aG8wEFbh2ouFioiISPFQ2CkGgV6uRHWrh/3lO03bGwxM6nYHgV6uVq5MRESk7NEE5WLyaLMQWtfw49i5dMIquCnoiIiIWInCTjEK9HJVyBEREbEyncYSERERm6awIyIiIjZNYUdERERsmsKOiIiI2DSFHREREbFpCjsiIiJi0xR2RERExKYp7IiIiIhNU9gRERERm6awIyIiIjZNYUdERERsmsKOiIiI2DSFHREREbFpCjsiIiJi0xR2RERExKYp7IiIiIhNU9gRERERm6awIyIiIjZNYUdERERsmsKOiIiI2DSFHREREbFpCjsiIiJi0xR2RERExKYp7IiIiIhNU9gRERERm6awIyIiIjZNYUdERERsmsKOiIiI2DSFHREREbFpCjsiIiJi0xR2RERExKYp7IiIiIhNU9gRERERm6awIyIiIjZNYUdERERsmsKOiIiI2DSFHREREbFpCjsiIiJi0xR2RERExKYp7IiIiIhNU9gRERERm6awIyIiIjZNYUdERERsmsKOiIiI2DSFHREREbFpCjsiIiJi0xR2RERExKYp7IiIiIhNU9gRERERm6awIyIiIjatRIedqKgomjVrhoeHBxUrVqRLly4cOHDAYp+MjAyGDBmCr68v5cqVo3v37sTFxVmpYhERESlpSnTYWbduHUOGDGHz5s2sWrWK7Oxs2rVrR1pamnmfESNG8L///Y8lS5awbt06zpw5Q7du3axYtYiIiJQkBpPJZLJ2EQV19uxZKlasyLp162jdujXJycn4+fmxcOFCHnnkEQD2799P7dq12bRpE82bNy9QvykpKXh5eZGcnIynp2dxHoKIiIgUkYJ+f5fokZ1/S05OBsDHxweAHTt2kJ2dTWRkpHmfWrVqERISwqZNm67ZT2ZmJikpKRYPERERsU2lJuwYjUaGDx9Oq1atuOOOOwCIjY3FyckJb29vi339/f2JjY29Zl9RUVF4eXmZH8HBwcVZuoiIiFhRqQk7Q4YMYe/evXz11Ve33Ne4ceNITk42P06ePFkEFYqIiEhJ5GDtAgpi6NChfP/996xfv57KlSub2wMCAsjKyiIpKclidCcuLo6AgIBr9ufs7Iyzs3NxliwiIiIlRIke2TGZTAwdOpRly5axdu1awsPDLbY3adIER0dH1qxZY247cOAAJ06coEWLFre7XBERESmBSvTIzpAhQ1i4cCHffvstHh4e5nk4Xl5euLq64uXlxYABAxg5ciQ+Pj54enry3HPP0aJFiwJfiSUiIiK2rURfem4wGPJtnzt3Lv379wdybyr4wgsv8OWXX5KZmUn79u35+OOPr3sa69906bmIiEjpU9Dv7xIddm4XhR0REZHSxybvsyMiIiJSWAo7IiIiYtMUdkRERMSmKeyIiIiITVPYEREREZumsCMiIiI2TWFHREREbJrCjoiIiNg0hR0RERGxaQo7IiIiYtMUdkRERMSmKeyIiIiITVPYEREREZumsCMiIiI2TWFHREREbJrCjoiIiNg0hR0RERGxaQo7IiIiYtMUdkRERMSmKeyIiIiITVPYEREREZumsCMiIiI2TWFHREREbJrCjoiIiNg0hR0RERGxaQo7IiIiYtMUdkRERMSmKeyIiIiITVPYEREREZumsCMiIiI2TWFHREREbJrCjoiIiNg0hR0RERGxaQo7IiIiYtMUdkRERMSmKeyIiIiITVPYEREREZumsCMiIiI2TWFHREREbJrCjoiIiNg0hR0RERGxaQo7IiIiYtMUdkRERMSmKeyIiIiITVPYEREREZumsCMiIiI2TWFHREREbJrCjoiIiNg0hR0RERGxaQo7IiIiYtMUdkRERMSmKeyIiIiITVPYEREREZumsCMiIiI2TWFHREREbJrCjoiIiNg0hR0RERGxaQo7IiIiYtMUdkRERMSmKeyIiIiITVPYEREREZumsCMiIiI2TWFHREREbJrNhJ2PPvqIsLAwXFxciIiIYOvWrdYuSUREREoAmwg7ixYtYuTIkbz22mvs3LmTBg0a0L59e+Lj461dmoiIiFiZTYSd9957j4EDB/LEE09Qp04dZs2ahZubG5999pm1SxMRERErc7B2AbcqKyuLHTt2MG7cOHObnZ0dkZGRbNq0Kd/XZGZmkpmZaX6enJwMQEpKSvEWKyIiIkXmyve2yWS67n6lPuycO3eOnJwc/P39Ldr9/f3Zv39/vq+JiorijTfeyNMeHBxcLDWKiIhI8blw4QJeXl7X3F7qw87NGDduHCNHjjQ/NxqNJCYm4uvri8FgKLL3SUlJITg4mJMnT+Lp6Vlk/crN0edR8ugzKVn0eZQs+jxuzGQyceHCBYKCgq67X6kPOxUqVMDe3p64uDiL9ri4OAICAvJ9jbOzM87OzhZt3t7exVUinp6e+g+1BNHnUfLoMylZ9HmULPo8ru96IzpXlPoJyk5OTjRp0oQ1a9aY24xGI2vWrKFFixZWrExERERKglI/sgMwcuRI+vXrR9OmTbnzzjuZNm0aaWlpPPHEE9YuTURERKzMJsLOo48+ytmzZ3n11VeJjY2lYcOGrFixIs+k5dvN2dmZ1157Lc8pM7EOfR4ljz6TkkWfR8miz6PoGEw3ul5LREREpBQr9XN2RERERK5HYUdERERsmsKOiIiI2DSFHREREbFpCjvF6KOPPiIsLAwXFxciIiLYunWrtUsqk6KiomjWrBkeHh5UrFiRLl26cODAAWuXJZdNnjwZg8HA8OHDrV1KmXX69Gkee+wxfH19cXV1pV69emzfvt3aZZVZOTk5jB8/nvDwcFxdXalatSoTJky44fpPcm0KO8Vk0aJFjBw5ktdee42dO3fSoEED2rdvT3x8vLVLK3PWrVvHkCFD2Lx5M6tWrSI7O5t27dqRlpZm7dLKvG3btvHJJ59Qv359a5dSZp0/f55WrVrh6OjITz/9xL59+3j33XcpX768tUsrs6ZMmcLMmTOZMWMGf//9N1OmTGHq1Kl8+OGH1i6t1NKl58UkIiKCZs2aMWPGDCD3rs7BwcE899xzjB071srVlW1nz56lYsWKrFu3jtatW1u7nDIrNTWVxo0b8/HHHzNx4kQaNmzItGnTrF1WmTN27Fg2btzIb7/9Zu1S5LIHH3wQf39/5syZY27r3r07rq6ufPHFF1asrPTSyE4xyMrKYseOHURGRprb7OzsiIyMZNOmTVasTACSk5MB8PHxsXIlZduQIUPo1KmTxf9O5Pb77rvvaNq0KT169KBixYo0atSITz/91NpllWktW7ZkzZo1HDx4EIA//viDDRs20LFjRytXVnrZxB2US5pz586Rk5OT5w7O/v7+7N+/30pVCeSOsA0fPpxWrVpxxx13WLucMuurr75i586dbNu2zdqllHlHjhxh5syZjBw5kpdeeolt27YxbNgwnJyc6Nevn7XLK5PGjh1LSkoKtWrVwt7enpycHN566y369Olj7dJKLYUdKVOGDBnC3r172bBhg7VLKbNOnjzJ888/z6pVq3BxcbF2OWWe0WikadOmTJo0CYBGjRqxd+9eZs2apbBjJYsXL2bBggUsXLiQunXrsnv3boYPH05QUJA+k5uksFMMKlSogL29PXFxcRbtcXFxBAQEWKkqGTp0KN9//z3r16+ncuXK1i6nzNqxYwfx8fE0btzY3JaTk8P69euZMWMGmZmZ2NvbW7HCsiUwMJA6depYtNWuXZtvvvnGShXJ6NGjGTt2LL169QKgXr16HD9+nKioKIWdm6Q5O8XAycmJJk2asGbNGnOb0WhkzZo1tGjRwoqVlU0mk4mhQ4eybNky1q5dS3h4uLVLKtPatm3Lnj172L17t/nRtGlT+vTpw+7duxV0brNWrVrluRXDwYMHCQ0NtVJFkp6ejp2d5dezvb09RqPRShWVfhrZKSYjR46kX79+NG3alDvvvJNp06aRlpbGE088Ye3SypwhQ4awcOFCvv32Wzw8PIiNjQXAy8sLV1dXK1dX9nh4eOSZL+Xu7o6vr6/mUVnBiBEjaNmyJZMmTaJnz55s3bqV2bNnM3v2bGuXVmZ17tyZt956i5CQEOrWrcuuXbt47733ePLJJ61dWqmlS8+L0YwZM3j77beJjY2lYcOGTJ8+nYiICGuXVeYYDIZ82+fOnUv//v1vbzGSr3vvvVeXnlvR999/z7hx4zh06BDh4eGMHDmSgQMHWrusMuvChQuMHz+eZcuWER8fT1BQEL179+bVV1/FycnJ2uWVSgo7IiIiYtM0Z0dERERsmsKOiIiI2DSFHREREbFpCjsiIiJi0xR2RERExKYp7IiIiIhNU9gRERERm6awIyLyL7/++isGg4GkpCRrlyIiRUBhR0RERGyawo6IiIjYNIUdESlxjEYjUVFRhIeH4+rqSoMGDfj666+Bf04x/fDDD9SvXx8XFxeaN2/O3r17Lfr45ptvqFu3Ls7OzoSFhfHuu+9abM/MzGTMmDEEBwfj7OxMtWrVmDNnjsU+O3bsoGnTpri5udGyZcs8q4OLSOmgsCMiJU5UVBSff/45s2bN4q+//mLEiBE89thjrFu3zrzP6NGjeffdd9m2bRt+fn507tyZ7OxsIDek9OzZk169erFnzx5ef/11xo8fz7x588yvf/zxx/nyyy+ZPn06f//9N5988gnlypWzqOPll1/m3XffZfv27Tg4OGjVaZFSSguBikiJkpmZiY+PD6tXr6ZFixbm9qeeeor09HQGDRpEmzZt+Oqrr3j00UcBSExMpHLlysybN4+ePXvSp08fzp49y88//2x+/YsvvsgPP/zAX3/9xcGDB6lZsyarVq0iMjIyTw2//vorbdq0YfXq1bRt2xaAH3/8kU6dOnHx4kVcXFyK+V9BRIqSRnZEpEQ5fPgw6enp3H///ZQrV878+Pzzz4mOjjbvd3UQ8vHxoWbNmvz9998A/P3337Rq1cqi31atWnHo0CFycnLYvXs39vb23HPPPdetpX79+ubfAwMDAYiPj7/lYxSR28vB2gWIiFwtNTUVgB9++IFKlSpZbHN2drYIPDfL1dW1QPs5OjqafzcYDEDufCIRKV00siMiJUqdOnVwdnbmxIkTVKtWzeIRHBxs3m/z5s3m38+fP8/BgwepXbs2ALVr12bjxo0W/W7cuJEaNWpgb29PvXr1MBqNFnOARMR2aWRHREoUDw8PRo0axYgRIzAajdx1110kJyezceNGPD09CQ0NBeDNN9/E19cXf39/Xn75ZSpUqECXLl0AeOGFF2jWrBkTJkzg0UcfZdOmTcyYMYOPP/4YgLCwMPr168eTTz7J9OnTadCgAcePHyc+Pp6ePXta69BFpJgo7IhIiTNhwgT8/PyIioriyJEjeHt707hxY1566SXzaaTJkyfz/PPPc+jQIRo2bMj//vc/nJycAGjcuDGLFy/m1VdfZcKECQQGBvLmm2/Sv39/83vMnDmTl156icGDB5OQkEBISAgvvfSSNQ5XRIqZrsYSkVLlypVS58+fx9vb29rliEgpoDk7IiIiYtMUdkRERMSm6TSWiIiI2DSN7IiIiIhNU9gRERERm6awIyIiIjZNYUdERERsmsKOiIiI2DSFHREREbFpCjsiIiJi0xR2RERExKYp7IiIiIhN+3+y6lfWWi9Y7AAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plt.plot(np.arange(len(epochs_x)), epochs_acc, marker = '.')\n", + "plt.xlabel('epoch')\n", + "plt.ylabel('test accuracy')\n", + "plt.ylim(0, 100)\n", + "for i, txt in enumerate(epochs_acc):\n", + " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom')\n", + "plt.show()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "speck-rescnn", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 69a7f8ae08135c25dc592383959516405908ee17 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Sun, 28 Apr 2024 12:48:02 +0200 Subject: [PATCH 053/379] CNN 2 skip conns. 85% | acc baseline CNN 4070% acc --- .../baseline-SCNN-example_3.ipynb | 453 ++++++++++++++++-- .../non-sequential-SCNN-example_3.ipynb | 449 +++++++++++++++-- 2 files changed, 802 insertions(+), 100 deletions(-) diff --git a/tests/test_nonsequential/baseline-SCNN-example_3.ipynb b/tests/test_nonsequential/baseline-SCNN-example_3.ipynb index 800ef861..8567adc5 100644 --- a/tests/test_nonsequential/baseline-SCNN-example_3.ipynb +++ b/tests/test_nonsequential/baseline-SCNN-example_3.ipynb @@ -6,7 +6,7 @@ "metadata": {}, "outputs": [], "source": [ - "import torch\n", + "import torch, random\n", "import torch.nn as nn\n", "import sinabs.layers as sl\n", "from tqdm.notebook import tqdm\n", @@ -27,20 +27,13 @@ "cell_type": "code", "execution_count": 2, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "torch.manual_seed(0)" + "torch.backends.cudnn.deterministic = True\n", + "random.seed(1)\n", + "torch.manual_seed(1)\n", + "torch.cuda.manual_seed(1)\n", + "np.random.seed(1)" ] }, { @@ -58,7 +51,7 @@ "source": [ "batch_size = 3\n", "num_workers = 1\n", - "epochs = 10\n", + "epochs = 20\n", "lr = 1e-3" ] }, @@ -366,7 +359,357 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "d1e6254a097e41c198599735d8d1a5c2", + "model_id": "57e76b2737034b53b317bac992a7d524", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/359 [00:00" ] @@ -743,19 +1086,23 @@ "plt.xlabel('epoch')\n", "plt.ylabel('average loss')\n", "plt.ylim(0,)\n", + "plt.xticks(np.arange(len(epochs_x)))\n", "for i, txt in enumerate(y_avg):\n", - " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom')\n", + " if i%2 == 0:\n", + " pass\n", + " else:\n", + " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", "plt.show()" ] }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 36, "metadata": {}, "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAG2CAYAAACZEEfAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABX5UlEQVR4nO3dd1QU198G8GdZqohUaUpTFESkKIioMVGwxSS2xPIziS3GJNiNUYyoiQU0amyxvtbEnsQeC0HFhoggdrGggkpRyiIgxd15/0A32SAKCiwMz+ecPUfuzg7fEWWfvXOLRBAEAUREREQipaHuAoiIiIgqEsMOERERiRrDDhEREYkaww4RERGJGsMOERERiRrDDhEREYkaww4RERGJGsMOERERiRrDDhEREYkaww4RERGJmlrDzvHjx/Hhhx/C2toaEokEu3btUnleEARMnToVVlZW0NPTg7+/P27evKlyTHp6OgYMGIA6derAyMgIQ4cORXZ2diVeBREREVVlag07OTk5cHd3xy+//PLS5+fOnYvFixdjxYoViIyMhL6+Pjp37oy8vDzlMQMGDMCVK1cQGhqKffv24fjx4/jyyy8r6xKIiIioipNUlY1AJRIJdu7ciR49egAo6tWxtrbG+PHj8e233wIAZDIZLCwssH79evTr1w/Xrl2Di4sLoqKi4OXlBQA4ePAg3n//fdy/fx/W1tbquhwiIiKqIjTVXUBJ7ty5g+TkZPj7+yvbDA0N4ePjg4iICPTr1w8REREwMjJSBh0A8Pf3h4aGBiIjI9GzZ8+Xnjs/Px/5+fnKrxUKBdLT02FqagqJRFJxF0VERETlRhAEPHnyBNbW1tDQKPlmVZUNO8nJyQAACwsLlXYLCwvlc8nJyTA3N1d5XlNTEyYmJspjXiY4OBg//PBDOVdMRERE6pCYmIj69euX+HyVDTsVKTAwEOPGjVN+LZPJYGtri8TERNSpU0eNlREREVFpZWVlwcbGBgYGBq88rsqGHUtLSwBASkoKrKyslO0pKSnw8PBQHpOamqryumfPniE9PV35+pfR0dGBjo5OsfY6deow7BAREVUzrxuCUmXX2XFwcIClpSXCwsKUbVlZWYiMjISvry8AwNfXF5mZmYiOjlYec+TIESgUCvj4+FR6zURERFT1qLVnJzs7G7du3VJ+fefOHcTGxsLExAS2trYYM2YMZs6ciUaNGsHBwQFBQUGwtrZWzthq0qQJunTpgmHDhmHFihUoLCzEiBEj0K9fP87EIiIiIgBqDjvnzp1D+/btlV+/GEczcOBArF+/Ht999x1ycnLw5ZdfIjMzE23btsXBgwehq6urfM2mTZswYsQI+Pn5QUNDA71798bixYsr/VqIiIioaqoy6+yoU1ZWFgwNDSGTyThmh4iIqJoo7ft3lR2zQ0RERFQeGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNSqdNiRy+UICgqCg4MD9PT00LBhQ8yYMQOCICiPEQQBU6dOhZWVFfT09ODv74+bN2+qsWoiIiKqSqp02JkzZw6WL1+OpUuX4tq1a5gzZw7mzp2LJUuWKI+ZO3cuFi9ejBUrViAyMhL6+vro3Lkz8vLy1Fg5ERERVRUS4d/dJFXMBx98AAsLC6xZs0bZ1rt3b+jp6eG3336DIAiwtrbG+PHj8e233wIAZDIZLCwssH79evTr169U3ycrKwuGhoaQyWSoU6dOhVwLERERla/Svn9X6Z6d1q1bIywsDDdu3AAAXLhwASdPnkTXrl0BAHfu3EFycjL8/f2VrzE0NISPjw8iIiJKPG9+fj6ysrJUHkRERCROmuou4FUmTZqErKwsODs7QyqVQi6XY9asWRgwYAAAIDk5GQBgYWGh8joLCwvlcy8THByMH374oeIKJyIioiqjSvfsbN++HZs2bcLmzZsRExODDRs2YN68ediwYcNbnTcwMBAymUz5SExMLKeKiYiIqKqp0j07EyZMwKRJk5Rjb5o1a4Z79+4hODgYAwcOhKWlJQAgJSUFVlZWytelpKTAw8OjxPPq6OhAR0enQmsnIiKiqqFK9+zk5uZCQ0O1RKlUCoVCAQBwcHCApaUlwsLClM9nZWUhMjISvr6+lVorERERVU1Vumfnww8/xKxZs2Bra4umTZvi/PnzWLBgAYYMGQIAkEgkGDNmDGbOnIlGjRrBwcEBQUFBsLa2Ro8ePdRbPBEREVUJVTrsLFmyBEFBQfjmm2+QmpoKa2trDB8+HFOnTlUe89133yEnJwdffvklMjMz0bZtWxw8eBC6urpqrJyIiIiqiiq9zk5l4To7RERE1Y8o1tkhIiIielsMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERPRSDx48wKeffgpTU1Po6emhWbNmOHfunMox165dw0cffQRDQ0Po6+vD29sbCQkJJZ5z9erVeOedd2BsbAxjY2P4+/vj7NmzKsekpKRg0KBBsLa2Rq1atdClSxfcvHnzja+DYYeIiIiKycjIQJs2baClpYUDBw7g6tWrmD9/PoyNjZXH3L59G23btoWzszOOHTuGixcvIigoCLq6uiWe99ixY+jfvz+OHj2KiIgI2NjYoFOnTnjw4AEAQBAE9OjRA/Hx8di9ezfOnz8POzs7+Pv7Iycn542uRSIIgvBGrxSRrKwsGBoaQiaToU6dOuouh4iISO0mTZqEU6dO4cSJEyUe069fP2hpaeHXX3994+8jl8thbGyMpUuX4vPPP8eNGzfg5OSEy5cvo2nTpgAAhUIBS0tLzJ49G1988YXytaV9/2bPDhERERWzZ88eeHl54ZNPPoG5uTk8PT2xevVq5fMKhQL79+9H48aN0blzZ5ibm8PHxwe7du0q0/fJzc1FYWEhTExMAAD5+fkAoNI7pKGhAR0dHZw8efKNroVhh4iIiIqJj4/H8uXL0ahRIxw6dAhff/01Ro0ahQ0bNgAAUlNTkZ2djZCQEHTp0gWHDx9Gz5490atXL4SHh5f6+0ycOBHW1tbw9/cHADg7O8PW1haBgYHIyMhAQUEB5syZg/v37yMpKemNroW3scDbWERERP+lra0NLy8vnD59Wtk2atQoREVFISIiAg8fPkS9evXQv39/bN68WXnMRx99BH19fWzZsuW13yMkJARz587FsWPH4ObmpmyPjo7G0KFDceHCBUilUvj7+0NDQwOCIODAgQPK43gbi4iIiN6YlZUVXFxcVNqaNGminGllZmYGTU3NVx7zKvPmzUNISAgOHz6sEnQAoEWLFoiNjUVmZiaSkpJw8OBBpKWloUGDBm90LQw7REREVEybNm0QFxen0nbjxg3Y2dkBKOr58fb2fuUxJZk7dy5mzJiBgwcPwsvLq8TjDA0NUbduXdy8eRPnzp1D9+7d3+haGHaIqEYrzToiL3z11VeQSCRYuHDha8/7yy+/wN7eHrq6uvDx8VFZRyQ9PR0jR46Ek5MT9PT0YGtri1GjRkEmk5XXZRG9tbFjx+LMmTOYPXs2bt26hc2bN2PVqlUICAhQHjNhwgRs27YNq1evxq1bt7B06VLs3bsX33zzjfKYzz//HIGBgcqv58yZg6CgIKxduxb29vZITk5GcnIysrOzlcfs2LEDx44dU04/79ixI3r06IFOnTq92cUIJMhkMgGAIJPJ1F0KEVWi9PR0wc7OThg0aJAQGRkpxMfHC4cOHRJu3bpV7Ng///xTcHd3F6ytrYWff/75lefdunWroK2tLaxdu1a4cuWKMGzYMMHIyEhISUkRBEEQLl26JPTq1UvYs2ePcOvWLSEsLExo1KiR0Lt374q4TKI3tnfvXsHV1VXQ0dERnJ2dhVWrVhU7Zs2aNYKjo6Ogq6sruLu7C7t27VJ5/t133xUGDhyo/NrOzk4AUOwxbdo05TGLFi0S6tevL2hpaQm2trbClClThPz8/GLfu7Tv3xygDA5QJqqpSrOOCFDU++Pj44NDhw6hW7duGDNmDMaMGVPi8T4+PvD29sbSpUsBFE3RtbGxwciRIzFp0qSXvmbHjh349NNPkZOTA01NzTe+JqKqJkn2FHce58DBTB9Whnrlem4OUCYieo3XrSMCFAWVzz77DBMmTFAucPYqBQUFiI6OVk6jBYrWCPH390dERESJr3vxy5pBh8RkW1QC2oQcwf9WR6JNyBFsi3r9wOWKwLBDRDXW69YRAYrGF2hqamLUqFGlOufjx48hl8thYWGh0m5hYYHk5OQSXzNjxgx8+eWXb34xRFXMg4xcTPrjEhTP7x8pBGDyn5eRJHta6bXwIwQR1VgKhQJeXl6YPXs2AMDT0xOXL1/GihUrMHDgQERHR2PRokWIiYmBRCKpkBqysrLQrVs3uLi4YPr06RXyPYgqW7IsD1/9Fo3/jpORCwLuPs4t99tZr8OeHSKqsV63jsiJEyeQmpoKW1tbaGpqQlNTE/fu3cP48eNhb2//0nOamZlBKpUiJSVFpT0lJQWWlpYqbU+ePEGXLl1gYGCAnTt3QktLq/wujkhN9l18iM4Lj+PSg6xiz0klEtib1ar0mhh2iKjGet06Ip999hkuXryI2NhY5cPa2hoTJkzAoUOHXnpObW1ttGjRAmFhYco2hUKBsLAw+Pr6KtuysrLQqVMnaGtrY8+ePa/cJZqoOsjKK8TYbbEYsfk8ZE8L0ayeIcZ3bAzp815RqUSC2b1cK71XB+BtLCKqwcaOHYvWrVtj9uzZ6NOnD86ePYtVq1Zh1apVAABTU1OYmpqqvEZLSwuWlpZwcnJStvn5+aFnz54YMWIEAGDcuHEYOHAgvLy80LJlSyxcuBA5OTkYPHgwgH+CTm5uLn777TdkZWUhK6voU3DdunUhlUor4/KJys2Z+DSM334BDzKfQkMCBLR3xCi/RtCSauBjr/q4+zgX9ma11BJ0AIYdIqrBvL29sXPnTgQGBuLHH3+Eg4MDFi5ciAEDBpTpPLdv38bjx4+VX/ft2xePHj3C1KlTkZycDA8PDxw8eFA5aDkmJgaRkZEAAEdHR5Vz3blzp8RbZERVTf4zORYcvoFVJ+IhCICtSS383NcdLexMlMdYGeqpLeS8wHV2wHV2iOjtVOQ6IkRVVVzyE4zZFotrSUW9kv28bTDlAxfU1qm8fpTSvn+zZ4eI6C1si0pA4J9F02s1JEBwr2bo622r7rKIKoxCIWDtqTuYezAOBXIFTPS1EdKrGTo1tXz9i9WEA5SJiN7QnUc5xdYRCfzzEi7ez6z0Wl63x5cgCJg6dSqsrKygp6cHf39/3Lx585XnfPLkCcaMGQM7Ozvo6emhdevWiIqKUjkmOzsbI0aMQP369aGnpwcXFxesWLGixl+HWD3MfIpP10Ri5v5rKJAr0MHZHIfGtKvSQQdg2CGqdBXxy7w05/3zzz/RqVMnmJqaQiKRIDY2tiIur0bIK5Rjzck76LHsVLF1RBQC8NHSU2gTcgQjt5zHulN3cCExE4VyRYXVk5GRgTZt2kBLSwsHDhzA1atXMX/+fBgbGyuPmTt3LhYvXowVK1YgMjIS+vr66Ny5M/Ly8ko87xdffIHQ0FD8+uuvuHTpEjp16gR/f388ePBAecy4ceNw8OBB/Pbbb7h27RrGjBmDESNGYM+ePTX2OsRqd+wDdFl4HKdvp0FPS4pZPV2xZqAX6hroqLu01+KYHXDMDlWejIwMeHp6on379vj6669Rt25d3Lx5Ew0bNkTDhg0BFK3YGxwcjA0bNsDBwQFBQUG4dOkSrl69WuL05NKc99dff8WdO3dgbW2NYcOG4fz58/Dw8KisSxeFvEI5NkcmYHn4bTx6kl/icRKgWAjS1dKAWz0jeNoZobmtMZrbGpfbm8Tr9vgSBAHW1tYYP348vv32WwBF21NYWFhg/fr16NevX7HXPH36FAYGBti9eze6deumbG/RogW6du2KmTNnAgBcXV3Rt29fBAUFlXhMTbsOsZHlFiJo92XsufAQAOBuY4Sf+7ijQd3aaq6MY3aIqqQ5c+bAxsYG69atU7Y5ODgo/ywIAhYuXIgpU6age/fuAICNGzfCwsICu3bteukv89KcFyhaMwYA7t69W16XU2PkFcqx9WwClh27jdTnIaeekR5GdHCEQiFg6u4rkAuCch2Rbm7WuJCYiZh7GYhJyEBMQiZkTwtx9m46zt5NV57X1qQWmtsaobldUfhxtjSAprTsHe579uxB586d8cknnyA8PBz16tXDN998g2HDhgEomuGVnJyssl+XoaEhfHx8EBER8dJ/V8+ePYNcLi8WsPX09HDy5Enl161bt8aePXswZMgQWFtb49ixY7hx4wZ+/vnnGnsdYnL61mOM33EBSbI8SDUkGNnBESPaO77Rv1N1YtghqkQV8cu8NOelN5NXKMe2qEQsO3YLKVn/hJyA9o74uEV9aGsW/cLv0MS82DoibRzN0MbRDEDRgM74xzmIScjA+YQMxNzLxI3UJ0hIz0VCei52xRZ9Yq6lLYVbfUM0tzVGCztjeNoaw0Rf+7V1vtjja9y4cZg8eTKioqIwatQoaGtrY+DAgco9ucqyX5eBgQF8fX0xY8YMNGnSBBYWFtiyZQsiIiJUpssvWbIEX375JerXrw9NTU1oaGhg9erVaNeuXVn+qkV1HWKQVyjHT4fisObkHQCAg5k+FvRxh6et8WteWTUx7BBVoor4ZV6a81LZ5D+TY3tUIn45ehvJWUVjQawMdRHQ3hGfeNWHjqbqon+vW0dEQ0MCR/PacDSvjT5eNgCKVpuNTchU9vycT8jAk7xnOBOfjjPx//T+OJjpw9PWSBmAGlsYQKqhuk/X6/b4elO//vorhgwZgnr16kEqlaJ58+bo378/oqOjlccsWbIEZ86cwZ49e2BnZ4fjx48jICAA1tbWKqG9NMRyHdXd1YdZGLstFnEpTwAA//OxxZRuTVBLu/pGhupbOVE1VFG/zCvqvDVN/jM5dpy7j2VHb+Gh7J+Q8017R/R5Sch5G3V0tdCucV20a1wXQFHvz61H2cpbX9H3MnD7UQ7uPC56/BlTNJi2to4m3G2Ken+a2xmjuY1xiXt8/fHHHwCg3JMrJSUFVlZWymNSUlJeOW6rYcOGCA8PR05ODrKysmBlZYW+ffuiQYMGAIrGw0yePBk7d+5Ujodxc3NDbGws5s2bV+aQIJbrqK7kCgH/dyIe8w/fQIFcAbPa2pj7sRs6OFu8/sVVXJnDztGjR9G+ffuKqIVI9Crql/nrzkuvVvBMgR3RifjlyD8hx6KODgLaO6Kvt025hpySaGhI0NjCAI0tDNCvZdE6PZm5BTifmInz9/7p/cnOf4ZTt9Jw6laa8rX5po44eCoGW88moLmdMRzr1lbZ48vBwQGWlpYICwtT/jvKyspCZGQkvv7669fWpq+vD319fWRkZODQoUOYO3cuAKCwsBCFhYXQ0FAdvyGVSqFQlH322ev2Kqsu11Ed3c/IxfjtFxB5p6hXsaOLBUJ6NYNp7ao/06o0yhx2unTpgvr162Pw4MEYOHAgbGxsKqIuIlGqqF/mrzsvvVyhXIHfo+9j6ZFbeJD5FABgbqCDb95riH4tbaGrpd49qoxqaaO9kznaO5kDKPrkfSPlSdGtr3tFt8DuPM6BpNkHePDbBHw9IQi1nNtC4/FtJO1bgf7jZuD4jUfwsDXCmDFjMHPmTDRq1Eg5y8/a2ho9evRQfr//7vF16NAhCIIAJycn3Lp1CxMmTICzs7Nyj686derg3XffxYQJE6Cnpwc7OzuEh4dj48aNWLBgQZmv93V7lUkkkmpxHdWJIAjYef4Bpu2+gif5z1BLW4ppH7qgj5cNJBLJ609QXQhl9OjRI2HBggWCu7u7oKmpKXTq1EnYtm2bkJ+fX9ZTVRkymUwAIMhkMnWXQiJ39uxZQVNTU5g1a5Zw8+ZNYdOmTUKtWrWE3377TXlMSEiIYGRkJOzevVu4ePGi0L17d8HBwUF4+vSp8pgOHToIS5YsKdN509LShPPnzwv79+8XAAhbt24Vzp8/LyQlJVXOxVchBc/kwtaz94Q2IWGC3cR9gt3EfYLXzFBh7cl44WnBM3WXVyZp2fnC31eThUE/LBMMrRsIEqmWoGlSXzDpPEJ5bfaT9gn+848KrT/+UjA0qSto6+gIfn5+QlxcnMq56tvYCkNGThAeZuYKgiAI27ZtExo0aCBoa2sLlpaWQkBAgJCZmanymqSkJGHQoEGCtbW1oKurKzg5OQnz588XFArFG13P3r17BVdXV0FHR0dwdnYWVq1apfK8QqEQgoKCBAsLC0GnCl9HdZCRky9881u08t9Jz19OCncfZ6u7rDIp7fv3W62zExMTg3Xr1mHLli0AgP/9738YOnQo3N3dyyeJVRKus0OVad++fQgMDMTNmzfh4OCAcePGqcyaEgQB06ZNw6pVq5CZmYm2bdti2bJlaNy4sfIYe3t7DBo0CNOnTy/1edevX6/8JPtv06ZNUzmPmBXKFdgZ8wBLjt5EYnpRT45ZbR18/V5DDPBRf09OeXgmV+B68hOcfz7uJyYhEwnpucWOM9TTgqetEVo8H/tzKzUbP+y9Uu23veD2HaVz4uYjfLvjAlKy8qGpIcEY/0b46t2G1W5KeWnfv996UcGHDx9i1apVCAkJgaamJvLy8uDr64sVK1agadOmb3PqSsOwQ9UNN54sm2dyBXaef4AlR24p3/jNamvjq3cbYoCPHfS0q3/IeZVHT/KLwk9CBs7fy8SF+5nIf/b6sSg+DibQqUYBML9Qrhxz8oKGBDg1qQP/nzyXVyhHyIHrWH/6LgCgQV19LOzrAbf6Rmqt601VaNgpLCzE7t27sXbtWoSGhsLLywtDhw5F//798ejRI0yZMgUxMTG4evXqW10EULQE/sSJE3HgwAHk5ubC0dER69atg5eXF4B/PgWvXr0amZmZaNOmDZYvX45GjRqV+nsw7FB1sunMPUzZfRkCP7m+1jO5ArtiH2LJkZu4l1YUckz1i0LOp63EH3JKUihX4FpSlrLnJ+LWYzzOKVB3WRWmlYMJ+rW0RXsncxjW0lJ3OWpz+YEMY7bF4lZqNgDgc187BHZtUq3/H1RY2Bk5ciS2bNkCQRDw2Wef4YsvvoCrq6vKMcnJybC2tn7rUewVtbT+fzHsUHVxLSkLXRepLqUvATD9Ixd0dbWCeZ3S/ZsXu2dyBfZceIglR27hzuMcAICJvjaGt2uAz3ztqvV6IRUhSfYUbUKOKDc0BQCJBAjq1gRGtV6/qGFVkZlbgBn7rhXbquMFqYYELe1N0NHFAh1dLGBjUqtS61MXuULAivDbWPj3DRTKBdQ10MFPH7vhvecD36uzCgs7fn5++OKLL9CrVy/o6Lx8StqzZ89w6tQpvPvuu2Wr+j8qYp+Ul2HYoerg7uMc9Ft1RrnI3cs0qKuPVg1Mnz9MYG5QMeFn+vTp+OGHH1TanJyccP36ddy9e7fYVhUvbN++HZ988kmx9sLCQkyZMgV//fUX4uPjYWhoCH9/f4SEhMDa2lp5XHp6OkaOHIm9e/dCQ0MDvXv3xqJFi1C7dtEePXKFgD0XHmBJ2C3E/yvkfNmuAT5rZQd9HYackmyLSsDkPy+rbHtRHXsMVa8D+Pq9olWSQ6+mKBfJe8HZ0kAZfFytDaGhIaLZR88lpudi3PZYRN3NAAB0aWqJ2b2alWpl7uqg0sbsVCQXFxd07twZ9+/ff+kS+PHx8WjYsGGxDQ3fffddeHh4YNGiRS89b35+PvLz/9nELysrCzY2Ngw7VGVF30vHFxvOISO3sNhzEgCNLWvjRko2/vu/ueHz8OPb0BQ+DqbltvHk9OnT8fvvv+Pvv/9WtmlqasLMzAxyuRyPHj1SOX7VqlX46aefkJSUpAwm/yaTyfDxxx9j2LBhcHd3R0ZGBkaPHg25XK6yc3vXrl2RlJSElStXorCwEIMHD4a3tzd+/W0T9l18iEVhNxH/qCjkGNfSwrB2DTDQ154hp5SSZE+LbXtRHZV0HQlpuQi9loLQq8mIupsB+b+6sizq6MC/SVHw8W1o+lZrK73qw8ALERER+P777xEZGQmpVAoPDw8cOnQIenov/3uXy+WYPn06fvvtN+Xdk0GDBmHKlCmQSCQv/cDQyNMXyU69ka9tiNo6mpj+UVP0bl5PVFPKK2wj0ODgYFhYWGDIkCEq7WvXrsWjR48wceLEsldbgopaWj84OLjYP0SiqmrfxYcYt/0CCp4p4FbfEB+6WyPkr+vFPoHLnhbi7J10nIlPw5n4NFxNysLtRzm4/SgHmyITAACNzGur9Py8zYJhmpqaykUQ/00qlRZr37lzJ/r06fPSoAMU7f8VGhqq0rZ06VK0bNkSCQkJsLW1xbVr13Dw4EFERUUpx+wtXLQYH37QDQmOvXC/oOhNwqiWFoa90wADW9ujNkNOmbxu24vqoqTrsDWthaFtHTC0rQMycwtwNC4VoVdTEB73CClZ+dgUmYBNkQnQ15biXae66OhigfZO5m90K69p06bFPgy8EBERgS5duiAwMBBLliyBpqYmLly4UGxRw3+bM2cOli9fjg0bNqBp06Y4d+4cBg8eDENDQ4waNQq5ubmIiYlBUFAQ7Bo1QciuaPy1OhiIuYiPpm3Agj4eNea23cuU+TfBypUrsXnz5mLtTZs2Rb9+/co17FTUEviBgYEYN26c8usXPTtEVYkgCFh5PB4hB4o+DXZ0scCifh6opa2JD9ysin1yNdTTUnbJA0XjF87eSUdEfBrOxKfjWlIWbqZm42ZqNn49cw8A0NiiKPz4NjCFTwPTMnVt37x5E9bW1tDV1YWvry+Cg4Nha1v8tkd0dDRiY2Pxyy+/lOn6ZTIZJBIJjIyMABS9QRgZGcHLywsKhYD9l5Lw8xVtCJDgxuXzsHJ/F8PeccDA1vYw0K25g1CpdIxqaaOnZ3309KyP/GdyRNxOQ+jVFPx9LQUpWfn461Iy/rqUDKmGBN72xujoYomOTSxga1q6wFDShwGgaPHEUaNGYdKkSco2JyenV57v9OnT6N69u3I7C3t7e2zZsgVnz54F8M8HhqNxqRjx+0U8yq8L885f4/76sZjbpV6NDjrAG4Sd5ORklWXsX6hbty6SkpLKpagXKmppfR0dnRLHGxFVBc/kCkzdcwWbn/fIDG5jjyndXJQbQJbmE7hRLW10amqJTk2L/p9k5BQg8l89P9eTn+BGSjZupGRjY0RR+HG2NFD2+vg4mMK4hPDj4+OD9evXw8nJCUlJSfjhhx/wzjvv4PLlyzAwMFA5ds2aNWjSpAlat25d6uvPy8vDxIkT0b9/f2XXdHJyMszNzYtuV/19EzefzyiR6hngPRttrJnYniGH3oiOphTvOZnjPSdzzOjuiksPZPj7WgpCr6bgevIT5easM/ZdhZNF0TgffxcLuNUreZxPSR8GUlNTERkZiQEDBqB169a4ffs2nJ2dMWvWLLRt27bEGlu3bo1Vq1bhxo0baNy4MS5cuICTJ08qV3h+WiDH7L+uKT/INDKvjT4eDhi+QQJTk+q5U3l5KnPYsbGxwalTp4oNQDx16pTKQMLyUNH7pBBVRdn5zxCwKQbhNx49nxHjgiFtXz7gtyyM9bXRxdUSXVyLwk96TgHO3klDxO2inp+4lCe4nlz0eLEGh7OlAXwbFt328nEwUXbnd+3aVXleNzc3+Pj4wM7ODtu3b8fQoUOVzz19+hSbN29GUFBQqessLCxEnz59IAgCli9fDqBok8y45Ce4n/EUIzafBwAY6Grii7YNMHOtNto7mzPoULnQ0JDA3cYI7jZGGN/JCQlpucrgc/Zu0f+TuJQnWHr0FswNdOD/vDfVt4GpclHKV30YiI+PB1A0rmfevHnw8PDAxo0b4efnh8uXL5e4bMqkSZOQlZUFZ2dnSKVSyOVyzJo1CwMGDMDF+5kYsy1WOV5tcBt7jH7PHn7vtVP5wFCTlTnsDBs2DGPGjEFhYSE6dOgAAAgLC8N3332H8ePHl2tx5bVPClF1kSR7iiHrz+FaUhZ0tTSwuJ+nsmemvJnoa6OLqxW6uBb1iqZl56v0/NxIyVaGn3Wn7kIiAZpY1lEOeG5pb6Jcs8TIyAiNGzfGrVu3VL7H77//jtzcXHz++eelqulF0Ll37x6OHDmC2rUNcOBSEhaF3cS5W7nIy0qHga4mhrZ1wOA2DtDXkmB8enqJtwuI3pataS0MaeuAIc/H+RyLe4TQqyk4FpeK1Cf52ByZgM3Px/m0a/x8nE87P2Wv6H8/DDRp0gQAMHz4cOWK5p6enggLC8PatWsRHBz80jq2b9+OTZs2YfPmzWjatCliY2MxZswYXMnUwBlpMzxTCLCoo4N5n7ijlb0RevfurfKBoaYrc9iZMGEC0tLS8M0336CgoGgRKl1dXUycOBGBgYHlWpy3tzd27tyJwMBA/Pjjj3BwcMDChQsxYMAA5THfffcdcnJy8OWXXyqX1j948GCp19ghqiquPszCkPVRSM7Kg1ltHawZ6AV3G6NK+/6mtXXwfjMrvN+sKPw8zs5HZHw6IuIf40x8Om6lZuNqUhauJmVh7ak7kEgAF6s68G1gCncLXdy6fRufffaZyjnXrFmDjz76CHXr1n3t938RdG7evIkjR44gKqkQi347iWtJWQAAY3tXpOXnYGlHI7zbpmjrjMOHD0OhUMDHx6ec/zaIijOqpY0envXQw7Me8p/JcSY+HaFXk/H31VQkZ+XhwOVkHLhcNM7Hy85YOYbOzvSfDwMvOgleNkQjISGhxO89YcIETJo0SbmkSh3rBrDYfhzb/m8p6g1bgW5uVpjVwxX6WhKVDwzs1SnyxlPPs7Ozce3aNejp6aFRo0bVegwM19khdTsWl4qATTHIKZDD0bw21g3yrnIDClOf5CEyvqjn59dFM1Fg7QlNQ3M8e5IO2clNKEiNh9+U3/Cue0P4NjSFsTwdLZo1xV9//YUuXboUO5+zszOCg4PRs2dPFBYW4uOPP0ZMTAwmL1yP368WjScCAAMjI3zRrhGGtm2Afr0/QkpKClasWKGceu7l5fXSSRNElUUQhKJxPldTcPj5OJ9/a2CkgTOz+mPEt5MQPOU72NraYMiQIZgxY4byGE9PT3Tt2lU5Iee/TE1NMXPmTHz11VfYFpWIH/ddRVL4FuReDsO2vyPR3cMaz549U35gOHr0aKk+ZFR3olhnp7Iw7JA6bY5MQNDuy5ArBLRuaIrln7aAoV7VHn/Sr18/HAsPR1paOvQMjKBn0xSaPv+DlvE/EwUyj29AwfVwfL/xCFo71oW3g4nKVHCJRIKff1mJlp17QZrzCK3cXV72rbD3wGF80KUjgKJFBUeMGKGyqODixYtLnNJOpA7DR4yGsbMvrj7RQfS120g/XvRhwPqL5bCyMEed24dxZscKrFr9f2jp1RwbNmzAvHnzcPnyZeXuAH5+fujZsydGjBgBABg0aBAOh/6NZn2/xZWndVCQchtZocswZMgQLFs0X+UDw759+1SWZDExMYG2tjgWEfyvCg07586dw/bt25GQkKC8lfXCn3/+WfZq1Yxhh9RBoRAw91AcVoTfBgD0bl4fwb2aQVuzeu06/EKyLA+Rd4rG+0TcTsPdNNWdtqUaErjWM4Tv89le99Jylbts/5u+thQDW9tj2DsNSpwNRlSV9evXD8ePH0daWhrMzOqigWsL2HUZgthMXWTnPwMAyM7sQPb5/UBeNho4N8XP837C+x3bK89hb2+PQYMGYfr06QCAfdG3MXTkBKRdOQlFrgym5hb4ctBnmDZtGrS1tV+5cvnRo0fx3nvvVfRlq0WFhZ2tW7fi888/R+fOnXH48GF06tQJN27cQEpKCnr27Il169a9dfGVjWGHKlteoRzjd1zA/otFyzWM9W+MUX6OolrZNEn2tGjMz+00nLmTptyI81U+a2WHsR0bi2Ype6J/ezHO5+/n6/kkyf7Z+kVDAnjZm6Dj81Wc7c30kSR7iutJWdgd+xC7Yh8CAJwsDLCwnweaWPG9CqjAsOPm5obhw4cjICAABgYGuHDhAhwcHDB8+HBYWVlVy5WJGXaoMqXnFGDYxnOIvpcBLakEc3q7oVfz+uouq8I9zHyqnOl1NC4Vj54U32V7y7BW8G1oqobqiCqXIAi4/CDr+fYVKcqB+C+YG+jg0ZN8lU1Nh73jgPGdnJRT3KkCw46+vj6uXLkCe3t7mJqa4tixY2jWrBmuXbuGDh06lPvCgpWBYYcqy53HORi87izupuWijq4mVn7mVSPf3F+2y7ZUIsHJSe1FsV0BUVklphet5/P3tRScuZ0G+X/emTUkwKlJHfj/4z9K+/5d5sEBxsbGePKkaKR5vXr1cPnyZQBAZmYmcnNf301NVFOdu5uOXstO4W5aLuob6+HPb1rXyKADFK0AHdyrGaTPb9u92OOLv8ipprIxqYXBbRyw6YtWWPGpV7HnFQJw9zHfY99UmdfZadeuHUJDQ9GsWTN88sknGD16NI4cOYLQ0FD4+flVRI1E1d7eCw8xfkfRZp7u9Q3xfwO9y20H8uqqr7ct2jWuK4pdtonKk2v9OtCQoFjPp71Z1VqOojop822s9PR05OXlwdraGgqFAnPnzsXp06fRqFEjTJkyBcbG1W8PDt7GoooiCAJWhMdjzsGizTw7uVhgUT9P6GnznjsRlWxbVAIm/3kZckFQ9nz29S6+0W5NVyFjdp49e4bNmzejc+fOKnP4qzuGHaoIhXIFpu6+jC1nEwEAQ9o44PtuTZSbeRIRvUqS7Cl7Pl+jtO/fZbqNpampia+++grXrl176wKJxOxJXiECNp/H8RuPoCEBpn7ggkFt3n4zTyKqOawM9RhyykmZx+y0bNkSsbGxyp3HiUhVkuwpBq+LwvXkJ9DTkmJxf090dBFPTygRUXVT5rDzzTffYNy4cUhMTESLFi2gr6+v8rybm1u5FUdU3Vx5KMOQ9VFIycpHXQMdrB3ojWb1DdVdFhFRjVbmAcoaGsVnq0skEgiCAIlEArlcXm7FVRaO2aHycDQuFSOeb+bZ2KI21g7yRn1jzp4gIqooFTJmBwDu3LnzVoURidGmyHuYuvtKtdrMk4iopihz2OFYHaJ/KBQC5hy6jpXh8QCAj1vUx+ye1XczTyIiMSpz2Nm4ceMrn//888/fuBii6iSvUI7x2y9g/6WiLVLGdWyMkR3EtZknEZEYlHnMzn8XDSwsLERubi60tbVRq1YtpKenl2uBlYFjdqis0rLzMWzjOcQkZEJLKsHcj93Q01P8m3kSEVUlFTZmJyMjo1jbzZs38fXXX2PChAllPR1RtRP/KBuD10fh3vPNPFd97oVWDWrmHldERNVBmcPOyzRq1AghISH49NNPcf369fI4JVGVFHU3HcM2nkNmbiFsTPSwbpA3HM0N1F0WERG9QrmEHaBodeWHDx+W1+mIqpw9Fx7i2+0XUCBXwN3GCGsGesGsds3ezJOIqDooc9jZs2ePyteCICApKQlLly5FmzZtyq0woqpCEAQsO3YbPx2KAwB0bmqBhX25mScRUXVR5vmxPXr0UHn06tUL06dPh5ubG9auXVsRNRKpCAkJgUQiwZgxY5RtycnJ+Oyzz2BpaQl9fX00b94cf/zxxyvPI5fLERQUBAcHB+jp6aFhw4aYMWMG/j1mv1CuwLDFuzHhywFI+LkPHi78BDGLv8aj5AcVdXlERFTOytyzo1AoKqIOolKJiorCypUri21L8vnnnyMzMxN79uyBmZkZNm/ejD59+uDcuXPw9PR86bnmzJmD5cuXY8OGDWjatCnOnTuHwYMHw9DQEKNGjcKTvEJ89vMe7P1xMAzcOyLw+6n43zvOuHLlCnR1dSvjcomIqByU25gdooqWnZ2NAQMGYPXq1Zg5c6bKc6dPn8by5cvRsmVLAMCUKVPw888/Izo6usSwc/r0aXTv3h3dunUDANjb22PLli04e/YsHmY+xZD1UTixcRFqO3rjj3XL4NekaDPPhg0bVuBVEhFReSvzbazevXtjzpw5xdrnzp2LTz75pFyKInqZgIAAdOvWDf7+/sWea926NbZt24b09HQoFAps3boVeXl5eO+990o8X+vWrREWFoYbN24AAC5cuICTJ0/C1edd9Fx2CteSZMiLP4fPOrfC3DGfw9zcHD4+Pti1a1cFXSEREVWEMoed48eP4/333y/W3rVrVxw/frxciiL6r61btyImJgbBwcEvfX779u0oLCyEqakpdHR0MHz4cOzcuROOjo4lnnPSpEno168fnJ2doaWlBU9PT3w04AtsfGSDlKx82NcqhKLgKdYtX4guXbrg8OHD6NmzJ3r16oXw8PCKulQiIipnZb6NlZ2dDW1t7WLtWlpayMrKKpeiiP4tMTERo0ePRmhoaIljZYKCgpCZmYm///4bZmZm2LVrF/r06YMTJ06gWbNmL33N9u3bsWnTJmzevBlNmzbFL7//jdVzp8G4w1N06dkXQR0s4Twd6N69O8aOHQsA8PDwwOnTp7FixQq8++67FXXJRERUjsrcs9OsWTNs27atWPvWrVvh4uJSLkUR/Vt0dDRSU1PRvHlzaGpqQlNTE+Hh4Vi8eDE0NTVx+/ZtLF26FGvXroWfnx/c3d0xbdo0eHl54ZdffinxvBMmTMCkSZPQp09f7E3UxMH8xjDw7g4hdifWDfaGQ30raGpqFvt33aRJEyQkJFT0ZRMRUTkpc89OUFAQevXqhdu3b6NDhw4AgLCwMGzZsgU7duwo9wKJ/Pz8cOnSJZW2wYMHw9nZGRMnTkRubi4AQENDNbtLpdJXzh7Mzc2FXABGbInBX5eSAQDtGpvjerIWtKQagFQb3t7eiIuLU3ndjRs3YGdnVx6XRkRElaDMYefDDz/Erl27MHv2bPz+++/Q09ODm5sb/v77b3brU4UwMDCAq6urSpu+vj5MTU3h6uqKwsJCODo6Yvjw4Zg3bx5MTU2xa9cuhIaGYt++fcrX+Pn5oWfPnhgxYgQAoFOX9zEx6AcY+H8DfXN79G3wDGvWbMSQIUOUr5kwYQL69u2Ldu3aoX379jh48CD27t2LY8eOVcq1ExHR23ujqefdunVTTtclUjctLS389ddfmDRpEj788ENkZ2fD0dERGzZsUBlMf/v2bTx+/Ljoz4+ycc+5H7TispAZuhyyvCzsqGeN4cOHY+rUqcrX9OzZEytWrEBwcDBGjRoFJycn/PHHH2jbtm2lXycREb0ZifDv5WJLISoqCgqFAj4+PirtkZGRkEql8PLyKtcCK0Npt4in6i1J9hR3HudAlluISX9eguxpIWxNamHdYG80rFtb3eUREVEZlfb9u8wDlAMCApCYmFis/cGDBwgICCjr6YgqxbaoBLQJOYL/rY7E15tiIHtaCA8bI/z5TWsGHSIikSvzbayrV6+iefPmxdo9PT1x9erVcimKqDw9yMjFpD8v4b99mAv7enDXciKiGqDMPTs6OjpISUkp1p6UlARNTe4+QVVHalYefjl6C72Wny4WdAAgSZZX+UUREVGlK3M66dSpEwIDA7F7924YGhoCADIzMzF58mR07Nix3AskKotncgXCbzzC1qhEHLmeCrni5UPSpBIJ7M1qVXJ1RESkDmUOO/PmzUO7du1gZ2en3GAxNjYWFhYW+PXXX8u9QKLSSEjLxfZzidgRnYiUrHxlu5edMfp62yCvUI7pe65CLgiQSiSY3csVVoZ6aqyYiIgqS5lnYwFATk4ONm3ahAsXLijX2enfvz+0tLQqosYKx9lY1VP+MzkOXUnBtqgEnLqVpmw30ddGL8966NfSBo7mBsr2JNlT3H2cC3uzWgw6REQiUNr37zcKO2LDsFO9xCU/wbaoRPx5/j4ycwsBABIJ0NbRDP28beHvYg4dTamaqyQioopW2vfvNx5RfPXqVSQkJKCgoECl/aOPPnrTUxKVKCf/GfZdfIitUYk4n5CpbLcy1MUnXjb4pEV92JhwDA4RERVX5rATHx+Pnj174tKlS5BIJHjRMSSRSAAAcrm8fCukGksQBFy4L8PWswnYe+EhcgqK/m1pakjg38QCfVvaoF2jupBqSNRcKRERVWVlnno+evRoODg4IDU1FbVq1cKVK1dw/PhxeHl5cb+gKi4kJAQSiQRjxoxRtg0fPhwNGzaEnp4e6tati+7du+P69euvPM+gQYMgkUhUHl26dCl23P79++Hj4wM9PT0YGxujR48epaozM7cA607dQddFJ9Djl1PYGpWInAI5HMz0MamrM04HdsCKz1qgvZM5gw4REb1WmXt2IiIicOTIEZiZmUFDQwMaGhpo27atcu+g8+fPV0Sd9JaioqKwcuVKuLm5qbS3aNECAwYMgK2tLdLT0zF9+nR06tQJd+7cgVRa8riXLl26YN26dcqvdXRUF+f7448/MGzYMMyePRsdOnTAs2fPcPny5RLPp1AIOBOfhq1RiTh4JRkFz4p2K9fR1EC3Zlbo622Dlg4myh5EIiKi0ipz2JHL5TAwKJrhYmZmhocPH8LJyQl2dnaIi4sr9wLp7WVnZ2PAgAFYvXo1Zs6cqfLcl19+qfyzvb09Zs6cCXd3d9y9excNGzYs8Zw6OjqwtLR86XPPnj3D6NGj8dNPP2Ho0KHKdhcXl2LHpmTl4ffo+9h+LhH30nKV7U2s6qB/Sxt0d68Hw1rVc5YfERFVDWUOO66urrhw4QIcHBzg4+ODuXPnQltbG6tWrUKDBg0qokZ6SwEBAejWrRv8/f2LhZ1/y8nJwbp16+Dg4AAbG5tXnvPYsWMwNzeHsbExOnTogJkzZ8LU1BQAEBMTgwcPHkBDQwOenp5ITk6Gh4cHfvrpJ7i6uuKZXIFjcUUL/x2N+2fhv9o6mujuYY1+3rZwrVeHvThERFQuyhx2pkyZgpycHADAjz/+iA8++ADvvPMOTE1NsW3btnIvkN7O1q1bERMTg6ioqBKPWbZsGb777jvk5OTAyckJoaGh0NbWLvH4Ll26oFevXnBwcMDt27cxefJkdO3aFREREZBKpYiPjwcATJ8+HQsWLIC9vT3mz5+Pdu++i9Er92N/XDZSn/yz8J+3vTH6etvi/WaWqKXNLUeIiKh8lfmdpXPnzso/Ozo64vr160hPT4exsTE/iVcxiYmJGD16NEJDQ6Grq1vicQMGDEDHjh2RlJSEefPmoU+fPjh16lSJr+nXr5/yz82aNYObmxsaNmyIY8eOwc/PDwpF0Xib77//Ht0+6oHDV1OgeOcryHbtx+L/+w0GHl1hoq+N3s3roa+36sJ/RERE5a1cPkabmJiUx2monEVHRyM1NVVll3q5XI7jx49j6dKlyM/Ph1QqhaGhIQwNDdGoUSO0atUKxsbG2LlzJ/r371+q79OgQQOYmZnh1q1b8PPzg5WVFQDgTIYelgWHKRf+0zSyhLVmDuYMaA7/JhbQ1izzZEAiIqIy4z0DEfPz88OlS5dU2gYPHgxnZ2dMnDjxpbOtBEGAIAjIz88v9lxJ7t+/j7S0NBibmWNbVAJ+jSkEpFrYfewcDNw7wcpQF708LDF3bQZGfuSL95tZvfW1ERERlRbDjogZGBjA1dVVpU1fXx+mpqZwdXVFfHw8tm3bhk6dOqFu3bq4f/8+QkJCoKenh/fff1/5GmdnZwQHB6Nnz57Izs7GDz/8gN69e8PS0hK3bt3CyLHfwsjSBtOipXgaVRSuDD3fR37kVkz8uA0+aN0QC+bPg4ZEgk8++aRS/w6IiIgYdmowXV1dnDhxAgsXLkRGRgYsLCzQrl07nD59Gubm5srj4uLiIJPJAABSqRQXL17E+vUbkJGZCS0DU2jausOo97d4qpCigZk++nrb4MPvfsOC2dMxf/JIzHz6FD4+Pjhy5AiMjY3VdblERFRDlXkj0OPHj6N169bQ1FTNSc+ePcPp06fRrl27ci2wMnAj0FdLkj3Fncc5sDOphXtpudgSlYhDl5NRIFdd+K9fS1t423OgOhERVY4K2wi0ffv2SEpKUvnkDwAymQzt27fn3lgisy0qAYF/XoLiJZHY5fnCfx951IOhHhf+IyKiqqnMYUcQhJd+ck9LS4O+vn65FEVVQ5LsKSb9eQn/7fvr6WmNoW0bwLWeoXoKIyIiKoNSh51evXoBKNrdfNCgQSp7Icnlcly8eBGtW7cu/wpJLQqeKTBr/7ViQQcA+njZMugQEVG1UeqwY2hY9OYmCAIMDAygp6enfE5bWxutWrXCsGHDyr9CqnT30nIwast5XLgvK/acVCKBvVktNVRFRET0Zkoddl7scG1vb49vv/2Wt6xEatf5B5iy6zKy85+hjm7RXlWbIxMhFwRIJRLM7uUKK0O915+IiIioiijzbKynT59CEATUqlX06f7evXvYuXMnXFxc0KlTpwopsqJxNhaQnf8MU3dfxp8xDwAALe1N8HM/D9Qz0kOS7CnuPs6FvVktBh0iIqoyKmw2Vvfu3dGrVy989dVXyMzMRMuWLaGtrY3Hjx9jwYIF+Prrr9+qcKp8l+7LMHJLDO6m5UJDAozya4QR7R2hKS3azsHKUI8hh4iIqq0yb04UExODd955BwDw+++/w9LSEvfu3cPGjRuxePHici+QKo5CIWD18Xj0Wn4Kd9NyYW2oi61f+mKMf2Nl0CEiIqruytyzk5ubCwODol2qDx8+jF69ekFDQwOtWrXCvXv3yr1AqhiPnuRj/I4LOH7jEQCgS1NLhPRuBqNa2mqujIiIqHyV+eO7o6Mjdu3ahcTERBw6dEg5Tic1NbXGjnepbsJvPELXRcdx/MYj6GhqYFZPVyz/tDmDDhERiVKZe3amTp2K//3vfxg7diw6dOgAX19fAEW9PJ6enuVeIJWfgmcKzDsch1XH4wEAThYGWPI/TzS2MFBzZURERBWnzD07H3/8MRISEnDu3DkcOnRI2e7n54eff/65XIv7r5CQEEgkEowZM0bZlpeXh4CAAJiamqJ27dro3bs3UlJSKrSO6ujO4xz0Xn5aGXQ+a2WH3SPaMOgQEZHovdEoVEtLSxgYGCA0NBRPnz4FAHh7e8PZ2blci/u3qKgorFy5Em5ubirtY8eOxd69e7Fjxw6Eh4fj4cOHytWeqcifMffxweITuPRABkM9Laz8rAVm9HCFrpZU3aURERFVuDKHnbS0NPj5+aFx48Z4//33kZSUBAAYOnQoxo8fX+4FAkB2djYGDBiA1atXw9jYWNkuk8mwZs0aLFiwAB06dECLFi2wbt06nD59GmfOnKmQWqqTJ3mFGLstFuO2X0BOgRwtHUxwYPQ76NzUUt2lERERVZoyh52xY8dCS0sLCQkJyoUFAaBv3744ePBguRb3QkBAALp16wZ/f3+V9ujoaBQWFqq0Ozs7w9bWFhERESWeLz8/H1lZWSoPsbmQmIkPlpzEzvMPoCEBxnVsjC3DWsHaiOvlEBFRzVLmAcqHDx/GoUOHUL9+fZX2Ro0aVcjU861btyImJgZRUVHFnktOToa2tjaMjIxU2i0sLJCcnFziOYODg/HDDz+Ud6lVgkIhYPWJePx0KA7PFALqGelhUT8PeNmbqLs0IiIitShz2MnJyVHp0XkhPT1dZSf08pCYmIjRo0cjNDQUurq65XbewMBAjBs3Tvl1VlYWbGxsyu386pL6JA/jt1/AiZuPAQDvN7NEcE83GNbSUnNlRERE6lPm21jvvPMONm7cqPxaIpFAoVBg7ty5aN++fbkWFx0djdTUVDRv3hyamprQ1NREeHg4Fi9eDE1NTVhYWKCgoACZmZkqr0tJSYGlZcnjUnR0dFCnTh2VR3V3NC4VXReewImbj6GrpYHgXs3wy/+aM+gQEVGNV+aenblz58LPzw/nzp1DQUEBvvvuO1y5cgXp6ek4depUuRbn5+eHS5cuqbQNHjwYzs7OmDhxImxsbKClpYWwsDD07t0bABAXF4eEhATl+j9il/9MjrkH47Dm5B0AgLOlAZb090QjTiknIiIC8AZhx9XVFTdu3MDSpUthYGCA7Oxs9OrVCwEBAbCysirX4gwMDODq6qrSpq+vD1NTU2X70KFDMW7cOJiYmKBOnToYOXIkfH190apVq3KtpSqKf5SNUVvP4/KDogHWA33tEPh+E04pJyIi+pcyh52EhATY2Njg+++/f+lztra25VJYaf3888/Q0NBA7969kZ+fj86dO2PZsmWVWkNlEwQBv0ffx7Q9V5BbIIdRLS389LE7OrpYqLs0IiKiKkciCIJQlhdIpVIkJSXB3NxcpT0tLQ3m5uaQy+XlWmBlyMrKgqGhIWQyWZUfv/MkrxDf77yMPRceAgBaNTDBwr6esDQsvwHcRERE1UFp37/L3LMjCAIkEkmx9uzs7HKdMUXFnU/IwKit55GY/hRSDQnG+jfC1+85QqpR/OdBRERERUoddl5M1ZZIJAgKClKZfi6XyxEZGQkPD49yL5CK1s5Zcfw2Fhy+oVw7Z3F/D7Sw49o5REREr1PqsHP+/HkART07ly5dgra2tvI5bW1tuLu749tvvy3/Cmu41Kw8jN0ei1O30gAA3dysMLtnMxjqcUo5ERFRaZQ67Bw9ehRA0dTvRYsWVfmxLWJw5HoKvt1xEek5BdDTkmL6Ry7o42Xz0tuIRERE9HJlXlRw3bp1NSLoLF++HG5ubspFB319fXHgwAHl88OHD0fDhg2hp6eHunXronv37rh+/forzymRSF76+Omnn5THzJo1C76+vtDS0YO/R0Ok5xSgiVUd7B3ZBn29bRl0iIiIyqjMYaemqF+/PkJCQhAdHY1z586hQ4cO6N69O65cuQIAyh3Wr127hkOHDkEQBHTq1OmVs9GSkpJUHmvXroVEIlEuiAgAyRnZyLT0gp57FwDAoNb22PlNaziac5FAIiKiN1HmqediVNqpayYmJvjpp58wdOjQYs9dvHgR7u7uuHXrFho2bFiq79ujRw88efIEYWFhEAQBO84VrZ3ztFAORdxRpIetRnaW7I2vi4iISMwqbOp5TSSXy7Fjxw7k5OS8dBuKnJwcrFu3Dg4ODqXeUDQlJQX79+/Hhg0bkJVXiMl/XsK+i0kAgNYNTeFj54RpR3nLioiI6G3xNtYrXLp0CbVr14aOjg6++uor7Ny5Ey4uLsrnly1bhtq1a6N27do4cOAAQkNDVWapvcqGDRtgYGAAB6/2eH/RCey7mASphgTfdXHCr0N9ONuKiIionDDsvIKTkxNiY2MRGRmJr7/+GgMHDsTVq1eVzw8YMADnz59HeHg4GjdujD59+iAvL69U5167di2avfsBPl1/HvcznqK+sR52fOWLb7hIIBERUbnibaxX0NbWhqOjI4CiAclRUVFYtGgRVq5cCQAwNDSEoaEhGjVqhFatWsHY2Bg7d+5E//79X3neXQf+RlxcHLJaB0BbIeBDd2vM6umKOrrszSEiIipvDDtloFAokJ+f/9LnBEGAIAglPv/C31dTMGTyXGhbOsKwXiP82L0pPm5Rn1PKiYiIKgjDTgkCAwPRtWtX2Nra4smTJ9i8eTOOHTuGQ4cOIT4+Htu2bUOnTp1Qt25d3L9/HyEhIdDT08P777+vPIezszOCg4PRs2dP5BXKEXLgOtYevYrMK8fh0mMEdo9qi4Z1a6t834SEBKSnpyMhIQFyuRyxsbEAAEdHR9SurXosERERvR7DTglSU1Px+eefIykpCYaGhnBzc8OhQ4fQsWNHPHz4ECdOnMDChQuRkZEBCwsLtGvXDqdPn1bZDT4uLg7nbt6H5e3H+GHvVVxPfoKca8ehKZHgyPLJMDctHl6mTp2KDRs2KL/29PQEULSC9XvvvVfh101ERCQ2XGcHpZ+nXxbbohIQ+OclKP71t2uqr415n7ijvbN5yS8kIiKiUuE6O2qUJHtaLOgAwPrB3mhW30gtNREREdVUnHpeAe48zikWdAAgO7/krSSIiIioYjDsVAAHM338d6kcqUQCe7Na6imIiIioBmPYqQBWhnoI7tUM0ufTyaUSCWb3coWVoZ6aKyMiIqp5OGangvT1tkW7xnVx93Eu7M1qMegQERGpCcNOBbIy1GPIISIiUjPexiIiIiJRY9ghIiIiUWPYISIiIlFj2CEiIiJRY9ghIiIiUWPYISIiIlFj2CEiIiJRY9ghIiIiUWPYISIiIlFj2CEiIiJRY9ghIiIiUWPYISIiIlFj2CEiIiJRY9ghIiIiUWPYISIiIlFj2CEiIiJRY9ghIiIiUWPYISIiIlFj2CEiIiJRY9ghIiIiUWPYISIiIlFj2CEiIiJRY9ghIiIiUWPYISIiIlFj2CEiIiJRY9ghIiIiUWPYISIiIlFj2CEiIiJRY9ghIiIiUWPYISIiIlFj2CEiIiJRY9ghIiIiUWPYISIiIlFj2CEiIiJRY9ghIiIiUWPYISIiIlFj2CEiIiJRY9ghIiIiUWPYISIiIlFj2CEiIiJRY9ghIiIiUWPYISIiIlGr0mEnODgY3t7eMDAwgLm5OXr06IG4uDiVY/Ly8hAQEABTU1PUrl0bvXv3RkpKipoqJiIioqqmSoed8PBwBAQE4MyZMwgNDUVhYSE6deqEnJwc5TFjx47F3r17sWPHDoSHh+Phw4fo1auXGqsmIiKiqkQiCIKg7iJK69GjRzA3N0d4eDjatWsHmUyGunXrYvPmzfj4448BANevX0eTJk0QERGBVq1aleq8WVlZMDQ0hEwmQ506dSryEoiIiKiclPb9u0r37PyXTCYDAJiYmAAAoqOjUVhYCH9/f+Uxzs7OsLW1RURERInnyc/PR1ZWlsqDiIiIxKnahB2FQoExY8agTZs2cHV1BQAkJydDW1sbRkZGKsdaWFggOTm5xHMFBwfD0NBQ+bCxsanI0omIiEiNqk3YCQgIwOXLl7F169a3PldgYCBkMpnykZiYWA4VEhERUVWkqe4CSmPEiBHYt28fjh8/jvr16yvbLS0tUVBQgMzMTJXenZSUFFhaWpZ4Ph0dHejo6FRkyURERFRFVOmeHUEQMGLECOzcuRNHjhyBg4ODyvMtWrSAlpYWwsLClG1xcXFISEiAr69vZZdLREREVVCV7tkJCAjA5s2bsXv3bhgYGCjH4RgaGkJPTw+GhoYYOnQoxo0bBxMTE9SpUwcjR46Er69vqWdiERERkbhV6annEonkpe3r1q3DoEGDABQtKjh+/Hhs2bIF+fn56Ny5M5YtW/bK21j/xannRERE1U9p37+rdNipLAw7RERE1Y8o19khIiIiKiuGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNdGEnV9++QX29vbQ1dWFj48Pzp49q+6SiIiIqAoQRdjZtm0bxo0bh2nTpiEmJgbu7u7o3LkzUlNT1V0aERERqZkows6CBQswbNgwDB48GC4uLlixYgVq1aqFtWvXqrs0IiIiUjNNdRfwtgoKChAdHY3AwEBlm4aGBvz9/REREfHS1+Tn5yM/P1/5tUwmAwBkZWVVbLFERERUbl68bwuC8Mrjqn3Yefz4MeRyOSwsLFTaLSwscP369Ze+Jjg4GD/88EOxdhsbmwqpkYiIiCrOkydPYGhoWOLz1T7svInAwECMGzdO+bVCoUB6ejpMTU0hkUjK7ftkZWXBxsYGiYmJqFOnTrmdl94Mfx5VD38mVQt/HlULfx6vJwgCnjx5Amtr61ceV+3DjpmZGaRSKVJSUlTaU1JSYGlp+dLX6OjoQEdHR6XNyMiookpEnTp1+A+1CuHPo+rhz6Rq4c+jauHP49Ve1aPzQrUfoKytrY0WLVogLCxM2aZQKBAWFgZfX181VkZERERVQbXv2QGAcePGYeDAgfDy8kLLli2xcOFC5OTkYPDgweoujYiIiNRMFGGnb9++ePToEaZOnYrk5GR4eHjg4MGDxQYtVzYdHR1Mmzat2C0zUg/+PKoe/kyqFv48qhb+PMqPRHjdfC0iIiKiaqzaj9khIiIiehWGHSIiIhI1hh0iIiISNYYdIiIiEjWGnQr0yy+/wN7eHrq6uvDx8cHZs2fVXVKNFBwcDG9vbxgYGMDc3Bw9evRAXFycusui50JCQiCRSDBmzBh1l1JjPXjwAJ9++ilMTU2hp6eHZs2a4dy5c+ouq8aSy+UICgqCg4MD9PT00LBhQ8yYMeO1+z9RyRh2Ksi2bdswbtw4TJs2DTExMXB3d0fnzp2Rmpqq7tJqnPDwcAQEBODMmTMIDQ1FYWEhOnXqhJycHHWXVuNFRUVh5cqVcHNzU3cpNVZGRgbatGkDLS0tHDhwAFevXsX8+fNhbGys7tJqrDlz5mD58uVYunQprl27hjlz5mDu3LlYsmSJukurtjj1vIL4+PjA29sbS5cuBVC0qrONjQ1GjhyJSZMmqbm6mu3Ro0cwNzdHeHg42rVrp+5yaqzs7Gw0b94cy5Ytw8yZM+Hh4YGFCxequ6waZ9KkSTh16hROnDih7lLouQ8++AAWFhZYs2aNsq13797Q09PDb7/9psbKqi/27FSAgoICREdHw9/fX9mmoaEBf39/REREqLEyAgCZTAYAMDExUXMlNVtAQAC6deum8v+EKt+ePXvg5eWFTz75BObm5vD09MTq1avVXVaN1rp1a4SFheHGjRsAgAsXLuDkyZPo2rWrmiurvkSxgnJV8/jxY8jl8mIrOFtYWOD69etqqoqAoh62MWPGoE2bNnB1dVV3OTXW1q1bERMTg6ioKHWXUuPFx8dj+fLlGDduHCZPnoyoqCiMGjUK2traGDhwoLrLq5EmTZqErKwsODs7QyqVQi6XY9asWRgwYIC6S6u2GHaoRgkICMDly5dx8uRJdZdSYyUmJmL06NEIDQ2Frq6uusup8RQKBby8vDB79mwAgKenJy5fvowVK1Yw7KjJ9u3bsWnTJmzevBlNmzZFbGwsxowZA2tra/5M3hDDTgUwMzODVCpFSkqKSntKSgosLS3VVBWNGDEC+/btw/Hjx1G/fn11l1NjRUdHIzU1Fc2bN1e2yeVyHD9+HEuXLkV+fj6kUqkaK6xZrKys4OLiotLWpEkT/PHHH2qqiCZMmIBJkyahX79+AIBmzZrh3r17CA4OZth5QxyzUwG0tbXRokULhIWFKdsUCgXCwsLg6+urxspqJkEQMGLECOzcuRNHjhyBg4ODukuq0fz8/HDp0iXExsYqH15eXhgwYABiY2MZdCpZmzZtii3FcOPGDdjZ2ampIsrNzYWGhurbs1QqhUKhUFNF1R97dirIuHHjMHDgQHh5eaFly5ZYuHAhcnJyMHjwYHWXVuMEBARg8+bN2L17NwwMDJCcnAwAMDQ0hJ6enpqrq3kMDAyKjZfS19eHqakpx1GpwdixY9G6dWvMnj0bffr0wdmzZ7Fq1SqsWrVK3aXVWB9++CFmzZoFW1tbNG3aFOfPn8eCBQswZMgQdZdWbXHqeQVaunQpfvrpJyQnJ8PDwwOLFy+Gj4+PusuqcSQSyUvb161bh0GDBlVuMfRS7733Hqeeq9G+ffsQGBiImzdvwsHBAePGjcOwYcPUXVaN9eTJEwQFBWHnzp1ITU2FtbU1+vfvj6lTp0JbW1vd5VVLDDtEREQkahyzQ0RERKLGsENERESixrBDREREosawQ0RERKLGsENERESixrBDREREosawQ0RERKLGsENE9B/Hjh2DRCJBZmamukshonLAsENERESixrBDREREosawQ0RVjkKhQHBwMBwcHKCnpwd3d3f8/vvvAP65xbR//364ublBV1cXrVq1wuXLl1XO8ccff6Bp06bQ0dGBvb095s+fr/J8fn4+Jk6cCBsbG+jo6MDR0RFr1qxROSY6OhpeXl6oVasWWrduXWx3cCKqHhh2iKjKCQ4OxsaNG7FixQpcuXIFY8eOxaefforw8HDlMRMmTMD8+fMRFRWFunXr4sMPP0RhYSGAopDSp08f9OvXD5cuXcL06dMRFBSE9evXK1//+eefY8uWLVi8eDGuXbuGlStXonbt2ip1fP/995g/fz7OnTsHTU1N7jpNVE1xI1AiqlLy8/NhYmKCv//+G76+vsr2L774Arm5ufjyyy/Rvn17bN26FX379gUApKeno379+li/fj369OmDAQMG4NGjRzh8+LDy9d999x3279+PK1eu4MaNG3ByckJoaCj8/f2L1XDs2DG0b98ef//9N/z8/AAAf/31F7p164anT59CV1e3gv8WiKg8sWeHiKqUW7duITc3Fx07dkTt2rWVj40bN+L27dvK4/4dhExMTODk5IRr164BAK5du4Y2bdqonLdNmza4efMm5HI5YmNjIZVK8e67776yFjc3N+WfraysAACpqalvfY1EVLk01V0AEdG/ZWdnAwD279+PevXqqTyno6OjEnjelJ6eXqmO09LSUv5ZIpEAKBpPRETVC3t2iKhKcXFxgY6ODhISEuDo6KjysLGxUR535swZ5Z8zMjJw48YNNGnSBADQpEkTnDp1SuW8p06dQuPGjSGVStGsWTMoFAqVMUBEJF7s2SGiKsXAwADffvstxo4dC4VCgbZt20Imk+HUqVOoU6cO7OzsAAA//vgjTE1NYWFhge+//x5mZmbo0aMHAGD8+PHw9vbGjBkz0LdvX0RERGDp0qVYtmwZAMDe3h4DBw7EkCFDsHjxYri7u+PevXtITU1Fnz591HXpRFRBGHaIqMqZMWMG6tati+DgYMTHx8PIyAjNmzfH5MmTlbeRQkJCMHr0aNy8eRMeHh7Yu3cvtLW1AQDNmzfH9u3bMXXqVMyYMQNWVlb48ccfMWjQIOX3WL58OSZPnoxvvvkGaWlpsLW1xeTJk9VxuURUwTgbi4iqlRczpTIyMmBkZKTucoioGuCYHSIiIhI1hh0iIiISNd7GIiIiIlFjzw4RERGJGsMOERERiRrDDhEREYkaww4RERGJGsMOERERiRrDDhEREYkaww4RERGJGsMOERERiRrDDhEREYna/wObtA88RS/mpQAAAABJRU5ErkJggg==", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAG2CAYAAACZEEfAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABjDUlEQVR4nO3dd3zM9x8H8NdlRxbZCTIQYsaIETGKVKiqVauoVTqiRNRs0UXQGrVLCWpWa1V/RgSxI0RIiiSIJMgQSS5L5n1/f6Q5ToacXNbX6/l43KPyve997n2ac6/7fD9DIgiCACIiIiKRUqvqAoiIiIgqEsMOERERiRrDDhEREYkaww4RERGJGsMOERERiRrDDhEREYkaww4RERGJGsMOERERiRrDDhEREYkaww4RERGJWpWGnXPnzqF///6wtraGRCLBoUOHFO4XBAELFiyAlZUVdHV14ebmhoiICIVzkpKSMGrUKBgaGqJ27dqYOHEi0tPTK/FVEBERUXVWpWEnIyMDTk5OWLduXbH3L1u2DKtXr8bGjRsREBAAPT09uLu7IysrS37OqFGj8O+//8LX1xdHjx7FuXPnMHny5Mp6CURERFTNSarLRqASiQQHDx7EwIEDART06lhbW2PGjBn46quvAABSqRQWFhbYtm0bRowYgTt37qBZs2YIDAyEs7MzAOD48eN477338OjRI1hbW1fVyyEiIqJqQqOqCyhJZGQk4uLi4ObmJj9mZGSEjh074vLlyxgxYgQuX76M2rVry4MOALi5uUFNTQ0BAQEYNGhQsW1nZ2cjOztb/rNMJkNSUhJMTEwgkUgq7kURERGRygiCgLS0NFhbW0NNreSLVdU27MTFxQEALCwsFI5bWFjI74uLi4O5ubnC/RoaGjA2NpafUxxvb2989913Kq6YiIiIqkJMTAzq1atX4v3VNuxUpLlz58LLy0v+s1QqhY2NDWJiYmBoaFiFlREREVFZpaamon79+jAwMCj1vGobdiwtLQEA8fHxsLKykh+Pj49H69at5eckJCQoPC4vLw9JSUnyxxdHW1sb2traRY4bGhoy7BAREdUwrxuCUm3X2bG3t4elpSX8/Pzkx1JTUxEQEAAXFxcAgIuLC1JSUnD9+nX5OadPn4ZMJkPHjh0rvWYiIiKqfqq0Zyc9PR337t2T/xwZGYng4GAYGxvDxsYGnp6e+PHHH+Hg4AB7e3vMnz8f1tbW8hlbTZs2RZ8+fTBp0iRs3LgRubm5mDJlCkaMGMGZWERERASgisPOtWvX0KNHD/nPheNoxo4di23btmHWrFnIyMjA5MmTkZKSgi5duuD48ePQ0dGRP2bXrl2YMmUKevXqBTU1NQwZMgSrV6+u9NdCRERE1VO1WWenKqWmpsLIyAhSqZRjdoiIiGqIsn5+V9sxO0RERESqwLBDREREosawQ0RERKLGsENERESixrBDREREosawQ0RERKLGsENERESixrBDREREosawQ0RERKLGsENERESixrBDREREosawQ0RERKLGsENERESixrBDREREosawQ0RERKLGsENERESixrBDREREosawQ0RERKLGsENERESixrBDREREosawQ0RERKLGsENERESixrBDREREosawQ0RERKLGsENERESixrBDREREosawQ0RERKLGsENERESixrBDREREosawQ0RERKLGsENERESixrBDREREosawQ0RERKLGsENERESixrBDREREosawQ0RERKLGsENERESixrBDREREosawQ0RERKLGsENERESixrBDREREosawQ0RERKLGsENERESixrBDREREosawQ0RERKLGsENERESixrBDREREosawQ0RERKLGsENERESixrBDREREosawQ0RERKLGsENERESixrBDREREosawQ0RERKLGsENERESixrBDREREosawQ0RERKLGsENERESixrBDREREosawQ0RERKLGsENERESixrBDREREosawQ0RERKJWrcNOfn4+5s+fD3t7e+jq6qJhw4b44YcfIAiC/BxBELBgwQJYWVlBV1cXbm5uiIiIqMKqiYiIqDqp1mFn6dKl2LBhA9auXYs7d+5g6dKlWLZsGdasWSM/Z9myZVi9ejU2btyIgIAA6Onpwd3dHVlZWVVYOREREVUXEuHlbpJq5v3334eFhQW2bNkiPzZkyBDo6upi586dEAQB1tbWmDFjBr766isAgFQqhYWFBbZt24YRI0aU6XlSU1NhZGQEqVQKQ0PDCnktREREpFpl/fyu1j07nTt3hp+fH8LDwwEAN2/exIULF9C3b18AQGRkJOLi4uDm5iZ/jJGRETp27IjLly+X2G52djZSU1MVbkRERCROGlVdQGnmzJmD1NRUODo6Ql1dHfn5+Vi0aBFGjRoFAIiLiwMAWFhYKDzOwsJCfl9xvL298d1331Vc4URERFRtVOuenT/++AO7du3C7t27ERQUhO3bt+Pnn3/G9u3by9Xu3LlzIZVK5beYmBgVVUxERETVTbXu2Zk5cybmzJkjH3vTsmVLREVFwdvbG2PHjoWlpSUAID4+HlZWVvLHxcfHo3Xr1iW2q62tDW1t7QqtnYiIiKqHat2zk5mZCTU1xRLV1dUhk8kAAPb29rC0tISfn5/8/tTUVAQEBMDFxaVSayUiojcXK32OS/cTESt9XtWlkAhV656d/v37Y9GiRbCxsUHz5s1x48YNrFixAhMmTAAASCQSeHp64scff4SDgwPs7e0xf/58WFtbY+DAgVVbPBERlcm+wGjMPRACmQCoSQDvwS0xvL1NVZdFIlKtw86aNWswf/58fPHFF0hISIC1tTU+/fRTLFiwQH7OrFmzkJGRgcmTJyMlJQVdunTB8ePHoaOjU4WVExFRWcRKn8uDDgDIBGDegVB0a2wGKyPdqi2ORKNar7NTWbjODhFR1dgXGI3Zf4UUOb5nUie4NDSpgoqoJhHFOjtERCRef15/hAWHQ4u9z8yAk0hIdRh2iIioUmXl5mPOX7fw1f6byM4T4GChDzWJ4jlrT0eAFx5IVar1mB0iIhKX6GeZ+HzXdfz7JBUSCeDZqzGm9GyEhLQsPEzMROrzHHyx+wYOBT+BayNTDHWuX9Ulkwgw7BARUaXwvR0Prz+CkZaVB2M9LfwyojW6OpgBAKyMdOUDkr3ezcBPJ8Kw4PC/aGNTG43MDaqybBIBXsYiIhI5Ozs7SCSSIjcPDw8AwKeffoqGDRtCV1cXZmZmGDBgAO7evVvm9j/77DNIJBKsWrVK4Xh4eDgGDBgAU1NT6NTSR//ePfE0LAhtbWrj6Jdd5EHnVZ93b4gujUzxPDcfU3bfQFZu/hu/diKAYYeISPQCAwMRGxsrv/n6+gIAhg4dCgBo164dfHx8cOfOHZw4cQKCIKB3797Iz399yDh48CCuXLkCa2vrIve9//77yMzKQdvPV8Jk9Apomtsj6eAPWPWBPaxrlzytXE1NghXDnWCqr4W7cWn44ejtN3zlRAUYdoiIRM7MzAyWlpby29GjR9GwYUN0794dADB58mR069YNdnZ2aNu2LX788UfExMTg4cOHpbb7+PFjfPnll9i1axc0NTUV7ktMTERERASi6/dGeK4xalvaYOu6lcjNfo7wu68PL+YGOlgxrDUAYFdANP4XEvtGr50IYNghInqr5OTkYOfOnZgwYQIkEkmR+zMyMuDj4wN7e3vUr1/y4GCZTIYxY8Zg5syZaN68+Sv3CfgjJAWaxvXwJPA4GtbWwIHPOyHq4mGYm5ujXbt2Zaq1W2MzfP5OQwDA7L9uISYpU4lXSvQCww4R0Vvk0KFDSElJwbhx4xSOr1+/Hvr6+tDX18exY8fg6+sLLS2tEttZunQpNDQ0MHXqVIXj0sxcTP79GpadCIP58B9RKz0GZ+b1RfP6plixYgWOHz+OOnXqlLler3cbo61NbaRl5eHLPTeQmy9T6vUSAQw7RERlUtog36SkJHz55Zdo0qQJdHV1YWNjg6lTp0IqlZba5rfffgtHR0fo6emhTp06cHNzQ0BAwGufd8mSJW/8OrZs2YK+ffsWGWMzatQo3LhxA/7+/mjcuDGGDRuGrKysYtu4fv06fvnlF2zbtk2hdyhW+hzvrz2PU3cSoKkugWXoLrRrYofz58/j6tWrGDhwIPr374/Y2LJfktJUV8PqkW1gqKOB4JgU/Hwy7M1eOL3VuF0EuF0EEb3e06dPFQbshoaG4t1338WZM2dgamqKhQsXYty4cWjWrBmioqLw2WefoVWrVvjzzz9LbHP37t0wNzdHgwYN8Pz5c6xcuRL79+/HvXv3YGZWMFPJzs4OEydOxKRJk+SPMzAwgJ6entKvISoqCg0aNMCBAwcwYMCAEs/LyclBnTp18Ntvv2HkyJFF7l+1ahW8vLygpvbi+3J+fj4gUYO6gSlcvt6Dsbbp+PSjQUhOTlb4d9XBwQETJ07EnDlzlKr9eGgsPtsZBADYNr493mlirtTjSZzK+vnNdXaIiMqgMHwUWrJkiXyQr0QiwV9//SW/r2HDhli0aBFGjx6NvLw8aGgU/0/tRx99pPDzihUrsGXLFty6dQu9evWSHzcwMIClpWW5X4OPjw/Mzc3Rr1+/Us8TBAGCICA7O7vY+8eMGQM3NzcAwPPcPKzxu4fd330KveY98e7A4dgypSvO+R0HAIVAVPizTKb8pag+LawwppMtfr8ShRl/3MT/pnWFhSE3fKay4WUsIiIlvW6QLwD5N82Sgk5xbW7atAlGRkZwcnJSuG/JkiUwMTFBmzZt8NNPPyEvL0/pmmUyGXx8fDB27FiFmh48eABvb29cv34d0dHRuHTpEoYOHQpdXV2899578vMcHR1x8OBBAICJiQlatGgBXQs7fHM2FecSdSFR00Bv5ybYP3sIjGppwsXFBXXq1MHYsWNx8+ZNhIeHY+bMmYiMjHxt2CrJ1/2aoqmVIZ5l5GD6vmDky976CxNURgw7RERKKmmQb6HExET88MMPmDx58mvbOnr0KPT19aGjo4OVK1fC19cXpqam8vunTp2KvXv34syZM/j000+xePFizJo1S+maT506hejoaEyYMEHhuI6ODs6fP4/33nsPjRo1wvDhw2FgYIBLly7B3PzFpaKwsDCFMUhHbz3BB2suICw+Dab62jA10IZrI1N5+DM1NcXx48eRnp6Onj17wtnZGRcuXMDhw4eLhLmy0tFUx9qP2qCWljou3X+G9WfuvVE79PbhmB1wzA4RKcfd3R1aWlr4+++/i9yXmpqKd999F8bGxjhy5EiR9WdelZGRgdjYWCQmJmLz5s04ffo0AgICFILGy7Zu3YpPP/0U6enp0Nau3J3BY6XPERGfjr9vPsH+648AAB3tjbFmZBuYV+IlpT+vP8JX+29CTQLsneyCDvbGlfbcVL2U9fObPTtEREqIiorCqVOn8MknnxS5Ly0tDX369IGBgQEOHjz42qADAHp6emjUqBE6deqELVu2QENDA1u2bCnx/I4dOyIvL++1C/6p2r7AaLguOY2Pt16VB53P32mIXZ90rNSgAwAftquHwW3qQiYA0/beQHJGTqU+P9U8DDtEREooaZBvamoqevfuDS0tLRw5cgQ6Om8WAGQyWYkDgwEgODgYampqJfb8qFK+TEDoYyl+ORWO2X+F4OUhMmoS4GMXW2ioV83HyA8DW8DeVA+x0izM/PMmeJGiYrxuX7WsrCx4eHjAxMQE+vr6GDJkCOLj48vcfkn7qql6yQXOxiIiekms9DkiEzNgb6on34W7UEmDfAuDTmZmJnbu3InU1FSkpqYCKJjFpa6uDqBgkK+3tzcGDRqEjIwMLFq0CB988AGsrKyQmJiIdevW4fHjx/I9qy5fvoyAgAD06NEDBgYGuHz5MqZPn47Ro0crtTBfWWXl5uNGdAquPUzC1YdJuBGdgvTs4gdDywTgYWJmkb+jyqKnrYG1H7XBoHWXcOpOAnwuPsSELvZVUouYBQYGFrvkQuHv6PTp0/HPP/9g//79MDIywpQpUzB48GBcvHjxtW2Xtq8aAHz//fdFllx4U+zZISL6T+Glmo82B8B1yWnsC4xWuL+kQb5BQUEICAhASEgIGjVqBCsrK/ktJiZGfl5YWBiuRTxCrPQ51NXVcffuXQwZMgSNGzdG//798ezZM5w/f16+/YK2tjb27t2L7t27o3nz5li0aBGmT5+OTZs2qeT1JmfkwPd2PBb/7w4Grb+Ilt+ewMjNV7DcNxznIxKRnp0HA20NdLI3xqtzztQlEtiZ1lJJHa/z+PFjjB49GiYmJtDV1UXLli1x7do1NLc2wtf9miI/IxmeX0yCuYUVatWqhT59+iAiIqLUNt95551ieyxe7rETBAELFiyAlZUVdHV14ebmVmK7sdLnuHQ/EbHS52Wuv1B8fDzGjRsHa2vrKqu/JKXtqyaVSrFlyxasWLECPXv2lG8oe+nSJVy5cqXUdkvbV61Q4ZILhbc3WVuqEAcogwOUiQg4G5aAcT6BRY6b6Wujdi1NGOhowEBHE4a6hX/WgKHOiz8baL84p/A+fR0NqKsVxIR9gdGYe6DgUpCaBPAe3BLD29tU2usTBAGPkp8j8GESAh8mI/BhEu4lpBc5z8JQG+3tjNHezhjOdnXgaGkIdTUJ9gVGY96BUOQLAtQlEiwe3KJS6k9OTkabNm3Qo0cPfP755zAzM0NERAQaNmyIhg0bQiaTwapxa0izZWg2yAO/fdIVv65bg+PHj+P27dslfkAmJSUhJ+fFWJ9nz57ByckJv/32m3yW3dKlS+Ht7Y3t27fD3t4e8+fPR0hICG7fvq1wmbK0/7evq18QBHTu3BmamppYvnw5DA0N5dtqVFb9ZZWTkwNra2t4eXlh3rx5OH36NHr16oXk5GTUrl1bfp6trS08PT0xffr0YtuRyWRwc3PDgAEDMG3aNNjZ2cHT0xOenp7yc+zs7JCVlYXc3FzY2Njgo48+wvTp04ss5cBFBYmo2nn8+DFmz56NY8eOITMzE40aNYKPjw+cnZ0BFHzDnT17Nk6ePImUlBR069YNa9asgYODQ4ltHjhwAIsXL8a9e/eQm5sLBwcHzJgxA2PGjAEA5Obm4ptvvsH//vc/PHjwAEZGRnBzc8OSJUtgbW2NyMQMLD8ZhqO3it/C4Gl6Np6mlzyG5nX0tTVQS0sdCWkv2pAJwJy/QnDrsRTWRrqvhKeXApSOJgy0NaCmVvxaPq96+RKcuYEOwuLScC0qCVcjk3DtYTLiUotu/9DIXB/t7erIA069OrrFrh00vL0NujU2w8PETNiZ1qq0y1dLly5F/fr14ePjIz9mb//ictW9e/eQcD8ErT23IEnbAttv52L9+vWwsrLCnj17ih1IDgDGxoozuPbu3YtatWrJL88IgoBVq1bhm2++ka82vWPHDlhYWODQoUMYMWIEAOBJSibmHAhBYbeBTADmHQhFt8ZmsDLSfW39ERERuHLlCkJDQ+U9ehs2bIClpWWl1K+MV5dciIuLg5aWlkLQAQALCwvExcWV2E5J+6q9bOrUqWjbti2MjY1x6dIlzJ07F7GxsVixYoXSdQMMO0RUSZKTk+Hq6ooePXrg2LFj8m+4hWNPBEHAwIEDoampicOHD8u/4bq5uZX6DdfY2Bhff/01HB0doaWlhaNHj2L8+PEwNzeHu7s7MjMzERQUhPnz58PJyQnJycmYNm0a+vbrj34LtmFfYEyJi9OpSYDfxjpDW0MdaVm5SM3KQ1pWHtKychX+myr/OU9+Xk5ewSrB6dl5xY57EQDsuhJd5Hhx9LU1FAKQ4Us9SIX/jYhPw+HgJyh8JdoaasjOU1ypWENNghZ1jdDB3hjOtnXgbGcMY72SN/t8lZWRbqWP0Tly5Ajc3d0xdOhQ+Pv7o27duvjiiy/kYzkKB3N/O8gJXsfjcTj4CVwbmkJbWxsXLlwoMSy8asuWLRgxYoT89ywyMhJxcXHylaIBwMjICB07dsTly5fRf9CHOBT8GL/6P8Cr10fyBQF/BMbgix6Nylz/yz0tampqFV7/m4SdkvZVU0bhvmpBQUElLsgJAF5eXvI/t2rVClpaWvj000/h7e39RksuMOwQUaWoqG+477zzjsLP06ZNw/bt23HhwgW4u7vDyMgIvr6+8vulmblw/ugrrPcchme+16BhaI6ejub4qncThDxOKXKppqejxRu93uy8fHkAikxMx8Tt1xQ+FCUAhjnXh0wQCs7LfiUwPc9DTr5iYIotfV/RV55fBl1NNTj/12PT3s4YrevXhq6W+hu9nqry4MEDbNiwQX7pJDAwEFOnToWWlhbGjh0LR0dH2NjYYNfapfhs9GysPR8DjzkLkfjoUZk3HL169SpCQ0MVpvwX9kxYWCj+/9czMsGp62HwW+xX4uBtAFh5KgK7AqIRce8+Hjx4ff1z587Fr7/+Cj09PaxcuRKPKqj+1/W6lKRwyYUDBw7Ij1laWiInJwcpKSkKvTvx8fElbm9y/vx5JCQkwMbmxSXQ/Px8zJgxA6tWrSpxSYWXl1xo0qSJ0vUz7BBRpaiMb7iCIOD06dMICwvD0qVLFe7LzMmDz8WH2Oh/Hwl3wwFI0K5RPXw9qJ18Ubpm1oYqu1SjraEObX11mOprw95UD0sGt1R6zEtWbv4rPUnF9ybdf5oG//DEIo/f/LEzujiYFdNyzSGTyeDs7IzFixcDANq0aYPQ0FBs3LgRY8eOhaamJg4cOICJEydi/wBnSNTUoW3rBFPHjkApPQcv27JlC1q2bIkOHToUe39Ongwnb8dh55Uo+Ic/BSQSmGXnwd5UD6M62kBdTYIfj95BviBATQJ0b2yGkMdSJKRlIy9fBnXzRkhuNgSZBvUxaVLrEus3NjaGuro63Nzc0Ldv3zJPp39d/apQ3JIL7dq1g6amJvz8/DBkyBAABYPwo6Oj4eLiUmw7L++rVsjd3R1jxozB+PHjS3z+8i65wLBDRJWirN/Q3+QbrlQqRd26dZGdnQ11dXWsX78e7777LgAgN1+GvYExWO0Xgadp2RDycpB1cQd6vT8Ih6a7FelKr6hLNW8y5kVHUx06muowMyi92z5W+hyuS04rrIOjLpGgobl+ecuuclZWVmjWrJnCsaZNmypsvNquXTsEBwdDKpXicVIqPt51FyHrpyBRYvfa9jMyMrB37158//33CscLeyZ+PhSAc0n6ePrfmCtZZgoaNW2O9RM7wLWhqXw8VZ8Wlgr/b3PyZDj+bxxGbjaBmokNjoXG4VhoHBqY6aGuphmiol5cwny5/pycHJiZmaFjx47ysWzlqT8+Ph5WVlby4/Hx8WjdurXCuaUttwCUvOSCkZERJk6cCC8vLxgbG8PQ0BBffvklXFxc0KlTJ/l5Ly+5YGJiAhMTE4X2NTU1YWlpKe+xqYglFzj1nIgqhUwmQ9u2bbF48WK0adMGkydPxqRJk7Bx40YAkH/DDQ8Ph7GxMWrVqoUzZ86gb9++RXbOfpWBgQGCg4MRGBiIRYsWwcvLC6dPn8Hh4MdwW+GP+YdC8TQtG3WNNGERuAENzfRwYJdPqWMGKoKVkS5cGpqoPExZGenCe3BLqP/3egp7jqpqDRxVcnV1RVhYmMKx8PBw2NraFjnXyMgIzezrw6ujIXLi7uGRYXP8U8LA80L79+9HdnY2Ro8eDQCQyQScj3gK7/PPoK5XBz77/8bTtGyYGWhjckdLCAkR+Hr8QHR1MFMYOP7q/1stDTV84GSNAe490UQnDWM62UJPSx0Pnmbg8LnrSFY3wuw/byH08Ytrk0ZGRvKxbNeuXZMPLFam/kL29vawtLSEn5+f/FhqaioCAgIUel1eXW5hz9Wi48hKWnIBAFauXIn3338fQ4YMQbdu3WBpaalwqQtQXHKhLCpiyQVOPQennlPN8roZTSV9gC9btgwzZ84s9r78/Hx8++232LlzJ+Li4mBtbY1x48bhm2++kbcnCAIWLlyIzZs3IyUlBa6urtiwYUOpM6VeZmtri3fffRe//fab/NiGDRvw448/4vHjxwrnFvcNd926dWV6HkEQ8P6w0QgIiYD+wIUAAFN9bXzR3RaHl8/Ew8hInD59usi3SzGIlT6v9NlSFS0wMBCdO3fGd999h2HDhuHq1auYNGkSNm3ahFGjRgEo+MA3MzODjY0NQkJCMG3aNOjXa4z0LtNgoK2B/03riq89P0PdunXh7e2t0H7Xrl1Rt25dbNy6A39ef4RdAdGITMwAAEiv/ImMwL8w23s1BnVtg++/W4hbt24pNXX75fr7DRyMjftPYvPi2ajd2wP6zXsAAMyeBuH9Do4Y9k5rhN+9jWnTpqFdu3YKvVcff/xxqfXv3bu3yHMvXboUS5YsUZh6fiP4Jn77+xwik3MQFJ2C46FFx+/oaanDSPflZRZemSGoMFC+8M8vztHTejGDsKKXXODUcyIRet2MJgBFLvkcO3YMEydOlF9TL87SpUuxYcMGbN++Hc2bN8e1a9cwfvx4GBkZyaeHLlu2DKtXr1b4h9Pd3b3M//Ar+w0dgPwb7g8//PDa9gHgelQSlh4Pg39YAvIyn8NKWwOfdm+A0R3qYfyYj3D/3j2cOXNGlEEHqJrZUhWtffv2OHjwIObOnYvvv/8e9vb2WLVqlTzoAAW/815eXvJLNh9//DHmzPsaY3yuIyg6BVP23EBKVHSRHsKwsDBcuHABH36zAR0X+8lnrxloa2Bw27oY5fkLdq6vj02L5+CnlBR06dIFx48fV2qNmuLqX7/mF7Tp/SF+vxyFY6GxuB/1CD/sX43vMlNgZGyGMR+Pwc+LFX/no6NLrv/kyZNFnjczJw/vjpiEq+FP8NHYCXienoZaNs1g2HsuJu26VWrNGTn5yMjJxxNp0aUKykIiKZhBqKelobDcwavT8isTe3bAnh2qOebMmYOLFy/i/PnzZX7MwIEDkZaWptCd/ar3338fFhYWCrM5hgwZAl1dXezcuROCIMDa2hozZszAV199BaCg98XCwgLbtm0r0zTWN/2G/rpvuN7e3rBo0AzHowVcDIvD8weBSPbfjoFTFmLr4tnQ15Lgww8/RFBQEI4ePaowO8XY2BhaWmWfek01y6PkTLz3y3mkZuXho442eL+VFexN9VBbVwt/33qCnVeicOvRi8tITa0MMaaTLQa0toaeduX0BTxNy8Yf12KwOyAaj1NeXObp1tgMYzrZoqejOdTVJCWOq8mXCXj4LANhcWm4G5uKu3FpCItPQ3RSZpEp8UBB74q9qR4cLQ1hXVsHv52PhPDK/XsmdYKOpvpLSykUDoovuuzCq4Pmc/NfHyn2TOoEl4aq+cLBnh0iEXrdjKZXxcfH459//sH27dtLbbdz587YtGkTwsPD0bhxY9y8eRMXLlyQL+ClijU73vQb+vz58xXaefkbbkxSJg4FPkDQT2uQn/YMEg0tWNk2xOLffPDZ+IJFBR8+fIgjR44AQJGBmWfOnCkydZ3Eo16dWlj2YSt8tjMIuwOisTsgGhIUDPx+nluw35OWuhr6tbLC6E62aGtTu9LHcZkZaMOjRyN81r0hztxNwO9XonAu4inOhRfcrI100LKuEXzvxEMmFPSavNfCCrpa6giLS0N4fFqR9ZRebtvR0gBNLAzgaGUIR0sDNDLXh47mi+UHGpnrF5kl2LHBmwURQRCQnSeTh6PIxAxM2qG45EJlbjPyMvbsgD07VHMUdp97eXlh6NChCAwMxLRp0+TTWF+1bNkyLFmyBE+ePCm1610mk2HevHlYtmwZ1NXVkZ+fj0WLFmHu3LkAgEuXLsHV1RVPnjxRmNkxbNgwSCQS7Nu3T8WvtGSx0ucIjk7B6bsJOBT8WP5Nsl9LK3j1boyGZjV/BhKpTqz0OTp7n8arH3SWRjoY62KHYc71YKKv/CJ1FSnqWQZ2B0Rj37UYpGTmvvZ8XU11NLbQRxNLAzhaFoSaJpYGZX5dFTnWq6K3GWHPDpEIvW7NkVdt3boVo0aNeu0Ygz/++AO7du3C7t270bx5cwQHB8PT0xPW1tbFtlsVsnLzscI3HJvPPVD44OrqYIqZ7k3Qql7tqiqNqrHIxIwiQQcAlg91gmsj00qvpyxsTfQw972mmP5uY/ziF4ENZ+8XOWdI27p4t5kFHC0NUd+4lnwPtjdRkWO9qmqbkVcx7BDVIGVZc6TQ+fPnERYWVqZel5kzZ2LOnDnyy1EtW7ZEVFQUvL29MXbsWKXW7FCV5IwcXItKxrWHSQh8mIRbj1Lwam+9mgRY9mEr0Q3KJdWxN9WDmgRF1iBqYPbmO2hXFh1NdXzsYotf/e8Xqf8r9yY15ve+OgycZ9ghqkGUmdG0ZcsWtGvXDk5OTq9tNzMzs8hMD3V1dchkBeni5TU7CsNN4Zodn3/++Ru+mhcKd+S+FvXfjtyRSYgoZkfuV8kE4GFiZpX/Q0rVV+EaRK9eSqkpvzM1vf7qgmGHqAaZPn06OnfujMWLF8tnNG3atKnIYlupqanYv38/li9fXmw7vXr1wqBBgzBlyhQAQP/+/bFo0SLY2NigefPmuHHjBlasWCFfREwikcDT0xM//vgjHBwc5FPPra2tMXDgQKVfh0wmICw+DdceJuHqw4Lem9hiprk2NNP7b9NKY9iZ1sLQjZeLfMOtisGOVLNUl0spb6qm118dMOwQ1SBlmdEEAHv37oUgCBg5cmSRNmKlz3E7LAKtH71Yj2fNmjWYP38+vvjiCyQkJMDa2hqffvopFixYID9n1qxZyMjIwOTJk5FSypojxU2RzcrNR8hjKa5GJuHawyRci0pGWpbiJoqFO3K3t6uD9nbGaGdbp8gAS37DpTdVHS6llEdNr7+qcTYWOBuL3h4VvZrpy+1L/tsQMSM7DzdjpPIdvAvpaamjrW1BsHG2q4PW9Wujltbrv3+JcZVgInozZf38ZtgBww6JW2J6NsLi0nA1Mgm/+EUUub+2rqbCHj9vSiYTkPK85Gmypvra6GBfB862xmhvZ4ymVgbQUOf2fET05jj1nOgtk5Wbj4j4dNyJS0VYXFrBiqpxaUhMzy71caUFFFX4tFsDjOxgA1uTWpW+YBsREcCwQ1QtlbQ0PFDQgxKTnIk7sQWBJiw+FXdj0/DwWYbC4N1CEglga1wLtia1cC48scjS8DsndoSZQfkXVXualo3RWwKKDCAe52rHy01EVKUYdoiqmVfH1UzsYg/r2roIi0vDnbg0RMSnITMnv9jHGutp/bc0vMF/q6gaorGFvnwsTHGrmXZW0cJqDhYGHEBMRNUSx+yAY3ao+oiVPofrktPF9tC8TEtDrWB5eIuCpeEdrQqWhzfT137tpaKKHuDLAcREVFk4ZoeoBroYkVhs0GlnWweuDU3QxNIQTSwNYGdS640H91b0FFZOkSWi6oZhh6iauJeQjkX/u1vkuLpEgrUftWGAICJ6Q5z3SVQN3EtIx4hNV5CcmQNLQ20UzgTnuBciovJjzw6JUmmzmaqbewnpGLn5ChLTs+FoaYDdkzohOy+f416IiFSEPTv0Rh4/fozRo0fDxMQEurq6aNmyJa5duwYAyM3NxezZs9GyZUvo6enB2toaH3/8MZ48eVJqm+fOnUP//v1hbW0NiUSCQ4cOKdxf1nb3BUbDdclpfLQ5AK5LTmNfYLRKX7sq3X9aEHSepr0IOsZ6WrAy0oVLQxMGHSIiFWDYIaUlJyfD1dUVmpqaOHbsGG7fvo3ly5ejTp06AAp20A4KCsL8+fMRFBSEAwcOICwsDB988EGp7WZkZMDJyQnr1q0r9v6ytBsRn4Y5f4XIB/nKBGDegVDESp+r5sWr0P2nBZeuXg06RESkWpx6Dk49V9acOXNw8eJFnD9/vsyPCQwMRIcOHRAVFQUbm9fvxSSRSHDw4MHX7qhd2O6N2+E4GZUPn4uReJ4rK3Lejgnt0a2xeZnrrWj3n6Zj5KYrSPgv6Oz6pGORTS+JiKh0Zf38Zs8OKe3IkSNwdnbG0KFDYW5ujjZt2mDz5s2lPkYqlUIikaB27doqrSUiJh6QSPDh1ltYf/Z+sUEHALyPheFxSvXo3XnwUtBpYsGgQ0RU0Rh2SGkPHjzAhg0b4ODggBMnTuDzzz/H1KlTsX379mLPz8rKwuzZszFy5EiV9ZyFx6dh6u8BGO/hiVpNuyFPXQdO9Wvj1zHt4D24JdT/W1hPTQLoaKjhTmwq+q0+j7NhCSp5/jf14L8xOvKgM4lBh4ioonE2FilNJpPB2dkZixcvBgC0adMGoaGh2LhxI8aOHatwbm5uLoYNGwZBELBhw4ZyP3dQdDLWn7kP39DHeHpoMSAA73+2ANP6toJLQxP56sHvNDGTz2bKyxfw+a7rCH2civHbAvFlTwdM6+UAdRXs9K2MyMQMjNx8BfGp2WhsoY9dkzrClEGHiKjCMeyQ0qysrNCsWTOFY02bNsVff/2lcKww6ERFReH06dNv3KsjCALORyRi/dl7uPIgCUJ+HhIPL0Gt7GQc8D2Jbi0bFK3xlVV8//ysM344ehu7AqKx2i8CQVHJ+GVE60rrVYlMzMCITZflQWf3pE4MOkRElYSXsUhprq6uCAsLUzgWHh4OW1tb+c+FQSciIgKnTp2CiYmJ0s8jkwn451Ys+q+9gI+3XsWVB0lQF/Kh5f8L6qpJcef6xWKDTnF0NNWxaFBLrBjmBI3nyTi0cg4szc2go6M4bR4oCFcLFiyAlZUVdHV14ebmhoiIiFLb9/b2Rvv27WFgYABzc3MMHDhQ/nf0MDEDIzddwe0/liPht8k4P68PmtrXw4ABA3D3btEVk4mISLUYdkhp06dPx5UrV7B48WLcu3cPu3fvxqZNm+Dh4QGgIOh8+OGHuHbtGnbt2oX8/HzExcUhLi4OOTk58nZ69eqFtWvXyn9OT09HcHAwAq8FAQBmbzuFSSv/RPCde9DVVMfYjvXQKGQTsmIjsH/fnhLbLU0Pez08P/A1DGrpwPTDb2ExYR3eHT9TYeD0smXLsHr1amzcuBEBAQHQ09ODu7s7srKySmzX398fHh4euHLlCnx9fZGbm4vevXvjdlQCRmy6grjULNRzaI69O7fjzp07OHHiBARBQO/evZGfX/wO5kREpCICCVKpVAAgSKXSqi6lxvj777+FFi1aCNra2oKjo6OwadMm+X2RkZECgGJvZ86ckZ9na2srLFy4UP7z/06cKvYxzu8OEp6lZ5e53dLMnj1b6NKli5CWlSt8seu6YDv7qGA7+6jw+c5rQurzHEEmkwmWlpbCTz/9JH9MSkqKoK2tLezZs6fMfz8JCQkCAKHpJysE29lHBbflZ4WE1CyFc27evCkAEO7du1fmdomI6IWyfn5zzA69kffffx/vv/9+sffZ2dlBeM3yTbHS59jtdw32pnpIzsjBtksPsf1KPmxnHwUAWBhqY1LXBhjRwQb62gW/psZ6r2/3dY4cOQJ3d3eMHz0S/v7+0K1thueNeuF/cMed2DTM61oHcXFxcHNzkz/GyMgIHTt2xOXLlzFixIgyPc/th7EAgBSZFpqZF4zRMTN4MUYnIyMDPj4+sLe3R/369cv1moiIqHQMO1Tp9gVGY+6BglWOJQA01SXIyS8IMXYmtfBZ94YY1LYutDXUVf7chdPmvby8MG/ePAQGBmLq1Gkw1NNFJLrhkw3XAQAWFhYKj7OwsEBcXFyZniPyaRoGjpkM7brN0LRZc+ye1FEedNavX49Zs2YhIyMDTZo0ga+vL7S0uGoyEVFFUnrMzpkzZyqiDhK55IwcXL7/DL+cCsfsl7ZzEADk5AtobKGPtR+1gd+MdzCig02FBB2gYNp827ZtsXjxYrRp0waTJ0/G5MmTUCfaH90amyE7r6CwRf/cRlau8mNpop5lwOX9j5AWG4l247/FnsmdYG6gI79/1KhRuHHjBvz9/dG4cWMMGzas1LFARERUfkr37PTp0wf16tXD+PHjMXbsWHbBk4LsvHzcS0hHWFwa7v53C4tLRXxqdqmP++6D5nBpaFrh9ZU2bT5gXHss1EzDj7uAAxdvIzJTCxtGtYONSS3Ex8ejdevWpbYd/SwTHd8fhcQ7V9BhymocnNVfIegABZfEjIyM4ODggE6dOqFOnTo4ePAgRo4cqeqXSkRE/1E67Dx+/Bi///47tm/fju+++w49e/bExIkTMXDgQHbHi0ys9DkiEzNgb6pXZPdtQRDwKPm5PMwUBpvIxAzky4ofV1PfWBe2xnq4eC8RL5+hLpHAzlSvAl/JC6VNm1dTk+D70T2xfoY5JLGh+NeiAfqtOY8f3muIgIAAfP755yW2G5WYgY79RyEx9DycPX7BoTmDigSdVwmCAEEQkJ1dehAkIqLyKddGoEFBQfDx8cGePXsAAB999BEmTpwIJycnlRVYGbgRaFEvj6tRkwCTujZA3Tq6BaEmNhXh8elIz84r9rFGuppoYmmAppYGaGJpiCaWBmhiaSAfaLwvMBrzDoQiXxCgLpFg8eAWGN7+9ZuDqkJgYCA6d+6M7777DsOGDcPVq1cxadIkbNq0CaNGjQIALF26FN7eS9B0xBxE5eoj5fxO6KQ9QvS9MOjr1QJQMG1+0KBBmDJlCqKfZaJDv5FICPaD04Qf8fv0gTD9L+gYGRlBV1cXDx48wL59+9C7d2+YmZnh0aNHWLJkCS5evIg7d+7A3Lz6bFJKRFRTlPXzu9y7nj958gSbNm3CkiVLoKGhgaysLLi4uGDjxo1o3rx5eZquNAw7imKlz+G65DRK6KCR01SXoKGZPppavQg0TS0NYWGoLd+2obTnKNzO4dVeo4p29OhRzJ07FxEREbC3t4eXlxcmTZokv18QBCxcuBCbNm3Cs6RkaFg3hXHvL+DariXWjmwDc0Md2NnZYdy4cZg4dRZGbLqCS3N7FftcPj4+GDduHJ48eYJPPvkE169fR3JyMiwsLNCtWzcsWLAATZo0qayXTkQkKhUadnJzc3H48GFs3boVvr6+cHZ2xsSJEzFy5Eg8ffoU33zzDYKCgnD79u1yvQig4LLZ7NmzcezYMWRmZqJRo0bw8fGBs7MzgBcfTJs3b0ZKSgpcXV3lm1SWFcOOokv3E/HR5oAix9vWr41ODU3gaGUIR0sD2JvqQVNd/OtS/i8kFrP+vIX07DyY6mtjzcg2sDOthasPkuB97C7iUrPQwFQPeyZ3goVh6ZeuiIhIdcr6+a30mJ0vv/wSe/bsgSAIGDNmDJYtW4YWLVrI79fT08PPP/8Ma2vrN6v8JcnJyXB1dUWPHj1w7NgxmJmZISIiAnXq1JGfU7ja7fbt22Fvb4/58+fD3d0dt2/fho4OP3jeRHFjTdQlEqwb3bbSe2Gqg/daWsHR0gBf7ArC3bg0fLT5CgDIxx2Z6msx6BARVWNKh53bt29jzZo1GDx4MLS1i9/I0NTUVCVT1JcuXYr69evDx8dHfsze3l7+Z0EQsGrVKnzzzTcYMGAAAGDHjh2wsLDAoUOHyrwAHCm6eC9R4efCcTVvY9Ap1MBMHwe/cMX0P4JxPFRxvZ2kjBzIyrnYIRERVRylr0H4+flh5MiRJQYdANDQ0ED37t3LVRhQsNqts7Mzhg4dCnNzc7Rp0wabN2+W3x8ZGVnqarclyc7ORmpqqsKNCshkArZdeggAmNG7MfZM6oQLc3pU2gDi6kxXSx0fd7ItclwmAA8TM6ugIiIiKgulw463tze2bt1a5PjWrVuxdOlSlRRVqHC1WwcHB5w4cQKff/45pk6diu3btwOAfEVbZVe79fb2lq93YmRkxLWCXnI2PAGRiRkw0NHABFd7uDQ0eat7dF5lb6YHtVfGXhdMna9VNQUREdFrKR12fv31Vzg6OhY53rx5c2zcuFElRRUqbrXbSZMmlft55s6dC6lUKr/FxMSoqOKaz+fiQwDAiPb1oafN3UReZWWkC+/BLaH+32wzXuIjIqr+lP40i4uLg5WVVZHjZmZmiI2NVUlRhUpb7RYALC0tAQDx8fEKNb1utVttbe1SL8O9rcLj03A+IhFqEuBjF7uqLqfaGt7eBt0am1XZ1HkiIlKO0j079evXx8WLF4scv3jxokpmYL2stNVugYLBypaWlvDz85Pfn5qaioCAALi4uKi0lrdBYa9O72aWqG/MyzKlsTLS5SU+IqIaQumenUmTJsHT0xO5ubno2bMngIJBy7NmzcKMGTNUWtz06dPRuXNnLF68WL7a7aZNm7Bp0yYAgEQigaenJ3788Uc4ODjIp55bW1tj4MCBKq1F7FIyc3DwxiMAwHhXu6othoiISIWUDjszZ87Es2fP8MUXXyAnJwcAoKOjg9mzZ2Pu3LkqLa59+/Y4ePAg5s6di++//x729vZYtWqVfFl/AJg1axYyMjIwefJkpKSkoEuXLjh+/DjX2FHSnqsxyMqVoZmVITrYG1d1OURERCrzxttFpKen486dO9DV1YWDg0ONHgPztq+gnJsvQ7dlZxArzcJPH7bCUGfOTiMiouqvwlZQLqSvr4/27du/6cOpGjnxbxxipVkw1ddCfyfVjrsiIiKqam8Udq5du4Y//vgD0dHR8ktZhQ4cOKCSwqjyFA5M/qijLXQ01au2GCIiIhVTejbW3r170blzZ9y5cwcHDx5Ebm4u/v33X5w+fRpGRkYVUSNVoJsxKbgelQxNdQlGd+IqyUREJD5Kh53Fixdj5cqV+Pvvv6GlpYVffvkFd+/exbBhw2Bjww/LmsbnYiQA4P1W1sVuAEpERFTTKR127t+/j379+gEAtLS0kJGRAYlEgunTp8unhFPNkJCahX9CChaCnOBq/5qziYiIaialw06dOnWQlpYGAKhbty5CQ0MBACkpKcjM5GaINcnOK1HIzRfgbFsHLevxEiQREYmT0gOUu3XrBl9fX7Rs2RJDhw7FtGnTcPr0afj6+qJXr14VUSNVgKzcfOwKiAYAjGevDhERiZjSYWft2rXIysoCAHz99dfQ1NTEpUuXMGTIEHzzzTcqL5AqxpGbT/AsIwfWRjpwb27x+gcQERHVUEqFnby8PBw9ehTu7u4AADU1NcyZM6dCCqOKIwiCfLr5x53toKGu9NVMIiKiGkOpTzkNDQ189tln8p4dqpkCIpNwJzYVOppqGNGeqyUTEZG4Kf2VvkOHDggODq6AUqiyFE43H9y2HmrX0qriaoiIiCqW0mN2vvjiC3h5eSEmJgbt2rWDnp6ewv2tWrVSWXGkejFJmTh5Ox4AML6zXdUWQ0REVAmUDjsjRowAAEydOlV+TCKRQBAESCQS5Ofnq646Urntlx5CEICuDqZwsDCo6nKIiIgqnNJhJzIysiLqoEqQkZ2HfddiAHARQSIiensoHXZsbW0rog6qBH8FPUJaVh4amOqhe2Ozqi6HiIioUigddnbs2FHq/R9//PEbF0MVRyZ7Md18nKsd1NQkVVsQERFRJVE67EybNk3h59zcXGRmZkJLSwu1atVi2Kmm/MOfIjIxAwY6GhjStl5Vl0NERFRplJ56npycrHBLT09HWFgYunTpgj179lREjaQCW/+bbj7cuT70tJXOuERERDWWSpbOdXBwwJIlS4r0+lD1EBGfhvMRiVCTAGM53ZyIiN4yKtsnQENDA0+ePFFVc6RCPpceAgDebWaB+sa1qrYYIiKiSqb09YwjR44o/CwIAmJjY7F27Vq4urqqrDBSjZTMHBwIegSAu5sTEdHbSemwM3DgQIWfJRIJzMzM0LNnTyxfvlxVdZGK7A2MQVauDE2tDNHR3riqyyEiIqp0SocdmUxWEXVQBcjLl2HHf5ewJrjaQSLhdHMiInr7qGzMDlU/J/6NxxNpFkz0tNDfybqqyyEiIqoSSoedIUOGYOnSpUWOL1u2DEOHDlVJUaQahbubj+poAx1N9SquhoiIqGooHXbOnTuH9957r8jxvn374ty5cyopisov5JEU16KSoakuwehO3OKDiIjeXkqHnfT0dGhpaRU5rqmpidTUVJUUReVX2KvzfitrmBvqVHE1REREVUfpsNOyZUvs27evyPG9e/eiWbNmKimKyichNQt/3ypY82i8q13VFkNERFTFlJ6NNX/+fAwePBj3799Hz549AQB+fn7Ys2cP9u/fr/ICSXk7A6KRmy+gnW0dtKpXu6rLISIiqlJKh53+/fvj0KFDWLx4Mf7880/o6uqiVatWOHXqFLp3714RNZISsnLzsTsgCgB7dYiIiIA3CDsA0K9fP/Tr10/VtZAK/H3zCRLTc2BtpIM+zS2ruhwiIqIqp/SYncDAQAQEBBQ5HhAQgGvXrqmkqLfNkiVLIJFI4OnpqXD88uXL6NmzJ/T09GBoaIhu3brh+fPnJbazePFiTBrcG9ErhyJk2TB8OGQwwsLCFM7ZtGkT3nnnHRgaGkIikSAlJaUCXhEREVH1oXTY8fDwQExMTJHjjx8/hoeHh0qKepsEBgbi119/RatWrRSOX758GX369EHv3r1x9epVBAYGYsqUKVBTK/l/2d/H/aDRsg9sx6/A/46dQG5uLnr37o2MjAz5OZmZmejTpw/mzZtXYa+JiIioOlH6Mtbt27fRtm3bIsfbtGmD27dvq6Sot0V6ejpGjRqFzZs348cff1S4b/r06Zg6dSrmzJkjP9akSZNS23OatAyx/8ZhRAcbdO3UEo7btsHc3BzXr19Ht27dAEDee3T27FmVvhYiIqLqSumeHW1tbcTHxxc5HhsbCw2NNxoC9Nby8PBAv3794ObmpnA8ISEBAQEBMDc3R+fOnWFhYYHu3bvjwoULJbYVk5SJk7fjABTsgwUAUqkUAGBszA1AiYjo7aV02Onduzfmzp0r/yAFgJSUFMybNw/vvvuuSosTs7179yIoKAje3t5F7nvw4AEA4Ntvv8WkSZNw/PhxtG3bFr169UJERESx7e24/BAyAejqYAoHCwPIZDJ4enrC1dUVLVq0qNDXQkREVJ0p3RXz888/o1u3brC1tUWbNm0AAMHBwbCwsMDvv/+u8gLFKCYmBtOmTYOvry90dIqubly4s/ynn36K8ePHAyi4TOjn54etW7cWCUgZ2XnYG1gwjqpwurmHhwdCQ0NL7Q0iIiJ6GygddurWrYtbt25h165duHnzJnR1dTF+/HiMHDkSmpqaFVGj6Fy/fh0JCQkKY5/y8/Nx7tw5rF27Vj6D6tUVqZs2bYro6Ogi7R0IeoS0rDzYm+rhncbmmDJlCo4ePYpz586hXr16FftiiIiIqrk3GmSjp6eHyZMnq7qWt0avXr0QEhKicGz8+PFwdHTE7Nmz0aBBA1hbWxeZNh4eHo6+ffsqHJPJBPhcfAgAGOtii6lTv8TBgwdx9uxZ2NvbV+jrICIiqgneeETx7du3ER0djZycHIXjH3zwQbmLEjsDA4Mi42j09PRgYmIiPz5z5kwsXLgQTk5OaN26NbZv3467d+/izz//lD+mV69eaN7ZDQ9yW8FAWwMXdyzF/n17cfjwYRgYGCAurmDAspGREXR1dQEAcXFxiIuLw7179wAAISEhMDAwgI2NDQcyExGRKCkddh48eIBBgwYhJCQEEokEgiAAACQSCYCCyzFUfp6ensjKysL06dORlJQEJycn+Pr6omHDhvJz7t+/j0T9BkDTVhjWvj4W9P8VAPDOO+8otOXj44Nx48YBADZu3IjvvvtOfl/hlPSXzyEiIhITiVCYVsqof//+UFdXx2+//QZ7e3tcvXoVz549w4wZM/Dzzz+ja9euFVVrhUlNTYWRkRGkUikMDQ2rupwyu3w/ESM3B0AC4NysHqhvXKuqSyIiIqo0Zf38Vnrq+eXLl/H999/D1NQUampqUFNTQ5cuXeDt7Y2pU6eWq2gqu32B0Ri5uWDbDgHApfuJVVsQERFRNaV02MnPz4eBgQEAwNTUFE+ePAEA2NraFhlQSxUjVvoccw8oDnCedyAUsdKS980iIiJ6Wyk9ZqdFixa4efMm7O3t0bFjRyxbtgxaWlrYtGkTGjRoUBE10ivuJ2RA9srFx3xBwMPETFgZ6VZNUURERNWU0mHnm2++kW8s+f333+P9999H165dYWJign379qm8QCqquEtW6hIJ7Ew5ZoeIiOhVSocdd3d3+Z8bNWqEu3fvIikpCXXq1JHPyKKKE/pYis3nC7aTkEgAQSgIOosHt2CvDhERUTFUsnMn12epHM9z8jFt7w3k5gtwb26Bhf2bIerZc9iZ1mLQISIiKgG3Ka9BvI/dwf2nGTA30MaSwa1QR08L1rV56YqIiKg0Ss/Goqpx5m4CdlyOAgAsH+aEOnpaVVwRERFRzcCwUwMkpmdj5p83AQATXO3R1cGsiisiIiKqOZQOO+fOnUNeXl6R43l5eTh37pxKiqIXBEHA7D9vITE9B00sDDCrT5OqLomIiKhGUTrs9OjRA0lJSUWOS6VS9OjRQyVF0Qu7AqLhdzcBWupqWDWiNXQ01au6JCIiohpF6bAjCEKxU8yfPXsGPT09lRRFBe4/TceP/9wGAMzq0wRNrWrOvl1ERETVRZlnYw0ePBhAwe7m48aNg7a2tvy+/Px83Lp1C507d1Z9hW+pnDwZPPcGIytXhi6NTDHB1b6qSyIiIqqRyhx2jIyMABT07BgYGEBX98W6LlpaWujUqRMmTZqk+grfUqtOhSPksRS1a2li+TAnqKlxwUYiIqI3Ueaw4+PjAwCws7PDV199xUtWFSjgwTNs8L8PAFgyuCUsDHWquCIiIqKaS+kxO7NmzVIYsxMVFYVVq1bh5MmTKi3sbSV9nguvP25CEICh7eqhTwurqi6JiIioRlM67AwYMAA7duwAAKSkpKBDhw5Yvnw5BgwYgA0bNqi8wLfNgsOheJzyHLYmtbDwg+ZVXQ4REVGNp3TYCQoKQteuXQEAf/75JywtLREVFYUdO3Zg9erVKi/wbXI4+DEOBz+BupoEK4e3hr42d/MgIiIqL6XDTmZmJgwMDAAAJ0+exODBg6GmpoZOnTohKipK5QW+LR4lZ+KbQ6EAgC97NkJbmzpVXBEREZE4KB12GjVqhEOHDiEmJgYnTpxA7969AQAJCQkwNOQ6MG8iXybA64+bSMvKQ1ub2pjSo1FVl0RERCQaSoedBQsW4KuvvoKdnR06dOgAFxcXAAW9PG3atFF5gW+DX8/dx9XIJOhpqWPl8NbQUOeWZURERKqi9Kfqhx9+iOjoaFy7dg0nTpyQH+/VqxdWrlyp0uJetWTJEkgkEnh6esqPZWVlwcPDAyYmJtDX18eQIUMQHx9foXWoUsgjKVacDAcALPygOWxNOKWfiIhIld6oC8HS0hIGBgbw9fXF8+fPAQDt27eHo6OjSot7WWBgIH799Ve0atVK4fj06dPx999/Y//+/fD398eTJ0/kqz1Xd89z8jFt3w3kyQT0bWGJoe3qVXVJREREoqN02Hn27Bl69eqFxo0b47333kNsbCwAYOLEiZgxY4bKCwSA9PR0jBo1Cps3b0adOi8G7kqlUmzZsgUrVqxAz5490a5dO/j4+ODSpUu4cuVKhdSiSov+dxsPnmbAwlAbiwe1LHbPMSIiIiofpcPO9OnToampiejoaNSqVUt+fPjw4Th+/LhKiyvk4eGBfv36wc3NTeH49evXkZubq3Dc0dERNjY2uHz5contZWdnIzU1VeFW2fzuxGPnlWgAwPKhrVFHT6vSayAiInobKL2Qy8mTJ3HixAnUq6d4ycXBwaFCpp7v3bsXQUFBCAwMLHJfXFwctLS0ULt2bYXjFhYWiIuLK7FNb29vfPfdd6outcyepmVj1p+3AACfdLFHFwfTKquFiIhI7JTu2cnIyFDo0SmUlJSksBO6KsTExGDatGnYtWsXdHRUtz/U3LlzIZVK5beYmBiVtf06giBg1p838SwjB46WBvjKvUmlPTcREdHbSOmw07VrV/l2EQAgkUggk8mwbNky9OjRQ6XFXb9+HQkJCWjbti00NDSgoaEBf39/rF69GhoaGrCwsEBOTg5SUlIUHhcfHw9LS8sS29XW1oahoaHCrbLsvBKFM2FPoaWhhl9GtIGOpnqlPTcREdHbSOnLWMuWLUOvXr1w7do15OTkYNasWfj333+RlJSEixcvqrS4Xr16ISQkROHY+PHj4ejoiNmzZ6N+/frQ1NSEn58fhgwZAgAICwtDdHS0fP2f6uReQhp+/OcOAGBOH0c0sTSo4oqIiIjET+mw06JFC4SHh2Pt2rUwMDBAeno6Bg8eDA8PD1hZqXaHbgMDA7Ro0ULhmJ6eHkxMTOTHJ06cCC8vLxgbG8PQ0BBffvklXFxc0KlTJ5XWUl45eTJM2xuM7DwZujqYYlxnu6ouiYiI6K2gdNiJjo5G/fr18fXXXxd7n42NjUoKK6uVK1dCTU0NQ4YMQXZ2Ntzd3bF+/fpKraEsVviG498nqahTSxPLhzpBTY3TzImIiCqDRBAEQZkHqKurIzY2Fubm5grHnz17BnNzc+Tn56u0wMqQmpoKIyMjSKXSChm/c/n+M3z02xUIArBxdDv0aVHyeCIiIiIqm7J+fis9QFkQhGIXv0tPT1fpjCmxkGbmYsYfwRAEYLhzfQYdIiKiSlbmy1heXl4ACmZfzZ8/X2H6eX5+PgICAtC6dWuVF1iTPUnJxIz9t/BEmgU7k1pY0L9ZVZdERET01ilz2Llx4waAgp6dkJAQaGm9WPFXS0sLTk5O+Oqrr1RfYQ21LzAac/4KQeE1wn4traCnrfQQKSIiIiqnMn/6njlzBkDB1O9ffvmlUtemqWlipc8x98CLoAMAG/0fYLSLLayMdKusLiIioreR0mN2fHx8GHReIzIxA7JXhn3nCwIeJmZWTUFERERvMaXDDr2evakeXp1Zri6RwM606DYbREREVLEYdiqAlZEuvAe3hPp/s9bUJRIsHtyCl7CIiIiqAEfMVpDh7W3QrbEZHiZmws60FoMOERFRFWHYqUBWRroMOURERFWMl7GIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUqnXY8fb2Rvv27WFgYABzc3MMHDgQYWFhCudkZWXBw8MDJiYm0NfXx5AhQxAfH19FFRMREVF1U63Djr+/Pzw8PHDlyhX4+voiNzcXvXv3RkZGhvyc6dOn4++//8b+/fvh7++PJ0+eYPDgwVVYNREREVUnEkEQhKouoqyePn0Kc3Nz+Pv7o1u3bpBKpTAzM8Pu3bvx4YcfAgDu3r2Lpk2b4vLly+jUqVOZ2k1NTYWRkRGkUikMDQ0r8iUQERGRipT187ta9+y8SiqVAgCMjY0BANevX0dubi7c3Nzk5zg6OsLGxgaXL18usZ3s7GykpqYq3IiIiEicakzYkclk8PT0hKurK1q0aAEAiIuLg5aWFmrXrq1wroWFBeLi4kpsy9vbG0ZGRvJb/fr1K7J0IiIiqkI1Jux4eHggNDQUe/fuLXdbc+fOhVQqld9iYmJUUCERERFVRxpVXUBZTJkyBUePHsW5c+dQr149+XFLS0vk5OQgJSVFoXcnPj4elpaWJbanra0NbW3tiiyZiIiIqolq3bMjCAKmTJmCgwcP4vTp07C3t1e4v127dtDU1ISfn5/8WFhYGKKjo+Hi4lLZ5RIREVE1VK17djw8PLB7924cPnwYBgYG8nE4RkZG0NXVhZGRESZOnAgvLy8YGxvD0NAQX375JVxcXMo8E4uIiIjErVpPPZdIJMUe9/Hxwbhx4wAULCo4Y8YM7NmzB9nZ2XB3d8f69etLvYz1Kk49JyIiqnnK+vldrcNOZWHYISIiqnlEuc4OERERkbIYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1EQTdtatWwc7Ozvo6OigY8eOuHr1alWXRERERNWAKMLOvn374OXlhYULFyIoKAhOTk5wd3dHQkJCVZdGREREVUwUYWfFihWYNGkSxo8fj2bNmmHjxo2oVasWtm7dWtWlERERURXTqOoCyisnJwfXr1/H3Llz5cfU1NTg5uaGy5cvF/uY7OxsZGdny3+WSqUAgNTU1IotloiIiFSm8HNbEIRSz6vxYScxMRH5+fmwsLBQOG5hYYG7d+8W+xhvb2989913RY7Xr1+/QmokIiKiipOWlgYjI6MS76/xYedNzJ07F15eXvKfZTIZkpKSYGJiAolEorLnSU1NRf369RETEwNDQ0OVtSuG9mty7TW9/Zpce01vvybXXtHt1+Taa3r7Nbl2QRCQlpYGa2vrUs+r8WHH1NQU6urqiI+PVzgeHx8PS0vLYh+jra0NbW1thWO1a9euqBJhaGhYIb9AYmi/Jtde09uvybXX9PZrcu0V3X5Nrr2mt19Tay+tR6dQjR+grKWlhXbt2sHPz09+TCaTwc/PDy4uLlVYGREREVUHNb5nBwC8vLwwduxYODs7o0OHDli1ahUyMjIwfvz4qi6NiIiIqpgows7w4cPx9OlTLFiwAHFxcWjdujWOHz9eZNByZdPW1sbChQuLXDJj+zW79prefk2uvaa3X5Nrr+j2a3LtNb39mlx7WUmE183XIiIiIqrBavyYHSIiIqLSMOwQERGRqDHsEBERkagx7BAREZGoMexUoHXr1sHOzg46Ojro2LEjrl69qpJ2z507h/79+8Pa2hoSiQSHDh1SSbtAwVYa7du3h4GBAczNzTFw4ECEhYWprP0NGzagVatW8sWlXFxccOzYMZW1/7IlS5ZAIpHA09NTZW1+++23kEgkCjdHR0eVtf/48WOMHj0aJiYm0NXVRcuWLXHt2jWVtG1nZ1ekdolEAg8PD5W0n5+fj/nz58Pe3h66urpo2LAhfvjhh9fuWVNWaWlp8PT0hK2tLXR1ddG5c2cEBga+UVuvew8JgoAFCxbAysoKurq6cHNzQ0REhMraP3DgAHr37i1ftT04OFhl9efm5mL27Nlo2bIl9PT0YG1tjY8//hhPnjxRSe3ffvstHB0doaenhzp16sDNzQ0BAQEqqf1Vn332GSQSCVatWqWy9seNG1fkPdCnTx+V1n/nzh188MEHMDIygp6eHtq3b4/o6Ohyt13c+1cikeCnn35SSe3p6emYMmUK6tWrB11dXfnG2mX1uvbj4+Mxbtw4WFtbo1atWujTp49S76vyYNipIPv27YOXlxcWLlyIoKAgODk5wd3dHQkJCeVuOyMjA05OTli3bp0KKlXk7+8PDw8PXLlyBb6+vsjNzUXv3r2RkZGhkvbr1auHJUuW4Pr167h27Rp69uyJAQMG4N9//1VJ+4UCAwPx66+/olWrViptFwCaN2+O2NhY+e3ChQsqaTc5ORmurq7Q1NTEsWPHcPv2bSxfvhx16tRRSfuBgYEKdfv6+gIAhg4dqpL2ly5dig0bNmDt2rW4c+cOli5dimXLlmHNmjUqaf+TTz6Br68vfv/9d4SEhKB3795wc3PD48ePlW7rde+hZcuWYfXq1di4cSMCAgKgp6cHd3d3ZGVlqaT9jIwMdOnSBUuXLlW69te1n5mZiaCgIMyfPx9BQUE4cOAAwsLC8MEHH6ik9saNG2Pt2rUICQnBhQsXYGdnh969e+Pp06cqab/QwYMHceXKldduA/Am7ffp00fhvbBnzx6VtX///n106dIFjo6OOHv2LG7duoX58+dDR0en3G2/XHNsbCy2bt0KiUSCIUOGqKR2Ly8vHD9+HDt37sSdO3fg6emJKVOm4MiRI+VuXxAEDBw4EA8ePMDhw4dx48YN2Nraws3NTWWfL6USqEJ06NBB8PDwkP+cn58vWFtbC97e3ip9HgDCwYMHVdrmyxISEgQAgr+/f4U9R506dYTffvtNZe2lpaUJDg4Ogq+vr9C9e3dh2rRpKmt74cKFgpOTk8rae9ns2bOFLl26VEjbxZk2bZrQsGFDQSaTqaS9fv36CRMmTFA4NnjwYGHUqFHlbjszM1NQV1cXjh49qnC8bdu2wtdff12utl99D8lkMsHS0lL46aef5MdSUlIEbW1tYc+ePeVu/2WRkZECAOHGjRtKt1uW9gtdvXpVACBERUWpvG2pVCoAEE6dOqVU26W1/+jRI6Fu3bpCaGioYGtrK6xcuVLptktqf+zYscKAAQPeqL2ytD98+HBh9OjRFdL2qwYMGCD07NlTZe03b95c+P777xWOvel77NX2w8LCBABCaGio/Fh+fr5gZmYmbN68Wen2lcWenQqQk5OD69evw83NTX5MTU0Nbm5uuHz5chVWpjypVAoAMDY2Vnnb+fn52Lt3LzIyMlS6tYeHhwf69eun8PevShEREbC2tkaDBg0watSoMnVPl8WRI0fg7OyMoUOHwtzcHG3atMHmzZtV0varcnJysHPnTkyYMEFlm9927twZfn5+CA8PBwDcvHkTFy5cQN++fcvddl5eHvLz84t8O9bV1VVZz1qhyMhIxMXFKfz+GBkZoWPHjjXu/VtIKpVCIpGofA/AnJwcbNq0CUZGRnByclJJmzKZDGPGjMHMmTPRvHlzlbT5qrNnz8Lc3BxNmjTB559/jmfPnqmkXZlMhn/++QeNGzeGu7s7zM3N0bFjR5UONSgUHx+Pf/75BxMnTlRZm507d8aRI0fw+PFjCIKAM2fOIDw8HL179y5329nZ2QCg8B5WU1ODtra2yt/DxWHYqQCJiYnIz88vsoKzhYUF4uLiqqgq5clkMnh6esLV1RUtWrRQWbshISHQ19eHtrY2PvvsMxw8eBDNmjVTSdt79+5FUFAQvL29VdLeqzp27Iht27bh+PHj2LBhAyIjI9G1a1ekpaWVu+0HDx5gw4YNcHBwwIkTJ/D5559j6tSp2L59uwoqV3To0CGkpKRg3LhxKmtzzpw5GDFiBBwdHaGpqYk2bdrA09MTo0aNKnfbBgYGcHFxwQ8//IAnT54gPz8fO3fuxOXLlxEbG6uC6l8ofI/W9PdvoaysLMyePRsjR45U2SaMR48ehb6+PnR0dLBy5Ur4+vrC1NRUJW0vXboUGhoamDp1qkrae1WfPn2wY8cO+Pn5YenSpfD390ffvn2Rn59f7rYTEhKQnp6OJUuWoE+fPjh58iQGDRqEwYMHw9/fXwXVv7B9+3YYGBhg8ODBKmtzzZo1aNasGerVqwctLS306dMH69atQ7du3crdtqOjI2xsbDB37lwkJycjJycHS5cuxaNHj1T+Hi6OKLaLoIrh4eGB0NBQlafuJk2aIDg4GFKpFH/++SfGjh0Lf3//cgeemJgYTJs2Db6+vmW6Pv4mXu6laNWqFTp27AhbW1v88ccf5f6GJZPJ4OzsjMWLFwMA2rRpg9DQUGzcuBFjx44tV9uv2rJlC/r27av0eIjS/PHHH9i1axd2796N5s2bIzg4GJ6enrC2tlZJ/b///jsmTJiAunXrQl1dHW3btsXIkSNx/fp1FVQvTrm5uRg2bBgEQcCGDRtU1m6PHj0QHByMxMREbN68GcOGDUNAQADMzc3L1e7169fxyy+/ICgoSGU9jq8aMWKE/M8tW7ZEq1at0LBhQ5w9exa9evUqV9symQwAMGDAAEyfPh0A0Lp1a1y6dAkbN25E9+7dy9X+y7Zu3YpRo0ap9N+6NWvW4MqVKzhy5AhsbW1x7tw5eHh4wNrautw95Zqamjhw4AAmTpwIY2NjqKurw83NDX379lXZJIbSsGenApiamkJdXR3x8fEKx+Pj42FpaVlFVSlnypQpOHr0KM6cOYN69eqptG0tLS00atQI7dq1g7e3N5ycnPDLL7+Uu93r168jISEBbdu2hYaGBjQ0NODv74/Vq1dDQ0NDJd/cXlW7dm00btwY9+7dK3dbVlZWRQJf06ZNVXaZrFBUVBROnTqFTz75RKXtzpw5U96707JlS4wZMwbTp09XWS9bw4YN4e/vj/T0dMTExODq1avIzc1FgwYNVNJ+ocL3aE1+/wIvgk5UVBR8fX1V1qsDAHp6emjUqBE6deqELVu2QENDA1u2bCl3u+fPn0dCQgJsbGzk7+GoqCjMmDEDdnZ25S+8GA0aNICpqalK3sOmpqbQ0NCo8Pfx+fPnERYWptL38PPnzzFv3jysWLEC/fv3R6tWrTBlyhQMHz4cP//8s0qeo127dggODkZKSgpiY2Nx/PhxPHv2TOXv4eIw7FQALS0ttGvXDn5+fvJjMpkMfn5+Kh2bUhEEQcCUKVNw8OBBnD59Gvb29hX+nDKZTH49tzx69eqFkJAQBAcHy2/Ozs4YNWoUgoODoa6uroJqFaWnp+P+/fuwsrIqd1uurq5FpvmHh4fD1ta23G2/zMfHB+bm5ujXr59K283MzISamuI/Kerq6vJvu6qip6cHKysrJCcn48SJExgwYIBK27e3t4elpaXC+zc1NRUBAQHV/v1bqDDoRERE4NSpUzAxManQ51PVe3jMmDG4deuWwnvY2toaM2fOxIkTJ1RQaVGPHj3Cs2fPVPIe1tLSQvv27Sv8fbxlyxa0a9dOZeOkgILfmdzc3Ep5DxsZGcHMzAwRERG4du2ayt/DxeFlrAri5eWFsWPHwtnZGR06dMCqVauQkZGB8ePHl7vt9PR0hW8hkZGRCA4OhrGxMWxsbMrVtoeHB3bv3o3Dhw/DwMBAPkbByMgIurq65WobAObOnYu+ffvCxsYGaWlp2L17N86ePauSf8gMDAyKjC3S09ODiYmJysYcffXVV+jfvz9sbW3x5MkTLFy4EOrq6hg5cmS5254+fTo6d+6MxYsXY9iwYbh69So2bdqETZs2qaDyAjKZDD4+Phg7diw0NFT79u/fvz8WLVoEGxsbNG/eHDdu3MCKFSswYcIElbR/4sQJCIKAJk2a4N69e5g5cyYcHR3f6D31uveQp6cnfvzxRzg4OMDe3h7z58+HtbU1Bg4cqJL2k5KSEB0dLV/7pvDD0dLSsky9R6W1b2VlhQ8//BBBQUE4evQo8vPz5e9jY2NjaGlpvXHbJiYmWLRoET744ANYWVkhMTER69atw+PHj8u8hMHr/m5eDWaampqwtLREkyZNyt2+sbExvvvuOwwZMgSWlpa4f/8+Zs2ahUaNGsHd3V0l9c+cORPDhw9Ht27d0KNHDxw/fhx///03zp49W+62gYLgvX//fixfvrxM9SrTfvfu3TFz5kzo6urC1tYW/v7+2LFjB1asWKGS9vfv3w8zMzPY2NggJCQE06ZNw8CBA1UyAPq1Kny+11tszZo1go2NjaClpSV06NBBuHLlikraPXPmjACgyG3s2LHlbru4dgEIPj4+5W5bEARhwoQJgq2traClpSWYmZkJvXr1Ek6ePKmStouj6qnnw4cPF6ysrAQtLS2hbt26wvDhw4V79+6prP2///5baNGihaCtrS04OjoKmzZtUlnbgiAIJ06cEAAIYWFhKm1XEAQhNTVVmDZtmmBjYyPo6OgIDRo0EL7++mshOztbJe3v27dPaNCggaClpSVYWloKHh4eQkpKyhu19br3kEwmE+bPny9YWFgI2traQq9evZT6O3td+z4+PsXev3DhwnK3XzidvbjbmTNnytX28+fPhUGDBgnW1taClpaWYGVlJXzwwQfC1atXVfZ38yplp56X1n5mZqbQu3dvwczMTNDU1BRsbW2FSZMmCXFxcSqtf8uWLUKjRo0EHR0dwcnJSTh06JDK2v71118FXV3dN/rdf137sbGxwrhx4wRra2tBR0dHaNKkibB8+fIyL0/xuvZ/+eUXoV69eoKmpqZgY2MjfPPNNyr79+F1JIJQCSODiIiIiKoIx+wQERGRqDHsEBERkagx7BAREZGoMewQERGRqDHsEBERkagx7BAREZGoMewQERGRqDHsEBG94uzZs5BIJEhJSanqUohIBRh2iIiISNQYdoiIiEjUGHaIqNqRyWTw9vaGvb09dHV14eTkhD///BPAi0tM//zzD1q1agUdHR106tQJoaGhCm389ddfaN68ObS1tWFnZ1dk48Ts7GzMnj0b9evXh7a2Nho1aoQtW7YonHP9+nU4OzujVq1a6Ny5c5HdrImoZmDYIaJqx9vbGzt27MDGjRvx77//Yvr06Rg9ejT8/f3l58ycORPLly9HYGAgzMzM0L9/f+Tm5gIoCCnDhg3DiBEjEBISgm+//Rbz58/Htm3b5I//+OOPsWfPHqxevRp37tzBr7/+Cn19fYU6vv76ayxfvhzXrl2DhoaGynZwJ6LKxY1Aiahayc7OhrGxMU6dOgUXFxf58U8++QSZmZmYPHkyevTogb1792L48OEAgKSkJNSrVw/btm3DsGHDMGrUKDx9+hQnT56UP37WrFn4559/8O+//yI8PBxNmjSBr68v3NzcitRw9uxZ9OjRA6dOnUKvXr0AAP/73//Qr18/PH/+HDo6OhX8t0BEqsSeHSKqVu7du4fMzEy8++670NfXl9927NiB+/fvy897OQgZGxujSZMmuHPnDgDgzp07cHV1VWjX1dUVERERyM/PR3BwMNTV1dG9e/dSa2nVqpX8z1ZWVgCAhISEcr9GIqpcGlVdABHRy9LT0wEA//zzD+rWratwn7a2tkLgeVO6urplOk9TU1P+Z4lEAqBgPBER1Szs2SGiaqVZs2bQ1tZGdHQ0GjVqpHCrX7++/LwrV67I/5ycnIzw8HA0bdoUANC0aVNcvHhRod2LFy+icePGUFdXR8uWLSGTyRTGABGReLFnh4iqFQMDA3z11VeYPn06ZDIZunTpAqlUiosXL8LQ0BC2trYAgO+//x4mJiawsLDA119/DVNTUwwcOBAAMGPGDLRv3x4//PADhg8fjsuXL2Pt2rVYv349AMDOzg5jx47FhAkTsHr1ajg5OSEqKgoJCQkYNmxYVb10IqogDDtEVO388MMPMDMzg7e3Nx48eIDatWujbdu2mDdvnvwy0pIlSzBt2jRERESgdevW+Pvvv6GlpQUAaNu2Lf744w8sWLAAP/zwA6ysrPD9999j3Lhx8ufYsGED5s2bhy+++ALPnj2DjY0N5s2bVxUvl4gqGGdjEVGNUjhTKjk5GbVr167qcoioBuCYHSIiIhI1hh0iIiISNV7GIiIiIlFjzw4RERGJGsMOERERiRrDDhEREYkaww4RERGJGsMOERERiRrDDhEREYkaww4RERGJGsMOERERiRrDDhEREYna/wEBwNi0U0qrFAAAAABJRU5ErkJggg==", "text/plain": [ "
" ] @@ -769,8 +1116,12 @@ "plt.xlabel('epoch')\n", "plt.ylabel('test accuracy')\n", "plt.ylim(0, 100)\n", + "plt.xticks(np.arange(len(epochs_x)))\n", "for i, txt in enumerate(epochs_acc):\n", - " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom')\n", + " if i%2 == 0:\n", + " pass\n", + " else:\n", + " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", "plt.show()" ] } diff --git a/tests/test_nonsequential/non-sequential-SCNN-example_3.ipynb b/tests/test_nonsequential/non-sequential-SCNN-example_3.ipynb index f3ac5d89..87457657 100644 --- a/tests/test_nonsequential/non-sequential-SCNN-example_3.ipynb +++ b/tests/test_nonsequential/non-sequential-SCNN-example_3.ipynb @@ -6,7 +6,7 @@ "metadata": {}, "outputs": [], "source": [ - "import torch\n", + "import torch, random\n", "import torch.nn as nn\n", "import sinabs.layers as sl\n", "from tqdm.notebook import tqdm\n", @@ -27,20 +27,13 @@ "cell_type": "code", "execution_count": 2, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "torch.manual_seed(0)" + "torch.backends.cudnn.deterministic = True\n", + "random.seed(1)\n", + "torch.manual_seed(1)\n", + "torch.cuda.manual_seed(1)\n", + "np.random.seed(1)" ] }, { @@ -58,7 +51,7 @@ "source": [ "batch_size = 3\n", "num_workers = 1\n", - "epochs = 10\n", + "epochs = 20\n", "lr = 1e-3" ] }, @@ -375,7 +368,357 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "43cb492ef2204104a2f1048eb2ae3a85", + "model_id": "45a1007196c04fa1ae9165909607816e", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/359 [00:00" ] @@ -752,8 +1095,12 @@ "plt.xlabel('epoch')\n", "plt.ylabel('average loss')\n", "plt.ylim(0,)\n", + "plt.xticks(np.arange(len(epochs_x)))\n", "for i, txt in enumerate(y_avg):\n", - " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom')\n", + " if i%2 == 0:\n", + " pass\n", + " else:\n", + " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", "plt.show()" ] }, @@ -764,7 +1111,7 @@ "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAG2CAYAAACZEEfAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABYNklEQVR4nO3deZyNdf/H8deZfTGLGWMWZrMv2WkspWSyJGWJuCVKVEiIUGkjgzZJkX5CdxQKdbeQpYjsS5FsY2cWZsyMmTGLOef3x3BymsEMM87Mmffz8TiPmfO9rvM9n8vpvs97vtf3ur4Gk8lkQkRERMRG2Vm7ABEREZHipLAjIiIiNk1hR0RERGyawo6IiIjYNIUdERERsWkKOyIiImLTFHZERETEpinsiIiIiE1T2BERERGbprAjIiIiNs2qYWf9+vV07tyZoKAgDAYDy5cvt9huMpl49dVXCQwMxNXVlcjISA4dOmSxT2JiIn369MHT0xNvb28GDBhAamrqbTwKERERKcmsGnbS0tJo0KABH330Ub7bp06dyvTp05k1axZbtmzB3d2d9u3bk5GRYd6nT58+/PXXX6xatYrvv/+e9evXM2jQoNt1CCIiIlLCGUrKQqAGg4Fly5bRpUsXIHdUJygoiBdeeIFRo0YBkJycjL+/P/PmzaNXr178/fff1KlTh23bttG0aVMAVqxYwQMPPMCpU6cICgqy1uGIiIhICeFg7QKu5ejRo8TGxhIZGWlu8/LyIiIigk2bNtGrVy82bdqEt7e3OegAREZGYmdnx5YtW+jatWu+fWdmZpKZmWl+bjQaSUxMxNfXF4PBUHwHJSIiIkXGZDJx4cIFgoKCsLO79smqEht2YmNjAfD397do9/f3N2+LjY2lYsWKFtsdHBzw8fEx75OfqKgo3njjjSKuWERERKzh5MmTVK5c+ZrbS2zYKU7jxo1j5MiR5ufJycmEhIRw8uRJPD09rViZiIiIFFRKSgrBwcF4eHhcd78SG3YCAgIAiIuLIzAw0NweFxdHw4YNzfvEx8dbvO7SpUskJiaaX58fZ2dnnJ2d87R7enoq7IiIiJQyN5qCUmLvsxMeHk5AQABr1qwxt6WkpLBlyxZatGgBQIsWLUhKSmLHjh3mfdauXYvRaCQiIuK21ywiIiIlj1VHdlJTUzl8+LD5+dGjR9m9ezc+Pj6EhIQwfPhwJk6cSPXq1QkPD2f8+PEEBQWZr9iqXbs2HTp0YODAgcyaNYvs7GyGDh1Kr169dCWWiIiIAFYOO9u3b6dNmzbm51fm0fTr14958+bx4osvkpaWxqBBg0hKSuKuu+5ixYoVuLi4mF+zYMEChg4dStu2bbGzs6N79+5Mnz79th+LiIiIlEwl5j471pSSkoKXlxfJycmasyMiIlJKFPT7u8TO2REREREpCgo7IiIiYtMUdkRERMSmKeyIiIiITVPYEREREZumsCMiIiI2TWFHREREbJrCjoiIiNg0hR0RERGxaQo7IiIiYtMUdkRERMSmKeyIiIiITVPYEREREZumsCMiIiI2TWFHREREbJrCjoiIiNg0hR0RERGxaQo7IiIiYtMUdkRERMSmKeyIiIiITVPYEREREZumsCMiIiI2TWFHREREbJrCjoiIiNg0hR0RERGxaQo7IiIiYtMUdkRERMSmKeyIiIiITVPYEREREZumsCMiIiI2TWFHREREbJrCjoiIiNg0hR0RERGxaQo7IiIiYtMUdkRERMSmKeyIiIiITVPYEREREZumsCMiIiI2TWFHREREbJrCjoiIiNg0hR0RERGxaQo7IiIiYtMUdkRERMSmKeyIiIiITVPYEREREZumsCMiIiI2TWFHREREbJrCjoiIiNg0hR0RERGxaQo7IiIiYtMUdkRERMSmKeyIiIiITVPYEREREZumsCMiIiI2TWFHREREbJrCjoiIiNg0hR0RERGxaQo7IiIiYtMUdkRERMSmKeyIiIiITVPYEREREZumsCMiIiI2TWFHREREbFqJDjs5OTmMHz+e8PBwXF1dqVq1KhMmTMBkMpn3MZlMvPrqqwQGBuLq6kpkZCSHDh2yYtUiIiJSkpTosDNlyhRmzpzJjBkz+Pvvv5kyZQpTp07lww8/NO8zdepUpk+fzqxZs9iyZQvu7u60b9+ejIwMK1YuIiIiJYXBdPUwSQnz4IMP4u/vz5w5c8xt3bt3x9XVlS+++AKTyURQUBAvvPACo0aNAiA5ORl/f3/mzZtHr169CvQ+KSkpeHl5kZycjKenZ7Eci4iIiBStgn5/l+iRnZYtW7JmzRoOHjwIwB9//MGGDRvo2LEjAEePHiU2NpbIyEjza7y8vIiIiGDTpk3X7DczM5OUlBSLh4iIiNgmB2sXcD1jx44lJSWFWrVqYW9vT05ODm+99RZ9+vQBIDY2FgB/f3+L1/n7+5u35ScqKoo33nij+AoXERGREqNEj+wsXryYBQsWsHDhQnbu3Mn8+fN55513mD9//i31O27cOJKTk82PkydPFlHFIiIiUtKU6LAzevRoxo4dS69evahXrx59+/ZlxIgRREVFARAQEABAXFycxevi4uLM2/Lj7OyMp6enxUNERET+ERYWhsFgyPMYMmSIeZ9NmzZx33334e7ujqenJ61bt+bixYvX7HP9+vV07tyZoKAgDAYDy5cvv24NzzzzDAaDgWnTpt3SsZTosJOeno6dnWWJ9vb2GI1GAMLDwwkICGDNmjXm7SkpKWzZsoUWLVrc1lpFRERsybZt24iJiTE/Vq1aBUCPHj2A3KDToUMH2rVrx9atW9m2bRtDhw7N8719tbS0NBo0aMBHH310w/dftmwZmzdvJigo6JaPpUTP2encuTNvvfUWISEh1K1bl127dvHee+/x5JNPAmAwGBg+fDgTJ06kevXqhIeHM378eIKCgujSpYt1ixcRESnF/Pz8LJ5PnjyZqlWrcs899wAwYsQIhg0bxtixY8371KxZ87p9duzY0XyR0fWcPn2a5557jpUrV9KpU6ebqN5SiR7Z+fDDD3nkkUcYPHgwtWvXZtSoUTz99NNMmDDBvM+LL77Ic889x6BBg2jWrBmpqamsWLECFxcXK1YuIiJiO7Kysvjiiy948sknMRgMxMfHs2XLFipWrEjLli3x9/fnnnvuYcOGDbf8Xkajkb59+zJ69Gjq1q1bBNWX8LDj4eHBtGnTOH78OBcvXiQ6OpqJEyfi5ORk3sdgMPDmm28SGxtLRkYGq1evpkaNGlasWkREyrIbzXW5995782x75plnrttnQVYLSExMpE+fPnh6euLt7c2AAQNITU0tkmNavnw5SUlJ9O/fH4AjR44A8PrrrzNw4EBWrFhB48aNadu27S2vYjBlyhQcHBwYNmzYrZZtVqLDjoiISGlzo7kuAAMHDrTYZ+rUqdftsyCrBfTp04e//vqLVatW8f3337N+/XoGDRpUJMc0Z84cOnbsaJ4/c2Xu7NNPP80TTzxBo0aNeP/996lZsyafffbZTb/Pjh07+OCDD5g3bx4Gg6FIaocSPmdHRESktLnRXBcANze36141fDWTycS0adN45ZVXePjhhwH4/PPP8ff3Z/ny5fTq1Yu///6bFStWsG3bNpo2bQrkTgV54IEHeOedd25pku/x48dZvXo1S5cuNbcFBgYCUKdOHYt9a9euzYkTJ276vX777Tfi4+MJCQkxt+Xk5PDCCy8wbdo0jh07dlP9amRHRESkmPx7rssVCxYsoEKFCtxxxx2MGzeO9PT0a/ZRkNUCNm3ahLe3tznoAERGRmJnZ8eWLVtu6Rjmzp1LxYoVLSYKh4WFERQUxIEDByz2PXjwIKGhoTf9Xn379uXPP/9k9+7d5kdQUBCjR49m5cqVN92vRnZERESKyb/nugD85z//ITQ0lKCgIP7880/GjBnDgQMHLEZOrlaQ1QJiY2OpWLGixXYHBwd8fHyuu6LAjRiNRubOnUu/fv1wcPgnMhgMBkaPHs1rr71GgwYNaNiwIfPnz2f//v18/fXX5v3atm1L165dGTp0KACpqakcPnzYvP3o0aPs3r0bHx8fQkJC8PX1xdfX16IGR0dHAgICbnil1/Uo7IiIiBSTf891ASzm0dSrV4/AwEDatm1LdHQ0VatWtUaZ17R69WpOnDhhvuXL1YYPH05GRgYjRowgMTGRBg0asGrVKotjiI6O5uipGH6PPkd4BXcO7NpOmzZtzNtHjhwJQL9+/Zg3b16xHYfCjoiISDHIb65LfiIiIgA4fPhwvmHn6tUCrsyVufK8YcOG5n3i4+MtXnfp0iUSExMLPDcoP+3atcNkMl1z+9ixYy3us/NvU5asZ9zSPXzz6RbsDBDVrd51+8vPzc7TuZrCjoiISDHIb65Lfnbv3g1gEWSudvVqAVfCzZXVAp599lkAWrRoQVJSEjt27KBJkyYArF27FqPRaA5TxeVSjpHz6dkkpGWSmJpFQloWCamZHE9IZ+7vx8z7GU3w0tK9tK7hR6CXa7HW9G8KOyIiIkXsWnNdoqOjWbhwIQ888AC+vr78+eefjBgxgtatW1O/fn3zfrVq1SIqKoquXbsWaLWA2rVr06FDBwYOHMisWbPIzs5m6NCh9OrVq9BXYuUYTSSlXwktWbkhJi2Lc6lZJKZlXm7LDTSJaVkkXcymoIM1OSYTx86lK+yIiIiUdtea6+Lk5MTq1auZNm0aaWlpBAcH0717d1555RWL/Q4cOEBycrL5+YsvvkhaWhqDBg0iKSmJu+66K89qAQsWLGDo0KG0bdsWOzs7unfvzvTp0zEaTSRdzCYxLfNyYMkNKlfCTG6QyQ0uiWlZJKZnFTi8XGEwgLerI77lnPFxd6JCOSecHexYvusMV3dlbzAQVsGtcJ0XAYOpsCfPbFBKSgpeXl4kJydrBXQRESkRYpIvcvRcGuEV3C1GQoxGEykZ2XmCS34h5sqojPEmvum93Rxzg4t7boDxLeeEr7vT5d+d8b3808fdifJujjjY572bzaJtJ3hp6V5yTCbsDQYmdbuDR5uF5PNuN6eg398a2RERESlBjEYT09ce4oPVh8yjItUquuNgZ2cONTk3kV48XRyocDmcWAYWp8ujMf+EmvJuTjjmE14K69FmIbSu4cexc+mEVXC77aevrlDYERERsSKTycTBuFQ2H0lg85EENkUnkHQx22Kfw/FpeV7n4exgDiqWwcWZClfa3Z3N4cXJwTr3EQ70crVayLlCYUdEREqEsLAwjh8/nqd98ODBfPTRR8yePZuFCxeyc+dOLly4wPnz5/H29r5un6+//jpvvPGGRVvNmjXZv3+/+Xl0dDSjRo1iw4YNZGZm0qFDBz788MM8N/ErKiaTicPxueFm05EEthxJJCEt64avG9exFq2qVTAHHGcH+2KpzxYp7IiISImwbds2cnJyzM/37t3L/fffb15AMz09nQ4dOtChQwfGjRtX4H7r1q3L6tWrzc+vvjoqLS2Ndu3a0aBBA9auXQvA+PHj6dy5M5s3b8bO7tZHQ0wmE9Fn09h0eeRmy5EEzqVahhsXRzuahvrQoqov1Su688wXOy3m2dgbDDzUMMjqIySllcKOiIiUCDdaQHP48OEA/Prrr4Xq18HB4Zo31tu4cSPHjh1j165d5gmu8+fPp3z58qxdu9ZiPaqCMplMHDmXZj4ltflIIudSMy32cXawo2lYeZqH+9K8qi8NKntbnGaK6lYvz8ReBZ2bp7AjIiIlzpUFNEeOHGmxgObNOHToEEFBQbi4uNCiRQuioqLMq2pnZmZiMBhwdnY27+/i4oKdnR0bNmwoUNgxmUwcS0i/HGxyH/EXLMONk4MdTULK06KqL82r+NIg2Ou6p6FKysReW6GwIyIiJU5+C2jejIiICObNm0fNmjWJiYnhjTfe4O6772bv3r14eHjQvHlz3N3dGTNmDJMmTcJkMjF27FhycnKIiYnJt0+TycTxhHTznJvNRxKIS8kbbhqHeNO8Sm64aRjsjYtj4ebYlISJvbZCYUdEREqc/BbQvBkdO3Y0/16/fn0iIiIIDQ1l8eLFDBgwAD8/P5YsWcKzzz7L9OnTsbOzo3fv3jRu3Ng8X8dkMnEy8SKbjpxj85FENh9JICY5w+J9nOztaBjiTYvL4aZRSOHDjRQfhR0RERtw+vRpxowZw08//UR6ejrVqlVj7ty5NG3aFMhdNHLMmDH8/PPPJCUl0bp1az788EOqV69+zT7vvfde1q1bl6f9gQce4IcffgByg8Brr73Gp59+SlJSEq1atWLmzJnX7fdGCrqA5s3w9vamRo0aHD582NzWrl07oqOjOXfuHA4ODnh7e1PR35/ad3Vg5KLdbD6SwJl/hRtHewONgsvTvIoPzav40ji0vMJNCaawIyJSyp0/f55WrVrRpk0bfvrpJ/z8/Dh06BDly5cHcgNJly5dcHR05Ntvv8XT05P33nuPyMhI9u3bh7u7e779Ll26lKysf64aSkhIoEGDBuarowCmTp3K9OnTmT9/vnnNpvbt27Nv3z6LpQwKo6ALaN6M1NRUoqOj6du3r0X7qfPpbDp2kc1HElm5ag5n48/yS3oIjrtOA7nhpkFlb/Ocm8Yh5XF1UrgpLRR2RERKuSlTphAcHMzcuXPNbeHh4ebfDx06xObNm9m7dy9169YFYObMmQQEBPDll1/y1FNP5duvj4+PxfOvvvoKNzc3c9gxmUxMmzaNV155hYcffhiAzz//HH9/f5YvX06vXr0KfSzXWkATIDY2ltjYWPOozJ49e/Dw8CAkJMRca9u2benatStDhw4FYNSoUXTu3JnQ0FDOnDnDa6+9hr29Pfc+0IVvdpxi05EEvl28gFQXf+zcvMg8s5/zq2fjdWcXmjeuZx65aRJaHjcnfWWWVvrkRERKue+++4727dvTo0cP1q1bR6VKlRg8eDADBw4Ecq84AixGWuzs7HB2dmbDhg3XDDv/NmfOHHr16mUeCTp69CixsbEWVyx5eXkRERHBpk2bbirsXGsBTYBZs2ZZ3CCwdevWQO5I0JWJzNHR0Rw9FcPv0ecIr+DOqVOn6N27NwkJCZTz9qFitQaEPfk+D/3fHnM/508eJW3vpxgzUvENqMQzw0cx5bVxlHNxLHT9UjJpIVC0EKiIlG5XQszIkSPp0aMH27Zt4/nnn2fWrFn069eP7OxsqlWrRkREBJ988gnu7u68//77jB07lnbt2rFy5cobvsfWrVuJiIhgy5Yt3HnnnQD8/vvvtGrVijNnzhAYGGjet2fPnhgMBhYtWlQ8B3wdi7adYNzSPRhNYACahpYnPjWT4wnpFvvZ2xmoV8nr8tVSPjQN86Gcs/7+L220EKiISBlhNBpp2rQpkyZNAqBRo0bs3bvXHHYcHR1ZunQpAwYMwMfHB3t7eyIjI+nYsSMF/Xt3zpw51KtXzxx0SoL0rEucSEzn2Ll0jieksS8mhW93nzFvNwHbjp8HwM5Abri5POemaWh5PDRyU2Yo7IiIlHKBgYHUqVPHoq127dp888035udNmjRh9+7dJCcnk5WVhZ+fHxEREearta4nLS2Nr776ijfffNOi/cpdiePi4ixGduLi4mjYsOEtHNE/ki9mcyIhnWMJaRxPSON4QjrHLz//9437ruXF9jXo2yJM4aYMU9gRESnlWrVqxYEDByzaDh48SGhoaJ59vby8gNxJy9u3b2fChAk37H/JkiVkZmby2GOPWbSHh4cTEBDAmjVrzOEmJSWFLVu28OyzzxaodpPJRGJaFscS0q8KM2nm5+fTs6/7em83R0J93Qn1ccPX3Yl5vx/j6rEqe4OBro0rK+iUcQo7IiKl3IgRI2jZsiWTJk2iZ8+ebN26ldmzZzN79mzzPkuWLMHPz4+QkBD27NnD888/T5cuXWjXrp15n8cff5xKlSoRFRVl0f+cOXPo0qULvr6+Fu0Gg4Hhw4czceJEqlevbr70PCgoiC5dupj3MxpNxF/INIeZY1dCTWIax8+lcyHz0nWPr0I5Z8J83Qj1dSfM140QXzfCfN0J9XXD283JYt9agR5aU0ryUNgRESnlmjVrxrJlyxg3bhxvvvkm4eHhTJs2jT59+pj3iYmJYeTIkeZTTo8//jjjx4+36OfEiRN5Vvk+cOAAGzZs4Oeff873vV988UXS0tIYNGgQSUlJ3NE4gmcnz+HdNUfMp5yOJ6aRkW287jEEebnkjtBcFWpCfd0J8XUr1MRhrSkl+dHVWOhqLBERgJjkixw9l0Z4Bfd8Q0LWJSOnzqfnOdV0PCGdk+fTyc659teJvZ2BSt6uhF41KnPlZ7CPm+4+LDdFV2OJiEiBXX3Jtp0BHosIJdjHzeKU0+nzFzFe589jJ3s7gn1cCbs8InN1qKlU3hVHe7trv1ikGCnsiIiUcZuPJDD2mz3mib1GE3y++Xi++7o62l8+1XQlzLibnwd6uWJvZ7h9hYsUkMKOiEgZlJCayf/+OMOyXaf541Ryvvs0r+JDk9Dyl+fQ5M6j8fNwxmBQoJHSRWFHRKSMyMjOYfXfcSzbeZp1B89y6fI5KTsDeU5P2RsMvP9oQ03wFZugsCMiYsOMRhNbjyWybOdpftwTY3GZd71KXnRtVInODYJYuz9Ol2yLzVLYERGxQYfjU1m26xTLd53hdNJFc3slb1cebhhEt8aVqFbRw9yuS7bFlinsiIjYiHNXzcP586p5OB7ODjxQL5AujSoREe6D3TUmEQd6uSrkiE1S2BERKcUysnNYtS+OZbty5+HkXJ5842Bn4J4afnRtXInI2v66j42UabrpgYiUaadPn+axxx7D19cXV1dX6tWrx/bt2/Pd95lnnsFgMDBt2rTr9hkWFobBYMjzGDJkCADHjh3Ld7vBYGDJkiU3rNloNLEpOoEXv/6DZhNX89yXu1i7P54co4n6lb14vXMdtrzUljn9m/Fg/SAFHSnzNLIjImXW+fPnadWqFW3atOGnn37Cz8+PQ4cOUb58+Tz7Llu2jM2bNxMUFHTDfrdt20ZOTo75+d69e7n//vvp0aMHAMHBwcTExFi8Zvbs2bz99tt07Njxmv0ejr/A0p2n+XZ33nk4XRoF0bVRZapVLHfD+kTKGoUdESmzpkyZQnBwMHPnzjW3hYeH59nv9OnTPPfcc6xcuZJOnTrdsF8/Pz+L55MnT6Zq1arcc889ANjb2xMQEGCxz7Jly+jZsyflylmGlXOpmXy3O3cezp7TV83DcXGgU71AujaqRLOwa8/DERGFHREpw7777jvat29Pjx49WLduHZUqVWLw4MEMHDjQvI/RaKRv376MHj2aunXrFvo9srKy+OKLLxg5cuQ1b8a3Y8cOdu/ezUcffQTkzsP5eV8cy3aeYv2hcxbzcO6t6UfXRpVpW7uiTk+JFJDCjoiUWUeOHGHmzJmMHDmSl156iW3btjFs2DCcnJzo168fkDv64+DgwLBhw27qPZYvX05SUhL9+/e/5j5z5syhdu3a4F+D0Uv+4Ke9saRedT+cBsHedGtUiQfrB+Jbzvmm6hApyxR2RKTMMhqNNG3alEmTJgHQqFEj9u7dy6xZs+jXrx87duzggw8+YOfOnTe9RMKcOXPo2LHjNef67DkWz9zPv8C/9X/4z6dbzO2Vy7vStVElujSqRFU/zcMRuRUKOyJSZgUGBlKnTh2Lttq1a/PNN98A8NtvvxEfH09ISIh5e05ODi+88ALTpk3j2LFj1+3/+PHjrF69mqVLl1q0n72QyXd/nGHZrlNsXrmcjIsXyanWGm8XBx6sH0jXRpVpGlpe83BEiojCjoiUWa1ateLAgQMWbQcPHiQ0NBSAvn37EhkZabG9ffv29O3blyeeeOKG/c+dO5eKFSvSqVMnLmbl8PO+WJbtOs1vV83DSduzirDGrfloYBvuq6V5OCLFQWFHRMqsESNG0LJlSyZNmkTPnj3ZunUrs2fPZvbs2QD4+vri6+tr8RpHR0cCAgKoWbOmua1t27Z07dqVoUOHmtuMRiNz584l8qEejF32Fyv+NQ+nYbA3rSpmM2bqXmZ+8iMd6gUW89GKlF0KOyJSZjVr1oxly5Yxbtw43nzzTcLDw5k2bRp9+vQpVD8HDx1m18ETxCRfJNDLlYNxF5j8f4s5ceIEa0134LjjFADBPq50bZg7D6eKXzleeuklKleuTLt27Yrj8ETkMoPJZDJZuwhrS0lJwcvLi+TkZDw9Pa1djoiUIou2nWDc0j0YTWAAAr1cOJOcYd7u6eJAp/q5C282DS1/0xOdRSSvgn5/a2RHROQmHT2bxthv9nDlL0YTcCY5Awc7aFPLn26NKtFG83BErE5hR0SkEHKMJjYfSWDpztP88OcZ8hsan9mnCffXDchni4hYg8KOiEgB7I9NYdnldaliUzKuuZ+9wcAdlb1uY2UiciMKOyIi1xCfksF3f5xh6c7T7ItJMbd7uTrSqX4g3RpV4nB8Ki8v20uOyYS9wcCkbncQ6OVqxapF5N8UdkRErpKedYmf/4pj6a7TbDh0lsu3w8HR3kCbmhXp1jh3Ho6zQ+48nKZhPtxT049j59IJq+CmoCNSAinsiEiZl2M08Xv0OZbtOs2KvbGkZ+WYtzUO8aZr48o8WC+Q8u5O+b4+0MtVIUekBFPYEZEy6++YFJbtOs23u08Tl5Jpbg/1daNLw0p0bVSJsAruVqxQRIqCnbULEJHS5/XXX8dgMFg8atWqZd4eHR1N165d8fPzw9PTk549exIXF3fdPnNychg/fjzh4eG4urpStWpVJkyYwNW3Auvfv3+e9+3QoUOhao9PyeDT9UfoMG09HT/4jdnrjxCXkomXqyN9IkL45tkW/DrqXkbcX0NBR8RGaGRHRG5K3bp1Wb16tfm5g0Pu/52kpaXRrl07GjRowNq1awEYP348nTt3ZvPmzdjZ5f831pQpU5g5cybz58+nbt26bN++nSeeeAIvLy+GDRtm3q9Dhw7MnTvX/NzZ2fmGtaZlXuLnfbEs3XmajYfPWczDua9WRbo2qkybWn7meTgiYlsUdkTkpjg4OBAQkPdeMhs3buTYsWPs2rXLfEfT+fPnU758edauXZtnYc0rfv/9dx5++GE6deoEQFhYGF9++SVbt2612M/Z2Tnf9/23HKOJjYdz5+Gs/MtyHk6T0PJ0bVSJB+sH4u2W/zwcEbEdhT6N9csvvxRHHSJSyhw6dIigoCCqVKlCnz59OHHiBACZmZkYDAaLERcXFxfs7OzYsGHDNftr2bIla9as4eDBgwD88ccfbNiwgY4dO1rs9+uvv1KxYkVq1qzJs88+S0JCgsX2fWdSeOuHfbSIWsPjn21l2a7TpGflEOrrxvDI6qwbfS/fPNuSx5qHKuiIlBGFHtnp0KEDlStX5oknnqBfv34EBwcXR10iUoJFREQwb948atasSUxMDG+88QZ33303e/fupXnz5ri7uzNmzBgmTZqEyWRi7Nix5OTkEBMTc80+x44dS0pKCrVq1cLe3p6cnBzeeusti0U5O3ToQLdu3QgPDyc6OpqXXnqJjh07snzlL/zvz1iW7TrN/tgL5v293Rx5sH4gXRtVpnGIt9alEimjCh12Tp8+zX//+1/mz5/PG2+8wX333ceAAQPo0qULTk76K0mkLLh6tKV+/fpEREQQGhrK4sWLGTBgAEuWLOHZZ59l+vTp2NnZ0bt3bxo3bnzN+ToAixcvZsGCBSxcuJC6deuye/duhg8fTlBQEP369QOgV69e5v2r1KjNGYMfQ7rcRePB03AJbQiAk71d7jycxpVoU7MiTg66DkOkrCt02KlQoQIjRoxgxIgR7Ny5k7lz5zJ48GAGDx7Mf/7zHwYMGECDBg2Ko1YRKaG8vb2pUaMGhw8fBqBdu3ZER0dz7tw5HBwc8Pb2JiAggCpVqlyzj9GjRzN27FhzoKlXrx7Hjx8nKirKHHZyjCY2HD7Hsp2nWPlXHBezc7Bz9ST7fAx3tW5D18aVeLBeEF5ujsV/0CJSatzSBOXGjRsTEBCAr68vkydP5rPPPuPjjz+mRYsWzJo1i7p16xZVnSJSgqWmphIdHU3fvn0t2itUqADA2rVriY+P56GHHrpmH+np6XlGfuzt7TEajfx1Jjl3Xao/znD2wj/3wwm0T+NExgWmPnYPAx5rWYRHJCK25KbGd7Ozs/n666954IEHCA0NZeXKlcyYMYO4uDgOHz5MaGgoPXr0KJICT58+zWOPPYavry+urq7Uq1eP7du3m7ebTCZeffVVAgMDcXV1JTIykkOHDhXJe4tI/kaNGsW6des4duwYv//+O127dsXe3p7evXsDMHfuXDZv3kx0dDRffPEFPXr0YMSIEdSsWdPcR9u2bZkxY4b5eefOnXnrrbf44YcfOHbsGJ99sYhJU94mu3JTOk3fwOy1+zj43UwcEw7TuYoDY+tnk7ViCtWqVeOxHtcOUSIihR7Zee655/jyyy8xmUz07duXqVOncscdd5i3u7u788477xAUFHTLxZ0/f55WrVrRpk0bfvrpJ/z8/Dh06BDly5c37zN16lSmT5/O/PnzCQ8PZ/z48bRv3559+/bh4uJyyzWISF6nTp2id+/eJCQk4Ofnx1133cXmzZvx8/MD4MCBA4wbN47ExETCwsJ4+eWXGTFihEUfV05zXfHhhx8y9qWX6f/U05xPOIvB3Qe3Ou0wNumJs70d99wRyK4tCZz49i0+mZ9EUFAQ7dq1Y8KECQW6146IlF0G09W3Jy2Atm3b8tRTT9GtW7dr/h/MpUuX2LhxI/fcc88tFTd27Fg2btzIb7/9lu92k8lEUFAQL7zwAqNGjQIgOTkZf39/5s2bZzGZ8XpSUlLw8vIiOTnZfF8QESl+MckXORyfytkLmaw7eJafL8/DuaJZWHm6NqpMp3qBmocjInkU9Pu70GHndqpTpw7t27fn1KlTrFu3jkqVKjF48GAGDhwIwJEjR6hatSq7du2iYcOG5tfdc889NGzYkA8++CDffjMzM8nM/Oe8f0pKCsHBwQo7IrfRrF8PM2XFAf79f0DhFdzp2ih3XapgHzer1CYipUNBw06h5+xERUXx2Wef5Wn/7LPPmDJlSmG7u64jR44wc+ZMqlevzsqVK3n22WcZNmwY8+fPByA2NhYAf39/i9f5+/ubt+UnKioKLy8v80P3ChK5fU4nXWT4V7uY/K+gYwA+fbwJa1+4h2FtqyvoiEiRKXTY+eSTTywW/Luibt26zJo1q0iKusJoNNK4cWMmTZpEo0aNGDRoEAMHDrzl9xk3bhzJycnmx8mTJ4uoYhG5ljNJF3ll+R7uffsXlu8+k2e7CSjn7Kgb/4lIkSv0BOXY2FgCAwPztPv5+V337qg3IzAwkDp16li01a5dm2+++QbAvD5OXFycRU1xcXEWp7X+zdnZWRMaRW6TmOSLfPxLNIu2nSQrxwhA4xBvdp1M4uqT6PYGA2EVNJojIkWv0CM7wcHBbNy4MU/7xo0bi+QKrKu1atWKAwcOWLQdPHiQ0NBQAMLDwwkICGDNmjXm7SkpKWzZsoUWLVoUaS0iUjixyRm89u1e7pn6K//dfJysHCMR4T58ObA5Swe3YnK3ethfHsWxNxiY1O0OAr1crVy1iNiiQo/sDBw4kOHDh5Odnc19990HwJo1a3jxxRd54YUXirS4ESNG0LJlSyZNmkTPnj3ZunUrs2fPZvbs2QAYDAaGDx/OxIkTqV69uvnS86CgILp06VKktYhIwcSlZDDz12gWbj1B1qXckZw7w3wYfn91WlatYN7v0WYhtK7hx7Fz6YRVcFPQEZFiU+iwM3r0aBISEhg8eDBZWVlA7orGY8aMYdy4cUVaXLNmzVi2bBnjxo3jzTffJDw8nGnTplksDPjiiy+SlpbGoEGDSEpK4q677mLFihW6x47IbRafksHMddEs3HKCzMshp1lYeUZE1qBFVd985+IEerkq5IhIsbvpS89TU1P5+++/cXV1pXr16qV6DozusyNy8+IvZDDr1yMs2HLcHHKahOaGnFbV8g85IiJFoaDf3ze9Nla5cuVo1qzZzb5cREq5sxcy+WRdNF9sOU5G9j8Tj0fcX4O7qlVQyBGREuOmws727dtZvHgxJ06cMJ/KumLp0qVFUpiIlEznUjOZvf4In286Zg45DYNzQ07r6go5IlLyFDrsfPXVVzz++OO0b9+en3/+mXbt2nHw4EHi4uLo2rVrcdQoIiVAgjnkHDcv6dAg2JsRkdW5p4afQo6IlFiFvvR80qRJvP/++/zvf//DycmJDz74gP3799OzZ09CQkKKo0YRXn/9dQwGg8Xj6ptbPv3001StWhVXV1f8/Px4+OGH2b9//3X7/Hd/Vx5vv/22eZ+dO3dy//334+3tja+vL4MGDSI1NbXYjrMkSkzLYvJP+7l76i98sv4IF7NzqF/Zi7n9m7F8cEvurVlRQUdESrRCh53o6Gg6deoEgJOTE2lpaRgMBkaMGGG+JFykONStW5eYmBjzY8OGDeZtTZo0Ye7cufz999+sXLkSk8lEu3btyMnJuWZ/V/cVExPDZ599hsFgoHv37gCcOXOGyMhIqlWrxpYtW1ixYgV//fUX/fv3L+5DLRHOp2UxZcV+7pqyllnroknPyqFeJS8+69+Ub4e0ok0thRwRKR0KfRqrfPnyXLhwAYBKlSqxd+9e6tWrR1JSEunp6UVeoMgVDg4O5rtm/9ugQYPMv4eFhTFx4kQaNGjAsWPHqFq1ar6v+Xdf3377LW3atKFKlSoAfP/99zg6OvLRRx9hZ5f7d8GsWbOoX78+hw8fplq1akVxWCVOUnoWn/52hHkbj5GWlRsW76jkyfC2NWhbWwFHREqfQo/stG7dmlWrVgHQo0cPnn/+eQYOHEjv3r1p27ZtkRcocsWhQ4cICgqiSpUq9OnThxMnTuS7X1paGnPnziU8PLzAi7zGxcXxww8/MGDAAHNbZmYmTk5O5qAD4Oqae0+Yq0eVCqM4TseZTCZeffVVAgMDcXV1JTIykkOHDlns89BDDxESEoKLiwuBgYH07duXM2cs16dKSs/inZUHuGvKL3z0SzRpWTnUDfLk08eb8r+hdxFZx19BR0RKpUKHnRkzZtCrVy8AXn75ZUaOHElcXBzdu3dnzpw5RV6gCEBERATz5s1jxYoVzJw5k6NHj3L33XebRxkBPv74Y8qVK0e5cuX46aefWLVqFU5OTgXqf/78+Xh4eNCtWzdz23333UdsbCxvv/02WVlZnD9/nrFjxwLc0jpwRX06burUqUyfPp1Zs2axZcsW3N3dad++PRkZGeZ92rRpw+LFizlw4ADffPMN0dHRPPLIIwAkp2fz3s8HuHvKL8z45TCpmZeoHejJJ32b8P1zd3G/Qo6IlHamQsjOzjbNnz/fFBsbW5iXlXjJyckmwJScnGztUqSAzp8/b/L09DT93//9n7ktKSnJdPDgQdO6detMnTt3NjVu3Nh08eLFAvVXs2ZN09ChQ/O0L1iwwOTv72+yt7c3OTk5mUaNGmXy9/c3TZ48+abqfu2110wNGjQo8P5//PGHCTAdPnw43+1Go9EUEBBgevvtt81tSUlJJmdnZ9OXX355zX6//fZbk8FgML39417THa+uMIWO+d4UOuZ7U/v315l+2hNjyskxFrhGERFrKej3d6FGdhwcHHjmmWcs/mIUsQZvb29q1KjB4cOHzW1eXl5Ur16d1q1b8/XXX7N//36WLVt2w75+++03Dhw4wFNPPZVn23/+8x9iY2M5ffo0CQkJvP7665w9e9Y8r+dmFOXpuKNHjxIbG0tkZKS5zcvLi4iICDZt2pTva46dieO192bhWrk2M9Yd40LmJWr6ezCzT2N+HHY3He4IwM5OIzkiYjsKfRrrzjvvZPfu3cVQihS1G80PmT17Nvfeey+enp4YDAaSkpJu2GdOTg7jx48nPDwcV1dXqlatyoQJEzBdtepIXFwc/fv3JygoCDc3Nzp06JBnDsmtSk1NJTo6msDAwHy3m0wmTCYTmZmZN+xrzpw5NGnShAYNGlxzH39/f8qVK8eiRYtwcXHh/vvvv6m6i/p0XGxsrLm+f9d7ZdsVI14YhbOrG+GVAth36Ag+XV+hhn85Pu7TmJ+ev5uO9QIVckTEJhX6aqzBgwczcuRITp48SZMmTXB3d7fYXr9+/SIrTm5d3bp1Wb16tfm5g8M/H3l6ejodOnSgQ4cOBV7EdcqUKcycOZP58+dTt25dtm/fzhNPPIGXlxfDhg3DZDLRpUsXHB0d+fbbb/H09OS9994jMjKSffv25fnvpaBGjRpF586dCQ0N5cyZM7z22mvY29vTu3dvjhw5wqJFi2jXrh1+fn6cOnWKyZMn4+rqygMPPGDuo1atWkRFRVnc/DIlJYUlS5bw7rvv5vu+M2bMoGXLlpQrV45Vq1YxevRoJk+ejLe3900dR8eOHc2/169fn4iICEJDQ1m8eLF5cnSfPn24//77iYmJ4Z133qFnz55s3Ljxphe3vZCRzbyNx/jZPoIKfWtwKTmezK2LCNj1f/z03irs7Qv9N4+ISKlS6LBzZXLysGHDzG0GgwGTyYTBYLjuREq5/a53ufbw4cMB+PXXXwvc3++//87DDz9svtdSWFgYX375JVu3bgVyT9Fs3ryZvXv3UrduXQBmzpxJQEAAX375Zb6nigri1KlT9O7dm4SEBPz8/LjrrrvYvHkzfn5+ZGdn89tvvzFt2jTOnz+Pv78/rVu35vfff6dixYrmPg4cOEBycrJFv1999RUmk4nevXvn+75bt27ltddeIzU1lVq1avHJJ5/Qt2/fmzqG/FzrdNyVU3LNmzenfPnyLFu2LN8ar3y2cXFxFqNccXFx1K1Xn49+Ocynvx0hKT0b7NyoXasiz7d9kPrl+xEWGsLWrVto0aJFkR2PiEhJVOiwc/To0eKoQ4rJlfkhLi4utGjRgqioqFu603XLli2ZPXs2Bw8epEaNGvzxxx9s2LCB9957D8B82ujqUQg7OzucnZ3ZsGHDTYedr7766prbgoKC+PHHH2/Yx5mkdI6eSyMm+SKBXrmXkA8aNMjiHj3/9vnnnxe+2EK4cjruWgHqRqfjwsPDCQgIYM2aNTRs2BCAM2cT+X3TZo5VbMWPKw8AUNXPnWFtq/Ng/SDs7QzmeUIFOc0nIlLaFTrshIaGFkcdUgyuzA+pWbMmMTExvPHGG9x9993s3bsXDw+Pm+pz7NixpKSkUKtWLezt7cnJyeGtt96iT58+QO6popCQEMaNG8cnn3yCu7s777//PqdOnbqly7Vv1aJtJxi3dA9GE9gZ4K2u9eh95+1f3qSoT8cZDAaGDx/OxIkTCQ6rwp/Jznz49kRwKw+hzaji507HihdwTtpDKN6cOnmJ6Ohoxo8fT9WqVTWqIyJlQqHDzo3+0n388cdvuhgpWgWZH1JYixcvZsGCBSxcuJC6deuye/duhg8fTlBQEP369cPR0ZGlS5cyYMAAfHx8sLe3JzIyko4dO1pMYi4uGdk5nEhM59i5tNyfCWkcjL3A1mPnzfsYTTBu6R7e/N9feLo64uHiSDlnBzxcHK766Ug5Fwc8L7eVc3Gw2O/Kvu5ODoWa1Fscp+OGDh/J+n2n6NN/AJcupuJSuQ6Nn36bsb2a8VCDSuz7ay/PP/8+b7z+OmlpaQQGBtKhQwdeeeUVnJ2di+YfXkSkBDOYCvkNVL58eYvn2dnZpKen4+TkhJubG4mJiUVa4O2QkpKCl5cXycnJeHp6WrucYtWsWTMiIyOJiooyt/3666+0adOG8+fP33DibXBwMGPHjmXIkCHmtokTJ/LFF1/kudNvcnIyWVlZ+Pn5ERERQdOmTfnoo49u+RguZGRzPCHdHGaOn8v9eSIxnZjk239bhKuD0pVQ5OF8VXC6vM3TxdH8+z+BKbfN3cm+UDfui0m+yP6YFHYcP8+XW0+SkJYFQJivG8PaVuehBkE4aOKxiNi4gn5/F3pk5/z583naDh06xLPPPsvo0aML253cRjeaH1IQ6enpFssnANjb22M0GvPs6+XlBeT+97F9+3YmTJhQ4PdJSs/iWEI6xxPSOHYu9+fxxNyf51KzrvtaD2cHwiq4E+rrRqivG16ujkT9tJ+rY72dARY93RxXRwcuZFwiNfMSFzKyL/+86vnl31MyLpl/v5CRzYWMS1wy5naYmpnbfivsDODu7ICH8+URpDyh6J+RpX0xKXyz4xRX/5US6uvGc/dVp0tDhRwRkX8rdNjJT/Xq1Zk8eTKPPfbYDdfxkdvnevNDIPceLbGxseYrgfbs2YOHhwchISH4+PgA0LZtW7p27crQoUMB6Ny5M2+99RYhISHUrVuXXbt28d577/Hkk0+a33fJkiX4+fkREhLCnj17eP755+nSpQvt2rUz72MymTibmsnxhKtPOV0JN2mkZFw/PPi6OxHi60aYb26oufIz1Ned8m6OeUZJvFwdeWnpXnJMJuwNBiZ1u4NmYb43/W9rMpnIvGT8JxxlXOJCZrb5d3Moyrz6+ZX9LINVjtGE0QQXMnKfU8jRKYMB/jvgTkJ8bu6yfhERW1ckYQdyL3H+98KCYl3Xmx8CuSt4v/HGG+b9W7duDcDcuXPp378/ANHR0Zw7d868z4cffsj48eMZPHgw8fHxBAUF8fTTT/Pqq6+a94mJiTGvmVbRP4DIh3vQtvezRP30t8Upp/Ss69+mwN/TmVBfd8Iuh5groSbE1w1PF8dC/Vs82iyE1jX8OHYunbAKbuarsW6WwWDAxdEeF0d7KpS7+XkvJpOJjGwjFzKzLULRtUaXjiWksSk64V99wOnzGQo7IiLXUOg5O999953Fc5PJRExMDDNmzCA4OJiffvqpSAu8HcrSnJ2bEZN8kaPn0giv4J4nJGTnGDmTdNHilNOJxDSOXZ5Tk3Up7+mtK+wMEOTtetWojNvlcONOiI8brk72xX1opU5M8kVaTV6L8ar/1dobDGwY2+aWA5yISGlTbHN2unTpYvHcYDDg5+fHfffdd8270ErpdfUl2wYDdKoXiI+7kzncnDp/kRzjtfOyo72B4PL/BJmrTzlVLu+Gk4PmlxRGoJcrUd3q5Tklp6AjInJthR7ZsUUa2clffqMI+XF2sDOfXgq7anQm1NeNQC8XTZgtBjHJF4vslJyISGlVbCM7UnYcPZeWb9B5uEEQLav5mkNNRQ9nLSB5mwV6uSrkiIgUUKH/5O7evTtTpkzJ0z516lR69OhRJEVJybDjeN7bDNgbDIx9oBaPNguheRVfArxcFHRERKREK3TYWb9+vcWt66/o2LEj69evL5KixPp+PRDP+6sOAnAlymh+iIiIlEaFPo2VmpqKk5NTnnZHR0dSUlKKpCixrv2xKQxduAujCXo0qcyI+6tzPOGi5oeIiEipVOiRnXr16rFo0aI87V999RV16tQpkqLEeuIvZDBg3nZSMy/RvIoPb3WtR5C3Gy2q+iroiIhIqVTokZ3x48fTrVs3oqOjue+++wBYs2YNX375JUuWLCnyAuX2uZiVw8D52zmddJEqFdyZ9VgTXRouIiKlXqHDTufOnVm+fDmTJk3i66+/xtXVlfr167N69Wruueee4qhRbgOj0cTIxbv541Qy3m6OfNa/Gd5ueU9XioiIlDY3del5p06d6NSpU1HXIlb09s8H+GlvLI72Bmb3bUpYBS09ICIitqHQ5yi2bdvGli1b8rRv2bKF7du3F0lRcnst3naSmb9GAzD1kfrcGe5j5YpERESKTqHDzpAhQzh58mSe9tOnTzNkyJAiKUpun9+jz/HSsj0ADGtbna6NKlu5IhERkaJV6LCzb98+GjdunKe9UaNG7Nu3r0iKktsj+mwqz/x3B5eMJh5qEMSIyOrWLklERKTIFTrsODs7ExcXl6c9JiYGBwetPlFaJKZl8eS8baRkXKJxiDdTH6mPwaA7IYuIiO0pdNhp164d48aNIzk52dyWlJTESy+9xP3331+kxUnxyLyUw9P/3c7xhHSCfVz59PGmuDjaW7ssERGRYlHooZh33nmH1q1bExoaSqNGjQDYvXs3/v7+/Pe//y3yAqVomUwmxnz9J9uOncfDxYHP+jXDt5yztcsSEREpNoUOO5UqVeLPP/9kwYIF/PHHH7i6uvLEE0/Qu3dvHB0di6NGKULT1xxm+e4z2NsZmNmnCdX9PaxdkoiISLG6qUk27u7uDBo0qKhrkWL27e7TvL86d3HPiV3u4K7qFaxckYiISPG76RnF+/bt48SJE2RlZVm0P/TQQ7dclBS97ccSGb3kTwAGta5C7ztDrFyRiIjI7VHosHPkyBG6du3Knj17MBgMmEwmAPOVPDk5OUVbodyyEwnpDPrvDrJyjLSr48/YDrWsXZKIiMhtU+irsZ5//nnCw8OJj4/Hzc2Nv/76i/Xr19O0aVN+/fXXYihRbkXyxWyemLeVxLQs6lXyYlqvhtjZ6RJzEREpOwo9srNp0ybWrl1LhQoVsLOzw87OjrvuuouoqCiGDRvGrl27iqNOuQnZOUYGL9hB9Nk0Ar1c+L9+TXFz0r2QRESkbCn0yE5OTg4eHrlX8FSoUIEzZ84AEBoayoEDB4q2OrlpJpOJ8cv3svFwAu5O9szp1wx/TxdrlyUiInLbFfrP/DvuuIM//viD8PBwIiIimDp1Kk5OTsyePZsqVaoUR41yE2avP8JX205iZ4AP/9OIOkGe1i5JRETEKgoddl555RXS0tIAePPNN3nwwQe5++678fX1ZdGiRUVeoBTeir0xTF6xH4DxD9bhvlr+Vq5IRETEegymK5dT3YLExETKly9fatdWSklJwcvLi+TkZDw9S/cIyJ+nkuj5ySYyso083iKUNx6qW2o/FxERkesp6Pd3kcxW9fHxKYpu5BadSbrIgPnbycg2cm9NP159sI6CjoiIlHmFnqAsJVNq5iWenLeNsxcyqRXgwYe9G+Fgr49XRERE34Y24FKOkecW7mR/7AUqlHNmTv9meLhonTIRERFQ2LEJE3/4m18OnMXF0Y45/ZpSydvV2iWJiIiUGIUOO+vXr+fSpUt52i9dusT69euLpCgpuHkbjzLv92MAvN+zIQ2Cva1aj4iISElT6LDTpk0bEhMT87QnJyfTpk2bIilKCuaX/fG8+f0+AMZ0qEXHeoFWrkhERKTkKXTYMZlM+V7hk5CQgLu7e5EUJTf2d0wKQxfuxGiCnk0r88w9uqGjiIhIfgp86Xm3bt2A3NXN+/fvj7Ozs3lbTk4Of/75Jy1btiz6CiWP+JQMBszbRlpWDi2q+DKxSz1dYi4iInINBQ47Xl5eQO7IjoeHB66u/0yCdXJyonnz5gwcOLDoKxQLF7NyeOrz7ZxJzqCKnzuzHmuCk4PmmYuIiFxLgcPO3LlzAQgLC2PUqFE6ZWUFRqOJEYt28+epZMq7OTK3fzO83HSJuYiIyPUUekjgxRdftDhlcvz4caZNm8bPP/9cpIVJXlNXHmDFX7E42dsx+/GmhPoqcIqIiNxIocPOww8/zOeffw5AUlISd955J++++y4PP/wwM2fOLPICJddXW08wa100AFMfqU+zMC3RISIiUhCFDjs7d+7k7rvvBuDrr78mICCA48eP8/nnnzN9+vQiL1Bg4+FzvLJ8LwDPt61Ol0aVrFyRiIhI6VHosJOeno6HhwcAP//8M926dcPOzo7mzZtz/PjxIi+wrDscf4FnvtjBJaOJhxoEMTyyurVLEhERKVUKHXaqVavG8uXLOXnyJCtXrqRdu3YAxMfHX3d5dSm8hNRMnpy3nQsZl2gSWp6pj9TXJeYiIiKFVOiw8+qrrzJq1CjCwsK48847adGiBZA7ytOoUaMiL7CsysjO4en/7uBEYjrBPq7M7tsEF0d7a5clIiJS6hQ67DzyyCOcOHGC7du3s3LlSnN727Ztef/994u0uH+bPHkyBoOB4cOHm9syMjIYMmQIvr6+lCtXju7duxMXF1esdRQ3k8nEmG/+ZPvx83i4ODC3fzN8yznf+IUiIiKSx03djS4gIAAPDw9WrVrFxYsXAWjWrBm1atUq0uKutm3bNj755BPq169v0T5ixAj+97//sWTJEtatW8eZM2fMd3surT5Yc4hvd5/Bwc7ArMeaUK2ih7VLEhERKbUKHXYSEhJo27YtNWrU4IEHHiAmJgaAAQMG8MILLxR5gQCpqan06dOHTz/9lPLly5vbk5OTmTNnDu+99x733XcfTZo0Ye7cufz+++9s3ry5WGopbst3nWba6kMATOxyB62qVbByRSIiIqVbocPOiBEjcHR05MSJE7i5uZnbH330UVasWFGkxV0xZMgQOnXqRGRkpEX7jh07yM7OtmivVasWISEhbNq06Zr9ZWZmkpKSYvEoCbYdS+TFr/8E4Ol7qtDrzhArVyQiIlL6FXi5iCt+/vlnVq5cSeXKlS3aq1evXiyXnn/11Vfs3LmTbdu25dkWGxuLk5MT3t7eFu3+/v7ExsZes8+oqCjeeOONoi71lhxPSGPQ59vJyjHSoW4AY9oX3ylBERGRsqTQIztpaWkWIzpXJCYmWqyEXhROnjzJ888/z4IFC3BxcSmyfseNG0dycrL5cfLkySLr+2Ykp2fzxLxtnE/Ppn5lL95/tCF2drrEXEREpCgUOuzcfffd5uUiAAwGA0ajkalTp9KmTZsiLW7Hjh3Ex8fTuHFjHBwccHBwYN26dUyfPh0HBwf8/f3JysoiKSnJ4nVxcXEEBARcs19nZ2c8PT0tHlebOXMm9evXN29r0aIFP/30k3n7vffei8FgsHg888wz1z2WuLg4+vfvT1BQEG5ubnTo0IFDhw6RdcnIM1/s4MjZNHzt0nHe8DHhIZVwd3encePGfPPNN4X/hxMRERGzQp/Gmjp1Km3btmX79u1kZWXx4osv8tdff5GYmMjGjRuLtLi2bduyZ88ei7YnnniCWrVqMWbMGIKDg3F0dGTNmjV0794dgAMHDnDixAnz/X9uRuXKlZk8eTLVq1fHZDIxf/58Hn74YXbt2kXdunUBGDhwIG+++ab5NfmNdl1hMpno0qULjo6OfPvtt3h6evLee+8RGRlJt7cWselIAu5O9rism8nxi6l89913VKhQgYULF9KzZ0+2b9+uexiJiIjcJIPJZDIV9kXJycnMmDGDP/74g9TUVBo3bsyQIUMIDAwsjhot3HvvvTRs2JBp06YB8Oyzz/Ljjz8yb948PD09ee655wD4/fffC9xnSkoKXl5eJCcnX/Mu0D4+Prz99tsMGDAgTw03cvDgQWrWrMnevXvNYcloNOLl64dT8z54NWzP//VrykNNqzJz5kz69u1rfq2vry9TpkzhqaeeKvDxiIiIlAUF+f6GmxjZOXHiBMHBwbz88sv5bgsJub1XEL3//vvY2dnRvXt3MjMzad++PR9//HGR9Z+Tk8OSJUtIS0uzGC1asGABX3zxBQEBAXTu3Jnx48dfc3QnMzMTwGLe0c/74riYY4fp1D5efWUE99Xyp2XLlixatIhOnTrh7e3N4sWLycjI4N577y2y4xERESlrCj2yY29vT0xMDBUrVrRoT0hIoGLFiuTk5BRpgbdDfslwz549tGjRgoyMDMqVK8fChQt54IEHAJg9ezahoaEEBQXx559/MmbMGO68806WLl2ab//Z2dlUq1aNiIgIPvnkE6ITs+k4aCzn1s6lasOWHN6Ve/ovKSmJRx99lJ9//hkHBwfc3NxYsmSJef0xERER+UexjeyYTKZ8F6NMTU0t0iumrK1mzZrs3r2b5ORkvv76a/r168e6deuoU6cOgwYNMu9Xr149AgMDadu2LdHR0VStWjVPX46OjixdupQBAwbg4+MDdna4hDakUr0WVPP/5+7I48ePJykpidWrV1OhQgWWL19Oz549+e2336hXr95tOW4RERFbU+CRnZEjRwLwwQcfMHDgQItTNjk5OWzZsgV7e/sin6R8OxQkGUZGRlK1alU++eSTPNvS0tIoV64cK1asoH379td8nwsZ2XR5fxUHY5KoW6UyCQtHcWezZnz00UdER0dTrVo1i3k9V963WrVqzJo169YPVERExIYU+cjOrl27gNyRnT179uDk5GTe5uTkRIMGDRg1atQtlFyyGY1G89ybf9u9ezfAdSdoX8ox8tyXu4hONhHgX5GX7y7PvS/s4K2JEwFIT08HwM7O8m4A9vb2GI3GIjgCERGRsqnAYeeXX34Bci/9/uCDD66boEq7cePG0bFjR0JCQrhw4QILFy7k119/ZeXKlURHR5vn7/j6+vLnn38yYsQIWrdubbFIaa1atYiKiqJr164A9B73Pr+dysLNx58+1V3o2/0punTpYp6PU6tWLapVq8bTTz/NO++8g6+vL8uXL2fVqlV8//33Vvl3EBERsQWFnrMzd+7c4qijRImPj+fxxx8nJiYGLy8v6tevz8qVK7n//vs5efIkq1evZtq0aaSlpREcHEz37t155ZVXLPo4cOAAycnJAMzbeJQ1Ow+SsnUphovJvB8UyOOPP8748ePN+zs6OvLjjz8yduxYOnfuTGpqKtWqVWP+/PnmidEiIiJSeDd1nx1bU9BzfoUVk3yRpTtP8c7Kg5iAsR1r8cw9eScwi4iISOEV29VYUjCLtp1g7NI9XImSzULL83TrKtYtSkREpAwq9NpYcmMxyRcZd1XQAdhx4jyxKRnWK0pERKSMUtgpBkfPpWH818lBowmOnUu3TkEiIiJlmMJOMQiv4I7dv+67aG8wEFbh2ouFioiISPFQ2CkGgV6uRHWrh/3lO03bGwxM6nYHgV6uVq5MRESk7NEE5WLyaLMQWtfw49i5dMIquCnoiIiIWInCTjEK9HJVyBEREbEyncYSERERm6awIyIiIjZNYUdERERsmsKOiIiI2DSFHREREbFpCjsiIiJi0xR2RERExKYp7IiIiIhNU9gRERERm6awIyIiIjZNYUdERERsmsKOiIiI2DSFHREREbFpCjsiIiJi0xR2RERExKYp7IiIiIhNU9gRERERm6awIyIiIjZNYUdERERsmsKOiIiI2DSFHREREbFpCjsiIiJi0xR2RERExKYp7IiIiIhNU9gRERERm6awIyIiIjZNYUdERERsmsKOiIiI2DSFHREREbFpCjsiIiJi0xR2RERExKYp7IiIiIhNU9gRERERm6awIyIiIjZNYUdERERsmsKOiIiI2DSFHREREbFpCjsiIiJi0xR2RERExKYp7IiIiIhNU9gRERERm6awIyIiIjZNYUdERERsmsKOiIiI2DSFHREREbFpCjsiIiJi0xR2RERExKYp7IiIiIhNU9gRERERm6awIyIiIjatRIedqKgomjVrhoeHBxUrVqRLly4cOHDAYp+MjAyGDBmCr68v5cqVo3v37sTFxVmpYhERESlpSnTYWbduHUOGDGHz5s2sWrWK7Oxs2rVrR1pamnmfESNG8L///Y8lS5awbt06zpw5Q7du3axYtYiIiJQkBpPJZLJ2EQV19uxZKlasyLp162jdujXJycn4+fmxcOFCHnnkEQD2799P7dq12bRpE82bNy9QvykpKXh5eZGcnIynp2dxHoKIiIgUkYJ+f5fokZ1/S05OBsDHxweAHTt2kJ2dTWRkpHmfWrVqERISwqZNm67ZT2ZmJikpKRYPERERsU2lJuwYjUaGDx9Oq1atuOOOOwCIjY3FyckJb29vi339/f2JjY29Zl9RUVF4eXmZH8HBwcVZuoiIiFhRqQk7Q4YMYe/evXz11Ve33Ne4ceNITk42P06ePFkEFYqIiEhJ5GDtAgpi6NChfP/996xfv57KlSub2wMCAsjKyiIpKclidCcuLo6AgIBr9ufs7Iyzs3NxliwiIiIlRIke2TGZTAwdOpRly5axdu1awsPDLbY3adIER0dH1qxZY247cOAAJ06coEWLFre7XBERESmBSvTIzpAhQ1i4cCHffvstHh4e5nk4Xl5euLq64uXlxYABAxg5ciQ+Pj54enry3HPP0aJFiwJfiSUiIiK2rURfem4wGPJtnzt3Lv379wdybyr4wgsv8OWXX5KZmUn79u35+OOPr3sa69906bmIiEjpU9Dv7xIddm4XhR0REZHSxybvsyMiIiJSWAo7IiIiYtMUdkRERMSmKeyIiIiITVPYEREREZumsCMiIiI2TWFHREREbJrCjoiIiNg0hR0RERGxaQo7IiIiYtMUdkRERMSmKeyIiIiITVPYEREREZumsCMiIiI2TWFHREREbJrCjoiIiNg0hR0RERGxaQo7IiIiYtMUdkRERMSmKeyIiIiITVPYEREREZumsCMiIiI2TWFHREREbJrCjoiIiNg0hR0RERGxaQo7IiIiYtMUdkRERMSmKeyIiIiITVPYEREREZumsCMiIiI2TWFHREREbJrCjoiIiNg0hR0RERGxaQo7IiIiYtMUdkRERMSmKeyIiIiITVPYEREREZumsCMiIiI2TWFHREREbJrCjoiIiNg0hR0RERGxaQo7IiIiYtMUdkRERMSmKeyIiIiITVPYEREREZumsCMiIiI2TWFHREREbJrCjoiIiNg0hR0RERGxaQo7IiIiYtMUdkRERMSmKeyIiIiITVPYEREREZumsCMiIiI2TWFHREREbJrCjoiIiNg0hR0RERGxaQo7IiIiYtMUdkRERMSmKeyIiIiITVPYEREREZumsCMiIiI2TWFHREREbJrNhJ2PPvqIsLAwXFxciIiIYOvWrdYuSUREREoAmwg7ixYtYuTIkbz22mvs3LmTBg0a0L59e+Lj461dmoiIiFiZTYSd9957j4EDB/LEE09Qp04dZs2ahZubG5999pm1SxMRERErc7B2AbcqKyuLHTt2MG7cOHObnZ0dkZGRbNq0Kd/XZGZmkpmZaX6enJwMQEpKSvEWKyIiIkXmyve2yWS67n6lPuycO3eOnJwc/P39Ldr9/f3Zv39/vq+JiorijTfeyNMeHBxcLDWKiIhI8blw4QJeXl7X3F7qw87NGDduHCNHjjQ/NxqNJCYm4uvri8FgKLL3SUlJITg4mJMnT+Lp6Vlk/crN0edR8ugzKVn0eZQs+jxuzGQyceHCBYKCgq67X6kPOxUqVMDe3p64uDiL9ri4OAICAvJ9jbOzM87OzhZt3t7exVUinp6e+g+1BNHnUfLoMylZ9HmULPo8ru96IzpXlPoJyk5OTjRp0oQ1a9aY24xGI2vWrKFFixZWrExERERKglI/sgMwcuRI+vXrR9OmTbnzzjuZNm0aaWlpPPHEE9YuTURERKzMJsLOo48+ytmzZ3n11VeJjY2lYcOGrFixIs+k5dvN2dmZ1157Lc8pM7EOfR4ljz6TkkWfR8miz6PoGEw3ul5LREREpBQr9XN2RERERK5HYUdERERsmsKOiIiI2DSFHREREbFpCjvF6KOPPiIsLAwXFxciIiLYunWrtUsqk6KiomjWrBkeHh5UrFiRLl26cODAAWuXJZdNnjwZg8HA8OHDrV1KmXX69Gkee+wxfH19cXV1pV69emzfvt3aZZVZOTk5jB8/nvDwcFxdXalatSoTJky44fpPcm0KO8Vk0aJFjBw5ktdee42dO3fSoEED2rdvT3x8vLVLK3PWrVvHkCFD2Lx5M6tWrSI7O5t27dqRlpZm7dLKvG3btvHJJ59Qv359a5dSZp0/f55WrVrh6OjITz/9xL59+3j33XcpX768tUsrs6ZMmcLMmTOZMWMGf//9N1OmTGHq1Kl8+OGH1i6t1NKl58UkIiKCZs2aMWPGDCD3rs7BwcE899xzjB071srVlW1nz56lYsWKrFu3jtatW1u7nDIrNTWVxo0b8/HHHzNx4kQaNmzItGnTrF1WmTN27Fg2btzIb7/9Zu1S5LIHH3wQf39/5syZY27r3r07rq6ufPHFF1asrPTSyE4xyMrKYseOHURGRprb7OzsiIyMZNOmTVasTACSk5MB8PHxsXIlZduQIUPo1KmTxf9O5Pb77rvvaNq0KT169KBixYo0atSITz/91NpllWktW7ZkzZo1HDx4EIA//viDDRs20LFjRytXVnrZxB2US5pz586Rk5OT5w7O/v7+7N+/30pVCeSOsA0fPpxWrVpxxx13WLucMuurr75i586dbNu2zdqllHlHjhxh5syZjBw5kpdeeolt27YxbNgwnJyc6Nevn7XLK5PGjh1LSkoKtWrVwt7enpycHN566y369Olj7dJKLYUdKVOGDBnC3r172bBhg7VLKbNOnjzJ888/z6pVq3BxcbF2OWWe0WikadOmTJo0CYBGjRqxd+9eZs2apbBjJYsXL2bBggUsXLiQunXrsnv3boYPH05QUJA+k5uksFMMKlSogL29PXFxcRbtcXFxBAQEWKkqGTp0KN9//z3r16+ncuXK1i6nzNqxYwfx8fE0btzY3JaTk8P69euZMWMGmZmZ2NvbW7HCsiUwMJA6depYtNWuXZtvvvnGShXJ6NGjGTt2LL169QKgXr16HD9+nKioKIWdm6Q5O8XAycmJJk2asGbNGnOb0WhkzZo1tGjRwoqVlU0mk4mhQ4eybNky1q5dS3h4uLVLKtPatm3Lnj172L17t/nRtGlT+vTpw+7duxV0brNWrVrluRXDwYMHCQ0NtVJFkp6ejp2d5dezvb09RqPRShWVfhrZKSYjR46kX79+NG3alDvvvJNp06aRlpbGE088Ye3SypwhQ4awcOFCvv32Wzw8PIiNjQXAy8sLV1dXK1dX9nh4eOSZL+Xu7o6vr6/mUVnBiBEjaNmyJZMmTaJnz55s3bqV2bNnM3v2bGuXVmZ17tyZt956i5CQEOrWrcuuXbt47733ePLJJ61dWqmlS8+L0YwZM3j77beJjY2lYcOGTJ8+nYiICGuXVeYYDIZ82+fOnUv//v1vbzGSr3vvvVeXnlvR999/z7hx4zh06BDh4eGMHDmSgQMHWrusMuvChQuMHz+eZcuWER8fT1BQEL179+bVV1/FycnJ2uWVSgo7IiIiYtM0Z0dERERsmsKOiIiI2DSFHREREbFpCjsiIiJi0xR2RERExKYp7IiIiIhNU9gRERERm6awIyLyL7/++isGg4GkpCRrlyIiRUBhR0RERGyawo6IiIjYNIUdESlxjEYjUVFRhIeH4+rqSoMGDfj666+Bf04x/fDDD9SvXx8XFxeaN2/O3r17Lfr45ptvqFu3Ls7OzoSFhfHuu+9abM/MzGTMmDEEBwfj7OxMtWrVmDNnjsU+O3bsoGnTpri5udGyZcs8q4OLSOmgsCMiJU5UVBSff/45s2bN4q+//mLEiBE89thjrFu3zrzP6NGjeffdd9m2bRt+fn507tyZ7OxsIDek9OzZk169erFnzx5ef/11xo8fz7x588yvf/zxx/nyyy+ZPn06f//9N5988gnlypWzqOPll1/m3XffZfv27Tg4OGjVaZFSSguBikiJkpmZiY+PD6tXr6ZFixbm9qeeeor09HQGDRpEmzZt+Oqrr3j00UcBSExMpHLlysybN4+ePXvSp08fzp49y88//2x+/YsvvsgPP/zAX3/9xcGDB6lZsyarVq0iMjIyTw2//vorbdq0YfXq1bRt2xaAH3/8kU6dOnHx4kVcXFyK+V9BRIqSRnZEpEQ5fPgw6enp3H///ZQrV878+Pzzz4mOjjbvd3UQ8vHxoWbNmvz9998A/P3337Rq1cqi31atWnHo0CFycnLYvXs39vb23HPPPdetpX79+ubfAwMDAYiPj7/lYxSR28vB2gWIiFwtNTUVgB9++IFKlSpZbHN2drYIPDfL1dW1QPs5OjqafzcYDEDufCIRKV00siMiJUqdOnVwdnbmxIkTVKtWzeIRHBxs3m/z5s3m38+fP8/BgwepXbs2ALVr12bjxo0W/W7cuJEaNWpgb29PvXr1MBqNFnOARMR2aWRHREoUDw8PRo0axYgRIzAajdx1110kJyezceNGPD09CQ0NBeDNN9/E19cXf39/Xn75ZSpUqECXLl0AeOGFF2jWrBkTJkzg0UcfZdOmTcyYMYOPP/4YgLCwMPr168eTTz7J9OnTadCgAcePHyc+Pp6ePXta69BFpJgo7IhIiTNhwgT8/PyIioriyJEjeHt707hxY1566SXzaaTJkyfz/PPPc+jQIRo2bMj//vc/nJycAGjcuDGLFy/m1VdfZcKECQQGBvLmm2/Sv39/83vMnDmTl156icGDB5OQkEBISAgvvfSSNQ5XRIqZrsYSkVLlypVS58+fx9vb29rliEgpoDk7IiIiYtMUdkRERMSm6TSWiIiI2DSN7IiIiIhNU9gRERERm6awIyIiIjZNYUdERERsmsKOiIiI2DSFHREREbFpCjsiIiJi0xR2RERExKYp7IiIiIhN+3+y6lfWWi9Y7AAAAABJRU5ErkJggg==", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAG2CAYAAACZEEfAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABpBElEQVR4nO3deXhM5/sG8Hsy2SMJ2RNkISTEvsROkSbU115b1V7VNlpL7a2lVUIXVKmtBLVXbaUVkRK1BSGIEhEkEVlEZLIvZt7fH2nmZyTIyGQb9+e65rrMmTPPPBOZnHvOec97JEIIASIiIiItpVPRDRARERGVJYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moVGnZOnTqF3r17w8HBARKJBAcOHFB5XAiBefPmwd7eHkZGRvDy8kJkZKTKOikpKRg+fDjMzMxQvXp1jBs3DhkZGeX4LoiIiKgyq9Cwk5mZiaZNm2L16tXFPv7tt99i5cqVWLt2LUJCQmBiYgIfHx/k5OQo1xk+fDhu3LiBwMBAHD58GKdOncKHH35YXm+BiIiIKjlJZbkQqEQiwf79+9GvXz8ABXt1HBwc8Pnnn2PatGkAAJlMBltbW2zevBlDhw7FzZs30bBhQ1y8eBGtWrUCABw9ehTvvPMOHjx4AAcHh4p6O0RERFRJ6FZ0Ay9y7949JCQkwMvLS7nM3Nwcbdq0wblz5zB06FCcO3cO1atXVwYdAPDy8oKOjg5CQkLQv3//Ymvn5uYiNzdXeV+hUCAlJQWWlpaQSCRl96aIiIhIY4QQSE9Ph4ODA3R0XnywqtKGnYSEBACAra2tynJbW1vlYwkJCbCxsVF5XFdXFxYWFsp1iuPn54evvvpKwx0TERFRRYiNjUWtWrVe+HilDTtlafbs2Zg6daryvkwmg6OjI2JjY2FmZlaBnREREVFJpaWloXbt2jA1NX3pepU27NjZ2QEAEhMTYW9vr1yemJiIZs2aKddJSkpSed7Tp0+RkpKifH5xDAwMYGBgUGS5mZkZww4REVEV86ohKJV2nh0XFxfY2dkhKChIuSwtLQ0hISFo164dAKBdu3ZITU1FaGiocp2///4bCoUCbdq0KfeeiYiIqPKp0D07GRkZuHPnjvL+vXv3EBYWBgsLCzg6OmLy5Mn45ptvUK9ePbi4uGDu3LlwcHBQnrHVoEED9OjRA+PHj8fatWuRn5+PiRMnYujQoTwTi4iIiABUcNi5dOkSunbtqrxfOI5m1KhR2Lx5M2bMmIHMzEx8+OGHSE1NRceOHXH06FEYGhoqn7N9+3ZMnDgR3bt3h46ODgYOHIiVK1eW+3shIiKiyqnSzLNTkdLS0mBubg6ZTMYxO0RERFVESbfflXbMDhEREZEmMOwQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQERGRVmPYISIioiLkcjnmzp0LFxcXGBkZoW7duli4cCGEEMp1Ro8eDYlEonLr0aPHK2vHxcXh/fffh6WlJYyMjNC4cWNcunRJ+fi+ffvg7e0NS0tLSCQShIWFleq96Jbq2URERKSVli5dijVr1mDLli3w8PDApUuXMGbMGJibm+Ozzz5TrtejRw/4+/sr7xsYGLy07pMnT9ChQwd07doVf/31F6ytrREZGYkaNWoo18nMzETHjh0xePBgjB8/vtTvhXt2iIheoSTfcJ/10UcfQSKRYMWKFaWu+7rfnLWpf6oYZ8+eRd++fdGrVy84Ozvj3Xffhbe3Ny5cuKCynoGBAezs7JS3Z0NLcZYuXYratWvD398fnp6ecHFxgbe3N+rWratcZ8SIEZg3bx68vLw08l64Z4eI6BVK+g0XAPbv34/z58/DwcFBY3XV/easbf1TxWjfvj3Wr1+P27dvo379+rh69SpOnz6NZcuWqax38uRJ2NjYoEaNGujWrRu++eYbWFpavrDuoUOH4OPjg0GDBiE4OBg1a9bEJ598opE9OC/CsENE9ArPfsMFAGdnZ+zcubPIN9y4uDh8+umnCAgIUK6ribqF35zf1P6pYsyaNQtpaWlwd3eHVCqFXC7HokWLMHz4cOU6PXr0wIABA+Di4oKoqCjMmTMHPXv2xLlz5yCVSoute/fuXaxZswZTp07FnDlzcPHiRXz22WfQ19fHqFGjyuS98DAWEdErtG/fHkFBQbh9+zYAKL/h9uzZU7mOQqHAiBEjMH36dHh4eGisLvD/35zd3Nzw8ccf4/Hjx29U/1VZWR1CBIDVq1fD2dkZhoaGaNOmTZGQGRUVhf79+8Pa2hpmZmYYPHgwEhMTS9z7nj17sH37duzYsQOXL1/Gli1b8P3332PLli3KdYYOHYo+ffqgcePG6NevHw4fPoyLFy/i5MmTL6yrUCjQokULLF68GM2bN8eHH36I8ePHY+3atSXuTW2ChEwmEwCETCar6FaIqBKSy+Vi5syZQiKRCF1dXSGRSMTixYtV1lm8eLF4++23hUKhEEII4eTkJJYvX17qujt37hQHDx4U165dE/v37xcNGjQQrVu3Fk+fPn1j+q/KFi1aJCwtLcXhw4fFvXv3xG+//SaqVasmfvzxxyLr7tu3TzRt2lQ4ODi88me/a9cuoa+vLzZt2iRu3Lghxo8fL6pXry4SExOFEEJkZGSIOnXqiP79+4tr166Ja9euib59+4rWrVsLuVxeot5r1aolVq1apbJs4cKFws3N7aXPs7KyEmvXrn3h446OjmLcuHEqy37++Wfh4OBQZN179+4JAOLKlSvF1irp9pthRzDsENHL7dy5U9SqVUvs3LlTXLt2TWzdulVYWFiIzZs3CyGEuHTpkrC1tRVxcXHK55QkLLyqbnGioqIEAHH8+PE3ov+nT5+KL7/8Ujg7OwtDQ0NRp04d8fXXXytDmRBCzJ8/X7i5uQljY2NRvXp10b17d3H+/PmX1nVychIAitw++eQT5Trr1q0TXbp0EaampgKAePLkSYl6flavXr3E2LFjVZYNGDBADB8+XGXZgwcPRM2aNUV4eHiJfvaenp7C19dXeV8ulwsHBwfh5+cnhBAiICBA6OjoqGzXUlNThUQiEYGBgSXq3cLCQvz8888qyxYvXizq1av3wufExsYKiUQiDh48+MJ1hg0bJjp27KiybPLkyaJdu3ZF1mXY0SCGHSJ6mVd9w12+fLmQSCRCKpUqbwCEjo6OcHJyeu26L/Kqb87a1H9J9oxs375dBAYGiqioKBEeHi7GjRsnzMzMRFJS0gvrJiUlifj4eOUtMDBQABAnTpxQrrN8+XLh5+cn/Pz8XjvsLFq0SDg5OYmIiAghhBBhYWHCxsZGbNu2TbmOXC4XXbt2FStWrBBCvDpo5ubmCqlUKvbv36+yfOTIkaJPnz5CCCEOHTokpFKpyMnJUT6ek5MjpFKpmD9/fol6HzVqlKhZs6byZ79v3z5hZWUlZsyYIYQQIj09XUybNk2cO3dO3Lt3Txw/fly0aNFC1KtXT+V1u3XrJn766Sfl/QsXLghdXV2xaNEiERkZKbZv3y6MjY1VfiaPHz8WV65cEUeOHBEAxK5du8SVK1dEfHy8So8l3X5zgDIR0StkZWVBR0d1iKNUKoVCoQBQcJrs86fI+vj4YMSIERgzZsxr1y3OgwcP8PjxY9jb278R/ZdkEPR7772n8pxly5Zh48aNuHbtGrp3715sXWtra5X7S5YsQd26ddGlSxflssmTJwPAS8efvEpJBvkuXboUurq6Rc6Me5Hk5GTI5XLY2tqqLLe1tcWtW7cAAG3btoWJiQlmzpyJxYsXQwiBWbNmQS6XIz4+vkSv89NPP2Hu3Ln45JNPkJSUBAcHB0yYMAHz5s0DUPB/fe3aNWzZsgWpqalwcHCAt7c3Fi5cqHLG3e3IO7hyOwbxsmzYmxuhdevW2L9/P2bPno2vv/4aLi4uWLFihcrP5NChQyq/e0OHDgUAzJ8/HwsWLChR/ypKFO+0HPfsENHLvOobbnGK+3b+/DdcTX1z1ub+S7Jn5Fm5ubniu+++E+bm5uLRo0cleo3c3FxhaWkpFi1aVOzjJ06ceO09O2VxCDEuLk4AEGfPnlVZPn36dOHp6am8HxAQIOrUqaPca/f++++LFi1aiI8++kjt9/G6dl2IFi6zDgunmYeFy6zDYteFaI3W52EsNTDsENHLpKWliUmTJglHR0fluJEvvvhC5ObmvvA5xW2wnJycVA4hvKpuVlaW8Pb2FtbW1kJPT084OTmJ8ePHi4SEhDem/5IMghZCiD/++EOYmJgIiUQiHBwcxIULF0r8Grt37xZSqVQlcDyrNGGnLA4hluQw1rMePXqk7N3W1lZ8++23ar+P1/EwNUsZdApvdWYdEQ9TszT2GiXdfkuEeMH5b2+QtLQ0mJubQyaTwczMrKLbISItFS/Lxr3kTLhYmcDe3Kii21FbRfS/a9cuTJ8+Hd999x08PDwQFhaGyZMnY9myZSpzsmRmZiI+Ph7JycnYsGED/v77b4SEhMDGxuaVr+Hj4wN9fX388ccfxT5+8uRJdO3aFU+ePEH16tXV6t/S0hLffPMNPv74Y+UyPz8/+Pv74/bt23j8+HGRw0rPHkJ0c3Mrtm6bNm3g6emJn376CUDB6dyOjo6YOHEiZs2aVexz/v77b3h5eeHmzZsvrKtJZ6OS8d6GkCLLd45vi3Z1XzzpoDpKuv3mmB0ionKw+2IMZu+7DoUAdCSA34DGGNLasaLbKrGK6n/69OmYNWuWcsxG48aNER0dDT8/P5WwY2JiAldXV7i6uqJt27aoV68eNm7ciNmzZ7+0fnR0NI4fP459+/aVSf+9e/fGokWL4OjoCA8PD1y5cgXLli3D2LFjARSEoednG9bT04OdnZ1KIOnevTv69++PiRMnAgCmTp2KUaNGoVWrVvD09MSKFSuQmZmpMs7F398fDRo0gLW1Nc6dO4dJkyZhypQp5RJ0ACDkbkqRZVKJBM5WxuXy+s9i2CEiKmPxsmxlUAAAhQDm7AtH5/rWVWIPT7wsG7P2XYd4pv/Z+66XS/+vMwgaKNjTkZub+8r6/v7+sLGxKdGM0a/jVYN8SyoqKgrJycnK+0OGDMGjR48wb948JCQkoFmzZjh69KjKoOWIiAjMnj0bKSkpcHZ2xhdffIEpU6Zo7L29zJ6LsfgxKBIAIEHBef1SiQSLBzSqkN95HsYCD2MRUdnJfSrHvIM3sPtibJHHNLk7vywdvBKHSbvDiizvUNcSH3Sqg471rKAnLZsJ+UePHo3jx49j3bp1yj0jH374IcaOHYulS5ciMzMTixYtQp8+fWBvb4/k5GSsXr0aO3bsQGhoqHI26Of3jAAFgcjFxQXDhg3DkiVLirx2QkICEhIScOnSJYwfPx6nTp2CqakpHB0dYWFhUSbv90Wq0iHQI9fi8enOy1AI4MPOdTC6vROiH2fD2cpY473zMBYRUQU7cSsJXx/+F/eSM4t93LKafjl3pL4nmXn4ITCi2MfORD3GmajHsDDRR6/G9ujbzAEtnWpAIpFo7PVLcvrzrVu3sGXLFiQnJ8PS0hKtW7fGP//8o3LZi+dPfwaA48ePIyYmRnlI6Xlr167FV199pbzfuXNnAAV7g0aPHq2x9/gqVekQ6ImIJEzefQUKAQzzdMTsnu6QSCRwqF7+h66exT074J4dItKs6MeZ+PqPfxF0KwkAYG1qgK5u1vg99AHkz/zFbVfHEpvHtoaBbvEXTKxoOflyvLfhPC7HpMLcSBfpOU+hEIBUAnz0lisyc5/i8LWHSM7IUz6nVg0j9G3mgL7NaqK+rWkFdv//qlJYeF54nAz/++m0yjKpRILTs7pWuj08IXcfY+SmC8h9qkDvpg5YMaQZpDqaC77FKen2m2EHDDtEpBlZeU+x+sQdbDh1D3lyBXR1JBjb0QWfdnOFqaEe4mXZuJ+chbyncvjuuIKM3Kf4XxN7rBzaHDplvFFQl1wh8NG2UAT+mwgzQ138/nF7VDPUxf3kLJXDEU/lCpyNeowDYXEICE9AZp5cWaOBvRn6NnNAn6YOcKhe/hvmR+m5OBuVjMm7wvDshq6yhoVn5csV2HouGt8H3EJ2ftHxSZXtEOi1B6l4b0MIMnKforu7DdaOaFlmhzafxbCjBoYdIioNIQQOX4vH4j9vIl6WAwDoVM8K83t7wNWmWrHPOR2ZjDGbLyBfLvBBRxd8+b+G5dnySwkh8OWBcGwPiYG+rg62jWsDT5dXj1HJyZfj+M1EHLjyEMG3k5D/324siQTwdLZA32Y18U5jO1Q31uzhu+w8OSKT0nErIR234tMRkZiGiIR0lT1Oz1s5tBn6NKup0T405eydZMw/dAORSRkvXGflsGbo07Ry9B+ZmI7B687hSVY+2tWxhP+Y1jDUK5+9lQw7amDYIaLXdSshDQsO3cD5/06zrVXDCHP/1xDeDW1fOXblwJU4TP5v4O+XvRrgg051yrrdEln1dyS+P3YbEgmwZngL9GhU8ktTFErNysOf1xNwMCwOIff+/xRkPakEb7nZoG8zB3g1sFVro6hQCMSkZBWEmoSCQBORkI57jzNR3JZMIgFqVTdC7JPsIo/p6Ujwfjsn+HZ1hVU1g6JPrgBxqdlYdORf/Hk9AQBQw1gPM3q4AwC+3B8O+TNvUqojwfzeDTGirZNGx0ipK+ZxFt5dexZJ6bloWrs6tn/QBtUMym84MMOOGhh2iEhdsux8LA+8jV/PR0OuEDDQ1cEnb7liQpc6am3A1wVHwe+vgusZrRzWHH2aOpRVyyWy51IsZuy9BgD4qo8HRrV3LnXNh6nZOHT1IQ6GPcTN+DTl8moGuvD2sEW/ZjXhYmWM2CfZyrONUjLzcCs+Dbf+CzS3EtNxOyEd2fnyYl/D0kQfbnamcLMzRQM7M7jZmaKebTUY6+ti98UYzNlXEBZ0JEAdaxPcSSoYNG6iL8W4TnUwvpMLTA31Sv1eX0dOvhzrT93FzyfvICdfAR0JMKKtE6a+7QZz44KeCg+B2psbYmVQJPZdiQMAvNfGEV/18SiXQ0bPS5DlYNC6s4hNyYabrSl2T2ir8b12r8KwowaGHSIqKYVCYM+lWHwbEIGUzILDJD0b2eGLXg1Qq4b6Z5wIIfD14X/hf+Y+9KQSbBnjifauVppuu0RORCThgy2XIFcIfPxWXcz8b6+CJt1OTMeBK3E4GPYQcalF97gABSEoI/dpsY8Z6Oqgnm01uNuZwf2/cONuZwZr05fvnSkMC4XjjU5HJuPbgFu49kAGoGAvim9XV7zf1qncDsEIIRD4byIWHvkXsSkFPwtPFwt81ccDDexfvC0SQmDdqbtYevQWhADauFhgzfstYWFSfkEjJTMPg9edw52kDDhZGuO3Ce1gY2ZYbq9fiGFHDQw7RJpTleYDUdeVmCeYf+iGcgPpalMNX/XxQIdShhOFQuDTnVdw5Ho8TA10sXtCOzR0KN+/RVdjUzF0/Xlk58sxoEVN/DCoaZkeHhFCIDT6CbaHRGP/lYfFruNkaQw3W9P/Qo0Z3O1N4WxporEzfIQQOBqegO+OReDuo4I9PQ7mhpj8dn0MaF4TumW4tyTqUQa++uNfnLr9CABgZ2aIOb0aoHcT+xL/3INuJuKznVeQmSdHbQsjbBzVulzOgEvLycd7G84jPC4N9uaG2DOhHWpbVMyp5Qw7amDYIdKMqnyK78s8Ss/F0qO3sDf0AQDA1EAXk7zqYVR7Z40dPsjJl2PUpgsIuZcCG1MD7Puk/WvtKXod95MzMXDNWTzOzEOnelbYNLp1uR0WedH1k/xHt0ZX91df10oTnsoV+P3yAywPjERCWsEAc1ebapjm7QYfj1ePvVJHRu5T/BQUiU1n7iFfLqAv1cEHnVzg29UVJq8x1uV2Yjo+2HIJMSlZMNGX4sehzeHV0PbVT3xN2XlyjNwUgov3n8DSRB+7J7R74SD88lDS7Xf5H+SjKs/Z2RkSiaTIzdfXF0DBrKMjRoyAnZ0dTExM0KJFC/z++++lqgkUTJfev39/WFtbw8zMDIMHD0ZiYmKZvlcquRddEiFeVvyhiqogX67AL//cRbfvTyqDzrstayFoWhd80KmORgOBoZ4U60e2Qn3bakhKz8Vo/4tIzXrx2USakpyRi1H+F/A4Mw+Napphzfvlc8pwIRcrEzy/o0YqkcDdvvzm6NGV6mBIa0ecnP4WvninAaob6+FOUgY+2haKfj+fxdmo5FcXeQUhBPZfeYBu35/EulN3kS8X6OZug4ApnTGjh/trBR0AqG9rigO+HdC2jgUy8+QY/+slrDkZhbLYj5H3VIGPtoXi4v0nMDXUxZaxnhUadNTBsENqu3jxIuLj45W3wMBAAMCgQYMAACNHjkRERAQOHTqE69evY8CAARg8eDCuXLny2jUzMzPh7e0NiUSCv//+G2fOnEFeXh569+79ymvkUPkIjnikDDqF5ELgfnJWxTT0muJl2TgblYxDV+Pwzo//4JsjN5Ge+xRNaplj3yft8f2gprAxLZuxCeZGetgy1hP25oa4k5SBD7ZcQs4LBuRqQmbuU4zdfBHRj7NQ28IIm0a3LtczaQDA3twIfgMaQ/rf3pOKvH6SoZ4U4zvXwakZXfFpN1cY6UlxNbZg/pgRG0Nw/b/Dl+oKj5Nh0NpzmLL7KpLSc+FkaYyNo1ph0+jWcLEyKXXfFib6+HVcGwxv4wghgKVHb2Hqnqsa/d15Kldg8u4rCL79CEZ6UviPbo1GNc01Vr+s8TAWeBirtCZPnozDhw8jMjISEokE1apVw5o1azBixAjlOpaWlli6dCk++OCD16p57Ngx9OzZE0+ePFH+H8lkMtSoUQPHjh2Dl5dXmbw3ejWFQmDTmXtY+tct5D+fdgBsGNkSbze0q4DO1PfsYbhCFib6mOHjhsGtapfbxH+3E9Px7pqzSMt5Cu+GtljzfkuNz0SbL1dg/NZLOBnxCDWM9fD7x+1Rx7rivqU/P4C4MkhKz8Hqv+9gx4UY5ZxBvZrY4/O365foZ/UkMw/fH4vAzgsxUAjASE+Kid1cMa6jS5kNgv713H0s+ONfyBUCzWpXx/oRLUs9cFihEJj5+zX8FvoA+lIdbBzdCp3qWWuo49LhYSwqF3l5edi2bRvGjh2rPK7dvn177N69GykpKVAoFNi1axdycnLw1ltvvXbN3NxcSCQSGBj8/xkXhoaG0NHRwenTp4vUKPx2/vwhlLI4BAcAq1evhrOzMwwNDdGmTRtcuHBB5fEJEyagbt26MDIygrW1Nfr27Ytbt26V6OdRmSWm5WCU/wV8c+Qm8hUCDexMixyS8N1xBfuvPKiYBtUQ9yQLs35XDToSALvGt8VQT8dyneG4vq0pNoxsBX2pDo79m4gFh25o9LCEEAJz9l3HyYhHMNTTwabRrSs06AAFe3ja1bWsNEEHAGxMDfFV30YImvoW+jevCYmk4CKXby8/hdn7riPhvwkknydXCPx6PhpdfziJ7SEFQed/TewR9HkX+HZ1LdOzvUa0c8bWsZ4wN9JDWGwq+qw689p7pICC35WFR/7Fb6EPINWRYOWw5pUm6KijUocduVyOuXPnwsXFBUZGRqhbty4WLlyo8qEXQmDevHmwt7eHkZERvLy8EBkZWYFdv1kOHDiA1NRUlYvi7dmzB/n5+bC0tISBgQEmTJiA/fv3w9XV9bVrtm3bFiYmJpg5cyaysrKQmZmJadOmQS6XIz4+XuX5G/65i/Z+f+O9DSHosORv7L4Yo3ysLA7B7d69G1OnTsX8+fNx+fJlNG3aFD4+PkhKSlKu07JlS/j7++PmzZsICAiAEALe3t6Qy8vuEEVZOxoeD58Vp/BPZDIM9XTwTb9G+HNSJ5yZ1Q07x7fF8ald8HZDW+Q9VWDK7qtY8tctyIvZ81MZJKbl4KNtoXi+OwHgcWbZj5spTps6llgxtBkkEuDX89H4+WSUxmovC7yN30IfQEcCrH6vBZo71tBYbW3kaGmM5UOa4c/POqG7uw3kCoGdF2LQ5bsT8PvrJlKz8pRfsI6Gx6P3T6cx90A4UrPy4W5nip3j22LVey3K7ZIZHVytcNC3A+pamyAhrWAunD+uFn/G26ssPx4J/zP3AQDfDmyCHo2qxl7aIkQltmjRImFpaSkOHz4s7t27J3777TdRrVo18eOPPyrXWbJkiTA3NxcHDhwQV69eFX369BEuLi4iOzu7xK8jk8kEACGTycribWg1b29v8b///U9l2cSJE4Wnp6c4fvy4CAsLEwsWLBDm5ubi2rVrr11TCCECAgJEnTp1hEQiEVKpVLz//vuiRYsWYsKECeJq7BOxPDBC+CwPFk4zD6vcnGceFsduxAuFQlGk5qRJk0TdunWVj5mYmIitW7eqrGNhYSE2bNjwwn49PT2Fr6+v8r5cLhcODg7Cz8/vhc+5evWqACDu3Lnzyp/Hs5ycnAQKtsEqt08++UTcu3ev2McAiD179hRbLy8vT8yYMUM0atRIGBsbC3t7ezFixAgRFxenXOfEiRMvrGs3cpnotfKUiExML7a+XK4Q3x69qfy/GOt/QaRl56n1nsvaX9fjRbOvAor83jjNPCzqzDoiHqZmVWh//qfvKvv57VJsqev9eu6+st7OkGgNdPjmuXjvsXh3zRnlz7H+F38K5+d+dxrPPyr8T98V+U/lFdanLDtPjNoUouzph4BbQi4v+nfwRTacilI+d8vZe2XXaCmUdPtdqcfs/O9//4OtrS02btyoXDZw4EAYGRlh27ZtEELAwcEBn3/+OaZNmwagYByHra0tNm/ejKFDh5bodThm5/VER0ejTp062LdvH/r27Qug4IwpV1dXhIeHw8PDQ7mul5cXXF1dsXbtWrVrPi85ORk5ciD80VMM7NgIFm0HQNqs+HWf1aimGUa0dUKfpjVhpC9FXl4eHBwcMHXqVMyZMwcA4O3tDX19fWzduhXVq1fHnj17MG7cOFy9erXYPVN5eXkwNjbG3r170a9fP+XyUaNGITU1FQcPHizynMzMTHz55Zc4ePAgbt26BX39kk8E9ujRI5W9QeHh4Xj77bdx4sQJdOrUCY8ePVJZf/369fjuu+8QHx+PatWKHqaQyWR49913MX78eDRt2hRPnjzBpEmTIJfLcenSJeV7TEkpmO4/PE6GeQfDcf3geuREX8XXO07gc2836Ou+fCfxwbA4zNh7DblPFahvWw2/jGwNR8uKmZejUGbuUyw8/C92XYwFAHg4mKGHhx1WHI+EXAjlQNnKcOq83183sS74LqQ6Emwc1Qpvub3eKdkBNxLw8bZQKAQw2aseJnvV13Cnbw4hBE5GPMI3R/5F1H9z9BSSADjyWUc0dKj4AbxyhcDSo7ew/tRdAEAPDzssG9IUxvovH4i+80LB+DUAmO7jBt+uJdszX95Kuv0u32H3amrfvj3Wr1+P27dvo379+rh69SpOnz6NZcuWAQDu3buHhIQElcGp5ubmaNOmDc6dO/fCsJObm4vc3Fzl/bS0tGLXo5fz9/eHjY0NevXqpVyWlVVw5o2OjurGTyqVluisqeJqFop+nImgm0k4EZGE83cfI+1uGLJkKaheuyXM9KXoVM8aLRyrY8nRW0XGXehKJQiPS8PM36/jmyM38W7LWqiRGFrsIbghQ4bA0tISurq6MDY2fukhuOTkZMjlctjaqs5rYWtrW2RMzs8//4wZM2YgMzMTbm5uCAwMVCvoAIC1teqx8iVLlqBu3bro0qULJBIJ7OxUdzHv378fgwcPLjboAAWfl8JDeYVWrVoFT09PxMTEwNHREfr6+rC2scXPJ+5gRVAUnubrIifqAsaM/wiz32lQor77NqsJZ0sTjN96CbcTM9B39Wn8PLxlhV21+WpsKibtuoL7j7MgkQATOtfF1LfrQ19XB++2qlXpBsrO9HFHUlou9l+JwyfbL2PXh23RpFZ1tWqERqfgs51XoBDA0Na1Mal7vbJp9g0hkUjQ1d0G+lIdDN+oOk+QACDLLn4G6PIm1ZFgzjsNUN/WFHP2XcfRGwmIXpOFDSNbvnAep0NXH2LO/oKgM6FLHXzyVt3ybLlMVOqwM2vWLKSlpcHd3R1SqRRyuRyLFi3C8OHDARQMJgVQ7Iam8LHi+Pn54auvviq7xt8ACoUC/v7+GDVqFHR1///XyN3dHa6urpgwYQK+//57WFpa4sCBAwgMDMThw4eV63Xv3h39+/fHxIkTX1gzX67AxfspOHErCUG3knAt6AD0LGtDx9gcuQ9vQfb3BrTrOxLfzhiI1i41YKBbMOjP3FhPeR2cwm/n3g3t8FtoLLadj0FMShb8z9xH4u4VsPVoi6uPJbC2VUBPqoO5c+ciNTUVx48fh5WVFQ4cOIDBgwfjn3/+QePGjUv1Mxs+fDjefvttxMfH4/vvv8fgwYNx5swZGBq+3pkShQO5p06dWuykZ6GhoQgLC8Pq1avVqiuTySCRSFC9enUAQGxKFqbsDsOl6CcAgIb5t/EgOw3zPvd9SZWi+nZqhujoaABANID28wuWf/LJJ8oez507hy+++AIhISGQSqVo1qwZAgICYGRUfOhYsGBBkc+ym5ubMmimpKRg/vz5OHbsGGJiYmBtbY06rd5CtHNvCH1j2JsbYtngZiqhy97cqNKEnEI6OhIsHdgEyRm5+CcyGWM3X8TvH7eHk2XJTlu+k5SBcVsuIfepAt3dbfBNv0YVevFIbVLHpmCeoGe/YEklEjhbVeyey+e927IWXKyMMeHXUNyMT0O/1Wew9v2WaOWsejX7v28lYuruMAgBDG/jiFk93LXjd6UcDqm9tp07d4patWqJnTt3imvXromtW7cKCwsLsXnzZiGEEGfOnBEAxMOHD1WeN2jQIDF48OAX1s3JyREymUx5i42N5ZgdNQUEBAgAIiIioshjt2/fFgMGDBA2NjbC2NhYNGnSpMg4GCcnJzF//vxia678PVh8si1UNJp3VOUYePW27wpDMwsh1dUTznXqiu+//77YcThCCPEwNUucvZNcZLyFXK4QJyOSxODvDwlIdIT1gC+F08zDwnNRoPhiS6AAIMLDw1We0717dzFhwoRiXyc3N1dIpVKxf/9+leUjR44Uffr0KfY5hc8zNjYWO3bseOE6r7J7924hlUpVxtc86+OPPxYNGjRQq2Z2drZo0aKFeO+994RCoRC/h8YKj//+HzzmHRW/h8aKnj17ip49e6rdb1JSkoiPjxf3Yh6IsT8fEzZDvhEAxIiFm0TeU7k4e/asMDMzE35+fiI8PFzcunVL7N69W+Tk5Lyw5vz584WHh4eIj49X3h49eqR8/Pr162LAgAHi0KFD4tSl66LzpJVCt4aDMK7fXvhuDxWpmZVr/NCrpOfki3d+PCWcZh4WXb79WzxKf/HPplCCLFu09wsSTjMPi76rTovM3Pxy6PTNsutCtKgz64hynNeuC5V3LNSDJ1mi54qC3yHXOUfE7osxysfO3kkW9b/4UzjNPCw+23lZrfE9FaWkY3Yq9Z6d6dOnY9asWcrDUY0bN0Z0dDT8/PwwatQo5S77xMRE2NvbK5+XmJiIZs2avbCugYGByinMpD5vb+8Xngpbr169V56ufe7qTdxLzsTD1Cw8ycov2HtzxwTOsw7jhwvpANIBFFzJ+C03G3Rzt0GnBd4wK+FViV/07VxHR4Iu9a1xIj0UNjbWmDJuCPZejkdiWi42Bd8HACw8fBOfGdiiXV1LSCSSlx6C09fXR8uWLREUFKQcs6NQKBAUFKSy1+p5QggIIVQOp6pr48aN6NmzJxwcil4lOzs7Gzt27MDcuXNLXC8/Px+DBw+GEAJLlq3EZ7vClGdwtHKqgeVDmkGSlYJBAQHYs2eP2v0+ewjul48c0O34dqRUt0dwujVG+1/A7fWT8dlnn2HWrFnK9dzc3F5ZV1dXt8jhu0KNGjXC77//jgNX4vDJgXCkG9aBXffRSDj4PZYPagw9vYq5yvXrqmagC/8xrTHg57O4/zgL4zZfxM4P275w/EV6Tj5G+19EXGrB1cQ3jmr1yrEapL4hrR3Rub51pTv8WZya1Y2w9+N2+HzPVfwVnoAZe6/hSvQTuDuYYcmfN5H7VAGvBrb4flDTcp1uoaxV6t/6rKysl479cHFxgZ2dHYKCgpThJi0tDSEhIfj444/Lu10qoeImbnuWh4MZurkXBJymtapr/ANXeLhszOjRmPVOI0z1boijNxKw9bQZDu63x4FVCxB8Yyyca9rCMf3fVx6Cmzp1KkaNGoVWrVrB09MTK1asQGZmJsaMGQMAuHv3Lnbv3g1vb29YW1vjwYMHWLJkCYyMjPDOO++81nuIjo7G8ePHsW/fvmIf37t3L7KysjBy5MgS1SsMOtHR0fh2014M2RSGh7IcSHUkmNy9Hj5+qy50pTpYuPoHWFpaok+fPq/V97Ovdz34MN4fOR4hBro4dTUKDy5eQM9+76J9+/aIioqCu7s7Fi1ahI4dO760VmRkJBwcHGBoaIh27drBz88Pjo4Fg4pl2fmYeyAch/4LbS2daqCNfU18d9KsygWdQjamhtgy1hPvrjmLqw9k8N1+GRtGtipy0crCqf1vxqfBqpoBtozxhGU1fskrK5Xx8OeLGOvrYvV7LfBjUCR+DIrEzv8G6QNAXWsTrHqvebleMqQ8VOqw07t3byxatAiOjo7w8PDAlStXsGzZMowdOxZAwQCxyZMn45tvvkG9evXg4uKCuXPnwsHBQeXMGKo84mXZmLXvOp7fKdTR1Qq9mtijq5sN7MzLZir+QsePH0dMTIzy90hfVwd9mjqgT1MHHG1+BJM/n447+xYiMS8bodXtYdt7Kk5l14JjnAyNapojKioK9x7E42xUMlysTDBkyBA8evQI8+bNQ0JCApo1a4ajR48qx5IZGhrin3/+wYoVK/DkyRPY2tqic+fOOHv2LGxsXu+smpcN5AYK9vr06dOnyIDm4hQGnduRkRjw5Xp8tPc2hACc/5tbpHAOFiEE/P39MXLkyFIHhcK5lBbN/BRpOtUwZOFWPACw+JuFmPrlQqzu2Rlbt25F9+7dER4ejnr1ih9M26ZNG2zevBlubm6Ij4/HV199hU6dOiE8PBz/PsrD1D1XEZeaDamOBJO618PgRuZo4zkMH374Yan6r2h1rath4+jWeG/DeZyIeIQ5+69j6cAmyrEVCoXA9L1XcebOY5joS7F5TMWf/UaVi46OBEM9a2NlUKTK/FL3kjPxJCuvygS3kqrUp56np6dj7ty52L9/P5KSkuDg4IBhw4Zh3rx5yrNYhBCYP38+1q9fj9TUVHTs2BE///wz6tcv+SmVPPW8/Gw+ew8LDv1bZPnO8W0r7Myc4mTkPsX+K3HYdi4aEYnpyuXNaldHfdtq2Bv6oMKu7K1QKODi4oJhw4ZhyZIlRR6/c+cO6tevjz///BM9evQo8ri7uzv8/PzQv39/5Ofn491338WFS6GoN2Ih7mUUhJg+TR3wzdC2qGH6/xvIoKAgeHl54ebNm3B3dy/Ve/Dx8YG+vj7++OMPAMBfx0/inbe7wqztIFi+NQqzezbAB51c0LRpU/Tq1Qt+fn4lqpuamgonJye8PXY6Qg2bQwjAydIYK4Y0Q93qUrz99tuwsLDAoUOHquyenWcF/puICb9egkIAn3Wvh6lvF/zd8/vzJtadugtdHQk2jW6NzvWr3oy3VPZedMX5yvb3+GVKuv2u1GGnvDDslI9/Ih9h/JZLyHmqOv5FKpHg9KyulfKbhBACl6Kf4Ndz0fgrPF55fZxnSSTA0gFN0LiWORyqG8HMULdMz144duwYfHx8EBERUWyonzNnDrZt24b79+8XOQxc0K8E/v7+GD16NO7du4c6deoU+zonTpxQucTHe++9h+joaJw5c6ZU/Rc3l1JhH//7bBGuGzUFUHD2SPRvi6Cvp4ft27eXqHbUowy0bNUawqERanQZjcGtamFebw+IvGz4+PjA2NgYhw8ffu0z4CqjHSExytOEZ/RwQ3xqDn49X3DW27LBTTGgRa2KbI8qsXhZNjos+bvImWSV9e9xcRh21MCwU/YCbiTg0x1XkCdXoJ5tNUQlZUAhUKkmbnuVR+m5+D4gArsvxb50vWoGunCoboia1Y3g8N+t8N81axjB1tSgyPiK58XLsnEvORMuViYa/6NTWNvcSA/LA2/j+M2Cy1p0dLXCD4ObwraUFw18lQULFmDdunWIjY1VTlsghECtWrUwZswY1O05DgsP/wuFAFK3T8HoIf2w4vtvX1pTCIEdF2Lw9b7LuLNyJOy6jsDGb+eiRyN7pKWlwcfHBwYGBvjzzz9hbKx9h3OWBd7GyiDVy+T0aGSLte+3qqCOqKrYfTGmyFQdVeHvcSGtmFSQtMP+Kw8w7bdrkCsEejayw4qhzZCSmVclzlx4lrWpASa/XQ+/hcYWGVztbmeKR+m5eJyZh4zcp7idmIHbiRnF1tGRAHZmhsrw8/+BqGBZyN3H+OqPf8vkMFlxg8P1pTqY0cMNYzu4lPnZFy+an0kikWD69OmYP38+NjZrhkXda2PS1z8iLSEap9AY4f+Nl3p+cPi0adPQ2csH28OzERx2G7LT26Gnq4uAn2ajUd2CoOPt7Y2srCxs27YNaWlpyklEra2tIZWW3QUZy9PQ1rWKhJ3AG4mIl2VXmc8XVYyqdCZZaTDsUJnadj4acw+GQ4iCwxJLBjSGrlSnSp258Cx7cyP4DWj8wm9C2XlyPJRl42FqwS3uSTbiUnMK7suyEZ+agzy5Ag9lOXgoy1FO1PciCgHM/P06/P68VeogolAIpGbnF1n+y6hW5Tam4/nB4c+aPHkycnJyMGXKFKSkpMCtYSPU+vB7pOhaYNDac1g2uCmioqKQnJysfM7lm3ewcr0/8rPSoGtsjuat22J74G+oV7d2weOXLyMkpGBMwvOzYN+7dw/Ozs5l92bL0f3HWUWWyQVwPzmrSn7OqHxV1b/H6uBhLPAwVllZczIKS48WzGQ7ur0z5v2vodbM2xAvy36tb0IKhUByRi7iUrPx8L8QFPff7WFqNqIfZyIjt3yvhF6ZByPKsvIxcedl/BNZEHCmvl0f77asiduJGThyLR6/hT4AANS3rYYVQ5qjocOb+fnVhrEXRK+DY3bUwLCjWUIIfH8sAqtPRAEAJnZ1xefe9bVjyvEyVtxGS0cCbBvXBtampZsj5VF6Lt7fGFLlNohP5Qos/vMWNp25V+zjo9s7Y1ZPdxjqacchqddV1cdeEL0Ohh01MOxojkIh8NUfN7DlXMHZILN6uuOjLlX/InLlqSw3WlV5g7guOAp+f6leXFVHApyZ1a1Sh7Xy9Lp7HImqKg5QpnL3VK7AzN+v4/fLDyCRAAv7NsL7bZ0quq0qpywHDFblwYiNa5kXWabguBQVb8LYC6LXwbBDGpH7VI7Ju8LwV3gCpDoS/DCoKfo1r1nRbVVZZbnRqqobRBerqnF1aSKqfLTr4hdUIbLz5Bi/NRR/hSdAX6qDNcNbMOiQxhWeCSf9b+xX4WG4qhjciKh8cc8OlUpaTj7Gbb6Ii/efwEhPig0jW6FjPauKbou0VFU+DEdEFYdhh15bSmYeRm4KQXhcGkwNdbF5TGu0dLKo6LZIy1XVw3BEVHEYdui1JKbl4P1fQhCZlAFLE31sHecJD4eiA0iJiIgqGsMOqS3mcRaGbzyP2JRs2Jsb4tdxbeBqU62i2yIiIioWww6pJTIxHe9vDEFiWi6cLI2x/YM2qFWDZ8MQEVHlxbBDJRYeJ8PITReQkpkHN1tT/DrOEzZlfIVsIiKi0mLYoRK5eD8FY/0vIj33KZrWMsfmMZ6oYaJf0W0RERG9EsMOvVLw7UeY8Osl5OQr0MbFAr+MagVTQ72KbouIiKhEGHboheJl2dhzMRY//R2Jpwqgq5s11rzf8o2/4CIREVUtDDtUrN0XYzDr9+sonJm/cU1zrBvRCvq6nHSbiIiqFm65qIh4WTZm7fv/oAMANx7K8Dgzt8J6IiIiel0MO1TEveRMCKG6rPDq0kRERFUNww4Vkf9UUWQZry5NRERVFcMOFXEw7CEAQPLffV5dmoiIqjIOUCYVCbIcHLpaEHY2jm4FIz1dXl2aiIiqNIYdUrH13H08VQh4Olugm7ttRbdDRERUajyMRUpZeU+xPSQGADCuk0sFd0NERKQZDDuk9HvoA8iy8+FkaQyvBtyrQ0RE2oFhhwAAcoXAxtP3AADjOrpAqiN5xTOIiIiqBoYdAgAE3UzE/cdZMDfSw7sta1V0O0RERBrDsEMAgF/+26vzXhtHGOtz3DoREWkPhh3CtQepuHAvBbo6Eoxq51zR7RAREWkUww7hl38K9ur0buoAO3PDCu6GiIhIsxh23nAPU7Nx5Ho8gIKByURERNqGYecNt+XsfcgVAu3qWKJRTfOKboeIiEjjGHbeYBm5T7HjQsEkgh9wEkEiItJSDDtvsN8uxSI95ynqWJmgq5tNRbdDRERUJhh23lByhcCmMwUDk8d2dIEOJxEkIiItxbDzhjp2IwGxKdmoYayHgS04iSAREWkvhp03VOEkgu+3dYKRvrSCuyEiIio7DDtvoMsxTxAa/QT6Uh2MaOdU0e0QERGVKYadN1DhBT/7NHOAjSknESQiIu3GsPOGiU3Jwl+cRJCIiN4gDDtvmM1n70MhgI6uVmhgb1bR7RAREZU5hp03SFpOPnZfjAXASQSJiOjNwbDzBtlzMRYZuU9Rz6YautS3ruh2iIiIygXDzhviqVwB/zP3ARSM1ZFIOIkgERG9GRh23hB/hScgLjUblib66Ne8ZkW3Q0REVG4Ydt4AQgj88s9dAAWTCBrqcRJBIiJ6czDsvAFCo5/g6gMZ9HU5iSAREb15GHbeAL/8UzCJ4IDmNWFVzaCCuyEiIipfDDtaLvpxJgL+TQBQcHVzIiKiNw3DjpbzP3MfQgBd6lujvq1pRbdDRERU7hh2tJgsKx97LnESQSIierMx7GixnRdjkJUnh7udKTq6WlV0O0RERBWCYUdL5csV2MxJBImIiBh2tNWf1+ORkJYDq2oG6NPMoaLbISIiqjAMO1pICIEN/00iOKqdEwx0OYkgERG9uRh2tFDIvRSEx6XBQFcHw9tyEkEiInqzqR12Tpw4URZ9kAYVTiI4sGUtWJjoV3A3REREFUvtsNOjRw/UrVsX33zzDWJjY8uiJyqFu48yEHQrEUDBwGQiIqI3ndphJy4uDhMnTsTevXtRp04d+Pj4YM+ePcjLyyuL/rTSggULIJFIVG7u7u7KxxMSEjBixAjY2dnBxMQELVq0wO+///7SmmvWrEGTJk3QwMkO0csGIWvvLNy+9I/KOhMmTEDdunVhZGQEa2tr9O3bF7du3SqT90hERFRZqB12rKysMGXKFISFhSEkJAT169fHJ598AgcHB3z22We4evVqWfSpdTw8PBAfH6+8nT59WvnYyJEjERERgUOHDuH69esYMGAABg8ejCtXrrywXq1atfDlgoWoOWYF7EetQE9vL/Tt2xc3btxQrtOyZUv4+/vj5s2bCAgIgBAC3t7ekMvlZfpeiYiIKlKpBii3aNECs2fPxsSJE5GRkYFNmzahZcuW6NSpk8pGlorS1dWFnZ2d8mZl9f+T/p09exaffvopPD09UadOHXz55ZeoXr06QkNDX1ivd+/eeFTDAwpTezT1aIDNq5ehWrVqOH/+vHKdDz/8EJ07d4azszNatGihPBR5//79snyrREREFeq1wk5+fj727t2Ld955B05OTggICMCqVauQmJiIO3fuwMnJCYMGDdJIg3FxcXj//fdhaWkJIyMjNG7cGJcuXVI+LoTAvHnzYG9vDyMjI3h5eSEyMlIjr12WIiMj4eDggDp16mD48OGIiYlRPta+fXvs3r0bKSkpUCgU2LVrF3JycvDWW2+9sF7uUzk2n70PABjT3hG7d+9GZmYm2rVrV+z6mZmZ8Pf3h4uLC2rXrq3Jt0ZERFS5CDVNnDhRWFpaCgsLCzFp0iRx/fr1IuvEx8cLiUSibukiUlJShJOTkxg9erQICQkRd+/eFQEBAeLOnTvKdZYsWSLMzc3FgQMHxNWrV0WfPn2Ei4uLyM7OLvHryGQyAUDIZLJS91wSf/75p9izZ4+4evWqOHr0qGjXrp1wdHQUaWlpQgghnjx5Iry9vQUAoaurK8zMzERAQMBLa+69FCvsx64SOvqGQiqVCnNzc3HkyJEi661evVqYmJgIAMLNzU3lZ0lERFSVlHT7rXbY6datm9ixY4fIycl54Tr5+fni5MmT6pYuYubMmaJjx44vfFyhUAg7Ozvx3XffKZelpqYKAwMDsXPnzhK/TnmHnec9efJEmJmZiV9++UUIURAoPT09xfHjx0VYWJhYsGCBMDc3F9euXSv2+QqFQvgsDxaO0/aL+duOi0uXLolZs2YJKysrcePGDZV1U1NTxe3bt0VwcLDo3bu3aNGihVrBkIiIqLIo6fZbIoQQFbln6WUaNmwIHx8fPHjwAMHBwahZsyY++eQTjB8/HgBw9+5d1K1bF1euXEGzZs2Uz+vSpQuaNWuGH3/8sdi6ubm5yM3NVd5PS0tD7dq1IZPJYGZmVqbv6UVat24NLy8vfPDBB3B1dUV4eDg8PDyUj3t5ecHV1RVr164t8twzd5Ix/JcQGOlJcW52N1Q31lc+p27duli3bl2xr5mXl4caNWrgl19+wbBhw8rmjREREZWRtLQ0mJubv3L7rfaYHT8/P2zatKnI8k2bNmHp0qXqlnupu3fvYs2aNahXrx4CAgLw8ccf47PPPsOWLVsAFJyiDQC2trYqz7O1tVU+Vhw/Pz+Ym5srbxU9ZiUjIwNRUVGwt7dHVlYWAEBHR/W/RiqVQqFQFPv8X/67NMSgVrWUQQcAFAqFSqh7nijYs/fSdYiIiKo6tcPOunXrVOaEKeTh4VHsXofSUCgUaNGiBRYvXozmzZvjww8/xPjx40v9OrNnz4ZMJlPeyntyxGnTpiE4OBj379/H2bNn0b9/f0ilUgwbNgzu7u5wdXXFhAkTcOHCBURFReGHH35AYGAg+vXrp6zRvXt3rFq1CneS0nEi4hFSgzfDQxKH+/fv4/r165g9ezZOnjyJ4cOHAygIjn5+fggNDUVMTAzOnj2LQYMGwcjICO+88065vn8iIqLypKvuExISEmBvb19kubW1NeLj4zXSVCF7e3s0bNhQZVmDBg2UE+zZ2dkBABITE1V6SkxMVDms9TwDAwMYGBhotFd1PHjwAMOGDcPjx49hbW2Njh074vz587C2tgYA/Pnnn5g1axZ69+6NjIwMuLq6YsuWLSqhJCoqCsnJydh4+j4AwFY/DzM/m4D4+HiYm5ujSZMmCAgIwNtvvw0AMDQ0xD///IMVK1bgyZMnsLW1RefOnXH27FnY2NiU+8+AiIiovKgddmrXro0zZ87AxUX1UgRnzpyBg4ODxhoDgA4dOiAiIkJl2e3bt+HkVHBxSxcXF9jZ2SEoKEgZbtLS0hASEoKPP/5Yo71o0q5du176eL169V45Y/L9+/fx70MZ+q46AwD4dbM/PF0sXri+g4MD/vzzT/WbJSIiquLUDjvjx4/H5MmTkZ+fj27dugEAgoKCMGPGDHz++ecabW7KlClo3749Fi9ejMGDB+PChQtYv3491q9fDwCQSCSYPHkyvvnmG9SrVw8uLi6YO3cuHBwcVA75aKPdF2Mw6/frKBxdfvdRxkvDDhER0ZtK7bOxhBCYNWsWVq5cqbwelqGhIWbOnIl58+ZpvMHDhw9j9uzZiIyMhIuLC6ZOnao8G6uwn/nz52P9+vVITU1Fx44d8fPPP6N+/folfo2SjuauLOJl2eiw5G8onvmfk0okOD2rK+zNjSquMSIionJU0u33a596npGRgZs3b8LIyAj16tWr0DEwpVXVws7ZqGS8tyGkyPKd49uiXV3LCuiIiIio/JV0+632YaxC1apVQ+vWrV/36VQKEkiKLJNKJHC2Mq6AboiIiCq31wo7ly5dwp49exATE6M8lFVo3759GmmMiieEwKoTqtf+kkokWDygEQ9hERERFUPtsLNr1y6MHDkSPj4+OHbsGLy9vXH79m0kJiaif//+ZdEjPePwtXicufMY+ro62P6BJ57KAWcrYwYdIiKiF1A77CxevBjLly+Hr68vTE1N8eOPP8LFxQUTJkwodv4d0pyM3Kf45si/AIBP3qqL1s4cn0NERPQqas+gHBUVhV69egEA9PX1kZmZCYlEgilTpihPCaey8ePx20hMy4WTpTE+6lK3otshIiKqEtQOOzVq1EB6ejoAoGbNmggPDwcApKamKq/rRJoXkZCOTWfuAwAW9PGAoZ60YhsiIiKqItQ+jNW5c2cEBgaicePGGDRoECZNmoS///4bgYGB6N69e1n0+MYTQmDuwXDIFQI+Hrbo6sbLOxAREZWU2mFn1apVyMnJAQB88cUX0NPTw9mzZzFw4EB8+eWXGm+QgP1X4nDhXgqM9KSY19ujotshIiKqUtQKO0+fPsXhw4fh4+MDANDR0cGsWbPKpDEqIMvOx+I/bwIAPu3uiprVedYVERGROtQas6Orq4uPPvpIuWeHyt6yYxFIzshDXWsTfNCxTkW3Q0REVOWoPUDZ09MTYWFhZdAKPS88ToZfz0cDABb2bQR9XbX/u4iIiN54ao/Z+eSTTzB16lTExsaiZcuWMDExUXm8SZMmGmvuTaZQCHx5IBwKAfRu6oD2rlYV3RIREVGVpPaFQHV0iu5dkEgkEEJAIpFALpdrrLnyUhkvBLrrQgxm7buOaga6CPq8C2zNDCu6JSIiokqlzC4Eeu/evVI1Rq/2JDMPS4/eAgBM9qrHoENERFQKaocdJyensuiDnvFtwC08ycqHu50pRrd3ruh2iIiIqjS1w87WrVtf+vjIkSNfuxkCrsQ8wa6LsQCAhf0aQVfKQclERESloXbYmTRpksr9/Px8ZGVlQV9fH8bGxgw7pSBXFMyULAQwsEUttHa2qOiWiIiIqjy1dxs8efJE5ZaRkYGIiAh07NgRO3fuLIse3xjbQ6IRHpcGU0NdzH7HvaLbISIi0goaOUZSr149LFmypMheHyq5R+m5+C4gAgAw3ccNVtUMKrgjIiIi7aCxASG6urp4+PChpsq9cfz+uon0nKdoVNMMw9twEDgREZGmqD1m59ChQyr3hRCIj4/HqlWr0KFDB4019ia5cC8F+y7HQSIpmClZqiOp6JaIiIi0htphp1+/fir3JRIJrK2t0a1bN/zwww+a6uuNkS9XYO6BcADA0Na10dyxRgV3REREpF3UDjsKhaIs+nhjbTl7HxGJ6ahhrIcZPhyUTEREpGmcxKUCJchysDzwNgBgZg931DDRr+COiIiItI/aYWfgwIFYunRpkeXffvstBg0apJGm3hSL/ryJzDw5mjtWx+BWtSu6HSIiIq2kdtg5deoU3nnnnSLLe/bsiVOnTmmkqTfBmTvJ+OPqQ+j8NyhZh4OSiYiIyoTaYScjIwP6+kUPt+jp6SEtLU0jTWm7vKcKzDtYMCh5RFsnNKppXsEdERERaS+1w07jxo2xe/fuIst37dqFhg0baqQpbffL6buIepQJq2oGmOrtVtHtEBERaTW1z8aaO3cuBgwYgKioKHTr1g0AEBQUhJ07d+K3337TeIPaJi41Gz8F3QEAzHnHHeZGehXcERERkXZTO+z07t0bBw4cwOLFi7F3714YGRmhSZMmOH78OLp06VIWPWqVr/+4gex8OTxdLNC/ec2KboeIiEjrqR12AKBXr17o1auXpnvReicikhBwIxFSHQkW9m0EiYSDkomIiMqa2mN2Ll68iJCQkCLLQ0JCcOnSJY00pY1y8uVYcOgGAGBsB2e42ZlWcEdERERvBrXDjq+vL2JjY4ssj4uLg6+vr0aa0kZrg6MQ/TgLtmYGmORVv6LbISIiemOoHXb+/fdftGjRosjy5s2b499//9VIU9om+nEmfj4ZBQCY+7+GqGbwWkcPiYiI6DWoHXYMDAyQmJhYZHl8fDx0dbkRf54QAgsO3UDeUwU6ulqhV2P7im6JiIjojaJ22PH29sbs2bMhk8mUy1JTUzFnzhy8/fbbGm2uqouXZeOnv+/gRMQj6Ekl+KqvBwclExERlTO1d8V8//336Ny5M5ycnNC8eXMAQFhYGGxtbfHrr79qvMGqavfFGMzedx0KUXC/o6sV6lpXq9imiIiI3kBqh52aNWvi2rVr2L59O65evQojIyOMGTMGw4YNg54eJ8gDCvboPBt0ACD49iPEy7Jhb25UcY0RERG9gV5rkI2JiQk+/PBDTfeiNe4lZ6oEHQBQCOB+chbDDhERUTl77RHF//77L2JiYpCXl6eyvE+fPqVuqqpzsTKBjgQqgUcqkcDZyrjimiIiInpDqR127t69i/79++P69euQSCQQomCLXjjwVi6Xa7bDKsje3Ah+Axpjzr5wyIWAVCLB4gGNuFeHiIioAqgddiZNmgQXFxcEBQXBxcUFFy5cwOPHj/H555/j+++/L4seq6QhrR3Rub417idnwdnKmEGHiIiogqgdds6dO4e///4bVlZW0NHRgY6ODjp27Ag/Pz989tlnuHLlSln0WSXZmxsx5BAREVUwtefZkcvlMDUtuK6TlZUVHj58CABwcnJCRESEZrsjIiIiKiW19+w0atQIV69ehYuLC9q0aYNvv/0W+vr6WL9+PerUqVMWPRIRERG9NrXDzpdffonMzEwAwNdff43//e9/6NSpEywtLbF7926NN0hERERUGhJReDpVKaSkpKBGjRpV9lIIaWlpMDc3h0wmg5mZWUW3Q0RERCVQ0u23Rq7caWFhoYkyRERERBqn9gBlIiIioqqEYYeIiIi0GsMOERERaTW1w86pU6fw9OnTIsufPn2KU6dOaaQpIiIiIk1RO+x07doVKSkpRZbLZDJ07dpVI00RERERaYraYUcIUewp5o8fP4aJiYlGmiIiIiLSlBKfej5gwAAABVc3Hz16NAwMDJSPyeVyXLt2De3bt9d8h0RERESlUOKwY25uDqBgz46pqSmMjP7/Apf6+vpo27Ytxo8fr/kOiYiIiEqhxGHH398fAODs7Ixp06bxkBURERFVCWqP2ZkxY4bKmJ3o6GisWLECx44d02hjRERERJqgdtjp27cvtm7dCgBITU2Fp6cnfvjhB/Tt2xdr1qzReINEREREpaF22Ll8+TI6deoEANi7dy/s7OwQHR2NrVu3YuXKlRpvkIiIiKg01A47WVlZMDU1BQAcO3YMAwYMgI6ODtq2bYvo6GiNN0hERERUGmqHHVdXVxw4cACxsbEICAiAt7c3ACApKemll1cnIiIiqghqh5158+Zh2rRpcHZ2hqenJ9q1awegYC9P8+bNNd4gERERUWmoHXbeffddxMTE4NKlSwgICFAu7969O5YvX67R5p63ZMkSSCQSTJ48WbksJycHvr6+sLS0RLVq1TBw4EAkJiaWaR9ERERUdbzWVc/t7OxgamqKwMBAZGdnAwBat24Nd3d3jTb3rIsXL2LdunVo0qSJyvIpU6bgjz/+wG+//Ybg4GA8fPhQOdszERERkdph5/Hjx+jevTvq16+Pd955B/Hx8QCAcePG4fPPP9d4gwCQkZGB4cOHY8OGDahRo4ZyuUwmw8aNG7Fs2TJ069YNLVu2hL+/P86ePYvz58+XSS9ERERUtagddqZMmQI9PT3ExMTA2NhYuXzIkCE4evSoRpsr5Ovri169esHLy0tleWhoKPLz81WWu7u7w9HREefOnXthvdzcXKSlpanciIiISDuV+HIRhY4dO4aAgADUqlVLZXm9evXK5NTzXbt24fLly7h48WKRxxISEqCvr4/q1aurLLe1tUVCQsILa/r5+eGrr77SdKtERERUCam9ZyczM1Nlj06hlJQUlSuha0JsbCwmTZqE7du3w9DQUGN1Z8+eDZlMprzFxsZqrDYRERFVLmqHnU6dOikvFwEAEokECoUC3377Lbp27arR5kJDQ5GUlIQWLVpAV1cXurq6CA4OxsqVK6GrqwtbW1vk5eUhNTVV5XmJiYmws7N7YV0DAwOYmZmp3IiIiEg7qX0Y69tvv0X37t1x6dIl5OXlYcaMGbhx4wZSUlJw5swZjTbXvXt3XL9+XWXZmDFj4O7ujpkzZ6J27drQ09NDUFAQBg4cCACIiIhATEyMcv4fIiIierOpHXYaNWqE27dvY9WqVTA1NUVGRgYGDBgAX19f2Nvba7Q5U1NTNGrUSGWZiYkJLC0tlcvHjRuHqVOnwsLCAmZmZvj000/Rrl07tG3bVqO9EBERUdWkdtiJiYlB7dq18cUXXxT7mKOjo0YaK6nly5dDR0cHAwcORG5uLnx8fPDzzz+Xaw9ERERUeUmEEEKdJ0ilUsTHx8PGxkZl+ePHj2FjYwO5XK7RBstDWloazM3NIZPJOH6HiIioiijp9lvtAcpCCEgkkiLLMzIyNHrGFBEREZEmlPgw1tSpUwEUnH01d+5cldPP5XI5QkJC0KxZM403SERERFQaJQ47V65cAVCwZ+f69evQ19dXPqavr4+mTZti2rRpmu+QiIiIqBRKHHZOnDgBoODU7x9//JFjW4iIiKhKUPtsLH9//7Log4iIiKhMqD1AmYiIiKgqYdghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIirVapw46fnx9at24NU1NT2NjYoF+/foiIiFBZJycnB76+vrC0tES1atUwcOBAJCYmVlDHREREVNlU6rATHBwMX19fnD9/HoGBgcjPz4e3tzcyMzOV60yZMgV//PEHfvvtNwQHB+Phw4cYMGBABXZNRERElYlECCEquomSevToEWxsbBAcHIzOnTtDJpPB2toaO3bswLvvvgsAuHXrFho0aIBz586hbdu2JaqblpYGc3NzyGQymJmZleVbICIiIg0p6fa7Uu/ZeZ5MJgMAWFhYAABCQ0ORn58PLy8v5Tru7u5wdHTEuXPnXlgnNzcXaWlpKjciIiLSTlUm7CgUCkyePBkdOnRAo0aNAAAJCQnQ19dH9erVVda1tbVFQkLCC2v5+fnB3Nxceatdu3ZZtk5EREQVqMqEHV9fX4SHh2PXrl2lrjV79mzIZDLlLTY2VgMdEhERUWWkW9ENlMTEiRNx+PBhnDp1CrVq1VIut7OzQ15eHlJTU1X27iQmJsLOzu6F9QwMDGBgYFCWLRMREVElUan37AghMHHiROzfvx9///03XFxcVB5v2bIl9PT0EBQUpFwWERGBmJgYtGvXrrzbJSIiokqoUu/Z8fX1xY4dO3Dw4EGYmpoqx+GYm5vDyMgI5ubmGDduHKZOnQoLCwuYmZnh008/Rbt27Up8JhYRERFpt0p96rlEIil2ub+/P0aPHg2gYFLBzz//HDt37kRubi58fHzw888/v/Qw1vN46jkREVHVU9Ltd6UOO+WFYYeIiKjq0cp5doiIiIjUxbBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIq2lN2Fm9ejWcnZ1haGiINm3a4MKFCxXdEhEREVUCWhF2du/ejalTp2L+/Pm4fPkymjZtCh8fHyQlJVV0a0RERFTBtCLsLFu2DOPHj8eYMWPQsGFDrF27FsbGxti0aVNFt0ZEREQVTLeiGyitvLw8hIaGYvbs2cplOjo68PLywrlz54p9Tm5uLnJzc5X3ZTIZACAtLa1smyUiIiKNKdxuCyFeul6VDzvJycmQy+WwtbVVWW5ra4tbt24V+xw/Pz989dVXRZbXrl27THokIiKispOeng5zc/MXPl7lw87rmD17NqZOnaq8r1AokJKSAktLS0gkEo29TlpaGmrXro3Y2FiYmZlprK421K/KvVf1+lW596pevyr3Xtb1q3LvVb1+Ve5dCIH09HQ4ODi8dL0qH3asrKwglUqRmJiosjwxMRF2dnbFPsfAwAAGBgYqy6pXr15WLcLMzKxMfoG0oX5V7r2q16/KvVf1+lW597KuX5V7r+r1q2rvL9ujU6jKD1DW19dHy5YtERQUpFymUCgQFBSEdu3aVWBnREREVBlU+T07ADB16lSMGjUKrVq1gqenJ1asWIHMzEyMGTOmolsjIiKiCqYVYWfIkCF49OgR5s2bh4SEBDRr1gxHjx4tMmi5vBkYGGD+/PlFDpmxftXuvarXr8q9V/X6Vbn3sq5flXuv6vWrcu8lJRGvOl+LiIiIqAqr8mN2iIiIiF6GYYeIiIi0GsMOERERaTWGHSIiItJqDDtlaPXq1XB2doahoSHatGmDCxcuaKTuqVOn0Lt3bzg4OEAikeDAgQMaqQsUXEqjdevWMDU1hY2NDfr164eIiAiN1V+zZg2aNGminFyqXbt2+OuvvzRW/1lLliyBRCLB5MmTNVZzwYIFkEgkKjd3d3eN1Y+Li8P7778PS0tLGBkZoXHjxrh06ZJGajs7OxfpXSKRwNfXVyP15XI55s6dCxcXFxgZGaFu3bpYuHDhK69ZU1Lp6emYPHkynJycYGRkhPbt2+PixYuvVetVnyEhBObNmwd7e3sYGRnBy8sLkZGRGqu/b98+eHt7K2dtDwsL01j/+fn5mDlzJho3bgwTExM4ODhg5MiRePjwoUZ6X7BgAdzd3WFiYoIaNWrAy8sLISEhGun9eR999BEkEglWrFihsfqjR48u8hno0aOHRvu/efMm+vTpA3Nzc5iYmKB169aIiYkpde3iPr8SiQTfffedRnrPyMjAxIkTUatWLRgZGSkvrF1Sr6qfmJiI0aNHw8HBAcbGxujRo4dan6vSYNgpI7t378bUqVMxf/58XL58GU2bNoWPjw+SkpJKXTszMxNNmzbF6tWrNdCpquDgYPj6+uL8+fMIDAxEfn4+vL29kZmZqZH6tWrVwpIlSxAaGopLly6hW7du6Nu3L27cuKGR+oUuXryIdevWoUmTJhqtCwAeHh6Ij49X3k6fPq2Ruk+ePEGHDh2gp6eHv/76C//++y9++OEH1KhRQyP1L168qNJ3YGAgAGDQoEEaqb906VKsWbMGq1atws2bN7F06VJ8++23+OmnnzRS/4MPPkBgYCB+/fVXXL9+Hd7e3vDy8kJcXJzatV71Gfr222+xcuVKrF27FiEhITAxMYGPjw9ycnI0Uj8zMxMdO3bE0qVL1e79VfWzsrJw+fJlzJ07F5cvX8a+ffsQERGBPn36aKT3+vXrY9WqVbh+/TpOnz4NZ2dneHt749GjRxqpX2j//v04f/78Ky8D8Dr1e/ToofJZ2Llzp8bqR0VFoWPHjnB3d8fJkydx7do1zJ07F4aGhqWu/WzP8fHx2LRpEyQSCQYOHKiR3qdOnYqjR49i27ZtuHnzJiZPnoyJEyfi0KFDpa4vhEC/fv1w9+5dHDx4EFeuXIGTkxO8vLw0tn15KUFlwtPTU/j6+irvy+Vy4eDgIPz8/DT6OgDE/v37NVrzWUlJSQKACA4OLrPXqFGjhvjll180Vi89PV3Uq1dPBAYGii5duohJkyZprPb8+fNF06ZNNVbvWTNnzhQdO3Ysk9rFmTRpkqhbt65QKBQaqderVy8xduxYlWUDBgwQw4cPL3XtrKwsIZVKxeHDh1WWt2jRQnzxxRelqv38Z0ihUAg7Ozvx3XffKZelpqYKAwMDsXPnzlLXf9a9e/cEAHHlyhW165akfqELFy4IACI6OlrjtWUymQAgjh8/rlbtl9V/8OCBqFmzpggPDxdOTk5i+fLlatd+Uf1Ro0aJvn37vla9ktQfMmSIeP/998uk9vP69u0runXrprH6Hh4e4uuvv1ZZ9rqfsefrR0RECAAiPDxcuUwulwtra2uxYcMGteuri3t2ykBeXh5CQ0Ph5eWlXKajowMvLy+cO3euAjtTn0wmAwBYWFhovLZcLseuXbuQmZmp0Ut7+Pr6olevXio/f02KjIyEg4MD6tSpg+HDh5do93RJHDp0CK1atcKgQYNgY2OD5s2bY8OGDRqp/by8vDxs27YNY8eO1djFb9u3b4+goCDcvn0bAHD16lWcPn0aPXv2LHXtp0+fQi6XF/l2bGRkpLE9a4Xu3buHhIQEld8fc3NztGnTpsp9fgvJZDJIJBKNXwMwLy8P69evh7m5OZo2baqRmgqFAiNGjMD06dPh4eGhkZrPO3nyJGxsbODm5oaPP/4Yjx8/1khdhUKBI0eOoH79+vDx8YGNjQ3atGmj0aEGhRITE3HkyBGMGzdOYzXbt2+PQ4cOIS4uDkIInDhxArdv34a3t3epa+fm5gKAymdYR0cHBgYGGv8MF4dhpwwkJydDLpcXmcHZ1tYWCQkJFdSV+hQKBSZPnowOHTqgUaNGGqt7/fp1VKtWDQYGBvjoo4+wf/9+NGzYUCO1d+3ahcuXL8PPz08j9Z7Xpk0bbN68GUePHsWaNWtw7949dOrUCenp6aWufffuXaxZswb16tVDQEAAPv74Y3z22WfYsmWLBjpXdeDAAaSmpmL06NEaqzlr1iwMHToU7u7u0NPTQ/PmzTF58mQMHz681LVNTU3Rrl07LFy4EA8fPoRcLse2bdtw7tw5xMfHa6D7/1f4Ga3qn99COTk5mDlzJoYNG6axizAePnwY1apVg6GhIZYvX47AwEBYWVlppPbSpUuhq6uLzz77TCP1ntejRw9s3boVQUFBWLp0KYKDg9GzZ0/I5fJS105KSkJGRgaWLFmCHj164NixY+jfvz8GDBiA4OBgDXT//7Zs2QJTU1MMGDBAYzV/+uknNGzYELVq1YK+vj569OiB1atXo3PnzqWu7e7uDkdHR8yePRtPnjxBXl4eli5digcPHmj8M1wcrbhcBJUNX19fhIeHazx1u7m5ISwsDDKZDHv37sWoUaMQHBxc6sATGxuLSZMmITAwsETHx1/Hs3spmjRpgjZt2sDJyQl79uwp9TcshUKBVq1aYfHixQCA5s2bIzw8HGvXrsWoUaNKVft5GzduRM+ePdUeD/Eye/bswfbt27Fjxw54eHggLCwMkydPhoODg0b6//XXXzF27FjUrFkTUqkULVq0wLBhwxAaGqqB7rVTfn4+Bg8eDCEE1qxZo7G6Xbt2RVhYGJKTk7FhwwYMHjwYISEhsLGxKVXd0NBQ/Pjjj7h8+bLG9jg+b+jQocp/N27cGE2aNEHdunVx8uRJdO/evVS1FQoFAKBv376YMmUKAKBZs2Y4e/Ys1q5diy5dupSq/rM2bdqE4cOHa/Rv3U8//YTz58/j0KFDcHJywqlTp+Dr6wsHB4dS7ynX09PDvn37MG7cOFhYWEAqlcLLyws9e/bU2EkML8M9O2XAysoKUqkUiYmJKssTExNhZ2dXQV2pZ+LEiTh8+DBOnDiBWrVqabS2vr4+XF1d0bJlS/j5+aFp06b48ccfS103NDQUSUlJaNGiBXR1daGrq4vg4GCsXLkSurq6Gvnm9rzq1aujfv36uHPnTqlr2dvbFwl8DRo00NhhskLR0dE4fvw4PvjgA43WnT59unLvTuPGjTFixAhMmTJFY3vZ6tati+DgYGRkZCA2NhYXLlxAfn4+6tSpo5H6hQo/o1X58wv8f9CJjo5GYGCgxvbqAICJiQlcXV3Rtm1bbNy4Ebq6uti4cWOp6/7zzz9ISkqCo6Oj8jMcHR2Nzz//HM7OzqVvvBh16tSBlZWVRj7DVlZW0NXVLfPP8T///IOIiAiNfoazs7MxZ84cLFu2DL1790aTJk0wceJEDBkyBN9//71GXqNly5YICwtDamoq4uPjcfToUTx+/Fjjn+HiMOyUAX19fbRs2RJBQUHKZQqFAkFBQRodm1IWhBCYOHEi9u/fj7///hsuLi5l/poKhUJ5PLc0unfvjuvXryMsLEx5a9WqFYYPH46wsDBIpVINdKsqIyMDUVFRsLe3L3WtDh06FDnN//bt23Bycip17Wf5+/vDxsYGvXr10mjdrKws6Oio/kmRSqXKb7uaYmJiAnt7ezx58gQBAQHo27evRuu7uLjAzs5O5fOblpaGkJCQSv/5LVQYdCIjI3H8+HFYWlqW6etp6jM8YsQIXLt2TeUz7ODggOnTpyMgIEADnRb14MEDPH78WCOfYX19fbRu3brMP8cbN25Ey5YtNTZOCij4ncnPzy+Xz7C5uTmsra0RGRmJS5cuafwzXBwexiojU6dOxahRo9CqVSt4enpixYoVyMzMxJgxY0pdOyMjQ+VbyL179xAWFgYLCws4OjqWqravry927NiBgwcPwtTUVDlGwdzcHEZGRqWqDQCzZ89Gz5494ejoiPT0dOzYsQMnT57UyB8yU1PTImOLTExMYGlpqbExR9OmTUPv3r3h5OSEhw8fYv78+ZBKpRg2bFipa0+ZMgXt27fH4sWLMXjwYFy4cAHr16/H+vXrNdB5AYVCAX9/f4waNQq6upr9+Pfu3RuLFi2Co6MjPDw8cOXKFSxbtgxjx47VSP2AgAAIIeDm5oY7d+5g+vTpcHd3f63P1Ks+Q5MnT8Y333yDevXqwcXFBXPnzoWDgwP69eunkfopKSmIiYlRzn1TuHG0s7Mr0d6jl9W3t7fHu+++i8uXL+Pw4cOQy+XKz7GFhQX09fVfu7alpSUWLVqEPn36wN7eHsnJyVi9ejXi4uJKPIXBq342zwczPT092NnZwc3NrdT1LSws8NVXX2HgwIGws7NDVFQUZsyYAVdXV/j4+Gik/+nTp2PIkCHo3LkzunbtiqNHj+KPP/7AyZMnS10bKAjev/32G3744YcS9atO/S5dumD69OkwMjKCk5MTgoODsXXrVixbtkwj9X/77TdYW1vD0dER169fx6RJk9CvXz+NDIB+pTI/3+sN9tNPPwlHR0ehr68vPD09xfnz5zVS98SJEwJAkduoUaNKXbu4ugCEv79/qWsLIcTYsWOFk5OT0NfXF9bW1qJ79+7i2LFjGqldHE2fej5kyBBhb28v9PX1Rc2aNcWQIUPEnTt3NFb/jz/+EI0aNRIGBgbC3d1drF+/XmO1hRAiICBAABAREREarSuEEGlpaWLSpEnC0dFRGBoaijp16ogvvvhC5ObmaqT+7t27RZ06dYS+vr6ws7MTvr6+IjU19bVqveozpFAoxNy5c4Wtra0wMDAQ3bt3V+tn9qr6/v7+xT4+f/78UtcvPJ29uNuJEydKVTs7O1v0799fODg4CH19fWFvby/69OkjLly4oLGfzfPUPfX8ZfWzsrKEt7e3sLa2Fnp6esLJyUmMHz9eJCQkaLT/jRs3CldXV2FoaCiaNm0qDhw4oLHa69atE0ZGRq/1u/+q+vHx8WL06NHCwcFBGBoaCjc3N/HDDz+UeHqKV9X/8ccfRa1atYSenp5wdHQUX375pcb+PryKRIhyGBlEREREVEE4ZoeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0T0nJMnT0IikSA1NbWiWyEiDWDYISIiIq3GsENERERajWGHiCodhUIBPz8/uLi4wMjICE2bNsXevXsB/P8hpiNHjqBJkyYwNDRE27ZtER4erlLj999/h4eHBwwMDODs7Fzkwom5ubmYOXMmateuDQMDA7i6umLjxo0q64SGhqJVq1YwNjZG+/bti1zNmoiqBoYdIqp0/Pz8sHXrVqxduxY3btzAlClT8P777yM4OFi5zvTp0/HDDz/g4sWLsLa2Ru/evZGfnw+gIKQMHjwYQ4cOxfXr17FgwQLMnTsXmzdvVj5/5MiR2LlzJ1auXImbN29i3bp1qFatmkofX3zxBX744QdcunQJurq6GruCOxGVL14IlIgqldzcXFhYWOD48eNo166dcvkHH3yArKwsfPjhh+jatSt27dqFIUOGAABSUlJQq1YtbN68GYMHD8bw4cPx6NEjHDt2TPn8GTNm4MiRI7hx4wZu374NNzc3BAYGwsvLq0gPJ0+eRNeuXXH8+HF0794dAPDnn3+iV69eyM7OhqGhYRn/FIhIk7hnh4gqlTt37iArKwtvv/02qlWrprxt3boVUVFRyvWeDUIWFhZwc3PDzZs3AQA3b95Ehw4dVOp26NABkZGRkMvlCAsLg1QqRZcuXV7aS5MmTZT/tre3BwAkJSWV+j0SUfnSregGiIielZGRAQA4cuQIatasqfKYgYGBSuB5XUZGRiVaT09PT/lviUQCoGA8ERFVLdyzQ0SVSsOGDWFgYICYmBi4urqq3GrXrq1c7/z588p/P3nyBLdv30aDBg0AAA0aNMCZM2dU6p45cwb169eHVCpF48aNoVAoVMYAEZH24p4dIqpUTE1NMW3aNEyZMgUKhQIdO3aETCbDmTNnYGZmBicnJwDA119/DUtLS9ja2uKLL76AlZUV+vXrBwD4/PPP0bp1ayxcuBBDhgzBuXPnsGrVKvz8888AAGdnZ4waNQpjx47FypUr0bRpU0RHRyMpKQmDBw+uqLdORGWEYYeIKp2FCxfC2toafn5+uHv3LqpXr44WLVpgzpw5ysNIS5YswaRJkxAZGYlmzZrhjz/+gL6+PgCgRYsW2LNnD+bNm4eFCxfC3t4eX3/9NUaPHq18jTVr1mDOnDn45JNP8PjxYzg6OmLOnDkV8XaJqIzxbCwiqlIKz5R68uQJqlevXtHtEFEVwDE7REREpNUYdoiIiEir8TAWERERaTXu2SEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKt9n8fQDs8MfsJIQAAAABJRU5ErkJggg==", "text/plain": [ "
" ] @@ -778,8 +1125,12 @@ "plt.xlabel('epoch')\n", "plt.ylabel('test accuracy')\n", "plt.ylim(0, 100)\n", + "plt.xticks(np.arange(len(epochs_x)))\n", "for i, txt in enumerate(epochs_acc):\n", - " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom')\n", + " if i%2 == 0:\n", + " pass\n", + " else:\n", + " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", "plt.show()" ] } From 8ba07fbfc58ab9ee6fa4b8b29e33e131ce7558f1 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Sun, 28 Apr 2024 20:54:01 +0200 Subject: [PATCH 054/379] non-seq version has improved accuracy (13% better) over baseline --- .../baseline-SCNN-example_3.ipynb | 440 +++++++++++++++-- .../non-sequential-SCNN-example_3.ipynb | 452 ++++++++++++++++-- 2 files changed, 796 insertions(+), 96 deletions(-) diff --git a/tests/test_nonsequential/baseline-SCNN-example_3.ipynb b/tests/test_nonsequential/baseline-SCNN-example_3.ipynb index 8567adc5..dc53313a 100644 --- a/tests/test_nonsequential/baseline-SCNN-example_3.ipynb +++ b/tests/test_nonsequential/baseline-SCNN-example_3.ipynb @@ -51,7 +51,7 @@ "source": [ "batch_size = 3\n", "num_workers = 1\n", - "epochs = 20\n", + "epochs = 30\n", "lr = 1e-3" ] }, @@ -359,7 +359,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "57e76b2737034b53b317bac992a7d524", + "model_id": "ebf2bea3d0124365abf1f2aa248ceab6", "version_major": 2, "version_minor": 0 }, @@ -373,7 +373,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "784ddd8e4fd942a3ab0926d62e7a2b0e", + "model_id": "faa557ac9bb54c02897848d65fb01cb7", "version_major": 2, "version_minor": 0 }, @@ -394,7 +394,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "a4c4e84fcd94445babc42b7c2e59e815", + "model_id": "5740aa705e294dff96add0963c733568", "version_major": 2, "version_minor": 0 }, @@ -408,7 +408,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "438eb0b917b641518f983effd0f1a0ca", + "model_id": "c3e026123d464e838af0e981152dc61c", "version_major": 2, "version_minor": 0 }, @@ -429,7 +429,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "263fbc8321354a3783da8dbef468b6c2", + "model_id": "2757957b8f2d4a53a3cee8aad91343d9", "version_major": 2, "version_minor": 0 }, @@ -443,7 +443,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "5cb854835e764b1cb2345757c9a38575", + "model_id": "047151a26d8346dbb0579972f5fcb324", "version_major": 2, "version_minor": 0 }, @@ -464,7 +464,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "993c7997c4564c8f82badee623f3a611", + "model_id": "4a0036cab9ee4aeb8f6d3c1b1985070b", "version_major": 2, "version_minor": 0 }, @@ -478,7 +478,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "1fdac09e1284489e96c3d15689f9e70d", + "model_id": "d207154f438f4215bcfd67717e72e838", "version_major": 2, "version_minor": 0 }, @@ -499,7 +499,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "5f2f08d152784f929bb12c77fc14dfe9", + "model_id": "c0dadbe693ce41a4aa06d09d9f521ef5", "version_major": 2, "version_minor": 0 }, @@ -513,7 +513,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "500d1323ed3a4d198b9a4e75fc12f1a7", + "model_id": "b47c85e708354f9399bb016932d0a4c6", "version_major": 2, "version_minor": 0 }, @@ -534,7 +534,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "41ff11b617954927b1bf138ff2321154", + "model_id": "7e23105d5c194961b4858db05d466e41", "version_major": 2, "version_minor": 0 }, @@ -548,7 +548,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "1a7f7ce0ad4049878167a738e4d0aeb1", + "model_id": "6b608aea788a46a590ea1e3440d0f005", "version_major": 2, "version_minor": 0 }, @@ -569,7 +569,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "3e66f687ce9d458582a9c8cb42905ffe", + "model_id": "17f668e20bf447c5b94099647a7cedc7", "version_major": 2, "version_minor": 0 }, @@ -583,7 +583,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "8e688520d88a4398b0d7018b4d97d89d", + "model_id": "534a4922584646c2861428df60daaf20", "version_major": 2, "version_minor": 0 }, @@ -604,7 +604,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "54b3d1da434847d5a4d78264686ecec7", + "model_id": "4764a80723ea40209e9e7158117c1898", "version_major": 2, "version_minor": 0 }, @@ -618,7 +618,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "1aa7e1804583444d8f8780bc88e84f2c", + "model_id": "65300a14e6d94459adeb770b6b268a91", "version_major": 2, "version_minor": 0 }, @@ -639,7 +639,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "0367206cfc414f6e9ce3245d60bb1dc2", + "model_id": "fc6fb5195c144daca12c96c24fd77655", "version_major": 2, "version_minor": 0 }, @@ -653,7 +653,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "162b85a11a0746ffb4ea697ffc514329", + "model_id": "9eeb3dfb97074188a4492e637c7da642", "version_major": 2, "version_minor": 0 }, @@ -674,7 +674,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "04e04a9bc5814118a19989bfa5c581a8", + "model_id": "1976efa33e2c4329b8078862fe018856", "version_major": 2, "version_minor": 0 }, @@ -688,7 +688,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "5ffbd79e43c148daac36e85fc8c557fd", + "model_id": "2ec118840b6a41678ceae4c8a2c3e640", "version_major": 2, "version_minor": 0 }, @@ -709,7 +709,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "d776863c3eaa4230bfce51fdc81c43cf", + "model_id": "3d14dd584ec84a7d8f766f4e7347efea", "version_major": 2, "version_minor": 0 }, @@ -723,7 +723,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "000df9888cca45ed9996dd9c97f568b9", + "model_id": "781187e60a764025bf9a95672440b191", "version_major": 2, "version_minor": 0 }, @@ -744,7 +744,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "bd6d1f79b0e74a44a82a5278225f9c18", + "model_id": "cd9ffb5c23d741b5a9ee1b0f77a3fb04", "version_major": 2, "version_minor": 0 }, @@ -758,7 +758,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "b9bed9c8c41047c5a4abd8db920becfc", + "model_id": "fb5a4f0aae964636a26819e57bd9acfb", "version_major": 2, "version_minor": 0 }, @@ -779,7 +779,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "8a4c0621aab84a6b9029db58a7c4b06f", + "model_id": "2cfd5abcc6e44794996933e0c6b7fe5e", "version_major": 2, "version_minor": 0 }, @@ -793,7 +793,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "5b31b3eeb215416498101201e67de690", + "model_id": "78343c4e0849457d895a3bf80b2c786c", "version_major": 2, "version_minor": 0 }, @@ -814,7 +814,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "eead401e1808442985c916b9008872c4", + "model_id": "41dad080ca4441b580bb8e77077ce5ed", "version_major": 2, "version_minor": 0 }, @@ -828,7 +828,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "264a4117ba5f4657a81fe570d3ab980c", + "model_id": "9012039db9064b91b7c7930ee3302f83", "version_major": 2, "version_minor": 0 }, @@ -849,7 +849,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "c4e113cf4efd4f0c8330a1cd8fc7ea21", + "model_id": "29477e853def44d9986e8f4b6ffe4379", "version_major": 2, "version_minor": 0 }, @@ -863,7 +863,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "0d46cfeba047469596a76b23b37e7211", + "model_id": "a97da5f79020463bac64bdf80db1fd2f", "version_major": 2, "version_minor": 0 }, @@ -884,7 +884,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "a4941f91e2514f17877259ab5e91747d", + "model_id": "b4d30264425840238b94a9d8f09979f6", "version_major": 2, "version_minor": 0 }, @@ -898,7 +898,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "9f300f7b53b64719868503d03aa96035", + "model_id": "ebbe8618017c434f917b93c2e4549098", "version_major": 2, "version_minor": 0 }, @@ -919,7 +919,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "d222b60e09ca4a79a1188276f22b0b9e", + "model_id": "ec03d092029843719fbb3c216a9f0433", "version_major": 2, "version_minor": 0 }, @@ -933,7 +933,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "f519f592de19457b89625adcde920b7d", + "model_id": "c76dde5446bc4cc9a8d4e4d876904725", "version_major": 2, "version_minor": 0 }, @@ -954,7 +954,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "e621fb2a4f8640e4bace1aa81f113b06", + "model_id": "2be56e494d264b52b3241e504488438e", "version_major": 2, "version_minor": 0 }, @@ -968,7 +968,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "1ad59265c65648d790a13b04e5bcf707", + "model_id": "4f165ce5dca94f699e46b4ebaa200100", "version_major": 2, "version_minor": 0 }, @@ -989,7 +989,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "fca9f69ad3b34004828684bea501f067", + "model_id": "4ce0b39beed8472a80af3b06b44be5b7", "version_major": 2, "version_minor": 0 }, @@ -1003,7 +1003,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "6b4914894b424ecf988b9e73db8c347c", + "model_id": "823f2347cedf4431a73e1ba87b6df989", "version_major": 2, "version_minor": 0 }, @@ -1024,7 +1024,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "9cfe42dbd5fa41acb8b3dde3349a4e5a", + "model_id": "31689f1975be4d4dbc8efdc99f19294d", "version_major": 2, "version_minor": 0 }, @@ -1038,7 +1038,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "ef0ddee884b74ae68e5aeae3d0af32e1", + "model_id": "b66a37ac35424593a7b060e637070501", "version_major": 2, "version_minor": 0 }, @@ -1055,6 +1055,356 @@ "text": [ "Epoch 19 accuracy: 70.45454545454545\n" ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "f852bd8d1eed4b1ea2f35bd1e66fa85d", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/359 [00:00" ] @@ -1097,12 +1447,12 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 17, "metadata": {}, "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAG2CAYAAACZEEfAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABjDUlEQVR4nO3dd3zM9x8H8NdlRxbZCTIQYsaIETGKVKiqVauoVTqiRNRs0UXQGrVLCWpWa1V/RgSxI0RIiiSIJMgQSS5L5n1/f6Q5ToacXNbX6/l43KPyve997n2ac6/7fD9DIgiCACIiIiKRUqvqAoiIiIgqEsMOERERiRrDDhEREYkaww4RERGJGsMOERERiRrDDhEREYkaww4RERGJGsMOERERiRrDDhEREYkaww4RERGJWpWGnXPnzqF///6wtraGRCLBoUOHFO4XBAELFiyAlZUVdHV14ebmhoiICIVzkpKSMGrUKBgaGqJ27dqYOHEi0tPTK/FVEBERUXVWpWEnIyMDTk5OWLduXbH3L1u2DKtXr8bGjRsREBAAPT09uLu7IysrS37OqFGj8O+//8LX1xdHjx7FuXPnMHny5Mp6CURERFTNSarLRqASiQQHDx7EwIEDART06lhbW2PGjBn46quvAABSqRQWFhbYtm0bRowYgTt37qBZs2YIDAyEs7MzAOD48eN477338OjRI1hbW1fVyyEiIqJqQqOqCyhJZGQk4uLi4ObmJj9mZGSEjh074vLlyxgxYgQuX76M2rVry4MOALi5uUFNTQ0BAQEYNGhQsW1nZ2cjOztb/rNMJkNSUhJMTEwgkUgq7kURERGRygiCgLS0NFhbW0NNreSLVdU27MTFxQEALCwsFI5bWFjI74uLi4O5ubnC/RoaGjA2NpafUxxvb2989913Kq6YiIiIqkJMTAzq1atX4v3VNuxUpLlz58LLy0v+s1QqhY2NDWJiYmBoaFiFlREREVFZpaamon79+jAwMCj1vGobdiwtLQEA8fHxsLKykh+Pj49H69at5eckJCQoPC4vLw9JSUnyxxdHW1sb2traRY4bGhoy7BAREdUwrxuCUm3X2bG3t4elpSX8/Pzkx1JTUxEQEAAXFxcAgIuLC1JSUnD9+nX5OadPn4ZMJkPHjh0rvWYiIiKqfqq0Zyc9PR337t2T/xwZGYng4GAYGxvDxsYGnp6e+PHHH+Hg4AB7e3vMnz8f1tbW8hlbTZs2RZ8+fTBp0iRs3LgRubm5mDJlCkaMGMGZWERERASgisPOtWvX0KNHD/nPheNoxo4di23btmHWrFnIyMjA5MmTkZKSgi5duuD48ePQ0dGRP2bXrl2YMmUKevXqBTU1NQwZMgSrV6+u9NdCRERE1VO1WWenKqWmpsLIyAhSqZRjdoiIiGqIsn5+V9sxO0RERESqwLBDREREosawQ0RERKLGsENERESixrBDREREosawQ0RERKLGsENERESixrBDREREosawQ0RERKLGsENERESixrBDREREosawQ0RERKLGsENERESixrBDREREosawQ0RERKLGsENERESixrBDREREosawQ0RERKLGsENERESixrBDREREosawQ0RERKLGsENERESixrBDREREosawQ0RERKLGsENERESixrBDREREosawQ0RERKLGsENERESixrBDREREosawQ0RERKLGsENERESixrBDREREosawQ0RERKLGsENERESixrBDREREosawQ0RERKLGsENERESixrBDREREosawQ0RERKLGsENERESixrBDREREosawQ0RERKLGsENERESixrBDREREosawQ0RERKLGsENERESixrBDREREosawQ0RERKLGsENERESixrBDREREosawQ0RERKLGsENERESixrBDREREosawQ0RERKLGsENERESixrBDREREosawQ0RERKLGsENERESixrBDREREosawQ0RERKLGsENERESixrBDREREosawQ0RERKJWrcNOfn4+5s+fD3t7e+jq6qJhw4b44YcfIAiC/BxBELBgwQJYWVlBV1cXbm5uiIiIqMKqiYiIqDqp1mFn6dKl2LBhA9auXYs7d+5g6dKlWLZsGdasWSM/Z9myZVi9ejU2btyIgIAA6Onpwd3dHVlZWVVYOREREVUXEuHlbpJq5v3334eFhQW2bNkiPzZkyBDo6upi586dEAQB1tbWmDFjBr766isAgFQqhYWFBbZt24YRI0aU6XlSU1NhZGQEqVQKQ0PDCnktREREpFpl/fyu1j07nTt3hp+fH8LDwwEAN2/exIULF9C3b18AQGRkJOLi4uDm5iZ/jJGRETp27IjLly+X2G52djZSU1MVbkRERCROGlVdQGnmzJmD1NRUODo6Ql1dHfn5+Vi0aBFGjRoFAIiLiwMAWFhYKDzOwsJCfl9xvL298d1331Vc4URERFRtVOuenT/++AO7du3C7t27ERQUhO3bt+Pnn3/G9u3by9Xu3LlzIZVK5beYmBgVVUxERETVTbXu2Zk5cybmzJkjH3vTsmVLREVFwdvbG2PHjoWlpSUAID4+HlZWVvLHxcfHo3Xr1iW2q62tDW1t7QqtnYiIiKqHat2zk5mZCTU1xRLV1dUhk8kAAPb29rC0tISfn5/8/tTUVAQEBMDFxaVSayUiojcXK32OS/cTESt9XtWlkAhV656d/v37Y9GiRbCxsUHz5s1x48YNrFixAhMmTAAASCQSeHp64scff4SDgwPs7e0xf/58WFtbY+DAgVVbPBERlcm+wGjMPRACmQCoSQDvwS0xvL1NVZdFIlKtw86aNWswf/58fPHFF0hISIC1tTU+/fRTLFiwQH7OrFmzkJGRgcmTJyMlJQVdunTB8ePHoaOjU4WVExFRWcRKn8uDDgDIBGDegVB0a2wGKyPdqi2ORKNar7NTWbjODhFR1dgXGI3Zf4UUOb5nUie4NDSpgoqoJhHFOjtERCRef15/hAWHQ4u9z8yAk0hIdRh2iIioUmXl5mPOX7fw1f6byM4T4GChDzWJ4jlrT0eAFx5IVar1mB0iIhKX6GeZ+HzXdfz7JBUSCeDZqzGm9GyEhLQsPEzMROrzHHyx+wYOBT+BayNTDHWuX9Ulkwgw7BARUaXwvR0Prz+CkZaVB2M9LfwyojW6OpgBAKyMdOUDkr3ezcBPJ8Kw4PC/aGNTG43MDaqybBIBXsYiIhI5Ozs7SCSSIjcPDw8AwKeffoqGDRtCV1cXZmZmGDBgAO7evVvm9j/77DNIJBKsWrVK4Xh4eDgGDBgAU1NT6NTSR//ePfE0LAhtbWrj6Jdd5EHnVZ93b4gujUzxPDcfU3bfQFZu/hu/diKAYYeISPQCAwMRGxsrv/n6+gIAhg4dCgBo164dfHx8cOfOHZw4cQKCIKB3797Iz399yDh48CCuXLkCa2vrIve9//77yMzKQdvPV8Jk9Apomtsj6eAPWPWBPaxrlzytXE1NghXDnWCqr4W7cWn44ejtN3zlRAUYdoiIRM7MzAyWlpby29GjR9GwYUN0794dADB58mR069YNdnZ2aNu2LX788UfExMTg4cOHpbb7+PFjfPnll9i1axc0NTUV7ktMTERERASi6/dGeK4xalvaYOu6lcjNfo7wu68PL+YGOlgxrDUAYFdANP4XEvtGr50IYNghInqr5OTkYOfOnZgwYQIkEkmR+zMyMuDj4wN7e3vUr1/y4GCZTIYxY8Zg5syZaN68+Sv3CfgjJAWaxvXwJPA4GtbWwIHPOyHq4mGYm5ujXbt2Zaq1W2MzfP5OQwDA7L9uISYpU4lXSvQCww4R0Vvk0KFDSElJwbhx4xSOr1+/Hvr6+tDX18exY8fg6+sLLS2tEttZunQpNDQ0MHXqVIXj0sxcTP79GpadCIP58B9RKz0GZ+b1RfP6plixYgWOHz+OOnXqlLler3cbo61NbaRl5eHLPTeQmy9T6vUSAQw7RERlUtog36SkJHz55Zdo0qQJdHV1YWNjg6lTp0IqlZba5rfffgtHR0fo6emhTp06cHNzQ0BAwGufd8mSJW/8OrZs2YK+ffsWGWMzatQo3LhxA/7+/mjcuDGGDRuGrKysYtu4fv06fvnlF2zbtk2hdyhW+hzvrz2PU3cSoKkugWXoLrRrYofz58/j6tWrGDhwIPr374/Y2LJfktJUV8PqkW1gqKOB4JgU/Hwy7M1eOL3VuF0EuF0EEb3e06dPFQbshoaG4t1338WZM2dgamqKhQsXYty4cWjWrBmioqLw2WefoVWrVvjzzz9LbHP37t0wNzdHgwYN8Pz5c6xcuRL79+/HvXv3YGZWMFPJzs4OEydOxKRJk+SPMzAwgJ6entKvISoqCg0aNMCBAwcwYMCAEs/LyclBnTp18Ntvv2HkyJFF7l+1ahW8vLygpvbi+3J+fj4gUYO6gSlcvt6Dsbbp+PSjQUhOTlb4d9XBwQETJ07EnDlzlKr9eGgsPtsZBADYNr493mlirtTjSZzK+vnNdXaIiMqgMHwUWrJkiXyQr0QiwV9//SW/r2HDhli0aBFGjx6NvLw8aGgU/0/tRx99pPDzihUrsGXLFty6dQu9evWSHzcwMIClpWW5X4OPjw/Mzc3Rr1+/Us8TBAGCICA7O7vY+8eMGQM3NzcAwPPcPKzxu4fd330KveY98e7A4dgypSvO+R0HAIVAVPizTKb8pag+LawwppMtfr8ShRl/3MT/pnWFhSE3fKay4WUsIiIlvW6QLwD5N82Sgk5xbW7atAlGRkZwcnJSuG/JkiUwMTFBmzZt8NNPPyEvL0/pmmUyGXx8fDB27FiFmh48eABvb29cv34d0dHRuHTpEoYOHQpdXV2899578vMcHR1x8OBBAICJiQlatGgBXQs7fHM2FecSdSFR00Bv5ybYP3sIjGppwsXFBXXq1MHYsWNx8+ZNhIeHY+bMmYiMjHxt2CrJ1/2aoqmVIZ5l5GD6vmDky976CxNURgw7RERKKmmQb6HExET88MMPmDx58mvbOnr0KPT19aGjo4OVK1fC19cXpqam8vunTp2KvXv34syZM/j000+xePFizJo1S+maT506hejoaEyYMEHhuI6ODs6fP4/33nsPjRo1wvDhw2FgYIBLly7B3PzFpaKwsDCFMUhHbz3BB2suICw+Dab62jA10IZrI1N5+DM1NcXx48eRnp6Onj17wtnZGRcuXMDhw4eLhLmy0tFUx9qP2qCWljou3X+G9WfuvVE79PbhmB1wzA4RKcfd3R1aWlr4+++/i9yXmpqKd999F8bGxjhy5EiR9WdelZGRgdjYWCQmJmLz5s04ffo0AgICFILGy7Zu3YpPP/0U6enp0Nau3J3BY6XPERGfjr9vPsH+648AAB3tjbFmZBuYV+IlpT+vP8JX+29CTQLsneyCDvbGlfbcVL2U9fObPTtEREqIiorCqVOn8MknnxS5Ly0tDX369IGBgQEOHjz42qADAHp6emjUqBE6deqELVu2QENDA1u2bCnx/I4dOyIvL++1C/6p2r7AaLguOY2Pt16VB53P32mIXZ90rNSgAwAftquHwW3qQiYA0/beQHJGTqU+P9U8DDtEREooaZBvamoqevfuDS0tLRw5cgQ6Om8WAGQyWYkDgwEgODgYampqJfb8qFK+TEDoYyl+ORWO2X+F4OUhMmoS4GMXW2ioV83HyA8DW8DeVA+x0izM/PMmeJGiYrxuX7WsrCx4eHjAxMQE+vr6GDJkCOLj48vcfkn7qql6yQXOxiIiekms9DkiEzNgb6on34W7UEmDfAuDTmZmJnbu3InU1FSkpqYCKJjFpa6uDqBgkK+3tzcGDRqEjIwMLFq0CB988AGsrKyQmJiIdevW4fHjx/I9qy5fvoyAgAD06NEDBgYGuHz5MqZPn47Ro0crtTBfWWXl5uNGdAquPUzC1YdJuBGdgvTs4gdDywTgYWJmkb+jyqKnrYG1H7XBoHWXcOpOAnwuPsSELvZVUouYBQYGFrvkQuHv6PTp0/HPP/9g//79MDIywpQpUzB48GBcvHjxtW2Xtq8aAHz//fdFllx4U+zZISL6T+Glmo82B8B1yWnsC4xWuL+kQb5BQUEICAhASEgIGjVqBCsrK/ktJiZGfl5YWBiuRTxCrPQ51NXVcffuXQwZMgSNGzdG//798ezZM5w/f16+/YK2tjb27t2L7t27o3nz5li0aBGmT5+OTZs2qeT1JmfkwPd2PBb/7w4Grb+Ilt+ewMjNV7DcNxznIxKRnp0HA20NdLI3xqtzztQlEtiZ1lJJHa/z+PFjjB49GiYmJtDV1UXLli1x7do1NLc2wtf9miI/IxmeX0yCuYUVatWqhT59+iAiIqLUNt95551ieyxe7rETBAELFiyAlZUVdHV14ebmVmK7sdLnuHQ/EbHS52Wuv1B8fDzGjRsHa2vrKqu/JKXtqyaVSrFlyxasWLECPXv2lG8oe+nSJVy5cqXUdkvbV61Q4ZILhbc3WVuqEAcogwOUiQg4G5aAcT6BRY6b6Wujdi1NGOhowEBHE4a6hX/WgKHOiz8baL84p/A+fR0NqKsVxIR9gdGYe6DgUpCaBPAe3BLD29tU2usTBAGPkp8j8GESAh8mI/BhEu4lpBc5z8JQG+3tjNHezhjOdnXgaGkIdTUJ9gVGY96BUOQLAtQlEiwe3KJS6k9OTkabNm3Qo0cPfP755zAzM0NERAQaNmyIhg0bQiaTwapxa0izZWg2yAO/fdIVv65bg+PHj+P27dslfkAmJSUhJ+fFWJ9nz57ByckJv/32m3yW3dKlS+Ht7Y3t27fD3t4e8+fPR0hICG7fvq1wmbK0/7evq18QBHTu3BmamppYvnw5DA0N5dtqVFb9ZZWTkwNra2t4eXlh3rx5OH36NHr16oXk5GTUrl1bfp6trS08PT0xffr0YtuRyWRwc3PDgAEDMG3aNNjZ2cHT0xOenp7yc+zs7JCVlYXc3FzY2Njgo48+wvTp04ss5cBFBYmo2nn8+DFmz56NY8eOITMzE40aNYKPjw+cnZ0BFHzDnT17Nk6ePImUlBR069YNa9asgYODQ4ltHjhwAIsXL8a9e/eQm5sLBwcHzJgxA2PGjAEA5Obm4ptvvsH//vc/PHjwAEZGRnBzc8OSJUtgbW2NyMQMLD8ZhqO3it/C4Gl6Np6mlzyG5nX0tTVQS0sdCWkv2pAJwJy/QnDrsRTWRrqvhKeXApSOJgy0NaCmVvxaPq96+RKcuYEOwuLScC0qCVcjk3DtYTLiUotu/9DIXB/t7erIA069OrrFrh00vL0NujU2w8PETNiZ1qq0y1dLly5F/fr14ePjIz9mb//ictW9e/eQcD8ErT23IEnbAttv52L9+vWwsrLCnj17ih1IDgDGxoozuPbu3YtatWrJL88IgoBVq1bhm2++ka82vWPHDlhYWODQoUMYMWIEAOBJSibmHAhBYbeBTADmHQhFt8ZmsDLSfW39ERERuHLlCkJDQ+U9ehs2bIClpWWl1K+MV5dciIuLg5aWlkLQAQALCwvExcWV2E5J+6q9bOrUqWjbti2MjY1x6dIlzJ07F7GxsVixYoXSdQMMO0RUSZKTk+Hq6ooePXrg2LFj8m+4hWNPBEHAwIEDoampicOHD8u/4bq5uZX6DdfY2Bhff/01HB0doaWlhaNHj2L8+PEwNzeHu7s7MjMzERQUhPnz58PJyQnJycmYNm0a+vbrj34LtmFfYEyJi9OpSYDfxjpDW0MdaVm5SM3KQ1pWHtKychX+myr/OU9+Xk5ewSrB6dl5xY57EQDsuhJd5Hhx9LU1FAKQ4Us9SIX/jYhPw+HgJyh8JdoaasjOU1ypWENNghZ1jdDB3hjOtnXgbGcMY72SN/t8lZWRbqWP0Tly5Ajc3d0xdOhQ+Pv7o27duvjiiy/kYzkKB3N/O8gJXsfjcTj4CVwbmkJbWxsXLlwoMSy8asuWLRgxYoT89ywyMhJxcXHylaIBwMjICB07dsTly5fRf9CHOBT8GL/6P8Cr10fyBQF/BMbgix6Nylz/yz0tampqFV7/m4SdkvZVU0bhvmpBQUElLsgJAF5eXvI/t2rVClpaWvj000/h7e39RksuMOwQUaWoqG+477zzjsLP06ZNw/bt23HhwgW4u7vDyMgIvr6+8vulmblw/ugrrPcchme+16BhaI6ejub4qncThDxOKXKppqejxRu93uy8fHkAikxMx8Tt1xQ+FCUAhjnXh0wQCs7LfiUwPc9DTr5iYIotfV/RV55fBl1NNTj/12PT3s4YrevXhq6W+hu9nqry4MEDbNiwQX7pJDAwEFOnToWWlhbGjh0LR0dH2NjYYNfapfhs9GysPR8DjzkLkfjoUZk3HL169SpCQ0MVpvwX9kxYWCj+/9czMsGp62HwW+xX4uBtAFh5KgK7AqIRce8+Hjx4ff1z587Fr7/+Cj09PaxcuRKPKqj+1/W6lKRwyYUDBw7Ij1laWiInJwcpKSkKvTvx8fElbm9y/vx5JCQkwMbmxSXQ/Px8zJgxA6tWrSpxSYWXl1xo0qSJ0vUz7BBRpaiMb7iCIOD06dMICwvD0qVLFe7LzMmDz8WH2Oh/Hwl3wwFI0K5RPXw9qJ18Ubpm1oYqu1SjraEObX11mOprw95UD0sGt1R6zEtWbv4rPUnF9ybdf5oG//DEIo/f/LEzujiYFdNyzSGTyeDs7IzFixcDANq0aYPQ0FBs3LgRY8eOhaamJg4cOICJEydi/wBnSNTUoW3rBFPHjkApPQcv27JlC1q2bIkOHToUe39Ongwnb8dh55Uo+Ic/BSQSmGXnwd5UD6M62kBdTYIfj95BviBATQJ0b2yGkMdSJKRlIy9fBnXzRkhuNgSZBvUxaVLrEus3NjaGuro63Nzc0Ldv3zJPp39d/apQ3JIL7dq1g6amJvz8/DBkyBAABYPwo6Oj4eLiUmw7L++rVsjd3R1jxozB+PHjS3z+8i65wLBDRJWirN/Q3+QbrlQqRd26dZGdnQ11dXWsX78e7777LgAgN1+GvYExWO0Xgadp2RDycpB1cQd6vT8Ih6a7FelKr6hLNW8y5kVHUx06muowMyi92z5W+hyuS04rrIOjLpGgobl+ecuuclZWVmjWrJnCsaZNmypsvNquXTsEBwdDKpXicVIqPt51FyHrpyBRYvfa9jMyMrB37158//33CscLeyZ+PhSAc0n6ePrfmCtZZgoaNW2O9RM7wLWhqXw8VZ8Wlgr/b3PyZDj+bxxGbjaBmokNjoXG4VhoHBqY6aGuphmiol5cwny5/pycHJiZmaFjx47ysWzlqT8+Ph5WVlby4/Hx8WjdurXCuaUttwCUvOSCkZERJk6cCC8vLxgbG8PQ0BBffvklXFxc0KlTJ/l5Ly+5YGJiAhMTE4X2NTU1YWlpKe+xqYglFzj1nIgqhUwmQ9u2bbF48WK0adMGkydPxqRJk7Bx40YAkH/DDQ8Ph7GxMWrVqoUzZ86gb9++RXbOfpWBgQGCg4MRGBiIRYsWwcvLC6dPn8Hh4MdwW+GP+YdC8TQtG3WNNGERuAENzfRwYJdPqWMGKoKVkS5cGpqoPExZGenCe3BLqP/3egp7jqpqDRxVcnV1RVhYmMKx8PBw2NraFjnXyMgIzezrw6ujIXLi7uGRYXP8U8LA80L79+9HdnY2Ro8eDQCQyQScj3gK7/PPoK5XBz77/8bTtGyYGWhjckdLCAkR+Hr8QHR1MFMYOP7q/1stDTV84GSNAe490UQnDWM62UJPSx0Pnmbg8LnrSFY3wuw/byH08Ytrk0ZGRvKxbNeuXZMPLFam/kL29vawtLSEn5+f/FhqaioCAgIUel1eXW5hz9Wi48hKWnIBAFauXIn3338fQ4YMQbdu3WBpaalwqQtQXHKhLCpiyQVOPQennlPN8roZTSV9gC9btgwzZ84s9r78/Hx8++232LlzJ+Li4mBtbY1x48bhm2++kbcnCAIWLlyIzZs3IyUlBa6urtiwYUOpM6VeZmtri3fffRe//fab/NiGDRvw448/4vHjxwrnFvcNd926dWV6HkEQ8P6w0QgIiYD+wIUAAFN9bXzR3RaHl8/Ew8hInD59usi3SzGIlT6v9NlSFS0wMBCdO3fGd999h2HDhuHq1auYNGkSNm3ahFGjRgEo+MA3MzODjY0NQkJCMG3aNOjXa4z0LtNgoK2B/03riq89P0PdunXh7e2t0H7Xrl1Rt25dbNy6A39ef4RdAdGITMwAAEiv/ImMwL8w23s1BnVtg++/W4hbt24pNXX75fr7DRyMjftPYvPi2ajd2wP6zXsAAMyeBuH9Do4Y9k5rhN+9jWnTpqFdu3YKvVcff/xxqfXv3bu3yHMvXboUS5YsUZh6fiP4Jn77+xwik3MQFJ2C46FFx+/oaanDSPflZRZemSGoMFC+8M8vztHTejGDsKKXXODUcyIRet2MJgBFLvkcO3YMEydOlF9TL87SpUuxYcMGbN++Hc2bN8e1a9cwfvx4GBkZyaeHLlu2DKtXr1b4h9Pd3b3M//Ar+w0dgPwb7g8//PDa9gHgelQSlh4Pg39YAvIyn8NKWwOfdm+A0R3qYfyYj3D/3j2cOXNGlEEHqJrZUhWtffv2OHjwIObOnYvvv/8e9vb2WLVqlTzoAAW/815eXvJLNh9//DHmzPsaY3yuIyg6BVP23EBKVHSRHsKwsDBcuHABH36zAR0X+8lnrxloa2Bw27oY5fkLdq6vj02L5+CnlBR06dIFx48fV2qNmuLqX7/mF7Tp/SF+vxyFY6GxuB/1CD/sX43vMlNgZGyGMR+Pwc+LFX/no6NLrv/kyZNFnjczJw/vjpiEq+FP8NHYCXienoZaNs1g2HsuJu26VWrNGTn5yMjJxxNp0aUKykIiKZhBqKelobDcwavT8isTe3bAnh2qOebMmYOLFy/i/PnzZX7MwIEDkZaWptCd/ar3338fFhYWCrM5hgwZAl1dXezcuROCIMDa2hozZszAV199BaCg98XCwgLbtm0r0zTWN/2G/rpvuN7e3rBo0AzHowVcDIvD8weBSPbfjoFTFmLr4tnQ15Lgww8/RFBQEI4ePaowO8XY2BhaWmWfek01y6PkTLz3y3mkZuXho442eL+VFexN9VBbVwt/33qCnVeicOvRi8tITa0MMaaTLQa0toaeduX0BTxNy8Yf12KwOyAaj1NeXObp1tgMYzrZoqejOdTVJCWOq8mXCXj4LANhcWm4G5uKu3FpCItPQ3RSZpEp8UBB74q9qR4cLQ1hXVsHv52PhPDK/XsmdYKOpvpLSykUDoovuuzCq4Pmc/NfHyn2TOoEl4aq+cLBnh0iEXrdjKZXxcfH459//sH27dtLbbdz587YtGkTwsPD0bhxY9y8eRMXLlyQL+ClijU73vQb+vz58xXaefkbbkxSJg4FPkDQT2uQn/YMEg0tWNk2xOLffPDZ+IJFBR8+fIgjR44AQJGBmWfOnCkydZ3Eo16dWlj2YSt8tjMIuwOisTsgGhIUDPx+nluw35OWuhr6tbLC6E62aGtTu9LHcZkZaMOjRyN81r0hztxNwO9XonAu4inOhRfcrI100LKuEXzvxEMmFPSavNfCCrpa6giLS0N4fFqR9ZRebtvR0gBNLAzgaGUIR0sDNDLXh47mi+UHGpnrF5kl2LHBmwURQRCQnSeTh6PIxAxM2qG45EJlbjPyMvbsgD07VHMUdp97eXlh6NChCAwMxLRp0+TTWF+1bNkyLFmyBE+ePCm1610mk2HevHlYtmwZ1NXVkZ+fj0WLFmHu3LkAgEuXLsHV1RVPnjxRmNkxbNgwSCQS7Nu3T8WvtGSx0ucIjk7B6bsJOBT8WP5Nsl9LK3j1boyGZjV/BhKpTqz0OTp7n8arH3SWRjoY62KHYc71YKKv/CJ1FSnqWQZ2B0Rj37UYpGTmvvZ8XU11NLbQRxNLAzhaFoSaJpYGZX5dFTnWq6K3GWHPDpEIvW7NkVdt3boVo0aNeu0Ygz/++AO7du3C7t270bx5cwQHB8PT0xPW1tbFtlsVsnLzscI3HJvPPVD44OrqYIqZ7k3Qql7tqiqNqrHIxIwiQQcAlg91gmsj00qvpyxsTfQw972mmP5uY/ziF4ENZ+8XOWdI27p4t5kFHC0NUd+4lnwPtjdRkWO9qmqbkVcx7BDVIGVZc6TQ+fPnERYWVqZel5kzZ2LOnDnyy1EtW7ZEVFQUvL29MXbsWKXW7FCV5IwcXItKxrWHSQh8mIRbj1Lwam+9mgRY9mEr0Q3KJdWxN9WDmgRF1iBqYPbmO2hXFh1NdXzsYotf/e8Xqf8r9yY15ve+OgycZ9ghqkGUmdG0ZcsWtGvXDk5OTq9tNzMzs8hMD3V1dchkBeni5TU7CsNN4Zodn3/++Ru+mhcKd+S+FvXfjtyRSYgoZkfuV8kE4GFiZpX/Q0rVV+EaRK9eSqkpvzM1vf7qgmGHqAaZPn06OnfujMWLF8tnNG3atKnIYlupqanYv38/li9fXmw7vXr1wqBBgzBlyhQAQP/+/bFo0SLY2NigefPmuHHjBlasWCFfREwikcDT0xM//vgjHBwc5FPPra2tMXDgQKVfh0wmICw+DdceJuHqw4Lem9hiprk2NNP7b9NKY9iZ1sLQjZeLfMOtisGOVLNUl0spb6qm118dMOwQ1SBlmdEEAHv37oUgCBg5cmSRNmKlz3E7LAKtH71Yj2fNmjWYP38+vvjiCyQkJMDa2hqffvopFixYID9n1qxZyMjIwOTJk5FSypojxU2RzcrNR8hjKa5GJuHawyRci0pGWpbiJoqFO3K3t6uD9nbGaGdbp8gAS37DpTdVHS6llEdNr7+qcTYWOBuL3h4VvZrpy+1L/tsQMSM7DzdjpPIdvAvpaamjrW1BsHG2q4PW9Wujltbrv3+JcZVgInozZf38ZtgBww6JW2J6NsLi0nA1Mgm/+EUUub+2rqbCHj9vSiYTkPK85Gmypvra6GBfB862xmhvZ4ymVgbQUOf2fET05jj1nOgtk5Wbj4j4dNyJS0VYXFrBiqpxaUhMzy71caUFFFX4tFsDjOxgA1uTWpW+YBsREcCwQ1QtlbQ0PFDQgxKTnIk7sQWBJiw+FXdj0/DwWYbC4N1CEglga1wLtia1cC48scjS8DsndoSZQfkXVXualo3RWwKKDCAe52rHy01EVKUYdoiqmVfH1UzsYg/r2roIi0vDnbg0RMSnITMnv9jHGutp/bc0vMF/q6gaorGFvnwsTHGrmXZW0cJqDhYGHEBMRNUSx+yAY3ao+oiVPofrktPF9tC8TEtDrWB5eIuCpeEdrQqWhzfT137tpaKKHuDLAcREVFk4ZoeoBroYkVhs0GlnWweuDU3QxNIQTSwNYGdS640H91b0FFZOkSWi6oZhh6iauJeQjkX/u1vkuLpEgrUftWGAICJ6Q5z3SVQN3EtIx4hNV5CcmQNLQ20UzgTnuBciovJjzw6JUmmzmaqbewnpGLn5ChLTs+FoaYDdkzohOy+f416IiFSEPTv0Rh4/fozRo0fDxMQEurq6aNmyJa5duwYAyM3NxezZs9GyZUvo6enB2toaH3/8MZ48eVJqm+fOnUP//v1hbW0NiUSCQ4cOKdxf1nb3BUbDdclpfLQ5AK5LTmNfYLRKX7sq3X9aEHSepr0IOsZ6WrAy0oVLQxMGHSIiFWDYIaUlJyfD1dUVmpqaOHbsGG7fvo3ly5ejTp06AAp20A4KCsL8+fMRFBSEAwcOICwsDB988EGp7WZkZMDJyQnr1q0r9v6ytBsRn4Y5f4XIB/nKBGDegVDESp+r5sWr0P2nBZeuXg06RESkWpx6Dk49V9acOXNw8eJFnD9/vsyPCQwMRIcOHRAVFQUbm9fvxSSRSHDw4MHX7qhd2O6N2+E4GZUPn4uReJ4rK3Lejgnt0a2xeZnrrWj3n6Zj5KYrSPgv6Oz6pGORTS+JiKh0Zf38Zs8OKe3IkSNwdnbG0KFDYW5ujjZt2mDz5s2lPkYqlUIikaB27doqrSUiJh6QSPDh1ltYf/Z+sUEHALyPheFxSvXo3XnwUtBpYsGgQ0RU0Rh2SGkPHjzAhg0b4ODggBMnTuDzzz/H1KlTsX379mLPz8rKwuzZszFy5EiV9ZyFx6dh6u8BGO/hiVpNuyFPXQdO9Wvj1zHt4D24JdT/W1hPTQLoaKjhTmwq+q0+j7NhCSp5/jf14L8xOvKgM4lBh4ioonE2FilNJpPB2dkZixcvBgC0adMGoaGh2LhxI8aOHatwbm5uLoYNGwZBELBhw4ZyP3dQdDLWn7kP39DHeHpoMSAA73+2ANP6toJLQxP56sHvNDGTz2bKyxfw+a7rCH2civHbAvFlTwdM6+UAdRXs9K2MyMQMjNx8BfGp2WhsoY9dkzrClEGHiKjCMeyQ0qysrNCsWTOFY02bNsVff/2lcKww6ERFReH06dNv3KsjCALORyRi/dl7uPIgCUJ+HhIPL0Gt7GQc8D2Jbi0bFK3xlVV8//ysM344ehu7AqKx2i8CQVHJ+GVE60rrVYlMzMCITZflQWf3pE4MOkRElYSXsUhprq6uCAsLUzgWHh4OW1tb+c+FQSciIgKnTp2CiYmJ0s8jkwn451Ys+q+9gI+3XsWVB0lQF/Kh5f8L6qpJcef6xWKDTnF0NNWxaFBLrBjmBI3nyTi0cg4szc2go6M4bR4oCFcLFiyAlZUVdHV14ebmhoiIiFLb9/b2Rvv27WFgYABzc3MMHDhQ/nf0MDEDIzddwe0/liPht8k4P68PmtrXw4ABA3D3btEVk4mISLUYdkhp06dPx5UrV7B48WLcu3cPu3fvxqZNm+Dh4QGgIOh8+OGHuHbtGnbt2oX8/HzExcUhLi4OOTk58nZ69eqFtWvXyn9OT09HcHAwAq8FAQBmbzuFSSv/RPCde9DVVMfYjvXQKGQTsmIjsH/fnhLbLU0Pez08P/A1DGrpwPTDb2ExYR3eHT9TYeD0smXLsHr1amzcuBEBAQHQ09ODu7s7srKySmzX398fHh4euHLlCnx9fZGbm4vevXvjdlQCRmy6grjULNRzaI69O7fjzp07OHHiBARBQO/evZGfX/wO5kREpCICCVKpVAAgSKXSqi6lxvj777+FFi1aCNra2oKjo6OwadMm+X2RkZECgGJvZ86ckZ9na2srLFy4UP7z/06cKvYxzu8OEp6lZ5e53dLMnj1b6NKli5CWlSt8seu6YDv7qGA7+6jw+c5rQurzHEEmkwmWlpbCTz/9JH9MSkqKoK2tLezZs6fMfz8JCQkCAKHpJysE29lHBbflZ4WE1CyFc27evCkAEO7du1fmdomI6IWyfn5zzA69kffffx/vv/9+sffZ2dlBeM3yTbHS59jtdw32pnpIzsjBtksPsf1KPmxnHwUAWBhqY1LXBhjRwQb62gW/psZ6r2/3dY4cOQJ3d3eMHz0S/v7+0K1thueNeuF/cMed2DTM61oHcXFxcHNzkz/GyMgIHTt2xOXLlzFixIgyPc/th7EAgBSZFpqZF4zRMTN4MUYnIyMDPj4+sLe3R/369cv1moiIqHQMO1Tp9gVGY+6BglWOJQA01SXIyS8IMXYmtfBZ94YY1LYutDXUVf7chdPmvby8MG/ePAQGBmLq1Gkw1NNFJLrhkw3XAQAWFhYKj7OwsEBcXFyZniPyaRoGjpkM7brN0LRZc+ye1FEedNavX49Zs2YhIyMDTZo0ga+vL7S0uGoyEVFFUnrMzpkzZyqiDhK55IwcXL7/DL+cCsfsl7ZzEADk5AtobKGPtR+1gd+MdzCig02FBB2gYNp827ZtsXjxYrRp0waTJ0/G5MmTUCfaH90amyE7r6CwRf/cRlau8mNpop5lwOX9j5AWG4l247/FnsmdYG6gI79/1KhRuHHjBvz9/dG4cWMMGzas1LFARERUfkr37PTp0wf16tXD+PHjMXbsWHbBk4LsvHzcS0hHWFwa7v53C4tLRXxqdqmP++6D5nBpaFrh9ZU2bT5gXHss1EzDj7uAAxdvIzJTCxtGtYONSS3Ex8ejdevWpbYd/SwTHd8fhcQ7V9BhymocnNVfIegABZfEjIyM4ODggE6dOqFOnTo4ePAgRo4cqeqXSkRE/1E67Dx+/Bi///47tm/fju+++w49e/bExIkTMXDgQHbHi0ys9DkiEzNgb6pXZPdtQRDwKPm5PMwUBpvIxAzky4ofV1PfWBe2xnq4eC8RL5+hLpHAzlSvAl/JC6VNm1dTk+D70T2xfoY5JLGh+NeiAfqtOY8f3muIgIAAfP755yW2G5WYgY79RyEx9DycPX7BoTmDigSdVwmCAEEQkJ1dehAkIqLyKddGoEFBQfDx8cGePXsAAB999BEmTpwIJycnlRVYGbgRaFEvj6tRkwCTujZA3Tq6BaEmNhXh8elIz84r9rFGuppoYmmAppYGaGJpiCaWBmhiaSAfaLwvMBrzDoQiXxCgLpFg8eAWGN7+9ZuDqkJgYCA6d+6M7777DsOGDcPVq1cxadIkbNq0CaNGjQIALF26FN7eS9B0xBxE5eoj5fxO6KQ9QvS9MOjr1QJQMG1+0KBBmDJlCqKfZaJDv5FICPaD04Qf8fv0gTD9L+gYGRlBV1cXDx48wL59+9C7d2+YmZnh0aNHWLJkCS5evIg7d+7A3Lz6bFJKRFRTlPXzu9y7nj958gSbNm3CkiVLoKGhgaysLLi4uGDjxo1o3rx5eZquNAw7imKlz+G65DRK6KCR01SXoKGZPppavQg0TS0NYWGoLd+2obTnKNzO4dVeo4p29OhRzJ07FxEREbC3t4eXlxcmTZokv18QBCxcuBCbNm3Cs6RkaFg3hXHvL+DariXWjmwDc0Md2NnZYdy4cZg4dRZGbLqCS3N7FftcPj4+GDduHJ48eYJPPvkE169fR3JyMiwsLNCtWzcsWLAATZo0qayXTkQkKhUadnJzc3H48GFs3boVvr6+cHZ2xsSJEzFy5Eg8ffoU33zzDYKCgnD79u1yvQig4LLZ7NmzcezYMWRmZqJRo0bw8fGBs7MzgBcfTJs3b0ZKSgpcXV3lm1SWFcOOokv3E/HR5oAix9vWr41ODU3gaGUIR0sD2JvqQVNd/OtS/i8kFrP+vIX07DyY6mtjzcg2sDOthasPkuB97C7iUrPQwFQPeyZ3goVh6ZeuiIhIdcr6+a30mJ0vv/wSe/bsgSAIGDNmDJYtW4YWLVrI79fT08PPP/8Ma2vrN6v8JcnJyXB1dUWPHj1w7NgxmJmZISIiAnXq1JGfU7ja7fbt22Fvb4/58+fD3d0dt2/fho4OP3jeRHFjTdQlEqwb3bbSe2Gqg/daWsHR0gBf7ArC3bg0fLT5CgDIxx2Z6msx6BARVWNKh53bt29jzZo1GDx4MLS1i9/I0NTUVCVT1JcuXYr69evDx8dHfsze3l7+Z0EQsGrVKnzzzTcYMGAAAGDHjh2wsLDAoUOHyrwAHCm6eC9R4efCcTVvY9Ap1MBMHwe/cMX0P4JxPFRxvZ2kjBzIyrnYIRERVRylr0H4+flh5MiRJQYdANDQ0ED37t3LVRhQsNqts7Mzhg4dCnNzc7Rp0wabN2+W3x8ZGVnqarclyc7ORmpqqsKNCshkArZdeggAmNG7MfZM6oQLc3pU2gDi6kxXSx0fd7ItclwmAA8TM6ugIiIiKgulw463tze2bt1a5PjWrVuxdOlSlRRVqHC1WwcHB5w4cQKff/45pk6diu3btwOAfEVbZVe79fb2lq93YmRkxLWCXnI2PAGRiRkw0NHABFd7uDQ0eat7dF5lb6YHtVfGXhdMna9VNQUREdFrKR12fv31Vzg6OhY53rx5c2zcuFElRRUqbrXbSZMmlft55s6dC6lUKr/FxMSoqOKaz+fiQwDAiPb1oafN3UReZWWkC+/BLaH+32wzXuIjIqr+lP40i4uLg5WVVZHjZmZmiI2NVUlRhUpb7RYALC0tAQDx8fEKNb1utVttbe1SL8O9rcLj03A+IhFqEuBjF7uqLqfaGt7eBt0am1XZ1HkiIlKO0j079evXx8WLF4scv3jxokpmYL2stNVugYLBypaWlvDz85Pfn5qaioCAALi4uKi0lrdBYa9O72aWqG/MyzKlsTLS5SU+IqIaQumenUmTJsHT0xO5ubno2bMngIJBy7NmzcKMGTNUWtz06dPRuXNnLF68WL7a7aZNm7Bp0yYAgEQigaenJ3788Uc4ODjIp55bW1tj4MCBKq1F7FIyc3DwxiMAwHhXu6othoiISIWUDjszZ87Es2fP8MUXXyAnJwcAoKOjg9mzZ2Pu3LkqLa59+/Y4ePAg5s6di++//x729vZYtWqVfFl/AJg1axYyMjIwefJkpKSkoEuXLjh+/DjX2FHSnqsxyMqVoZmVITrYG1d1OURERCrzxttFpKen486dO9DV1YWDg0ONHgPztq+gnJsvQ7dlZxArzcJPH7bCUGfOTiMiouqvwlZQLqSvr4/27du/6cOpGjnxbxxipVkw1ddCfyfVjrsiIiKqam8Udq5du4Y//vgD0dHR8ktZhQ4cOKCSwqjyFA5M/qijLXQ01au2GCIiIhVTejbW3r170blzZ9y5cwcHDx5Ebm4u/v33X5w+fRpGRkYVUSNVoJsxKbgelQxNdQlGd+IqyUREJD5Kh53Fixdj5cqV+Pvvv6GlpYVffvkFd+/exbBhw2Bjww/LmsbnYiQA4P1W1sVuAEpERFTTKR127t+/j379+gEAtLS0kJGRAYlEgunTp8unhFPNkJCahX9CChaCnOBq/5qziYiIaialw06dOnWQlpYGAKhbty5CQ0MBACkpKcjM5GaINcnOK1HIzRfgbFsHLevxEiQREYmT0gOUu3XrBl9fX7Rs2RJDhw7FtGnTcPr0afj6+qJXr14VUSNVgKzcfOwKiAYAjGevDhERiZjSYWft2rXIysoCAHz99dfQ1NTEpUuXMGTIEHzzzTcqL5AqxpGbT/AsIwfWRjpwb27x+gcQERHVUEqFnby8PBw9ehTu7u4AADU1NcyZM6dCCqOKIwiCfLr5x53toKGu9NVMIiKiGkOpTzkNDQ189tln8p4dqpkCIpNwJzYVOppqGNGeqyUTEZG4Kf2VvkOHDggODq6AUqiyFE43H9y2HmrX0qriaoiIiCqW0mN2vvjiC3h5eSEmJgbt2rWDnp6ewv2tWrVSWXGkejFJmTh5Ox4AML6zXdUWQ0REVAmUDjsjRowAAEydOlV+TCKRQBAESCQS5Ofnq646Urntlx5CEICuDqZwsDCo6nKIiIgqnNJhJzIysiLqoEqQkZ2HfddiAHARQSIiensoHXZsbW0rog6qBH8FPUJaVh4amOqhe2Ozqi6HiIioUigddnbs2FHq/R9//PEbF0MVRyZ7Md18nKsd1NQkVVsQERFRJVE67EybNk3h59zcXGRmZkJLSwu1atVi2Kmm/MOfIjIxAwY6GhjStl5Vl0NERFRplJ56npycrHBLT09HWFgYunTpgj179lREjaQCW/+bbj7cuT70tJXOuERERDWWSpbOdXBwwJIlS4r0+lD1EBGfhvMRiVCTAGM53ZyIiN4yKtsnQENDA0+ePFFVc6RCPpceAgDebWaB+sa1qrYYIiKiSqb09YwjR44o/CwIAmJjY7F27Vq4urqqrDBSjZTMHBwIegSAu5sTEdHbSemwM3DgQIWfJRIJzMzM0LNnTyxfvlxVdZGK7A2MQVauDE2tDNHR3riqyyEiIqp0SocdmUxWEXVQBcjLl2HHf5ewJrjaQSLhdHMiInr7qGzMDlU/J/6NxxNpFkz0tNDfybqqyyEiIqoSSoedIUOGYOnSpUWOL1u2DEOHDlVJUaQahbubj+poAx1N9SquhoiIqGooHXbOnTuH9957r8jxvn374ty5cyopisov5JEU16KSoakuwehO3OKDiIjeXkqHnfT0dGhpaRU5rqmpidTUVJUUReVX2KvzfitrmBvqVHE1REREVUfpsNOyZUvs27evyPG9e/eiWbNmKimKyichNQt/3ypY82i8q13VFkNERFTFlJ6NNX/+fAwePBj3799Hz549AQB+fn7Ys2cP9u/fr/ICSXk7A6KRmy+gnW0dtKpXu6rLISIiqlJKh53+/fvj0KFDWLx4Mf7880/o6uqiVatWOHXqFLp3714RNZISsnLzsTsgCgB7dYiIiIA3CDsA0K9fP/Tr10/VtZAK/H3zCRLTc2BtpIM+zS2ruhwiIqIqp/SYncDAQAQEBBQ5HhAQgGvXrqmkqLfNkiVLIJFI4OnpqXD88uXL6NmzJ/T09GBoaIhu3brh+fPnJbazePFiTBrcG9ErhyJk2TB8OGQwwsLCFM7ZtGkT3nnnHRgaGkIikSAlJaUCXhEREVH1oXTY8fDwQExMTJHjjx8/hoeHh0qKepsEBgbi119/RatWrRSOX758GX369EHv3r1x9epVBAYGYsqUKVBTK/l/2d/H/aDRsg9sx6/A/46dQG5uLnr37o2MjAz5OZmZmejTpw/mzZtXYa+JiIioOlH6Mtbt27fRtm3bIsfbtGmD27dvq6Sot0V6ejpGjRqFzZs348cff1S4b/r06Zg6dSrmzJkjP9akSZNS23OatAyx/8ZhRAcbdO3UEo7btsHc3BzXr19Ht27dAEDee3T27FmVvhYiIqLqSumeHW1tbcTHxxc5HhsbCw2NNxoC9Nby8PBAv3794ObmpnA8ISEBAQEBMDc3R+fOnWFhYYHu3bvjwoULJbYVk5SJk7fjABTsgwUAUqkUAGBszA1AiYjo7aV02Onduzfmzp0r/yAFgJSUFMybNw/vvvuuSosTs7179yIoKAje3t5F7nvw4AEA4Ntvv8WkSZNw/PhxtG3bFr169UJERESx7e24/BAyAejqYAoHCwPIZDJ4enrC1dUVLVq0qNDXQkREVJ0p3RXz888/o1u3brC1tUWbNm0AAMHBwbCwsMDvv/+u8gLFKCYmBtOmTYOvry90dIqubly4s/ynn36K8ePHAyi4TOjn54etW7cWCUgZ2XnYG1gwjqpwurmHhwdCQ0NL7Q0iIiJ6GygddurWrYtbt25h165duHnzJnR1dTF+/HiMHDkSmpqaFVGj6Fy/fh0JCQkKY5/y8/Nx7tw5rF27Vj6D6tUVqZs2bYro6Ogi7R0IeoS0rDzYm+rhncbmmDJlCo4ePYpz586hXr16FftiiIiIqrk3GmSjp6eHyZMnq7qWt0avXr0QEhKicGz8+PFwdHTE7Nmz0aBBA1hbWxeZNh4eHo6+ffsqHJPJBPhcfAgAGOtii6lTv8TBgwdx9uxZ2NvbV+jrICIiqgneeETx7du3ER0djZycHIXjH3zwQbmLEjsDA4Mi42j09PRgYmIiPz5z5kwsXLgQTk5OaN26NbZv3467d+/izz//lD+mV69eaN7ZDQ9yW8FAWwMXdyzF/n17cfjwYRgYGCAurmDAspGREXR1dQEAcXFxiIuLw7179wAAISEhMDAwgI2NDQcyExGRKCkddh48eIBBgwYhJCQEEokEgiAAACQSCYCCyzFUfp6ensjKysL06dORlJQEJycn+Pr6omHDhvJz7t+/j0T9BkDTVhjWvj4W9P8VAPDOO+8otOXj44Nx48YBADZu3IjvvvtOfl/hlPSXzyEiIhITiVCYVsqof//+UFdXx2+//QZ7e3tcvXoVz549w4wZM/Dzzz+ja9euFVVrhUlNTYWRkRGkUikMDQ2rupwyu3w/ESM3B0AC4NysHqhvXKuqSyIiIqo0Zf38Vnrq+eXLl/H999/D1NQUampqUFNTQ5cuXeDt7Y2pU6eWq2gqu32B0Ri5uWDbDgHApfuJVVsQERFRNaV02MnPz4eBgQEAwNTUFE+ePAEA2NraFhlQSxUjVvoccw8oDnCedyAUsdKS980iIiJ6Wyk9ZqdFixa4efMm7O3t0bFjRyxbtgxaWlrYtGkTGjRoUBE10ivuJ2RA9srFx3xBwMPETFgZ6VZNUURERNWU0mHnm2++kW8s+f333+P9999H165dYWJign379qm8QCqquEtW6hIJ7Ew5ZoeIiOhVSocdd3d3+Z8bNWqEu3fvIikpCXXq1JHPyKKKE/pYis3nC7aTkEgAQSgIOosHt2CvDhERUTFUsnMn12epHM9z8jFt7w3k5gtwb26Bhf2bIerZc9iZ1mLQISIiKgG3Ka9BvI/dwf2nGTA30MaSwa1QR08L1rV56YqIiKg0Ss/Goqpx5m4CdlyOAgAsH+aEOnpaVVwRERFRzcCwUwMkpmdj5p83AQATXO3R1cGsiisiIiKqOZQOO+fOnUNeXl6R43l5eTh37pxKiqIXBEHA7D9vITE9B00sDDCrT5OqLomIiKhGUTrs9OjRA0lJSUWOS6VS9OjRQyVF0Qu7AqLhdzcBWupqWDWiNXQ01au6JCIiohpF6bAjCEKxU8yfPXsGPT09lRRFBe4/TceP/9wGAMzq0wRNrWrOvl1ERETVRZlnYw0ePBhAwe7m48aNg7a2tvy+/Px83Lp1C507d1Z9hW+pnDwZPPcGIytXhi6NTDHB1b6qSyIiIqqRyhx2jIyMABT07BgYGEBX98W6LlpaWujUqRMmTZqk+grfUqtOhSPksRS1a2li+TAnqKlxwUYiIqI3Ueaw4+PjAwCws7PDV199xUtWFSjgwTNs8L8PAFgyuCUsDHWquCIiIqKaS+kxO7NmzVIYsxMVFYVVq1bh5MmTKi3sbSV9nguvP25CEICh7eqhTwurqi6JiIioRlM67AwYMAA7duwAAKSkpKBDhw5Yvnw5BgwYgA0bNqi8wLfNgsOheJzyHLYmtbDwg+ZVXQ4REVGNp3TYCQoKQteuXQEAf/75JywtLREVFYUdO3Zg9erVKi/wbXI4+DEOBz+BupoEK4e3hr42d/MgIiIqL6XDTmZmJgwMDAAAJ0+exODBg6GmpoZOnTohKipK5QW+LR4lZ+KbQ6EAgC97NkJbmzpVXBEREZE4KB12GjVqhEOHDiEmJgYnTpxA7969AQAJCQkwNOQ6MG8iXybA64+bSMvKQ1ub2pjSo1FVl0RERCQaSoedBQsW4KuvvoKdnR06dOgAFxcXAAW9PG3atFF5gW+DX8/dx9XIJOhpqWPl8NbQUOeWZURERKqi9Kfqhx9+iOjoaFy7dg0nTpyQH+/VqxdWrlyp0uJetWTJEkgkEnh6esqPZWVlwcPDAyYmJtDX18eQIUMQHx9foXWoUsgjKVacDAcALPygOWxNOKWfiIhIld6oC8HS0hIGBgbw9fXF8+fPAQDt27eHo6OjSot7WWBgIH799Ve0atVK4fj06dPx999/Y//+/fD398eTJ0/kqz1Xd89z8jFt3w3kyQT0bWGJoe3qVXVJREREoqN02Hn27Bl69eqFxo0b47333kNsbCwAYOLEiZgxY4bKCwSA9PR0jBo1Cps3b0adOi8G7kqlUmzZsgUrVqxAz5490a5dO/j4+ODSpUu4cuVKhdSiSov+dxsPnmbAwlAbiwe1LHbPMSIiIiofpcPO9OnToampiejoaNSqVUt+fPjw4Th+/LhKiyvk4eGBfv36wc3NTeH49evXkZubq3Dc0dERNjY2uHz5contZWdnIzU1VeFW2fzuxGPnlWgAwPKhrVFHT6vSayAiInobKL2Qy8mTJ3HixAnUq6d4ycXBwaFCpp7v3bsXQUFBCAwMLHJfXFwctLS0ULt2bYXjFhYWiIuLK7FNb29vfPfdd6outcyepmVj1p+3AACfdLFHFwfTKquFiIhI7JTu2cnIyFDo0SmUlJSksBO6KsTExGDatGnYtWsXdHRUtz/U3LlzIZVK5beYmBiVtf06giBg1p838SwjB46WBvjKvUmlPTcREdHbSOmw07VrV/l2EQAgkUggk8mwbNky9OjRQ6XFXb9+HQkJCWjbti00NDSgoaEBf39/rF69GhoaGrCwsEBOTg5SUlIUHhcfHw9LS8sS29XW1oahoaHCrbLsvBKFM2FPoaWhhl9GtIGOpnqlPTcREdHbSOnLWMuWLUOvXr1w7do15OTkYNasWfj333+RlJSEixcvqrS4Xr16ISQkROHY+PHj4ejoiNmzZ6N+/frQ1NSEn58fhgwZAgAICwtDdHS0fP2f6uReQhp+/OcOAGBOH0c0sTSo4oqIiIjET+mw06JFC4SHh2Pt2rUwMDBAeno6Bg8eDA8PD1hZqXaHbgMDA7Ro0ULhmJ6eHkxMTOTHJ06cCC8vLxgbG8PQ0BBffvklXFxc0KlTJ5XWUl45eTJM2xuM7DwZujqYYlxnu6ouiYiI6K2gdNiJjo5G/fr18fXXXxd7n42NjUoKK6uVK1dCTU0NQ4YMQXZ2Ntzd3bF+/fpKraEsVviG498nqahTSxPLhzpBTY3TzImIiCqDRBAEQZkHqKurIzY2Fubm5grHnz17BnNzc+Tn56u0wMqQmpoKIyMjSKXSChm/c/n+M3z02xUIArBxdDv0aVHyeCIiIiIqm7J+fis9QFkQhGIXv0tPT1fpjCmxkGbmYsYfwRAEYLhzfQYdIiKiSlbmy1heXl4ACmZfzZ8/X2H6eX5+PgICAtC6dWuVF1iTPUnJxIz9t/BEmgU7k1pY0L9ZVZdERET01ilz2Llx4waAgp6dkJAQaGm9WPFXS0sLTk5O+Oqrr1RfYQ21LzAac/4KQeE1wn4traCnrfQQKSIiIiqnMn/6njlzBkDB1O9ffvmlUtemqWlipc8x98CLoAMAG/0fYLSLLayMdKusLiIioreR0mN2fHx8GHReIzIxA7JXhn3nCwIeJmZWTUFERERvMaXDDr2evakeXp1Zri6RwM606DYbREREVLEYdiqAlZEuvAe3hPp/s9bUJRIsHtyCl7CIiIiqAEfMVpDh7W3QrbEZHiZmws60FoMOERFRFWHYqUBWRroMOURERFWMl7GIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUqnXY8fb2Rvv27WFgYABzc3MMHDgQYWFhCudkZWXBw8MDJiYm0NfXx5AhQxAfH19FFRMREVF1U63Djr+/Pzw8PHDlyhX4+voiNzcXvXv3RkZGhvyc6dOn4++//8b+/fvh7++PJ0+eYPDgwVVYNREREVUnEkEQhKouoqyePn0Kc3Nz+Pv7o1u3bpBKpTAzM8Pu3bvx4YcfAgDu3r2Lpk2b4vLly+jUqVOZ2k1NTYWRkRGkUikMDQ0r8iUQERGRipT187ta9+y8SiqVAgCMjY0BANevX0dubi7c3Nzk5zg6OsLGxgaXL18usZ3s7GykpqYq3IiIiEicakzYkclk8PT0hKurK1q0aAEAiIuLg5aWFmrXrq1wroWFBeLi4kpsy9vbG0ZGRvJb/fr1K7J0IiIiqkI1Jux4eHggNDQUe/fuLXdbc+fOhVQqld9iYmJUUCERERFVRxpVXUBZTJkyBUePHsW5c+dQr149+XFLS0vk5OQgJSVFoXcnPj4elpaWJbanra0NbW3tiiyZiIiIqolq3bMjCAKmTJmCgwcP4vTp07C3t1e4v127dtDU1ISfn5/8WFhYGKKjo+Hi4lLZ5RIREVE1VK17djw8PLB7924cPnwYBgYG8nE4RkZG0NXVhZGRESZOnAgvLy8YGxvD0NAQX375JVxcXMo8E4uIiIjErVpPPZdIJMUe9/Hxwbhx4wAULCo4Y8YM7NmzB9nZ2XB3d8f69etLvYz1Kk49JyIiqnnK+vldrcNOZWHYISIiqnlEuc4OERERkbIYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1Bh2iIiISNQYdoiIiEjUGHaIiIhI1EQTdtatWwc7Ozvo6OigY8eOuHr1alWXRERERNWAKMLOvn374OXlhYULFyIoKAhOTk5wd3dHQkJCVZdGREREVUwUYWfFihWYNGkSxo8fj2bNmmHjxo2oVasWtm7dWtWlERERURXTqOoCyisnJwfXr1/H3Llz5cfU1NTg5uaGy5cvF/uY7OxsZGdny3+WSqUAgNTU1IotloiIiFSm8HNbEIRSz6vxYScxMRH5+fmwsLBQOG5hYYG7d+8W+xhvb2989913RY7Xr1+/QmokIiKiipOWlgYjI6MS76/xYedNzJ07F15eXvKfZTIZkpKSYGJiAolEorLnSU1NRf369RETEwNDQ0OVtSuG9mty7TW9/Zpce01vvybXXtHt1+Taa3r7Nbl2QRCQlpYGa2vrUs+r8WHH1NQU6urqiI+PVzgeHx8PS0vLYh+jra0NbW1thWO1a9euqBJhaGhYIb9AYmi/Jtde09uvybXX9PZrcu0V3X5Nrr2mt19Tay+tR6dQjR+grKWlhXbt2sHPz09+TCaTwc/PDy4uLlVYGREREVUHNb5nBwC8vLwwduxYODs7o0OHDli1ahUyMjIwfvz4qi6NiIiIqpgows7w4cPx9OlTLFiwAHFxcWjdujWOHz9eZNByZdPW1sbChQuLXDJj+zW79prefk2uvaa3X5Nrr+j2a3LtNb39mlx7WUmE183XIiIiIqrBavyYHSIiIqLSMOwQERGRqDHsEBERkagx7BAREZGoMexUoHXr1sHOzg46Ojro2LEjrl69qpJ2z507h/79+8Pa2hoSiQSHDh1SSbtAwVYa7du3h4GBAczNzTFw4ECEhYWprP0NGzagVatW8sWlXFxccOzYMZW1/7IlS5ZAIpHA09NTZW1+++23kEgkCjdHR0eVtf/48WOMHj0aJiYm0NXVRcuWLXHt2jWVtG1nZ1ekdolEAg8PD5W0n5+fj/nz58Pe3h66urpo2LAhfvjhh9fuWVNWaWlp8PT0hK2tLXR1ddG5c2cEBga+UVuvew8JgoAFCxbAysoKurq6cHNzQ0REhMraP3DgAHr37i1ftT04OFhl9efm5mL27Nlo2bIl9PT0YG1tjY8//hhPnjxRSe3ffvstHB0doaenhzp16sDNzQ0BAQEqqf1Vn332GSQSCVatWqWy9seNG1fkPdCnTx+V1n/nzh188MEHMDIygp6eHtq3b4/o6Ohyt13c+1cikeCnn35SSe3p6emYMmUK6tWrB11dXfnG2mX1uvbj4+Mxbtw4WFtbo1atWujTp49S76vyYNipIPv27YOXlxcWLlyIoKAgODk5wd3dHQkJCeVuOyMjA05OTli3bp0KKlXk7+8PDw8PXLlyBb6+vsjNzUXv3r2RkZGhkvbr1auHJUuW4Pr167h27Rp69uyJAQMG4N9//1VJ+4UCAwPx66+/olWrViptFwCaN2+O2NhY+e3ChQsqaTc5ORmurq7Q1NTEsWPHcPv2bSxfvhx16tRRSfuBgYEKdfv6+gIAhg4dqpL2ly5dig0bNmDt2rW4c+cOli5dimXLlmHNmjUqaf+TTz6Br68vfv/9d4SEhKB3795wc3PD48ePlW7rde+hZcuWYfXq1di4cSMCAgKgp6cHd3d3ZGVlqaT9jIwMdOnSBUuXLlW69te1n5mZiaCgIMyfPx9BQUE4cOAAwsLC8MEHH6ik9saNG2Pt2rUICQnBhQsXYGdnh969e+Pp06cqab/QwYMHceXKldduA/Am7ffp00fhvbBnzx6VtX///n106dIFjo6OOHv2LG7duoX58+dDR0en3G2/XHNsbCy2bt0KiUSCIUOGqKR2Ly8vHD9+HDt37sSdO3fg6emJKVOm4MiRI+VuXxAEDBw4EA8ePMDhw4dx48YN2Nraws3NTWWfL6USqEJ06NBB8PDwkP+cn58vWFtbC97e3ip9HgDCwYMHVdrmyxISEgQAgr+/f4U9R506dYTffvtNZe2lpaUJDg4Ogq+vr9C9e3dh2rRpKmt74cKFgpOTk8rae9ns2bOFLl26VEjbxZk2bZrQsGFDQSaTqaS9fv36CRMmTFA4NnjwYGHUqFHlbjszM1NQV1cXjh49qnC8bdu2wtdff12utl99D8lkMsHS0lL46aef5MdSUlIEbW1tYc+ePeVu/2WRkZECAOHGjRtKt1uW9gtdvXpVACBERUWpvG2pVCoAEE6dOqVU26W1/+jRI6Fu3bpCaGioYGtrK6xcuVLptktqf+zYscKAAQPeqL2ytD98+HBh9OjRFdL2qwYMGCD07NlTZe03b95c+P777xWOvel77NX2w8LCBABCaGio/Fh+fr5gZmYmbN68Wen2lcWenQqQk5OD69evw83NTX5MTU0Nbm5uuHz5chVWpjypVAoAMDY2Vnnb+fn52Lt3LzIyMlS6tYeHhwf69eun8PevShEREbC2tkaDBg0watSoMnVPl8WRI0fg7OyMoUOHwtzcHG3atMHmzZtV0varcnJysHPnTkyYMEFlm9927twZfn5+CA8PBwDcvHkTFy5cQN++fcvddl5eHvLz84t8O9bV1VVZz1qhyMhIxMXFKfz+GBkZoWPHjjXu/VtIKpVCIpGofA/AnJwcbNq0CUZGRnByclJJmzKZDGPGjMHMmTPRvHlzlbT5qrNnz8Lc3BxNmjTB559/jmfPnqmkXZlMhn/++QeNGzeGu7s7zM3N0bFjR5UONSgUHx+Pf/75BxMnTlRZm507d8aRI0fw+PFjCIKAM2fOIDw8HL179y5329nZ2QCg8B5WU1ODtra2yt/DxWHYqQCJiYnIz88vsoKzhYUF4uLiqqgq5clkMnh6esLV1RUtWrRQWbshISHQ19eHtrY2PvvsMxw8eBDNmjVTSdt79+5FUFAQvL29VdLeqzp27Iht27bh+PHj2LBhAyIjI9G1a1ekpaWVu+0HDx5gw4YNcHBwwIkTJ/D5559j6tSp2L59uwoqV3To0CGkpKRg3LhxKmtzzpw5GDFiBBwdHaGpqYk2bdrA09MTo0aNKnfbBgYGcHFxwQ8//IAnT54gPz8fO3fuxOXLlxEbG6uC6l8ofI/W9PdvoaysLMyePRsjR45U2SaMR48ehb6+PnR0dLBy5Ur4+vrC1NRUJW0vXboUGhoamDp1qkrae1WfPn2wY8cO+Pn5YenSpfD390ffvn2Rn59f7rYTEhKQnp6OJUuWoE+fPjh58iQGDRqEwYMHw9/fXwXVv7B9+3YYGBhg8ODBKmtzzZo1aNasGerVqwctLS306dMH69atQ7du3crdtqOjI2xsbDB37lwkJycjJycHS5cuxaNHj1T+Hi6OKLaLoIrh4eGB0NBQlafuJk2aIDg4GFKpFH/++SfGjh0Lf3//cgeemJgYTJs2Db6+vmW6Pv4mXu6laNWqFTp27AhbW1v88ccf5f6GJZPJ4OzsjMWLFwMA2rRpg9DQUGzcuBFjx44tV9uv2rJlC/r27av0eIjS/PHHH9i1axd2796N5s2bIzg4GJ6enrC2tlZJ/b///jsmTJiAunXrQl1dHW3btsXIkSNx/fp1FVQvTrm5uRg2bBgEQcCGDRtU1m6PHj0QHByMxMREbN68GcOGDUNAQADMzc3L1e7169fxyy+/ICgoSGU9jq8aMWKE/M8tW7ZEq1at0LBhQ5w9exa9evUqV9symQwAMGDAAEyfPh0A0Lp1a1y6dAkbN25E9+7dy9X+y7Zu3YpRo0ap9N+6NWvW4MqVKzhy5AhsbW1x7tw5eHh4wNrautw95Zqamjhw4AAmTpwIY2NjqKurw83NDX379lXZJIbSsGenApiamkJdXR3x8fEKx+Pj42FpaVlFVSlnypQpOHr0KM6cOYN69eqptG0tLS00atQI7dq1g7e3N5ycnPDLL7+Uu93r168jISEBbdu2hYaGBjQ0NODv74/Vq1dDQ0NDJd/cXlW7dm00btwY9+7dK3dbVlZWRQJf06ZNVXaZrFBUVBROnTqFTz75RKXtzpw5U96707JlS4wZMwbTp09XWS9bw4YN4e/vj/T0dMTExODq1avIzc1FgwYNVNJ+ocL3aE1+/wIvgk5UVBR8fX1V1qsDAHp6emjUqBE6deqELVu2QENDA1u2bCl3u+fPn0dCQgJsbGzk7+GoqCjMmDEDdnZ25S+8GA0aNICpqalK3sOmpqbQ0NCo8Pfx+fPnERYWptL38PPnzzFv3jysWLEC/fv3R6tWrTBlyhQMHz4cP//8s0qeo127dggODkZKSgpiY2Nx/PhxPHv2TOXv4eIw7FQALS0ttGvXDn5+fvJjMpkMfn5+Kh2bUhEEQcCUKVNw8OBBnD59Gvb29hX+nDKZTH49tzx69eqFkJAQBAcHy2/Ozs4YNWoUgoODoa6uroJqFaWnp+P+/fuwsrIqd1uurq5FpvmHh4fD1ta23G2/zMfHB+bm5ujXr59K283MzISamuI/Kerq6vJvu6qip6cHKysrJCcn48SJExgwYIBK27e3t4elpaXC+zc1NRUBAQHV/v1bqDDoRERE4NSpUzAxManQ51PVe3jMmDG4deuWwnvY2toaM2fOxIkTJ1RQaVGPHj3Cs2fPVPIe1tLSQvv27Sv8fbxlyxa0a9dOZeOkgILfmdzc3Ep5DxsZGcHMzAwRERG4du2ayt/DxeFlrAri5eWFsWPHwtnZGR06dMCqVauQkZGB8ePHl7vt9PR0hW8hkZGRCA4OhrGxMWxsbMrVtoeHB3bv3o3Dhw/DwMBAPkbByMgIurq65WobAObOnYu+ffvCxsYGaWlp2L17N86ePauSf8gMDAyKjC3S09ODiYmJysYcffXVV+jfvz9sbW3x5MkTLFy4EOrq6hg5cmS5254+fTo6d+6MxYsXY9iwYbh69So2bdqETZs2qaDyAjKZDD4+Phg7diw0NFT79u/fvz8WLVoEGxsbNG/eHDdu3MCKFSswYcIElbR/4sQJCIKAJk2a4N69e5g5cyYcHR3f6D31uveQp6cnfvzxRzg4OMDe3h7z58+HtbU1Bg4cqJL2k5KSEB0dLV/7pvDD0dLSsky9R6W1b2VlhQ8//BBBQUE4evQo8vPz5e9jY2NjaGlpvXHbJiYmWLRoET744ANYWVkhMTER69atw+PHj8u8hMHr/m5eDWaampqwtLREkyZNyt2+sbExvvvuOwwZMgSWlpa4f/8+Zs2ahUaNGsHd3V0l9c+cORPDhw9Ht27d0KNHDxw/fhx///03zp49W+62gYLgvX//fixfvrxM9SrTfvfu3TFz5kzo6urC1tYW/v7+2LFjB1asWKGS9vfv3w8zMzPY2NggJCQE06ZNw8CBA1UyAPq1Kny+11tszZo1go2NjaClpSV06NBBuHLlikraPXPmjACgyG3s2LHlbru4dgEIPj4+5W5bEARhwoQJgq2traClpSWYmZkJvXr1Ek6ePKmStouj6qnnw4cPF6ysrAQtLS2hbt26wvDhw4V79+6prP2///5baNGihaCtrS04OjoKmzZtUlnbgiAIJ06cEAAIYWFhKm1XEAQhNTVVmDZtmmBjYyPo6OgIDRo0EL7++mshOztbJe3v27dPaNCggaClpSVYWloKHh4eQkpKyhu19br3kEwmE+bPny9YWFgI2traQq9evZT6O3td+z4+PsXev3DhwnK3XzidvbjbmTNnytX28+fPhUGDBgnW1taClpaWYGVlJXzwwQfC1atXVfZ38yplp56X1n5mZqbQu3dvwczMTNDU1BRsbW2FSZMmCXFxcSqtf8uWLUKjRo0EHR0dwcnJSTh06JDK2v71118FXV3dN/rdf137sbGxwrhx4wRra2tBR0dHaNKkibB8+fIyL0/xuvZ/+eUXoV69eoKmpqZgY2MjfPPNNyr79+F1JIJQCSODiIiIiKoIx+wQERGRqDHsEBERkagx7BAREZGoMewQERGRqDHsEBERkagx7BAREZGoMewQERGRqDHsEBG94uzZs5BIJEhJSanqUohIBRh2iIiISNQYdoiIiEjUGHaIqNqRyWTw9vaGvb09dHV14eTkhD///BPAi0tM//zzD1q1agUdHR106tQJoaGhCm389ddfaN68ObS1tWFnZ1dk48Ts7GzMnj0b9evXh7a2Nho1aoQtW7YonHP9+nU4OzujVq1a6Ny5c5HdrImoZmDYIaJqx9vbGzt27MDGjRvx77//Yvr06Rg9ejT8/f3l58ycORPLly9HYGAgzMzM0L9/f+Tm5gIoCCnDhg3DiBEjEBISgm+//Rbz58/Htm3b5I//+OOPsWfPHqxevRp37tzBr7/+Cn19fYU6vv76ayxfvhzXrl2DhoaGynZwJ6LKxY1Aiahayc7OhrGxMU6dOgUXFxf58U8++QSZmZmYPHkyevTogb1792L48OEAgKSkJNSrVw/btm3DsGHDMGrUKDx9+hQnT56UP37WrFn4559/8O+//yI8PBxNmjSBr68v3NzcitRw9uxZ9OjRA6dOnUKvXr0AAP/73//Qr18/PH/+HDo6OhX8t0BEqsSeHSKqVu7du4fMzEy8++670NfXl9927NiB+/fvy897OQgZGxujSZMmuHPnDgDgzp07cHV1VWjX1dUVERERyM/PR3BwMNTV1dG9e/dSa2nVqpX8z1ZWVgCAhISEcr9GIqpcGlVdABHRy9LT0wEA//zzD+rWratwn7a2tkLgeVO6urplOk9TU1P+Z4lEAqBgPBER1Szs2SGiaqVZs2bQ1tZGdHQ0GjVqpHCrX7++/LwrV67I/5ycnIzw8HA0bdoUANC0aVNcvHhRod2LFy+icePGUFdXR8uWLSGTyRTGABGReLFnh4iqFQMDA3z11VeYPn06ZDIZunTpAqlUiosXL8LQ0BC2trYAgO+//x4mJiawsLDA119/DVNTUwwcOBAAMGPGDLRv3x4//PADhg8fjsuXL2Pt2rVYv349AMDOzg5jx47FhAkTsHr1ajg5OSEqKgoJCQkYNmxYVb10IqogDDtEVO388MMPMDMzg7e3Nx48eIDatWujbdu2mDdvnvwy0pIlSzBt2jRERESgdevW+Pvvv6GlpQUAaNu2Lf744w8sWLAAP/zwA6ysrPD9999j3Lhx8ufYsGED5s2bhy+++ALPnj2DjY0N5s2bVxUvl4gqGGdjEVGNUjhTKjk5GbVr167qcoioBuCYHSIiIhI1hh0iIiISNV7GIiIiIlFjzw4RERGJGsMOERERiRrDDhEREYkaww4RERGJGsMOERERiRrDDhEREYkaww4RERGJGsMOERERiRrDDhEREYna/wEBwNi0U0qrFAAAAABJRU5ErkJggg==", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAG2CAYAAACZEEfAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAAB9HElEQVR4nO3dd1RUx98G8GfpRUB6kaqg2HvB3oIt9qiJJjHGWH5irLFgYkw0ETWJJsbE9ipqrDGxG3vBhgiKXSmKglQLLL3uvH8QNq60XYrg+nzO2XNk7p07c3evu987d4pECCFAREREpKY0qroCRERERJWJwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqbUqDXbOnTuH/v37w87ODhKJBPv27VPYLoTA119/DVtbW+jr66Nnz54ICwtT2OfFixcYNWoUjI2NUbNmTYwdOxapqamv8SyIiIioOqvSYCctLQ1NmzbFb7/9VuT2ZcuWYeXKlVizZg0CAgJgaGiIXr16ITMzU77PqFGjcOfOHZw4cQKHDh3CuXPnMH78+Nd1CkRERFTNSarLQqASiQR79+7FoEGDAOS36tjZ2WHmzJn44osvAABSqRTW1tbYtGkT3n//fdy7dw8NGjRAYGAgWrVqBQA4evQo+vbtiydPnsDOzq6qToeIiIiqCa2qrkBxIiIiEBcXh549e8rTTExM0LZtW/j7++P999+Hv78/atasKQ90AKBnz57Q0NBAQEAABg8eXOSxs7KykJWVJf9bJpPhxYsXMDc3h0QiqbyTIiIiogojhEBKSgrs7OygoVH8w6pqG+zExcUBAKytrRXSra2t5dvi4uJgZWWlsF1LSwtmZmbyfYri4+ODb7/9toJrTERERFUhKioK9vb2xW6vtsFOZfL29saMGTPkf0ulUjg6OiIqKgrGxsZVWDMiIiJSVnJyMhwcHGBkZFTiftU22LGxsQEAxMfHw9bWVp4eHx+PZs2ayfdJSEhQyJebm4sXL17I8xdFV1cXurq6hdKNjY0Z7BAREb1hSuuCUm3n2XFxcYGNjQ1OnTolT0tOTkZAQAA8PDwAAB4eHkhKSsLVq1fl+5w+fRoymQxt27Z97XUmIiKi6qdKW3ZSU1MRHh4u/zsiIgLXr1+HmZkZHB0dMW3aNHz33Xdwc3ODi4sL5s+fDzs7O/mIrfr166N3794YN24c1qxZg5ycHEyePBnvv/8+R2IRERERgCoOdoKCgtCtWzf53wX9aEaPHo1NmzZh9uzZSEtLw/jx45GUlISOHTvi6NGj0NPTk+fZtm0bJk+ejB49ekBDQwNDhw7FypUrX/u5EBERUfVUbebZqUrJyckwMTGBVCplnx0iIqI3hLK/39W2zw4RERFRRWCwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGptWod7OTl5WH+/PlwcXGBvr4+6tSpg0WLFkEIId9HCIGvv/4atra20NfXR8+ePREWFlaFtSYiIqLqpFoHO0uXLsXq1auxatUq3Lt3D0uXLsWyZcvw66+/yvdZtmwZVq5ciTVr1iAgIACGhobo1asXMjMzq7DmREREVF1IxMvNJNXMu+++C2tra2zYsEGeNnToUOjr62Pr1q0QQsDOzg4zZ87EF198AQCQSqWwtrbGpk2b8P777ytVTnJyMkxMTCCVSmFsbFwp50JEREQVS9nf72rdstO+fXucOnUKoaGhAIAbN27gwoUL6NOnDwAgIiICcXFx6NmzpzyPiYkJ2rZtC39//2KPm5WVheTkZIUXERERqSetqq5ASebOnYvk5GS4u7tDU1MTeXl5+P777zFq1CgAQFxcHADA2tpaIZ+1tbV8W1F8fHzw7bffVl7FiYiIqNqo1i07f/75J7Zt24bt27fj2rVr2Lx5M3788Uds3ry5XMf19vaGVCqVv6KioiqoxkRERFTdVOuWnVmzZmHu3LnyvjeNGzfG48eP4ePjg9GjR8PGxgYAEB8fD1tbW3m++Ph4NGvWrNjj6urqQldXt1LrTkRERNVDtW7ZSU9Ph4aGYhU1NTUhk8kAAC4uLrCxscGpU6fk25OTkxEQEAAPD4/XWlciqhqx0gxcevAMsdKMqq4KVSJnZ2dIJJJCLy8vLwBA165dC22bOHFiiccs6ngSiQQ//PCDwn6HDx9G27Ztoa+vD1NTUwwaNKiyTpMqSbVu2enfvz++//57ODo6omHDhggODsby5cvx6aefAsi/UKdNm4bvvvsObm5ucHFxwfz582FnZ8eLkegtsCswEt57bkEmAA0J4DOkMUa0dqzqalElCAwMRF5envzv27dv45133sGwYcPkaePGjcPChQvlfxsYGJR4zNjYWIW/jxw5grFjx2Lo0KHytL///hvjxo3D4sWL0b17d+Tm5uL27dvlPR16zap1sPPrr79i/vz5mDRpEhISEmBnZ4cJEybg66+/lu8ze/ZspKWlYfz48UhKSkLHjh1x9OhR6OnpVWHNiaiyxUozMHfPLRRMniETwLw9t9G5riVsTfSrtnJU4SwtLRX+XrJkCerUqYMuXbrI0wwMDOTdG5Tx6r779+9Ht27dULt2bQBAbm4upk6dih9++AFjx46V79egQYOynAJVoWr9GMvIyAg///wzHj9+jIyMDDx48ADfffcddHR05PtIJBIsXLgQcXFxyMzMxMmTJ1G3bt0qrDURvQ7XHifi1VnC8oTAo2fpVVMhem2ys7OxdetWfPrpp5BIJPL0bdu2wcLCAo0aNYK3tzfS05W/FuLj43H48GGFoObatWuIjo6GhoYGmjdvDltbW/Tp04ctO2+gat2yQ0RUlPTsXKw8XXhZGE0J4GxR8qMLevPt27cPSUlJ+OSTT+RpI0eOhJOTE+zs7HDz5k3MmTMHISEh2LNnj1LH3Lx5M4yMjDBkyBB52sOHDwEA33zzDZYvXw5nZ2f89NNP6Nq1K0JDQ2FmZlah50WVh8EOEb1R8mQCU3ZcR0hcKgx1NJGRkwfZvy08ozu48BHWW2DDhg3o06cP7Ozs5Gnjx4+X/7tx48awtbVFjx498ODBA9SpU6fUY27cuBGjRo1S6AJRMBjmyy+/lPfj8fX1hb29PXbv3o0JEyZU1ClRJavWj7GIiF7l8889nLwXDx0tDWwZ2xYX53ZH30b5fS9uRiWhGq+AQxXg8ePHOHnyJD777LMS92vbti0AIDw8vNRjnj9/HiEhIYWOWTClyct9dHR1dVG7dm1ERkaqWnWqQgx2iOiNsfXyY/zfhQgAwE/DmqKlkylsTfSxYEBD6GhqIOhxIgIiXlRxLaky+fr6wsrKCv369Stxv+vXrwOAwhxsxdmwYQNatmyJpk2bKqS3bNkSurq6CAkJkafl5OTg0aNHcHJyUr3yVGUY7BBRmZU298mECRNQp04d6Ovrw9LSEgMHDsT9+/eVPv7EiRMhkUjw888/wy/0KRYcuAMA+Li+Nv7v6//BwsICxsbGGNq3J1rrxgAAfjtT+p08vZlkMhl8fX0xevRoaGn91wvjwYMHWLRoEa5evYpHjx7hwIED+Pjjj9G5c2c0adJEvp+7uzv27t2rcMzk5GTs3r27yJYiY2NjTJw4ERMmTJBf2zo6OkhISMCECRMq5Tp/WWhoKAYOHCi/zjt27IgzZ84ofVz6D4MdIiqzwMBAxMbGyl8nTpwAAPncJy1btoSvry/u3buHY8eOQQgBT09PhflSirN3715cvnwZdnZ2SEjOhNe2a8iTCQxtYY+d309Gbm4uTp8+jatXr6Jp06bYu3QKRHoizoc9Q3BkYqWeN1WNkydPIjIyUj7XWgEdHR2cPHkSnp6ecHd3x8yZMzF06FAcPHhQYb+QkBBIpVKFtJ07d0IIgQ8++KDIMn/44QeMHz8e5ubmMDQ0RKdOnbB+/XoAFX+dv+rdd98tdJ2/++67Ja79SMUQJKRSqQAgpFJpVVeF6I02depUUadOHSGTyYrcfuPGDQFAhIeHl3icJ0+eiFq1aonbt28LB0dH4dxvknCac0gMX3NJRMfGCwDi3Llz8v2Tk5MFADH0yzXCac4hMXbTlQo9L8rn5OQkABR6TZo0SQghxPjx40Xt2rWFnp6esLCwEAMGDBD37t0r8Zh///23eOedd4SZmZkAIIKDgwvts3btWtGlSxdhZGQkAIjExMQyn0NMUrq4GP5UxCSll/kYlXGdOzk5iRUrVsi3PX36tNjr/MSJE2WuuyrehM9b2d9vtuwQUYUobu6TAmlpafD19YWLiwscHByKPY5MJsNHH32EWbNmobabO56lZkOakQMXC0Os/aglbK0tUa9ePWzZsgVpaWnIzc3F2rVrYWVlhXmj+0IiAU7eS8DdmOTKPN23UmW05KWlpaFjx45YunRpsfukp6ejd+/emDdvXrnqvyswEh2WnMbI9QHosOQ0dgWq3sm4Mq7zhg0bFtpubm6OevXqoW/fvvJHaMbGxgCAd955B15eXnjx4gU+//xz1KtXD/r6+nB0dMSUKVMKtV69SpllMgYMGICcnBzo6OjA0tISQ4cOxY4dOwC8OZ+3glICu7cCW3aIym/Xrl1CU1NTREdHK6T/9ttvwtDQUAAQ9erVK/Vud/HixeKdd94Rubl5YuIfQULT2ErY9Z4gHj5Nle8TFRUlWrZsKSQSidDU1BS2trbi2rVrQgghvLZdFU5zDolJ265W/EmSgopq4RBCiIiIiGLv9AucOXOmzC07MUnpwnnuIeE0579X7bmHVW7hqejrvOC9e7VlR4j867xJkyYCgNDQ0BBWVlZi9erVAoA4c+aMuHXrlhgyZIg4cOCACA8PF6dOnRJubm5i6NChJZYdGxur8Nq4caOQSCTiwYMH8n2WL18u/P39xaNHj8TFixeFh4eHsLGxqXafN1t2iN4SJXUSLuud3zfffAN3d3cYGhrC1NQUPXv2REBAQInljhgxAm5uboX6HowaNQrBwcHw8/ND3bp1MXz4cGRmZhZZ7tWrV/HLL79g06ZN+OF4KI7cjoNEAgxv5QgXC0MAgBACXl5esLKywvnz53HlyhUMGjQI/fv3R2xsLLy6uQIA/rkViwdPU8v6tlIpKqqF43WJeJZWITNuFzXHD1D267yo9w747zqvVasWLly4gMDAQAwdOhRffPEFnJ2d0aVLFzRq1Ah///03+vfvjzp16qB79+74/vvvcfDgQeTm5hZ7DjY2NgqvV5fJAIDp06ejXbt2cHJyQvv27TFz5kzExcVh9OjRb8TnXUip4ddbgC079CZLSEhQuEs7ceJEue/8tm3bJk6cOCEePHggbt++LcaOHSuMjY1FQkKCfB8nJyexcOFCERsbK65cuSI0NDTEzp07SzxuVlaWMDAwENu3by9y+4oVK4REIhEampoCEo381793tU5OTkIIIU6ePCk0NDQK/X91dXUVPj4+Qgghxm4KFE5zDokZu66X9vZRGVVUC0eBym7ZiU5MU2jVKUvLzqNHj4SGhobYt29fifspe51ramrKX8pc51lZWUJDQ0N4enoWW/b69euFhYWF0ucUFxcntLS0xLZt24rd5/nz58LDw0MAqHafN1t2iN4SlpaWCndphw4dki+QWNY7v5EjR6Jnz56oXbs2GjZsiOXLlyM5ORk3b95U2M/IyAg2NjY4fPgwrKysFFaLLooQAkIIZGVlFbn9o48+wuZDfrAbsxK2Y1biizX7YWdnh1mzZuHYsWMAIF/vSEND8etLQ0NDPuPt5O75rTv7rkcj6gXXyqoMFdHC8TrFJRe+5ub2cVdpxm1l5/hR5jq/efMmrl+/Ln8pc53v27cPMpkMzZs3L/K4z549w6JFixRmky5NUctkFJgzZw4MDQ1hbm6Ou3fvwtPT8435vF/FYIdIjZT2aAEApFIpjI2NFeYpKe2Y69atg4mJSaFJ15YsWQIzMzMsXrwY7u7uCtsePnwIHx8fXL16FZGRkbh06RKGDRsGfX199O3bV77fy3OfJObpYNmVdGiaO2HYO+2xbFw/aGtrw8bGBvXq1QMAeHh4wNTUFKNHj8aNGzcQGhqKWbNmISIiQv4j1MyhJjq5WSBPJrDG74Fybx4praRZjE1MTODm5obOnTvjr7/+wv379wvNbVMVDt2IBQB4NrBGPesaAIC07OID/lcVN8dPWa5zc3NzNGrUSOGlzHU+Z84cSCSSIofJJycno1+/fmjQoAG++eYbpc+rqGUyCsyaNQvBwcH4448/IJVK8fz580IzlFfXz/tVDHaI1EhRCyS+TJU7v0OHDqFGjRrQ09PDihUrcOLECVhYWMi3T5kyBTt37sTixYuRk5ODq1evYvbs2fLtenp6OH/+PPr27QtXV1eMGDECRkZGuHTpEqysrOT7Fcx98iw1C2M2BSIlMxetnEyxdGiTIgM2CwsLHD16FKmpqejevTtatWqFCxcuYP/+/QrB2OR/++7sDnqCOGn1u9N8k1VUC8frIpMJ/HMrP9gZ1soBk/69NnZeiUJunkypYxQ3x4+q17myXr3OW7RogUePHuGrr74qdNORkpKC3r17w8jICHv37oW2trZSZRS3TMbLdahbty7Cw8NhZWWFq1ev4vLly8Uer7p83kVS6uGammOfHVIXnp6e4t133y1ym1QqFW3atBG9e/cW2dnZpR4rNTVVhIWFCX9/f/Hpp58KZ2dnER8fX+z+GzZsEFpaWiIzM1OlOsckpYsz9+NFv1/OCac5h0SnpafFsxTVjlGc91ZfFE5zDomFB+9UyPFIiLy8POHo6CjmzJmjkP7gwQOxePFiERQUJB4/fiwuXrwo+vfvL8zMzBSum3r16ok9e/bI/37+/LkIDg4Whw8fFgDEzp07RXBwsIiNjZXvExsbK4KDg8X69evlc88EBweL58+fK1XnwIjnwmnOIdHo66MiIztXZObkihYLjwunOYfEkVuxpR+ggpRnjp8FCxYIGxsbkZOTo5AulUpFu3btRJcuXURaWppKxxw9erRo2bJlifsUfN7/+9//5H0Bhag+n7eyv98MdgSDHVIPJXWeTE5OFh4eHqJHjx4iIyOjTMd3dXUVixcvLnb77du3BQBx//59pY+588pj4fLScOB6X/4jwuJTylS/opwNScg/7lf/VFgA9bY7duyYACBCQkIU0qOjo0WfPn2ElZWV0NbWFvb29mLkyJGFrgcAwtfXV/63r69vkRPXLViwQL7PggULitzn5eOUZMH+28JpziExfWewPG3JkXvCac4h8eH/XVb1LSiTl691l7mHxM4rj5XOW1yAKZVKRdu2bUXjxo1FeHi4wkCF3Nxc+X6vBhwFeQ0MDMTq1asLlXf58mXx66+/iuDgYLFlyxYBQDRv3lzUqVNHfjNTXT5vBjsqYLBD6qAy7vxeVrt2bYUvpFdt3bpVaGhoiBcvXih1vIinqcL5ldExLnMPlWtm21fJZDLR/9fzwmnOIbH0SMkzu74pSpvVNiMjQ0yaNEmYmZkJQ0NDMWTIEBEXF6f08SdMmCAAFJrzpahyC0a/qaoiZjFWVl6eTLT+7oRwmnNInLz73/sQ+TxNPu/Oy3M4VYaYpHSFoF7VkWDFBZgFo5WKekVERMj3KypQWLt2rdDX1xdJSUmFyrt586bo1q2bMDMzE7q6usLZ2Vl8/OlnYt+Fm2X6zCrz81b291u5HopEVK3JZDL834aN6DlgGJ6m5cDWJP+/dnJyMjw9PZGeno6tW7ciOTkZycn5MwtbWlpCU1MTQH7nSR8fHwwePBhpaWn4/vvvMWDAANja2uLZs2f47bffEB0dLZ851d/fHwEBAejWrRuMjIzg7++P6dOn48MPP4SpqWkJ9RQIiHiBfcHROHAjGuLV7QJ49CxdpREyJZFIJPDq5ooJf1zFFv/HmNC5DkwMlOvPUF0FBgYqzFB7+/ZtvPPOO/LPZvr06Th8+DB2794NExMTTJ48GUOGDMHFixdLPXZJ6zQBwMKFCzFu3Dj530ZGRirXf3vAY3y59zYEAA0J4DOkMUa0dlT5OMoKepyIhJQsGOlpoaPbf33OHMwM0K2eFU7fT8C2y4/x1bsNKq0OEc/SICtmjh9lrnVPT89CHYMBoGvXrkWmv0oIgVhpBi49eAYXC0PYmuhj/PjxRfbdE0LAvnY9rN2xH09Ts/AsNRsn7sTh0M1Y+B2MhORgJLrWs0QDO+NSywWAuzHJOBvy9LV93sVhsEOkBr76bTuin0ThtGiEDktOy79Qrl27Jp8M0NXVVSFPREQEnJ2dAeR3ngwKe4J20gyY6mri/v372Lx5M549ewZzc3O0bt0a58+fl09rr6uri507d+Kbb75BVlYWXFxcMH36dMyYMaPI+oXGp2BvcDT2B0cjpoTOwpoSCZwtDJQ+7+joaMyZMwdHjhxBeno6XF1d4evri1atWgEA4uPjsX3ZXMTtO4Ts9FS0+6cNDmzfADc3t2KP2bVrV/j5+RVK79u3Lw4fPgwg/wdh+vTpWLN2LbIyM6GhoQFXV1ds27ZNoew5c+bg+PHjSEpKQufOnfHrr7+Wu2wLCwssWLAA69evR1JSEiwtLeHo6IguXbpAKpViw4YN2L59O7p37w4gvzNx/fr1cfnyZbRr167E9/Lzzz/HsWPHiu14XDDVQFnFSjPkgQ6QH9zO23MbnetaVliA+6pDN2MAAJ4NbKCrpamw7cN2jjh9PwG7rz7BF73qQU9bs6hDlJuFoW6hNA0JVLrWy2NXYCS899yCTAASCTCilQNcrWrkBzMp2XiWmiV/PU/NRu6rkdlLBIAzIU9xJuSpyvV4HZ93cRjsEL3hYpLSsSPaFE5zDgHI/0KZ8/ctbLr0GLVqGmLOXzdgUUMXFjV0YGmkB4saOrAw0oVpDV0IIfBnUBRc5h7CthfAjn8DpT179pRYZosWLUoclQEA8cmZOHA9BnuDo3E39r91qoz0tNCvsS0GNa+FiGdp+GrvbeQJAU2JBIuHNFL6SzAxMREdOnRAt27dcOTIEVhaWiIsLEzesiSEwKBBg6CtrY3vf9+C5WejkBB8AD169MS9e3dhaGhY5HH37NmD7Oxs+d/Pnz9H06ZN5S0nQP4M0ytXroSuQ2OYN3kHabdOIv5ZDAwMDAqVvX//fhgbG2P58uXo2bMn7t4tX9nLli3DypUrsXnzZtjb26N9+/YwNDREVlYWrl69ipycHPTs2VO+v7u7OxwdHeHv719ssFPaOk0FlixZgkWLFsHR0REjR47E9OnTlZ7CAADO3E8o1JqnSguHqvJkAv/cyl8h/N2mtoW2d6lrBXtTfTxJzMDBGzEY1qpyZv7dez26UJqbVQ3YGBce7l3RYqUZ8kAHAIQAdgZGlZrPWE8LFka60NHUwP24lELbezeyLvUzi03KwNE78Qpplfl5l4TBDlEFUKaFQdW7/D179mDx4sUIDw9HTk4O3NzcMHPmTHz00UcAgJycHMz1nocNO/Yg+Wk0NHQNoefUFDW7fAItI3Pci03GvdiSF8PU1pAg56W7OJkA5v59C7FJmahtVSM/QKqhC4sauqhpoF3s3D2x0gxEPEuDlZEebkQlYd/1aFwMfyb/gtXWlKBrPSsMbl4L3d2t5HfQ7Wqbo2s9Szx6lg5nCwOVvgCXLl0KBwcH+Pr6ytNcXFzk/w4LC8Ply5dx+/ZtuNdvgEOxfnhoNhGJ6z7Bjh07ih1ua2ZmpvD3zp07YWBgIA84hBD48ccfoWlsBesPFgMADFzb4MmvH8LvchAaNGigUHZB8LB69WrY2NiUu+yff/4ZX331FQYOHIg///wTeXl5SEtLk084p6Ojg5o1ayocx9raGnFxcSW+l1paWpgyZUqx+0yZMgUtWrSAmZkZLl26BG9vb8TGxmL58uXF5nlZTp4Mmy49KpSuamueKq5EvMCz1CyY6GujQx2LQts1NSQY2dYRy46GYGtAZKUEOwnJmfC9GAEAWDq0CTQ1AO+/byEkPhV7g6MxpIV9hZf5socJhR+hAYBHHXO42xjBooZu/v9xI51/b4p0YV5DR94KFivNQIclpxWOoSmRYEH/hqUHO9IMHL8bXyjv62rRehmDHaJyUqWFQZW7fDMzM3z55Zdwd3eHjo4ODh06hDFjxsDKygq9evVCeMxzbD54FtqthsHWygWyzFS8OLUOT/csQq1PfobP4MbIFaJQM/Wz1Gw8S8lCSlauQqBTQAD4+VRYoXQtDQnMa+R/IVoa6cq/GKNepOOfW7GF7tgBoKWTKQY3r4V+jW1haqhT5HnamuiX6S7vwIED6NWrF4YNGwY/Pz/UqlULkyZNkvcpKZjrQ09PD5oaEvyvax3M/usmMmWa8Dt3vtiA41UbNmzA+++/L/+cHj58iPT0dBg2aIOn+3yQGXUbmjXMoWlsiTPnLuB/n36sUHYBDQ0N6Orq4sKFC2UuOyIiAnFxcfKWm4JZjJOTk+Hv74+2bdsqddyXFazTdO3atWKDWQAKjyibNGkCHR0dTJgwAT4+PtDVLfyY5lXrzj1EaHwq9LU1kZWbJ/8BnOlZt9Lu8g/fyn+E1auhNXS0ip5WbngrB6w4EYobUUm49USKxvYmFVqHX0+HIzNHhhaONTG8lT0kEgnik7Pww7EQfHvwLjq6WcDKqPJaeC4+eFYoTVMiwfLhTZV6321N9OEzpDHm7VG9BbY8eSsagx2iclKlhUGVu/yuXbsq/D116lRs3rwZFy5cQE23Vpi07RZqDP4G5oY6eK+lPf7vfATwzkTEbZmBme3NMKJNyZ0AM3PycDdGiqFr/BUWSJQA6OZuhbSsXHlwJM3IQa5MID45C/FFTLv/qnGdXPBRO2c4mlfeHdzDhw+xevVqzJgxA/PmzUNgYCCmTJkCHR0djB49Wv74xtvbG2vXrkW/hpaYNX8fcpKf4npIhFJlXLlyBbdv38aGDRsAAJHP0+G14QwAIC3kIoxbD4K1x3BkxYbhxfHfccrvImKSMgqVbWhoiBUrVuDJkyeIjY0tU9kA5K0z1tbW8lmM9+zZg23btiEuLg42NjbIzs5GUlKSQutOfHx8sX1tzp8/j4SEBDg6/ne95OXlYebMmfj555/x6NGjIvO1bdsWubm5ePTokXzW3+KEJ6Tgl38D6O8GNUJ7V3NM+OMqbj6R4kVadol5yyo3T4ajt/Pfr35Niu5wDQAWNXTRt7Et9l+PwdbLj7H0vSYVVofHz9Ow40okAGB2b3d5MDm+c238cysWd2KSsWD/Haz+sGWFlfmyq48Tse7cQwD5fXWEQJkCjhGtHdG5btlaYMuTtyJxBmWicjpw4ABatWqFYcOGwcrKCs2bN8f69evl20u7y1eGEAKnTp1CSEgIsi3rYdT/BeB5WjYa2hnjwOcd4d23Pi7M7QbvHk6QSCT4qEvx/S4K6GlrooWTGZYMaQzNf7+ENSUSLBnaGBs/aY1dEzxwamZX3FjgidDv+sDfuzsOTO6AjZ+0wrKhTTCrVz30bmhd5LG7u1tXaqAD5PczadGiBRYvXozmzZtj/PjxGDduHNasWQMA0NbWxp49exAaGgozMzOYGNWASWIo9Gq3xJOkTGTnlj5z7oYNG9C4cWM0b9kKa/wewPNnP1yPTAIAWDi4wrzraOhY14Fxs97QqmmNpLgovLPcD1uvPMHuv/6Wl21gYIAzZ86gT58+hdb0Kq3sNm3aFLm9qFmMW7ZsCW1tbZw6dUqeFhISgsjISHh4eBR5HGXWaSrK9evXoaGhoTBLcFHyZAKz/7qJ7FwZutazxJAWtWBroo/pPesCAP4MikK6Css2KCv/EVY2ahpoo30d8xL3/bCdEwBg/41oSDNyKqwOK06EIlcm0LmuJdrV/q8O2poaWPZeE2hpSHDkdhyO3FIuAFaFND0HU3YEI1cm8G4TW1yc0w07xrXDhbndyjQaytZEHx51zMsUrJQnb0Vhyw5ROanawqDKXb5UKkWtWrWQlZUFTU1NdB/rjV0xpgAE+je1w7KhTaCvk/9s3VRXgs2/fI8PPvgAxsbKDQsFlLvz0tHSKPJxU1U+k7e1tUWDBorDhevXr4+///5b/nfLli1x/fp1SKVSZGdnw6imGSxqN0KuSR3sDX5S4pd+Wloadu7ciQnTvdH/1wvyTpqt67vgEICOzerh97nd5e+bxy59PElLRFp2Hr49eBdN7U2w7R8/2Bvmry9maWmJtm3byvtxlaSg7IULFyqkF7TOxMbGKkw1EB8fj2bNmsHExARjx47FjBkzYGZmBmNjY3z++efw8PBQ6Jz88lQD5ubmMDdXDAZeXaeprFMNAMDmS49wLTIJNXS1sHhwY3nrRpe6lnA2N8Cj5+nYGxyNUW2dSn1fVHHo3wCid0MbaGuWHGC2cjJFPWsjhMSnYM+1JxjTwaXE/ZVxLzYZ+2/kP0ab3atwy1dDOxNM7FIHq86EY/7+O2hX27zYR72qEkJg9t83EJ2UAUczA/gMaQwjPW3Y1Xz9fWWqC7bsEJWTqi0MqtzlGxkZ4fr16zh29gLq9huHoxt+QGbkTczt446V7zeTBzo5OTkYPnw4hBBYvXq1yudQ1juvgmfyL7cMva5n8h06dEBISIhCWmhoKJycCv9ompiYwNLSElGPHiI9JhQGbm2x+uyDEtdF+mP7TqRlZGJnkhPux6WgpoE2fnivCfZ7D4aenh5u3Lghf98MJTl4EvUYtZ0c8N2gRjDS1cKNJ1IMWHURv1+MgaGJKcLCwhAUFISBAweWem67d+9GVlYWPvzwQ4V0FxcX2NjYYNaiH+VTDXh8ewj+lwPkLTcrVqzAu+++i6FDh6Jz586wsbEpNLouJCQEj2Of4tKDZ4iVZpRan4KpBrp06YKGDRvi+++/x/Tp07Fu3boS80U+T8cPx/I/o7l93GFX87/rQkNDgo88nAHkB0TKzBejrJcfYb1bwiOsAhKJBB965F83Wy8/rpC6/HgsBEIA/ZrYolGtovsBfd7DFa5WNfAsNQuLDt0td5kFtvg/xrE78dDWlGDVyOYw0nuz55aqEBU+neEbiDMoq4cnT56IUaNGCTMzM6GnpycaNWokAgMD5dtRzEyjy5YtK/aYubm54quvvhLOzs5CT09P1K5dWyxcuFDIZDL5Po6OjqJZs2bCxsZG6OnpiR49eohvvvlG2NnZFTpeUlKSSEhIEEII0aZNG/mstyW5EZUo2n5/UjjNOSRMm/cSrTp0VdienZ0tBg0aJJo0aSKePXtW6vEqQ0xSurgU/uy1zIhb4MqVK0JLS0t8//33IiwsTGzbtk0YGBiIrVu3yvf5888/xZkzZ8SDBw/Evn37hJOTkxgwaLBo9u0x4TTnkNgX/ER89NFHYu7cuQrHPnY7Vhg5NRIG7p3kywy8vNyEl5eXACA+/PBDcejQIdGyZUshkUjExo0bhRBCxEkzhOfnS4T1B4uF3YT/E+4fLRTWdg5iyJAhCuUUVbYQQnTs2FGMGDFCIS0vTyZuPUkSvcfMEBq6hsJyyHxh++kqoe/WTmiZWIsuPsfEqPWXxdQd18Sig3fE6rPh4q+gKHE2JEHcjk4S8dIMkZObJ4Qo39IFypLJZOKDdf7Cac4hMWLtJZGXJyu0jzQjW9Sff0Q4zTkkLoY9rbCyz4XmLxPSfOFx+TmXJiUzRzQoqEt4+epSsBZXbe/D4kFCycufXH38Qj6T8+l7xa89p6xbT5KE27x/hNOcQ2LD+YflPl51xxmU6a1S2ogoAIUeGR05cgRjx47F0KFDiz3u0qVLsXr1amzevBkNGzZEUFAQxowZAxMTE/kwXVNTU9y6dQt///03XFxcMH/+fCxfvhz169cvdDwTk/w7vIK7/EWLFpV4XnuuPcHcPbeQnStDHUtDmLlb4mnMf3NkFLTohIWF4cyZM4UeR7wuZR1RVR6tW7fG3r174e3tjYULF8LFxQU///wzRo0aJd8nNjYWM2bMQHx8PGxtbfHxxx9j/vz5WHfhMX48HorfzoQjOzJS3sIWJ83EggO3cejcVaQ8vo1GY5fh97Ft0MnNUqHsX3/9Fc+ePcPOnTuxdetWGBgYYOHChRgzZgwAwNpYD/1c9RG8eykSniYg3tAUNRp2h82gGXiakgVLo/zRS5EvlV0gJCQEFy5cwPHjxxGdlIELYU9xPuwZLj14jhdp2RCW3WDUMgbPj/0KWWYa9OwbwGr4QjxKysGjpMIjb14mkQAmetpIeqlfSmVN9LYzMAqXHjyHnrYGlg5tAg2NwiO9jPW0MbSFPf64/BibLj1Ce9fCw8PL4vDNfx9hNbKBVimPsArU0NXCoOa1sC0gEtsuR6J9EUPVlSGEwLKj+a1Zw1vZo7ZljRL3b+Foik87uGDDhQjM23sLx6d3LnNLTGpWLiZvv4bsPBl61rfGmA7OZTqOOpIIUYFth2+o5ORkmJiYQCqVqtTXgaqPuXPn4uLFizh//rzSeQYNGoSUlBSFzpyvevfdd2Ftba0wImbo0KHQ19fH1q1bIYSApaUlEhMTsWjRIgwfPhxnz57FuHHj4OXlhVWrVgHIfyxRMNPtrVu3MHXqVLRs2VKhf8nHH3+MWrVqwcfHB7l5MvT5dCZuZppDy9QW7RyN0FLjERbM/xKrV6/GZ599hpycHLz33nu4du0aDh06BGvr/zoLm5mZQUenYp7/qyNpRg46LjmNlKxcTO9ZF0Nb1sLp+wlYdjQEqVm50NKQYFzn2pjS3U3+qLCs0rJy8dPxUGy6FAGZyJ+sbV7f+hjeygHxKZmIeJYmn8I/JTMHlx++yA9wwp/h4dM0hWMZ6miiuWNNXAx/rjDUX0MC/DS8GWQyoTjFQGoWnqbk//tFWlaR860U2DGuHTxK6cirrFhpBjyXn0NKVi6+6lcfn3WqXey+4Qkp6Ln8HDQkgN+sbnAwK1+/kpw8GVp/fxJJ6TnY/llblQKoe7HJ6PPLeWhpSHBpbndYlWHSvzMhCRjjGwgdLQ34zeqqVACZkZ2HXj+fQ+SLdIxq64jvBzdWuVwhBKbtuo7912NgZ6KHf6Z2Qk0D9f8OUPb3my07pBZKm3PlVfHx8Th8+DA2b95c4nHbt2+PdevWITQ0FHXr1sWNGzdw4cIF+URqEREReP78OX755ResX79e3sLg5uYmX3cKKL6F4WUFd/lJ6dn4fEcwroTFIv3+LiDtBc4YGiDW3R1bt27FiBEjAORPZHjgwAEAQLNmzRSOdebMmUJD1+k/JvraaO1shtMhCVhxMhQrTobKtzV3rAmfIY3hblMxNz6Gulr4un8DDGpuB+89t3AnJhlz99zCWr+HePwif8I3CQBHMwM8ScpA3ksRiYYEaOpQE51cLdCpriWaOdSEtqYGdgVGFpq7ZHDzWiXWI08mkJiejXuxyfh445VC0w04mVdMq44QAl/tvY2UrFw0c6hZamdfVysjdHKzwPmwZ9h6+TG8+xZuEVXFpQfPkZSeA4saOmjjYlZ6hpfUtzVGKydTBD1OxM7AKEzpUfykn0WRyf5r1fmkvbPSLWX6OppYMrQxRq4PwLaASLzbxE7lwHN30BPsvx4DTQ0JVn7Q/K0IdFTBlh2wZUcdFAzrnjFjBoYNG4bAwEBMnToVa9aswejRowvtv2zZMixZsgQxMTEKQ8JfJZPJMG/ePCxbtgyamprIy8vD999/D29vbwDApUuX0KFDB8TExMDW9r/p6IcPHw6JRIJdu3YpfQ6x0gycD3uGn0+GIiYpE/ramvhxWFP0a1J4mnsqn6JmhQWALzzr4n9dXaFZxCOXipCbJ4PvxUf46XgIMosZ+u5sboCObhbo6GoJjzrmMNEv+pFGrDSjzHOX5AdLt5D30vmPauuIhQMblfvc91+PxtSd16GjqYFDUzqirnXpi4WevBuPz7YEwURfG5e9e5SrNW32XzfwZ9ATfNjOEd8NUr2FZF9wNKbtug5bEz2cn91N6cdgAHDgRgym7AiGka4Wzs3upvLoqnl7b2F7QCSczA1wdGpnpd+HsPgU9F91AZk5MszqVQ9e3VxLz6Qm2LJDbxWZTIZWrVph8eL86fubN2+O27dvFxvsbNy4EaNGjSox0AGAP//8E9u2bcP27dvRsGFDXL9+HdOmTYOdnV2Rxy2rLZceYcGBO/JHE6YG2tj2WTulVxYm1RS1CjUAtHQyq7RABwC0NDUwrnNtWBnpYuqu64W2r3y/GQY0K7mFpkB5+km9PN1A0OMXWH4iFNsCIvEsNQu/vN+8zAtiPkvNwjcH7gAAPu/uqlSgA+RPYuloZoDIF+nYdz0aH5QyIWZxsnNlOPbvWkzKjMIqSp/GNlh4SAex0kycvp8Az4bKLXyakyfDT8fzW3XGda5dpmHk3n3cceZ+Ah4/T8dPx0OUWok9IzsPXtuvITNHhk5uFvhflzoql/s24NBzUgvFzbkSGRlZaN/z588jJCREqSn7Z82ahblz5+L9999H48aN8dFHH2H69Onw8fEB8N+8J/HxiovdlTRjbYHcPBn8Qp9iwpYgfP1SoAPk9ykxNeRw0criYmGIV2Oa17lmT5vaZkWW31rFxy7lUTBs/vPubvhtZAvoaGrg2J14fLzhCqTpZZtYb8GBO0hMz0F9W2NM7Kr8j66mhgQf/zv0uzzD0C+GP4M0IweWRrpo7Vy291JXSxPD/10j64/Lj5XO92dQFB4/T4e5oQ7GdizbPD1GetpY/G9/nY0XI3AtMrHUPAsP3UFofCosauhi+fBmRXYEJwY7pCZUmXNlw4YNaNmyJZo2bVrqcdPT0wuNltHU1IRMlv8IomDek5c7OScnJyMgIKDIGWuFELj1RIqFB++inc9pjN54BcfuxhfaTyaAR8/SS60flU1Vzg9UHcp/Vd/Gttj8aRsY6WrhyqMXGL7WX6n5d1527E4cDt+MhaaGBD+816TUifxeNayVA/S1NXE/LgWXH75QKW+BQ/+OwurbyKZcLXSj2jpCIgHOhz3Do2dppe6fkZ2Hlf8uhzG5uysMdcv+0KSbe/6CuTIBzP7rJrJy84rd98CNGOy4EgWJBPjl/WbyUX5UGB9jkVqYPn062rdvj8WLF2P48OG4cuUK1q1bV2jSs+TkZOzevRs//fRTkcfp0aMHBg8ejMmTJwMA+vfvj++//x6Ojo5o2LAhgoODsXz5cnz66acA8icjmzZtGr777ju4ubnJh57b2dlh0KBB8uNGvUjHgRsx2BscjfCEVHm6qYE2urtbYU9wtEKH0apaGfhtUtVr9lR1+a/yqGOOPyd6YPTGKwiJT8HQ3y9hy9g2cLUq/VGUND0HX+27DQCY0Ll2sZPolcREXxtDWuQP/d586ZHKHXSzcvNw/G7pa2Epw8HMAF3qWuJsyFNsvxKJeaV0mt7s/wjxyVmoVVMfI9uW7RHcy75+twHOhz1FeEIqVp0Ox0zPwjMwP3qWhnl7bgEAJndzRYcKGravrhjskFpQZs4VANi5cyeEEPjggw+KPE5oWDiCQyMRK82ArYk+fv31V8yfPx+TJk1CQkIC7OzsMGHCBHz99dfyPLNnz0ZaWhrGjx+PpKQkNG7ZFlt270OWTBN7r0Ri77VoXHn0352qrpYG3mlgjcHNa6FzXUtoa2qgjYtZtVgZ+G1TFfMDVafyX1Xf1hh7JrXHxxuv4OHTNAxd7Y+Nn7RCS6eSHwl9d/gunqZkoY6locojmF42ur0ztgVE4vjdOEQnZaBWTeXfmwthz5CSmQsrI120cip5CQtlfNTOCWdDnuLPoCjMeKdusf2YpBk5WH32AQBg+jt1oatVvqkKAMDUUAcLBzbCpG3XsPrsA/RpZKvQfy8rNw+Td1xDalYu2jibYWo53vO3BUdjgaOxKN+uwEh477kFmcgf8uszpLFKC+a9nB/Ib53J+/e/l0QCeNQ2x6DmtdC7kQ2Mi5g0rDyja4gq0ou0bHy6KRDXo5Kgq6WBVSNb4J0GRS/66hf6FKM3XoFEAvw10aPUwKg0I9dfxqUHzzGxSx3M7eOudL4Zu65jT3A0PmnvjG8GlL4QbmnyZAKdl51BdFIGfhrWFENb2he53w/H7uO3Mw/gZlUDR6d1rtAO7hP/uIqjd+LQ0M4Y+7w6yB8NfnvwDnwvPoKpgTb+mdrprf6+4GgsIiVl5uTh6O04zP37lryTsEwAc/6+BZ9/7ivV4U8mEwqz0gJAnhCoY2mI4a0cMKCZXalfSNXtLp/eXmaGOtg+ri28tl3DmZCnmPBHEBYPboz3XxkllZqVK3+UMtrDudyBDpA/P82lB8+xMzAS03q6KTUyLDMnDyf+7fvWv2nFTNWgqSHByLaO+OFYCLYGPC4y2ElIycTGC48AAF/0qlfhI/kWDmoI/4fPcScmGevOPYRXN1ccvxMH34v5Zf40vCm/M5TEYIfeOjKZwN3YZJwPe4YL4U8R+CgR2cXMefJqAKOq7wY1rrBZaYleJwMdLaz7uBW899zCX1fzly15mpKFyd1d5SuXLzt6H9FJGXAw08fs3oX7lZRFj/rWsDfVx5PEDBy4HoPhrR1KzXMu9ClSsnJha6KH5g7lf4RVYHgrB/x8MhTBkUm4HS0t1Bfpt9PhyMjJQzOHmvAspuWrPKyM9PD1uw0wc/cN/HwyFJoaEvx2JhwA8FlHF3R3r/gy1RWDHVIrsdIMhen3C8QkZeBC2DOcC3sqX2PoZVZGukhIyVJI05AAW8e2VWqEw9OULHy4IUBh7hZ2MqY3nbamBn54rwmsjXXx25kH+OlEKOJTMjGxSx0cvxOPLf75Q7OXDGkCA52K+TnR1JDgo3ZO8DlyH76XHmFYK3t5cFWcw7f+HYXV2LZCh15bGumidyNbHLwRg20Bj+EzpIl8W9SLdGy/kj+1xeze9UqtY1kNaVELa889QGh8KpYcuQ8AsDfVx+zeyj/iIwY7pEZe7XPzYTsnSIBi1xjyqGOOjq4W6OhmiTqWhvgzKKpQJ2Fl19VxszaCz5DG7GRMakcikWBWL3dY1tDFt4fuYuvlSGy9/N/8Va2dTSt8JNCI1g5YcTIU92KTEfgoscRlHzJz8nDy30dYlTHb+IdtHXHwRgz2BcfAu299eX+7FSdCkZMn0MnNosyLhiojLjlTYQQnkH/z9jwti98vKmCwQ2ohVpqh0DlYJiC/6wSKX2PoZeUdClzdhhITVaRPOrhAU1OC+fvuKKRffZwoH71YUWoa6GBw81rYcSUKmy89KjHYORvyFGnZeahVUx/NHWpWWB0KtHExQ13rGgiNT8Xea9EY3d4Z9+OSsfd6NABgVq+KeXxXnKJm+y6Yh4vfMcpjsENqobjp/3vWt8J7LR1KXGPoZeXtJMxOxqTO6ljWKJRWWT+8o9s7Y8eVKBy9E4eYpAzYFTMM/b9HWDaV8ihJIpHgw3ZO+Hr/HWy9/Bgfezjhx2OhECK/zCb2NSu8zJcVzPbNR+TlwxmUSS24WBji1a85TYkEiwY1Qu9GNkoFOkRUste5zIa7jTHa1TZDnkxgW0DRyzZkZOfh1L3yrYWljMHNa8FARxNhCalY7fcAJ+/FQ0MCzHinclt1gOo32/abisEOqYVnKdl4OdrhFwJRxXvdP7yftHcGAOy4EoXMnMLLJpwNSUB6dh7sTfXRxF71WZuVZaSnjUHN8xdoXXY0f1maYS0d4GpVuKWrMoxo7YgLc7thx7h2uDC3m0rzf1E+PsaiClfciKjKkp0rw6y/bkAIoIe7FT7rVJt9Zogqyevsm9azvjXsTPQQI83EwRsxGNZKcRh6wVpY/ZrYVtpoqAKWNRRHZda2NKzU8l7FR+Tlw5YdNRQdHY0PP/wQ5ubm0NfXR+PGjREUFAQAyMnJwZw5c9C4cWMYGhrCzs4OH3/8MWJiYko85rlz59C/f3/Y2dlBIpFg3759CtsLjuvo6o5alqbo3Kwe6nboh9X/BFbWacqt8XuA+3EpMDXQxrL3msCjjjm/FIgqUcGK6ZX9/0xLUwMfeTgDADa9shp6enYuTt3/9xFW48p7hAXk38D9ejpMIW3Z0RCVF0ulqsNgR80kJiaiQ4cO0NbWxpEjR3D37l389NNPMDXNn2grPT0d165dw/z583Ht2jXs2bMHISEhGDBgQInHTUtLQ9OmTfHbb78VuT09PR3+V4KQ2WgQbEf/AstB85D9IhrTxo6s1C+E0PgU+ZfQNwMawrwGV/0lUifvt3aArpYG7sQk4+rjRHn66fsJyMyRwdHMAI1qVe4yP0UNgMgTAo+epVdquVRx+BhLzSxduhQODg7w9fWVp7m4uMj/bWJighMnTijkWbVqFdq0aYPIyEg4Ohb9LLhPnz7o06dPseWamJjggwVrsfTf59kAYPbORMRtmYHLN0MxuFPTsp5SsfJkArP+uomcPIGe9a0woGnl3t0R0etnaqiDgc3s8GfQE2y69AitnPOHoR9+jY+wOCLqzceWHTVz4MABtGrVCsOGDYOVlRWaN2+O9evXl5hHKpVCIpGgZs2aZSozKzcPCw/eVQh0AECWlQ5AAoMalXPXtfFCBG5EJcFIVwvfDWpc6V94RFQ1Rv/bUfno7TjESTORlpWL0/cTAADvVsJEgq/iiKg3H1t21MzDhw+xevVqzJgxA/PmzUNgYCCmTJkCHR0djB49utD+mZmZmDNnDj744IMyrfgenpCCz3dcx73YZABA+zrmuPzwOfJyspF01hcGDTpj4bEItHC1U2rZBWU9epaGH4/nB1df9qsPGxO9Cjs2EVUvDe1M0MbZDFcevcC2gMdwszZCVq4MLhaGaGBbuY+wCnDS0Dcbgx01I5PJ0KpVKyxevBgA0Lx5c9y+fRtr1qwpFOzk5ORg+PDhEEJg9erVKpUjhMD2gEgsPHQHmTkymBvq4IdhTdDd3RqRz5IxasRwGFoawnzELEQlZeCzLUHYNb6dUisYl36OAnP+vomsXBk6uJpjhBILBRLRm210e2dcefQC2wMi5cPM+zWu/EdYL+OIqDcXH2OpGVtbWzRo0EAhrX79+oiMjFRIKwh0Hj9+jBMnTqjcqrPG7yHm7b2FzBwZOrlZ4MjUTujubo2cnBxMHTcayc9icf7saWyZ2BU1DbRxIyoJM/68DllR0xyraPuVSAREvIC+tiaWDGnCx1dEbwHPhtawNdHD87RsnAl5CqBy1sIi9cRgR8106NABISGKfWdCQ0Ph5OQk/7sg0AkLC8PJkydhbm6u9PH9HzwHAARHJkJbU4Iv+9bH5jFtYGWsh5ycHPTv3x9nzpxBVFQU7O3tMaBbO0xrogFtTQn+uRWHZcfu4+uvv4atrS309fXRs2dPhIWFlVimj48PWrduDSMjI1hYWmHS6A+Q8/wJZveuBwez/A6CEyZMQJ06daCvrw9LS0sMHDgQ9+/fV/q8iKh609bUKDRx4I2opKqpDL1xGOyomenTp+Py5ctYvHgxwsPDsX37dqxbtw5eXl4A8gOd9957D0FBQdi2bRvy8vIQFxeHuLg4ZGdny4/To0cPrFq1Sv53ojQZ03/fh6HfbwcAmOQmwqeLEXo5a0FDQ4KcnBwMGDAAJ0+eRJcuXbBt2zb4+fnhyy+/RNt6tbBkSBMAwNKly/DTil+wZs0aBAQEwNDQEL169UJmZmax5+Tn5wcvLy/4+/vDY/Jy5ObmIHHPArzXxFK+T8uWLeHr64t79+7h2LFjEELA09MTeXmFZ10lojdPrDQDJ/5d3bzAl3tvc64bUo4gIZVKBQAhlUqruioV4uDBg6JRo0ZCV1dXuLu7i3Xr1sm3RURECABFvs6cOSPfz8nJSSxYsEAIIcTjZ2miw5SVReYZPXq00sf98eg9oWloKsy7fyouhj0VQgiRlJQkdHV1xY4dO0o9r7+CooTTnEPCZdoOAUD4+fkVu++NGzcEABEeHq7am0dE1dLF8KfCac6hQq9L4c+qumpUhZT9/WYHZTX07rvv4t133y1ym7Ozs8IspMXxv3EPEc/SsOliBH48HopU/dpovOAofIY0KfI5ubOzM+rXr49evXrhyZMn8PPzQ61atTBp0iR07doVADDIVRtfpCVCx7EZJm69ij2TOsDVygRt27aFv78/3n///WLrk5CSiYWH7gIAPm5liW8BmJmZFblvWloafH194eLiAgcHdl4mUgec64bKg4+xqJBdgZHosOQ0Rq4PwDcH7yI1KxetnU1xZFrnEjsEFgx7d3Nzw7Fjx/C///0PU6ZMwebNmwEA8fH5TdDN6jkhOTMXYzZdwfPULFhbWyMuLq7EOn297w6kGTloaFsDgduXo0OHDmjUqJHCPr///jtq1KiBGjVq4MiRIzhx4gR0dHTK+W4QUXXAuW6oPCRCmdv8l5w5cwbdunWrrPpUieTkZJiYmEAqlZZprhl1IU3PweFbMZi397ZCugTAudld4WBW8sJ3Ojo6aNWqFS5duiRPmzJlCgIDA+Hv749Lly6hQ4cOuB0Wgf/tiUDki3S0dDIFTq2AlqYGdu3aVeRx/7kVi0nbrkFLQ4JWT/7G5XOncOHCBdjb2yvWXypFQkICYmNj8eOPPyI6OhoXL16Enh7n4CFSF7HSDM51Q3LK/n6r/Bird+/esLe3x5gxYzB69Gg+JniDZefKcC0yERfCnuF8+DPcepJUaP0XIL/jzZPEzFKDneKGvf/9998AABsbGwBATmoSNn7SGoN/v4irjxORc+chhr7TochjJqZl4+v9+cGX9e1tuBh8DufOnSsU6AD5S1aYmJjAzc0N7dq1g6mpKfbu3YsPPvigtLeCiN4QnOuGykLlx1jR0dGYPHky/vrrL9SuXRu9evXCn3/+qTCSh6perDQDlx48UxipIIRAaHwKNlyIwBjfK2i28DjeX3cZq86E40ZUfqDjbG6AV2etUfa5eGnD3l1cXGBjY4NTp07B1aoG1n7YEho5GYgJu4XEGs5FHnPhobt4mpKFvAv/h0dXz+L06dMKa30VRwgBIQSysrJK3ZeIiNSbyi07FhYWmD59OqZPn45r167B19cXkyZNwqRJkzBy5EiMHTsWTZtW/KKPpLxdgZHw3nMLMgFoSIDhrRyQkydwIfwp4pMVf/wtauigg6sFOrpaoKObBWxN9LErMBLz9txGnhAqPRefPn062rdvj8WLF2P48OG4cuUK1q1bh3Xr1gEAJBIJpk2bhu+++w5ubm5wcXGBedB6RNUww7lsF/x99QmGtrRHjx49MHjwYDTsORx7g6OReGI1xIMLOHTgAIyMjOT9e0xMTKCvr4+HDx9i165d8PT0hKWlJZ48eYIlS5ZAX18fffv2rfg3mIiI3igq99l5VUxMDNatW4clS5ZAS0sLmZmZ8PDwwJo1a9CwYcOKqmelUqc+O7HSDHRYcrrIx1EAoKulgTYuZujoaoFObpZwtzGChkbhGYjL+lz80KFD8Pb2RlhYGFxcXDBjxgyMGzdOvl0IgQULFmDdunVISkpCx44d0ez9mfgrXAZtTQm2jm2LEd1b4oMPP8JZw66IlWbi8dKiR5b5+vrik08+QUxMDD777DNcvXoViYmJsLa2RufOnfH111+jXr16StediIjeLMr+fpcp2MnJycH+/fuxceNGnDhxAq1atcLYsWPxwQcf4OnTp/jqq69w7do13L17t1wnAeQ/NpszZw6OHDmC9PR0uLq6wtfXF61atQLw34/n+vXrkZSUhA4dOshHBClLnYKdSw+eYeT6gELp/ZvaYkQrR7RyNq2Q9akqkkwmMHnHNfxzKw41DbSx7qOWWOP3EKfvJ8DJ3ABHp3aGvk71qjMREVW9Suug/Pnnn2PHjh0QQuCjjz7CsmXLFIYAGxoa4scff4SdnV3Zav6SxMREdOjQAd26dcORI0dgaWmJsLAwmJqayvdZtmwZVq5cic2bN8PFxQXz589Hr169cPfu3bdyFE5xc1HM61u/2nbq09CQYPnwZohOuowbUUkYvvayfFuvBjYMdIiIqFxUbtnp0aMHPvvsMwwZMgS6urpF7pObm4uLFy+iS5cu5arc3LlzcfHiRZw/f77I7UII2NnZYebMmfjiiy8A5A8/tra2xqZNm0qcpO5l6tSyAwDTdgZj3/UYAP/NRTGitWMV16p0t6OlePfXCwppmhIJLsztVm0DNSIiqjrK/n6rPBrr1KlT+OCDD4oNdABAS0ur3IEOABw4cACtWrXCsGHDYGVlhebNm2P9+vXy7REREYiLi0PPnj3laSYm/83IW5ysrCwkJycrvNRJSmYuAGBEawdcmNvtjQh0ACA5M6dQWp4QePQsvQpqQ0RE6kLlYMfHxwcbN24slL5x40YsXbq0QipVoLQZeQtG5VhbWyvkK21GXh8fH/mcLCYmJmo1V1BWbh4u/bsy+UftnN6oFpGCR3Av43TwRERUXioHO2vXroW7u3uh9IYNG2LNmjUVUqkCMpkMLVq0wOLFi9G8eXOMHz8e48aNK3c53t7ekEql8ldUVFQF1bjqBUYkIiMnD5ZGumho92Y9kuN08EREVBlU7qAcFxcHW9vC6yNZWloiNja2QipVQNkZeePj4xXqFB8fj2bNmhV7XF1d3RIfw73JzoYkAAC61LWERFJ4SHl1N6K1IzrXteR08EREVGFUbtlxcHDAxYsXC6VfvHixQkZgvUyVGXkLJCcnIyAgAB4eHhValzfF2dCnAICu9SyruCZlZ2uiD4865gx0iIioQqjcsjNu3DhMmzYNOTk56N69O4D8TsuzZ8/GzJkzK7RyZZmRd/78+bCzs8OgQYMqtC5vgieJ6QhPSIWGBOjk+uYGO0RERBVJ5WBn1qxZeP78OSZNmiRfD0tPTw9z5syBt7d3hVaudevW2Lt3L7y9vbFw4UK4uLjg559/xqhRo+T7zJ49G2lpaRg/frx8Rt6jR4++lXPsnA3Jb9Vp4WgKEwPtKq4NERFR9VDm5SJSU1Nx79496Ovrw83N7Y3uA6Mu8+x8tjkIJ+/F4wvPupjcXfkZpImIiN5ElTaDcoEaNWqgdevWZc1OFSx/yPkzAEDXelZVXBsiIqLqo0zBTlBQEP78809ERkbKH2UV2LNnT4VUjFQT9CgR6dl5sKihiwa2b27rFBERUUVTeTTWzp070b59e9y7dw979+5FTk4O7ty5g9OnT8PExKQy6khKeHnIeVGrmBMREb2tVA52Fi9ejBUrVuDgwYPQ0dHBL7/8gvv372P48OFwdHwzliVQRwWdk7u8wUPOiYiIKoPKwc6DBw/Qr18/AICOjg7S0tIgkUgwffp0+ZBwer2ikzIQ9u+Q885uFlVdHSIiompF5WDH1NQUKSkpAIBatWrh9u3bAICkpCSkp3PBxqrg92+rTjOHmqhpoFPFtSEiIqpeVO6g3LlzZ5w4cQKNGzfGsGHDMHXqVJw+fRonTpxAjx49KqOOVIqC/jochUVERFSYysHOqlWrkJmZCQD48ssvoa2tjUuXLmHo0KH46quvKryCVLLsXBkuhhcMOWd/HSIiolepFOzk5ubi0KFD6NWrFwBAQ0MDc+fOrZSKkXKCHr9AWnYeLGrooJEdR8MRERG9SqU+O1paWpg4caK8ZYeqXkF/nc5uHHJORERUFJU7KLdp0wbXr1+vhKpQWXDIORERUclU7rMzadIkzJgxA1FRUWjZsiUMDQ0Vtjdp0qTCKkcli0nKQEh8yr9DzhnsEBERFUXlYOf9998HAEyZMkWeJpFIIISARCJBXl5exdWOSuQXmt+q09ShJkwNOeSciIioKCoHOxEREZVRDyqDl5eIICIioqKpHOw4OTlVRj1IRTl5MlwMfw6A8+sQERGVROVgZ8uWLSVu//jjj8tcGVLe1ceJSM3KhZmhDprU4pBzIiKi4qgc7EydOlXh75ycHKSnp0NHRwcGBgYMdl6Ts/Ih5xYcck5ERFQClYeeJyYmKrxSU1MREhKCjh07YseOHZVRRyoCl4ggIiJSjsrBTlHc3NywZMmSQq0+VDnipJm4H5cCiQTozM7JREREJaqQYAfIn105Jiamog5HJfALzW/VaWJfE2Ycck5ERFQilfvsHDhwQOFvIQRiY2OxatUqdOjQocIqRsUr6K/Tla06REREpVI52Bk0aJDC3xKJBJaWlujevTt++umniqoXFSMnT4YLYVzlnIiISFkqBzsymawy6kFKuvY4ESlZuTA10EYT+5pVXR0iIqJqr8L67NDrcfbfJSI6uVlCk0POiYiISqVysDN06FAsXbq0UPqyZcswbNiwCqkUFc+voL8OH2EREREpReVg59y5c+jbt2+h9D59+uDcuXMVUikqWkJyJu7GJgPgkHMiIiJlqRzspKamQken8HBnbW1tJCcnV0ilqGgFj7Ca2JvAooZuFdeGiIjozaBysNO4cWPs2rWrUPrOnTvRoEGDCqkUFc2PQ86JiIhUpvJorPnz52PIkCF48OABunfvDgA4deoUduzYgd27d1d4BSlfbp4M58Pyg50uXCKCiIhIaSoHO/3798e+ffuwePFi/PXXX9DX10eTJk1w8uRJdOnSpTLqSACCo5KQnJmLmgbaaOZQs6qrQ0RE9MZQOdgBgH79+qFfv34VXRcqQcHCnxxyTkREpBqV++wEBgYiICCgUHpAQACCgoIqpFLqZsmSJZBIJJg2bZpCur+/P7p37w5DQ0MYGxujc+fOyMjIKPIYZ0OeQur/J4589wmMjIxgZWWFQYMGISQkRGG/devWoWvXrjA2NoZEIkFSUlIlnRUREdGbQeVgx8vLC1FRUYXSo6Oj4eXlVSGVUieBgYFYu3YtmjRpopDu7++P3r17w9PTE1euXEFgYCAmT54MDY3CH0lCSibuxCQjM+o2pn4+GZcvX8aJEyeQk5MDT09PpKWlyfdNT09H7969MW/evEo/NyIiojeByo+x7t69ixYtWhRKb968Oe7evVshlVIXqampGDVqFNavX4/vvvtOYdv06dMxZcoUzJ07V55Wr169Io9TMAqrx/Sf8fnETvL0TZs2wcrKClevXkXnzp0BQN56dPbs2Qo8EyIiojeXyi07urq6iI+PL5QeGxsLLa0ydQFSW15eXujXrx969uypkJ6QkICAgABYWVmhffv2sLa2RpcuXXDhwoUij1Mwv07XuoqjsKRSKQDAzMysEmpPRESkHlQOdjw9PeHt7S3/oQWApKQkzJs3D++8806FVu5NtnPnTly7dg0+Pj6Ftj18+BAA8M0332DcuHE4evQoWrRogR49eiAsLExh39xiVjmXyWSYNm0aOnTogEaNGlXimRAREb3ZVG6K+fHHH9G5c2c4OTmhefPmAIDr16/D2toaf/zxR4VX8E0UFRWFqVOn4sSJE9DT0yu0vWDl+AkTJmDMmDEA8h8Dnjp1Chs3blQIkG48SYI0IwfGeloKQ869vLxw+/btYluDiIiIKJ/KwU6tWrVw8+ZNbNu2DTdu3IC+vj7GjBmDDz74ANra2pVRxzfO1atXkZCQoNC3KS8vD+fOncOqVavkI6henXG6fv36iIyMVEg7+29/nU51LaGlmd8QN3nyZBw6dAjnzp2Dvb19ZZ4KERHRG69MnWwMDQ0xfvz4iq6L2ujRowdu3bqlkDZmzBi4u7tjzpw5qF27Nuzs7AoNGw8NDUWfPn0U0s6+tESEEAKff/459u7di7Nnz8LFxaVyT4SIiEgNlLlH8d27dxEZGYns7GyF9AEDBpS7Um86IyOjQv1oDA0NYW5uLk+fNWsWFixYgKZNm6JZs2bYvHkz7t+/j7/++kuep3PXbritXQ/GLfujSz1LeHl5Yfv27di/fz+MjIwQFxcHADAxMYG+vj4AIC4uDnFxcQgPDwcA3Lp1C0ZGRnB0dGRHZiIieiupHOw8fPgQgwcPxq1btyCRSCCEAABIJPmz+ubl5VVsDdXUtGnTkJmZienTp+PFixdo2rQpTpw4gTp16sj3uR8aDlltWzS0M4aVkR5Wr14NAOjatavCsXx9ffHJJ58AANasWYNvv/1Wvq1gSPrL+xAREb1NJKIgWlFS//79oampif/7v/+Di4sLrly5gufPn2PmzJn48ccf0alTp9IPUs0kJyfDxMQEUqkUxsbGVV0duc82B+HkvXiM9nDCtwM54oqIiOhlyv5+qzz03N/fHwsXLoSFhQU0NDSgoaGBjh07wsfHB1OmTClXpek/O65E4uS9/PmMtlx+jF2BkaXkICIioqKoHOzk5eXByMgIAGBhYYGYmBgAgJOTU6EOt1Q2sdIMzNv7XwdnIYB5e24jVlr0ullERERUPJX77DRq1Ag3btyAi4sL2rZti2XLlkFHRwfr1q1D7dq1K6OOb52IZ2l49eFinhB49Cwdtib6VVMpIiKiN5TKwc5XX30lX3hy4cKFePfdd9GpUyeYm5tj165dFV7Bt5Hmv529X01ztjCogtoQERG92VQOdnr16iX/t6urK+7fv48XL17A1NRUPiKLymff9RiFvzUlEiwe0oitOkRERGVQISt3cv6WipOQkom/rz0BAKwe1QI1DXTgbGHAQIeIiKiMuEx5NbPp4iNk58rQwrEmejeyYWsZERFROak8GosqT2pWLv64/BgAMKFLHQY6REREFYDBTjWyIyASKZm5qG1piHfqW1d1dYiIiNSCysHOuXPnkJubWyg9NzcX586dq5BKvY2yc2XYcCECADChc21oaLBVh4iIqCKoHOx069YNL168KJQulUrRrVu3CqnU22j/9WjEJWfCykgXg5rXqurqEBERqQ2Vgx0hRJF9SZ4/fw5DQ8MKqdTbRiYTWHfuIQDg044u0NXSrOIaERERqQ+lR2MNGTIEQP7q5p988gl0dXXl2/Ly8nDz5k20b9++4mv4Fjh9PwFhCakw0tXCyLaOVV0dIiIitaJ0sGNiYgIgv2XHyMgI+vr/zfuio6ODdu3aYdy4cRVfw7fA2nMPAAAj2znCWE+7imtDRESkXpQOdnx9fQEAzs7O+OKLL/jIqoJcffwCgY8SoaOpgU87uFR1dYiIiNSOyn12Zs+erdBn5/Hjx/j5559x/PjxCq3Y22KNX35fncHNa8HaWK+Ka0NERKR+VA52Bg4ciC1btgAAkpKS0KZNG/z0008YOHAgVq9eXeEVVGfhCak4cTceEgkwrjNXjCciIqoMKgc7165dQ6dOnQAAf/31F2xsbPD48WNs2bIFK1eurPAKqrN1//bVeae+NVytalRxbYiIiNSTysFOeno6jIyMAADHjx/HkCFDoKGhgXbt2uHx48cVXkF1FZ+cib3B0QDyl4YgIiKiyqFysOPq6op9+/YhKioKx44dg6enJwAgISEBxsbGFV5BdbXxQgRy8gTaOJuhpZNpVVeHiIhIbakc7Hz99df44osv4OzsjDZt2sDDwwNAfitP8+bNK7yC6ig5MwfbAiIBABO6sK8OERFRZVI52HnvvfcQGRmJoKAgHDt2TJ7eo0cPrFixokIr96olS5ZAIpFg2rRp8rTMzEx4eXnB3NwcNWrUwNChQxEfH1+p9SivbZcjkZqVCzerGuhWz6qqq0NERKTWyrTquY2NDYyMjHDixAlkZGQAAFq3bg13d/cKrdzLAgMDsXbtWjRp0kQhffr06Th48CB2794NPz8/xMTEyGd7ro6ycvOw8eK/C352qcMFP4mIiCqZysHO8+fP0aNHD9StWxd9+/ZFbGwsAGDs2LGYOXNmhVcQAFJTUzFq1CisX78epqb/9W+RSqXYsGEDli9fju7du6Nly5bw9fXFpUuXcPny5UqpS3ntC47G05Qs2JroYUBTu6quDhERkdpTOdiZPn06tLW1ERkZCQMDA3n6iBEjcPTo0QqtXAEvLy/069cPPXv2VEi/evUqcnJyFNLd3d3h6OgIf3//Yo+XlZWF5ORkhdfrIJMJrP13wc+xHV2go1WmhjUiIiJSgdLLRRQ4fvw4jh07Bnt7e4V0Nze3Shl6vnPnTly7dg2BgYGFtsXFxUFHRwc1a9ZUSLe2tkZcXFyxx/Tx8cG3335b0VUt1Yl78Xj4NA3Gelp4vw0X/CQiInodVG5aSEtLU2jRKfDixQuFldArQlRUFKZOnYpt27ZBT6/illLw9vaGVCqVv6Kioirs2MURQmCNX/4kgh95OKGGrspxJhEREZWBysFOp06d5MtFAIBEIoFMJsOyZcvQrVu3Cq3c1atXkZCQgBYtWkBLSwtaWlrw8/PDypUroaWlBWtra2RnZyMpKUkhX3x8PGxsbIo9rq6uLoyNjRVelS3wUSKCI5Ogo6WBT9pzwU8iIqLXReXmhWXLlqFHjx4ICgpCdnY2Zs+ejTt37uDFixe4ePFihVauR48euHXrlkLamDFj4O7ujjlz5sDBwQHa2to4deoUhg4dCgAICQlBZGSkfP6f6qKgVee9lvawNKrYFjAiIiIqnsrBTqNGjRAaGopVq1bByMgIqampGDJkCLy8vGBra1uhlTMyMkKjRo0U0gwNDWFubi5PHzt2LGbMmAEzMzMYGxvj888/h4eHB9q1a1ehdSmPkLgUnL6fkL/gZydOIkhERPQ6qRzsREZGwsHBAV9++WWR2xwdX2/H2xUrVkBDQwNDhw5FVlYWevXqhd9///211qE0a/9d8LN3Qxu4WBhWcW2IiIjeLhIhhFAlg6amJmJjY2FlpTjz7/Pnz2FlZYW8vLwKreDrkJycDBMTE0il0grvvxOTlIHOy84gVyaw36sDmjrUrNDjExERva2U/f1WuYOyEAISSeFZf1NTUyt0xJS62HghArkygXa1zRjoEBERVQGlH2PNmDEDQP7oq/nz5ysMP8/Ly0NAQACaNWtW4RV8k4XGpWDr5fy5hyZ2qVPFtSEiIno7KR3sBAcHA8hv2bl16xZ0dHTk23R0dNC0aVN88cUXFV/DN9SuwEjM/fsWCp4Rxkkzq7Q+REREbyulg50zZ84AyB/6/csvv7yWuWneVLHSDHjv+S/QAYAv995Gl3qWsDXRr7J6ERERvY1U7rPj6+vLQKcUEc/SIHul23eeEHj0LL1qKkRERPQW40qUlcDFwhAar/Th1pRI4GxReJkNIiIiqlwMdiqBrYk+fIY0hua/o9Y0JRIsHtKIj7CIiIiqAFejrCQjWjuic11LPHqWDmcLAwY6REREVYTBTiWyNdFnkENERFTF+BiLiIiI1BqDHSIiIlJrDHaIiIhIrTHYISIiIrXGYIeIiIjUGoMdIiIiUmsMdoiIiEitMdghIiIitcZgh4iIiNQagx0iIiJSawx2iIiISK0x2CEiIiK1xmCHiIiI1BqDHSIiIlJrDHaIiIhIrTHYISIiIrXGYIeIiIjUGoMdIiIiUmsMdoiIiEitMdghIiIitcZgh4iIiNQagx0iIiJSawx2iIiISK0x2CEiIiK1xmCHiIiI1BqDHSIiIlJrDHaIiIhIrTHYISIiIrXGYIeIiIjUGoMdIiIiUmsMdoiIiEitMdghIiIitcZgh4iIiNQagx0iIiJSawx2iIiISK0x2CEiIiK1xmCHiIiI1BqDHSIiIlJrDHaIiIhIrTHYISIiIrXGYIeIiIjUGoMdIiIiUmsMdoiIiEitMdghIiIitcZgh4iIiNQagx0iIiJSawx2iIiISK0x2CEiIiK1xmCHiIiI1BqDHSIiIlJrDHaIiIhIrTHYISIiIrXGYIeIiIjUGoMdIiIiUmvVOtjx8fFB69atYWRkBCsrKwwaNAghISEK+2RmZsLLywvm5uaoUaMGhg4divj4+CqqMREREVU31TrY8fPzg5eXFy5fvowTJ04gJycHnp6eSEtLk+8zffp0HDx4ELt374afnx9iYmIwZMiQKqw1ERERVScSIYSo6koo6+nTp7CysoKfnx86d+4MqVQKS0tLbN++He+99x4A4P79+6hfvz78/f3Rrl07pY6bnJwMExMTSKVSGBsbV+YpEBERUQVR9ve7WrfsvEoqlQIAzMzMAABXr15FTk4OevbsKd/H3d0djo6O8Pf3L/Y4WVlZSE5OVngRERGRenpjgh2ZTIZp06ahQ4cOaNSoEQAgLi4OOjo6qFmzpsK+1tbWiIuLK/ZYPj4+MDExkb8cHBwqs+pERERUhd6YYMfLywu3b9/Gzp07y30sb29vSKVS+SsqKqoCakhERETVkVZVV0AZkydPxqFDh3Du3DnY29vL021sbJCdnY2kpCSF1p34+HjY2NgUezxdXV3o6upWZpWJiIiomqjWLTtCCEyePBl79+7F6dOn4eLiorC9ZcuW0NbWxqlTp+RpISEhiIyMhIeHx+uuLhEREVVD1bplx8vLC9u3b8f+/fthZGQk74djYmICfX19mJiYYOzYsZgxYwbMzMxgbGyMzz//HB4eHkqPxCIiIiL1Vq2HnkskkiLTfX198cknnwDIn1Rw5syZ2LFjB7KystCrVy/8/vvvJT7GehWHnhMREb15lP39rtbBzuvCYIeIiOjNo5bz7BARERGpisEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTW1CbY+e233+Ds7Aw9PT20bdsWV65cqeoqERERUTWgFsHOrl27MGPGDCxYsADXrl1D06ZN0atXLyQkJFR11YiIiKiKqUWws3z5cowbNw5jxoxBgwYNsGbNGhgYGGDjxo1VXTUiIiKqYlpVXYHyys7OxtWrV+Ht7S1P09DQQM+ePeHv719knqysLGRlZcn/lkqlAIDk5OTKrSwRERFVmILfbSFEifu98cHOs2fPkJeXB2tra4V0a2tr3L9/v8g8Pj4++PbbbwulOzg4VEodiYiIqPKkpKTAxMSk2O1vfLBTFt7e3pgxY4b8b5lMhhcvXsDc3BwSiaTCyklOToaDgwOioqJgbGz8WvOz7Ndfdnnzs+y3q+zy5mfZLPtNyV/esksihEBKSgrs7OxK3O+ND3YsLCygqamJ+Ph4hfT4+HjY2NgUmUdXVxe6uroKaTVr1qysKsLY2LhcH3B58rPs1192efOz7Ler7PLmZ9ks+03JX96yi1NSi06BN76Dso6ODlq2bIlTp07J02QyGU6dOgUPD48qrBkRERFVB298yw4AzJgxA6NHj0arVq3Qpk0b/Pzzz0hLS8OYMWOqumpERERUxdQi2BkxYgSePn2Kr7/+GnFxcWjWrBmOHj1aqNPy66arq4sFCxYUemT2OvKz7Ndfdnnzs+y3q+zy5mfZLPtNyV/esiuCRJQ2XouIiIjoDfbG99khIiIiKgmDHSIiIlJrDHaIiIhIrTHYISIiIrXGYKcS/fbbb3B2doaenh7atm2LK1euKJXv3Llz6N+/P+zs7CCRSLBv3z6ly/Tx8UHr1q1hZGQEKysrDBo0CCEhIUrnX716NZo0aSKf/MnDwwNHjhxROv/LlixZAolEgmnTpim1/zfffAOJRKLwcnd3V7q86OhofPjhhzA3N4e+vj4aN26MoKAgpfI6OzsXKlsikcDLy6vUvHl5eZg/fz5cXFygr6+POnXqYNGiRaWu1fKylJQUTJs2DU5OTtDX10f79u0RGBhYaL/Srg0hBL7++mvY2tpCX18fPXv2RFhYmNL59+zZA09PT/ls4tevX1e6/JycHMyZMweNGzeGoaEh7Ozs8PHHHyMmJkapsr/55hu4u7vD0NAQpqam6NmzJwICApSu+8smTpwIiUSCn3/+Wam8n3zySaHPvnfv3iqVfe/ePQwYMAAmJiYwNDRE69atERkZWWreoq47iUSCH374QamyU1NTMXnyZNjb20NfX1++GLIyeePj4/HJJ5/Azs4OBgYG6N27t/x6Uea7JDMzE15eXjA3N0eNGjUwdOhQ+QSvyuRft24dunbtCmNjY0gkEiQlJcm3lZb/xYsX+Pzzz1GvXj3o6+vD0dERU6ZMgVQqVarsCRMmoE6dOtDX14elpSUGDhwoX2JIle9RIQT69Okjf3+Vydu1a9dCn/fEiRNVKtvf3x/du3eHoaEhjI2N0blzZyxcuLDEvI8ePSr2etu9e7dSZcfFxeGjjz6CjY0NDA0N0aJFC/z9999K5X3w4AEGDx4MS0tLGBsbY/jw4YUmBK4sDHYqya5duzBjxgwsWLAA165dQ9OmTdGrVy8kJCSUmjctLQ1NmzbFb7/9pnK5fn5+8PLywuXLl3HixAnk5OTA09MTaWlpSuW3t7fHkiVLcPXqVQQFBaF79+4YOHAg7ty5o1I9AgMDsXbtWjRp0kSlfA0bNkRsbKz8deHCBaXyJSYmokOHDtDW1saRI0dw9+5d/PTTTzA1NVW6vi+Xe+LECQDAsGHDSs27dOlSrF69GqtWrcK9e/ewdOlSLFu2DL/++qtSZQPAZ599hhMnTuCPP/7ArVu34OnpiZ49eyI6Olphv9KujWXLlmHlypVYs2YNAgICYGhoiF69eiEzM1Op/GlpaejYsSOWLl1a7Pbi8qenp+PatWuYP38+rl27hj179iAkJAQDBgxQquy6deti1apVuHXrFi5cuABnZ2d4enri6dOnSuUvsHfvXly+fFlh+nhl8vbu3VvhGtixY4fS+R88eICOHTvC3d0dZ8+exc2bNzF//nzo6emVmvflMmNjY7Fx40ZIJBIMHTpUqbJnzJiBo0ePYuvWrbh37x6mTZuGyZMn48CBAyXmFUJg0KBBePjwIfbv34/g4GA4OTmhZ8+eSEtLU+q7ZPr06Th48CB2794NPz8/xMTEYMiQIQCU+y5KT09H7969MW/evEL1Ky1/TEwMYmJi8OOPP+L27dvYtGkTjh49irFjxypVdsuWLeHr64t79+7h2LFjEELA09MTeXl5Kn2P/vzzzwrLDCmbd9y4cQqf+7Jly5TO7+/vj969e8PT0xNXrlxBYGAgJk+ejAsXLpSY18HBodD19u2336JGjRro06ePUmV//PHHCAkJwYEDB3Dr1i0MGTIEw4cPx8GDB0vMm5aWBk9PT0gkEpw+fRoXL15EdnY2+vfvD5lMVuh9rXCCKkWbNm2El5eX/O+8vDxhZ2cnfHx8VDoOALF3794y1yMhIUEAEH5+fmU+hqmpqfi///s/pfdPSUkRbm5u4sSJE6JLly5i6tSpSuVbsGCBaNq0aZnqOGfOHNGxY8cy5S3K1KlTRZ06dYRMJit13379+olPP/1UIW3IkCFi1KhRSpWVnp4uNDU1xaFDhxTSW7RoIb788sti8716bchkMmFjYyN++OEHeVpSUpLQ1dUVO3bsKDX/yyIiIgQAERwcrHT5Rbly5YoAIB4/fqxyXqlUKgCIkydPKl32kydPRK1atcTt27eFk5OTWLFihVJ5R48eLQYOHFhifUrKP2LECPHhhx+WKe+rBg4cKLp37650/oYNG4qFCxcqpBV17byaNyQkRAAQt2/flqfl5eUJS0tLsX79+kJlv/pdkpSUJLS1tcXu3bvl+9y7d08AEP7+/qXmf9mZM2cEAJGYmFjkeZeWv8Cff/4pdHR0RE5Ojsp5b9y4IQCI8PBwpcsODg4WtWrVErGxscV+tkXlVeV7saj8bdu2FV999VWZ8r6qWbNmhb6/SspvaGgotmzZorCfmZlZoWvm1bzHjh0TGhoaQiqVyvdJSkoSEolEnDhxotRzKS+27FSC7OxsXL16FT179pSnaWhooGfPnvD393+tdZFKpQAAMzMzlfPm5eVh586dSEtLU2npDS8vL/Tr10/h/JUVFhYGOzs71K5dG6NGjUJkZKRS+Q4cOIBWrVph2LBhsLKyQvPmzbF+/XqVywfyP7+tW7fi008/VWph2Pbt2+PUqVMIDQ0FANy4cQMXLlxAnz59lCovNzcXeXl50NPTU0jX19dXumULACIiIhAXF6fwvpuYmKBt27av/borIJVKIZFIVF57Ljs7G+vWrYOJiQmaNm2qVB6ZTIaPPvoIs2bNQsOGDVWu69mzZ2FlZYV69erhf//7H54/f650uYcPH0bdunXRq1cvWFlZoW3btio9fi4QHx+Pw4cPY+zYsUrnad++PQ4cOIDo6GgIIXDmzBmEhobC09OzxHxZWVkAoHDdaWhoQFdXt8jr7tXvkqtXryInJ0fhenN3d4ejo2OR11t5vouUzS+VSmFsbAwtLa1C6SXlTUtLg6+vL1xcXODg4KBU2enp6Rg5ciR+++23YtdhLKnsbdu2wcLCAo0aNYK3tzfS09OVyp+QkICAgABYWVmhffv2sLa2RpcuXZT6zF519epVXL9+vdjrraj87du3x65du/DixQvIZDLs3LkTmZmZ6Nq1a4l5s7KyIJFIFCYW1NPTg4aGhkrfc2VW6eHUWyg6OloAEJcuXVJInzVrlmjTpo1Kx0I5Wnby8vJEv379RIcOHVTKd/PmTWFoaCg0NTWFiYmJOHz4sNJ5d+zYIRo1aiQyMjKEEKrdwfzzzz/izz//FDdu3BBHjx4VHh4ewtHRUSQnJ5eaV1dXV+jq6gpvb29x7do1sXbtWqGnpyc2bdqkdN0L7Nq1S2hqaoro6Gil9s/LyxNz5swREolEaGlpCYlEIhYvXqxSmR4eHqJLly4iOjpa5Obmij/++ENoaGiIunXrFpvn1Wvj4sWLAoCIiYlR2G/YsGFi+PDhpeZ/WUW07GRkZIgWLVqIkSNHKp334MGDwtDQUEgkEmFnZyeuXLmidNmLFy8W77zzjrw1TpWWnR07doj9+/eLmzdvir1794r69euL1q1bi9zc3FLzF9zVGxgYiOXLl4vg4GDh4+MjJBKJOHv2rFLnXWDp0qXC1NRU/v9HmbpnZmaKjz/+WAAQWlpaQkdHR2zevLnUvNnZ2cLR0VEMGzZMvHjxQmRlZYklS5YIAMLT01Mhb1HfJdu2bRM6OjqFymndurWYPXt2qflfVlrLjjLfZU+fPhWOjo5i3rx5Suf97bffhKGhoQAg6tWrV2SrTnH5x48fL8aOHSv/u6jPpri8a9euFUePHhU3b94UW7duFbVq1RKDBw9Wqmx/f38BQJiZmYmNGzeKa9euiWnTpgkdHR0RGhqq1HkX+N///ifq169f5Lbi8icmJgpPT0/59WZsbCyOHTtWat6EhARhbGwspk6dKtLS0kRqaqqYPHmyACDGjx9fbB0rCoOdSlBdgp2JEycKJycnERUVpVK+rKwsERYWJoKCgsTcuXOFhYWFuHPnTqn5IiMjhZWVlbhx44Y8TZVg51WJiYnC2NhYqUdo2trawsPDQyHt888/F+3atVO5XE9PT/Huu+8qvf+OHTuEvb292LFjh7h586bYsmWLMDMzUynQCg8PF507dxYAhKampmjdurUYNWqUcHd3LzZPdQ52srOzRf/+/UXz5s0Vmq1Ly5uamirCwsKEv7+/+PTTT4Wzs7OIj48vNX9QUJCwtrZWCFBVCXZe9eDBA6UfoRX8f//ggw8U9uvfv794//33VSq7Xr16YvLkycVuLyr/Dz/8IOrWrSsOHDggbty4IX799VdRo0aNQo8GisobFBQkmjZtKr/uevXqJfr06SN69+6tsF9R3yWqBDulfReVFuyUll8qlYo2bdqI3r17i+zsbKXzJiUlidDQUOHn5yf69+8vWrRoUSjQLCr//v37haurq0hJSZGnFfX+KvsdfOrUqSIfoRWVv+D/ube3t8K+jRs3FnPnzlW67PT0dGFiYiJ+/PHHIrcXl3/y5MmiTZs24uTJk+L69evim2++ESYmJuLmzZul5j127JioXbu2kEgkQlNTU3z44YeiRYsWYuLEiSW8OxWDwU4lyMrKEpqamoUu/I8//lgMGDBApWOVNdjx8vIS9vb24uHDhyrnfVWPHj2Uirz37t0r/9IseAGQX9hF3SWXplWrVgr/gYvj6OiocJclhBC///67sLOzU6m8R48eCQ0NDbFv3z6l89jb24tVq1YppC1atEjUq1dPpbKFyP+xLwhWhg8fLvr27Vvsvq9eGwU/0K8GKJ07dxZTpkwpNf/LyhPsZGdni0GDBokmTZqIZ8+eqZT3Va6urkW2kr2af8WKFfLr7OVrT0NDQzg5OZWpbAsLC7FmzZpSy87KyhJaWlpi0aJFCvvNnj1btG/fXumyz507JwCI69evF1unV/Onp6cLbW3tQv29xo4dK3r16qV02UlJSSIhIUEIkd/fcNKkSfJtxX2XFPxAvxqgODo6iuXLl5ea/2UlBTul5U9OThYeHh6iR48ehQIVVb4Hs7KyhIGBgdi+fXup+adOnVrs9dalSxeVy05NTRUAxNGjR0st++HDhwKA+OOPPxTShw8fLm9FVabsLVu2CG1tbfnn/rLi8oeHhxfq5yVE/m/EhAkTlC776dOn8s/a2tpaLFu2rNh9Kwr77FQCHR0dtGzZEqdOnZKnyWQynDp1SqW+L2UhhMDkyZOxd+9enD59Gi4uLuU+pkwmkz/fL0mPHj1w69YtXL9+Xf5q1aoVRo0ahevXr0NTU1OlclNTU/HgwQPY2tqWum+HDh0KDXMMDQ2Fk5OTSmX6+vrCysoK/fr1UzpPeno6NDQU/ytpamqWaYSBoaEhbG1tkZiYiGPHjmHgwIFK53VxcYGNjY3CdZecnIyAgIBKv+4K5OTkYPjw4QgLC8PJkydhbm5eruMpe+199NFHuHnzpsK1Z2dnh1mzZuHYsWMql/vkyRM8f/5cqWtPR0cHrVu3Lvf1t2HDBrRs2VLpPkpA/vudk5NT7uvPxMQElpaWCAsLQ1BQEAYOHFjqd0nLli2hra2tcL2FhIQgMjISHh4e5f4uUiZ/cnIyPD09oaOjgwMHDsj7H5WlbJF/84+srKxS88+dO7fQ9QYAK1aswMaNG1UuuyC/ra1tqWU7OzvDzs6uyOvN0dFR6bI3bNiAAQMGwNLSUuE9KCl/Qb+ioq63vLw8pcu2sLBAzZo1cfr0aSQkJMhHbFaqSg+n3lI7d+4Uurq6YtOmTeLu3bti/PjxombNmiIuLq7UvCkpKSI4OFgEBwcLAPJ+AK+OaCnK//73P2FiYiLOnj0rYmNj5a/09HSl6j137lzh5+cnIiIixM2bN8XcuXOFRCIRx48fVyr/q1R5jDVz5kxx9uxZERERIS5evCh69uwpLCwsirzzeNWVK1eElpaW+P7770VYWJjYtm2bMDAwEFu3blW6rnl5ecLR0VHMmTNH6TxC5I/kqVWrljh06JCIiIgQe/bsERYWFoWa8kty9OhRceTIEfHw4UNx/Phx0bRpU9G2bdtCTfKlXRtLliwRNWvWlPc/GThwoHBxcZHf8ZaW//nz5yI4OFgcPnxYABA7d+4UwcHBIjY2ttT82dnZYsCAAcLe3l5cv35d4frLysoqMW9qaqrw9vYW/v7+4tGjRyIoKEiMGTNG6Orqyu8iVf1/8fJjrJLypqSkiC+++EL4+/uLiIgIcfLkSdGiRQvh5uYmMjMzlSp7z549QltbW6xbt06EhYWJX3/9VWhqaorz588rVW+pVCoMDAzE6tWrC51Hafm7dOkiGjZsKM6cOSMePnwofH19hZ6envj9999Lzfvnn3+KM2fOiAcPHoh9+/YJJycnMWTIECGEct8lEydOFI6OjuL06dMiKChIeHh4yB8nK5M/NjZWBAcHi/Xr1wsA4ty5cyI4OFg8f/681PxSqVS0bdtWNG7cWISHhyvsM3HixBLzPnjwQCxevFgEBQWJx48fi4sXL4r+/fsLMzMzER8fX6bvUfzbclZa3vDwcLFw4UIRFBQkIiIixP79+0Xt2rVF586dlX7fVqxYIYyNjcXu3btFWFiY+Oqrr4Senp4YOXKkUvUOCwsTEolEHDlyRCG9tLKzs7OFq6ur6NSpkwgICBDh4eHixx9/FBKJRPTt27fUsjdu3Cj8/f1FeHi4+OOPP4SZmZmYMWNGse9pRWKwU4l+/fVX4ejoKHR0dESbNm3E5cuXlcpX0KT76mv06NGl5i0qHwDh6+urVNmffvqpcHJyEjo6OsLS0lL06NGjzIGOEKoFOyNGjBC2trZCR0dH1KpVS4wYMaLIDoPFOXjwoGjUqJHQ1dUV7u7uYt26dSrV9dixYwKACAkJUSlfcnKymDp1qnB0dBR6enqidu3a4ssvvxRZWVlKH2PXrl2idu3aQkdHR9jY2AgvLy+RlJRUaL/Srg2ZTCbmz58vrK2tha6urujRo4fC+ZSW39fXt8jtCxYsKDV/waOvol5nzpwpMW9GRoYYPHiwsLOzEzo6OsLW1lYMGDBAoYOyqv8vXg52Ssqbnp4uPD09haWlpdDW1hZOTk5i3LhxCjcmypS9YcMG4erqKvT09ETTpk3lj0KVybt27Vqhr69fps88NjZWfPLJJ8LOzk7o6emJevXqiZ9++knIZLJS8/7yyy/C3t5eaGtrC0dHR/HVV1/Jr1tlvksyMjLEpEmThKmpqTAwMBCDBw+WB8bK5F+wYEGx+5SWv7hzK+lVkDc6Olr06dNHWFlZCW1tbWFvby9Gjhwp7t+/r3TdX1UQ7JSWNzIyUnTu3FmYmZkJXV1d4erqKmbNmiXv26Zs2T4+PsLe3l4YGBgIDw8Pcf78eaXzent7CwcHB5GXl1foHErLHxoaKoYMGSKsrKyEgYGBaNKkidiyZYtSeefMmSOsra2Ftra2cHNzk1+nr4Pk3xMkIiIiUkvss0NERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BARveLs2bOQSCRISkqq6qoQUQVgsENERERqjcEOERERqTUGO0RU7chkMvj4+MDFxQX6+vpo2rQp/vrrLwD/PWI6fPgwmjRpAj09PbRr1w63b99WOMbff/+Nhg0bQldXF87Ozvjpp58UtmdlZWHOnDlwcHCArq4uXF1dsWHDBoV9rl69ilatWsHAwADt27cvtNI0Eb0ZGOwQUbXj4+ODLVu2YM2aNbhz5w6mT5+ODz/8EH5+fvJ9Zs2ahZ9++gmBgYGwtLRE//79kZOTAyA/SBk+fDjef/993Lp1C9988w3mz5+PTZs2yfN//PHH2LFjB1auXIl79+5h7dq1qFGjhkI9vvzyS/z0008ICgqClpYWPv3009dy/kRUsbgQKBFVK1lZWTAzM8PJkyfh4eEhT//ss8+Qnp6O8ePHo1u3bti5cydGjBgBAHjx4gXs7e2xadMmDB8+HKNGjcLTp09x/Phxef7Zs2fj8OHDuHPnDkJDQ1GvXj2cOHECPXv2LFSHs2fPolu3bjh58iR69OgBAPjnn3/Qr18/ZGRkQE9Pr5LfBSKqSGzZIaJqJTw8HOnp6XjnnXdQo0YN+WvLli148OCBfL+XAyEzMzPUq1cP9+7dAwDcu3cPHTp0UDhuhw4dEBYWhry8PFy/fh2ampro0qVLiXVp0qSJ/N+2trYAgISEhHKfIxG9XlpVXQEiopelpqYCAA4fPoxatWopbNPV1VUIeMpKX19fqf20tbXl/5ZIJADy+xMR0ZuFLTtEVK00aNAAurq6iIyMhKurq8LLwcFBvt/ly5fl/05MTERoaCjq168PAKhfvz4uXryocNyLFy+ibt260NTUROPGjSGTyRT6ABGR+mLLDhFVK0ZGRvjiiy8wffp0yGQydOzYEVKpFBcvXoSxsTGcnJwAAAsXLoS5uTmsra3x5ZdfwsLCAoMGDQIAzJw5E61bt8aiRYswYsQI+Pv7Y9WqVfj9998BAM7Ozhg9ejQ+/fRTrFy5Ek2bNsXjx4+RkJCA4cOHV9WpE1ElYbBDRNXOokWLYGlpCR8fHzx8+BA1a9ZEixYtMG/ePPljpCVLlmDq1KkICwtDs2bNcPDgQejo6AAAWrRogT///BNff/01Fi1aBFtbWyxcuBCffPKJvIzVq1dj3rx5mDRpEp4/fw5HR0fMmzevKk6XiCoZR2MR0RulYKRUYmIiatasWdXVIaI3APvsEBERkVpjsENERERqjY+xiIiISK2xZYeIiIjUGoMdIiIiUmsMdoiIiEitMdghIiIitcZgh4iIiNQagx0iIiJSawx2iIiISK0x2CEiIiK1xmCHiIiI1Nr/A45Ofu2hDDlOAAAAAElFTkSuQmCC", "text/plain": [ "
" ] diff --git a/tests/test_nonsequential/non-sequential-SCNN-example_3.ipynb b/tests/test_nonsequential/non-sequential-SCNN-example_3.ipynb index 87457657..956dfb88 100644 --- a/tests/test_nonsequential/non-sequential-SCNN-example_3.ipynb +++ b/tests/test_nonsequential/non-sequential-SCNN-example_3.ipynb @@ -51,7 +51,7 @@ "source": [ "batch_size = 3\n", "num_workers = 1\n", - "epochs = 20\n", + "epochs = 30\n", "lr = 1e-3" ] }, @@ -368,7 +368,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "45a1007196c04fa1ae9165909607816e", + "model_id": "12d134e3b89e41888c9c47892b8e6491", "version_major": 2, "version_minor": 0 }, @@ -382,7 +382,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "7ea2c1334aed43caae515c98f6c52613", + "model_id": "dad714bd3c9c44178729964e549b1349", "version_major": 2, "version_minor": 0 }, @@ -403,7 +403,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "658d33a8d118468da81a8aa69cf87cad", + "model_id": "86551db99b2f431fa1808c939bf9c76a", "version_major": 2, "version_minor": 0 }, @@ -417,7 +417,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "2a8b6a64493742a59809d41f1c4966c5", + "model_id": "e65960569a0f405986462743730225f5", "version_major": 2, "version_minor": 0 }, @@ -438,7 +438,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "ef7900458d00429db2059380a78791bd", + "model_id": "f494a16eed7847d5a4508cb4229eb5c3", "version_major": 2, "version_minor": 0 }, @@ -452,7 +452,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "49a89617326549fea8aa9e3cb8c45f58", + "model_id": "df363ac148bc4c11aa6a0946d03cd107", "version_major": 2, "version_minor": 0 }, @@ -473,7 +473,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "2637020c90044f8491e361ba04fb4ad3", + "model_id": "fa613935f3a748c7a70b30f5d8b12dce", "version_major": 2, "version_minor": 0 }, @@ -487,7 +487,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "929d0196596949b4b3872dc9ee3a832a", + "model_id": "76ac8052ee86420d9976c1c4914de629", "version_major": 2, "version_minor": 0 }, @@ -508,7 +508,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "877d9a3f07c74adf8bbea7c9f1e824ab", + "model_id": "a58539406c5e44be97c7bde3a77ab9b6", "version_major": 2, "version_minor": 0 }, @@ -522,7 +522,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "37854898408b4e6b8881e1289b383578", + "model_id": "c8225a8c10f04c37a84c89697de55ce1", "version_major": 2, "version_minor": 0 }, @@ -543,7 +543,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "347d383af65d42e680e4b63ae483ca31", + "model_id": "e308d4881a234b569c8a421c85c7d070", "version_major": 2, "version_minor": 0 }, @@ -557,7 +557,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "fbfdf26e181c4b7a9409f66c1e3fdadb", + "model_id": "07ebd6f1b5cb4f2190eed59bd1e7b5e5", "version_major": 2, "version_minor": 0 }, @@ -578,7 +578,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "c591461535214b4297233ed222a45917", + "model_id": "3d36f57679424fb6b74680a1a3a62e84", "version_major": 2, "version_minor": 0 }, @@ -592,7 +592,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "face11d4c6154225a2e86b5219337f10", + "model_id": "6e1648bc2958408fadc6bc5ae3aa3256", "version_major": 2, "version_minor": 0 }, @@ -613,7 +613,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "c2c6ac2419c94f7fab58d68a09a47114", + "model_id": "a5a767dca24f4545bf84111c5807314d", "version_major": 2, "version_minor": 0 }, @@ -627,7 +627,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "5833e29af9c44851bf01d005df36b867", + "model_id": "672840d4d3af4bc78c75e364c3d69963", "version_major": 2, "version_minor": 0 }, @@ -648,7 +648,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "2afa91087e1d4733ba4b3473bc436cc6", + "model_id": "01a26da2bdce4bca9d39dd8984d042b5", "version_major": 2, "version_minor": 0 }, @@ -662,7 +662,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "c52fbf464920441d8bb0f76749f70c2b", + "model_id": "177fd37e0eab4795bfc59d6bd97a9aad", "version_major": 2, "version_minor": 0 }, @@ -683,7 +683,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "4b353e35d828486387b13f52eb67fcdf", + "model_id": "afe69feb614442dcbd658ff365af973b", "version_major": 2, "version_minor": 0 }, @@ -697,7 +697,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "0974622ad5144710ad9bdf0875356453", + "model_id": "6444a13035014508bc47550bb1fb3c90", "version_major": 2, "version_minor": 0 }, @@ -718,7 +718,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "69a472b798f342438e07ce8be2348161", + "model_id": "4f36db6994a540d4ac158065ab0b1f7b", "version_major": 2, "version_minor": 0 }, @@ -732,7 +732,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "70583030430d4bdcbf23234dccdb6034", + "model_id": "a786978ce8ba4dd282ff8428e1491a48", "version_major": 2, "version_minor": 0 }, @@ -753,7 +753,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "15d31b5963bb4c859c4cfb5b5df10a22", + "model_id": "26b4d34e0f344e81b33d43fc1b5a86ef", "version_major": 2, "version_minor": 0 }, @@ -767,7 +767,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "1aa8ab5e7a15447e9e6ff314f0d212aa", + "model_id": "67d64e70242e488b8d5bf13fe1e9f1b3", "version_major": 2, "version_minor": 0 }, @@ -788,7 +788,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "58858ba00662421497ca4311c4e28c95", + "model_id": "663e7f38eb8a4ed0b1f6cde5dfb3d431", "version_major": 2, "version_minor": 0 }, @@ -802,7 +802,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "b719e05acda94ed09c6d1cb94035cf39", + "model_id": "f3f4a084b59348d18341497916f46b9c", "version_major": 2, "version_minor": 0 }, @@ -823,7 +823,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "db9a69b1d1054e4da4bdb7672cb988d7", + "model_id": "478eb58da5454eed9de461a91107846f", "version_major": 2, "version_minor": 0 }, @@ -837,7 +837,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "9bfe2be7c8cd41ffbe4637dca15c4231", + "model_id": "45c6cb33606b46bc877c437fb8cd5754", "version_major": 2, "version_minor": 0 }, @@ -858,7 +858,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "f2a2cd7932704a5597f0d982f8e298b0", + "model_id": "dadbf061a1f14ded927d6935b9935596", "version_major": 2, "version_minor": 0 }, @@ -872,7 +872,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "fa72f47975dd47a6bd47f0a42b7ca9ab", + "model_id": "c4aaedbf57d9415abe757bac082f1e34", "version_major": 2, "version_minor": 0 }, @@ -893,7 +893,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "52b53a55b1b64f11a37e32c55899a723", + "model_id": "a7603a6174414132b23c65a9942b5a2d", "version_major": 2, "version_minor": 0 }, @@ -907,7 +907,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "10b6d7c7fbf941d7883fe6c6a3e201d1", + "model_id": "5e6ee57580a9484d86bdb93c6632d3bb", "version_major": 2, "version_minor": 0 }, @@ -928,7 +928,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "ef5c7e3ffa174f8ca31b8270a551e689", + "model_id": "f9c3cd8d3662410e9cb717a589f2fe37", "version_major": 2, "version_minor": 0 }, @@ -942,7 +942,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "4df102b34cec42bcacef09b7c150a85c", + "model_id": "eaa3334ff93c4ca88c821742d8318b0d", "version_major": 2, "version_minor": 0 }, @@ -963,7 +963,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "7bea771bde7e4c298a3eea098c18b0b3", + "model_id": "c4b3a84a81d248eb9114745ad67d140e", "version_major": 2, "version_minor": 0 }, @@ -977,7 +977,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "aaca6e2774c44bdaa8cbd89424d309a5", + "model_id": "ec719eac2f0f45cdafa00a2a0dbac285", "version_major": 2, "version_minor": 0 }, @@ -998,7 +998,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "60fdaacfeeef447abdf47180e8ebbbda", + "model_id": "27fd1b40937745938d996b9cf53d8752", "version_major": 2, "version_minor": 0 }, @@ -1012,7 +1012,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "84040fabb0be4920a7d882fb6294562f", + "model_id": "296f1b9cdf6a44f491c8387d7570098d", "version_major": 2, "version_minor": 0 }, @@ -1033,7 +1033,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "b1a3bd77451244508356fa7831abb486", + "model_id": "d9bde1c861fc4e14a8bad10a665baeb4", "version_major": 2, "version_minor": 0 }, @@ -1047,7 +1047,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "4b4ba12c3ce54bbdb784de9ef47352a8", + "model_id": "c44e170962a546c3a52d014dc50586f2", "version_major": 2, "version_minor": 0 }, @@ -1064,6 +1064,356 @@ "text": [ "Epoch 19 accuracy: 85.60606060606061\n" ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "7aa75420aaf2498db249299e4d5f7225", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/359 [00:00" ] @@ -1097,21 +1447,21 @@ "plt.ylim(0,)\n", "plt.xticks(np.arange(len(epochs_x)))\n", "for i, txt in enumerate(y_avg):\n", - " if i%2 == 0:\n", - " pass\n", - " else:\n", + " if i%5 == 0:\n", " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", + " else:\n", + " pass\n", "plt.show()" ] }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 22, "metadata": {}, "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAG2CAYAAACZEEfAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABpBElEQVR4nO3deXhM5/sG8Hsy2SMJ2RNkISTEvsROkSbU115b1V7VNlpL7a2lVUIXVKmtBLVXbaUVkRK1BSGIEhEkEVlEZLIvZt7fH2nmZyTIyGQb9+e65rrMmTPPPBOZnHvOec97JEIIASIiIiItpVPRDRARERGVJYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moVGnZOnTqF3r17w8HBARKJBAcOHFB5XAiBefPmwd7eHkZGRvDy8kJkZKTKOikpKRg+fDjMzMxQvXp1jBs3DhkZGeX4LoiIiKgyq9Cwk5mZiaZNm2L16tXFPv7tt99i5cqVWLt2LUJCQmBiYgIfHx/k5OQo1xk+fDhu3LiBwMBAHD58GKdOncKHH35YXm+BiIiIKjlJZbkQqEQiwf79+9GvXz8ABXt1HBwc8Pnnn2PatGkAAJlMBltbW2zevBlDhw7FzZs30bBhQ1y8eBGtWrUCABw9ehTvvPMOHjx4AAcHh4p6O0RERFRJ6FZ0Ay9y7949JCQkwMvLS7nM3Nwcbdq0wblz5zB06FCcO3cO1atXVwYdAPDy8oKOjg5CQkLQv3//Ymvn5uYiNzdXeV+hUCAlJQWWlpaQSCRl96aIiIhIY4QQSE9Ph4ODA3R0XnywqtKGnYSEBACAra2tynJbW1vlYwkJCbCxsVF5XFdXFxYWFsp1iuPn54evvvpKwx0TERFRRYiNjUWtWrVe+HilDTtlafbs2Zg6daryvkwmg6OjI2JjY2FmZlaBnREREVFJpaWloXbt2jA1NX3pepU27NjZ2QEAEhMTYW9vr1yemJiIZs2aKddJSkpSed7Tp0+RkpKifH5xDAwMYGBgUGS5mZkZww4REVEV86ohKJV2nh0XFxfY2dkhKChIuSwtLQ0hISFo164dAKBdu3ZITU1FaGiocp2///4bCoUCbdq0KfeeiYiIqPKp0D07GRkZuHPnjvL+vXv3EBYWBgsLCzg6OmLy5Mn45ptvUK9ePbi4uGDu3LlwcHBQnrHVoEED9OjRA+PHj8fatWuRn5+PiRMnYujQoTwTi4iIiABUcNi5dOkSunbtqrxfOI5m1KhR2Lx5M2bMmIHMzEx8+OGHSE1NRceOHXH06FEYGhoqn7N9+3ZMnDgR3bt3h46ODgYOHIiVK1eW+3shIiKiyqnSzLNTkdLS0mBubg6ZTMYxO0RERFVESbfflXbMDhEREZEmMOwQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQERGRVmPYISIioiLkcjnmzp0LFxcXGBkZoW7duli4cCGEEMp1Ro8eDYlEonLr0aPHK2vHxcXh/fffh6WlJYyMjNC4cWNcunRJ+fi+ffvg7e0NS0tLSCQShIWFleq96Jbq2URERKSVli5dijVr1mDLli3w8PDApUuXMGbMGJibm+Ozzz5TrtejRw/4+/sr7xsYGLy07pMnT9ChQwd07doVf/31F6ytrREZGYkaNWoo18nMzETHjh0xePBgjB8/vtTvhXt2iIheoSTfcJ/10UcfQSKRYMWKFaWu+7rfnLWpf6oYZ8+eRd++fdGrVy84Ozvj3Xffhbe3Ny5cuKCynoGBAezs7JS3Z0NLcZYuXYratWvD398fnp6ecHFxgbe3N+rWratcZ8SIEZg3bx68vLw08l64Z4eI6BVK+g0XAPbv34/z58/DwcFBY3XV/easbf1TxWjfvj3Wr1+P27dvo379+rh69SpOnz6NZcuWqax38uRJ2NjYoEaNGujWrRu++eYbWFpavrDuoUOH4OPjg0GDBiE4OBg1a9bEJ598opE9OC/CsENE9ArPfsMFAGdnZ+zcubPIN9y4uDh8+umnCAgIUK6ribqF35zf1P6pYsyaNQtpaWlwd3eHVCqFXC7HokWLMHz4cOU6PXr0wIABA+Di4oKoqCjMmTMHPXv2xLlz5yCVSoute/fuXaxZswZTp07FnDlzcPHiRXz22WfQ19fHqFGjyuS98DAWEdErtG/fHkFBQbh9+zYAKL/h9uzZU7mOQqHAiBEjMH36dHh4eGisLvD/35zd3Nzw8ccf4/Hjx29U/1VZWR1CBIDVq1fD2dkZhoaGaNOmTZGQGRUVhf79+8Pa2hpmZmYYPHgwEhMTS9z7nj17sH37duzYsQOXL1/Gli1b8P3332PLli3KdYYOHYo+ffqgcePG6NevHw4fPoyLFy/i5MmTL6yrUCjQokULLF68GM2bN8eHH36I8ePHY+3atSXuTW2ChEwmEwCETCar6FaIqBKSy+Vi5syZQiKRCF1dXSGRSMTixYtV1lm8eLF4++23hUKhEEII4eTkJJYvX17qujt37hQHDx4U165dE/v37xcNGjQQrVu3Fk+fPn1j+q/KFi1aJCwtLcXhw4fFvXv3xG+//SaqVasmfvzxxyLr7tu3TzRt2lQ4ODi88me/a9cuoa+vLzZt2iRu3Lghxo8fL6pXry4SExOFEEJkZGSIOnXqiP79+4tr166Ja9euib59+4rWrVsLuVxeot5r1aolVq1apbJs4cKFws3N7aXPs7KyEmvXrn3h446OjmLcuHEqy37++Wfh4OBQZN179+4JAOLKlSvF1irp9pthRzDsENHL7dy5U9SqVUvs3LlTXLt2TWzdulVYWFiIzZs3CyGEuHTpkrC1tRVxcXHK55QkLLyqbnGioqIEAHH8+PE3ov+nT5+KL7/8Ujg7OwtDQ0NRp04d8fXXXytDmRBCzJ8/X7i5uQljY2NRvXp10b17d3H+/PmX1nVychIAitw++eQT5Trr1q0TXbp0EaampgKAePLkSYl6flavXr3E2LFjVZYNGDBADB8+XGXZgwcPRM2aNUV4eHiJfvaenp7C19dXeV8ulwsHBwfh5+cnhBAiICBA6OjoqGzXUlNThUQiEYGBgSXq3cLCQvz8888qyxYvXizq1av3wufExsYKiUQiDh48+MJ1hg0bJjp27KiybPLkyaJdu3ZF1mXY0SCGHSJ6mVd9w12+fLmQSCRCKpUqbwCEjo6OcHJyeu26L/Kqb87a1H9J9oxs375dBAYGiqioKBEeHi7GjRsnzMzMRFJS0gvrJiUlifj4eOUtMDBQABAnTpxQrrN8+XLh5+cn/Pz8XjvsLFq0SDg5OYmIiAghhBBhYWHCxsZGbNu2TbmOXC4XXbt2FStWrBBCvDpo5ubmCqlUKvbv36+yfOTIkaJPnz5CCCEOHTokpFKpyMnJUT6ek5MjpFKpmD9/fol6HzVqlKhZs6byZ79v3z5hZWUlZsyYIYQQIj09XUybNk2cO3dO3Lt3Txw/fly0aNFC1KtXT+V1u3XrJn766Sfl/QsXLghdXV2xaNEiERkZKbZv3y6MjY1VfiaPHz8WV65cEUeOHBEAxK5du8SVK1dEfHy8So8l3X5zgDIR0StkZWVBR0d1iKNUKoVCoQBQcJrs86fI+vj4YMSIERgzZsxr1y3OgwcP8PjxY9jb278R/ZdkEPR7772n8pxly5Zh48aNuHbtGrp3715sXWtra5X7S5YsQd26ddGlSxflssmTJwPAS8efvEpJBvkuXboUurq6Rc6Me5Hk5GTI5XLY2tqqLLe1tcWtW7cAAG3btoWJiQlmzpyJxYsXQwiBWbNmQS6XIz4+vkSv89NPP2Hu3Ln45JNPkJSUBAcHB0yYMAHz5s0DUPB/fe3aNWzZsgWpqalwcHCAt7c3Fi5cqHLG3e3IO7hyOwbxsmzYmxuhdevW2L9/P2bPno2vv/4aLi4uWLFihcrP5NChQyq/e0OHDgUAzJ8/HwsWLChR/ypKFO+0HPfsENHLvOobbnGK+3b+/DdcTX1z1ub+S7Jn5Fm5ubniu+++E+bm5uLRo0cleo3c3FxhaWkpFi1aVOzjJ06ceO09O2VxCDEuLk4AEGfPnlVZPn36dOHp6am8HxAQIOrUqaPca/f++++LFi1aiI8++kjt9/G6dl2IFi6zDgunmYeFy6zDYteFaI3W52EsNTDsENHLpKWliUmTJglHR0fluJEvvvhC5ObmvvA5xW2wnJycVA4hvKpuVlaW8Pb2FtbW1kJPT084OTmJ8ePHi4SEhDem/5IMghZCiD/++EOYmJgIiUQiHBwcxIULF0r8Grt37xZSqVQlcDyrNGGnLA4hluQw1rMePXqk7N3W1lZ8++23ar+P1/EwNUsZdApvdWYdEQ9TszT2GiXdfkuEeMH5b2+QtLQ0mJubQyaTwczMrKLbISItFS/Lxr3kTLhYmcDe3Kii21FbRfS/a9cuTJ8+Hd999x08PDwQFhaGyZMnY9myZSpzsmRmZiI+Ph7JycnYsGED/v77b4SEhMDGxuaVr+Hj4wN9fX388ccfxT5+8uRJdO3aFU+ePEH16tXV6t/S0hLffPMNPv74Y+UyPz8/+Pv74/bt23j8+HGRw0rPHkJ0c3Mrtm6bNm3g6emJn376CUDB6dyOjo6YOHEiZs2aVexz/v77b3h5eeHmzZsvrKtJZ6OS8d6GkCLLd45vi3Z1XzzpoDpKuv3mmB0ionKw+2IMZu+7DoUAdCSA34DGGNLasaLbKrGK6n/69OmYNWuWcsxG48aNER0dDT8/P5WwY2JiAldXV7i6uqJt27aoV68eNm7ciNmzZ7+0fnR0NI4fP459+/aVSf+9e/fGokWL4OjoCA8PD1y5cgXLli3D2LFjARSEoednG9bT04OdnZ1KIOnevTv69++PiRMnAgCmTp2KUaNGoVWrVvD09MSKFSuQmZmpMs7F398fDRo0gLW1Nc6dO4dJkyZhypQp5RJ0ACDkbkqRZVKJBM5WxuXy+s9i2CEiKmPxsmxlUAAAhQDm7AtH5/rWVWIPT7wsG7P2XYd4pv/Z+66XS/+vMwgaKNjTkZub+8r6/v7+sLGxKdGM0a/jVYN8SyoqKgrJycnK+0OGDMGjR48wb948JCQkoFmzZjh69KjKoOWIiAjMnj0bKSkpcHZ2xhdffIEpU6Zo7L29zJ6LsfgxKBIAIEHBef1SiQSLBzSqkN95HsYCD2MRUdnJfSrHvIM3sPtibJHHNLk7vywdvBKHSbvDiizvUNcSH3Sqg471rKAnLZsJ+UePHo3jx49j3bp1yj0jH374IcaOHYulS5ciMzMTixYtQp8+fWBvb4/k5GSsXr0aO3bsQGhoqHI26Of3jAAFgcjFxQXDhg3DkiVLirx2QkICEhIScOnSJYwfPx6nTp2CqakpHB0dYWFhUSbv90Wq0iHQI9fi8enOy1AI4MPOdTC6vROiH2fD2cpY473zMBYRUQU7cSsJXx/+F/eSM4t93LKafjl3pL4nmXn4ITCi2MfORD3GmajHsDDRR6/G9ujbzAEtnWpAIpFo7PVLcvrzrVu3sGXLFiQnJ8PS0hKtW7fGP//8o3LZi+dPfwaA48ePIyYmRnlI6Xlr167FV199pbzfuXNnAAV7g0aPHq2x9/gqVekQ6ImIJEzefQUKAQzzdMTsnu6QSCRwqF7+h66exT074J4dItKs6MeZ+PqPfxF0KwkAYG1qgK5u1vg99AHkz/zFbVfHEpvHtoaBbvEXTKxoOflyvLfhPC7HpMLcSBfpOU+hEIBUAnz0lisyc5/i8LWHSM7IUz6nVg0j9G3mgL7NaqK+rWkFdv//qlJYeF54nAz/++m0yjKpRILTs7pWuj08IXcfY+SmC8h9qkDvpg5YMaQZpDqaC77FKen2m2EHDDtEpBlZeU+x+sQdbDh1D3lyBXR1JBjb0QWfdnOFqaEe4mXZuJ+chbyncvjuuIKM3Kf4XxN7rBzaHDplvFFQl1wh8NG2UAT+mwgzQ138/nF7VDPUxf3kLJXDEU/lCpyNeowDYXEICE9AZp5cWaOBvRn6NnNAn6YOcKhe/hvmR+m5OBuVjMm7wvDshq6yhoVn5csV2HouGt8H3EJ2ftHxSZXtEOi1B6l4b0MIMnKforu7DdaOaFlmhzafxbCjBoYdIioNIQQOX4vH4j9vIl6WAwDoVM8K83t7wNWmWrHPOR2ZjDGbLyBfLvBBRxd8+b+G5dnySwkh8OWBcGwPiYG+rg62jWsDT5dXj1HJyZfj+M1EHLjyEMG3k5D/324siQTwdLZA32Y18U5jO1Q31uzhu+w8OSKT0nErIR234tMRkZiGiIR0lT1Oz1s5tBn6NKup0T405eydZMw/dAORSRkvXGflsGbo07Ry9B+ZmI7B687hSVY+2tWxhP+Y1jDUK5+9lQw7amDYIaLXdSshDQsO3cD5/06zrVXDCHP/1xDeDW1fOXblwJU4TP5v4O+XvRrgg051yrrdEln1dyS+P3YbEgmwZngL9GhU8ktTFErNysOf1xNwMCwOIff+/xRkPakEb7nZoG8zB3g1sFVro6hQCMSkZBWEmoSCQBORkI57jzNR3JZMIgFqVTdC7JPsIo/p6Ujwfjsn+HZ1hVU1g6JPrgBxqdlYdORf/Hk9AQBQw1gPM3q4AwC+3B8O+TNvUqojwfzeDTGirZNGx0ipK+ZxFt5dexZJ6bloWrs6tn/QBtUMym84MMOOGhh2iEhdsux8LA+8jV/PR0OuEDDQ1cEnb7liQpc6am3A1wVHwe+vgusZrRzWHH2aOpRVyyWy51IsZuy9BgD4qo8HRrV3LnXNh6nZOHT1IQ6GPcTN+DTl8moGuvD2sEW/ZjXhYmWM2CfZyrONUjLzcCs+Dbf+CzS3EtNxOyEd2fnyYl/D0kQfbnamcLMzRQM7M7jZmaKebTUY6+ti98UYzNlXEBZ0JEAdaxPcSSoYNG6iL8W4TnUwvpMLTA31Sv1eX0dOvhzrT93FzyfvICdfAR0JMKKtE6a+7QZz44KeCg+B2psbYmVQJPZdiQMAvNfGEV/18SiXQ0bPS5DlYNC6s4hNyYabrSl2T2ir8b12r8KwowaGHSIqKYVCYM+lWHwbEIGUzILDJD0b2eGLXg1Qq4b6Z5wIIfD14X/hf+Y+9KQSbBnjifauVppuu0RORCThgy2XIFcIfPxWXcz8b6+CJt1OTMeBK3E4GPYQcalF97gABSEoI/dpsY8Z6Oqgnm01uNuZwf2/cONuZwZr05fvnSkMC4XjjU5HJuPbgFu49kAGoGAvim9XV7zf1qncDsEIIRD4byIWHvkXsSkFPwtPFwt81ccDDexfvC0SQmDdqbtYevQWhADauFhgzfstYWFSfkEjJTMPg9edw52kDDhZGuO3Ce1gY2ZYbq9fiGFHDQw7RJpTleYDUdeVmCeYf+iGcgPpalMNX/XxQIdShhOFQuDTnVdw5Ho8TA10sXtCOzR0KN+/RVdjUzF0/Xlk58sxoEVN/DCoaZkeHhFCIDT6CbaHRGP/lYfFruNkaQw3W9P/Qo0Z3O1N4WxporEzfIQQOBqegO+OReDuo4I9PQ7mhpj8dn0MaF4TumW4tyTqUQa++uNfnLr9CABgZ2aIOb0aoHcT+xL/3INuJuKznVeQmSdHbQsjbBzVulzOgEvLycd7G84jPC4N9uaG2DOhHWpbVMyp5Qw7amDYIdKMqnyK78s8Ss/F0qO3sDf0AQDA1EAXk7zqYVR7Z40dPsjJl2PUpgsIuZcCG1MD7Puk/WvtKXod95MzMXDNWTzOzEOnelbYNLp1uR0WedH1k/xHt0ZX91df10oTnsoV+P3yAywPjERCWsEAc1ebapjm7QYfj1ePvVJHRu5T/BQUiU1n7iFfLqAv1cEHnVzg29UVJq8x1uV2Yjo+2HIJMSlZMNGX4sehzeHV0PbVT3xN2XlyjNwUgov3n8DSRB+7J7R74SD88lDS7Xf5H+SjKs/Z2RkSiaTIzdfXF0DBrKMjRoyAnZ0dTExM0KJFC/z++++lqgkUTJfev39/WFtbw8zMDIMHD0ZiYmKZvlcquRddEiFeVvyhiqogX67AL//cRbfvTyqDzrstayFoWhd80KmORgOBoZ4U60e2Qn3bakhKz8Vo/4tIzXrx2USakpyRi1H+F/A4Mw+Napphzfvlc8pwIRcrEzy/o0YqkcDdvvzm6NGV6mBIa0ecnP4WvninAaob6+FOUgY+2haKfj+fxdmo5FcXeQUhBPZfeYBu35/EulN3kS8X6OZug4ApnTGjh/trBR0AqG9rigO+HdC2jgUy8+QY/+slrDkZhbLYj5H3VIGPtoXi4v0nMDXUxZaxnhUadNTBsENqu3jxIuLj45W3wMBAAMCgQYMAACNHjkRERAQOHTqE69evY8CAARg8eDCuXLny2jUzMzPh7e0NiUSCv//+G2fOnEFeXh569+79ymvkUPkIjnikDDqF5ELgfnJWxTT0muJl2TgblYxDV+Pwzo//4JsjN5Ge+xRNaplj3yft8f2gprAxLZuxCeZGetgy1hP25oa4k5SBD7ZcQs4LBuRqQmbuU4zdfBHRj7NQ28IIm0a3LtczaQDA3twIfgMaQ/rf3pOKvH6SoZ4U4zvXwakZXfFpN1cY6UlxNbZg/pgRG0Nw/b/Dl+oKj5Nh0NpzmLL7KpLSc+FkaYyNo1ph0+jWcLEyKXXfFib6+HVcGwxv4wghgKVHb2Hqnqsa/d15Kldg8u4rCL79CEZ6UviPbo1GNc01Vr+s8TAWeBirtCZPnozDhw8jMjISEokE1apVw5o1azBixAjlOpaWlli6dCk++OCD16p57Ngx9OzZE0+ePFH+H8lkMtSoUQPHjh2Dl5dXmbw3ejWFQmDTmXtY+tct5D+fdgBsGNkSbze0q4DO1PfsYbhCFib6mOHjhsGtapfbxH+3E9Px7pqzSMt5Cu+GtljzfkuNz0SbL1dg/NZLOBnxCDWM9fD7x+1Rx7rivqU/P4C4MkhKz8Hqv+9gx4UY5ZxBvZrY4/O365foZ/UkMw/fH4vAzgsxUAjASE+Kid1cMa6jS5kNgv713H0s+ONfyBUCzWpXx/oRLUs9cFihEJj5+zX8FvoA+lIdbBzdCp3qWWuo49LhYSwqF3l5edi2bRvGjh2rPK7dvn177N69GykpKVAoFNi1axdycnLw1ltvvXbN3NxcSCQSGBj8/xkXhoaG0NHRwenTp4vUKPx2/vwhlLI4BAcAq1evhrOzMwwNDdGmTRtcuHBB5fEJEyagbt26MDIygrW1Nfr27Ytbt26V6OdRmSWm5WCU/wV8c+Qm8hUCDexMixyS8N1xBfuvPKiYBtUQ9yQLs35XDToSALvGt8VQT8dyneG4vq0pNoxsBX2pDo79m4gFh25o9LCEEAJz9l3HyYhHMNTTwabRrSs06AAFe3ja1bWsNEEHAGxMDfFV30YImvoW+jevCYmk4CKXby8/hdn7riPhvwkknydXCPx6PhpdfziJ7SEFQed/TewR9HkX+HZ1LdOzvUa0c8bWsZ4wN9JDWGwq+qw689p7pICC35WFR/7Fb6EPINWRYOWw5pUm6KijUocduVyOuXPnwsXFBUZGRqhbty4WLlyo8qEXQmDevHmwt7eHkZERvLy8EBkZWYFdv1kOHDiA1NRUlYvi7dmzB/n5+bC0tISBgQEmTJiA/fv3w9XV9bVrtm3bFiYmJpg5cyaysrKQmZmJadOmQS6XIz4+XuX5G/65i/Z+f+O9DSHosORv7L4Yo3ysLA7B7d69G1OnTsX8+fNx+fJlNG3aFD4+PkhKSlKu07JlS/j7++PmzZsICAiAEALe3t6Qy8vuEEVZOxoeD58Vp/BPZDIM9XTwTb9G+HNSJ5yZ1Q07x7fF8ald8HZDW+Q9VWDK7qtY8tctyIvZ81MZJKbl4KNtoXi+OwHgcWbZj5spTps6llgxtBkkEuDX89H4+WSUxmovC7yN30IfQEcCrH6vBZo71tBYbW3kaGmM5UOa4c/POqG7uw3kCoGdF2LQ5bsT8PvrJlKz8pRfsI6Gx6P3T6cx90A4UrPy4W5nip3j22LVey3K7ZIZHVytcNC3A+pamyAhrWAunD+uFn/G26ssPx4J/zP3AQDfDmyCHo2qxl7aIkQltmjRImFpaSkOHz4s7t27J3777TdRrVo18eOPPyrXWbJkiTA3NxcHDhwQV69eFX369BEuLi4iOzu7xK8jk8kEACGTycribWg1b29v8b///U9l2cSJE4Wnp6c4fvy4CAsLEwsWLBDm5ubi2rVrr11TCCECAgJEnTp1hEQiEVKpVLz//vuiRYsWYsKECeJq7BOxPDBC+CwPFk4zD6vcnGceFsduxAuFQlGk5qRJk0TdunWVj5mYmIitW7eqrGNhYSE2bNjwwn49PT2Fr6+v8r5cLhcODg7Cz8/vhc+5evWqACDu3Lnzyp/Hs5ycnAQKtsEqt08++UTcu3ev2McAiD179hRbLy8vT8yYMUM0atRIGBsbC3t7ezFixAgRFxenXOfEiRMvrGs3cpnotfKUiExML7a+XK4Q3x69qfy/GOt/QaRl56n1nsvaX9fjRbOvAor83jjNPCzqzDoiHqZmVWh//qfvKvv57VJsqev9eu6+st7OkGgNdPjmuXjvsXh3zRnlz7H+F38K5+d+dxrPPyr8T98V+U/lFdanLDtPjNoUouzph4BbQi4v+nfwRTacilI+d8vZe2XXaCmUdPtdqcfs/O9//4OtrS02btyoXDZw4EAYGRlh27ZtEELAwcEBn3/+OaZNmwagYByHra0tNm/ejKFDh5bodThm5/VER0ejTp062LdvH/r27Qug4IwpV1dXhIeHw8PDQ7mul5cXXF1dsXbtWrVrPi85ORk5ciD80VMM7NgIFm0HQNqs+HWf1aimGUa0dUKfpjVhpC9FXl4eHBwcMHXqVMyZMwcA4O3tDX19fWzduhXVq1fHnj17MG7cOFy9erXYPVN5eXkwNjbG3r170a9fP+XyUaNGITU1FQcPHizynMzMTHz55Zc4ePAgbt26BX39kk8E9ujRI5W9QeHh4Xj77bdx4sQJdOrUCY8ePVJZf/369fjuu+8QHx+PatWKHqaQyWR49913MX78eDRt2hRPnjzBpEmTIJfLcenSJeV7TEkpmO4/PE6GeQfDcf3geuREX8XXO07gc2836Ou+fCfxwbA4zNh7DblPFahvWw2/jGwNR8uKmZejUGbuUyw8/C92XYwFAHg4mKGHhx1WHI+EXAjlQNnKcOq83183sS74LqQ6Emwc1Qpvub3eKdkBNxLw8bZQKAQw2aseJnvV13Cnbw4hBE5GPMI3R/5F1H9z9BSSADjyWUc0dKj4AbxyhcDSo7ew/tRdAEAPDzssG9IUxvovH4i+80LB+DUAmO7jBt+uJdszX95Kuv0u32H3amrfvj3Wr1+P27dvo379+rh69SpOnz6NZcuWAQDu3buHhIQElcGp5ubmaNOmDc6dO/fCsJObm4vc3Fzl/bS0tGLXo5fz9/eHjY0NevXqpVyWlVVw5o2OjurGTyqVluisqeJqFop+nImgm0k4EZGE83cfI+1uGLJkKaheuyXM9KXoVM8aLRyrY8nRW0XGXehKJQiPS8PM36/jmyM38W7LWqiRGFrsIbghQ4bA0tISurq6MDY2fukhuOTkZMjlctjaqs5rYWtrW2RMzs8//4wZM2YgMzMTbm5uCAwMVCvoAIC1teqx8iVLlqBu3bro0qULJBIJ7OxUdzHv378fgwcPLjboAAWfl8JDeYVWrVoFT09PxMTEwNHREfr6+rC2scXPJ+5gRVAUnubrIifqAsaM/wiz32lQor77NqsJZ0sTjN96CbcTM9B39Wn8PLxlhV21+WpsKibtuoL7j7MgkQATOtfF1LfrQ19XB++2qlXpBsrO9HFHUlou9l+JwyfbL2PXh23RpFZ1tWqERqfgs51XoBDA0Na1Mal7vbJp9g0hkUjQ1d0G+lIdDN+oOk+QACDLLn4G6PIm1ZFgzjsNUN/WFHP2XcfRGwmIXpOFDSNbvnAep0NXH2LO/oKgM6FLHXzyVt3ybLlMVOqwM2vWLKSlpcHd3R1SqRRyuRyLFi3C8OHDARQMJgVQ7Iam8LHi+Pn54auvviq7xt8ACoUC/v7+GDVqFHR1///XyN3dHa6urpgwYQK+//57WFpa4sCBAwgMDMThw4eV63Xv3h39+/fHxIkTX1gzX67AxfspOHErCUG3knAt6AD0LGtDx9gcuQ9vQfb3BrTrOxLfzhiI1i41YKBbMOjP3FhPeR2cwm/n3g3t8FtoLLadj0FMShb8z9xH4u4VsPVoi6uPJbC2VUBPqoO5c+ciNTUVx48fh5WVFQ4cOIDBgwfjn3/+QePGjUv1Mxs+fDjefvttxMfH4/vvv8fgwYNx5swZGBq+3pkShQO5p06dWuykZ6GhoQgLC8Pq1avVqiuTySCRSFC9enUAQGxKFqbsDsOl6CcAgIb5t/EgOw3zPvd9SZWi+nZqhujoaABANID28wuWf/LJJ8oez507hy+++AIhISGQSqVo1qwZAgICYGRUfOhYsGBBkc+ym5ubMmimpKRg/vz5OHbsGGJiYmBtbY06rd5CtHNvCH1j2JsbYtngZiqhy97cqNKEnEI6OhIsHdgEyRm5+CcyGWM3X8TvH7eHk2XJTlu+k5SBcVsuIfepAt3dbfBNv0YVevFIbVLHpmCeoGe/YEklEjhbVeyey+e927IWXKyMMeHXUNyMT0O/1Wew9v2WaOWsejX7v28lYuruMAgBDG/jiFk93LXjd6UcDqm9tp07d4patWqJnTt3imvXromtW7cKCwsLsXnzZiGEEGfOnBEAxMOHD1WeN2jQIDF48OAX1s3JyREymUx5i42N5ZgdNQUEBAgAIiIioshjt2/fFgMGDBA2NjbC2NhYNGnSpMg4GCcnJzF//vxia678PVh8si1UNJp3VOUYePW27wpDMwsh1dUTznXqiu+//77YcThCCPEwNUucvZNcZLyFXK4QJyOSxODvDwlIdIT1gC+F08zDwnNRoPhiS6AAIMLDw1We0717dzFhwoRiXyc3N1dIpVKxf/9+leUjR44Uffr0KfY5hc8zNjYWO3bseOE6r7J7924hlUpVxtc86+OPPxYNGjRQq2Z2drZo0aKFeO+994RCoRC/h8YKj//+HzzmHRW/h8aKnj17ip49e6rdb1JSkoiPjxf3Yh6IsT8fEzZDvhEAxIiFm0TeU7k4e/asMDMzE35+fiI8PFzcunVL7N69W+Tk5Lyw5vz584WHh4eIj49X3h49eqR8/Pr162LAgAHi0KFD4tSl66LzpJVCt4aDMK7fXvhuDxWpmZVr/NCrpOfki3d+PCWcZh4WXb79WzxKf/HPplCCLFu09wsSTjMPi76rTovM3Pxy6PTNsutCtKgz64hynNeuC5V3LNSDJ1mi54qC3yHXOUfE7osxysfO3kkW9b/4UzjNPCw+23lZrfE9FaWkY3Yq9Z6d6dOnY9asWcrDUY0bN0Z0dDT8/PwwatQo5S77xMRE2NvbK5+XmJiIZs2avbCugYGByinMpD5vb+8Xngpbr169V56ufe7qTdxLzsTD1Cw8ycov2HtzxwTOsw7jhwvpANIBFFzJ+C03G3Rzt0GnBd4wK+FViV/07VxHR4Iu9a1xIj0UNjbWmDJuCPZejkdiWi42Bd8HACw8fBOfGdiiXV1LSCSSlx6C09fXR8uWLREUFKQcs6NQKBAUFKSy1+p5QggIIVQOp6pr48aN6NmzJxwcil4lOzs7Gzt27MDcuXNLXC8/Px+DBw+GEAJLlq3EZ7vClGdwtHKqgeVDmkGSlYJBAQHYs2eP2v0+ewjul48c0O34dqRUt0dwujVG+1/A7fWT8dlnn2HWrFnK9dzc3F5ZV1dXt8jhu0KNGjXC77//jgNX4vDJgXCkG9aBXffRSDj4PZYPagw9vYq5yvXrqmagC/8xrTHg57O4/zgL4zZfxM4P275w/EV6Tj5G+19EXGrB1cQ3jmr1yrEapL4hrR3Rub51pTv8WZya1Y2w9+N2+HzPVfwVnoAZe6/hSvQTuDuYYcmfN5H7VAGvBrb4flDTcp1uoaxV6t/6rKysl479cHFxgZ2dHYKCgpThJi0tDSEhIfj444/Lu10qoeImbnuWh4MZurkXBJymtapr/ANXeLhszOjRmPVOI0z1boijNxKw9bQZDu63x4FVCxB8Yyyca9rCMf3fVx6Cmzp1KkaNGoVWrVrB09MTK1asQGZmJsaMGQMAuHv3Lnbv3g1vb29YW1vjwYMHWLJkCYyMjPDOO++81nuIjo7G8ePHsW/fvmIf37t3L7KysjBy5MgS1SsMOtHR0fh2014M2RSGh7IcSHUkmNy9Hj5+qy50pTpYuPoHWFpaok+fPq/V97Ovdz34MN4fOR4hBro4dTUKDy5eQM9+76J9+/aIioqCu7s7Fi1ahI4dO760VmRkJBwcHGBoaIh27drBz88Pjo4Fg4pl2fmYeyAch/4LbS2daqCNfU18d9KsygWdQjamhtgy1hPvrjmLqw9k8N1+GRtGtipy0crCqf1vxqfBqpoBtozxhGU1fskrK5Xx8OeLGOvrYvV7LfBjUCR+DIrEzv8G6QNAXWsTrHqvebleMqQ8VOqw07t3byxatAiOjo7w8PDAlStXsGzZMowdOxZAwQCxyZMn45tvvkG9evXg4uKCuXPnwsHBQeXMGKo84mXZmLXvOp7fKdTR1Qq9mtijq5sN7MzLZir+QsePH0dMTIzy90hfVwd9mjqgT1MHHG1+BJM/n447+xYiMS8bodXtYdt7Kk5l14JjnAyNapojKioK9x7E42xUMlysTDBkyBA8evQI8+bNQ0JCApo1a4ajR48qx5IZGhrin3/+wYoVK/DkyRPY2tqic+fOOHv2LGxsXu+smpcN5AYK9vr06dOnyIDm4hQGnduRkRjw5Xp8tPc2hACc/5tbpHAOFiEE/P39MXLkyFIHhcK5lBbN/BRpOtUwZOFWPACw+JuFmPrlQqzu2Rlbt25F9+7dER4ejnr1ih9M26ZNG2zevBlubm6Ij4/HV199hU6dOiE8PBz/PsrD1D1XEZeaDamOBJO618PgRuZo4zkMH374Yan6r2h1rath4+jWeG/DeZyIeIQ5+69j6cAmyrEVCoXA9L1XcebOY5joS7F5TMWf/UaVi46OBEM9a2NlUKTK/FL3kjPxJCuvygS3kqrUp56np6dj7ty52L9/P5KSkuDg4IBhw4Zh3rx5yrNYhBCYP38+1q9fj9TUVHTs2BE///wz6tcv+SmVPPW8/Gw+ew8LDv1bZPnO8W0r7Myc4mTkPsX+K3HYdi4aEYnpyuXNaldHfdtq2Bv6oMKu7K1QKODi4oJhw4ZhyZIlRR6/c+cO6tevjz///BM9evQo8ri7uzv8/PzQv39/5Ofn491338WFS6GoN2Ih7mUUhJg+TR3wzdC2qGH6/xvIoKAgeHl54ebNm3B3dy/Ve/Dx8YG+vj7++OMPAMBfx0/inbe7wqztIFi+NQqzezbAB51c0LRpU/Tq1Qt+fn4lqpuamgonJye8PXY6Qg2bQwjAydIYK4Y0Q93qUrz99tuwsLDAoUOHquyenWcF/puICb9egkIAn3Wvh6lvF/zd8/vzJtadugtdHQk2jW6NzvWr3oy3VPZedMX5yvb3+GVKuv2u1GGnvDDslI9/Ih9h/JZLyHmqOv5FKpHg9KyulfKbhBACl6Kf4Ndz0fgrPF55fZxnSSTA0gFN0LiWORyqG8HMULdMz144duwYfHx8EBERUWyonzNnDrZt24b79+8XOQxc0K8E/v7+GD16NO7du4c6deoU+zonTpxQucTHe++9h+joaJw5c6ZU/Rc3l1JhH//7bBGuGzUFUHD2SPRvi6Cvp4ft27eXqHbUowy0bNUawqERanQZjcGtamFebw+IvGz4+PjA2NgYhw8ffu0z4CqjHSExytOEZ/RwQ3xqDn49X3DW27LBTTGgRa2KbI8qsXhZNjos+bvImWSV9e9xcRh21MCwU/YCbiTg0x1XkCdXoJ5tNUQlZUAhUKkmbnuVR+m5+D4gArsvxb50vWoGunCoboia1Y3g8N+t8N81axjB1tSgyPiK58XLsnEvORMuViYa/6NTWNvcSA/LA2/j+M2Cy1p0dLXCD4ObwraUFw18lQULFmDdunWIjY1VTlsghECtWrUwZswY1O05DgsP/wuFAFK3T8HoIf2w4vtvX1pTCIEdF2Lw9b7LuLNyJOy6jsDGb+eiRyN7pKWlwcfHBwYGBvjzzz9hbKx9h3OWBd7GyiDVy+T0aGSLte+3qqCOqKrYfTGmyFQdVeHvcSGtmFSQtMP+Kw8w7bdrkCsEejayw4qhzZCSmVclzlx4lrWpASa/XQ+/hcYWGVztbmeKR+m5eJyZh4zcp7idmIHbiRnF1tGRAHZmhsrw8/+BqGBZyN3H+OqPf8vkMFlxg8P1pTqY0cMNYzu4lPnZFy+an0kikWD69OmYP38+NjZrhkXda2PS1z8iLSEap9AY4f+Nl3p+cPi0adPQ2csH28OzERx2G7LT26Gnq4uAn2ajUd2CoOPt7Y2srCxs27YNaWlpyklEra2tIZWW3QUZy9PQ1rWKhJ3AG4mIl2VXmc8XVYyqdCZZaTDsUJnadj4acw+GQ4iCwxJLBjSGrlSnSp258Cx7cyP4DWj8wm9C2XlyPJRl42FqwS3uSTbiUnMK7suyEZ+agzy5Ag9lOXgoy1FO1PciCgHM/P06/P68VeogolAIpGbnF1n+y6hW5Tam4/nB4c+aPHkycnJyMGXKFKSkpMCtYSPU+vB7pOhaYNDac1g2uCmioqKQnJysfM7lm3ewcr0/8rPSoGtsjuat22J74G+oV7d2weOXLyMkpGBMwvOzYN+7dw/Ozs5l92bL0f3HWUWWyQVwPzmrSn7OqHxV1b/H6uBhLPAwVllZczIKS48WzGQ7ur0z5v2vodbM2xAvy36tb0IKhUByRi7iUrPx8L8QFPff7WFqNqIfZyIjt3yvhF6ZByPKsvIxcedl/BNZEHCmvl0f77asiduJGThyLR6/hT4AANS3rYYVQ5qjocOb+fnVhrEXRK+DY3bUwLCjWUIIfH8sAqtPRAEAJnZ1xefe9bVjyvEyVtxGS0cCbBvXBtampZsj5VF6Lt7fGFLlNohP5Qos/vMWNp25V+zjo9s7Y1ZPdxjqacchqddV1cdeEL0Ohh01MOxojkIh8NUfN7DlXMHZILN6uuOjLlX/InLlqSw3WlV5g7guOAp+f6leXFVHApyZ1a1Sh7Xy9Lp7HImqKg5QpnL3VK7AzN+v4/fLDyCRAAv7NsL7bZ0quq0qpywHDFblwYiNa5kXWabguBQVb8LYC6LXwbBDGpH7VI7Ju8LwV3gCpDoS/DCoKfo1r1nRbVVZZbnRqqobRBerqnF1aSKqfLTr4hdUIbLz5Bi/NRR/hSdAX6qDNcNbMOiQxhWeCSf9b+xX4WG4qhjciKh8cc8OlUpaTj7Gbb6Ii/efwEhPig0jW6FjPauKbou0VFU+DEdEFYdhh15bSmYeRm4KQXhcGkwNdbF5TGu0dLKo6LZIy1XVw3BEVHEYdui1JKbl4P1fQhCZlAFLE31sHecJD4eiA0iJiIgqGsMOqS3mcRaGbzyP2JRs2Jsb4tdxbeBqU62i2yIiIioWww6pJTIxHe9vDEFiWi6cLI2x/YM2qFWDZ8MQEVHlxbBDJRYeJ8PITReQkpkHN1tT/DrOEzZlfIVsIiKi0mLYoRK5eD8FY/0vIj33KZrWMsfmMZ6oYaJf0W0RERG9EsMOvVLw7UeY8Osl5OQr0MbFAr+MagVTQ72KbouIiKhEGHboheJl2dhzMRY//R2Jpwqgq5s11rzf8o2/4CIREVUtDDtUrN0XYzDr9+sonJm/cU1zrBvRCvq6nHSbiIiqFm65qIh4WTZm7fv/oAMANx7K8Dgzt8J6IiIiel0MO1TEveRMCKG6rPDq0kRERFUNww4Vkf9UUWQZry5NRERVFcMOFXEw7CEAQPLffV5dmoiIqjIOUCYVCbIcHLpaEHY2jm4FIz1dXl2aiIiqNIYdUrH13H08VQh4Olugm7ttRbdDRERUajyMRUpZeU+xPSQGADCuk0sFd0NERKQZDDuk9HvoA8iy8+FkaQyvBtyrQ0RE2oFhhwAAcoXAxtP3AADjOrpAqiN5xTOIiIiqBoYdAgAE3UzE/cdZMDfSw7sta1V0O0RERBrDsEMAgF/+26vzXhtHGOtz3DoREWkPhh3CtQepuHAvBbo6Eoxq51zR7RAREWkUww7hl38K9ur0buoAO3PDCu6GiIhIsxh23nAPU7Nx5Ho8gIKByURERNqGYecNt+XsfcgVAu3qWKJRTfOKboeIiEjjGHbeYBm5T7HjQsEkgh9wEkEiItJSDDtvsN8uxSI95ynqWJmgq5tNRbdDRERUJhh23lByhcCmMwUDk8d2dIEOJxEkIiItxbDzhjp2IwGxKdmoYayHgS04iSAREWkvhp03VOEkgu+3dYKRvrSCuyEiIio7DDtvoMsxTxAa/QT6Uh2MaOdU0e0QERGVKYadN1DhBT/7NHOAjSknESQiIu3GsPOGiU3Jwl+cRJCIiN4gDDtvmM1n70MhgI6uVmhgb1bR7RAREZU5hp03SFpOPnZfjAXASQSJiOjNwbDzBtlzMRYZuU9Rz6YautS3ruh2iIiIygXDzhviqVwB/zP3ARSM1ZFIOIkgERG9GRh23hB/hScgLjUblib66Ne8ZkW3Q0REVG4Ydt4AQgj88s9dAAWTCBrqcRJBIiJ6czDsvAFCo5/g6gMZ9HU5iSAREb15GHbeAL/8UzCJ4IDmNWFVzaCCuyEiIipfDDtaLvpxJgL+TQBQcHVzIiKiNw3DjpbzP3MfQgBd6lujvq1pRbdDRERU7hh2tJgsKx97LnESQSIierMx7GixnRdjkJUnh7udKTq6WlV0O0RERBWCYUdL5csV2MxJBImIiBh2tNWf1+ORkJYDq2oG6NPMoaLbISIiqjAMO1pICIEN/00iOKqdEwx0OYkgERG9uRh2tFDIvRSEx6XBQFcHw9tyEkEiInqzqR12Tpw4URZ9kAYVTiI4sGUtWJjoV3A3REREFUvtsNOjRw/UrVsX33zzDWJjY8uiJyqFu48yEHQrEUDBwGQiIqI3ndphJy4uDhMnTsTevXtRp04d+Pj4YM+ePcjLyyuL/rTSggULIJFIVG7u7u7KxxMSEjBixAjY2dnBxMQELVq0wO+///7SmmvWrEGTJk3QwMkO0csGIWvvLNy+9I/KOhMmTEDdunVhZGQEa2tr9O3bF7du3SqT90hERFRZqB12rKysMGXKFISFhSEkJAT169fHJ598AgcHB3z22We4evVqWfSpdTw8PBAfH6+8nT59WvnYyJEjERERgUOHDuH69esYMGAABg8ejCtXrrywXq1atfDlgoWoOWYF7EetQE9vL/Tt2xc3btxQrtOyZUv4+/vj5s2bCAgIgBAC3t7ekMvlZfpeiYiIKlKpBii3aNECs2fPxsSJE5GRkYFNmzahZcuW6NSpk8pGlorS1dWFnZ2d8mZl9f+T/p09exaffvopPD09UadOHXz55ZeoXr06QkNDX1ivd+/eeFTDAwpTezT1aIDNq5ehWrVqOH/+vHKdDz/8EJ07d4azszNatGihPBR5//79snyrREREFeq1wk5+fj727t2Ld955B05OTggICMCqVauQmJiIO3fuwMnJCYMGDdJIg3FxcXj//fdhaWkJIyMjNG7cGJcuXVI+LoTAvHnzYG9vDyMjI3h5eSEyMlIjr12WIiMj4eDggDp16mD48OGIiYlRPta+fXvs3r0bKSkpUCgU2LVrF3JycvDWW2+9sF7uUzk2n70PABjT3hG7d+9GZmYm2rVrV+z6mZmZ8Pf3h4uLC2rXrq3Jt0ZERFS5CDVNnDhRWFpaCgsLCzFp0iRx/fr1IuvEx8cLiUSibukiUlJShJOTkxg9erQICQkRd+/eFQEBAeLOnTvKdZYsWSLMzc3FgQMHxNWrV0WfPn2Ei4uLyM7OLvHryGQyAUDIZLJS91wSf/75p9izZ4+4evWqOHr0qGjXrp1wdHQUaWlpQgghnjx5Iry9vQUAoaurK8zMzERAQMBLa+69FCvsx64SOvqGQiqVCnNzc3HkyJEi661evVqYmJgIAMLNzU3lZ0lERFSVlHT7rXbY6datm9ixY4fIycl54Tr5+fni5MmT6pYuYubMmaJjx44vfFyhUAg7Ozvx3XffKZelpqYKAwMDsXPnzhK/TnmHnec9efJEmJmZiV9++UUIURAoPT09xfHjx0VYWJhYsGCBMDc3F9euXSv2+QqFQvgsDxaO0/aL+duOi0uXLolZs2YJKysrcePGDZV1U1NTxe3bt0VwcLDo3bu3aNGihVrBkIiIqLIo6fZbIoQQFbln6WUaNmwIHx8fPHjwAMHBwahZsyY++eQTjB8/HgBw9+5d1K1bF1euXEGzZs2Uz+vSpQuaNWuGH3/8sdi6ubm5yM3NVd5PS0tD7dq1IZPJYGZmVqbv6UVat24NLy8vfPDBB3B1dUV4eDg8PDyUj3t5ecHV1RVr164t8twzd5Ix/JcQGOlJcW52N1Q31lc+p27duli3bl2xr5mXl4caNWrgl19+wbBhw8rmjREREZWRtLQ0mJubv3L7rfaYHT8/P2zatKnI8k2bNmHp0qXqlnupu3fvYs2aNahXrx4CAgLw8ccf47PPPsOWLVsAFJyiDQC2trYqz7O1tVU+Vhw/Pz+Ym5srbxU9ZiUjIwNRUVGwt7dHVlYWAEBHR/W/RiqVQqFQFPv8X/67NMSgVrWUQQcAFAqFSqh7nijYs/fSdYiIiKo6tcPOunXrVOaEKeTh4VHsXofSUCgUaNGiBRYvXozmzZvjww8/xPjx40v9OrNnz4ZMJlPeyntyxGnTpiE4OBj379/H2bNn0b9/f0ilUgwbNgzu7u5wdXXFhAkTcOHCBURFReGHH35AYGAg+vXrp6zRvXt3rFq1CneS0nEi4hFSgzfDQxKH+/fv4/r165g9ezZOnjyJ4cOHAygIjn5+fggNDUVMTAzOnj2LQYMGwcjICO+88065vn8iIqLypKvuExISEmBvb19kubW1NeLj4zXSVCF7e3s0bNhQZVmDBg2UE+zZ2dkBABITE1V6SkxMVDms9TwDAwMYGBhotFd1PHjwAMOGDcPjx49hbW2Njh074vz587C2tgYA/Pnnn5g1axZ69+6NjIwMuLq6YsuWLSqhJCoqCsnJydh4+j4AwFY/DzM/m4D4+HiYm5ujSZMmCAgIwNtvvw0AMDQ0xD///IMVK1bgyZMnsLW1RefOnXH27FnY2NiU+8+AiIiovKgddmrXro0zZ87AxUX1UgRnzpyBg4ODxhoDgA4dOiAiIkJl2e3bt+HkVHBxSxcXF9jZ2SEoKEgZbtLS0hASEoKPP/5Yo71o0q5du176eL169V45Y/L9+/fx70MZ+q46AwD4dbM/PF0sXri+g4MD/vzzT/WbJSIiquLUDjvjx4/H5MmTkZ+fj27dugEAgoKCMGPGDHz++ecabW7KlClo3749Fi9ejMGDB+PChQtYv3491q9fDwCQSCSYPHkyvvnmG9SrVw8uLi6YO3cuHBwcVA75aKPdF2Mw6/frKBxdfvdRxkvDDhER0ZtK7bOxhBCYNWsWVq5cqbwelqGhIWbOnIl58+ZpvMHDhw9j9uzZiIyMhIuLC6ZOnao8G6uwn/nz52P9+vVITU1Fx44d8fPPP6N+/folfo2SjuauLOJl2eiw5G8onvmfk0okOD2rK+zNjSquMSIionJU0u33a596npGRgZs3b8LIyAj16tWr0DEwpVXVws7ZqGS8tyGkyPKd49uiXV3LCuiIiIio/JV0+632YaxC1apVQ+vWrV/36VQKEkiKLJNKJHC2Mq6AboiIiCq31wo7ly5dwp49exATE6M8lFVo3759GmmMiieEwKoTqtf+kkokWDygEQ9hERERFUPtsLNr1y6MHDkSPj4+OHbsGLy9vXH79m0kJiaif//+ZdEjPePwtXicufMY+ro62P6BJ57KAWcrYwYdIiKiF1A77CxevBjLly+Hr68vTE1N8eOPP8LFxQUTJkwodv4d0pyM3Kf45si/AIBP3qqL1s4cn0NERPQqas+gHBUVhV69egEA9PX1kZmZCYlEgilTpihPCaey8ePx20hMy4WTpTE+6lK3otshIiKqEtQOOzVq1EB6ejoAoGbNmggPDwcApKamKq/rRJoXkZCOTWfuAwAW9PGAoZ60YhsiIiKqItQ+jNW5c2cEBgaicePGGDRoECZNmoS///4bgYGB6N69e1n0+MYTQmDuwXDIFQI+Hrbo6sbLOxAREZWU2mFn1apVyMnJAQB88cUX0NPTw9mzZzFw4EB8+eWXGm+QgP1X4nDhXgqM9KSY19ujotshIiKqUtQKO0+fPsXhw4fh4+MDANDR0cGsWbPKpDEqIMvOx+I/bwIAPu3uiprVedYVERGROtQas6Orq4uPPvpIuWeHyt6yYxFIzshDXWsTfNCxTkW3Q0REVOWoPUDZ09MTYWFhZdAKPS88ToZfz0cDABb2bQR9XbX/u4iIiN54ao/Z+eSTTzB16lTExsaiZcuWMDExUXm8SZMmGmvuTaZQCHx5IBwKAfRu6oD2rlYV3RIREVGVpPaFQHV0iu5dkEgkEEJAIpFALpdrrLnyUhkvBLrrQgxm7buOaga6CPq8C2zNDCu6JSIiokqlzC4Eeu/evVI1Rq/2JDMPS4/eAgBM9qrHoENERFQKaocdJyensuiDnvFtwC08ycqHu50pRrd3ruh2iIiIqjS1w87WrVtf+vjIkSNfuxkCrsQ8wa6LsQCAhf0aQVfKQclERESloXbYmTRpksr9/Px8ZGVlQV9fH8bGxgw7pSBXFMyULAQwsEUttHa2qOiWiIiIqjy1dxs8efJE5ZaRkYGIiAh07NgRO3fuLIse3xjbQ6IRHpcGU0NdzH7HvaLbISIi0goaOUZSr149LFmypMheHyq5R+m5+C4gAgAw3ccNVtUMKrgjIiIi7aCxASG6urp4+PChpsq9cfz+uon0nKdoVNMMw9twEDgREZGmqD1m59ChQyr3hRCIj4/HqlWr0KFDB4019ia5cC8F+y7HQSIpmClZqiOp6JaIiIi0htphp1+/fir3JRIJrK2t0a1bN/zwww+a6uuNkS9XYO6BcADA0Na10dyxRgV3REREpF3UDjsKhaIs+nhjbTl7HxGJ6ahhrIcZPhyUTEREpGmcxKUCJchysDzwNgBgZg931DDRr+COiIiItI/aYWfgwIFYunRpkeXffvstBg0apJGm3hSL/ryJzDw5mjtWx+BWtSu6HSIiIq2kdtg5deoU3nnnnSLLe/bsiVOnTmmkqTfBmTvJ+OPqQ+j8NyhZh4OSiYiIyoTaYScjIwP6+kUPt+jp6SEtLU0jTWm7vKcKzDtYMCh5RFsnNKppXsEdERERaS+1w07jxo2xe/fuIst37dqFhg0baqQpbffL6buIepQJq2oGmOrtVtHtEBERaTW1z8aaO3cuBgwYgKioKHTr1g0AEBQUhJ07d+K3337TeIPaJi41Gz8F3QEAzHnHHeZGehXcERERkXZTO+z07t0bBw4cwOLFi7F3714YGRmhSZMmOH78OLp06VIWPWqVr/+4gex8OTxdLNC/ec2KboeIiEjrqR12AKBXr17o1auXpnvReicikhBwIxFSHQkW9m0EiYSDkomIiMqa2mN2Ll68iJCQkCLLQ0JCcOnSJY00pY1y8uVYcOgGAGBsB2e42ZlWcEdERERvBrXDjq+vL2JjY4ssj4uLg6+vr0aa0kZrg6MQ/TgLtmYGmORVv6LbISIiemOoHXb+/fdftGjRosjy5s2b499//9VIU9om+nEmfj4ZBQCY+7+GqGbwWkcPiYiI6DWoHXYMDAyQmJhYZHl8fDx0dbkRf54QAgsO3UDeUwU6ulqhV2P7im6JiIjojaJ22PH29sbs2bMhk8mUy1JTUzFnzhy8/fbbGm2uqouXZeOnv+/gRMQj6Ekl+KqvBwclExERlTO1d8V8//336Ny5M5ycnNC8eXMAQFhYGGxtbfHrr79qvMGqavfFGMzedx0KUXC/o6sV6lpXq9imiIiI3kBqh52aNWvi2rVr2L59O65evQojIyOMGTMGw4YNg54eJ8gDCvboPBt0ACD49iPEy7Jhb25UcY0RERG9gV5rkI2JiQk+/PBDTfeiNe4lZ6oEHQBQCOB+chbDDhERUTl77RHF//77L2JiYpCXl6eyvE+fPqVuqqpzsTKBjgQqgUcqkcDZyrjimiIiInpDqR127t69i/79++P69euQSCQQomCLXjjwVi6Xa7bDKsje3Ah+Axpjzr5wyIWAVCLB4gGNuFeHiIioAqgddiZNmgQXFxcEBQXBxcUFFy5cwOPHj/H555/j+++/L4seq6QhrR3Rub417idnwdnKmEGHiIiogqgdds6dO4e///4bVlZW0NHRgY6ODjp27Ag/Pz989tlnuHLlSln0WSXZmxsx5BAREVUwtefZkcvlMDUtuK6TlZUVHj58CABwcnJCRESEZrsjIiIiKiW19+w0atQIV69ehYuLC9q0aYNvv/0W+vr6WL9+PerUqVMWPRIRERG9NrXDzpdffonMzEwAwNdff43//e9/6NSpEywtLbF7926NN0hERERUGhJReDpVKaSkpKBGjRpV9lIIaWlpMDc3h0wmg5mZWUW3Q0RERCVQ0u23Rq7caWFhoYkyRERERBqn9gBlIiIioqqEYYeIiIi0GsMOERERaTW1w86pU6fw9OnTIsufPn2KU6dOaaQpIiIiIk1RO+x07doVKSkpRZbLZDJ07dpVI00RERERaYraYUcIUewp5o8fP4aJiYlGmiIiIiLSlBKfej5gwAAABVc3Hz16NAwMDJSPyeVyXLt2De3bt9d8h0RERESlUOKwY25uDqBgz46pqSmMjP7/Apf6+vpo27Ytxo8fr/kOiYiIiEqhxGHH398fAODs7Ixp06bxkBURERFVCWqP2ZkxY4bKmJ3o6GisWLECx44d02hjRERERJqgdtjp27cvtm7dCgBITU2Fp6cnfvjhB/Tt2xdr1qzReINEREREpaF22Ll8+TI6deoEANi7dy/s7OwQHR2NrVu3YuXKlRpvkIiIiKg01A47WVlZMDU1BQAcO3YMAwYMgI6ODtq2bYvo6GiNN0hERERUGmqHHVdXVxw4cACxsbEICAiAt7c3ACApKemll1cnIiIiqghqh5158+Zh2rRpcHZ2hqenJ9q1awegYC9P8+bNNd4gERERUWmoHXbeffddxMTE4NKlSwgICFAu7969O5YvX67R5p63ZMkSSCQSTJ48WbksJycHvr6+sLS0RLVq1TBw4EAkJiaWaR9ERERUdbzWVc/t7OxgamqKwMBAZGdnAwBat24Nd3d3jTb3rIsXL2LdunVo0qSJyvIpU6bgjz/+wG+//Ybg4GA8fPhQOdszERERkdph5/Hjx+jevTvq16+Pd955B/Hx8QCAcePG4fPPP9d4gwCQkZGB4cOHY8OGDahRo4ZyuUwmw8aNG7Fs2TJ069YNLVu2hL+/P86ePYvz58+XSS9ERERUtagddqZMmQI9PT3ExMTA2NhYuXzIkCE4evSoRpsr5Ovri169esHLy0tleWhoKPLz81WWu7u7w9HREefOnXthvdzcXKSlpanciIiISDuV+HIRhY4dO4aAgADUqlVLZXm9evXK5NTzXbt24fLly7h48WKRxxISEqCvr4/q1aurLLe1tUVCQsILa/r5+eGrr77SdKtERERUCam9ZyczM1Nlj06hlJQUlSuha0JsbCwmTZqE7du3w9DQUGN1Z8+eDZlMprzFxsZqrDYRERFVLmqHnU6dOikvFwEAEokECoUC3377Lbp27arR5kJDQ5GUlIQWLVpAV1cXurq6CA4OxsqVK6GrqwtbW1vk5eUhNTVV5XmJiYmws7N7YV0DAwOYmZmp3IiIiEg7qX0Y69tvv0X37t1x6dIl5OXlYcaMGbhx4wZSUlJw5swZjTbXvXt3XL9+XWXZmDFj4O7ujpkzZ6J27drQ09NDUFAQBg4cCACIiIhATEyMcv4fIiIierOpHXYaNWqE27dvY9WqVTA1NUVGRgYGDBgAX19f2Nvba7Q5U1NTNGrUSGWZiYkJLC0tlcvHjRuHqVOnwsLCAmZmZvj000/Rrl07tG3bVqO9EBERUdWkdtiJiYlB7dq18cUXXxT7mKOjo0YaK6nly5dDR0cHAwcORG5uLnx8fPDzzz+Xaw9ERERUeUmEEEKdJ0ilUsTHx8PGxkZl+ePHj2FjYwO5XK7RBstDWloazM3NIZPJOH6HiIioiijp9lvtAcpCCEgkkiLLMzIyNHrGFBEREZEmlPgw1tSpUwEUnH01d+5cldPP5XI5QkJC0KxZM403SERERFQaJQ47V65cAVCwZ+f69evQ19dXPqavr4+mTZti2rRpmu+QiIiIqBRKHHZOnDgBoODU7x9//JFjW4iIiKhKUPtsLH9//7Log4iIiKhMqD1AmYiIiKgqYdghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIirVapw46fnx9at24NU1NT2NjYoF+/foiIiFBZJycnB76+vrC0tES1atUwcOBAJCYmVlDHREREVNlU6rATHBwMX19fnD9/HoGBgcjPz4e3tzcyMzOV60yZMgV//PEHfvvtNwQHB+Phw4cYMGBABXZNRERElYlECCEquomSevToEWxsbBAcHIzOnTtDJpPB2toaO3bswLvvvgsAuHXrFho0aIBz586hbdu2JaqblpYGc3NzyGQymJmZleVbICIiIg0p6fa7Uu/ZeZ5MJgMAWFhYAABCQ0ORn58PLy8v5Tru7u5wdHTEuXPnXlgnNzcXaWlpKjciIiLSTlUm7CgUCkyePBkdOnRAo0aNAAAJCQnQ19dH9erVVda1tbVFQkLCC2v5+fnB3Nxceatdu3ZZtk5EREQVqMqEHV9fX4SHh2PXrl2lrjV79mzIZDLlLTY2VgMdEhERUWWkW9ENlMTEiRNx+PBhnDp1CrVq1VIut7OzQ15eHlJTU1X27iQmJsLOzu6F9QwMDGBgYFCWLRMREVElUan37AghMHHiROzfvx9///03XFxcVB5v2bIl9PT0EBQUpFwWERGBmJgYtGvXrrzbJSIiokqoUu/Z8fX1xY4dO3Dw4EGYmpoqx+GYm5vDyMgI5ubmGDduHKZOnQoLCwuYmZnh008/Rbt27Up8JhYRERFpt0p96rlEIil2ub+/P0aPHg2gYFLBzz//HDt37kRubi58fHzw888/v/Qw1vN46jkREVHVU9Ltd6UOO+WFYYeIiKjq0cp5doiIiIjUxbBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIq2lN2Fm9ejWcnZ1haGiINm3a4MKFCxXdEhEREVUCWhF2du/ejalTp2L+/Pm4fPkymjZtCh8fHyQlJVV0a0RERFTBtCLsLFu2DOPHj8eYMWPQsGFDrF27FsbGxti0aVNFt0ZEREQVTLeiGyitvLw8hIaGYvbs2cplOjo68PLywrlz54p9Tm5uLnJzc5X3ZTIZACAtLa1smyUiIiKNKdxuCyFeul6VDzvJycmQy+WwtbVVWW5ra4tbt24V+xw/Pz989dVXRZbXrl27THokIiKispOeng5zc/MXPl7lw87rmD17NqZOnaq8r1AokJKSAktLS0gkEo29TlpaGmrXro3Y2FiYmZlprK421K/KvVf1+lW596pevyr3Xtb1q3LvVb1+Ve5dCIH09HQ4ODi8dL0qH3asrKwglUqRmJiosjwxMRF2dnbFPsfAwAAGBgYqy6pXr15WLcLMzKxMfoG0oX5V7r2q16/KvVf1+lW597KuX5V7r+r1q2rvL9ujU6jKD1DW19dHy5YtERQUpFymUCgQFBSEdu3aVWBnREREVBlU+T07ADB16lSMGjUKrVq1gqenJ1asWIHMzEyMGTOmolsjIiKiCqYVYWfIkCF49OgR5s2bh4SEBDRr1gxHjx4tMmi5vBkYGGD+/PlFDpmxftXuvarXr8q9V/X6Vbn3sq5flXuv6vWrcu8lJRGvOl+LiIiIqAqr8mN2iIiIiF6GYYeIiIi0GsMOERERaTWGHSIiItJqDDtlaPXq1XB2doahoSHatGmDCxcuaKTuqVOn0Lt3bzg4OEAikeDAgQMaqQsUXEqjdevWMDU1hY2NDfr164eIiAiN1V+zZg2aNGminFyqXbt2+OuvvzRW/1lLliyBRCLB5MmTNVZzwYIFkEgkKjd3d3eN1Y+Li8P7778PS0tLGBkZoXHjxrh06ZJGajs7OxfpXSKRwNfXVyP15XI55s6dCxcXFxgZGaFu3bpYuHDhK69ZU1Lp6emYPHkynJycYGRkhPbt2+PixYuvVetVnyEhBObNmwd7e3sYGRnBy8sLkZGRGqu/b98+eHt7K2dtDwsL01j/+fn5mDlzJho3bgwTExM4ODhg5MiRePjwoUZ6X7BgAdzd3WFiYoIaNWrAy8sLISEhGun9eR999BEkEglWrFihsfqjR48u8hno0aOHRvu/efMm+vTpA3Nzc5iYmKB169aIiYkpde3iPr8SiQTfffedRnrPyMjAxIkTUatWLRgZGSkvrF1Sr6qfmJiI0aNHw8HBAcbGxujRo4dan6vSYNgpI7t378bUqVMxf/58XL58GU2bNoWPjw+SkpJKXTszMxNNmzbF6tWrNdCpquDgYPj6+uL8+fMIDAxEfn4+vL29kZmZqZH6tWrVwpIlSxAaGopLly6hW7du6Nu3L27cuKGR+oUuXryIdevWoUmTJhqtCwAeHh6Ij49X3k6fPq2Ruk+ePEGHDh2gp6eHv/76C//++y9++OEH1KhRQyP1L168qNJ3YGAgAGDQoEEaqb906VKsWbMGq1atws2bN7F06VJ8++23+OmnnzRS/4MPPkBgYCB+/fVXXL9+Hd7e3vDy8kJcXJzatV71Gfr222+xcuVKrF27FiEhITAxMYGPjw9ycnI0Uj8zMxMdO3bE0qVL1e79VfWzsrJw+fJlzJ07F5cvX8a+ffsQERGBPn36aKT3+vXrY9WqVbh+/TpOnz4NZ2dneHt749GjRxqpX2j//v04f/78Ky8D8Dr1e/ToofJZ2Llzp8bqR0VFoWPHjnB3d8fJkydx7do1zJ07F4aGhqWu/WzP8fHx2LRpEyQSCQYOHKiR3qdOnYqjR49i27ZtuHnzJiZPnoyJEyfi0KFDpa4vhEC/fv1w9+5dHDx4EFeuXIGTkxO8vLw0tn15KUFlwtPTU/j6+irvy+Vy4eDgIPz8/DT6OgDE/v37NVrzWUlJSQKACA4OLrPXqFGjhvjll180Vi89PV3Uq1dPBAYGii5duohJkyZprPb8+fNF06ZNNVbvWTNnzhQdO3Ysk9rFmTRpkqhbt65QKBQaqderVy8xduxYlWUDBgwQw4cPL3XtrKwsIZVKxeHDh1WWt2jRQnzxxRelqv38Z0ihUAg7Ozvx3XffKZelpqYKAwMDsXPnzlLXf9a9e/cEAHHlyhW165akfqELFy4IACI6OlrjtWUymQAgjh8/rlbtl9V/8OCBqFmzpggPDxdOTk5i+fLlatd+Uf1Ro0aJvn37vla9ktQfMmSIeP/998uk9vP69u0runXrprH6Hh4e4uuvv1ZZ9rqfsefrR0RECAAiPDxcuUwulwtra2uxYcMGteuri3t2ykBeXh5CQ0Ph5eWlXKajowMvLy+cO3euAjtTn0wmAwBYWFhovLZcLseuXbuQmZmp0Ut7+Pr6olevXio/f02KjIyEg4MD6tSpg+HDh5do93RJHDp0CK1atcKgQYNgY2OD5s2bY8OGDRqp/by8vDxs27YNY8eO1djFb9u3b4+goCDcvn0bAHD16lWcPn0aPXv2LHXtp0+fQi6XF/l2bGRkpLE9a4Xu3buHhIQEld8fc3NztGnTpsp9fgvJZDJIJBKNXwMwLy8P69evh7m5OZo2baqRmgqFAiNGjMD06dPh4eGhkZrPO3nyJGxsbODm5oaPP/4Yjx8/1khdhUKBI0eOoH79+vDx8YGNjQ3atGmj0aEGhRITE3HkyBGMGzdOYzXbt2+PQ4cOIS4uDkIInDhxArdv34a3t3epa+fm5gKAymdYR0cHBgYGGv8MF4dhpwwkJydDLpcXmcHZ1tYWCQkJFdSV+hQKBSZPnowOHTqgUaNGGqt7/fp1VKtWDQYGBvjoo4+wf/9+NGzYUCO1d+3ahcuXL8PPz08j9Z7Xpk0bbN68GUePHsWaNWtw7949dOrUCenp6aWufffuXaxZswb16tVDQEAAPv74Y3z22WfYsmWLBjpXdeDAAaSmpmL06NEaqzlr1iwMHToU7u7u0NPTQ/PmzTF58mQMHz681LVNTU3Rrl07LFy4EA8fPoRcLse2bdtw7tw5xMfHa6D7/1f4Ga3qn99COTk5mDlzJoYNG6axizAePnwY1apVg6GhIZYvX47AwEBYWVlppPbSpUuhq6uLzz77TCP1ntejRw9s3boVQUFBWLp0KYKDg9GzZ0/I5fJS105KSkJGRgaWLFmCHj164NixY+jfvz8GDBiA4OBgDXT//7Zs2QJTU1MMGDBAYzV/+uknNGzYELVq1YK+vj569OiB1atXo3PnzqWu7e7uDkdHR8yePRtPnjxBXl4eli5digcPHmj8M1wcrbhcBJUNX19fhIeHazx1u7m5ISwsDDKZDHv37sWoUaMQHBxc6sATGxuLSZMmITAwsETHx1/Hs3spmjRpgjZt2sDJyQl79uwp9TcshUKBVq1aYfHixQCA5s2bIzw8HGvXrsWoUaNKVft5GzduRM+ePdUeD/Eye/bswfbt27Fjxw54eHggLCwMkydPhoODg0b6//XXXzF27FjUrFkTUqkULVq0wLBhwxAaGqqB7rVTfn4+Bg8eDCEE1qxZo7G6Xbt2RVhYGJKTk7FhwwYMHjwYISEhsLGxKVXd0NBQ/Pjjj7h8+bLG9jg+b+jQocp/N27cGE2aNEHdunVx8uRJdO/evVS1FQoFAKBv376YMmUKAKBZs2Y4e/Ys1q5diy5dupSq/rM2bdqE4cOHa/Rv3U8//YTz58/j0KFDcHJywqlTp+Dr6wsHB4dS7ynX09PDvn37MG7cOFhYWEAqlcLLyws9e/bU2EkML8M9O2XAysoKUqkUiYmJKssTExNhZ2dXQV2pZ+LEiTh8+DBOnDiBWrVqabS2vr4+XF1d0bJlS/j5+aFp06b48ccfS103NDQUSUlJaNGiBXR1daGrq4vg4GCsXLkSurq6Gvnm9rzq1aujfv36uHPnTqlr2dvbFwl8DRo00NhhskLR0dE4fvw4PvjgA43WnT59unLvTuPGjTFixAhMmTJFY3vZ6tati+DgYGRkZCA2NhYXLlxAfn4+6tSpo5H6hQo/o1X58wv8f9CJjo5GYGCgxvbqAICJiQlcXV3Rtm1bbNy4Ebq6uti4cWOp6/7zzz9ISkqCo6Oj8jMcHR2Nzz//HM7OzqVvvBh16tSBlZWVRj7DVlZW0NXVLfPP8T///IOIiAiNfoazs7MxZ84cLFu2DL1790aTJk0wceJEDBkyBN9//71GXqNly5YICwtDamoq4uPjcfToUTx+/Fjjn+HiMOyUAX19fbRs2RJBQUHKZQqFAkFBQRodm1IWhBCYOHEi9u/fj7///hsuLi5l/poKhUJ5PLc0unfvjuvXryMsLEx5a9WqFYYPH46wsDBIpVINdKsqIyMDUVFRsLe3L3WtDh06FDnN//bt23Bycip17Wf5+/vDxsYGvXr10mjdrKws6Oio/kmRSqXKb7uaYmJiAnt7ezx58gQBAQHo27evRuu7uLjAzs5O5fOblpaGkJCQSv/5LVQYdCIjI3H8+HFYWlqW6etp6jM8YsQIXLt2TeUz7ODggOnTpyMgIEADnRb14MEDPH78WCOfYX19fbRu3brMP8cbN25Ey5YtNTZOCij4ncnPzy+Xz7C5uTmsra0RGRmJS5cuafwzXBwexiojU6dOxahRo9CqVSt4enpixYoVyMzMxJgxY0pdOyMjQ+VbyL179xAWFgYLCws4OjqWqravry927NiBgwcPwtTUVDlGwdzcHEZGRqWqDQCzZ89Gz5494ejoiPT0dOzYsQMnT57UyB8yU1PTImOLTExMYGlpqbExR9OmTUPv3r3h5OSEhw8fYv78+ZBKpRg2bFipa0+ZMgXt27fH4sWLMXjwYFy4cAHr16/H+vXrNdB5AYVCAX9/f4waNQq6upr9+Pfu3RuLFi2Co6MjPDw8cOXKFSxbtgxjx47VSP2AgAAIIeDm5oY7d+5g+vTpcHd3f63P1Ks+Q5MnT8Y333yDevXqwcXFBXPnzoWDgwP69eunkfopKSmIiYlRzn1TuHG0s7Mr0d6jl9W3t7fHu+++i8uXL+Pw4cOQy+XKz7GFhQX09fVfu7alpSUWLVqEPn36wN7eHsnJyVi9ejXi4uJKPIXBq342zwczPT092NnZwc3NrdT1LSws8NVXX2HgwIGws7NDVFQUZsyYAVdXV/j4+Gik/+nTp2PIkCHo3LkzunbtiqNHj+KPP/7AyZMnS10bKAjev/32G3744YcS9atO/S5dumD69OkwMjKCk5MTgoODsXXrVixbtkwj9X/77TdYW1vD0dER169fx6RJk9CvXz+NDIB+pTI/3+sN9tNPPwlHR0ehr68vPD09xfnz5zVS98SJEwJAkduoUaNKXbu4ugCEv79/qWsLIcTYsWOFk5OT0NfXF9bW1qJ79+7i2LFjGqldHE2fej5kyBBhb28v9PX1Rc2aNcWQIUPEnTt3NFb/jz/+EI0aNRIGBgbC3d1drF+/XmO1hRAiICBAABAREREarSuEEGlpaWLSpEnC0dFRGBoaijp16ogvvvhC5ObmaqT+7t27RZ06dYS+vr6ws7MTvr6+IjU19bVqveozpFAoxNy5c4Wtra0wMDAQ3bt3V+tn9qr6/v7+xT4+f/78UtcvPJ29uNuJEydKVTs7O1v0799fODg4CH19fWFvby/69OkjLly4oLGfzfPUPfX8ZfWzsrKEt7e3sLa2Fnp6esLJyUmMHz9eJCQkaLT/jRs3CldXV2FoaCiaNm0qDhw4oLHa69atE0ZGRq/1u/+q+vHx8WL06NHCwcFBGBoaCjc3N/HDDz+UeHqKV9X/8ccfRa1atYSenp5wdHQUX375pcb+PryKRIhyGBlEREREVEE4ZoeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0T0nJMnT0IikSA1NbWiWyEiDWDYISIiIq3GsENERERajWGHiCodhUIBPz8/uLi4wMjICE2bNsXevXsB/P8hpiNHjqBJkyYwNDRE27ZtER4erlLj999/h4eHBwwMDODs7Fzkwom5ubmYOXMmateuDQMDA7i6umLjxo0q64SGhqJVq1YwNjZG+/bti1zNmoiqBoYdIqp0/Pz8sHXrVqxduxY3btzAlClT8P777yM4OFi5zvTp0/HDDz/g4sWLsLa2Ru/evZGfnw+gIKQMHjwYQ4cOxfXr17FgwQLMnTsXmzdvVj5/5MiR2LlzJ1auXImbN29i3bp1qFatmkofX3zxBX744QdcunQJurq6GruCOxGVL14IlIgqldzcXFhYWOD48eNo166dcvkHH3yArKwsfPjhh+jatSt27dqFIUOGAABSUlJQq1YtbN68GYMHD8bw4cPx6NEjHDt2TPn8GTNm4MiRI7hx4wZu374NNzc3BAYGwsvLq0gPJ0+eRNeuXXH8+HF0794dAPDnn3+iV69eyM7OhqGhYRn/FIhIk7hnh4gqlTt37iArKwtvv/02qlWrprxt3boVUVFRyvWeDUIWFhZwc3PDzZs3AQA3b95Ehw4dVOp26NABkZGRkMvlCAsLg1QqRZcuXV7aS5MmTZT/tre3BwAkJSWV+j0SUfnSregGiIielZGRAQA4cuQIatasqfKYgYGBSuB5XUZGRiVaT09PT/lviUQCoGA8ERFVLdyzQ0SVSsOGDWFgYICYmBi4urqq3GrXrq1c7/z588p/P3nyBLdv30aDBg0AAA0aNMCZM2dU6p45cwb169eHVCpF48aNoVAoVMYAEZH24p4dIqpUTE1NMW3aNEyZMgUKhQIdO3aETCbDmTNnYGZmBicnJwDA119/DUtLS9ja2uKLL76AlZUV+vXrBwD4/PPP0bp1ayxcuBBDhgzBuXPnsGrVKvz8888AAGdnZ4waNQpjx47FypUr0bRpU0RHRyMpKQmDBw+uqLdORGWEYYeIKp2FCxfC2toafn5+uHv3LqpXr44WLVpgzpw5ysNIS5YswaRJkxAZGYlmzZrhjz/+gL6+PgCgRYsW2LNnD+bNm4eFCxfC3t4eX3/9NUaPHq18jTVr1mDOnDn45JNP8PjxYzg6OmLOnDkV8XaJqIzxbCwiqlIKz5R68uQJqlevXtHtEFEVwDE7REREpNUYdoiIiEir8TAWERERaTXu2SEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKt9n8fQDs8MfsJIQAAAABJRU5ErkJggg==", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAG2CAYAAACZEEfAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABq60lEQVR4nO3dd1hT1+MG8DfsHUW2TAfixK3gqqO46q6jtVWr1bZi62idraPVFrWt7dfROn5urauOOloXrqqooOBEBERBZYhI2DPn9weaGlmJBIH4fp4nT8vNPfecQMx9c+6550iEEAJEREREWkqnohtAREREVJ4YdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirVWjYOXPmDPr06QMHBwdIJBLs27dP6XkhBObMmQN7e3sYGxujW7duCA8PV9onKSkJw4cPh4WFBapVq4YxY8YgLS3tNb4KIiIiqswqNOykp6fD09MTK1asKPL5xYsXY+nSpVi5ciUuXrwIU1NTdO/eHVlZWYp9hg8fjps3b+LYsWM4ePAgzpw5g3Hjxr2ul0BERESVnKSyLAQqkUiwd+9e9O/fH0BBr46DgwO+/PJLfPXVVwAAmUwGW1tbbNiwAcOGDUNoaCgaNGiAwMBAtGzZEgBw+PBh9OrVCw8ePICDg0NFvRwiIiKqJPQqugHFiYqKQlxcHLp166bYJpVK0aZNGwQEBGDYsGEICAhAtWrVFEEHALp16wYdHR1cvHgRAwYMKPLY2dnZyM7OVvwsl8uRlJSEGjVqQCKRlN+LIiIiIo0RQiA1NRUODg7Q0Sn+YlWlDTtxcXEAAFtbW6Xttra2iufi4uJgY2Oj9Lyenh4sLS0V+xTFz88P3377rYZbTERERBUhJiYGjo6OxT5facNOeZo5cyamTJmi+Fkmk8HZ2RkxMTGwsLCowJYRERGRqlJSUuDk5ARzc/MS96u0YcfOzg4AEB8fD3t7e8X2+Ph4NG3aVLFPQkKCUrm8vDwkJSUpyhfF0NAQhoaGhbZbWFgw7BAREVUxpQ1BqbTz7Li5ucHOzg7+/v6KbSkpKbh48SK8vLwAAF5eXkhOTsbly5cV+5w4cQJyuRxt2rR57W0mIiKiyqdCe3bS0tIQERGh+DkqKgohISGwtLSEs7MzJk2ahAULFqBu3bpwc3PD7Nmz4eDgoLhjq379+ujRowfGjh2LlStXIjc3FxMmTMCwYcN4JxYREREBqOCwExQUhM6dOyt+fj6OZuTIkdiwYQOmTZuG9PR0jBs3DsnJyWjfvj0OHz4MIyMjRZmtW7diwoQJ6Nq1K3R0dDBo0CAsXbr0tb8WIiIiqpwqzTw7FSklJQVSqRQymYxjdoiIiKoIVc/flXbMDhEREZEmMOwQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEqQn5+P2bNnw83NDcbGxqhduzbmz58PIYRin1GjRkEikSg9evToUeJxU1NTMWnSJLi4uMDY2Bje3t4IDAxU2ictLQ0TJkyAo6MjjI2N0aBBA6xcubJcXqc206voBhAREVVmixYtwu+//46NGzeiYcOGCAoKwkcffQSpVIovvvhCsV+PHj2wfv16xc+GhoYlHvfjjz/GjRs3sHnzZjg4OGDLli3o1q0bbt26hZo1awIApkyZghMnTmDLli1wdXXF0aNHMX78eDg4OKBv377l84K1EHt2iIiISnD+/Hn069cPvXv3hqurK9599134+Pjg0qVLSvsZGhrCzs5O8ahevXqxx8zMzMTu3buxePFidOzYEXXq1MG8efNQp04d/P7770p1jxw5Em+99RZcXV0xbtw4eHp6FqqbSsawQ0T0hiivyzF+fn5o1aoVzM3NYWNjg/79+yMsLExpn08++QS1a9eGsbExalhZo0O3njgTGFIeL1PjvL294e/vjzt37gAArl69irNnz6Jnz55K+506dQo2NjaoV68ePvvsMzx58qTYY+bl5SE/Px9GRkZK242NjXH27Fmluvfv34+HDx9CCIGTJ0/izp078PHx0eArfAMIEjKZTAAQMpmsoptCRFRuvv/+e1GjRg1x8OBBERUVJXbt2iXMzMzE//73P8U+I0eOFD169BCxsbGKR1JSUonH7d69u1i/fr24ceOGCAkJEb169RLOzs4iLS1Nsc+qVavE6dOnxdJ9Z4XDqF+FcZ3WQtfCWvwRcLfcXq+m5Ofni+nTpwuJRCJ09fSERCIRP/zwg9I+27ZtE3/99Ze4du2a2Lt3r6hfv75o1aqVyMvLK/a4Xl5eolOnTuLhw4ciLy9PbN68Wejo6Ah3d3fFPllZWWLEiBECgNDT0xMGBgZi48aN5fZaqxpVz98MO4Jhh4jeDL179xajR49W2jZw4EAxfPhwxc8jR44U/fr1K1M9CQkJAoA4ffq00vbIhFThOv2gcJl+UNh/tEwAEE6f/J94lJxRpvrK27Zt24Sljb2w7jtV2I9eLqzemSLMLKqJDRs2FFsmMjJSABDHjx8vdp+IiAjRsWNHAUDo6uqKVq1aieHDhwsPDw/FPj/++KNwd3cX+/fvF1evXhXLli0TZmZm4tixYxp9jVWVqudvXsYiInpDlMflmKLIZDIAgKWlpWLbydsJGLIyAAKAPCcLadePQ09qC4l5DdxLzCjbCytnU776ChLPfjCp3wkG1q4wbdgFep7vYP73PxRbplatWrCyskJERESx+9SuXRunT59GWloaYmJicOnSJeTm5qJWrVoACsb1zJo1C0uWLEGfPn3QpEkTTJgwAUOHDsVPP/2k8ddZklhZJs5HJiJWlvla69UU3o1FRPSGmDFjBlJSUuDh4QFdXV3k5+fj+++/x/DhwxX79OjRAwMHDoSbmxsiIyMxa9Ys9OzZEwEBAdDV1S21DrlcjkmTJqFdu3Zo1KgRElKz8N2BWzh4LRapVw7h6an1ELlZ0LN0hM3QBdDV04erlUl5vuwyEUJAlpIOI8lLfQMSHcTLMuEfGo+O7tbQ11V+/sGDB3jy5Ans7e1LrcPU1BSmpqZ4+vQpjhw5gsWLFwMAcnNzkZubCx0d5WPr6upCLper9TpiZZmISkyHm5Up7KXGpe4vhEBadh4S03KwIzAaq87chRCAjgTwG9gYQ1s5q1V/RWPYISJ6Q+zcuRNbt27FH3/8gYYNGyIkJASTJk2Cg4MDRo4cCQAYNmyYYv/GjRujSZMmqF27Nk6dOoWuXbuWWoevry9u3LiBM2f+xbZL0fD7OxQpWXnQkQDjPx4Ji48H45e/ApF8aTcS/1oI51E/IyMnv9xec1lcikqC3z+hkLi0gOz8DuhaWMPAyhk58ZFICdwHsyZvY8zGIEj18mF6cy8+GfkeOnrWxd27dzFt2jTUqVMH3bt3Vxyva9euGDBgACZMmAAAOHLkCIQQqFevHiIiIjB16lR4eHjgo48+AgBYWFigU6dOmDp1KoyNjeHi4oLTp09j06ZNWLJkicqvY0dgNGbuuQ75s7Ay+W13tHa1RGJaDhLTshWPx6nKP2flFg5UcgFM330dJ8Meo4VzdXjYm8PDzgLW5iXfZl/RJEK8MAz/DZWSkgKpVAqZTAYLC4uKbg4RqSA/Px/z5s3Dli1bEBcXBwcHB4waNQrffPMNJBIJAGDevHnYvn07YmJiYGBggBYtWuD7779HmzZtij2uq6sr7t+/X2j7+PHjsWLFCgDA6tWr8ccff+DKlStITU3F06dPUa1atXJ5nZrk5OSEGTNmwNfXV7FtwYIF2LJlC27fvl1sOWtrayxYsACffPJJicefMGEC/vrrL2za8zd+u5yKwHtPAQCNa0rhN7AxGtWUAijoZbj9IAlvN6+D6t0/h+dbvbHPtx3MDCvH9+878alYfPg2jocmAAAM5NnQD9mFsIv+kGfIoGtmiU49+8P73U/w961EJDxNweM9C5CTcBciOx3VrGzR3edt/PrjQtja2iqO6+rqilGjRmHevHkACsLnzJkz8eDBA1haWmLQoEH4/vvvIZVKFWXi4uIwc+ZMHD16FElJSXBxccG4ceMwefJkxfv8ZUIIPHiaidtxqQiMeoLV/0a98u/CUE8H2Xml9yJZmRmivr05POwKwo+HvTnq2JjBUE9X7V4ldah6/q4c7ywiIjWpMtGbu7s7li9fjlq1aiEzMxO//PILfHx8EBERAWtr6yKPGxgYiPz8/3oabty4gbfffhuDBw9WbMvIyECPHj3Qo0cPzJw5s3xfqAZlZGSofUlElcsxQgh8/vnn2LN3L0Z+vx7j9sUgN1/AxEAXU952xyhvV+i9cJnHXmoMSyMrGOhKYGEARCSk4audV/H7B82LPYG/DrGyTPxy7A7+vPwAcgHo6kgwrJUTJnatCxuLgYiVZeJeYgZcrUwUJ+05/eQ4F/kEe1utxZGb8cjMLXjvBAD4dHcEBjTLxDtNHGBpaoCAq6GISkxHrCwT9lJjDBkyBEOGDCmxTXZ2dli/fn2xgSE1Kxd34lMRGpuK23EpuB2bittxqUjLzivxuLYWhnCqbgIrM0NYmRvAyswQ1uaGBT+bGcL62XZZZi7aLTwB+QvdIjoSYEx7NzxMzsTt2FREPUlHYlo2/g3Pxr/hiYr99HQkqGFmgPiUbEW5iroExp4dsGeHqCp65513YGtri7Vr1yq2DRo0CMbGxtiyZUuRZZ7/Wz9+/LhKl2QAYNKkSTh48CDCw8MLnYhPnTqFzp07V5menVGjRuH48eNYtWoVrJ1qw//cRfw0+yt8PGY0Fi1ahLS0NHz77bcYNGgQ7OzsEBkZiWnTpiE1NRXXr19XzAj88uWY8ePHY/OWrajzwbdI1LECALSrWwPzB7dGXYcauHv3Lnbs2AEfHx9YW1vjwYMHWLhwIc6dO4edxwLw2e5I5OTLMbV7Pfh2rvPafy+yjFz8djoCG87dU/Ri9Gxkh6+610NtazOVj5OenYejt+KwN/gRzoY/VgQEPR0J6tqa4XZsKgQKTvrTutfDO54OKh334NVHWHwkDHIBSAB08bCBjo4Et+NSEJNU9IBhfV0J6tiYw8XSBEduxuHFE72uRIKzMzqr3MuyIzAas/bcQL4Q0JVI8MPARkqBJSMnD+HxaQiNTcHtuFTFf2WZuYWOpW7dpWHPDhFpNW9vb6xevRp37tyBu7u74s6i4sYy5OTkYPXq1ZBKpfD09FSpjpycHGzZsgVTpkyp0B4HTVm2bBlmz56NUR9/gieJj6FrZgnTBl3RpN9YAAW9PNeuXcPGjRuRnJwMBwcH+Pj4YP78+UpLH0RGRiIxseAbfHJGjmLG35DfJyv22QGgh3Q96o4aBSMjI/z777/49ddf8fTpU9ja2qJjx444f/486tVzx7d5Rpi55zp+OhqGRjWl6ORedK+bpmXl5mNTwD2sOBmpODG3drPEjJ4eaO5c/OzHxTE11MOAZo4Y0MwRCalZ2B/yCPtCHuLGwxSExqYq9pMLYOHhMCw8HFbC0YomAPjfTlDaZmdhpBg7U//Zf2tZmyoGTRcVVtQJG0NbOaOju3WhXq3nTAz04OlUDZ5O1f5rpxA4dD0WE/4IVto3XwjcS8zQ+OWs0rBnB+zZIaqK5HI5Zs2ahcWLFyvdWfTyZaWDBw9i2LBhyMjIgL29Pfbt24dWrVqpVMfOnTvx/vvvIzo6Gg4Ohb+FV7WeHaDgUs3LlyUkAGb28kBdG3PFZY0apoYw0Ct6dpJYWSaiHqcjPCEVy05EIDEtBwAwvI0zpvXwgNRYX+12zdxzDdsuxUBqrI8DE9rDuUb53KEVK8tEZEI6QmNlWH/uHh7JsgAA9WzNMb1nPXSuZ6PxYLsrKAZT/7xWaLu+jgQ6OiXXJZcL5MoLn6ZHtHVBj8Z28LCzgKWpQaltKOoSXHkr6r3Gnh0iqjDlOYCwvOpW5c4iAOjcuTNCQkKQmJiINWvWYMiQIbh48SJsbGxKrWPt2rXo2bNnkUGnqop6nI6Xz50CwA9/Fx6gLDXWh5WZwbMAVDCOIzY5E0dvxStdFqlrYwa/gY3R0tWy0DFUNa9vQ4TGpiIkJhnjNgdhz3hvmBho9hS1IzAaM/Zcx4tf8R2kRpjiUw8DmtWEbinB41W1r2sFHQkKnfTPTC/9pF9cYPisc221/r3YS41f+79te6kx/AY2LlOvkqawZwfs2aE328u3pb7OAYRlqftV7yyqW7cuRo8eXerA4vv376NWrVrYs2cP+vXrV+Q+VbFnZ8uF+/hm3w2lbRIUXL5JzcpDYlo2nqTnIL+I3oSiSACcnvoWnGuYlrltsbJM9Fl2FolpOejr6YD/DWuqsV6WqzFP0W/FeaVtEgAnv3oLrlZlb3tpShv3Ul5lK4Py7FVizw4RlSpWlqkIG0DBN89Ze26go7t1uX/7KqrumXuuq1z3q9xZBBRc/srOzi71+OvXr4eNjQ169+5d6r5VRWpWLpb6hwMoONELoMiTp1wukJyZWzDfSmo2HqdlIzEtB1djkrH/6iOlYwoAD5OzNBJ27KXGWPF+cwz/v4vYf/URmjhK8XGHWmU6Zr5cYMuF+/D7J7TQcwJArCzrtYSd0sa9lFfZyqAiepVexrBD9Aa7HZtS6JLG6xpAGJVY+HKKXABbL9zHlLfrlTqWoU+fPvj+++/h7OyMhg0bIjg4GEuWLMHo0aMBAOnp6fj+++/Rt29f2NvbIzExEStWrMDDhw+VbiN/+c4ioCAQrV+/HiNHjoSeXuGPybi4OMTFxSmWArh+/TrMzc3h7OystERCaV735cOfj95BQmo2XGuYYMNHrZ+d6AufPHV0JLA0NYClqQHcbc2V2nvw2qNCl1Q0OQNym1o18HXv+vj2wC34/XMbDRws4F3b6pWOFRqbgpl7riMkJrnI5zXd9tKU5aRfGQJDVca1sYgqWH5+PmbPng03NzcYGxujdu3amD9/Pl68wrxnzx74+PigRo0akEgkCAkJUenYu3btgoeHB4yMjNC4cWP8/fffiueS0nOw+Mgd5CbGIGH3d4j+ZQiilwxC7MbJ0MtUby2kV+FWzLfp5Scj0Wf5Wfwb/rjE8suWLcO7776L8ePHo379+vjqq6/wySefYP78+QAKenlu376NQYMGwd3dHX369MGTJ0/w77//omHDhgAKTt63wsIR9SBW6djHjx9HdHS0Iji9bOXKlWjWrBnGji24i6ljx45o1qwZ9u/fr/Lr3xEYjXYLT+D9NRfRbuEJ7AiMVrnsq7j+QIZNAfcAAPP7N4KrlSm8atdQe9yH38DG0H12aam8xmCM8nbFgGY1kS8X+PyPYDxKVm89psycfCz85zb6LDuLkJhkmBvqYX7/Rq+l7VQ5ccwOOGaHKtYPP/yAJUuWFJoc7/vvv1dMjrd582ZERUXBwcEBY8eORXBwMJo2bVricc+fP4+OHTvCz88P77zzDv744w8sWrQIV65cgdShFkauu4Sw8AjEbZ4C8yZvw6R+J0gMTGCW8QgnF4+Dg71dub7uM3ceY8S6S4qfdSTA2w1scT7iCVKfTYjWoa4VpvfwUMy8q0kVOVbp4dMMtF90skxzn6gjXy4w4LdzuPZAhr6eDlj6XrMyHe913NmTmZOPQb+fx63YFDRxlGLnJ14w0i99ba5/wx/j6703EJ1UsLhor8Z2mNunIWwtjF5b2+n1UfX8zbADhh2qWOpMjnfv3j24ubmpFHaGDh2K9PR0HDx4ULGtbdu2cKnbAHc93sfj1Gyk/f0T2tezxe//tx43HsgwZddVpGblYX6/hvjQy1WTL1NJbr4cPf/3LyIS0jCkZcG8JM9PPknpOVh+IgKbL9xDbn7Bx1NfTwd85VNPY7cjRyelo9OPp5TuyinPsPGi7Lx8jFx3CRfuJhV6btvYtvCqXUPjdW4KuIc5f92EuZEe/L/sBBtzI43XUR5ikjLQZ/lZJGfkYkhLRywa1KTYActP0rKx4FAo9gY/BFBwl9V3/RqhWwPbIvcn7aDq+ZuXsYgqmLe3N/z9/XHnzh0AUEyO17NnzzIdNyAgAN26dVPa1rBVB+w/dgqPU7Phbm2KrKggtGjSEKOG9Mfwzk2QsWs6Mu4E4Odjd/A0PadM9ZdkU8B9RCSkwdLUAF/3bqB0OcXS1ABz+jTAiS/fwoBmNSGRAPuvPkLXJacwb/9NPEkrfXDxy4QQiEhIxfpzURizIRA+S87g5a95z8cqlafUrFyM3hBYZNDRkaBcxo8kpGThx2eT103rXq/KBB0AcLI0wbL3mkFHAuwMeoCtFwtf6hNCYGdQDLouOY29wQ+hIwE+aueKo1M6MeiQAgcoE1WwGTNmICUlBR4eHkqT4w0fPrxMx42Li1NahPDQtVgcCM9ATupTtHazxAIfR9T7Kg0LFy7EggULsGjRIvz99z/4+puvoWNkhiXHHDC/f6OyvrxCEtOy8euxgmA3rXu9Yiegc7I0wS9Dm+LjDm5YdDgMZ+48xobz9/Dn5Qf4pGMtjOngVuI8LE/SsnE2IhFnwxNxNiIRsc8mjyvJoeuxaO1mWS7zrSSkZmHUukDcik2BqYEuhrV2xvpzUYrBvpamBuWyEOb8Q6FIzc6Dp6MU77dx0fjxy1uHutaY2t0Diw7fxrcHbqK+vQVauBTMbnz3cRpm7b2uCI8N7C2wcFBjNHGsVoEtpsqIYYeogqk6OV5ZbDx/D/MO3ES+XMBQTwebRrdG0uN4AEC/fv0weXLBNP9NmzbFP/6ncSXkH2x1aYz3WjujgYNmL+3+eDgMqdl5aFTTAoNbOpW6f0MHKTaNbo1zEYlY+M9tXH8ow8/H7mDThfuY2LUuOrpb4cHTTDhIjfHgaSb+DX+Mf8MTcSs2Rek4Bno6aO1qiQ51rdC+rhWuPZDhm70Fc5c8vw17y4X7ePA0A/8b1uyVZgEuTlRiOkasu4iYpExYmRlg/ajWaOwoxccd3HA1Jhlz/rqJhNRsTNl5Fas+aFHqnWiqOnPnMQ5cfQQdCfD9gMblNmleefu0Uy1cf5iMv6/HYdymIMzt0wDXH8qwMeA+cvLkMNbXxeS362J0OzelBUeJnmPYIapgU6dOxYwZMzBs2DAAQOPGjXH//n34+fmVKezY2dkhLi4OPx65jRUnIwEADarLkeLqCCN9XVhZWUFPTw8NGjRQKufd0hMRe49ALoBvD9zE9nFtNTax27UHydh5OQYAMK9PQ7VOvu3qWOEv33Y4dD0WPx4JQ3RSRqHJ8V5W394CHZ+Fm1aulkoDXBs6SPFWvf/mLrkUlYRpf17DqbDH6L/iHNaMaIE6NuYlHF01V2OS8dGGQCSl58Clhgk2jW4Nl2dz0jy/ndheaozBqwJw7FY8VpyMwOdd65a53qzcfMz+q+D3M9LbtVwGeb8uEokEi9/1RNC9p0hIzcYX20MUz3Vyt8aC/o3gZPn6biGnqocRmEiDYmWZOB+ZiFiZ6rfKvurkeKVp07Ytlm/Zpwg6X77tjvyYa/Dy8gIAGBgYoFWrVggLU16M8M6dO2jTxB1G+jq4GJWEQ9djCx37VcjlAvP234QQwIBmNV9paQEdHQn6eDrg+JROmPK2e5H79Gpsh/8Na4rAr7vhn4kdMLNXfXSoa13knTz2UmPFeKF+TWti92fecJAaISoxHf1XnMfxW/Fqt/FFp8IS8N6aC0hKz0HjmlLs/sxbEXRe5OlUDQv6FVwyXHL8Dk6GJRTaR12/nYrE/ScZsLUwLPZ3VZWkZhVMcvgiiQTwG8igQ6Vj2CHSkFedN+X55HiHDh3CvXv3sHfvXixZsgQDBgxQ7JOUlISQkBDcunULABAWFoaQkBDExcUp9hkxYoRiCYSMnDxk1PXB3ZBzSL20B583N8aTf7ciKChIafK8qVOnYseOHVizZg0iIiKwfPlyHDhwAF9N+gKfdaoDAPjhUCgyc/LL/PvZF/IQV6KTYWKgixk9Pcp0LAM9HbR0LXpV6g/buqJf05qwNjcs8vmSNKopxf7P26O1qyXSsvMwdnMQlp8Ix6vctLrnygN8vDEIGTn56FDXCtvGtYWVWfFtGtLKCcPbOEMIYOK2YNxLTFe7zuciH6dh5amCkDvnnYYwN9LcJbmKUtQklEIA95+oNwcPvZkYdog0oLhlF1Tp4SltcjwA2L9/P5o1a6ZYumDYsGFo1qwZVq5cqdgnOjoasbGxSErPwftrLuJmvj3s+0+DSdRpzPqwJ/7880/s27cPjRr9N+h4wIABWLlyJRYvXozGjRvj//7v/7B79260b98en3SqhZrVjPFIloXfT0eW6feTlp2Hhf8UrFc1oUsdxZwnZeFmZYqXr4JpYkZcKzNDbPm4DT5oWxA8fjp6B75/XEH6s7l/SiOEwKrTkZiy8yry5AL9mzpg7chWKg0+ntunIZo7V0NKVh4+2XxZ5Tpfrn/2vhvIyZejk7s1ejUu3/mSXpfy+nvTm4Hz7IDz7FDZ7b/6CF9sCy60vbzmTSlKrCwTgVFJ+OloGKKTMiE11se6US3RwuXVV6L+53osPtt6BQZ6OvCf0umVLxcs/Oc2Vp6OhEsNExyd3BGGeqVPDqeK8l4gcdulaMz56wZy8wU87MyxZkTLEn8HcrnA93+HYu3ZKADA2A5umNmzvloDjuNTsvDOsrN4nJqN3k3ssfy9ZmqNmdoX/BCTdoTAUE8HRyd3LPKyWVVV1RfEJM3jpIJqYNihsniSlo0Bv51DdJJyL46OBDg3o8trmaV1R2A0Zuy5rpg7Rmqsh92feZd5gK0QAsP/7yLORz5Bj4Z2WPlhC7WPEZWYDp9fTiM3X+D/RrTU+Nwn5T0jbtC9JHy65QoS07JR3UQfK95vDu86hddqys7Lx1e7ruHAs4Uyv+5VH2M7vtoiloH3kvDe6gvIkwvM6uWBcR1rq1ROlpGLrktOITEtB1/5uGNCl7IPdK5sOAMyvYiTCtJr5+rqColEUujh6+sLoGDelw8//BB2dnYwNTVF8+bNsXv37lKPu2LFCri6usLIyAht2rTBpUuXlJ7/5JNPULt2bRgbG8Pa2hr9+vXD7du3y+U1viw1Kxcj11961pOip9TN3sRRCjsNXK4pTawsUynoFLQrD6YamLNFIpFg7rO7pg7fjMO5iES1j7Hg4C3k5gt0crdG1/o2ZW7Ty14cZFweWrpa4sDn7dDEUYqnGbn4cN0lbDgXpTSO5/lkgQeuPoK+rgS/Dm36ykEHAFq5WmJun4K75Bb+c1vl3/viI7eRmJaD2tamZaq/Mivvvzdpp0oddlRZIFEIgTlz5sDe3h7Gxsbo1q0bwsPDK7DVb67AwECs/DsQjhM2w9F3M+yGLQAAxQrTI0aMQFhYGPbv34/r169j4MCBGDJkCIKDC1/+eW7Hjh2YMmUK5s6diytXrsDT0xPdu3dHQsJ/d6u0aNEC69evR2hoKI4cOQIhBHx8fJCfX/ZBtSXJys3H2E1BuPEwBTVMDbBnfDucm9EFc96pD10JEBIjw7pz98q1DQDwb3hiodmA5QIamw24np05PmxbMBndtwduIi9f9bvEToYlwP92AvR0JJjTp4HGbmF/3eylxtj5iZdiccp5B25h2p/XcP9JOv6+/giDfj+PcxFPYGqgi3WjWqF/s5plrvODti54t4Uj5AKY8McVPHha8t8zOPop/rhUMCh+Qf/GGrtUSKQNKnXYWbRoEX7//XcsX74coaGhWLRoERYvXoxly5Yp9lm8eDGWLl2KlStX4uLFizA1NUX37t2RlVX6bKmkOflygeN3M+B3Oh66ptWha1Yd6eGXoFfNHnU9WwEoWJjy888/R+vWrVGrVi188803qFatGi5fvlzscZcsWYKxY8fio48+QoMGDbBy5UqYmJhg3bp1in3GjRuHjh07wtXVFc2bN8eCBQsQExODe/fuldvrzcuXY8IfwbhwNwlmhnrYOLo1alubwV5qjNHta+Gbdwq+lf/wdygCIstvBfGYpAwsPly4F0vTAzcnd3NHdRN93IlPw5YL91Uqk5Mnx/wDBXePfdTOFbWtzTTWnopgpK+LJUM88U3v+tCRALsuP0CnH09h/NZg3IlPg6mBLraP80KHutYaqU8ikWBB/0ZoXLOgR+nTLZeRlVt0gM/Ll+PrvTcgBDCwec3XNk6MqKqo1GHn/Pnz6NevH3r37g1XV1e8++678PHxUVzGEELg119/xTfffIN+/fqhSZMm2LRpEx49eoR9+/ZVbOPfEBk5edh4/h66/HwKM/f+N8GbyM9F+q1TMGvyNj7eeBmRj9Pg7e2NHTt2ICkpCXK5HNu3b0dWVhbeeuutIo+dk5ODy5cvK63vpKOjg27duiEgIKDIMunp6Vi/fj3c3Nzg5FT67LyvQi4XmL77Oo6HxsNATwf/N7JloQnbRnm7KnoBJvxxBY+SNX977OPUbHy49iIS03JgY26ouIT2fOCmJrv5pSb6+Kp7PQDAkmN3VFqfasP5KNxNTIeVmSG+0MAkeZWBRCLBxx1q4ZehTQs9l5mbDytzA43WZ6Svi5UftoClqQFuPEx5FmgKD7PcGHAft2JTIDXWx6xe9TXaBiJtUKnDTmkLJEZFRSEuLk7pZCiVStGmTZtiT4YAkJ2djZSUFKUHqSchJQs/HrkNL78TmLv/Ju4/yYCFkR6eX6TIuHMB8qw0mDbqitC4VPT89V90Hu+H7Jwc1KhRA4aGhvjkk0+wd+9e1KlTp8g6EhMTkZ+fr7S+EwDY2toqzS8DAL/99hvMzMxgZmaGf/75B8eOHYOBgWZPPEBBwP7+71DsvvIAujoSrHi/OdrWKvwtWiKR4IcBjdHA3gJP0nPwWQnfyl9FSlYuRq67hHtPMuBY3Rj7J7THuRldsG1sW5yd0blc7lAZ1soZDewtkJKVh5+O3ilx34TULCz1jwAATO9RTyvmeXlRUXP4aPLS4YtqVjPG8meLYe6+8gCbX+pZi5VlYsnRgokhZ/T0KHEuH6I3VaUOO8+n0Pfw8IC+vj6aNWuGSZMmKRZIfH7CU+Vk+CI/Pz9IpVLFo7x6ALRRWFwqpu66ivaLTmLFyUjIMnPhUsME3/VriAuzumLhoMbQlUiQdu0oTGq1xNeDvdHJ3Ro5+XL4zZ+HS7djsHzzHgQFBWHKlCkYMmQIrl+/XuZ2DR8+HMHBwTh9+jTc3d0xZMiQcrmUueJkhOK24sWDmuDtEu4sMjbQxaoPW6CaiT6uPpBh7l83X2lyupdl5uTj4w1BuBWbAiszA2we0wZ2UqNyH7ipqyPBvL4NAQDbA6Nx46Gs2H0XHw5DWnYePJ2qYVBzx3JpT0V63XO+eNexwsyeBT023x24hcB7/62a/t2BW0jPyUdz52oYqsJaY0Rvokoddl5cIPHKlSvYuHEjfvrpJ2zcuLFMx505cyZkMpniERMTo6EWaychBM6GJ2LEukvo/usZ7Lr8ADn5crR0qY6VH7TAiS/fwggvV5gY6GFoK2dsf782cqKvYvl3X+Kzt+pgw0etMLN9daReOQjTtz/HjzcMsD1SgklTZ6Fly5ZYsWJFkfVaWVlBV1cX8fHKU/bHx8fDzk55ojSpVIq6deuiY8eO+PPPP3H79m3s3btXo7+HzRfuK3o0Zr/TAINalH4Sd7I0wbJn38p3BMUoBpC+qtx8OXz/uIJL95Jg/myskJvV65tHpbWbJfp6OkAIPFv6oXB4C45+ij8vPwAAzOvTQGOLWlYm9lJj+A0sCPZA+Vw6fNnHHdzQx9MBeXKB8VuvID4lCydux+OfG3HQ1ZHg+wGNtfJ3TaQJlXoh0NIWSHx+wouPj4e9vb2iXHx8PJo2bVrscQ0NDWFoyK7eksTKMhEen4bw+FTsuvwAt+NSARTMHdOzkT0+7uCGZs5FT9d/aPc22NjY4MMhAwEUXNLxdimY76VXEwccjwW2XYrBsVsJkGfkFXvXlIGBAVq0aAF/f3/0798fACCXy+Hv76+05MHLhBAQQiA7u/RxJao6cPUR5jxbVPHzLnUwpr2bymU71LXG1O4eWHT4NubtvwkPOwu0cCn6d1cSuVxg6q6rOHE7AYZ6Olg7qhUaOrz+xR1n9vLAsVvxCLr/FPuvPkK/pv/defR8/SsAeLeFY7HvEW0wtJUzOrpbv7Y5XyQSCRYNaozw+FTcjkvF6A2BiE8p6L0c094N9e05RxhRcSp1z05pCyS6ubnBzs4O/v7+iudTUlJw8eJFxWKHpL7tl6Lh7XcCI9ZdwvxDobgdlwoTA12M8nbF6amdsWJ482JPYnK5HOvXr8fIkSOhp/dflvbw8ECdOnVwZ/cSzGljAHtJMu6e2IbggDOIMmuAmKSCsQ5du3bF8uXLFeWmTJmCNWvWYOPGjQgNDcVnn32G9PR0fPTRRwCAu3fvws/PD5cvX0Z0dDTOnz+PwYMHw9jYGL169dLI7+P0nceYsjMEQgAftnV5pUUVP+1UC70a2yE3X2D81stISFXvEpsQAt8dvIV9IY+gpyPB7x80R2u3V58ZuSzspcbw7VwwyZ3f37eVljTYfeUBrj6QwcxQD9N61KuQ9r1Or3vOFxMDPaz6sAWM9HRw81EKEtNyAACO1TjnDFFJKnXYKW2BRIlEgkmTJmHBggWKuVtGjBgBBwcHRU8Aqef5Gk8vXpyQANg3vh3m9W1Y6nIBx48fR3R0NEaPHq20XV9fH3///Tesra0x89MPcG3pOBjfPwfbPlMQYegOn1/OYM2Zu4iMjETUg1jFyuFDhw7FTz/9hDlz5qBp06YICQnB4cOHFeO0jIyM8O+//6JXr16oU6cOhg4dCnNzc5w/fx42NmWfwO7y/SR8uvkycvMF+ng64Nu+DV9prhiJRILF73qiro0Z4lOy4bv1CnLyVJ+vZql/BDacvwcA+GmwJ7p4aHYWYnV93KEWnCyNEZeShd9OFQxETsnKxaLDBQNlv+haBzbm5T+h4pvIQE8H2S/NdfTtgVsqrcNG9Kaq1MtFpKamYvbs2di7dy8SEhLg4OCA9957D3PmzFHcaSOEwNy5c7F69WokJyejffv2+O233+Durvq3by4X8Z/dl2Pw5a5rhbaX1xpPEQlpmLX3Oi5FFQy4dJAaITYlC0IUXDLzG9i4wta+uR2XgiErA5CSlYdO7tZYM6IlDPTK9v3g7uM09Ft+DqnZeRjp5YJv+zUqtczG8/cw99mloW/7NsRIb9cytUFTjt6Mw7jNl2Ggq4NjUzpiy4X7WPNvFGpZmeLwpI5l/l1R0c5HJuL9NRcLbX+d67ARVRZcG0sNDDsF8vLl6LP8HEJjlW/F15VIcHZG53LrqpfLBXZdjsGCQ6FIzVJe5VlHAvzl2w6Nakpf6+y70U8yMGjleTxOzUYLl+rYPKY1TAw0M8TNPzQeYzYGAQB+HuxZ4kDnv0IeYuL2EADApG51Mamb+pfQyosQAiPWXcK/4Ylo5GCBW7EpkAtg/Uet0Lme5peFoAKxsky0W3gC8hc+ucv73yhRZcW1sUhtK05GIjQ2BUZ6OuU6Qd3LdHQkGNrKGT++26TQc3IB9Fl+Di0WHMf7ay7guwO3sCsoBjceyoqctyZWlqm4BPYqYmWZ+PvaIwxbE4DHqdnwsDPHupGtNBZ0AKBrfVtMfDbJ3qy914u9hfvk7QR8ufMqgIJJCidWson5CtbNagAdCXDjUYri5JuQwtnLy1NF3AlGVNVV6rux6PW5GpOMpScK1hRb9G4TtHazfO0rC3s6VYOOBErfWIGCMUNJ6Tk4H/kE519YekFXRwI3K1N42Jmjvr0FHqdmYVPAfchf8RLYjsBozNxzXVG/pakBNo1uDamJ5ifEm9i1Lm48lMH/dgI+2XwZBz5vD0vT/yZBDLyXhE+3XEaeXGBAs5qY807lXFfK1FCv0Lpcs/bcQEd3a558y9HrvhOMqKrjZSzwMlZmTj56L/sXdx+n450m9lj+fvMKa8uOwGjM2nMD+UIovrH2a1oT4fFpCI1LQWhsCm7HpiI0LgXJGbmlHq+asb5Kc4/I5QLJmcrH05EA52Z0KbcTiSwzF/1XnENUYjq8a9fAptGtoaerg1uPUjB0dQBSs/LQ1cMGKz9sAX3dytkJy/EjRFSRVD1/s2eHsPCfUNx9nA5bC0Ms6F/6gNnyVNw31saOUjR2/G9OGSEEElKzC8JPXCr+vfMY54pYcPPlAKOO59P/l1fYkRrrY9WHLdB/xTmcj3yCOftvopVLdXx3sGDsUmtXS6wY3rzSBh3gv5mEXx4/Ul4zCRMRvQr27ODN7tk5fecxRq4rWFh185jWGlux+XUratCmjgTYMqZNkesYvexxajY+WHuxQgZ9/n09FuO3XlHaZi81wpHJHWFRBdaUKqo3rqLuoCOiNwt7dqhUyRk5mLrrvwGwVTXoAP8N2nz5pOtdx0ql8nVtzYss/zrGQjRzrgYJoDS3UXxKFtKz86pE2OH4ESKq7Bh23lBCCHy97wYSUrNRy9oU03t4VHSTyqysJ92KOmlHJabj5e7V8r6Epmn2UuMq01YievMw7Lyh9l99hEPXYqGnI8GvQ5vC2EC3opukEWU96VbESZvjXoiIylflHflI5eZRcia+2VewqOUXXeuiiWO1im3QG47zphARlS/27Lxh5HKBqX9eRWpWHpo6VcP4t2pXdJMIHPdCRFSeGHbeMBvO38O5iCcw1tfFkiGe0KvEtzW/aTjuhYiofPBM9wYJj0/FwsO3AQBf966PWtZmFdwiIiKi8sew84bIyZNj8s4Q5OTJ8VY9awxvw3lQiIjozcCw84ZY6h+OGw9TUM1EH4sHNamU6ywRERGVB4adN8Dl+0n47VQEAMBvQGPYWBhVcIuIiIheH4YdLZeenYcpO69CLoCBzWqiZ2P7im4SERHRa8Wwo+UWHArF/ScZqFnNGPP6Nazo5hAREb12vPVcS8XKMrHn8gNsuxQNiQT4abBnlVhniYiISNMYdrTQjsBozNxzXbH8QPvaVvCqXaNiG0VERFRBeBlLy8TKMpWCDgCci0xErCyz4hpFRERUgRh2tExUYrpS0AH+W0GbiIjoTcSwo2XcrEzx8hQ6XEGbiIjeZAw7WsZeaoz2dawUP3MFbSIietNxgLIWepRcMD5nYte6GNbaiUGHiIjeaAw7Wib6SQYiH6dDV0eC0e3dIDXm7eZERPRm42UsLXMyLAEA0NKlOoMOERERGHa0zonbBWGni4dNBbeEiIiocmDY0SIZOXkIuPsEAMMOERHRcww7WuR8xBPk5MnhWN0YdWzMKro5RERElQLDjhZ5Pl6ni4cNJC9PtkNERPSGYtjREkIInHw2XqdzPV7CIiIieo5hR0uExafikSwLRvo6XPSTiIjoBQw7WuL5XVjeta1gpK9bwa0hIiKqPBh2tITiEhbvwiIiIlLCsKMFkjNycPn+UwBA53rWFdwaIiKiyoVhRwucCU+EXADutmZwrM7VzYmIiF7EsKMFeAmLiIioeAw7VVy+XODU8/l1eMs5ERFRIQw7VVxITDKeZuTCwkgPLVyqV3RziIiIKh2GnSru+SWsju7W0NPln5OIiOhlPDtWcS8uEUFERESFMexUYfEpWbj5KAUSCdDJnbecExERFYVhpwp7fgnL07EaapgZVnBriIiIKieGnSrs+RIRvIRFRERUPIadKio7Lx9nIxIBMOwQERGVhGGniroUlYSMnHzYmBuioYNFRTeHiIio0mLYqaJO3n4MAHirnjUkEkkFt4aIiKjyYtiponjLORERkWoYdqqgqMR0RCWmQ19XgvZ1ecs5ERFRSRh2qqDnd2G1drOEmaFeBbeGiIiocmPYqYIUq5xz4U8iIqJSMexUMenZebgY9QQAx+sQERGpgmGnijkbkYjcfAGXGiZwszKt6OYQERFVegw7VcyLl7B4yzkREVHpGHaqECEEbzknIiJSE8NOFXLzUQriU7JhYqCLNrUsK7o5REREVQLDThVy6lmvTrs6VjDU063g1hAREVUNDDtVyAneck5ERKQ2tcPOyZMny6MdVIqk9BwExyQDADp7cNZkIiIiVakddnr06IHatWtjwYIFiImJKY82URFO30mAEEB9ewvYS40rujlERERVhtph5+HDh5gwYQL+/PNP1KpVC927d8fOnTuRk5NTHu2jZ048W+W8C3t1iIiI1KJ22LGyssLkyZMREhKCixcvwt3dHePHj4eDgwO++OILXL16tTza+UbLy5fjNG85JyIieiVlGqDcvHlzzJw5ExMmTEBaWhrWrVuHFi1aoEOHDrh586am2vjGC45JRkpWHqqZ6KOpU/WKbg4REVGV8kphJzc3F3/++Sd69eoFFxcXHDlyBMuXL0d8fDwiIiLg4uKCwYMHa6SBDx8+xAcffIAaNWrA2NgYjRs3RlBQkOJ5IQTmzJkDe3t7GBsbo1u3bggPD9dI3ZXF87uwOrlbQ1eHsyYTERGpQ+2w8/nnn8Pe3h6ffPIJ3N3dERwcjICAAHz88ccwNTWFq6srfvrpJ9y+fbvMjXv69CnatWsHfX19/PPPP7h16xZ+/vlnVK/+X+/G4sWLsXTpUqxcuRIXL16EqakpunfvjqysrDLXX1k8XyKCl7CIiIjUp6dugVu3bmHZsmUYOHAgDA0Ni9zHyspKI7eoL1q0CE5OTli/fr1im5ubm+L/hRD49ddf8c0336Bfv34AgE2bNsHW1hb79u3DsGHDytyGivYwORO341KhIyno2SEiIiL1qN2z4+/vj/fee6/YoAMAenp66NSpU5kaBgD79+9Hy5YtMXjwYNjY2KBZs2ZYs2aN4vmoqCjExcWhW7duim1SqRRt2rRBQEBAscfNzs5GSkqK0qOyet6r09y5OqqZGFRwa4iIiKoetcOOn58f1q1bV2j7unXrsGjRIo006rm7d+/i999/R926dXHkyBF89tln+OKLL7Bx40YAQFxcHADA1tZWqZytra3iuaL4+flBKpUqHk5OThpttyY9XyKiMy9hERERvRK1w86qVavg4eFRaHvDhg2xcuVKjTTqOblcjubNm+OHH35As2bNMG7cOIwdO7bM9cycORMymUzxqKyTI2bl5uNcxBMAXCKCiIjoVakdduLi4mBvb19ou7W1NWJjYzXSqOfs7e3RoEEDpW3169dHdHQ0AMDOzg4AEB8fr7RPfHy84rmiGBoawsLCQulRGV24+wSZufmwlxqhvr15RTeHiIioSlI77Dg5OeHcuXOFtp87dw4ODg4aadRz7dq1Q1hYmNK2O3fuwMXFBUDBYGU7Ozv4+/srnk9JScHFixfh5eWl0bZUhINXHwEAWrtZQiLhLedERESvQu27scaOHYtJkyYhNzcXXbp0AVAwaHnatGn48ssvNdq4yZMnw9vbGz/88AOGDBmCS5cuYfXq1Vi9ejUAQCKRYNKkSViwYAHq1q0LNzc3zJ49Gw4ODujfv79G2/K6bb8UjT+vPAQA7L/6CN61a2BoK+cKbhUREVHVIxFCCHUKCCEwY8YMLF26VLEelpGREaZPn445c+ZovIEHDx7EzJkzER4eDjc3N0yZMgVjx45Vas/cuXOxevVqJCcno3379vjtt9/g7u6uch0pKSmQSqWQyWSV4pJWrCwT7RaegPyFv4yuRIKzMzpzEVAiIqJnVD1/qx12nktLS0NoaCiMjY1Rt27dEm9Fr+wqW9g5H5mI99dcLLR929i28KpdowJaREREVPmoev5W+zLWc2ZmZmjVqtWrFqcSOFYv3HujK5HA1cqkAlpDRERUtb1S2AkKCsLOnTsRHR2tuJT13J49ezTSsDfZrUepSj/rSiT4YWAjXsIiIiJ6BWrfjbV9+3Z4e3sjNDQUe/fuRW5uLm7evIkTJ05AKpWWRxvfONsuFdxa/2FbF2wb2xZnZ3Tm4GQiIqJXpHbY+eGHH/DLL7/gwIEDMDAwwP/+9z/cvn0bQ4YMgbMzT8hl9eBpBs6EPwYAfNzBDV61a7BHh4iIqAzUDjuRkZHo3bs3AMDAwADp6emQSCSYPHmy4pZwenU7A2MgBNCuTg241DCt6OYQERFVeWqHnerVqyM1tWBMSc2aNXHjxg0AQHJyMjIyMjTbujdMXr4cO4IKlq54rzV7yYiIiDRB7QHKHTt2xLFjx9C4cWMMHjwYEydOxIkTJ3Ds2DF07dq1PNr4xjgV9hjxKdmwNDXA2w1sSy9AREREpVI77CxfvhxZWVkAgK+//hr6+vo4f/48Bg0ahG+++UbjDXyTPB+Y/G4LRxjq6VZwa4iIiLSDWmEnLy8PBw8eRPfu3QEAOjo6mDFjRrk07E0TK8vEybAEAMDQVk4V3BoiIiLtodaYHT09PXz66aeKnh3SnJ2BDyAXQBs3S9S2Nqvo5hAREWkNtQcot27dGiEhIeXQlDdXvlxg57OBye+34cBkIiIiTVJ7zM748eMxZcoUxMTEoEWLFjA1Vb49ukmTJhpr3JviTPhjPEzORDUTfXRvaFfRzSEiItIqaoedYcOGAQC++OILxTaJRAIhBCQSCfLz8zXXujfE9mcDkwc2c4SRPgcmExERaZLaYScqKqo82vHGSkjJwvHQgoHJ77XmwGQiIiJNUzvsuLi4lEc73li7Lj9AvlygpUt11LU1r+jmEBERaR21w86mTZtKfH7EiBGv3Jg3jVwusD2w4BLWMM6YTEREVC7UDjsTJ05U+jk3NxcZGRkwMDCAiYkJw44azkUmIiYpE+ZGeujd2L6im0NERKSV1L71/OnTp0qPtLQ0hIWFoX379ti2bVt5tFFrbb9UcLv5gGY1YWzAgclERETlQe2wU5S6deti4cKFhXp9qHiJadk4eisOADCsFS9hERERlReNhB2gYHblR48eaepwWm/35QfIzRfwdKqGBg4WFd0cIiIiraX2mJ39+/cr/SyEQGxsLJYvX4527dpprGHaTAiB7YHPZkzm7eZERETlSu2w079/f6WfJRIJrK2t0aVLF/z888+aapdWu3A3CVGJ6TAz1MM7TRwqujlERERaTe2wI5fLy6Mdb5Rtz2ZM7tvUAaaGav8JiIiISA0aG7NDqnmanoPDNwoGJr/PuXWIiIjKndphZ9CgQVi0aFGh7YsXL8bgwYM10ihttvvKA+Tky9GopgUa1ZRWdHOIiIi0ntph58yZM+jVq1eh7T179sSZM2c00iht9eLA5PfYq0NERPRaqB120tLSYGBgUGi7vr4+UlJSNNKoymbhwoWQSCSYNGkSAODevXuQSCRFPnbt2lXscXR0dOD/5Vu4v+gdfNDWVVHmxx9/VOzTt29fODs7w8jICPb29vjwww95Sz8REVEZqB12GjdujB07dhTavn37djRo0EAjjapMAgMDsWrVKjRp0kSxzcnJCbGxsUqPb7/9FmZmZujZs2exxxq38hgcfTdj/Gp/xMbGYt26dZBIJBg0aJBin86dO2Pnzp0ICwvD7t27ERkZiXfffbdcXyMREZE2U/tWoNmzZ2PgwIGIjIxEly5dAAD+/v7Ytm1bib0aVVFaWhqGDx+ONWvWYMGCBYrturq6sLOzU9p37969GDJkCMzMzIo8liwjF6dicqFrVh0fd28GO7vq+Ouvv9C5c2fUqlVLsd/kyZMV/+/i4oIZM2agf//+yM3Nhb6+voZfIRERkfZTu2enT58+2LdvHyIiIjB+/Hh8+eWXePDgAY4fP15oDp6qztfXF71790a3bt1K3O/y5csICQnBmDFjit1nb/ADZOfJ4WFnjqZO1RAfH49Dhw6VWCYpKQlbt26Ft7c3gw4REdEreqVJXnr37o3evXtrui2Vyvbt23HlyhUEBgaWuu/atWtRv359eHt7F/n8ywOTJRIJNm7cCHNzcwwcOLDQ/tOnT8fy5cuRkZGBtm3b4uDBg2V7MURERG8wtXt2AgMDcfHixULbL168iKCgII00qqLFxMRg4sSJ2Lp1K4yMjErcNzMzE3/88UeJPTTBMcm4HZcKQz0d9G9WEwCwbt06DB8+vMjjT506FcHBwTh69Ch0dXUxYsQICCHK9qKIiIjeUGqHHV9fX8TExBTa/vDhQ/j6+mqkURXt8uXLSEhIQPPmzaGnpwc9PT2cPn0aS5cuhZ6eHvLz8xX7/vnnn8jIyMCIESOKPd72ZzMmv9PEAVJjffz7778ICwvDxx9/XOT+VlZWcHd3x9tvv43t27fj77//xoULFzT7IomIiN4Qal/GunXrFpo3b15oe7NmzXDr1i2NNKqide3aFdevX1fa9tFHH8HDwwPTp0+Hrq6uYvvatWvRt29fWFtbF3ms1KxcHLgaCwB479min2vXrkWLFi3g6elZalueL8+RnZ39Sq+FiIjoTad22DE0NER8fLzSHUQAEBsbCz097VjnydzcHI0aNVLaZmpqiho1aihtj4iIwJkzZ/D3338XeRwPDw/0GDUZmbmOqGtjhhYu1ZGSkoJdu3YVuWjqxYsXERgYiPbt26N69eqIjIzE7NmzUbt2bXh5eWn2RRIREb0h1L6M5ePjg5kzZ0Imkym2JScnY9asWXj77bc12rjKbt26dXB0dISPj0+Rz4eFheFw8F0AwLBnA5O3b98OIQTee++9QvubmJhgz5496Nq1K+rVq4cxY8agSZMmOH36NAwNDcv1tRAREWkriVBz5OvDhw/RsWNHPHnyBM2aNQMAhISEwNbWFseOHYOTk1O5NLQ8paSkQCqVQiaTwcLCQiPH3BEYjRl7ruP5b3fOOw0wur2bRo5NREREqp+/1Q47AJCeno6tW7fi6tWrMDY2RpMmTfDee+9V2blgNB12YmWZaLfwBOQv/GZ1JRKcndEZ9lLjMh+fiIiIVD9/v9IgG1NTU4wbN+6VG6ftohLTlYIOAOQLgXuJGQw7REREr9krjyi+desWoqOjkZOTo7S9b9++ZW5UVedmZQodCQr17LhamVRco4iIiN5Qaoedu3fvYsCAAbh+/TokEolisjuJRAIASnPQvKnspcbwG9gYs/bcQL4Q0JVI8MPARuzVISIiqgBqh52JEyfCzc0N/v7+cHNzw6VLl/DkyRN8+eWX+Omnn8qjjVXS0FbO6OhujXuJGXC1MmHQISIiqiBqh52AgACcOHECVlZW0NHRgY6ODtq3bw8/Pz988cUXCA4OLo92Vkn2UmOGHCIiogqm9jw7+fn5MDc3B1CwrMGjR48AAC4uLggLC9Ns64iIiIjKSO2enUaNGuHq1atwc3NDmzZtsHjxYhgYGGD16tWFZlUmIiIiqmhqh51vvvkG6enpAIDvvvsO77zzDjp06IAaNWpgx44dGm8gERERUVm80qSCL0tKSkL16tUVd2RVNeUxgzIRERGVr3KdVPBllpaWmjgMERERkcapPUCZiIiIqCph2CEiIiKtxrBDREREWk3tsHPmzBnk5eUV2p6Xl4czZ85opFFEREREmqJ22OncuTOSkpIKbZfJZOjcubNGGkVERESkKWqHHSFEkbeYP3nyBKamphppFBEREZGmqHzr+cCBAwEUrG4+atQoGBoaKp7Lz8/HtWvX4O3trfkWEhEREZWBymFHKpUCKOjZMTc3h7HxfwtcGhgYoG3bthg7dqzmW0hERERUBiqHnfXr1wMAXF1d8dVXX/GSFREREVUJao/ZmTZtmtKYnfv37+PXX3/F0aNHNdowIiIiIk1QO+z069cPmzZtAgAkJyejdevW+Pnnn9GvXz/8/vvvGm8gERERUVmoHXauXLmCDh06AAD+/PNP2NnZ4f79+9i0aROWLl2q8QYSERERlYXaYScjIwPm5uYAgKNHj2LgwIHQ0dFB27Ztcf/+fY03kIiIiKgs1A47derUwb59+xATE4MjR47Ax8cHAJCQkFDi8upEREREFUHtsDNnzhx89dVXcHV1RevWreHl5QWgoJenWbNmGm8gERERUVmoHXbeffddREdHIygoCEeOHFFs79q1K3755ReNNu5lCxcuhEQiwaRJkxTbsrKy4Ovrixo1asDMzAyDBg1CfHx8ubaDiIiIqo5XWvXczs4O5ubmOHbsGDIzMwEArVq1goeHh0Yb96LAwECsWrUKTZo0Udo+efJkHDhwALt27cLp06fx6NEjxWzPRERERGqHnSdPnqBr165wd3dHr169EBsbCwAYM2YMvvzyS403EADS0tIwfPhwrFmzBtWrV1dsl8lkWLt2LZYsWYIuXbqgRYsWWL9+Pc6fP48LFy6US1uIiIioalE77EyePBn6+vqIjo6GiYmJYvvQoUNx+PBhjTbuOV9fX/Tu3RvdunVT2n758mXk5uYqbffw8ICzszMCAgKKPV52djZSUlKUHkRERKSdVF4u4rmjR4/iyJEjcHR0VNpet27dcrn1fPv27bhy5QoCAwMLPRcXFwcDAwNUq1ZNabutrS3i4uKKPaafnx++/fZbTTeViIiIKiG1e3bS09OVenSeS0pKUloJXRNiYmIwceJEbN26FUZGRho77syZMyGTyRSPmJgYjR2biIiIKhe1w06HDh0Uy0UAgEQigVwux+LFi9G5c2eNNu7y5ctISEhA8+bNoaenBz09PZw+fRpLly6Fnp4ebG1tkZOTg+TkZKVy8fHxsLOzK/a4hoaGsLCwUHoQERGRdlL7MtbixYvRtWtXBAUFIScnB9OmTcPNmzeRlJSEc+fOabRxXbt2xfXr15W2ffTRR/Dw8MD06dPh5OQEfX19+Pv7Y9CgQQCAsLAwREdHK+b/ISIiojeb2mGnUaNGuHPnDpYvXw5zc3OkpaVh4MCB8PX1hb29vUYbZ25ujkaNGiltMzU1RY0aNRTbx4wZgylTpsDS0hIWFhb4/PPP4eXlhbZt22q0LURERFQ1qR12oqOj4eTkhK+//rrI55ydnTXSMFX98ssv0NHRwaBBg5CdnY3u3bvjt99+e61tICIiospLIoQQ6hTQ1dVFbGwsbGxslLY/efIENjY2yM/P12gDX4eUlBRIpVLIZDKO3yEiIqoiVD1/qz1AWQgBiURSaHtaWppG75giIiIi0gSVL2NNmTIFQMHdV7Nnz1a6/Tw/Px8XL15E06ZNNd5AIiIiorJQOewEBwcDKOjZuX79OgwMDBTPGRgYwNPTE1999ZXmW0hERERUBiqHnZMnTwIouPX7f//7H8e2EBERUZWg9t1Y69evL492EBEREZULtQcoExEREVUlDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Sp12PHz80OrVq1gbm4OGxsb9O/fH2FhYUr7ZGVlwdfXFzVq1ICZmRkGDRqE+Pj4CmoxERERVTaVOuycPn0avr6+uHDhAo4dO4bc3Fz4+PggPT1dsc/kyZNx4MAB7Nq1C6dPn8ajR48wcODACmw1ERERVSYSIYSo6Eao6vHjx7CxscHp06fRsWNHyGQyWFtb448//sC7774LALh9+zbq16+PgIAAtG3bVqXjpqSkQCqVQiaTwcLCojxfAhEREWmIqufvSt2z8zKZTAYAsLS0BABcvnwZubm56Natm2IfDw8PODs7IyAgoNjjZGdnIyUlRelBRERE2qnKhB25XI5JkyahXbt2aNSoEQAgLi4OBgYGqFatmtK+tra2iIuLK/ZYfn5+kEqlioeTk1N5Np2IiIgqUJUJO76+vrhx4wa2b99e5mPNnDkTMplM8YiJidFAC4mIiKgy0qvoBqhiwoQJOHjwIM6cOQNHR0fFdjs7O+Tk5CA5OVmpdyc+Ph52dnbFHs/Q0BCGhobl2WQiIiKqJCp1z44QAhMmTMDevXtx4sQJuLm5KT3fokUL6Ovrw9/fX7EtLCwM0dHR8PLyet3NJSIiokqoUvfs+Pr64o8//sBff/0Fc3NzxTgcqVQKY2NjSKVSjBkzBlOmTIGlpSUsLCzw+eefw8vLS+U7sYiIiEi7VepbzyUSSZHb169fj1GjRgEomFTwyy+/xLZt25CdnY3u3bvjt99+K/Ey1st46zkREVHVo+r5u1KHndeFYYeIiKjq0cp5doiIiIjUxbBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIq2lN2FmxYgVcXV1hZGSENm3a4NKlSxXdJCIiIqoEtCLs7NixA1OmTMHcuXNx5coVeHp6onv37khISKjophEREVEF04qws2TJEowdOxYfffQRGjRogJUrV8LExATr1q2r6KYRERFRBdOr6AaUVU5ODi5fvoyZM2cqtuno6KBbt24ICAgoskx2djays7MVP8tkMgBASkpK+TaWiIiINOb5eVsIUeJ+VT7sJCYmIj8/H7a2tkrbbW1tcfv27SLL+Pn54dtvvy203cnJqVzaSEREROUnNTUVUqm02OerfNh5FTNnzsSUKVMUP8vlciQlJaFGjRqQSCQaqyclJQVOTk6IiYmBhYXFay3Pul9/3WUtz7rfrLrLWp51s+6qUr6sdZdECIHU1FQ4ODiUuF+VDztWVlbQ1dVFfHy80vb4+HjY2dkVWcbQ0BCGhoZK26pVq1ZeTYSFhUWZ/sBlKc+6X3/dZS3Put+sustannWz7qpSvqx1F6ekHp3nqvwAZQMDA7Ro0QL+/v6KbXK5HP7+/vDy8qrAlhEREVFlUOV7dgBgypQpGDlyJFq2bInWrVvj119/RXp6Oj766KOKbhoRERFVMK0IO0OHDsXjx48xZ84cxMXFoWnTpjh8+HChQcuvm6GhIebOnVvoktnrKM+6X3/dZS3Put+sustannWz7qpSvqx1a4JElHa/FhEREVEVVuXH7BARERGVhGGHiIiItBrDDhEREWk1hh0iIiLSagw75WjFihVwdXWFkZER2rRpg0uXLqlU7syZM+jTpw8cHBwgkUiwb98+lev08/NDq1atYG5uDhsbG/Tv3x9hYWEql//999/RpEkTxeRPXl5e+Oeff1Qu/6KFCxdCIpFg0qRJKu0/b948SCQSpYeHh4fK9T18+BAffPABatSoAWNjYzRu3BhBQUEqlXV1dS1Ut0Qiga+vb6ll8/PzMXv2bLi5ucHY2Bi1a9fG/PnzS12r5UWpqamYNGkSXFxcYGxsDG9vbwQGBhbar7T3hhACc+bMgb29PYyNjdGtWzeEh4erXH7Pnj3w8fFRzCYeEhKicv25ubmYPn06GjduDFNTUzg4OGDEiBF49OiRSnXPmzcPHh4eMDU1RfXq1dGtWzdcvHhR5ba/6NNPP4VEIsGvv/6qUtlRo0YV+tv36NFDrbpDQ0PRt29fSKVSmJqaolWrVoiOji61bFHvO4lEgh9//FGlutPS0jBhwgQ4OjrC2NhYsRiyKmXj4+MxatQoODg4wMTEBD169FC8X1T5LMnKyoKvry9q1KgBMzMzDBo0SDHBqyrlV69ejbfeegsWFhaQSCRITk5WPFda+aSkJHz++eeoV68ejI2N4ezsjC+++AIymUyluj/55BPUrl0bxsbGsLa2Rr9+/RRLDKnzOSqEQM+ePRW/X1XKvvXWW4X+3p9++qladQcEBKBLly4wNTWFhYUFOnbsiO+++67Esvfu3Sv2/bZr1y6V6o6Li8OHH34IOzs7mJqaonnz5ti9e7dKZSMjIzFgwABYW1vDwsICQ4YMKTQhcHlh2CknO3bswJQpUzB37lxcuXIFnp6e6N69OxISEkotm56eDk9PT6xYsULtek+fPg1fX19cuHABx44dQ25uLnx8fJCenq5SeUdHRyxcuBCXL19GUFAQunTpgn79+uHmzZtqtSMwMBCrVq1CkyZN1CrXsGFDxMbGKh5nz55VqdzTp0/Rrl076Ovr459//sGtW7fw888/o3r16iq398V6jx07BgAYPHhwqWUXLVqE33//HcuXL0doaCgWLVqExYsXY9myZSrVDQAff/wxjh07hs2bN+P69evw8fFBt27d8PDhQ6X9SntvLF68GEuXLsXKlStx8eJFmJqaonv37sjKylKpfHp6Otq3b49FixYV+3xx5TMyMnDlyhXMnj0bV65cwZ49exAWFoa+ffuqVLe7uzuWL1+O69ev4+zZs3B1dYWPjw8eP36sUvnn9u7diwsXLihNH69K2R49eii9B7Zt26Zy+cjISLRv3x4eHh44deoUrl27htmzZ8PIyKjUsi/WGRsbi3Xr1kEikWDQoEEq1T1lyhQcPnwYW7ZsQWhoKCZNmoQJEyZg//79JZYVQqB///64e/cu/vrrLwQHB8PFxQXdunVDenq6Sp8lkydPxoEDB7Br1y6cPn0ajx49wsCBAwGo9lmUkZGBHj16YNasWYXaV1r5R48e4dGjR/jpp59w48YNbNiwAYcPH8aYMWNUqrtFixZYv349QkNDceTIEQgh4OPjg/z8fLU+R3/99VelZYZULTt27Filv/vixYtVLh8QEIAePXrAx8cHly5dQmBgICZMmICzZ8+WWNbJyanQ++3bb7+FmZkZevbsqVLdI0aMQFhYGPbv34/r169j4MCBGDJkCA4cOFBi2fT0dPj4+EAikeDEiRM4d+4ccnJy0KdPH8jl8kK/V40TVC5at24tfH19FT/n5+cLBwcH4efnp9ZxAIi9e/e+cjsSEhIEAHH69OlXPkb16tXF//3f/6m8f2pqqqhbt644duyY6NSpk5g4caJK5ebOnSs8PT1fqY3Tp08X7du3f6WyRZk4caKoXbu2kMvlpe7bu3dvMXr0aKVtAwcOFMOHD1eproyMDKGrqysOHjyotL158+bi66+/Lrbcy+8NuVwu7OzsxI8//qjYlpycLAwNDcW2bdtKLf+iqKgoAUAEBwerXH9RLl26JACI+/fvq11WJpMJAOL48eMq1/3gwQNRs2ZNcePGDeHi4iJ++eUXlcqOHDlS9OvXr8T2lFR+6NCh4oMPPnilsi/r16+f6NKli8rlGzZsKL777julbUW9d14uGxYWJgCIGzduKLbl5+cLa2trsWbNmkJ1v/xZkpycLPT19cWuXbsU+4SGhgoAIiAgoNTyLzp58qQAIJ4+fVrk6y6t/HM7d+4UBgYGIjc3V+2yV69eFQBERESEynUHBweLmjVritjY2GL/tkWVVedzsajybdq0Ed98880rlX1Z06ZNC31+lVTe1NRUbNq0SWk/S0vLQu+Zl8seOXJE6OjoCJlMptgnOTlZSCQScezYsVJfS1mxZ6cc5OTk4PLly+jWrZtim46ODrp164aAgIDX2haZTAYAsLS0VLtsfn4+tm/fjvT0dLWW3vD19UXv3r2VXr+qwsPD4eDggFq1amH48OGIjo5Wqdz+/fvRsmVLDB48GDY2NmjWrBnWrFmjdv1Awd9vy5YtGD16tEoLw3p7e8Pf3x937twBAFy9ehVnz55Fz549VaovLy8P+fn5MDIyUtpubGyscs8WAERFRSEuLk7p9y6VStGmTZvX/r57TiaTQSKRqL32XE5ODlavXg2pVApPT0+Vysjlcnz44YeYOnUqGjZsqHZbT506BRsbG9SrVw+fffYZnjx5onK9hw4dgru7O7p37w4bGxu0adNGrcvPz8XHx+PQoUMYM2aMymW8vb2xf/9+PHz4EEIInDx5Enfu3IGPj0+J5bKzswFA6X2no6MDQ0PDIt93L3+WXL58Gbm5uUrvNw8PDzg7Oxf5fivLZ5Gq5WUyGSwsLKCnp1doe0ll09PTsX79eri5ucHJyUmlujMyMvD+++9jxYoVxa7DWFLdW7duhZWVFRo1aoSZM2ciIyNDpfIJCQm4ePEibGxs4O3tDVtbW3Tq1Emlv9nLLl++jJCQkGLfb0WV9/b2xo4dO5CUlAS5XI7t27cjKysLb731Volls7OzIZFIlCYWNDIygo6Ojlqfc6+s3OPUG+jhw4cCgDh//rzS9qlTp4rWrVurdSyUoWcnPz9f9O7dW7Rr106tcteuXROmpqZCV1dXSKVScejQIZXLbtu2TTRq1EhkZmYKIdT7BvP333+LnTt3iqtXr4rDhw8LLy8v4ezsLFJSUkota2hoKAwNDcXMmTPFlStXxKpVq4SRkZHYsGGDym1/bseOHUJXV1c8fPhQpf3z8/PF9OnThUQiEXp6ekIikYgffvhBrTq9vLxEp06dxMOHD0VeXp7YvHmz0NHREe7u7sWWefm9ce7cOQFAPHr0SGm/wYMHiyFDhpRa/kWa6NnJzMwUzZs3F++//77KZQ8cOCBMTU2FRCIRDg4O4tKlSyrX/cMPP4i3335b0RunTs/Otm3bxF9//SWuXbsm9u7dK+rXry9atWol8vLySi3//Fu9iYmJWLJkiQgODhZ+fn5CIpGIU6dOqfS6n1u0aJGoXr264t+PKm3PysoSI0aMEACEnp6eMDAwEBs3biy1bE5OjnB2dhaDBw8WSUlJIjs7WyxcuFAAED4+Pkpli/os2bp1qzAwMChUT6tWrcS0adNKLf+i0np2VPkse/z4sXB2dhazZs1SueyKFSuEqampACDq1atXZK9OceXHjRsnxowZo/i5qL9NcWVXrVolDh8+LK5duya2bNkiatasKQYMGKBS3QEBAQKAsLS0FOvWrRNXrlwRkyZNEgYGBuLOnTsqve7nPvvsM1G/fv0inyuu/NOnT4WPj4/i/WZhYSGOHDlSatmEhARhYWEhJk6cKNLT00VaWpqYMGGCACDGjRtXbBs1hWGnHFSWsPPpp58KFxcXERMTo1a57OxsER4eLoKCgsSMGTOElZWVuHnzZqnloqOjhY2Njbh69apimzph52VPnz4VFhYWKl1C09fXF15eXkrbPv/8c9G2bVu16/Xx8RHvvPOOyvtv27ZNODo6im3btolr166JTZs2CUtLS7WCVkREhOjYsaMAIHR1dUWrVq3E8OHDhYeHR7FlKnPYycnJEX369BHNmjVT6rYurWxaWpoIDw8XAQEBYvTo0cLV1VXEx8eXWj4oKEjY2toqBVR1ws7LIiMjVb6E9vzf+3vvvae0X58+fcSwYcPUqrtevXpiwoQJxT5fVPkff/xRuLu7i/3794urV6+KZcuWCTMzs0KXBooqGxQUJDw9PRXvu+7du4uePXuKHj16KO1X1GeJOmGntM+i0sJOaeVlMplo3bq16NGjh8jJyVG5bHJysrhz5444ffq06NOnj2jevHmhoFlU+b/++kvUqVNHpKamKrYV9ftV9TPY39+/yEtoRZV//u985syZSvs2btxYzJgxQ+W6MzIyhFQqFT/99FORzxdXfsKECaJ169bi+PHjIiQkRMybN09IpVJx7dq1UsseOXJE1KpVS0gkEqGrqys++OAD0bx5c/Hpp5+W8NvRDIadcpCdnS10dXULvfFHjBgh+vbtq9axXjXs+Pr6CkdHR3H37l21y76sa9euKiXvvXv3Kj40nz8AKN7YRX1LLk3Lli2V/gEXx9nZWelblhBC/Pbbb8LBwUGt+u7duyd0dHTEvn37VC7j6Ogoli9frrRt/vz5ol69emrVLUTByf55WBkyZIjo1atXsfu+/N54foJ+OaB07NhRfPHFF6WWf1FZwk5OTo7o37+/aNKkiUhMTFSr7Mvq1KlTZC/Zy+V/+eUXxfvsxfeejo6OcHFxeaW6raysxMqVK0utOzs7W+jp6Yn58+cr7Tdt2jTh7e2tct1nzpwRAERISEixbXq5fEZGhtDX1y803mvMmDGie/fuKtednJwsEhIShBAF4w3Hjx+veK64z5LnJ+iXA4qzs7NYsmRJqeVfVFLYKa18SkqK8PLyEl27di0UVNT5HMzOzhYmJibijz/+KLX8xIkTi32/derUSe2609LSBABx+PDhUuu+e/euACA2b96stH3IkCGKXlRV6t60aZPQ19dX/N1fVFz5iIiIQuO8hCg4R3zyyScq1/348WPF39rW1lYsXry42H01hWN2yoGBgQFatGgBf39/xTa5XA5/f3+1xr68CiEEJkyYgL179+LEiRNwc3Mr8zHlcrni+n5JunbtiuvXryMkJETxaNmyJYYPH46QkBDo6uqqVW9aWhoiIyNhb29f6r7t2rUrdJvjnTt34OLiolad69evh42NDXr37q1ymYyMDOjoKP9T0tXVfaU7DExNTWFvb4+nT5/iyJEj6Nevn8pl3dzcYGdnp/S+S0lJwcWLF8v9ffdcbm4uhgwZgvDwcBw/fhw1atQo0/FUfe99+OGHuHbtmtJ7z8HBAVOnTsWRI0fUrvfBgwd48uSJSu89AwMDtGrVqszvv7Vr16JFixYqj1ECCn7fubm5ZX7/SaVSWFtbIzw8HEFBQejXr1+pnyUtWrSAvr6+0vstLCwM0dHR8PLyKvNnkSrlU1JS4OPjAwMDA+zfv18x/uhV6hYFX/6RnZ1davkZM2YUer8BwC+//IJ169apXffz8vb29qXW7erqCgcHhyLfb87OzirXvXbtWvTt2xfW1tZKv4OSyj8fV1TU+y0/P1/luq2srFCtWjWcOHECCQkJijs2y1W5x6k31Pbt24WhoaHYsGGDuHXrlhg3bpyoVq2aiIuLK7VsamqqCA4OFsHBwQKAYhzAy3e0FOWzzz4TUqlUnDp1SsTGxioeGRkZKrV7xowZ4vTp0yIqKkpcu3ZNzJgxQ0gkEnH06FGVyr9MnctYX375pTh16pSIiooS586dE926dRNWVlZFfvN42aVLl4Senp74/vvvRXh4uNi6daswMTERW7ZsUbmt+fn5wtnZWUyfPl3lMkIU3MlTs2ZNcfDgQREVFSX27NkjrKysCnXll+Tw4cPin3/+EXfv3hVHjx4Vnp6eok2bNoW65Et7byxcuFBUq1ZNMf6kX79+ws3NTfGNt7TyT548EcHBweLQoUMCgNi+fbsIDg4WsbGxpZbPyckRffv2FY6OjiIkJETp/ZednV1i2bS0NDFz5kwREBAg7t27J4KCgsRHH30kDA0NFd8i1f138eJlrJLKpqamiq+++koEBASIqKgocfz4cdG8eXNRt25dkZWVpVLde/bsEfr6+mL16tUiPDxcLFu2TOjq6op///1XpXbLZDJhYmIifv/990Kvo7TynTp1Eg0bNhQnT54Ud+/eFevXrxdGRkbit99+K7Xszp07xcmTJ0VkZKTYt2+fcHFxEQMHDhRCqPZZ8umnnwpnZ2dx4sQJERQUJLy8vBSXk1UpHxsbK4KDg8WaNWsEAHHmzBkRHBwsnjx5Ump5mUwm2rRpIxo3biwiIiKU9vn0009LLBsZGSl++OEHERQUJO7fvy/OnTsn+vTpIywtLUV8fPwrfY7iWc9ZaWUjIiLEd999J4KCgkRUVJT466+/RK1atUTHjh1V/r398ssvwsLCQuzatUuEh4eLb775RhgZGYn3339fpXaHh4cLiUQi/vnnH6XtpdWdk5Mj6tSpIzp06CAuXrwoIiIixE8//SQkEono1atXqXWvW7dOBAQEiIiICLF582ZhaWkppkyZUuzvVJMYdsrRsmXLhLOzszAwMBCtW7cWFy5cUKnc8y7dlx8jR44stWxR5QCI9evXq1T36NGjhYuLizAwMBDW1taia9eurxx0hFAv7AwdOlTY29sLAwMDUbNmTTF06NAiBwwW58CBA6JRo0bC0NBQeHh4iNWrV6vV1iNHjggAIiwsTK1yKSkpYuLEicLZ2VkYGRmJWrVqia+//lpkZ2erfIwdO3aIWrVqCQMDA2FnZyd8fX1FcnJyof1Ke2/I5XIxe/ZsYWtrKwwNDUXXrl2VXk9p5devX1/k83Pnzi21/PNLX0U9Tp48WWLZzMxMMWDAAOHg4CAMDAyEvb296Nu3r9IAZXX/XbwYdkoqm5GRIXx8fIS1tbXQ19cXLi4uYuzYsUpfTFSpe+3ataJOnTrCyMhIeHp6Ki6FqlJ21apVwtjY+JX+5rGxsWLUqFHCwcFBGBkZiXr16omff/5ZyOXyUsv+73//E46OjkJfX184OzuLb775RvG+VeWzJDMzU4wfP15Ur15dmJiYiAEDBiiCsSrl586dW+w+pZUv7rWV9Hhe9uHDh6Jnz57CxsZG6OvrC0dHR/H++++L27dvq9z2lz0PO6WVjY6OFh07dhSWlpbC0NBQ1KlTR0ydOlUxtk3Vuv38/ISjo6MwMTERXl5e4t9//1W57MyZM4WTk5PIz88v9BpKK3/nzh0xcOBAYWNjI0xMTESTJk3Epk2bVCo7ffp0YWtrK/T19UXdunUV79PXQfLsBRIRERFpJY7ZISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQEb3k1KlTkEgkSE5OruimEJEGMOwQERGRVmPYISIiIq3GsENElY5cLoefnx/c3NxgbGwMT09P/PnnnwD+u8R06NAhNGnSBEZGRmjbti1u3LihdIzdu3ejYcOGMDQ0hKurK37++Wel57OzszF9+nQ4OTnB0NAQderUwdq1a5X2uXz5Mlq2bAkTExN4e3sXWmmaiKoGhh0iqnT8/PywadMmrFy5Ejdv3sTkyZPxwQcf4PTp04p9pk6dip9//hmBgYGwtrZGnz59kJubC6AgpAwZMgTDhg3D9evXMW/ePMyePRsbNmxQlB8xYgS2bduGpUuXIjQ0FKtWrYKZmZlSO77++mv8/PPPCAoKgp6eHkaPHv1aXj8RaRYXAiWiSiU7OxuWlpY4fvw4vLy8FNs//vhjZGRkYNy4cejcuTO2b9+OoUOHAgCSkpLg6OiIDRs2YMiQIRg+fDgeP36Mo0ePKspPmzYNhw4dws2bN3Hnzh3Uq1cPx44dQ7du3Qq14dSpU+jcuTOOHz+Orl27AgD+/vtv9O7dG5mZmTAyMirn3wIRaRJ7doioUomIiEBGRgbefvttmJmZKR6bNm1CZGSkYr8Xg5ClpSXq1auH0NBQAEBoaCjatWundNx27dohPDwc+fn5CAkJga6uLjp16lRiW5o0aaL4f3t7ewBAQkJCmV8jEb1eehXdACKiF6WlpQEADh06hJo1ayo9Z2hoqBR4XpWxsbFK++nr6yv+XyKRACgYT0REVQt7doioUmnQoAEMDQ0RHR2NOnXqKD2cnJwU+124cEHx/0+fPsWdO3dQv359AED9+vVx7tw5peOeO3cO7u7u0NXVRePGjSGXy5XGABGR9mLPDhFVKubm5vjqq68wefJkyOVytG/fHjKZDOfOnYOFhQVcXFwAAN999x1q1KgBW1tbfP3117CyskL//v0BAF9++SVatWqF+fPnY+jQoQgICMDy5cvx22+/AQBcXV0xcuRIjB49GkuXLoWnpyfu37+PhIQEDBkypKJeOhGVE4YdIqp05s+fD2tra/j5+eHu3buoVq0amjdvjlmzZikuIy1cuBATJ05EeHg4mjZtigMHDsDAwAAA0Lx5c+zcuRNz5szB/PnzYW9vj++++w6jRo1S1PH7779j1qxZGD9+PJ48eQJnZ2fMmjWrIl4uEZUz3o1FRFXK8zulnj59imrVqlV0c4ioCuCYHSIiItJqDDtERESk1XgZi4iIiLQae3aIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIq/0/aNuF5S+4oysAAAAASUVORK5CYII=", "text/plain": [ "
" ] @@ -1127,10 +1477,10 @@ "plt.ylim(0, 100)\n", "plt.xticks(np.arange(len(epochs_x)))\n", "for i, txt in enumerate(epochs_acc):\n", - " if i%2 == 0:\n", - " pass\n", - " else:\n", + " if i%5 == 0:\n", " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", + " else:\n", + " pass\n", "plt.show()" ] } From 26c98c7d958969a3a495125266b2630b42cac0fe Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Mon, 29 Apr 2024 14:04:25 +0200 Subject: [PATCH 055/379] baseline - testing transfer learning --- .../transfer-learning/baseline-SCNN-3.ipynb | 546 ++++++++++++++++++ 1 file changed, 546 insertions(+) create mode 100644 tests/test_nonsequential/transfer-learning/baseline-SCNN-3.ipynb diff --git a/tests/test_nonsequential/transfer-learning/baseline-SCNN-3.ipynb b/tests/test_nonsequential/transfer-learning/baseline-SCNN-3.ipynb new file mode 100644 index 00000000..2e941d62 --- /dev/null +++ b/tests/test_nonsequential/transfer-learning/baseline-SCNN-3.ipynb @@ -0,0 +1,546 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "import torch, random\n", + "import torch.nn as nn\n", + "import sinabs.layers as sl\n", + "from tqdm.notebook import tqdm\n", + "\n", + "from tonic.transforms import ToFrame\n", + "from torch.utils.data import DataLoader\n", + "from torch.nn import CrossEntropyLoss\n", + "from torch.optim import Adam\n", + "\n", + "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "torch.backends.cudnn.deterministic = True\n", + "random.seed(1)\n", + "torch.manual_seed(1)\n", + "torch.cuda.manual_seed(1)\n", + "np.random.seed(1)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "batch_size = 3\n", + "num_workers = 1\n", + "epochs_pretrain = 5\n", + "epochs = 30\n", + "lr = 1e-3" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Dataset for transfer learning\n", + "\n", + "Dataset used to pre-train the network such that its parameters are set within a \"good\" region of the parameters space (i.e., hopefully training on a \"simpler\" dataset sets the wheights to values that improve the training on a harder dataset)." + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "from tonic.datasets.nmnist import NMNIST" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "root_dir = \"../NMNIST\"\n", + "_ = NMNIST(save_to=root_dir, train=True)\n", + "_ = NMNIST(save_to=root_dir, train=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "n_time_steps = 50\n", + "to_raster = ToFrame(sensor_size=(128,128, 2), n_time_bins=n_time_steps)\n", + "# to_raster = ToFrame(sensor_size=NMNIST.sensor_size, n_time_bins=n_time_steps)\n", + "\n", + "snn_train_dataset_pre = NMNIST(save_to=root_dir, train=True, transform=to_raster)\n", + "snn_test_dataset_pre = NMNIST(save_to=root_dir, train=False, transform=to_raster)" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The transformed array is in shape [Time-Step, Channel, Height, Width] --> (50, 2, 128, 128)\n" + ] + } + ], + "source": [ + "sample_data, label = snn_train_dataset_pre[0]\n", + "print(f\"The transformed array is in shape [Time-Step, Channel, Height, Width] --> {sample_data.shape}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [], + "source": [ + "snn_train_dataloader_pre = DataLoader(snn_train_dataset_pre, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True)\n", + "snn_test_dataloader_pre = DataLoader(snn_test_dataset_pre, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=False)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Training dataset\n", + "\n", + "This is the dataset to be used for training after the network has gone through a pre-training phase." + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [], + "source": [ + "from tonic.datasets.dvsgesture import DVSGesture" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [], + "source": [ + "root_dir = \"../DVSGESTURE\"\n", + "_ = DVSGesture(save_to=root_dir, train=True)\n", + "_ = DVSGesture(save_to=root_dir, train=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [], + "source": [ + "n_time_steps = 50\n", + "to_raster = ToFrame(sensor_size=DVSGesture.sensor_size, n_time_bins=n_time_steps)\n", + "\n", + "snn_train_dataset = DVSGesture(save_to=root_dir, train=True, transform=to_raster)\n", + "snn_test_dataset = DVSGesture(save_to=root_dir, train=False, transform=to_raster)" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The transformed array is in shape [Time-Step, Channel, Height, Width] --> (50, 2, 128, 128)\n" + ] + } + ], + "source": [ + "sample_data, label = snn_train_dataset[0]\n", + "print(f\"The transformed array is in shape [Time-Step, Channel, Height, Width] --> {sample_data.shape}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [], + "source": [ + "snn_train_dataloader = DataLoader(snn_train_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True)\n", + "snn_test_dataloader = DataLoader(snn_test_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=False)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Network Module\n", + "\n", + "We need to define a `nn.Module` implementing the network we want the chip to reproduce." + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "device: NVIDIA GeForce RTX 3070 Ti\n" + ] + } + ], + "source": [ + "if torch.cuda.is_available():\n", + " device = torch.device('cuda:0')\n", + " print('device: ', torch.cuda.get_device_name(0))\n", + "else:\n", + " device = torch.device('cpu')" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [], + "source": [ + "class SNN(nn.Module):\n", + " def __init__(self) -> None:\n", + " super().__init__()\n", + "\n", + " self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False)\n", + " self.iaf1 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + " self.pool1 = nn.AvgPool2d(2,2)\n", + "\n", + " self.conv2 = nn.Conv2d(10, 10, 2, 1, bias=False)\n", + " self.iaf2 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + " self.pool2 = nn.AvgPool2d(3,3)\n", + "\n", + " self.conv3 = nn.Conv2d(10, 10, 3, 1, bias=False)\n", + " self.iaf3 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + " self.pool3 = nn.AvgPool2d(2,2)\n", + "\n", + " self.flat = nn.Flatten()\n", + "\n", + " self.fc1 = nn.Linear(810, 100, bias=False)\n", + " self.iaf4 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + "\n", + " self.fc2 = nn.Linear(100, 100, bias=False)\n", + " self.iaf5 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + "\n", + " self.fc3 = nn.Linear(100, 100, bias=False)\n", + " self.iaf6 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + "\n", + " self.fc4 = nn.Linear(100, 11, bias=False)\n", + " self.iaf7 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + "\n", + " def detach_neuron_states(self):\n", + " for name, layer in self.named_modules():\n", + " if name != '':\n", + " if isinstance(layer, sl.StatefulLayer):\n", + " for name, buffer in layer.named_buffers():\n", + " buffer.detach_()\n", + "\n", + " def init_weights(self):\n", + " for name, layer in self.named_modules():\n", + " if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear):\n", + " nn.init.xavier_normal_(layer.weight.data)\n", + "\n", + " def forward(self, x):\n", + " \n", + " con1_out = self.conv1(x)\n", + " iaf1_out = self.iaf1(con1_out)\n", + " pool1_out = self.pool1(iaf1_out)\n", + "\n", + " conv2_out = self.conv2(pool1_out)\n", + " iaf2_out = self.iaf2(conv2_out)\n", + " pool2_out = self.pool2(iaf2_out)\n", + "\n", + " conv3_out = self.conv3(pool2_out)\n", + " iaf3_out = self.iaf3(conv3_out)\n", + " pool3_out = self.pool3(iaf3_out)\n", + "\n", + " flat_out = self.flat(pool3_out)\n", + " \n", + " fc1_out = self.fc1(flat_out)\n", + " iaf4_out = self.iaf4(fc1_out)\n", + "\n", + " fc2_out = self.fc2(iaf4_out)\n", + " iaf5_out = self.iaf5(fc2_out)\n", + "\n", + " fc3_out = self.fc3(iaf5_out)\n", + " iaf6_out = self.iaf6(fc3_out)\n", + "\n", + " fc4_out = self.fc4(iaf6_out)\n", + " iaf7_out = self.iaf7(fc4_out)\n", + "\n", + " return iaf7_out" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [], + "source": [ + "snn = SNN().to(device)" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [], + "source": [ + "snn.init_weights()" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": {}, + "outputs": [], + "source": [ + "optimizer = Adam(snn.parameters(), lr=lr, betas=(0.9, 0.999), eps=1e-8)\n", + "loss_fn = CrossEntropyLoss()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Define train and test functions" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": {}, + "outputs": [], + "source": [ + "def train(dataloader, model, loss_fn, optimizer, epochs, test_func, dataloader_test):\n", + " epochs_y = []\n", + " epochs_x = []\n", + " epochs_acc = []\n", + " model.train()\n", + "\n", + " for e in range(epochs):\n", + " losses = []\n", + " batches = []\n", + " batch_count = 0\n", + " train_p_bar = tqdm(snn_train_dataloader)\n", + "\n", + " for X, y in train_p_bar:\n", + " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", + " X = X.reshape(-1, 2, 128, 128).to(dtype=torch.float, device=device)\n", + " y = y.to(dtype=torch.long, device=device)\n", + "\n", + " # forward\n", + " pred = model(X)\n", + "\n", + " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", + " pred = pred.reshape(batch_size, n_time_steps, -1)\n", + "\n", + " # accumulate all time-steps output for final prediction\n", + " pred = pred.sum(dim = 1)\n", + " loss = loss_fn(pred, y)\n", + "\n", + " # gradient update\n", + " optimizer.zero_grad()\n", + " loss.backward()\n", + " optimizer.step()\n", + "\n", + " # detach the neuron states and activations from current computation graph(necessary)\n", + " model.detach_neuron_states()\n", + "\n", + " train_p_bar.set_description(f\"Epoch {e} - BPTT Training Loss: {round(loss.item(), 4)}\")\n", + "\n", + " batch_count += 1\n", + " losses.append(loss.item())\n", + " batches.append(batch_count)\n", + "\n", + " epochs_y.append(losses)\n", + " epochs_x.append(batches)\n", + "\n", + " acc = test_func(dataloader_test, model)\n", + " print(f'Epoch {e} accuracy: {acc}')\n", + " epochs_acc.append(acc)\n", + "\n", + " return epochs_x, epochs_y, epochs_acc\n" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": {}, + "outputs": [], + "source": [ + "def test(dataloader, model):\n", + " correct_predictions = []\n", + " with torch.no_grad():\n", + " test_p_bar = tqdm(dataloader)\n", + " for X, y in test_p_bar:\n", + " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", + " X = X.reshape(-1, 2, 128, 128).to(dtype=torch.float, device=device)\n", + " y = y.to(dtype=torch.long, device=device)\n", + "\n", + " # forward\n", + " output = model(X)\n", + "\n", + " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", + " output = output.reshape(batch_size, n_time_steps, -1)\n", + "\n", + " # accumulate all time-steps output for final prediction\n", + " output = output.sum(dim=1)\n", + "\n", + " # calculate accuracy\n", + " pred = output.argmax(dim=1, keepdim=True)\n", + "\n", + " # compute the total correct predictions\n", + " correct_predictions.append(pred.eq(y.view_as(pred)))\n", + "\n", + " test_p_bar.set_description(f\"Testing Model...\")\n", + " \n", + " correct_predictions = torch.cat(correct_predictions)\n", + " return correct_predictions.sum().item()/(len(correct_predictions))*100" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Pre-training loop (transfer learning)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "4303748d66154bd0aaba4c9798099e3d", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/359 [00:00" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "y_avg = []\n", + "for y in epochs_y:\n", + " y_avg.append(np.mean(y))\n", + "\n", + "plt.plot(np.arange(len(epochs_x)), y_avg, marker = '.')\n", + "plt.xlabel('epoch')\n", + "plt.ylabel('average loss')\n", + "plt.ylim(0,)\n", + "plt.xticks(np.arange(len(epochs_x)))\n", + "for i, txt in enumerate(y_avg):\n", + " if i%2 == 0:\n", + " pass\n", + " else:\n", + " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAG2CAYAAACZEEfAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAAB9HElEQVR4nO3dd1RUx98G8GfpRUB6kaqg2HvB3oIt9qiJJjHGWH5irLFgYkw0ETWJJsbE9ipqrDGxG3vBhgiKXSmKglQLLL3uvH8QNq60XYrg+nzO2XNk7p07c3evu987d4pECCFAREREpKY0qroCRERERJWJwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqbUqDXbOnTuH/v37w87ODhKJBPv27VPYLoTA119/DVtbW+jr66Nnz54ICwtT2OfFixcYNWoUjI2NUbNmTYwdOxapqamv8SyIiIioOqvSYCctLQ1NmzbFb7/9VuT2ZcuWYeXKlVizZg0CAgJgaGiIXr16ITMzU77PqFGjcOfOHZw4cQKHDh3CuXPnMH78+Nd1CkRERFTNSarLQqASiQR79+7FoEGDAOS36tjZ2WHmzJn44osvAABSqRTW1tbYtGkT3n//fdy7dw8NGjRAYGAgWrVqBQA4evQo+vbtiydPnsDOzq6qToeIiIiqCa2qrkBxIiIiEBcXh549e8rTTExM0LZtW/j7++P999+Hv78/atasKQ90AKBnz57Q0NBAQEAABg8eXOSxs7KykJWVJf9bJpPhxYsXMDc3h0QiqbyTIiIiogojhEBKSgrs7OygoVH8w6pqG+zExcUBAKytrRXSra2t5dvi4uJgZWWlsF1LSwtmZmbyfYri4+ODb7/9toJrTERERFUhKioK9vb2xW6vtsFOZfL29saMGTPkf0ulUjg6OiIqKgrGxsZVWDMiIiJSVnJyMhwcHGBkZFTiftU22LGxsQEAxMfHw9bWVp4eHx+PZs2ayfdJSEhQyJebm4sXL17I8xdFV1cXurq6hdKNjY0Z7BAREb1hSuuCUm3n2XFxcYGNjQ1OnTolT0tOTkZAQAA8PDwAAB4eHkhKSsLVq1fl+5w+fRoymQxt27Z97XUmIiKi6qdKW3ZSU1MRHh4u/zsiIgLXr1+HmZkZHB0dMW3aNHz33Xdwc3ODi4sL5s+fDzs7O/mIrfr166N3794YN24c1qxZg5ycHEyePBnvv/8+R2IRERERgCoOdoKCgtCtWzf53wX9aEaPHo1NmzZh9uzZSEtLw/jx45GUlISOHTvi6NGj0NPTk+fZtm0bJk+ejB49ekBDQwNDhw7FypUrX/u5EBERUfVUbebZqUrJyckwMTGBVCplnx0iIqI3hLK/39W2zw4RERFRRWCwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGptWod7OTl5WH+/PlwcXGBvr4+6tSpg0WLFkEIId9HCIGvv/4atra20NfXR8+ePREWFlaFtSYiIqLqpFoHO0uXLsXq1auxatUq3Lt3D0uXLsWyZcvw66+/yvdZtmwZVq5ciTVr1iAgIACGhobo1asXMjMzq7DmREREVF1IxMvNJNXMu+++C2tra2zYsEGeNnToUOjr62Pr1q0QQsDOzg4zZ87EF198AQCQSqWwtrbGpk2b8P777ytVTnJyMkxMTCCVSmFsbFwp50JEREQVS9nf72rdstO+fXucOnUKoaGhAIAbN27gwoUL6NOnDwAgIiICcXFx6NmzpzyPiYkJ2rZtC39//2KPm5WVheTkZIUXERERqSetqq5ASebOnYvk5GS4u7tDU1MTeXl5+P777zFq1CgAQFxcHADA2tpaIZ+1tbV8W1F8fHzw7bffVl7FiYiIqNqo1i07f/75J7Zt24bt27fj2rVr2Lx5M3788Uds3ry5XMf19vaGVCqVv6KioiqoxkRERFTdVOuWnVmzZmHu3LnyvjeNGzfG48eP4ePjg9GjR8PGxgYAEB8fD1tbW3m++Ph4NGvWrNjj6urqQldXt1LrTkRERNVDtW7ZSU9Ph4aGYhU1NTUhk8kAAC4uLrCxscGpU6fk25OTkxEQEAAPD4/XWlciqhqx0gxcevAMsdKMqq4KVSJnZ2dIJJJCLy8vLwBA165dC22bOHFiiccs6ngSiQQ//PCDwn6HDx9G27Ztoa+vD1NTUwwaNKiyTpMqSbVu2enfvz++//57ODo6omHDhggODsby5cvx6aefAsi/UKdNm4bvvvsObm5ucHFxwfz582FnZ8eLkegtsCswEt57bkEmAA0J4DOkMUa0dqzqalElCAwMRF5envzv27dv45133sGwYcPkaePGjcPChQvlfxsYGJR4zNjYWIW/jxw5grFjx2Lo0KHytL///hvjxo3D4sWL0b17d+Tm5uL27dvlPR16zap1sPPrr79i/vz5mDRpEhISEmBnZ4cJEybg66+/lu8ze/ZspKWlYfz48UhKSkLHjh1x9OhR6OnpVWHNiaiyxUozMHfPLRRMniETwLw9t9G5riVsTfSrtnJU4SwtLRX+XrJkCerUqYMuXbrI0wwMDOTdG5Tx6r779+9Ht27dULt2bQBAbm4upk6dih9++AFjx46V79egQYOynAJVoWr9GMvIyAg///wzHj9+jIyMDDx48ADfffcddHR05PtIJBIsXLgQcXFxyMzMxMmTJ1G3bt0qrDURvQ7XHifi1VnC8oTAo2fpVVMhem2ys7OxdetWfPrpp5BIJPL0bdu2wcLCAo0aNYK3tzfS05W/FuLj43H48GGFoObatWuIjo6GhoYGmjdvDltbW/Tp04ctO2+gat2yQ0RUlPTsXKw8XXhZGE0J4GxR8qMLevPt27cPSUlJ+OSTT+RpI0eOhJOTE+zs7HDz5k3MmTMHISEh2LNnj1LH3Lx5M4yMjDBkyBB52sOHDwEA33zzDZYvXw5nZ2f89NNP6Nq1K0JDQ2FmZlah50WVh8EOEb1R8mQCU3ZcR0hcKgx1NJGRkwfZvy08ozu48BHWW2DDhg3o06cP7Ozs5Gnjx4+X/7tx48awtbVFjx498ODBA9SpU6fUY27cuBGjRo1S6AJRMBjmyy+/lPfj8fX1hb29PXbv3o0JEyZU1ClRJavWj7GIiF7l8889nLwXDx0tDWwZ2xYX53ZH30b5fS9uRiWhGq+AQxXg8ePHOHnyJD777LMS92vbti0AIDw8vNRjnj9/HiEhIYWOWTClyct9dHR1dVG7dm1ERkaqWnWqQgx2iOiNsfXyY/zfhQgAwE/DmqKlkylsTfSxYEBD6GhqIOhxIgIiXlRxLaky+fr6wsrKCv369Stxv+vXrwOAwhxsxdmwYQNatmyJpk2bKqS3bNkSurq6CAkJkafl5OTg0aNHcHJyUr3yVGUY7BBRmZU298mECRNQp04d6Ovrw9LSEgMHDsT9+/eVPv7EiRMhkUjw888/wy/0KRYcuAMA+Li+Nv7v6//BwsICxsbGGNq3J1rrxgAAfjtT+p08vZlkMhl8fX0xevRoaGn91wvjwYMHWLRoEa5evYpHjx7hwIED+Pjjj9G5c2c0adJEvp+7uzv27t2rcMzk5GTs3r27yJYiY2NjTJw4ERMmTJBf2zo6OkhISMCECRMq5Tp/WWhoKAYOHCi/zjt27IgzZ84ofVz6D4MdIiqzwMBAxMbGyl8nTpwAAPncJy1btoSvry/u3buHY8eOQQgBT09PhflSirN3715cvnwZdnZ2SEjOhNe2a8iTCQxtYY+d309Gbm4uTp8+jatXr6Jp06bYu3QKRHoizoc9Q3BkYqWeN1WNkydPIjIyUj7XWgEdHR2cPHkSnp6ecHd3x8yZMzF06FAcPHhQYb+QkBBIpVKFtJ07d0IIgQ8++KDIMn/44QeMHz8e5ubmMDQ0RKdOnbB+/XoAFX+dv+rdd98tdJ2/++67Ja79SMUQJKRSqQAgpFJpVVeF6I02depUUadOHSGTyYrcfuPGDQFAhIeHl3icJ0+eiFq1aonbt28LB0dH4dxvknCac0gMX3NJRMfGCwDi3Llz8v2Tk5MFADH0yzXCac4hMXbTlQo9L8rn5OQkABR6TZo0SQghxPjx40Xt2rWFnp6esLCwEAMGDBD37t0r8Zh///23eOedd4SZmZkAIIKDgwvts3btWtGlSxdhZGQkAIjExMQyn0NMUrq4GP5UxCSll/kYlXGdOzk5iRUrVsi3PX36tNjr/MSJE2WuuyrehM9b2d9vtuwQUYUobu6TAmlpafD19YWLiwscHByKPY5MJsNHH32EWbNmobabO56lZkOakQMXC0Os/aglbK0tUa9ePWzZsgVpaWnIzc3F2rVrYWVlhXmj+0IiAU7eS8DdmOTKPN23UmW05KWlpaFjx45YunRpsfukp6ejd+/emDdvXrnqvyswEh2WnMbI9QHosOQ0dgWq3sm4Mq7zhg0bFtpubm6OevXqoW/fvvJHaMbGxgCAd955B15eXnjx4gU+//xz1KtXD/r6+nB0dMSUKVMKtV69SpllMgYMGICcnBzo6OjA0tISQ4cOxY4dOwC8OZ+3glICu7cCW3aIym/Xrl1CU1NTREdHK6T/9ttvwtDQUAAQ9erVK/Vud/HixeKdd94Rubl5YuIfQULT2ErY9Z4gHj5Nle8TFRUlWrZsKSQSidDU1BS2trbi2rVrQgghvLZdFU5zDolJ265W/EmSgopq4RBCiIiIiGLv9AucOXOmzC07MUnpwnnuIeE0579X7bmHVW7hqejrvOC9e7VlR4j867xJkyYCgNDQ0BBWVlZi9erVAoA4c+aMuHXrlhgyZIg4cOCACA8PF6dOnRJubm5i6NChJZYdGxur8Nq4caOQSCTiwYMH8n2WL18u/P39xaNHj8TFixeFh4eHsLGxqXafN1t2iN4SJXUSLuud3zfffAN3d3cYGhrC1NQUPXv2REBAQInljhgxAm5uboX6HowaNQrBwcHw8/ND3bp1MXz4cGRmZhZZ7tWrV/HLL79g06ZN+OF4KI7cjoNEAgxv5QgXC0MAgBACXl5esLKywvnz53HlyhUMGjQI/fv3R2xsLLy6uQIA/rkViwdPU8v6tlIpKqqF43WJeJZWITNuFzXHD1D267yo9w747zqvVasWLly4gMDAQAwdOhRffPEFnJ2d0aVLFzRq1Ah///03+vfvjzp16qB79+74/vvvcfDgQeTm5hZ7DjY2NgqvV5fJAIDp06ejXbt2cHJyQvv27TFz5kzExcVh9OjRb8TnXUip4ddbgC079CZLSEhQuEs7ceJEue/8tm3bJk6cOCEePHggbt++LcaOHSuMjY1FQkKCfB8nJyexcOFCERsbK65cuSI0NDTEzp07SzxuVlaWMDAwENu3by9y+4oVK4REIhEampoCEo381793tU5OTkIIIU6ePCk0NDQK/X91dXUVPj4+Qgghxm4KFE5zDokZu66X9vZRGVVUC0eBym7ZiU5MU2jVKUvLzqNHj4SGhobYt29fifspe51ramrKX8pc51lZWUJDQ0N4enoWW/b69euFhYWF0ucUFxcntLS0xLZt24rd5/nz58LDw0MAqHafN1t2iN4SlpaWCndphw4dki+QWNY7v5EjR6Jnz56oXbs2GjZsiOXLlyM5ORk3b95U2M/IyAg2NjY4fPgwrKysFFaLLooQAkIIZGVlFbn9o48+wuZDfrAbsxK2Y1biizX7YWdnh1mzZuHYsWMAIF/vSEND8etLQ0NDPuPt5O75rTv7rkcj6gXXyqoMFdHC8TrFJRe+5ub2cVdpxm1l5/hR5jq/efMmrl+/Ln8pc53v27cPMpkMzZs3L/K4z549w6JFixRmky5NUctkFJgzZw4MDQ1hbm6Ou3fvwtPT8435vF/FYIdIjZT2aAEApFIpjI2NFeYpKe2Y69atg4mJSaFJ15YsWQIzMzMsXrwY7u7uCtsePnwIHx8fXL16FZGRkbh06RKGDRsGfX199O3bV77fy3OfJObpYNmVdGiaO2HYO+2xbFw/aGtrw8bGBvXq1QMAeHh4wNTUFKNHj8aNGzcQGhqKWbNmISIiQv4j1MyhJjq5WSBPJrDG74Fybx4praRZjE1MTODm5obOnTvjr7/+wv379wvNbVMVDt2IBQB4NrBGPesaAIC07OID/lcVN8dPWa5zc3NzNGrUSOGlzHU+Z84cSCSSIofJJycno1+/fmjQoAG++eYbpc+rqGUyCsyaNQvBwcH4448/IJVK8fz580IzlFfXz/tVDHaI1EhRCyS+TJU7v0OHDqFGjRrQ09PDihUrcOLECVhYWMi3T5kyBTt37sTixYuRk5ODq1evYvbs2fLtenp6OH/+PPr27QtXV1eMGDECRkZGuHTpEqysrOT7Fcx98iw1C2M2BSIlMxetnEyxdGiTIgM2CwsLHD16FKmpqejevTtatWqFCxcuYP/+/QrB2OR/++7sDnqCOGn1u9N8k1VUC8frIpMJ/HMrP9gZ1soBk/69NnZeiUJunkypYxQ3x4+q17myXr3OW7RogUePHuGrr74qdNORkpKC3r17w8jICHv37oW2trZSZRS3TMbLdahbty7Cw8NhZWWFq1ev4vLly8Uer7p83kVS6uGammOfHVIXnp6e4t133y1ym1QqFW3atBG9e/cW2dnZpR4rNTVVhIWFCX9/f/Hpp58KZ2dnER8fX+z+GzZsEFpaWiIzM1OlOsckpYsz9+NFv1/OCac5h0SnpafFsxTVjlGc91ZfFE5zDomFB+9UyPFIiLy8POHo6CjmzJmjkP7gwQOxePFiERQUJB4/fiwuXrwo+vfvL8zMzBSum3r16ok9e/bI/37+/LkIDg4Whw8fFgDEzp07RXBwsIiNjZXvExsbK4KDg8X69evlc88EBweL58+fK1XnwIjnwmnOIdHo66MiIztXZObkihYLjwunOYfEkVuxpR+ggpRnjp8FCxYIGxsbkZOTo5AulUpFu3btRJcuXURaWppKxxw9erRo2bJlifsUfN7/+9//5H0Bhag+n7eyv98MdgSDHVIPJXWeTE5OFh4eHqJHjx4iIyOjTMd3dXUVixcvLnb77du3BQBx//59pY+588pj4fLScOB6X/4jwuJTylS/opwNScg/7lf/VFgA9bY7duyYACBCQkIU0qOjo0WfPn2ElZWV0NbWFvb29mLkyJGFrgcAwtfXV/63r69vkRPXLViwQL7PggULitzn5eOUZMH+28JpziExfWewPG3JkXvCac4h8eH/XVb1LSiTl691l7mHxM4rj5XOW1yAKZVKRdu2bUXjxo1FeHi4wkCF3Nxc+X6vBhwFeQ0MDMTq1asLlXf58mXx66+/iuDgYLFlyxYBQDRv3lzUqVNHfjNTXT5vBjsqYLBD6qAy7vxeVrt2bYUvpFdt3bpVaGhoiBcvXih1vIinqcL5ldExLnMPlWtm21fJZDLR/9fzwmnOIbH0SMkzu74pSpvVNiMjQ0yaNEmYmZkJQ0NDMWTIEBEXF6f08SdMmCAAFJrzpahyC0a/qaoiZjFWVl6eTLT+7oRwmnNInLz73/sQ+TxNPu/Oy3M4VYaYpHSFoF7VkWDFBZgFo5WKekVERMj3KypQWLt2rdDX1xdJSUmFyrt586bo1q2bMDMzE7q6usLZ2Vl8/OlnYt+Fm2X6zCrz81b291u5HopEVK3JZDL834aN6DlgGJ6m5cDWJP+/dnJyMjw9PZGeno6tW7ciOTkZycn5MwtbWlpCU1MTQH7nSR8fHwwePBhpaWn4/vvvMWDAANja2uLZs2f47bffEB0dLZ851d/fHwEBAejWrRuMjIzg7++P6dOn48MPP4SpqWkJ9RQIiHiBfcHROHAjGuLV7QJ49CxdpREyJZFIJPDq5ooJf1zFFv/HmNC5DkwMlOvPUF0FBgYqzFB7+/ZtvPPOO/LPZvr06Th8+DB2794NExMTTJ48GUOGDMHFixdLPXZJ6zQBwMKFCzFu3Dj530ZGRirXf3vAY3y59zYEAA0J4DOkMUa0dlT5OMoKepyIhJQsGOlpoaPbf33OHMwM0K2eFU7fT8C2y4/x1bsNKq0OEc/SICtmjh9lrnVPT89CHYMBoGvXrkWmv0oIgVhpBi49eAYXC0PYmuhj/PjxRfbdE0LAvnY9rN2xH09Ts/AsNRsn7sTh0M1Y+B2MhORgJLrWs0QDO+NSywWAuzHJOBvy9LV93sVhsEOkBr76bTuin0ThtGiEDktOy79Qrl27Jp8M0NXVVSFPREQEnJ2dAeR3ngwKe4J20gyY6mri/v372Lx5M549ewZzc3O0bt0a58+fl09rr6uri507d+Kbb75BVlYWXFxcMH36dMyYMaPI+oXGp2BvcDT2B0cjpoTOwpoSCZwtDJQ+7+joaMyZMwdHjhxBeno6XF1d4evri1atWgEA4uPjsX3ZXMTtO4Ts9FS0+6cNDmzfADc3t2KP2bVrV/j5+RVK79u3Lw4fPgwg/wdh+vTpWLN2LbIyM6GhoQFXV1ds27ZNoew5c+bg+PHjSEpKQufOnfHrr7+Wu2wLCwssWLAA69evR1JSEiwtLeHo6IguXbpAKpViw4YN2L59O7p37w4gvzNx/fr1cfnyZbRr167E9/Lzzz/HsWPHiu14XDDVQFnFSjPkgQ6QH9zO23MbnetaVliA+6pDN2MAAJ4NbKCrpamw7cN2jjh9PwG7rz7BF73qQU9bs6hDlJuFoW6hNA0JVLrWy2NXYCS899yCTAASCTCilQNcrWrkBzMp2XiWmiV/PU/NRu6rkdlLBIAzIU9xJuSpyvV4HZ93cRjsEL3hYpLSsSPaFE5zDgHI/0KZ8/ctbLr0GLVqGmLOXzdgUUMXFjV0YGmkB4saOrAw0oVpDV0IIfBnUBRc5h7CthfAjn8DpT179pRYZosWLUoclQEA8cmZOHA9BnuDo3E39r91qoz0tNCvsS0GNa+FiGdp+GrvbeQJAU2JBIuHNFL6SzAxMREdOnRAt27dcOTIEVhaWiIsLEzesiSEwKBBg6CtrY3vf9+C5WejkBB8AD169MS9e3dhaGhY5HH37NmD7Oxs+d/Pnz9H06ZN5S0nQP4M0ytXroSuQ2OYN3kHabdOIv5ZDAwMDAqVvX//fhgbG2P58uXo2bMn7t4tX9nLli3DypUrsXnzZtjb26N9+/YwNDREVlYWrl69ipycHPTs2VO+v7u7OxwdHeHv719ssFPaOk0FlixZgkWLFsHR0REjR47E9OnTlZ7CAADO3E8o1JqnSguHqvJkAv/cyl8h/N2mtoW2d6lrBXtTfTxJzMDBGzEY1qpyZv7dez26UJqbVQ3YGBce7l3RYqUZ8kAHAIQAdgZGlZrPWE8LFka60NHUwP24lELbezeyLvUzi03KwNE78Qpplfl5l4TBDlEFUKaFQdW7/D179mDx4sUIDw9HTk4O3NzcMHPmTHz00UcAgJycHMz1nocNO/Yg+Wk0NHQNoefUFDW7fAItI3Pci03GvdiSF8PU1pAg56W7OJkA5v59C7FJmahtVSM/QKqhC4sauqhpoF3s3D2x0gxEPEuDlZEebkQlYd/1aFwMfyb/gtXWlKBrPSsMbl4L3d2t5HfQ7Wqbo2s9Szx6lg5nCwOVvgCXLl0KBwcH+Pr6ytNcXFzk/w4LC8Ply5dx+/ZtuNdvgEOxfnhoNhGJ6z7Bjh07ih1ua2ZmpvD3zp07YWBgIA84hBD48ccfoWlsBesPFgMADFzb4MmvH8LvchAaNGigUHZB8LB69WrY2NiUu+yff/4ZX331FQYOHIg///wTeXl5SEtLk084p6Ojg5o1ayocx9raGnFxcSW+l1paWpgyZUqx+0yZMgUtWrSAmZkZLl26BG9vb8TGxmL58uXF5nlZTp4Mmy49KpSuamueKq5EvMCz1CyY6GujQx2LQts1NSQY2dYRy46GYGtAZKUEOwnJmfC9GAEAWDq0CTQ1AO+/byEkPhV7g6MxpIV9hZf5socJhR+hAYBHHXO42xjBooZu/v9xI51/b4p0YV5DR94KFivNQIclpxWOoSmRYEH/hqUHO9IMHL8bXyjv62rRehmDHaJyUqWFQZW7fDMzM3z55Zdwd3eHjo4ODh06hDFjxsDKygq9evVCeMxzbD54FtqthsHWygWyzFS8OLUOT/csQq1PfobP4MbIFaJQM/Wz1Gw8S8lCSlauQqBTQAD4+VRYoXQtDQnMa+R/IVoa6cq/GKNepOOfW7GF7tgBoKWTKQY3r4V+jW1haqhT5HnamuiX6S7vwIED6NWrF4YNGwY/Pz/UqlULkyZNkvcpKZjrQ09PD5oaEvyvax3M/usmMmWa8Dt3vtiA41UbNmzA+++/L/+cHj58iPT0dBg2aIOn+3yQGXUbmjXMoWlsiTPnLuB/n36sUHYBDQ0N6Orq4sKFC2UuOyIiAnFxcfKWm4JZjJOTk+Hv74+2bdsqddyXFazTdO3atWKDWQAKjyibNGkCHR0dTJgwAT4+PtDVLfyY5lXrzj1EaHwq9LU1kZWbJ/8BnOlZt9Lu8g/fyn+E1auhNXS0ip5WbngrB6w4EYobUUm49USKxvYmFVqHX0+HIzNHhhaONTG8lT0kEgnik7Pww7EQfHvwLjq6WcDKqPJaeC4+eFYoTVMiwfLhTZV6321N9OEzpDHm7VG9BbY8eSsagx2iclKlhUGVu/yuXbsq/D116lRs3rwZFy5cQE23Vpi07RZqDP4G5oY6eK+lPf7vfATwzkTEbZmBme3NMKJNyZ0AM3PycDdGiqFr/BUWSJQA6OZuhbSsXHlwJM3IQa5MID45C/FFTLv/qnGdXPBRO2c4mlfeHdzDhw+xevVqzJgxA/PmzUNgYCCmTJkCHR0djB49Wv74xtvbG2vXrkW/hpaYNX8fcpKf4npIhFJlXLlyBbdv38aGDRsAAJHP0+G14QwAIC3kIoxbD4K1x3BkxYbhxfHfccrvImKSMgqVbWhoiBUrVuDJkyeIjY0tU9kA5K0z1tbW8lmM9+zZg23btiEuLg42NjbIzs5GUlKSQutOfHx8sX1tzp8/j4SEBDg6/ne95OXlYebMmfj555/x6NGjIvO1bdsWubm5ePTokXzW3+KEJ6Tgl38D6O8GNUJ7V3NM+OMqbj6R4kVadol5yyo3T4ajt/Pfr35Niu5wDQAWNXTRt7Et9l+PwdbLj7H0vSYVVofHz9Ow40okAGB2b3d5MDm+c238cysWd2KSsWD/Haz+sGWFlfmyq48Tse7cQwD5fXWEQJkCjhGtHdG5btlaYMuTtyJxBmWicjpw4ABatWqFYcOGwcrKCs2bN8f69evl20u7y1eGEAKnTp1CSEgIsi3rYdT/BeB5WjYa2hnjwOcd4d23Pi7M7QbvHk6QSCT4qEvx/S4K6GlrooWTGZYMaQzNf7+ENSUSLBnaGBs/aY1dEzxwamZX3FjgidDv+sDfuzsOTO6AjZ+0wrKhTTCrVz30bmhd5LG7u1tXaqAD5PczadGiBRYvXozmzZtj/PjxGDduHNasWQMA0NbWxp49exAaGgozMzOYGNWASWIo9Gq3xJOkTGTnlj5z7oYNG9C4cWM0b9kKa/wewPNnP1yPTAIAWDi4wrzraOhY14Fxs97QqmmNpLgovLPcD1uvPMHuv/6Wl21gYIAzZ86gT58+hdb0Kq3sNm3aFLm9qFmMW7ZsCW1tbZw6dUqeFhISgsjISHh4eBR5HGXWaSrK9evXoaGhoTBLcFHyZAKz/7qJ7FwZutazxJAWtWBroo/pPesCAP4MikK6Css2KCv/EVY2ahpoo30d8xL3/bCdEwBg/41oSDNyKqwOK06EIlcm0LmuJdrV/q8O2poaWPZeE2hpSHDkdhyO3FIuAFaFND0HU3YEI1cm8G4TW1yc0w07xrXDhbndyjQaytZEHx51zMsUrJQnb0Vhyw5ROanawqDKXb5UKkWtWrWQlZUFTU1NdB/rjV0xpgAE+je1w7KhTaCvk/9s3VRXgs2/fI8PPvgAxsbKDQsFlLvz0tHSKPJxU1U+k7e1tUWDBorDhevXr4+///5b/nfLli1x/fp1SKVSZGdnw6imGSxqN0KuSR3sDX5S4pd+Wloadu7ciQnTvdH/1wvyTpqt67vgEICOzerh97nd5e+bxy59PElLRFp2Hr49eBdN7U2w7R8/2Bvmry9maWmJtm3byvtxlaSg7IULFyqkF7TOxMbGKkw1EB8fj2bNmsHExARjx47FjBkzYGZmBmNjY3z++efw8PBQ6Jz88lQD5ubmMDdXDAZeXaeprFMNAMDmS49wLTIJNXS1sHhwY3nrRpe6lnA2N8Cj5+nYGxyNUW2dSn1fVHHo3wCid0MbaGuWHGC2cjJFPWsjhMSnYM+1JxjTwaXE/ZVxLzYZ+2/kP0ab3atwy1dDOxNM7FIHq86EY/7+O2hX27zYR72qEkJg9t83EJ2UAUczA/gMaQwjPW3Y1Xz9fWWqC7bsEJWTqi0MqtzlGxkZ4fr16zh29gLq9huHoxt+QGbkTczt446V7zeTBzo5OTkYPnw4hBBYvXq1yudQ1juvgmfyL7cMva5n8h06dEBISIhCWmhoKJycCv9ompiYwNLSElGPHiI9JhQGbm2x+uyDEtdF+mP7TqRlZGJnkhPux6WgpoE2fnivCfZ7D4aenh5u3Lghf98MJTl4EvUYtZ0c8N2gRjDS1cKNJ1IMWHURv1+MgaGJKcLCwhAUFISBAweWem67d+9GVlYWPvzwQ4V0FxcX2NjYYNaiH+VTDXh8ewj+lwPkLTcrVqzAu+++i6FDh6Jz586wsbEpNLouJCQEj2Of4tKDZ4iVZpRan4KpBrp06YKGDRvi+++/x/Tp07Fu3boS80U+T8cPx/I/o7l93GFX87/rQkNDgo88nAHkB0TKzBejrJcfYb1bwiOsAhKJBB965F83Wy8/rpC6/HgsBEIA/ZrYolGtovsBfd7DFa5WNfAsNQuLDt0td5kFtvg/xrE78dDWlGDVyOYw0nuz55aqEBU+neEbiDMoq4cnT56IUaNGCTMzM6GnpycaNWokAgMD5dtRzEyjy5YtK/aYubm54quvvhLOzs5CT09P1K5dWyxcuFDIZDL5Po6OjqJZs2bCxsZG6OnpiR49eohvvvlG2NnZFTpeUlKSSEhIEEII0aZNG/mstyW5EZUo2n5/UjjNOSRMm/cSrTp0VdienZ0tBg0aJJo0aSKePXtW6vEqQ0xSurgU/uy1zIhb4MqVK0JLS0t8//33IiwsTGzbtk0YGBiIrVu3yvf5888/xZkzZ8SDBw/Evn37hJOTkxgwaLBo9u0x4TTnkNgX/ER89NFHYu7cuQrHPnY7Vhg5NRIG7p3kywy8vNyEl5eXACA+/PBDcejQIdGyZUshkUjExo0bhRBCxEkzhOfnS4T1B4uF3YT/E+4fLRTWdg5iyJAhCuUUVbYQQnTs2FGMGDFCIS0vTyZuPUkSvcfMEBq6hsJyyHxh++kqoe/WTmiZWIsuPsfEqPWXxdQd18Sig3fE6rPh4q+gKHE2JEHcjk4S8dIMkZObJ4Qo39IFypLJZOKDdf7Cac4hMWLtJZGXJyu0jzQjW9Sff0Q4zTkkLoY9rbCyz4XmLxPSfOFx+TmXJiUzRzQoqEt4+epSsBZXbe/D4kFCycufXH38Qj6T8+l7xa89p6xbT5KE27x/hNOcQ2LD+YflPl51xxmU6a1S2ogoAIUeGR05cgRjx47F0KFDiz3u0qVLsXr1amzevBkNGzZEUFAQxowZAxMTE/kwXVNTU9y6dQt///03XFxcMH/+fCxfvhz169cvdDwTk/w7vIK7/EWLFpV4XnuuPcHcPbeQnStDHUtDmLlb4mnMf3NkFLTohIWF4cyZM4UeR7wuZR1RVR6tW7fG3r174e3tjYULF8LFxQU///wzRo0aJd8nNjYWM2bMQHx8PGxtbfHxxx9j/vz5WHfhMX48HorfzoQjOzJS3sIWJ83EggO3cejcVaQ8vo1GY5fh97Ft0MnNUqHsX3/9Fc+ePcPOnTuxdetWGBgYYOHChRgzZgwAwNpYD/1c9RG8eykSniYg3tAUNRp2h82gGXiakgVLo/zRS5EvlV0gJCQEFy5cwPHjxxGdlIELYU9xPuwZLj14jhdp2RCW3WDUMgbPj/0KWWYa9OwbwGr4QjxKysGjpMIjb14mkQAmetpIeqlfSmVN9LYzMAqXHjyHnrYGlg5tAg2NwiO9jPW0MbSFPf64/BibLj1Ce9fCw8PL4vDNfx9hNbKBVimPsArU0NXCoOa1sC0gEtsuR6J9EUPVlSGEwLKj+a1Zw1vZo7ZljRL3b+Foik87uGDDhQjM23sLx6d3LnNLTGpWLiZvv4bsPBl61rfGmA7OZTqOOpIIUYFth2+o5ORkmJiYQCqVqtTXgaqPuXPn4uLFizh//rzSeQYNGoSUlBSFzpyvevfdd2Ftba0wImbo0KHQ19fH1q1bIYSApaUlEhMTsWjRIgwfPhxnz57FuHHj4OXlhVWrVgHIfyxRMNPtrVu3MHXqVLRs2VKhf8nHH3+MWrVqwcfHB7l5MvT5dCZuZppDy9QW7RyN0FLjERbM/xKrV6/GZ599hpycHLz33nu4du0aDh06BGvr/zoLm5mZQUenYp7/qyNpRg46LjmNlKxcTO9ZF0Nb1sLp+wlYdjQEqVm50NKQYFzn2pjS3U3+qLCs0rJy8dPxUGy6FAGZyJ+sbV7f+hjeygHxKZmIeJYmn8I/JTMHlx++yA9wwp/h4dM0hWMZ6miiuWNNXAx/rjDUX0MC/DS8GWQyoTjFQGoWnqbk//tFWlaR860U2DGuHTxK6cirrFhpBjyXn0NKVi6+6lcfn3WqXey+4Qkp6Ln8HDQkgN+sbnAwK1+/kpw8GVp/fxJJ6TnY/llblQKoe7HJ6PPLeWhpSHBpbndYlWHSvzMhCRjjGwgdLQ34zeqqVACZkZ2HXj+fQ+SLdIxq64jvBzdWuVwhBKbtuo7912NgZ6KHf6Z2Qk0D9f8OUPb3my07pBZKm3PlVfHx8Th8+DA2b95c4nHbt2+PdevWITQ0FHXr1sWNGzdw4cIF+URqEREReP78OX755ResX79e3sLg5uYmX3cKKL6F4WUFd/lJ6dn4fEcwroTFIv3+LiDtBc4YGiDW3R1bt27FiBEjAORPZHjgwAEAQLNmzRSOdebMmUJD1+k/JvraaO1shtMhCVhxMhQrTobKtzV3rAmfIY3hblMxNz6Gulr4un8DDGpuB+89t3AnJhlz99zCWr+HePwif8I3CQBHMwM8ScpA3ksRiYYEaOpQE51cLdCpriWaOdSEtqYGdgVGFpq7ZHDzWiXWI08mkJiejXuxyfh445VC0w04mVdMq44QAl/tvY2UrFw0c6hZamdfVysjdHKzwPmwZ9h6+TG8+xZuEVXFpQfPkZSeA4saOmjjYlZ6hpfUtzVGKydTBD1OxM7AKEzpUfykn0WRyf5r1fmkvbPSLWX6OppYMrQxRq4PwLaASLzbxE7lwHN30BPsvx4DTQ0JVn7Q/K0IdFTBlh2wZUcdFAzrnjFjBoYNG4bAwEBMnToVa9aswejRowvtv2zZMixZsgQxMTEKQ8JfJZPJMG/ePCxbtgyamprIy8vD999/D29vbwDApUuX0KFDB8TExMDW9r/p6IcPHw6JRIJdu3YpfQ6x0gycD3uGn0+GIiYpE/ramvhxWFP0a1J4mnsqn6JmhQWALzzr4n9dXaFZxCOXipCbJ4PvxUf46XgIMosZ+u5sboCObhbo6GoJjzrmMNEv+pFGrDSjzHOX5AdLt5D30vmPauuIhQMblfvc91+PxtSd16GjqYFDUzqirnXpi4WevBuPz7YEwURfG5e9e5SrNW32XzfwZ9ATfNjOEd8NUr2FZF9wNKbtug5bEz2cn91N6cdgAHDgRgym7AiGka4Wzs3upvLoqnl7b2F7QCSczA1wdGpnpd+HsPgU9F91AZk5MszqVQ9e3VxLz6Qm2LJDbxWZTIZWrVph8eL86fubN2+O27dvFxvsbNy4EaNGjSox0AGAP//8E9u2bcP27dvRsGFDXL9+HdOmTYOdnV2Rxy2rLZceYcGBO/JHE6YG2tj2WTulVxYm1RS1CjUAtHQyq7RABwC0NDUwrnNtWBnpYuqu64W2r3y/GQY0K7mFpkB5+km9PN1A0OMXWH4iFNsCIvEsNQu/vN+8zAtiPkvNwjcH7gAAPu/uqlSgA+RPYuloZoDIF+nYdz0aH5QyIWZxsnNlOPbvWkzKjMIqSp/GNlh4SAex0kycvp8Az4bKLXyakyfDT8fzW3XGda5dpmHk3n3cceZ+Ah4/T8dPx0OUWok9IzsPXtuvITNHhk5uFvhflzoql/s24NBzUgvFzbkSGRlZaN/z588jJCREqSn7Z82ahblz5+L9999H48aN8dFHH2H69Onw8fEB8N+8J/HxiovdlTRjbYHcPBn8Qp9iwpYgfP1SoAPk9ykxNeRw0criYmGIV2Oa17lmT5vaZkWW31rFxy7lUTBs/vPubvhtZAvoaGrg2J14fLzhCqTpZZtYb8GBO0hMz0F9W2NM7Kr8j66mhgQf/zv0uzzD0C+GP4M0IweWRrpo7Vy291JXSxPD/10j64/Lj5XO92dQFB4/T4e5oQ7GdizbPD1GetpY/G9/nY0XI3AtMrHUPAsP3UFofCosauhi+fBmRXYEJwY7pCZUmXNlw4YNaNmyJZo2bVrqcdPT0wuNltHU1IRMlv8IomDek5c7OScnJyMgIKDIGWuFELj1RIqFB++inc9pjN54BcfuxhfaTyaAR8/SS60flU1Vzg9UHcp/Vd/Gttj8aRsY6WrhyqMXGL7WX6n5d1527E4cDt+MhaaGBD+816TUifxeNayVA/S1NXE/LgWXH75QKW+BQ/+OwurbyKZcLXSj2jpCIgHOhz3Do2dppe6fkZ2Hlf8uhzG5uysMdcv+0KSbe/6CuTIBzP7rJrJy84rd98CNGOy4EgWJBPjl/WbyUX5UGB9jkVqYPn062rdvj8WLF2P48OG4cuUK1q1bV2jSs+TkZOzevRs//fRTkcfp0aMHBg8ejMmTJwMA+vfvj++//x6Ojo5o2LAhgoODsXz5cnz66acA8icjmzZtGr777ju4ubnJh57b2dlh0KBB8uNGvUjHgRsx2BscjfCEVHm6qYE2urtbYU9wtEKH0apaGfhtUtVr9lR1+a/yqGOOPyd6YPTGKwiJT8HQ3y9hy9g2cLUq/VGUND0HX+27DQCY0Ll2sZPolcREXxtDWuQP/d586ZHKHXSzcvNw/G7pa2Epw8HMAF3qWuJsyFNsvxKJeaV0mt7s/wjxyVmoVVMfI9uW7RHcy75+twHOhz1FeEIqVp0Ox0zPwjMwP3qWhnl7bgEAJndzRYcKGravrhjskFpQZs4VANi5cyeEEPjggw+KPE5oWDiCQyMRK82ArYk+fv31V8yfPx+TJk1CQkIC7OzsMGHCBHz99dfyPLNnz0ZaWhrGjx+PpKQkNG7ZFlt270OWTBN7r0Ri77VoXHn0352qrpYG3mlgjcHNa6FzXUtoa2qgjYtZtVgZ+G1TFfMDVafyX1Xf1hh7JrXHxxuv4OHTNAxd7Y+Nn7RCS6eSHwl9d/gunqZkoY6locojmF42ur0ztgVE4vjdOEQnZaBWTeXfmwthz5CSmQsrI120cip5CQtlfNTOCWdDnuLPoCjMeKdusf2YpBk5WH32AQBg+jt1oatVvqkKAMDUUAcLBzbCpG3XsPrsA/RpZKvQfy8rNw+Td1xDalYu2jibYWo53vO3BUdjgaOxKN+uwEh477kFmcgf8uszpLFKC+a9nB/Ib53J+/e/l0QCeNQ2x6DmtdC7kQ2Mi5g0rDyja4gq0ou0bHy6KRDXo5Kgq6WBVSNb4J0GRS/66hf6FKM3XoFEAvw10aPUwKg0I9dfxqUHzzGxSx3M7eOudL4Zu65jT3A0PmnvjG8GlL4QbmnyZAKdl51BdFIGfhrWFENb2he53w/H7uO3Mw/gZlUDR6d1rtAO7hP/uIqjd+LQ0M4Y+7w6yB8NfnvwDnwvPoKpgTb+mdrprf6+4GgsIiVl5uTh6O04zP37lryTsEwAc/6+BZ9/7ivV4U8mEwqz0gJAnhCoY2mI4a0cMKCZXalfSNXtLp/eXmaGOtg+ri28tl3DmZCnmPBHEBYPboz3XxkllZqVK3+UMtrDudyBDpA/P82lB8+xMzAS03q6KTUyLDMnDyf+7fvWv2nFTNWgqSHByLaO+OFYCLYGPC4y2ElIycTGC48AAF/0qlfhI/kWDmoI/4fPcScmGevOPYRXN1ccvxMH34v5Zf40vCm/M5TEYIfeOjKZwN3YZJwPe4YL4U8R+CgR2cXMefJqAKOq7wY1rrBZaYleJwMdLaz7uBW899zCX1fzly15mpKFyd1d5SuXLzt6H9FJGXAw08fs3oX7lZRFj/rWsDfVx5PEDBy4HoPhrR1KzXMu9ClSsnJha6KH5g7lf4RVYHgrB/x8MhTBkUm4HS0t1Bfpt9PhyMjJQzOHmvAspuWrPKyM9PD1uw0wc/cN/HwyFJoaEvx2JhwA8FlHF3R3r/gy1RWDHVIrsdIMhen3C8QkZeBC2DOcC3sqX2PoZVZGukhIyVJI05AAW8e2VWqEw9OULHy4IUBh7hZ2MqY3nbamBn54rwmsjXXx25kH+OlEKOJTMjGxSx0cvxOPLf75Q7OXDGkCA52K+TnR1JDgo3ZO8DlyH76XHmFYK3t5cFWcw7f+HYXV2LZCh15bGumidyNbHLwRg20Bj+EzpIl8W9SLdGy/kj+1xeze9UqtY1kNaVELa889QGh8KpYcuQ8AsDfVx+zeyj/iIwY7pEZe7XPzYTsnSIBi1xjyqGOOjq4W6OhmiTqWhvgzKKpQJ2Fl19VxszaCz5DG7GRMakcikWBWL3dY1tDFt4fuYuvlSGy9/N/8Va2dTSt8JNCI1g5YcTIU92KTEfgoscRlHzJz8nDy30dYlTHb+IdtHXHwRgz2BcfAu299eX+7FSdCkZMn0MnNosyLhiojLjlTYQQnkH/z9jwti98vKmCwQ2ohVpqh0DlYJiC/6wSKX2PoZeUdClzdhhITVaRPOrhAU1OC+fvuKKRffZwoH71YUWoa6GBw81rYcSUKmy89KjHYORvyFGnZeahVUx/NHWpWWB0KtHExQ13rGgiNT8Xea9EY3d4Z9+OSsfd6NABgVq+KeXxXnKJm+y6Yh4vfMcpjsENqobjp/3vWt8J7LR1KXGPoZeXtJMxOxqTO6ljWKJRWWT+8o9s7Y8eVKBy9E4eYpAzYFTMM/b9HWDaV8ihJIpHgw3ZO+Hr/HWy9/Bgfezjhx2OhECK/zCb2NSu8zJcVzPbNR+TlwxmUSS24WBji1a85TYkEiwY1Qu9GNkoFOkRUste5zIa7jTHa1TZDnkxgW0DRyzZkZOfh1L3yrYWljMHNa8FARxNhCalY7fcAJ+/FQ0MCzHinclt1gOo32/abisEOqYVnKdl4OdrhFwJRxXvdP7yftHcGAOy4EoXMnMLLJpwNSUB6dh7sTfXRxF71WZuVZaSnjUHN8xdoXXY0f1maYS0d4GpVuKWrMoxo7YgLc7thx7h2uDC3m0rzf1E+PsaiClfciKjKkp0rw6y/bkAIoIe7FT7rVJt9Zogqyevsm9azvjXsTPQQI83EwRsxGNZKcRh6wVpY/ZrYVtpoqAKWNRRHZda2NKzU8l7FR+Tlw5YdNRQdHY0PP/wQ5ubm0NfXR+PGjREUFAQAyMnJwZw5c9C4cWMYGhrCzs4OH3/8MWJiYko85rlz59C/f3/Y2dlBIpFg3759CtsLjuvo6o5alqbo3Kwe6nboh9X/BFbWacqt8XuA+3EpMDXQxrL3msCjjjm/FIgqUcGK6ZX9/0xLUwMfeTgDADa9shp6enYuTt3/9xFW48p7hAXk38D9ejpMIW3Z0RCVF0ulqsNgR80kJiaiQ4cO0NbWxpEjR3D37l389NNPMDXNn2grPT0d165dw/z583Ht2jXs2bMHISEhGDBgQInHTUtLQ9OmTfHbb78VuT09PR3+V4KQ2WgQbEf/AstB85D9IhrTxo6s1C+E0PgU+ZfQNwMawrwGV/0lUifvt3aArpYG7sQk4+rjRHn66fsJyMyRwdHMAI1qVe4yP0UNgMgTAo+epVdquVRx+BhLzSxduhQODg7w9fWVp7m4uMj/bWJighMnTijkWbVqFdq0aYPIyEg4Ohb9LLhPnz7o06dPseWamJjggwVrsfTf59kAYPbORMRtmYHLN0MxuFPTsp5SsfJkArP+uomcPIGe9a0woGnl3t0R0etnaqiDgc3s8GfQE2y69AitnPOHoR9+jY+wOCLqzceWHTVz4MABtGrVCsOGDYOVlRWaN2+O9evXl5hHKpVCIpGgZs2aZSozKzcPCw/eVQh0AECWlQ5AAoMalXPXtfFCBG5EJcFIVwvfDWpc6V94RFQ1Rv/bUfno7TjESTORlpWL0/cTAADvVsJEgq/iiKg3H1t21MzDhw+xevVqzJgxA/PmzUNgYCCmTJkCHR0djB49utD+mZmZmDNnDj744IMyrfgenpCCz3dcx73YZABA+zrmuPzwOfJyspF01hcGDTpj4bEItHC1U2rZBWU9epaGH4/nB1df9qsPGxO9Cjs2EVUvDe1M0MbZDFcevcC2gMdwszZCVq4MLhaGaGBbuY+wCnDS0Dcbgx01I5PJ0KpVKyxevBgA0Lx5c9y+fRtr1qwpFOzk5ORg+PDhEEJg9erVKpUjhMD2gEgsPHQHmTkymBvq4IdhTdDd3RqRz5IxasRwGFoawnzELEQlZeCzLUHYNb6dUisYl36OAnP+vomsXBk6uJpjhBILBRLRm210e2dcefQC2wMi5cPM+zWu/EdYL+OIqDcXH2OpGVtbWzRo0EAhrX79+oiMjFRIKwh0Hj9+jBMnTqjcqrPG7yHm7b2FzBwZOrlZ4MjUTujubo2cnBxMHTcayc9icf7saWyZ2BU1DbRxIyoJM/68DllR0xyraPuVSAREvIC+tiaWDGnCx1dEbwHPhtawNdHD87RsnAl5CqBy1sIi9cRgR8106NABISGKfWdCQ0Ph5OQk/7sg0AkLC8PJkydhbm6u9PH9HzwHAARHJkJbU4Iv+9bH5jFtYGWsh5ycHPTv3x9nzpxBVFQU7O3tMaBbO0xrogFtTQn+uRWHZcfu4+uvv4atrS309fXRs2dPhIWFlVimj48PWrduDSMjI1hYWmHS6A+Q8/wJZveuBwez/A6CEyZMQJ06daCvrw9LS0sMHDgQ9+/fV/q8iKh609bUKDRx4I2opKqpDL1xGOyomenTp+Py5ctYvHgxwsPDsX37dqxbtw5eXl4A8gOd9957D0FBQdi2bRvy8vIQFxeHuLg4ZGdny4/To0cPrFq1Sv53ojQZ03/fh6HfbwcAmOQmwqeLEXo5a0FDQ4KcnBwMGDAAJ0+eRJcuXbBt2zb4+fnhyy+/RNt6tbBkSBMAwNKly/DTil+wZs0aBAQEwNDQEL169UJmZmax5+Tn5wcvLy/4+/vDY/Jy5ObmIHHPArzXxFK+T8uWLeHr64t79+7h2LFjEELA09MTeXmFZ10lojdPrDQDJ/5d3bzAl3tvc64bUo4gIZVKBQAhlUqruioV4uDBg6JRo0ZCV1dXuLu7i3Xr1sm3RURECABFvs6cOSPfz8nJSSxYsEAIIcTjZ2miw5SVReYZPXq00sf98eg9oWloKsy7fyouhj0VQgiRlJQkdHV1xY4dO0o9r7+CooTTnEPCZdoOAUD4+fkVu++NGzcEABEeHq7am0dE1dLF8KfCac6hQq9L4c+qumpUhZT9/WYHZTX07rvv4t133y1ym7Ozs8IspMXxv3EPEc/SsOliBH48HopU/dpovOAofIY0KfI5ubOzM+rXr49evXrhyZMn8PPzQ61atTBp0iR07doVADDIVRtfpCVCx7EZJm69ij2TOsDVygRt27aFv78/3n///WLrk5CSiYWH7gIAPm5liW8BmJmZFblvWloafH194eLiAgcHdl4mUgec64bKg4+xqJBdgZHosOQ0Rq4PwDcH7yI1KxetnU1xZFrnEjsEFgx7d3Nzw7Fjx/C///0PU6ZMwebNmwEA8fH5TdDN6jkhOTMXYzZdwfPULFhbWyMuLq7EOn297w6kGTloaFsDgduXo0OHDmjUqJHCPr///jtq1KiBGjVq4MiRIzhx4gR0dHTK+W4QUXXAuW6oPCRCmdv8l5w5cwbdunWrrPpUieTkZJiYmEAqlZZprhl1IU3PweFbMZi397ZCugTAudld4WBW8sJ3Ojo6aNWqFS5duiRPmzJlCgIDA+Hv749Lly6hQ4cOuB0Wgf/tiUDki3S0dDIFTq2AlqYGdu3aVeRx/7kVi0nbrkFLQ4JWT/7G5XOncOHCBdjb2yvWXypFQkICYmNj8eOPPyI6OhoXL16Enh7n4CFSF7HSDM51Q3LK/n6r/Bird+/esLe3x5gxYzB69Gg+JniDZefKcC0yERfCnuF8+DPcepJUaP0XIL/jzZPEzFKDneKGvf/9998AABsbGwBATmoSNn7SGoN/v4irjxORc+chhr7TochjJqZl4+v9+cGX9e1tuBh8DufOnSsU6AD5S1aYmJjAzc0N7dq1g6mpKfbu3YsPPvigtLeCiN4QnOuGykLlx1jR0dGYPHky/vrrL9SuXRu9evXCn3/+qTCSh6perDQDlx48UxipIIRAaHwKNlyIwBjfK2i28DjeX3cZq86E40ZUfqDjbG6AV2etUfa5eGnD3l1cXGBjY4NTp07B1aoG1n7YEho5GYgJu4XEGs5FHnPhobt4mpKFvAv/h0dXz+L06dMKa30VRwgBIQSysrJK3ZeIiNSbyi07FhYWmD59OqZPn45r167B19cXkyZNwqRJkzBy5EiMHTsWTZtW/KKPpLxdgZHw3nMLMgFoSIDhrRyQkydwIfwp4pMVf/wtauigg6sFOrpaoKObBWxN9LErMBLz9txGnhAqPRefPn062rdvj8WLF2P48OG4cuUK1q1bh3Xr1gEAJBIJpk2bhu+++w5ubm5wcXGBedB6RNUww7lsF/x99QmGtrRHjx49MHjwYDTsORx7g6OReGI1xIMLOHTgAIyMjOT9e0xMTKCvr4+HDx9i165d8PT0hKWlJZ48eYIlS5ZAX18fffv2rfg3mIiI3igq99l5VUxMDNatW4clS5ZAS0sLmZmZ8PDwwJo1a9CwYcOKqmelUqc+O7HSDHRYcrrIx1EAoKulgTYuZujoaoFObpZwtzGChkbhGYjL+lz80KFD8Pb2RlhYGFxcXDBjxgyMGzdOvl0IgQULFmDdunVISkpCx44d0ez9mfgrXAZtTQm2jm2LEd1b4oMPP8JZw66IlWbi8dKiR5b5+vrik08+QUxMDD777DNcvXoViYmJsLa2RufOnfH111+jXr16StediIjeLMr+fpcp2MnJycH+/fuxceNGnDhxAq1atcLYsWPxwQcf4OnTp/jqq69w7do13L17t1wnAeQ/NpszZw6OHDmC9PR0uLq6wtfXF61atQLw34/n+vXrkZSUhA4dOshHBClLnYKdSw+eYeT6gELp/ZvaYkQrR7RyNq2Q9akqkkwmMHnHNfxzKw41DbSx7qOWWOP3EKfvJ8DJ3ABHp3aGvk71qjMREVW9Suug/Pnnn2PHjh0QQuCjjz7CsmXLFIYAGxoa4scff4SdnV3Zav6SxMREdOjQAd26dcORI0dgaWmJsLAwmJqayvdZtmwZVq5cic2bN8PFxQXz589Hr169cPfu3bdyFE5xc1HM61u/2nbq09CQYPnwZohOuowbUUkYvvayfFuvBjYMdIiIqFxUbtnp0aMHPvvsMwwZMgS6urpF7pObm4uLFy+iS5cu5arc3LlzcfHiRZw/f77I7UII2NnZYebMmfjiiy8A5A8/tra2xqZNm0qcpO5l6tSyAwDTdgZj3/UYAP/NRTGitWMV16p0t6OlePfXCwppmhIJLsztVm0DNSIiqjrK/n6rPBrr1KlT+OCDD4oNdABAS0ur3IEOABw4cACtWrXCsGHDYGVlhebNm2P9+vXy7REREYiLi0PPnj3laSYm/83IW5ysrCwkJycrvNRJSmYuAGBEawdcmNvtjQh0ACA5M6dQWp4QePQsvQpqQ0RE6kLlYMfHxwcbN24slL5x40YsXbq0QipVoLQZeQtG5VhbWyvkK21GXh8fH/mcLCYmJmo1V1BWbh4u/bsy+UftnN6oFpGCR3Av43TwRERUXioHO2vXroW7u3uh9IYNG2LNmjUVUqkCMpkMLVq0wOLFi9G8eXOMHz8e48aNK3c53t7ekEql8ldUVFQF1bjqBUYkIiMnD5ZGumho92Y9kuN08EREVBlU7qAcFxcHW9vC6yNZWloiNja2QipVQNkZeePj4xXqFB8fj2bNmhV7XF1d3RIfw73JzoYkAAC61LWERFJ4SHl1N6K1IzrXteR08EREVGFUbtlxcHDAxYsXC6VfvHixQkZgvUyVGXkLJCcnIyAgAB4eHhValzfF2dCnAICu9SyruCZlZ2uiD4865gx0iIioQqjcsjNu3DhMmzYNOTk56N69O4D8TsuzZ8/GzJkzK7RyZZmRd/78+bCzs8OgQYMqtC5vgieJ6QhPSIWGBOjk+uYGO0RERBVJ5WBn1qxZeP78OSZNmiRfD0tPTw9z5syBt7d3hVaudevW2Lt3L7y9vbFw4UK4uLjg559/xqhRo+T7zJ49G2lpaRg/frx8Rt6jR4++lXPsnA3Jb9Vp4WgKEwPtKq4NERFR9VDm5SJSU1Nx79496Ovrw83N7Y3uA6Mu8+x8tjkIJ+/F4wvPupjcXfkZpImIiN5ElTaDcoEaNWqgdevWZc1OFSx/yPkzAEDXelZVXBsiIqLqo0zBTlBQEP78809ERkbKH2UV2LNnT4VUjFQT9CgR6dl5sKihiwa2b27rFBERUUVTeTTWzp070b59e9y7dw979+5FTk4O7ty5g9OnT8PExKQy6khKeHnIeVGrmBMREb2tVA52Fi9ejBUrVuDgwYPQ0dHBL7/8gvv372P48OFwdHwzliVQRwWdk7u8wUPOiYiIKoPKwc6DBw/Qr18/AICOjg7S0tIgkUgwffp0+ZBwer2ikzIQ9u+Q885uFlVdHSIiompF5WDH1NQUKSkpAIBatWrh9u3bAICkpCSkp3PBxqrg92+rTjOHmqhpoFPFtSEiIqpeVO6g3LlzZ5w4cQKNGzfGsGHDMHXqVJw+fRonTpxAjx49KqOOVIqC/jochUVERFSYysHOqlWrkJmZCQD48ssvoa2tjUuXLmHo0KH46quvKryCVLLsXBkuhhcMOWd/HSIiolepFOzk5ubi0KFD6NWrFwBAQ0MDc+fOrZSKkXKCHr9AWnYeLGrooJEdR8MRERG9SqU+O1paWpg4caK8ZYeqXkF/nc5uHHJORERUFJU7KLdp0wbXr1+vhKpQWXDIORERUclU7rMzadIkzJgxA1FRUWjZsiUMDQ0Vtjdp0qTCKkcli0nKQEh8yr9DzhnsEBERFUXlYOf9998HAEyZMkWeJpFIIISARCJBXl5exdWOSuQXmt+q09ShJkwNOeSciIioKCoHOxEREZVRDyqDl5eIICIioqKpHOw4OTlVRj1IRTl5MlwMfw6A8+sQERGVROVgZ8uWLSVu//jjj8tcGVLe1ceJSM3KhZmhDprU4pBzIiKi4qgc7EydOlXh75ycHKSnp0NHRwcGBgYMdl6Ts/Ih5xYcck5ERFQClYeeJyYmKrxSU1MREhKCjh07YseOHZVRRyoCl4ggIiJSjsrBTlHc3NywZMmSQq0+VDnipJm4H5cCiQTozM7JREREJaqQYAfIn105Jiamog5HJfALzW/VaWJfE2Ycck5ERFQilfvsHDhwQOFvIQRiY2OxatUqdOjQocIqRsUr6K/Tla06REREpVI52Bk0aJDC3xKJBJaWlujevTt++umniqoXFSMnT4YLYVzlnIiISFkqBzsymawy6kFKuvY4ESlZuTA10EYT+5pVXR0iIqJqr8L67NDrcfbfJSI6uVlCk0POiYiISqVysDN06FAsXbq0UPqyZcswbNiwCqkUFc+voL8OH2EREREpReVg59y5c+jbt2+h9D59+uDcuXMVUikqWkJyJu7GJgPgkHMiIiJlqRzspKamQken8HBnbW1tJCcnV0ilqGgFj7Ca2JvAooZuFdeGiIjozaBysNO4cWPs2rWrUPrOnTvRoEGDCqkUFc2PQ86JiIhUpvJorPnz52PIkCF48OABunfvDgA4deoUduzYgd27d1d4BSlfbp4M58Pyg50uXCKCiIhIaSoHO/3798e+ffuwePFi/PXXX9DX10eTJk1w8uRJdOnSpTLqSACCo5KQnJmLmgbaaOZQs6qrQ0RE9MZQOdgBgH79+qFfv34VXRcqQcHCnxxyTkREpBqV++wEBgYiICCgUHpAQACCgoIqpFLqZsmSJZBIJJg2bZpCur+/P7p37w5DQ0MYGxujc+fOyMjIKPIYZ0OeQur/J4589wmMjIxgZWWFQYMGISQkRGG/devWoWvXrjA2NoZEIkFSUlIlnRUREdGbQeVgx8vLC1FRUYXSo6Oj4eXlVSGVUieBgYFYu3YtmjRpopDu7++P3r17w9PTE1euXEFgYCAmT54MDY3CH0lCSibuxCQjM+o2pn4+GZcvX8aJEyeQk5MDT09PpKWlyfdNT09H7969MW/evEo/NyIiojeByo+x7t69ixYtWhRKb968Oe7evVshlVIXqampGDVqFNavX4/vvvtOYdv06dMxZcoUzJ07V55Wr169Io9TMAqrx/Sf8fnETvL0TZs2wcrKClevXkXnzp0BQN56dPbs2Qo8EyIiojeXyi07urq6iI+PL5QeGxsLLa0ydQFSW15eXujXrx969uypkJ6QkICAgABYWVmhffv2sLa2RpcuXXDhwoUij1Mwv07XuoqjsKRSKQDAzMysEmpPRESkHlQOdjw9PeHt7S3/oQWApKQkzJs3D++8806FVu5NtnPnTly7dg0+Pj6Ftj18+BAA8M0332DcuHE4evQoWrRogR49eiAsLExh39xiVjmXyWSYNm0aOnTogEaNGlXimRAREb3ZVG6K+fHHH9G5c2c4OTmhefPmAIDr16/D2toaf/zxR4VX8E0UFRWFqVOn4sSJE9DT0yu0vWDl+AkTJmDMmDEA8h8Dnjp1Chs3blQIkG48SYI0IwfGeloKQ869vLxw+/btYluDiIiIKJ/KwU6tWrVw8+ZNbNu2DTdu3IC+vj7GjBmDDz74ANra2pVRxzfO1atXkZCQoNC3KS8vD+fOncOqVavkI6henXG6fv36iIyMVEg7+29/nU51LaGlmd8QN3nyZBw6dAjnzp2Dvb19ZZ4KERHRG69MnWwMDQ0xfvz4iq6L2ujRowdu3bqlkDZmzBi4u7tjzpw5qF27Nuzs7AoNGw8NDUWfPn0U0s6+tESEEAKff/459u7di7Nnz8LFxaVyT4SIiEgNlLlH8d27dxEZGYns7GyF9AEDBpS7Um86IyOjQv1oDA0NYW5uLk+fNWsWFixYgKZNm6JZs2bYvHkz7t+/j7/++kuep3PXbritXQ/GLfujSz1LeHl5Yfv27di/fz+MjIwQFxcHADAxMYG+vj4AIC4uDnFxcQgPDwcA3Lp1C0ZGRnB0dGRHZiIieiupHOw8fPgQgwcPxq1btyCRSCCEAABIJPmz+ubl5VVsDdXUtGnTkJmZienTp+PFixdo2rQpTpw4gTp16sj3uR8aDlltWzS0M4aVkR5Wr14NAOjatavCsXx9ffHJJ58AANasWYNvv/1Wvq1gSPrL+xAREb1NJKIgWlFS//79oampif/7v/+Di4sLrly5gufPn2PmzJn48ccf0alTp9IPUs0kJyfDxMQEUqkUxsbGVV0duc82B+HkvXiM9nDCtwM54oqIiOhlyv5+qzz03N/fHwsXLoSFhQU0NDSgoaGBjh07wsfHB1OmTClXpek/O65E4uS9/PmMtlx+jF2BkaXkICIioqKoHOzk5eXByMgIAGBhYYGYmBgAgJOTU6EOt1Q2sdIMzNv7XwdnIYB5e24jVlr0ullERERUPJX77DRq1Ag3btyAi4sL2rZti2XLlkFHRwfr1q1D7dq1K6OOb52IZ2l49eFinhB49Cwdtib6VVMpIiKiN5TKwc5XX30lX3hy4cKFePfdd9GpUyeYm5tj165dFV7Bt5Hmv529X01ztjCogtoQERG92VQOdnr16iX/t6urK+7fv48XL17A1NRUPiKLymff9RiFvzUlEiwe0oitOkRERGVQISt3cv6WipOQkom/rz0BAKwe1QI1DXTgbGHAQIeIiKiMuEx5NbPp4iNk58rQwrEmejeyYWsZERFROak8GosqT2pWLv64/BgAMKFLHQY6REREFYDBTjWyIyASKZm5qG1piHfqW1d1dYiIiNSCysHOuXPnkJubWyg9NzcX586dq5BKvY2yc2XYcCECADChc21oaLBVh4iIqCKoHOx069YNL168KJQulUrRrVu3CqnU22j/9WjEJWfCykgXg5rXqurqEBERqQ2Vgx0hRJF9SZ4/fw5DQ8MKqdTbRiYTWHfuIQDg044u0NXSrOIaERERqQ+lR2MNGTIEQP7q5p988gl0dXXl2/Ly8nDz5k20b9++4mv4Fjh9PwFhCakw0tXCyLaOVV0dIiIitaJ0sGNiYgIgv2XHyMgI+vr/zfuio6ODdu3aYdy4cRVfw7fA2nMPAAAj2znCWE+7imtDRESkXpQOdnx9fQEAzs7O+OKLL/jIqoJcffwCgY8SoaOpgU87uFR1dYiIiNSOyn12Zs+erdBn5/Hjx/j5559x/PjxCq3Y22KNX35fncHNa8HaWK+Ka0NERKR+VA52Bg4ciC1btgAAkpKS0KZNG/z0008YOHAgVq9eXeEVVGfhCak4cTceEgkwrjNXjCciIqoMKgc7165dQ6dOnQAAf/31F2xsbPD48WNs2bIFK1eurPAKqrN1//bVeae+NVytalRxbYiIiNSTysFOeno6jIyMAADHjx/HkCFDoKGhgXbt2uHx48cVXkF1FZ+cib3B0QDyl4YgIiKiyqFysOPq6op9+/YhKioKx44dg6enJwAgISEBxsbGFV5BdbXxQgRy8gTaOJuhpZNpVVeHiIhIbakc7Hz99df44osv4OzsjDZt2sDDwwNAfitP8+bNK7yC6ig5MwfbAiIBABO6sK8OERFRZVI52HnvvfcQGRmJoKAgHDt2TJ7eo0cPrFixokIr96olS5ZAIpFg2rRp8rTMzEx4eXnB3NwcNWrUwNChQxEfH1+p9SivbZcjkZqVCzerGuhWz6qqq0NERKTWyrTquY2NDYyMjHDixAlkZGQAAFq3bg13d/cKrdzLAgMDsXbtWjRp0kQhffr06Th48CB2794NPz8/xMTEyGd7ro6ycvOw8eK/C352qcMFP4mIiCqZysHO8+fP0aNHD9StWxd9+/ZFbGwsAGDs2LGYOXNmhVcQAFJTUzFq1CisX78epqb/9W+RSqXYsGEDli9fju7du6Nly5bw9fXFpUuXcPny5UqpS3ntC47G05Qs2JroYUBTu6quDhERkdpTOdiZPn06tLW1ERkZCQMDA3n6iBEjcPTo0QqtXAEvLy/069cPPXv2VEi/evUqcnJyFNLd3d3h6OgIf3//Yo+XlZWF5ORkhdfrIJMJrP13wc+xHV2go1WmhjUiIiJSgdLLRRQ4fvw4jh07Bnt7e4V0Nze3Shl6vnPnTly7dg2BgYGFtsXFxUFHRwc1a9ZUSLe2tkZcXFyxx/Tx8cG3335b0VUt1Yl78Xj4NA3Gelp4vw0X/CQiInodVG5aSEtLU2jRKfDixQuFldArQlRUFKZOnYpt27ZBT6/illLw9vaGVCqVv6Kioirs2MURQmCNX/4kgh95OKGGrspxJhEREZWBysFOp06d5MtFAIBEIoFMJsOyZcvQrVu3Cq3c1atXkZCQgBYtWkBLSwtaWlrw8/PDypUroaWlBWtra2RnZyMpKUkhX3x8PGxsbIo9rq6uLoyNjRVelS3wUSKCI5Ogo6WBT9pzwU8iIqLXReXmhWXLlqFHjx4ICgpCdnY2Zs+ejTt37uDFixe4ePFihVauR48euHXrlkLamDFj4O7ujjlz5sDBwQHa2to4deoUhg4dCgAICQlBZGSkfP6f6qKgVee9lvawNKrYFjAiIiIqnsrBTqNGjRAaGopVq1bByMgIqampGDJkCLy8vGBra1uhlTMyMkKjRo0U0gwNDWFubi5PHzt2LGbMmAEzMzMYGxvj888/h4eHB9q1a1ehdSmPkLgUnL6fkL/gZydOIkhERPQ6qRzsREZGwsHBAV9++WWR2xwdX2/H2xUrVkBDQwNDhw5FVlYWevXqhd9///211qE0a/9d8LN3Qxu4WBhWcW2IiIjeLhIhhFAlg6amJmJjY2FlpTjz7/Pnz2FlZYW8vLwKreDrkJycDBMTE0il0grvvxOTlIHOy84gVyaw36sDmjrUrNDjExERva2U/f1WuYOyEAISSeFZf1NTUyt0xJS62HghArkygXa1zRjoEBERVQGlH2PNmDEDQP7oq/nz5ysMP8/Ly0NAQACaNWtW4RV8k4XGpWDr5fy5hyZ2qVPFtSEiIno7KR3sBAcHA8hv2bl16xZ0dHTk23R0dNC0aVN88cUXFV/DN9SuwEjM/fsWCp4Rxkkzq7Q+REREbyulg50zZ84AyB/6/csvv7yWuWneVLHSDHjv+S/QAYAv995Gl3qWsDXRr7J6ERERvY1U7rPj6+vLQKcUEc/SIHul23eeEHj0LL1qKkRERPQW40qUlcDFwhAar/Th1pRI4GxReJkNIiIiqlwMdiqBrYk+fIY0hua/o9Y0JRIsHtKIj7CIiIiqAFejrCQjWjuic11LPHqWDmcLAwY6REREVYTBTiWyNdFnkENERFTF+BiLiIiI1BqDHSIiIlJrDHaIiIhIrTHYISIiIrXGYIeIiIjUGoMdIiIiUmsMdoiIiEitMdghIiIitcZgh4iIiNQagx0iIiJSawx2iIiISK0x2CEiIiK1xmCHiIiI1BqDHSIiIlJrDHaIiIhIrTHYISIiIrXGYIeIiIjUGoMdIiIiUmsMdoiIiEitMdghIiIitcZgh4iIiNQagx0iIiJSawx2iIiISK0x2CEiIiK1xmCHiIiI1BqDHSIiIlJrDHaIiIhIrTHYISIiIrXGYIeIiIjUGoMdIiIiUmsMdoiIiEitMdghIiIitcZgh4iIiNQagx0iIiJSawx2iIiISK0x2CEiIiK1xmCHiIiI1BqDHSIiIlJrDHaIiIhIrTHYISIiIrXGYIeIiIjUGoMdIiIiUmsMdoiIiEitMdghIiIitcZgh4iIiNQagx0iIiJSawx2iIiISK0x2CEiIiK1xmCHiIiI1BqDHSIiIlJrDHaIiIhIrTHYISIiIrXGYIeIiIjUGoMdIiIiUmvVOtjx8fFB69atYWRkBCsrKwwaNAghISEK+2RmZsLLywvm5uaoUaMGhg4divj4+CqqMREREVU31TrY8fPzg5eXFy5fvowTJ04gJycHnp6eSEtLk+8zffp0HDx4ELt374afnx9iYmIwZMiQKqw1ERERVScSIYSo6koo6+nTp7CysoKfnx86d+4MqVQKS0tLbN++He+99x4A4P79+6hfvz78/f3Rrl07pY6bnJwMExMTSKVSGBsbV+YpEBERUQVR9ve7WrfsvEoqlQIAzMzMAABXr15FTk4OevbsKd/H3d0djo6O8Pf3L/Y4WVlZSE5OVngRERGRenpjgh2ZTIZp06ahQ4cOaNSoEQAgLi4OOjo6qFmzpsK+1tbWiIuLK/ZYPj4+MDExkb8cHBwqs+pERERUhd6YYMfLywu3b9/Gzp07y30sb29vSKVS+SsqKqoCakhERETVkVZVV0AZkydPxqFDh3Du3DnY29vL021sbJCdnY2kpCSF1p34+HjY2NgUezxdXV3o6upWZpWJiIiomqjWLTtCCEyePBl79+7F6dOn4eLiorC9ZcuW0NbWxqlTp+RpISEhiIyMhIeHx+uuLhEREVVD1bplx8vLC9u3b8f+/fthZGQk74djYmICfX19mJiYYOzYsZgxYwbMzMxgbGyMzz//HB4eHkqPxCIiIiL1Vq2HnkskkiLTfX198cknnwDIn1Rw5syZ2LFjB7KystCrVy/8/vvvJT7GehWHnhMREb15lP39rtbBzuvCYIeIiOjNo5bz7BARERGpisEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTW1CbY+e233+Ds7Aw9PT20bdsWV65cqeoqERERUTWgFsHOrl27MGPGDCxYsADXrl1D06ZN0atXLyQkJFR11YiIiKiKqUWws3z5cowbNw5jxoxBgwYNsGbNGhgYGGDjxo1VXTUiIiKqYlpVXYHyys7OxtWrV+Ht7S1P09DQQM+ePeHv719knqysLGRlZcn/lkqlAIDk5OTKrSwRERFVmILfbSFEifu98cHOs2fPkJeXB2tra4V0a2tr3L9/v8g8Pj4++PbbbwulOzg4VEodiYiIqPKkpKTAxMSk2O1vfLBTFt7e3pgxY4b8b5lMhhcvXsDc3BwSiaTCyklOToaDgwOioqJgbGz8WvOz7Ndfdnnzs+y3q+zy5mfZLPtNyV/esksihEBKSgrs7OxK3O+ND3YsLCygqamJ+Ph4hfT4+HjY2NgUmUdXVxe6uroKaTVr1qysKsLY2LhcH3B58rPs1192efOz7Ler7PLmZ9ks+03JX96yi1NSi06BN76Dso6ODlq2bIlTp07J02QyGU6dOgUPD48qrBkRERFVB298yw4AzJgxA6NHj0arVq3Qpk0b/Pzzz0hLS8OYMWOqumpERERUxdQi2BkxYgSePn2Kr7/+GnFxcWjWrBmOHj1aqNPy66arq4sFCxYUemT2OvKz7Ndfdnnzs+y3q+zy5mfZLPtNyV/esiuCRJQ2XouIiIjoDfbG99khIiIiKgmDHSIiIlJrDHaIiIhIrTHYISIiIrXGYKcS/fbbb3B2doaenh7atm2LK1euKJXv3Llz6N+/P+zs7CCRSLBv3z6ly/Tx8UHr1q1hZGQEKysrDBo0CCEhIUrnX716NZo0aSKf/MnDwwNHjhxROv/LlixZAolEgmnTpim1/zfffAOJRKLwcnd3V7q86OhofPjhhzA3N4e+vj4aN26MoKAgpfI6OzsXKlsikcDLy6vUvHl5eZg/fz5cXFygr6+POnXqYNGiRaWu1fKylJQUTJs2DU5OTtDX10f79u0RGBhYaL/Srg0hBL7++mvY2tpCX18fPXv2RFhYmNL59+zZA09PT/ls4tevX1e6/JycHMyZMweNGzeGoaEh7Ozs8PHHHyMmJkapsr/55hu4u7vD0NAQpqam6NmzJwICApSu+8smTpwIiUSCn3/+Wam8n3zySaHPvnfv3iqVfe/ePQwYMAAmJiYwNDRE69atERkZWWreoq47iUSCH374QamyU1NTMXnyZNjb20NfX1++GLIyeePj4/HJJ5/Azs4OBgYG6N27t/x6Uea7JDMzE15eXjA3N0eNGjUwdOhQ+QSvyuRft24dunbtCmNjY0gkEiQlJcm3lZb/xYsX+Pzzz1GvXj3o6+vD0dERU6ZMgVQqVarsCRMmoE6dOtDX14elpSUGDhwoX2JIle9RIQT69Okjf3+Vydu1a9dCn/fEiRNVKtvf3x/du3eHoaEhjI2N0blzZyxcuLDEvI8ePSr2etu9e7dSZcfFxeGjjz6CjY0NDA0N0aJFC/z9999K5X3w4AEGDx4MS0tLGBsbY/jw4YUmBK4sDHYqya5duzBjxgwsWLAA165dQ9OmTdGrVy8kJCSUmjctLQ1NmzbFb7/9pnK5fn5+8PLywuXLl3HixAnk5OTA09MTaWlpSuW3t7fHkiVLcPXqVQQFBaF79+4YOHAg7ty5o1I9AgMDsXbtWjRp0kSlfA0bNkRsbKz8deHCBaXyJSYmokOHDtDW1saRI0dw9+5d/PTTTzA1NVW6vi+Xe+LECQDAsGHDSs27dOlSrF69GqtWrcK9e/ewdOlSLFu2DL/++qtSZQPAZ599hhMnTuCPP/7ArVu34OnpiZ49eyI6Olphv9KujWXLlmHlypVYs2YNAgICYGhoiF69eiEzM1Op/GlpaejYsSOWLl1a7Pbi8qenp+PatWuYP38+rl27hj179iAkJAQDBgxQquy6deti1apVuHXrFi5cuABnZ2d4enri6dOnSuUvsHfvXly+fFlh+nhl8vbu3VvhGtixY4fS+R88eICOHTvC3d0dZ8+exc2bNzF//nzo6emVmvflMmNjY7Fx40ZIJBIMHTpUqbJnzJiBo0ePYuvWrbh37x6mTZuGyZMn48CBAyXmFUJg0KBBePjwIfbv34/g4GA4OTmhZ8+eSEtLU+q7ZPr06Th48CB2794NPz8/xMTEYMiQIQCU+y5KT09H7969MW/evEL1Ky1/TEwMYmJi8OOPP+L27dvYtGkTjh49irFjxypVdsuWLeHr64t79+7h2LFjEELA09MTeXl5Kn2P/vzzzwrLDCmbd9y4cQqf+7Jly5TO7+/vj969e8PT0xNXrlxBYGAgJk+ejAsXLpSY18HBodD19u2336JGjRro06ePUmV//PHHCAkJwYEDB3Dr1i0MGTIEw4cPx8GDB0vMm5aWBk9PT0gkEpw+fRoXL15EdnY2+vfvD5lMVuh9rXCCKkWbNm2El5eX/O+8vDxhZ2cnfHx8VDoOALF3794y1yMhIUEAEH5+fmU+hqmpqfi///s/pfdPSUkRbm5u4sSJE6JLly5i6tSpSuVbsGCBaNq0aZnqOGfOHNGxY8cy5S3K1KlTRZ06dYRMJit13379+olPP/1UIW3IkCFi1KhRSpWVnp4uNDU1xaFDhxTSW7RoIb788sti8716bchkMmFjYyN++OEHeVpSUpLQ1dUVO3bsKDX/yyIiIgQAERwcrHT5Rbly5YoAIB4/fqxyXqlUKgCIkydPKl32kydPRK1atcTt27eFk5OTWLFihVJ5R48eLQYOHFhifUrKP2LECPHhhx+WKe+rBg4cKLp37650/oYNG4qFCxcqpBV17byaNyQkRAAQt2/flqfl5eUJS0tLsX79+kJlv/pdkpSUJLS1tcXu3bvl+9y7d08AEP7+/qXmf9mZM2cEAJGYmFjkeZeWv8Cff/4pdHR0RE5Ojsp5b9y4IQCI8PBwpcsODg4WtWrVErGxscV+tkXlVeV7saj8bdu2FV999VWZ8r6qWbNmhb6/SspvaGgotmzZorCfmZlZoWvm1bzHjh0TGhoaQiqVyvdJSkoSEolEnDhxotRzKS+27FSC7OxsXL16FT179pSnaWhooGfPnvD393+tdZFKpQAAMzMzlfPm5eVh586dSEtLU2npDS8vL/Tr10/h/JUVFhYGOzs71K5dG6NGjUJkZKRS+Q4cOIBWrVph2LBhsLKyQvPmzbF+/XqVywfyP7+tW7fi008/VWph2Pbt2+PUqVMIDQ0FANy4cQMXLlxAnz59lCovNzcXeXl50NPTU0jX19dXumULACIiIhAXF6fwvpuYmKBt27av/borIJVKIZFIVF57Ljs7G+vWrYOJiQmaNm2qVB6ZTIaPPvoIs2bNQsOGDVWu69mzZ2FlZYV69erhf//7H54/f650uYcPH0bdunXRq1cvWFlZoW3btio9fi4QHx+Pw4cPY+zYsUrnad++PQ4cOIDo6GgIIXDmzBmEhobC09OzxHxZWVkAoHDdaWhoQFdXt8jr7tXvkqtXryInJ0fhenN3d4ejo2OR11t5vouUzS+VSmFsbAwtLa1C6SXlTUtLg6+vL1xcXODg4KBU2enp6Rg5ciR+++23YtdhLKnsbdu2wcLCAo0aNYK3tzfS09OVyp+QkICAgABYWVmhffv2sLa2RpcuXZT6zF519epVXL9+vdjrraj87du3x65du/DixQvIZDLs3LkTmZmZ6Nq1a4l5s7KyIJFIFCYW1NPTg4aGhkrfc2VW6eHUWyg6OloAEJcuXVJInzVrlmjTpo1Kx0I5Wnby8vJEv379RIcOHVTKd/PmTWFoaCg0NTWFiYmJOHz4sNJ5d+zYIRo1aiQyMjKEEKrdwfzzzz/izz//FDdu3BBHjx4VHh4ewtHRUSQnJ5eaV1dXV+jq6gpvb29x7do1sXbtWqGnpyc2bdqkdN0L7Nq1S2hqaoro6Gil9s/LyxNz5swREolEaGlpCYlEIhYvXqxSmR4eHqJLly4iOjpa5Obmij/++ENoaGiIunXrFpvn1Wvj4sWLAoCIiYlR2G/YsGFi+PDhpeZ/WUW07GRkZIgWLVqIkSNHKp334MGDwtDQUEgkEmFnZyeuXLmidNmLFy8W77zzjrw1TpWWnR07doj9+/eLmzdvir1794r69euL1q1bi9zc3FLzF9zVGxgYiOXLl4vg4GDh4+MjJBKJOHv2rFLnXWDp0qXC1NRU/v9HmbpnZmaKjz/+WAAQWlpaQkdHR2zevLnUvNnZ2cLR0VEMGzZMvHjxQmRlZYklS5YIAMLT01Mhb1HfJdu2bRM6OjqFymndurWYPXt2qflfVlrLjjLfZU+fPhWOjo5i3rx5Suf97bffhKGhoQAg6tWrV2SrTnH5x48fL8aOHSv/u6jPpri8a9euFUePHhU3b94UW7duFbVq1RKDBw9Wqmx/f38BQJiZmYmNGzeKa9euiWnTpgkdHR0RGhqq1HkX+N///ifq169f5Lbi8icmJgpPT0/59WZsbCyOHTtWat6EhARhbGwspk6dKtLS0kRqaqqYPHmyACDGjx9fbB0rCoOdSlBdgp2JEycKJycnERUVpVK+rKwsERYWJoKCgsTcuXOFhYWFuHPnTqn5IiMjhZWVlbhx44Y8TZVg51WJiYnC2NhYqUdo2trawsPDQyHt888/F+3atVO5XE9PT/Huu+8qvf+OHTuEvb292LFjh7h586bYsmWLMDMzUynQCg8PF507dxYAhKampmjdurUYNWqUcHd3LzZPdQ52srOzRf/+/UXz5s0Vmq1Ly5uamirCwsKEv7+/+PTTT4Wzs7OIj48vNX9QUJCwtrZWCFBVCXZe9eDBA6UfoRX8f//ggw8U9uvfv794//33VSq7Xr16YvLkycVuLyr/Dz/8IOrWrSsOHDggbty4IX799VdRo0aNQo8GisobFBQkmjZtKr/uevXqJfr06SN69+6tsF9R3yWqBDulfReVFuyUll8qlYo2bdqI3r17i+zsbKXzJiUlidDQUOHn5yf69+8vWrRoUSjQLCr//v37haurq0hJSZGnFfX+KvsdfOrUqSIfoRWVv+D/ube3t8K+jRs3FnPnzlW67PT0dGFiYiJ+/PHHIrcXl3/y5MmiTZs24uTJk+L69evim2++ESYmJuLmzZul5j127JioXbu2kEgkQlNTU3z44YeiRYsWYuLEiSW8OxWDwU4lyMrKEpqamoUu/I8//lgMGDBApWOVNdjx8vIS9vb24uHDhyrnfVWPHj2Uirz37t0r/9IseAGQX9hF3SWXplWrVgr/gYvj6OiocJclhBC///67sLOzU6m8R48eCQ0NDbFv3z6l89jb24tVq1YppC1atEjUq1dPpbKFyP+xLwhWhg8fLvr27Vvsvq9eGwU/0K8GKJ07dxZTpkwpNf/LyhPsZGdni0GDBokmTZqIZ8+eqZT3Va6urkW2kr2af8WKFfLr7OVrT0NDQzg5OZWpbAsLC7FmzZpSy87KyhJaWlpi0aJFCvvNnj1btG/fXumyz507JwCI69evF1unV/Onp6cLbW3tQv29xo4dK3r16qV02UlJSSIhIUEIkd/fcNKkSfJtxX2XFPxAvxqgODo6iuXLl5ea/2UlBTul5U9OThYeHh6iR48ehQIVVb4Hs7KyhIGBgdi+fXup+adOnVrs9dalSxeVy05NTRUAxNGjR0st++HDhwKA+OOPPxTShw8fLm9FVabsLVu2CG1tbfnn/rLi8oeHhxfq5yVE/m/EhAkTlC776dOn8s/a2tpaLFu2rNh9Kwr77FQCHR0dtGzZEqdOnZKnyWQynDp1SqW+L2UhhMDkyZOxd+9enD59Gi4uLuU+pkwmkz/fL0mPHj1w69YtXL9+Xf5q1aoVRo0ahevXr0NTU1OlclNTU/HgwQPY2tqWum+HDh0KDXMMDQ2Fk5OTSmX6+vrCysoK/fr1UzpPeno6NDQU/ytpamqWaYSBoaEhbG1tkZiYiGPHjmHgwIFK53VxcYGNjY3CdZecnIyAgIBKv+4K5OTkYPjw4QgLC8PJkydhbm5eruMpe+199NFHuHnzpsK1Z2dnh1mzZuHYsWMql/vkyRM8f/5cqWtPR0cHrVu3Lvf1t2HDBrRs2VLpPkpA/vudk5NT7uvPxMQElpaWCAsLQ1BQEAYOHFjqd0nLli2hra2tcL2FhIQgMjISHh4e5f4uUiZ/cnIyPD09oaOjgwMHDsj7H5WlbJF/84+srKxS88+dO7fQ9QYAK1aswMaNG1UuuyC/ra1tqWU7OzvDzs6uyOvN0dFR6bI3bNiAAQMGwNLSUuE9KCl/Qb+ioq63vLw8pcu2sLBAzZo1cfr0aSQkJMhHbFaqSg+n3lI7d+4Uurq6YtOmTeLu3bti/PjxombNmiIuLq7UvCkpKSI4OFgEBwcLAPJ+AK+OaCnK//73P2FiYiLOnj0rYmNj5a/09HSl6j137lzh5+cnIiIixM2bN8XcuXOFRCIRx48fVyr/q1R5jDVz5kxx9uxZERERIS5evCh69uwpLCwsirzzeNWVK1eElpaW+P7770VYWJjYtm2bMDAwEFu3blW6rnl5ecLR0VHMmTNH6TxC5I/kqVWrljh06JCIiIgQe/bsERYWFoWa8kty9OhRceTIEfHw4UNx/Phx0bRpU9G2bdtCTfKlXRtLliwRNWvWlPc/GThwoHBxcZHf8ZaW//nz5yI4OFgcPnxYABA7d+4UwcHBIjY2ttT82dnZYsCAAcLe3l5cv35d4frLysoqMW9qaqrw9vYW/v7+4tGjRyIoKEiMGTNG6Orqyu8iVf1/8fJjrJLypqSkiC+++EL4+/uLiIgIcfLkSdGiRQvh5uYmMjMzlSp7z549QltbW6xbt06EhYWJX3/9VWhqaorz588rVW+pVCoMDAzE6tWrC51Hafm7dOkiGjZsKM6cOSMePnwofH19hZ6envj9999Lzfvnn3+KM2fOiAcPHoh9+/YJJycnMWTIECGEct8lEydOFI6OjuL06dMiKChIeHh4yB8nK5M/NjZWBAcHi/Xr1wsA4ty5cyI4OFg8f/681PxSqVS0bdtWNG7cWISHhyvsM3HixBLzPnjwQCxevFgEBQWJx48fi4sXL4r+/fsLMzMzER8fX6bvUfzbclZa3vDwcLFw4UIRFBQkIiIixP79+0Xt2rVF586dlX7fVqxYIYyNjcXu3btFWFiY+Oqrr4Senp4YOXKkUvUOCwsTEolEHDlyRCG9tLKzs7OFq6ur6NSpkwgICBDh4eHixx9/FBKJRPTt27fUsjdu3Cj8/f1FeHi4+OOPP4SZmZmYMWNGse9pRWKwU4l+/fVX4ejoKHR0dESbNm3E5cuXlcpX0KT76mv06NGl5i0qHwDh6+urVNmffvqpcHJyEjo6OsLS0lL06NGjzIGOEKoFOyNGjBC2trZCR0dH1KpVS4wYMaLIDoPFOXjwoGjUqJHQ1dUV7u7uYt26dSrV9dixYwKACAkJUSlfcnKymDp1qnB0dBR6enqidu3a4ssvvxRZWVlKH2PXrl2idu3aQkdHR9jY2AgvLy+RlJRUaL/Srg2ZTCbmz58vrK2tha6urujRo4fC+ZSW39fXt8jtCxYsKDV/waOvol5nzpwpMW9GRoYYPHiwsLOzEzo6OsLW1lYMGDBAoYOyqv8vXg52Ssqbnp4uPD09haWlpdDW1hZOTk5i3LhxCjcmypS9YcMG4erqKvT09ETTpk3lj0KVybt27Vqhr69fps88NjZWfPLJJ8LOzk7o6emJevXqiZ9++knIZLJS8/7yyy/C3t5eaGtrC0dHR/HVV1/Jr1tlvksyMjLEpEmThKmpqTAwMBCDBw+WB8bK5F+wYEGx+5SWv7hzK+lVkDc6Olr06dNHWFlZCW1tbWFvby9Gjhwp7t+/r3TdX1UQ7JSWNzIyUnTu3FmYmZkJXV1d4erqKmbNmiXv26Zs2T4+PsLe3l4YGBgIDw8Pcf78eaXzent7CwcHB5GXl1foHErLHxoaKoYMGSKsrKyEgYGBaNKkidiyZYtSeefMmSOsra2Ftra2cHNzk1+nr4Pk3xMkIiIiUkvss0NERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BARveLs2bOQSCRISkqq6qoQUQVgsENERERqjcEOERERqTUGO0RU7chkMvj4+MDFxQX6+vpo2rQp/vrrLwD/PWI6fPgwmjRpAj09PbRr1w63b99WOMbff/+Nhg0bQldXF87Ozvjpp58UtmdlZWHOnDlwcHCArq4uXF1dsWHDBoV9rl69ilatWsHAwADt27cvtNI0Eb0ZGOwQUbXj4+ODLVu2YM2aNbhz5w6mT5+ODz/8EH5+fvJ9Zs2ahZ9++gmBgYGwtLRE//79kZOTAyA/SBk+fDjef/993Lp1C9988w3mz5+PTZs2yfN//PHH2LFjB1auXIl79+5h7dq1qFGjhkI9vvzyS/z0008ICgqClpYWPv3009dy/kRUsbgQKBFVK1lZWTAzM8PJkyfh4eEhT//ss8+Qnp6O8ePHo1u3bti5cydGjBgBAHjx4gXs7e2xadMmDB8+HKNGjcLTp09x/Phxef7Zs2fj8OHDuHPnDkJDQ1GvXj2cOHECPXv2LFSHs2fPolu3bjh58iR69OgBAPjnn3/Qr18/ZGRkQE9Pr5LfBSKqSGzZIaJqJTw8HOnp6XjnnXdQo0YN+WvLli148OCBfL+XAyEzMzPUq1cP9+7dAwDcu3cPHTp0UDhuhw4dEBYWhry8PFy/fh2ampro0qVLiXVp0qSJ/N+2trYAgISEhHKfIxG9XlpVXQEiopelpqYCAA4fPoxatWopbNPV1VUIeMpKX19fqf20tbXl/5ZIJADy+xMR0ZuFLTtEVK00aNAAurq6iIyMhKurq8LLwcFBvt/ly5fl/05MTERoaCjq168PAKhfvz4uXryocNyLFy+ibt260NTUROPGjSGTyRT6ABGR+mLLDhFVK0ZGRvjiiy8wffp0yGQydOzYEVKpFBcvXoSxsTGcnJwAAAsXLoS5uTmsra3x5ZdfwsLCAoMGDQIAzJw5E61bt8aiRYswYsQI+Pv7Y9WqVfj9998BAM7Ozhg9ejQ+/fRTrFy5Ek2bNsXjx4+RkJCA4cOHV9WpE1ElYbBDRNXOokWLYGlpCR8fHzx8+BA1a9ZEixYtMG/ePPljpCVLlmDq1KkICwtDs2bNcPDgQejo6AAAWrRogT///BNff/01Fi1aBFtbWyxcuBCffPKJvIzVq1dj3rx5mDRpEp4/fw5HR0fMmzevKk6XiCoZR2MR0RulYKRUYmIiatasWdXVIaI3APvsEBERkVpjsENERERqjY+xiIiISK2xZYeIiIjUGoMdIiIiUmsMdoiIiEitMdghIiIitcZgh4iIiNQagx0iIiJSawx2iIiISK0x2CEiIiK1xmCHiIiI1Nr/A45Ofu2hDDlOAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plt.plot(np.arange(len(epochs_x)), epochs_acc, marker = '.')\n", + "plt.xlabel('epoch')\n", + "plt.ylabel('test accuracy')\n", + "plt.ylim(0, 100)\n", + "plt.xticks(np.arange(len(epochs_x)))\n", + "for i, txt in enumerate(epochs_acc):\n", + " if i%2 == 0:\n", + " pass\n", + " else:\n", + " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", + "plt.show()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "speck-rescnn", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From fd1f38eb338baa4d93df594e9172ef6395afae92 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Mon, 29 Apr 2024 15:09:02 +0200 Subject: [PATCH 056/379] baseline with transfer-learning --- .../transfer-learning/baseline-SCNN-3.ipynb | 571 +++++++++--------- .../transfer-learning/seq_model.py | 86 +++ 2 files changed, 361 insertions(+), 296 deletions(-) create mode 100644 tests/test_nonsequential/transfer-learning/seq_model.py diff --git a/tests/test_nonsequential/transfer-learning/baseline-SCNN-3.ipynb b/tests/test_nonsequential/transfer-learning/baseline-SCNN-3.ipynb index 2e941d62..5e27fd6f 100644 --- a/tests/test_nonsequential/transfer-learning/baseline-SCNN-3.ipynb +++ b/tests/test_nonsequential/transfer-learning/baseline-SCNN-3.ipynb @@ -2,13 +2,12 @@ "cells": [ { "cell_type": "code", - "execution_count": 13, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import torch, random\n", "import torch.nn as nn\n", - "import sinabs.layers as sl\n", "from tqdm.notebook import tqdm\n", "\n", "from tonic.transforms import ToFrame\n", @@ -16,15 +15,15 @@ "from torch.nn import CrossEntropyLoss\n", "from torch.optim import Adam\n", "\n", - "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential\n", - "\n", "import matplotlib.pyplot as plt\n", - "import numpy as np" + "import numpy as np\n", + "\n", + "from seq_model import SNN" ] }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -37,452 +36,443 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "batch_size = 3\n", + "batch_size_pre = 32\n", "num_workers = 1\n", - "epochs_pretrain = 5\n", + "epochs_pretrain = 1\n", "epochs = 30\n", - "lr = 1e-3" + "lr = 1e-3\n", + "n_time_steps = 50" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": 4, "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "device: NVIDIA GeForce RTX 3070 Ti\n" + ] + } + ], "source": [ - "## Dataset for transfer learning\n", - "\n", - "Dataset used to pre-train the network such that its parameters are set within a \"good\" region of the parameters space (i.e., hopefully training on a \"simpler\" dataset sets the wheights to values that improve the training on a harder dataset)." + "if torch.cuda.is_available():\n", + " device = torch.device('cuda:0')\n", + " print('device: ', torch.cuda.get_device_name(0))\n", + "else:\n", + " device = torch.device('cpu')" ] }, { - "cell_type": "code", - "execution_count": 16, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "from tonic.datasets.nmnist import NMNIST" + "## Training/Testing helper functions" ] }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ - "root_dir = \"../NMNIST\"\n", - "_ = NMNIST(save_to=root_dir, train=True)\n", - "_ = NMNIST(save_to=root_dir, train=False)" + "def train(batch_size, feature_map_size, dataloader_train, model, loss_fn, optimizer, epochs, test_func, dataloader_test, phase):\n", + " epochs_y = []\n", + " epochs_x = []\n", + " epochs_acc = []\n", + " model.train()\n", + "\n", + " for e in range(epochs):\n", + " losses = []\n", + " batches = []\n", + " batch_count = 0\n", + " train_p_bar = tqdm(dataloader_train)\n", + "\n", + " for X, y in train_p_bar:\n", + " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", + " X = X.reshape(-1, feature_map_size[2], feature_map_size[0], feature_map_size[1]).to(dtype=torch.float, device=device)\n", + " y = y.to(dtype=torch.long, device=device)\n", + "\n", + " # forward\n", + " pred = model(X)\n", + "\n", + " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", + " pred = pred.reshape(batch_size, n_time_steps, -1)\n", + "\n", + " # accumulate all time-steps output for final prediction\n", + " pred = pred.sum(dim = 1)\n", + " loss = loss_fn(pred, y)\n", + "\n", + " # gradient update\n", + " optimizer.zero_grad()\n", + " loss.backward()\n", + " optimizer.step()\n", + "\n", + " # detach the neuron states and activations from current computation graph(necessary)\n", + " model.detach_neuron_states()\n", + "\n", + " train_p_bar.set_description(f\"{phase} - Epoch {e} - BPTT Training Loss: {round(loss.item(), 4)}\")\n", + "\n", + " batch_count += 1\n", + " losses.append(loss.item())\n", + " batches.append(batch_count)\n", + "\n", + " epochs_y.append(losses)\n", + " epochs_x.append(batches)\n", + "\n", + " acc = test_func(feature_map_size, dataloader_test, model)\n", + " print(f'{phase} - Epoch {e} accuracy: {acc}')\n", + " epochs_acc.append(acc)\n", + "\n", + " return epochs_x, epochs_y, epochs_acc\n" ] }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ - "n_time_steps = 50\n", - "to_raster = ToFrame(sensor_size=(128,128, 2), n_time_bins=n_time_steps)\n", - "# to_raster = ToFrame(sensor_size=NMNIST.sensor_size, n_time_bins=n_time_steps)\n", + "def test(feature_map_size, dataloader, model):\n", + " correct_predictions = []\n", + " with torch.no_grad():\n", + " test_p_bar = tqdm(dataloader)\n", + " for X, y in test_p_bar:\n", + " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", + " X = X.reshape(-1, feature_map_size[2], feature_map_size[0], feature_map_size[1]).to(dtype=torch.float, device=device)\n", + " y = y.to(dtype=torch.long, device=device)\n", "\n", - "snn_train_dataset_pre = NMNIST(save_to=root_dir, train=True, transform=to_raster)\n", - "snn_test_dataset_pre = NMNIST(save_to=root_dir, train=False, transform=to_raster)" + " # forward\n", + " output = model(X)\n", + "\n", + " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", + " output = output.reshape(batch_size, n_time_steps, -1)\n", + "\n", + " # accumulate all time-steps output for final prediction\n", + " output = output.sum(dim=1)\n", + "\n", + " # calculate accuracy\n", + " pred = output.argmax(dim=1, keepdim=True)\n", + "\n", + " # compute the total correct predictions\n", + " correct_predictions.append(pred.eq(y.view_as(pred)))\n", + "\n", + " test_p_bar.set_description(f\"Testing Model...\")\n", + " \n", + " correct_predictions = torch.cat(correct_predictions)\n", + " return correct_predictions.sum().item()/(len(correct_predictions))*100" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Pre-training loop" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Loading the pre-training data. Dataset used to pre-train the network such that its parameters are set within a \"good\" region of the parameters space (i.e., hopefully training on a \"simpler\" dataset sets the wheights to values that improve the training on a harder dataset)." ] }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 7, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "The transformed array is in shape [Time-Step, Channel, Height, Width] --> (50, 2, 128, 128)\n" + "The transformed array is in shape [Time-Step, Channel, Height, Width] --> (50, 2, 34, 34)\n" ] } ], "source": [ + "from tonic.datasets.nmnist import NMNIST\n", + "\n", + "root_dir = \"../NMNIST\"\n", + "_ = NMNIST(save_to=root_dir, train=True)\n", + "_ = NMNIST(save_to=root_dir, train=False)\n", + "\n", + "to_raster = ToFrame(sensor_size=NMNIST.sensor_size, n_time_bins=n_time_steps)\n", + "\n", + "snn_train_dataset_pre = NMNIST(save_to=root_dir, train=True, transform=to_raster)\n", + "snn_test_dataset_pre = NMNIST(save_to=root_dir, train=False, transform=to_raster)\n", + "\n", "sample_data, label = snn_train_dataset_pre[0]\n", - "print(f\"The transformed array is in shape [Time-Step, Channel, Height, Width] --> {sample_data.shape}\")" + "print(f\"The transformed array is in shape [Time-Step, Channel, Height, Width] --> {sample_data.shape}\")\n", + "\n", + "snn_train_dataloader_pre = DataLoader(snn_train_dataset_pre, batch_size=batch_size_pre, num_workers=num_workers, drop_last=True, shuffle=True)\n", + "snn_test_dataloader_pre = DataLoader(snn_test_dataset_pre, batch_size=batch_size_pre, num_workers=num_workers, drop_last=True, shuffle=False)" ] }, { - "cell_type": "code", - "execution_count": 20, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "snn_train_dataloader_pre = DataLoader(snn_train_dataset_pre, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True)\n", - "snn_test_dataloader_pre = DataLoader(snn_test_dataset_pre, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=False)" + "instantiating model..." ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": 8, "metadata": {}, + "outputs": [], "source": [ - "## Training dataset\n", - "\n", - "This is the dataset to be used for training after the network has gone through a pre-training phase." + "snn = SNN(10, 10, batch_size_pre).to(device)\n", + "snn.init_weights()" ] }, { - "cell_type": "code", - "execution_count": 21, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "from tonic.datasets.dvsgesture import DVSGesture" + "loss and optimizer..." ] }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ - "root_dir = \"../DVSGESTURE\"\n", - "_ = DVSGesture(save_to=root_dir, train=True)\n", - "_ = DVSGesture(save_to=root_dir, train=False)" + "optimizer = Adam(snn.parameters(), lr=lr, betas=(0.9, 0.999), eps=1e-8)\n", + "loss_fn = CrossEntropyLoss()" ] }, { - "cell_type": "code", - "execution_count": 23, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "n_time_steps = 50\n", - "to_raster = ToFrame(sensor_size=DVSGesture.sensor_size, n_time_bins=n_time_steps)\n", - "\n", - "snn_train_dataset = DVSGesture(save_to=root_dir, train=True, transform=to_raster)\n", - "snn_test_dataset = DVSGesture(save_to=root_dir, train=False, transform=to_raster)" + "pre-training the model..." ] }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 10, "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "The transformed array is in shape [Time-Step, Channel, Height, Width] --> (50, 2, 128, 128)\n" - ] + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "a5bfe44485924a4a85b3357e62c24e75", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/1875 [00:00 {sample_data.shape}\")" + "epochs_x_pre, epochs_y_pre, epochs_acc_pre = train(\n", + " batch_size_pre,\n", + " NMNIST.sensor_size, \n", + " snn_train_dataloader_pre, \n", + " snn, \n", + " loss_fn, \n", + " optimizer, \n", + " epochs_pretrain, \n", + " test, \n", + " snn_test_dataloader_pre,\n", + " 'pre-training'\n", + " )\n", + "\n", + "snn.export_conv_params()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "plotting..." ] }, { "cell_type": "code", - "execution_count": 25, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "snn_train_dataloader = DataLoader(snn_train_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True)\n", - "snn_test_dataloader = DataLoader(snn_test_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=False)" + "y_avg = []\n", + "for y in epochs_y_pre:\n", + " y_avg.append(np.mean(y))\n", + "\n", + "plt.plot(np.arange(len(epochs_x_pre)), y_avg, marker = '.')\n", + "plt.xlabel('epoch')\n", + "plt.ylabel('average loss')\n", + "plt.ylim(0,)\n", + "plt.xticks(np.arange(len(epochs_x_pre)))\n", + "for i, txt in enumerate(y_avg):\n", + " if i%2 == 0:\n", + " pass\n", + " else:\n", + " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "plt.plot(np.arange(len(epochs_x_pre)), epochs_acc_pre, marker = '.')\n", + "plt.xlabel('epoch')\n", + "plt.ylabel('test accuracy')\n", + "plt.ylim(0, 100)\n", + "plt.xticks(np.arange(len(epochs_x_pre)))\n", + "for i, txt in enumerate(epochs_acc_pre):\n", + " if i%2 == 0:\n", + " pass\n", + " else:\n", + " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", + "plt.show()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Network Module\n", - "\n", - "We need to define a `nn.Module` implementing the network we want the chip to reproduce." + "## \"Post-training\" loop" ] }, { - "cell_type": "code", - "execution_count": 26, + "cell_type": "markdown", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "device: NVIDIA GeForce RTX 3070 Ti\n" - ] - } - ], "source": [ - "if torch.cuda.is_available():\n", - " device = torch.device('cuda:0')\n", - " print('device: ', torch.cuda.get_device_name(0))\n", - "else:\n", - " device = torch.device('cpu')" + "loading the data..." ] }, { "cell_type": "code", - "execution_count": 29, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "class SNN(nn.Module):\n", - " def __init__(self) -> None:\n", - " super().__init__()\n", - "\n", - " self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False)\n", - " self.iaf1 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - " self.pool1 = nn.AvgPool2d(2,2)\n", - "\n", - " self.conv2 = nn.Conv2d(10, 10, 2, 1, bias=False)\n", - " self.iaf2 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - " self.pool2 = nn.AvgPool2d(3,3)\n", - "\n", - " self.conv3 = nn.Conv2d(10, 10, 3, 1, bias=False)\n", - " self.iaf3 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - " self.pool3 = nn.AvgPool2d(2,2)\n", - "\n", - " self.flat = nn.Flatten()\n", + "from tonic.datasets.dvsgesture import DVSGesture\n", "\n", - " self.fc1 = nn.Linear(810, 100, bias=False)\n", - " self.iaf4 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - "\n", - " self.fc2 = nn.Linear(100, 100, bias=False)\n", - " self.iaf5 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - "\n", - " self.fc3 = nn.Linear(100, 100, bias=False)\n", - " self.iaf6 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - "\n", - " self.fc4 = nn.Linear(100, 11, bias=False)\n", - " self.iaf7 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - "\n", - " def detach_neuron_states(self):\n", - " for name, layer in self.named_modules():\n", - " if name != '':\n", - " if isinstance(layer, sl.StatefulLayer):\n", - " for name, buffer in layer.named_buffers():\n", - " buffer.detach_()\n", - "\n", - " def init_weights(self):\n", - " for name, layer in self.named_modules():\n", - " if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear):\n", - " nn.init.xavier_normal_(layer.weight.data)\n", - "\n", - " def forward(self, x):\n", - " \n", - " con1_out = self.conv1(x)\n", - " iaf1_out = self.iaf1(con1_out)\n", - " pool1_out = self.pool1(iaf1_out)\n", - "\n", - " conv2_out = self.conv2(pool1_out)\n", - " iaf2_out = self.iaf2(conv2_out)\n", - " pool2_out = self.pool2(iaf2_out)\n", - "\n", - " conv3_out = self.conv3(pool2_out)\n", - " iaf3_out = self.iaf3(conv3_out)\n", - " pool3_out = self.pool3(iaf3_out)\n", - "\n", - " flat_out = self.flat(pool3_out)\n", - " \n", - " fc1_out = self.fc1(flat_out)\n", - " iaf4_out = self.iaf4(fc1_out)\n", + "root_dir = \"../DVSGESTURE\"\n", + "_ = DVSGesture(save_to=root_dir, train=True)\n", + "_ = DVSGesture(save_to=root_dir, train=False)\n", "\n", - " fc2_out = self.fc2(iaf4_out)\n", - " iaf5_out = self.iaf5(fc2_out)\n", + "to_raster = ToFrame(sensor_size=DVSGesture.sensor_size, n_time_bins=n_time_steps)\n", "\n", - " fc3_out = self.fc3(iaf5_out)\n", - " iaf6_out = self.iaf6(fc3_out)\n", + "snn_train_dataset = DVSGesture(save_to=root_dir, train=True, transform=to_raster)\n", + "snn_test_dataset = DVSGesture(save_to=root_dir, train=False, transform=to_raster)\n", "\n", - " fc4_out = self.fc4(iaf6_out)\n", - " iaf7_out = self.iaf7(fc4_out)\n", + "sample_data, label = snn_train_dataset[0]\n", + "print(f\"The transformed array is in shape [Time-Step, Channel, Height, Width] --> {sample_data.shape}\")\n", "\n", - " return iaf7_out" + "snn_train_dataloader = DataLoader(snn_train_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True)\n", + "snn_test_dataloader = DataLoader(snn_test_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=False)" ] }, { - "cell_type": "code", - "execution_count": 30, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "snn = SNN().to(device)" + "instantiating model..." ] }, { "cell_type": "code", - "execution_count": 31, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ + "snn = SNN(11, 810, batch_size).to(device)\n", "snn.init_weights()" ] }, { - "cell_type": "code", - "execution_count": 32, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "optimizer = Adam(snn.parameters(), lr=lr, betas=(0.9, 0.999), eps=1e-8)\n", - "loss_fn = CrossEntropyLoss()" + "loading weights from pre-training..." ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "metadata": {}, + "outputs": [], "source": [ - "## Define train and test functions" + "snn.load_conv_params()" ] }, { - "cell_type": "code", - "execution_count": 33, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "def train(dataloader, model, loss_fn, optimizer, epochs, test_func, dataloader_test):\n", - " epochs_y = []\n", - " epochs_x = []\n", - " epochs_acc = []\n", - " model.train()\n", - "\n", - " for e in range(epochs):\n", - " losses = []\n", - " batches = []\n", - " batch_count = 0\n", - " train_p_bar = tqdm(snn_train_dataloader)\n", - "\n", - " for X, y in train_p_bar:\n", - " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", - " X = X.reshape(-1, 2, 128, 128).to(dtype=torch.float, device=device)\n", - " y = y.to(dtype=torch.long, device=device)\n", - "\n", - " # forward\n", - " pred = model(X)\n", - "\n", - " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", - " pred = pred.reshape(batch_size, n_time_steps, -1)\n", - "\n", - " # accumulate all time-steps output for final prediction\n", - " pred = pred.sum(dim = 1)\n", - " loss = loss_fn(pred, y)\n", - "\n", - " # gradient update\n", - " optimizer.zero_grad()\n", - " loss.backward()\n", - " optimizer.step()\n", - "\n", - " # detach the neuron states and activations from current computation graph(necessary)\n", - " model.detach_neuron_states()\n", - "\n", - " train_p_bar.set_description(f\"Epoch {e} - BPTT Training Loss: {round(loss.item(), 4)}\")\n", - "\n", - " batch_count += 1\n", - " losses.append(loss.item())\n", - " batches.append(batch_count)\n", - "\n", - " epochs_y.append(losses)\n", - " epochs_x.append(batches)\n", - "\n", - " acc = test_func(dataloader_test, model)\n", - " print(f'Epoch {e} accuracy: {acc}')\n", - " epochs_acc.append(acc)\n", - "\n", - " return epochs_x, epochs_y, epochs_acc\n" + "loss and optimizer..." ] }, { "cell_type": "code", - "execution_count": 34, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "def test(dataloader, model):\n", - " correct_predictions = []\n", - " with torch.no_grad():\n", - " test_p_bar = tqdm(dataloader)\n", - " for X, y in test_p_bar:\n", - " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", - " X = X.reshape(-1, 2, 128, 128).to(dtype=torch.float, device=device)\n", - " y = y.to(dtype=torch.long, device=device)\n", - "\n", - " # forward\n", - " output = model(X)\n", - "\n", - " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", - " output = output.reshape(batch_size, n_time_steps, -1)\n", - "\n", - " # accumulate all time-steps output for final prediction\n", - " output = output.sum(dim=1)\n", - "\n", - " # calculate accuracy\n", - " pred = output.argmax(dim=1, keepdim=True)\n", - "\n", - " # compute the total correct predictions\n", - " correct_predictions.append(pred.eq(y.view_as(pred)))\n", - "\n", - " test_p_bar.set_description(f\"Testing Model...\")\n", - " \n", - " correct_predictions = torch.cat(correct_predictions)\n", - " return correct_predictions.sum().item()/(len(correct_predictions))*100" + "optimizer = Adam(snn.parameters(), lr=lr, betas=(0.9, 0.999), eps=1e-8)\n", + "loss_fn = CrossEntropyLoss()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Pre-training loop (transfer learning)" + "training the model..." ] }, { "cell_type": "code", - "execution_count": 35, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "4303748d66154bd0aaba4c9798099e3d", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - " 0%| | 0/359 [00:00" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "y_avg = []\n", - "for y in epochs_y:\n", + "for y in epochs_y_dvs128:\n", " y_avg.append(np.mean(y))\n", "\n", - "plt.plot(np.arange(len(epochs_x)), y_avg, marker = '.')\n", + "plt.plot(np.arange(len(epochs_x_dvs128)), y_avg, marker = '.')\n", "plt.xlabel('epoch')\n", "plt.ylabel('average loss')\n", "plt.ylim(0,)\n", - "plt.xticks(np.arange(len(epochs_x)))\n", + "plt.xticks(np.arange(len(epochs_x_dvs128)))\n", "for i, txt in enumerate(y_avg):\n", " if i%2 == 0:\n", " pass\n", @@ -493,27 +483,16 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAG2CAYAAACZEEfAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAAB9HElEQVR4nO3dd1RUx98G8GfpRUB6kaqg2HvB3oIt9qiJJjHGWH5irLFgYkw0ETWJJsbE9ipqrDGxG3vBhgiKXSmKglQLLL3uvH8QNq60XYrg+nzO2XNk7p07c3evu987d4pECCFAREREpKY0qroCRERERJWJwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqbUqDXbOnTuH/v37w87ODhKJBPv27VPYLoTA119/DVtbW+jr66Nnz54ICwtT2OfFixcYNWoUjI2NUbNmTYwdOxapqamv8SyIiIioOqvSYCctLQ1NmzbFb7/9VuT2ZcuWYeXKlVizZg0CAgJgaGiIXr16ITMzU77PqFGjcOfOHZw4cQKHDh3CuXPnMH78+Nd1CkRERFTNSarLQqASiQR79+7FoEGDAOS36tjZ2WHmzJn44osvAABSqRTW1tbYtGkT3n//fdy7dw8NGjRAYGAgWrVqBQA4evQo+vbtiydPnsDOzq6qToeIiIiqCa2qrkBxIiIiEBcXh549e8rTTExM0LZtW/j7++P999+Hv78/atasKQ90AKBnz57Q0NBAQEAABg8eXOSxs7KykJWVJf9bJpPhxYsXMDc3h0QiqbyTIiIiogojhEBKSgrs7OygoVH8w6pqG+zExcUBAKytrRXSra2t5dvi4uJgZWWlsF1LSwtmZmbyfYri4+ODb7/9toJrTERERFUhKioK9vb2xW6vtsFOZfL29saMGTPkf0ulUjg6OiIqKgrGxsZVWDMiIiJSVnJyMhwcHGBkZFTiftU22LGxsQEAxMfHw9bWVp4eHx+PZs2ayfdJSEhQyJebm4sXL17I8xdFV1cXurq6hdKNjY0Z7BAREb1hSuuCUm3n2XFxcYGNjQ1OnTolT0tOTkZAQAA8PDwAAB4eHkhKSsLVq1fl+5w+fRoymQxt27Z97XUmIiKi6qdKW3ZSU1MRHh4u/zsiIgLXr1+HmZkZHB0dMW3aNHz33Xdwc3ODi4sL5s+fDzs7O/mIrfr166N3794YN24c1qxZg5ycHEyePBnvv/8+R2IRERERgCoOdoKCgtCtWzf53wX9aEaPHo1NmzZh9uzZSEtLw/jx45GUlISOHTvi6NGj0NPTk+fZtm0bJk+ejB49ekBDQwNDhw7FypUrX/u5EBERUfVUbebZqUrJyckwMTGBVCplnx0iIqI3hLK/39W2zw4RERFRRWCwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGptWod7OTl5WH+/PlwcXGBvr4+6tSpg0WLFkEIId9HCIGvv/4atra20NfXR8+ePREWFlaFtSYiIqLqpFoHO0uXLsXq1auxatUq3Lt3D0uXLsWyZcvw66+/yvdZtmwZVq5ciTVr1iAgIACGhobo1asXMjMzq7DmREREVF1IxMvNJNXMu+++C2tra2zYsEGeNnToUOjr62Pr1q0QQsDOzg4zZ87EF198AQCQSqWwtrbGpk2b8P777ytVTnJyMkxMTCCVSmFsbFwp50JEREQVS9nf72rdstO+fXucOnUKoaGhAIAbN27gwoUL6NOnDwAgIiICcXFx6NmzpzyPiYkJ2rZtC39//2KPm5WVheTkZIUXERERqSetqq5ASebOnYvk5GS4u7tDU1MTeXl5+P777zFq1CgAQFxcHADA2tpaIZ+1tbV8W1F8fHzw7bffVl7FiYiIqNqo1i07f/75J7Zt24bt27fj2rVr2Lx5M3788Uds3ry5XMf19vaGVCqVv6KioiqoxkRERFTdVOuWnVmzZmHu3LnyvjeNGzfG48eP4ePjg9GjR8PGxgYAEB8fD1tbW3m++Ph4NGvWrNjj6urqQldXt1LrTkRERNVDtW7ZSU9Ph4aGYhU1NTUhk8kAAC4uLrCxscGpU6fk25OTkxEQEAAPD4/XWlciqhqx0gxcevAMsdKMqq4KVSJnZ2dIJJJCLy8vLwBA165dC22bOHFiiccs6ngSiQQ//PCDwn6HDx9G27Ztoa+vD1NTUwwaNKiyTpMqSbVu2enfvz++//57ODo6omHDhggODsby5cvx6aefAsi/UKdNm4bvvvsObm5ucHFxwfz582FnZ8eLkegtsCswEt57bkEmAA0J4DOkMUa0dqzqalElCAwMRF5envzv27dv45133sGwYcPkaePGjcPChQvlfxsYGJR4zNjYWIW/jxw5grFjx2Lo0KHytL///hvjxo3D4sWL0b17d+Tm5uL27dvlPR16zap1sPPrr79i/vz5mDRpEhISEmBnZ4cJEybg66+/lu8ze/ZspKWlYfz48UhKSkLHjh1x9OhR6OnpVWHNiaiyxUozMHfPLRRMniETwLw9t9G5riVsTfSrtnJU4SwtLRX+XrJkCerUqYMuXbrI0wwMDOTdG5Tx6r779+9Ht27dULt2bQBAbm4upk6dih9++AFjx46V79egQYOynAJVoWr9GMvIyAg///wzHj9+jIyMDDx48ADfffcddHR05PtIJBIsXLgQcXFxyMzMxMmTJ1G3bt0qrDURvQ7XHifi1VnC8oTAo2fpVVMhem2ys7OxdetWfPrpp5BIJPL0bdu2wcLCAo0aNYK3tzfS05W/FuLj43H48GGFoObatWuIjo6GhoYGmjdvDltbW/Tp04ctO2+gat2yQ0RUlPTsXKw8XXhZGE0J4GxR8qMLevPt27cPSUlJ+OSTT+RpI0eOhJOTE+zs7HDz5k3MmTMHISEh2LNnj1LH3Lx5M4yMjDBkyBB52sOHDwEA33zzDZYvXw5nZ2f89NNP6Nq1K0JDQ2FmZlah50WVh8EOEb1R8mQCU3ZcR0hcKgx1NJGRkwfZvy08ozu48BHWW2DDhg3o06cP7Ozs5Gnjx4+X/7tx48awtbVFjx498ODBA9SpU6fUY27cuBGjRo1S6AJRMBjmyy+/lPfj8fX1hb29PXbv3o0JEyZU1ClRJavWj7GIiF7l8889nLwXDx0tDWwZ2xYX53ZH30b5fS9uRiWhGq+AQxXg8ePHOHnyJD777LMS92vbti0AIDw8vNRjnj9/HiEhIYWOWTClyct9dHR1dVG7dm1ERkaqWnWqQgx2iOiNsfXyY/zfhQgAwE/DmqKlkylsTfSxYEBD6GhqIOhxIgIiXlRxLaky+fr6wsrKCv369Stxv+vXrwOAwhxsxdmwYQNatmyJpk2bKqS3bNkSurq6CAkJkafl5OTg0aNHcHJyUr3yVGUY7BBRmZU298mECRNQp04d6Ovrw9LSEgMHDsT9+/eVPv7EiRMhkUjw888/wy/0KRYcuAMA+Li+Nv7v6//BwsICxsbGGNq3J1rrxgAAfjtT+p08vZlkMhl8fX0xevRoaGn91wvjwYMHWLRoEa5evYpHjx7hwIED+Pjjj9G5c2c0adJEvp+7uzv27t2rcMzk5GTs3r27yJYiY2NjTJw4ERMmTJBf2zo6OkhISMCECRMq5Tp/WWhoKAYOHCi/zjt27IgzZ84ofVz6D4MdIiqzwMBAxMbGyl8nTpwAAPncJy1btoSvry/u3buHY8eOQQgBT09PhflSirN3715cvnwZdnZ2SEjOhNe2a8iTCQxtYY+d309Gbm4uTp8+jatXr6Jp06bYu3QKRHoizoc9Q3BkYqWeN1WNkydPIjIyUj7XWgEdHR2cPHkSnp6ecHd3x8yZMzF06FAcPHhQYb+QkBBIpVKFtJ07d0IIgQ8++KDIMn/44QeMHz8e5ubmMDQ0RKdOnbB+/XoAFX+dv+rdd98tdJ2/++67Ja79SMUQJKRSqQAgpFJpVVeF6I02depUUadOHSGTyYrcfuPGDQFAhIeHl3icJ0+eiFq1aonbt28LB0dH4dxvknCac0gMX3NJRMfGCwDi3Llz8v2Tk5MFADH0yzXCac4hMXbTlQo9L8rn5OQkABR6TZo0SQghxPjx40Xt2rWFnp6esLCwEAMGDBD37t0r8Zh///23eOedd4SZmZkAIIKDgwvts3btWtGlSxdhZGQkAIjExMQyn0NMUrq4GP5UxCSll/kYlXGdOzk5iRUrVsi3PX36tNjr/MSJE2WuuyrehM9b2d9vtuwQUYUobu6TAmlpafD19YWLiwscHByKPY5MJsNHH32EWbNmobabO56lZkOakQMXC0Os/aglbK0tUa9ePWzZsgVpaWnIzc3F2rVrYWVlhXmj+0IiAU7eS8DdmOTKPN23UmW05KWlpaFjx45YunRpsfukp6ejd+/emDdvXrnqvyswEh2WnMbI9QHosOQ0dgWq3sm4Mq7zhg0bFtpubm6OevXqoW/fvvJHaMbGxgCAd955B15eXnjx4gU+//xz1KtXD/r6+nB0dMSUKVMKtV69SpllMgYMGICcnBzo6OjA0tISQ4cOxY4dOwC8OZ+3glICu7cCW3aIym/Xrl1CU1NTREdHK6T/9ttvwtDQUAAQ9erVK/Vud/HixeKdd94Rubl5YuIfQULT2ErY9Z4gHj5Nle8TFRUlWrZsKSQSidDU1BS2trbi2rVrQgghvLZdFU5zDolJ265W/EmSgopq4RBCiIiIiGLv9AucOXOmzC07MUnpwnnuIeE0579X7bmHVW7hqejrvOC9e7VlR4j867xJkyYCgNDQ0BBWVlZi9erVAoA4c+aMuHXrlhgyZIg4cOCACA8PF6dOnRJubm5i6NChJZYdGxur8Nq4caOQSCTiwYMH8n2WL18u/P39xaNHj8TFixeFh4eHsLGxqXafN1t2iN4SJXUSLuud3zfffAN3d3cYGhrC1NQUPXv2REBAQInljhgxAm5uboX6HowaNQrBwcHw8/ND3bp1MXz4cGRmZhZZ7tWrV/HLL79g06ZN+OF4KI7cjoNEAgxv5QgXC0MAgBACXl5esLKywvnz53HlyhUMGjQI/fv3R2xsLLy6uQIA/rkViwdPU8v6tlIpKqqF43WJeJZWITNuFzXHD1D267yo9w747zqvVasWLly4gMDAQAwdOhRffPEFnJ2d0aVLFzRq1Ah///03+vfvjzp16qB79+74/vvvcfDgQeTm5hZ7DjY2NgqvV5fJAIDp06ejXbt2cHJyQvv27TFz5kzExcVh9OjRb8TnXUip4ddbgC079CZLSEhQuEs7ceJEue/8tm3bJk6cOCEePHggbt++LcaOHSuMjY1FQkKCfB8nJyexcOFCERsbK65cuSI0NDTEzp07SzxuVlaWMDAwENu3by9y+4oVK4REIhEampoCEo381793tU5OTkIIIU6ePCk0NDQK/X91dXUVPj4+Qgghxm4KFE5zDokZu66X9vZRGVVUC0eBym7ZiU5MU2jVKUvLzqNHj4SGhobYt29fifspe51ramrKX8pc51lZWUJDQ0N4enoWW/b69euFhYWF0ucUFxcntLS0xLZt24rd5/nz58LDw0MAqHafN1t2iN4SlpaWCndphw4dki+QWNY7v5EjR6Jnz56oXbs2GjZsiOXLlyM5ORk3b95U2M/IyAg2NjY4fPgwrKysFFaLLooQAkIIZGVlFbn9o48+wuZDfrAbsxK2Y1biizX7YWdnh1mzZuHYsWMAIF/vSEND8etLQ0NDPuPt5O75rTv7rkcj6gXXyqoMFdHC8TrFJRe+5ub2cVdpxm1l5/hR5jq/efMmrl+/Ln8pc53v27cPMpkMzZs3L/K4z549w6JFixRmky5NUctkFJgzZw4MDQ1hbm6Ou3fvwtPT8435vF/FYIdIjZT2aAEApFIpjI2NFeYpKe2Y69atg4mJSaFJ15YsWQIzMzMsXrwY7u7uCtsePnwIHx8fXL16FZGRkbh06RKGDRsGfX199O3bV77fy3OfJObpYNmVdGiaO2HYO+2xbFw/aGtrw8bGBvXq1QMAeHh4wNTUFKNHj8aNGzcQGhqKWbNmISIiQv4j1MyhJjq5WSBPJrDG74Fybx4praRZjE1MTODm5obOnTvjr7/+wv379wvNbVMVDt2IBQB4NrBGPesaAIC07OID/lcVN8dPWa5zc3NzNGrUSOGlzHU+Z84cSCSSIofJJycno1+/fmjQoAG++eYbpc+rqGUyCsyaNQvBwcH4448/IJVK8fz580IzlFfXz/tVDHaI1EhRCyS+TJU7v0OHDqFGjRrQ09PDihUrcOLECVhYWMi3T5kyBTt37sTixYuRk5ODq1evYvbs2fLtenp6OH/+PPr27QtXV1eMGDECRkZGuHTpEqysrOT7Fcx98iw1C2M2BSIlMxetnEyxdGiTIgM2CwsLHD16FKmpqejevTtatWqFCxcuYP/+/QrB2OR/++7sDnqCOGn1u9N8k1VUC8frIpMJ/HMrP9gZ1soBk/69NnZeiUJunkypYxQ3x4+q17myXr3OW7RogUePHuGrr74qdNORkpKC3r17w8jICHv37oW2trZSZRS3TMbLdahbty7Cw8NhZWWFq1ev4vLly8Uer7p83kVS6uGammOfHVIXnp6e4t133y1ym1QqFW3atBG9e/cW2dnZpR4rNTVVhIWFCX9/f/Hpp58KZ2dnER8fX+z+GzZsEFpaWiIzM1OlOsckpYsz9+NFv1/OCac5h0SnpafFsxTVjlGc91ZfFE5zDomFB+9UyPFIiLy8POHo6CjmzJmjkP7gwQOxePFiERQUJB4/fiwuXrwo+vfvL8zMzBSum3r16ok9e/bI/37+/LkIDg4Whw8fFgDEzp07RXBwsIiNjZXvExsbK4KDg8X69evlc88EBweL58+fK1XnwIjnwmnOIdHo66MiIztXZObkihYLjwunOYfEkVuxpR+ggpRnjp8FCxYIGxsbkZOTo5AulUpFu3btRJcuXURaWppKxxw9erRo2bJlifsUfN7/+9//5H0Bhag+n7eyv98MdgSDHVIPJXWeTE5OFh4eHqJHjx4iIyOjTMd3dXUVixcvLnb77du3BQBx//59pY+588pj4fLScOB6X/4jwuJTylS/opwNScg/7lf/VFgA9bY7duyYACBCQkIU0qOjo0WfPn2ElZWV0NbWFvb29mLkyJGFrgcAwtfXV/63r69vkRPXLViwQL7PggULitzn5eOUZMH+28JpziExfWewPG3JkXvCac4h8eH/XVb1LSiTl691l7mHxM4rj5XOW1yAKZVKRdu2bUXjxo1FeHi4wkCF3Nxc+X6vBhwFeQ0MDMTq1asLlXf58mXx66+/iuDgYLFlyxYBQDRv3lzUqVNHfjNTXT5vBjsqYLBD6qAy7vxeVrt2bYUvpFdt3bpVaGhoiBcvXih1vIinqcL5ldExLnMPlWtm21fJZDLR/9fzwmnOIbH0SMkzu74pSpvVNiMjQ0yaNEmYmZkJQ0NDMWTIEBEXF6f08SdMmCAAFJrzpahyC0a/qaoiZjFWVl6eTLT+7oRwmnNInLz73/sQ+TxNPu/Oy3M4VYaYpHSFoF7VkWDFBZgFo5WKekVERMj3KypQWLt2rdDX1xdJSUmFyrt586bo1q2bMDMzE7q6usLZ2Vl8/OlnYt+Fm2X6zCrz81b291u5HopEVK3JZDL834aN6DlgGJ6m5cDWJP+/dnJyMjw9PZGeno6tW7ciOTkZycn5MwtbWlpCU1MTQH7nSR8fHwwePBhpaWn4/vvvMWDAANja2uLZs2f47bffEB0dLZ851d/fHwEBAejWrRuMjIzg7++P6dOn48MPP4SpqWkJ9RQIiHiBfcHROHAjGuLV7QJ49CxdpREyJZFIJPDq5ooJf1zFFv/HmNC5DkwMlOvPUF0FBgYqzFB7+/ZtvPPOO/LPZvr06Th8+DB2794NExMTTJ48GUOGDMHFixdLPXZJ6zQBwMKFCzFu3Dj530ZGRirXf3vAY3y59zYEAA0J4DOkMUa0dlT5OMoKepyIhJQsGOlpoaPbf33OHMwM0K2eFU7fT8C2y4/x1bsNKq0OEc/SICtmjh9lrnVPT89CHYMBoGvXrkWmv0oIgVhpBi49eAYXC0PYmuhj/PjxRfbdE0LAvnY9rN2xH09Ts/AsNRsn7sTh0M1Y+B2MhORgJLrWs0QDO+NSywWAuzHJOBvy9LV93sVhsEOkBr76bTuin0ThtGiEDktOy79Qrl27Jp8M0NXVVSFPREQEnJ2dAeR3ngwKe4J20gyY6mri/v372Lx5M549ewZzc3O0bt0a58+fl09rr6uri507d+Kbb75BVlYWXFxcMH36dMyYMaPI+oXGp2BvcDT2B0cjpoTOwpoSCZwtDJQ+7+joaMyZMwdHjhxBeno6XF1d4evri1atWgEA4uPjsX3ZXMTtO4Ts9FS0+6cNDmzfADc3t2KP2bVrV/j5+RVK79u3Lw4fPgwg/wdh+vTpWLN2LbIyM6GhoQFXV1ds27ZNoew5c+bg+PHjSEpKQufOnfHrr7+Wu2wLCwssWLAA69evR1JSEiwtLeHo6IguXbpAKpViw4YN2L59O7p37w4gvzNx/fr1cfnyZbRr167E9/Lzzz/HsWPHiu14XDDVQFnFSjPkgQ6QH9zO23MbnetaVliA+6pDN2MAAJ4NbKCrpamw7cN2jjh9PwG7rz7BF73qQU9bs6hDlJuFoW6hNA0JVLrWy2NXYCS899yCTAASCTCilQNcrWrkBzMp2XiWmiV/PU/NRu6rkdlLBIAzIU9xJuSpyvV4HZ93cRjsEL3hYpLSsSPaFE5zDgHI/0KZ8/ctbLr0GLVqGmLOXzdgUUMXFjV0YGmkB4saOrAw0oVpDV0IIfBnUBRc5h7CthfAjn8DpT179pRYZosWLUoclQEA8cmZOHA9BnuDo3E39r91qoz0tNCvsS0GNa+FiGdp+GrvbeQJAU2JBIuHNFL6SzAxMREdOnRAt27dcOTIEVhaWiIsLEzesiSEwKBBg6CtrY3vf9+C5WejkBB8AD169MS9e3dhaGhY5HH37NmD7Oxs+d/Pnz9H06ZN5S0nQP4M0ytXroSuQ2OYN3kHabdOIv5ZDAwMDAqVvX//fhgbG2P58uXo2bMn7t4tX9nLli3DypUrsXnzZtjb26N9+/YwNDREVlYWrl69ipycHPTs2VO+v7u7OxwdHeHv719ssFPaOk0FlixZgkWLFsHR0REjR47E9OnTlZ7CAADO3E8o1JqnSguHqvJkAv/cyl8h/N2mtoW2d6lrBXtTfTxJzMDBGzEY1qpyZv7dez26UJqbVQ3YGBce7l3RYqUZ8kAHAIQAdgZGlZrPWE8LFka60NHUwP24lELbezeyLvUzi03KwNE78Qpplfl5l4TBDlEFUKaFQdW7/D179mDx4sUIDw9HTk4O3NzcMHPmTHz00UcAgJycHMz1nocNO/Yg+Wk0NHQNoefUFDW7fAItI3Pci03GvdiSF8PU1pAg56W7OJkA5v59C7FJmahtVSM/QKqhC4sauqhpoF3s3D2x0gxEPEuDlZEebkQlYd/1aFwMfyb/gtXWlKBrPSsMbl4L3d2t5HfQ7Wqbo2s9Szx6lg5nCwOVvgCXLl0KBwcH+Pr6ytNcXFzk/w4LC8Ply5dx+/ZtuNdvgEOxfnhoNhGJ6z7Bjh07ih1ua2ZmpvD3zp07YWBgIA84hBD48ccfoWlsBesPFgMADFzb4MmvH8LvchAaNGigUHZB8LB69WrY2NiUu+yff/4ZX331FQYOHIg///wTeXl5SEtLk084p6Ojg5o1ayocx9raGnFxcSW+l1paWpgyZUqx+0yZMgUtWrSAmZkZLl26BG9vb8TGxmL58uXF5nlZTp4Mmy49KpSuamueKq5EvMCz1CyY6GujQx2LQts1NSQY2dYRy46GYGtAZKUEOwnJmfC9GAEAWDq0CTQ1AO+/byEkPhV7g6MxpIV9hZf5socJhR+hAYBHHXO42xjBooZu/v9xI51/b4p0YV5DR94KFivNQIclpxWOoSmRYEH/hqUHO9IMHL8bXyjv62rRehmDHaJyUqWFQZW7fDMzM3z55Zdwd3eHjo4ODh06hDFjxsDKygq9evVCeMxzbD54FtqthsHWygWyzFS8OLUOT/csQq1PfobP4MbIFaJQM/Wz1Gw8S8lCSlauQqBTQAD4+VRYoXQtDQnMa+R/IVoa6cq/GKNepOOfW7GF7tgBoKWTKQY3r4V+jW1haqhT5HnamuiX6S7vwIED6NWrF4YNGwY/Pz/UqlULkyZNkvcpKZjrQ09PD5oaEvyvax3M/usmMmWa8Dt3vtiA41UbNmzA+++/L/+cHj58iPT0dBg2aIOn+3yQGXUbmjXMoWlsiTPnLuB/n36sUHYBDQ0N6Orq4sKFC2UuOyIiAnFxcfKWm4JZjJOTk+Hv74+2bdsqddyXFazTdO3atWKDWQAKjyibNGkCHR0dTJgwAT4+PtDVLfyY5lXrzj1EaHwq9LU1kZWbJ/8BnOlZt9Lu8g/fyn+E1auhNXS0ip5WbngrB6w4EYobUUm49USKxvYmFVqHX0+HIzNHhhaONTG8lT0kEgnik7Pww7EQfHvwLjq6WcDKqPJaeC4+eFYoTVMiwfLhTZV6321N9OEzpDHm7VG9BbY8eSsagx2iclKlhUGVu/yuXbsq/D116lRs3rwZFy5cQE23Vpi07RZqDP4G5oY6eK+lPf7vfATwzkTEbZmBme3NMKJNyZ0AM3PycDdGiqFr/BUWSJQA6OZuhbSsXHlwJM3IQa5MID45C/FFTLv/qnGdXPBRO2c4mlfeHdzDhw+xevVqzJgxA/PmzUNgYCCmTJkCHR0djB49Wv74xtvbG2vXrkW/hpaYNX8fcpKf4npIhFJlXLlyBbdv38aGDRsAAJHP0+G14QwAIC3kIoxbD4K1x3BkxYbhxfHfccrvImKSMgqVbWhoiBUrVuDJkyeIjY0tU9kA5K0z1tbW8lmM9+zZg23btiEuLg42NjbIzs5GUlKSQutOfHx8sX1tzp8/j4SEBDg6/ne95OXlYebMmfj555/x6NGjIvO1bdsWubm5ePTokXzW3+KEJ6Tgl38D6O8GNUJ7V3NM+OMqbj6R4kVadol5yyo3T4ajt/Pfr35Niu5wDQAWNXTRt7Et9l+PwdbLj7H0vSYVVofHz9Ow40okAGB2b3d5MDm+c238cysWd2KSsWD/Haz+sGWFlfmyq48Tse7cQwD5fXWEQJkCjhGtHdG5btlaYMuTtyJxBmWicjpw4ABatWqFYcOGwcrKCs2bN8f69evl20u7y1eGEAKnTp1CSEgIsi3rYdT/BeB5WjYa2hnjwOcd4d23Pi7M7QbvHk6QSCT4qEvx/S4K6GlrooWTGZYMaQzNf7+ENSUSLBnaGBs/aY1dEzxwamZX3FjgidDv+sDfuzsOTO6AjZ+0wrKhTTCrVz30bmhd5LG7u1tXaqAD5PczadGiBRYvXozmzZtj/PjxGDduHNasWQMA0NbWxp49exAaGgozMzOYGNWASWIo9Gq3xJOkTGTnlj5z7oYNG9C4cWM0b9kKa/wewPNnP1yPTAIAWDi4wrzraOhY14Fxs97QqmmNpLgovLPcD1uvPMHuv/6Wl21gYIAzZ86gT58+hdb0Kq3sNm3aFLm9qFmMW7ZsCW1tbZw6dUqeFhISgsjISHh4eBR5HGXWaSrK9evXoaGhoTBLcFHyZAKz/7qJ7FwZutazxJAWtWBroo/pPesCAP4MikK6Css2KCv/EVY2ahpoo30d8xL3/bCdEwBg/41oSDNyKqwOK06EIlcm0LmuJdrV/q8O2poaWPZeE2hpSHDkdhyO3FIuAFaFND0HU3YEI1cm8G4TW1yc0w07xrXDhbndyjQaytZEHx51zMsUrJQnb0Vhyw5ROanawqDKXb5UKkWtWrWQlZUFTU1NdB/rjV0xpgAE+je1w7KhTaCvk/9s3VRXgs2/fI8PPvgAxsbKDQsFlLvz0tHSKPJxU1U+k7e1tUWDBorDhevXr4+///5b/nfLli1x/fp1SKVSZGdnw6imGSxqN0KuSR3sDX5S4pd+Wloadu7ciQnTvdH/1wvyTpqt67vgEICOzerh97nd5e+bxy59PElLRFp2Hr49eBdN7U2w7R8/2Bvmry9maWmJtm3byvtxlaSg7IULFyqkF7TOxMbGKkw1EB8fj2bNmsHExARjx47FjBkzYGZmBmNjY3z++efw8PBQ6Jz88lQD5ubmMDdXDAZeXaeprFMNAMDmS49wLTIJNXS1sHhwY3nrRpe6lnA2N8Cj5+nYGxyNUW2dSn1fVHHo3wCid0MbaGuWHGC2cjJFPWsjhMSnYM+1JxjTwaXE/ZVxLzYZ+2/kP0ab3atwy1dDOxNM7FIHq86EY/7+O2hX27zYR72qEkJg9t83EJ2UAUczA/gMaQwjPW3Y1Xz9fWWqC7bsEJWTqi0MqtzlGxkZ4fr16zh29gLq9huHoxt+QGbkTczt446V7zeTBzo5OTkYPnw4hBBYvXq1yudQ1juvgmfyL7cMva5n8h06dEBISIhCWmhoKJycCv9ompiYwNLSElGPHiI9JhQGbm2x+uyDEtdF+mP7TqRlZGJnkhPux6WgpoE2fnivCfZ7D4aenh5u3Lghf98MJTl4EvUYtZ0c8N2gRjDS1cKNJ1IMWHURv1+MgaGJKcLCwhAUFISBAweWem67d+9GVlYWPvzwQ4V0FxcX2NjYYNaiH+VTDXh8ewj+lwPkLTcrVqzAu+++i6FDh6Jz586wsbEpNLouJCQEj2Of4tKDZ4iVZpRan4KpBrp06YKGDRvi+++/x/Tp07Fu3boS80U+T8cPx/I/o7l93GFX87/rQkNDgo88nAHkB0TKzBejrJcfYb1bwiOsAhKJBB965F83Wy8/rpC6/HgsBEIA/ZrYolGtovsBfd7DFa5WNfAsNQuLDt0td5kFtvg/xrE78dDWlGDVyOYw0nuz55aqEBU+neEbiDMoq4cnT56IUaNGCTMzM6GnpycaNWokAgMD5dtRzEyjy5YtK/aYubm54quvvhLOzs5CT09P1K5dWyxcuFDIZDL5Po6OjqJZs2bCxsZG6OnpiR49eohvvvlG2NnZFTpeUlKSSEhIEEII0aZNG/mstyW5EZUo2n5/UjjNOSRMm/cSrTp0VdienZ0tBg0aJJo0aSKePXtW6vEqQ0xSurgU/uy1zIhb4MqVK0JLS0t8//33IiwsTGzbtk0YGBiIrVu3yvf5888/xZkzZ8SDBw/Evn37hJOTkxgwaLBo9u0x4TTnkNgX/ER89NFHYu7cuQrHPnY7Vhg5NRIG7p3kywy8vNyEl5eXACA+/PBDcejQIdGyZUshkUjExo0bhRBCxEkzhOfnS4T1B4uF3YT/E+4fLRTWdg5iyJAhCuUUVbYQQnTs2FGMGDFCIS0vTyZuPUkSvcfMEBq6hsJyyHxh++kqoe/WTmiZWIsuPsfEqPWXxdQd18Sig3fE6rPh4q+gKHE2JEHcjk4S8dIMkZObJ4Qo39IFypLJZOKDdf7Cac4hMWLtJZGXJyu0jzQjW9Sff0Q4zTkkLoY9rbCyz4XmLxPSfOFx+TmXJiUzRzQoqEt4+epSsBZXbe/D4kFCycufXH38Qj6T8+l7xa89p6xbT5KE27x/hNOcQ2LD+YflPl51xxmU6a1S2ogoAIUeGR05cgRjx47F0KFDiz3u0qVLsXr1amzevBkNGzZEUFAQxowZAxMTE/kwXVNTU9y6dQt///03XFxcMH/+fCxfvhz169cvdDwTk/w7vIK7/EWLFpV4XnuuPcHcPbeQnStDHUtDmLlb4mnMf3NkFLTohIWF4cyZM4UeR7wuZR1RVR6tW7fG3r174e3tjYULF8LFxQU///wzRo0aJd8nNjYWM2bMQHx8PGxtbfHxxx9j/vz5WHfhMX48HorfzoQjOzJS3sIWJ83EggO3cejcVaQ8vo1GY5fh97Ft0MnNUqHsX3/9Fc+ePcPOnTuxdetWGBgYYOHChRgzZgwAwNpYD/1c9RG8eykSniYg3tAUNRp2h82gGXiakgVLo/zRS5EvlV0gJCQEFy5cwPHjxxGdlIELYU9xPuwZLj14jhdp2RCW3WDUMgbPj/0KWWYa9OwbwGr4QjxKysGjpMIjb14mkQAmetpIeqlfSmVN9LYzMAqXHjyHnrYGlg5tAg2NwiO9jPW0MbSFPf64/BibLj1Ce9fCw8PL4vDNfx9hNbKBVimPsArU0NXCoOa1sC0gEtsuR6J9EUPVlSGEwLKj+a1Zw1vZo7ZljRL3b+Foik87uGDDhQjM23sLx6d3LnNLTGpWLiZvv4bsPBl61rfGmA7OZTqOOpIIUYFth2+o5ORkmJiYQCqVqtTXgaqPuXPn4uLFizh//rzSeQYNGoSUlBSFzpyvevfdd2Ftba0wImbo0KHQ19fH1q1bIYSApaUlEhMTsWjRIgwfPhxnz57FuHHj4OXlhVWrVgHIfyxRMNPtrVu3MHXqVLRs2VKhf8nHH3+MWrVqwcfHB7l5MvT5dCZuZppDy9QW7RyN0FLjERbM/xKrV6/GZ599hpycHLz33nu4du0aDh06BGvr/zoLm5mZQUenYp7/qyNpRg46LjmNlKxcTO9ZF0Nb1sLp+wlYdjQEqVm50NKQYFzn2pjS3U3+qLCs0rJy8dPxUGy6FAGZyJ+sbV7f+hjeygHxKZmIeJYmn8I/JTMHlx++yA9wwp/h4dM0hWMZ6miiuWNNXAx/rjDUX0MC/DS8GWQyoTjFQGoWnqbk//tFWlaR860U2DGuHTxK6cirrFhpBjyXn0NKVi6+6lcfn3WqXey+4Qkp6Ln8HDQkgN+sbnAwK1+/kpw8GVp/fxJJ6TnY/llblQKoe7HJ6PPLeWhpSHBpbndYlWHSvzMhCRjjGwgdLQ34zeqqVACZkZ2HXj+fQ+SLdIxq64jvBzdWuVwhBKbtuo7912NgZ6KHf6Z2Qk0D9f8OUPb3my07pBZKm3PlVfHx8Th8+DA2b95c4nHbt2+PdevWITQ0FHXr1sWNGzdw4cIF+URqEREReP78OX755ResX79e3sLg5uYmX3cKKL6F4WUFd/lJ6dn4fEcwroTFIv3+LiDtBc4YGiDW3R1bt27FiBEjAORPZHjgwAEAQLNmzRSOdebMmUJD1+k/JvraaO1shtMhCVhxMhQrTobKtzV3rAmfIY3hblMxNz6Gulr4un8DDGpuB+89t3AnJhlz99zCWr+HePwif8I3CQBHMwM8ScpA3ksRiYYEaOpQE51cLdCpriWaOdSEtqYGdgVGFpq7ZHDzWiXWI08mkJiejXuxyfh445VC0w04mVdMq44QAl/tvY2UrFw0c6hZamdfVysjdHKzwPmwZ9h6+TG8+xZuEVXFpQfPkZSeA4saOmjjYlZ6hpfUtzVGKydTBD1OxM7AKEzpUfykn0WRyf5r1fmkvbPSLWX6OppYMrQxRq4PwLaASLzbxE7lwHN30BPsvx4DTQ0JVn7Q/K0IdFTBlh2wZUcdFAzrnjFjBoYNG4bAwEBMnToVa9aswejRowvtv2zZMixZsgQxMTEKQ8JfJZPJMG/ePCxbtgyamprIy8vD999/D29vbwDApUuX0KFDB8TExMDW9r/p6IcPHw6JRIJdu3YpfQ6x0gycD3uGn0+GIiYpE/ramvhxWFP0a1J4mnsqn6JmhQWALzzr4n9dXaFZxCOXipCbJ4PvxUf46XgIMosZ+u5sboCObhbo6GoJjzrmMNEv+pFGrDSjzHOX5AdLt5D30vmPauuIhQMblfvc91+PxtSd16GjqYFDUzqirnXpi4WevBuPz7YEwURfG5e9e5SrNW32XzfwZ9ATfNjOEd8NUr2FZF9wNKbtug5bEz2cn91N6cdgAHDgRgym7AiGka4Wzs3upvLoqnl7b2F7QCSczA1wdGpnpd+HsPgU9F91AZk5MszqVQ9e3VxLz6Qm2LJDbxWZTIZWrVph8eL86fubN2+O27dvFxvsbNy4EaNGjSox0AGAP//8E9u2bcP27dvRsGFDXL9+HdOmTYOdnV2Rxy2rLZceYcGBO/JHE6YG2tj2WTulVxYm1RS1CjUAtHQyq7RABwC0NDUwrnNtWBnpYuqu64W2r3y/GQY0K7mFpkB5+km9PN1A0OMXWH4iFNsCIvEsNQu/vN+8zAtiPkvNwjcH7gAAPu/uqlSgA+RPYuloZoDIF+nYdz0aH5QyIWZxsnNlOPbvWkzKjMIqSp/GNlh4SAex0kycvp8Az4bKLXyakyfDT8fzW3XGda5dpmHk3n3cceZ+Ah4/T8dPx0OUWok9IzsPXtuvITNHhk5uFvhflzoql/s24NBzUgvFzbkSGRlZaN/z588jJCREqSn7Z82ahblz5+L9999H48aN8dFHH2H69Onw8fEB8N+8J/HxiovdlTRjbYHcPBn8Qp9iwpYgfP1SoAPk9ykxNeRw0criYmGIV2Oa17lmT5vaZkWW31rFxy7lUTBs/vPubvhtZAvoaGrg2J14fLzhCqTpZZtYb8GBO0hMz0F9W2NM7Kr8j66mhgQf/zv0uzzD0C+GP4M0IweWRrpo7Vy291JXSxPD/10j64/Lj5XO92dQFB4/T4e5oQ7GdizbPD1GetpY/G9/nY0XI3AtMrHUPAsP3UFofCosauhi+fBmRXYEJwY7pCZUmXNlw4YNaNmyJZo2bVrqcdPT0wuNltHU1IRMlv8IomDek5c7OScnJyMgIKDIGWuFELj1RIqFB++inc9pjN54BcfuxhfaTyaAR8/SS60flU1Vzg9UHcp/Vd/Gttj8aRsY6WrhyqMXGL7WX6n5d1527E4cDt+MhaaGBD+816TUifxeNayVA/S1NXE/LgWXH75QKW+BQ/+OwurbyKZcLXSj2jpCIgHOhz3Do2dppe6fkZ2Hlf8uhzG5uysMdcv+0KSbe/6CuTIBzP7rJrJy84rd98CNGOy4EgWJBPjl/WbyUX5UGB9jkVqYPn062rdvj8WLF2P48OG4cuUK1q1bV2jSs+TkZOzevRs//fRTkcfp0aMHBg8ejMmTJwMA+vfvj++//x6Ojo5o2LAhgoODsXz5cnz66acA8icjmzZtGr777ju4ubnJh57b2dlh0KBB8uNGvUjHgRsx2BscjfCEVHm6qYE2urtbYU9wtEKH0apaGfhtUtVr9lR1+a/yqGOOPyd6YPTGKwiJT8HQ3y9hy9g2cLUq/VGUND0HX+27DQCY0Ll2sZPolcREXxtDWuQP/d586ZHKHXSzcvNw/G7pa2Epw8HMAF3qWuJsyFNsvxKJeaV0mt7s/wjxyVmoVVMfI9uW7RHcy75+twHOhz1FeEIqVp0Ox0zPwjMwP3qWhnl7bgEAJndzRYcKGravrhjskFpQZs4VANi5cyeEEPjggw+KPE5oWDiCQyMRK82ArYk+fv31V8yfPx+TJk1CQkIC7OzsMGHCBHz99dfyPLNnz0ZaWhrGjx+PpKQkNG7ZFlt270OWTBN7r0Ri77VoXHn0352qrpYG3mlgjcHNa6FzXUtoa2qgjYtZtVgZ+G1TFfMDVafyX1Xf1hh7JrXHxxuv4OHTNAxd7Y+Nn7RCS6eSHwl9d/gunqZkoY6locojmF42ur0ztgVE4vjdOEQnZaBWTeXfmwthz5CSmQsrI120cip5CQtlfNTOCWdDnuLPoCjMeKdusf2YpBk5WH32AQBg+jt1oatVvqkKAMDUUAcLBzbCpG3XsPrsA/RpZKvQfy8rNw+Td1xDalYu2jibYWo53vO3BUdjgaOxKN+uwEh477kFmcgf8uszpLFKC+a9nB/Ib53J+/e/l0QCeNQ2x6DmtdC7kQ2Mi5g0rDyja4gq0ou0bHy6KRDXo5Kgq6WBVSNb4J0GRS/66hf6FKM3XoFEAvw10aPUwKg0I9dfxqUHzzGxSx3M7eOudL4Zu65jT3A0PmnvjG8GlL4QbmnyZAKdl51BdFIGfhrWFENb2he53w/H7uO3Mw/gZlUDR6d1rtAO7hP/uIqjd+LQ0M4Y+7w6yB8NfnvwDnwvPoKpgTb+mdrprf6+4GgsIiVl5uTh6O04zP37lryTsEwAc/6+BZ9/7ivV4U8mEwqz0gJAnhCoY2mI4a0cMKCZXalfSNXtLp/eXmaGOtg+ri28tl3DmZCnmPBHEBYPboz3XxkllZqVK3+UMtrDudyBDpA/P82lB8+xMzAS03q6KTUyLDMnDyf+7fvWv2nFTNWgqSHByLaO+OFYCLYGPC4y2ElIycTGC48AAF/0qlfhI/kWDmoI/4fPcScmGevOPYRXN1ccvxMH34v5Zf40vCm/M5TEYIfeOjKZwN3YZJwPe4YL4U8R+CgR2cXMefJqAKOq7wY1rrBZaYleJwMdLaz7uBW899zCX1fzly15mpKFyd1d5SuXLzt6H9FJGXAw08fs3oX7lZRFj/rWsDfVx5PEDBy4HoPhrR1KzXMu9ClSsnJha6KH5g7lf4RVYHgrB/x8MhTBkUm4HS0t1Bfpt9PhyMjJQzOHmvAspuWrPKyM9PD1uw0wc/cN/HwyFJoaEvx2JhwA8FlHF3R3r/gy1RWDHVIrsdIMhen3C8QkZeBC2DOcC3sqX2PoZVZGukhIyVJI05AAW8e2VWqEw9OULHy4IUBh7hZ2MqY3nbamBn54rwmsjXXx25kH+OlEKOJTMjGxSx0cvxOPLf75Q7OXDGkCA52K+TnR1JDgo3ZO8DlyH76XHmFYK3t5cFWcw7f+HYXV2LZCh15bGumidyNbHLwRg20Bj+EzpIl8W9SLdGy/kj+1xeze9UqtY1kNaVELa889QGh8KpYcuQ8AsDfVx+zeyj/iIwY7pEZe7XPzYTsnSIBi1xjyqGOOjq4W6OhmiTqWhvgzKKpQJ2Fl19VxszaCz5DG7GRMakcikWBWL3dY1tDFt4fuYuvlSGy9/N/8Va2dTSt8JNCI1g5YcTIU92KTEfgoscRlHzJz8nDy30dYlTHb+IdtHXHwRgz2BcfAu299eX+7FSdCkZMn0MnNosyLhiojLjlTYQQnkH/z9jwti98vKmCwQ2ohVpqh0DlYJiC/6wSKX2PoZeUdClzdhhITVaRPOrhAU1OC+fvuKKRffZwoH71YUWoa6GBw81rYcSUKmy89KjHYORvyFGnZeahVUx/NHWpWWB0KtHExQ13rGgiNT8Xea9EY3d4Z9+OSsfd6NABgVq+KeXxXnKJm+y6Yh4vfMcpjsENqobjp/3vWt8J7LR1KXGPoZeXtJMxOxqTO6ljWKJRWWT+8o9s7Y8eVKBy9E4eYpAzYFTMM/b9HWDaV8ihJIpHgw3ZO+Hr/HWy9/Bgfezjhx2OhECK/zCb2NSu8zJcVzPbNR+TlwxmUSS24WBji1a85TYkEiwY1Qu9GNkoFOkRUste5zIa7jTHa1TZDnkxgW0DRyzZkZOfh1L3yrYWljMHNa8FARxNhCalY7fcAJ+/FQ0MCzHinclt1gOo32/abisEOqYVnKdl4OdrhFwJRxXvdP7yftHcGAOy4EoXMnMLLJpwNSUB6dh7sTfXRxF71WZuVZaSnjUHN8xdoXXY0f1maYS0d4GpVuKWrMoxo7YgLc7thx7h2uDC3m0rzf1E+PsaiClfciKjKkp0rw6y/bkAIoIe7FT7rVJt9Zogqyevsm9azvjXsTPQQI83EwRsxGNZKcRh6wVpY/ZrYVtpoqAKWNRRHZda2NKzU8l7FR+Tlw5YdNRQdHY0PP/wQ5ubm0NfXR+PGjREUFAQAyMnJwZw5c9C4cWMYGhrCzs4OH3/8MWJiYko85rlz59C/f3/Y2dlBIpFg3759CtsLjuvo6o5alqbo3Kwe6nboh9X/BFbWacqt8XuA+3EpMDXQxrL3msCjjjm/FIgqUcGK6ZX9/0xLUwMfeTgDADa9shp6enYuTt3/9xFW48p7hAXk38D9ejpMIW3Z0RCVF0ulqsNgR80kJiaiQ4cO0NbWxpEjR3D37l389NNPMDXNn2grPT0d165dw/z583Ht2jXs2bMHISEhGDBgQInHTUtLQ9OmTfHbb78VuT09PR3+V4KQ2WgQbEf/AstB85D9IhrTxo6s1C+E0PgU+ZfQNwMawrwGV/0lUifvt3aArpYG7sQk4+rjRHn66fsJyMyRwdHMAI1qVe4yP0UNgMgTAo+epVdquVRx+BhLzSxduhQODg7w9fWVp7m4uMj/bWJighMnTijkWbVqFdq0aYPIyEg4Ohb9LLhPnz7o06dPseWamJjggwVrsfTf59kAYPbORMRtmYHLN0MxuFPTsp5SsfJkArP+uomcPIGe9a0woGnl3t0R0etnaqiDgc3s8GfQE2y69AitnPOHoR9+jY+wOCLqzceWHTVz4MABtGrVCsOGDYOVlRWaN2+O9evXl5hHKpVCIpGgZs2aZSozKzcPCw/eVQh0AECWlQ5AAoMalXPXtfFCBG5EJcFIVwvfDWpc6V94RFQ1Rv/bUfno7TjESTORlpWL0/cTAADvVsJEgq/iiKg3H1t21MzDhw+xevVqzJgxA/PmzUNgYCCmTJkCHR0djB49utD+mZmZmDNnDj744IMyrfgenpCCz3dcx73YZABA+zrmuPzwOfJyspF01hcGDTpj4bEItHC1U2rZBWU9epaGH4/nB1df9qsPGxO9Cjs2EVUvDe1M0MbZDFcevcC2gMdwszZCVq4MLhaGaGBbuY+wCnDS0Dcbgx01I5PJ0KpVKyxevBgA0Lx5c9y+fRtr1qwpFOzk5ORg+PDhEEJg9erVKpUjhMD2gEgsPHQHmTkymBvq4IdhTdDd3RqRz5IxasRwGFoawnzELEQlZeCzLUHYNb6dUisYl36OAnP+vomsXBk6uJpjhBILBRLRm210e2dcefQC2wMi5cPM+zWu/EdYL+OIqDcXH2OpGVtbWzRo0EAhrX79+oiMjFRIKwh0Hj9+jBMnTqjcqrPG7yHm7b2FzBwZOrlZ4MjUTujubo2cnBxMHTcayc9icf7saWyZ2BU1DbRxIyoJM/68DllR0xyraPuVSAREvIC+tiaWDGnCx1dEbwHPhtawNdHD87RsnAl5CqBy1sIi9cRgR8106NABISGKfWdCQ0Ph5OQk/7sg0AkLC8PJkydhbm6u9PH9HzwHAARHJkJbU4Iv+9bH5jFtYGWsh5ycHPTv3x9nzpxBVFQU7O3tMaBbO0xrogFtTQn+uRWHZcfu4+uvv4atrS309fXRs2dPhIWFlVimj48PWrduDSMjI1hYWmHS6A+Q8/wJZveuBwez/A6CEyZMQJ06daCvrw9LS0sMHDgQ9+/fV/q8iKh609bUKDRx4I2opKqpDL1xGOyomenTp+Py5ctYvHgxwsPDsX37dqxbtw5eXl4A8gOd9957D0FBQdi2bRvy8vIQFxeHuLg4ZGdny4/To0cPrFq1Sv53ojQZ03/fh6HfbwcAmOQmwqeLEXo5a0FDQ4KcnBwMGDAAJ0+eRJcuXbBt2zb4+fnhyy+/RNt6tbBkSBMAwNKly/DTil+wZs0aBAQEwNDQEL169UJmZmax5+Tn5wcvLy/4+/vDY/Jy5ObmIHHPArzXxFK+T8uWLeHr64t79+7h2LFjEELA09MTeXmFZ10lojdPrDQDJ/5d3bzAl3tvc64bUo4gIZVKBQAhlUqruioV4uDBg6JRo0ZCV1dXuLu7i3Xr1sm3RURECABFvs6cOSPfz8nJSSxYsEAIIcTjZ2miw5SVReYZPXq00sf98eg9oWloKsy7fyouhj0VQgiRlJQkdHV1xY4dO0o9r7+CooTTnEPCZdoOAUD4+fkVu++NGzcEABEeHq7am0dE1dLF8KfCac6hQq9L4c+qumpUhZT9/WYHZTX07rvv4t133y1ym7Ozs8IspMXxv3EPEc/SsOliBH48HopU/dpovOAofIY0KfI5ubOzM+rXr49evXrhyZMn8PPzQ61atTBp0iR07doVADDIVRtfpCVCx7EZJm69ij2TOsDVygRt27aFv78/3n///WLrk5CSiYWH7gIAPm5liW8BmJmZFblvWloafH194eLiAgcHdl4mUgec64bKg4+xqJBdgZHosOQ0Rq4PwDcH7yI1KxetnU1xZFrnEjsEFgx7d3Nzw7Fjx/C///0PU6ZMwebNmwEA8fH5TdDN6jkhOTMXYzZdwfPULFhbWyMuLq7EOn297w6kGTloaFsDgduXo0OHDmjUqJHCPr///jtq1KiBGjVq4MiRIzhx4gR0dHTK+W4QUXXAuW6oPCRCmdv8l5w5cwbdunWrrPpUieTkZJiYmEAqlZZprhl1IU3PweFbMZi397ZCugTAudld4WBW8sJ3Ojo6aNWqFS5duiRPmzJlCgIDA+Hv749Lly6hQ4cOuB0Wgf/tiUDki3S0dDIFTq2AlqYGdu3aVeRx/7kVi0nbrkFLQ4JWT/7G5XOncOHCBdjb2yvWXypFQkICYmNj8eOPPyI6OhoXL16Enh7n4CFSF7HSDM51Q3LK/n6r/Bird+/esLe3x5gxYzB69Gg+JniDZefKcC0yERfCnuF8+DPcepJUaP0XIL/jzZPEzFKDneKGvf/9998AABsbGwBATmoSNn7SGoN/v4irjxORc+chhr7TochjJqZl4+v9+cGX9e1tuBh8DufOnSsU6AD5S1aYmJjAzc0N7dq1g6mpKfbu3YsPPvigtLeCiN4QnOuGykLlx1jR0dGYPHky/vrrL9SuXRu9evXCn3/+qTCSh6perDQDlx48UxipIIRAaHwKNlyIwBjfK2i28DjeX3cZq86E40ZUfqDjbG6AV2etUfa5eGnD3l1cXGBjY4NTp07B1aoG1n7YEho5GYgJu4XEGs5FHnPhobt4mpKFvAv/h0dXz+L06dMKa30VRwgBIQSysrJK3ZeIiNSbyi07FhYWmD59OqZPn45r167B19cXkyZNwqRJkzBy5EiMHTsWTZtW/KKPpLxdgZHw3nMLMgFoSIDhrRyQkydwIfwp4pMVf/wtauigg6sFOrpaoKObBWxN9LErMBLz9txGnhAqPRefPn062rdvj8WLF2P48OG4cuUK1q1bh3Xr1gEAJBIJpk2bhu+++w5ubm5wcXGBedB6RNUww7lsF/x99QmGtrRHjx49MHjwYDTsORx7g6OReGI1xIMLOHTgAIyMjOT9e0xMTKCvr4+HDx9i165d8PT0hKWlJZ48eYIlS5ZAX18fffv2rfg3mIiI3igq99l5VUxMDNatW4clS5ZAS0sLmZmZ8PDwwJo1a9CwYcOKqmelUqc+O7HSDHRYcrrIx1EAoKulgTYuZujoaoFObpZwtzGChkbhGYjL+lz80KFD8Pb2RlhYGFxcXDBjxgyMGzdOvl0IgQULFmDdunVISkpCx44d0ez9mfgrXAZtTQm2jm2LEd1b4oMPP8JZw66IlWbi8dKiR5b5+vrik08+QUxMDD777DNcvXoViYmJsLa2RufOnfH111+jXr16StediIjeLMr+fpcp2MnJycH+/fuxceNGnDhxAq1atcLYsWPxwQcf4OnTp/jqq69w7do13L17t1wnAeQ/NpszZw6OHDmC9PR0uLq6wtfXF61atQLw34/n+vXrkZSUhA4dOshHBClLnYKdSw+eYeT6gELp/ZvaYkQrR7RyNq2Q9akqkkwmMHnHNfxzKw41DbSx7qOWWOP3EKfvJ8DJ3ABHp3aGvk71qjMREVW9Suug/Pnnn2PHjh0QQuCjjz7CsmXLFIYAGxoa4scff4SdnV3Zav6SxMREdOjQAd26dcORI0dgaWmJsLAwmJqayvdZtmwZVq5cic2bN8PFxQXz589Hr169cPfu3bdyFE5xc1HM61u/2nbq09CQYPnwZohOuowbUUkYvvayfFuvBjYMdIiIqFxUbtnp0aMHPvvsMwwZMgS6urpF7pObm4uLFy+iS5cu5arc3LlzcfHiRZw/f77I7UII2NnZYebMmfjiiy8A5A8/tra2xqZNm0qcpO5l6tSyAwDTdgZj3/UYAP/NRTGitWMV16p0t6OlePfXCwppmhIJLsztVm0DNSIiqjrK/n6rPBrr1KlT+OCDD4oNdABAS0ur3IEOABw4cACtWrXCsGHDYGVlhebNm2P9+vXy7REREYiLi0PPnj3laSYm/83IW5ysrCwkJycrvNRJSmYuAGBEawdcmNvtjQh0ACA5M6dQWp4QePQsvQpqQ0RE6kLlYMfHxwcbN24slL5x40YsXbq0QipVoLQZeQtG5VhbWyvkK21GXh8fH/mcLCYmJmo1V1BWbh4u/bsy+UftnN6oFpGCR3Av43TwRERUXioHO2vXroW7u3uh9IYNG2LNmjUVUqkCMpkMLVq0wOLFi9G8eXOMHz8e48aNK3c53t7ekEql8ldUVFQF1bjqBUYkIiMnD5ZGumho92Y9kuN08EREVBlU7qAcFxcHW9vC6yNZWloiNja2QipVQNkZeePj4xXqFB8fj2bNmhV7XF1d3RIfw73JzoYkAAC61LWERFJ4SHl1N6K1IzrXteR08EREVGFUbtlxcHDAxYsXC6VfvHixQkZgvUyVGXkLJCcnIyAgAB4eHhValzfF2dCnAICu9SyruCZlZ2uiD4865gx0iIioQqjcsjNu3DhMmzYNOTk56N69O4D8TsuzZ8/GzJkzK7RyZZmRd/78+bCzs8OgQYMqtC5vgieJ6QhPSIWGBOjk+uYGO0RERBVJ5WBn1qxZeP78OSZNmiRfD0tPTw9z5syBt7d3hVaudevW2Lt3L7y9vbFw4UK4uLjg559/xqhRo+T7zJ49G2lpaRg/frx8Rt6jR4++lXPsnA3Jb9Vp4WgKEwPtKq4NERFR9VDm5SJSU1Nx79496Ovrw83N7Y3uA6Mu8+x8tjkIJ+/F4wvPupjcXfkZpImIiN5ElTaDcoEaNWqgdevWZc1OFSx/yPkzAEDXelZVXBsiIqLqo0zBTlBQEP78809ERkbKH2UV2LNnT4VUjFQT9CgR6dl5sKihiwa2b27rFBERUUVTeTTWzp070b59e9y7dw979+5FTk4O7ty5g9OnT8PExKQy6khKeHnIeVGrmBMREb2tVA52Fi9ejBUrVuDgwYPQ0dHBL7/8gvv372P48OFwdHwzliVQRwWdk7u8wUPOiYiIKoPKwc6DBw/Qr18/AICOjg7S0tIgkUgwffp0+ZBwer2ikzIQ9u+Q885uFlVdHSIiompF5WDH1NQUKSkpAIBatWrh9u3bAICkpCSkp3PBxqrg92+rTjOHmqhpoFPFtSEiIqpeVO6g3LlzZ5w4cQKNGzfGsGHDMHXqVJw+fRonTpxAjx49KqOOVIqC/jochUVERFSYysHOqlWrkJmZCQD48ssvoa2tjUuXLmHo0KH46quvKryCVLLsXBkuhhcMOWd/HSIiolepFOzk5ubi0KFD6NWrFwBAQ0MDc+fOrZSKkXKCHr9AWnYeLGrooJEdR8MRERG9SqU+O1paWpg4caK8ZYeqXkF/nc5uHHJORERUFJU7KLdp0wbXr1+vhKpQWXDIORERUclU7rMzadIkzJgxA1FRUWjZsiUMDQ0Vtjdp0qTCKkcli0nKQEh8yr9DzhnsEBERFUXlYOf9998HAEyZMkWeJpFIIISARCJBXl5exdWOSuQXmt+q09ShJkwNOeSciIioKCoHOxEREZVRDyqDl5eIICIioqKpHOw4OTlVRj1IRTl5MlwMfw6A8+sQERGVROVgZ8uWLSVu//jjj8tcGVLe1ceJSM3KhZmhDprU4pBzIiKi4qgc7EydOlXh75ycHKSnp0NHRwcGBgYMdl6Ts/Ih5xYcck5ERFQClYeeJyYmKrxSU1MREhKCjh07YseOHZVRRyoCl4ggIiJSjsrBTlHc3NywZMmSQq0+VDnipJm4H5cCiQTozM7JREREJaqQYAfIn105Jiamog5HJfALzW/VaWJfE2Ycck5ERFQilfvsHDhwQOFvIQRiY2OxatUqdOjQocIqRsUr6K/Tla06REREpVI52Bk0aJDC3xKJBJaWlujevTt++umniqoXFSMnT4YLYVzlnIiISFkqBzsymawy6kFKuvY4ESlZuTA10EYT+5pVXR0iIqJqr8L67NDrcfbfJSI6uVlCk0POiYiISqVysDN06FAsXbq0UPqyZcswbNiwCqkUFc+voL8OH2EREREpReVg59y5c+jbt2+h9D59+uDcuXMVUikqWkJyJu7GJgPgkHMiIiJlqRzspKamQken8HBnbW1tJCcnV0ilqGgFj7Ca2JvAooZuFdeGiIjozaBysNO4cWPs2rWrUPrOnTvRoEGDCqkUFc2PQ86JiIhUpvJorPnz52PIkCF48OABunfvDgA4deoUduzYgd27d1d4BSlfbp4M58Pyg50uXCKCiIhIaSoHO/3798e+ffuwePFi/PXXX9DX10eTJk1w8uRJdOnSpTLqSACCo5KQnJmLmgbaaOZQs6qrQ0RE9MZQOdgBgH79+qFfv34VXRcqQcHCnxxyTkREpBqV++wEBgYiICCgUHpAQACCgoIqpFLqZsmSJZBIJJg2bZpCur+/P7p37w5DQ0MYGxujc+fOyMjIKPIYZ0OeQur/J4589wmMjIxgZWWFQYMGISQkRGG/devWoWvXrjA2NoZEIkFSUlIlnRUREdGbQeVgx8vLC1FRUYXSo6Oj4eXlVSGVUieBgYFYu3YtmjRpopDu7++P3r17w9PTE1euXEFgYCAmT54MDY3CH0lCSibuxCQjM+o2pn4+GZcvX8aJEyeQk5MDT09PpKWlyfdNT09H7969MW/evEo/NyIiojeByo+x7t69ixYtWhRKb968Oe7evVshlVIXqampGDVqFNavX4/vvvtOYdv06dMxZcoUzJ07V55Wr169Io9TMAqrx/Sf8fnETvL0TZs2wcrKClevXkXnzp0BQN56dPbs2Qo8EyIiojeXyi07urq6iI+PL5QeGxsLLa0ydQFSW15eXujXrx969uypkJ6QkICAgABYWVmhffv2sLa2RpcuXXDhwoUij1Mwv07XuoqjsKRSKQDAzMysEmpPRESkHlQOdjw9PeHt7S3/oQWApKQkzJs3D++8806FVu5NtnPnTly7dg0+Pj6Ftj18+BAA8M0332DcuHE4evQoWrRogR49eiAsLExh39xiVjmXyWSYNm0aOnTogEaNGlXimRAREb3ZVG6K+fHHH9G5c2c4OTmhefPmAIDr16/D2toaf/zxR4VX8E0UFRWFqVOn4sSJE9DT0yu0vWDl+AkTJmDMmDEA8h8Dnjp1Chs3blQIkG48SYI0IwfGeloKQ869vLxw+/btYluDiIiIKJ/KwU6tWrVw8+ZNbNu2DTdu3IC+vj7GjBmDDz74ANra2pVRxzfO1atXkZCQoNC3KS8vD+fOncOqVavkI6henXG6fv36iIyMVEg7+29/nU51LaGlmd8QN3nyZBw6dAjnzp2Dvb19ZZ4KERHRG69MnWwMDQ0xfvz4iq6L2ujRowdu3bqlkDZmzBi4u7tjzpw5qF27Nuzs7AoNGw8NDUWfPn0U0s6+tESEEAKff/459u7di7Nnz8LFxaVyT4SIiEgNlLlH8d27dxEZGYns7GyF9AEDBpS7Um86IyOjQv1oDA0NYW5uLk+fNWsWFixYgKZNm6JZs2bYvHkz7t+/j7/++kuep3PXbritXQ/GLfujSz1LeHl5Yfv27di/fz+MjIwQFxcHADAxMYG+vj4AIC4uDnFxcQgPDwcA3Lp1C0ZGRnB0dGRHZiIieiupHOw8fPgQgwcPxq1btyCRSCCEAABIJPmz+ubl5VVsDdXUtGnTkJmZienTp+PFixdo2rQpTpw4gTp16sj3uR8aDlltWzS0M4aVkR5Wr14NAOjatavCsXx9ffHJJ58AANasWYNvv/1Wvq1gSPrL+xAREb1NJKIgWlFS//79oampif/7v/+Di4sLrly5gufPn2PmzJn48ccf0alTp9IPUs0kJyfDxMQEUqkUxsbGVV0duc82B+HkvXiM9nDCtwM54oqIiOhlyv5+qzz03N/fHwsXLoSFhQU0NDSgoaGBjh07wsfHB1OmTClXpek/O65E4uS9/PmMtlx+jF2BkaXkICIioqKoHOzk5eXByMgIAGBhYYGYmBgAgJOTU6EOt1Q2sdIMzNv7XwdnIYB5e24jVlr0ullERERUPJX77DRq1Ag3btyAi4sL2rZti2XLlkFHRwfr1q1D7dq1K6OOb52IZ2l49eFinhB49Cwdtib6VVMpIiKiN5TKwc5XX30lX3hy4cKFePfdd9GpUyeYm5tj165dFV7Bt5Hmv529X01ztjCogtoQERG92VQOdnr16iX/t6urK+7fv48XL17A1NRUPiKLymff9RiFvzUlEiwe0oitOkRERGVQISt3cv6WipOQkom/rz0BAKwe1QI1DXTgbGHAQIeIiKiMuEx5NbPp4iNk58rQwrEmejeyYWsZERFROak8GosqT2pWLv64/BgAMKFLHQY6REREFYDBTjWyIyASKZm5qG1piHfqW1d1dYiIiNSCysHOuXPnkJubWyg9NzcX586dq5BKvY2yc2XYcCECADChc21oaLBVh4iIqCKoHOx069YNL168KJQulUrRrVu3CqnU22j/9WjEJWfCykgXg5rXqurqEBERqQ2Vgx0hRJF9SZ4/fw5DQ8MKqdTbRiYTWHfuIQDg044u0NXSrOIaERERqQ+lR2MNGTIEQP7q5p988gl0dXXl2/Ly8nDz5k20b9++4mv4Fjh9PwFhCakw0tXCyLaOVV0dIiIitaJ0sGNiYgIgv2XHyMgI+vr/zfuio6ODdu3aYdy4cRVfw7fA2nMPAAAj2znCWE+7imtDRESkXpQOdnx9fQEAzs7O+OKLL/jIqoJcffwCgY8SoaOpgU87uFR1dYiIiNSOyn12Zs+erdBn5/Hjx/j5559x/PjxCq3Y22KNX35fncHNa8HaWK+Ka0NERKR+VA52Bg4ciC1btgAAkpKS0KZNG/z0008YOHAgVq9eXeEVVGfhCak4cTceEgkwrjNXjCciIqoMKgc7165dQ6dOnQAAf/31F2xsbPD48WNs2bIFK1eurPAKqrN1//bVeae+NVytalRxbYiIiNSTysFOeno6jIyMAADHjx/HkCFDoKGhgXbt2uHx48cVXkF1FZ+cib3B0QDyl4YgIiKiyqFysOPq6op9+/YhKioKx44dg6enJwAgISEBxsbGFV5BdbXxQgRy8gTaOJuhpZNpVVeHiIhIbakc7Hz99df44osv4OzsjDZt2sDDwwNAfitP8+bNK7yC6ig5MwfbAiIBABO6sK8OERFRZVI52HnvvfcQGRmJoKAgHDt2TJ7eo0cPrFixokIr96olS5ZAIpFg2rRp8rTMzEx4eXnB3NwcNWrUwNChQxEfH1+p9SivbZcjkZqVCzerGuhWz6qqq0NERKTWyrTquY2NDYyMjHDixAlkZGQAAFq3bg13d/cKrdzLAgMDsXbtWjRp0kQhffr06Th48CB2794NPz8/xMTEyGd7ro6ycvOw8eK/C352qcMFP4mIiCqZysHO8+fP0aNHD9StWxd9+/ZFbGwsAGDs2LGYOXNmhVcQAFJTUzFq1CisX78epqb/9W+RSqXYsGEDli9fju7du6Nly5bw9fXFpUuXcPny5UqpS3ntC47G05Qs2JroYUBTu6quDhERkdpTOdiZPn06tLW1ERkZCQMDA3n6iBEjcPTo0QqtXAEvLy/069cPPXv2VEi/evUqcnJyFNLd3d3h6OgIf3//Yo+XlZWF5ORkhdfrIJMJrP13wc+xHV2go1WmhjUiIiJSgdLLRRQ4fvw4jh07Bnt7e4V0Nze3Shl6vnPnTly7dg2BgYGFtsXFxUFHRwc1a9ZUSLe2tkZcXFyxx/Tx8cG3335b0VUt1Yl78Xj4NA3Gelp4vw0X/CQiInodVG5aSEtLU2jRKfDixQuFldArQlRUFKZOnYpt27ZBT6/illLw9vaGVCqVv6Kioirs2MURQmCNX/4kgh95OKGGrspxJhEREZWBysFOp06d5MtFAIBEIoFMJsOyZcvQrVu3Cq3c1atXkZCQgBYtWkBLSwtaWlrw8/PDypUroaWlBWtra2RnZyMpKUkhX3x8PGxsbIo9rq6uLoyNjRVelS3wUSKCI5Ogo6WBT9pzwU8iIqLXReXmhWXLlqFHjx4ICgpCdnY2Zs+ejTt37uDFixe4ePFihVauR48euHXrlkLamDFj4O7ujjlz5sDBwQHa2to4deoUhg4dCgAICQlBZGSkfP6f6qKgVee9lvawNKrYFjAiIiIqnsrBTqNGjRAaGopVq1bByMgIqampGDJkCLy8vGBra1uhlTMyMkKjRo0U0gwNDWFubi5PHzt2LGbMmAEzMzMYGxvj888/h4eHB9q1a1ehdSmPkLgUnL6fkL/gZydOIkhERPQ6qRzsREZGwsHBAV9++WWR2xwdX2/H2xUrVkBDQwNDhw5FVlYWevXqhd9///211qE0a/9d8LN3Qxu4WBhWcW2IiIjeLhIhhFAlg6amJmJjY2FlpTjz7/Pnz2FlZYW8vLwKreDrkJycDBMTE0il0grvvxOTlIHOy84gVyaw36sDmjrUrNDjExERva2U/f1WuYOyEAISSeFZf1NTUyt0xJS62HghArkygXa1zRjoEBERVQGlH2PNmDEDQP7oq/nz5ysMP8/Ly0NAQACaNWtW4RV8k4XGpWDr5fy5hyZ2qVPFtSEiIno7KR3sBAcHA8hv2bl16xZ0dHTk23R0dNC0aVN88cUXFV/DN9SuwEjM/fsWCp4Rxkkzq7Q+REREbyulg50zZ84AyB/6/csvv7yWuWneVLHSDHjv+S/QAYAv995Gl3qWsDXRr7J6ERERvY1U7rPj6+vLQKcUEc/SIHul23eeEHj0LL1qKkRERPQW40qUlcDFwhAar/Th1pRI4GxReJkNIiIiqlwMdiqBrYk+fIY0hua/o9Y0JRIsHtKIj7CIiIiqAFejrCQjWjuic11LPHqWDmcLAwY6REREVYTBTiWyNdFnkENERFTF+BiLiIiI1BqDHSIiIlJrDHaIiIhIrTHYISIiIrXGYIeIiIjUGoMdIiIiUmsMdoiIiEitMdghIiIitcZgh4iIiNQagx0iIiJSawx2iIiISK0x2CEiIiK1xmCHiIiI1BqDHSIiIlJrDHaIiIhIrTHYISIiIrXGYIeIiIjUGoMdIiIiUmsMdoiIiEitMdghIiIitcZgh4iIiNQagx0iIiJSawx2iIiISK0x2CEiIiK1xmCHiIiI1BqDHSIiIlJrDHaIiIhIrTHYISIiIrXGYIeIiIjUGoMdIiIiUmsMdoiIiEitMdghIiIitcZgh4iIiNQagx0iIiJSawx2iIiISK0x2CEiIiK1xmCHiIiI1BqDHSIiIlJrDHaIiIhIrTHYISIiIrXGYIeIiIjUGoMdIiIiUmsMdoiIiEitMdghIiIitcZgh4iIiNQagx0iIiJSawx2iIiISK0x2CEiIiK1xmCHiIiI1BqDHSIiIlJrDHaIiIhIrTHYISIiIrXGYIeIiIjUGoMdIiIiUmvVOtjx8fFB69atYWRkBCsrKwwaNAghISEK+2RmZsLLywvm5uaoUaMGhg4divj4+CqqMREREVU31TrY8fPzg5eXFy5fvowTJ04gJycHnp6eSEtLk+8zffp0HDx4ELt374afnx9iYmIwZMiQKqw1ERERVScSIYSo6koo6+nTp7CysoKfnx86d+4MqVQKS0tLbN++He+99x4A4P79+6hfvz78/f3Rrl07pY6bnJwMExMTSKVSGBsbV+YpEBERUQVR9ve7WrfsvEoqlQIAzMzMAABXr15FTk4OevbsKd/H3d0djo6O8Pf3L/Y4WVlZSE5OVngRERGRenpjgh2ZTIZp06ahQ4cOaNSoEQAgLi4OOjo6qFmzpsK+1tbWiIuLK/ZYPj4+MDExkb8cHBwqs+pERERUhd6YYMfLywu3b9/Gzp07y30sb29vSKVS+SsqKqoCakhERETVkVZVV0AZkydPxqFDh3Du3DnY29vL021sbJCdnY2kpCSF1p34+HjY2NgUezxdXV3o6upWZpWJiIiomqjWLTtCCEyePBl79+7F6dOn4eLiorC9ZcuW0NbWxqlTp+RpISEhiIyMhIeHx+uuLhEREVVD1bplx8vLC9u3b8f+/fthZGQk74djYmICfX19mJiYYOzYsZgxYwbMzMxgbGyMzz//HB4eHkqPxCIiIiL1Vq2HnkskkiLTfX198cknnwDIn1Rw5syZ2LFjB7KystCrVy/8/vvvJT7GehWHnhMREb15lP39rtbBzuvCYIeIiOjNo5bz7BARERGpisEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTW1CbY+e233+Ds7Aw9PT20bdsWV65cqeoqERERUTWgFsHOrl27MGPGDCxYsADXrl1D06ZN0atXLyQkJFR11YiIiKiKqUWws3z5cowbNw5jxoxBgwYNsGbNGhgYGGDjxo1VXTUiIiKqYlpVXYHyys7OxtWrV+Ht7S1P09DQQM+ePeHv719knqysLGRlZcn/lkqlAIDk5OTKrSwRERFVmILfbSFEifu98cHOs2fPkJeXB2tra4V0a2tr3L9/v8g8Pj4++PbbbwulOzg4VEodiYiIqPKkpKTAxMSk2O1vfLBTFt7e3pgxY4b8b5lMhhcvXsDc3BwSiaTCyklOToaDgwOioqJgbGz8WvOz7Ndfdnnzs+y3q+zy5mfZLPtNyV/esksihEBKSgrs7OxK3O+ND3YsLCygqamJ+Ph4hfT4+HjY2NgUmUdXVxe6uroKaTVr1qysKsLY2LhcH3B58rPs1192efOz7Ler7PLmZ9ks+03JX96yi1NSi06BN76Dso6ODlq2bIlTp07J02QyGU6dOgUPD48qrBkRERFVB298yw4AzJgxA6NHj0arVq3Qpk0b/Pzzz0hLS8OYMWOqumpERERUxdQi2BkxYgSePn2Kr7/+GnFxcWjWrBmOHj1aqNPy66arq4sFCxYUemT2OvKz7Ndfdnnzs+y3q+zy5mfZLPtNyV/esiuCRJQ2XouIiIjoDfbG99khIiIiKgmDHSIiIlJrDHaIiIhIrTHYISIiIrXGYKcS/fbbb3B2doaenh7atm2LK1euKJXv3Llz6N+/P+zs7CCRSLBv3z6ly/Tx8UHr1q1hZGQEKysrDBo0CCEhIUrnX716NZo0aSKf/MnDwwNHjhxROv/LlixZAolEgmnTpim1/zfffAOJRKLwcnd3V7q86OhofPjhhzA3N4e+vj4aN26MoKAgpfI6OzsXKlsikcDLy6vUvHl5eZg/fz5cXFygr6+POnXqYNGiRaWu1fKylJQUTJs2DU5OTtDX10f79u0RGBhYaL/Srg0hBL7++mvY2tpCX18fPXv2RFhYmNL59+zZA09PT/ls4tevX1e6/JycHMyZMweNGzeGoaEh7Ozs8PHHHyMmJkapsr/55hu4u7vD0NAQpqam6NmzJwICApSu+8smTpwIiUSCn3/+Wam8n3zySaHPvnfv3iqVfe/ePQwYMAAmJiYwNDRE69atERkZWWreoq47iUSCH374QamyU1NTMXnyZNjb20NfX1++GLIyeePj4/HJJ5/Azs4OBgYG6N27t/x6Uea7JDMzE15eXjA3N0eNGjUwdOhQ+QSvyuRft24dunbtCmNjY0gkEiQlJcm3lZb/xYsX+Pzzz1GvXj3o6+vD0dERU6ZMgVQqVarsCRMmoE6dOtDX14elpSUGDhwoX2JIle9RIQT69Okjf3+Vydu1a9dCn/fEiRNVKtvf3x/du3eHoaEhjI2N0blzZyxcuLDEvI8ePSr2etu9e7dSZcfFxeGjjz6CjY0NDA0N0aJFC/z9999K5X3w4AEGDx4MS0tLGBsbY/jw4YUmBK4sDHYqya5duzBjxgwsWLAA165dQ9OmTdGrVy8kJCSUmjctLQ1NmzbFb7/9pnK5fn5+8PLywuXLl3HixAnk5OTA09MTaWlpSuW3t7fHkiVLcPXqVQQFBaF79+4YOHAg7ty5o1I9AgMDsXbtWjRp0kSlfA0bNkRsbKz8deHCBaXyJSYmokOHDtDW1saRI0dw9+5d/PTTTzA1NVW6vi+Xe+LECQDAsGHDSs27dOlSrF69GqtWrcK9e/ewdOlSLFu2DL/++qtSZQPAZ599hhMnTuCPP/7ArVu34OnpiZ49eyI6Olphv9KujWXLlmHlypVYs2YNAgICYGhoiF69eiEzM1Op/GlpaejYsSOWLl1a7Pbi8qenp+PatWuYP38+rl27hj179iAkJAQDBgxQquy6deti1apVuHXrFi5cuABnZ2d4enri6dOnSuUvsHfvXly+fFlh+nhl8vbu3VvhGtixY4fS+R88eICOHTvC3d0dZ8+exc2bNzF//nzo6emVmvflMmNjY7Fx40ZIJBIMHTpUqbJnzJiBo0ePYuvWrbh37x6mTZuGyZMn48CBAyXmFUJg0KBBePjwIfbv34/g4GA4OTmhZ8+eSEtLU+q7ZPr06Th48CB2794NPz8/xMTEYMiQIQCU+y5KT09H7969MW/evEL1Ky1/TEwMYmJi8OOPP+L27dvYtGkTjh49irFjxypVdsuWLeHr64t79+7h2LFjEELA09MTeXl5Kn2P/vzzzwrLDCmbd9y4cQqf+7Jly5TO7+/vj969e8PT0xNXrlxBYGAgJk+ejAsXLpSY18HBodD19u2336JGjRro06ePUmV//PHHCAkJwYEDB3Dr1i0MGTIEw4cPx8GDB0vMm5aWBk9PT0gkEpw+fRoXL15EdnY2+vfvD5lMVuh9rXCCKkWbNm2El5eX/O+8vDxhZ2cnfHx8VDoOALF3794y1yMhIUEAEH5+fmU+hqmpqfi///s/pfdPSUkRbm5u4sSJE6JLly5i6tSpSuVbsGCBaNq0aZnqOGfOHNGxY8cy5S3K1KlTRZ06dYRMJit13379+olPP/1UIW3IkCFi1KhRSpWVnp4uNDU1xaFDhxTSW7RoIb788sti8716bchkMmFjYyN++OEHeVpSUpLQ1dUVO3bsKDX/yyIiIgQAERwcrHT5Rbly5YoAIB4/fqxyXqlUKgCIkydPKl32kydPRK1atcTt27eFk5OTWLFihVJ5R48eLQYOHFhifUrKP2LECPHhhx+WKe+rBg4cKLp37650/oYNG4qFCxcqpBV17byaNyQkRAAQt2/flqfl5eUJS0tLsX79+kJlv/pdkpSUJLS1tcXu3bvl+9y7d08AEP7+/qXmf9mZM2cEAJGYmFjkeZeWv8Cff/4pdHR0RE5Ojsp5b9y4IQCI8PBwpcsODg4WtWrVErGxscV+tkXlVeV7saj8bdu2FV999VWZ8r6qWbNmhb6/SspvaGgotmzZorCfmZlZoWvm1bzHjh0TGhoaQiqVyvdJSkoSEolEnDhxotRzKS+27FSC7OxsXL16FT179pSnaWhooGfPnvD393+tdZFKpQAAMzMzlfPm5eVh586dSEtLU2npDS8vL/Tr10/h/JUVFhYGOzs71K5dG6NGjUJkZKRS+Q4cOIBWrVph2LBhsLKyQvPmzbF+/XqVywfyP7+tW7fi008/VWph2Pbt2+PUqVMIDQ0FANy4cQMXLlxAnz59lCovNzcXeXl50NPTU0jX19dXumULACIiIhAXF6fwvpuYmKBt27av/borIJVKIZFIVF57Ljs7G+vWrYOJiQmaNm2qVB6ZTIaPPvoIs2bNQsOGDVWu69mzZ2FlZYV69erhf//7H54/f650uYcPH0bdunXRq1cvWFlZoW3btio9fi4QHx+Pw4cPY+zYsUrnad++PQ4cOIDo6GgIIXDmzBmEhobC09OzxHxZWVkAoHDdaWhoQFdXt8jr7tXvkqtXryInJ0fhenN3d4ejo2OR11t5vouUzS+VSmFsbAwtLa1C6SXlTUtLg6+vL1xcXODg4KBU2enp6Rg5ciR+++23YtdhLKnsbdu2wcLCAo0aNYK3tzfS09OVyp+QkICAgABYWVmhffv2sLa2RpcuXZT6zF519epVXL9+vdjrraj87du3x65du/DixQvIZDLs3LkTmZmZ6Nq1a4l5s7KyIJFIFCYW1NPTg4aGhkrfc2VW6eHUWyg6OloAEJcuXVJInzVrlmjTpo1Kx0I5Wnby8vJEv379RIcOHVTKd/PmTWFoaCg0NTWFiYmJOHz4sNJ5d+zYIRo1aiQyMjKEEKrdwfzzzz/izz//FDdu3BBHjx4VHh4ewtHRUSQnJ5eaV1dXV+jq6gpvb29x7do1sXbtWqGnpyc2bdqkdN0L7Nq1S2hqaoro6Gil9s/LyxNz5swREolEaGlpCYlEIhYvXqxSmR4eHqJLly4iOjpa5Obmij/++ENoaGiIunXrFpvn1Wvj4sWLAoCIiYlR2G/YsGFi+PDhpeZ/WUW07GRkZIgWLVqIkSNHKp334MGDwtDQUEgkEmFnZyeuXLmidNmLFy8W77zzjrw1TpWWnR07doj9+/eLmzdvir1794r69euL1q1bi9zc3FLzF9zVGxgYiOXLl4vg4GDh4+MjJBKJOHv2rFLnXWDp0qXC1NRU/v9HmbpnZmaKjz/+WAAQWlpaQkdHR2zevLnUvNnZ2cLR0VEMGzZMvHjxQmRlZYklS5YIAMLT01Mhb1HfJdu2bRM6OjqFymndurWYPXt2qflfVlrLjjLfZU+fPhWOjo5i3rx5Suf97bffhKGhoQAg6tWrV2SrTnH5x48fL8aOHSv/u6jPpri8a9euFUePHhU3b94UW7duFbVq1RKDBw9Wqmx/f38BQJiZmYmNGzeKa9euiWnTpgkdHR0RGhqq1HkX+N///ifq169f5Lbi8icmJgpPT0/59WZsbCyOHTtWat6EhARhbGwspk6dKtLS0kRqaqqYPHmyACDGjx9fbB0rCoOdSlBdgp2JEycKJycnERUVpVK+rKwsERYWJoKCgsTcuXOFhYWFuHPnTqn5IiMjhZWVlbhx44Y8TZVg51WJiYnC2NhYqUdo2trawsPDQyHt888/F+3atVO5XE9PT/Huu+8qvf+OHTuEvb292LFjh7h586bYsmWLMDMzUynQCg8PF507dxYAhKampmjdurUYNWqUcHd3LzZPdQ52srOzRf/+/UXz5s0Vmq1Ly5uamirCwsKEv7+/+PTTT4Wzs7OIj48vNX9QUJCwtrZWCFBVCXZe9eDBA6UfoRX8f//ggw8U9uvfv794//33VSq7Xr16YvLkycVuLyr/Dz/8IOrWrSsOHDggbty4IX799VdRo0aNQo8GisobFBQkmjZtKr/uevXqJfr06SN69+6tsF9R3yWqBDulfReVFuyUll8qlYo2bdqI3r17i+zsbKXzJiUlidDQUOHn5yf69+8vWrRoUSjQLCr//v37haurq0hJSZGnFfX+KvsdfOrUqSIfoRWVv+D/ube3t8K+jRs3FnPnzlW67PT0dGFiYiJ+/PHHIrcXl3/y5MmiTZs24uTJk+L69evim2++ESYmJuLmzZul5j127JioXbu2kEgkQlNTU3z44YeiRYsWYuLEiSW8OxWDwU4lyMrKEpqamoUu/I8//lgMGDBApWOVNdjx8vIS9vb24uHDhyrnfVWPHj2Uirz37t0r/9IseAGQX9hF3SWXplWrVgr/gYvj6OiocJclhBC///67sLOzU6m8R48eCQ0NDbFv3z6l89jb24tVq1YppC1atEjUq1dPpbKFyP+xLwhWhg8fLvr27Vvsvq9eGwU/0K8GKJ07dxZTpkwpNf/LyhPsZGdni0GDBokmTZqIZ8+eqZT3Va6urkW2kr2af8WKFfLr7OVrT0NDQzg5OZWpbAsLC7FmzZpSy87KyhJaWlpi0aJFCvvNnj1btG/fXumyz507JwCI69evF1unV/Onp6cLbW3tQv29xo4dK3r16qV02UlJSSIhIUEIkd/fcNKkSfJtxX2XFPxAvxqgODo6iuXLl5ea/2UlBTul5U9OThYeHh6iR48ehQIVVb4Hs7KyhIGBgdi+fXup+adOnVrs9dalSxeVy05NTRUAxNGjR0st++HDhwKA+OOPPxTShw8fLm9FVabsLVu2CG1tbfnn/rLi8oeHhxfq5yVE/m/EhAkTlC776dOn8s/a2tpaLFu2rNh9Kwr77FQCHR0dtGzZEqdOnZKnyWQynDp1SqW+L2UhhMDkyZOxd+9enD59Gi4uLuU+pkwmkz/fL0mPHj1w69YtXL9+Xf5q1aoVRo0ahevXr0NTU1OlclNTU/HgwQPY2tqWum+HDh0KDXMMDQ2Fk5OTSmX6+vrCysoK/fr1UzpPeno6NDQU/ytpamqWaYSBoaEhbG1tkZiYiGPHjmHgwIFK53VxcYGNjY3CdZecnIyAgIBKv+4K5OTkYPjw4QgLC8PJkydhbm5eruMpe+199NFHuHnzpsK1Z2dnh1mzZuHYsWMql/vkyRM8f/5cqWtPR0cHrVu3Lvf1t2HDBrRs2VLpPkpA/vudk5NT7uvPxMQElpaWCAsLQ1BQEAYOHFjqd0nLli2hra2tcL2FhIQgMjISHh4e5f4uUiZ/cnIyPD09oaOjgwMHDsj7H5WlbJF/84+srKxS88+dO7fQ9QYAK1aswMaNG1UuuyC/ra1tqWU7OzvDzs6uyOvN0dFR6bI3bNiAAQMGwNLSUuE9KCl/Qb+ioq63vLw8pcu2sLBAzZo1cfr0aSQkJMhHbFaqSg+n3lI7d+4Uurq6YtOmTeLu3bti/PjxombNmiIuLq7UvCkpKSI4OFgEBwcLAPJ+AK+OaCnK//73P2FiYiLOnj0rYmNj5a/09HSl6j137lzh5+cnIiIixM2bN8XcuXOFRCIRx48fVyr/q1R5jDVz5kxx9uxZERERIS5evCh69uwpLCwsirzzeNWVK1eElpaW+P7770VYWJjYtm2bMDAwEFu3blW6rnl5ecLR0VHMmTNH6TxC5I/kqVWrljh06JCIiIgQe/bsERYWFoWa8kty9OhRceTIEfHw4UNx/Phx0bRpU9G2bdtCTfKlXRtLliwRNWvWlPc/GThwoHBxcZHf8ZaW//nz5yI4OFgcPnxYABA7d+4UwcHBIjY2ttT82dnZYsCAAcLe3l5cv35d4frLysoqMW9qaqrw9vYW/v7+4tGjRyIoKEiMGTNG6Orqyu8iVf1/8fJjrJLypqSkiC+++EL4+/uLiIgIcfLkSdGiRQvh5uYmMjMzlSp7z549QltbW6xbt06EhYWJX3/9VWhqaorz588rVW+pVCoMDAzE6tWrC51Hafm7dOkiGjZsKM6cOSMePnwofH19hZ6envj9999Lzfvnn3+KM2fOiAcPHoh9+/YJJycnMWTIECGEct8lEydOFI6OjuL06dMiKChIeHh4yB8nK5M/NjZWBAcHi/Xr1wsA4ty5cyI4OFg8f/681PxSqVS0bdtWNG7cWISHhyvsM3HixBLzPnjwQCxevFgEBQWJx48fi4sXL4r+/fsLMzMzER8fX6bvUfzbclZa3vDwcLFw4UIRFBQkIiIixP79+0Xt2rVF586dlX7fVqxYIYyNjcXu3btFWFiY+Oqrr4Senp4YOXKkUvUOCwsTEolEHDlyRCG9tLKzs7OFq6ur6NSpkwgICBDh4eHixx9/FBKJRPTt27fUsjdu3Cj8/f1FeHi4+OOPP4SZmZmYMWNGse9pRWKwU4l+/fVX4ejoKHR0dESbNm3E5cuXlcpX0KT76mv06NGl5i0qHwDh6+urVNmffvqpcHJyEjo6OsLS0lL06NGjzIGOEKoFOyNGjBC2trZCR0dH1KpVS4wYMaLIDoPFOXjwoGjUqJHQ1dUV7u7uYt26dSrV9dixYwKACAkJUSlfcnKymDp1qnB0dBR6enqidu3a4ssvvxRZWVlKH2PXrl2idu3aQkdHR9jY2AgvLy+RlJRUaL/Srg2ZTCbmz58vrK2tha6urujRo4fC+ZSW39fXt8jtCxYsKDV/waOvol5nzpwpMW9GRoYYPHiwsLOzEzo6OsLW1lYMGDBAoYOyqv8vXg52Ssqbnp4uPD09haWlpdDW1hZOTk5i3LhxCjcmypS9YcMG4erqKvT09ETTpk3lj0KVybt27Vqhr69fps88NjZWfPLJJ8LOzk7o6emJevXqiZ9++knIZLJS8/7yyy/C3t5eaGtrC0dHR/HVV1/Jr1tlvksyMjLEpEmThKmpqTAwMBCDBw+WB8bK5F+wYEGx+5SWv7hzK+lVkDc6Olr06dNHWFlZCW1tbWFvby9Gjhwp7t+/r3TdX1UQ7JSWNzIyUnTu3FmYmZkJXV1d4erqKmbNmiXv26Zs2T4+PsLe3l4YGBgIDw8Pcf78eaXzent7CwcHB5GXl1foHErLHxoaKoYMGSKsrKyEgYGBaNKkidiyZYtSeefMmSOsra2Ftra2cHNzk1+nr4Pk3xMkIiIiUkvss0NERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BARveLs2bOQSCRISkqq6qoQUQVgsENERERqjcEOERERqTUGO0RU7chkMvj4+MDFxQX6+vpo2rQp/vrrLwD/PWI6fPgwmjRpAj09PbRr1w63b99WOMbff/+Nhg0bQldXF87Ozvjpp58UtmdlZWHOnDlwcHCArq4uXF1dsWHDBoV9rl69ilatWsHAwADt27cvtNI0Eb0ZGOwQUbXj4+ODLVu2YM2aNbhz5w6mT5+ODz/8EH5+fvJ9Zs2ahZ9++gmBgYGwtLRE//79kZOTAyA/SBk+fDjef/993Lp1C9988w3mz5+PTZs2yfN//PHH2LFjB1auXIl79+5h7dq1qFGjhkI9vvzyS/z0008ICgqClpYWPv3009dy/kRUsbgQKBFVK1lZWTAzM8PJkyfh4eEhT//ss8+Qnp6O8ePHo1u3bti5cydGjBgBAHjx4gXs7e2xadMmDB8+HKNGjcLTp09x/Phxef7Zs2fj8OHDuHPnDkJDQ1GvXj2cOHECPXv2LFSHs2fPolu3bjh58iR69OgBAPjnn3/Qr18/ZGRkQE9Pr5LfBSKqSGzZIaJqJTw8HOnp6XjnnXdQo0YN+WvLli148OCBfL+XAyEzMzPUq1cP9+7dAwDcu3cPHTp0UDhuhw4dEBYWhry8PFy/fh2ampro0qVLiXVp0qSJ/N+2trYAgISEhHKfIxG9XlpVXQEiopelpqYCAA4fPoxatWopbNPV1VUIeMpKX19fqf20tbXl/5ZIJADy+xMR0ZuFLTtEVK00aNAAurq6iIyMhKurq8LLwcFBvt/ly5fl/05MTERoaCjq168PAKhfvz4uXryocNyLFy+ibt260NTUROPGjSGTyRT6ABGR+mLLDhFVK0ZGRvjiiy8wffp0yGQydOzYEVKpFBcvXoSxsTGcnJwAAAsXLoS5uTmsra3x5ZdfwsLCAoMGDQIAzJw5E61bt8aiRYswYsQI+Pv7Y9WqVfj9998BAM7Ozhg9ejQ+/fRTrFy5Ek2bNsXjx4+RkJCA4cOHV9WpE1ElYbBDRNXOokWLYGlpCR8fHzx8+BA1a9ZEixYtMG/ePPljpCVLlmDq1KkICwtDs2bNcPDgQejo6AAAWrRogT///BNff/01Fi1aBFtbWyxcuBCffPKJvIzVq1dj3rx5mDRpEp4/fw5HR0fMmzevKk6XiCoZR2MR0RulYKRUYmIiatasWdXVIaI3APvsEBERkVpjsENERERqjY+xiIiISK2xZYeIiIjUGoMdIiIiUmsMdoiIiEitMdghIiIitcZgh4iIiNQagx0iIiJSawx2iIiISK0x2CEiIiK1xmCHiIiI1Nr/A45Ofu2hDDlOAAAAAElFTkSuQmCC", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ - "plt.plot(np.arange(len(epochs_x)), epochs_acc, marker = '.')\n", + "plt.plot(np.arange(len(epochs_x_dvs128)), epochs_acc_dvs128, marker = '.')\n", "plt.xlabel('epoch')\n", "plt.ylabel('test accuracy')\n", "plt.ylim(0, 100)\n", - "plt.xticks(np.arange(len(epochs_x)))\n", - "for i, txt in enumerate(epochs_acc):\n", + "plt.xticks(np.arange(len(epochs_x_dvs128)))\n", + "for i, txt in enumerate(epochs_acc_dvs128):\n", " if i%2 == 0:\n", " pass\n", " else:\n", diff --git a/tests/test_nonsequential/transfer-learning/seq_model.py b/tests/test_nonsequential/transfer-learning/seq_model.py new file mode 100644 index 00000000..a109c47e --- /dev/null +++ b/tests/test_nonsequential/transfer-learning/seq_model.py @@ -0,0 +1,86 @@ +import torch +import torch.nn as nn +import sinabs.layers as sl +from sinabs.activation.surrogate_gradient_fn import PeriodicExponential + +class SNN(nn.Module): + def __init__(self, nb_classes, pool2lin_size, batch_size) -> None: + super().__init__() + + self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False) + self.iaf1 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential()) + self.pool1 = nn.AvgPool2d(2,2) + + self.conv2 = nn.Conv2d(10, 10, 2, 1, bias=False) + self.iaf2 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential()) + self.pool2 = nn.AvgPool2d(3,3) + + self.conv3 = nn.Conv2d(10, 10, 3, 1, bias=False) + self.iaf3 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential()) + self.pool3 = nn.AvgPool2d(2,2) + + self.flat = nn.Flatten() + + self.fc1 = nn.Linear(pool2lin_size, 100, bias=False) + self.iaf4 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential()) + + self.fc2 = nn.Linear(100, 100, bias=False) + self.iaf5 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential()) + + self.fc3 = nn.Linear(100, 100, bias=False) + self.iaf6 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential()) + + self.fc4 = nn.Linear(100, nb_classes, bias=False) + self.iaf7 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential()) + + def export_conv_params(self): + torch.save(self.conv1.state_dict(), 'seq_conv1_weights.pth') + torch.save(self.conv2.state_dict(), 'seq_conv2_weights.pth') + torch.save(self.conv3.state_dict(), 'seq_conv3_weights.pth') + + def load_conv_params(self): + self.conv1.load_state_dict(torch.load('seq_conv1_weights.pth')) + self.conv2.load_state_dict(torch.load('seq_conv2_weights.pth')) + self.conv3.load_state_dict(torch.load('seq_conv3_weights.pth')) + + def detach_neuron_states(self): + for name, layer in self.named_modules(): + if name != '': + if isinstance(layer, sl.StatefulLayer): + for name, buffer in layer.named_buffers(): + buffer.detach_() + + def init_weights(self): + for name, layer in self.named_modules(): + if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear): + nn.init.xavier_normal_(layer.weight.data) + + def forward(self, x): + + con1_out = self.conv1(x) + iaf1_out = self.iaf1(con1_out) + pool1_out = self.pool1(iaf1_out) + + conv2_out = self.conv2(pool1_out) + iaf2_out = self.iaf2(conv2_out) + pool2_out = self.pool2(iaf2_out) + + conv3_out = self.conv3(pool2_out) + iaf3_out = self.iaf3(conv3_out) + pool3_out = self.pool3(iaf3_out) + + flat_out = self.flat(pool3_out) + + fc1_out = self.fc1(flat_out) + iaf4_out = self.iaf4(fc1_out) + + fc2_out = self.fc2(iaf4_out) + iaf5_out = self.iaf5(fc2_out) + + fc3_out = self.fc3(iaf5_out) + iaf6_out = self.iaf6(fc3_out) + + fc4_out = self.fc4(iaf6_out) + iaf7_out = self.iaf7(fc4_out) + + return iaf7_out \ No newline at end of file From 71882d23a76fe76ad5e6d0b1b3fc7b71f931c78f Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Mon, 29 Apr 2024 19:29:55 +0200 Subject: [PATCH 057/379] experiments devided in sets of params explored --- .../exp_set_A/baseline-SCNN-example_3.ipynb | 1500 ++++++++++++++++ .../non-sequential-SCNN-example_3.ipynb | 1509 +++++++++++++++++ .../exp_set_B/baseline-SCNN-example_3.ipynb | 1500 ++++++++++++++++ .../non-sequential-SCNN-example_3.ipynb | 1509 +++++++++++++++++ 4 files changed, 6018 insertions(+) create mode 100644 tests/test_nonsequential/exp_set_A/baseline-SCNN-example_3.ipynb create mode 100644 tests/test_nonsequential/exp_set_A/non-sequential-SCNN-example_3.ipynb create mode 100644 tests/test_nonsequential/exp_set_B/baseline-SCNN-example_3.ipynb create mode 100644 tests/test_nonsequential/exp_set_B/non-sequential-SCNN-example_3.ipynb diff --git a/tests/test_nonsequential/exp_set_A/baseline-SCNN-example_3.ipynb b/tests/test_nonsequential/exp_set_A/baseline-SCNN-example_3.ipynb new file mode 100644 index 00000000..dc53313a --- /dev/null +++ b/tests/test_nonsequential/exp_set_A/baseline-SCNN-example_3.ipynb @@ -0,0 +1,1500 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import torch, random\n", + "import torch.nn as nn\n", + "import sinabs.layers as sl\n", + "from tqdm.notebook import tqdm\n", + "\n", + "from tonic.datasets.dvsgesture import DVSGesture\n", + "from tonic.transforms import ToFrame\n", + "from torch.utils.data import DataLoader\n", + "from torch.nn import CrossEntropyLoss\n", + "from torch.optim import Adam\n", + "\n", + "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "torch.backends.cudnn.deterministic = True\n", + "random.seed(1)\n", + "torch.manual_seed(1)\n", + "torch.cuda.manual_seed(1)\n", + "np.random.seed(1)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Loading Data" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "batch_size = 3\n", + "num_workers = 1\n", + "epochs = 30\n", + "lr = 1e-3" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "root_dir = \"./DVSGESTURE\"\n", + "_ = DVSGesture(save_to=root_dir, train=True)\n", + "_ = DVSGesture(save_to=root_dir, train=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "n_time_steps = 50\n", + "to_raster = ToFrame(sensor_size=DVSGesture.sensor_size, n_time_bins=n_time_steps)\n", + "\n", + "snn_train_dataset = DVSGesture(save_to=root_dir, train=True, transform=to_raster)\n", + "snn_test_dataset = DVSGesture(save_to=root_dir, train=False, transform=to_raster)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The transformed array is in shape [Time-Step, Channel, Height, Width] --> (50, 2, 128, 128)\n" + ] + } + ], + "source": [ + "sample_data, label = snn_train_dataset[0]\n", + "print(f\"The transformed array is in shape [Time-Step, Channel, Height, Width] --> {sample_data.shape}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "snn_train_dataloader = DataLoader(snn_train_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True)\n", + "snn_test_dataloader = DataLoader(snn_test_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=False)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Network Module\n", + "\n", + "We need to define a `nn.Module` implementing the network we want the chip to reproduce." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "device: NVIDIA RTX A4000\n" + ] + } + ], + "source": [ + "if torch.cuda.is_available():\n", + " device = torch.device('cuda:0')\n", + " print('device: ', torch.cuda.get_device_name(0))\n", + "else:\n", + " device = torch.device('cpu')" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "class SNN(nn.Module):\n", + " def __init__(self) -> None:\n", + " super().__init__()\n", + "\n", + " self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False)\n", + " self.iaf1 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + " self.pool1 = nn.AvgPool2d(2,2)\n", + "\n", + " self.conv2 = nn.Conv2d(10, 10, 2, 1, bias=False)\n", + " self.iaf2 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + " self.pool2 = nn.AvgPool2d(3,3)\n", + "\n", + " self.conv3 = nn.Conv2d(10, 10, 3, 1, bias=False)\n", + " self.iaf3 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + " self.pool3 = nn.AvgPool2d(2,2)\n", + "\n", + " self.flat = nn.Flatten()\n", + "\n", + " self.fc1 = nn.Linear(810, 100, bias=False)\n", + " self.iaf4 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + "\n", + " self.fc2 = nn.Linear(100, 100, bias=False)\n", + " self.iaf5 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + "\n", + " self.fc3 = nn.Linear(100, 100, bias=False)\n", + " self.iaf6 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + "\n", + " self.fc4 = nn.Linear(100, 11, bias=False)\n", + " self.iaf7 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + "\n", + " def detach_neuron_states(self):\n", + " for name, layer in self.named_modules():\n", + " if name != '':\n", + " if isinstance(layer, sl.StatefulLayer):\n", + " for name, buffer in layer.named_buffers():\n", + " buffer.detach_()\n", + "\n", + " def init_weights(self):\n", + " for name, layer in self.named_modules():\n", + " if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear):\n", + " nn.init.xavier_normal_(layer.weight.data)\n", + "\n", + " def forward(self, x):\n", + " \n", + " con1_out = self.conv1(x)\n", + " iaf1_out = self.iaf1(con1_out)\n", + " pool1_out = self.pool1(iaf1_out)\n", + "\n", + " conv2_out = self.conv2(pool1_out)\n", + " iaf2_out = self.iaf2(conv2_out)\n", + " pool2_out = self.pool2(iaf2_out)\n", + "\n", + " conv3_out = self.conv3(pool2_out)\n", + " iaf3_out = self.iaf3(conv3_out)\n", + " pool3_out = self.pool3(iaf3_out)\n", + "\n", + " flat_out = self.flat(pool3_out)\n", + " \n", + " fc1_out = self.fc1(flat_out)\n", + " iaf4_out = self.iaf4(fc1_out)\n", + "\n", + " fc2_out = self.fc2(iaf4_out)\n", + " iaf5_out = self.iaf5(fc2_out)\n", + "\n", + " fc3_out = self.fc3(iaf5_out)\n", + " iaf6_out = self.iaf6(fc3_out)\n", + "\n", + " fc4_out = self.fc4(iaf6_out)\n", + " iaf7_out = self.iaf7(fc4_out)\n", + "\n", + " return iaf7_out" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "snn = SNN().to(device)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "snn.init_weights()" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "optimizer = Adam(snn.parameters(), lr=lr, betas=(0.9, 0.999), eps=1e-8)\n", + "loss_fn = CrossEntropyLoss()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Define train and test" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "def train(dataloader, model, loss_fn, optimizer, epochs, test_func, dataloader_test):\n", + " epochs_y = []\n", + " epochs_x = []\n", + " epochs_acc = []\n", + " model.train()\n", + "\n", + " for e in range(epochs):\n", + " losses = []\n", + " batches = []\n", + " batch_count = 0\n", + " train_p_bar = tqdm(snn_train_dataloader)\n", + "\n", + " for X, y in train_p_bar:\n", + " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", + " X = X.reshape(-1, 2, 128, 128).to(dtype=torch.float, device=device)\n", + " y = y.to(dtype=torch.long, device=device)\n", + "\n", + " # forward\n", + " pred = model(X)\n", + "\n", + " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", + " pred = pred.reshape(batch_size, n_time_steps, -1)\n", + "\n", + " # accumulate all time-steps output for final prediction\n", + " pred = pred.sum(dim = 1)\n", + " loss = loss_fn(pred, y)\n", + "\n", + " # gradient update\n", + " optimizer.zero_grad()\n", + " loss.backward()\n", + " optimizer.step()\n", + "\n", + " # detach the neuron states and activations from current computation graph(necessary)\n", + " model.detach_neuron_states()\n", + "\n", + " train_p_bar.set_description(f\"Epoch {e} - BPTT Training Loss: {round(loss.item(), 4)}\")\n", + "\n", + " batch_count += 1\n", + " losses.append(loss.item())\n", + " batches.append(batch_count)\n", + "\n", + " epochs_y.append(losses)\n", + " epochs_x.append(batches)\n", + "\n", + " acc = test_func(dataloader_test, model)\n", + " print(f'Epoch {e} accuracy: {acc}')\n", + " epochs_acc.append(acc)\n", + "\n", + " return epochs_x, epochs_y, epochs_acc\n" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "def test(dataloader, model):\n", + " correct_predictions = []\n", + " with torch.no_grad():\n", + " test_p_bar = tqdm(dataloader)\n", + " for X, y in test_p_bar:\n", + " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", + " X = X.reshape(-1, 2, 128, 128).to(dtype=torch.float, device=device)\n", + " y = y.to(dtype=torch.long, device=device)\n", + "\n", + " # forward\n", + " output = model(X)\n", + "\n", + " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", + " output = output.reshape(batch_size, n_time_steps, -1)\n", + "\n", + " # accumulate all time-steps output for final prediction\n", + " output = output.sum(dim=1)\n", + "\n", + " # calculate accuracy\n", + " pred = output.argmax(dim=1, keepdim=True)\n", + "\n", + " # compute the total correct predictions\n", + " correct_predictions.append(pred.eq(y.view_as(pred)))\n", + "\n", + " test_p_bar.set_description(f\"Testing Model...\")\n", + " \n", + " correct_predictions = torch.cat(correct_predictions)\n", + " return correct_predictions.sum().item()/(len(correct_predictions))*100" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Training loop" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "ebf2bea3d0124365abf1f2aa248ceab6", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/359 [00:00" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "y_avg = []\n", + "for y in epochs_y:\n", + " y_avg.append(np.mean(y))\n", + "\n", + "plt.plot(np.arange(len(epochs_x)), y_avg, marker = '.')\n", + "plt.xlabel('epoch')\n", + "plt.ylabel('average loss')\n", + "plt.ylim(0,)\n", + "plt.xticks(np.arange(len(epochs_x)))\n", + "for i, txt in enumerate(y_avg):\n", + " if i%2 == 0:\n", + " pass\n", + " else:\n", + " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAG2CAYAAACZEEfAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAAB9HElEQVR4nO3dd1RUx98G8GfpRUB6kaqg2HvB3oIt9qiJJjHGWH5irLFgYkw0ETWJJsbE9ipqrDGxG3vBhgiKXSmKglQLLL3uvH8QNq60XYrg+nzO2XNk7p07c3evu987d4pECCFAREREpKY0qroCRERERJWJwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqbUqDXbOnTuH/v37w87ODhKJBPv27VPYLoTA119/DVtbW+jr66Nnz54ICwtT2OfFixcYNWoUjI2NUbNmTYwdOxapqamv8SyIiIioOqvSYCctLQ1NmzbFb7/9VuT2ZcuWYeXKlVizZg0CAgJgaGiIXr16ITMzU77PqFGjcOfOHZw4cQKHDh3CuXPnMH78+Nd1CkRERFTNSarLQqASiQR79+7FoEGDAOS36tjZ2WHmzJn44osvAABSqRTW1tbYtGkT3n//fdy7dw8NGjRAYGAgWrVqBQA4evQo+vbtiydPnsDOzq6qToeIiIiqCa2qrkBxIiIiEBcXh549e8rTTExM0LZtW/j7++P999+Hv78/atasKQ90AKBnz57Q0NBAQEAABg8eXOSxs7KykJWVJf9bJpPhxYsXMDc3h0QiqbyTIiIiogojhEBKSgrs7OygoVH8w6pqG+zExcUBAKytrRXSra2t5dvi4uJgZWWlsF1LSwtmZmbyfYri4+ODb7/9toJrTERERFUhKioK9vb2xW6vtsFOZfL29saMGTPkf0ulUjg6OiIqKgrGxsZVWDMiIiJSVnJyMhwcHGBkZFTiftU22LGxsQEAxMfHw9bWVp4eHx+PZs2ayfdJSEhQyJebm4sXL17I8xdFV1cXurq6hdKNjY0Z7BAREb1hSuuCUm3n2XFxcYGNjQ1OnTolT0tOTkZAQAA8PDwAAB4eHkhKSsLVq1fl+5w+fRoymQxt27Z97XUmIiKi6qdKW3ZSU1MRHh4u/zsiIgLXr1+HmZkZHB0dMW3aNHz33Xdwc3ODi4sL5s+fDzs7O/mIrfr166N3794YN24c1qxZg5ycHEyePBnvv/8+R2IRERERgCoOdoKCgtCtWzf53wX9aEaPHo1NmzZh9uzZSEtLw/jx45GUlISOHTvi6NGj0NPTk+fZtm0bJk+ejB49ekBDQwNDhw7FypUrX/u5EBERUfVUbebZqUrJyckwMTGBVCplnx0iIqI3hLK/39W2zw4RERFRRWCwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGptWod7OTl5WH+/PlwcXGBvr4+6tSpg0WLFkEIId9HCIGvv/4atra20NfXR8+ePREWFlaFtSYiIqLqpFoHO0uXLsXq1auxatUq3Lt3D0uXLsWyZcvw66+/yvdZtmwZVq5ciTVr1iAgIACGhobo1asXMjMzq7DmREREVF1IxMvNJNXMu+++C2tra2zYsEGeNnToUOjr62Pr1q0QQsDOzg4zZ87EF198AQCQSqWwtrbGpk2b8P777ytVTnJyMkxMTCCVSmFsbFwp50JEREQVS9nf72rdstO+fXucOnUKoaGhAIAbN27gwoUL6NOnDwAgIiICcXFx6NmzpzyPiYkJ2rZtC39//2KPm5WVheTkZIUXERERqSetqq5ASebOnYvk5GS4u7tDU1MTeXl5+P777zFq1CgAQFxcHADA2tpaIZ+1tbV8W1F8fHzw7bffVl7FiYiIqNqo1i07f/75J7Zt24bt27fj2rVr2Lx5M3788Uds3ry5XMf19vaGVCqVv6KioiqoxkRERFTdVOuWnVmzZmHu3LnyvjeNGzfG48eP4ePjg9GjR8PGxgYAEB8fD1tbW3m++Ph4NGvWrNjj6urqQldXt1LrTkRERNVDtW7ZSU9Ph4aGYhU1NTUhk8kAAC4uLrCxscGpU6fk25OTkxEQEAAPD4/XWlciqhqx0gxcevAMsdKMqq4KVSJnZ2dIJJJCLy8vLwBA165dC22bOHFiiccs6ngSiQQ//PCDwn6HDx9G27Ztoa+vD1NTUwwaNKiyTpMqSbVu2enfvz++//57ODo6omHDhggODsby5cvx6aefAsi/UKdNm4bvvvsObm5ucHFxwfz582FnZ8eLkegtsCswEt57bkEmAA0J4DOkMUa0dqzqalElCAwMRF5envzv27dv45133sGwYcPkaePGjcPChQvlfxsYGJR4zNjYWIW/jxw5grFjx2Lo0KHytL///hvjxo3D4sWL0b17d+Tm5uL27dvlPR16zap1sPPrr79i/vz5mDRpEhISEmBnZ4cJEybg66+/lu8ze/ZspKWlYfz48UhKSkLHjh1x9OhR6OnpVWHNiaiyxUozMHfPLRRMniETwLw9t9G5riVsTfSrtnJU4SwtLRX+XrJkCerUqYMuXbrI0wwMDOTdG5Tx6r779+9Ht27dULt2bQBAbm4upk6dih9++AFjx46V79egQYOynAJVoWr9GMvIyAg///wzHj9+jIyMDDx48ADfffcddHR05PtIJBIsXLgQcXFxyMzMxMmTJ1G3bt0qrDURvQ7XHifi1VnC8oTAo2fpVVMhem2ys7OxdetWfPrpp5BIJPL0bdu2wcLCAo0aNYK3tzfS05W/FuLj43H48GGFoObatWuIjo6GhoYGmjdvDltbW/Tp04ctO2+gat2yQ0RUlPTsXKw8XXhZGE0J4GxR8qMLevPt27cPSUlJ+OSTT+RpI0eOhJOTE+zs7HDz5k3MmTMHISEh2LNnj1LH3Lx5M4yMjDBkyBB52sOHDwEA33zzDZYvXw5nZ2f89NNP6Nq1K0JDQ2FmZlah50WVh8EOEb1R8mQCU3ZcR0hcKgx1NJGRkwfZvy08ozu48BHWW2DDhg3o06cP7Ozs5Gnjx4+X/7tx48awtbVFjx498ODBA9SpU6fUY27cuBGjRo1S6AJRMBjmyy+/lPfj8fX1hb29PXbv3o0JEyZU1ClRJavWj7GIiF7l8889nLwXDx0tDWwZ2xYX53ZH30b5fS9uRiWhGq+AQxXg8ePHOHnyJD777LMS92vbti0AIDw8vNRjnj9/HiEhIYWOWTClyct9dHR1dVG7dm1ERkaqWnWqQgx2iOiNsfXyY/zfhQgAwE/DmqKlkylsTfSxYEBD6GhqIOhxIgIiXlRxLaky+fr6wsrKCv369Stxv+vXrwOAwhxsxdmwYQNatmyJpk2bKqS3bNkSurq6CAkJkafl5OTg0aNHcHJyUr3yVGUY7BBRmZU298mECRNQp04d6Ovrw9LSEgMHDsT9+/eVPv7EiRMhkUjw888/wy/0KRYcuAMA+Li+Nv7v6//BwsICxsbGGNq3J1rrxgAAfjtT+p08vZlkMhl8fX0xevRoaGn91wvjwYMHWLRoEa5evYpHjx7hwIED+Pjjj9G5c2c0adJEvp+7uzv27t2rcMzk5GTs3r27yJYiY2NjTJw4ERMmTJBf2zo6OkhISMCECRMq5Tp/WWhoKAYOHCi/zjt27IgzZ84ofVz6D4MdIiqzwMBAxMbGyl8nTpwAAPncJy1btoSvry/u3buHY8eOQQgBT09PhflSirN3715cvnwZdnZ2SEjOhNe2a8iTCQxtYY+d309Gbm4uTp8+jatXr6Jp06bYu3QKRHoizoc9Q3BkYqWeN1WNkydPIjIyUj7XWgEdHR2cPHkSnp6ecHd3x8yZMzF06FAcPHhQYb+QkBBIpVKFtJ07d0IIgQ8++KDIMn/44QeMHz8e5ubmMDQ0RKdOnbB+/XoAFX+dv+rdd98tdJ2/++67Ja79SMUQJKRSqQAgpFJpVVeF6I02depUUadOHSGTyYrcfuPGDQFAhIeHl3icJ0+eiFq1aonbt28LB0dH4dxvknCac0gMX3NJRMfGCwDi3Llz8v2Tk5MFADH0yzXCac4hMXbTlQo9L8rn5OQkABR6TZo0SQghxPjx40Xt2rWFnp6esLCwEAMGDBD37t0r8Zh///23eOedd4SZmZkAIIKDgwvts3btWtGlSxdhZGQkAIjExMQyn0NMUrq4GP5UxCSll/kYlXGdOzk5iRUrVsi3PX36tNjr/MSJE2WuuyrehM9b2d9vtuwQUYUobu6TAmlpafD19YWLiwscHByKPY5MJsNHH32EWbNmobabO56lZkOakQMXC0Os/aglbK0tUa9ePWzZsgVpaWnIzc3F2rVrYWVlhXmj+0IiAU7eS8DdmOTKPN23UmW05KWlpaFjx45YunRpsfukp6ejd+/emDdvXrnqvyswEh2WnMbI9QHosOQ0dgWq3sm4Mq7zhg0bFtpubm6OevXqoW/fvvJHaMbGxgCAd955B15eXnjx4gU+//xz1KtXD/r6+nB0dMSUKVMKtV69SpllMgYMGICcnBzo6OjA0tISQ4cOxY4dOwC8OZ+3glICu7cCW3aIym/Xrl1CU1NTREdHK6T/9ttvwtDQUAAQ9erVK/Vud/HixeKdd94Rubl5YuIfQULT2ErY9Z4gHj5Nle8TFRUlWrZsKSQSidDU1BS2trbi2rVrQgghvLZdFU5zDolJ265W/EmSgopq4RBCiIiIiGLv9AucOXOmzC07MUnpwnnuIeE0579X7bmHVW7hqejrvOC9e7VlR4j867xJkyYCgNDQ0BBWVlZi9erVAoA4c+aMuHXrlhgyZIg4cOCACA8PF6dOnRJubm5i6NChJZYdGxur8Nq4caOQSCTiwYMH8n2WL18u/P39xaNHj8TFixeFh4eHsLGxqXafN1t2iN4SJXUSLuud3zfffAN3d3cYGhrC1NQUPXv2REBAQInljhgxAm5uboX6HowaNQrBwcHw8/ND3bp1MXz4cGRmZhZZ7tWrV/HLL79g06ZN+OF4KI7cjoNEAgxv5QgXC0MAgBACXl5esLKywvnz53HlyhUMGjQI/fv3R2xsLLy6uQIA/rkViwdPU8v6tlIpKqqF43WJeJZWITNuFzXHD1D267yo9w747zqvVasWLly4gMDAQAwdOhRffPEFnJ2d0aVLFzRq1Ah///03+vfvjzp16qB79+74/vvvcfDgQeTm5hZ7DjY2NgqvV5fJAIDp06ejXbt2cHJyQvv27TFz5kzExcVh9OjRb8TnXUip4ddbgC079CZLSEhQuEs7ceJEue/8tm3bJk6cOCEePHggbt++LcaOHSuMjY1FQkKCfB8nJyexcOFCERsbK65cuSI0NDTEzp07SzxuVlaWMDAwENu3by9y+4oVK4REIhEampoCEo381793tU5OTkIIIU6ePCk0NDQK/X91dXUVPj4+Qgghxm4KFE5zDokZu66X9vZRGVVUC0eBym7ZiU5MU2jVKUvLzqNHj4SGhobYt29fifspe51ramrKX8pc51lZWUJDQ0N4enoWW/b69euFhYWF0ucUFxcntLS0xLZt24rd5/nz58LDw0MAqHafN1t2iN4SlpaWCndphw4dki+QWNY7v5EjR6Jnz56oXbs2GjZsiOXLlyM5ORk3b95U2M/IyAg2NjY4fPgwrKysFFaLLooQAkIIZGVlFbn9o48+wuZDfrAbsxK2Y1biizX7YWdnh1mzZuHYsWMAIF/vSEND8etLQ0NDPuPt5O75rTv7rkcj6gXXyqoMFdHC8TrFJRe+5ub2cVdpxm1l5/hR5jq/efMmrl+/Ln8pc53v27cPMpkMzZs3L/K4z549w6JFixRmky5NUctkFJgzZw4MDQ1hbm6Ou3fvwtPT8435vF/FYIdIjZT2aAEApFIpjI2NFeYpKe2Y69atg4mJSaFJ15YsWQIzMzMsXrwY7u7uCtsePnwIHx8fXL16FZGRkbh06RKGDRsGfX199O3bV77fy3OfJObpYNmVdGiaO2HYO+2xbFw/aGtrw8bGBvXq1QMAeHh4wNTUFKNHj8aNGzcQGhqKWbNmISIiQv4j1MyhJjq5WSBPJrDG74Fybx4praRZjE1MTODm5obOnTvjr7/+wv379wvNbVMVDt2IBQB4NrBGPesaAIC07OID/lcVN8dPWa5zc3NzNGrUSOGlzHU+Z84cSCSSIofJJycno1+/fmjQoAG++eYbpc+rqGUyCsyaNQvBwcH4448/IJVK8fz580IzlFfXz/tVDHaI1EhRCyS+TJU7v0OHDqFGjRrQ09PDihUrcOLECVhYWMi3T5kyBTt37sTixYuRk5ODq1evYvbs2fLtenp6OH/+PPr27QtXV1eMGDECRkZGuHTpEqysrOT7Fcx98iw1C2M2BSIlMxetnEyxdGiTIgM2CwsLHD16FKmpqejevTtatWqFCxcuYP/+/QrB2OR/++7sDnqCOGn1u9N8k1VUC8frIpMJ/HMrP9gZ1soBk/69NnZeiUJunkypYxQ3x4+q17myXr3OW7RogUePHuGrr74qdNORkpKC3r17w8jICHv37oW2trZSZRS3TMbLdahbty7Cw8NhZWWFq1ev4vLly8Uer7p83kVS6uGammOfHVIXnp6e4t133y1ym1QqFW3atBG9e/cW2dnZpR4rNTVVhIWFCX9/f/Hpp58KZ2dnER8fX+z+GzZsEFpaWiIzM1OlOsckpYsz9+NFv1/OCac5h0SnpafFsxTVjlGc91ZfFE5zDomFB+9UyPFIiLy8POHo6CjmzJmjkP7gwQOxePFiERQUJB4/fiwuXrwo+vfvL8zMzBSum3r16ok9e/bI/37+/LkIDg4Whw8fFgDEzp07RXBwsIiNjZXvExsbK4KDg8X69evlc88EBweL58+fK1XnwIjnwmnOIdHo66MiIztXZObkihYLjwunOYfEkVuxpR+ggpRnjp8FCxYIGxsbkZOTo5AulUpFu3btRJcuXURaWppKxxw9erRo2bJlifsUfN7/+9//5H0Bhag+n7eyv98MdgSDHVIPJXWeTE5OFh4eHqJHjx4iIyOjTMd3dXUVixcvLnb77du3BQBx//59pY+588pj4fLScOB6X/4jwuJTylS/opwNScg/7lf/VFgA9bY7duyYACBCQkIU0qOjo0WfPn2ElZWV0NbWFvb29mLkyJGFrgcAwtfXV/63r69vkRPXLViwQL7PggULitzn5eOUZMH+28JpziExfWewPG3JkXvCac4h8eH/XVb1LSiTl691l7mHxM4rj5XOW1yAKZVKRdu2bUXjxo1FeHi4wkCF3Nxc+X6vBhwFeQ0MDMTq1asLlXf58mXx66+/iuDgYLFlyxYBQDRv3lzUqVNHfjNTXT5vBjsqYLBD6qAy7vxeVrt2bYUvpFdt3bpVaGhoiBcvXih1vIinqcL5ldExLnMPlWtm21fJZDLR/9fzwmnOIbH0SMkzu74pSpvVNiMjQ0yaNEmYmZkJQ0NDMWTIEBEXF6f08SdMmCAAFJrzpahyC0a/qaoiZjFWVl6eTLT+7oRwmnNInLz73/sQ+TxNPu/Oy3M4VYaYpHSFoF7VkWDFBZgFo5WKekVERMj3KypQWLt2rdDX1xdJSUmFyrt586bo1q2bMDMzE7q6usLZ2Vl8/OlnYt+Fm2X6zCrz81b291u5HopEVK3JZDL834aN6DlgGJ6m5cDWJP+/dnJyMjw9PZGeno6tW7ciOTkZycn5MwtbWlpCU1MTQH7nSR8fHwwePBhpaWn4/vvvMWDAANja2uLZs2f47bffEB0dLZ851d/fHwEBAejWrRuMjIzg7++P6dOn48MPP4SpqWkJ9RQIiHiBfcHROHAjGuLV7QJ49CxdpREyJZFIJPDq5ooJf1zFFv/HmNC5DkwMlOvPUF0FBgYqzFB7+/ZtvPPOO/LPZvr06Th8+DB2794NExMTTJ48GUOGDMHFixdLPXZJ6zQBwMKFCzFu3Dj530ZGRirXf3vAY3y59zYEAA0J4DOkMUa0dlT5OMoKepyIhJQsGOlpoaPbf33OHMwM0K2eFU7fT8C2y4/x1bsNKq0OEc/SICtmjh9lrnVPT89CHYMBoGvXrkWmv0oIgVhpBi49eAYXC0PYmuhj/PjxRfbdE0LAvnY9rN2xH09Ts/AsNRsn7sTh0M1Y+B2MhORgJLrWs0QDO+NSywWAuzHJOBvy9LV93sVhsEOkBr76bTuin0ThtGiEDktOy79Qrl27Jp8M0NXVVSFPREQEnJ2dAeR3ngwKe4J20gyY6mri/v372Lx5M549ewZzc3O0bt0a58+fl09rr6uri507d+Kbb75BVlYWXFxcMH36dMyYMaPI+oXGp2BvcDT2B0cjpoTOwpoSCZwtDJQ+7+joaMyZMwdHjhxBeno6XF1d4evri1atWgEA4uPjsX3ZXMTtO4Ts9FS0+6cNDmzfADc3t2KP2bVrV/j5+RVK79u3Lw4fPgwg/wdh+vTpWLN2LbIyM6GhoQFXV1ds27ZNoew5c+bg+PHjSEpKQufOnfHrr7+Wu2wLCwssWLAA69evR1JSEiwtLeHo6IguXbpAKpViw4YN2L59O7p37w4gvzNx/fr1cfnyZbRr167E9/Lzzz/HsWPHiu14XDDVQFnFSjPkgQ6QH9zO23MbnetaVliA+6pDN2MAAJ4NbKCrpamw7cN2jjh9PwG7rz7BF73qQU9bs6hDlJuFoW6hNA0JVLrWy2NXYCS899yCTAASCTCilQNcrWrkBzMp2XiWmiV/PU/NRu6rkdlLBIAzIU9xJuSpyvV4HZ93cRjsEL3hYpLSsSPaFE5zDgHI/0KZ8/ctbLr0GLVqGmLOXzdgUUMXFjV0YGmkB4saOrAw0oVpDV0IIfBnUBRc5h7CthfAjn8DpT179pRYZosWLUoclQEA8cmZOHA9BnuDo3E39r91qoz0tNCvsS0GNa+FiGdp+GrvbeQJAU2JBIuHNFL6SzAxMREdOnRAt27dcOTIEVhaWiIsLEzesiSEwKBBg6CtrY3vf9+C5WejkBB8AD169MS9e3dhaGhY5HH37NmD7Oxs+d/Pnz9H06ZN5S0nQP4M0ytXroSuQ2OYN3kHabdOIv5ZDAwMDAqVvX//fhgbG2P58uXo2bMn7t4tX9nLli3DypUrsXnzZtjb26N9+/YwNDREVlYWrl69ipycHPTs2VO+v7u7OxwdHeHv719ssFPaOk0FlixZgkWLFsHR0REjR47E9OnTlZ7CAADO3E8o1JqnSguHqvJkAv/cyl8h/N2mtoW2d6lrBXtTfTxJzMDBGzEY1qpyZv7dez26UJqbVQ3YGBce7l3RYqUZ8kAHAIQAdgZGlZrPWE8LFka60NHUwP24lELbezeyLvUzi03KwNE78Qpplfl5l4TBDlEFUKaFQdW7/D179mDx4sUIDw9HTk4O3NzcMHPmTHz00UcAgJycHMz1nocNO/Yg+Wk0NHQNoefUFDW7fAItI3Pci03GvdiSF8PU1pAg56W7OJkA5v59C7FJmahtVSM/QKqhC4sauqhpoF3s3D2x0gxEPEuDlZEebkQlYd/1aFwMfyb/gtXWlKBrPSsMbl4L3d2t5HfQ7Wqbo2s9Szx6lg5nCwOVvgCXLl0KBwcH+Pr6ytNcXFzk/w4LC8Ply5dx+/ZtuNdvgEOxfnhoNhGJ6z7Bjh07ih1ua2ZmpvD3zp07YWBgIA84hBD48ccfoWlsBesPFgMADFzb4MmvH8LvchAaNGigUHZB8LB69WrY2NiUu+yff/4ZX331FQYOHIg///wTeXl5SEtLk084p6Ojg5o1ayocx9raGnFxcSW+l1paWpgyZUqx+0yZMgUtWrSAmZkZLl26BG9vb8TGxmL58uXF5nlZTp4Mmy49KpSuamueKq5EvMCz1CyY6GujQx2LQts1NSQY2dYRy46GYGtAZKUEOwnJmfC9GAEAWDq0CTQ1AO+/byEkPhV7g6MxpIV9hZf5socJhR+hAYBHHXO42xjBooZu/v9xI51/b4p0YV5DR94KFivNQIclpxWOoSmRYEH/hqUHO9IMHL8bXyjv62rRehmDHaJyUqWFQZW7fDMzM3z55Zdwd3eHjo4ODh06hDFjxsDKygq9evVCeMxzbD54FtqthsHWygWyzFS8OLUOT/csQq1PfobP4MbIFaJQM/Wz1Gw8S8lCSlauQqBTQAD4+VRYoXQtDQnMa+R/IVoa6cq/GKNepOOfW7GF7tgBoKWTKQY3r4V+jW1haqhT5HnamuiX6S7vwIED6NWrF4YNGwY/Pz/UqlULkyZNkvcpKZjrQ09PD5oaEvyvax3M/usmMmWa8Dt3vtiA41UbNmzA+++/L/+cHj58iPT0dBg2aIOn+3yQGXUbmjXMoWlsiTPnLuB/n36sUHYBDQ0N6Orq4sKFC2UuOyIiAnFxcfKWm4JZjJOTk+Hv74+2bdsqddyXFazTdO3atWKDWQAKjyibNGkCHR0dTJgwAT4+PtDVLfyY5lXrzj1EaHwq9LU1kZWbJ/8BnOlZt9Lu8g/fyn+E1auhNXS0ip5WbngrB6w4EYobUUm49USKxvYmFVqHX0+HIzNHhhaONTG8lT0kEgnik7Pww7EQfHvwLjq6WcDKqPJaeC4+eFYoTVMiwfLhTZV6321N9OEzpDHm7VG9BbY8eSsagx2iclKlhUGVu/yuXbsq/D116lRs3rwZFy5cQE23Vpi07RZqDP4G5oY6eK+lPf7vfATwzkTEbZmBme3NMKJNyZ0AM3PycDdGiqFr/BUWSJQA6OZuhbSsXHlwJM3IQa5MID45C/FFTLv/qnGdXPBRO2c4mlfeHdzDhw+xevVqzJgxA/PmzUNgYCCmTJkCHR0djB49Wv74xtvbG2vXrkW/hpaYNX8fcpKf4npIhFJlXLlyBbdv38aGDRsAAJHP0+G14QwAIC3kIoxbD4K1x3BkxYbhxfHfccrvImKSMgqVbWhoiBUrVuDJkyeIjY0tU9kA5K0z1tbW8lmM9+zZg23btiEuLg42NjbIzs5GUlKSQutOfHx8sX1tzp8/j4SEBDg6/ne95OXlYebMmfj555/x6NGjIvO1bdsWubm5ePTokXzW3+KEJ6Tgl38D6O8GNUJ7V3NM+OMqbj6R4kVadol5yyo3T4ajt/Pfr35Niu5wDQAWNXTRt7Et9l+PwdbLj7H0vSYVVofHz9Ow40okAGB2b3d5MDm+c238cysWd2KSsWD/Haz+sGWFlfmyq48Tse7cQwD5fXWEQJkCjhGtHdG5btlaYMuTtyJxBmWicjpw4ABatWqFYcOGwcrKCs2bN8f69evl20u7y1eGEAKnTp1CSEgIsi3rYdT/BeB5WjYa2hnjwOcd4d23Pi7M7QbvHk6QSCT4qEvx/S4K6GlrooWTGZYMaQzNf7+ENSUSLBnaGBs/aY1dEzxwamZX3FjgidDv+sDfuzsOTO6AjZ+0wrKhTTCrVz30bmhd5LG7u1tXaqAD5PczadGiBRYvXozmzZtj/PjxGDduHNasWQMA0NbWxp49exAaGgozMzOYGNWASWIo9Gq3xJOkTGTnlj5z7oYNG9C4cWM0b9kKa/wewPNnP1yPTAIAWDi4wrzraOhY14Fxs97QqmmNpLgovLPcD1uvPMHuv/6Wl21gYIAzZ86gT58+hdb0Kq3sNm3aFLm9qFmMW7ZsCW1tbZw6dUqeFhISgsjISHh4eBR5HGXWaSrK9evXoaGhoTBLcFHyZAKz/7qJ7FwZutazxJAWtWBroo/pPesCAP4MikK6Css2KCv/EVY2ahpoo30d8xL3/bCdEwBg/41oSDNyKqwOK06EIlcm0LmuJdrV/q8O2poaWPZeE2hpSHDkdhyO3FIuAFaFND0HU3YEI1cm8G4TW1yc0w07xrXDhbndyjQaytZEHx51zMsUrJQnb0Vhyw5ROanawqDKXb5UKkWtWrWQlZUFTU1NdB/rjV0xpgAE+je1w7KhTaCvk/9s3VRXgs2/fI8PPvgAxsbKDQsFlLvz0tHSKPJxU1U+k7e1tUWDBorDhevXr4+///5b/nfLli1x/fp1SKVSZGdnw6imGSxqN0KuSR3sDX5S4pd+Wloadu7ciQnTvdH/1wvyTpqt67vgEICOzerh97nd5e+bxy59PElLRFp2Hr49eBdN7U2w7R8/2Bvmry9maWmJtm3byvtxlaSg7IULFyqkF7TOxMbGKkw1EB8fj2bNmsHExARjx47FjBkzYGZmBmNjY3z++efw8PBQ6Jz88lQD5ubmMDdXDAZeXaeprFMNAMDmS49wLTIJNXS1sHhwY3nrRpe6lnA2N8Cj5+nYGxyNUW2dSn1fVHHo3wCid0MbaGuWHGC2cjJFPWsjhMSnYM+1JxjTwaXE/ZVxLzYZ+2/kP0ab3atwy1dDOxNM7FIHq86EY/7+O2hX27zYR72qEkJg9t83EJ2UAUczA/gMaQwjPW3Y1Xz9fWWqC7bsEJWTqi0MqtzlGxkZ4fr16zh29gLq9huHoxt+QGbkTczt446V7zeTBzo5OTkYPnw4hBBYvXq1yudQ1juvgmfyL7cMva5n8h06dEBISIhCWmhoKJycCv9ompiYwNLSElGPHiI9JhQGbm2x+uyDEtdF+mP7TqRlZGJnkhPux6WgpoE2fnivCfZ7D4aenh5u3Lghf98MJTl4EvUYtZ0c8N2gRjDS1cKNJ1IMWHURv1+MgaGJKcLCwhAUFISBAweWem67d+9GVlYWPvzwQ4V0FxcX2NjYYNaiH+VTDXh8ewj+lwPkLTcrVqzAu+++i6FDh6Jz586wsbEpNLouJCQEj2Of4tKDZ4iVZpRan4KpBrp06YKGDRvi+++/x/Tp07Fu3boS80U+T8cPx/I/o7l93GFX87/rQkNDgo88nAHkB0TKzBejrJcfYb1bwiOsAhKJBB965F83Wy8/rpC6/HgsBEIA/ZrYolGtovsBfd7DFa5WNfAsNQuLDt0td5kFtvg/xrE78dDWlGDVyOYw0nuz55aqEBU+neEbiDMoq4cnT56IUaNGCTMzM6GnpycaNWokAgMD5dtRzEyjy5YtK/aYubm54quvvhLOzs5CT09P1K5dWyxcuFDIZDL5Po6OjqJZs2bCxsZG6OnpiR49eohvvvlG2NnZFTpeUlKSSEhIEEII0aZNG/mstyW5EZUo2n5/UjjNOSRMm/cSrTp0VdienZ0tBg0aJJo0aSKePXtW6vEqQ0xSurgU/uy1zIhb4MqVK0JLS0t8//33IiwsTGzbtk0YGBiIrVu3yvf5888/xZkzZ8SDBw/Evn37hJOTkxgwaLBo9u0x4TTnkNgX/ER89NFHYu7cuQrHPnY7Vhg5NRIG7p3kywy8vNyEl5eXACA+/PBDcejQIdGyZUshkUjExo0bhRBCxEkzhOfnS4T1B4uF3YT/E+4fLRTWdg5iyJAhCuUUVbYQQnTs2FGMGDFCIS0vTyZuPUkSvcfMEBq6hsJyyHxh++kqoe/WTmiZWIsuPsfEqPWXxdQd18Sig3fE6rPh4q+gKHE2JEHcjk4S8dIMkZObJ4Qo39IFypLJZOKDdf7Cac4hMWLtJZGXJyu0jzQjW9Sff0Q4zTkkLoY9rbCyz4XmLxPSfOFx+TmXJiUzRzQoqEt4+epSsBZXbe/D4kFCycufXH38Qj6T8+l7xa89p6xbT5KE27x/hNOcQ2LD+YflPl51xxmU6a1S2ogoAIUeGR05cgRjx47F0KFDiz3u0qVLsXr1amzevBkNGzZEUFAQxowZAxMTE/kwXVNTU9y6dQt///03XFxcMH/+fCxfvhz169cvdDwTk/w7vIK7/EWLFpV4XnuuPcHcPbeQnStDHUtDmLlb4mnMf3NkFLTohIWF4cyZM4UeR7wuZR1RVR6tW7fG3r174e3tjYULF8LFxQU///wzRo0aJd8nNjYWM2bMQHx8PGxtbfHxxx9j/vz5WHfhMX48HorfzoQjOzJS3sIWJ83EggO3cejcVaQ8vo1GY5fh97Ft0MnNUqHsX3/9Fc+ePcPOnTuxdetWGBgYYOHChRgzZgwAwNpYD/1c9RG8eykSniYg3tAUNRp2h82gGXiakgVLo/zRS5EvlV0gJCQEFy5cwPHjxxGdlIELYU9xPuwZLj14jhdp2RCW3WDUMgbPj/0KWWYa9OwbwGr4QjxKysGjpMIjb14mkQAmetpIeqlfSmVN9LYzMAqXHjyHnrYGlg5tAg2NwiO9jPW0MbSFPf64/BibLj1Ce9fCw8PL4vDNfx9hNbKBVimPsArU0NXCoOa1sC0gEtsuR6J9EUPVlSGEwLKj+a1Zw1vZo7ZljRL3b+Foik87uGDDhQjM23sLx6d3LnNLTGpWLiZvv4bsPBl61rfGmA7OZTqOOpIIUYFth2+o5ORkmJiYQCqVqtTXgaqPuXPn4uLFizh//rzSeQYNGoSUlBSFzpyvevfdd2Ftba0wImbo0KHQ19fH1q1bIYSApaUlEhMTsWjRIgwfPhxnz57FuHHj4OXlhVWrVgHIfyxRMNPtrVu3MHXqVLRs2VKhf8nHH3+MWrVqwcfHB7l5MvT5dCZuZppDy9QW7RyN0FLjERbM/xKrV6/GZ599hpycHLz33nu4du0aDh06BGvr/zoLm5mZQUenYp7/qyNpRg46LjmNlKxcTO9ZF0Nb1sLp+wlYdjQEqVm50NKQYFzn2pjS3U3+qLCs0rJy8dPxUGy6FAGZyJ+sbV7f+hjeygHxKZmIeJYmn8I/JTMHlx++yA9wwp/h4dM0hWMZ6miiuWNNXAx/rjDUX0MC/DS8GWQyoTjFQGoWnqbk//tFWlaR860U2DGuHTxK6cirrFhpBjyXn0NKVi6+6lcfn3WqXey+4Qkp6Ln8HDQkgN+sbnAwK1+/kpw8GVp/fxJJ6TnY/llblQKoe7HJ6PPLeWhpSHBpbndYlWHSvzMhCRjjGwgdLQ34zeqqVACZkZ2HXj+fQ+SLdIxq64jvBzdWuVwhBKbtuo7912NgZ6KHf6Z2Qk0D9f8OUPb3my07pBZKm3PlVfHx8Th8+DA2b95c4nHbt2+PdevWITQ0FHXr1sWNGzdw4cIF+URqEREReP78OX755ResX79e3sLg5uYmX3cKKL6F4WUFd/lJ6dn4fEcwroTFIv3+LiDtBc4YGiDW3R1bt27FiBEjAORPZHjgwAEAQLNmzRSOdebMmUJD1+k/JvraaO1shtMhCVhxMhQrTobKtzV3rAmfIY3hblMxNz6Gulr4un8DDGpuB+89t3AnJhlz99zCWr+HePwif8I3CQBHMwM8ScpA3ksRiYYEaOpQE51cLdCpriWaOdSEtqYGdgVGFpq7ZHDzWiXWI08mkJiejXuxyfh445VC0w04mVdMq44QAl/tvY2UrFw0c6hZamdfVysjdHKzwPmwZ9h6+TG8+xZuEVXFpQfPkZSeA4saOmjjYlZ6hpfUtzVGKydTBD1OxM7AKEzpUfykn0WRyf5r1fmkvbPSLWX6OppYMrQxRq4PwLaASLzbxE7lwHN30BPsvx4DTQ0JVn7Q/K0IdFTBlh2wZUcdFAzrnjFjBoYNG4bAwEBMnToVa9aswejRowvtv2zZMixZsgQxMTEKQ8JfJZPJMG/ePCxbtgyamprIy8vD999/D29vbwDApUuX0KFDB8TExMDW9r/p6IcPHw6JRIJdu3YpfQ6x0gycD3uGn0+GIiYpE/ramvhxWFP0a1J4mnsqn6JmhQWALzzr4n9dXaFZxCOXipCbJ4PvxUf46XgIMosZ+u5sboCObhbo6GoJjzrmMNEv+pFGrDSjzHOX5AdLt5D30vmPauuIhQMblfvc91+PxtSd16GjqYFDUzqirnXpi4WevBuPz7YEwURfG5e9e5SrNW32XzfwZ9ATfNjOEd8NUr2FZF9wNKbtug5bEz2cn91N6cdgAHDgRgym7AiGka4Wzs3upvLoqnl7b2F7QCSczA1wdGpnpd+HsPgU9F91AZk5MszqVQ9e3VxLz6Qm2LJDbxWZTIZWrVph8eL86fubN2+O27dvFxvsbNy4EaNGjSox0AGAP//8E9u2bcP27dvRsGFDXL9+HdOmTYOdnV2Rxy2rLZceYcGBO/JHE6YG2tj2WTulVxYm1RS1CjUAtHQyq7RABwC0NDUwrnNtWBnpYuqu64W2r3y/GQY0K7mFpkB5+km9PN1A0OMXWH4iFNsCIvEsNQu/vN+8zAtiPkvNwjcH7gAAPu/uqlSgA+RPYuloZoDIF+nYdz0aH5QyIWZxsnNlOPbvWkzKjMIqSp/GNlh4SAex0kycvp8Az4bKLXyakyfDT8fzW3XGda5dpmHk3n3cceZ+Ah4/T8dPx0OUWok9IzsPXtuvITNHhk5uFvhflzoql/s24NBzUgvFzbkSGRlZaN/z588jJCREqSn7Z82ahblz5+L9999H48aN8dFHH2H69Onw8fEB8N+8J/HxiovdlTRjbYHcPBn8Qp9iwpYgfP1SoAPk9ykxNeRw0criYmGIV2Oa17lmT5vaZkWW31rFxy7lUTBs/vPubvhtZAvoaGrg2J14fLzhCqTpZZtYb8GBO0hMz0F9W2NM7Kr8j66mhgQf/zv0uzzD0C+GP4M0IweWRrpo7Vy291JXSxPD/10j64/Lj5XO92dQFB4/T4e5oQ7GdizbPD1GetpY/G9/nY0XI3AtMrHUPAsP3UFofCosauhi+fBmRXYEJwY7pCZUmXNlw4YNaNmyJZo2bVrqcdPT0wuNltHU1IRMlv8IomDek5c7OScnJyMgIKDIGWuFELj1RIqFB++inc9pjN54BcfuxhfaTyaAR8/SS60flU1Vzg9UHcp/Vd/Gttj8aRsY6WrhyqMXGL7WX6n5d1527E4cDt+MhaaGBD+816TUifxeNayVA/S1NXE/LgWXH75QKW+BQ/+OwurbyKZcLXSj2jpCIgHOhz3Do2dppe6fkZ2Hlf8uhzG5uysMdcv+0KSbe/6CuTIBzP7rJrJy84rd98CNGOy4EgWJBPjl/WbyUX5UGB9jkVqYPn062rdvj8WLF2P48OG4cuUK1q1bV2jSs+TkZOzevRs//fRTkcfp0aMHBg8ejMmTJwMA+vfvj++//x6Ojo5o2LAhgoODsXz5cnz66acA8icjmzZtGr777ju4ubnJh57b2dlh0KBB8uNGvUjHgRsx2BscjfCEVHm6qYE2urtbYU9wtEKH0apaGfhtUtVr9lR1+a/yqGOOPyd6YPTGKwiJT8HQ3y9hy9g2cLUq/VGUND0HX+27DQCY0Ll2sZPolcREXxtDWuQP/d586ZHKHXSzcvNw/G7pa2Epw8HMAF3qWuJsyFNsvxKJeaV0mt7s/wjxyVmoVVMfI9uW7RHcy75+twHOhz1FeEIqVp0Ox0zPwjMwP3qWhnl7bgEAJndzRYcKGravrhjskFpQZs4VANi5cyeEEPjggw+KPE5oWDiCQyMRK82ArYk+fv31V8yfPx+TJk1CQkIC7OzsMGHCBHz99dfyPLNnz0ZaWhrGjx+PpKQkNG7ZFlt270OWTBN7r0Ri77VoXHn0352qrpYG3mlgjcHNa6FzXUtoa2qgjYtZtVgZ+G1TFfMDVafyX1Xf1hh7JrXHxxuv4OHTNAxd7Y+Nn7RCS6eSHwl9d/gunqZkoY6locojmF42ur0ztgVE4vjdOEQnZaBWTeXfmwthz5CSmQsrI120cip5CQtlfNTOCWdDnuLPoCjMeKdusf2YpBk5WH32AQBg+jt1oatVvqkKAMDUUAcLBzbCpG3XsPrsA/RpZKvQfy8rNw+Td1xDalYu2jibYWo53vO3BUdjgaOxKN+uwEh477kFmcgf8uszpLFKC+a9nB/Ib53J+/e/l0QCeNQ2x6DmtdC7kQ2Mi5g0rDyja4gq0ou0bHy6KRDXo5Kgq6WBVSNb4J0GRS/66hf6FKM3XoFEAvw10aPUwKg0I9dfxqUHzzGxSx3M7eOudL4Zu65jT3A0PmnvjG8GlL4QbmnyZAKdl51BdFIGfhrWFENb2he53w/H7uO3Mw/gZlUDR6d1rtAO7hP/uIqjd+LQ0M4Y+7w6yB8NfnvwDnwvPoKpgTb+mdrprf6+4GgsIiVl5uTh6O04zP37lryTsEwAc/6+BZ9/7ivV4U8mEwqz0gJAnhCoY2mI4a0cMKCZXalfSNXtLp/eXmaGOtg+ri28tl3DmZCnmPBHEBYPboz3XxkllZqVK3+UMtrDudyBDpA/P82lB8+xMzAS03q6KTUyLDMnDyf+7fvWv2nFTNWgqSHByLaO+OFYCLYGPC4y2ElIycTGC48AAF/0qlfhI/kWDmoI/4fPcScmGevOPYRXN1ccvxMH34v5Zf40vCm/M5TEYIfeOjKZwN3YZJwPe4YL4U8R+CgR2cXMefJqAKOq7wY1rrBZaYleJwMdLaz7uBW899zCX1fzly15mpKFyd1d5SuXLzt6H9FJGXAw08fs3oX7lZRFj/rWsDfVx5PEDBy4HoPhrR1KzXMu9ClSsnJha6KH5g7lf4RVYHgrB/x8MhTBkUm4HS0t1Bfpt9PhyMjJQzOHmvAspuWrPKyM9PD1uw0wc/cN/HwyFJoaEvx2JhwA8FlHF3R3r/gy1RWDHVIrsdIMhen3C8QkZeBC2DOcC3sqX2PoZVZGukhIyVJI05AAW8e2VWqEw9OULHy4IUBh7hZ2MqY3nbamBn54rwmsjXXx25kH+OlEKOJTMjGxSx0cvxOPLf75Q7OXDGkCA52K+TnR1JDgo3ZO8DlyH76XHmFYK3t5cFWcw7f+HYXV2LZCh15bGumidyNbHLwRg20Bj+EzpIl8W9SLdGy/kj+1xeze9UqtY1kNaVELa889QGh8KpYcuQ8AsDfVx+zeyj/iIwY7pEZe7XPzYTsnSIBi1xjyqGOOjq4W6OhmiTqWhvgzKKpQJ2Fl19VxszaCz5DG7GRMakcikWBWL3dY1tDFt4fuYuvlSGy9/N/8Va2dTSt8JNCI1g5YcTIU92KTEfgoscRlHzJz8nDy30dYlTHb+IdtHXHwRgz2BcfAu299eX+7FSdCkZMn0MnNosyLhiojLjlTYQQnkH/z9jwti98vKmCwQ2ohVpqh0DlYJiC/6wSKX2PoZeUdClzdhhITVaRPOrhAU1OC+fvuKKRffZwoH71YUWoa6GBw81rYcSUKmy89KjHYORvyFGnZeahVUx/NHWpWWB0KtHExQ13rGgiNT8Xea9EY3d4Z9+OSsfd6NABgVq+KeXxXnKJm+y6Yh4vfMcpjsENqobjp/3vWt8J7LR1KXGPoZeXtJMxOxqTO6ljWKJRWWT+8o9s7Y8eVKBy9E4eYpAzYFTMM/b9HWDaV8ihJIpHgw3ZO+Hr/HWy9/Bgfezjhx2OhECK/zCb2NSu8zJcVzPbNR+TlwxmUSS24WBji1a85TYkEiwY1Qu9GNkoFOkRUste5zIa7jTHa1TZDnkxgW0DRyzZkZOfh1L3yrYWljMHNa8FARxNhCalY7fcAJ+/FQ0MCzHinclt1gOo32/abisEOqYVnKdl4OdrhFwJRxXvdP7yftHcGAOy4EoXMnMLLJpwNSUB6dh7sTfXRxF71WZuVZaSnjUHN8xdoXXY0f1maYS0d4GpVuKWrMoxo7YgLc7thx7h2uDC3m0rzf1E+PsaiClfciKjKkp0rw6y/bkAIoIe7FT7rVJt9Zogqyevsm9azvjXsTPQQI83EwRsxGNZKcRh6wVpY/ZrYVtpoqAKWNRRHZda2NKzU8l7FR+Tlw5YdNRQdHY0PP/wQ5ubm0NfXR+PGjREUFAQAyMnJwZw5c9C4cWMYGhrCzs4OH3/8MWJiYko85rlz59C/f3/Y2dlBIpFg3759CtsLjuvo6o5alqbo3Kwe6nboh9X/BFbWacqt8XuA+3EpMDXQxrL3msCjjjm/FIgqUcGK6ZX9/0xLUwMfeTgDADa9shp6enYuTt3/9xFW48p7hAXk38D9ejpMIW3Z0RCVF0ulqsNgR80kJiaiQ4cO0NbWxpEjR3D37l389NNPMDXNn2grPT0d165dw/z583Ht2jXs2bMHISEhGDBgQInHTUtLQ9OmTfHbb78VuT09PR3+V4KQ2WgQbEf/AstB85D9IhrTxo6s1C+E0PgU+ZfQNwMawrwGV/0lUifvt3aArpYG7sQk4+rjRHn66fsJyMyRwdHMAI1qVe4yP0UNgMgTAo+epVdquVRx+BhLzSxduhQODg7w9fWVp7m4uMj/bWJighMnTijkWbVqFdq0aYPIyEg4Ohb9LLhPnz7o06dPseWamJjggwVrsfTf59kAYPbORMRtmYHLN0MxuFPTsp5SsfJkArP+uomcPIGe9a0woGnl3t0R0etnaqiDgc3s8GfQE2y69AitnPOHoR9+jY+wOCLqzceWHTVz4MABtGrVCsOGDYOVlRWaN2+O9evXl5hHKpVCIpGgZs2aZSozKzcPCw/eVQh0AECWlQ5AAoMalXPXtfFCBG5EJcFIVwvfDWpc6V94RFQ1Rv/bUfno7TjESTORlpWL0/cTAADvVsJEgq/iiKg3H1t21MzDhw+xevVqzJgxA/PmzUNgYCCmTJkCHR0djB49utD+mZmZmDNnDj744IMyrfgenpCCz3dcx73YZABA+zrmuPzwOfJyspF01hcGDTpj4bEItHC1U2rZBWU9epaGH4/nB1df9qsPGxO9Cjs2EVUvDe1M0MbZDFcevcC2gMdwszZCVq4MLhaGaGBbuY+wCnDS0Dcbgx01I5PJ0KpVKyxevBgA0Lx5c9y+fRtr1qwpFOzk5ORg+PDhEEJg9erVKpUjhMD2gEgsPHQHmTkymBvq4IdhTdDd3RqRz5IxasRwGFoawnzELEQlZeCzLUHYNb6dUisYl36OAnP+vomsXBk6uJpjhBILBRLRm210e2dcefQC2wMi5cPM+zWu/EdYL+OIqDcXH2OpGVtbWzRo0EAhrX79+oiMjFRIKwh0Hj9+jBMnTqjcqrPG7yHm7b2FzBwZOrlZ4MjUTujubo2cnBxMHTcayc9icf7saWyZ2BU1DbRxIyoJM/68DllR0xyraPuVSAREvIC+tiaWDGnCx1dEbwHPhtawNdHD87RsnAl5CqBy1sIi9cRgR8106NABISGKfWdCQ0Ph5OQk/7sg0AkLC8PJkydhbm6u9PH9HzwHAARHJkJbU4Iv+9bH5jFtYGWsh5ycHPTv3x9nzpxBVFQU7O3tMaBbO0xrogFtTQn+uRWHZcfu4+uvv4atrS309fXRs2dPhIWFlVimj48PWrduDSMjI1hYWmHS6A+Q8/wJZveuBwez/A6CEyZMQJ06daCvrw9LS0sMHDgQ9+/fV/q8iKh609bUKDRx4I2opKqpDL1xGOyomenTp+Py5ctYvHgxwsPDsX37dqxbtw5eXl4A8gOd9957D0FBQdi2bRvy8vIQFxeHuLg4ZGdny4/To0cPrFq1Sv53ojQZ03/fh6HfbwcAmOQmwqeLEXo5a0FDQ4KcnBwMGDAAJ0+eRJcuXbBt2zb4+fnhyy+/RNt6tbBkSBMAwNKly/DTil+wZs0aBAQEwNDQEL169UJmZmax5+Tn5wcvLy/4+/vDY/Jy5ObmIHHPArzXxFK+T8uWLeHr64t79+7h2LFjEELA09MTeXmFZ10lojdPrDQDJ/5d3bzAl3tvc64bUo4gIZVKBQAhlUqruioV4uDBg6JRo0ZCV1dXuLu7i3Xr1sm3RURECABFvs6cOSPfz8nJSSxYsEAIIcTjZ2miw5SVReYZPXq00sf98eg9oWloKsy7fyouhj0VQgiRlJQkdHV1xY4dO0o9r7+CooTTnEPCZdoOAUD4+fkVu++NGzcEABEeHq7am0dE1dLF8KfCac6hQq9L4c+qumpUhZT9/WYHZTX07rvv4t133y1ym7Ozs8IspMXxv3EPEc/SsOliBH48HopU/dpovOAofIY0KfI5ubOzM+rXr49evXrhyZMn8PPzQ61atTBp0iR07doVADDIVRtfpCVCx7EZJm69ij2TOsDVygRt27aFv78/3n///WLrk5CSiYWH7gIAPm5liW8BmJmZFblvWloafH194eLiAgcHdl4mUgec64bKg4+xqJBdgZHosOQ0Rq4PwDcH7yI1KxetnU1xZFrnEjsEFgx7d3Nzw7Fjx/C///0PU6ZMwebNmwEA8fH5TdDN6jkhOTMXYzZdwfPULFhbWyMuLq7EOn297w6kGTloaFsDgduXo0OHDmjUqJHCPr///jtq1KiBGjVq4MiRIzhx4gR0dHTK+W4QUXXAuW6oPCRCmdv8l5w5cwbdunWrrPpUieTkZJiYmEAqlZZprhl1IU3PweFbMZi397ZCugTAudld4WBW8sJ3Ojo6aNWqFS5duiRPmzJlCgIDA+Hv749Lly6hQ4cOuB0Wgf/tiUDki3S0dDIFTq2AlqYGdu3aVeRx/7kVi0nbrkFLQ4JWT/7G5XOncOHCBdjb2yvWXypFQkICYmNj8eOPPyI6OhoXL16Enh7n4CFSF7HSDM51Q3LK/n6r/Bird+/esLe3x5gxYzB69Gg+JniDZefKcC0yERfCnuF8+DPcepJUaP0XIL/jzZPEzFKDneKGvf/9998AABsbGwBATmoSNn7SGoN/v4irjxORc+chhr7TochjJqZl4+v9+cGX9e1tuBh8DufOnSsU6AD5S1aYmJjAzc0N7dq1g6mpKfbu3YsPPvigtLeCiN4QnOuGykLlx1jR0dGYPHky/vrrL9SuXRu9evXCn3/+qTCSh6perDQDlx48UxipIIRAaHwKNlyIwBjfK2i28DjeX3cZq86E40ZUfqDjbG6AV2etUfa5eGnD3l1cXGBjY4NTp07B1aoG1n7YEho5GYgJu4XEGs5FHnPhobt4mpKFvAv/h0dXz+L06dMKa30VRwgBIQSysrJK3ZeIiNSbyi07FhYWmD59OqZPn45r167B19cXkyZNwqRJkzBy5EiMHTsWTZtW/KKPpLxdgZHw3nMLMgFoSIDhrRyQkydwIfwp4pMVf/wtauigg6sFOrpaoKObBWxN9LErMBLz9txGnhAqPRefPn062rdvj8WLF2P48OG4cuUK1q1bh3Xr1gEAJBIJpk2bhu+++w5ubm5wcXGBedB6RNUww7lsF/x99QmGtrRHjx49MHjwYDTsORx7g6OReGI1xIMLOHTgAIyMjOT9e0xMTKCvr4+HDx9i165d8PT0hKWlJZ48eYIlS5ZAX18fffv2rfg3mIiI3igq99l5VUxMDNatW4clS5ZAS0sLmZmZ8PDwwJo1a9CwYcOKqmelUqc+O7HSDHRYcrrIx1EAoKulgTYuZujoaoFObpZwtzGChkbhGYjL+lz80KFD8Pb2RlhYGFxcXDBjxgyMGzdOvl0IgQULFmDdunVISkpCx44d0ez9mfgrXAZtTQm2jm2LEd1b4oMPP8JZw66IlWbi8dKiR5b5+vrik08+QUxMDD777DNcvXoViYmJsLa2RufOnfH111+jXr16StediIjeLMr+fpcp2MnJycH+/fuxceNGnDhxAq1atcLYsWPxwQcf4OnTp/jqq69w7do13L17t1wnAeQ/NpszZw6OHDmC9PR0uLq6wtfXF61atQLw34/n+vXrkZSUhA4dOshHBClLnYKdSw+eYeT6gELp/ZvaYkQrR7RyNq2Q9akqkkwmMHnHNfxzKw41DbSx7qOWWOP3EKfvJ8DJ3ABHp3aGvk71qjMREVW9Suug/Pnnn2PHjh0QQuCjjz7CsmXLFIYAGxoa4scff4SdnV3Zav6SxMREdOjQAd26dcORI0dgaWmJsLAwmJqayvdZtmwZVq5cic2bN8PFxQXz589Hr169cPfu3bdyFE5xc1HM61u/2nbq09CQYPnwZohOuowbUUkYvvayfFuvBjYMdIiIqFxUbtnp0aMHPvvsMwwZMgS6urpF7pObm4uLFy+iS5cu5arc3LlzcfHiRZw/f77I7UII2NnZYebMmfjiiy8A5A8/tra2xqZNm0qcpO5l6tSyAwDTdgZj3/UYAP/NRTGitWMV16p0t6OlePfXCwppmhIJLsztVm0DNSIiqjrK/n6rPBrr1KlT+OCDD4oNdABAS0ur3IEOABw4cACtWrXCsGHDYGVlhebNm2P9+vXy7REREYiLi0PPnj3laSYm/83IW5ysrCwkJycrvNRJSmYuAGBEawdcmNvtjQh0ACA5M6dQWp4QePQsvQpqQ0RE6kLlYMfHxwcbN24slL5x40YsXbq0QipVoLQZeQtG5VhbWyvkK21GXh8fH/mcLCYmJmo1V1BWbh4u/bsy+UftnN6oFpGCR3Av43TwRERUXioHO2vXroW7u3uh9IYNG2LNmjUVUqkCMpkMLVq0wOLFi9G8eXOMHz8e48aNK3c53t7ekEql8ldUVFQF1bjqBUYkIiMnD5ZGumho92Y9kuN08EREVBlU7qAcFxcHW9vC6yNZWloiNja2QipVQNkZeePj4xXqFB8fj2bNmhV7XF1d3RIfw73JzoYkAAC61LWERFJ4SHl1N6K1IzrXteR08EREVGFUbtlxcHDAxYsXC6VfvHixQkZgvUyVGXkLJCcnIyAgAB4eHhValzfF2dCnAICu9SyruCZlZ2uiD4865gx0iIioQqjcsjNu3DhMmzYNOTk56N69O4D8TsuzZ8/GzJkzK7RyZZmRd/78+bCzs8OgQYMqtC5vgieJ6QhPSIWGBOjk+uYGO0RERBVJ5WBn1qxZeP78OSZNmiRfD0tPTw9z5syBt7d3hVaudevW2Lt3L7y9vbFw4UK4uLjg559/xqhRo+T7zJ49G2lpaRg/frx8Rt6jR4++lXPsnA3Jb9Vp4WgKEwPtKq4NERFR9VDm5SJSU1Nx79496Ovrw83N7Y3uA6Mu8+x8tjkIJ+/F4wvPupjcXfkZpImIiN5ElTaDcoEaNWqgdevWZc1OFSx/yPkzAEDXelZVXBsiIqLqo0zBTlBQEP78809ERkbKH2UV2LNnT4VUjFQT9CgR6dl5sKihiwa2b27rFBERUUVTeTTWzp070b59e9y7dw979+5FTk4O7ty5g9OnT8PExKQy6khKeHnIeVGrmBMREb2tVA52Fi9ejBUrVuDgwYPQ0dHBL7/8gvv372P48OFwdHwzliVQRwWdk7u8wUPOiYiIKoPKwc6DBw/Qr18/AICOjg7S0tIgkUgwffp0+ZBwer2ikzIQ9u+Q885uFlVdHSIiompF5WDH1NQUKSkpAIBatWrh9u3bAICkpCSkp3PBxqrg92+rTjOHmqhpoFPFtSEiIqpeVO6g3LlzZ5w4cQKNGzfGsGHDMHXqVJw+fRonTpxAjx49KqOOVIqC/jochUVERFSYysHOqlWrkJmZCQD48ssvoa2tjUuXLmHo0KH46quvKryCVLLsXBkuhhcMOWd/HSIiolepFOzk5ubi0KFD6NWrFwBAQ0MDc+fOrZSKkXKCHr9AWnYeLGrooJEdR8MRERG9SqU+O1paWpg4caK8ZYeqXkF/nc5uHHJORERUFJU7KLdp0wbXr1+vhKpQWXDIORERUclU7rMzadIkzJgxA1FRUWjZsiUMDQ0Vtjdp0qTCKkcli0nKQEh8yr9DzhnsEBERFUXlYOf9998HAEyZMkWeJpFIIISARCJBXl5exdWOSuQXmt+q09ShJkwNOeSciIioKCoHOxEREZVRDyqDl5eIICIioqKpHOw4OTlVRj1IRTl5MlwMfw6A8+sQERGVROVgZ8uWLSVu//jjj8tcGVLe1ceJSM3KhZmhDprU4pBzIiKi4qgc7EydOlXh75ycHKSnp0NHRwcGBgYMdl6Ts/Ih5xYcck5ERFQClYeeJyYmKrxSU1MREhKCjh07YseOHZVRRyoCl4ggIiJSjsrBTlHc3NywZMmSQq0+VDnipJm4H5cCiQTozM7JREREJaqQYAfIn105Jiamog5HJfALzW/VaWJfE2Ycck5ERFQilfvsHDhwQOFvIQRiY2OxatUqdOjQocIqRsUr6K/Tla06REREpVI52Bk0aJDC3xKJBJaWlujevTt++umniqoXFSMnT4YLYVzlnIiISFkqBzsymawy6kFKuvY4ESlZuTA10EYT+5pVXR0iIqJqr8L67NDrcfbfJSI6uVlCk0POiYiISqVysDN06FAsXbq0UPqyZcswbNiwCqkUFc+voL8OH2EREREpReVg59y5c+jbt2+h9D59+uDcuXMVUikqWkJyJu7GJgPgkHMiIiJlqRzspKamQken8HBnbW1tJCcnV0ilqGgFj7Ca2JvAooZuFdeGiIjozaBysNO4cWPs2rWrUPrOnTvRoEGDCqkUFc2PQ86JiIhUpvJorPnz52PIkCF48OABunfvDgA4deoUduzYgd27d1d4BSlfbp4M58Pyg50uXCKCiIhIaSoHO/3798e+ffuwePFi/PXXX9DX10eTJk1w8uRJdOnSpTLqSACCo5KQnJmLmgbaaOZQs6qrQ0RE9MZQOdgBgH79+qFfv34VXRcqQcHCnxxyTkREpBqV++wEBgYiICCgUHpAQACCgoIqpFLqZsmSJZBIJJg2bZpCur+/P7p37w5DQ0MYGxujc+fOyMjIKPIYZ0OeQur/J4589wmMjIxgZWWFQYMGISQkRGG/devWoWvXrjA2NoZEIkFSUlIlnRUREdGbQeVgx8vLC1FRUYXSo6Oj4eXlVSGVUieBgYFYu3YtmjRpopDu7++P3r17w9PTE1euXEFgYCAmT54MDY3CH0lCSibuxCQjM+o2pn4+GZcvX8aJEyeQk5MDT09PpKWlyfdNT09H7969MW/evEo/NyIiojeByo+x7t69ixYtWhRKb968Oe7evVshlVIXqampGDVqFNavX4/vvvtOYdv06dMxZcoUzJ07V55Wr169Io9TMAqrx/Sf8fnETvL0TZs2wcrKClevXkXnzp0BQN56dPbs2Qo8EyIiojeXyi07urq6iI+PL5QeGxsLLa0ydQFSW15eXujXrx969uypkJ6QkICAgABYWVmhffv2sLa2RpcuXXDhwoUij1Mwv07XuoqjsKRSKQDAzMysEmpPRESkHlQOdjw9PeHt7S3/oQWApKQkzJs3D++8806FVu5NtnPnTly7dg0+Pj6Ftj18+BAA8M0332DcuHE4evQoWrRogR49eiAsLExh39xiVjmXyWSYNm0aOnTogEaNGlXimRAREb3ZVG6K+fHHH9G5c2c4OTmhefPmAIDr16/D2toaf/zxR4VX8E0UFRWFqVOn4sSJE9DT0yu0vWDl+AkTJmDMmDEA8h8Dnjp1Chs3blQIkG48SYI0IwfGeloKQ869vLxw+/btYluDiIiIKJ/KwU6tWrVw8+ZNbNu2DTdu3IC+vj7GjBmDDz74ANra2pVRxzfO1atXkZCQoNC3KS8vD+fOncOqVavkI6henXG6fv36iIyMVEg7+29/nU51LaGlmd8QN3nyZBw6dAjnzp2Dvb19ZZ4KERHRG69MnWwMDQ0xfvz4iq6L2ujRowdu3bqlkDZmzBi4u7tjzpw5qF27Nuzs7AoNGw8NDUWfPn0U0s6+tESEEAKff/459u7di7Nnz8LFxaVyT4SIiEgNlLlH8d27dxEZGYns7GyF9AEDBpS7Um86IyOjQv1oDA0NYW5uLk+fNWsWFixYgKZNm6JZs2bYvHkz7t+/j7/++kuep3PXbritXQ/GLfujSz1LeHl5Yfv27di/fz+MjIwQFxcHADAxMYG+vj4AIC4uDnFxcQgPDwcA3Lp1C0ZGRnB0dGRHZiIieiupHOw8fPgQgwcPxq1btyCRSCCEAABIJPmz+ubl5VVsDdXUtGnTkJmZienTp+PFixdo2rQpTpw4gTp16sj3uR8aDlltWzS0M4aVkR5Wr14NAOjatavCsXx9ffHJJ58AANasWYNvv/1Wvq1gSPrL+xAREb1NJKIgWlFS//79oampif/7v/+Di4sLrly5gufPn2PmzJn48ccf0alTp9IPUs0kJyfDxMQEUqkUxsbGVV0duc82B+HkvXiM9nDCtwM54oqIiOhlyv5+qzz03N/fHwsXLoSFhQU0NDSgoaGBjh07wsfHB1OmTClXpek/O65E4uS9/PmMtlx+jF2BkaXkICIioqKoHOzk5eXByMgIAGBhYYGYmBgAgJOTU6EOt1Q2sdIMzNv7XwdnIYB5e24jVlr0ullERERUPJX77DRq1Ag3btyAi4sL2rZti2XLlkFHRwfr1q1D7dq1K6OOb52IZ2l49eFinhB49Cwdtib6VVMpIiKiN5TKwc5XX30lX3hy4cKFePfdd9GpUyeYm5tj165dFV7Bt5Hmv529X01ztjCogtoQERG92VQOdnr16iX/t6urK+7fv48XL17A1NRUPiKLymff9RiFvzUlEiwe0oitOkRERGVQISt3cv6WipOQkom/rz0BAKwe1QI1DXTgbGHAQIeIiKiMuEx5NbPp4iNk58rQwrEmejeyYWsZERFROak8GosqT2pWLv64/BgAMKFLHQY6REREFYDBTjWyIyASKZm5qG1piHfqW1d1dYiIiNSCysHOuXPnkJubWyg9NzcX586dq5BKvY2yc2XYcCECADChc21oaLBVh4iIqCKoHOx069YNL168KJQulUrRrVu3CqnU22j/9WjEJWfCykgXg5rXqurqEBERqQ2Vgx0hRJF9SZ4/fw5DQ8MKqdTbRiYTWHfuIQDg044u0NXSrOIaERERqQ+lR2MNGTIEQP7q5p988gl0dXXl2/Ly8nDz5k20b9++4mv4Fjh9PwFhCakw0tXCyLaOVV0dIiIitaJ0sGNiYgIgv2XHyMgI+vr/zfuio6ODdu3aYdy4cRVfw7fA2nMPAAAj2znCWE+7imtDRESkXpQOdnx9fQEAzs7O+OKLL/jIqoJcffwCgY8SoaOpgU87uFR1dYiIiNSOyn12Zs+erdBn5/Hjx/j5559x/PjxCq3Y22KNX35fncHNa8HaWK+Ka0NERKR+VA52Bg4ciC1btgAAkpKS0KZNG/z0008YOHAgVq9eXeEVVGfhCak4cTceEgkwrjNXjCciIqoMKgc7165dQ6dOnQAAf/31F2xsbPD48WNs2bIFK1eurPAKqrN1//bVeae+NVytalRxbYiIiNSTysFOeno6jIyMAADHjx/HkCFDoKGhgXbt2uHx48cVXkF1FZ+cib3B0QDyl4YgIiKiyqFysOPq6op9+/YhKioKx44dg6enJwAgISEBxsbGFV5BdbXxQgRy8gTaOJuhpZNpVVeHiIhIbakc7Hz99df44osv4OzsjDZt2sDDwwNAfitP8+bNK7yC6ig5MwfbAiIBABO6sK8OERFRZVI52HnvvfcQGRmJoKAgHDt2TJ7eo0cPrFixokIr96olS5ZAIpFg2rRp8rTMzEx4eXnB3NwcNWrUwNChQxEfH1+p9SivbZcjkZqVCzerGuhWz6qqq0NERKTWyrTquY2NDYyMjHDixAlkZGQAAFq3bg13d/cKrdzLAgMDsXbtWjRp0kQhffr06Th48CB2794NPz8/xMTEyGd7ro6ycvOw8eK/C352qcMFP4mIiCqZysHO8+fP0aNHD9StWxd9+/ZFbGwsAGDs2LGYOXNmhVcQAFJTUzFq1CisX78epqb/9W+RSqXYsGEDli9fju7du6Nly5bw9fXFpUuXcPny5UqpS3ntC47G05Qs2JroYUBTu6quDhERkdpTOdiZPn06tLW1ERkZCQMDA3n6iBEjcPTo0QqtXAEvLy/069cPPXv2VEi/evUqcnJyFNLd3d3h6OgIf3//Yo+XlZWF5ORkhdfrIJMJrP13wc+xHV2go1WmhjUiIiJSgdLLRRQ4fvw4jh07Bnt7e4V0Nze3Shl6vnPnTly7dg2BgYGFtsXFxUFHRwc1a9ZUSLe2tkZcXFyxx/Tx8cG3335b0VUt1Yl78Xj4NA3Gelp4vw0X/CQiInodVG5aSEtLU2jRKfDixQuFldArQlRUFKZOnYpt27ZBT6/illLw9vaGVCqVv6Kioirs2MURQmCNX/4kgh95OKGGrspxJhEREZWBysFOp06d5MtFAIBEIoFMJsOyZcvQrVu3Cq3c1atXkZCQgBYtWkBLSwtaWlrw8/PDypUroaWlBWtra2RnZyMpKUkhX3x8PGxsbIo9rq6uLoyNjRVelS3wUSKCI5Ogo6WBT9pzwU8iIqLXReXmhWXLlqFHjx4ICgpCdnY2Zs+ejTt37uDFixe4ePFihVauR48euHXrlkLamDFj4O7ujjlz5sDBwQHa2to4deoUhg4dCgAICQlBZGSkfP6f6qKgVee9lvawNKrYFjAiIiIqnsrBTqNGjRAaGopVq1bByMgIqampGDJkCLy8vGBra1uhlTMyMkKjRo0U0gwNDWFubi5PHzt2LGbMmAEzMzMYGxvj888/h4eHB9q1a1ehdSmPkLgUnL6fkL/gZydOIkhERPQ6qRzsREZGwsHBAV9++WWR2xwdX2/H2xUrVkBDQwNDhw5FVlYWevXqhd9///211qE0a/9d8LN3Qxu4WBhWcW2IiIjeLhIhhFAlg6amJmJjY2FlpTjz7/Pnz2FlZYW8vLwKreDrkJycDBMTE0il0grvvxOTlIHOy84gVyaw36sDmjrUrNDjExERva2U/f1WuYOyEAISSeFZf1NTUyt0xJS62HghArkygXa1zRjoEBERVQGlH2PNmDEDQP7oq/nz5ysMP8/Ly0NAQACaNWtW4RV8k4XGpWDr5fy5hyZ2qVPFtSEiIno7KR3sBAcHA8hv2bl16xZ0dHTk23R0dNC0aVN88cUXFV/DN9SuwEjM/fsWCp4Rxkkzq7Q+REREbyulg50zZ84AyB/6/csvv7yWuWneVLHSDHjv+S/QAYAv995Gl3qWsDXRr7J6ERERvY1U7rPj6+vLQKcUEc/SIHul23eeEHj0LL1qKkRERPQW40qUlcDFwhAar/Th1pRI4GxReJkNIiIiqlwMdiqBrYk+fIY0hua/o9Y0JRIsHtKIj7CIiIiqAFejrCQjWjuic11LPHqWDmcLAwY6REREVYTBTiWyNdFnkENERFTF+BiLiIiI1BqDHSIiIlJrDHaIiIhIrTHYISIiIrXGYIeIiIjUGoMdIiIiUmsMdoiIiEitMdghIiIitcZgh4iIiNQagx0iIiJSawx2iIiISK0x2CEiIiK1xmCHiIiI1BqDHSIiIlJrDHaIiIhIrTHYISIiIrXGYIeIiIjUGoMdIiIiUmsMdoiIiEitMdghIiIitcZgh4iIiNQagx0iIiJSawx2iIiISK0x2CEiIiK1xmCHiIiI1BqDHSIiIlJrDHaIiIhIrTHYISIiIrXGYIeIiIjUGoMdIiIiUmsMdoiIiEitMdghIiIitcZgh4iIiNQagx0iIiJSawx2iIiISK0x2CEiIiK1xmCHiIiI1BqDHSIiIlJrDHaIiIhIrTHYISIiIrXGYIeIiIjUGoMdIiIiUmsMdoiIiEitMdghIiIitcZgh4iIiNQagx0iIiJSawx2iIiISK0x2CEiIiK1xmCHiIiI1BqDHSIiIlJrDHaIiIhIrTHYISIiIrXGYIeIiIjUGoMdIiIiUmvVOtjx8fFB69atYWRkBCsrKwwaNAghISEK+2RmZsLLywvm5uaoUaMGhg4divj4+CqqMREREVU31TrY8fPzg5eXFy5fvowTJ04gJycHnp6eSEtLk+8zffp0HDx4ELt374afnx9iYmIwZMiQKqw1ERERVScSIYSo6koo6+nTp7CysoKfnx86d+4MqVQKS0tLbN++He+99x4A4P79+6hfvz78/f3Rrl07pY6bnJwMExMTSKVSGBsbV+YpEBERUQVR9ve7WrfsvEoqlQIAzMzMAABXr15FTk4OevbsKd/H3d0djo6O8Pf3L/Y4WVlZSE5OVngRERGRenpjgh2ZTIZp06ahQ4cOaNSoEQAgLi4OOjo6qFmzpsK+1tbWiIuLK/ZYPj4+MDExkb8cHBwqs+pERERUhd6YYMfLywu3b9/Gzp07y30sb29vSKVS+SsqKqoCakhERETVkVZVV0AZkydPxqFDh3Du3DnY29vL021sbJCdnY2kpCSF1p34+HjY2NgUezxdXV3o6upWZpWJiIiomqjWLTtCCEyePBl79+7F6dOn4eLiorC9ZcuW0NbWxqlTp+RpISEhiIyMhIeHx+uuLhEREVVD1bplx8vLC9u3b8f+/fthZGQk74djYmICfX19mJiYYOzYsZgxYwbMzMxgbGyMzz//HB4eHkqPxCIiIiL1Vq2HnkskkiLTfX198cknnwDIn1Rw5syZ2LFjB7KystCrVy/8/vvvJT7GehWHnhMREb15lP39rtbBzuvCYIeIiOjNo5bz7BARERGpisEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTW1CbY+e233+Ds7Aw9PT20bdsWV65cqeoqERERUTWgFsHOrl27MGPGDCxYsADXrl1D06ZN0atXLyQkJFR11YiIiKiKqUWws3z5cowbNw5jxoxBgwYNsGbNGhgYGGDjxo1VXTUiIiKqYlpVXYHyys7OxtWrV+Ht7S1P09DQQM+ePeHv719knqysLGRlZcn/lkqlAIDk5OTKrSwRERFVmILfbSFEifu98cHOs2fPkJeXB2tra4V0a2tr3L9/v8g8Pj4++PbbbwulOzg4VEodiYiIqPKkpKTAxMSk2O1vfLBTFt7e3pgxY4b8b5lMhhcvXsDc3BwSiaTCyklOToaDgwOioqJgbGz8WvOz7Ndfdnnzs+y3q+zy5mfZLPtNyV/esksihEBKSgrs7OxK3O+ND3YsLCygqamJ+Ph4hfT4+HjY2NgUmUdXVxe6uroKaTVr1qysKsLY2LhcH3B58rPs1192efOz7Ler7PLmZ9ks+03JX96yi1NSi06BN76Dso6ODlq2bIlTp07J02QyGU6dOgUPD48qrBkRERFVB298yw4AzJgxA6NHj0arVq3Qpk0b/Pzzz0hLS8OYMWOqumpERERUxdQi2BkxYgSePn2Kr7/+GnFxcWjWrBmOHj1aqNPy66arq4sFCxYUemT2OvKz7Ndfdnnzs+y3q+zy5mfZLPtNyV/esiuCRJQ2XouIiIjoDfbG99khIiIiKgmDHSIiIlJrDHaIiIhIrTHYISIiIrXGYKcS/fbbb3B2doaenh7atm2LK1euKJXv3Llz6N+/P+zs7CCRSLBv3z6ly/Tx8UHr1q1hZGQEKysrDBo0CCEhIUrnX716NZo0aSKf/MnDwwNHjhxROv/LlixZAolEgmnTpim1/zfffAOJRKLwcnd3V7q86OhofPjhhzA3N4e+vj4aN26MoKAgpfI6OzsXKlsikcDLy6vUvHl5eZg/fz5cXFygr6+POnXqYNGiRaWu1fKylJQUTJs2DU5OTtDX10f79u0RGBhYaL/Srg0hBL7++mvY2tpCX18fPXv2RFhYmNL59+zZA09PT/ls4tevX1e6/JycHMyZMweNGzeGoaEh7Ozs8PHHHyMmJkapsr/55hu4u7vD0NAQpqam6NmzJwICApSu+8smTpwIiUSCn3/+Wam8n3zySaHPvnfv3iqVfe/ePQwYMAAmJiYwNDRE69atERkZWWreoq47iUSCH374QamyU1NTMXnyZNjb20NfX1++GLIyeePj4/HJJ5/Azs4OBgYG6N27t/x6Uea7JDMzE15eXjA3N0eNGjUwdOhQ+QSvyuRft24dunbtCmNjY0gkEiQlJcm3lZb/xYsX+Pzzz1GvXj3o6+vD0dERU6ZMgVQqVarsCRMmoE6dOtDX14elpSUGDhwoX2JIle9RIQT69Okjf3+Vydu1a9dCn/fEiRNVKtvf3x/du3eHoaEhjI2N0blzZyxcuLDEvI8ePSr2etu9e7dSZcfFxeGjjz6CjY0NDA0N0aJFC/z9999K5X3w4AEGDx4MS0tLGBsbY/jw4YUmBK4sDHYqya5duzBjxgwsWLAA165dQ9OmTdGrVy8kJCSUmjctLQ1NmzbFb7/9pnK5fn5+8PLywuXLl3HixAnk5OTA09MTaWlpSuW3t7fHkiVLcPXqVQQFBaF79+4YOHAg7ty5o1I9AgMDsXbtWjRp0kSlfA0bNkRsbKz8deHCBaXyJSYmokOHDtDW1saRI0dw9+5d/PTTTzA1NVW6vi+Xe+LECQDAsGHDSs27dOlSrF69GqtWrcK9e/ewdOlSLFu2DL/++qtSZQPAZ599hhMnTuCPP/7ArVu34OnpiZ49eyI6Olphv9KujWXLlmHlypVYs2YNAgICYGhoiF69eiEzM1Op/GlpaejYsSOWLl1a7Pbi8qenp+PatWuYP38+rl27hj179iAkJAQDBgxQquy6deti1apVuHXrFi5cuABnZ2d4enri6dOnSuUvsHfvXly+fFlh+nhl8vbu3VvhGtixY4fS+R88eICOHTvC3d0dZ8+exc2bNzF//nzo6emVmvflMmNjY7Fx40ZIJBIMHTpUqbJnzJiBo0ePYuvWrbh37x6mTZuGyZMn48CBAyXmFUJg0KBBePjwIfbv34/g4GA4OTmhZ8+eSEtLU+q7ZPr06Th48CB2794NPz8/xMTEYMiQIQCU+y5KT09H7969MW/evEL1Ky1/TEwMYmJi8OOPP+L27dvYtGkTjh49irFjxypVdsuWLeHr64t79+7h2LFjEELA09MTeXl5Kn2P/vzzzwrLDCmbd9y4cQqf+7Jly5TO7+/vj969e8PT0xNXrlxBYGAgJk+ejAsXLpSY18HBodD19u2336JGjRro06ePUmV//PHHCAkJwYEDB3Dr1i0MGTIEw4cPx8GDB0vMm5aWBk9PT0gkEpw+fRoXL15EdnY2+vfvD5lMVuh9rXCCKkWbNm2El5eX/O+8vDxhZ2cnfHx8VDoOALF3794y1yMhIUEAEH5+fmU+hqmpqfi///s/pfdPSUkRbm5u4sSJE6JLly5i6tSpSuVbsGCBaNq0aZnqOGfOHNGxY8cy5S3K1KlTRZ06dYRMJit13379+olPP/1UIW3IkCFi1KhRSpWVnp4uNDU1xaFDhxTSW7RoIb788sti8716bchkMmFjYyN++OEHeVpSUpLQ1dUVO3bsKDX/yyIiIgQAERwcrHT5Rbly5YoAIB4/fqxyXqlUKgCIkydPKl32kydPRK1atcTt27eFk5OTWLFihVJ5R48eLQYOHFhifUrKP2LECPHhhx+WKe+rBg4cKLp37650/oYNG4qFCxcqpBV17byaNyQkRAAQt2/flqfl5eUJS0tLsX79+kJlv/pdkpSUJLS1tcXu3bvl+9y7d08AEP7+/qXmf9mZM2cEAJGYmFjkeZeWv8Cff/4pdHR0RE5Ojsp5b9y4IQCI8PBwpcsODg4WtWrVErGxscV+tkXlVeV7saj8bdu2FV999VWZ8r6qWbNmhb6/SspvaGgotmzZorCfmZlZoWvm1bzHjh0TGhoaQiqVyvdJSkoSEolEnDhxotRzKS+27FSC7OxsXL16FT179pSnaWhooGfPnvD393+tdZFKpQAAMzMzlfPm5eVh586dSEtLU2npDS8vL/Tr10/h/JUVFhYGOzs71K5dG6NGjUJkZKRS+Q4cOIBWrVph2LBhsLKyQvPmzbF+/XqVywfyP7+tW7fi008/VWph2Pbt2+PUqVMIDQ0FANy4cQMXLlxAnz59lCovNzcXeXl50NPTU0jX19dXumULACIiIhAXF6fwvpuYmKBt27av/borIJVKIZFIVF57Ljs7G+vWrYOJiQmaNm2qVB6ZTIaPPvoIs2bNQsOGDVWu69mzZ2FlZYV69erhf//7H54/f650uYcPH0bdunXRq1cvWFlZoW3btio9fi4QHx+Pw4cPY+zYsUrnad++PQ4cOIDo6GgIIXDmzBmEhobC09OzxHxZWVkAoHDdaWhoQFdXt8jr7tXvkqtXryInJ0fhenN3d4ejo2OR11t5vouUzS+VSmFsbAwtLa1C6SXlTUtLg6+vL1xcXODg4KBU2enp6Rg5ciR+++23YtdhLKnsbdu2wcLCAo0aNYK3tzfS09OVyp+QkICAgABYWVmhffv2sLa2RpcuXZT6zF519epVXL9+vdjrraj87du3x65du/DixQvIZDLs3LkTmZmZ6Nq1a4l5s7KyIJFIFCYW1NPTg4aGhkrfc2VW6eHUWyg6OloAEJcuXVJInzVrlmjTpo1Kx0I5Wnby8vJEv379RIcOHVTKd/PmTWFoaCg0NTWFiYmJOHz4sNJ5d+zYIRo1aiQyMjKEEKrdwfzzzz/izz//FDdu3BBHjx4VHh4ewtHRUSQnJ5eaV1dXV+jq6gpvb29x7do1sXbtWqGnpyc2bdqkdN0L7Nq1S2hqaoro6Gil9s/LyxNz5swREolEaGlpCYlEIhYvXqxSmR4eHqJLly4iOjpa5Obmij/++ENoaGiIunXrFpvn1Wvj4sWLAoCIiYlR2G/YsGFi+PDhpeZ/WUW07GRkZIgWLVqIkSNHKp334MGDwtDQUEgkEmFnZyeuXLmidNmLFy8W77zzjrw1TpWWnR07doj9+/eLmzdvir1794r69euL1q1bi9zc3FLzF9zVGxgYiOXLl4vg4GDh4+MjJBKJOHv2rFLnXWDp0qXC1NRU/v9HmbpnZmaKjz/+WAAQWlpaQkdHR2zevLnUvNnZ2cLR0VEMGzZMvHjxQmRlZYklS5YIAMLT01Mhb1HfJdu2bRM6OjqFymndurWYPXt2qflfVlrLjjLfZU+fPhWOjo5i3rx5Suf97bffhKGhoQAg6tWrV2SrTnH5x48fL8aOHSv/u6jPpri8a9euFUePHhU3b94UW7duFbVq1RKDBw9Wqmx/f38BQJiZmYmNGzeKa9euiWnTpgkdHR0RGhqq1HkX+N///ifq169f5Lbi8icmJgpPT0/59WZsbCyOHTtWat6EhARhbGwspk6dKtLS0kRqaqqYPHmyACDGjx9fbB0rCoOdSlBdgp2JEycKJycnERUVpVK+rKwsERYWJoKCgsTcuXOFhYWFuHPnTqn5IiMjhZWVlbhx44Y8TZVg51WJiYnC2NhYqUdo2trawsPDQyHt888/F+3atVO5XE9PT/Huu+8qvf+OHTuEvb292LFjh7h586bYsmWLMDMzUynQCg8PF507dxYAhKampmjdurUYNWqUcHd3LzZPdQ52srOzRf/+/UXz5s0Vmq1Ly5uamirCwsKEv7+/+PTTT4Wzs7OIj48vNX9QUJCwtrZWCFBVCXZe9eDBA6UfoRX8f//ggw8U9uvfv794//33VSq7Xr16YvLkycVuLyr/Dz/8IOrWrSsOHDggbty4IX799VdRo0aNQo8GisobFBQkmjZtKr/uevXqJfr06SN69+6tsF9R3yWqBDulfReVFuyUll8qlYo2bdqI3r17i+zsbKXzJiUlidDQUOHn5yf69+8vWrRoUSjQLCr//v37haurq0hJSZGnFfX+KvsdfOrUqSIfoRWVv+D/ube3t8K+jRs3FnPnzlW67PT0dGFiYiJ+/PHHIrcXl3/y5MmiTZs24uTJk+L69evim2++ESYmJuLmzZul5j127JioXbu2kEgkQlNTU3z44YeiRYsWYuLEiSW8OxWDwU4lyMrKEpqamoUu/I8//lgMGDBApWOVNdjx8vIS9vb24uHDhyrnfVWPHj2Uirz37t0r/9IseAGQX9hF3SWXplWrVgr/gYvj6OiocJclhBC///67sLOzU6m8R48eCQ0NDbFv3z6l89jb24tVq1YppC1atEjUq1dPpbKFyP+xLwhWhg8fLvr27Vvsvq9eGwU/0K8GKJ07dxZTpkwpNf/LyhPsZGdni0GDBokmTZqIZ8+eqZT3Va6urkW2kr2af8WKFfLr7OVrT0NDQzg5OZWpbAsLC7FmzZpSy87KyhJaWlpi0aJFCvvNnj1btG/fXumyz507JwCI69evF1unV/Onp6cLbW3tQv29xo4dK3r16qV02UlJSSIhIUEIkd/fcNKkSfJtxX2XFPxAvxqgODo6iuXLl5ea/2UlBTul5U9OThYeHh6iR48ehQIVVb4Hs7KyhIGBgdi+fXup+adOnVrs9dalSxeVy05NTRUAxNGjR0st++HDhwKA+OOPPxTShw8fLm9FVabsLVu2CG1tbfnn/rLi8oeHhxfq5yVE/m/EhAkTlC776dOn8s/a2tpaLFu2rNh9Kwr77FQCHR0dtGzZEqdOnZKnyWQynDp1SqW+L2UhhMDkyZOxd+9enD59Gi4uLuU+pkwmkz/fL0mPHj1w69YtXL9+Xf5q1aoVRo0ahevXr0NTU1OlclNTU/HgwQPY2tqWum+HDh0KDXMMDQ2Fk5OTSmX6+vrCysoK/fr1UzpPeno6NDQU/ytpamqWaYSBoaEhbG1tkZiYiGPHjmHgwIFK53VxcYGNjY3CdZecnIyAgIBKv+4K5OTkYPjw4QgLC8PJkydhbm5eruMpe+199NFHuHnzpsK1Z2dnh1mzZuHYsWMql/vkyRM8f/5cqWtPR0cHrVu3Lvf1t2HDBrRs2VLpPkpA/vudk5NT7uvPxMQElpaWCAsLQ1BQEAYOHFjqd0nLli2hra2tcL2FhIQgMjISHh4e5f4uUiZ/cnIyPD09oaOjgwMHDsj7H5WlbJF/84+srKxS88+dO7fQ9QYAK1aswMaNG1UuuyC/ra1tqWU7OzvDzs6uyOvN0dFR6bI3bNiAAQMGwNLSUuE9KCl/Qb+ioq63vLw8pcu2sLBAzZo1cfr0aSQkJMhHbFaqSg+n3lI7d+4Uurq6YtOmTeLu3bti/PjxombNmiIuLq7UvCkpKSI4OFgEBwcLAPJ+AK+OaCnK//73P2FiYiLOnj0rYmNj5a/09HSl6j137lzh5+cnIiIixM2bN8XcuXOFRCIRx48fVyr/q1R5jDVz5kxx9uxZERERIS5evCh69uwpLCwsirzzeNWVK1eElpaW+P7770VYWJjYtm2bMDAwEFu3blW6rnl5ecLR0VHMmTNH6TxC5I/kqVWrljh06JCIiIgQe/bsERYWFoWa8kty9OhRceTIEfHw4UNx/Phx0bRpU9G2bdtCTfKlXRtLliwRNWvWlPc/GThwoHBxcZHf8ZaW//nz5yI4OFgcPnxYABA7d+4UwcHBIjY2ttT82dnZYsCAAcLe3l5cv35d4frLysoqMW9qaqrw9vYW/v7+4tGjRyIoKEiMGTNG6Orqyu8iVf1/8fJjrJLypqSkiC+++EL4+/uLiIgIcfLkSdGiRQvh5uYmMjMzlSp7z549QltbW6xbt06EhYWJX3/9VWhqaorz588rVW+pVCoMDAzE6tWrC51Hafm7dOkiGjZsKM6cOSMePnwofH19hZ6envj9999Lzfvnn3+KM2fOiAcPHoh9+/YJJycnMWTIECGEct8lEydOFI6OjuL06dMiKChIeHh4yB8nK5M/NjZWBAcHi/Xr1wsA4ty5cyI4OFg8f/681PxSqVS0bdtWNG7cWISHhyvsM3HixBLzPnjwQCxevFgEBQWJx48fi4sXL4r+/fsLMzMzER8fX6bvUfzbclZa3vDwcLFw4UIRFBQkIiIixP79+0Xt2rVF586dlX7fVqxYIYyNjcXu3btFWFiY+Oqrr4Senp4YOXKkUvUOCwsTEolEHDlyRCG9tLKzs7OFq6ur6NSpkwgICBDh4eHixx9/FBKJRPTt27fUsjdu3Cj8/f1FeHi4+OOPP4SZmZmYMWNGse9pRWKwU4l+/fVX4ejoKHR0dESbNm3E5cuXlcpX0KT76mv06NGl5i0qHwDh6+urVNmffvqpcHJyEjo6OsLS0lL06NGjzIGOEKoFOyNGjBC2trZCR0dH1KpVS4wYMaLIDoPFOXjwoGjUqJHQ1dUV7u7uYt26dSrV9dixYwKACAkJUSlfcnKymDp1qnB0dBR6enqidu3a4ssvvxRZWVlKH2PXrl2idu3aQkdHR9jY2AgvLy+RlJRUaL/Srg2ZTCbmz58vrK2tha6urujRo4fC+ZSW39fXt8jtCxYsKDV/waOvol5nzpwpMW9GRoYYPHiwsLOzEzo6OsLW1lYMGDBAoYOyqv8vXg52Ssqbnp4uPD09haWlpdDW1hZOTk5i3LhxCjcmypS9YcMG4erqKvT09ETTpk3lj0KVybt27Vqhr69fps88NjZWfPLJJ8LOzk7o6emJevXqiZ9++knIZLJS8/7yyy/C3t5eaGtrC0dHR/HVV1/Jr1tlvksyMjLEpEmThKmpqTAwMBCDBw+WB8bK5F+wYEGx+5SWv7hzK+lVkDc6Olr06dNHWFlZCW1tbWFvby9Gjhwp7t+/r3TdX1UQ7JSWNzIyUnTu3FmYmZkJXV1d4erqKmbNmiXv26Zs2T4+PsLe3l4YGBgIDw8Pcf78eaXzent7CwcHB5GXl1foHErLHxoaKoYMGSKsrKyEgYGBaNKkidiyZYtSeefMmSOsra2Ftra2cHNzk1+nr4Pk3xMkIiIiUkvss0NERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BARveLs2bOQSCRISkqq6qoQUQVgsENERERqjcEOERERqTUGO0RU7chkMvj4+MDFxQX6+vpo2rQp/vrrLwD/PWI6fPgwmjRpAj09PbRr1w63b99WOMbff/+Nhg0bQldXF87Ozvjpp58UtmdlZWHOnDlwcHCArq4uXF1dsWHDBoV9rl69ilatWsHAwADt27cvtNI0Eb0ZGOwQUbXj4+ODLVu2YM2aNbhz5w6mT5+ODz/8EH5+fvJ9Zs2ahZ9++gmBgYGwtLRE//79kZOTAyA/SBk+fDjef/993Lp1C9988w3mz5+PTZs2yfN//PHH2LFjB1auXIl79+5h7dq1qFGjhkI9vvzyS/z0008ICgqClpYWPv3009dy/kRUsbgQKBFVK1lZWTAzM8PJkyfh4eEhT//ss8+Qnp6O8ePHo1u3bti5cydGjBgBAHjx4gXs7e2xadMmDB8+HKNGjcLTp09x/Phxef7Zs2fj8OHDuHPnDkJDQ1GvXj2cOHECPXv2LFSHs2fPolu3bjh58iR69OgBAPjnn3/Qr18/ZGRkQE9Pr5LfBSKqSGzZIaJqJTw8HOnp6XjnnXdQo0YN+WvLli148OCBfL+XAyEzMzPUq1cP9+7dAwDcu3cPHTp0UDhuhw4dEBYWhry8PFy/fh2ampro0qVLiXVp0qSJ/N+2trYAgISEhHKfIxG9XlpVXQEiopelpqYCAA4fPoxatWopbNPV1VUIeMpKX19fqf20tbXl/5ZIJADy+xMR0ZuFLTtEVK00aNAAurq6iIyMhKurq8LLwcFBvt/ly5fl/05MTERoaCjq168PAKhfvz4uXryocNyLFy+ibt260NTUROPGjSGTyRT6ABGR+mLLDhFVK0ZGRvjiiy8wffp0yGQydOzYEVKpFBcvXoSxsTGcnJwAAAsXLoS5uTmsra3x5ZdfwsLCAoMGDQIAzJw5E61bt8aiRYswYsQI+Pv7Y9WqVfj9998BAM7Ozhg9ejQ+/fRTrFy5Ek2bNsXjx4+RkJCA4cOHV9WpE1ElYbBDRNXOokWLYGlpCR8fHzx8+BA1a9ZEixYtMG/ePPljpCVLlmDq1KkICwtDs2bNcPDgQejo6AAAWrRogT///BNff/01Fi1aBFtbWyxcuBCffPKJvIzVq1dj3rx5mDRpEp4/fw5HR0fMmzevKk6XiCoZR2MR0RulYKRUYmIiatasWdXVIaI3APvsEBERkVpjsENERERqjY+xiIiISK2xZYeIiIjUGoMdIiIiUmsMdoiIiEitMdghIiIitcZgh4iIiNQagx0iIiJSawx2iIiISK0x2CEiIiK1xmCHiIiI1Nr/A45Ofu2hDDlOAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plt.plot(np.arange(len(epochs_x)), epochs_acc, marker = '.')\n", + "plt.xlabel('epoch')\n", + "plt.ylabel('test accuracy')\n", + "plt.ylim(0, 100)\n", + "plt.xticks(np.arange(len(epochs_x)))\n", + "for i, txt in enumerate(epochs_acc):\n", + " if i%2 == 0:\n", + " pass\n", + " else:\n", + " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", + "plt.show()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "speck-rescnn", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tests/test_nonsequential/exp_set_A/non-sequential-SCNN-example_3.ipynb b/tests/test_nonsequential/exp_set_A/non-sequential-SCNN-example_3.ipynb new file mode 100644 index 00000000..956dfb88 --- /dev/null +++ b/tests/test_nonsequential/exp_set_A/non-sequential-SCNN-example_3.ipynb @@ -0,0 +1,1509 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import torch, random\n", + "import torch.nn as nn\n", + "import sinabs.layers as sl\n", + "from tqdm.notebook import tqdm\n", + "\n", + "from tonic.datasets.dvsgesture import DVSGesture\n", + "from tonic.transforms import ToFrame\n", + "from torch.utils.data import DataLoader\n", + "from torch.nn import CrossEntropyLoss\n", + "from torch.optim import Adam\n", + "\n", + "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "torch.backends.cudnn.deterministic = True\n", + "random.seed(1)\n", + "torch.manual_seed(1)\n", + "torch.cuda.manual_seed(1)\n", + "np.random.seed(1)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Loading Data" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "batch_size = 3\n", + "num_workers = 1\n", + "epochs = 30\n", + "lr = 1e-3" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "root_dir = \"./DVSGESTURE\"\n", + "_ = DVSGesture(save_to=root_dir, train=True)\n", + "_ = DVSGesture(save_to=root_dir, train=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "n_time_steps = 50\n", + "to_raster = ToFrame(sensor_size=DVSGesture.sensor_size, n_time_bins=n_time_steps)\n", + "\n", + "snn_train_dataset = DVSGesture(save_to=root_dir, train=True, transform=to_raster)\n", + "snn_test_dataset = DVSGesture(save_to=root_dir, train=False, transform=to_raster)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The transformed array is in shape [Time-Step, Channel, Height, Width] --> (50, 2, 128, 128)\n" + ] + } + ], + "source": [ + "sample_data, label = snn_train_dataset[0]\n", + "print(f\"The transformed array is in shape [Time-Step, Channel, Height, Width] --> {sample_data.shape}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "snn_train_dataloader = DataLoader(snn_train_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True)\n", + "snn_test_dataloader = DataLoader(snn_test_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=False)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Network Module\n", + "\n", + "We need to define a `nn.Module` implementing the network we want the chip to reproduce." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "device: NVIDIA RTX A4000\n" + ] + } + ], + "source": [ + "if torch.cuda.is_available():\n", + " device = torch.device('cuda:0')\n", + " print('device: ', torch.cuda.get_device_name(0))\n", + "else:\n", + " device = torch.device('cpu')" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "class SNN(nn.Module):\n", + " def __init__(self) -> None:\n", + " super().__init__()\n", + "\n", + " self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False)\n", + " self.iaf1 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + " self.pool1 = nn.AvgPool2d(2,2)\n", + " self.pool1a = nn.AvgPool2d(6,6)\n", + "\n", + " self.conv2 = nn.Conv2d(10, 10, 2, 1, bias=False)\n", + " self.iaf2 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + " self.pool2 = nn.AvgPool2d(3,3)\n", + "\n", + " self.conv3 = nn.Conv2d(10, 10, 3, 1, bias=False)\n", + " self.iaf3 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + " self.pool3 = nn.AvgPool2d(2,2)\n", + "\n", + " self.flat = nn.Flatten()\n", + "\n", + " self.fc1 = nn.Linear(810, 100, bias=False)\n", + " self.iaf4 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + "\n", + " self.fc2 = nn.Linear(100, 100, bias=False)\n", + " self.iaf5 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + "\n", + " self.fc3 = nn.Linear(100, 100, bias=False)\n", + " self.iaf6 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + "\n", + " self.fc4 = nn.Linear(100, 11, bias=False)\n", + " self.iaf7 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + "\n", + " self.merge_fc = sl.Merge()\n", + " self.merge_conv = sl.Merge()\n", + "\n", + " def detach_neuron_states(self):\n", + " for name, layer in self.named_modules():\n", + " if name != '':\n", + " if isinstance(layer, sl.StatefulLayer):\n", + " for name, buffer in layer.named_buffers():\n", + " buffer.detach_()\n", + "\n", + " def init_weights(self):\n", + " for name, layer in self.named_modules():\n", + " if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear):\n", + " nn.init.xavier_normal_(layer.weight.data)\n", + "\n", + " def forward(self, x):\n", + " \n", + " con1_out = self.conv1(x)\n", + " iaf1_out = self.iaf1(con1_out)\n", + " pool1_out = self.pool1(iaf1_out)\n", + " pool1a_out = self.pool1a(iaf1_out)\n", + "\n", + " conv2_out = self.conv2(pool1_out)\n", + " iaf2_out = self.iaf2(conv2_out)\n", + " pool2_out = self.pool2(iaf2_out)\n", + "\n", + " merged_conv_out = self.merge_conv(pool1a_out, pool2_out)\n", + "\n", + " conv3_out = self.conv3(merged_conv_out)\n", + " iaf3_out = self.iaf3(conv3_out)\n", + " pool3_out = self.pool3(iaf3_out)\n", + "\n", + " flat_out = self.flat(pool3_out)\n", + " \n", + " fc1_out = self.fc1(flat_out)\n", + " iaf4_out = self.iaf4(fc1_out)\n", + "\n", + " fc2_out = self.fc2(iaf4_out)\n", + " iaf5_out = self.iaf5(fc2_out)\n", + "\n", + " fc3_out = self.fc3(iaf5_out)\n", + " iaf6_out = self.iaf6(fc3_out)\n", + "\n", + " merge_fc_out = self.merge_fc(iaf4_out, iaf6_out)\n", + "\n", + " fc4_out = self.fc4(merge_fc_out)\n", + " iaf7_out = self.iaf7(fc4_out)\n", + "\n", + " return iaf7_out" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "snn = SNN().to(device)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "snn.init_weights()" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "optimizer = Adam(snn.parameters(), lr=lr, betas=(0.9, 0.999), eps=1e-8)\n", + "loss_fn = CrossEntropyLoss()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Define train and test" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "def train(dataloader, model, loss_fn, optimizer, epochs, test_func, dataloader_test):\n", + " epochs_y = []\n", + " epochs_x = []\n", + " epochs_acc = []\n", + " model.train()\n", + "\n", + " for e in range(epochs):\n", + " losses = []\n", + " batches = []\n", + " batch_count = 0\n", + " train_p_bar = tqdm(snn_train_dataloader)\n", + "\n", + " for X, y in train_p_bar:\n", + " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", + " X = X.reshape(-1, 2, 128, 128).to(dtype=torch.float, device=device)\n", + " y = y.to(dtype=torch.long, device=device)\n", + "\n", + " # forward\n", + " pred = model(X)\n", + "\n", + " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", + " pred = pred.reshape(batch_size, n_time_steps, -1)\n", + "\n", + " # accumulate all time-steps output for final prediction\n", + " pred = pred.sum(dim = 1)\n", + " loss = loss_fn(pred, y)\n", + "\n", + " # gradient update\n", + " optimizer.zero_grad()\n", + " loss.backward()\n", + " optimizer.step()\n", + "\n", + " # detach the neuron states and activations from current computation graph(necessary)\n", + " model.detach_neuron_states()\n", + "\n", + " train_p_bar.set_description(f\"Epoch {e} - BPTT Training Loss: {round(loss.item(), 4)}\")\n", + "\n", + " batch_count += 1\n", + " losses.append(loss.item())\n", + " batches.append(batch_count)\n", + "\n", + " epochs_y.append(losses)\n", + " epochs_x.append(batches)\n", + "\n", + " acc = test_func(dataloader_test, model)\n", + " print(f'Epoch {e} accuracy: {acc}')\n", + " epochs_acc.append(acc)\n", + "\n", + " return epochs_x, epochs_y, epochs_acc\n" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "def test(dataloader, model):\n", + " correct_predictions = []\n", + " with torch.no_grad():\n", + " test_p_bar = tqdm(dataloader)\n", + " for X, y in test_p_bar:\n", + " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", + " X = X.reshape(-1, 2, 128, 128).to(dtype=torch.float, device=device)\n", + " y = y.to(dtype=torch.long, device=device)\n", + "\n", + " # forward\n", + " output = model(X)\n", + "\n", + " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", + " output = output.reshape(batch_size, n_time_steps, -1)\n", + "\n", + " # accumulate all time-steps output for final prediction\n", + " output = output.sum(dim=1)\n", + "\n", + " # calculate accuracy\n", + " pred = output.argmax(dim=1, keepdim=True)\n", + "\n", + " # compute the total correct predictions\n", + " correct_predictions.append(pred.eq(y.view_as(pred)))\n", + "\n", + " test_p_bar.set_description(f\"Testing Model...\")\n", + " \n", + " correct_predictions = torch.cat(correct_predictions)\n", + " return correct_predictions.sum().item()/(len(correct_predictions))*100" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Training loop (HPO)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "12d134e3b89e41888c9c47892b8e6491", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/359 [00:00" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "y_avg = []\n", + "for y in epochs_y:\n", + " y_avg.append(np.mean(y))\n", + "\n", + "plt.plot(np.arange(len(epochs_x)), y_avg, marker = '.')\n", + "plt.xlabel('epoch')\n", + "plt.ylabel('average loss')\n", + "plt.ylim(0,)\n", + "plt.xticks(np.arange(len(epochs_x)))\n", + "for i, txt in enumerate(y_avg):\n", + " if i%5 == 0:\n", + " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", + " else:\n", + " pass\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAG2CAYAAACZEEfAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABq60lEQVR4nO3dd1hT1+MG8DfsHUW2TAfixK3gqqO46q6jtVWr1bZi62idraPVFrWt7dfROn5urauOOloXrqqooOBEBERBZYhI2DPn9weaGlmJBIH4fp4nT8vNPfecQMx9c+6550iEEAJEREREWkqnohtAREREVJ4YdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirVWjYOXPmDPr06QMHBwdIJBLs27dP6XkhBObMmQN7e3sYGxujW7duCA8PV9onKSkJw4cPh4WFBapVq4YxY8YgLS3tNb4KIiIiqswqNOykp6fD09MTK1asKPL5xYsXY+nSpVi5ciUuXrwIU1NTdO/eHVlZWYp9hg8fjps3b+LYsWM4ePAgzpw5g3Hjxr2ul0BERESVnKSyLAQqkUiwd+9e9O/fH0BBr46DgwO+/PJLfPXVVwAAmUwGW1tbbNiwAcOGDUNoaCgaNGiAwMBAtGzZEgBw+PBh9OrVCw8ePICDg0NFvRwiIiKqJPQqugHFiYqKQlxcHLp166bYJpVK0aZNGwQEBGDYsGEICAhAtWrVFEEHALp16wYdHR1cvHgRAwYMKPLY2dnZyM7OVvwsl8uRlJSEGjVqQCKRlN+LIiIiIo0RQiA1NRUODg7Q0Sn+YlWlDTtxcXEAAFtbW6Xttra2iufi4uJgY2Oj9Lyenh4sLS0V+xTFz88P3377rYZbTERERBUhJiYGjo6OxT5facNOeZo5cyamTJmi+Fkmk8HZ2RkxMTGwsLCowJYRERGRqlJSUuDk5ARzc/MS96u0YcfOzg4AEB8fD3t7e8X2+Ph4NG3aVLFPQkKCUrm8vDwkJSUpyhfF0NAQhoaGhbZbWFgw7BAREVUxpQ1BqbTz7Li5ucHOzg7+/v6KbSkpKbh48SK8vLwAAF5eXkhOTsbly5cV+5w4cQJyuRxt2rR57W0mIiKiyqdCe3bS0tIQERGh+DkqKgohISGwtLSEs7MzJk2ahAULFqBu3bpwc3PD7Nmz4eDgoLhjq379+ujRowfGjh2LlStXIjc3FxMmTMCwYcN4JxYREREBqOCwExQUhM6dOyt+fj6OZuTIkdiwYQOmTZuG9PR0jBs3DsnJyWjfvj0OHz4MIyMjRZmtW7diwoQJ6Nq1K3R0dDBo0CAsXbr0tb8WIiIiqpwqzTw7FSklJQVSqRQymYxjdoiIiKoIVc/flXbMDhEREZEmMOwQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEqQn5+P2bNnw83NDcbGxqhduzbmz58PIYRin1GjRkEikSg9evToUeJxU1NTMWnSJLi4uMDY2Bje3t4IDAxU2ictLQ0TJkyAo6MjjI2N0aBBA6xcubJcXqc206voBhAREVVmixYtwu+//46NGzeiYcOGCAoKwkcffQSpVIovvvhCsV+PHj2wfv16xc+GhoYlHvfjjz/GjRs3sHnzZjg4OGDLli3o1q0bbt26hZo1awIApkyZghMnTmDLli1wdXXF0aNHMX78eDg4OKBv377l84K1EHt2iIiISnD+/Hn069cPvXv3hqurK9599134+Pjg0qVLSvsZGhrCzs5O8ahevXqxx8zMzMTu3buxePFidOzYEXXq1MG8efNQp04d/P7770p1jxw5Em+99RZcXV0xbtw4eHp6FqqbSsawQ0T0hiivyzF+fn5o1aoVzM3NYWNjg/79+yMsLExpn08++QS1a9eGsbExalhZo0O3njgTGFIeL1PjvL294e/vjzt37gAArl69irNnz6Jnz55K+506dQo2NjaoV68ePvvsMzx58qTYY+bl5SE/Px9GRkZK242NjXH27Fmluvfv34+HDx9CCIGTJ0/izp078PHx0eArfAMIEjKZTAAQMpmsoptCRFRuvv/+e1GjRg1x8OBBERUVJXbt2iXMzMzE//73P8U+I0eOFD169BCxsbGKR1JSUonH7d69u1i/fr24ceOGCAkJEb169RLOzs4iLS1Nsc+qVavE6dOnxdJ9Z4XDqF+FcZ3WQtfCWvwRcLfcXq+m5Ofni+nTpwuJRCJ09fSERCIRP/zwg9I+27ZtE3/99Ze4du2a2Lt3r6hfv75o1aqVyMvLK/a4Xl5eolOnTuLhw4ciLy9PbN68Wejo6Ah3d3fFPllZWWLEiBECgNDT0xMGBgZi48aN5fZaqxpVz98MO4Jhh4jeDL179xajR49W2jZw4EAxfPhwxc8jR44U/fr1K1M9CQkJAoA4ffq00vbIhFThOv2gcJl+UNh/tEwAEE6f/J94lJxRpvrK27Zt24Sljb2w7jtV2I9eLqzemSLMLKqJDRs2FFsmMjJSABDHjx8vdp+IiAjRsWNHAUDo6uqKVq1aieHDhwsPDw/FPj/++KNwd3cX+/fvF1evXhXLli0TZmZm4tixYxp9jVWVqudvXsYiInpDlMflmKLIZDIAgKWlpWLbydsJGLIyAAKAPCcLadePQ09qC4l5DdxLzCjbCytnU776ChLPfjCp3wkG1q4wbdgFep7vYP73PxRbplatWrCyskJERESx+9SuXRunT59GWloaYmJicOnSJeTm5qJWrVoACsb1zJo1C0uWLEGfPn3QpEkTTJgwAUOHDsVPP/2k8ddZklhZJs5HJiJWlvla69UU3o1FRPSGmDFjBlJSUuDh4QFdXV3k5+fj+++/x/DhwxX79OjRAwMHDoSbmxsiIyMxa9Ys9OzZEwEBAdDV1S21DrlcjkmTJqFdu3Zo1KgRElKz8N2BWzh4LRapVw7h6an1ELlZ0LN0hM3QBdDV04erlUl5vuwyEUJAlpIOI8lLfQMSHcTLMuEfGo+O7tbQ11V+/sGDB3jy5Ans7e1LrcPU1BSmpqZ4+vQpjhw5gsWLFwMAcnNzkZubCx0d5WPr6upCLper9TpiZZmISkyHm5Up7KXGpe4vhEBadh4S03KwIzAaq87chRCAjgTwG9gYQ1s5q1V/RWPYISJ6Q+zcuRNbt27FH3/8gYYNGyIkJASTJk2Cg4MDRo4cCQAYNmyYYv/GjRujSZMmqF27Nk6dOoWuXbuWWoevry9u3LiBM2f+xbZL0fD7OxQpWXnQkQDjPx4Ji48H45e/ApF8aTcS/1oI51E/IyMnv9xec1lcikqC3z+hkLi0gOz8DuhaWMPAyhk58ZFICdwHsyZvY8zGIEj18mF6cy8+GfkeOnrWxd27dzFt2jTUqVMH3bt3Vxyva9euGDBgACZMmAAAOHLkCIQQqFevHiIiIjB16lR4eHjgo48+AgBYWFigU6dOmDp1KoyNjeHi4oLTp09j06ZNWLJkicqvY0dgNGbuuQ75s7Ay+W13tHa1RGJaDhLTshWPx6nKP2flFg5UcgFM330dJ8Meo4VzdXjYm8PDzgLW5iXfZl/RJEK8MAz/DZWSkgKpVAqZTAYLC4uKbg4RqSA/Px/z5s3Dli1bEBcXBwcHB4waNQrffPMNJBIJAGDevHnYvn07YmJiYGBggBYtWuD7779HmzZtij2uq6sr7t+/X2j7+PHjsWLFCgDA6tWr8ccff+DKlStITU3F06dPUa1atXJ5nZrk5OSEGTNmwNfXV7FtwYIF2LJlC27fvl1sOWtrayxYsACffPJJicefMGEC/vrrL2za8zd+u5yKwHtPAQCNa0rhN7AxGtWUAijoZbj9IAlvN6+D6t0/h+dbvbHPtx3MDCvH9+878alYfPg2jocmAAAM5NnQD9mFsIv+kGfIoGtmiU49+8P73U/w961EJDxNweM9C5CTcBciOx3VrGzR3edt/PrjQtja2iqO6+rqilGjRmHevHkACsLnzJkz8eDBA1haWmLQoEH4/vvvIZVKFWXi4uIwc+ZMHD16FElJSXBxccG4ceMwefJkxfv8ZUIIPHiaidtxqQiMeoLV/0a98u/CUE8H2Xml9yJZmRmivr05POwKwo+HvTnq2JjBUE9X7V4ldah6/q4c7ywiIjWpMtGbu7s7li9fjlq1aiEzMxO//PILfHx8EBERAWtr6yKPGxgYiPz8/3oabty4gbfffhuDBw9WbMvIyECPHj3Qo0cPzJw5s3xfqAZlZGSofUlElcsxQgh8/vnn2LN3L0Z+vx7j9sUgN1/AxEAXU952xyhvV+i9cJnHXmoMSyMrGOhKYGEARCSk4audV/H7B82LPYG/DrGyTPxy7A7+vPwAcgHo6kgwrJUTJnatCxuLgYiVZeJeYgZcrUwUJ+05/eQ4F/kEe1utxZGb8cjMLXjvBAD4dHcEBjTLxDtNHGBpaoCAq6GISkxHrCwT9lJjDBkyBEOGDCmxTXZ2dli/fn2xgSE1Kxd34lMRGpuK23EpuB2bittxqUjLzivxuLYWhnCqbgIrM0NYmRvAyswQ1uaGBT+bGcL62XZZZi7aLTwB+QvdIjoSYEx7NzxMzsTt2FREPUlHYlo2/g3Pxr/hiYr99HQkqGFmgPiUbEW5iroExp4dsGeHqCp65513YGtri7Vr1yq2DRo0CMbGxtiyZUuRZZ7/Wz9+/LhKl2QAYNKkSTh48CDCw8MLnYhPnTqFzp07V5menVGjRuH48eNYtWoVrJ1qw//cRfw0+yt8PGY0Fi1ahLS0NHz77bcYNGgQ7OzsEBkZiWnTpiE1NRXXr19XzAj88uWY8ePHY/OWrajzwbdI1LECALSrWwPzB7dGXYcauHv3Lnbs2AEfHx9YW1vjwYMHWLhwIc6dO4edxwLw2e5I5OTLMbV7Pfh2rvPafy+yjFz8djoCG87dU/Ri9Gxkh6+610NtazOVj5OenYejt+KwN/gRzoY/VgQEPR0J6tqa4XZsKgQKTvrTutfDO54OKh334NVHWHwkDHIBSAB08bCBjo4Et+NSEJNU9IBhfV0J6tiYw8XSBEduxuHFE72uRIKzMzqr3MuyIzAas/bcQL4Q0JVI8MPARkqBJSMnD+HxaQiNTcHtuFTFf2WZuYWOpW7dpWHPDhFpNW9vb6xevRp37tyBu7u74s6i4sYy5OTkYPXq1ZBKpfD09FSpjpycHGzZsgVTpkyp0B4HTVm2bBlmz56NUR9/gieJj6FrZgnTBl3RpN9YAAW9PNeuXcPGjRuRnJwMBwcH+Pj4YP78+UpLH0RGRiIxseAbfHJGjmLG35DfJyv22QGgh3Q96o4aBSMjI/z777/49ddf8fTpU9ja2qJjx444f/486tVzx7d5Rpi55zp+OhqGRjWl6ORedK+bpmXl5mNTwD2sOBmpODG3drPEjJ4eaO5c/OzHxTE11MOAZo4Y0MwRCalZ2B/yCPtCHuLGwxSExqYq9pMLYOHhMCw8HFbC0YomAPjfTlDaZmdhpBg7U//Zf2tZmyoGTRcVVtQJG0NbOaOju3WhXq3nTAz04OlUDZ5O1f5rpxA4dD0WE/4IVto3XwjcS8zQ+OWs0rBnB+zZIaqK5HI5Zs2ahcWLFyvdWfTyZaWDBw9i2LBhyMjIgL29Pfbt24dWrVqpVMfOnTvx/vvvIzo6Gg4Ohb+FV7WeHaDgUs3LlyUkAGb28kBdG3PFZY0apoYw0Ct6dpJYWSaiHqcjPCEVy05EIDEtBwAwvI0zpvXwgNRYX+12zdxzDdsuxUBqrI8DE9rDuUb53KEVK8tEZEI6QmNlWH/uHh7JsgAA9WzNMb1nPXSuZ6PxYLsrKAZT/7xWaLu+jgQ6OiXXJZcL5MoLn6ZHtHVBj8Z28LCzgKWpQaltKOoSXHkr6r3Gnh0iqjDlOYCwvOpW5c4iAOjcuTNCQkKQmJiINWvWYMiQIbh48SJsbGxKrWPt2rXo2bNnkUGnqop6nI6Xz50CwA9/Fx6gLDXWh5WZwbMAVDCOIzY5E0dvxStdFqlrYwa/gY3R0tWy0DFUNa9vQ4TGpiIkJhnjNgdhz3hvmBho9hS1IzAaM/Zcx4tf8R2kRpjiUw8DmtWEbinB41W1r2sFHQkKnfTPTC/9pF9cYPisc221/r3YS41f+79te6kx/AY2LlOvkqawZwfs2aE328u3pb7OAYRlqftV7yyqW7cuRo8eXerA4vv376NWrVrYs2cP+vXrV+Q+VbFnZ8uF+/hm3w2lbRIUXL5JzcpDYlo2nqTnIL+I3oSiSACcnvoWnGuYlrltsbJM9Fl2FolpOejr6YD/DWuqsV6WqzFP0W/FeaVtEgAnv3oLrlZlb3tpShv3Ul5lK4Py7FVizw4RlSpWlqkIG0DBN89Ze26go7t1uX/7KqrumXuuq1z3q9xZBBRc/srOzi71+OvXr4eNjQ169+5d6r5VRWpWLpb6hwMoONELoMiTp1wukJyZWzDfSmo2HqdlIzEtB1djkrH/6iOlYwoAD5OzNBJ27KXGWPF+cwz/v4vYf/URmjhK8XGHWmU6Zr5cYMuF+/D7J7TQcwJArCzrtYSd0sa9lFfZyqAiepVexrBD9Aa7HZtS6JLG6xpAGJVY+HKKXABbL9zHlLfrlTqWoU+fPvj+++/h7OyMhg0bIjg4GEuWLMHo0aMBAOnp6fj+++/Rt29f2NvbIzExEStWrMDDhw+VbiN/+c4ioCAQrV+/HiNHjoSeXuGPybi4OMTFxSmWArh+/TrMzc3h7OystERCaV735cOfj95BQmo2XGuYYMNHrZ+d6AufPHV0JLA0NYClqQHcbc2V2nvw2qNCl1Q0OQNym1o18HXv+vj2wC34/XMbDRws4F3b6pWOFRqbgpl7riMkJrnI5zXd9tKU5aRfGQJDVca1sYgqWH5+PmbPng03NzcYGxujdu3amD9/Pl68wrxnzx74+PigRo0akEgkCAkJUenYu3btgoeHB4yMjNC4cWP8/fffiueS0nOw+Mgd5CbGIGH3d4j+ZQiilwxC7MbJ0MtUby2kV+FWzLfp5Scj0Wf5Wfwb/rjE8suWLcO7776L8ePHo379+vjqq6/wySefYP78+QAKenlu376NQYMGwd3dHX369MGTJ0/w77//omHDhgAKTt63wsIR9SBW6djHjx9HdHS0Iji9bOXKlWjWrBnGji24i6ljx45o1qwZ9u/fr/Lr3xEYjXYLT+D9NRfRbuEJ7AiMVrnsq7j+QIZNAfcAAPP7N4KrlSm8atdQe9yH38DG0H12aam8xmCM8nbFgGY1kS8X+PyPYDxKVm89psycfCz85zb6LDuLkJhkmBvqYX7/Rq+l7VQ5ccwOOGaHKtYPP/yAJUuWFJoc7/vvv1dMjrd582ZERUXBwcEBY8eORXBwMJo2bVricc+fP4+OHTvCz88P77zzDv744w8sWrQIV65cgdShFkauu4Sw8AjEbZ4C8yZvw6R+J0gMTGCW8QgnF4+Dg71dub7uM3ceY8S6S4qfdSTA2w1scT7iCVKfTYjWoa4VpvfwUMy8q0kVOVbp4dMMtF90skxzn6gjXy4w4LdzuPZAhr6eDlj6XrMyHe913NmTmZOPQb+fx63YFDRxlGLnJ14w0i99ba5/wx/j6703EJ1UsLhor8Z2mNunIWwtjF5b2+n1UfX8zbADhh2qWOpMjnfv3j24ubmpFHaGDh2K9PR0HDx4ULGtbdu2cKnbAHc93sfj1Gyk/f0T2tezxe//tx43HsgwZddVpGblYX6/hvjQy1WTL1NJbr4cPf/3LyIS0jCkZcG8JM9PPknpOVh+IgKbL9xDbn7Bx1NfTwd85VNPY7cjRyelo9OPp5TuyinPsPGi7Lx8jFx3CRfuJhV6btvYtvCqXUPjdW4KuIc5f92EuZEe/L/sBBtzI43XUR5ikjLQZ/lZJGfkYkhLRywa1KTYActP0rKx4FAo9gY/BFBwl9V3/RqhWwPbIvcn7aDq+ZuXsYgqmLe3N/z9/XHnzh0AUEyO17NnzzIdNyAgAN26dVPa1rBVB+w/dgqPU7Phbm2KrKggtGjSEKOG9Mfwzk2QsWs6Mu4E4Odjd/A0PadM9ZdkU8B9RCSkwdLUAF/3bqB0OcXS1ABz+jTAiS/fwoBmNSGRAPuvPkLXJacwb/9NPEkrfXDxy4QQiEhIxfpzURizIRA+S87g5a95z8cqlafUrFyM3hBYZNDRkaBcxo8kpGThx2eT103rXq/KBB0AcLI0wbL3mkFHAuwMeoCtFwtf6hNCYGdQDLouOY29wQ+hIwE+aueKo1M6MeiQAgcoE1WwGTNmICUlBR4eHkqT4w0fPrxMx42Li1NahPDQtVgcCM9ATupTtHazxAIfR9T7Kg0LFy7EggULsGjRIvz99z/4+puvoWNkhiXHHDC/f6OyvrxCEtOy8euxgmA3rXu9Yiegc7I0wS9Dm+LjDm5YdDgMZ+48xobz9/Dn5Qf4pGMtjOngVuI8LE/SsnE2IhFnwxNxNiIRsc8mjyvJoeuxaO1mWS7zrSSkZmHUukDcik2BqYEuhrV2xvpzUYrBvpamBuWyEOb8Q6FIzc6Dp6MU77dx0fjxy1uHutaY2t0Diw7fxrcHbqK+vQVauBTMbnz3cRpm7b2uCI8N7C2wcFBjNHGsVoEtpsqIYYeogqk6OV5ZbDx/D/MO3ES+XMBQTwebRrdG0uN4AEC/fv0weXLBNP9NmzbFP/6ncSXkH2x1aYz3WjujgYNmL+3+eDgMqdl5aFTTAoNbOpW6f0MHKTaNbo1zEYlY+M9tXH8ow8/H7mDThfuY2LUuOrpb4cHTTDhIjfHgaSb+DX+Mf8MTcSs2Rek4Bno6aO1qiQ51rdC+rhWuPZDhm70Fc5c8vw17y4X7ePA0A/8b1uyVZgEuTlRiOkasu4iYpExYmRlg/ajWaOwoxccd3HA1Jhlz/rqJhNRsTNl5Fas+aFHqnWiqOnPnMQ5cfQQdCfD9gMblNmleefu0Uy1cf5iMv6/HYdymIMzt0wDXH8qwMeA+cvLkMNbXxeS362J0OzelBUeJnmPYIapgU6dOxYwZMzBs2DAAQOPGjXH//n34+fmVKezY2dkhLi4OPx65jRUnIwEADarLkeLqCCN9XVhZWUFPTw8NGjRQKufd0hMRe49ALoBvD9zE9nFtNTax27UHydh5OQYAMK9PQ7VOvu3qWOEv33Y4dD0WPx4JQ3RSRqHJ8V5W394CHZ+Fm1aulkoDXBs6SPFWvf/mLrkUlYRpf17DqbDH6L/iHNaMaIE6NuYlHF01V2OS8dGGQCSl58Clhgk2jW4Nl2dz0jy/ndheaozBqwJw7FY8VpyMwOdd65a53qzcfMz+q+D3M9LbtVwGeb8uEokEi9/1RNC9p0hIzcYX20MUz3Vyt8aC/o3gZPn6biGnqocRmEiDYmWZOB+ZiFiZ6rfKvurkeKVp07Ytlm/Zpwg6X77tjvyYa/Dy8gIAGBgYoFWrVggLU16M8M6dO2jTxB1G+jq4GJWEQ9djCx37VcjlAvP234QQwIBmNV9paQEdHQn6eDrg+JROmPK2e5H79Gpsh/8Na4rAr7vhn4kdMLNXfXSoa13knTz2UmPFeKF+TWti92fecJAaISoxHf1XnMfxW/Fqt/FFp8IS8N6aC0hKz0HjmlLs/sxbEXRe5OlUDQv6FVwyXHL8Dk6GJRTaR12/nYrE/ScZsLUwLPZ3VZWkZhVMcvgiiQTwG8igQ6Vj2CHSkFedN+X55HiHDh3CvXv3sHfvXixZsgQDBgxQ7JOUlISQkBDcunULABAWFoaQkBDExcUp9hkxYoRiCYSMnDxk1PXB3ZBzSL20B583N8aTf7ciKChIafK8qVOnYseOHVizZg0iIiKwfPlyHDhwAF9N+gKfdaoDAPjhUCgyc/LL/PvZF/IQV6KTYWKgixk9Pcp0LAM9HbR0LXpV6g/buqJf05qwNjcs8vmSNKopxf7P26O1qyXSsvMwdnMQlp8Ix6vctLrnygN8vDEIGTn56FDXCtvGtYWVWfFtGtLKCcPbOEMIYOK2YNxLTFe7zuciH6dh5amCkDvnnYYwN9LcJbmKUtQklEIA95+oNwcPvZkYdog0oLhlF1Tp4SltcjwA2L9/P5o1a6ZYumDYsGFo1qwZVq5cqdgnOjoasbGxSErPwftrLuJmvj3s+0+DSdRpzPqwJ/7880/s27cPjRr9N+h4wIABWLlyJRYvXozGjRvj//7v/7B79260b98en3SqhZrVjPFIloXfT0eW6feTlp2Hhf8UrFc1oUsdxZwnZeFmZYqXr4JpYkZcKzNDbPm4DT5oWxA8fjp6B75/XEH6s7l/SiOEwKrTkZiy8yry5AL9mzpg7chWKg0+ntunIZo7V0NKVh4+2XxZ5Tpfrn/2vhvIyZejk7s1ejUu3/mSXpfy+nvTm4Hz7IDz7FDZ7b/6CF9sCy60vbzmTSlKrCwTgVFJ+OloGKKTMiE11se6US3RwuXVV6L+53osPtt6BQZ6OvCf0umVLxcs/Oc2Vp6OhEsNExyd3BGGeqVPDqeK8l4gcdulaMz56wZy8wU87MyxZkTLEn8HcrnA93+HYu3ZKADA2A5umNmzvloDjuNTsvDOsrN4nJqN3k3ssfy9ZmqNmdoX/BCTdoTAUE8HRyd3LPKyWVVV1RfEJM3jpIJqYNihsniSlo0Bv51DdJJyL46OBDg3o8trmaV1R2A0Zuy5rpg7Rmqsh92feZd5gK0QAsP/7yLORz5Bj4Z2WPlhC7WPEZWYDp9fTiM3X+D/RrTU+Nwn5T0jbtC9JHy65QoS07JR3UQfK95vDu86hddqys7Lx1e7ruHAs4Uyv+5VH2M7vtoiloH3kvDe6gvIkwvM6uWBcR1rq1ROlpGLrktOITEtB1/5uGNCl7IPdK5sOAMyvYiTCtJr5+rqColEUujh6+sLoGDelw8//BB2dnYwNTVF8+bNsXv37lKPu2LFCri6usLIyAht2rTBpUuXlJ7/5JNPULt2bRgbG8Pa2hr9+vXD7du3y+U1viw1Kxcj11961pOip9TN3sRRCjsNXK4pTawsUynoFLQrD6YamLNFIpFg7rO7pg7fjMO5iES1j7Hg4C3k5gt0crdG1/o2ZW7Ty14cZFweWrpa4sDn7dDEUYqnGbn4cN0lbDgXpTSO5/lkgQeuPoK+rgS/Dm36ykEHAFq5WmJun4K75Bb+c1vl3/viI7eRmJaD2tamZaq/Mivvvzdpp0oddlRZIFEIgTlz5sDe3h7Gxsbo1q0bwsPDK7DVb67AwECs/DsQjhM2w9F3M+yGLQAAxQrTI0aMQFhYGPbv34/r169j4MCBGDJkCIKDC1/+eW7Hjh2YMmUK5s6diytXrsDT0xPdu3dHQsJ/d6u0aNEC69evR2hoKI4cOQIhBHx8fJCfX/ZBtSXJys3H2E1BuPEwBTVMDbBnfDucm9EFc96pD10JEBIjw7pz98q1DQDwb3hiodmA5QIamw24np05PmxbMBndtwduIi9f9bvEToYlwP92AvR0JJjTp4HGbmF/3eylxtj5iZdiccp5B25h2p/XcP9JOv6+/giDfj+PcxFPYGqgi3WjWqF/s5plrvODti54t4Uj5AKY8McVPHha8t8zOPop/rhUMCh+Qf/GGrtUSKQNKnXYWbRoEX7//XcsX74coaGhWLRoERYvXoxly5Yp9lm8eDGWLl2KlStX4uLFizA1NUX37t2RlVX6bKmkOflygeN3M+B3Oh66ptWha1Yd6eGXoFfNHnU9WwEoWJjy888/R+vWrVGrVi188803qFatGi5fvlzscZcsWYKxY8fio48+QoMGDbBy5UqYmJhg3bp1in3GjRuHjh07wtXVFc2bN8eCBQsQExODe/fuldvrzcuXY8IfwbhwNwlmhnrYOLo1alubwV5qjNHta+Gbdwq+lf/wdygCIstvBfGYpAwsPly4F0vTAzcnd3NHdRN93IlPw5YL91Uqk5Mnx/wDBXePfdTOFbWtzTTWnopgpK+LJUM88U3v+tCRALsuP0CnH09h/NZg3IlPg6mBLraP80KHutYaqU8ikWBB/0ZoXLOgR+nTLZeRlVt0gM/Ll+PrvTcgBDCwec3XNk6MqKqo1GHn/Pnz6NevH3r37g1XV1e8++678PHxUVzGEELg119/xTfffIN+/fqhSZMm2LRpEx49eoR9+/ZVbOPfEBk5edh4/h66/HwKM/f+N8GbyM9F+q1TMGvyNj7eeBmRj9Pg7e2NHTt2ICkpCXK5HNu3b0dWVhbeeuutIo+dk5ODy5cvK63vpKOjg27duiEgIKDIMunp6Vi/fj3c3Nzg5FT67LyvQi4XmL77Oo6HxsNATwf/N7JloQnbRnm7KnoBJvxxBY+SNX977OPUbHy49iIS03JgY26ouIT2fOCmJrv5pSb6+Kp7PQDAkmN3VFqfasP5KNxNTIeVmSG+0MAkeZWBRCLBxx1q4ZehTQs9l5mbDytzA43WZ6Svi5UftoClqQFuPEx5FmgKD7PcGHAft2JTIDXWx6xe9TXaBiJtUKnDTmkLJEZFRSEuLk7pZCiVStGmTZtiT4YAkJ2djZSUFKUHqSchJQs/HrkNL78TmLv/Ju4/yYCFkR6eX6TIuHMB8qw0mDbqitC4VPT89V90Hu+H7Jwc1KhRA4aGhvjkk0+wd+9e1KlTp8g6EhMTkZ+fr7S+EwDY2toqzS8DAL/99hvMzMxgZmaGf/75B8eOHYOBgWZPPEBBwP7+71DsvvIAujoSrHi/OdrWKvwtWiKR4IcBjdHA3gJP0nPwWQnfyl9FSlYuRq67hHtPMuBY3Rj7J7THuRldsG1sW5yd0blc7lAZ1soZDewtkJKVh5+O3ilx34TULCz1jwAATO9RTyvmeXlRUXP4aPLS4YtqVjPG8meLYe6+8gCbX+pZi5VlYsnRgokhZ/T0KHEuH6I3VaUOO8+n0Pfw8IC+vj6aNWuGSZMmKRZIfH7CU+Vk+CI/Pz9IpVLFo7x6ALRRWFwqpu66ivaLTmLFyUjIMnPhUsME3/VriAuzumLhoMbQlUiQdu0oTGq1xNeDvdHJ3Ro5+XL4zZ+HS7djsHzzHgQFBWHKlCkYMmQIrl+/XuZ2DR8+HMHBwTh9+jTc3d0xZMiQcrmUueJkhOK24sWDmuDtEu4sMjbQxaoPW6CaiT6uPpBh7l83X2lyupdl5uTj4w1BuBWbAiszA2we0wZ2UqNyH7ipqyPBvL4NAQDbA6Nx46Gs2H0XHw5DWnYePJ2qYVBzx3JpT0V63XO+eNexwsyeBT023x24hcB7/62a/t2BW0jPyUdz52oYqsJaY0Rvokoddl5cIPHKlSvYuHEjfvrpJ2zcuLFMx505cyZkMpniERMTo6EWaychBM6GJ2LEukvo/usZ7Lr8ADn5crR0qY6VH7TAiS/fwggvV5gY6GFoK2dsf782cqKvYvl3X+Kzt+pgw0etMLN9daReOQjTtz/HjzcMsD1SgklTZ6Fly5ZYsWJFkfVaWVlBV1cX8fHKU/bHx8fDzk55ojSpVIq6deuiY8eO+PPPP3H79m3s3btXo7+HzRfuK3o0Zr/TAINalH4Sd7I0wbJn38p3BMUoBpC+qtx8OXz/uIJL95Jg/myskJvV65tHpbWbJfp6OkAIPFv6oXB4C45+ij8vPwAAzOvTQGOLWlYm9lJj+A0sCPZA+Vw6fNnHHdzQx9MBeXKB8VuvID4lCydux+OfG3HQ1ZHg+wGNtfJ3TaQJlXoh0NIWSHx+wouPj4e9vb2iXHx8PJo2bVrscQ0NDWFoyK7eksTKMhEen4bw+FTsuvwAt+NSARTMHdOzkT0+7uCGZs5FT9d/aPc22NjY4MMhAwEUXNLxdimY76VXEwccjwW2XYrBsVsJkGfkFXvXlIGBAVq0aAF/f3/0798fACCXy+Hv76+05MHLhBAQQiA7u/RxJao6cPUR5jxbVPHzLnUwpr2bymU71LXG1O4eWHT4NubtvwkPOwu0cCn6d1cSuVxg6q6rOHE7AYZ6Olg7qhUaOrz+xR1n9vLAsVvxCLr/FPuvPkK/pv/defR8/SsAeLeFY7HvEW0wtJUzOrpbv7Y5XyQSCRYNaozw+FTcjkvF6A2BiE8p6L0c094N9e05RxhRcSp1z05pCyS6ubnBzs4O/v7+iudTUlJw8eJFxWKHpL7tl6Lh7XcCI9ZdwvxDobgdlwoTA12M8nbF6amdsWJ482JPYnK5HOvXr8fIkSOhp/dflvbw8ECdOnVwZ/cSzGljAHtJMu6e2IbggDOIMmuAmKSCsQ5du3bF8uXLFeWmTJmCNWvWYOPGjQgNDcVnn32G9PR0fPTRRwCAu3fvws/PD5cvX0Z0dDTOnz+PwYMHw9jYGL169dLI7+P0nceYsjMEQgAftnV5pUUVP+1UC70a2yE3X2D81stISFXvEpsQAt8dvIV9IY+gpyPB7x80R2u3V58ZuSzspcbw7VwwyZ3f37eVljTYfeUBrj6QwcxQD9N61KuQ9r1Or3vOFxMDPaz6sAWM9HRw81EKEtNyAACO1TjnDFFJKnXYKW2BRIlEgkmTJmHBggWKuVtGjBgBBwcHRU8Aqef5Gk8vXpyQANg3vh3m9W1Y6nIBx48fR3R0NEaPHq20XV9fH3///Tesra0x89MPcG3pOBjfPwfbPlMQYegOn1/OYM2Zu4iMjETUg1jFyuFDhw7FTz/9hDlz5qBp06YICQnB4cOHFeO0jIyM8O+//6JXr16oU6cOhg4dCnNzc5w/fx42NmWfwO7y/SR8uvkycvMF+ng64Nu+DV9prhiJRILF73qiro0Z4lOy4bv1CnLyVJ+vZql/BDacvwcA+GmwJ7p4aHYWYnV93KEWnCyNEZeShd9OFQxETsnKxaLDBQNlv+haBzbm5T+h4pvIQE8H2S/NdfTtgVsqrcNG9Kaq1MtFpKamYvbs2di7dy8SEhLg4OCA9957D3PmzFHcaSOEwNy5c7F69WokJyejffv2+O233+Durvq3by4X8Z/dl2Pw5a5rhbaX1xpPEQlpmLX3Oi5FFQy4dJAaITYlC0IUXDLzG9i4wta+uR2XgiErA5CSlYdO7tZYM6IlDPTK9v3g7uM09Ft+DqnZeRjp5YJv+zUqtczG8/cw99mloW/7NsRIb9cytUFTjt6Mw7jNl2Ggq4NjUzpiy4X7WPNvFGpZmeLwpI5l/l1R0c5HJuL9NRcLbX+d67ARVRZcG0sNDDsF8vLl6LP8HEJjlW/F15VIcHZG53LrqpfLBXZdjsGCQ6FIzVJe5VlHAvzl2w6Nakpf6+y70U8yMGjleTxOzUYLl+rYPKY1TAw0M8TNPzQeYzYGAQB+HuxZ4kDnv0IeYuL2EADApG51Mamb+pfQyosQAiPWXcK/4Ylo5GCBW7EpkAtg/Uet0Lme5peFoAKxsky0W3gC8hc+ucv73yhRZcW1sUhtK05GIjQ2BUZ6OuU6Qd3LdHQkGNrKGT++26TQc3IB9Fl+Di0WHMf7ay7guwO3sCsoBjceyoqctyZWlqm4BPYqYmWZ+PvaIwxbE4DHqdnwsDPHupGtNBZ0AKBrfVtMfDbJ3qy914u9hfvk7QR8ufMqgIJJCidWson5CtbNagAdCXDjUYri5JuQwtnLy1NF3AlGVNVV6rux6PW5GpOMpScK1hRb9G4TtHazfO0rC3s6VYOOBErfWIGCMUNJ6Tk4H/kE519YekFXRwI3K1N42Jmjvr0FHqdmYVPAfchf8RLYjsBozNxzXVG/pakBNo1uDamJ5ifEm9i1Lm48lMH/dgI+2XwZBz5vD0vT/yZBDLyXhE+3XEaeXGBAs5qY807lXFfK1FCv0Lpcs/bcQEd3a558y9HrvhOMqKrjZSzwMlZmTj56L/sXdx+n450m9lj+fvMKa8uOwGjM2nMD+UIovrH2a1oT4fFpCI1LQWhsCm7HpiI0LgXJGbmlHq+asb5Kc4/I5QLJmcrH05EA52Z0KbcTiSwzF/1XnENUYjq8a9fAptGtoaerg1uPUjB0dQBSs/LQ1cMGKz9sAX3dytkJy/EjRFSRVD1/s2eHsPCfUNx9nA5bC0Ms6F/6gNnyVNw31saOUjR2/G9OGSEEElKzC8JPXCr+vfMY54pYcPPlAKOO59P/l1fYkRrrY9WHLdB/xTmcj3yCOftvopVLdXx3sGDsUmtXS6wY3rzSBh3gv5mEXx4/Ul4zCRMRvQr27ODN7tk5fecxRq4rWFh185jWGlux+XUratCmjgTYMqZNkesYvexxajY+WHuxQgZ9/n09FuO3XlHaZi81wpHJHWFRBdaUKqo3rqLuoCOiNwt7dqhUyRk5mLrrvwGwVTXoAP8N2nz5pOtdx0ql8nVtzYss/zrGQjRzrgYJoDS3UXxKFtKz86pE2OH4ESKq7Bh23lBCCHy97wYSUrNRy9oU03t4VHSTyqysJ92KOmlHJabj5e7V8r6Epmn2UuMq01YievMw7Lyh9l99hEPXYqGnI8GvQ5vC2EC3opukEWU96VbESZvjXoiIylflHflI5eZRcia+2VewqOUXXeuiiWO1im3QG47zphARlS/27Lxh5HKBqX9eRWpWHpo6VcP4t2pXdJMIHPdCRFSeGHbeMBvO38O5iCcw1tfFkiGe0KvEtzW/aTjuhYiofPBM9wYJj0/FwsO3AQBf966PWtZmFdwiIiKi8sew84bIyZNj8s4Q5OTJ8VY9awxvw3lQiIjozcCw84ZY6h+OGw9TUM1EH4sHNamU6ywRERGVB4adN8Dl+0n47VQEAMBvQGPYWBhVcIuIiIheH4YdLZeenYcpO69CLoCBzWqiZ2P7im4SERHRa8Wwo+UWHArF/ScZqFnNGPP6Nazo5hAREb12vPVcS8XKMrHn8gNsuxQNiQT4abBnlVhniYiISNMYdrTQjsBozNxzXbH8QPvaVvCqXaNiG0VERFRBeBlLy8TKMpWCDgCci0xErCyz4hpFRERUgRh2tExUYrpS0AH+W0GbiIjoTcSwo2XcrEzx8hQ6XEGbiIjeZAw7WsZeaoz2dawUP3MFbSIietNxgLIWepRcMD5nYte6GNbaiUGHiIjeaAw7Wib6SQYiH6dDV0eC0e3dIDXm7eZERPRm42UsLXMyLAEA0NKlOoMOERERGHa0zonbBWGni4dNBbeEiIiocmDY0SIZOXkIuPsEAMMOERHRcww7WuR8xBPk5MnhWN0YdWzMKro5RERElQLDjhZ5Pl6ni4cNJC9PtkNERPSGYtjREkIInHw2XqdzPV7CIiIieo5hR0uExafikSwLRvo6XPSTiIjoBQw7WuL5XVjeta1gpK9bwa0hIiKqPBh2tITiEhbvwiIiIlLCsKMFkjNycPn+UwBA53rWFdwaIiKiyoVhRwucCU+EXADutmZwrM7VzYmIiF7EsKMFeAmLiIioeAw7VVy+XODU8/l1eMs5ERFRIQw7VVxITDKeZuTCwkgPLVyqV3RziIiIKh2GnSru+SWsju7W0NPln5OIiOhlPDtWcS8uEUFERESFMexUYfEpWbj5KAUSCdDJnbecExERFYVhpwp7fgnL07EaapgZVnBriIiIKieGnSrs+RIRvIRFRERUPIadKio7Lx9nIxIBMOwQERGVhGGniroUlYSMnHzYmBuioYNFRTeHiIio0mLYqaJO3n4MAHirnjUkEkkFt4aIiKjyYtiponjLORERkWoYdqqgqMR0RCWmQ19XgvZ1ecs5ERFRSRh2qqDnd2G1drOEmaFeBbeGiIiocmPYqYIUq5xz4U8iIqJSMexUMenZebgY9QQAx+sQERGpgmGnijkbkYjcfAGXGiZwszKt6OYQERFVegw7VcyLl7B4yzkREVHpGHaqECEEbzknIiJSE8NOFXLzUQriU7JhYqCLNrUsK7o5REREVQLDThVy6lmvTrs6VjDU063g1hAREVUNDDtVyAneck5ERKQ2tcPOyZMny6MdVIqk9BwExyQDADp7cNZkIiIiVakddnr06IHatWtjwYIFiImJKY82URFO30mAEEB9ewvYS40rujlERERVhtph5+HDh5gwYQL+/PNP1KpVC927d8fOnTuRk5NTHu2jZ048W+W8C3t1iIiI1KJ22LGyssLkyZMREhKCixcvwt3dHePHj4eDgwO++OILXL16tTza+UbLy5fjNG85JyIieiVlGqDcvHlzzJw5ExMmTEBaWhrWrVuHFi1aoEOHDrh586am2vjGC45JRkpWHqqZ6KOpU/WKbg4REVGV8kphJzc3F3/++Sd69eoFFxcXHDlyBMuXL0d8fDwiIiLg4uKCwYMHa6SBDx8+xAcffIAaNWrA2NgYjRs3RlBQkOJ5IQTmzJkDe3t7GBsbo1u3bggPD9dI3ZXF87uwOrlbQ1eHsyYTERGpQ+2w8/nnn8Pe3h6ffPIJ3N3dERwcjICAAHz88ccwNTWFq6srfvrpJ9y+fbvMjXv69CnatWsHfX19/PPPP7h16xZ+/vlnVK/+X+/G4sWLsXTpUqxcuRIXL16EqakpunfvjqysrDLXX1k8XyKCl7CIiIjUp6dugVu3bmHZsmUYOHAgDA0Ni9zHyspKI7eoL1q0CE5OTli/fr1im5ubm+L/hRD49ddf8c0336Bfv34AgE2bNsHW1hb79u3DsGHDytyGivYwORO341KhIyno2SEiIiL1qN2z4+/vj/fee6/YoAMAenp66NSpU5kaBgD79+9Hy5YtMXjwYNjY2KBZs2ZYs2aN4vmoqCjExcWhW7duim1SqRRt2rRBQEBAscfNzs5GSkqK0qOyet6r09y5OqqZGFRwa4iIiKoetcOOn58f1q1bV2j7unXrsGjRIo006rm7d+/i999/R926dXHkyBF89tln+OKLL7Bx40YAQFxcHADA1tZWqZytra3iuaL4+flBKpUqHk5OThpttyY9XyKiMy9hERERvRK1w86qVavg4eFRaHvDhg2xcuVKjTTqOblcjubNm+OHH35As2bNMG7cOIwdO7bM9cycORMymUzxqKyTI2bl5uNcxBMAXCKCiIjoVakdduLi4mBvb19ou7W1NWJjYzXSqOfs7e3RoEEDpW3169dHdHQ0AMDOzg4AEB8fr7RPfHy84rmiGBoawsLCQulRGV24+wSZufmwlxqhvr15RTeHiIioSlI77Dg5OeHcuXOFtp87dw4ODg4aadRz7dq1Q1hYmNK2O3fuwMXFBUDBYGU7Ozv4+/srnk9JScHFixfh5eWl0bZUhINXHwEAWrtZQiLhLedERESvQu27scaOHYtJkyYhNzcXXbp0AVAwaHnatGn48ssvNdq4yZMnw9vbGz/88AOGDBmCS5cuYfXq1Vi9ejUAQCKRYNKkSViwYAHq1q0LNzc3zJ49Gw4ODujfv79G2/K6bb8UjT+vPAQA7L/6CN61a2BoK+cKbhUREVHVIxFCCHUKCCEwY8YMLF26VLEelpGREaZPn445c+ZovIEHDx7EzJkzER4eDjc3N0yZMgVjx45Vas/cuXOxevVqJCcno3379vjtt9/g7u6uch0pKSmQSqWQyWSV4pJWrCwT7RaegPyFv4yuRIKzMzpzEVAiIqJnVD1/qx12nktLS0NoaCiMjY1Rt27dEm9Fr+wqW9g5H5mI99dcLLR929i28KpdowJaREREVPmoev5W+zLWc2ZmZmjVqtWrFqcSOFYv3HujK5HA1cqkAlpDRERUtb1S2AkKCsLOnTsRHR2tuJT13J49ezTSsDfZrUepSj/rSiT4YWAjXsIiIiJ6BWrfjbV9+3Z4e3sjNDQUe/fuRW5uLm7evIkTJ05AKpWWRxvfONsuFdxa/2FbF2wb2xZnZ3Tm4GQiIqJXpHbY+eGHH/DLL7/gwIEDMDAwwP/+9z/cvn0bQ4YMgbMzT8hl9eBpBs6EPwYAfNzBDV61a7BHh4iIqAzUDjuRkZHo3bs3AMDAwADp6emQSCSYPHmy4pZwenU7A2MgBNCuTg241DCt6OYQERFVeWqHnerVqyM1tWBMSc2aNXHjxg0AQHJyMjIyMjTbujdMXr4cO4IKlq54rzV7yYiIiDRB7QHKHTt2xLFjx9C4cWMMHjwYEydOxIkTJ3Ds2DF07dq1PNr4xjgV9hjxKdmwNDXA2w1sSy9AREREpVI77CxfvhxZWVkAgK+//hr6+vo4f/48Bg0ahG+++UbjDXyTPB+Y/G4LRxjq6VZwa4iIiLSDWmEnLy8PBw8eRPfu3QEAOjo6mDFjRrk07E0TK8vEybAEAMDQVk4V3BoiIiLtodaYHT09PXz66aeKnh3SnJ2BDyAXQBs3S9S2Nqvo5hAREWkNtQcot27dGiEhIeXQlDdXvlxg57OBye+34cBkIiIiTVJ7zM748eMxZcoUxMTEoEWLFjA1Vb49ukmTJhpr3JviTPhjPEzORDUTfXRvaFfRzSEiItIqaoedYcOGAQC++OILxTaJRAIhBCQSCfLz8zXXujfE9mcDkwc2c4SRPgcmExERaZLaYScqKqo82vHGSkjJwvHQgoHJ77XmwGQiIiJNUzvsuLi4lEc73li7Lj9AvlygpUt11LU1r+jmEBERaR21w86mTZtKfH7EiBGv3Jg3jVwusD2w4BLWMM6YTEREVC7UDjsTJ05U+jk3NxcZGRkwMDCAiYkJw44azkUmIiYpE+ZGeujd2L6im0NERKSV1L71/OnTp0qPtLQ0hIWFoX379ti2bVt5tFFrbb9UcLv5gGY1YWzAgclERETlQe2wU5S6deti4cKFhXp9qHiJadk4eisOADCsFS9hERERlReNhB2gYHblR48eaepwWm/35QfIzRfwdKqGBg4WFd0cIiIiraX2mJ39+/cr/SyEQGxsLJYvX4527dpprGHaTAiB7YHPZkzm7eZERETlSu2w079/f6WfJRIJrK2t0aVLF/z888+aapdWu3A3CVGJ6TAz1MM7TRwqujlERERaTe2wI5fLy6Mdb5Rtz2ZM7tvUAaaGav8JiIiISA0aG7NDqnmanoPDNwoGJr/PuXWIiIjKndphZ9CgQVi0aFGh7YsXL8bgwYM10ihttvvKA+Tky9GopgUa1ZRWdHOIiIi0ntph58yZM+jVq1eh7T179sSZM2c00iht9eLA5PfYq0NERPRaqB120tLSYGBgUGi7vr4+UlJSNNKoymbhwoWQSCSYNGkSAODevXuQSCRFPnbt2lXscXR0dOD/5Vu4v+gdfNDWVVHmxx9/VOzTt29fODs7w8jICPb29vjwww95Sz8REVEZqB12GjdujB07dhTavn37djRo0EAjjapMAgMDsWrVKjRp0kSxzcnJCbGxsUqPb7/9FmZmZujZs2exxxq38hgcfTdj/Gp/xMbGYt26dZBIJBg0aJBin86dO2Pnzp0ICwvD7t27ERkZiXfffbdcXyMREZE2U/tWoNmzZ2PgwIGIjIxEly5dAAD+/v7Ytm1bib0aVVFaWhqGDx+ONWvWYMGCBYrturq6sLOzU9p37969GDJkCMzMzIo8liwjF6dicqFrVh0fd28GO7vq+Ouvv9C5c2fUqlVLsd/kyZMV/+/i4oIZM2agf//+yM3Nhb6+voZfIRERkfZTu2enT58+2LdvHyIiIjB+/Hh8+eWXePDgAY4fP15oDp6qztfXF71790a3bt1K3O/y5csICQnBmDFjit1nb/ADZOfJ4WFnjqZO1RAfH49Dhw6VWCYpKQlbt26Ft7c3gw4REdEreqVJXnr37o3evXtrui2Vyvbt23HlyhUEBgaWuu/atWtRv359eHt7F/n8ywOTJRIJNm7cCHNzcwwcOLDQ/tOnT8fy5cuRkZGBtm3b4uDBg2V7MURERG8wtXt2AgMDcfHixULbL168iKCgII00qqLFxMRg4sSJ2Lp1K4yMjErcNzMzE3/88UeJPTTBMcm4HZcKQz0d9G9WEwCwbt06DB8+vMjjT506FcHBwTh69Ch0dXUxYsQICCHK9qKIiIjeUGqHHV9fX8TExBTa/vDhQ/j6+mqkURXt8uXLSEhIQPPmzaGnpwc9PT2cPn0aS5cuhZ6eHvLz8xX7/vnnn8jIyMCIESOKPd72ZzMmv9PEAVJjffz7778ICwvDxx9/XOT+VlZWcHd3x9tvv43t27fj77//xoULFzT7IomIiN4Qal/GunXrFpo3b15oe7NmzXDr1i2NNKqide3aFdevX1fa9tFHH8HDwwPTp0+Hrq6uYvvatWvRt29fWFtbF3ms1KxcHLgaCwB479min2vXrkWLFi3g6elZalueL8+RnZ39Sq+FiIjoTad22DE0NER8fLzSHUQAEBsbCz097VjnydzcHI0aNVLaZmpqiho1aihtj4iIwJkzZ/D3338XeRwPDw/0GDUZmbmOqGtjhhYu1ZGSkoJdu3YVuWjqxYsXERgYiPbt26N69eqIjIzE7NmzUbt2bXh5eWn2RRIREb0h1L6M5ePjg5kzZ0Imkym2JScnY9asWXj77bc12rjKbt26dXB0dISPj0+Rz4eFheFw8F0AwLBnA5O3b98OIQTee++9QvubmJhgz5496Nq1K+rVq4cxY8agSZMmOH36NAwNDcv1tRAREWkriVBz5OvDhw/RsWNHPHnyBM2aNQMAhISEwNbWFseOHYOTk1O5NLQ8paSkQCqVQiaTwcLCQiPH3BEYjRl7ruP5b3fOOw0wur2bRo5NREREqp+/1Q47AJCeno6tW7fi6tWrMDY2RpMmTfDee+9V2blgNB12YmWZaLfwBOQv/GZ1JRKcndEZ9lLjMh+fiIiIVD9/v9IgG1NTU4wbN+6VG6ftohLTlYIOAOQLgXuJGQw7REREr9krjyi+desWoqOjkZOTo7S9b9++ZW5UVedmZQodCQr17LhamVRco4iIiN5Qaoedu3fvYsCAAbh+/TokEolisjuJRAIASnPQvKnspcbwG9gYs/bcQL4Q0JVI8MPARuzVISIiqgBqh52JEyfCzc0N/v7+cHNzw6VLl/DkyRN8+eWX+Omnn8qjjVXS0FbO6OhujXuJGXC1MmHQISIiqiBqh52AgACcOHECVlZW0NHRgY6ODtq3bw8/Pz988cUXCA4OLo92Vkn2UmOGHCIiogqm9jw7+fn5MDc3B1CwrMGjR48AAC4uLggLC9Ns64iIiIjKSO2enUaNGuHq1atwc3NDmzZtsHjxYhgYGGD16tWFZlUmIiIiqmhqh51vvvkG6enpAIDvvvsO77zzDjp06IAaNWpgx44dGm8gERERUVm80qSCL0tKSkL16tUVd2RVNeUxgzIRERGVr3KdVPBllpaWmjgMERERkcapPUCZiIiIqCph2CEiIiKtxrBDREREWk3tsHPmzBnk5eUV2p6Xl4czZ85opFFEREREmqJ22OncuTOSkpIKbZfJZOjcubNGGkVERESkKWqHHSFEkbeYP3nyBKamphppFBEREZGmqHzr+cCBAwEUrG4+atQoGBoaKp7Lz8/HtWvX4O3trfkWEhEREZWBymFHKpUCKOjZMTc3h7HxfwtcGhgYoG3bthg7dqzmW0hERERUBiqHnfXr1wMAXF1d8dVXX/GSFREREVUJao/ZmTZtmtKYnfv37+PXX3/F0aNHNdowIiIiIk1QO+z069cPmzZtAgAkJyejdevW+Pnnn9GvXz/8/vvvGm8gERERUVmoHXauXLmCDh06AAD+/PNP2NnZ4f79+9i0aROWLl2q8QYSERERlYXaYScjIwPm5uYAgKNHj2LgwIHQ0dFB27Ztcf/+fY03kIiIiKgs1A47derUwb59+xATE4MjR47Ax8cHAJCQkFDi8upEREREFUHtsDNnzhx89dVXcHV1RevWreHl5QWgoJenWbNmGm8gERERUVmoHXbeffddREdHIygoCEeOHFFs79q1K3755ReNNu5lCxcuhEQiwaRJkxTbsrKy4Ovrixo1asDMzAyDBg1CfHx8ubaDiIiIqo5XWvXczs4O5ubmOHbsGDIzMwEArVq1goeHh0Yb96LAwECsWrUKTZo0Udo+efJkHDhwALt27cLp06fx6NEjxWzPRERERGqHnSdPnqBr165wd3dHr169EBsbCwAYM2YMvvzyS403EADS0tIwfPhwrFmzBtWrV1dsl8lkWLt2LZYsWYIuXbqgRYsWWL9+Pc6fP48LFy6US1uIiIioalE77EyePBn6+vqIjo6GiYmJYvvQoUNx+PBhjTbuOV9fX/Tu3RvdunVT2n758mXk5uYqbffw8ICzszMCAgKKPV52djZSUlKUHkRERKSdVF4u4rmjR4/iyJEjcHR0VNpet27dcrn1fPv27bhy5QoCAwMLPRcXFwcDAwNUq1ZNabutrS3i4uKKPaafnx++/fZbTTeViIiIKiG1e3bS09OVenSeS0pKUloJXRNiYmIwceJEbN26FUZGRho77syZMyGTyRSPmJgYjR2biIiIKhe1w06HDh0Uy0UAgEQigVwux+LFi9G5c2eNNu7y5ctISEhA8+bNoaenBz09PZw+fRpLly6Fnp4ebG1tkZOTg+TkZKVy8fHxsLOzK/a4hoaGsLCwUHoQERGRdlL7MtbixYvRtWtXBAUFIScnB9OmTcPNmzeRlJSEc+fOabRxXbt2xfXr15W2ffTRR/Dw8MD06dPh5OQEfX19+Pv7Y9CgQQCAsLAwREdHK+b/ISIiojeb2mGnUaNGuHPnDpYvXw5zc3OkpaVh4MCB8PX1hb29vUYbZ25ujkaNGiltMzU1RY0aNRTbx4wZgylTpsDS0hIWFhb4/PPP4eXlhbZt22q0LURERFQ1qR12oqOj4eTkhK+//rrI55ydnTXSMFX98ssv0NHRwaBBg5CdnY3u3bvjt99+e61tICIiospLIoQQ6hTQ1dVFbGwsbGxslLY/efIENjY2yM/P12gDX4eUlBRIpVLIZDKO3yEiIqoiVD1/qz1AWQgBiURSaHtaWppG75giIiIi0gSVL2NNmTIFQMHdV7Nnz1a6/Tw/Px8XL15E06ZNNd5AIiIiorJQOewEBwcDKOjZuX79OgwMDBTPGRgYwNPTE1999ZXmW0hERERUBiqHnZMnTwIouPX7f//7H8e2EBERUZWg9t1Y69evL492EBEREZULtQcoExEREVUlDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Sp12PHz80OrVq1gbm4OGxsb9O/fH2FhYUr7ZGVlwdfXFzVq1ICZmRkGDRqE+Pj4CmoxERERVTaVOuycPn0avr6+uHDhAo4dO4bc3Fz4+PggPT1dsc/kyZNx4MAB7Nq1C6dPn8ajR48wcODACmw1ERERVSYSIYSo6Eao6vHjx7CxscHp06fRsWNHyGQyWFtb448//sC7774LALh9+zbq16+PgIAAtG3bVqXjpqSkQCqVQiaTwcLCojxfAhEREWmIqufvSt2z8zKZTAYAsLS0BABcvnwZubm56Natm2IfDw8PODs7IyAgoNjjZGdnIyUlRelBRERE2qnKhB25XI5JkyahXbt2aNSoEQAgLi4OBgYGqFatmtK+tra2iIuLK/ZYfn5+kEqlioeTk1N5Np2IiIgqUJUJO76+vrhx4wa2b99e5mPNnDkTMplM8YiJidFAC4mIiKgy0qvoBqhiwoQJOHjwIM6cOQNHR0fFdjs7O+Tk5CA5OVmpdyc+Ph52dnbFHs/Q0BCGhobl2WQiIiKqJCp1z44QAhMmTMDevXtx4sQJuLm5KT3fokUL6Ovrw9/fX7EtLCwM0dHR8PLyet3NJSIiokqoUvfs+Pr64o8//sBff/0Fc3NzxTgcqVQKY2NjSKVSjBkzBlOmTIGlpSUsLCzw+eefw8vLS+U7sYiIiEi7VepbzyUSSZHb169fj1GjRgEomFTwyy+/xLZt25CdnY3u3bvjt99+K/Ey1st46zkREVHVo+r5u1KHndeFYYeIiKjq0cp5doiIiIjUxbBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIq2lN2FmxYgVcXV1hZGSENm3a4NKlSxXdJCIiIqoEtCLs7NixA1OmTMHcuXNx5coVeHp6onv37khISKjophEREVEF04qws2TJEowdOxYfffQRGjRogJUrV8LExATr1q2r6KYRERFRBdOr6AaUVU5ODi5fvoyZM2cqtuno6KBbt24ICAgoskx2djays7MVP8tkMgBASkpK+TaWiIiINOb5eVsIUeJ+VT7sJCYmIj8/H7a2tkrbbW1tcfv27SLL+Pn54dtvvy203cnJqVzaSEREROUnNTUVUqm02OerfNh5FTNnzsSUKVMUP8vlciQlJaFGjRqQSCQaqyclJQVOTk6IiYmBhYXFay3Pul9/3WUtz7rfrLrLWp51s+6qUr6sdZdECIHU1FQ4ODiUuF+VDztWVlbQ1dVFfHy80vb4+HjY2dkVWcbQ0BCGhoZK26pVq1ZeTYSFhUWZ/sBlKc+6X3/dZS3Put+sustannWz7qpSvqx1F6ekHp3nqvwAZQMDA7Ro0QL+/v6KbXK5HP7+/vDy8qrAlhEREVFlUOV7dgBgypQpGDlyJFq2bInWrVvj119/RXp6Oj766KOKbhoRERFVMK0IO0OHDsXjx48xZ84cxMXFoWnTpjh8+HChQcuvm6GhIebOnVvoktnrKM+6X3/dZS3Put+sustannWz7qpSvqx1a4JElHa/FhEREVEVVuXH7BARERGVhGGHiIiItBrDDhEREWk1hh0iIiLSagw75WjFihVwdXWFkZER2rRpg0uXLqlU7syZM+jTpw8cHBwgkUiwb98+lev08/NDq1atYG5uDhsbG/Tv3x9hYWEql//999/RpEkTxeRPXl5e+Oeff1Qu/6KFCxdCIpFg0qRJKu0/b948SCQSpYeHh4fK9T18+BAffPABatSoAWNjYzRu3BhBQUEqlXV1dS1Ut0Qiga+vb6ll8/PzMXv2bLi5ucHY2Bi1a9fG/PnzS12r5UWpqamYNGkSXFxcYGxsDG9vbwQGBhbar7T3hhACc+bMgb29PYyNjdGtWzeEh4erXH7Pnj3w8fFRzCYeEhKicv25ubmYPn06GjduDFNTUzg4OGDEiBF49OiRSnXPmzcPHh4eMDU1RfXq1dGtWzdcvHhR5ba/6NNPP4VEIsGvv/6qUtlRo0YV+tv36NFDrbpDQ0PRt29fSKVSmJqaolWrVoiOji61bFHvO4lEgh9//FGlutPS0jBhwgQ4OjrC2NhYsRiyKmXj4+MxatQoODg4wMTEBD169FC8X1T5LMnKyoKvry9q1KgBMzMzDBo0SDHBqyrlV69ejbfeegsWFhaQSCRITk5WPFda+aSkJHz++eeoV68ejI2N4ezsjC+++AIymUyluj/55BPUrl0bxsbGsLa2Rr9+/RRLDKnzOSqEQM+ePRW/X1XKvvXWW4X+3p9++qladQcEBKBLly4wNTWFhYUFOnbsiO+++67Esvfu3Sv2/bZr1y6V6o6Li8OHH34IOzs7mJqaonnz5ti9e7dKZSMjIzFgwABYW1vDwsICQ4YMKTQhcHlh2CknO3bswJQpUzB37lxcuXIFnp6e6N69OxISEkotm56eDk9PT6xYsULtek+fPg1fX19cuHABx44dQ25uLnx8fJCenq5SeUdHRyxcuBCXL19GUFAQunTpgn79+uHmzZtqtSMwMBCrVq1CkyZN1CrXsGFDxMbGKh5nz55VqdzTp0/Rrl076Ovr459//sGtW7fw888/o3r16iq398V6jx07BgAYPHhwqWUXLVqE33//HcuXL0doaCgWLVqExYsXY9myZSrVDQAff/wxjh07hs2bN+P69evw8fFBt27d8PDhQ6X9SntvLF68GEuXLsXKlStx8eJFmJqaonv37sjKylKpfHp6Otq3b49FixYV+3xx5TMyMnDlyhXMnj0bV65cwZ49exAWFoa+ffuqVLe7uzuWL1+O69ev4+zZs3B1dYWPjw8eP36sUvnn9u7diwsXLihNH69K2R49eii9B7Zt26Zy+cjISLRv3x4eHh44deoUrl27htmzZ8PIyKjUsi/WGRsbi3Xr1kEikWDQoEEq1T1lyhQcPnwYW7ZsQWhoKCZNmoQJEyZg//79JZYVQqB///64e/cu/vrrLwQHB8PFxQXdunVDenq6Sp8lkydPxoEDB7Br1y6cPn0ajx49wsCBAwGo9lmUkZGBHj16YNasWYXaV1r5R48e4dGjR/jpp59w48YNbNiwAYcPH8aYMWNUqrtFixZYv349QkNDceTIEQgh4OPjg/z8fLU+R3/99VelZYZULTt27Filv/vixYtVLh8QEIAePXrAx8cHly5dQmBgICZMmICzZ8+WWNbJyanQ++3bb7+FmZkZevbsqVLdI0aMQFhYGPbv34/r169j4MCBGDJkCA4cOFBi2fT0dPj4+EAikeDEiRM4d+4ccnJy0KdPH8jl8kK/V40TVC5at24tfH19FT/n5+cLBwcH4efnp9ZxAIi9e/e+cjsSEhIEAHH69OlXPkb16tXF//3f/6m8f2pqqqhbt644duyY6NSpk5g4caJK5ebOnSs8PT1fqY3Tp08X7du3f6WyRZk4caKoXbu2kMvlpe7bu3dvMXr0aKVtAwcOFMOHD1eproyMDKGrqysOHjyotL158+bi66+/Lrbcy+8NuVwu7OzsxI8//qjYlpycLAwNDcW2bdtKLf+iqKgoAUAEBwerXH9RLl26JACI+/fvq11WJpMJAOL48eMq1/3gwQNRs2ZNcePGDeHi4iJ++eUXlcqOHDlS9OvXr8T2lFR+6NCh4oMPPnilsi/r16+f6NKli8rlGzZsKL777julbUW9d14uGxYWJgCIGzduKLbl5+cLa2trsWbNmkJ1v/xZkpycLPT19cWuXbsU+4SGhgoAIiAgoNTyLzp58qQAIJ4+fVrk6y6t/HM7d+4UBgYGIjc3V+2yV69eFQBERESEynUHBweLmjVritjY2GL/tkWVVedzsajybdq0Ed98880rlX1Z06ZNC31+lVTe1NRUbNq0SWk/S0vLQu+Zl8seOXJE6OjoCJlMptgnOTlZSCQScezYsVJfS1mxZ6cc5OTk4PLly+jWrZtim46ODrp164aAgIDX2haZTAYAsLS0VLtsfn4+tm/fjvT0dLWW3vD19UXv3r2VXr+qwsPD4eDggFq1amH48OGIjo5Wqdz+/fvRsmVLDB48GDY2NmjWrBnWrFmjdv1Awd9vy5YtGD16tEoLw3p7e8Pf3x937twBAFy9ehVnz55Fz549VaovLy8P+fn5MDIyUtpubGyscs8WAERFRSEuLk7p9y6VStGmTZvX/r57TiaTQSKRqL32XE5ODlavXg2pVApPT0+Vysjlcnz44YeYOnUqGjZsqHZbT506BRsbG9SrVw+fffYZnjx5onK9hw4dgru7O7p37w4bGxu0adNGrcvPz8XHx+PQoUMYM2aMymW8vb2xf/9+PHz4EEIInDx5Enfu3IGPj0+J5bKzswFA6X2no6MDQ0PDIt93L3+WXL58Gbm5uUrvNw8PDzg7Oxf5fivLZ5Gq5WUyGSwsLKCnp1doe0ll09PTsX79eri5ucHJyUmlujMyMvD+++9jxYoVxa7DWFLdW7duhZWVFRo1aoSZM2ciIyNDpfIJCQm4ePEibGxs4O3tDVtbW3Tq1Emlv9nLLl++jJCQkGLfb0WV9/b2xo4dO5CUlAS5XI7t27cjKysLb731Volls7OzIZFIlCYWNDIygo6Ojlqfc6+s3OPUG+jhw4cCgDh//rzS9qlTp4rWrVurdSyUoWcnPz9f9O7dW7Rr106tcteuXROmpqZCV1dXSKVScejQIZXLbtu2TTRq1EhkZmYKIdT7BvP333+LnTt3iqtXr4rDhw8LLy8v4ezsLFJSUkota2hoKAwNDcXMmTPFlStXxKpVq4SRkZHYsGGDym1/bseOHUJXV1c8fPhQpf3z8/PF9OnThUQiEXp6ekIikYgffvhBrTq9vLxEp06dxMOHD0VeXp7YvHmz0NHREe7u7sWWefm9ce7cOQFAPHr0SGm/wYMHiyFDhpRa/kWa6NnJzMwUzZs3F++//77KZQ8cOCBMTU2FRCIRDg4O4tKlSyrX/cMPP4i3335b0RunTs/Otm3bxF9//SWuXbsm9u7dK+rXry9atWol8vLySi3//Fu9iYmJWLJkiQgODhZ+fn5CIpGIU6dOqfS6n1u0aJGoXr264t+PKm3PysoSI0aMEACEnp6eMDAwEBs3biy1bE5OjnB2dhaDBw8WSUlJIjs7WyxcuFAAED4+Pkpli/os2bp1qzAwMChUT6tWrcS0adNKLf+i0np2VPkse/z4sXB2dhazZs1SueyKFSuEqampACDq1atXZK9OceXHjRsnxowZo/i5qL9NcWVXrVolDh8+LK5duya2bNkiatasKQYMGKBS3QEBAQKAsLS0FOvWrRNXrlwRkyZNEgYGBuLOnTsqve7nPvvsM1G/fv0inyuu/NOnT4WPj4/i/WZhYSGOHDlSatmEhARhYWEhJk6cKNLT00VaWpqYMGGCACDGjRtXbBs1hWGnHFSWsPPpp58KFxcXERMTo1a57OxsER4eLoKCgsSMGTOElZWVuHnzZqnloqOjhY2Njbh69apimzph52VPnz4VFhYWKl1C09fXF15eXkrbPv/8c9G2bVu16/Xx8RHvvPOOyvtv27ZNODo6im3btolr166JTZs2CUtLS7WCVkREhOjYsaMAIHR1dUWrVq3E8OHDhYeHR7FlKnPYycnJEX369BHNmjVT6rYurWxaWpoIDw8XAQEBYvTo0cLV1VXEx8eXWj4oKEjY2toqBVR1ws7LIiMjVb6E9vzf+3vvvae0X58+fcSwYcPUqrtevXpiwoQJxT5fVPkff/xRuLu7i/3794urV6+KZcuWCTMzs0KXBooqGxQUJDw9PRXvu+7du4uePXuKHj16KO1X1GeJOmGntM+i0sJOaeVlMplo3bq16NGjh8jJyVG5bHJysrhz5444ffq06NOnj2jevHmhoFlU+b/++kvUqVNHpKamKrYV9ftV9TPY39+/yEtoRZV//u985syZSvs2btxYzJgxQ+W6MzIyhFQqFT/99FORzxdXfsKECaJ169bi+PHjIiQkRMybN09IpVJx7dq1UsseOXJE1KpVS0gkEqGrqys++OAD0bx5c/Hpp5+W8NvRDIadcpCdnS10dXULvfFHjBgh+vbtq9axXjXs+Pr6CkdHR3H37l21y76sa9euKiXvvXv3Kj40nz8AKN7YRX1LLk3Lli2V/gEXx9nZWelblhBC/Pbbb8LBwUGt+u7duyd0dHTEvn37VC7j6Ogoli9frrRt/vz5ol69emrVLUTByf55WBkyZIjo1atXsfu+/N54foJ+OaB07NhRfPHFF6WWf1FZwk5OTo7o37+/aNKkiUhMTFSr7Mvq1KlTZC/Zy+V/+eUXxfvsxfeejo6OcHFxeaW6raysxMqVK0utOzs7W+jp6Yn58+cr7Tdt2jTh7e2tct1nzpwRAERISEixbXq5fEZGhtDX1y803mvMmDGie/fuKtednJwsEhIShBAF4w3Hjx+veK64z5LnJ+iXA4qzs7NYsmRJqeVfVFLYKa18SkqK8PLyEl27di0UVNT5HMzOzhYmJibijz/+KLX8xIkTi32/derUSe2609LSBABx+PDhUuu+e/euACA2b96stH3IkCGKXlRV6t60aZPQ19dX/N1fVFz5iIiIQuO8hCg4R3zyyScq1/348WPF39rW1lYsXry42H01hWN2yoGBgQFatGgBf39/xTa5XA5/f3+1xr68CiEEJkyYgL179+LEiRNwc3Mr8zHlcrni+n5JunbtiuvXryMkJETxaNmyJYYPH46QkBDo6uqqVW9aWhoiIyNhb29f6r7t2rUrdJvjnTt34OLiolad69evh42NDXr37q1ymYyMDOjoKP9T0tXVfaU7DExNTWFvb4+nT5/iyJEj6Nevn8pl3dzcYGdnp/S+S0lJwcWLF8v9ffdcbm4uhgwZgvDwcBw/fhw1atQo0/FUfe99+OGHuHbtmtJ7z8HBAVOnTsWRI0fUrvfBgwd48uSJSu89AwMDtGrVqszvv7Vr16JFixYqj1ECCn7fubm5ZX7/SaVSWFtbIzw8HEFBQejXr1+pnyUtWrSAvr6+0vstLCwM0dHR8PLyKvNnkSrlU1JS4OPjAwMDA+zfv18x/uhV6hYFX/6RnZ1davkZM2YUer8BwC+//IJ169apXffz8vb29qXW7erqCgcHhyLfb87OzirXvXbtWvTt2xfW1tZKv4OSyj8fV1TU+y0/P1/luq2srFCtWjWcOHECCQkJijs2y1W5x6k31Pbt24WhoaHYsGGDuHXrlhg3bpyoVq2aiIuLK7VsamqqCA4OFsHBwQKAYhzAy3e0FOWzzz4TUqlUnDp1SsTGxioeGRkZKrV7xowZ4vTp0yIqKkpcu3ZNzJgxQ0gkEnH06FGVyr9MnctYX375pTh16pSIiooS586dE926dRNWVlZFfvN42aVLl4Senp74/vvvRXh4uNi6daswMTERW7ZsUbmt+fn5wtnZWUyfPl3lMkIU3MlTs2ZNcfDgQREVFSX27NkjrKysCnXll+Tw4cPin3/+EXfv3hVHjx4Vnp6eok2bNoW65Et7byxcuFBUq1ZNMf6kX79+ws3NTfGNt7TyT548EcHBweLQoUMCgNi+fbsIDg4WsbGxpZbPyckRffv2FY6OjiIkJETp/ZednV1i2bS0NDFz5kwREBAg7t27J4KCgsRHH30kDA0NFd8i1f138eJlrJLKpqamiq+++koEBASIqKgocfz4cdG8eXNRt25dkZWVpVLde/bsEfr6+mL16tUiPDxcLFu2TOjq6op///1XpXbLZDJhYmIifv/990Kvo7TynTp1Eg0bNhQnT54Ud+/eFevXrxdGRkbit99+K7Xszp07xcmTJ0VkZKTYt2+fcHFxEQMHDhRCqPZZ8umnnwpnZ2dx4sQJERQUJLy8vBSXk1UpHxsbK4KDg8WaNWsEAHHmzBkRHBwsnjx5Ump5mUwm2rRpIxo3biwiIiKU9vn0009LLBsZGSl++OEHERQUJO7fvy/OnTsn+vTpIywtLUV8fPwrfY7iWc9ZaWUjIiLEd999J4KCgkRUVJT466+/RK1atUTHjh1V/r398ssvwsLCQuzatUuEh4eLb775RhgZGYn3339fpXaHh4cLiUQi/vnnH6XtpdWdk5Mj6tSpIzp06CAuXrwoIiIixE8//SQkEono1atXqXWvW7dOBAQEiIiICLF582ZhaWkppkyZUuzvVJMYdsrRsmXLhLOzszAwMBCtW7cWFy5cUKnc8y7dlx8jR44stWxR5QCI9evXq1T36NGjhYuLizAwMBDW1taia9eurxx0hFAv7AwdOlTY29sLAwMDUbNmTTF06NAiBwwW58CBA6JRo0bC0NBQeHh4iNWrV6vV1iNHjggAIiwsTK1yKSkpYuLEicLZ2VkYGRmJWrVqia+//lpkZ2erfIwdO3aIWrVqCQMDA2FnZyd8fX1FcnJyof1Ke2/I5XIxe/ZsYWtrKwwNDUXXrl2VXk9p5devX1/k83Pnzi21/PNLX0U9Tp48WWLZzMxMMWDAAOHg4CAMDAyEvb296Nu3r9IAZXX/XbwYdkoqm5GRIXx8fIS1tbXQ19cXLi4uYuzYsUpfTFSpe+3ataJOnTrCyMhIeHp6Ki6FqlJ21apVwtjY+JX+5rGxsWLUqFHCwcFBGBkZiXr16omff/5ZyOXyUsv+73//E46OjkJfX184OzuLb775RvG+VeWzJDMzU4wfP15Ur15dmJiYiAEDBiiCsSrl586dW+w+pZUv7rWV9Hhe9uHDh6Jnz57CxsZG6OvrC0dHR/H++++L27dvq9z2lz0PO6WVjY6OFh07dhSWlpbC0NBQ1KlTR0ydOlUxtk3Vuv38/ISjo6MwMTERXl5e4t9//1W57MyZM4WTk5PIz88v9BpKK3/nzh0xcOBAYWNjI0xMTESTJk3Epk2bVCo7ffp0YWtrK/T19UXdunUV79PXQfLsBRIRERFpJY7ZISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQEb3k1KlTkEgkSE5OruimEJEGMOwQERGRVmPYISIiIq3GsENElY5cLoefnx/c3NxgbGwMT09P/PnnnwD+u8R06NAhNGnSBEZGRmjbti1u3LihdIzdu3ejYcOGMDQ0hKurK37++Wel57OzszF9+nQ4OTnB0NAQderUwdq1a5X2uXz5Mlq2bAkTExN4e3sXWmmaiKoGhh0iqnT8/PywadMmrFy5Ejdv3sTkyZPxwQcf4PTp04p9pk6dip9//hmBgYGwtrZGnz59kJubC6AgpAwZMgTDhg3D9evXMW/ePMyePRsbNmxQlB8xYgS2bduGpUuXIjQ0FKtWrYKZmZlSO77++mv8/PPPCAoKgp6eHkaPHv1aXj8RaRYXAiWiSiU7OxuWlpY4fvw4vLy8FNs//vhjZGRkYNy4cejcuTO2b9+OoUOHAgCSkpLg6OiIDRs2YMiQIRg+fDgeP36Mo0ePKspPmzYNhw4dws2bN3Hnzh3Uq1cPx44dQ7du3Qq14dSpU+jcuTOOHz+Orl27AgD+/vtv9O7dG5mZmTAyMirn3wIRaRJ7doioUomIiEBGRgbefvttmJmZKR6bNm1CZGSkYr8Xg5ClpSXq1auH0NBQAEBoaCjatWundNx27dohPDwc+fn5CAkJga6uLjp16lRiW5o0aaL4f3t7ewBAQkJCmV8jEb1eehXdACKiF6WlpQEADh06hJo1ayo9Z2hoqBR4XpWxsbFK++nr6yv+XyKRACgYT0REVQt7doioUmnQoAEMDQ0RHR2NOnXqKD2cnJwU+124cEHx/0+fPsWdO3dQv359AED9+vVx7tw5peOeO3cO7u7u0NXVRePGjSGXy5XGABGR9mLPDhFVKubm5vjqq68wefJkyOVytG/fHjKZDOfOnYOFhQVcXFwAAN999x1q1KgBW1tbfP3117CyskL//v0BAF9++SVatWqF+fPnY+jQoQgICMDy5cvx22+/AQBcXV0xcuRIjB49GkuXLoWnpyfu37+PhIQEDBkypKJeOhGVE4YdIqp05s+fD2tra/j5+eHu3buoVq0amjdvjlmzZikuIy1cuBATJ05EeHg4mjZtigMHDsDAwAAA0Lx5c+zcuRNz5szB/PnzYW9vj++++w6jRo1S1PH7779j1qxZGD9+PJ48eQJnZ2fMmjWrIl4uEZUz3o1FRFXK8zulnj59imrVqlV0c4ioCuCYHSIiItJqDDtERESk1XgZi4iIiLQae3aIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIq/0/aNuF5S+4oysAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plt.plot(np.arange(len(epochs_x)), epochs_acc, marker = '.')\n", + "plt.xlabel('epoch')\n", + "plt.ylabel('test accuracy')\n", + "plt.ylim(0, 100)\n", + "plt.xticks(np.arange(len(epochs_x)))\n", + "for i, txt in enumerate(epochs_acc):\n", + " if i%5 == 0:\n", + " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", + " else:\n", + " pass\n", + "plt.show()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "speck-rescnn", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tests/test_nonsequential/exp_set_B/baseline-SCNN-example_3.ipynb b/tests/test_nonsequential/exp_set_B/baseline-SCNN-example_3.ipynb new file mode 100644 index 00000000..0020ec07 --- /dev/null +++ b/tests/test_nonsequential/exp_set_B/baseline-SCNN-example_3.ipynb @@ -0,0 +1,1500 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import torch, random\n", + "import torch.nn as nn\n", + "import sinabs.layers as sl\n", + "from tqdm.notebook import tqdm\n", + "\n", + "from tonic.datasets.dvsgesture import DVSGesture\n", + "from tonic.transforms import ToFrame\n", + "from torch.utils.data import DataLoader\n", + "from torch.nn import CrossEntropyLoss\n", + "from torch.optim import Adam\n", + "\n", + "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "torch.backends.cudnn.deterministic = True\n", + "random.seed(1)\n", + "torch.manual_seed(1)\n", + "torch.cuda.manual_seed(1)\n", + "np.random.seed(1)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Loading Data" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "batch_size = 3\n", + "num_workers = 1\n", + "epochs = 30\n", + "lr = 5e-4" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "root_dir = \"./DVSGESTURE\"\n", + "_ = DVSGesture(save_to=root_dir, train=True)\n", + "_ = DVSGesture(save_to=root_dir, train=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "n_time_steps = 50\n", + "to_raster = ToFrame(sensor_size=DVSGesture.sensor_size, n_time_bins=n_time_steps)\n", + "\n", + "snn_train_dataset = DVSGesture(save_to=root_dir, train=True, transform=to_raster)\n", + "snn_test_dataset = DVSGesture(save_to=root_dir, train=False, transform=to_raster)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The transformed array is in shape [Time-Step, Channel, Height, Width] --> (50, 2, 128, 128)\n" + ] + } + ], + "source": [ + "sample_data, label = snn_train_dataset[0]\n", + "print(f\"The transformed array is in shape [Time-Step, Channel, Height, Width] --> {sample_data.shape}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "snn_train_dataloader = DataLoader(snn_train_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True)\n", + "snn_test_dataloader = DataLoader(snn_test_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=False)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Network Module\n", + "\n", + "We need to define a `nn.Module` implementing the network we want the chip to reproduce." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "device: NVIDIA RTX A4000\n" + ] + } + ], + "source": [ + "if torch.cuda.is_available():\n", + " device = torch.device('cuda:0')\n", + " print('device: ', torch.cuda.get_device_name(0))\n", + "else:\n", + " device = torch.device('cpu')" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "class SNN(nn.Module):\n", + " def __init__(self) -> None:\n", + " super().__init__()\n", + "\n", + " self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False)\n", + " self.iaf1 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + " self.pool1 = nn.AvgPool2d(2,2)\n", + "\n", + " self.conv2 = nn.Conv2d(10, 10, 2, 1, bias=False)\n", + " self.iaf2 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + " self.pool2 = nn.AvgPool2d(3,3)\n", + "\n", + " self.conv3 = nn.Conv2d(10, 10, 3, 1, bias=False)\n", + " self.iaf3 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + " self.pool3 = nn.AvgPool2d(2,2)\n", + "\n", + " self.flat = nn.Flatten()\n", + "\n", + " self.fc1 = nn.Linear(810, 100, bias=False)\n", + " self.iaf4 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + "\n", + " self.fc2 = nn.Linear(100, 100, bias=False)\n", + " self.iaf5 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + "\n", + " self.fc3 = nn.Linear(100, 100, bias=False)\n", + " self.iaf6 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + "\n", + " self.fc4 = nn.Linear(100, 11, bias=False)\n", + " self.iaf7 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + "\n", + " def detach_neuron_states(self):\n", + " for name, layer in self.named_modules():\n", + " if name != '':\n", + " if isinstance(layer, sl.StatefulLayer):\n", + " for name, buffer in layer.named_buffers():\n", + " buffer.detach_()\n", + "\n", + " def init_weights(self):\n", + " for name, layer in self.named_modules():\n", + " if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear):\n", + " nn.init.xavier_normal_(layer.weight.data)\n", + "\n", + " def forward(self, x):\n", + " \n", + " con1_out = self.conv1(x)\n", + " iaf1_out = self.iaf1(con1_out)\n", + " pool1_out = self.pool1(iaf1_out)\n", + "\n", + " conv2_out = self.conv2(pool1_out)\n", + " iaf2_out = self.iaf2(conv2_out)\n", + " pool2_out = self.pool2(iaf2_out)\n", + "\n", + " conv3_out = self.conv3(pool2_out)\n", + " iaf3_out = self.iaf3(conv3_out)\n", + " pool3_out = self.pool3(iaf3_out)\n", + "\n", + " flat_out = self.flat(pool3_out)\n", + " \n", + " fc1_out = self.fc1(flat_out)\n", + " iaf4_out = self.iaf4(fc1_out)\n", + "\n", + " fc2_out = self.fc2(iaf4_out)\n", + " iaf5_out = self.iaf5(fc2_out)\n", + "\n", + " fc3_out = self.fc3(iaf5_out)\n", + " iaf6_out = self.iaf6(fc3_out)\n", + "\n", + " fc4_out = self.fc4(iaf6_out)\n", + " iaf7_out = self.iaf7(fc4_out)\n", + "\n", + " return iaf7_out" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "snn = SNN().to(device)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "snn.init_weights()" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "optimizer = Adam(snn.parameters(), lr=lr, betas=(0.9, 0.999), eps=1e-8)\n", + "loss_fn = CrossEntropyLoss()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Define train and test" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "def train(dataloader, model, loss_fn, optimizer, epochs, test_func, dataloader_test):\n", + " epochs_y = []\n", + " epochs_x = []\n", + " epochs_acc = []\n", + " model.train()\n", + "\n", + " for e in range(epochs):\n", + " losses = []\n", + " batches = []\n", + " batch_count = 0\n", + " train_p_bar = tqdm(snn_train_dataloader)\n", + "\n", + " for X, y in train_p_bar:\n", + " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", + " X = X.reshape(-1, 2, 128, 128).to(dtype=torch.float, device=device)\n", + " y = y.to(dtype=torch.long, device=device)\n", + "\n", + " # forward\n", + " pred = model(X)\n", + "\n", + " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", + " pred = pred.reshape(batch_size, n_time_steps, -1)\n", + "\n", + " # accumulate all time-steps output for final prediction\n", + " pred = pred.sum(dim = 1)\n", + " loss = loss_fn(pred, y)\n", + "\n", + " # gradient update\n", + " optimizer.zero_grad()\n", + " loss.backward()\n", + " optimizer.step()\n", + "\n", + " # detach the neuron states and activations from current computation graph(necessary)\n", + " model.detach_neuron_states()\n", + "\n", + " train_p_bar.set_description(f\"Epoch {e} - BPTT Training Loss: {round(loss.item(), 4)}\")\n", + "\n", + " batch_count += 1\n", + " losses.append(loss.item())\n", + " batches.append(batch_count)\n", + "\n", + " epochs_y.append(losses)\n", + " epochs_x.append(batches)\n", + "\n", + " acc = test_func(dataloader_test, model)\n", + " print(f'Epoch {e} accuracy: {acc}')\n", + " epochs_acc.append(acc)\n", + "\n", + " return epochs_x, epochs_y, epochs_acc\n" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "def test(dataloader, model):\n", + " correct_predictions = []\n", + " with torch.no_grad():\n", + " test_p_bar = tqdm(dataloader)\n", + " for X, y in test_p_bar:\n", + " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", + " X = X.reshape(-1, 2, 128, 128).to(dtype=torch.float, device=device)\n", + " y = y.to(dtype=torch.long, device=device)\n", + "\n", + " # forward\n", + " output = model(X)\n", + "\n", + " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", + " output = output.reshape(batch_size, n_time_steps, -1)\n", + "\n", + " # accumulate all time-steps output for final prediction\n", + " output = output.sum(dim=1)\n", + "\n", + " # calculate accuracy\n", + " pred = output.argmax(dim=1, keepdim=True)\n", + "\n", + " # compute the total correct predictions\n", + " correct_predictions.append(pred.eq(y.view_as(pred)))\n", + "\n", + " test_p_bar.set_description(f\"Testing Model...\")\n", + " \n", + " correct_predictions = torch.cat(correct_predictions)\n", + " return correct_predictions.sum().item()/(len(correct_predictions))*100" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Training loop" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "ebf2bea3d0124365abf1f2aa248ceab6", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/359 [00:00" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "y_avg = []\n", + "for y in epochs_y:\n", + " y_avg.append(np.mean(y))\n", + "\n", + "plt.plot(np.arange(len(epochs_x)), y_avg, marker = '.')\n", + "plt.xlabel('epoch')\n", + "plt.ylabel('average loss')\n", + "plt.ylim(0,)\n", + "plt.xticks(np.arange(len(epochs_x)))\n", + "for i, txt in enumerate(y_avg):\n", + " if i%2 == 0:\n", + " pass\n", + " else:\n", + " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAG2CAYAAACZEEfAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAAB9HElEQVR4nO3dd1RUx98G8GfpRUB6kaqg2HvB3oIt9qiJJjHGWH5irLFgYkw0ETWJJsbE9ipqrDGxG3vBhgiKXSmKglQLLL3uvH8QNq60XYrg+nzO2XNk7p07c3evu987d4pECCFAREREpKY0qroCRERERJWJwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqbUqDXbOnTuH/v37w87ODhKJBPv27VPYLoTA119/DVtbW+jr66Nnz54ICwtT2OfFixcYNWoUjI2NUbNmTYwdOxapqamv8SyIiIioOqvSYCctLQ1NmzbFb7/9VuT2ZcuWYeXKlVizZg0CAgJgaGiIXr16ITMzU77PqFGjcOfOHZw4cQKHDh3CuXPnMH78+Nd1CkRERFTNSarLQqASiQR79+7FoEGDAOS36tjZ2WHmzJn44osvAABSqRTW1tbYtGkT3n//fdy7dw8NGjRAYGAgWrVqBQA4evQo+vbtiydPnsDOzq6qToeIiIiqCa2qrkBxIiIiEBcXh549e8rTTExM0LZtW/j7++P999+Hv78/atasKQ90AKBnz57Q0NBAQEAABg8eXOSxs7KykJWVJf9bJpPhxYsXMDc3h0QiqbyTIiIiogojhEBKSgrs7OygoVH8w6pqG+zExcUBAKytrRXSra2t5dvi4uJgZWWlsF1LSwtmZmbyfYri4+ODb7/9toJrTERERFUhKioK9vb2xW6vtsFOZfL29saMGTPkf0ulUjg6OiIqKgrGxsZVWDMiIiJSVnJyMhwcHGBkZFTiftU22LGxsQEAxMfHw9bWVp4eHx+PZs2ayfdJSEhQyJebm4sXL17I8xdFV1cXurq6hdKNjY0Z7BAREb1hSuuCUm3n2XFxcYGNjQ1OnTolT0tOTkZAQAA8PDwAAB4eHkhKSsLVq1fl+5w+fRoymQxt27Z97XUmIiKi6qdKW3ZSU1MRHh4u/zsiIgLXr1+HmZkZHB0dMW3aNHz33Xdwc3ODi4sL5s+fDzs7O/mIrfr166N3794YN24c1qxZg5ycHEyePBnvv/8+R2IRERERgCoOdoKCgtCtWzf53wX9aEaPHo1NmzZh9uzZSEtLw/jx45GUlISOHTvi6NGj0NPTk+fZtm0bJk+ejB49ekBDQwNDhw7FypUrX/u5EBERUfVUbebZqUrJyckwMTGBVCplnx0iIqI3hLK/39W2zw4RERFRRWCwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGptWod7OTl5WH+/PlwcXGBvr4+6tSpg0WLFkEIId9HCIGvv/4atra20NfXR8+ePREWFlaFtSYiIqLqpFoHO0uXLsXq1auxatUq3Lt3D0uXLsWyZcvw66+/yvdZtmwZVq5ciTVr1iAgIACGhobo1asXMjMzq7DmREREVF1IxMvNJNXMu+++C2tra2zYsEGeNnToUOjr62Pr1q0QQsDOzg4zZ87EF198AQCQSqWwtrbGpk2b8P777ytVTnJyMkxMTCCVSmFsbFwp50JEREQVS9nf72rdstO+fXucOnUKoaGhAIAbN27gwoUL6NOnDwAgIiICcXFx6NmzpzyPiYkJ2rZtC39//2KPm5WVheTkZIUXERERqSetqq5ASebOnYvk5GS4u7tDU1MTeXl5+P777zFq1CgAQFxcHADA2tpaIZ+1tbV8W1F8fHzw7bffVl7FiYiIqNqo1i07f/75J7Zt24bt27fj2rVr2Lx5M3788Uds3ry5XMf19vaGVCqVv6KioiqoxkRERFTdVOuWnVmzZmHu3LnyvjeNGzfG48eP4ePjg9GjR8PGxgYAEB8fD1tbW3m++Ph4NGvWrNjj6urqQldXt1LrTkRERNVDtW7ZSU9Ph4aGYhU1NTUhk8kAAC4uLrCxscGpU6fk25OTkxEQEAAPD4/XWlciqhqx0gxcevAMsdKMqq4KVSJnZ2dIJJJCLy8vLwBA165dC22bOHFiiccs6ngSiQQ//PCDwn6HDx9G27Ztoa+vD1NTUwwaNKiyTpMqSbVu2enfvz++//57ODo6omHDhggODsby5cvx6aefAsi/UKdNm4bvvvsObm5ucHFxwfz582FnZ8eLkegtsCswEt57bkEmAA0J4DOkMUa0dqzqalElCAwMRF5envzv27dv45133sGwYcPkaePGjcPChQvlfxsYGJR4zNjYWIW/jxw5grFjx2Lo0KHytL///hvjxo3D4sWL0b17d+Tm5uL27dvlPR16zap1sPPrr79i/vz5mDRpEhISEmBnZ4cJEybg66+/lu8ze/ZspKWlYfz48UhKSkLHjh1x9OhR6OnpVWHNiaiyxUozMHfPLRRMniETwLw9t9G5riVsTfSrtnJU4SwtLRX+XrJkCerUqYMuXbrI0wwMDOTdG5Tx6r779+9Ht27dULt2bQBAbm4upk6dih9++AFjx46V79egQYOynAJVoWr9GMvIyAg///wzHj9+jIyMDDx48ADfffcddHR05PtIJBIsXLgQcXFxyMzMxMmTJ1G3bt0qrDURvQ7XHifi1VnC8oTAo2fpVVMhem2ys7OxdetWfPrpp5BIJPL0bdu2wcLCAo0aNYK3tzfS05W/FuLj43H48GGFoObatWuIjo6GhoYGmjdvDltbW/Tp04ctO2+gat2yQ0RUlPTsXKw8XXhZGE0J4GxR8qMLevPt27cPSUlJ+OSTT+RpI0eOhJOTE+zs7HDz5k3MmTMHISEh2LNnj1LH3Lx5M4yMjDBkyBB52sOHDwEA33zzDZYvXw5nZ2f89NNP6Nq1K0JDQ2FmZlah50WVh8EOEb1R8mQCU3ZcR0hcKgx1NJGRkwfZvy08ozu48BHWW2DDhg3o06cP7Ozs5Gnjx4+X/7tx48awtbVFjx498ODBA9SpU6fUY27cuBGjRo1S6AJRMBjmyy+/lPfj8fX1hb29PXbv3o0JEyZU1ClRJavWj7GIiF7l8889nLwXDx0tDWwZ2xYX53ZH30b5fS9uRiWhGq+AQxXg8ePHOHnyJD777LMS92vbti0AIDw8vNRjnj9/HiEhIYWOWTClyct9dHR1dVG7dm1ERkaqWnWqQgx2iOiNsfXyY/zfhQgAwE/DmqKlkylsTfSxYEBD6GhqIOhxIgIiXlRxLaky+fr6wsrKCv369Stxv+vXrwOAwhxsxdmwYQNatmyJpk2bKqS3bNkSurq6CAkJkafl5OTg0aNHcHJyUr3yVGUY7BBRmZU298mECRNQp04d6Ovrw9LSEgMHDsT9+/eVPv7EiRMhkUjw888/wy/0KRYcuAMA+Li+Nv7v6//BwsICxsbGGNq3J1rrxgAAfjtT+p08vZlkMhl8fX0xevRoaGn91wvjwYMHWLRoEa5evYpHjx7hwIED+Pjjj9G5c2c0adJEvp+7uzv27t2rcMzk5GTs3r27yJYiY2NjTJw4ERMmTJBf2zo6OkhISMCECRMq5Tp/WWhoKAYOHCi/zjt27IgzZ84ofVz6D4MdIiqzwMBAxMbGyl8nTpwAAPncJy1btoSvry/u3buHY8eOQQgBT09PhflSirN3715cvnwZdnZ2SEjOhNe2a8iTCQxtYY+d309Gbm4uTp8+jatXr6Jp06bYu3QKRHoizoc9Q3BkYqWeN1WNkydPIjIyUj7XWgEdHR2cPHkSnp6ecHd3x8yZMzF06FAcPHhQYb+QkBBIpVKFtJ07d0IIgQ8++KDIMn/44QeMHz8e5ubmMDQ0RKdOnbB+/XoAFX+dv+rdd98tdJ2/++67Ja79SMUQJKRSqQAgpFJpVVeF6I02depUUadOHSGTyYrcfuPGDQFAhIeHl3icJ0+eiFq1aonbt28LB0dH4dxvknCac0gMX3NJRMfGCwDi3Llz8v2Tk5MFADH0yzXCac4hMXbTlQo9L8rn5OQkABR6TZo0SQghxPjx40Xt2rWFnp6esLCwEAMGDBD37t0r8Zh///23eOedd4SZmZkAIIKDgwvts3btWtGlSxdhZGQkAIjExMQyn0NMUrq4GP5UxCSll/kYlXGdOzk5iRUrVsi3PX36tNjr/MSJE2WuuyrehM9b2d9vtuwQUYUobu6TAmlpafD19YWLiwscHByKPY5MJsNHH32EWbNmobabO56lZkOakQMXC0Os/aglbK0tUa9ePWzZsgVpaWnIzc3F2rVrYWVlhXmj+0IiAU7eS8DdmOTKPN23UmW05KWlpaFjx45YunRpsfukp6ejd+/emDdvXrnqvyswEh2WnMbI9QHosOQ0dgWq3sm4Mq7zhg0bFtpubm6OevXqoW/fvvJHaMbGxgCAd955B15eXnjx4gU+//xz1KtXD/r6+nB0dMSUKVMKtV69SpllMgYMGICcnBzo6OjA0tISQ4cOxY4dOwC8OZ+3glICu7cCW3aIym/Xrl1CU1NTREdHK6T/9ttvwtDQUAAQ9erVK/Vud/HixeKdd94Rubl5YuIfQULT2ErY9Z4gHj5Nle8TFRUlWrZsKSQSidDU1BS2trbi2rVrQgghvLZdFU5zDolJ265W/EmSgopq4RBCiIiIiGLv9AucOXOmzC07MUnpwnnuIeE0579X7bmHVW7hqejrvOC9e7VlR4j867xJkyYCgNDQ0BBWVlZi9erVAoA4c+aMuHXrlhgyZIg4cOCACA8PF6dOnRJubm5i6NChJZYdGxur8Nq4caOQSCTiwYMH8n2WL18u/P39xaNHj8TFixeFh4eHsLGxqXafN1t2iN4SJXUSLuud3zfffAN3d3cYGhrC1NQUPXv2REBAQInljhgxAm5uboX6HowaNQrBwcHw8/ND3bp1MXz4cGRmZhZZ7tWrV/HLL79g06ZN+OF4KI7cjoNEAgxv5QgXC0MAgBACXl5esLKywvnz53HlyhUMGjQI/fv3R2xsLLy6uQIA/rkViwdPU8v6tlIpKqqF43WJeJZWITNuFzXHD1D267yo9w747zqvVasWLly4gMDAQAwdOhRffPEFnJ2d0aVLFzRq1Ah///03+vfvjzp16qB79+74/vvvcfDgQeTm5hZ7DjY2NgqvV5fJAIDp06ejXbt2cHJyQvv27TFz5kzExcVh9OjRb8TnXUip4ddbgC079CZLSEhQuEs7ceJEue/8tm3bJk6cOCEePHggbt++LcaOHSuMjY1FQkKCfB8nJyexcOFCERsbK65cuSI0NDTEzp07SzxuVlaWMDAwENu3by9y+4oVK4REIhEampoCEo381793tU5OTkIIIU6ePCk0NDQK/X91dXUVPj4+Qgghxm4KFE5zDokZu66X9vZRGVVUC0eBym7ZiU5MU2jVKUvLzqNHj4SGhobYt29fifspe51ramrKX8pc51lZWUJDQ0N4enoWW/b69euFhYWF0ucUFxcntLS0xLZt24rd5/nz58LDw0MAqHafN1t2iN4SlpaWCndphw4dki+QWNY7v5EjR6Jnz56oXbs2GjZsiOXLlyM5ORk3b95U2M/IyAg2NjY4fPgwrKysFFaLLooQAkIIZGVlFbn9o48+wuZDfrAbsxK2Y1biizX7YWdnh1mzZuHYsWMAIF/vSEND8etLQ0NDPuPt5O75rTv7rkcj6gXXyqoMFdHC8TrFJRe+5ub2cVdpxm1l5/hR5jq/efMmrl+/Ln8pc53v27cPMpkMzZs3L/K4z549w6JFixRmky5NUctkFJgzZw4MDQ1hbm6Ou3fvwtPT8435vF/FYIdIjZT2aAEApFIpjI2NFeYpKe2Y69atg4mJSaFJ15YsWQIzMzMsXrwY7u7uCtsePnwIHx8fXL16FZGRkbh06RKGDRsGfX199O3bV77fy3OfJObpYNmVdGiaO2HYO+2xbFw/aGtrw8bGBvXq1QMAeHh4wNTUFKNHj8aNGzcQGhqKWbNmISIiQv4j1MyhJjq5WSBPJrDG74Fybx4praRZjE1MTODm5obOnTvjr7/+wv379wvNbVMVDt2IBQB4NrBGPesaAIC07OID/lcVN8dPWa5zc3NzNGrUSOGlzHU+Z84cSCSSIofJJycno1+/fmjQoAG++eYbpc+rqGUyCsyaNQvBwcH4448/IJVK8fz580IzlFfXz/tVDHaI1EhRCyS+TJU7v0OHDqFGjRrQ09PDihUrcOLECVhYWMi3T5kyBTt37sTixYuRk5ODq1evYvbs2fLtenp6OH/+PPr27QtXV1eMGDECRkZGuHTpEqysrOT7Fcx98iw1C2M2BSIlMxetnEyxdGiTIgM2CwsLHD16FKmpqejevTtatWqFCxcuYP/+/QrB2OR/++7sDnqCOGn1u9N8k1VUC8frIpMJ/HMrP9gZ1soBk/69NnZeiUJunkypYxQ3x4+q17myXr3OW7RogUePHuGrr74qdNORkpKC3r17w8jICHv37oW2trZSZRS3TMbLdahbty7Cw8NhZWWFq1ev4vLly8Uer7p83kVS6uGammOfHVIXnp6e4t133y1ym1QqFW3atBG9e/cW2dnZpR4rNTVVhIWFCX9/f/Hpp58KZ2dnER8fX+z+GzZsEFpaWiIzM1OlOsckpYsz9+NFv1/OCac5h0SnpafFsxTVjlGc91ZfFE5zDomFB+9UyPFIiLy8POHo6CjmzJmjkP7gwQOxePFiERQUJB4/fiwuXrwo+vfvL8zMzBSum3r16ok9e/bI/37+/LkIDg4Whw8fFgDEzp07RXBwsIiNjZXvExsbK4KDg8X69evlc88EBweL58+fK1XnwIjnwmnOIdHo66MiIztXZObkihYLjwunOYfEkVuxpR+ggpRnjp8FCxYIGxsbkZOTo5AulUpFu3btRJcuXURaWppKxxw9erRo2bJlifsUfN7/+9//5H0Bhag+n7eyv98MdgSDHVIPJXWeTE5OFh4eHqJHjx4iIyOjTMd3dXUVixcvLnb77du3BQBx//59pY+588pj4fLScOB6X/4jwuJTylS/opwNScg/7lf/VFgA9bY7duyYACBCQkIU0qOjo0WfPn2ElZWV0NbWFvb29mLkyJGFrgcAwtfXV/63r69vkRPXLViwQL7PggULitzn5eOUZMH+28JpziExfWewPG3JkXvCac4h8eH/XVb1LSiTl691l7mHxM4rj5XOW1yAKZVKRdu2bUXjxo1FeHi4wkCF3Nxc+X6vBhwFeQ0MDMTq1asLlXf58mXx66+/iuDgYLFlyxYBQDRv3lzUqVNHfjNTXT5vBjsqYLBD6qAy7vxeVrt2bYUvpFdt3bpVaGhoiBcvXih1vIinqcL5ldExLnMPlWtm21fJZDLR/9fzwmnOIbH0SMkzu74pSpvVNiMjQ0yaNEmYmZkJQ0NDMWTIEBEXF6f08SdMmCAAFJrzpahyC0a/qaoiZjFWVl6eTLT+7oRwmnNInLz73/sQ+TxNPu/Oy3M4VYaYpHSFoF7VkWDFBZgFo5WKekVERMj3KypQWLt2rdDX1xdJSUmFyrt586bo1q2bMDMzE7q6usLZ2Vl8/OlnYt+Fm2X6zCrz81b291u5HopEVK3JZDL834aN6DlgGJ6m5cDWJP+/dnJyMjw9PZGeno6tW7ciOTkZycn5MwtbWlpCU1MTQH7nSR8fHwwePBhpaWn4/vvvMWDAANja2uLZs2f47bffEB0dLZ851d/fHwEBAejWrRuMjIzg7++P6dOn48MPP4SpqWkJ9RQIiHiBfcHROHAjGuLV7QJ49CxdpREyJZFIJPDq5ooJf1zFFv/HmNC5DkwMlOvPUF0FBgYqzFB7+/ZtvPPOO/LPZvr06Th8+DB2794NExMTTJ48GUOGDMHFixdLPXZJ6zQBwMKFCzFu3Dj530ZGRirXf3vAY3y59zYEAA0J4DOkMUa0dlT5OMoKepyIhJQsGOlpoaPbf33OHMwM0K2eFU7fT8C2y4/x1bsNKq0OEc/SICtmjh9lrnVPT89CHYMBoGvXrkWmv0oIgVhpBi49eAYXC0PYmuhj/PjxRfbdE0LAvnY9rN2xH09Ts/AsNRsn7sTh0M1Y+B2MhORgJLrWs0QDO+NSywWAuzHJOBvy9LV93sVhsEOkBr76bTuin0ThtGiEDktOy79Qrl27Jp8M0NXVVSFPREQEnJ2dAeR3ngwKe4J20gyY6mri/v372Lx5M549ewZzc3O0bt0a58+fl09rr6uri507d+Kbb75BVlYWXFxcMH36dMyYMaPI+oXGp2BvcDT2B0cjpoTOwpoSCZwtDJQ+7+joaMyZMwdHjhxBeno6XF1d4evri1atWgEA4uPjsX3ZXMTtO4Ts9FS0+6cNDmzfADc3t2KP2bVrV/j5+RVK79u3Lw4fPgwg/wdh+vTpWLN2LbIyM6GhoQFXV1ds27ZNoew5c+bg+PHjSEpKQufOnfHrr7+Wu2wLCwssWLAA69evR1JSEiwtLeHo6IguXbpAKpViw4YN2L59O7p37w4gvzNx/fr1cfnyZbRr167E9/Lzzz/HsWPHiu14XDDVQFnFSjPkgQ6QH9zO23MbnetaVliA+6pDN2MAAJ4NbKCrpamw7cN2jjh9PwG7rz7BF73qQU9bs6hDlJuFoW6hNA0JVLrWy2NXYCS899yCTAASCTCilQNcrWrkBzMp2XiWmiV/PU/NRu6rkdlLBIAzIU9xJuSpyvV4HZ93cRjsEL3hYpLSsSPaFE5zDgHI/0KZ8/ctbLr0GLVqGmLOXzdgUUMXFjV0YGmkB4saOrAw0oVpDV0IIfBnUBRc5h7CthfAjn8DpT179pRYZosWLUoclQEA8cmZOHA9BnuDo3E39r91qoz0tNCvsS0GNa+FiGdp+GrvbeQJAU2JBIuHNFL6SzAxMREdOnRAt27dcOTIEVhaWiIsLEzesiSEwKBBg6CtrY3vf9+C5WejkBB8AD169MS9e3dhaGhY5HH37NmD7Oxs+d/Pnz9H06ZN5S0nQP4M0ytXroSuQ2OYN3kHabdOIv5ZDAwMDAqVvX//fhgbG2P58uXo2bMn7t4tX9nLli3DypUrsXnzZtjb26N9+/YwNDREVlYWrl69ipycHPTs2VO+v7u7OxwdHeHv719ssFPaOk0FlixZgkWLFsHR0REjR47E9OnTlZ7CAADO3E8o1JqnSguHqvJkAv/cyl8h/N2mtoW2d6lrBXtTfTxJzMDBGzEY1qpyZv7dez26UJqbVQ3YGBce7l3RYqUZ8kAHAIQAdgZGlZrPWE8LFka60NHUwP24lELbezeyLvUzi03KwNE78Qpplfl5l4TBDlEFUKaFQdW7/D179mDx4sUIDw9HTk4O3NzcMHPmTHz00UcAgJycHMz1nocNO/Yg+Wk0NHQNoefUFDW7fAItI3Pci03GvdiSF8PU1pAg56W7OJkA5v59C7FJmahtVSM/QKqhC4sauqhpoF3s3D2x0gxEPEuDlZEebkQlYd/1aFwMfyb/gtXWlKBrPSsMbl4L3d2t5HfQ7Wqbo2s9Szx6lg5nCwOVvgCXLl0KBwcH+Pr6ytNcXFzk/w4LC8Ply5dx+/ZtuNdvgEOxfnhoNhGJ6z7Bjh07ih1ua2ZmpvD3zp07YWBgIA84hBD48ccfoWlsBesPFgMADFzb4MmvH8LvchAaNGigUHZB8LB69WrY2NiUu+yff/4ZX331FQYOHIg///wTeXl5SEtLk084p6Ojg5o1ayocx9raGnFxcSW+l1paWpgyZUqx+0yZMgUtWrSAmZkZLl26BG9vb8TGxmL58uXF5nlZTp4Mmy49KpSuamueKq5EvMCz1CyY6GujQx2LQts1NSQY2dYRy46GYGtAZKUEOwnJmfC9GAEAWDq0CTQ1AO+/byEkPhV7g6MxpIV9hZf5socJhR+hAYBHHXO42xjBooZu/v9xI51/b4p0YV5DR94KFivNQIclpxWOoSmRYEH/hqUHO9IMHL8bXyjv62rRehmDHaJyUqWFQZW7fDMzM3z55Zdwd3eHjo4ODh06hDFjxsDKygq9evVCeMxzbD54FtqthsHWygWyzFS8OLUOT/csQq1PfobP4MbIFaJQM/Wz1Gw8S8lCSlauQqBTQAD4+VRYoXQtDQnMa+R/IVoa6cq/GKNepOOfW7GF7tgBoKWTKQY3r4V+jW1haqhT5HnamuiX6S7vwIED6NWrF4YNGwY/Pz/UqlULkyZNkvcpKZjrQ09PD5oaEvyvax3M/usmMmWa8Dt3vtiA41UbNmzA+++/L/+cHj58iPT0dBg2aIOn+3yQGXUbmjXMoWlsiTPnLuB/n36sUHYBDQ0N6Orq4sKFC2UuOyIiAnFxcfKWm4JZjJOTk+Hv74+2bdsqddyXFazTdO3atWKDWQAKjyibNGkCHR0dTJgwAT4+PtDVLfyY5lXrzj1EaHwq9LU1kZWbJ/8BnOlZt9Lu8g/fyn+E1auhNXS0ip5WbngrB6w4EYobUUm49USKxvYmFVqHX0+HIzNHhhaONTG8lT0kEgnik7Pww7EQfHvwLjq6WcDKqPJaeC4+eFYoTVMiwfLhTZV6321N9OEzpDHm7VG9BbY8eSsagx2iclKlhUGVu/yuXbsq/D116lRs3rwZFy5cQE23Vpi07RZqDP4G5oY6eK+lPf7vfATwzkTEbZmBme3NMKJNyZ0AM3PycDdGiqFr/BUWSJQA6OZuhbSsXHlwJM3IQa5MID45C/FFTLv/qnGdXPBRO2c4mlfeHdzDhw+xevVqzJgxA/PmzUNgYCCmTJkCHR0djB49Wv74xtvbG2vXrkW/hpaYNX8fcpKf4npIhFJlXLlyBbdv38aGDRsAAJHP0+G14QwAIC3kIoxbD4K1x3BkxYbhxfHfccrvImKSMgqVbWhoiBUrVuDJkyeIjY0tU9kA5K0z1tbW8lmM9+zZg23btiEuLg42NjbIzs5GUlKSQutOfHx8sX1tzp8/j4SEBDg6/ne95OXlYebMmfj555/x6NGjIvO1bdsWubm5ePTokXzW3+KEJ6Tgl38D6O8GNUJ7V3NM+OMqbj6R4kVadol5yyo3T4ajt/Pfr35Niu5wDQAWNXTRt7Et9l+PwdbLj7H0vSYVVofHz9Ow40okAGB2b3d5MDm+c238cysWd2KSsWD/Haz+sGWFlfmyq48Tse7cQwD5fXWEQJkCjhGtHdG5btlaYMuTtyJxBmWicjpw4ABatWqFYcOGwcrKCs2bN8f69evl20u7y1eGEAKnTp1CSEgIsi3rYdT/BeB5WjYa2hnjwOcd4d23Pi7M7QbvHk6QSCT4qEvx/S4K6GlrooWTGZYMaQzNf7+ENSUSLBnaGBs/aY1dEzxwamZX3FjgidDv+sDfuzsOTO6AjZ+0wrKhTTCrVz30bmhd5LG7u1tXaqAD5PczadGiBRYvXozmzZtj/PjxGDduHNasWQMA0NbWxp49exAaGgozMzOYGNWASWIo9Gq3xJOkTGTnlj5z7oYNG9C4cWM0b9kKa/wewPNnP1yPTAIAWDi4wrzraOhY14Fxs97QqmmNpLgovLPcD1uvPMHuv/6Wl21gYIAzZ86gT58+hdb0Kq3sNm3aFLm9qFmMW7ZsCW1tbZw6dUqeFhISgsjISHh4eBR5HGXWaSrK9evXoaGhoTBLcFHyZAKz/7qJ7FwZutazxJAWtWBroo/pPesCAP4MikK6Css2KCv/EVY2ahpoo30d8xL3/bCdEwBg/41oSDNyKqwOK06EIlcm0LmuJdrV/q8O2poaWPZeE2hpSHDkdhyO3FIuAFaFND0HU3YEI1cm8G4TW1yc0w07xrXDhbndyjQaytZEHx51zMsUrJQnb0Vhyw5ROanawqDKXb5UKkWtWrWQlZUFTU1NdB/rjV0xpgAE+je1w7KhTaCvk/9s3VRXgs2/fI8PPvgAxsbKDQsFlLvz0tHSKPJxU1U+k7e1tUWDBorDhevXr4+///5b/nfLli1x/fp1SKVSZGdnw6imGSxqN0KuSR3sDX5S4pd+Wloadu7ciQnTvdH/1wvyTpqt67vgEICOzerh97nd5e+bxy59PElLRFp2Hr49eBdN7U2w7R8/2Bvmry9maWmJtm3byvtxlaSg7IULFyqkF7TOxMbGKkw1EB8fj2bNmsHExARjx47FjBkzYGZmBmNjY3z++efw8PBQ6Jz88lQD5ubmMDdXDAZeXaeprFMNAMDmS49wLTIJNXS1sHhwY3nrRpe6lnA2N8Cj5+nYGxyNUW2dSn1fVHHo3wCid0MbaGuWHGC2cjJFPWsjhMSnYM+1JxjTwaXE/ZVxLzYZ+2/kP0ab3atwy1dDOxNM7FIHq86EY/7+O2hX27zYR72qEkJg9t83EJ2UAUczA/gMaQwjPW3Y1Xz9fWWqC7bsEJWTqi0MqtzlGxkZ4fr16zh29gLq9huHoxt+QGbkTczt446V7zeTBzo5OTkYPnw4hBBYvXq1yudQ1juvgmfyL7cMva5n8h06dEBISIhCWmhoKJycCv9ompiYwNLSElGPHiI9JhQGbm2x+uyDEtdF+mP7TqRlZGJnkhPux6WgpoE2fnivCfZ7D4aenh5u3Lghf98MJTl4EvUYtZ0c8N2gRjDS1cKNJ1IMWHURv1+MgaGJKcLCwhAUFISBAweWem67d+9GVlYWPvzwQ4V0FxcX2NjYYNaiH+VTDXh8ewj+lwPkLTcrVqzAu+++i6FDh6Jz586wsbEpNLouJCQEj2Of4tKDZ4iVZpRan4KpBrp06YKGDRvi+++/x/Tp07Fu3boS80U+T8cPx/I/o7l93GFX87/rQkNDgo88nAHkB0TKzBejrJcfYb1bwiOsAhKJBB965F83Wy8/rpC6/HgsBEIA/ZrYolGtovsBfd7DFa5WNfAsNQuLDt0td5kFtvg/xrE78dDWlGDVyOYw0nuz55aqEBU+neEbiDMoq4cnT56IUaNGCTMzM6GnpycaNWokAgMD5dtRzEyjy5YtK/aYubm54quvvhLOzs5CT09P1K5dWyxcuFDIZDL5Po6OjqJZs2bCxsZG6OnpiR49eohvvvlG2NnZFTpeUlKSSEhIEEII0aZNG/mstyW5EZUo2n5/UjjNOSRMm/cSrTp0VdienZ0tBg0aJJo0aSKePXtW6vEqQ0xSurgU/uy1zIhb4MqVK0JLS0t8//33IiwsTGzbtk0YGBiIrVu3yvf5888/xZkzZ8SDBw/Evn37hJOTkxgwaLBo9u0x4TTnkNgX/ER89NFHYu7cuQrHPnY7Vhg5NRIG7p3kywy8vNyEl5eXACA+/PBDcejQIdGyZUshkUjExo0bhRBCxEkzhOfnS4T1B4uF3YT/E+4fLRTWdg5iyJAhCuUUVbYQQnTs2FGMGDFCIS0vTyZuPUkSvcfMEBq6hsJyyHxh++kqoe/WTmiZWIsuPsfEqPWXxdQd18Sig3fE6rPh4q+gKHE2JEHcjk4S8dIMkZObJ4Qo39IFypLJZOKDdf7Cac4hMWLtJZGXJyu0jzQjW9Sff0Q4zTkkLoY9rbCyz4XmLxPSfOFx+TmXJiUzRzQoqEt4+epSsBZXbe/D4kFCycufXH38Qj6T8+l7xa89p6xbT5KE27x/hNOcQ2LD+YflPl51xxmU6a1S2ogoAIUeGR05cgRjx47F0KFDiz3u0qVLsXr1amzevBkNGzZEUFAQxowZAxMTE/kwXVNTU9y6dQt///03XFxcMH/+fCxfvhz169cvdDwTk/w7vIK7/EWLFpV4XnuuPcHcPbeQnStDHUtDmLlb4mnMf3NkFLTohIWF4cyZM4UeR7wuZR1RVR6tW7fG3r174e3tjYULF8LFxQU///wzRo0aJd8nNjYWM2bMQHx8PGxtbfHxxx9j/vz5WHfhMX48HorfzoQjOzJS3sIWJ83EggO3cejcVaQ8vo1GY5fh97Ft0MnNUqHsX3/9Fc+ePcPOnTuxdetWGBgYYOHChRgzZgwAwNpYD/1c9RG8eykSniYg3tAUNRp2h82gGXiakgVLo/zRS5EvlV0gJCQEFy5cwPHjxxGdlIELYU9xPuwZLj14jhdp2RCW3WDUMgbPj/0KWWYa9OwbwGr4QjxKysGjpMIjb14mkQAmetpIeqlfSmVN9LYzMAqXHjyHnrYGlg5tAg2NwiO9jPW0MbSFPf64/BibLj1Ce9fCw8PL4vDNfx9hNbKBVimPsArU0NXCoOa1sC0gEtsuR6J9EUPVlSGEwLKj+a1Zw1vZo7ZljRL3b+Foik87uGDDhQjM23sLx6d3LnNLTGpWLiZvv4bsPBl61rfGmA7OZTqOOpIIUYFth2+o5ORkmJiYQCqVqtTXgaqPuXPn4uLFizh//rzSeQYNGoSUlBSFzpyvevfdd2Ftba0wImbo0KHQ19fH1q1bIYSApaUlEhMTsWjRIgwfPhxnz57FuHHj4OXlhVWrVgHIfyxRMNPtrVu3MHXqVLRs2VKhf8nHH3+MWrVqwcfHB7l5MvT5dCZuZppDy9QW7RyN0FLjERbM/xKrV6/GZ599hpycHLz33nu4du0aDh06BGvr/zoLm5mZQUenYp7/qyNpRg46LjmNlKxcTO9ZF0Nb1sLp+wlYdjQEqVm50NKQYFzn2pjS3U3+qLCs0rJy8dPxUGy6FAGZyJ+sbV7f+hjeygHxKZmIeJYmn8I/JTMHlx++yA9wwp/h4dM0hWMZ6miiuWNNXAx/rjDUX0MC/DS8GWQyoTjFQGoWnqbk//tFWlaR860U2DGuHTxK6cirrFhpBjyXn0NKVi6+6lcfn3WqXey+4Qkp6Ln8HDQkgN+sbnAwK1+/kpw8GVp/fxJJ6TnY/llblQKoe7HJ6PPLeWhpSHBpbndYlWHSvzMhCRjjGwgdLQ34zeqqVACZkZ2HXj+fQ+SLdIxq64jvBzdWuVwhBKbtuo7912NgZ6KHf6Z2Qk0D9f8OUPb3my07pBZKm3PlVfHx8Th8+DA2b95c4nHbt2+PdevWITQ0FHXr1sWNGzdw4cIF+URqEREReP78OX755ResX79e3sLg5uYmX3cKKL6F4WUFd/lJ6dn4fEcwroTFIv3+LiDtBc4YGiDW3R1bt27FiBEjAORPZHjgwAEAQLNmzRSOdebMmUJD1+k/JvraaO1shtMhCVhxMhQrTobKtzV3rAmfIY3hblMxNz6Gulr4un8DDGpuB+89t3AnJhlz99zCWr+HePwif8I3CQBHMwM8ScpA3ksRiYYEaOpQE51cLdCpriWaOdSEtqYGdgVGFpq7ZHDzWiXWI08mkJiejXuxyfh445VC0w04mVdMq44QAl/tvY2UrFw0c6hZamdfVysjdHKzwPmwZ9h6+TG8+xZuEVXFpQfPkZSeA4saOmjjYlZ6hpfUtzVGKydTBD1OxM7AKEzpUfykn0WRyf5r1fmkvbPSLWX6OppYMrQxRq4PwLaASLzbxE7lwHN30BPsvx4DTQ0JVn7Q/K0IdFTBlh2wZUcdFAzrnjFjBoYNG4bAwEBMnToVa9aswejRowvtv2zZMixZsgQxMTEKQ8JfJZPJMG/ePCxbtgyamprIy8vD999/D29vbwDApUuX0KFDB8TExMDW9r/p6IcPHw6JRIJdu3YpfQ6x0gycD3uGn0+GIiYpE/ramvhxWFP0a1J4mnsqn6JmhQWALzzr4n9dXaFZxCOXipCbJ4PvxUf46XgIMosZ+u5sboCObhbo6GoJjzrmMNEv+pFGrDSjzHOX5AdLt5D30vmPauuIhQMblfvc91+PxtSd16GjqYFDUzqirnXpi4WevBuPz7YEwURfG5e9e5SrNW32XzfwZ9ATfNjOEd8NUr2FZF9wNKbtug5bEz2cn91N6cdgAHDgRgym7AiGka4Wzs3upvLoqnl7b2F7QCSczA1wdGpnpd+HsPgU9F91AZk5MszqVQ9e3VxLz6Qm2LJDbxWZTIZWrVph8eL86fubN2+O27dvFxvsbNy4EaNGjSox0AGAP//8E9u2bcP27dvRsGFDXL9+HdOmTYOdnV2Rxy2rLZceYcGBO/JHE6YG2tj2WTulVxYm1RS1CjUAtHQyq7RABwC0NDUwrnNtWBnpYuqu64W2r3y/GQY0K7mFpkB5+km9PN1A0OMXWH4iFNsCIvEsNQu/vN+8zAtiPkvNwjcH7gAAPu/uqlSgA+RPYuloZoDIF+nYdz0aH5QyIWZxsnNlOPbvWkzKjMIqSp/GNlh4SAex0kycvp8Az4bKLXyakyfDT8fzW3XGda5dpmHk3n3cceZ+Ah4/T8dPx0OUWok9IzsPXtuvITNHhk5uFvhflzoql/s24NBzUgvFzbkSGRlZaN/z588jJCREqSn7Z82ahblz5+L9999H48aN8dFHH2H69Onw8fEB8N+8J/HxiovdlTRjbYHcPBn8Qp9iwpYgfP1SoAPk9ykxNeRw0criYmGIV2Oa17lmT5vaZkWW31rFxy7lUTBs/vPubvhtZAvoaGrg2J14fLzhCqTpZZtYb8GBO0hMz0F9W2NM7Kr8j66mhgQf/zv0uzzD0C+GP4M0IweWRrpo7Vy291JXSxPD/10j64/Lj5XO92dQFB4/T4e5oQ7GdizbPD1GetpY/G9/nY0XI3AtMrHUPAsP3UFofCosauhi+fBmRXYEJwY7pCZUmXNlw4YNaNmyJZo2bVrqcdPT0wuNltHU1IRMlv8IomDek5c7OScnJyMgIKDIGWuFELj1RIqFB++inc9pjN54BcfuxhfaTyaAR8/SS60flU1Vzg9UHcp/Vd/Gttj8aRsY6WrhyqMXGL7WX6n5d1527E4cDt+MhaaGBD+816TUifxeNayVA/S1NXE/LgWXH75QKW+BQ/+OwurbyKZcLXSj2jpCIgHOhz3Do2dppe6fkZ2Hlf8uhzG5uysMdcv+0KSbe/6CuTIBzP7rJrJy84rd98CNGOy4EgWJBPjl/WbyUX5UGB9jkVqYPn062rdvj8WLF2P48OG4cuUK1q1bV2jSs+TkZOzevRs//fRTkcfp0aMHBg8ejMmTJwMA+vfvj++//x6Ojo5o2LAhgoODsXz5cnz66acA8icjmzZtGr777ju4ubnJh57b2dlh0KBB8uNGvUjHgRsx2BscjfCEVHm6qYE2urtbYU9wtEKH0apaGfhtUtVr9lR1+a/yqGOOPyd6YPTGKwiJT8HQ3y9hy9g2cLUq/VGUND0HX+27DQCY0Ll2sZPolcREXxtDWuQP/d586ZHKHXSzcvNw/G7pa2Epw8HMAF3qWuJsyFNsvxKJeaV0mt7s/wjxyVmoVVMfI9uW7RHcy75+twHOhz1FeEIqVp0Ox0zPwjMwP3qWhnl7bgEAJndzRYcKGravrhjskFpQZs4VANi5cyeEEPjggw+KPE5oWDiCQyMRK82ArYk+fv31V8yfPx+TJk1CQkIC7OzsMGHCBHz99dfyPLNnz0ZaWhrGjx+PpKQkNG7ZFlt270OWTBN7r0Ri77VoXHn0352qrpYG3mlgjcHNa6FzXUtoa2qgjYtZtVgZ+G1TFfMDVafyX1Xf1hh7JrXHxxuv4OHTNAxd7Y+Nn7RCS6eSHwl9d/gunqZkoY6locojmF42ur0ztgVE4vjdOEQnZaBWTeXfmwthz5CSmQsrI120cip5CQtlfNTOCWdDnuLPoCjMeKdusf2YpBk5WH32AQBg+jt1oatVvqkKAMDUUAcLBzbCpG3XsPrsA/RpZKvQfy8rNw+Td1xDalYu2jibYWo53vO3BUdjgaOxKN+uwEh477kFmcgf8uszpLFKC+a9nB/Ib53J+/e/l0QCeNQ2x6DmtdC7kQ2Mi5g0rDyja4gq0ou0bHy6KRDXo5Kgq6WBVSNb4J0GRS/66hf6FKM3XoFEAvw10aPUwKg0I9dfxqUHzzGxSx3M7eOudL4Zu65jT3A0PmnvjG8GlL4QbmnyZAKdl51BdFIGfhrWFENb2he53w/H7uO3Mw/gZlUDR6d1rtAO7hP/uIqjd+LQ0M4Y+7w6yB8NfnvwDnwvPoKpgTb+mdrprf6+4GgsIiVl5uTh6O04zP37lryTsEwAc/6+BZ9/7ivV4U8mEwqz0gJAnhCoY2mI4a0cMKCZXalfSNXtLp/eXmaGOtg+ri28tl3DmZCnmPBHEBYPboz3XxkllZqVK3+UMtrDudyBDpA/P82lB8+xMzAS03q6KTUyLDMnDyf+7fvWv2nFTNWgqSHByLaO+OFYCLYGPC4y2ElIycTGC48AAF/0qlfhI/kWDmoI/4fPcScmGevOPYRXN1ccvxMH34v5Zf40vCm/M5TEYIfeOjKZwN3YZJwPe4YL4U8R+CgR2cXMefJqAKOq7wY1rrBZaYleJwMdLaz7uBW899zCX1fzly15mpKFyd1d5SuXLzt6H9FJGXAw08fs3oX7lZRFj/rWsDfVx5PEDBy4HoPhrR1KzXMu9ClSsnJha6KH5g7lf4RVYHgrB/x8MhTBkUm4HS0t1Bfpt9PhyMjJQzOHmvAspuWrPKyM9PD1uw0wc/cN/HwyFJoaEvx2JhwA8FlHF3R3r/gy1RWDHVIrsdIMhen3C8QkZeBC2DOcC3sqX2PoZVZGukhIyVJI05AAW8e2VWqEw9OULHy4IUBh7hZ2MqY3nbamBn54rwmsjXXx25kH+OlEKOJTMjGxSx0cvxOPLf75Q7OXDGkCA52K+TnR1JDgo3ZO8DlyH76XHmFYK3t5cFWcw7f+HYXV2LZCh15bGumidyNbHLwRg20Bj+EzpIl8W9SLdGy/kj+1xeze9UqtY1kNaVELa889QGh8KpYcuQ8AsDfVx+zeyj/iIwY7pEZe7XPzYTsnSIBi1xjyqGOOjq4W6OhmiTqWhvgzKKpQJ2Fl19VxszaCz5DG7GRMakcikWBWL3dY1tDFt4fuYuvlSGy9/N/8Va2dTSt8JNCI1g5YcTIU92KTEfgoscRlHzJz8nDy30dYlTHb+IdtHXHwRgz2BcfAu299eX+7FSdCkZMn0MnNosyLhiojLjlTYQQnkH/z9jwti98vKmCwQ2ohVpqh0DlYJiC/6wSKX2PoZeUdClzdhhITVaRPOrhAU1OC+fvuKKRffZwoH71YUWoa6GBw81rYcSUKmy89KjHYORvyFGnZeahVUx/NHWpWWB0KtHExQ13rGgiNT8Xea9EY3d4Z9+OSsfd6NABgVq+KeXxXnKJm+y6Yh4vfMcpjsENqobjp/3vWt8J7LR1KXGPoZeXtJMxOxqTO6ljWKJRWWT+8o9s7Y8eVKBy9E4eYpAzYFTMM/b9HWDaV8ihJIpHgw3ZO+Hr/HWy9/Bgfezjhx2OhECK/zCb2NSu8zJcVzPbNR+TlwxmUSS24WBji1a85TYkEiwY1Qu9GNkoFOkRUste5zIa7jTHa1TZDnkxgW0DRyzZkZOfh1L3yrYWljMHNa8FARxNhCalY7fcAJ+/FQ0MCzHinclt1gOo32/abisEOqYVnKdl4OdrhFwJRxXvdP7yftHcGAOy4EoXMnMLLJpwNSUB6dh7sTfXRxF71WZuVZaSnjUHN8xdoXXY0f1maYS0d4GpVuKWrMoxo7YgLc7thx7h2uDC3m0rzf1E+PsaiClfciKjKkp0rw6y/bkAIoIe7FT7rVJt9Zogqyevsm9azvjXsTPQQI83EwRsxGNZKcRh6wVpY/ZrYVtpoqAKWNRRHZda2NKzU8l7FR+Tlw5YdNRQdHY0PP/wQ5ubm0NfXR+PGjREUFAQAyMnJwZw5c9C4cWMYGhrCzs4OH3/8MWJiYko85rlz59C/f3/Y2dlBIpFg3759CtsLjuvo6o5alqbo3Kwe6nboh9X/BFbWacqt8XuA+3EpMDXQxrL3msCjjjm/FIgqUcGK6ZX9/0xLUwMfeTgDADa9shp6enYuTt3/9xFW48p7hAXk38D9ejpMIW3Z0RCVF0ulqsNgR80kJiaiQ4cO0NbWxpEjR3D37l389NNPMDXNn2grPT0d165dw/z583Ht2jXs2bMHISEhGDBgQInHTUtLQ9OmTfHbb78VuT09PR3+V4KQ2WgQbEf/AstB85D9IhrTxo6s1C+E0PgU+ZfQNwMawrwGV/0lUifvt3aArpYG7sQk4+rjRHn66fsJyMyRwdHMAI1qVe4yP0UNgMgTAo+epVdquVRx+BhLzSxduhQODg7w9fWVp7m4uMj/bWJighMnTijkWbVqFdq0aYPIyEg4Ohb9LLhPnz7o06dPseWamJjggwVrsfTf59kAYPbORMRtmYHLN0MxuFPTsp5SsfJkArP+uomcPIGe9a0woGnl3t0R0etnaqiDgc3s8GfQE2y69AitnPOHoR9+jY+wOCLqzceWHTVz4MABtGrVCsOGDYOVlRWaN2+O9evXl5hHKpVCIpGgZs2aZSozKzcPCw/eVQh0AECWlQ5AAoMalXPXtfFCBG5EJcFIVwvfDWpc6V94RFQ1Rv/bUfno7TjESTORlpWL0/cTAADvVsJEgq/iiKg3H1t21MzDhw+xevVqzJgxA/PmzUNgYCCmTJkCHR0djB49utD+mZmZmDNnDj744IMyrfgenpCCz3dcx73YZABA+zrmuPzwOfJyspF01hcGDTpj4bEItHC1U2rZBWU9epaGH4/nB1df9qsPGxO9Cjs2EVUvDe1M0MbZDFcevcC2gMdwszZCVq4MLhaGaGBbuY+wCnDS0Dcbgx01I5PJ0KpVKyxevBgA0Lx5c9y+fRtr1qwpFOzk5ORg+PDhEEJg9erVKpUjhMD2gEgsPHQHmTkymBvq4IdhTdDd3RqRz5IxasRwGFoawnzELEQlZeCzLUHYNb6dUisYl36OAnP+vomsXBk6uJpjhBILBRLRm210e2dcefQC2wMi5cPM+zWu/EdYL+OIqDcXH2OpGVtbWzRo0EAhrX79+oiMjFRIKwh0Hj9+jBMnTqjcqrPG7yHm7b2FzBwZOrlZ4MjUTujubo2cnBxMHTcayc9icf7saWyZ2BU1DbRxIyoJM/68DllR0xyraPuVSAREvIC+tiaWDGnCx1dEbwHPhtawNdHD87RsnAl5CqBy1sIi9cRgR8106NABISGKfWdCQ0Ph5OQk/7sg0AkLC8PJkydhbm6u9PH9HzwHAARHJkJbU4Iv+9bH5jFtYGWsh5ycHPTv3x9nzpxBVFQU7O3tMaBbO0xrogFtTQn+uRWHZcfu4+uvv4atrS309fXRs2dPhIWFlVimj48PWrduDSMjI1hYWmHS6A+Q8/wJZveuBwez/A6CEyZMQJ06daCvrw9LS0sMHDgQ9+/fV/q8iKh609bUKDRx4I2opKqpDL1xGOyomenTp+Py5ctYvHgxwsPDsX37dqxbtw5eXl4A8gOd9957D0FBQdi2bRvy8vIQFxeHuLg4ZGdny4/To0cPrFq1Sv53ojQZ03/fh6HfbwcAmOQmwqeLEXo5a0FDQ4KcnBwMGDAAJ0+eRJcuXbBt2zb4+fnhyy+/RNt6tbBkSBMAwNKly/DTil+wZs0aBAQEwNDQEL169UJmZmax5+Tn5wcvLy/4+/vDY/Jy5ObmIHHPArzXxFK+T8uWLeHr64t79+7h2LFjEELA09MTeXmFZ10lojdPrDQDJ/5d3bzAl3tvc64bUo4gIZVKBQAhlUqruioV4uDBg6JRo0ZCV1dXuLu7i3Xr1sm3RURECABFvs6cOSPfz8nJSSxYsEAIIcTjZ2miw5SVReYZPXq00sf98eg9oWloKsy7fyouhj0VQgiRlJQkdHV1xY4dO0o9r7+CooTTnEPCZdoOAUD4+fkVu++NGzcEABEeHq7am0dE1dLF8KfCac6hQq9L4c+qumpUhZT9/WYHZTX07rvv4t133y1ym7Ozs8IspMXxv3EPEc/SsOliBH48HopU/dpovOAofIY0KfI5ubOzM+rXr49evXrhyZMn8PPzQ61atTBp0iR07doVADDIVRtfpCVCx7EZJm69ij2TOsDVygRt27aFv78/3n///WLrk5CSiYWH7gIAPm5liW8BmJmZFblvWloafH194eLiAgcHdl4mUgec64bKg4+xqJBdgZHosOQ0Rq4PwDcH7yI1KxetnU1xZFrnEjsEFgx7d3Nzw7Fjx/C///0PU6ZMwebNmwEA8fH5TdDN6jkhOTMXYzZdwfPULFhbWyMuLq7EOn297w6kGTloaFsDgduXo0OHDmjUqJHCPr///jtq1KiBGjVq4MiRIzhx4gR0dHTK+W4QUXXAuW6oPCRCmdv8l5w5cwbdunWrrPpUieTkZJiYmEAqlZZprhl1IU3PweFbMZi397ZCugTAudld4WBW8sJ3Ojo6aNWqFS5duiRPmzJlCgIDA+Hv749Lly6hQ4cOuB0Wgf/tiUDki3S0dDIFTq2AlqYGdu3aVeRx/7kVi0nbrkFLQ4JWT/7G5XOncOHCBdjb2yvWXypFQkICYmNj8eOPPyI6OhoXL16Enh7n4CFSF7HSDM51Q3LK/n6r/Bird+/esLe3x5gxYzB69Gg+JniDZefKcC0yERfCnuF8+DPcepJUaP0XIL/jzZPEzFKDneKGvf/9998AABsbGwBATmoSNn7SGoN/v4irjxORc+chhr7TochjJqZl4+v9+cGX9e1tuBh8DufOnSsU6AD5S1aYmJjAzc0N7dq1g6mpKfbu3YsPPvigtLeCiN4QnOuGykLlx1jR0dGYPHky/vrrL9SuXRu9evXCn3/+qTCSh6perDQDlx48UxipIIRAaHwKNlyIwBjfK2i28DjeX3cZq86E40ZUfqDjbG6AV2etUfa5eGnD3l1cXGBjY4NTp07B1aoG1n7YEho5GYgJu4XEGs5FHnPhobt4mpKFvAv/h0dXz+L06dMKa30VRwgBIQSysrJK3ZeIiNSbyi07FhYWmD59OqZPn45r167B19cXkyZNwqRJkzBy5EiMHTsWTZtW/KKPpLxdgZHw3nMLMgFoSIDhrRyQkydwIfwp4pMVf/wtauigg6sFOrpaoKObBWxN9LErMBLz9txGnhAqPRefPn062rdvj8WLF2P48OG4cuUK1q1bh3Xr1gEAJBIJpk2bhu+++w5ubm5wcXGBedB6RNUww7lsF/x99QmGtrRHjx49MHjwYDTsORx7g6OReGI1xIMLOHTgAIyMjOT9e0xMTKCvr4+HDx9i165d8PT0hKWlJZ48eYIlS5ZAX18fffv2rfg3mIiI3igq99l5VUxMDNatW4clS5ZAS0sLmZmZ8PDwwJo1a9CwYcOKqmelUqc+O7HSDHRYcrrIx1EAoKulgTYuZujoaoFObpZwtzGChkbhGYjL+lz80KFD8Pb2RlhYGFxcXDBjxgyMGzdOvl0IgQULFmDdunVISkpCx44d0ez9mfgrXAZtTQm2jm2LEd1b4oMPP8JZw66IlWbi8dKiR5b5+vrik08+QUxMDD777DNcvXoViYmJsLa2RufOnfH111+jXr16StediIjeLMr+fpcp2MnJycH+/fuxceNGnDhxAq1atcLYsWPxwQcf4OnTp/jqq69w7do13L17t1wnAeQ/NpszZw6OHDmC9PR0uLq6wtfXF61atQLw34/n+vXrkZSUhA4dOshHBClLnYKdSw+eYeT6gELp/ZvaYkQrR7RyNq2Q9akqkkwmMHnHNfxzKw41DbSx7qOWWOP3EKfvJ8DJ3ABHp3aGvk71qjMREVW9Suug/Pnnn2PHjh0QQuCjjz7CsmXLFIYAGxoa4scff4SdnV3Zav6SxMREdOjQAd26dcORI0dgaWmJsLAwmJqayvdZtmwZVq5cic2bN8PFxQXz589Hr169cPfu3bdyFE5xc1HM61u/2nbq09CQYPnwZohOuowbUUkYvvayfFuvBjYMdIiIqFxUbtnp0aMHPvvsMwwZMgS6urpF7pObm4uLFy+iS5cu5arc3LlzcfHiRZw/f77I7UII2NnZYebMmfjiiy8A5A8/tra2xqZNm0qcpO5l6tSyAwDTdgZj3/UYAP/NRTGitWMV16p0t6OlePfXCwppmhIJLsztVm0DNSIiqjrK/n6rPBrr1KlT+OCDD4oNdABAS0ur3IEOABw4cACtWrXCsGHDYGVlhebNm2P9+vXy7REREYiLi0PPnj3laSYm/83IW5ysrCwkJycrvNRJSmYuAGBEawdcmNvtjQh0ACA5M6dQWp4QePQsvQpqQ0RE6kLlYMfHxwcbN24slL5x40YsXbq0QipVoLQZeQtG5VhbWyvkK21GXh8fH/mcLCYmJmo1V1BWbh4u/bsy+UftnN6oFpGCR3Av43TwRERUXioHO2vXroW7u3uh9IYNG2LNmjUVUqkCMpkMLVq0wOLFi9G8eXOMHz8e48aNK3c53t7ekEql8ldUVFQF1bjqBUYkIiMnD5ZGumho92Y9kuN08EREVBlU7qAcFxcHW9vC6yNZWloiNja2QipVQNkZeePj4xXqFB8fj2bNmhV7XF1d3RIfw73JzoYkAAC61LWERFJ4SHl1N6K1IzrXteR08EREVGFUbtlxcHDAxYsXC6VfvHixQkZgvUyVGXkLJCcnIyAgAB4eHhValzfF2dCnAICu9SyruCZlZ2uiD4865gx0iIioQqjcsjNu3DhMmzYNOTk56N69O4D8TsuzZ8/GzJkzK7RyZZmRd/78+bCzs8OgQYMqtC5vgieJ6QhPSIWGBOjk+uYGO0RERBVJ5WBn1qxZeP78OSZNmiRfD0tPTw9z5syBt7d3hVaudevW2Lt3L7y9vbFw4UK4uLjg559/xqhRo+T7zJ49G2lpaRg/frx8Rt6jR4++lXPsnA3Jb9Vp4WgKEwPtKq4NERFR9VDm5SJSU1Nx79496Ovrw83N7Y3uA6Mu8+x8tjkIJ+/F4wvPupjcXfkZpImIiN5ElTaDcoEaNWqgdevWZc1OFSx/yPkzAEDXelZVXBsiIqLqo0zBTlBQEP78809ERkbKH2UV2LNnT4VUjFQT9CgR6dl5sKihiwa2b27rFBERUUVTeTTWzp070b59e9y7dw979+5FTk4O7ty5g9OnT8PExKQy6khKeHnIeVGrmBMREb2tVA52Fi9ejBUrVuDgwYPQ0dHBL7/8gvv372P48OFwdHwzliVQRwWdk7u8wUPOiYiIKoPKwc6DBw/Qr18/AICOjg7S0tIgkUgwffp0+ZBwer2ikzIQ9u+Q885uFlVdHSIiompF5WDH1NQUKSkpAIBatWrh9u3bAICkpCSkp3PBxqrg92+rTjOHmqhpoFPFtSEiIqpeVO6g3LlzZ5w4cQKNGzfGsGHDMHXqVJw+fRonTpxAjx49KqOOVIqC/jochUVERFSYysHOqlWrkJmZCQD48ssvoa2tjUuXLmHo0KH46quvKryCVLLsXBkuhhcMOWd/HSIiolepFOzk5ubi0KFD6NWrFwBAQ0MDc+fOrZSKkXKCHr9AWnYeLGrooJEdR8MRERG9SqU+O1paWpg4caK8ZYeqXkF/nc5uHHJORERUFJU7KLdp0wbXr1+vhKpQWXDIORERUclU7rMzadIkzJgxA1FRUWjZsiUMDQ0Vtjdp0qTCKkcli0nKQEh8yr9DzhnsEBERFUXlYOf9998HAEyZMkWeJpFIIISARCJBXl5exdWOSuQXmt+q09ShJkwNOeSciIioKCoHOxEREZVRDyqDl5eIICIioqKpHOw4OTlVRj1IRTl5MlwMfw6A8+sQERGVROVgZ8uWLSVu//jjj8tcGVLe1ceJSM3KhZmhDprU4pBzIiKi4qgc7EydOlXh75ycHKSnp0NHRwcGBgYMdl6Ts/Ih5xYcck5ERFQClYeeJyYmKrxSU1MREhKCjh07YseOHZVRRyoCl4ggIiJSjsrBTlHc3NywZMmSQq0+VDnipJm4H5cCiQTozM7JREREJaqQYAfIn105Jiamog5HJfALzW/VaWJfE2Ycck5ERFQilfvsHDhwQOFvIQRiY2OxatUqdOjQocIqRsUr6K/Tla06REREpVI52Bk0aJDC3xKJBJaWlujevTt++umniqoXFSMnT4YLYVzlnIiISFkqBzsymawy6kFKuvY4ESlZuTA10EYT+5pVXR0iIqJqr8L67NDrcfbfJSI6uVlCk0POiYiISqVysDN06FAsXbq0UPqyZcswbNiwCqkUFc+voL8OH2EREREpReVg59y5c+jbt2+h9D59+uDcuXMVUikqWkJyJu7GJgPgkHMiIiJlqRzspKamQken8HBnbW1tJCcnV0ilqGgFj7Ca2JvAooZuFdeGiIjozaBysNO4cWPs2rWrUPrOnTvRoEGDCqkUFc2PQ86JiIhUpvJorPnz52PIkCF48OABunfvDgA4deoUduzYgd27d1d4BSlfbp4M58Pyg50uXCKCiIhIaSoHO/3798e+ffuwePFi/PXXX9DX10eTJk1w8uRJdOnSpTLqSACCo5KQnJmLmgbaaOZQs6qrQ0RE9MZQOdgBgH79+qFfv34VXRcqQcHCnxxyTkREpBqV++wEBgYiICCgUHpAQACCgoIqpFLqZsmSJZBIJJg2bZpCur+/P7p37w5DQ0MYGxujc+fOyMjIKPIYZ0OeQur/J4589wmMjIxgZWWFQYMGISQkRGG/devWoWvXrjA2NoZEIkFSUlIlnRUREdGbQeVgx8vLC1FRUYXSo6Oj4eXlVSGVUieBgYFYu3YtmjRpopDu7++P3r17w9PTE1euXEFgYCAmT54MDY3CH0lCSibuxCQjM+o2pn4+GZcvX8aJEyeQk5MDT09PpKWlyfdNT09H7969MW/evEo/NyIiojeByo+x7t69ixYtWhRKb968Oe7evVshlVIXqampGDVqFNavX4/vvvtOYdv06dMxZcoUzJ07V55Wr169Io9TMAqrx/Sf8fnETvL0TZs2wcrKClevXkXnzp0BQN56dPbs2Qo8EyIiojeXyi07urq6iI+PL5QeGxsLLa0ydQFSW15eXujXrx969uypkJ6QkICAgABYWVmhffv2sLa2RpcuXXDhwoUij1Mwv07XuoqjsKRSKQDAzMysEmpPRESkHlQOdjw9PeHt7S3/oQWApKQkzJs3D++8806FVu5NtnPnTly7dg0+Pj6Ftj18+BAA8M0332DcuHE4evQoWrRogR49eiAsLExh39xiVjmXyWSYNm0aOnTogEaNGlXimRAREb3ZVG6K+fHHH9G5c2c4OTmhefPmAIDr16/D2toaf/zxR4VX8E0UFRWFqVOn4sSJE9DT0yu0vWDl+AkTJmDMmDEA8h8Dnjp1Chs3blQIkG48SYI0IwfGeloKQ869vLxw+/btYluDiIiIKJ/KwU6tWrVw8+ZNbNu2DTdu3IC+vj7GjBmDDz74ANra2pVRxzfO1atXkZCQoNC3KS8vD+fOncOqVavkI6henXG6fv36iIyMVEg7+29/nU51LaGlmd8QN3nyZBw6dAjnzp2Dvb19ZZ4KERHRG69MnWwMDQ0xfvz4iq6L2ujRowdu3bqlkDZmzBi4u7tjzpw5qF27Nuzs7AoNGw8NDUWfPn0U0s6+tESEEAKff/459u7di7Nnz8LFxaVyT4SIiEgNlLlH8d27dxEZGYns7GyF9AEDBpS7Um86IyOjQv1oDA0NYW5uLk+fNWsWFixYgKZNm6JZs2bYvHkz7t+/j7/++kuep3PXbritXQ/GLfujSz1LeHl5Yfv27di/fz+MjIwQFxcHADAxMYG+vj4AIC4uDnFxcQgPDwcA3Lp1C0ZGRnB0dGRHZiIieiupHOw8fPgQgwcPxq1btyCRSCCEAABIJPmz+ubl5VVsDdXUtGnTkJmZienTp+PFixdo2rQpTpw4gTp16sj3uR8aDlltWzS0M4aVkR5Wr14NAOjatavCsXx9ffHJJ58AANasWYNvv/1Wvq1gSPrL+xAREb1NJKIgWlFS//79oampif/7v/+Di4sLrly5gufPn2PmzJn48ccf0alTp9IPUs0kJyfDxMQEUqkUxsbGVV0duc82B+HkvXiM9nDCtwM54oqIiOhlyv5+qzz03N/fHwsXLoSFhQU0NDSgoaGBjh07wsfHB1OmTClXpek/O65E4uS9/PmMtlx+jF2BkaXkICIioqKoHOzk5eXByMgIAGBhYYGYmBgAgJOTU6EOt1Q2sdIMzNv7XwdnIYB5e24jVlr0ullERERUPJX77DRq1Ag3btyAi4sL2rZti2XLlkFHRwfr1q1D7dq1K6OOb52IZ2l49eFinhB49Cwdtib6VVMpIiKiN5TKwc5XX30lX3hy4cKFePfdd9GpUyeYm5tj165dFV7Bt5Hmv529X01ztjCogtoQERG92VQOdnr16iX/t6urK+7fv48XL17A1NRUPiKLymff9RiFvzUlEiwe0oitOkRERGVQISt3cv6WipOQkom/rz0BAKwe1QI1DXTgbGHAQIeIiKiMuEx5NbPp4iNk58rQwrEmejeyYWsZERFROak8GosqT2pWLv64/BgAMKFLHQY6REREFYDBTjWyIyASKZm5qG1piHfqW1d1dYiIiNSCysHOuXPnkJubWyg9NzcX586dq5BKvY2yc2XYcCECADChc21oaLBVh4iIqCKoHOx069YNL168KJQulUrRrVu3CqnU22j/9WjEJWfCykgXg5rXqurqEBERqQ2Vgx0hRJF9SZ4/fw5DQ8MKqdTbRiYTWHfuIQDg044u0NXSrOIaERERqQ+lR2MNGTIEQP7q5p988gl0dXXl2/Ly8nDz5k20b9++4mv4Fjh9PwFhCakw0tXCyLaOVV0dIiIitaJ0sGNiYgIgv2XHyMgI+vr/zfuio6ODdu3aYdy4cRVfw7fA2nMPAAAj2znCWE+7imtDRESkXpQOdnx9fQEAzs7O+OKLL/jIqoJcffwCgY8SoaOpgU87uFR1dYiIiNSOyn12Zs+erdBn5/Hjx/j5559x/PjxCq3Y22KNX35fncHNa8HaWK+Ka0NERKR+VA52Bg4ciC1btgAAkpKS0KZNG/z0008YOHAgVq9eXeEVVGfhCak4cTceEgkwrjNXjCciIqoMKgc7165dQ6dOnQAAf/31F2xsbPD48WNs2bIFK1eurPAKqrN1//bVeae+NVytalRxbYiIiNSTysFOeno6jIyMAADHjx/HkCFDoKGhgXbt2uHx48cVXkF1FZ+cib3B0QDyl4YgIiKiyqFysOPq6op9+/YhKioKx44dg6enJwAgISEBxsbGFV5BdbXxQgRy8gTaOJuhpZNpVVeHiIhIbakc7Hz99df44osv4OzsjDZt2sDDwwNAfitP8+bNK7yC6ig5MwfbAiIBABO6sK8OERFRZVI52HnvvfcQGRmJoKAgHDt2TJ7eo0cPrFixokIr96olS5ZAIpFg2rRp8rTMzEx4eXnB3NwcNWrUwNChQxEfH1+p9SivbZcjkZqVCzerGuhWz6qqq0NERKTWyrTquY2NDYyMjHDixAlkZGQAAFq3bg13d/cKrdzLAgMDsXbtWjRp0kQhffr06Th48CB2794NPz8/xMTEyGd7ro6ycvOw8eK/C352qcMFP4mIiCqZysHO8+fP0aNHD9StWxd9+/ZFbGwsAGDs2LGYOXNmhVcQAFJTUzFq1CisX78epqb/9W+RSqXYsGEDli9fju7du6Nly5bw9fXFpUuXcPny5UqpS3ntC47G05Qs2JroYUBTu6quDhERkdpTOdiZPn06tLW1ERkZCQMDA3n6iBEjcPTo0QqtXAEvLy/069cPPXv2VEi/evUqcnJyFNLd3d3h6OgIf3//Yo+XlZWF5ORkhdfrIJMJrP13wc+xHV2go1WmhjUiIiJSgdLLRRQ4fvw4jh07Bnt7e4V0Nze3Shl6vnPnTly7dg2BgYGFtsXFxUFHRwc1a9ZUSLe2tkZcXFyxx/Tx8cG3335b0VUt1Yl78Xj4NA3Gelp4vw0X/CQiInodVG5aSEtLU2jRKfDixQuFldArQlRUFKZOnYpt27ZBT6/illLw9vaGVCqVv6Kioirs2MURQmCNX/4kgh95OKGGrspxJhEREZWBysFOp06d5MtFAIBEIoFMJsOyZcvQrVu3Cq3c1atXkZCQgBYtWkBLSwtaWlrw8/PDypUroaWlBWtra2RnZyMpKUkhX3x8PGxsbIo9rq6uLoyNjRVelS3wUSKCI5Ogo6WBT9pzwU8iIqLXReXmhWXLlqFHjx4ICgpCdnY2Zs+ejTt37uDFixe4ePFihVauR48euHXrlkLamDFj4O7ujjlz5sDBwQHa2to4deoUhg4dCgAICQlBZGSkfP6f6qKgVee9lvawNKrYFjAiIiIqnsrBTqNGjRAaGopVq1bByMgIqampGDJkCLy8vGBra1uhlTMyMkKjRo0U0gwNDWFubi5PHzt2LGbMmAEzMzMYGxvj888/h4eHB9q1a1ehdSmPkLgUnL6fkL/gZydOIkhERPQ6qRzsREZGwsHBAV9++WWR2xwdX2/H2xUrVkBDQwNDhw5FVlYWevXqhd9///211qE0a/9d8LN3Qxu4WBhWcW2IiIjeLhIhhFAlg6amJmJjY2FlpTjz7/Pnz2FlZYW8vLwKreDrkJycDBMTE0il0grvvxOTlIHOy84gVyaw36sDmjrUrNDjExERva2U/f1WuYOyEAISSeFZf1NTUyt0xJS62HghArkygXa1zRjoEBERVQGlH2PNmDEDQP7oq/nz5ysMP8/Ly0NAQACaNWtW4RV8k4XGpWDr5fy5hyZ2qVPFtSEiIno7KR3sBAcHA8hv2bl16xZ0dHTk23R0dNC0aVN88cUXFV/DN9SuwEjM/fsWCp4Rxkkzq7Q+REREbyulg50zZ84AyB/6/csvv7yWuWneVLHSDHjv+S/QAYAv995Gl3qWsDXRr7J6ERERvY1U7rPj6+vLQKcUEc/SIHul23eeEHj0LL1qKkRERPQW40qUlcDFwhAar/Th1pRI4GxReJkNIiIiqlwMdiqBrYk+fIY0hua/o9Y0JRIsHtKIj7CIiIiqAFejrCQjWjuic11LPHqWDmcLAwY6REREVYTBTiWyNdFnkENERFTF+BiLiIiI1BqDHSIiIlJrDHaIiIhIrTHYISIiIrXGYIeIiIjUGoMdIiIiUmsMdoiIiEitMdghIiIitcZgh4iIiNQagx0iIiJSawx2iIiISK0x2CEiIiK1xmCHiIiI1BqDHSIiIlJrDHaIiIhIrTHYISIiIrXGYIeIiIjUGoMdIiIiUmsMdoiIiEitMdghIiIitcZgh4iIiNQagx0iIiJSawx2iIiISK0x2CEiIiK1xmCHiIiI1BqDHSIiIlJrDHaIiIhIrTHYISIiIrXGYIeIiIjUGoMdIiIiUmsMdoiIiEitMdghIiIitcZgh4iIiNQagx0iIiJSawx2iIiISK0x2CEiIiK1xmCHiIiI1BqDHSIiIlJrDHaIiIhIrTHYISIiIrXGYIeIiIjUGoMdIiIiUmsMdoiIiEitMdghIiIitcZgh4iIiNQagx0iIiJSawx2iIiISK0x2CEiIiK1xmCHiIiI1BqDHSIiIlJrDHaIiIhIrTHYISIiIrXGYIeIiIjUGoMdIiIiUmvVOtjx8fFB69atYWRkBCsrKwwaNAghISEK+2RmZsLLywvm5uaoUaMGhg4divj4+CqqMREREVU31TrY8fPzg5eXFy5fvowTJ04gJycHnp6eSEtLk+8zffp0HDx4ELt374afnx9iYmIwZMiQKqw1ERERVScSIYSo6koo6+nTp7CysoKfnx86d+4MqVQKS0tLbN++He+99x4A4P79+6hfvz78/f3Rrl07pY6bnJwMExMTSKVSGBsbV+YpEBERUQVR9ve7WrfsvEoqlQIAzMzMAABXr15FTk4OevbsKd/H3d0djo6O8Pf3L/Y4WVlZSE5OVngRERGRenpjgh2ZTIZp06ahQ4cOaNSoEQAgLi4OOjo6qFmzpsK+1tbWiIuLK/ZYPj4+MDExkb8cHBwqs+pERERUhd6YYMfLywu3b9/Gzp07y30sb29vSKVS+SsqKqoCakhERETVkVZVV0AZkydPxqFDh3Du3DnY29vL021sbJCdnY2kpCSF1p34+HjY2NgUezxdXV3o6upWZpWJiIiomqjWLTtCCEyePBl79+7F6dOn4eLiorC9ZcuW0NbWxqlTp+RpISEhiIyMhIeHx+uuLhEREVVD1bplx8vLC9u3b8f+/fthZGQk74djYmICfX19mJiYYOzYsZgxYwbMzMxgbGyMzz//HB4eHkqPxCIiIiL1Vq2HnkskkiLTfX198cknnwDIn1Rw5syZ2LFjB7KystCrVy/8/vvvJT7GehWHnhMREb15lP39rtbBzuvCYIeIiOjNo5bz7BARERGpisEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTW1CbY+e233+Ds7Aw9PT20bdsWV65cqeoqERERUTWgFsHOrl27MGPGDCxYsADXrl1D06ZN0atXLyQkJFR11YiIiKiKqUWws3z5cowbNw5jxoxBgwYNsGbNGhgYGGDjxo1VXTUiIiKqYlpVXYHyys7OxtWrV+Ht7S1P09DQQM+ePeHv719knqysLGRlZcn/lkqlAIDk5OTKrSwRERFVmILfbSFEifu98cHOs2fPkJeXB2tra4V0a2tr3L9/v8g8Pj4++PbbbwulOzg4VEodiYiIqPKkpKTAxMSk2O1vfLBTFt7e3pgxY4b8b5lMhhcvXsDc3BwSiaTCyklOToaDgwOioqJgbGz8WvOz7Ndfdnnzs+y3q+zy5mfZLPtNyV/esksihEBKSgrs7OxK3O+ND3YsLCygqamJ+Ph4hfT4+HjY2NgUmUdXVxe6uroKaTVr1qysKsLY2LhcH3B58rPs1192efOz7Ler7PLmZ9ks+03JX96yi1NSi06BN76Dso6ODlq2bIlTp07J02QyGU6dOgUPD48qrBkRERFVB298yw4AzJgxA6NHj0arVq3Qpk0b/Pzzz0hLS8OYMWOqumpERERUxdQi2BkxYgSePn2Kr7/+GnFxcWjWrBmOHj1aqNPy66arq4sFCxYUemT2OvKz7Ndfdnnzs+y3q+zy5mfZLPtNyV/esiuCRJQ2XouIiIjoDfbG99khIiIiKgmDHSIiIlJrDHaIiIhIrTHYISIiIrXGYKcS/fbbb3B2doaenh7atm2LK1euKJXv3Llz6N+/P+zs7CCRSLBv3z6ly/Tx8UHr1q1hZGQEKysrDBo0CCEhIUrnX716NZo0aSKf/MnDwwNHjhxROv/LlixZAolEgmnTpim1/zfffAOJRKLwcnd3V7q86OhofPjhhzA3N4e+vj4aN26MoKAgpfI6OzsXKlsikcDLy6vUvHl5eZg/fz5cXFygr6+POnXqYNGiRaWu1fKylJQUTJs2DU5OTtDX10f79u0RGBhYaL/Srg0hBL7++mvY2tpCX18fPXv2RFhYmNL59+zZA09PT/ls4tevX1e6/JycHMyZMweNGzeGoaEh7Ozs8PHHHyMmJkapsr/55hu4u7vD0NAQpqam6NmzJwICApSu+8smTpwIiUSCn3/+Wam8n3zySaHPvnfv3iqVfe/ePQwYMAAmJiYwNDRE69atERkZWWreoq47iUSCH374QamyU1NTMXnyZNjb20NfX1++GLIyeePj4/HJJ5/Azs4OBgYG6N27t/x6Uea7JDMzE15eXjA3N0eNGjUwdOhQ+QSvyuRft24dunbtCmNjY0gkEiQlJcm3lZb/xYsX+Pzzz1GvXj3o6+vD0dERU6ZMgVQqVarsCRMmoE6dOtDX14elpSUGDhwoX2JIle9RIQT69Okjf3+Vydu1a9dCn/fEiRNVKtvf3x/du3eHoaEhjI2N0blzZyxcuLDEvI8ePSr2etu9e7dSZcfFxeGjjz6CjY0NDA0N0aJFC/z9999K5X3w4AEGDx4MS0tLGBsbY/jw4YUmBK4sDHYqya5duzBjxgwsWLAA165dQ9OmTdGrVy8kJCSUmjctLQ1NmzbFb7/9pnK5fn5+8PLywuXLl3HixAnk5OTA09MTaWlpSuW3t7fHkiVLcPXqVQQFBaF79+4YOHAg7ty5o1I9AgMDsXbtWjRp0kSlfA0bNkRsbKz8deHCBaXyJSYmokOHDtDW1saRI0dw9+5d/PTTTzA1NVW6vi+Xe+LECQDAsGHDSs27dOlSrF69GqtWrcK9e/ewdOlSLFu2DL/++qtSZQPAZ599hhMnTuCPP/7ArVu34OnpiZ49eyI6Olphv9KujWXLlmHlypVYs2YNAgICYGhoiF69eiEzM1Op/GlpaejYsSOWLl1a7Pbi8qenp+PatWuYP38+rl27hj179iAkJAQDBgxQquy6deti1apVuHXrFi5cuABnZ2d4enri6dOnSuUvsHfvXly+fFlh+nhl8vbu3VvhGtixY4fS+R88eICOHTvC3d0dZ8+exc2bNzF//nzo6emVmvflMmNjY7Fx40ZIJBIMHTpUqbJnzJiBo0ePYuvWrbh37x6mTZuGyZMn48CBAyXmFUJg0KBBePjwIfbv34/g4GA4OTmhZ8+eSEtLU+q7ZPr06Th48CB2794NPz8/xMTEYMiQIQCU+y5KT09H7969MW/evEL1Ky1/TEwMYmJi8OOPP+L27dvYtGkTjh49irFjxypVdsuWLeHr64t79+7h2LFjEELA09MTeXl5Kn2P/vzzzwrLDCmbd9y4cQqf+7Jly5TO7+/vj969e8PT0xNXrlxBYGAgJk+ejAsXLpSY18HBodD19u2336JGjRro06ePUmV//PHHCAkJwYEDB3Dr1i0MGTIEw4cPx8GDB0vMm5aWBk9PT0gkEpw+fRoXL15EdnY2+vfvD5lMVuh9rXCCKkWbNm2El5eX/O+8vDxhZ2cnfHx8VDoOALF3794y1yMhIUEAEH5+fmU+hqmpqfi///s/pfdPSUkRbm5u4sSJE6JLly5i6tSpSuVbsGCBaNq0aZnqOGfOHNGxY8cy5S3K1KlTRZ06dYRMJit13379+olPP/1UIW3IkCFi1KhRSpWVnp4uNDU1xaFDhxTSW7RoIb788sti8716bchkMmFjYyN++OEHeVpSUpLQ1dUVO3bsKDX/yyIiIgQAERwcrHT5Rbly5YoAIB4/fqxyXqlUKgCIkydPKl32kydPRK1atcTt27eFk5OTWLFihVJ5R48eLQYOHFhifUrKP2LECPHhhx+WKe+rBg4cKLp37650/oYNG4qFCxcqpBV17byaNyQkRAAQt2/flqfl5eUJS0tLsX79+kJlv/pdkpSUJLS1tcXu3bvl+9y7d08AEP7+/qXmf9mZM2cEAJGYmFjkeZeWv8Cff/4pdHR0RE5Ojsp5b9y4IQCI8PBwpcsODg4WtWrVErGxscV+tkXlVeV7saj8bdu2FV999VWZ8r6qWbNmhb6/SspvaGgotmzZorCfmZlZoWvm1bzHjh0TGhoaQiqVyvdJSkoSEolEnDhxotRzKS+27FSC7OxsXL16FT179pSnaWhooGfPnvD393+tdZFKpQAAMzMzlfPm5eVh586dSEtLU2npDS8vL/Tr10/h/JUVFhYGOzs71K5dG6NGjUJkZKRS+Q4cOIBWrVph2LBhsLKyQvPmzbF+/XqVywfyP7+tW7fi008/VWph2Pbt2+PUqVMIDQ0FANy4cQMXLlxAnz59lCovNzcXeXl50NPTU0jX19dXumULACIiIhAXF6fwvpuYmKBt27av/borIJVKIZFIVF57Ljs7G+vWrYOJiQmaNm2qVB6ZTIaPPvoIs2bNQsOGDVWu69mzZ2FlZYV69erhf//7H54/f650uYcPH0bdunXRq1cvWFlZoW3btio9fi4QHx+Pw4cPY+zYsUrnad++PQ4cOIDo6GgIIXDmzBmEhobC09OzxHxZWVkAoHDdaWhoQFdXt8jr7tXvkqtXryInJ0fhenN3d4ejo2OR11t5vouUzS+VSmFsbAwtLa1C6SXlTUtLg6+vL1xcXODg4KBU2enp6Rg5ciR+++23YtdhLKnsbdu2wcLCAo0aNYK3tzfS09OVyp+QkICAgABYWVmhffv2sLa2RpcuXZT6zF519epVXL9+vdjrraj87du3x65du/DixQvIZDLs3LkTmZmZ6Nq1a4l5s7KyIJFIFCYW1NPTg4aGhkrfc2VW6eHUWyg6OloAEJcuXVJInzVrlmjTpo1Kx0I5Wnby8vJEv379RIcOHVTKd/PmTWFoaCg0NTWFiYmJOHz4sNJ5d+zYIRo1aiQyMjKEEKrdwfzzzz/izz//FDdu3BBHjx4VHh4ewtHRUSQnJ5eaV1dXV+jq6gpvb29x7do1sXbtWqGnpyc2bdqkdN0L7Nq1S2hqaoro6Gil9s/LyxNz5swREolEaGlpCYlEIhYvXqxSmR4eHqJLly4iOjpa5Obmij/++ENoaGiIunXrFpvn1Wvj4sWLAoCIiYlR2G/YsGFi+PDhpeZ/WUW07GRkZIgWLVqIkSNHKp334MGDwtDQUEgkEmFnZyeuXLmidNmLFy8W77zzjrw1TpWWnR07doj9+/eLmzdvir1794r69euL1q1bi9zc3FLzF9zVGxgYiOXLl4vg4GDh4+MjJBKJOHv2rFLnXWDp0qXC1NRU/v9HmbpnZmaKjz/+WAAQWlpaQkdHR2zevLnUvNnZ2cLR0VEMGzZMvHjxQmRlZYklS5YIAMLT01Mhb1HfJdu2bRM6OjqFymndurWYPXt2qflfVlrLjjLfZU+fPhWOjo5i3rx5Suf97bffhKGhoQAg6tWrV2SrTnH5x48fL8aOHSv/u6jPpri8a9euFUePHhU3b94UW7duFbVq1RKDBw9Wqmx/f38BQJiZmYmNGzeKa9euiWnTpgkdHR0RGhqq1HkX+N///ifq169f5Lbi8icmJgpPT0/59WZsbCyOHTtWat6EhARhbGwspk6dKtLS0kRqaqqYPHmyACDGjx9fbB0rCoOdSlBdgp2JEycKJycnERUVpVK+rKwsERYWJoKCgsTcuXOFhYWFuHPnTqn5IiMjhZWVlbhx44Y8TZVg51WJiYnC2NhYqUdo2trawsPDQyHt888/F+3atVO5XE9PT/Huu+8qvf+OHTuEvb292LFjh7h586bYsmWLMDMzUynQCg8PF507dxYAhKampmjdurUYNWqUcHd3LzZPdQ52srOzRf/+/UXz5s0Vmq1Ly5uamirCwsKEv7+/+PTTT4Wzs7OIj48vNX9QUJCwtrZWCFBVCXZe9eDBA6UfoRX8f//ggw8U9uvfv794//33VSq7Xr16YvLkycVuLyr/Dz/8IOrWrSsOHDggbty4IX799VdRo0aNQo8GisobFBQkmjZtKr/uevXqJfr06SN69+6tsF9R3yWqBDulfReVFuyUll8qlYo2bdqI3r17i+zsbKXzJiUlidDQUOHn5yf69+8vWrRoUSjQLCr//v37haurq0hJSZGnFfX+KvsdfOrUqSIfoRWVv+D/ube3t8K+jRs3FnPnzlW67PT0dGFiYiJ+/PHHIrcXl3/y5MmiTZs24uTJk+L69evim2++ESYmJuLmzZul5j127JioXbu2kEgkQlNTU3z44YeiRYsWYuLEiSW8OxWDwU4lyMrKEpqamoUu/I8//lgMGDBApWOVNdjx8vIS9vb24uHDhyrnfVWPHj2Uirz37t0r/9IseAGQX9hF3SWXplWrVgr/gYvj6OiocJclhBC///67sLOzU6m8R48eCQ0NDbFv3z6l89jb24tVq1YppC1atEjUq1dPpbKFyP+xLwhWhg8fLvr27Vvsvq9eGwU/0K8GKJ07dxZTpkwpNf/LyhPsZGdni0GDBokmTZqIZ8+eqZT3Va6urkW2kr2af8WKFfLr7OVrT0NDQzg5OZWpbAsLC7FmzZpSy87KyhJaWlpi0aJFCvvNnj1btG/fXumyz507JwCI69evF1unV/Onp6cLbW3tQv29xo4dK3r16qV02UlJSSIhIUEIkd/fcNKkSfJtxX2XFPxAvxqgODo6iuXLl5ea/2UlBTul5U9OThYeHh6iR48ehQIVVb4Hs7KyhIGBgdi+fXup+adOnVrs9dalSxeVy05NTRUAxNGjR0st++HDhwKA+OOPPxTShw8fLm9FVabsLVu2CG1tbfnn/rLi8oeHhxfq5yVE/m/EhAkTlC776dOn8s/a2tpaLFu2rNh9Kwr77FQCHR0dtGzZEqdOnZKnyWQynDp1SqW+L2UhhMDkyZOxd+9enD59Gi4uLuU+pkwmkz/fL0mPHj1w69YtXL9+Xf5q1aoVRo0ahevXr0NTU1OlclNTU/HgwQPY2tqWum+HDh0KDXMMDQ2Fk5OTSmX6+vrCysoK/fr1UzpPeno6NDQU/ytpamqWaYSBoaEhbG1tkZiYiGPHjmHgwIFK53VxcYGNjY3CdZecnIyAgIBKv+4K5OTkYPjw4QgLC8PJkydhbm5eruMpe+199NFHuHnzpsK1Z2dnh1mzZuHYsWMql/vkyRM8f/5cqWtPR0cHrVu3Lvf1t2HDBrRs2VLpPkpA/vudk5NT7uvPxMQElpaWCAsLQ1BQEAYOHFjqd0nLli2hra2tcL2FhIQgMjISHh4e5f4uUiZ/cnIyPD09oaOjgwMHDsj7H5WlbJF/84+srKxS88+dO7fQ9QYAK1aswMaNG1UuuyC/ra1tqWU7OzvDzs6uyOvN0dFR6bI3bNiAAQMGwNLSUuE9KCl/Qb+ioq63vLw8pcu2sLBAzZo1cfr0aSQkJMhHbFaqSg+n3lI7d+4Uurq6YtOmTeLu3bti/PjxombNmiIuLq7UvCkpKSI4OFgEBwcLAPJ+AK+OaCnK//73P2FiYiLOnj0rYmNj5a/09HSl6j137lzh5+cnIiIixM2bN8XcuXOFRCIRx48fVyr/q1R5jDVz5kxx9uxZERERIS5evCh69uwpLCwsirzzeNWVK1eElpaW+P7770VYWJjYtm2bMDAwEFu3blW6rnl5ecLR0VHMmTNH6TxC5I/kqVWrljh06JCIiIgQe/bsERYWFoWa8kty9OhRceTIEfHw4UNx/Phx0bRpU9G2bdtCTfKlXRtLliwRNWvWlPc/GThwoHBxcZHf8ZaW//nz5yI4OFgcPnxYABA7d+4UwcHBIjY2ttT82dnZYsCAAcLe3l5cv35d4frLysoqMW9qaqrw9vYW/v7+4tGjRyIoKEiMGTNG6Orqyu8iVf1/8fJjrJLypqSkiC+++EL4+/uLiIgIcfLkSdGiRQvh5uYmMjMzlSp7z549QltbW6xbt06EhYWJX3/9VWhqaorz588rVW+pVCoMDAzE6tWrC51Hafm7dOkiGjZsKM6cOSMePnwofH19hZ6envj9999Lzfvnn3+KM2fOiAcPHoh9+/YJJycnMWTIECGEct8lEydOFI6OjuL06dMiKChIeHh4yB8nK5M/NjZWBAcHi/Xr1wsA4ty5cyI4OFg8f/681PxSqVS0bdtWNG7cWISHhyvsM3HixBLzPnjwQCxevFgEBQWJx48fi4sXL4r+/fsLMzMzER8fX6bvUfzbclZa3vDwcLFw4UIRFBQkIiIixP79+0Xt2rVF586dlX7fVqxYIYyNjcXu3btFWFiY+Oqrr4Senp4YOXKkUvUOCwsTEolEHDlyRCG9tLKzs7OFq6ur6NSpkwgICBDh4eHixx9/FBKJRPTt27fUsjdu3Cj8/f1FeHi4+OOPP4SZmZmYMWNGse9pRWKwU4l+/fVX4ejoKHR0dESbNm3E5cuXlcpX0KT76mv06NGl5i0qHwDh6+urVNmffvqpcHJyEjo6OsLS0lL06NGjzIGOEKoFOyNGjBC2trZCR0dH1KpVS4wYMaLIDoPFOXjwoGjUqJHQ1dUV7u7uYt26dSrV9dixYwKACAkJUSlfcnKymDp1qnB0dBR6enqidu3a4ssvvxRZWVlKH2PXrl2idu3aQkdHR9jY2AgvLy+RlJRUaL/Srg2ZTCbmz58vrK2tha6urujRo4fC+ZSW39fXt8jtCxYsKDV/waOvol5nzpwpMW9GRoYYPHiwsLOzEzo6OsLW1lYMGDBAoYOyqv8vXg52Ssqbnp4uPD09haWlpdDW1hZOTk5i3LhxCjcmypS9YcMG4erqKvT09ETTpk3lj0KVybt27Vqhr69fps88NjZWfPLJJ8LOzk7o6emJevXqiZ9++knIZLJS8/7yyy/C3t5eaGtrC0dHR/HVV1/Jr1tlvksyMjLEpEmThKmpqTAwMBCDBw+WB8bK5F+wYEGx+5SWv7hzK+lVkDc6Olr06dNHWFlZCW1tbWFvby9Gjhwp7t+/r3TdX1UQ7JSWNzIyUnTu3FmYmZkJXV1d4erqKmbNmiXv26Zs2T4+PsLe3l4YGBgIDw8Pcf78eaXzent7CwcHB5GXl1foHErLHxoaKoYMGSKsrKyEgYGBaNKkidiyZYtSeefMmSOsra2Ftra2cHNzk1+nr4Pk3xMkIiIiUkvss0NERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BARveLs2bOQSCRISkqq6qoQUQVgsENERERqjcEOERERqTUGO0RU7chkMvj4+MDFxQX6+vpo2rQp/vrrLwD/PWI6fPgwmjRpAj09PbRr1w63b99WOMbff/+Nhg0bQldXF87Ozvjpp58UtmdlZWHOnDlwcHCArq4uXF1dsWHDBoV9rl69ilatWsHAwADt27cvtNI0Eb0ZGOwQUbXj4+ODLVu2YM2aNbhz5w6mT5+ODz/8EH5+fvJ9Zs2ahZ9++gmBgYGwtLRE//79kZOTAyA/SBk+fDjef/993Lp1C9988w3mz5+PTZs2yfN//PHH2LFjB1auXIl79+5h7dq1qFGjhkI9vvzyS/z0008ICgqClpYWPv3009dy/kRUsbgQKBFVK1lZWTAzM8PJkyfh4eEhT//ss8+Qnp6O8ePHo1u3bti5cydGjBgBAHjx4gXs7e2xadMmDB8+HKNGjcLTp09x/Phxef7Zs2fj8OHDuHPnDkJDQ1GvXj2cOHECPXv2LFSHs2fPolu3bjh58iR69OgBAPjnn3/Qr18/ZGRkQE9Pr5LfBSKqSGzZIaJqJTw8HOnp6XjnnXdQo0YN+WvLli148OCBfL+XAyEzMzPUq1cP9+7dAwDcu3cPHTp0UDhuhw4dEBYWhry8PFy/fh2ampro0qVLiXVp0qSJ/N+2trYAgISEhHKfIxG9XlpVXQEiopelpqYCAA4fPoxatWopbNPV1VUIeMpKX19fqf20tbXl/5ZIJADy+xMR0ZuFLTtEVK00aNAAurq6iIyMhKurq8LLwcFBvt/ly5fl/05MTERoaCjq168PAKhfvz4uXryocNyLFy+ibt260NTUROPGjSGTyRT6ABGR+mLLDhFVK0ZGRvjiiy8wffp0yGQydOzYEVKpFBcvXoSxsTGcnJwAAAsXLoS5uTmsra3x5ZdfwsLCAoMGDQIAzJw5E61bt8aiRYswYsQI+Pv7Y9WqVfj9998BAM7Ozhg9ejQ+/fRTrFy5Ek2bNsXjx4+RkJCA4cOHV9WpE1ElYbBDRNXOokWLYGlpCR8fHzx8+BA1a9ZEixYtMG/ePPljpCVLlmDq1KkICwtDs2bNcPDgQejo6AAAWrRogT///BNff/01Fi1aBFtbWyxcuBCffPKJvIzVq1dj3rx5mDRpEp4/fw5HR0fMmzevKk6XiCoZR2MR0RulYKRUYmIiatasWdXVIaI3APvsEBERkVpjsENERERqjY+xiIiISK2xZYeIiIjUGoMdIiIiUmsMdoiIiEitMdghIiIitcZgh4iIiNQagx0iIiJSawx2iIiISK0x2CEiIiK1xmCHiIiI1Nr/A45Ofu2hDDlOAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plt.plot(np.arange(len(epochs_x)), epochs_acc, marker = '.')\n", + "plt.xlabel('epoch')\n", + "plt.ylabel('test accuracy')\n", + "plt.ylim(0, 100)\n", + "plt.xticks(np.arange(len(epochs_x)))\n", + "for i, txt in enumerate(epochs_acc):\n", + " if i%2 == 0:\n", + " pass\n", + " else:\n", + " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", + "plt.show()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "speck-rescnn", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tests/test_nonsequential/exp_set_B/non-sequential-SCNN-example_3.ipynb b/tests/test_nonsequential/exp_set_B/non-sequential-SCNN-example_3.ipynb new file mode 100644 index 00000000..fca63b7d --- /dev/null +++ b/tests/test_nonsequential/exp_set_B/non-sequential-SCNN-example_3.ipynb @@ -0,0 +1,1509 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import torch, random\n", + "import torch.nn as nn\n", + "import sinabs.layers as sl\n", + "from tqdm.notebook import tqdm\n", + "\n", + "from tonic.datasets.dvsgesture import DVSGesture\n", + "from tonic.transforms import ToFrame\n", + "from torch.utils.data import DataLoader\n", + "from torch.nn import CrossEntropyLoss\n", + "from torch.optim import Adam\n", + "\n", + "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "torch.backends.cudnn.deterministic = True\n", + "random.seed(1)\n", + "torch.manual_seed(1)\n", + "torch.cuda.manual_seed(1)\n", + "np.random.seed(1)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Loading Data" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "batch_size = 3\n", + "num_workers = 1\n", + "epochs = 30\n", + "lr = 5e-4" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "root_dir = \"./DVSGESTURE\"\n", + "_ = DVSGesture(save_to=root_dir, train=True)\n", + "_ = DVSGesture(save_to=root_dir, train=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "n_time_steps = 50\n", + "to_raster = ToFrame(sensor_size=DVSGesture.sensor_size, n_time_bins=n_time_steps)\n", + "\n", + "snn_train_dataset = DVSGesture(save_to=root_dir, train=True, transform=to_raster)\n", + "snn_test_dataset = DVSGesture(save_to=root_dir, train=False, transform=to_raster)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The transformed array is in shape [Time-Step, Channel, Height, Width] --> (50, 2, 128, 128)\n" + ] + } + ], + "source": [ + "sample_data, label = snn_train_dataset[0]\n", + "print(f\"The transformed array is in shape [Time-Step, Channel, Height, Width] --> {sample_data.shape}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "snn_train_dataloader = DataLoader(snn_train_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True)\n", + "snn_test_dataloader = DataLoader(snn_test_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=False)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Network Module\n", + "\n", + "We need to define a `nn.Module` implementing the network we want the chip to reproduce." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "device: NVIDIA RTX A4000\n" + ] + } + ], + "source": [ + "if torch.cuda.is_available():\n", + " device = torch.device('cuda:0')\n", + " print('device: ', torch.cuda.get_device_name(0))\n", + "else:\n", + " device = torch.device('cpu')" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "class SNN(nn.Module):\n", + " def __init__(self) -> None:\n", + " super().__init__()\n", + "\n", + " self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False)\n", + " self.iaf1 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + " self.pool1 = nn.AvgPool2d(2,2)\n", + " self.pool1a = nn.AvgPool2d(6,6)\n", + "\n", + " self.conv2 = nn.Conv2d(10, 10, 2, 1, bias=False)\n", + " self.iaf2 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + " self.pool2 = nn.AvgPool2d(3,3)\n", + "\n", + " self.conv3 = nn.Conv2d(10, 10, 3, 1, bias=False)\n", + " self.iaf3 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + " self.pool3 = nn.AvgPool2d(2,2)\n", + "\n", + " self.flat = nn.Flatten()\n", + "\n", + " self.fc1 = nn.Linear(810, 100, bias=False)\n", + " self.iaf4 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + "\n", + " self.fc2 = nn.Linear(100, 100, bias=False)\n", + " self.iaf5 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + "\n", + " self.fc3 = nn.Linear(100, 100, bias=False)\n", + " self.iaf6 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + "\n", + " self.fc4 = nn.Linear(100, 11, bias=False)\n", + " self.iaf7 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + "\n", + " self.merge_fc = sl.Merge()\n", + " self.merge_conv = sl.Merge()\n", + "\n", + " def detach_neuron_states(self):\n", + " for name, layer in self.named_modules():\n", + " if name != '':\n", + " if isinstance(layer, sl.StatefulLayer):\n", + " for name, buffer in layer.named_buffers():\n", + " buffer.detach_()\n", + "\n", + " def init_weights(self):\n", + " for name, layer in self.named_modules():\n", + " if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear):\n", + " nn.init.xavier_normal_(layer.weight.data)\n", + "\n", + " def forward(self, x):\n", + " \n", + " con1_out = self.conv1(x)\n", + " iaf1_out = self.iaf1(con1_out)\n", + " pool1_out = self.pool1(iaf1_out)\n", + " pool1a_out = self.pool1a(iaf1_out)\n", + "\n", + " conv2_out = self.conv2(pool1_out)\n", + " iaf2_out = self.iaf2(conv2_out)\n", + " pool2_out = self.pool2(iaf2_out)\n", + "\n", + " merged_conv_out = self.merge_conv(pool1a_out, pool2_out)\n", + "\n", + " conv3_out = self.conv3(merged_conv_out)\n", + " iaf3_out = self.iaf3(conv3_out)\n", + " pool3_out = self.pool3(iaf3_out)\n", + "\n", + " flat_out = self.flat(pool3_out)\n", + " \n", + " fc1_out = self.fc1(flat_out)\n", + " iaf4_out = self.iaf4(fc1_out)\n", + "\n", + " fc2_out = self.fc2(iaf4_out)\n", + " iaf5_out = self.iaf5(fc2_out)\n", + "\n", + " fc3_out = self.fc3(iaf5_out)\n", + " iaf6_out = self.iaf6(fc3_out)\n", + "\n", + " merge_fc_out = self.merge_fc(iaf4_out, iaf6_out)\n", + "\n", + " fc4_out = self.fc4(merge_fc_out)\n", + " iaf7_out = self.iaf7(fc4_out)\n", + "\n", + " return iaf7_out" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "snn = SNN().to(device)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "snn.init_weights()" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "optimizer = Adam(snn.parameters(), lr=lr, betas=(0.9, 0.999), eps=1e-8)\n", + "loss_fn = CrossEntropyLoss()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Define train and test" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "def train(dataloader, model, loss_fn, optimizer, epochs, test_func, dataloader_test):\n", + " epochs_y = []\n", + " epochs_x = []\n", + " epochs_acc = []\n", + " model.train()\n", + "\n", + " for e in range(epochs):\n", + " losses = []\n", + " batches = []\n", + " batch_count = 0\n", + " train_p_bar = tqdm(snn_train_dataloader)\n", + "\n", + " for X, y in train_p_bar:\n", + " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", + " X = X.reshape(-1, 2, 128, 128).to(dtype=torch.float, device=device)\n", + " y = y.to(dtype=torch.long, device=device)\n", + "\n", + " # forward\n", + " pred = model(X)\n", + "\n", + " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", + " pred = pred.reshape(batch_size, n_time_steps, -1)\n", + "\n", + " # accumulate all time-steps output for final prediction\n", + " pred = pred.sum(dim = 1)\n", + " loss = loss_fn(pred, y)\n", + "\n", + " # gradient update\n", + " optimizer.zero_grad()\n", + " loss.backward()\n", + " optimizer.step()\n", + "\n", + " # detach the neuron states and activations from current computation graph(necessary)\n", + " model.detach_neuron_states()\n", + "\n", + " train_p_bar.set_description(f\"Epoch {e} - BPTT Training Loss: {round(loss.item(), 4)}\")\n", + "\n", + " batch_count += 1\n", + " losses.append(loss.item())\n", + " batches.append(batch_count)\n", + "\n", + " epochs_y.append(losses)\n", + " epochs_x.append(batches)\n", + "\n", + " acc = test_func(dataloader_test, model)\n", + " print(f'Epoch {e} accuracy: {acc}')\n", + " epochs_acc.append(acc)\n", + "\n", + " return epochs_x, epochs_y, epochs_acc\n" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "def test(dataloader, model):\n", + " correct_predictions = []\n", + " with torch.no_grad():\n", + " test_p_bar = tqdm(dataloader)\n", + " for X, y in test_p_bar:\n", + " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", + " X = X.reshape(-1, 2, 128, 128).to(dtype=torch.float, device=device)\n", + " y = y.to(dtype=torch.long, device=device)\n", + "\n", + " # forward\n", + " output = model(X)\n", + "\n", + " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", + " output = output.reshape(batch_size, n_time_steps, -1)\n", + "\n", + " # accumulate all time-steps output for final prediction\n", + " output = output.sum(dim=1)\n", + "\n", + " # calculate accuracy\n", + " pred = output.argmax(dim=1, keepdim=True)\n", + "\n", + " # compute the total correct predictions\n", + " correct_predictions.append(pred.eq(y.view_as(pred)))\n", + "\n", + " test_p_bar.set_description(f\"Testing Model...\")\n", + " \n", + " correct_predictions = torch.cat(correct_predictions)\n", + " return correct_predictions.sum().item()/(len(correct_predictions))*100" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Training loop (HPO)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "12d134e3b89e41888c9c47892b8e6491", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/359 [00:00" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "y_avg = []\n", + "for y in epochs_y:\n", + " y_avg.append(np.mean(y))\n", + "\n", + "plt.plot(np.arange(len(epochs_x)), y_avg, marker = '.')\n", + "plt.xlabel('epoch')\n", + "plt.ylabel('average loss')\n", + "plt.ylim(0,)\n", + "plt.xticks(np.arange(len(epochs_x)))\n", + "for i, txt in enumerate(y_avg):\n", + " if i%5 == 0:\n", + " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", + " else:\n", + " pass\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAG2CAYAAACZEEfAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABq60lEQVR4nO3dd1hT1+MG8DfsHUW2TAfixK3gqqO46q6jtVWr1bZi62idraPVFrWt7dfROn5urauOOloXrqqooOBEBERBZYhI2DPn9weaGlmJBIH4fp4nT8vNPfecQMx9c+6550iEEAJEREREWkqnohtAREREVJ4YdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirVWjYOXPmDPr06QMHBwdIJBLs27dP6XkhBObMmQN7e3sYGxujW7duCA8PV9onKSkJw4cPh4WFBapVq4YxY8YgLS3tNb4KIiIiqswqNOykp6fD09MTK1asKPL5xYsXY+nSpVi5ciUuXrwIU1NTdO/eHVlZWYp9hg8fjps3b+LYsWM4ePAgzpw5g3Hjxr2ul0BERESVnKSyLAQqkUiwd+9e9O/fH0BBr46DgwO+/PJLfPXVVwAAmUwGW1tbbNiwAcOGDUNoaCgaNGiAwMBAtGzZEgBw+PBh9OrVCw8ePICDg0NFvRwiIiKqJPQqugHFiYqKQlxcHLp166bYJpVK0aZNGwQEBGDYsGEICAhAtWrVFEEHALp16wYdHR1cvHgRAwYMKPLY2dnZyM7OVvwsl8uRlJSEGjVqQCKRlN+LIiIiIo0RQiA1NRUODg7Q0Sn+YlWlDTtxcXEAAFtbW6Xttra2iufi4uJgY2Oj9Lyenh4sLS0V+xTFz88P3377rYZbTERERBUhJiYGjo6OxT5facNOeZo5cyamTJmi+Fkmk8HZ2RkxMTGwsLCowJYRERGRqlJSUuDk5ARzc/MS96u0YcfOzg4AEB8fD3t7e8X2+Ph4NG3aVLFPQkKCUrm8vDwkJSUpyhfF0NAQhoaGhbZbWFgw7BAREVUxpQ1BqbTz7Li5ucHOzg7+/v6KbSkpKbh48SK8vLwAAF5eXkhOTsbly5cV+5w4cQJyuRxt2rR57W0mIiKiyqdCe3bS0tIQERGh+DkqKgohISGwtLSEs7MzJk2ahAULFqBu3bpwc3PD7Nmz4eDgoLhjq379+ujRowfGjh2LlStXIjc3FxMmTMCwYcN4JxYREREBqOCwExQUhM6dOyt+fj6OZuTIkdiwYQOmTZuG9PR0jBs3DsnJyWjfvj0OHz4MIyMjRZmtW7diwoQJ6Nq1K3R0dDBo0CAsXbr0tb8WIiIiqpwqzTw7FSklJQVSqRQymYxjdoiIiKoIVc/flXbMDhEREZEmMOwQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEqQn5+P2bNnw83NDcbGxqhduzbmz58PIYRin1GjRkEikSg9evToUeJxU1NTMWnSJLi4uMDY2Bje3t4IDAxU2ictLQ0TJkyAo6MjjI2N0aBBA6xcubJcXqc206voBhAREVVmixYtwu+//46NGzeiYcOGCAoKwkcffQSpVIovvvhCsV+PHj2wfv16xc+GhoYlHvfjjz/GjRs3sHnzZjg4OGDLli3o1q0bbt26hZo1awIApkyZghMnTmDLli1wdXXF0aNHMX78eDg4OKBv377l84K1EHt2iIiISnD+/Hn069cPvXv3hqurK9599134+Pjg0qVLSvsZGhrCzs5O8ahevXqxx8zMzMTu3buxePFidOzYEXXq1MG8efNQp04d/P7770p1jxw5Em+99RZcXV0xbtw4eHp6FqqbSsawQ0T0hiivyzF+fn5o1aoVzM3NYWNjg/79+yMsLExpn08++QS1a9eGsbExalhZo0O3njgTGFIeL1PjvL294e/vjzt37gAArl69irNnz6Jnz55K+506dQo2NjaoV68ePvvsMzx58qTYY+bl5SE/Px9GRkZK242NjXH27Fmluvfv34+HDx9CCIGTJ0/izp078PHx0eArfAMIEjKZTAAQMpmsoptCRFRuvv/+e1GjRg1x8OBBERUVJXbt2iXMzMzE//73P8U+I0eOFD169BCxsbGKR1JSUonH7d69u1i/fr24ceOGCAkJEb169RLOzs4iLS1Nsc+qVavE6dOnxdJ9Z4XDqF+FcZ3WQtfCWvwRcLfcXq+m5Ofni+nTpwuJRCJ09fSERCIRP/zwg9I+27ZtE3/99Ze4du2a2Lt3r6hfv75o1aqVyMvLK/a4Xl5eolOnTuLhw4ciLy9PbN68Wejo6Ah3d3fFPllZWWLEiBECgNDT0xMGBgZi48aN5fZaqxpVz98MO4Jhh4jeDL179xajR49W2jZw4EAxfPhwxc8jR44U/fr1K1M9CQkJAoA4ffq00vbIhFThOv2gcJl+UNh/tEwAEE6f/J94lJxRpvrK27Zt24Sljb2w7jtV2I9eLqzemSLMLKqJDRs2FFsmMjJSABDHjx8vdp+IiAjRsWNHAUDo6uqKVq1aieHDhwsPDw/FPj/++KNwd3cX+/fvF1evXhXLli0TZmZm4tixYxp9jVWVqudvXsYiInpDlMflmKLIZDIAgKWlpWLbydsJGLIyAAKAPCcLadePQ09qC4l5DdxLzCjbCytnU776ChLPfjCp3wkG1q4wbdgFep7vYP73PxRbplatWrCyskJERESx+9SuXRunT59GWloaYmJicOnSJeTm5qJWrVoACsb1zJo1C0uWLEGfPn3QpEkTTJgwAUOHDsVPP/2k8ddZklhZJs5HJiJWlvla69UU3o1FRPSGmDFjBlJSUuDh4QFdXV3k5+fj+++/x/DhwxX79OjRAwMHDoSbmxsiIyMxa9Ys9OzZEwEBAdDV1S21DrlcjkmTJqFdu3Zo1KgRElKz8N2BWzh4LRapVw7h6an1ELlZ0LN0hM3QBdDV04erlUl5vuwyEUJAlpIOI8lLfQMSHcTLMuEfGo+O7tbQ11V+/sGDB3jy5Ans7e1LrcPU1BSmpqZ4+vQpjhw5gsWLFwMAcnNzkZubCx0d5WPr6upCLper9TpiZZmISkyHm5Up7KXGpe4vhEBadh4S03KwIzAaq87chRCAjgTwG9gYQ1s5q1V/RWPYISJ6Q+zcuRNbt27FH3/8gYYNGyIkJASTJk2Cg4MDRo4cCQAYNmyYYv/GjRujSZMmqF27Nk6dOoWuXbuWWoevry9u3LiBM2f+xbZL0fD7OxQpWXnQkQDjPx4Ji48H45e/ApF8aTcS/1oI51E/IyMnv9xec1lcikqC3z+hkLi0gOz8DuhaWMPAyhk58ZFICdwHsyZvY8zGIEj18mF6cy8+GfkeOnrWxd27dzFt2jTUqVMH3bt3Vxyva9euGDBgACZMmAAAOHLkCIQQqFevHiIiIjB16lR4eHjgo48+AgBYWFigU6dOmDp1KoyNjeHi4oLTp09j06ZNWLJkicqvY0dgNGbuuQ75s7Ay+W13tHa1RGJaDhLTshWPx6nKP2flFg5UcgFM330dJ8Meo4VzdXjYm8PDzgLW5iXfZl/RJEK8MAz/DZWSkgKpVAqZTAYLC4uKbg4RqSA/Px/z5s3Dli1bEBcXBwcHB4waNQrffPMNJBIJAGDevHnYvn07YmJiYGBggBYtWuD7779HmzZtij2uq6sr7t+/X2j7+PHjsWLFCgDA6tWr8ccff+DKlStITU3F06dPUa1atXJ5nZrk5OSEGTNmwNfXV7FtwYIF2LJlC27fvl1sOWtrayxYsACffPJJicefMGEC/vrrL2za8zd+u5yKwHtPAQCNa0rhN7AxGtWUAijoZbj9IAlvN6+D6t0/h+dbvbHPtx3MDCvH9+878alYfPg2jocmAAAM5NnQD9mFsIv+kGfIoGtmiU49+8P73U/w961EJDxNweM9C5CTcBciOx3VrGzR3edt/PrjQtja2iqO6+rqilGjRmHevHkACsLnzJkz8eDBA1haWmLQoEH4/vvvIZVKFWXi4uIwc+ZMHD16FElJSXBxccG4ceMwefJkxfv8ZUIIPHiaidtxqQiMeoLV/0a98u/CUE8H2Xml9yJZmRmivr05POwKwo+HvTnq2JjBUE9X7V4ldah6/q4c7ywiIjWpMtGbu7s7li9fjlq1aiEzMxO//PILfHx8EBERAWtr6yKPGxgYiPz8/3oabty4gbfffhuDBw9WbMvIyECPHj3Qo0cPzJw5s3xfqAZlZGSofUlElcsxQgh8/vnn2LN3L0Z+vx7j9sUgN1/AxEAXU952xyhvV+i9cJnHXmoMSyMrGOhKYGEARCSk4audV/H7B82LPYG/DrGyTPxy7A7+vPwAcgHo6kgwrJUTJnatCxuLgYiVZeJeYgZcrUwUJ+05/eQ4F/kEe1utxZGb8cjMLXjvBAD4dHcEBjTLxDtNHGBpaoCAq6GISkxHrCwT9lJjDBkyBEOGDCmxTXZ2dli/fn2xgSE1Kxd34lMRGpuK23EpuB2bittxqUjLzivxuLYWhnCqbgIrM0NYmRvAyswQ1uaGBT+bGcL62XZZZi7aLTwB+QvdIjoSYEx7NzxMzsTt2FREPUlHYlo2/g3Pxr/hiYr99HQkqGFmgPiUbEW5iroExp4dsGeHqCp65513YGtri7Vr1yq2DRo0CMbGxtiyZUuRZZ7/Wz9+/LhKl2QAYNKkSTh48CDCw8MLnYhPnTqFzp07V5menVGjRuH48eNYtWoVrJ1qw//cRfw0+yt8PGY0Fi1ahLS0NHz77bcYNGgQ7OzsEBkZiWnTpiE1NRXXr19XzAj88uWY8ePHY/OWrajzwbdI1LECALSrWwPzB7dGXYcauHv3Lnbs2AEfHx9YW1vjwYMHWLhwIc6dO4edxwLw2e5I5OTLMbV7Pfh2rvPafy+yjFz8djoCG87dU/Ri9Gxkh6+610NtazOVj5OenYejt+KwN/gRzoY/VgQEPR0J6tqa4XZsKgQKTvrTutfDO54OKh334NVHWHwkDHIBSAB08bCBjo4Et+NSEJNU9IBhfV0J6tiYw8XSBEduxuHFE72uRIKzMzqr3MuyIzAas/bcQL4Q0JVI8MPARkqBJSMnD+HxaQiNTcHtuFTFf2WZuYWOpW7dpWHPDhFpNW9vb6xevRp37tyBu7u74s6i4sYy5OTkYPXq1ZBKpfD09FSpjpycHGzZsgVTpkyp0B4HTVm2bBlmz56NUR9/gieJj6FrZgnTBl3RpN9YAAW9PNeuXcPGjRuRnJwMBwcH+Pj4YP78+UpLH0RGRiIxseAbfHJGjmLG35DfJyv22QGgh3Q96o4aBSMjI/z777/49ddf8fTpU9ja2qJjx444f/486tVzx7d5Rpi55zp+OhqGRjWl6ORedK+bpmXl5mNTwD2sOBmpODG3drPEjJ4eaO5c/OzHxTE11MOAZo4Y0MwRCalZ2B/yCPtCHuLGwxSExqYq9pMLYOHhMCw8HFbC0YomAPjfTlDaZmdhpBg7U//Zf2tZmyoGTRcVVtQJG0NbOaOju3WhXq3nTAz04OlUDZ5O1f5rpxA4dD0WE/4IVto3XwjcS8zQ+OWs0rBnB+zZIaqK5HI5Zs2ahcWLFyvdWfTyZaWDBw9i2LBhyMjIgL29Pfbt24dWrVqpVMfOnTvx/vvvIzo6Gg4Ohb+FV7WeHaDgUs3LlyUkAGb28kBdG3PFZY0apoYw0Ct6dpJYWSaiHqcjPCEVy05EIDEtBwAwvI0zpvXwgNRYX+12zdxzDdsuxUBqrI8DE9rDuUb53KEVK8tEZEI6QmNlWH/uHh7JsgAA9WzNMb1nPXSuZ6PxYLsrKAZT/7xWaLu+jgQ6OiXXJZcL5MoLn6ZHtHVBj8Z28LCzgKWpQaltKOoSXHkr6r3Gnh0iqjDlOYCwvOpW5c4iAOjcuTNCQkKQmJiINWvWYMiQIbh48SJsbGxKrWPt2rXo2bNnkUGnqop6nI6Xz50CwA9/Fx6gLDXWh5WZwbMAVDCOIzY5E0dvxStdFqlrYwa/gY3R0tWy0DFUNa9vQ4TGpiIkJhnjNgdhz3hvmBho9hS1IzAaM/Zcx4tf8R2kRpjiUw8DmtWEbinB41W1r2sFHQkKnfTPTC/9pF9cYPisc221/r3YS41f+79te6kx/AY2LlOvkqawZwfs2aE328u3pb7OAYRlqftV7yyqW7cuRo8eXerA4vv376NWrVrYs2cP+vXrV+Q+VbFnZ8uF+/hm3w2lbRIUXL5JzcpDYlo2nqTnIL+I3oSiSACcnvoWnGuYlrltsbJM9Fl2FolpOejr6YD/DWuqsV6WqzFP0W/FeaVtEgAnv3oLrlZlb3tpShv3Ul5lK4Py7FVizw4RlSpWlqkIG0DBN89Ze26go7t1uX/7KqrumXuuq1z3q9xZBBRc/srOzi71+OvXr4eNjQ169+5d6r5VRWpWLpb6hwMoONELoMiTp1wukJyZWzDfSmo2HqdlIzEtB1djkrH/6iOlYwoAD5OzNBJ27KXGWPF+cwz/v4vYf/URmjhK8XGHWmU6Zr5cYMuF+/D7J7TQcwJArCzrtYSd0sa9lFfZyqAiepVexrBD9Aa7HZtS6JLG6xpAGJVY+HKKXABbL9zHlLfrlTqWoU+fPvj+++/h7OyMhg0bIjg4GEuWLMHo0aMBAOnp6fj+++/Rt29f2NvbIzExEStWrMDDhw+VbiN/+c4ioCAQrV+/HiNHjoSeXuGPybi4OMTFxSmWArh+/TrMzc3h7OystERCaV735cOfj95BQmo2XGuYYMNHrZ+d6AufPHV0JLA0NYClqQHcbc2V2nvw2qNCl1Q0OQNym1o18HXv+vj2wC34/XMbDRws4F3b6pWOFRqbgpl7riMkJrnI5zXd9tKU5aRfGQJDVca1sYgqWH5+PmbPng03NzcYGxujdu3amD9/Pl68wrxnzx74+PigRo0akEgkCAkJUenYu3btgoeHB4yMjNC4cWP8/fffiueS0nOw+Mgd5CbGIGH3d4j+ZQiilwxC7MbJ0MtUby2kV+FWzLfp5Scj0Wf5Wfwb/rjE8suWLcO7776L8ePHo379+vjqq6/wySefYP78+QAKenlu376NQYMGwd3dHX369MGTJ0/w77//omHDhgAKTt63wsIR9SBW6djHjx9HdHS0Iji9bOXKlWjWrBnGji24i6ljx45o1qwZ9u/fr/Lr3xEYjXYLT+D9NRfRbuEJ7AiMVrnsq7j+QIZNAfcAAPP7N4KrlSm8atdQe9yH38DG0H12aam8xmCM8nbFgGY1kS8X+PyPYDxKVm89psycfCz85zb6LDuLkJhkmBvqYX7/Rq+l7VQ5ccwOOGaHKtYPP/yAJUuWFJoc7/vvv1dMjrd582ZERUXBwcEBY8eORXBwMJo2bVricc+fP4+OHTvCz88P77zzDv744w8sWrQIV65cgdShFkauu4Sw8AjEbZ4C8yZvw6R+J0gMTGCW8QgnF4+Dg71dub7uM3ceY8S6S4qfdSTA2w1scT7iCVKfTYjWoa4VpvfwUMy8q0kVOVbp4dMMtF90skxzn6gjXy4w4LdzuPZAhr6eDlj6XrMyHe913NmTmZOPQb+fx63YFDRxlGLnJ14w0i99ba5/wx/j6703EJ1UsLhor8Z2mNunIWwtjF5b2+n1UfX8zbADhh2qWOpMjnfv3j24ubmpFHaGDh2K9PR0HDx4ULGtbdu2cKnbAHc93sfj1Gyk/f0T2tezxe//tx43HsgwZddVpGblYX6/hvjQy1WTL1NJbr4cPf/3LyIS0jCkZcG8JM9PPknpOVh+IgKbL9xDbn7Bx1NfTwd85VNPY7cjRyelo9OPp5TuyinPsPGi7Lx8jFx3CRfuJhV6btvYtvCqXUPjdW4KuIc5f92EuZEe/L/sBBtzI43XUR5ikjLQZ/lZJGfkYkhLRywa1KTYActP0rKx4FAo9gY/BFBwl9V3/RqhWwPbIvcn7aDq+ZuXsYgqmLe3N/z9/XHnzh0AUEyO17NnzzIdNyAgAN26dVPa1rBVB+w/dgqPU7Phbm2KrKggtGjSEKOG9Mfwzk2QsWs6Mu4E4Odjd/A0PadM9ZdkU8B9RCSkwdLUAF/3bqB0OcXS1ABz+jTAiS/fwoBmNSGRAPuvPkLXJacwb/9NPEkrfXDxy4QQiEhIxfpzURizIRA+S87g5a95z8cqlafUrFyM3hBYZNDRkaBcxo8kpGThx2eT103rXq/KBB0AcLI0wbL3mkFHAuwMeoCtFwtf6hNCYGdQDLouOY29wQ+hIwE+aueKo1M6MeiQAgcoE1WwGTNmICUlBR4eHkqT4w0fPrxMx42Li1NahPDQtVgcCM9ATupTtHazxAIfR9T7Kg0LFy7EggULsGjRIvz99z/4+puvoWNkhiXHHDC/f6OyvrxCEtOy8euxgmA3rXu9Yiegc7I0wS9Dm+LjDm5YdDgMZ+48xobz9/Dn5Qf4pGMtjOngVuI8LE/SsnE2IhFnwxNxNiIRsc8mjyvJoeuxaO1mWS7zrSSkZmHUukDcik2BqYEuhrV2xvpzUYrBvpamBuWyEOb8Q6FIzc6Dp6MU77dx0fjxy1uHutaY2t0Diw7fxrcHbqK+vQVauBTMbnz3cRpm7b2uCI8N7C2wcFBjNHGsVoEtpsqIYYeogqk6OV5ZbDx/D/MO3ES+XMBQTwebRrdG0uN4AEC/fv0weXLBNP9NmzbFP/6ncSXkH2x1aYz3WjujgYNmL+3+eDgMqdl5aFTTAoNbOpW6f0MHKTaNbo1zEYlY+M9tXH8ow8/H7mDThfuY2LUuOrpb4cHTTDhIjfHgaSb+DX+Mf8MTcSs2Rek4Bno6aO1qiQ51rdC+rhWuPZDhm70Fc5c8vw17y4X7ePA0A/8b1uyVZgEuTlRiOkasu4iYpExYmRlg/ajWaOwoxccd3HA1Jhlz/rqJhNRsTNl5Fas+aFHqnWiqOnPnMQ5cfQQdCfD9gMblNmleefu0Uy1cf5iMv6/HYdymIMzt0wDXH8qwMeA+cvLkMNbXxeS362J0OzelBUeJnmPYIapgU6dOxYwZMzBs2DAAQOPGjXH//n34+fmVKezY2dkhLi4OPx65jRUnIwEADarLkeLqCCN9XVhZWUFPTw8NGjRQKufd0hMRe49ALoBvD9zE9nFtNTax27UHydh5OQYAMK9PQ7VOvu3qWOEv33Y4dD0WPx4JQ3RSRqHJ8V5W394CHZ+Fm1aulkoDXBs6SPFWvf/mLrkUlYRpf17DqbDH6L/iHNaMaIE6NuYlHF01V2OS8dGGQCSl58Clhgk2jW4Nl2dz0jy/ndheaozBqwJw7FY8VpyMwOdd65a53qzcfMz+q+D3M9LbtVwGeb8uEokEi9/1RNC9p0hIzcYX20MUz3Vyt8aC/o3gZPn6biGnqocRmEiDYmWZOB+ZiFiZ6rfKvurkeKVp07Ytlm/Zpwg6X77tjvyYa/Dy8gIAGBgYoFWrVggLU16M8M6dO2jTxB1G+jq4GJWEQ9djCx37VcjlAvP234QQwIBmNV9paQEdHQn6eDrg+JROmPK2e5H79Gpsh/8Na4rAr7vhn4kdMLNXfXSoa13knTz2UmPFeKF+TWti92fecJAaISoxHf1XnMfxW/Fqt/FFp8IS8N6aC0hKz0HjmlLs/sxbEXRe5OlUDQv6FVwyXHL8Dk6GJRTaR12/nYrE/ScZsLUwLPZ3VZWkZhVMcvgiiQTwG8igQ6Vj2CHSkFedN+X55HiHDh3CvXv3sHfvXixZsgQDBgxQ7JOUlISQkBDcunULABAWFoaQkBDExcUp9hkxYoRiCYSMnDxk1PXB3ZBzSL20B583N8aTf7ciKChIafK8qVOnYseOHVizZg0iIiKwfPlyHDhwAF9N+gKfdaoDAPjhUCgyc/LL/PvZF/IQV6KTYWKgixk9Pcp0LAM9HbR0LXpV6g/buqJf05qwNjcs8vmSNKopxf7P26O1qyXSsvMwdnMQlp8Ix6vctLrnygN8vDEIGTn56FDXCtvGtYWVWfFtGtLKCcPbOEMIYOK2YNxLTFe7zuciH6dh5amCkDvnnYYwN9LcJbmKUtQklEIA95+oNwcPvZkYdog0oLhlF1Tp4SltcjwA2L9/P5o1a6ZYumDYsGFo1qwZVq5cqdgnOjoasbGxSErPwftrLuJmvj3s+0+DSdRpzPqwJ/7880/s27cPjRr9N+h4wIABWLlyJRYvXozGjRvj//7v/7B79260b98en3SqhZrVjPFIloXfT0eW6feTlp2Hhf8UrFc1oUsdxZwnZeFmZYqXr4JpYkZcKzNDbPm4DT5oWxA8fjp6B75/XEH6s7l/SiOEwKrTkZiy8yry5AL9mzpg7chWKg0+ntunIZo7V0NKVh4+2XxZ5Tpfrn/2vhvIyZejk7s1ejUu3/mSXpfy+nvTm4Hz7IDz7FDZ7b/6CF9sCy60vbzmTSlKrCwTgVFJ+OloGKKTMiE11se6US3RwuXVV6L+53osPtt6BQZ6OvCf0umVLxcs/Oc2Vp6OhEsNExyd3BGGeqVPDqeK8l4gcdulaMz56wZy8wU87MyxZkTLEn8HcrnA93+HYu3ZKADA2A5umNmzvloDjuNTsvDOsrN4nJqN3k3ssfy9ZmqNmdoX/BCTdoTAUE8HRyd3LPKyWVVV1RfEJM3jpIJqYNihsniSlo0Bv51DdJJyL46OBDg3o8trmaV1R2A0Zuy5rpg7Rmqsh92feZd5gK0QAsP/7yLORz5Bj4Z2WPlhC7WPEZWYDp9fTiM3X+D/RrTU+Nwn5T0jbtC9JHy65QoS07JR3UQfK95vDu86hddqys7Lx1e7ruHAs4Uyv+5VH2M7vtoiloH3kvDe6gvIkwvM6uWBcR1rq1ROlpGLrktOITEtB1/5uGNCl7IPdK5sOAMyvYiTCtJr5+rqColEUujh6+sLoGDelw8//BB2dnYwNTVF8+bNsXv37lKPu2LFCri6usLIyAht2rTBpUuXlJ7/5JNPULt2bRgbG8Pa2hr9+vXD7du3y+U1viw1Kxcj11961pOip9TN3sRRCjsNXK4pTawsUynoFLQrD6YamLNFIpFg7rO7pg7fjMO5iES1j7Hg4C3k5gt0crdG1/o2ZW7Ty14cZFweWrpa4sDn7dDEUYqnGbn4cN0lbDgXpTSO5/lkgQeuPoK+rgS/Dm36ykEHAFq5WmJun4K75Bb+c1vl3/viI7eRmJaD2tamZaq/Mivvvzdpp0oddlRZIFEIgTlz5sDe3h7Gxsbo1q0bwsPDK7DVb67AwECs/DsQjhM2w9F3M+yGLQAAxQrTI0aMQFhYGPbv34/r169j4MCBGDJkCIKDC1/+eW7Hjh2YMmUK5s6diytXrsDT0xPdu3dHQsJ/d6u0aNEC69evR2hoKI4cOQIhBHx8fJCfX/ZBtSXJys3H2E1BuPEwBTVMDbBnfDucm9EFc96pD10JEBIjw7pz98q1DQDwb3hiodmA5QIamw24np05PmxbMBndtwduIi9f9bvEToYlwP92AvR0JJjTp4HGbmF/3eylxtj5iZdiccp5B25h2p/XcP9JOv6+/giDfj+PcxFPYGqgi3WjWqF/s5plrvODti54t4Uj5AKY8McVPHha8t8zOPop/rhUMCh+Qf/GGrtUSKQNKnXYWbRoEX7//XcsX74coaGhWLRoERYvXoxly5Yp9lm8eDGWLl2KlStX4uLFizA1NUX37t2RlVX6bKmkOflygeN3M+B3Oh66ptWha1Yd6eGXoFfNHnU9WwEoWJjy888/R+vWrVGrVi188803qFatGi5fvlzscZcsWYKxY8fio48+QoMGDbBy5UqYmJhg3bp1in3GjRuHjh07wtXVFc2bN8eCBQsQExODe/fuldvrzcuXY8IfwbhwNwlmhnrYOLo1alubwV5qjNHta+Gbdwq+lf/wdygCIstvBfGYpAwsPly4F0vTAzcnd3NHdRN93IlPw5YL91Uqk5Mnx/wDBXePfdTOFbWtzTTWnopgpK+LJUM88U3v+tCRALsuP0CnH09h/NZg3IlPg6mBLraP80KHutYaqU8ikWBB/0ZoXLOgR+nTLZeRlVt0gM/Ll+PrvTcgBDCwec3XNk6MqKqo1GHn/Pnz6NevH3r37g1XV1e8++678PHxUVzGEELg119/xTfffIN+/fqhSZMm2LRpEx49eoR9+/ZVbOPfEBk5edh4/h66/HwKM/f+N8GbyM9F+q1TMGvyNj7eeBmRj9Pg7e2NHTt2ICkpCXK5HNu3b0dWVhbeeuutIo+dk5ODy5cvK63vpKOjg27duiEgIKDIMunp6Vi/fj3c3Nzg5FT67LyvQi4XmL77Oo6HxsNATwf/N7JloQnbRnm7KnoBJvxxBY+SNX977OPUbHy49iIS03JgY26ouIT2fOCmJrv5pSb6+Kp7PQDAkmN3VFqfasP5KNxNTIeVmSG+0MAkeZWBRCLBxx1q4ZehTQs9l5mbDytzA43WZ6Svi5UftoClqQFuPEx5FmgKD7PcGHAft2JTIDXWx6xe9TXaBiJtUKnDTmkLJEZFRSEuLk7pZCiVStGmTZtiT4YAkJ2djZSUFKUHqSchJQs/HrkNL78TmLv/Ju4/yYCFkR6eX6TIuHMB8qw0mDbqitC4VPT89V90Hu+H7Jwc1KhRA4aGhvjkk0+wd+9e1KlTp8g6EhMTkZ+fr7S+EwDY2toqzS8DAL/99hvMzMxgZmaGf/75B8eOHYOBgWZPPEBBwP7+71DsvvIAujoSrHi/OdrWKvwtWiKR4IcBjdHA3gJP0nPwWQnfyl9FSlYuRq67hHtPMuBY3Rj7J7THuRldsG1sW5yd0blc7lAZ1soZDewtkJKVh5+O3ilx34TULCz1jwAATO9RTyvmeXlRUXP4aPLS4YtqVjPG8meLYe6+8gCbX+pZi5VlYsnRgokhZ/T0KHEuH6I3VaUOO8+n0Pfw8IC+vj6aNWuGSZMmKRZIfH7CU+Vk+CI/Pz9IpVLFo7x6ALRRWFwqpu66ivaLTmLFyUjIMnPhUsME3/VriAuzumLhoMbQlUiQdu0oTGq1xNeDvdHJ3Ro5+XL4zZ+HS7djsHzzHgQFBWHKlCkYMmQIrl+/XuZ2DR8+HMHBwTh9+jTc3d0xZMiQcrmUueJkhOK24sWDmuDtEu4sMjbQxaoPW6CaiT6uPpBh7l83X2lyupdl5uTj4w1BuBWbAiszA2we0wZ2UqNyH7ipqyPBvL4NAQDbA6Nx46Gs2H0XHw5DWnYePJ2qYVBzx3JpT0V63XO+eNexwsyeBT023x24hcB7/62a/t2BW0jPyUdz52oYqsJaY0Rvokoddl5cIPHKlSvYuHEjfvrpJ2zcuLFMx505cyZkMpniERMTo6EWaychBM6GJ2LEukvo/usZ7Lr8ADn5crR0qY6VH7TAiS/fwggvV5gY6GFoK2dsf782cqKvYvl3X+Kzt+pgw0etMLN9daReOQjTtz/HjzcMsD1SgklTZ6Fly5ZYsWJFkfVaWVlBV1cX8fHKU/bHx8fDzk55ojSpVIq6deuiY8eO+PPPP3H79m3s3btXo7+HzRfuK3o0Zr/TAINalH4Sd7I0wbJn38p3BMUoBpC+qtx8OXz/uIJL95Jg/myskJvV65tHpbWbJfp6OkAIPFv6oXB4C45+ij8vPwAAzOvTQGOLWlYm9lJj+A0sCPZA+Vw6fNnHHdzQx9MBeXKB8VuvID4lCydux+OfG3HQ1ZHg+wGNtfJ3TaQJlXoh0NIWSHx+wouPj4e9vb2iXHx8PJo2bVrscQ0NDWFoyK7eksTKMhEen4bw+FTsuvwAt+NSARTMHdOzkT0+7uCGZs5FT9d/aPc22NjY4MMhAwEUXNLxdimY76VXEwccjwW2XYrBsVsJkGfkFXvXlIGBAVq0aAF/f3/0798fACCXy+Hv76+05MHLhBAQQiA7u/RxJao6cPUR5jxbVPHzLnUwpr2bymU71LXG1O4eWHT4NubtvwkPOwu0cCn6d1cSuVxg6q6rOHE7AYZ6Olg7qhUaOrz+xR1n9vLAsVvxCLr/FPuvPkK/pv/defR8/SsAeLeFY7HvEW0wtJUzOrpbv7Y5XyQSCRYNaozw+FTcjkvF6A2BiE8p6L0c094N9e05RxhRcSp1z05pCyS6ubnBzs4O/v7+iudTUlJw8eJFxWKHpL7tl6Lh7XcCI9ZdwvxDobgdlwoTA12M8nbF6amdsWJ482JPYnK5HOvXr8fIkSOhp/dflvbw8ECdOnVwZ/cSzGljAHtJMu6e2IbggDOIMmuAmKSCsQ5du3bF8uXLFeWmTJmCNWvWYOPGjQgNDcVnn32G9PR0fPTRRwCAu3fvws/PD5cvX0Z0dDTOnz+PwYMHw9jYGL169dLI7+P0nceYsjMEQgAftnV5pUUVP+1UC70a2yE3X2D81stISFXvEpsQAt8dvIV9IY+gpyPB7x80R2u3V58ZuSzspcbw7VwwyZ3f37eVljTYfeUBrj6QwcxQD9N61KuQ9r1Or3vOFxMDPaz6sAWM9HRw81EKEtNyAACO1TjnDFFJKnXYKW2BRIlEgkmTJmHBggWKuVtGjBgBBwcHRU8Aqef5Gk8vXpyQANg3vh3m9W1Y6nIBx48fR3R0NEaPHq20XV9fH3///Tesra0x89MPcG3pOBjfPwfbPlMQYegOn1/OYM2Zu4iMjETUg1jFyuFDhw7FTz/9hDlz5qBp06YICQnB4cOHFeO0jIyM8O+//6JXr16oU6cOhg4dCnNzc5w/fx42NmWfwO7y/SR8uvkycvMF+ng64Nu+DV9prhiJRILF73qiro0Z4lOy4bv1CnLyVJ+vZql/BDacvwcA+GmwJ7p4aHYWYnV93KEWnCyNEZeShd9OFQxETsnKxaLDBQNlv+haBzbm5T+h4pvIQE8H2S/NdfTtgVsqrcNG9Kaq1MtFpKamYvbs2di7dy8SEhLg4OCA9957D3PmzFHcaSOEwNy5c7F69WokJyejffv2+O233+Durvq3by4X8Z/dl2Pw5a5rhbaX1xpPEQlpmLX3Oi5FFQy4dJAaITYlC0IUXDLzG9i4wta+uR2XgiErA5CSlYdO7tZYM6IlDPTK9v3g7uM09Ft+DqnZeRjp5YJv+zUqtczG8/cw99mloW/7NsRIb9cytUFTjt6Mw7jNl2Ggq4NjUzpiy4X7WPNvFGpZmeLwpI5l/l1R0c5HJuL9NRcLbX+d67ARVRZcG0sNDDsF8vLl6LP8HEJjlW/F15VIcHZG53LrqpfLBXZdjsGCQ6FIzVJe5VlHAvzl2w6Nakpf6+y70U8yMGjleTxOzUYLl+rYPKY1TAw0M8TNPzQeYzYGAQB+HuxZ4kDnv0IeYuL2EADApG51Mamb+pfQyosQAiPWXcK/4Ylo5GCBW7EpkAtg/Uet0Lme5peFoAKxsky0W3gC8hc+ucv73yhRZcW1sUhtK05GIjQ2BUZ6OuU6Qd3LdHQkGNrKGT++26TQc3IB9Fl+Di0WHMf7ay7guwO3sCsoBjceyoqctyZWlqm4BPYqYmWZ+PvaIwxbE4DHqdnwsDPHupGtNBZ0AKBrfVtMfDbJ3qy914u9hfvk7QR8ufMqgIJJCidWson5CtbNagAdCXDjUYri5JuQwtnLy1NF3AlGVNVV6rux6PW5GpOMpScK1hRb9G4TtHazfO0rC3s6VYOOBErfWIGCMUNJ6Tk4H/kE519YekFXRwI3K1N42Jmjvr0FHqdmYVPAfchf8RLYjsBozNxzXVG/pakBNo1uDamJ5ifEm9i1Lm48lMH/dgI+2XwZBz5vD0vT/yZBDLyXhE+3XEaeXGBAs5qY807lXFfK1FCv0Lpcs/bcQEd3a558y9HrvhOMqKrjZSzwMlZmTj56L/sXdx+n450m9lj+fvMKa8uOwGjM2nMD+UIovrH2a1oT4fFpCI1LQWhsCm7HpiI0LgXJGbmlHq+asb5Kc4/I5QLJmcrH05EA52Z0KbcTiSwzF/1XnENUYjq8a9fAptGtoaerg1uPUjB0dQBSs/LQ1cMGKz9sAX3dytkJy/EjRFSRVD1/s2eHsPCfUNx9nA5bC0Ms6F/6gNnyVNw31saOUjR2/G9OGSEEElKzC8JPXCr+vfMY54pYcPPlAKOO59P/l1fYkRrrY9WHLdB/xTmcj3yCOftvopVLdXx3sGDsUmtXS6wY3rzSBh3gv5mEXx4/Ul4zCRMRvQr27ODN7tk5fecxRq4rWFh185jWGlux+XUratCmjgTYMqZNkesYvexxajY+WHuxQgZ9/n09FuO3XlHaZi81wpHJHWFRBdaUKqo3rqLuoCOiNwt7dqhUyRk5mLrrvwGwVTXoAP8N2nz5pOtdx0ql8nVtzYss/zrGQjRzrgYJoDS3UXxKFtKz86pE2OH4ESKq7Bh23lBCCHy97wYSUrNRy9oU03t4VHSTyqysJ92KOmlHJabj5e7V8r6Epmn2UuMq01YievMw7Lyh9l99hEPXYqGnI8GvQ5vC2EC3opukEWU96VbESZvjXoiIylflHflI5eZRcia+2VewqOUXXeuiiWO1im3QG47zphARlS/27Lxh5HKBqX9eRWpWHpo6VcP4t2pXdJMIHPdCRFSeGHbeMBvO38O5iCcw1tfFkiGe0KvEtzW/aTjuhYiofPBM9wYJj0/FwsO3AQBf966PWtZmFdwiIiKi8sew84bIyZNj8s4Q5OTJ8VY9awxvw3lQiIjozcCw84ZY6h+OGw9TUM1EH4sHNamU6ywRERGVB4adN8Dl+0n47VQEAMBvQGPYWBhVcIuIiIheH4YdLZeenYcpO69CLoCBzWqiZ2P7im4SERHRa8Wwo+UWHArF/ScZqFnNGPP6Nazo5hAREb12vPVcS8XKMrHn8gNsuxQNiQT4abBnlVhniYiISNMYdrTQjsBozNxzXbH8QPvaVvCqXaNiG0VERFRBeBlLy8TKMpWCDgCci0xErCyz4hpFRERUgRh2tExUYrpS0AH+W0GbiIjoTcSwo2XcrEzx8hQ6XEGbiIjeZAw7WsZeaoz2dawUP3MFbSIietNxgLIWepRcMD5nYte6GNbaiUGHiIjeaAw7Wib6SQYiH6dDV0eC0e3dIDXm7eZERPRm42UsLXMyLAEA0NKlOoMOERERGHa0zonbBWGni4dNBbeEiIiocmDY0SIZOXkIuPsEAMMOERHRcww7WuR8xBPk5MnhWN0YdWzMKro5RERElQLDjhZ5Pl6ni4cNJC9PtkNERPSGYtjREkIInHw2XqdzPV7CIiIieo5hR0uExafikSwLRvo6XPSTiIjoBQw7WuL5XVjeta1gpK9bwa0hIiKqPBh2tITiEhbvwiIiIlLCsKMFkjNycPn+UwBA53rWFdwaIiKiyoVhRwucCU+EXADutmZwrM7VzYmIiF7EsKMFeAmLiIioeAw7VVy+XODU8/l1eMs5ERFRIQw7VVxITDKeZuTCwkgPLVyqV3RziIiIKh2GnSru+SWsju7W0NPln5OIiOhlPDtWcS8uEUFERESFMexUYfEpWbj5KAUSCdDJnbecExERFYVhpwp7fgnL07EaapgZVnBriIiIKieGnSrs+RIRvIRFRERUPIadKio7Lx9nIxIBMOwQERGVhGGniroUlYSMnHzYmBuioYNFRTeHiIio0mLYqaJO3n4MAHirnjUkEkkFt4aIiKjyYtiponjLORERkWoYdqqgqMR0RCWmQ19XgvZ1ecs5ERFRSRh2qqDnd2G1drOEmaFeBbeGiIiocmPYqYIUq5xz4U8iIqJSMexUMenZebgY9QQAx+sQERGpgmGnijkbkYjcfAGXGiZwszKt6OYQERFVegw7VcyLl7B4yzkREVHpGHaqECEEbzknIiJSE8NOFXLzUQriU7JhYqCLNrUsK7o5REREVQLDThVy6lmvTrs6VjDU063g1hAREVUNDDtVyAneck5ERKQ2tcPOyZMny6MdVIqk9BwExyQDADp7cNZkIiIiVakddnr06IHatWtjwYIFiImJKY82URFO30mAEEB9ewvYS40rujlERERVhtph5+HDh5gwYQL+/PNP1KpVC927d8fOnTuRk5NTHu2jZ048W+W8C3t1iIiI1KJ22LGyssLkyZMREhKCixcvwt3dHePHj4eDgwO++OILXL16tTza+UbLy5fjNG85JyIieiVlGqDcvHlzzJw5ExMmTEBaWhrWrVuHFi1aoEOHDrh586am2vjGC45JRkpWHqqZ6KOpU/WKbg4REVGV8kphJzc3F3/++Sd69eoFFxcXHDlyBMuXL0d8fDwiIiLg4uKCwYMHa6SBDx8+xAcffIAaNWrA2NgYjRs3RlBQkOJ5IQTmzJkDe3t7GBsbo1u3bggPD9dI3ZXF87uwOrlbQ1eHsyYTERGpQ+2w8/nnn8Pe3h6ffPIJ3N3dERwcjICAAHz88ccwNTWFq6srfvrpJ9y+fbvMjXv69CnatWsHfX19/PPPP7h16xZ+/vlnVK/+X+/G4sWLsXTpUqxcuRIXL16EqakpunfvjqysrDLXX1k8XyKCl7CIiIjUp6dugVu3bmHZsmUYOHAgDA0Ni9zHyspKI7eoL1q0CE5OTli/fr1im5ubm+L/hRD49ddf8c0336Bfv34AgE2bNsHW1hb79u3DsGHDytyGivYwORO341KhIyno2SEiIiL1qN2z4+/vj/fee6/YoAMAenp66NSpU5kaBgD79+9Hy5YtMXjwYNjY2KBZs2ZYs2aN4vmoqCjExcWhW7duim1SqRRt2rRBQEBAscfNzs5GSkqK0qOyet6r09y5OqqZGFRwa4iIiKoetcOOn58f1q1bV2j7unXrsGjRIo006rm7d+/i999/R926dXHkyBF89tln+OKLL7Bx40YAQFxcHADA1tZWqZytra3iuaL4+flBKpUqHk5OThpttyY9XyKiMy9hERERvRK1w86qVavg4eFRaHvDhg2xcuVKjTTqOblcjubNm+OHH35As2bNMG7cOIwdO7bM9cycORMymUzxqKyTI2bl5uNcxBMAXCKCiIjoVakdduLi4mBvb19ou7W1NWJjYzXSqOfs7e3RoEEDpW3169dHdHQ0AMDOzg4AEB8fr7RPfHy84rmiGBoawsLCQulRGV24+wSZufmwlxqhvr15RTeHiIioSlI77Dg5OeHcuXOFtp87dw4ODg4aadRz7dq1Q1hYmNK2O3fuwMXFBUDBYGU7Ozv4+/srnk9JScHFixfh5eWl0bZUhINXHwEAWrtZQiLhLedERESvQu27scaOHYtJkyYhNzcXXbp0AVAwaHnatGn48ssvNdq4yZMnw9vbGz/88AOGDBmCS5cuYfXq1Vi9ejUAQCKRYNKkSViwYAHq1q0LNzc3zJ49Gw4ODujfv79G2/K6bb8UjT+vPAQA7L/6CN61a2BoK+cKbhUREVHVIxFCCHUKCCEwY8YMLF26VLEelpGREaZPn445c+ZovIEHDx7EzJkzER4eDjc3N0yZMgVjx45Vas/cuXOxevVqJCcno3379vjtt9/g7u6uch0pKSmQSqWQyWSV4pJWrCwT7RaegPyFv4yuRIKzMzpzEVAiIqJnVD1/qx12nktLS0NoaCiMjY1Rt27dEm9Fr+wqW9g5H5mI99dcLLR929i28KpdowJaREREVPmoev5W+zLWc2ZmZmjVqtWrFqcSOFYv3HujK5HA1cqkAlpDRERUtb1S2AkKCsLOnTsRHR2tuJT13J49ezTSsDfZrUepSj/rSiT4YWAjXsIiIiJ6BWrfjbV9+3Z4e3sjNDQUe/fuRW5uLm7evIkTJ05AKpWWRxvfONsuFdxa/2FbF2wb2xZnZ3Tm4GQiIqJXpHbY+eGHH/DLL7/gwIEDMDAwwP/+9z/cvn0bQ4YMgbMzT8hl9eBpBs6EPwYAfNzBDV61a7BHh4iIqAzUDjuRkZHo3bs3AMDAwADp6emQSCSYPHmy4pZwenU7A2MgBNCuTg241DCt6OYQERFVeWqHnerVqyM1tWBMSc2aNXHjxg0AQHJyMjIyMjTbujdMXr4cO4IKlq54rzV7yYiIiDRB7QHKHTt2xLFjx9C4cWMMHjwYEydOxIkTJ3Ds2DF07dq1PNr4xjgV9hjxKdmwNDXA2w1sSy9AREREpVI77CxfvhxZWVkAgK+//hr6+vo4f/48Bg0ahG+++UbjDXyTPB+Y/G4LRxjq6VZwa4iIiLSDWmEnLy8PBw8eRPfu3QEAOjo6mDFjRrk07E0TK8vEybAEAMDQVk4V3BoiIiLtodaYHT09PXz66aeKnh3SnJ2BDyAXQBs3S9S2Nqvo5hAREWkNtQcot27dGiEhIeXQlDdXvlxg57OBye+34cBkIiIiTVJ7zM748eMxZcoUxMTEoEWLFjA1Vb49ukmTJhpr3JviTPhjPEzORDUTfXRvaFfRzSEiItIqaoedYcOGAQC++OILxTaJRAIhBCQSCfLz8zXXujfE9mcDkwc2c4SRPgcmExERaZLaYScqKqo82vHGSkjJwvHQgoHJ77XmwGQiIiJNUzvsuLi4lEc73li7Lj9AvlygpUt11LU1r+jmEBERaR21w86mTZtKfH7EiBGv3Jg3jVwusD2w4BLWMM6YTEREVC7UDjsTJ05U+jk3NxcZGRkwMDCAiYkJw44azkUmIiYpE+ZGeujd2L6im0NERKSV1L71/OnTp0qPtLQ0hIWFoX379ti2bVt5tFFrbb9UcLv5gGY1YWzAgclERETlQe2wU5S6deti4cKFhXp9qHiJadk4eisOADCsFS9hERERlReNhB2gYHblR48eaepwWm/35QfIzRfwdKqGBg4WFd0cIiIiraX2mJ39+/cr/SyEQGxsLJYvX4527dpprGHaTAiB7YHPZkzm7eZERETlSu2w079/f6WfJRIJrK2t0aVLF/z888+aapdWu3A3CVGJ6TAz1MM7TRwqujlERERaTe2wI5fLy6Mdb5Rtz2ZM7tvUAaaGav8JiIiISA0aG7NDqnmanoPDNwoGJr/PuXWIiIjKndphZ9CgQVi0aFGh7YsXL8bgwYM10ihttvvKA+Tky9GopgUa1ZRWdHOIiIi0ntph58yZM+jVq1eh7T179sSZM2c00iht9eLA5PfYq0NERPRaqB120tLSYGBgUGi7vr4+UlJSNNKoymbhwoWQSCSYNGkSAODevXuQSCRFPnbt2lXscXR0dOD/5Vu4v+gdfNDWVVHmxx9/VOzTt29fODs7w8jICPb29vjwww95Sz8REVEZqB12GjdujB07dhTavn37djRo0EAjjapMAgMDsWrVKjRp0kSxzcnJCbGxsUqPb7/9FmZmZujZs2exxxq38hgcfTdj/Gp/xMbGYt26dZBIJBg0aJBin86dO2Pnzp0ICwvD7t27ERkZiXfffbdcXyMREZE2U/tWoNmzZ2PgwIGIjIxEly5dAAD+/v7Ytm1bib0aVVFaWhqGDx+ONWvWYMGCBYrturq6sLOzU9p37969GDJkCMzMzIo8liwjF6dicqFrVh0fd28GO7vq+Ouvv9C5c2fUqlVLsd/kyZMV/+/i4oIZM2agf//+yM3Nhb6+voZfIRERkfZTu2enT58+2LdvHyIiIjB+/Hh8+eWXePDgAY4fP15oDp6qztfXF71790a3bt1K3O/y5csICQnBmDFjit1nb/ADZOfJ4WFnjqZO1RAfH49Dhw6VWCYpKQlbt26Ft7c3gw4REdEreqVJXnr37o3evXtrui2Vyvbt23HlyhUEBgaWuu/atWtRv359eHt7F/n8ywOTJRIJNm7cCHNzcwwcOLDQ/tOnT8fy5cuRkZGBtm3b4uDBg2V7MURERG8wtXt2AgMDcfHixULbL168iKCgII00qqLFxMRg4sSJ2Lp1K4yMjErcNzMzE3/88UeJPTTBMcm4HZcKQz0d9G9WEwCwbt06DB8+vMjjT506FcHBwTh69Ch0dXUxYsQICCHK9qKIiIjeUGqHHV9fX8TExBTa/vDhQ/j6+mqkURXt8uXLSEhIQPPmzaGnpwc9PT2cPn0aS5cuhZ6eHvLz8xX7/vnnn8jIyMCIESOKPd72ZzMmv9PEAVJjffz7778ICwvDxx9/XOT+VlZWcHd3x9tvv43t27fj77//xoULFzT7IomIiN4Qal/GunXrFpo3b15oe7NmzXDr1i2NNKqide3aFdevX1fa9tFHH8HDwwPTp0+Hrq6uYvvatWvRt29fWFtbF3ms1KxcHLgaCwB479min2vXrkWLFi3g6elZalueL8+RnZ39Sq+FiIjoTad22DE0NER8fLzSHUQAEBsbCz097VjnydzcHI0aNVLaZmpqiho1aihtj4iIwJkzZ/D3338XeRwPDw/0GDUZmbmOqGtjhhYu1ZGSkoJdu3YVuWjqxYsXERgYiPbt26N69eqIjIzE7NmzUbt2bXh5eWn2RRIREb0h1L6M5ePjg5kzZ0Imkym2JScnY9asWXj77bc12rjKbt26dXB0dISPj0+Rz4eFheFw8F0AwLBnA5O3b98OIQTee++9QvubmJhgz5496Nq1K+rVq4cxY8agSZMmOH36NAwNDcv1tRAREWkriVBz5OvDhw/RsWNHPHnyBM2aNQMAhISEwNbWFseOHYOTk1O5NLQ8paSkQCqVQiaTwcLCQiPH3BEYjRl7ruP5b3fOOw0wur2bRo5NREREqp+/1Q47AJCeno6tW7fi6tWrMDY2RpMmTfDee+9V2blgNB12YmWZaLfwBOQv/GZ1JRKcndEZ9lLjMh+fiIiIVD9/v9IgG1NTU4wbN+6VG6ftohLTlYIOAOQLgXuJGQw7REREr9krjyi+desWoqOjkZOTo7S9b9++ZW5UVedmZQodCQr17LhamVRco4iIiN5Qaoedu3fvYsCAAbh+/TokEolisjuJRAIASnPQvKnspcbwG9gYs/bcQL4Q0JVI8MPARuzVISIiqgBqh52JEyfCzc0N/v7+cHNzw6VLl/DkyRN8+eWX+Omnn8qjjVXS0FbO6OhujXuJGXC1MmHQISIiqiBqh52AgACcOHECVlZW0NHRgY6ODtq3bw8/Pz988cUXCA4OLo92Vkn2UmOGHCIiogqm9jw7+fn5MDc3B1CwrMGjR48AAC4uLggLC9Ns64iIiIjKSO2enUaNGuHq1atwc3NDmzZtsHjxYhgYGGD16tWFZlUmIiIiqmhqh51vvvkG6enpAIDvvvsO77zzDjp06IAaNWpgx44dGm8gERERUVm80qSCL0tKSkL16tUVd2RVNeUxgzIRERGVr3KdVPBllpaWmjgMERERkcapPUCZiIiIqCph2CEiIiKtxrBDREREWk3tsHPmzBnk5eUV2p6Xl4czZ85opFFEREREmqJ22OncuTOSkpIKbZfJZOjcubNGGkVERESkKWqHHSFEkbeYP3nyBKamphppFBEREZGmqHzr+cCBAwEUrG4+atQoGBoaKp7Lz8/HtWvX4O3trfkWEhEREZWBymFHKpUCKOjZMTc3h7HxfwtcGhgYoG3bthg7dqzmW0hERERUBiqHnfXr1wMAXF1d8dVXX/GSFREREVUJao/ZmTZtmtKYnfv37+PXX3/F0aNHNdowIiIiIk1QO+z069cPmzZtAgAkJyejdevW+Pnnn9GvXz/8/vvvGm8gERERUVmoHXauXLmCDh06AAD+/PNP2NnZ4f79+9i0aROWLl2q8QYSERERlYXaYScjIwPm5uYAgKNHj2LgwIHQ0dFB27Ztcf/+fY03kIiIiKgs1A47derUwb59+xATE4MjR47Ax8cHAJCQkFDi8upEREREFUHtsDNnzhx89dVXcHV1RevWreHl5QWgoJenWbNmGm8gERERUVmoHXbeffddREdHIygoCEeOHFFs79q1K3755ReNNu5lCxcuhEQiwaRJkxTbsrKy4Ovrixo1asDMzAyDBg1CfHx8ubaDiIiIqo5XWvXczs4O5ubmOHbsGDIzMwEArVq1goeHh0Yb96LAwECsWrUKTZo0Udo+efJkHDhwALt27cLp06fx6NEjxWzPRERERGqHnSdPnqBr165wd3dHr169EBsbCwAYM2YMvvzyS403EADS0tIwfPhwrFmzBtWrV1dsl8lkWLt2LZYsWYIuXbqgRYsWWL9+Pc6fP48LFy6US1uIiIioalE77EyePBn6+vqIjo6GiYmJYvvQoUNx+PBhjTbuOV9fX/Tu3RvdunVT2n758mXk5uYqbffw8ICzszMCAgKKPV52djZSUlKUHkRERKSdVF4u4rmjR4/iyJEjcHR0VNpet27dcrn1fPv27bhy5QoCAwMLPRcXFwcDAwNUq1ZNabutrS3i4uKKPaafnx++/fZbTTeViIiIKiG1e3bS09OVenSeS0pKUloJXRNiYmIwceJEbN26FUZGRho77syZMyGTyRSPmJgYjR2biIiIKhe1w06HDh0Uy0UAgEQigVwux+LFi9G5c2eNNu7y5ctISEhA8+bNoaenBz09PZw+fRpLly6Fnp4ebG1tkZOTg+TkZKVy8fHxsLOzK/a4hoaGsLCwUHoQERGRdlL7MtbixYvRtWtXBAUFIScnB9OmTcPNmzeRlJSEc+fOabRxXbt2xfXr15W2ffTRR/Dw8MD06dPh5OQEfX19+Pv7Y9CgQQCAsLAwREdHK+b/ISIiojeb2mGnUaNGuHPnDpYvXw5zc3OkpaVh4MCB8PX1hb29vUYbZ25ujkaNGiltMzU1RY0aNRTbx4wZgylTpsDS0hIWFhb4/PPP4eXlhbZt22q0LURERFQ1qR12oqOj4eTkhK+//rrI55ydnTXSMFX98ssv0NHRwaBBg5CdnY3u3bvjt99+e61tICIiospLIoQQ6hTQ1dVFbGwsbGxslLY/efIENjY2yM/P12gDX4eUlBRIpVLIZDKO3yEiIqoiVD1/qz1AWQgBiURSaHtaWppG75giIiIi0gSVL2NNmTIFQMHdV7Nnz1a6/Tw/Px8XL15E06ZNNd5AIiIiorJQOewEBwcDKOjZuX79OgwMDBTPGRgYwNPTE1999ZXmW0hERERUBiqHnZMnTwIouPX7f//7H8e2EBERUZWg9t1Y69evL492EBEREZULtQcoExEREVUlDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Sp12PHz80OrVq1gbm4OGxsb9O/fH2FhYUr7ZGVlwdfXFzVq1ICZmRkGDRqE+Pj4CmoxERERVTaVOuycPn0avr6+uHDhAo4dO4bc3Fz4+PggPT1dsc/kyZNx4MAB7Nq1C6dPn8ajR48wcODACmw1ERERVSYSIYSo6Eao6vHjx7CxscHp06fRsWNHyGQyWFtb448//sC7774LALh9+zbq16+PgIAAtG3bVqXjpqSkQCqVQiaTwcLCojxfAhEREWmIqufvSt2z8zKZTAYAsLS0BABcvnwZubm56Natm2IfDw8PODs7IyAgoNjjZGdnIyUlRelBRERE2qnKhB25XI5JkyahXbt2aNSoEQAgLi4OBgYGqFatmtK+tra2iIuLK/ZYfn5+kEqlioeTk1N5Np2IiIgqUJUJO76+vrhx4wa2b99e5mPNnDkTMplM8YiJidFAC4mIiKgy0qvoBqhiwoQJOHjwIM6cOQNHR0fFdjs7O+Tk5CA5OVmpdyc+Ph52dnbFHs/Q0BCGhobl2WQiIiKqJCp1z44QAhMmTMDevXtx4sQJuLm5KT3fokUL6Ovrw9/fX7EtLCwM0dHR8PLyet3NJSIiokqoUvfs+Pr64o8//sBff/0Fc3NzxTgcqVQKY2NjSKVSjBkzBlOmTIGlpSUsLCzw+eefw8vLS+U7sYiIiEi7VepbzyUSSZHb169fj1GjRgEomFTwyy+/xLZt25CdnY3u3bvjt99+K/Ey1st46zkREVHVo+r5u1KHndeFYYeIiKjq0cp5doiIiIjUxbBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIq2lN2FmxYgVcXV1hZGSENm3a4NKlSxXdJCIiIqoEtCLs7NixA1OmTMHcuXNx5coVeHp6onv37khISKjophEREVEF04qws2TJEowdOxYfffQRGjRogJUrV8LExATr1q2r6KYRERFRBdOr6AaUVU5ODi5fvoyZM2cqtuno6KBbt24ICAgoskx2djays7MVP8tkMgBASkpK+TaWiIiINOb5eVsIUeJ+VT7sJCYmIj8/H7a2tkrbbW1tcfv27SLL+Pn54dtvvy203cnJqVzaSEREROUnNTUVUqm02OerfNh5FTNnzsSUKVMUP8vlciQlJaFGjRqQSCQaqyclJQVOTk6IiYmBhYXFay3Pul9/3WUtz7rfrLrLWp51s+6qUr6sdZdECIHU1FQ4ODiUuF+VDztWVlbQ1dVFfHy80vb4+HjY2dkVWcbQ0BCGhoZK26pVq1ZeTYSFhUWZ/sBlKc+6X3/dZS3Put+sustannWz7qpSvqx1F6ekHp3nqvwAZQMDA7Ro0QL+/v6KbXK5HP7+/vDy8qrAlhEREVFlUOV7dgBgypQpGDlyJFq2bInWrVvj119/RXp6Oj766KOKbhoRERFVMK0IO0OHDsXjx48xZ84cxMXFoWnTpjh8+HChQcuvm6GhIebOnVvoktnrKM+6X3/dZS3Put+sustannWz7qpSvqx1a4JElHa/FhEREVEVVuXH7BARERGVhGGHiIiItBrDDhEREWk1hh0iIiLSagw75WjFihVwdXWFkZER2rRpg0uXLqlU7syZM+jTpw8cHBwgkUiwb98+lev08/NDq1atYG5uDhsbG/Tv3x9hYWEql//999/RpEkTxeRPXl5e+Oeff1Qu/6KFCxdCIpFg0qRJKu0/b948SCQSpYeHh4fK9T18+BAffPABatSoAWNjYzRu3BhBQUEqlXV1dS1Ut0Qiga+vb6ll8/PzMXv2bLi5ucHY2Bi1a9fG/PnzS12r5UWpqamYNGkSXFxcYGxsDG9vbwQGBhbar7T3hhACc+bMgb29PYyNjdGtWzeEh4erXH7Pnj3w8fFRzCYeEhKicv25ubmYPn06GjduDFNTUzg4OGDEiBF49OiRSnXPmzcPHh4eMDU1RfXq1dGtWzdcvHhR5ba/6NNPP4VEIsGvv/6qUtlRo0YV+tv36NFDrbpDQ0PRt29fSKVSmJqaolWrVoiOji61bFHvO4lEgh9//FGlutPS0jBhwgQ4OjrC2NhYsRiyKmXj4+MxatQoODg4wMTEBD169FC8X1T5LMnKyoKvry9q1KgBMzMzDBo0SDHBqyrlV69ejbfeegsWFhaQSCRITk5WPFda+aSkJHz++eeoV68ejI2N4ezsjC+++AIymUyluj/55BPUrl0bxsbGsLa2Rr9+/RRLDKnzOSqEQM+ePRW/X1XKvvXWW4X+3p9++qladQcEBKBLly4wNTWFhYUFOnbsiO+++67Esvfu3Sv2/bZr1y6V6o6Li8OHH34IOzs7mJqaonnz5ti9e7dKZSMjIzFgwABYW1vDwsICQ4YMKTQhcHlh2CknO3bswJQpUzB37lxcuXIFnp6e6N69OxISEkotm56eDk9PT6xYsULtek+fPg1fX19cuHABx44dQ25uLnx8fJCenq5SeUdHRyxcuBCXL19GUFAQunTpgn79+uHmzZtqtSMwMBCrVq1CkyZN1CrXsGFDxMbGKh5nz55VqdzTp0/Rrl076Ovr459//sGtW7fw888/o3r16iq398V6jx07BgAYPHhwqWUXLVqE33//HcuXL0doaCgWLVqExYsXY9myZSrVDQAff/wxjh07hs2bN+P69evw8fFBt27d8PDhQ6X9SntvLF68GEuXLsXKlStx8eJFmJqaonv37sjKylKpfHp6Otq3b49FixYV+3xx5TMyMnDlyhXMnj0bV65cwZ49exAWFoa+ffuqVLe7uzuWL1+O69ev4+zZs3B1dYWPjw8eP36sUvnn9u7diwsXLihNH69K2R49eii9B7Zt26Zy+cjISLRv3x4eHh44deoUrl27htmzZ8PIyKjUsi/WGRsbi3Xr1kEikWDQoEEq1T1lyhQcPnwYW7ZsQWhoKCZNmoQJEyZg//79JZYVQqB///64e/cu/vrrLwQHB8PFxQXdunVDenq6Sp8lkydPxoEDB7Br1y6cPn0ajx49wsCBAwGo9lmUkZGBHj16YNasWYXaV1r5R48e4dGjR/jpp59w48YNbNiwAYcPH8aYMWNUqrtFixZYv349QkNDceTIEQgh4OPjg/z8fLU+R3/99VelZYZULTt27Filv/vixYtVLh8QEIAePXrAx8cHly5dQmBgICZMmICzZ8+WWNbJyanQ++3bb7+FmZkZevbsqVLdI0aMQFhYGPbv34/r169j4MCBGDJkCA4cOFBi2fT0dPj4+EAikeDEiRM4d+4ccnJy0KdPH8jl8kK/V40TVC5at24tfH19FT/n5+cLBwcH4efnp9ZxAIi9e/e+cjsSEhIEAHH69OlXPkb16tXF//3f/6m8f2pqqqhbt644duyY6NSpk5g4caJK5ebOnSs8PT1fqY3Tp08X7du3f6WyRZk4caKoXbu2kMvlpe7bu3dvMXr0aKVtAwcOFMOHD1eproyMDKGrqysOHjyotL158+bi66+/Lrbcy+8NuVwu7OzsxI8//qjYlpycLAwNDcW2bdtKLf+iqKgoAUAEBwerXH9RLl26JACI+/fvq11WJpMJAOL48eMq1/3gwQNRs2ZNcePGDeHi4iJ++eUXlcqOHDlS9OvXr8T2lFR+6NCh4oMPPnilsi/r16+f6NKli8rlGzZsKL777julbUW9d14uGxYWJgCIGzduKLbl5+cLa2trsWbNmkJ1v/xZkpycLPT19cWuXbsU+4SGhgoAIiAgoNTyLzp58qQAIJ4+fVrk6y6t/HM7d+4UBgYGIjc3V+2yV69eFQBERESEynUHBweLmjVritjY2GL/tkWVVedzsajybdq0Ed98880rlX1Z06ZNC31+lVTe1NRUbNq0SWk/S0vLQu+Zl8seOXJE6OjoCJlMptgnOTlZSCQScezYsVJfS1mxZ6cc5OTk4PLly+jWrZtim46ODrp164aAgIDX2haZTAYAsLS0VLtsfn4+tm/fjvT0dLWW3vD19UXv3r2VXr+qwsPD4eDggFq1amH48OGIjo5Wqdz+/fvRsmVLDB48GDY2NmjWrBnWrFmjdv1Awd9vy5YtGD16tEoLw3p7e8Pf3x937twBAFy9ehVnz55Fz549VaovLy8P+fn5MDIyUtpubGyscs8WAERFRSEuLk7p9y6VStGmTZvX/r57TiaTQSKRqL32XE5ODlavXg2pVApPT0+Vysjlcnz44YeYOnUqGjZsqHZbT506BRsbG9SrVw+fffYZnjx5onK9hw4dgru7O7p37w4bGxu0adNGrcvPz8XHx+PQoUMYM2aMymW8vb2xf/9+PHz4EEIInDx5Enfu3IGPj0+J5bKzswFA6X2no6MDQ0PDIt93L3+WXL58Gbm5uUrvNw8PDzg7Oxf5fivLZ5Gq5WUyGSwsLKCnp1doe0ll09PTsX79eri5ucHJyUmlujMyMvD+++9jxYoVxa7DWFLdW7duhZWVFRo1aoSZM2ciIyNDpfIJCQm4ePEibGxs4O3tDVtbW3Tq1Emlv9nLLl++jJCQkGLfb0WV9/b2xo4dO5CUlAS5XI7t27cjKysLb731Volls7OzIZFIlCYWNDIygo6Ojlqfc6+s3OPUG+jhw4cCgDh//rzS9qlTp4rWrVurdSyUoWcnPz9f9O7dW7Rr106tcteuXROmpqZCV1dXSKVScejQIZXLbtu2TTRq1EhkZmYKIdT7BvP333+LnTt3iqtXr4rDhw8LLy8v4ezsLFJSUkota2hoKAwNDcXMmTPFlStXxKpVq4SRkZHYsGGDym1/bseOHUJXV1c8fPhQpf3z8/PF9OnThUQiEXp6ekIikYgffvhBrTq9vLxEp06dxMOHD0VeXp7YvHmz0NHREe7u7sWWefm9ce7cOQFAPHr0SGm/wYMHiyFDhpRa/kWa6NnJzMwUzZs3F++//77KZQ8cOCBMTU2FRCIRDg4O4tKlSyrX/cMPP4i3335b0RunTs/Otm3bxF9//SWuXbsm9u7dK+rXry9atWol8vLySi3//Fu9iYmJWLJkiQgODhZ+fn5CIpGIU6dOqfS6n1u0aJGoXr264t+PKm3PysoSI0aMEACEnp6eMDAwEBs3biy1bE5OjnB2dhaDBw8WSUlJIjs7WyxcuFAAED4+Pkpli/os2bp1qzAwMChUT6tWrcS0adNKLf+i0np2VPkse/z4sXB2dhazZs1SueyKFSuEqampACDq1atXZK9OceXHjRsnxowZo/i5qL9NcWVXrVolDh8+LK5duya2bNkiatasKQYMGKBS3QEBAQKAsLS0FOvWrRNXrlwRkyZNEgYGBuLOnTsqve7nPvvsM1G/fv0inyuu/NOnT4WPj4/i/WZhYSGOHDlSatmEhARhYWEhJk6cKNLT00VaWpqYMGGCACDGjRtXbBs1hWGnHFSWsPPpp58KFxcXERMTo1a57OxsER4eLoKCgsSMGTOElZWVuHnzZqnloqOjhY2Njbh69apimzph52VPnz4VFhYWKl1C09fXF15eXkrbPv/8c9G2bVu16/Xx8RHvvPOOyvtv27ZNODo6im3btolr166JTZs2CUtLS7WCVkREhOjYsaMAIHR1dUWrVq3E8OHDhYeHR7FlKnPYycnJEX369BHNmjVT6rYurWxaWpoIDw8XAQEBYvTo0cLV1VXEx8eXWj4oKEjY2toqBVR1ws7LIiMjVb6E9vzf+3vvvae0X58+fcSwYcPUqrtevXpiwoQJxT5fVPkff/xRuLu7i/3794urV6+KZcuWCTMzs0KXBooqGxQUJDw9PRXvu+7du4uePXuKHj16KO1X1GeJOmGntM+i0sJOaeVlMplo3bq16NGjh8jJyVG5bHJysrhz5444ffq06NOnj2jevHmhoFlU+b/++kvUqVNHpKamKrYV9ftV9TPY39+/yEtoRZV//u985syZSvs2btxYzJgxQ+W6MzIyhFQqFT/99FORzxdXfsKECaJ169bi+PHjIiQkRMybN09IpVJx7dq1UsseOXJE1KpVS0gkEqGrqys++OAD0bx5c/Hpp5+W8NvRDIadcpCdnS10dXULvfFHjBgh+vbtq9axXjXs+Pr6CkdHR3H37l21y76sa9euKiXvvXv3Kj40nz8AKN7YRX1LLk3Lli2V/gEXx9nZWelblhBC/Pbbb8LBwUGt+u7duyd0dHTEvn37VC7j6Ogoli9frrRt/vz5ol69emrVLUTByf55WBkyZIjo1atXsfu+/N54foJ+OaB07NhRfPHFF6WWf1FZwk5OTo7o37+/aNKkiUhMTFSr7Mvq1KlTZC/Zy+V/+eUXxfvsxfeejo6OcHFxeaW6raysxMqVK0utOzs7W+jp6Yn58+cr7Tdt2jTh7e2tct1nzpwRAERISEixbXq5fEZGhtDX1y803mvMmDGie/fuKtednJwsEhIShBAF4w3Hjx+veK64z5LnJ+iXA4qzs7NYsmRJqeVfVFLYKa18SkqK8PLyEl27di0UVNT5HMzOzhYmJibijz/+KLX8xIkTi32/derUSe2609LSBABx+PDhUuu+e/euACA2b96stH3IkCGKXlRV6t60aZPQ19dX/N1fVFz5iIiIQuO8hCg4R3zyyScq1/348WPF39rW1lYsXry42H01hWN2yoGBgQFatGgBf39/xTa5XA5/f3+1xr68CiEEJkyYgL179+LEiRNwc3Mr8zHlcrni+n5JunbtiuvXryMkJETxaNmyJYYPH46QkBDo6uqqVW9aWhoiIyNhb29f6r7t2rUrdJvjnTt34OLiolad69evh42NDXr37q1ymYyMDOjoKP9T0tXVfaU7DExNTWFvb4+nT5/iyJEj6Nevn8pl3dzcYGdnp/S+S0lJwcWLF8v9ffdcbm4uhgwZgvDwcBw/fhw1atQo0/FUfe99+OGHuHbtmtJ7z8HBAVOnTsWRI0fUrvfBgwd48uSJSu89AwMDtGrVqszvv7Vr16JFixYqj1ECCn7fubm5ZX7/SaVSWFtbIzw8HEFBQejXr1+pnyUtWrSAvr6+0vstLCwM0dHR8PLyKvNnkSrlU1JS4OPjAwMDA+zfv18x/uhV6hYFX/6RnZ1davkZM2YUer8BwC+//IJ169apXffz8vb29qXW7erqCgcHhyLfb87OzirXvXbtWvTt2xfW1tZKv4OSyj8fV1TU+y0/P1/luq2srFCtWjWcOHECCQkJijs2y1W5x6k31Pbt24WhoaHYsGGDuHXrlhg3bpyoVq2aiIuLK7VsamqqCA4OFsHBwQKAYhzAy3e0FOWzzz4TUqlUnDp1SsTGxioeGRkZKrV7xowZ4vTp0yIqKkpcu3ZNzJgxQ0gkEnH06FGVyr9MnctYX375pTh16pSIiooS586dE926dRNWVlZFfvN42aVLl4Senp74/vvvRXh4uNi6daswMTERW7ZsUbmt+fn5wtnZWUyfPl3lMkIU3MlTs2ZNcfDgQREVFSX27NkjrKysCnXll+Tw4cPin3/+EXfv3hVHjx4Vnp6eok2bNoW65Et7byxcuFBUq1ZNMf6kX79+ws3NTfGNt7TyT548EcHBweLQoUMCgNi+fbsIDg4WsbGxpZbPyckRffv2FY6OjiIkJETp/ZednV1i2bS0NDFz5kwREBAg7t27J4KCgsRHH30kDA0NFd8i1f138eJlrJLKpqamiq+++koEBASIqKgocfz4cdG8eXNRt25dkZWVpVLde/bsEfr6+mL16tUiPDxcLFu2TOjq6op///1XpXbLZDJhYmIifv/990Kvo7TynTp1Eg0bNhQnT54Ud+/eFevXrxdGRkbit99+K7Xszp07xcmTJ0VkZKTYt2+fcHFxEQMHDhRCqPZZ8umnnwpnZ2dx4sQJERQUJLy8vBSXk1UpHxsbK4KDg8WaNWsEAHHmzBkRHBwsnjx5Ump5mUwm2rRpIxo3biwiIiKU9vn0009LLBsZGSl++OEHERQUJO7fvy/OnTsn+vTpIywtLUV8fPwrfY7iWc9ZaWUjIiLEd999J4KCgkRUVJT466+/RK1atUTHjh1V/r398ssvwsLCQuzatUuEh4eLb775RhgZGYn3339fpXaHh4cLiUQi/vnnH6XtpdWdk5Mj6tSpIzp06CAuXrwoIiIixE8//SQkEono1atXqXWvW7dOBAQEiIiICLF582ZhaWkppkyZUuzvVJMYdsrRsmXLhLOzszAwMBCtW7cWFy5cUKnc8y7dlx8jR44stWxR5QCI9evXq1T36NGjhYuLizAwMBDW1taia9eurxx0hFAv7AwdOlTY29sLAwMDUbNmTTF06NAiBwwW58CBA6JRo0bC0NBQeHh4iNWrV6vV1iNHjggAIiwsTK1yKSkpYuLEicLZ2VkYGRmJWrVqia+//lpkZ2erfIwdO3aIWrVqCQMDA2FnZyd8fX1FcnJyof1Ke2/I5XIxe/ZsYWtrKwwNDUXXrl2VXk9p5devX1/k83Pnzi21/PNLX0U9Tp48WWLZzMxMMWDAAOHg4CAMDAyEvb296Nu3r9IAZXX/XbwYdkoqm5GRIXx8fIS1tbXQ19cXLi4uYuzYsUpfTFSpe+3ataJOnTrCyMhIeHp6Ki6FqlJ21apVwtjY+JX+5rGxsWLUqFHCwcFBGBkZiXr16omff/5ZyOXyUsv+73//E46OjkJfX184OzuLb775RvG+VeWzJDMzU4wfP15Ur15dmJiYiAEDBiiCsSrl586dW+w+pZUv7rWV9Hhe9uHDh6Jnz57CxsZG6OvrC0dHR/H++++L27dvq9z2lz0PO6WVjY6OFh07dhSWlpbC0NBQ1KlTR0ydOlUxtk3Vuv38/ISjo6MwMTERXl5e4t9//1W57MyZM4WTk5PIz88v9BpKK3/nzh0xcOBAYWNjI0xMTESTJk3Epk2bVCo7ffp0YWtrK/T19UXdunUV79PXQfLsBRIRERFpJY7ZISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQEb3k1KlTkEgkSE5OruimEJEGMOwQERGRVmPYISIiIq3GsENElY5cLoefnx/c3NxgbGwMT09P/PnnnwD+u8R06NAhNGnSBEZGRmjbti1u3LihdIzdu3ejYcOGMDQ0hKurK37++Wel57OzszF9+nQ4OTnB0NAQderUwdq1a5X2uXz5Mlq2bAkTExN4e3sXWmmaiKoGhh0iqnT8/PywadMmrFy5Ejdv3sTkyZPxwQcf4PTp04p9pk6dip9//hmBgYGwtrZGnz59kJubC6AgpAwZMgTDhg3D9evXMW/ePMyePRsbNmxQlB8xYgS2bduGpUuXIjQ0FKtWrYKZmZlSO77++mv8/PPPCAoKgp6eHkaPHv1aXj8RaRYXAiWiSiU7OxuWlpY4fvw4vLy8FNs//vhjZGRkYNy4cejcuTO2b9+OoUOHAgCSkpLg6OiIDRs2YMiQIRg+fDgeP36Mo0ePKspPmzYNhw4dws2bN3Hnzh3Uq1cPx44dQ7du3Qq14dSpU+jcuTOOHz+Orl27AgD+/vtv9O7dG5mZmTAyMirn3wIRaRJ7doioUomIiEBGRgbefvttmJmZKR6bNm1CZGSkYr8Xg5ClpSXq1auH0NBQAEBoaCjatWundNx27dohPDwc+fn5CAkJga6uLjp16lRiW5o0aaL4f3t7ewBAQkJCmV8jEb1eehXdACKiF6WlpQEADh06hJo1ayo9Z2hoqBR4XpWxsbFK++nr6yv+XyKRACgYT0REVQt7doioUmnQoAEMDQ0RHR2NOnXqKD2cnJwU+124cEHx/0+fPsWdO3dQv359AED9+vVx7tw5peOeO3cO7u7u0NXVRePGjSGXy5XGABGR9mLPDhFVKubm5vjqq68wefJkyOVytG/fHjKZDOfOnYOFhQVcXFwAAN999x1q1KgBW1tbfP3117CyskL//v0BAF9++SVatWqF+fPnY+jQoQgICMDy5cvx22+/AQBcXV0xcuRIjB49GkuXLoWnpyfu37+PhIQEDBkypKJeOhGVE4YdIqp05s+fD2tra/j5+eHu3buoVq0amjdvjlmzZikuIy1cuBATJ05EeHg4mjZtigMHDsDAwAAA0Lx5c+zcuRNz5szB/PnzYW9vj++++w6jRo1S1PH7779j1qxZGD9+PJ48eQJnZ2fMmjWrIl4uEZUz3o1FRFXK8zulnj59imrVqlV0c4ioCuCYHSIiItJqDDtERESk1XgZi4iIiLQae3aIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIq/0/aNuF5S+4oysAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plt.plot(np.arange(len(epochs_x)), epochs_acc, marker = '.')\n", + "plt.xlabel('epoch')\n", + "plt.ylabel('test accuracy')\n", + "plt.ylim(0, 100)\n", + "plt.xticks(np.arange(len(epochs_x)))\n", + "for i, txt in enumerate(epochs_acc):\n", + " if i%5 == 0:\n", + " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", + " else:\n", + " pass\n", + "plt.show()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "speck-rescnn", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 03ef85c41aee240cffb5b34092df483f6d21449b Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Mon, 29 Apr 2024 20:48:29 +0200 Subject: [PATCH 058/379] baseline exp. set B --- .../exp_set_B/baseline-SCNN-example_3.ipynb | 198 ++++++++++-------- 1 file changed, 105 insertions(+), 93 deletions(-) diff --git a/tests/test_nonsequential/exp_set_B/baseline-SCNN-example_3.ipynb b/tests/test_nonsequential/exp_set_B/baseline-SCNN-example_3.ipynb index 0020ec07..a29b616b 100644 --- a/tests/test_nonsequential/exp_set_B/baseline-SCNN-example_3.ipynb +++ b/tests/test_nonsequential/exp_set_B/baseline-SCNN-example_3.ipynb @@ -61,7 +61,7 @@ "metadata": {}, "outputs": [], "source": [ - "root_dir = \"./DVSGESTURE\"\n", + "root_dir = \"../DVSGESTURE\"\n", "_ = DVSGesture(save_to=root_dir, train=True)\n", "_ = DVSGesture(save_to=root_dir, train=False)" ] @@ -125,7 +125,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "device: NVIDIA RTX A4000\n" + "device: NVIDIA GeForce RTX 3070 Ti\n" ] } ], @@ -359,7 +359,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "ebf2bea3d0124365abf1f2aa248ceab6", + "model_id": "a1826a7c572749a4ae111dcc8af0ec3b", "version_major": 2, "version_minor": 0 }, @@ -373,7 +373,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "faa557ac9bb54c02897848d65fb01cb7", + "model_id": "cdee887ba5ab4a009e24a14073b6a27c", "version_major": 2, "version_minor": 0 }, @@ -388,13 +388,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch 0 accuracy: 37.121212121212125\n" + "Epoch 0 accuracy: 48.484848484848484\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "5740aa705e294dff96add0963c733568", + "model_id": "36e2efe00a574f0cbe38b359707f7156", "version_major": 2, "version_minor": 0 }, @@ -408,7 +408,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "c3e026123d464e838af0e981152dc61c", + "model_id": "77003c0df2924f0b8eb3a48471a9107d", "version_major": 2, "version_minor": 0 }, @@ -423,13 +423,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch 1 accuracy: 46.21212121212121\n" + "Epoch 1 accuracy: 53.78787878787878\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "2757957b8f2d4a53a3cee8aad91343d9", + "model_id": "361bb9ebe2224ff58cda47eadc3478b1", "version_major": 2, "version_minor": 0 }, @@ -443,7 +443,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "047151a26d8346dbb0579972f5fcb324", + "model_id": "3ffd1cd17aee4c64b8406e168e099165", "version_major": 2, "version_minor": 0 }, @@ -458,13 +458,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch 2 accuracy: 60.984848484848484\n" + "Epoch 2 accuracy: 58.333333333333336\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "4a0036cab9ee4aeb8f6d3c1b1985070b", + "model_id": "1fa772f587b84a009b20a6bcc0c92f24", "version_major": 2, "version_minor": 0 }, @@ -478,7 +478,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "d207154f438f4215bcfd67717e72e838", + "model_id": "993c1cc4960d435d942ab04b71336c11", "version_major": 2, "version_minor": 0 }, @@ -493,13 +493,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch 3 accuracy: 62.121212121212125\n" + "Epoch 3 accuracy: 59.46969696969697\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "c0dadbe693ce41a4aa06d09d9f521ef5", + "model_id": "c2850afc8d734b1895bf0e4f066be4fb", "version_major": 2, "version_minor": 0 }, @@ -513,7 +513,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "b47c85e708354f9399bb016932d0a4c6", + "model_id": "288e5f0e2b6c40a9a700838e311a780d", "version_major": 2, "version_minor": 0 }, @@ -528,13 +528,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch 4 accuracy: 65.15151515151516\n" + "Epoch 4 accuracy: 62.121212121212125\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "7e23105d5c194961b4858db05d466e41", + "model_id": "6ccfe28d0aaa4f98b5dea71fec5886ef", "version_major": 2, "version_minor": 0 }, @@ -548,7 +548,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "6b608aea788a46a590ea1e3440d0f005", + "model_id": "7f2ab5fd076d48639ba86dc2288581c9", "version_major": 2, "version_minor": 0 }, @@ -563,13 +563,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch 5 accuracy: 60.22727272727273\n" + "Epoch 5 accuracy: 68.18181818181817\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "17f668e20bf447c5b94099647a7cedc7", + "model_id": "dd6fe6838fbb48a79b3a0cca0b197c8a", "version_major": 2, "version_minor": 0 }, @@ -583,7 +583,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "534a4922584646c2861428df60daaf20", + "model_id": "320cb632552841cbb18ca5977c14eb9e", "version_major": 2, "version_minor": 0 }, @@ -598,13 +598,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch 6 accuracy: 66.66666666666666\n" + "Epoch 6 accuracy: 75.0\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "4764a80723ea40209e9e7158117c1898", + "model_id": "ab8f0d2b47814428a585d6846bfb12ba", "version_major": 2, "version_minor": 0 }, @@ -618,7 +618,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "65300a14e6d94459adeb770b6b268a91", + "model_id": "df5792c4cf8a4fd68fbf0b6be743023b", "version_major": 2, "version_minor": 0 }, @@ -633,13 +633,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch 7 accuracy: 67.8030303030303\n" + "Epoch 7 accuracy: 70.07575757575758\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "fc6fb5195c144daca12c96c24fd77655", + "model_id": "296a057dd86a4a56a78eda53475a9883", "version_major": 2, "version_minor": 0 }, @@ -653,7 +653,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "9eeb3dfb97074188a4492e637c7da642", + "model_id": "4fe836cf9b804de39012115be8808a2c", "version_major": 2, "version_minor": 0 }, @@ -668,13 +668,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch 8 accuracy: 67.8030303030303\n" + "Epoch 8 accuracy: 68.56060606060606\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "1976efa33e2c4329b8078862fe018856", + "model_id": "b5c290923e544deb9563c2a6e726a0e0", "version_major": 2, "version_minor": 0 }, @@ -688,7 +688,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "2ec118840b6a41678ceae4c8a2c3e640", + "model_id": "f21c90349862487dbc0466e08b75aa92", "version_major": 2, "version_minor": 0 }, @@ -703,13 +703,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch 9 accuracy: 69.31818181818183\n" + "Epoch 9 accuracy: 67.42424242424242\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "3d14dd584ec84a7d8f766f4e7347efea", + "model_id": "d6e1ac78091c458aa38768111432278c", "version_major": 2, "version_minor": 0 }, @@ -723,7 +723,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "781187e60a764025bf9a95672440b191", + "model_id": "8fb11a7b141b4ad3a3853cd3fb34c6a4", "version_major": 2, "version_minor": 0 }, @@ -744,7 +744,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "cd9ffb5c23d741b5a9ee1b0f77a3fb04", + "model_id": "1a294d1301d84784b8011b55397336b1", "version_major": 2, "version_minor": 0 }, @@ -758,7 +758,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "fb5a4f0aae964636a26819e57bd9acfb", + "model_id": "3479f807767c4544a92b048b4a68a487", "version_major": 2, "version_minor": 0 }, @@ -773,13 +773,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch 11 accuracy: 72.34848484848484\n" + "Epoch 11 accuracy: 66.66666666666666\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "2cfd5abcc6e44794996933e0c6b7fe5e", + "model_id": "0a4575aed4964f13a38b4cb180bd2e3c", "version_major": 2, "version_minor": 0 }, @@ -793,7 +793,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "78343c4e0849457d895a3bf80b2c786c", + "model_id": "a2634eaf43e348f4b5d256233bf32c15", "version_major": 2, "version_minor": 0 }, @@ -808,13 +808,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch 12 accuracy: 71.96969696969697\n" + "Epoch 12 accuracy: 70.83333333333334\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "41dad080ca4441b580bb8e77077ce5ed", + "model_id": "93d267f9e9ff4ac985b5d2e258f093ee", "version_major": 2, "version_minor": 0 }, @@ -828,7 +828,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "9012039db9064b91b7c7930ee3302f83", + "model_id": "d88a59c50b8c43e8a6a2589366d82157", "version_major": 2, "version_minor": 0 }, @@ -843,13 +843,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch 13 accuracy: 73.48484848484848\n" + "Epoch 13 accuracy: 63.63636363636363\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "29477e853def44d9986e8f4b6ffe4379", + "model_id": "edb9245d578248a8a6b2ddc4b9b2b7b5", "version_major": 2, "version_minor": 0 }, @@ -863,7 +863,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "a97da5f79020463bac64bdf80db1fd2f", + "model_id": "fc1b320aba1d4e06828843295efd781e", "version_major": 2, "version_minor": 0 }, @@ -878,13 +878,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch 14 accuracy: 78.4090909090909\n" + "Epoch 14 accuracy: 67.42424242424242\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "b4d30264425840238b94a9d8f09979f6", + "model_id": "65d8b8f584d3453bbe9d48bb6ff1c64f", "version_major": 2, "version_minor": 0 }, @@ -898,7 +898,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "ebbe8618017c434f917b93c2e4549098", + "model_id": "3454887f6bee4c20bb1f279e0e512edd", "version_major": 2, "version_minor": 0 }, @@ -913,13 +913,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch 15 accuracy: 69.6969696969697\n" + "Epoch 15 accuracy: 70.83333333333334\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "ec03d092029843719fbb3c216a9f0433", + "model_id": "6f50d226e3694abd862b3017e31c3970", "version_major": 2, "version_minor": 0 }, @@ -933,7 +933,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "c76dde5446bc4cc9a8d4e4d876904725", + "model_id": "6a26275ff48c40299546ec985466fc00", "version_major": 2, "version_minor": 0 }, @@ -948,13 +948,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch 16 accuracy: 72.34848484848484\n" + "Epoch 16 accuracy: 71.21212121212122\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "2be56e494d264b52b3241e504488438e", + "model_id": "38364d6fe1fe4c858ecf5b03abb3d56a", "version_major": 2, "version_minor": 0 }, @@ -968,7 +968,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "4f165ce5dca94f699e46b4ebaa200100", + "model_id": "e585604744044e6b8879483dac3001cb", "version_major": 2, "version_minor": 0 }, @@ -983,13 +983,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch 17 accuracy: 69.6969696969697\n" + "Epoch 17 accuracy: 70.07575757575758\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "4ce0b39beed8472a80af3b06b44be5b7", + "model_id": "86a946d91f13438aab026de9bb19fd13", "version_major": 2, "version_minor": 0 }, @@ -1003,7 +1003,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "823f2347cedf4431a73e1ba87b6df989", + "model_id": "17f1fb63c0fc4269a977f83c824205ed", "version_major": 2, "version_minor": 0 }, @@ -1018,13 +1018,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch 18 accuracy: 70.83333333333334\n" + "Epoch 18 accuracy: 74.24242424242425\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "31689f1975be4d4dbc8efdc99f19294d", + "model_id": "f5f27fd1ba324fe5a406e32e0547e2af", "version_major": 2, "version_minor": 0 }, @@ -1038,7 +1038,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "b66a37ac35424593a7b060e637070501", + "model_id": "0bb12d4285554f6b8bef98ddc848af27", "version_major": 2, "version_minor": 0 }, @@ -1053,13 +1053,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch 19 accuracy: 70.45454545454545\n" + "Epoch 19 accuracy: 72.34848484848484\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "f852bd8d1eed4b1ea2f35bd1e66fa85d", + "model_id": "c52103acdce841d287b35ea8deffd3c0", "version_major": 2, "version_minor": 0 }, @@ -1073,7 +1073,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "5acf774d4e044a2d888080d0cf8c6adb", + "model_id": "8f2909c0b245407ebecfcc6f030c347c", "version_major": 2, "version_minor": 0 }, @@ -1088,13 +1088,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch 20 accuracy: 66.66666666666666\n" + "Epoch 20 accuracy: 81.06060606060606\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "3adec41b81f647c9bf7927a728bcc4b8", + "model_id": "0532d37161b44cc38aeebd8c8b72b8a4", "version_major": 2, "version_minor": 0 }, @@ -1108,7 +1108,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "40affe91b1db44209d798346242e0c82", + "model_id": "7534768be1924502a609f5023c85d7e8", "version_major": 2, "version_minor": 0 }, @@ -1123,13 +1123,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch 21 accuracy: 73.10606060606061\n" + "Epoch 21 accuracy: 76.89393939393939\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "7a87176de46542938ecf685ce38b5ec8", + "model_id": "08990c76ea8847a6bd7adab6a2ddc649", "version_major": 2, "version_minor": 0 }, @@ -1143,7 +1143,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "b3723d4e8b45427b87a104988f9ce685", + "model_id": "01283fd94c53469da2ba1ee2adc12e6c", "version_major": 2, "version_minor": 0 }, @@ -1158,13 +1158,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch 22 accuracy: 63.25757575757576\n" + "Epoch 22 accuracy: 75.37878787878788\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "e413c6706d394c8b9faea44040d9d8fe", + "model_id": "172cadfad00343b48fa4759a207e9499", "version_major": 2, "version_minor": 0 }, @@ -1178,7 +1178,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "722516f61e8d40ee8a78a1da33f468cc", + "model_id": "62d668881e974e44822deb36df4fce7e", "version_major": 2, "version_minor": 0 }, @@ -1193,13 +1193,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch 23 accuracy: 75.75757575757575\n" + "Epoch 23 accuracy: 77.65151515151516\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "67d030099a8f476286d79f6dcc5e50c3", + "model_id": "c77214254bd546bd8501f1af927e2eb6", "version_major": 2, "version_minor": 0 }, @@ -1213,7 +1213,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "05f797d62f8748759cce8b6fea1ffd25", + "model_id": "3cd73e5da1214c248f574beda47d5b6c", "version_major": 2, "version_minor": 0 }, @@ -1228,13 +1228,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch 24 accuracy: 65.53030303030303\n" + "Epoch 24 accuracy: 78.03030303030303\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "343c1f7a8d664f7d9b84ca0740f7a51c", + "model_id": "c133852724254922a24292a764e8ccc9", "version_major": 2, "version_minor": 0 }, @@ -1248,7 +1248,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "a3428f78880a4db2bcee8583b423a6da", + "model_id": "28f22af338ce42e792f3b1f2aad23a7e", "version_major": 2, "version_minor": 0 }, @@ -1263,13 +1263,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch 25 accuracy: 73.48484848484848\n" + "Epoch 25 accuracy: 79.16666666666666\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "3a7734ea3ec948e0bc47c07cab2cedbc", + "model_id": "9fb7256cea1640cbb6914025c2a4c575", "version_major": 2, "version_minor": 0 }, @@ -1283,7 +1283,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "d38b5b0f556c49e5b0c535158a478088", + "model_id": "3c70c5f3740e4e14aa19a83ea5694802", "version_major": 2, "version_minor": 0 }, @@ -1298,13 +1298,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch 26 accuracy: 67.04545454545455\n" + "Epoch 26 accuracy: 73.48484848484848\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "bea88bf9082e4e998f576e3e2f33b280", + "model_id": "f3b18d45198f4ca3abd93be868db3817", "version_major": 2, "version_minor": 0 }, @@ -1318,7 +1318,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "2f525b86f7274f6988304508c84aad8d", + "model_id": "5af6f3d191244ff19966137a7f63e1bc", "version_major": 2, "version_minor": 0 }, @@ -1333,13 +1333,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch 27 accuracy: 72.72727272727273\n" + "Epoch 27 accuracy: 78.78787878787878\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "e1aa66c07cd6485298821c14f9855821", + "model_id": "654f368e72244385870716f1c1035a49", "version_major": 2, "version_minor": 0 }, @@ -1353,7 +1353,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "fc03d324544d4b74b1a87f187fe1bcb5", + "model_id": "158e123b4f6d4d7caefd81c0fbc0f998", "version_major": 2, "version_minor": 0 }, @@ -1368,13 +1368,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch 28 accuracy: 73.10606060606061\n" + "Epoch 28 accuracy: 75.75757575757575\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "dc255b0a112c400e83b66f9a0084fce3", + "model_id": "545bf0abdbe244a7b75a80a40e9bc373", "version_major": 2, "version_minor": 0 }, @@ -1388,7 +1388,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "ba4dd7bec7a54615b819d1eafcdb044b", + "model_id": "c0fef8d8ba3c430896da4a5a6d1be3d6", "version_major": 2, "version_minor": 0 }, @@ -1403,7 +1403,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch 29 accuracy: 73.10606060606061\n" + "Epoch 29 accuracy: 80.3030303030303\n" ] } ], @@ -1418,7 +1418,7 @@ "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAGwCAYAAABVdURTAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAAB5lUlEQVR4nO3dd3zN1/8H8Ne9NzsyZe8QEiNiRcTeoX5mi6rWqLZfLS3VqlGlOzqUb8uXVo2iRYfVUhojCDGC2CJkGdmRvXPP74/IrSsh98a9uRmv5+NxHw/53HM+n/cnue593zMlQggBIiIiogZCqusAiIiIiDSJyQ0RERE1KExuiIiIqEFhckNEREQNCpMbIiIialCY3BAREVGDwuSGiIiIGhQ9XQdQ2+RyOe7duwczMzNIJBJdh0NEREQqEEIgJycHTk5OkEqf3DbT6JKbe/fuwdXVVddhEBERUQ3cvn0bLi4uTyzT6JIbMzMzAOW/HHNzcx1HQ0RERKrIzs6Gq6ur4nP8SRpdclPRFWVubs7khoiIqJ5RZUgJBxQTERFRg8LkhoiIiBoUJjdERETUoDC5ISIiogaFyQ0RERE1KExuiIiIqEFhckNEREQNCpMbIiIialCY3BAREVGDwuRGgxKzCnDiVhoSswp0HQoREVGj1ei2X9CWbWcSMH/7JcgFIJUAwaN9Mc7fTddhERERNTpsudGAxKwCzHuQ2ACAXAALtl9mCw4REZEOMLnRgNi0PAihfKxMCMSl5esmICIiokaMyY0GeNqYQvrIJqUyiQQeNia6CYiIiKgRY3KjAY4Wxgge7av4WSIBPh/dFo4WxjqMioiIqHFicqMh4/zdML1PcwBADy8bDiYmIiLSESY3GjSwjQMA4OKdLMjloprSREREpA1MbjSorZM5TA1kyCoowbWkbF2HQ0RE1CgxudEgPZkU/p7WAIBTMRk6joaIiKhxYnKjYV2bNQUAnIxJ13EkREREjROTGw0LqGi5ic3guBsiIiIdYHKjYW2dLRTjbq4n5eg6HCIiokaHyY2G6cuk6OxR0XrDrikiIqLaxuRGCzjuhoiISHeY3GhBQDOOuyEiItIVJjda4OtsARMDGTLzSxCVzHE3REREtYnJjRYojbth1xQREVGtYnKjJV0fdE2d5GJ+REREtYrJjZYEeJYPKj4Vm85xN0RERLWIyY2WtHOxgLG+DPfzS3AjheNuiIiIaguTGy0pH3djBQA4eYvjboiIiGoLkxstqljv5lQsx90QERHVFiY3WtSV690QERHVOiY3WuTrbAljfRky8ooRnZKr63CIiIgaBSY3WmSg99C4G653Q0REVCuY3GjZv+NumNwQERHVBiY3Whbg+e9ifkJw3A0REZG2MbnRsnYuljDSl3LcDRERUS1hcqNlBnpSdHavaL1h1xQREZG2MbmpBYop4dxnioiISOuY3NSCgAeDik/GpHPcDRERkZYxuakF7VwsYKQvRXpeMW5y3A0REZFWMbmpBYZ6MnRy53o3REREtUGnyU1wcDD8/f1hZmYGOzs7jBw5ElFRUdXW++233+Dj4wMjIyP4+vpi7969tRDt0+nq+aBrivtMERERaZVOk5sjR45g+vTpOHnyJEJCQlBSUoJBgwYhLy/vsXVOnDiB8ePHY+rUqTh//jxGjhyJkSNH4vLly7UYufoqxt2c4rgbIiIirZKIOvRJm5qaCjs7Oxw5cgS9evWqssy4ceOQl5eHv/76S3Gsa9euaN++PVavXl2pfFFREYqKihQ/Z2dnw9XVFVlZWTA3N9f8TTxGUWkZ2n34D4pK5Tgwuxe87Mxq7dpERET1XXZ2NiwsLFT6/K5TY26ysrIAANbW1o8tEx4ejgEDBigdCwoKQnh4eJXlg4ODYWFhoXi4urpqLmA1PDzuJpxTwomIiLSmziQ3crkcs2bNQvfu3dG2bdvHlktKSoK9vb3SMXt7eyQlJVVZfv78+cjKylI8bt++rdG41dH1oa4pIiIi0g49XQdQYfr06bh8+TLCwsI0el5DQ0MYGhpq9Jw19eg+UxKJRMcRERERNTx1ouVmxowZ+Ouvv3D48GG4uLg8sayDgwOSk5OVjiUnJ8PBwUGbIWqEn6slDPWkSMstwq3Uxw+aJiIioprTaXIjhMCMGTOwY8cOHDp0CJ6entXWCQwMxMGDB5WOhYSEIDAwUFthaoyRvgwd3bjeDRERkTbpNLmZPn06Nm/ejF9++QVmZmZISkpCUlISCgoKFGUmTpyI+fPnK36eOXMm9u3bh6VLl+L69ev48MMPERERgRkzZujiFtTW9aGtGIiIiEjzdJrcrFq1CllZWejTpw8cHR0Vj23btinKJCQkIDExUfFzt27d8Msvv+CHH36An58ffv/9d+zcufOJg5DrkoCKTTRjM7jeDRERkRbUqXVuaoM68+S1obCkDO0++gfFpXIcfKc3mts2qfUYiIiI6pt6u85NY1A+7sYSALumiIiItIHJjQ78O+6Gi/kRERFpGpMbHQjw5D5TRERE2sLkRgc6uFnCQE+KlJwixKZxvRsiIiJNYnKjA0b6MnRwtQTArikiIiJNY3KjI1zvhoiISDuY3OjIv+vdcNwNERGRJjG50ZGOblYwkEmRnF2EuPR8XYdDRETUYDC50REjfRnac70bIiIijWNyowVHjx7FsGHD4OTkBIlEgp07d1ZZ7uFxNz///DP8/PxgYmICR0dHvPzyy0hP/zfpWbNmDXr27AkrKytYWVlhwIABOH36dG3cDhERUb3C5EYL8vLy4Ofnh5UrVz6xXFfP8nE3Bw4fxcSJEzF16lRcuXIFv/32G06fPo1XX31VUTY0NBTjx4/H4cOHER4eDldXVwwaNAh3797V6r0QERHVN9xbSsskEgl27NiBkSNHVnquoLgMfh/9g9QTv8M87jDiY2MUz3333Xf44osvcOfOnSrPW1ZWBisrK6xYsQITJ07UVvhERER1AveWqieMDWRo72oJQ2cf3L1zB3v37oUQAsnJyfj999/xzDPPPLZufn4+SkpKYG1tXYsRExER1X1MbnSsazNrGLm0RtD0TzFu3DgYGBjAwcEBFhYWT+zWmjt3LpycnDBgwIBajJaIiKjuY3KjYwHNmqI4LQEh677EBx98gLNnz2Lfvn2Ii4vDtGnTqqyzZMkSbN26FTt27ICRkVEtR0xERFS36ek6gMauo5sVck/9Bn0nH4ybOh3uTU3Rrl07mJqaomfPnvj000/h6OioKP/1119jyZIlOHDgANq1a6fDyImIiOomttzomLGBDOZ6ckAiVVrvRiaTAYDS6sVffvklPvnkE+zbtw+dO3eu9ViJiIjqAyY3WpCbm4vIyEhERkYCAGJjYxEZGYmEhAQAwPz585VmOPXoH4T8Gyfw/erViImJwfHjx/HWW2+hS5cucHJyAgB88cUX+OCDD7Bu3Tp4eHggKSkJSUlJyM3NrfX7IyIiqsuY3GhBREQEOnTogA4dOgAAZs+ejQ4dOmDRokUAgMTEREWiAwAz/vMqrPq9gvA/f0abtm0xZswYeHt7Y/v27Yoyq1atQnFxMZ577jk4OjoqHl9//XXt3hwREVEdx3Vu6oBN4XH4YNcVAIBUAgSP9sU4fzcdR0VERFR3cJ2beiQxqwCLd19R/CwXwILtl5GYVaDDqIiIiOovJjc6FpuWB/kjbWdlQiAujTuFExER1QSTGx3ztDGFVKJ8TCoBPGxMdBMQERFRPcfkRsccLYwRPNoXMsm/GY67tSkcLYx1GBUREVH9xeSmDhjn74aweX2x8oWO0JMCsel5iLydqeuwiIiI6iUmN3WEo4UxhrZzxIj2LgCANcdiqqlBREREVWFyU8e80tMTAPD3pUTczuCgYiIiInUxualjWjmao2cLG8gFsDYsVtfhEBER1TtMbuqg13o1AwD8GnEbWfklOo6GiIiofmFyUwf18LKBj4MZ8ovL8PPpeF2HQ0REVK8wuamDJBIJXu1Z3nqz4XgcikvlOo6IiIio/mByU0cN83OCvbkhUnKKsPvCPV2HQ0REVG8wuamjDPSkmNytfObUj8di0Mj2NyUiIqoxJjd12AsBbjA1kOF6Ug6ORafpOhwiIqJ6gclNHWZhrI+x/q4AuKgfERGRqpjc1HEvd/eEVAIci07D1XvZug6HiIiozmNyU8e5WptgiK8jAODHMLbeEBERVYfJTT3w2oNp4bsj7yEpq1DH0RAREdVtTG7qAT9XS3TxtEapXGDDiThdh0NERFSnMbmpJyoW9fv5VDxyi0p1HA0REVHdxeSmnujvY4dmNqbIKSzFtjO3dR0OERFRncXkpp6QSiWY2rN8Ub91YbEoLeOWDERERFVhclOPPNvRBdamBribWYC/LyfpOhwiIqI6iclNPWKkL8PEQHcA5Yv6cUsGIiKiypjc1DMvdXWHoZ4UF+9k4XRshq7DISIiqnOY3NQzTZsY4tlOLgC4JQMREVFVmNzUQ1N7lA8sPnAtBbdSc3UcDRERUd3C5KYeam7bBANa2QMAfjwWq+NoiIiI6hYmN/XUa73KF/X749wdpOUW6TgaIiKiuoPJTT3l72EFPxcLFJfKsSk8XtfhEBER1RlMbuopiUSCVx+03mw6GY/CkjIdR0RERFQ3MLmpxwa3cYCLlTEy8orxx7k7ug6HiIioTmByU4/pyaR4uXv5zKnVobdwPDoNiVkFOo6KiIhIt5jc1HNj/V1hpCfF7fsFmLD2FLovOYRtZxJ0HRYREZHOMLmp53IKS1BU+u8mmnIBLNh+mS04RETUaDG5qedi0/Lw6A5TZUIgLi1fJ/EQERHpGpObes7TxhRSifIxmUQCDxsT3QRERESkY0xu6jlHC2MEj/ZFRX4jAfD56LZwtDDWZVhEREQ6w+SmARjn74YFz/gAAHydLTDO303HEREREekOk5sGYmBrBwDA9aQcLuhHRESNGpObBsK9qQlsmhiguEyOy3ezdB0OERGRzjC5aSAkEgk6u1sDAM7E3ddxNERERLrD5KYB6exhBQCIiMvQcSRERES6w+SmAfH3KG+5OZtwH3L5o6vfEBERNQ5MbhqQ1k7mMNaXITO/BLdSc3UdDhERkU4wuWlA9GVStHe1BMBxN0RE1HgxuWlg/DnuhoiIGjkmNw1M5wfjbs7EM7khIqLGiclNA9PBzRJSCXA7owDJ2YW6DoeIiKjW6TS5OXr0KIYNGwYnJydIJBLs3LnzieVDQ0MhkUgqPZKSkmon4HrAzEgfPg7mAIAIjrshIqJGSKfJTV5eHvz8/LBy5Uq16kVFRSExMVHxsLOz01KE9VPFuJszHHdDRESNkJ4uLz5kyBAMGTJE7Xp2dnawtLRUqWxRURGKiooUP2dnZ6t9vfqms4c1fgqPRwTH3RARUSNUL8fctG/fHo6Ojhg4cCCOHz/+xLLBwcGwsLBQPFxdXWspSt2pWKn46r1s5BaV6jgaIiKi2lWvkhtHR0esXr0af/zxB/744w+4urqiT58+OHfu3GPrzJ8/H1lZWYrH7du3azFi3XC0MIazpTHkAohMyNR1OERERLVKp91S6vL29oa3t7fi527duuHWrVtYtmwZNm3aVGUdQ0NDGBoa1laIdYa/hxXuRhbgTFwGerSw0XU4REREtaZetdxUpUuXLrh586auw6hzKta74bgbIiJqbOp9chMZGQlHR0ddh1HnVGyieT4hE6Vlch1HQ0REVHt02i2Vm5ur1OoSGxuLyMhIWFtbw83NDfPnz8fdu3exceNGAMDy5cvh6emJNm3aoLCwED/++CMOHTqEf/75R1e3UGe1sGsCcyM9ZBeW4lpiDnxdLHQdEhERUa3QaXITERGBvn37Kn6ePXs2AGDSpEnYsGEDEhMTkZCQoHi+uLgY77zzDu7evQsTExO0a9cOBw4cUDoHlZNKJejkboXDUak4E5fB5IaIiBoNiRBC6DqI2pSdnQ0LCwtkZWXB3Nxc1+Fo1crDN/HV/ig84+uA/03opOtwiIiIakydz+96P+aGHq9i3M2ZuPtoZDksERE1YkxuGrB2LhYwkEmRmlOE2xkFug6HiIioVjC5acCM9GVo61zedMd9poiIqLFgctPA+XO9GyIiamSY3DRwnR8ad0NERNQYMLlp4Dq5l2+ieTMlF/fzinUcDRERkfapndwUFBQgPz9f8XN8fDyWL1/OhfTqKGtTAzS3NQUAnI1n6w0RETV8aic3I0aMUKwYnJmZiYCAACxduhQjRozAqlWrNB4gPT3FlHCOuyEiokZA7eTm3Llz6NmzJwDg999/h729PeLj47Fx40Z8++23Gg+Qnp5iE02OuyEiokZA7eQmPz8fZmZmAIB//vkHo0ePhlQqRdeuXREfH6/xAOnp+XuUj7u5dCcLhSVlOo6GiIhIu9RObry8vLBz507cvn0b+/fvx6BBgwAAKSkpDX47g/rKzdoENk0MUVwmx6W7WboOh4iISKvUTm4WLVqEd999Fx4eHggICEBgYCCA8lacDh06aDxAenoSiUTResPF/IiIqKFTe1fw5557Dj169EBiYiL8/PwUx/v3749Ro0ZpNDjSnM4e1vj7chLH3RARUYOndnIDAA4ODnBwcABQvkvnoUOH4O3tDR8fH40GR5pT0XITEZcBuVxAKpXoOCIiIiLtULtbauzYsVixYgWA8jVvOnfujLFjx6Jdu3b4448/NB4gaUYrR3MY68uQXViK6JRcXYdDRESkNWonN0ePHlVMBd+xYweEEMjMzMS3336LTz/9VOMBkmboy6To4GYJgPtMERFRw6Z2cpOVlQVr6/J1U/bt24dnn30WJiYmGDp0KKKjozUeIGkO17shIqLGQO3kxtXVFeHh4cjLy8O+ffsUU8Hv378PIyMjjQdImsMZU0RE1BiondzMmjULEyZMgIuLC5ycnNCnTx8A5d1Vvr6+mo6PNKiDmxWkEuDO/QIkZRXqOhwiIiKtUDu5eeONNxAeHo5169YhLCwMUmn5KZo1a8YxN3VcE0M9tHIsX2iR426IiKihqtFU8M6dO6Nz584QQkAIAYlEgqFDh2o6NtICfw9rXLmXjYi4+/i/dk66DoeIiEjj1G65AYCNGzfC19cXxsbGMDY2Rrt27bBp0yZNx0Za0JnjboiIqIFTu+Xmm2++wQcffIAZM2age/fuAICwsDBMmzYNaWlpePvttzUeJGlOZ/fyGVPXErORU1gCMyN9HUdERESkWWonN9999x1WrVqFiRMnKo4NHz4cbdq0wYcffsjkpo5zsDCCi5Ux7twvwPmETPRqaavrkIiIiDRK7W6pxMREdOvWrdLxbt26ITExUSNBkXb5V6x3E8/1boiIqOFRO7nx8vLCr7/+Wun4tm3b0KJFC40ERdrV+aF9poiIiBoatbulPvroI4wbNw5Hjx5VjLk5fvw4Dh48WGXSQ3VPRcvN+YRMlJTJoS+r0bhyIiKiOkntT7Vnn30Wp06dgo2NDXbu3ImdO3fCxsYGp0+fxqhRo7QRI2mYl20TmBvpoaCkDFfvZes6HCIiIo2q0To3nTp1wubNmzUdC9USqVSCzh7WOHQ9BRHx9+HnaqnrkIiIiDRGpeQmO1v1b/fm5uY1DoZqT2cPq/LkJi4DU3t46jocIiIijVGpW8rS0hJWVlZPfFSUId07evQohg0bBicnJ0gkEuzcubNSmYpxN2fi7uPevXt44YUX0LJlS0ilUsyaNatS+ZKSEnz88cdo3rw5jIyM4Ofnh3379mn5ToiIiNSnUsvN4cOHtR0HaVBeXh78/Pzw8ssvY/To0VWW8XW2gIFMirTcIsQkZcLW1hYLFy7EsmXLqiy/cOFCbN68GWvWrIGPjw/279+PUaNG4cSJE+jQoYM2b4eIiEgtKiU3vXv31nYcpEFDhgzBkCFDnljGSF8GXxcLnI2/j0S5Gf773/8CANatW1dl+U2bNuH999/HM888AwB4/fXXceDAASxdupTjr4iIqE7hHOBGrGK9m7MqLOZXVFQEIyMjpWPGxsYICwvTSmxEREQ1xeSmEfN3rxh3U/1ifkFBQfjmm28QHR0NuVyOkJAQbN++natSExFRncPkphHr5F7ecnMrNQ/puUVPLPvf//4XLVq0gI+PDwwMDDBjxgxMmTIFUilfQkREVLfwk6kRszI1gJddEwDVd03Z2tpi586dyMvLQ3x8PK5fv44mTZqgWbNmtREqERGRymqU3JSWluLAgQP4/vvvkZOTAwC4d+8ecnNzNRocaZ+/GuNuAMDIyAjOzs4oLS3FH3/8gREjRmgzPCIiIrWpvUJxfHw8Bg8ejISEBBQVFWHgwIEwMzPDF198gaKiIqxevVobcZIacnNzcfPmTcXPsbGxiIyMhLW1Ndzc3DB//nzcvXsXGzduRGd3a2w5fRshx05hiFMRcnNzkZqaisjISBgYGKB169YAgFOnTuHu3bto37497t69iw8//BByuRzvvfeerm6TiIioSmq33MycOROdO3fG/fv3YWxsrDg+atQoHDx4UKPBUc1ERESgQ4cOivVnZs+ejQ4dOmDRokUAgMTERCQkJAD4dzG/w0umoEOHDjh79ix++eUXdOjQQTHtGwAKCwuxcOFCtG7dGqNGjYKzszPCwsJgaWlZuzdHRERUDYkQQqhToWnTpjhx4gS8vb1hZmaGCxcuoFmzZoiLi0Pr1q2Rn5+vrVg1Ijs7GxYWFsjKyuJWEQC2nk7AvO2XAAASCbBktC/G+bvpOCoiIiJl6nx+q91yI5fLUVZWVun4nTt3YGZmpu7pSIcSswqwYMclxc9CAPO3X8K9zLqdoBIRET2J2snNoEGDsHz5csXPEokEubm5WLx4sVI3BtV9sWl5kD/SbicXwNSfInDpTpZugiIiInpKandL3blzB0FBQRBCIDo6Gp07d0Z0dDRsbGxw9OhR2NnZaStWjWC31L8SswrQfcmhSglOhVEdnPFukDecLY2rLkBERFRL1Pn8Vju5Acqngm/duhUXL15Ebm4uOnbsiAkTJigNMK6rmNwo23YmAQu2X0aZEJBJJJgT1BJRybnYcf4uAMBAT4qpPTzxep/mMDfS13G0RETUWGk9uanPmNxUlphVgLi0fHjYmMDRojxBvXQnC5/tvYqTMeVbM1ibGmBm/xZ4IcAN+jKu/UhERLVLq8nN7t27qz6RRAIjIyN4eXnB09NTnVPWKiY3qhNC4OC1FAT/fQ23UvMAAM1sTDF3iA8GtbaHRCLRcYRERNRYaDW5kUqlkEgkeLRaxTGJRIIePXpg586dsLKyUj96LWNyo77SMjm2nrmNZSE3kJ5XDKB8ZeP3h7ZGe1dLJGYVIDYtD542poqWHyIiIk3S6lTwkJAQ+Pv7IyQkBFlZWcjKykJISAgCAgLw119/4ejRo0hPT8e7775b4xugukVPJsWLXd0ROqcPZvT1gpG+FGfi7mPkyuMYsSIM3ZccwgtrTqH7kkPYdiZB1+ESEVEjV6MVir/55hv0798fZmZmMDMzQ//+/fHVV19hzpw56N69O5YvX46QkBBtxEs6ZGakj3eDvHH43T54rpMLAODCnSzFbCu5AN5evhUDBz8DJycnSCQS7Ny5s9rzhoaGomPHjjA0NISXlxc2bNig9LyHhwckEkmlx/Tp0zV8h0RE1BCondzcunWryuYgc3NzxMTEAABatGiBtLS0p4+O6iRHC2N8PcYPn49qW+m5suICODdvhZUrV6p0rtjYWAwdOhR9+/ZFZGQkZs2ahVdeeQX79+9XlDlz5gwSExMVj4rEecyYMZq5ISIialDU3jizU6dOmDNnDjZu3AhbW1sAQGpqKt577z34+/sDAKKjo+Hq6qrZSKnO6etjB6kESuvkGDfvDJeu7hgytJVK51i9ejU8PT2xdOlSAECrVq0QFhaGZcuWISgoCAAUr7MKS5YsQfPmzdG7d2/N3AgRETUoarfcrF27FrGxsXBxcYGXlxe8vLzg4uKCuLg4/PjjjwDKd6VeuHChxoOlusXRwhjBo30hezBrqmLu1OaT8Rj2XZhK5wgPD8eAAQOUjgUFBSE8PLzK8sXFxdi8eTNefvllztYiIqIqqd1y4+3tjatXr+Kff/7BjRs3FMcGDhwIqbQ8Vxo5cqRGg6S6a5y/G3q1tFWsk3PxThbe33EJ0Sm5AIBdkXfxzP/JYaBXdR6dlJQEe3t7pWP29vbIzs5GQUFBpYUhd+7ciczMTEyePFkr90NERPWf2skNUD4dfPDgwRg8eLCm46F6yNHCWDEF3NHCGP4e1li8+wpWANhzMRHDV4Rh6Vg/tHGyeOprrV27FkOGDIGTk9NTn4uIiBqmGiU3eXl5OHLkCBISElBcXKz03FtvvaWRwKj+sjY1wHfjO2DFC0ATIz1cT8rBiBXH8Wa/Fnijb3OlFY4dHByQnJysVD85ORnm5uaVWm3i4+Nx4MABbN++vVbug4iI6ie1k5vz58/jmWeeQX5+PvLy8mBtbY20tDSYmJjAzs6OyQ0p+XBYG5wodcC+K0lYduAGQq4lYemY9vB2MAMABAYGYu/evUp1QkJCEBgYWOlc69evh52dHYYOHVorsRMRUf2k9oDit99+G8OGDcP9+/dhbGyMkydPIj4+Hp06dcLXX3+tjRipHsnNzUVkZCQiIyMBAOlJd/CftlIs7G0LC2N9HPvlW/gPHImVh2+itEyOadOmISYmBu+99x6uX7+O//3vf/j111/x9ttvK51XLpdj/fr1mDRpEvT0atTgSEREjYVQk4WFhbh+/bri31evXhVCCHHy5Enh7e2t7ulqXVZWlgAgsrKydB1Kg3T48GEBoNJj0qRJIjmrQLToPlQYurYV7nP/EsNXhIno5Gzx+5/7RItWbYWBgYFo1qyZWL9+faXz7t+/XwAQUVFRtX9TRESkc+p8fqv9FVhfX18xK8rOzg4JCQlo1aoVLCwscPv2bY0lXVQ/9enTp9K+Yw+LOvYntp+7iw//vIILtzMRtPwY5HIBMXwJnCXA56N9Mc7frVK9QYMGPfG8REREFdTulurQoQPOnDkDAOjduzcWLVqEn3/+GbNmzULbtpVXrCV6mEQiwbOdXPDP273Q1dMaZXKBipRFLoAF2y8jMatApzESEVH9pnZy8/nnn8PR0REA8Nlnn8HKygqvv/46UlNT8cMPP2g8QGqYHC2M8Vb/FpWOlwmBuLR8HUREREQNhVrdUkII2NnZKVpo7OzssG/fPq0ERg2fp61ppe0bZBIJPGxMdBcUERHVe2q13Agh4OXlxbE1pBEV2zdUkAD4fHRbxYKARERENaFWciOVStGiRQukp6drKx5qZMb5u+G9wd4AgFaO5lUOJiYiIlKH2mNulixZgjlz5uDy5cvaiIcaoWc7ugAAriZmIyW7UMfREBFRfaf2VPCJEyciPz8ffn5+MDAwqLREfkZGhsaCo8bB3twIfq6WuHA7EweupeCFALbeEBFRzamd3CxfvlwLYVBjN6i1PS7czkTI1SQmN0RE9FTUTm4mTZqkjTiokRvY2h5f7Y/C8VvpyCsqhakht1ggIqKaUXvMDQDcunULCxcuxPjx45GSkgIA+Pvvv3HlyhW1znP06FEMGzYMTk5OkEgk2LlzZ7V1QkND0bFjRxgaGsLLywsbNmyowR1QXdPCrgncrE1QXCrHsehUXYdDRET1mNrJzZEjR+Dr64tTp05h+/btyM3NBQBcuHABixcvVutceXl58PPzw8qVK1UqHxsbi6FDh6Jv376IjIzErFmz8Morr2D//v3q3gbVMRKJBANb2wMA/rmarONoiIioPlO77X/evHn49NNPMXv2bJiZmSmO9+vXDytWrFDrXEOGDMGQIUNULr969Wp4enpi6dKlAIBWrVohLCwMy5YtQ1BQkFrXprpnYGt7rA2LxaHrKSgtk0NPVqOGRSIiauTU/vS4dOkSRo0aVem4nZ0d0tLSNBLU44SHh2PAgAFKx4KCghAeHv7YOkVFRcjOzlZ6UN3U2d0Klib6yMwvQUT8fV2HQ0RE9ZTayY2lpSUSExMrHT9//jycnZ01EtTjJCUlwd7eXumYvb09srOzUVBQ9WaLwcHBsLCwUDxcXV21GiPVnJ5Min4+dgCAEHZNERFRDamd3Dz//POYO3cukpKSIJFIIJfLcfz4cbz77ruYOHGiNmJ8KvPnz0dWVpbiwa0j6rZBD8bdhFxNhhCimtJERESV1WhXcB8fH7i6uiI3NxetW7dGr1690K1bNyxcuFAbMSo4ODggOVn5G31ycjLMzc0rLSZYwdDQEObm5koPqrt6trCFgZ4UCRn5uJGcq+twiIioHlI7uTEwMMCaNWtw69Yt/PXXX9i8eTOuX7+OTZs2QSaTaSNGhcDAQBw8eFDpWEhICAIDA7V6Xao9poZ66OFlAwA4cI1dU0REpD61k5uwsDAAgJubG5555hmMHTsWLVq0qNHFc3NzERkZicjISADlU70jIyORkJAAoLxL6eGurmnTpiEmJgbvvfcerl+/jv/973/49ddf8fbbb9fo+lQ3cUo4ERE9DbWTm379+sHT0xMLFizA1atXn+riERER6NChAzp06AAAmD17Njp06IBFixYBABITExWJDgB4enpiz549CAkJgZ+fH5YuXYoff/yR08AbmP4PBhVfuJ2JZG6kSUREapIINUdtpqWlYevWrdiyZQvCw8PRrl07TJgwAePHj4eLi4u24tSY7OxsWFhYICsri+Nv6rCRK48j8nYmPhvVFhMC3HUdDhER6Zg6n99qt9zY2NhgxowZOH78OG7duoUxY8bgp59+goeHB/r161fjoIkeNvChWVNERETqeKolYD09PTFv3jwsWbIEvr6+OHLkiKbiokauYkr4iZvpyC0q1XE0RERUn9Q4uTl+/DjeeOMNODo64oUXXkDbtm2xZ88eTcZGjZiXXRN4NDVBcZkcR29wI00iIlKd2snN/Pnz4enpiX79+iEhIQH//e9/kZSUhE2bNmHw4MHaiJEaoYc30mTXFBERqUPtjTOPHj2KOXPmYOzYsbCxsdFGTEQAgIGtHbDmGDfSJCIi9aid3Bw/flwbcRBV0sndClYm+rifX4IzcfcR2LyprkMiIqJ6QO3kpsLVq1eRkJCA4uJipePDhw9/6qCIAEAmlaCfjz3+OHcHIVeTmdwQEZFK1E5uYmJiMGrUKFy6dAkSiUSxuaFEIgEAlJWVaTZCatQGtn6Q3FxLwgf/10rxOiMiInoctQcxzJw5E56enkhJSYGJiQmuXLmCo0ePonPnzggNDdVCiNSY9WppA0M9KW5nFCAqOUfX4RARUT2gdnITHh6Ojz/+GDY2NpBKpZBKpejRoweCg4Px1ltvaSNGasRMDP7dSDPkCmdNERFR9dRObsrKymBmZgagfLXie/fuAQDc3d0RFRWl2eiI8NBqxdwlnIiIVKD2mJu2bdviwoUL8PT0REBAAL788ksYGBjghx9+QLNmzbQRIzVy/VvZQyK5hIt3spCUVQgHCyNdh0RERHWY2i03CxcuhFwuBwB8/PHHiI2NRc+ePbF37158++23Gg+QyNbMEB1cLQEAB9h6Q0RE1VC75SYoKEjxby8vL1y/fh0ZGRmwsrLiTBbSmoGtHXAuIRMhV5PxYlfuEk5ERI+nkSVfra2tmdiQVlWMuwm/xY00iYjoybiePdULzW1N4WljiuIyOY5EcSNNIiJ6PCY3VC8ob6SZpONoiIioLmNyQ/VGRXJz6HoKSsrkOo6GiIjqKiY3VG90dLNCU1MDZBeW4kxshq7DISKiOorJDdUb5Rtp2gHggn5ERPR4TG6oXvl33E2yYtNWIiKihzG5oXqlZwtbGOlLced+Aa4ncSNNIiJtW7lyJTw8PGBkZISAgACcPn36sWX79OkDiURS6TF06FBFmcmTJ1d6fvDgwRqNmckN1SvGBjL08LIFUN56Q0RE2rNt2zbMnj0bixcvxrlz5+Dn54egoCCkpKRUWX779u1ITExUPC5fvgyZTIYxY8YolRs8eLBSuS1btmg0biY3VO8MbP1g3A2TGyIirfrmm2/w6quvYsqUKWjdujVWr14NExMTrFu3rsry1tbWcHBwUDxCQkJgYmJSKbkxNDRUKmdlZaXRuJncUL3Tz8ceEglw6W4WErMKdB0OEVGDVFxcjLNnz2LAgAGKY1KpFAMGDEB4eLhK51i7di2ef/55mJqaKh0PDQ2FnZ0dvL298frrryM9PV2jsTO5oXrH1swQHd3Ks/wDbL0hItKKtLQ0lJWVwd7eXum4vb09kpKqX0z19OnTuHz5Ml555RWl44MHD8bGjRtx8OBBfPHFFzhy5AiGDBmCsrIyjcWu9saZRHXBwNb2OBt/H/9cTcZLgR66DoeIiB6xdu1a+Pr6okuXLkrHn3/+ecW/fX190a5dOzRv3hyhoaHo37+/Rq7NlhuqlyqmhJ+MSUdOYYmOoyEianhsbGwgk8mQnKzcQp6cnAwHB4cn1s3Ly8PWrVsxderUaq/TrFkz2NjY4ObNm08V78OY3FC91Ny2CZrZmqKkTGDNsRiOvSEi0jADAwN06tQJBw8eVByTy+U4ePAgAgMDn1j3t99+Q1FREV588cVqr3Pnzh2kp6fD0dHxqWOuwOSG6i1XK2MAwLcHb6L7kkPYdiZBxxERETUss2fPxpo1a/DTTz/h2rVreP3115GXl4cpU6YAACZOnIj58+dXqrd27VqMHDkSTZs2VTqem5uLOXPm4OTJk4iLi8PBgwcxYsQIeHl5ISgoSGNxM7mheikxqwBHo9MUP8sFsGD7ZXz+9TKVF5sCgMzMTEyfPh2Ojo4wNDREy5YtsXfvXsXzOTk5mDVrFtzd3WFsbIxu3brhzJkzWrsvIqK6ZNy4cfj666+xaNEitG/fHpGRkdi3b59ikHFCQgISExOV6hyLuICwsDCMfL5yq41MJsPFixcxfPhwtGzZElOnTkWnTp1w7NgxGBoaaixuiWhka9hnZ2fDwsICWVlZMDc313U4VEMnbqXhhTWnlI7lXTuKrL+X4/vvVyMgIADLly/Hb7/9hqioKNjZ2VU6R3FxMbp37w47OzssWLAAzs7OiI+Ph6WlJfz8/ACU/8e+fPkyVq1aBScnJ2zevBnLli3D1atX4ezsXCv3SkRUX2w7k4D52y9BLgCpBAge7Ytx/m4aObc6n99MbqheSswqQPclhyB/6NWbuHE2fHzbI/zPn2FioAe5XA5XV1e8+eabmDdvXqVzrF69Gl999RWuX78OfX39Ss8XFBTAzMwMu3btUlo6vFOnThgyZAg+/fRTrdwbEVF9VNX7skwiQdi8vnC0MH7q86vz+c1uKaqXHC2METzaFzKJpPxAWQmKk24i0bQFhn4bhsjbmdUuNrV7924EBgZi+vTpsLe3R9u2bfH5558r1looLS1FWVkZjIyMlOoZGxsjLCxMq/dHRFTfxKblKSU2AFAmBOLS8ms9Fq5zQ/XWOH839Gppi7i0fBiVZKLj13LY2tkjNi0Pz646gTf7ecHW1g7Xr1+vsn5MTAwOHTqECRMmYO/evbh58ybeeOMNlJSUYPHixTAzM0NgYCA++eQTtGrVCvb29tiyZQvCw8Ph5eVVy3dLRFS3edqYVjomk0jgYWNS67Gw5YbqNUcLYwQ2bwp78/Imz+Xj2mOYnxPK5ALLD0Rj14V7KCqVV1lXLpfDzs4OP/zwAzp16oRx48bh/fffx+rVqxVlNm3aBCEEnJ2dYWhoiG+//Rbjx4+HVMr/OkRED7MzM4Kxvkzxs0wiweej22qkS0pdbLmhBqFisam8rHR8N74nBrSyw8KdlxGblARJsR5+OZWA8V1cIanoxgLg6OgIfX19yGT//mds1aoVkpKSUFxcDAMDAzRv3hxHjhxBXl4esrOz4ejoiHHjxqFZs2a6uE0iojrr0t0sFJSUwdRQhu9f7ITmdk10ktgAbLmhBuLRxaZGtHfG3rd6QH7nEvQcvbFgxyW8ujECqTlFijrdu3fHzZs3IZf/27Jz48YNODo6wsDAQOn8pqamcHR0xP3797F//36MGDGidm6MiKieOH6zfHmOHl426NHCVmeJDcDkhhqQRxeb+nT+bBiiBAtnvQEDmRRbv5oL36GTFJttvv7668jIyMDMmTNx48YN7NmzB59//jmmT5+uOOf+/fuxb98+xMbGIiQkBH379oWPj49iASsiIip3LDoVQHlyo2vslqIGY9y4cUhNTcWiRYuQlJSE9u3bY9++fQgI6IJnArLRfdsC5N6X4pWNERjfxRULh7bGz3/sxrw572LNmjVwdnbGzJkzMXfuXMU5s7KyMH/+fNy5cwfW1tZ49tln8dlnn1U5dZyIqLHKLy7FufhMAECPFra6DQZc50bX4VAtKiwpw9J/ovBjWCyEAJqaGiAjvxhCC4tNERE1JqFRKZi8/gycLY0RNrev0vhGTeE6N0RVMNKX4f2hrfHzKwGwNzNEel55YgP8u30DN+AkIlLfw+NttJHYqIvJDTU63Zrb4NNRvpWOP+1iUytXrtTovlZHjx7FsGHD4OTkBIlEgp07d9bJaxMRHXuw11/3FrofbwMwuaFGqq2zOaSPfLmQSlDjxaa2bduG2bNnY/HixTh37hz8/PwQFBSElJSUKssXFxdj4MCBiIuLw++//46oqCjFuJ8KeXl58PPzw8qVK+vstYmIUnOKcD0pBwDQvXnTakrXEtHIZGVlCQAiKytL16GQjm09HS885/0l3OeWP97YfLbG5+rSpYuYPn264ueysjLh5OQkgoODqyy/atUq0axZM1FcXKzS+QGIHTt2KH6+l5kvjt9MFfcy82v92kSkfQ//H6/rdp6/I9zn/iWGLD+q1euo8/nNlhtqtMb5u+H4vH54pYcnAODg9WTEpuWpfZ7i4mKcPXsWAwYMUBx72n2tnmTbmQR0X3IIL6w5hW6f7UdELV6biLTv4f/j3ZccwrYzCboO6YkquqR61pEuKYDdUtTIOVoY4/2hrdDDywaFJXLM/f0i5I/u/FaNtLQ0lJWVwd7eXum4vb09kpKSqqwTExOD33//HWVlZdi7dy8++OADLF26tNqdxs/GZ2DuH5cUm9OV5GVDXlYGYaw8c0Ab1yYi7UvMKsD87f/+H6/rkx2EEP8OJmZyQ1R3SCQSBI/2hYmBDKfjMrAxPE7r11RlX6tHbTgeizGrq26NWbD9En49cxslZVXvo/W01yai2lGXdtZWxa3UPCRmFcJATwp/D2tdh6PA5IYIgKu1CeYP8QEAfLEvCgnpqr+RVOxrlZycrHQ8OTkZDg4OVdZxdHREy5YtH7uvVYWY1FzM/jUSAHDiVnqlNz2ZiTkgkSI9NRXv/XERA745gt/P3kFiUtJTX5uIap+7deVJDbraWVsVYQ9WJfb3sILRQ5tm6hqTG6IHJgS4o2szaxSUlOG9Py6o3D316L5WQHnryMGDBxEYGFhlner2tYpOzsHMrecx4Jsj2H7uLgCgrbMFtr/RDV886wvZg3Uk9PQM0KyVL9oiAU1NDRCfno93fj2PX3f/DXP31iir4h7U2VOLiGrXxTtZlY59Nko3O2urIuxmOgCgex3YcuFhTG6IHpBKJfjyWT8Y68twMiYDP59WfRDfo/tavf7668jLy1PsQTVx4kTMnz9fUf5x+1qNnTgV0385h0HLj2LH6VsoTIpBB5NMAMCwZnqQZsQj0B4Im9cXW17tirB5ffH5ovk4sfdX/MfpNia11kPewdUoLSxAqGiDgcuOoO//PYe58+ZVe+2H99TKzc1FZGQkIiMjAQCxsbGIjIxEQkLdHthIVJ+VyQWWHbgBABjv7wq9B+tVdPGsO909Dyspk+NkTHly09NL91suPIx7SxE9xK2pCeYO9saHf15F8N5r6NPSFq5VNBM/6nH7WlUMMk5ISIBU+u93CVdXV6V9rewcHOHR+znskPtDcjERAOBnlI6/NryFnQ/qzJ49GwAwadIkbNiwQfFNruLan338EZKSktDOzw9TvtmAkDRzxKTmIeliFKKSc9Hjwj0M9XWEq6sr9u/fj7fffhvt2rWrck+tiIgI9O3bV/Hzo9cmIs376+I93EjOhbmRHuY90wq37xcg7GYaDkelopltE12HV8nFO5nILSqFpYk+WjvVre2MuLcU0SPkcoHnfziJ03EZ6O7VFJunBmh8OfFtZxKUZkRUkEiAZ9o6YkY/L7RyfLrXZ05hCTYcj8OaYzHILiwFAHjbm2HmgBbwc7FAfEY+PG1MNdLcvXLlSnz11VdISkqCn58fvvvuO3Tp0qXKshs2bKi0q7qhoSEKCwsVP0+ePBk//fSTUpmgoCDs27fvqWMlqotKy+QYtOwoYtLy8O6glpjRrwV+PBaDT/dcQ88WNtg0NUDXIVay/MANLD8QjaG+jlg5oaPWr6fO5zdbbogeIZVK8MVz7TB4+VEcv5mOLadv44UAzW2o+ehUzwoDWtnjvcHeaGlvppHrmBnp483+LTCpuwfWhcVibVgsopJz8MbP5xRlNLFhaMUKyatXr0ZAQACWL1+OoKAgREVFwc7Orso65ubmiIqKUvxcVfI4ePBgrF+/XvGzoaFhjWMkqut2nL+LmLQ8WJnoY3L38rW3+njb4dM913AqJgP5xaUwMahbH9kVU8Dr2ngbgGNuiKrkaWOKOUHeAIDP917D3UzNrTHxW8SdSokNAEzt4amxxOZh5kb6mDWgJcLm9sOU7h5Kz2liDY1vvvkGr776KqZMmYLWrVtj9erVMDExwbp16x5bRyKRwMHBQfF4dI0goDyZebiMlZVVjWMkqstKyuT49lA0AGBa7+ZoYliexDS3NYWLlTGKy+Q48WDgbl2RW1SK8wmZAOrW4n0VmNwQPcaU7p7o6GaJ3KJSzN9+CU/bg1tYUoZFuy7jm5AblZ6rjameFsb6GNi6chLxNGto1GR1ZqB8wLK7uztcXV0xYsQIXLlypVKZ0NBQ2NnZwdvbG6+//jrS0+vWmzuRpvwWcQe3Mwpg08QQEwM9FMclEgn6epe3fobeqHqvOF05eSsdpXIB96YmKo1LrG1MbogeQyaV4KsxfjDQk+LojVT8FnGnxue6mZKDkSuPY2N4PACgVwsbxcadMokEn4+unamenjamlTYMlT3FhqE1WZ3Z29sb69atw65du7B582bI5XJ069YNd+78+/sdPHgwNm7ciIMHD+KLL77AkSNHMGTIEG4RQQ1OUWkZVjxotXmjT3MYGyivFdPXp3wW0uHrqU/9BUuTwupwlxTAMTdET9TctgneGdgSwX9fxyd7rqJnSxu1khAhBH6NuI0Pd19FQUkZbJoY4OsxfujjbYfErALEpeXDw8ak1tawcLQwRvBoX6UxP//p3bxW19AIDAxUWv+nW7duaNWqFb7//nt88sknAIDnn39e8byvry/atWuH5s2bIzQ0FP3796+1WIm0bevp27iXVQgHc6Mqx/YFNrOBgZ4UdzMLcCs1F152mu+6romK5KZnHU1u2HJDVI1XejaDn6slcgpLsUCN7qnswhK8ueU85v5xCQUlZejZwgZ7Z/ZEnwfNzI4Wxghs3rTWF+eq2DC014N+8qSswmpqPF5NVmd+lL6+Pjp06ICbN28+tkyzZs1gY2PzxDJE9U1hSRlWHi5/TU/v51XlCr/GBjJ0bdYUQHnrTV2QlFWImym5kEiAwOZNdR1OlZjcEFVDJpXg6+fawUAmxeGoVMWKwU9yPuE+hn57DH9dTISeVIJ5Q3zw05QusDMzqoWIq+doYYzZg8oHTP91MREZeTXbdqEmqzM/qqysDJcuXYKjo+Njy9y5cwfp6elPLENU32w+GY+UnCI4WxpjXGfXx5br0/JB11RU3Rh3U9Fq087ZApYmdXNVcyY3RCpo8WB9GAD46M8rSMmuurVDLhdYFXoLY1aH43ZGAVytjfHbtEBM690c0kcHu+hYe1dLtHOxQHGZHNvO3K7xedRdnfnjjz/GP//8g5iYGJw7dw4vvvgi4uPj8corrwAoH2w8Z84cnDx5EnFxcTh48CBGjBgBLy8vBAUFPd1NE9UReUWlWBV6CwDwVn8vGOg9/uO4r095a++ZuAzkFpXWSnxPUpengFdgckOkov/0agZfZwtkF5ZiwY7LlbqnUnIKMXHdaXyx7zpK5QLD/Jyw562e6OBWd6cwv9jVHQDw86n4KvehUsW4cePw9ddfY9GiRWjfvj0iIyMrrc6cmJioKH8nKRUTp0xFq1at8MwzzyA7OxsnTpxA69atAQAymQwXL17E8OHD0bJlS0ydOhWdOnXCsWPHuNYNNRg/hcchPa8Y7k1NMLqjyxPLetqYwr2pCUrKhCKx0BUhhKLlpkcdnAJegSsUE6khKikH//fdMZSUCfz3+fYY0d4ZABAalYJ3fr2A9LxiGOvL8NHwNhjT2UXjKxtrWkFxGboGH0RWQQnWTe6Mfj6Vp4pr0sMrM2tiAUGi+iinsAQ9vzyMzPwSfDPWr9rkBgA+3H0FG07EYXwXNwSP9q2FKKt2PSkbg5cfg5G+FBcWD4KhXu3tBK7O5zdbbojU4O1ghjf7lXdPfbDzMv68cA/zt1/E5PVnkJ5XDB8HM/z5ZneM9Xet84kNUD5YcUyn8jfWTQ+mqWvLoysza2IBQaL6aF1YHDLzS9Dc1lTxBak6vb3Lx92ERqXodEp4WHR5q00Xz6a1mtioi8kNkZpe79McjhZGyC4sxZtbzmPL6fLxKpO7eWDn9O51Zqqmqiq6pkJvpCIhvWaL+akiNi2v0srMT7OAYF2wcuVKeHh4wMjICAEBATh9+rRK9bZu3QqJRIKRI0cqHc/NzcWMGTPg4uICY2NjxYrP1HBk5Zfgx7AYAMCsAS0hU3EsXmCzpjDUkyIxqxA3knO1GeIT1fUp4BWY3BCpKS23CEmPDCiWSoD/9G5W5VTOus7DxhS9WtpCiPKxN9piaaxf6VhtrMysLRV7ai1evBjnzp2Dn58fgoKCkJLy5BktcXFxePfdd9GzZ89Kz82ePRv79u3D5s2bce3aNcyaNQszZszA7t27tXUbpAZNJLNrjsUgp7AUPg5mGN7eGRKJpNLjq6++qnQOI30Zuj2Ydq2rWVPFpXKciskAULcHEwNMbojUFpuWh0dbheUC9boF4qUHrTfbIm6jsEQ7qwBvrWJG1rtBLWt9nR9NqcmeWmVlZZgwYQI++ugjNGvWrNLzJ06cwKRJk9CnTx94eHjgtddeg5+fn8ofoqQ9mkhmi8vkWH88FkB5q01iYqLSY926dZBIJHj22WerPFfFGlmHr+smuTmXcF+xGKmPQ91uoWZyQ6SmqrcwqL8tEADQz8cOzpbGyMwvwZ6LidVXUNPNlFz8fCoBALBifAf4OpcPBqzp+jq6VtM9tT7++GPY2dlh6tSpVT7frVs37N69G3fv3oUQAocPH8aNGzcwaNAgjd8DqUcTyWxcWh7yisvQ1tkcQW3slTaGdXBwwK5du9C3b98qE18Ain2mzsbfR3ZhiVbu80kqZmp1a25T55a2eBSTGyI1VWxhIHswYLg294bSFplUolj6feNJzXdNLfn7GsrkAgNa2eP//Jwwa0BLAMC2M7eRX6z7dTvUVZM9tcLCwrB27VqsWbPmsef97rvv0Lp1a7i4uMDAwACDBw/GypUr0atXL43GT+rRRDJbWFKG2xnlrbvvDPSuNOEgOTkZe/bseWziCwBuTU3QzMYUpXKB49G1PyX8WHTdnwJeoU4kN+r0Y27YsKFS/6SRUd1Y9ZUaj3H+bgib1xdbXu2KsHl9G8R05nH+rtCXSXDhdiYu3snU2HlP3ErDgWspkD1YqRko/wbq3tQE2YWl2HG++hWf67ucnBy89NJLWLNmDWxsHv/B8N133+HkyZPYvXs3zp49i6VLl2L69Ok4cOBALUZLj9JEMns9KQdlAujgZok+D2Y+Peynn36CmZkZRo8e/cRYKrqmQqNqdyuGrPwSxftCjzo+3gaoA8lNTfoxzc3Nlfop4+O1O4WVqCq62htKW2yaGOIZ3/LtDTZrqPVGLhf4bM81AMCEADd42TUBAEilEkwM9AAA/HQirk7tdqwKdffUunXrFuLi4jBs2DDo6elBT08PGzduxO7du6Gnp4dbt26hoKAACxYswDfffINhw4ahXbt2mDFjhmKRRKo/Hk1mE7MKEJOWB6DqVhsAWLduHSZMmFDtl/WKxCj0Ru1OCQ+PSYdcAM1sTeFkWfff83Se3NSkH1MikSj1Uz6aTRNRzUwMLB9YvCvyHrLyn75Pf8f5u7hyLxtmhnqY2b+F0nNjOrvAxECGG8m5CL+V/tTXqk3q7qnl4+ODS5cuITIyUvEYPnw4+vbti8jISLi6uqKkpAQlJSWQSpXflmUyGeRyudbviR7vaZNZZ2sz5Fw8iIKbp9CnlQNu3bqlVP7YsWOIiopSbEHyJF08rWGsL0NydhGuJeY83Y2pIexmeUtRXZ8CXkGnyU1N+zFzc3Ph7u4OV1dXjBgxAleuXHls2aKiImRnZys9iKhqHd2s0MrRHEWlcvx2tub7TQHlqx9/tT8KQPmOx02bKG+dYG6kj2cfrMy6/kTcU11LF9TZU8vIyAht27ZVelhaWsLMzAxt27aFgYEBzM3N0bt3b8yZMwehoaGIjY3Fhg0bsHHjRowaNUqXt9roPU0yu+9IOFxe/hbGLQLQuWsPRTL7sLVr16JTp07w8/OrNhYjfRm6e9X+lPDjN8u/gNT1KeAVdJrc1KQf09vbG+vWrcOuXbuwefNmyOVydOvWDXfu3KmyfHBwMCwsLBSPR19URPQviUSimBa++WQ85DXcbwoA1obFICm7EM6WxpjczaPKMpO6lV/r4LVkxWDL+kLdPbUSswpw4lbaE1dk3rp1K/z9/TFhwgS0bt0aS5YswWeffYZp06Zp/X7oyWqazP6TaABpU3c42ljD2c5akcxWyM7Oxm+//aZSq02F3opxN7WT3Ny5n4/YtDzIpBJ0fbDWTl2n824pdQUGBmLixIlo3749evfuje3bt8PW1hbff/99leXnz5+PrKwsxeP27af7NkrU0I1o7wQzQz3EpecrViNVV0pOoWLH4/cGez92cUMvOzP0bGEDuQA2aWGWlrbNmDED8fHxKCoqwqlTpxAQEKB4LjQ0FBs2bABQvqdW9yWH8MKaU+i+5BC2nUnAhg0bsHPnTqXzOTg4YP369bh79y4KCgpw/fp1zJ49u8oxGppcHbmkpARz586Fr68vTE1N4eTkhIkTJ+LevXtq/04aKnWTWQA4HZuO38+Wf/Fu7WxR5Xm3bt0KIQTGjx+vcix9WpaPuzmXkKmR7uPqVGy54OdiAXOjyotx1kU6TW7U7cesir6+Pjp06ICbN29W+byhoSHMzc2VHkT0eKaGeni2Yr+pGiYcy0KikVdcBj9XSwz3c3pi2YpWna2nE+rltPDqVLWn1vztl55qTy1Nr46cn5+Pc+fO4YMPPsC5c+ewfft2REVFYfjw4TWOUVNUafGqLaoms0B5Qjv2+5OKv/uL7y2plMwCwGuvvYb8/HxYWFSd/FTF1doEXnZNUCYXOHZT+7Om/t0FvPIsr7pKp8mNuv2YVSkrK8OlS5fg6OiorTCJGp2K/aYOXkvG3Uz1PlSiknKw7Uz5gn0Lh7aqdgPRh6eF7zzf8FoKTtxMq7SnllwA/9l0FucT7tfonJpeHdnCwgIhISEYO3YsvL290bVrV6xYsQJnz55FQkJCjWLUhKpavOqDioT2YZreJLavYiNN7SY3crnAiQcD/uvDFPAKOu+WUqcfEyhfFOmff/5BTEwMzp07hxdffBHx8fFq9VcS0ZN52TVBt+ZNIRfAL2ruNxX89zXIBTC4jQP8PayrLS+V/jvOZ8OJ2Ho3LfxJ4tLyEPz39Sqfu3gnC6P+dwKT15/GhduZKp9TW6sjPyorKwsSiQSWlpYqx6ZJiVkFmFdPd5GPSsrR+iaxD6938zRj46pzNTEbGXnFMDWQoYObpdauo2k6T27U7ce8f/8+Xn31VbRq1QrPPPMMsrOzceLECbRu3VpXt0DUIFVMC9925jaKSlXbb+pYdCpCo1Kh99CCfaoY09m13k4Lf5z49DyMX3MSabnFsDMzVGzZIZNI8F6QN8Z0coFMKkFoVCpGrDyOlzecwaU7WdWeV1urIz+ssLAQc+fOxfjx43XWlf/LqYRKe7jVl13kq1qYUtNbtHT2sIKpgQxpuUW4ck97s4AruqQCmjWFvkznKYPK6kSk6vRjLlu2TFE2KSkJe/bsQYcOHXQQNVHDNqCVPezNDZGWW4x9l6v+0HxY2UML9r0U6A4PG1OVr2VhrI/RHZ0BABvq4bTwR93OyMf4H04iMasQzW1Nseetnjg+r59iRes3+nrhqzF+ODi7N57t6AKpBDh0PQXDVoThlZ/O4PLd6pMcVam6OnKFkpISjB07FkIIrFq1qsoy6gxm3r59Ozp37gxLS0uYmpqiffv22LRpk1KZ3NxczJgxAy4uLjA2NoaDuxc+X/ptpXNJJajze7j9GnEbuyLLu1cfTmg1vUWLoZ4M3R50E2lz1lTFflL1qUsKqCPJDRHVPXoyKV7oUt56sym8+q6p38/exvWkHJgbVV6wTxWTHqxYfKAeTgt/2J37+Ri/5iTuZRWimY0ptrzaFbZmhlWuaO1hY4qlY/1w8J0+GN3BGVIJcOBaCv7vuzC8ujECV+5VTnK0sTpyhYrEJj4+HiEhIVW22qg7mNna2hrvv/8+wsPDcfHiRUyZMgVTpkzB/v37FWVmz56Nffv2Yd2Gn/D8l7+jpNUQZISshkfuVaVNap0tjWFnVne327l8Nwsf7LwMAHhnYEulhFYbW7RUbKSprfVuCkvKcDo2AwDQsx7sJ/UwJjdE9Fjju7hCTypBRPx9XH1C03deUSmW/nMDAPBW/xawNDF4bNnHaWH/77RwTW3/UNvuZRZg/JqTuHO/AJ42ptjyWlfYmVf/YexpY4pvxrVHyOzeGNneCRIJEHI1GUO/DcN/NkXgWmK2YtZQekGZxldHBv5NbKKjo3HgwAE0bVr1eibqDmbu06cPRo0ahVatWqF58+aYOXMm2rVrh7CwMEWZEydO4LnnJ2D1DSMcvgtYdRwCj5at0cMyC8fn9cPKFzrA1FCG2/cLsDYsptrfpy5k5Zfg9Z/PoqhUjv4+dpje10vrW7RUbMUQeTsT9/OKNX7+iLj7KCqVw97cULF1Sn3B5IaIHsvO3AhBbctbAzY/YWDxD0djkJJTBDdrE7z0YKxOTVS03myph9PCk7IKMX7NSdzOKIB7UxP88moA7FVIbB7W3LYJlj/fASFv98Zwv/IkZ/+VZAz57zEEBpfPGuq25BA6PPOiRldHLikpwXPPPYeIiAj8/PPPKCsrQ1JSEpKSklBc/O+HZk0HM1cQQuDgwYOIiopS2um8dfvOWLFhG05fuQkzQxlmti5G2t04DBo0CI4WxhjazgmLh7UBAHz9zw3cTMlV6/eqbXK5wKxt53E7owBu1ib4Zmx7SKVPniWoCU6WxvC2N4NcAEejNT9rqmK8TXcvm2pnPdY1TG6I6IkqZjLtPH8X2YWVFwxLzi7ED0fLv03PHewDQ72qF+xTRV8fO7hZ179p4cnZ5YlNfHo+XK2NseXVrk/1bd3Lrgm+Hd8B/8zqhf4+dkrPCQHsK2gGq74vY9rbc9G2nR/2hp7E7KUbcCtHhtsZ+YhXc3Xku3fvYvfu3bhz5w7at28PR0dHxePEiROKcjUZzAyUz7xq0qQJDAwMMHToUHz33XcYOHAgAODQ9WRcchsNWDnj7v8m4/rnw/Du1HFYuXKlUgI0ppMLere0RXGpHHN+v4AyLc4QUteKwzdxOCoVhnpSrHqxIyxMam+huz4+5a03R7QwJbxiP6n6Nt4GAPR0HQAR1W0BntZoYdcE0Sm52H72DiZ391R6/uv9USgoKUMndys846va4puPI5NKMDHQHZ/uuYafTsRhfBfXOv+NMSWnPLGJTcuDs2V5YqOpXZNb2Jthak9PHLxeeUyFod9Q2PsNVfy89gaw9sYpAIB+9/cgtTLBpHWnUVxahpMxGRAoH+AaPNpXaZIGAHh4eGh1Cr6ZmRkiIyORm5uLgwcPYvbs2fD09ESsvgc+3XMV90/ugjT1Jn757Q+0aemFo0ePYvr06XByclK0EkkkEgSP9kXQsqM4n5CJdWGxeLVXs2qurH1HbqRi2YHyLtlPR7ZFGyfVF+PThD4t7fD9kRiE3iifEq6pFqOMvGLFLCwmN0TU4EgkErwU6I5Fu65g08l4TOrmoUg4rt7Lxu/nypeXf1+FBftUMaazK5b+cwNRyTkIj0lHt+Z19401NacIL6w5hZjUPDhZGGHra13hYqXZ2TyeNqaQSqC0bopUAmyaGoD84jLEp+eVt9Zk5CMhPR+37+ejpEwgNi0PsWl5SueqWCumV0vbGrUs1XRVealUCi8vLwBA+/btceXqVUydvRDyoAWQlxQh+9hG/LF9O0YOHwYAaNeuHSIjI/H1118rdYE5WRpj4f+1wtw/LuHrf6LQr5UdmtvqbizI7Yx8zNx6HkIALwS4YUzn2t+7sLOHFZoY6iEjrxgX72ahvaulRs574lYahABa2jdRadxYXcNuKSKq1qgOzjA1kOFWah7CY8rXoRFC4PO91yAEMLSdIzq6WWnkWhbG+ni204Np4cfjqi2v6WnJD5s2bRokEgmWL19e6bn03CJM+PEkbqbkwtHCCFte6wpXa81PU3a0MEbwaF/IHiSOsgctGN29bDCwtT1e6dkMH41oiw1TuuDQu31w/ZMhOD6vH355NQCv9vSsdL6nWStGE6vKZ+WX4EhUMu6lZ0MiAd4d0BzyslIY6it/15bJZJDL5ZXqj+3sil4tbVFUKsd7v1/UWfdUYUkZ3vj5HDLzS+DnYoHFw3Sz1pq+TKpoWdHklPB/p4DXny0XHsbkhoiqZWakj1EP1qGpmMkUGpWKsJtpMJBJMW+w6gv2qULVaeHamJZcYceOHTh58iScnCrvjZWRV4wJP57CjeRc2Jsb4pdXu8K9qerr+qhrnL8bwub1VWlasUwqgbOlMbo1t8HLPTxRVS/FtcSar6Oj7qrywcHBCAkJQUxMDA6cOItO42YiJnwfrPz644eXOuOtIX7o3bs35syZg9DQUMTGxmLDhg3YuHEjRo0aVen6EokES0b7oomhHs7G38f647E1vpen8dGfV3DpbhasTPTxvxc7PdVYs6fV98G4m8MaGncjhMCx6Ir9pOrHLuCPYnJDRCp5qasHgPLZO3czC/DZ3vIF+yZ399B4i0ULezP08Kp+Wrg2piUD5QNs33zzTfz888/Q11ceHJqZX57YXE/Kga1ZeWLjqcaChTVVk2nFj7b6VOQ5n+29rtLCjFVRd1X55IxMvPzqf9CqdRsMHtAHd84dRoux83Dw+48xsHV5na1bt8Lf3x8TJkxA69atsWTJEnz22WeYNm1alTE4WRpj4dBWAICv9kchJrV2Z0/9GnEbW07fhkQC/Pf5DnDW0BirmurdsnzQ+cU7mUjPLXrq852Jy8Cd+wXQkwIBnkxuiKgB83YwQxdPa5TJBcZ9fwI3U3JhaaKP6X29tHI9xW7hZ26joLjy9g/ampYsl8vx0ksvYc6cOWjTpo1SnRtJORi58jiuJWbDpokhtrzaVadjPlSh1Oozty/GdHJBmVzgrS3ncayG04dVXVV+25kE7NbrBdnz38Fx1u9weWsrghb8iLAfFqK1078LBDo4OGD9+vW4e/cuCgoKcP36dcyePbvKMVwV3ZCTe7VE7q/vITvh2mO7p6rrhiwpKcHcuXPh6+sLU1NTODk5YeLEibh37/Ez9R5eqG/2gJbo1bJ2um2e1P3qYGGEVo7mEA+mhD/NfW87k4Bx358EAJTKgb8u1p9Ziw9jckNEKvN68EF+534hgPJVSy2MtTPttWJaeFZBCXZGVt6rRxvTkgHgiy++gJ6eHt566y2let8evIFBy48iLr28m2xioFu9WdisotXH2coEwaN9MaStA4rL5Hht41mcjc/QyjUrdsZ+OOeQAPjvuPY1XmX40W7IoF4BSPltMU5dja1y247quiHz8/Nx7tw5fPDBBzh37hy2b9+OqKgoDB8+vMrrV7VQX21Qpfv14V3Ca3rfQ4b+H+Zvv4SH08T6slnpoySiIW3Bq4Ls7GxYWFggKytLZxvCEdVHiVkF6L7kUKVZO8fn9dPaCqw/HovBp3uuwdveDPtm9VT6Jn/v3j04OzvjxIkTSoNZ33vvPRw5cgSnTp2q8pxyuRwxMTGKacmffPIJdu7ciT59+uDs2bMYOnQozp07BycnJ+QXl8LDwxO2XUchr2WQ0nlkEgnC5vXV2r1rU1FpGV7deBZHb6TCzEgPW1/rqvEpzL9G3MZ7v1+sdHzLq10R2LxmXR0BAQHw9/fHihUrAJT/LW0cnCBaD4Z9z3H4e2avarsIO3bsiKFDh+KTTz6p8vkzZ86gS5cuiI+Ph5vbv2Ob5HKBqT+dweGoVLhZm+DPGT1qbT2bqu7b1dUVb775JubNmwcAOB2bgbHfh8PSRB9nFw6E7JHBVqret/Pr66Bnrry20tP8zTRJnc9vttwQkUpi0/LwaMu/XECruzSP6ewKY30ZopJzcDJGuYXhaaclt2/fHu+88w6ee+45BAcHAwCOHTuGlJQUuLm5QSrTg6mRIVIT7+DqzpW4s+plpfPUlx2qq2KoJ8P3L3aCv4cVcgpLMXHtadzS0LgVIQQ2hsdh4Y5LlZ57mp2xH9cNOWxIEJpkxqCwRI73fr8A+WNmTz2uG/JRWVlZkEgksLS0VDquq4X6VO1+7ehmCTMjPWTmlyDydqbiuCr3nVtUiu/2XQAggdRQuTVS07uZ1xYmN0Skkor1Vh6m7Tc+pWnhJ5RnxWhiWnJFnaKiImQXlsCgVR/0eG897Cb9Fw6T/wvHKd/CwLwpej/3MhzGfaxUr76+6VcwNpBh7WR/tHU2R3peMV788RTu3H+6ZC09twiv/BSBRbuuoLhMwNu+icZ2xn5SN6SNrACmBjKcibtfqXuqum7IhxUWFmLu3LkYP368UstAaFSKzhbqU7X7VU8mRa8WFasVp6h83wevJWPAFyHYtnIJTFr3QqcWTlrdzby2cBE/IlJJxcybBdsvo0yIWnvjmxTogc0nExByNRl37ucrLZI3e/ZsTJo0CZ07d0aXLl2wfPnyStOSnZ2dFS0zwcHB6Ny5M5o3b46ioiLs2bMHGzdtQr+XF6DLZwdQWCIHYIMmDnYIauuA8f6ueOHgVxjZrQ0cuw+p9XvXNnMjffw0pQvGfh+OW6l5ePHHU/h1WmCNxsQcuZGKd369gLTcIhjoSTF/iA8md/NAUnYh4tLy4WFjorXfl4GeFO8ObYX3d1zGl/uvo5+PHTwedE9VtTpys2bN0KdPH6VzVGwcKoTAqlWrAJR3xZ6OycAHuy7rdKE+VfXxtsWeS4k4HJWKWQNaPPG+U3IK8dHuq/gr8jZSd34OfZkEv21ahyEdmyExq0DrfzNtY3JDRCob5++GXi1ta/WNr2JaeNjNNGw6GY/5Q1r9G8+4cUhNTcWiRYuQlJSE9u3bV5qWLJX+20BdMS05NTkRMn1D6Fu7wOqZ2bhh1QUokcPLrgme93fF6I4usDZV3tlcF/deG5o2McTPr3TFc6tPIC49HxPXnsbW17qqvLN7YUkZvtwXhXUP1ptpaV++L5aPQ3nLh6OFsUZ+V9V1Q77QxQ17LibixK10vPf7RWx9rSukUkml1ZGvXbuG4OBgpeSmIrGJj4/HoUOHYG5ujm1nEpQGRLtYGetkoT51ul97PxhUfOluFtLzSqq87169emNbxG0E772GrLxCpO3+AhZlWTgdEQZXx/L/N5r6m+kSkxsiUosu3vgmdfNA2M00bD19G7P6t4Sxwb8Lps2YMQMzZsyosl5oaCiA8sGgq47cwi5ZL8ie74WHPxKM9KX4v3ZOGN/FFR3drCpNP46Li1P8uyG86VfFwcIIP78SgDGrw3E9KQeT15/B5lcC0MTwyR8R0ck5eHPLeVxPygEATAp0x/xnWsFIX/ML2j3cDTly5EgA/3ZDzpgxAxKJBF882w5By4/idFwGNobHVdoHraJOUdG/a8FUJDbR0dE4fPgwmjZtiri0PMzbfgkPT7e5l1mAjLziWv/7V3ffD7MzM0JbZ3NcvpuNIzdS8VwnF8Vzcrkcmbn5eH7NSZyOzYAoK0XRP0vhJMnEiVPHYGtbP1cifhwmN0RU5/XzsYOrtTFuZxTgpxNxaOdqAU8b0yo/aDLyinE9KRtRSTm4npiD68k5iErMRmFp5aX83w3yxsRAd5gb1d4uznWVe1NTbJoagHE/hCPydiZe2xiBdZP9q0xUhBDYfDIen+65hqJSOZqaGuCrMe3Qz8e+ijNrTnXdkO/PmobmaIJLTkPxxb4oXN23EQN7dVN0Q+7duxebNm1SdDuVlJTgueeew7lz5xD8/c9YfTgaEfGncOF2JoRhE0hk/74uKgbP6yK5Vaf7tU9LOxz//UdsksSjo9VgFBUVYfdff+GnjZtgE/QGkmMzYCwTMDq+AmkZsfj9r79QVlamGL9jbW0NAwPVWu3qMiY3RFTnyaQSTAr0wKd7rmHJvusAyqehz+jnBXdrU0Ql5+BaYnlCk5Kj+gqtndysmNg8xNvBDD9N6YIX1pzEiVvpmPHLOax6sRP0Zf927aXnFmHuHxdx4Fr5Giu9W9riqzHtarx2jTpU6YZ0d/dAYLOmCI9Jx67IGGzZ9BPSkhNhbGwMHx8fbN68Gc8+NwbnEu7jr+MXsHv3bgDAS0N7K13LfvznMHJrp/hZlwPI1el+7etji09KirDvh8+wb9k7MDA0gszKGdZDZ8O4VS/08bbFfzqaodvnBwCUd1k97PDhw5XGI9VHXOeGiOqFG8k5GLTsqEpl3axN4O1gBh8HM/g4mMPa1AATfjypNJW9Pq9To23ht9Ixef1pFJXKMaK9E94L8kZ8Rj5Ssovw2d5rSM0pgoFMirlDfDClmwekVW1gpUO3M/LRf2koisvK/+BSCTC9rxfMjPQQfisdZ+LuI7eoVKlOU1MDdG3eFN2aN0Vgs6Y4HZuB93coDyB/0p5edUWZXKDth/srrept08QAi4e1wf+1c6xy5ef6QJ3Pb7bcEFG9kPaYPXNaPdgWwtvBHD6OZmhpb1blWBFdzPSqrwKbN8WqFzvitY1nsSvyHnZFKi/B38KuCf77fAelLRTqEj2ZBCVl/2aycgF8d+imUhkLY310bWaNbs1tENi8KVrYNVH60G9m2wS9vevfAPKUnMJKiY0EwM+vBMDboW7+vbSByQ0R1QsV6+wot74A66b4q/TB01BnO2lLPx97LBrWGot2XVE6LgGwZmJnxVTruig2LQ9VdUl0dLXEM+0cEdi8KVo5mFfb4lQfB5DHpuVVOiYAZOSV1H4wOsTkhojqBU2ss1MfP6x0qaq9swSAxKzCOp3cVJUISyXAyhc7Nvi/f9VfAur3gpM1weSGiOoNtr7Urvr6QamrBSfrgsZ87w/jgGIiInqsbWcSKn1Q1oeBtQAaxEq7NdUQ712dz28mN0RE9EQN8YOS6h/OliIiIo3hWCWqb7grOBERETUoTG6IiIioQWFyQ0RERA0KkxsiIiJqUJjcEBERUYPC5IaIiIgaFCY3RERE1KAwuSEiIqIGhckNERERNShMboiIiKhBYXJDREREDQqTGyIiImpQmNwQERFRg8LkhoiIiBoUJjdERETUoDC5ISIiogaFyQ0RERE1KExuiIiIqEFhckNEREQNCpMbIiIialCY3BAREVGDwuSGiIiIGhQmN0RERNSgMLkhIiKiBoXJDRERETUoTG6IiIioQWFyQ0RERA0KkxsiIiJqUJjcEBERUYPC5IaIiIgaFCY3RERE1KAwuSEiIqIGhckNERERNShMboiIiKhBYXJDREREDQqTGyIiImpQmNwQERFRg8LkhoiIiBoUJjdERETUoDC5ISIiogaFyQ0RERE1KExuiIiIqEGpE8nNypUr4eHhASMjIwQEBOD06dNPLP/bb7/Bx8cHRkZG8PX1xd69e2spUiIiIqrrdJ7cbNu2DbNnz8bixYtx7tw5+Pn5ISgoCCkpKVWWP3HiBMaPH4+pU6fi/PnzGDlyJEaOHInLly/XcuRERERUF0mEEEKXAQQEBMDf3x8rVqwAAMjlcri6uuLNN9/EvHnzKpUfN24c8vLy8NdffymOde3aFe3bt8fq1aurvV52djYsLCyQlZUFc3Nzzd0IERERaY06n996tRRTlYqLi3H27FnMnz9fcUwqlWLAgAEIDw+vsk54eDhmz56tdCwoKAg7d+6ssnxRURGKiooUP2dlZQEo/yURERFR/VDxua1Km4xOk5u0tDSUlZXB3t5e6bi9vT2uX79eZZ2kpKQqyyclJVVZPjg4GB999FGl466urjWMmoiIiHQlJycHFhYWTyyj0+SmNsyfP1+ppUculyMjIwNNmzaFRCLR6LWys7Ph6uqK27dvq93l9TR16/O1n7Y+r924rv209XltXru+1G+s134SIQRycnLg5ORUbVmdJjc2NjaQyWRITk5WOp6cnAwHB4cq6zg4OKhV3tDQEIaGhkrHLC0tax60CszNzWv8B32auvX52k9bn9duXNd+2vq8Nq9dX+o31ms/TnUtNhV0OlvKwMAAnTp1wsGDBxXH5HI5Dh48iMDAwCrrBAYGKpUHgJCQkMeWJyIiosZF591Ss2fPxqRJk9C5c2d06dIFy5cvR15eHqZMmQIAmDhxIpydnREcHAwAmDlzJnr37o2lS5di6NCh2Lp1KyIiIvDDDz/o8jaIiIiojtB5cjNu3DikpqZi0aJFSEpKQvv27bFv3z7FoOGEhARIpf82MHXr1g2//PILFi5ciAULFqBFixbYuXMn2rZtq6tbUDA0NMTixYsrdYNpu259vvbT1ue1G9e1n7Y+r81r15f6jfXamqLzdW6IiIiINEnnKxQTERERaRKTGyIiImpQmNwQERFRg8LkhoiIiBoUJjcasnLlSnh4eMDIyAgBAQE4ffq0SvWOHj2KYcOGwcnJCRKJ5LF7ZD1OcHAw/P39YWZmBjs7O4wcORJRUVEq1V21ahXatWunWGgpMDAQf//9t1rXr7BkyRJIJBLMmjVLpfIffvghJBKJ0sPHx0eta969excvvvgimjZtCmNjY/j6+iIiIqLaeh4eHpWuLZFIMH36dJWuW1ZWhg8++ACenp4wNjZG8+bN8cknn6i03wlQvnT4rFmz4O7uDmNjY3Tr1g1nzpypsmx1rw8hBBYtWgRHR0cYGxtjwIABiI6OVrn+9u3bMWjQIMWK3ZGRkSrVLSkpwdy5c+Hr6wtTU1M4OTlh4sSJuHfvnsrX/vDDD+Hj4wNTU1NYWVlhwIABOHXqlEp1HzZt2jRIJBIsX75c5WtPnjy50t9/8ODBKl/72rVrGD58OCwsLGBqagp/f38kJCSoVL+q155EIsFXX31Vbd3c3FzMmDEDLi4uMDY2RuvWrZU2DK6ufnJyMiZPngwnJyeYmJhg8ODBiteLKu8lhYWFmD59Opo2bYomTZrg2WefVSyqqkr9H374AX369IG5uTkkEgkyMzNVqpuRkYE333wT3t7eMDY2hpubG9566y3FXoGqXPs///kPmjdvDmNjY9ja2mLEiBG4fv26Wu+hQggMGTJE6XerSv0+ffpU+ntPmzZN5WuHh4ejX79+MDU1hbm5OXr16oWCgoJq68fFxT329fbCCy9Ue+2kpCS89NJLcHBwgKmpKTp27Ig//vhD5fu+desWRo0aBVtbW5ibm2Ps2LGVFuHVFiY3GrBt2zbMnj0bixcvxrlz5+Dn54egoCCkpKRUWzcvLw9+fn5YuXJlja595MgRTJ8+HSdPnkRISAhKSkowaNAg5OXlVVvXxcUFS5YswdmzZxEREYF+/fphxIgRuHLliloxnDlzBt9//z3atWunVr02bdogMTFR8QgLC1O57v3799G9e3fo6+vj77//xtWrV7F06VJYWVmpFO/D1w0JCQEAjBkzRqVrf/HFF1i1ahVWrFiBa9eu4YsvvsCXX36J7777TqX6r7zyCkJCQrBp0yZcunQJgwYNwoABA3D37t1KZat7fXz55Zf49ttvsXr1apw6dQqmpqYICgpCYWGhSvXz8vLQo0cPfPHFF2pdOz8/H+fOncMHH3yAc+fOYfv27YiKisLw4cNVjr1ly5ZYsWIFLl26hLCwMHh4eGDQoEFITU1V+f/Fjh07cPLkyUrLsatSf/DgwUqvgy1btqhU99atW+jRowd8fHwQGhqKixcv4oMPPoCRkZFK9R++ZmJiItatWweJRIJnn3222rqzZ8/Gvn37sHnzZly7dg2zZs3CjBkzsHv37mqvLYTAyJEjERMTg127duH8+fNwd3fHgAEDkJeXp9J7ydtvv40///wTv/32G44cOYJ79+5h9OjRAFR7L8rPz8fgwYOxYMECpdiqq3vv3j3cu3cPX3/9NS5fvowNGzZg3759mDp1qsrX7tSpE9avX49r165h//79EEJg0KBBCA0NVfk9dPny5ZW27VH1PfjVV19V+rt/+eWXKtUNDw/H4MGDMWjQIJw+fRpnzpzBjBkzIJVKq63v6upa6fX20UcfoUmTJkhNTa322hMnTkRUVBR2796NS5cuYfTo0Rg7dizOnz9f7bXz8vIwaNAgSCQSHDp0CMePH0dxcTGGDRsGuVxe6XercYKeWpcuXcT06dMVP5eVlQknJycRHBys1nkAiB07djxVLCkpKQKAOHLkSI3qW1lZiR9//FHl8jk5OaJFixYiJCRE9O7dW8ycOVOleosXLxZ+fn41ilEIIebOnSt69OhR4/oPmzlzpmjevLmQy+UqlR86dKh4+eWXlY6NHj1aTJgwodq6+fn5QiaTib/++kvpeMeOHcX777//xLqPvj7kcrlwcHAQX331leJYZmamMDQ0FFu2bKm2/sNiY2MFAHH+/HmVrl2V06dPCwAiPj6+RvWzsrIEAHHgwAGV6t65c0c4OzuLy5cvC3d3d7Fs2TKVY580aZIYMWLEE+N5XN1x48aJF198sdq6T4r9YSNGjBD9+vVTqW6bNm3Exx9/rHTsca+dR+tHRUUJAOLy5cuKY2VlZcLW1lasWbOmUv1H30syMzOFvr6++O233xRlrl27JgCI8PDwaus/7PDhwwKAuH//fqXnqqtb4ddffxUGBgaipKSkRvUvXLggAIibN2+qVPf8+fPC2dlZJCYmPvHvWlV9Vd8bq6obEBAgFi5cWG3dJ8X+sPbt21d6/3pcXVNTU7Fx40alctbW1iq9Xvbv3y+kUqnIyspSlMnMzBQSiUSEhISodD9Pgy03T6m4uBhnz57FgAEDFMekUikGDBiA8PDwWo+nopnW2tparXplZWXYunUr8vLy1NrKYvr06Rg6dKjS/asqOjoaTk5OaNasGSZMmKBo1lfF7t270blzZ4wZMwZ2dnbo0KED1qxZo3YMxcXF2Lx5M15++WWVN1Lt1q0bDh48iBs3bgAALly4gLCwMAwZMqTauqWlpSgrK1N8y69gbGysVssVAMTGxiIpKUnpd29hYYGAgACdvfYkEkmN9m4rLi7GDz/8AAsLC/j5+VVbXi6X46WXXsKcOXPQpk2bGkQLhIaGws7ODt7e3nj99deRnp6u0nX37NmDli1bIigoCHZ2dggICFC7O7lCcnIy9uzZo2iBqE63bt2we/du3L17F0IIHD58GDdu3MCgQYOqrVtUVAQASq89qVQKQ0PDKl97j76XnD17FiUlJUqvNx8fH7i5uVX5eqvpe5GqdbOysmBubg49vcpr0VZXPy8vD+vXr4enpydcXV2rrZufn48XXngBK1eufOw+htVd++eff4aNjQ3atm2L+fPnIz8/v9q6KSkpOHXqFOzs7NCtWzfY29ujd+/ej32vqO6+z549i8jIyCpfb1XV7datG7Zt24aMjAzI5XJs3boVhYWF6NOnT7X1i4qKIJFIlBbyMzIyglQqVfu9rka0nj41cHfv3hUAxIkTJ5SOz5kzR3Tp0kWtc+EpW27KysrE0KFDRffu3VWuc/HiRWFqaipkMpmwsLAQe/bsUbnuli1bRNu2bUVBQYEQQvVvJ0IIsXfvXvHrr7+KCxcuiH379onAwEDh5uYmsrOzVapvaGgoDA0Nxfz588W5c+fE999/L4yMjMSGDRtUjl8IIbZt2yZkMpm4e/euynXKysrE3LlzhUQiEXp6ekIikYjPP/9c5fqBgYGid+/e4u7du6K0tFRs2rRJSKVS0bJlyyfWe/T1cfz4cQFA3Lt3T6ncmDFjxNixY6ut/7CnbbkpKCgQHTt2FC+88IJa9f/8809hamoqJBKJcHJyEqdPn1ap7ueffy4GDhyoaG1Tt+Vmy5YtYteuXeLixYtix44dolWrVsLf31+UlpY+sW7Ft3YTExPxzTffiPPnz4vg4GAhkUhEaGioyvdd4YsvvhBWVlaK/0PV1S0sLBQTJ04UAISenp4wMDAQP/30k0r3XVxcLNzc3MSYMWNERkaGKCoqEkuWLBEAxKBBg5TqVvVe8vPPPwsDA4NK1/H39xfvvfdetfUf9qSWG1Xex1JTU4Wbm5tYsGCBWvVXrlwpTE1NBQDh7e1dqdXmcXVfe+01MXXqVMXPj/u7Pq7+999/L/bt2ycuXrwoNm/eLJydncWoUaOqrRseHi4ACGtra7Fu3Tpx7tw5MWvWLGFgYCBu3Lih8n1XeP3110WrVq1Ujvv+/fti0KBBitebubm52L9/v0r1U1JShLm5uZg5c6bIy8sTubm5YsaMGQKAeO211x4bo6YwuXlKdSm5mTZtmnB3dxe3b99WuU5RUZGIjo4WERERYt68ecLGxkZcuXKl2noJCQnCzs5OXLhwQXFMneTmUffv3xfm5uYqd4np6+uLwMBApWNvvvmm6Nq1q1rXHTRokPi///s/teps2bJFuLi4iC1btoiLFy+KjRs3Cmtra5UTq5s3b4pevXoJAEImkwl/f38xYcIE4ePj88R6dTW5KS4uFsOGDRMdOnRQaoJWpX5ubq6Ijo4W4eHh4uWXXxYeHh4iOTn5iXUjIiKEvb29UkKqbnLzqFu3bqnUJVbx/338+PFK5YYNGyaef/55ta/t7e0tZsyYoXLcX331lWjZsqXYvXu3uHDhgvjuu+9EkyZNqmzmr6p+RESE8PPzU7z2goKCxJAhQ8TgwYOVylX1XqJOclPde9GTkpvq6mZlZYkuXbqIwYMHi+LiYrXqZ2Zmihs3bogjR46IYcOGiY4dOyolllXV3bVrl/Dy8hI5OTmKY4/7u6r6Hnzw4MFKXWJV1a34Pz5//nyl+r6+vmLevHlqXTs/P19YWFiIr7/+WuW4Z8yYIbp06SIOHDggIiMjxYcffigsLCzExYsXVaq/f/9+0axZMyGRSIRMJhMvvvii6Nixo5g2bdoTfjuaweTmKRUVFQmZTFbphT5x4kQxfPhwtc71NMnN9OnThYuLi4iJialR/Qr9+/dXKavesWOH4g2y4gFA8SJ+9BuwKjp37lzpP+zjuLm5KX2TEkKI//3vf8LJyUnl68XFxQmpVCp27typVpwuLi5ixYoVSsc++eQT4e3trdZ5cnNzFYnJ2LFjxTPPPPPE8o++Pio+kB9NSHr16iXeeuutaus/rKbJTXFxsRg5cqRo166dSEtLUzn2x/Hy8qrUCvZo3WXLlileZw+/9qRSqXB3d6/xtW1sbMTq1aufWLeoqEjo6emJTz75RKnce++9J7p166bWtY8ePSoAiMjIyCqff7Rufn6+0NfXrzRea+rUqSIoKEita2dmZoqUlBQhRPmYwTfeeEPx3OPeSyo+kB9NSNzc3MQ333xTbf2HPS65qa5udna2CAwMFP3796+ytUud98GioiJhYmIifvnllyfWnTlz5mNfb717967RtXNzcwUAsW/fvifWjYmJEQDEpk2blI6PHTtWqZVUlWtv3LhR6OvrK/7u1dW9efNmpTFaQpR/RvznP/9R69qpqamKv7W9vb348ssvH1tWUzjm5ikZGBigU6dOOHjwoOKYXC7HwYMH1Rq7UlNCCMyYMQM7duzAoUOH4Onp+VTnk8vlir75J+nfvz8uXbqEyMhIxaNz586YMGECIiMjIZPJ1Lpubm4ubt26BUdHR5XKd+/evdK0wxs3bsDd3V3la65fvx52dnYYOnSoWrHm5+crbeYKADKZTO0ZAKampnB0dMT9+/exf/9+jBgxQq36np6ecHBwUHrtZWdn49SpU7Xy2ispKcHYsWMRHR2NAwcOoGnTpk99TlVefy+99BIuXryo9NpzcnLCnDlzsH///hpd986dO0hPT6/29WdgYAB/f/+nfu0BwNq1a9GpUyeVxhgB5b/vkpISjbz2LCwsYGtri+joaERERGDEiBHVvpd06tQJ+vr6Sq+3qKgoJCQkIDAw8Knei1Spm52djUGDBsHAwAC7d+9WGjtUk2uL8i/3KCwsfGLdefPmVXq9AcCyZcuwfv36Gl274hwODg5PrOvh4QEnJ6fHvt7UufbatWsxfPhw2NraKu7/SXUrxgQ97vWmzrVtbGxgaWmJQ4cOISUlRWlWpdZoPX1qBLZu3SoMDQ3Fhg0bxNWrV8Vrr70mLC0tRVJSUrV1c3JyxPnz58X58+cFAEU/flUzTqry+uuvCwsLCxEaGioSExMVj/z8/Grrzps3Txw5ckTExsaKixcvinnz5gmJRCL++ecfla79KHW6pd555x0RGhoqYmNjxfHjx8WAAQOEjY1NpW8Vj3P69Gmhp6cnPvvsMxEdHS1+/vlnYWJiIjZv3qxS/bKyMuHm5ibmzp2rUvmHTZo0STg7O4u//vpLxMbGiu3btwsbG5tKTfOPs2/fPvH333+LmJgY8c8//wg/Pz8REBBQZRN7da+PJUuWCEtLS8X4kREjRghPT0/Ft9rq6qenp4vz58+LPXv2CABi69at4vz58yIxMfGJdYuLi8Xw4cOFi4uLiIyMVHrtFRUVVXvt3NxcMX/+fBEeHi7i4uJERESEmDJlijA0NBSXL19W+//Fo91ST6qfk5Mj3n33XREeHi5iY2PFgQMHRMeOHUWLFi1EYWFhtdfevn270NfXFz/88IOIjo4W3333nZDJZOLYsWMq/c6FKO9aMTExEatWrVLr7927d2/Rpk0bcfjwYRETEyPWr18vjIyMxP/+9z+V6v/666/i8OHD4tatW2Lnzp3C3d1djB49Wgih2nvJtGnThJubmzh06JCIiIgQgYGBiu5hVeonJiaK8+fPizVr1ggA4ujRo+L8+fNiypQpT6yblZUlAgIChK+vr7h586ZSmdLS0mqvfevWLfH555+LiIgIER8fL44fPy6GDRsmrK2txeTJk9V+D8VDrWLVXfvmzZvi448/FhERESI2Nlbs2rVLNGvWTPTq1Uul39myZcuEubm5+O2330R0dLRYuHChMDIyEjdv3lT5/T86OlpIJBLx999/K45VV7e4uFh4eXmJnj17ilOnTombN2+Kr7/+WkgkErFnzx6Vrr1u3ToRHh4ubt68KTZt2iSsra3F7NmzH/t71SQmNxry3XffCTc3N2FgYCC6dOkiTp48qVK9iubZRx+TJk1SqX5VdQGI9evXV1v35ZdfFu7u7sLAwEDY2tqK/v371zixEUK95GbcuHHC0dFRGBgYCGdnZzFu3LhKg/uq8+eff4q2bdsKQ0ND4ePjI3744QeV6+7fv18AEFFRUWpdU4jypvGZM2cKNzc3YWRkJJo1aybef/99xYd6dbZt2yaaNWsmDAwMhIODg5g+fbrIzMyssmx1rw+5XC4++OADYW9vLwwNDUX//v2V7qm6+uvXr6/y+cWLFz+xbkU3VlWPw4cPV3vtgoICMWrUKOHk5CQMDAyEo6OjGD58uGJAsbr/Lx5Nbp5UPz8/XwwaNEjY2toKfX194e7uLl599VXFlxFVrr127Vrh5eUljIyMhJ+fn1LXpir1v//+e2FsbFzp715d3cTERDF58mTh5OQkjIyMhLe3t1i6dKliYHV19f/73/8KFxcXoa+vL9zc3MTChQsVr1tV3ksKCgrEG2+8IaysrISJiYkYNWqUSExMVLn+4sWLH1vuSXUfd18AnvharKh/9+5dMWTIEGFnZyf09fWFi4uLeOGFF8T169dr9B76cHJTXf2EhATRq1cvYW1tLQwNDYWXl5eYM2eOYukDVa4dHBwsXFxchImJiQgMDFQk0qrWnz9/vnB1dRVlZWVK91Bd3Rs3bojRo0cLOzs7YWJiItq1a6eYGq5K/blz5wp7e3uhr68vWrRoofRa1TbJgyCJiIiIGgSOuSEiIqIGhckNERERNShMboiIiKhBYXJDREREDQqTGyIiImpQmNwQERFRg8LkhoiIiBoUJjdERETUoDC5IaJGLzQ0FBKJBJmZmboOhYg0gMkNERERNShMboiIiKhBYXJDRDonl8sRHBwMT09PGBsbw8/PD7///juAf7uM9uzZg3bt2sHIyAhdu3bF5cuXlc7xxx9/oE2bNjA0NISHhweWLl2q9HxRURHmzp0LV1dXGBoawsvLC2vXrlUqc/bsWXTu3BkmJibo1q0boqKitHvjRKQVTG6ISOeCg4OxceNGrF69GleuXMHbb7+NF198EUeOHFGUmTNnDpYuXYozZ87A1tYWw4YNQ0lJCYDypGTs2LF4/vnncenSJXz44Yf44IMPsGHDBkX9iRMnYsuWLfj2229x7do1fP/992jSpIlSHO+//z6WLl2KiIgI6Onp4eWXX66V+ycizeKu4ESkU0VFRbC2tsaBAwcQGBioOP7KK68gPz8fr732Gvr27YutW7di3LhxAICMjAy4uLhgw4YNGDt2LCZMmIDU1FT8888/ivrvvfce9uzZgytXruDGjRvw9vZGSEgIBgwYUCmG0NBQ9O3bFwcOHED//v0BAHv37sXQoUNRUFAAIyMjLf8WiEiT2HJDRDp18+ZN5OfnY+DAgWjSpInisXHjRty6dUtR7uHEx9raGt7e3rh27RoA4Nq1a+jevbvSebt3747o6GiUlZUhMjISMpkMvXv3fmIs7dq1U/zb0dERAJCSkvLU90hEtUtP1wEQUeOWm5sLANizZw+cnZ2VnjM0NFRKcGrK2NhYpXL6+vqKf0skEgDl44GIqH5hyw0R6VTr1q1haGiIhIQEeHl5KT1cXV0V5U6ePKn49/3793Hjxg20atUKANCqVSscP35c6bzHjx9Hy5YtIZPJ4OvrC7lcrjSGh4gaLrbcEJFOmZmZ4d1338Xbb78NuVyOHj16ICsrC8ePH4e5uTnc3d0BAB9//DGaNm0Ke3t7vP/++7CxscHIkSMBAO+88w78/f3xySefYNy4cQgPD8eKFSvwv//9DwDg4eGBSZMm4eWXX8a3334LPz8/xMfHIyUlBWPHjtXVrRORljC5ISKd++STT2Bra4vg4GDExMTA0tISHTt2xIIFCxTdQkuWLMHMmTMRHR2N9u3b488//4SBgQEAoGPHjvj111+xaNEifPLJJ3B0dMTHH3+MyZMnK66xatUqLFiwAG+88QbS09Ph5uaGBQsW6OJ2iUjLOFuKiOq0iplM9+/fh6Wlpa7DIaJ6gGNuiIiIqEFhckNEREQNCruliIiIqEFhyw0RERE1KExuiIiIqEFhckNEREQNCpMbIiIialCY3BAREVGDwuSGiIiIGhQmN0RERNSgMLkhIiKiBuX/AU6kMMY/SanhAAAAAElFTkSuQmCC", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkAAAAGwCAYAAABB4NqyAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAACD4UlEQVR4nO3deVxUVf8H8M+dYUcYVGRTNlfEBXEF10zFtFyykjIxn7Sy7JdLm6TmUrm0SWVqi8tji8uTpJaWYom4oKaCKyoqCsIgojKsgjDn9wcyObLNwAzD8nm/XvNK7pxz7/fiaebruWeRhBACRERERA2IzNQBEBEREdU0JkBERETU4DABIiIiogaHCRARERE1OEyAiIiIqMFhAkREREQNDhMgIiIianDMTB1AbaRWq5GSkgI7OztIkmTqcIiIiEgHQghkZWXBzc0NMlnFfTxMgMqQkpICd3d3U4dBREREVZCUlIQWLVpUWIYJUBns7OwAFP8C7e3tTRwNERER6SIzMxPu7u6a7/GKMAEqQ8ljL3t7eyZAREREdYwuw1c4CJqIiIgaHCZARERE1OAwASIiIqIGx6QJ0OLFi9GjRw/Y2dnByckJo0ePxoULFyqtt2/fPnTr1g1WVlZo2bIlVq1aVarMli1b4OvrC0tLS/j6+uLXX381xi0QERFRHWTSBGjfvn2YOnUqDh8+jIiICBQWFiIoKAg5OTnl1klISMDw4cPRr18/xMTE4L333sMbb7yBLVu2aMpER0cjODgYISEhOHnyJEJCQjB27FgcOXKkJm6LiIiIajlJCCFMHUSJmzdvwsnJCfv27UP//v3LLPPuu+9i+/btiIuL0xybMmUKTp48iejoaABAcHAwMjMz8ccff2jKPPbYY2jcuDE2bNhQaRyZmZlQKBRQqVScBUZERFRH6PP9XavGAKlUKgBAkyZNyi0THR2NoKAgrWNDhw7FsWPHcO/evQrLHDp0qMxz5ufnIzMzU+tFRERE9VetSYCEEJg5cyb69u2Ljh07llsuNTUVzs7OWsecnZ1RWFiI9PT0CsukpqaWec7FixdDoVBoXlwFmoiIqH6rNQnQ66+/jlOnTun0iOrhBY5KnuI9eLysMuUtjBQaGgqVSqV5JSUl6Rs+ERER1SG1YiXo//u//8P27dsRFRVV6d4dLi4upXpy0tLSYGZmhqZNm1ZY5uFeoRKWlpawtLSsxh0QERFRXWLSHiAhBF5//XWEh4fj77//hre3d6V1AgMDERERoXVs9+7d6N69O8zNzSss07t3b8MFT0RERHWWSROgqVOn4scff8TPP/8MOzs7pKamIjU1FXl5eZoyoaGhmDBhgubnKVOm4Nq1a5g5cybi4uKwZs0arF69Gm+99ZamzLRp07B7924sXboU58+fx9KlS7Fnzx5Mnz69Jm+vTEpVHg5dTodSlVd5YSIiIjIKk06DL29Mztq1azFx4kQAwMSJE3H16lVERkZq3t+3bx9mzJiBs2fPws3NDe+++y6mTJmidY5ffvkFc+bMwZUrV9CqVSt89NFHGDNmjE5xGWsa/H8PJWD+b+cgBCCTgMVjOiG4h4fBzk9ERNSQ6fP9XavWAaotjJEAKVV56L3kbzz425ZLEg7MGghXhbVBrkFERNSQ1dl1gOqzhPQcPJxqFgmBq+m5pgmIiIioAWMCVEO8HW0he+iJn1yS4OVoY5qAiIiIGjAmQDXEVWGNxWM6aX6WScCiMR35+IuIiMgEmADVoOAeHgjyLV6L6JUBrTgAmoiIyESYANWwjs0VAICbWfkmjoSIiKjhYgJUw1o1awQAuHwz28SREBERNVxMgGpYKydbAMDltGxwBQIiIiLTYAJUw7yaFs8Gy7xbiJvZfAxGRERkCkyAapiVuRzuTYqnvl9OyzFxNERERA0TEyAT4DggIiIi02ICZAKtmt0fB8QEiIiIyCSYAJnAvz1AfARGRERkCkyATKCV0/0EKI09QERERKbABMgEWt/vAUrOyENuQaGJoyEiImp4mACZQGNbCzSxtQAAXOFjMCIiohrHBMhEOBCaiIjIdJgAmQgHQhMREZkOEyAT4VpAREREpsMEyERacyYYERGRyTABMpGSHqAr6TkoUnNTVCIioprEBMhEmje2hoWZDAWFaiTfyTN1OERERA0KEyATkcsktHTkTDAiIiJTYAJkQhwITUREZBpMgEyIawERERGZBhMgEyrZE+wSZ4IRERHVKCZAJsTFEImIiEyDCZAJtbz/COx2TgFu5xSYOBoiIqKGgwmQCdlYmKG5gzUA4ArHAREREdUYJkAm1pIDoYmIiGocEyATa82B0ERERDWOCZCJcSA0ERFRzWMCZGJcDJGIiKjmMQEysVZOxWOAkm7n4u69IhNHQ0RE1DAwATKxZo0sYWdlBrUArt3KNXU4REREDYJJE6CoqCiMGDECbm5ukCQJW7durbD8xIkTIUlSqVeHDh00ZdatW1dmmbt37xr5bqpGkiQ+BiMiIqphJk2AcnJy4Ofnh+XLl+tU/osvvoBSqdS8kpKS0KRJEzzzzDNa5ezt7bXKKZVKWFlZGeMWDIIzwYiIiGqWmSkvPmzYMAwbNkzn8gqFAgqFQvPz1q1bcefOHfznP//RKidJElxcXAwWp7GxB4iIiKhm1ekxQKtXr8bgwYPh6empdTw7Oxuenp5o0aIFnnjiCcTExFR4nvz8fGRmZmq9ahJ3hSciIqpZdTYBUiqV+OOPPzB58mSt4z4+Pli3bh22b9+ODRs2wMrKCn369EF8fHy551q8eLGmd0mhUMDd3d3Y4Wsp2RX+cloO1GpRo9cmIiJqiOpsArRu3To4ODhg9OjRWscDAgIwfvx4+Pn5oV+/fti8eTPatm2Lr776qtxzhYaGQqVSaV5JSUlGjl6bRxMbmMkk5N0rQmpm7RysTUREVJ+YdAxQVQkhsGbNGoSEhMDCwqLCsjKZDD169KiwB8jS0hKWlpaGDlNn5nIZvBxtcSktG5fSsuF2f4NUIiIiMo462QO0b98+XLp0CZMmTaq0rBACsbGxcHV1rYHIqo7jgIiIiGqOSXuAsrOzcenSJc3PCQkJiI2NRZMmTeDh4YHQ0FAkJydj/fr1WvVWr16NXr16oWPHjqXOuWDBAgQEBKBNmzbIzMzEl19+idjYWHz99ddGv5/qKJ4JdoMJEBERUQ0waQJ07NgxDBw4UPPzzJkzAQAvvPAC1q1bB6VSicTERK06KpUKW7ZswRdffFHmOTMyMvDyyy8jNTUVCoUC/v7+iIqKQs+ePY13IwagmQqfxk1RiYiIjE0SQnDa0UMyMzOhUCigUqlgb29fI9eMTcrA6K8PwsnOEkdnD66RaxIREdUn+nx/18kxQPVRy/tjgNKy8pF5956JoyEiIqrfmADVEvZW5nC2L56JdplbYhARERkVE6Ba5N8tMTgOiIiIyJiYANUi3BOMiIioZjABqkU0awHxERgREZFRMQGqRTR7grEHiIiIyKiYANUire8nQNdu5eJekdrE0RAREdVfTIBqERd7K9hYyFGoFrh2K9fU4RAREdVbTIBqEUmSOBCaiIioBjABqmW4KSoREZHxMQGqZbgnGBERkfExAaplOBOMiIjI+JgA1TIlM8Eup2WD+9QSEREZBxOgWsazqQ1kEpCVX4ibWfmmDoeIiKheYgJUy1iayeHRxAYAcImPwYiIiIyCCVAtxE1RiYiIjIsJUC3U6oFxQERERGR4TIBqodZcDJGIiMiomADVQq2cuCs8ERGRMTEBqoVaOhb3AKWo7iInv9DE0RAREdU/TIBqoca2FmhqawEASEjnQGgiIiJDYwJUS3FTVCIiIuNhAlRLcRwQERGR8TABqqVKeoC4GCIREZHhMQGqpf5dC4hjgIiIiAyNCVAtVbIWUEJ6DorU3BSViIjIkJgA1VJuDtawNJOhoEiN63dyTR0OERFRvcIEqJaSyyR4O94fCM1xQERERAbFBKgWa31/HNAlzgQjIiIyKCZAtZhmLSAOhCYiIjIoJkC1mGYmGB+BERERGRQToFqsVTOOASIiIjIGJkC1WMmmqHdy7+F2ToGJoyEiIqo/mADVYtYWcjR3sAbAXiAiIiJDMmkCFBUVhREjRsDNzQ2SJGHr1q0Vlo+MjIQkSaVe58+f1yq3ZcsW+Pr6wtLSEr6+vvj111+NeBfGxZlgREREhmfSBCgnJwd+fn5Yvny5XvUuXLgApVKpebVp00bzXnR0NIKDgxESEoKTJ08iJCQEY8eOxZEjRwwdfo34dyYYEyAiIiJDMTPlxYcNG4Zhw4bpXc/JyQkODg5lvhcWFoYhQ4YgNDQUABAaGop9+/YhLCwMGzZsqE64JqHZFZ6PwIiIiAymTo4B8vf3h6urKwYNGoS9e/dqvRcdHY2goCCtY0OHDsWhQ4fKPV9+fj4yMzO1XrWFpgfoJtcCIiIiMpQ6lQC5urri22+/xZYtWxAeHo527dph0KBBiIqK0pRJTU2Fs7OzVj1nZ2ekpqaWe97FixdDoVBoXu7u7ka7B32VJEBJd3Jx916RiaMhIiKqH0z6CExf7dq1Q7t27TQ/BwYGIikpCZ9++in69++vOS5JklY9IUSpYw8KDQ3FzJkzNT9nZmbWmiTIsZEFFNbmUOXdQ0J6Dtq72ps6JCIiojqvTvUAlSUgIADx8fGan11cXEr19qSlpZXqFXqQpaUl7O3ttV61hSRJXBCRiIjIwOp8AhQTEwNXV1fNz4GBgYiIiNAqs3v3bvTu3bumQzMY7glGRERkWCZNgLKzsxEbG4vY2FgAQEJCAmJjY5GYmAig+NHUhAkTNOXDwsKwdetWxMfH4+zZswgNDcWWLVvw+uuva8pMmzYNu3fvxtKlS3H+/HksXboUe/bswfTp02vy1iqlzxpID+8JdvDgQZiZmaFLly5a5c6ePYunnnoKXl5ekCQJYWFhRoqeiIiobjNpAnTs2DH4+/vD398fADBz5kz4+/vj/fffBwAolUpNMgQABQUFeOutt9C5c2f069cPBw4cwI4dOzBmzBhNmd69e2Pjxo1Yu3YtOnfujHXr1mHTpk3o1atXzd5cJfRZA+nfmWDZUKlUmDBhAgYNGlSqXG5uLlq2bIklS5bAxcXF4DETERHVF5IQQpg6iNomMzMTCoUCKpWqRsYDSZKEX3/9FaNHjy7z/Ss3s/HoZ/tgbS5H54vr0LZtG8jlcmzdulXTe/YwLy8vTJ8+vdb1fBERERmLPt/fdX4MUEPg0cQG5nIJN4//ifPx8Zg3b56pQyIiIqrT6tQ0+IbKTC6Dk/oOruz7L8LC/4CZGf/aiIiIqoM9QHVAUVER4jctgkPfcSiwLX86PxEREemGXQl1QFZWFm4mnAOunsfkPd/gZQlQq9UQQsDMzAy7d+/Go48+auowiYiI6gz2ANUB9vb2WLYpAq7/+RJBc9YhNjYWU6ZMQbt27RAbG1vrZrgRERHVduwBMpHs7GxcunRJ83PJGkhNmjSBh4cHQkNDkZycjPXr10Mmk+HR3t0RdiIf6eaW6NixI5ycnGBlZYWOHTtqzlFQUIBz585p/pycnIzY2Fg0atQIrVu3rvF7JCIiqq3YA2Qi+q6B1PL+WkDp2fm4mJpV5jlTUlI051Qqlfj000/h7++PyZMnG/luiIiI6hauA1SGml4HSBeb/knEu1tOAwAkCVgyphOCe3iYOCoiIqLag+sA1TNKVR5Cw09rfhYCeC/8DJSqPBNGRUREVHcxAaoDEtJzoH6on65ICFxNzzVNQERERHUcE6A6wNvRFjJJ+5hMArwcbUwTEBERUR3HBKgOcFVYY/GYTpA/kAT9p483XBXWpguKiIioDmMCVEcE9/DAgVmP4lEfJwDAvSK1iSMiIiKqu5gA1SGuCms836t45teeczfACXxERERVwwSojunT2hFW5jKkqO7inDLT1OEQERHVSUyA6hgrczn6tWkGANhzLs3E0RAREdVNTIDqoCHti3eE3xN3w8SREBER1U1MgOqggT5OkCTgdLKKiyESERFVAROgOqiZnSX83R0AAH/F8TEYERGRvpgA1VGDfYsfg/3Fx2BERER6YwJUR5WMAzp4+RZy8gtNHA0REVHdwgSojmrt1AieTW1QUKjG/vh0U4dDRERUpzABqqMkScIgH84GIyIiqgomQHXYYN/ibTH+Pp+Gooe3iyciIqJyMQGqw3p4NYG9lRlu5xQgJvGOqcMhIiKqM5gA1WHmchkG3t8cNYKPwYiIiHTGBKiOG9y+ZDo81wMiIiLSFROgOm5Au2Ywk0m4lJaNhPQcU4dDRERUJzABquPsrcwR0LIpAC6KSEREpCsmQPXA4Pb3xwGdYwJERESkCyZA9cCg++OAjl27gzs5BSaOhoiIqPZjAlQPuDexgY+LHYrUApEXORiaiIioMkyA6omS2WB7zjEBIiIiqgwToHqiZHf4fRdvoqBQbeJoiIiIajeTJkBRUVEYMWIE3NzcIEkStm7dWmH58PBwDBkyBM2aNYO9vT0CAwOxa9curTLr1q2DJEmlXnfv3jXinZhe5+YKNLOzRHZ+IY4k3DJ1OERERLWaSROgnJwc+Pn5Yfny5TqVj4qKwpAhQ7Bz504cP34cAwcOxIgRIxATE6NVzt7eHkqlUutlZWVljFuoNWQySTMbbA9ngxEREVXITN8KeXl5EELAxsYGAHDt2jX8+uuv8PX1RVBQkF7nGjZsGIYNG6Zz+bCwMK2fFy1ahG3btuG3336Dv7+/5rgkSXBxcdErlvpgcHtnbDiahD1xaZg/UkCSJFOHREREVCvp3QM0atQorF+/HgCQkZGBXr164bPPPsOoUaOwcuVKgwdYEbVajaysLDRp0kTreHZ2Njw9PdGiRQs88cQTpXqIHpafn4/MzEytV13Up7UjrMxlSM7IQ5wyy9ThEBER1Vp6J0AnTpxAv379AAC//PILnJ2dce3aNaxfvx5ffvmlwQOsyGeffYacnByMHTtWc8zHxwfr1q3D9u3bsWHDBlhZWaFPnz6Ij48v9zyLFy+GQqHQvNzd3WsifIOzMpejX5tmAIA9XBWaiIioXHonQLm5ubCzswMA7N69G2PGjIFMJkNAQACuXbtm8ADLs2HDBsyfPx+bNm2Ck5OT5nhAQADGjx8PPz8/9OvXD5s3b0bbtm3x1VdflXuu0NBQqFQqzSspKakmbsEohpRMh2cCREREVC69E6DWrVtj69atSEpKwq5duzTjftLS0mBvb2/wAMuyadMmTJo0CZs3b8bgwYMrLCuTydCjR48Ke4AsLS1hb2+v9aqrBvo4QZKAU9dVuJFZv2e+ERERVZXeCdD777+Pt956C15eXujVqxcCAwMBFPcGPTgQ2Vg2bNiAiRMn4ueff8bjjz9eaXkhBGJjY+Hq6mr02GqDZnaW6OLuAAD4K46LIhIREZVF71lgTz/9NPr27QulUgk/Pz/N8UGDBuHJJ5/U61zZ2dm4dOmS5ueEhATExsaiSZMm8PDwQGhoKJKTkzWDrjds2IAJEybgiy++QEBAAFJTUwEA1tbWUCgUAIAFCxYgICAAbdq0QWZmJr788kvExsbi66+/1vdW66zB7Z0Rk5iBPXE3MK6Xh6nDISIiqnWqtA6Qi4sL/P39IZPJkJmZia1bt8LOzg4+Pj56nefYsWPw9/fX9BzNnDkT/v7+eP/99wEASqUSiYmJmvLffPMNCgsLMXXqVLi6umpe06ZN05TJyMjAyy+/jPbt2yMoKAjJycmIiopCz549q3KrddKQ+6tCH7iUjtyCQhNHQ0REVPtIQgihT4WxY8eif//+eP3115GXlwc/Pz9cvXoVQghs3LgRTz31lLFirTGZmZlQKBRQqVR1cjyQEAIDPolE4u1cfBPSDUM7NLw1kYiIqOHR5/tb7x6gqKgozTT4X3/9FUIIZGRk4Msvv8SHH35YtYjJoCRJemBzVM4GIyIiepjeCZBKpdIsPPjnn3/iqaeego2NDR5//PEKZ1pRzRrsW7w0wN/n01Ck1quTj4iIqN7TOwFyd3dHdHQ0cnJy8Oeff2qmwd+5c6fe77dVl/TwagJ7KzPcyilAbNIdU4dDRERUq+idAE2fPh3PP/88WrRoATc3NzzyyCMAih+NderUydDxURWZy2UY6HN/c1ROhyciItKidwL02muvITo6GmvWrMGBAwcgkxWfomXLlhwDVMsM4jggIiKiMum9DhAAdO/eHd27d4cQAkIU7zquy6KEVLMGtG0GM5mE+LRsXE3PgZejralDIiIiqhWqtA7Q+vXr0alTJ1hbW8Pa2hqdO3fGDz/8YOjYqJoU1ubo1bJ4wDr3BiMiIvqX3gnQ559/jldffRXDhw/H5s2bsWnTJjz22GOYMmUKli1bZowYqRoGc3NUIiKiUvReCNHb2xsLFizAhAkTtI7/97//xfz585GQkGDQAE2hri+E+KCk27no9/FeyGUSjs8ZDAcbC1OHREREZBRGXQhRqVSid+/epY737t0bSqVS39ORkbk3sYGPix2K1AKRF26aOhwiIqJaQe8EqHXr1ti8eXOp45s2bUKbNm0MEhQZVsljsAg+BiMiIgJQhVlgCxYsQHBwMKKiotCnTx9IkoQDBw7gr7/+KjMxItMb7OuM5XsvIerCTRQUqmFhVqWx70RERPWG3t+ETz31FI4cOQJHR0ds3boV4eHhcHR0xNGjR/Hkk08aI0aqps7NFWhmZ4ms/EKsO5gApSrP1CERERGZlN6DoBuC+jQIusQzqw7hn6vFW2LIJGDxmE4I7uFh4qiIiIgMR5/vb50egWVmZup88fqSMNQnSlUejl37dz8wtQDeCz+D/m2bwVVhbcLIiIiITEOnBMjBwQGSJFVYpmRF6KKiIoMERoaTkJ6Dh/v5ioTA1fRcJkBERNQg6ZQA7d2719hxkBF5O9pCJhX3/JSQS4CXo43pgiIiIjIhnRKgAQMGGDsOMiJXhTUWj+mE0PDTmiRoYh9v9v4QEVGDxfnQDURwDw8cnPUoHutYvCZQnFL3cV1ERET1DROgBsRVYY25T3SAmUzCocu3EJuUYeqQiIiITIIJUAPT3MEao7o0BwCsirxs4miIiIhMgwlQAzRlQEsAwK5zqbiUlm3iaIiIiGpelRKgwsJC7NmzB9988w2ysrIAACkpKcjO5pdpXdDG2Q5DfJ0hBPBtFHuBiIio4dE7Abp27Ro6deqEUaNGYerUqbh5s3iH8Y8//hhvvfWWwQMk43j1kVYAgF9jkrk1BhERNTh6J0DTpk1D9+7dcefOHVhb/zuN+sknn8Rff/1l0ODIeLp6NEYv7ya4VySwen+CqcMhIiKqUXonQAcOHMCcOXNgYWGhddzT0xPJyckGC4yMr6QX6OejicjILTBxNERERDVH7wRIrVaXud3F9evXYWdnZ5CgqGYMaNsM7V3tkVtQhPXR10wdDhERUY3ROwEaMmQIwsLCND9LkoTs7GzMmzcPw4cPN2RsZGSSJGl6gdYeTEBuQaGJIyIiIqoZeidAy5Ytw759++Dr64u7d+9i3Lhx8PLyQnJyMpYuXWqMGMmIhnd0gUcTG9zJvYfN/ySZOhwiIqIaIQnx8D7hlcvLy8OGDRtw4sQJqNVqdO3aFc8//7zWoOi6LDMzEwqFAiqVCvb29qYOx+h+PHwNc7aeQXMHa0S+/QjM5VweioiI6h59vr+rlADVdw0tAbp7rwh9l+5FenY+Ph/rhzFdW5g6JCIiIr3p8/2t027wD9q+fXuZxyVJgpWVFVq3bg1vb299T0smZGUux4t9vfDxnxewat9ljO7SHDKZZOqwiIiIjEbvBGj06NGQJAkPdxyVHJMkCX379sXWrVvRuHFjgwVKxjU+wBMr917GxRvZ+Pt8Ggb7Ops6JCIiIqPRe7BHREQEevTogYiICKhUKqhUKkRERKBnz574/fffERUVhVu3bnFV6DrG3soczwd4AgBWRF4qleASERHVJ1VaCfrzzz/HoEGDYGdnBzs7OwwaNAiffvop3n77bfTp0wdhYWGIiIio9FxRUVEYMWIE3NzcIEkStm7dWmmdffv2oVu3brCyskLLli2xatWqUmW2bNkCX19fWFpawtfXF7/++qu+t9kgvdjHCxZmMpxIzMA/V++YOhwiIiKj0TsBunz5cpkDi+zt7XHlyhUAQJs2bZCenl7puXJycuDn54fly5frdO2EhAQMHz4c/fr1Q0xMDN577z288cYb2LJli6ZMdHQ0goODERISgpMnTyIkJARjx47FkSNHdLzDhsvJ3gpPdyseAL0y8pKJoyEiIjIevROgbt264e2339ZsggoAN2/exDvvvIMePXoAAOLj49GiReUziYYNG4YPP/wQY8aM0enaq1atgoeHB8LCwtC+fXtMnjwZL774Ij799FNNmbCwMAwZMgShoaHw8fFBaGgoBg0apLV4IxVbsWIFvL29YWVlhW7dumH//v14uV9LyCRg74WbiFNmapX/+uuv0b59e1hbW6Ndu3ZYv359uefeuHEjJEnC6NGjjXwXRERE+tM7AVq9ejUSEhLQokULtG7dGm3atEGLFi1w9epVfP/99wCA7OxszJ071+DBRkdHIygoSOvY0KFDcezYMdy7d6/CMocOHSr3vPn5+cjMzNR61XebNm3C9OnTMXv2bMTExKBfv34YNmwYZLm3MLyTKwBg1b7LmvIrV65EaGgo5s+fj7Nnz2LBggWYOnUqfvvtt1LnvnbtGt566y3069evxu6HiIhIH3rPAmvXrh3i4uKwa9cuXLx4EUII+Pj4YMiQIZDJivMpY/2rPzU1Fc7O2rOTnJ2dUVhYiPT0dLi6upZbJjU1tdzzLl68GAsWLDBKzLXV559/jkmTJmHy5MkAinvOdu3ahZUrV2LK67Pw+yklfjuZgjeHtINHUxv88MMPeOWVVxAcHAwAaNmyJQ4fPoylS5dixIgRmvMWFRXh+eefx4IFC7B//35kZGSY4vaIiIgqVKUlfyVJwmOPPYY33ngD06ZNw9ChQzXJj7FJkvb6NCWzlR48XlaZh489KDQ0VDOjTaVSISmpfm8JUVBQgOPHj5fqKQsKCsKhQ4fQsbkC/ds2g1oA3+0vHteVn58PKysrrfLW1tY4evSopvcNABYuXIhmzZph0qRJxr8RIiKiKtK7BwgoHry8b98+JCYmoqCgQOu9N954wyCBlcXFxaVUT05aWhrMzMzQtGnTCss83Cv0IEtLS1haWho+4FoqPT0dRUVFFfaUvTqgFaIu3sTmY0l4Y1AbDB06FN9//z1Gjx6Nrl274vjx41izZg3u3bun6X07ePAgVq9ejdjYWBPcFRERke70ToBiYmIwfPhw5ObmIicnB02aNEF6ejpsbGzg5ORk1AQoMDCw1JiT3bt3o3v37jA3N9eUiYiIwIwZM7TK9O7d22hx1VUV9ZQFtGyCLu4OiE3KwLpDCZg7dy5SU1MREBAAIQScnZ0xceJEfPzxx5DL5cjKysL48ePx3XffwdHR0RS3Q0REpDO9n1vNmDEDI0aMwO3bt2FtbY3Dhw/j2rVr6Natm9ZsLF1kZ2cjNjZW02OQkJCA2NhYJCYmAih+NDVhwgRN+SlTpuDatWuYOXMm4uLisGbNGqxevVpr0cVp06Zh9+7dWLp0Kc6fP4+lS5diz549mD59ur63Wm85OjpCLpdX2FMmSRJefaQVAGB99DUUSmZYs2YNcnNzcfXqVSQmJsLLywt2dnZwdHTE5cuXcfXqVYwYMQJmZmYwMzPD+vXrsX37dpiZmeHy5cul4iAiIjIZoSeFQiHOnz+v+fO5c+eEEEIcPnxYtGvXTq9z7d27VwAo9XrhhReEEEK88MILYsCAAVp1IiMjhb+/v7CwsBBeXl5i5cqVpc77v//9T7Rr106Ym5sLHx8fsWXLFr3iUqlUAoBQqVR61atLevbsKV599VWtY+3btxezZs3S/FxUpBaPfrpXeL77u1gVeanUOfr37y+ee+45IYQQeXl54vTp01qvUaNGiUcffVScPn1a5OfnG/eGiIiowdPn+1vvR2Dm5uaaxyTOzs5ITExE+/btoVAoND03unrkkUcq3HJh3bp1pY4NGDAAJ06cqPC8Tz/9NJ5++mm9YmloZs6ciZCQEHTv3h2BgYH49ttvkZiYiClTpgAo7n1LTk7GlDc+wtu/nMLyrfthkXAQffsE4s6dO/j8889x5swZ/Pe//wUAWFlZoWPHjlrXcHBwAIBSx4mIiExN7wTI398fx44dQ9u2bTFw4EC8//77SE9Pxw8//IBOnToZI0YyguDgYNy6dQsLFy6EUqlEx44dsXPnTnh6Fu8HplQqkZiYiFFdmuPziItIvHwX733wBTKU12Bhbo6BAwfi0KFD8PLyMu2NEBERVYEkKuqCKcOxY8eQlZWFgQMH4ubNm3jhhRdw4MABtG7dGmvXroWfn5+xYq0xmZmZUCgUUKlUZW770dC8/vMJ/H5KCQCQScDiMZ0Q3MPDxFERERFp0+f7W68eICEEmjVrhg4dOgAAmjVrhp07d1Y9Uqr1lKo87Dyt1PysFsB74WfQv20zuCqsTRgZERFR1ek1C0wIgTZt2uD69evGiodqmYT0HKgf6iMsEgJX03NNExAREZEB6JUAyWQytGnTBrdu3TJWPFTLeDvaQvbQItoyCfBytDFNQERERAag9zpAH3/8Md5++22cOXPGGPFQLeOqsMbiMZ0gf2DRxCc6u/LxFxER1Wl6zwIbP348cnNz4efnBwsLC1hba38R3r5922DBUe0Q3MMD/ds2w8q9l7H+8DUk3s4zdUhERETVoncCFBYWZoQwqLZzVVjj9UGt8eORa4hNykBCeg68HW1NHRYREVGV6J0AvfDCC8aIg+oAJzsr9G3TDFEXb2JbbDKmD25r6pCIiIiqRO8xQABw+fJlzJkzB8899xzS0tIAAH/++SfOnj1r0OCo9hndxQ0AsC02pcJVvImIiGozvROgffv2oVOnTjhy5AjCw8ORnZ0NADh16hTmzZtn8ACpdgnq4AIrcxkS0nNw6rrK1OEQERFVid4J0KxZs/Dhhx8iIiICFhYWmuMDBw5EdHS0QYOj2qeRpRmG+LoAALbGJps4GiIioqrROwE6ffo0nnzyyVLHmzVrxvWBGogn/Ysfg/12MgWFRWoTR0NERKQ/vRMgBwcHKJXKUsdjYmLQvHlzgwRFtVu/Ns3QxNYC6dkFOHiZSS8REdU9eidA48aNw7vvvovU1FRIkgS1Wo2DBw/irbfewoQJE4wRI9Uy5nIZHu/kCgDYFsPHYEREVPfonQB99NFH8PDwQPPmzZGdnQ1fX1/0798fvXv3xpw5c4wRI9VCo+8/Btt1NhW5BYUmjoaIiEg/kqjiXObLly8jJiYGarUa/v7+aNOmjaFjM5nMzEwoFAqoVCrY29ubOpxaSQiB/p/sRdLtPHzxbBeM6sLHn0REZFr6fH9XaRo8ALRq1QpPP/00xo4dW6+SH9KNJEkYfT/p2RabYuJoiIiI9KN3AjRkyBB4eHhg1qxZ3BC1gSvp9Ym6eBO3cwpMHA0REZHu9E6AUlJS8M4772D//v3o3LkzOnfujI8//hjXr183RnxUi7V2aoSOze1RqBbYcYq9QEREVHfonQA5Ojri9ddfx8GDB3H58mUEBwdj/fr18PLywqOPPmqMGKkWK3kM9itngxERUR1Spb3ASnh7e2PWrFlYsmQJOnXqpBkfRA3HSD83yCTgRGIGEm/lmjocIiIinVQ5ATp48CBee+01uLq6Yty4cejQoQN+//13Q8ZGdYCTvRV6t3IEAGzj1hhERFRH6J0Avffee/D29sajjz6Ka9euISwsDKmpqfjxxx8xbNgwY8RItdyo+zvEb41N5g7xRERUJ+idAEVGRuKtt95CcnIyduzYgXHjxsHGxsYYsVEd8VhHF1iayXD5Zg7OJGeaOhwiIqJKmelb4dChQ8aIg+owOytzDPZ1xo5TSmyNTUanFgpTh0RERFQhvROgEufOnUNiYiIKCrTXfxk5cmS1g6K6Z3SX5thxSonfTqbgveHtIZdJpg6JiIioXHonQFeuXMGTTz6J06dPQ5IkzZgPSSr+wisqKjJshFQnDGjbDA425kjLykf05Vvo28bR1CERERGVS+8xQNOmTYO3tzdu3LgBGxsbnD17FlFRUejevTsiIyONECLVBRZm/+4Qr8uaQCtWrIC3tzesrKzQrVs37N+/v9yyEydOhCRJpV4dOnQos/zGjRuLt+oYPbpK90JERPWf3glQdHQ0Fi5ciGbNmkEmk0Emk6Fv375YvHgx3njjDWPESHXEaP/iRRF3nU3F3Xvl9wRu2rQJ06dPx+zZsxETE4N+/fph2LBhSExMLLP8F198AaVSqXklJSWhSZMmeOaZZ0qVvXbtGt566y3069fPMDdFRET1kt4JUFFRERo1agSgeFXolJTiLRA8PT1x4cIFw0ZHdUo3j8Zo7mCN7PxC7Im7UW65zz//HJMmTcLkyZPRvn17hIWFwd3dHStXriyzvEKhgIuLi+Z17Ngx3LlzB//5z3+0yhUVFeH555/HggUL0LJlS4PeGxER1S96J0AdO3bEqVOnAAC9evXCxx9/jIMHD2LhwoX80mngZDLp3zWBYsreG6ygoADHjx9HUFCQ1vGgoCCdZxiuXr0agwcPhqenp9bxkp7JSZMmVSF6IiJqSPQeBD1nzhzk5OQAAD788EM88cQT6NevH5o2bYpNmzYZPECqW570b44VkZex72Ia7uQUoLGthdb76enpKCoqgrOzs9ZxZ2dnpKamVnp+pVKJP/74Az///LPW8YMHD2L16tWIjY2t9j0QEVH9p3cCNHToUM2fW7ZsiXPnzuH27dto3LixZiYYNVxtnO3g62qPc8pM7DitxPgAzzLLPdxWhBA6tZ9169bBwcFBa4BzVlYWxo8fj++++w6Ojpx9RkRElavWZqglmjRpUuXkx9CzgdatW1dmmbt371YpPtLfaP/ix2Bl7Q3m6OgIuVxeqrcnLS2tVK/Qw4QQWLNmDUJCQmBh8W/P0uXLl3H16lWMGDECZmZmMDMzw/r167F9+3aYmZnh8uXLBrgrIiKqTwySAFWVsWYD2dvba5VTKpWwsrKqiVsiACP9mkOSgH+u3sH1O9o7xFtYWKBbt26IiIjQOh4REYHevXtXeN59+/bh0qVLpcb4+Pj44PTp04iNjdW8Ro4ciYEDByI2Nhbu7u6GuTEiIqo3qrwStCE8OBsIAMLCwrBr1y6sXLkSixcvLlVeoVBAofh3m4WtW7eWORtIkiS4uLgYN3gql4vCCoEtm+LQ5VvYFpuCqQNba70/c+ZMhISEoHv37ggMDMS3336LxMRETJkyBQAQGhqK5ORkrF+/Xqve6tWr0atXL3Ts2FHruJWVValjDg4OAFDqOBEREWDCHiBjzgbKzs6Gp6cnWrRogSeeeAIxMTEVnic/Px+ZmZlaL6qe0V2K1wTaGlN6h/jg4GCEhYVh4cKF6NKlC6KiorBz507N36NSqSzVC3ghMRX/+2ULnh4XUjM3QERE9ZokHv52qiEpKSlo3rw5Dh48qPXoY9GiRfjvf/9b6ZpCSqUS7u7u+PnnnzF27FjN8cOHD+PSpUvo1KkTMjMz8cUXX2Dnzp04efIk2rRpU+a55s+fjwULFpQ6rlKpYG9vX8U7bNgy795D9w/3oKBQjR1v9EUHt6pvkLrpn0SEhp+GWgAyCVg8phOCe3gYMFoiIqoPMjMzoVAodPr+NukYIMCws4EAICAgAOPHj4efnx/69euHzZs3o23btvjqq6/KPVdoaChUKpXmlZSUVKV7oX/ZW5ljkI8TAGBbbNlrAlWmsEiNHw9fw7tbipMfAFAL4L3wM1Cq8gwVKhERNUAmGwNkjNlAZZHJZOjRowfi4+PLLWNpaQlLS0vdgyedjPZvjj/OpGJ7bArefcxH5x3iVXn3sPFoIv576CpSVKVn7xUJgavpuXBVWBs6ZCIiaiBM1gNkjNlAZRFCIDY2Fq6urtWKl/T3SLtmsLcyQ2rmXRy5cqvS8gnpOXh/2xkELv4Li/84jxTVXThYm+PhtEkmAV6ONsYJmoiIGgSTzgIz9GwgAFiwYAECAgLQpk0bZGZm4ssvv0RsbCy+/vrrGrkn+pelmRyPd3bFhqNJ2BqbjN6tSy9SKIRA9OVbWH0gAX9fSEPJiDQfFzu82McbI7u4YVtssmYMEAB0bK6Aiz2XNSAioqozaQIUHByMW7duYeHChVAqlejYsWOls4FUKhW2bNmCL774osxzZmRk4OWXX0ZqaioUCgX8/f0RFRWFnj17Gv1+qLRRXZpjw9Ek/HE6FQtHdYSVuRwAcPdeEbafTMGaAwk4n5qlKT/Ixwkv9vVG71ZNNWPBgnt4oH/bZtgTl4Z5287g1HUVtp9Mwaj7M82IiIj0ZbJZYLWZPqPIqWJqtUDfpX8jRXUX0we1wWBfZ0Scu4GfjlxDenYBAMDaXI5nurfAxN5eaNmsUYXn+2JPPJbtuQg7KzP8Ob0/mjtwHBARERXT5/ubCVAZmAAZ1gtrjmLfxZuljrsprPBCby8828MDChtznc5VWKTG06uiEZuUgcCWTfHT5F6Q6Ti4moiI6rc6NQ2e6jelKg9R8aWTnw9Gd0DUOwPxyoBWOic/AGAml2FZcBdYm8sRfeUW1hxMMGS4RETUQDABIqNKSM9BWX2MrZvZwUxetebn7WiLuU/4AgA+/vMCzqdy5W4iItIPEyAyKm9HWzz8hEouSdWexv5cT3cM8nFCQZEa0zfGIr+wqFrnIyKihoUJEBmVq8Iai8d0gvz+jC65JGHRmI7VXsRQkiQseaozmtpa4HxqFj7ffdEQ4RIRUQPBQdBl4CBow1Oq8nA1PRdejjYGXcF599lUvPzDcUgSsOGlAAS0bGqwcxMRUd3CQdBU67gqrBHYqqnBt68I6uCC4O7uEAJ4c/NJZN69Z9DzExFR/cQEiOq8uSN84dHEBskZeZi37aypwyEiojqACRDVeY0szbAs2A8yCfg1Jhm/n6ra7vNERNRwMAGieqGbZxO89khrAMDsX88gtYxd5ImIiEowAaJ6Y9rgNujUXAFV3j28/ctJqNUc309ERGVjAkT1hvn9VaItzWTYH5+O9dFXTR0SERHVUkyAqF5p7dQI7w1vDwBY/Md5xN/IqqQGERE1REyAqN6ZEOiJ/m2bIb9QjembYlFQqDZ1SEREVMswAaJ6R5IkfPJ0ZzjYmONsSiY+2nEOhy6nQ6nKM3VoRERUSzABonrJ2d4Ki57sBAD4b/Q1jPvuCPos+Rub/kk0cWRERFQbMAGiesvfw0HrZ7UA3gs/w54gIiJiAkT1V0J6TqljRULganquCaIhIqLahAkQ1VvejraQSdrHZBLg5WhjmoCIiKjWYAJE9ZarwhqLx3SCXPo3C2pia4HGNhYmjIqIiGoDJkBUrwX38MCBWQPx/YTuaGJrgfTsAnz1d7ypwyIiIhNjAkT1nqvCGoN9nTWzwlbtu4IzySoTR0VERKbEBIgajMc6umB4JxcUqQXe+eUU7hVxgUQiooaKCRA1KAtGdoTC2hznlJn4NuqKqcNpMFasWAFvb29YWVmhW7du2L9/f7llIyMjIUlSqdf58+c1Zc6ePYunnnoKXl5ekCQJYWFhNXAXdY+hf+/fffcd+vXrh8aNG6Nx48YYPHgwjh49WhO3QmRwTICoQWlmZ4n3n/AFAHzxVzwupWWbOKL6b9OmTZg+fTpmz56NmJgY9OvXD8OGDUNiYsWLUl64cAFKpVLzatOmjea93NxctGzZEkuWLIGLi4uxb6FOMsbvPTIyEs899xz27t2L6OhoeHh4ICgoCMnJyca+HSKDk4QQwtRB1DaZmZlQKBRQqVSwt7c3dThkYEIITFz7D/ZdvInuno2x+ZVAyB6eL08G06tXL3Tt2hUrV67UHGvfvj1Gjx6NxYsXlyofGRmJgQMH4s6dO3BwcKj0/F5eXpg+fTqmT59uwKjrPmP/3gGgqKgIjRs3xvLlyzFhwgRDhU5UZfp8f7MHiBocSZKwaEwn2FrIcezaHfxw+JqpQ6q3CgoKcPz4cQQFBWkdDwoKwqFDhyqs6+/vD1dXVwwaNAh79+41Zpj1Tk393nNzc3Hv3j00adKk2jET1TQmQNQgNXewxqxhPgCApX+ex/U7XB3aGNLT01FUVARnZ2et487OzkhNTS2zjqurK7799lts2bIF4eHhaNeuHQYNGoSoqKiaCLleqKnf+6xZs9C8eXMMHjzYoPET1QQzUwdAZCrP9/LE9pMp+OfqHYSGn8b6F3tCkvgozBge/r0KIcr9Xbdr1w7t2rXT/BwYGIikpCR8+umn6N+/v1HjrG+M+Xv/+OOPsWHDBkRGRsLKysqwgRPVAPYAUYNTMjPGxsYacSunoijlHPbHp2PLiYoHch48eBBmZmbo0qVLqffCwsLQrl07WFtbw93dHTNmzMDdu3eNdAd1h6OjI+Ryealeh7S0tFK9ExUJCAhAfDwXsNSVsX/vn376KRYtWoTdu3ejc+fO1Y6XyBSYAFGD8vDMmMEDB+Dm/+ajMDMNH/x+DmlZZSctKpUKEyZMwKBBg0q999NPP2HWrFmYN28e4uLisHr1amzatAmhoaHGvp1az8LCAt26dUNERITW8YiICPTu3Vvn88TExMDV1dXQ4dVbxvy9f/LJJ/jggw/w559/onv37gaJl8gU+AiMGpTPP/8ckyZNwuTJkwEU99zs2rUL9+L/gsr+OczbdhYrx3crVe+VV17BuHHjIJfLsXXrVq33oqOj0adPH4wbNw5A8ayk5557juuj3Ddz5kyEhISge/fuCAwMxLfffovExERMmTIFABAaGork5GSsX78eQPHfiZeXFzp06ICCggL8+OOP2LJlC7Zs2aI5Z0FBAc6dO6f5c3JyMmJjY9GoUSO0bt265m+yFjLG7/3jjz/G3Llz8fPPP8PLy0vTw9SoUSM0atSo5m+SqDoElaJSqQQAoVKpTB0KGVB+fr6Qy+UiPDxc6/gbb7whugf0Ea1CdwjPd38XO0+laL2/Zs0a0b17d3Hv3j0xb9484efnp/X+hg0bhEKhEEeOHBFCCHH58mXh4+MjFi9eLFIycsXBSzdFSkauUe+ttvv666+Fp6ensLCwEF27dhX79u3TvPfCCy+IAQMGaH5eunSpaNWqlbCyshKNGzcWffv2FTt27NA6X0JCggBQ6vXgeUi/3/vsBR+K5h5eFf7ePT09y/y9z5s3r4buiKhi+nx/cx2gMnAdoPopJSUFzZs3x8GDB7UeAyxatAj//e9/8dKX27F87yU4NrLEnpn94WBjgfj4ePTt2xf79+9H27ZtMX/+fGzduhWxsbFa5/7qq6/w5ptvQgiBwsJCvPrqqxjwn1kIDT8NtQBkErB4TCcE9/Co4buum5SqPCSk58Db0RauCmtTh1PvbfonkW2V6oU6tQ6QoZdqB4AtW7bA19cXlpaW8PX1xa+//mrs26A6pLyZMf83qDVaNbNFenY+PtwRh6KiIowbNw4LFixA27Ztyz1fZGQkPvroI6xYsQInTpzAL79swf9+3YZXZr4H9f1/XqgFEBp+GkpVnjFvrV7Y9E8iei/5G+O+O4I+S/7Gpn8qXrmYqkepytMkP0BxW30v/AzbKtV7Jk2AjLFUe3R0NIKDgxESEoKTJ08iJCQEY8eOxZEjR4x9O1TLVTYzxtJMjo+f9oMkAb8cv44/YhJw7NgxvP766zAzM4OZmRkWLlyIkydPwszMDH///TcAYO7cuQgJCcGY5yYg+o4NvrrSGKL7c8g8/AuE+HfDVbUAPvokTOeE/0HlzUBbt25dmf8oqKsz0JSqPMwKPw3BL+Mak5Ceo0l+ShQJgavpXBuL6jeTJkAPDkht3749wsLC4O7urrV0e1mcnJzg4uKiecnlcs17YWFhGDJkCEJDQ+Hj44PQ0FAMGjSImyWSTjNjunk2xsTeXgCARbuv4sjxGMTGxmpeU6ZMQbt27RAbG4tevXpBCIH0jCzsvZCOgMV/YdHO87h6KxdW5mYABPDAE+acuCisWPI+xk+ZrlfCX9EMNACwt7fX+geBUqmss+uyxCkz8fBDeX4ZG5e3oy0eXhpILknwcrQxTUBENcRkCZCxlmqPjo4udc6hQ4dWeM78/HxkZmZqvah+mjlzJr7//nusWbMGcXFxmDFjRqmZMRc2LkaLxtZIyczHzuvm6Nixo+bl5OQEKysreLRuhy2nbuKxsP1QKnxxYtcm3Dm1Fy2tsjGueQYsT/6CXo8EwUxePNFSJgF5J7bBttMQbMxsg+MqGyxbtkynhL9kBlpgYGCZ70uSpPUPgrq8Oei22JRSx/hlbFyuCmsM76g91X3RmI4ce0X1nsmmwVdnqfZu3bohPz8fP/zwAwYNGoTIyEjNSqWpqal6nRMAFi9ejAULFlTzjqguCA4Oxq1bt7Bw4UIolUp07NgRO3fuhKenJwBAqVQiJfk6lszvjPGrj+C/0VcR2LIJ7KzN4e1oi7TMu1Cq7iJg0V/ILSgCADgPGIeOzR2QeOQXREd8hUvNmmHEiBH46KOPkCdZ4mp6LtzszdDqk0sY/NRknC8SmLP1DE4mZeDRwUMqTM7Xrl2Ly5cv48cff8SHH35YZpns7Gx4enqiqKgIXbp0wQcffAB/f3/D//KMbO+FNE0CJEn/dp6FDvfhl7GR5d0r0vp5WCeuuUT1n8nXATLGUu36nBMo/lf/zJkzNT9nZmbC3d1dr/uguuO1117Da6+9VuZ769at0/x5bPcW2HzsOl758cS/BSz6wTq4H3ILitDGqRGe7+WBJ7u2gML6iTLP54Dif2GnpKSgqKgIc58JxOlCF3z853n87/h1WF29i3spyjLrxsfHY9asWdi/fz/MzMr+X9XHxwfr1q1Dp06dkJmZiS+++AJ9+vTByZMntcbG1XYZuQV495dTAID/9PHCy/1bYvz3R3D5Zg7M5Safq1GvFRapcTThNgDAwkyGgkI1TiWp0LeNo4kjIzIuk32yGGupdhcXF73PaWlpCXt7e60X0eS+Lcs8HuTrjM2vBGL3jP6Y2McbCmtznc8pk8kwZUAr/DCpFxrbmOOG6i6SM/KwP/6mVjldZ6AFBARg/Pjx8PPzQ79+/bB582a0bdsWX331lc4x1QbvbzuLtKx8tGxmi3cfK+7xeb5Xca/cttiKtyih6jmTkons/ELYW5lhiG/x52Rs0h0TR0VkfCZLgIy1VHtgYGCpc+7evVuvcxIBQHpOfpnH/9PHGz29m+i1cerDCX+f1o74/Y1+aCzLA6wd8MKao1gReQkly3JlZWXpNAPtYTKZDD169KhT+2btOKXE9pMpkMskfD62C6zMiyc1PNHZFTIJOJGYgcRbHARtLNGXbwEAerVsiq4ejQEAsUkZJoyIqGaY9BGYMZZqnzZtGvr374+lS5di1KhR2LZtG/bs2YMDBw6Y5B6p7vJ2tIVMgtYU4aoOyH0w4X/yyScBAM0drGF54yy6dAtEqgA+/vMCTiZl4NNn/GBvb4/Tp09rnWPFihX4+++/8csvv8Db27vM6wghEBsbi06dOukdoymkZd3FnK3F9/naI63Qxd1B856TvRV6t3LEgUvp+O1UCqYO5BYXxhB9pTgBCmzZFH73f/+xSRmVDh0gqutMmgDpMiD1wSnCBQUFeOutt5CcnAxra2t06NABO3bswPDhwzVlevfujY0bN2LOnDmYO3cuWrVqhU2bNqFXr141fn9Ut7kqrLF4TCe8F34GRUJALknVmh1TVsKflJSIP//8A4duSJg6/S389Hs64tPex7ch3dCxY0et+iUz0B48vmDBAgQEBKBNmzbIzMzEl19+idjYWHz99dfVuveaIIRA6JbTuJN7Dx3c7PF/j5YeszSyixsOXErH1phkvPZIK34hG9i9IjWOXS0e/xPQsilaNrOFuVxCenYBrt/Jg3sTzr6j+svkg6B1HZAKAO+88w7eeeedSs/59NNP4+mnnzZEeNTABffwQP+2zXA1PRdejjbVmo1UUcLv6QlsaG6GyNRbuHIzB6OWH8Snz/ihi4eDZkuIsmRkZODll19GamoqFAoF/P39ERUVhZ49e1Y5zpryv2PX8df5NFjIZfh8bBdYmJV+Iv9YRxfM2XoG8WnZOJ+ahfauHJ9nSKeuZyC3oAiNbczh42IHmUxCe1d7nLquQmxSBhMgqte4F1gZuBcYmUp6dj5e//kEDl8p/le5hOLdJuvb/kxJt3Mx7Iv9yM4vxKxhPpgyoFW5Zaf8cBx/nk3FlAGtMGuYTw1GWf8t/zsen+6+iMc6uGBVSDcAwPvbzmB99DVM6uuNuU/4mjhCIv3Uqb3AiOhfjo0s8eOkXniuZ/EyDCX/OjHElhD67Lt34MAB9OnTB02bNoW1tTV8fHywbNmyUuXCwsLQrl07WFtbw93dHTNmzKh0Gw61WuDtX04iO78Q3T0b46V+Zc+2KzGqixsA4LeTKVA/vGcDVYtm/E+rpppjXR4YB0RUn5n8ERgRaTOTyzDCzw0bjiZpHS/ZEqIqj+FK9t1bsWIF+vTpg2+++QbDhg3DuXPn4OFRulfJ1tYWr7/+Ojp37gxbW1scOHAAr7zyCmxtbfHyyy8DAH766SfMmjULa9asQe/evXHx4kVMnDgRAMpMlkqsO3QVh6/chrW5HJ8+4we5rOJxPQN9nGBnaYbkjDwcT7yDHl5N9L5/Ki2/sAjHrhZPdy8rATqTrMK9IjXXYaJ6iy2bqBYqmYH2sMY2uq859CB9993z9/fHc889hw4dOsDLywvjx4/H0KFDtXqNoqOj0adPH4wbNw5eXl4ICgrCc889h2PHjpUbx6W0bCz98zwA4L3H28OrnLFND7Iyl2Nox+LtPbgmkOHEJmYgv1ANx0YWaOPUSHPc29EWCmtz5BeqcV6ZZcIIiYyLCRBRLVQyA03+0KynaRtjkZap307v1dl3r0RMTAwOHTqEAQMGaI717dsXx48fx9GjRwEAV65cwc6dO/H444+XeY7CIjXe/N9J5Beq0a+NI8b30n08U8ljsB2nlLhXpNa5HpWv5PFXr5ZNtWbXSZL0wHR4LohI9RcfgRHVUg/OQBNCYPqmWFy4kYWx30Tjx8m90KKxbjN0qrLvXokWLVrg5s2bKCwsxPz58zF58mTNe88++yxu3ryJvn37QgiBwsJCvPrqq5g1a1aZ51oZeRknkzJgZ2WGj5/urNeU9sCWTeHYyBLp2fk4EJ+OgT5OOtelsh1+YP2fh3Vxd0DUxZuIScpASNl78BLVeewBIqrFXBXWCGzVFL1bO+KXKb3RorE1rt7KxdhV0bhyM1uvc+m7Rx4A7N+/H8eOHcOqVasQFhaGDRs2aN6LjIzERx99hBUrVuDEiRMIDw/H77//jg8++KDUec4kq/DFX8WrUy8c1UHvcUxmchme6Fy84jsfg1Xf3XtFOJGYAUB7/E8Jfw6EpgaACRBRHeHR1Aa/TOmNVs1skaK6i7HfRCNOmVlpversu+ft7Y1OnTrhpZdewowZMzB//nzNe3PnzkVISAgmT56MTp064cknn8SiRYuwePFiqNX/PqbKLyzCm5tPolAt8FgHF4zu0ly/G7+v5DHY7nM3kFtQWKVzULETiXdQUKiGk50lWpYxDqvkEdiVmzlQ5d6r4eiIagYTIKI6xEVhhU2vBMLX1R7p2QUI/iYaMYkVj9Mw1L57Qgjk5/+7P1pubi5kMu2PELlcDiEEHlxe7POIi7hwIwuOjSzw0ZMdq7yacxd3B3g0sUFuQRH2xKVV6RxU7PDlf6e/l/X30cTWAp5Nix+xnryeUZOhEdUYJkBEdYxjI0tseDkAXT0ckHm3EOO/P6LZ0LI8M2fOxPfff481a9YgLi4OM2bMKLXv3oQJEzTlv/76a/z222+Ij49HfHw81q5di08//RTjx4/XlBkxYgRWrlyJjRs3IiEhAREREZg7dy5GjhwJubx4Q9NjV2/j26grAIBFT3ZC00aWVb5vSZI0vUDb+RisWqIrGP9TgusBUX3HQdBEdZDC2hw/TOqFl384hoOXbmHi2qNYOb4rHvUp+5GWvvvuqdVqvP3uLFy7ehXm5mZo3aoVlixZgldeeUVTZs6cOZAkCXPmzEFycjKaNWuGESNG4KOPPoJSlYc4ZSbmbj0DIYCnurZAUAeXat/3qC5u+OrvS4i8cBN3cgrQ2Nai2udsaPIKijRJTVnjf0p0cXfAttgUJkBUb3ErjDJwKwyqK+7eK8LrP8dgT9wNmMkkhD3bBU90dqv2eTf9k4jQ8NNQC/234XiwLgAorM2w/91HYW9VtTWMHjb8i/04p8zEoic7YZweU+mp2P74mwhZfRRuCiscnPVouY8kYxLv4MkVh9DE1gLH5wzmRrRUJ+jz/c0eIKI6zMpcjpXju+Kt/53EttgUvLEhBrn5RRjbw73K5zyXosKs8NMo+aeRWgCztpzG0YTbsLaQo0hdPB5ILQTUAlALAXH/v9l3C/HXee3xOVl3C5GTX2iwBGhUFzecU2ZiW2wyE6AqKHlcGlDO+J8Svm72sJDLcDunAEm38+DRlBujUv3CBIiojjO/v5u6jYUZNhxNxDtbTiE7vxAv9vWutG7W3Xs4nazCqesqnLqegZNJKiRnlN5vTADYcqJq427UAlXewqMsI/zcsPiP8zh69TZSMvLg5mCY8zYUuoz/AQBLMznau9njZFIGYpLuMAGieocJEFE9IJdJWPRkRzSylOO7/QlY+Ps5pKry8Eg7J3g3s4Wrwhp37xXhnDITp5IycOq6CievZ+BKeg50eQguAZjYxwsKa3PIJAkyCZDJpH//LEmQJAnZd+8hbE88HjylXJLg5Wi4L083B2v09G6Cowm38fupFLzcv/yd5Elbdn4hTl1XAah4/E8Jf3cHnEzKQGxSBkZVcfkCotqKCRBRPSFJEt4b3h6NLM2xbM9FfLs/Ad/uTwAAuCmskJaVj8IydlNv7mANP3cFOrdwQOcWCnRqrsDO00q8F34GRUJALklYNKajzmOAXBRWpeoaqvenxKgubjiacBvbYpkA6eOfq7dRpBZwb2Kt00ri1ZkJtmLFCnzyySdQKpXo0KEDwsLC0K9fvzLLHjhwAO+++y7Onz+P3NxceHp64pVXXsGMGTM0Zc6ePYv3338fx48fx7Vr17Bs2TJMnz5d77iISjABIqpHJEnC2B4tELbnolYvTIqqeP+wprYW6NxCAT93B/i1cECnFgo4ljE1/cFtOLwcbfRKYKpTV1fDO7pi3razOJuSiUtpWWjtZGfwa9RHmvV/Knn8VaIkATqbkomCQjUszHRbOWXTpk2YPn06VqxYgT59+uCbb77BsGHDcO7cOXh4lE6kbW1t8frrr6Nz586wtbXFgQMH8Morr8DW1hYvv/wygOJ1p1q2bIlnnnlGKzEiqiquA0RUzySk56Csp1pfPtcFx+YMxtr/9MT0wW0x0MepzOSnRMk2HFVJYKpTVxeNbS0woG0zAMD22BSjXMNQVqxYAW9vb1hZWaFbt27Yv39/uWUPHDiAPn36oGnTprC2toaPjw+WLVtWqtyWLVvg6+sLS0tL+Pr64tdff9UpFs34Hx0efwGAZ1MbNLYxR0GhWqdVx0t8/vnnmDRpEiZPnoz27dsjLCwM7u7uWLlyZZnl/f398dxzz6FDhw7w8vLC+PHjMXToUK3fVY8ePfDJJ5/g2WefhaVl1deTIirBBIionvF2tIXsock9cklCD68m9Woq88j7iyJuO5mC2rqaR0lPyOzZsxETE4N+/fph2LBhWmsuPaikJyQqKgpxcXGYM2cO5syZg2+//VZTJjo6GsHBwQgJCcHJkycREhKCsWPH4siRIxXGknn3Hs4k3x//09JRp/i1d4bP0KlOQUEBjh8/jqCgIK3jQUFBOHTokE7niImJwaFDhzBgwACdyhNVBRMgonrGVWGNxWM6QX4/2THWOBxTG+LrDGtzOa7dysXJ+wN7axtj9ISEhYVhyJAhCA0NhY+PD0JDQzFo0CCEhYVVGMvRK7ehFsUJsovCSud70HccUHp6OoqKikrtM+fs7FxqP7qHtWjRApaWlujevTumTp2KyZMn6xwnkb6YABHVQ8E9PHBg1kBseCkAB2YN1HkAc11iY2GGoA7FX7K1cYd4Y/WEREdHlzrn0KFDKz1nyeOvAB3H/5So6kDoh3sbhRCV9kDu378fx44dw6pVqxAWFoYNGzbodc3aQJ9HnuHh4RgyZAiaNWsGe3t7BAYGYteuXaXKhYWFoV27drC2toa7uztmzJiBu3fvGvM2GgQmQET1lLHH4dQGJXuD/XZSiaIyZriZkrF6QlJTU6t0zujL+o3/KVGSACWk5yAjt6DS8o6OjpDL5aXiSUtLKxX3w7y9vdGpUye89NJLmDFjBubPn69XrKam7yPPqKgoDBkyBDt37sTx48cxcOBAjBgxAjExMZoyP/30E2bNmoV58+YhLi4Oq1evxqZNmxAaGlpTt1VvMQEiojqrX5tmaGxjjvTs/Eo3hDUVY/SE6HvOjNwCxKUWD2IOaNlEn/DhYGMBb0dbALr1AllYWKBbt26IiIjQOh4REYHevXvrfF0hBPLz8/WK1dT0feQZFhaGd955Bz169ECbNm2waNEitGnTBr/99pumTHR0NPr06YNx48bBy8sLQUFBeO6553Ds2LGauq16iwkQEdVZ5nIZhndyBVD+YzBDP5J45JFHIN1f+PHB1+OPP65Vzlg9IS4uLnqf8/CV2xACaO3UCE52uo//KaHvY7CZM2fi+++/x5o1axAXF4cZM2YgMTERU6ZMAQCEhoZiwoQJmvJff/01fvvtN8THxyM+Ph5r167Fp59+ivHjx2vKFBQUIDY2FrGxsSgoKEBycjJiY2Nx6dIlve/HGAzxyFOtViMrKwtNmvybpPbt2xfHjx/H0aNHAQBXrlzBzp07S7U30h8TICKq00pWKP7zTCru3ivSes8YjyTCw8OhVCo1rzNnzkAul+OZZ57ROpexekICAwNLnXP37t0VnvOwZvyPfr0/JUoSoJjEDJ3KBwcHIywsDAsXLkSXLl0QFRWFnTt3wtPTEwCgVCq1/g7u5ORj2pvvwK9LF3Tv3h1fffUVlixZgoULF2rKpKSkwN/fH/7+/lAqlfj000/h7+9fawZKV+eRZ4nPPvsMOTk5GDt2rObYs88+iw8++AB9+/aFubk5WrVqhYEDB2LWrFkGjb9BElSKSqUSAIRKpTJ1KERUiaIitQhctEd4vvu7+ON0itZ7PXv2FFOmTNE65uPjI2bNmqXz+X19fcWCBQvKfX/ZsmXCzs5OZGdnl3pv48aNwtzcXKxevVqcO3dOTJ8+Xdja2oqrV68KIYSYNWuWCAkJ0ZRfvny52L59u7h48aK4ePGiWLNmjbC3txezZ8/WlDl48KCQy+ViyZIlIi4uTixZskSYmZmJw4cPlxtj0Of7hOe7v4vfT6aUW6YisYl3hOe7vwu/BbuEWq2u0jnKs/HoNeE963fh+e7vwnvW72Lj0WsGPX9NSU5OFgDEoUOHtI5/+OGHol27dpXW//nnn4WNjY2IiIjQOr53717h7OwsvvvuO3Hq1CkRHh4u3N3dxcKFCw0af3V9/fXXwsvLS1haWoquXbuKqKiocstu2bJFDB48WDg6Ogo7OzsREBAg/vzzT60ya9euFSjehlDrlZeXV2Ec+nx/cyVoIqrTZDIJI7q44Zt9V7AtNgWPdSx+JFbySOLhfylX95HEw1avXo1nn30Wtra2pd4LDg7GrVu3sHDhQiiVSnTs2LHSnpDP3nwHqcmJMDczQ6tWrbBkyRK88sormjK9e/fGxo0bMWfOHMydOxetWrXCpk2b0KtXrzLju5Wdjws3sgBUvQeovas9LMxkyMi9h6u3cjVjgqpLqcpDaPhplIxfVwvgvfAz6N+2WZ0bvF+dR56bNm3CpEmT8L///Q+DBw/Wem/u3LkICQnR9HR16tQJOTk5ePnllzF79mzIZKZ/kKPvyt8lPa2LFi2Cg4MD1q5dixEjRuDIkSPw9/fXlLO3t8eFCxe06lpZ6f8ItzxMgIiozhvl1xzf7LuCv86nIfPuPdhbmRvtkcSDjh49ijNnzmD16tXlnuO1117Da6+9VuZ769at0/x50z+JWHOnHdRjPoWLBCwe06nc5QuefvppPP300zrdw+ErtwEA7Zzt0LSClb8rYmEmQwc3e8QkZiA26Y7BEqCE9Bw8PHmvSAhcTc+tcwnQg488n3zySc3xiIgIjBo1qtx6GzZswIsvvogNGzaUOa4nNze3VJIjl8shhKg1C4A+OPgbKB7cvWvXLqxcuRKLFy8uVf7hNasWLVqEbdu24bffftNKgCRJgouLi9HiNn3qSERUTe1d7dDGqREKCtXYdUY7uanKLCyg+Itp/vz52LRpE5ycnMoss3r1anTs2BE9e/asevAovydEqcqr1nmBf8f/6Dv9/WGagdA6jgPShbejLR7+m5AkwMux8o1aayN9B39v2LABEyZMwGeffYaAgACkpqYiNTUVKtW/C3uOGDECK1euxMaNG5GQkICIiAjMnTsXI0eOhFwur/F7fJixBn8DQHZ2Njw9PdGiRQs88cQTWmPxDIEJEBHVeZIkadYE2n6yeG8wQzyS2Lx5c6lHEiVyc3OxceNGgwzCragnpLqqugDiw6qzM3x5GttYwMZCXupYRXvU1Wb6Dv7+5ptvUFhYiKlTp8LV1VXzmjZtmqbMnDlz8Oabb2LOnDnw9fXFpEmTMHToUHzzzTc1fn9lMVZPq4+PD9atW4ft27djw4YNsLKyQp8+fRAfH2+w2PkIjIjqhZF+zfHp7os4eCkdaVl34WRnZZRHEiU2b96M/Px8ranaVeViX3pcg4Tq94SkZd3FpbRsSFLVx/+U8HdvDAA4p8zE3XtFsDKvfu/D9pMpyCkogrOdJRaN6YS3/ncSt3MK8L9j1zGuV91cvVzXR54A8Oon65F4v+dPVs5jTzMzM8ybNw/z5s0zVsgGUd2e1m3btmn1tAYEBCAgIEDzc58+fdC1a1d89dVX+PLLLw0SM3uAiKhe8GhqA38PB6gFsOOUEoBxHkmUWL16NUaPHo2mTavXswL822v1IEkCbmZVbyHAkvE/7V3s4WBjUa1zuTexRhNbC9wrEjinx87w5RFCYM2BBADAi329Mai9M/7v0TYAgK/+ji+1pEF9o1TlYZaRHnvWJGP3tJaQyWTo0aOHQXuAmAARUb0xyq/4Mdj/jl3Hocvp6P/YSIM/kgCAixcv4sCBA5g0aVK1Y1aq8rBq32UAwAejO2DDS73wqI8T1AKYsSm2WolAVbe/KIskSQYdBxR95RbOp2bB2lyOZ+/3eozr5QFXhRWUqrv46UjZazXVF0fvL075IEM99qxJVV3vasOGDZg4cSJ+/vlnnRZ1FEIgNjYWrq6u1Y65BB+BEVG98XhnNyz47RzOKTMx7rsjkCRgSv+B2PR3MNQCUAsBtRDYd/Em1GqBkHeXokgt8OeZVKiFQO9pX+FaQCIEyn8kAQBt27Y12AycT/68gLv31Oju2Rjje3lCkiT4uNhjaFgULt/MwZI/zmP+yA5VOrdmAHQ1x/+U6OLugL/PpxlkHNCaA1cBAE93awGFjTkAwMpcjjcGtUFo+GmsjLyEZ3u4w9ay/n1N3b1XhK8jS69gLZekOjkAfObMmQgJCUH37t0RGBiIb7/9tlRPa3JyMtavXw/g357WL774QtPTCgDW1tZQKBQAgAULFiAgIABt2rRBZmYmvvzyS8TGxuLrr782WNwmb1krVqzAJ598AqVSiQ4dOiAsLAz9+vUrs2x4eDhWrlyJ2NhY5Ofno0OHDpg/fz6GDh2qKbNu3Tr85z//KVU3Ly/PoOsHEFHtU6hW48G0RAhg5b4rWLnvit7nqok1aWKTMhAeU7yFx9wnfDVjJhrbWuCTZ/zwwpqjWHfoKga1d0K/Ns30Oneq6i4S0nMgk4Ce1Rz/U8JQA6Gvpufgr/M3AAAT+3hpvfd0txZYte8yrt3KxbpDVzF1YOtqXas2mrftLC7eyIaNhRx5BUWaNjtruE+dm/4P6L/e1YM9rVOnTtUcf+GFFzTjpDIyMvDyyy8jNTUVCoUC/v7+iIqKqvaMyweZNAGqq4snEVHtlJCeU+ZxV4UVGlmaQSZJkCRALpMgkyTIZBJkUvG/vLPvFuL8/QUDSxhzTRohBD74/RwAYIx/c/jdTy5KDGjbDBMCPbE++hre+t9J7JreX69xPNFX0gEAHZsrYG9lbpCYS2JMvJ2LW9n5VV5XaN2hqxACGNiuGVo1a6T1nrlchhmD22L6plh8s+8yxgd4QmFtmPhrg03/JGLTsSRIEvBNSDe0dLTFhNVHcTk9p9z2WxcYevD3smXLsGzZMmOFC8DECVBdXTyJiGonb0dbyCRoTSmXSxLCX+tdaRKjVOWhz5K/teoaYiZWeX4/pcTxa3dgbS7H24+1K7NM6LD2OBCfjivpOZi77Sy+es6/zHJl0Yz/MdDjLwBQWJujZTNbXLmZg5PXM/CoT8WDXMuSefce/ncsCUDx4OeyjPBzw4rIS7h4Ixvf77+CN4PK/v3UNaevqzB321kAwJtD2mp69RY/1Rljv4nGxqOJmNjbC22d7UwZplHVptW/TTYIujYtnpSfn4/MzEytFxHVPa4Kaywe0wny+4+S5JKERWM66vTB+nBdoHjzoSP3Z1IZ0t17RVjyx3kAwCsDWpYbn7WFHMuCu0Auk/DbyZRyd7wvi2b9HwMMgH5QdQdCb/4nCTkFRWjj1Ah9WzuWWUYukzBzSHHSs+ZAAm5lV282XG2QkVuAV386joJCNQa3d8Jrj/z7aK+ndxMM7eAMtQAW74wzYZTGty02xWhrXunLZAlQbVo8afHixVAoFJqXu7t71W6KiEwuuIcHDswaiA0vBeDArIHlbidRWd3J93sn5mw9g8Rbhv1wXn0gAckZeXBVWOGV/q0qLOvn7oD/e7T4y3LuVt2mSV+/k4uk23mQyyT08DLM+J8S/iU7w1dhHFCRWuC/0VcBAP/p413hOjFDOzijU3MFcgqKsDLychUirT3UaoHpm2Jx/U4ePJrY4LOxXSCTad/7rGHtYSaTsPfCTRyITzdRpMZTpBb4POKiJvF/kKkGf5t8Gryhl6kPCAjA+PHj4efnh379+mHz5s1o27Ytvvrqq3LPFRoaCpVKpXklJSVV/YaIyORcFdYIbNW0Sl3qJXVnDfNBD6/GyM4vxLRNMSgsUhsktrSsu1ixt3gG0DuPtYO1ReULCk4d2Bp+7g7IvFuIt/53EuqH/wn9kJLHX51bKNDIwLOoutxfEPFkUkalcTxsT9wNJN3Og4ONOZ70b15hWUmS8GZQWwDAD4evIVV1t2oB1wJf/h2PyAs3YWkmw8rxXcsc0+TtaIvxAcWDhj/ccQ5Fev5ugeJJRd7e3rCyKl4EdP/+/eWWVSqVGDduHNq1aweZTIbp06eXKvPII49AkqRSL12mrT8oPTsfE9YcwZd/FXdE9PJugpL8T59eWkMzWQJUmxZPsrS0hL29vdaLiBo2M7kMy4K7wM7KDDGJGZoP7+r6bNdF5BQUwc/dAaP8Kk4CSpjLZVg21g9W5jIcvHRL04tSnmgDT39/kI+rHSzNZMi8W4iEW/oN2i1Z+HBcTw+dEr8BbZuhh1dj5BeqsXyv7r//2pQIRF5Iwxf3285HT3ZCBzdFuWWnDWoDeysznE/NwpYT1yu/0QeUTCqaPXs2YmJi0K9fPwwbNkxr9tWD8vPz0axZM8yePRt+fn5llgkPD4dSqdS8zpw5A7lcjmeeeUbnuI4m3MbwL/bj4KVbsDaXIyy4Cza9EoiDsx6tUi+tIZksAarLiycRUcPQorENFo/pBABYvvcSjtxPLKrqTLIKm48X9zC//0T7Uo9BKtKyWSPMftwXALDkj/OIf2jGWgkhBA4bcAHEh5nLZejYvPhLXJ9xQGeSVTiScBtmMgkhgZ461ZEkCW/dHwC98WgSkm5X/iiyNiUCSbdzMW1jLIQoXuTx6W4tKizf2NZCsxr2p7suILegsNL7LfHgpKL27dsjLCwM7u7uWLlyZZnlvby88MUXX2DChAmatXce1qRJE7i4uGheERERsLGx0SkBUqsFVu27jOe+O4y0rHy0dmqE7a/3wej7PX/V6aU1FJM+AjPGMvULFizArl27cOXKFcTGxmLSpEmIjY3VnJOISB9PdHbDM91aaFZmVuXeq9J5Sqa9CwE80dkV3Tz1H5szvpcHBrRthvxCNWZsjkVBYenHcom3c5GiugtzuYTuVbiGLqqyHtDag1cBAMM7uer1pderZVP0a+OIQrVA2J7Ke4FqSyJw914RXvvpBFR59+DXQoF5I3x1ut8JvT3h3sQaaVn5+C4qQac6hphUpIvVq1fj2Wefha2tbYXlVLn38PIPx7Dkj/MoUguM7uKGbVP7oE0tm91m0gTIGDvnliye1L59ewQFBSE5OdngiycRUcMyf2QHeDvaIkV1F6G/nqrSKtC7zt7AkYTbsDCTYdYwnyrFIUkSPnm6MxxszHEmObPMx3Il43+6uDvo9JipKvRNgNKy7uK3+/udlTf1vSIlvUC/xlzHpbSye76A2pUIzN9+FqeTVWhsY44V47vB0ky3vwtLMznefay4fXwTdRlpmZWPfTLEpKLKHD16FGfOnNEsW1Oek0kZePyr/dgTlwYLMxkWPdkJy4K71MoVvU0+CPq1117D1atXkZ+fj+PHj6N///6a99atW4fIyEjNz5GRkRBClHo9uMjSsmXLcO3aNeTn5yMtLQ27du1CYGBgDd4REdU3tpZm+OLZLjCTSdh5OhX/O6bf+Iz8wiIs/qN4evNL/bzRonHVZ7w42Vth0ZPFj+VWRF7C8Wva0/SNOf6nREkCFHd/Z/jK/HQ4EQVFanT1cNDU1YefuwOCfIuniS+LKL8XqLYkApv/ScLGf4oXO/ziWX80d9DvMc/jnVzh7+GA3IIifB5xUed6VZ1UpIvVq1ejY8eO5XYmCCGwPvoqnlkVrZntFv5qb4zr5WGwGAzN5AkQEVFd0LmFA94aWtwTMW/7WVy+ma1z3f8euoprt3LRzM4Srz5S/a0dhndyxRj/5lALYObmk8jJLx4rIoTQ9AAZev2fB7VobA3HRhYoVAucTVFVWPbuvSL8dOQagKr1/pSYGdQWkgTsOK3EmeSKr2nKROBMsgpzt50pjnlwW/Rvq98WJkBx/HMebw8A2HwsCedTK16brjqTinSRm5uLjRs3lpv0ZecX4o2NsXh/21kUFKkR5OuM3/6vr2asWG3FBIiISEcv92uJ3q2aIu9eEaZtjClzDM7DbmXn46u/iqe9vx3UzmDT0ueP6oDmDta4disXH+4o7l26kp6DtKx8WJjJ0NWjsUGuU5YHd4aPqWQg9G8nU5CeXQA3hRUe61D1Ffp9XOwxorMbAGBZOb0ipk4EVLn38OpPx5FfqMYgH6dq7WPWzbMJhndygVoAi3aWXjvnQVWdVKSrzZs3Iz8/H+PHj9ccU6rycOhyOvbH38TIrw7gt5MpMJMVJ27fhHSrE9uXMAEiItKRTCbh87Fd0Pj+GJzPdl+otM7nEReRlV+IDm72eKqSWUD6sLcyx6fP+EGSgA1HE/FX3A1N709XDwdYmRtn/E8JXcYBCSGw5v7g5wm9vWAmr95XzowhbSGXSfjrfBpOJN4p9b4pEoESxYsdxiDpdh7cm1jj8zIWO9TXu4/5wFwuIeriTey7eLPCsvpOKgKA2NhYxMbGIjs7Gzdv3kRsbCzOnTtX6tyrV6/G6NGj0bRpca/ipn8S0WfJ3xj33RGErD6KK+k5cLG3wqZXAjC5X8ta+8jrYUyAiIj04KKwwtKnOgMAvom6UuGqvRdSs7DhaPFEjrlP+EJezS/EhwW2aopJfYofK7275ZRmq4yOFaw1YyglCyJWlAAdvnIbccpMWJvL8WyP6q+w7+1oi6e7FieRn+4qO/msyUTgQcv3XsLeksUOn+8GhU31e0A8m9rihUAvAMCiHXEVLo6o76QiAPD394e/vz+OHz+On3/+Gf7+/hg+fLhWmYsXL+LAgQN4ZEQwtsUm48Pfz+HdLadLbWexZmL3Ks1sNCVJVGU6Qz2XmZkJhUIBlUrFRRGJqExztp7Gj4cT4WRniT+m9Su1M7oQAhPWHMX++HQ81sEFq0K6GSWOu/eKMHL5AVy88e+YJEkClpSxw7YhZd69B78FuyEEcGzOYDiWsTP8S+uPIeLcDYwP8MCHozsZ5LrJGXkY+EkkCorU+HlyL/QuYz+xFStW4OOPP4ZSqUTHjh2xbNkyzQSbiRMn4urVq5oJNkpVHtwcSg9K9/T0xNWrVzU/X7x4Ee3atcPu3bsxZMgQzXGlKg/bYlM0Wzx8/HRnjO1uuO2UMnILMOCTSKjy7mHJmE54tqdh/k43/ZOo2ZRUJgGzh7dHF4/GuHYrB1dv5Rb/N734z6q8ypd+2PBSgFHWndKXPt/fTIDKwASIiCqTV1CEEcsP4FJaNga3d8J3E7prdf3/ff4GXlx3DBZyGSJm9odn04rXTqmOfRfS8MLaf7SOySUJB2YNNOpCc4M/34dLadn4fkJ3DPbVHmNz7VYOHvk0EkIAe2YOQGunRga77vztZ7Hu0FX4ezgg/NXeVX7k8nASsOhJ/RKMB+sDQE/vxtj8SvUftT1s9YEEfPD7OTSzs0TkW49Ue0q5UpWH3kv+hj7f/i72VvBsagMnO0v8fkqJB6vWRFvTlT7f37VvYj4RUR1gbSHHl8/6Y/TXB7EnLg0/Hr6GkPuPK+4VqTUDk//Tx8uoyQ8AmJuVHs1QssO2Mb+Uurg74FJaNmKTMkolQOsOXYUQwCPtmhk0+QGA1wa2wsZ/EhGTmIG/z6dhUHv9BjgLIXD4yi3M2nJa80WuFsCs8NOYFX4acpkEmQTIJOn+q/jPkoT770lQC+BOboHWeY9dvQOlKs/gv/OQAE+sjy6eSfhN1BXMHNK2yudKVd3Fu1tOlZn8NLU1R1tne3g52sKrqQ08m9rCy9EGHk1sYGPxb7rQt00i3gs/gyIhTLqXV3UxASIiqiJfN3vMGuaDhb+fw4c74tDTuynaudjhx8PXcOVmDpraWmDqo9Wf9l4Zb0dbyCRojcuoiR22u7g74Jfj10uNA8q6e0+zVtKLfao+9b08TnZWmNjbG6v2XcZnuy9iYDunSgccF6kFYhLvYPe5G4g4dwMJ6eXvY1akFihe3Ui/ByRqAaMknRZmMsx6zAev/nQC30ZdxrieHnBRWOl1jpz8QnwTdQXfRV1BXhlrN8kk4Pc3+ukUe3APD/Rv2wxX03Ph5WhTJ5MfgAkQEVG1/KePF6LibyLywk28sSEGnzzdWTNAd8aQtrC3Mv50YFeFNRaP6VTj/yovmQlWsjN8SRKy+dh1ZOcXorVTI/RrU3qMjiG80r8lfjp8DeeUmfjjTCoe71x6v8e794qwPz4dEedS8VdcGm7l/NtjYy6TcO+hkbwyCdj2eh842VlBLQSK1AJCAGohoC75r7r4zzcy7+KFtUe1elKMmXQ+1tEF3T0b49i1O/hs9wV88kzZ+5Y9rEgt8MvxJHy6+yJuZuUDALp7Nkavlk2xKvJylduLq8K6ziY+JZgAERFVQ/H2FH4Y9kUULtzIwsivD2reM/CkrwqZ4l/lPi52sDKXISu/EFfSs9HayQ5FaoF1h4r3sHqxj7fRpkQ3trXApH7eCNsTj4//jIPC2hytnGxhaSbHX3E3sPvcDeyPv4m79/5dq8nOygyP+jhhiK8zBrRthp2nlaWSxk7NHXS6fjsXOyypwaRTkiTMfrw9nlxxCL+cuI6Jfbwq3FkeAPbH38RHO+JwPrV4+xDPpjaY9ZgPHuvoAkmSMD7Ao8734lQHB0GXgYOgiUhf4SeuY+bmk1rHatPgUGN5ZtUh/HP1Dj55ujOe6e6OXWdT8coPx+FgY47oWYOMth8ZUPyordeiv5Bb8O8jHQnaD66aO1hjiK8zhvg6o6d3E5g/tBaRUpVXrSSguvX19X8bYvDbyRT0ad0UP07qVWaCefFGFhbtjEPkheK1gxTW5vi/R1tjQqAXLMoYL1afcBA0EVENK2tMRk0MRDa1Lu4O+OfqHcQmZeCZ7u5Yc6C492dcTw+jJj9A8RYMeQXa41kEgDZOjTC8kyuCOjjD19W+wl6o6j7KqelHQe8MbYddZ1Jx8NItRF64iYE+Tpr3bmblY9mei9h4NBFqAZjLJUwI9ML/PdoaDjYWNRZjXcEEiIjIAEw1ENnUihdETEBsUgbOJKtwJOE2zGQSQgI9jX7thPScMocpLxzVsVasSWMM7k1s8J8+Xvgm6goW/HYWZnIJ7o1tsOO0EisjLyP7/r5wj3VwwaxhPvByNO4MxLqMCRARkQGYaiCyqXXxcAAAnE/NwsrIywCKN2utiftuqEnnawNb44fD13D1Vi5CVh/Ves+vhQKzH/dFT++6tSqzKTABIiIykPoyPVgfbgorNLOzxM2sfOw4rQRQvV3f9dFQk87cgtKP/gBgwUhfhAR4VXsPsoaCCRARkQHVh+nB+pAkCc0aWWqmWAPAhdRMzRR5Y2uISWd5j/7aOtsz+dFD/R4OTkRERqVU5SFOmal17L3wM1Cq8mosBleFNQJbNW0QyQ/w76O/BzWER3+GxgSIiIiqrKzeiJLZb2QcJY/+5PdntzWUR3+GxkdgRERUZQ11ILKpNcRHf4bGHiAiIqoy9kaYTkN79Gdo7AEiIqJqYW8E1UVMgIiIqNoa2uw3qvv4CIyIiIgaHCZARERE1OAwASIiIqIGhwkQERERNThMgIiIiKjBYQJEREREDQ4TICIiImpwmAARERFRg8MEiIiIiBocJkBERETU4DABIiIiogaHCRARERE1OCZPgFasWAFvb29YWVmhW7du2L9/f4Xl9+3bh27dusHKygotW7bEqlWrSpXZsmULfH19YWlpCV9fX/z666/GCp+IiIjqIJMmQJs2bcL06dMxe/ZsxMTEoF+/fhg2bBgSExPLLJ+QkIDhw4ejX79+iImJwXvvvYc33ngDW7Zs0ZSJjo5GcHAwQkJCcPLkSYSEhGDs2LE4cuRITd0WERER1XKSEEKY6uK9evVC165dsXLlSs2x9u3bY/To0Vi8eHGp8u+++y62b9+OuLg4zbEpU6bg5MmTiI6OBgAEBwcjMzMTf/zxh6bMY489hsaNG2PDhg06xZWZmQmFQgGVSgV7e/uq3h4RERHVIH2+v81qKKZSCgoKcPz4ccyaNUvreFBQEA4dOlRmnejoaAQFBWkdGzp0KFavXo179+7B3Nwc0dHRmDFjRqkyYWFh5caSn5+P/Px8zc8qlQpA8S+SiIiI6oaS721d+nZMlgClp6ejqKgIzs7OWsednZ2RmppaZp3U1NQyyxcWFiI9PR2urq7llinvnACwePFiLFiwoNRxd3d3XW+HiIiIaomsrCwoFIoKy5gsASohSZLWz0KIUscqK//wcX3PGRoaipkzZ2p+VqvVuH37Npo2bVphvarIzMyEu7s7kpKS9H68Vp26dfna1a3Pazesa1e3Pq/Na9eV+g312hURQiArKwtubm6VljVZAuTo6Ai5XF6qZyYtLa1UD04JFxeXMsubmZmhadOmFZYp75wAYGlpCUtLS61jDg4Out5Kldjb21f5L706devytatbn9duWNeubn1em9euK/Ub6rXLU1nPTwmTzQKzsLBAt27dEBERoXU8IiICvXv3LrNOYGBgqfK7d+9G9+7dYW5uXmGZ8s5JREREDY9JH4HNnDkTISEh6N69OwIDA/Htt98iMTERU6ZMAVD8aCo5ORnr168HUDzja/ny5Zg5cyZeeuklREdHY/Xq1Vqzu6ZNm4b+/ftj6dKlGDVqFLZt24Y9e/bgwIEDJrlHIiIiqn1MmgAFBwfj1q1bWLhwIZRKJTp27IidO3fC09MTAKBUKrXWBPL29sbOnTsxY8YMfP3113Bzc8OXX36Jp556SlOmd+/e2LhxI+bMmYO5c+eiVatW2LRpE3r16lXj91cWS0tLzJs3r9QjN2PXrcvXrm59XrthXbu69XltXruu1G+o1zYUk64DRERERGQKJt8Kg4iIiKimMQEiIiKiBocJEBERETU4TICIiIiowWECVINWrFgBb29vWFlZoVu3bti/f79O9aKiojBixAi4ublBkiRs3bpV52suXrwYPXr0gJ2dHZycnDB69GhcuHBB5/orV65E586dNYtVBQYGam00q4/FixdDkiRMnz5dp/Lz58+HJElaLxcXF72umZycjPHjx6Np06awsbFBly5dcPz48UrreXl5lbq2JEmYOnWqTtctLCzEnDlz4O3tDWtra7Rs2RILFy6EWq3WqX5WVhamT58OT09PWFtbo3fv3vjnn3/KLFtZ+xBCYP78+XBzc4O1tTUeeeQRnD17Vuf64eHhGDp0KBwdHSFJEmJjY3Wqe+/ePbz77rvo1KkTbG1t4ebmhgkTJiAlJUXna8+fPx8+Pj6wtbVF48aNMXjwYBw5ckSnug965ZVXIEmS1p6AldWfOHFiqb//gIAAna8dFxeHkSNHQqFQwM7ODgEBAZpZrZXVL6vtSZKETz75pNK62dnZeP3119GiRQtYW1ujffv2WhtOV1b/xo0bmDhxItzc3GBjY4PHHnsM8fHxAHT7PCmvvelSt6K2Vln9ytqbLtcvr73p+zn6cHvTpX557U3Xa5fX3nSpX157GzZsWKV1K2pvuly7ovZmbEyAasimTZswffp0zJ49GzExMejXrx+GDRumNc2/PDk5OfDz88Py5cv1vu6+ffswdepUHD58GBERESgsLERQUBBycnJ0qt+iRQssWbIEx44dw7Fjx/Doo49i1KhRWl+guvjnn3/w7bffonPnznrV69ChA5RKpeZ1+vRpneveuXMHffr0gbm5Of744w+cO3cOn332mU6rfP/zzz9a1y1ZXPOZZ57R6dpLly7FqlWrsHz5csTFxeHjjz/GJ598gq+++kqn+pMnT0ZERAR++OEHnD59GkFBQRg8eDCSk5NLla2sfXz88cf4/PPPsXz5cvzzzz9wcXHBkCFDkJWVpVP9nJwc9OnTB0uWLNHr2rm5uThx4gTmzp2LEydOIDw8HBcvXsTIkSN1jr1t27ZYvnw5Tp8+jQMHDsDLywtBQUG4efOmzv9fbN26FUeOHCm1NL4u9R977DGtdrBz506d6l6+fBl9+/aFj48PIiMjcfLkScydOxdWVlY61X/wmkqlEmvWrIEkSXjqqacqrTtjxgz8+eef+PHHHxEXF4cZM2bg//7v/7Bt27ZKry2EwOjRo3HlyhVs27YNMTEx8PT0xODBg5GTk6PT50l57e2vv/6qtG5Fba2ya1fW3nSJvbz2FhERofPnaFntTdfP4bLamy51K2pvutQvr73l5ORUWrei9lbZtStrb0YnqEb07NlTTJkyReuYj4+PmDVrll7nASB+/fXXKseRlpYmAIh9+/ZV+RyNGzcW33//vc7ls7KyRJs2bURERIQYMGCAmDZtmk715s2bJ/z8/KoWpBDi3XffFX379q1y/QdNmzZNtGrVSqjVap3KP/744+LFF1/UOjZmzBgxfvz4Suvm5uYKuVwufv/9d63jfn5+Yvbs2RXWfbh9qNVq4eLiIpYsWaI5dvfuXaFQKMSqVasqrf+ghIQEAUDExMTodO2yHD16VAAQ165dq1J9lUolAIg9e/boVPf69euiefPm4syZM8LT01MsW7ZM59hfeOEFMWrUqArjKa9ucHCwTn/XFcX+oFGjRolHH31Up7odOnQQCxcu1DrWtWtXMWfOnErrX7hwQQAQZ86c0RwrLCwUTZo0Ed99912p+g9/nujT3ir6LKqsrVVWv0RF7U2X+uW1t/Lq6treyqqva3srq64+7U2X+y6vvZVVV5/29nB9fdubobEHqAYUFBTg+PHjCAoK0joeFBSEQ4cO1WgsKpUKANCkSRO96xYVFWHjxo3IyclBYGCgzvWmTp2Kxx9/HIMHD9b7mvHx8XBzc4O3tzeeffZZXLlyRee627dvR/fu3fHMM8/AyckJ/v7++O677/SOoaCgAD/++CNefPFFnTfH7du3L/766y9cvHgRAHDy5EkcOHAAw4cPr7RuYWEhioqKNL0FJaytrfVe0TwhIQGpqalabc/S0hIDBgyo8bYHFLc/SZKqtNdeQUEBvv32WygUCvj5+VVaXq1WIyQkBG+//TY6dOhQhWiByMhIODk5oW3btnjppZeQlpam03V37NiBtm3bYujQoXByckKvXr30enT9oBs3bmDHjh2YNGmSTuX79u2L7du3Izk5GUII7N27FxcvXsTQoUMrrZufnw8AWm1PLpfDwsKizLb38OeJPu2tOp9FutavqL1VVr+i9lZWXX3aW3nX1qW9PVxX3/ZW2X1X1N7KqqtPe3u4vr7tzeCMnmKRSE5OFgDEwYMHtY5/9NFHom3btnqdC9XoAVKr1WLEiBF694qcOnVK2NraCrlcLhQKhdixY4fOdTds2CA6duwo8vLyhBBCrx6gnTt3il9++UWcOnVK03vk7Ows0tPTdapvaWkpLC0tRWhoqDhx4oRYtWqVsLKyEv/97391jl8IITZt2iTkcrlITk7WuY5arRazZs0SkiQJMzMzIUmSWLRokc71AwMDxYABA0RycrIoLCwUP/zwg5AkqdL28nD7OHjwoABQKvaXXnpJBAUFVVr/QdXtAcrLyxPdunUTzz//vF71f/vtN2FrayskSRJubm7i6NGjOtVdtGiRGDJkiKbXTt8eoI0bN4rff/9dnD59Wmzfvl34+fmJDh06iLt371ZYV6lUCgDCxsZGfP755yImJkYsXrxYSJIkIiMjdb7vEkuXLhWNGzfW/D9UWd38/HwxYcIEAUCYmZkJCwsLsX79ep3uu6CgQHh6eopnnnlG3L59W+Tn54vFixcLAKXaS1mfJ7q2t8o+iypra7p8llXU3iqqX1l7K6+uru2tvPq6tLey6urT3nT5vZXX3sqrq2t7K6u+Pu3NGJgA1YCSBOjQoUNaxz/88EPRrl07vc5VnQTotddeE56eniIpKUmvevn5+SI+Pl78888/YtasWcLR0VGcPXu20nqJiYnCyclJxMbGao7pkwA9LDs7Wzg7O4vPPvtMp/Lm5uYiMDBQ69j//d//iYCAAL2uGxQUJJ544gm96mzYsEG0aNFCbNiwQZw6dUqsX79eNGnSRKxbt06n+pcuXRL9+/cXAIRcLhc9evQQzz//vGjfvn2F9cpLgFJSUrTKTZ48WQwdOrTS+g+qTgJUUFAgRo0aJfz9/YVKpdKrfnZ2toiPjxfR0dHixRdfFF5eXuLGjRsV1j127JhwdnbW+iLWNwF6WEpKijA3NxdbtmypsG7J/+/PPfecVrkRI0aIZ599Vu9rt2vXTrz++us6x/3JJ5+Itm3biu3bt4uTJ0+Kr776SjRq1EhEREToVP/YsWPCz89P0/aGDh0qhg0bJoYNG6ZVrqzPE13bW2WfRZW1tcrqV9beKqpfWXsrq64+7U3Xz+Gy2ltZdfVpb7pcu7z2Vl5dXdtbefV1bW/GwASoBuTn5wu5XC7Cw8O1jr/xxhuif//+ep2rqgnQ66+/Llq0aCGuXLmid92HDRo0SLz88suVlvv11181jbrkBUBIkiTkcrkoLCzU+9qDBw8uNZaqPB4eHmLSpElax1asWCHc3Nx0vt7Vq1eFTCYTW7du1SvOFi1aiOXLl2sd++CDD/ROeLOzszVfJmPHjhXDhw+vsPzD7ePy5csCgDhx4oRWuZEjR4oJEyZUWv9BVU2ACgoKxOjRo0Xnzp0r7L3TtW23bt26VG/aw3WXLVumaWcPtj2ZTCY8PT2rde0Hx7eUVTc/P1+YmZmJDz74QKvcO++8I3r37q3XtaOiogQArX9EVFQ3NzdXmJublxo/NmnSJL0T3oyMDJGWliaEKB7D+Nprr2neK+/zRJf2pstnUUVtrbL6lbU3fT8LH2xv5dXVtb1V5dol7a28urq2N12uXV57K6+uru1Nl2tX1N6MhWOAaoCFhQW6deummUlUIiIiAr179zbqtYUQeP311xEeHo6///4b3t7eBjlnybPbigwaNAinT59GbGys5tW9e3c8//zziI2NhVwu1+u6+fn5iIuLg6urq07l+/TpU2rK5cWLFzWb7epi7dq1cHJywuOPP65XrLm5uZDJtP/3ksvlOk+DL2FrawtXV1fcuXMHu3btwqhRo/Sq7+3tDRcXF622V1BQgH379hm97QHFU5PHjh2L+Ph47NmzB02bNq32OXVpfyEhITh16pRW23Nzc8Pbb7+NXbt2Vem6t27dQlJSUqXtz8LCAj169Kh22wOA1atXo1u3bjqNeQKKf9/37t0zSNtTKBRo1qwZ4uPjcezYMYwaNarSz5OK2ltgYGC1Pot0+SyrqL1V9bNQCIG7d+9WWLey9laVa5e0NxcXlwrrVtbe9Ln2w+2tsrqVtTd9rl1WezM6o6dYJIQofr5rbm4uVq9eLc6dOyemT58ubG1txdWrVyutm5WVJWJiYkRMTIwAoHnOW9bMhoe9+uqrQqFQiMjISKFUKjWv3NxcneIODQ0VUVFRIiEhQZw6dUq89957QiaTid27d+tU/2H6PAJ78803RWRkpLhy5Yo4fPiweOKJJ4SdnZ1OvzMhimeAmJmZiY8++kjEx8eLn376SdjY2Igff/xRp/pFRUXCw8NDvPvuuzqVf9ALL7wgmjdvLn7//XeRkJAgwsPDhaOjo3jnnXd0qv/nn3+KP/74Q1y5ckXs3r1b+Pn5iZ49e4qCgoJSZStrH0uWLBEKhUKEh4eL06dPi+eee064urqKzMxMnerfunVLxMTEiB07dggAYuPGjSImJkYolcoK6967d0+MHDlStGjRQsTGxmq1v/z8/EqvnZ2dLUJDQ0V0dLS4evWqOH78uJg0aZKwtLQUZ86c0fv/i4cfSVRUPysrS7z55pvi0KFDIiEhQezdu1cEBgaK5s2bi8zMzEqvHR4eLszNzcW3334r4uPjxVdffSXkcrnYv3+/Tr9zIYpnINnY2IiVK1fq9fc9YMAA0aFDB7F3715x5coVsXbtWmFlZSVWrFihU/3NmzeLvXv3isuXL4utW7cKT09PMWbMGCGEbp8n5bW3SZMmVVq3orZW2bUra2+V1a+ovQUHB+v9Ofpge6vs2hW1N11+bxW1N12/A8pqb7rUrai96VK/ovZmbEyAatDXX38tPD09hYWFhejatavOU9H37t0rAJR6vfDCC5XWLaseALF27Vqdrv3iiy9qYm7WrJkYNGhQlZMfIfRLgIKDg4Wrq6swNzcXbm5uYsyYMTqNPXrQb7/9Jjp27CgsLS2Fj4+P+Pbbb3Wuu2vXLgFAXLhwQa9rCiFEZmammDZtmvDw8BBWVlaiZcuWYvbs2Zov/sps2rRJtGzZUlhYWAgXFxcxdepUkZGRUWbZytqHWq0W8+bNEy4uLsLS0lL0799fnD59Wuf6a9euLfP9efPmVVi35DFGWa+9e/dWeu28vDzx5JNPCjc3N2FhYSFcXV3FyJEjNYNS9f3/4uEEqKL6ubm5IigoSDRr1kyYm5sLDw8P8cILL4jExESdr7169WrRunVrYWVlJfz8/LQeo+pS/5tvvhHW1tal/t4rq6tUKsXEiROFm5ubsLKyEu3atROfffaZZnBuZfW/+OIL0aJFC819z5kzR9Nudfk8Ka+96VK3orZWWf3K2ltl9Stqb1X5HH2wvVVWv6L2puu1y2tvutYvq73pUrei9qZL/Yram7FJ92+SiIiIqMHgGCAiIiJqcJgAERERUYPDBIiIiIgaHCZARERE1OAwASIiIqIGhwkQERERNThMgIiIiKjBYQJEREREDQ4TICIiHURGRkKSJGRkZJg6FCIyACZARERE1OAwASIiIqIGhwkQEdUJQgh8/PHHaNmyJaytreHn54dffvkFwL+Pp3bs2AE/Pz9YWVmhV69eOH36tNY5tmzZgg4dOsDS0hJeXl747LPPtN7Pz8/HO++8A3d3d1haWqJNmzZYvXq1Vpnjx4+je/fusLGxQe/evXHhwgXj3jgRGQUTICKqE+bMmYO1a9di5cqVOHv2LGbMmIHx48dj3759mjJvv/02Pv30U/zzzz9wcnLCyJEjce/ePQDFicvYsWPx7LPP4vTp05g/fz7mzp2LdevWaepPmDABGzduxJdffom4uDisWrUKjRo10opj9uzZ+Oyzz3Ds2DGYmZnhxRdfrJH7JyLD4m7wRFTr5eTkwNHREX///TcCAwM1xydPnozc3Fy8/PLLGDhwIDZu3Ijg4GAAwO3bt9GiRQusW7cOY8eOxfPPP4+bN29i9+7dmvrvvPMOduzYgbNnz+LixYto164dIiIiMHjw4FIxREZGYuDAgdizZw8GDRoEANi5cycef/xx5OXlwcrKysi/BSIyJPYAEVGtd+7cOdy9exdDhgxBo0aNNK/169fj8uXLmnIPJkdNmjRBu3btEBcXBwCIi4tDnz59tM7bp08fxMfHo6ioCLGxsZDL5RgwYECFsXTu3FnzZ1dXVwBAWlpate+RiGqWmakDICKqjFqtBgDs2LEDzZs313rP0tJSKwl6mCRJAIrHEJX8ucSDHeDW1tY6xWJubl7q3CXxEVHdwR4gIqr1fH19YWlpicTERLRu3Vrr5e7uril3+PBhzZ/v3LmDixcvwsfHR3OOAwcOaJ330KFDaNu2LeRyOTp16gS1Wq01poiI6i/2ABFRrWdnZ4e33noLM2bMgFqtRt++fZGZmYlDhw6hUaNG8PT0BAAsXLgQTZs2hbOzM2bPng1HR0eMHj0aAPDmm2+iR48e+OCDDxAcHIzo6GgsX74cK1asAAB4eXnhhRdewIsvvogvv/wSfn5+uHbtGtLS0jB27FhT3ToRGQkTICKqEz744AM4OTlh8eLFuHLlChwcHNC1a1e89957mkdQS5YswbRp0xAfHw8/Pz9s374dFhYWAICuXbti8+bNeP/99/HBBx/A1dUVCxcuxMSJEzXXWLlyJd577z289tpruHXrFjw8PPDee++Z4naJyMg4C4yI6rySGVp37tyBg4ODqcMhojqAY4CIiIiowWECRERERA0OH4ERERFRg8MeICIiImpwmAARERFRg8MEiIiIiBocJkBERETU4DABIiIiogaHCRARERE1OEyAiIiIqMFhAkREREQNzv8Dnl8paupmY9oAAAAASUVORK5CYII=", "text/plain": [ "
" ] @@ -1452,7 +1452,7 @@ "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAG2CAYAAACZEEfAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAAB9HElEQVR4nO3dd1RUx98G8GfpRUB6kaqg2HvB3oIt9qiJJjHGWH5irLFgYkw0ETWJJsbE9ipqrDGxG3vBhgiKXSmKglQLLL3uvH8QNq60XYrg+nzO2XNk7p07c3evu987d4pECCFAREREpKY0qroCRERERJWJwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqbUqDXbOnTuH/v37w87ODhKJBPv27VPYLoTA119/DVtbW+jr66Nnz54ICwtT2OfFixcYNWoUjI2NUbNmTYwdOxapqamv8SyIiIioOqvSYCctLQ1NmzbFb7/9VuT2ZcuWYeXKlVizZg0CAgJgaGiIXr16ITMzU77PqFGjcOfOHZw4cQKHDh3CuXPnMH78+Nd1CkRERFTNSarLQqASiQR79+7FoEGDAOS36tjZ2WHmzJn44osvAABSqRTW1tbYtGkT3n//fdy7dw8NGjRAYGAgWrVqBQA4evQo+vbtiydPnsDOzq6qToeIiIiqCa2qrkBxIiIiEBcXh549e8rTTExM0LZtW/j7++P999+Hv78/atasKQ90AKBnz57Q0NBAQEAABg8eXOSxs7KykJWVJf9bJpPhxYsXMDc3h0QiqbyTIiIiogojhEBKSgrs7OygoVH8w6pqG+zExcUBAKytrRXSra2t5dvi4uJgZWWlsF1LSwtmZmbyfYri4+ODb7/9toJrTERERFUhKioK9vb2xW6vtsFOZfL29saMGTPkf0ulUjg6OiIqKgrGxsZVWDMiIiJSVnJyMhwcHGBkZFTiftU22LGxsQEAxMfHw9bWVp4eHx+PZs2ayfdJSEhQyJebm4sXL17I8xdFV1cXurq6hdKNjY0Z7BAREb1hSuuCUm3n2XFxcYGNjQ1OnTolT0tOTkZAQAA8PDwAAB4eHkhKSsLVq1fl+5w+fRoymQxt27Z97XUmIiKi6qdKW3ZSU1MRHh4u/zsiIgLXr1+HmZkZHB0dMW3aNHz33Xdwc3ODi4sL5s+fDzs7O/mIrfr166N3794YN24c1qxZg5ycHEyePBnvv/8+R2IRERERgCoOdoKCgtCtWzf53wX9aEaPHo1NmzZh9uzZSEtLw/jx45GUlISOHTvi6NGj0NPTk+fZtm0bJk+ejB49ekBDQwNDhw7FypUrX/u5EBERUfVUbebZqUrJyckwMTGBVCplnx0iIqI3hLK/39W2zw4RERFRRWCwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGptWod7OTl5WH+/PlwcXGBvr4+6tSpg0WLFkEIId9HCIGvv/4atra20NfXR8+ePREWFlaFtSYiIqLqpFoHO0uXLsXq1auxatUq3Lt3D0uXLsWyZcvw66+/yvdZtmwZVq5ciTVr1iAgIACGhobo1asXMjMzq7DmREREVF1IxMvNJNXMu+++C2tra2zYsEGeNnToUOjr62Pr1q0QQsDOzg4zZ87EF198AQCQSqWwtrbGpk2b8P777ytVTnJyMkxMTCCVSmFsbFwp50JEREQVS9nf72rdstO+fXucOnUKoaGhAIAbN27gwoUL6NOnDwAgIiICcXFx6NmzpzyPiYkJ2rZtC39//2KPm5WVheTkZIUXERERqSetqq5ASebOnYvk5GS4u7tDU1MTeXl5+P777zFq1CgAQFxcHADA2tpaIZ+1tbV8W1F8fHzw7bffVl7FiYiIqNqo1i07f/75J7Zt24bt27fj2rVr2Lx5M3788Uds3ry5XMf19vaGVCqVv6KioiqoxkRERFTdVOuWnVmzZmHu3LnyvjeNGzfG48eP4ePjg9GjR8PGxgYAEB8fD1tbW3m++Ph4NGvWrNjj6urqQldXt1LrTkRERNVDtW7ZSU9Ph4aGYhU1NTUhk8kAAC4uLrCxscGpU6fk25OTkxEQEAAPD4/XWlciqhqx0gxcevAMsdKMqq4KVSJnZ2dIJJJCLy8vLwBA165dC22bOHFiiccs6ngSiQQ//PCDwn6HDx9G27Ztoa+vD1NTUwwaNKiyTpMqSbVu2enfvz++//57ODo6omHDhggODsby5cvx6aefAsi/UKdNm4bvvvsObm5ucHFxwfz582FnZ8eLkegtsCswEt57bkEmAA0J4DOkMUa0dqzqalElCAwMRF5envzv27dv45133sGwYcPkaePGjcPChQvlfxsYGJR4zNjYWIW/jxw5grFjx2Lo0KHytL///hvjxo3D4sWL0b17d+Tm5uL27dvlPR16zap1sPPrr79i/vz5mDRpEhISEmBnZ4cJEybg66+/lu8ze/ZspKWlYfz48UhKSkLHjh1x9OhR6OnpVWHNiaiyxUozMHfPLRRMniETwLw9t9G5riVsTfSrtnJU4SwtLRX+XrJkCerUqYMuXbrI0wwMDOTdG5Tx6r779+9Ht27dULt2bQBAbm4upk6dih9++AFjx46V79egQYOynAJVoWr9GMvIyAg///wzHj9+jIyMDDx48ADfffcddHR05PtIJBIsXLgQcXFxyMzMxMmTJ1G3bt0qrDURvQ7XHifi1VnC8oTAo2fpVVMhem2ys7OxdetWfPrpp5BIJPL0bdu2wcLCAo0aNYK3tzfS05W/FuLj43H48GGFoObatWuIjo6GhoYGmjdvDltbW/Tp04ctO2+gat2yQ0RUlPTsXKw8XXhZGE0J4GxR8qMLevPt27cPSUlJ+OSTT+RpI0eOhJOTE+zs7HDz5k3MmTMHISEh2LNnj1LH3Lx5M4yMjDBkyBB52sOHDwEA33zzDZYvXw5nZ2f89NNP6Nq1K0JDQ2FmZlah50WVh8EOEb1R8mQCU3ZcR0hcKgx1NJGRkwfZvy08ozu48BHWW2DDhg3o06cP7Ozs5Gnjx4+X/7tx48awtbVFjx498ODBA9SpU6fUY27cuBGjRo1S6AJRMBjmyy+/lPfj8fX1hb29PXbv3o0JEyZU1ClRJavWj7GIiF7l8889nLwXDx0tDWwZ2xYX53ZH30b5fS9uRiWhGq+AQxXg8ePHOHnyJD777LMS92vbti0AIDw8vNRjnj9/HiEhIYWOWTClyct9dHR1dVG7dm1ERkaqWnWqQgx2iOiNsfXyY/zfhQgAwE/DmqKlkylsTfSxYEBD6GhqIOhxIgIiXlRxLaky+fr6wsrKCv369Stxv+vXrwOAwhxsxdmwYQNatmyJpk2bKqS3bNkSurq6CAkJkafl5OTg0aNHcHJyUr3yVGUY7BBRmZU298mECRNQp04d6Ovrw9LSEgMHDsT9+/eVPv7EiRMhkUjw888/wy/0KRYcuAMA+Li+Nv7v6//BwsICxsbGGNq3J1rrxgAAfjtT+p08vZlkMhl8fX0xevRoaGn91wvjwYMHWLRoEa5evYpHjx7hwIED+Pjjj9G5c2c0adJEvp+7uzv27t2rcMzk5GTs3r27yJYiY2NjTJw4ERMmTJBf2zo6OkhISMCECRMq5Tp/WWhoKAYOHCi/zjt27IgzZ84ofVz6D4MdIiqzwMBAxMbGyl8nTpwAAPncJy1btoSvry/u3buHY8eOQQgBT09PhflSirN3715cvnwZdnZ2SEjOhNe2a8iTCQxtYY+d309Gbm4uTp8+jatXr6Jp06bYu3QKRHoizoc9Q3BkYqWeN1WNkydPIjIyUj7XWgEdHR2cPHkSnp6ecHd3x8yZMzF06FAcPHhQYb+QkBBIpVKFtJ07d0IIgQ8++KDIMn/44QeMHz8e5ubmMDQ0RKdOnbB+/XoAFX+dv+rdd98tdJ2/++67Ja79SMUQJKRSqQAgpFJpVVeF6I02depUUadOHSGTyYrcfuPGDQFAhIeHl3icJ0+eiFq1aonbt28LB0dH4dxvknCac0gMX3NJRMfGCwDi3Llz8v2Tk5MFADH0yzXCac4hMXbTlQo9L8rn5OQkABR6TZo0SQghxPjx40Xt2rWFnp6esLCwEAMGDBD37t0r8Zh///23eOedd4SZmZkAIIKDgwvts3btWtGlSxdhZGQkAIjExMQyn0NMUrq4GP5UxCSll/kYlXGdOzk5iRUrVsi3PX36tNjr/MSJE2WuuyrehM9b2d9vtuwQUYUobu6TAmlpafD19YWLiwscHByKPY5MJsNHH32EWbNmobabO56lZkOakQMXC0Os/aglbK0tUa9ePWzZsgVpaWnIzc3F2rVrYWVlhXmj+0IiAU7eS8DdmOTKPN23UmW05KWlpaFjx45YunRpsfukp6ejd+/emDdvXrnqvyswEh2WnMbI9QHosOQ0dgWq3sm4Mq7zhg0bFtpubm6OevXqoW/fvvJHaMbGxgCAd955B15eXnjx4gU+//xz1KtXD/r6+nB0dMSUKVMKtV69SpllMgYMGICcnBzo6OjA0tISQ4cOxY4dOwC8OZ+3glICu7cCW3aIym/Xrl1CU1NTREdHK6T/9ttvwtDQUAAQ9erVK/Vud/HixeKdd94Rubl5YuIfQULT2ErY9Z4gHj5Nle8TFRUlWrZsKSQSidDU1BS2trbi2rVrQgghvLZdFU5zDolJ265W/EmSgopq4RBCiIiIiGLv9AucOXOmzC07MUnpwnnuIeE0579X7bmHVW7hqejrvOC9e7VlR4j867xJkyYCgNDQ0BBWVlZi9erVAoA4c+aMuHXrlhgyZIg4cOCACA8PF6dOnRJubm5i6NChJZYdGxur8Nq4caOQSCTiwYMH8n2WL18u/P39xaNHj8TFixeFh4eHsLGxqXafN1t2iN4SJXUSLuud3zfffAN3d3cYGhrC1NQUPXv2REBAQInljhgxAm5uboX6HowaNQrBwcHw8/ND3bp1MXz4cGRmZhZZ7tWrV/HLL79g06ZN+OF4KI7cjoNEAgxv5QgXC0MAgBACXl5esLKywvnz53HlyhUMGjQI/fv3R2xsLLy6uQIA/rkViwdPU8v6tlIpKqqF43WJeJZWITNuFzXHD1D267yo9w747zqvVasWLly4gMDAQAwdOhRffPEFnJ2d0aVLFzRq1Ah///03+vfvjzp16qB79+74/vvvcfDgQeTm5hZ7DjY2NgqvV5fJAIDp06ejXbt2cHJyQvv27TFz5kzExcVh9OjRb8TnXUip4ddbgC079CZLSEhQuEs7ceJEue/8tm3bJk6cOCEePHggbt++LcaOHSuMjY1FQkKCfB8nJyexcOFCERsbK65cuSI0NDTEzp07SzxuVlaWMDAwENu3by9y+4oVK4REIhEampoCEo381793tU5OTkIIIU6ePCk0NDQK/X91dXUVPj4+Qgghxm4KFE5zDokZu66X9vZRGVVUC0eBym7ZiU5MU2jVKUvLzqNHj4SGhobYt29fifspe51ramrKX8pc51lZWUJDQ0N4enoWW/b69euFhYWF0ucUFxcntLS0xLZt24rd5/nz58LDw0MAqHafN1t2iN4SlpaWCndphw4dki+QWNY7v5EjR6Jnz56oXbs2GjZsiOXLlyM5ORk3b95U2M/IyAg2NjY4fPgwrKysFFaLLooQAkIIZGVlFbn9o48+wuZDfrAbsxK2Y1biizX7YWdnh1mzZuHYsWMAIF/vSEND8etLQ0NDPuPt5O75rTv7rkcj6gXXyqoMFdHC8TrFJRe+5ub2cVdpxm1l5/hR5jq/efMmrl+/Ln8pc53v27cPMpkMzZs3L/K4z549w6JFixRmky5NUctkFJgzZw4MDQ1hbm6Ou3fvwtPT8435vF/FYIdIjZT2aAEApFIpjI2NFeYpKe2Y69atg4mJSaFJ15YsWQIzMzMsXrwY7u7uCtsePnwIHx8fXL16FZGRkbh06RKGDRsGfX199O3bV77fy3OfJObpYNmVdGiaO2HYO+2xbFw/aGtrw8bGBvXq1QMAeHh4wNTUFKNHj8aNGzcQGhqKWbNmISIiQv4j1MyhJjq5WSBPJrDG74Fybx4praRZjE1MTODm5obOnTvjr7/+wv379wvNbVMVDt2IBQB4NrBGPesaAIC07OID/lcVN8dPWa5zc3NzNGrUSOGlzHU+Z84cSCSSIofJJycno1+/fmjQoAG++eYbpc+rqGUyCsyaNQvBwcH4448/IJVK8fz580IzlFfXz/tVDHaI1EhRCyS+TJU7v0OHDqFGjRrQ09PDihUrcOLECVhYWMi3T5kyBTt37sTixYuRk5ODq1evYvbs2fLtenp6OH/+PPr27QtXV1eMGDECRkZGuHTpEqysrOT7Fcx98iw1C2M2BSIlMxetnEyxdGiTIgM2CwsLHD16FKmpqejevTtatWqFCxcuYP/+/QrB2OR/++7sDnqCOGn1u9N8k1VUC8frIpMJ/HMrP9gZ1soBk/69NnZeiUJunkypYxQ3x4+q17myXr3OW7RogUePHuGrr74qdNORkpKC3r17w8jICHv37oW2trZSZRS3TMbLdahbty7Cw8NhZWWFq1ev4vLly8Uer7p83kVS6uGammOfHVIXnp6e4t133y1ym1QqFW3atBG9e/cW2dnZpR4rNTVVhIWFCX9/f/Hpp58KZ2dnER8fX+z+GzZsEFpaWiIzM1OlOsckpYsz9+NFv1/OCac5h0SnpafFsxTVjlGc91ZfFE5zDomFB+9UyPFIiLy8POHo6CjmzJmjkP7gwQOxePFiERQUJB4/fiwuXrwo+vfvL8zMzBSum3r16ok9e/bI/37+/LkIDg4Whw8fFgDEzp07RXBwsIiNjZXvExsbK4KDg8X69evlc88EBweL58+fK1XnwIjnwmnOIdHo66MiIztXZObkihYLjwunOYfEkVuxpR+ggpRnjp8FCxYIGxsbkZOTo5AulUpFu3btRJcuXURaWppKxxw9erRo2bJlifsUfN7/+9//5H0Bhag+n7eyv98MdgSDHVIPJXWeTE5OFh4eHqJHjx4iIyOjTMd3dXUVixcvLnb77du3BQBx//59pY+588pj4fLScOB6X/4jwuJTylS/opwNScg/7lf/VFgA9bY7duyYACBCQkIU0qOjo0WfPn2ElZWV0NbWFvb29mLkyJGFrgcAwtfXV/63r69vkRPXLViwQL7PggULitzn5eOUZMH+28JpziExfWewPG3JkXvCac4h8eH/XVb1LSiTl691l7mHxM4rj5XOW1yAKZVKRdu2bUXjxo1FeHi4wkCF3Nxc+X6vBhwFeQ0MDMTq1asLlXf58mXx66+/iuDgYLFlyxYBQDRv3lzUqVNHfjNTXT5vBjsqYLBD6qAy7vxeVrt2bYUvpFdt3bpVaGhoiBcvXih1vIinqcL5ldExLnMPlWtm21fJZDLR/9fzwmnOIbH0SMkzu74pSpvVNiMjQ0yaNEmYmZkJQ0NDMWTIEBEXF6f08SdMmCAAFJrzpahyC0a/qaoiZjFWVl6eTLT+7oRwmnNInLz73/sQ+TxNPu/Oy3M4VYaYpHSFoF7VkWDFBZgFo5WKekVERMj3KypQWLt2rdDX1xdJSUmFyrt586bo1q2bMDMzE7q6usLZ2Vl8/OlnYt+Fm2X6zCrz81b291u5HopEVK3JZDL834aN6DlgGJ6m5cDWJP+/dnJyMjw9PZGeno6tW7ciOTkZycn5MwtbWlpCU1MTQH7nSR8fHwwePBhpaWn4/vvvMWDAANja2uLZs2f47bffEB0dLZ851d/fHwEBAejWrRuMjIzg7++P6dOn48MPP4SpqWkJ9RQIiHiBfcHROHAjGuLV7QJ49CxdpREyJZFIJPDq5ooJf1zFFv/HmNC5DkwMlOvPUF0FBgYqzFB7+/ZtvPPOO/LPZvr06Th8+DB2794NExMTTJ48GUOGDMHFixdLPXZJ6zQBwMKFCzFu3Dj530ZGRirXf3vAY3y59zYEAA0J4DOkMUa0dlT5OMoKepyIhJQsGOlpoaPbf33OHMwM0K2eFU7fT8C2y4/x1bsNKq0OEc/SICtmjh9lrnVPT89CHYMBoGvXrkWmv0oIgVhpBi49eAYXC0PYmuhj/PjxRfbdE0LAvnY9rN2xH09Ts/AsNRsn7sTh0M1Y+B2MhORgJLrWs0QDO+NSywWAuzHJOBvy9LV93sVhsEOkBr76bTuin0ThtGiEDktOy79Qrl27Jp8M0NXVVSFPREQEnJ2dAeR3ngwKe4J20gyY6mri/v372Lx5M549ewZzc3O0bt0a58+fl09rr6uri507d+Kbb75BVlYWXFxcMH36dMyYMaPI+oXGp2BvcDT2B0cjpoTOwpoSCZwtDJQ+7+joaMyZMwdHjhxBeno6XF1d4evri1atWgEA4uPjsX3ZXMTtO4Ts9FS0+6cNDmzfADc3t2KP2bVrV/j5+RVK79u3Lw4fPgwg/wdh+vTpWLN2LbIyM6GhoQFXV1ds27ZNoew5c+bg+PHjSEpKQufOnfHrr7+Wu2wLCwssWLAA69evR1JSEiwtLeHo6IguXbpAKpViw4YN2L59O7p37w4gvzNx/fr1cfnyZbRr167E9/Lzzz/HsWPHiu14XDDVQFnFSjPkgQ6QH9zO23MbnetaVliA+6pDN2MAAJ4NbKCrpamw7cN2jjh9PwG7rz7BF73qQU9bs6hDlJuFoW6hNA0JVLrWy2NXYCS899yCTAASCTCilQNcrWrkBzMp2XiWmiV/PU/NRu6rkdlLBIAzIU9xJuSpyvV4HZ93cRjsEL3hYpLSsSPaFE5zDgHI/0KZ8/ctbLr0GLVqGmLOXzdgUUMXFjV0YGmkB4saOrAw0oVpDV0IIfBnUBRc5h7CthfAjn8DpT179pRYZosWLUoclQEA8cmZOHA9BnuDo3E39r91qoz0tNCvsS0GNa+FiGdp+GrvbeQJAU2JBIuHNFL6SzAxMREdOnRAt27dcOTIEVhaWiIsLEzesiSEwKBBg6CtrY3vf9+C5WejkBB8AD169MS9e3dhaGhY5HH37NmD7Oxs+d/Pnz9H06ZN5S0nQP4M0ytXroSuQ2OYN3kHabdOIv5ZDAwMDAqVvX//fhgbG2P58uXo2bMn7t4tX9nLli3DypUrsXnzZtjb26N9+/YwNDREVlYWrl69ipycHPTs2VO+v7u7OxwdHeHv719ssFPaOk0FlixZgkWLFsHR0REjR47E9OnTlZ7CAADO3E8o1JqnSguHqvJkAv/cyl8h/N2mtoW2d6lrBXtTfTxJzMDBGzEY1qpyZv7dez26UJqbVQ3YGBce7l3RYqUZ8kAHAIQAdgZGlZrPWE8LFka60NHUwP24lELbezeyLvUzi03KwNE78Qpplfl5l4TBDlEFUKaFQdW7/D179mDx4sUIDw9HTk4O3NzcMHPmTHz00UcAgJycHMz1nocNO/Yg+Wk0NHQNoefUFDW7fAItI3Pci03GvdiSF8PU1pAg56W7OJkA5v59C7FJmahtVSM/QKqhC4sauqhpoF3s3D2x0gxEPEuDlZEebkQlYd/1aFwMfyb/gtXWlKBrPSsMbl4L3d2t5HfQ7Wqbo2s9Szx6lg5nCwOVvgCXLl0KBwcH+Pr6ytNcXFzk/w4LC8Ply5dx+/ZtuNdvgEOxfnhoNhGJ6z7Bjh07ih1ua2ZmpvD3zp07YWBgIA84hBD48ccfoWlsBesPFgMADFzb4MmvH8LvchAaNGigUHZB8LB69WrY2NiUu+yff/4ZX331FQYOHIg///wTeXl5SEtLk084p6Ojg5o1ayocx9raGnFxcSW+l1paWpgyZUqx+0yZMgUtWrSAmZkZLl26BG9vb8TGxmL58uXF5nlZTp4Mmy49KpSuamueKq5EvMCz1CyY6GujQx2LQts1NSQY2dYRy46GYGtAZKUEOwnJmfC9GAEAWDq0CTQ1AO+/byEkPhV7g6MxpIV9hZf5socJhR+hAYBHHXO42xjBooZu/v9xI51/b4p0YV5DR94KFivNQIclpxWOoSmRYEH/hqUHO9IMHL8bXyjv62rRehmDHaJyUqWFQZW7fDMzM3z55Zdwd3eHjo4ODh06hDFjxsDKygq9evVCeMxzbD54FtqthsHWygWyzFS8OLUOT/csQq1PfobP4MbIFaJQM/Wz1Gw8S8lCSlauQqBTQAD4+VRYoXQtDQnMa+R/IVoa6cq/GKNepOOfW7GF7tgBoKWTKQY3r4V+jW1haqhT5HnamuiX6S7vwIED6NWrF4YNGwY/Pz/UqlULkyZNkvcpKZjrQ09PD5oaEvyvax3M/usmMmWa8Dt3vtiA41UbNmzA+++/L/+cHj58iPT0dBg2aIOn+3yQGXUbmjXMoWlsiTPnLuB/n36sUHYBDQ0N6Orq4sKFC2UuOyIiAnFxcfKWm4JZjJOTk+Hv74+2bdsqddyXFazTdO3atWKDWQAKjyibNGkCHR0dTJgwAT4+PtDVLfyY5lXrzj1EaHwq9LU1kZWbJ/8BnOlZt9Lu8g/fyn+E1auhNXS0ip5WbngrB6w4EYobUUm49USKxvYmFVqHX0+HIzNHhhaONTG8lT0kEgnik7Pww7EQfHvwLjq6WcDKqPJaeC4+eFYoTVMiwfLhTZV6321N9OEzpDHm7VG9BbY8eSsagx2iclKlhUGVu/yuXbsq/D116lRs3rwZFy5cQE23Vpi07RZqDP4G5oY6eK+lPf7vfATwzkTEbZmBme3NMKJNyZ0AM3PycDdGiqFr/BUWSJQA6OZuhbSsXHlwJM3IQa5MID45C/FFTLv/qnGdXPBRO2c4mlfeHdzDhw+xevVqzJgxA/PmzUNgYCCmTJkCHR0djB49Wv74xtvbG2vXrkW/hpaYNX8fcpKf4npIhFJlXLlyBbdv38aGDRsAAJHP0+G14QwAIC3kIoxbD4K1x3BkxYbhxfHfccrvImKSMgqVbWhoiBUrVuDJkyeIjY0tU9kA5K0z1tbW8lmM9+zZg23btiEuLg42NjbIzs5GUlKSQutOfHx8sX1tzp8/j4SEBDg6/ne95OXlYebMmfj555/x6NGjIvO1bdsWubm5ePTokXzW3+KEJ6Tgl38D6O8GNUJ7V3NM+OMqbj6R4kVadol5yyo3T4ajt/Pfr35Niu5wDQAWNXTRt7Et9l+PwdbLj7H0vSYVVofHz9Ow40okAGB2b3d5MDm+c238cysWd2KSsWD/Haz+sGWFlfmyq48Tse7cQwD5fXWEQJkCjhGtHdG5btlaYMuTtyJxBmWicjpw4ABatWqFYcOGwcrKCs2bN8f69evl20u7y1eGEAKnTp1CSEgIsi3rYdT/BeB5WjYa2hnjwOcd4d23Pi7M7QbvHk6QSCT4qEvx/S4K6GlrooWTGZYMaQzNf7+ENSUSLBnaGBs/aY1dEzxwamZX3FjgidDv+sDfuzsOTO6AjZ+0wrKhTTCrVz30bmhd5LG7u1tXaqAD5PczadGiBRYvXozmzZtj/PjxGDduHNasWQMA0NbWxp49exAaGgozMzOYGNWASWIo9Gq3xJOkTGTnlj5z7oYNG9C4cWM0b9kKa/wewPNnP1yPTAIAWDi4wrzraOhY14Fxs97QqmmNpLgovLPcD1uvPMHuv/6Wl21gYIAzZ86gT58+hdb0Kq3sNm3aFLm9qFmMW7ZsCW1tbZw6dUqeFhISgsjISHh4eBR5HGXWaSrK9evXoaGhoTBLcFHyZAKz/7qJ7FwZutazxJAWtWBroo/pPesCAP4MikK6Css2KCv/EVY2ahpoo30d8xL3/bCdEwBg/41oSDNyKqwOK06EIlcm0LmuJdrV/q8O2poaWPZeE2hpSHDkdhyO3FIuAFaFND0HU3YEI1cm8G4TW1yc0w07xrXDhbndyjQaytZEHx51zMsUrJQnb0Vhyw5ROanawqDKXb5UKkWtWrWQlZUFTU1NdB/rjV0xpgAE+je1w7KhTaCvk/9s3VRXgs2/fI8PPvgAxsbKDQsFlLvz0tHSKPJxU1U+k7e1tUWDBorDhevXr4+///5b/nfLli1x/fp1SKVSZGdnw6imGSxqN0KuSR3sDX5S4pd+Wloadu7ciQnTvdH/1wvyTpqt67vgEICOzerh97nd5e+bxy59PElLRFp2Hr49eBdN7U2w7R8/2Bvmry9maWmJtm3byvtxlaSg7IULFyqkF7TOxMbGKkw1EB8fj2bNmsHExARjx47FjBkzYGZmBmNjY3z++efw8PBQ6Jz88lQD5ubmMDdXDAZeXaeprFMNAMDmS49wLTIJNXS1sHhwY3nrRpe6lnA2N8Cj5+nYGxyNUW2dSn1fVHHo3wCid0MbaGuWHGC2cjJFPWsjhMSnYM+1JxjTwaXE/ZVxLzYZ+2/kP0ab3atwy1dDOxNM7FIHq86EY/7+O2hX27zYR72qEkJg9t83EJ2UAUczA/gMaQwjPW3Y1Xz9fWWqC7bsEJWTqi0MqtzlGxkZ4fr16zh29gLq9huHoxt+QGbkTczt446V7zeTBzo5OTkYPnw4hBBYvXq1yudQ1juvgmfyL7cMva5n8h06dEBISIhCWmhoKJycCv9ompiYwNLSElGPHiI9JhQGbm2x+uyDEtdF+mP7TqRlZGJnkhPux6WgpoE2fnivCfZ7D4aenh5u3Lghf98MJTl4EvUYtZ0c8N2gRjDS1cKNJ1IMWHURv1+MgaGJKcLCwhAUFISBAweWem67d+9GVlYWPvzwQ4V0FxcX2NjYYNaiH+VTDXh8ewj+lwPkLTcrVqzAu+++i6FDh6Jz586wsbEpNLouJCQEj2Of4tKDZ4iVZpRan4KpBrp06YKGDRvi+++/x/Tp07Fu3boS80U+T8cPx/I/o7l93GFX87/rQkNDgo88nAHkB0TKzBejrJcfYb1bwiOsAhKJBB965F83Wy8/rpC6/HgsBEIA/ZrYolGtovsBfd7DFa5WNfAsNQuLDt0td5kFtvg/xrE78dDWlGDVyOYw0nuz55aqEBU+neEbiDMoq4cnT56IUaNGCTMzM6GnpycaNWokAgMD5dtRzEyjy5YtK/aYubm54quvvhLOzs5CT09P1K5dWyxcuFDIZDL5Po6OjqJZs2bCxsZG6OnpiR49eohvvvlG2NnZFTpeUlKSSEhIEEII0aZNG/mstyW5EZUo2n5/UjjNOSRMm/cSrTp0VdienZ0tBg0aJJo0aSKePXtW6vEqQ0xSurgU/uy1zIhb4MqVK0JLS0t8//33IiwsTGzbtk0YGBiIrVu3yvf5888/xZkzZ8SDBw/Evn37hJOTkxgwaLBo9u0x4TTnkNgX/ER89NFHYu7cuQrHPnY7Vhg5NRIG7p3kywy8vNyEl5eXACA+/PBDcejQIdGyZUshkUjExo0bhRBCxEkzhOfnS4T1B4uF3YT/E+4fLRTWdg5iyJAhCuUUVbYQQnTs2FGMGDFCIS0vTyZuPUkSvcfMEBq6hsJyyHxh++kqoe/WTmiZWIsuPsfEqPWXxdQd18Sig3fE6rPh4q+gKHE2JEHcjk4S8dIMkZObJ4Qo39IFypLJZOKDdf7Cac4hMWLtJZGXJyu0jzQjW9Sff0Q4zTkkLoY9rbCyz4XmLxPSfOFx+TmXJiUzRzQoqEt4+epSsBZXbe/D4kFCycufXH38Qj6T8+l7xa89p6xbT5KE27x/hNOcQ2LD+YflPl51xxmU6a1S2ogoAIUeGR05cgRjx47F0KFDiz3u0qVLsXr1amzevBkNGzZEUFAQxowZAxMTE/kwXVNTU9y6dQt///03XFxcMH/+fCxfvhz169cvdDwTk/w7vIK7/EWLFpV4XnuuPcHcPbeQnStDHUtDmLlb4mnMf3NkFLTohIWF4cyZM4UeR7wuZR1RVR6tW7fG3r174e3tjYULF8LFxQU///wzRo0aJd8nNjYWM2bMQHx8PGxtbfHxxx9j/vz5WHfhMX48HorfzoQjOzJS3sIWJ83EggO3cejcVaQ8vo1GY5fh97Ft0MnNUqHsX3/9Fc+ePcPOnTuxdetWGBgYYOHChRgzZgwAwNpYD/1c9RG8eykSniYg3tAUNRp2h82gGXiakgVLo/zRS5EvlV0gJCQEFy5cwPHjxxGdlIELYU9xPuwZLj14jhdp2RCW3WDUMgbPj/0KWWYa9OwbwGr4QjxKysGjpMIjb14mkQAmetpIeqlfSmVN9LYzMAqXHjyHnrYGlg5tAg2NwiO9jPW0MbSFPf64/BibLj1Ce9fCw8PL4vDNfx9hNbKBVimPsArU0NXCoOa1sC0gEtsuR6J9EUPVlSGEwLKj+a1Zw1vZo7ZljRL3b+Foik87uGDDhQjM23sLx6d3LnNLTGpWLiZvv4bsPBl61rfGmA7OZTqOOpIIUYFth2+o5ORkmJiYQCqVqtTXgaqPuXPn4uLFizh//rzSeQYNGoSUlBSFzpyvevfdd2Ftba0wImbo0KHQ19fH1q1bIYSApaUlEhMTsWjRIgwfPhxnz57FuHHj4OXlhVWrVgHIfyxRMNPtrVu3MHXqVLRs2VKhf8nHH3+MWrVqwcfHB7l5MvT5dCZuZppDy9QW7RyN0FLjERbM/xKrV6/GZ599hpycHLz33nu4du0aDh06BGvr/zoLm5mZQUenYp7/qyNpRg46LjmNlKxcTO9ZF0Nb1sLp+wlYdjQEqVm50NKQYFzn2pjS3U3+qLCs0rJy8dPxUGy6FAGZyJ+sbV7f+hjeygHxKZmIeJYmn8I/JTMHlx++yA9wwp/h4dM0hWMZ6miiuWNNXAx/rjDUX0MC/DS8GWQyoTjFQGoWnqbk//tFWlaR860U2DGuHTxK6cirrFhpBjyXn0NKVi6+6lcfn3WqXey+4Qkp6Ln8HDQkgN+sbnAwK1+/kpw8GVp/fxJJ6TnY/llblQKoe7HJ6PPLeWhpSHBpbndYlWHSvzMhCRjjGwgdLQ34zeqqVACZkZ2HXj+fQ+SLdIxq64jvBzdWuVwhBKbtuo7912NgZ6KHf6Z2Qk0D9f8OUPb3my07pBZKm3PlVfHx8Th8+DA2b95c4nHbt2+PdevWITQ0FHXr1sWNGzdw4cIF+URqEREReP78OX755ResX79e3sLg5uYmX3cKKL6F4WUFd/lJ6dn4fEcwroTFIv3+LiDtBc4YGiDW3R1bt27FiBEjAORPZHjgwAEAQLNmzRSOdebMmUJD1+k/JvraaO1shtMhCVhxMhQrTobKtzV3rAmfIY3hblMxNz6Gulr4un8DDGpuB+89t3AnJhlz99zCWr+HePwif8I3CQBHMwM8ScpA3ksRiYYEaOpQE51cLdCpriWaOdSEtqYGdgVGFpq7ZHDzWiXWI08mkJiejXuxyfh445VC0w04mVdMq44QAl/tvY2UrFw0c6hZamdfVysjdHKzwPmwZ9h6+TG8+xZuEVXFpQfPkZSeA4saOmjjYlZ6hpfUtzVGKydTBD1OxM7AKEzpUfykn0WRyf5r1fmkvbPSLWX6OppYMrQxRq4PwLaASLzbxE7lwHN30BPsvx4DTQ0JVn7Q/K0IdFTBlh2wZUcdFAzrnjFjBoYNG4bAwEBMnToVa9aswejRowvtv2zZMixZsgQxMTEKQ8JfJZPJMG/ePCxbtgyamprIy8vD999/D29vbwDApUuX0KFDB8TExMDW9r/p6IcPHw6JRIJdu3YpfQ6x0gycD3uGn0+GIiYpE/ramvhxWFP0a1J4mnsqn6JmhQWALzzr4n9dXaFZxCOXipCbJ4PvxUf46XgIMosZ+u5sboCObhbo6GoJjzrmMNEv+pFGrDSjzHOX5AdLt5D30vmPauuIhQMblfvc91+PxtSd16GjqYFDUzqirnXpi4WevBuPz7YEwURfG5e9e5SrNW32XzfwZ9ATfNjOEd8NUr2FZF9wNKbtug5bEz2cn91N6cdgAHDgRgym7AiGka4Wzs3upvLoqnl7b2F7QCSczA1wdGpnpd+HsPgU9F91AZk5MszqVQ9e3VxLz6Qm2LJDbxWZTIZWrVph8eL86fubN2+O27dvFxvsbNy4EaNGjSox0AGAP//8E9u2bcP27dvRsGFDXL9+HdOmTYOdnV2Rxy2rLZceYcGBO/JHE6YG2tj2WTulVxYm1RS1CjUAtHQyq7RABwC0NDUwrnNtWBnpYuqu64W2r3y/GQY0K7mFpkB5+km9PN1A0OMXWH4iFNsCIvEsNQu/vN+8zAtiPkvNwjcH7gAAPu/uqlSgA+RPYuloZoDIF+nYdz0aH5QyIWZxsnNlOPbvWkzKjMIqSp/GNlh4SAex0kycvp8Az4bKLXyakyfDT8fzW3XGda5dpmHk3n3cceZ+Ah4/T8dPx0OUWok9IzsPXtuvITNHhk5uFvhflzoql/s24NBzUgvFzbkSGRlZaN/z588jJCREqSn7Z82ahblz5+L9999H48aN8dFHH2H69Onw8fEB8N+8J/HxiovdlTRjbYHcPBn8Qp9iwpYgfP1SoAPk9ykxNeRw0criYmGIV2Oa17lmT5vaZkWW31rFxy7lUTBs/vPubvhtZAvoaGrg2J14fLzhCqTpZZtYb8GBO0hMz0F9W2NM7Kr8j66mhgQf/zv0uzzD0C+GP4M0IweWRrpo7Vy291JXSxPD/10j64/Lj5XO92dQFB4/T4e5oQ7GdizbPD1GetpY/G9/nY0XI3AtMrHUPAsP3UFofCosauhi+fBmRXYEJwY7pCZUmXNlw4YNaNmyJZo2bVrqcdPT0wuNltHU1IRMlv8IomDek5c7OScnJyMgIKDIGWuFELj1RIqFB++inc9pjN54BcfuxhfaTyaAR8/SS60flU1Vzg9UHcp/Vd/Gttj8aRsY6WrhyqMXGL7WX6n5d1527E4cDt+MhaaGBD+816TUifxeNayVA/S1NXE/LgWXH75QKW+BQ/+OwurbyKZcLXSj2jpCIgHOhz3Do2dppe6fkZ2Hlf8uhzG5uysMdcv+0KSbe/6CuTIBzP7rJrJy84rd98CNGOy4EgWJBPjl/WbyUX5UGB9jkVqYPn062rdvj8WLF2P48OG4cuUK1q1bV2jSs+TkZOzevRs//fRTkcfp0aMHBg8ejMmTJwMA+vfvj++//x6Ojo5o2LAhgoODsXz5cnz66acA8icjmzZtGr777ju4ubnJh57b2dlh0KBB8uNGvUjHgRsx2BscjfCEVHm6qYE2urtbYU9wtEKH0apaGfhtUtVr9lR1+a/yqGOOPyd6YPTGKwiJT8HQ3y9hy9g2cLUq/VGUND0HX+27DQCY0Ll2sZPolcREXxtDWuQP/d586ZHKHXSzcvNw/G7pa2Epw8HMAF3qWuJsyFNsvxKJeaV0mt7s/wjxyVmoVVMfI9uW7RHcy75+twHOhz1FeEIqVp0Ox0zPwjMwP3qWhnl7bgEAJndzRYcKGravrhjskFpQZs4VANi5cyeEEPjggw+KPE5oWDiCQyMRK82ArYk+fv31V8yfPx+TJk1CQkIC7OzsMGHCBHz99dfyPLNnz0ZaWhrGjx+PpKQkNG7ZFlt270OWTBN7r0Ri77VoXHn0352qrpYG3mlgjcHNa6FzXUtoa2qgjYtZtVgZ+G1TFfMDVafyX1Xf1hh7JrXHxxuv4OHTNAxd7Y+Nn7RCS6eSHwl9d/gunqZkoY6locojmF42ur0ztgVE4vjdOEQnZaBWTeXfmwthz5CSmQsrI120cip5CQtlfNTOCWdDnuLPoCjMeKdusf2YpBk5WH32AQBg+jt1oatVvqkKAMDUUAcLBzbCpG3XsPrsA/RpZKvQfy8rNw+Td1xDalYu2jibYWo53vO3BUdjgaOxKN+uwEh477kFmcgf8uszpLFKC+a9nB/Ib53J+/e/l0QCeNQ2x6DmtdC7kQ2Mi5g0rDyja4gq0ou0bHy6KRDXo5Kgq6WBVSNb4J0GRS/66hf6FKM3XoFEAvw10aPUwKg0I9dfxqUHzzGxSx3M7eOudL4Zu65jT3A0PmnvjG8GlL4QbmnyZAKdl51BdFIGfhrWFENb2he53w/H7uO3Mw/gZlUDR6d1rtAO7hP/uIqjd+LQ0M4Y+7w6yB8NfnvwDnwvPoKpgTb+mdrprf6+4GgsIiVl5uTh6O04zP37lryTsEwAc/6+BZ9/7ivV4U8mEwqz0gJAnhCoY2mI4a0cMKCZXalfSNXtLp/eXmaGOtg+ri28tl3DmZCnmPBHEBYPboz3XxkllZqVK3+UMtrDudyBDpA/P82lB8+xMzAS03q6KTUyLDMnDyf+7fvWv2nFTNWgqSHByLaO+OFYCLYGPC4y2ElIycTGC48AAF/0qlfhI/kWDmoI/4fPcScmGevOPYRXN1ccvxMH34v5Zf40vCm/M5TEYIfeOjKZwN3YZJwPe4YL4U8R+CgR2cXMefJqAKOq7wY1rrBZaYleJwMdLaz7uBW899zCX1fzly15mpKFyd1d5SuXLzt6H9FJGXAw08fs3oX7lZRFj/rWsDfVx5PEDBy4HoPhrR1KzXMu9ClSsnJha6KH5g7lf4RVYHgrB/x8MhTBkUm4HS0t1Bfpt9PhyMjJQzOHmvAspuWrPKyM9PD1uw0wc/cN/HwyFJoaEvx2JhwA8FlHF3R3r/gy1RWDHVIrsdIMhen3C8QkZeBC2DOcC3sqX2PoZVZGukhIyVJI05AAW8e2VWqEw9OULHy4IUBh7hZ2MqY3nbamBn54rwmsjXXx25kH+OlEKOJTMjGxSx0cvxOPLf75Q7OXDGkCA52K+TnR1JDgo3ZO8DlyH76XHmFYK3t5cFWcw7f+HYXV2LZCh15bGumidyNbHLwRg20Bj+EzpIl8W9SLdGy/kj+1xeze9UqtY1kNaVELa889QGh8KpYcuQ8AsDfVx+zeyj/iIwY7pEZe7XPzYTsnSIBi1xjyqGOOjq4W6OhmiTqWhvgzKKpQJ2Fl19VxszaCz5DG7GRMakcikWBWL3dY1tDFt4fuYuvlSGy9/N/8Va2dTSt8JNCI1g5YcTIU92KTEfgoscRlHzJz8nDy30dYlTHb+IdtHXHwRgz2BcfAu299eX+7FSdCkZMn0MnNosyLhiojLjlTYQQnkH/z9jwti98vKmCwQ2ohVpqh0DlYJiC/6wSKX2PoZeUdClzdhhITVaRPOrhAU1OC+fvuKKRffZwoH71YUWoa6GBw81rYcSUKmy89KjHYORvyFGnZeahVUx/NHWpWWB0KtHExQ13rGgiNT8Xea9EY3d4Z9+OSsfd6NABgVq+KeXxXnKJm+y6Yh4vfMcpjsENqobjp/3vWt8J7LR1KXGPoZeXtJMxOxqTO6ljWKJRWWT+8o9s7Y8eVKBy9E4eYpAzYFTMM/b9HWDaV8ihJIpHgw3ZO+Hr/HWy9/Bgfezjhx2OhECK/zCb2NSu8zJcVzPbNR+TlwxmUSS24WBji1a85TYkEiwY1Qu9GNkoFOkRUste5zIa7jTHa1TZDnkxgW0DRyzZkZOfh1L3yrYWljMHNa8FARxNhCalY7fcAJ+/FQ0MCzHinclt1gOo32/abisEOqYVnKdl4OdrhFwJRxXvdP7yftHcGAOy4EoXMnMLLJpwNSUB6dh7sTfXRxF71WZuVZaSnjUHN8xdoXXY0f1maYS0d4GpVuKWrMoxo7YgLc7thx7h2uDC3m0rzf1E+PsaiClfciKjKkp0rw6y/bkAIoIe7FT7rVJt9Zogqyevsm9azvjXsTPQQI83EwRsxGNZKcRh6wVpY/ZrYVtpoqAKWNRRHZda2NKzU8l7FR+Tlw5YdNRQdHY0PP/wQ5ubm0NfXR+PGjREUFAQAyMnJwZw5c9C4cWMYGhrCzs4OH3/8MWJiYko85rlz59C/f3/Y2dlBIpFg3759CtsLjuvo6o5alqbo3Kwe6nboh9X/BFbWacqt8XuA+3EpMDXQxrL3msCjjjm/FIgqUcGK6ZX9/0xLUwMfeTgDADa9shp6enYuTt3/9xFW48p7hAXk38D9ejpMIW3Z0RCVF0ulqsNgR80kJiaiQ4cO0NbWxpEjR3D37l389NNPMDXNn2grPT0d165dw/z583Ht2jXs2bMHISEhGDBgQInHTUtLQ9OmTfHbb78VuT09PR3+V4KQ2WgQbEf/AstB85D9IhrTxo6s1C+E0PgU+ZfQNwMawrwGV/0lUifvt3aArpYG7sQk4+rjRHn66fsJyMyRwdHMAI1qVe4yP0UNgMgTAo+epVdquVRx+BhLzSxduhQODg7w9fWVp7m4uMj/bWJighMnTijkWbVqFdq0aYPIyEg4Ohb9LLhPnz7o06dPseWamJjggwVrsfTf59kAYPbORMRtmYHLN0MxuFPTsp5SsfJkArP+uomcPIGe9a0woGnl3t0R0etnaqiDgc3s8GfQE2y69AitnPOHoR9+jY+wOCLqzceWHTVz4MABtGrVCsOGDYOVlRWaN2+O9evXl5hHKpVCIpGgZs2aZSozKzcPCw/eVQh0AECWlQ5AAoMalXPXtfFCBG5EJcFIVwvfDWpc6V94RFQ1Rv/bUfno7TjESTORlpWL0/cTAADvVsJEgq/iiKg3H1t21MzDhw+xevVqzJgxA/PmzUNgYCCmTJkCHR0djB49utD+mZmZmDNnDj744IMyrfgenpCCz3dcx73YZABA+zrmuPzwOfJyspF01hcGDTpj4bEItHC1U2rZBWU9epaGH4/nB1df9qsPGxO9Cjs2EVUvDe1M0MbZDFcevcC2gMdwszZCVq4MLhaGaGBbuY+wCnDS0Dcbgx01I5PJ0KpVKyxevBgA0Lx5c9y+fRtr1qwpFOzk5ORg+PDhEEJg9erVKpUjhMD2gEgsPHQHmTkymBvq4IdhTdDd3RqRz5IxasRwGFoawnzELEQlZeCzLUHYNb6dUisYl36OAnP+vomsXBk6uJpjhBILBRLRm210e2dcefQC2wMi5cPM+zWu/EdYL+OIqDcXH2OpGVtbWzRo0EAhrX79+oiMjFRIKwh0Hj9+jBMnTqjcqrPG7yHm7b2FzBwZOrlZ4MjUTujubo2cnBxMHTcayc9icf7saWyZ2BU1DbRxIyoJM/68DllR0xyraPuVSAREvIC+tiaWDGnCx1dEbwHPhtawNdHD87RsnAl5CqBy1sIi9cRgR8106NABISGKfWdCQ0Ph5OQk/7sg0AkLC8PJkydhbm6u9PH9HzwHAARHJkJbU4Iv+9bH5jFtYGWsh5ycHPTv3x9nzpxBVFQU7O3tMaBbO0xrogFtTQn+uRWHZcfu4+uvv4atrS309fXRs2dPhIWFlVimj48PWrduDSMjI1hYWmHS6A+Q8/wJZveuBwez/A6CEyZMQJ06daCvrw9LS0sMHDgQ9+/fV/q8iKh609bUKDRx4I2opKqpDL1xGOyomenTp+Py5ctYvHgxwsPDsX37dqxbtw5eXl4A8gOd9957D0FBQdi2bRvy8vIQFxeHuLg4ZGdny4/To0cPrFq1Sv53ojQZ03/fh6HfbwcAmOQmwqeLEXo5a0FDQ4KcnBwMGDAAJ0+eRJcuXbBt2zb4+fnhyy+/RNt6tbBkSBMAwNKly/DTil+wZs0aBAQEwNDQEL169UJmZmax5+Tn5wcvLy/4+/vDY/Jy5ObmIHHPArzXxFK+T8uWLeHr64t79+7h2LFjEELA09MTeXmFZ10lojdPrDQDJ/5d3bzAl3tvc64bUo4gIZVKBQAhlUqruioV4uDBg6JRo0ZCV1dXuLu7i3Xr1sm3RURECABFvs6cOSPfz8nJSSxYsEAIIcTjZ2miw5SVReYZPXq00sf98eg9oWloKsy7fyouhj0VQgiRlJQkdHV1xY4dO0o9r7+CooTTnEPCZdoOAUD4+fkVu++NGzcEABEeHq7am0dE1dLF8KfCac6hQq9L4c+qumpUhZT9/WYHZTX07rvv4t133y1ym7Ozs8IspMXxv3EPEc/SsOliBH48HopU/dpovOAofIY0KfI5ubOzM+rXr49evXrhyZMn8PPzQ61atTBp0iR07doVADDIVRtfpCVCx7EZJm69ij2TOsDVygRt27aFv78/3n///WLrk5CSiYWH7gIAPm5liW8BmJmZFblvWloafH194eLiAgcHdl4mUgec64bKg4+xqJBdgZHosOQ0Rq4PwDcH7yI1KxetnU1xZFrnEjsEFgx7d3Nzw7Fjx/C///0PU6ZMwebNmwEA8fH5TdDN6jkhOTMXYzZdwfPULFhbWyMuLq7EOn297w6kGTloaFsDgduXo0OHDmjUqJHCPr///jtq1KiBGjVq4MiRIzhx4gR0dHTK+W4QUXXAuW6oPCRCmdv8l5w5cwbdunWrrPpUieTkZJiYmEAqlZZprhl1IU3PweFbMZi397ZCugTAudld4WBW8sJ3Ojo6aNWqFS5duiRPmzJlCgIDA+Hv749Lly6hQ4cOuB0Wgf/tiUDki3S0dDIFTq2AlqYGdu3aVeRx/7kVi0nbrkFLQ4JWT/7G5XOncOHCBdjb2yvWXypFQkICYmNj8eOPPyI6OhoXL16Enh7n4CFSF7HSDM51Q3LK/n6r/Bird+/esLe3x5gxYzB69Gg+JniDZefKcC0yERfCnuF8+DPcepJUaP0XIL/jzZPEzFKDneKGvf/9998AABsbGwBATmoSNn7SGoN/v4irjxORc+chhr7TochjJqZl4+v9+cGX9e1tuBh8DufOnSsU6AD5S1aYmJjAzc0N7dq1g6mpKfbu3YsPPvigtLeCiN4QnOuGykLlx1jR0dGYPHky/vrrL9SuXRu9evXCn3/+qTCSh6perDQDlx48UxipIIRAaHwKNlyIwBjfK2i28DjeX3cZq86E40ZUfqDjbG6AV2etUfa5eGnD3l1cXGBjY4NTp07B1aoG1n7YEho5GYgJu4XEGs5FHnPhobt4mpKFvAv/h0dXz+L06dMKa30VRwgBIQSysrJK3ZeIiNSbyi07FhYWmD59OqZPn45r167B19cXkyZNwqRJkzBy5EiMHTsWTZtW/KKPpLxdgZHw3nMLMgFoSIDhrRyQkydwIfwp4pMVf/wtauigg6sFOrpaoKObBWxN9LErMBLz9txGnhAqPRefPn062rdvj8WLF2P48OG4cuUK1q1bh3Xr1gEAJBIJpk2bhu+++w5ubm5wcXGBedB6RNUww7lsF/x99QmGtrRHjx49MHjwYDTsORx7g6OReGI1xIMLOHTgAIyMjOT9e0xMTKCvr4+HDx9i165d8PT0hKWlJZ48eYIlS5ZAX18fffv2rfg3mIiI3igq99l5VUxMDNatW4clS5ZAS0sLmZmZ8PDwwJo1a9CwYcOKqmelUqc+O7HSDHRYcrrIx1EAoKulgTYuZujoaoFObpZwtzGChkbhGYjL+lz80KFD8Pb2RlhYGFxcXDBjxgyMGzdOvl0IgQULFmDdunVISkpCx44d0ez9mfgrXAZtTQm2jm2LEd1b4oMPP8JZw66IlWbi8dKiR5b5+vrik08+QUxMDD777DNcvXoViYmJsLa2RufOnfH111+jXr16StediIjeLMr+fpcp2MnJycH+/fuxceNGnDhxAq1atcLYsWPxwQcf4OnTp/jqq69w7do13L17t1wnAeQ/NpszZw6OHDmC9PR0uLq6wtfXF61atQLw34/n+vXrkZSUhA4dOshHBClLnYKdSw+eYeT6gELp/ZvaYkQrR7RyNq2Q9akqkkwmMHnHNfxzKw41DbSx7qOWWOP3EKfvJ8DJ3ABHp3aGvk71qjMREVW9Suug/Pnnn2PHjh0QQuCjjz7CsmXLFIYAGxoa4scff4SdnV3Zav6SxMREdOjQAd26dcORI0dgaWmJsLAwmJqayvdZtmwZVq5cic2bN8PFxQXz589Hr169cPfu3bdyFE5xc1HM61u/2nbq09CQYPnwZohOuowbUUkYvvayfFuvBjYMdIiIqFxUbtnp0aMHPvvsMwwZMgS6urpF7pObm4uLFy+iS5cu5arc3LlzcfHiRZw/f77I7UII2NnZYebMmfjiiy8A5A8/tra2xqZNm0qcpO5l6tSyAwDTdgZj3/UYAP/NRTGitWMV16p0t6OlePfXCwppmhIJLsztVm0DNSIiqjrK/n6rPBrr1KlT+OCDD4oNdABAS0ur3IEOABw4cACtWrXCsGHDYGVlhebNm2P9+vXy7REREYiLi0PPnj3laSYm/83IW5ysrCwkJycrvNRJSmYuAGBEawdcmNvtjQh0ACA5M6dQWp4QePQsvQpqQ0RE6kLlYMfHxwcbN24slL5x40YsXbq0QipVoLQZeQtG5VhbWyvkK21GXh8fH/mcLCYmJmo1V1BWbh4u/bsy+UftnN6oFpGCR3Av43TwRERUXioHO2vXroW7u3uh9IYNG2LNmjUVUqkCMpkMLVq0wOLFi9G8eXOMHz8e48aNK3c53t7ekEql8ldUVFQF1bjqBUYkIiMnD5ZGumho92Y9kuN08EREVBlU7qAcFxcHW9vC6yNZWloiNja2QipVQNkZeePj4xXqFB8fj2bNmhV7XF1d3RIfw73JzoYkAAC61LWERFJ4SHl1N6K1IzrXteR08EREVGFUbtlxcHDAxYsXC6VfvHixQkZgvUyVGXkLJCcnIyAgAB4eHhValzfF2dCnAICu9SyruCZlZ2uiD4865gx0iIioQqjcsjNu3DhMmzYNOTk56N69O4D8TsuzZ8/GzJkzK7RyZZmRd/78+bCzs8OgQYMqtC5vgieJ6QhPSIWGBOjk+uYGO0RERBVJ5WBn1qxZeP78OSZNmiRfD0tPTw9z5syBt7d3hVaudevW2Lt3L7y9vbFw4UK4uLjg559/xqhRo+T7zJ49G2lpaRg/frx8Rt6jR4++lXPsnA3Jb9Vp4WgKEwPtKq4NERFR9VDm5SJSU1Nx79496Ovrw83N7Y3uA6Mu8+x8tjkIJ+/F4wvPupjcXfkZpImIiN5ElTaDcoEaNWqgdevWZc1OFSx/yPkzAEDXelZVXBsiIqLqo0zBTlBQEP78809ERkbKH2UV2LNnT4VUjFQT9CgR6dl5sKihiwa2b27rFBERUUVTeTTWzp070b59e9y7dw979+5FTk4O7ty5g9OnT8PExKQy6khKeHnIeVGrmBMREb2tVA52Fi9ejBUrVuDgwYPQ0dHBL7/8gvv372P48OFwdHwzliVQRwWdk7u8wUPOiYiIKoPKwc6DBw/Qr18/AICOjg7S0tIgkUgwffp0+ZBwer2ikzIQ9u+Q885uFlVdHSIiompF5WDH1NQUKSkpAIBatWrh9u3bAICkpCSkp3PBxqrg92+rTjOHmqhpoFPFtSEiIqpeVO6g3LlzZ5w4cQKNGzfGsGHDMHXqVJw+fRonTpxAjx49KqOOVIqC/jochUVERFSYysHOqlWrkJmZCQD48ssvoa2tjUuXLmHo0KH46quvKryCVLLsXBkuhhcMOWd/HSIiolepFOzk5ubi0KFD6NWrFwBAQ0MDc+fOrZSKkXKCHr9AWnYeLGrooJEdR8MRERG9SqU+O1paWpg4caK8ZYeqXkF/nc5uHHJORERUFJU7KLdp0wbXr1+vhKpQWXDIORERUclU7rMzadIkzJgxA1FRUWjZsiUMDQ0Vtjdp0qTCKkcli0nKQEh8yr9DzhnsEBERFUXlYOf9998HAEyZMkWeJpFIIISARCJBXl5exdWOSuQXmt+q09ShJkwNOeSciIioKCoHOxEREZVRDyqDl5eIICIioqKpHOw4OTlVRj1IRTl5MlwMfw6A8+sQERGVROVgZ8uWLSVu//jjj8tcGVLe1ceJSM3KhZmhDprU4pBzIiKi4qgc7EydOlXh75ycHKSnp0NHRwcGBgYMdl6Ts/Ih5xYcck5ERFQClYeeJyYmKrxSU1MREhKCjh07YseOHZVRRyoCl4ggIiJSjsrBTlHc3NywZMmSQq0+VDnipJm4H5cCiQTozM7JREREJaqQYAfIn105Jiamog5HJfALzW/VaWJfE2Ycck5ERFQilfvsHDhwQOFvIQRiY2OxatUqdOjQocIqRsUr6K/Tla06REREpVI52Bk0aJDC3xKJBJaWlujevTt++umniqoXFSMnT4YLYVzlnIiISFkqBzsymawy6kFKuvY4ESlZuTA10EYT+5pVXR0iIqJqr8L67NDrcfbfJSI6uVlCk0POiYiISqVysDN06FAsXbq0UPqyZcswbNiwCqkUFc+voL8OH2EREREpReVg59y5c+jbt2+h9D59+uDcuXMVUikqWkJyJu7GJgPgkHMiIiJlqRzspKamQken8HBnbW1tJCcnV0ilqGgFj7Ca2JvAooZuFdeGiIjozaBysNO4cWPs2rWrUPrOnTvRoEGDCqkUFc2PQ86JiIhUpvJorPnz52PIkCF48OABunfvDgA4deoUduzYgd27d1d4BSlfbp4M58Pyg50uXCKCiIhIaSoHO/3798e+ffuwePFi/PXXX9DX10eTJk1w8uRJdOnSpTLqSACCo5KQnJmLmgbaaOZQs6qrQ0RE9MZQOdgBgH79+qFfv34VXRcqQcHCnxxyTkREpBqV++wEBgYiICCgUHpAQACCgoIqpFLqZsmSJZBIJJg2bZpCur+/P7p37w5DQ0MYGxujc+fOyMjIKPIYZ0OeQur/J4589wmMjIxgZWWFQYMGISQkRGG/devWoWvXrjA2NoZEIkFSUlIlnRUREdGbQeVgx8vLC1FRUYXSo6Oj4eXlVSGVUieBgYFYu3YtmjRpopDu7++P3r17w9PTE1euXEFgYCAmT54MDY3CH0lCSibuxCQjM+o2pn4+GZcvX8aJEyeQk5MDT09PpKWlyfdNT09H7969MW/evEo/NyIiojeByo+x7t69ixYtWhRKb968Oe7evVshlVIXqampGDVqFNavX4/vvvtOYdv06dMxZcoUzJ07V55Wr169Io9TMAqrx/Sf8fnETvL0TZs2wcrKClevXkXnzp0BQN56dPbs2Qo8EyIiojeXyi07urq6iI+PL5QeGxsLLa0ydQFSW15eXujXrx969uypkJ6QkICAgABYWVmhffv2sLa2RpcuXXDhwoUij1Mwv07XuoqjsKRSKQDAzMysEmpPRESkHlQOdjw9PeHt7S3/oQWApKQkzJs3D++8806FVu5NtnPnTly7dg0+Pj6Ftj18+BAA8M0332DcuHE4evQoWrRogR49eiAsLExh39xiVjmXyWSYNm0aOnTogEaNGlXimRAREb3ZVG6K+fHHH9G5c2c4OTmhefPmAIDr16/D2toaf/zxR4VX8E0UFRWFqVOn4sSJE9DT0yu0vWDl+AkTJmDMmDEA8h8Dnjp1Chs3blQIkG48SYI0IwfGeloKQ869vLxw+/btYluDiIiIKJ/KwU6tWrVw8+ZNbNu2DTdu3IC+vj7GjBmDDz74ANra2pVRxzfO1atXkZCQoNC3KS8vD+fOncOqVavkI6henXG6fv36iIyMVEg7+29/nU51LaGlmd8QN3nyZBw6dAjnzp2Dvb19ZZ4KERHRG69MnWwMDQ0xfvz4iq6L2ujRowdu3bqlkDZmzBi4u7tjzpw5qF27Nuzs7AoNGw8NDUWfPn0U0s6+tESEEAKff/459u7di7Nnz8LFxaVyT4SIiEgNlLlH8d27dxEZGYns7GyF9AEDBpS7Um86IyOjQv1oDA0NYW5uLk+fNWsWFixYgKZNm6JZs2bYvHkz7t+/j7/++kuep3PXbritXQ/GLfujSz1LeHl5Yfv27di/fz+MjIwQFxcHADAxMYG+vj4AIC4uDnFxcQgPDwcA3Lp1C0ZGRnB0dGRHZiIieiupHOw8fPgQgwcPxq1btyCRSCCEAABIJPmz+ubl5VVsDdXUtGnTkJmZienTp+PFixdo2rQpTpw4gTp16sj3uR8aDlltWzS0M4aVkR5Wr14NAOjatavCsXx9ffHJJ58AANasWYNvv/1Wvq1gSPrL+xAREb1NJKIgWlFS//79oampif/7v/+Di4sLrly5gufPn2PmzJn48ccf0alTp9IPUs0kJyfDxMQEUqkUxsbGVV0duc82B+HkvXiM9nDCtwM54oqIiOhlyv5+qzz03N/fHwsXLoSFhQU0NDSgoaGBjh07wsfHB1OmTClXpek/O65E4uS9/PmMtlx+jF2BkaXkICIioqKoHOzk5eXByMgIAGBhYYGYmBgAgJOTU6EOt1Q2sdIMzNv7XwdnIYB5e24jVlr0ullERERUPJX77DRq1Ag3btyAi4sL2rZti2XLlkFHRwfr1q1D7dq1K6OOb52IZ2l49eFinhB49Cwdtib6VVMpIiKiN5TKwc5XX30lX3hy4cKFePfdd9GpUyeYm5tj165dFV7Bt5Hmv529X01ztjCogtoQERG92VQOdnr16iX/t6urK+7fv48XL17A1NRUPiKLymff9RiFvzUlEiwe0oitOkRERGVQISt3cv6WipOQkom/rz0BAKwe1QI1DXTgbGHAQIeIiKiMuEx5NbPp4iNk58rQwrEmejeyYWsZERFROak8GosqT2pWLv64/BgAMKFLHQY6REREFYDBTjWyIyASKZm5qG1piHfqW1d1dYiIiNSCysHOuXPnkJubWyg9NzcX586dq5BKvY2yc2XYcCECADChc21oaLBVh4iIqCKoHOx069YNL168KJQulUrRrVu3CqnU22j/9WjEJWfCykgXg5rXqurqEBERqQ2Vgx0hRJF9SZ4/fw5DQ8MKqdTbRiYTWHfuIQDg044u0NXSrOIaERERqQ+lR2MNGTIEQP7q5p988gl0dXXl2/Ly8nDz5k20b9++4mv4Fjh9PwFhCakw0tXCyLaOVV0dIiIitaJ0sGNiYgIgv2XHyMgI+vr/zfuio6ODdu3aYdy4cRVfw7fA2nMPAAAj2znCWE+7imtDRESkXpQOdnx9fQEAzs7O+OKLL/jIqoJcffwCgY8SoaOpgU87uFR1dYiIiNSOyn12Zs+erdBn5/Hjx/j5559x/PjxCq3Y22KNX35fncHNa8HaWK+Ka0NERKR+VA52Bg4ciC1btgAAkpKS0KZNG/z0008YOHAgVq9eXeEVVGfhCak4cTceEgkwrjNXjCciIqoMKgc7165dQ6dOnQAAf/31F2xsbPD48WNs2bIFK1eurPAKqrN1//bVeae+NVytalRxbYiIiNSTysFOeno6jIyMAADHjx/HkCFDoKGhgXbt2uHx48cVXkF1FZ+cib3B0QDyl4YgIiKiyqFysOPq6op9+/YhKioKx44dg6enJwAgISEBxsbGFV5BdbXxQgRy8gTaOJuhpZNpVVeHiIhIbakc7Hz99df44osv4OzsjDZt2sDDwwNAfitP8+bNK7yC6ig5MwfbAiIBABO6sK8OERFRZVI52HnvvfcQGRmJoKAgHDt2TJ7eo0cPrFixokIr96olS5ZAIpFg2rRp8rTMzEx4eXnB3NwcNWrUwNChQxEfH1+p9SivbZcjkZqVCzerGuhWz6qqq0NERKTWyrTquY2NDYyMjHDixAlkZGQAAFq3bg13d/cKrdzLAgMDsXbtWjRp0kQhffr06Th48CB2794NPz8/xMTEyGd7ro6ycvOw8eK/C352qcMFP4mIiCqZysHO8+fP0aNHD9StWxd9+/ZFbGwsAGDs2LGYOXNmhVcQAFJTUzFq1CisX78epqb/9W+RSqXYsGEDli9fju7du6Nly5bw9fXFpUuXcPny5UqpS3ntC47G05Qs2JroYUBTu6quDhERkdpTOdiZPn06tLW1ERkZCQMDA3n6iBEjcPTo0QqtXAEvLy/069cPPXv2VEi/evUqcnJyFNLd3d3h6OgIf3//Yo+XlZWF5ORkhdfrIJMJrP13wc+xHV2go1WmhjUiIiJSgdLLRRQ4fvw4jh07Bnt7e4V0Nze3Shl6vnPnTly7dg2BgYGFtsXFxUFHRwc1a9ZUSLe2tkZcXFyxx/Tx8cG3335b0VUt1Yl78Xj4NA3Gelp4vw0X/CQiInodVG5aSEtLU2jRKfDixQuFldArQlRUFKZOnYpt27ZBT6/illLw9vaGVCqVv6Kioirs2MURQmCNX/4kgh95OKGGrspxJhEREZWBysFOp06d5MtFAIBEIoFMJsOyZcvQrVu3Cq3c1atXkZCQgBYtWkBLSwtaWlrw8/PDypUroaWlBWtra2RnZyMpKUkhX3x8PGxsbIo9rq6uLoyNjRVelS3wUSKCI5Ogo6WBT9pzwU8iIqLXReXmhWXLlqFHjx4ICgpCdnY2Zs+ejTt37uDFixe4ePFihVauR48euHXrlkLamDFj4O7ujjlz5sDBwQHa2to4deoUhg4dCgAICQlBZGSkfP6f6qKgVee9lvawNKrYFjAiIiIqnsrBTqNGjRAaGopVq1bByMgIqampGDJkCLy8vGBra1uhlTMyMkKjRo0U0gwNDWFubi5PHzt2LGbMmAEzMzMYGxvj888/h4eHB9q1a1ehdSmPkLgUnL6fkL/gZydOIkhERPQ6qRzsREZGwsHBAV9++WWR2xwdX2/H2xUrVkBDQwNDhw5FVlYWevXqhd9///211qE0a/9d8LN3Qxu4WBhWcW2IiIjeLhIhhFAlg6amJmJjY2FlpTjz7/Pnz2FlZYW8vLwKreDrkJycDBMTE0il0grvvxOTlIHOy84gVyaw36sDmjrUrNDjExERva2U/f1WuYOyEAISSeFZf1NTUyt0xJS62HghArkygXa1zRjoEBERVQGlH2PNmDEDQP7oq/nz5ysMP8/Ly0NAQACaNWtW4RV8k4XGpWDr5fy5hyZ2qVPFtSEiIno7KR3sBAcHA8hv2bl16xZ0dHTk23R0dNC0aVN88cUXFV/DN9SuwEjM/fsWCp4Rxkkzq7Q+REREbyulg50zZ84AyB/6/csvv7yWuWneVLHSDHjv+S/QAYAv995Gl3qWsDXRr7J6ERERvY1U7rPj6+vLQKcUEc/SIHul23eeEHj0LL1qKkRERPQW40qUlcDFwhAar/Th1pRI4GxReJkNIiIiqlwMdiqBrYk+fIY0hua/o9Y0JRIsHtKIj7CIiIiqAFejrCQjWjuic11LPHqWDmcLAwY6REREVYTBTiWyNdFnkENERFTF+BiLiIiI1BqDHSIiIlJrDHaIiIhIrTHYISIiIrXGYIeIiIjUGoMdIiIiUmsMdoiIiEitMdghIiIitcZgh4iIiNQagx0iIiJSawx2iIiISK0x2CEiIiK1xmCHiIiI1BqDHSIiIlJrDHaIiIhIrTHYISIiIrXGYIeIiIjUGoMdIiIiUmsMdoiIiEitMdghIiIitcZgh4iIiNQagx0iIiJSawx2iIiISK0x2CEiIiK1xmCHiIiI1BqDHSIiIlJrDHaIiIhIrTHYISIiIrXGYIeIiIjUGoMdIiIiUmsMdoiIiEitMdghIiIitcZgh4iIiNQagx0iIiJSawx2iIiISK0x2CEiIiK1xmCHiIiI1BqDHSIiIlJrDHaIiIhIrTHYISIiIrXGYIeIiIjUGoMdIiIiUmsMdoiIiEitMdghIiIitcZgh4iIiNQagx0iIiJSawx2iIiISK0x2CEiIiK1xmCHiIiI1BqDHSIiIlJrDHaIiIhIrTHYISIiIrXGYIeIiIjUGoMdIiIiUmvVOtjx8fFB69atYWRkBCsrKwwaNAghISEK+2RmZsLLywvm5uaoUaMGhg4divj4+CqqMREREVU31TrY8fPzg5eXFy5fvowTJ04gJycHnp6eSEtLk+8zffp0HDx4ELt374afnx9iYmIwZMiQKqw1ERERVScSIYSo6koo6+nTp7CysoKfnx86d+4MqVQKS0tLbN++He+99x4A4P79+6hfvz78/f3Rrl07pY6bnJwMExMTSKVSGBsbV+YpEBERUQVR9ve7WrfsvEoqlQIAzMzMAABXr15FTk4OevbsKd/H3d0djo6O8Pf3L/Y4WVlZSE5OVngRERGRenpjgh2ZTIZp06ahQ4cOaNSoEQAgLi4OOjo6qFmzpsK+1tbWiIuLK/ZYPj4+MDExkb8cHBwqs+pERERUhd6YYMfLywu3b9/Gzp07y30sb29vSKVS+SsqKqoCakhERETVkVZVV0AZkydPxqFDh3Du3DnY29vL021sbJCdnY2kpCSF1p34+HjY2NgUezxdXV3o6upWZpWJiIiomqjWLTtCCEyePBl79+7F6dOn4eLiorC9ZcuW0NbWxqlTp+RpISEhiIyMhIeHx+uuLhEREVVD1bplx8vLC9u3b8f+/fthZGQk74djYmICfX19mJiYYOzYsZgxYwbMzMxgbGyMzz//HB4eHkqPxCIiIiL1Vq2HnkskkiLTfX198cknnwDIn1Rw5syZ2LFjB7KystCrVy/8/vvvJT7GehWHnhMREb15lP39rtbBzuvCYIeIiOjNo5bz7BARERGpisEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTW1CbY+e233+Ds7Aw9PT20bdsWV65cqeoqERERUTWgFsHOrl27MGPGDCxYsADXrl1D06ZN0atXLyQkJFR11YiIiKiKqUWws3z5cowbNw5jxoxBgwYNsGbNGhgYGGDjxo1VXTUiIiKqYlpVXYHyys7OxtWrV+Ht7S1P09DQQM+ePeHv719knqysLGRlZcn/lkqlAIDk5OTKrSwRERFVmILfbSFEifu98cHOs2fPkJeXB2tra4V0a2tr3L9/v8g8Pj4++PbbbwulOzg4VEodiYiIqPKkpKTAxMSk2O1vfLBTFt7e3pgxY4b8b5lMhhcvXsDc3BwSiaTCyklOToaDgwOioqJgbGz8WvOz7Ndfdnnzs+y3q+zy5mfZLPtNyV/esksihEBKSgrs7OxK3O+ND3YsLCygqamJ+Ph4hfT4+HjY2NgUmUdXVxe6uroKaTVr1qysKsLY2LhcH3B58rPs1192efOz7Ler7PLmZ9ks+03JX96yi1NSi06BN76Dso6ODlq2bIlTp07J02QyGU6dOgUPD48qrBkRERFVB298yw4AzJgxA6NHj0arVq3Qpk0b/Pzzz0hLS8OYMWOqumpERERUxdQi2BkxYgSePn2Kr7/+GnFxcWjWrBmOHj1aqNPy66arq4sFCxYUemT2OvKz7Ndfdnnzs+y3q+zy5mfZLPtNyV/esiuCRJQ2XouIiIjoDfbG99khIiIiKgmDHSIiIlJrDHaIiIhIrTHYISIiIrXGYKcS/fbbb3B2doaenh7atm2LK1euKJXv3Llz6N+/P+zs7CCRSLBv3z6ly/Tx8UHr1q1hZGQEKysrDBo0CCEhIUrnX716NZo0aSKf/MnDwwNHjhxROv/LlixZAolEgmnTpim1/zfffAOJRKLwcnd3V7q86OhofPjhhzA3N4e+vj4aN26MoKAgpfI6OzsXKlsikcDLy6vUvHl5eZg/fz5cXFygr6+POnXqYNGiRaWu1fKylJQUTJs2DU5OTtDX10f79u0RGBhYaL/Srg0hBL7++mvY2tpCX18fPXv2RFhYmNL59+zZA09PT/ls4tevX1e6/JycHMyZMweNGzeGoaEh7Ozs8PHHHyMmJkapsr/55hu4u7vD0NAQpqam6NmzJwICApSu+8smTpwIiUSCn3/+Wam8n3zySaHPvnfv3iqVfe/ePQwYMAAmJiYwNDRE69atERkZWWreoq47iUSCH374QamyU1NTMXnyZNjb20NfX1++GLIyeePj4/HJJ5/Azs4OBgYG6N27t/x6Uea7JDMzE15eXjA3N0eNGjUwdOhQ+QSvyuRft24dunbtCmNjY0gkEiQlJcm3lZb/xYsX+Pzzz1GvXj3o6+vD0dERU6ZMgVQqVarsCRMmoE6dOtDX14elpSUGDhwoX2JIle9RIQT69Okjf3+Vydu1a9dCn/fEiRNVKtvf3x/du3eHoaEhjI2N0blzZyxcuLDEvI8ePSr2etu9e7dSZcfFxeGjjz6CjY0NDA0N0aJFC/z9999K5X3w4AEGDx4MS0tLGBsbY/jw4YUmBK4sDHYqya5duzBjxgwsWLAA165dQ9OmTdGrVy8kJCSUmjctLQ1NmzbFb7/9pnK5fn5+8PLywuXLl3HixAnk5OTA09MTaWlpSuW3t7fHkiVLcPXqVQQFBaF79+4YOHAg7ty5o1I9AgMDsXbtWjRp0kSlfA0bNkRsbKz8deHCBaXyJSYmokOHDtDW1saRI0dw9+5d/PTTTzA1NVW6vi+Xe+LECQDAsGHDSs27dOlSrF69GqtWrcK9e/ewdOlSLFu2DL/++qtSZQPAZ599hhMnTuCPP/7ArVu34OnpiZ49eyI6Olphv9KujWXLlmHlypVYs2YNAgICYGhoiF69eiEzM1Op/GlpaejYsSOWLl1a7Pbi8qenp+PatWuYP38+rl27hj179iAkJAQDBgxQquy6deti1apVuHXrFi5cuABnZ2d4enri6dOnSuUvsHfvXly+fFlh+nhl8vbu3VvhGtixY4fS+R88eICOHTvC3d0dZ8+exc2bNzF//nzo6emVmvflMmNjY7Fx40ZIJBIMHTpUqbJnzJiBo0ePYuvWrbh37x6mTZuGyZMn48CBAyXmFUJg0KBBePjwIfbv34/g4GA4OTmhZ8+eSEtLU+q7ZPr06Th48CB2794NPz8/xMTEYMiQIQCU+y5KT09H7969MW/evEL1Ky1/TEwMYmJi8OOPP+L27dvYtGkTjh49irFjxypVdsuWLeHr64t79+7h2LFjEELA09MTeXl5Kn2P/vzzzwrLDCmbd9y4cQqf+7Jly5TO7+/vj969e8PT0xNXrlxBYGAgJk+ejAsXLpSY18HBodD19u2336JGjRro06ePUmV//PHHCAkJwYEDB3Dr1i0MGTIEw4cPx8GDB0vMm5aWBk9PT0gkEpw+fRoXL15EdnY2+vfvD5lMVuh9rXCCKkWbNm2El5eX/O+8vDxhZ2cnfHx8VDoOALF3794y1yMhIUEAEH5+fmU+hqmpqfi///s/pfdPSUkRbm5u4sSJE6JLly5i6tSpSuVbsGCBaNq0aZnqOGfOHNGxY8cy5S3K1KlTRZ06dYRMJit13379+olPP/1UIW3IkCFi1KhRSpWVnp4uNDU1xaFDhxTSW7RoIb788sti8716bchkMmFjYyN++OEHeVpSUpLQ1dUVO3bsKDX/yyIiIgQAERwcrHT5Rbly5YoAIB4/fqxyXqlUKgCIkydPKl32kydPRK1atcTt27eFk5OTWLFihVJ5R48eLQYOHFhifUrKP2LECPHhhx+WKe+rBg4cKLp37650/oYNG4qFCxcqpBV17byaNyQkRAAQt2/flqfl5eUJS0tLsX79+kJlv/pdkpSUJLS1tcXu3bvl+9y7d08AEP7+/qXmf9mZM2cEAJGYmFjkeZeWv8Cff/4pdHR0RE5Ojsp5b9y4IQCI8PBwpcsODg4WtWrVErGxscV+tkXlVeV7saj8bdu2FV999VWZ8r6qWbNmhb6/SspvaGgotmzZorCfmZlZoWvm1bzHjh0TGhoaQiqVyvdJSkoSEolEnDhxotRzKS+27FSC7OxsXL16FT179pSnaWhooGfPnvD393+tdZFKpQAAMzMzlfPm5eVh586dSEtLU2npDS8vL/Tr10/h/JUVFhYGOzs71K5dG6NGjUJkZKRS+Q4cOIBWrVph2LBhsLKyQvPmzbF+/XqVywfyP7+tW7fi008/VWph2Pbt2+PUqVMIDQ0FANy4cQMXLlxAnz59lCovNzcXeXl50NPTU0jX19dXumULACIiIhAXF6fwvpuYmKBt27av/borIJVKIZFIVF57Ljs7G+vWrYOJiQmaNm2qVB6ZTIaPPvoIs2bNQsOGDVWu69mzZ2FlZYV69erhf//7H54/f650uYcPH0bdunXRq1cvWFlZoW3btio9fi4QHx+Pw4cPY+zYsUrnad++PQ4cOIDo6GgIIXDmzBmEhobC09OzxHxZWVkAoHDdaWhoQFdXt8jr7tXvkqtXryInJ0fhenN3d4ejo2OR11t5vouUzS+VSmFsbAwtLa1C6SXlTUtLg6+vL1xcXODg4KBU2enp6Rg5ciR+++23YtdhLKnsbdu2wcLCAo0aNYK3tzfS09OVyp+QkICAgABYWVmhffv2sLa2RpcuXZT6zF519epVXL9+vdjrraj87du3x65du/DixQvIZDLs3LkTmZmZ6Nq1a4l5s7KyIJFIFCYW1NPTg4aGhkrfc2VW6eHUWyg6OloAEJcuXVJInzVrlmjTpo1Kx0I5Wnby8vJEv379RIcOHVTKd/PmTWFoaCg0NTWFiYmJOHz4sNJ5d+zYIRo1aiQyMjKEEKrdwfzzzz/izz//FDdu3BBHjx4VHh4ewtHRUSQnJ5eaV1dXV+jq6gpvb29x7do1sXbtWqGnpyc2bdqkdN0L7Nq1S2hqaoro6Gil9s/LyxNz5swREolEaGlpCYlEIhYvXqxSmR4eHqJLly4iOjpa5Obmij/++ENoaGiIunXrFpvn1Wvj4sWLAoCIiYlR2G/YsGFi+PDhpeZ/WUW07GRkZIgWLVqIkSNHKp334MGDwtDQUEgkEmFnZyeuXLmidNmLFy8W77zzjrw1TpWWnR07doj9+/eLmzdvir1794r69euL1q1bi9zc3FLzF9zVGxgYiOXLl4vg4GDh4+MjJBKJOHv2rFLnXWDp0qXC1NRU/v9HmbpnZmaKjz/+WAAQWlpaQkdHR2zevLnUvNnZ2cLR0VEMGzZMvHjxQmRlZYklS5YIAMLT01Mhb1HfJdu2bRM6OjqFymndurWYPXt2qflfVlrLjjLfZU+fPhWOjo5i3rx5Suf97bffhKGhoQAg6tWrV2SrTnH5x48fL8aOHSv/u6jPpri8a9euFUePHhU3b94UW7duFbVq1RKDBw9Wqmx/f38BQJiZmYmNGzeKa9euiWnTpgkdHR0RGhqq1HkX+N///ifq169f5Lbi8icmJgpPT0/59WZsbCyOHTtWat6EhARhbGwspk6dKtLS0kRqaqqYPHmyACDGjx9fbB0rCoOdSlBdgp2JEycKJycnERUVpVK+rKwsERYWJoKCgsTcuXOFhYWFuHPnTqn5IiMjhZWVlbhx44Y8TZVg51WJiYnC2NhYqUdo2trawsPDQyHt888/F+3atVO5XE9PT/Huu+8qvf+OHTuEvb292LFjh7h586bYsmWLMDMzUynQCg8PF507dxYAhKampmjdurUYNWqUcHd3LzZPdQ52srOzRf/+/UXz5s0Vmq1Ly5uamirCwsKEv7+/+PTTT4Wzs7OIj48vNX9QUJCwtrZWCFBVCXZe9eDBA6UfoRX8f//ggw8U9uvfv794//33VSq7Xr16YvLkycVuLyr/Dz/8IOrWrSsOHDggbty4IX799VdRo0aNQo8GisobFBQkmjZtKr/uevXqJfr06SN69+6tsF9R3yWqBDulfReVFuyUll8qlYo2bdqI3r17i+zsbKXzJiUlidDQUOHn5yf69+8vWrRoUSjQLCr//v37haurq0hJSZGnFfX+KvsdfOrUqSIfoRWVv+D/ube3t8K+jRs3FnPnzlW67PT0dGFiYiJ+/PHHIrcXl3/y5MmiTZs24uTJk+L69evim2++ESYmJuLmzZul5j127JioXbu2kEgkQlNTU3z44YeiRYsWYuLEiSW8OxWDwU4lyMrKEpqamoUu/I8//lgMGDBApWOVNdjx8vIS9vb24uHDhyrnfVWPHj2Uirz37t0r/9IseAGQX9hF3SWXplWrVgr/gYvj6OiocJclhBC///67sLOzU6m8R48eCQ0NDbFv3z6l89jb24tVq1YppC1atEjUq1dPpbKFyP+xLwhWhg8fLvr27Vvsvq9eGwU/0K8GKJ07dxZTpkwpNf/LyhPsZGdni0GDBokmTZqIZ8+eqZT3Va6urkW2kr2af8WKFfLr7OVrT0NDQzg5OZWpbAsLC7FmzZpSy87KyhJaWlpi0aJFCvvNnj1btG/fXumyz507JwCI69evF1unV/Onp6cLbW3tQv29xo4dK3r16qV02UlJSSIhIUEIkd/fcNKkSfJtxX2XFPxAvxqgODo6iuXLl5ea/2UlBTul5U9OThYeHh6iR48ehQIVVb4Hs7KyhIGBgdi+fXup+adOnVrs9dalSxeVy05NTRUAxNGjR0st++HDhwKA+OOPPxTShw8fLm9FVabsLVu2CG1tbfnn/rLi8oeHhxfq5yVE/m/EhAkTlC776dOn8s/a2tpaLFu2rNh9Kwr77FQCHR0dtGzZEqdOnZKnyWQynDp1SqW+L2UhhMDkyZOxd+9enD59Gi4uLuU+pkwmkz/fL0mPHj1w69YtXL9+Xf5q1aoVRo0ahevXr0NTU1OlclNTU/HgwQPY2tqWum+HDh0KDXMMDQ2Fk5OTSmX6+vrCysoK/fr1UzpPeno6NDQU/ytpamqWaYSBoaEhbG1tkZiYiGPHjmHgwIFK53VxcYGNjY3CdZecnIyAgIBKv+4K5OTkYPjw4QgLC8PJkydhbm5eruMpe+199NFHuHnzpsK1Z2dnh1mzZuHYsWMql/vkyRM8f/5cqWtPR0cHrVu3Lvf1t2HDBrRs2VLpPkpA/vudk5NT7uvPxMQElpaWCAsLQ1BQEAYOHFjqd0nLli2hra2tcL2FhIQgMjISHh4e5f4uUiZ/cnIyPD09oaOjgwMHDsj7H5WlbJF/84+srKxS88+dO7fQ9QYAK1aswMaNG1UuuyC/ra1tqWU7OzvDzs6uyOvN0dFR6bI3bNiAAQMGwNLSUuE9KCl/Qb+ioq63vLw8pcu2sLBAzZo1cfr0aSQkJMhHbFaqSg+n3lI7d+4Uurq6YtOmTeLu3bti/PjxombNmiIuLq7UvCkpKSI4OFgEBwcLAPJ+AK+OaCnK//73P2FiYiLOnj0rYmNj5a/09HSl6j137lzh5+cnIiIixM2bN8XcuXOFRCIRx48fVyr/q1R5jDVz5kxx9uxZERERIS5evCh69uwpLCwsirzzeNWVK1eElpaW+P7770VYWJjYtm2bMDAwEFu3blW6rnl5ecLR0VHMmTNH6TxC5I/kqVWrljh06JCIiIgQe/bsERYWFoWa8kty9OhRceTIEfHw4UNx/Phx0bRpU9G2bdtCTfKlXRtLliwRNWvWlPc/GThwoHBxcZHf8ZaW//nz5yI4OFgcPnxYABA7d+4UwcHBIjY2ttT82dnZYsCAAcLe3l5cv35d4frLysoqMW9qaqrw9vYW/v7+4tGjRyIoKEiMGTNG6Orqyu8iVf1/8fJjrJLypqSkiC+++EL4+/uLiIgIcfLkSdGiRQvh5uYmMjMzlSp7z549QltbW6xbt06EhYWJX3/9VWhqaorz588rVW+pVCoMDAzE6tWrC51Hafm7dOkiGjZsKM6cOSMePnwofH19hZ6envj9999Lzfvnn3+KM2fOiAcPHoh9+/YJJycnMWTIECGEct8lEydOFI6OjuL06dMiKChIeHh4yB8nK5M/NjZWBAcHi/Xr1wsA4ty5cyI4OFg8f/681PxSqVS0bdtWNG7cWISHhyvsM3HixBLzPnjwQCxevFgEBQWJx48fi4sXL4r+/fsLMzMzER8fX6bvUfzbclZa3vDwcLFw4UIRFBQkIiIixP79+0Xt2rVF586dlX7fVqxYIYyNjcXu3btFWFiY+Oqrr4Senp4YOXKkUvUOCwsTEolEHDlyRCG9tLKzs7OFq6ur6NSpkwgICBDh4eHixx9/FBKJRPTt27fUsjdu3Cj8/f1FeHi4+OOPP4SZmZmYMWNGse9pRWKwU4l+/fVX4ejoKHR0dESbNm3E5cuXlcpX0KT76mv06NGl5i0qHwDh6+urVNmffvqpcHJyEjo6OsLS0lL06NGjzIGOEKoFOyNGjBC2trZCR0dH1KpVS4wYMaLIDoPFOXjwoGjUqJHQ1dUV7u7uYt26dSrV9dixYwKACAkJUSlfcnKymDp1qnB0dBR6enqidu3a4ssvvxRZWVlKH2PXrl2idu3aQkdHR9jY2AgvLy+RlJRUaL/Srg2ZTCbmz58vrK2tha6urujRo4fC+ZSW39fXt8jtCxYsKDV/waOvol5nzpwpMW9GRoYYPHiwsLOzEzo6OsLW1lYMGDBAoYOyqv8vXg52Ssqbnp4uPD09haWlpdDW1hZOTk5i3LhxCjcmypS9YcMG4erqKvT09ETTpk3lj0KVybt27Vqhr69fps88NjZWfPLJJ8LOzk7o6emJevXqiZ9++knIZLJS8/7yyy/C3t5eaGtrC0dHR/HVV1/Jr1tlvksyMjLEpEmThKmpqTAwMBCDBw+WB8bK5F+wYEGx+5SWv7hzK+lVkDc6Olr06dNHWFlZCW1tbWFvby9Gjhwp7t+/r3TdX1UQ7JSWNzIyUnTu3FmYmZkJXV1d4erqKmbNmiXv26Zs2T4+PsLe3l4YGBgIDw8Pcf78eaXzent7CwcHB5GXl1foHErLHxoaKoYMGSKsrKyEgYGBaNKkidiyZYtSeefMmSOsra2Ftra2cHNzk1+nr4Pk3xMkIiIiUkvss0NERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BARveLs2bOQSCRISkqq6qoQUQVgsENERERqjcEOERERqTUGO0RU7chkMvj4+MDFxQX6+vpo2rQp/vrrLwD/PWI6fPgwmjRpAj09PbRr1w63b99WOMbff/+Nhg0bQldXF87Ozvjpp58UtmdlZWHOnDlwcHCArq4uXF1dsWHDBoV9rl69ilatWsHAwADt27cvtNI0Eb0ZGOwQUbXj4+ODLVu2YM2aNbhz5w6mT5+ODz/8EH5+fvJ9Zs2ahZ9++gmBgYGwtLRE//79kZOTAyA/SBk+fDjef/993Lp1C9988w3mz5+PTZs2yfN//PHH2LFjB1auXIl79+5h7dq1qFGjhkI9vvzyS/z0008ICgqClpYWPv3009dy/kRUsbgQKBFVK1lZWTAzM8PJkyfh4eEhT//ss8+Qnp6O8ePHo1u3bti5cydGjBgBAHjx4gXs7e2xadMmDB8+HKNGjcLTp09x/Phxef7Zs2fj8OHDuHPnDkJDQ1GvXj2cOHECPXv2LFSHs2fPolu3bjh58iR69OgBAPjnn3/Qr18/ZGRkQE9Pr5LfBSKqSGzZIaJqJTw8HOnp6XjnnXdQo0YN+WvLli148OCBfL+XAyEzMzPUq1cP9+7dAwDcu3cPHTp0UDhuhw4dEBYWhry8PFy/fh2ampro0qVLiXVp0qSJ/N+2trYAgISEhHKfIxG9XlpVXQEiopelpqYCAA4fPoxatWopbNPV1VUIeMpKX19fqf20tbXl/5ZIJADy+xMR0ZuFLTtEVK00aNAAurq6iIyMhKurq8LLwcFBvt/ly5fl/05MTERoaCjq168PAKhfvz4uXryocNyLFy+ibt260NTUROPGjSGTyRT6ABGR+mLLDhFVK0ZGRvjiiy8wffp0yGQydOzYEVKpFBcvXoSxsTGcnJwAAAsXLoS5uTmsra3x5ZdfwsLCAoMGDQIAzJw5E61bt8aiRYswYsQI+Pv7Y9WqVfj9998BAM7Ozhg9ejQ+/fRTrFy5Ek2bNsXjx4+RkJCA4cOHV9WpE1ElYbBDRNXOokWLYGlpCR8fHzx8+BA1a9ZEixYtMG/ePPljpCVLlmDq1KkICwtDs2bNcPDgQejo6AAAWrRogT///BNff/01Fi1aBFtbWyxcuBCffPKJvIzVq1dj3rx5mDRpEp4/fw5HR0fMmzevKk6XiCoZR2MR0RulYKRUYmIiatasWdXVIaI3APvsEBERkVpjsENERERqjY+xiIiISK2xZYeIiIjUGoMdIiIiUmsMdoiIiEitMdghIiIitcZgh4iIiNQagx0iIiJSawx2iIiISK0x2CEiIiK1xmCHiIiI1Nr/A45Ofu2hDDlOAAAAAElFTkSuQmCC", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAG2CAYAAACZEEfAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAAB7hklEQVR4nO3deXhM1/8H8PdkskeE7DNJJEESBLEWsRdRVUupXe2qpSXUvrSqJGiptoryU2uLllhqj5bYiSVEBCEhEYkIyWTfz++PfDM1ss2QiIz363nmaXPvPcuNY+7HuWeRCCEEiIiIiLSUTkVXgIiIiKg8MdghIiIircZgh4iIiLQagx0iIiLSagx2iIiISKsx2CEiIiKtxmCHiIiItBqDHSIiItJqDHaIiIhIqzHYISIiIq1WocHOyZMn0aNHD8jlckgkEuzZs0flvBAC8+fPh1wuh5GRETp06ICQkBCVazIzM/HFF1/A0tISJiYm6NmzJx4+fPga74KIiIjeZBUa7KSmpsLDwwMrV64s8vzSpUuxfPlyrFy5EoGBgbC1tUWXLl2QnJysvMbb2xu7d+/G9u3bcfr0aaSkpOCDDz5Abm7u67oNIiIieoNJ3pSNQCUSCXbv3o3evXsDyO/Vkcvl8Pb2xowZMwDk9+LY2NhgyZIlGDduHBQKBaysrLBlyxYMGDAAAPDo0SM4ODjg4MGD6Nq1a0XdDhEREb0hdCu6AsWJiIhAbGwsvLy8lMcMDAzQvn17nD17FuPGjcPly5eRnZ2tco1cLkf9+vVx9uzZYoOdzMxMZGZmKn/Oy8vDs2fPYGFhAYlEUn43RURERGVGCIHk5GTI5XLo6BT/suqNDXZiY2MBADY2NirHbWxs8ODBA+U1+vr6qF69eqFrCtIXxdfXF998800Z15iIiIgqQlRUFOzt7Ys9/8YGOwVe7GkRQpTa+1LaNbNmzcKUKVOUPysUCtSoUQNRUVGoWrXqq1WYiIiIXoukpCQ4ODjA1NS0xOve2GDH1tYWQH7vjUwmUx6Pi4tT9vbY2toiKysLCQkJKr07cXFx8PT0LDZvAwMDGBgYFDpetWpVBjtERESVTGmdIG/sOjvOzs6wtbWFv7+/8lhWVhYCAgKUgUzTpk2hp6enck1MTAxu3LhRYrBDREREb48K7dlJSUnB3bt3lT9HREQgKCgI5ubmqFGjBry9veHj4wMXFxe4uLjAx8cHxsbGGDx4MADAzMwMo0ePxpdffgkLCwuYm5tj6tSpaNCgATp37lxRt0VERERvkAoNdi5duoSOHTsqfy4YRzN8+HBs3LgR06dPR3p6OsaPH4+EhAS0aNECR48eVXk398MPP0BXVxf9+/dHeno6OnXqhI0bN0Iqlb72+yEiIqI3zxuzzk5FSkpKgpmZGRQKBcfsEBERVRLqPr/f2DE7RERERGWBwQ4RERFpNQY7REREpNUY7BAREZFWY7BDREREWo3BDhEREWk1BjtERESk1RjsEBERkVZjsENERERajcEOERERaTUGO0RERKTVGOwQERGRVmOwQ0RERFqNwQ4RERFpNQY7REREpNUY7BAREZFWY7BDREREWo3BDhEREWk1BjtERESk1RjsEBERkVZjsENERERajcEOERERaTUGO0RERKTVGOwQERGRVmOwQ0RERFqNwQ4RERFpNQY7REREpNUY7BAREZFWY7BDREREWo3BDhEREWk1BjtERESk1RjsEBERkVZjsENERERajcEOERERaTUGO0RERKTVGOwQERGRVmOwQ0RERFqNwQ4RERFpNQY7REREpNUY7BAREZFWY7BDREREWo3BDhEREWk1BjtERESk1RjsEBERkVZjsENERERajcEOERERaTUGO0RERKTVGOwQERGRVmOwQ0RERFqNwQ4RERFpNQY7REREpNUY7BAREZFWY7BDREREWo3BDhERERWSk5ODuXPnwtnZGUZGRqhZsyYWLFiAvLw85TVCCMyfPx9yuRxGRkbo0KEDQkJCSszXz88PzZo1Q7Vq1WBiYoJGjRphy5Ytha5btWoVnJ2dYWhoiKZNm+LUqVMvfS8MdoiIiKiQJUuWYM2aNVi5ciVCQ0OxdOlSfPfdd/j555+V1yxduhTLly/HypUrERgYCFtbW3Tp0gXJycnF5mtubo45c+bg3LlzuH79OkaOHImRI0fiyJEjymt27NgBb29vzJkzB1evXkXbtm3RrVs3REZGvtS9SIQQ4qVSapGkpCSYmZlBoVCgatWqFV0dIiKiCvfBBx/AxsYG69evVx7r27cvjI2NsWXLFgghIJfL4e3tjRkzZgAAMjMzYWNjgyVLlmDcuHFql9WkSRN0794d3377LQCgRYsWaNKkCVavXq28pm7duujduzd8fX2Vx9R9frNnh4iIiApp06YN/vnnH9y5cwcAcO3aNZw+fRrvv/8+ACAiIgKxsbHw8vJSpjEwMED79u1x9uxZtcoQQuCff/7B7du30a5dOwBAVlYWLl++rJIvAHh5eamd74t0XyoVERERabUZM2ZAoVCgTp06kEqlyM3NxaJFizBo0CAAQGxsLADAxsZGJZ2NjQ0ePHhQYt4KhQJ2dnbIzMyEVCrFqlWr0KVLFwBAfHw8cnNzi8y3oExNsWeHiIioBE5OTpBIJIU+EyZMAAA8fvwYI0aMgFwuh7GxMd577z2EhYWVmGdISAj69u2rzHvFihUal1veduzYga1bt+KPP/7AlStXsGnTJnz//ffYtGmTynUSiUTlZyFEoWMvMjU1RVBQEAIDA7Fo0SJMmTIFJ06ceOV8i8Ngh4iIqASBgYGIiYlRfvz9/QEA/fr1gxACvXv3Rnh4OPbu3YurV6/C0dERnTt3RmpqarF5pqWloWbNmli8eDFsbW2LLdfe3r7Q8VWrVikDnpSUFHz++eewt7eHkZER6tatqzLOpSgdOnQoMojq3r278prk5GSMHTsWqampGDlyJMaNG4c6depg8uTJyjEzBfV+sbclLi6uUK/Mi3R0dFC7dm00atQIX375JT766CNlvpaWlpBKpS+Vb7HlvVQqIiJ6a8Uo0nH2XjxiFOkVXZXXwsrKCra2tsrP/v37UatWLbRv3x5hYWE4f/48Vq9ejebNm8PNzQ2rVq1CSkoKtm3bVmyezZs3x3fffYeBAwfCwMCg2HKvXLmiDLLGjBkDmUwGID/QAoDJkyfj8OHD2Lp1K0JDQzF58mR88cUX2Lt3b7Fl+/n5qQRvN27cgFQqVeYJAGPGjEF6ejpGjBiB4OBgeHl5KQO4gqnnzs7OsLW1VQZ/QP54m4CAAHh6eqr/C0Z+r01mZiYAQF9fH02bNlXJFwD8/f01zvf5At56CoVCABAKhaKiq0JE9EbbfvGBcJ65XzjO2C+cZ+4X2y8+qOgqvVaZmZnCwsJCLFq0SAghxPXr1wUAcffuXZXrbG1txfDhw9XK09HRUfzwww9qlevp6Slq1aol8vLyhBBCuLu7iwULFqhc26RJEzF37lz1bkgI8cMPPwhTU1ORkpIihBAiLS1NSKVS0alTJ2FnZyf2798vIiIihJOTkzA2NhbTp09Xpl28eLEwMzMTfn5+Ijg4WAwaNEjIZDKRlJSkvObjjz8WM2fOVP7s4+Mjjh49Ku7duydCQ0PFsmXLhK6urli3bp3ymu3btws9PT2xfv16cfPmTeHt7S1MTEzE/fv3Vequ7vObA5SJiEgtMYp0zPILRt7/FizJE8Bsvxto52oFmZlRxVbuNdmzZw8SExMxYsQIAECdOnXg6OiIWbNm4ddff4WJiQmWL1+O2NhYxMTElGm5CQkJCA0NxdSpU5VjV9q0aYN9+/Zh1KhRkMvlOHHiBO7cuYMff/xR7bzXr1+PgQMHwsTEBED+YoK5ubmYNGkS/vnnH4wfPx5xcXEQQsDGxkY5PRwApk+fjvT0dIwfPx4JCQlo0aIFjh49ClNTU+U1kZGR0NH570VSamoqxo8fj4cPH8LIyAh16tTB1q1bMWDAAOU1AwYMwNOnT7FgwQLExMSgfv36OHjwIBwdHV/uF6h26FcBsrOzxZw5c4STk5MwNDQUzs7O4ptvvhG5ubnKa/Ly8sTXX38tZDKZMDQ0FO3btxc3btzQqBz27BARle7M3SfCccb+Qp+zd+PLvWxHR0cBoNBn/PjxQghR5DkAYunSpSXmm5CQIMaPHy9sbW2FgYGBqFOnjjhw4IDy/Ndff10oT319fZU8Ll26JDw8PAQAIZVKRdeuXUW3bt1Et27d1L630np2vLy8RJMmTYRUKhXR0dHK45mZmWLYsGECgNDV1RX6+vpi8+bNapUrhBAXLlwQAMSFCxdUjrdq1Uq0b99eREdHi5ycHLFlyxYhkUiEq6ur2nkXeJSYJs7cfSIeJaZpnLY0WtGzU7B646ZNm+Du7o5Lly5h5MiRMDMzw6RJkwD8t3rjxo0b4erqioULF6JLly64ffu2SmRJRESvxtnSBBLkP/Gfdz8+Fa1qWZRr2YGBgcjNzVX+fOPGDXTp0kU5zuTFXpRDhw5h9OjR6Nu3b7F5ZmVloUuXLrC2tsbOnTthb2+PqKioQs8Od3d3HDt2DFFRUWjZsiXWrVuncr5p06YICgqCQqFAVlYWrKys0KJFCzRr1uxVbxsA8ODBAxw7dgwNGzZEt27dIJfLled++uknnD9/Hvv27YOjoyNOnjyJ8ePHQyaToXPnzqXmvX79etSvXx/vvPOOyvEtW7Zg1KhRsLOzg1QqRZMmTTB48GBcuXJFo7rvCIxU9gbqSADfPg0woHkNjfIoC290sHPu3Dn06tVLOULcyckJ27Ztw6VLlwDkD2hasWIF5syZgz59+gAANm3aBBsbG/zxxx8ard5IREQls6xiADMjPSSmZwOAMvCZt/cGqhnroVsDWbmVbWVlpfLz4sWLlYOEARSa0bR371507NgRNWvWLDbP3377Dc+ePcPZs2ehp6cHAEW+JtHV1YWtrS3WrFkDa2trDB48uMj8zMzMAABhYWG4dOmSyuueV7FhwwZYWFjg+vXrmD9/vvJ4eno6Zs+ejd27dyufkw0bNkRQUBC+//77UoOdtLQ0bN++HQsWLCh0rlatWggICEBqaiqSkpIgk8kwYMAAODs7q13vN+m15xs9G6u8Vm/MzMxEUlKSyoeIiEpm5+CIa/O74sGSD/BgyQe4/7//Pj68Cp9vu4oD12MQGhqKnj17wszMDKampmjZsmWp+xmtWLECbm5uMDIygoODAyZPnoyMjAzl+eTkZHh7e8PR0RFGRkZo2bIlNm7ciFGjRhW57srjx49x4MABjB49usRy9+3bh1atWmHChAmwsbFB/fr14ePjo9KDBOQHLzKZDAsXLkS1atUK3c9ff/2FEydOKKefd+nSBb1791Z5Ng0bNgyzZs1S/pyVlYWgoCAEBQUhKysL0dHRCAoKwt27d1XyzsvLw4YNG+Dq6gpra2uV6eHZ2dnIzs5WGQ8DAFKpVGWzzuL8+eefyMzMxNChQ4u9xsTEBDKZDAkJCThy5Ah69epVar4FIp6kKgOdArlC4H58mtp5lJkyf4FWhvLy8sTMmTOFRCIRurq6QiKRCB8fH+X5M2fOCAAq7y+FEGLs2LHCy8ur2HyLegcLjtkhIipWTm6eaP31bmE/YYtYvOuciImJEf7+/gKA+GjeOuE4Y79w+PT/hKlZNTFt2jRx5coVce/ePbF//37x+PHjYvPdunWrMDAwEL///ruIiIgQR44cETKZTHh7eyuv6d+/v6hXr54ICAgQYWFh4qOPPhIAxKVLl4rMc8mSJaJ69eoiPT29xHtyc3MTBgYGYtSoUeLSpUti27ZtwtzcXHzzzTfKaw4ePCh27twp1qxZIwCI5s2bCxsbGxEf/984pR9//FHY29sLPT09UaNGDTF37lyRmZmpUlb79u1F/0FDlWNXIiIiinwOtW/fXiXdkSNHBAAhl8vFjBkzCt1D+/bthbu7uzh+/LgIDw8XGzZsEIaGhmLVqlXKa16cDVWgTZs2YsCAAUX+bg4fPiwOHTokwsPDxdGjR4WHh4d45513RFZWVom/0+fN3R1caHxXzZkHynTsjrpjdt7oYGfbtm3C3t5ebNu2TVy/fl1s3rxZmJubi40bNwoh/gt2Hj16pJJuzJgxomvXrsXmm5GRIRQKhfITFRXFYIeIqAR/X4sWjjP2i4bzj4jkjGwhhBCTJk0StWrVEtk5uWLKjiBhXKetMHHvKPZcfah2vhMmTBDvvvuuyrEpU6aINm3aCCH+mwa9f/9+5XkvLy9RtWpVMWfOnCLzdHNzE59//nmpZbu4uAgHBweRk5OjPLZs2TJha2tbbJqUlBRhY2Mjli1bVmr+z3uVKfsFAc/t27cLnYuJiREjRowQcrlcGBoaCjc3N7Fs2TLl1HQh8gOiF6fB3759WwAQR48eLbLMHTt2iJo1awp9fX1ha2srJkyYIBITE9Wu868Bd5UBjtPM/wKdsl6qQCsGKE+bNg0zZ87EwIEDAQANGjTAgwcP4Ovri+HDh6us3liw0BJQ+iqLBgYGxS7iREREqvLyBFb+m/96ZWRrJ1Qx0EVWVha2bt2KKVOmQFeqg8V96uPn4Zdh3OxDDOrTA/qJkXBzqYVZs2ahd+/exebdpk0bbN26FRcvXsQ777yD8PBwHDx4EMOHDwfw3zRoQ0NDAP8N1q1duzZOnz5dKL9Tp07h9u3b2LFjR6n3JZPJoKenB6lUqjxWt25dxMbGIisrC/r6+oXSmJiYoEGDBqVuB1EgKycPx27GYuauYOXAbk3Hrnh5eUGIF4eF57O1tcWGDRtKTH/ixAnlQpDOliaQmRnB1dW12DwBoH///ujfv3+pdSvKn5ei4HPwFgBgxnt10LuxHPfj0+BkaVxhSxS80cFOWlpaie8in1+9sXHjxgD+W71xyZIlr72+RETa6J9bcbgVm4wqBroY4ekEoPB6M0/jnyA7Iw1pgbtQpfVQGDmOhGuVR+jTpw+OHz+uHEj8ooEDB+LJkydo06YNhBDIycnBZ599hpkzZwLI30OpVatW+Pbbb1G3bl2sX78epqamyrGcL1q/fj2aNm0KDw+PUu+rdevW+OOPP5CXl6d81ty5cwcymazIQAfIH/MZGhqKtm3bFnleCIF7T1JwKiwep8PicT78KVKzcgtdVzB25XU8/F/njKjDN2Ixc9d1AMC4djXxWYdaAFDh6zC90cFOjx49sGjRItSoUQPu7u64evUqli9fjlGjRgHI3yTM29sbPj4+cHFxgYuLC3x8fGBsbFzsaHkiIlKfEAIrj+f36gxt6YhqxvlBwPr161WmQRf8I7Tvhx/Cqd8X2HYxCqclNdGkzSWsWbOm2GDnxIkTWLRoEVatWoUWLVrg7t27mDRpEmQyGebNmwdAdRo0kN+b8cEHHxSaBp2UlIS//voLy5YtK7KsYcOGwc7OTrkH02effYaff/4ZkyZNwhdffIGwsDD4+Phg4sSJyjRTp05Fjx49UKNGDcTFxWHu198gIVGBbh/+twBefEomztyNVwY4sUkZKuVWe24GWwEJACdL42J+62WnqBlRs/yCy2VG1Nm78Zi47SryBDCgmQNmdqtTpvm/ijc62Pn5558xb9485eqNcrkc48aNw1dffaW8Rp3VG4mI6OWcvhuPa1GJMNTTwZi2+dOOC14l+fn5Ka+ztLSErq4u3N3rYXbvBtCRSPD7hUiEZZkh+Wbxr3zmzZuHjz/+GGPGjAGQP1whNTUVn3zyCebMmQMdHR3lNOh9+/ahV69eCAgIwLx58wpNg96+fTuEEBg0aFCRZb24kq+DgwOOHj2KyZMno2HDhrCzs8OkSZMwY8YM5TUPHz7EoEGDEB8fjyrVzJFZvRbMBi5F/z/uol3tRMSlZCE0RnVGr76uDt5xMkcbF0u0qW2JerKq+OtyFGb73UDuc6+OHiakl3uPx+2Y5EIzovIEsObEPczv6f7Su4i/6FpUIsZuvoSs3Dy8526LRR/WL7O8y4JElPTS7i2RlJQEMzMzKBQKVK1ataKrQ0T0xhjw6zlciHiGka2d8HUPdwDA/Pnz8euvvyIqKgq6uv/9m9nT0xO1atXCli1bIITAV3tDsGz6WOjoGWD9xs1Fvjpp2rQpOnfurDL0YNu2bRg1ahRSUlJUxtMUSEhIgLOzM5YuXYpPPvlEo/uJUaQjIj5VOXblRbl5AglpWYhPyUR8cv5/nyRn4v7TVPx+ofgp9HVlVdHOxRJtXCzR3MkchnqF6x2jSMf9+FRsOvsAh0NiITMzxMGJbVHdpOhXZq9KCIFPt1zGkZuPizzf0c0K3/XzgGWVVxvDejcuGf3WnENCWjY8a1ngtxHNi7z/8qDu8/uN7tkhIqKKczHiGS5EPIOeVIJP2uUvzlew7svw4cNVAh0gf1LJgAED0K5dO3Ts2BHWUceRcS8Q1oN8MGNX/quUQyvnqrxK6tGjB5YvX47GjRsrX2PNmzcPPXv2VAY6R44cgRACbm5uuHv3LqZNmwY3NzeMHDlSo/t5fuyKBEAbF0tYVjFQBjTxKVl4lppZqCekJJ93rIXhns6wMi09YJCZGUFmZoSG9tVw5+dkhMenYupf1/B/w5uVSy/IhjP3ceTmY0gk+febJwCpBOjeUI7DIbE4fvsJ3ltxCsv7e6Cdq1Wp+RUlOjEdH6+/iIS0bHjYm2HtsGavLdDRBIMdIiIqUsFYnY+aOih7QY4dO4bIyEjl2Mnnffjhh1izZg18fX0xceJEuLm5wW/XTlyTumDDmfuY5RcMoxt3VF4lzZ07FxKJBHPnzkV0dDSsrKyU4zULKBQKzJo1Cw8fPoS5uTn69u2LRYsWKVc9VsfVyATM2BWs/FkAOBUWX+z15ib6sKyiD8sqBrCsYgAjPSn+vBSlslWGVCLBkJaOagU6zzMx0MXPgxvjw1Vn8c+tOPx25j5Gt1F/ZWJ1nA9/ikUHQwEA87rXQ7cGtiozom7FJmHitqu48zgFw367iLFtnTG1qxsMdNUPVOJTMvHx/11AjCIDta2rYMPId1DF4M0MK/gaC3yNRUT0ousPE9Fz5RlIdSQ4/mUH1LB4+cG0QggsPBCK9acjAABTvVzRxLF6sa+SypIiLRurTtzF+tMRyCmiy2bQOw5o6mgOyyr6sDI1gFUVA5ib6ENXWniDgR2BkcpxN1KJBD596r/SrKYt5+5j3t4Q6Ekl2PWZJxraV3vpvJ73KDEdPX4+jaepWejdSI4fBjQqsucoIzsXiw6EYsv5BwAAd3lV/DSoMWpZVSm1jOSMbAxadx43opNgV80IOz9rVSEzrtR9fjPYAYMdIqIXfbL5Eo7efIw+je2wfECjV85PCAGfg6FYdypCeaw8p0FnZOdi09n7+OX4XSRl5BR5jVQiwemZHTV6SOePuymbNWOEEBj/+xUcuhGLGubG2D+xDaoaqt9bVZSM7FwM+PUcrj1UoJ6sKnZ95gkj/ZJ7a/xvPsb0ndeQkJYNIz0p5vesh/7NHIp9tZaRnYsRGy7ifPgzWJjo469PW6GmGgFSeVD3+f1G741FRESv3+3YZBz931iP8R1rlUmeEokEI1s74fnHZ54AZuwKxrYLkcjMKbwWzcvIzRP481IUOn5/Ar6HbiEpIwduNqbYMKI5FvdpAOn/HuAFPTOaBiwyMyO0qmVRJr0YEokEi/s2hH11I0Q+S8Msv+ASF/orjRACX+8NwbWHClQz1sOvHzctNdABgC71bHBoUjt41rJAenYuZuwKxoQ/rkCRll3o2pzcPHz+x1WcD3+GKga62DTqnQoLdDTxZr5cIyKiCvPL/8bqdKtvi9rWZbeMx/2naSjqUT5rdzAWH76F7g1l+LCxHZrWqA4dHc0G7Aoh8O+tOCw5fAt3HqcAAORmhpji5YYPG9tB+r/82rtZVfhqvs8zM9LDz4Mao9+aczhwPQata1licIuX6+n642IkdlyKgo4E+GlgYziYq//q0dbMEFtHt8DaU+H4/shtHAyORVBkIlYMbIx3nM0B5K+kPWNXMI6FPoaBrg7+b3gz1Lcze6m6vm58jQW+xiIiKhARn4pOy04gTwAHJraBu7zsHmYxinS0XvyvymwnCQCLKvqIT8lSHrOvboTejezQu7EdaluX3mtwJTIBiw/ewsX7zwDkBxATOtbCsFZOb+TMoKKsOxmORQdDYaCrg72ft0YdW82eRZcfJGDg2nPIzhWY/p4bxneo/dJ1uRaViEnbr+L+0zToSIDP33VBv6Z28Dl4C4duxEKqI8GvQ5uic73it2V6XThmRwMMdoiI8k3feQ1/XnqITnWssX5E8zLPv6hBvh81dcD58KfYfTUah4JjVLZXaGhvht6N7NDDQ66c9VSwVo6ORIKNZ+7jcEgsAMBAVwcjWzvjs/a1YGb8amNfXre8PIHRmwJx/PYT1LIywd9ftIGxvnovX+KSM9Dj59N4nJSJ9xvY4pfBTV55KntKZg7m7wvBzssPC53r38weSz8qfTuO14HBjgYY7BARAQ8T0tDhuxPIyRPwG++JJjWql0s5JQ3yTc/KxbHQx9h9NRoBd54g93/dQFIdCdrUtoTMzBB/XopS6R3SkQAfNbWHd2dXyKtV/Kupl/UsNQvdfjyJx0mZ+KipPb7vV3pAkZWThyH/dx6B9xPgYl0Fuye0LtPp35vO3cfXe0NUjr3MwO7ywgHK9NYq2N03RpFe0VUhKnNOTk6QSCSFPhMmTEB2djZmzJiBBg0awMTEBHK5HMOGDcOjR49KzNPPzw/NmjVDbXtbhH/fB4rfJyMkYL/KNfPnzy9Upq2t7UvdQ0mDfI30pejhIcdvI5rj4uxO+KanOxo5VENunkDAnSfYHhhVaNG/LaNbYOlHHpU60AHy1/b5aWBj6EiAnZcfwu9K4V6VFy06cBOB9xNgaqCLXz9uWubr3LgU8RqxYBPTyoQDlEmr/HHhAebsvgGB8t/dl6giBAYGIjf3v9c8N27cQJcuXdCvXz+kpaXhypUrmDdvHjw8PJCQkABvb2/07NkTly5dKjZPc3NzfD55GuafSkS2kKKfdRxGjhwJa2trdO3aVXmdu7s7jh07pvy5qK0cypJFFQMM93TCcE8nRMSn4ud/wuB3NbrQdTpv0B5Mr6pFTQt4d3bFcv87mLvnBjwcqhW77s3Oyw+x6Vz+GjkrBjYql1lRzpYm0JFAJcCUSiSvZRPTssSeHdIaMYp0ZaAD5P/lnO13gz08pFWsrKxga2sLYWSG8FRd7Ni1B7Vq1UL79u1hZmYGf39/9O/fH25ubmjZsiV+/vlnXL58GZGRxe/r1KFDB0RXawBUs0fLRvXww4JZaNiwIU6fPq1yna6uLmxtbZUfK6uX22LgZThbmmDae254cZJWZXzwlmZCx9rwrGWBtKxcTPj9CjKyC0/LD36owOzd+StCT+rkgk51y2ewsMzMCL5lMGW/ojHYIa1xIfxZoWmtlbG7lag0OwIj0Xrxvxi05jR+27QFzbz6FDsgVaFQQCKRoFq1asXm9yw1S7nJ5YSOtfDvv//i9u3baNeuncp1YWFhkMvlcHZ2xsCBAxEeHl5m96QObXnwlkaqI8GKAY1gYaKPW7HJWHQgVOX805RMfLr1MrJy8tCpjjUmdXIp1/oMaF4Dp2d2xLaxLXF6ZsdK2VvO11ikFYQQ2B5Y9L9cZWaGr7k2ROUnRpGu3Mwy7c555GWk4JxOfcQo0gs99DMyMjBz5kwMHjy4xMGbG85EICU5CY9WjUDXZdmQSqVYtWoVunTporymRYsW2Lx5M1xdXfH48WMsXLgQnp6eCAkJgYWFRbnd74sGNK+Bdq5v1lo55cG6qiGWD2iE4b9dxJbzD+BZywLdGsiQk5uHL7ZdRXRiOpwtTbB8QCON1yR6GQWbmFZW7NkhrXD4RizOhz+DVCIp1M29zP/OK61KSm+mkgbqAvkB8Pz58yGXy2FkZIQOHTogJCSklFyBFStWwM3NDUZGRnBwcMDkyZORkZGhPL969Wo0bNgQVatWRdWqVdGqVSscOnSo3O7zRRHxqcrxEynXj8KoZlNIqljgt9MRKq87srOzMXDgQOTl5WHVqlXF5peUkY2NZ+9Dom+EtbuPITAwEIsWLcKUKVNw4sQJ5XXdunVD37590aBBA3Tu3BkHDhwAAGzatKlc7rMkZbmK8ZusvasVPuuQv4L19F3XcfnBM3jvCMLZe09hrC/Frx83hZlR5ZpiX1HYs0OVXkpmDub/nf8Qm9CxFga1qIH78WmIT8nA5B3X8Pe1R3C2MMYUL7cKrimVpZIG6gLA0qVLsXz5cmzcuBGurq5YuHAhunTpgtu3b8PUtOhVgX///XfMnDkTv/32Gzw9PXHnzh2MGDECAPDDDz8AAOzt7bF48WLUrp2/aNumTZvQq1cvXL16Fe7u7uV4x/luxSQBAHIUcch4cA1WH84GAKw7FYHdV6PxcUsnDGwmx6cjhyIiIgL//vtvib06W849QHJGDlxtqmLEe62goyNBo0aNEBoaCl9fX3To0KHIdCYmJmjQoAHCwsLK/B7pP1O6uOJC+FNciUxE39XnlMc/bGwHV5uyW91a27Fnhyq95Ufv4HFSJhwtjDG+Y23lv/p6eNjB58MGAICf/r2LXUUsjkWVV8FA3YLP/v37lQN1hRBYsWIF5syZgz59+qB+/frYtGkT0tLS8McffxSb57lz59C6dWsMHjwYTk5O8PLywqBBg1RmMvXo0QPvv/8+XF1d4erqikWLFqFKlSo4f/58ud/zzssP8e3/xm+kBPtDamwGk9rN0b2BDHbVjBCfkoXlR26iVosuOH05GP+3fU+Jr5jSsnKUO5FP6Fhb5XWIEAKZmZnFps3MzERoaChkMlkZ3R0VRU+qg7nd6xY6vv1iFCdfaIDBDlVqN6IV2Hg2Ag9Xj8LJ6e/CSF9X5ZVGwMbF+KxDLQghMNZ7BqxsbDV6pbFr1y7Uq1cPBgYGqFevHnbv3q1yPicnB3PnzoWzszOMjIxQs2ZNLFiwAHl5eeV1y8V6m9cXysrKwtatWzFq1ChIJBJEREQgNjYWXl5eymsMDAzQvn17nD17tth82rRpg8uXL+PixYsAgPDwcBw8eBDdu3dXue7FV2gJCQkYM2bMK79CK6m9/XkpClP/vIJnAVuQ8NtYKM5sg5FOLj6SBuLnQY1wYloHLP+oPjIPf4+MmDDodZ6EPqvOYPCPh3Hwwk2VwGXYsGGYNWsW/rgQiWepWZBc2wPDxzcQHh6OW7duYfny5di8eTOGDh2qTDN16lQEBAQgIiICFy5cwEcffYSkpCQMHz681PuiV5ORU/j7hJMvNCRIKBQKAUAoFIqKrgppICc3T/T8+ZRwnLFfjFp9TMTExCg//v7+AoA4fvy4yM3NEy0HfC4k+kbCsf88cTDgghgwYICQyWQiKSmp2PzPnj0rpFKp8PHxEaGhocLHx0fo6uqK8+fPK69ZuHChsLCwEPv37xcRERHir7/+ElWqVBErVqx4Hb8Cpe0XHwjnmfuF44z9wnnmfrH94oPXWn5F27Fjh5BKpSI6OloIIcSZM2cEAOXPBcaOHSu8vLxKzOunn34Senp6QldXVwAQn332WaFrTpw4IYyMjISOjo4wNTUVCxcuVLY3IYRYvHixMDU1Fbt27RLBwcGv3N62X3wgnGbuF9XafiyMTKuJb775RgAQP/74o0p7i4iIEACK/HhO/EnsC4oW2Tm5on379qLvgCHCY/4R4Thjv/hw5Oeidu3awtDQUFSvXl20atVKbN++XaV+Bfegp6cn5HK56NOnjwgJCSn1z4Ze3aPENOXf74JPzZkHxKPEtIquWoVT9/nN7SLA7SIqqy3nH2DenhswNdDFsS/bw6bqf7OuvL29sX//fuV4ArlcjuoteiOtzgdwsjDG9tHNUKemA5YsWYJx48YVmf+AAQOQlJSkMvj0vffeQ/Xq1bFt2zYAwAcffAAbGxusX79eeU3fvn1hbGyMLVu2lMdtF1LU5opv0nLur0PXrl2hr6+Pv//+GwBw9uxZtG7dGo8ePVJ5zTJ27FhERUXh8OHDReZz4sQJDBw4EAsXLkSLFi1w9+5dTJo0CWPHjsW8efOU12VlZSEyMhKJiYnYtWsXVqxYASsrKzx4kL/Am1wuh7e3N2bMmAEg/5WPjY3NS7W3pDwDPGryCQBA/5+laNOgFn777TflNSW1t7txKVh/OgJ+Vx4i83+9A3bVjNDIoRoOBscol2pY1Ls+hrR0LPF3TBWrqD3FKuMU8LLG7SJIq8UlZ2Dp4VsAgKld3VQCneJeaaycOgJ21Yxw/2kaJv55A23btSvxlca5c+dUXoMA+Q/V59O0adMG//zzD+7cuQMAuHbtGk6fPo3333+/LG+3RLdikgotn/82dXE/ePAAx44dw5gxY5THCrYxiI2NVbk2Li4ONjbFL742b948fPzxxxgzZgwaNGiADz/8ED4+PvD19VV5Namvr4/atWujWbNm+Oabb5CbmwtbW9tXeoVWVHuzcGuOixfyB6WObO2EkX3ew7///qt2e6ttXQW+fRrg7Mx34d3ZBeYm+ohOTMeB5wIdAPhqb8hb+fqzMtGGtW4qEmdjUaXkcyAUyRk5aGBnhqEv/It0z549SExMVM6iKXjg1anpgA0upui76iwu3n8G01Q9ZGXGvpi1UmxsbKEHo42NjcoDdMaMGVAoFKhTpw6kUilyc3OxaNEiDBo0qIzutGRCCPx+IarQcR0JtG5V2eJs2LAB1tbWKuNqnJ2dYWtrC39/fzRu3BhAfhAcEBCAJUuWFJtXWloadHRU/w0olUohhCh2+YI9e/YgJycHTk5OAP5rb0W1nYKen6K82N62nLuPoxEZyE1NwOg2zvmDVD+oh6SkJI3bm0UVA3h3dsWn7WvhuyO3lYOSCxQEx29LT2BlVdnXuqlI7NmhSufM3XjsCXoEHQng82EDSF9YWGf9+vXo1q0b5HK5ynGJRAJXG1P8MqQJpDoS3I9PQeSzkv81++KqtEIIlWM7duzA1q1b8ccff+DKlSvYtGkTvv/++9e29sj60xE4FvoYOhKorC9krC+FvlT7/3rn5eVhw4YNGD58OHR1//u3m0Qigbe3N3x8fLB7927cuHEDI0aMgLGxMQYPHqy8rmCgboEePXpg9erV2L59OyIiIuDv74958+ahZ8+eyn2gZs+ejVOnTuH+/fsIDg7GzJkz8wfAjx2rUrfS2k5RCs5vOnsf8/aGQAgBXZ382TgSieSV25uhnhRj2jq/FVsuED2PPTtUqWRk52LunhsAgGGtnNDA3kzlfMErDT8/P+Wx519pyGQytHO1woJe7hj7lwJReSbYGxSNXo3sCpVla2tb6muQadOmYebMmRg4cCAAoEGDBnjw4AF8fX3LfZbK2Xvx8D2U/yrvqw/qoWt9W9x5nIxv9t1EeHwqZvkF49ePm5b6gK3Mjh07hsjISIwaNarQuenTpyM9PR3jx49HQkICWrRogaNHj6qssRMZGanSkzN37lxIJBLMnTsX0dHRsLKyQo8ePbBo0SLlNY8fP8bHH3+MmJgYGJtUQWLCM3w5c45yteEX21uB0l6hFbS3305HYMH+mwCAFjJd3JDZKv8My6K9FWy58OL4D/YYkDbT/n/60WsVHR2NoUOHwsLCAsbGxmjUqBEuX76sPJ+SkoLPP/8c9vb2MDIyQt26dbF69eoS8wwJCUHfvn3h5OQEI31dXDv0B6xNDTDFy1V5TcEUcA8PD+Tl5cHb21s5Bfz5VxoF+jWWQcTchIFdXUz76zou3X9WqNxWrVqppAGAo0ePwtPTU/lzca89ynvqeXRiOj7/4ypy8wT6NLbDcE8nyMyM0N7VGisHN4GeVIKjNx/jr9e0tlBFTXv38vKCEAKurq6FzkkkEsyfPx8xMTHIyMhAQEAA6tevr3LNiRMn4PvjamXddXV18fXXX+Pu3btIT09HZGQkfvnlF1SrVg3ZuXl4nJSByQuWY+ORi5j4+0WIel0hNamOnXnvYNvF/O1KimpvBa/Qnm87L2rVqhV+27FXGeiM71ALeVHXyqW9cfwHvW3Ys0NlJiEhAa1bt0bHjh1x6NAhWFtb4969eyobEE6ePBnHjx/H1q1b4eTkhKNHj2L8+PGQy+Xo1atXkfmmpaWhZs2aaP9eD0yZPAUA8FWPeqhq+N8y6UuWLMHq1auhr6+PTz/9FJ06dcLIkSNhZmaGSZMmKV9puLi4wMXFBT4+PqhetQq8PuyHExEp+GTLZTjf2ADXmo7w9fUFAEyaNAnt2rXDkiVL0KtXL+zduxfHjh1T2Qm64F/9NWrUgLu7O65evYrly5cX2dNQVjKyc/HZ1st4lpoFd3lV+PRpoNJ7U09eFVO6uGHJ4VtY8PdNtKppAQfz8ntFsSMwUrlXk44E8O3ToNI8PJ+vu0QC9G1sD2crE8SnZCI+JQvxyZn/+/9MJKRlq6QVIg8pwcdgUr8TIJFill8wjtyIRed6Nhj+yfhC7a2oV2h2dnbK9ubc/iP8+flAVDN0xtih/ZEXtKdc2xvHf9DbhFPPwannZWXmzJk4c+YMTp06Vew19evXx4ABA1Sm8TZt2hTvv/8+vv3222LTCSEw7LeL2DalJxq/PxgX/1iu8oD/4IMPkJOTgyNHjuD27dtwdXVVmZIrhMA333yDX3/9VflK45dffkFN1zoY8Ot5BEcroPhrDrq1aog/tm5W5rtz507MnTsX4eHhqFWrFhYtWoQ+ffoozycnJ2PevHnYvXs34uLiIJfLMWjQIHz11VfQ19d/2V9lib+HaTuvY+flh6hurId9n7cpMpDJzRMYuPYcAu8n4B0nc2z7pGWhsU1loTJPe3+UmIbWi49Dky9AHUn+YF9DXR3cuXIGcX9+BfnYX6FnrvoaVAgBceUvPLt0EFlpyWj+zjv4dfUqlZ6lDh06wMnJCRs3bsSagHtYfOgWUm+dhuTyDiQ+fvhGtDeiN526z28GO2CwU1bq1auHrl274uHDhwgICICdnR3Gjx+vMnDz008/xeXLl7Fnzx7I5XKcOHECPXv2xKFDh9CmTZti89537REmbruK6DWjMGval1gwZ7rK+cWLF2PNmjU4evQoXF1dce3aNXh5eWHFihWlzlR5nJSB3r+cQYwiA01qVIN3Z1e42FTR6GEdHR2NGTNm4NChQ0hPT4erqyvWr1+Ppk2bAig8WLXA0qVLMW3atFLz3759OwYNGoQmbbvgqeck6EiAzaNa4NTOdfDz88OtW7dgZGQET09PLFmyBG5uboh6lob3VpxEalYuZnarg0/b11L7ftTld+Uhpvx5rdDxbWNbolWt17cTtqbSsnIwamMgzocXfn3ZztUS9WRmsKyiDytTA1hWKfjoo7qxPnR0JEUGeToSYHQbZ1yLUuBKZAJynjupIwEa2FdD29qWaONiiSY1qkNfVwcxinSsOBaGHYH5M+q8O7vAu3PhV3JEVDR1n998jUVlJjw8HKtXr8aUKVMwe/ZsXLx4ERMnToSBgQGGDRsGAPjpp58wduxY2NvbQ1dXFzo6Ovi///u/EgMdRXo2vv3fOIaqhnowNyn8L9hXmQJuU9UQv41ojl4rT+NKZCKG/XZRo9cx6ry+i4mJUUlz6NAhjB49Gn379i01/wcPHmDq1Klo3LwVbsUmwwrA9PfqoI2LJRYGBGDChAlo3rw5cnJyMGfOHHh5eeHmzZtwMDfB1z3cMX3XdSw7ehvtXKxQT152wfy9JylY9L99mp73pk97f5iQhk82X8bN/22o+TypRIIlfRuWGugWN8i3oL2kZObgQvhTnAqLx+m78bgbl4JrUYm4FpWIlcfvwlhfCofqRrj9OEWZZ5d6Ngx0iMoJgx0qM3l5eWjWrBl8fHwAAI0bN0ZISAhWr16tEuycP38e+/btg6OjI06ePInx48dDJpOhc+fORea77OhtPEnORE0rE9wzLLrJPj8l193dHUFBQfD29oZcLldrlko1Yz1kP/cv8TwBzNwVjGpGevByty1xRtOSJUvg4OCADRs2KI8VrLlSoGCGToG9e/eiY8eOqFmzZon1ys3NxZAhQ/DlzLlY9JsfBJLRvYEM49rlp3txJeCCNWcuX76Mdu3aoV8ze/iHPob/zceY8mcQ9kxoDUM9aYllquNuXAoGrTuPp6lZsKlqgCfJmcpeDicLE9g+t8jjm+R8+FOM//0KnqVmwcJEH/2a2WPdyYiXmpU0oHkNtHO1wv34NDhZGqukq2Kgi051bdCpbv7sqxhFOk7/L/A5HRaPp6lZKoEOAPwbGocYRfob//qPqDJisENlRiaToV69eirH6tati127dgEA0tPTMXv2bOzevVu5AFzDhg0RFBSE77//vshg51pUIracz1+IbWHv+hj8c9Flv+qU3Ij4VLz4QlcAGLf1ChzMjfBhIzv0amyHWlZVCqXdt28funbtin79+hX7+u55jx8/xoEDB9RaG2XBggWwsLTEaWlDZGTvRBUDXSz9qGGh4KvgNdqBAwcAAJ988gl+//13NG3aFL59GuBqZAKCb9xE4zbz8ejWZeTl5cHd3R1//vknatQovvcqMTERc+bMgZ+fHxISEuDs7Iwv532LNeHVEJ+Sicdrx+BBgur0/AcAelwajv3bN5Z6f6+LEAJbzj/Agr9vIidPoL5dVfz6cTPYVTPCcE+nIgMWdag7yFdmZoR+zRzQr5kD8vIEtgdGYvbuGyrXcGE/ovLDYIfKTOvWrXH79m2VY3fu3IGjY/4Kx9nZ2cjOzlZ76mxObh5m7w6GEMCHje3gWcuy2LJfdUqus6UJdCRQGYMhAWCop4OoZ+n46d+7+Onfu/CwN0Pvxnbo4SGHZRUDAOq9vnvepk2bYGpqqjLwtChnzpzB+vXr0d/nD/jdTICeVIJG8mowMVD9a/v8a7T69esjIyMDixcvVr5Gs6xigInNqmLkoulI8eiCn7buQVt3R4SGhsLQsPgemKysLHTp0gXW1tbYuXMn7O3tcebabSw8Eo40UyPUlVXF0SuXUNXwv56i+ZsO49eZI3G/akOkZ+XCSP/Ve5FeVWZOLr7aE4Idl/LHxfRqJMfiPg2VdXvds5J0dCToWMe6UHvjwn5E5YfBDpWZyZMnw9PTEz4+Pujfvz8uXryItWvXYu3atQCAqlWron379pg2bRqMjIzg6OiIgIAAbN68GcuXL1fmUzAl1/WDTxDyKAlV9AR62GchKCgIWVlZiI6ORlBQEKpUqYLatWsDePUpucWNwejpYYejN2Ox52o0TobF49pDBa49VGDhgVC0c7FE78Z2Kq/vYhTpaFDVAYOHXVN5ffe83377DUOGDCkx0EhOTsbQoUPx8TQfbLuZDIkEeMfZHEYis9C1Ba/RjI2NERUVhdOnT8Pe3l7lmoMbf0Ddd9ohpdUorA0R+LCLA7qX8grtt99+w7Nnz3D27Fno6enhdmwyll2TIM3UAfVkVfH7mBao/sL4Kd2HV2FgLkdydVf8evJehY9BiUvKwKdbL+NKZCJ0JMDMbnUwtm3NCl9okQv7Eb1enI0FzsYqS/v378esWbMQFhYGZ2dnTJkyReV1TmxsLGbNmoWjR4/i2bNncHR0xCeffILJkycrH0AdOnSAjdwBIS5DkZKZA+8WZpjcp22hstq3b48TJ04AKLspuTGK9GJfacSnZGL/tUfYHfQI16ISlccfrRkFtyaeGDBlEX47E4E8AaRcPYi8q7vwLE71Fc+pU6fQrl07BAUFwcPDo9h6BAUF5e/pJMnvrdKRSCBEfi+Vjo4Obt++jVq18mdX1atXD/r6+rhz5w6MjY3h4OCg8hotLy8PZmZm8J4yFSu37UdSdBis5Q5Y/d236N27d7F1eP/992Fubg5jY2P47d6DVB1jGNZpD88+I/H7WE9UM1b9vWZlZUEul+P9wWNw0rgtDHR18M+X7WFfvWJ6K65FJWLclsuITcpAVUNd/Dy4Cdq7WlVIXYpTUnsjotJx6rkGGOy8WWIU6fDeHoQLEc/QuEY17PrUEzrlsEbMqwh/koI9V6OxOygaVzYuQG7yE9gOWao8/+yfdciKuY2IkCsqD7ERI0bgxo0buHTpUon5P4xX4INv/0JcSiZa1rTA/B7u+OqreUhOTsaPP/4IV1dX6OvrQwgBPT095ObmYty4cRg3bhwuXrwIb29v/Prrrxg2bJhy2wJjY2OMmzIbO6KrIi38MhQnN+P48eNo3759kXWoU6cO7t+/j+4f9sMNs5Z49ugBFP/8iqlTvOHz7TeFrv/zzz8xePBgPHjwAFP2R+JCxDN0byDDL0OavORv+eXtuvwQs3YHIysnD7Wtq2DdsGZwtjR57fUgovKl7vOb20XQG2VHYCQ8F/+LCxH565+0c7F64wIdAKhpVQVTvNxwclpHrF48F1kxt6E49yeyEx4h9eYJpFw7jCqNu2P4bxfxg/8dXLr/DE8TEvHXX39hzJgxReZZsCllTm4epvrdRKKRDHXqumPjl33RsGEDVKtWDaampqhfv76yt2rChAnIzc2Fu7s75s+fD5lMhl69emHkyJHKbTgKxi316tULy7+dg0n9u8CsZT+YurbAip9/KfYe8/LyUN3CCnfdBiOzmhM8vXri66/mYsP/rS3y+oINWO3s7DC/pzt0JMCB4BicvRf/Kr9qjeTk5uHb/Tfx5V/XkJWTh851rbF7vCcDHaK3HIMdemPEKNIxyy9YZVbUyn/vvvb9ljQhkUgwrGdnbPx9B1JDA/Bo/QQkntmO6u+ORRX3jrjzOAU//hOGj9acQ4OPv0ZmTi5ETU/ce5KCFztVIyMjERMTg8WHbuF8+DOY6Evx68dNVbbFeFFBQBMSEgKZTKb8pKSkIDIyf68mS0tL6OrqKmfKeXd2RT1ZVaCaHc5eu12oHgWqmlshxcASiow8NHKohi1jWqBJw/qIjY1FVlaWyrUFG7AWBHJ1ZVUxpEX+wPQFf99ETm757hUGAIlpWRixIRDrT0cAACa+WxtrP24G0xJ+f0T0duAAZXpjRMSnqsxOASrPdNxhA/rCoGZzlQGn07q6opqxPk7djceZu/FIrO8F+/peWPJvFJb8GwW5mSHaulihjYslWte2xLa9h/D7+QdYefweAOD7fh5wsflvh+6NGzcWKlcIgcGDByMqKkplm47JkycrZ8Hp6+ujefPmyply+ro6WDGwEZqseoQUver442KkMjApEPxQgUg9B6Q/PY5G9lWxefQ7qGqohzt37kAmkxUaB1Wwvk/BkgIAMKWLK/6+/gi3YpPxx8VIDGvl9Eq/4+LEKNJx8k48fvznDh4lZsBYX4pl/TzQrYGs9MRE9FZgsENvjAfxaYWOVabpuMUtMjfwnRrIzRMIeaTIX1E3LB6XHyTgkSIDOy5FKadEP6+Dm5XaD+vSZsEB+esQDRgwAO3atUPHjh1x9PBhpN+7CKuBPli4PxSetSzx9ZTPYGdnh4HjZ+Dj9Reg3+A9SAL3wSZkO2IjbXAqLAw+Pj6YOHGiSvl5eXnYsGEDhg8fDl3d/75Sqpvo40svN8zbcwPLjt5Bj4byQrO3XtWOwEjM3BWs3N+quoketo1tiTq2HHtHRP/hAGVwgPKbIC45A++tOIVnqVmQIH9BvxeX4NcmaVk5uBjxDKfD4nH8dhzuPUlVOa8jAc7MfFftHq3SZsEB+VPJfX198fDhQ7i5ueHrr+fjzye2OBf+FO7yqnj0+0xYyewR02gUkjNy0NypOj6tm4s5M6YhKCgIdnZ2GD16NGbMmAGp9L/1c44ePYquXbsqN2B9Xm6eQPefTuFWbDKGtqyBhb0bvORvrLAYRTo8ff9V2chT098bEVVunI2lAQY7FUsIgTGbLuGfW3GoK6uKNUOb4FFixlszHffsvXgMXneh0PHXsZlmdGI63v3+BDJzVMfUvONkjg0jmxdawPBlnA9/ioFrz0NHAvz9RRu4y81eOU8hBCbvCMKeoEeFzr3pm5ASUdnhbCyqNLYHRuGfW3HQl+pgxYBGcLQwQataFm9FoAP8t3rz817X6zsdCZCVU3jwsG+fBmoHOtHR0Rg6dCgsLCxgbGyMRo0a4fLly8rzhzf/DMXmCbi/rC8a13ZA586dceFC4eDuRYmJiZgwYQJkMhkMDQ1Rt25dHDx4EDm5eZix67pKoKM49yceLPkACf+sqzSvPYno9eGYHapQD56mKnc0n/6eG9xsTUtJoX0qcjXdiPhUFNW1G5eciVrWhfcBe5E6O767urpi9apfMPvYY2SkZ0An4TS8vLxw9+5dWFkVvchfUVtVREVFQc/QGOO2XMY/t+KgI8nfRmT7wRNIvnYE+lbOaF377QmSiUh9DHaowuTk5mHyjiCkZeWiZU1zjGrtXNFVqjAl7aBdnoraE0yTXiV1dnwfPHgwACDONAzL/e8gwd4eSbu34fr16+jUqVOR+b64VQUAVLWUYdSmQFyNTISBrg5WDm6CVjVMsGd2Pyz/aRW2r10BV5u3L1gmotLxNRZVmF9PhuNKZCJMDXTxfT+PN3LxwNdJZmb02l/fFfQqSf+3VYemvUr79u1Ds2bN0K9fP1hbW6Nx48ZYt25dkdd+0q4m5Ka6uHtyLwxNTEvcLmPfvn1o1aoVJkyYABsbG7jVrYem/cbjyv2nMDPSw+9jWqBLPRtMmDABPXt8gInD+kBfl19nRFQ09uxQhbgRrcAP/ncAAN/0cq+w/ZPo1XqV1N3xff/+/Rg4cGD+7vQm1SHv/y3SdYr/Mw8PD8e///6LIUOG4JfNf2LGb0fxYN/PkGflYOfvP8PFxhTbt2/HlStXEBgY+Er3T0Taj8EOvXYZ2bnw3hGEnDyBbvVt8WFju4qu0ltPZmb0Uj1Kz+/4DgCNGzdGSEhIoR3fO3bsiKCgIDx58gRDp/nggZ8v5jSti80TvIrN19raGqNm+uLT368i19kTtb0SkXhhF1xsTBEVFYVJkybh6NGjJe4eT0QE8DUWVYClh2/jblwKrEwNsOjDBsrdzqnykclkym0oCtStW1e5VUUBExMT1K5dG61atcKhXX9AoiPF33/+jtNhRe+bJZPJYC53xMiNl5Vr/iwa8R7iHj9GVlYWLl++jLi4ODRt2hS6urrQ1dVFQEAAfvrpJ+jq6iI3N7fc7pmIKh8GO/Ranbkbj9/O5O9dtPSjhjAv4xV16fVq3bq1chuKAnfu3FFuVVEUVxtTmBpIIXKz8c3fIcguYt8sM6f6uBF6B5k5OfCqZ4Mto1sgOjJcuVVFp06dEBwcjKCgIOWnWbNmGDJkCIKCglQWPSQi4mssem0UadmY+tc1AMCQFjXQ0c26gmtEr6q0rSpSU1OxaNEi9OzZEzKZDE+fPsWqVauQlhAH58YdERaXgi3nHuDE2q9hZ2cHHx8fLDt6B1dMmiMvYwOsgrdh8siv8c/RwypbVRTs/v48ExMTWFhYFDpORMRgh16br/bdQIwiA04WxpjTvW5FV4fKQPPmzbF7927MmjULCxYsgLOzM1asWIEhQ4YAAKRSKW7duoVNmzYhPj4eFhYWaN68OU6dOoW7wgaz/ILxw7E7qBpxH5BIMGPXdfx56SF0q1ph4ncbcf6P5WjcyAN2dnaYNGkSZsyYUbE3TESVEreLALeLeB3+vvYIX2y7CqmOBDs/bYXGNapXdJWoguXmCfRceRohj5LwnrstHidl4GpUInQkwMLeDTC4hfbtiUZEZYvbRdAbI1aRgbl7bgAAJnSoxUCHAABSHQm+6ekOADgcEourUYkAgKEtHBnoEFGZYrBD5UoIgWk7r0GRno0Gdmb4opNLRVeJ3iB21QtPd//9QiRiFOkVUBsi0lYMdqhcbTn/AKfC4mGgq4MfBjSCnpRNjv4TEZ9a6FiuELgfn1YBtSEibcUnD5Wbu3Ep8DkYCgCY1a0OaquxsSS9XSpyx3ciensw2NEy8+fPh0QiUfnY2toqzz9+/BgjRoyAXC6HsbEx3nvvPYSFhamd//bt2yGRSNC7d2+V405OToXKdbExRfSBX9DWxRLDWjmV0R2SNnnVvbmIiNTBqedayN3dHceOHVP+XLDAmhACvXv3hp6eHvbu3YuqVati+fLl6Ny5M27evAkTE5MS833w4AGmTp2Ktm3bFjoXGBioXLV279WH+HrTETzeMRcmdVqjrYvVW7/JJxWvonZ8J6K3h8Y9OydOnCiHalBZ0tXVha2trfJjZWUFAAgLC8P58+exevVqNG/eHG5ubli1ahVSUlKwbdu2EvPMzc3FkCFD8M0336BmzZqFzltZWcHW1ha5BlWxOOAx0u5ehG41GQwcGmDJoVsccEolqogd34no7aFxsPPee++hVq1aWLhwIaKiosqjTvSKwsLCIJfL4ezsjIEDByI8PBwAkJmZCQAqGydKpVLo6+vj9OnTJea5YMECWFlZYfTo0cVeczcuGaM3XkJebjZSb55AlYZdIJFIOOCUiIgqlMbBzqNHjzBp0iT4+fnB2dkZXbt2xZ9//omsrKzyqB9pqEWLFti8eTOOHDmCdevWITY2Fp6ennj69Cnq1KkDR0dHzJo1CwkJCcjKysLixYsRGxuLmJiYYvM8c+YM1q9fj3Xr1hV5PiM7F8v976Dbj6dw63Ey0u6cR15GCkzqdwLAAadERFSxNA52zM3NMXHiRFy5cgWXLl2Cm5sbJkyYAJlMhokTJ+LatWvlUU9SU7du3dC3b180aNAAnTt3xoEDBwAAmzZtgp6eHnbt2oU7d+7A3NwcxsbGOHHiBLp161bsxonJyckYOnQo1q1bB0tLy0Lnz4c/xfs/ncJP/4QhO1fg3TrWsHp0BsY1m0HX1IIDTomIqMK98nYRjx49wtq1a7F48WLo6uoiIyMDrVq1wpo1a+Du7l5W9SxX2r5dRJcuXVC7dm2sXr1aeUyhUCArKwtWVlZo0aIFmjVrhl9++aVQ2qCgIDRu3FglGMrL+98u1RIdyMasgV51GSyrGGB+z3qoXzULtWrVwv9t2YY6LTpxwCkREZWbct0uIjs7Gzt37sT7778PR0dHHDlyBCtXrsTjx48REREBBwcH9OvX76Ur/7zo6GgMHToUFhYWMDY2RqNGjXD58mXleSEE5s+fD7lcDiMjI3To0AEhISFlUrY2yMzMRGhoKGQymcpxMzMzWFlZISwsDJcuXUKvXr2KTF+nTh0EBwcjKCgIQUFBuHr1Kpq390IVZw/YjvgRulUtMbhFDfzzZXt80FCOjRs3wtraGh/378MBp0RE9EbQeOr5F198oZy5M3ToUCxduhT169dXnjcxMcHixYvh5OT0ypVLSEhA69at0bFjRxw6dAjW1ta4d+8eqlWrprxm6dKlWL58OTZu3AhXV1csXLgQXbp0we3bt2FqavrKdahspk6dih49eqBGjRqIi4vDwoULkZSUhOHDhwMA/vrrL1hZWaFGjRoIDg7GpEmT0Lt3b3h5eSnzGDZsGOzs7ODr6wtDQ0Pln2/UszTM2XMDIfE5yJMawt29Pnz6NEBzJ3MA+T0+GzZswPDhw6Gry1UNiIjozaDxE+nmzZv4+eef0bdvX+jr6xd5jVwux/Hjx1+5ckuWLIGDgwM2bNigPPZ8ECWEwIoVKzBnzhz06dMHQP7YFBsbG/zxxx8YN27cK9ehsnn48CEGDRqE+Ph4WFlZoWXLljh//jwcHR0BADExMZgyZQoeP34MmUyGYcOGYd68eSp5REZGQkfnv06/7Nw8rD8dgRXH7iAjOw86OhLUtDLBgYltoa/733XHjh1DZGQkRo0a9XpuloiISA2vPGanPNWrVw9du3bFw4cPERAQADs7O4wfPx5jx44FAISHh6NWrVq4cuUKGjdurEzXq1cvVKtWDZs2bSoy38zMTOU0bCD/nZ+Dg4PWjtl5GTGKdETEpyItKxffH7mNW7HJAICWNc3h82ED1LTi1g9ERFSx1B2zo3HPjq+vL2xsbAr96/23337DkydPMGPGDM1rW4zw8HCsXr0aU6ZMwezZs3Hx4kVMnDgRBgYGGDZsGGJjYwEANjY2KulsbGzw4MGDEu/hm2++KbN6apsdgZGY5ReMvOfC4GrGepjzfl181NQeEglXQyYiospD4wHKv/76K+rUqVPouLu7O9asWVMmlSqQl5eHJk2awMfHB40bN8a4ceMwduxYlVlFAAo9fIUQJT6QZ82aBYVCofxwcUQgL0/gRrQCSw/fwoxdqoGOBMAfY1qgXzMHBjpERFTpaNyzExsbW2hmD5C/XUBJC9O9DJlMhnr16qkcq1u3Lnbt2gUAyg0uX6xTXFxcod6e5xkYGMDAwKBM61oZxSjScSosHqfD4nHmbjyepha9MKQAoEjPeb2VIyIiKiMaBzsODg44c+YMnJ2dVY6fOXMGcrm8zCoGAK1bt8bt27dVjt25c0c52NbZ2Rm2trbw9/dXjtnJyspCQEAAlixZUqZ1qYwKxt04W5pAZmaElMwcnL/3FKfvxuNU2BPce5Kqcr2xvhSNHKrh3L2neH4gF1dAJiKiykzjYGfMmDHw9vZGdnY23n33XQDAP//8g+nTp+PLL78s08pNnjwZnp6e8PHxQf/+/XHx4kWsXbsWa9euBZD/+srb2xs+Pj5wcXGBi4sLfHx8YGxsjMGDB5dpXSqb58fdSAA4WZggKiENOc+9n9KRAA3sq6GdiyXa1LZE4xrVoa+rgx2BkZjtdwO5QnAFZCIiqvQ0no0lhMDMmTPx008/KffDMjQ0xIwZM/DVV1+VeQX379+PWbNmISwsDM7OzpgyZYpyNlZBfb755hv8+uuvSEhIQIsWLfDLL7+orP1TGm1bQTlGkQ7Pxf+iqD/ZGubGaONiiba1LeFZyxJmxnrF5nE/Po0rIBMR0RtL3ef3S089T0lJQWhoKIyMjODi4lKpx8BoW7Cz7uQ9LDp4q9DxHwc0Qq/GdhVQIyIiorJXblPPC1SpUgXNmzd/2eRUTq5FJWK5/51Cx6USCd6paV4BNSIiIqpYLxXsBAYG4q+//kJkZKTyVVYBPz+/MqkYae5uXDJGbLiI9Ow81LIyQUR8KvIEOO6GiIjeahoHO9u3b8ewYcPg5eUFf39/eHl5ISwsDLGxsfjwww/Lo46khujEdHy8/iIS0rLhYW+G38e2RHJGNsfdEBHRW0/jRQV9fHzwww8/YP/+/dDX18ePP/6I0NBQ9O/fHzVq1CiPOlY68+fPh0QiUfkUrAlUcL5OnTowMTFB9erV0blzZ1y4cKHEPDt06FAoT4lEgu7duyM+JRMf/98FRMc9Q97ZDbiydAisqpmib7dO0H0WzkCHiIjeahoHO/fu3UP37t0B5C/Ol5qaColEgsmTJyunhFP+itIxMTHKT3BwsPKcq6srVq5cieDgYJw+fRpOTk7w8vLCkydPis3Pz89PJb8bN25AKpWiR+8+GLHhIsLjU5H2zyoYPbmJ37duQXBwMLy8vNC5c2dER0e/jlsmIiJ6I2kc7JibmyM5OX9TSDs7O9y4cQMAkJiYiLS0tLKtXSWmq6sLW1tb5cfKykp5bvDgwejcuTNq1qwJd3d3LF++HElJSbh+/Xqx+Zmbm6vk5+/vD2NjY/inO+NGdBKq6wsk3jyF5d9/h3bt2qF27dqYP38+nJ2dC22vQURE9DbRONhp27Yt/P39AQD9+/fHpEmTMHbsWAwaNAidOnUq8wpWVmFhYZDL5XB2dsbAgQMRHh5e5HVZWVlYu3YtzMzM4OHhoXb+/7d+PeRNOuHyo3RUMdDFmiGNkJubC0NDQ5XrjIyMcPr06Ve6FyIiospM4wHKK1euREZGBoD8DTX19PRw+vRp9OnTB/PmzSvzClZGLVq0wObNm+Hq6orHjx9j4cKF8PT0REhICCwsLADkL5Y4cOBApKWlQSaTwd/fH5aWlmrlf/78BYTcuAHbj0eiqq4O/m94M7SoaYFWrVrh22+/Rd26dWFjY4Nt27bhwoULcHFxKc/bJSIieqNptKhgTk4Ofv/9d3Tt2lVlwG1lV96LCqampqJWrVqYPn06pkyZojwWExOD+Ph4rFu3Dv/++y8uXLgAa2vrEvMSQqB5twG4fvkiHMauwq9Dm6JzvfxNT+/du4dRo0bh5MmTkEqlaNKkCVxdXXHlyhXcvHmzzO+LiIioIqn7/NboNZauri4+++wzZGZmvnIF3yYmJiZo0KABwsLCVI7Vrl0bLVu2xPr166Grq4v169eXmtcPh4Jx5fgBVPHwwncfNVQGOgBQq1YtBAQEICUlBVFRUbh48SKys7MLbdpKRET0NtF4zE6LFi1w9erV8qiL1srMzERoaChkMlmx1wghSg0it5x/AJ9fNkDkZmP+5E/Rp4l9kdeZmJhAJpMhISEBR44cQa9evV6p/kRERJWZxmN2xo8fjy+//BIPHz5E06ZNYWJionK+YcOGZVa5ymrq1Kno0aMHatSogbi4OCxcuBBJSUkYPnw4UlNTsWjRIvTs2RMymQxPnz7FqlWr8PDhQ/Tr10+Zx7Bhw2BnZwdfX18AwL5rj/DV3htIuX4Ujdp0wcTuTQqVe+TIEQgh4Obmhrt372LatGlwc3PDyJEjX9u9ExERvWk0DnYGDBgAAJg4caLymEQigRACEokEubm5ZVe7Surhw4cYNGgQ4uPjYWVlhZYtW+L8+fNwdHRERkYGbt26hU2bNiE+Ph4WFhZo3rw5Tp06BXd3d2UekZGR0NHJ73g7cTsOU3YEIetpNDIf3sSS9T8UWa5CocCsWbPw8OFDmJubo2/fvli0aBH09Ire2ZyIiOhtoPGu5w8ePCjxvKOj4ytVqCK8qbuexyjScSQkFr4HQ5GZI9DDQ44fBzSCjo6koqtGRERU4cpt1/PKGMxURjsCIzHTLxgFoairTRUs6+fBQIeIiEhDGgc7mzdvLvH8sGHDXroylC9GkY5ZzwU6AHA3LgVPUzO5zxUREZGGNA52Jk2apPJzdnY20tLSoK+vD2NjYwY7ZSDscQryXni5mCeA+/FpDHaIiIg0pPHU84SEBJVPSkoKbt++jTZt2mDbtm3lUce3zu6rDwsdk0okcLI0roDaEBERVW4aBztFcXFxweLFiwv1+pDm/gyMwu6rjwAABcNzpBIJfPrUZ68OERHRS9D4NVZxpFIpHj16VFbZvZWuRSVi7p78XeSndHFFv2b2uB+fBidLYwY6REREL0njYGffvn0qPwshEBMTg5UrV6J169ZlVrG3TXxKJj7dehlZuXnoXNcGn3esDR0dCYMcIiKiV6RxsNO7d2+VnyUSCaysrPDuu+9i2bJlZVWvt0pObh4+/+MKYhQZqGlpguUDOMWciIiorGgc7OTl5ZVHPd5qvodu4Xz4M5joS7F2WFNUNeSKx0RERGWlTAYo08vbGxSN9acjAADL+nugtrVpBdeIiIhIu2gc7Hz00UdYvHhxoePfffedykaWVLqbj5IwY9d1AMD4DrXwXv3id0UnIiKil6NxsBMQEIDu3bsXOv7ee+/h5MmTZVKpt0FiWhbGbb2EjOw8tHO1wpdebhVdJSIiIq2kcbCTkpICfX39Qsf19PSQlJRUJpXSdrl5AhO3ByHqWToczI3w08BGkHJAMhERUbnQONipX78+duzYUej49u3bUa9evTKplLZb7n8bJ+88gaGeDn4d2gzVjAsHj0RERFQ2NJ6NNW/ePPTt2xf37t3Du+++CwD4559/sG3bNvz1119lXkFtc/hGDH45fg8AsKRvQ9STF78lPREREb06jYOdnj17Ys+ePfDx8cHOnTthZGSEhg0b4tixY2jfvn151FFrhD1Oxpd/XgMAjG7jjF6N7Cq4RkRERNpPIoQQpV+m3ZKSkmBmZgaFQoGqVcunpyUpIxu9V55BeHwqWtY0x9bRLaAr5cx/IiKil6Xu81vjp21gYCAuXLhQ6PiFCxdw6dIlTbN7K+TlCUzZcQ3h8amQmRli5eAmDHSIiIheE42fuBMmTEBUVFSh49HR0ZgwYUKZVErbrDx+F8dCH0NfVwdrhjaFZRWDiq4SERHRW0PjMTs3b95EkyZNCh1v3Lgxbt68WSaV0hYxinTsvhKN5f53AAALe9WHh0O1iq0UERHRW0bjYMfAwACPHz9GzZo1VY7HxMRAV1fj7LTWjsBIzPILRt7/RkS1cDZH/+YOFVspIiKit5DGr7G6dOmCWbNmQaFQKI8lJiZi9uzZ6NKlS5lWrrKKUaSrBDoAEHj/GWIU6RVXKSIioreUxl0xy5YtQ7t27eDo6IjGjRsDAIKCgmBjY4MtW7aUeQUro4j4VJVABwDyBHA/Pg0yM6OKqRQREdFbSuNgx87ODtevX8fvv/+Oa9euwcjICCNHjsSgQYOgp6dXHnWsdJwtTaAjgUrAI5VI4GRpXHGVIiIieku91CAbExMTfPLJJ2VdF60hMzOCb58GmO13A7lCQCqRwKdPffbqEBERVYCXHlF88+ZNREZGIisrS+V4z549X7lS2mBA8xpo52qF+/FpcLI0ZqBDRERUQTQOdsLDw/Hhhx8iODgYEokEBQswSyT5u3bn5uaWbQ0rMZmZEYMcIiKiCqbxbKxJkybB2dkZjx8/hrGxMUJCQnDy5Ek0a9YMJ06cKIcqEhEREb08jXt2zp07h3///RdWVlbQ0dGBjo4O2rRpA19fX0ycOBFXr14tj3oSERERvRSNe3Zyc3NRpUoVAIClpSUePXoEAHB0dMTt27fLtnZEREREr0jjnp369evj+vXrqFmzJlq0aIGlS5dCX18fa9euLbSqMhEREVFF0zjYmTt3LlJTUwEACxcuxAcffIC2bdvCwsICO3bsKPMKEhEREb0KiSiYTvUKnj17hurVqytnZFU2SUlJMDMzg0KhQNWqVSu6OkRERKQGdZ/fZbJzp7m5eVlkQ0RERFTmNB6gTERERFSZMNghIiIircZgh4iIiLSaxsHOyZMnkZOTU+h4Tk4OTp48WSaVIiIiIiorGgc7HTt2xLNnzwodVygU6NixY5lUioiIiKisaBzsCCGKnGL+9OlTmJiYlEmliIiIiMqK2lPP+/TpAyB/d/MRI0bAwMBAeS43NxfXr1+Hp6dn2deQiIiI6BWoHeyYmZkByO/ZMTU1hZGRkfKcvr4+WrZsibFjx5Z9DYmIiIhegdrBzoYNGwAATk5OmDp1Kl9ZERERUaWg8Zid6dOnq4zZefDgAVasWIGjR4+WacWIiIiIyoLGwU6vXr2wefNmAEBiYiLeeecdLFu2DL169cLq1avLvIJEREREr0LjYOfKlSto27YtAGDnzp2wtbXFgwcPsHnzZvz0009lXkEiIiKiV6FxsJOWlgZTU1MAwNGjR9GnTx/o6OigZcuWePDgQZlXkIiIiOhVaBzs1K5dG3v27EFUVBSOHDkCLy8vAEBcXFyJ26sTERERVQSNg52vvvoKU6dOhZOTE9555x20atUKQH4vT+PGjcu8gkRERESvQuNg56OPPkJkZCQuXbqEI0eOKI936tQJP/zwQ5lW7kW+vr6QSCTw9vZWHhNCYP78+ZDL5TAyMkKHDh0QEhJSrvUgIiKiyuOldj23tbWFqakp/P39kZ6eDgBo3rw56tSpU6aVe15gYCDWrl2Lhg0bqhxfunQpli9fjpUrVyIwMBC2trbo0qULkpOTy60uREREVHloHOw8ffoUnTp1gqurK95//33ExMQAAMaMGYMvv/yyzCsIACkpKRgyZAjWrVuH6tWrK48LIbBixQrMmTMHffr0Qf369bFp0yakpaXhjz/+KJe6EBERUeWicbAzefJk6OnpITIyEsbGxsrjAwYMwOHDh8u0cgUmTJiA7t27o3PnzirHIyIiEBsbqxwkDQAGBgZo3749zp49W2x+mZmZSEpKUvkQERGRdlJ7u4gCR48exZEjR2Bvb69y3MXFpVymnm/fvh1XrlxBYGBgoXOxsbEAABsbG5XjNjY2JdbF19cX33zzTdlWlIiIiN5IGvfspKamqvToFIiPj1fZCb0sREVFYdKkSdi6dSsMDQ2Lve757SuA/NdbLx573qxZs6BQKJSfqKioMqszERERvVk0DnbatWun3C4CyA808vLy8N1336Fjx45lWrnLly8jLi4OTZs2ha6uLnR1dREQEICffvoJurq6yh6dgh6eAnFxcYV6e55nYGCAqlWrqnyIiIhIO2n8Guu7775Dhw4dcOnSJWRlZWH69OkICQnBs2fPcObMmTKtXKdOnRAcHKxybOTIkahTpw5mzJiBmjVrwtbWFv7+/so1frKyshAQEIAlS5aUaV2IiIioctI42KlXrx6uX7+O1atXQyqVIjU1FX369MGECRMgk8nKtHKmpqaoX7++yjETExNYWFgoj3t7e8PHxwcuLi5wcXGBj48PjI2NMXjw4DKtCxEREVVOGgc7kZGRcHBwKHKAb2RkJGrUqFEmFVPX9OnTkZ6ejvHjxyMhIQEtWrTA0aNHlft3ERER0dtNIoQQmiSQSqWIiYmBtbW1yvGnT5/C2toaubm5ZVrB1yEpKQlmZmZQKBQcv0NERFRJqPv81niAcnEznVJSUkqcMUVERERUEdR+jTVlyhQA+bOv5s2bpzL9PDc3FxcuXECjRo3KvIJEREREr0LtYOfq1asA8nt2goODoa+vrzynr68PDw8PTJ06texrSERERPQK1A52jh8/DiB/6vePP/7IsS1ERERUKWg8G2vDhg3lUQ8iIiKicqHxAGUiIiKiyoTBDhEREWk1BjtERESk1RjsEBERkVZjsENERERajcEOERERaTUGO0RERKTVGOwQERGRVmOwQ0RERFqNwQ4RERFpNQY7REREpNUY7BAREZFWY7BDREREWo3BDhEREWk1BjtERESk1RjsEBERkVZjsENERERajcEOERERaTUGO0RERKTVGOwQERGRVmOwQ0RERFqNwQ4RERFpNQY7REREpNUY7BAREZFWY7BDREREWo3BDhEREWk1BjtERESk1RjsEBERkVZjsENERERajcEOERERaTUGO0RERKTVGOwQERGRVmOwQ0RERFqNwQ4RERFpNQY7REREpNUY7BAREZFWY7BDREREWo3BDhEREWk1BjtERESk1RjsEBERkVZjsENERERajcEOERERaTUGO0RERKTVGOwQERGRVmOwQ0RERFqNwQ4RERFpNQY7REREpNUY7BAREZFWY7BDREREWo3BDhEREWk1BjtERESk1RjsEBERkVZjsENERERajcEOERERaTUGO0RERKTVGOwQERGRVmOwQ0RERFqNwQ4RERFptTc62PH19UXz5s1hamoKa2tr9O7dG7dv31a5RgiB+fPnQy6Xw8jICB06dEBISEgF1ZiIiIjeNG90sBMQEIAJEybg/Pnz8Pf3R05ODry8vJCamqq8ZunSpVi+fDlWrlyJwMBA2NraokuXLkhOTq7AmhMREdGbQiKEEBVdCXU9efIE1tbWCAgIQLt27SCEgFwuh7e3N2bMmAEAyMzMhI2NDZYsWYJx48aplW9SUhLMzMygUChQtWrV8rwFIiIiKiPqPr/f6J6dFykUCgCAubk5ACAiIgKxsbHw8vJSXmNgYID27dvj7NmzxeaTmZmJpKQklQ8RERFpp0oT7AghMGXKFLRp0wb169cHAMTGxgIAbGxsVK61sbFRniuKr68vzMzMlB8HB4fyqzgRERFVqEoT7Hz++ee4fv06tm3bVuicRCJR+VkIUejY82bNmgWFQqH8REVFlXl9iYiI6M2gW9EVUMcXX3yBffv24eTJk7C3t1cet7W1BZDfwyOTyZTH4+LiCvX2PM/AwAAGBgblV2EiIiJ6Y7zRPTtCCHz++efw8/PDv//+C2dnZ5Xzzs7OsLW1hb+/v/JYVlYWAgIC4Onp+bqrS0RERG+gN7pnZ8KECfjjjz+wd+9emJqaKsfhmJmZwcjICBKJBN7e3vDx8YGLiwtcXFzg4+MDY2NjDB48uIJrT0RERG+CNzrYWb16NQCgQ4cOKsc3bNiAESNGAACmT5+O9PR0jB8/HgkJCWjRogWOHj0KU1PT11xbIiIiehNVqnV2ygvX2SEiIqp8tHKdHSIiIiJNMdghIiIircZgh4iIiLQagx0iIiLSagx2iIiISKsx2CEiIiKtxmCHiIiItBqDHSIiItJqDHaIiIhIqzHYISIiIq3GYIeIiIi0GoMdIiIi0moMdoiIiEirMdghIiIircZgh4iIiLQagx0iIiLSagx2iIiISKsx2CEiIiKtxmCHiIiItBqDHSIiItJqDHaIiIhIqzHYISIiIq3GYIeIiIi0GoMdIiIi0moMdoiIiEirMdghIiIircZgh4iIiLQagx0iIiLSagx2iIiISKsx2CEiIiKtxmCHiIiItBqDHSIiItJqDHaIiIhIqzHYISIiIq3GYIeIiIi0GoMdIiIi0moMdoiIiEirMdghIiIircZgh4iIiLQagx0iIiLSagx2iIiISKsx2CEiIiKtxmCHiIiItBqDHSIiItJqDHaIiIhIqzHYISIiIq3GYIeIiIi0GoMdIiIi0moMdoiIiEirMdghIiIircZgh4iIiLQagx0iIiLSagx2iIiISKsx2CEiIiKtxmCHiIiItBqDHSIiItJqDHaIiIhIqzHYISIiIq3GYIeIiIi0GoMdIiIi0moMdoiIiEirMdghIiIircZgh4iIiLSa1gQ7q1atgrOzMwwNDdG0aVOcOnWqoqtEREREbwCtCHZ27NgBb29vzJkzB1evXkXbtm3RrVs3REZGVnTViIiIqIJJhBCioivxqlq0aIEmTZpg9erVymN169ZF79694evrW2r6pKQkmJmZQaFQoGrVquVZVSIiIioj6j6/dV9jncpFVlYWLl++jJkzZ6oc9/LywtmzZ4tMk5mZiczMTOXPCoUCQP4vjYiIiCqHgud2af02lT7YiY+PR25uLmxsbFSO29jYIDY2tsg0vr6++Oabbwodd3BwKJc6EhERUflJTk6GmZlZsecrfbBTQCKRqPwshCh0rMCsWbMwZcoU5c95eXl49uwZLCwsik3zMpKSkuDg4ICoqKiXej1WkelZNsuuLOlZ9ttV9qumZ9mVr+ySCCGQnJwMuVxe4nWVPtixtLSEVCot1IsTFxdXqLengIGBAQwMDFSOVatWrbyqiKpVq77SH3BFpmfZLLuypGfZb1fZr5qeZVe+sotTUo9OgUo/G0tfXx9NmzaFv7+/ynF/f394enpWUK2IiIjoTVHpe3YAYMqUKfj444/RrFkztGrVCmvXrkVkZCQ+/fTTiq4aERERVTCtCHYGDBiAp0+fYsGCBYiJiUH9+vVx8OBBODo6Vmi9DAwM8PXXXxd6ZVYZ0rNsll1Z0rPst6vsV03Psitf2WVBK9bZISIiIipOpR+zQ0RERFQSBjtERESk1RjsEBERkVZjsENERERajcFOOVq1ahWcnZ1haGiIpk2b4tSpU2qlO3nyJHr06AG5XA6JRII9e/aoXaavry+aN28OU1NTWFtbo3fv3rh9+7ba6VevXo2GDRsqF39q1aoVDh06pHb6F+sikUjg7e2t1vXz58+HRCJR+dja2qpdXnR0NIYOHQoLCwsYGxujUaNGuHz5slppnZycCpUtkUgwYcKEUtPm5ORg7ty5cHZ2hpGREWrWrIkFCxYgLy9P7bonJyfD29sbjo6OMDIygqenJwIDA4u8trT2IYTA/PnzIZfLYWRkhA4dOiAkJESttH5+fujatSssLS0hkUgQFBSkdtnZ2dmYMWMGGjRoABMTE8jlcgwbNgyPHj1Sq+z58+ejTp06MDExQfXq1dG5c2dcuHBB7ft+3rhx4yCRSLBixQq10o4YMaLQn33Lli01Kjs0NBQ9e/aEmZkZTE1N0bJlS0RGRpaatqh2J5FI8N1336lVdkpKCj7//HPY29vDyMgIdevWVdkUubT0jx8/xogRIyCXy2FsbIz33nsPYWFhANT7PimuvamTtqT2Vlr6ktqbOmWX1N40/R59vr2pk7ak9qZu2UW1txkzZpSatqT2pk7ZxbU3ddKW1NbKG4OdcrJjxw54e3tjzpw5uHr1Ktq2bYtu3bohMjKy1LSpqanw8PDAypUrNS43ICAAEyZMwPnz5+Hv74+cnBx4eXkhNTVVrfT29vZYvHgxLl26hEuXLuHdd99Fr169lA9LdQUGBmLt2rVo2LChRunc3d0RExOj/AQHB6uVLiEhAa1bt4aenh4OHTqEmzdvYtmyZWqvjB0YGKhSbsEilf369Ss17ZIlS7BmzRqsXLkSoaGhWLp0Kb777jv8/PPPapUNAGPGjIG/vz+2bNmC4OBgeHl5oXPnzoiOji50bWntY+nSpVi+fDlWrlyJwMBA2NraokuXLkhOTi41bWpqKlq3bo3FixcXe7649Glpabhy5QrmzZuHK1euwM/PD3fu3EHPnj3VqrerqytWrlyJ4OBgnD59Gk5OTvDy8sKTJ0/USl9gz549uHDhgsry8eqkfe+991TawMGDB9VOf+/ePbRp0wZ16tTBiRMncO3aNcybNw+Ghoalpn2+zJiYGPz222+QSCTo27evWmVPnjwZhw8fxtatWxEaGorJkyfjiy++wN69e0tNL4RA7969ER4ejr179+Lq1atwdHRE586dkZqaqtb3SXHt7Z9//ik1bUntrbSyS2pv6tS7pPamyffoi+1N3bTFtTd10hfX3gIDA0tNW1J7U6fs4trbX3/9VWLa0tpauRNULt555x3x6aefqhyrU6eOmDlzpkb5ABC7d+9+6XrExcUJACIgIOCl86hevbr4v//7P7WvT05OFi4uLsLf31+0b99eTJo0Sa10X3/9tfDw8HipOs6YMUO0adPmpdIWZdKkSaJWrVoiLy+v1Gu7d+8uRo0apXKsT58+YujQoWqVlZaWJqRSqdi/f7/KcQ8PDzFnzpwS077YPvLy8oStra1YvHix8lhGRoYwMzMTa9asKTHt8yIiIgQAcfXqVbXLLsrFixcFAPHgwQON0yoUCgFAHDt2TO2yHz58KOzs7MSNGzeEo6Oj+OGHH9RKO3z4cNGrV68S61NS+gEDBqj1563Offfq1Uu8++67aqd3d3cXCxYsUDnWpEkTMXfu3FLT3759WwAQN27cUB7LyckR5ubmYt26dYXSv/h9okl7K+m7SJ32ps53WXHtTZ20JbW34tKr096KSqtJeysqvbrtTZ37Lqm9FZVe3fb2YlpN21pZY89OOcjKysLly5fh5eWlctzLywtnz559rXVRKBQAAHNzc43T5ubmYvv27UhNTUWrVq3UTjdhwgR0794dnTt31rjMsLAwyOVyODs7Y+DAgQgPD1cr3b59+9CsWTP069cP1tbWaNy4MdatW6dx+UD+n9/WrVsxatQotTaGbdOmDf755x/cuXMHAHDt2jWcPn0a77//vlrl5eTkIDc3F4aGhirHjYyMcPr0aY3qHhERgdjYWJW2Z2BggPbt27/2tgfktz+JRKLx3nNZWVlYu3YtzMzM4OHhoVaavLw8fPzxx5g2bRrc3d01ruuJEydgbW0NV1dXjB07FnFxcWqXe+DAAbi6uqJr166wtrZGixYtNHr9XODx48c4cOAARo8erXaaNm3aYN++fYiOjoYQAsePH8edO3fQtWvXUtNmZmYCgErbk0ql0NfXL7Ltvfh9okl7e5XvInXTF9feSktbWnsrKr267a24stVtby+m16S9lXbfpbW3otKr295eTKtpWytz5R5OvYWio6MFAHHmzBmV44sWLRKurq4a5YVX6NnJy8sTPXr00LjH4/r168LExERIpVJhZmYmDhw4oHbabdu2ifr164v09HQhhNCoZ+fgwYNi586d4vr168peIRsbGxEfH19qWgMDA2FgYCBmzZolrly5ItasWSMMDQ3Fpk2b1K57gR07dgipVCqio6PVuj4vL0/MnDlTSCQSoaurKyQSifDx8dGozFatWon27duL6OhokZOTI7Zs2SIkEkmp7eXF9nHmzBkBoFDdx44dK7y8vEpM+7yy6NlJT08XTZs2FUOGDFE77d9//y1MTEyERCIRcrlcXLx4Ue2yfXx8RJcuXZS9cZr07Gzfvl3s379fBAcHi3379gkPDw/h7u4uMjIySk0fExMjAAhjY2OxfPlycfXqVeHr6yskEok4ceKEWvddYMmSJaJ69erKvz/q1D0zM1MMGzZMABC6urpCX19fbN68Wa30WVlZwtHRUfTr1088e/ZMZGZmCl9fXwGgUHsp6vtE3fZW2ndRae1Nne+y4tpbSWnVaW/FpVenvRWXVt32VlR6ddubOr+zktpbcenVaW9FpdWkrZUHBjvloCDYOXv2rMrxhQsXCjc3N43yepVgZ/z48cLR0VFERUVplC4zM1OEhYWJwMBAMXPmTGFpaSlCQkJKTRcZGSmsra1FUFCQ8pgmwc6LUlJShI2NjVi2bFmp1+rp6YlWrVqpHPviiy9Ey5YtNS7Xy8tLfPDBB2pfv23bNmFvby+2bdsmrl+/LjZv3izMzc3Fxo0b1c7j7t27ol27dgKAkEqlonnz5mLIkCGibt26JaYrLth59OiRynVjxowRXbt2LTHt81412MnKyhK9evUSjRs3FgqFQu20KSkpIiwsTJw7d06MGjVKODk5icePH5ea/tKlS8LGxkbloatJsPOiR48eCT09PbFr165S0xf8fR80aJDKdT169BADBw7UqGw3Nzfx+eefF3u+qPTfffedcHV1Ffv27RPXrl0TP//8s6hSpYrw9/dXK/2lS5eEh4eHsu117dpVdOvWTXTr1k3luqK+T9Rtb6V9F5XW3kpLX1J7KymtOu2tqPTqtjd1v4OLa29FpVe3valTdkntrbj06rS34tKq29bKA4OdcpCZmSmkUqnw8/NTOT5x4kTRrl07jfJ62WDn888/F/b29iI8PFzjtC/q1KmT+OSTT0q9bvfu3cpGXPABICQSiZBKpSInJ0fjsjt37lxo7FNRatSoIUaPHq1ybNWqVUIul2tU3v3794WOjo7Ys2eP2mns7e3FypUrVY59++23Gge2QuR/+RY8OPr37y/ef//9Eq9/sX3cu3dPABBXrlxRua5nz55i2LBhJaZ93qsEO1lZWaJ3796iYcOGxfbKqduua9euXWQv2Yvpf/jhB2U7e77t6ejoCEdHx5cu+/mxKMWlz8zMFLq6uuLbb79VuW769OnC09NT7bJPnjwpAKj8Y6G0stPS0oSenl6h8V6jR48uFNyWVn5iYqKIi4sTQuSPORw/frzyXHHfJ+q0N3W+i0pqb6WlL6m9afo9+GJ7Ky69Ou3tZcp+vr0Vl16d9qZO2SW1t+LSq9Pe1Cm7pLZWXjhmpxzo6+ujadOmyhk9Bfz9/eHp6VmuZQsh8Pnnn8PPzw///vsvnJ2dyyTPgvetJenUqROCg4MRFBSk/DRr1gxDhgxBUFAQpFKpRuVmZmYiNDQUMpms1Gtbt25daJrjnTt3NN4MdsOGDbC2tkb37t3VTpOWlgYdHdW/SlKpVKOp5wVMTEwgk8mQkJCAI0eOoFevXhqld3Z2hq2trUrby8rKQkBAQLm3PSB/OnD//v0RFhaGY8eOwcLC4pXyU7ftffzxx7h+/bpK25PL5Zg2bRqOHDmicblPnz5FVFSUWm1PX18fzZs3f+X2t379ejRt2lTtMUpA/u87Ozu7TNqfmZkZrKysEBYWhkuXLqFXr16lfp+U1N5atWr1St9F6nyXFdfeXvZ7sKC9lZa+pPZ2+PBhjct+vr2VVnZJ7a1GjRpql11Ueyut7JLaW25urtplF9XWyl25h1Nvqe3btws9PT2xfv16cfPmTeHt7S1MTEzE/fv3S02bnJwsrl69Kq5evSoAKN/LvjjDoCifffaZMDMzEydOnBAxMTHKT1pamlr1njVrljh58qSIiIgQ169fF7NnzxY6Ojri6NGjaqV/kSavsb788ktx4sQJER4eLs6fPy8++OADYWpqqtbv7OLFi0JXV1csWrRIhIWFid9//10YGxuLrVu3ql3X3NxcUaNGDTFjxgy10wiRP7PCzs5O7N+/X0RERAg/Pz9haWkppk+frnYehw8fFocOHRLh4eHi6NGjwsPDQ7zzzjsiKyur0LWltY/FixcLMzMz4efnJ4KDg8WgQYOETCYTSUlJpaZ9+vSpuHr1qjhw4IAAILZv3y6uXr0qYmJiSi07Oztb9OzZU9jb24ugoCCV9peZmVli2pSUFDFr1ixx7tw5cf/+fXH58mUxevRoYWBgoJy9oenfi+dfK5SUNjk5WXz55Zfi7NmzIiIiQhw/fly0atVK2NnZiaSkJLXK9vPzE3p6emLt2rUiLCxM/Pzzz0IqlYpTp06pVW+FQiGMjY3F6tWrNf7zbt++vXB3dxfHjx8X4eHhYsOGDcLQ0FCsWrVKrfR//vmnOH78uLh3757Ys2ePcHR0FH369BFCqPd9Ulx7Gz16dKlpS2pvpZVdUnv75JNPSkxbWnt7me/RgvZWWtrS2ps6ZRfX3nr37q1WvYtrb+qUXVx7a9u2balpS2pr5Y3BTjn65ZdfhKOjo9DX1xdNmjRRe/r38ePHBYBCn+HDh5eatqh0AMSGDRvUKnvUqFHKOltZWYlOnTq9dKAjhGbBzoABA4RMJhN6enpCLpeLPn36qDVWqMDff/8t6tevLwwMDESdOnXE2rVrNarrkSNHBABx+/ZtjdIlJSWJSZMmiRo1aghDQ0NRs2ZNMWfOHJGZmal2Hjt27BA1a9YU+vr6wtbWVkyYMEEkJiYWeW1p7SMvL098/fXXwtbWVhgYGIh27dqJ4OBgtdJu2LChyPNff/11qekLXkUU9Tl+/HiJadPT08WHH34o5HK50NfXFzKZTPTs2VNlwKimfy+eD3ZKSpuWlia8vLyElZWV0NPTEzVq1BDDhw8XkZGRGpW9fv16Ubt2bWFoaCg8PDyUr0LVSfvrr78KIyOjIv/MS0sfExMjRowYIeRyuTA0NBRubm5i2bJlyoGzpaX/8ccfhb29vfLe586dq2y76nyfFNfe1ElbUnsrLX1J7a20tKW1t5f5Hi1ob6WlLa29qVt2Ue1N3bTFtTd10hfX3tRJW1JbK2+S/90gERERkVbimB0iIiLSagx2iIiISKsx2CEiIiKtxmCHiIiItBqDHSIiItJqDHaIiIhIqzHYISIiIq3GYIeIqAgnTpyARCJBYmJiRVeFiF4Rgx0iIiLSagx2iIiISKsx2CGiN5IQAkuXLkXNmjVhZGQEDw8P7Ny5E8B/r5gOHDgADw8PGBoaokWLFggODlbJY9euXXB3d4eBgQGcnJywbNkylfOZmZmYPn06HBwcYGBgABcXF6xfv17lmsuXL6NZs2YwNjaGp6dnod2miejNx2CHiN5Ic+fOxYYNG7B69WqEhIRg8uTJGDp0KAICApTXTJs2Dd9//z0CAwNhbW2Nnj17Ijs7G0B+kNK/f38MHDgQwcHBmD9/PubNm4eNGzcq0w8bNgzbt2/HTz/9hNDQUKxZswZVqlRRqcecOXOwbNkyXLp0Cbq6uhg1atRruX8iKjvcCJSI3jipqamwtLTEv//+i1atWimPjxkzBmlpafjkk0/QsWNHbN++HQMGDAAAPHv2DPb29ti4cSP69++PIUOG4MmTJzh69Kgy/fTp03HgwAGEhITgzp07cHNzg7+/Pzp37lyoDidOnEDHjh1x7NgxdOrUCQBw8OBBdO/eHenp6TA0NCzn3wIRlRX27BDRG+fmzZvIyMhAly5dUKVKFeVn8+bNuHfvnvK65wMhc3NzuLm5ITQ0FAAQGhqK1q1bq+TbunVrhIWFITc3F0FBQZBKpWjfvn2JdWnYsKHy/2UyGQAgLi7ule+RiF4f3YquABHRi/Ly8gAABw4cgJ2dnco5AwMDlYDnRRKJBED+mJ+C/y/wfEe2kZGRWnXR09MrlHdB/YiocmDPDhG9cerVqwcDAwNERkaidu3aKh8HBwfldefPn1f+f0JCAu7cuYM6deoo8zh9+rRKvmfPnoWrqyukUikaNGiAvLw8lTFARKSd2LNDRG8cU1NTTJ06FZMnT0ZeXh7atGmDpKQknD17FlWqVIGjoyMAYMGCBbCwsICNjQ3mzJkDS0tL9O7dGwDw5Zdfonnz5vj2228xYMAAnDt3DitXrsSqVasAAE5OThg+fDhGjRqFn376CR4eHnjw4AHi4uLQv3//irp1IioHDHaI6I307bffwtraGr6+vggPD0e1atXQpEkTzJ49W/kaafHixZg0aRLCwsLg4eGBffv2QV9fHwDQpEkT/Pnnn/jqq6/w7bffQiaTYcGCBRgxYoSyjNWrV2P27NkYP348nj59iho1amD27NkVcbtEVI44G4uIKp2CmVIJCQmoVq1aRVeHiN5wHLNDREREWo3BDhEREWk1vsYiIiIircaeHSIiItJqDHaIiIhIqzHYISIiIq3GYIeIiIi0GoMdIiIi0moMdoiIiEirMdghIiIircZgh4iIiLQagx0iIiLSav8Pap8fE7Joju4AAAAASUVORK5CYII=", "text/plain": [ "
" ] @@ -1474,6 +1474,18 @@ " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", "plt.show()" ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "with open('baseline_exp_set_B_training_metrics.npy', 'wb') as f:\n", + " np.save(f, np.array(epochs_x))\n", + " np.save(f, np.array(epochs_y))\n", + " np.save(f, np.array(epochs_acc))" + ] } ], "metadata": { From 3e5c67d077faec262a719359e214adb01488b19d Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Mon, 29 Apr 2024 20:51:47 +0200 Subject: [PATCH 059/379] nonseq. exp. set B --- .../non-sequential-SCNN-example_3.ipynb | 200 ++++++++++-------- 1 file changed, 106 insertions(+), 94 deletions(-) diff --git a/tests/test_nonsequential/exp_set_B/non-sequential-SCNN-example_3.ipynb b/tests/test_nonsequential/exp_set_B/non-sequential-SCNN-example_3.ipynb index fca63b7d..f8b7f3ff 100644 --- a/tests/test_nonsequential/exp_set_B/non-sequential-SCNN-example_3.ipynb +++ b/tests/test_nonsequential/exp_set_B/non-sequential-SCNN-example_3.ipynb @@ -61,7 +61,7 @@ "metadata": {}, "outputs": [], "source": [ - "root_dir = \"./DVSGESTURE\"\n", + "root_dir = \"../DVSGESTURE\"\n", "_ = DVSGesture(save_to=root_dir, train=True)\n", "_ = DVSGesture(save_to=root_dir, train=False)" ] @@ -368,7 +368,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "12d134e3b89e41888c9c47892b8e6491", + "model_id": "81911e3a7aa94b6299e5943b8f751b65", "version_major": 2, "version_minor": 0 }, @@ -382,7 +382,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "dad714bd3c9c44178729964e549b1349", + "model_id": "29732fa3a1054216b4e25fea82c6522a", "version_major": 2, "version_minor": 0 }, @@ -397,13 +397,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch 0 accuracy: 47.72727272727273\n" + "Epoch 0 accuracy: 45.45454545454545\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "86551db99b2f431fa1808c939bf9c76a", + "model_id": "430bcd18833146398afebad6aea87271", "version_major": 2, "version_minor": 0 }, @@ -417,7 +417,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "e65960569a0f405986462743730225f5", + "model_id": "873fb089042e48f4b0c37f8b8bec90d8", "version_major": 2, "version_minor": 0 }, @@ -432,13 +432,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch 1 accuracy: 58.333333333333336\n" + "Epoch 1 accuracy: 64.01515151515152\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "f494a16eed7847d5a4508cb4229eb5c3", + "model_id": "70f5a16e13a6431ca284b03bdca61fc3", "version_major": 2, "version_minor": 0 }, @@ -452,7 +452,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "df363ac148bc4c11aa6a0946d03cd107", + "model_id": "c15dc70df5274d34b4e4c3f331926968", "version_major": 2, "version_minor": 0 }, @@ -467,13 +467,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch 2 accuracy: 72.72727272727273\n" + "Epoch 2 accuracy: 67.04545454545455\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "fa613935f3a748c7a70b30f5d8b12dce", + "model_id": "4f9e936f30e1449d9461f6349a47c17f", "version_major": 2, "version_minor": 0 }, @@ -487,7 +487,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "76ac8052ee86420d9976c1c4914de629", + "model_id": "c6f9b12606cf469692154bf393218b32", "version_major": 2, "version_minor": 0 }, @@ -502,13 +502,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch 3 accuracy: 78.78787878787878\n" + "Epoch 3 accuracy: 78.03030303030303\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "a58539406c5e44be97c7bde3a77ab9b6", + "model_id": "bf27776a50204852958e29df3e9d6543", "version_major": 2, "version_minor": 0 }, @@ -522,7 +522,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "c8225a8c10f04c37a84c89697de55ce1", + "model_id": "49d66dc3327542feaf78b278c9e3a645", "version_major": 2, "version_minor": 0 }, @@ -537,13 +537,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch 4 accuracy: 80.68181818181817\n" + "Epoch 4 accuracy: 75.37878787878788\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "e308d4881a234b569c8a421c85c7d070", + "model_id": "b8f7857dbae941588fd5167cb87a93ce", "version_major": 2, "version_minor": 0 }, @@ -557,7 +557,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "07ebd6f1b5cb4f2190eed59bd1e7b5e5", + "model_id": "b345336cd5b34f95808cc3603d73a14f", "version_major": 2, "version_minor": 0 }, @@ -572,13 +572,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch 5 accuracy: 78.03030303030303\n" + "Epoch 5 accuracy: 79.54545454545455\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "3d36f57679424fb6b74680a1a3a62e84", + "model_id": "049a3484f53949c695819049a16f979e", "version_major": 2, "version_minor": 0 }, @@ -592,7 +592,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "6e1648bc2958408fadc6bc5ae3aa3256", + "model_id": "87f5865d10d94a91b4f3c71b822e02a4", "version_major": 2, "version_minor": 0 }, @@ -607,13 +607,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch 6 accuracy: 77.27272727272727\n" + "Epoch 6 accuracy: 73.86363636363636\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "a5a767dca24f4545bf84111c5807314d", + "model_id": "2346c5b2b1544887b1c9b781c98efe86", "version_major": 2, "version_minor": 0 }, @@ -627,7 +627,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "672840d4d3af4bc78c75e364c3d69963", + "model_id": "7e380dc6852b4e618a566b4d0b6d455e", "version_major": 2, "version_minor": 0 }, @@ -642,13 +642,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch 7 accuracy: 77.27272727272727\n" + "Epoch 7 accuracy: 78.78787878787878\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "01a26da2bdce4bca9d39dd8984d042b5", + "model_id": "f532bd8971e64de581eee39ca8210d73", "version_major": 2, "version_minor": 0 }, @@ -662,7 +662,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "177fd37e0eab4795bfc59d6bd97a9aad", + "model_id": "57c41939e5ce411e967bdff5706bafff", "version_major": 2, "version_minor": 0 }, @@ -677,13 +677,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch 8 accuracy: 81.81818181818183\n" + "Epoch 8 accuracy: 83.33333333333334\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "afe69feb614442dcbd658ff365af973b", + "model_id": "bb58831f51514b929707c9d00386b00b", "version_major": 2, "version_minor": 0 }, @@ -697,7 +697,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "6444a13035014508bc47550bb1fb3c90", + "model_id": "ae36ee7ef7244204949172e38a1e3c3f", "version_major": 2, "version_minor": 0 }, @@ -712,13 +712,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch 9 accuracy: 76.51515151515152\n" + "Epoch 9 accuracy: 78.03030303030303\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "4f36db6994a540d4ac158065ab0b1f7b", + "model_id": "ba081ddbf96044b4866a88375bb5d132", "version_major": 2, "version_minor": 0 }, @@ -732,7 +732,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "a786978ce8ba4dd282ff8428e1491a48", + "model_id": "b9b2152aa90d4e03b864485cc8c4a3cd", "version_major": 2, "version_minor": 0 }, @@ -747,13 +747,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch 10 accuracy: 81.06060606060606\n" + "Epoch 10 accuracy: 73.10606060606061\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "26b4d34e0f344e81b33d43fc1b5a86ef", + "model_id": "673fbc9bfb724ac6bd92e17b1d305271", "version_major": 2, "version_minor": 0 }, @@ -767,7 +767,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "67d64e70242e488b8d5bf13fe1e9f1b3", + "model_id": "023886a9985e4061888248b705b378db", "version_major": 2, "version_minor": 0 }, @@ -782,13 +782,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch 11 accuracy: 84.84848484848484\n" + "Epoch 11 accuracy: 80.68181818181817\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "663e7f38eb8a4ed0b1f6cde5dfb3d431", + "model_id": "37c001f3de6f4456aadc89fc0a8fd30f", "version_major": 2, "version_minor": 0 }, @@ -802,7 +802,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "f3f4a084b59348d18341497916f46b9c", + "model_id": "485e07e924f74595b8dc6e16fd39c074", "version_major": 2, "version_minor": 0 }, @@ -817,13 +817,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch 12 accuracy: 77.65151515151516\n" + "Epoch 12 accuracy: 76.51515151515152\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "478eb58da5454eed9de461a91107846f", + "model_id": "bab78a6c9a4b409ea3c346e5cc3bc670", "version_major": 2, "version_minor": 0 }, @@ -837,7 +837,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "45c6cb33606b46bc877c437fb8cd5754", + "model_id": "9c65309f65e9405a9654b875925a3dec", "version_major": 2, "version_minor": 0 }, @@ -852,13 +852,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch 13 accuracy: 84.84848484848484\n" + "Epoch 13 accuracy: 75.75757575757575\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "dadbf061a1f14ded927d6935b9935596", + "model_id": "d6697b3a3060422bbdcb33a6c4bcbc90", "version_major": 2, "version_minor": 0 }, @@ -872,7 +872,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "c4aaedbf57d9415abe757bac082f1e34", + "model_id": "0e4cb4adf5854ccda0c0f026af6f7f44", "version_major": 2, "version_minor": 0 }, @@ -887,13 +887,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch 14 accuracy: 82.57575757575758\n" + "Epoch 14 accuracy: 83.71212121212122\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "a7603a6174414132b23c65a9942b5a2d", + "model_id": "7986c2025d084d49b7531930bcdcf1ac", "version_major": 2, "version_minor": 0 }, @@ -907,7 +907,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "5e6ee57580a9484d86bdb93c6632d3bb", + "model_id": "f711f2ad3906490b8a069def1f0d8aa6", "version_major": 2, "version_minor": 0 }, @@ -922,13 +922,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch 15 accuracy: 83.71212121212122\n" + "Epoch 15 accuracy: 79.92424242424242\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "f9c3cd8d3662410e9cb717a589f2fe37", + "model_id": "f3d39c70649244ed927d03f212e79036", "version_major": 2, "version_minor": 0 }, @@ -942,7 +942,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "eaa3334ff93c4ca88c821742d8318b0d", + "model_id": "d5a750a56d5d4fe0a5b21b07e0b3a9c1", "version_major": 2, "version_minor": 0 }, @@ -957,13 +957,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch 16 accuracy: 80.3030303030303\n" + "Epoch 16 accuracy: 82.95454545454545\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "c4b3a84a81d248eb9114745ad67d140e", + "model_id": "a6fd6dd2aa424987bffa1cca283014b4", "version_major": 2, "version_minor": 0 }, @@ -977,7 +977,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "ec719eac2f0f45cdafa00a2a0dbac285", + "model_id": "a2657ea1fcd542adbb2bea6f0b261aa5", "version_major": 2, "version_minor": 0 }, @@ -992,13 +992,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch 17 accuracy: 84.0909090909091\n" + "Epoch 17 accuracy: 80.68181818181817\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "27fd1b40937745938d996b9cf53d8752", + "model_id": "b58214f17d2b494ebf5a6eecd6417ea9", "version_major": 2, "version_minor": 0 }, @@ -1012,7 +1012,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "296f1b9cdf6a44f491c8387d7570098d", + "model_id": "4e9a70fb6d3945b2a57baeff27a7b4ee", "version_major": 2, "version_minor": 0 }, @@ -1027,13 +1027,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch 18 accuracy: 78.78787878787878\n" + "Epoch 18 accuracy: 81.06060606060606\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "d9bde1c861fc4e14a8bad10a665baeb4", + "model_id": "b37e5e3275e94ea188db474000c088de", "version_major": 2, "version_minor": 0 }, @@ -1047,7 +1047,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "c44e170962a546c3a52d014dc50586f2", + "model_id": "ea639c48291b4c4d86d4174cf7a46ee8", "version_major": 2, "version_minor": 0 }, @@ -1062,13 +1062,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch 19 accuracy: 85.60606060606061\n" + "Epoch 19 accuracy: 81.06060606060606\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "7aa75420aaf2498db249299e4d5f7225", + "model_id": "ae33f1c03d7a4971977e57e996fe953a", "version_major": 2, "version_minor": 0 }, @@ -1082,7 +1082,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "e1208e706a4345c3ad192826666ac541", + "model_id": "25c9fb2de9ff4d6cbab6836f3fde7b1c", "version_major": 2, "version_minor": 0 }, @@ -1097,13 +1097,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch 20 accuracy: 85.22727272727273\n" + "Epoch 20 accuracy: 80.68181818181817\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "d93ae0e2ee434f43a2becdd5424c82c6", + "model_id": "6b11079d52a34cda84e1e19b1d0f24aa", "version_major": 2, "version_minor": 0 }, @@ -1117,7 +1117,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "0584bda6877440e596252828c72380b6", + "model_id": "d82fa88bec2e4acd980548de34d7b003", "version_major": 2, "version_minor": 0 }, @@ -1132,13 +1132,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch 21 accuracy: 87.87878787878788\n" + "Epoch 21 accuracy: 80.68181818181817\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "6786bd222f394804b8678fd2cf372b8d", + "model_id": "003db61c67e64c58a210a174eb3678a0", "version_major": 2, "version_minor": 0 }, @@ -1152,7 +1152,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "e62bb84b7eb64590a10618e33c315f39", + "model_id": "4cd63dd1446f4bf8b74516eae948b2ef", "version_major": 2, "version_minor": 0 }, @@ -1167,13 +1167,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch 22 accuracy: 82.57575757575758\n" + "Epoch 22 accuracy: 79.54545454545455\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "e91fc0a219b9465fbbeee66fed5c1af7", + "model_id": "d1218d30f7b147e8902d28ec5bcbee82", "version_major": 2, "version_minor": 0 }, @@ -1187,7 +1187,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "9cbcf2ab593f4d55a38e5e0aeea42153", + "model_id": "ef9faf3fff9a478a8dfb167ba8e3e197", "version_major": 2, "version_minor": 0 }, @@ -1202,13 +1202,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch 23 accuracy: 85.22727272727273\n" + "Epoch 23 accuracy: 84.0909090909091\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "0764987324f44311a62ca84098a845e5", + "model_id": "c2bf97036cb94c32a231d4fccc554388", "version_major": 2, "version_minor": 0 }, @@ -1222,7 +1222,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "ca02cdc810544e20b359a46e6ad34c03", + "model_id": "529cf2f7739540a2a275bbae700fd116", "version_major": 2, "version_minor": 0 }, @@ -1237,13 +1237,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch 24 accuracy: 87.5\n" + "Epoch 24 accuracy: 80.3030303030303\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "6c8d128578a644b5b33e66f5b45b9da6", + "model_id": "f35036ca021c4912b3be0429d105d3ff", "version_major": 2, "version_minor": 0 }, @@ -1257,7 +1257,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "175dfd50e03d4820a86a26654e452e23", + "model_id": "abb86443efe54497957c0ef13a6a3d13", "version_major": 2, "version_minor": 0 }, @@ -1272,13 +1272,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch 25 accuracy: 85.98484848484848\n" + "Epoch 25 accuracy: 85.60606060606061\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "7ae02c2119c94d1c948977c84da72131", + "model_id": "77357f17c8564c0e9d8a400a8d3557dd", "version_major": 2, "version_minor": 0 }, @@ -1292,7 +1292,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "d53b3f9ea091486ea258116c4be41ffa", + "model_id": "71703b46583d46f7a4234af64dfc4714", "version_major": 2, "version_minor": 0 }, @@ -1313,7 +1313,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "0581d34ef4554b1ab8d81c357ad161fe", + "model_id": "22ca37f732c743c3b46da79541192205", "version_major": 2, "version_minor": 0 }, @@ -1327,7 +1327,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "8d41d94535f64c1b993a6af42c7a879c", + "model_id": "7304b54df0b649deb2b85227bc679dca", "version_major": 2, "version_minor": 0 }, @@ -1342,13 +1342,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch 27 accuracy: 86.74242424242425\n" + "Epoch 27 accuracy: 82.95454545454545\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "0af51891f0764f4dada46d6a2aa36bfe", + "model_id": "8474b8a943af43c48e9346a88c467457", "version_major": 2, "version_minor": 0 }, @@ -1362,7 +1362,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "fe6e620344bd489396ca5b47e1181c6f", + "model_id": "16ba56c35b18468c8755fc8c9da003f2", "version_major": 2, "version_minor": 0 }, @@ -1377,13 +1377,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch 28 accuracy: 87.12121212121212\n" + "Epoch 28 accuracy: 81.43939393939394\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "e5f2eebd7e474c6885747117699c2410", + "model_id": "99fa082a72ad4ada8cc2a84c6ed79fbc", "version_major": 2, "version_minor": 0 }, @@ -1397,7 +1397,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "909798a1d58441b38bd980e971638e36", + "model_id": "45dad1191d42469f9eca6f7167dae4e8", "version_major": 2, "version_minor": 0 }, @@ -1412,7 +1412,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch 29 accuracy: 86.36363636363636\n" + "Epoch 29 accuracy: 81.43939393939394\n" ] } ], @@ -1422,12 +1422,12 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 16, "metadata": {}, "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAGwCAYAAABVdURTAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABllklEQVR4nO3deVhUZf8G8PvMsCMgyC4gKO4LKiKBa4pomYr2qpmFptVbaVlWbpn25q9QyzLT3Mqtci2X0lIRcw13cBdxxYVVZdjXeX5/IKMj2wzOMCz357rmKg7nmfMdHIfb5zyLJIQQICIiIqolZIYugIiIiEiXGG6IiIioVmG4ISIiolqF4YaIiIhqFYYbIiIiqlUYboiIiKhWYbghIiKiWsXI0AVUNaVSibt378LKygqSJBm6HCIiItKAEALp6elwdXWFTFZ+30ydCzd3796Fu7u7ocsgIiKiSrh16xbc3NzKPafOhRsrKysART8ca2trA1dDREREmkhLS4O7u7vq93h56ly4Kb4VZW1tzXBDRERUw2gypIQDiomIiKhWYbghIiKiWoXhhoiIiGoVhpsyhIWFwc/PD1ZWVnB0dERISAhiYmI0br9+/XpIkoSQkBC145Iklfr46quvdPwKiIiI6iaGmzLs378f48aNw5EjRxAeHo78/HwEBwcjMzOzwrY3btzARx99hG7dupX4Xnx8vNpjxYoVkCQJL774oj5eBhERUZ0jCSGEoYuoSmlpabCxsYFCodBqtlRycjIcHR2xf/9+dO/evczzCgsL0b17d4wZMwYHDx5Eamoqtm7dWub5ISEhSE9PR0REhDYvg4iIqE7R5vc3e240pFAoAAB2dnblnvf555/D0dERY8eOrfA5ExMTsWPHDo3OJSIiIs0YNNxUZlzLqlWrSoxXMTMz02udSqUS77//Prp06YI2bdqUed6hQ4fw008/Yfny5Ro97+rVq2FlZYUhQ4boqlQiIqI6z6CL+BWPa/Hz80NBQQGmTZuG4OBgXLhwAZaWlmW2s7a2VgtB+t4jaty4cTh37hwOHTpU5jnp6el49dVXsXz5ctjb22v0vCtWrMDIkSP1Hs6IiIjqEoOGm507d6p9vWrVKjg6OuLkyZPljmuRJAnOzs4aXSM3Nxe5ubmqr9PS0rSqcfz48di+fTsOHDhQ7l4WV69exY0bNzBgwADVMaVSCQAwMjJCTEwMmjRpovrewYMHERMTgw0bNmhVDxEREZWvWo250XRcS0ZGBho1agR3d3cMGjQI58+fL/PcsLAw2NjYqB6abpophMD48eOxZcsW7N27F15eXuWe36JFC5w9exbR0dGqx8CBA/Hss88iOjq6xHV/+ukn+Pr6wsfHR6N6iIiISDPVZraUUqnEwIEDkZqaWu7tn8jISMTGxqJdu3ZQKBT4+uuvceDAAZw/f77UnpXSem7c3d0rHG39zjvvYO3atdi2bRuaN2+uOm5jYwNzc3MAQGhoKBo2bIiwsLBSn2P06NGlzpZKS0uDi4sL5s2bh7feeqvMGoiIiKiINrOlqs3GmZqMawGAgIAABAQEqL4ODAxEy5YtsXTpUsyaNavE+aampjA1NdW6nsWLFwMAevbsqXZ85cqVGD16NAAgLi4OMtmjzq94RTaup2TCy94SLjbmZT73+vXrIYTAiBEjtK6LiIiIylctem7Gjx+Pbdu24cCBAxXe/inN0KFDYWRkhHXr1lV4bmXXuanIikPXMWvHBQgByCQgbEhbDPfz0NnzExER1WU1Zp0bbce1lKawsBBnz56Fi4uLHirUTLwiWxVsAEApgGmbzyFekW2wmoiIiOoqg4abcePG4ZdffsHatWthZWWFhIQEJCQkIDv7USgIDQ3F1KlTVV9//vnn2L17N65du4ZTp07hlVdewc2bN/H6668b4iUAAK6nZOLJ/q9CIXAjJcswBREREdVhBh1zU5lxLQ8ePMAbb7yBhIQE2NrawtfXF//++y9atWpVVWWX4GVvCZlU1GNTTC5J8LS3MFhNREREdVW1GHNTlfQ15mbD8ThM/v0sAI65ISIi0rUaM+amNhnu5wG3+kUrDS8c0YHBhoiIyEAYbnTI1bboNlRBneoLIyIiql4YbnTI0apoPZ2ktBwDV0JERFR3MdzokKNV0W2p5PTcCs4kIiIifWG40SEn64c9Nww3REREBsNwo0OOD8NNIm9LERERGQzDjQ4V35Zizw0REZHhMNzokOq2FHtuiIiIDIbhRoccHvbcpOUUICe/0MDVEBER1U0MNzpkbWYEU6OiH2lSGm9NERERGQLDjQ5JkgQn6+JxN7w1RUREZAgMNzpWvJBfIntuiIiIDILhRsccVWvdsOeGiIjIEBhudIzTwYmIiAyL4UbHVD03vC1FRERkEAw3Ovao54a3pYiIiAyB4UbHHu0Mzp4bIiIiQ2C40TFOBSciIjIshhsdK+65eZCVj9wCrlJMRERU1RhudKy+hTFM5EU/1mTOmCIiIqpyDDc6JkkSHIrH3TDcEBERVTmGGz1w5O7gREREBsNwoweO7LkhIiIyGIYbPVCtdcPp4ERERFWO4UYPnLi/FBERkcEw3OhBcc8NdwYnIiKqegw3euBgzTE3REREhsJwowdOD3tuknlbioiIqMox3OhB8VTwlIw85BcqDVwNERFR3cJwowd2FiYwkkkAgJQM3poiIiKqSgw3eiCTPbZKMQcVExERVSmGGz0pXsgvkasUExERVSmGGz1xKF7IjzOmiIiIqhTDjZ44cTo4ERGRQTDc6Ikjp4MTEREZBMONnhRPB+cqxURERFWL4UZPHu0Mzp4bIiKiqsRwoydO1twZnIiIyBAYbvSkuOcmJSMXhUph4GqIiIjqDoYbPWlQzxQyCVAK4B5XKSYiIqoyDDd6IpdJsK/H6eBERERVjeFGjx7NmOKgYiIioqrCcKNHjlylmIiIqMox3OiRapVizpgiIiKqMgw3evRofyneliIiIqoqDDd69GhncPbcEBERVRWGGz0qDjfcX4qIiKjqMNzokWqVYg4oJiIiqjIMN3pUPBU8OT0XSq5STEREVCUYbvTIvp4pJAkoUArcz8ozdDlERER1AsONHhnLZWhgaQKA08GJiIiqCsONnhVPB0/koGIiIqIqwXCjZ6oZU+y5ISIiqhIMN3qmWqWYPTdERERVguFGz4r3l+JCfkRERFWD4UbPHNlzQ0REVKUYbvSMO4MTERFVLYYbPXPkzuBERERViuFGzx7tL5ULIbhKMRERkb4x3OiZw8Nwk1eoRGpWvoGrISIiqv0MGm7CwsLg5+cHKysrODo6IiQkBDExMRW227RpE1q0aAEzMzO0bdsWf/31VxVUWzmmRnLYWhgD4LgbIiKiqmDQcLN//36MGzcOR44cQXh4OPLz8xEcHIzMzMwy2/z7778YMWIExo4di6ioKISEhCAkJATnzp2rwsq182g6OGdMERER6ZskqtFAkOTkZDg6OmL//v3o3r17qecMHz4cmZmZ2L59u+rYM888g/bt22PJkiUVXiMtLQ02NjZQKBSwtrbWWe3lefWnozgYm4Kvh/rgP75uVXJNIiKi2kSb39/VasyNQqEAANjZ2ZV5TmRkJIKCgtSO9e3bF5GRkaWen5ubi7S0NLVHVXs0HZw9N0RERPpWbcKNUqnE+++/jy5duqBNmzZlnpeQkAAnJye1Y05OTkhISCj1/LCwMNjY2Kge7u7uOq1bE5wOTkREVHWqTbgZN24czp07h/Xr1+v0eadOnQqFQqF63Lp1S6fPr4ni6eDsuSEiItI/I0MXAADjx4/H9u3bceDAAbi5lT8mxdnZGYmJiWrHEhMT4ezsXOr5pqamMDU11VmtleFk/fC2FHtuiIiI9M6gPTdCCIwfPx5btmzB3r174eXlVWGbgIAAREREqB0LDw9HQECAvsp8ao96bhhuiIiI9M2gPTfjxo3D2rVrsW3bNlhZWanGzdjY2MDc3BwAEBoaioYNGyIsLAwAMGHCBPTo0QPz5s1D//79sX79epw4cQLLli0z2OuoyONTwYUQkCTJwBURERHVXgbtuVm8eDEUCgV69uwJFxcX1WPDhg2qc+Li4hAfH6/6OjAwEGvXrsWyZcvg4+OD3377DVu3bi13ELKhFQ8ozi1QIi2nwMDVEBER1W4G7bnRZImdffv2lTg2dOhQDB06VA8V6YeZsRzWZkZIyylAcnoObMyNDV0SERFRrVVtZkvVdo7WxbemOO6GiIhInxhuqgingxMREVUNhpsqwungREREVYPhpooU99zwthQREZF+MdxUEQfeliIiIqoSDDdVRHVbigv5ERER6RXDTRUpvi2VzHBDRESkVww3VeTRVHDeliIiItInhpsqUtxzk5VXiIxcrlJMRESkLww3VcTS1Aj1TIsWhE5i7w0REZHeMNxUIU4HJyIi0j+GmyrE6eBERET6x3BThYqng3PGFBERkf4w3FShR7el2HNDRESkLww3VcjRuvi2FHtuiIiI9IXhpgpx80wiIiL9Y7ipQhxQTEREpH8MN1XI0Yo9N0RERPrGcFOFnB6OuUnPLUB2XqGBqyEiIqqdGG6qUD1TI5gbywHw1hQREZG+MNxUIUmSVDOmuEoxERGRfjDcVDGn4nE37LkhIiLSC4abKuZQvNYNe26IiIj0guGmiqlWKWbPDRERkV4w3FSx4ungyey5ISIi0guGmyrmxC0YiIiI9Irhpoo5ckAxERGRXjHcVDFOBSciItIvhpsqVjwVXJGdj5x8rlJMRESkaww3Vcza3AgmRkU/9mSOuyEiItI5hpsqJkmSajo4x90QERHpHsONAThZc3dwIiIifWG4MYBHPTcMN0RERLrGcGMAqlWK03hbioiISNcYbgzAsfi2FHtuiIiIdI7hxgB4W4qIiEh/GG4MQNVzw9tSREREOsdwYwDsuSEiItIfhhsDKJ4Kfj8zD3kFSgNXQ0REVLsw3BiArYUxjOUSACAlg703REREusRwYwCSJMGhHqeDExER6QPDjYFwOjgREZF+MNwYCAcVExER6QfDjYE4Wj8MN7wtRUREpFMMNwbiZMXNM4mIiPSB4cZAVD036ey5ISIi0iWGGwNxtOKAYiIiIn1guDEQB9XO4Aw3REREusRwYyDFqxTfy8xFQSFXKSYiItIVhhsDaWBpArlMghDAvcw8Q5dDRERUazDcGIhMJsG+ngkArlJMRESkS1qHm+zsbGRlZam+vnnzJubPn4/du3frtLC6oPjWFKeDExER6Y7W4WbQoEFYs2YNACA1NRX+/v6YN28eBg0ahMWLF+u8wNqMqxQTERHpntbh5tSpU+jWrRsA4LfffoOTkxNu3ryJNWvWYMGCBTovsDZzeDgdnLeliIiIdEfrcJOVlQUrKysAwO7duzFkyBDIZDI888wzuHnzps4LrM2crNlzQ0REpGtahxtvb29s3boVt27dwq5duxAcHAwASEpKgrW1tc4LrM2KF/JL5irFREREOqN1uJkxYwY++ugjeHp6wt/fHwEBAQCKenE6dOig8wJrM465ISIi0j0jbRv85z//QdeuXREfHw8fHx/V8d69e2Pw4ME6La62K54txTE3REREuqN1uAEAZ2dnODs7AwDS0tKwd+9eNG/eHC1atNBpcbVd8eaZKRl5KFQKyGWSgSsiIiKq+bS+LTVs2DAsXLgQQNGaN506dcKwYcPQrl07/P777zovsDZrYGkCSQIKlQL3uUoxERGRTmgdbg4cOKCaCr5lyxYIIZCamooFCxbg//7v/7R+rgEDBsDV1RWSJGHr1q3lnr9v3z5IklTikZCQoO3LqBaM5DI0sCzeQJO3poiIiHRB63CjUChgZ2cHANi5cydefPFFWFhYoH///oiNjdXquTIzM+Hj44NFixZp1S4mJgbx8fGqh6Ojo1btq5Pi6eDJHFRMRESkE1qPuXF3d0dkZCTs7Oywc+dOrF+/HgDw4MEDmJmZafVczz33HJ577jltS4CjoyPq16+vdbvqyNHKFOcBJHE6OBERkU5o3XPz/vvvY+TIkXBzc4Orqyt69uwJoOgWU9u2bXVdX6nat28PFxcX9OnTB4cPHy733NzcXKSlpak9qhNH1SrF7LkhIiLSBa3DzTvvvIPIyEisWLEChw4dgkxW9BSNGzfWesyNtlxcXLBkyRL8/vvv+P333+Hu7o6ePXvi1KlTZbYJCwuDjY2N6uHu7q7XGrX1aJVi9twQERHpgiSEEJVtXNxUkp5+CrMkSdiyZQtCQkK0atejRw94eHjg559/LvX7ubm5yM191CuSlpYGd3d3KBSKarGi8s9HbuLTrecQ3MoJy0I7GbocIiKiaiktLQ02NjYa/f7WuucGANasWYO2bdvC3Nwc5ubmaNeuXZnhQt86d+6MK1eulPl9U1NTWFtbqz2qE65STEREpFtaDyj+5ptv8Omnn2L8+PHo0qULAODQoUN46623kJKSgg8++EDnRZYnOjoaLi4uVXpNXSpepTiJU8GJiIh0Qutw8/3332Px4sUIDQ1VHRs4cCBat26Nzz77TKtwk5GRodbrcv36dURHR8POzg4eHh6YOnUq7ty5gzVr1gAA5s+fDy8vL7Ru3Ro5OTn48ccfsXfvXuzevVvbl1FtFPfcJGfkQgihk1t8REREdZnW4SY+Ph6BgYEljgcGBiI+Pl6r5zpx4gSeffZZ1dcTJ04EAIwaNQqrVq1CfHw84uLiVN/Py8vDhx9+iDt37sDCwgLt2rXDnj171J6jprGvVxRu8gsFHmTlw87SxMAVERER1Wxahxtvb29s3LgR06ZNUzu+YcMGNG3aVKvn6tmzJ8obz7xq1Sq1rydNmoRJkyZpdY3qzsRIBjtLE9zPzENiWg7DDRER0VPSOtz873//w/Dhw3HgwAHVmJvDhw8jIiICGzdu1HmBdYGjlSnuZ+YhKT0XLWvu8CEiIqJqQevZUi+++CKOHj0Ke3t7bN26FVu3boW9vT2OHTuGwYMH66PGWs+Rg4qJiIh0RuueGwDw9fXFL7/8outa6ixOByciItIdjcKNNlsWVLd1ZGoC1SrF7LkhIiJ6ahqFm/r161c4Rbl4GnNhYaFOCqtLiveXYs8NERHR09Mo3Pzzzz/6rqNO420pIiIi3dEo3PTo0UPfddRpxQOKE3lbioiI6KlVam8p0q3He26eYh9TIiIiAsNNteDwMNzkFSiRll1g4GqIiIhqNoabasDMWA5rs6I7hOfuKgxcDRERUc3GcFMNbDgeh7Scoh6bV346ig3H4ypoQURERGWpVLgpKCjAnj17sHTpUqSnpwMA7t69i4yMDJ0WVxfEK7IxdfNZ1ddCANM2n0O8ItuAVREREdVcWq9QfPPmTfTr1w9xcXHIzc1Fnz59YGVlhTlz5iA3NxdLlizRR5211vWUTCifGENcKARupGTBxcbcMEURERHVYFr33EyYMAGdOnXCgwcPYG7+6Jfv4MGDERERodPi6gIve0vInlgfUS5J8LS3MExBRERENZzW4ebgwYOYPn06TExM1I57enrizp07OiusrnCxMUfYkLZqAWdicDP22hAREVWS1uFGqVSWusXC7du3YWVlpZOi6prhfh44PKUXWrkU7ctVz7RS+5kSERERKhFugoODMX/+fNXXkiQhIyMDM2fOxPPPP6/L2uoUFxtzDPBxBQDsv5xs4GqIiIhqLq3Dzbx583D48GG0atUKOTk5ePnll1W3pObMmaOPGuuMHs0cAACRV+8ht4AbkBIREVWG1vc/3NzccPr0aaxfvx5nzpxBRkYGxo4di5EjR6oNMCbttXSxgoOVKZLTc3HixgN08bY3dElEREQ1TqUGdxgZGeGVV17RdS11niRJ6N7UAb+fuo39l5MZboiIiCpB63Dzxx9/lHpckiSYmZnB29sbXl5eT11YXdWjeVG4OXA5GdOeb2nocoiIiGocrcNNSEgIJEkqsXt18TFJktC1a1ds3boVtra2Oiu0rujmbQ9JAi4lpCNBkQNnGzNDl0RERFSjaD2gODw8HH5+fggPD4dCoYBCoUB4eDj8/f2xfft2HDhwAPfu3cNHH32kj3prPVtLE7Rzqw8AOMBZU0RERFrTuudmwoQJWLZsGQIDA1XHevfuDTMzM7z55ps4f/485s+fjzFjxui00LqkRzMHnL6Viv2xyRjm527ocoiIiGoUrXturl69Cmtr6xLHra2tce3aNQBA06ZNkZKS8vTV1VHFU8IPxaagoFBp4GqIiIhqFq3Dja+vLz7++GMkJz+6ZZKcnIxJkybBz88PABAbGwt3d/Y4VJaPmw2szYygyM7H6dsKQ5dDRERUo2gdbn766Sdcv34dbm5u8Pb2hre3N9zc3HDjxg38+OOPAICMjAxMnz5d58XWFUZyGbo1Leq94bgbIiIi7Wg95qZ58+a4cOECdu/ejcuXL6uO9enTBzJZUVYKCQnRaZF1UY9mDthxNh77Lyfjgz7NDF0OERFRjVGpRfxkMhn69euHfv366boeeqhbs6IF/E7fTsWDzDzYWppU0IKIiIiASoabzMxM7N+/H3FxccjLy1P73nvvvaeTwuo6FxtzNHeyQkxiOg5dSVFtqklERETl0zrcREVF4fnnn0dWVhYyMzNhZ2eHlJQUWFhYwNHRkeFGh7o3s0dMYjr2X05muCEiItKQ1gOKP/jgAwwYMAAPHjyAubk5jhw5gps3b8LX1xdff/21Pmqss3o0cwRQNKj4yRWhiYiIqHRah5vo6Gh8+OGHkMlkkMvlyM3Nhbu7O+bOnYtp06bpo8Y6q5OnLcyN5UhKz8WlhHRDl0NERFQjaB1ujI2NVbOiHB0dERcXBwCwsbHBrVu3dFtdHWdmLMczje0AAPs5JZyIiEgjWoebDh064Pjx4wCAHj16YMaMGfj111/x/vvvo02bNjovsK4rXq2Y690QERFpRutw8+WXX8LFxQUA8MUXX8DW1hZvv/02kpOTsWzZMp0XWNf1aF407ub4jfvIzC0wcDVERETVn1azpYQQcHR0VPXQODo6YufOnXopjIp4NrCAu505bt3PRuTVewhq5WTokoiIiKo1rXpuhBDw9vbm2JoqJEnSo1tTsbw1RUREVBGtwo1MJkPTpk1x7949fdVDpSieEs5BxURERBXTeszN7Nmz8fHHH+PcuXP6qIdKEdCkAYxkEm7ey8KNlExDl0NERFStab1CcWhoKLKysuDj4wMTExOYm5urff/+/fs6K46K1DM1QidPWxy5dh8HYpPhaW9p6JKIiIiqLa3Dzfz58/VQBlWkRzNHHLl2H/tjkhEa4GnocoiIiKotrcPNqFGj9FEHVaB7M3vM2Qn8e/UecgsKYWokN3RJRERE1ZLWY24A4OrVq5g+fTpGjBiBpKQkAMDff/+N8+fP67Q4eqSVizUcrEyRnV+IkzceGLocIiKiakvrcLN//360bdsWR48exebNm5GRkQEAOH36NGbOnKnzAqmIJEno3rRoSjhnTREREZVN63AzZcoU/N///R/Cw8NhYmKiOt6rVy8cOXJEp8WRuu7N7AEw3BAREZVH63Bz9uxZDB48uMRxR0dHpKSk6KQoKl23pg6QJOBSQjoS03IMXQ4REVG1pHW4qV+/PuLj40scj4qKQsOGDXVSFJXOztIE7dzqA2DvDRERUVm0DjcvvfQSJk+ejISEBEiSBKVSicOHD+Ojjz5CaGioPmqkx/RoyltTRERE5anUruAtWrSAu7s7MjIy0KpVK3Tv3h2BgYGYPn26Pmqkx/RoXjSo+FBsCgqVwsDVEBERVT9ar3NjYmKC5cuX49NPP8W5c+eQkZGBDh06oGnTpvqoj57g41YfVmZGUGTn4/TtVHT0sDV0SURERNWK1uHm0KFD6Nq1Kzw8PODh4aGPmqgcRnIZujW1x19nE7A/JpnhhoiI6Ala35bq1asXvLy8MG3aNFy4cEEfNVEFejQrujV1IJbjboiIiJ6kdbi5e/cuPvzwQ+zfvx9t2rRB+/bt8dVXX+H27dv6qI9K0f1huDl9KxUPMvMMXA0REVH1onW4sbe3x/jx43H48GFcvXoVQ4cOxerVq+Hp6YlevXrpo0Z6gouNOZo51YNSAIeucG0hIiKix1Vqb6liXl5emDJlCmbPno22bdti//79uqqLKqC6NcUp4URERGoqHW4OHz6Md955By4uLnj55ZfRpk0b7NixQ5e1UTmKb03tv5wMITglnIiIqJjWs6WmTp2K9evX4+7du+jTpw++++47DBo0CBYWFvqoj8rg52kHM2MZktJzcSkhHS1drA1dEhERUbWgdbg5cOAAPv74YwwbNgz29vb6qIk0YGYsR0DjBvgnJhkHLicz3BARET2kdbg5fPiwPuqgSujezAH/xCRj/+Vk/LdHE0OXQ0REVC1oHW6KXbhwAXFxccjLU5+KPHDgwKcuijRTPKj4+I37yMwtgKVppf84iYiIag2tBxRfu3YNPj4+aNOmDfr374+QkBCEhIRg8ODBGDx4sFbPdeDAAQwYMACurq6QJAlbt26tsM2+ffvQsWNHmJqawtvbG6tWrdL2JdQaXvaWcLczR36hwJFr9wxdDhERUbWgdbiZMGECvLy8kJSUBAsLC5w/fx4HDhxAp06dsG/fPq2eKzMzEz4+Pli0aJFG51+/fh39+/fHs88+i+joaLz//vt4/fXXsWvXLm1fRq0gSRK6N300a4qIiIgqcVsqMjISe/fuhb29PWQyGWQyGbp27YqwsDC89957iIqK0vi5nnvuOTz33HMan79kyRJ4eXlh3rx5AICWLVvi0KFD+Pbbb9G3b99S2+Tm5iI3N1f1dVpamsbXqwl6NHPAr0fjGG6IiIge0rrnprCwEFZWVgCKViu+e/cuAKBRo0aIiYnRbXVPiIyMRFBQkNqxvn37IjIyssw2YWFhsLGxUT3c3d31WmNVC/S2h5FMws17WbiRkmnocoiIiAxO63DTpk0bnD59GgDg7++PuXPn4vDhw/j888/RuHFjnRf4uISEBDg5Oakdc3JyQlpaGrKzs0ttM3XqVCgUCtXj1q1beq2xqtUzNYJvo6KdwVf9ewPxitJ/DkRERHWF1relpk+fjszMoh6Czz//HC+88AK6deuGBg0aYMOGDTov8GmZmprC1NTU0GXola2lMYCicLMm8gbChrTFcD8PA1dFRERkGFqHm8fHtnh7e+PSpUu4f/8+bG1tIUmSTot7krOzMxITE9WOJSYmwtraGubm5nq9dnUVr8jGrvOPfiZKAUzbfA7dmznAxaZu/kyIiKhue6qNM4vZ2dnpPdgAQEBAACIiItSOhYeHIyAgQO/Xrq6up2Tiya2lCoXAjZQswxRERERkYDoJN5WVkZGB6OhoREdHAyia6h0dHY24uDgAReNlQkNDVee/9dZbuHbtGiZNmoRLly7hhx9+wMaNG/HBBx8YovxqwcveErIncqVckuBpz72+iIiobjJouDlx4gQ6dOiADh06AAAmTpyIDh06YMaMGQCA+Ph4VdABAC8vL+zYsQPh4eHw8fHBvHnz8OOPP5Y5DbwucLExR9iQtmoBZ+rzLXhLioiI6ixJiCdvatRuaWlpsLGxgUKhgLV17dlsMl6RjVeWH8XVlEx8NqAVRnfxMnRJREREOqPN72+D9tyQ7rjYmGPkM40AAFuj7xq4GiIiIsNhuKlFBvi4Qi6TEH0rFde5oB8REdVRDDe1iIOVKbp42wMAtkbdMXA1REREhsFwU8sM7uAKANgWfQd1bDgVERERAIabWie4lTPMjeW4cS8L0bdSDV0OERFRlWO4qWUsTY0Q3Lpo/61tHFhMRER1EMNNLRTSviEA4M/Td5FfqDRwNURERFWL4aYW6trUHg0sTXAvMw+HrqQYuhwiIqIqxXBTCxnLZXihnQsAzpoiIqK6h+GmlgrpUHRravf5RGTmFhi4GiIioqrDcFNLtXevj0YNLJCdX4jdFxIMXQ4REVGVYbippSRJUg0s3hrFWVNERFR3MNzUYsW3pg7GJiM5PdfA1RAREVUNhptazMveEj7u9aEUwPYz7L0hIqK6geGmlhvcvmg7Bs6aIiKiuoLhppZ74eFO4advK3AtOcPQ5RAREekdw00tZ1/PFN2aPtwpnNsxEBFRHcBwUwcUz5riTuFERFQXMNzUAcGtnWBhIsfNe1mI4k7hRERUyzHc1AEWJkYIbvVwp3AOLCYiolqO4aaOKF7z5s8z8dwpnIiIajWGmzqiq7c97OuZ4H5mHg7GJhu6HCIiIr1huKkjjOQyvNCueM0bzpoiIqLai+GmDlHtFH4hARncKZyIiGophps6xMfNBl72lsjJV2L3ee4UTkREtRPDTR3y+E7hWzhrioiIaimGmzpm0MO9pg5fSUFSeo6BqyEiItI9hps6xtPeEh08inYK//N0vKHLISIi0jmGmzro8e0YiIiIahuGmzrohXYukMsknLmtwFXuFE5ERLUMw00d1KCeKbo/3Cmc2zEQEVFtw3BTRxWvebM1+i53CiciolqF4aaO6tOqaKfwuPtZOBWXauhyiIiIdIbhpo6yMDFCv9bOAICtvDVFRES1CMNNHTbo4a2p7WfucqdwIiKqNRhu6rAuTRrAvp4pHmTl48Bl7hRORES1A8NNHWYkl2GAjwsA4MdD1xGvyDZwRURERE+P4aaOszQ1AgBEXr2HLrP3YsPxOANXRERE9HQYbuqweEU2fvjniuprpQCmbT7HHhwiIqrRGG7qsOspmVA+scRNoRC4kZJlmIKIiIh0gOGmDvOyt4RMKnnc3c686oshIiLSEYabOszFxhxhQ9pCLqknnG3Rdw1UERER0dMzMnQBZFjD/TzQvZkDbqRk4dydVHzx1yV8G34ZXb3t4eNe39DlERERaY09NwQXG3MENGmA17s1Rv+2LihQCry/IRqZuQWGLo2IiEhrDDekIkkSvhjcBi42ZriekolZ2y8YuiQiIiKtMdyQmvoWJpg3zAeSBKw/fgs7z8UbuiQiIiKtMNxQCYFN7PHf7k0AAFM2n0WCIsfAFREREWmO4YZKNbFPM7RtaIPUrHx8uCkayicXxCEiIqqmGG6oVCZGMsx/qT3MjeU4fOUefjp03dAlERERaYThhsrUxKEeZgxoBQCYu+sSzt9VGLgiIiKiijHcULle8nNHcCsn5BcKTFgfjey8QkOXREREVC6GGyqXJEmY/WI7OFqZ4kpSBr74i9PDiYioemO4oQrZWRZNDweAX47EYc+FRANXREREVDaGG9JIt6YOeL2rFwBg0u9nkJTO6eFERFQ9MdyQxj7u1xwtnK1wPzMPH206w+nhRERULTHckMZMjeRYMKIDTI1kOHA5Gasjbxi6JCIiohIYbkgrzZys8En/lgCAsL8v4VJCmoErIiIiUsdwQ1p79ZlGeLa5A/IKlJiwLho5+ZweTkRE1QfDDWlNkiTM/Y8P7OuZICYxHTO2ncO/V1MQr8g2dGlEREQMN1Q5Dlam+Oo/RdPDN564jZeXH0WX2Xux4XicgSsjIqK6rlqEm0WLFsHT0xNmZmbw9/fHsWPHyjx31apVkCRJ7WFmZlaF1VKxFi5WkB77WimAaZvPsQeHiIgMyuDhZsOGDZg4cSJmzpyJU6dOwcfHB3379kVSUlKZbaytrREfH6963Lx5sworpmLXUzLx5GTwQiFwIyXLIPUQEREB1SDcfPPNN3jjjTfw2muvoVWrVliyZAksLCywYsWKMttIkgRnZ2fVw8nJqcxzc3NzkZaWpvYg3fCyt4RMUj8mAfC0tzBIPURERICBw01eXh5OnjyJoKAg1TGZTIagoCBERkaW2S4jIwONGjWCu7s7Bg0ahPPnz5d5blhYGGxsbFQPd3d3nb6GuszFxhxhQ9pCLj1KOJIEJKXlGrAqIiKq6wwablJSUlBYWFii58XJyQkJCQmltmnevDlWrFiBbdu24ZdffoFSqURgYCBu375d6vlTp06FQqFQPW7duqXz11GXDffzwKEpz2LdG/54toUDlAJ4d10U0nPyDV0aERHVUUaGLkBbAQEBCAgIUH0dGBiIli1bYunSpZg1a1aJ801NTWFqalqVJdY5LjbmcLExRytXGzz/3UHE3c/CJ1vO4buX2kOSpIqfgIiISIcM2nNjb28PuVyOxET1XaYTExPh7Oys0XMYGxujQ4cOuHLlij5KJC3YmBtjwYgOkMsk/HH6LjadLL03jYiISJ8MGm5MTEzg6+uLiIgI1TGlUomIiAi13pnyFBYW4uzZs3BxcdFXmaQF30a2mNinGQBg5rbzuJKUYeCKiIiorjH4bKmJEydi+fLlWL16NS5evIi3334bmZmZeO211wAAoaGhmDp1qur8zz//HLt378a1a9dw6tQpvPLKK7h58yZef/11Q70EesJbPZogsEkDZOcX4t11UdyegYiIqpTBx9wMHz4cycnJmDFjBhISEtC+fXvs3LlTNcg4Li4OMtmjDPbgwQO88cYbSEhIgK2tLXx9ffHvv/+iVatWhnoJ9AS5TMK3w9vj+e8O4mJ8Gmb/fQmfDWxt6LKIiKiOkIQQT67DVqulpaXBxsYGCoUC1tbWhi6nVlq0aBG++uor3I1PgNSgEeyC/ovVk19GcOuS46g2b96ML7/8EleuXEF+fj6aNm2KDz/8EK+++qrqnLIGJc+dOxcff/yx3l4HERFVH9r8/jb4bSmqXR5fcTo66hTatWuHpI0z8MHqA6Vuy2BnZ4dPPvkEkZGROHPmDF577TW89tpr2LVrl+qcx1ejjo+Px4oVKyBJEl588cWqfGlERFRDsOeGdMrf3x9+fn5YuHAhACAnrwD1HV1g3r4/+oz4L9a98QzkTy5r/ISOHTuif//+pU7tB4CQkBCkp6erDUQnIqLajT03ZBClrThtZmKE/n2DURAfg2PX7+P7vbFlthdCICIiAjExMejevXup5yQmJmLHjh0YO3aszusnIqLageGGdKasFaebNGoIF5McAMCCiFgcvXZP7fsKhQL16tWDiYkJ+vfvj++//x59+vQp9RqrV6+GlZUVhgwZop8XQURENR7DDVUJWwsTvNjRDUoBTFgfjQeZearvWVlZITo6GsePH8cXX3yBiRMnYt++faU+z4oVKzBy5EiYmZlVUeVERFTTGHwqONUeFa04/fmg1oiKe4BrKZn4+LczWB7qC0mSIJPJ4O3tDQBo3749Ll68iLCwMPTs2VPteQ4ePIiYmBhs2LChql4SERHVQOy5IZ2paMVpS1MjLBjRASZyGfZcTMSayJulPo9SqURubsmdxX/66Sf4+vrCx8dHb6+BiIhqPvbckE5NnDgRo0aNQqdOndC5c2fMnz9fbcXpuVPfRVOlJc67vYAv/rqI6D9X4vleXdGkSRPk5ubir7/+ws8//4zFixerPW9aWho2bdqEefPmGeJlERFRDcJwQzqlyYrTnp6e6N3CERGXkrDlxDVs2fAr4u/egYW5OVq2bIFffvkFw4cPV3ve9evXQwiBESNG6LTeeEU2rqdkwsveEi425jp9biIiMgyuc0MGcT8zD899dwCJaY9uP8kkIGxIWwz386iSGtYevYlPtp6DEFV/bSIi0g7XuaFqz87SBJ/2V98PTCmAaZvPlbqSsS4plQK/HLmJaVuKgk1VXpuIiPSPt6XIYOzqmZQ4VigEzt9J08stIiEEDsam4KtdMTh7R1HqtW+kZPH2FBFRDcdwQwbjZW8JmVTUa/K4CeujMKarF8Z29UJ9i5IBqDKi4h5g7s4YRD5cQNDcWIacfCUev7RMAjztLXRyPSIiMhzeliKDcbExR9iQtpA/3PVbJgHO1qbIzCvE93uvoOucf/D1rhi1Bf+0dTkxHW+uOYHBP/yLyGv3YCKXYUwXLxya3AuzX3x0bQCwMDGCsZx/JYiIajoOKCaDi1dk40ZKFjztLeBkZYbdFxIwf08sLiWkAwAsTeQYFeiJ17s1hp2lZj05t+5nYf6eWGyOuq0aMPxiRzdMCGoKN9tHvTPximxcTkjHZ3+ex/WULPRo5oCVo/0gq2BzTyIiqlra/P5muKFqSakU2H0hEd9FxOJifBqAopATGuiJN8oJOSkZuVi49wp+PXoT+YVFb+1+rZ3xUd9m8Ha0KvN6MQnpGLjwEHILlJjevyVe79ZY9y+KiIgqjbOlqMaTyST0a+OMHe92xdJXfdHKxRqZeYVYvO8qus7Zi9l/X8K9jFzEK7Lx79UUXE5Mxze7Y9B97j9Y9e8N5BcKdPFugK3jumDJq77lBhsAaO5shU9fKJq9NWfnJZy9XXLAsb4tWrQInp6eMDMzg7+/P44dO1bmucuXL0e3bt1ga2sLW1tbBAUFlTh/9OjRkCRJ7dGvXz99vwwiIoNjzw3VCEIIhD/syTl/t6gnx0QuIb9Q4Mk3cDs3G0zq2wJdm9prfY23fjmJXecT4dnAAtvf64Z6plUz5n7Dhg0IDQ3FkiVL4O/vj/nz52PTpk2IiYmBo6NjifNHjhyJLl26IDAwEGZmZpgzZw62bNmC8+fPo2HDhgCKwk1iYiJWrlypamdqagpbW9sqeU1ERLrE21LlYLip2YQQiLiYhK92X0JMQkaJ7385uA1GdPaAJFVuzExqVh6e/+4g7ipyMKRDQ3wzvP1TVqwZf39/+Pn5YeHChQCK9tdyd3fHu+++iylTplTYvrCwELa2tli4cCFCQ0MBFIWb1NRUbN26VZ+lExFVCd6WolpLkiQEtXLCzBdal/p9L/t6lQ42AFDfwgTzX+oAmQRsjrqDzaduV/q5NJWXl4eTJ08iKChIdUwmkyEoKAiRkZEaPUdWVhby8/NhZ2endnzfvn1wdHRE8+bN8fbbb+PevXs6rZ2IqDpiuKEaycuhaI2cx8klSSfr1HT2ssOE3s0AAJ9uPYfrKZlP/ZzlSUlJQWFhoWr/rWJOTk5ISEjQ6DkmT54MV1dXtYDUr18/rFmzBhEREZgzZw7279+P5557DoWFhTqtn4ioumG4oRrpyTVy5JKEL4e00dnqwuN7eaOzlx0y8wrx3roo5BUodfK8+jB79mysX78eW7ZsgZmZmer4Sy+9hIEDB6Jt27YICQnB9u3bcfz4cezbt89wxRIRVQGGG6qxhvt54NCUZ7HujWdwaMqzOt30Ui6T8N1L7VHfwhhn7yjw1a5LOnvuJ9nb20MulyMxMVHteGJiIpydnctt+/XXX2P27NnYvXs32rVrV+65jRs3hr29Pa5cufLUNRMRVWcMN1SjudiYI6BJA73sB+ViY465LxYFhuUHr+OfmCSdXwMATExM4Ovri4iICNUxpVKJiIgIBAQElNlu7ty5mDVrFnbu3IlOnTpVeJ3bt2/j3r17cHFx0UndRETVFcMNUTmCWztjVEAjAMBHG08jKT1HL9eZOHEili9fjtWrV+PixYt4++23kZmZiddeew0AEBoaiqlTp6rOnzNnDj799FOsWLECnp6eSEhIQEJCAjIyimaQZWRk4OOPP8aRI0dw48YNREREYNCgQfD29kbfvn318hqIiKoLhhuiCkx9viVaOFvhXmYeJm44DeWTO33qwPDhw/H1119jxowZaN++PaKjo7Fz507VIOO4uDjEx8erzl+46Afk5eXhP//5D1xcXFSPr7/+GgAgl8tx5swZDBw4EM2aNcPYsWPh6+uLgwcPwtTUVOf1ExFVJ1znhkgDV5LSMeD7w8jOL8Tkfi3wds8mBqtlw/E4TN18FsqHe2aFDWmr0/FGRETVEde5IdIxb0crfDawaHuGebtjEBX3oEquK4RAgiIHey4k4rs9sXj1p6OY/HtRsAEApQCmbT6HeEV2ldRDRFQTVM3a8kS1wLBO7jgQm4IdZ+Lx3voo7HivG6zNjCv9fPGKbFxPyYSXvSVcbMwhhMCt+9k4d1eBc3cUOH83DefvKpCSkVfu8xQKgRspWXoZVE1EVBMx3BBpSJIkfDm4LU7fSsWt+9mYuOE0xnTxhJeDpVbBIregEGv+vYkv/74IIQAJgJe9JZIzcpGeU1DifJkENHW0QmtXa7jbWWDB3lg8eTO5YX2zEu2IiOoqjrkh0tLJmw/wnyX/qgKGTCpa9M/fqwHuZ+YhNSsP9zPz8SAr7+EjHw8yH/5/Zh4y88peIdhELkNzZyu0aWiNVq42aONqjRbO1jA3kavO2XA8DtM2n0PhY391X+/qhekPdzUnIqqNtPn9zZ4bIi251jfD41uRKwWwIOIKgKdbHG/2kLYY0tENJkblD4Ub7ueB7s0ccCMlC3H3MjF581n8eOg6ujS1x7PNS+4grmtP3k4jIqpuGG6ItHQ9JROldXc2rG+GhrYWsLUwhp2lCepbmMDOwgT1H//a0gT5BUr0++4AHp9RLpck9GjuUGGwKeZiY65awPBCfBpWR97ERxtP4+8J3eBorb9bVJypRUQ1AcMNkZa87Is27XwynPz2dqDGPRlhQ9qqbi097b5YU59viWM3HuBifBo+2BiNn8f4Q/bkrqI6EK/IVgUboOj1T9l8Fp09G8DLwVLn1yMiqixOBSfSki427dTlvlhmxnJ8P6IDzI3lOHzlHhbvv1rp5yrPlaQMPLl+oRDA8wsOYNqWszh9KxV1bAhflYpXZOPfqymc9k+kAQ4oJqqkeEU2bqRkwdPeolqMPdl4/BYm/X4GcpmEjf8NgG8jW509d36hEv/9+ST2Xip/f60WzlYY2skdgzs0hJ2lic6uX9fxdiCRdr+/GW6IagkhBN5bH40/T99Fw/rm+GtCN9iYV34dnmJ5BUq8u+4Udp1PhFwqGkutFEU9Vv83uA087Cyw8cQt/H0uAXkFSgCAsVxCcCtnDO3khm5NHSDXw20yQzDEYOrbD7LQbc4/auO85JKEQ1OerRahmqiqMNyUg+GGarO0nHz0X3AQt+5no39bFyx8uQMkqfLBIie/EON+PYWIS0kwkcuw5NWOaOliXWqPlSIrH9tO38HGE7dw7k6a6rirjRn+4+uGoZ3c4W5nUWNnWxmi9yQmIR3v/HoSV5MzS3xv3RvPIKBJA71en6g6YbgpB8MN1XZRcQ8wdEkkCpQCs4e0xUudK/cLOCe/EG/+fBIHLifD1EiG5aGd0L2Zg0Ztz99VYNOJ29gSdQeK7HzV8SYOlriWXDTbrCbdXolXZKPL7L0lBpHrq/ckt6AQi/ZeweL9V5FfWPIjWiYBh6f0qlHhkOhpcW8pojqsg4ctPgxuDgD47M/ziE1M1/o5svIKMGbVcRy4nAxzYzlWjvbTONgAQGtXG3w2sDWOTuuNBSM6oFtTewDA1eRH0+hr0r5YpQ2mLhQC10rpUXlax2/cx/PfHcSCvVeQXygQ1NIJU59voRrADhStln03NUfn1yaqLRhuiGqh/3ZvjG5N7ZGTr8S766KQk1/2qshPysgtwOiVx/Hv1XuwNJFj9ZjOCPS2r1QdZsZyDPRxxc9j/fHd8PYlvl8oBGIStA9fVW3X+cRSj0/dfAa/n7yNgkLlU18jLScf07eexdAlkbianAn7eqb4YWRHLA/1xX+7N8GhKc/i19f90dXbHoVKgTfWnMDNe7oPV0S1AcMNUS0kk0mYN8wH9vVMcCkhHV/suKhRu7ScfIxacQzHrt+HlakR1oz1R2cvO53U1LmxHUobV/zp1nM4VUW7rFfG3kuJ+OXITQBQ1S8BMDeRI+5+Nj7cdBq95u3H+mNxqgHV2tp9PgHB3xzAL0fiAADDO7kjYmIPPN/WRTVmysXGHF287bEs1BdtGlrjfmYeXlt1HKlZ5W+sSlQXccwNUS22/3IyRq04BgBY8oov+rVxLvNcRVY+Qlcew+lbqbA2M8LPY/3h415fp/U8vi+WTALqmRohLacAMgl4p6c33uvdVONVmqvCndRs9F9wEKlZ+Rgd6In/9misGkxtZWaMnyNv4seD13AvsyhgNKxvjrd6NMbQTu4wM5ZX8OxAUnoOPvvjPP46mwAA8GxggS+HtEVgk/J7ypLSchCy6DDuKnLg72WHNWM7w9So4usR1WQcUFwOhhuqa7786yKWHbgGG3Nj/DWhGxrWLzkI9UFmHl5dcRTn7qShvoUxfhnrjzYNbfRSz+PrA1kYG2HmH+ewNfouAKCVizW+Ge6DFs6G/7uZX6jE8KWROBWXCh83G2x8K6DUAJGVV4C1R+Ow9MA1JKfnAgCcrE3xZvcmeLmzh9qmp8WEENh44ha+2HERaTkFkMskvNm9MSb0bqpRKAKASwlp+M/iSGTkFmBIh4aYN8znqWbGEVV3HFBMRCofBTeHj5sNFNn5eH99VInxISkZuRix/AjO3UlDA0sTrH/zGb0FGwCqPbFcbMxhY2GM+S91wA8jO8LWwhgX4tMw8PvDWLr/KgqfHMFbxb7aFYNTcUW9WAtf7lhmz4iFiRFe79YYByc9i88HtYaLjRkS03Ixa/sFdJu7F0v2X0VmboFqheFj1+/h5eVHMfn3s0jLKUDbhjb4Y3wXTO7XQuNgAwAtnK3xw8iOkMskbI66g+8iYjVuu2jRInh6esLMzAz+/v44duxYuedv2rQJLVq0gJmZGdq2bYu//vpL7fujR4+GJElqj379+mlcT3m4MjNVBntuiOqAm/cy0X/BIWTkFuC93k0xsU8zAEW3RUYuP4rYpAw4WJli7ev+aOpkZZAak9JzMPX3s4h4uAqyn6ctvh7qg0YNqn7fqj0XEvH6mhMAgKWv+qJv67Jv5z0pt6AQv5+8gx/2XcHtB0W/kM1N5MjJK1RbiM/MWIYP+zTHa108YSSv/L8z1x0rWn8HAL4Z5oMhHd3KPX/Dhg0IDQ3FkiVL4O/vj/nz52PTpk2IiYmBo2PJXeX//fdfdO/eHWFhYXjhhRewdu1azJkzB6dOnUKbNm0AFIWbxMRErFy5UtXO1NQUtra2T7WuEVdmpsfxtlQ5GG6ortoWfQcT1kdDJgHfj+gAAJjz9yXEPciGs7UZ1r7hj8YO9QxaoxACm07cxv/+PI/MvEJYmMgxvX8rjOjsXmW3XG4/yEL/BYegyM7HmC5emDGgVaWeJ79Qia1Rd7AgIha3HpTsdfjtrQB08tTNYO3Zf1/Ckv1XYSyX8PNYfzzTuOzF/fz9/eHn54eFCxcCAJRKJdzd3fHuu+9iypQpJc4fPnw4MjMzsX37dtWxZ555Bu3bt8eSJUsAFIWb1NRUbN26Va1taeFkqK87svILkZVbgIzcAmTmFiIzrwCZuQXIzCss+m9uARIUOfjp0HWuzEwq2vz+5q7gRHXEoPYNcTA2Bb+dvI1xa6NUx+ubG2PDf58xSA/JkyRJwjA/dwQ0aYAPN53Gsev3MW3LWey+kIA5L7aDk7WZXq+fV6DE+LVRUGTnw8e9PqY816LSz2Usl2FoJ3e42JjjlZ+Olvh+aYvzVdakvs1x634WdpyNx39/PonN7wSiSSlBNS8vDydPnsTUqVNVx2QyGYKCghAZGVnqc0dGRmLixIlqx/r27VsiyOzbtw+Ojo6wtbVFr169MH7SJyV2kZ/8+1lM/v1spV9noRC4kZLFcEMV4pgbojrk7R5NShxLy8mvVjOUAMDdzgLr33gG0/u3hImRDPtikhH87QH8efpuiTEYuhw/MmfnJexbtwgJP76F8EnBcHJogKCgIBw9WjKcaKqJo2WJKfBySYKnvUWln/NJxVP/O3jUhyI7H6+tPI57GbklzktJSUFhYSGcnJzUjjs5OSEhIaHU505ISKjw/H79+mHNmjWIiIjAnDlz8M++/Qjq0w+FhWWvrySTACtTIzhbm6GJgyXaudkgoHEDBLV0xKD2rghp74rS+urO3uHu81Qx9twQ1SGJ6SVXtVUKVMt/DctkEl7v1hg9mjngg43ROHcnDe+ue9TjJJOAAVY3sOR/E9XGj/Tt27fc8SMjRoxQGz8SEhKCU6dO4a5kj58OXYexXUNMnfMNhj7ri+zsbHz77bcIDg7GlStX4OCg+SrNxVxszBE2pK1qCrxckvDlkDY6/3mbGcuxPLQTBv9wGHH3s/DGmhNY+8YzWg1SrqyXXnoJQNF4oxNp9WD83BQkfDsKjnFnYe7ZXnWeTAL+GN8VTRzqwcxYVuGtxoAmDVQ/t2Jf/nUJR6/dR9iQtnDUc08e1Vwcc0NUh1T1Hkm6kl+oxJd/XcTKwzfUjiesmYgRLzyLlcuKxn5UdvyId8s2ONNoKNJyCvBGNy980v/ROJviz4w9e/agd+/elX4Nj0+B1+fP+kpSBob8cBhpOQXo39YF34/oANnDrqO8vDxYWFjgt99+Q0hIiKrNqFGjkJqaim3btpV4Pg8PD0ycOBHvv/++6tjMmTOxdetWnD59GgBQUKgsmrG1JxZ3Uot61O4uHIleI99FrN0zKHy4i/yXQ9poPSC4+OfmbmeO7Wfi8c3uy8grVMLG3BifD2qNgT6unAJfR3AqOBGVqrgXoXifIn31IuiasVyGPq3Ub42IwnzkJlyBrXdH1TFNxo8EBQWpHQvqE4w/d+9DWk4BOnjUx6R+j8bZ5OXlYdmyZbCxsYGPj89TvYbHp8Drk7djPSx9tROM5RJ2nI3HV7tjVN8zMTGBr68vIiIiVMeUSiUiIiIQEBBQ6vMFBASonQ8A4eHhCAgIgBACf5+NR9/5BzDptzO4k5oNJ2tTTOzSAAVZaXinvx8OTemFdW88g0NTnq3UTKfin5ubrQXe6tEE29/rirYNi5Y2mLA+GuPWnir1Fpw+6HoKvRACM2bMgIuLC8zNzREUFITYWM2n9FPZGG6I6pjhfh44NOXZp/qFYwhe9upjVwqz0gChxLqzafjsj/NIyynafVzb8SNRKQIZqfdQ38IYC1/uCGO5DNu3b0e9evVgZmaGb7/9FuHh4bC3r9z+WoYQ0KQBZg9pBwBYvO8qluy7qhqnNHHiRCxfvhyrV6/GxYsX8fbbbyMzMxOvvfYaACA0NFRtwPGECROwc+dOzJs3D5cuXcJnn32GEydO4JkXXsbAhYfx35WHcWLj9zC+dwVvdqiHz3yVWP3ZOHh7e6Nv3746D3XNnKyw+Z1AfBDUDEYyCX+dTUDwtwew81zpf+a6smHDBkycOBEzZ87EqVOn4OPjg759+yIpKanU84tvgY4dOxZRUVEICQlBSEgIzp07pzpn7ty5WLBgAZYsWYKjR4/C0tISffv2RU4ON0V9WrwtRUQ1xuPbNygz7uHWolFwfuUrmDZsCft6ppjevyUOr52PAwcOlDoI2MTEBKtXr8aIESMAADvPxWPkxFlIPbwOu09cQu+WRcEnMzMT8fHxSElJwfLly7F3714cPXq01HE81dm34ZfVFvcrno6dfPQPfPXVV0hISED79u2xYMEC+Pv7AwB69uwJT09PrFq1StVu2epf8X//m4nEO7fg7tkYrsGvI86yqIfLTCqACP8K9+MuQ5GaCldXVwQHB2PWrFklgqSunbujwIcbTyPm4c73gzs0xGcDWsPGwljn19L1FHohBFxdXfHhhx/io48+AgAoFAo4OTlh1apVqnFM9AjXuSkHww1RzVY8BsPV2ghNXBpg1vcrEJ7lgWvJRTtkS/sXoYmNhD07d5Ro+/j4kbh7Wej//UHcCl8N8/hTiL92qcxrNm3aFGPGjFHr0agJ7qZmIXD2PyWOd2tqj4b1zdGgngnsLE3RwNLk4f+bwL6eKWwtTFQz6B5fq+ZxJnIZRj7jgXHPesO+nmlVvJxS5RYUYv6eWCzdfxVKUbT1xewX2+HZ5roLovoYq3Tt2jU0adIEUVFRaN++veqcHj16oH379vjuu+90Vn9twXVuiKjWcrExV93i8PX1xd0Lx/H3tyPx48HrWBARg6vnjiGt0wuY/fclvNfbGxYmjz7misePvD3+XYxbewrpOQWQx5/FgD49yr2mUqlEbm7VjOvQpRv3sko9fjA2pcK2VmZGqG9uXOoChC+0c8aU51rCzVZ309kry9RIjsn9WqBPKyd8tPE0rqVk4rWVx/GSnzvGdvVCckZupVZHflx5U+gvXSo9FFc0hb74v9pMyyfNMdwQUY01ceJEjBo1Cp06dUKvzp3x7+3NuKHMg0WbICzZfxXffPIenu3YHOuXL4AkSZgwYQJ69OiBAW9MwXmjJii8chjpt2Pw3rvrABTdjvriiy8wcOBAuLi4ICUlBYsWLcKdO3cwdOhQA79a7RWPU3q810UmAR8GN0dBocD9zFzcy8zDvYw83M/Mw73MPNzPzIVSAOk5BUjPKSj1eUf6e1aLYPO4jh622PFeN3y1KwYrDl/H+uO3sP74LQDcuqEuYrghohpr+PDhSE5OxowZM1TjRw7sDUe6lSc++/M8TqTEY8dRJcasOo7/DWwDr9Yd8PKUr7H2h69QoEiEZ+Mm2Lp1q2qPJLlcjkuXLmH16tVISUlBgwYN4Ofnh4MHD6J169YGfrXaK2uNnfJ+ySuVAorsfNzLzMPlhDSMWxtVYgsEXS5AqEvmJnLMGNAKvo3qq63CrRTAlN/PwsXGHN2a2ms9ddze3h5yuRyJiYlqxxMTE+HsXPq+Y87OzuWeX/zfxMREuLi4qJ3z+G0qqhyOuSGiWik7rxCL/rmCpQeuIr9QQC6ToFQK1S/qns0dsOq1zgatsao8zRo7jw/iruxaNVXt36speHl56atKu9uZI6ilE/q0ckJnTzuNNy319/dH586d8f333wMoulXp4eGB8ePHlzmgOCsrC3/++afqWGBgINq1a6c2oPijjz7Chx9+CKDo95OjoyMHFJeBA4rLwXBDVLdcTc7AlN/P4PiNB2rHZRJweEqvar/GT3VQVQsQ6kppi1UCgIlcQt5je3rZmBujVwtH9GnlhO7NHFDPtOybGRs2bMCoUaOwdOlSdO7cGfPnz8fGjRtx6dIlODk5ITQ0FA0bNkRYWBiAoqngPXr0wOzZs9G/f3+sX78eX375pdpu6nPmzMHs2bOxevVqeHl54dNPP8WZM2dw4cIFmJlx9eUn1bhF/HS9MBIRUbEmDvXwQVCzEseLt52gilXVAoS6UtpilXNebIvomcFY+qov/uPrBlsLYyiy87El6g7e+fUUOn4ejtErj+HXozeRlFa0zszj+5gNHz4cX3/9NWbMmIH27dsjOjoaO3fuVA0IjouLQ3x8vKqGwMBALPpxFb5btBg+Pj747bff1G6BAsCkSZPw7rvv4s0334Sfnx8yMjKwc+dOBhsdMHjPzYYNGxAaGqq2N8ymTZvK3Rume/fuanvDzJkzRy0Nl4c9N0R1T03ddoKeTnk9ToVKgZM3HyD8QgJ2X0jEzSdmlrnbmuP2g2wIAJIETOjVFCEdGsLMWA5zYznMTGQwkZe9P9bjU+grM6A5XpGN6ymZlZ7p9TTtDXnt8tSo21K6XhipIgw3RHVTTRw7QlVDCIHYpAyEX0hE+IVERN9K1aidJAHmxWHHWA4zYxnMTeSQQcKZOwr1cwE839YZ1ubGkMskGMlkMJJJkMslGMtkkMskGMslyGUynLujwF9n44uCFYBB7V3h62kH6eE1i55PgiRBdUyC6hs4fv0+fjt5W9X+P75u6OxlV/Ra1V74Ez8HCBy7fh+bT91RtR3SsaGqbfF1S/9hFP3n2PX7+P3htXU9S63GhBt9beL2uNzcXLX1KRQKBTw8PHDr1i2GG6I6JkGRjbh72fBoYA5n9thQGXafi8fETWdKHDc1lqGgUKDwycE8VCa5JGHXB9108vctLS0N7u7uSE1NhY2NTbnnGnQquD4WRnpSWFgY/ve//5U47u7uXsmqiYiISFPNv9Xt86Wnp1fvcFMVpk6diokTJ6q+ViqVuH//Pho0aKD1WgcVKU6VlekVepq2NfnaT9ue165b137a9rw2r11T2tfVa5dHCIH09HS4urpWeK5Bw40+FkZ6kqmpKUxN1fc9qV+/fuWL1oC1tXWl/0Cfpm1NvvbTtue169a1n7Y9r81r15T2dfXaZamox6aYQaeCm5iYwNfXFxEREapjSqUSERERCAgIKLVN8d4wjwsPDy/zfCIiIqpbDH5b6vG9YYoXRsrMzMRrr70GACUWRireG2bevHmqhZFOnDiBZcuWGfJlEBERUTVh8HBT2t4wTy6MJJM96mAKDAzE2rVrMX36dEybNg1NmzYtsTCSoZiammLmzJklboPpu21NvvbTtue169a1n7Y9r81r15T2dfXaumLwdW6IiIiIdKlabL9AREREpCsMN0RERFSrMNwQERFRrcJwQ0RERLUKw42OLFq0CJ6enjAzM4O/vz+OHTumUbsDBw5gwIABcHV1hSRJ2Lp1q1bXDQsLg5+fH6ysrODo6IiQkBDExMRo1Hbx4sVo166daqGlgIAA/P3331pdv9js2bMhSZLanl/l+eyzzyBJktqjRYsWWl3zzp07eOWVV9CgQQOYm5ujbdu2OHHiRIXtPD09S1xbkiSMGzdOo+sWFhbi008/hZeXF8zNzdGkSRPMmjULmo7NT09Px/vvv49GjRrB3NwcgYGBOH78eKnnVvT+EEJgxowZcHFxgbm5OYKCghAbG6tx+82bNyM4OFi1Ynd0dLRGbfPz8zF58mS0bdsWlpaWcHV1RWhoKO7evavxtT/77DO0aNEClpaWsLW1RVBQEI4ePapR28e99dZbkCQJ8+fP1/jao0ePLvHn369fP42vffHiRQwcOBA2NjawtLSEn58f4uLiNGpf2ntPkiR89dVXFbbNyMjA+PHj4ebmBnNzc7Rq1Uptw+CK2icmJmL06NFwdXWFhYUF+vXrp3q/aPJZkpOTg3HjxqFBgwaoV68eXnzxRdWiqpq0X7ZsGXr27Alra2tIkoTU1FSN2t6/fx/vvvsumjdvDnNzc3h4eOC9996DQqHQ+Nr//e9/0aRJE5ibm8PBwQGDBg3CpUuXtPoMFULgueeeU/vZatK+Z8+eJf6833rrLY2vHRkZiV69esHS0hLW1tbo3r07srOzK2x/48aNMt9vL7/8coXXTkhIwKuvvgpnZ2dYWlqiY8eO+P333zV+3VevXsXgwYPh4OAAa2trDBs2rMQivPrCcKMDGzZswMSJEzFz5kycOnUKPj4+6Nu3L5KSkipsm5mZCR8fHyxatKhS196/fz/GjRuHI0eOIDw8HPn5+QgODkZmZmaFbd3c3DB79mycPHkSJ06cQK9evTBo0CCcP39eqxqOHz+OpUuXol27dlq1a926NeLj41WPQ4cOadz2wYMH6NKlC4yNjfH333/jwoULmDdvHmxtbTWq9/HrhoeHAwCGDh2q0bXnzJmDxYsXY+HChbh48SLmzJmDuXPn4vvvv9eo/euvv47w8HD8/PPPOHv2LIKDgxEUFIQ7d+6UOLei98fcuXOxYMECLFmyBEePHoWlpSX69u2LnJwcjdpnZmaia9eumDNnjlbXzsrKwqlTp/Dpp5/i1KlT2Lx5M2JiYjBw4ECNa2/WrBkWLlyIs2fP4tChQ/D09ERwcDCSk5M1/nuxZcsWHDlypMRy7Jq079evn9r7YN26dRq1vXr1Krp27YoWLVpg3759OHPmDD799FOYmZlp1P7xa8bHx2PFihWQJAkvvvhihW0nTpyInTt34pdffsHFixfx/vvvY/z48fjjjz8qvLYQAiEhIbh27Rq2bduGqKgoNGrUCEFBQcjMzNTos+SDDz7An3/+iU2bNmH//v24e/cuhgwZAkCzz6KsrCz069cP06ZNU6utorZ3797F3bt38fXXX+PcuXNYtWoVdu7cibFjx2p8bV9fX6xcuRIXL17Erl27IIRAcHAw9u3bp/Fn6Pz580ts26PpZ/Abb7yh9uc+d+5cjdpGRkaiX79+CA4OxrFjx3D8+HGMHz8eMpmswvbu7u4l3m//+9//UK9ePSQnJ1d47dDQUMTExOCPP/7A2bNnMWTIEAwbNgxRUVEVXjszMxPBwcGQJAl79+7F4cOHkZeXhwEDBkCpVJb42eqcoKfWuXNnMW7cONXXhYWFwtXVVYSFhWn1PADEli1bnqqWpKQkAUDs37+/Uu1tbW3Fjz/+qPH56enpomnTpiI8PFz06NFDTJgwQaN2M2fOFD4+PpWqUQghJk+eLLp27Vrp9o+bMGGCaNKkiVAqlRqd379/fzFmzBi1Y0OGDBEjR46ssG1WVpaQy+Vi+/btasc7duwoPvnkk3LbPvn+UCqVwtnZWXz11VeqY6mpqcLU1FSsW7euwvaPu379ugAgoqKiNLp2aY4dOyYAiJs3b1aqvUKhEADEnj17NGp7+/Zt0bBhQ3Hu3DnRqFEj8e2332pc+6hRo8SgQYPKraestsOHDxevvPJKhW3Lq/1xgwYNEr169dKobevWrcXnn3+udqys986T7WNiYgQAce7cOdWxwsJC4eDgIJYvX16i/ZOfJampqcLY2Fhs2rRJdc7FixcFABEZGVlh+8f9888/AoB48OBBie9V1LbYxo0bhYmJicjPz69U+9OnTwsA4sqVKxq1jYqKEg0bNhTx8fHl/rmW1l7Tz8bS2vr7+4vp06dX2La82h/Xvn37Ep9fZbW1tLQUa9asUTvPzs5Oo/fLrl27hEwmEwqFQnVOamqqkCRJhIeHa/R6ngZ7bp5SXl4eTp48iaCgINUxmUyGoKAgREZGVnk9xd20dnZ2WrUrLCzE+vXrkZmZqdVWFuPGjUP//v3VXr+mYmNj4erqisaNG2PkyJGqbn1N/PHHH+jUqROGDh0KR0dHdOjQAcuXL9e6hry8PPzyyy8YM2aMxhupBgYGIiIiApcvXwYAnD59GocOHcJzzz1XYduCggIUFhaq/pVfzNzcXKueKwC4fv06EhIS1H72NjY28Pf3N9h7T5KkSu3dlpeXh2XLlsHGxgY+Pj4Vnq9UKvHqq6/i448/RuvWrStRLbBv3z44OjqiefPmePvtt3Hv3j2Nrrtjxw40a9YMffv2haOjI/z9/bW+nVwsMTERO3bsUPVAVCQwMBB//PEH7ty5AyEE/vnnH1y+fBnBwcEVts3NzQUAtfeeTCaDqalpqe+9Jz9LTp48ifz8fLX3W4sWLeDh4VHq+62yn0WatlUoFLC2toaRUcm1aCtqn5mZiZUrV8LLywvu7u4Vts3KysLLL7+MRYsWlbmPYUXX/vXXX2Fvb482bdpg6tSpyMrKqrBtUlISjh49CkdHRwQGBsLJyQk9evQo87Oiotd98uRJREdHl/p+K61tYGAgNmzYgPv370OpVGL9+vXIyclBz549K2yfm5sLSZLUFvIzMzODTCbT+rOuUvQen2q5O3fuCADi33//VTv+8ccfi86dO2v1XHjKnpvCwkLRv39/0aVLF43bnDlzRlhaWgq5XC5sbGzEjh07NG67bt060aZNG5GdnS2E0PxfJ0II8ddff4mNGzeK06dPi507d4qAgADh4eEh0tLSNGpvamoqTE1NxdSpU8WpU6fE0qVLhZmZmVi1apXG9QshxIYNG4RcLhd37tzRuE1hYaGYPHmykCRJGBkZCUmSxJdffqlx+4CAANGjRw9x584dUVBQIH7++Wchk8lEs2bNym335Pvj8OHDAoC4e/eu2nlDhw4Vw4YNq7D945625yY7O1t07NhRvPzyy1q1//PPP4WlpaWQJEm4urqKY8eOadT2yy+/FH369FH1tmnbc7Nu3Tqxbds2cebMGbFlyxbRsmVL4efnJwoKCsptW/yvdgsLC/HNN9+IqKgoERYWJiRJEvv27dP4dRebM2eOsLW1Vf0dqqhtTk6OCA0NFQCEkZGRMDExEatXr9bodefl5QkPDw8xdOhQcf/+fZGbmytmz54tAIjg4GC1tqV9lvz666/CxMSkxHX8/PzEpEmTKmz/uPJ6bjT5HEtOThYeHh5i2rRpWrVftGiRsLS0FABE8+bNS/TalNX2zTffFGPHjlV9Xdafa1ntly5dKnbu3CnOnDkjfvnlF9GwYUMxePDgCttGRkYKAMLOzk6sWLFCnDp1Srz//vvCxMREXL58WePXXeztt98WLVu21LjuBw8eiODgYNX7zdraWuzatUuj9klJScLa2lpMmDBBZGZmioyMDDF+/HgBQLz55ptl1qgrDDdPqTqFm7feeks0atRI3Lp1S+M2ubm5IjY2Vpw4cUJMmTJF2Nvbi/Pnz1fYLi4uTjg6OorTp0+rjmkTbp704MEDYW1trfEtMWNjYxEQEKB27N133xXPPPOMVtcNDg4WL7zwglZt1q1bJ9zc3MS6devEmTNnxJo1a4SdnZ3GwerKlSuie/fuAoCQy+XCz89PjBw5UrRo0aLcdtU13OTl5YkBAwaIDh06qHVBa9I+IyNDxMbGisjISDFmzBjh6ekpEhMTy2174sQJ4eTkpBZItQ03T7p69apGt8SK/76PGDFC7bwBAwaIl156SetrN2/eXIwfP17jur/66ivRrFkz8ccff4jTp0+L77//XtSrV6/Ubv7S2p84cUL4+Pio3nt9+/YVzz33nOjXr5/aeaV9lmgTbir6LCov3FTUVqFQiM6dO4t+/fqJvLw8rdqnpqaKy5cvi/3794sBAwaIjh07qgXL0tpu27ZNeHt7i/T0dNWxsv5cNf0MjoiIKHFLrLS2xX/Hp06dqta+bdu2YsqUKVpdOysrS9jY2Iivv/5a47rHjx8vOnfuLPbs2SOio6PFZ599JmxsbMSZM2c0ar9r1y7RuHFjIUmSkMvl4pVXXhEdO3YUb731Vjk/Hd1guHlKubm5Qi6Xl3ijh4aGioEDB2r1XE8TbsaNGyfc3NzEtWvXKtW+WO/evTVK1Vu2bFF9QBY/AKjexE/+C1gTnTp1KvEXtiweHh5q/5ISQogffvhBuLq6any9GzduCJlMJrZu3apVnW5ubmLhwoVqx2bNmiWaN2+u1fNkZGSogsmwYcPE888/X+75T74/in8hPxlIunfvLt57770K2z+usuEmLy9PhISEiHbt2omUlBSNay+Lt7d3iV6wJ9t+++23qvfZ4+89mUwmGjVqVOlr29vbiyVLlpTbNjc3VxgZGYlZs2apnTdp0iQRGBio1bUPHDggAIjo6OhSv/9k26ysLGFsbFxivNbYsWNF3759tbp2amqqSEpKEkIUjRl85513VN8r67Ok+Bfyk4HEw8NDfPPNNxW2f1xZ4aaitmlpaSIgIED07t271N4ubT4Hc3NzhYWFhVi7dm25bSdMmFDm+61Hjx6VunZGRoYAIHbu3Flu22vXrgkA4ueff1Y7PmzYMLVeUk2uvWbNGmFsbKz6c6+o7ZUrV0qM0RKi6HfEf//7X62unZycrPqzdnJyEnPnzi3zXF3hmJunZGJiAl9fX0RERKiOKZVKREREaDV2pbKEEBg/fjy2bNmCvXv3wsvL66meT6lUqu7Nl6d37944e/YsoqOjVY9OnTph5MiRiI6Ohlwu1+q6GRkZuHr1KlxcXDQ6v0uXLiWmHV6+fBmNGjXS+JorV66Eo6Mj+vfvr1WtWVlZapu5AoBcLtd6BoClpSVcXFzw4MED7Nq1C4MGDdKqvZeXF5ydndXee2lpaTh69GiVvPfy8/MxbNgwxMbGYs+ePWjQoMFTP6cm779XX30VZ86cUXvvubq64uOPP8auXbsqdd3bt2/j3r17Fb7/TExM4Ofn99TvPQD46aef4Ovrq9EYI6Do552fn6+T956NjQ0cHBwQGxuLEydOYNCgQRV+lvj6+sLY2Fjt/RYTE4O4uDgEBAQ81WeRJm3T0tIQHBwMExMT/PHHH2pjhypzbVH0j3vk5OSU23bKlCkl3m8A8O2332LlypWVunbxczg7O5fb1tPTE66urmW+37S59k8//YSBAwfCwcFB9frLa1s8Jqis95s217a3t0f9+vWxd+9eJCUlqc2q1Bu9x6c6YP369cLU1FSsWrVKXLhwQbz55puifv36IiEhocK26enpIioqSkRFRQkAqvv4pc04Kc3bb78tbGxsxL59+0R8fLzqkZWVVWHbKVOmiP3794vr16+LM2fOiClTpghJksTu3bs1uvaTtLkt9eGHH4p9+/aJ69evi8OHD4ugoCBhb29f4l8VZTl27JgwMjISX3zxhYiNjRW//vqrsLCwEL/88otG7QsLC4WHh4eYPHmyRuc/btSoUaJhw4Zi+/bt4vr162Lz5s3C3t6+RNd8WXbu3Cn+/vtvce3aNbF7927h4+Mj/P39S+1ir+j9MXv2bFG/fn3V+JFBgwYJLy8v1b9qK2p/7949ERUVJXbs2CEAiPXr14uoqCgRHx9fbtu8vDwxcOBA4ebmJqKjo9Xee7m5uRVeOyMjQ0ydOlVERkaKGzduiBMnTojXXntNmJqainPnzmn99+LJ21LltU9PTxcfffSRiIyMFNevXxd79uwRHTt2FE2bNhU5OTkVXnvz5s3C2NhYLFu2TMTGxorvv/9eyOVycfDgQY1+5kIU3VqxsLAQixcv1urPu0ePHqJ169bin3/+EdeuXRMrV64UZmZm4ocfftCo/caNG8U///wjrl69KrZu3SoaNWokhgwZIoTQ7LPkrbfeEh4eHmLv3r3ixIkTIiAgQHV7WJP28fHxIioqSixfvlwAEAcOHBBRUVHitddeK7etQqEQ/v7+om3btuLKlStq5xQUFFR47atXr4ovv/xSnDhxQty8eVMcPnxYDBgwQNjZ2YnRo0dr/RmKx3rFKrr2lStXxOeffy5OnDghrl+/LrZt2yYaN24sunfvrtHP7NtvvxXW1tZi06ZNIjY2VkyfPl2YmZmJK1euaPz5HxsbKyRJEn///bfqWEVt8/LyhLe3t+jWrZs4evSouHLlivj666+FJElix44dGl17xYoVIjIyUly5ckX8/PPPws7OTkycOLHMn6suMdzoyPfffy88PDyEiYmJ6Ny5szhy5IhG7Yq7Z598jBo1SqP2pbUFIFauXFlh2zFjxohGjRoJExMT4eDgIHr37l3pYCOEduFm+PDhwsXFRZiYmIiGDRuK4cOHlxjcV5E///xTtGnTRpiamooWLVqIZcuWadx2165dAoCIiYnR6ppCFHWNT5gwQXh4eAgzMzPRuHFj8cknn6h+qVdkw4YNonHjxsLExEQ4OzuLcePGidTU1FLPrej9oVQqxaeffiqcnJyEqamp6N27t9prqqj9ypUrS/3+zJkzy21bfBurtMc///xT4bWzs7PF4MGDhaurqzAxMREuLi5i4MCBqgHF2v69eDLclNc+KytLBAcHCwcHB2FsbCwaNWok3njjDdU/RjS59k8//SS8vb2FmZmZ8PHxUbu1qUn7pUuXCnNz8xJ/7hW1jY+PF6NHjxaurq7CzMxMNG/eXMybN081sLqi9t99951wc3MTxsbGwsPDQ0yfPl31vtXksyQ7O1u88847wtbWVlhYWIjBgweL+Ph4jdvPnDmzzPPKa1vW6wJQ7nuxuP2dO3fEc889JxwdHYWxsbFwc3MTL7/8srh06VKlPkMfDzcVtY+LixPdu3cXdnZ2wtTUVHh7e4uPP/5YtfSBJtcOCwsTbm5uwsLCQgQEBKiCtKbtp06dKtzd3UVhYaHaa6io7eXLl8WQIUOEo6OjsLCwEO3atVNNDdek/eTJk4WTk5MwNjYWTZs2VXuv6pv0sEgiIiKiWoFjboiIiKhWYbghIiKiWoXhhoiIiGoVhhsiIiKqVRhuiIiIqFZhuCEiIqJaheGGiIiIahWGGyIiIqpVGG6IqM7bt28fJElCamqqoUshIh1guCEiIqJaheGGiIiIahWGGyIyOKVSibCwMHh5ecHc3Bw+Pj747bffADy6ZbRjxw60a9cOZmZmeOaZZ3Du3Dm15/j999/RunVrmJqawtPTE/PmzVP7fm5uLiZPngx3d3eYmprC29sbP/30k9o5J0+eRKdOnWBhYYHAwEDExMTo94UTkV4w3BCRwYWFhWHNmjVYsmQJzp8/jw8++ACvvPIK9u/frzrn448/xrx583D8+HE4ODhgwIAByM/PB1AUSoYNG4aXXnoJZ8+exWeffYZPP/0Uq1atUrUPDQ3FunXrsGDBAly8eBFLly5FvXr11Or45JNPMG/ePJw4cQJGRkYYM2ZMlbx+ItIt7gpORAaVm5sLOzs77NmzBwEBAarjr7/+OrKysvDmm2/i2Wefxfr16zF8+HAAwP379+Hm5oZVq1Zh2LBhGDlyJJKTk7F7925V+0mTJmHHjh04f/48Ll++jObNmyM8PBxBQUElati3bx+effZZ7NmzB7179wYA/PXXX+jfvz+ys7NhZmam558CEekSe26IyKCuXLmCrKws9OnTB/Xq1VM91qxZg6tXr6rOezz42NnZoXnz5rh48SIA4OLFi+jSpYva83bp0gWxsbEoLCxEdHQ05HI5evToUW4t7dq1U/2/i4sLACApKempXyMRVS0jQxdARHVbRkYGAGDHjh1o2LCh2vdMTU3VAk5lmZuba3SesbGx6v8lSQJQNB6IiGoW9twQkUG1atUKpqamiIuLg7e3t9rD3d1ddd6RI0dU///gwQNcvnwZLVu2BAC0bNkShw8fVnvew4cPo1mzZpDL5Wjbti2USqXaGB4iqr3Yc0NEBmVlZYWPPvoIH3zwAZRKJbp27QqFQoHDhw/D2toajRo1AgB8/vnnaNCgAZycnPDJJ5/A3t4eISEhAIAPP/wQfn5+mDVrFoYPH47IyEgsXLgQP/zwAwDA09MTo0aNwpgxY7BgwQL4+Pjg5s2bSEpKwrBhwwz10olITxhuiMjgZs2aBQcHB4SFheHatWuoX78+OnbsiGnTpqluC82ePRsTJkxAbGws2rdvjz///BMmJiYAgI4dO2Ljxo2YMWMGZs2aBRcXF3z++ecYPXq06hqLFy/GtGnT8M477+DevXvw8PDAtGnTDPFyiUjPOFuKiKq14plMDx48QP369Q1dDhHVABxzQ0RERLUKww0RERHVKrwtRURERLUKe26IiIioVmG4ISIiolqF4YaIiIhqFYYbIiIiqlUYboiIiKhWYbghIiKiWoXhhoiIiGoVhhsiIiKqVf4fF9pZU7pglpUAAAAASUVORK5CYII=", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAGwCAYAAABVdURTAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABj0klEQVR4nO3dd3xTVf8H8M9N2qS7dO8FZRdKgVIKCAhYQGT6CCoK7gUK4gIeEdfPIoryqMhyIKKAg6UoyEagbAoUGWW2dFBKabpXcn5/lEZKV1KS3pJ+3q9XXsrtPbnf26bJp+ece64khBAgIiIishAKuQsgIiIiMiWGGyIiIrIoDDdERERkURhuiIiIyKIw3BAREZFFYbghIiIii8JwQ0RERBbFSu4CGppOp0NqaiocHR0hSZLc5RAREZEBhBDIzc2Fr68vFIra+2aaXLhJTU1FQECA3GUQERFRPSQnJ8Pf37/WfZpcuHF0dARQ/s1xcnKSuRoiIiIyRE5ODgICAvSf47VpcuGmYijKycmJ4YaIiOgOY8iUEk4oJiIiIovCcENEREQWheGmGrGxsYiMjISjoyM8PT0xYsQInD59utY2ffv2hSRJVR5DhgzR77Nq1SrExMTAzc0NkiQhPj7ezGdCRETU9DDcVGPHjh2YMGEC9u7di02bNqG0tBQxMTHIz8+vsc2qVauQlpamfyQkJECpVOKBBx7Q75Ofn49evXrhww8/bIjTICIiapKa3IRiQ2zYsKHSv5csWQJPT08cOnQIvXv3rraNq6trpX+vWLECdnZ2lcLNo48+CgC4ePGiaQsmIiIiPfbcGECj0QCoGmBq8/XXX+PBBx+Evb29ucoiIiKiarDnpg46nQ6TJ09Gz549ERYWZlCb/fv3IyEhAV9//bWZqyMiIqJbMdzUYcKECUhISMCuXbsMbvP111+jQ4cO6NatmxkrIyIioupwWKoWEydOxO+//45t27bVudRzhfz8fKxYsQJPPvmkmasjIiKi6rDnphpCCLz44otYvXo1tm/fjpCQEIPb/vzzzyguLsYjjzxixgqJiIioJgw31ZgwYQJ+/PFHrF27Fo6OjkhPTwcAODs7w9bWFgAwbtw4+Pn5ITY2tlLbr7/+GiNGjICbm1uV583KykJSUhJSU1MBQL92jre3N7y9vc15SkRERE0Gh6WqMX/+fGg0GvTt2xc+Pj76x8qVK/X7JCUlIS0trVK7vw8exa5duzDiwep7bdatW4eIiAj9wn4PPvggIiIisGDBAvOdDBERURMjCSGE3EU0pJycHDg7O0Oj0Zj0xpkrDyRh2qrj0AlAIQGxozpgTGSgyZ6fiIioKTPm85s9NyaQpinUBxsA0Alg+qoEpGkK5S2MiIioCWK4MYELmfn6YFNBKwQuZhbIUxAREVETxnBjAiHu9lBIlbcpJQnB7nbyFERERNSEMdyYgI+zLd4Z9u/qxQoJ+GBUGHycbWWsioiIqGliuDGRR6ODYK9SAgB+eKo7JxMTERHJhOHGhLycbeQugYiIqMljuDEhDwc1AOBqXrHMlRARETVdDDcm5OF4I9zkMtwQERHJheHGhCrCTUZukcyVEBERNV0MNybk6Vg+54Y9N0RERPJhuDEhDksRERHJj+HGhBhuiIiI5MdwY0KeDDdERESyY7gxoYqem6yCEpRqdTJXQ0RE1DQx3JiQi50KSoUEIYCs/BK5yyEiImqSGG5MSKmQ4GavAsChKSIiIrkw3JiYpxPXuiEiIpITw42J6W/BwJ4bIiIiWTDcmBgvByciIpIXw42JMdwQERHJi+HGxCpuwZDBcENERCQLhhsTY88NERGRvBhuTEwfbvIYboiIiOTAcGNivFqKiIhIXgw3JlbRc1NQokVecZnM1RARETU9soab2NhYREZGwtHREZ6enhgxYgROnz5da5slS5ZAkqRKDxsbmwaquG72aivYq5QA2HtDREQkB1nDzY4dOzBhwgTs3bsXmzZtQmlpKWJiYpCfn19rOycnJ6Slpekfly5daqCKDcNJxURERPKxkvPgGzZsqPTvJUuWwNPTE4cOHULv3r1rbCdJEry9vQ06RnFxMYqL/w0ZOTk59SvWCB6Oaly8VsBwQ0REJINGNedGo9EAAFxdXWvdLy8vD0FBQQgICMDw4cNx4sSJGveNjY2Fs7Oz/hEQEGDSmqvz71o3vL8UERFRQ2s04Uan02Hy5Mno2bMnwsLCatyvdevW+Oabb7B27VosW7YMOp0OPXr0wOXLl6vdf9q0adBoNPpHcnKyuU5Bj8NSRERE8pF1WOpmEyZMQEJCAnbt2lXrftHR0YiOjtb/u0ePHmjbti0WLlyI9957r8r+arUaarXa5PXWhuGGiIhIPo0i3EycOBG///47du7cCX9/f6PaWltbIyIiAmfPnjVTdcbTr3XDhfyIiIganKzDUkIITJw4EatXr8bWrVsREhJi9HNotVocP34cPj4+ZqiwfjycysNNRg7DDRERUUOTtedmwoQJ+PHHH7F27Vo4OjoiPT0dAODs7AxbW1sAwLhx4+Dn54fY2FgAwLvvvovu3bsjNDQU2dnZ+Oijj3Dp0iU89dRTsp3HrdhzQ0REJB9Zw838+fMBAH379q20/dtvv8Vjjz0GAEhKSoJC8W8H0/Xr1/H0008jPT0dLi4u6NKlC/bs2YN27do1VNl18rwx5+ZaXjG0OgGlQpK5IiIioqZDEkIIuYtoSDk5OXB2doZGo4GTk5NZjlGm1aHlm39CCGD/f/vrLw0nIiKi+jHm87vRXApuSayUCrjZ84opIiIiOTDcmAkvByciIpIHw42ZMNwQERHJg+HGTCqumMpguCEiImpQDDdm4unEnhsiIiI5MNyYCde6ISIikgfDjZlwzg0REZE8GG7MxJPhhoiISBYMN2bCnhsiIiJ5MNyYSUW4ySsuQ0FJmczVEBERNR0MN2bioLaCjXX5tzczt0TmaoiIiJoOhhszkSRJf0+pjNwimashIiJqOhhuzIjzboiIiBoew40Zca0bIiKihsdwY0bsuSEiImp4DDdmVLHWTUYOww0REVFDYbgxI33PDYeliIiIGgzDjRlxWIqIiKjhMdyYEcMNERFRw2O4MaOKdW4y84qh0wmZqyEiImoaGG7MyM1BBQAo0wlcL+AqxURERA2B4caMrJUKuNqXBxxOKiYiImoYDDdmpl/Ij/NuiIiIGgTDjZl5OnGtGyIioobEcGNmvAUDERFRw2K4MTNeDk5ERNSwGG7MjOGGiIioYTHcmFlFuMnILZK5EiIioqaB4cbM2HNDRETUsBhuzMyT4YaIiKhBMdyYmYdD+S0YcorKUFSqlbkaIiIiy8dwY2ZOtlZQWZV/m9l7Q0REZH4MN2YmSRLXuiEiImpADDcNgJOKiYiIGg7DTQNguCEiImo4DDcNwFO/1g3DDRERkbkx3DQA9twQERE1HIabBsBwQ0RE1HAYbhqAp2P5WjdXeQsGIiIis2O4aQDsuSEiImo4DDcNQB9u8oohhJC5GiIiIsvGcNMA3B1UAIBSrYCmsFTmaoiIiCwbw00DUFsp0czOGgAvByciIjI3hpsGor8FA8MNERGRWTHcNBBOKiYiImoYDDcNhOGGiIioYTDcNJB/b8HAtW6IiIjMieGmgbDnhoiIqGEw3DSQm9e6ISIiIvNhuGkgHg4Vt2BguCEiIjInhpsG4ulUMeeG4YaIiMicGG4aSMU6N9kFpSgu08pcDRERkeViuGkgzrbWsFZKAIBreSUyV0NERGS5GG4aiEIhwZ2rFBMREZmdrOEmNjYWkZGRcHR0hKenJ0aMGIHTp0/X2e7nn39GmzZtYGNjgw4dOuCPP/5ogGpv379r3TDcEBERmYus4WbHjh2YMGEC9u7di02bNqG0tBQxMTHIz8+vsc2ePXvw0EMP4cknn8SRI0cwYsQIjBgxAgkJCQ1Yef1wrRsiIiLzk4QQQu4iKly9ehWenp7YsWMHevfuXe0+Y8aMQX5+Pn7//Xf9tu7du6NTp05YsGBBlf2Li4tRXPxvmMjJyUFAQAA0Gg2cnJxMfxK1mLbqGJbvT8bLA1ph0oCWDXpsIiKiO1lOTg6cnZ0N+vxuVHNuNBoNAMDV1bXGfeLi4jBgwIBK2wYOHIi4uLhq94+NjYWzs7P+ERAQYLqCjaS/M3geb8FARERkLo0m3Oh0OkyePBk9e/ZEWFhYjfulp6fDy8ur0jYvLy+kp6dXu/+0adOg0Wj0j+TkZJPWbQwPp/KF/DJyOCxFRERkLlZyF1BhwoQJSEhIwK5du0z6vGq1Gmq12qTPWV//9tww3BAREZlLowg3EydOxO+//46dO3fC39+/1n29vb1x5cqVStuuXLkCb29vc5ZoEpxQTEREZH6yDksJITBx4kSsXr0aW7duRUhISJ1toqOjsWXLlkrbNm3ahOjoaHOVaTKeN4WbRjSPm4iIyKLI2nMzYcIE/Pjjj1i7di0cHR3182acnZ1ha2sLABg3bhz8/PwQGxsLAJg0aRL69OmDOXPmYMiQIVixYgUOHjyIRYsWyXYehqrouSku0yGnqAzOttYyV0RERGR5ZO25mT9/PjQaDfr27QsfHx/9Y+XKlfp9kpKSkJaWpv93jx498OOPP2LRokUIDw/HL7/8gjVr1tQ6CbmxsLFWwtGmPE9yaIqIiMg8ZO25MWRoZvv27VW2PfDAA3jggQfMUJH5eTiqkVtUhqu5xQj1dJC7HCIiIovTaC4Fbyp4xRQREZF5Mdw0ME/9WjdcyI+IiMgcGG4aGHtuiIiIzIvhpoFxrRsiIiLzYrhpYJ4MN0RERGbFcNPA2HNDRERkXgw3DYzhhoiIyLwYbhpYRbjJKihBqVYnczVERESWh+GmgbnaqaBUSBACuJZXInc5REREFofhpoEpFBLcHVQAODRFRERkDgw3MtDPu8njQn5ERESmxnAjA/1Cfuy5ISIiMjmGGxl4OlbcgoHhhoiIyNQYbmTw77AUww0REZGpMdzIgGvdEBERmQ/DjQwYboiIiMyH4UYGFfeXymC4ISIiMjmGGxnc3HMjhJC5GiIiIsvCcCMD9xuXgheWapFfopW5GiIiIsvCcCMDe7UV7FVKAJx3Q0REZGoMNzLxdKpY64arFBMREZkSw41M9KsUc60bIiIik2K4kQkvByciIjIPhhuZMNwQERGZB8ONTDy41g0REZFZMNzIhD03RERE5mF0uCksLERBQYH+35cuXcLcuXPx119/mbQwS8dwQ0REZB5Gh5vhw4dj6dKlAIDs7GxERUVhzpw5GD58OObPn2/yAi0Vr5YiIiIyD6PDzeHDh3HXXXcBAH755Rd4eXnh0qVLWLp0KT777DOTF2ipPJ3Kw821vGJodbwFAxERkakYHW4KCgrg6OgIAPjrr78watQoKBQKdO/eHZcuXTJ5gZbKzV4NhQToBHAtn703REREpmJ0uAkNDcWaNWuQnJyMjRs3IiYmBgCQkZEBJycnkxdoqZQKCa72nHdDRERkakaHm7feeguvvvoqgoODERUVhejoaADlvTgREREmL9CScVIxERGR6VkZ2+A///kPevXqhbS0NISHh+u39+/fHyNHjjRpcZbO01GNk2lc64aIiMiUjA43AODt7Q1vb28AQE5ODrZu3YrWrVujTZs2Ji3O0rHnhoiIyPSMHpYaPXo0vvjiCwDla9507doVo0ePRseOHfHrr7+avEBLxnBDRERkekaHm507d+ovBV+9ejWEEMjOzsZnn32G999/3+QFWjJPR651Q0REZGpGhxuNRgNXV1cAwIYNG3D//ffDzs4OQ4YMQWJioskLtGT6npschhsiIiJTMTrcBAQEIC4uDvn5+diwYYP+UvDr16/DxsbG5AVaMq5STEREZHpGTyiePHkyxo4dCwcHBwQFBaFv374AyoerOnToYOr6LBrn3BAREZme0eHmhRdeQLdu3ZCcnIx77rkHCkV550/z5s0558ZInk7lPV15xWUoKCmDnapeF68RERHRTer1adq1a1d07doVQggIISBJEoYMGWLq2iyevUoJW2slCku1uJpbjCA3hhsiIqLbZfScGwBYunQpOnToAFtbW9ja2qJjx474/vvvTV2bxZMkiUNTREREJmZ0V8Enn3yCGTNmYOLEiejZsycAYNeuXXjuueeQmZmJl19+2eRFWjIPRzWSsgoYboiIiEzE6HDz+eefY/78+Rg3bpx+27Bhw9C+fXu8/fbbDDdGqljrhrdgICIiMg2jh6XS0tLQo0ePKtt79OiBtLQ0kxTVlHBYioiIyLSMDjehoaH46aefqmxfuXIlWrZsaZKimhL9WjcMN0RERCZh9LDUO++8gzFjxmDnzp36OTe7d+/Gli1bqg09VDsP3oKBiIjIpIzuubn//vuxb98+uLu7Y82aNVizZg3c3d2xf/9+jBw50hw1WjRPp4o5N0UyV0JERGQZ6rWwSpcuXbBs2TJT19IkeTiUL+THYSkiIiLTMCjc5OTkGPyETk5O9S6mKaoYlsrMK4FOJ6BQSDJXREREdGczKNw0a9YMklT7h27FSsVardYkhTUVbg4qSBKg1QlcLyiB240JxkRERFQ/BoWbbdu2mbuOJstaqYCrnQrX8kuQkVvMcENERHSbDAo3ffr0MXcdTZqHoxrX8ktwNbcYbX3kroaIiOjOVq97S5FpcSE/IiIi05E13OzcuRNDhw6Fr68vJEnCmjVrat1/+/btkCSpyiM9Pb1hCjYT/UJ+XOuGiIjotskabvLz8xEeHo558+YZ1e706dNIS0vTPzw9Pc1UYcPwqFjrJofhhoiI6HbVa50bUxk8eDAGDx5sdDtPT080a9bMoH2Li4tRXPxvaDDmsvaGwp4bIiIi06lXz01ZWRk2b96MhQsXIjc3FwCQmpqKvLw8kxZXk06dOsHHxwf33HMPdu/eXeu+sbGxcHZ21j8CAgIapEZj/DvnhqsUExER3S6jw82lS5fQoUMHDB8+HBMmTMDVq1cBAB9++CFeffVVkxd4Mx8fHyxYsAC//vorfv31VwQEBKBv3744fPhwjW2mTZsGjUajfyQnJ5u1xvrghGIiIiLTMXpYatKkSejatSuOHj0KNzc3/faRI0fi6aefNmlxt2rdujVat26t/3ePHj1w7tw5fPrpp/j++++rbaNWq6FWN+61Yzwdy2/BkMFwQ0REdNuMDjd///039uzZA5VKVWl7cHAwUlJSTFaYobp164Zdu3Y1+HFNqaLnJreoDEWlWthYK2WuiIiI6M5l9LCUTqer9hYLly9fhqOjo0mKMkZ8fDx8fO7sle+cbKygsir/UXBoioiI6PYY3XMTExODuXPnYtGiRQAASZKQl5eHmTNn4t577zXqufLy8nD27Fn9vy9cuID4+Hi4uroiMDAQ06ZNQ0pKCpYuXQoAmDt3LkJCQtC+fXsUFRXhq6++wtatW/HXX38ZexqNiiRJcLVTIT2nCCfTcxDgaid3SURERHcso8PNnDlzMHDgQLRr1w5FRUV4+OGHkZiYCHd3dyxfvtyo5zp48CDuvvtu/b+nTJkCABg/fjyWLFmCtLQ0JCUl6b9eUlKCV155BSkpKbCzs0PHjh2xefPmSs9xJ1p5IAnpOeVXSj279BBm3d8BYyIDZa6KiIjoziQJIYSxjcrKyrBixQocO3YMeXl56Ny5M8aOHQtbW1tz1GhSOTk5cHZ2hkajgZOTk9zlIE1TiJ6ztkJ3009BKUnYNfVu+Dg3/u8nERFRQzDm87tei/hZWVnhkUceqVdxVNmFzPxKwQYAtELgYmYBww0REVE9GB1u1q1bV+12SZJgY2OD0NBQhISE3HZhTUWIuz0UEm7puQGC3TnvhoiIqD6MDjcjRoyAJEm4dTSrYpskSejVqxfWrFkDFxcXkxVqqXycbRE7qgOmrTquDziTB7Rirw0REVE9GX0p+KZNmxAZGYlNmzbpV/3dtGkToqKi8Pvvv2Pnzp24du2a2VcrtiRjIgOxe2o/tPUpv5Te1UFVRwsiIiKqSb1WKF60aBF69Oih39a/f3/Y2NjgmWeewYkTJzB37lw88cQTJi3U0vk42yKmnTdOpuVi3/ksjI0KkrskIiKiO5LRPTfnzp2rdpayk5MTzp8/DwBo2bIlMjMzb7+6JiYqxBUAsP9CVpVhPyIiIjKM0eGmS5cueO211/Q3zASAq1ev4vXXX0dkZCQAIDExsVHefbuxiwh0gbVSQnpOEZKyCuQuh4iI6I5kdLj5+uuvceHCBfj7+yM0NBShoaHw9/fHxYsX8dVXXwEoX3n4zTffNHmxls5WpURH/2YAgH0XsuQthoiI6A5l9Jyb1q1b459//sFff/2FM2fO6Lfdc889UCjKs9KIESNMWmRTEhXiikOXrmPf+SyM7sreLyIiImPVaxE/hUKBQYMGYdCgQaaup8nrFuKKL7efw/6L1+QuhYiI6I5Ur3CTn5+PHTt2ICkpCSUlJZW+9tJLL5mksKaqa7ArlAoJyVmFSM0uhG8zrndDRERkDKPDzZEjR3DvvfeioKAA+fn5cHV1RWZmJuzs7ODp6clwc5sc1FYI83XC0csa7LtwDSMj/OUuiYiI6I5i9ITil19+GUOHDsX169dha2uLvXv34tKlS+jSpQs+/vhjc9TY5HS76ZJwIiIiMo7R4SY+Ph6vvPIKFAoFlEoliouLERAQgNmzZ2P69OnmqLHJiQpxAwDsO89wQ0REZCyjw421tbX+qihPT08kJSUBAJydnZGcnGza6pqoyGBXSBJwPjMfGblFcpdDRER0RzE63ERERODAgQMAgD59+uCtt97CDz/8gMmTJyMsLMzkBTZFznbWaONdvgo0h6aIiIiMY3S4+eCDD+Dj4wMA+L//+z+4uLjg+eefx9WrV7Fo0SKTF9hUVdyKgUNTRERExjHqaikhBDw9PfU9NJ6entiwYYNZCmvqokJcsWTPRfbcEBERGcmonhshBEJDQzm3pgFUXDF1+kousvJL6tibiIiIKhgVbhQKBVq2bIlr17h6rrm5OagR6ukAADhwkb03REREhjJ6zs2sWbPw2muvISEhwRz10E0474aIiMh4Rq9QPG7cOBQUFCA8PBwqlQq2tpVvD5CVxQ9iU+kW4oof9iXxPlNERERGMDrczJ071wxlUHW6Ny9fzO+f1BzkFJXCycZa5oqIiIgaP6PDzfjx481RB1XDy8kGwW52uHitAAcvZqFfGy+5SyIiImr0jJ5zAwDnzp3Dm2++iYceeggZGRkAgD///BMnTpwwaXH071VT+3hJOBERkUGMDjc7duxAhw4dsG/fPqxatQp5eXkAgKNHj2LmzJkmL7Cp432miIiIjGN0uJk6dSref/99bNq0CSqVSr+9X79+2Lt3r0mLo397bhJSNMgvLpO5GiIiosbP6HBz/PhxjBw5ssp2T09PZGZmmqQo+leAqx38mtmiTCdwOOm63OUQERE1ekaHm2bNmiEtLa3K9iNHjsDPz88kRVFl3bjeDRERkcGMDjcPPvgg3njjDaSnp0OSJOh0OuzevRuvvvoqxo0bZ44am7yKxfx4nykiIqK61euu4G3atEFAQADy8vLQrl079O7dGz169MCbb75pjhqbvKgb693EJ2ejqFQrczVERESNm9Hr3KhUKixevBgzZsxAQkIC8vLyEBERgZYtW5qjPgIQ7GYHD0c1ruYWIz45W7+4HxEREVVldLjZtWsXevXqhcDAQAQGBpqjJrqFJEmICnHF78fSsO98FsMNERFRLYwelurXrx9CQkIwffp0/PPPP+aoiaqhn3fD+0wRERHVyuhwk5qaildeeQU7duxAWFgYOnXqhI8++giXL182R310Q8W8m0OXrqOkTCdzNURERI2X0eHG3d0dEydOxO7du3Hu3Dk88MAD+O677xAcHIx+/fqZo0YCEOrhABc7axSV6nA8JVvucoiIiBqtet1bqkJISAimTp2KWbNmoUOHDtixY4ep6qJbKBQS7zNFRERkgHqHm927d+OFF16Aj48PHn74YYSFhWH9+vWmrI1uwftMERER1c3oq6WmTZuGFStWIDU1Fffccw/+97//Yfjw4bCzszNHfXSTip6bQ5euo0yrg5XytjreiIiILJLR4Wbnzp147bXXMHr0aLi7u5ujJqpBWx8nONpYIbeoDP+k5aCjfzO5SyIiImp0jA43u3fvNkcdZAClQkJksCu2nsrAvvNZDDdERETVMDrcVPjnn3+QlJSEkpKSStuHDRt220VRzaJCboSbC1l4undzucshIiJqdIwON+fPn8fIkSNx/PhxSJIEIQSA8lV0AUCr5b2PzKlivZsDF7Og0wkoFJLMFRERETUuRs9InTRpEkJCQpCRkQE7OzucOHECO3fuRNeuXbF9+3YzlEg3a+/rBDuVEprCUpy+kit3OURERI2O0eEmLi4O7777Ltzd3aFQKKBQKNCrVy/ExsbipZdeMkeNdBNrpQJdglwAAPvO81YMREREtzI63Gi1Wjg6OgIoX604NTUVABAUFITTp0+btjqq1r/3meJ6N0RERLcyes5NWFgYjh49ipCQEERFRWH27NlQqVRYtGgRmjfnBNeGUDHvZv+FLAgh9POdiIiIqB7h5s0330R+fj4A4N1338V9992Hu+66C25ubli5cqXJC6SqOvo7Q22lQGZeCc5dzUOop6PcJRERETUaRoebgQMH6v8/NDQUp06dQlZWFlxcXNiD0EDUVkpEBDbD3vNZ2Hchi+GGiIjoJiZZv9/V1ZXBpoHxPlNERETV482J7lD6ScU35t0QERFROYabO1REoAuslRLSc4qQlFUgdzlERESNBsPNHcpWpdTfW4pDU0RERP9iuLmDVQxN7bvAcENERFRB1nCzc+dODB06FL6+vpAkCWvWrKmzzfbt29G5c2eo1WqEhoZiyZIlZq+zseqmDzdcqZiIiKiCrOEmPz8f4eHhmDdvnkH7X7hwAUOGDMHdd9+N+Ph4TJ48GU899RQ2btxo5kobp67BrlAqJFy+XoiU7EK5yyEiImoUjF7nxpQGDx6MwYMHG7z/ggULEBISgjlz5gAA2rZti127duHTTz+ttP5OU+GgtkKYrxOOXtZg/4VrGBnhL3dJREREsruj5tzExcVhwIABlbYNHDgQcXFxNbYpLi5GTk5OpYcl6XbTJeFERER0h4Wb9PR0eHl5Vdrm5eWFnJwcFBZWPywTGxsLZ2dn/SMgIKAhSm0wXMyPiIiosjsq3NTHtGnToNFo9I/k5GS5SzKpyGBXSBJwPjMfGTlFcpdDREQkuzsq3Hh7e+PKlSuVtl25cgVOTk6wtbWtto1arYaTk1OlhyVxtrNGG+/yc1q27xLSNJxYTERETdsdFW6io6OxZcuWSts2bdqE6OhomSpqHFzsrAEAn205i56ztmLlgSSZKyIiIpKPrOEmLy8P8fHxiI+PB1B+qXd8fDySkso/nKdNm4Zx48bp93/uuedw/vx5vP766zh16hS+/PJL/PTTT3j55ZflKL9RSNMUIu7cv+vc6AQwfVUCe3CIiKjJkjXcHDx4EBEREYiIiAAATJkyBREREXjrrbcAAGlpafqgAwAhISFYv349Nm3ahPDwcMyZMwdfffVVk7wMvMKFzHzcettMrRC4mMn7TRERUdMkiSZ2S+mcnBw4OztDo9FYxPybNE0hes7aCt1NP0WlJGHX1Lvh41z9PCQiIqI7jTGf33fUnBuqysfZFrGjOkCS/t32wagwBhsiImqyGG4swJjIQPzx0l1Q3Ag4kcGu8hZEREQkI4YbC9HWxwm9WnoAAH4/liZzNURERPJhuLEg93X0AQD8fixV5kqIiIjkw3BjQQa284a1UsKZK3k4cyVX7nKIiIhkwXBjQZztrNG7YmjqKHtviIioaWK4sTD3hVcMTaWhiV3lT0REBIDhxuIMaOsFlZUC5zPz8U9ajtzlEBERNTiGGwvjaGONu1vzqikiImq6GG4s0H0dfQGUXzXFoSkiImpqGG4sUP+2nrC1ViI5qxDHLmvkLoeIiKhBMdxYIDuVFfq19QTANW+IiKjpYbixUENvLOi3/lgadDoOTRERUdPBcGOh+rb2hL1KiVRNEY4kX5e7HCIiogbDcGOhbKyVuKedFwBeNUVERE0Lw40Fq7hq6o/jHJoiIqKmg+HGgt3Vyh2ONla4klOMAxez5C6HiIioQTDcWDC1lRID23sD4NAUERE1HQw3Fu6+G1dN/ZmQhjKtTuZqiIiIzI/hxsL1DHWHi501MvNKsO8Ch6aIiMjyMdxYOGulAoPCKoamuKAfERFZPoabJqDiqqk/E9JRyqEpIiKycAw3TUBUiCvcHVTILijF7rOZcpdDRERkVgw3TYCVUoHBYeUTi3nVFBERWTqGmyai4qqpjSfSUVymlbkaIiIi82G4aSIig13h5aRGblEZ/j7DoSkiIrJcDDdNhEIh4d4OFUNTvGqKiIgsF8NNE1Jx1dSmf66gqJRDU0REZJkYbpqQzoHN4NfMFvklWmw/nSF3OURERGbBcNOESJKEITcmFv/Gq6aIiMhCMdw0MRVXTW09mYGCkjKZqyEiIjI9hpsmpoOfMwJd7VBYqsWWkxyaIiIiy8Nw08RIkqTvveFVU0REZIkYbpqgiqumtp2+ityiUpmrISIiMi2GmyaorY8jmnvYo6RMh80nr8hdDhERkUkx3DRB5UNT5b03vx/lVVNERGRZGG6aqKE35t3sTLwKTQGHpoiIyHIw3DRRLb0c0drLEaVagY3/pMtdDhERkckw3DRh/141xaEpIiKyHAw3Tdh94eXzbnafzURWfonM1RAREZkGw00TFuJuj/a+TtDqBDYkcGiKiIgsA8NNE1dx1dQP+y4hTVMoczVERES3j+GmyRMAgBOpOeg5aytWHkiSuR4iIqLbw3DThKVpCvHRxtP6f+sEMG3VcfbgEBHRHY3hpgm7kJkPnai8TSeA+ORsWeohIiIyBYabJizE3R4Kqer2GWsScOjS9YYviIiIyAQYbpowH2dbxI7qAKVUnnAUEuDpqEZmXgkeXBSHZXsvQQhRx7MQERE1LpJoYp9eOTk5cHZ2hkajgZOTk9zlNAppmkJczCxAsLsdHG2s8fovR/HH8fJLwx/o4o/3RoTBxlopc5VERNSUGfP5zXBDVQghsGjneXy44RR0Aujg54z5j3SGv4ud3KUREVETZcznN4elqApJkvBsnxZY+kQUXOyscTxFg6Gf78KuxEy5SyMiIqoTww3VqFdLd/z2Yi908HPG9YJSjPtmHxbsOMd5OERE1Kgx3FCt/F3s8PNz0Xigiz90Apj15ylM+PEw8orL5C6NiIioWgw3VCcbayVm/6cj3h8RBmulhD+Op2PkvN04fzVP7tKIiIiqYLghg0iShEe6B2HFM9HwclIjMSMPw7/YjU3/XEGaphB7zmVyZWMiImoUGkW4mTdvHoKDg2FjY4OoqCjs37+/xn2XLFkCSZIqPWxsbBqw2qatS5ALfnuxF7oFuyK3uAxPLz2IHrFb8fDifbw3FRERNQqyh5uVK1diypQpmDlzJg4fPozw8HAMHDgQGRkZNbZxcnJCWlqa/nHp0qUGrJg8HW3ww9NReKCLP4CKW2+W37ph+qoE9uAQEZGsZA83n3zyCZ5++mk8/vjjaNeuHRYsWAA7Ozt88803NbaRJAne3t76h5eXVwNWTABgrVRgZGe/Ktu1QuBiZoEMFREREZWTNdyUlJTg0KFDGDBggH6bQqHAgAEDEBcXV2O7vLw8BAUFISAgAMOHD8eJEydq3Le4uBg5OTmVHmQaNd2bys1B1fDFEBER3SBruMnMzIRWq63S8+Ll5YX09PRq27Ru3RrffPMN1q5di2XLlkGn06FHjx64fPlytfvHxsbC2dlZ/wgICDD5eTRVt96bqsKrPx9FdkGJTFUREVFTJ/uwlLGio6Mxbtw4dOrUCX369MGqVavg4eGBhQsXVrv/tGnToNFo9I/k5OQGrtiyjYkMxK6pd2P5093x3eORcLGzxu51P8DHP8igCeI3W7FiBSRJwogRIyptF0Lgrbfego+PD2xtbTFgwAAkJiaa4WyIiMgSyBpu3N3doVQqceXKlUrbr1y5Am9vb4Oew9raGhERETh79my1X1er1XBycqr0INPycbZFdAs39GntiXFeqbi+9WvYdx+DiJcWolW7sDoniAPAxYsX8eqrr+Kuu+6q8rXZs2fjs88+w4IFC7Bv3z7Y29tj4MCBKCoqMtcpERHRHUzWcKNSqdClSxds2bJFv02n02HLli2Ijo426Dm0Wi2OHz8OHx8fc5VJRljxzXyMHf84WvS6D2kKd1xuOxY2tra1ThDXarUYO3Ys3nnnHTRv3rzS14QQmDt3Lt58800MHz4cHTt2xNKlS5Gamoo1a9aY+WyIiOhOJPuw1JQpU7B48WJ89913OHnyJJ5//nnk5+fj8ccfBwCMGzcO06ZN0+//7rvv4q+//sL58+dx+PBhPPLII7h06RKeeuopuU6BbqiYIH7/0MH46dlo+DWzxYWsQuh8OmDrzl01tnv33Xfh6emJJ598ssrXLly4gPT09EqTzp2dnREVFVXrpHMiImq6rOQuYMyYMbh69SreeustpKeno1OnTtiwYYN+knFSUhIUin8z2PXr1/H0008jPT0dLi4u6NKlC/bs2YN27drJdQp0w80TxIPc7LHy2e54ePE+HLVyQNzxf3DpWj6C3Owrtdm1axe+/vprxMfHV/ucFRPLjZl0TkRETZvs4QYAJk6ciIkTJ1b7te3bt1f696effopPP/20Aaqi2+XvYoefno1Gtz8W4opWh9EL4/Dj093RwsMBAJCbm4tHH30Uixcvhru7u8zVEhGRpWgU4YYsQ3UTxL2dbRDta4VtmR64klOMMQv34oenotDa2xHnzp3DxYsXMXToUP3+Op0OAGBlZYXTp0/rJ5ZfuXKl0ryqK1euoFOnTg1zYkREdEeRfc4NWY6aJojv3rkdz40ejHY+TsjMK8aDi+KQkKJBmzZtcPz4ccTHx+sfw4YNw9133434+HgEBAQgJCQE3t7elZ4zJycH+/btM3jSORERNS3suSGTmjJlCsaPH4+uXbuiW7dumDt3LvLz8zHh2aegdnRBh77DcF7piId1At8/GYXwsLBK7Zs1awYACLtp++TJk/H++++jZcuWCAkJwYwZM+Dr61tlPRwiIiKA4YZMrK4J4sE2hUgqVSCnqAxjv9qHJY9Hws/FFhcy8xHibl/tc77++uvIz8/HM888g+zsbPTq1QsbNmzg3eCJiKhakhBC1L2b5cjJyYGzszM0Gg0X9JNJfnEZnvzuAPaez4JKKaFUJyAEoJCA2FEdMCYyUO4SiYiokTHm85tzbqjB2aut8O1j3dAt2AUl2vJgAwA6AUxflYA0TaG8BRIR0R2N4YZkYatS4oW7Q6ts1wqBb3dfgKawVIaqiIjIEnDODcmmtbcjFFJ5j83NFu28gCV7LuGetl4YEeGHPq08oLJiDiciIsNwzg3JauWBJExflQCtEFBIQEx7L5zLyEdiRp5+Hxc7a9zX0RcjIvzQObAZJEmSsWIiIpKDMZ/fDDckuzRNIS5mFiDY3Q4+zrYQQuCftBysPpyCtUdTcTW3WL9vkJsdRnTyw4gIP4S42yNNU6i/0srH2VbGsyAiInPihGK6o/g42yK6hZs+nEiShPa+znjzvnbYO60/lj7RDaMi/GCnUuLStQL8b0si7v54O3rP3ooesVvx8OJ96DlrK1YeSJL5TG7PvHnzEBwcDBsbG0RFRWH//v017rt48WLcddddcHFxgYuLCwYMGFBl/7y8PEycOBH+/v6wtbVFu3btsGDBAnOfBhGR7BhuqFFTKiT0buWBT8Z0wsE3B2DumE7o08oDEoCkrEJUdDuWX2l1/I690mrlypWYMmUKZs6cicOHDyM8PBwDBw5ERkZGtftv374dDz30ELZt24a4uDgEBAQgJiYGKSkp+n2mTJmCDRs2YNmyZTh58iQmT56MiRMnYt26dQ11WkREsuCwFN2R/jieihd+OFJl+/yxnTG4g081LRq3qKgoREZG4osvvgBQftuKgIAAvPjii5g6dWqd7bVaLVxcXPDFF19g3LhxAMpXeR4zZgxmzJih369Lly4YPHgw3n//ffOcCBGRmXBYiixeRKALFNXMK37j12P4/Vhqwxd0G0pKSnDo0CEMGDBAv02hUGDAgAGIi4sz6DkKCgpQWloKV1dX/bYePXpg3bp1SElJgRAC27Ztw5kzZxATE2PycyAiakwYbuiO5ONsi9hRHaC8ceWUQgJ8m9kgp6gME388gpeWH0F2QYnMVRomMzMTWq1Wf4uKCl5eXkhPTzfoOd544w34+vpWCkiff/452rVrB39/f6hUKgwaNAjz5s1D7969TVo/EVFjw3Vu6I41JjIQvVt56K+0cndQ4/OtZzFv21msO5qKfReuYfZ/wtGnlYfcpVar4kov29Lbmyc0a9YsrFixAtu3b690v63PP/8ce/fuxbp16xAUFISdO3diwoQJVUIQEZGlYbihO5qPs22lS8Cn3NMK/dp4YspP8Th/NR/jv9mPR7oHYvq9bWGnajwv9292XcB7v/8DAUDSlUKhVOLKlSuV9rly5Qq8vb1rfZ6PP/4Ys2bNwubNm9GxY0f99sLCQkyfPh2rV6/GkCFDAAAdO3ZEfHw8Pv74Y4YbIrJoHJYii9MpoBnWv3gXHusRDABYtjcJg//3Nw5dypK1rlKtDpv+uYLx3+zDuzeCDQAIhTWsPFtg9jc/42hyNrQ6AZ1Ohy1btiA6OrrG55s9ezbee+89bNiwAV27dq18rNJSlJaWQqGo/CuuVCqh0+lMfWpERI1K4/lTlsiEbFVKvD2sPe5p54VXfz6KS9cK8MCCODzXpwUmD2jVoLdz+Cc1B78cuoy18Sm4ll/9PCCnyBGIW/8p+j/nBffm7aA88Seu5+QiZsQYAMC4cePg5+eH2NhYAMCHH36It956Cz/++COCg4P1c3McHBzg4OAAJycn9OnTB6+99hpsbW0RFBSEHTt2YOnSpfjkk08a5sSJiGTCS8HJ4mkKS/HObyew6nD5GjBtfZzwyehwtPUx388/M68Ya+NT8cuhyziZlqPf7u6gwj1tvbDiYDJu/s2TJMAjaRvi13+P0rwsqDybw3XAs1D7tkaIuz3OL3kNrUKb49cVy+BkY42AwCBcTq66aOHMmTPx9ttvAwDS09Mxbdo0/PXXX8jKykJQUBCeeeYZvPzyy7yFBRHdcXj7hVow3DRdGxLSMH11ArLyS6BSKjAlphXu6+iDpKyCet2+4dZbP5SU6bD1VAZ+OXQZ209noOzGHUFVSgUGtPPE/Z390buVB6yVikr31FJKEj4YFYYxkYEo0+pw9HI2/k7MxK7ETBy5MUxVQamQ4N/MFklZBRAov0osdlQHjIkMNOW3ioio0WG4qQXDTdN2NbcY01Ydx+aTlSfvSgDG9wjCoDAfOKitYK+2gr1aCQe1FWytlVV6OlYeSMK0VcehE+W9LtHN3XAyLQfXC0r1+4T7O+M/XfwxNNwXzexUVWq59Z5a1ckpKsXec9ew62x52DmfmV9lH6UkYdfUu3lvLSKyaAw3tWC4ISEEFv99Hh/8ccqg/RUSYK8qDzwONlawVioqDTXdzNNRjZGd/fCfzv5o6eVoyrIBAOviU/DSivgq25c/3R3RLdxMfjwiosbCmM9vTiimJkeSJIT5OVf7NR9nG+iEQH6xFvklZRCi/L5VucVlyC0uA6rPNACANwa1xtN3NYeV0nyTlSNDXKGQymu6mRkPSUR0x2G4oSYpxN2+SkhQShJWvdBDP7yj0wkUlmqRV1yGvOIy5N/4b3JWIab+egw35wulJGFEhJ9Zgw3w78rMFfN1KkxddRyrX+gJZ1trsx6fiOhOwHBDTdKtIaFiUu/N81YUCunG3BsrVLoxQgsAELW2NaebV2Z2sFHi2aWHcP5qPl5cfgTfjO9q9oBFRNTYcc4NNWmGTOo1R1tTSkjR4IEFcSgs1eLxnsGYObS9bLUQEZkL7wpOZCAfZ1tEt3CrVzi5nbamFObnjE9GhwMAvt19Ecv3V13/xpLNmzcPwcHBsLGxQVRUFPbv31/jvidOnMD999+P4OBgSJKEuXPn1vrcs2bNgiRJmDx5smmLJiKzYrghsgCDO/hgyj2tAAAz1iRg7/lrMlfUMFauXIkpU6Zg5syZOHz4MMLDwzFw4EBkZGRUu39BQQGaN2+OWbNm1XnfrgMHDmDhwoWV7tlFRHcGhhsiC/Fiv1AMDfdFmU7g+WWHkHStQO6SzO6TTz7B008/jccffxzt2rXDggULYGdnh2+++aba/SMjI/HRRx/hwQcfhFqtrvF58/LyMHbsWCxevBguLi7mKl82pu7tmj9/Pjp27AgnJyc4OTkhOjoaf/75pxnPgKh2DDdEFkKSJHz0n47o6O+M6wWlePK7A8gtKq274R2qpKQEhw4dqnSHc4VCgQEDBiAuLu62nnvChAkYMmSIRd493Ry9Xf7+/pg1axYOHTqEgwcPol+/fhg+fDhOnDhhzlMhqhHDDZEFsbFWYvG4rvByUiMxIw+TVsRXun2DJcnMzIRWq4WXV6Vr2eDl5aW/kWh9rFixAocPH9bfpNTSmKO3a+jQobj33nvRsmVLtGrVCv/3f/8HBwcH7N2715ynQlQjhhsiC+PlZINFj3aF2kqBracy8OEGw1ZiJiA5ORmTJk3CDz/8ABsbG7nLMTlz9nZV0Gq1WLFiBfLz8xEdHW2S5yQyFsMNkQUKD2iGjx8ov4Jq0c7z+PlgsswVmZ67uzuUSiWuXKl8n7ArV67UOVm4JocOHUJGRgY6d+4MKysrWFlZYceOHfjss89gZWUFrVZritJlY67eLgA4fvw4HBwcoFar8dxzz2H16tVo167dbT0nUX0x3BBZqKHhvnipXygA4L+rE3DwYpbMFZmWSqVCly5dsGXLFv02nU6HLVu21LvHoH///jh+/Dji4+P1j65du2Ls2LGIj4+HUqk0VfkWp3Xr1oiPj8e+ffvw/PPPY/z48fjnn3/kLouaKK5QTGTBJg9ohcSMPPyZkI5nvz+EtRN7wt/FTu6yTGbKlCkYP348unbtim7dumHu3LnIz8/H448/DgAYN24c/Pz89PNnSkpK9B+4JSUlSElJQXx8PBwcHBAaGgpHR0eEhYVVOoa9vT3c3NyqbL8TmaO3q4JKpUJoaHmY7tKlCw4cOID//e9/eHv2XFzIzEeIu73sa0JR08GeGyILplBImDM6HO18nHAtvwRPfXcQ+cVlcpdlMmPGjMHHH3+Mt956C506dUJ8fDw2bNigH3ZJSkpCWlqafv/4U+cRERGBiIgIpKWl4eOPP0ZERASeeuops9eapinEnnOZSNMUmv1YNTFHb1dNdDodTqVkoeesrXh48T70nLUVKw80rQUmST68/QJRE5CaXYhhX+xGZl4x7mnnhYWPdIFCIcldVoNaeSAJ01Ydh04ACgmIHdUBYyIDLf7YVWpZuRLjx4/HwoUL9b1dP/30E06dOgUvL69ae7vuvfdejB07FmPHjtX3dgHAtGnTMHjwYAQGBiI3NxeLvl2KeXPnwPOBd2ETEqE/tkICdk/txx4cqhdjPr8ZboiaiMNJ1/Hgor0oKdNhXPcgDOrgLctQQZqmsMGHKdI0heg5a2ulu8A3xAetEAIbT6TjuWWHK21XSMD21/oi0NXebMeuzRdffIGPPvoI6enp6NSpEz777DNERUUBAPr27Yvg4GAsWbIEAHDx4kWEhIRUeY4+ffpg+/bt0BSWYuy4xxD39w5kX8uAQm0PK/cgOEX9B7Y3BZsKz/dpgVcHtoayiYVrun0MN7VguKGmbPWRy3h55VH9vyUJeG94GB7pHtQgx5ejB0OnE3hrbQKW7as6JBLkaodHo4MwrJMvPB1Nd+n35esFWH04Bb8cvoxLNawU7WRjjYe6BWB0ZABaeDiY7NimduvP7Pk+LeDhqMaxyxrEX87G+av5VdooFRKau9sjMSOv2uds6emAV2JaY2B7L0gSQw4ZhuGmFgw31JSlaQrRI3Yrbv2lb+Fhj86BLgjzc0aYnxPa+jjBTnX71xtczy/Buat5OJuRh6PJ2Vh+oPIl6UpJwq6pd5ut9yRNU4hXfjqKPedqv9eWQgLuaumBUZ39ENPOG7Yq46+KKigpw4aEdPxy6HKl49lYK1BUqqu1bWSwC0Z3DcCQjj4m+b6bSnJWPnp/tB11fUoEutohPKAZwv2d0SmgGdr7OsNWpcTKA0mYvioBWiGgkICB7b2x59w1aArLV84OD2iG1we2Rs9Q9wY4m9s3b948fY9XeHg4Pv/8c3Tr1q3G/X/++WfMmDEDFy9eRMuWLfHhhx/i3nvv1X/9sccew3fffVepzcCBA7FhwwazncOdjOGmFgw31JTtOZeJhxfvq3M/hQS08HC4EXacEebrhHa+TnC0sa4yrKTTCaRqCnHuaj7OZpQHmXNX83AuIw/X8kvqPNYXD0XgvnBfU5xeJeuOpuLN1ceRU1QGW2slBoV5YV18KrSiPFT9d0hbWFspsPrwZRxOyta3s1cpMSjMB6M6+6F7c7dah090OoEDF7Pwy6HL+ON4GvJL/l0HJ7q5G/7TxR+Dwrzx+7FU/Ye8UpLw3oj2cHNQ46cDydh2OkM/XOagtsLQcF+MiQxAuL+zLL0axWVa7Dl7DX8mpOGP42nIK666tk9EQDP0be2JjgHOCPdvBld7VY3Pl6YpxMXMAgS728HH2RaawlIs3nke3+y+gIIb36+eoW54NaY1IgIb7328Vq5ciXHjxmHBggWIiorC3Llz8fPPP+P06dPw9PSssv+ePXvQu3dvxMbG4r777sOPP/6IDz/8EIcPH9ZfeffYY4/hypUr+Pbbb/Xt1Gq17Pczk2Po2BAMN7VguKGmrKa5J7EjOyAluxAJqTlISNEgI7e42vbuDipk5v0bWHyb2eB6fikKS2te3M6vmS2ae9jDx9kGPx+8XKXXSG0l4dneLfBsnxawV99+r4WmsBRvrU3A2vhUAEC4vzM+HdMJzT0cqnzQVriYmY/VR1Kw+kgKkrL+HUbydrLB8AhfjIrwR2tvR/2bvtpKgb8TM/Hr4ctIzvr36qdAVzv8p4s/Rkb4IcC18iX3NR07XVOEXw9fxsoDyZWO3cbbEaO7BmBkhB9c7FVm/cApLNFix5mr2JCQhi0nM5BbyxV1SgnYZYK5SldzizFv21n8uC8JJdrynq2Ydl54JaY1Wns73tZzm0NUVBQiIyPxxRdfACi/GiwgIAAvvvgipk6dWmX/MWPGID8/H7///rt+W/fu3dGpUycsWLAAQHm4yc7Oxpo1axrkHAzRmCa/34rhphYMN9TU3TxUoJQkfDAqrMqbV0ZOERJSNUhIycHxFA1OpGiQqimq8TmtFBKC3e0R6uGAUE8HtPC0R6iHI5p72FcKLLcOUwS52uPCtfI5G56Oarw2sDXu7+xf7yu59pzLxKs/HUWqpghKhYQJd4fixX6hsFYatuqFEAKHLl3HqiMp+P1oKnKK/v2Q93G2QbqmqEo4c1BbYUgHH/ynqz+6BrnUu7dFpxPYe+EafjqQjD8S0lFSVv6Br1Iq0MbHEcdTNBD1/MCpLhjlFZdh66kMbEhIw7ZTVysFVE9HNQa298bgMG9cvJaPGWtO1Pp6uR2Xrxdg7uZErDp8GTpRPg9sZCc/TB7QCoFudrcV6kwVCEtKSmBnZ4dffvkFI0aM0G8fP348srOzsXbt2iptAgMDMWXKFEyePFm/bebMmVizZg2OHi2f9/bYY49hzZo1UKlUcHFxQb9+/fD+++/Dzc2t3rXejur++DH30LExGG5qwXBDVHMvQm02JKRVueoHAOaMDsewcF+DA8TNx/Z2ssGGhHR88OdJfQ9ImJ8TZgxph6jmhr/BF5dpMeevM1j893kIAQS52eHTMZ3Q+TaGOYrLtNh2KgOrDqdg66krKKtm2szbQ9thTGRgvebo1EZTUIq1R1OwYn8y/knLqXafiMBmCHCxg6ejGp5Oang62vz7/042cFRbQZKkKn+J39/FH9fzS7AzMVMfoIDyHrZBYeWBpnOgS6WAWZ/Xi7HOZuRizl9n8GdC+W0grJUSugS5YP+FrFp7Ecq0OhSUalFUokVh6Y1HiRZ/HE/DV7su1DsQ3iw1NRV+fn7Ys2dPpfWAXn/9dezYsQP79lUd6lWpVPjuu+/w0EMP6bd9+eWXeOedd/SLKK5YsQJ2dnYICQnBuXPnMH36dDg4OCAuLq5BV8MWQmDv+Sx8vjWx2vlpT/QMxqT+reBsZ91gNVWH4aYWDDdE9WPOv+qKy7RYsvsiPt96Fnk3hkQGh3lj2uC2CHSrfUXlU+k5mLwiHqfScwEAD3ULwJtD2plkiKvCXyfS8cz3h6psX/50d0S3MO9f2T/uu4TpqxOMbmdjrYCrvQqp2TX3uIW42+sDTQc/eeb43OrY5Wx8tPE0/k7MrPbrfs1sUKoVKCzVoqhUi1Kt4R9hj0UH496OPogIbGZwGAfMF25udf78ebRo0QKbN29G//79Da6vvjJyivDL4cv46UAyLtZwVV8FlZUCg9p7Y3TXAPRo4SbLOlnGfH43nmn5RNSo+TjbInZUhypDWqb4S15tpcSzfVrg/i7++GTTGazYn4Q/E9Kx5WQGHu8ZjAn9QuFkU/mvRp1O4JvdFzB7w2mUaHVws1dh1v0dcU87rxqOUn8d/J2hkFAl2AW7m/9WFne38axybIUEzBzaDiVlAhm5RcjILUZGTrH+/3OLylBUqqsx2Nzf2Q9P926O1l6OjSLQ3KyjfzN8/2QUvvr7PN5ff7LK11NqOCdJAuyslfpetJvnhlVYEncRS+IuwlFthegWbujdygN9WnlUmR91q/rctsLb29vo21w0b94c7u7uOHv2rNnCTZlWh52JV7F8fzK2nsqA9sYLy16lxLBOfnBzUGH+trPQ3ujxGtLBB4kZeTiVnot1R1Ox7mgq/JrZ4j9d/PGfLv51fu/kwp4bIjJKQwxRnErPwf+tP6n/693NXoWX72mFByMDcDWvGAcvXseSPRdw6FI2AKBfG098eH9HeDiqzVIPYNhcpcZy7MISLTJyi3AyLQfPLztcaZ5QY5pDUZuaJr8veKQz/FzsYHsjyNhaK2FjrYTaSqEPatW1lSTgnrZeOHjpOrJuuYqvubs9erfyQO9W7uje3E1/Of7Nc3ZGxPRFt27d8PnnnwMon1AcGBiIiRMn1jihuKCgAL/99pt+W48ePdCxY0f9hOJbXb58GYGBgVizZg2GDRtWr+9bTZKzCvDTwWT8fPAy0nP+DYhdglwwJjIAQzr46Hs7b/0dF0IgISUHPx1Mxpr4FOTeNBetZ6gbRncNwMD23rCxVurbm2PyO4elasFwQ3RnEEJg2+kMvL/+pH6hOC8nNTJyivUf1tZKCW8Pa4+HuwU2SA9EQwQ7Ux9bzlB2u26n9pra6nQCCaka7DxzFTvPZOJQ0nV97wVQPoE7MsQFzjbW+PNEun7OzlDHi1j47isG37Ziz5496NOnD2bNmoUhQ4ZgxYoV+OCDD/SXgufl5eGdd97B/fffD29vb5w7dw6vv/46cnNzcfz4cajVtxfU0zSFOHMlF0lZBfjrxBXsOpupX6/Ixc4aozr748HIALT0Mu7KtKJSLTaeSMfPBy9j19l/hw6dbKwwvJMfmtlZY962s2a52orhphYMN0R3llKtDj/svYRPNp2pdPUSwHsVGUrOUHa7bqd2Q9rmFJViz9lr2Jl4FTvPXMXl6zXf2DQofSeO/7kMudcz0bJtGN778GPc2683bFXKKretAIBF3/2A99+ZiSspyWjZsiVmz56tX8SvsLAQI0aMwJEjR5CdnQ1fX1/ExMTgvffe09/41RhlWh3SNEVIvl6AXw9dxqrDKVWu7LurpTvGRAbgnnZeUFvd/oTl5KwC/HLoMn45dBkp2dV/30zZU8hwUwuGG6I706YT6Xhapkm91DQIIXAhMx9L9lzE0rhLBrdrZmcNX2db+DazgY+zLXya2SDpWgFWHkzW9/xMv7ctxkYFwcZaYVAv461DO0IIaApLkZRVgOSsQiRlFdz4/wIkXy9AyvVClOmq/ziXAPzyfDS6BLkafE7G0OkE9py7hi+3n632aitT/Y5yQjERWZwwGSf1UtMgSRKaezjg+b4tsGzvpSpzdp7qFYK84jKkZBchLbsQaZoi5BWXIbugFNkFpTVetq8TwPvrT+L99SehkAB7lRXs1ErYq6xgr7aCnUoJe3X5/9urlLh8vRC7z2bqe158nG2QV1xWaa5LdVRKBdwcVEi7ZU0qAaCkzHz9GAqFhF4t3dHC077aKyrl+B1luCGiO4I5r9YiullNr7Xq5o7kFJUiNbsQadlFSNWU//docjb+Plv9pew6AeQWl91YBbr6lcBvdXNY8XRUI8DVDoGudvr/lv+/LbwcbXAlt0i2gNGYfkc5LEVEd5Q7ef4I3Vnq+1qrfk0oYOPLfeBkY4X8Ei3yi8uQX1yGghIt8kvKbvxbi1PpOfjp4OUqz/nxfzpiSEdfgxaMlHsSubl+R++4YSlT32mViCyXj7MtQw01iPq+1mrqwQj1dKizbZqmEL8culyl56VnS3eDV8IeExmI3q08ZPsjoDH8jhq+RKOZrFy5ElOmTMHMmTNx+PBhhIeHY+DAgcjIyKh2/z179uChhx7Ck08+iSNHjmDEiBEYMWIEEhKMX8GTiIjIHMZEBmLX1Lux/Onu2DX1boN7TiqCkfLGpOP6Du34ONsiuoWb7CFDLrIPS5njTqu14bAUERE1dhx+reqOGZYqKSnBoUOHMG3aNP02hUKBAQMGIC4urto2cXFxmDJlSqVtAwcOrPGW8cXFxSgu/nfSlkajAVD+TSIiImqM7CWgvYc1gFLk5JTKXU6jUPG5bUifjKzhJjMzE1qttsqCRV5eXjh16lS1bdLT06vdPz09vdr9Y2Nj8c4771TZHhAQUM+qiYiISC65ublwdnaudZ9GMaHYnKZNm1app0en0yErKwtubm4mX649JycHAQEBSE5ONnrI63ba3snHvt32PHbTOvbttuexeew7pX1TPXZthBDIzc2Fr69vnfvKGm4a4k6rarW6yj06mjVrVv+iDeDk5FTvH+jttL2Tj3277XnspnXs223PY/PYd0r7pnrsmtTVY1NB1qulVCoVunTpgi1btui36XQ6bNmyBdHR0dW2iY6OrrQ/AGzatKnG/YmIiKhpkX1YasqUKRg/fjy6du2qv9Nqfn4+Hn/8cQCocqfVSZMmoU+fPpgzZ47+TqsHDx7EokWL5DwNIiIiaiRkDzdjxozB1atX8dZbbyE9PR2dOnXChg0b9JOGk5KSoFD828HUo0cP/Pjjj3jzzTcxffp0tGzZEmvWrEFYWJhcp6CnVqsxc+bMet2q/nba3snHvt32PHbTOvbttuexeew7pX1TPbapyL7ODREREZEpyb5CMREREZEpMdwQERGRRWG4ISIiIovCcENEREQWheHGRObNm4fg4GDY2NggKioK+/fvN6jdzp07MXToUPj6+kKSpBrvkVWT2NhYREZGwtHREZ6enhgxYgROnz5tUNv58+ejY8eO+oWWoqOj8eeffxp1/AqzZs2CJEmYPHmyQfu//fbbkCSp0qNNmzZGHTMlJQWPPPII3NzcYGtriw4dOuDgwYN1tgsODq5ybEmSMGHCBIOOq9VqMWPGDISEhMDW1hYtWrTAe++9Z9D9ToDypcMnT56MoKAg2NraokePHjhw4EC1+9b1+hBC4K233oKPjw9sbW0xYMAAJCYmGtx+1apViImJ0a/YHR8fb1Db0tJSvPHGG+jQoQPs7e3h6+uLcePGITU11eBjv/3222jTpg3s7e3h4uKCAQMGYN++fQa1vdlzzz0HSZIwd+5cg4/92GOPVfn5Dxo0yOBjnzx5EsOGDYOzszPs7e0RGRmJpKQkg9pX99qTJAkfffRRnW3z8vIwceJE+Pv7w9bWFu3atat0w+C62l+5cgWPPfYYfH19YWdnh0GDBulfL4a8lxQVFWHChAlwc3ODg4MD7r//fv2iqoa0X7RoEfr27QsnJydIkoTs7GyD2mZlZeHFF19E69atYWtri8DAQLz00kv6ewUacuxnn30WLVq0gK2tLTw8PDB8+HCcOnXKqPdQIQQGDx5c6XtrSPu+fftW+Xk/99xzBh87Li4O/fr1g729PZycnNC7d28UFhbW2f7ixYs1vt4efvjhOo+dnp6ORx99FN7e3rC3t0fnzp3x66+/Gnze586dw8iRI+Hh4QEnJyeMHj26yiK85sJwYwIrV67ElClTMHPmTBw+fBjh4eEYOHAgMjIy6mybn5+P8PBwzJs3r17H3rFjByZMmIC9e/di06ZNKC0tRUxMDPLz8+ts6+/vj1mzZuHQoUM4ePAg+vXrh+HDh+PEiRNG1XDgwAEsXLgQHTt2NKpd+/btkZaWpn/s2rXL4LbXr19Hz549YW1tjT///BP//PMP5syZAxcXF4Pqvfm4mzZtAgA88MADBh37ww8/xPz58/HFF1/g5MmT+PDDDzF79mx8/vnnBrV/6qmnsGnTJnz//fc4fvw4YmJiMGDAAKSkpFTZt67Xx+zZs/HZZ59hwYIF2LdvH+zt7TFw4EAUFRUZ1D4/Px+9evXChx9+aNSxCwoKcPjwYcyYMQOHDx/GqlWrcPr0aQwbNszg2lu1aoUvvvgCx48fx65duxAcHIyYmBhcvXrV4N+L1atXY+/evVWWYzek/aBBgyq9DpYvX25Q23PnzqFXr15o06YNtm/fjmPHjmHGjBmwsbExqP3Nx0xLS8M333wDSZJw//3319l2ypQp2LBhA5YtW4aTJ09i8uTJmDhxItatW1fnsYUQGDFiBM6fP4+1a9fiyJEjCAoKwoABA5Cfn2/Qe8nLL7+M3377DT///DN27NiB1NRUjBo1CoBh70UFBQUYNGgQpk+fXqm2utqmpqYiNTUVH3/8MRISErBkyRJs2LABTz75pMHH7tKlC7799lucPHkSGzduhBACMTEx2L59u8HvoXPnzq1y2x5D34OffvrpSj/32bNnG9Q2Li4OgwYNQkxMDPbv348DBw5g4sSJUCgUdbYPCAio8np755134ODggKtXr9Z57HHjxuH06dNYt24djh8/jlGjRmH06NE4cuRIncfOz89HTEwMJEnC1q1bsXv3bpSUlGDo0KHQ6XRVvrcmJ+i2devWTUyYMEH/b61WK3x9fUVsbKxRzwNArF69+rZqycjIEADEjh076tXexcVFfPXVVwbvn5ubK1q2bCk2bdok+vTpIyZNmmRQu5kzZ4rw8PB61SiEEG+88Ybo1atXvdvfbNKkSaJFixZCp9MZtP+QIUPEE088UWnbqFGjxNixY+tsW1BQIJRKpfj9998rbe/cubP473//W2vbW18fOp1OeHt7i48++ki/LTs7W6jVarF8+fI629/swoULAoA4cuSIQceuzv79+wUAcenSpXq112g0AoDYvHmzQW0vX74s/Pz8REJCgggKChKffvqpwbWPHz9eDB8+vNZ6amo7ZswY8cgjj9TZtrbabzZ8+HDRr18/g9q2b99evPvuu5W21fTaubX96dOnBQCRkJCg36bVaoWHh4dYvHhxlfa3vpdkZ2cLa2tr8fPPP+v3OXnypAAg4uLi6mx/s23btgkA4vr161W+VlfbCj/99JNQqVSitLS0Xu2PHj0qAIizZ88a1PbIkSPCz89PpKWl1fpzra69oe+N1bWNiooSb775Zp1ta6v9Zp06dary/lVTW3t7e7F06dJK+7m6uhr0etm4caNQKBRCo9Ho98nOzhaSJIlNmzYZdD63gz03t6mkpASHDh3CgAED9NsUCgUGDBiAuLi4Bq+nopvW1dXVqHZarRYrVqxAfn6+UbeymDBhAoYMGVLp/A2VmJgIX19fNG/eHGPHjtV36xti3bp16Nq1Kx544AF4enoiIiICixcvNrqGkpISLFu2DE888YTBN1Lt0aMHtmzZgjNnzgAAjh49il27dmHw4MF1ti0rK4NWq9X/lV/B1tbWqJ4rALhw4QLS09Mrfe+dnZ0RFRUl22tPkqR63butpKQEixYtgrOzM8LDw+vcX6fT4dFHH8Vrr72G9u3b16NaYPv27fD09ETr1q3x/PPP49q1awYdd/369WjVqhUGDhwIT09PREVFGT2cXOHKlStYv369vgeiLj169MC6deuQkpICIQS2bduGM2fOICYmps62xcXFAFDptadQKKBWq6t97d36XnLo0CGUlpZWer21adMGgYGB1b7e6vteZGhbjUYDJycnWFlVXYu2rvb5+fn49ttvERISgoCAgDrbFhQU4OGHH8a8efNqvI9hXcf+4Ycf4O7ujrCwMEybNg0FBQV1ts3IyMC+ffvg6emJHj16wMvLC3369KnxvaKu8z506BDi4+Orfb1V17ZHjx5YuXIlsrKyoNPpsGLFChQVFaFv3751ti8uLoYkSZUW8rOxsYFCoTD6va5ezB6fLFxKSooAIPbs2VNp+2uvvSa6detm1HPhNntutFqtGDJkiOjZs6fBbY4dOybs7e2FUqkUzs7OYv369Qa3Xb58uQgLCxOFhYVCCMP/OhFCiD/++EP89NNP4ujRo2LDhg0iOjpaBAYGipycHIPaq9VqoVarxbRp08Thw4fFwoULhY2NjViyZInB9QshxMqVK4VSqRQpKSkGt9FqteKNN94QkiQJKysrIUmS+OCDDwxuHx0dLfr06SNSUlJEWVmZ+P7774VCoRCtWrWqtd2tr4/du3cLACI1NbXSfg888IAYPXp0ne1vdrs9N4WFhaJz587i4YcfNqr9b7/9Juzt7YUkScLX11fs37/foLYffPCBuOeee/S9bcb23CxfvlysXbtWHDt2TKxevVq0bdtWREZGirKyslrbVvzVbmdnJz755BNx5MgRERsbKyRJEtu3bzf4vCt8+OGHwsXFRf87VFfboqIiMW7cOAFAWFlZCZVKJb777juDzrukpEQEBgaKBx54QGRlZYni4mIxa9YsAUDExMRUalvde8kPP/wgVCpVleNERkaK119/vc72N6ut58aQ97GrV6+KwMBAMX36dKPaz5s3T9jb2wsAonXr1lV6bWpq+8wzz4gnn3xS/++afq41tV+4cKHYsGGDOHbsmFi2bJnw8/MTI0eOrLNtXFycACBcXV3FN998Iw4fPiwmT54sVCqVOHPmjMHnXeH5558Xbdu2Nbju69evi5iYGP3rzcnJSWzcuNGg9hkZGcLJyUlMmjRJ5Ofni7y8PDFx4kQBQDzzzDM11mgqDDe3qTGFm+eee04EBQWJ5ORkg9sUFxeLxMREcfDgQTF16lTh7u4uTpw4UWe7pKQk4enpKY4eParfZky4udX169eFk5OTwUNi1tbWIjo6utK2F198UXTv3t2o48bExIj77rvPqDbLly8X/v7+Yvny5eLYsWNi6dKlwtXV1eBgdfbsWdG7d28BQCiVShEZGSnGjh0r2rRpU2u7xhpuSkpKxNChQ0VERESlLmhD2ufl5YnExEQRFxcnnnjiCREcHCyuXLlSa9uDBw8KLy+vSoHU2HBzq3Pnzhk0JFbx+/7QQw9V2m/o0KHiwQcfNPrYrVu3FhMnTjS47o8++ki0atVKrFu3Thw9elR8/vnnwsHBodpu/uraHzx4UISHh+tfewMHDhSDBw8WgwYNqrRfde8lxoSbut6Lags3dbXVaDSiW7duYtCgQaKkpMSo9tnZ2eLMmTNix44dYujQoaJz586VgmV1bdeuXStCQ0NFbm6ufltNP1dD34O3bNlSZUisurYVv+PTpk2r1L5Dhw5i6tSpRh27oKBAODs7i48//tjguidOnCi6desmNm/eLOLj48Xbb78tnJ2dxbFjxwxqv3HjRtG8eXMhSZJQKpXikUceEZ07dxbPPfdcLd8d02C4uU3FxcVCqVRWeaGPGzdODBs2zKjnup1wM2HCBOHv7y/Onz9fr/YV+vfvb1CqXr16tf4NsuIBQP8ivvUvYEN07dq1yi9sTQIDAyv9JSWEEF9++aXw9fU1+HgXL14UCoVCrFmzxqg6/f39xRdffFFp23vvvSdat25t1PPk5eXpg8no0aPFvffeW+v+t74+Kj6Qbw0kvXv3Fi+99FKd7W9W33BTUlIiRowYITp27CgyMzMNrr0moaGhVXrBbm376aef6l9nN7/2FAqFCAoKqvex3d3dxYIFC2ptW1xcLKysrMR7771Xab/XX39d9OjRw6hj79y5UwAQ8fHx1X791rYFBQXC2tq6ynytJ598UgwcONCoY2dnZ4uMjAwhRPmcwRdeeEH/tZreSyo+kG8NJIGBgeKTTz6ps/3Nago3dbXNyckR0dHRon///tX2dhnzPlhcXCzs7OzEjz/+WGvbSZMm1fh669OnT72OnZeXJwCIDRs21Nr2/PnzAoD4/vvvK20fPXp0pV5SQ469dOlSYW1trf+519X27NmzVeZoCVH+GfHss88adeyrV6/qf9ZeXl5i9uzZNe5rKpxzc5tUKhW6dOmCLVu26LfpdDps2bLFqLkr9SWEwMSJE7F69Wps3boVISEht/V8Op1OPzZfm/79++P48eOIj4/XP7p27YqxY8ciPj4eSqXSqOPm5eXh3Llz8PHxMWj/nj17Vrns8MyZMwgKCjL4mN9++y08PT0xZMgQo2otKCiodDNXAFAqlUZfAWBvbw8fHx9cv34dGzduxPDhw41qHxISAm9v70qvvZycHOzbt69BXnulpaUYPXo0EhMTsXnzZri5ud32cxry+nv00Udx7NixSq89X19fvPbaa9i4cWO9jnv58mVcu3atztefSqVCZGTkbb/2AODrr79Gly5dDJpjBJR/v0tLS03y2nN2doaHhwcSExNx8OBBDB8+vM73ki5dusDa2rrS6+306dNISkpCdHT0bb0XGdI2JycHMTExUKlUWLduXaW5Q/U5tij/4x5FRUW1tp06dWqV1xsAfPrpp/j222/rdeyK5/D29q61bXBwMHx9fWt8vRlz7K+//hrDhg2Dh4eH/vxra1sxJ6im15sxx3Z3d0ezZs2wdetWZGRkVLqq0mzMHp+agBUrVgi1Wi2WLFki/vnnH/HMM8+IZs2aifT09Drb5ubmiiNHjogjR44IAPpx/OquOKnO888/L5ydncX27dtFWlqa/lFQUFBn26lTp4odO3aICxcuiGPHjompU6cKSZLEX3/9ZdCxb2XMsNQrr7witm/fLi5cuCB2794tBgwYINzd3av8VVGT/fv3CysrK/F///d/IjExUfzwww/Czs5OLFu2zKD2Wq1WBAYGijfeeMOg/W82fvx44efnJ37//Xdx4cIFsWrVKuHu7l6la74mGzZsEH/++ac4f/68+Ouvv0R4eLiIioqqtou9rtfHrFmzRLNmzfTzR4YPHy5CQkL0f9XW1f7atWviyJEjYv369QKAWLFihThy5IhIS0urtW1JSYkYNmyY8Pf3F/Hx8ZVee8XFxXUeOy8vT0ybNk3ExcWJixcvioMHD4rHH39cqNVqkZCQYPTvxa3DUrW1z83NFa+++qqIi4sTFy5cEJs3bxadO3cWLVu2FEVFRXUee9WqVcLa2losWrRIJCYmis8//1wolUrx999/G/Q9F6J8aMXOzk7Mnz/fqJ93nz59RPv27cW2bdvE+fPnxbfffitsbGzEl19+aVD7n376SWzbtk2cO3dOrFmzRgQFBYlRo0YJIQx7L3nuuedEYGCg2Lp1qzh48KCIjo7WDw8b0j4tLU0cOXJELF68WAAQO3fuFEeOHBGPP/54rW01Go2IiooSHTp0EGfPnq20T1lZWZ3HPnfunPjggw/EwYMHxaVLl8Tu3bvF0KFDhaurq3jssceMfg/FTb1idR377Nmz4t133xUHDx4UFy5cEGvXrhXNmzcXvXv3Nuh79umnnwonJyfx888/i8TERPHmm28KGxsbcfbsWYPf/xMTE4UkSeLPP//Ub6urbUlJiQgNDRV33XWX2Ldvnzh79qz4+OOPhSRJYv369QYd+5tvvhFxcXHi7Nmz4vvvvxeurq5iypQpNX5fTYnhxkQ+//xzERgYKFQqlejWrZvYu3evQe0qumdvfYwfP96g9tW1BSC+/fbbOts+8cQTIigoSKhUKuHh4SH69+9f72AjhHHhZsyYMcLHx0eoVCrh5+cnxowZU2VyX11+++03ERYWJtRqtWjTpo1YtGiRwW03btwoAIjTp08bdUwhyrvGJ02aJAIDA4WNjY1o3ry5+O9//6v/UK/LypUrRfPmzYVKpRLe3t5iwoQJIjs7u9p963p96HQ6MWPGDOHl5SXUarXo379/pXOqq/23335b7ddnzpxZa9uKYazqHtu2bavz2IWFhWLkyJHC19dXqFQq4ePjI4YNG6afUGzs78Wt4aa29gUFBSImJkZ4eHgIa2trERQUJJ5++mn9HyOGHPvrr78WoaGhwsbGRoSHh1ca2jSk/cKFC4WtrW2Vn3tdbdPS0sRjjz0mfH19hY2NjWjdurWYM2eOfmJ1Xe3/97//CX9/f2FtbS0CAwPFm2++qX/dGvJeUlhYKF544QXh4uIi7OzsxMiRI0VaWprB7WfOnFnjfrW1rem8ANT6Wqxon5KSIgYPHiw8PT2FtbW18Pf3Fw8//LA4depUvd5Dbw43dbVPSkoSvXv3Fq6urkKtVovQ0FDx2muv6Zc+MOTYsbGxwt/fX9jZ2Yno6Gh9kDa0/bRp00RAQIDQarWVzqGutmfOnBGjRo0Snp6ews7OTnTs2FF/abgh7d944w3h5eUlrK2tRcuWLSu9Vs1NulEkERERkUXgnBsiIiKyKAw3REREZFEYboiIiMiiMNwQERGRRWG4ISIiIovCcENEREQWheGGiIiILArDDREREVkUhhsiavK2b98OSZKQnZ0tdylEZAIMN0RERGRRGG6IiIjIojDcEJHsdDodYmNjERISAltbW4SHh+OXX34B8O+Q0fr169GxY0fY2Nige/fuSEhIqPQcv/76K9q3bw+1Wo3g4GDMmTOn0teLi4vxxhtvICAgAGq1GqGhofj6668r7XPo0CF07doVdnZ26NGjB06fPm3eEycis2C4ISLZxcbGYunSpViwYAFOnDiBl19+GY888gh27Nih3+e1117DnDlzcODAAXh4eGDo0KEoLS0FUB5KRo8ejQcffBDHjx/H22+/jRkzZmDJkiX69uPGjcPy5cvx2Wef4eTJk1i4cCEcHBwq1fHf//4Xc+bMwcGDB2FlZYUnnniiQc6fiEyLdwUnIlkVFxfD1dUVmzdvRnR0tH77U089hYKCAjzzzDO4++67sWLFCowZMwYAkJWVBX9/fyxZsgSjR4/G2LFjcfXqVfz111/69q+//jrWr1+PEydO4MyZM2jdujU2bdqEAQMGVKlh+/btuPvuu7F582b0798fAPDHH39gyJAhKCwshI2NjZm/C0RkSuy5ISJZnT17FgUFBbjnnnvg4OCgfyxduhTnzp3T73dz8HF1dUXr1q1x8uRJAMDJkyfRs2fPSs/bs2dPJCYmQqvVIj4+HkqlEn369Km1lo4dO+r/38fHBwCQkZFx2+dIRA3LSu4CiKhpy8vLAwCsX78efn5+lb6mVqsrBZz6srW1NWg/a2tr/f9LkgSgfD4QEd1Z2HNDRLJq164d1Go1kpKSEBoaWukREBCg32/v3r36/79+/TrOnDmDtm3bAgDatm2L3bt3V3re3bt3o1WrVlAqlejQoQN0Ol2lOTxEZLnYc0NEsnJ0dMSrr76Kl19+GTqdDr169YJGo8Hu3bvh5OSEoKAgAMC7774LNzc3eHl54b///S/c3d0xYsQIAMArr7yCyMhIvPfeexgzZgzi4uLwxRdf4MsvvwQABAcHY/z48XjiiSfw2WefITw8HJcuXUJGRgZGjx4t16kTkZkw3BCR7N577z14eHggNjYW58+fR7NmzdC5c2dMnz5dPyw0a9YsTJo0CYmJiejUqRN+++03qFQqAEDnzp3x008/4a233sJ7770HHx8fvPvuu3jsscf0x5g/fz6mT5+OF154AdeuXUNgYCCmT58ux+kSkZnxaikiatQqrmS6fv06mjVrJnc5RHQH4JwbIiIisigMN0RERGRROCxFREREFoU9N0RERGRRGG6IiIjIojDcEBERkUVhuCEiIiKLwnBDREREFoXhhoiIiCwKww0RERFZFIYbIiIisij/D2R59X2VjT8hAAAAAElFTkSuQmCC", "text/plain": [ "
" ] @@ -1456,12 +1456,12 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 17, "metadata": {}, "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAG2CAYAAACZEEfAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABq60lEQVR4nO3dd1hT1+MG8DfsHUW2TAfixK3gqqO46q6jtVWr1bZi62idraPVFrWt7dfROn5urauOOloXrqqooOBEBERBZYhI2DPn9weaGlmJBIH4fp4nT8vNPfecQMx9c+6550iEEAJEREREWkqnohtAREREVJ4YdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirVWjYOXPmDPr06QMHBwdIJBLs27dP6XkhBObMmQN7e3sYGxujW7duCA8PV9onKSkJw4cPh4WFBapVq4YxY8YgLS3tNb4KIiIiqswqNOykp6fD09MTK1asKPL5xYsXY+nSpVi5ciUuXrwIU1NTdO/eHVlZWYp9hg8fjps3b+LYsWM4ePAgzpw5g3Hjxr2ul0BERESVnKSyLAQqkUiwd+9e9O/fH0BBr46DgwO+/PJLfPXVVwAAmUwGW1tbbNiwAcOGDUNoaCgaNGiAwMBAtGzZEgBw+PBh9OrVCw8ePICDg0NFvRwiIiKqJPQqugHFiYqKQlxcHLp166bYJpVK0aZNGwQEBGDYsGEICAhAtWrVFEEHALp16wYdHR1cvHgRAwYMKPLY2dnZyM7OVvwsl8uRlJSEGjVqQCKRlN+LIiIiIo0RQiA1NRUODg7Q0Sn+YlWlDTtxcXEAAFtbW6Xttra2iufi4uJgY2Oj9Lyenh4sLS0V+xTFz88P3377rYZbTERERBUhJiYGjo6OxT5facNOeZo5cyamTJmi+Fkmk8HZ2RkxMTGwsLCowJYRERGRqlJSUuDk5ARzc/MS96u0YcfOzg4AEB8fD3t7e8X2+Ph4NG3aVLFPQkKCUrm8vDwkJSUpyhfF0NAQhoaGhbZbWFgw7BAREVUxpQ1BqbTz7Li5ucHOzg7+/v6KbSkpKbh48SK8vLwAAF5eXkhOTsbly5cV+5w4cQJyuRxt2rR57W0mIiKiyqdCe3bS0tIQERGh+DkqKgohISGwtLSEs7MzJk2ahAULFqBu3bpwc3PD7Nmz4eDgoLhjq379+ujRowfGjh2LlStXIjc3FxMmTMCwYcN4JxYREREBqOCwExQUhM6dOyt+fj6OZuTIkdiwYQOmTZuG9PR0jBs3DsnJyWjfvj0OHz4MIyMjRZmtW7diwoQJ6Nq1K3R0dDBo0CAsXbr0tb8WIiIiqpwqzTw7FSklJQVSqRQymYxjdoiIiKoIVc/flXbMDhEREZEmMOwQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEqQn5+P2bNnw83NDcbGxqhduzbmz58PIYRin1GjRkEikSg9evToUeJxU1NTMWnSJLi4uMDY2Bje3t4IDAxU2ictLQ0TJkyAo6MjjI2N0aBBA6xcubJcXqc206voBhAREVVmixYtwu+//46NGzeiYcOGCAoKwkcffQSpVIovvvhCsV+PHj2wfv16xc+GhoYlHvfjjz/GjRs3sHnzZjg4OGDLli3o1q0bbt26hZo1awIApkyZghMnTmDLli1wdXXF0aNHMX78eDg4OKBv377l84K1EHt2iIiISnD+/Hn069cPvXv3hqurK9599134+Pjg0qVLSvsZGhrCzs5O8ahevXqxx8zMzMTu3buxePFidOzYEXXq1MG8efNQp04d/P7770p1jxw5Em+99RZcXV0xbtw4eHp6FqqbSsawQ0T0hiivyzF+fn5o1aoVzM3NYWNjg/79+yMsLExpn08++QS1a9eGsbExalhZo0O3njgTGFIeL1PjvL294e/vjzt37gAArl69irNnz6Jnz55K+506dQo2NjaoV68ePvvsMzx58qTYY+bl5SE/Px9GRkZK242NjXH27Fmluvfv34+HDx9CCIGTJ0/izp078PHx0eArfAMIEjKZTAAQMpmsoptCRFRuvv/+e1GjRg1x8OBBERUVJXbt2iXMzMzE//73P8U+I0eOFD169BCxsbGKR1JSUonH7d69u1i/fr24ceOGCAkJEb169RLOzs4iLS1Nsc+qVavE6dOnxdJ9Z4XDqF+FcZ3WQtfCWvwRcLfcXq+m5Ofni+nTpwuJRCJ09fSERCIRP/zwg9I+27ZtE3/99Ze4du2a2Lt3r6hfv75o1aqVyMvLK/a4Xl5eolOnTuLhw4ciLy9PbN68Wejo6Ah3d3fFPllZWWLEiBECgNDT0xMGBgZi48aN5fZaqxpVz98MO4Jhh4jeDL179xajR49W2jZw4EAxfPhwxc8jR44U/fr1K1M9CQkJAoA4ffq00vbIhFThOv2gcJl+UNh/tEwAEE6f/J94lJxRpvrK27Zt24Sljb2w7jtV2I9eLqzemSLMLKqJDRs2FFsmMjJSABDHjx8vdp+IiAjRsWNHAUDo6uqKVq1aieHDhwsPDw/FPj/++KNwd3cX+/fvF1evXhXLli0TZmZm4tixYxp9jVWVqudvXsYiInpDlMflmKLIZDIAgKWlpWLbydsJGLIyAAKAPCcLadePQ09qC4l5DdxLzCjbCytnU776ChLPfjCp3wkG1q4wbdgFep7vYP73PxRbplatWrCyskJERESx+9SuXRunT59GWloaYmJicOnSJeTm5qJWrVoACsb1zJo1C0uWLEGfPn3QpEkTTJgwAUOHDsVPP/2k8ddZklhZJs5HJiJWlvla69UU3o1FRPSGmDFjBlJSUuDh4QFdXV3k5+fj+++/x/DhwxX79OjRAwMHDoSbmxsiIyMxa9Ys9OzZEwEBAdDV1S21DrlcjkmTJqFdu3Zo1KgRElKz8N2BWzh4LRapVw7h6an1ELlZ0LN0hM3QBdDV04erlUl5vuwyEUJAlpIOI8lLfQMSHcTLMuEfGo+O7tbQ11V+/sGDB3jy5Ans7e1LrcPU1BSmpqZ4+vQpjhw5gsWLFwMAcnNzkZubCx0d5WPr6upCLper9TpiZZmISkyHm5Up7KXGpe4vhEBadh4S03KwIzAaq87chRCAjgTwG9gYQ1s5q1V/RWPYISJ6Q+zcuRNbt27FH3/8gYYNGyIkJASTJk2Cg4MDRo4cCQAYNmyYYv/GjRujSZMmqF27Nk6dOoWuXbuWWoevry9u3LiBM2f+xbZL0fD7OxQpWXnQkQDjPx4Ji48H45e/ApF8aTcS/1oI51E/IyMnv9xec1lcikqC3z+hkLi0gOz8DuhaWMPAyhk58ZFICdwHsyZvY8zGIEj18mF6cy8+GfkeOnrWxd27dzFt2jTUqVMH3bt3Vxyva9euGDBgACZMmAAAOHLkCIQQqFevHiIiIjB16lR4eHjgo48+AgBYWFigU6dOmDp1KoyNjeHi4oLTp09j06ZNWLJkicqvY0dgNGbuuQ75s7Ay+W13tHa1RGJaDhLTshWPx6nKP2flFg5UcgFM330dJ8Meo4VzdXjYm8PDzgLW5iXfZl/RJEK8MAz/DZWSkgKpVAqZTAYLC4uKbg4RqSA/Px/z5s3Dli1bEBcXBwcHB4waNQrffPMNJBIJAGDevHnYvn07YmJiYGBggBYtWuD7779HmzZtij2uq6sr7t+/X2j7+PHjsWLFCgDA6tWr8ccff+DKlStITU3F06dPUa1atXJ5nZrk5OSEGTNmwNfXV7FtwYIF2LJlC27fvl1sOWtrayxYsACffPJJicefMGEC/vrrL2za8zd+u5yKwHtPAQCNa0rhN7AxGtWUAijoZbj9IAlvN6+D6t0/h+dbvbHPtx3MDCvH9+878alYfPg2jocmAAAM5NnQD9mFsIv+kGfIoGtmiU49+8P73U/w961EJDxNweM9C5CTcBciOx3VrGzR3edt/PrjQtja2iqO6+rqilGjRmHevHkACsLnzJkz8eDBA1haWmLQoEH4/vvvIZVKFWXi4uIwc+ZMHD16FElJSXBxccG4ceMwefJkxfv8ZUIIPHiaidtxqQiMeoLV/0a98u/CUE8H2Xml9yJZmRmivr05POwKwo+HvTnq2JjBUE9X7V4ldah6/q4c7ywiIjWpMtGbu7s7li9fjlq1aiEzMxO//PILfHx8EBERAWtr6yKPGxgYiPz8/3oabty4gbfffhuDBw9WbMvIyECPHj3Qo0cPzJw5s3xfqAZlZGSofUlElcsxQgh8/vnn2LN3L0Z+vx7j9sUgN1/AxEAXU952xyhvV+i9cJnHXmoMSyMrGOhKYGEARCSk4audV/H7B82LPYG/DrGyTPxy7A7+vPwAcgHo6kgwrJUTJnatCxuLgYiVZeJeYgZcrUwUJ+05/eQ4F/kEe1utxZGb8cjMLXjvBAD4dHcEBjTLxDtNHGBpaoCAq6GISkxHrCwT9lJjDBkyBEOGDCmxTXZ2dli/fn2xgSE1Kxd34lMRGpuK23EpuB2bittxqUjLzivxuLYWhnCqbgIrM0NYmRvAyswQ1uaGBT+bGcL62XZZZi7aLTwB+QvdIjoSYEx7NzxMzsTt2FREPUlHYlo2/g3Pxr/hiYr99HQkqGFmgPiUbEW5iroExp4dsGeHqCp65513YGtri7Vr1yq2DRo0CMbGxtiyZUuRZZ7/Wz9+/LhKl2QAYNKkSTh48CDCw8MLnYhPnTqFzp07V5menVGjRuH48eNYtWoVrJ1qw//cRfw0+yt8PGY0Fi1ahLS0NHz77bcYNGgQ7OzsEBkZiWnTpiE1NRXXr19XzAj88uWY8ePHY/OWrajzwbdI1LECALSrWwPzB7dGXYcauHv3Lnbs2AEfHx9YW1vjwYMHWLhwIc6dO4edxwLw2e5I5OTLMbV7Pfh2rvPafy+yjFz8djoCG87dU/Ri9Gxkh6+610NtazOVj5OenYejt+KwN/gRzoY/VgQEPR0J6tqa4XZsKgQKTvrTutfDO54OKh334NVHWHwkDHIBSAB08bCBjo4Et+NSEJNU9IBhfV0J6tiYw8XSBEduxuHFE72uRIKzMzqr3MuyIzAas/bcQL4Q0JVI8MPARkqBJSMnD+HxaQiNTcHtuFTFf2WZuYWOpW7dpWHPDhFpNW9vb6xevRp37tyBu7u74s6i4sYy5OTkYPXq1ZBKpfD09FSpjpycHGzZsgVTpkyp0B4HTVm2bBlmz56NUR9/gieJj6FrZgnTBl3RpN9YAAW9PNeuXcPGjRuRnJwMBwcH+Pj4YP78+UpLH0RGRiIxseAbfHJGjmLG35DfJyv22QGgh3Q96o4aBSMjI/z777/49ddf8fTpU9ja2qJjx444f/486tVzx7d5Rpi55zp+OhqGRjWl6ORedK+bpmXl5mNTwD2sOBmpODG3drPEjJ4eaO5c/OzHxTE11MOAZo4Y0MwRCalZ2B/yCPtCHuLGwxSExqYq9pMLYOHhMCw8HFbC0YomAPjfTlDaZmdhpBg7U//Zf2tZmyoGTRcVVtQJG0NbOaOju3WhXq3nTAz04OlUDZ5O1f5rpxA4dD0WE/4IVto3XwjcS8zQ+OWs0rBnB+zZIaqK5HI5Zs2ahcWLFyvdWfTyZaWDBw9i2LBhyMjIgL29Pfbt24dWrVqpVMfOnTvx/vvvIzo6Gg4Ohb+FV7WeHaDgUs3LlyUkAGb28kBdG3PFZY0apoYw0Ct6dpJYWSaiHqcjPCEVy05EIDEtBwAwvI0zpvXwgNRYX+12zdxzDdsuxUBqrI8DE9rDuUb53KEVK8tEZEI6QmNlWH/uHh7JsgAA9WzNMb1nPXSuZ6PxYLsrKAZT/7xWaLu+jgQ6OiXXJZcL5MoLn6ZHtHVBj8Z28LCzgKWpQaltKOoSXHkr6r3Gnh0iqjDlOYCwvOpW5c4iAOjcuTNCQkKQmJiINWvWYMiQIbh48SJsbGxKrWPt2rXo2bNnkUGnqop6nI6Xz50CwA9/Fx6gLDXWh5WZwbMAVDCOIzY5E0dvxStdFqlrYwa/gY3R0tWy0DFUNa9vQ4TGpiIkJhnjNgdhz3hvmBho9hS1IzAaM/Zcx4tf8R2kRpjiUw8DmtWEbinB41W1r2sFHQkKnfTPTC/9pF9cYPisc221/r3YS41f+79te6kx/AY2LlOvkqawZwfs2aE328u3pb7OAYRlqftV7yyqW7cuRo8eXerA4vv376NWrVrYs2cP+vXrV+Q+VbFnZ8uF+/hm3w2lbRIUXL5JzcpDYlo2nqTnIL+I3oSiSACcnvoWnGuYlrltsbJM9Fl2FolpOejr6YD/DWuqsV6WqzFP0W/FeaVtEgAnv3oLrlZlb3tpShv3Ul5lK4Py7FVizw4RlSpWlqkIG0DBN89Ze26go7t1uX/7KqrumXuuq1z3q9xZBBRc/srOzi71+OvXr4eNjQ169+5d6r5VRWpWLpb6hwMoONELoMiTp1wukJyZWzDfSmo2HqdlIzEtB1djkrH/6iOlYwoAD5OzNBJ27KXGWPF+cwz/v4vYf/URmjhK8XGHWmU6Zr5cYMuF+/D7J7TQcwJArCzrtYSd0sa9lFfZyqAiepVexrBD9Aa7HZtS6JLG6xpAGJVY+HKKXABbL9zHlLfrlTqWoU+fPvj+++/h7OyMhg0bIjg4GEuWLMHo0aMBAOnp6fj+++/Rt29f2NvbIzExEStWrMDDhw+VbiN/+c4ioCAQrV+/HiNHjoSeXuGPybi4OMTFxSmWArh+/TrMzc3h7OystERCaV735cOfj95BQmo2XGuYYMNHrZ+d6AufPHV0JLA0NYClqQHcbc2V2nvw2qNCl1Q0OQNym1o18HXv+vj2wC34/XMbDRws4F3b6pWOFRqbgpl7riMkJrnI5zXd9tKU5aRfGQJDVca1sYgqWH5+PmbPng03NzcYGxujdu3amD9/Pl68wrxnzx74+PigRo0akEgkCAkJUenYu3btgoeHB4yMjNC4cWP8/fffiueS0nOw+Mgd5CbGIGH3d4j+ZQiilwxC7MbJ0MtUby2kV+FWzLfp5Scj0Wf5Wfwb/rjE8suWLcO7776L8ePHo379+vjqq6/wySefYP78+QAKenlu376NQYMGwd3dHX369MGTJ0/w77//omHDhgAKTt63wsIR9SBW6djHjx9HdHS0Iji9bOXKlWjWrBnGji24i6ljx45o1qwZ9u/fr/Lr3xEYjXYLT+D9NRfRbuEJ7AiMVrnsq7j+QIZNAfcAAPP7N4KrlSm8atdQe9yH38DG0H12aam8xmCM8nbFgGY1kS8X+PyPYDxKVm89psycfCz85zb6LDuLkJhkmBvqYX7/Rq+l7VQ5ccwOOGaHKtYPP/yAJUuWFJoc7/vvv1dMjrd582ZERUXBwcEBY8eORXBwMJo2bVricc+fP4+OHTvCz88P77zzDv744w8sWrQIV65cgdShFkauu4Sw8AjEbZ4C8yZvw6R+J0gMTGCW8QgnF4+Dg71dub7uM3ceY8S6S4qfdSTA2w1scT7iCVKfTYjWoa4VpvfwUMy8q0kVOVbp4dMMtF90skxzn6gjXy4w4LdzuPZAhr6eDlj6XrMyHe913NmTmZOPQb+fx63YFDRxlGLnJ14w0i99ba5/wx/j6703EJ1UsLhor8Z2mNunIWwtjF5b2+n1UfX8zbADhh2qWOpMjnfv3j24ubmpFHaGDh2K9PR0HDx4ULGtbdu2cKnbAHc93sfj1Gyk/f0T2tezxe//tx43HsgwZddVpGblYX6/hvjQy1WTL1NJbr4cPf/3LyIS0jCkZcG8JM9PPknpOVh+IgKbL9xDbn7Bx1NfTwd85VNPY7cjRyelo9OPp5TuyinPsPGi7Lx8jFx3CRfuJhV6btvYtvCqXUPjdW4KuIc5f92EuZEe/L/sBBtzI43XUR5ikjLQZ/lZJGfkYkhLRywa1KTYActP0rKx4FAo9gY/BFBwl9V3/RqhWwPbIvcn7aDq+ZuXsYgqmLe3N/z9/XHnzh0AUEyO17NnzzIdNyAgAN26dVPa1rBVB+w/dgqPU7Phbm2KrKggtGjSEKOG9Mfwzk2QsWs6Mu4E4Odjd/A0PadM9ZdkU8B9RCSkwdLUAF/3bqB0OcXS1ABz+jTAiS/fwoBmNSGRAPuvPkLXJacwb/9NPEkrfXDxy4QQiEhIxfpzURizIRA+S87g5a95z8cqlafUrFyM3hBYZNDRkaBcxo8kpGThx2eT103rXq/KBB0AcLI0wbL3mkFHAuwMeoCtFwtf6hNCYGdQDLouOY29wQ+hIwE+aueKo1M6MeiQAgcoE1WwGTNmICUlBR4eHkqT4w0fPrxMx42Li1NahPDQtVgcCM9ATupTtHazxAIfR9T7Kg0LFy7EggULsGjRIvz99z/4+puvoWNkhiXHHDC/f6OyvrxCEtOy8euxgmA3rXu9Yiegc7I0wS9Dm+LjDm5YdDgMZ+48xobz9/Dn5Qf4pGMtjOngVuI8LE/SsnE2IhFnwxNxNiIRsc8mjyvJoeuxaO1mWS7zrSSkZmHUukDcik2BqYEuhrV2xvpzUYrBvpamBuWyEOb8Q6FIzc6Dp6MU77dx0fjxy1uHutaY2t0Diw7fxrcHbqK+vQVauBTMbnz3cRpm7b2uCI8N7C2wcFBjNHGsVoEtpsqIYYeogqk6OV5ZbDx/D/MO3ES+XMBQTwebRrdG0uN4AEC/fv0weXLBNP9NmzbFP/6ncSXkH2x1aYz3WjujgYNmL+3+eDgMqdl5aFTTAoNbOpW6f0MHKTaNbo1zEYlY+M9tXH8ow8/H7mDThfuY2LUuOrpb4cHTTDhIjfHgaSb+DX+Mf8MTcSs2Rek4Bno6aO1qiQ51rdC+rhWuPZDhm70Fc5c8vw17y4X7ePA0A/8b1uyVZgEuTlRiOkasu4iYpExYmRlg/ajWaOwoxccd3HA1Jhlz/rqJhNRsTNl5Fas+aFHqnWiqOnPnMQ5cfQQdCfD9gMblNmleefu0Uy1cf5iMv6/HYdymIMzt0wDXH8qwMeA+cvLkMNbXxeS362J0OzelBUeJnmPYIapgU6dOxYwZMzBs2DAAQOPGjXH//n34+fmVKezY2dkhLi4OPx65jRUnIwEADarLkeLqCCN9XVhZWUFPTw8NGjRQKufd0hMRe49ALoBvD9zE9nFtNTax27UHydh5OQYAMK9PQ7VOvu3qWOEv33Y4dD0WPx4JQ3RSRqHJ8V5W394CHZ+Fm1aulkoDXBs6SPFWvf/mLrkUlYRpf17DqbDH6L/iHNaMaIE6NuYlHF01V2OS8dGGQCSl58Clhgk2jW4Nl2dz0jy/ndheaozBqwJw7FY8VpyMwOdd65a53qzcfMz+q+D3M9LbtVwGeb8uEokEi9/1RNC9p0hIzcYX20MUz3Vyt8aC/o3gZPn6biGnqocRmEiDYmWZOB+ZiFiZ6rfKvurkeKVp07Ytlm/Zpwg6X77tjvyYa/Dy8gIAGBgYoFWrVggLU16M8M6dO2jTxB1G+jq4GJWEQ9djCx37VcjlAvP234QQwIBmNV9paQEdHQn6eDrg+JROmPK2e5H79Gpsh/8Na4rAr7vhn4kdMLNXfXSoa13knTz2UmPFeKF+TWti92fecJAaISoxHf1XnMfxW/Fqt/FFp8IS8N6aC0hKz0HjmlLs/sxbEXRe5OlUDQv6FVwyXHL8Dk6GJRTaR12/nYrE/ScZsLUwLPZ3VZWkZhVMcvgiiQTwG8igQ6Vj2CHSkFedN+X55HiHDh3CvXv3sHfvXixZsgQDBgxQ7JOUlISQkBDcunULABAWFoaQkBDExcUp9hkxYoRiCYSMnDxk1PXB3ZBzSL20B583N8aTf7ciKChIafK8qVOnYseOHVizZg0iIiKwfPlyHDhwAF9N+gKfdaoDAPjhUCgyc/LL/PvZF/IQV6KTYWKgixk9Pcp0LAM9HbR0LXpV6g/buqJf05qwNjcs8vmSNKopxf7P26O1qyXSsvMwdnMQlp8Ix6vctLrnygN8vDEIGTn56FDXCtvGtYWVWfFtGtLKCcPbOEMIYOK2YNxLTFe7zuciH6dh5amCkDvnnYYwN9LcJbmKUtQklEIA95+oNwcPvZkYdog0oLhlF1Tp4SltcjwA2L9/P5o1a6ZYumDYsGFo1qwZVq5cqdgnOjoasbGxSErPwftrLuJmvj3s+0+DSdRpzPqwJ/7880/s27cPjRr9N+h4wIABWLlyJRYvXozGjRvj//7v/7B79260b98en3SqhZrVjPFIloXfT0eW6feTlp2Hhf8UrFc1oUsdxZwnZeFmZYqXr4JpYkZcKzNDbPm4DT5oWxA8fjp6B75/XEH6s7l/SiOEwKrTkZiy8yry5AL9mzpg7chWKg0+ntunIZo7V0NKVh4+2XxZ5Tpfrn/2vhvIyZejk7s1ejUu3/mSXpfy+nvTm4Hz7IDz7FDZ7b/6CF9sCy60vbzmTSlKrCwTgVFJ+OloGKKTMiE11se6US3RwuXVV6L+53osPtt6BQZ6OvCf0umVLxcs/Oc2Vp6OhEsNExyd3BGGeqVPDqeK8l4gcdulaMz56wZy8wU87MyxZkTLEn8HcrnA93+HYu3ZKADA2A5umNmzvloDjuNTsvDOsrN4nJqN3k3ssfy9ZmqNmdoX/BCTdoTAUE8HRyd3LPKyWVVV1RfEJM3jpIJqYNihsniSlo0Bv51DdJJyL46OBDg3o8trmaV1R2A0Zuy5rpg7Rmqsh92feZd5gK0QAsP/7yLORz5Bj4Z2WPlhC7WPEZWYDp9fTiM3X+D/RrTU+Nwn5T0jbtC9JHy65QoS07JR3UQfK95vDu86hddqys7Lx1e7ruHAs4Uyv+5VH2M7vtoiloH3kvDe6gvIkwvM6uWBcR1rq1ROlpGLrktOITEtB1/5uGNCl7IPdK5sOAMyvYiTCtJr5+rqColEUujh6+sLoGDelw8//BB2dnYwNTVF8+bNsXv37lKPu2LFCri6usLIyAht2rTBpUuXlJ7/5JNPULt2bRgbG8Pa2hr9+vXD7du3y+U1viw1Kxcj11961pOip9TN3sRRCjsNXK4pTawsUynoFLQrD6YamLNFIpFg7rO7pg7fjMO5iES1j7Hg4C3k5gt0crdG1/o2ZW7Ty14cZFweWrpa4sDn7dDEUYqnGbn4cN0lbDgXpTSO5/lkgQeuPoK+rgS/Dm36ykEHAFq5WmJun4K75Bb+c1vl3/viI7eRmJaD2tamZaq/Mivvvzdpp0oddlRZIFEIgTlz5sDe3h7Gxsbo1q0bwsPDK7DVb67AwECs/DsQjhM2w9F3M+yGLQAAxQrTI0aMQFhYGPbv34/r169j4MCBGDJkCIKDC1/+eW7Hjh2YMmUK5s6diytXrsDT0xPdu3dHQsJ/d6u0aNEC69evR2hoKI4cOQIhBHx8fJCfX/ZBtSXJys3H2E1BuPEwBTVMDbBnfDucm9EFc96pD10JEBIjw7pz98q1DQDwb3hiodmA5QIamw24np05PmxbMBndtwduIi9f9bvEToYlwP92AvR0JJjTp4HGbmF/3eylxtj5iZdiccp5B25h2p/XcP9JOv6+/giDfj+PcxFPYGqgi3WjWqF/s5plrvODti54t4Uj5AKY8McVPHha8t8zOPop/rhUMCh+Qf/GGrtUSKQNKnXYWbRoEX7//XcsX74coaGhWLRoERYvXoxly5Yp9lm8eDGWLl2KlStX4uLFizA1NUX37t2RlVX6bKmkOflygeN3M+B3Oh66ptWha1Yd6eGXoFfNHnU9WwEoWJjy888/R+vWrVGrVi188803qFatGi5fvlzscZcsWYKxY8fio48+QoMGDbBy5UqYmJhg3bp1in3GjRuHjh07wtXVFc2bN8eCBQsQExODe/fuldvrzcuXY8IfwbhwNwlmhnrYOLo1alubwV5qjNHta+Gbdwq+lf/wdygCIstvBfGYpAwsPly4F0vTAzcnd3NHdRN93IlPw5YL91Uqk5Mnx/wDBXePfdTOFbWtzTTWnopgpK+LJUM88U3v+tCRALsuP0CnH09h/NZg3IlPg6mBLraP80KHutYaqU8ikWBB/0ZoXLOgR+nTLZeRlVt0gM/Ll+PrvTcgBDCwec3XNk6MqKqo1GHn/Pnz6NevH3r37g1XV1e8++678PHxUVzGEELg119/xTfffIN+/fqhSZMm2LRpEx49eoR9+/ZVbOPfEBk5edh4/h66/HwKM/f+N8GbyM9F+q1TMGvyNj7eeBmRj9Pg7e2NHTt2ICkpCXK5HNu3b0dWVhbeeuutIo+dk5ODy5cvK63vpKOjg27duiEgIKDIMunp6Vi/fj3c3Nzg5FT67LyvQi4XmL77Oo6HxsNATwf/N7JloQnbRnm7KnoBJvxxBY+SNX977OPUbHy49iIS03JgY26ouIT2fOCmJrv5pSb6+Kp7PQDAkmN3VFqfasP5KNxNTIeVmSG+0MAkeZWBRCLBxx1q4ZehTQs9l5mbDytzA43WZ6Svi5UftoClqQFuPEx5FmgKD7PcGHAft2JTIDXWx6xe9TXaBiJtUKnDTmkLJEZFRSEuLk7pZCiVStGmTZtiT4YAkJ2djZSUFKUHqSchJQs/HrkNL78TmLv/Ju4/yYCFkR6eX6TIuHMB8qw0mDbqitC4VPT89V90Hu+H7Jwc1KhRA4aGhvjkk0+wd+9e1KlTp8g6EhMTkZ+fr7S+EwDY2toqzS8DAL/99hvMzMxgZmaGf/75B8eOHYOBgWZPPEBBwP7+71DsvvIAujoSrHi/OdrWKvwtWiKR4IcBjdHA3gJP0nPwWQnfyl9FSlYuRq67hHtPMuBY3Rj7J7THuRldsG1sW5yd0blc7lAZ1soZDewtkJKVh5+O3ilx34TULCz1jwAATO9RTyvmeXlRUXP4aPLS4YtqVjPG8meLYe6+8gCbX+pZi5VlYsnRgokhZ/T0KHEuH6I3VaUOO8+n0Pfw8IC+vj6aNWuGSZMmKRZIfH7CU+Vk+CI/Pz9IpVLFo7x6ALRRWFwqpu66ivaLTmLFyUjIMnPhUsME3/VriAuzumLhoMbQlUiQdu0oTGq1xNeDvdHJ3Ro5+XL4zZ+HS7djsHzzHgQFBWHKlCkYMmQIrl+/XuZ2DR8+HMHBwTh9+jTc3d0xZMiQcrmUueJkhOK24sWDmuDtEu4sMjbQxaoPW6CaiT6uPpBh7l83X2lyupdl5uTj4w1BuBWbAiszA2we0wZ2UqNyH7ipqyPBvL4NAQDbA6Nx46Gs2H0XHw5DWnYePJ2qYVBzx3JpT0V63XO+eNexwsyeBT023x24hcB7/62a/t2BW0jPyUdz52oYqsJaY0Rvokoddl5cIPHKlSvYuHEjfvrpJ2zcuLFMx505cyZkMpniERMTo6EWaychBM6GJ2LEukvo/usZ7Lr8ADn5crR0qY6VH7TAiS/fwggvV5gY6GFoK2dsf782cqKvYvl3X+Kzt+pgw0etMLN9daReOQjTtz/HjzcMsD1SgklTZ6Fly5ZYsWJFkfVaWVlBV1cX8fHKU/bHx8fDzk55ojSpVIq6deuiY8eO+PPPP3H79m3s3btXo7+HzRfuK3o0Zr/TAINalH4Sd7I0wbJn38p3BMUoBpC+qtx8OXz/uIJL95Jg/myskJvV65tHpbWbJfp6OkAIPFv6oXB4C45+ij8vPwAAzOvTQGOLWlYm9lJj+A0sCPZA+Vw6fNnHHdzQx9MBeXKB8VuvID4lCydux+OfG3HQ1ZHg+wGNtfJ3TaQJlXoh0NIWSHx+wouPj4e9vb2iXHx8PJo2bVrscQ0NDWFoyK7eksTKMhEen4bw+FTsuvwAt+NSARTMHdOzkT0+7uCGZs5FT9d/aPc22NjY4MMhAwEUXNLxdimY76VXEwccjwW2XYrBsVsJkGfkFXvXlIGBAVq0aAF/f3/0798fACCXy+Hv76+05MHLhBAQQiA7u/RxJao6cPUR5jxbVPHzLnUwpr2bymU71LXG1O4eWHT4NubtvwkPOwu0cCn6d1cSuVxg6q6rOHE7AYZ6Olg7qhUaOrz+xR1n9vLAsVvxCLr/FPuvPkK/pv/defR8/SsAeLeFY7HvEW0wtJUzOrpbv7Y5XyQSCRYNaozw+FTcjkvF6A2BiE8p6L0c094N9e05RxhRcSp1z05pCyS6ubnBzs4O/v7+iudTUlJw8eJFxWKHpL7tl6Lh7XcCI9ZdwvxDobgdlwoTA12M8nbF6amdsWJ482JPYnK5HOvXr8fIkSOhp/dflvbw8ECdOnVwZ/cSzGljAHtJMu6e2IbggDOIMmuAmKSCsQ5du3bF8uXLFeWmTJmCNWvWYOPGjQgNDcVnn32G9PR0fPTRRwCAu3fvws/PD5cvX0Z0dDTOnz+PwYMHw9jYGL169dLI7+P0nceYsjMEQgAftnV5pUUVP+1UC70a2yE3X2D81stISFXvEpsQAt8dvIV9IY+gpyPB7x80R2u3V58ZuSzspcbw7VwwyZ3f37eVljTYfeUBrj6QwcxQD9N61KuQ9r1Or3vOFxMDPaz6sAWM9HRw81EKEtNyAACO1TjnDFFJKnXYKW2BRIlEgkmTJmHBggWKuVtGjBgBBwcHRU8Aqef5Gk8vXpyQANg3vh3m9W1Y6nIBx48fR3R0NEaPHq20XV9fH3///Tesra0x89MPcG3pOBjfPwfbPlMQYegOn1/OYM2Zu4iMjETUg1jFyuFDhw7FTz/9hDlz5qBp06YICQnB4cOHFeO0jIyM8O+//6JXr16oU6cOhg4dCnNzc5w/fx42NmWfwO7y/SR8uvkycvMF+ng64Nu+DV9prhiJRILF73qiro0Z4lOy4bv1CnLyVJ+vZql/BDacvwcA+GmwJ7p4aHYWYnV93KEWnCyNEZeShd9OFQxETsnKxaLDBQNlv+haBzbm5T+h4pvIQE8H2S/NdfTtgVsqrcNG9Kaq1MtFpKamYvbs2di7dy8SEhLg4OCA9957D3PmzFHcaSOEwNy5c7F69WokJyejffv2+O233+Durvq3by4X8Z/dl2Pw5a5rhbaX1xpPEQlpmLX3Oi5FFQy4dJAaITYlC0IUXDLzG9i4wta+uR2XgiErA5CSlYdO7tZYM6IlDPTK9v3g7uM09Ft+DqnZeRjp5YJv+zUqtczG8/cw99mloW/7NsRIb9cytUFTjt6Mw7jNl2Ggq4NjUzpiy4X7WPNvFGpZmeLwpI5l/l1R0c5HJuL9NRcLbX+d67ARVRZcG0sNDDsF8vLl6LP8HEJjlW/F15VIcHZG53LrqpfLBXZdjsGCQ6FIzVJe5VlHAvzl2w6Nakpf6+y70U8yMGjleTxOzUYLl+rYPKY1TAw0M8TNPzQeYzYGAQB+HuxZ4kDnv0IeYuL2EADApG51Mamb+pfQyosQAiPWXcK/4Ylo5GCBW7EpkAtg/Uet0Lme5peFoAKxsky0W3gC8hc+ucv73yhRZcW1sUhtK05GIjQ2BUZ6OuU6Qd3LdHQkGNrKGT++26TQc3IB9Fl+Di0WHMf7ay7guwO3sCsoBjceyoqctyZWlqm4BPYqYmWZ+PvaIwxbE4DHqdnwsDPHupGtNBZ0AKBrfVtMfDbJ3qy914u9hfvk7QR8ufMqgIJJCidWson5CtbNagAdCXDjUYri5JuQwtnLy1NF3AlGVNVV6rux6PW5GpOMpScK1hRb9G4TtHazfO0rC3s6VYOOBErfWIGCMUNJ6Tk4H/kE519YekFXRwI3K1N42Jmjvr0FHqdmYVPAfchf8RLYjsBozNxzXVG/pakBNo1uDamJ5ifEm9i1Lm48lMH/dgI+2XwZBz5vD0vT/yZBDLyXhE+3XEaeXGBAs5qY807lXFfK1FCv0Lpcs/bcQEd3a558y9HrvhOMqKrjZSzwMlZmTj56L/sXdx+n450m9lj+fvMKa8uOwGjM2nMD+UIovrH2a1oT4fFpCI1LQWhsCm7HpiI0LgXJGbmlHq+asb5Kc4/I5QLJmcrH05EA52Z0KbcTiSwzF/1XnENUYjq8a9fAptGtoaerg1uPUjB0dQBSs/LQ1cMGKz9sAX3dytkJy/EjRFSRVD1/s2eHsPCfUNx9nA5bC0Ms6F/6gNnyVNw31saOUjR2/G9OGSEEElKzC8JPXCr+vfMY54pYcPPlAKOO59P/l1fYkRrrY9WHLdB/xTmcj3yCOftvopVLdXx3sGDsUmtXS6wY3rzSBh3gv5mEXx4/Ul4zCRMRvQr27ODN7tk5fecxRq4rWFh185jWGlux+XUratCmjgTYMqZNkesYvexxajY+WHuxQgZ9/n09FuO3XlHaZi81wpHJHWFRBdaUKqo3rqLuoCOiNwt7dqhUyRk5mLrrvwGwVTXoAP8N2nz5pOtdx0ql8nVtzYss/zrGQjRzrgYJoDS3UXxKFtKz86pE2OH4ESKq7Bh23lBCCHy97wYSUrNRy9oU03t4VHSTyqysJ92KOmlHJabj5e7V8r6Epmn2UuMq01YievMw7Lyh9l99hEPXYqGnI8GvQ5vC2EC3opukEWU96VbESZvjXoiIylflHflI5eZRcia+2VewqOUXXeuiiWO1im3QG47zphARlS/27Lxh5HKBqX9eRWpWHpo6VcP4t2pXdJMIHPdCRFSeGHbeMBvO38O5iCcw1tfFkiGe0KvEtzW/aTjuhYiofPBM9wYJj0/FwsO3AQBf966PWtZmFdwiIiKi8sew84bIyZNj8s4Q5OTJ8VY9awxvw3lQiIjozcCw84ZY6h+OGw9TUM1EH4sHNamU6ywRERGVB4adN8Dl+0n47VQEAMBvQGPYWBhVcIuIiIheH4YdLZeenYcpO69CLoCBzWqiZ2P7im4SERHRa8Wwo+UWHArF/ScZqFnNGPP6Nazo5hAREb12vPVcS8XKMrHn8gNsuxQNiQT4abBnlVhniYiISNMYdrTQjsBozNxzXbH8QPvaVvCqXaNiG0VERFRBeBlLy8TKMpWCDgCci0xErCyz4hpFRERUgRh2tExUYrpS0AH+W0GbiIjoTcSwo2XcrEzx8hQ6XEGbiIjeZAw7WsZeaoz2dawUP3MFbSIietNxgLIWepRcMD5nYte6GNbaiUGHiIjeaAw7Wib6SQYiH6dDV0eC0e3dIDXm7eZERPRm42UsLXMyLAEA0NKlOoMOERERGHa0zonbBWGni4dNBbeEiIiocmDY0SIZOXkIuPsEAMMOERHRcww7WuR8xBPk5MnhWN0YdWzMKro5RERElQLDjhZ5Pl6ni4cNJC9PtkNERPSGYtjREkIInHw2XqdzPV7CIiIieo5hR0uExafikSwLRvo6XPSTiIjoBQw7WuL5XVjeta1gpK9bwa0hIiKqPBh2tITiEhbvwiIiIlLCsKMFkjNycPn+UwBA53rWFdwaIiKiyoVhRwucCU+EXADutmZwrM7VzYmIiF7EsKMFeAmLiIioeAw7VVy+XODU8/l1eMs5ERFRIQw7VVxITDKeZuTCwkgPLVyqV3RziIiIKh2GnSru+SWsju7W0NPln5OIiOhlPDtWcS8uEUFERESFMexUYfEpWbj5KAUSCdDJnbecExERFYVhpwp7fgnL07EaapgZVnBriIiIKieGnSrs+RIRvIRFRERUPIadKio7Lx9nIxIBMOwQERGVhGGniroUlYSMnHzYmBuioYNFRTeHiIio0mLYqaJO3n4MAHirnjUkEkkFt4aIiKjyYtiponjLORERkWoYdqqgqMR0RCWmQ19XgvZ1ecs5ERFRSRh2qqDnd2G1drOEmaFeBbeGiIiocmPYqYIUq5xz4U8iIqJSMexUMenZebgY9QQAx+sQERGpgmGnijkbkYjcfAGXGiZwszKt6OYQERFVegw7VcyLl7B4yzkREVHpGHaqECEEbzknIiJSE8NOFXLzUQriU7JhYqCLNrUsK7o5REREVQLDThVy6lmvTrs6VjDU063g1hAREVUNDDtVyAneck5ERKQ2tcPOyZMny6MdVIqk9BwExyQDADp7cNZkIiIiVakddnr06IHatWtjwYIFiImJKY82URFO30mAEEB9ewvYS40rujlERERVhtph5+HDh5gwYQL+/PNP1KpVC927d8fOnTuRk5NTHu2jZ048W+W8C3t1iIiI1KJ22LGyssLkyZMREhKCixcvwt3dHePHj4eDgwO++OILXL16tTza+UbLy5fjNG85JyIieiVlGqDcvHlzzJw5ExMmTEBaWhrWrVuHFi1aoEOHDrh586am2vjGC45JRkpWHqqZ6KOpU/WKbg4REVGV8kphJzc3F3/++Sd69eoFFxcXHDlyBMuXL0d8fDwiIiLg4uKCwYMHa6SBDx8+xAcffIAaNWrA2NgYjRs3RlBQkOJ5IQTmzJkDe3t7GBsbo1u3bggPD9dI3ZXF87uwOrlbQ1eHsyYTERGpQ+2w8/nnn8Pe3h6ffPIJ3N3dERwcjICAAHz88ccwNTWFq6srfvrpJ9y+fbvMjXv69CnatWsHfX19/PPPP7h16xZ+/vlnVK/+X+/G4sWLsXTpUqxcuRIXL16EqakpunfvjqysrDLXX1k8XyKCl7CIiIjUp6dugVu3bmHZsmUYOHAgDA0Ni9zHyspKI7eoL1q0CE5OTli/fr1im5ubm+L/hRD49ddf8c0336Bfv34AgE2bNsHW1hb79u3DsGHDytyGivYwORO341KhIyno2SEiIiL1qN2z4+/vj/fee6/YoAMAenp66NSpU5kaBgD79+9Hy5YtMXjwYNjY2KBZs2ZYs2aN4vmoqCjExcWhW7duim1SqRRt2rRBQEBAscfNzs5GSkqK0qOyet6r09y5OqqZGFRwa4iIiKoetcOOn58f1q1bV2j7unXrsGjRIo006rm7d+/i999/R926dXHkyBF89tln+OKLL7Bx40YAQFxcHADA1tZWqZytra3iuaL4+flBKpUqHk5OThpttyY9XyKiMy9hERERvRK1w86qVavg4eFRaHvDhg2xcuVKjTTqOblcjubNm+OHH35As2bNMG7cOIwdO7bM9cycORMymUzxqKyTI2bl5uNcxBMAXCKCiIjoVakdduLi4mBvb19ou7W1NWJjYzXSqOfs7e3RoEEDpW3169dHdHQ0AMDOzg4AEB8fr7RPfHy84rmiGBoawsLCQulRGV24+wSZufmwlxqhvr15RTeHiIioSlI77Dg5OeHcuXOFtp87dw4ODg4aadRz7dq1Q1hYmNK2O3fuwMXFBUDBYGU7Ozv4+/srnk9JScHFixfh5eWl0bZUhINXHwEAWrtZQiLhLedERESvQu27scaOHYtJkyYhNzcXXbp0AVAwaHnatGn48ssvNdq4yZMnw9vbGz/88AOGDBmCS5cuYfXq1Vi9ejUAQCKRYNKkSViwYAHq1q0LNzc3zJ49Gw4ODujfv79G2/K6bb8UjT+vPAQA7L/6CN61a2BoK+cKbhUREVHVIxFCCHUKCCEwY8YMLF26VLEelpGREaZPn445c+ZovIEHDx7EzJkzER4eDjc3N0yZMgVjx45Vas/cuXOxevVqJCcno3379vjtt9/g7u6uch0pKSmQSqWQyWSV4pJWrCwT7RaegPyFv4yuRIKzMzpzEVAiIqJnVD1/qx12nktLS0NoaCiMjY1Rt27dEm9Fr+wqW9g5H5mI99dcLLR929i28KpdowJaREREVPmoev5W+zLWc2ZmZmjVqtWrFqcSOFYv3HujK5HA1cqkAlpDRERUtb1S2AkKCsLOnTsRHR2tuJT13J49ezTSsDfZrUepSj/rSiT4YWAjXsIiIiJ6BWrfjbV9+3Z4e3sjNDQUe/fuRW5uLm7evIkTJ05AKpWWRxvfONsuFdxa/2FbF2wb2xZnZ3Tm4GQiIqJXpHbY+eGHH/DLL7/gwIEDMDAwwP/+9z/cvn0bQ4YMgbMzT8hl9eBpBs6EPwYAfNzBDV61a7BHh4iIqAzUDjuRkZHo3bs3AMDAwADp6emQSCSYPHmy4pZwenU7A2MgBNCuTg241DCt6OYQERFVeWqHnerVqyM1tWBMSc2aNXHjxg0AQHJyMjIyMjTbujdMXr4cO4IKlq54rzV7yYiIiDRB7QHKHTt2xLFjx9C4cWMMHjwYEydOxIkTJ3Ds2DF07dq1PNr4xjgV9hjxKdmwNDXA2w1sSy9AREREpVI77CxfvhxZWVkAgK+//hr6+vo4f/48Bg0ahG+++UbjDXyTPB+Y/G4LRxjq6VZwa4iIiLSDWmEnLy8PBw8eRPfu3QEAOjo6mDFjRrk07E0TK8vEybAEAMDQVk4V3BoiIiLtodaYHT09PXz66aeKnh3SnJ2BDyAXQBs3S9S2Nqvo5hAREWkNtQcot27dGiEhIeXQlDdXvlxg57OBye+34cBkIiIiTVJ7zM748eMxZcoUxMTEoEWLFjA1Vb49ukmTJhpr3JviTPhjPEzORDUTfXRvaFfRzSEiItIqaoedYcOGAQC++OILxTaJRAIhBCQSCfLz8zXXujfE9mcDkwc2c4SRPgcmExERaZLaYScqKqo82vHGSkjJwvHQgoHJ77XmwGQiIiJNUzvsuLi4lEc73li7Lj9AvlygpUt11LU1r+jmEBERaR21w86mTZtKfH7EiBGv3Jg3jVwusD2w4BLWMM6YTEREVC7UDjsTJ05U+jk3NxcZGRkwMDCAiYkJw44azkUmIiYpE+ZGeujd2L6im0NERKSV1L71/OnTp0qPtLQ0hIWFoX379ti2bVt5tFFrbb9UcLv5gGY1YWzAgclERETlQe2wU5S6deti4cKFhXp9qHiJadk4eisOADCsFS9hERERlReNhB2gYHblR48eaepwWm/35QfIzRfwdKqGBg4WFd0cIiIiraX2mJ39+/cr/SyEQGxsLJYvX4527dpprGHaTAiB7YHPZkzm7eZERETlSu2w079/f6WfJRIJrK2t0aVLF/z888+aapdWu3A3CVGJ6TAz1MM7TRwqujlERERaTe2wI5fLy6Mdb5Rtz2ZM7tvUAaaGav8JiIiISA0aG7NDqnmanoPDNwoGJr/PuXWIiIjKndphZ9CgQVi0aFGh7YsXL8bgwYM10ihttvvKA+Tky9GopgUa1ZRWdHOIiIi0ntph58yZM+jVq1eh7T179sSZM2c00iht9eLA5PfYq0NERPRaqB120tLSYGBgUGi7vr4+UlJSNNKoymbhwoWQSCSYNGkSAODevXuQSCRFPnbt2lXscXR0dOD/5Vu4v+gdfNDWVVHmxx9/VOzTt29fODs7w8jICPb29vjwww95Sz8REVEZqB12GjdujB07dhTavn37djRo0EAjjapMAgMDsWrVKjRp0kSxzcnJCbGxsUqPb7/9FmZmZujZs2exxxq38hgcfTdj/Gp/xMbGYt26dZBIJBg0aJBin86dO2Pnzp0ICwvD7t27ERkZiXfffbdcXyMREZE2U/tWoNmzZ2PgwIGIjIxEly5dAAD+/v7Ytm1bib0aVVFaWhqGDx+ONWvWYMGCBYrturq6sLOzU9p37969GDJkCMzMzIo8liwjF6dicqFrVh0fd28GO7vq+Ouvv9C5c2fUqlVLsd/kyZMV/+/i4oIZM2agf//+yM3Nhb6+voZfIRERkfZTu2enT58+2LdvHyIiIjB+/Hh8+eWXePDgAY4fP15oDp6qztfXF71790a3bt1K3O/y5csICQnBmDFjit1nb/ADZOfJ4WFnjqZO1RAfH49Dhw6VWCYpKQlbt26Ft7c3gw4REdEreqVJXnr37o3evXtrui2Vyvbt23HlyhUEBgaWuu/atWtRv359eHt7F/n8ywOTJRIJNm7cCHNzcwwcOLDQ/tOnT8fy5cuRkZGBtm3b4uDBg2V7MURERG8wtXt2AgMDcfHixULbL168iKCgII00qqLFxMRg4sSJ2Lp1K4yMjErcNzMzE3/88UeJPTTBMcm4HZcKQz0d9G9WEwCwbt06DB8+vMjjT506FcHBwTh69Ch0dXUxYsQICCHK9qKIiIjeUGqHHV9fX8TExBTa/vDhQ/j6+mqkURXt8uXLSEhIQPPmzaGnpwc9PT2cPn0aS5cuhZ6eHvLz8xX7/vnnn8jIyMCIESOKPd72ZzMmv9PEAVJjffz7778ICwvDxx9/XOT+VlZWcHd3x9tvv43t27fj77//xoULFzT7IomIiN4Qal/GunXrFpo3b15oe7NmzXDr1i2NNKqide3aFdevX1fa9tFHH8HDwwPTp0+Hrq6uYvvatWvRt29fWFtbF3ms1KxcHLgaCwB479min2vXrkWLFi3g6elZalueL8+RnZ39Sq+FiIjoTad22DE0NER8fLzSHUQAEBsbCz097VjnydzcHI0aNVLaZmpqiho1aihtj4iIwJkzZ/D3338XeRwPDw/0GDUZmbmOqGtjhhYu1ZGSkoJdu3YVuWjqxYsXERgYiPbt26N69eqIjIzE7NmzUbt2bXh5eWn2RRIREb0h1L6M5ePjg5kzZ0Imkym2JScnY9asWXj77bc12rjKbt26dXB0dISPj0+Rz4eFheFw8F0AwLBnA5O3b98OIQTee++9QvubmJhgz5496Nq1K+rVq4cxY8agSZMmOH36NAwNDcv1tRAREWkriVBz5OvDhw/RsWNHPHnyBM2aNQMAhISEwNbWFseOHYOTk1O5NLQ8paSkQCqVQiaTwcLCQiPH3BEYjRl7ruP5b3fOOw0wur2bRo5NREREqp+/1Q47AJCeno6tW7fi6tWrMDY2RpMmTfDee+9V2blgNB12YmWZaLfwBOQv/GZ1JRKcndEZ9lLjMh+fiIiIVD9/v9IgG1NTU4wbN+6VG6ftohLTlYIOAOQLgXuJGQw7REREr9krjyi+desWoqOjkZOTo7S9b9++ZW5UVedmZQodCQr17LhamVRco4iIiN5Qaoedu3fvYsCAAbh+/TokEolisjuJRAIASnPQvKnspcbwG9gYs/bcQL4Q0JVI8MPARuzVISIiqgBqh52JEyfCzc0N/v7+cHNzw6VLl/DkyRN8+eWX+Omnn8qjjVXS0FbO6OhujXuJGXC1MmHQISIiqiBqh52AgACcOHECVlZW0NHRgY6ODtq3bw8/Pz988cUXCA4OLo92Vkn2UmOGHCIiogqm9jw7+fn5MDc3B1CwrMGjR48AAC4uLggLC9Ns64iIiIjKSO2enUaNGuHq1atwc3NDmzZtsHjxYhgYGGD16tWFZlUmIiIiqmhqh51vvvkG6enpAIDvvvsO77zzDjp06IAaNWpgx44dGm8gERERUVm80qSCL0tKSkL16tUVd2RVNeUxgzIRERGVr3KdVPBllpaWmjgMERERkcapPUCZiIiIqCph2CEiIiKtxrBDREREWk3tsHPmzBnk5eUV2p6Xl4czZ85opFFEREREmqJ22OncuTOSkpIKbZfJZOjcubNGGkVERESkKWqHHSFEkbeYP3nyBKamphppFBEREZGmqHzr+cCBAwEUrG4+atQoGBoaKp7Lz8/HtWvX4O3trfkWEhEREZWBymFHKpUCKOjZMTc3h7HxfwtcGhgYoG3bthg7dqzmW0hERERUBiqHnfXr1wMAXF1d8dVXX/GSFREREVUJao/ZmTZtmtKYnfv37+PXX3/F0aNHNdowIiIiIk1QO+z069cPmzZtAgAkJyejdevW+Pnnn9GvXz/8/vvvGm8gERERUVmoHXauXLmCDh06AAD+/PNP2NnZ4f79+9i0aROWLl2q8QYSERERlYXaYScjIwPm5uYAgKNHj2LgwIHQ0dFB27Ztcf/+fY03kIiIiKgs1A47derUwb59+xATE4MjR47Ax8cHAJCQkFDi8upEREREFUHtsDNnzhx89dVXcHV1RevWreHl5QWgoJenWbNmGm8gERERUVmoHXbeffddREdHIygoCEeOHFFs79q1K3755ReNNu5lCxcuhEQiwaRJkxTbsrKy4Ovrixo1asDMzAyDBg1CfHx8ubaDiIiIqo5XWvXczs4O5ubmOHbsGDIzMwEArVq1goeHh0Yb96LAwECsWrUKTZo0Udo+efJkHDhwALt27cLp06fx6NEjxWzPRERERGqHnSdPnqBr165wd3dHr169EBsbCwAYM2YMvvzyS403EADS0tIwfPhwrFmzBtWrV1dsl8lkWLt2LZYsWYIuXbqgRYsWWL9+Pc6fP48LFy6US1uIiIioalE77EyePBn6+vqIjo6GiYmJYvvQoUNx+PBhjTbuOV9fX/Tu3RvdunVT2n758mXk5uYqbffw8ICzszMCAgKKPV52djZSUlKUHkRERKSdVF4u4rmjR4/iyJEjcHR0VNpet27dcrn1fPv27bhy5QoCAwMLPRcXFwcDAwNUq1ZNabutrS3i4uKKPaafnx++/fZbTTeViIiIKiG1e3bS09OVenSeS0pKUloJXRNiYmIwceJEbN26FUZGRho77syZMyGTyRSPmJgYjR2biIiIKhe1w06HDh0Uy0UAgEQigVwux+LFi9G5c2eNNu7y5ctISEhA8+bNoaenBz09PZw+fRpLly6Fnp4ebG1tkZOTg+TkZKVy8fHxsLOzK/a4hoaGsLCwUHoQERGRdlL7MtbixYvRtWtXBAUFIScnB9OmTcPNmzeRlJSEc+fOabRxXbt2xfXr15W2ffTRR/Dw8MD06dPh5OQEfX19+Pv7Y9CgQQCAsLAwREdHK+b/ISIiojeb2mGnUaNGuHPnDpYvXw5zc3OkpaVh4MCB8PX1hb29vUYbZ25ujkaNGiltMzU1RY0aNRTbx4wZgylTpsDS0hIWFhb4/PPP4eXlhbZt22q0LURERFQ1qR12oqOj4eTkhK+//rrI55ydnTXSMFX98ssv0NHRwaBBg5CdnY3u3bvjt99+e61tICIiospLIoQQ6hTQ1dVFbGwsbGxslLY/efIENjY2yM/P12gDX4eUlBRIpVLIZDKO3yEiIqoiVD1/qz1AWQgBiURSaHtaWppG75giIiIi0gSVL2NNmTIFQMHdV7Nnz1a6/Tw/Px8XL15E06ZNNd5AIiIiorJQOewEBwcDKOjZuX79OgwMDBTPGRgYwNPTE1999ZXmW0hERERUBiqHnZMnTwIouPX7f//7H8e2EBERUZWg9t1Y69evL492EBEREZULtQcoExEREVUlDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Sp12PHz80OrVq1gbm4OGxsb9O/fH2FhYUr7ZGVlwdfXFzVq1ICZmRkGDRqE+Pj4CmoxERERVTaVOuycPn0avr6+uHDhAo4dO4bc3Fz4+PggPT1dsc/kyZNx4MAB7Nq1C6dPn8ajR48wcODACmw1ERERVSYSIYSo6Eao6vHjx7CxscHp06fRsWNHyGQyWFtb448//sC7774LALh9+zbq16+PgIAAtG3bVqXjpqSkQCqVQiaTwcLCojxfAhEREWmIqufvSt2z8zKZTAYAsLS0BABcvnwZubm56Natm2IfDw8PODs7IyAgoNjjZGdnIyUlRelBRERE2qnKhB25XI5JkyahXbt2aNSoEQAgLi4OBgYGqFatmtK+tra2iIuLK/ZYfn5+kEqlioeTk1N5Np2IiIgqUJUJO76+vrhx4wa2b99e5mPNnDkTMplM8YiJidFAC4mIiKgy0qvoBqhiwoQJOHjwIM6cOQNHR0fFdjs7O+Tk5CA5OVmpdyc+Ph52dnbFHs/Q0BCGhobl2WQiIiKqJCp1z44QAhMmTMDevXtx4sQJuLm5KT3fokUL6Ovrw9/fX7EtLCwM0dHR8PLyet3NJSIiokqoUvfs+Pr64o8//sBff/0Fc3NzxTgcqVQKY2NjSKVSjBkzBlOmTIGlpSUsLCzw+eefw8vLS+U7sYiIiEi7VepbzyUSSZHb169fj1GjRgEomFTwyy+/xLZt25CdnY3u3bvjt99+K/Ey1st46zkREVHVo+r5u1KHndeFYYeIiKjq0cp5doiIiIjUxbBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIq2lN2FmxYgVcXV1hZGSENm3a4NKlSxXdJCIiIqoEtCLs7NixA1OmTMHcuXNx5coVeHp6onv37khISKjophEREVEF04qws2TJEowdOxYfffQRGjRogJUrV8LExATr1q2r6KYRERFRBdOr6AaUVU5ODi5fvoyZM2cqtuno6KBbt24ICAgoskx2djays7MVP8tkMgBASkpK+TaWiIiINOb5eVsIUeJ+VT7sJCYmIj8/H7a2tkrbbW1tcfv27SLL+Pn54dtvvy203cnJqVzaSEREROUnNTUVUqm02OerfNh5FTNnzsSUKVMUP8vlciQlJaFGjRqQSCQaqyclJQVOTk6IiYmBhYXFay3Pul9/3WUtz7rfrLrLWp51s+6qUr6sdZdECIHU1FQ4ODiUuF+VDztWVlbQ1dVFfHy80vb4+HjY2dkVWcbQ0BCGhoZK26pVq1ZeTYSFhUWZ/sBlKc+6X3/dZS3Put+sustannWz7qpSvqx1F6ekHp3nqvwAZQMDA7Ro0QL+/v6KbXK5HP7+/vDy8qrAlhEREVFlUOV7dgBgypQpGDlyJFq2bInWrVvj119/RXp6Oj766KOKbhoRERFVMK0IO0OHDsXjx48xZ84cxMXFoWnTpjh8+HChQcuvm6GhIebOnVvoktnrKM+6X3/dZS3Put+sustannWz7qpSvqx1a4JElHa/FhEREVEVVuXH7BARERGVhGGHiIiItBrDDhEREWk1hh0iIiLSagw75WjFihVwdXWFkZER2rRpg0uXLqlU7syZM+jTpw8cHBwgkUiwb98+lev08/NDq1atYG5uDhsbG/Tv3x9hYWEql//999/RpEkTxeRPXl5e+Oeff1Qu/6KFCxdCIpFg0qRJKu0/b948SCQSpYeHh4fK9T18+BAffPABatSoAWNjYzRu3BhBQUEqlXV1dS1Ut0Qiga+vb6ll8/PzMXv2bLi5ucHY2Bi1a9fG/PnzS12r5UWpqamYNGkSXFxcYGxsDG9vbwQGBhbar7T3hhACc+bMgb29PYyNjdGtWzeEh4erXH7Pnj3w8fFRzCYeEhKicv25ubmYPn06GjduDFNTUzg4OGDEiBF49OiRSnXPmzcPHh4eMDU1RfXq1dGtWzdcvHhR5ba/6NNPP4VEIsGvv/6qUtlRo0YV+tv36NFDrbpDQ0PRt29fSKVSmJqaolWrVoiOji61bFHvO4lEgh9//FGlutPS0jBhwgQ4OjrC2NhYsRiyKmXj4+MxatQoODg4wMTEBD169FC8X1T5LMnKyoKvry9q1KgBMzMzDBo0SDHBqyrlV69ejbfeegsWFhaQSCRITk5WPFda+aSkJHz++eeoV68ejI2N4ezsjC+++AIymUyluj/55BPUrl0bxsbGsLa2Rr9+/RRLDKnzOSqEQM+ePRW/X1XKvvXWW4X+3p9++qladQcEBKBLly4wNTWFhYUFOnbsiO+++67Esvfu3Sv2/bZr1y6V6o6Li8OHH34IOzs7mJqaonnz5ti9e7dKZSMjIzFgwABYW1vDwsICQ4YMKTQhcHlh2CknO3bswJQpUzB37lxcuXIFnp6e6N69OxISEkotm56eDk9PT6xYsULtek+fPg1fX19cuHABx44dQ25uLnx8fJCenq5SeUdHRyxcuBCXL19GUFAQunTpgn79+uHmzZtqtSMwMBCrVq1CkyZN1CrXsGFDxMbGKh5nz55VqdzTp0/Rrl076Ovr459//sGtW7fw888/o3r16iq398V6jx07BgAYPHhwqWUXLVqE33//HcuXL0doaCgWLVqExYsXY9myZSrVDQAff/wxjh07hs2bN+P69evw8fFBt27d8PDhQ6X9SntvLF68GEuXLsXKlStx8eJFmJqaonv37sjKylKpfHp6Otq3b49FixYV+3xx5TMyMnDlyhXMnj0bV65cwZ49exAWFoa+ffuqVLe7uzuWL1+O69ev4+zZs3B1dYWPjw8eP36sUvnn9u7diwsXLihNH69K2R49eii9B7Zt26Zy+cjISLRv3x4eHh44deoUrl27htmzZ8PIyKjUsi/WGRsbi3Xr1kEikWDQoEEq1T1lyhQcPnwYW7ZsQWhoKCZNmoQJEyZg//79JZYVQqB///64e/cu/vrrLwQHB8PFxQXdunVDenq6Sp8lkydPxoEDB7Br1y6cPn0ajx49wsCBAwGo9lmUkZGBHj16YNasWYXaV1r5R48e4dGjR/jpp59w48YNbNiwAYcPH8aYMWNUqrtFixZYv349QkNDceTIEQgh4OPjg/z8fLU+R3/99VelZYZULTt27Filv/vixYtVLh8QEIAePXrAx8cHly5dQmBgICZMmICzZ8+WWNbJyanQ++3bb7+FmZkZevbsqVLdI0aMQFhYGPbv34/r169j4MCBGDJkCA4cOFBi2fT0dPj4+EAikeDEiRM4d+4ccnJy0KdPH8jl8kK/V40TVC5at24tfH19FT/n5+cLBwcH4efnp9ZxAIi9e/e+cjsSEhIEAHH69OlXPkb16tXF//3f/6m8f2pqqqhbt644duyY6NSpk5g4caJK5ebOnSs8PT1fqY3Tp08X7du3f6WyRZk4caKoXbu2kMvlpe7bu3dvMXr0aKVtAwcOFMOHD1eproyMDKGrqysOHjyotL158+bi66+/Lrbcy+8NuVwu7OzsxI8//qjYlpycLAwNDcW2bdtKLf+iqKgoAUAEBwerXH9RLl26JACI+/fvq11WJpMJAOL48eMq1/3gwQNRs2ZNcePGDeHi4iJ++eUXlcqOHDlS9OvXr8T2lFR+6NCh4oMPPnilsi/r16+f6NKli8rlGzZsKL777julbUW9d14uGxYWJgCIGzduKLbl5+cLa2trsWbNmkJ1v/xZkpycLPT19cWuXbsU+4SGhgoAIiAgoNTyLzp58qQAIJ4+fVrk6y6t/HM7d+4UBgYGIjc3V+2yV69eFQBERESEynUHBweLmjVritjY2GL/tkWVVedzsajybdq0Ed98880rlX1Z06ZNC31+lVTe1NRUbNq0SWk/S0vLQu+Zl8seOXJE6OjoCJlMptgnOTlZSCQScezYsVJfS1mxZ6cc5OTk4PLly+jWrZtim46ODrp164aAgIDX2haZTAYAsLS0VLtsfn4+tm/fjvT0dLWW3vD19UXv3r2VXr+qwsPD4eDggFq1amH48OGIjo5Wqdz+/fvRsmVLDB48GDY2NmjWrBnWrFmjdv1Awd9vy5YtGD16tEoLw3p7e8Pf3x937twBAFy9ehVnz55Fz549VaovLy8P+fn5MDIyUtpubGyscs8WAERFRSEuLk7p9y6VStGmTZvX/r57TiaTQSKRqL32XE5ODlavXg2pVApPT0+Vysjlcnz44YeYOnUqGjZsqHZbT506BRsbG9SrVw+fffYZnjx5onK9hw4dgru7O7p37w4bGxu0adNGrcvPz8XHx+PQoUMYM2aMymW8vb2xf/9+PHz4EEIInDx5Enfu3IGPj0+J5bKzswFA6X2no6MDQ0PDIt93L3+WXL58Gbm5uUrvNw8PDzg7Oxf5fivLZ5Gq5WUyGSwsLKCnp1doe0ll09PTsX79eri5ucHJyUmlujMyMvD+++9jxYoVxa7DWFLdW7duhZWVFRo1aoSZM2ciIyNDpfIJCQm4ePEibGxs4O3tDVtbW3Tq1Emlv9nLLl++jJCQkGLfb0WV9/b2xo4dO5CUlAS5XI7t27cjKysLb731Volls7OzIZFIlCYWNDIygo6Ojlqfc6+s3OPUG+jhw4cCgDh//rzS9qlTp4rWrVurdSyUoWcnPz9f9O7dW7Rr106tcteuXROmpqZCV1dXSKVScejQIZXLbtu2TTRq1EhkZmYKIdT7BvP333+LnTt3iqtXr4rDhw8LLy8v4ezsLFJSUkota2hoKAwNDcXMmTPFlStXxKpVq4SRkZHYsGGDym1/bseOHUJXV1c8fPhQpf3z8/PF9OnThUQiEXp6ekIikYgffvhBrTq9vLxEp06dxMOHD0VeXp7YvHmz0NHREe7u7sWWefm9ce7cOQFAPHr0SGm/wYMHiyFDhpRa/kWa6NnJzMwUzZs3F++//77KZQ8cOCBMTU2FRCIRDg4O4tKlSyrX/cMPP4i3335b0RunTs/Otm3bxF9//SWuXbsm9u7dK+rXry9atWol8vLySi3//Fu9iYmJWLJkiQgODhZ+fn5CIpGIU6dOqfS6n1u0aJGoXr264t+PKm3PysoSI0aMEACEnp6eMDAwEBs3biy1bE5OjnB2dhaDBw8WSUlJIjs7WyxcuFAAED4+Pkpli/os2bp1qzAwMChUT6tWrcS0adNKLf+i0np2VPkse/z4sXB2dhazZs1SueyKFSuEqampACDq1atXZK9OceXHjRsnxowZo/i5qL9NcWVXrVolDh8+LK5duya2bNkiatasKQYMGKBS3QEBAQKAsLS0FOvWrRNXrlwRkyZNEgYGBuLOnTsqve7nPvvsM1G/fv0inyuu/NOnT4WPj4/i/WZhYSGOHDlSatmEhARhYWEhJk6cKNLT00VaWpqYMGGCACDGjRtXbBs1hWGnHFSWsPPpp58KFxcXERMTo1a57OxsER4eLoKCgsSMGTOElZWVuHnzZqnloqOjhY2Njbh69apimzph52VPnz4VFhYWKl1C09fXF15eXkrbPv/8c9G2bVu16/Xx8RHvvPOOyvtv27ZNODo6im3btolr166JTZs2CUtLS7WCVkREhOjYsaMAIHR1dUWrVq3E8OHDhYeHR7FlKnPYycnJEX369BHNmjVT6rYurWxaWpoIDw8XAQEBYvTo0cLV1VXEx8eXWj4oKEjY2toqBVR1ws7LIiMjVb6E9vzf+3vvvae0X58+fcSwYcPUqrtevXpiwoQJxT5fVPkff/xRuLu7i/3794urV6+KZcuWCTMzs0KXBooqGxQUJDw9PRXvu+7du4uePXuKHj16KO1X1GeJOmGntM+i0sJOaeVlMplo3bq16NGjh8jJyVG5bHJysrhz5444ffq06NOnj2jevHmhoFlU+b/++kvUqVNHpKamKrYV9ftV9TPY39+/yEtoRZV//u985syZSvs2btxYzJgxQ+W6MzIyhFQqFT/99FORzxdXfsKECaJ169bi+PHjIiQkRMybN09IpVJx7dq1UsseOXJE1KpVS0gkEqGrqys++OAD0bx5c/Hpp5+W8NvRDIadcpCdnS10dXULvfFHjBgh+vbtq9axXjXs+Pr6CkdHR3H37l21y76sa9euKiXvvXv3Kj40nz8AKN7YRX1LLk3Lli2V/gEXx9nZWelblhBC/Pbbb8LBwUGt+u7duyd0dHTEvn37VC7j6Ogoli9frrRt/vz5ol69emrVLUTByf55WBkyZIjo1atXsfu+/N54foJ+OaB07NhRfPHFF6WWf1FZwk5OTo7o37+/aNKkiUhMTFSr7Mvq1KlTZC/Zy+V/+eUXxfvsxfeejo6OcHFxeaW6raysxMqVK0utOzs7W+jp6Yn58+cr7Tdt2jTh7e2tct1nzpwRAERISEixbXq5fEZGhtDX1y803mvMmDGie/fuKtednJwsEhIShBAF4w3Hjx+veK64z5LnJ+iXA4qzs7NYsmRJqeVfVFLYKa18SkqK8PLyEl27di0UVNT5HMzOzhYmJibijz/+KLX8xIkTi32/derUSe2609LSBABx+PDhUuu+e/euACA2b96stH3IkCGKXlRV6t60aZPQ19dX/N1fVFz5iIiIQuO8hCg4R3zyyScq1/348WPF39rW1lYsXry42H01hWN2yoGBgQFatGgBf39/xTa5XA5/f3+1xr68CiEEJkyYgL179+LEiRNwc3Mr8zHlcrni+n5JunbtiuvXryMkJETxaNmyJYYPH46QkBDo6uqqVW9aWhoiIyNhb29f6r7t2rUrdJvjnTt34OLiolad69evh42NDXr37q1ymYyMDOjoKP9T0tXVfaU7DExNTWFvb4+nT5/iyJEj6Nevn8pl3dzcYGdnp/S+S0lJwcWLF8v9ffdcbm4uhgwZgvDwcBw/fhw1atQo0/FUfe99+OGHuHbtmtJ7z8HBAVOnTsWRI0fUrvfBgwd48uSJSu89AwMDtGrVqszvv7Vr16JFixYqj1ECCn7fubm5ZX7/SaVSWFtbIzw8HEFBQejXr1+pnyUtWrSAvr6+0vstLCwM0dHR8PLyKvNnkSrlU1JS4OPjAwMDA+zfv18x/uhV6hYFX/6RnZ1davkZM2YUer8BwC+//IJ169apXffz8vb29qXW7erqCgcHhyLfb87OzirXvXbtWvTt2xfW1tZKv4OSyj8fV1TU+y0/P1/luq2srFCtWjWcOHECCQkJijs2y1W5x6k31Pbt24WhoaHYsGGDuHXrlhg3bpyoVq2aiIuLK7VsamqqCA4OFsHBwQKAYhzAy3e0FOWzzz4TUqlUnDp1SsTGxioeGRkZKrV7xowZ4vTp0yIqKkpcu3ZNzJgxQ0gkEnH06FGVyr9MnctYX375pTh16pSIiooS586dE926dRNWVlZFfvN42aVLl4Senp74/vvvRXh4uNi6daswMTERW7ZsUbmt+fn5wtnZWUyfPl3lMkIU3MlTs2ZNcfDgQREVFSX27NkjrKysCnXll+Tw4cPin3/+EXfv3hVHjx4Vnp6eok2bNoW65Et7byxcuFBUq1ZNMf6kX79+ws3NTfGNt7TyT548EcHBweLQoUMCgNi+fbsIDg4WsbGxpZbPyckRffv2FY6OjiIkJETp/ZednV1i2bS0NDFz5kwREBAg7t27J4KCgsRHH30kDA0NFd8i1f138eJlrJLKpqamiq+++koEBASIqKgocfz4cdG8eXNRt25dkZWVpVLde/bsEfr6+mL16tUiPDxcLFu2TOjq6op///1XpXbLZDJhYmIifv/990Kvo7TynTp1Eg0bNhQnT54Ud+/eFevXrxdGRkbit99+K7Xszp07xcmTJ0VkZKTYt2+fcHFxEQMHDhRCqPZZ8umnnwpnZ2dx4sQJERQUJLy8vBSXk1UpHxsbK4KDg8WaNWsEAHHmzBkRHBwsnjx5Ump5mUwm2rRpIxo3biwiIiKU9vn0009LLBsZGSl++OEHERQUJO7fvy/OnTsn+vTpIywtLUV8fPwrfY7iWc9ZaWUjIiLEd999J4KCgkRUVJT466+/RK1atUTHjh1V/r398ssvwsLCQuzatUuEh4eLb775RhgZGYn3339fpXaHh4cLiUQi/vnnH6XtpdWdk5Mj6tSpIzp06CAuXrwoIiIixE8//SQkEono1atXqXWvW7dOBAQEiIiICLF582ZhaWkppkyZUuzvVJMYdsrRsmXLhLOzszAwMBCtW7cWFy5cUKnc8y7dlx8jR44stWxR5QCI9evXq1T36NGjhYuLizAwMBDW1taia9eurxx0hFAv7AwdOlTY29sLAwMDUbNmTTF06NAiBwwW58CBA6JRo0bC0NBQeHh4iNWrV6vV1iNHjggAIiwsTK1yKSkpYuLEicLZ2VkYGRmJWrVqia+//lpkZ2erfIwdO3aIWrVqCQMDA2FnZyd8fX1FcnJyof1Ke2/I5XIxe/ZsYWtrKwwNDUXXrl2VXk9p5devX1/k83Pnzi21/PNLX0U9Tp48WWLZzMxMMWDAAOHg4CAMDAyEvb296Nu3r9IAZXX/XbwYdkoqm5GRIXx8fIS1tbXQ19cXLi4uYuzYsUpfTFSpe+3ataJOnTrCyMhIeHp6Ki6FqlJ21apVwtjY+JX+5rGxsWLUqFHCwcFBGBkZiXr16omff/5ZyOXyUsv+73//E46OjkJfX184OzuLb775RvG+VeWzJDMzU4wfP15Ur15dmJiYiAEDBiiCsSrl586dW+w+pZUv7rWV9Hhe9uHDh6Jnz57CxsZG6OvrC0dHR/H++++L27dvq9z2lz0PO6WVjY6OFh07dhSWlpbC0NBQ1KlTR0ydOlUxtk3Vuv38/ISjo6MwMTERXl5e4t9//1W57MyZM4WTk5PIz88v9BpKK3/nzh0xcOBAYWNjI0xMTESTJk3Epk2bVCo7ffp0YWtrK/T19UXdunUV79PXQfLsBRIRERFpJY7ZISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQEb3k1KlTkEgkSE5OruimEJEGMOwQERGRVmPYISIiIq3GsENElY5cLoefnx/c3NxgbGwMT09P/PnnnwD+u8R06NAhNGnSBEZGRmjbti1u3LihdIzdu3ejYcOGMDQ0hKurK37++Wel57OzszF9+nQ4OTnB0NAQderUwdq1a5X2uXz5Mlq2bAkTExN4e3sXWmmaiKoGhh0iqnT8/PywadMmrFy5Ejdv3sTkyZPxwQcf4PTp04p9pk6dip9//hmBgYGwtrZGnz59kJubC6AgpAwZMgTDhg3D9evXMW/ePMyePRsbNmxQlB8xYgS2bduGpUuXIjQ0FKtWrYKZmZlSO77++mv8/PPPCAoKgp6eHkaPHv1aXj8RaRYXAiWiSiU7OxuWlpY4fvw4vLy8FNs//vhjZGRkYNy4cejcuTO2b9+OoUOHAgCSkpLg6OiIDRs2YMiQIRg+fDgeP36Mo0ePKspPmzYNhw4dws2bN3Hnzh3Uq1cPx44dQ7du3Qq14dSpU+jcuTOOHz+Orl27AgD+/vtv9O7dG5mZmTAyMirn3wIRaRJ7doioUomIiEBGRgbefvttmJmZKR6bNm1CZGSkYr8Xg5ClpSXq1auH0NBQAEBoaCjatWundNx27dohPDwc+fn5CAkJga6uLjp16lRiW5o0aaL4f3t7ewBAQkJCmV8jEb1eehXdACKiF6WlpQEADh06hJo1ayo9Z2hoqBR4XpWxsbFK++nr6yv+XyKRACgYT0REVQt7doioUmnQoAEMDQ0RHR2NOnXqKD2cnJwU+124cEHx/0+fPsWdO3dQv359AED9+vVx7tw5peOeO3cO7u7u0NXVRePGjSGXy5XGABGR9mLPDhFVKubm5vjqq68wefJkyOVytG/fHjKZDOfOnYOFhQVcXFwAAN999x1q1KgBW1tbfP3117CyskL//v0BAF9++SVatWqF+fPnY+jQoQgICMDy5cvx22+/AQBcXV0xcuRIjB49GkuXLoWnpyfu37+PhIQEDBkypKJeOhGVE4YdIqp05s+fD2tra/j5+eHu3buoVq0amjdvjlmzZikuIy1cuBATJ05EeHg4mjZtigMHDsDAwAAA0Lx5c+zcuRNz5szB/PnzYW9vj++++w6jRo1S1PH7779j1qxZGD9+PJ48eQJnZ2fMmjWrIl4uEZUz3o1FRFXK8zulnj59imrVqlV0c4ioCuCYHSIiItJqDDtERESk1XgZi4iIiLQae3aIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIq/0/aNuF5S+4oysAAAAASUVORK5CYII=", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAG2CAYAAACZEEfAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABqLElEQVR4nO3dd1RUR8MG8GfpRUDpINWKir2iRmMJthhbLLGXaEww1lgwMTExippoiia2L2KNLVFjib1GgwgodhGwoEgRkaW33fn+4GXjStuVpbg8v3P2nHD3zp25uNn7MHfujEQIIUBERESkpXQqugFEREREZYlhh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLRahYad8+fPo2/fvnB0dIREIsH+/fuV3hdC4Msvv4SDgwOMjY3RvXt3hIeHK+2TmJiIESNGwNzcHNWrV8eECROQmppajmdBRERElVmFhp20tDQ0bdoUv/zyS6HvL1++HD///DPWrl2LwMBAmJqaokePHsjMzFTsM2LECNy6dQsnTpzAoUOHcP78eUyaNKm8ToGIiIgqOUllWQhUIpFg37596N+/P4C8Xh1HR0fMmjULn332GQBAKpXCzs4OmzZtwrBhw3Dnzh00bNgQQUFBaNWqFQDg6NGj6N27N548eQJHR8eKOh0iIiKqJPQqugFFefDgAWJjY9G9e3fFNgsLC7Rt2xYBAQEYNmwYAgICUL16dUXQAYDu3btDR0cHgYGBGDBgQKHHzsrKQlZWluJnuVyOxMREWFlZQSKRlN1JERERkcYIIZCSkgJHR0fo6BR9s6rShp3Y2FgAgJ2dndJ2Ozs7xXuxsbGwtbVVel9PTw+WlpaKfQrj5+eHr7/+WsMtJiIioorw+PFjODk5Ffl+pQ07ZcnX1xczZ85U/CyVSuHi4oLHjx/D3Ny8AltGREREqkpOToazszPMzMyK3a/Shh17e3sAQFxcHBwcHBTb4+Li0KxZM8U+8fHxSuVyc3ORmJioKF8YQ0NDGBoaFthubm7OsENERPSGKWkISqWdZ8fd3R329vY4deqUYltycjICAwPh5eUFAPDy8kJSUhJCQkIU+5w+fRpyuRxt27Yt9zYTERFR5VOhPTupqamIiIhQ/PzgwQOEhobC0tISLi4umD59Or799lvUrVsX7u7uWLBgARwdHRVPbDVo0AA9e/bExIkTsXbtWuTk5GDKlCkYNmwYn8QiIiIiABUcdoKDg9GlSxfFz/njaMaMGYNNmzZhzpw5SEtLw6RJk5CUlISOHTvi6NGjMDIyUpTZvn07pkyZgm7dukFHRweDBg3Czz//XO7nQkRERJVTpZlnpyIlJyfDwsICUqmUY3aIiIjeEKpevyvtmB0iIiIiTWDYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQERGRVmPYISIiIq3GsENERFQMmUyGBQsWwN3dHcbGxqhduzYWLVoEIYRin7Fjx0IikSi9evbsWeKxo6OjMXLkSFhZWcHY2BiNGzdGcHCw4v29e/fC29sbVlZWkEgkCA0NLYtT1Hp6Fd0AIiKiymzZsmVYs2YNNm/ejEaNGiE4OBjjxo2DhYUFpk6dqtivZ8+e8Pf3V/xsaGhY7HFfvHiBDh06oEuXLjhy5AhsbGwQHh6OGjVqKPZJS0tDx44dMWTIEEycOFHzJ1dFMOwQEREV499//0W/fv3Qp08fAICbmxt27NiBy5cvK+1naGgIe3t7lY+7bNkyODs7KwUkd3d3pX1GjRoFAHj48OFrtp4A3sYiIiIqVvv27XHq1Cncu3cPAHDt2jVcuHABvXr1Utrv7NmzsLW1Rf369fHxxx/j+fPnxR73wIEDaNWqFQYPHgxbW1s0b94cGzZsKLPzqMrYs0NERFSMefPmITk5GR4eHtDV1YVMJsPixYsxYsQIxT49e/bEwIED4e7ujsjISMyfPx+9evVCQEAAdHV1Cz3u/fv3sWbNGsycORPz589HUFAQpk6dCgMDA4wZM6a8Tq9KYNghIiIqxu7du7F9+3b8/vvvaNSoEUJDQzF9+nQ4OjoqQsmwYcMU+zdu3BhNmjRB7dq1cfbsWXTr1q3Q48rlcrRq1QpLliwBADRv3hw3b97E2rVrGXY0jLexiIiIijF79mzMmzcPw4YNQ+PGjTFq1CjMmDEDfn5+RZapVasWrK2tERERUeQ+Dg4OaNiwodK2Bg0aICoqSmNtpzwMO0RERMVIT0+Hjo7y5VJXVxdyubzIMk+ePMHz58/h4OBQ5D4dOnRAWFiY0rZ79+7B1dW1dA2mAhh2iIiIitG3b18sXrwYhw8fxsOHD7Fv3z6sXLkSAwYMAACkpqZi9uzZuHTpEh4+fIhTp06hX79+qFOnDnr06KE4Trdu3bB69WrFzzNmzMClS5ewZMkSRERE4Pfff8f69evh4+Oj2CcxMRGhoaG4EHQVAHDp6g2EhoYiNja2nM5eSwgSUqlUABBSqbSim0JERJVMcnKymDZtmnBxcRFGRkaiVq1a4vPPPxdZWVlCCCHS09OFt7e3sLGxEfr6+sLV1VVMnDhRxMbGKh3HydlFjP90tnialK7YdvDgQeHp6SkMDQ2Fh4eHWL9+vVIZf39/AaDA66uvvirz834TqHr9lgjx0hSQVVRycjIsLCwglUphbm5e0c0hIiItsysoCr57b0AuAB0JsGRAY7zb1BEJKVlISM17PUvNRkJKFp6lZim2xyZn4mlSptKxdCTAxXld4WBhXEFnU3moev1m2AHDDhERlZ0YaQY6LD0NuQavtkv6e2J4O47tUfX6zUfPiYiIytCDhLQig46xvi6szQxgXc1Q8bKpZgAbs7z/lkiAT7ZfKVD+ywM3kZqdiw871oKOjqTsT+INx7BDRERUhu7FphTYpiMBTszojNq21Uos7zewMebvvQmZENCRAA0dzHHzaTKW/H0X/4QnYMXgprA1NyqLpmsN3sYCb2MREVHZuPVUikFr/kVmjhwS5I0u1pVIsGSgJ4a2dlH5ODHSDDxMSIebtQnszY2wM+gxvj54C5k5cliaGuC795ugWwO7MjuPyopjdtTAsENEVDXESDPwICEN7tamZT7A90VaNvquvoAnLzLQuZ4NFg/wxOPEDLhZm2ik7oj4FHy6IxR3YpIBAGPbu2FeLw8Y6Re+PIU2YthRA8MO0ZurPC9e9GZ79Ykov4GN1epdUYdMLjDW/zL+CU+Ai6UJDk7pCAsTfY3Xk5Urw7IjYdh48QEAwMPeDD9/0Bz17Mw0XldlpOr1m5MKEtEba1dQFDosPY3hGwLRYelp7AriNPtUuBhpBub9L+gAgFwA8/feRIw0o0zq+/54GP4JT4Cxvi7WjWpZJkEHAAz1dPFl34bwH9ca1tUMcDc2BX1XXcDWS4/Avoz/MOwQaVCMNAP/RiaU2Rco/SdGmqH4Kx0o+4uXNpDJZFiwYAHc3d1hbGyM2rVrY9GiRUoXRSEEvvzySzg4OMDY2Bjdu3dHeHh4iceOjo7GyJEjYWVlBWNjYzRu3BjBwcGK91NTUzFlyhQ4OTnB2NgYDRs2xNq1a8vkPAtz+k48Xr32y4TAw4R0jdd15EYM1pyNBAAse78JGjiU/R2DLvVtcWRaJ3SuZ4OsXDkW7L+JiVtCkJiWXeZ1vwkYdog0ZMflKLRnL0O5Kexx3rK6eBXlTQu3y5Ytw5o1a7B69WrcuXMHy5Ytw/Lly7Fq1SrFPsuXL8fPP/+MtWvXIjAwEKampujRowcyMzOLPO6LFy/QoUMH6Ovr48iRI7h9+zZWrFiBGjVqKPaZOXMmjh49im3btuHOnTuYPn06pkyZggMHDpTpOQNA5LNUfHcsrND30rNzNVpXeFwKPttzDQDwYUd3vNfUUaPHL46NmSH8x7bGgncbwkBXByfvxKHXT+fxb0TCG/dZ1TSO2QHH7FDpRb9IR4dlZ5S26UokuDCvC8eRlJGHCWl4+/uzBbafntUZtWxKfpy3tMpz/IemvPvuu7Czs8Nvv/2m2DZo0CAYGxtj27ZtEELA0dERs2bNwmeffQYAkEqlsLOzw6ZNmzBs2LBCjztv3jxcvHgR//zzT5F1e3p6YujQoViwYIFiW8uWLdGrVy98++23GjrDgp4mZeD9Nf/iqTQTNasbI0aaoRSSjfV1sWZkC7xd37bUdSVn5qDf6ot4kJAGr1pW2DqhDfR0K6ZP4Wa0FNN2XkXkszQAUDwJ9qZ8VlXFMTtE5UQIgW8P3ymwvbx7GaqaVo3r49Gydwu8ug0ajcS0bERGRmLAgAGwsbGBubk5hgwZgri4uGKPmZKSgunTp8PV1RXGxsZo3749goKCFO/n5ORg7ty5aNDIEx90qIeo1aORcGgFspOfvxG30Nq3b49Tp07h3r17AIBr167hwoUL6NWrFwDgwYMHiI2NRffu3RVlLCws0LZtWwQEBBR53AMHDqBVq1YYPHgwbG1t0bx5c2zYsKFA3QcOHEB0dDSEEDhz5gzu3bsHb2/vMjjTPIlp2Rj1WyCeSjNRy8YUB6Z0wMV5XbFjYjucmtkZb9W1RkaODB9uDsafIU9KVZdcLjBzVygeJKTB0cIIq4c3r7CgAwCeNS1w8NOO6Ncsr2cpP9/JBeC790al/6xqGsMOaYybmxskEkmBV/4Kvq9z8Vm4cGGB43l4eCjt8/bbbxfYZ/LkyWV2nq/65UwEjtwsuAKxjgRwszYpt3ZUJcmZObAfvRJOPluxfG8ADgbcgt+GnQCADKfWGPDTKXTt/g4kEglOnz6NixcvIjs7G3379oVcLi/yuB9++CFOnDiBrVu34saNG/D29kb37t0RHR0NAEhPT8eVK1fQrO94OIz5CTb95yMnMRrP9i6CTAj8cy+hXM7/dc2bNw/Dhg2Dh4cH9PX10bx5c0yfPh0jRowAAMVK2nZ2yvO12NnZFbvK9v3797FmzRrUrVsXx44dw8cff4ypU6di8+bNin1WrVqFhg0bwsnJCQYGBujZsyd++eUXdOrUqQzOFEjNysVY/8uIfJYXPrZNaAuraoZwsDCGV20r1Latht/GtMaA5jWRKxeYteca1p6LfO1BvatOR+DknXgY6Olg7aiWsKpmqOEzUp+JgR6GtnYusF0ugImbg/FHyBOkZmn2Nl5lxbBDGhMUFISYmBjF68SJEwCAwYMHIy0tDd7e3mpffACgUaNGSse9cOFCgX0mTpyotM/y5cvL5BxftfXSI3x/PO+v5L5NHKAr+W/adj0dCZIzqsYXSXnbcP4+UiWmqF/LGTPea4N32zVE7M0AuLi5w92zNe5eC0bUo0fwXfozGjdujMaNG2Pz5s0IDg7G6dOnCz1mRkYG/vzzTyxfvhydOnVCnTp1sHDhQtSpUwdr1qzJ28nABPXGLUMA6kPfygmGNT1g+c5kZMdGIDc5HnP/vI7Ze67haVLl/Kt59+7d2L59O37//XdcuXIFmzdvxvfff68USl6HXC5HixYtsGTJEjRv3hyTJk3CxIkTlQYgr1q1CpcuXcKBAwcQEhKCFStWwMfHBydPniztaRWQmSPDxM3BuP5ECktTA2yZ0BaO1QveTjbQ08GKwU0xqVMtAMDSI3ex6NAdyNVcxOr03Tj8eCrve2Bxf080cape6nPQFHdrUxS2msTNp8n4bM81tPr2BKbuuIozd+ORIyv+u/hNxrBDGmNjYwN7e3vF69ChQ6hduzY6d+6Mixcv4uHDh9i0aZPKF598enp6Sse1trYusI+JiYnSPuUx9urAtaf48q+bAIBPu9bBquEtcGFeF2yb0Aat3KojWybw0dZgSDNyyrwtVcmzlCz8diFvTpHPvOtBT1cH2dnZ2LZtGz6a+CH2+XSEo1neSjhjN19FQORzAICRkRF0dHQKDcsAkJubC5lMBiMj5Wn3jY2NceHCBQQ9TETvn/7B4esx0NORoKenPXQkgDwrHYAEnm4OEAD2hDzB29+fhd/fdyBNr1z/9rNnz1b07jRu3BijRo3CjBkz4OfnBwCwt7cHgAI9rnFxcYr3CuPg4ICGDRsqbWvQoAGiovIG6WdkZGD+/PlYuXIl+vbtiyZNmmDKlCkYOnQovv/+e02eInJlckzdcRUB95/D1EAXm8a1Rp1ilmTQ0ZFgfu8G+KJPAwDAxosPMHXnVWTlylSq70FCGqbtDIUQwKh2rhjcqmBPSkVysDCG38DGij/EdCUSzOlRH7PeqYda1qbIzJHjwLWnGLcpCO2WnMLCA7cQ+jhJ6x5bZ9ihMpF/8Rk/fjwkEgmysrIgkUhgaPhf125JF5984eHhcHR0RK1atTBixAjFF+jLtm/fDmtra3h6esLX1xfp6WU7VuZsWDxm7vrvC27mO/UA5H2xdKxrg/WjWqNmdWM8fJ6OmbtC1f5LkYr2y5kIpGfL0NTJAj0a5V2A9+/fj6SkJIwdOxb2Fkb4Y8EY6Bka4/Gx/8OodeexNzASn332GWQyGWJiYgo9rpmZGby8vLBo0SI8ffoUMpkM27ZtQ0BAAG5HPsLQdQGITsqAq5UJ/vy4PdaObIlT09ujxs3dGDB4CI7O6Yl9n7RHG3dLZOfKse78fby1/DTWnYtEZo5qF86ylp6eDh0d5a99XV1dRe+qu7s77O3tcerUKcX7ycnJCAwMhJeXV5HH7dChA8LClJ92unfvHlxd81blzsnJQU5OTrF1a4IQAr57b+D47TgY6Olgw5hWKveyfPhWLfw0rBn0dSU4dD0G4/yDkJJZfFhNy8rFpC3BSMnMRSvXGljwbsNi968oQ1u74MK8LtgxsR0uzOuCT7rUwafd6uLUrM74y6cDxrZ3g5WpAZ6nZWPTvw/R/5eL6LbiHH4+FY6o53nfpaV5mqsyPAnGp7HAp7HKwu7duzF8+HBERUXB0dERz549Q506dTBu3DgsWbIEQgjMmzcPq1evxqRJk7Bu3bpCj3PkyBGkpqaifv36iImJwddff43o6GjcvHkTZmZ5M4SuX78erq6ucHR0xPXr1zF37ly0adMGe/fuLZNzC3mUiBH/F4jMHDn6NnXET0ObFbrq8M3ovDVxsnLlmNatLmb8LxDR63ucmI6uK84iRyaw/cO26FAnr5evR48eMDAwwMGDBxX7Hvr7CIaPm4iU+KeARIL23u8h81kU2rRp899tqVdERkZi/PjxOH/+PHR1deHZpBkS9awQE3kbNSeuxcAWNfFNP09UM9RDTk4OBg0ahCdPnuDs2bOK7w4hBM6GPcPSI3cRFpe3AKSjhRFmvFMPA1s4QVdDK1S/zszRY8eOxcmTJ7Fu3TrYONfGqYuB+H7BZ/hwwngsW7YMQN7j6UuXLsXmzZvh7u6OBQsW4Pr167h9+7ai16tbt24YMGAApkyZAiDvFnb79u3x9ddfY8iQIbh8+TImTpyI9evXK8YDvf3220hISMDq1avh6uqKc+fO4eOPP8bKlSvx8ccfl/r3IYTAkr/vYMM/D6AjAdaMbKkIw+r4J/wZJm8NQVq2DA0dzLFpfGvYmhVcZFMIAZ/fr+DvG7GwNTPEoU87vtGLcebI5LgQkYB9V6Jx/HYsMnP+C6GuliaISkyHQN5TXSPbuSj+3yvJxYgEbLsUVWZPgnG5CDUw7GheYRef48eP4+OPP8aDBw+go6ODDz74ALdv3y724vOqpKQkuLq6YuXKlZgwYUKh+5w+fRrdunVDREQEateurZHzyXcnJhlD1wUgOTMXnevZYMPoVjDQK7qD9M+QJ5j1vzk3/m90K3RvWPUW6tOkmbtDsfdKNDrUscL2D9sBAB49eoRatWph79696Nevn9L+MrnAZ1v/wZ9XY6BjVA3P14/FgnmzMWfOnGLrSUtLwx8B9/D9hXjc37UYOrmZ2LZnH/o1qwkgr6diyJAhuH//Pk6fPg0rK6sCx5DJBfZdjcbK42F4Ks2bo6aeXTXM7emBrh62kEheP/S87mPvKSkpWLBgAbbv+gPPE55Bt5olTBt2xi/ffYsR7esAyLuIf/XVV1i/fj2SkpLQsWNH/Prrr6hX77+w7ubmhrFjx2LhwoWKbYcOHYKvry/Cw8Ph7u6OmTNnYuLEiYr3Y2Nj4evri+PHjyMxMRGurq6YNGkSZsyYUarfRb5fzkQo5tL57v0mpbqddDNairH+l5GQmg1nS2NsGd8W7tamSvusPReJpUfuQl9Xgp2T2qGlq2Wp2l+ZpGbl4tjNWOwPjcaF8ARoMiRoekoOhh01MOxoVnEXHwBISEiAnp4eqlevDnt7e8yaNQuzZ89W+fitW7dG9+7dFeMMXpWWloZq1arh6NGj6NGjx2ufx6uinqdj0Np/8SwlCy1da2DbhLYwNih5wb2v/rqJzQGPYGaoh7+mdCiXOWDU9SasLxUWm4KeP52HEMBfPh3Q1Lk6gLwn9tatW4fHjx9DT0+vQDkhBFadjsDi/9uD+J1fYPyP+7BuSt8iHwtOzcrFwgO38EfIE8gyUxG3/kN8s9gP82bk9WLkB53w8HCcOXMGNjY2xbY7M0eGrQGPsPpMhGL8Vht3S8zr5QEHC6MCv3chBJIzcvEsNQsJ+a+ULCSkZiMhNQtPXqTjQsTzAvVUN9YvtIfxVXK5QFIh48gGtagJVytTWFczhHU1A1ibGcKmmiGsqxkW+JyX9vOi6c/b9sBH+Hxf3vi5L/o0wIdv1Sr1MR89T8PojZfx6Hk6LE0N4D+2teIz90/4M4zZeBlyASzq74lR7VxLXV9ldfj6U/j8frXA9np21WBuVPwSGMmZObgXl1pg+46J7eBVu+AfCK9D1et3wW+GSkQmk2HhwoXYtm0bYmNj4ejoiLFjx+KLL75Q/CWQ/1fIhg0bkJSUhA4dOigegaSK4e/vD1tbW/Tp06fQ9/MHGJ8+fRrx8fF47733VD52amoqIiMjMWrUqCL3CQ0NBZA3aFJT4pMzMfK3QDxLyYKHvRk2jmmtUtABgC/ebYjbMckIevgCH20NwT6fDqhmWHn+13tTJsf7/ngYhAB6NrJXXHTkcjn8/f0xZsyYAkHH398fDRo0gI2NDSxjApH+93cwb90Pp2L0MGlrCH4Z3gLv9vJWuh2zZtufWHM2As/1rCFPioFO0HY0b9wIs6Z8BCAv6Lz//vu4cuUKDh06BJlMpngk29LSEgYGBgXabaSvi4mdamFIK2esORcJ/4sPcPlBIgb++q/SfjWrG0MuBJ6nZiP7NZ6KKSzAqOPPK9FFvmdqoAtrs7zgk5kjw62neatsSwC8Xd8GDR1V/yPx9tNknA17prHbGoeuP8UX+/OCjk+X2hoJOgDgamWKPz9uj3H+QbgRLcWw9ZeweIAndCUSLPjrJuQCGNzSCSPbVr7/VzSphWuNvIH4L3WL6Eok2Dy+TYlBNUaagQ5LTxcoWxFTclTqnp0lS5Zg5cqV2Lx5Mxo1aoTg4GCMGzcOixcvxtSpUwHk3V/28/NTur9848YNpfvLJWHPjubI5XK4u7vjgw8+wNKlS5Xee/niExAQgGnTpmHs2LFYsWKFYp9XxwJ89tln6Nu3L1xdXfH06VN89dVXCA0Nxe3bt2FjY4PIyEj8/vvv6N27N6ysrHD9+nXMmDEDTk5OOHfunEbOSZqeg6HrA3A3NgUulib4Y7KX2vfm41My0XfVBcQlZ6GXpz1+HdFCI133pVXYl5GOBLg4r2ul6uG5EvUCA3/9FzoS4PiMTqhjmzde6/jx4+jRowfCwsKUbrMAeXPKbNq0CYmJiXBzc8PkyZPh2WM4Pt1xFVm5cjR3qY7LfsPRc8BQfPP1Qhy8HoOFP/4fnp/dBFnKc1ha1sCwIYOxePFiWFhYAAAePnwId3f3Qtt45swZvP322yWey9OkDCz5+zYOXS963hoAMDPSU/SsWJsZ/K/HxRD6ujpYfvSu0q0FHQmwbUJb2JiVPLfLs5QsjPwtUOnfXCIBxrRzQ2auDAmpWXiWmo2ElCw8S81Cdm7ZP448oq0Lenrao7WbJYz0VfsjAgDO3XuGDzcHIUcmMLytCxb399T4/1epWbn4eFsI/glXnkPJqYYxTs7srFZ731S7gqIwf+9NyISArkSCJQM9VQ6opSmrCq24jVVWU5u/imFHc9S9+Lx6v/7VsQDDhg3D+fPn8fz5c9jY2KBjx45YvHixYizO48ePMXLkSNy8eRNpaWlwdnbGgAED8MUXX2jk3zI9OxejfruMkEcvYGtmiD8mt4eL1ev9VRLy6AWGrQ9AjkxgTs/6+OTtOqVuX2n9G5mA4RsCC2zfOr4N3qpX/O2Z8iKEwAcbLuHS/UQMbumE7wY3LdXxgh8mYsLmoqcE6OVpj6UDm5TZKtVA0b/3Rf090dXDFlamBsVeREt7AVG1vBACKVm5ittoFyMS8NOpgouC9vS0UykcxyRl4OitoicSNdTTQWs3S3Ssa4236lqjgb15kbfmQh69wMj/C0RGjgzvNnHAT8Oaa2zw96uinqeh03dnlbZVxj8KylKMNAMPE9LhZm2i9jmXpmxJtCLsLFmyBOvXr8fx48dRr149XLt2Dd7e3li5ciVGjBiB+/fvo3bt2rh69SqaNWumKNe5c2c0a9YMP/30U6HHzcrKQlZWluLn5ORkODs7M+xUEpVl/Eh2rhwTtwTj3L1nMDfSw+7JXvCwL93nI39sgY4E2DSuDTpVcKC4/jgJ7/1yscD2t+pa47cxrYsdfF1ezt97htEbL8NAVwdnZr+NmoVMDqeufyMSMPz/CoYN354emNS5Vpn3uhXVva/OwM3SXkBep3xp211YeYkkL2BeeZSE2GTlxUatTA3QoY61Ivw4WOStbXX+XgK+PXwbKZm56FTPBv9XwoMCpVVUONXk2BN6PVoxZmfevHlITk6Gh4cHdHV1IZPJsHjx4lJPbe7n54evv/667BpOr62yjB+R/W/6+HP3nsFYXxf+49qUOugAwPA2Lrj+WIpdwY/x6Y6rOPRpRzhbVtySElsvPVL6WUcCSCQS/BOegE+2h+CXES1gqFdx3fRCCMUTNiPbuWok6ADIG2xSiCbO1cvl9mL+RG+v9q6oE1ocLIxL9cfA65QvbbuLKj+0tQuEEIh8lop/whPwT3gCLt1/judp2Thw7SkOXHsKALCpZoCE1GzFLTwXS2OsHdmizEN5/izElWHsCb2eSh12Xp7avFGjRggNDcX06dPh6OiIMWPGvPZxfX19MXPmTMXP+T07VLFipBmY9+cNpQXr5u+9iU71bMq1h0cIga8O3MTBa0+hryvB2lEt0dK1hkaOLZFI8HW/Rrgbm4xrT6SYtDUEez9ur/JgZ026/TQZf1zJW/xw/eiWMDPUh5u1CcLjUjFxSzBO3onHx9uu4NcRLSpsXMKRm7G4ES2FqYEufLpobhqBynDxGtraBZ3q2ZRZ935ZKW27iyovkUhQx9YMdWzNMK6DO7Jz5bga9QIXIvLCz7XHSXiWmq10rCcvMiDNyIGJQdleyjQRTqliVeqw8/LU5gDQuHFjPHr0CH5+fhgzZozS1OYvP3kTFxendFvrVYaGhkoz+VLlsP3SowLzOeSvHF5eXyox0gwsP3oX+64+hUQCrBzSDJ01fKvJSF8Xa0a2RN9VF3AnJhm+e6/jh6HNynXAcv4EbEIAfZo4wLvhf5OvOVgY47cxrfHhliCcvhuPydtCsHZky3IPPLkyOb4/nter8+FbtTS6sGJluXiVtnemopRHr5KBng7a1rJC21pWmOVdHyduxWHi1mClfeQC5fb98KaGU8pT8Tfki1FWU5tT5bP/ajR+ORNZYLsE5bdy+K6gKLT3O419V/O6zPs3c0Tfpo5lUpdjdWP8MqIFdHUk2B/6FP4XH5ZJPUU5d+8ZLkQkwEBXB3N7eBR4v2Nda2wc0xpG+jo4G/YMk7aGlPuSB39eeYL7z9JQw0QfH75V+BNQpfHqFPqV8XF7+o+nk3mBBS3Luzcuf8V0Bp03T6UOO3379sXixYtx+PBhPHz4EPv27cPKlSsxYMAAAHndntOnT8e3336LAwcO4MaNGxg9ejQcHR3Rv3//im38G6481zLZe+UJZu4OhQDQ2rUGdF/6QhPIu91S1l69hQYAB0JjyvT829WywvzeeYsPLv77Dg5ff1ouv/NcmRxL/r4DABjT3rXIp8va17GG/9g2MNbXxfl7zzBxS3C5BZ7MHBl+PJn31I9PlzowK2HystfFi9ebo7AFLXkriVRVqW9jrVq1CgsWLMAnn3yC+Ph4ODo64qOPPsKXX36p2GfOnDlIS0vDpEmTFFObHz16VOU5dqig8hwk/EfIE8z+4xqEAD5o44zF/RsjLiUTDxPSsf9qNHYFP8asPddwZNpbZfqldvJ2XIXcQhvfwQ03niRhf+h/s5SW9e98T8gT3ItLhYWxPqZ0KX7yTa/aVvAf1xrjNwXhn/AETNgchP8brfqEiq9r26VHiJFmwsHCCCO1eHZaUg9vJdHrqtSPnpcXzrPzH008Equq3cGPMffP6xAib1KxRf08lebUyMqV4f01AbgRLUVrtxrYMbFdkVP8l8aTF+nou+oiXqQrD34sq/N+1YNnaeiy4my51J2WlYvO351FQmoWFrzbEBM6qnZ76PKDRIz1v4z0bBm8alnht7GtymxQaEpmDjotP4MX6TlYNqhyzuZMRJWDqtfvSn0bi8rfg4Q0paAD/NfDoUm7gqIUQWdUO1d829+zwORhhnq6WD28OaoZ6iHo4YtCJzMrreTMHEzYFIwX6dlwsDBSjAkozy7ymOSCt63K4ncOAOvO30dCahZcrUzUWs+njbsltoxvA1MDXQTcf47xm4KQnp2r8fYBwIZ/HuBFeg5q2ZhiUAunMqmDiKoWhh1S8urKvvl2XH4EaXrp1t7J93tgFOb+eQNCAGO8XPFNv0ZFPonkamUKv4GNAQCrz0TgwitTtpdGrkyOKb9fRVhcCmzNDPHnx+1xcV7Xch+wmv8Y9MvKYmB2rDQT68/nDQKf29ND7blJWrlZYsuENqhmqIdL9xMx1j8IaVmaDTzPU7Pw2z/3AQCfedcvk548Iqp6+E1CSmSvdOvkX4MPXItBt5VnceDaU5Tmzuf2wEeYv+8GAGBcBzcsfK/ooJOvb1NHfNDGBUIA03eF4llKVrH7qyJvLp1bOP+/SQN/G9MajtWNK2TA6qsDL4G8gdkXC1nZujRWnghDZo4cLV1roJenfckFCtHSNS/wmBnqKW5tpWow8PxyJhJp2TI0rmnx2m0kInoVww4p+TMkb+XjFi7VsWNiO/zr2xV7Jnuhjm01JKRmY+qOqxjrH4THierfYtka8BCf78tbnXhCR3d8+W5DleeW+apvQ9S3M0NCahZm7AqF/NV7bWr67cIDbA+MgkQC/DSsGRo7WZTqeKX18mPQo/93e8l373UERGom8Nx+mow9IXkTCH7ep0Gp5vRp4VIDWz9sCzOjvNuLYzZeRnhcSqmfJHvyIh3b/jej85ye9SvFQqlEpB0YdkhBLhfYE/IYADDKy1XRw9HazRKHp3bEzHfqwUBXB+fuPcM7P5zD+vORyJWptiLy5n8fYsFftwAAE99yxxdqXnCN9PPG7xjr6+JCRALWnCs4J4+qjt+KxeL/PXr9ee8G8G5UOXoQ8nuVFr7XCH0aOyBHJjB5WwjuP0st1XFfnUCwhUvpZ4Nu5lwd2ybkBZ6QRy/wzg/nMXxDIDosPY1dQVGvdcyfToYjWyaHVy0rdKxjXeo2EhHlY9ghhUv3n+PJiwyYGeqhZyMHpfcM9XQxtVtdHJ3+FtrVskRmjhxL/r6L91ZfxLXHScUe1//iA3x1IC/ofNSpFub3fr2ehbp2Zvi6XyMAwMoT9xD0MFHtY9x4IsW0naGKJ8BUfRqpPOnoSLBiSFM0c64OaUYOxm8KQmJadskFi5A/gaC+rqTQCQRfV1Pn6vhpWDOlbXIBzPvzBpYdvYMzd+MRI81Q6bZneFwK/vzf0hXs1SEiTWPYIYXdwXm9Ou81cyxyHpVaNtWwY2I7LH+/Caqb6ON2TDIG/HoRXx+8VejYjf/75z6+PngbAPDx27Uxr5dHqS5kg1s6YUDzmpDJBabuuIoXaoSAGGkGJmwOQkaODG/VtVZpvFB5cHNzg0QiUXoZG+jBMnQrnGoYI2T7cri4ucPY2Bg2Njbo168f7t69W+wx9+7dC29vb1hZWaGLhx2y4+5jjJeb0gSC69evx9tvvw1zc3NIJBIkJSWp3fbClpAQANacvY9xm4Lg5Xcazb45gaHrArDwwC3sCorCtcdJyMj+b3LCGGkG5u/Lm9fJu6Edmmug54mI6GWVelJBKj/SjBwcuZm3UvyQVsUviiqRSDCklTO6etji20O3FcsdHLsZi2/6eaJRTXM8SEhDQMRzrDoTAQDw6VIbn3mX/i92iUSCRf09Efo4CQ8S0jD7j2vYMLpVicdNzcrF+E3BiE/JQj27avhlRAvoV5InfYKCgiCT/Xfxv3nzJt555x2MHjEMNRu0xNv/1IOs0dvo3a4xfNrb4euvv4a3tzcePHgAXd3CQ2laWho6duwIt9bdsGHJPFQz1MOUrnWU9klPT0fPnj3Rs2dP+Pr6vlbbC1tQUwKgewM7PEpMQ+SzNEgzchD4IBGBD/7riZNIAHcrU5ga6uJmdLJiQsdGNav2PFdEVDY4qSA4qSCQN2PtF/tvor6dGY5Of0utUHL+3jN8vv8GHicWPjh1atc6mPFOPY32otx6KsWAX/9Fdq68xMnxcmVyTNoagtN342FdzQD7fTrAqUb5raejrunTp+PQoUMIDw+HRCLBP+HPMNY/CDK5wIzu9dDFNgNNmzZFREQEatcueiXwtKxceH2+EzdWjMKXGw/h63F9Ct3v7Nmz6NKlC168eIHq1aur3d5dQVEFFtTMf2w/M0eGyGepuBOTgrsxybgbm4I7Mcl4XkSPXHlN5EhE2kHV6zd7dggAsOd/t7AGt3JSO5R0qmeD49M7Y/HfdxRP0+STAPigrYvGbxc1crTAF30a4Mu/bmHpkTto7VYDTZyqF7rvt4fv4PTdeBjq6WDD6FaVOuhkZ2dj27ZtmDlzpuJ39lZdGyzq54n5+25gxd/XcSr+KNzd3eHsXHwP3Lrz9xVjfd5t7FDsvqVR3BT+Rvq6aORogUaOyk+7PUvJwt4rT+B3RPl2XHmvck9EVUPl6MenCnU3NhnXnkihpyPBgOY1X+sYxga66N244FNNAiiTmYCBvJmXezayR45MYMrvV5GcWXDSw83/PsSmfx8CAH4Y2qzSjwfZv38/kpKSMHbsWKXtSSGHEPPTYDz+4X0cPXIUK/33wMDAoMjjvDyBIADoqzmBoLrUnZ/IxswQ7zVzrPBVrImoamDYIewJznsKpnsDO1hVM3zt4xQ2E3BZXrwkEgmWvd8ENasbIyoxHb57byg9+XP6bhy+Ppj3FNjcnh7oXYa9G5ry22+/oVevXnB0dFTaPmLECFwPDcW7vuugZ+mI4R8MQ1h00XPw5E8g2Mix8t6W5SrWRFReGHaquOxcOfZdzZtIcEjr0q1DVBEXLwtjfawa3hx6OhIcvh6DHZfzbsfdfpqMT3+/CrkAhrZyxuTOtcqsDZry6NEjnDx5Eh9++GGB9ywsLFC/fj3sWjgeXT5ZisyExxg498dCl/C4E/PfBIIlrWpe0V6eTLE8l+ggoqqFY3aquNN345CYlg1bM0N0qmtT6uMVN36jrLRwqYHZPerD78hdLDxwE8mZOdh44QHSsmVoX9sK3w7wrBSPmJfE398ftra26NOn8IHEAGBioIe1I1vAbS4Q9yIVk7eFYPP4NkrrXL08gWBFzwytCgcLY/bmEFGZYs9OFbf7f7ewBrV00tiiixWxvtTEt2qhvl01ZMsElh65i/iULNhUM8CaES0rzSPmxZHL5fD398eYMWOgp/ff3yD379+Hn58fQkJCEBUVhX///Rc+E0bBrJoJrDzaIuD+c3y+7wY8PDywb98+nLv3DP+EJ0AnOxXvOmbh9u28OY7CwsIQGhqK2NhYxbFjY2MRGhqKiIi86QFu3LiB0NBQJCaqP1kjEVFlVvmvAlRm4pIzcTYsHkDeZH1vsriUTITHKy+r8DwtG+k5ml2Vu6ycPHkSUVFRGD9+vNJ2IyMj/PPPP+jduzfq1KmDoUOHwszMDIEBAVg7sSt0JMCekCcICwvDixdJWHI4bxmMZvII9O7SXtFLNGzYMDRv3hxr165VHHvt2rVo3rw5Jk6cCADo1KkTmjdvjgMHDpTTWRMRlQ/Os4OqO8/Or2cjsPxoGFq71cCeye0rujml8m9kAoZvCCywfcfEdvCqbVUBLSofWwIe4sv/rTnW1cMWp+/Gw8xID//M6YLqJkU/rUVEpA1UvX6zZ6eKEkIonsIaXMKMyW+C8n4SrLIY7eWGcR3cAACn7+b10qVm5uLYrdhiShERVS0MO1VU8KMXeJCQBhMDXfR5Ax7JLklVfoz51dmjBYD5e28iRlr4jNZERFUNn8aqonYH5T2i/W4TB5gaasfHoCKeBKsMohILTtrImYiJiP6jHVc5UktqVi4O34gBUPKin2+aqvgYc2GLcVaFW3hERKribawq6O/rMUjPlqGWtSlaulbu5ROoZFX5Fh4RkSrYs1MF7VYs+un8Rky2RyWrqrfwiIhUwbBTxUTEpyL40Qvo6kgwqMXrLfpJlVNVvIVHRKQK3saqYvaE5PXqvF3PBrbmRhXcGiIiorLHsFOF5Mjk+DMkb9FPbZhbh4iISBUMO1XIubBnSEjNgpWpAbp62FZ0c4iIiMoFw04Vkj8weUDzmkqrZBMREWkzXvGqiGcpWYrlBHgLi4iIqhKGnSpi/9Vo5MoFmjpXR317s4puDhERUblh2KkChBCKW1hDWjlVcGuIiIjKF8NOFRD6OAnh8akw1NNB36aOFd0cIiKicsWwUwXsDn4CAOjd2AHmRvoV3BoiIqLyxbCj5TKyZTh47SkAYDBvYRERURXEsKPljtyMQWpWLlwsTdDO3aqim0NERFTuGHa0nGLRz5ZO0NHhop9ERFT1MOxosUfP03DpfiIkEmBQS97CIiKiqolhR4v9EZI3MPmtujZwrM7VsImIqGpi2NFST16kY3vgIwCcW4eIiKo2vYpuAGnerqAozNt7A0Lk/ZyUnlOxDSIiIqpA7NnRMjHSDPi+FHQA4Ku/biFGmlFxjSIiIqpADDtaZvulR5AL5W0yIfAwIb1iGkRERFTBeBtLS6Rk5uCrv25h79XoAu/pSiRwszapgFYRERFVPPbsaIHQx0no8/MF7L0aDR0J0L2BLfKn1NGVSLBkoCccLPg0FhERVU3s2XmDyeQCa89F4ocT95ArF6hZ3Rg/DWuGVm6WiJFm4GFCOtysTRh0iIioSmPYeUPFSjMxY1coAu4/BwC828QBiwc0hoVx3kKfDhbGDDlERERg2HkjHbsVi7l/XkdSeg5MDHTx9XuN8H5LJ0gkXA6CiIjoVQw7b5CMbBm+PXwb2wOjAACNa1rgp2HNUMumWgW3jIiIqPJi2HlD3IlJxtQdVxEenwoA+KhTLczyrg8DPY4xJyIiKg7DTiUWI83Ag2dpCHqYiF/ORiI7Vw4bM0OsHNIUb9W1qejmERERvREYdiqpXUFR8N17Q2mCwG4etlj+fhNYVTOsuIYRERG9YXgPpBLKX/Lh5aAjkQCL+jdi0CEiIlITw04l9CAhrcCSD0IAj55zfSsiIiJ1MexUQu7WpgW2cckHIiKi18OwUwk5WBjD0tRA8TOXfCAiInp9HKBcCcWnZCIxLRsA8NuYVmjoaM6gQ0RE9JoYdiqhK4+SAAAe9mbo1sCuYhtDRET0huNtrEroStQLAEAL1xoV3BIiIqI3n9ph58yZM2XRDnpJyKO8sNPShWGHiIiotNQOOz179kTt2rXx7bff4vHjx2XRpiotK1eGG0+kAICW7NkhIiIqNbXDTnR0NKZMmYI//vgDtWrVQo8ePbB7925kZ2eXRfuqnJvRyciWyWFlagBXKz5qTkREVFpqhx1ra2vMmDEDoaGhCAwMRL169fDJJ5/A0dERU6dOxbVr18qinVXGlUf/jdeRSCQV3BoiIqI3X6kGKLdo0QK+vr6YMmUKUlNTsXHjRrRs2RJvvfUWbt26pak2VimK8Tq8hUVERKQRrxV2cnJy8Mcff6B3795wdXXFsWPHsHr1asTFxSEiIgKurq4YPHiwRhoYHR2NkSNHwsrKCsbGxmjcuDGCg4MV7wsh8OWXX8LBwQHGxsbo3r07wsPDNVJ3eRNCICSKYYeIiEiT1A47n376KRwcHPDRRx+hXr16uHr1KgICAvDhhx/C1NQUbm5u+P7773H37t1SN+7Fixfo0KED9PX1ceTIEdy+fRsrVqxAjRr/BYHly5fj559/xtq1axEYGAhTU1P06NEDmZmZpa6/vD15kYFnKVnQ15WgcU2Lim4OERGRVlB7UsHbt29j1apVGDhwIAwNC1+B29raWiOPqC9btgzOzs7w9/dXbHN3d1f8txACP/74I7744gv069cPALBlyxbY2dlh//79GDZsWKnbUJ7yb2E1crSAkb5uBbeGiIhIO6jds3Pq1Cl88MEHRQYdANDT00Pnzp1L1TAAOHDgAFq1aoXBgwfD1tYWzZs3x4YNGxTvP3jwALGxsejevbtim4WFBdq2bYuAgIAij5uVlYXk5GSlV2XA8TpERESap3bY8fPzw8aNGwts37hxI5YtW6aRRuW7f/8+1qxZg7p16+LYsWP4+OOPMXXqVGzevBkAEBsbCwCws1NeUsHOzk7xXmH8/PxgYWGheDk7O2u03a+LYYeIiEjz1A4769atg4eHR4HtjRo1wtq1azXSqHxyuRwtWrTAkiVL0Lx5c0yaNAkTJ04sdT2+vr6QSqWKV2WYHDE1Kxd3Y/N6mBh2iIiINEftsBMbGwsHB4cC221sbBATE6ORRuVzcHBAw4YNlbY1aNAAUVFRAAB7e3sAQFxcnNI+cXFxivcKY2hoCHNzc6VXRbv2OAlyAdSsbgw7c6OKbg4REZHWUDvsODs74+LFiwW2X7x4EY6OjhppVL4OHTogLCxMadu9e/fg6uoKIG+wsr29PU6dOqV4Pzk5GYGBgfDy8tJoW8oab2ERERGVDbWfxpo4cSKmT5+OnJwcdO3aFUDeoOU5c+Zg1qxZGm3cjBkz0L59eyxZsgRDhgzB5cuXsX79eqxfvx4AIJFIMH36dHz77beoW7cu3N3dsWDBAjg6OqJ///4abUtZY9ghIiIqG2qHndmzZ+P58+f45JNPFOthGRkZYe7cufD19dVo41q3bo19+/bB19cX33zzDdzd3fHjjz9ixIgRin3mzJmDtLQ0TJo0CUlJSejYsSOOHj0KI6M351aQXC5whZMJEhERlQmJEEK8TsHU1FTcuXMHxsbGqFu3brGPold2ycnJsLCwgFQqrZDxO/fiUuD9w3kY6+vixkJv6OmWahUPIiKiKkHV67faPTv5qlWrhtatW79ucXpJ/i2sZs7VGXSIiIg07LXCTnBwMHbv3o2oqCjFrax8e/fu1UjDqhKO1yEiIio7ancj7Ny5E+3bt8edO3ewb98+5OTk4NatWzh9+jQsLLie0+u4wrBDRERUZtQOO0uWLMEPP/yAgwcPwsDAAD/99BPu3r2LIUOGwMXFpSzaqNUS07JxPyENANDcpXrFNoaIiEgLqR12IiMj0adPHwCAgYEB0tLSIJFIMGPGDMUj4aS6/F6dOrbVUN3EoIJbQ0REpH3UDjs1atRASkoKAKBmzZq4efMmACApKQnp6emabV0VEJL/yLkLb2ERERGVBbUHKHfq1AknTpxA48aNMXjwYEybNg2nT5/GiRMn0K1bt7Joo1bj4GQiIqKypXbYWb16NTIzMwEAn3/+OfT19fHvv/9i0KBB+OKLLzTeQG2WI5Pj2uMkAEALhh0iIqIyoVbYyc3NxaFDh9CjRw8AgI6ODubNm1cmDasKbj9NRlauHNVN9FHL2rSim0NERKSV1Bqzo6enh8mTJyt6dqh08m9htXCpAR0dSQW3hoiISDupPUC5TZs2CA0NLYOmVD0hXA+LiIiozKk9ZueTTz7BzJkz8fjxY7Rs2RKmpsq3X5o0aaKxxmm7Ky/17BAREVHZUDvsDBs2DAAwdepUxTaJRAIhBCQSCWQymeZap8WeJmUgRpoJXR0Jmjpz5mkiIqKyonbYefDgQVm0o8rJH6/T0MEcJgavvR4rERERlUDtq6yrq2tZtKPK4fw6RERE5UPtsLNly5Zi3x89evRrN6YqYdghIiIqH2qHnWnTpin9nJOTg/T0dBgYGMDExIRhRwXp2bm4HZMMgGGHiIiorKn96PmLFy+UXqmpqQgLC0PHjh2xY8eOsmij1rn2WAqZXMDBwgiO1Y0rujlERERaTe2wU5i6deti6dKlBXp9qHBX/je/DpeIICIiKnsaCTtA3uzKT58+1dThtJpivA7n1yEiIipzao/ZOXDggNLPQgjExMRg9erV6NChg8Yapq3kcqHo2eF4HSIiorKndtjp37+/0s8SiQQ2Njbo2rUrVqxYoal2aa37CWlISs+Bkb4OGjqaV3RziIiItJ7aYUcul5dFO6qM/CUimjhVh76uxu4iEhERURF4tS1nnF+HiIiofKkddgYNGoRly5YV2L58+XIMHjxYI43SZoqVzjk4mYiIqFyoHXbOnz+P3r17F9jeq1cvnD9/XiON0lZJ6dmIiE8FwMfOiYiIyovaYSc1NRUGBgYFtuvr6yM5OVkjjdJWV6OSAAC1rE1haVrwd0hERESap3bYady4MXbt2lVg+86dO9GwYUONNEpb5Y/XYa8OERFR+VH7aawFCxZg4MCBiIyMRNeuXQEAp06dwo4dO7Bnzx6NN1CbcHAyERFR+VM77PTt2xf79+/HkiVL8Mcff8DY2BhNmjTByZMn0blz57Joo1bIlckR+jgJAMMOERFReVI77ABAnz590KdPH023RavdjU1BRo4MZkZ6qGNTraKbQ0REVGWoPWYnKCgIgYGBBbYHBgYiODhYI43SRorxOi41oKMjqeDWEBERVR1qhx0fHx88fvy4wPbo6Gj4+PhopFHaiON1iIiIKobaYef27dto0aJFge3NmzfH7du3NdIobcSwQ0REVDHUDjuGhoaIi4srsD0mJgZ6eq81BKjSW7p0KSQSCaZPn67Y9vbbb0MikSi9Jk+eXGj5WGkmopMyoCMBmjpXBwBMnjwZEokEP/74o9K+bm5uBY67dOnSMjozIiIi7ad2OvH29oavry/++usvWFhYAACSkpIwf/58vPPOOxpvYEULCgrCunXr0KRJkwLvTZw4Ed98843iZxMTk0KPceV/S0R42JujmqEe9u3bh0uXLsHR0bHQ/b/55htMnDhR8bOZmVlpToGIiKhKUzvsfP/99+jUqRNcXV3RvHlzAEBoaCjs7OywdetWjTewIqWmpmLEiBHYsGEDvv322wLvm5iYwN7evsTjvHwLKzo6Gp9++imOHTtW5BNtZmZmKh2XiIiISqb2bayaNWvi+vXrWL58ORo2bIiWLVvip59+wo0bN+Ds7FwWbawwPj4+6NOnD7p3717o+9u3b4e1tTU8PT3h6+uL9PT0QvfLDzvNnS0watQozJ49G40aNSqy3qVLl8LKygrNmzfHd999h9zc3NKfDBERURX1WoNsTE1NMWnSJE23pVLZuXMnrly5gqCgoELfHz58OFxdXeHo6Ijr169j7ty5CAsLw969e5X2y8yR4dZTKQAgcP9G6OnpYerUqUXWO3XqVLRo0QKWlpb4999/4evri5iYGKxcuVJzJ0dERFSFvPaI4tu3byMqKgrZ2dlK2997771SN6qiPX78GNOmTcOJEydgZGRU6D4vh73GjRvDwcEB3bp1Q2RkJGrXrq1470a0FDkyAZPkR9i8dw2uXLkCiaToeXZmzpyp+O8mTZrAwMAAH330Efz8/GBoaKiBsyMiIqpa1A479+/fx4ABA3Djxg1IJBIIIQBAcQGXyWSabWEFCAkJQXx8vNIj9jKZDOfPn8fq1auRlZUFXV1dpTJt27YFAERERCiFnfxbWNWlkbgbHw8XFxelY86aNQs//vgjHj58WGhb2rZti9zcXDx8+BD169fX1CkSERFVGWqP2Zk2bRrc3d0RHx8PExMT3Lp1C+fPn0erVq1w9uzZMmhi+evWrRtu3LiB0NBQxatVq1YYMWIEQkNDCwQdIG+QNgA4ODgobc8PO4OHDcf169eVjuno6IjZs2fj2LFjRbYlNDQUOjo6sLW11dwJEhERVSFq9+wEBATg9OnTsLa2ho6ODnR0dNCxY0f4+flh6tSpuHr1alm0s1yZmZnB09NTaZupqSmsrKzg6emJyMhI/P777+jduzesrKxw/fp1zJgxA506dVJ6RN3DwwM5zYcCrm3QqUkteL4yoaC+vj7s7e0VPTYBAQEIDAxEly5dYGZmhoCAAMyYMQMjR45EjRqcjJCIiOh1qB12ZDKZYt4Xa2trPH36FPXr14erqyvCwsI03sDKyMDAACdPnsSPP/6ItLQ0ODs7Y9CgQfjiiy+U9gsLC4NV7WRY6urAs6Z5icc1NDTEzp07sXDhQmRlZcHd3R0zZsxQGsdDRERE6lE77Hh6euLatWtwd3dH27ZtsXz5chgYGGD9+vWoVatWWbSxUnj5Fp2zszPOnTtXYpn/Ox+JRYfvoL69GQz1Ct76enWcTosWLXDp0qXSNpWIiIheovaYnS+++AJyuRxA3ky/Dx48wFtvvYW///4bP//8s8Yb+KbaFRSFRYfvAABuRkuxKyiqgltERERUNUlE/uNUpZCYmIgaNWoU+0h1ZZacnAwLCwtIpVKYm5d8u6kkMdIMdFh6GvKXfrO6EgkuzOsCBwvjUh+fiIiIVL9+q92zUxhLS8s3NuiUhQcJaUpBBwBkQuBhQuEzLBMREVHZ0UjYIWXu1qbQeSX76UokcLMufKFQIiIiKjsMO2XAwcIYfgMbQ/d/vV26EgmWDPTkLSwiIqIK8NrLRVDxhrZ2Qad6NniYkA43axMGHSIiogqids/O+fPnC12FOzc3F+fPn9dIo7SFg4UxvGpbMegQERFVILXDTpcuXZCYmFhgu1QqRZcuXTTSKCIiIiJNUTvsCCEKffLq+fPnMDU11UijiIiIiDRF5TE7AwcOBJC3uvnYsWNhaGioeE8mk+H69eto37695ltIREREVAoqhx0LCwsAeT07ZmZmMDb+bxyKgYEB2rVrh4kTJ2q+hURERESloHLY8ff3BwC4ubnhs88+4y0rIiIieiOoPWZnzpw5SmN2Hj16hB9//BHHjx/XaMOIiIiINEHtsNOvXz9s2bIFAJCUlIQ2bdpgxYoV6NevH9asWaPxBhIRERGVhtph58qVK3jrrbcAAH/88Qfs7e3x6NEjbNmyhaueExERUaWjdthJT0+HmZkZAOD48eMYOHAgdHR00K5dOzx69EjjDSQiIiIqDbXDTp06dbB//348fvwYx44dg7e3NwAgPj6+2OXViYiIiCqC2mHnyy+/xGeffQY3Nze0adMGXl5eAPJ6eZo3b67xBhIRERGVhtph5/3330dUVBSCg4Nx7NgxxfZu3brhhx9+0GjjXrV06VJIJBJMnz5dsS0zMxM+Pj6wsrJCtWrVMGjQIMTFxZVpO4iIiOjNoXbYAQB7e3uYmZnhxIkTyMjIAAC0bt0aHh4eGm3cy4KCgrBu3To0adJEafuMGTNw8OBB7NmzB+fOncPTp08Vsz0TERERqR12nj9/jm7duqFevXro3bs3YmJiAAATJkzArFmzNN5AAEhNTcWIESOwYcMG1KhRQ7FdKpXit99+w8qVK9G1a1e0bNkS/v7++Pfff3Hp0qUyaQsRERG9WdQOOzNmzIC+vj6ioqJgYmKi2D506FAcPXpUo43L5+Pjgz59+qB79+5K20NCQpCTk6O03cPDAy4uLggICCjyeFlZWUhOTlZ6ERERkXZSebmIfMePH8exY8fg5OSktL1u3bpl8uj5zp07ceXKFQQFBRV4LzY2FgYGBqhevbrSdjs7O8TGxhZ5TD8/P3z99deabioRERFVQmr37KSlpSn16ORLTExUWgldEx4/foxp06Zh+/btMDIy0thxfX19IZVKFa/Hjx9r7NhERERUuagddt566y3FchEAIJFIIJfLsXz5cnTp0kWjjQsJCUF8fDxatGgBPT096Onp4dy5c/j555+hp6cHOzs7ZGdnIykpSalcXFwc7O3tizyuoaEhzM3NlV5ERESkndS+jbV8+XJ069YNwcHByM7Oxpw5c3Dr1i0kJibi4sWLGm1ct27dcOPGDaVt48aNg4eHB+bOnQtnZ2fo6+vj1KlTGDRoEAAgLCwMUVFRivl/iIiIqGpTO+x4enri3r17WL16NczMzJCamoqBAwfCx8cHDg4OGm2cmZkZPD09lbaZmprCyspKsX3ChAmYOXMmLC0tYW5ujk8//RReXl5o166dRttCREREbya1w05UVBScnZ3x+eefF/qei4uLRhqmqh9++AE6OjoYNGgQsrKy0KNHD/z666/l2gYiIiKqvCRCCKFOAV1dXcTExMDW1lZp+/Pnz2FrawuZTKbRBpaH5ORkWFhYQCqVcvwOERHRG0LV67faA5SFEJBIJAW2p6amavSJKSIiIiJNUPk21syZMwHkPX21YMECpcfPZTIZAgMD0axZM403kIiIiKg0VA47V69eBZDXs3Pjxg0YGBgo3jMwMEDTpk3x2Wefab6FRERERKWgctg5c+YMgLxHv3/66SeObSEiIqI3gtpPY/n7+5dFO4iIiIjKhNoDlImIiIjeJAw7REREpNUYdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUqddjx8/ND69atYWZmBltbW/Tv3x9hYWFK+2RmZsLHxwdWVlaoVq0aBg0ahLi4uApqMREREVU2lTrsnDt3Dj4+Prh06RJOnDiBnJwceHt7Iy0tTbHPjBkzcPDgQezZswfnzp3D06dPMXDgwApsNREREVUmEiGEqOhGqOrZs2ewtbXFuXPn0KlTJ0ilUtjY2OD333/H+++/DwC4e/cuGjRogICAALRr106l4yYnJ8PCwgJSqRTm5uZleQpERESkIapevyt1z86rpFIpAMDS0hIAEBISgpycHHTv3l2xj4eHB1xcXBAQEFDkcbKyspCcnKz0IiIiIu30xoQduVyO6dOno0OHDvD09AQAxMbGwsDAANWrV1fa187ODrGxsUUey8/PDxYWFoqXs7NzWTadiIiIKtAbE3Z8fHxw8+ZN7Ny5s9TH8vX1hVQqVbweP36sgRYSERFRZaRX0Q1QxZQpU3Do0CGcP38eTk5Oiu329vbIzs5GUlKSUu9OXFwc7O3tizyeoaEhDA0Ny7LJREREVElU6p4dIQSmTJmCffv24fTp03B3d1d6v2XLltDX18epU6cU28LCwhAVFQUvL6/ybi4RERFVQpW6Z8fHxwe///47/vrrL5iZmSnG4VhYWMDY2BgWFhaYMGECZs6cCUtLS5ibm+PTTz+Fl5eXyk9iERERkXar1I+eSySSQrf7+/tj7NixAPImFZw1axZ27NiBrKws9OjRA7/++muxt7FexUfPiYiI3jyqXr8rddgpLww7REREbx6tnGeHiIiISF0MO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLSa1oSdX375BW5ubjAyMkLbtm1x+fLlim4SERERVQJaEXZ27dqFmTNn4quvvsKVK1fQtGlT9OjRA/Hx8RXdNCIiIqpgWhF2Vq5ciYkTJ2LcuHFo2LAh1q5dCxMTE2zcuLGim0ZEREQVTK+iG1Ba2dnZCAkJga+vr2Kbjo4OunfvjoCAgELLZGVlISsrS/GzVCoFACQnJ5dtY4mIiEhj8q/bQohi93vjw05CQgJkMhns7OyUttvZ2eHu3buFlvHz88PXX39dYLuzs3OZtJGIiIjKTkpKCiwsLIp8/40PO6/D19cXM2fOVPwsl8uRmJgIKysrSCQSjdWTnJwMZ2dnPH78GObm5uVannWXf92lLc+6q1bdpS3Puln3m1K+tHUXRwiBlJQUODo6FrvfGx92rK2toauri7i4OKXtcXFxsLe3L7SMoaEhDA0NlbZVr169rJoIc3PzUv0Dl6Y86y7/uktbnnVXrbpLW551s+43pXxp6y5KcT06+d74AcoGBgZo2bIlTp06pdgml8tx6tQpeHl5VWDLiIiIqDJ443t2AGDmzJkYM2YMWrVqhTZt2uDHH39EWloaxo0bV9FNIyIiogqmFWFn6NChePbsGb788kvExsaiWbNmOHr0aIFBy+XN0NAQX331VYFbZuVRnnWXf92lLc+6q1bdpS3Puln3m1K+tHVrgkSU9LwWERER0RvsjR+zQ0RERFQchh0iIiLSagw7REREpNUYdoiIiEirMeyUoV9++QVubm4wMjJC27ZtcfnyZZXKnT9/Hn379oWjoyMkEgn279+vcp1+fn5o3bo1zMzMYGtri/79+yMsLEzl8mvWrEGTJk0Ukz95eXnhyJEjKpd/2dKlSyGRSDB9+nSV9l+4cCEkEonSy8PDQ+X6oqOjMXLkSFhZWcHY2BiNGzdGcHCwSmXd3NwK1C2RSODj41NiWZlMhgULFsDd3R3GxsaoXbs2Fi1aVOJaLS9LSUnB9OnT4erqCmNjY7Rv3x5BQUEF9ivpsyGEwJdffgkHBwcYGxuje/fuCA8PV7n83r174e3trZhNPDQ0VOX6c3JyMHfuXDRu3BimpqZwdHTE6NGj8fTpU5XqXrhwITw8PGBqaooaNWqge/fuCAwMVLntL5s8eTIkEgl+/PFHlcqOHTu2wL99z5491ar7zp07eO+992BhYQFTU1O0bt0aUVFRJZYt7HMnkUjw3XffqVR3amoqpkyZAicnJxgbGysWQ1albFxcHMaOHQtHR0eYmJigZ8+eis+LKt8lmZmZ8PHxgZWVFapVq4ZBgwYpJnhVpfz69evx9ttvw9zcHBKJBElJSYr3SiqfmJiITz/9FPXr14exsTFcXFwwdepUSKVSler+6KOPULt2bRgbG8PGxgb9+vVTLDGkzveoEAK9evVS/H5VKfv2228X+PeePHmyWnUHBASga9euMDU1hbm5OTp16oRvvvmm2LIPHz4s8vO2Z88eleqOjY3FqFGjYG9vD1NTU7Ro0QJ//vmnSmUjIyMxYMAA2NjYwNzcHEOGDCkwIXBZYdgpI7t27cLMmTPx1Vdf4cqVK2jatCl69OiB+Pj4EsumpaWhadOm+OWXX9Su99y5c/Dx8cGlS5dw4sQJ5OTkwNvbG2lpaSqVd3JywtKlSxESEoLg4GB07doV/fr1w61bt9RqR1BQENatW4cmTZqoVa5Ro0aIiYlRvC5cuKBSuRcvXqBDhw7Q19fHkSNHcPv2baxYsQI1atRQub0v13vixAkAwODBg0ssu2zZMqxZswarV6/GnTt3sGzZMixfvhyrVq1SqW4A+PDDD3HixAls3boVN27cgLe3N7p3747o6Gil/Ur6bCxfvhw///wz1q5di8DAQJiamqJHjx7IzMxUqXxaWho6duyIZcuWFfl+UeXT09Nx5coVLFiwAFeuXMHevXsRFhaG9957T6W669Wrh9WrV+PGjRu4cOEC3Nzc4O3tjWfPnqlUPt++fftw6dIlpenjVSnbs2dPpc/Ajh07VC4fGRmJjh07wsPDA2fPnsX169exYMECGBkZlVj25TpjYmKwceNGSCQSDBo0SKW6Z86ciaNHj2Lbtm24c+cOpk+fjilTpuDAgQPFlhVCoH///rh//z7++usvXL16Fa6urujevTvS0tJU+i6ZMWMGDh48iD179uDcuXN4+vQpBg4cCEC176L09HT07NkT8+fPL9C+kso/ffoUT58+xffff4+bN29i06ZNOHr0KCZMmKBS3S1btoS/vz/u3LmDY8eOQQgBb29vyGQytb5Hf/zxR6VlhlQtO3HiRKV/9+XLl6tcPiAgAD179oS3tzcuX76MoKAgTJkyBRcuXCi2rLOzc4HP29dff41q1aqhV69eKtU9evRohIWF4cCBA7hx4wYGDhyIIUOG4ODBg8WWTUtLg7e3NyQSCU6fPo2LFy8iOzsbffv2hVwuL/B71ThBZaJNmzbCx8dH8bNMJhOOjo7Cz89PreMAEPv27XvtdsTHxwsA4ty5c699jBo1aoj/+7//U3n/lJQUUbduXXHixAnRuXNnMW3aNJXKffXVV6Jp06av1ca5c+eKjh07vlbZwkybNk3Url1byOXyEvft06ePGD9+vNK2gQMHihEjRqhUV3p6utDV1RWHDh1S2t6iRQvx+eefF1nu1c+GXC4X9vb24rvvvlNsS0pKEoaGhmLHjh0lln/ZgwcPBABx9epVlesvzOXLlwUA8ejRI7XLSqVSAUCcPHlS5bqfPHkiatasKW7evClcXV3FDz/8oFLZMWPGiH79+hXbnuLKDx06VIwcOfK1yr6qX79+omvXriqXb9Sokfjmm2+UthX22Xm1bFhYmAAgbt68qdgmk8mEjY2N2LBhQ4G6X/0uSUpKEvr6+mLPnj2Kfe7cuSMAiICAgBLLv+zMmTMCgHjx4kWh511S+Xy7d+8WBgYGIicnR+2y165dEwBERESEynVfvXpV1KxZU8TExBT5b1tYWXW+Fwsr37ZtW/HFF1+8VtlXNWvWrMD3V3HlTU1NxZYtW5T2s7S0LPCZebXssWPHhI6OjpBKpYp9kpKShEQiESdOnCjxXEqLPTtlIDs7GyEhIejevbtim46ODrp3746AgIBybYtUKgUAWFpaql1WJpNh586dSEtLU2vpDR8fH/Tp00fp/FUVHh4OR0dH1KpVCyNGjEBUVJRK5Q4cOIBWrVph8ODBsLW1RfPmzbFhwwa16wfy/v22bduG8ePHq7QwbPv27XHq1Cncu3cPAHDt2jVcuHABvXr1Uqm+3NxcyGQyGBkZKW03NjZWuWcLAB48eIDY2Fil37uFhQXatm1b7p+7fFKpFBKJRO2157Kzs7F+/XpYWFigadOmKpWRy+UYNWoUZs+ejUaNGqnd1rNnz8LW1hb169fHxx9/jOfPn6tc7+HDh1GvXj306NEDtra2aNu2rVq3n/PFxcXh8OHDmDBhgspl2rdvjwMHDiA6OhpCCJw5cwb37t2Dt7d3seWysrIAQOlzp6OjA0NDw0I/d69+l4SEhCAnJ0fp8+bh4QEXF5dCP2+l+S5StbxUKoW5uTn09PQKbC+ubFpaGvz9/eHu7g5nZ2eV6k5PT8fw4cPxyy+/FLkOY3F1b9++HdbW1vD09ISvry/S09NVKh8fH4/AwEDY2tqiffv2sLOzQ+fOnVX6N3tVSEgIQkNDi/y8FVa+ffv22LVrFxITEyGXy7Fz505kZmbi7bffLrZsVlYWJBKJ0sSCRkZG0NHRUet77rWVeZyqgqKjowUA8e+//yptnz17tmjTpo1ax0IpenZkMpno06eP6NChg1rlrl+/LkxNTYWurq6wsLAQhw8fVrnsjh07hKenp8jIyBBCqPcXzN9//y12794trl27Jo4ePSq8vLyEi4uLSE5OLrGsoaGhMDQ0FL6+vuLKlSti3bp1wsjISGzatEnltufbtWuX0NXVFdHR0SrtL5PJxNy5c4VEIhF6enpCIpGIJUuWqFWnl5eX6Ny5s4iOjha5ubli69atQkdHR9SrV6/IMq9+Ni5evCgAiKdPnyrtN3jwYDFkyJASy79MEz07GRkZokWLFmL48OEqlz148KAwNTUVEolEODo6isuXL6tc95IlS8Q777yj6I1Tp2dnx44d4q+//hLXr18X+/btEw0aNBCtW7cWubm5JZbP/6vexMRErFy5Uly9elX4+fkJiUQizp49q9J551u2bJmoUaOG4v8fVdqemZkpRo8eLQAIPT09YWBgIDZv3lxi2ezsbOHi4iIGDx4sEhMTRVZWlli6dKkAILy9vZXKFvZdsn37dmFgYFCgntatW4s5c+aUWP5lJfXsqPJd9uzZM+Hi4iLmz5+vctlffvlFmJqaCgCifv36hfbqFFV+0qRJYsKECYqfC/u3KarsunXrxNGjR8X169fFtm3bRM2aNcWAAQNUqjsgIEAAEJaWlmLjxo3iypUrYvr06cLAwEDcu3dPpfPO9/HHH4sGDRoU+l5R5V+8eCG8vb0Vnzdzc3Nx7NixEsvGx8cLc3NzMW3aNJGWliZSU1PFlClTBAAxadKkItuoKQw7ZaCyhJ3JkycLV1dX8fjxY7XKZWVlifDwcBEcHCzmzZsnrK2txa1bt0osFxUVJWxtbcW1a9cU29QJO6968eKFMDc3V+kWmr6+vvDy8lLa9umnn4p27dqpXa+3t7d49913Vd5/x44dwsnJSezYsUNcv35dbNmyRVhaWqoVtCIiIkSnTp0EAKGrqytat24tRowYITw8PIosU5nDTnZ2tujbt69o3ry5Urd1SWVTU1NFeHi4CAgIEOPHjxdubm4iLi6uxPLBwcHCzs5OKaCqE3ZeFRkZqfIttPz/3z/44AOl/fr27SuGDRumVt3169cXU6ZMKfL9wsp/9913ol69euLAgQPi2rVrYtWqVaJatWoFbg0UVjY4OFg0bdpU8bnr0aOH6NWrl+jZs6fSfoV9l6gTdkr6Liop7JRUXiqVijZt2oiePXuK7OxslcsmJSWJe/fuiXPnzom+ffuKFi1aFAiahZX/66+/RJ06dURKSopiW2G/X1W/g0+dOlXoLbTCyuf/f+7r66u0b+PGjcW8efNUrjs9PV1YWFiI77//vtD3iyo/ZcoU0aZNG3Hy5EkRGhoqFi5cKCwsLMT169dLLHvs2DFRq1YtIZFIhK6urhg5cqRo0aKFmDx5cjG/Hc1g2CkDWVlZQldXt8AHf/To0eK9995T61ivG3Z8fHyEk5OTuH//vtplX9WtWzeVkve+ffsUX5r5LwCKD3ZhfyWXpFWrVkr/AxfFxcVF6a8sIYT49ddfhaOjo1r1PXz4UOjo6Ij9+/erXMbJyUmsXr1aaduiRYtE/fr11apbiLyLfX5YGTJkiOjdu3eR+7762ci/QL8aUDp16iSmTp1aYvmXlSbsZGdni/79+4smTZqIhIQEtcq+qk6dOoX2kr1a/ocfflB8zl7+7Ono6AhXV9fXqtva2lqsXbu2xLqzsrKEnp6eWLRokdJ+c+bMEe3bt1e57vPnzwsAIjQ0tMg2vVo+PT1d6OvrFxjvNWHCBNGjRw+V605KShLx8fFCiLzxhp988onivaK+S/Iv0K8GFBcXF7Fy5coSy7+suLBTUvnk5GTh5eUlunXrViCoqPM9mJWVJUxMTMTvv/9eYvlp06YV+Xnr3Lmz2nWnpqYKAOLo0aMl1n3//n0BQGzdulVp+5AhQxS9qKrUvWXLFqGvr6/4d39ZUeUjIiIKjPMSIu8a8dFHH6lc97NnzxT/1nZ2dmL58uVF7qspHLNTBgwMDNCyZUucOnVKsU0ul+PUqVNqjX15HUIITJkyBfv27cPp06fh7u5e6mPK5XLF/f3idOvWDTdu3EBoaKji1apVK4wYMQKhoaHQ1dVVq97U1FRERkbCwcGhxH07dOhQ4DHHe/fuwdXVVa06/f39YWtriz59+qhcJj09HTo6yv8r6erqvtYTBqampnBwcMCLFy9w7Ngx9OvXT+Wy7u7usLe3V/rcJScnIzAwsMw/d/lycnIwZMgQhIeH4+TJk7CysirV8VT97I0aNQrXr19X+uw5Ojpi9uzZOHbsmNr1PnnyBM+fP1fps2dgYIDWrVuX+vP322+/oWXLliqPUQLyft85OTml/vxZWFjAxsYG4eHhCA4ORr9+/Ur8LmnZsiX09fWVPm9hYWGIioqCl5dXqb+LVCmfnJwMb29vGBgY4MCBA4rxR69Tt8j74x9ZWVkllp83b16BzxsA/PDDD9i4caPadeeXd3BwKLFuNzc3ODo6Fvp5c3FxUbnu3377De+99x5sbGyUfgfFlc8fV1TY500mk6lct7W1NapXr47Tp08jPj5e8cRmmSrzOFVF7dy5UxgaGopNmzaJ27dvi0mTJonq1auL2NjYEsumpKSIq1eviqtXrwoAinEArz7RUpiPP/5YWFhYiLNnz4qYmBjFKz09XaV2z5s3T5w7d048ePBAXL9+XcybN09IJBJx/Phxlcq/Sp3bWLNmzRJnz54VDx48EBcvXhTdu3cX1tbWhf7l8arLly8LPT09sXjxYhEeHi62b98uTExMxLZt21Ruq0wmEy4uLmLu3LkqlxEi70memjVrikOHDokHDx6IvXv3Cmtr6wJd+cU5evSoOHLkiLh//744fvy4aNq0qWjbtm2BLvmSPhtLly4V1atXV4w/6devn3B3d1f8xVtS+efPn4urV6+Kw4cPCwBi586d4urVqyImJqbE8tnZ2eK9994TTk5OIjQ0VOnzl5WVVWzZ1NRU4evrKwICAsTDhw9FcHCwGDdunDA0NFT8Fanu/xcv38YqrmxKSor47LPPREBAgHjw4IE4efKkaNGihahbt67IzMxUqe69e/cKfX19sX79ehEeHi5WrVoldHV1xT///KNSu6VSqTAxMRFr1qwpcB4lle/cubNo1KiROHPmjLh//77w9/cXRkZG4tdffy2x7O7du8WZM2dEZGSk2L9/v3B1dRUDBw4UQqj2XTJ58mTh4uIiTp8+LYKDg4WXl5fidrIq5WNiYsTVq1fFhg0bBABx/vx5cfXqVfH8+fMSy0ulUtG2bVvRuHFjERERobTP5MmTiy0bGRkplixZIoKDg8WjR4/ExYsXRd++fYWlpaWIi4t7re9R/K/nrKSyERER4ptvvhHBwcHiwYMH4q+//hK1atUSnTp1Uvn39sMPPwhzc3OxZ88eER4eLr744gthZGQkhg8frlK7w8PDhUQiEUeOHFHaXlLd2dnZok6dOuKtt94SgYGBIiIiQnz//fdCIpGI3r17l1j3xo0bRUBAgIiIiBBbt24VlpaWYubMmUX+TjWJYacMrVq1Sri4uAgDAwPRpk0bcenSJZXK5XfpvvoaM2ZMiWULKwdA+Pv7q1T3+PHjhaurqzAwMBA2NjaiW7durx10hFAv7AwdOlQ4ODgIAwMDUbNmTTF06NBCBwwW5eDBg8LT01MYGhoKDw8PsX79erXaeuzYMQFAhIWFqVUuOTlZTJs2Tbi4uAgjIyNRq1Yt8fnnn4usrCyVj7Fr1y5Rq1YtYWBgIOzt7YWPj49ISkoqsF9Jnw25XC4WLFgg7OzshKGhoejWrZvS+ZRU3t/fv9D3v/rqqxLL59/6Kux15syZYstmZGSIAQMGCEdHR2FgYCAcHBzEe++9pzRAWd3/L14OO8WVTU9PF97e3sLGxkbo6+sLV1dXMXHiRKU/TFSp+7fffhN16tQRRkZGomnTpopboaqUXbdunTA2Nn6tf/OYmBgxduxY4ejoKIyMjET9+vXFihUrhFwuL7HsTz/9JJycnIS+vr5wcXERX3zxheJzq8p3SUZGhvjkk09EjRo1hImJiRgwYIAiGKtS/quvvipyn5LKF3Vuxb3yy0ZHR4tevXoJW1tboa+vL5ycnMTw4cPF3bt3VW77q/LDTkllo6KiRKdOnYSlpaUwNDQUderUEbNnz1aMbVO1bj8/P+Hk5CRMTEyEl5eX+Oeff1Qu6+vrK5ydnYVMJitwDiWVv3fvnhg4cKCwtbUVJiYmokmTJmLLli0qlZ07d66ws7MT+vr6om7duorPaXmQ/O8EiYiIiLQSx+wQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIXnH27FlIJBIkJSVVdFOISAMYdoiIiEirMewQERGRVmPYIaJKRy6Xw8/PD+7u7jA2NkbTpk3xxx9/APjvFtPhw4fRpEkTGBkZoV27drh586bSMf788080atQIhoaGcHNzw4oVK5Tez8rKwty5c+Hs7AxDQ0PUqVMHv/32m9I+ISEhaNWqFUxMTNC+ffsCK00T0ZuBYYeIKh0/Pz9s2bIFa9euxa1btzBjxgyMHDkS586dU+wze/ZsrFixAkFBQbCxsUHfvn2Rk5MDIC+kDBkyBMOGDcONGzewcOFCLFiwAJs2bVKUHz16NHbs2IGff/4Zd+7cwbp161CtWjWldnz++edYsWIFgoODoaenh/Hjx5fL+RORZnEhUCKqVLKysmBpaYmTJ0/Cy8tLsf3DDz9Eeno6Jk2ahC5dumDnzp0YOnQoACAxMRFOTk7YtGkThgwZghEjRuDZs2c4fvy4ovycOXNw+PBh3Lp1C/fu3UP9+vVx4sQJdO/evUAbzp49iy5duuDkyZPo1q0bAODvv/9Gnz59kJGRASMjozL+LRCRJrFnh4gqlYiICKSnp+Odd95BtWrVFK8tW7YgMjJSsd/LQcjS0hL169fHnTt3AAB37txBhw4dlI7boUMHhIeHQyaTITQ0FLq6uujcuXOxbWnSpInivx0cHAAA8fHxpT5HIipfehXdACKil6WmpgIADh8+jJo1ayq9Z2hoqBR4XpexsbFK++nr6yv+WyKRAMgbT0REbxb27BBRpdKwYUMYGhoiKioKderUUXo5Ozsr9rt06ZLiv1+8eIF79+6hQYMGAIAGDRrg4sWLSse9ePEi6tWrB11dXTRu3BhyuVxpDBARaS/27BBRpWJmZobPPvsMM2bMgFwuR8eOHSGVSnHx4kWYm5vD1dUVAPDNN9/AysoKdnZ2+Pzzz2FtbY3+/fsDAGbNmoXWrVtj0aJFGDp0KAICArB69Wr8+uuvAAA3NzeMGTMG48ePx88//4ymTZvi0aNHiI+Px5AhQyrq1ImojDDsEFGls2jRItjY2MDPzw/3799H9erV0aJFC8yfP19xG2np0qWYNm0awsPD0axZMxw8eBAGBgYAgBYtWmD37t348ssvsWjRIjg4OOCbb77B2LFjFXWsWbMG8+fPxyeffILnz5/DxcUF8+fPr4jTJaIyxqexiOiNkv+k1IsXL1C9evWKbg4RvQE4ZoeIiIi0GsMOERERaTXexiIiIiKtxp4dIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0mr/DyhqLZkmhFwSAAAAAElFTkSuQmCC", "text/plain": [ "
" ] @@ -1483,6 +1483,18 @@ " pass\n", "plt.show()" ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "with open('nonseq_exp_set_B_training_metrics.npy', 'wb') as f:\n", + " np.save(f, np.array(epochs_x))\n", + " np.save(f, np.array(epochs_y))\n", + " np.save(f, np.array(epochs_acc))" + ] } ], "metadata": { From d1f2dee9db674c29ed4fbfd7416ed72ed77767a3 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Mon, 29 Apr 2024 20:53:24 +0200 Subject: [PATCH 060/379] exp. set B1 --- .../exp_set_B1/baseline-SCNN-example_3.ipynb | 1071 ++++++++++++ .../non-sequential-SCNN-example_3.ipynb | 1509 +++++++++++++++++ 2 files changed, 2580 insertions(+) create mode 100644 tests/test_nonsequential/exp_set_B1/baseline-SCNN-example_3.ipynb create mode 100644 tests/test_nonsequential/exp_set_B1/non-sequential-SCNN-example_3.ipynb diff --git a/tests/test_nonsequential/exp_set_B1/baseline-SCNN-example_3.ipynb b/tests/test_nonsequential/exp_set_B1/baseline-SCNN-example_3.ipynb new file mode 100644 index 00000000..2eb99dfc --- /dev/null +++ b/tests/test_nonsequential/exp_set_B1/baseline-SCNN-example_3.ipynb @@ -0,0 +1,1071 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import torch, random\n", + "import torch.nn as nn\n", + "import sinabs.layers as sl\n", + "from tqdm.notebook import tqdm\n", + "\n", + "from tonic.datasets.dvsgesture import DVSGesture\n", + "from tonic.transforms import ToFrame\n", + "from torch.utils.data import DataLoader\n", + "from torch.nn import CrossEntropyLoss\n", + "from torch.optim import Adam\n", + "\n", + "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "torch.backends.cudnn.deterministic = True\n", + "random.seed(1)\n", + "torch.manual_seed(1)\n", + "torch.cuda.manual_seed(1)\n", + "np.random.seed(1)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Loading Data" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "batch_size = 3\n", + "num_workers = 1\n", + "epochs = 30\n", + "lr = 5e-4" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "root_dir = \"../DVSGESTURE\"\n", + "_ = DVSGesture(save_to=root_dir, train=True)\n", + "_ = DVSGesture(save_to=root_dir, train=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "n_time_steps = 50\n", + "to_raster = ToFrame(sensor_size=DVSGesture.sensor_size, n_time_bins=n_time_steps)\n", + "\n", + "snn_train_dataset = DVSGesture(save_to=root_dir, train=True, transform=to_raster)\n", + "snn_test_dataset = DVSGesture(save_to=root_dir, train=False, transform=to_raster)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The transformed array is in shape [Time-Step, Channel, Height, Width] --> (50, 2, 128, 128)\n" + ] + } + ], + "source": [ + "sample_data, label = snn_train_dataset[0]\n", + "print(f\"The transformed array is in shape [Time-Step, Channel, Height, Width] --> {sample_data.shape}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "snn_train_dataloader = DataLoader(snn_train_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True)\n", + "snn_test_dataloader = DataLoader(snn_test_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=False)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Network Module\n", + "\n", + "We need to define a `nn.Module` implementing the network we want the chip to reproduce." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "device: NVIDIA GeForce RTX 3070 Ti\n" + ] + } + ], + "source": [ + "if torch.cuda.is_available():\n", + " device = torch.device('cuda:0')\n", + " print('device: ', torch.cuda.get_device_name(0))\n", + "else:\n", + " device = torch.device('cpu')" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "class SNN(nn.Module):\n", + " def __init__(self) -> None:\n", + " super().__init__()\n", + "\n", + " self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False)\n", + " self.iaf1 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential(), spike_threshold=0.5)\n", + " self.pool1 = nn.AvgPool2d(2,2)\n", + "\n", + " self.conv2 = nn.Conv2d(10, 10, 2, 1, bias=False)\n", + " self.iaf2 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential(), spike_threshold=0.5)\n", + " self.pool2 = nn.AvgPool2d(3,3)\n", + "\n", + " self.conv3 = nn.Conv2d(10, 10, 3, 1, bias=False)\n", + " self.iaf3 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential(), spike_threshold=0.5)\n", + " self.pool3 = nn.AvgPool2d(2,2)\n", + "\n", + " self.flat = nn.Flatten()\n", + "\n", + " self.fc1 = nn.Linear(810, 100, bias=False)\n", + " self.iaf4 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential(), spike_threshold=0.5)\n", + "\n", + " self.fc2 = nn.Linear(100, 100, bias=False)\n", + " self.iaf5 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential(), spike_threshold=0.5)\n", + "\n", + " self.fc3 = nn.Linear(100, 100, bias=False)\n", + " self.iaf6 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential(), spike_threshold=0.5)\n", + "\n", + " self.fc4 = nn.Linear(100, 11, bias=False)\n", + " self.iaf7 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential(), spike_threshold=0.5)\n", + "\n", + " def detach_neuron_states(self):\n", + " for name, layer in self.named_modules():\n", + " if name != '':\n", + " if isinstance(layer, sl.StatefulLayer):\n", + " for name, buffer in layer.named_buffers():\n", + " buffer.detach_()\n", + "\n", + " def init_weights(self):\n", + " for name, layer in self.named_modules():\n", + " if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear):\n", + " nn.init.xavier_normal_(layer.weight.data)\n", + "\n", + " def forward(self, x):\n", + " \n", + " con1_out = self.conv1(x)\n", + " iaf1_out = self.iaf1(con1_out)\n", + " pool1_out = self.pool1(iaf1_out)\n", + "\n", + " conv2_out = self.conv2(pool1_out)\n", + " iaf2_out = self.iaf2(conv2_out)\n", + " pool2_out = self.pool2(iaf2_out)\n", + "\n", + " conv3_out = self.conv3(pool2_out)\n", + " iaf3_out = self.iaf3(conv3_out)\n", + " pool3_out = self.pool3(iaf3_out)\n", + "\n", + " flat_out = self.flat(pool3_out)\n", + " \n", + " fc1_out = self.fc1(flat_out)\n", + " iaf4_out = self.iaf4(fc1_out)\n", + "\n", + " fc2_out = self.fc2(iaf4_out)\n", + " iaf5_out = self.iaf5(fc2_out)\n", + "\n", + " fc3_out = self.fc3(iaf5_out)\n", + " iaf6_out = self.iaf6(fc3_out)\n", + "\n", + " fc4_out = self.fc4(iaf6_out)\n", + " iaf7_out = self.iaf7(fc4_out)\n", + "\n", + " return iaf7_out" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "snn = SNN().to(device)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "snn.init_weights()" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "optimizer = Adam(snn.parameters(), lr=lr, betas=(0.9, 0.999), eps=1e-8)\n", + "loss_fn = CrossEntropyLoss()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Define train and test" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "def train(dataloader, model, loss_fn, optimizer, epochs, test_func, dataloader_test):\n", + " epochs_y = []\n", + " epochs_x = []\n", + " epochs_acc = []\n", + " model.train()\n", + "\n", + " for e in range(epochs):\n", + " losses = []\n", + " batches = []\n", + " batch_count = 0\n", + " train_p_bar = tqdm(snn_train_dataloader)\n", + "\n", + " for X, y in train_p_bar:\n", + " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", + " X = X.reshape(-1, 2, 128, 128).to(dtype=torch.float, device=device)\n", + " y = y.to(dtype=torch.long, device=device)\n", + "\n", + " # forward\n", + " pred = model(X)\n", + "\n", + " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", + " pred = pred.reshape(batch_size, n_time_steps, -1)\n", + "\n", + " # accumulate all time-steps output for final prediction\n", + " pred = pred.sum(dim = 1)\n", + " loss = loss_fn(pred, y)\n", + "\n", + " # gradient update\n", + " optimizer.zero_grad()\n", + " loss.backward()\n", + " optimizer.step()\n", + "\n", + " # detach the neuron states and activations from current computation graph(necessary)\n", + " model.detach_neuron_states()\n", + "\n", + " train_p_bar.set_description(f\"Epoch {e} - BPTT Training Loss: {round(loss.item(), 4)}\")\n", + "\n", + " batch_count += 1\n", + " losses.append(loss.item())\n", + " batches.append(batch_count)\n", + "\n", + " epochs_y.append(losses)\n", + " epochs_x.append(batches)\n", + "\n", + " acc = test_func(dataloader_test, model)\n", + " print(f'Epoch {e} accuracy: {acc}')\n", + " epochs_acc.append(acc)\n", + "\n", + " return epochs_x, epochs_y, epochs_acc\n" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "def test(dataloader, model):\n", + " correct_predictions = []\n", + " with torch.no_grad():\n", + " test_p_bar = tqdm(dataloader)\n", + " for X, y in test_p_bar:\n", + " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", + " X = X.reshape(-1, 2, 128, 128).to(dtype=torch.float, device=device)\n", + " y = y.to(dtype=torch.long, device=device)\n", + "\n", + " # forward\n", + " output = model(X)\n", + "\n", + " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", + " output = output.reshape(batch_size, n_time_steps, -1)\n", + "\n", + " # accumulate all time-steps output for final prediction\n", + " output = output.sum(dim=1)\n", + "\n", + " # calculate accuracy\n", + " pred = output.argmax(dim=1, keepdim=True)\n", + "\n", + " # compute the total correct predictions\n", + " correct_predictions.append(pred.eq(y.view_as(pred)))\n", + "\n", + " test_p_bar.set_description(f\"Testing Model...\")\n", + " \n", + " correct_predictions = torch.cat(correct_predictions)\n", + " return correct_predictions.sum().item()/(len(correct_predictions))*100" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Training loop" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "7c174908d3784728bf9706cd7683230e", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/359 [00:00" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "y_avg = []\n", + "for y in epochs_y:\n", + " y_avg.append(np.mean(y))\n", + "\n", + "plt.plot(np.arange(len(epochs_x)), y_avg, marker = '.')\n", + "plt.xlabel('epoch')\n", + "plt.ylabel('average loss')\n", + "plt.ylim(0,)\n", + "plt.xticks(np.arange(len(epochs_x)))\n", + "for i, txt in enumerate(y_avg):\n", + " if i%2 == 0:\n", + " pass\n", + " else:\n", + " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAG2CAYAAACZEEfAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAAB7hklEQVR4nO3deXhM1/8H8PdkskeE7DNJJEESBLEWsRdRVUupXe2qpSXUvrSqJGiptoryU2uLllhqj5bYiSVEBCEhEYkIyWTfz++PfDM1ss2QiIz363nmaXPvPcuNY+7HuWeRCCEEiIiIiLSUTkVXgIiIiKg8MdghIiIircZgh4iIiLQagx0iIiLSagx2iIiISKsx2CEiIiKtxmCHiIiItBqDHSIiItJqDHaIiIhIqzHYISIiIq1WocHOyZMn0aNHD8jlckgkEuzZs0flvBAC8+fPh1wuh5GRETp06ICQkBCVazIzM/HFF1/A0tISJiYm6NmzJx4+fPga74KIiIjeZBUa7KSmpsLDwwMrV64s8vzSpUuxfPlyrFy5EoGBgbC1tUWXLl2QnJysvMbb2xu7d+/G9u3bcfr0aaSkpOCDDz5Abm7u67oNIiIieoNJ3pSNQCUSCXbv3o3evXsDyO/Vkcvl8Pb2xowZMwDk9+LY2NhgyZIlGDduHBQKBaysrLBlyxYMGDAAAPDo0SM4ODjg4MGD6Nq1a0XdDhEREb0hdCu6AsWJiIhAbGwsvLy8lMcMDAzQvn17nD17FuPGjcPly5eRnZ2tco1cLkf9+vVx9uzZYoOdzMxMZGZmKn/Oy8vDs2fPYGFhAYlEUn43RURERGVGCIHk5GTI5XLo6BT/suqNDXZiY2MBADY2NirHbWxs8ODBA+U1+vr6qF69eqFrCtIXxdfXF998800Z15iIiIgqQlRUFOzt7Ys9/8YGOwVe7GkRQpTa+1LaNbNmzcKUKVOUPysUCtSoUQNRUVGoWrXqq1WYiIiIXoukpCQ4ODjA1NS0xOve2GDH1tYWQH7vjUwmUx6Pi4tT9vbY2toiKysLCQkJKr07cXFx8PT0LDZvAwMDGBgYFDpetWpVBjtERESVTGmdIG/sOjvOzs6wtbWFv7+/8lhWVhYCAgKUgUzTpk2hp6enck1MTAxu3LhRYrBDREREb48K7dlJSUnB3bt3lT9HREQgKCgI5ubmqFGjBry9veHj4wMXFxe4uLjAx8cHxsbGGDx4MADAzMwMo0ePxpdffgkLCwuYm5tj6tSpaNCgATp37lxRt0VERERvkAoNdi5duoSOHTsqfy4YRzN8+HBs3LgR06dPR3p6OsaPH4+EhAS0aNECR48eVXk398MPP0BXVxf9+/dHeno6OnXqhI0bN0Iqlb72+yEiIqI3zxuzzk5FSkpKgpmZGRQKBcfsEBERVRLqPr/f2DE7RERERGWBwQ4RERFpNQY7REREpNUY7BAREZFWY7BDREREWo3BDhEREWk1BjtERESk1RjsEBERkVZjsENERERajcEOERERaTUGO0RERKTVGOwQERGRVmOwQ0RERFqNwQ4RERFpNQY7REREpNUY7BAREZFWY7BDREREWo3BDhEREWk1BjtERESk1RjsEBERkVZjsENERERajcEOERERaTUGO0RERKTVGOwQERGRVmOwQ0RERFqNwQ4RERFpNQY7REREpNUY7BAREZFWY7BDREREWo3BDhEREWk1BjtERESk1RjsEBERkVZjsENERERajcEOERERaTUGO0RERKTVGOwQERGRVmOwQ0RERFqNwQ4RERFpNQY7REREpNUY7BAREZFWY7BDREREWo3BDhEREWk1BjtERESk1RjsEBERkVZjsENERERajcEOERERaTUGO0RERKTVGOwQERGRVmOwQ0RERFqNwQ4RERFpNQY7REREpNUY7BAREZFWY7BDREREWo3BDhERERWSk5ODuXPnwtnZGUZGRqhZsyYWLFiAvLw85TVCCMyfPx9yuRxGRkbo0KEDQkJCSszXz88PzZo1Q7Vq1WBiYoJGjRphy5Ytha5btWoVnJ2dYWhoiKZNm+LUqVMvfS8MdoiIiKiQJUuWYM2aNVi5ciVCQ0OxdOlSfPfdd/j555+V1yxduhTLly/HypUrERgYCFtbW3Tp0gXJycnF5mtubo45c+bg3LlzuH79OkaOHImRI0fiyJEjymt27NgBb29vzJkzB1evXkXbtm3RrVs3REZGvtS9SIQQ4qVSapGkpCSYmZlBoVCgatWqFV0dIiKiCvfBBx/AxsYG69evVx7r27cvjI2NsWXLFgghIJfL4e3tjRkzZgAAMjMzYWNjgyVLlmDcuHFql9WkSRN0794d3377LQCgRYsWaNKkCVavXq28pm7duujduzd8fX2Vx9R9frNnh4iIiApp06YN/vnnH9y5cwcAcO3aNZw+fRrvv/8+ACAiIgKxsbHw8vJSpjEwMED79u1x9uxZtcoQQuCff/7B7du30a5dOwBAVlYWLl++rJIvAHh5eamd74t0XyoVERERabUZM2ZAoVCgTp06kEqlyM3NxaJFizBo0CAAQGxsLADAxsZGJZ2NjQ0ePHhQYt4KhQJ2dnbIzMyEVCrFqlWr0KVLFwBAfHw8cnNzi8y3oExNsWeHiIioBE5OTpBIJIU+EyZMAAA8fvwYI0aMgFwuh7GxMd577z2EhYWVmGdISAj69u2rzHvFihUal1veduzYga1bt+KPP/7AlStXsGnTJnz//ffYtGmTynUSiUTlZyFEoWMvMjU1RVBQEAIDA7Fo0SJMmTIFJ06ceOV8i8Ngh4iIqASBgYGIiYlRfvz9/QEA/fr1gxACvXv3Rnh4OPbu3YurV6/C0dERnTt3RmpqarF5pqWloWbNmli8eDFsbW2LLdfe3r7Q8VWrVikDnpSUFHz++eewt7eHkZER6tatqzLOpSgdOnQoMojq3r278prk5GSMHTsWqampGDlyJMaNG4c6depg8uTJyjEzBfV+sbclLi6uUK/Mi3R0dFC7dm00atQIX375JT766CNlvpaWlpBKpS+Vb7HlvVQqIiJ6a8Uo0nH2XjxiFOkVXZXXwsrKCra2tsrP/v37UatWLbRv3x5hYWE4f/48Vq9ejebNm8PNzQ2rVq1CSkoKtm3bVmyezZs3x3fffYeBAwfCwMCg2HKvXLmiDLLGjBkDmUwGID/QAoDJkyfj8OHD2Lp1K0JDQzF58mR88cUX2Lt3b7Fl+/n5qQRvN27cgFQqVeYJAGPGjEF6ejpGjBiB4OBgeHl5KQO4gqnnzs7OsLW1VQZ/QP54m4CAAHh6eqr/C0Z+r01mZiYAQF9fH02bNlXJFwD8/f01zvf5At56CoVCABAKhaKiq0JE9EbbfvGBcJ65XzjO2C+cZ+4X2y8+qOgqvVaZmZnCwsJCLFq0SAghxPXr1wUAcffuXZXrbG1txfDhw9XK09HRUfzwww9qlevp6Slq1aol8vLyhBBCuLu7iwULFqhc26RJEzF37lz1bkgI8cMPPwhTU1ORkpIihBAiLS1NSKVS0alTJ2FnZyf2798vIiIihJOTkzA2NhbTp09Xpl28eLEwMzMTfn5+Ijg4WAwaNEjIZDKRlJSkvObjjz8WM2fOVP7s4+Mjjh49Ku7duydCQ0PFsmXLhK6urli3bp3ymu3btws9PT2xfv16cfPmTeHt7S1MTEzE/fv3Vequ7vObA5SJiEgtMYp0zPILRt7/FizJE8Bsvxto52oFmZlRxVbuNdmzZw8SExMxYsQIAECdOnXg6OiIWbNm4ddff4WJiQmWL1+O2NhYxMTElGm5CQkJCA0NxdSpU5VjV9q0aYN9+/Zh1KhRkMvlOHHiBO7cuYMff/xR7bzXr1+PgQMHwsTEBED+YoK5ubmYNGkS/vnnH4wfPx5xcXEQQsDGxkY5PRwApk+fjvT0dIwfPx4JCQlo0aIFjh49ClNTU+U1kZGR0NH570VSamoqxo8fj4cPH8LIyAh16tTB1q1bMWDAAOU1AwYMwNOnT7FgwQLExMSgfv36OHjwIBwdHV/uF6h26FcBsrOzxZw5c4STk5MwNDQUzs7O4ptvvhG5ubnKa/Ly8sTXX38tZDKZMDQ0FO3btxc3btzQqBz27BARle7M3SfCccb+Qp+zd+PLvWxHR0cBoNBn/PjxQghR5DkAYunSpSXmm5CQIMaPHy9sbW2FgYGBqFOnjjhw4IDy/Ndff10oT319fZU8Ll26JDw8PAQAIZVKRdeuXUW3bt1Et27d1L630np2vLy8RJMmTYRUKhXR0dHK45mZmWLYsGECgNDV1RX6+vpi8+bNapUrhBAXLlwQAMSFCxdUjrdq1Uq0b99eREdHi5ycHLFlyxYhkUiEq6ur2nkXeJSYJs7cfSIeJaZpnLY0WtGzU7B646ZNm+Du7o5Lly5h5MiRMDMzw6RJkwD8t3rjxo0b4erqioULF6JLly64ffu2SmRJRESvxtnSBBLkP/Gfdz8+Fa1qWZRr2YGBgcjNzVX+fOPGDXTp0kU5zuTFXpRDhw5h9OjR6Nu3b7F5ZmVloUuXLrC2tsbOnTthb2+PqKioQs8Od3d3HDt2DFFRUWjZsiXWrVuncr5p06YICgqCQqFAVlYWrKys0KJFCzRr1uxVbxsA8ODBAxw7dgwNGzZEt27dIJfLled++uknnD9/Hvv27YOjoyNOnjyJ8ePHQyaToXPnzqXmvX79etSvXx/vvPOOyvEtW7Zg1KhRsLOzg1QqRZMmTTB48GBcuXJFo7rvCIxU9gbqSADfPg0woHkNjfIoC290sHPu3Dn06tVLOULcyckJ27Ztw6VLlwDkD2hasWIF5syZgz59+gAANm3aBBsbG/zxxx8ard5IREQls6xiADMjPSSmZwOAMvCZt/cGqhnroVsDWbmVbWVlpfLz4sWLlYOEARSa0bR371507NgRNWvWLDbP3377Dc+ePcPZs2ehp6cHAEW+JtHV1YWtrS3WrFkDa2trDB48uMj8zMzMAABhYWG4dOmSyuueV7FhwwZYWFjg+vXrmD9/vvJ4eno6Zs+ejd27dyufkw0bNkRQUBC+//77UoOdtLQ0bN++HQsWLCh0rlatWggICEBqaiqSkpIgk8kwYMAAODs7q13vN+m15xs9G6u8Vm/MzMxEUlKSyoeIiEpm5+CIa/O74sGSD/BgyQe4/7//Pj68Cp9vu4oD12MQGhqKnj17wszMDKampmjZsmWp+xmtWLECbm5uMDIygoODAyZPnoyMjAzl+eTkZHh7e8PR0RFGRkZo2bIlNm7ciFGjRhW57srjx49x4MABjB49usRy9+3bh1atWmHChAmwsbFB/fr14ePjo9KDBOQHLzKZDAsXLkS1atUK3c9ff/2FEydOKKefd+nSBb1791Z5Ng0bNgyzZs1S/pyVlYWgoCAEBQUhKysL0dHRCAoKwt27d1XyzsvLw4YNG+Dq6gpra2uV6eHZ2dnIzs5WGQ8DAFKpVGWzzuL8+eefyMzMxNChQ4u9xsTEBDKZDAkJCThy5Ah69epVar4FIp6kKgOdArlC4H58mtp5lJkyf4FWhvLy8sTMmTOFRCIRurq6QiKRCB8fH+X5M2fOCAAq7y+FEGLs2LHCy8ur2HyLegcLjtkhIipWTm6eaP31bmE/YYtYvOuciImJEf7+/gKA+GjeOuE4Y79w+PT/hKlZNTFt2jRx5coVce/ePbF//37x+PHjYvPdunWrMDAwEL///ruIiIgQR44cETKZTHh7eyuv6d+/v6hXr54ICAgQYWFh4qOPPhIAxKVLl4rMc8mSJaJ69eoiPT29xHtyc3MTBgYGYtSoUeLSpUti27ZtwtzcXHzzzTfKaw4ePCh27twp1qxZIwCI5s2bCxsbGxEf/984pR9//FHY29sLPT09UaNGDTF37lyRmZmpUlb79u1F/0FDlWNXIiIiinwOtW/fXiXdkSNHBAAhl8vFjBkzCt1D+/bthbu7uzh+/LgIDw8XGzZsEIaGhmLVqlXKa16cDVWgTZs2YsCAAUX+bg4fPiwOHTokwsPDxdGjR4WHh4d45513RFZWVom/0+fN3R1caHxXzZkHynTsjrpjdt7oYGfbtm3C3t5ebNu2TVy/fl1s3rxZmJubi40bNwoh/gt2Hj16pJJuzJgxomvXrsXmm5GRIRQKhfITFRXFYIeIqAR/X4sWjjP2i4bzj4jkjGwhhBCTJk0StWrVEtk5uWLKjiBhXKetMHHvKPZcfah2vhMmTBDvvvuuyrEpU6aINm3aCCH+mwa9f/9+5XkvLy9RtWpVMWfOnCLzdHNzE59//nmpZbu4uAgHBweRk5OjPLZs2TJha2tbbJqUlBRhY2Mjli1bVmr+z3uVKfsFAc/t27cLnYuJiREjRowQcrlcGBoaCjc3N7Fs2TLl1HQh8gOiF6fB3759WwAQR48eLbLMHTt2iJo1awp9fX1ha2srJkyYIBITE9Wu868Bd5UBjtPM/wKdsl6qQCsGKE+bNg0zZ87EwIEDAQANGjTAgwcP4Ovri+HDh6us3liw0BJQ+iqLBgYGxS7iREREqvLyBFb+m/96ZWRrJ1Qx0EVWVha2bt2KKVOmQFeqg8V96uPn4Zdh3OxDDOrTA/qJkXBzqYVZs2ahd+/exebdpk0bbN26FRcvXsQ777yD8PBwHDx4EMOHDwfw3zRoQ0NDAP8N1q1duzZOnz5dKL9Tp07h9u3b2LFjR6n3JZPJoKenB6lUqjxWt25dxMbGIisrC/r6+oXSmJiYoEGDBqVuB1EgKycPx27GYuauYOXAbk3Hrnh5eUGIF4eF57O1tcWGDRtKTH/ixAnlQpDOliaQmRnB1dW12DwBoH///ujfv3+pdSvKn5ei4HPwFgBgxnt10LuxHPfj0+BkaVxhSxS80cFOWlpaie8in1+9sXHjxgD+W71xyZIlr72+RETa6J9bcbgVm4wqBroY4ekEoPB6M0/jnyA7Iw1pgbtQpfVQGDmOhGuVR+jTpw+OHz+uHEj8ooEDB+LJkydo06YNhBDIycnBZ599hpkzZwLI30OpVatW+Pbbb1G3bl2sX78epqamyrGcL1q/fj2aNm0KDw+PUu+rdevW+OOPP5CXl6d81ty5cwcymazIQAfIH/MZGhqKtm3bFnleCIF7T1JwKiwep8PicT78KVKzcgtdVzB25XU8/F/njKjDN2Ixc9d1AMC4djXxWYdaAFDh6zC90cFOjx49sGjRItSoUQPu7u64evUqli9fjlGjRgHI3yTM29sbPj4+cHFxgYuLC3x8fGBsbFzsaHkiIlKfEAIrj+f36gxt6YhqxvlBwPr161WmQRf8I7Tvhx/Cqd8X2HYxCqclNdGkzSWsWbOm2GDnxIkTWLRoEVatWoUWLVrg7t27mDRpEmQyGebNmwdAdRo0kN+b8cEHHxSaBp2UlIS//voLy5YtK7KsYcOGwc7OTrkH02effYaff/4ZkyZNwhdffIGwsDD4+Phg4sSJyjRTp05Fjx49UKNGDcTFxWHu198gIVGBbh/+twBefEomztyNVwY4sUkZKuVWe24GWwEJACdL42J+62WnqBlRs/yCy2VG1Nm78Zi47SryBDCgmQNmdqtTpvm/ijc62Pn5558xb9485eqNcrkc48aNw1dffaW8Rp3VG4mI6OWcvhuPa1GJMNTTwZi2+dOOC14l+fn5Ka+ztLSErq4u3N3rYXbvBtCRSPD7hUiEZZkh+Wbxr3zmzZuHjz/+GGPGjAGQP1whNTUVn3zyCebMmQMdHR3lNOh9+/ahV69eCAgIwLx58wpNg96+fTuEEBg0aFCRZb24kq+DgwOOHj2KyZMno2HDhrCzs8OkSZMwY8YM5TUPHz7EoEGDEB8fjyrVzJFZvRbMBi5F/z/uol3tRMSlZCE0RnVGr76uDt5xMkcbF0u0qW2JerKq+OtyFGb73UDuc6+OHiakl3uPx+2Y5EIzovIEsObEPczv6f7Su4i/6FpUIsZuvoSs3Dy8526LRR/WL7O8y4JElPTS7i2RlJQEMzMzKBQKVK1ataKrQ0T0xhjw6zlciHiGka2d8HUPdwDA/Pnz8euvvyIqKgq6uv/9m9nT0xO1atXCli1bIITAV3tDsGz6WOjoGWD9xs1Fvjpp2rQpOnfurDL0YNu2bRg1ahRSUlJUxtMUSEhIgLOzM5YuXYpPPvlEo/uJUaQjIj5VOXblRbl5AglpWYhPyUR8cv5/nyRn4v7TVPx+ofgp9HVlVdHOxRJtXCzR3MkchnqF6x2jSMf9+FRsOvsAh0NiITMzxMGJbVHdpOhXZq9KCIFPt1zGkZuPizzf0c0K3/XzgGWVVxvDejcuGf3WnENCWjY8a1ngtxHNi7z/8qDu8/uN7tkhIqKKczHiGS5EPIOeVIJP2uUvzlew7svw4cNVAh0gf1LJgAED0K5dO3Ts2BHWUceRcS8Q1oN8MGNX/quUQyvnqrxK6tGjB5YvX47GjRsrX2PNmzcPPXv2VAY6R44cgRACbm5uuHv3LqZNmwY3NzeMHDlSo/t5fuyKBEAbF0tYVjFQBjTxKVl4lppZqCekJJ93rIXhns6wMi09YJCZGUFmZoSG9tVw5+dkhMenYupf1/B/w5uVSy/IhjP3ceTmY0gk+febJwCpBOjeUI7DIbE4fvsJ3ltxCsv7e6Cdq1Wp+RUlOjEdH6+/iIS0bHjYm2HtsGavLdDRBIMdIiIqUsFYnY+aOih7QY4dO4bIyEjl2Mnnffjhh1izZg18fX0xceJEuLm5wW/XTlyTumDDmfuY5RcMoxt3VF4lzZ07FxKJBHPnzkV0dDSsrKyU4zULKBQKzJo1Cw8fPoS5uTn69u2LRYsWKVc9VsfVyATM2BWs/FkAOBUWX+z15ib6sKyiD8sqBrCsYgAjPSn+vBSlslWGVCLBkJaOagU6zzMx0MXPgxvjw1Vn8c+tOPx25j5Gt1F/ZWJ1nA9/ikUHQwEA87rXQ7cGtiozom7FJmHitqu48zgFw367iLFtnTG1qxsMdNUPVOJTMvHx/11AjCIDta2rYMPId1DF4M0MK/gaC3yNRUT0ousPE9Fz5RlIdSQ4/mUH1LB4+cG0QggsPBCK9acjAABTvVzRxLF6sa+SypIiLRurTtzF+tMRyCmiy2bQOw5o6mgOyyr6sDI1gFUVA5ib6ENXWniDgR2BkcpxN1KJBD596r/SrKYt5+5j3t4Q6Ekl2PWZJxraV3vpvJ73KDEdPX4+jaepWejdSI4fBjQqsucoIzsXiw6EYsv5BwAAd3lV/DSoMWpZVSm1jOSMbAxadx43opNgV80IOz9rVSEzrtR9fjPYAYMdIqIXfbL5Eo7efIw+je2wfECjV85PCAGfg6FYdypCeaw8p0FnZOdi09n7+OX4XSRl5BR5jVQiwemZHTV6SOePuymbNWOEEBj/+xUcuhGLGubG2D+xDaoaqt9bVZSM7FwM+PUcrj1UoJ6sKnZ95gkj/ZJ7a/xvPsb0ndeQkJYNIz0p5vesh/7NHIp9tZaRnYsRGy7ifPgzWJjo469PW6GmGgFSeVD3+f1G741FRESv3+3YZBz931iP8R1rlUmeEokEI1s74fnHZ54AZuwKxrYLkcjMKbwWzcvIzRP481IUOn5/Ar6HbiEpIwduNqbYMKI5FvdpAOn/HuAFPTOaBiwyMyO0qmVRJr0YEokEi/s2hH11I0Q+S8Msv+ASF/orjRACX+8NwbWHClQz1sOvHzctNdABgC71bHBoUjt41rJAenYuZuwKxoQ/rkCRll3o2pzcPHz+x1WcD3+GKga62DTqnQoLdDTxZr5cIyKiCvPL/8bqdKtvi9rWZbeMx/2naSjqUT5rdzAWH76F7g1l+LCxHZrWqA4dHc0G7Aoh8O+tOCw5fAt3HqcAAORmhpji5YYPG9tB+r/82rtZVfhqvs8zM9LDz4Mao9+aczhwPQata1licIuX6+n642IkdlyKgo4E+GlgYziYq//q0dbMEFtHt8DaU+H4/shtHAyORVBkIlYMbIx3nM0B5K+kPWNXMI6FPoaBrg7+b3gz1Lcze6m6vm58jQW+xiIiKhARn4pOy04gTwAHJraBu7zsHmYxinS0XvyvymwnCQCLKvqIT8lSHrOvboTejezQu7EdaluX3mtwJTIBiw/ewsX7zwDkBxATOtbCsFZOb+TMoKKsOxmORQdDYaCrg72ft0YdW82eRZcfJGDg2nPIzhWY/p4bxneo/dJ1uRaViEnbr+L+0zToSIDP33VBv6Z28Dl4C4duxEKqI8GvQ5uic73it2V6XThmRwMMdoiI8k3feQ1/XnqITnWssX5E8zLPv6hBvh81dcD58KfYfTUah4JjVLZXaGhvht6N7NDDQ66c9VSwVo6ORIKNZ+7jcEgsAMBAVwcjWzvjs/a1YGb8amNfXre8PIHRmwJx/PYT1LIywd9ftIGxvnovX+KSM9Dj59N4nJSJ9xvY4pfBTV55KntKZg7m7wvBzssPC53r38weSz8qfTuO14HBjgYY7BARAQ8T0tDhuxPIyRPwG++JJjWql0s5JQ3yTc/KxbHQx9h9NRoBd54g93/dQFIdCdrUtoTMzBB/XopS6R3SkQAfNbWHd2dXyKtV/Kupl/UsNQvdfjyJx0mZ+KipPb7vV3pAkZWThyH/dx6B9xPgYl0Fuye0LtPp35vO3cfXe0NUjr3MwO7ywgHK9NYq2N03RpFe0VUhKnNOTk6QSCSFPhMmTEB2djZmzJiBBg0awMTEBHK5HMOGDcOjR49KzNPPzw/NmjVDbXtbhH/fB4rfJyMkYL/KNfPnzy9Upq2t7UvdQ0mDfI30pejhIcdvI5rj4uxO+KanOxo5VENunkDAnSfYHhhVaNG/LaNbYOlHHpU60AHy1/b5aWBj6EiAnZcfwu9K4V6VFy06cBOB9xNgaqCLXz9uWubr3LgU8RqxYBPTyoQDlEmr/HHhAebsvgGB8t/dl6giBAYGIjf3v9c8N27cQJcuXdCvXz+kpaXhypUrmDdvHjw8PJCQkABvb2/07NkTly5dKjZPc3NzfD55GuafSkS2kKKfdRxGjhwJa2trdO3aVXmdu7s7jh07pvy5qK0cypJFFQMM93TCcE8nRMSn4ud/wuB3NbrQdTpv0B5Mr6pFTQt4d3bFcv87mLvnBjwcqhW77s3Oyw+x6Vz+GjkrBjYql1lRzpYm0JFAJcCUSiSvZRPTssSeHdIaMYp0ZaAD5P/lnO13gz08pFWsrKxga2sLYWSG8FRd7Ni1B7Vq1UL79u1hZmYGf39/9O/fH25ubmjZsiV+/vlnXL58GZGRxe/r1KFDB0RXawBUs0fLRvXww4JZaNiwIU6fPq1yna6uLmxtbZUfK6uX22LgZThbmmDae254cZJWZXzwlmZCx9rwrGWBtKxcTPj9CjKyC0/LD36owOzd+StCT+rkgk51y2ewsMzMCL5lMGW/ojHYIa1xIfxZoWmtlbG7lag0OwIj0Xrxvxi05jR+27QFzbz6FDsgVaFQQCKRoFq1asXm9yw1S7nJ5YSOtfDvv//i9u3baNeuncp1YWFhkMvlcHZ2xsCBAxEeHl5m96QObXnwlkaqI8GKAY1gYaKPW7HJWHQgVOX805RMfLr1MrJy8tCpjjUmdXIp1/oMaF4Dp2d2xLaxLXF6ZsdK2VvO11ikFYQQ2B5Y9L9cZWaGr7k2ROUnRpGu3Mwy7c555GWk4JxOfcQo0gs99DMyMjBz5kwMHjy4xMGbG85EICU5CY9WjUDXZdmQSqVYtWoVunTporymRYsW2Lx5M1xdXfH48WMsXLgQnp6eCAkJgYWFRbnd74sGNK+Bdq5v1lo55cG6qiGWD2iE4b9dxJbzD+BZywLdGsiQk5uHL7ZdRXRiOpwtTbB8QCON1yR6GQWbmFZW7NkhrXD4RizOhz+DVCIp1M29zP/OK61KSm+mkgbqAvkB8Pz58yGXy2FkZIQOHTogJCSklFyBFStWwM3NDUZGRnBwcMDkyZORkZGhPL969Wo0bNgQVatWRdWqVdGqVSscOnSo3O7zRRHxqcrxEynXj8KoZlNIqljgt9MRKq87srOzMXDgQOTl5WHVqlXF5peUkY2NZ+9Dom+EtbuPITAwEIsWLcKUKVNw4sQJ5XXdunVD37590aBBA3Tu3BkHDhwAAGzatKlc7rMkZbmK8ZusvasVPuuQv4L19F3XcfnBM3jvCMLZe09hrC/Frx83hZlR5ZpiX1HYs0OVXkpmDub/nf8Qm9CxFga1qIH78WmIT8nA5B3X8Pe1R3C2MMYUL7cKrimVpZIG6gLA0qVLsXz5cmzcuBGurq5YuHAhunTpgtu3b8PUtOhVgX///XfMnDkTv/32Gzw9PXHnzh2MGDECAPDDDz8AAOzt7bF48WLUrp2/aNumTZvQq1cvXL16Fe7u7uV4x/luxSQBAHIUcch4cA1WH84GAKw7FYHdV6PxcUsnDGwmx6cjhyIiIgL//vtvib06W849QHJGDlxtqmLEe62goyNBo0aNEBoaCl9fX3To0KHIdCYmJmjQoAHCwsLK/B7pP1O6uOJC+FNciUxE39XnlMc/bGwHV5uyW91a27Fnhyq95Ufv4HFSJhwtjDG+Y23lv/p6eNjB58MGAICf/r2LXUUsjkWVV8FA3YLP/v37lQN1hRBYsWIF5syZgz59+qB+/frYtGkT0tLS8McffxSb57lz59C6dWsMHjwYTk5O8PLywqBBg1RmMvXo0QPvv/8+XF1d4erqikWLFqFKlSo4f/58ud/zzssP8e3/xm+kBPtDamwGk9rN0b2BDHbVjBCfkoXlR26iVosuOH05GP+3fU+Jr5jSsnKUO5FP6Fhb5XWIEAKZmZnFps3MzERoaChkMlkZ3R0VRU+qg7nd6xY6vv1iFCdfaIDBDlVqN6IV2Hg2Ag9Xj8LJ6e/CSF9X5ZVGwMbF+KxDLQghMNZ7BqxsbDV6pbFr1y7Uq1cPBgYGqFevHnbv3q1yPicnB3PnzoWzszOMjIxQs2ZNLFiwAHl5eeV1y8V6m9cXysrKwtatWzFq1ChIJBJEREQgNjYWXl5eymsMDAzQvn17nD17tth82rRpg8uXL+PixYsAgPDwcBw8eBDdu3dXue7FV2gJCQkYM2bMK79CK6m9/XkpClP/vIJnAVuQ8NtYKM5sg5FOLj6SBuLnQY1wYloHLP+oPjIPf4+MmDDodZ6EPqvOYPCPh3Hwwk2VwGXYsGGYNWsW/rgQiWepWZBc2wPDxzcQHh6OW7duYfny5di8eTOGDh2qTDN16lQEBAQgIiICFy5cwEcffYSkpCQMHz681PuiV5ORU/j7hJMvNCRIKBQKAUAoFIqKrgppICc3T/T8+ZRwnLFfjFp9TMTExCg//v7+AoA4fvy4yM3NEy0HfC4k+kbCsf88cTDgghgwYICQyWQiKSmp2PzPnj0rpFKp8PHxEaGhocLHx0fo6uqK8+fPK69ZuHChsLCwEPv37xcRERHir7/+ElWqVBErVqx4Hb8Cpe0XHwjnmfuF44z9wnnmfrH94oPXWn5F27Fjh5BKpSI6OloIIcSZM2cEAOXPBcaOHSu8vLxKzOunn34Senp6QldXVwAQn332WaFrTpw4IYyMjISOjo4wNTUVCxcuVLY3IYRYvHixMDU1Fbt27RLBwcGv3N62X3wgnGbuF9XafiyMTKuJb775RgAQP/74o0p7i4iIEACK/HhO/EnsC4oW2Tm5on379qLvgCHCY/4R4Thjv/hw5Oeidu3awtDQUFSvXl20atVKbN++XaV+Bfegp6cn5HK56NOnjwgJCSn1z4Ze3aPENOXf74JPzZkHxKPEtIquWoVT9/nN7SLA7SIqqy3nH2DenhswNdDFsS/bw6bqf7OuvL29sX//fuV4ArlcjuoteiOtzgdwsjDG9tHNUKemA5YsWYJx48YVmf+AAQOQlJSkMvj0vffeQ/Xq1bFt2zYAwAcffAAbGxusX79eeU3fvn1hbGyMLVu2lMdtF1LU5opv0nLur0PXrl2hr6+Pv//+GwBw9uxZtG7dGo8ePVJ5zTJ27FhERUXh8OHDReZz4sQJDBw4EAsXLkSLFi1w9+5dTJo0CWPHjsW8efOU12VlZSEyMhKJiYnYtWsXVqxYASsrKzx4kL/Am1wuh7e3N2bMmAEg/5WPjY3NS7W3pDwDPGryCQBA/5+laNOgFn777TflNSW1t7txKVh/OgJ+Vx4i83+9A3bVjNDIoRoOBscol2pY1Ls+hrR0LPF3TBWrqD3FKuMU8LLG7SJIq8UlZ2Dp4VsAgKld3VQCneJeaaycOgJ21Yxw/2kaJv55A23btSvxlca5c+dUXoMA+Q/V59O0adMG//zzD+7cuQMAuHbtGk6fPo3333+/LG+3RLdikgotn/82dXE/ePAAx44dw5gxY5THCrYxiI2NVbk2Li4ONjbFL742b948fPzxxxgzZgwaNGiADz/8ED4+PvD19VV5Namvr4/atWujWbNm+Oabb5CbmwtbW9tXeoVWVHuzcGuOixfyB6WObO2EkX3ew7///qt2e6ttXQW+fRrg7Mx34d3ZBeYm+ohOTMeB5wIdAPhqb8hb+fqzMtGGtW4qEmdjUaXkcyAUyRk5aGBnhqEv/It0z549SExMVM6iKXjg1anpgA0upui76iwu3n8G01Q9ZGXGvpi1UmxsbKEHo42NjcoDdMaMGVAoFKhTpw6kUilyc3OxaNEiDBo0qIzutGRCCPx+IarQcR0JtG5V2eJs2LAB1tbWKuNqnJ2dYWtrC39/fzRu3BhAfhAcEBCAJUuWFJtXWloadHRU/w0olUohhCh2+YI9e/YgJycHTk5OAP5rb0W1nYKen6K82N62nLuPoxEZyE1NwOg2zvmDVD+oh6SkJI3bm0UVA3h3dsWn7WvhuyO3lYOSCxQEx29LT2BlVdnXuqlI7NmhSufM3XjsCXoEHQng82EDSF9YWGf9+vXo1q0b5HK5ynGJRAJXG1P8MqQJpDoS3I9PQeSzkv81++KqtEIIlWM7duzA1q1b8ccff+DKlSvYtGkTvv/++9e29sj60xE4FvoYOhKorC9krC+FvlT7/3rn5eVhw4YNGD58OHR1//u3m0Qigbe3N3x8fLB7927cuHEDI0aMgLGxMQYPHqy8rmCgboEePXpg9erV2L59OyIiIuDv74958+ahZ8+eyn2gZs+ejVOnTuH+/fsIDg7GzJkz8wfAjx2rUrfS2k5RCs5vOnsf8/aGQAgBXZ382TgSieSV25uhnhRj2jq/FVsuED2PPTtUqWRk52LunhsAgGGtnNDA3kzlfMErDT8/P+Wx519pyGQytHO1woJe7hj7lwJReSbYGxSNXo3sCpVla2tb6muQadOmYebMmRg4cCAAoEGDBnjw4AF8fX3LfZbK2Xvx8D2U/yrvqw/qoWt9W9x5nIxv9t1EeHwqZvkF49ePm5b6gK3Mjh07hsjISIwaNarQuenTpyM9PR3jx49HQkICWrRogaNHj6qssRMZGanSkzN37lxIJBLMnTsX0dHRsLKyQo8ePbBo0SLlNY8fP8bHH3+MmJgYGJtUQWLCM3w5c45yteEX21uB0l6hFbS3305HYMH+mwCAFjJd3JDZKv8My6K9FWy58OL4D/YYkDbT/n/60WsVHR2NoUOHwsLCAsbGxmjUqBEuX76sPJ+SkoLPP/8c9vb2MDIyQt26dbF69eoS8wwJCUHfvn3h5OQEI31dXDv0B6xNDTDFy1V5TcEUcA8PD+Tl5cHb21s5Bfz5VxoF+jWWQcTchIFdXUz76zou3X9WqNxWrVqppAGAo0ePwtPTU/lzca89ynvqeXRiOj7/4ypy8wT6NLbDcE8nyMyM0N7VGisHN4GeVIKjNx/jr9e0tlBFTXv38vKCEAKurq6FzkkkEsyfPx8xMTHIyMhAQEAA6tevr3LNiRMn4PvjamXddXV18fXXX+Pu3btIT09HZGQkfvnlF1SrVg3ZuXl4nJSByQuWY+ORi5j4+0WIel0hNamOnXnvYNvF/O1KimpvBa/Qnm87L2rVqhV+27FXGeiM71ALeVHXyqW9cfwHvW3Ys0NlJiEhAa1bt0bHjh1x6NAhWFtb4969eyobEE6ePBnHjx/H1q1b4eTkhKNHj2L8+PGQy+Xo1atXkfmmpaWhZs2aaP9eD0yZPAUA8FWPeqhq+N8y6UuWLMHq1auhr6+PTz/9FJ06dcLIkSNhZmaGSZMmKV9puLi4wMXFBT4+PqhetQq8PuyHExEp+GTLZTjf2ADXmo7w9fUFAEyaNAnt2rXDkiVL0KtXL+zduxfHjh1T2Qm64F/9NWrUgLu7O65evYrly5cX2dNQVjKyc/HZ1st4lpoFd3lV+PRpoNJ7U09eFVO6uGHJ4VtY8PdNtKppAQfz8ntFsSMwUrlXk44E8O3ToNI8PJ+vu0QC9G1sD2crE8SnZCI+JQvxyZn/+/9MJKRlq6QVIg8pwcdgUr8TIJFill8wjtyIRed6Nhj+yfhC7a2oV2h2dnbK9ubc/iP8+flAVDN0xtih/ZEXtKdc2xvHf9DbhFPPwannZWXmzJk4c+YMTp06Vew19evXx4ABA1Sm8TZt2hTvv/8+vv3222LTCSEw7LeL2DalJxq/PxgX/1iu8oD/4IMPkJOTgyNHjuD27dtwdXVVmZIrhMA333yDX3/9VflK45dffkFN1zoY8Ot5BEcroPhrDrq1aog/tm5W5rtz507MnTsX4eHhqFWrFhYtWoQ+ffoozycnJ2PevHnYvXs34uLiIJfLMWjQIHz11VfQ19d/2V9lib+HaTuvY+flh6hurId9n7cpMpDJzRMYuPYcAu8n4B0nc2z7pGWhsU1loTJPe3+UmIbWi49Dky9AHUn+YF9DXR3cuXIGcX9+BfnYX6FnrvoaVAgBceUvPLt0EFlpyWj+zjv4dfUqlZ6lDh06wMnJCRs3bsSagHtYfOgWUm+dhuTyDiQ+fvhGtDeiN526z28GO2CwU1bq1auHrl274uHDhwgICICdnR3Gjx+vMnDz008/xeXLl7Fnzx7I5XKcOHECPXv2xKFDh9CmTZti89537REmbruK6DWjMGval1gwZ7rK+cWLF2PNmjU4evQoXF1dce3aNXh5eWHFihWlzlR5nJSB3r+cQYwiA01qVIN3Z1e42FTR6GEdHR2NGTNm4NChQ0hPT4erqyvWr1+Ppk2bAig8WLXA0qVLMW3atFLz3759OwYNGoQmbbvgqeck6EiAzaNa4NTOdfDz88OtW7dgZGQET09PLFmyBG5uboh6lob3VpxEalYuZnarg0/b11L7ftTld+Uhpvx5rdDxbWNbolWt17cTtqbSsnIwamMgzocXfn3ZztUS9WRmsKyiDytTA1hWKfjoo7qxPnR0JEUGeToSYHQbZ1yLUuBKZAJynjupIwEa2FdD29qWaONiiSY1qkNfVwcxinSsOBaGHYH5M+q8O7vAu3PhV3JEVDR1n998jUVlJjw8HKtXr8aUKVMwe/ZsXLx4ERMnToSBgQGGDRsGAPjpp58wduxY2NvbQ1dXFzo6Ovi///u/EgMdRXo2vv3fOIaqhnowNyn8L9hXmQJuU9UQv41ojl4rT+NKZCKG/XZRo9cx6ry+i4mJUUlz6NAhjB49Gn379i01/wcPHmDq1Klo3LwVbsUmwwrA9PfqoI2LJRYGBGDChAlo3rw5cnJyMGfOHHh5eeHmzZtwMDfB1z3cMX3XdSw7ehvtXKxQT152wfy9JylY9L99mp73pk97f5iQhk82X8bN/22o+TypRIIlfRuWGugWN8i3oL2kZObgQvhTnAqLx+m78bgbl4JrUYm4FpWIlcfvwlhfCofqRrj9OEWZZ5d6Ngx0iMoJgx0qM3l5eWjWrBl8fHwAAI0bN0ZISAhWr16tEuycP38e+/btg6OjI06ePInx48dDJpOhc+fORea77OhtPEnORE0rE9wzLLrJPj8l193dHUFBQfD29oZcLldrlko1Yz1kP/cv8TwBzNwVjGpGevByty1xRtOSJUvg4OCADRs2KI8VrLlSoGCGToG9e/eiY8eOqFmzZon1ys3NxZAhQ/DlzLlY9JsfBJLRvYEM49rlp3txJeCCNWcuX76Mdu3aoV8ze/iHPob/zceY8mcQ9kxoDUM9aYllquNuXAoGrTuPp6lZsKlqgCfJmcpeDicLE9g+t8jjm+R8+FOM//0KnqVmwcJEH/2a2WPdyYiXmpU0oHkNtHO1wv34NDhZGqukq2Kgi051bdCpbv7sqxhFOk7/L/A5HRaPp6lZKoEOAPwbGocYRfob//qPqDJisENlRiaToV69eirH6tati127dgEA0tPTMXv2bOzevVu5AFzDhg0RFBSE77//vshg51pUIracz1+IbWHv+hj8c9Flv+qU3Ij4VLz4QlcAGLf1ChzMjfBhIzv0amyHWlZVCqXdt28funbtin79+hX7+u55jx8/xoEDB9RaG2XBggWwsLTEaWlDZGTvRBUDXSz9qGGh4KvgNdqBAwcAAJ988gl+//13NG3aFL59GuBqZAKCb9xE4zbz8ejWZeTl5cHd3R1//vknatQovvcqMTERc+bMgZ+fHxISEuDs7Iwv532LNeHVEJ+Sicdrx+BBgur0/AcAelwajv3bN5Z6f6+LEAJbzj/Agr9vIidPoL5dVfz6cTPYVTPCcE+nIgMWdag7yFdmZoR+zRzQr5kD8vIEtgdGYvbuGyrXcGE/ovLDYIfKTOvWrXH79m2VY3fu3IGjY/4Kx9nZ2cjOzlZ76mxObh5m7w6GEMCHje3gWcuy2LJfdUqus6UJdCRQGYMhAWCop4OoZ+n46d+7+Onfu/CwN0Pvxnbo4SGHZRUDAOq9vnvepk2bYGpqqjLwtChnzpzB+vXr0d/nD/jdTICeVIJG8mowMVD9a/v8a7T69esjIyMDixcvVr5Gs6xigInNqmLkoulI8eiCn7buQVt3R4SGhsLQsPgemKysLHTp0gXW1tbYuXMn7O3tcebabSw8Eo40UyPUlVXF0SuXUNXwv56i+ZsO49eZI3G/akOkZ+XCSP/Ve5FeVWZOLr7aE4Idl/LHxfRqJMfiPg2VdXvds5J0dCToWMe6UHvjwn5E5YfBDpWZyZMnw9PTEz4+Pujfvz8uXryItWvXYu3atQCAqlWron379pg2bRqMjIzg6OiIgIAAbN68GcuXL1fmUzAl1/WDTxDyKAlV9AR62GchKCgIWVlZiI6ORlBQEKpUqYLatWsDePUpucWNwejpYYejN2Ox52o0TobF49pDBa49VGDhgVC0c7FE78Z2Kq/vYhTpaFDVAYOHXVN5ffe83377DUOGDCkx0EhOTsbQoUPx8TQfbLuZDIkEeMfZHEYis9C1Ba/RjI2NERUVhdOnT8Pe3l7lmoMbf0Ddd9ohpdUorA0R+LCLA7qX8grtt99+w7Nnz3D27Fno6enhdmwyll2TIM3UAfVkVfH7mBao/sL4Kd2HV2FgLkdydVf8evJehY9BiUvKwKdbL+NKZCJ0JMDMbnUwtm3NCl9okQv7Eb1enI0FzsYqS/v378esWbMQFhYGZ2dnTJkyReV1TmxsLGbNmoWjR4/i2bNncHR0xCeffILJkycrH0AdOnSAjdwBIS5DkZKZA+8WZpjcp22hstq3b48TJ04AKLspuTGK9GJfacSnZGL/tUfYHfQI16ISlccfrRkFtyaeGDBlEX47E4E8AaRcPYi8q7vwLE71Fc+pU6fQrl07BAUFwcPDo9h6BAUF5e/pJMnvrdKRSCBEfi+Vjo4Obt++jVq18mdX1atXD/r6+rhz5w6MjY3h4OCg8hotLy8PZmZm8J4yFSu37UdSdBis5Q5Y/d236N27d7F1eP/992Fubg5jY2P47d6DVB1jGNZpD88+I/H7WE9UM1b9vWZlZUEul+P9wWNw0rgtDHR18M+X7WFfvWJ6K65FJWLclsuITcpAVUNd/Dy4Cdq7WlVIXYpTUnsjotJx6rkGGOy8WWIU6fDeHoQLEc/QuEY17PrUEzrlsEbMqwh/koI9V6OxOygaVzYuQG7yE9gOWao8/+yfdciKuY2IkCsqD7ERI0bgxo0buHTpUon5P4xX4INv/0JcSiZa1rTA/B7u+OqreUhOTsaPP/4IV1dX6OvrQwgBPT095ObmYty4cRg3bhwuXrwIb29v/Prrrxg2bJhy2wJjY2OMmzIbO6KrIi38MhQnN+P48eNo3759kXWoU6cO7t+/j+4f9sMNs5Z49ugBFP/8iqlTvOHz7TeFrv/zzz8xePBgPHjwAFP2R+JCxDN0byDDL0OavORv+eXtuvwQs3YHIysnD7Wtq2DdsGZwtjR57fUgovKl7vOb20XQG2VHYCQ8F/+LCxH565+0c7F64wIdAKhpVQVTvNxwclpHrF48F1kxt6E49yeyEx4h9eYJpFw7jCqNu2P4bxfxg/8dXLr/DE8TEvHXX39hzJgxReZZsCllTm4epvrdRKKRDHXqumPjl33RsGEDVKtWDaampqhfv76yt2rChAnIzc2Fu7s75s+fD5lMhl69emHkyJHKbTgKxi316tULy7+dg0n9u8CsZT+YurbAip9/KfYe8/LyUN3CCnfdBiOzmhM8vXri66/mYsP/rS3y+oINWO3s7DC/pzt0JMCB4BicvRf/Kr9qjeTk5uHb/Tfx5V/XkJWTh851rbF7vCcDHaK3HIMdemPEKNIxyy9YZVbUyn/vvvb9ljQhkUgwrGdnbPx9B1JDA/Bo/QQkntmO6u+ORRX3jrjzOAU//hOGj9acQ4OPv0ZmTi5ETU/ce5KCFztVIyMjERMTg8WHbuF8+DOY6Evx68dNVbbFeFFBQBMSEgKZTKb8pKSkIDIyf68mS0tL6OrqKmfKeXd2RT1ZVaCaHc5eu12oHgWqmlshxcASiow8NHKohi1jWqBJw/qIjY1FVlaWyrUFG7AWBHJ1ZVUxpEX+wPQFf99ETm757hUGAIlpWRixIRDrT0cAACa+WxtrP24G0xJ+f0T0duAAZXpjRMSnqsxOASrPdNxhA/rCoGZzlQGn07q6opqxPk7djceZu/FIrO8F+/peWPJvFJb8GwW5mSHaulihjYslWte2xLa9h/D7+QdYefweAOD7fh5wsflvh+6NGzcWKlcIgcGDByMqKkplm47JkycrZ8Hp6+ujefPmyply+ro6WDGwEZqseoQUver442KkMjApEPxQgUg9B6Q/PY5G9lWxefQ7qGqohzt37kAmkxUaB1Wwvk/BkgIAMKWLK/6+/gi3YpPxx8VIDGvl9Eq/4+LEKNJx8k48fvznDh4lZsBYX4pl/TzQrYGs9MRE9FZgsENvjAfxaYWOVabpuMUtMjfwnRrIzRMIeaTIX1E3LB6XHyTgkSIDOy5FKadEP6+Dm5XaD+vSZsEB+esQDRgwAO3atUPHjh1x9PBhpN+7CKuBPli4PxSetSzx9ZTPYGdnh4HjZ+Dj9Reg3+A9SAL3wSZkO2IjbXAqLAw+Pj6YOHGiSvl5eXnYsGEDhg8fDl3d/75Sqpvo40svN8zbcwPLjt5Bj4byQrO3XtWOwEjM3BWs3N+quoketo1tiTq2HHtHRP/hAGVwgPKbIC45A++tOIVnqVmQIH9BvxeX4NcmaVk5uBjxDKfD4nH8dhzuPUlVOa8jAc7MfFftHq3SZsEB+VPJfX198fDhQ7i5ueHrr+fjzye2OBf+FO7yqnj0+0xYyewR02gUkjNy0NypOj6tm4s5M6YhKCgIdnZ2GD16NGbMmAGp9L/1c44ePYquXbsqN2B9Xm6eQPefTuFWbDKGtqyBhb0bvORvrLAYRTo8ff9V2chT098bEVVunI2lAQY7FUsIgTGbLuGfW3GoK6uKNUOb4FFixlszHffsvXgMXneh0PHXsZlmdGI63v3+BDJzVMfUvONkjg0jmxdawPBlnA9/ioFrz0NHAvz9RRu4y81eOU8hBCbvCMKeoEeFzr3pm5ASUdnhbCyqNLYHRuGfW3HQl+pgxYBGcLQwQataFm9FoAP8t3rz817X6zsdCZCVU3jwsG+fBmoHOtHR0Rg6dCgsLCxgbGyMRo0a4fLly8rzhzf/DMXmCbi/rC8a13ZA586dceFC4eDuRYmJiZgwYQJkMhkMDQ1Rt25dHDx4EDm5eZix67pKoKM49yceLPkACf+sqzSvPYno9eGYHapQD56mKnc0n/6eG9xsTUtJoX0qcjXdiPhUFNW1G5eciVrWhfcBe5E6O767urpi9apfMPvYY2SkZ0An4TS8vLxw9+5dWFkVvchfUVtVREVFQc/QGOO2XMY/t+KgI8nfRmT7wRNIvnYE+lbOaF377QmSiUh9DHaowuTk5mHyjiCkZeWiZU1zjGrtXNFVqjAl7aBdnoraE0yTXiV1dnwfPHgwACDONAzL/e8gwd4eSbu34fr16+jUqVOR+b64VQUAVLWUYdSmQFyNTISBrg5WDm6CVjVMsGd2Pyz/aRW2r10BV5u3L1gmotLxNRZVmF9PhuNKZCJMDXTxfT+PN3LxwNdJZmb02l/fFfQqSf+3VYemvUr79u1Ds2bN0K9fP1hbW6Nx48ZYt25dkdd+0q4m5Ka6uHtyLwxNTEvcLmPfvn1o1aoVJkyYABsbG7jVrYem/cbjyv2nMDPSw+9jWqBLPRtMmDABPXt8gInD+kBfl19nRFQ09uxQhbgRrcAP/ncAAN/0cq+w/ZPo1XqV1N3xff/+/Rg4cGD+7vQm1SHv/y3SdYr/Mw8PD8e///6LIUOG4JfNf2LGb0fxYN/PkGflYOfvP8PFxhTbt2/HlStXEBgY+Er3T0Taj8EOvXYZ2bnw3hGEnDyBbvVt8WFju4qu0ltPZmb0Uj1Kz+/4DgCNGzdGSEhIoR3fO3bsiKCgIDx58gRDp/nggZ8v5jSti80TvIrN19raGqNm+uLT368i19kTtb0SkXhhF1xsTBEVFYVJkybh6NGjJe4eT0QE8DUWVYClh2/jblwKrEwNsOjDBsrdzqnykclkym0oCtStW1e5VUUBExMT1K5dG61atcKhXX9AoiPF33/+jtNhRe+bJZPJYC53xMiNl5Vr/iwa8R7iHj9GVlYWLl++jLi4ODRt2hS6urrQ1dVFQEAAfvrpJ+jq6iI3N7fc7pmIKh8GO/Ranbkbj9/O5O9dtPSjhjAv4xV16fVq3bq1chuKAnfu3FFuVVEUVxtTmBpIIXKz8c3fIcguYt8sM6f6uBF6B5k5OfCqZ4Mto1sgOjJcuVVFp06dEBwcjKCgIOWnWbNmGDJkCIKCglQWPSQi4mssem0UadmY+tc1AMCQFjXQ0c26gmtEr6q0rSpSU1OxaNEi9OzZEzKZDE+fPsWqVauQlhAH58YdERaXgi3nHuDE2q9hZ2cHHx8fLDt6B1dMmiMvYwOsgrdh8siv8c/RwypbVRTs/v48ExMTWFhYFDpORMRgh16br/bdQIwiA04WxpjTvW5FV4fKQPPmzbF7927MmjULCxYsgLOzM1asWIEhQ4YAAKRSKW7duoVNmzYhPj4eFhYWaN68OU6dOoW7wgaz/ILxw7E7qBpxH5BIMGPXdfx56SF0q1ph4ncbcf6P5WjcyAN2dnaYNGkSZsyYUbE3TESVEreLALeLeB3+vvYIX2y7CqmOBDs/bYXGNapXdJWoguXmCfRceRohj5LwnrstHidl4GpUInQkwMLeDTC4hfbtiUZEZYvbRdAbI1aRgbl7bgAAJnSoxUCHAABSHQm+6ekOADgcEourUYkAgKEtHBnoEFGZYrBD5UoIgWk7r0GRno0Gdmb4opNLRVeJ3iB21QtPd//9QiRiFOkVUBsi0lYMdqhcbTn/AKfC4mGgq4MfBjSCnpRNjv4TEZ9a6FiuELgfn1YBtSEibcUnD5Wbu3Ep8DkYCgCY1a0OaquxsSS9XSpyx3ciensw2NEy8+fPh0QiUfnY2toqzz9+/BgjRoyAXC6HsbEx3nvvPYSFhamd//bt2yGRSNC7d2+V405OToXKdbExRfSBX9DWxRLDWjmV0R2SNnnVvbmIiNTBqedayN3dHceOHVP+XLDAmhACvXv3hp6eHvbu3YuqVati+fLl6Ny5M27evAkTE5MS833w4AGmTp2Ktm3bFjoXGBioXLV279WH+HrTETzeMRcmdVqjrYvVW7/JJxWvonZ8J6K3h8Y9OydOnCiHalBZ0tXVha2trfJjZWUFAAgLC8P58+exevVqNG/eHG5ubli1ahVSUlKwbdu2EvPMzc3FkCFD8M0336BmzZqFzltZWcHW1ha5BlWxOOAx0u5ehG41GQwcGmDJoVsccEolqogd34no7aFxsPPee++hVq1aWLhwIaKiosqjTvSKwsLCIJfL4ezsjIEDByI8PBwAkJmZCQAqGydKpVLo6+vj9OnTJea5YMECWFlZYfTo0cVeczcuGaM3XkJebjZSb55AlYZdIJFIOOCUiIgqlMbBzqNHjzBp0iT4+fnB2dkZXbt2xZ9//omsrKzyqB9pqEWLFti8eTOOHDmCdevWITY2Fp6ennj69Cnq1KkDR0dHzJo1CwkJCcjKysLixYsRGxuLmJiYYvM8c+YM1q9fj3Xr1hV5PiM7F8v976Dbj6dw63Ey0u6cR15GCkzqdwLAAadERFSxNA52zM3NMXHiRFy5cgWXLl2Cm5sbJkyYAJlMhokTJ+LatWvlUU9SU7du3dC3b180aNAAnTt3xoEDBwAAmzZtgp6eHnbt2oU7d+7A3NwcxsbGOHHiBLp161bsxonJyckYOnQo1q1bB0tLy0Lnz4c/xfs/ncJP/4QhO1fg3TrWsHp0BsY1m0HX1IIDTomIqMK98nYRjx49wtq1a7F48WLo6uoiIyMDrVq1wpo1a+Du7l5W9SxX2r5dRJcuXVC7dm2sXr1aeUyhUCArKwtWVlZo0aIFmjVrhl9++aVQ2qCgIDRu3FglGMrL+98u1RIdyMasgV51GSyrGGB+z3qoXzULtWrVwv9t2YY6LTpxwCkREZWbct0uIjs7Gzt37sT7778PR0dHHDlyBCtXrsTjx48REREBBwcH9OvX76Ur/7zo6GgMHToUFhYWMDY2RqNGjXD58mXleSEE5s+fD7lcDiMjI3To0AEhISFlUrY2yMzMRGhoKGQymcpxMzMzWFlZISwsDJcuXUKvXr2KTF+nTh0EBwcjKCgIQUFBuHr1Kpq390IVZw/YjvgRulUtMbhFDfzzZXt80FCOjRs3wtraGh/378MBp0RE9EbQeOr5F198oZy5M3ToUCxduhT169dXnjcxMcHixYvh5OT0ypVLSEhA69at0bFjRxw6dAjW1ta4d+8eqlWrprxm6dKlWL58OTZu3AhXV1csXLgQXbp0we3bt2FqavrKdahspk6dih49eqBGjRqIi4vDwoULkZSUhOHDhwMA/vrrL1hZWaFGjRoIDg7GpEmT0Lt3b3h5eSnzGDZsGOzs7ODr6wtDQ0Pln2/UszTM2XMDIfE5yJMawt29Pnz6NEBzJ3MA+T0+GzZswPDhw6Gry1UNiIjozaDxE+nmzZv4+eef0bdvX+jr6xd5jVwux/Hjx1+5ckuWLIGDgwM2bNigPPZ8ECWEwIoVKzBnzhz06dMHQP7YFBsbG/zxxx8YN27cK9ehsnn48CEGDRqE+Ph4WFlZoWXLljh//jwcHR0BADExMZgyZQoeP34MmUyGYcOGYd68eSp5REZGQkfnv06/7Nw8rD8dgRXH7iAjOw86OhLUtDLBgYltoa/733XHjh1DZGQkRo0a9XpuloiISA2vPGanPNWrVw9du3bFw4cPERAQADs7O4wfPx5jx44FAISHh6NWrVq4cuUKGjdurEzXq1cvVKtWDZs2bSoy38zMTOU0bCD/nZ+Dg4PWjtl5GTGKdETEpyItKxffH7mNW7HJAICWNc3h82ED1LTi1g9ERFSx1B2zo3HPjq+vL2xsbAr96/23337DkydPMGPGDM1rW4zw8HCsXr0aU6ZMwezZs3Hx4kVMnDgRBgYGGDZsGGJjYwEANjY2KulsbGzw4MGDEu/hm2++KbN6apsdgZGY5ReMvOfC4GrGepjzfl181NQeEglXQyYiospD4wHKv/76K+rUqVPouLu7O9asWVMmlSqQl5eHJk2awMfHB40bN8a4ceMwduxYlVlFAAo9fIUQJT6QZ82aBYVCofxwcUQgL0/gRrQCSw/fwoxdqoGOBMAfY1qgXzMHBjpERFTpaNyzExsbW2hmD5C/XUBJC9O9DJlMhnr16qkcq1u3Lnbt2gUAyg0uX6xTXFxcod6e5xkYGMDAwKBM61oZxSjScSosHqfD4nHmbjyepha9MKQAoEjPeb2VIyIiKiMaBzsODg44c+YMnJ2dVY6fOXMGcrm8zCoGAK1bt8bt27dVjt25c0c52NbZ2Rm2trbw9/dXjtnJyspCQEAAlixZUqZ1qYwKxt04W5pAZmaElMwcnL/3FKfvxuNU2BPce5Kqcr2xvhSNHKrh3L2neH4gF1dAJiKiykzjYGfMmDHw9vZGdnY23n33XQDAP//8g+nTp+PLL78s08pNnjwZnp6e8PHxQf/+/XHx4kWsXbsWa9euBZD/+srb2xs+Pj5wcXGBi4sLfHx8YGxsjMGDB5dpXSqb58fdSAA4WZggKiENOc+9n9KRAA3sq6GdiyXa1LZE4xrVoa+rgx2BkZjtdwO5QnAFZCIiqvQ0no0lhMDMmTPx008/KffDMjQ0xIwZM/DVV1+VeQX379+PWbNmISwsDM7OzpgyZYpyNlZBfb755hv8+uuvSEhIQIsWLfDLL7+orP1TGm1bQTlGkQ7Pxf+iqD/ZGubGaONiiba1LeFZyxJmxnrF5nE/Po0rIBMR0RtL3ef3S089T0lJQWhoKIyMjODi4lKpx8BoW7Cz7uQ9LDp4q9DxHwc0Qq/GdhVQIyIiorJXblPPC1SpUgXNmzd/2eRUTq5FJWK5/51Cx6USCd6paV4BNSIiIqpYLxXsBAYG4q+//kJkZKTyVVYBPz+/MqkYae5uXDJGbLiI9Ow81LIyQUR8KvIEOO6GiIjeahoHO9u3b8ewYcPg5eUFf39/eHl5ISwsDLGxsfjwww/Lo46khujEdHy8/iIS0rLhYW+G38e2RHJGNsfdEBHRW0/jRQV9fHzwww8/YP/+/dDX18ePP/6I0NBQ9O/fHzVq1CiPOlY68+fPh0QiUfkUrAlUcL5OnTowMTFB9erV0blzZ1y4cKHEPDt06FAoT4lEgu7duyM+JRMf/98FRMc9Q97ZDbiydAisqpmib7dO0H0WzkCHiIjeahoHO/fu3UP37t0B5C/Ol5qaColEgsmTJyunhFP+itIxMTHKT3BwsPKcq6srVq5cieDgYJw+fRpOTk7w8vLCkydPis3Pz89PJb8bN25AKpWiR+8+GLHhIsLjU5H2zyoYPbmJ37duQXBwMLy8vNC5c2dER0e/jlsmIiJ6I2kc7JibmyM5OX9TSDs7O9y4cQMAkJiYiLS0tLKtXSWmq6sLW1tb5cfKykp5bvDgwejcuTNq1qwJd3d3LF++HElJSbh+/Xqx+Zmbm6vk5+/vD2NjY/inO+NGdBKq6wsk3jyF5d9/h3bt2qF27dqYP38+nJ2dC22vQURE9DbRONhp27Yt/P39AQD9+/fHpEmTMHbsWAwaNAidOnUq8wpWVmFhYZDL5XB2dsbAgQMRHh5e5HVZWVlYu3YtzMzM4OHhoXb+/7d+PeRNOuHyo3RUMdDFmiGNkJubC0NDQ5XrjIyMcPr06Ve6FyIiospM4wHKK1euREZGBoD8DTX19PRw+vRp9OnTB/PmzSvzClZGLVq0wObNm+Hq6orHjx9j4cKF8PT0REhICCwsLADkL5Y4cOBApKWlQSaTwd/fH5aWlmrlf/78BYTcuAHbj0eiqq4O/m94M7SoaYFWrVrh22+/Rd26dWFjY4Nt27bhwoULcHFxKc/bJSIieqNptKhgTk4Ofv/9d3Tt2lVlwG1lV96LCqampqJWrVqYPn06pkyZojwWExOD+Ph4rFu3Dv/++y8uXLgAa2vrEvMSQqB5twG4fvkiHMauwq9Dm6JzvfxNT+/du4dRo0bh5MmTkEqlaNKkCVxdXXHlyhXcvHmzzO+LiIioIqn7/NboNZauri4+++wzZGZmvnIF3yYmJiZo0KABwsLCVI7Vrl0bLVu2xPr166Grq4v169eXmtcPh4Jx5fgBVPHwwncfNVQGOgBQq1YtBAQEICUlBVFRUbh48SKys7MLbdpKRET0NtF4zE6LFi1w9erV8qiL1srMzERoaChkMlmx1wghSg0it5x/AJ9fNkDkZmP+5E/Rp4l9kdeZmJhAJpMhISEBR44cQa9evV6p/kRERJWZxmN2xo8fjy+//BIPHz5E06ZNYWJionK+YcOGZVa5ymrq1Kno0aMHatSogbi4OCxcuBBJSUkYPnw4UlNTsWjRIvTs2RMymQxPnz7FqlWr8PDhQ/Tr10+Zx7Bhw2BnZwdfX18AwL5rj/DV3htIuX4Ujdp0wcTuTQqVe+TIEQgh4Obmhrt372LatGlwc3PDyJEjX9u9ExERvWk0DnYGDBgAAJg4caLymEQigRACEokEubm5ZVe7Surhw4cYNGgQ4uPjYWVlhZYtW+L8+fNwdHRERkYGbt26hU2bNiE+Ph4WFhZo3rw5Tp06BXd3d2UekZGR0NHJ73g7cTsOU3YEIetpNDIf3sSS9T8UWa5CocCsWbPw8OFDmJubo2/fvli0aBH09Ire2ZyIiOhtoPGu5w8ePCjxvKOj4ytVqCK8qbuexyjScSQkFr4HQ5GZI9DDQ44fBzSCjo6koqtGRERU4cpt1/PKGMxURjsCIzHTLxgFoairTRUs6+fBQIeIiEhDGgc7mzdvLvH8sGHDXroylC9GkY5ZzwU6AHA3LgVPUzO5zxUREZGGNA52Jk2apPJzdnY20tLSoK+vD2NjYwY7ZSDscQryXni5mCeA+/FpDHaIiIg0pPHU84SEBJVPSkoKbt++jTZt2mDbtm3lUce3zu6rDwsdk0okcLI0roDaEBERVW4aBztFcXFxweLFiwv1+pDm/gyMwu6rjwAABcNzpBIJfPrUZ68OERHRS9D4NVZxpFIpHj16VFbZvZWuRSVi7p78XeSndHFFv2b2uB+fBidLYwY6REREL0njYGffvn0qPwshEBMTg5UrV6J169ZlVrG3TXxKJj7dehlZuXnoXNcGn3esDR0dCYMcIiKiV6RxsNO7d2+VnyUSCaysrPDuu+9i2bJlZVWvt0pObh4+/+MKYhQZqGlpguUDOMWciIiorGgc7OTl5ZVHPd5qvodu4Xz4M5joS7F2WFNUNeSKx0RERGWlTAYo08vbGxSN9acjAADL+nugtrVpBdeIiIhIu2gc7Hz00UdYvHhxoePfffedykaWVLqbj5IwY9d1AMD4DrXwXv3id0UnIiKil6NxsBMQEIDu3bsXOv7ee+/h5MmTZVKpt0FiWhbGbb2EjOw8tHO1wpdebhVdJSIiIq2kcbCTkpICfX39Qsf19PSQlJRUJpXSdrl5AhO3ByHqWToczI3w08BGkHJAMhERUbnQONipX78+duzYUej49u3bUa9evTKplLZb7n8bJ+88gaGeDn4d2gzVjAsHj0RERFQ2NJ6NNW/ePPTt2xf37t3Du+++CwD4559/sG3bNvz1119lXkFtc/hGDH45fg8AsKRvQ9STF78lPREREb06jYOdnj17Ys+ePfDx8cHOnTthZGSEhg0b4tixY2jfvn151FFrhD1Oxpd/XgMAjG7jjF6N7Cq4RkRERNpPIoQQpV+m3ZKSkmBmZgaFQoGqVcunpyUpIxu9V55BeHwqWtY0x9bRLaAr5cx/IiKil6Xu81vjp21gYCAuXLhQ6PiFCxdw6dIlTbN7K+TlCUzZcQ3h8amQmRli5eAmDHSIiIheE42fuBMmTEBUVFSh49HR0ZgwYUKZVErbrDx+F8dCH0NfVwdrhjaFZRWDiq4SERHRW0PjMTs3b95EkyZNCh1v3Lgxbt68WSaV0hYxinTsvhKN5f53AAALe9WHh0O1iq0UERHRW0bjYMfAwACPHz9GzZo1VY7HxMRAV1fj7LTWjsBIzPILRt7/RkS1cDZH/+YOFVspIiKit5DGr7G6dOmCWbNmQaFQKI8lJiZi9uzZ6NKlS5lWrrKKUaSrBDoAEHj/GWIU6RVXKSIioreUxl0xy5YtQ7t27eDo6IjGjRsDAIKCgmBjY4MtW7aUeQUro4j4VJVABwDyBHA/Pg0yM6OKqRQREdFbSuNgx87ODtevX8fvv/+Oa9euwcjICCNHjsSgQYOgp6dXHnWsdJwtTaAjgUrAI5VI4GRpXHGVIiIieku91CAbExMTfPLJJ2VdF60hMzOCb58GmO13A7lCQCqRwKdPffbqEBERVYCXHlF88+ZNREZGIisrS+V4z549X7lS2mBA8xpo52qF+/FpcLI0ZqBDRERUQTQOdsLDw/Hhhx8iODgYEokEBQswSyT5u3bn5uaWbQ0rMZmZEYMcIiKiCqbxbKxJkybB2dkZjx8/hrGxMUJCQnDy5Ek0a9YMJ06cKIcqEhEREb08jXt2zp07h3///RdWVlbQ0dGBjo4O2rRpA19fX0ycOBFXr14tj3oSERERvRSNe3Zyc3NRpUoVAIClpSUePXoEAHB0dMTt27fLtnZEREREr0jjnp369evj+vXrqFmzJlq0aIGlS5dCX18fa9euLbSqMhEREVFF0zjYmTt3LlJTUwEACxcuxAcffIC2bdvCwsICO3bsKPMKEhEREb0KiSiYTvUKnj17hurVqytnZFU2SUlJMDMzg0KhQNWqVSu6OkRERKQGdZ/fZbJzp7m5eVlkQ0RERFTmNB6gTERERFSZMNghIiIircZgh4iIiLSaxsHOyZMnkZOTU+h4Tk4OTp48WSaVIiIiIiorGgc7HTt2xLNnzwodVygU6NixY5lUioiIiKisaBzsCCGKnGL+9OlTmJiYlEmliIiIiMqK2lPP+/TpAyB/d/MRI0bAwMBAeS43NxfXr1+Hp6dn2deQiIiI6BWoHeyYmZkByO/ZMTU1hZGRkfKcvr4+WrZsibFjx5Z9DYmIiIhegdrBzoYNGwAATk5OmDp1Kl9ZERERUaWg8Zid6dOnq4zZefDgAVasWIGjR4+WacWIiIiIyoLGwU6vXr2wefNmAEBiYiLeeecdLFu2DL169cLq1avLvIJEREREr0LjYOfKlSto27YtAGDnzp2wtbXFgwcPsHnzZvz0009lXkEiIiKiV6FxsJOWlgZTU1MAwNGjR9GnTx/o6OigZcuWePDgQZlXkIiIiOhVaBzs1K5dG3v27EFUVBSOHDkCLy8vAEBcXFyJ26sTERERVQSNg52vvvoKU6dOhZOTE9555x20atUKQH4vT+PGjcu8gkRERESvQuNg56OPPkJkZCQuXbqEI0eOKI936tQJP/zwQ5lW7kW+vr6QSCTw9vZWHhNCYP78+ZDL5TAyMkKHDh0QEhJSrvUgIiKiyuOldj23tbWFqakp/P39kZ6eDgBo3rw56tSpU6aVe15gYCDWrl2Lhg0bqhxfunQpli9fjpUrVyIwMBC2trbo0qULkpOTy60uREREVHloHOw8ffoUnTp1gqurK95//33ExMQAAMaMGYMvv/yyzCsIACkpKRgyZAjWrVuH6tWrK48LIbBixQrMmTMHffr0Qf369bFp0yakpaXhjz/+KJe6EBERUeWicbAzefJk6OnpITIyEsbGxsrjAwYMwOHDh8u0cgUmTJiA7t27o3PnzirHIyIiEBsbqxwkDQAGBgZo3749zp49W2x+mZmZSEpKUvkQERGRdlJ7u4gCR48exZEjR2Bvb69y3MXFpVymnm/fvh1XrlxBYGBgoXOxsbEAABsbG5XjNjY2JdbF19cX33zzTdlWlIiIiN5IGvfspKamqvToFIiPj1fZCb0sREVFYdKkSdi6dSsMDQ2Lve757SuA/NdbLx573qxZs6BQKJSfqKioMqszERERvVk0DnbatWun3C4CyA808vLy8N1336Fjx45lWrnLly8jLi4OTZs2ha6uLnR1dREQEICffvoJurq6yh6dgh6eAnFxcYV6e55nYGCAqlWrqnyIiIhIO2n8Guu7775Dhw4dcOnSJWRlZWH69OkICQnBs2fPcObMmTKtXKdOnRAcHKxybOTIkahTpw5mzJiBmjVrwtbWFv7+/so1frKyshAQEIAlS5aUaV2IiIioctI42KlXrx6uX7+O1atXQyqVIjU1FX369MGECRMgk8nKtHKmpqaoX7++yjETExNYWFgoj3t7e8PHxwcuLi5wcXGBj48PjI2NMXjw4DKtCxEREVVOGgc7kZGRcHBwKHKAb2RkJGrUqFEmFVPX9OnTkZ6ejvHjxyMhIQEtWrTA0aNHlft3ERER0dtNIoQQmiSQSqWIiYmBtbW1yvGnT5/C2toaubm5ZVrB1yEpKQlmZmZQKBQcv0NERFRJqPv81niAcnEznVJSUkqcMUVERERUEdR+jTVlyhQA+bOv5s2bpzL9PDc3FxcuXECjRo3KvIJEREREr0LtYOfq1asA8nt2goODoa+vrzynr68PDw8PTJ06texrSERERPQK1A52jh8/DiB/6vePP/7IsS1ERERUKWg8G2vDhg3lUQ8iIiKicqHxAGUiIiKiyoTBDhEREWk1BjtERESk1RjsEBERkVZjsENERERajcEOERERaTUGO0RERKTVGOwQERGRVmOwQ0RERFqNwQ4RERFpNQY7REREpNUY7BAREZFWY7BDREREWo3BDhEREWk1BjtERESk1RjsEBERkVZjsENERERajcEOERERaTUGO0RERKTVGOwQERGRVmOwQ0RERFqNwQ4RERFpNQY7REREpNUY7BAREZFWY7BDREREWo3BDhEREWk1BjtERESk1RjsEBERkVZjsENERERajcEOERERaTUGO0RERKTVGOwQERGRVmOwQ0RERFqNwQ4RERFpNQY7REREpNUY7BAREZFWY7BDREREWo3BDhEREWk1BjtERESk1RjsEBERkVZjsENERERajcEOERERaTUGO0RERKTVGOwQERGRVmOwQ0RERFqNwQ4RERFpNQY7REREpNUY7BAREZFWY7BDREREWo3BDhEREWk1BjtERESk1RjsEBERkVZjsENERERajcEOERERaTUGO0RERKTVGOwQERGRVmOwQ0RERFqNwQ4RERFptTc62PH19UXz5s1hamoKa2tr9O7dG7dv31a5RgiB+fPnQy6Xw8jICB06dEBISEgF1ZiIiIjeNG90sBMQEIAJEybg/Pnz8Pf3R05ODry8vJCamqq8ZunSpVi+fDlWrlyJwMBA2NraokuXLkhOTq7AmhMREdGbQiKEEBVdCXU9efIE1tbWCAgIQLt27SCEgFwuh7e3N2bMmAEAyMzMhI2NDZYsWYJx48aplW9SUhLMzMygUChQtWrV8rwFIiIiKiPqPr/f6J6dFykUCgCAubk5ACAiIgKxsbHw8vJSXmNgYID27dvj7NmzxeaTmZmJpKQklQ8RERFpp0oT7AghMGXKFLRp0wb169cHAMTGxgIAbGxsVK61sbFRniuKr68vzMzMlB8HB4fyqzgRERFVqEoT7Hz++ee4fv06tm3bVuicRCJR+VkIUejY82bNmgWFQqH8REVFlXl9iYiI6M2gW9EVUMcXX3yBffv24eTJk7C3t1cet7W1BZDfwyOTyZTH4+LiCvX2PM/AwAAGBgblV2EiIiJ6Y7zRPTtCCHz++efw8/PDv//+C2dnZ5Xzzs7OsLW1hb+/v/JYVlYWAgIC4Onp+bqrS0RERG+gN7pnZ8KECfjjjz+wd+9emJqaKsfhmJmZwcjICBKJBN7e3vDx8YGLiwtcXFzg4+MDY2NjDB48uIJrT0RERG+CNzrYWb16NQCgQ4cOKsc3bNiAESNGAACmT5+O9PR0jB8/HgkJCWjRogWOHj0KU1PT11xbIiIiehNVqnV2ygvX2SEiIqp8tHKdHSIiIiJNMdghIiIircZgh4iIiLQagx0iIiLSagx2iIiISKsx2CEiIiKtxmCHiIiItBqDHSIiItJqDHaIiIhIqzHYISIiIq3GYIeIiIi0GoMdIiIi0moMdoiIiEirMdghIiIircZgh4iIiLQagx0iIiLSagx2iIiISKsx2CEiIiKtxmCHiIiItBqDHSIiItJqDHaIiIhIqzHYISIiIq3GYIeIiIi0GoMdIiIi0moMdoiIiEirMdghIiIircZgh4iIiLQagx0iIiLSagx2iIiISKsx2CEiIiKtxmCHiIiItBqDHSIiItJqDHaIiIhIqzHYISIiIq3GYIeIiIi0GoMdIiIi0moMdoiIiEirMdghIiIircZgh4iIiLQagx0iIiLSagx2iIiISKsx2CEiIiKtxmCHiIiItBqDHSIiItJqDHaIiIhIqzHYISIiIq3GYIeIiIi0GoMdIiIi0moMdoiIiEirMdghIiIircZgh4iIiLQagx0iIiLSagx2iIiISKsx2CEiIiKtxmCHiIiItBqDHSIiItJqDHaIiIhIqzHYISIiIq3GYIeIiIi0GoMdIiIi0moMdoiIiEirMdghIiIircZgh4iIiLSa1gQ7q1atgrOzMwwNDdG0aVOcOnWqoqtEREREbwCtCHZ27NgBb29vzJkzB1evXkXbtm3RrVs3REZGVnTViIiIqIJJhBCioivxqlq0aIEmTZpg9erVymN169ZF79694evrW2r6pKQkmJmZQaFQoGrVquVZVSIiIioj6j6/dV9jncpFVlYWLl++jJkzZ6oc9/LywtmzZ4tMk5mZiczMTOXPCoUCQP4vjYiIiCqHgud2af02lT7YiY+PR25uLmxsbFSO29jYIDY2tsg0vr6++Oabbwodd3BwKJc6EhERUflJTk6GmZlZsecrfbBTQCKRqPwshCh0rMCsWbMwZcoU5c95eXl49uwZLCwsik3zMpKSkuDg4ICoqKiXej1WkelZNsuuLOlZ9ttV9qumZ9mVr+ySCCGQnJwMuVxe4nWVPtixtLSEVCot1IsTFxdXqLengIGBAQwMDFSOVatWrbyqiKpVq77SH3BFpmfZLLuypGfZb1fZr5qeZVe+sotTUo9OgUo/G0tfXx9NmzaFv7+/ynF/f394enpWUK2IiIjoTVHpe3YAYMqUKfj444/RrFkztGrVCmvXrkVkZCQ+/fTTiq4aERERVTCtCHYGDBiAp0+fYsGCBYiJiUH9+vVx8OBBODo6Vmi9DAwM8PXXXxd6ZVYZ0rNsll1Z0rPst6vsV03Psitf2WVBK9bZISIiIipOpR+zQ0RERFQSBjtERESk1RjsEBERkVZjsENERERajcFOOVq1ahWcnZ1haGiIpk2b4tSpU2qlO3nyJHr06AG5XA6JRII9e/aoXaavry+aN28OU1NTWFtbo3fv3rh9+7ba6VevXo2GDRsqF39q1aoVDh06pHb6F+sikUjg7e2t1vXz58+HRCJR+dja2qpdXnR0NIYOHQoLCwsYGxujUaNGuHz5slppnZycCpUtkUgwYcKEUtPm5ORg7ty5cHZ2hpGREWrWrIkFCxYgLy9P7bonJyfD29sbjo6OMDIygqenJwIDA4u8trT2IYTA/PnzIZfLYWRkhA4dOiAkJESttH5+fujatSssLS0hkUgQFBSkdtnZ2dmYMWMGGjRoABMTE8jlcgwbNgyPHj1Sq+z58+ejTp06MDExQfXq1dG5c2dcuHBB7ft+3rhx4yCRSLBixQq10o4YMaLQn33Lli01Kjs0NBQ9e/aEmZkZTE1N0bJlS0RGRpaatqh2J5FI8N1336lVdkpKCj7//HPY29vDyMgIdevWVdkUubT0jx8/xogRIyCXy2FsbIz33nsPYWFhANT7PimuvamTtqT2Vlr6ktqbOmWX1N40/R59vr2pk7ak9qZu2UW1txkzZpSatqT2pk7ZxbU3ddKW1NbKG4OdcrJjxw54e3tjzpw5uHr1Ktq2bYtu3bohMjKy1LSpqanw8PDAypUrNS43ICAAEyZMwPnz5+Hv74+cnBx4eXkhNTVVrfT29vZYvHgxLl26hEuXLuHdd99Fr169lA9LdQUGBmLt2rVo2LChRunc3d0RExOj/AQHB6uVLiEhAa1bt4aenh4OHTqEmzdvYtmyZWqvjB0YGKhSbsEilf369Ss17ZIlS7BmzRqsXLkSoaGhWLp0Kb777jv8/PPPapUNAGPGjIG/vz+2bNmC4OBgeHl5oXPnzoiOji50bWntY+nSpVi+fDlWrlyJwMBA2NraokuXLkhOTi41bWpqKlq3bo3FixcXe7649Glpabhy5QrmzZuHK1euwM/PD3fu3EHPnj3VqrerqytWrlyJ4OBgnD59Gk5OTvDy8sKTJ0/USl9gz549uHDhgsry8eqkfe+991TawMGDB9VOf+/ePbRp0wZ16tTBiRMncO3aNcybNw+Ghoalpn2+zJiYGPz222+QSCTo27evWmVPnjwZhw8fxtatWxEaGorJkyfjiy++wN69e0tNL4RA7969ER4ejr179+Lq1atwdHRE586dkZqaqtb3SXHt7Z9//ik1bUntrbSyS2pv6tS7pPamyffoi+1N3bTFtTd10hfX3gIDA0tNW1J7U6fs4trbX3/9VWLa0tpauRNULt555x3x6aefqhyrU6eOmDlzpkb5ABC7d+9+6XrExcUJACIgIOCl86hevbr4v//7P7WvT05OFi4uLsLf31+0b99eTJo0Sa10X3/9tfDw8HipOs6YMUO0adPmpdIWZdKkSaJWrVoiLy+v1Gu7d+8uRo0apXKsT58+YujQoWqVlZaWJqRSqdi/f7/KcQ8PDzFnzpwS077YPvLy8oStra1YvHix8lhGRoYwMzMTa9asKTHt8yIiIgQAcfXqVbXLLsrFixcFAPHgwQON0yoUCgFAHDt2TO2yHz58KOzs7MSNGzeEo6Oj+OGHH9RKO3z4cNGrV68S61NS+gEDBqj1563Offfq1Uu8++67aqd3d3cXCxYsUDnWpEkTMXfu3FLT3759WwAQN27cUB7LyckR5ubmYt26dYXSv/h9okl7K+m7SJ32ps53WXHtTZ20JbW34tKr096KSqtJeysqvbrtTZ37Lqm9FZVe3fb2YlpN21pZY89OOcjKysLly5fh5eWlctzLywtnz559rXVRKBQAAHNzc43T5ubmYvv27UhNTUWrVq3UTjdhwgR0794dnTt31rjMsLAwyOVyODs7Y+DAgQgPD1cr3b59+9CsWTP069cP1tbWaNy4MdatW6dx+UD+n9/WrVsxatQotTaGbdOmDf755x/cuXMHAHDt2jWcPn0a77//vlrl5eTkIDc3F4aGhirHjYyMcPr0aY3qHhERgdjYWJW2Z2BggPbt27/2tgfktz+JRKLx3nNZWVlYu3YtzMzM4OHhoVaavLw8fPzxx5g2bRrc3d01ruuJEydgbW0NV1dXjB07FnFxcWqXe+DAAbi6uqJr166wtrZGixYtNHr9XODx48c4cOAARo8erXaaNm3aYN++fYiOjoYQAsePH8edO3fQtWvXUtNmZmYCgErbk0ql0NfXL7Ltvfh9okl7e5XvInXTF9feSktbWnsrKr267a24stVtby+m16S9lXbfpbW3otKr295eTKtpWytz5R5OvYWio6MFAHHmzBmV44sWLRKurq4a5YVX6NnJy8sTPXr00LjH4/r168LExERIpVJhZmYmDhw4oHbabdu2ifr164v09HQhhNCoZ+fgwYNi586d4vr168peIRsbGxEfH19qWgMDA2FgYCBmzZolrly5ItasWSMMDQ3Fpk2b1K57gR07dgipVCqio6PVuj4vL0/MnDlTSCQSoaurKyQSifDx8dGozFatWon27duL6OhokZOTI7Zs2SIkEkmp7eXF9nHmzBkBoFDdx44dK7y8vEpM+7yy6NlJT08XTZs2FUOGDFE77d9//y1MTEyERCIRcrlcXLx4Ue2yfXx8RJcuXZS9cZr07Gzfvl3s379fBAcHi3379gkPDw/h7u4uMjIySk0fExMjAAhjY2OxfPlycfXqVeHr6yskEok4ceKEWvddYMmSJaJ69erKvz/q1D0zM1MMGzZMABC6urpCX19fbN68Wa30WVlZwtHRUfTr1088e/ZMZGZmCl9fXwGgUHsp6vtE3fZW2ndRae1Nne+y4tpbSWnVaW/FpVenvRWXVt32VlR6ddubOr+zktpbcenVaW9FpdWkrZUHBjvloCDYOXv2rMrxhQsXCjc3N43yepVgZ/z48cLR0VFERUVplC4zM1OEhYWJwMBAMXPmTGFpaSlCQkJKTRcZGSmsra1FUFCQ8pgmwc6LUlJShI2NjVi2bFmp1+rp6YlWrVqpHPviiy9Ey5YtNS7Xy8tLfPDBB2pfv23bNmFvby+2bdsmrl+/LjZv3izMzc3Fxo0b1c7j7t27ol27dgKAkEqlonnz5mLIkCGibt26JaYrLth59OiRynVjxowRXbt2LTHt81412MnKyhK9evUSjRs3FgqFQu20KSkpIiwsTJw7d06MGjVKODk5icePH5ea/tKlS8LGxkbloatJsPOiR48eCT09PbFr165S0xf8fR80aJDKdT169BADBw7UqGw3Nzfx+eefF3u+qPTfffedcHV1Ffv27RPXrl0TP//8s6hSpYrw9/dXK/2lS5eEh4eHsu117dpVdOvWTXTr1k3luqK+T9Rtb6V9F5XW3kpLX1J7KymtOu2tqPTqtjd1v4OLa29FpVe3valTdkntrbj06rS34tKq29bKA4OdcpCZmSmkUqnw8/NTOT5x4kTRrl07jfJ62WDn888/F/b29iI8PFzjtC/q1KmT+OSTT0q9bvfu3cpGXPABICQSiZBKpSInJ0fjsjt37lxo7FNRatSoIUaPHq1ybNWqVUIul2tU3v3794WOjo7Ys2eP2mns7e3FypUrVY59++23Gge2QuR/+RY8OPr37y/ef//9Eq9/sX3cu3dPABBXrlxRua5nz55i2LBhJaZ93qsEO1lZWaJ3796iYcOGxfbKqduua9euXWQv2Yvpf/jhB2U7e77t6ejoCEdHx5cu+/mxKMWlz8zMFLq6uuLbb79VuW769OnC09NT7bJPnjwpAKj8Y6G0stPS0oSenl6h8V6jR48uFNyWVn5iYqKIi4sTQuSPORw/frzyXHHfJ+q0N3W+i0pqb6WlL6m9afo9+GJ7Ky69Ou3tZcp+vr0Vl16d9qZO2SW1t+LSq9Pe1Cm7pLZWXjhmpxzo6+ujadOmyhk9Bfz9/eHp6VmuZQsh8Pnnn8PPzw///vsvnJ2dyyTPgvetJenUqROCg4MRFBSk/DRr1gxDhgxBUFAQpFKpRuVmZmYiNDQUMpms1Gtbt25daJrjnTt3NN4MdsOGDbC2tkb37t3VTpOWlgYdHdW/SlKpVKOp5wVMTEwgk8mQkJCAI0eOoFevXhqld3Z2hq2trUrby8rKQkBAQLm3PSB/OnD//v0RFhaGY8eOwcLC4pXyU7ftffzxx7h+/bpK25PL5Zg2bRqOHDmicblPnz5FVFSUWm1PX18fzZs3f+X2t379ejRt2lTtMUpA/u87Ozu7TNqfmZkZrKysEBYWhkuXLqFXr16lfp+U1N5atWr1St9F6nyXFdfeXvZ7sKC9lZa+pPZ2+PBhjct+vr2VVnZJ7a1GjRpql11Ueyut7JLaW25urtplF9XWyl25h1Nvqe3btws9PT2xfv16cfPmTeHt7S1MTEzE/fv3S02bnJwsrl69Kq5evSoAKN/LvjjDoCifffaZMDMzEydOnBAxMTHKT1pamlr1njVrljh58qSIiIgQ169fF7NnzxY6Ojri6NGjaqV/kSavsb788ktx4sQJER4eLs6fPy8++OADYWpqqtbv7OLFi0JXV1csWrRIhIWFid9//10YGxuLrVu3ql3X3NxcUaNGDTFjxgy10wiRP7PCzs5O7N+/X0RERAg/Pz9haWkppk+frnYehw8fFocOHRLh4eHi6NGjwsPDQ7zzzjsiKyur0LWltY/FixcLMzMz4efnJ4KDg8WgQYOETCYTSUlJpaZ9+vSpuHr1qjhw4IAAILZv3y6uXr0qYmJiSi07Oztb9OzZU9jb24ugoCCV9peZmVli2pSUFDFr1ixx7tw5cf/+fXH58mUxevRoYWBgoJy9oenfi+dfK5SUNjk5WXz55Zfi7NmzIiIiQhw/fly0atVK2NnZiaSkJLXK9vPzE3p6emLt2rUiLCxM/Pzzz0IqlYpTp06pVW+FQiGMjY3F6tWrNf7zbt++vXB3dxfHjx8X4eHhYsOGDcLQ0FCsWrVKrfR//vmnOH78uLh3757Ys2ePcHR0FH369BFCqPd9Ulx7Gz16dKlpS2pvpZVdUnv75JNPSkxbWnt7me/RgvZWWtrS2ps6ZRfX3nr37q1WvYtrb+qUXVx7a9u2balpS2pr5Y3BTjn65ZdfhKOjo9DX1xdNmjRRe/r38ePHBYBCn+HDh5eatqh0AMSGDRvUKnvUqFHKOltZWYlOnTq9dKAjhGbBzoABA4RMJhN6enpCLpeLPn36qDVWqMDff/8t6tevLwwMDESdOnXE2rVrNarrkSNHBABx+/ZtjdIlJSWJSZMmiRo1aghDQ0NRs2ZNMWfOHJGZmal2Hjt27BA1a9YU+vr6wtbWVkyYMEEkJiYWeW1p7SMvL098/fXXwtbWVhgYGIh27dqJ4OBgtdJu2LChyPNff/11qekLXkUU9Tl+/HiJadPT08WHH34o5HK50NfXFzKZTPTs2VNlwKimfy+eD3ZKSpuWlia8vLyElZWV0NPTEzVq1BDDhw8XkZGRGpW9fv16Ubt2bWFoaCg8PDyUr0LVSfvrr78KIyOjIv/MS0sfExMjRowYIeRyuTA0NBRubm5i2bJlyoGzpaX/8ccfhb29vfLe586dq2y76nyfFNfe1ElbUnsrLX1J7a20tKW1t5f5Hi1ob6WlLa29qVt2Ue1N3bTFtTd10hfX3tRJW1JbK2+S/90gERERkVbimB0iIiLSagx2iIiISKsx2CEiIiKtxmCHiIiItBqDHSIiItJqDHaIiIhIqzHYISIiIq3GYIeIqAgnTpyARCJBYmJiRVeFiF4Rgx0iIiLSagx2iIiISKsx2CGiN5IQAkuXLkXNmjVhZGQEDw8P7Ny5E8B/r5gOHDgADw8PGBoaokWLFggODlbJY9euXXB3d4eBgQGcnJywbNkylfOZmZmYPn06HBwcYGBgABcXF6xfv17lmsuXL6NZs2YwNjaGp6dnod2miejNx2CHiN5Ic+fOxYYNG7B69WqEhIRg8uTJGDp0KAICApTXTJs2Dd9//z0CAwNhbW2Nnj17Ijs7G0B+kNK/f38MHDgQwcHBmD9/PubNm4eNGzcq0w8bNgzbt2/HTz/9hNDQUKxZswZVqlRRqcecOXOwbNkyXLp0Cbq6uhg1atRruX8iKjvcCJSI3jipqamwtLTEv//+i1atWimPjxkzBmlpafjkk0/QsWNHbN++HQMGDAAAPHv2DPb29ti4cSP69++PIUOG4MmTJzh69Kgy/fTp03HgwAGEhITgzp07cHNzg7+/Pzp37lyoDidOnEDHjh1x7NgxdOrUCQBw8OBBdO/eHenp6TA0NCzn3wIRlRX27BDRG+fmzZvIyMhAly5dUKVKFeVn8+bNuHfvnvK65wMhc3NzuLm5ITQ0FAAQGhqK1q1bq+TbunVrhIWFITc3F0FBQZBKpWjfvn2JdWnYsKHy/2UyGQAgLi7ule+RiF4f3YquABHRi/Ly8gAABw4cgJ2dnco5AwMDlYDnRRKJBED+mJ+C/y/wfEe2kZGRWnXR09MrlHdB/YiocmDPDhG9cerVqwcDAwNERkaidu3aKh8HBwfldefPn1f+f0JCAu7cuYM6deoo8zh9+rRKvmfPnoWrqyukUikaNGiAvLw8lTFARKSd2LNDRG8cU1NTTJ06FZMnT0ZeXh7atGmDpKQknD17FlWqVIGjoyMAYMGCBbCwsICNjQ3mzJkDS0tL9O7dGwDw5Zdfonnz5vj2228xYMAAnDt3DitXrsSqVasAAE5OThg+fDhGjRqFn376CR4eHnjw4AHi4uLQv3//irp1IioHDHaI6I307bffwtraGr6+vggPD0e1atXQpEkTzJ49W/kaafHixZg0aRLCwsLg4eGBffv2QV9fHwDQpEkT/Pnnn/jqq6/w7bffQiaTYcGCBRgxYoSyjNWrV2P27NkYP348nj59iho1amD27NkVcbtEVI44G4uIKp2CmVIJCQmoVq1aRVeHiN5wHLNDREREWo3BDhEREWk1vsYiIiIircaeHSIiItJqDHaIiIhIqzHYISIiIq3GYIeIiIi0GoMdIiIi0moMdoiIiEirMdghIiIircZgh4iIiLQagx0iIiLSav8Pap8fE7Joju4AAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plt.plot(np.arange(len(epochs_x)), epochs_acc, marker = '.')\n", + "plt.xlabel('epoch')\n", + "plt.ylabel('test accuracy')\n", + "plt.ylim(0, 100)\n", + "plt.xticks(np.arange(len(epochs_x)))\n", + "for i, txt in enumerate(epochs_acc):\n", + " if i%2 == 0:\n", + " pass\n", + " else:\n", + " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "with open('baseline_exp_set_B1_training_metrics.npy', 'wb') as f:\n", + " np.save(f, np.array(epochs_x))\n", + " np.save(f, np.array(epochs_y))\n", + " np.save(f, np.array(epochs_acc))" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "speck-rescnn", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tests/test_nonsequential/exp_set_B1/non-sequential-SCNN-example_3.ipynb b/tests/test_nonsequential/exp_set_B1/non-sequential-SCNN-example_3.ipynb new file mode 100644 index 00000000..fca63b7d --- /dev/null +++ b/tests/test_nonsequential/exp_set_B1/non-sequential-SCNN-example_3.ipynb @@ -0,0 +1,1509 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import torch, random\n", + "import torch.nn as nn\n", + "import sinabs.layers as sl\n", + "from tqdm.notebook import tqdm\n", + "\n", + "from tonic.datasets.dvsgesture import DVSGesture\n", + "from tonic.transforms import ToFrame\n", + "from torch.utils.data import DataLoader\n", + "from torch.nn import CrossEntropyLoss\n", + "from torch.optim import Adam\n", + "\n", + "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "torch.backends.cudnn.deterministic = True\n", + "random.seed(1)\n", + "torch.manual_seed(1)\n", + "torch.cuda.manual_seed(1)\n", + "np.random.seed(1)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Loading Data" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "batch_size = 3\n", + "num_workers = 1\n", + "epochs = 30\n", + "lr = 5e-4" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "root_dir = \"./DVSGESTURE\"\n", + "_ = DVSGesture(save_to=root_dir, train=True)\n", + "_ = DVSGesture(save_to=root_dir, train=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "n_time_steps = 50\n", + "to_raster = ToFrame(sensor_size=DVSGesture.sensor_size, n_time_bins=n_time_steps)\n", + "\n", + "snn_train_dataset = DVSGesture(save_to=root_dir, train=True, transform=to_raster)\n", + "snn_test_dataset = DVSGesture(save_to=root_dir, train=False, transform=to_raster)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The transformed array is in shape [Time-Step, Channel, Height, Width] --> (50, 2, 128, 128)\n" + ] + } + ], + "source": [ + "sample_data, label = snn_train_dataset[0]\n", + "print(f\"The transformed array is in shape [Time-Step, Channel, Height, Width] --> {sample_data.shape}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "snn_train_dataloader = DataLoader(snn_train_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True)\n", + "snn_test_dataloader = DataLoader(snn_test_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=False)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Network Module\n", + "\n", + "We need to define a `nn.Module` implementing the network we want the chip to reproduce." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "device: NVIDIA RTX A4000\n" + ] + } + ], + "source": [ + "if torch.cuda.is_available():\n", + " device = torch.device('cuda:0')\n", + " print('device: ', torch.cuda.get_device_name(0))\n", + "else:\n", + " device = torch.device('cpu')" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "class SNN(nn.Module):\n", + " def __init__(self) -> None:\n", + " super().__init__()\n", + "\n", + " self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False)\n", + " self.iaf1 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + " self.pool1 = nn.AvgPool2d(2,2)\n", + " self.pool1a = nn.AvgPool2d(6,6)\n", + "\n", + " self.conv2 = nn.Conv2d(10, 10, 2, 1, bias=False)\n", + " self.iaf2 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + " self.pool2 = nn.AvgPool2d(3,3)\n", + "\n", + " self.conv3 = nn.Conv2d(10, 10, 3, 1, bias=False)\n", + " self.iaf3 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + " self.pool3 = nn.AvgPool2d(2,2)\n", + "\n", + " self.flat = nn.Flatten()\n", + "\n", + " self.fc1 = nn.Linear(810, 100, bias=False)\n", + " self.iaf4 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + "\n", + " self.fc2 = nn.Linear(100, 100, bias=False)\n", + " self.iaf5 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + "\n", + " self.fc3 = nn.Linear(100, 100, bias=False)\n", + " self.iaf6 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + "\n", + " self.fc4 = nn.Linear(100, 11, bias=False)\n", + " self.iaf7 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + "\n", + " self.merge_fc = sl.Merge()\n", + " self.merge_conv = sl.Merge()\n", + "\n", + " def detach_neuron_states(self):\n", + " for name, layer in self.named_modules():\n", + " if name != '':\n", + " if isinstance(layer, sl.StatefulLayer):\n", + " for name, buffer in layer.named_buffers():\n", + " buffer.detach_()\n", + "\n", + " def init_weights(self):\n", + " for name, layer in self.named_modules():\n", + " if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear):\n", + " nn.init.xavier_normal_(layer.weight.data)\n", + "\n", + " def forward(self, x):\n", + " \n", + " con1_out = self.conv1(x)\n", + " iaf1_out = self.iaf1(con1_out)\n", + " pool1_out = self.pool1(iaf1_out)\n", + " pool1a_out = self.pool1a(iaf1_out)\n", + "\n", + " conv2_out = self.conv2(pool1_out)\n", + " iaf2_out = self.iaf2(conv2_out)\n", + " pool2_out = self.pool2(iaf2_out)\n", + "\n", + " merged_conv_out = self.merge_conv(pool1a_out, pool2_out)\n", + "\n", + " conv3_out = self.conv3(merged_conv_out)\n", + " iaf3_out = self.iaf3(conv3_out)\n", + " pool3_out = self.pool3(iaf3_out)\n", + "\n", + " flat_out = self.flat(pool3_out)\n", + " \n", + " fc1_out = self.fc1(flat_out)\n", + " iaf4_out = self.iaf4(fc1_out)\n", + "\n", + " fc2_out = self.fc2(iaf4_out)\n", + " iaf5_out = self.iaf5(fc2_out)\n", + "\n", + " fc3_out = self.fc3(iaf5_out)\n", + " iaf6_out = self.iaf6(fc3_out)\n", + "\n", + " merge_fc_out = self.merge_fc(iaf4_out, iaf6_out)\n", + "\n", + " fc4_out = self.fc4(merge_fc_out)\n", + " iaf7_out = self.iaf7(fc4_out)\n", + "\n", + " return iaf7_out" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "snn = SNN().to(device)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "snn.init_weights()" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "optimizer = Adam(snn.parameters(), lr=lr, betas=(0.9, 0.999), eps=1e-8)\n", + "loss_fn = CrossEntropyLoss()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Define train and test" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "def train(dataloader, model, loss_fn, optimizer, epochs, test_func, dataloader_test):\n", + " epochs_y = []\n", + " epochs_x = []\n", + " epochs_acc = []\n", + " model.train()\n", + "\n", + " for e in range(epochs):\n", + " losses = []\n", + " batches = []\n", + " batch_count = 0\n", + " train_p_bar = tqdm(snn_train_dataloader)\n", + "\n", + " for X, y in train_p_bar:\n", + " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", + " X = X.reshape(-1, 2, 128, 128).to(dtype=torch.float, device=device)\n", + " y = y.to(dtype=torch.long, device=device)\n", + "\n", + " # forward\n", + " pred = model(X)\n", + "\n", + " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", + " pred = pred.reshape(batch_size, n_time_steps, -1)\n", + "\n", + " # accumulate all time-steps output for final prediction\n", + " pred = pred.sum(dim = 1)\n", + " loss = loss_fn(pred, y)\n", + "\n", + " # gradient update\n", + " optimizer.zero_grad()\n", + " loss.backward()\n", + " optimizer.step()\n", + "\n", + " # detach the neuron states and activations from current computation graph(necessary)\n", + " model.detach_neuron_states()\n", + "\n", + " train_p_bar.set_description(f\"Epoch {e} - BPTT Training Loss: {round(loss.item(), 4)}\")\n", + "\n", + " batch_count += 1\n", + " losses.append(loss.item())\n", + " batches.append(batch_count)\n", + "\n", + " epochs_y.append(losses)\n", + " epochs_x.append(batches)\n", + "\n", + " acc = test_func(dataloader_test, model)\n", + " print(f'Epoch {e} accuracy: {acc}')\n", + " epochs_acc.append(acc)\n", + "\n", + " return epochs_x, epochs_y, epochs_acc\n" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "def test(dataloader, model):\n", + " correct_predictions = []\n", + " with torch.no_grad():\n", + " test_p_bar = tqdm(dataloader)\n", + " for X, y in test_p_bar:\n", + " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", + " X = X.reshape(-1, 2, 128, 128).to(dtype=torch.float, device=device)\n", + " y = y.to(dtype=torch.long, device=device)\n", + "\n", + " # forward\n", + " output = model(X)\n", + "\n", + " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", + " output = output.reshape(batch_size, n_time_steps, -1)\n", + "\n", + " # accumulate all time-steps output for final prediction\n", + " output = output.sum(dim=1)\n", + "\n", + " # calculate accuracy\n", + " pred = output.argmax(dim=1, keepdim=True)\n", + "\n", + " # compute the total correct predictions\n", + " correct_predictions.append(pred.eq(y.view_as(pred)))\n", + "\n", + " test_p_bar.set_description(f\"Testing Model...\")\n", + " \n", + " correct_predictions = torch.cat(correct_predictions)\n", + " return correct_predictions.sum().item()/(len(correct_predictions))*100" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Training loop (HPO)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "12d134e3b89e41888c9c47892b8e6491", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/359 [00:00" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "y_avg = []\n", + "for y in epochs_y:\n", + " y_avg.append(np.mean(y))\n", + "\n", + "plt.plot(np.arange(len(epochs_x)), y_avg, marker = '.')\n", + "plt.xlabel('epoch')\n", + "plt.ylabel('average loss')\n", + "plt.ylim(0,)\n", + "plt.xticks(np.arange(len(epochs_x)))\n", + "for i, txt in enumerate(y_avg):\n", + " if i%5 == 0:\n", + " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", + " else:\n", + " pass\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAG2CAYAAACZEEfAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABq60lEQVR4nO3dd1hT1+MG8DfsHUW2TAfixK3gqqO46q6jtVWr1bZi62idraPVFrWt7dfROn5urauOOloXrqqooOBEBERBZYhI2DPn9weaGlmJBIH4fp4nT8vNPfecQMx9c+6550iEEAJEREREWkqnohtAREREVJ4YdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirVWjYOXPmDPr06QMHBwdIJBLs27dP6XkhBObMmQN7e3sYGxujW7duCA8PV9onKSkJw4cPh4WFBapVq4YxY8YgLS3tNb4KIiIiqswqNOykp6fD09MTK1asKPL5xYsXY+nSpVi5ciUuXrwIU1NTdO/eHVlZWYp9hg8fjps3b+LYsWM4ePAgzpw5g3Hjxr2ul0BERESVnKSyLAQqkUiwd+9e9O/fH0BBr46DgwO+/PJLfPXVVwAAmUwGW1tbbNiwAcOGDUNoaCgaNGiAwMBAtGzZEgBw+PBh9OrVCw8ePICDg0NFvRwiIiKqJPQqugHFiYqKQlxcHLp166bYJpVK0aZNGwQEBGDYsGEICAhAtWrVFEEHALp16wYdHR1cvHgRAwYMKPLY2dnZyM7OVvwsl8uRlJSEGjVqQCKRlN+LIiIiIo0RQiA1NRUODg7Q0Sn+YlWlDTtxcXEAAFtbW6Xttra2iufi4uJgY2Oj9Lyenh4sLS0V+xTFz88P3377rYZbTERERBUhJiYGjo6OxT5facNOeZo5cyamTJmi+Fkmk8HZ2RkxMTGwsLCowJYRERGRqlJSUuDk5ARzc/MS96u0YcfOzg4AEB8fD3t7e8X2+Ph4NG3aVLFPQkKCUrm8vDwkJSUpyhfF0NAQhoaGhbZbWFgw7BAREVUxpQ1BqbTz7Li5ucHOzg7+/v6KbSkpKbh48SK8vLwAAF5eXkhOTsbly5cV+5w4cQJyuRxt2rR57W0mIiKiyqdCe3bS0tIQERGh+DkqKgohISGwtLSEs7MzJk2ahAULFqBu3bpwc3PD7Nmz4eDgoLhjq379+ujRowfGjh2LlStXIjc3FxMmTMCwYcN4JxYREREBqOCwExQUhM6dOyt+fj6OZuTIkdiwYQOmTZuG9PR0jBs3DsnJyWjfvj0OHz4MIyMjRZmtW7diwoQJ6Nq1K3R0dDBo0CAsXbr0tb8WIiIiqpwqzTw7FSklJQVSqRQymYxjdoiIiKoIVc/flXbMDhEREZEmMOwQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEqQn5+P2bNnw83NDcbGxqhduzbmz58PIYRin1GjRkEikSg9evToUeJxU1NTMWnSJLi4uMDY2Bje3t4IDAxU2ictLQ0TJkyAo6MjjI2N0aBBA6xcubJcXqc206voBhAREVVmixYtwu+//46NGzeiYcOGCAoKwkcffQSpVIovvvhCsV+PHj2wfv16xc+GhoYlHvfjjz/GjRs3sHnzZjg4OGDLli3o1q0bbt26hZo1awIApkyZghMnTmDLli1wdXXF0aNHMX78eDg4OKBv377l84K1EHt2iIiISnD+/Hn069cPvXv3hqurK9599134+Pjg0qVLSvsZGhrCzs5O8ahevXqxx8zMzMTu3buxePFidOzYEXXq1MG8efNQp04d/P7770p1jxw5Em+99RZcXV0xbtw4eHp6FqqbSsawQ0T0hiivyzF+fn5o1aoVzM3NYWNjg/79+yMsLExpn08++QS1a9eGsbExalhZo0O3njgTGFIeL1PjvL294e/vjzt37gAArl69irNnz6Jnz55K+506dQo2NjaoV68ePvvsMzx58qTYY+bl5SE/Px9GRkZK242NjXH27Fmluvfv34+HDx9CCIGTJ0/izp078PHx0eArfAMIEjKZTAAQMpmsoptCRFRuvv/+e1GjRg1x8OBBERUVJXbt2iXMzMzE//73P8U+I0eOFD169BCxsbGKR1JSUonH7d69u1i/fr24ceOGCAkJEb169RLOzs4iLS1Nsc+qVavE6dOnxdJ9Z4XDqF+FcZ3WQtfCWvwRcLfcXq+m5Ofni+nTpwuJRCJ09fSERCIRP/zwg9I+27ZtE3/99Ze4du2a2Lt3r6hfv75o1aqVyMvLK/a4Xl5eolOnTuLhw4ciLy9PbN68Wejo6Ah3d3fFPllZWWLEiBECgNDT0xMGBgZi48aN5fZaqxpVz98MO4Jhh4jeDL179xajR49W2jZw4EAxfPhwxc8jR44U/fr1K1M9CQkJAoA4ffq00vbIhFThOv2gcJl+UNh/tEwAEE6f/J94lJxRpvrK27Zt24Sljb2w7jtV2I9eLqzemSLMLKqJDRs2FFsmMjJSABDHjx8vdp+IiAjRsWNHAUDo6uqKVq1aieHDhwsPDw/FPj/++KNwd3cX+/fvF1evXhXLli0TZmZm4tixYxp9jVWVqudvXsYiInpDlMflmKLIZDIAgKWlpWLbydsJGLIyAAKAPCcLadePQ09qC4l5DdxLzCjbCytnU776ChLPfjCp3wkG1q4wbdgFep7vYP73PxRbplatWrCyskJERESx+9SuXRunT59GWloaYmJicOnSJeTm5qJWrVoACsb1zJo1C0uWLEGfPn3QpEkTTJgwAUOHDsVPP/2k8ddZklhZJs5HJiJWlvla69UU3o1FRPSGmDFjBlJSUuDh4QFdXV3k5+fj+++/x/DhwxX79OjRAwMHDoSbmxsiIyMxa9Ys9OzZEwEBAdDV1S21DrlcjkmTJqFdu3Zo1KgRElKz8N2BWzh4LRapVw7h6an1ELlZ0LN0hM3QBdDV04erlUl5vuwyEUJAlpIOI8lLfQMSHcTLMuEfGo+O7tbQ11V+/sGDB3jy5Ans7e1LrcPU1BSmpqZ4+vQpjhw5gsWLFwMAcnNzkZubCx0d5WPr6upCLper9TpiZZmISkyHm5Up7KXGpe4vhEBadh4S03KwIzAaq87chRCAjgTwG9gYQ1s5q1V/RWPYISJ6Q+zcuRNbt27FH3/8gYYNGyIkJASTJk2Cg4MDRo4cCQAYNmyYYv/GjRujSZMmqF27Nk6dOoWuXbuWWoevry9u3LiBM2f+xbZL0fD7OxQpWXnQkQDjPx4Ji48H45e/ApF8aTcS/1oI51E/IyMnv9xec1lcikqC3z+hkLi0gOz8DuhaWMPAyhk58ZFICdwHsyZvY8zGIEj18mF6cy8+GfkeOnrWxd27dzFt2jTUqVMH3bt3Vxyva9euGDBgACZMmAAAOHLkCIQQqFevHiIiIjB16lR4eHjgo48+AgBYWFigU6dOmDp1KoyNjeHi4oLTp09j06ZNWLJkicqvY0dgNGbuuQ75s7Ay+W13tHa1RGJaDhLTshWPx6nKP2flFg5UcgFM330dJ8Meo4VzdXjYm8PDzgLW5iXfZl/RJEK8MAz/DZWSkgKpVAqZTAYLC4uKbg4RqSA/Px/z5s3Dli1bEBcXBwcHB4waNQrffPMNJBIJAGDevHnYvn07YmJiYGBggBYtWuD7779HmzZtij2uq6sr7t+/X2j7+PHjsWLFCgDA6tWr8ccff+DKlStITU3F06dPUa1atXJ5nZrk5OSEGTNmwNfXV7FtwYIF2LJlC27fvl1sOWtrayxYsACffPJJicefMGEC/vrrL2za8zd+u5yKwHtPAQCNa0rhN7AxGtWUAijoZbj9IAlvN6+D6t0/h+dbvbHPtx3MDCvH9+878alYfPg2jocmAAAM5NnQD9mFsIv+kGfIoGtmiU49+8P73U/w961EJDxNweM9C5CTcBciOx3VrGzR3edt/PrjQtja2iqO6+rqilGjRmHevHkACsLnzJkz8eDBA1haWmLQoEH4/vvvIZVKFWXi4uIwc+ZMHD16FElJSXBxccG4ceMwefJkxfv8ZUIIPHiaidtxqQiMeoLV/0a98u/CUE8H2Xml9yJZmRmivr05POwKwo+HvTnq2JjBUE9X7V4ldah6/q4c7ywiIjWpMtGbu7s7li9fjlq1aiEzMxO//PILfHx8EBERAWtr6yKPGxgYiPz8/3oabty4gbfffhuDBw9WbMvIyECPHj3Qo0cPzJw5s3xfqAZlZGSofUlElcsxQgh8/vnn2LN3L0Z+vx7j9sUgN1/AxEAXU952xyhvV+i9cJnHXmoMSyMrGOhKYGEARCSk4audV/H7B82LPYG/DrGyTPxy7A7+vPwAcgHo6kgwrJUTJnatCxuLgYiVZeJeYgZcrUwUJ+05/eQ4F/kEe1utxZGb8cjMLXjvBAD4dHcEBjTLxDtNHGBpaoCAq6GISkxHrCwT9lJjDBkyBEOGDCmxTXZ2dli/fn2xgSE1Kxd34lMRGpuK23EpuB2bittxqUjLzivxuLYWhnCqbgIrM0NYmRvAyswQ1uaGBT+bGcL62XZZZi7aLTwB+QvdIjoSYEx7NzxMzsTt2FREPUlHYlo2/g3Pxr/hiYr99HQkqGFmgPiUbEW5iroExp4dsGeHqCp65513YGtri7Vr1yq2DRo0CMbGxtiyZUuRZZ7/Wz9+/LhKl2QAYNKkSTh48CDCw8MLnYhPnTqFzp07V5menVGjRuH48eNYtWoVrJ1qw//cRfw0+yt8PGY0Fi1ahLS0NHz77bcYNGgQ7OzsEBkZiWnTpiE1NRXXr19XzAj88uWY8ePHY/OWrajzwbdI1LECALSrWwPzB7dGXYcauHv3Lnbs2AEfHx9YW1vjwYMHWLhwIc6dO4edxwLw2e5I5OTLMbV7Pfh2rvPafy+yjFz8djoCG87dU/Ri9Gxkh6+610NtazOVj5OenYejt+KwN/gRzoY/VgQEPR0J6tqa4XZsKgQKTvrTutfDO54OKh334NVHWHwkDHIBSAB08bCBjo4Et+NSEJNU9IBhfV0J6tiYw8XSBEduxuHFE72uRIKzMzqr3MuyIzAas/bcQL4Q0JVI8MPARkqBJSMnD+HxaQiNTcHtuFTFf2WZuYWOpW7dpWHPDhFpNW9vb6xevRp37tyBu7u74s6i4sYy5OTkYPXq1ZBKpfD09FSpjpycHGzZsgVTpkyp0B4HTVm2bBlmz56NUR9/gieJj6FrZgnTBl3RpN9YAAW9PNeuXcPGjRuRnJwMBwcH+Pj4YP78+UpLH0RGRiIxseAbfHJGjmLG35DfJyv22QGgh3Q96o4aBSMjI/z777/49ddf8fTpU9ja2qJjx444f/486tVzx7d5Rpi55zp+OhqGRjWl6ORedK+bpmXl5mNTwD2sOBmpODG3drPEjJ4eaO5c/OzHxTE11MOAZo4Y0MwRCalZ2B/yCPtCHuLGwxSExqYq9pMLYOHhMCw8HFbC0YomAPjfTlDaZmdhpBg7U//Zf2tZmyoGTRcVVtQJG0NbOaOju3WhXq3nTAz04OlUDZ5O1f5rpxA4dD0WE/4IVto3XwjcS8zQ+OWs0rBnB+zZIaqK5HI5Zs2ahcWLFyvdWfTyZaWDBw9i2LBhyMjIgL29Pfbt24dWrVqpVMfOnTvx/vvvIzo6Gg4Ohb+FV7WeHaDgUs3LlyUkAGb28kBdG3PFZY0apoYw0Ct6dpJYWSaiHqcjPCEVy05EIDEtBwAwvI0zpvXwgNRYX+12zdxzDdsuxUBqrI8DE9rDuUb53KEVK8tEZEI6QmNlWH/uHh7JsgAA9WzNMb1nPXSuZ6PxYLsrKAZT/7xWaLu+jgQ6OiXXJZcL5MoLn6ZHtHVBj8Z28LCzgKWpQaltKOoSXHkr6r3Gnh0iqjDlOYCwvOpW5c4iAOjcuTNCQkKQmJiINWvWYMiQIbh48SJsbGxKrWPt2rXo2bNnkUGnqop6nI6Xz50CwA9/Fx6gLDXWh5WZwbMAVDCOIzY5E0dvxStdFqlrYwa/gY3R0tWy0DFUNa9vQ4TGpiIkJhnjNgdhz3hvmBho9hS1IzAaM/Zcx4tf8R2kRpjiUw8DmtWEbinB41W1r2sFHQkKnfTPTC/9pF9cYPisc221/r3YS41f+79te6kx/AY2LlOvkqawZwfs2aE328u3pb7OAYRlqftV7yyqW7cuRo8eXerA4vv376NWrVrYs2cP+vXrV+Q+VbFnZ8uF+/hm3w2lbRIUXL5JzcpDYlo2nqTnIL+I3oSiSACcnvoWnGuYlrltsbJM9Fl2FolpOejr6YD/DWuqsV6WqzFP0W/FeaVtEgAnv3oLrlZlb3tpShv3Ul5lK4Py7FVizw4RlSpWlqkIG0DBN89Ze26go7t1uX/7KqrumXuuq1z3q9xZBBRc/srOzi71+OvXr4eNjQ169+5d6r5VRWpWLpb6hwMoONELoMiTp1wukJyZWzDfSmo2HqdlIzEtB1djkrH/6iOlYwoAD5OzNBJ27KXGWPF+cwz/v4vYf/URmjhK8XGHWmU6Zr5cYMuF+/D7J7TQcwJArCzrtYSd0sa9lFfZyqAiepVexrBD9Aa7HZtS6JLG6xpAGJVY+HKKXABbL9zHlLfrlTqWoU+fPvj+++/h7OyMhg0bIjg4GEuWLMHo0aMBAOnp6fj+++/Rt29f2NvbIzExEStWrMDDhw+VbiN/+c4ioCAQrV+/HiNHjoSeXuGPybi4OMTFxSmWArh+/TrMzc3h7OystERCaV735cOfj95BQmo2XGuYYMNHrZ+d6AufPHV0JLA0NYClqQHcbc2V2nvw2qNCl1Q0OQNym1o18HXv+vj2wC34/XMbDRws4F3b6pWOFRqbgpl7riMkJrnI5zXd9tKU5aRfGQJDVca1sYgqWH5+PmbPng03NzcYGxujdu3amD9/Pl68wrxnzx74+PigRo0akEgkCAkJUenYu3btgoeHB4yMjNC4cWP8/fffiueS0nOw+Mgd5CbGIGH3d4j+ZQiilwxC7MbJ0MtUby2kV+FWzLfp5Scj0Wf5Wfwb/rjE8suWLcO7776L8ePHo379+vjqq6/wySefYP78+QAKenlu376NQYMGwd3dHX369MGTJ0/w77//omHDhgAKTt63wsIR9SBW6djHjx9HdHS0Iji9bOXKlWjWrBnGji24i6ljx45o1qwZ9u/fr/Lr3xEYjXYLT+D9NRfRbuEJ7AiMVrnsq7j+QIZNAfcAAPP7N4KrlSm8atdQe9yH38DG0H12aam8xmCM8nbFgGY1kS8X+PyPYDxKVm89psycfCz85zb6LDuLkJhkmBvqYX7/Rq+l7VQ5ccwOOGaHKtYPP/yAJUuWFJoc7/vvv1dMjrd582ZERUXBwcEBY8eORXBwMJo2bVricc+fP4+OHTvCz88P77zzDv744w8sWrQIV65cgdShFkauu4Sw8AjEbZ4C8yZvw6R+J0gMTGCW8QgnF4+Dg71dub7uM3ceY8S6S4qfdSTA2w1scT7iCVKfTYjWoa4VpvfwUMy8q0kVOVbp4dMMtF90skxzn6gjXy4w4LdzuPZAhr6eDlj6XrMyHe913NmTmZOPQb+fx63YFDRxlGLnJ14w0i99ba5/wx/j6703EJ1UsLhor8Z2mNunIWwtjF5b2+n1UfX8zbADhh2qWOpMjnfv3j24ubmpFHaGDh2K9PR0HDx4ULGtbdu2cKnbAHc93sfj1Gyk/f0T2tezxe//tx43HsgwZddVpGblYX6/hvjQy1WTL1NJbr4cPf/3LyIS0jCkZcG8JM9PPknpOVh+IgKbL9xDbn7Bx1NfTwd85VNPY7cjRyelo9OPp5TuyinPsPGi7Lx8jFx3CRfuJhV6btvYtvCqXUPjdW4KuIc5f92EuZEe/L/sBBtzI43XUR5ikjLQZ/lZJGfkYkhLRywa1KTYActP0rKx4FAo9gY/BFBwl9V3/RqhWwPbIvcn7aDq+ZuXsYgqmLe3N/z9/XHnzh0AUEyO17NnzzIdNyAgAN26dVPa1rBVB+w/dgqPU7Phbm2KrKggtGjSEKOG9Mfwzk2QsWs6Mu4E4Odjd/A0PadM9ZdkU8B9RCSkwdLUAF/3bqB0OcXS1ABz+jTAiS/fwoBmNSGRAPuvPkLXJacwb/9NPEkrfXDxy4QQiEhIxfpzURizIRA+S87g5a95z8cqlafUrFyM3hBYZNDRkaBcxo8kpGThx2eT103rXq/KBB0AcLI0wbL3mkFHAuwMeoCtFwtf6hNCYGdQDLouOY29wQ+hIwE+aueKo1M6MeiQAgcoE1WwGTNmICUlBR4eHkqT4w0fPrxMx42Li1NahPDQtVgcCM9ATupTtHazxAIfR9T7Kg0LFy7EggULsGjRIvz99z/4+puvoWNkhiXHHDC/f6OyvrxCEtOy8euxgmA3rXu9Yiegc7I0wS9Dm+LjDm5YdDgMZ+48xobz9/Dn5Qf4pGMtjOngVuI8LE/SsnE2IhFnwxNxNiIRsc8mjyvJoeuxaO1mWS7zrSSkZmHUukDcik2BqYEuhrV2xvpzUYrBvpamBuWyEOb8Q6FIzc6Dp6MU77dx0fjxy1uHutaY2t0Diw7fxrcHbqK+vQVauBTMbnz3cRpm7b2uCI8N7C2wcFBjNHGsVoEtpsqIYYeogqk6OV5ZbDx/D/MO3ES+XMBQTwebRrdG0uN4AEC/fv0weXLBNP9NmzbFP/6ncSXkH2x1aYz3WjujgYNmL+3+eDgMqdl5aFTTAoNbOpW6f0MHKTaNbo1zEYlY+M9tXH8ow8/H7mDThfuY2LUuOrpb4cHTTDhIjfHgaSb+DX+Mf8MTcSs2Rek4Bno6aO1qiQ51rdC+rhWuPZDhm70Fc5c8vw17y4X7ePA0A/8b1uyVZgEuTlRiOkasu4iYpExYmRlg/ajWaOwoxccd3HA1Jhlz/rqJhNRsTNl5Fas+aFHqnWiqOnPnMQ5cfQQdCfD9gMblNmleefu0Uy1cf5iMv6/HYdymIMzt0wDXH8qwMeA+cvLkMNbXxeS362J0OzelBUeJnmPYIapgU6dOxYwZMzBs2DAAQOPGjXH//n34+fmVKezY2dkhLi4OPx65jRUnIwEADarLkeLqCCN9XVhZWUFPTw8NGjRQKufd0hMRe49ALoBvD9zE9nFtNTax27UHydh5OQYAMK9PQ7VOvu3qWOEv33Y4dD0WPx4JQ3RSRqHJ8V5W394CHZ+Fm1aulkoDXBs6SPFWvf/mLrkUlYRpf17DqbDH6L/iHNaMaIE6NuYlHF01V2OS8dGGQCSl58Clhgk2jW4Nl2dz0jy/ndheaozBqwJw7FY8VpyMwOdd65a53qzcfMz+q+D3M9LbtVwGeb8uEokEi9/1RNC9p0hIzcYX20MUz3Vyt8aC/o3gZPn6biGnqocRmEiDYmWZOB+ZiFiZ6rfKvurkeKVp07Ytlm/Zpwg6X77tjvyYa/Dy8gIAGBgYoFWrVggLU16M8M6dO2jTxB1G+jq4GJWEQ9djCx37VcjlAvP234QQwIBmNV9paQEdHQn6eDrg+JROmPK2e5H79Gpsh/8Na4rAr7vhn4kdMLNXfXSoa13knTz2UmPFeKF+TWti92fecJAaISoxHf1XnMfxW/Fqt/FFp8IS8N6aC0hKz0HjmlLs/sxbEXRe5OlUDQv6FVwyXHL8Dk6GJRTaR12/nYrE/ScZsLUwLPZ3VZWkZhVMcvgiiQTwG8igQ6Vj2CHSkFedN+X55HiHDh3CvXv3sHfvXixZsgQDBgxQ7JOUlISQkBDcunULABAWFoaQkBDExcUp9hkxYoRiCYSMnDxk1PXB3ZBzSL20B583N8aTf7ciKChIafK8qVOnYseOHVizZg0iIiKwfPlyHDhwAF9N+gKfdaoDAPjhUCgyc/LL/PvZF/IQV6KTYWKgixk9Pcp0LAM9HbR0LXpV6g/buqJf05qwNjcs8vmSNKopxf7P26O1qyXSsvMwdnMQlp8Ix6vctLrnygN8vDEIGTn56FDXCtvGtYWVWfFtGtLKCcPbOEMIYOK2YNxLTFe7zuciH6dh5amCkDvnnYYwN9LcJbmKUtQklEIA95+oNwcPvZkYdog0oLhlF1Tp4SltcjwA2L9/P5o1a6ZYumDYsGFo1qwZVq5cqdgnOjoasbGxSErPwftrLuJmvj3s+0+DSdRpzPqwJ/7880/s27cPjRr9N+h4wIABWLlyJRYvXozGjRvj//7v/7B79260b98en3SqhZrVjPFIloXfT0eW6feTlp2Hhf8UrFc1oUsdxZwnZeFmZYqXr4JpYkZcKzNDbPm4DT5oWxA8fjp6B75/XEH6s7l/SiOEwKrTkZiy8yry5AL9mzpg7chWKg0+ntunIZo7V0NKVh4+2XxZ5Tpfrn/2vhvIyZejk7s1ejUu3/mSXpfy+nvTm4Hz7IDz7FDZ7b/6CF9sCy60vbzmTSlKrCwTgVFJ+OloGKKTMiE11se6US3RwuXVV6L+53osPtt6BQZ6OvCf0umVLxcs/Oc2Vp6OhEsNExyd3BGGeqVPDqeK8l4gcdulaMz56wZy8wU87MyxZkTLEn8HcrnA93+HYu3ZKADA2A5umNmzvloDjuNTsvDOsrN4nJqN3k3ssfy9ZmqNmdoX/BCTdoTAUE8HRyd3LPKyWVVV1RfEJM3jpIJqYNihsniSlo0Bv51DdJJyL46OBDg3o8trmaV1R2A0Zuy5rpg7Rmqsh92feZd5gK0QAsP/7yLORz5Bj4Z2WPlhC7WPEZWYDp9fTiM3X+D/RrTU+Nwn5T0jbtC9JHy65QoS07JR3UQfK95vDu86hddqys7Lx1e7ruHAs4Uyv+5VH2M7vtoiloH3kvDe6gvIkwvM6uWBcR1rq1ROlpGLrktOITEtB1/5uGNCl7IPdK5sOAMyvYiTCtJr5+rqColEUujh6+sLoGDelw8//BB2dnYwNTVF8+bNsXv37lKPu2LFCri6usLIyAht2rTBpUuXlJ7/5JNPULt2bRgbG8Pa2hr9+vXD7du3y+U1viw1Kxcj11961pOip9TN3sRRCjsNXK4pTawsUynoFLQrD6YamLNFIpFg7rO7pg7fjMO5iES1j7Hg4C3k5gt0crdG1/o2ZW7Ty14cZFweWrpa4sDn7dDEUYqnGbn4cN0lbDgXpTSO5/lkgQeuPoK+rgS/Dm36ykEHAFq5WmJun4K75Bb+c1vl3/viI7eRmJaD2tamZaq/Mivvvzdpp0oddlRZIFEIgTlz5sDe3h7Gxsbo1q0bwsPDK7DVb67AwECs/DsQjhM2w9F3M+yGLQAAxQrTI0aMQFhYGPbv34/r169j4MCBGDJkCIKDC1/+eW7Hjh2YMmUK5s6diytXrsDT0xPdu3dHQsJ/d6u0aNEC69evR2hoKI4cOQIhBHx8fJCfX/ZBtSXJys3H2E1BuPEwBTVMDbBnfDucm9EFc96pD10JEBIjw7pz98q1DQDwb3hiodmA5QIamw24np05PmxbMBndtwduIi9f9bvEToYlwP92AvR0JJjTp4HGbmF/3eylxtj5iZdiccp5B25h2p/XcP9JOv6+/giDfj+PcxFPYGqgi3WjWqF/s5plrvODti54t4Uj5AKY8McVPHha8t8zOPop/rhUMCh+Qf/GGrtUSKQNKnXYWbRoEX7//XcsX74coaGhWLRoERYvXoxly5Yp9lm8eDGWLl2KlStX4uLFizA1NUX37t2RlVX6bKmkOflygeN3M+B3Oh66ptWha1Yd6eGXoFfNHnU9WwEoWJjy888/R+vWrVGrVi188803qFatGi5fvlzscZcsWYKxY8fio48+QoMGDbBy5UqYmJhg3bp1in3GjRuHjh07wtXVFc2bN8eCBQsQExODe/fuldvrzcuXY8IfwbhwNwlmhnrYOLo1alubwV5qjNHta+Gbdwq+lf/wdygCIstvBfGYpAwsPly4F0vTAzcnd3NHdRN93IlPw5YL91Uqk5Mnx/wDBXePfdTOFbWtzTTWnopgpK+LJUM88U3v+tCRALsuP0CnH09h/NZg3IlPg6mBLraP80KHutYaqU8ikWBB/0ZoXLOgR+nTLZeRlVt0gM/Ll+PrvTcgBDCwec3XNk6MqKqo1GHn/Pnz6NevH3r37g1XV1e8++678PHxUVzGEELg119/xTfffIN+/fqhSZMm2LRpEx49eoR9+/ZVbOPfEBk5edh4/h66/HwKM/f+N8GbyM9F+q1TMGvyNj7eeBmRj9Pg7e2NHTt2ICkpCXK5HNu3b0dWVhbeeuutIo+dk5ODy5cvK63vpKOjg27duiEgIKDIMunp6Vi/fj3c3Nzg5FT67LyvQi4XmL77Oo6HxsNATwf/N7JloQnbRnm7KnoBJvxxBY+SNX977OPUbHy49iIS03JgY26ouIT2fOCmJrv5pSb6+Kp7PQDAkmN3VFqfasP5KNxNTIeVmSG+0MAkeZWBRCLBxx1q4ZehTQs9l5mbDytzA43WZ6Svi5UftoClqQFuPEx5FmgKD7PcGHAft2JTIDXWx6xe9TXaBiJtUKnDTmkLJEZFRSEuLk7pZCiVStGmTZtiT4YAkJ2djZSUFKUHqSchJQs/HrkNL78TmLv/Ju4/yYCFkR6eX6TIuHMB8qw0mDbqitC4VPT89V90Hu+H7Jwc1KhRA4aGhvjkk0+wd+9e1KlTp8g6EhMTkZ+fr7S+EwDY2toqzS8DAL/99hvMzMxgZmaGf/75B8eOHYOBgWZPPEBBwP7+71DsvvIAujoSrHi/OdrWKvwtWiKR4IcBjdHA3gJP0nPwWQnfyl9FSlYuRq67hHtPMuBY3Rj7J7THuRldsG1sW5yd0blc7lAZ1soZDewtkJKVh5+O3ilx34TULCz1jwAATO9RTyvmeXlRUXP4aPLS4YtqVjPG8meLYe6+8gCbX+pZi5VlYsnRgokhZ/T0KHEuH6I3VaUOO8+n0Pfw8IC+vj6aNWuGSZMmKRZIfH7CU+Vk+CI/Pz9IpVLFo7x6ALRRWFwqpu66ivaLTmLFyUjIMnPhUsME3/VriAuzumLhoMbQlUiQdu0oTGq1xNeDvdHJ3Ro5+XL4zZ+HS7djsHzzHgQFBWHKlCkYMmQIrl+/XuZ2DR8+HMHBwTh9+jTc3d0xZMiQcrmUueJkhOK24sWDmuDtEu4sMjbQxaoPW6CaiT6uPpBh7l83X2lyupdl5uTj4w1BuBWbAiszA2we0wZ2UqNyH7ipqyPBvL4NAQDbA6Nx46Gs2H0XHw5DWnYePJ2qYVBzx3JpT0V63XO+eNexwsyeBT023x24hcB7/62a/t2BW0jPyUdz52oYqsJaY0Rvokoddl5cIPHKlSvYuHEjfvrpJ2zcuLFMx505cyZkMpniERMTo6EWaychBM6GJ2LEukvo/usZ7Lr8ADn5crR0qY6VH7TAiS/fwggvV5gY6GFoK2dsf782cqKvYvl3X+Kzt+pgw0etMLN9daReOQjTtz/HjzcMsD1SgklTZ6Fly5ZYsWJFkfVaWVlBV1cX8fHKU/bHx8fDzk55ojSpVIq6deuiY8eO+PPPP3H79m3s3btXo7+HzRfuK3o0Zr/TAINalH4Sd7I0wbJn38p3BMUoBpC+qtx8OXz/uIJL95Jg/myskJvV65tHpbWbJfp6OkAIPFv6oXB4C45+ij8vPwAAzOvTQGOLWlYm9lJj+A0sCPZA+Vw6fNnHHdzQx9MBeXKB8VuvID4lCydux+OfG3HQ1ZHg+wGNtfJ3TaQJlXoh0NIWSHx+wouPj4e9vb2iXHx8PJo2bVrscQ0NDWFoyK7eksTKMhEen4bw+FTsuvwAt+NSARTMHdOzkT0+7uCGZs5FT9d/aPc22NjY4MMhAwEUXNLxdimY76VXEwccjwW2XYrBsVsJkGfkFXvXlIGBAVq0aAF/f3/0798fACCXy+Hv76+05MHLhBAQQiA7u/RxJao6cPUR5jxbVPHzLnUwpr2bymU71LXG1O4eWHT4NubtvwkPOwu0cCn6d1cSuVxg6q6rOHE7AYZ6Olg7qhUaOrz+xR1n9vLAsVvxCLr/FPuvPkK/pv/defR8/SsAeLeFY7HvEW0wtJUzOrpbv7Y5XyQSCRYNaozw+FTcjkvF6A2BiE8p6L0c094N9e05RxhRcSp1z05pCyS6ubnBzs4O/v7+iudTUlJw8eJFxWKHpL7tl6Lh7XcCI9ZdwvxDobgdlwoTA12M8nbF6amdsWJ482JPYnK5HOvXr8fIkSOhp/dflvbw8ECdOnVwZ/cSzGljAHtJMu6e2IbggDOIMmuAmKSCsQ5du3bF8uXLFeWmTJmCNWvWYOPGjQgNDcVnn32G9PR0fPTRRwCAu3fvws/PD5cvX0Z0dDTOnz+PwYMHw9jYGL169dLI7+P0nceYsjMEQgAftnV5pUUVP+1UC70a2yE3X2D81stISFXvEpsQAt8dvIV9IY+gpyPB7x80R2u3V58ZuSzspcbw7VwwyZ3f37eVljTYfeUBrj6QwcxQD9N61KuQ9r1Or3vOFxMDPaz6sAWM9HRw81EKEtNyAACO1TjnDFFJKnXYKW2BRIlEgkmTJmHBggWKuVtGjBgBBwcHRU8Aqef5Gk8vXpyQANg3vh3m9W1Y6nIBx48fR3R0NEaPHq20XV9fH3///Tesra0x89MPcG3pOBjfPwfbPlMQYegOn1/OYM2Zu4iMjETUg1jFyuFDhw7FTz/9hDlz5qBp06YICQnB4cOHFeO0jIyM8O+//6JXr16oU6cOhg4dCnNzc5w/fx42NmWfwO7y/SR8uvkycvMF+ng64Nu+DV9prhiJRILF73qiro0Z4lOy4bv1CnLyVJ+vZql/BDacvwcA+GmwJ7p4aHYWYnV93KEWnCyNEZeShd9OFQxETsnKxaLDBQNlv+haBzbm5T+h4pvIQE8H2S/NdfTtgVsqrcNG9Kaq1MtFpKamYvbs2di7dy8SEhLg4OCA9957D3PmzFHcaSOEwNy5c7F69WokJyejffv2+O233+Durvq3by4X8Z/dl2Pw5a5rhbaX1xpPEQlpmLX3Oi5FFQy4dJAaITYlC0IUXDLzG9i4wta+uR2XgiErA5CSlYdO7tZYM6IlDPTK9v3g7uM09Ft+DqnZeRjp5YJv+zUqtczG8/cw99mloW/7NsRIb9cytUFTjt6Mw7jNl2Ggq4NjUzpiy4X7WPNvFGpZmeLwpI5l/l1R0c5HJuL9NRcLbX+d67ARVRZcG0sNDDsF8vLl6LP8HEJjlW/F15VIcHZG53LrqpfLBXZdjsGCQ6FIzVJe5VlHAvzl2w6Nakpf6+y70U8yMGjleTxOzUYLl+rYPKY1TAw0M8TNPzQeYzYGAQB+HuxZ4kDnv0IeYuL2EADApG51Mamb+pfQyosQAiPWXcK/4Ylo5GCBW7EpkAtg/Uet0Lme5peFoAKxsky0W3gC8hc+ucv73yhRZcW1sUhtK05GIjQ2BUZ6OuU6Qd3LdHQkGNrKGT++26TQc3IB9Fl+Di0WHMf7ay7guwO3sCsoBjceyoqctyZWlqm4BPYqYmWZ+PvaIwxbE4DHqdnwsDPHupGtNBZ0AKBrfVtMfDbJ3qy914u9hfvk7QR8ufMqgIJJCidWson5CtbNagAdCXDjUYri5JuQwtnLy1NF3AlGVNVV6rux6PW5GpOMpScK1hRb9G4TtHazfO0rC3s6VYOOBErfWIGCMUNJ6Tk4H/kE519YekFXRwI3K1N42Jmjvr0FHqdmYVPAfchf8RLYjsBozNxzXVG/pakBNo1uDamJ5ifEm9i1Lm48lMH/dgI+2XwZBz5vD0vT/yZBDLyXhE+3XEaeXGBAs5qY807lXFfK1FCv0Lpcs/bcQEd3a558y9HrvhOMqKrjZSzwMlZmTj56L/sXdx+n450m9lj+fvMKa8uOwGjM2nMD+UIovrH2a1oT4fFpCI1LQWhsCm7HpiI0LgXJGbmlHq+asb5Kc4/I5QLJmcrH05EA52Z0KbcTiSwzF/1XnENUYjq8a9fAptGtoaerg1uPUjB0dQBSs/LQ1cMGKz9sAX3dytkJy/EjRFSRVD1/s2eHsPCfUNx9nA5bC0Ms6F/6gNnyVNw31saOUjR2/G9OGSEEElKzC8JPXCr+vfMY54pYcPPlAKOO59P/l1fYkRrrY9WHLdB/xTmcj3yCOftvopVLdXx3sGDsUmtXS6wY3rzSBh3gv5mEXx4/Ul4zCRMRvQr27ODN7tk5fecxRq4rWFh185jWGlux+XUratCmjgTYMqZNkesYvexxajY+WHuxQgZ9/n09FuO3XlHaZi81wpHJHWFRBdaUKqo3rqLuoCOiNwt7dqhUyRk5mLrrvwGwVTXoAP8N2nz5pOtdx0ql8nVtzYss/zrGQjRzrgYJoDS3UXxKFtKz86pE2OH4ESKq7Bh23lBCCHy97wYSUrNRy9oU03t4VHSTyqysJ92KOmlHJabj5e7V8r6Epmn2UuMq01YievMw7Lyh9l99hEPXYqGnI8GvQ5vC2EC3opukEWU96VbESZvjXoiIylflHflI5eZRcia+2VewqOUXXeuiiWO1im3QG47zphARlS/27Lxh5HKBqX9eRWpWHpo6VcP4t2pXdJMIHPdCRFSeGHbeMBvO38O5iCcw1tfFkiGe0KvEtzW/aTjuhYiofPBM9wYJj0/FwsO3AQBf966PWtZmFdwiIiKi8sew84bIyZNj8s4Q5OTJ8VY9awxvw3lQiIjozcCw84ZY6h+OGw9TUM1EH4sHNamU6ywRERGVB4adN8Dl+0n47VQEAMBvQGPYWBhVcIuIiIheH4YdLZeenYcpO69CLoCBzWqiZ2P7im4SERHRa8Wwo+UWHArF/ScZqFnNGPP6Nazo5hAREb12vPVcS8XKMrHn8gNsuxQNiQT4abBnlVhniYiISNMYdrTQjsBozNxzXbH8QPvaVvCqXaNiG0VERFRBeBlLy8TKMpWCDgCci0xErCyz4hpFRERUgRh2tExUYrpS0AH+W0GbiIjoTcSwo2XcrEzx8hQ6XEGbiIjeZAw7WsZeaoz2dawUP3MFbSIietNxgLIWepRcMD5nYte6GNbaiUGHiIjeaAw7Wib6SQYiH6dDV0eC0e3dIDXm7eZERPRm42UsLXMyLAEA0NKlOoMOERERGHa0zonbBWGni4dNBbeEiIiocmDY0SIZOXkIuPsEAMMOERHRcww7WuR8xBPk5MnhWN0YdWzMKro5RERElQLDjhZ5Pl6ni4cNJC9PtkNERPSGYtjREkIInHw2XqdzPV7CIiIieo5hR0uExafikSwLRvo6XPSTiIjoBQw7WuL5XVjeta1gpK9bwa0hIiKqPBh2tITiEhbvwiIiIlLCsKMFkjNycPn+UwBA53rWFdwaIiKiyoVhRwucCU+EXADutmZwrM7VzYmIiF7EsKMFeAmLiIioeAw7VVy+XODU8/l1eMs5ERFRIQw7VVxITDKeZuTCwkgPLVyqV3RziIiIKh2GnSru+SWsju7W0NPln5OIiOhlPDtWcS8uEUFERESFMexUYfEpWbj5KAUSCdDJnbecExERFYVhpwp7fgnL07EaapgZVnBriIiIKieGnSrs+RIRvIRFRERUPIadKio7Lx9nIxIBMOwQERGVhGGniroUlYSMnHzYmBuioYNFRTeHiIio0mLYqaJO3n4MAHirnjUkEkkFt4aIiKjyYtiponjLORERkWoYdqqgqMR0RCWmQ19XgvZ1ecs5ERFRSRh2qqDnd2G1drOEmaFeBbeGiIiocmPYqYIUq5xz4U8iIqJSMexUMenZebgY9QQAx+sQERGpgmGnijkbkYjcfAGXGiZwszKt6OYQERFVegw7VcyLl7B4yzkREVHpGHaqECEEbzknIiJSE8NOFXLzUQriU7JhYqCLNrUsK7o5REREVQLDThVy6lmvTrs6VjDU063g1hAREVUNDDtVyAneck5ERKQ2tcPOyZMny6MdVIqk9BwExyQDADp7cNZkIiIiVakddnr06IHatWtjwYIFiImJKY82URFO30mAEEB9ewvYS40rujlERERVhtph5+HDh5gwYQL+/PNP1KpVC927d8fOnTuRk5NTHu2jZ048W+W8C3t1iIiI1KJ22LGyssLkyZMREhKCixcvwt3dHePHj4eDgwO++OILXL16tTza+UbLy5fjNG85JyIieiVlGqDcvHlzzJw5ExMmTEBaWhrWrVuHFi1aoEOHDrh586am2vjGC45JRkpWHqqZ6KOpU/WKbg4REVGV8kphJzc3F3/++Sd69eoFFxcXHDlyBMuXL0d8fDwiIiLg4uKCwYMHa6SBDx8+xAcffIAaNWrA2NgYjRs3RlBQkOJ5IQTmzJkDe3t7GBsbo1u3bggPD9dI3ZXF87uwOrlbQ1eHsyYTERGpQ+2w8/nnn8Pe3h6ffPIJ3N3dERwcjICAAHz88ccwNTWFq6srfvrpJ9y+fbvMjXv69CnatWsHfX19/PPPP7h16xZ+/vlnVK/+X+/G4sWLsXTpUqxcuRIXL16EqakpunfvjqysrDLXX1k8XyKCl7CIiIjUp6dugVu3bmHZsmUYOHAgDA0Ni9zHyspKI7eoL1q0CE5OTli/fr1im5ubm+L/hRD49ddf8c0336Bfv34AgE2bNsHW1hb79u3DsGHDytyGivYwORO341KhIyno2SEiIiL1qN2z4+/vj/fee6/YoAMAenp66NSpU5kaBgD79+9Hy5YtMXjwYNjY2KBZs2ZYs2aN4vmoqCjExcWhW7duim1SqRRt2rRBQEBAscfNzs5GSkqK0qOyet6r09y5OqqZGFRwa4iIiKoetcOOn58f1q1bV2j7unXrsGjRIo006rm7d+/i999/R926dXHkyBF89tln+OKLL7Bx40YAQFxcHADA1tZWqZytra3iuaL4+flBKpUqHk5OThpttyY9XyKiMy9hERERvRK1w86qVavg4eFRaHvDhg2xcuVKjTTqOblcjubNm+OHH35As2bNMG7cOIwdO7bM9cycORMymUzxqKyTI2bl5uNcxBMAXCKCiIjoVakdduLi4mBvb19ou7W1NWJjYzXSqOfs7e3RoEEDpW3169dHdHQ0AMDOzg4AEB8fr7RPfHy84rmiGBoawsLCQulRGV24+wSZufmwlxqhvr15RTeHiIioSlI77Dg5OeHcuXOFtp87dw4ODg4aadRz7dq1Q1hYmNK2O3fuwMXFBUDBYGU7Ozv4+/srnk9JScHFixfh5eWl0bZUhINXHwEAWrtZQiLhLedERESvQu27scaOHYtJkyYhNzcXXbp0AVAwaHnatGn48ssvNdq4yZMnw9vbGz/88AOGDBmCS5cuYfXq1Vi9ejUAQCKRYNKkSViwYAHq1q0LNzc3zJ49Gw4ODujfv79G2/K6bb8UjT+vPAQA7L/6CN61a2BoK+cKbhUREVHVIxFCCHUKCCEwY8YMLF26VLEelpGREaZPn445c+ZovIEHDx7EzJkzER4eDjc3N0yZMgVjx45Vas/cuXOxevVqJCcno3379vjtt9/g7u6uch0pKSmQSqWQyWSV4pJWrCwT7RaegPyFv4yuRIKzMzpzEVAiIqJnVD1/qx12nktLS0NoaCiMjY1Rt27dEm9Fr+wqW9g5H5mI99dcLLR929i28KpdowJaREREVPmoev5W+zLWc2ZmZmjVqtWrFqcSOFYv3HujK5HA1cqkAlpDRERUtb1S2AkKCsLOnTsRHR2tuJT13J49ezTSsDfZrUepSj/rSiT4YWAjXsIiIiJ6BWrfjbV9+3Z4e3sjNDQUe/fuRW5uLm7evIkTJ05AKpWWRxvfONsuFdxa/2FbF2wb2xZnZ3Tm4GQiIqJXpHbY+eGHH/DLL7/gwIEDMDAwwP/+9z/cvn0bQ4YMgbMzT8hl9eBpBs6EPwYAfNzBDV61a7BHh4iIqAzUDjuRkZHo3bs3AMDAwADp6emQSCSYPHmy4pZwenU7A2MgBNCuTg241DCt6OYQERFVeWqHnerVqyM1tWBMSc2aNXHjxg0AQHJyMjIyMjTbujdMXr4cO4IKlq54rzV7yYiIiDRB7QHKHTt2xLFjx9C4cWMMHjwYEydOxIkTJ3Ds2DF07dq1PNr4xjgV9hjxKdmwNDXA2w1sSy9AREREpVI77CxfvhxZWVkAgK+//hr6+vo4f/48Bg0ahG+++UbjDXyTPB+Y/G4LRxjq6VZwa4iIiLSDWmEnLy8PBw8eRPfu3QEAOjo6mDFjRrk07E0TK8vEybAEAMDQVk4V3BoiIiLtodaYHT09PXz66aeKnh3SnJ2BDyAXQBs3S9S2Nqvo5hAREWkNtQcot27dGiEhIeXQlDdXvlxg57OBye+34cBkIiIiTVJ7zM748eMxZcoUxMTEoEWLFjA1Vb49ukmTJhpr3JviTPhjPEzORDUTfXRvaFfRzSEiItIqaoedYcOGAQC++OILxTaJRAIhBCQSCfLz8zXXujfE9mcDkwc2c4SRPgcmExERaZLaYScqKqo82vHGSkjJwvHQgoHJ77XmwGQiIiJNUzvsuLi4lEc73li7Lj9AvlygpUt11LU1r+jmEBERaR21w86mTZtKfH7EiBGv3Jg3jVwusD2w4BLWMM6YTEREVC7UDjsTJ05U+jk3NxcZGRkwMDCAiYkJw44azkUmIiYpE+ZGeujd2L6im0NERKSV1L71/OnTp0qPtLQ0hIWFoX379ti2bVt5tFFrbb9UcLv5gGY1YWzAgclERETlQe2wU5S6deti4cKFhXp9qHiJadk4eisOADCsFS9hERERlReNhB2gYHblR48eaepwWm/35QfIzRfwdKqGBg4WFd0cIiIiraX2mJ39+/cr/SyEQGxsLJYvX4527dpprGHaTAiB7YHPZkzm7eZERETlSu2w079/f6WfJRIJrK2t0aVLF/z888+aapdWu3A3CVGJ6TAz1MM7TRwqujlERERaTe2wI5fLy6Mdb5Rtz2ZM7tvUAaaGav8JiIiISA0aG7NDqnmanoPDNwoGJr/PuXWIiIjKndphZ9CgQVi0aFGh7YsXL8bgwYM10ihttvvKA+Tky9GopgUa1ZRWdHOIiIi0ntph58yZM+jVq1eh7T179sSZM2c00iht9eLA5PfYq0NERPRaqB120tLSYGBgUGi7vr4+UlJSNNKoymbhwoWQSCSYNGkSAODevXuQSCRFPnbt2lXscXR0dOD/5Vu4v+gdfNDWVVHmxx9/VOzTt29fODs7w8jICPb29vjwww95Sz8REVEZqB12GjdujB07dhTavn37djRo0EAjjapMAgMDsWrVKjRp0kSxzcnJCbGxsUqPb7/9FmZmZujZs2exxxq38hgcfTdj/Gp/xMbGYt26dZBIJBg0aJBin86dO2Pnzp0ICwvD7t27ERkZiXfffbdcXyMREZE2U/tWoNmzZ2PgwIGIjIxEly5dAAD+/v7Ytm1bib0aVVFaWhqGDx+ONWvWYMGCBYrturq6sLOzU9p37969GDJkCMzMzIo8liwjF6dicqFrVh0fd28GO7vq+Ouvv9C5c2fUqlVLsd/kyZMV/+/i4oIZM2agf//+yM3Nhb6+voZfIRERkfZTu2enT58+2LdvHyIiIjB+/Hh8+eWXePDgAY4fP15oDp6qztfXF71790a3bt1K3O/y5csICQnBmDFjit1nb/ADZOfJ4WFnjqZO1RAfH49Dhw6VWCYpKQlbt26Ft7c3gw4REdEreqVJXnr37o3evXtrui2Vyvbt23HlyhUEBgaWuu/atWtRv359eHt7F/n8ywOTJRIJNm7cCHNzcwwcOLDQ/tOnT8fy5cuRkZGBtm3b4uDBg2V7MURERG8wtXt2AgMDcfHixULbL168iKCgII00qqLFxMRg4sSJ2Lp1K4yMjErcNzMzE3/88UeJPTTBMcm4HZcKQz0d9G9WEwCwbt06DB8+vMjjT506FcHBwTh69Ch0dXUxYsQICCHK9qKIiIjeUGqHHV9fX8TExBTa/vDhQ/j6+mqkURXt8uXLSEhIQPPmzaGnpwc9PT2cPn0aS5cuhZ6eHvLz8xX7/vnnn8jIyMCIESOKPd72ZzMmv9PEAVJjffz7778ICwvDxx9/XOT+VlZWcHd3x9tvv43t27fj77//xoULFzT7IomIiN4Qal/GunXrFpo3b15oe7NmzXDr1i2NNKqide3aFdevX1fa9tFHH8HDwwPTp0+Hrq6uYvvatWvRt29fWFtbF3ms1KxcHLgaCwB479min2vXrkWLFi3g6elZalueL8+RnZ39Sq+FiIjoTad22DE0NER8fLzSHUQAEBsbCz097VjnydzcHI0aNVLaZmpqiho1aihtj4iIwJkzZ/D3338XeRwPDw/0GDUZmbmOqGtjhhYu1ZGSkoJdu3YVuWjqxYsXERgYiPbt26N69eqIjIzE7NmzUbt2bXh5eWn2RRIREb0h1L6M5ePjg5kzZ0Imkym2JScnY9asWXj77bc12rjKbt26dXB0dISPj0+Rz4eFheFw8F0AwLBnA5O3b98OIQTee++9QvubmJhgz5496Nq1K+rVq4cxY8agSZMmOH36NAwNDcv1tRAREWkriVBz5OvDhw/RsWNHPHnyBM2aNQMAhISEwNbWFseOHYOTk1O5NLQ8paSkQCqVQiaTwcLCQiPH3BEYjRl7ruP5b3fOOw0wur2bRo5NREREqp+/1Q47AJCeno6tW7fi6tWrMDY2RpMmTfDee+9V2blgNB12YmWZaLfwBOQv/GZ1JRKcndEZ9lLjMh+fiIiIVD9/v9IgG1NTU4wbN+6VG6ftohLTlYIOAOQLgXuJGQw7REREr9krjyi+desWoqOjkZOTo7S9b9++ZW5UVedmZQodCQr17LhamVRco4iIiN5Qaoedu3fvYsCAAbh+/TokEolisjuJRAIASnPQvKnspcbwG9gYs/bcQL4Q0JVI8MPARuzVISIiqgBqh52JEyfCzc0N/v7+cHNzw6VLl/DkyRN8+eWX+Omnn8qjjVXS0FbO6OhujXuJGXC1MmHQISIiqiBqh52AgACcOHECVlZW0NHRgY6ODtq3bw8/Pz988cUXCA4OLo92Vkn2UmOGHCIiogqm9jw7+fn5MDc3B1CwrMGjR48AAC4uLggLC9Ns64iIiIjKSO2enUaNGuHq1atwc3NDmzZtsHjxYhgYGGD16tWFZlUmIiIiqmhqh51vvvkG6enpAIDvvvsO77zzDjp06IAaNWpgx44dGm8gERERUVm80qSCL0tKSkL16tUVd2RVNeUxgzIRERGVr3KdVPBllpaWmjgMERERkcapPUCZiIiIqCph2CEiIiKtxrBDREREWk3tsHPmzBnk5eUV2p6Xl4czZ85opFFEREREmqJ22OncuTOSkpIKbZfJZOjcubNGGkVERESkKWqHHSFEkbeYP3nyBKamphppFBEREZGmqHzr+cCBAwEUrG4+atQoGBoaKp7Lz8/HtWvX4O3trfkWEhEREZWBymFHKpUCKOjZMTc3h7HxfwtcGhgYoG3bthg7dqzmW0hERERUBiqHnfXr1wMAXF1d8dVXX/GSFREREVUJao/ZmTZtmtKYnfv37+PXX3/F0aNHNdowIiIiIk1QO+z069cPmzZtAgAkJyejdevW+Pnnn9GvXz/8/vvvGm8gERERUVmoHXauXLmCDh06AAD+/PNP2NnZ4f79+9i0aROWLl2q8QYSERERlYXaYScjIwPm5uYAgKNHj2LgwIHQ0dFB27Ztcf/+fY03kIiIiKgs1A47derUwb59+xATE4MjR47Ax8cHAJCQkFDi8upEREREFUHtsDNnzhx89dVXcHV1RevWreHl5QWgoJenWbNmGm8gERERUVmoHXbeffddREdHIygoCEeOHFFs79q1K3755ReNNu5lCxcuhEQiwaRJkxTbsrKy4Ovrixo1asDMzAyDBg1CfHx8ubaDiIiIqo5XWvXczs4O5ubmOHbsGDIzMwEArVq1goeHh0Yb96LAwECsWrUKTZo0Udo+efJkHDhwALt27cLp06fx6NEjxWzPRERERGqHnSdPnqBr165wd3dHr169EBsbCwAYM2YMvvzyS403EADS0tIwfPhwrFmzBtWrV1dsl8lkWLt2LZYsWYIuXbqgRYsWWL9+Pc6fP48LFy6US1uIiIioalE77EyePBn6+vqIjo6GiYmJYvvQoUNx+PBhjTbuOV9fX/Tu3RvdunVT2n758mXk5uYqbffw8ICzszMCAgKKPV52djZSUlKUHkRERKSdVF4u4rmjR4/iyJEjcHR0VNpet27dcrn1fPv27bhy5QoCAwMLPRcXFwcDAwNUq1ZNabutrS3i4uKKPaafnx++/fZbTTeViIiIKiG1e3bS09OVenSeS0pKUloJXRNiYmIwceJEbN26FUZGRho77syZMyGTyRSPmJgYjR2biIiIKhe1w06HDh0Uy0UAgEQigVwux+LFi9G5c2eNNu7y5ctISEhA8+bNoaenBz09PZw+fRpLly6Fnp4ebG1tkZOTg+TkZKVy8fHxsLOzK/a4hoaGsLCwUHoQERGRdlL7MtbixYvRtWtXBAUFIScnB9OmTcPNmzeRlJSEc+fOabRxXbt2xfXr15W2ffTRR/Dw8MD06dPh5OQEfX19+Pv7Y9CgQQCAsLAwREdHK+b/ISIiojeb2mGnUaNGuHPnDpYvXw5zc3OkpaVh4MCB8PX1hb29vUYbZ25ujkaNGiltMzU1RY0aNRTbx4wZgylTpsDS0hIWFhb4/PPP4eXlhbZt22q0LURERFQ1qR12oqOj4eTkhK+//rrI55ydnTXSMFX98ssv0NHRwaBBg5CdnY3u3bvjt99+e61tICIiospLIoQQ6hTQ1dVFbGwsbGxslLY/efIENjY2yM/P12gDX4eUlBRIpVLIZDKO3yEiIqoiVD1/qz1AWQgBiURSaHtaWppG75giIiIi0gSVL2NNmTIFQMHdV7Nnz1a6/Tw/Px8XL15E06ZNNd5AIiIiorJQOewEBwcDKOjZuX79OgwMDBTPGRgYwNPTE1999ZXmW0hERERUBiqHnZMnTwIouPX7f//7H8e2EBERUZWg9t1Y69evL492EBEREZULtQcoExEREVUlDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Sp12PHz80OrVq1gbm4OGxsb9O/fH2FhYUr7ZGVlwdfXFzVq1ICZmRkGDRqE+Pj4CmoxERERVTaVOuycPn0avr6+uHDhAo4dO4bc3Fz4+PggPT1dsc/kyZNx4MAB7Nq1C6dPn8ajR48wcODACmw1ERERVSYSIYSo6Eao6vHjx7CxscHp06fRsWNHyGQyWFtb448//sC7774LALh9+zbq16+PgIAAtG3bVqXjpqSkQCqVQiaTwcLCojxfAhEREWmIqufvSt2z8zKZTAYAsLS0BABcvnwZubm56Natm2IfDw8PODs7IyAgoNjjZGdnIyUlRelBRERE2qnKhB25XI5JkyahXbt2aNSoEQAgLi4OBgYGqFatmtK+tra2iIuLK/ZYfn5+kEqlioeTk1N5Np2IiIgqUJUJO76+vrhx4wa2b99e5mPNnDkTMplM8YiJidFAC4mIiKgy0qvoBqhiwoQJOHjwIM6cOQNHR0fFdjs7O+Tk5CA5OVmpdyc+Ph52dnbFHs/Q0BCGhobl2WQiIiKqJCp1z44QAhMmTMDevXtx4sQJuLm5KT3fokUL6Ovrw9/fX7EtLCwM0dHR8PLyet3NJSIiokqoUvfs+Pr64o8//sBff/0Fc3NzxTgcqVQKY2NjSKVSjBkzBlOmTIGlpSUsLCzw+eefw8vLS+U7sYiIiEi7VepbzyUSSZHb169fj1GjRgEomFTwyy+/xLZt25CdnY3u3bvjt99+K/Ey1st46zkREVHVo+r5u1KHndeFYYeIiKjq0cp5doiIiIjUxbBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIq2lN2FmxYgVcXV1hZGSENm3a4NKlSxXdJCIiIqoEtCLs7NixA1OmTMHcuXNx5coVeHp6onv37khISKjophEREVEF04qws2TJEowdOxYfffQRGjRogJUrV8LExATr1q2r6KYRERFRBdOr6AaUVU5ODi5fvoyZM2cqtuno6KBbt24ICAgoskx2djays7MVP8tkMgBASkpK+TaWiIiINOb5eVsIUeJ+VT7sJCYmIj8/H7a2tkrbbW1tcfv27SLL+Pn54dtvvy203cnJqVzaSEREROUnNTUVUqm02OerfNh5FTNnzsSUKVMUP8vlciQlJaFGjRqQSCQaqyclJQVOTk6IiYmBhYXFay3Pul9/3WUtz7rfrLrLWp51s+6qUr6sdZdECIHU1FQ4ODiUuF+VDztWVlbQ1dVFfHy80vb4+HjY2dkVWcbQ0BCGhoZK26pVq1ZeTYSFhUWZ/sBlKc+6X3/dZS3Put+sustannWz7qpSvqx1F6ekHp3nqvwAZQMDA7Ro0QL+/v6KbXK5HP7+/vDy8qrAlhEREVFlUOV7dgBgypQpGDlyJFq2bInWrVvj119/RXp6Oj766KOKbhoRERFVMK0IO0OHDsXjx48xZ84cxMXFoWnTpjh8+HChQcuvm6GhIebOnVvoktnrKM+6X3/dZS3Put+sustannWz7qpSvqx1a4JElHa/FhEREVEVVuXH7BARERGVhGGHiIiItBrDDhEREWk1hh0iIiLSagw75WjFihVwdXWFkZER2rRpg0uXLqlU7syZM+jTpw8cHBwgkUiwb98+lev08/NDq1atYG5uDhsbG/Tv3x9hYWEql//999/RpEkTxeRPXl5e+Oeff1Qu/6KFCxdCIpFg0qRJKu0/b948SCQSpYeHh4fK9T18+BAffPABatSoAWNjYzRu3BhBQUEqlXV1dS1Ut0Qiga+vb6ll8/PzMXv2bLi5ucHY2Bi1a9fG/PnzS12r5UWpqamYNGkSXFxcYGxsDG9vbwQGBhbar7T3hhACc+bMgb29PYyNjdGtWzeEh4erXH7Pnj3w8fFRzCYeEhKicv25ubmYPn06GjduDFNTUzg4OGDEiBF49OiRSnXPmzcPHh4eMDU1RfXq1dGtWzdcvHhR5ba/6NNPP4VEIsGvv/6qUtlRo0YV+tv36NFDrbpDQ0PRt29fSKVSmJqaolWrVoiOji61bFHvO4lEgh9//FGlutPS0jBhwgQ4OjrC2NhYsRiyKmXj4+MxatQoODg4wMTEBD169FC8X1T5LMnKyoKvry9q1KgBMzMzDBo0SDHBqyrlV69ejbfeegsWFhaQSCRITk5WPFda+aSkJHz++eeoV68ejI2N4ezsjC+++AIymUyluj/55BPUrl0bxsbGsLa2Rr9+/RRLDKnzOSqEQM+ePRW/X1XKvvXWW4X+3p9++qladQcEBKBLly4wNTWFhYUFOnbsiO+++67Esvfu3Sv2/bZr1y6V6o6Li8OHH34IOzs7mJqaonnz5ti9e7dKZSMjIzFgwABYW1vDwsICQ4YMKTQhcHlh2CknO3bswJQpUzB37lxcuXIFnp6e6N69OxISEkotm56eDk9PT6xYsULtek+fPg1fX19cuHABx44dQ25uLnx8fJCenq5SeUdHRyxcuBCXL19GUFAQunTpgn79+uHmzZtqtSMwMBCrVq1CkyZN1CrXsGFDxMbGKh5nz55VqdzTp0/Rrl076Ovr459//sGtW7fw888/o3r16iq398V6jx07BgAYPHhwqWUXLVqE33//HcuXL0doaCgWLVqExYsXY9myZSrVDQAff/wxjh07hs2bN+P69evw8fFBt27d8PDhQ6X9SntvLF68GEuXLsXKlStx8eJFmJqaonv37sjKylKpfHp6Otq3b49FixYV+3xx5TMyMnDlyhXMnj0bV65cwZ49exAWFoa+ffuqVLe7uzuWL1+O69ev4+zZs3B1dYWPjw8eP36sUvnn9u7diwsXLihNH69K2R49eii9B7Zt26Zy+cjISLRv3x4eHh44deoUrl27htmzZ8PIyKjUsi/WGRsbi3Xr1kEikWDQoEEq1T1lyhQcPnwYW7ZsQWhoKCZNmoQJEyZg//79JZYVQqB///64e/cu/vrrLwQHB8PFxQXdunVDenq6Sp8lkydPxoEDB7Br1y6cPn0ajx49wsCBAwGo9lmUkZGBHj16YNasWYXaV1r5R48e4dGjR/jpp59w48YNbNiwAYcPH8aYMWNUqrtFixZYv349QkNDceTIEQgh4OPjg/z8fLU+R3/99VelZYZULTt27Filv/vixYtVLh8QEIAePXrAx8cHly5dQmBgICZMmICzZ8+WWNbJyanQ++3bb7+FmZkZevbsqVLdI0aMQFhYGPbv34/r169j4MCBGDJkCA4cOFBi2fT0dPj4+EAikeDEiRM4d+4ccnJy0KdPH8jl8kK/V40TVC5at24tfH19FT/n5+cLBwcH4efnp9ZxAIi9e/e+cjsSEhIEAHH69OlXPkb16tXF//3f/6m8f2pqqqhbt644duyY6NSpk5g4caJK5ebOnSs8PT1fqY3Tp08X7du3f6WyRZk4caKoXbu2kMvlpe7bu3dvMXr0aKVtAwcOFMOHD1eproyMDKGrqysOHjyotL158+bi66+/Lrbcy+8NuVwu7OzsxI8//qjYlpycLAwNDcW2bdtKLf+iqKgoAUAEBwerXH9RLl26JACI+/fvq11WJpMJAOL48eMq1/3gwQNRs2ZNcePGDeHi4iJ++eUXlcqOHDlS9OvXr8T2lFR+6NCh4oMPPnilsi/r16+f6NKli8rlGzZsKL777julbUW9d14uGxYWJgCIGzduKLbl5+cLa2trsWbNmkJ1v/xZkpycLPT19cWuXbsU+4SGhgoAIiAgoNTyLzp58qQAIJ4+fVrk6y6t/HM7d+4UBgYGIjc3V+2yV69eFQBERESEynUHBweLmjVritjY2GL/tkWVVedzsajybdq0Ed98880rlX1Z06ZNC31+lVTe1NRUbNq0SWk/S0vLQu+Zl8seOXJE6OjoCJlMptgnOTlZSCQScezYsVJfS1mxZ6cc5OTk4PLly+jWrZtim46ODrp164aAgIDX2haZTAYAsLS0VLtsfn4+tm/fjvT0dLWW3vD19UXv3r2VXr+qwsPD4eDggFq1amH48OGIjo5Wqdz+/fvRsmVLDB48GDY2NmjWrBnWrFmjdv1Awd9vy5YtGD16tEoLw3p7e8Pf3x937twBAFy9ehVnz55Fz549VaovLy8P+fn5MDIyUtpubGyscs8WAERFRSEuLk7p9y6VStGmTZvX/r57TiaTQSKRqL32XE5ODlavXg2pVApPT0+Vysjlcnz44YeYOnUqGjZsqHZbT506BRsbG9SrVw+fffYZnjx5onK9hw4dgru7O7p37w4bGxu0adNGrcvPz8XHx+PQoUMYM2aMymW8vb2xf/9+PHz4EEIInDx5Enfu3IGPj0+J5bKzswFA6X2no6MDQ0PDIt93L3+WXL58Gbm5uUrvNw8PDzg7Oxf5fivLZ5Gq5WUyGSwsLKCnp1doe0ll09PTsX79eri5ucHJyUmlujMyMvD+++9jxYoVxa7DWFLdW7duhZWVFRo1aoSZM2ciIyNDpfIJCQm4ePEibGxs4O3tDVtbW3Tq1Emlv9nLLl++jJCQkGLfb0WV9/b2xo4dO5CUlAS5XI7t27cjKysLb731Volls7OzIZFIlCYWNDIygo6Ojlqfc6+s3OPUG+jhw4cCgDh//rzS9qlTp4rWrVurdSyUoWcnPz9f9O7dW7Rr106tcteuXROmpqZCV1dXSKVScejQIZXLbtu2TTRq1EhkZmYKIdT7BvP333+LnTt3iqtXr4rDhw8LLy8v4ezsLFJSUkota2hoKAwNDcXMmTPFlStXxKpVq4SRkZHYsGGDym1/bseOHUJXV1c8fPhQpf3z8/PF9OnThUQiEXp6ekIikYgffvhBrTq9vLxEp06dxMOHD0VeXp7YvHmz0NHREe7u7sWWefm9ce7cOQFAPHr0SGm/wYMHiyFDhpRa/kWa6NnJzMwUzZs3F++//77KZQ8cOCBMTU2FRCIRDg4O4tKlSyrX/cMPP4i3335b0RunTs/Otm3bxF9//SWuXbsm9u7dK+rXry9atWol8vLySi3//Fu9iYmJWLJkiQgODhZ+fn5CIpGIU6dOqfS6n1u0aJGoXr264t+PKm3PysoSI0aMEACEnp6eMDAwEBs3biy1bE5OjnB2dhaDBw8WSUlJIjs7WyxcuFAAED4+Pkpli/os2bp1qzAwMChUT6tWrcS0adNKLf+i0np2VPkse/z4sXB2dhazZs1SueyKFSuEqampACDq1atXZK9OceXHjRsnxowZo/i5qL9NcWVXrVolDh8+LK5duya2bNkiatasKQYMGKBS3QEBAQKAsLS0FOvWrRNXrlwRkyZNEgYGBuLOnTsqve7nPvvsM1G/fv0inyuu/NOnT4WPj4/i/WZhYSGOHDlSatmEhARhYWEhJk6cKNLT00VaWpqYMGGCACDGjRtXbBs1hWGnHFSWsPPpp58KFxcXERMTo1a57OxsER4eLoKCgsSMGTOElZWVuHnzZqnloqOjhY2Njbh69apimzph52VPnz4VFhYWKl1C09fXF15eXkrbPv/8c9G2bVu16/Xx8RHvvPOOyvtv27ZNODo6im3btolr166JTZs2CUtLS7WCVkREhOjYsaMAIHR1dUWrVq3E8OHDhYeHR7FlKnPYycnJEX369BHNmjVT6rYurWxaWpoIDw8XAQEBYvTo0cLV1VXEx8eXWj4oKEjY2toqBVR1ws7LIiMjVb6E9vzf+3vvvae0X58+fcSwYcPUqrtevXpiwoQJxT5fVPkff/xRuLu7i/3794urV6+KZcuWCTMzs0KXBooqGxQUJDw9PRXvu+7du4uePXuKHj16KO1X1GeJOmGntM+i0sJOaeVlMplo3bq16NGjh8jJyVG5bHJysrhz5444ffq06NOnj2jevHmhoFlU+b/++kvUqVNHpKamKrYV9ftV9TPY39+/yEtoRZV//u985syZSvs2btxYzJgxQ+W6MzIyhFQqFT/99FORzxdXfsKECaJ169bi+PHjIiQkRMybN09IpVJx7dq1UsseOXJE1KpVS0gkEqGrqys++OAD0bx5c/Hpp5+W8NvRDIadcpCdnS10dXULvfFHjBgh+vbtq9axXjXs+Pr6CkdHR3H37l21y76sa9euKiXvvXv3Kj40nz8AKN7YRX1LLk3Lli2V/gEXx9nZWelblhBC/Pbbb8LBwUGt+u7duyd0dHTEvn37VC7j6Ogoli9frrRt/vz5ol69emrVLUTByf55WBkyZIjo1atXsfu+/N54foJ+OaB07NhRfPHFF6WWf1FZwk5OTo7o37+/aNKkiUhMTFSr7Mvq1KlTZC/Zy+V/+eUXxfvsxfeejo6OcHFxeaW6raysxMqVK0utOzs7W+jp6Yn58+cr7Tdt2jTh7e2tct1nzpwRAERISEixbXq5fEZGhtDX1y803mvMmDGie/fuKtednJwsEhIShBAF4w3Hjx+veK64z5LnJ+iXA4qzs7NYsmRJqeVfVFLYKa18SkqK8PLyEl27di0UVNT5HMzOzhYmJibijz/+KLX8xIkTi32/derUSe2609LSBABx+PDhUuu+e/euACA2b96stH3IkCGKXlRV6t60aZPQ19dX/N1fVFz5iIiIQuO8hCg4R3zyyScq1/348WPF39rW1lYsXry42H01hWN2yoGBgQFatGgBf39/xTa5XA5/f3+1xr68CiEEJkyYgL179+LEiRNwc3Mr8zHlcrni+n5JunbtiuvXryMkJETxaNmyJYYPH46QkBDo6uqqVW9aWhoiIyNhb29f6r7t2rUrdJvjnTt34OLiolad69evh42NDXr37q1ymYyMDOjoKP9T0tXVfaU7DExNTWFvb4+nT5/iyJEj6Nevn8pl3dzcYGdnp/S+S0lJwcWLF8v9ffdcbm4uhgwZgvDwcBw/fhw1atQo0/FUfe99+OGHuHbtmtJ7z8HBAVOnTsWRI0fUrvfBgwd48uSJSu89AwMDtGrVqszvv7Vr16JFixYqj1ECCn7fubm5ZX7/SaVSWFtbIzw8HEFBQejXr1+pnyUtWrSAvr6+0vstLCwM0dHR8PLyKvNnkSrlU1JS4OPjAwMDA+zfv18x/uhV6hYFX/6RnZ1davkZM2YUer8BwC+//IJ169apXffz8vb29qXW7erqCgcHhyLfb87OzirXvXbtWvTt2xfW1tZKv4OSyj8fV1TU+y0/P1/luq2srFCtWjWcOHECCQkJijs2y1W5x6k31Pbt24WhoaHYsGGDuHXrlhg3bpyoVq2aiIuLK7VsamqqCA4OFsHBwQKAYhzAy3e0FOWzzz4TUqlUnDp1SsTGxioeGRkZKrV7xowZ4vTp0yIqKkpcu3ZNzJgxQ0gkEnH06FGVyr9MnctYX375pTh16pSIiooS586dE926dRNWVlZFfvN42aVLl4Senp74/vvvRXh4uNi6daswMTERW7ZsUbmt+fn5wtnZWUyfPl3lMkIU3MlTs2ZNcfDgQREVFSX27NkjrKysCnXll+Tw4cPin3/+EXfv3hVHjx4Vnp6eok2bNoW65Et7byxcuFBUq1ZNMf6kX79+ws3NTfGNt7TyT548EcHBweLQoUMCgNi+fbsIDg4WsbGxpZbPyckRffv2FY6OjiIkJETp/ZednV1i2bS0NDFz5kwREBAg7t27J4KCgsRHH30kDA0NFd8i1f138eJlrJLKpqamiq+++koEBASIqKgocfz4cdG8eXNRt25dkZWVpVLde/bsEfr6+mL16tUiPDxcLFu2TOjq6op///1XpXbLZDJhYmIifv/990Kvo7TynTp1Eg0bNhQnT54Ud+/eFevXrxdGRkbit99+K7Xszp07xcmTJ0VkZKTYt2+fcHFxEQMHDhRCqPZZ8umnnwpnZ2dx4sQJERQUJLy8vBSXk1UpHxsbK4KDg8WaNWsEAHHmzBkRHBwsnjx5Ump5mUwm2rRpIxo3biwiIiKU9vn0009LLBsZGSl++OEHERQUJO7fvy/OnTsn+vTpIywtLUV8fPwrfY7iWc9ZaWUjIiLEd999J4KCgkRUVJT466+/RK1atUTHjh1V/r398ssvwsLCQuzatUuEh4eLb775RhgZGYn3339fpXaHh4cLiUQi/vnnH6XtpdWdk5Mj6tSpIzp06CAuXrwoIiIixE8//SQkEono1atXqXWvW7dOBAQEiIiICLF582ZhaWkppkyZUuzvVJMYdsrRsmXLhLOzszAwMBCtW7cWFy5cUKnc8y7dlx8jR44stWxR5QCI9evXq1T36NGjhYuLizAwMBDW1taia9eurxx0hFAv7AwdOlTY29sLAwMDUbNmTTF06NAiBwwW58CBA6JRo0bC0NBQeHh4iNWrV6vV1iNHjggAIiwsTK1yKSkpYuLEicLZ2VkYGRmJWrVqia+//lpkZ2erfIwdO3aIWrVqCQMDA2FnZyd8fX1FcnJyof1Ke2/I5XIxe/ZsYWtrKwwNDUXXrl2VXk9p5devX1/k83Pnzi21/PNLX0U9Tp48WWLZzMxMMWDAAOHg4CAMDAyEvb296Nu3r9IAZXX/XbwYdkoqm5GRIXx8fIS1tbXQ19cXLi4uYuzYsUpfTFSpe+3ataJOnTrCyMhIeHp6Ki6FqlJ21apVwtjY+JX+5rGxsWLUqFHCwcFBGBkZiXr16omff/5ZyOXyUsv+73//E46OjkJfX184OzuLb775RvG+VeWzJDMzU4wfP15Ur15dmJiYiAEDBiiCsSrl586dW+w+pZUv7rWV9Hhe9uHDh6Jnz57CxsZG6OvrC0dHR/H++++L27dvq9z2lz0PO6WVjY6OFh07dhSWlpbC0NBQ1KlTR0ydOlUxtk3Vuv38/ISjo6MwMTERXl5e4t9//1W57MyZM4WTk5PIz88v9BpKK3/nzh0xcOBAYWNjI0xMTESTJk3Epk2bVCo7ffp0YWtrK/T19UXdunUV79PXQfLsBRIRERFpJY7ZISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQEb3k1KlTkEgkSE5OruimEJEGMOwQERGRVmPYISIiIq3GsENElY5cLoefnx/c3NxgbGwMT09P/PnnnwD+u8R06NAhNGnSBEZGRmjbti1u3LihdIzdu3ejYcOGMDQ0hKurK37++Wel57OzszF9+nQ4OTnB0NAQderUwdq1a5X2uXz5Mlq2bAkTExN4e3sXWmmaiKoGhh0iqnT8/PywadMmrFy5Ejdv3sTkyZPxwQcf4PTp04p9pk6dip9//hmBgYGwtrZGnz59kJubC6AgpAwZMgTDhg3D9evXMW/ePMyePRsbNmxQlB8xYgS2bduGpUuXIjQ0FKtWrYKZmZlSO77++mv8/PPPCAoKgp6eHkaPHv1aXj8RaRYXAiWiSiU7OxuWlpY4fvw4vLy8FNs//vhjZGRkYNy4cejcuTO2b9+OoUOHAgCSkpLg6OiIDRs2YMiQIRg+fDgeP36Mo0ePKspPmzYNhw4dws2bN3Hnzh3Uq1cPx44dQ7du3Qq14dSpU+jcuTOOHz+Orl27AgD+/vtv9O7dG5mZmTAyMirn3wIRaRJ7doioUomIiEBGRgbefvttmJmZKR6bNm1CZGSkYr8Xg5ClpSXq1auH0NBQAEBoaCjatWundNx27dohPDwc+fn5CAkJga6uLjp16lRiW5o0aaL4f3t7ewBAQkJCmV8jEb1eehXdACKiF6WlpQEADh06hJo1ayo9Z2hoqBR4XpWxsbFK++nr6yv+XyKRACgYT0REVQt7doioUmnQoAEMDQ0RHR2NOnXqKD2cnJwU+124cEHx/0+fPsWdO3dQv359AED9+vVx7tw5peOeO3cO7u7u0NXVRePGjSGXy5XGABGR9mLPDhFVKubm5vjqq68wefJkyOVytG/fHjKZDOfOnYOFhQVcXFwAAN999x1q1KgBW1tbfP3117CyskL//v0BAF9++SVatWqF+fPnY+jQoQgICMDy5cvx22+/AQBcXV0xcuRIjB49GkuXLoWnpyfu37+PhIQEDBkypKJeOhGVE4YdIqp05s+fD2tra/j5+eHu3buoVq0amjdvjlmzZikuIy1cuBATJ05EeHg4mjZtigMHDsDAwAAA0Lx5c+zcuRNz5szB/PnzYW9vj++++w6jRo1S1PH7779j1qxZGD9+PJ48eQJnZ2fMmjWrIl4uEZUz3o1FRFXK8zulnj59imrVqlV0c4ioCuCYHSIiItJqDDtERESk1XgZi4iIiLQae3aIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIq/0/aNuF5S+4oysAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plt.plot(np.arange(len(epochs_x)), epochs_acc, marker = '.')\n", + "plt.xlabel('epoch')\n", + "plt.ylabel('test accuracy')\n", + "plt.ylim(0, 100)\n", + "plt.xticks(np.arange(len(epochs_x)))\n", + "for i, txt in enumerate(epochs_acc):\n", + " if i%5 == 0:\n", + " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", + " else:\n", + " pass\n", + "plt.show()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "speck-rescnn", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 8a69b7281825e500d892ddf7566ffc14f9e8d1bc Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Mon, 29 Apr 2024 21:07:12 +0200 Subject: [PATCH 061/379] baseline exp. set B --- .../baseline_exp_set_B_training_metrics.npy | Bin 0 -> 172944 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/test_nonsequential/exp_set_B/baseline_exp_set_B_training_metrics.npy diff --git a/tests/test_nonsequential/exp_set_B/baseline_exp_set_B_training_metrics.npy b/tests/test_nonsequential/exp_set_B/baseline_exp_set_B_training_metrics.npy new file mode 100644 index 0000000000000000000000000000000000000000..eafbf40a061d3e9f1e60476b58a21fd33ee6d88e GIT binary patch literal 172944 zcmeF$RhV4Ixd!U71I`X7JDez)pu^yhgBme2bBmdoTg=SdVrFJ$W|l_G%zQkx-hHmm zbFN!AuX|zDSFnD`LkgBCP`J!{DJ@dkepsz~)2dBA{AcQi|E~S)YSlB2=d4_>Y4uNzH?38i@q#pMNhU`tv?N z#ZU7y{477<=lDPTJiov%@=N?OzrwHbYy3L@m*3zw`7M5%-{E)pJ$|1*U?%>MKVm9> z%%AY5{271FU+_czlE31w`5XS0zvJ)u2mXb1)}!F*oxtFY_@!3$P#yu`r9UD2uT;ORywMu{6uDEX%PxE3hIfF~-WQ z!m6ys>a4+1Y{k}W!?tY4_UyopjI$FvvkSYj z8@sayd$JdMvk&{SANz9v2XYVxa|nlW7>9ENM{*QLa}39F9LIA46P(CNoXjbl%4wX= z8Jx*koXt6$%Xys71zgBQT+Ah0%4J;6613bt>Jj^3J%40mv6FkXNJk2va%X2)>bY9>^UgBk5;Z84j-r{ZE z;a%S2eLmnrKH_6O;Zr{2bH3n9zT#`X;ak3A%6rj&yvO(XDSn!t;b-{)Kga*!{}mEn z@cWDW62Hu^@T>e9zs~>VH~39{i{IvV_+5UF-{%jQi9h6zn93jXC;TaY#-H;S{E)xo zulQ^JhQHg@*Dgnzr}C!JNz!c$M5q8%)}q^ zM@;39`4j$>KjY8&3x3F7@>l#df5YGMcliSA5Mke9L!C*3D56$+|hpAz3#^IV9`mD2GfD zDL>?on93jXC;TaY#-H;S{E)xoulQ^JhQH3! z4w)j7b+a5YGmTl8mD!k`Ihd2Vn45W+m-(2V1z3=USeQjvl*L$_C0LTBSej*6mgQKU z625(tir0S#_FuWnykgzti!sj$NFr*hHS*fY{I5&#^!9nmTbk=Y{Rx}$M)>N zj*PPtJF^SBvKzaz2Ya#?d$SMwvLE|%00(jq2XhFAau|nm1V?fdM{^9vavaBV0u!9b zNu10noXTmO&KaD^S)9!|oXdHf&jnn_MO@4!T*_r!&J|qARb0(AT+4M_&kfwjP29{a z+{$g-&K=yzUEIw*+{=C3&jUQjLp;nQJj!D{&J#SzQ#{QxJj-)D&vahkMPA}%Ug1?< z<8|KPP2S>d-r-%|<9$BhLq6hTKH*b7<8!{?OTOZ3zTsQGW3q0Jf=JfQQ4YzvIm#hf zH%B>Sib&SYa>&dyW?@!lV|M0XPUd26=3!puV}2H3K^9_R7GY5qV{w*XNtR-1mSI_z zV|i9!MOI>rm05*VS&h|MgEd);wONOCS&#MEfDPG*joE}v*^JHEf-TvKt=Wcc*^cem zfgKrVCw68Rc4aqqXAkydFZO01_GLfz=Kv1mAP(jb4&^Wo=LnAED30bBj^#Lx=L9A= zk&`%?Q#h5=IGr;%le0LRb2yjtIG+o+kc+sOOSqKFxST7vlB>9yYq*x{xSkuhk(;=g zTey|mxScz=le@T^d$^bTxSt1jkcW7fM|hOSc$_DAlBal@XLy$9c%JFJz>B=Z%e=y? zyvFOi!JE9r+q}cOyvO@|z=wRq$9%%4e8%T|!Iyl+*L=gbe8*(n90ifAo1+|(b#s(M zvTlxY$P|&Ro8^$1Y0Sc`%*O1@!JN#++|0wg%*XsJz=ABq!Ysm~EXLw2!ICV+(k#QW zEXVS!z>2KI7%Q_1tFjuavj%Ij7HhK(>#`o}vjH2j5gW4!o3a_3vjtnS6)0*Ks{Ja3eQy zGq-Rnw{bgna3^!V%Px*|``GPO`im&;GZ~2bNx;Y9WSvN;HB?WG&Wa9oA(%)@K7YWFt0a6Eau{Zm$FZ;1S2XG(72otoWfJjBC1!lOLK<2=EWJjK&I!?Qfc^GxRjUgRZS z<`rJ$HD2cp-sCOb<{jSUJ>KU7KI9`l<`X{UGd|}FzT_*u<{Q4{J0|PqD2Qa;9OaO# zo1+|(b#s(Mrif(SEQicYV-{v*HfCoI=43ABW*+8cKIUfu7Gxn7W)T);F&1YDmSicG zW*L@cIhJPyR%9i{SeaE=mDO0CHCU6iSetcNm-Sem4cL&4*qBY&l+Djng@UGdYX1IfrvOkMp^J3%Q7kxr9r(jLW%#E4hlRxrS@Gj_bLB z8@Y*_xrJM~joZ0{JGqOyxrckXkNbIm2YHBxd4xxKjK_I`CwYped4^|sj^~-q3%tln zyv!@S%4@vN8@$O|yv;kj%X_@f2Ykp!e9R|&%4dAe7ktTAe9bp}%XduH%~24^x;e@r zSvN;HB*jHnZ_*4%52Qe9L&jF%*{N^%Y4kw0xZZvEX*P-%3>_e5-iD5 zEX^`3%W^Ew3arRVjIlDSuqvyuI%}{dYq2)#urBMdJ{zzh8?iB)uqm6dIa{zLTd_6U zur1rMJv*=?#_sIFp6tcm?8Cn7$Nn6^fgHra9KxX-#^D^nksQU*9K*33 z$MKxN1SfJ5Cvys?avG;|24`{>XLAncavtY%0T*%+7jp@hav7I%1y^zvS91;5avj%m z12=LLH**WOavQgE2X}H8cXJQ-av%5e01xsI5Az6*@)(cv1W)o5PxB1V@*K}IofmkK zmw1_1c$L?9oi})sw|JX(c$fEhpAYzukNB8R_>|B1oGojI73xtN=In3wsOp9NTug;fCD**gE@plIgG!UvoI^OF*|cGCv!13^Dr;-F+U5iAPccDi?Aq*u{cYx zBulY0%djlVu{bWw6FajDyRsX*vj=;!7kjf0`?4SVa{vc&5C?MzhjJK)a|B0n6i0Im z$8sFUa{?2b$Vr^cDV)k_oX#1X$yuDuIh@ORoX-VZ$VFVtC0xp7T+S6-$yHpM$W7eLE!@g&+|C``$z9ydJ>1KE+|L6%$U{8LBRtAuJkAq5$x}SdGd#<4JkNAq z;6+~IWnSS`UgLG%;7#7*ZQkKs-s62f;6py*V?Nh8VP1%gi*@7+E zimlm(ZP||P*?}DyXD4=M7j|Vgc4rUvWH0t+ANFNG_U8Z&?yQj^_j>IFXY$nNv8G(>R?oIFqwDn{zmq^EjUixR8sum`k{n%eb5?xRR^5nrpb0 z>$sj9xRINH=Xjp! zyugdR#LK+GtGveRyuq8i#oN5YyS&Hye87i%#K(NXr+miee8HD|#n*hpw|vKB-5dpx ztec}8l67;GL$Yp;a>x{stefSKnQ6?ztjxyj%)y+@#oWxpyv)b^EWm;+#KJ7XqAbSZ zEWwg2#nLRpvMk5)tiXz_#272H3ahdjtFs1cvKDKz4(qZW>$3qHvJo4z37fJRo3jO5 zvK3pi4coFE+p_~ZGR{uy%r5N8ZtTt;?8#p2%|7hQe(cWy9LPZ&%pn}gVI0m89LZ4} z%`qIyaU9PHOmHG6aWbcHDyMNeXK*HGaW?00F6VJR7jPjLaWR*0DVK3MS8yd)aW&U) zE!S~9H*h02aWl7YE4OhwcW@_naX0sHFZXdj5AYxl@i33@D39?tPw*s9@ifoyEYI;g z(|Lgxd5M>Kg;#lv*Lj0Cd5gDshj)38_xXSi`G}ACgira5&-sEc`HHXkhHv?f$+|fT zB3U;_IV9`mD2HU-9OaNHB3U=fAv4pMg;|-6*_nemnTxrZhk2Qg`B{JkS%`&Mghg45 z#aV(SS&F4uhGkif8n5#PZ}Jvz^A7Lw9`Ex3AMz0&^9i5w8K3h7U-A`S^9|qf9g}r) z6hyLaj&exW%~1}?x;e@rQ$(_EmP2NyF$=RY8?!S9b21lmGY|7JAM>*S3$hRkvj~f_ z7>lz6OR^M8vkc3!9Luu;E3y(}tjsE`%4)368m!4$tj#*C%X+NO25iViY|JKX%4TfN z7Hr8@Y|S=o%XVzf4(!M{JFzpnuq(TCi2XQcma43gyI7e_K zM{zXAa4g4hJSQ;0iJZjAoWiM`#_62FnViMhoWr@C$N5~qgiSA5Mke9L!C z*3D56$+|hpAz3#^IV9`mD2GfD$+}q%nVH5c%*t%c&K%6iT+Gcp%*%Yt&jKvSLM+T8 zEXram&JrxiQY_6fEX#5%&kC%_N{q2GtFS7ou{vw8CTp=a>##2Cu|6BHAsewVo3JUH zu{m3?C0nsI+psO$u{}GmBjfDE&g{aj?8ffw!Jh2J-t5D^?8p8bz=0gZ!5qS&9LC`s z!I2!r(Hz6E9LMpTzyv395+`#Cr*ayna|UN}7H4w~=W-tBa{(7}5f^g_mvR}Ga|Ks& z6<2c&*K!@#a|1VW6E|}Uw{jb|a|d^F7k6_H_i`Wi^8gR>5D)VRkMbCg^8`=w6i@RE z&+;74Go2TBk(YRxS9q1zc%3(Rlec)AcX*fgc%KjWkdOG7PxzG2_?$2JlCSuhZ}^t) zn5>(lAd+=+ltZ#^j&exW%~1}SB9e8p95OSFS(ugCn4LM8lew6id6<{^n4bk$kcC*7 zMOc)@SezwTlBHOhWmuNwSe_MFk(C%@WmaKTR%3P6U`^IyZPsC3)?V$^ zHe++PU`w`QYqnuqwqtvCU`NK;iJjSnUD=J@*@HdVi@n*0ec6xwIe-H>h=VzVLphAY zIf5fOilaG(V>yoFIe`gIZs!i}!9`5Bn?&kp>@Fs8ZHt+B*@9{n#@F5@ZF`w`$pYb_g@FidIHQ(?p z-!WM?M?oa(<|v0`-5lkRtec}8GDRfoW;tYL8nZAfvoSk!Feh^{H}fzr^D#dQupkSu zFpID#i?KLMup~>dG|R9o%dtEwup%ol#>%Y1s;tK9tihVB#oDaHx~#|gY`}(W#KvsG zrfkOMY{8an#nx=Ywrt1t?7)tUvlBbB3%jx#yR!#-vKM=^5Bsto`*Q#Xau5e|2#0bQ zhjRo+aui2%499XD$8!P`oXAO>%qg78X`Id(oXJ_7%{iRQd7RG$T*yUS%q3jPWn9h` zT**~j%{5%hbzIL4+{jJb%q`r?ZQRZs+{sl%p*L?V?53iJjqi$ z%`-g9b3D&dpRbJzD-r!B%;%(mHUEbq;KHx(>;$uGHQ$FK!zTiu~;%mO) zTfSqmZjORT*3D54$+|hpAz3#^Ib@1R*3ELr%rs_UR%T;%=3q|dVs7SPUgl$d7GOaZ zVqq3xQ5IuymS9PiVriCPS(amYR$xU|VvLnpg;iON)meizS&OwAb*;yu{1A!mGT->%766yv5tR!@Io4`+UHMe8k6m!l!)3=X}AJe8ty% z!?%3LWZfJEk*u4e9Flc<Z#^j&jHpk*u5LkeO-B!mP~3?99QO%*EWy!@SJL{4BtN zEX2Yr!lEq3;w-_EEXC3+!?G;L@~ps$ti%{AvkI%S8mqGgYqAz=vkvRB9_zCK8?q4_ zvk9BB8Jn{OTe1~fvklv_9ow@5J2K8r?949g%5Ln=9_-0p?9D#x%YN+70UXFd9Lymc z%3&PN5gf@;9L+Ht%W)jf2~2PzCvh^Ta4M&9I%jYuXK^;?a4zR@J{NEy7jZF{a4DB@ zIahEcS8+Aha4pwyJvVS8H*qt!a4WZQJ9ls=cX2oOa4+|9KM(LA5AiUM@F)!-$ju|EfJAO~?U zhj1u|aX3eCBu8;H$8apiaXcq5!HJy2$(+KeoW|*#!I_-J*_^|G!IfOa)m+21T*vj?z>VC*&D_GR+{W$P!JXX2-Q2^y+{gVqz=J%*!#u*HJjUZZ z!IM12(>%koJje4)=LKHmC0^zgUgb4j=MCQEE#BrG-sL^s=L0_EBR=L6KIJn$=L^2% zE57C%zU4b6>*gqkWZfL)kgS`d9Flc<ZS7WZf)>%uHhzW@R>JXAb6MF6L$)=4C$S zX8{&uAr@v47G*IOX9<>MDVAm#mSs7XX9ZSdCB|5pRalkPSe-RkleJizby%16Sf35p zkd4@wP1uyp*qklclC9X9ZP=FW*q$BOk#TlnXLey%c4K$;U{Cg9Z}wqd_G5nz;6M)I zU=HC>4&!i+;7E?*XpZ4nj^lVvV1g4liIX{nQ#p;(IfFAfi?cb0b2*Rmxqu6~h>N*| zOSz28xq>UXimSPXYq^f=xq%zGiJQ5FTe*$fxq~~oi@Ujpd%2JMd4LCbh=+NEM|q6L zd4eZ-il=#oXL*k2na&Hm$Vb5JG{$#yw3-G$VYt4Cw$6he9jkq z$ya>MH+;)?OxDd&5XrhZ${|@dM>!Mm%+4Il$z06MJj}~{ z%+CTW$U-d4A}q>cEY1=v$x#;r? zupt|U62#@j@ zkMjgi@)S?=4A1f$&oiADc#)TQnOAs~*La;bc$2qyn|FAZ_jsQV_>hnIm{0hW&-k1# z_>!;qns4})@0hHcqac!XbCg4}ZjN$D*3D54nIe*Pvm7!rjaitL*_fROmghGRL7<2iu|PUIv`<`holG*0IX&g3l4<{ZxDJkI9=F61IE z<`ORDGA`!|uH-7N<{GZ$I<{6&lIi6=aFYqES@iMRQDzEW6Z}28>@iy=9F7NR^AMha`@iCw9DWCB< zU+^Vg@ipJ@E#EO&H%CDv>*gqjWZfL)kgS`d95O{D>t;D*W*W0FE3+{>b1)}!F*oxt zFY_@!3$P#yu`r9UD2uT;ORywMu{6uDEX%PxE3hIfF~-WQ!m6ys>a4+1Y{k}W!?tY4_UyopjI$FvvkSYj8@sayd$JdMvk&{SANz9v z2XYVxa|nlW7>9ENM{*QLa}39F9LIA46P(CNoXjbl%4wX=8Jx*koXt6$%Xys71zgBQ zT+Ah0%4J;6613bt>Jj^3J z%40mv6FkXNJk2va%X2)>bY9>^UgBk5;Z84j-r{ZE;a%S2eLmnrKH_6O;Zr{2 zbH3n9zT#`X;ak3AvTlxoNY>3!4#~PX${|@dM>%AQNY>4A$jmfmVOC~icIIGC=3;K< zVP58AeimRs7Ghx*VNn)iah707mSSm^VOf@Ac~)RWR$`2mS%pkyCwJJBN{>krO{`rr;P5tv< z|M9EernXQ0|M%zfpR_hM`G?<3`^5O4fBNkVQ}SkeFT=FrML(Tk;yWLIBg51U_fs=W z@6s`xPoH)4a~Z#5_g;P_9RKV`?`N3!#^BFn82?Jvl<@m2Pd*h~+%y=!GxtU2PtM2w zF*aP6*fS#-Yj-`IkJXqM?n@c+hYvFDOB5#_67XIvNi)z;wu``oFE-ez1MFM1+scdpXmy{S)* zhU*d~JA~)P=l?Eh=Yh%hGpSy%*7VHhGujvLRV7>>Te2X^eagP@o|HokzmRc0rBb=5z5Vw^ z?fZG@sNHwlKg)Q}hRau?_7*B0j>o=w>&qGU#ov7D{R~r1GGp+f6AT7(YT)6 z^3{y<@hd~ab5g3+iQc=dTDUG&w$PU{&c}NGBIF+b*7|T=;%h0tXjeje(m(&p z3mNT6k9{kQb9%>OQEoq28qL37{XUFSdZulmzo~!vY&34w8=eZ|_4mC|zUi}~`L(e| zXlLxFXP#!9Px)YeXh%w>W#N4K59VA7$5Y>ra{YFvX#PH`8qTLJsQWPEeEc^@Lwh#t zDj(Kc{86pD8OKw{d@&lgrBB28*f+X{exRb)&h!^P;-jn*5*Wr2bb48==X%uVH{#@Uq$Qf#)a_y#Kj(A-KKngdvw0ff#`k3t=qpx?MR7li`MNVY zTl%MtugSPRvF5vBKE%&o2-l@9{7>jtqQ>c68RyeC{4C@W%kt`S#_tXH%Y^xn@~B3b z*Qss4zAd~r-sOMOGR{A09CA%*IV3zM<@%Sx`{HfahV_y1@$T>)Nolb*T5tK6gz?<4 ztz&w4-ALs-X&DNjRA>22d-y{6wB`+mlAV$=T`a!$|xFk0WQ zhKG5Z_{#2(Q+nGq;d$wQIuKnyctQ9bJ5n%Ouif4Y;~8)J@qvu?#|v+{68bUjyTSDQ zyQ1}4V@ouC#ny#+ke+vInD>cwHKO@3xl8yiq@L^+a!aJm2>nkP_RVO2$n@th-{Zgk zMW~C?zW$eJ{ST@b#yz!kwBN`2d_S}&7Aq62-(URaWXSnD|BmKk!_J`}iCRAi^C<1k zXQKCX`9kzv{a1rff2B@(Kg^%BsRzS2r~h(6w630i5bZOutYKaxKI$IL>+_?-x{PmK z7M`E-L6l$Y+#jc9yg&8Pi_pLSt6SoWeh}^FU-%#x|MmOP{C)3wFrIB<^d0)y&%(a) zf8Y7om48GzCH@iSXG-oRA;+|hwL?E*2l_|`zXYYjT;zvJ@^1j#nVrcgdj)do@e)N+tkJ7(8H~J1w?ia0#;v+&kQ@vd--LCM znE$iL{KdlfrWDx{o|o2PeCS`|Y}K${QhxX4u+Cz+x`cHRoBo?0rDcrg7q*1`A~E7? z=feE@>H6rt-&_y(#W(!@TE_8s_QoOC^q>DQv_EY^)o7d!|12C&{nYrV|Ia>-+F#_? z(faFt$okt7&D%qdqVf1^(P-Y6x%Rb;cBHMT8}^ad%9Lmwj>;X@XS~_Okaw*2&EpyE zNXa`P^e-jlZnVF2svGi-|F&q@M^k^aH?%Lk-n=uR-9H!`u8V#DS+wtzTo7`KckL4G zr!A+3@r&&*AKH^Lqg2Qtk>hT(|9$I&=sBf&MfKL-I*0j_I-_aW7vhI^NBzF``OwaI zo>-XoDSZ}3>+DmXIg-(i)I!5Te-cafhdfjNa6Igj8{Uo&Ii`;J?VixyU!+_O{rpp< zFyB&6PK(yhtd?Qi<3HLIa!4QXKOwJpk!)MSec$LB^=DqyFz-^1?+Wcs`Nqig;r%}w z7III?H!IpdJM}-BaXeP@Kg)t^s)csMzf?Z-C%yXC!x_gDvpcTKaKpnN917Rf&mPuG ztbYAyKYlYXdT*_0KTZ8-;cz^5t>f5?>r*Qq4*N#z!9Sww8~rF6-`Joq?y)LkLS35j zJXQi{|AY_eA^YneT>vZy28z?Mvs6gyV^`!=n9q%Yb0|%repb^4Y(K{VHYH z%4l6)j78tO4}TKo?S?{s4bM-_ZlCUxBjlVq`*YF0dgMR*L;Kbx!uY4Cr_=vuahQj( zR;$CjjJ60NJ(wZnHHcBxBL zub#*j^4{=4g=n4iiii20-r#Rxd}61Dggj$4uZHnYEZrL2*Kp?6jQ6IVY%(Lm#BZ{P zd6ZaNH|&GyKYtw7S!&z%Vco?h#KL@!ogNqZpL+K3`)QxtmvZ!PQM>A_2=gTM8|_0* ziFtFvb?H4W9}Lg`)W|j&rdZD@g$sT??USF0Z~rA)m-D0XNvXN;hZ)yx_}Yxg;ri6~ zLc3G`cqH0y9`xxFp7+TL3 zCKl$6&d>N|v@d-sSJ-FLW`1u&#`9y@o`w7p4PS*kQXUSE-dCVbv@X9CkNUkSd)VjF zKXo&_H?_>zZ)Nl=W$nFap1$};^!==RC%ivx_w2C0q-TzY{-pi$qj24ZtT)2%_y@z+ zWV}B;$C5CQV!xdm+Ld_qFpO_%mC4b5y)mlC;@|(}xp4h=c87MPRC^cR7ym+;Xul{o zJz76`ejI(zo)-xFV|qt_AN@(wF#lpz_K(SUPU_>_p&hA@8%FPa_palAKRF(&_#mve z)Eni(`IO?XqJL+Z+GlOZ>q5scE-6b|emy*=X3=PUH2y`{?-Qw?`F`3b&q?`cMrcoL z$NF$SWpcaF{@73IhkY=m|AVNXt!sr`;ys#&+)}=rDXiO+{q>`HdNUeYpV!?>oF>N+pu{?vmbqWvuIS3({sOSgu3m)dh~^qqP0 zk1&4|pPjHNyl-5YX#RcgQph2t@pr<0l2RmdH2!}W7WSRgE48Eiv&;(Pk~+9|*bmaD zjthCG6dM-Jlh1z`?GvS=zt1PW`zo{}ajbGQpQoM*`$wYn7sLLVxLz;pXX#7oMcAxNm z)_;7#cQ$1Fj!(QE>W!3*)uR38J2Rto*|*!l@cd5?3E#_<+}{iROTU>om2o_EWAy!q zFTEDc-{s%g9iDfr<@s>F*YDm>`{cTm&we~9{4Rea>SzCT(Y(v~ING=KT?_kd%E$kX z_KWx*L+-ILABTKmFZPG_rqsV0@=X2u`e;7hC>g$|@ofj9_Wh!Lcwg%0{u9lssU@Rv z?Dvh3OZ=C$qy22fxhT&eRiire?0=$l(I@(MvDn7bp&zkKN1}X6M9)phvOBEv#Jx<> zJo;tsuz#nn{A}n){OOmXee=gJqWM3lLo{!)ek1fZHfBroUCIC9(TqG2c|RNWt&{=x zqw}r48^$s5yJw-DY5)CCw9eMl4)2YR&JxzyhR)r>c*YM`iSnqiB&>(nxiO*NiK{z9 ze`9lVgz-=LYt)|fsbxaD(+52c_oq+m5$0p;RKYN=Dfe@S`_j*UI%;>kdNfalpE#Y- z-juvW!+MGRxkfl1>l%F*5}$7z=0jrVKcji@+4n>4@o~?;llIAN@x0fTgz;~`D0=_= zVIjBp!+lY`QQ+&Lf2reoMB~2nN!UN*jjycAxG#N9>c#N>*~4;%-|sz(*84{}LmiNK z^uz7p`TtoNo|ib)@)sGuW3@jW+OwhK>}X%P{xakdo027(cM}rfd|J&vg&bq^8%O)z zwm(F5aC(h!K6UEnLLO;nnnk()`b+PpWjz1qBSRkP)8~ICy#E(FLcbF~8yChi{nI7F zxTO4LO!(gYUp<%JX;*aq%+b(}l*b!FKjYi}7VU3Sw@nG{AGS1{kA3@V`7(Z||8`?E z?@zCa*5zj^MeYA(!K&fFhJ9RDW zlPMdYMtOYtboRgvtNYgCjNLS znkStWhIyJO*(l7D#Np3GIi_C@?Mk0MYhK3pFaDV>Vcw;`+Y+tETD6X4yg%``&qd?0 zE&49T3oHozip}|2bbq&9(LO(7U#pDkQ+n)++Lx(ww2$q0IymEadTP$F-^AWN4A-TO z=pFXOlxgpUeKoa2iRR%u+~%8gLcf0y{r{fS$-ScYe>g6DU*eOxgx~R<@1G0LjkiCL zVPa>i@V!fUTsQ0^v92l6`2#z`^(haVhJ7U7Z*KIR`ts?JL!v<5*BST6KWrM!r-%2W zzwb@C7Uo&(&aXnQ@quf?zP90DqiEa?T#e5E@SEYj*y8)qzW1AYVLqk)_AgVI2=_9vR%3V&~jo!JxK zn>znW*tb%@+9<5k)WI{SW;{Q&X7lL%tvZJ1#2Y+~=HI+4YctNr1~ojJ;fCA43wgyR z4h!=q@m|enzq>Ufyf6J9WuyKz9~T|};B@%^CaK@q6y{0lsOayNv7gTf=VP4)oea;P zo;6xu3#P6P?Kzb{^f!I>h0w0_q2CJaiMRMLj87~}v*j7rC615WkzvZS@?qY^_WU>+ zrz_Q?eeT=QzgHw4tqsqM-}qM8-xCMBh5pBfmYg2${~%NNPNdcT#rtWWJTARQ@la2s ze6Q`{@Sb6_qVH6pN?|>vZr>g9NG)ICe8&0ozby>om0H91J1wd!6ZN)+_NATqXUHw} zTNR@9^x5cpocQ0jq5g|i%@>{*`|~#!XFM-;?ZBuWzy7%}o+*p6M*HH(w2*H~q3?uT zV!eulb-JNj-S^Wzxi8WEyJ23(ru}7p#&hD^@`d>t`~3%N!gE(gxg;k2Z_RN2qpM*) z#j3stoaBbh3NO^>O|+KJ`U|n zt@pL4oj0FF`(6DXgnq}5&W+|-x!Pe~B|i0)@cy(H-;e6@1vA5XNaVZrlZ^hPEM6P! zYrpAnB%{5FJL94pH!qIX!QjvTEaSQj&o(Z~IG>)Qdzgo@YUf&o{?4uz+MoDyl`!5Z zEeb~a&x18p!t*}YI(#Qma^(v1Ctm1g7|+CsqT%0TQ>&$hc^*GmEgIhr(f=ok-yIkF zoBF+J`!k*spI$0_$5Q^eH0+7M()0Z{%*XhO zkE3<`o2#M!v7U9J^;F?YTf_5rTnO_z^wkSLN^hCw7b7&hEyp zYxl4!g-)(R!_HE!F|Q+n2mtp7vGW*`z0PwT1kuu#wP1 zl062UblffZVb2Z(x5_tS8V5vYGEzqwiH{VvMJH-v{}846T1DpML0% zym$!h@KL9MKW}{!beAt{Kt6a#2GEW1m zum?(xAifGOPvcg!1f5@)Tm}5CXJ)`S=%Oz%j?6s+_K$U~gi!pddLQ)fdLMCJJr2fI zylM@3mMAD@4NKiIkB=!Z<$26W|v z<2?dCX?$P@g{unsuda;(esax!XotCJT|n-e2R@W60`OHp1<=`DF8^S(|EU-7_v~K~ z+~L3PAYGl`VM1BQ_om@5*P<5C!^+hH{3Hh4#JDK0RG=$ARTTDsr^`Y1qw}WfroMW0 z6#S(7(!R2o;ZOCmwM2hds{`Om`CX74ojU|MlfhXapFG`?#lWNCQQ)aI{625uVoFQm zw*qJ3XYlfQA*ZZUEbj8;^Ed^GPdxvdPvrRse~P!eigxwpLel{^`v&q84&**!;<|WI z&{h780KfC<$4i@Zc62H9pKf1+^kMO6^wa%iR=COM`HCQvo9;q?@tkMTo*Z2faHM}J z@T<5r1NxOOOai*8#tR}*zd#kxlVyHI_Pou3<0f6kZScW_D!uPr6N(~x$$sRI4|>Vf zdqEdIY7*uFt_nVr0WZJ@?de+UVaK#QN&GmyDfF`%+XUs+APxGtrWL_B@u*~^AAFji zpL*9PDz|17#zmbg0z6cXNr>~psX=G?{S)wH-3ox;`TIU01`qc45y{oy*5GHcU^T{% zjTuIKk?gIP%Tr8lD@OUNUhYP{JDEP2P@anhd~r7k_*w>T`VZ-T=fSV~Dy@slrjyVf zFaK_U$>%xafu5qSuQvSazP6O;xcK{ilds!fN4`$~81lhWEpnN3RX9GzMVF|E_IaXO zsHdB*`2x7z{)1hR%Y87uVt^On{e2_kiXWo&c~NN%^dcYkaF?ma*e%j0b@!9Kd2*TL zZS_vFcdJWbT=czsXh-i(PW*E21L&!OQi2}@OGxYWz4qKRaAl?67!Mxr3iJl=Qxo;w z#RgD+oBJ-teBj|S*im(%B=j3Uyc>E_lFI-`tm#GL+%OpU zsxuOFVLgM<58a;T&8i{kJ9c3<_(@DI3;d;>ugO)xXjhe34LazJWbZ`3HDgSBqA}TB z@jd|Kt+ocPFnGy|@u9c$r9|+D^q%XWhsa6iuz0G}<4t{?CqDSY-D@f2O^qOY)TZVe zO+HHzfc`VP?xQZOxsHAYccAeL@c9Ec+1$jxgV&(E=s6<7q{}tkF}`}q=MgCPnuCAb z?~bEg_N5-_p^lNncQ-iT@|p=SKCHx^c7W6P6!ZzfC9_#>F`5 znm)?Jm7x7;S+;H<;@{JOpY($7;9Ie#3iOA%7z4dvK;gTbfbdC}=m_AyiVJy`0m*0_ zgPu_TuG78++ZG%BR+s%?x5fOUH2zEL055j37UF!yk*&bwJ676ML37?KG&_&m~j(Q^981S>+kRSF#E?Groe2$AFH!Mp~+WpdI`dz2G>? zs|t5v55z(@^q1IGg~n_40rX2`-3NR{o8$1q_~65#1}+P;>jo#A9z{D9S1&Z7Xps-? zs6%g$nYb&^X~eL9l(BG?~v7rD2AFLIW7D@ zgqs7;7`?%dZG_!oBL^bXH`Y=+qiFqz-=Td0k?p3J%Tr!7@uGFT)Z=LVsAhWVXXt_M zz^4+e_vxu^fxozQ40P9XmZQGv@r3w(G0C^;{0MxhydRQW`<6vN^s<%U3)ZU{_=&T% zJq>*EHWt~3TW!vwABir2Z*;-`NdEo{#=3!A-?6^QXTjUhF6(`O^o?(Ssy{p9Ti~1Q zB(2}%S=*>MdkS5H(fgrc<_~%As0Hw zKER*r(6*6&FVsifI4emZ9` z_+9;uaT~s1jRG)_U|V-0UEXX@dMRhm0l;H_=PST>&sy-6`1BP0Vx==(FzGDl*k%*D z)7yRihg~93{`$B(Ce%N+KpynjiqJ!%>pjvZWmeur{g+m6wW9g3ZdJb>((8QLX+qgN zEzxn zjE~qo_c-9L)1-eMHO08N^DJosI1_%OJvHTn?Crr9H`WO9DTLJ?yP9gn2AU*N* zrzTY6hGINq)Bm7P#NYjp3!ZH0Jmjwl1^?3Z*c7y*3$FkktPQO*>D*IE zpZT|f9#tbr@3JkeF>ZQm3F5B`578f;yUK9W4m&y@`k47fq5rJZ1IRg>x}W%n{|9=o zp^@M}7MU3Qp_>*aI^E7k^*&VvAG+5R$_@P2e?izlWgpJ%g{v^ zZ%yO<={4j@)d{6~{&qePyFKZr=GW#MxZ(!wOR2JtsQ+6RfIhP42;e6x{{TGx`3K>3 zeh2uBC&&ms7w1j_Pnjt#`ll1p`D+&2jd~(i1JrZHN{n)bMCKFx0m0+(vYN<+abEU*b$4;D|;cz+ZS>%!_&%j*?tVT8Q!GjlD?! zr5#A+^HoN=URn-(!F+uugRfp`nlHtFlLhn{+=0e_k{V#*uH%(}FOSm>euz9e3wB#P zEWHWscC`M@=D<@1o*cQ7`agXR=)=B^gq(==e~+5-{N!`ugMb|9m-1-_xmF{_0zVe+ z<>m5}6HDnl8t*U+IIl$XrI!cqsX6^_&weh&`-MUeoN|oGfa6g zcjYe=vf$p}C-z|g?2cNucpmbnucvW|b06*LH~z4zqQYdrmCI=TMP_;le}E<47!U9& z4Ip}DegJx~4qG5EGWtB^MJ%NCbKP$NmZ=)a$y$ z=gob08@j5sCoxX)Y8vWiokYl2W$e08{Su%f4ZStTD&`FMPqoS33HfU;V;-w}#rJY~@MB5KjY0b4 z2GIyjn&;!6^8sJ+IsSW-uJgCXdIxj8|AcY@?}_dQGhH-s^{NT%Ay4)YdQN`Y34Y<5 z_R~5+jDHx)wfY8os*V(f+{-t4h#p^`lK#;*U|-a9D@T*%FtqOqMY^6l0`0O)qrev` z^u&A6nHM?_y!OADfbx4vfDW!H4<4DgIAQnOI^Ll1O}}=RNmt36px+{MFvefxItctd zS@=Ym5R%VVPY^O!0G$`}jz|0d8_3^aISZ2AcsRSdfy-lJmKwZtBJZ1M=T6C=Ce(Ec z(K)l4NrI8?_YHg~3ooB%;xcJ$z+r3==*hB{03D_Oani>PS`nX~sZ4sfQ)~DwtQhg3 z7)k4S;_?d2i(S3wyp3!-2mH^U#|GYP(o$vG<342r(C)!XXiuE44tnq>!$tzXp}xNn zURgqV=++eA%Lngg*A*sNv5|C=tN9Z_l% z_}6{S-^=C6)E;Y3mt0wT$E0Q3t^3ZyzKC{f~ zi~bkT?gYC&8#E4a)%FtXqQ2V=^mhd;M?3D`xi*;cqVzb(r)rg87ust)5`4nHo`wF@ z8~bA%`N@+MMw0&{3;Us*O4A2=T90WBy~2x6huz>`1~mixwVR2LQ@N>~%67hcsU7Go z8~p}ceK}1i@PGE~mmoR?fS#jt+_mSseDEF-t#zjv`d)mb1(asnz_MrAy z6X*WrK!08#73jr|F2OiC`{**qTF}S+>mlIDP`mEh?GN>P$z%9Q>{BW56Pq%24BEXy zewPS|`D)~q&uxHlmkr{fKfLKaFPBMAMEcuxCk4irg%sUv>Z`CxvrMS74kJ1bK7(;# zv5tXG|L|L%5BRIX&B3?)LNws3l4N(q?&P4isGpALI3pP!b>%^JJ&V`%=9ppRG17h7r$cJdq9drMvbr-tWmXpIk})h-&$n{FMut5z5cYz_0AcVbYsPdlG%>*GBtN z)_|O;4Xq&u>Q5t*^LK-0fF23cfZsgu%K(qs8?T#?_gId8um^3x@3KyjawtFc0r72) znKZt|Znrn-V!)H*2>m`m-bJU++fm++=wxHhfD&S{PGZ*Z#YPNvvaQu_Akw3gV`pbId zLg=2i7Ib1+8eyE-5%Nn!al&663cG3QdG5KrLAzTs(m3s0HvsSw+#-73KSuOTLh~Hq zDvok`rQQE5-j?jZtOcNh7!eG5>6+s(ZjOKEo-cP9xNPIjjcBi6Ka8LHkNBR&J%@I9 zz)9rmkWy|_PK|s+{ZDT9qpM`E0XUBp+3l{wUz@mSPV2{VfSu>nT1|LH>?L^<<&m&8BrtsTV>2xo7X5 zhy34FpcmDgf^;5rTNaG38u^0cwDLlPqC#Tesl2b!J>S{cQ zPsHNge-MA&8GItESH^gWHqFrcdyrX0^l7 zjW3@v^_@?>rgdf=6%2mWImvGprC$+!dVj*Ws&GYmB5O4GmbWPf`irQUs3*EKfgaN5 z%bx^(?c-s-!R)z2S$7}!UdGrDYqDuYZ1{b=KHj!U5;igdJ!na}VIHu7(hw zcApMAEMFG*i*~=0{g-pdKIw<|ucChJLy#Z2tl~nzsSybNlH-frq zt_olFnNXBlPJFa-2kbEWZz$SR)n*%5Rt`k} zMDS?97ctGdoANqoCD317Y< zS2gwBZHk~BAZ6gS**Ok)HI=Yex(DrdsP4-ap}l;xk0tBOMLT+Cfz2jeJqz1oLRmRK z_`*3CpzFtn+~{k4Adjxw?>?aX-Jhs0-_t!o^?~Gqy^jH%RP{R8_he&RO-8$gLq`Dq zvq{fQ$RFk=J(H~|_?Fd93HcJ|reK_8ty_?9xitpz$BWT-^KB%$7WHx(ifnY*~Bp5E#g%|KlRv!pqFbrt!L^+@i6X6MS-4ds~AgydLRILHnUR_ypPUPN|m|ypmIo*)(m8Xs(U&$`eqpZ|L&{20Kz3%Y& zKRL@x>*c!m>W`*>EEml~RF43R7e5;2HtDiI@w=)%>>J8uqxRUkEyTZFGLs$g^UdHg zaP;eGrW`L;8uG7?zlI#EIbX=W+*<{@>HdE3Lq z05|z=vU6Tt!GCg22jC$iXFyKmR(n3aS3;zV{T<2AI85t0I@M;3pJ>>AnStl}Is^T7 z*P(NvY-4aewEJiR;PLmG+N(wDY%(%)XOqvSS0TBXe+c$oN9T%M) zo~kk3XA+rvL!ZdXX-gR?q1d$im#Hs4W{fnU9GnmIQlk=jxlH=h%ot~}eK+XE zuMdGdX#Zz`$IlllfqHevX9Ju%@}LQIY1+TwX%B2iyxsfjCgfei_nA=Mn7jk=lq)d4 zGG7GIXCkdGkMf=9{TN>95|!Wojr7jME0A~gg7(40l%!-YXOMiV zrWGKU%%wvO99NpZq(^cEpndTr5uI~O+5a!_Yh(4{#-u(dA6%IHpi5PLo4BYM8~W9C ziQbP;RZ?Tz)%N7TM--#|dY<|v=*p-5kfxqT-&_K`fbttmC{vyj>(}qJ*ZrsO@_jAGU6Xwxpg*ah`ujNLE3{uz9t z?L5lm>)RFO5?2IWoOK`We`KS93Xyc~}D{QVE;BRxL?vc)wrzodPDCRt!W2IaGF z4>qBg&;fMgewHunJ$O-P1lnU+XS9B;)UzNpL3 zn{qruU-B#7j|1IQ!WH0Ok%#W@vz8AqK5DP^!@5~{Irz)6X&JtN;C{Ti-K4ws?|?mH=jh%M zyR<&sq{}w6Ue9XXgnz3V(tJk*?|>av4YPsXq6z6my+3s`Q%|%Wm&}BGb7|-w<`-)- z;PuY{x`-g!U*XptLav!N`7!KxDeylNzkwHjd|ozE06 zZCJ?qkMcapdAWEudhqrWKzF`B^HBqr7wq;N@S75!IqM@T0j=|xE}Bnd z-!spYbI-2_J*_^{dL!R>aG^<8b$1L${j^a9<69DdkJy1Y#ODRV764wiRP_;#Zdlla($D(c`_F-Yo%>!on8r(#PfO!u z_n+A8I?($p_Eg}ZYFIyEm)8yhPqnT~dbEY5Ur(}!ZoeGR8~hL5yCLj7bgzWK%%E|lJnowPy*%0x$lfjp)bbr!snL2Dm`4x0NLM5_(>4mevi&wo4 zcpiTM_ZQs|keKY!t_CKR=Z4zeGu8?vi3e(LYFQ>gcHz)2Ibc@pc} zdLr$ci=0Ium#SR};HM^$oQOY79+~oTMgz!^wEHJ&{0WSYyg#)V+Rc&)`D|J{=o>MB z?lZAOONl=QTe%(nb(|^h@@Y$UCt0FM6W4nq`kIhm`bGWjNB2fV9P$f9Er0NVR7Z#p z{_KkjI1^if&Z?)iOB?Nb8L3VKKOMK(Yg10tnhpJ_d+dii=!|P2XPzv4yjpllQ%)u+ zFHOjL#HD%SlX>Ld#3uhx?KnHi%c4{E%n$qF67q7ftsy`8T^?`?wG2 z;@QsyUg~2C!1bK#+5>)S-Jkkl&!x*4I`7Q4^(Q^I_$Ty?Bj4=qBFMY`MDX>?*9Chabd;*?B;h?C=BFxY##Qu2|}0Ce&-Q^)sPJN_JNl z@gqL0MeC8WS2y@!Jo#T8Q{#XR| zOO#oU@ZY+{NUwK(P5t$?blce){p2~14Kn3qk?R`~hL@-Q z_3cXfFyR7{$LaQ*o?Sl`i)j3G(RMD>>--(?Me;qg&U&Rb>Zz>X0bg}Hk9mW7M0%9T ziL*^T=35$k=hy-F?;q6xFB|c%DPlg}ItceYSc%o>uS!1~e8WnSK6EAj{uuQV{s!Ok zf!7eqt8+mQ(QP;6kk_a=-jw6%x`98~Ad-9L*Msy=&@RxyZQr#JlLPky&%D!Nmsp=K z7+2ZM4LOqsA6-WNCm+%;4SIl&ScB#Hklwu_%ISRy?WJ76L)WW zPIj=9J%>3s0qCpiCP6!@c3;>Rk&@0Oh<)Qp?{u%U(A4MWX#I~Jc;3*&^^Mb@r}};f z<1W*s?SXbHYz7_q@Kq%56CR^~I@y*zs5gk_39`LkClhCN%Y#qZHoK1B%=Zw+bFbZ} z_qv5~Hbu>+ubhTFNbecxO?_UgGTFD$)j$v4nfA|E(aX~SKbC!G?wCF2c!%yI1DJtl z*PVIAj?kxaYsU~%Pp%HcJxAgA-S4N&H|hHN*X<@0H%Ly^8`^(U{X#)!HNP+L<-hn& zQ;wHw{ucF~#v{G5p5AxjUo##z>9SfQ;6|s68M`BoU_Z+V@sca z?&9ZV$g`Nye<0GA*!$GY=-i~fvP+VqbP77RquyNwU0HLo z!@O+IaMYXg>V+9+{^1z-#q%#dHI?j$3?+Rnf=^+5yQ#0j+C%>M zT`!aq*~pHGYSy0ru0r&wkpX&2jolAAu#PmZQobP{(4WfWcc{^=u>NY%VXo*{G~X;1 z3H>8yE+Dyd3d|+HLtdKH*uascqaZh|U@6dzU+)J$SG}SA3RNHz^d>7w>u5aLjZm~7 z=Op-71h$xN;;vTA)E{~B5$LRY(LNyGNB2bBgYA14c^?4281w++#Ea2A z76)DT?eW%V!k11E7b^p1S=fb?DqI=`ce=EOM2fX>8+x4ytnWS6(oe%pvy z&~LI7*+bUW`;lo!g}pw7_8t*Ft_2@KH{Q4?_BHr|0<`Yf<|X0trSeZxUXG&kQ+z|w zvw%}vVqU|ZJq5k>{1j+U{m8M@0Yaf8xQo#6yc zzXtCm{DMfrA>$&Yi^=N`=^KjL;b*eQ|Y z(hHR98sz2jz;kB^2HkbzjNk{|W*_op2U;g&pTglUse=cgU)Y?ruoq%{71%j;=o#v> z$uXdZnz?h3|*_esQ=XQR;TeihG=)$xvC?hOCPNRD(Y}hTxr3j?+jk?uK^6NWT zeYnZa1MXh{U3AIE$3d6TIWUfD*H+L?Bo(j&&ib-A))Dt*jhmuLZba!1yt zb=+jME~PY`Z(}W@(2fYUeASxXNtfqy(tP_fy$_+P(tLs6=tq2ag8Hx9ksV~YN?tH{ ziphBZPZnx-8}SkG;omv_k={*y3u|um&aO7_$K>CbHHKew(=>EH_s8lS=yzS;Yp|zb6Y|v^yj-48q_OlXI|lN@74?%JTD0Ai zbKgmccG!~JTTI+_BVe5g#hT5fO(?%T%5Opz&(3?7H^98--@dg^=SC=>v}R|NGi5!W zSWW(v?(rCULl?K}4IOu9ae49`JajDk&f)JQ-lo1>XZ#rzGqHf3g(* z3okYoeiRxr?L~AUzM1p~@}kQ`!v3jZKCn9~OC`t!D?;c0*n=0)A8hm<;H~2=yJ7Hg zK1a{;nKy|}K^$@_W6}P#DtL9CDX)T-2bfUACpxG`54)JSd_eb{bsaiCqV9}hm`PG5+cxt)HlF^zMZ$VAtQP_=EpsmBmlN@5lUL z-}#U$hk;N2QlN)(&r1cJ1l?5os!dJ4^Zh%|;k*x9iOz?Lr?(yfPHTEMP=pnPzbq2i z`MI4>s6L76n{ul1wM__%m!WaXZP$G_6JM(v@9zTM4Ihk;3_lLPn`NFw?Ht@k@|5Ap zNtDY~8vW;I(n6oB12q3qfwb<=a~uNQgmaEJjLPvgO~9Y>KeA_h1=(r0_hZ_BJ3R~d z>iBejT2`_5fhry8Yue=le#}O=b~yQ6rD%}UvL9RM9G-)@kv+1#d^c zE5W{ppnaBv&Iz*hdEuw&p0r=(tRJ~g(E7VU(R`{t(ZlWg5PlWEOXm++IeX6d%0AGU z`Nu#W-F6>RCL{eKi!4Mxl>6~B(8sGmDHEzEWPjYLvku2%=$wrc*PFhPf9Gm@)W8=vF8(s1UP1e$`gn8V<3rRg&qDa?-gIt)#bK}q z^3UaCrXCwl>wnI^x+}>#;H6)TgI}ym(*1gN(C4iwFRR=PM0>|Jg1=OR)sORQfnVih zdtRp({f>;d$jLq*_z3-=VwGHuc4a2;n~ESm!)@>7^Wkf+n|$tT?clh*cTj#-3;09& zg*_Man)dbh*FMneN}Po~7Hi1v^Bdi1o|fSf`5*bJ1Q|Hug>SS8_3g_TN8TnD>=WO! z2L175;S-ye6aRIgdptVx@~5Vr>UodKJ+R}FkbJNCmrzFCV4^|FA~8 z&H=7{KhYia3HqH!*n3*Pqlmw+oGp)X89#yku3Q>)R5KQ%UD4;oag#4YuTC`~%W3s* z(MRAjS)dc_r4*;g&YkWKyRUu*08f#-G5XKzksecLX?|cpnUB|B27XahvJrnf?@5K0 zK!4bX8ARU!v<}K@T?UGjpVOKr>;`6-kj`_8&OMbulMs($$V+fCk=uv?8%dNrVqT)+ zeuX?a>u+k=8@lINIs@pXFJFNEWfyvoKT;$;_)T7)c-OR}QVt_M^pfuH=)#$ye^m_q zz7X$t5%lHH=8r-DHnqKldM`SFAJq?97nhIDz<**7E@K>3wTSzepS2~pDoy&ErXG(s zucir|b8zBfnG`0@hSGXGdsbE<-)9r?bM}{zb3VhKs|+bY@}8~RM${`v`)B&H*LK9W z$N7NzAMLv1lXbx+&c~HNfBD@b;B!8@H}s!udclo)Q_mkZpzMZg8pm%m z-%?XsQNIVb%LTZ7W^xlc{@lNL$e5DE4|ZK%Zu#)RBZoC}*j)qOw(r!*#MKQAxpK}gh`T$%$GpbV zc_v@f-_Q>I2n<|+@jqtQy?hH`JlSm;KOpP*!|TT-Ju-X?_*M0%dtZ(}t#l=fk9gIJ z+IdEPBCB~0__`)Mf}R!L$JUy5<=FV)z_arT8t2^y!6&X1?-rPJwn`(edhdZ;=;IxJ z8a)54yB)UYc8`!>&fNdIZsV$Pe-HZCGy?phHqkopSm-rm&q63FA_hsuk7XW z#MvqGdvyakAL!}de!y8)t)~f9?taRI&d2M#Cp}t-=F=i6L%-O4(pNIF8|1)QCuRY( z&L^_JMnCoHO&2>hg0w#0ag_H=%k z-=T5PTQdS*c04ob`y3_5n09oX*rQD-yU~4LX8i%VY%=^nHhf%DldrO6f_@PT^P@lf zChc$V?a|*%KI8o{&gNh93FZa=OkGcm@seVdKgtE1djfdF!ofG3(LMw#wQoJrHw7Ly z*#<_YR;PbizFmAeM5T%dn*ah(H{*fL^@*`^z zKFasYc9YL*k^XhfsJa31S<9Q7Q2b5z4q-}7cKkM-Z*u>j-zm}=vg9@Ss>2_^VcX9` z|G1*6PcZ3h{$s@5U#ekV>^gE7`0+Dq|Ds<0D6*e+J|HFC^W*mSOPu=#jz6x#FTOJM zoOK>o!#Om5Ju`hpd(L@?o7eUtZu!u2?hW$D&yjv*#an}3B2W64rkt!s?;NO;R8D)Z zhCH*nA-zq$7*F!dqGv6Lz2=vBVmo`>q2>n!RIs0%sr%4#^gDY%@1ck|H%ZSF z+6Ft!#@+y5I(Ck4%ytj>eE4zRgszi00Z(PJ``-4v0oz6Qc$hc&mD0XjAY=8w`0H~g zfuBgcZkK_>Qzk)wS+Mnw$9_it#8|)YXm1zo?y5J@dIDa0iKm37I=>I)X7uC@4o{y=<$^7q-uQzdQHa(0)NVCv7q1N>P@8g zU-W|AkxA_Pl~3utDE5)$L7c6DaS#jX9VOA9_D6Y^c)_L}5nd5+#k&h+C+zthm26!L zlh4B*5}z);2f2|GQqs7N&I~;vLhU^89j(u+vxy&>dak)kiSO3T1wZJp3*>h_`3Af_ zMey-X%b^ch7@ePxcK)NYEg^oHa|+|?lxGVw!;Y)!Mc_Bfug9L5_T*#2OEp-4_SN*F ziA?%G`5%xFcbW}E(GGh;eveK|@3XM9J0a)n<|>T0?0TKXdu)5?F~<(_o_>Q&yJ9rm zYhYPPzS*C~t4zA8?RyvP)gXS-DPBO|^0<-Q}{V#ienrUkmoa@grDD zk_R32pg-#6Yu*{~!y1$^p+2*&hY7{gN*G67^fl~`4p|So#U9Z<9M4AkXkycwAk@p- z^n(fch|lPsm}~dztbZUrT=Q~y^5u#U!1L%m^q2pc3VS0u(mYl-7>@NDzRvo=8xC&; z{5QW|n2_~&Gt`7);l!m#|4|(6%B>f`FQPP^Uvt)9+0daF7ZFJILfUmn(ezSWQ(r~d zb+$xaq>t;k(^n7OU63}_-w0xcLUeGyrcsB2~(gablUR7SAg!JYf3tg z!B&nxYRa)RA^T0p^Im~Il;>!DkX8JMe0ixC^e9W;X9D0p?;C1D`L;!*301DFgx~v) z|Dz`ze~hh50zLT8er0Pr4E3D#;u!m0hW|$B0TsLkp{Qp2pK}-dDfaU65>tUWzPIz5^wCDMK%H6o|>q;(Vdh3vb|{}uGnOFx1?*trdf zfxmO^e()XefgC~W>0*n0hkxrj8n-5KqD^`6aUSGN7NBz)?)`RMVhWwxQLo|?o;|Gn zEJygrzt>9r<)cF)9+02xfx>_IF8=oiPsUn zGw2)?|L_j_TcxG*=OQkhi_wMc{z1TLjGO3^33TLP5B|U}cueO{^kVvbE(gA|j?D*? zJ{IRU_crbG>g<#W-4kdYFFh|XyF77M=PAvA-{S71hc^YDhJKCJb*M?#VIy}SKc~gV z-iH%MwpB9e{HXOm3wDKmRaYyVLwbK&kNvlfKVhCU>8_9~3rr|Bl3&k?Sic}$IgFop zy|IAF|7W+Hdkl~1oqM_WpqIY{v^?y&3JIahDjPesI>cEHPd|7IJVi*aqpO7_<3Ls{%zBfzOZ_j35( zr6j)%=)SYFe#c+V1mDXghkly!tlTchxB1t6s_`r0x1;v_QWfG)+3_IsyZh^QFPEpB zp1f=x;XMU@znWzIo5`n1p6$M_3fX~tF85P=S8qq5p8Y)mm79J~*LjD|eYngAlh5y3 zJKlioFi%bErL3M;sL9ux2D~?+sz>Xw@`qjDaQwR~f6y=KT>^9vk)L1}RmKfS7tIRN z{KmO=xW>B{+HGLhMZd>jJ>1g-K2a&wShO2N_r7I6JJ0As_fK`*O3zI`bN1z&`)n(w z1CH8#0P-e+Ucg_FkM?8UC<2av9_o|*J*=Cw|Do(Tcr}UcZ?iJr2>)tpKzBaGt{?SF z0X-q`^avpXU`eBc5Nko+cg*ZU#6yacb#)P>ew#WIli0p zD<4kpin*q*qj`Q(_aFmbJdC=8uwpMC^nWeQTil0^gAZ8wQ_#;cTlRg(Ur6V&*f!D= zyl5oGL#J;9I%zu}5sM2%LH{}P{k|dKAMNbVHOpNd@CuLkWkQ~Z<~6*o_21su`%|ar zoVxB!zh5l|doMTTm0y0+Z}(3h1Ds#|(7v$#35z}eJ;dHx|MKl!j34`L&v#inqdMAt zRBr=#YG+6b1MlB_#Ln;3=NRl)vqyH^`Xq+l@f5|Uvyk1C9q+v}^>vcPw9mCS5a+7? z+5H{tCh-5;cMb54UQXjbygU3+b)_fxP_SW}OupD2KN{in8dFWE{iDVpyi50jRp%hQ z|07!+ApYGFc-GKePwY+j?P-B`3DgGqT@W^TJ;uxRtl=aB$N6MG+J9l+V=ke7-|xr& ze?Bij{T6n8PTd(c(d4VD)*snigZP7I7-iCBMf>|_wgH*R-2~~==yxW?>DaK#;x4^M z#)h?r9T2l<{YsDDMt;G8G|)fpzcG6aUTot1KY%mo`D?U)k=EB)v8Nb6-TVstY7J)YyNgfIcoh8v z|HfGl;_v8vV)2&rn27bhB+4J8eP`2z`FMkqr1w*gJYn*+|E9-if8b5%XZ@_(E~MAZ z??U?Y=AkB3(WT)($iK9|sb4gL{EIvn!Phd)f*L4q8u5Jm;Ct9@H8(Hlr{~aogWVy$ zERK4E{`%W@^oPA}0Qpzh=)SFZ*BO4R^Dc!PmK^+`tpB3zeihsDwhG#PLg#puT~BuI zg^O17t|wvT>F<)I=}_LY(O^QE#=bweWex0>GcHbf`KL7aT3h?6r;SEEmhA$@S$wee zrS}Wa?caTDTV8$lhJ3kd`=Ec~_IvmxGT;^UCoTI3dCOjNxP6chxxqN2Mb9apa0` z3EYSLmZX=&o@et=uIfSfak_lZYbGvI`hP+Cv-zz|=(NxO#^Bvu_R*gA9rp+Rlcxwi zJ(hkiNj+MHc?$2mxs0hVM|E!vcz)x+7rauxZYJ*TSO{{(7F&NS`0PoO&K8%Rg?dp_ zE1OVsjSWA@*C*ans`C@jlgFX=G5CqE&_m9-7(J^8#=*V50M1jo^V7W| z_ANE^it@JattQC}y{7jaB|4<{pKsddV^Y9gvNBshFSao^>DBPou%G-yGSm}o)1Vw5 zypH&3wS7@8%QfQn zTP?9(z;@c_y~7DIBY_mX6belDyyIVovV8r4n5}i9S)NF?j5t4a-!Dqn2;lNfqhem@SsON|-_dE#&L zfe+Z0VCWNZmd*>Xaz`MK>{a102EIs4^k!3OexiDHC%Qbd^V0tDMgdPl8_(z5lMyeX z$&a(|lX81LN;iz4^UK!n)k8Vth8?8$3J8j4To~DZJtgop`epreJ+Ue7S@5>@ch?Kj zd`VXP`qSW{>^(tN;3N1+<=a*h_*wqdRcKxz%1yz2KY4`ae|itug@5Nl570U)8~7@P zsV}!zp|Dh}_5Z)W^2_+9CN5go^{|6>Kc*qQFUiVHBKh|#4|^lZ(0p3Ywfe=^mwq?o zedYKZLac)*`g) zdVF9}lDG8D`4N0I!fAL)CVxSB`rT=0eQrkFTO*N600uV$Z6zEI_N z2zQg+(oHu*?!`u0C(seJE+g6olV5!xfXcti6>jRuKRAWqgK<{?x#=h{=)n(fMkP&o#$@%?aR%1Nrky57?892@-dnxos7yb_W z#a|R9ev3T}`j;o#3;K#Fp_ffP?binSMJ6a8Wa4Tv?Gy5p1L1!#pCLO34KR`H7@QoeIu- zGSk1%`v2-3eNB0u;1c>Himm4+?wU#VmRbEL-_EIk^gz45AAb+&ZwKzXek32UW`l03 z{yoHHw>9TXJyCuG(#4BxIOi!o(Eb!39EJ5>b*)Ehl)JST^2d8N#JGrO#nea(^1?C-tl=M40cb*UWxJQ94XQf;R6ZdDPAguMey`^@>> z?*%{59?#i3x66~P_FSuE^3_56J4hYx-9>y+6y(&CMPHimhe)@-hpyLT2s3e=o7R1G z?`q(C_VgS1>2EUtZ^yr6dtbFf`9{fWn~>$*w+-;@{Ylk;^oaaR_bu45wabw|fX=t+ z>_cNA&hw*Ry!MInCN7$v@N#)V-FhbQb}Q z0$lr^1Ka->dQ5(9ihNcr65}N!FTtO1U5X7pbZ^cJJXBL+?xKbE_f8>>$nW)FN6xQsg)^ak&jbBRE!{mG3ktMzMH9z=ShSBf3i8)sh7pK#FN4Z6XR---ZS9z-9Q=KkD zU-0&H-riYX(LcsEHRZK^H%PR!<5PVF^e1!nG3|N{|NEKjw(kS@3!+T3KvSPz8jA69 z_WeXhYbVzS{Rg~!J^)WW+TOS9k*hw^v)zZ^$d{1+q>@a(W70+YJFsU!-u&q~p7>)G ztwX8W2Wz3+pfjsX$kNj9rOMAW;sdVtAK3MO?^raix*z3V(Yl73_zvUoZ(Vm4?UTxV zqeu^Y>IJ;T89EncnlPU%uz#$9qdQ+aW>)C-Jf;ykM8SELUdJKBUS&X%DFo%ac9(eKdmfL)Mp ze!B;7(x1KV&K`b5q?kPUFttRgzB z>LL`ieXu?&wpL$k(%FF!z~hPfKz?|u`ON^|-n(`+px>>KA8wpC z>H2c&#DLd&+Y!WjT&{+AWiM|N%9!%tLtUCH6BjZ0sN8~t@RxM0#SaZVXhO#MuBkN6X6kp#LVxe9GEk?nn1d#DO-Tzqb8lc0EP! zu6o755r4?PR~HUWGI8zR`vUN%ZM|e+duI(-NHE^BrpzRFz{ehc4X@3p7he$tdzBWRweiqSbiF@ETDlkR+K%qyIy;!z}b zJOiEQ6`Q_8&h+b;Cy;|f){lO1;5P86lm>pKE2}+M``?zOCS5muvzygAStCQ)LHUDO11?69vk{{)jAY&`15X_rM-E2j4sIKB%!>dI0a24@r-vOAR^GUt=yC z{M5Wa;Nz^<=;E0GUoE2fzivtMNb#KZg_QjrEb*)e?36C^nDoUtdf(1juUA<~uX}#r zk^H{iE77jwuX*0t0zc|?Nka{OVqGBSQ+#gSM!0C1#*uM17+te5L&f1qcWdnEifJ~kD}d6`0)U7m8{_YjP~OYZhEac7<*n$kNe zVpSl1C(x7iKc71M5cXKNw(nQi^>{}wtMxfs8a!O9FG9}wRUi1z@{--}O+xc*XFp10 zNc_f>SHV-DM_93b(3ei5a`$kux3dO+L^*p-CN-qE@yOZOCfsE+J|JUH)v$+I-SVND-{A9dX1XVCuGuJBL9hQ2#ZT-AvL ze<^3*-u@n*+GfuKW*tU)bjr;`z+-dZLCD*PHlUwpeWHV#1low&|P_u4!l@rh;Sei-=d)`Wa$ z$L_}}yY9jA(RqA#zAvDIwC^Q~r;8E*|MPYoZvS+QzoXaGrc)R{89?@!&9Z*P>)Y_l znDqnsK>EEPz2#R)1JCuC-k)-{NC1CF*!??EgMJ@LSL?b3_>Qjw`E%R%0mLWyAL`q4 zOz52FkI0beg-Pd8wC<^0)*d_W{vF&)cz-GZx#qrS%A4}?Zl4Aw6tCJrK3KXk@YmJ% zUflq1VQ1)bHjnTSb7`MRM0QDO^4X~j&?k<(nwXrE^4lBkcn# zPWPr9{@@{xs-m1dzu~EZPt?ko&ZP5ZD=@#Z$)5X&cAXnY#+&PV?9jTmP+`Oy-o+uxD$xmy2XFEMeizUA8gS+McP_vi*$aLfYp@Y==>bjO? z*Dj8B`Jm2UOk8B5eFYVf;T!UYq}pym=TqK8vY5D|FP;4iwv5h!@hh^a$=3rqjzs+^ z+8=i0QlznZ)j5CGo6aTbur??!*4q0|vJdc9i|KcQ+-L9N9Id=<&jb4Xg}(Zy|Lpq# z;x4@pqe~K<#Ot=d3?AZsW{f*aO!IyHc^UjdU54)A%HCekBhI?MIBn1MrSA>Aoc9ex zszAt#nv>w1fy)ljyi~8aN_^kT^6N@E@9iF#WjERvv>(IOQ`ix4zQbV9r`bHf)2R;b zLj2;s5EJS%{;+?n)z`8}x9hUb`{X?NTJV#2+aB;(qVB{$mJa%4^deJF9Io}jglz2$ ztS^WOnPU)-r1Q*<9@4uX{D!^sDLT{StHj=6CX}(ruITFxpy$+B+J}`tVlF}t&iqaL zAGOP;K)oNN-}R4H(I&1ZbogUJw!X<26Y{!|rA;WepMd^VtID@Cagm{3Z^W{8ltK0t{E}uRI{c_eF z_!fFU%-!wm1mNMEt9JH-GT8eqJ!qZYwYWO^CAQKze-%mVzAE#6_-lF!&7)OaI$tNw z(mDqFMfVHXs^QRIYV}l%GwVb4m~Rfm@3yc?#J9@(An4-wb-V-lr|c%Z2clMv;09k; ztoo1x<(&?E!0rC9b01l>wDaUYhnAc2DqFu`6FPFEcL$Q)ebE7azgm!k_}F>R?@S}e z1^-(W^>kNyPh4lO77BQ6?Yy))twV}KEpXpgcBS*1JWU(S|9BX^7cP!xfW8y=#(gsU zBOdIpW8mm4d5DkG(76CMCo}AYGp|>(hkKiH&d24f3uu4dGL+L@q8FIB<45ypg`u~c zesX`>k72{Ef^S5AbrJ2J%>}-6`~s2J7xOq>^daH>Hw^UPd-KB%ssiH(|6YBd7ufQ8 z@Hw$qJ6u5;`EE9I<*+(t)I)k&3k6z ztOEHBGR+NY@AGZYOFiumy~eNHhP{;+Z;~CDLwutBat<{0MIUQ7?6~S!be~D=yNdF% zKK-7PXWyX<+WUtP{lni3r+eIoMjryahBUsiQp5w(Z)d(EZ;f`Fe0}o~;5zd=^>b?) zlkWIq|K4MXpz~77;Uj8}X-W6@`77Jcl;p3;(p}MCXZ?e{Oy1`I!<7f=UKLO01Nz8d zIw$J*e-=EGzP?zGbJ=(D^p>+kx7`~^uIxEhr#wrZf%vg|h3V-3g0|1m?$pIkOvn}{ zg8b;j^lq(KNb6Xt#XI=fY@%IfzDw)ruE*m(fd4;t0N=8v--utGbIN&qao%6V+^J~b ztMSFRq8)DUDckqn)R27Wr+$$6DDoS&j%Pwg4%DPn;CC6Be2qzW_V48g`yQ9`Ui6v0 z=TWZ^{eF&PubJ2RElBt0KPV)-q1qFl^Swl0X5Z6h_V?7}xxN@D^)svWC+)hBb55ZI z&7T~*%BS|*W8kwaN8mrn35UlbUMVBwUO(D9-^3j`{a5dbeNW83A0-AZZ3;XuKAYZ~^4zyCV(RHH>tLTme)7{qH_BJ@(}n?VLOPemw%gx_S{((y zMA-chmbE3;8Fbb%kRvsV=1HuYednSYt?TG5fq@u*@1x()uJi5;u=3z#`M}v<6LU$P zMYpZlOnFhN75I<8iMI>!bq9U|zYi8aXTA4EV(1~c#`<;PG=EYj>E1aHpmieQy$W_s zHTFFNyrz$bep7Ggoe@WGIdUXE#9%#y+jA-+7Ws7+eb0NR(aQ}!&UqYd-&0fV)?>ZS z8Ar9Q5%^UtwD+gBCx$%}U+F%KqyPWS*WK6e1Fn@ft{Y)~q!``Pbo7nrQIqiREytm} zMa-e6w!d4zGJU~0EnS*^_d&foKh2bP_}|^)W%d85AKq!biK~DQkXtb!(HBE6Hpg$Y zNf+mH-b0wizUQBi*7cano-_BOd8VoI|36jd9psdk?A+MYcfYguSe^5S&N_rs$3m&~{TyoTsY@`{CRl<3Y*Cqg-A4POZIPC>qi}lDqyN@bUk3 z_T}+D7vKLwmXuwHEK}LDgs7;zWs9=AB+AIXl(qUKF|rgTOGWk}>qVB5vc4@9A`!|y z*+ped*86+U^PKbgbbTJ*e}3;j?%a9J%$e=XIcLtCIio+g3+Gb01HH?p`;!0T2rur- zzdOKDFC>9)+V|#Q9lBz4?jCz-0OU7WkA}|^{E-D%-)`q|jHisdbkpdDplf>dY{(n& z$)AGH>6Yg}mu!>TokG2_bA#KVe^qEduM50~`c;l2fU63SK3C0227asTWRKt`{ss7M z{u$t%U(gA9K_~C94y(~Wz_n9J?o;hEVSfEe@-w8f_Uwm#?myw`UFYnJ#5euYI}GS0U~;y$v|B zFH_Sto(SLl%u`j-zBZ?U4??%pw5#Kh&hOsIdbFZfn_4YnNVi$_7a`P*u3?=eynGx9 z_-?y>dpdk~^Hz;U_%zuSi=NqkynAB!e)}(~CuJka^WD|}AI0v(k0JkD<|g?kbF&X( z-u2Ad=&$aar3>oIknOb)nkSi$RG{&+>^Iq!UW5IQ=)>Ft*=e4R52Sb1PY}MT)ptPd z*E0))4*80so(gbv?Qd6v(4M4yoNKc071X<$?oX+o$<9a3qjbCLlPMux<)wQF<_q?Z zSiD_te9v}or4X9Ww_Za$>!+;m6<3{j3&-c^JXXGbN<)`~9<|%s5#N}uGfu;3n zTEBsDw5tyl3~*if$(RqjaxC=Hk>2kqwg)}^>Kf!Ln{)~D=*jH zUJvu+5zzcJ*C?9>kN{S$_xntcg+uJ7(KJLD63u3lt$s{+Y2V$Wc{EDZdz zBa1=4vSo&%oJoHJ^Q$M*y=J|I{5qMMn>U~zy|-Y0s#cKPE$h+Lruztb(2|c(f9aPm zpuK(g9P}4DhZQ>s!FSL8wB6JBh<^$F?$ovn%Cr9tX?OgOwMWDJ=!X^L?)b%Sm}kZN zP&HvTt!o|^-De}(EBhVU-}+X(Ka=s}&^$}$4-vokl-94WbE8mbk7`W!m2K~edqdpR zx#xTcMUDztQvKM@+Fv&$N6WpRjBSXX-b|T~{v2F@ez@|TfcK^~?fZ0v7cvF-y6)XY zLMZz^sZYy2rZi>xgmgbS!^0s|w`OA9>RVreKB*nFF7&dZn0KFl7mW+wgD{6^f8^7= z(G2yle}DJfRLFgP4e6s~Tj`(s_g`eBFk0o1gU@OG;oJ>pavCq3=3(P4G9H z#OM25k=gY##$T0th3IO#!>0l~ccBc%RnH^2Q{YJZbfc~s&q`keV8q|5%-jy)0!aaDrtJXVrj zdH585LU$)1;e9w9Y$sw|y^fi(@Tw}UN;Ljf?2(I?|#JiTqL^{`&lyV!iBF29{bfShb*9TJHI*ePoZ4wpL>A+t_aCdetc7` z6T5ji_=5S9*Jo#b7pnVk(9KBhPID6FO;S?9SOT8x^L4iyp921EdLmnj-V=}d*Capi zOE)b<os#=!W^@DS4OGsF4+J=x_1cMZQ0O7_jA^9z*rQR8sY?5is2>lKp8@yRF5+Ke z|H=1w)X^WI7xiCehy1MM-qqbRA#bX#-_U#>sIWQI@1J7%mG4oQ8Z`fQF8R$f&FI|~ z{S@E#%5w{JC2qRV`TcQvugrhM{U0~JZ-6U5{W#tK_dn=H=&$%a=#~TXYhU_=+SBeo zprf|;cI?aC$U#IG_&$#6c>O5ayS>8D5XP(XyPduGo-0@A-|7Z<{_f=`(2k$(>WAX80Of|DW~5df?_J>=H>Y;pQyQ9r8zX?uA^CH}(6)yP%z& z<6A?Wl2GP zKXDb{sdyHek5+FI{DVYy+`D8yW}22ifbvaOqz|F5O>{;0y!$vA_rRU_{d)F`;US-E zPUp++z3BvZKr!ln<<8Linv4w&p*?rAeZo&{m&N!jbVzru4Sr+m^Z*#LEj_L5#As{a=8yC^lT_*&@CWZz=1 zG{k(nwfnJ-^tL*npTZ~nHzPo&^fOb$Z+=g_SL%P%> z=VN|*nYY6CNKWB_@imJyvTk}{Gi)hZ$6B9U^*@P zN@=IcIvH|eZ0nG7A)mXS^mL*}lzb6he9!0HtCoVU`Df_;mV_R;73$#vE)&sf`|G>K z09X9_^Si}vC+pFcWN!HHa<(nmm#O9%=>FoqX=uM~x)ylgEZm+E^4mC_51W^% zK9lWNvLBpG@1cquulgT?{!#pJsKsxOL%nyBd=bgFyq}1D-Q9(B!2?r}^cQMLR_F)( zfR8(dbbYlg-LK;Y>&1-Ua*pySt$SC!WPjwpbYXM|-JoYNuO??sqOU`85WgE+m-tHd zCBQ5DBHQuFc?SE-)HBk~4sgtf7YNSH^`HZ0`i4D#U#AD?mc8%;$ssvMU>&JJWLFwH zME$ls7cL9={pk1-gwq#7p3$Qwp`Fry?q1?OpWf)d>QD%DFuJ$Ec5<>mbq#p^jpzHF zImbY6qgK`{7wXZkmBsia;KT2efBpL!sDB*2t7q2nyyV_a{B1MoCG_C!nLD9y^ z?E4mUQqIpz)w-B(cTZ#Bzx2cJ-jNsO$I*NWy-^tt6vX#?zW|rCUNw5pn&-{+_ z96z->;rD%K3xxc3-D@R67$4pJrx5x%fBYRnUH0_DAylu>`FcWcH!CLGWa-I-74$mTcN2g*57s4DzoMR?p^q;b&&9$t+0 z@cBgIT!_wtdEMKQeD1{Zy_rn=Ayq5sceMAzE5Iwu@5%cXe`CLC+dM$y@R@oF@GlM| z``bdKe|2kkpH=P9g^-@G7a{p#G`$yTe|ew^%AF$q5vzC(_+(i>rQVc!Gj?*vHP)3&GzHyhab*%)XkS-y!H8)=Y@R2M-1afVjls2FDCm}z$?x7 zfV`^=dB~&?9TGi_$p7}z{6Ybq8+-)#YP*wP5I<#V@$lWR_;e@QRfY9yEiPdG&D3Pv zlQ8v&|0+xP8GCFQ^gR0Bv~<49{?q;L-(C-J;(b`o_$l9_5KqX7C$ay}x9oZd?WV%G zkCONoKg;JfZsJaocRqa;=Sk82>DnJX0~~jf?{jueDih-JGndH!i`Z$}i)2@=r!@i` z$tU}U#D0&)Pxehyza=?k;9RUfpKT@S1-YPq#H>Jl*XZ0x-epvCFU`Ssw!aj7v`>sUmiTL~?!d?R9jtF-`YPoTd=L0wre-b@>M`kG#rknwX+7&_lCU0K zO|qxdU$S4NDW&k;v>Af_2>rD$@_NoZ=BEHp&rGIw(7D3f&*hzEzP7Ul^dA13p)^0+ z*#0Re(<}Cqt9aio`$o|V=(N|eg!<%OgPW3p^qlN>!Jo?qxOzy=?if#Qk9%bx=#si} z2J(cwr_ISqvA<@qxAC{;lN}t(pKjgBk^!FY!g5O8NO|(#V9PJW`1wx1ynyy{eWvO% z*u#iEBlZ#Lvt(Xuy<-oc-N&b_454b91NsU7G1*hQ^mE<;yyUOYZhJI^&h748j(IYR zienv_`F!pa67>G-`Wy@BtWvj0EzWp&`aZodQT z#8=-l8tuJ0m*mI-G%oSE7eU``m%o7T`X$y!wCD`FtC;V(eW~@x!iOdFgWoBJ{?^-G zALBvIPuwTXGBLzuzssMM&)O8@F5@ZV9Dn{W){7hJ#{nLnOSuj#4>JH?oBeKz-@r5F zA@9YX?~QdHe{LT7Ytp3yo$+(aWk}U7xrf2}bNl`yT~J^5yu`N)(E7B`v0U1@WaW^q zbL|1&vGYa@M0_9Lo3BdphJTIDXN(;|deJEZv3}K_KOv{7Iz=$9t~$Llqgti`fAKrX zua|9Ehx&hz;0hg=adQ{?9{G1a#Qx1xBfBU6KJj^>M>4MZ@Kc>bIX`{FaD+!m4tHWl zU-cQRS2;&^!)W}XbIg|}%|yNH=zTJ`Z1{~3m-4Yn>9Mb{yx&*7w!kh$^pN_VM?tsb z{64mY?4Im^rNFyr-KM0B2YujuU&N1#?S`K##rntCjp~;|{pLNgt1^pL1E2g@!e2Ep z8Sk{15f#p$-Ll`{^&$I0_f&O~g9{gZ8}Rtt*9=?nSBUF0Cy^e_!*ufB;?`!qC#1Up zyk4FpyFFKj-XqoGH%IKM5_!WfI#*s#=j@u#6GcC8`+Cp==JyZv$o|)}KEO|*`&?${ zbE(#wtG8)&$nU#V|2>4#&&0p@Nn9^p3w?)t_xndcK6As4o)6_z_GP_7=%ys&zNMVw zxN7V-svYZVcw8eo@Yg`tSLl`-G~jF`I}_h!J>(hL=SOzX+yE#3gT+363ZJts-G%jL zCX-#5wBJ|!^IU*0bj7S?dls=jlzGfw`bYGmOT)})kLXiAO@@7Sqz{sIF#YFfbUyD} z`~&`>`!2)2SJEYPZUXdJ;s?RKNqRtOpQ}!E+f{x6{70vdou0_QqDOb-iH<~Y7boN0 z*=YP(4M>94me)kc=b+L~cj^PIBmYTG(u;2= zJ&2^+$^+_za{f`0o8vu*Z~3*&i^sxuzBjJx5`MVFmi!fPJ^IR)$(VNr=<+^40`VuO zV!g*>U}I*3&71=0q|W1 z@+at~Z%&K!xi^lXo_0JBCkqx0arv3m{~3(G1@tnMw9^5?B4$Zv~$hyQXA+ zV`h;(n7fzcKAn^_6#J>uWY21IlHIKt)C1$MQ)oZnucqML5mk%stJpsOVE?N2vR|U7 z-FS@CBziBz)!RHb#Qm5epnt*-qjko5UQ=S%hLA7*;lVh{*G_sXgqHm@#)|U&+A;i2 z#=YPEhJ1YQNY2|-k5O5}cNtHeyc+V45v!49DPr_kBh z<{!XUP1khYP|pG8ulKUw=S4RbgztU`y&Gs&kpDTeg!TWSZHb@R)PGILXYNRfh0wBn zjww$5Q)7S9J36XEmRhLCHiG=_vJS&|CeHuwk0r|aJD+WYrFTr?-%dk2^x67o zuWL>Eso1qNXlJB1tU&rnm*HXTLrqc={PFrcM91_+x>xCsf3*btN&c=+2+f$rgFts--JGX;hEl)KU*G(1!lzeESCrq>7VXe= z>D@E)4C(74J;NWo4~nH<9P(@R(BFVRoZb!8`E3!vk&5^prK~5%dLl_S7d8|M`9=S# z&e6P^Q-7=ne70vX`IcMMQ;+;n=uO`~72>{R&K=0t=+#{zv~xZ`h40k6^vb(dLyA%YxZemxSJR;=t7s(#dq$9arU4L?A z$Y<}_S}ugX)=TKWyFurNeq~wUtE+alJK!bqgrCTDz83lYv~pjE{py-7L{IG~e>{|n zf4$xSKHJ~8qNLAF_~)Z@GC=*m|M)qFGludy&G(H`wWB4=O_z@WuWUa(E~L9VpWcal zS&sp4Wgnte&qF(6HFLv`)3&?_er0}6B05o*-%Y&9ezdtsrqXLVoc@kM zjtIE}ZhnUCAs(IEEH94zjQxuI4X7NSf?m1tCq6^IKWku~M6Pt*$ls>RdK%;Ho8$sL zHv8vIPE`;4edKDDeKHg5jC?MVk94aS2(RYerU&8z{+ZkKwhL1#2KchSvTRSI7I!!s zzMFaEm)e(SyYqp(jz+aUhjy`ESlW}IpO`;=d^-4;Iz#@=V|O2Df_hkwu8wwt9!}My z`?+F2Bye2MDKrn_C$i5z8jm4l_vyOR{mtmyX-t<60lsGak(vA>=(U=$Js;Y6{*F^2 zG|k@wzUuV9V1C@yY2bH$(#w#06LfcJz@J!$!JyabK!4CTeZRM?-m?Z#3awTC%Hj^XME){jnJ7@joZEL;VX<FVTOn-ZzrtgkRoBvlI07KQ1cb@ox*l~}2bvWSOCa1`Jxwp5_JZAMI&o11NY8|O> zZXsRlk2X<|0cv$NwU+A73BMp;alPt zokO1@J2RPoRg~@_YknWk{;_v8>Z|ix1BBc!tInf7wX74?IYCX}UvBb)WNYq*KG5W1 z|JN0>fq$ycb8SMq*7JUQ6x}P*1*a096hIu>o(E8moBuiJsUJ&zyJVju>&*R<pz z-zD~Y^t%f2Ysc$KkEC-08Fzhs z;BN4%8Oujw{C8}j^~2?(agvVQ_T5clJ%Q9WuQA3)^b-C6omZK?*H5DV{xJM02spEq z{E|vsm8?zW8!Q68iayj29Wa5$?|0BEwd%#nA#MvLV_z%%F_~szzJy-5Q)LhRK^8r8YMCjAa$Vx{;JoXFmH@_j7#zzWr=+|7X67s2YBS4RYpQ-b7UK?dU zMt-Ie=M`92YUvYIQEoxmRv|QvN8E|}YOx%CN0uK#T;wY0pR99tf$U=CzJ&W~P`;2~ z&J|oQ=GUwTw|uYG+(^MYKdyBf)UP{rdM=cU?`lvzgzBM_c>zcCa}9FRy4p0I^Z+4G z;%A!vo*wy%d_w%wkO!gsxt>`C1o2fpCD)4P`DIp%K_ z3!Xwg*;j@(gkOL1U+C+RpCqCC>Ro!b!9T|KV`-ZnM)^MXfv!gMfuJa#s~z^k;_ptM zOQCZ$KHpNTzgAaBZ{fz0{|-H_O*X)NG7I&mC9NkP`<&#wu5{nc?3#EnKyhfTzx`+@C)Qo|K)~-Azhs!yEsX=ZPP<9X-?=3NN;i%ozHRxQ~V*&NmGI36a579 zn^r$<4f!3vcjk65ANZgY=!F~fD(!bgzN<`r&!k?LJIjGkUO$q)aR|+TE2RK$JK38> z_YTjsh==d$JfGKXU-DLn%X%~E)IG68TdcRY`Wd6xEZx>GOECo_%4A-X@tem+d| zYE4lu?>oisP;B0pqtUMOwY!ASozFErgpq&8CP~W!|Mq3cZ-LxTcgOY&Lb>nxT?0A4 zokQcAD1x8x72zAcXE&N>PPZf2f850_G|pFhz7pUj#*_BxH&S4iDR89!uHdiGuc-2w z&xHK)GyUnkKRsvNHT0wBP06pLdwOm=#9yO(6>e}h$TyblwcKO0pHg4%CVG33 z_9v!$UEqt<6Kj43@|1pVKj}}`ys#K>tI+wSpZ+1_QFn&*L?7hnhxXim@qXl6KYwEg z<>w|m0D5B|m_YLW(!RMvx~+74TnJ5k3ixO&IX%(;ja_>o{odxG}H&r-{gzW9J6fGhCW4$v0T zn0Mi0vd-Psv=03oQgs*FHN6h-RL>$mEiUPaYN_CreVh2wt^6dPa6N8yP8w%^FF@zW zebN zrS^iZvfne$_AV|D=`VZ-y3cO9E*Tr@w;Ky}3!(hf~Xde_fBN_uE_tuf>lH-`K{n}wv; zP87jUe_Mmr-Dk=0n=Ruh`zn)iFYsUNu+>NP$UoS=nlsU!AIY9kC08gL;_g?HGi>{* zkncpERtt6l-(~#NnVjcCei>K)T`!`ma=(WC8Tg|EPla)|>F8WtiM)f>C+a)(5!Q{p zM)XenQRxz2b`0?4C+*M`bCbWm<+Se1u;QQ>?$cG@qd!e2f^Pe#$?i0+))U@-d)I8F zzw$EZqHpv9jq3u^f2vYx!B3-e(lt#mu2KEGkBs8~Y+es|eZIzemvJz)VyWneVn4ky zZhk;p;D?=(485Bw`Vi6S&$~jeDe!gi@@S7=JrwD>aqp_3J~x#82+4im0d$W|^notl zhoF1<*0J;TX`7mcPFF4q=)dnsa+mZ&=!(SMJAEDx^|?$$Z|(Ve zKv&fx!y#|PI-kM(*v3SMWc}Jgbbc>zd^XY-o0_EOHti;+3vlD}$X+pempMD1TkI!! z#q@52@sj&or_w0}u z^dV{;*)c}>+tK^Z3A#K3@&G58hj-)tLX>ZFxnZF{emnX7R0n5mz&v%~cNE9bJLGog zws*sK(X*;wNv{~eVY(su1@+=fwE#ylaHwDBdlejaXMU_4@>zaISo~yo_DA4F-&p$E zLco*p6FSAARAb~G_&=oY7J1ja{1Dc)Ua|mmT+aQx`ik&)75N?Z$DZ$v_M~K|btZDz zkJ&NKrq|gvTQC7)K_eew!u$|;ru`^2OxuH*r^by=p)3Hc5^oQr!iCs_&)}Jxm zF&FM6{u0Ys4)q>rF$3ioj^0(A@M2n9$RlpkaL_Xu4<+x`h+PaPxXv3v@72oAB)^C~ z=H_8t0(^O2!@N%S3k06~V9$z>F8#4<*?yV%g{qeU_+$G#(IMnB_37M4%Eww(ZIAE# z4viUK5bN0;$oWC|ZZpr^5kfuhBJznJG5gO$eZ%)y=@y{demmWpcNzKoiQgOcvq&%G zifqBYMb%BM$CCZMgwig5av;vjU9Pm>p?(>U$0xQ6abL1E;Y9~pr}jCLFWfkKhh5-^ zeXzS&XBFV^`ZK!J-GH-`_LpiB`*Glc`f@M87dQ#J_+H4b7M8|7A^N^?Eb#~4-+rp}Q{s9Sg#D_snpTm-nWnaerhR`sbHyzN$}D`Neg}gYWi-ijX%{ocL4ZkMQv%^7|v_O056Y>)Bpo z5#hNXpSR?{$4%C`_?1%o>AiZ9lLXJ=6>dO|@v+jNH*U$R&|CQdd=HSj)EZLKPpFPV4W{44a+ zGQLOh+X3#eT-~W{ca;U!~J%j8Ith+#ZhlQolZlNPm%zpVk!Ryx% zbiph;Rsis4C9es5OymKP$Hc#4RG$5vsBLXBV4j3uJiDIo;@PBK;D@}vrJQ5?SDDY- z@l58?{Xy@m>qoQWoZa<42|8mtwE!I!e3tPN{j*EMb``hDp{DNTHE1U{MCgaKOYq2E zA^*lQpN7||DNpA`wivJPkIO=zsHc*DK*0l*J;8xAz!pl+DrDi3B4=PZ?S9oncn?$Kc&DAcf1JkvHyX~71%lRL1)|ovVYg? zFIDQ7dA)sp%v-;LKL3xL#dZWy`5Yedo!d-ul^vFX{foIp>%nejyka>*%0=Uu%=T~{ z58sDzx4f><%bvh8>=ze4`EO7oO9DboL z^4(MKX^eY^G_+sia{xOsOR9aA+d+OwM9&%7C-6BztPh==NImg|w9hbOy3##Awj*&j zX@4&KO7gqkQb4Co>1HmpNA^XME}_h`;JwVBq)X__ox!}=4)+27ZI%Ho(BEG=mWd@Y z-A)E)lvS@~GtVQ#wG7lY2_C|99C* z-foNe3&8z%w zv?4nSoA(FEx312YZTgRt~5jW*vPS%C%OM8=lFtuZd`+NXve(D zg(#$RQa5lT>aqLj99sILR?~hf{zHeIA-_JgmhgSXiLoIrKRtxU=_dIdHS9-4(%lS_ zGwq?aj7Ou=1D^QBWO>0H>3AINVtt_L&F4hy$J5pMa3IRPxvn$vF&>EBu6v{)(e+|v zKOfaE@5A#4ll|8EP+rad>T|%^KLzy2R#|pA;GgZ93G$se^Jgmj(bo%*{8y3JP5Ujx zcjTQ5hA;EWQDs**fqZM@nJItYG>Dr*J0O49V{O1M4Etv_tT(p<=>2QGY%chhnsNu- zhq{CAf2)FZFrVh{T{veG`V`R@zNcymvV6o1(jSm~l*pgze9pa0@}WkH+y}zNwfu>Bs6M%(rXBc5J7M{vFcICVIb7)TNm#a$fw}Ygiv{P5p() z$NZ9;{oi{z%jkUry=4mF8S67;9@Xv>IA_t_$nGrOzbmbuCCy2`+xZpFnRN3Pu}_in zQ+cP`elQ$#K}{(~^6j)?z<2u@%VQ^BfIOw^lHX|aY@wQ=y>4!A;;Z@j{%XZa*U*2l ze|eDZ^~Nu6=#Bhm2NV5dyNlSU@ryAYYxrJP&XK9~a?v<6OPUhk$}jd_yiq6@KR0`H z2=yU4PxPnKz9b$>C&|Cahbj%I@~E!LzK%+esX-lrf-F~KTG%6 zT+tp_2e!{4qSJiuS>XG3i4W@Bw69QYOM#AQwo|v2c)!l?z=~cyqK~h}3&s+_T$80Z zXRw{wZfSd;o@j^Iv#fow1LDofV1F$9#T6mGE$?MV`@G!Oa)tc1!Rf*X-yefZN3t8rd^<>H+>L^04B4pp^5IH-rE9T6-aHnCsw8+<-3beqRr@FVhD z_u_l7OHo_MFOWHY7;>Xcq4QDu8~H~yzY`xYciu(vz|(QuqchE(Kl$(aMNbmhSMt3k zsaN*JVuvg5$-Co4H(a>_v(Q7= zhWw=4PtLva?{>;Mj8(3L^=&`<>&}p_JD0?`3LN#))h^+?yPQJ$2p;Iz0`iNmm(%;$ zGX82S*;Tk4TLDM2{hiQ1$tQFp@)yASUvqaq%%`+Z=!Dq6NWa7{gut`sf1HK>@%!-c z%cNiR!-kT4&hNkYMQra_ko{^&y@EG6$^Jm(Vi^aaCsLpKbTZ~$4<~s~ZM$?A#&6;G z#Y5rRfe&gg z>2al=$e#p%5V_sPo}>KyUDszjNs*`CrG0U1Ne;*ls@x}MLqB9bH0yH(k7b>@CiISu z#C6^eKSle6zF*%9x~baFNE#x=y&cYPZlSzK@ z$80*TMZ3*%(yNKyP57JSm-&@?RFn0fvm*EEr}hzD=l8>JobTA1MCar^Ab)c*_a=)Z=_$s@;B2_Dep3jf_!IFib39$ z@r|F3fnUk|s3g9}!3CurX|KRjzjk>j)GI&1JIOEe>wa!EC*+fRreZ%Q<>b9u|Mr+_ z$jAFv{qh9JV{$Lgmd*jXWLS@+Z?g+^tWR()=-gM}3m?&w49V;3*#FSx4o>1v#QrU|^T#6LyU;y5@nhhvYeD{hP48L28`qTl zR{P`RFI)U^`Ol}3JhGPji756XDD~>zH*A0(`7x7s`PlEVU48jlNVk9d1-dTt8;XRV z-FAQ8SRyo?I=qec^LtGp4ZrmCE~7-<$i=DXi}-Cca-StC$RvaA{zDNA`iG|qTo7TURe0!T-Sg+votdD6O$1B_m zKHzK9J>LK-{LHF%7Nec>=-nOqn)q9f{6e@g&Ecm-zUzgwzmvFI%KORF?_xak_h7zB(}vutOC)8sV2RukLa3lP+|{y`HQ>d-Vbf|Jt%HrJhJm;Sc6D)<1Gw z()AZ9SSOzScnCj=Cnx8OCGy(bWPicMLhwK=@yAX74*OB{#1iuNlbo!>cab}_yo(pU zC4PdJ_Gz-WH0;k==$fy1{1C=p^f**3de6>?-su+Gi+1@3@~SGdFKsN5SH>&$;wZ`DOJBhhm>&-XMD@ zZApKt9$8l_ly^6d{ss8i)`Gs652;;#E#31~H~8IPq5pS2dLiVOpT6<>TC^i?LGVf2 zv=sD2w&j+a=ud^o^N_zl_kAJs`DmY}cD_jCw}<2gy{HZH`#$7v)SPVq{v+>&xv7O= zH*DCCfc^W|e*bPqEaM;dgt*|P&2t`l1an_j*pujv$LLq2^uWq0U@yH$wd#bclSyI#3>q-);a5iN=rZGou)dhx4EA03)J5v59U?qV)Ty3oJF9Yy4d|c zhkWL@te_+22HBO{@85&`mq^0TR4)8u$QQ{yV!twx?9NrDD%dy6I@68V-m*p3M*yGw zPQ@>zz+TEfuQ9&5!tTx?UFZID4f1up2lyf9gtqgWBu}({lEx!%0i^3J7v>^=x^tyM zDE)FZ=pKst@=KEUSUwecqi(RCl)214rpZ_z zkseI^(aL+_bMGD$;H#cno1lHwli=6Sm(PZI)T6$`_{uwTw$ci;)3>B^6|DzWYuspl5Q!Ug?VOS5xqQbY#E4b`JKX z_E^utcWo(;a-Q{Ms?}Xsw|42Dz<<+??7d?{$IyD}`QSl}*TPcJTgbYL;IZCNvE9A+ zLsje9&So3oqrSf(=(70dlkraQf9Pl9X_sJL+%39y;mgoHVm+kPpMl<4elOaWCOf-G zZV>;g6Y4+?5O*)O$JB=q!93iLBl{`UQ!D;mT68unKs<82>df2VwIC14ZzmW1~C zR>#0MBYTA=E3hwDd|su$yp{@I&3y6)?`rPG`y=N3Mx@KQ-tM0<;-}yINqQhYFOhb* zHUBgU^Cb7Ze9j+!K%C!2G^@LUU-+>bONZ~8@Ba$E8nH82H5PQ=b{j+MeF5!H;$>H2 zeshgeJe>!JJ zwD0w^D?(_~R094b>@wiD*!**o_EEoA!?@X>)*+u``$>_T06X!=W&G+yfFt87-zBvC zF0y*^7x+t*{`ubHYo=-^7Z5unx9u?KoJ!*RF)8G)MUS9!yo7%N=sRRRy4-`{KTqm) zg9eb?&+nP|akMWLekA)C$**#@#`=`+{!P9wCFB1>26|t8_#Les|^9^WJ7 zcG;<9mk_;o%KKHvenDm21g^c!Z{`2Y^K-;+MPF|Ql7BU6zx$>Q*+1Vi{UpZcO3D_j zTVD{5;=>7I|EcD{wfJS2+kTyxz77@mEL=()D>C_k8MokQ*7l zhI+-{*1iISXTQ)p7dGFem4G7^Pp|)Yi2EsrAqU83BYxIKyh`#Quj44)pRWNpybo~& z>Hb}UKj6I#$>%2h4nO=7m(c%pvUe!2_n!rx$#?tC#>r^Uk%^EyWd0QUyNac0bsqJO zBl{_r#P@kekv{?7uKs+aGr;)fF6Tl#UN1f9tJEX-8~b)8$uFEB`l*PokIY1L?&F_H zKIeS4kOLhuO;3Fp;M?gm|8@!apA`5Bx(NPe7JiHR<-5u|8uFzbTMu%XzB~(bH>$tJ zG0-)0c`N9Q;d@ki%W{%8M^-L|e$Fc2KZIt&8*d|Ie}uZ)F~C#2A2-c9L5`JvnG!4) zGX3CgCC-6~KSq51lTx6wYE7fp1ALd9jCaB8rwaRHz2;fa4>j;!k_WT@I05Zo{L?c^ z{T%RIx6KUxsLS6R64K-CceDpQJ_iy0p?;-v1haM)>;_a>qEqG(zVG_zfBp#N)Oz+; z^XQg4QD4vR36C$;?|}5*u3;TWe-r+nCWdyHv~?lZ$T-_)`(yuNelAS@;ot6aA(Z#M z(xE?YLmsRb!}|fx{yN0Jk#9kM7(`z#^v5;KjC#~He&=5H>#VmmYcDNCJ6H58gZe5G zUA67$ok4Z=gD1jw`%ez=N5^(Vk-YK*`FTrZ!Ox6r)(mj2lK$D19fI{G^u+P|+xC|t zkA{3EgTgqO=H1awGw`t&P~YN>gomtm6S+g~%lUsu@1XUyk;wn-65yvl`5NeiDR^i< zz6+h7P!Q`!@IE%;q znqKyu?E?Jh_Zii%e&0AOd^aC&1YK}*OAbhtPYN6-J3_aE_D8zW$@@b-dx7^Y`Hnzt z&~GqZyOy%{-+aP-Mfp@2UMI~+&d^Phz6kl`T>>rg;ha&K0(=<y=sT=ZpIsg zr-}c=&(4^=4eb$oYCc!8|(JAPd602O|^c2m$@m)zh3z#Co`RI6>ha`H!16kjbj|o4 zn??IXQ=0W}7msBKaOLN6k{=m~%Y9h$7uh@5<6QpS;L_+1kAu(w8Bd`@CWX#FeW5wP zBendwd7+$dcOLSwxp@uz>2^6&>h~FFU#+TTB>u<#Uu@Z(pof~r$&dOS?*~ac^x=%Z zg?hyAtDg5R@zq_MLEp?LSuk(Hf93w8s#6>CgRY!Rc+L0tWqzalaz4!KSL~|XEIJQQ z49=2iM9=RS)40ogVS4hA^#ey6XK z8~nvIZbkmmnO_TBp^p+5d~kdFU>!?6lJ1tS0bOu!kX@k6qon(5DX=rJjgxRMSJH*v zMfq6H68^2bZ8(APUUUZXYGNLDV4TN2h<$?jg5*Kfp8Rgxe>SHYA60TK@I>J2)=f3W zb4qs5&1n4C?o9HzmaqRA^63k7kI^;y9qZTcrF}*0z=wm;F6qyp5$Qr)@GRKq#`k`*kB>K(rt{uCD?4r61g&xFwmQ480`+C2Q-Xr(t z>D`g2KGCzMC%z!~m%sz;tMol|&n#Yu{B!HVxrzVYb}!D%Y*9Ikn{Aqm`SiUmmJe~o zcCnt{Ig)w>ucbcu9`CUo@`v=(KmJNmfD>=Q`*6|s&ExxZt3IN77=X}QfhY7*;^OyG z_`GeF5%Phg$LfB9{j|BBLjI`b-j_@VKQt%TfZje>9h zFlTi2uEfH3`N@4doAq1Zhn=0aWk{EEHknAJW75UA)2OiC35Bo8eNnlGp|`L7$E@>Z>L=4j_u`MA zqTY$LPZT=kO0Ydq@ky&gdUU?b=lJH-K->qD{`)%QCsn`pJN$S=>um?gV`9fB{Sx`d zG+{j~x6^0M40%MgOCtYCx8L8{_eL?ahXF-+{0+ZrBK8x!ZlrxuuY?J?3hxWVpJ025 z37_l;aBrVS|LWRnsITt<8nA|IZyhA$B74z_CkYlACCNtSZn#pUSACUZNFue<-`Momr1?@+o z^nvXEh2@I~uXZjy1iTP?Z?y5uO9nBRp>tMe`}`y#2>TR_xm+z z!RPF_OA}JTnO*KK)b}dwTkNSx8vwTf$(MRzUf{FLqrGrfose$oOw5XM%U^@Mp?{+H zab&%V{i(D|H$DaW5g-0(qfk!Pg_$=Dc&t0u#riXaX&hv{M33QH(RotzUF0{Z;Qt6c z)7AKQB%ej+o{Y~f#~9*^a{k}C6zrJXo(rky%k6Yg6=V6D?RvzYeS%NGU$*-rCAHt| z)BJ9+=$k}7XFJ{7n=gFRea-%mZp$Ei)XbR*`N2Ou{=QfukN%PD5QQG-bd_!edLlpf z<~q!$=nu{BrJIELus@JJlgJkd`ZO<;llwEi>T9s8b8H`OR*Hke-I@90gD1oZ0<7 z3HmekQGbj}Yyq7I$-M@rPeVWF>XV-VJ^P`Vp`6GauEgx7sQIDLf9stjU$`gf-DBH!?FzJ){d<{1C5TQRSpfTB&-cIo zT^0VXLH@3UzWJm%z+;n^-jlUkCrk-&&9q}PyE9PT~KT%KCy?>Cw!2y5A}J%tF?;cS}#}efc!-9X+1TZTunf+vjI@ z#eO4xf#1=aRSJ3^Rq03IxA+~@Exx*d_GTl$_mQ0OU;QZj8S5d*xIZQBkkF=7Avs`f zKgc_F0p0Uc$#kFG#ro1YQkCK0+bZ2=;H#N5nDk(`?ZcyYj=N=qoME0H+BdY%FF6dm zeD(Gk@Kb$(&zA~!fZXBMk-ej=Z@rEE9jTF^W76IPJt-65$-QeeZEXg?XFYMOSob0! zEXrJkq6I@AmN$j`hbKcKhdI0sAC3HVrFJ*fl*-x?G24DTT0imAg9X0{T zq1fF>fBf@nhll(Tf4g#!#*_2w++#+rH|4wI6pR7y1w-niF`mCVil3ncV z)$N{QH}G`CPfg zTadSX-h=Z(IluEBjFWzd&UZ{j(zELa8?M55FrO1Wi2W!V*(L0xcVwgS>!GloV=E7m z9L(pf(qAY3bw(%byFO}=RKHQnH0pKrR`K>7`n2m^yoxUVM90U9m)6BQ_UPTWciUIn z^yuBbL-=01ZTCJM65soDZrig%B3`26!=+1>)^F?o|KD9t{o@B`d+M23`;Y2Z-g~TG zY<_a$SE_!j|G9M;3$Lml%Z>~UWAB$;acaihjbrFUyh*G>$UmfUY)H05h6fwRCUsxB zx!C?jv2uxQ@0V>73t*Cy8^xL@C4S2q$9B}p`(oc64WoSWy+P%@?LXSqID&`zMm33r z&-L>(A^0_##X_|5<4t2|?ypT^TZ$!W?9w#0K4anj=LR*U`nEKUC1UF`wrLVev?h_E TX)JW3|G6fy(A@U9n#KMf7dXz^ literal 0 HcmV?d00001 From 6af1fdcbb71aa67ccb8a9ff4439a3275bb2d2de1 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Tue, 30 Apr 2024 09:57:46 +0200 Subject: [PATCH 062/379] trasnfer learning combinations --- .../exp_set_TA1/main_loop.py | 5 + .../exp_set_TA1/nonseq_model.py | 117 ++++++++++++++ .../exp_set_TA1/train_script.py | 152 ++++++++++++++++++ 3 files changed, 274 insertions(+) create mode 100644 tests/test_nonsequential/exp_set_TA1/main_loop.py create mode 100644 tests/test_nonsequential/exp_set_TA1/nonseq_model.py create mode 100644 tests/test_nonsequential/exp_set_TA1/train_script.py diff --git a/tests/test_nonsequential/exp_set_TA1/main_loop.py b/tests/test_nonsequential/exp_set_TA1/main_loop.py new file mode 100644 index 00000000..c328a576 --- /dev/null +++ b/tests/test_nonsequential/exp_set_TA1/main_loop.py @@ -0,0 +1,5 @@ +import os + +for w_load in range(3, 9): + print(w_load) + os.system(f'python train_script.py {w_load}') \ No newline at end of file diff --git a/tests/test_nonsequential/exp_set_TA1/nonseq_model.py b/tests/test_nonsequential/exp_set_TA1/nonseq_model.py new file mode 100644 index 00000000..7e65c172 --- /dev/null +++ b/tests/test_nonsequential/exp_set_TA1/nonseq_model.py @@ -0,0 +1,117 @@ +import torch +import torch.nn as nn +import sinabs.layers as sl +from sinabs.activation.surrogate_gradient_fn import PeriodicExponential + +class SNN(nn.Module): + def __init__(self, nb_classes, pool2lin_size, batch_size) -> None: + super().__init__() + + self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False) + self.iaf1 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential()) + self.pool1 = nn.AvgPool2d(2,2) + self.pool1a = nn.AvgPool2d(6,6) + + self.conv2 = nn.Conv2d(10, 10, 2, 1, bias=False) + self.iaf2 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential()) + self.pool2 = nn.AvgPool2d(3,3) + + self.conv3 = nn.Conv2d(10, 10, 3, 1, bias=False) + self.iaf3 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential()) + self.pool3 = nn.AvgPool2d(2,2) + + self.flat = nn.Flatten() + + self.fc1 = nn.Linear(pool2lin_size, 100, bias=False) + self.iaf4 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential()) + + self.fc2 = nn.Linear(100, 100, bias=False) + self.iaf5 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential()) + + self.fc3 = nn.Linear(100, 100, bias=False) + self.iaf6 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential()) + + self.fc4 = nn.Linear(100, nb_classes, bias=False) + self.iaf7 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential()) + + self.merge_fc = sl.Merge() + self.merge_conv = sl.Merge() + + def export_conv_params(self): + torch.save(self.conv1.state_dict(), 'nonseq_conv1_weights.pth') + torch.save(self.conv2.state_dict(), 'nonseq_conv2_weights.pth') + torch.save(self.conv3.state_dict(), 'nonseq_conv3_weights.pth') + torch.save(self.fc2.state_dict(), 'nonseq_fc2_weights.pth') + torch.save(self.fc3.state_dict(), 'nonseq_fc3_weights.pth') + + def load_conv_params(self, w_load): + if w_load == 0: + self.conv1.load_state_dict(torch.load('nonseq_conv1_weights.pth')) + elif w_load == 1: + self.conv2.load_state_dict(torch.load('nonseq_conv2_weights.pth')) + elif w_load == 2: + self.conv3.load_state_dict(torch.load('nonseq_conv3_weights.pth')) + elif w_load == 4: + self.conv1.load_state_dict(torch.load('nonseq_conv1_weights.pth')) + self.conv2.load_state_dict(torch.load('nonseq_conv2_weights.pth')) + self.conv3.load_state_dict(torch.load('nonseq_conv3_weights.pth')) + elif w_load == 5: + self.fc2.load_state_dict(torch.load('nonseq_fc2_weights.pth')) + elif w_load == 6: + self.fc3.load_state_dict(torch.load('nonseq_fc3_weights.pth')) + elif w_load == 7: + self.fc2.load_state_dict(torch.load('nonseq_fc2_weights.pth')) + self.fc3.load_state_dict(torch.load('nonseq_fc3_weights.pth')) + elif w_load == 8: + self.conv1.load_state_dict(torch.load('nonseq_conv1_weights.pth')) + self.conv2.load_state_dict(torch.load('nonseq_conv2_weights.pth')) + self.conv3.load_state_dict(torch.load('nonseq_conv3_weights.pth')) + self.fc2.load_state_dict(torch.load('nonseq_fc2_weights.pth')) + self.fc3.load_state_dict(torch.load('nonseq_fc3_weights.pth')) + + def detach_neuron_states(self): + for name, layer in self.named_modules(): + if name != '': + if isinstance(layer, sl.StatefulLayer): + for name, buffer in layer.named_buffers(): + buffer.detach_() + + def init_weights(self): + for name, layer in self.named_modules(): + if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear): + nn.init.xavier_normal_(layer.weight.data) + + def forward(self, x): + + con1_out = self.conv1(x) + iaf1_out = self.iaf1(con1_out) + pool1_out = self.pool1(iaf1_out) + pool1a_out = self.pool1a(iaf1_out) + + conv2_out = self.conv2(pool1_out) + iaf2_out = self.iaf2(conv2_out) + pool2_out = self.pool2(iaf2_out) + + merged_conv_out = self.merge_conv(pool1a_out, pool2_out) + + conv3_out = self.conv3(merged_conv_out) + iaf3_out = self.iaf3(conv3_out) + pool3_out = self.pool3(iaf3_out) + + flat_out = self.flat(pool3_out) + + fc1_out = self.fc1(flat_out) + iaf4_out = self.iaf4(fc1_out) + + fc2_out = self.fc2(iaf4_out) + iaf5_out = self.iaf5(fc2_out) + + fc3_out = self.fc3(iaf5_out) + iaf6_out = self.iaf6(fc3_out) + + merge_fc_out = self.merge_fc(iaf4_out, iaf6_out) + + fc4_out = self.fc4(merge_fc_out) + iaf7_out = self.iaf7(fc4_out) + + return iaf7_out \ No newline at end of file diff --git a/tests/test_nonsequential/exp_set_TA1/train_script.py b/tests/test_nonsequential/exp_set_TA1/train_script.py new file mode 100644 index 00000000..a752422b --- /dev/null +++ b/tests/test_nonsequential/exp_set_TA1/train_script.py @@ -0,0 +1,152 @@ +import torch, random, sys +import torch.nn as nn +from tqdm.notebook import tqdm + +from tonic.transforms import ToFrame +from torch.utils.data import DataLoader +from torch.nn import CrossEntropyLoss +from torch.optim import Adam + +import numpy as np + +from nonseq_model import SNN + +torch.backends.cudnn.deterministic = True +random.seed(1) +torch.manual_seed(1) +torch.cuda.manual_seed(1) +np.random.seed(1) + +batch_size = 3 +num_workers = 1 +epochs = 30 +lr = 1e-3 +n_time_steps = 50 + +if torch.cuda.is_available(): + device = torch.device('cuda:0') + print('device: ', torch.cuda.get_device_name(0)) +else: + device = torch.device('cpu') + +def train(batch_size, feature_map_size, dataloader_train, model, loss_fn, optimizer, epochs, test_func, dataloader_test, phase): + epochs_y = [] + epochs_x = [] + epochs_acc = [] + model.train() + + for e in range(epochs): + losses = [] + batches = [] + batch_count = 0 + train_p_bar = tqdm(dataloader_train) + + for X, y in train_p_bar: + # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width] + X = X.reshape(-1, feature_map_size[2], feature_map_size[0], feature_map_size[1]).to(dtype=torch.float, device=device) + y = y.to(dtype=torch.long, device=device) + + # forward + pred = model(X) + + # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes] + pred = pred.reshape(batch_size, n_time_steps, -1) + + # accumulate all time-steps output for final prediction + pred = pred.sum(dim = 1) + loss = loss_fn(pred, y) + + # gradient update + optimizer.zero_grad() + loss.backward() + optimizer.step() + + # detach the neuron states and activations from current computation graph(necessary) + model.detach_neuron_states() + + train_p_bar.set_description(f"{phase} - Epoch {e} - BPTT Training Loss: {round(loss.item(), 4)}") + + batch_count += 1 + losses.append(loss.item()) + batches.append(batch_count) + + epochs_y.append(losses) + epochs_x.append(batches) + + acc = test_func(batch_size, feature_map_size, dataloader_test, model) + print(f'{phase} - Epoch {e} accuracy: {acc}') + epochs_acc.append(acc) + + return epochs_x, epochs_y, epochs_acc + +def test(batch_size, feature_map_size, dataloader, model): + correct_predictions = [] + with torch.no_grad(): + test_p_bar = tqdm(dataloader) + for X, y in test_p_bar: + # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width] + X = X.reshape(-1, feature_map_size[2], feature_map_size[0], feature_map_size[1]).to(dtype=torch.float, device=device) + y = y.to(dtype=torch.long, device=device) + + # forward + output = model(X) + + # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes] + output = output.reshape(batch_size, n_time_steps, -1) + + # accumulate all time-steps output for final prediction + output = output.sum(dim=1) + + # calculate accuracy + pred = output.argmax(dim=1, keepdim=True) + + # compute the total correct predictions + correct_predictions.append(pred.eq(y.view_as(pred))) + + test_p_bar.set_description(f"Testing Model...") + + correct_predictions = torch.cat(correct_predictions) + return correct_predictions.sum().item()/(len(correct_predictions))*100 + +from tonic.datasets.dvsgesture import DVSGesture + +root_dir = "../DVSGESTURE" +_ = DVSGesture(save_to=root_dir, train=True) +_ = DVSGesture(save_to=root_dir, train=False) + +to_raster = ToFrame(sensor_size=DVSGesture.sensor_size, n_time_bins=n_time_steps) + +snn_train_dataset = DVSGesture(save_to=root_dir, train=True, transform=to_raster) +snn_test_dataset = DVSGesture(save_to=root_dir, train=False, transform=to_raster) + +sample_data, label = snn_train_dataset[0] +print(f"The transformed array is in shape [Time-Step, Channel, Height, Width] --> {sample_data.shape}") + +snn_train_dataloader = DataLoader(snn_train_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True) +snn_test_dataloader = DataLoader(snn_test_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=False) + +snn = SNN(11, 810, batch_size).to(device) +snn.init_weights() + +snn.load_conv_params(int(sys.argv[1])) + +optimizer = Adam(snn.parameters(), lr=lr, betas=(0.9, 0.999), eps=1e-8) +loss_fn = CrossEntropyLoss() + +epochs_x, epochs_y, epochs_acc = train( + batch_size, + DVSGesture.sensor_size, + snn_train_dataloader, + snn, + loss_fn, + optimizer, + epochs, + test, + snn_test_dataloader, + 'post-training' + ) + +with open(f'nonseq_TA1_w_load_{sys.argv[1]}_training_metrics.npy', 'wb') as f: + np.save(f, np.array(epochs_x)) + np.save(f, np.array(epochs_y)) + np.save(f, np.array(epochs_acc)) \ No newline at end of file From eb48ef2bb2f2901bd1e885c4eeef0775fe640321 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Tue, 30 Apr 2024 10:00:39 +0200 Subject: [PATCH 063/379] transfer learning parameters --- .../exp_set_TA1/nonseq_conv1_weights.pth | Bin 0 -> 1693 bytes .../exp_set_TA1/nonseq_conv2_weights.pth | Bin 0 -> 2973 bytes .../exp_set_TA1/nonseq_conv3_weights.pth | Bin 0 -> 4957 bytes .../exp_set_TA1/nonseq_fc2_weights.pth | Bin 0 -> 41299 bytes .../exp_set_TA1/nonseq_fc3_weights.pth | Bin 0 -> 41299 bytes 5 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/test_nonsequential/exp_set_TA1/nonseq_conv1_weights.pth create mode 100644 tests/test_nonsequential/exp_set_TA1/nonseq_conv2_weights.pth create mode 100644 tests/test_nonsequential/exp_set_TA1/nonseq_conv3_weights.pth create mode 100644 tests/test_nonsequential/exp_set_TA1/nonseq_fc2_weights.pth create mode 100644 tests/test_nonsequential/exp_set_TA1/nonseq_fc3_weights.pth diff --git a/tests/test_nonsequential/exp_set_TA1/nonseq_conv1_weights.pth b/tests/test_nonsequential/exp_set_TA1/nonseq_conv1_weights.pth new file mode 100644 index 0000000000000000000000000000000000000000..a4ff9cf6e910398edaef8f119a23cac63e8bb7a9 GIT binary patch literal 1693 zcmWIWW@cev;NW1u0J03M40-u^#i@ny$@zI@hVkX8nduoN#ri3UC5d_k**R`bj0{l? zOv%alIXS7xC7D3AT>eEVsYR(NE}6+CT!jppL4}MFY(SGCS__$yOY)17GxXw1OEPnc zx#EjblS(slQsPTe^NRC};>(P<3Yj%DBG`dCih(K<(^CssAX>QGa`F>Pf+2!jg{%>b zKs5%1Y+%!qOH&f93<}x314;@x0=*eDyt%xYK(M5cQ^T9Xn+3>bEGgvb%mBHFyQGjO zsF1g~wvaD^6KG0&ZfZ#)$WMj*V0nQEpddq`U~Qog*d=ADMa4kB6$*PZ78e&M=>>SR za|Cxi5BdaD1;PP1Lr{*v33~`8RhFdYgF?#9$&Mk4B9N0=Qj(Jja#O4AV@Xh0gD`G4 zNi!H?cM~WC^bOpcbV+lN%im`E^LqE~S=&tP5C1i=pZ@*p-d}22`+N>;*mr)yK|2jc zJKHsP|Lt40XRaOV6ApX1(~I~0YcRBX_Kk1fg6@WWw(ZC5a_Ya@P0zR4Z(4X{pMyEy z9wtpLJ1arey-WE%?-f%DvHz9WZC?>|%3d&Vy?x`f>wE9=zux<+b=AI!G7WaCvVQN2 zd%(VbnfJ_n<|@wnnPa5wKRG#^ zr}Dh*`eN(7uMCyMVo%ms46#d+zWOu%#l!nj?a&7g|i1$yAbQ=D3qnV6GV zl?W;d;xkj+oD>N=$iURx%)->d!qnW@!qn2z%*for$N&fo4UG)V%`8og%}fm}EsRY; zuH10dH&X-XLJ$t{W&}AFUUDJlNLi2s3cy>Ap&Nyqk>pT}0Ty`J0|1_((al1RT4@xs zuHiBZk`d7jLXIIB6oV!(<8Tx-U7(wU9E#c~CJ6y+0<0m0J$?hc+1PZT2FNk%!i{2u qvS4&G(5E1<186)50QG@Ncm``wNd*KT=>Tt5Fpm{j>44Nj)B*re#oZqO literal 0 HcmV?d00001 diff --git a/tests/test_nonsequential/exp_set_TA1/nonseq_conv2_weights.pth b/tests/test_nonsequential/exp_set_TA1/nonseq_conv2_weights.pth new file mode 100644 index 0000000000000000000000000000000000000000..8b615944c77d94845ff8d0b501c4e634d51b9799 GIT binary patch literal 2973 zcmbtW3sjBi8s6P@w=I?Fc2vqJxs<)P(#8JYKi%0P3@Yii+p?#ded&e{x=}-eQ*?+r zg+zxVUBv$1kHpM`aY8MpB}SN0t1xmIxt+Z+7EaDtv(Edk_y523fA9CK=X<|zt>+05 zYH71rdU~w?2pg6TD>)@uE|x}#Qj*j7QR!ky{BnhSN?eR0#$`o9Vt|hZD}1e{NR*P8 zC>ALss#=4&vN*9!9OowyDGa2n3G<~Ip==fM+oe)Xks?JVTJ92+s*ohg4WeY?*i=bk zT$DnbEKiX|rST1M!hNN$T#!Gp{k!OsY#a9^bM6Xu}rSooOF~>LoS!cx`YUI^_r`9iw0Fy>J0g8 z3vF35hHqhPrb3*eZdHH}#R~s_B2l7HB#PCP@~B%@>Rzk!^OQ!jc*9dtH(&}kz{mNI z@kzH#^{kq%!94Ui%al)k%dt96r74A4 z?4yK;h5Mksi$(nD0pz(yK4mNKg3n{jG^kMhm?+P6~>A;;811~NM8F=jk6}i zgM&TRbilM6@_L|WI@h!uzrNByWgY5ehR#J$7X8-riindCJeJL*|LsdA{#0i3mGYg? zxut~S-P!_i9=h~-ybYP>3W>su}=l>AkQSN@7^=y?4IlRCPZ8j7|@-${um3#!Gs8+OX7 zggn@i(*bQ=-O#*#Hhs!5pXms+LcyFIsy#9f+i7FEU-<>CU0e)pXLV@%L+#AaGIQ9J zaFIMRp4lXVUHg$qmu1O`u(-kYtDMqoe_xkrM*hmyZLBw zrT})R?zL%Q6g=-($84w#0pk{Hs{iIiaMl#i$=`P|yxn}LeafQ;>PORG{z8)ri;7^2 zMFHa-xr$cqD1g}8k#tL*IW6AN4c#3cgf&os-)UHo?HfDrl7j)&{B1nARz!gzuLQh0 zT*&FpKatk4{dgeb3?A!tB#p&8$WVG2+5D=B+zq*dezKRC`z@Q)tWl7^mcCN5esyJL zhK;8jgOT!0?t_sQY(bT`8Eg}B$r5WD;^sV_)SXyL)^K)_9;qJ_cB2kn=%)bP+JIW) zw!xkyfTirmkk@_$gZNF#j6Q9uV9y5TXznt&dgBCDd-j#G*}b3IrpF;iB!+Zx*58@T z;0OpvZpOwkL(q_4#-5eEjK2u|7cg0E39#bQRxE7vgryDns1s&{y}mJ|R#Qaj6`7L> z))J`PxfiL6YcarPG06xZcsw$nnf{1Fdc@PIAZCcE>}zGRj~{`uBNvcaMXq$Bd-C6>qx5P+XeEq-58%{3H#fvNy!`z zEDP?!g_a_GHoBg&TV2NFD4ik0CJkSmF(GS$7c&)R0@UlehP-+wQXZWGmFvt<>y8!u z%)^rI`>lZ1@!3oV3H$L`eJ(DM`_PZZ?*?f#SIIP8#md4Q_%&}M*!6@_jYUp)j}`-$ z>Y*aMbFe9}M6hd73+NY_)An^nO&=h#9WJ{f&S1xPTwBeQOojwQn@c)=} zy^-Ci25JWFyWm0(NA25GmRpCVu76wmZ<6Ow`Tzg` literal 0 HcmV?d00001 diff --git a/tests/test_nonsequential/exp_set_TA1/nonseq_conv3_weights.pth b/tests/test_nonsequential/exp_set_TA1/nonseq_conv3_weights.pth new file mode 100644 index 0000000000000000000000000000000000000000..8e179172d529aec383c7cae44f5c5fd693b64cb3 GIT binary patch literal 4957 zcmbtYc|6rw*uU10T^ZR*Mb^l^-1A&i%3u;jr6J2D5iVCri#;upv`W$*Ew_|r;{MKa zD~ZyC5-Kgyh?a>drHz+q-ucYTTkrCDpWo;CoO6EXIp6a=&pH1*UvCcyNft{+hV@@e zhb6^|iH%9%@`6HRW0qM3E$2o?EaoSehXwP4C&xuaySa+77Nm-YhQ>xmb3^%&L$PwR z;={P{+^}hpp?o;kXM^2|7E}OgI zv}6=Kcn7>JFW5g_8hmdI!q#U8X;-Ty7;tpyz>`NLFIiIbvim%G z6}vD33uRm^-^K7&R5Mx%6EJkp1O}IEr9tk?P=|3whmeoVV}n>&_N)km5BE{Go8~lL zRS#3IT%{XE8=>T{Z^(*sb>u;Q4Ae{H!{j&_(j%6^)TD@G!$B!DYFjIkw@bhmSAC%P z!g?^aE(haGAxSJ*2=h(VV2#B_OjDJ|f|02Zd?<}pn=N5FfA=T*CBt#nQ+3QtE29&4 zWr0Y`9Zb@!VT5%XUD|mc9zS#y`k6>$`hE_r-})ndO1npl88&7cABONxaU{@pE*VoH z%e=GZ;&#D9dO_|Z6`Xu5BzGl5FD+F$mvRKufi26a`}hP_dj(=gp&ZRvz7Pa*nxxLw zoKxj2#aSY?MRdnXofG@qo~#Wp6q#2HphoWzn7%NV+)a4J-jMwojKr#FkBu3}bNvL) zhwK(?p4ZLr_wOgpnKwZ85RaUT_avF|@5%I)5_oH}FK)drLyf$n;PlEEuuzf33hS4| zAZ8I{1#Ci}owJFv+)6AOm;twDO(aI2%4l|n4f~8#FzoTtfc8DPI7jIcsT`lhmR+8Q zie9c@WO+fDdu!CS&jLGso<4$mEpfCsO)konim zaCQ9mBz1c_`{|3{*rInIh2CcbY{kn(SfFqeQd{yc#F!1ES~G~#soO9qcNeWKamTma z6F3|5M~H0R-Nl+(S-Rv}0sg46omP!*V$V$S6scC?wv&98-f05$!N=ell<(6R4|f8Ob#fC&wDYfn}IUdUe$y_jw+XG1^M>G)$xClE>zB?=SCGcj_E3dkkHZa z0_8d4ez#yxTM(3&Nir`t9i`!2o-kD+7dkFQz^asZMv&w}_RZ4+^NCf$@I4On!8Jwt zu#^z~kpe96-^O^DZNtk~t5M;d7EYNgg(Y%(Nh1F}Q?=tHS;^`r*Y2o;TGT$a@AyWt zNVSeMnrq^l#u4b|U(ZhIUCL2#$R@srLr8BYAG?xnVb6-q)GYC5bhJsw68A(J+j9^% zbx4Q?KY0@IHCu6GNG>XbBQ9^5DhfNX9!q;|Ft>3w$9t0!G{Z7u##>%yA1)-_8LX4|7R%RySEGrHD}L2c33G z$cU|i%8S)BWLgJFkUc_bYV!#%?=yLll|zEW)X7^<1-c}!7PPC!;f1tQ#F{sWDtG=! zuRKb}9P?eM-8mP^m%T*0U*FKF{p;{fSq1vr)G;56Pm*8m&81^v456o10#?c!G4sk5 zKxcFjjbD^b^4pJ4XX8T3+BXODZs~z_rz!k2y?~BL^CnM9)=@#D8oYR@RH(48?g6v*aPxry^XAObpkpj2pToT%ea-GV$oT;*6 z7+Tzs6a_DQk5BfBi>j;r@J(_q+*0kLtOuzmKKdw?-?@oCsy#!;cnQeLeuf@RoQ@L{ zPeH+BCw%B?LHFxPLt$p+gQAiK4R$#ReZp?Nl3?EVpZHGzG7jz7LpFLm$^Y9ga$-Kk`1HnC)v zk^Y+!WEwvjO?pOPj9fA4dwGm@hrVJ5JRz`k4GYJ&9fd%_d6;(C5FhIO!mhco1w%tO z(r54WATDS&ri^N3lI=r?d2|q5ODaH)X81Wt#o!b2;jSJ1s_ zU@iZi?CwusjJeCPt*{W2d=&wOGx74?wdnU|ANJ=Qrw`W+I?4Tf4p_P~h`&n)U7(;@IXs+ z!0Q?#c$|*@@|WpS-Ghdj_;q>60`R%qIgPsR7IrB23j zWPiqKGNFC~)O4+-rG-1d>yEML?d)o-d!dcZZRe@uB54s#I*ZfaxPqigBIf?!OL~e8 z@yGE3D&F~=33#MO2J0@8<9>DYeD?;&oE@@oT7L_fn|p#?;%!W)`yQh?4N0)kUkwee z_MuUO01OPwVe2u#kiwiENI(*tHh~pafup%^F8ri$9d^$@1V*`%oXd`p zpyQti`l;#AB0kN@FeHFRat_13!bfC=l{80TZw^T02f~3GYiL$D$0WB-Al!Ex>W-#( zJbw{5FO0^;Q3iBfz+0z`(2w-(?Nj9Q@eHV3GmB1{XbNSo{FtZG8aVO&YbV2`&9uRC zJ#5^p1n=tNNuSMZGAKAq&U&f+h7a0p~|P}atF zlF%{}9XIE~W{K5wW#E3|voHwq8j6^zs5lZHdy+0(af#lv*9M!GD}DiAX;K{2@QV^01&8rHjzjhqB>wG4c_#^Rg&0ca#piXa&7)6^L&EVuQGqU6U8=S;{ zg6}PyP&;=K7*q*KtmP9D6*85SEDnckUl4Q=ed;S_p8lN2|p z;M%CYG~-$~^Ijz%b3V$7&YCu1_}j(UMLfax-g@Y~DZ_bi_ID7@?}6yi8kl~2?NBYV z1eaJ@krv+&s#i4;Al{UAT7PDLmK+o=7`2tSZJ$7&R$Jr3<~Xp>OoQ@nXD~^27SA!MEgauX*W_=d`8Nt3~cZAXLkmuizeM4Cu&+!i!;=D z80PGYT2Wcpp{a|dk8J4KDWP%-NcUPn>b-QF{MHxHvkbmoiT+t{ySa}4UzDO&a>F|| z?v0o$K2-M(`?q&AVd;N;$H{*bzX{y<$l&P6XKQa`Z*OB~ zWpCr)U~6e-Z)q`PQ*108>@01qtSl|;?Q9*ahgsg-P+wp;#BkWXJ;a8|{-wP9T~N>& zp7@9Tw>r$fq5due=>7v{>X2&kRj9x8h<{`K{VKKofz|vitS>^szk&XKL)!lUTBQ69 zQh&aKe zCNOACP~hZ{fN1$hF$=FqabIbX<^S#+DG?AIu_j=Jwg1}akgzCu|209&)`o-y`bP(? zii%j{zs^=ZQqtVqS4Pw$O4M@c@}Nkm|L!6`B`jiT^ql{`DIYn=S6tM_CQ|yptOu+O zTb;W;PL?Jp-(JN9ZYD=WDubil}e|S*z z(*F!5Qsuv%tNMy=ibW3I5~=oIFR(6XO_XRzkwZMhqoSggS$lfO$Y$=gdEPB*^`F!8 z|LHhAF{S^>amzMD2SxnHJyTrd{^Pm-KEgtxqr-y!V>}VMbpGS3|D6AC<2qt?|C90m z3}m#;6qnKe$MXOFhyVXodqn^_^mnV=CAi=U{>}T=w zEAeXlK3M8^7j$1;hpp>-`7Y`4EP32byvkX@oks_uvBzCF)_W)mbw5TLYdj#JcOq-4 zF~Gr&H~8YaT{>{QC>n(_zRRu`r|)FpS+)`vM9j z<@n}EI?g@fOpElDxVs022rAZj3kK!v#T%bCQG4bX(jTz_*Qh4&2UdT?RsQBA-EdwQ zG0Y6i7MH{HeRgcow=s#7IWRi&3F&+-AocOHDD%x}HY`7f8IG04 z!Mn70kIrx=*EWt-p3B0?KiXg&Q)i=mKlAyW?YQYHpp8=zCjHq-M~4pZ4`&T#*-d76 zUP4PSeWC-XzqF!|KuzqhEPw&cJVEZIFCd$?5Dq*yV^iA{>0?G8R<71z%S)~b=#dr~ zndtH1Klf1M(|)edIgX|7*oB)<%p+;D(_k<(1M<)5(zv+}xb%T8dbO>^f>|^0hkF;S z*>qVra>a5^A^c&T%DXvWQTiVDy)c4r#fhxcV;ZjPQm0|w*8KZ>dN9t+l&!YP#8huB z_Iha%UcPdVzcQ$p2EPi0MWHVtTwIxLUoxEO`M$xc?dhPGV@_W8KH{sr%W(Lv0FZUn zWi>A=Iq`I>y4-P6B<(v70{7@*!qpyFa&8x2U1>_H4Fs|;Teu0F0xfLR;~Uf_vYEqg z!itRLSafn1|NHe=w%*Bwv(FkzN8%*#QoSzCIbDjD4XTu^TO|DZu#vyR#$oH`Eu`0+ z#dVxr&N=-KLRWh?w$HGivyIpxcyGHERy&;J!!?dTc%~h{!l)bW1b6YjKA(f3e+ILq zAG%R^rGQ(tUyfCGl#uVWwRH9SQ7ma%L~D9?!;qb9yOM{z=14c zD!8Tp%t56lf;C#3LfzNHd}YT2K73ahR%&&ij`n1@+PDBWyGl{5!xY-;CWUUxesZC2 z7*%aLf+fSG$c8VaeeH!9;ub_jpWSHW*IHUQs);MU>rQjL?ICiCKL+ocj_DtEamugE zXvdS`P!gof28L_XkmVR&DoR@5(|dWQmi!RA&B?%!rj_Wyz5^}O4ho?Pp*@qO(abPp^4n1C*w$Z z(;$ktwUPd;2tbd?%Rs*(hKA*PfTN)XHLg_y;pzR{kA(ZsDxZy?EQlWwu?YumjbTL=CvaQ&W;C4QKodNp zu=Sn=Q(0IP2(ym$)~Y=M zjwO~Qy!gydID9FvlRewG^bJOIJ2eJ>Z*8W;l5^aQHInppkUPD8Z$~bV@}W2H7}gx$ zj%pn~%+$YyyV^aN?7CFp%Rx1EVUr_d)_YRF%r;n;ufh6%tOK?C+Wg!>gTd;z0h9lD znwwTQk||7DLpk!|8jkOxOz`nWcY0j*3YImfVde*4J}D`TrYNQZS8pSVSIfA} z;}_9E%LA7t)KJ^aVtDacoLzQ4hXZ+qFj6U(^Gx0eyJn=3|C(Cts`Pmw zeS~(wNERv3VLb(gG`6;w50(z4z2oYjFkgtZaXz%>p(gt^{y5*%H4-y)x1sLaxiDQ) zmPv)a2BF$fJbY{}%Nn1~HV;&ATaTL2xS;zmKG6vh9xuT4_HSWm*m1nIe1Lncmkd%H zG9j^Y7kjH~%cNGe7T8tq{63Mi`T9$u@3^MG3xfcmclZ4H#}^ zMC?BsY9EO84zQFm~VnKj}tw#513zYculOL&I zOJCNO@L>NKDjqc9<)|FCul+oYU#`VmPW;1zh6?Pk$2>T3n(}c$EyVU-Srb4Furp*iLuOa1bGQhf{brw%)b`hH*V$Zg>@9jOH_$uiMeU zP7$;AUjpqj7ja8ZF=zEygW69^lWBc9GsbCxLlfGB*gYpn>t>c&~mMtp6FzY~v%jV-KX+u;uNT7Z<}VXdS?xE;@KFZZH;PJV39E20Zw| z5jz@HP;>GIcFTEW!hLit?(Jy29Qnzyr^f^VNOW%VrOAO!!UT0)oqId1f;Np#sSo}7CYkwvE+TbVo`e)ot`aP$?}Y7~Rf z3$DZI0%a+P(?La0h zKEQ{MT!yoj$dQtKDpxs52|UZBL1B~+3)V@&1k)l6DN13S`dnJqHWST*S5unr1hN>W zM5f{@P&F=x`fuxCgw7wlmv;%WjMm|mgx|Q|vKu56p3|GOQnY9s$;=l`VK?G`VPt7N z_%{aQ&-hUg@MjGSgl?o|4zoymUmEFv3)tWBWrw#WQd_eeyRv%`w4aD(ZoA#slac0h zdr~1>%+O;RA)Yul>p1@~^NpbA(PB{6b7eE8?SYVH6S{Ry74LamL$BCfD8^Rf58t75 z=vE#E4jaTu-e1kmOliroM9`T-)qS%8|X4`pdLVUUoivG=Zbldsgyo`;0Lz zeFN*}-tv5gIWt|qhLgMp2&vQ!P~({FdYq$S3l-j%1v zgB4ich`m@a#Fj37NT;>C{kY}h5pCW&(^f4f8tFfmA6;@3xjY0g4P(#p`}%f9lw>%$35-mig+ANvtk zTFim5FN$ji0xE?Y7MP&=`Hd{9dtq{@WN?ecVb<8(Ns-H)%$Rj{(88+KZ*rl0A46thm8xz@+iki^~CYx@<;?)t*5 zw(k(reUY2}PLu2x{)RHaGgKUJjI-st;h*hYEV=(20&RUz^Xx^o>dJYvZZE=>38&cI zQHD&iWIo-!a~cA>(qU_UF=gI<$S+=IO?Ubq@Sponq0b^c*4rz=+(ww9di6b$dHk8n z{IZ)>ev=mEpK&zllnrqtwKkeLvU!5G`qXhnECQGxG^-5$}PN|qV_i7-*5LD(8J8`xf4Wg&9yPd(Jd9stXN^f92Z`!dhqpG~CFB;er;T#F*GpHbpT*$nf-Icvyn=cSAL6yh z82<0Ar=;_0GD{wVbeh-3zZtHyzukxO!m40ib03@;wUh>h%%VU^D|G8V#MOlAP}$uj zX!|1sKmKiiwyRdm^s^$_K^`p1PhfAoWT@Isoz>|6#!avX)HlC_e=CCE?C`PZb#gFO zJMV*w^O}X0Qx?;DI~%IlEJqtY{=~(lJZg0g6>5Grp|qoOFlyD9oE>iOY`!LTPS9e-A9QqBTdn&NN&b6)^j>Ic+Sggs-hb936f=f(!L-Br$0;Ce}y^lXn;MH!Dl=+sggW z#`Ngc?qsSmI{*Xmp_owd1OF%=fF;G3Fi3nVU46L{dyQ1wZm%~MYF zdSEEs6rT^LIDZoQ7?9@O$=rg8`k0tUf(`qm=w0C^)bO-m4g2~*`-UclEbzhDGZ}(Y zUKwQB@h_(q-74xd(h#nlMF1xj-scxtn$djW zC^VI?LZy>CaOs2`z9*>)DlL_0-Nt)xXYDL#zcU1rj?RFSvCHZCxU<3q+DSMj{URw^ zGitwg1E$`wVx!+BYgo9yj>id(z;jdOSS41H_GT;kV2WL(}-- z^y}|9I`vSAO!j_8*{6|I;(Z-Qza2$r0_5?o1KUo#fcVICh&_Lb+qiunQ;HJsgMaO& zQ+X3fV3|!J_P$^gkVqqD1Yp$0L#SF&MPa1MyccEAt2=%8)WU~eiuPRQQ*L9|l|oc* z9L9Ew4`LmYYH+!<0lTJI$&PydK+AqvmbyU~zqxqRnK#MUxqK3KT`1s(|DH*Dd1Bad zK$qEE@SvIz5i~(^63?a!Scms8czt;}ewtj0M}ID&I|f;3y}%N*LbF)YKTRg(9!d+3 zenMkcb=Ip|2^BAlSZc&vmil!MEXl~ivC1phf~+O*X!~8L-TxkFPppd=`*jQ-ardyHPD#XJMh+xL%8UG3Zt)snC?I!Rh*5Xq2nSc z`h^0kaa#;;G@U5kG?0z^mPgZW_HZ_D6JgOH4N5UNg871Awtw0x`m%L92K*S$R`1fo z_Tk4-Qff7qwBZ!DOr{849$Zcz%_i{PVc&V%Vg*(`{S*It)FfD9y_H%rim-LO9KZ8| z6b&T&hDJ1IeJQ7~c|jh_c^;2*JP)zd(?dz2ZzLMNiHBSA6VM?iRg{-JgELp=;ixyd zwTr~Yu%oH_VE-u>7XLzut=$nv&m6M(=_z_tmX<^d{f@FUD<^uczY!nS8Pn!E13Gul zkL@t-Vo{1g`Yutrsz$LIWr(57>k;Ok*(h8-qtMN)t z00cgf0)xtO>RdaP{FE-z)p;YxCMy(O1M^w)*I1-8A5qV(ic8rzne%G3r}KYSGbN32 zOig_PWXmVwj_hAtf7%z^KJyE-Z%}9ISL|5(Y)3Y+>lSQLh~^D{ufYr9KBSg>9QBm6 z=*yeOC^)S}f#18ZHp83-T^fV8q?B-+_HnlKx;PvCs}EMnsZqt734Bw2D~#Lx1-*I> zve{~ruzPARh^-W18fEiQ@`e#aJ$GhpZrR|JA_or=_oMqvKUR5QAzmrB!&if}=(X|` zx*;`1VEYc2_Vj=Z@HlY^^R

Kcjzn_Ob0%{Lbn-B zC{?};D?d(Tsj0`H@FZH_m>)GdwsYwi4LAFQ1vmsRQuOX8}{?x1ywuJKZ%krg3FQ>!!LL!DrhvghzrNq3__0 z=soQe7HFKt#$0Vu&Jbr6AJr*JgmL`3I1V$+HHn0kj`^S5DaY-JAbRd;oRVkB6f>-8 zp=~|iY~BNFhNfb|^BH{Ev0>DB&zAP4jKQnUQ`nR7uV`$^FU~bBnVeSUl7nM0mlEsA zJYI_Stvv=pEl&xSlfRqo7#zX$4s>Dfi^25zsU=(Z^*A&!Z~D-`jmn0{(uboWeDIJH z{mgOU3z!Bg_-+OKypg0jtAN>mF=n1WZ$o-xGZ&WB33DCp2^Xg{!k?0V(D^)t-TU_n z1&L9xZVp%P|5l9dy%liNSIfdt>0InD*+_ZrOJHi#ME=_+AHH2biKYydroIOXu+GGT zDVQp<#~k+c@x2*|xN!R%RBM>U zs$3HJ`m$Lp`HL7;pS_NKYnwn*tcf-^o`)j)Y%;K4N*6PJK*(lg%63+vz&Saf(lQpt zSaiXH>Yu#l>^1Q9nln{ydB`6z$fk$&e*D3VxwK#WiZErU1mm_?;W=1Hr)oZfZLm4M z&nu!U)kko1rVwO%3bAgr6F)=u6HQXwMAh0JSheRk9?(6>`;`@A4}Xg55icXV*tMLO z?G!pYA_uL_2D4$ij^XSe13I|flT|JD!i-`SlrM@WK428*%3AQzsI}yF_!~$*Jcv{{ z0C5T`#K}z;R{WfWG8<%==F7!+qB)73oZ7~PzRY0VGa@kW^LtJ~WjfuJRzdTQ3tWv} z2bL_2r)Zs>IR2!C&@IA|rpJ$`j%QnF?g$e&WuZjr0g~+Pa|Ieb>Mma!?hA9*v{LvJ z4Z2+Y2L`R|bMop`Cyo5IIG|R8zJvCH-Zwp%8G4rc`%RAJ27Kf$pR>V3SA{VD;4c)& z&F1enh2TutuTYw6itW!YLO|y;XpY?VGV6dL@_Z*DIW^ zG@L2|M$plie6nzfV)Fa9q1vL!G<3um)YW-XSGq_7%wMlZ>!qo@&s8_lJ^qIl&XXg@ zzb|oX(<|=WKPk2WVz9x+gbf=s0J~yr&?DObR}HU)lon-JkZ>M%l^F@*-iNYDkF@D# zcmzg`?W6}g$Fb#sW7y5#ulUKT*)Z`)A>Iy|#_VP-fxwVW~l=ZY%Wqwh|o zb$f8(b8Ec(Yz-b)sKkT)*6i=e7N}i1mCe}>sEg}atW7QVG}(f_H=RfOl{@I<=~uWb zSe{S2W6mseHJFKA8vQ!`4LNyGlcc%1H?7K<@M#9xxNy6c;hOT4ObJ*lbig}zctt) z{tlMwck>e0=irT-X-sR{QjFYujej?09!*FvM!osL|DY>v za;)Tvvrp29zhMHS`PHyNN{XBh-^G7N4)N}jyjgMj0)C|XB$76Fg^9u~5aHMh|8)QI z{@1OU^uOx@`KUo`k6;^f$yZ|KtyP@o_Y6vL8^dR1r}JCXx6#~>JnTs}B8M3U@Ww`r z&2-PguGmk!+$NsxztW&(YZkGNlZN=BTp5R|>cPKL7x^wd6|$VQ2>kUlC|t~%38jwU zuLCpagvU(MvD?QVz28QCk35jC&IOZ_&zzIAIhL$zCC|1L3{LoovNtclD47J3uP7j| zbAEJF$(|C@jB&I0QwkZ#0ez46_^e|B4Ln%HE2kQ;3yxES1IzO<&GRn4@vMW4?fc>U zj7H&yw9#y%&1&vL7s1uQLI|(4qR~Qk*e9>R)?9l)k5`7!@(F|4jBW{XIlq!Vc9?L6 zfs^>|c}4j9aSu29h8&eVk!6c2tZ>V`ZZ2eYHGl6<6z1pbLd_Oc8rGEuaanWNDxVi% zwYrxErMSR|?mT`?+g+?ZwUSSMB!g2YR&r5AU3jHfmVDZDDe~a~c)neQHpj-1`@}(7I9sM1Yc`r;b3u>rtjHH;M!TSR?oHvP z@afbkw-?r&+lwK8Y~Ula#cjh!VVkZh#d*Dw~>b{#*^pG!(brL#Pk)}(6&RJW>2i4P-QES9Hhj|VG2o1IgPQ#{-n3ZkERI| zNPq4zKK!#1AKb#>++{MD{~>}u2)Xq2Rxdt$84Y8O`BU?f>)2m$6Lkx>L7(*%;rNI8 zO#6$GXns42V$C18u`&`8wl7EbEwc4b15NoWxhq-2L4~>lb_rBH-HocxnBb4V1MoIo znqr4$V!QW7H0*mPLJfjx*0qz+U@BRsqxPHEcp=IS%uC_s(k^azvouREzX5(4K$F7b zaGy|#bYfi4x?>PC3b%oYYYyQMFB5;xxl}E6PFw+9&zEt-(XLpK*iDD~0p*_kco^fF9qorz__Q zVb!s56x~`%13|6eSSe0k!v)+sMHyT*JA$p9kWX*FhJ)(VLJ$fRS!1v!`kgw9Bl+E= z8eEPtr*5L&=yot*!{BJmn zo?OQtQ`kk?^H;F38lQR9wOLd$r5GCzJ5V)_qoc!)LTtMQXKH(lmMx3q7fVVrR&K;% z!%MJU_6*3Mm#29*#!{GUwUsLK|}O$w1>0nh09)$gHOrazci-c%_ny{F@`;@NMkkbncSd}9`sIU2e#}_ z8DXLOR?xg+lVHbuBX8N5$~w2k9#6~zilk-cvws9_Iln}ZU;*?KZ|ic?Qs3B z4w#_p#>!`lla;ely{C3Bj438*?xg{(fC30(ht$$Byq~LUk>lbf8x}9t_dYGmO56x-zS)= z5J`P^ooSTXK7?O6q%e3rDJ>tvj?H>clit7Pd^MuE@r!rUcIipfTke3nQ+!#-lHXW3 z{tj1Z`VNCmOA3~(GiF6MDro53WL#Cbl=x0h<`R=2m|gf79o~hqrB;D-OT-&~nKzV) z_ZTqks1z6#Ax#Elt|I-5KHHLYo4b`b4ot2_alhZXVETbf{Czc_f|oC++uJPJqJ!Td zb;wRk4jaQe{|Tl3`Jceg@)Bph;W+o;pb@dD9<+S6CiiugonUahEvV}YSW;#LMd!rB zo^yTZvs@q5ZSKRwfLSo3JOYwS4xpt(3{4YZFfnNtF=uHiEd9}o%F04c%A^9<%o3wJ zQ)hEMM&i7+oh2Pt3q{YUYdGnpKK4yV&cNY1RHb%`a+IZ%QYFL8Lca3;HMY=p>lOb| z-x%+$(Ez*a79^;c%g$S>vp0{Dxj*i<^hrMfPamH^VfPB4;&KY7=d8qh+a^%auZK9o zCY3(A#G}Es7&iZX7hV-}q@(3!ILOzYKec)n6xKcfFMC~Dks^w9Q|HqQ?PFlDMGhkT zW$Bp5Tj+Xi%`T0oLYbDGxc=)@3aXk+k%LyDl!FK(+bGIEN*XcQC5W$kCBp{H&M@ta zgIH7EOhK=o;Y!!xG%7-j1 z!=mc5kUo7ZDTO~oiz8of?D87L8=`3N(Fqi(E^d!X_ZdlY`g=JBvA=Ww$ z3SutvAEv2N|CuKIIp0b&M_-}jx4HPq?K6Ms@j;Y`8$n~QyWzS%dH8+*1IWieMT74X zu~yZeDd;ES%?rP9@7rxO{C7UPAkzggx9o7)5jEOv8t-)Z(;()&#*M^PM3|zmf;nbw zX5uAjWa>AZVXhj;`}Sdw-8X)hi$8l4a)W>T(vpo<9>Z3@a)5hv2gqSd0tURO#PYA( zxa0Evto!u^xU+pJE^f?a9~&Yu|FR1^zU3Q~yp#e3NQ5Ugk+`Zqo^$D}!G|KA_3GR4 z82^<(uI>T%%XBMy`rs;8@MI+EuWv!WsWo)BN0XiCtHgCzmO;Y$STZy95M;PZvT3&o z%`3{G85+RN<_cwPo6D^oCdW3IDUq*wAq{qZ3av&*;P&qM=%t^`)SF@$ir{rT`sy z_dJZ=K2V~nU%OB`;0Sv7Oa(g|TXbJ-NIP83Ve@eZUb5&AZ9cq)?#4O5qL~Uh2t=GSZLt`s(^V+k* z{ULoq<(4F>Snv`TTdc>5MjsI`yBS^6x59`K4LDB9$Laip2-yAU(T^C zPlSclcc$?QE)QwncjVn$$3Rr|891|3AH5VCnSFm9x^B*7T1kEQ=|BQ?sjVQJoZo^y zg_3OJNok5%a1=fmB*T#Ha_r&W?ZTP2E70Hi57=sK<3|LXr)%bmaM;~_@MP97<~sj5 z4*BFq^_uH>8z0en_eYhJI@XChyZ-VDiqpBDix$vQ(f4xq{2aP6LY39ji`wf-vf|kv zxX$l|qIukciBXNj`IwWF*%g%2l3|0Y0{Gf$Bl@)>9vY zmMP)LN|vkh>4{7DkkuY^XKgOJ{`vte%N2xNUIIFcFg4k{Pf+SKg%z)HBzN&J2D3KM z=8k{DoV_F2t&iWa*;Nyw$8_-PEAm;g#XjNZ#$A9T-RQ&LMmW^Y(cRCFKtfpych;?d znUxlpwKWrWB#$M>)X^+YW;Ok@Izro9oY~iNogpK3fA*0Fz17*6fa-9VE`Q0IFLrt1J zZKzY%VI%Mgzl3H-oxyDIes1~S#mrIDg-*;>WuZz*+|-S_xazhp25oo@2_Y-^8yhn@ zPs2aBt6Pi9-F%iUQ(D1vHtVv^m17yZB%tU*6Z#kw1p=>nh`Oar%A-@*`?`GgYMvXn zPfU#5JDyYZoH?v~>K9Pk^cS!8g`kO63Gok`_-QkZ1+tntVCne_N@FHqvdT0T{w|zt z`)NWFG9y{fdw(8%2phMopgd1Wc4!IVut%w+IaZuiORTN?K6)Y^s(TApHywjfC%dsa z;TTlx4TdL?BF^!FD0g~kL{DZD3|jRGC#{Rb#QJc0VloyP9p3(b^rtQPvSGt zI;42P80Nq1fm^?nsCKe;-TDe2zU|sgGV)$WG1Eo~t0J!Ai>egbQqqOlsikmAQJUto zopdTU6k$@`{{#wiL#gHMNnGc72>!I>QP~& z4n4u0ynGH-R{Ij4SH%r^?@Tw&HNb^iU3~L^GMC$u0r53){GpK?)37kR9q!zO;*M# z6n9??oDK4j{ksCrf1R<~`zesjKmOQ&2%p*0BM7-+iYMO{^GNyJOzZRbMW!1&M0iJA zo1@^<_6;!NF3l6vej9^PLF-W|AcoZ{NAvGAQ{eSZPn4LplQ1@vT4TPLy;T!XmbtvMLiRvL7xE0k(HIpHCxc$X|K-W3>Y#3x1%9u@3zW z&ZfoR@1c?5OU{12mPnjoMwu6i;joSyY_F4}_v?k|uvVKopN?iRKc}&jGD*5-70c3p ze#C24MdaME2Y!1L(sM~kQYxCyuKms92SNqt81ezH2OI#eB2_Y-bDGZ(Z6Icb?T1~_ zi4e7)!1z@uH$hHr)8? zyzqpQG+pYf;Kaf-S^uDJerXxdiP10cihdD)Z>0u>RL#cib7rB%kBb<;p#f_$!kB{L zOg8)ZDprt`27i~WR2^@UAKl7M*R|fXPe1(WGPPgDMz`zZWIw=3-1c1k@U|JbTA3V#Mll>J9ZAu zHzlxOZGU$5z8O8Ai)pAePm(h& z_(wD5JfwTK`}w#-!8mS5F+QEVlpa5=!)}K)I8A?~aQ%%egq-Ve@4*-r5ipfCDIbKX z<1eCk+8PL~yNk1;-7$3iZn(JjA7WoOxVKGbU$tIBX31)x)3WI1(1ocBT!<}chGdcc zbJF&gc&F2?Y!!B?WKklT%g>&Jez>L<9-33igE8zfV%$IF^2d#x0*lbapZ5=&??hmnHwfQd~ zvhfbO{PpH{%V^{9A<1lIj4^&wuE8B@1(>;~2tA7=>y4uhklkcyI-AZ`lLI>X% z%dxA%8*s>>8fAS%brO-{RJvC{dp1izRDln_MYmnx7IqHq8=pb<=X2ojWg|`_X)2p_ zW(Ak0tH(SBct`zHog$9hk0e_!Lz9~`|8XJlgI0O4@5=jxuGzNrF^B7DjBp6aE)k$y zwHn-BHJF0t9e~@zdT^h!=fJ8 zTlSXQm{ow?e{-2Z$^m$$`~yE|RpKk-@oZSaRy-eU2*0--f}t9DY@BL6y-$>2r&RLs zj_*uNwN=9QHWMa3?-xp+5v`k*_wZ4DHNM~fh@&INK*oIpB>CvjtYv1X_}l}g2P%Mi z{!HGl;46fDO(chR4Ew)^z>5uQp=H35Dn4g|->{h&9-)P2URv|w-%>f(Jbg^bkzyV? zIrv_r(Kpg5!iCM_z;t&$x_RuuIgaX}snSYul4|rU><(Hcs=y_68OpqMhTP1}nZ4Y0 zkq##vBeJol$%oZ!8P4qO2Y8#A+GKyC4Mwh%RH`IV(Q)_R47-()@k&^)$+AuKYlD- z>`5Y_)iOxE=fpfG+TiiGV1^Up;Z zgnJj`P^&qZD$KGexabP3bjgL$Z(BH>%|mI3==ts7G`8aF57gXTOIKqHFubD>XHLDt z*X=b!eceG2g`A9WRJ(9B1!k8DVfD6dIO$Wu4vzf6`x?%rn*8DPspls6 zch|$)lu%*OvvTNPFqmv-Z^YTNw{aiuHP$Ar-cI6{X3X_omGGy;67E~T9lk=O&3^IN zm>W~{gWn&xl%*OZ(WkYgbS=q(ZED)d8V!`GD=&@a4LQU=vOC5+E>35@0bAH-gQeUO z(-iRjF2%jSHXki}6v-?4G!@r3;!uZt)>*ZZR7ASVzBi&evHCC687aatv?HiWgqt`S zSd!xQV(|EF!6JwC^Sd3ix$d`_ICfYrYx*ljk?Z8y@gF;#;y zq^mfFnxP!t#~RUxkEU#z0;ywlj8`3g9gX=q`Lb)RY zDI;$@4(WXaU)LK^(AiIDJI0OnXS-5gt`@g-&3t&;^B0#a;t{?@;e?Mxc<$3_2+}#o zlw_n?=oV`xc1nw;YKAk54=xydcrhnAXqRYQajYm&iIUY7$jHi$RLja~e3=v7owo}g zJFjpGdo_awoBES>YcM6r{^PyVW>HV>dZ${$R+3W_VO$zlVA0Ftyl>A-?nmV?XuGBj zwp#5tr1>wV_x3^6+6$b);8y-p)DN(V$z`2nj-RS#plOYKefInuw5k|EI>!%SxgZJq z)$%A)eJ+39h!@ptox$oU1#tOd6r6P4$-b=k$vv!4rp(R^{B}K>S>~L>&|BiH!}=1= zOAseME}1oydlP%mi2>7hf=KRr{ z{tk*yJjrg591brPL*MgrG0;DVpZPix`0t(Mz)u0FxywE~k7Oqr%xLdnpxBk_%y74M zy;J;j_R41jfK;p~zSl9^Z7NK)s!lMJVF=XjD*&gK^^JI%T68q9o} z#o3*!2U)4AEE^m2Td+mz7ur603)8>5L+7C5cvChG?#ds6=ctd%lX^KD=}N4zlcE0d zNo-!|ec`=V^Vsmsli9Vi%NYKU6ZtCxq=4z1g8Hqrl0eBdAH3e*i6b4yL%qb>HNTVcRl%)JO1FOEgoz|t~#@M{DT(U+ed#^ zhGOiB419fMGhG>aonwD9aqVIWO8WZ)HIzjBW!^=R=2V?3Zq;GbWo@|QvJ%eh;K*)N z99?Q#N84v#!Cm1@`iiy1WP$t~DpKz-+MHw-voFjE0*M6?{za z5RiV|iKaS^@I!17^~IOL(9)4?`}ivY>qnbe+MKiO@QjV1+kKi!e9mEWyBP>ntXTdZ zT_zit$^NB{$LF^XQO}zv)Fi9SU07reY3GD!7JL#MHdvD0{16(jv&3+T9Pa+h5V*2Z zlc|pkr`|7%f$eVQi}QV$lSqr{c%KvTmBW~`sTnwbRH3gwrO0#KUOYKip5*qf<}YXM z6zFUTq|q<*n9sOO9Fg<|o8QIL$TeL;t+|OfPSTsUwHhs+u*Qd>`D}yU5TuWu z7}QVcxyk-w`}jzhEd&@vJB)6%WTtTstv$p=N! z;G-b2`37VTs>7=5FHm8l63L#B=ZEg(*{jaAXlqvEibE8G*ZgL#!U;P#CNqc~Ej$R~g zn78FP?v;{c<#CSO9=&;}~{~5lTIttWRtI*Dz z1f2Z7mUG}Q3Px|gP7U@YOj1-&GV+@X8P(+D|6=I8Zbp^cy-Z|>PQ1}rx8e-RLkn+@eO!;L`p&7fG@0Msb_kJCB*2NXV z)lP}BVtTkIgMM*`UfOZ~%Yx}&E)RWW?p(~|Vf6TxGARXk@!P&w@a7=m2fn$EtF4T& z%E*-BJ`Lt-f4;z26J3_4>5TlR4&K1#GS+*jb|&Ux)mIUx>Tjhrn&gB3Rte3DbgiajiXzh(*=oQ0XuHiHSOF&zCp+ zx9S6I{kwbI!S@oZ)HfgE)imIcfh8{MSxqnhW!JjYiD9CC6aRFz5zV~f!R2;Fvt7US zslM$9tghkEKG7Ag?`lV>Ig@C8YCSg1v7q4_jzin(BY4@z0rot~Mgu!BztBa2=5GFg zFv$zoyDp*676rIM=mEc(`Uv^BNcb1j0y}!IqtlU8obaq2e-z2F6+Yfn9bwN77FWT* zn}A+Z4fx|0ryvIT0p zTmnmJZ}=GTi?@lp!PZ|{OpX_TXx%348ugaTJ2?~I_876TW`Q`?`vk9DD&+5JN5Q_~ z8Eh-f!_CiTIY5;lfGc!pD7>b}EfVPKd$Qfk!bcVHuu}Rb#ym zlK2Y^Rrp=~Ecf93e$MH0Bxv<0VfXwm=yo>*ZpztE?f60b=&n6@QsO^utX?P#Ht>Ky zPBW=>iuFj96Os66){J_w9Lw|S3+%tfV*3PB6*4Z>(ZxtEcJc)B%KB3iPBe)-b zk0jSEfq7Ev!7%eMPTBT^dT-t)g?VWhzsP`wY40Z3FqHP)8$`BJ_AEO`ig@V+uHda0 z3sz)7Qrj@vKF^D6!iA3H%WdG}ZwsLcn)IVMifSGw(V4H~D8}wFmmuz>_WjWq-*gKX zE`9?q?g)K`|4d=v=z1O_>-jvtShV*_zIfr#esPz*_2(y zs6ZvuHXVi6oe#+VwHNlh2l)KmJ(yOz0)ofRCkM|Sl*?0uaS_S1;qr0(mf^=F51F#n z?e=(}_ya!q`x(n-%%xxLquD0K#Z2dy9WCqJM|-=eII#`4;T4Nn|xNiWH@``AVG4I8Qb-w2Sn`?*#`ILoZ5{bUQ1>q zE#zyVbx#}GsD9*4d)?XU`u+G~c{n*JU&G)h$H@1=Qr`NE7wO(SjOQAZF)im2wtcMR zw^+W!OBZ)gci4LI+dqp)_uzqv%%bVESsXwVD7UDnsj?S?^QT~KN6S%wWD&$ZSHu^@%>{yBFu=&c9x3`GsDql zTMiskb7zOHC(}u}S+qjvng<+AKyg(Etas=ljjzL*fy6XkrB9i@b*oV9f?UqWHkwUd zaR}OP6>_^H*091Gz5+Xw%8Ks1Cd~C_shcg?73qiU!xBw!zSs=m-4gZsS8LJ#MrbL{xdeT87vrA2=ds;xG~+X037O(V_-7tV zI`Y@Jt1Fb5b+7|Hp7$}UK|?f%@jq*WWlU^(Qxdjkl$H_V!PE;GOKOl zw4_$^)sC+z!Ooq%kIaSv{fP0#(rj++PIfq~SX_HM7y=INfrF!EDalpnYlaPHP2Uco z)|j(+u42FVUS2&aS6h;%u^xVU*Cfhqm`N>O&+zwnI~M+3hfVpu2J1wp*%;6N*i_}6 z)K{__#r|Pz#vo7jJmeJ~US~*e)O9HMur3R$@ul~pTy-$P9M2;f#Jz{{Whd>3}mXvh?@1hgd0T=e-Df!9OVuLO{<2b(7 zrT}|y>eIx@{%lfZJcix)>!3aDGM@CegTl~r%wzZoI(bP<_A0d)@FkZP*+fv@uW&Ff zHKlVOUh@AW^VszT8^woP<;Af}df=DCJUl7ZDvyj}K0Up7=fi&xEo51DgM!(@4ht|; zUkM|Y-{X=dAEnmu4lrVyB8%%7OokDLV7x7!uN(G|kOH~=PAHIqU4(!5R zhGXflx`uX2d1(1s*iFd|hA~^iP_tqx{F$rCdS1l9=71>tZ-+T9WJh@e|0uLh zb)(uDiBxEoCzcIKhLm*`;;F47n&5N@&ra>)lwu^=+Y25rEz=#V^xfg4!wqs+whv2xM8Z3xp8_Lc|Voq&{)woG|T2_BfdgS+Xg&y<=i~%U$a{DR!@S}#wBB|cO;!=om_gl3OUIi#|JI* zNVBs`Y*kqRiyAkvD;ez&5<4B_A4E{#{bO{r_b1FByAXq+wip)QHF-E0i(iX=Acy2QTA01Vp^5Y3Od$u;yDBZ|P zF&Eu!J=v6)U>0~N65f3CV?z!f=Cj&wVv%AwINy521zTkEEC23-kQq<;tz#G|Tg|6# zjZL(2u9?Qj&}iV(JF zz+p7$TE~3lwaIjf3%%QXh`;$(ibXkH!oP-VAaq|M9N80%fA=I2na^eYeeZBraSRpc z)N`)OlDLS!!$I|fz|?teq9l{M(DUXfj?BG_&%Jf&v5^;M3pwTH@O@a3Zzfu^;VT&Y zDTk(0;mE5J@6RFX;X6*|lq;D1E*F(+>xgUI`Y|p(U-Waq1$^S)NDY;Zu+Z3uThY@F zesZHB*{_C;JX``3qH<}<)|sTedITNby_@+z4WhD3m(g*;OZfS+nbuu1q=DJ1NZ#zW zczmE1nt&56*{DIK;~e?c4H?vXO^HoTw}Fm`zv$JmpR=o zI?vk#C0)|>o9FIBsdF#*@cd*f+dhPiY~Bx#JZ{msZ<|1@qD>ROZ=$+OB8uI1h1VbT ziR!Bx&|}GBG!D81DoPK8EH+r+zlp#1c`Qwpv4@0jLrAhZmUgA>rploQ*q{|naN1!y zOEQmVQ-rJ_*mW9RsTff|w~S-cZ+*j8@{(YB&XnCaU(4GGc}J`DB6L|VPx3Ld+3ElV zYCq)6T5`3~Y_f<}3I7|fU$%nr&Pr5}a~RteE5j1M1~^`A$H|W|r7z36$zyr~RPN|T z^+oAS;e`gQ_ZOI-j}({bdxX*V`hKc>iOF6`RIakO@!FE+?|(V{ofP=p+XAnDVTmc;$^5I;9B71Fb%hn4#eZduHp6^pcJHy-5 zU}2I%y{#^J({wfOPKc9xPdf%(0Nz!()DpkVsDxdgwt;Ow=txe3vdAx1U{ua! zv&cJT5Li$Mv##4=V$37{V33f{zS1X~-Q%b+u@3a&#*x(-RR_naH83;X8Jr{nagO3M zoUWb$$&;2dxh-4Kqx(3!AJD@eVOOcg_b_?uzvOf7`?0LmcH#$~o;d1KG%jDRg_5sA z394LJa&;8Ubz3W%v#9|_zaC1;*C((OIs@x(Pd0#oE=TE)#&dY2RKitX^uWyPvD6}6 zgyji;uu1(8{QUP2^Rs*S_)AUTGW#PpYMUJ%+h&Tn$x?iU=PIgttq2pt_Q9I@^YPIF z8G5Yog^yE@f;;~iLifaYs(7#pHO?roe}keZ_Mt7YtrB$q$2U0tQt%fU>|rH^7OZKg z0phGwPQy@zT{e6H@0HJ@>B8Zh)@Ns0w9yr>P2NC8)q&JBa38e$^Jus03_PsRrzyTl z0;d`UM<37*4o{)pIukVp9))G4S`?<|12g-auxwQcZcI9lE7Jxr z|KKS!`fdbloiiQ^?%#8`|8oU@>}DD{E$e0S)#}u4s}7HR#^K2-O=`$F1_=sE^-E$+ zv3Q0W^jF8Rv@I%_s@Kf9{n2-j?p#XSofo6c7#~qzOCYY${3`zYY$itBEMm6iiu|vow|M72 z<4JbNE^w58g4-S*;3Ctlup=-YMjTh5R}J;3l0Auho($s6xf>{6a*ES&kzxjR``J}{ z6&AJ8kS@9}hWL(^{Nojp#037W=l0f$C*s+wKWD?c3{=-J)&y| z?QqrMGmv%k0BrA4rv1@xK(ob$+ABICsUuA2u^xgSH`S@_lL5)xJP*IV?t_YRR!sM0 zEiZAFhrBlfX@sFIxAo~E8kTwwEq2(^j}{NM`$#pDI6shftr8fN@h{Qfg~vylVMW42S|zK_?pO~JVdh8_tsO%%Ln^3f*$jk%$FP2P8@@cU81G&ecE~$I z;Br+h+zGn}pJwb~bJLQz)aCMc{Z=eC<2JUajjSky5ZmkU|h-bW8m z@=ZE5+6S|)<;`4K?E#uM?h$b^sjMMTiSgG5P`u7d2-EpT!K0e$f)y6hF0F3);}$fofzz*Ag8oLbWD+P% zqHPB3e&__w{Z0b*&sK(z7Z>@Jfu^{A|3gT;rHitvx)d>CAw2)|8+)d%#wqWYFtB1kw| z&>&k&sLT0@!_pYnop2Z*{)xj(^EMb`H=jcDtHG;$JZX1LVTbjSQBy|^%_o;p*z+i| zOb~jknMo{hw>^2~){vK70*$l`Wof~=bjDtnwFthJ4+oAxds+f@nSEjw1wx*_%-xm^})VrvX+{cXdHg6~4!lRNOFT@o#)Cu7hf zZH(+_xqv^o?^@G2Qf{1@H8eg=pq+P2!@adUT z4*&L+!u#8ayz&oIwD|Q7=QMxB{|p1bZ1O;EOXeCl=&R4Z^+=HPm_}~-lqNjZwUSS{ z8%i^V%p^-yJ#u$`hor!HdaU{-WZH&3wCjBL6<$ zjXkuk;}1PM28SKO;h*JqZn5qXdi=P6{^`x-=IY0zWd_G?H4bCXk0hh9W)P%mLRK@D{T|OUkEoE&a0z@gdT+0j@_7*2_Lu;bxXtoPm=4hI&Anfel$#R>W z?{lE8Brpx(xZGQgdN!AHd)M2r_x|_s++J&FYI#K(|6rWNhW z&yn2;Pl2^_roeOp-QreIxm`fV&p44nwGy{da2X}{N1^51 zlgN)7#9ltO#p)j@?)9)rT=J> zc8$QDeIUKiMeO^q0qj#pxA2DfY6S?|qUYH$l8KOvXjxt8O;ZM9`>`##~* znfb7_c_j;Z{}e+bU1&_|Sx3*QdSVyDC2g5oE^$97cch4?(6P~#R{bm=9|kthU9 zbr}d~d4&>NL(x*~DBg9qiS-ALC5sMsbPR~bU(F5h+vFt}nKocnnJzmZbpL`s$T7Rl zV5auG5)}-cF!9VR=6$i4A6A<~2lN9$>)~U*^T87cIQ@m&(X3Cm7RQ1A;I(A;MZSKa z+;xgdt3l~Yaxf;^pPRk068r1>5DVR4P?-9=ck0SFQiT77W1K2T8_RZ?t{l^8}kZ)Z|9L4esUt`TcQ(P;*oj-rW9R{6m;j*lqn2Xm@ z?v||Z+5TY|USY@Kq0V1?5_1!d@)EGp=^NU5Yk}L)clcRN@SW;KWAMaMi2q%M?`A8B zoetYTaL9d-ay)>Vlk}PUr}fY<-HADE;Y2sjJBZ#i?}g*Khik8O%VEdfe0p(v1#TOq zMssdNl2B=ZC!gflg}Vl9*N-09e2K%{!~4J?;HNmyLYXaSzmNMQpFs4!A@!GM?t>5u zfmhw9#KvZ6ldZ`DDrlca7?*)nR+sTu<}m8h%7w1iCn?(JCR|ZXAV-}UX!&O(t=KSJ zaF&0i0pZo`h;=uV|6?R3 zmUZ)q8vVFEc^WRh(=WL8M$?gP$qsBp1b9>pLhait>5Nt`7OZ8oXZi;8Q+WzOmlL_Q zYi@83*Dj)u^*C}hilRGp_T1WuBT3#Z6&^|latdwI%)~I1L|HocN>&<_gc&rn)ssGt z%IBA-tJB|&wlwb1IezIfVK&W4#Olj7G{;>Ndj}`*Go&=B^S}r3%3XT&@2nMVDvX85 zQ$0*I{xRH9QKh7|`|xLb8J8KIC1j9`!S~lTK1ogR_HAdBSE$Dx@A=2iQJKXbo%I@5 zDc|RxK3R*IP0=uT*&lIO`E>dmnGaHRrFbT(8#Pbu#;0GMnZmPbzW1~_OfFfF+0;<~)k_zj`eG zPi!T+*f|SQqxWOEe>Qwco=TMRAGpLcV~EahlAj0^YvE3#ZVX^8rTgfk=?Mrq=S61u z|B-p(dr_QUHXZKy$<1B$5mp}<#fqwD;C>GekHZGV&vbt{U%IkG>-7AB$chf~Z z^$J>N9z-=UOL+IpH28K(M7M6opjmM=oxAu8FK3vtqQ+;!_gW)L-{gUIZ+Fm_v_j1C z)rRXiimYk63x(Y0ae$eI$p6=T7+J&9>`Uvgiq3N9GgUb)&!u>~s|kaR7P8&43E&`? zMQ-Y8yp_oolV=D$6pbWWJXwMY_spgdH>N{pfE*on(Zi6jPdV=TRvdmxmwvW*qpzho zPS`(@-`=#78n4x3q+&as68t;iX8R!~cswmwGK7`+ZRHmRP2{p>?B{2z&7l3UOQEvR z9fsTPMbE;WsMxiOP8vtD#H=QW44*^i7rYdIZBJrat(8Jk&Rl}(?#0c1ugG0pfFaBRN7b98=!Py9}t`$Cd! zvk`i%qqUg4WG|H^X)({40?Uwm1|v)pnful<*3^3#b=oxOzqeDEgL*GG%`zY}D@EF8 zp+s0oE*H0Z*wzCNrmmAT^x*~LiFW{-9 zhCB0(+4f`vDZ^@%T&m0zDrM2YC7O1X2uwq%97rGB%W8(@k#2xNnC6t@)6?g%SjB*RE*MZoYAv^6 z#xk;hu>|J~HDtFl*0L-O4K{m353gYH6|Mxz;PP{Mc)tD)ehRDO#mnwc(2godF!_a~ z{1d1;5k}ep4jopG`c*y-4{u44vq~V%i;!R~o0a%K7i!QUpc$44+hWNXQz>v_An*RB zpZA+ojK5+6NcQIq2oh$7)Qa7uLR}*--zDi&{$B}3HId=2G1T1-wj)NT+qM}a+ z+*j&AD=B9tZJP_n&Ms$agD3HqPu)Yaf^A%9Ng%nOj3ey>`@r7f1INZT@j?HJ@QU#O z_&eJSuIin@p*c@sa7~=(l-6M8HEs!9)*VkrhrJTXKluuY`uoT_sS_53RZ@7jJ{H}b z3a)-Kcx^`7js7jeRi+K4&W}Q z$|~F?Xl%w8w6B|i?HiX-*d|Zf#1CP+Bs1~Fpc2S^ZNsm-ewB~er^?#5E@xL7g}$O_ zK6iXx5{}$s!NR7R(lD)R%u%sw z{P(hFkaGSZy!p_>pV(?kZ(DD}m0IRMkz@)0Z{G3CE z^z_v$bf{Kh;oVMXmu~^@PBD5b_+1)*y3o33H$aWE;=ai0b9=6Nf~D!2raj6CTZ7{x#Pn zem!9VeN*YcO`?0at*4XUF)e`xDB9EFeqHQvGsVBRx1qi7RKB|M3|G3)oHFfK68-lX ze3g8JTMa9yxMU8Ue0~jcM1z1U?8W|4b5d-XMPr0a%;kwCB~6}BXKI|-r4ldF$ZG*7 z!Z^vQ`(|d_KCTyc4Id9>{ zc}Z%RyOOV(qDLOT=aKh5eZg@8u#!`wJ;Up{wc5e(^|%$e3Uk+$JN z+$;9{kV27A>gW*0v?H==8_7nNg9qXZ;nH$TWi=!x0z5e zJc%2p-%JBCHEDzBBW!!3!n|L85a-^Vi{&Rjv6^REFrdd6PBvS@m=-UJe8J(a+C!Y* z+qq!aPBcUjpww7`%MUli%SH$3)r=K*xUY+Ea1(YHe%r9qzn>dlSqk&+%kitl^Dvti zUMo+x(34T=Fg;)J(nQO^=gSkxcSHfp*Y&~HNLw=L-^8Z8IEt=4+Hid7OP;;mgW1yW zFk<0)GKrO81*Q8jW?em&OXtz$DGsb}?_17t@C;TV-0v}I-O5c0mlt=hJdItg*5p|7 z0lkjD2Q#_p*i&;5m#7bBQnNT%r}ULym*kEfDkoq@#&Osq_2G`#f)XA(P@S(i0% zZ3ff9xFa0@ekx_ZwCv&KjY7Ep_A_@-;N(?qp2p>|_rdn1&>vfJitTRsi5{|s0`qnn zv@#-5D=44#H3@9aUIahATj?&Rj&D8xnupn2zPS$;-MpYf!x_1QjElc=3BjDxI@|)CZ{0!QVR^rbQiq6Jti9gSoI9 zP)#L!*JwIrFOA`$*mEu@|K{6hjRE}{C`wj zJC<$x;fQT}jo2T-QxZNQkE?o^3kO9NIHIPHk4haX^aOjsu)_t%^dfeqUS36;Co@iS;MMqx1_XSF#F*<_f-xxPXpg zne3!6s8SvYUwh8*OM;cD>6vg-qbY(Sc8|d1&mLfXKa7s*LOSCb;nKEWi5m^tA?|_X;C0)oPil8N|34j z6K|cGN)7JYnO8?GSH02(UPVc<$)U#uM@~LVt-8m}nN-9!iAC_EVkmvedIei29|5b1 zQ5e%(FY?=vh|f+sAu1oE4PX7~lj|#7KWz^b-@XNQp<1+j?=JqR@EROFXgi+Eai=Wn zXHeLzL>5_-sIy{leZVEbMWbF1J(6m8=6Nst&>ANCXty8s^H%Koh<&{83tgQ2MT$Dh z(m~WO#&;91;w`HI?B0cQq8dG!dW+f0EN^2lp4sb14YS7c>+468obDEgsu>7Pmg6Y9 zRGtM6PKK^mN9f|{jVy8MTZat`)#%9nkI?<12CMFcg4*`;?1HevxSYF-jGPG;9sMJ! zU19_e%8O`Ay%ja~NU#A0qnV+}9_)QSlC|Hg1b);2!3Q;mUKBlpo$4K^cVr57KmP&K zOzY8TEkWy;MKs6n03?1m3{y?JxDbmHZguc{%6+qrcf2v126-N&7yT2M{Z3;F`YXlC zEaUNo;NA0c-hzD_b!fL%gj1}=R-Ha zj9r6S$=$Uqa_U*kY>0yGx}8w*I;k#qcQj5LdY|iSJ&vdDG(&v(A~tDU3e0bd!j?#Zjr3KNDg%Ol+m&RT^RB(nmNNu2r+b`^Y;1p zT;q9d-qbhZ^VcUp$h%6up-PRhgY|H}&Jj{vE6C|OD8y1M+VkUQcx4}Nf5$8ajk8A96T9(0nEV|PaGWVY7! z!t-1K+pl9q$6m~3Mu}O>ik5QLP4j8~`b`v5Ur!%`tudndGk0lW9Qwq5!p6&vkRPxT z|Aefff39}?6sr`fHB6+7&%fZx8J*noSG{7b+>Pvq&{5WT8YEgUqm&+;{myM)kjP|* z?V`q=SybhC2CUNuQqg)VYOoai1aa2D-Lj*=1JP_t!ceBzoFwFpd*H)k|O?`HWk23ARoHl!I^V{dx%vO<{v{Zf7q#Q)!TK?s-81i0FBkGZ~ zq1#_IDEin_Fih)5 zw2VLabGHioipV6HO|i@-VkE!qN|ZWt87`zrvzbe?xpPv1tcdwBsX-lN z&^w5>`A&tQ*{`{yf;v(_Gsex)(r$U7p}2h-TfH-RYqc@t-9rp^z!Cl+p%_%rdpCT^X{UN&U*Y%+c= zUtjS0Al*oB37?C3vmAMez13J| zeIGAXOH$sbXUIuzr%#&)iB!3}TvE??x-id7+;LZjmUF}Ci^VyZ?3u|Teh$SWNi*nD zU<%VfcSds;Fu8=uba03si@0+<4}oyERj}ZBTN*?Z9opbsMM#}pcBFk7YY2=D}3$={mF-$ zFTp!u-kKZs3RbH;#LX+UK=*JL`kw5}glR#76PQ^KyF|b%)fjhh-0`HH_z{jEatlg%F8i&|W@6*Y+b(#V@uQdoR)Q+bb z*I|^LYK=F_m1xDTA=r_gM+J5XTzh^o)HH6O*0FZnE0fEVZ21TpORVrkpcjo(UPRiD zGco;iHZ2Re$$38#ZZ%z1W9qp-K+8HChwUCp7k3omJ+~l!b3+_@JEziw>3)KDF+r54 zEh;-@Izy7;!| z!q*Ul+g0QECxeJjAvt!m@U4Sd=%FI`&`KF>lp3)=!81|ZNTma-@x_n0yf=Rmz|b3XOqTUrUflm zD8+UKSZR*p@>`5(SZ571pSXf?W!s?2NqA1%6%MwZGuff7Kl!zr@1Q|xE{&^o_W$8`|U-gmwY)0?wDezWzK0R4xh;iS`A#M96c0DYJc@2M! z7a&yp+2}TGm3fCguXcN=Iou%(9_8dI84Qx zj!W#t8`@@U%;r{7~$t7$`hB_)n9TeYOy#}IEPE$k86@1o`fD*-GDy%wWZ2 z1n2eO;+#nZf$exO&YQ*_Or?EA%UP|~d3VZ3XHeWFCKl9)i^6wM<*~Q; zSk8ofAI_M`3@Q3pARf{!D@3#u$~5eSEI=t z4lwm&H))2KCTuicgZ}MfXk4mV{rsxwFujEluJNUjS7q7TopOQ@QjYztJBWL1@4|{3 zs*H2H>s_Q4@LAW!K#sE^Yn>p0MKwjpjP^7AelsCrl4j;YALVgwE9(#KhAqR) zs46vv9n5UQ5VuO!nz<4T?9(7x;0-UI$fv%F7VfxOA=S+a0d1*3YCZD+X0@pD509-O zPG=FjJkX!=4!`4ccUJQj&ha8`OL}*w152>?hG}n`plnzitrarmjI{}H?Aa;i z_+FZ=a38^3Lo)dl#>VuqYC2oATez_`;~@I{lcvt&s&xKB5PY8z%AUQ>q;%OO{PYpG zDL3OiPLh!)tH&x-;3>tvy{d!9TNG&W5+8Arp#mwZe!_V({mJ+Hb^bxNDqUW%7Oz=1 zaEe-O;Gd?-4!b#Xe=R3*Qy1}^$Y%Mx}C`2#6xQx&%RT@i=c|H4CaTo42NVVm<& zEW29{rAaM#Meu#PbQJP2GJm+V?GEr^*JR4Ss7+l1mauQ`lVQlN1)Rd)Xx88*LAy?e zp{%^{?6bX|Y6~LaYeO|g{Mkgk+LBNzR|X39|AA_U4!`$O4!EqpMNNDZ~QaVS}jKzH#nGmw~D*?Vm|Fk%HjOfBI!@QlaP0ehl_4n^g$y9+r6*zH+6@@ z*78Sq@PI3=+~q^AQq#!GJ{run90FSliTc=1hVcWJ^0iWTnFpN3VizYiDJh1GE+1w+ ze}0K$CoX{zReBVnWrnWd77*m=fNj49)W@8-&YRr6%8wd$3{1YB72NXwi4F~vBXfhR zU~FZA+ai>qT{lH!urm+?$2PF%6LP^XE{Fykts<%I>tL<5CwYurgn35}L3dROT5o8^ zHQmc#lu)+088k(-;z}@V*y+SJ^rxWq%Q3<{Vu3S~C~_t0}1d!WCIcl{a4 zw_CJu?|U}U^xHS^{Hu+ua(WRBjPnCSH3g;|UJrry>LKloJjtl%^P-=|Z1fEu)>9%$ z-Y+azQr&;#yTp?9$+Xa_gl+6%ryZVe`h#@`?$MD~XCZP_2S0XQGXCeEKt6Y4xr1lz zVQ#o2UzC>&)0?l8yKF1ByX^tb3K~eJGZIZ+`{InKPh!sR2823KB26yxF$z{XYpco z?`b|id|4RYO5IBnV`Jg@dAlc|5YCjUWx zDg@VNa6fVox+XQ@MZ13x=GcsZv%2upm1(r~nFQM{e;wXfCcvYM(%gW8L1dDs&W%`X zjENP~xZIX`EFejVycO1$CKE%OP*QaEr>^ z5ekRzr^oA}`G}|h@;Ge+(ji*ht&Rz>uYVfX7?q8iP0sVOW_w`lap87xS(<2|o&t?> z{Ed%}9K(|?rgUsi0{=#J5^0r{z@_RMC^wx?H)egopE;GZLe#}J_2t6=e`RJK-wfM@ z2&TS*vCQntXwjU4v0-~)qH7nez2?Zeo)zE-i?Phf4tbMz44h=6*^q~3Y^6#7$t~9B z^GxTXu0aD{u|5EEM^2*9uFs55A5PaiS~(Zxndsn`AvRSTE6hccs3m9^oqJ}8V>h3H zs19j*=&%x|{#i*5Ju);i;56_0u!lQ)+J~kDtz%oYJ8+4av`EUs0rG7uQ7ml^_wKY~ zhA`L6evw9RHEPg6Z3u5RKnJHkx&f2o-oa&cb&7I!V@daAsp5V=Iyo3pOF<8W9v_K! zmh5KVmp_N-8zb?{*;CwmZyhYLUk}qtv*Fz06mVKziw0G<;cw*#=nZ?13u1r4!CQs! z;khYW-}njV@B0dyE^ijg&Ju2S>|2gw_7~$KxkA`+vw+-H1zY(VveF@}5q2e#LR&e^%Zn_^J3oePL;kAtpE*BYb(|RP} zi{@e)+>~a)A^G7sksqTZjss8*-^%Gq3n>*ag9nm!9ehTL_{34FO*v((8HWqwncW_jr zJK5-}i|QUqA*dR_Xm2}`y4+U#{*ffEaxB6%3p=1_N-$pX8VEt7$5P>kO@ga+G>v^y z#AHSZ4AfX}Si8oB(@Ned*2`66W0jwQ%Ii_2=@W?-6BC(UUl9sq4;9HAqHAA>OZ>mH zGXbZnYyUqoOJ*`e5}K6EoV^xNQB=lClQJ|QAr%@-nbKs=SSTcws0?S{w?t7=sUA-n z^rS?ZG^uF#?f&ok|6T9<{-6JK{jT3S*SgL**E#z>_gQ7Y=ip z1I{Rz6pTHK0TEGf+})LyeNP1&?PT^?r5>D~>_e8^y2kJRdISthb@1n>pX`PUUig>n zHe4;6Nd(5&p`rX;qF{WGT4O$0Gj9AL`t*BuGO?XEO1>>m zAgh{>6LAwowC+hG{0D0IXUien5!}Zb4}1rWXkWVDO_AK4R0W$(Zb!4D$KiQU4x_(5 zorc{O1Z`_2yqq?TW6?FE-j&HTI?J3!rO5DY@Cb_J48W&9-V?V5DeOBQL3hl%f-)(c zz}_okf4q}mrN;Dg{ijUSIM1CggkF&R{6To}Q~~{V`lEI|=X+hW42q7NVG9FBL2jfH zJmz}w)3yL>d}TJgDcw!}_;na!49rpbdIh!#b+VUjZQ&TlIMC44q^_LT{MyD&=E<8t z8rGeOIVM)@#7OH~!<-h-D}o#0$H=Wq~4njgX9vo_fOrVpaJLh0Fq zy?piaIrXd%x87vj7vhuRi1%v(Fx~VV>vYkQuOg$(=^>tzVPieELrIUG2w9Ko8J6uA z6QYtmt&IOnZz>t~C%h}(MU>NzQ-`YeWc(k~=xbXEoa&-Xu4k2l{8cTCJid%pdaWTj zLw?NRI~OqL)I@Z4b%D%7hl!nkA^)+6KV$!LB36cvgU~rM@Rl&ABYrJ_mj$&*?)X<| zw4jB^r8S{cMiFIG+DR;U(qE^i@{&h=@y4@W7@ky1aybvUN89yaXl6DMo0CDW)xHL| zh;UFiI?6;v9fG1_EvQ=aom-2JjPXVZD$u6_lUyfIRkx!se;hX^EBp{#H?fe{WyiSb zD*(B=m|p1CCjrkU8VGmdjs z6Ur`EX(VZTX43G*3D~t(f$HmLvnO@aQ0|~PZn&UC?4_>5oKkzZyyP%BT4f5AwdqV( z+G7ykQVOf?YQpk!m9(vpB}#{Gf_hs${<0SaOS3}qxjB!<%&`aap2rX~Uli;j&k|hf ziD?xjz~}tQDaXR-ix7l~?H}1EN-eyMvMcaZCWG=}Kbbff3$vfgQ}?*%^z?!XXozSf zN9^Lr>aqQt4)hIYfT&M*@bB`Pm&Xw&=b6;0L4(>olA!gA^SJ)a1+XlPATIuwYZqt7 zlC(}G`f-g12;IoU==Xs{Vj9MOH+ZVcqNjmP*?J|wtN z8HMk=G+g%9L^eg41}}ZfPWN!YdzNR})RsP6H|r{$J>wQ+CfZ<0!X`-JwAZqnSNMnS zN1(LKm%JFy`2cy%gUl}lwB&OL8+TBG$_T$^Od}(y`kZo-#uq{HXM?M zd8RbEvVuKjt<6{*;QYSc3E-RHGMxH{I}!RNfEpt!dHUxyas0gzwlLKhW_VVTOg4=3 z$9n;1^K7Wq`fRczGm0ccpQbm%?Qz9~@nCT2B-v_tnPhk9gN<1)Z?A(E)wn@XAh{Mk z=uE?sZ70Zf#d2!-po9cfMBzYjF&UhG9bVtfCeu8OvAcCS4NB-|_^G*gptqf5>r6-a zAp>MCDO0zv4QzD1HM|+LgsW5YN&G$@_WzWowyt-HlZpyW<#?QXbxOcYv4a?w8)2h{ z82xejI>#$hppQrPVO!i*I7)ccJt#ika8%wOI^5_YSb8Ui-j8w?=&coeXeiDhA zWVVTCPaMqJV2*b&sx=+LfbA(5duu=1SE!?8U_Y5%sHE+ZwKPO86&!-9b~D(Vb4^6E*nH*2{Ba9Qpj^GbLc){X)87A2%)?u#!&LJBJpy zXLI9L)2W!72szDp5Q^nkf%eh)4fxy1nbr+$2?J30Pc$7`0TX~4evS%ouv`n{Obysv2q*z)?7!93MHWN*y%JN_6ZT5 zeSu_;oFJRqd@-zK0}W7UgJahw)59FwGsS8g4!nxs@8mo$c5hOq(L{qL>LnA$6b;y| z+C${}#?#yS!Vnd{0w={P)6mv#DC!v?T5m3cVW&N=&Qc{U?t6*d*ViP3F(#2F!mwGb zi+5Rf46&Nu%r2L!L;=NO7&Y(!kuwz3E3&XFDhRdsi$QvNJ6~>UGnqVig6Z&R!k&|% zbZD5tkH^&;)$`+sR_`|Wd{YM7>{n91f;n`RZ9e$T@L|S1O@U0=wNPF}(d79r7?M&W zk~d^9`Aa;h+tF$H>68=3`G$kWGZ9R$lx_@6Ifa%|fAX7sUV%i|7p8AnF?Dj}la@f+9E81(?GZWx66D>~H0%ZsME$3T?Z4l3I}2?;+Lb!zpfiI5pJyZ9P5 z{V7a`lU7iRfxEzk~rAq#c_oc@c!O7v{-r#eof-Gd*}pfE^_lF zg^Ov3$bRBs(+M7#S&&$h&9pYX!xa&Kv0|x*$-(h4mS=kEp=e|ZF-6YHA^bb6UT6*8 z-`11fD0ht4alp-;Srr*LV(! z+0#*__%>!tS;{C@IFbPG8g@?22zli$%J|C1@xwgg=nTDwFyx)g=v#4p+?@eL+0PdD zNQ=@l4;3(cw441AR7pNMRuCnf3sC(ui9KB-2M0WtkuzG+m|I>9^%uusfk`oWvds|8 zJ-c!C5`yo`EIEDpAq*6piQRrAkAT-3M^8P9FBDtt1>F3P9o*HmyM)sN0I2*wsiYi5_V zHI3iLeWtKEttq#mCpz7+eSSGpQY=dERnrNs611=0r;(#xe6}DO ze(rn&iuYQeH+>GaZg-}w0g33BxfV`D3gg^gBE-OM1VYbE!=zXn-15PU8qzr2c*GXo zmRoVWwQ45u$3?Q@UKQ{^o(KJ(99#9y8aTIj157X!LxcK*tony!aOSk$o;Ss*%0h9X zS=mlPtz2+>dJ&--8usXd0eu^x{8rM_EOY^{m`%~D^Gxxxw(G^d3sbEq^ApLr}nZ8xo z3j=zl^w+z2R7mO_G+FD?;I$ODs-GeQPsZ>BS05pD$@hrvwFy-4-9+5+trR_drPy~{ z#?Xig6I9$;je52P__=W*xxo3fYM5A1h1v*aWc>`vCd&~k%>`73M)Eb~JaOxgC!Mt@ zi5$_iB=2n_i020p5M|ZT_~JzxZJtP$yFMnfOVv>Kb_F(a{5he~TB7Z79}ZrUAj)53 zLC;$T$%}Qg{jDxlS*OTm9&07i>QjgezYc0SyIn_S7B`p<0nITt$FF9H20moe(3ofs|=qEf=?WN&i~s3xdD zmtZxbB?7MT05VLR|=5Z8cKLWsF@fC9)Q(c-zs%ZFtNKkm6lIDK%Ta* z!Q97tKq@1hL_}th{enMu0om*5*h*J8HX(=@rgDsfAz#i$`>SGxr6O4kva{sektvk2c#JwLOY@qvtjv6a zNKaQh3Wy4z#qQ4_X=+7X)}LktWO<;hYfjSrTZmix6XM*~%&}I|iMnqewVgJPwLkQL z*;PA0I_uTAvA)eXah4O^@+OQX@u$*;)5zYB$z#u`x51-bYIM=LTb#aoKdw+phQngg z*fu^JuS|W%ILu7r`auy^uaxFd*5nrKR9iz8dW*1jeH_5ZI(+>^oY>Fj{-cE((7U?o zFsDYUF{49<-Va(+YY_X0Jl!%0?ex!ppvpTE^7Sqp8r)9C%8EeUQW<=ip+l?J*)cU| zL&@O=cQoUe`O;fC@7s}4R>FT3UR+vCHfb9*j_vZpJjOfpCHV|B#K zyb98vuOR6ru^8g91;=?Wz#yk4dJAV0k7Ku3m%O(in$d}uTB?AiaPvtUtZC(}Utlx4 z1u{PuLGW5VDzrxnCOf@gK1GU<%0fvR7^+V6UW(zNVj~(>sn}Q^x`0;nDdBOaI&g_E zg!V?j}MjTu}7U68y5t2e(XIOZs1~#-9xz;ZA4? zv+dp^=;_d*<8wmL*>V>-wpFsR;(;(uN$`WL*&SryM~8 z^^jx=LblODXBLJC4O@Y8Av$KpQm3*N2AAzdME-ftR}Zs_9kYSxmT zm3N5$UI+3f|1IbFx|awoKTS#%U9kGG1}!QM*$Pzcw4g^iO4=ZI%h?kCf4l zJ04w@Sfc#p-MH-PTueH540<`8s_3y+xGBDoNV>Jb{eT3GAc}a4P$tEK+8LIxyPoKhWuFeytcC=#)U_0R+$SI}Od74qUXZt2GRTS*S#amE zD?Pr?i^$E7B~m9G=zjH6utvO{%w2H?lI4XlSKo}D=Qts%o29Txa|zk1P)*uA2O#F# zM0z^yJ4k*xfTMmt$#;7hR5BhLmpfT=D_5aoX%wFIo2hp1R^I? zHE5Q0g6Ch?2>;t^v|1L+3>7I;jVT{MXYY4-Vs{aie_Me%IU20rW@AuKoeTb0KutzJ z63ZYd;u~Q_D}{CO*BwtP5&x1m*1(OnL2}Y>gQ`lXmG+aAdD_8O`*5PzB@Ru4t*}it3H=+a=yCBp5?pSGpFJ`lNSEV- z7vEvdE^=nPge38Gd>u?PmTEAT90A7{qV&btDcna zyb~S;1HB1&=;8!AQSJ=UiTMa3-70jcQVJ<;tHFq46Zk21x^(NqUob~SlT=4v0(q&s zxI9n?eMB8`U$Q%2TO_~w)p zZo9u2_uk&a`e*OPwyT2h&_xpq)NI*jlKc4A^<*HiR~^k8YhdH{R;E~9l0KO0jz;>^ zQS0OoX_wAm{e0|jgGY(g!gKNb8`_HK)nJCf7q?=s?7;deHJf8L9`&a@X34P+q)mPZsx?KaNvtO0T9b)2Z;JRm zV+LXIvZpYG^QiecC5sIiHAJ0}QFz{8NaDLD5}nX7ct1*rE-Yg}cI#GhB6=#7_8aE7 zOO4Ea*8>gpW~pT3^-B=#AxAsSlWEVqC&cN%0c^@|1q)MElC+j!^{O_0O79&g+rFP3 z_4#vj7o_kR5m@V1(IheGZ62+lmEfOp#OIn%=BFswrwuJinh5l!xf8DLB{}w6h?<4)6b~OCYF#o!5)c!4ubK&16^v@&WKjZvs z>rMT)ICuU&&fo7>XGc-d8UNf|bBw`%2NL<`qyKv!!F|WyCH&|=?z!#6em{DD>>tm6 VzwGQN^1GP@M7Z1U_y6O){{=!6a^?U4 literal 0 HcmV?d00001 diff --git a/tests/test_nonsequential/exp_set_TA1/nonseq_fc3_weights.pth b/tests/test_nonsequential/exp_set_TA1/nonseq_fc3_weights.pth new file mode 100644 index 0000000000000000000000000000000000000000..64ca4119f3b8831591efb8201563032e21aa1371 GIT binary patch literal 41299 zcmaHSc{Enx_pdQSiA zoYJ6F8fXyJeZRloy6gUNfA^lX&N}Zp@AK?)o_(IRhtFs41x}JuVq&tgV*l4MLQF<1 zI3zgCFVxfD$I^3?UtqxM@Gvu9uW+vk>(>U&aF7&Rx^!ujgpW^1P>`Qbc%W#k{QM2R zejEIJ9Rq#BRx%6^m0jp0=H%<-n-D4|x`-!)%I{kJpBEV%9y-J&R3U78=+MP-qRF1?{KCEd zvzSoD|N38PvFN5)=&5vzBKa}??t2jb1wLQ zDsH5h!vCZ=?=9heA^%a&3>`zItOlvJ;$x7hU`^o2!H(4EIMzJ2}drpQf@>kbgoK*%vbHu zb#yo0UG|u_Hg%y{iycThbT89J3%-9Jf^M2EyZx>Z8?||E-L(uf9iPb^yO7IwxX93n z(NE~w?iS8ZaVIOevm3v>kY>_8TF|)F7G_G1WsCP&()7B1{)ws_`j~Lkr8Zpf?A|w! z=ET^m{R4P%jup$xF=2&+T$#RxE``*urgu56R8zAKWOeV-E6pLaEBO&WGyNKtZCykT zt2J0k&``D^Ie-=%+>dt_+A?w0S-rCT6gHN~K>ZjUq)BrivtSMv5IKULZ8M>bmBG+0 zFTxE6;G6D>4&2*x9$Kr#bv?aoo4v_ zQYx)~r^hxMM!=B%6nGokL@%+7+w=A-xX7&|-=meVp>!+_$yX-JPlXs?_Z04RYSFqL zC1Fyl55a3~a8h)w{!(f|$Bd#NN`*xK3D|8ofYSG<5%kEtQY<)Jk0PYKCM&Bf+GZ5Sq-jGDiuvhvCQ zAZhjp=ALNAn#B%M`Z_b{-zA1_J*pgP-o;zW@}Tjo9vu~;Al!E)n!Pq)T7LyB;^GYq zvwzPu#oIAXG6ai@&FPt0J|`|z!F@fmDDa~(JvLc}r>%-1WEcof9}K`De}dV0m+fro zTSr{ddIajsp9vSe&4s8y8Jco?CYv)dnN016(y*QdZ24ZI(>>up&!TvH_7G3*Jwq>! zEM!&UTKw-=O9-1AN8Qaws6NCSrVL{g_Ct;LzqS-FNQptQK0F!0{IOM z5H7DvCw5Arim4htZ}^o& zFO}OI?z~QI8QW2(L4lq-%;6^`UVx>A;*?urz{nJdOJsB z<20bkXhu)8$1>T6^Uy5Km`)x#!X2Ay4aY6tqTN#suzE6-nKdn-mlAIv`Q|8!6GTyN zusTG&)n=vg;x&=UKCsHE8sZbw=!In^s(i?S2~*_RqS!O&;&BY#Y&rz?S2oaxzJ0iH znG1D~)a8Y3b}&c#4IIiz!DSXNpvm4%wHi1Q^ zkgum&rYGgmM@8)lV?$#3Z&pUlgu1E(QCQ_SLCnD7O7Q%lHN$R<9a$c234b{(jm;< z^${E{n9!?k4fe7-o#u3X0pX7XdX%{kbCxRL;%A!hv(brf(H;Zi2TAblISZg^*KfQt zM1b};kHPZUMYwY8b8e{iAXDyBTp&!)Q z&gyD3{`drq)E&Tn`E`8obq)7$>LY0Um8?1RS*_aV7Xj7>V^Le^n@&~!l+ww2%D zb6503xu+q|ebFH=l}EVn;Z1(HLnguan=o*FFKM;s;fo+&ws)m7f2t!y5N(}@$BV7t zhFJ$0UK4b8_P>mAk)7cM^^#!{<{8Le9)DC*xtl5XFrW2qglq7BJIeJ)OlZZM`t6h zZl1wbTl8SbECbfwH-t&Gt--z>IdBzX$;4zY>X#@}+tYje$Gu*7=jKYf^Ei|#mFL3F zK|=2Ay>8BO?*%MNj}~kQ(1ruv`$6lRH1qm-iq~_9g)9k2)=_p9oYx-#i0wor%b`qQ z^F|=rlI*79C#Vfex1GH+nN6#!#5M;Nn(ktU;})I3*z6>l{_hbKTP|ZNv8&OoN{#8Q zbt85x2)1S4LrufYR8)MCHE1lw%c)Ix=-UUrI#HaolJa5Xg5hu?bUwT0okpI24CzeU zCtPp6kNc97$)*&(g;y)1+19RD+H|lE70kNOFT4cqId(wky@Qx9Cy#l40ODFE zQ@W`efA6-Xjq&p&GIqQJJNnM>i5F%gjXi-I?wGI=IWOEYb30k8Me%R;269&pOR@Q1 zw(vIItI4`$7F(CsCA1j*2>T0E+4_ii^($SFc@}%CqNUD_e^)a~{H%d*5(k z_yb;d>O3qTu@8%~&DjMBHFk9EMf|xi7Sn?l!qEt2PIb$8w)pxdzNqPput?!04EYv< z*AHC;Z}Wp-)P4qr*eKDJq%J5AN(32m!m5}bP&E8Js(Re!6|S#=J?!N%@LP;eeigYPBN$kayk%;@B#^tV7(v;@nO%D^(`Y3So4M=f8&SzG&a zFzy|S2J%Z_=dC*YD?J={Sqx$EZIx(0mfLoD+ z2i8c?2Qh*}8tR-)Ssn$*&877()2Lt1fu0nXQm>&N^}r7pvSKHV?XRTi3sYcQ(*Ue} zE5*;f+#`xNj=|ppij-A-5HE?ygXBbl>RQ~OPkeUm}dgIRs6!M3*=~d)Clrh zBnJH3GC1%yo=wem#>Bn$zUi#i(f4!S5_I zpi~uIF7((Un!oZDOdYXB(yv>4LngH>qlaUL^AO42CD z$#i$K2AwGC<6TxXVeDcJ!QV6LG-{72O;n4bh_0JhG24$9Ub+WOY8Oc*ZzKhuJI0QB z{)UrV{VCJig0?)+re`+>v;9>^@Xez(d~I(5vz8C18%J~bAa#x!R6n zQ^UYarGSRNDgsHznS9o#1K4h1ObvN6(IR9Ny*On^dmod-f3E)^SoM%&!OWWU~yjbCA9yKxPmJMA!eERo?pb%&s-`DU2x7Y}b+<-9{@Kz|m31 zz+dh+V8=9Er2G?h2g<yZGk4Q1YFx8&-uOW2VvEP)(|~TL*eJ`&dT#RsL@L4D#J! zO&9D8anR^#%tdw(DW$r>U3UqR*HUKwN6u1OfeoczN~5KQ&8Vq<9j5o^Q=I(>D!4e8 z63$Fx8H;_nJ!Zpcf9EV(-#H#_w6!R^V-ZA1rBi9|YEp^(hEIn{(l&h~T5<0QDQcFY zui09j6~I|C|5mvJYK<7ShqpdhAT>WLYa4n&?3jYn52wCp8}4gs_>@Mw77#@*a_s;7Ow< zHe9bEZDAcGCIoSZ4PDug;UjQJmM>Y*HsUuTey&rYM@kojR#bs;Y6{$y&ui%T_jBx) z;5fQmsG;Jgt^5%hL-v7Uv?MHt`L!ovV}c)Q&d_HqFG#4_lg7PWR?bg2s6wIsi%>Z~ z7KdAiQ*z-@L9?tYa^H}9-L{-f3U(lr11 zUCuSFoOOEoFzY9;F)$*XFAR3zZD-b#$wO<@@^B~9k>_lZ;!=^@X+*E97qHUBHW)MO z4Ysw0(fUp6KtgLU=6}%JhtdwsCt-skyBsA? z!@O)svaAIc{@h8)(Ti#LzjQY2Oa?`6cV_0s(Kyano?e^xf%;@C2)|*<7QY{kCo%)5 ze04d_93Bb2mp*|{_iJn@I)Eda7#$wH3(q9%hG7j6g4lVbWE1cT-~LLb8RvnPorvN1 zjBpn9ZVr7--AGgJZN-d+2uywU9-nsy(*~0$`lp^j89#2|y1YU`pZ{mBmE7q^fEb^B zV+tK<>%^WSS&EN)0%mI7u=m7T&iZ#8jb5S5j)fSo6LycFx-XO7jZ&l!SG8!-wxysg zCCU5OOxt*i$dor=?YqLPgYmEQi!FJy=oL6)RHD-3R^PoBZ+P;pc}a<_+)V}v@ps%5L7@;N2R&EL9UhsKfZ;ibE*Veq zt{@w!c)IyRnwJ0lgT;m?VQSa~*y!;ag&*c{XB}7I)~Mt7$FUlRn0yyFj;P@CCXA!3 zXDitDUPsznSPz5j`(W_VY*4;43g=i`GpWNb@y&WQHvdo`9`-v%E4yv*SMwlT+UX3x zv`tvlo+PNT)1}BeJ7HOj1~>kW8;w}L2XfQ<1X~iLAhK&J?R)&i_HkbajBnY-d~CF7 zuzw}_9C;6>tj`7uE5j8cy-)zhYoSv@0d8@gcDjhd8mKV3eJyO-CfoVxFcd zTe~wDDnrvaxspG;y;K+WF&(mPG=e?yQ6y!27JI%m^LuM1P`J`kEZn}E@>eK;$*5uo zV-Eb%TPrZSIF7n9Gx9}0?v@{iU`#HLm^Bqr{ zcw>l?DQLW+`#l)b32~Cz43-s59-e%z$s&qtUy553Rfr zLm?WfB-S*#=5I(j>8}pLdm5P}H)SDf_YkK;Z+DQpbry!?JjRZD=@={h9()Jb z82Lk=mfX3D2fMPEr-nV6>>LMc9lr8I`u~Bmr8pc^+e)n}0WeQzF1+!3f~va?;e+-+ z*t_@?W_er^?kw)d38MpGmz+0ouiRnD!fTRFx;iM!wpG6MI*X%)evgOn&5eZ9Rpn%@brbX~EG=GhvQ&23G#A z0|#LPtW~N9Wsi&W)Ju|14joPD{)L>FmLt>sD*`1v7)xPx?8Sf-^R0XiHiu(u^9Dbq z;`SYo>wOkS-)|6X-TMQRE<4h^2pM)<*^n|< z`D6)}`ZL%uc-4z^w-EG!Qq6 z?YWyyce>6YEy^Shaa{^3?Z)YE+XOeYcj1hkjr_N^SblA%CfiXM4-w0@(bIWn;dSN{ zu3}*>&56)w@B7VhkGCs7UoRIWp0r}^sW7&pXd8`5Fh zbh~&q%~MLGsfrrhn6SZ=zSfB@%P$A-Kb>^p)I;>EI{~`8Y@vR-1N)tO1(bA8vNwq( zYvFMkPR-0pNX`;7`SoVdDLOh$yeM#Ny~27QQMkqY5YPQ&8WUC;*#{IfZe+2M)G zuq^H}Tv+KzW7MC(ew(A{l-B}_PtL@+_1gI0juP8aHH*I4$uoD`0i2dJjeTBd!KSM8 zW0nYet*?8>1%F(K>UR5R-arCg{%4gW%J@!C8dw3V)Jp6ZDpYg9hVXoOX z*sWuSX6|QTk$O5iWu;72%h$6-yMjsi)l9m->?q8s$$?6*G$_BZ5M3LFlc04Gjav2` z8*1#?Y~6F%9CwOT+EVe5^+R}iP6F*uT_*b@D`D5KAr$3`v}KJlMLo@+%iAV@uo{T~!4((^r^F=G>N(vb~izS~LTwLVq<8Agk8 zpOKU(uQ;=~0IT=j;wo2ew->r#z%)5btC6E6x$&IywQ4?V(Q#Oz_z_%=RN{T7 zOse&{go{^NGJ|s_3AZ4>l;>b?dJ~yIgZyW?us466qTQSGSo6&i1Gg?6&Ix+e>lVQQ0UC>TjoGmY)`z@HmvyYz~7v7mY|< zFp~T?6hhL+og@hMg`$Q>*c2niZF^fKtol@i?{vX{S;<1;Rsvg6tc>I zmfw+`8Xd#)uYM_z*S3D}wY!O@%pW4|lV%fZ$AHB+8CtS(4-~G&~0WpzL+B9 zvR!kbHM5epxOWXU)tqO(yAtr~_mS}BMgr>p(*z?tM5W3_*rX*vud1xr?7a)X!S)M0 z9$LtG$Q=EtJXN>@m9*#~Q?%ISbGoiFwuf(LtRam&HsoONz7_^BhTyYL=uE@e<>rYqes z+DzB(s8Y>DkzM-iM=He+;Bc%fvwX?OStk8l|f!HGN$-k7^$M=bNm(zKZ zDSCS{P4L?$2;L|L%V%ZM=!jH0!RAv!<9ZBpd;&+$Nl~7uGM$pjB5CjI)rV!G>Fon| zoO@K34)j(*zrkR3HZq~Aq`(+-G>YkHU^KlN_Y#{h7kzurz^=bWbZlHcwT_Ktx_)yh z#iazVq)M?n)yv7_=W85p=|FWoX&9cNO`TesXqHtd#9GU-O{PoP$*mn+(0DPnuXimo zx0fJSy>GDg?>5vqlnJ47u6QJLEWFyX63b4Gpf4iK)@G*-H7!i!w@Rmzf=n$}9eE9u z{RSYq$rdW#6jSy2X6~qi1Nf|t;V$m*phxEAGkbFbjI+IhhW$%x5d5uF#wS z3EXQpn3`Q$Q1|x~IzL;!dgKBp5l*CpeR<=l@{u0Rn|m3Y=Feue4`L`UQTcBR%{FL{vE~4j+TU?&(4hXxx6i@$1 z!)eZYX!n@=_~?@vD^OlVGqxn)kAZt!uuBERA5owY8Me40(}2}B1q+=jQ?RaW8D6aa z$j1yZr?*WX;kR^5p>5lkLWTGS0-+mCC%{t*^Am@)F64 zUw}7?=JduP7W?OqU_GVz>}i%Nt?rr4gl3wY;k9_->U-B|?>;H^GtL|P+Ac=Yx96Y&<{nhp_owhJfV~ z{Nta&+|UKID{C}p9usjl))MSPtr4y3-;Bw>_S;U@F~V(rRw(mT6JxhHQNx;K&{J%~ zIhKE5_ka^t&M`+xKVx=W;uuICIf0`$Ux3I$7kFo>OI3yu?D54<`1xv#@UgHP-wdAx ze#iIX$5tbHX6H}6;!n8u3+7Sp&$Bd6#A8JLQs%~14xt;veq+drM5eY}g|y5%Aopz$ zZj@UOqwcD+4~v!f`pH|!XG8)`NVXz_XJ0r_)u*`{W-wY$9ZP~FFfLk+UZszxiLuG- z>2(>*YDnR_^M|ktpUNm+oUnROA8(p`l7;wWV&H)c+}I1S)GAZ({n=o);c{*DxHXbE zY~LPgI4#0Q%vI>1dza8N8-g>@ zQGP5N5feoPO9!#*Ze?`LIElbOWqVnr2isIuZRik<(Ao765rQkX4& zM!pX!%V)D?#>t?tZ#d?y%V5FkF;oz74=vpGGDkIO);Ir#EkYN#s~q5aM_$3c@Ew9} zFXj+8sS{IT3=RquC z<57%M|IID@Xh_kc`#EC#^n5o(o{?u`e+jvrYFD^F z!CLhDa})QiI*=NM*E7ZV4qUITg9~05lG`^YUTfD?&VJwkG?m>Ieh6^Gg>ptr_GSp^ zLjX>nupN##E#ZG??!~PqhH#5|^SB4sfzMnr1fCq|gdGYVIB39^uGp(EGj@#cSzdwj z-JD^GJ$yybLH>_}7!{S<325~}nB8i?x^uNrN93O(O%q7f^#vEKC*tx` zM7W`dn~gvIK%f{A1&tO5VVppjc3PTZZJjmqn)C!+LgiRWuRVPI_MR*5JO}Si#n5BL zM7D;rqTX*~*uVQ(s2v*%?MDy}{~S*XF0X?0pY_;cZp==F48_O$_1Se3br7UWGXA*~ ziI+~Msp400S*Hml`YvP`ugdNz>A>bLi zI+aoW?q{@89l)b~VPNyK8?xm@T#M;n30myh6_KA>ya%N- z1frO#0vqrc=a_qhH~XB11rZ-Ww`4t4${z&tEG=f!If)H=`We>S>+%x(U^qU>nEl$* z0w>JXX~!thcgUYYVQ~pGrNxa69ixaqo z&(k1MK_1H$g6K=)21*sa!>2!un6vVB8h*nDvY#xVe3|Lg`)N8nlvATcGB#K&@eVt3 zUU3~w=IqwU0Tg$;2$lw-I+tA|*~M80A+WoaJ1d;a*f~=cnLG&>1YG1VCV$56`=WYR zpJXoN^tO2oPonyc96w_XVs%Kf8j}|GMaNV60aKmgu{*ppl!)-m}PPsK?UlzO=8p98c^m#AoEr$zm+X+;+ax;|wYXxiLrkIH59LX>4Ov=?MatILE4P?Gqa*U`NAsy>`82q8ZV|jZHIu5xs?f4!f)TQ>Ag!mJwjP;A z{rk1q@FRzCkCinYIy(auBCBBJ*c&h|QiUxYI||S0d87Aw58NHs0(akTqkkU`WByc2 zig{tg<{!JxA8OU0tsBGHq7(A0=l0`MWxi9#&zISna?SA1Yn=ANo(_s=*To3yauX4j9 z0=d`6j#5Rrg!?acMZ)vnvb>BG+K*z8Ns! zY0iatU*%t^@8hOzNo2>&2C)XML)_i8I$%2aFePR%d3hSs%~p9z6UDPTHcqD2kq4=< zZ5|GJJ&vk%PLh<`JQV6D(aRDemXs`U`@|35biaIDUZ01k4xXc*1-MS44kks};DD(PsTG{z8mo`c zv1)gA`^$0uY`i*cUiTdLK9I!j6k}TWR*SaxOoFP`x18R?0`~98SLm;~1KAFv>A`8_ z$IV;?4Qv}OS@;t5QU3a68xbbv6ISQiiQc zSxzcHJ@7$9GQGU~8vW;=hO*hanZcnPj@B!%%MxBVZ{Bq#`Fl4Aw~wN;H7c}x+Yx-W z{UyFv8;|9Uh)UxuFx5hi&FtS(t-5$BoY_@MQ*`bK&PRyT6VGxMwmI~6Ofb9c-Uuxx&0*S(2PpGr3B0&Cm^nW;rtys-0?jWQsAAAy zuzdOo9)7BYCb7=~*@9oZSCbwsjMAYAck)2FPLgha{0JBCSkWVy9KPN?MQBoN!v2MQ zuda;z!5d2qz~5~r;hOew`n!J>95UO@>KyI3`5xgE8a%uz$f6mBoXIEeN^knw6vo|= zx{1bjwDIKO3)t7}PM_T>`J#)}IMnkE7!8-A#)>s;!>hsM^R*KCLaVq_qFTJgTlMMt zsI5#Z40+dIp0xa)D;#Wn1SZbWG~ZB$j1$cTebHuYcg_l|)E`G*_UvZMS}w4g=Dk8E zmkqqKcr>aPeaDL6I^MTJnIsC7*(?3IXfwiGgom8NJu~9@Z6gn(a<3)JpV-K~mPmr{ zYvORs#QUH;c@&%Z%#eNO9)WCoI8ODLOvCbavk&HFEKFIPC6%t?j~@_%^tK=rKU2k} z#a%-6ZELW#dk?@9Yq~X~2HzNE@LqwS93k<)hOx6c~qKjoTS*})KZ5|=)x#w=$E+Nl+e@U@aqe=8t zREW6U<{X!rt;lx&o(FY5RB8D*Kh$eh$61bAY;HAAjhol8!h*51?ou+!x0$lG zBOmas`D=)D&_xv`2hRTe1I*m86gQVQpx(U0+{Is4x&4RI;O30MZ1Jg;q}s6r{2OXv z(8JT9+Zjb2- zJ>@W%&NrgEUHx2FjWOGv^%N@pCBv=0a(J6kMDMc1nCzPE>`(D62cll*oBI{sU|Tw^A2=^ct$Q#$O^*4#4yMQIDp0SlIhK8{gXBkAI5jGk zSIAySl{IHD@Y4whD=y~FJru=AW?TL}npDB6#Qa0)R7*c!K z&s|oULK};9`NT=GOzc}_wXBLP919%Amf0;7+5081r-UOH2_5t(N~EIoy0l`zfc9_x z#&62fq02QJSpO;+xGT|y`$hS@K}HX(5-PHsV?dLS#IPmlU$~+5wtSf3F(}NHqYV)g z@xAgm=r?o3?RiJwU9kWXqyaLAw}6*j3Om)746|1>^Sv;W&2ZfgR^?Tg)hwh&{akeZ zp~)V8Is)v=YHI(Q$NXQ^;xU7H^kt_%JnbK#N!0HZ~M66T5S?!k7YTPXwM4?97}yDa*ldW$;K#WDHwX|{ITTsFKo zfOTythT1vHXxrs2#9gytFXKH0$={1uwbE?5_TG>MINydpLj@pa{1G4O-i8X30IVP1 zE$rf^lB(@V_Sx}P)nX41`g`;!E?c3FC(8@Ld$b$v(OQe84I?P)gg$JPR0W-s3jDkJ z4EOj%2i-fr3}(#P!ngcYq@1}Yq2Ixd;zS&jYon-WRK#ic&s3!L{n1o)fJ47c95a6! z%_{fqB3SGTYWEU=P3aKvnpSMhR&g38aA$2xGjPm-32e3eB(|%246SgxL4Oskn494n zysQ&W87S*Tw=F_75&;M0l099t4f^%3NBghzDPOa_kZjUO=B^`IMA4oRZd^#m9G^0xDj_hE8wZZT=w~o#I8e_+)<4 z^tQmH}#5CZnFSiI3>@7LB;P?1cOqP;mcVhI447zsPj9F~l6@^$!f# zdUFl-V%Tl$m>vRdotx|KKkDk0l z2TRYA*`rmU`EMF3?^A^G{0_Vo{}T_LAa0NQcxnmIrWq&G8Hw^S1wjDk7NbLfcd97j z{9m-wn?dp^Jk+Zlh3YpNv}1x;O~v?aEOBEVcOzg82_t~ciTY$pGNyX{oe!}_!BiwbIF?%BmT3u$x?#$MO@=3~^ ztkirqRDBp#%bC+E?{ucywt^)Eh*}oiB-unkF!Uu5Mu!TIv9YX{4Kln^{Bzc*vU{NncJW#khDX&xDn|0UHP_;K$e|j_&>n1|n zjtN{-->*zcXV8`&a1+kh|YB4K>!8}3V~G}~2s9S7e}B;VyZaCw~;E?RRM zeTI4C=v-HHo~cPW2OhJ<5~bXn<62ByYCH8^%A)APOc;oiB*pp;oFy%agPoq@D)BLF zf=7iwv402+sfgtgTTfB4{T|qw@JMK@r@(ZwHqoJHoAIBFJGXM)2o`8l&B|w#V)9%a z=701I443NU170|jY{p@_^-u*$G==oVG=OD|v!axD2k`VcXPBV1ot=})7fc;AmMsyl zhoIJzBp5FaE00w}_s>bZ;qF9=Ian!>2yutN-e_*4MIhcxQD6$QR?^dv^2s0kG zU};AM)_r?|TILn7+trZegtg*QhhE%%yGY=d>`Y|^n)Iaixvq0=n8rQt=Htyi!>#*+*~`{HSm9!R zsrq;qsO#OpbE`iK|F(+!{+I`>!zP)#xOg93dduL}5@TF?!H}lj8OJZap$U(3BC9hl z7W1Qiya2856Dd!o0ai6RFvS^hG`BSi=Ks_qsdvQ~U8jNZwkCMdAc{QtYw+EpF*G|{ z78f|Z=j(djabZ1{6iB}?ZdE#w-&XCvGsvoSXY|OZwoL7wGdN!dF5|a zR_9M_JhTH;LiWhU)b|p7Uf1wWfu>&V$=Q+w4~IT+k1aKI`b=N z#v~1PQhyulyUFPDo;ZGXDh{aASHe=&3(emVV7950o$wUsnP z5=Du|bI&(PLLo&+$%w2%WJW42m8SMEqLi{C_1tqJ6rx{|Q8wA5kVxov|AD7o&-1$X zd%owq-=C8vdY5F$J{_Dyd-P_*h{zyxc>aR#JG~!2p7GqMTAqFbwX_#e~u*0J!kP79CcaBa3i6bQ>anXu-AKgoVI~U=fGxm_GB+KS0L}J0# zp~5+P7Me?klDAtEY!Ms)v3wl-*JsNVK8Uy@2N$xZD$eY%=_Bs&>^hVk5JP)gok*_W zE98ti2nP=J^0nr>@RZUWR9=zF5*s2|wO&4q+6T13=MB!hoI-6186^5IiXWZ56q^h+ zxxC{)`0;O+V^e}K2i}n_sDWABpdLc2Io7oA#D036HG>^fw#LQmIJ`2N%S-ilk=>1I zl6b$FM!y-%dSDhjtslpdj2kIdavx)7l3}7Z(Gt--)_>v@otACJu@_iyn=PCx%voFPP*gT*+ao|st)`88Vu!>G`y+aBXB81a9&s&*u5? zzV0)d>8lMH1u3|4%qwV9w}ZI%U~P8g~z}qZjs&vV8!lZcc)`7|iB}nxNm}HT;E#ci=&>1ij3B zjK>G(!nG~(+`X7QJY;(Xt5(*c)1wxA;WU5@@`h4a;~>`UQ3ZEjI%D57U@s(Epw>X( zsNDqSe1Ipo_w0fH5-&r7z^OUTbHY{f18KIj1M-q?(4x2>ZECLL*xqCK@=G5kYI-qg z(I-&cNf#{y{>|0-SsO0o_#ghiSX+3LfVpl)G-pY~v%D z#9n=9?8?Khg)g9Y>v*`apqT9x=98xutps~5OPV+1J*s+I^JW6SGi}sEREQiyy^jZA zNQPpxM6h-COp)ZzgV;J}GmA3#hLMMtVg8~Q)oPx7T;8cd{$%?&a1c1m z%w?x}{lYP*<0DIB<}IM)Tc)%r&XB&Wio|3Y7lH?A=$Y(>9dB>pu##+;uyg^Pb}QqW zAFI)YH~##sz?ry4oLuwpsyTRfNKm<(B0V3VNnh5U#LvNBu;KG^I#4~Hw#P(3@HjR0 zXIMJ=DCkjyuh2icdl7ysDn30T;ZxZm6S{^^R zGLgGnScn=Q#L(`tnDs@<K4%0T&CNu)iPDsqwVvNvwI5zT zNPwJh#%@Y(0oZM42S+-G&`R}sC|IBZN*}_>p*R7K*<3;o9ma)qzd{3pKlrTe06t;|v1j2T zmiO*8x(a8K`@k+u~>Nlc+AO%s1q9~@9s1<+nK=%ItxBAH@L9> z((hESzkZZ0S~3xM(_?kta)&Z~LRz ztC#y2cY-&GzJQ-Du7w{iqsZESF|U~MkER9%vZcAt@W;CkfHnqj*H#bolT+B2d^7rY zbSJPY-?+)y$GGmd(UjBhk{){QpktT9`Tl1~?2_VLTt0jv`Nt=tRMU7CIDI=jFOi|E zYI5+pcR#19_W+-kt)r~CJ>=@5L<*bN(}we3(bVrba?P21!%e}t)Y8Gmo%g}6W*DhR z_lu{sE09NHENs$vOZK~UX}Vt*epX9^+-auBTOAd=_EXF-`ZHKojK_!mL+OcdUf({Y zK$_0q__TRpoR+^0g&m*48ZPT%<$qfAQ~Q}PE8of{ot!N2&`6T(_4Q)Ld0@f89R;| zmCH~vYNLH@b_YMldKK8Oy2gLs6N`J4wxHE}XXagf3^&#V<9NBz^u6jB8GrMm=qY-1 zgNp)LS4-A+|0@5%Z7;+;tHf0T`5^_S&lS;vp(XG)F%`SMd+>9jtFh?tYgq1U&HR=8nPSm6zHHeou-~x@P7Df#Kj|{0 z(7y{dE&nL=HBxYubS&G}bO!fmD6l+bOL)4>h-dw2?Aqcn;Fu-2-AT$c;kPgP&NQQz zQVX_?y%4=xdlu$-X)x1GCM;p?P~_H~7569YL>wK0>zfQT$*|9T2`sKU4Tl)?@C|W^cz)Rw$nEll9IfYITy$TwtT&q` z2pLwd%4c&`Rnuu=sv~`!V$ZMhvu3#qjj(mV2aFy(ioSH0lSzj;{hDM=W;Nl^ds{@= z&8NuDU@<*EDut!nn}r^08ooLs=;k*<(f_O>yJ9dKdHGFv-nJX~MQiBl{>>t-zx!F^ z)Jo1^O*yxEZxokO8o~T4f55XVNdyB2;fcXmv!M4CHf)!p)Ay<|s8EsJvRO?%ogXkl z;NkSvDMQDy9emcg?XXE%gKVxi;K67;eD6IU^E(@Hfx>p2A7X$@8W9z8!ePtV@0^p1 z0W1GK3L}r~qs;{~XeH#~lTKj{t8WiqT^MaoFQ(}3$$WNp98W4|VQ1w+bo(3&@nuiA zv~SgTJ1iO9_ejGe`)5eK%kb=%wa_v<7~>0du_?2TG-aZ2;)*I*YE?vg#p~els%`YI zX(ao=-QjwJgYZbgbF|gaBiHi-NVYQpmd-y7-M5!v$(;Yt_3&YMyK)mH8tI~jRSgbO z_=>JF$=H2w0Mz{uddc&iaFOQ>XxEDz-eBTm^fd5>k(F_D{fG;CiehNhv`zS;z=)>k zI5V$KF`O&=3l&_yz>0396?YOz;ruN6k$oTEef4JH{h6H4nyGMbmKS~PxKp2Z&Gn%CbdSZQdCOSKeVGa^Tly|xco~X-E z(tcC2Y`cR^J<;$vY%^TE-GaXMTI@sU46)gs5X_mrhlxrQ;JxM&SWw*s@mDj&V`fez z_2jjzey=tf{}Nd2#{!2eeE)AejhU)r1sW_UfM0_Q`6)a0mOSd(m&@HQqiKMQbKUg1So%_u}S#?wL4=vt8uFep=Xq z^2$-PVwo}=oU6l6Kda3)uGD6%rVg)cmZ7;5GVt|s6S%V~nf}c*VneUF!G8Y}XkfR7 zo)1?Ll4f3^x_u$0s0psj!7$eQx(qv`=2Ps+r||D-GhUdQ2?0s+Aj=g{$M*FUGc$rD zFKnXd?-zKfc2hLs%&1|HE!s^{rnz35=&j`>xEmdWUso5i-w_jWdte1$sv(JI{NiY* z=@ss4$6@@o`Y|4{41j5}moZs;G@Fq&k{doP8nRpvc3KR=+cx82gvWE7^I|Qgzt4cv z$<@53SsHkRbO|n*B^0{mQ^cLepqH})E`~`kIoW1z*F+uGI@Fijy!98XtvCYG!gC|v z$Jl`2AGl%ecsRFiKZIW%OcO zBuMx=ldgC!PK-~4Nws<4KVmH#E|0Lgv=TjIzH+zjE72rWyKuap1+)kCYeU@Uc7De!X*XQ$t`p?73Pz&7n`T~mEW%%V?iFEfUaD|_{L1yPH z<~&uAru|Gom8-9DOiT*r+-D8{p1+3e&i-`LK$8y6)xeCEn(Wy*#89Mk3iZw!!j`V-!@qye(-z6kI7^hxi8c$3!V#nB z)AeB5{PZSw*x|K^+@#6(s2z3eFTqTW5!j=BrMl3~QJ4qS;hc%WS$x@peOPS6&Tvn7 zl?^elvelSU9pB*M8Jaleurov`OeB*oNv2ik&Ocgb&kn1_qwPW?*k&~ZOpG_--TJ36 zGjK3-T^~vx+g`!#mjCR(T3zANw(sMApLM5WN3P>%f%A4&%La=wMbL~=VDSoDP=3$= zYCjdhg4SGsZQ2IxSY3mVQ!oGy_|Je(ru*@dx)KFgpQ4M8qai3i2c3havE$1w!UOlc ze7e*e+IeFpN;dD{)e<+cnePVB#QH#bEA)ImeQU2a`D6=b2bD?B?jOJLIxo^)`VorX zdcsrlFj}%Ao-W5nfa9q$uB-VYXSQuRGfpVy)5~u`VB&sw`Kca;T$oG~zDEk#Rqjly z$dQ>FI$@T1E;gMBWUC%(ki~StX_M5ams#Cdt0~XST1z49&jr4|fJ6p24cD&6BOc-Ke#i=%H#L{HcrYE?=TF8{6P|#);y@S>Zp4Bo zCgQvzb#h%!@WMlZw%OhTX$^CxuOLeU?nyi7zxu}4%&?+Y>DHF{6L*y}_Q6A^5Jp^_0;`j}<>L~9KNU}E@_`DB&5b#29 zsH>)se8K=yZkkLgC)`;_%PM;EuLPP;l%bo6TF`Z5&$0u6sWL znXw7X)nq(=sk#m6<6JR3Y6eyb&x|NvpKkyD44s>2lfA1xsRlcNP0$dmoiK=f&M{_( zyG=N)Qz!AO!)EGf66Rk6%*a;zKhB^s3);^`p?l_aT%SLNyOS9yo*ml@4^|Ciigx}i zYE>PK9WPG@?!?y2Ua!qO9jn<34Lh2mI*76Cbau`vfU@0XX{z7^`s5YjOvf;xUJa*S z&H^hQSL3eAN~WeWn&hSn`HV)t#G@y))@-dBCQelC;L2yqFk3mmlmn6QQ)elc6p~D? zA3lN3x|#T=K8-&z%^TJavSv@NoZu32t1!ml0)3jdmL3?6!ZSkwf}-0+jrQ-j$q$;~ zxpOyH)|-YmOiHNz{7Fu!_%)n;QbExd$FrLDD9XlJK5`x_lArk zi>)y=iSfP|s&E_P&aKB(vs3)zQZJ@dzJf*?8nA*=DOTu{PH&V(FipQhe8{T;Y7g24 z--jxb{t{iHXO~crvLQTUJpJ|>giE$1vBgE}z-!fhyerJq<>X>W@9Z^jzbge5uk`8L zK?apd!|17SXUUs9oof|!;t8(_l=)SGZ}B@#6Z31i*ojx5v3CIbGO!)j-m|6Kce811 z(0cl_KL9jTrsH&NL(0CULJ>{M%%LI*Myaev&BiWq>S|Y-d;1eN!@3077H2p;R+_$s z2tCM!1!7~n8LY}DpY`?BL4ca@Id9_e_2?I%uF}DmMa&_;Gbwn(aR`2V_}TvUp*@_{ zrqwX~vQmf8zm zRV>>*B1<&$I}b|LvGk_32%l>?h@I4*;@#&rq0J>8N6qqQ|C%Z}6X#rbvoM_tk+K!} zpKr%3fjzF9_z0%!FJfv1ne6E&H~yOhVDB|QI91%rKd6!+y`0Zj+cA=DdJ%?ORMU9b z$VA$}r$P6?B<}5-Eu>w4jz-ri(lQ-43<~JNqy59!LzT^ZL4N>SC=(%GnU=uLZdn39 zeLSFMOed%6c@r0I(PMKOfgYGAu|E@zvqF=2^gZ^IQ^*kV2u9|kc8vv_8$K49r7^7R zyMZ=S!su)BAetaOopJ?V!AJ8ccW1~^+-^Ff=G=Hm@(8I1FFi}r7+r~v-{_N*b}8;V zQ_cz&2_F3L)f6~A-o7+@6aibrmsK|7LdC^2bCed@|1rkweUqpGH`53!8`No3#D%Zc z*w25<=*2pH*lf9vL@l;xeqI%80^=y{~`?&v4J3r+7;v2mX{h zGyB0oNctD?-KKHSvO$7gEV6-#Mmn_3qX+mATlr%-OHi#m5L%K=ImPf)bei*?mpNDf zaf|med-I2!(z*~XJIWos=RaZ_RtK;vK7kOn?J1f(i=>)+cI0%o53TDC(e{>VDx4?F z`ebdG()|w@Xypg;%64?8>JT|fzZd2Js$?tV0Bo8f&+ST0M`y_wqQr((ERQ#)f>vL= z8wa#=Q#yS05`6KaXW-q=(RlIVT(Io55$zr(bi6CZz|7_DY-n>7ceV5djvd&{&sR`_ z52dDT`LQOrE^rqX3VK{qlLTlN>C%l9BP!Un3c7WrQ6hE${`&;r@8A!YB_4rp?nQ74 zHNXwiZ=MB{jXZr@7$HyDVg)?(ywVqPKz$`yGKc#bWTDEzCFFwjrw5E7Kj* z;VdmRhi^&`VaHbmqfG9af3i_QWdOkE0&FJI&GVgAj4QJu+Gi_Fuc7n#e z`wY&Dy;(-aQ@C_Qkrp11sXcqS6K`MB!MqvmsFo9inTfKjTT4jIDn3aioxAygG9RGi z!)kC^W+r^k5~!Uf$-1HxnR|&Q|8K@l!E;b#+7E45;oE9ZRrUr`pJD9$gS*_Fo)qr& z*dnqVcAwuTaSRIvnzGoe5B!IyI6C{)fSt8C&JDdM%^I(TLb{AP>Ydggz2ow@a(gbZaxOl;IDuA}DzL;o%gJuvBXIC&#Gk>Z@$S^)7`3$kjO>@feW#_YYTph1)wWoa ze=>w0vn!W((!%_6P&>DX%gT0V@`IY7Xsau}_Rhy^u{k&? zem#JspPoI&?V8f`jyZ=0)!Dgi@ICUkZQUO^%ID*F@P@dHlTZ<@EPVA&hh> zBM|QJeG4Xw{OTk?|D`f5IHpTj>P`Ey*cHV-w#6X!KA( z2^4v30^OgiCS;#Rk?9a$^3m`o^C#bM*_8vF*RXy}u880{;dgXsY=HuS*;TzMjipXL zFzZtg{XDW?V8=X}vO^|Dt?TE`Lvd{YGo{XyAM}azX|+JD+BeRx2suwu%NkaMhGn*EHeiOeb3R zbrdYGUBhZRcR=cFM!P-6(ht9(;@KT4qVwjy74#-(4H>P901cgBenF5i z({S}8nWvF#N%M5JN@Xnbwut1Nl zZ84n-sw0Xx_tA4{#DL`j({mc92d>~2A8IF3pN9vkGVx^`p{&#$Xbq8NB_Xo#Zm0%o{cEZ$*tk`}g2$Ej8mS&%cMH))dSRXYW)(L$CAQ_$eC zo4_|8ye=3Xkd(|RJruK@N*cyGIJS%mvqZ$W`Y zBMz#YhtZdo<9;z?PL2JrV|yH(-w;Wy+8UgddoBL5)C0rxsX}&lDTy2n*^UPiY?{_e z=J{4En#IklHVSFv-L|acX8T6r7bAJ{-4jhq_aDHazusctks18(=TfX`=MpZ87J@@< zC+}Js0>%bYnZXR<|MNYDUlvPZYD@)J`uQ_1+-SnKN87UN&U?inBUG8`T5a4vs*VdW zvEzH6uBM=@(eUX?6IJ&MjOUIQ*wJJLlE3-|mevtYkMlwAXrTsPsmk0X^`Nq9B?WOw zTwqcsD(zm+$G*)YyT@11+P4&5x5z<$36JKI6X}D>5GFr&FXUdHOpXe!l#;v{COuTZ zVzXP&Xfq!i)~mBI)_J(*vKj>rPiKys6wqEJh}sGySvs1VaRaROCHO=E91SK&qP+k%5#gVLe1s3S+0?zl@ZX<Gz3jwUEmc@Lru9uSzqK&j)_{{#Vdia~bEJFlXas52NR>Fz%N96`UPb z#=bjo>~x15ELs`~(XUdW<&z8hG5j*Fq+FcSqb4vyLZal;bh61iM-Lv~0RHhHN*6h@ z9kpJR*i;0=U)G}c#{}4{D)2_n#?a{h>Tu$ffozlRMf{T0U(>kyqd3H12I~DC2Degd z!71lF#41};O6+NvDu0a4QIue!wv%L^rHPiyZa{I?W&V5iC}`@CU{gc$sBr2#&~A09 z**#*rz@>Pwswp?2Ap9VIWMvH%4>}?`H>L`%x4ncB76J!7>moNv+=%O}Eii7RI{WAM z563@uVFrTApXS|xPhGCUKHC+nqxL5sC4CO7mm1Ou2Z8D6nF`v2)aX;vDH?J{ zg)Bap()e2nz*eKW1FYwHWR{BRRg zHB4qBKKXLHP7Gr^jCSDBK2;$bQi>CQCY?#7o|kX9bb1 ze8==<(;$`8()k*g%R{tpsfujK+r!XCav%!DK)%iTB^eHKlJT z-&&e(l$-LR%SC7>aRCRFo1ua3Q|wK?Kr5FFqXh#u^6Be7iy~L9g3~{=$#TL)e0$mq zTg+c_E86oZXtc0XqN)!C`3FN*EucT&^x)c|`OtJ~mw4TuF*SCF=Zkz6)L@aBEvyk{ zStlRPplP2cVZPlKQh!T_H z@NStcldc#|6Q0k(l$1rReP1vgwsE5k5w@gw-iU4WSV-@el=B-uT2N*AL0HVbL;2fy z-s#ps%F>Af+m*M(R*s4AdG1nvOS2v`8vPO;$mWA)k00zYnt(;qG^oU)5HqAc^KZ`k0Pqr?drT%{rY@^{28~1E!PLxjYE4jHauPN3n;? zHbH_o6$Y2h0R3JjF7XT(@~bwIrE zqQ&oYj6b5zD#j$z&fZwqF-yo_coIW5)aKBu)pvN^e!!2~)wqQVhjS(4sKv98^9=I^ zPGArvcRYup!lV2;r&D~h{ske|UWMH~n!s;=uLV-S{@^MrE7mz+KimCChwW4B0oBfR zY;(>)ni_Tka)-^v-f^QTr)x1TsTe>rhR&fa+U9W2U?2J^{Nm({c0ua45SlwAg-)A% z#(@tdvFAn+njR7Sy(NjbCR&3HY3s)G*Hj_IUZ1V8-;Rxu!}-51iC80K%LvRm=xNWU zEWeSM-8Y!^k2+iPq&1H3y!jTMWn^HAuZT8H+>6g0w$p#^HG*GlNbzCqR3XftJZ#ly z{!t~e3g1gUz5{A>H0F@A%0&LnSaV7+h=S(IBKBxx3?90@l1inOU{GfjZ#nrJJnD8q zXPuoOZzfC1pT0qg>M*7{-irNEP^EL_WtTb7V2FAt0jYsm#+lL4DZKuR9{HLJlWi2{> zb2{r*9*ZM8Z{yf$O>}SKVL0+Ft)_m;82o%%$kSh`PE$n3VPWi8%-_^Vo;>ieuUByj zS!0=6^CV_fA5FUDGPGv66_$G662I4;$(1?u!8KQoE-FOvdMQytE{7ODdCQ85ywzBn zgbQ_Xw)87Up5h9xQqNR1*1o2K#uT4|gjb94hIBF_sXJ8I#(gTU?GB5)nF z0OYxM{P=N|{G@+dxPUx07EG3+j~6ruc|{6+zl(m}JB^>`oW#XRnOHp}1iO{zh|M1t z;>Tub%GP~>DG%dl@66v=^+=m)?WVG*;h%Z`L_vxG9-f1Zq;4jIJD=WgGIaV}( z!yc+jsX#41XX?_*!hmVp@T+r0x3J7KP& zkRzV;57+DLryuj?Q?1~zuPE)nCz)d?cjg$f6+pZiJ2BTNw~Br@hmozyAU?!+FFX_6 z%_pI8_~d3c7P<@GRz|h`JEJlX+q&UH>x+1B_N$tDl6A6S8@>^doL7gY+OKpo!a2ZffY2fM}@t(MSlmn!z7e1!)NP($_u#4va%`}!FYEsjf+tsW;l17#$a|9r zfg?*{u-ZYiK71HAEn3Wv;$^Aj^G(c8E+XfW;k0buVJ=sB32rpefvgaOL3+=)Gheqt z`cgd_w>=%^*nZ)jRE}oVlgGgPM_}(Ub0c3B6oA8YDo zag$jI9-X4Yu7_#CpV_wTkb*X@Njbx|l&pmj|0T1BUE1^`Vl=b3ZNTmgF2=#H@A0v9 zl_X=-Nj{@E{>R0UX9l|a* zAH+HvQ!uPCX7AUQ;HW#s^k$9{XkrR%JuhSo2=&XXANMhUJIH#4brv&K7BK%8u{3MH zn7&-f#SuaV#Rsj4Z2gKv(v(O?ePM5b%>-9^`ptrc3R!u!5l{JBV^>42_BB|@NwLs| zh3xE69W0epXH{cE*hqUfcrsMTI2V|1zgy#Zn_Z*mvyKL3t}vzth5yiKY6EuJ4Psv- zX3?;LCPHq^a;#V*-0L*w!j(S*Nn|d@%)tYBgL+w}{r899N$D`YVmdRwcphF%ZNc2# zLeDkQi1QxdN|X0~0|U`D2Xe7k}%pEevGQ*SF&GIEop_c5giS< zin=ko(Jb{H_vxpeke88)2RiTJ-JZuV==pFms}QnAABT`}^G{Chsu;D5&R})RKpNqE z0xSM)By$NBYLyoly8X6%u+d6*h!*t@l;F;cYkIyHEwtVy9K`SeV`sqf;>1?xDT5DhC}%9x%8`HGqF{d0H)pq zj2VhUmYpH1A?w)0{YHGsi5qxb{w1Fm;lw769E+#sgi?l-6C2`?gWm*4Y_XF(@Ah9M zY(A*0Z&vv-Gsr`^0n#HtmehTi797M{>&rzIe#zoFHxcPjxY#ZGKsR@xUk)d8|mG+0#RK} z3D(4PV(N;c5c#YamW4^ue;by-BW@bI!d*e7VK(&X-WQPiyMbn;?832oro()R1>8eT zYf{=6gUZUE@b9Sv@JsWv-@C$}il-{FZ8aiPY_)~QJL)*CZzHL_(3s;=wLmcK6~X{`+7V&S~j6{A110`q6O^uqcC8w|v1o{b;V(!kq*Y85Ec7 zp>2AStg~}9gn9hqRhCCWa@S){=iyMMX`ozdpC3=pPRxMD+1gCy(^%&2p-dSqJMo`L z$iS<5A?kj6fJOvnQtJsT8fX3=j1(Q>Gdv<;P53-!u}qSgJaeZM-8t0i6$K|g?nVF9 z(==7nm)~?@8lLJYNB!}dEV|KyO$jr@BX>8TlVu6M%vjIvED)T&_GHnQSZgNPwG{u_ zAIH;wKH=FbF{~nX4l53sKvQIM$v46kmt?-?$DfvGi)Cu~ePe1tLDfWT*S8lX{Dqof zya5(PrEmc)8q|5nlFFkbMWa`SL2s@C^ZPu7ZOlxjufm?7%kQRuhGP;YDw?BHvjUm9 z=b%Hk2WVBflSsZ3uhg{S$1Nhx^v7L_zJ8G+!jj;TVmVwo?a8b<&T^%abGV7V^>9A& z6(5@S4J0~-!@SRquqNmiEZ#H+eT4hmBq1{~Qbz-FRYtOsjptCyUj`G6I>l#v$AkQ2 zqA`{V_AeKS_?FHAbh_e0sL)?ZuLoCXut*rQ8D-)c$Ez=u_~rog9rd32^Z zmOeiY6;(w@(ttZUA+ z?hRK@aNCvZdFQm_G;_%aI&O9mEt`X>;ly5ikh_%|`7Ibv%3K!@H3u4UFoedPzKo65 zIW%!jH1!Gd10%gCDx7@<2bZs=_02DFgViC@!bHm1t4Z0`)hI7yW{i}7j&(P4s6=xQ z9UpN5XVi70xymlGc~{TBxN#mYN=s3?+*;}xqfSmSTew`in?lCP629WpcV0SR5;gB# zK;Kv=?$RrR73)kX;;{+bKQg#>n4uI)S))W({tF?|#j%`2pad4&Jcu8!?jSSM#rXVO zH7`HkpkNmeg_A+JtfV}+fX-ZIjwl8OtF`SGqb&0 zC@yU@_WO^duxZEf-=v{rwQD}g6+A$Gd#(6d=qSuAH=*mgr@4ZT40zdh7GCsU76rOy zg5wHbzI%iO+P|8Eez=ErA62Juim6m6^t7+aDA1)-L+Q849E$z(9zQmG0CrVs!F*qp3$!YQ-0iOAp`nCIBEYFf_VM|K3H_WGL$0zjLGpMHHi*2){^TD%aAS}7 z{+Z!4(NEyU2FBy8e}_bG5|VJ6x(+iR9WJhWyB3m%BvpNsdBO#)4JYSo8Q3$@9NTs{ z(mksbF6_f5>`yoZD>5RPd(~^i`Xu&O!H)`mjH6-avtZEq(P%GZT1~Qv=CZv7j!@$; zCeHi7t!cO8Umiug<7r0AN3KNOt@_MjwhIdztVc?=V~80OZOZh3pd;$=&@YL?uLj_C zUrBa)e?3}v3TK$fO1x%v1{CxTq0@_7yk4?8L>7#K*EJ{cy!&93U6P7v)mijthdQ0g z-$p)vdeGTQmM==W#$Pe(!_dt@`iCFzpNA~Pk%B9?s|4}Z_6SnB0rWn8Dt}wMl~{cw zss~@io!5Ti8A)9llX{h3tQv!hw&~OOE;owJsppF(`(d1QJ%(y0@K%q6vw3tm<@y>@ z;XYr=;3rb{eHUm^vLd(RO;9O{#inOvIAua9eY49&yIVWS#yk(teC@>wt)W<8d{Gn_ zk%_l`$HD-eTKq9dj?TTYpb6XnbQ^e(%67+-RJ9}(Pb%g11YX6lbAamGVyU=iE6Rl4 z!QI31aOlv-c)))n^ZoD~Pd;u&&$y0iABBGI!;#zgfv&@&h>bMf%b!kJm*c#TZTN1H zI!2CmC*OmH?8c4f*fHb+X8hpMGH)4)<1=8(`565B@)hjxUrXPAdy%U{3%2<CAT?0cM^Y}sI-eIuCczD$JR^*d%0Z)89CsMb)h@Ii;tgOp~wK|2d z;_U0<)P+Zojz8q@hdbk~KY6sxbqCH^;s)}Ribo$_rP}@Ou)sPKwU;~*7rgt$m*gHs z!zc07g!3_Hiz;;m%;N422u0OXBSF5+8%EisqEFm<{$<>G+%a$riFTRL%TQM~V)tIW z)1yk!4n=g!dOlb2$Q%r2Zr~<4ynvg!-$lzjH0gZh8j$tuf({`+a7%$be%5jlZ_x2( z@nL^?wf|m%+NCMvZSP2>bOOr`yhMpydD_2wDU3*$1;77lAogT8OjzSVx=uRG+fRu% zT<*oD8TpGEDtFM`Kz$sh@SLuix#IAk9Jcq^07|gGL~q*cpwg=hr*(Rv)`S71Ib$+} zw>ApZwT3|oI}BU!C$4vO^-H2zk`~^w`^jIucvbYxJc!$7F^pa|5sY@@UeQ5%b1%t-!s_A<cLJGda*5BGnKh+ zV-F!5IwBEvl?N{8D3$b1bs@>V{y5M zA+d9;u%uj4`|$He zc^2$)59jupG9P&}{P*1(Jlpp``;8Fj@eqjB5#cn%_B+Jrj)j?j2BKx(eVAe{&$Q=p zRNf_pde$HC#Zg1L*lokyJ8657RTU z5TOJ&Zn=Z?fkw>5$B3Tf8G`<*aS$eC4V4LQN(LKCkN>sfwbBIq;-Amg%v?+3e%|EE zyFQ5{<$AGsQXW;v+!yukm8EQ^qsyH_t}Pe! zoRFj?xh1$=_7kpi*hFvM>SMR$UfjIW374iyl3lqi(^xPI?jF*o^!r1}LL(Fh22H0Q zF=jmXcL1AnZlA!yy#$@m9e8zjG(T~!Gxgptrt8m7LVz2A=jDO)Jz^hlLYBt;U$dD^ z$|x3hEr4%Z<-o_M4#I0%K6q~75qKGWpW8ZV8MO-#yUlAaxc#UJ+olX++fM1Tgbzyi zd4LuDp0Ec_WsL`^W_SCd^lX@vKbD_*P?y1HLwbZRB>T>k`m6Rp%1jBW^Nr?z`(5Jo z&TR&#*ib4L`fY9A?(qJUCPmIjMWaquWs7j`@y!^?;6;p?@DAid@>IRBYKUuS=VusuWBzTwgAzK0pTyg87$ z>WrZC%NtS6{v7$;mrsdESyAeczGwGI5&KT4!kDo^f($$Z48 zuW)9@b~>7*&I+o1q1@ve=lw9l-nieLMOV02*G`|w@{%PXrP`2reAtRd8k}%^eiCy$ z;l+B_=+NTtuVHEFSe9I(PQ!KX!=B72ex#L!@S=`KeYp_Ed)>mcgmafn+c#_9#Kl=5Q{Y|t&h%7g0oblo2~EbMgNl@SjU+-#tzIfkY+ z9H&}=*_4WpW7q4{Yu72}^9|a0Fz~-18lYAW!H4zO@Ue?P4JQ$K3cbL;X2RU(GMI4b zLXPAfht7Tu^K=Kx?ZupE zi5h!*$Cd4y=Kx<4cCa7(I?-b`jQXdY;qqQRq`~=n>9PM5dNrz2Y-t+A0v^_LAa@xb zsZV8B`h|I;%_6#d@g_b~ETFk(f+*&98eJ{5WUFV+1Bxv7F%Kk_qd@v8+4egF~;^7LrcIBwGDERqV; zr#Tnp*^P-o0{fMK5z@)*?%@nDwOYs~T3w_5MfteomJt*36%cR+ z6%{HO3W-Xou&?7Np)@19bKhy6$A<`M_^#gP`QG*1&v$=oz3V;qI@jKN?R_2l+~>K6 z>$;BP|NG-PuPl1o>k4`kd~i>n1bzFooY)~L$o}~p&uPd(op+lk{`FTbZ~bNQoYI3h zd50vlx5(1%Q#-{s5^aTJvN-Y$|A-0AbEvNEv)E1j0u*0T&Ve_u zD5C%>ZoI_qtuE*=T#ckYgkV^m5^I~=En2u$7iD(sV7X}`7N2nzuO?}-=mHVBM9<-# zomQYt^$X}h;}?PBnklgB09LRPCil7=-!~bupTnsIv>Lhm$jm~u}yGC#+SV1 z3Lw`zn3^W)!2~pfK9}(npJEBMpD$wdh--LAbt=IYUsOE%4u1~LLOa=HJTyjwq+Z0* z=HQ+1r0QLlQ0jwW)g0?R zwFR%-@}Vo=8$q_Yl2>38ur%))zqI5Y|MG_ywY;4}a+g2j&&C*96XVTY%YBbIg`EPQ zPX&{$MI<@0N3^?o3!K1}_}%9Yr1{&ky<=q1!})^k1KTrj z%d;0h{}@POI8F2I^SDjbNnjV)$+br>rojf+z;?a`Cw@GFF6WKsY9scrJp+%6{4Sc} zKZMVD?>9gCaXgG);y8qq8VVrZaR-|A+sI0m45jC>T5Ja1gp7mkBx!4j`4vaNsImop zT^6wVeRn}2a3?kFauoSiZr~oB7(CCLWKbPiZ z3QT4?7+BuInVn{^d3gY7|E|b=Jp2Y}&7ZJL$hlv4m!cJa%83>_DKnWqPs}S4E-5G6hH6{w3%*))SJE7t3I9_w;8d8bMyG< zVsmOPishUXGT=y8Cj==-k$Rmvt(oP@R2NH<*ii!)jTChH;v(qV+RVRd902aKO8ePrbg!Fy%oAe1^j~ zBViIYWK1BxtD8h?J?GQh77Iu-yN!-Eb6M(!vj|(t`PhQ7xX%757o6y5~w*|%h< z@Mn$Ka=AaPQPGCzQ}P1y{TS?1_25rLt%ngkHZgxdveTDV&17>CwTZ@syM)_|pU~;V1Xgf%|D9v_5?gHfkDEP4SAE#@IWf^lMo*{BjnScl9E5H)F|Z zC2|$=t!GbeAxCR}TKwB_aO!VCud`lNYQZ-4Ln0Y7>MjV{=;>I}wt-d)Y;SK*DSEoz zgd9^>l1-BfS(|0UF-v=BZyQJc&-9p%{9A$JQP2Mwwie%P8Oc<*M)(wQ6qYYvh7})W zNyxxM?7}fz{77|NcqxM9dx!Cxm4uvfb02hk84E9uuE$1kIC(rVCrPNGp5V=##iV_Z z=kyTQG$*01x-J#9rQ(x^Tlo~jKhQ2_0ym6bNk1Z0(NpyTj#GXFW4g}bh)tu&e$r=v zsSiXBv4)`E%CP3hHPi_|ii1zcvCqw^Ons3gW*E)n{cME)s~s(97#Km?p_BO$@rg{U z{~e}q#DNt?dDDpxM%4AxOwf2Kz^2Z-BIQJZ)B3CdzMjnFmrc8Xx}ldi_wwtco1sgq zYZhQYekhIH5<$DfCZOT?5z{M$^XprvVrE`~?dOrpxWYzbn5-}C7=YGfJd9%?%qah2h4PN?i13ZMv@?M;)ttFzuxtYl2*7) z*@OCD$VB=BIgS*iC1zlvzF}*1k#>MP}=Kjgd==3qso6z;l+ifnt zV4c7oy>=85HYQU$+8~-LGEcepP$D;%f^WWr+KWTk+tOlw=qxGLa(Nt;49|rf<9nfE zl_V`sauGb-Zex{8gbgpgf?2coBPa@UAX7#~0|!FK(mL_&M^-fIu_dGw3cq`PmUyl3 zCg-;77NXtH;-OlG+fWRp4pXtJ^1%J6`KI2)MYZ0lh7)He= zZ9;w80(j85m3{Br&63NPGtHr)P@eS~9y+hZ$dmoq=g<<8dfWhk3WDE~YbCmmNn}tW z4Mw}2VEIKg{!FO!6^#2M5BN$l=~3zKY)l?<>ba>~SO5@-ZJT9rnPDqlU1w zQ5|BHjj8bRfC7{*=ub;uFX24~&7-J~a~ZF8n^K3}#LyBu*m0`>bwdWwq~WCytK$Rv zKlS4`*7U=VbptB>T~*QBV=JY~nX-d5ZlZ6ML%6)etxy~yc%j`-=kM;;W+R_EkfdZR zBuS+Z^K3%%N9MF?!+5BDY)M0EQmL}+3T~eB1OEt;f^AC=V9Tc@d}ebIrl=Kji@U~i z?KgDUY+Zi}PThq;)rV=k*$Yvi;K9?f!H(iKwcxs8n`qOzrI7K+j8xvX;?R}mY_G8= z7v(>Mem)ya^SyS`Pw2+N;4;3YxejNgPvy3)ID&Rp;^1D31cW>qM6aX|V$g04n%XiL zi$9!)yIFlW_h+3rEasg^(_PStTc%@(-C!!q{)j#MOW}-2k9V(i0M+;wyn zoG+5*3X089ThbL`Y@$T@$u+jm7CF+T+k(f6tsGsv=gxC`8~C29MSR!Ad30{UBd#U4 zlzJNH;K^aSaNwIaJn8311)h>@+M>H)cH$T*J&qxj`PEnZ37!>7w$r>oMDG$vT!A1$yZ?`=|aPUS8h6xc8!o6qt7f$7-vQlIT`%ECmS z!JKL4K&JY75Jg<*!f$Z_B!BlIcs;oYj%*qI;xJu}Z$RG)2eLH~v@NC+>1(+GUhUL_ zu>spCRoGwSOXM(AJrWgSq?vrC3RCG`WgAZZ(d7n^%_pF; zoGZk6YSY*>ZQN3?!U9{Y;9bQR;5HS*(ZmGNEFHkDNmY71TRB;ngeK`NXUXT(*`nJM*2u>&JDrH!m7zo*0P< zYk%X1Wly5A-40Z1w26+B7dLwHL0r|Z7PkLk4@rx!z;1(g+^}$WQt*mpo?qHfY2#kab*B%` z)1W=+7Cx90$Da<1!X;btX-`=WeAP7~13{ChbvA(;1>OA6=$Uw{br2TzsXMpCFcy1DRgi`13patimKsh}Dmn%xPcs|!r(8nZ!yYC?OvFkm2bCqFQD{Z;F zXjPUSp8|d-{(zRRZmizmE#yWRGHdA~7CL7t`E?$IF6TNv+e-tVPc`Qnof3FIt&!9s zc19~nS#;D%hwjiL;HUeJTSK2v+clLfD_F?qlnUot27Ltim5F?m-Y${isXJ&;R7c)h z6W9Rv9LO}8gO?39kf(5*7n~cyt;lbJVO9rm`?y)qtGgf6p32e3>LZ+?tQxENP4Leg zeHMiT2HyMg46V7p2*PNP1qV%p@YCjmQfMDTZUqFO)F z&-5felP30P`C)QM3jhQ0UTRI<4kPEpgWBF{?0}ew5AN>*t6obkrBej=tbcMha2|CO z3tU3e@o>X(H@oBaN?=qFgdb?ZD>06u-IqqPmqum$RkM7oD2bq@UE`?rvkc2RA;wY3 zGMv#LQA}}Z2Y8h^!<3o}^uAo3ZT>ol>ed@juKsBVEGy$;W#{wRl9IGPJ)O7mJxkRm zzv0hJfqj1b0m_D3vC9sfILi7xbfo057mxzS<-XFe!7-?L=LrTjCUe;XPh)@SR}`8P z##WyfY6+c>z=uDTDb%JIhMU&#Dl?NQEXx9Q%^jKF#CT4J1TEx^)BFtU9BgpUMC*fY zywToxuzW8CgT0@_%wh}b=W++vJ=utn%N2!nSwTbAcVSy?2b!!;GcmlHN71*(^4qdyN z$X!k!&BtEntLGTf?9=bK+&jghhGiD4M8$%>b$gJNMJ?X&(qKbuhqL~|cWuq#M}k-E zHeTzh3;ND`4POs+<9e@nu0j+AO}V1VRP(tI>UtYIzRaN7d&=B-p)P*0nHAo=^#exa zb_mSemALWYRaz5WDe_9O2k+mcnZZfH15wcdOkoPl$eqIGpSGko^^#!LUC&kKCSma2 z>EOI&CS^{DL@T2U0=r;3H(dDdqbr~BK*$x$Zy&*uCrLBwFrlV;`c6tr7){A{{iyum zF32>pr&yn1OyR2nvzt)FIX&6KDPJhZf4rK`t=H?OlGlpDG58R=`E4@0Te6MjZ61ZE z2AAWL$uVdsc-48on@SDunxVMt2W<7XCH0mPT)0k>{5`DcMbUFi=Zksm#xyQgehtk} z>V7_b=+8jKBc?Ty042!()=JZK?Rf)_DeKZ zO@&u+r&-csEgC9sNcwA6(793mH2wN_T(sdmmvB}MAJRVdSbGxn)1HJ1$6TS^iLo;> zgJ`0+EBo_mHtKlhif4ZQlr{K{0fk3|Iaq_f&xCs5!*YCY?@;QMSVa2^D=dZL^x}W;6;EOUYQ+B0^H4E6b4TkK!pg9{bzzL$j0-f~(=zE(D-ccJy6FUC^ znl3U-W!)_8N3H><11lm?@M0LYtebn9ForU#~e9nRukSQN^fr9;{xu{o8zb1 z)!ZViQ;S1Qa}BUrS%5b>_JI78<8WZAkhh-UOg=mNQ_Smgczc`~EzeP)q8aBv+3YYz z_gv!_2Tg#dSNgDbMKHSDnF@>itKix7FmX$$JxJ=j#;z0#${Y0>ll+Ce@@%23CgUJ} z?zt>_a;})Q_Kl&Jw^Lcw1!a1(+>Tz%yNCDkr<0V9kQ+R%1Nl0&I9`;AjUOaf{jPCU z7slq&WXbtVRvRn5b|P0O;g~+8V2s)dZ07F1AVPYJqK77AKr;7UA$u~Nqw9w(&p?D}Ysao8bN zteZz6HsAR*_suEWsZ!uaI?>(hC0y-@RC-dZj73jhpjPG+Tvj~|+XOFe(_wvRQTr4+ zlF~)9-(KdTO77GCICFA!_ad#RV2G>`e8zshvyCkK0P)-9$Vjw|H8usYvd#K5%SBFp zzUr7MlMhKq{OYq@q@*Pz$Ow!u?-e>(>K+X@4_szXoj-zHIq_2HFnKG-QR1fA!&i z31TBD@po^3VRU5h%7~!gi2scA@5`$3mq`2nKGJ{9qsf1U`S-Qa{7aZw*?&8re|;kU zGtR%yp5b5OH2r;?zhBo`F0!(}{cCd_sR{obNakO+{_l0P@E(7)bw~brEo>+E>sEWt YKkxr~c$SOIuW6Q$5iY-;|Ig?CA4?LY@Bjb+ literal 0 HcmV?d00001 From cd6d44cdd287d2a2e3da3076c422f14dd9c2852e Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Tue, 30 Apr 2024 14:39:39 +0200 Subject: [PATCH 064/379] using SumPool2d --- .../using_SumPool2d/Res-SCNN3.ipynb | 1509 +++++++++++++++++ 1 file changed, 1509 insertions(+) create mode 100644 tests/test_nonsequential/using_SumPool2d/Res-SCNN3.ipynb diff --git a/tests/test_nonsequential/using_SumPool2d/Res-SCNN3.ipynb b/tests/test_nonsequential/using_SumPool2d/Res-SCNN3.ipynb new file mode 100644 index 00000000..2e507d45 --- /dev/null +++ b/tests/test_nonsequential/using_SumPool2d/Res-SCNN3.ipynb @@ -0,0 +1,1509 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import torch, random\n", + "import torch.nn as nn\n", + "import sinabs.layers as sl\n", + "from tqdm.notebook import tqdm\n", + "\n", + "from tonic.datasets.dvsgesture import DVSGesture\n", + "from tonic.transforms import ToFrame\n", + "from torch.utils.data import DataLoader\n", + "from torch.nn import CrossEntropyLoss\n", + "from torch.optim import Adam\n", + "\n", + "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "torch.backends.cudnn.deterministic = True\n", + "random.seed(1)\n", + "torch.manual_seed(1)\n", + "torch.cuda.manual_seed(1)\n", + "np.random.seed(1)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Loading Data" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "batch_size = 3\n", + "num_workers = 1\n", + "epochs = 30\n", + "lr = 1e-4" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "root_dir = \"../DVSGESTURE\"\n", + "_ = DVSGesture(save_to=root_dir, train=True)\n", + "_ = DVSGesture(save_to=root_dir, train=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "n_time_steps = 50\n", + "to_raster = ToFrame(sensor_size=DVSGesture.sensor_size, n_time_bins=n_time_steps)\n", + "\n", + "snn_train_dataset = DVSGesture(save_to=root_dir, train=True, transform=to_raster)\n", + "snn_test_dataset = DVSGesture(save_to=root_dir, train=False, transform=to_raster)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The transformed array is in shape [Time-Step, Channel, Height, Width] --> (50, 2, 128, 128)\n" + ] + } + ], + "source": [ + "sample_data, label = snn_train_dataset[0]\n", + "print(f\"The transformed array is in shape [Time-Step, Channel, Height, Width] --> {sample_data.shape}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "snn_train_dataloader = DataLoader(snn_train_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True)\n", + "snn_test_dataloader = DataLoader(snn_test_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=False)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Network Module\n", + "\n", + "We need to define a `nn.Module` implementing the network we want the chip to reproduce." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "device: NVIDIA RTX A4000\n" + ] + } + ], + "source": [ + "if torch.cuda.is_available():\n", + " device = torch.device('cuda:0')\n", + " print('device: ', torch.cuda.get_device_name(0))\n", + "else:\n", + " device = torch.device('cpu')" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "class SNN(nn.Module):\n", + " def __init__(self) -> None:\n", + " super().__init__()\n", + "\n", + " self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False)\n", + " self.iaf1 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + " self.pool1 = sl.SumPool2d(2,2)\n", + " self.pool1a = sl.SumPool2d(6,6)\n", + "\n", + " self.conv2 = nn.Conv2d(10, 10, 2, 1, bias=False)\n", + " self.iaf2 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + " self.pool2 = sl.SumPool2d(3,3)\n", + "\n", + " self.conv3 = nn.Conv2d(10, 10, 3, 1, bias=False)\n", + " self.iaf3 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + " self.pool3 = sl.SumPool2d(2,2)\n", + "\n", + " self.flat = nn.Flatten()\n", + "\n", + " self.fc1 = nn.Linear(810, 100, bias=False)\n", + " self.iaf4 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + "\n", + " self.fc2 = nn.Linear(100, 100, bias=False)\n", + " self.iaf5 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + "\n", + " self.fc3 = nn.Linear(100, 100, bias=False)\n", + " self.iaf6 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + "\n", + " self.fc4 = nn.Linear(100, 11, bias=False)\n", + " self.iaf7 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + "\n", + " self.merge_fc = sl.Merge()\n", + " self.merge_conv = sl.Merge()\n", + "\n", + " def detach_neuron_states(self):\n", + " for name, layer in self.named_modules():\n", + " if name != '':\n", + " if isinstance(layer, sl.StatefulLayer):\n", + " for name, buffer in layer.named_buffers():\n", + " buffer.detach_()\n", + "\n", + " def init_weights(self):\n", + " for name, layer in self.named_modules():\n", + " if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear):\n", + " nn.init.xavier_normal_(layer.weight.data)\n", + "\n", + " def forward(self, x):\n", + " \n", + " con1_out = self.conv1(x)\n", + " iaf1_out = self.iaf1(con1_out)\n", + " pool1_out = self.pool1(iaf1_out)\n", + " pool1a_out = self.pool1a(iaf1_out)\n", + "\n", + " conv2_out = self.conv2(pool1_out)\n", + " iaf2_out = self.iaf2(conv2_out)\n", + " pool2_out = self.pool2(iaf2_out)\n", + "\n", + " merged_conv_out = self.merge_conv(pool1a_out, pool2_out)\n", + "\n", + " conv3_out = self.conv3(merged_conv_out)\n", + " iaf3_out = self.iaf3(conv3_out)\n", + " pool3_out = self.pool3(iaf3_out)\n", + "\n", + " flat_out = self.flat(pool3_out)\n", + " \n", + " fc1_out = self.fc1(flat_out)\n", + " iaf4_out = self.iaf4(fc1_out)\n", + "\n", + " fc2_out = self.fc2(iaf4_out)\n", + " iaf5_out = self.iaf5(fc2_out)\n", + "\n", + " fc3_out = self.fc3(iaf5_out)\n", + " iaf6_out = self.iaf6(fc3_out)\n", + "\n", + " merge_fc_out = self.merge_fc(iaf4_out, iaf6_out)\n", + "\n", + " fc4_out = self.fc4(merge_fc_out)\n", + " iaf7_out = self.iaf7(fc4_out)\n", + "\n", + " return iaf7_out" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "snn = SNN().to(device)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "snn.init_weights()" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "optimizer = Adam(snn.parameters(), lr=lr, betas=(0.9, 0.999), eps=1e-8)\n", + "loss_fn = CrossEntropyLoss()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Define train and test" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "def train(dataloader, model, loss_fn, optimizer, epochs, test_func, dataloader_test):\n", + " epochs_y = []\n", + " epochs_x = []\n", + " epochs_acc = []\n", + " model.train()\n", + "\n", + " for e in range(epochs):\n", + " losses = []\n", + " batches = []\n", + " batch_count = 0\n", + " train_p_bar = tqdm(snn_train_dataloader)\n", + "\n", + " for X, y in train_p_bar:\n", + " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", + " X = X.reshape(-1, 2, 128, 128).to(dtype=torch.float, device=device)\n", + " y = y.to(dtype=torch.long, device=device)\n", + "\n", + " # forward\n", + " pred = model(X)\n", + "\n", + " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", + " pred = pred.reshape(batch_size, n_time_steps, -1)\n", + "\n", + " # accumulate all time-steps output for final prediction\n", + " pred = pred.sum(dim = 1)\n", + " loss = loss_fn(pred, y)\n", + "\n", + " # gradient update\n", + " optimizer.zero_grad()\n", + " loss.backward()\n", + " optimizer.step()\n", + "\n", + " # detach the neuron states and activations from current computation graph(necessary)\n", + " model.detach_neuron_states()\n", + "\n", + " train_p_bar.set_description(f\"Epoch {e} - BPTT Training Loss: {round(loss.item(), 4)}\")\n", + "\n", + " batch_count += 1\n", + " losses.append(loss.item())\n", + " batches.append(batch_count)\n", + "\n", + " epochs_y.append(losses)\n", + " epochs_x.append(batches)\n", + "\n", + " acc = test_func(dataloader_test, model)\n", + " print(f'Epoch {e} accuracy: {acc}')\n", + " epochs_acc.append(acc)\n", + "\n", + " return epochs_x, epochs_y, epochs_acc\n" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "def test(dataloader, model):\n", + " correct_predictions = []\n", + " with torch.no_grad():\n", + " test_p_bar = tqdm(dataloader)\n", + " for X, y in test_p_bar:\n", + " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", + " X = X.reshape(-1, 2, 128, 128).to(dtype=torch.float, device=device)\n", + " y = y.to(dtype=torch.long, device=device)\n", + "\n", + " # forward\n", + " output = model(X)\n", + "\n", + " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", + " output = output.reshape(batch_size, n_time_steps, -1)\n", + "\n", + " # accumulate all time-steps output for final prediction\n", + " output = output.sum(dim=1)\n", + "\n", + " # calculate accuracy\n", + " pred = output.argmax(dim=1, keepdim=True)\n", + "\n", + " # compute the total correct predictions\n", + " correct_predictions.append(pred.eq(y.view_as(pred)))\n", + "\n", + " test_p_bar.set_description(f\"Testing Model...\")\n", + " \n", + " correct_predictions = torch.cat(correct_predictions)\n", + " return correct_predictions.sum().item()/(len(correct_predictions))*100" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Training loop (HPO)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "12d134e3b89e41888c9c47892b8e6491", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/359 [00:00" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "y_avg = []\n", + "for y in epochs_y:\n", + " y_avg.append(np.mean(y))\n", + "\n", + "plt.plot(np.arange(len(epochs_x)), y_avg, marker = '.')\n", + "plt.xlabel('epoch')\n", + "plt.ylabel('average loss')\n", + "plt.ylim(0,)\n", + "plt.xticks(np.arange(len(epochs_x)))\n", + "for i, txt in enumerate(y_avg):\n", + " if i%5 == 0:\n", + " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", + " else:\n", + " pass\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAG2CAYAAACZEEfAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABq60lEQVR4nO3dd1hT1+MG8DfsHUW2TAfixK3gqqO46q6jtVWr1bZi62idraPVFrWt7dfROn5urauOOloXrqqooOBEBERBZYhI2DPn9weaGlmJBIH4fp4nT8vNPfecQMx9c+6550iEEAJEREREWkqnohtAREREVJ4YdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirVWjYOXPmDPr06QMHBwdIJBLs27dP6XkhBObMmQN7e3sYGxujW7duCA8PV9onKSkJw4cPh4WFBapVq4YxY8YgLS3tNb4KIiIiqswqNOykp6fD09MTK1asKPL5xYsXY+nSpVi5ciUuXrwIU1NTdO/eHVlZWYp9hg8fjps3b+LYsWM4ePAgzpw5g3Hjxr2ul0BERESVnKSyLAQqkUiwd+9e9O/fH0BBr46DgwO+/PJLfPXVVwAAmUwGW1tbbNiwAcOGDUNoaCgaNGiAwMBAtGzZEgBw+PBh9OrVCw8ePICDg0NFvRwiIiKqJPQqugHFiYqKQlxcHLp166bYJpVK0aZNGwQEBGDYsGEICAhAtWrVFEEHALp16wYdHR1cvHgRAwYMKPLY2dnZyM7OVvwsl8uRlJSEGjVqQCKRlN+LIiIiIo0RQiA1NRUODg7Q0Sn+YlWlDTtxcXEAAFtbW6Xttra2iufi4uJgY2Oj9Lyenh4sLS0V+xTFz88P3377rYZbTERERBUhJiYGjo6OxT5facNOeZo5cyamTJmi+Fkmk8HZ2RkxMTGwsLCowJYRERGRqlJSUuDk5ARzc/MS96u0YcfOzg4AEB8fD3t7e8X2+Ph4NG3aVLFPQkKCUrm8vDwkJSUpyhfF0NAQhoaGhbZbWFgw7BAREVUxpQ1BqbTz7Li5ucHOzg7+/v6KbSkpKbh48SK8vLwAAF5eXkhOTsbly5cV+5w4cQJyuRxt2rR57W0mIiKiyqdCe3bS0tIQERGh+DkqKgohISGwtLSEs7MzJk2ahAULFqBu3bpwc3PD7Nmz4eDgoLhjq379+ujRowfGjh2LlStXIjc3FxMmTMCwYcN4JxYREREBqOCwExQUhM6dOyt+fj6OZuTIkdiwYQOmTZuG9PR0jBs3DsnJyWjfvj0OHz4MIyMjRZmtW7diwoQJ6Nq1K3R0dDBo0CAsXbr0tb8WIiIiqpwqzTw7FSklJQVSqRQymYxjdoiIiKoIVc/flXbMDhEREZEmMOwQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEqQn5+P2bNnw83NDcbGxqhduzbmz58PIYRin1GjRkEikSg9evToUeJxU1NTMWnSJLi4uMDY2Bje3t4IDAxU2ictLQ0TJkyAo6MjjI2N0aBBA6xcubJcXqc206voBhAREVVmixYtwu+//46NGzeiYcOGCAoKwkcffQSpVIovvvhCsV+PHj2wfv16xc+GhoYlHvfjjz/GjRs3sHnzZjg4OGDLli3o1q0bbt26hZo1awIApkyZghMnTmDLli1wdXXF0aNHMX78eDg4OKBv377l84K1EHt2iIiISnD+/Hn069cPvXv3hqurK9599134+Pjg0qVLSvsZGhrCzs5O8ahevXqxx8zMzMTu3buxePFidOzYEXXq1MG8efNQp04d/P7770p1jxw5Em+99RZcXV0xbtw4eHp6FqqbSsawQ0T0hiivyzF+fn5o1aoVzM3NYWNjg/79+yMsLExpn08++QS1a9eGsbExalhZo0O3njgTGFIeL1PjvL294e/vjzt37gAArl69irNnz6Jnz55K+506dQo2NjaoV68ePvvsMzx58qTYY+bl5SE/Px9GRkZK242NjXH27Fmluvfv34+HDx9CCIGTJ0/izp078PHx0eArfAMIEjKZTAAQMpmsoptCRFRuvv/+e1GjRg1x8OBBERUVJXbt2iXMzMzE//73P8U+I0eOFD169BCxsbGKR1JSUonH7d69u1i/fr24ceOGCAkJEb169RLOzs4iLS1Nsc+qVavE6dOnxdJ9Z4XDqF+FcZ3WQtfCWvwRcLfcXq+m5Ofni+nTpwuJRCJ09fSERCIRP/zwg9I+27ZtE3/99Ze4du2a2Lt3r6hfv75o1aqVyMvLK/a4Xl5eolOnTuLhw4ciLy9PbN68Wejo6Ah3d3fFPllZWWLEiBECgNDT0xMGBgZi48aN5fZaqxpVz98MO4Jhh4jeDL179xajR49W2jZw4EAxfPhwxc8jR44U/fr1K1M9CQkJAoA4ffq00vbIhFThOv2gcJl+UNh/tEwAEE6f/J94lJxRpvrK27Zt24Sljb2w7jtV2I9eLqzemSLMLKqJDRs2FFsmMjJSABDHjx8vdp+IiAjRsWNHAUDo6uqKVq1aieHDhwsPDw/FPj/++KNwd3cX+/fvF1evXhXLli0TZmZm4tixYxp9jVWVqudvXsYiInpDlMflmKLIZDIAgKWlpWLbydsJGLIyAAKAPCcLadePQ09qC4l5DdxLzCjbCytnU776ChLPfjCp3wkG1q4wbdgFep7vYP73PxRbplatWrCyskJERESx+9SuXRunT59GWloaYmJicOnSJeTm5qJWrVoACsb1zJo1C0uWLEGfPn3QpEkTTJgwAUOHDsVPP/2k8ddZklhZJs5HJiJWlvla69UU3o1FRPSGmDFjBlJSUuDh4QFdXV3k5+fj+++/x/DhwxX79OjRAwMHDoSbmxsiIyMxa9Ys9OzZEwEBAdDV1S21DrlcjkmTJqFdu3Zo1KgRElKz8N2BWzh4LRapVw7h6an1ELlZ0LN0hM3QBdDV04erlUl5vuwyEUJAlpIOI8lLfQMSHcTLMuEfGo+O7tbQ11V+/sGDB3jy5Ans7e1LrcPU1BSmpqZ4+vQpjhw5gsWLFwMAcnNzkZubCx0d5WPr6upCLper9TpiZZmISkyHm5Up7KXGpe4vhEBadh4S03KwIzAaq87chRCAjgTwG9gYQ1s5q1V/RWPYISJ6Q+zcuRNbt27FH3/8gYYNGyIkJASTJk2Cg4MDRo4cCQAYNmyYYv/GjRujSZMmqF27Nk6dOoWuXbuWWoevry9u3LiBM2f+xbZL0fD7OxQpWXnQkQDjPx4Ji48H45e/ApF8aTcS/1oI51E/IyMnv9xec1lcikqC3z+hkLi0gOz8DuhaWMPAyhk58ZFICdwHsyZvY8zGIEj18mF6cy8+GfkeOnrWxd27dzFt2jTUqVMH3bt3Vxyva9euGDBgACZMmAAAOHLkCIQQqFevHiIiIjB16lR4eHjgo48+AgBYWFigU6dOmDp1KoyNjeHi4oLTp09j06ZNWLJkicqvY0dgNGbuuQ75s7Ay+W13tHa1RGJaDhLTshWPx6nKP2flFg5UcgFM330dJ8Meo4VzdXjYm8PDzgLW5iXfZl/RJEK8MAz/DZWSkgKpVAqZTAYLC4uKbg4RqSA/Px/z5s3Dli1bEBcXBwcHB4waNQrffPMNJBIJAGDevHnYvn07YmJiYGBggBYtWuD7779HmzZtij2uq6sr7t+/X2j7+PHjsWLFCgDA6tWr8ccff+DKlStITU3F06dPUa1atXJ5nZrk5OSEGTNmwNfXV7FtwYIF2LJlC27fvl1sOWtrayxYsACffPJJicefMGEC/vrrL2za8zd+u5yKwHtPAQCNa0rhN7AxGtWUAijoZbj9IAlvN6+D6t0/h+dbvbHPtx3MDCvH9+878alYfPg2jocmAAAM5NnQD9mFsIv+kGfIoGtmiU49+8P73U/w961EJDxNweM9C5CTcBciOx3VrGzR3edt/PrjQtja2iqO6+rqilGjRmHevHkACsLnzJkz8eDBA1haWmLQoEH4/vvvIZVKFWXi4uIwc+ZMHD16FElJSXBxccG4ceMwefJkxfv8ZUIIPHiaidtxqQiMeoLV/0a98u/CUE8H2Xml9yJZmRmivr05POwKwo+HvTnq2JjBUE9X7V4ldah6/q4c7ywiIjWpMtGbu7s7li9fjlq1aiEzMxO//PILfHx8EBERAWtr6yKPGxgYiPz8/3oabty4gbfffhuDBw9WbMvIyECPHj3Qo0cPzJw5s3xfqAZlZGSofUlElcsxQgh8/vnn2LN3L0Z+vx7j9sUgN1/AxEAXU952xyhvV+i9cJnHXmoMSyMrGOhKYGEARCSk4audV/H7B82LPYG/DrGyTPxy7A7+vPwAcgHo6kgwrJUTJnatCxuLgYiVZeJeYgZcrUwUJ+05/eQ4F/kEe1utxZGb8cjMLXjvBAD4dHcEBjTLxDtNHGBpaoCAq6GISkxHrCwT9lJjDBkyBEOGDCmxTXZ2dli/fn2xgSE1Kxd34lMRGpuK23EpuB2bittxqUjLzivxuLYWhnCqbgIrM0NYmRvAyswQ1uaGBT+bGcL62XZZZi7aLTwB+QvdIjoSYEx7NzxMzsTt2FREPUlHYlo2/g3Pxr/hiYr99HQkqGFmgPiUbEW5iroExp4dsGeHqCp65513YGtri7Vr1yq2DRo0CMbGxtiyZUuRZZ7/Wz9+/LhKl2QAYNKkSTh48CDCw8MLnYhPnTqFzp07V5menVGjRuH48eNYtWoVrJ1qw//cRfw0+yt8PGY0Fi1ahLS0NHz77bcYNGgQ7OzsEBkZiWnTpiE1NRXXr19XzAj88uWY8ePHY/OWrajzwbdI1LECALSrWwPzB7dGXYcauHv3Lnbs2AEfHx9YW1vjwYMHWLhwIc6dO4edxwLw2e5I5OTLMbV7Pfh2rvPafy+yjFz8djoCG87dU/Ri9Gxkh6+610NtazOVj5OenYejt+KwN/gRzoY/VgQEPR0J6tqa4XZsKgQKTvrTutfDO54OKh334NVHWHwkDHIBSAB08bCBjo4Et+NSEJNU9IBhfV0J6tiYw8XSBEduxuHFE72uRIKzMzqr3MuyIzAas/bcQL4Q0JVI8MPARkqBJSMnD+HxaQiNTcHtuFTFf2WZuYWOpW7dpWHPDhFpNW9vb6xevRp37tyBu7u74s6i4sYy5OTkYPXq1ZBKpfD09FSpjpycHGzZsgVTpkyp0B4HTVm2bBlmz56NUR9/gieJj6FrZgnTBl3RpN9YAAW9PNeuXcPGjRuRnJwMBwcH+Pj4YP78+UpLH0RGRiIxseAbfHJGjmLG35DfJyv22QGgh3Q96o4aBSMjI/z777/49ddf8fTpU9ja2qJjx444f/486tVzx7d5Rpi55zp+OhqGRjWl6ORedK+bpmXl5mNTwD2sOBmpODG3drPEjJ4eaO5c/OzHxTE11MOAZo4Y0MwRCalZ2B/yCPtCHuLGwxSExqYq9pMLYOHhMCw8HFbC0YomAPjfTlDaZmdhpBg7U//Zf2tZmyoGTRcVVtQJG0NbOaOju3WhXq3nTAz04OlUDZ5O1f5rpxA4dD0WE/4IVto3XwjcS8zQ+OWs0rBnB+zZIaqK5HI5Zs2ahcWLFyvdWfTyZaWDBw9i2LBhyMjIgL29Pfbt24dWrVqpVMfOnTvx/vvvIzo6Gg4Ohb+FV7WeHaDgUs3LlyUkAGb28kBdG3PFZY0apoYw0Ct6dpJYWSaiHqcjPCEVy05EIDEtBwAwvI0zpvXwgNRYX+12zdxzDdsuxUBqrI8DE9rDuUb53KEVK8tEZEI6QmNlWH/uHh7JsgAA9WzNMb1nPXSuZ6PxYLsrKAZT/7xWaLu+jgQ6OiXXJZcL5MoLn6ZHtHVBj8Z28LCzgKWpQaltKOoSXHkr6r3Gnh0iqjDlOYCwvOpW5c4iAOjcuTNCQkKQmJiINWvWYMiQIbh48SJsbGxKrWPt2rXo2bNnkUGnqop6nI6Xz50CwA9/Fx6gLDXWh5WZwbMAVDCOIzY5E0dvxStdFqlrYwa/gY3R0tWy0DFUNa9vQ4TGpiIkJhnjNgdhz3hvmBho9hS1IzAaM/Zcx4tf8R2kRpjiUw8DmtWEbinB41W1r2sFHQkKnfTPTC/9pF9cYPisc221/r3YS41f+79te6kx/AY2LlOvkqawZwfs2aE328u3pb7OAYRlqftV7yyqW7cuRo8eXerA4vv376NWrVrYs2cP+vXrV+Q+VbFnZ8uF+/hm3w2lbRIUXL5JzcpDYlo2nqTnIL+I3oSiSACcnvoWnGuYlrltsbJM9Fl2FolpOejr6YD/DWuqsV6WqzFP0W/FeaVtEgAnv3oLrlZlb3tpShv3Ul5lK4Py7FVizw4RlSpWlqkIG0DBN89Ze26go7t1uX/7KqrumXuuq1z3q9xZBBRc/srOzi71+OvXr4eNjQ169+5d6r5VRWpWLpb6hwMoONELoMiTp1wukJyZWzDfSmo2HqdlIzEtB1djkrH/6iOlYwoAD5OzNBJ27KXGWPF+cwz/v4vYf/URmjhK8XGHWmU6Zr5cYMuF+/D7J7TQcwJArCzrtYSd0sa9lFfZyqAiepVexrBD9Aa7HZtS6JLG6xpAGJVY+HKKXABbL9zHlLfrlTqWoU+fPvj+++/h7OyMhg0bIjg4GEuWLMHo0aMBAOnp6fj+++/Rt29f2NvbIzExEStWrMDDhw+VbiN/+c4ioCAQrV+/HiNHjoSeXuGPybi4OMTFxSmWArh+/TrMzc3h7OystERCaV735cOfj95BQmo2XGuYYMNHrZ+d6AufPHV0JLA0NYClqQHcbc2V2nvw2qNCl1Q0OQNym1o18HXv+vj2wC34/XMbDRws4F3b6pWOFRqbgpl7riMkJrnI5zXd9tKU5aRfGQJDVca1sYgqWH5+PmbPng03NzcYGxujdu3amD9/Pl68wrxnzx74+PigRo0akEgkCAkJUenYu3btgoeHB4yMjNC4cWP8/fffiueS0nOw+Mgd5CbGIGH3d4j+ZQiilwxC7MbJ0MtUby2kV+FWzLfp5Scj0Wf5Wfwb/rjE8suWLcO7776L8ePHo379+vjqq6/wySefYP78+QAKenlu376NQYMGwd3dHX369MGTJ0/w77//omHDhgAKTt63wsIR9SBW6djHjx9HdHS0Iji9bOXKlWjWrBnGji24i6ljx45o1qwZ9u/fr/Lr3xEYjXYLT+D9NRfRbuEJ7AiMVrnsq7j+QIZNAfcAAPP7N4KrlSm8atdQe9yH38DG0H12aam8xmCM8nbFgGY1kS8X+PyPYDxKVm89psycfCz85zb6LDuLkJhkmBvqYX7/Rq+l7VQ5ccwOOGaHKtYPP/yAJUuWFJoc7/vvv1dMjrd582ZERUXBwcEBY8eORXBwMJo2bVricc+fP4+OHTvCz88P77zzDv744w8sWrQIV65cgdShFkauu4Sw8AjEbZ4C8yZvw6R+J0gMTGCW8QgnF4+Dg71dub7uM3ceY8S6S4qfdSTA2w1scT7iCVKfTYjWoa4VpvfwUMy8q0kVOVbp4dMMtF90skxzn6gjXy4w4LdzuPZAhr6eDlj6XrMyHe913NmTmZOPQb+fx63YFDRxlGLnJ14w0i99ba5/wx/j6703EJ1UsLhor8Z2mNunIWwtjF5b2+n1UfX8zbADhh2qWOpMjnfv3j24ubmpFHaGDh2K9PR0HDx4ULGtbdu2cKnbAHc93sfj1Gyk/f0T2tezxe//tx43HsgwZddVpGblYX6/hvjQy1WTL1NJbr4cPf/3LyIS0jCkZcG8JM9PPknpOVh+IgKbL9xDbn7Bx1NfTwd85VNPY7cjRyelo9OPp5TuyinPsPGi7Lx8jFx3CRfuJhV6btvYtvCqXUPjdW4KuIc5f92EuZEe/L/sBBtzI43XUR5ikjLQZ/lZJGfkYkhLRywa1KTYActP0rKx4FAo9gY/BFBwl9V3/RqhWwPbIvcn7aDq+ZuXsYgqmLe3N/z9/XHnzh0AUEyO17NnzzIdNyAgAN26dVPa1rBVB+w/dgqPU7Phbm2KrKggtGjSEKOG9Mfwzk2QsWs6Mu4E4Odjd/A0PadM9ZdkU8B9RCSkwdLUAF/3bqB0OcXS1ABz+jTAiS/fwoBmNSGRAPuvPkLXJacwb/9NPEkrfXDxy4QQiEhIxfpzURizIRA+S87g5a95z8cqlafUrFyM3hBYZNDRkaBcxo8kpGThx2eT103rXq/KBB0AcLI0wbL3mkFHAuwMeoCtFwtf6hNCYGdQDLouOY29wQ+hIwE+aueKo1M6MeiQAgcoE1WwGTNmICUlBR4eHkqT4w0fPrxMx42Li1NahPDQtVgcCM9ATupTtHazxAIfR9T7Kg0LFy7EggULsGjRIvz99z/4+puvoWNkhiXHHDC/f6OyvrxCEtOy8euxgmA3rXu9Yiegc7I0wS9Dm+LjDm5YdDgMZ+48xobz9/Dn5Qf4pGMtjOngVuI8LE/SsnE2IhFnwxNxNiIRsc8mjyvJoeuxaO1mWS7zrSSkZmHUukDcik2BqYEuhrV2xvpzUYrBvpamBuWyEOb8Q6FIzc6Dp6MU77dx0fjxy1uHutaY2t0Diw7fxrcHbqK+vQVauBTMbnz3cRpm7b2uCI8N7C2wcFBjNHGsVoEtpsqIYYeogqk6OV5ZbDx/D/MO3ES+XMBQTwebRrdG0uN4AEC/fv0weXLBNP9NmzbFP/6ncSXkH2x1aYz3WjujgYNmL+3+eDgMqdl5aFTTAoNbOpW6f0MHKTaNbo1zEYlY+M9tXH8ow8/H7mDThfuY2LUuOrpb4cHTTDhIjfHgaSb+DX+Mf8MTcSs2Rek4Bno6aO1qiQ51rdC+rhWuPZDhm70Fc5c8vw17y4X7ePA0A/8b1uyVZgEuTlRiOkasu4iYpExYmRlg/ajWaOwoxccd3HA1Jhlz/rqJhNRsTNl5Fas+aFHqnWiqOnPnMQ5cfQQdCfD9gMblNmleefu0Uy1cf5iMv6/HYdymIMzt0wDXH8qwMeA+cvLkMNbXxeS362J0OzelBUeJnmPYIapgU6dOxYwZMzBs2DAAQOPGjXH//n34+fmVKezY2dkhLi4OPx65jRUnIwEADarLkeLqCCN9XVhZWUFPTw8NGjRQKufd0hMRe49ALoBvD9zE9nFtNTax27UHydh5OQYAMK9PQ7VOvu3qWOEv33Y4dD0WPx4JQ3RSRqHJ8V5W394CHZ+Fm1aulkoDXBs6SPFWvf/mLrkUlYRpf17DqbDH6L/iHNaMaIE6NuYlHF01V2OS8dGGQCSl58Clhgk2jW4Nl2dz0jy/ndheaozBqwJw7FY8VpyMwOdd65a53qzcfMz+q+D3M9LbtVwGeb8uEokEi9/1RNC9p0hIzcYX20MUz3Vyt8aC/o3gZPn6biGnqocRmEiDYmWZOB+ZiFiZ6rfKvurkeKVp07Ytlm/Zpwg6X77tjvyYa/Dy8gIAGBgYoFWrVggLU16M8M6dO2jTxB1G+jq4GJWEQ9djCx37VcjlAvP234QQwIBmNV9paQEdHQn6eDrg+JROmPK2e5H79Gpsh/8Na4rAr7vhn4kdMLNXfXSoa13knTz2UmPFeKF+TWti92fecJAaISoxHf1XnMfxW/Fqt/FFp8IS8N6aC0hKz0HjmlLs/sxbEXRe5OlUDQv6FVwyXHL8Dk6GJRTaR12/nYrE/ScZsLUwLPZ3VZWkZhVMcvgiiQTwG8igQ6Vj2CHSkFedN+X55HiHDh3CvXv3sHfvXixZsgQDBgxQ7JOUlISQkBDcunULABAWFoaQkBDExcUp9hkxYoRiCYSMnDxk1PXB3ZBzSL20B583N8aTf7ciKChIafK8qVOnYseOHVizZg0iIiKwfPlyHDhwAF9N+gKfdaoDAPjhUCgyc/LL/PvZF/IQV6KTYWKgixk9Pcp0LAM9HbR0LXpV6g/buqJf05qwNjcs8vmSNKopxf7P26O1qyXSsvMwdnMQlp8Ix6vctLrnygN8vDEIGTn56FDXCtvGtYWVWfFtGtLKCcPbOEMIYOK2YNxLTFe7zuciH6dh5amCkDvnnYYwN9LcJbmKUtQklEIA95+oNwcPvZkYdog0oLhlF1Tp4SltcjwA2L9/P5o1a6ZYumDYsGFo1qwZVq5cqdgnOjoasbGxSErPwftrLuJmvj3s+0+DSdRpzPqwJ/7880/s27cPjRr9N+h4wIABWLlyJRYvXozGjRvj//7v/7B79260b98en3SqhZrVjPFIloXfT0eW6feTlp2Hhf8UrFc1oUsdxZwnZeFmZYqXr4JpYkZcKzNDbPm4DT5oWxA8fjp6B75/XEH6s7l/SiOEwKrTkZiy8yry5AL9mzpg7chWKg0+ntunIZo7V0NKVh4+2XxZ5Tpfrn/2vhvIyZejk7s1ejUu3/mSXpfy+nvTm4Hz7IDz7FDZ7b/6CF9sCy60vbzmTSlKrCwTgVFJ+OloGKKTMiE11se6US3RwuXVV6L+53osPtt6BQZ6OvCf0umVLxcs/Oc2Vp6OhEsNExyd3BGGeqVPDqeK8l4gcdulaMz56wZy8wU87MyxZkTLEn8HcrnA93+HYu3ZKADA2A5umNmzvloDjuNTsvDOsrN4nJqN3k3ssfy9ZmqNmdoX/BCTdoTAUE8HRyd3LPKyWVVV1RfEJM3jpIJqYNihsniSlo0Bv51DdJJyL46OBDg3o8trmaV1R2A0Zuy5rpg7Rmqsh92feZd5gK0QAsP/7yLORz5Bj4Z2WPlhC7WPEZWYDp9fTiM3X+D/RrTU+Nwn5T0jbtC9JHy65QoS07JR3UQfK95vDu86hddqys7Lx1e7ruHAs4Uyv+5VH2M7vtoiloH3kvDe6gvIkwvM6uWBcR1rq1ROlpGLrktOITEtB1/5uGNCl7IPdK5sOAMyvYiTCtJr5+rqColEUujh6+sLoGDelw8//BB2dnYwNTVF8+bNsXv37lKPu2LFCri6usLIyAht2rTBpUuXlJ7/5JNPULt2bRgbG8Pa2hr9+vXD7du3y+U1viw1Kxcj11961pOip9TN3sRRCjsNXK4pTawsUynoFLQrD6YamLNFIpFg7rO7pg7fjMO5iES1j7Hg4C3k5gt0crdG1/o2ZW7Ty14cZFweWrpa4sDn7dDEUYqnGbn4cN0lbDgXpTSO5/lkgQeuPoK+rgS/Dm36ykEHAFq5WmJun4K75Bb+c1vl3/viI7eRmJaD2tamZaq/Mivvvzdpp0oddlRZIFEIgTlz5sDe3h7Gxsbo1q0bwsPDK7DVb67AwECs/DsQjhM2w9F3M+yGLQAAxQrTI0aMQFhYGPbv34/r169j4MCBGDJkCIKDC1/+eW7Hjh2YMmUK5s6diytXrsDT0xPdu3dHQsJ/d6u0aNEC69evR2hoKI4cOQIhBHx8fJCfX/ZBtSXJys3H2E1BuPEwBTVMDbBnfDucm9EFc96pD10JEBIjw7pz98q1DQDwb3hiodmA5QIamw24np05PmxbMBndtwduIi9f9bvEToYlwP92AvR0JJjTp4HGbmF/3eylxtj5iZdiccp5B25h2p/XcP9JOv6+/giDfj+PcxFPYGqgi3WjWqF/s5plrvODti54t4Uj5AKY8McVPHha8t8zOPop/rhUMCh+Qf/GGrtUSKQNKnXYWbRoEX7//XcsX74coaGhWLRoERYvXoxly5Yp9lm8eDGWLl2KlStX4uLFizA1NUX37t2RlVX6bKmkOflygeN3M+B3Oh66ptWha1Yd6eGXoFfNHnU9WwEoWJjy888/R+vWrVGrVi188803qFatGi5fvlzscZcsWYKxY8fio48+QoMGDbBy5UqYmJhg3bp1in3GjRuHjh07wtXVFc2bN8eCBQsQExODe/fuldvrzcuXY8IfwbhwNwlmhnrYOLo1alubwV5qjNHta+Gbdwq+lf/wdygCIstvBfGYpAwsPly4F0vTAzcnd3NHdRN93IlPw5YL91Uqk5Mnx/wDBXePfdTOFbWtzTTWnopgpK+LJUM88U3v+tCRALsuP0CnH09h/NZg3IlPg6mBLraP80KHutYaqU8ikWBB/0ZoXLOgR+nTLZeRlVt0gM/Ll+PrvTcgBDCwec3XNk6MqKqo1GHn/Pnz6NevH3r37g1XV1e8++678PHxUVzGEELg119/xTfffIN+/fqhSZMm2LRpEx49eoR9+/ZVbOPfEBk5edh4/h66/HwKM/f+N8GbyM9F+q1TMGvyNj7eeBmRj9Pg7e2NHTt2ICkpCXK5HNu3b0dWVhbeeuutIo+dk5ODy5cvK63vpKOjg27duiEgIKDIMunp6Vi/fj3c3Nzg5FT67LyvQi4XmL77Oo6HxsNATwf/N7JloQnbRnm7KnoBJvxxBY+SNX977OPUbHy49iIS03JgY26ouIT2fOCmJrv5pSb6+Kp7PQDAkmN3VFqfasP5KNxNTIeVmSG+0MAkeZWBRCLBxx1q4ZehTQs9l5mbDytzA43WZ6Svi5UftoClqQFuPEx5FmgKD7PcGHAft2JTIDXWx6xe9TXaBiJtUKnDTmkLJEZFRSEuLk7pZCiVStGmTZtiT4YAkJ2djZSUFKUHqSchJQs/HrkNL78TmLv/Ju4/yYCFkR6eX6TIuHMB8qw0mDbqitC4VPT89V90Hu+H7Jwc1KhRA4aGhvjkk0+wd+9e1KlTp8g6EhMTkZ+fr7S+EwDY2toqzS8DAL/99hvMzMxgZmaGf/75B8eOHYOBgWZPPEBBwP7+71DsvvIAujoSrHi/OdrWKvwtWiKR4IcBjdHA3gJP0nPwWQnfyl9FSlYuRq67hHtPMuBY3Rj7J7THuRldsG1sW5yd0blc7lAZ1soZDewtkJKVh5+O3ilx34TULCz1jwAATO9RTyvmeXlRUXP4aPLS4YtqVjPG8meLYe6+8gCbX+pZi5VlYsnRgokhZ/T0KHEuH6I3VaUOO8+n0Pfw8IC+vj6aNWuGSZMmKRZIfH7CU+Vk+CI/Pz9IpVLFo7x6ALRRWFwqpu66ivaLTmLFyUjIMnPhUsME3/VriAuzumLhoMbQlUiQdu0oTGq1xNeDvdHJ3Ro5+XL4zZ+HS7djsHzzHgQFBWHKlCkYMmQIrl+/XuZ2DR8+HMHBwTh9+jTc3d0xZMiQcrmUueJkhOK24sWDmuDtEu4sMjbQxaoPW6CaiT6uPpBh7l83X2lyupdl5uTj4w1BuBWbAiszA2we0wZ2UqNyH7ipqyPBvL4NAQDbA6Nx46Gs2H0XHw5DWnYePJ2qYVBzx3JpT0V63XO+eNexwsyeBT023x24hcB7/62a/t2BW0jPyUdz52oYqsJaY0Rvokoddl5cIPHKlSvYuHEjfvrpJ2zcuLFMx505cyZkMpniERMTo6EWaychBM6GJ2LEukvo/usZ7Lr8ADn5crR0qY6VH7TAiS/fwggvV5gY6GFoK2dsf782cqKvYvl3X+Kzt+pgw0etMLN9daReOQjTtz/HjzcMsD1SgklTZ6Fly5ZYsWJFkfVaWVlBV1cX8fHKU/bHx8fDzk55ojSpVIq6deuiY8eO+PPPP3H79m3s3btXo7+HzRfuK3o0Zr/TAINalH4Sd7I0wbJn38p3BMUoBpC+qtx8OXz/uIJL95Jg/myskJvV65tHpbWbJfp6OkAIPFv6oXB4C45+ij8vPwAAzOvTQGOLWlYm9lJj+A0sCPZA+Vw6fNnHHdzQx9MBeXKB8VuvID4lCydux+OfG3HQ1ZHg+wGNtfJ3TaQJlXoh0NIWSHx+wouPj4e9vb2iXHx8PJo2bVrscQ0NDWFoyK7eksTKMhEen4bw+FTsuvwAt+NSARTMHdOzkT0+7uCGZs5FT9d/aPc22NjY4MMhAwEUXNLxdimY76VXEwccjwW2XYrBsVsJkGfkFXvXlIGBAVq0aAF/f3/0798fACCXy+Hv76+05MHLhBAQQiA7u/RxJao6cPUR5jxbVPHzLnUwpr2bymU71LXG1O4eWHT4NubtvwkPOwu0cCn6d1cSuVxg6q6rOHE7AYZ6Olg7qhUaOrz+xR1n9vLAsVvxCLr/FPuvPkK/pv/defR8/SsAeLeFY7HvEW0wtJUzOrpbv7Y5XyQSCRYNaozw+FTcjkvF6A2BiE8p6L0c094N9e05RxhRcSp1z05pCyS6ubnBzs4O/v7+iudTUlJw8eJFxWKHpL7tl6Lh7XcCI9ZdwvxDobgdlwoTA12M8nbF6amdsWJ482JPYnK5HOvXr8fIkSOhp/dflvbw8ECdOnVwZ/cSzGljAHtJMu6e2IbggDOIMmuAmKSCsQ5du3bF8uXLFeWmTJmCNWvWYOPGjQgNDcVnn32G9PR0fPTRRwCAu3fvws/PD5cvX0Z0dDTOnz+PwYMHw9jYGL169dLI7+P0nceYsjMEQgAftnV5pUUVP+1UC70a2yE3X2D81stISFXvEpsQAt8dvIV9IY+gpyPB7x80R2u3V58ZuSzspcbw7VwwyZ3f37eVljTYfeUBrj6QwcxQD9N61KuQ9r1Or3vOFxMDPaz6sAWM9HRw81EKEtNyAACO1TjnDFFJKnXYKW2BRIlEgkmTJmHBggWKuVtGjBgBBwcHRU8Aqef5Gk8vXpyQANg3vh3m9W1Y6nIBx48fR3R0NEaPHq20XV9fH3///Tesra0x89MPcG3pOBjfPwfbPlMQYegOn1/OYM2Zu4iMjETUg1jFyuFDhw7FTz/9hDlz5qBp06YICQnB4cOHFeO0jIyM8O+//6JXr16oU6cOhg4dCnNzc5w/fx42NmWfwO7y/SR8uvkycvMF+ng64Nu+DV9prhiJRILF73qiro0Z4lOy4bv1CnLyVJ+vZql/BDacvwcA+GmwJ7p4aHYWYnV93KEWnCyNEZeShd9OFQxETsnKxaLDBQNlv+haBzbm5T+h4pvIQE8H2S/NdfTtgVsqrcNG9Kaq1MtFpKamYvbs2di7dy8SEhLg4OCA9957D3PmzFHcaSOEwNy5c7F69WokJyejffv2+O233+Durvq3by4X8Z/dl2Pw5a5rhbaX1xpPEQlpmLX3Oi5FFQy4dJAaITYlC0IUXDLzG9i4wta+uR2XgiErA5CSlYdO7tZYM6IlDPTK9v3g7uM09Ft+DqnZeRjp5YJv+zUqtczG8/cw99mloW/7NsRIb9cytUFTjt6Mw7jNl2Ggq4NjUzpiy4X7WPNvFGpZmeLwpI5l/l1R0c5HJuL9NRcLbX+d67ARVRZcG0sNDDsF8vLl6LP8HEJjlW/F15VIcHZG53LrqpfLBXZdjsGCQ6FIzVJe5VlHAvzl2w6Nakpf6+y70U8yMGjleTxOzUYLl+rYPKY1TAw0M8TNPzQeYzYGAQB+HuxZ4kDnv0IeYuL2EADApG51Mamb+pfQyosQAiPWXcK/4Ylo5GCBW7EpkAtg/Uet0Lme5peFoAKxsky0W3gC8hc+ucv73yhRZcW1sUhtK05GIjQ2BUZ6OuU6Qd3LdHQkGNrKGT++26TQc3IB9Fl+Di0WHMf7ay7guwO3sCsoBjceyoqctyZWlqm4BPYqYmWZ+PvaIwxbE4DHqdnwsDPHupGtNBZ0AKBrfVtMfDbJ3qy914u9hfvk7QR8ufMqgIJJCidWson5CtbNagAdCXDjUYri5JuQwtnLy1NF3AlGVNVV6rux6PW5GpOMpScK1hRb9G4TtHazfO0rC3s6VYOOBErfWIGCMUNJ6Tk4H/kE519YekFXRwI3K1N42Jmjvr0FHqdmYVPAfchf8RLYjsBozNxzXVG/pakBNo1uDamJ5ifEm9i1Lm48lMH/dgI+2XwZBz5vD0vT/yZBDLyXhE+3XEaeXGBAs5qY807lXFfK1FCv0Lpcs/bcQEd3a558y9HrvhOMqKrjZSzwMlZmTj56L/sXdx+n450m9lj+fvMKa8uOwGjM2nMD+UIovrH2a1oT4fFpCI1LQWhsCm7HpiI0LgXJGbmlHq+asb5Kc4/I5QLJmcrH05EA52Z0KbcTiSwzF/1XnENUYjq8a9fAptGtoaerg1uPUjB0dQBSs/LQ1cMGKz9sAX3dytkJy/EjRFSRVD1/s2eHsPCfUNx9nA5bC0Ms6F/6gNnyVNw31saOUjR2/G9OGSEEElKzC8JPXCr+vfMY54pYcPPlAKOO59P/l1fYkRrrY9WHLdB/xTmcj3yCOftvopVLdXx3sGDsUmtXS6wY3rzSBh3gv5mEXx4/Ul4zCRMRvQr27ODN7tk5fecxRq4rWFh185jWGlux+XUratCmjgTYMqZNkesYvexxajY+WHuxQgZ9/n09FuO3XlHaZi81wpHJHWFRBdaUKqo3rqLuoCOiNwt7dqhUyRk5mLrrvwGwVTXoAP8N2nz5pOtdx0ql8nVtzYss/zrGQjRzrgYJoDS3UXxKFtKz86pE2OH4ESKq7Bh23lBCCHy97wYSUrNRy9oU03t4VHSTyqysJ92KOmlHJabj5e7V8r6Epmn2UuMq01YievMw7Lyh9l99hEPXYqGnI8GvQ5vC2EC3opukEWU96VbESZvjXoiIylflHflI5eZRcia+2VewqOUXXeuiiWO1im3QG47zphARlS/27Lxh5HKBqX9eRWpWHpo6VcP4t2pXdJMIHPdCRFSeGHbeMBvO38O5iCcw1tfFkiGe0KvEtzW/aTjuhYiofPBM9wYJj0/FwsO3AQBf966PWtZmFdwiIiKi8sew84bIyZNj8s4Q5OTJ8VY9awxvw3lQiIjozcCw84ZY6h+OGw9TUM1EH4sHNamU6ywRERGVB4adN8Dl+0n47VQEAMBvQGPYWBhVcIuIiIheH4YdLZeenYcpO69CLoCBzWqiZ2P7im4SERHRa8Wwo+UWHArF/ScZqFnNGPP6Nazo5hAREb12vPVcS8XKMrHn8gNsuxQNiQT4abBnlVhniYiISNMYdrTQjsBozNxzXbH8QPvaVvCqXaNiG0VERFRBeBlLy8TKMpWCDgCci0xErCyz4hpFRERUgRh2tExUYrpS0AH+W0GbiIjoTcSwo2XcrEzx8hQ6XEGbiIjeZAw7WsZeaoz2dawUP3MFbSIietNxgLIWepRcMD5nYte6GNbaiUGHiIjeaAw7Wib6SQYiH6dDV0eC0e3dIDXm7eZERPRm42UsLXMyLAEA0NKlOoMOERERGHa0zonbBWGni4dNBbeEiIiocmDY0SIZOXkIuPsEAMMOERHRcww7WuR8xBPk5MnhWN0YdWzMKro5RERElQLDjhZ5Pl6ni4cNJC9PtkNERPSGYtjREkIInHw2XqdzPV7CIiIieo5hR0uExafikSwLRvo6XPSTiIjoBQw7WuL5XVjeta1gpK9bwa0hIiKqPBh2tITiEhbvwiIiIlLCsKMFkjNycPn+UwBA53rWFdwaIiKiyoVhRwucCU+EXADutmZwrM7VzYmIiF7EsKMFeAmLiIioeAw7VVy+XODU8/l1eMs5ERFRIQw7VVxITDKeZuTCwkgPLVyqV3RziIiIKh2GnSru+SWsju7W0NPln5OIiOhlPDtWcS8uEUFERESFMexUYfEpWbj5KAUSCdDJnbecExERFYVhpwp7fgnL07EaapgZVnBriIiIKieGnSrs+RIRvIRFRERUPIadKio7Lx9nIxIBMOwQERGVhGGniroUlYSMnHzYmBuioYNFRTeHiIio0mLYqaJO3n4MAHirnjUkEkkFt4aIiKjyYtiponjLORERkWoYdqqgqMR0RCWmQ19XgvZ1ecs5ERFRSRh2qqDnd2G1drOEmaFeBbeGiIiocmPYqYIUq5xz4U8iIqJSMexUMenZebgY9QQAx+sQERGpgmGnijkbkYjcfAGXGiZwszKt6OYQERFVegw7VcyLl7B4yzkREVHpGHaqECEEbzknIiJSE8NOFXLzUQriU7JhYqCLNrUsK7o5REREVQLDThVy6lmvTrs6VjDU063g1hAREVUNDDtVyAneck5ERKQ2tcPOyZMny6MdVIqk9BwExyQDADp7cNZkIiIiVakddnr06IHatWtjwYIFiImJKY82URFO30mAEEB9ewvYS40rujlERERVhtph5+HDh5gwYQL+/PNP1KpVC927d8fOnTuRk5NTHu2jZ048W+W8C3t1iIiI1KJ22LGyssLkyZMREhKCixcvwt3dHePHj4eDgwO++OILXL16tTza+UbLy5fjNG85JyIieiVlGqDcvHlzzJw5ExMmTEBaWhrWrVuHFi1aoEOHDrh586am2vjGC45JRkpWHqqZ6KOpU/WKbg4REVGV8kphJzc3F3/++Sd69eoFFxcXHDlyBMuXL0d8fDwiIiLg4uKCwYMHa6SBDx8+xAcffIAaNWrA2NgYjRs3RlBQkOJ5IQTmzJkDe3t7GBsbo1u3bggPD9dI3ZXF87uwOrlbQ1eHsyYTERGpQ+2w8/nnn8Pe3h6ffPIJ3N3dERwcjICAAHz88ccwNTWFq6srfvrpJ9y+fbvMjXv69CnatWsHfX19/PPPP7h16xZ+/vlnVK/+X+/G4sWLsXTpUqxcuRIXL16EqakpunfvjqysrDLXX1k8XyKCl7CIiIjUp6dugVu3bmHZsmUYOHAgDA0Ni9zHyspKI7eoL1q0CE5OTli/fr1im5ubm+L/hRD49ddf8c0336Bfv34AgE2bNsHW1hb79u3DsGHDytyGivYwORO341KhIyno2SEiIiL1qN2z4+/vj/fee6/YoAMAenp66NSpU5kaBgD79+9Hy5YtMXjwYNjY2KBZs2ZYs2aN4vmoqCjExcWhW7duim1SqRRt2rRBQEBAscfNzs5GSkqK0qOyet6r09y5OqqZGFRwa4iIiKoetcOOn58f1q1bV2j7unXrsGjRIo006rm7d+/i999/R926dXHkyBF89tln+OKLL7Bx40YAQFxcHADA1tZWqZytra3iuaL4+flBKpUqHk5OThpttyY9XyKiMy9hERERvRK1w86qVavg4eFRaHvDhg2xcuVKjTTqOblcjubNm+OHH35As2bNMG7cOIwdO7bM9cycORMymUzxqKyTI2bl5uNcxBMAXCKCiIjoVakdduLi4mBvb19ou7W1NWJjYzXSqOfs7e3RoEEDpW3169dHdHQ0AMDOzg4AEB8fr7RPfHy84rmiGBoawsLCQulRGV24+wSZufmwlxqhvr15RTeHiIioSlI77Dg5OeHcuXOFtp87dw4ODg4aadRz7dq1Q1hYmNK2O3fuwMXFBUDBYGU7Ozv4+/srnk9JScHFixfh5eWl0bZUhINXHwEAWrtZQiLhLedERESvQu27scaOHYtJkyYhNzcXXbp0AVAwaHnatGn48ssvNdq4yZMnw9vbGz/88AOGDBmCS5cuYfXq1Vi9ejUAQCKRYNKkSViwYAHq1q0LNzc3zJ49Gw4ODujfv79G2/K6bb8UjT+vPAQA7L/6CN61a2BoK+cKbhUREVHVIxFCCHUKCCEwY8YMLF26VLEelpGREaZPn445c+ZovIEHDx7EzJkzER4eDjc3N0yZMgVjx45Vas/cuXOxevVqJCcno3379vjtt9/g7u6uch0pKSmQSqWQyWSV4pJWrCwT7RaegPyFv4yuRIKzMzpzEVAiIqJnVD1/qx12nktLS0NoaCiMjY1Rt27dEm9Fr+wqW9g5H5mI99dcLLR929i28KpdowJaREREVPmoev5W+zLWc2ZmZmjVqtWrFqcSOFYv3HujK5HA1cqkAlpDRERUtb1S2AkKCsLOnTsRHR2tuJT13J49ezTSsDfZrUepSj/rSiT4YWAjXsIiIiJ6BWrfjbV9+3Z4e3sjNDQUe/fuRW5uLm7evIkTJ05AKpWWRxvfONsuFdxa/2FbF2wb2xZnZ3Tm4GQiIqJXpHbY+eGHH/DLL7/gwIEDMDAwwP/+9z/cvn0bQ4YMgbMzT8hl9eBpBs6EPwYAfNzBDV61a7BHh4iIqAzUDjuRkZHo3bs3AMDAwADp6emQSCSYPHmy4pZwenU7A2MgBNCuTg241DCt6OYQERFVeWqHnerVqyM1tWBMSc2aNXHjxg0AQHJyMjIyMjTbujdMXr4cO4IKlq54rzV7yYiIiDRB7QHKHTt2xLFjx9C4cWMMHjwYEydOxIkTJ3Ds2DF07dq1PNr4xjgV9hjxKdmwNDXA2w1sSy9AREREpVI77CxfvhxZWVkAgK+//hr6+vo4f/48Bg0ahG+++UbjDXyTPB+Y/G4LRxjq6VZwa4iIiLSDWmEnLy8PBw8eRPfu3QEAOjo6mDFjRrk07E0TK8vEybAEAMDQVk4V3BoiIiLtodaYHT09PXz66aeKnh3SnJ2BDyAXQBs3S9S2Nqvo5hAREWkNtQcot27dGiEhIeXQlDdXvlxg57OBye+34cBkIiIiTVJ7zM748eMxZcoUxMTEoEWLFjA1Vb49ukmTJhpr3JviTPhjPEzORDUTfXRvaFfRzSEiItIqaoedYcOGAQC++OILxTaJRAIhBCQSCfLz8zXXujfE9mcDkwc2c4SRPgcmExERaZLaYScqKqo82vHGSkjJwvHQgoHJ77XmwGQiIiJNUzvsuLi4lEc73li7Lj9AvlygpUt11LU1r+jmEBERaR21w86mTZtKfH7EiBGv3Jg3jVwusD2w4BLWMM6YTEREVC7UDjsTJ05U+jk3NxcZGRkwMDCAiYkJw44azkUmIiYpE+ZGeujd2L6im0NERKSV1L71/OnTp0qPtLQ0hIWFoX379ti2bVt5tFFrbb9UcLv5gGY1YWzAgclERETlQe2wU5S6deti4cKFhXp9qHiJadk4eisOADCsFS9hERERlReNhB2gYHblR48eaepwWm/35QfIzRfwdKqGBg4WFd0cIiIiraX2mJ39+/cr/SyEQGxsLJYvX4527dpprGHaTAiB7YHPZkzm7eZERETlSu2w079/f6WfJRIJrK2t0aVLF/z888+aapdWu3A3CVGJ6TAz1MM7TRwqujlERERaTe2wI5fLy6Mdb5Rtz2ZM7tvUAaaGav8JiIiISA0aG7NDqnmanoPDNwoGJr/PuXWIiIjKndphZ9CgQVi0aFGh7YsXL8bgwYM10ihttvvKA+Tky9GopgUa1ZRWdHOIiIi0ntph58yZM+jVq1eh7T179sSZM2c00iht9eLA5PfYq0NERPRaqB120tLSYGBgUGi7vr4+UlJSNNKoymbhwoWQSCSYNGkSAODevXuQSCRFPnbt2lXscXR0dOD/5Vu4v+gdfNDWVVHmxx9/VOzTt29fODs7w8jICPb29vjwww95Sz8REVEZqB12GjdujB07dhTavn37djRo0EAjjapMAgMDsWrVKjRp0kSxzcnJCbGxsUqPb7/9FmZmZujZs2exxxq38hgcfTdj/Gp/xMbGYt26dZBIJBg0aJBin86dO2Pnzp0ICwvD7t27ERkZiXfffbdcXyMREZE2U/tWoNmzZ2PgwIGIjIxEly5dAAD+/v7Ytm1bib0aVVFaWhqGDx+ONWvWYMGCBYrturq6sLOzU9p37969GDJkCMzMzIo8liwjF6dicqFrVh0fd28GO7vq+Ouvv9C5c2fUqlVLsd/kyZMV/+/i4oIZM2agf//+yM3Nhb6+voZfIRERkfZTu2enT58+2LdvHyIiIjB+/Hh8+eWXePDgAY4fP15oDp6qztfXF71790a3bt1K3O/y5csICQnBmDFjit1nb/ADZOfJ4WFnjqZO1RAfH49Dhw6VWCYpKQlbt26Ft7c3gw4REdEreqVJXnr37o3evXtrui2Vyvbt23HlyhUEBgaWuu/atWtRv359eHt7F/n8ywOTJRIJNm7cCHNzcwwcOLDQ/tOnT8fy5cuRkZGBtm3b4uDBg2V7MURERG8wtXt2AgMDcfHixULbL168iKCgII00qqLFxMRg4sSJ2Lp1K4yMjErcNzMzE3/88UeJPTTBMcm4HZcKQz0d9G9WEwCwbt06DB8+vMjjT506FcHBwTh69Ch0dXUxYsQICCHK9qKIiIjeUGqHHV9fX8TExBTa/vDhQ/j6+mqkURXt8uXLSEhIQPPmzaGnpwc9PT2cPn0aS5cuhZ6eHvLz8xX7/vnnn8jIyMCIESOKPd72ZzMmv9PEAVJjffz7778ICwvDxx9/XOT+VlZWcHd3x9tvv43t27fj77//xoULFzT7IomIiN4Qal/GunXrFpo3b15oe7NmzXDr1i2NNKqide3aFdevX1fa9tFHH8HDwwPTp0+Hrq6uYvvatWvRt29fWFtbF3ms1KxcHLgaCwB479min2vXrkWLFi3g6elZalueL8+RnZ39Sq+FiIjoTad22DE0NER8fLzSHUQAEBsbCz097VjnydzcHI0aNVLaZmpqiho1aihtj4iIwJkzZ/D3338XeRwPDw/0GDUZmbmOqGtjhhYu1ZGSkoJdu3YVuWjqxYsXERgYiPbt26N69eqIjIzE7NmzUbt2bXh5eWn2RRIREb0h1L6M5ePjg5kzZ0Imkym2JScnY9asWXj77bc12rjKbt26dXB0dISPj0+Rz4eFheFw8F0AwLBnA5O3b98OIQTee++9QvubmJhgz5496Nq1K+rVq4cxY8agSZMmOH36NAwNDcv1tRAREWkriVBz5OvDhw/RsWNHPHnyBM2aNQMAhISEwNbWFseOHYOTk1O5NLQ8paSkQCqVQiaTwcLCQiPH3BEYjRl7ruP5b3fOOw0wur2bRo5NREREqp+/1Q47AJCeno6tW7fi6tWrMDY2RpMmTfDee+9V2blgNB12YmWZaLfwBOQv/GZ1JRKcndEZ9lLjMh+fiIiIVD9/v9IgG1NTU4wbN+6VG6ftohLTlYIOAOQLgXuJGQw7REREr9krjyi+desWoqOjkZOTo7S9b9++ZW5UVedmZQodCQr17LhamVRco4iIiN5Qaoedu3fvYsCAAbh+/TokEolisjuJRAIASnPQvKnspcbwG9gYs/bcQL4Q0JVI8MPARuzVISIiqgBqh52JEyfCzc0N/v7+cHNzw6VLl/DkyRN8+eWX+Omnn8qjjVXS0FbO6OhujXuJGXC1MmHQISIiqiBqh52AgACcOHECVlZW0NHRgY6ODtq3bw8/Pz988cUXCA4OLo92Vkn2UmOGHCIiogqm9jw7+fn5MDc3B1CwrMGjR48AAC4uLggLC9Ns64iIiIjKSO2enUaNGuHq1atwc3NDmzZtsHjxYhgYGGD16tWFZlUmIiIiqmhqh51vvvkG6enpAIDvvvsO77zzDjp06IAaNWpgx44dGm8gERERUVm80qSCL0tKSkL16tUVd2RVNeUxgzIRERGVr3KdVPBllpaWmjgMERERkcapPUCZiIiIqCph2CEiIiKtxrBDREREWk3tsHPmzBnk5eUV2p6Xl4czZ85opFFEREREmqJ22OncuTOSkpIKbZfJZOjcubNGGkVERESkKWqHHSFEkbeYP3nyBKamphppFBEREZGmqHzr+cCBAwEUrG4+atQoGBoaKp7Lz8/HtWvX4O3trfkWEhEREZWBymFHKpUCKOjZMTc3h7HxfwtcGhgYoG3bthg7dqzmW0hERERUBiqHnfXr1wMAXF1d8dVXX/GSFREREVUJao/ZmTZtmtKYnfv37+PXX3/F0aNHNdowIiIiIk1QO+z069cPmzZtAgAkJyejdevW+Pnnn9GvXz/8/vvvGm8gERERUVmoHXauXLmCDh06AAD+/PNP2NnZ4f79+9i0aROWLl2q8QYSERERlYXaYScjIwPm5uYAgKNHj2LgwIHQ0dFB27Ztcf/+fY03kIiIiKgs1A47derUwb59+xATE4MjR47Ax8cHAJCQkFDi8upEREREFUHtsDNnzhx89dVXcHV1RevWreHl5QWgoJenWbNmGm8gERERUVmoHXbeffddREdHIygoCEeOHFFs79q1K3755ReNNu5lCxcuhEQiwaRJkxTbsrKy4Ovrixo1asDMzAyDBg1CfHx8ubaDiIiIqo5XWvXczs4O5ubmOHbsGDIzMwEArVq1goeHh0Yb96LAwECsWrUKTZo0Udo+efJkHDhwALt27cLp06fx6NEjxWzPRERERGqHnSdPnqBr165wd3dHr169EBsbCwAYM2YMvvzyS403EADS0tIwfPhwrFmzBtWrV1dsl8lkWLt2LZYsWYIuXbqgRYsWWL9+Pc6fP48LFy6US1uIiIioalE77EyePBn6+vqIjo6GiYmJYvvQoUNx+PBhjTbuOV9fX/Tu3RvdunVT2n758mXk5uYqbffw8ICzszMCAgKKPV52djZSUlKUHkRERKSdVF4u4rmjR4/iyJEjcHR0VNpet27dcrn1fPv27bhy5QoCAwMLPRcXFwcDAwNUq1ZNabutrS3i4uKKPaafnx++/fZbTTeViIiIKiG1e3bS09OVenSeS0pKUloJXRNiYmIwceJEbN26FUZGRho77syZMyGTyRSPmJgYjR2biIiIKhe1w06HDh0Uy0UAgEQigVwux+LFi9G5c2eNNu7y5ctISEhA8+bNoaenBz09PZw+fRpLly6Fnp4ebG1tkZOTg+TkZKVy8fHxsLOzK/a4hoaGsLCwUHoQERGRdlL7MtbixYvRtWtXBAUFIScnB9OmTcPNmzeRlJSEc+fOabRxXbt2xfXr15W2ffTRR/Dw8MD06dPh5OQEfX19+Pv7Y9CgQQCAsLAwREdHK+b/ISIiojeb2mGnUaNGuHPnDpYvXw5zc3OkpaVh4MCB8PX1hb29vUYbZ25ujkaNGiltMzU1RY0aNRTbx4wZgylTpsDS0hIWFhb4/PPP4eXlhbZt22q0LURERFQ1qR12oqOj4eTkhK+//rrI55ydnTXSMFX98ssv0NHRwaBBg5CdnY3u3bvjt99+e61tICIiospLIoQQ6hTQ1dVFbGwsbGxslLY/efIENjY2yM/P12gDX4eUlBRIpVLIZDKO3yEiIqoiVD1/qz1AWQgBiURSaHtaWppG75giIiIi0gSVL2NNmTIFQMHdV7Nnz1a6/Tw/Px8XL15E06ZNNd5AIiIiorJQOewEBwcDKOjZuX79OgwMDBTPGRgYwNPTE1999ZXmW0hERERUBiqHnZMnTwIouPX7f//7H8e2EBERUZWg9t1Y69evL492EBEREZULtQcoExEREVUlDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Sp12PHz80OrVq1gbm4OGxsb9O/fH2FhYUr7ZGVlwdfXFzVq1ICZmRkGDRqE+Pj4CmoxERERVTaVOuycPn0avr6+uHDhAo4dO4bc3Fz4+PggPT1dsc/kyZNx4MAB7Nq1C6dPn8ajR48wcODACmw1ERERVSYSIYSo6Eao6vHjx7CxscHp06fRsWNHyGQyWFtb448//sC7774LALh9+zbq16+PgIAAtG3bVqXjpqSkQCqVQiaTwcLCojxfAhEREWmIqufvSt2z8zKZTAYAsLS0BABcvnwZubm56Natm2IfDw8PODs7IyAgoNjjZGdnIyUlRelBRERE2qnKhB25XI5JkyahXbt2aNSoEQAgLi4OBgYGqFatmtK+tra2iIuLK/ZYfn5+kEqlioeTk1N5Np2IiIgqUJUJO76+vrhx4wa2b99e5mPNnDkTMplM8YiJidFAC4mIiKgy0qvoBqhiwoQJOHjwIM6cOQNHR0fFdjs7O+Tk5CA5OVmpdyc+Ph52dnbFHs/Q0BCGhobl2WQiIiKqJCp1z44QAhMmTMDevXtx4sQJuLm5KT3fokUL6Ovrw9/fX7EtLCwM0dHR8PLyet3NJSIiokqoUvfs+Pr64o8//sBff/0Fc3NzxTgcqVQKY2NjSKVSjBkzBlOmTIGlpSUsLCzw+eefw8vLS+U7sYiIiEi7VepbzyUSSZHb169fj1GjRgEomFTwyy+/xLZt25CdnY3u3bvjt99+K/Ey1st46zkREVHVo+r5u1KHndeFYYeIiKjq0cp5doiIiIjUxbBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIq2lN2FmxYgVcXV1hZGSENm3a4NKlSxXdJCIiIqoEtCLs7NixA1OmTMHcuXNx5coVeHp6onv37khISKjophEREVEF04qws2TJEowdOxYfffQRGjRogJUrV8LExATr1q2r6KYRERFRBdOr6AaUVU5ODi5fvoyZM2cqtuno6KBbt24ICAgoskx2djays7MVP8tkMgBASkpK+TaWiIiINOb5eVsIUeJ+VT7sJCYmIj8/H7a2tkrbbW1tcfv27SLL+Pn54dtvvy203cnJqVzaSEREROUnNTUVUqm02OerfNh5FTNnzsSUKVMUP8vlciQlJaFGjRqQSCQaqyclJQVOTk6IiYmBhYXFay3Pul9/3WUtz7rfrLrLWp51s+6qUr6sdZdECIHU1FQ4ODiUuF+VDztWVlbQ1dVFfHy80vb4+HjY2dkVWcbQ0BCGhoZK26pVq1ZeTYSFhUWZ/sBlKc+6X3/dZS3Put+sustannWz7qpSvqx1F6ekHp3nqvwAZQMDA7Ro0QL+/v6KbXK5HP7+/vDy8qrAlhEREVFlUOV7dgBgypQpGDlyJFq2bInWrVvj119/RXp6Oj766KOKbhoRERFVMK0IO0OHDsXjx48xZ84cxMXFoWnTpjh8+HChQcuvm6GhIebOnVvoktnrKM+6X3/dZS3Put+sustannWz7qpSvqx1a4JElHa/FhEREVEVVuXH7BARERGVhGGHiIiItBrDDhEREWk1hh0iIiLSagw75WjFihVwdXWFkZER2rRpg0uXLqlU7syZM+jTpw8cHBwgkUiwb98+lev08/NDq1atYG5uDhsbG/Tv3x9hYWEql//999/RpEkTxeRPXl5e+Oeff1Qu/6KFCxdCIpFg0qRJKu0/b948SCQSpYeHh4fK9T18+BAffPABatSoAWNjYzRu3BhBQUEqlXV1dS1Ut0Qiga+vb6ll8/PzMXv2bLi5ucHY2Bi1a9fG/PnzS12r5UWpqamYNGkSXFxcYGxsDG9vbwQGBhbar7T3hhACc+bMgb29PYyNjdGtWzeEh4erXH7Pnj3w8fFRzCYeEhKicv25ubmYPn06GjduDFNTUzg4OGDEiBF49OiRSnXPmzcPHh4eMDU1RfXq1dGtWzdcvHhR5ba/6NNPP4VEIsGvv/6qUtlRo0YV+tv36NFDrbpDQ0PRt29fSKVSmJqaolWrVoiOji61bFHvO4lEgh9//FGlutPS0jBhwgQ4OjrC2NhYsRiyKmXj4+MxatQoODg4wMTEBD169FC8X1T5LMnKyoKvry9q1KgBMzMzDBo0SDHBqyrlV69ejbfeegsWFhaQSCRITk5WPFda+aSkJHz++eeoV68ejI2N4ezsjC+++AIymUyluj/55BPUrl0bxsbGsLa2Rr9+/RRLDKnzOSqEQM+ePRW/X1XKvvXWW4X+3p9++qladQcEBKBLly4wNTWFhYUFOnbsiO+++67Esvfu3Sv2/bZr1y6V6o6Li8OHH34IOzs7mJqaonnz5ti9e7dKZSMjIzFgwABYW1vDwsICQ4YMKTQhcHlh2CknO3bswJQpUzB37lxcuXIFnp6e6N69OxISEkotm56eDk9PT6xYsULtek+fPg1fX19cuHABx44dQ25uLnx8fJCenq5SeUdHRyxcuBCXL19GUFAQunTpgn79+uHmzZtqtSMwMBCrVq1CkyZN1CrXsGFDxMbGKh5nz55VqdzTp0/Rrl076Ovr459//sGtW7fw888/o3r16iq398V6jx07BgAYPHhwqWUXLVqE33//HcuXL0doaCgWLVqExYsXY9myZSrVDQAff/wxjh07hs2bN+P69evw8fFBt27d8PDhQ6X9SntvLF68GEuXLsXKlStx8eJFmJqaonv37sjKylKpfHp6Otq3b49FixYV+3xx5TMyMnDlyhXMnj0bV65cwZ49exAWFoa+ffuqVLe7uzuWL1+O69ev4+zZs3B1dYWPjw8eP36sUvnn9u7diwsXLihNH69K2R49eii9B7Zt26Zy+cjISLRv3x4eHh44deoUrl27htmzZ8PIyKjUsi/WGRsbi3Xr1kEikWDQoEEq1T1lyhQcPnwYW7ZsQWhoKCZNmoQJEyZg//79JZYVQqB///64e/cu/vrrLwQHB8PFxQXdunVDenq6Sp8lkydPxoEDB7Br1y6cPn0ajx49wsCBAwGo9lmUkZGBHj16YNasWYXaV1r5R48e4dGjR/jpp59w48YNbNiwAYcPH8aYMWNUqrtFixZYv349QkNDceTIEQgh4OPjg/z8fLU+R3/99VelZYZULTt27Filv/vixYtVLh8QEIAePXrAx8cHly5dQmBgICZMmICzZ8+WWNbJyanQ++3bb7+FmZkZevbsqVLdI0aMQFhYGPbv34/r169j4MCBGDJkCA4cOFBi2fT0dPj4+EAikeDEiRM4d+4ccnJy0KdPH8jl8kK/V40TVC5at24tfH19FT/n5+cLBwcH4efnp9ZxAIi9e/e+cjsSEhIEAHH69OlXPkb16tXF//3f/6m8f2pqqqhbt644duyY6NSpk5g4caJK5ebOnSs8PT1fqY3Tp08X7du3f6WyRZk4caKoXbu2kMvlpe7bu3dvMXr0aKVtAwcOFMOHD1eproyMDKGrqysOHjyotL158+bi66+/Lrbcy+8NuVwu7OzsxI8//qjYlpycLAwNDcW2bdtKLf+iqKgoAUAEBwerXH9RLl26JACI+/fvq11WJpMJAOL48eMq1/3gwQNRs2ZNcePGDeHi4iJ++eUXlcqOHDlS9OvXr8T2lFR+6NCh4oMPPnilsi/r16+f6NKli8rlGzZsKL777julbUW9d14uGxYWJgCIGzduKLbl5+cLa2trsWbNmkJ1v/xZkpycLPT19cWuXbsU+4SGhgoAIiAgoNTyLzp58qQAIJ4+fVrk6y6t/HM7d+4UBgYGIjc3V+2yV69eFQBERESEynUHBweLmjVritjY2GL/tkWVVedzsajybdq0Ed98880rlX1Z06ZNC31+lVTe1NRUbNq0SWk/S0vLQu+Zl8seOXJE6OjoCJlMptgnOTlZSCQScezYsVJfS1mxZ6cc5OTk4PLly+jWrZtim46ODrp164aAgIDX2haZTAYAsLS0VLtsfn4+tm/fjvT0dLWW3vD19UXv3r2VXr+qwsPD4eDggFq1amH48OGIjo5Wqdz+/fvRsmVLDB48GDY2NmjWrBnWrFmjdv1Awd9vy5YtGD16tEoLw3p7e8Pf3x937twBAFy9ehVnz55Fz549VaovLy8P+fn5MDIyUtpubGyscs8WAERFRSEuLk7p9y6VStGmTZvX/r57TiaTQSKRqL32XE5ODlavXg2pVApPT0+Vysjlcnz44YeYOnUqGjZsqHZbT506BRsbG9SrVw+fffYZnjx5onK9hw4dgru7O7p37w4bGxu0adNGrcvPz8XHx+PQoUMYM2aMymW8vb2xf/9+PHz4EEIInDx5Enfu3IGPj0+J5bKzswFA6X2no6MDQ0PDIt93L3+WXL58Gbm5uUrvNw8PDzg7Oxf5fivLZ5Gq5WUyGSwsLKCnp1doe0ll09PTsX79eri5ucHJyUmlujMyMvD+++9jxYoVxa7DWFLdW7duhZWVFRo1aoSZM2ciIyNDpfIJCQm4ePEibGxs4O3tDVtbW3Tq1Emlv9nLLl++jJCQkGLfb0WV9/b2xo4dO5CUlAS5XI7t27cjKysLb731Volls7OzIZFIlCYWNDIygo6Ojlqfc6+s3OPUG+jhw4cCgDh//rzS9qlTp4rWrVurdSyUoWcnPz9f9O7dW7Rr106tcteuXROmpqZCV1dXSKVScejQIZXLbtu2TTRq1EhkZmYKIdT7BvP333+LnTt3iqtXr4rDhw8LLy8v4ezsLFJSUkota2hoKAwNDcXMmTPFlStXxKpVq4SRkZHYsGGDym1/bseOHUJXV1c8fPhQpf3z8/PF9OnThUQiEXp6ekIikYgffvhBrTq9vLxEp06dxMOHD0VeXp7YvHmz0NHREe7u7sWWefm9ce7cOQFAPHr0SGm/wYMHiyFDhpRa/kWa6NnJzMwUzZs3F++//77KZQ8cOCBMTU2FRCIRDg4O4tKlSyrX/cMPP4i3335b0RunTs/Otm3bxF9//SWuXbsm9u7dK+rXry9atWol8vLySi3//Fu9iYmJWLJkiQgODhZ+fn5CIpGIU6dOqfS6n1u0aJGoXr264t+PKm3PysoSI0aMEACEnp6eMDAwEBs3biy1bE5OjnB2dhaDBw8WSUlJIjs7WyxcuFAAED4+Pkpli/os2bp1qzAwMChUT6tWrcS0adNKLf+i0np2VPkse/z4sXB2dhazZs1SueyKFSuEqampACDq1atXZK9OceXHjRsnxowZo/i5qL9NcWVXrVolDh8+LK5duya2bNkiatasKQYMGKBS3QEBAQKAsLS0FOvWrRNXrlwRkyZNEgYGBuLOnTsqve7nPvvsM1G/fv0inyuu/NOnT4WPj4/i/WZhYSGOHDlSatmEhARhYWEhJk6cKNLT00VaWpqYMGGCACDGjRtXbBs1hWGnHFSWsPPpp58KFxcXERMTo1a57OxsER4eLoKCgsSMGTOElZWVuHnzZqnloqOjhY2Njbh69apimzph52VPnz4VFhYWKl1C09fXF15eXkrbPv/8c9G2bVu16/Xx8RHvvPOOyvtv27ZNODo6im3btolr166JTZs2CUtLS7WCVkREhOjYsaMAIHR1dUWrVq3E8OHDhYeHR7FlKnPYycnJEX369BHNmjVT6rYurWxaWpoIDw8XAQEBYvTo0cLV1VXEx8eXWj4oKEjY2toqBVR1ws7LIiMjVb6E9vzf+3vvvae0X58+fcSwYcPUqrtevXpiwoQJxT5fVPkff/xRuLu7i/3794urV6+KZcuWCTMzs0KXBooqGxQUJDw9PRXvu+7du4uePXuKHj16KO1X1GeJOmGntM+i0sJOaeVlMplo3bq16NGjh8jJyVG5bHJysrhz5444ffq06NOnj2jevHmhoFlU+b/++kvUqVNHpKamKrYV9ftV9TPY39+/yEtoRZV//u985syZSvs2btxYzJgxQ+W6MzIyhFQqFT/99FORzxdXfsKECaJ169bi+PHjIiQkRMybN09IpVJx7dq1UsseOXJE1KpVS0gkEqGrqys++OAD0bx5c/Hpp5+W8NvRDIadcpCdnS10dXULvfFHjBgh+vbtq9axXjXs+Pr6CkdHR3H37l21y76sa9euKiXvvXv3Kj40nz8AKN7YRX1LLk3Lli2V/gEXx9nZWelblhBC/Pbbb8LBwUGt+u7duyd0dHTEvn37VC7j6Ogoli9frrRt/vz5ol69emrVLUTByf55WBkyZIjo1atXsfu+/N54foJ+OaB07NhRfPHFF6WWf1FZwk5OTo7o37+/aNKkiUhMTFSr7Mvq1KlTZC/Zy+V/+eUXxfvsxfeejo6OcHFxeaW6raysxMqVK0utOzs7W+jp6Yn58+cr7Tdt2jTh7e2tct1nzpwRAERISEixbXq5fEZGhtDX1y803mvMmDGie/fuKtednJwsEhIShBAF4w3Hjx+veK64z5LnJ+iXA4qzs7NYsmRJqeVfVFLYKa18SkqK8PLyEl27di0UVNT5HMzOzhYmJibijz/+KLX8xIkTi32/derUSe2609LSBABx+PDhUuu+e/euACA2b96stH3IkCGKXlRV6t60aZPQ19dX/N1fVFz5iIiIQuO8hCg4R3zyyScq1/348WPF39rW1lYsXry42H01hWN2yoGBgQFatGgBf39/xTa5XA5/f3+1xr68CiEEJkyYgL179+LEiRNwc3Mr8zHlcrni+n5JunbtiuvXryMkJETxaNmyJYYPH46QkBDo6uqqVW9aWhoiIyNhb29f6r7t2rUrdJvjnTt34OLiolad69evh42NDXr37q1ymYyMDOjoKP9T0tXVfaU7DExNTWFvb4+nT5/iyJEj6Nevn8pl3dzcYGdnp/S+S0lJwcWLF8v9ffdcbm4uhgwZgvDwcBw/fhw1atQo0/FUfe99+OGHuHbtmtJ7z8HBAVOnTsWRI0fUrvfBgwd48uSJSu89AwMDtGrVqszvv7Vr16JFixYqj1ECCn7fubm5ZX7/SaVSWFtbIzw8HEFBQejXr1+pnyUtWrSAvr6+0vstLCwM0dHR8PLyKvNnkSrlU1JS4OPjAwMDA+zfv18x/uhV6hYFX/6RnZ1davkZM2YUer8BwC+//IJ169apXffz8vb29qXW7erqCgcHhyLfb87OzirXvXbtWvTt2xfW1tZKv4OSyj8fV1TU+y0/P1/luq2srFCtWjWcOHECCQkJijs2y1W5x6k31Pbt24WhoaHYsGGDuHXrlhg3bpyoVq2aiIuLK7VsamqqCA4OFsHBwQKAYhzAy3e0FOWzzz4TUqlUnDp1SsTGxioeGRkZKrV7xowZ4vTp0yIqKkpcu3ZNzJgxQ0gkEnH06FGVyr9MnctYX375pTh16pSIiooS586dE926dRNWVlZFfvN42aVLl4Senp74/vvvRXh4uNi6daswMTERW7ZsUbmt+fn5wtnZWUyfPl3lMkIU3MlTs2ZNcfDgQREVFSX27NkjrKysCnXll+Tw4cPin3/+EXfv3hVHjx4Vnp6eok2bNoW65Et7byxcuFBUq1ZNMf6kX79+ws3NTfGNt7TyT548EcHBweLQoUMCgNi+fbsIDg4WsbGxpZbPyckRffv2FY6OjiIkJETp/ZednV1i2bS0NDFz5kwREBAg7t27J4KCgsRHH30kDA0NFd8i1f138eJlrJLKpqamiq+++koEBASIqKgocfz4cdG8eXNRt25dkZWVpVLde/bsEfr6+mL16tUiPDxcLFu2TOjq6op///1XpXbLZDJhYmIifv/990Kvo7TynTp1Eg0bNhQnT54Ud+/eFevXrxdGRkbit99+K7Xszp07xcmTJ0VkZKTYt2+fcHFxEQMHDhRCqPZZ8umnnwpnZ2dx4sQJERQUJLy8vBSXk1UpHxsbK4KDg8WaNWsEAHHmzBkRHBwsnjx5Ump5mUwm2rRpIxo3biwiIiKU9vn0009LLBsZGSl++OEHERQUJO7fvy/OnTsn+vTpIywtLUV8fPwrfY7iWc9ZaWUjIiLEd999J4KCgkRUVJT466+/RK1atUTHjh1V/r398ssvwsLCQuzatUuEh4eLb775RhgZGYn3339fpXaHh4cLiUQi/vnnH6XtpdWdk5Mj6tSpIzp06CAuXrwoIiIixE8//SQkEono1atXqXWvW7dOBAQEiIiICLF582ZhaWkppkyZUuzvVJMYdsrRsmXLhLOzszAwMBCtW7cWFy5cUKnc8y7dlx8jR44stWxR5QCI9evXq1T36NGjhYuLizAwMBDW1taia9eurxx0hFAv7AwdOlTY29sLAwMDUbNmTTF06NAiBwwW58CBA6JRo0bC0NBQeHh4iNWrV6vV1iNHjggAIiwsTK1yKSkpYuLEicLZ2VkYGRmJWrVqia+//lpkZ2erfIwdO3aIWrVqCQMDA2FnZyd8fX1FcnJyof1Ke2/I5XIxe/ZsYWtrKwwNDUXXrl2VXk9p5devX1/k83Pnzi21/PNLX0U9Tp48WWLZzMxMMWDAAOHg4CAMDAyEvb296Nu3r9IAZXX/XbwYdkoqm5GRIXx8fIS1tbXQ19cXLi4uYuzYsUpfTFSpe+3ataJOnTrCyMhIeHp6Ki6FqlJ21apVwtjY+JX+5rGxsWLUqFHCwcFBGBkZiXr16omff/5ZyOXyUsv+73//E46OjkJfX184OzuLb775RvG+VeWzJDMzU4wfP15Ur15dmJiYiAEDBiiCsSrl586dW+w+pZUv7rWV9Hhe9uHDh6Jnz57CxsZG6OvrC0dHR/H++++L27dvq9z2lz0PO6WVjY6OFh07dhSWlpbC0NBQ1KlTR0ydOlUxtk3Vuv38/ISjo6MwMTERXl5e4t9//1W57MyZM4WTk5PIz88v9BpKK3/nzh0xcOBAYWNjI0xMTESTJk3Epk2bVCo7ffp0YWtrK/T19UXdunUV79PXQfLsBRIRERFpJY7ZISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQEb3k1KlTkEgkSE5OruimEJEGMOwQERGRVmPYISIiIq3GsENElY5cLoefnx/c3NxgbGwMT09P/PnnnwD+u8R06NAhNGnSBEZGRmjbti1u3LihdIzdu3ejYcOGMDQ0hKurK37++Wel57OzszF9+nQ4OTnB0NAQderUwdq1a5X2uXz5Mlq2bAkTExN4e3sXWmmaiKoGhh0iqnT8/PywadMmrFy5Ejdv3sTkyZPxwQcf4PTp04p9pk6dip9//hmBgYGwtrZGnz59kJubC6AgpAwZMgTDhg3D9evXMW/ePMyePRsbNmxQlB8xYgS2bduGpUuXIjQ0FKtWrYKZmZlSO77++mv8/PPPCAoKgp6eHkaPHv1aXj8RaRYXAiWiSiU7OxuWlpY4fvw4vLy8FNs//vhjZGRkYNy4cejcuTO2b9+OoUOHAgCSkpLg6OiIDRs2YMiQIRg+fDgeP36Mo0ePKspPmzYNhw4dws2bN3Hnzh3Uq1cPx44dQ7du3Qq14dSpU+jcuTOOHz+Orl27AgD+/vtv9O7dG5mZmTAyMirn3wIRaRJ7doioUomIiEBGRgbefvttmJmZKR6bNm1CZGSkYr8Xg5ClpSXq1auH0NBQAEBoaCjatWundNx27dohPDwc+fn5CAkJga6uLjp16lRiW5o0aaL4f3t7ewBAQkJCmV8jEb1eehXdACKiF6WlpQEADh06hJo1ayo9Z2hoqBR4XpWxsbFK++nr6yv+XyKRACgYT0REVQt7doioUmnQoAEMDQ0RHR2NOnXqKD2cnJwU+124cEHx/0+fPsWdO3dQv359AED9+vVx7tw5peOeO3cO7u7u0NXVRePGjSGXy5XGABGR9mLPDhFVKubm5vjqq68wefJkyOVytG/fHjKZDOfOnYOFhQVcXFwAAN999x1q1KgBW1tbfP3117CyskL//v0BAF9++SVatWqF+fPnY+jQoQgICMDy5cvx22+/AQBcXV0xcuRIjB49GkuXLoWnpyfu37+PhIQEDBkypKJeOhGVE4YdIqp05s+fD2tra/j5+eHu3buoVq0amjdvjlmzZikuIy1cuBATJ05EeHg4mjZtigMHDsDAwAAA0Lx5c+zcuRNz5szB/PnzYW9vj++++w6jRo1S1PH7779j1qxZGD9+PJ48eQJnZ2fMmjWrIl4uEZUz3o1FRFXK8zulnj59imrVqlV0c4ioCuCYHSIiItJqDDtERESk1XgZi4iIiLQae3aIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIq/0/aNuF5S+4oysAAAAASUVORK5CYII=", + "text/plain": [ + "

" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plt.plot(np.arange(len(epochs_x)), epochs_acc, marker = '.')\n", + "plt.xlabel('epoch')\n", + "plt.ylabel('test accuracy')\n", + "plt.ylim(0, 100)\n", + "plt.xticks(np.arange(len(epochs_x)))\n", + "for i, txt in enumerate(epochs_acc):\n", + " if i%5 == 0:\n", + " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", + " else:\n", + " pass\n", + "plt.show()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "speck-rescnn", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From acb3adcc281bf5895ea04a58775268f0971f5417 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Wed, 1 May 2024 15:31:41 +0200 Subject: [PATCH 065/379] testing different architectures --- .../ARCHITECTURES_SEARCH/Res-SCNN3.ipynb | 393 ++++++++++++++++++ .../using_SumPool2d/models/ResCSNN3.py | 122 ++++++ .../using_SumPool2d/models/SCNN.py | 130 ++++++ .../test_nonsequential/utils/train_test_fn.py | 81 ++++ 4 files changed, 726 insertions(+) create mode 100644 tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/Res-SCNN3.ipynb create mode 100644 tests/test_nonsequential/using_SumPool2d/models/ResCSNN3.py create mode 100644 tests/test_nonsequential/using_SumPool2d/models/SCNN.py create mode 100644 tests/test_nonsequential/utils/train_test_fn.py diff --git a/tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/Res-SCNN3.ipynb b/tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/Res-SCNN3.ipynb new file mode 100644 index 00000000..d7817d15 --- /dev/null +++ b/tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/Res-SCNN3.ipynb @@ -0,0 +1,393 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import torch, random, sys\n", + "\n", + "from tqdm.notebook import tqdm\n", + "\n", + "import tonic\n", + "from tonic.datasets.dvsgesture import DVSGesture\n", + "from tonic.transforms import ToFrame\n", + "from torch.utils.data import DataLoader\n", + "from torch.nn import CrossEntropyLoss\n", + "from torch.optim import Adam\n", + "\n", + "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential, MultiGaussian\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "\n", + "sys.path.append('../../utils')\n", + "sys.path.append('../models')\n", + "\n", + "from SCNN import SCNN\n", + "# from CSNN3 import CSNN3\n", + "from train_test_fn import training_loop" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "torch.backends.cudnn.enabled = False\n", + "torch.backends.cudnn.deterministic = True\n", + "random.seed(1)\n", + "torch.manual_seed(1)\n", + "torch.cuda.manual_seed(1)\n", + "np.random.seed(1)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Loading Data" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "batch_size = 8\n", + "num_workers = 4\n", + "epochs = 30\n", + "lr = 5e-5\n", + "\n", + "spk_thr = 2.0\n", + "v_min = -0.313\n", + "\n", + "grad_scale = 1.534\n", + "grad_width = 0.759" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "root_dir = \"../../DVSGESTURE\"\n", + "_ = DVSGesture(save_to=root_dir, train=True)\n", + "_ = DVSGesture(save_to=root_dir, train=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "n_time_steps = 50\n", + "to_raster = ToFrame(sensor_size=DVSGesture.sensor_size, n_time_bins=n_time_steps)\n", + "\n", + "snn_train_dataset = DVSGesture(save_to=root_dir, train=True, transform=to_raster)\n", + "snn_test_dataset = DVSGesture(save_to=root_dir, train=False, transform=to_raster)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The transformed array is in shape [Time-Step, Channel, Height, Width] --> (50, 2, 128, 128)\n" + ] + } + ], + "source": [ + "sample_data, label = snn_train_dataset[0]\n", + "print(f\"The transformed array is in shape [Time-Step, Channel, Height, Width] --> {sample_data.shape}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "disk caching samples..." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "disk_cache_train = tonic.DiskCachedDataset(\n", + " dataset=snn_train_dataset,\n", + " cache_path='./cached_train'\n", + ")\n", + "\n", + "disk_cache_test = tonic.DiskCachedDataset(\n", + " dataset=snn_test_dataset,\n", + " cache_path='./cached_test'\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "snn_train_dataloader = DataLoader(disk_cache_train, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True)\n", + "snn_test_dataloader = DataLoader(disk_cache_test, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=False)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Network Module\n", + "\n", + "We need to define a `nn.Module` implementing the network we want the chip to reproduce." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "device: NVIDIA GeForce RTX 3070 Ti\n" + ] + } + ], + "source": [ + "if torch.cuda.is_available():\n", + " device = torch.device('cuda:0')\n", + " print('device: ', torch.cuda.get_device_name(0))\n", + "else:\n", + " device = torch.device('cpu')" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "flat_s: 784\n" + ] + } + ], + "source": [ + "snn = SCNN(DVSGesture.sensor_size, 11, batch_size, PeriodicExponential(grad_scale=grad_scale, grad_width=grad_width), v_min, spk_thr).to(device)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "snn.init_weights()" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "optimizer = Adam(snn.parameters(), lr=lr, betas=(0.9, 0.999), eps=1e-8)\n", + "loss_fn = CrossEntropyLoss()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Define train and test" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Training loop (HPO)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "e2d31de3202340d6898918b79f8f2f90", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/134 [00:00 1\u001b[0m epochs_x, epochs_y, epochs_acc \u001b[38;5;241m=\u001b[39m \u001b[43mtraining_loop\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 2\u001b[0m \u001b[43m \u001b[49m\u001b[43mdevice\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[1;32m 3\u001b[0m \u001b[43m \u001b[49m\u001b[43mn_time_steps\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 4\u001b[0m \u001b[43m \u001b[49m\u001b[43mbatch_size\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 5\u001b[0m \u001b[43m \u001b[49m\u001b[43mDVSGesture\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msensor_size\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 6\u001b[0m \u001b[43m \u001b[49m\u001b[43msnn_train_dataloader\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[1;32m 7\u001b[0m \u001b[43m \u001b[49m\u001b[43msnn\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[1;32m 8\u001b[0m \u001b[43m \u001b[49m\u001b[43mloss_fn\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[1;32m 9\u001b[0m \u001b[43m \u001b[49m\u001b[43moptimizer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[1;32m 10\u001b[0m \u001b[43m \u001b[49m\u001b[43mepochs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[1;32m 11\u001b[0m \u001b[43m \u001b[49m\u001b[43msnn_test_dataloader\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/Documents/github/sinabs/tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/../../utils/train_test_fn.py:33\u001b[0m, in \u001b[0;36mtraining_loop\u001b[0;34m(device, nb_time_steps, batch_size, feature_map_size, dataloader_train, model, loss_fn, optimizer, epochs, dataloader_test)\u001b[0m\n\u001b[1;32m 31\u001b[0m \u001b[38;5;66;03m# gradient update\u001b[39;00m\n\u001b[1;32m 32\u001b[0m optimizer\u001b[38;5;241m.\u001b[39mzero_grad()\n\u001b[0;32m---> 33\u001b[0m \u001b[43mloss\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbackward\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 34\u001b[0m optimizer\u001b[38;5;241m.\u001b[39mstep()\n\u001b[1;32m 36\u001b[0m \u001b[38;5;66;03m# detach the neuron states and activations from current computation graph(necessary)\u001b[39;00m\n", + "File \u001b[0;32m~/.pyenv/versions/3.10.0/lib/python3.10/site-packages/torch/_tensor.py:525\u001b[0m, in \u001b[0;36mTensor.backward\u001b[0;34m(self, gradient, retain_graph, create_graph, inputs)\u001b[0m\n\u001b[1;32m 515\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m has_torch_function_unary(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m 516\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m handle_torch_function(\n\u001b[1;32m 517\u001b[0m Tensor\u001b[38;5;241m.\u001b[39mbackward,\n\u001b[1;32m 518\u001b[0m (\u001b[38;5;28mself\u001b[39m,),\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 523\u001b[0m inputs\u001b[38;5;241m=\u001b[39minputs,\n\u001b[1;32m 524\u001b[0m )\n\u001b[0;32m--> 525\u001b[0m \u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mautograd\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbackward\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 526\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mgradient\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mretain_graph\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcreate_graph\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minputs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43minputs\u001b[49m\n\u001b[1;32m 527\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/.pyenv/versions/3.10.0/lib/python3.10/site-packages/torch/autograd/__init__.py:267\u001b[0m, in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[0m\n\u001b[1;32m 262\u001b[0m retain_graph \u001b[38;5;241m=\u001b[39m create_graph\n\u001b[1;32m 264\u001b[0m \u001b[38;5;66;03m# The reason we repeat the same comment below is that\u001b[39;00m\n\u001b[1;32m 265\u001b[0m \u001b[38;5;66;03m# some Python versions print out the first line of a multi-line function\u001b[39;00m\n\u001b[1;32m 266\u001b[0m \u001b[38;5;66;03m# calls in the traceback and some print out the last line\u001b[39;00m\n\u001b[0;32m--> 267\u001b[0m \u001b[43m_engine_run_backward\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 268\u001b[0m \u001b[43m \u001b[49m\u001b[43mtensors\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 269\u001b[0m \u001b[43m \u001b[49m\u001b[43mgrad_tensors_\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 270\u001b[0m \u001b[43m \u001b[49m\u001b[43mretain_graph\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 271\u001b[0m \u001b[43m \u001b[49m\u001b[43mcreate_graph\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 272\u001b[0m \u001b[43m \u001b[49m\u001b[43minputs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 273\u001b[0m \u001b[43m \u001b[49m\u001b[43mallow_unreachable\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 274\u001b[0m \u001b[43m \u001b[49m\u001b[43maccumulate_grad\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 275\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/.pyenv/versions/3.10.0/lib/python3.10/site-packages/torch/autograd/graph.py:744\u001b[0m, in \u001b[0;36m_engine_run_backward\u001b[0;34m(t_outputs, *args, **kwargs)\u001b[0m\n\u001b[1;32m 742\u001b[0m unregister_hooks \u001b[38;5;241m=\u001b[39m _register_logging_hooks_on_whole_graph(t_outputs)\n\u001b[1;32m 743\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 744\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mVariable\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_execution_engine\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun_backward\u001b[49m\u001b[43m(\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;66;43;03m# Calls into the C++ engine to run the backward pass\u001b[39;49;00m\n\u001b[1;32m 745\u001b[0m \u001b[43m \u001b[49m\u001b[43mt_outputs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\n\u001b[1;32m 746\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;66;03m# Calls into the C++ engine to run the backward pass\u001b[39;00m\n\u001b[1;32m 747\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[1;32m 748\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m attach_logging_hooks:\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m: " + ] + } + ], + "source": [ + "epochs_x, epochs_y, epochs_acc = training_loop(\n", + " device, \n", + " n_time_steps,\n", + " batch_size,\n", + " DVSGesture.sensor_size,\n", + " snn_train_dataloader, \n", + " snn, \n", + " loss_fn, \n", + " optimizer, \n", + " epochs, \n", + " snn_test_dataloader)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjIAAAGwCAYAAACzXI8XAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAAB4uElEQVR4nO3dd1hT1/8H8HcSIGErIkuG4MCNA0TQqlUErVWr/TmqrVatVot1te6qrVpRW61VW1fdto5+66qzTqzWhYJ74EQRN0NAVnJ+f2CuRFCIBiL4fj1PHuHmfu49gQhvzjn3XJkQQoCIiIioGJIbuwFEREREr4pBhoiIiIotBhkiIiIqthhkiIiIqNhikCEiIqJii0GGiIiIii0GGSIiIiq2TIzdgMKm0Whw+/ZtWFtbQyaTGbs5REREVABCCDx+/BguLi6Qy1/c71Lig8zt27fh5uZm7GYQERHRK7h58yZcXV1f+HyJDzLW1tYAsr8QNjY2Rm4NERERFURSUhLc3Nyk3+MvUuKDjHY4ycbGhkGGiIiomMlvWggn+xIREVGxxSBDRERExdZbGWTUajXGjh0LT09PmJubo0KFCpg4cSKevxH4+fPn0bZtW9ja2sLS0hJ+fn6IiYl54XEzMzMxYcIEVKhQASqVCj4+Pti+fXuu/WJjY/Hxxx+jTJkyMDc3R82aNREREWHw10lERFTSlfg5MnmZOnUq5s6di2XLlqF69eqIiIhAz549YWtri4EDBwIArly5gkaNGqF379747rvvYGNjg7Nnz0KlUr3wuN988w1WrlyJhQsXokqVKtixYwfat2+P//77D3Xq1AEAxMfHo2HDhnj33Xexbds2lC1bFtHR0ShdunSRvHYiIqKSRCae74YoYZKSkmBra4vExERpsu/7778PR0dHLFq0SNrvww8/hLm5OVauXAkA6NKlC0xNTbFixYoCn8vFxQVjxoxBaGjoC487cuRIHDx4EP/++68hXh4REVGJlNfv77y8lUNLgYGB2L17Ny5dugQAOHnyJA4cOIBWrVoByF5Eb8uWLahcuTJCQkLg4OAAf39/bNiw4aXHTU9Pz9VjY25ujgMHDkifb9q0Cb6+vujYsSMcHBxQp04dLFy40LAvkIiI6C3xVgaZkSNHokuXLqhSpQpMTU1Rp04dDB48GN26dQMA3Lt3D8nJyZgyZQpatmyJf/75B+3bt0eHDh0QHh7+wuOGhIRgxowZiI6Ohkajwc6dO7Fu3TrExcVJ+1y9ehVz585FpUqVsGPHDvTv3x8DBw7EsmXLCv11ExERlTiihEtMTBQARGJiorRt1apVwtXVVaxatUqcOnVKLF++XNjZ2YmlS5cKIYSIjY0VAMRHH32kc6w2bdqILl26vPBc9+7dE+3atRNyuVwoFApRuXJl8cUXXwiVSiXtY2pqKgICAnTqvvzyS9GgQQNDvFwiIqISIa/f33l5K3tkhg0bJvXK1KxZE5988gmGDBmCsLAwAIC9vT1MTExQrVo1nbqqVau+9KqlsmXLYsOGDUhJScGNGzdw4cIFWFlZwcvLS9rH2dlZ7+MSERFR3t7KIJOamprrBlQKhQIajQYAYGZmBj8/P1y8eFFnn0uXLsHDwyPf46tUKpQrVw5ZWVn466+/0K5dO+m5hg0bvvJxiYiISNdbefl1mzZt8P3338Pd3R3Vq1dHZGQkZsyYgV69ekn7DBs2DJ07d0bjxo3x7rvvYvv27fj777+xb98+aZ/u3bujXLlyUk/OkSNHEBsbi9q1ayM2NhbffvstNBoNhg8fLtUMGTIEgYGBmDx5Mjp16oSjR49iwYIFWLBgQZG9fiIiohKjiIa6jCavMbakpCQxaNAg4e7uLlQqlfDy8hJjxowR6enpOrWLFi0SFStWFCqVSvj4+IgNGzboPN+kSRPRo0cP6fN9+/aJqlWrCqVSKcqUKSM++eQTERsbm6tNf//9t6hRo4ZQKpWiSpUqYsGCBYZ90URERMVcQefIvJXryBhCXOITXHuQAk97SzjbmhvsuERERFTw399v5dDS61pzLAYj152GEIBMBkzpUBOd/dyN3SwiIqK3zls52fd1xCU+wainIQYAhABGrzuDuMQnxm0YERHRW4hBRk/XHqRA89xgnFoIXH+QapwGERERvcUYZPTkaW8JuUx3m0ImQ3l7C+M0iIiI6C3GIKMnZ1tzhHWoCdnTMCMDMLlDDU74JSIiMgIGmVfQ2c8do1pWAQAEVCjDib5ERERGwiDzilztsoeSstQl+up1IiKiNxqDzCuyUmZfuf44PcvILSEiInp7Mci8IitVdpBJTs80ckuIiIjeXgwyr8j6aY9Mchp7ZIiIiIyFQeYVPeuRyUIJv8sDERHRG4tB5hVp58hkqgXSszRGbg0REdHbiUHmFVmaPbtNVTIn/BIRERkFg8wrkstlUq8M58kQEREZB4PMa5CCDHtkiIiIjIJB5jVYP53w+5g9MkREREbBIPMacl65REREREWPQeY1PBta4qJ4RERExsAg8xq0Q0uc7EtERGQcDDKvgfdbIiIiMi4GmddgpTQFwB4ZIiIiY2GQeQ1WvGqJiIjIqBhkXoM115EhIiIyKgaZ18AeGSIiIuNikHkNvPyaiIjIuBhkXgMXxCMiIjIuowYZtVqNsWPHwtPTE+bm5qhQoQImTpwIIYS0jxAC48aNg7OzM8zNzREUFITo6GgjtvoZa940koiIyKiMGmSmTp2KuXPnYs6cOTh//jymTp2KadOmYfbs2dI+06ZNw6xZszBv3jwcOXIElpaWCAkJQVpamhFbno09MkRERMZlYsyT//fff2jXrh1at24NAChfvjxWrVqFo0ePAsjujZk5cya++eYbtGvXDgCwfPlyODo6YsOGDejSpYvR2g7kWBCPPTJERERGYdQemcDAQOzevRuXLl0CAJw8eRIHDhxAq1atAADXrl3DnTt3EBQUJNXY2trC398fhw4dyvOY6enpSEpK0nkUFuunC+KlZ2mQkaUptPMQERFR3ozaIzNy5EgkJSWhSpUqUCgUUKvV+P7779GtWzcAwJ07dwAAjo6OOnWOjo7Sc88LCwvDd999V7gNf8pSqZA+TknPgpmJWZGcl4iIiLIZtUdm7dq1+P333/HHH3/gxIkTWLZsGX788UcsW7bslY85atQoJCYmSo+bN28asMW6TBRymJtmhxnOkyEiIip6Ru2RGTZsGEaOHCnNdalZsyZu3LiBsLAw9OjRA05OTgCAu3fvwtnZWaq7e/cuateunecxlUollEplobddy1plgieZas6TISIiMgKj9sikpqZCLtdtgkKhgEaTPd/E09MTTk5O2L17t/R8UlISjhw5goCAgCJt64vwyiUiIiLjMWqPTJs2bfD999/D3d0d1atXR2RkJGbMmIFevXoBAGQyGQYPHoxJkyahUqVK8PT0xNixY+Hi4oIPPvjAmE2XWHN1XyIiIqMxapCZPXs2xo4diy+++AL37t2Di4sLPv/8c4wbN07aZ/jw4UhJSUHfvn2RkJCARo0aYfv27VCpVEZs+TO83xIREZHxyETOZXRLoKSkJNja2iIxMRE2NjYGP/7nKyKw4+xdfN++Brr5exj8+ERERG+jgv7+5r2WXpPV07VkeJsCIiKioscg85qsObRERERkNAwyr8lKyauWiIiIjIVB5jVxsi8REZHxMMi8Jitefk1ERGQ0DDKvyZoL4hERERkNg8xrknpkOLRERERU5BhkXpM2yDxmjwwREVGRY5B5TdK9ltgjQ0REVOQYZF6TtXZBPPbIEBERFTkGmdek7ZFJzVBDrSnRd3sgIiJ64zDIvCbtHBmAvTJERERFjUHmNZmZyKE0yf4yMsgQEREVLQYZA7DmhF8iIiKjYJAxAK7uS0REZBwMMgbA+y0REREZB4OMAfAO2ERERMbBIGMAVtq1ZNgjQ0REVKQYZAzAmkNLRERERsEgYwC83xIREZFxMMgYAO+3REREZBwMMgbAy6+JiIiMg0HGAKQF8Ti0REREVKQYZAxAmiPDoSUiIqIixSBjAFxHhoiIyDgYZAyAk32JiIiMg0HGAKy1C+KxR4aIiKhIMcgYAHtkiIiIjINBxgCkOTIZWdBohJFbQ0RE9PZgkDEA7eXXQgCpmWojt4aIiOjtwSBjAEoTOUwVMgAcXiIiIipKDDIGIJPJuLovERGRETDIGIgV74BNRERU5BhkDMSKl2ATEREVOQYZA7FW8hJsIiKiosYgYyAcWiIiIip6DDIGIt04kkNLRERERYZBxkC4ui8REVHRY5AxEGtefk1ERFTkGGQM5Nk6MuyRISIiKioMMgbCyb5ERERFj0HGQNgjQ0REVPQYZAzEmpN9iYiIihyDjIFwZV8iIqKixyBjIJwjQ0REVPQYZAyEc2SIiIiKHoOMgUhzZNKzIIQwcmuIiIjeDgwyBqINMmqNQFqmxsitISIiejswyBiIuakCcln2x4+5ui8REVGRYJAxEJlM9myeDCf8EhERFQkGGQOyVvESbCIioqLEIGNA7JEhIiIqWgwyBqRdSyaJQYaIiKhIMMgYENeSISIiKloMMgZkJd1viVctERERFQUGGQOyZo8MERFRkWKQMSDt0NJjBhkiIqIiwSBjQM+GlhhkiIiIigKDjAFxsi8REVHRYpAxIGv2yBARERUpBhkDslJmr+zLOTJERERFg0HGgDhHhoiIqGgxyBgQ58gQEREVLQYZA5LmyDDIEBERFQkGGQPiZF8iIqKixSBjQNqhpQy1BulZaiO3hoiIqORjkDEgSzMT6WP2yhARERU+BhkDkstlnPBLRERUhBhkDEy63xJ7ZIiIiAodg4yBadeSYZAhIiIqfEYPMrGxsfj4449RpkwZmJubo2bNmoiIiJCeF0Jg3LhxcHZ2hrm5OYKCghAdHW3EFr8ch5aIiIiKjlGDTHx8PBo2bAhTU1Ns27YN586dw/Tp01G6dGlpn2nTpmHWrFmYN28ejhw5AktLS4SEhCAtLc2ILX+xZ2vJZBq5JURERCWfSf67FJ6pU6fCzc0NS5YskbZ5enpKHwshMHPmTHzzzTdo164dAGD58uVwdHTEhg0b0KVLlyJvc36kHhkOLRERERU6o/bIbNq0Cb6+vujYsSMcHBxQp04dLFy4UHr+2rVruHPnDoKCgqRttra28Pf3x6FDh/I8Znp6OpKSknQeRUma7MuhJSIiokJn1CBz9epVzJ07F5UqVcKOHTvQv39/DBw4EMuWLQMA3LlzBwDg6OioU+fo6Cg997ywsDDY2tpKDzc3t8J9Ec/hjSOJiIiKjlGDjEajQd26dTF58mTUqVMHffv2RZ8+fTBv3rxXPuaoUaOQmJgoPW7evGnAFufPmpN9iYiIioxRg4yzszOqVaums61q1aqIiYkBADg5OQEA7t69q7PP3bt3peeep1QqYWNjo/MoSuyRISIiKjpGDTINGzbExYsXdbZdunQJHh4eALIn/jo5OWH37t3S80lJSThy5AgCAgKKtK0FZaU0BcA5MkREREXBqFctDRkyBIGBgZg8eTI6deqEo0ePYsGCBViwYAEAQCaTYfDgwZg0aRIqVaoET09PjB07Fi4uLvjggw+M2fQXYo8MERFR0TFqkPHz88P69esxatQoTJgwAZ6enpg5cya6desm7TN8+HCkpKSgb9++SEhIQKNGjbB9+3aoVCojtvzFOEeGiIio6MiEEMLYjShMSUlJsLW1RWJiYpHMlzl2/RE6zjsET3tL7P26aaGfj4iIqCQq6O9vo9+ioKSx5r2WiIiIigyDjIE9u9cSb1FARERU2BhkDMz66VVLaZkaZKo1Rm4NERFRycYgY2CWSoX0cQon/BIRERUqBhkDM1HIYW6aHWY4T4aIiKhwMcgUAitO+CUiIioSDDKFgGvJEBERFQ0GmUIgre7LK5eIiIgKFYNMIdBegs2hJSIiosKld5B58uQJUlNTpc9v3LiBmTNn4p9//jFow4ozKw4tERERFQm9g0y7du2wfPlyAEBCQgL8/f0xffp0tGvXDnPnzjV4A4sj3jiSiIioaOgdZE6cOIF33nkHAPC///0Pjo6OuHHjBpYvX45Zs2YZvIHFESf7EhERFQ29g0xqaiqsra0BAP/88w86dOgAuVyOBg0a4MaNGwZvYHHEy6+JiIiKht5BpmLFitiwYQNu3ryJHTt2IDg4GABw7969Irm7dHFg9fQ2BeyRISIiKlx6B5lx48bh66+/Rvny5eHv74+AgAAA2b0zderUMXgDiyPOkSEiIioaJvoW/N///R8aNWqEuLg4+Pj4SNubN2+O9u3bG7RxxRXnyBARERUNvYMMADg5OcHJyQkAkJSUhD179sDb2xtVqlQxaOOKK2kdGQYZIiKiQqX30FKnTp0wZ84cANlryvj6+qJTp06oVasW/vrrL4M3sDh6NrTElX2JiIgKk95BZv/+/dLl1+vXr4cQAgkJCZg1axYmTZpk8AYWR9YqDi0REREVBb2DTGJiIuzs7AAA27dvx4cffggLCwu0bt0a0dHRBm9gcWStvWqJk32JiIgKld5Bxs3NDYcOHUJKSgq2b98uXX4dHx8PlUpl8AYWR9qhpZQMNdQaYeTWEBERlVx6T/YdPHgwunXrBisrK3h4eKBp06YAsoecatasaej2FUuWSoX0cUpGFmxUpkZsDRERUcmld5D54osvUL9+fdy8eRMtWrSAXJ7dqePl5cU5Mk8pTRQwM5EjI0uDx2kMMkRERIXllS6/9vX1ha+vL4QQEEJAJpOhdevWhm5bsWatNMHDrAzOkyEiIipEes+RAYDly5ejZs2aMDc3h7m5OWrVqoUVK1YYum3FmnQJdjovwSYiIiosevfIzJgxA2PHjsWAAQPQsGFDAMCBAwfQr18/PHjwAEOGDDF4I4sjaVE89sgQEREVGr2DzOzZszF37lx0795d2ta2bVtUr14d3377LYPMU1a8TQEREVGh03toKS4uDoGBgbm2BwYGIi4uziCNKgmseeNIIiKiQqd3kKlYsSLWrl2ba/uaNWtQqVIlgzSqJGCPDBERUeHTe2jpu+++Q+fOnbF//35pjszBgwexe/fuPAPO20o72ZdzZIiIiAqP3j0yH374IY4cOQJ7e3ts2LABGzZsgL29PY4ePYr27dsXRhuLJSvtbQrYI0NERFRoXmkdmXr16mHlypWGbkuJwjkyREREha9AQSYpKanAB7SxsXnlxpQknCNDRERU+AoUZEqVKgWZTPbSfbQr/KrVaoM0rLiT1pFhkCEiIio0BQoye/fuLex2lDjSyr5pXNmXiIiosBQoyDRp0qSw21HiSHNk2CNDRERUaF7pXkuUP2vtVUuc7EtERFRoGGQKibSODHtkiIiICg2DTCHJedWSEMLIrSEiIiqZGGQKiXaOjBBASgav5CIiIioMrxRksrKysGvXLsyfPx+PHz8GANy+fRvJyckGbVxxpjSRw0Sefck658kQEREVDr1X9r1x4wZatmyJmJgYpKeno0WLFrC2tsbUqVORnp6OefPmFUY7ix2ZTAYrlQkSUjORnJ4JQGXsJhEREZU4evfIDBo0CL6+voiPj4e5ubm0vX379ti9e7dBG1fcSYvisUeGiIioUOjdI/Pvv//iv//+g5mZmc728uXLIzY21mANKwl4mwIiIqLCpXePjEajyfM2BLdu3YK1tbVBGlVS8MaRREREhUvvIBMcHIyZM2dKn8tkMiQnJ2P8+PF47733DNm2Yo/3WyIiIipceg8tTZ8+HSEhIahWrRrS0tLQtWtXREdHw97eHqtWrSqMNhZbViqu7ktERFSY9A4yrq6uOHnyJFavXo1Tp04hOTkZvXv3Rrdu3XQm/xLnyBARERU2vYMMAJiYmODjjz82dFtKHN44koiIqHDpHWQ2bdqU53aZTAaVSoWKFSvC09PztRtWEvDyayIiosKld5D54IMPIJPJct0/SLtNJpOhUaNG2LBhA0qXLm2whhZHHFoiIiIqXHpftbRz5074+flh586dSExMRGJiInbu3Al/f39s3rwZ+/fvx8OHD/H1118XRnuLFSvp8utMI7eEiIioZNK7R2bQoEFYsGABAgMDpW3NmzeHSqVC3759cfbsWcycORO9evUyaEOLIxvOkSEiIipUevfIXLlyBTY2Nrm229jY4OrVqwCASpUq4cGDB6/fumLOSpl9+TXnyBARERUOvYNMvXr1MGzYMNy/f1/adv/+fQwfPhx+fn4AgOjoaLi5uRmulcWUFXtkiIiICpXeQ0uLFi1Cu3bt4OrqKoWVmzdvwsvLCxs3bgQAJCcn45tvvjFsS4shTvYlIiIqXHoHGW9vb5w7dw7//PMPLl26JG1r0aIF5PLsDp4PPvjAoI0srrTryDxOy5Ku6CIiIiLDeaUF8eRyOVq2bImWLVsauj0lirZHRq0RSMvUwNxMYeQWERERlSyvFGRSUlIQHh6OmJgYZGRk6Dw3cOBAgzSsJLAwU0AmA4QAHqdnMsgQEREZmN5BJjIyEu+99x5SU1ORkpICOzs7PHjwABYWFnBwcGCQyUEmk8FKaYLHaVlITsuCg7WxW0RERFSy6H3V0pAhQ9CmTRvEx8fD3Nwchw8fxo0bN1CvXj38+OOPhdHGYs2aE36JiIgKjd5BJioqCl999RXkcjkUCgXS09Ph5uaGadOmYfTo0YXRxmLt2eq+DDJERESGpneQMTU1la5OcnBwQExMDADA1tYWN2/eNGzrSgDpxpHskSEiIjI4vefI1KlTB8eOHUOlSpXQpEkTjBs3Dg8ePMCKFStQo0aNwmhjsWalyl7dlz0yREREhqd3j8zkyZPh7OwMAPj+++9RunRp9O/fH/fv38eCBQsM3sDijnNkiIiICo9ePTJCCDg4OEg9Lw4ODti+fXuhNKyk4Oq+REREhUevHhkhBCpWrMi5MHqwyrG6LxERERmWXkFGLpejUqVKePjwYWG1p8R51iOTaeSWEBERlTx6z5GZMmUKhg0bhjNnzhRGe0oca15+TUREVGj0vmqpe/fuSE1NhY+PD8zMzGBubq7z/KNHjwzWuJJACjKcI0NERGRwegeZmTNnFkIzsnt6Ro0ahUGDBknnSEtLw1dffYXVq1cjPT0dISEh+PXXX+Ho6FgobSgMVsrsy685R4aIiMjw9A4yPXr0MHgjjh07hvnz56NWrVo624cMGYItW7bgzz//hK2tLQYMGIAOHTrg4MGDBm9DYbFijwwREVGh0XuODABcuXIF33zzDT766CPcu3cPALBt2zacPXtW72MlJyejW7duWLhwIUqXLi1tT0xMxKJFizBjxgw0a9YM9erVw5IlS/Dff//h8OHDr9Jso+Dl10RERIVH7yATHh6OmjVr4siRI1i3bh2Sk5MBACdPnsT48eP1bkBoaChat26NoKAgne3Hjx9HZmamzvYqVarA3d0dhw4deuHx0tPTkZSUpPMwJmtefk1ERFRo9A4yI0eOxKRJk7Bz506YmZlJ25s1a6Z3T8nq1atx4sQJhIWF5Xruzp07MDMzQ6lSpXS2Ozo64s6dOy88ZlhYGGxtbaWHm5ubXm0yNKlHhkGGiIjI4PQOMqdPn0b79u1zbXdwcMCDBw8KfJybN29i0KBB+P3336FSqfRtxguNGjUKiYmJ0sPYi/dp58hkqDVIz1IbtS1EREQljd5BplSpUoiLi8u1PTIyEuXKlSvwcY4fP4579+6hbt26MDExgYmJCcLDwzFr1iyYmJjA0dERGRkZSEhI0Km7e/cunJycXnhcpVIJGxsbnYcxWZo9m0/NXhkiIiLD0jvIdOnSBSNGjMCdO3cgk8mg0Whw8OBBfP311+jevXuBj9O8eXOcPn0aUVFR0sPX1xfdunWTPjY1NcXu3bulmosXLyImJgYBAQH6NttoFHIZLM0UADjhl4iIyND0vvx68uTJCA0NhZubG9RqNapVqwa1Wo2uXbvim2++KfBxrK2tpZtPallaWqJMmTLS9t69e2Po0KGws7ODjY0NvvzySwQEBKBBgwb6NtuorFQmSMlQc8IvERGRgekdZMzMzLBw4UKMHTsWZ86cQXJyMurUqYNKlSoZvHE//fQT5HI5PvzwQ50F8YobK6UJ7iKdPTJEREQGJhNCCH0KDhw4gEaNGhVWewwuKSkJtra2SExMNNp8mXa/HMTJmwn4rbsvgqoVn1WJiYiIjKWgv7/1niPTrFkzeHp6YvTo0Th37txrNfJtYc1F8YiIiAqF3kHm9u3b+OqrrxAeHo4aNWqgdu3a+OGHH3Dr1q3CaF+JoF1L5jGDDBERkUHpHWTs7e0xYMAAHDx4EFeuXEHHjh2xbNkylC9fHs2aNSuMNhZ70v2WONmXiIjIoF7pXktanp6eGDlyJKZMmYKaNWsiPDzcUO0qUZ7dbynTyC0hIiIqWV45yBw8eBBffPEFnJ2d0bVrV9SoUQNbtmwxZNtKDGv2yBARERUKvS+/HjVqFFavXo3bt2+jRYsW+Pnnn9GuXTtYWFgURvtKBM6RISIiKhx6B5n9+/dj2LBh6NSpE+zt7QujTSWOtcoUAHtkiIiIDE3vIHPw4MHCaEeJJk32ZY8MERGRQekdZLTOnTuHmJgYZGRk6Gxv27btazeqpOE6MkRERIVD7yBz9epVtG/fHqdPn4ZMJoN2YWCZTAYAUKvVhm1hCaDtkeG9loiIiAxL76uWBg0aBE9PT9y7dw8WFhY4e/Ys9u/fD19fX+zbt68Qmlj8SZN9GWSIiIgMSu8emUOHDmHPnj2wt7eHXC6HXC5Ho0aNEBYWhoEDByIyMrIw2lmscR0ZIiKiwqF3j4xarYa1tTWA7FV+b9++DQDw8PDAxYsXDdu6EkK7jkxapgaZao2RW0NERFRy6N0jU6NGDZw8eRKenp7w9/fHtGnTYGZmhgULFsDLy6sw2ljsWSqffZlT0rNQysLMiK0hIiIqOfQOMt988w1SUlIAABMmTMD777+Pd955B2XKlMGaNWsM3sCSwFQhh8pUjrRMDR6nMcgQEREZit5BJiQkRPq4YsWKuHDhAh49eoTSpUtLVy5RblZKU6RlpvMSbCIiIgN6rZtGatnZ2THE5MOai+IREREZnEGCDOVPunKJl2ATEREZDINMEeGNI4mIiAyPQaaISPdbYo8MERGRwTDIFBFrLopHRERkcAwyRYQ9MkRERIbHIFNEOEeGiIjI8Bhkioi1yhQAe2SIiIgMiUGmiFhxHRkiIiKDY5ApIs8m+zLIEBERGQqDTBHRzpFJ4tASERGRwTDIFJFnVy3x8msiIiJDYZApIlYcWiIiIjI4BpkiYs11ZIiIiAyOQaaIaHtkUjLUUGuEkVtDRERUMjDIFBHtHBkASMlgrwwREZEhMMgUEaWJAmaK7C83h5eIiIgMg0GmCHFRPCIiIsNikClC0v2W2CNDRERkEAwyRYiXYBMRERkWg0wRsuIl2ERERAbFIFOEnt1viav7EhERGQKDTBHS9shwjgwREZFhMMgUIc6RISIiMiwGmSJkrTIFwDkyREREhsIgU4SsuY4MERGRQTHIFCFpHRkGGSIiIoNgkClCXBCPiIjIsBhkitCzdWR4+TUREZEhMMgUIWtetURERGRQDDJFiCv7EhERGRaDTBHiZF8iIiLDYpApQlY5Lr8WQhi5NURERMUfg0wRslZmL4gnBJCaoTZya4iIiIo/BpkipDKVQyGXAeCEXyIiIkNgkClCMpmMa8kQEREZEINMEeONI4mIiAyHQaaIWfMSbCIiIoNhkCliz3pkuLovERHR62KQKWLaS7A5R4aIiOj1McgUMc6RISIiMhwGmSJmrcpeS4ZzZIiIiF4fg0wRs1axR4aIiMhQGGSKGO+3REREZDgMMkWMC+IREREZDoNMEZNuHJnGy6+JiIheF4NMEbPmVUtEREQGwyBTxLiODBERkeEwyBQxriNDRERkOAwyRYyXXxMRERkOg0wRs1I+WxBPCGHk1hARERVvDDJFTDtHJksjkJ6lMXJriIiIijcGmSJmYaqATJb9MSf8EhERvR4GmSIml8tgZcZ5MkRERIZg1CATFhYGPz8/WFtbw8HBAR988AEuXryos09aWhpCQ0NRpkwZWFlZ4cMPP8Tdu3eN1GLDeLYoHoMMERHR6zBqkAkPD0doaCgOHz6MnTt3IjMzE8HBwUhJSZH2GTJkCP7++2/8+eefCA8Px+3bt9GhQwcjtvr1PbvfElf3JSIieh0mxjz59u3bdT5funQpHBwccPz4cTRu3BiJiYlYtGgR/vjjDzRr1gwAsGTJElStWhWHDx9GgwYNjNHs18YeGSIiIsN4o+bIJCYmAgDs7OwAAMePH0dmZiaCgoKkfapUqQJ3d3ccOnQoz2Okp6cjKSlJ5/Gm4aJ4REREhvHGBBmNRoPBgwejYcOGqFGjBgDgzp07MDMzQ6lSpXT2dXR0xJ07d/I8TlhYGGxtbaWHm5ubwdu6f/9+tGnTBi4uLpDJZNiwYYPO8zKZLM/HDz/8AODFi+L98ssvKF++PFQqFfz9/XH06FGd5z///HNUqFAB5ubmKFu2LNq1a4cLFy4Y/PUREREVF29MkAkNDcWZM2ewevXq1zrOqFGjkJiYKD1u3rxpoBY+k5KSAh8fH/zyyy95Ph8XF6fzWLx4MWQyGT788EMAgPXTRfFyXn69Zs0aDB06FOPHj8eJEyfg4+ODkJAQ3Lt3T9qnXr16WLJkCc6fP48dO3ZACIHg4GCo1WqDv0YiIqLiwKhzZLQGDBiAzZs3Y//+/XB1dZW2Ozk5ISMjAwkJCTq9Mnfv3oWTk1Oex1IqlVAqlYXa3latWqFVq1YvfP75tm3cuBHvvvsuvLy8AOSYI5OjR2bGjBno06cPevbsCQCYN28etmzZgsWLF2PkyJEAgL59+0r7ly9fHpMmTYKPjw+uX7+OChUqGObFERERFSNG7ZERQmDAgAFYv3499uzZA09PT53n69WrB1NTU+zevVvadvHiRcTExCAgIKCom/tK7t69iy1btqB3797SNumqpbTsq5YyMjJw/PhxnblAcrkcQUFBL5wLlJKSgiVLlsDT07NQhs+IiIiKA6P2yISGhuKPP/7Axo0bYW1tLc17sbW1hbm5OWxtbdG7d28MHToUdnZ2sLGxwZdffomAgIBic8XSsmXLYG1trXPJuPVzVy09ePAAarUajo6OOrWOjo655sD8+uuvGD58OFJSUuDt7Y2dO3fCzMyskF8FERHRm8moPTJz585FYmIimjZtCmdnZ+mxZs0aaZ+ffvoJ77//Pj788EM0btwYTk5OWLdunRFbrZ/FixejW7duUKlU0rbXuWqpW7duiIyMRHh4OCpXroxOnTohLS3NYO0lIiIqTozaI1OQuz+rVCr88ssvL5xY+yb7999/cfHiRZ1gBjybI6Od7Gtvbw+FQpFrxeK85gJpr8aqVKkSGjRogNKlS2P9+vX46KOPCvGVEBERvZnemKuWSqJFixahXr168PHx0dn+fI+MmZkZ6tWrpzMXSKPRYPfu3S+dCySEgBAC6enphdB6IiKiN98bcdVScZOcnIzLly9Ln1+7dg1RUVGws7ODu7s7ACApKQl//vknpk+fnqveWmWCu6tHw6R2E2DgOwCAoUOHokePHvD19UX9+vUxc+ZMpKSkSFcxXb16FWvWrEFwcDDKli2LW7duYcqUKTA3N8d7771XBK+aiIjozcMg8woiIiLw7rvvSp8PHToUANCjRw8sXboUALB69WoIIfIc8rFSmiIz/g4eJzxCXOITONuao3Pnzrh//z7GjRuHO3fuoHbt2ti+fbs0AVilUuHff//FzJkzER8fD0dHRzRu3Bj//fcfHBwcCv9FExERvYFkoiATVYqxpKQk2NraIjExETY2NsZuDgBgXvgVTNmWfTWSXAaEdaiJzn7uRm4VERHRm6Ogv785R6aIxSU+wbTtzy6p1ghg9LoziEt8YsRWERERFU8MMkXs2oMUaJ7rA1MLgesPUo3TICIiomKMQaaIedpbQi7T3aaQyVDe3sI4DSIiIirGGGSKmLOtOcI61NQJM8NbesPZ1tx4jSIiIiqmGGSMoLOfOw6ObIaqTtYAsoeWiIiISH8MMkbibGuOng2zb5K57kRsgVY5JiIiIl0MMkbUqqYTlCZyXL6XjDOxScZuDhERUbHDIGNE1ipTtKiWveDdushbRm4NERFR8cMgY2Qd6pYDAPx98jYy1Rojt4aIiKh4YZAxsncqlYW9lRkeJGfg3+j7xm4OERFRscIgY2SmCjna+LgAyJ70S0RERAXHIPMG6FDHFQCw89xdJKVlGrk1RERExQeDzBugRjkbVHSwQnqWBttP3zF2c4iIiIoNBpk3gEwmkyb9/nWCVy8REREVFIPMG+KD2uUgkwFHrj3CrXjeQJKIiKggGGTeEC6lzNHAswwAYGPUbSO3hoiIqHhgkHmDtH86vLTuxC3esoCIiKgAGGTeIK1qOEFlKseV+yk4dSvR2M0hIiJ64zHIvEGsVaYIruYEAFgfyTVliIiI8sMg84bRDi9t4i0LiIiI8sUg84Z5p6I97K2UeJSSgfCLvGUBERHRyzDIvGFMFHK0q519ywIOLxEREb0cg8wbqH2d7OGlnefvIvEJb1lARET0Igwyb6DqLjao7GiFjCwNtp2OM3ZziIiI3lgMMm8gmUyG9k9vJMk7YhMREb0Yg8wb6oM6LpDJgKPXH+HmI96ygIiIKC8MMm8oZ1tzBFbIvmXBBk76JSIiyhODzBtMO7y0PjKWtywgIiLKA4PMG6zl01sWXH2QgqibCcZuDhER0RuHQeYNZqU0QcvqvGUBERHRizDIvOHa180eXvr75G1kZPGWBURERDkxyLwBYmNj8fHHH6NMmTIwNzdHzZo1ERERAQBoWKEMylorEZ+aifBLuW9ZcPDgQZiYmKB27do62x8/fozBgwfDw8MD5ubmCAwMxLFjx4ri5RARERUZBhkji4+PR8OGDWFqaopt27bh3LlzmD59OkqXLg3g6S0LfLJvWbDuxC2d2oSEBHTv3h3NmzfPddzPPvsMO3fuxIoVK3D69GkEBwcjKCgIsbEcoiIiopJDJkr45TBJSUmwtbVFYmIibGxsjN2cXEaOHImDBw/i33//feE+524n4b1Z/8JMIcexMUGwtTAFAHTp0gWVKlWCQqHAhg0bEBUVBQB48uQJrK2tsXHjRrRu3Vo6Tr169dCqVStMmjSpUF8TERHR6yro72/2yBjZpk2b4Ovri44dO8LBwQF16tTBwoULdfap5mKDKk7WyFBrsOXpLQuWLFmCq1evYvz48bmOmZWVBbVaDZVKpbPd3NwcBw4cKLwXQ0REVMQYZIzs6tWrmDt3LipVqoQdO3agf//+GDhwIJYtW6azn/ZGkutO3EJ0dDRGjhyJlStXwsTEJNcxra2tERAQgIkTJ+L27dtQq9VYuXIlDh06hLg43ruJiIhKDgYZI9NoNKhbty4mT56MOnXqoG/fvujTpw/mzZuns1+72uUgkwHHrj3A/3Xqgu+++w6VK1d+4XFXrFgBIQTKlSsHpVKJWbNm4aOPPoJczm85ERGVHPytZmTOzs6oVq2azraqVasiJiZGZ5uTrQqNKtpDZDzBqagTGDBgAExMTGBiYoIJEybg5MmTMDExwZ49ewAAFSpUQHh4OJKTk3Hz5k0cPXoUmZmZ8PLyKrLXRkREVNgYZIysYcOGuHjxos62S5cuwcPDI9e+7euUg0xpAd+hixAZGYmoqChERUWhX79+8Pb2RlRUFPz9/XVqLC0t4ezsjPj4eOzYsQPt2rUr1NdDRERUlHJPsKAiNWTIEAQGBmLy5Mno1KkTjh49igULFmDBggXSPqNGjUJsbCzmLlwMCzNT3Jc5IsPGFfU8si/RdnBwgEqlQo0aNaSaHTt2QAgBb29vXL58GcOGDUOVKlXQs2fPIn+NREREhYU9Mkbm5+eH9evXY9WqVahRowYmTpyImTNnolu3btI+cXFxiImJgaXSBC1rZN+yYOXh6/jvygPEJT7J87iJiYkIDQ1FlSpV0L17dzRq1Ag7duyAqalpkbwuIiKiosB1ZIqZf6Pv45NFR6XP5TIgrENNdPZzN2KriIiIDIvryJRQnvaWOp9rBDB63ZkX9swQERGVZAwyxUzMo1Sdzx9HbsXNRaGoWM4BNjY2CAgIwLZt215Y37RpU8hkslyPnCsAf/vtt6hSpQosLS1RunRpBAUF4ciRI4X2moiIiF4Vg0wx42lvCbns2ecK6zIo3aQH1mzbh4iICDRr1gzt2rXD2bNn86xft24d4uLipMeZM2egUCjQsWNHaZ/KlStjzpw5OH36NA4cOIDy5csjODgY9+/nvmklERGRMXGOTDG05lgMRq87DXWO71wZSzP82NEH71ZxgJ2dHX744Qf07t0732PNnDkT48aNQ1xcHCwtLfPcR/s13LVrV543qCQiIjI0zpEpwTr7uePAyGZY1acB/vjMH1WcrPEwJQOfLj6M/xv2A1JSUhAQEJDvccLCwjB69GikpaXB09MTH3zwQa41bTIyMrBgwQLY2trCx8cHQHb48fb2hrm5Odzc3DBkyBCkpaVJNXPnzkWtWrVgY2NToOEuIiKiV8V1ZIopZ1tzONuaAwAmNbZBYMOWyExPx00zc9TpMRGmZdzyPcamTZvw5MkTrFq1ClWrVsXo0aMRHByMc+fOYe/evejSpQtSU1Ph7OyMnTt3wt7eHn/88QdGjhyJxYsXIzAwEJcuXcKnn34KmUyGGTNmAABcXV0xZcoUVKpUCUIILFu2DO3atUNkZCSqV69eqF8XIiJ6y4gSLjExUQAQiYmJxm5KoUlPTxfR0dFi7v92CMd3Ogu5uY0o//k8sfLwdaHRaF5Y17dvX1GzZk3p83v37gkAIjw8XCQnJ4vo6Ghx6NAh0atXL1G+fHlx9+5dERoaKpo1ayaEEGLy5MnC19dXmJqaClNTU9GuXTtx4cKFPM9VunRp8dtvv4m//vpL1KtXT9ja2goLCwvh4+Mjli9f/sI2fv755wKA+Omnn17ti0NERMVSQX9/c2ipBDAzM0PFihXR78NgnNqyFA7lK+PB4Q0Ys/4MPl9xHPEpGblqUlJSsHr1ap15NImJiQAAOzs7WFpaomLFimjQoAEWLVoEExMTLFq0CIGBgTh+/DiOHj2K8PBwdOzYEa6urvj888+RmZmJ4OBgpKSkSMdUq9VYvXq1NNxlZ2eHMWPGoGfPnqhQoQIuXLiA7t27IzAwMNew1vr163H48GG4uLgAAM6ePYsPP/wQ5cuXh0wmw8yZM3O9rrCwMPj5+cHa2hoODg55DpcREVHJwSBTwjhYq1DVyRo1nS1gqpDhn3N30ernf/HflQc6+/35559IT0/Hxx9/DCD7LtyDBw9Gw4YNdW51oKXRaJCeno6uXbtiwoQJaNSoEXbv3o0RI0YgJCQEs2fPxtKlSxETE4Pjx4/j9OnTsLKyglKpRL9+/bB+/XpUq1YNTZs2Rfv27XH+/HkMHToUx48fh7e3N+7fv68TgmJjY/Hll1/i999/l1YjTk1NhZeXF6ZMmQI7OzssXLgQLi4ukMlk2LBhAwAgPDwcoaGhOHz4MHbu3KkTrn755RdUrVpVmttTp06dXPUJCQkIDQ2Fs7MzlEolKleujK1btwIAfv/9d/j4+MDCwgLOzs7o1asXHj58mOf3YfXq1ZDJZPjggw9e91tKREQvU0Q9REZT0oeWRo4cKcLDw8W1a9fEqVOnxMiRI4VMJhP//POPOH0rQTjWayFsGvyfKD9ys5i67bzIyFILIYRo1KiR6Ny5s3Scfv36CQ8PD3Hx4kUxatQocejQIXH9+nUREREhevbsKZRKpThz5ozYu3evcHR0FAsXLhSnTp0S69atE25ubmLChAkiOjpaABCnT5+WhrsiIiLEyJEjhb29vTh79qxO2zUajdi1a5ewsLAQa9eulYa11Gq1ePfdd8XMmTOFEEJ4eHjkGlpycHAQLVq0EOvWrRMAxPr16/P8+miHy4YMGSKsra3F6tWrxZUrV8SIESOEqampGDVqlFSfnp4ufH19xXvvvScOHDggrl27Jvbt2yeioqLEgQMHhFwuFz///LO4evWqmD17trCyshIqlSrX+a9duybKlSsn3nnnHdGuXTshhBA9evQQAPJ8aGvHjx+f6zlvb28hhBBpaWli9OjRwt3dXZiYmAhzc3Nha2sr1Z85c0Z06NBBeHh45BqK27t3b57njYuL0/la3bp1S3Tr1k3Y2dkJlUolatSoIY4dO1aQtyERkcEV9Pc3g0wx16tXL+Hh4SHMzMxE2bJlRfPmzcU///wjPf9O48aiRpO2wmPEZuExYrNoO+eA+HPXIQFArFr/txBCiNDQUOHq6iquXr0qnjx5Itq3by9cXFyEmZmZcHZ2Fm3bthVHjx4VQmQHoK+//lqnDStWrBAqlUq89957omHDhnm2s3nz5qJv375CCCESEhKEpaWlMDExEUqlUixatEgnBE2ePFm0aNFCmt+TV5DJue1lQUZ7XB8fn1ztHjp0qGjYsKFUP3fuXOHl5SUyMjJyHeeHH34QXl5e0udbt24VwcHBws7OTuf8WVlZIjAwUPz222+iR48eUpBJSEgQcXFxIi4uTqxcuVJ8+eWXwsrKKleQqV69urRfXFycuH//vhBCiLZt2wp/f3+xc+dOsWTJEtGjRw/x/fffS/VHjx4VX3/9tVi1apVwcnLKM8hcvHhRrFu3TrRo0UI4OjrqnPvRo0fCw8NDfPrpp+LIkSPi6tWrYseOHWLlypV5hqDn6+fMmSM8PDyEUqkU9evXF0eOHMn1NZwzZ44AIExMTHRqw8PDxfvvvy+cnZ1f+L3UBkkbG5tc+73o3H/99ZcICgoS9vb2wtraWjRo0EBs375d57hJSUli0KBBwt3dXahUKhEQECC914nIuDhH5i2xaNEiXL9+Henp6bh37x527dqFFi1aSM/vDw/H6X0b8UvXurBRmeDkzQR8vfMhPEZsxqjDQEinHli/fj327NkDT09PqFQqrFu3DrGxsUhPT8ft27exceNG+Pn5Acge3pHLdd82CoUCmZmZOHv2LFavXp1nO7VDUwBgbW2NqKgoHDt2DN9//z2GDBmCTz75BA0bNkR6ejp+/vlnLF26FDKZLM9jFVTO4TKFQgGVSqXzvLm5OY4efXbfqk2bNiEgIAChoaFwdHREjRo1MHnyZKjVagQEBODmzZvYunUrhBCoW7cu0tLS8OGHH+occ8KECXBwcMi1ho+trS2cnJzg5OSEbt26oVmzZjpzibRMTEyk/ZycnGBvb4/t27cjPDwcW7duRVBQED799FMsXboUo0ePlur8/Pzwww8/oEuXLlAqlXl+PbR3Sa9fvz7mzp2r89zUqVPh5uaGJUuWoH79+vD09ERwcDDKlSsHALh48SLi4uKwcuVKDBo0CL/88otUu2bNGgwdOhTjx4/HiRMn4OPjg5CQENy7d0/aJyEhAZMmTYKnpydcXV11zp2SkgIfHx+dY+aUkJCA7t27w8fHByqVqsDn3r9/P1q0aIGtW7di3rx5ePDgAVq2bKkzlPjZZ59h586dWLFiBU6fPo3g4GAEBQVh4cKFaNGiBcqWLQsbGxtUr14dDRo0yDUUqTVlyhTIZDIMHjwYQPaNXrt27YrKlStDJpPBy8srV+2LVtlu3ry5VCuXy9GxY0e0adNGp37hwoV45513ULp0aWn17Zzv5X379qFu3bpQKpWoWLEili5dmuvrGhsbi48//hhlypSBubk5atasiYiIiDy/B0RvtKLJVcZT0ntk9HHixiOpZ8ZjxGZhVec9IVNaivdHzROz/j4qwqMuiVuxt0VqaqpU88knn4iRI0dKn48fP15YW1uLVatWiatXr4p//vlH2NjYCHNzc3H16lUhxMuHu/JSpUoVoVKpxM2bN8VPP/0kZDKZUCgU0gOAkMvlwsPDQ6opSI+Mdrjs5s2bYtSoUcLJyUlEREQIjUYjjh07JvUqaOu9vb2FUqkUvXr1EhEREWL16tXCzs5OfPvtt0IIIdauXSusrKykHoU2bdqIjIwMqf7ff/8V5cqVk3pRcvbIPO/9998XLVq0yNUjY2FhIZydnYWnp6fo2rWruHHjhujfv79o3ry5GDFihHBxcRGVKlUSX331lUhNTc3ztT/fg6XtkfHw8BBOTk4iKChIHDhwQKe2atWqYvDgweL//u//RNmyZUXt2rXFggULpNr4+Phcr0FbX79+fREaGiptV6vVwsXFRYSFhUnbOnfuLL755hsxfvx44ePj88LvWV7bn6/V99xCZPegjRkzRri6ukq1qampQqFQiM2bN+vsW7duXeHn5yemTp0qjh49Ki5duiQ6deok5HK5+PHHH3O18ejRo6J8+fKiVq1aYtCgQUKI7OHFgQMHimXLlgkvLy/h5+eXaxj04cOHOr1vZ86cEQqFQvzwww9Sbe3atUW7du3EmDFjdOq7du0qfvnlFxEZGSnOnz8vPv30U2Fraytu3bolrl69KiwsLMTQoUPFuXPnxKBBgwQAnd7DF/XAXb58WXrP1KlTR5iZmQl7e3tRpkyZXD1eefWEZWRkiO+++054eXkJpVIpatWqJb744gud/T777LMXDqEWpH7evHkv7cH76aefROXKlYVKpRKlSpUS1tbWudr+omHUnLWurq5i8ODB4smTJ9KxC9J7SIbDoaWnGGSeOXj5vk6Qef6HifYR2HOsmL37kjh05YF4p3Fj0aNHD+kYmZmZ4quRY0Q59/JCqVQKKysrYWlpKSIiIqR98hvuyik0NFRYWFiI+vXrCyGEePDggTh9+rTOw8XFRYwYMULn0u78gkzO4TIhhEhNTRU9e/YUJiYmQqFQCBcXFzF8+HCdIFOpUiXh5uYmsrKypONMnz5dODk5ibNnzwpnZ2cxbdo0cfLkSbF9+3ZRs2ZN0atXLwFA/PHHH6J8+fJi69atUu2LgkxsbKxQKBRizZo1Om3funWrWLt2rXT8gIAA4e7uLpo3by6USqVo3bq1OHLkiNiyZYv0S6ggQebChQti3rx5IiIiQhw8eFD6OuSsVSqVQqlUilGjRokTJ06I+fPnC5VKJUaMGJFnCNJ+3deuXSsUCkWuNnTv3l20bdtWCCHE4sWLhZ+fn8jMzNQ7yORVq8+5c1Kr1cLNzU06R1JSkgAgdu3apbNfw4YNRZMmTXLVV6tWTXz33Xc6bXz8+LGoVKmS2Llzp2jSpIkUZHLKuf1lv/x++uknYW1tLZKTk/OsfVl9VlaWsLa2FsuWLRPDhw8X1atXl57bunWrqFatmqhdu7ZUP2LECNGoUaM825EzCE2fPl2YmJgImUwm5s+fL/r06SNKlSol5s+fL8zMzMTixYvF2bNnpe0DBgwQLi4uYsuWLeLKlSvS/4/x48dL+6lUKuHt7Z3nEOrw4cPzrbeyshKDBw/Oc37c77//LpRKpfj999/F7NmzhampqbC1tRXdu3eX2njx4sU8Q9yMGTOk2mvXrokdO3YIGxsbYWNjIwWhn376KVeozPm9yuvnqVwul0JUXnPlQkJCXlhrbm4ulEql8PPzE5988omoUaOG9MfOJ598ImJjY6Xzx8fHiy+++EI4OTkJhUIhTExMhKmpaYECXM5a7c9tBweHPIeKw8LCBADpfblkyZJc7VYqlTrvqRf9rpk2bVqe70Gtgv7+5oJ4bxHtfZo0IvtzjxGbIZcBPRuWx8U7yYiMiUdKhhqxAH785xIAwKThCNiVs8X3W86hnocdbsWnYh0CYPJRAMz++RXq6APYumUTypUrhzt37gAA5syZA3Pz7MX6unfvjnLlyknDXWFhYfD19YWXlxfGjRuHzZs3Iz09HX369AEAlClTBmXKlNFpt6mpKZycnODt7Z3vaxRC4Msvv8T69euxb98+eHp6AsgeRlq8eDHmz5+Pu3fvwtnZGQsWLIC1tTUeP34MAHB2doapqSkUCoV0vKpVq+LOnTv4/vvv0bBhQwwbNgwAUKtWLVhaWuKdd94BANy5cwfXr19HmzZtpFqNRpP9NTQxwcWLF1GhQgUAwLJly1CqVKlcVzS1atVK+rhWrVrw9/eHh4cH4uLiIJPJ8Pvvv8PW1hYAMGPGDPzf//1fvl8PAPD29tb52gUGBuLKlSvYv3+/Tlt9fX0xefJkAECdOnVw5swZbN++HfPmzYOvry/S09Px22+/oWnTptJNRB8/fgy1Wg1HR0edczo6OuLChQuIjo7GyJEj8e+//8LERL8fN/nV5nfu5/34449ITk6WPre2tkZAQAAmTpyIqlWrwtHREatWrcKhQ4dQsWJFnVqNRoPHjx/Dzs5OZ3toaChat26NoKAgTJo0Sa/X97xFixahS5cuL7xVyMukpqYiMzMTdnZ2OHToEIKCgqTnWrVqhTt37kjDXkD2MGpISAg6duyI8PBwlCtXDl988QX69OmDefPmwdPTE9OnT4e/vz8+//xzPHjwAOvWrcPWrVuxZcsWTJw4EX369EHPnj0BAPPmzcOWLVuwbNkyhIWF4b333gMAnDlzBl5eXrh8+TKqVauGefPmYfXq1UhMTISTk1Ou17FixQqMGTPmpfVbtmyBo6Mj2rdvn6v+v//+Q8OGDdG1a1f4+/ujb9++UCqVOHLkCPbv348tW7agb9++0jCqlqenJzZt2iTVAsCRI0eQkpICT09PbNy4ETNnzsR3332HixcvwsHBIde5161bh4yM7KUuNm7ciC+//BJZWVmYMGECbty4gZCQEAQHB6Nly5Y651YqlRBCSLUAsHLlSgwbNgwfffQRvvrqK0ybNg2///47fv31VzRu3Bjx8fEYNGgQ2rZti4iICGRkZKBFixZwcHBA//79pSH7+vXrY/v27QgJCcGRI0cQHByMd999F9u2bUPZsmURHR0NS0tLqfZ///sfIiMjMXToUIwZMwYdO3bEzJkzERISgosXL+LGjRuYP38+atWqpfPabWxsdJa5eH5aQFxcnM7n27ZtQ+/evXMNzb8qzpF5izjbmiOsQ00onr7JFDIZwjrUxNj3q2PlZ/44OT4Ym79shG/bVEPrWs5wtFEiSyMQdTMBC/+9hn4rj2PSlvNSEHocuRWpyUlo2rQpnJ2dpceaNWukc8bExOi8ie8+SkCvPp/D29sbq1atgqurK3755Re8//77uHPnDp48eSLt2717d4waNUr6PCMjA1FRUYiKikJGRgZiY2MRFRWl8xpDQ0OxcuVK/PHHH7C2tsadO3d0jmtqagpXV1coFAqsXr0a77//vlTbsGFDXL58WQogAHDp0iU4OzsjLS0tz7lBWuXKlcPp06el9kVFRaFt27Z49913ERUVBTe37JWWhRBYvHgxPvnkE5iZmb30+1WqVClpnkS5cuWkEANkByzxGrdJq1+/vs7nzs7OqFatms62qlWr4v79+/j8889Rr149BAYGSis6//TTT/meQwiBrl274rvvvkPlypX1ap9arX7l2rz88ccf+O6777B27Vqd7StWrIAQAuXKlYNSqcSsWbPw0Ucf5fpea0NQp06dpG2rV6/GiRMnEBYW9trtO3r0KM6cOYPPPvvslepHjBgBFxcXBAUF4c6dO3mGu6SkJOnzq1evYu7cuahUqRJ27NiB/v37Y+DAgVi2bJkUhDIyMnD8+HEEBQUhJCQEhw4dglwuR7NmzXDr1i2dsCSXyxEUFIS0tDRpLpq23tXVFQcOHJD28/Lywv379+Hi4gIvLy9069YNMTExAID09PR864OCgnDo0KE8vw7ada4OHjyI48ePo0aNGti6dSvee+89qfb48ePw9fVFx44d4eDggDp16mDhwoU6a2QB2X90WVtbo2fPnlKIsrCwwOLFi/M8t52dnTS3bfHixfD394eVlRUGDx4s1V6+fBlKpVJnHlzp0qV1ap2cnDB79myYmppi1qxZqFatGhYvXgwHBwc8fPgQ3t7eaNCgAebMmYPjx48jJiYGixcvxqNHj7BhwwZs2bIFffr0wZQpU9ChQwfp3DkDXM55cOHh4VJtw4YNsWLFCvTt2xfjx4/Xed1z585Ft27dsHDhQpQuXVrntctkMp32P//+y/mck5MTNm7ciHfffRdeXl4veVcXHHtk3jKd/dzRuHJZXH+QivL2FtJtDgDARCFHjXK2qFHOFp829IQQArfinyDixiMcux6Pfy/dx834Z0HDY8TmXMcvbWGKvx5b4Ojvx+FW2gK9w5bC1c4Cl+89xuGrj7DJpDEUXRpDPTU7QJw7dw79+vVDv379AABLlizBp59+CiA7BMnlcly/fh0AcP36ddSpU0c6148//ogff/wRAHDt2jVERUVJk1ibNm2q066wsDC4urrC398f8fHxmDp1KqKiojBixAisWrUK165dQ+PGjTFr1iwMGjQIGRkZiI6OxtmzZzFw4ECUK1cOffr0wdy5cxESEoK4uDgMHjwY9evXx9GjR2FmZpZr/Z1SpUoBgM728PBwXL58uUA39ExOTsaVK1cQHByMTZs2ITk5GVZWVgCyA5ZcLtcJXfp4PgA2bNgw18KBly5dgoeHR67a+vXrS79YrK2toVAocPfuXZ197t69C3t7e2zbtg2RkZEYMGAAgOyeDW0AO3Xq1AvX2Xn8+DEiIiLyrNX2zrzs3Dn/2l+9ejU+++wz/Pnnnzq/fAGgQoUKCA8PR0pKCpKSkuDs7IzOnTvr/IDVhqCNGzdKf4k/ePAAY8aMwc6dO3NNIn8VixYtQs2aNXMFzIKYMmUKVq9ejX379hW4LS/qgZs3bx4ePnwIR0dHPHjwQOrxMjMzQ1JSEp48eQJra2sAyDMsWVtbY8aMGWjcuDHMzc2hVqtx+PBhnf0qVqyIlJQU/PXXX4iLi8N3332Hd955B2fOnEFISEi+9S/qcQOArl274sGDB2jatCnUajX69++Pfv36SRPjHR0dkZKSgrlz52Lo0KEYPXo0jh07hoEDB2LevHnSGlkajQZqtRohISFSbX4hSksbwMqVK6fTwxYUFIS9e/fiypUrcHBwQOnSpdGsWTNMmjRJpxc6IyMDMTExaNGihVSb17kTExMhk8lQqlQp6UKFfv364ejRo4iLi4OLiwtGjBgBhUKBoKAgrFu3Dp999lmuXricFzls2LAB9+/fR6VKlaBWq6FQKKRz//bbb/i///u/PHsfk5OT4eHhAY1Gg7p162Ly5MkvvB3N3bt3pd47Q2GQeQvlvE/Ty8hkMrjZWcDNzgLt67giLvEJGk7ZI/XIaFVxssadpDQkpGYiPjUT8amJOB2b+NJja0PQezWd4GijgrXKFDYqE6iUJthyKg7WKhPMWL4eNioT3E1Kg5XSBIfvyeE5cjM0AkiPOYU7q55dtTN06FAAQI8ePbB06VJ8+umnuH79Ovbt2wcAOH/+PDp27oLL0dEwMzNFrZo1kZSUhNatW+vUv/feezh27BiOHTsGMzMzjB07VvphcOveI0yb8TOGfvUVbG1sULduXQwePBghISFSkLKzs4O7uztGjRqFf//9FzVr1tR53YsWLYKvry+ysrKkMKGt/fXXX9GtWzesWbMG165dQ1ZWFhQKBcLCwnDw4EH07NkT3333HWJiYjBw4EC0a9cO69evx7Vr13Ds2DE8ePAAzs7OePDgARYuXIimTZvCysoKmzdvhqenJ6pXr460tDT89ttv2LNnj067hgwZgsDAQEyePBmdOnXC0aNHsWDBAixYsCDX9y4qKgrOzs4Asnu46tWrh927d0uhRKPRYPfu3fjiiy8wbdo0ndpff/0Ve/bswcWLF1/a02JjY4PTp0/nWfu///0PNWvWfOm5teFn1apV6NWrF1avXi19r/NiaWkJS0tLxMfHY8eOHVK7XxSCrly5gnv37qFu3brSNrVajf3792POnDlIT0/X6bF7Ge0q2xMmTCjQ/jn9+OOPmDJlCnbt2iV19zs5OeUZ7mxsbKRemRf1wP3111+vNLSl5eHhAXd3d1SpUkXa9v7770uLSgKAl5cXbt68iVq1aukMoa5duxY///wz+vTp89L6l9m3bx8mT56MsLAwDBs2DGFhYfj1118xceJEjB07VtpP+8sWeBbipk6dikePHuHXX3+Fp6cngoKCEBkZqVP7shClpQ2AMTExOj1s2kC4fPlyeHp64sqVKxg9ejRatWqFQ4cOSe+XnTt3AgA6d+6sc9yc505LS8OIESPw0UcfwcbGBlevXsWePXuk4bY+ffpg+vTpyMzMxPjx418a4EqVKoX4+Hh069YNy5cvR6tWrbBp0yZMmjQJ48ePBwA8evQIDx8+zLP30dvbG4sXL0atWrWQmJiIH3/8EYGBgTh79myuKxSB7KF1a2trdOjQ4aVfR30wyFCBaYemRq87A7UQUMhkmNyhBjr7uQMAHqdl4lb8E9x8lIqb8U9wKz4VNx9l/3v9QQrSsnL3Hmw9feeV2qJ0rwWPEZvh42oLWwszmJvKYWFmAnMzBSZuPoeaXUfBz1SBJQevwdxUgaibmUh5bzKcAMhkwAfvVsRiHxcoTeRQmSqgNJFDaZL9r1ye+7LvNcdisDjeG+LD6XCWAR97JGNi/y7SXb2fD1JxcXFwc3OTLrWNS3yC09fi8L+//kLoF1/o9Cxpa8uXL4/Nmzfj7t27MDExQfv27XH48GF4enri93V/46shg1HP1xdWlpZ48OCB1B2vrc/p3LlzqFOnDpo0aYJGzVrgx4GD8fDeHZibm8PLywvz5s1D3759pRDl6OiI9evXo2fPnhg7diwqV66MmTNn4v79+9i4caMUgn799Vfs3r0bc+fOlUJUhw4dMG7cOPj6+uLAgQNSD0fv3r11/mpPTk7W6ZGJi4tDVFQUlEol0tPTMWvWLADZwe7UqVNSKNQqVaoUhBDIysqS9nvRuXv27Ik//vgDPXr0wM8//wx/f39pDldOO3bsgBAC3t7euHz5MoYNG4YqVaqgZ8+eLw1BtWrVyhW0evbsiSpVqkjBt6CeX2W7oKZNm4bvv/8eO3bsgK+vr7Q9ICAg1y/+nTt3IiAgADt27ADw8h44ExMTqUdN2+MVHx8PGxsbmJubS3PK8gpL2vd8Wloa4uLipBvH5uzher7HTDuEevnyZZQtW1bv+pzGjh2LTz75BAMHDsTIkSNRpUoVTJ48GX379sWYMWNw9+5dqFSqPEPc/PnzMXDgQHz22We4ffs2AKBfv34ICwvDmDFjcg035sfLyytXD1uZMmXQtm1bAEDNmjVRq1YtVKhQAfv27UPz5s0BZIdvALnaqJWZmYlOnTpBCCH1QGs0Gjg4OGDatGlYvXo1goKCYGlpiR9++EEKI0DeAW7hwoVwcHDAggULpO/pp59+innz5mH8+PG4efMm9uzZgwoVKuTZ4xcQEICAgADp88DAQOnrOXHixFz7L168GN26dTNIT6YWgwzp5WVDU9YqU1R1NkVVZ5tcdbcTUtFo6l6d3hyZDPi8sRdkMhkep2UiOS0Lj7WP9Cw8TsvE47QsJKdnQf18N9BTJ2+9vOcnL0IAc/Zcxpw9l/N83kwhzw42TwOOiRy48ejZkJpGAMuvW6H7osOwVpnCVCGHqUL29F85Jm89j+ofjYSPXI5f9l7G2duJ2Hb6DgQAp4F/wszfHTt7DYO5mQIq0+wApTLN/tjcVPuxAoqngWrNsRiMWhcLTaNhcH4HCOtQUwqP+cmuPQ2Tj+boBLC+ffsCyB3AWrdurdOTNW3aNAweMhS3b9+GpYUF3N3dIISQhgK19fXr18e4ceNw69YtWFpaYufOnVKIiUt8gmsPUhB3/jjmz58vtU1bGxISIv2Cfb5NYT/PxbUHKfC0t8Tt27dx6dIlKQTmd+4FCxYgKysLoaGhCA0N1fm6aAPclStXMH36dFy/fh1mZmbo3bs3vv/+e/z55586IejKlSu4du2atEbPnTt3ULVqVZ0euJs3b+rc4kPb45aYmIhLly5Jc8ee770bO3YsnJycdIYXtLXJycm4ffs21qxZIw2pXbt2DYMGDcLcuXOxatUq/P7775g+fTpmz54NKysr9OvXD3PmzMHw4cPRq1cv7NmzB2vXrsWWLVukr/PLeuBOnTqFrVu3wszMTOrxevjwIQICAqDRaLB37164urq+tCdMpVLB09MTdevWxe7du6Wv//P7aV/jlStX8Mknn0jb9KnPSbvOVc62BwYGAsjuMdu9ezcqV66cZ4gzNTWVwoo2xGlDmzZ8vyxEaWkvdHh+eDuvWi8vL9jb2+Py5cto3rw5UlJSsGnTJsjl8jyDooODAzp16oQbN25gz549sLHJ/lmrvVDB0dFRCp/aCxUyMjJeGuA0Gg0qV64MhUIhvW5bW1up9vjx40hNTcWZM2ek9+DLeh9NTU1Rp04dXL6c++frv//+i4sXL+rMozQEBhnSW0GHpnJyKWXx0t6clxFC4NqDFATNCNcJQnIZMOmDGlCZKpCaocaTDDWeZKqRmqFGWqYaqRlZeJKpQWx8Kk7EJOQ6ro3KBGqNQHqWBlk5Dpyh1iBDrcHj9KyXtiv80oOXPp/nawHwx5EY/HEkJt99TRUymCnkSMlQS9s0Ahjx12ksOnANKlMF5DIZTOQyyOUyKGQyKOTPHhlZGhy4/ECndsV1KwxcdQI2KlOYKLJrTRRymMhl+HlXNPx6fIMAuQy//XsVpgo5bru3ADpVhzMAGYD3/d3xg7eDTnAzUchgKpfD1EQGE7kcZk+33X+cjs2nbmPi5nPQiOzv1/L/rqOjb3Z3s/bCBhlkkMmyj6+92kEGYG3ETWkoUy4DwkIn5Lmw24vs27dPJ0T9X5uW0nM5w9KVK1ekocg5c+YAwEtD0PP12h64tLQ0nX1y9rpFRUXl2Xs3atQo3Lp1K9dVIDlrjx8/jj///DPXuQHoXLn2v//9D+PHj8e3336L5WvWYdTwr/Hzzz/DxcUF48aNk4LltWvXUL58ecyfPx8//fQTxo0bB0tLS8ycORPdunVDYGCgFIQ6d+6MESNGQK1WY/78+ejfvz9SUlIwZcoUDBo0CCdOnEC1atUgl8ulhQ3XrVuH2rVrIzY2FqmpqXj8+DHc3Nxw/vx5zJw5Ew8ePIC3tzeuX7+O/v3748KFC1AoFPjoo49w5MgRxMbGvrQ+JSUFfn5+UtibPXs2ypcvDzs7O7Rp0wYzZsxAnTp18PHHH+Orr77CmjVr0KRJEwwYMAApKSn47bff0KZNG9SpUwf+/v5o3LgxFixYgJCQEMydO1faXrFiRSxatAht2rSBQqGARqPBrl278OGHH+YaFs7Ze7hx40bIZDKdHpwXBbBbt27h4cOH0lDtn3/+iYyMDNSuXTvPoGhrawuZTIa9e/fqBN+GDRvijz/+gImJiRTgKlasCGdnZ5iYmLw0wJUtW1a6yEEbAMPDw+Hs7AwzMzO8++67cHBwwEcffSQNlb2s91GtVuP06dPSlWc5LVq0CPXq1YOPj0+u516HTLzOpQ/FQFJSEmxtbZGYmCilVzKeuMQnefbmFMSaYzGvFITymtujkMlwYOS7Uhuy1BqkZ2U/0jLVTz9WIy1Tg9vxqQhdFQnxXG/SsBBvWJgqkKkWyFBrkKnWIEstkPk0CGWqNbj16An2Xbqfq00V7C1hopAjLSs7dD3JUCMtS4OMPIbf6JkylmYwN3s2FGhmkt17ZpZjaDC7N02OGw9ScOjqIwhkB6NmVRxQ09UWCtnT4Pc0/MnlMshlgEIug/xpGDx+PR5/nbgl1X7cwB1NKjtkh7anwc/kaU+cifzpv0+3myrk2HzqNiZvPS+FsAntauCj+u6Qy3Jfmvo8bS+aFOD06IHLq17bC/e8F80ni0t8gr82/4N5U8ch+uIFaXLv48ePUbt2bcyaNQv+/v6YM2cOvv76a2RmZsLX1xezZs1CWloa+nzeDzeuXYOllRXeb/0evL29sWDBAty5cwe1a9eGjY0Nzp49K91w1cXFBbt27ZImXudX36NHjzzDZY8ePTBx+mx8N2ESdv/9P9yJuw2VSoWsrCwpHGjbvnnzZnTu3BlpaWmoXLkyhg4dip49e2Lk2G+x+o/f8fDeHVhaWiIhIQGzZ89G06ZNMXPmTPzxxx86l/A//7UEgHfeeQdqtRonTpzA/PnzUb9+fcycORNr167FRx99hO7du2PatGlQKpW4dOkSHj9+jNOnT0OpVOKdd95BuXLl0L59e/To0UOqnzFjBpYtWyZNoJ80aRKcnJwwZswY2NnZ4e7du6hevTp69OiBChUqYPjw4VCpVOjZsyfS0tKwdu1arFq1Cm3atEGNGjWkANenTx+EhYXhm2++QY8ePfDll19i0aJFmDZtGjp06IBJkyZJbb9w4QIcHR3RvXt37Nu3Dx06dMDMmTMxYcIENGjQABUrVkRCQgJ++OEHbNiwAcePH9fpAdJOpp8+fbrUo5ufgv7+ZpChYuVVg9CrhqDXrS9IiMpJ87SHKC0zu3fp5qNUdFl4WCdEyWXAjx19UMrCFFlqAY0QUGuALI1G+lit0eBRSiambb+AnP/BZQA+e8cT5qYKZGkE1BqBTLWAWqNBpkZArRbI1Gig1gjcSUzDkWuPcrWxQllLmJspkJmVvW/OAJf59N+sp+GOcpPLABO5HHI5cvWiAcCD5IxcNZUcrGBulj3caPJ0XxO5XPdzhQwZWQK7zusOScgAdPZzg625qbS/iUK31vTp55Ex8Vh3IlYKcF393fFOJfvsnj/Fs6CnDYGKp72BJk9D4K5zdzFrT7QUor4O9kbb2i5SSJTqZTLIcrx+7fY/I25i9PpXC3GGDoBhHWri/pFN+OGHH6QQpQ1CQPbQUfny5aUAE5f4BPuOnkTX4AD8888/uHjxok7ttGnTMHnyZERGRuLBgwewsLBA165dMXHiRDg6OuLfiJNo7Fcbq9b/jS4fvI85c+ZI9VWrVsXJkyfzbPfevXvhXccff/+zD/OnjcfZ06dgZWUFtVqN1NTUfANcnz598PfOfRg57GtcuXAG5cqVQ7Vq1XDq1KkXvu4LFy6gS5cumDlzJvp+8SU2bdyA+Af3ULp0adSrVw+TJk3S6VUEsns5Bw8ejLi4OJ2lJF6GQeYpBhnSep3eoNepN1aIet1afUPY8/KaFyWXAf8MaQJHm+y5JgLZc5YgAPE0cgkB3ElKQ+tZ/+aqXfppfVibmyD9ae/Vs3/VOh9fvpeMtRG3crUpqKoDylorodZkB77s4JcdBrUf33+cgRMx8blqK5S1hMpUkR3aNNlhLUudHQCztGFOk92GF0zpoldgbiqXgpdOD1qOnjWhEYjJsTSEVq1ytlCZKXKEPxkU8uyeM4Ui+3gmchnSs9TY8tyFBzIAnzTwQCkLUymsSbVPg6P2mBE34vG/iFu5AqBMpg1weNrupx/LsodTteFu74W7mLf/KoTI7u0d2KwSWtdyloaOFc8/ZM/avz4yFuM2njFogCuq8JifEhVkfvnlFymZ+vj4YPbs2QVeb4FBht4ExgpRr1v7Noaw1w1wedXLZcC2Qe+gjJUSGo2AWogcvWlPH0LgbmIaPl16LFcP3IxOtWFrbvq0F00j9aZlqZ/++3T7o5QMzNwVnasX7uMGHlCZynP1wmXlOMbdx2mIuJ47wFV2tIKFmYnUzufbrP34SYYaCU8yc9WbyrMnQGXX5vvlIwOQIXuoVPZ0KFMGSCFKJns2Jw0AktJyzwV0K20uXXBg8nT49Pneuyy1BgevPNSp0+f/SUGUmCCzZs0adO/eHfPmzYO/vz9mzpyJP//884XLRD+PQYbo9byNIay4BrjXqS+MAPd8vRACQkAKQDk/vpP4BK1+zt0Lt7pvA9hbKXMMm+YIU0JAoxG4l5SWax6bXAZ8/0ENWJubSmErK+e/6meBMOFJBubtu5orAHb0ddUZhtX9V/O0By89z4sJKjtawUppAo3Ift0agac9f9lDyNpewNR0NeKS0nLVW6tMIJfJnoXGHMHxTbaqTwMEVCiT/44FUGKCjL+/P/z8/KSrCTQaDdzc3PDll19i5MiR+dYzyBC9vYwVoorruY0Z4F63viQHwJy0oShLo8Ht+DQ0n7EvV/jbGNoQDjYqaJ4GR+2/0sfI/vdeUjq6/XY4V/2v3erC1twsu/dOo4FanR3esp6Gtyy1wIPkdEzZpjsHjz0yecjIyICFhQX+97//6Sxl3qNHDyQkJGDjxo25atLT05Geni59npiYCHd3d9y8eZNBhogoH3cSnyDm4RO4lzGH0yv8QjJmvbHO/dfxm5jw93kpBI1rUxUf1nMrkvrifO78JCUlwc3NDQkJCS+fIPzSe2MbWWxsrAAg/vvvP53tw4YNE/Xr18+zZvz48S+8ZTgffPDBBx988FG8Hjdv3nxpVihxC+KNGjVKZ8EojUaDR48eoUyZMvmu36APbVJ81Z4eY9bz3Dx3cannud+uc79uPc9d/M79MkIIPH78GC4uLi/d740OMjnv9ZHTy5aJViqV0jLiWtq7EBcGGxub1/rmGbOe5+a5i0s9z/12nft163nu4nfuFynImjP63QWriOW8X4aWdqnmnDepIiIiorfTG90jA2TfV6RHjx7w9fWVlnrW3t2WiIiI3m5vfJDp3Lkz7t+/j3HjxknLJW/fvl26AZqxKJVKjB8/PtcwVnGo57l57uJSz3O/Xed+3Xqeu/id2xDe6MuviYiIiF7mjZ4jQ0RERPQyDDJERERUbDHIEBERUbHFIENERETFFoPMK/rll19Qvnx5qFQq+Pv74+jRowWq279/P9q0aQMXFxfIZDJs2LChwOcMCwuDn58frK2t4eDggA8++AAXL14scP3cuXNRq1YtaeGigIAAbNu2rcD1OU2ZMgUymQyDBw8u0P7ffvvt09vHP3tUqVKlwOeLjY3Fxx9/jDJlysDc3Bw1a9ZEREREgWrLly+f69wymQyhoaH51qrVaowdOxaenp4wNzdHhQoVMHHiROgzR/7x48cYPHgwPDw8YG5ujsDAQBw7dizPffN7fwghMG7cODg7O8Pc3BxBQUGIjo4uUO26desQHBwsrXIdFRVV4HNnZmZixIgRqFmzJiwtLeHi4oLu3bvj9u3bBTr3t99+iypVqsDS0hKlS5dGUFAQjhw5UuDXnVO/fv0gk8kwc+bMAtV++umnub73LVu21Ovc58+fR9u2bWFrawtLS0v4+fkhJiamQPV5vfdkMhl++OGHfGuTk5MxYMAAuLq6wtzcHNWqVcO8efMK3Pa7d+/i008/hYuLCywsLNCyZUvp/VKQnydpaWkIDQ1FmTJlYGVlhQ8//BB3794tUO2CBQvQtGlT2NjYQCaTISEhQXouv/pHjx7hyy+/hLe3N8zNzeHu7o6BAwciMTGxQOf+/PPPUaFCBZibm6Ns2bJo164dLly4UODXrSWEQKtWraSvbUFqmzZtmut73a9fP73OfejQITRr1gyWlpawsbFB48aN8eTJk3zrr1+//sL3W9euXfM99507d/DJJ5/AyckJlpaWqFu3Lv76668Ct/3KlSto3749ypYtCxsbG3Tq1CnXgraFgUHmFaxZswZDhw7F+PHjceLECfj4+CAkJAT37t3LtzYlJQU+Pj745Zdf9D5veHg4QkNDcfjwYezcuROZmZkIDg5GSkpKgepdXV0xZcoUHD9+HBEREWjWrBnatWuHs2fP6tWOY8eOYf78+ahVq5ZeddWrV0dcXJz0OHDgQIHq4uPj0bBhQ5iammLbtm04d+4cpk+fjtKlSxe4vTnPu3PnTgBAx44d862dOnUq5s6dizlz5uD8+fOYOnUqpk2bhtmzZxfo3ADw2WefYefOnVixYgVOnz6N4OBgBAUFITY2Nte++b0/pk2bhlmzZmHevHk4cuQILC0tERISgrS0tHxrU1JS0KhRI0ydOvWFz7+oPjU1FSdOnMDYsWNx4sQJrFu3DhcvXkTbtm0L1O7KlStjzpw5OH36NA4cOIDy5csjODgY9+/fL1C91vr163H48GGdJcsLUtuyZUud98CqVasKXH/lyhU0atQIVapUwb59+3Dq1CmMHTsWKpWqQPU5zxsXF4fFixdDJpPhww8/zLd26NCh2L59O1auXInz589j8ODBGDBgADZt2pTvuYUQ+OCDD3D16lVs3LgRkZGR8PDwQFBQEFJSUgr082TIkCH4+++/8eeffyI8PBy3b99Ghw4dClSbmpqKli1bYvTo0bnall/97du3cfv2bfz44484c+YMli5diu3bt6N3794FOne9evWwZMkSnD9/Hjt27IAQAsHBwVCr1Xr9HJ05c6bOrW0KWtunTx+d7/m0adMKXH/o0CG0bNkSwcHBOHr0KI4dO4YBAwZALpfnW+/m5pbr/fbdd9/BysoK9+/fz/fc3bt3x8WLF7Fp0yacPn0aHTp0QKdOnRAZGZnvuVNSUhAcHAyZTIY9e/bg4MGDyMjIQJs2baDRaHJ9bQ3qte/s+BaqX7++CA0NlT5Xq9XCxcVFhIWF6XUcAGL9+vWv3I579+4JACI8PPyVj1G6dGnx22+/FXj/x48fi0qVKomdO3eKJk2aiEGDBhWobvz48cLHx+eV2jhixAjRqFGjV6rNy6BBg0SFChWERqPJd9/WrVuLXr166Wzr0KGD6NatW4HOlZqaKhQKhdi8ebPO9rp164oxY8a8tPb594dGoxFOTk7ihx9+kLYlJCQIpVIpVq1a9dLanK5duyYAiMjIyAKfOy9Hjx4VAMSNGzf0rk1MTBQAxK5duwp87lu3boly5cqJM2fOCA8PD/HTTz8VqLZHjx6iXbt2L23Py+o7d+4sPv7441euf167du1Es2bNClRbvXp1MWHCBJ1tL3rvPF9/8eJFAUCcOXNG2qZWq0XZsmXFwoULc9U///MkISFBmJqaij///FPa5/z58wKAOHTo0Etrc9q7d68AIOLj43M9V5B6rbVr1wozMzORmZmpd+3JkycFAHH58uUCnzsyMlKUK1dOxMXFvfD7mletPj8X86r39/cX33zzzSvXP6927dq5foa9qNbS0lIsX75cZz87O7sCvV927Ngh5HK5SExMlPZJSEgQMplM7Ny5s0Cv51WxR0ZPGRkZOH78OIKCgqRtcrkcQUFBOHToUJG2JTExEQBgZ2end61arcbq1auRkpKi1+0eQkND0bp1a53XX1DR0dFwcXGBl5cXunXrJnXN52fTpk3w9fVFx44d4eDggDp16mDhwoV6nx/I/v6tXLkSvXr1KtBNRAMDA7F7925cunQJAHDy5EkcOHAArVq1KtD5srKyoFarpb/etczNzQvcI6V17do13LlzR+drb2trC39//yJ/7wHZ7z+ZTKb3vcwyMjKwYMEC2NrawsfHp0A1Go0Gn3zyCYYNG4bq1avr3dZ9+/bBwcEB3t7e6N+/Px4+fFjg827ZsgWVK1dGSEgIHBwc4O/vr9eQcE53797Fli1b0Lt37wLtHxgYiE2bNiE2NhZCCOzduxeXLl1CcHBwvrXp6ekAoPPek8vlUCqVeb73nv95cvz4cWRmZuq836pUqQJ3d/dc77fX+VlU0PrExETY2NjAxMQk1/aX1aakpGDJkiXw9PSEm5tbgc6dmpqKrl274pdffnnhff1edu7ff/8d9vb2qFGjBkaNGoXU1NQC1d+7dw9HjhyBg4MDAgMD4ejoiCZNmrzwZ0V+r/348eOIiorK8/2WV21gYCDWrFmDR48eQaPRYPXq1UhLS0PTpk3zrU9PT4dMJtNZGE+lUkEul+v9s05vhRqTSqDY2FgBQPz3338624cNGybq16+v17HwGj0yarVatG7dWjRs2FCvulOnTglLS0uhUCiEra2t2LJlS4FrV61aJWrUqCGePHkihNDvL4+tW7eKtWvXipMnT4rt27eLgIAA4e7uLpKSkvKtVSqVQqlUilGjRokTJ06I+fPnC5VKJZYuXVrgtmutWbNGKBQKERsbW6D91Wq1GDFihJDJZMLExETIZDIxefJkvc4ZEBAgmjRpImJjY0VWVpZYsWKFkMvlonLlyi+te/79cfDgQQFA3L59W2e/jh07ik6dOr20NidD9Mg8efJE1K1bV3Tt2rXAtX///bewtLQUMplMuLi4iKNHjxb43JMnTxYtWrSQetH06ZFZtWqV2Lhxozh16pRYv369qFq1qvDz8xNZWVn51mv/GrewsBAzZswQkZGRIiwsTMhkMrFv374Cv3atqVOnitKlS0v/h/KrTUtLE927dxcAhImJiTAzMxPLli3L89jP12dkZAh3d3fRsWNH8ejRI5Geni6mTJkiAIjg4GCd2rx+nvz+++/CzMws13n8/PzE8OHDX1qbU349MgX5WXb//n3h7u4uRo8eXeDaX375RVhaWgoAwtvbO8/emBfV9+3bV/Tu3Vv6PK/vzYtq58+fL7Zv3y5OnTolVq5cKcqVKyfat29foHMfOnRIABB2dnZi8eLF4sSJE2Lw4MHCzMxMXLp0qcCvXat///6iatWqBX7d8fHxIjg4WHq/2djYiB07dhSo/t69e8LGxkYMGjRIpKSkiOTkZDFgwAABQPTt2/eFbTQEBhk9vSlBpl+/fsLDw0PcvHlTr7r09HQRHR0tIiIixMiRI4W9vb04e/ZsvnUxMTHCwcFBnDx5UtqmT5B5Xnx8vLCxsSnQsJapqakICAjQ2fbll1+KBg0a6H3e4OBg8f777xd4/1WrVglXV1exatUqcerUKbF8+XJhZ2enV4i6fPmyaNy4sQAgFAqF8PPzE926dRNVqlR5ad2bGmQyMjJEmzZtRJ06dXS6kfOrTU5OFtHR0eLQoUOiV69eonz58uLu3bv51kdERAhHR0ed8KlPkHnelStXCjyspf3//tFHH+ns16ZNG9GlSxe9z+/t7S0GDBiQ53N51f7www+icuXKYtOmTeLkyZNi9uzZwsrKKs+u+rzqIyIihI+Pj/TeCwkJEa1atRItW7bU2S+vnycFDTL5/SzKL8jkV5+YmCjq168vWrZsKTIyMgpcm5CQIC5duiTCw8NFmzZtRN26dXMFyLzqN27cKCpWrCgeP34sbcvra1vQn8G7d+/Oc1grr3rt//FRo0bp7FuzZk0xcuRIvc6fmpoqbG1txY8//pjruRfVDhgwQNSvX1/s2rVLREVFiW+//VbY2tqKU6dOFah+x44dwsvLS8hkMqFQKMTHH38s6tatK/r16/eCr45hMMjoKT09XSgUilxv6u7du4u2bdvqdaxXDTKhoaHC1dVVXL16Ve/a5zVv3rxAaXn9+vXSD0PtA4D0hs3rr9v8+Pr65vrPmRd3d3edv46EEOLXX38VLi4uep3v+vXrQi6Xiw0bNhS4xtXVVcyZM0dn28SJE4W3t7de5xYi+xe5NoR06tRJvPfeey/d//n3h/YX8PMBpHHjxmLgwIEvrc3pdYJMRkaG+OCDD0StWrXEgwcP9Kp9XsWKFfPs3Xq+/qeffpLeZznfe3K5XHh4eLzSue3t7cW8efPyPXd6erowMTEREydO1Nlv+PDhIjAwMN/6nPbv3y8AiKioqDyff742NTVVmJqa5ppf1bt3bxESEqLXuRMSEsS9e/eEENlz/L744gvpuRf9PNH+An4+gLi7u4sZM2a8tDanlwWZ/OqTkpJEQECAaN68ea4Qos/PwfT0dGFhYSH++OOPfOsHDRr0wvdbkyZN9D53cnKyACC2b9+e77mvXr0qAIgVK1bobO/UqZNO72dBzr98+XJhamoqfd/zq718+XKuOVVCZP+O+Pzzz/U69/3796Xvt6Ojo5g2bdoL9zUEzpHRk5mZGerVq4fdu3dL2zQaDXbv3q3XXJNXIYTAgAEDsH79euzZsweenp6vfUyNRiONpb9M8+bNcfr0aURFRUkPX19fdOvWDVFRUVAoFHqdNzk5GVeuXIGzs3O++zZs2DDXZX6XLl2Ch4eHXudcsmQJHBwc0Lp16wLXpKamQi7X/W+iUCheaRa+paUlnJ2dER8fjx07dqBdu3Z61Xt6esLJyUnnvZeUlIQjR44U+nsPyL4Eu1OnToiOjsauXbtQpkyZ1zpeQd97n3zyCU6dOqXz3nNxccGwYcOwY8cOvc9769YtPHz4sEDvPTMzM/j5+Rnk/bdo0SLUq1evwPOCMjMzkZmZaZD3n62tLcqWLYvo6GhERESgXbt2+f48qVevHkxNTXXebxcvXkRMTAwaNGjwWj+LCvKzLCkpCcHBwTAzM8OmTZukuT6v8nNQZP/RjvT09HzrR44cmev9BgA//fQTFi9erPe5tfXOzs75nrt8+fJwcXF54ftNn9e+aNEitG3bFmXLlpW+Bi+r1c7jedH7TZ9z29vbo1SpUtizZw/u3bsnXd1YaAo1JpVQq1evFkqlUixdulScO3dO9O3bV5QqVUrcuXMn39rHjx+LyMhIERkZKQBI4+7PX/mRl/79+wtbW1uxb98+ERcXJz1SU1ML1O6RI0eK8PBwce3aNXHq1CkxcuRIIZPJxD///FOg+ufpM7T01VdfiX379olr166JgwcPiqCgIGFvb5/rr4W8HD16VJiYmIjvv/9eREdHi99//11YWFiIlStXFritarVauLu7ixEjRhS4RojsK17KlSsnNm/eLK5duybWrVsn7O3tdbrW87N9+3axbds2cfXqVfHPP/8IHx8f4e/vn6ubXIj83x9TpkwRpUqVkuZ8tGvXTnh6eoonT57kW/vw4UMRGRkptmzZIgCI1atXi8jISBEXF5fvuTMyMkTbtm2Fq6uriIqK0nn/paenv7Q2OTlZjBo1Shw6dEhcv35dREREiJ49ewqlUin99afv/4ucQ0svq338+LH4+uuvxaFDh8S1a9fErl27RN26dUWlSpVEWlpagc69bt06YWpqKhYsWCCio6PF7NmzhUKhEP/++2+B256YmCgsLCzE3Llz9fp+N2nSRFSvXl3s3btXXL16VSxZskSoVCrx66+/Fqh+7dq1Yu/eveLKlStiw4YNwsPDQ3To0EEIUbCfJ/369RPu7u5iz549IiIiQgQEBIiAgIAC1cbFxYnIyEixcOFCAUDs379fREZGiocPH+Zbn5iYKPz9/UXNmjXF5cuXdfbp16/fS2uvXLkiJk+eLCIiIsSNGzfEwYMHRZs2bYSdnZ24e/fuK/0cxdPervxqL1++LCZMmCAiIiLEtWvXxMaNG4WXl5do3Lhxgb/mP/30k7CxsRF//vmniI6OFt98841QqVTi8uXLBW57dHS0kMlkYtu2bdK2/GozMjJExYoVxTvvvCOOHDkiLl++LH788Uchk8nEli1bCnTuxYsXi0OHDonLly+LFStWCDs7OzF06NAXfl0NhUHmFc2ePVu4u7sLMzMzUb9+fXH48OEC1Wm7WZ9/9OjRI9/avOoAiCVLlhTo3L169RIeHh7CzMxMlC1bVjRv3vyVQ4wQ+gWZzp07C2dnZ2FmZibKlSsnOnfunOfkuxf5+++/RY0aNYRSqRRVqlQRCxYs0KutO3bsEADExYsX9apLSkoSgwYNEu7u7kKlUgkvLy8xZswYkZ6eXuBjrFmzRnh5eQkzMzPh5OQkQkNDRUJCQp775vf+0Gg0YuzYscLR0VEolUrRvHlz6TXlV7tkyZI8nx8/fny+9drhqLwee/fufWntkydPRPv27YWLi4swMzMTzs7Oom3btjqTffX9f5EzyLysNjU1VQQHB4uyZcsKU1NT4eHhIfr06aPzR0dBzr1o0SJRsWJFoVKphI+Pj87wZEHq58+fL8zNzXN93/OrjYuLE59++qlwcXERKpVKeHt7i+nTp0uTnvOr//nnn4Wrq6swNTUV7u7u4ptvvpHeuwX5efLkyRPxxRdfiNKlSwsLCwvRvn17aQJ0frXjx49/4T751b/odb3soa2NjY0VrVq1Eg4ODsLU1FS4urqKrl27igsXLhT4dT9PG2Tyq42JiRGNGzcWdnZ2QqlUiooVK4phw4ZJc8kKeu6wsDDh6uoqLCwsREBAgBSaC1o/atQo4ebmJtRqtc5ryK/20qVLokOHDsLBwUFYWFiIWrVqSZdjF6R+xIgRwtHRUZiamopKlSrpvFcLk+xpA4mIiIiKHc6RISIiomKLQYaIiIiKLQYZIiIiKrYYZIiIiKjYYpAhIiKiYotBhoiIiIotBhkiIiIqthhkiIiIqNhikCGit86+ffsgk8mQkJBg7KYQ0WtikCEiIqJii0GGiIiIii0GGSIqchqNBmFhYfD09IS5uTl8fHzwv//9D8CzYZ8tW7agVq1aUKlUaNCgAc6cOaNzjL/++gvVq1eHUqlE+fLlMX36dJ3n09PTMWLECLi5uUGpVKJixYpYtGiRzj7Hjx+Hr68vLCwsEBgYiIsXLxbuCycig2OQIaIiFxYWhuXLl2PevHk4e/YshgwZgo8//hjh4eHSPsOGDcP06dNx7NgxlC1bFm3atEFmZiaA7ADSqVMndOnSBadPn8a3336LsWPHYunSpVJ99+7dsWrVKsyaNQvnz5/H/PnzYWVlpdOOMWPGYPr06YiIiICJiQl69epVJK+fiAyHd78moiKVnp4OOzs77Nq1CwEBAdL2zz77DKmpqejbty/effddrF69Gp07dwYAPHr0CK6urli6dCk6deqEbt264f79+/jnn3+k+uHDh2PLli04e/YsLl26BG9vb+zcuRNBQUG52rBv3z68++672LVrF5o3bw4A2Lp1K1q3bo0nT55ApVIV8leBiAyFPTJEVKQuX76M1NRUtGjRAlZWVtJj+fLluHLlirRfzpBjZ2cHb29vnD9/HgBw/vx5NGzYUOe4DRs2RHR0NNRqNaKioqBQKNCkSZOXtqVWrVrSx87OzgCAe/fuvfZrJKKiY2LsBhDR2yU5ORkAsGXLFpQrV07nOaVSqRNmXpW5uXmB9jM1NZU+lslkALLn7xBR8cEeGSIqUtWqVYNSqURMTAwqVqyo83Bzc5P2O3z4sPRxfHw8Ll26hKpVqwIAqlatioMHD+oc9+DBg6hcuTIUCgVq1qwJjUajM+eGiEom9sgQUZGytrbG119/jSFDhkCj0aBRo0ZITEzEwYMHYWNjAw8PDwDAhAkTUKZMGTg6OmLMmDGwt7fHBx98AAD46quv4Ofnh4kTJ6Jz5844dOgQ5syZg19//RUAUL58efTo0QO9evXCrFmz4OPjgxs3buDevXvo1KmTsV46ERUCBhkiKnITJ05E2bJlERYWhqtXr6JUqVKoW7cuRo8eLQ3tTJkyBYMGDUJ0dDRq166Nv//+G2ZmZgCAunXrYu3atRg3bhwmTpwIZ2dnTJgwAZ9++ql0jrlz52L06NH44osv8PDhQ7i7u2P06NHGeLlEVIh41RIRvVG0VxTFx8ejVKlSxm4OEb3hOEeGiIiIii0GGSIiIiq2OLRERERExRZ7ZIiIiKjYYpAhIiKiYotBhoiIiIotBhkiIiIqthhkiIiIqNhikCEiIqJii0GGiIiIii0GGSIiIiq2/h+5yQxUdZWhjwAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "y_avg = []\n", + "for y in epochs_y:\n", + " y_avg.append(np.mean(y))\n", + "\n", + "plt.plot(np.arange(len(epochs_x)), y_avg, marker = '.')\n", + "plt.xlabel('epoch')\n", + "plt.ylabel('average loss')\n", + "plt.ylim(0,)\n", + "plt.xticks(np.arange(len(epochs_x)))\n", + "for i, txt in enumerate(y_avg):\n", + " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAG2CAYAAACZEEfAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABmHUlEQVR4nO3deVxUVeMG8GeGZdhBdpBF3MBdcQX34EXN3FMzU1PTLCyXLKXSst5EzdLXNLdcW9xKzTT3NRUVFBRNERCFkEVEhn1x5vz+IOfnyDaDIDA+389nPjX33rMMDjMP9557jkQIIUBERESko6Q13QEiIiKi6sSwQ0RERDqNYYeIiIh0GsMOERER6TSGHSIiItJpDDtERESk0xh2iIiISKcx7BAREZFOY9ghIiIincawQ0RERDqtRsPO6dOnMWDAADg7O0MikWDPnj1q+4UQmDdvHpycnGBsbAx/f39ER0erHZOeno7Ro0fDwsICVlZWmDhxIrKzs5/jqyAiIqLarEbDTk5ODtq0aYOVK1eWun/x4sVYvnw5Vq9ejQsXLsDU1BR9+vRBfn6+6pjRo0fj+vXrOHLkCPbt24fTp09j8uTJz+slEBERUS0nqS0LgUokEuzevRuDBw8GUHxWx9nZGR988AFmzZoFAJDL5XBwcMCmTZvw2muv4caNG2jevDlCQ0PRoUMHAMDBgwfx8ssv459//oGzs3NNvRwiIiKqJfRrugNliYuLQ3JyMvz9/VXbLC0t0blzZ4SEhOC1115DSEgIrKysVEEHAPz9/SGVSnHhwgUMGTKk1LoLCgpQUFCgeq5UKpGeng4bGxtIJJLqe1FERERUZYQQyMrKgrOzM6TSsi9W1dqwk5ycDABwcHBQ2+7g4KDal5ycDHt7e7X9+vr6sLa2Vh1TmuDgYMyfP7+Ke0xEREQ1ISEhAS4uLmXur7VhpzoFBQVh5syZqudyuRxubm5ISEiAhYVFDfaMiIiINJWZmQlXV1eYm5uXe1ytDTuOjo4AgJSUFDg5Oam2p6SkoG3btqpjUlNT1co9evQI6enpqvKlkclkkMlkJbZbWFgw7BAREdUxFQ1BqbXz7Hh4eMDR0RHHjh1TbcvMzMSFCxfg4+MDAPDx8UFGRgYuXbqkOub48eNQKpXo3Lnzc+8zERER1T41emYnOzsbMTExqudxcXGIiIiAtbU13NzcMH36dPz3v/9FkyZN4OHhgblz58LZ2Vl1x1azZs3Qt29fTJo0CatXr0ZRURGmTp2K1157jXdiEREREYAaDjthYWHo3bu36vnjcTTjxo3Dpk2b8NFHHyEnJweTJ09GRkYGunXrhoMHD8LIyEhV5ueff8bUqVPh5+cHqVSKYcOGYfny5c/9tRAREVHtVGvm2alJmZmZsLS0hFwu55gdIiKiOkLT7+9aO2aHiIiIqCow7BAREZFOY9ghIiIincawQ0RERDqNYYeIiIh0GsMOERER6TSGHSIiItJpDDtERESk0xh2iIiISKcx7BAREZFOY9ghIiIincawQ0RERDqNYYeIiIh0GsMOERER6TSGHSIiItJpDDtERESk0xh2iIiISKcx7BAREZFOY9ghIiIincawQ0RERDqNYYeIiIh0GsMOERER6TSGHSIiItJpDDtERESk0xh2iIiISKcx7BAREZFOY9ghIiIincawQ0RERDqNYYeIiIh0GsMOERER6TSGHSIiItJpDDtERESk0xh2iIiISKcx7BAREZFOY9ghIiIincawQ0RERDqNYYeIiIh0GsMOERER6TSGHSIiItJpDDtERESk0xh2iIiISKcx7BAREZFOY9ghIiIincawQ0RERDqNYYeIiIh0GsMOERER6TSGHSIiItJpDDtERESk0xh2iIiISKcx7BAREZFOY9ghIiIincawQ0RERDqNYYeIiIh0GsMOERER6TSGHSIiItJpDDtERESk0xh2iIiISKcx7BAREZFOY9ghIiIincawQ0RERDqNYYeIiIh0GsMOERER6TSGHSIiItJptTrsKBQKzJ07Fx4eHjA2NkajRo3w5ZdfQgihOkYIgXnz5sHJyQnGxsbw9/dHdHR0DfaaiIiIapNaHXYWLVqEVatWYcWKFbhx4wYWLVqExYsX47vvvlMds3jxYixfvhyrV6/GhQsXYGpqij59+iA/P78Ge05ERES1hUQ8eZqklnnllVfg4OCA9evXq7YNGzYMxsbG+OmnnyCEgLOzMz744APMmjULACCXy+Hg4IBNmzbhtdde06idzMxMWFpaQi6Xw8LColpeCxEREVUtTb+/a/WZHV9fXxw7dgy3bt0CAFy5cgVnzpxBv379AABxcXFITk6Gv7+/qoylpSU6d+6MkJCQMustKChAZmam2oOIiIh0k35Nd6A8c+bMQWZmJry8vKCnpweFQoGvvvoKo0ePBgAkJycDABwcHNTKOTg4qPaVJjg4GPPnz6++jhMREVGtUavP7OzYsQM///wzfvnlF1y+fBmbN2/GkiVLsHnz5meqNygoCHK5XPVISEiooh4TERFRbVOrz+x8+OGHmDNnjmrsTatWrXD37l0EBwdj3LhxcHR0BACkpKTAyclJVS4lJQVt27Yts16ZTAaZTFatfSciIqLaoVaf2cnNzYVUqt5FPT09KJVKAICHhwccHR1x7Ngx1f7MzExcuHABPj4+z7WvREREVDvV6jM7AwYMwFdffQU3Nze0aNEC4eHh+PbbbzFhwgQAgEQiwfTp0/Hf//4XTZo0gYeHB+bOnQtnZ2cMHjy4ZjtPREREtUKtDjvfffcd5s6di3fffRepqalwdnbG22+/jXnz5qmO+eijj5CTk4PJkycjIyMD3bp1w8GDB2FkZFSDPSciIqLaolbPs/O8cJ4dIiKiukcn5tkhIiIielYMO0RERKTTGHaIiIhIpzHsEBERkU5j2CEiIiKdxrBDREREOo1hh4iIiHQaww4RERHpNIYdIiIi0mkMO0RERKTTGHaIiIhIpzHsEBERkU5j2CEiIqISGjRoAIlEUuIRGBgIAFi7di169eoFCwsLSCQSZGRkVFjn559/XqI+Ly8vtWNiY2MxZMgQ2NnZwcLCAiNGjEBKSsozvRaGHSIiIiohNDQUSUlJqseRI0cAAMOHDwcA5Obmom/fvvj444+1qrdFixZq9Z45c0a1LycnBwEBAZBIJDh+/DjOnj2LwsJCDBgwAEqlstKvRb/SJYmIiEhn2dnZqT1fuHAhGjVqhJ49ewIApk+fDgA4efKkVvXq6+vD0dGx1H1nz57FnTt3EB4eDgsLCwDA5s2bUa9ePRw/fhz+/v7avYh/8cwOERERlauwsBA//fQTJkyYAIlE8kx1RUdHw9nZGQ0bNsTo0aMRHx+v2ldQUACJRAKZTKbaZmRkBKlUqnYGSFsMO0RERFSuPXv2ICMjA2+++eYz1dO5c2ds2rQJBw8exKpVqxAXF4fu3bsjKysLANClSxeYmppi9uzZyM3NRU5ODmbNmgWFQoGkpKRKt8uwQ0REROVav349+vXrB2dn52eqp1+/fhg+fDhat26NPn364M8//0RGRgZ27NgBoPjS2c6dO/HHH3/AzMwMlpaWyMjIgLe3N6TSykcWjtkhIiKiMt29exdHjx7Frl27qrxuKysrNG3aFDExMaptAQEBiI2NRVpaGvT19WFlZQVHR0c0bNiw0u3wzA4RERGVaePGjbC3t0f//v2rvO7s7GzExsbCycmpxD5bW1tYWVnh+PHjSE1NxcCBAyvdDsMOERERlUqpVGLjxo0YN24c9PXVLwYlJycjIiJCdVYmMjISERERSE9PVx3j5+eHFStWqJ7PmjULp06dwp07d3Du3DkMGTIEenp6GDVqlOqYjRs34vz584iNjcVPP/2E4cOHY8aMGfD09Kz06+BlLCIiIirV0aNHER8fjwkTJpTYt3r1asyfP1/1vEePHgCKw8rjgcyxsbGI+ycJ52LT4GFrin/++QejRo3CgwcPYGdnh27duuH8+fNqt7lHRUUhKCgI6enpaNCgAT755BPMmDHjmV6HRAghnqkGHZCZmQlLS0vI5XLVff1ERET0bLaHxiNoVySUApBKgOChrTCyo1uV1a/p9zcvYxEREVUgMTERb7zxBmxsbGBsbIxWrVohLCxMtT87OxtTp06Fi4sLjI2N0bx5c6xevbrcOjdt2lRi6QQjI6MSx924cQMDBw6EpaUlTE1N0bFjR7W5aWqjgkcK/HElEXN+Kw46AKAUwMe7riFJnvfc+8PLWERE9MJIkuchLi0HHramcLI01qjMw4cP0bVrV/Tu3RsHDhyAnZ0doqOjUa9ePdUxM2fOxPHjx/HTTz+hQYMGOHz4MN599104OzuXO7DWwsICUVFRqudPT9gXGxuLbt26YeLEiZg/fz4sLCxw/fr1UkNRVb9ubQghEJWShTPRaTgdnYaLcQ+QX1RyeQeFELiTllstfSgPww4REb0QKntJZdGiRXB1dcXGjRtV2zw8PNSOOXfuHMaNG4devXoBACZPnow1a9bg4sWL5YYdiURS5tIJAPDJJ5/g5ZdfxuLFi1XbGjVqVGGfn1QVl5JKC0upmfn4KzoNZ2KKH/ezCtTKWJsaIj2nUG2bnkSCBrYmWrVdFRh2iIhI5yXJ8zDnt0g8HqT6+JJKj6Z2FZ5l2Lt3L/r06YPhw4fj1KlTqF+/Pt59911MmjRJdYyvry/27t2LCRMmwNnZGSdPnsStW7ewdOnScuvOzs6Gu7s7lEolvL29sWDBArRo0aK4j0ol9u/fj48++gh9+vRBeHg4PDw8EBQUhMGDB5dan0IpEJ+ei5tJmbiRlInwhAz8FZ2m2q8UwOzfIrHurzg4WRrBzkwGW3MZbM0MYWsm+/+HuSFsTGXQk0rUwpJEAnRtZIv7WQWISslSa9vIQIrOHjbo3sQW3ZrYwtPBHDvCEvDxrmtQCAE9iQQLhrZ87md1AA5QBsABykREuuyRQolp28KxPzK5xL6tk7rAp5FNueUfXzKaOXMmhg8fjtDQUEybNg2rV6/GuHHjABSv6TR58mRs2bIF+vr6kEqlWLduHcaOHVtmvSEhIYiOjkbr1q0hl8uxZMkSnD59GtevX4eLiwuSk5Ph5OQEExMTfPTJZ3Bv1RG3Lp3Bwi8+w4kTJ9Cmow9uJmfhZlImbiZn4UZyFm4lZyGvSPEMP63/J5UAlsYGeJhbVOp+iQRo6WyJbk1s0b2JLdq714NMX6/EcUnyPNxJy0UDW5MqDzqafn8z7IBhh4hIV8lzizB162W1sxtPmu7XBNP8m5S7uKWhoSE6dOiAc+fOqba9//77CA0NRUhICABgyZIlWLduHZYsWQJ3d3ecPn0aQUFB2L17t8YrdRcVFaFZs2YYNWoUvvzyS9y7dw/169eHb8BAJHlPLj6zAiBn3wIo9WQw6/dBqfUYGUjh6WAOL0cLOFsZYdmxaDz5TS+VAEuGt4FSAGnZBbifVYC07H8fWYVIyy5Aem4hyksH7/s1xpu+HrA2NdTotVUXTb+/eRmLiIh00q2ULEzaEoa7D3JhbKCHod71se1iAhRCQAJAAFh2LBoJD/Pw1ZCWMDIoeVYCAJycnNC8eXO1bc2aNcNvv/0GAMjLy8PHH3+M3bt3q2YZbt26NSIiIrBkyRKNw46BgQHatWunmqTP3Koe9PT0cT3PAlb/Bg8BoMDMCQX//A0zAC71jOHlaIFmTuaq/7rbmEJP+v/hzdHSqMSlpKHeLuX25ZFCifTcQty8l4Vxmy6qBR89iQSjOrnVeNDRBsMOERHpnMPXkzFjewRyChVwqWeMtWM6oLmzBaa+1Bh30nLhbmOMA9dS8NX+v/Hb5X8Qcz8ba8e0h4NFybucunbtqnbHFADcunUL7u7uAIrPyBQVFZVYqFJPTw9KZck7ksqiUCgQGRkJ764vIWjXVey/mgR9h8Z4lJ6odlxReiLaNmuMw58HwMLIoMJ6R3Z0Q4+mdlpdStLXk8Le3Aj2nkZYOLRVrRh38ywYdoiISGcolQLfHY/B0qO3AAA+DW2wcrS36iyEk6Wx6ot6YjcPeDqYI/CXy7iSkIFXvjuD1W+0R3v3emp1zpgxA76+vliwYAFGjBiBixcvYu3atVi7di2A4tvHe/bsiQ8//BDGxsZwd3fHqVOnsGXLFnz77beqesaOHYv69esjODgYAPDFF1+gS5cuaNy4MSJvJ+Kz/y5EdGwcMru1guHFBACAa6+RiNn2FWQuLWDk3hp5ty8hL+Yi5iw/rFHQeezJ162tyoSl2oZjdsAxO0REuiC74BE+2BGBQ9dTAABv+jbAJ/2bwUCv/Plz7z7IweQtlxCVkgVDPSn+O7glRnR0VTtm3759CAoKQnR0NDw8PDBz5ky1u7GSk5MRFBSEw4cPIz09He7u7pg8eTJmzJihGg/k260HrBycsX7DRjhZGuPtwPfw62+7kJF2HzAyhcyhMax6jIGNmyf6tXLE4Hb10cXDBu9+tgTrVy6DIisNBtb18f6HH+PrWW9V8U+vbuIAZS0w7BAR1W13H+Rg0pYw3ErJLg4sQ1piRAfXigv+K6fgET7YcQUHrxffsaVpUNKU2u3bABo7mCE2NVs1u7C+VIJenvYY0q4+/JrZlxg/VJ13NNVlDDtaYNghIqq7zkSnIfCXy5DnFcHeXIbVY9rD261exQWfolQKrDgRg2+PlH4JrLKuJDzE4JXnUNqXrbebFYa0q4/+rZ3r1IDf2oJhRwsMO0REdY8QAuvPxGHBnzegFEBbVyusKWOQsTbKGtysqeyCR7hw+wH+ik7DX9H3EXs/p9Tjlo5ogyEV3BVF5WPY0QLDDhFR3ZJfpMDHuyKxK7z4TqVX27vgv4PLvn1cW0/ftr5keBv0b+1U6rGPFEpcTZTjTHQazkSn4XL8QzxS/v9X6+Pb3J+kJ5HgzJzevCT1jDjPDhER6ZwkeR4u3X2IFcdjcDM5C3pSCT7t3wxv+jYod2JAbTV1MMfvgV3x3tZw/PXvZbKLce4IaO6IhvamKHykLF4XKjoNZ2PTkJX/SK28m7VJ8czCjW3h28gWB68n1fnbt+syntkBz+wQ0YsjMTERs2fPxoEDB5Cbm4vGjRtj48aN6NChQ4ljp0yZgjVr1mDp0qWYPn16mXWuWrUKq1atwp07dwAALVq0wLx589CvXz/VMbGxsZg1axbOnDmDgoIC9O3bF9999x0cHBw07vu2i/EI2h2pmuDO2FAP68d2gG9jW43r0NYjhRKLD0Vh7enb5R5nYaSPro1t/w04dnCzKbnYJQcZVz2e2SEiIjUPHz5E165d0bt3bxw4cAB2dnaIjo5GvXolB/Pu3r0b58+fh7Ozc4X1uri4YOHChWjSpAmEENi8eTMGDRqE8PBwtGjRAjk5OQgICECbNm1w/PhxAMDcuXMxYMAAnD9/vsRkfE8reKTA5nN3sODPm+rbixTwsDPV4iegPX09KcZ3bYB1p2+XuBTV1tUKfl726N7UDq3qW6rNWlyaZ5nrhp4Nww4R0Qti0aJFcHV1xcaNG1XbPDw8ShyXmJiI9957D4cOHVItf1CeAQMGqD3/6quvsGrVKpw/fx4tWrTA2bNncefOHYSHh6v++t68eTPq1auH48ePl7mcQkZuIX6+EI9N5+7gflZBif1KAdxJy632ABGXllPqnVSz+3pVuIgo1Q5VM4EAERHVenv37kWHDh0wfPhw2Nvbo127dli3bp3aMUqlEmPGjMGHH36IFi1aaN2GQqHAtm3bkJOTAx8fHwDFK4JLJBLIZDLVcUZGRpBKpThz5kyJOu4+yMG836/BJ/g4vj4UhftZBbAzk+Hp8yZ6Egka2Ja8XFTVPGxN8fRJm+fVNlUNhh0iohfE7du3sWrVKjRp0gSHDh3CO++8g/fffx+bN29WHbNo0SLo6+vj/fff16ruyMhImJmZQSaTYcqUKdi9e7dq8cwuXbrA1NQUs2fPRm5uLnJycjBr1iwoFAokJSWp6rh0Nx1TfryEXktOYkvIXeQVKdDcyQJLR7bB2TkvYeGwVtD7dxDy8xzk62RpjOChNdM2VQ1exiIiekEolUp06NABCxYsAAC0a9cO165dw+rVqzFu3DhcunQJ//vf/3D58mWt72zy9PREREQE5HI5fv31V4wbNw6nTp1C8+bNYWdnh507d+Kdd97B8uXLIZVKMWrUKHh7e0MikeBAZBLW/nUb4fEZqvp6edphcveG8Glko+pLTa7RpAvrQ73IeDcWeDcWEb0Y3N3d8Z///Ac//PCDatuqVavw3//+F4mJiVi2bBlmzpypNmBYoVBAKpXC1dVVdbeVJvz9/dGoUSOsWbNGbXtaWhrScopwL0+Kwb4tYeszFMpWAwEAhnpSDGlXH29190ATB/Nne7H0QuDdWEREpKZr166IiopS23br1i24u7sDAMaMGVNisHCfPn0wZswYjB8/Xqu2lEolCgrUBxUXKZRYH3ofq0/dRu7dK8h6+ADmzt6wMzHAmC7uGOPjDnvzZ5v9mKg0DDtERC+IGTNmwNfXFwsWLMCIESNw8eJFrF27FmvXrgUA2NjYwMZG/e4iAwMDODo6wtPTU7XNz88PQ4YMwdSpUwEAQUFB6NevH9zc3JCVlYVffvkFJ0+exMGDBxF7PxtnotOwfsMGJCjrodDAHAX3buLh0bUw7zgIhrYu2P1O12q/hZxebAw7REQviI4dO2L37t0ICgrCF198AQ8PDyxbtgyjR4/Wqp7Y2FikpaWpnqempmLs2LFISkqChYUl6jfyxPBPVuHzS3q4d/wUAODhtRvIvnYUyrxs6Fvaw9JnBMw7DoYQQHJmPsMOVSuO2QHH7BARaSNJnoe4tBx42JqinokhLt19WLx0Qsx9XL+XiSe/VQz1pOjQoB66NbFFM0cLTNwciieWjeIaUfRMOGaHiIiq3PbQeATtilQFFn09CR4p1P9m9nI0R7fGtuje1A6dGljD2PD/F+cMHtqKa0TRc8ewQ0REFUrLLsDP5+9i6dFote2PFAI2pobo6WmH7k1s0bWxbbmDjHkLN9UEhh0iIipVXqECh/9Oxp7wRJyOToNCWfqohxWvt4NPI80X4+QaUfS8MewQEZGKQikQEvsAu8MTcfBaEnIKFap9zRzNcTM5S22dqOJlEzi4mGo3hh0iohfQk4OMHS2McCMpC7vD/8HvEfeQ+sSim67WxhjStj4GtauPRnZm2B4azzE3VOcw7BARvWCeHGQsAWBvIUNK5v8HHCsTA7zS2glD2tWHt1s9taUjOOaG6iKGHSKiSnry7Mjz/tKvqG2FUuBhbiHSsguQlvXvf7MLEPcgBz+fj1cdJwCkZBbAQE+C/zR3wOC29dHL0x6G+mWvE80xN1TXMOwQEVXCk2dHpJLiW6pHdnR7Lm1vOheH+X/8DfHvmZluTWxhayZDWnYB7mcVIC27EOk5BShjPHGpVr3RHv7NHKqtz0Q1iWGHiEhLSfI8zNkVqZo8TymA2b9F4veIe2jragUvJws0czSHh60p9PXKPkNSEYVSIC4tBzeTM3EzKQs3kzNxLTETyZn5qmMEgL+i00otL5EA9UwMYWtmCFszGWzNZDA20MOOsIQSg4xbOHNCVdJdWoedEydOoHfv3tXRFyKiOuF87AOUNvf8udgHOBf7QPXcUF+KJvZmaOZkAS9Hc9V/bcxkANQvRRnp6+HGE6HmRlIWbqVkoeCRUqM+vd7JFe3drWFrLoOtmSHszGSwNjUsNWx5u1txkDG9ULReLkImk8HFxQXjx4/HuHHj4OrqWl19e264XAQRaSojtxADvzuD+Id5atulEmDmf5rinjwfN5MyEZWcpXbb9pPszGWwMjZAdGp2he0ZG+jB09EczZzM4eVoATszQ0zdGv7MSy4kyfM4yJjqvGpbLiIxMRE//vgjNm/ejPnz5+Oll17CxIkTMXjwYBgaGj5Tp4mIarPCR0q8/eMlxD/Mg5WxATLzi6AUUJ0deXLMjlIp8M/DPNXZmhtJmbiZnIm76bm4n1U8tuZpzlZGaFXfEl6OFqpw42ZtAqlUonZccMGjZz4zw0HG9CJ5poVAL1++jI0bN2Lr1q0AgNdffx0TJ05EmzZtqqyDzwPP7BBRRYQQmLXzKn67/A/MZPr47R1fWBjra312JKfgEX67/A/m/X69xL6tk7rAp5GNRvXwzAyR5t/flR85B8Db2xtBQUGYOnUqsrOzsWHDBrRv3x7du3fH9eslf5GJiOqqlSdi8Nvlf6AnlWDlaG94OprDydIYPo1stAobpjJ9/Ke5A546WfPvTMQmGtdTmbaJXlSVCjtFRUX49ddf8fLLL8Pd3R2HDh3CihUrkJKSgpiYGLi7u2P48OFV0sHExES88cYbsLGxgbGxMVq1aoWwsDDVfiEE5s2bBycnJxgbG8Pf3x/R0dHl1EhEuuDzzz+HRCJRe3h5eQEA7ty5U2Lf48fOnTvLrPPNN98scXzfvn3xx5V7WHL4FgDg7bbG+PbDt2BrawsLCwt069YNJ06c0KrvTpbGCB7aCnr/TtbHQcJE1UvrMTvvvfcetm7dCiEExowZg8WLF6Nly5aq/aampliyZAmcnZ2fuXMPHz5E165d0bt3bxw4cAB2dnaIjo5GvXr1VMcsXrwYy5cvx+bNm+Hh4YG5c+eiT58++Pvvv2FkVPbKu0RU97Vo0QJHjx5VPdfXL/5Ic3V1RVJSktqxa9euxddff41+/fqVW2ffvn2xceNG1fMbqXl4e8cVAMDEbh74YeZgNGnSBMePH4exsTGWLVuGV155BbGxsXB0dNS475yJmOj50Trs/P333/juu+8wdOhQyGSyUo+xtbXV+i+d0ixatAiurq5qHzweHh6q/xdCYNmyZfj0008xaNAgAMCWLVvg4OCAPXv24LXXXnvmPhBR7aWvr19qwNDT0yuxfffu3RgxYgTMzMzKrVMmk6nKxj/IxazfI1H4SAn/Zg6Y3MkO86KjsX79erRu3RoAsHDhQnz//fe4du2aVmEH4CBhoudF68tYx44dw6hRo8oMOkDxB1DPnj2fqWMAsHfvXnTo0AHDhw+Hvb092rVrh3Xr1qn2x8XFITk5Gf7+/qptlpaW6Ny5M0JCQsqst6CgAJmZmWoPIqp7oqOj4ezsjIYNG2L06NGIj48v9bhLly4hIiICEydOrLDOkydPwt7eHk2aNoXvgFFITUtDy/oWWD6qLeztbOHp6YktW7YgJycHjx49wpo1a2Bvb4/27dtX9csjoiqiddgJDg7Ghg0bSmzfsGEDFi1aVCWdeuz27dtYtWoVmjRpgkOHDuGdd97B+++/j82bNwMAkpOTAQAODupTnDs4OKj2lSY4OBiWlpaqhy7MFUT0ouncuTM2bdqEgwcPYtWqVYiLi0P37t2RlZVV4tj169ejWbNm8PX1LbfOvn37YsuWLTh4+Aic/jMRqbfC8fC3+Vj7hjdMDPUhkUhw9OhRhIeHw9zcHEZGRvj2229x8OBBtcvrRFS7aB121qxZoxoE+KQWLVpg9erVVdKpx5RKJby9vbFgwQK0a9cOkydPxqRJk565naCgIMjlctUjISGhinpMRM9Lv379MHz4cLRu3Rp9+vTBn3/+iYyMDOzYsUPtuLy8PPzyyy8andV57bXXMGDAAGyPkSDevCXcRs1HTmIUblw+D6D40nlgYCDs7e3x119/4eLFixg8eDAGDBhQYowQEdUeWoed5ORkODk5ldhuZ2dX5b/sTk5OaN68udq2Zs2aqU5VP74+npKSonZMSkpKudfOZTIZLCws1B5EVLdZWVmhadOmiImJUdv+66+/Ijc3F2PHjtWonjWnb2N7WAKkEmBt4MuwtbVV1Xn8+HHs27cP27ZtQ9euXeHt7Y3vv/8exsbGqjPORFT7aB12XF1dcfbs2RLbz549WyV3YD2pa9euiIqKUtt269YtuLu7AygerOzo6Ihjx46p9mdmZuLChQvw8fGp0r4QUe2WnZ2N2NjYEn+MrV+/HgMHDoSdnV2FdRyITMLCAzcBAPNeaY6mZkV48OCBqs7c3FwAgFSq/tEplUqhVGq2hhUR1QChpUWLFgkbGxuxYcMGcefOHXHnzh2xfv16YWNjIxYsWKBtdeW6ePGi0NfXF1999ZWIjo4WP//8szAxMRE//fST6piFCxcKKysr8fvvv4urV6+KQYMGCQ8PD5GXl6dxO3K5XAAQcrm8SvtPRNXngw8+ECdPnhRxcXHi7Nmzwt/fX9ja2orU1FTVMdHR0UIikYgDBw6UWoenp6fYtWuXEEKIszcSRL0uw4TjG0vE+2sOiqNHjwpvb2/RpEkTkZ+fL4QQ4v79+8LGxkYMHTpUREREiKioKDFr1ixhYGAgIiIiqv9FE5EaTb+/tQ47SqVSfPTRR8LIyEhIpVIhlUqFiYmJmD9/fqU7W54//vhDtGzZUshkMuHl5SXWrl1boj9z584VDg4OQiaTCT8/PxEVFaVVGww7RHXPyJEjhZOTkzA0NBT169cXI0eOFDExMWrHBAUFCVdXV6FQKEqtA4DYuHGjSEjPEe3m7RNGDdoJI/N6wsDAQLi7u4tJkyaJ5ORktTKhoaEiICBAWFtbC3Nzc9GlSxfx559/VtvrJKKyafr9Xem1sbKzs3Hjxg0YGxujSZMm5d6KXttxbSyiuitJnoe4tBx42JpqPWdNkjwP1+9lYsH+G7idlgMvR3P8+o4vzGRaT0FGRDWg2lY9f8zMzAwdO3asbHEiome2PTQeQbsioRSAVAJ81McTr7TRbOzgviv3sPhQFJT//rlnLtPHhjc7MugQ6aBK/VaHhYVhx44diI+PR2Fhodq+Xbt2VUnHiIjKExKbhtm/RaqeKwWw8GAUFh6MKqdU2XIKH0Eiqfg4Iqp7tL4ba9u2bfD19cWNGzewe/duFBUV4fr16zh+/DgsLS2ro49ERACK57k5F5uGCZtCMWrdhVKPMZBKINOXlvsweHrJcRSHpTtpudX9EoioBmh9ZmfBggVYunQpAgMDYW5ujv/973/w8PDA22+/Xer8O0REz6pIocSfkUlY99dtXEsse3kXPYkEp2f3rnDsTpI8D10XHlddwnpctoGtSVV1mYhqEa3P7MTGxqJ///4AAENDQ+Tk5EAikWDGjBlYu3ZtlXeQiF5cmflFWHs6Fj0Wn8C0bRG4lpgJIwMp3ujihhOzemHRsFbQ+/fak55EggVDW2o0SNnJ0hjBQytXlojqHq3P7NSrV0+19kz9+vVx7do1tGrVChkZGaoJt4iInkViRh42nonDttAEZBc8AgDYmskwzscdo7u4w9rUEADgYWuKHk3tcCctFw1sTbQKKyM7ulW6LBHVLVqHnR49euDIkSNo1aoVhg8fjmnTpuH48eM4cuQI/Pz8qqOPRKTDnrx1/H5WAdb9FYc/I5Og+PcaUxN7M7zV3QOD2taHkYFeifJOlsaVDirPUpaI6g6tw86KFSuQn58PAPjkk09gYGCAc+fOYdiwYfj000+rvINEpLuevHX8aV0b2+Ct7g3Rs4kdpKUMKCYi0pRWYefRo0fYt28f+vTpA6B4PZg5c+ZUS8eISLfdy8jFnF2ReHpa074tHPGeX2O0cObdnURUNbQaoKyvr48pU6aozuwQEVXG1X8yMGnLpRJBBwDG+TZg0CGiKqX1ZaxOnTohIiJCtfI4EZGm7qTlYMnhKOy7mlTqft7+TUTVQeuw8+6772LmzJlISEhA+/btYWpqqra/devWVdY5ItINadkFWH4sGr9ciMcjpYBEAgxpWx9NHMyw5NAtKITg7d9EVG20XghUKi155UsikUAIAYlEAoVCUWWde164EChR9cgueIQf/rqNdadvI6ew+LOhZ1M7zO7rhebOxb9rSfI83v5NRJVSbQuBxsXFPVPHiEj3FSmU2HYxHv87Fo207OL181q7WGJOXy/4NrZVO5a3fxNRddM67HCsDhE97fFcOQ1sTHA5PgNLDkXhzoPiSUbdbUzwYR9P9G/lBAlX2iSiGqB12NmyZUu5+8eOHVvpzhBR3VPWXDm2ZoaY5tcEr3Vyg4Ge1ivTEBFVGa3H7NSrV0/teVFREXJzc2FoaAgTExOkp6dXaQefB47ZIaqc0hbUBICJ3Rpg5n88YSrT+u8pIiKNafr9rfWfWw8fPlR7ZGdnIyoqCt26dcPWrVufqdNEVHcUPlJi2ZHoUmc/9m/myKBDRLVGlXwaNWnSBAsXLsQbb7yBmzdvVkWVRFSLXbr7EB/vikRUSlaJfZwrh4hqmyr700tfXx/37t2rquqIqBbKzC/C4oM38fOFeAgBWJsa4j/N7fFr2D9QCHCuHCKqlbQOO3v37lV7LoRAUlISVqxYga5du1ZZx4io9hBC4OC1ZHy29zpSswoAAK+2d8HHLzeDtakhpvs35Vw5RFRraR12Bg8erPZcIpHAzs4OL730Er755puq6hcR1RL3MvIw7/drOHojFQDgYWuKrwa3VJsvh3PlEFFtpnXYUSqV1dEPIqplFEqBzefuYMnhKOQWKmCgJ8GUno0Q2LsxjAz0arp7REQa4+QXRLXMwoULIZFIMH36dNW25ORkjBkzBo6OjjA1NYW3tzd+++23Z6qzvHqvJcox5Puz+GLf38gtVKC9ez3sf787PgjwZNAhojpH6zM7w4YNQ6dOnTB79my17YsXL0ZoaCh27txZZZ0jetGEhoZizZo1JRbUHTt2LDIyMrB3717Y2tril19+wYgRIxAWFoZ27dpVqs4n693wyw7kSEwQcngvho8YAedxy6Bv3xDmRvqY088Lozq6QSrl7MdEVDdpfWbn9OnTePnll0ts79evH06fPl0lnSJ6EWVnZ2P06NFYt25dick7z507h/feew+dOnVCw4YN8emnn8LKygqXLl2qdJ2P6+3Y/3VMPZKFDw+lYJfoDImhKfKSotG/lROOzeyJ0Z3dGXSIqE7TOuxkZ2fD0NCwxHYDAwNkZmZWSaeIXkSBgYHo378//P39S+zz9fXF9u3bkZ6eDqVSiW3btiE/Px+9evWqdJ0A0L5TF2z88RcU5WZBCCVy/j4FoSjE528Px8rR3rC3MKqKl0ZEVKO0vozVqlUrbN++HfPmzVPbvm3bNjRv3rzKOkb0Itm2bRsuhl3C9zsPIkmeV2L/jh07MHLkSNjY2EBfXx8mJibYvXs3GjduXG6dly9fRmhoaKn7UzPz4frqx7j49Sz8s3wUINWDRF8GuyGfoEMr/i4Tke7QOuzMnTsXQ4cORWxsLF566SUAwLFjx7B161aO1yGqhISEBEwJfA9mgz/D+C1XIJUARv/OZfPY3LlzkZGRgaNHj8LW1hZ79uzBiBEj8Ndff6FVq1al1jlt2jQcOXIERkbqZ2ey8ouw7vRtrPsrDol/roSyIAf2I/8LPRML5N46j7TfFyFvWl+gkU21vm4ioudF64VAAWD//v1YsGABIiIiYGxsjNatW+Ozzz5Dz549q6OP1Y4LgVJN2vDzdkx84zVA8sRVZaGERCKBVCpFVFQUGjdujGvXrqFFixaqQ/z9/dG4cWOsXr26RJ179uzBkCFDoKf3/3dOKRQKSCQSCIkEbh/sxiN5Ku6tnYT3v/8d++L1oRACehIJTI4tQFfvlqXWS0RUm2j6/V2p5SL69++P/v37V7pzRPT/Hlo2hdOEFWrbHvz5PxjZumDIm4E4F5UIAJBK1YfY6enplTnvlZ+fHyIjIwEASqXAyVup+GRGIISlMyw6D0MjewsM72KGd9cCU3o1wRznBqoZkN+8/D/Op0VEOkXrAcqhoaG4cOFCie0XLlxAWFhYlXSK6EUghMDKEzH47sw9GNo1UHtIDGQQMnOcvG+ET088hJFNfbw8fAx2HDiB2NhYfPPNNzhy5IjajOZ+fn5YsaI4NJmbm6Nly5Z4KHNA0MkMfBuWjyKJAUwtrPD15FdwaEYPvDWgOxo3boy3334bCVGRsEcGfvnh+xL1EhHVdVqHncDAQCQkJJTYnpiYiMDAwCrpFJGuyy18hKlbw/H1oSgAQJeG1nh8d7eeRIKGdmZ4uZUTRnVyhYWpEayHzkNqkQyjXh0Kz+YtsXTVD/jfqh/UpoGIjY1FWloaAOBaohxj1l/AmPUXcf1eJsxk+nC1NsGr7V0wurM7DPSkMDAwwJ9//gk7OzsMGDAArVu3xpYtW7B58+ZSp5cgIqqrtB6zY2ZmhqtXr6Jhw4Zq2+Pi4tC6dWtkZWVVaQefB47ZoecpIT0Xk3+8hBtJmTDQk2D+wJZ4vbMbkuR5pS6mmV+kwMmoVOy6nIgTUakoUhT/ykokgG8jGwxuWx99Wzoiu+ARLtxOx/6rSThyIwUAYKAnwRtd3DG1d2PYmMlq5PUSEVWXahuzI5PJkJKSUiLsJCUlQV+/UkOAiF4YIbEP8O7Pl/Awtwi2ZoZY9UZ7dGxgDaDsxTSNDPTQt6UT+rZ0QkZuIfZHJmFPeCJC7zzE2ZgHOBvzAEG7I/FIof53y+C2zvggwBOu1ibP5bUREdVWWp/ZGTVqFJKSkvD777/D0tISAJCRkYHBgwfD3t4eO3bsqJaOViee2aHqJkTxoppf7r8BhVKgVX1LrBnTHs5WlV8pPCE9F3vCE7EzLAHxD9Xn5pFKgLNzXuJK5ESk06rtzM6SJUvQo0cPuLu7q9bkiYiIgIODA3788cfK95hIRxU8UmDunmvYEfYPAGBIu/oIHtrqmRfUdLU2wXt+TdDevR5e/0H9pgGlAO6k5TLsEBGhEmGnfv36uHr1Kn7++WdcuXIFxsbGGD9+PEaNGgUDA4Pq6CNRnZWamY+3f7qE8PgMSCVAUL9meKu7BySSqltrysPOFFJJccB5TE8iQQNbXr4iIgIqcTcWAJiammLy5MlYuXIllixZgrFjxzLoUAkLFy6ERCLB9OnTS+wTQqBfv36QSCTYs2dPufV8/vnn8PLygqmpKerVqwd/f3+16Q/u3LmDiRMnwsPDA8bGxmjUqBE+++wzFBYWVvEr0k54/EMMWHEG4fEZsDDSx6bxnTCpR8MqDTpA8Vif4KGtoPdvvXoSCRYMbcmzOkRE/6r0iOK///4b8fHxJb5QBg4c+MydorovNDQUa9asQevWrUvdv2zZMo2/9Js2bYoVK1agYcOGyMvLw9KlSxEQEICYmBjY2dnh5s2bUCqVWLNmjWqm4UmTJiEnJwdLliypypelsZ1hCfhk9zUUKpRoYm+GdWM7oIGtabW1N7KjG3o0tSv1bi4iohed1gOUb9++jSFDhiAyMrJ46nnx+DbY4i8uhUJR9b2sZhygXLWys7Ph7e2N77//Hv/973/Rtm1bLFu2TLU/IiICr7zyCsLCwuDk5ITdu3drNYnd43+vo0ePws/Pr9Rjvv76a6xatQq3b99+xlejnYT0HHy1/wYOXi++9TuguQO+HdkWZjLeqUhEVNU0/f7W+jLWtGnT4OHhgdTUVJiYmOD69es4ffo0OnTogJMnTz5Ln0lHBAYGon///vD39y+xLzc3F6+//jpWrlwJR0dHresuLCzE2rVrYWlpiTZt2pR5nFwuh7W1tdb1P4s1p2LRffFJVdDxa2aP1W+0Z9AhIqphWn8Kh4SE4Pjx47C1tYVUKoVUKkW3bt0QHByM999/H+Hh4dXRT6ojtm3bhsuXLyM0NLTU/TNmzICvry8GDRqkVb379u3Da6+9htzcXDg5OeHIkSOwtbUt9diYmBh89913z+0Sljy3CIsP38TP5+PVtp+8eR8pWfm8pEREVMO0DjsKhQLm5uYAAFtbW9y7dw+enp5wd3dHVFRUlXeQ6o6EhARMmzYNR44cgZGRUYn9e/fuxfHjxysViHv37o2IiAikpaVh3bp1GDFiBC5cuAB7e3u14xITE9G3b18MHz4ckyZNqvRr0UR+kQKbz93ByhMxyMx/VGK/Qgje/k1EVAtofRmrZcuWuHLlCgCgc+fOWLx4Mc6ePYsvvviixKzK9GK5dOkSUlNT4e3tDX19fejr6+PUqVNYvnw59PX1ceTIEcTGxsLKykq1HwCGDRuGXr16lVu3qakpGjdujC5dumD9+vXQ19fH+vXr1Y65d+8eevfuDV9fX6xdu7a6XiYUSoGdYQl4aclJBB+4icz8R2hoa4qnx1vz9m8iotpB6zM7n376KXJycgAAX3zxBV555RV0794dNjY22L59e5V3kOoOPz8/REZGqm0bP348vLy8MHv2bNja2uLtt99W29+qVSssXboUAwYM0KotpVKJgoIC1fPExET07t0b7du3x8aNGyGVVmpWhXIJIXD8ZioWHbyJWynZAABnSyPMDPDEkHb18eulBHy86xoUQvD2byKiWkTrsNOnTx/V/zdu3Bg3b95Eeno66tWrV+Xzh1DdYm5ujpYtW6ptMzU1hY2NjWp7aYOS3dzc4OHhoXru5eWF4OBgDBkyBDk5Ofjqq68wcOBAODk5IS0tDStXrkRiYiKGDx8OoDjo9OrVC+7u7liyZAnu37+vqqsyg6BLczn+IRYeuImLcekAAEtjAwT2boSxPg1UMyHz9m8iotqpSm4Ted53vZBui4qKglwuBwDo6enh5s2b2Lx5M9LS0mBjY4OOHTvir7/+QosWLQAAR44cQUxMDGJiYuDi4qJWl5YzK5QQez8bXx+MwsHryQAAmb4U47t64J2ejWBpUnIizbIW8yQiopqj9Tw7uojz7NQuSfI8xKXlwMPW9LkHh8dtm8v0sTU0AdtDE6BQCkglwKvtXTDdv+kzLd5JRERVp9oWAiWqTttD4xG0KxJKUbxyd/DQVhjZ0e25t/0k/2b2+LCPFzwdzZ9LP4iIqGox7FCtkSTPw5xdkXh8rlEpgDm/RcLBwgg9mthBKq36MWEKpcC1RDkOXEvC6lMlZ1teNdob/Vo5VXm7RET0/Ggddk6fPg1fX1/VbcOPPXr0COfOnUOPHj2qrHP0Yvnt0j94+qKqAPDmxlBYmxrCt5ENujexRbcmdqj/DJeSEtJzcSYmDWei03A2Ng0ZuUVlHmtlYljpdoiIqHbQOuz07t0bSUlJJSZzk8vl6N27d51cG4tq3vbQeHx75Fap+0wM9ZCeU4h9V5Ow72oSAKChrSm6NbFFt8a28GlkA3Oj4sHCpY33kecVIST2Ac7E3MeZ6DTceZCrVr+5TB9t3axwJjoNT2YtzpNDRKQbtA47QohSbzF/8OABTE2rb1Vn0k1CCKw4HoNv/g06Hdzr4XL8QygFVHPVDPV2wZWEDJyOTsOZ6Pu48o8ct9NycDstB1tC7kJPKkE7VytYmRjg2M1UCAFIJMBLXvZ4mFOIK//IoXhiII6eVIK2rlbo3sQW3ZvYoo2LFfT1pNgeGs95coiIdJDGd2MNHToUAPD777+jb9++kMlkqn0KhQJXr16Fp6cnDh48WD09rUa8G6tmKJQCn++9jh/P3wUABPZuhFkBnkjOzC93rprM/H/P1ESn4a/o+yXO1JTmyTNBXRrZwMKo5G3jQPGZIc6TQ0RUN1T53ViWlpYAiv8SNzc3h7Hx/38RGBoaokuXLtW+FhHpjvwiBWZsj8CBa8mQSIDPB7TAON8GACqeq8bCyAB9WjiiT4viCQMT0nOxOeQOfvgrrsSxk3t4YJyvh8ZjfDhPDhGR7tE47GzcuBEA0KBBA8yaNYuXrKjS5HlFmLQlDBfj0mGoJ8XSkW3Rv3Xl73hytTbBxG4e2HAmTu22cT2JBOO7ejC8EBG94LReQOijjz5SG7Nz9+5dLFu2DIcPH67SjpFuSpbnY+SaEFyMS4e5TB+bJnR8pqDzmJOlMYKHtoLev+9NjrkhIqLHtB6gPGjQIAwdOhRTpkxBRkYGOnXqBENDQ6SlpeHbb7/FO++8Ux39JB0Qk5qNcRsuIjEjD/bmMmwa3wnNnatujBTXpiIiotJofWbn8uXL6N69OwDg119/haOjI+7evYstW7Zg+fLlVd5B0g2X7j7Eq6vPITEjDw1tTfHbO75VGnQec7I0hk8jGwYdIiJS0frMTm5uLszNi6fNP3z4MIYOHQqpVIouXbrg7t27Vd5BqvuO3UhB4C+XkV+kRBtXK2x8syOsTTlZHxERPR9an9lp3Lgx9uzZg4SEBBw6dAgBAQEAgNTUVN62TSXsCE3A5B8vIb9Iid6edtg6qTODDhERPVdan9mZN28eXn/9dcyYMQMvvfQSfHx8ABSf5WnXrl2Vd5DqniR5HuLu5+DUrVSsOV18O/ir7V0QPLQVDPS0ztdERETPROtvnldffRXx8fEICwvDoUOHVNv9/PywdOnSKu3c0xYuXAiJRILp06ertuXn5yMwMBA2NjYwMzPDsGHDkJKSUq39oLJtD41H14XH8foPF1RB591ejfD1q60ZdIiIqEZU6tvH0dER5ubmOHLkCPLy8gAAHTt2hJeXV5V27kmhoaFYs2YNWrdurbZ9xowZ+OOPP7Bz506cOnUK9+7dU832TM9XkjwPQbsi1ea6kUiAMT7upS4xQkRE9DxoHXYePHgAPz8/NG3aFC+//DKSkooXZpw4cSI++OCDKu8gAGRnZ2P06NFYt24d6tWrp9oul8uxfv16fPvtt3jppZfQvn17bNy4EefOncP58+erpS9UusJHSnxz+JZa0AEAIYA7aRUv50BERFRdtA47M2bMgIGBAeLj42Fi8v8rQo8cObLa1sUKDAxE//794e/vr7b90qVLKCoqUtvu5eUFNzc3hISElFlfQUEBMjMz1R5UeaF30vHy8r/w66V/SuzjyuFERFTTtB6gfPjwYRw6dAguLi5q25s0aVItt55v27YNly9fRmhoaIl9ycnJMDQ0hJWVldp2BwcHJCcnl1lncHAw5s+fX9VdfeHIc4uw8OBNbL0YDwCwNTOEfzMH7AxLgEJwFmMiIqodtA47OTk5amd0HktPT1dbCb0qJCQkYNq0aThy5AiMjIyqrN6goCDMnDlT9TwzMxOurq5VVr+uE0Jg39UkzP/jb6RlFwAAXuvoijn9vGBlYohp/k04izEREdUaWoed7t27Y8uWLfjyyy8BABKJBEqlEosXL0bv3r2rtHOXLl1CamoqvL29VdsUCgVOnz6NFStW4NChQygsLERGRoba2Z2UlBQ4OjqWWa9MJqvyYPai+OdhLubuuYYTUfcBAA3tTBE8pBU6N7RRHcOVw4mIqDbROuwsXrwYfn5+CAsLQ2FhIT766CNcv34d6enpOHv2bJV2zs/PD5GRkWrbxo8fDy8vL8yePRuurq4wMDDAsWPHMGzYMABAVFQU4uPjVfP/UNV4pFBi07k7+ObwLeQVKWCoJ8W7vRvhnV6NINPXq+nuERERlUnrsNOyZUvcunULK1asgLm5ObKzszF06FAEBgbCyenZV69+krm5OVq2bKm2zdTUFDY2NqrtEydOxMyZM2FtbQ0LCwu899578PHxQZcuXaq0Ly+yyH/kCNp9FdcSiwdyd/KwxoIhrdDY3qyGe0ZERFQxrcNOfHw8XF1d8cknn5S6z83NrUo6pqmlS5dCKpVi2LBhKCgoQJ8+ffD9998/1z7ooiR5Hm4kZeLw9RTsCEuAUgCWxgb4+GUvDG/vCqmU8+YQEVHdIBFCiIoP+396enpISkqCvb292vYHDx7A3t4eCoWiSjv4PGRmZsLS0hJyuZzre6F4FuQ5uyLx5DtjYBtnzH2lOezMOdaJiIhqB02/v7U+syOEKHU23Ozs7Cq9Y4pqRtz9HMz+TX2clFQCBL3sxaBDRER1ksZh5/Gt2hKJBHPnzlW7/VyhUODChQto27ZtlXeQnp+/72Xirc0l5zNS/jsLMu+wIiKiukjjsBMeHg6g+MxOZGQkDA0NVfsMDQ3Rpk0bzJo1q+p7SNVOCIGNZ+9g4YGbKFQoS+znLMhERFSXaRx2Tpw4AaD41u///e9/HNuiI+5nFeDDX6/g5L/z5vg3c4BvIxt8tf8GFEJwFmQiIqrztB6grIte1AHKp27dxwc7riAtuwAyfSk+faU53ujsBolEgiR5HmdBJiKiWq3aBihT3VfwSIGvD0bhhzNxAABPB3MsH9UOno7mqmM4CzIREekKhp0XTExqNt7fGo6/k4onCHzTtwHm9POCkQFnQSYiIt3EsPOCEEJge2gC5v/xN/KKFLA2NcTXr7aGXzOHmu4aERFRtWLYeQFk5BYiaFckDlxLBgB0b2KLb4a3gb0F50UiIiLdx7Cjo5LkeYhLy0FGTiG+3H8DSfJ8GOhJ8GEfT7zVrSGXeyAiohcGw44O2h4aj6BdkVA+cZ+dh60plr/WDq1cLGuuY0RERDWAYUfHJMnzSgQdCYAfxnZAI65STkRELyBpTXeAqlZUUpZa0AEAASA1q6BG+kNERFTTGHZ0SG7hIyw/Hl1iO5d7ICKiFxnDjo7ILXyECZtCcTk+AzJ9KR6PP+ZyD0RE9KLjmB0dkFPwCOM3heJiXDrMZfrYPLETnCyNuNwDERERGHbqvOyCRxi/8SJC7zyEuUwfWyZ2Qju3egDAkENERASGnTotK78Ib24MxaW7D2FupI+fJnZGG1ermu4WERFRrcKwU0dl5hfhzQ0XcTk+AxZG+vjprc5o7WJV090iIiKqdRh26qDM/CKMXX8REQkZsDQ2wM9vdUbL+pwskIiIqDQMO3WMPK8IYzdcxJWEDFiZGOCniQw6RERE5WHYqUPkuUUYs+ECrv4jRz0TA/z8Vhc0d7ao6W4RERHVagw7dURGbiHeWH8B1xIzYW1qiJ/f6oxmTgw6REREFWHYqQMe5hRi9A8X8HdSJmxMDfHLpC7wdDSv6W4RERHVCQw7tViSPA9X/5Hj60NRiEnNhq1ZcdBp6sCgQ0REpCmGnVpqe2i82urlZjJ9bJ3UBU0YdIiIiLTCtbFqoSR5nlrQAYrXvjIzYjYlIiLSFsNOLRSXlqMWdABAKYA7abk10yEiIqI6jGGnFnKxKrmmlZ5Egga2JjXQGyIiorqNYacWCrv7UO25nkSCBUNbcmFPIiKiSuAgkFpGCIEf/ooDAEzp2RA9m9qjga0Jgw4REVElMezUMiG3H+DvpEwYGUjxdo9GqGdqWNNdIiIiqtN4GauWWf/vWZ1X27sw6BAREVUBhp1a5Pb9bBy7mQoAGN/Vo4Z7Q0REpBsYdmqRDWeLz+r4edmjkZ1ZDfeGiIhINzDs1BIPcwrx66V/AAATu/OsDhERUVVh2KklfrkYj/wiJZo7WcCnoU1Nd4eIiEhnMOzUAoWPlNh87g4A4K3uHpBIJDXbISIiIh3CsFML7Lt6D6lZBbA3l+GV1s413R0iIiKdwrBTw4QQWH+meGDyON8GMNTnPwkREVFV4jdrDTt/Ox3X7xVPIvh6J7ea7g4REZHOYdipYevP3AbASQSJiIiqC8NODbp9PxtHb3ASQSIiourEsFODNp69A4CTCBIREVUnhp0akpFbiJ2XEgBwEkEiIqLqxLBTQ36+wEkEiYiIngeGnRpQ+EiJLSF3AAATu3ESQSIiourEsFMD9kfeQ0pm8SSCA9pwEkEiIqLqxLDznAkh8MNfnESQiIjoeeE37XPGSQSJiIieL4adUgQHB6Njx44wNzeHvb09Bg8ejKioKLVj1q5di169esHCwgISiQQZGRka1TvwP90Rv3Q4EpaPxvjRI6qkXiIiIiobw04pTp06hcDAQJw/fx5HjhxBUVERAgICkJOTozomNzcXffv2xccff6xxvQeOHIdo1geObyzB9j37q6xeIiIiKptECCFquhM1LTMzE5aWlpDL5bCwsCix//79+7C3t8epU6fQo0cPtX0nT55E79698fDhQ1hZWZXbztw91/Dj+bvw87LH+jc7Vlm9REREL6KKvr8f45kdDcjlcgCAtbV1pevIyC3Er5f+AfD/kwhWRb1ERERUPoadCiiVSkyfPh1du3ZFy5YtK13PLxfjkVekQLN/JxGsqnqJiIiofPo13YHaLjAwENeuXcOZM2cqXUfhIyU2n7sDAHjr30kE33333Weul4iIiCrGsFOOqVOnYt++fTh9+jRcXFwqXc+fkUlqkwhWVb1ERERUMYadUggh8N5772H37t04efIkPDwqv1CnEAI/nLkNABjr446Z09+vknqJiIhIMww7pQgMDMQvv/yC33//Hebm5khOTgYAWFpawtjYGACQnJyM5ORkxMTEAAAiIyNhbm4ONzc31YBjPz8/tO0egGv5LWFkIEX41iX4bef2Z66XiIiINMdbz1Hy1rWyFubcuHEj3nzzTQDA559/jvnz55d7jKubO8xa+SOv1VCM7uyGBUNbV0m9REREpPmt5ww70PyHpY3tofEI2hUJ5b8/3Q8CmuK9l5pUSd1ERETEeXZqVJI8Ty3oAMCyI9FIkufVXKeIiIheUAw71SAuLUct6ACAQgjcScutmQ4RERG9wGp12NFkQc78/HwEBgbCxsYGZmZmGDZsGFJSUmqox8U8bE0hfWrYj55Egga2JjXTISIiohdYrQ47mizIOWPGDPzxxx/YuXMnTp06hXv37mHo0KE12GvAydIYwUNbQe/fgc56EgkWDG0JJ0vjGu0XERHRi6hODVB+euFMuVwOOzs7/PLLL3j11VcBADdv3kSzZs0QEhKCLl26aFRvdQxQBorH7txJy0UDWxMGHSIioiqm6fd3nZpn5+mFMy9duoSioiL4+/urjvHy8oKbm1u5YaegoAAFBQWq55mZmdXSXydLY4YcIiKiGlarL2M9qbSFM5OTk2FoaAgrKyu1Yx0cHFQT9pUmODgYlpaWqoerq2t1dp2IiIhqUJ0JO48X5Ny2bdsz1xUUFAS5XK56JCQkVEEPiYiIqDaqE5exylo409HREYWFhcjIyFA7u5OSkgJHR8cy65PJZJDJZNXZZSIiIqolavWZHSEEpk6dit27d+P48eMlFs5s3749DAwMcOzYMdW2qKgoxMfHw8fH53l3l4iIiGqhWn1mp6IFOS0tLTFx4kTMnDkT1tbWsLCwwHvvvQcfHx+N78QiIiIi3Varbz3XZEHO/Px8fPDBB9i6dSsKCgrQp08ffP/99+Vexnpadd16TkRERNWHC4FqgWGHiIio7uFCoERERERg2CEiIiIdx7BDREREOo1hh4iIiHQaww4RERHpNIYdIiIi0mkMO0RERKTTGHaIiIhIpzHsEBERkU5j2CEiIiKdxrBDREREOo1hh4iIiHQaww4RERHpNIYdIiIi0mkMO0RERKTTGHaIiIhIpzHsEBERkU5j2CEiIiKdxrBDREREOo1hh4iIiHQaww4RERHpNIYdIiIi0mkMO0RERKTTGHaIiIhIpzHsEBERkU5j2CEiIiKdxrBDREREOo1hh4iIiHQaww4RERHpNIYdIiIi0mkMO0RERKTTGHaIiIhIpzHsEBERkU5j2CEiIiKdxrBDREREOo1hh4iIiHQaww4RERHpNIYdIiIi0mkMO0RERKTTGHaIiIhIpzHsEBERkU5j2CEiIiKdxrBDREREOo1hh4iIiHQaww4RERHpNIYdIiIi0mkMO0RERKTTGHaIiIhIpzHsEBERkU5j2CEiIiKdxrBDREREOo1hh4iIiHQaww4RERHpNIYdIiIi0mkMO0RERKTTGHaIiIhIpzHsEBERkU5j2CEiIiKdxrBDREREOo1hh4iIiHQaww4RERHpNIYdIiIi0mkMO0RERKTTdCbsrFy5Eg0aNICRkRE6d+6Mixcv1nSXiIiIqBbQibCzfft2zJw5E5999hkuX76MNm3aoE+fPkhNTa3prhEREVEN04mw8+2332LSpEkYP348mjdvjtWrV8PExAQbNmyo6a4RERFRDdOv6Q48q8LCQly6dAlBQUGqbVKpFP7+/ggJCSm1TEFBAQoKClTP5XI5ACAzM7N6O0tERERV5vH3thCi3OPqfNhJS0uDQqGAg4OD2nYHBwfcvHmz1DLBwcGYP39+ie2urq7V0kciIiKqPllZWbC0tCxzf50PO5URFBSEmTNnqp4rlUqkp6fDxsYGEomkytrJzMyEq6srEhISYGFh8VzLs+3n3/azlmfbL1bbz1qebbPtulL+WdsujxACWVlZcHZ2Lve4Oh92bG1toaenh5SUFLXtKSkpcHR0LLWMTCaDTCZT22ZlZVVdXYSFhcUz/QM/S3m2/fzbftbybPvFavtZy7Nttl1Xyj9r22Up74zOY3V+gLKhoSHat2+PY8eOqbYplUocO3YMPj4+NdgzIiIiqg3q/JkdAJg5cybGjRuHDh06oFOnTli2bBlycnIwfvz4mu4aERER1TCdCDsjR47E/fv3MW/ePCQnJ6Nt27Y4ePBgiUHLz5tMJsNnn31W4pLZ8yjPtp9/289anm2/WG0/a3m2zbbrSvlnbbsqSERF92sRERER1WF1fswOERERUXkYdoiIiEinMewQERGRTmPYISIiIp3GsFONVq5ciQYNGsDIyAidO3fGxYsXNSp3+vRpDBgwAM7OzpBIJNizZ4/GbQYHB6Njx44wNzeHvb09Bg8ejKioKI3Lr1q1Cq1bt1ZN/uTj44MDBw5oXP5JCxcuhEQiwfTp0zU6/vPPP4dEIlF7eHl5adxeYmIi3njjDdjY2MDY2BitWrVCWFiYRmUbNGhQom2JRILAwMAKyyoUCsydOxceHh4wNjZGo0aN8OWXX1a4VsuTsrKyMH36dLi7u8PY2Bi+vr4IDQ0tcVxF7w0hBObNmwcnJycYGxvD398f0dHRGpfftWsXAgICVLOJR0REaNx+UVERZs+ejVatWsHU1BTOzs4YO3Ys7t27p1Hbn3/+Oby8vGBqaop69erB398fFy5c0LjvT5oyZQokEgmWLVumUdk333yzxL993759tWr7xo0bGDhwICwtLWFqaoqOHTsiPj6+wrKlve8kEgm+/vprjdrOzs7G1KlT4eLiAmNjY9ViyJqUTUlJwZtvvglnZ2eYmJigb9++qveLJp8l+fn5CAwMhI2NDczMzDBs2DDVBK+alF+7di169eoFCwsLSCQSZGRkqPZVVD49PR3vvfcePD09YWxsDDc3N7z//vuQy+Uatf3222+jUaNGMDY2hp2dHQYNGqRaYkibz1EhBPr166f6+WpStlevXiX+vadMmaJV2yEhIXjppZdgamoKCwsL9OjRA1988UW5Ze/cuVPm+23nzp0atZ2cnIwxY8bA0dERpqam8Pb2xm+//aZR2djYWAwZMgR2dnawsLDAiBEjSkwIXF0YdqrJ9u3bMXPmTHz22We4fPky2rRpgz59+iA1NbXCsjk5OWjTpg1WrlypdbunTp1CYGAgzp8/jyNHjqCoqAgBAQHIycnRqLyLiwsWLlyIS5cuISwsDC+99BIGDRqE69eva9WP0NBQrFmzBq1bt9aqXIsWLZCUlKR6nDlzRqNyDx8+RNeuXWFgYIADBw7g77//xjfffIN69epp3N8n2z1y5AgAYPjw4RWWXbRoEVatWoUVK1bgxo0bWLRoERYvXozvvvtOo7YB4K233sKRI0fw448/IjIyEgEBAfD390diYqLacRW9NxYvXozly5dj9erVuHDhAkxNTdGnTx/k5+drVD4nJwfdunXDokWLytxfVvnc3FxcvnwZc+fOxeXLl7Fr1y5ERUVh4MCBGrXdtGlTrFixApGRkThz5gwaNGiAgIAA3L9/X6Pyj+3evRvnz59Xmz5ek7J9+/ZVew9s3bpV4/KxsbHo1q0bvLy8cPLkSVy9ehVz586FkZFRhWWfbDMpKQkbNmyARCLBsGHDNGp75syZOHjwIH766SfcuHED06dPx9SpU7F3795yywohMHjwYNy+fRu///47wsPD4e7uDn9/f+Tk5Gj0WTJjxgz88ccf2LlzJ06dOoV79+5h6NChADT7LMrNzUXfvn3x8ccfl+hfReXv3buHe/fuYcmSJbh27Ro2bdqEgwcPYuLEiRq13b59e2zcuBE3btzAoUOHIIRAQEAAFAqFVp+jy5YtU1tmSNOykyZNUvt3X7x4scblQ0JC0LdvXwQEBODixYsIDQ3F1KlTcebMmXLLurq6lni/zZ8/H2ZmZujXr59GbY8dOxZRUVHYu3cvIiMjMXToUIwYMQJ//PFHuWVzcnIQEBAAiUSC48eP4+zZsygsLMSAAQOgVCpL/FyrnKBq0alTJxEYGKh6rlAohLOzswgODtaqHgBi9+7dle5HamqqACBOnTpV6Trq1asnfvjhB42Pz8rKEk2aNBFHjhwRPXv2FNOmTdOo3GeffSbatGlTqT7Onj1bdOvWrVJlSzNt2jTRqFEjoVQqKzy2f//+YsKECWrbhg4dKkaPHq1RW7m5uUJPT0/s27dPbbu3t7f45JNPyiz39HtDqVQKR0dH8fXXX6u2ZWRkCJlMJrZu3Vph+SfFxcUJACI8PFzj9ktz8eJFAUDcvXtX67JyuVwAEEePHtW47X/++UfUr19fXLt2Tbi7u4ulS5dqVHbcuHFi0KBB5fanvPIjR44Ub7zxRqXKPm3QoEHipZde0rh8ixYtxBdffKG2rbT3ztNlo6KiBABx7do11TaFQiHs7OzEunXrSrT99GdJRkaGMDAwEDt37lQdc+PGDQFAhISEVFj+SSdOnBAAxMOHD0t93RWVf2zHjh3C0NBQFBUVaV32ypUrAoCIiYnRuO3w8HBRv359kZSUVOa/bWlltflcLK18586dxaefflqpsk9r27Ztic+v8sqbmpqKLVu2qB1nbW1d4j3zdNlDhw4JqVQq5HK56piMjAwhkUjEkSNHKnwtz4pndqpBYWEhLl26BH9/f9U2qVQKf39/hISEPNe+yOVyAIC1tbXWZRUKBbZt24acnBytlt4IDAxE//791V6/pqKjo+Hs7IyGDRti9OjRiI+P16jc3r170aFDBwwfPhz29vZo164d1q1bp3X7QPG/308//YQJEyZotDCsr68vjh07hlu3bgEArly5gjNnzqBfv34atffo0SMoFAoYGRmpbTc2Ntb4zBYAxMXFITk5We3nbmlpic6dOz/3991jcrkcEolE67XnCgsLsXbtWlhaWqJNmzYalVEqlRgzZgw+/PBDtGjRQuu+njx5Evb29vD09MQ777yDBw8eaNzu/v370bRpU/Tp0wf29vbo3LmzVpefH0tJScH+/fsxceJEjcv4+vpi7969SExMhBACJ06cwK1btxAQEFBuuYKCAgBQe99JpVLIZLJS33dPf5ZcunQJRUVFau83Ly8vuLm5lfp+e5bPIk3Ly+VyWFhYQF9fv8T28srm5ORg48aN8PDwgKurq0Zt5+bm4vXXX8fKlSvLXIexvLZ//vln2NraomXLlggKCkJubq5G5VNTU3HhwgXY29vD19cXDg4O6Nmzp0b/Zk+7dOkSIiIiyny/lVbe19cX27dvR3p6OpRKJbZt24b8/Hz06tWr3LIFBQWQSCRqEwsaGRlBKpVq9TlXadUep15AiYmJAoA4d+6c2vYPP/xQdOrUSau68AxndhQKhejfv7/o2rWrVuWuXr0qTE1NhZ6enrC0tBT79+/XuOzWrVtFy5YtRV5enhBCu79g/vzzT7Fjxw5x5coVcfDgQeHj4yPc3NxEZmZmhWVlMpmQyWQiKChIXL58WaxZs0YYGRmJTZs2adz3x7Zv3y709PREYmKiRscrFAoxe/ZsIZFIhL6+vpBIJGLBggVatenj4yN69uwpEhMTxaNHj8SPP/4opFKpaNq0aZllnn5vnD17VgAQ9+7dUztu+PDhYsSIERWWf1JVnNnJy8sT3t7e4vXXX9e47B9//CFMTU2FRCIRzs7O4uLFixq3vWDBAvGf//xHdTZOmzM7W7duFb///ru4evWq2L17t2jWrJno2LGjePToUYXlH/9Vb2JiIr799lsRHh4ugoODhUQiESdPntTodT+2aNEiUa9ePdXvjyZ9z8/PF2PHjhUAhL6+vjA0NBSbN2+usGxhYaFwc3MTw4cPF+np6aKgoEAsXLhQABABAQFqZUv7LPn555+FoaFhiXY6duwoPvroowrLP6miMzuafJbdv39fuLm5iY8//ljjsitXrhSmpqYCgPD09Cz1rE5Z5SdPniwmTpyoel7av01ZZdesWSMOHjworl69Kn766SdRv359MWTIEI3aDgkJEQCEtbW12LBhg7h8+bKYPn26MDQ0FLdu3dLodT/2zjvviGbNmpW6r6zyDx8+FAEBAar3m4WFhTh06FCFZVNTU4WFhYWYNm2ayMnJEdnZ2WLq1KkCgJg8eXKZfawqDDvVoLaEnSlTpgh3d3eRkJCgVbmCggIRHR0twsLCxJw5c4Stra24fv16heXi4+OFvb29uHLlimqbNmHnaQ8fPhQWFhYaXUIzMDAQPj4+atvee+890aVLF63bDQgIEK+88orGx2/dulW4uLiIrVu3iqtXr4otW7YIa2trrYJWTEyM6NGjhwAg9PT0RMeOHcXo0aOFl5dXmWVqc9gpLCwUAwYMEO3atVM7bV1R2ezsbBEdHS1CQkLEhAkTRIMGDURKSkqF5cPCwoSDg4NaQNUm7DwtNjZW40toj3/fR40apXbcgAEDxGuvvaZV256enmLq1Kll7i+t/Ndffy2aNm0q9u7dK65cuSK+++47YWZmVuLSQGllw8LCRJs2bVTvuz59+oh+/fqJvn37qh1X2meJNmGnos+iisJOReXlcrno1KmT6Nu3rygsLNS4bEZGhrh165Y4deqUGDBggPD29i4RNEsr//vvv4vGjRuLrKws1bbSfr6afgYfO3as1EtopZV//HseFBSkdmyrVq3EnDlzNG47NzdXWFpaiiVLlpS6v6zyU6dOFZ06dRJHjx4VERER4vPPPxeWlpbi6tWrFZY9dOiQaNiwoZBIJEJPT0+88cYbwtvbW0yZMqWcn07VYNipBgUFBUJPT6/EG3/s2LFi4MCBWtVV2bATGBgoXFxcxO3bt7Uu+zQ/Pz+Nkvfu3btVH5qPHwBUb+zS/kquSIcOHdR+gcvi5uam9leWEEJ8//33wtnZWav27ty5I6RSqdizZ4/GZVxcXMSKFSvUtn355ZfC09NTq7aFKP6yfxxWRowYIV5++eUyj336vfH4C/rpgNKjRw/x/vvvV1j+Sc8SdgoLC8XgwYNF69atRVpamlZln9a4ceNSz5I9XX7p0qWq99mT7z2pVCrc3d0r1batra1YvXp1hW0XFBQIfX198eWXX6od99FHHwlfX1+N2z59+rQAICIiIsrs09Plc3NzhYGBQYnxXhMnThR9+vTRuO2MjAyRmpoqhCgeb/juu++q9pX1WfL4C/rpgOLm5ia+/fbbCss/qbywU1H5zMxM4ePjI/z8/EoEFW0+BwsKCoSJiYn45ZdfKiw/bdq0Mt9vPXv21Lrt7OxsAUAcPHiwwrZv374tAIgff/xRbfuIESNUZ1E1aXvLli3CwMBA9e/+pLLKx8TElBjnJUTxd8Tbb7+tcdv3799X/Vs7ODiIxYsXl3lsVeGYnWpgaGiI9u3b49ixY6ptSqUSx44d02rsS2UIITB16lTs3r0bx48fh4eHxzPXqVQqVdf3y+Pn54fIyEhERESoHh06dMDo0aMREREBPT09rdrNzs5GbGwsnJycKjy2a9euJW5zvHXrFtzd3bVqc+PGjbC3t0f//v01LpObmwupVP1XSU9Pr1J3GJiamsLJyQkPHz7EoUOHMGjQII3Lenh4wNHRUe19l5mZiQsXLlT7++6xoqIijBgxAtHR0Th69ChsbGyeqT5N33tjxozB1atX1d57zs7O+PDDD3Ho0CGt2/3nn3/w4MEDjd57hoaG6Nix4zO//9avX4/27dtrPEYJKP55FxUVPfP7z9LSEnZ2doiOjkZYWBgGDRpU4WdJ+/btYWBgoPZ+i4qKQnx8PHx8fJ75s0iT8pmZmQgICIChoSH27t2rGn9UmbZF8R//KCgoqLD8nDlzSrzfAGDp0qXYsGGD1m0/Lu/k5FRh2w0aNICzs3Op7zc3NzeN216/fj0GDhwIOzs7tZ9BeeUfjysq7f2mUCg0btvW1hZWVlY4fvw4UlNTVXdsVqtqj1MvqG3btgmZTCY2bdok/v77bzF58mRhZWUlkpOTKyyblZUlwsPDRXh4uACgGgfw9B0tpXnnnXeEpaWlOHnypEhKSlI9cnNzNer3nDlzxKlTp0RcXJy4evWqmDNnjpBIJOLw4cMalX+aNpexPvjgA3Hy5EkRFxcnzp49K/z9/YWtrW2pf3k87eLFi0JfX1989dVXIjo6Wvz888/CxMRE/PTTTxr3VaFQCDc3NzF79myNywhRfCdP/fr1xb59+0RcXJzYtWuXsLW1LXEqvzwHDx4UBw4cELdv3xaHDx8Wbdq0EZ07dy5xSr6i98bChQuFlZWVavzJoEGDhIeHh+ov3orKP3jwQISHh4v9+/cLAGLbtm0iPDxcJCUlVVi+sLBQDBw4ULi4uIiIiAi1919BQUG5ZbOzs0VQUJAICQkRd+7cEWFhYWL8+PFCJpOp/orU9vfiyctY5ZXNysoSs2bNEiEhISIuLk4cPXpUeHt7iyZNmoj8/HyN2t61a5cwMDAQa9euFdHR0eK7774Tenp64q+//tKo33K5XJiYmIhVq1aVeB0Vle/Zs6do0aKFOHHihLh9+7bYuHGjMDIyEt9//32FZXfs2CFOnDghYmNjxZ49e4S7u7sYOnSoEEKzz5IpU6YINzc3cfz4cREWFiZ8fHxUl5M1KZ+UlCTCw8PFunXrBABx+vRpER4eLh48eFBheblcLjp37ixatWolYmJi1I6ZMmVKuWVjY2PFggULRFhYmLh79644e/asGDBggLC2thYpKSmV+hzFv2fOKiobExMjvvjiCxEWFibi4uLE77//Lho2bCh69Oih8c9t6dKlwsLCQuzcuVNER0eLTz/9VBgZGYnXX39do35HR0cLiUQiDhw4oLa9orYLCwtF48aNRffu3cWFCxdETEyMWLJkiZBIJOLll1+usO0NGzaIkJAQERMTI3788UdhbW0tZs6cWebPtCox7FSj7777Tri5uQlDQ0PRqVMncf78eY3KPT6l+/Rj3LhxFZYtrRwAsXHjRo3anjBhgnB3dxeGhobCzs5O+Pn5VTroCKFd2Bk5cqRwcnIShoaGon79+mLkyJGlDhgsyx9//CFatmwpZDKZ8PLyEmvXrtWqr4cOHRIARFRUlFblMjMzxbRp04Sbm5swMjISDRs2FJ988okoKCjQuI7t27eLhg0bCkNDQ+Ho6CgCAwNFRkZGieMqem8olUoxd+5c4eDgIGQymfDz81N7PRWV37hxY6n7P/vsswrLP770VdrjxIkT5ZbNy8sTQ4YMEc7OzsLQ0FA4OTmJgQMHqg1Q1vb34smwU17Z3NxcERAQIOzs7ISBgYFwd3cXkyZNUvvDRJO2169fLxo3biyMjIxEmzZtVJdCNSm7Zs0aYWxsXKl/86SkJPHmm28KZ2dnYWRkJDw9PcU333wjlEplhWX/97//CRcXF2FgYCDc3NzEp59+qnrfavJZkpeXJ959911Rr149YWJiIoYMGaIKxpqU/+yzz8o8pqLyZb228h6PyyYmJop+/foJe3t7YWBgIFxcXMTrr78ubt68qXHfn/Y47FRUNj4+XvTo0UNYW1sLmUwmGjduLD788EPV2DZN2w4ODhYuLi7CxMRE+Pj4iL/++kvjskFBQcLV1VUoFIoSr6Gi8rdu3RJDhw4V9vb2wsTERLRu3Vps2bJFo7KzZ88WDg4OwsDAQDRp0kT1Pn0eJP++QCIiIiKdxDE7REREpNMYdoiIiEinMewQERGRTmPYISIiIp3GsENEREQ6jWGHiIiIdBrDDhEREek0hh0ioqecPHkSEokEGRkZNd0VIqoCDDtERESk0xh2iIiISKcx7BBRraNUKhEcHAwPDw8YGxujTZs2+PXXXwH8/yWm/fv3o3Xr1jAyMkKXLl1w7do1tTp+++03tGjRAjKZDA0aNMA333yjtr+goACzZ8+Gq6srZDIZGjdujPXr16sdc+nSJXTo0AEmJibw9fUtsdI0EdUNDDtEVOsEBwdjy5YtWL16Na5fv44ZM2bgjTfewKlTp1THfPjhh/jmm28QGhoKOzs7DBgwAEVFRQCKQ8qIESPw2muvITIyEp9//jnmzp2LTZs2qcqPHTsWW7duxfLly3Hjxg2sWbMGZmZmav345JNP8M033yAsLAz6+vqYMGHCc3n9RFS1uBAoEdUqBQUFsLa2xtGjR+Hj46Pa/tZbbyE3NxeTJ09G7969sW3bNowcORIAkJ6eDhcXF2zatAkjRozA6NGjcf/+fRw+fFhV/qOPPsL+/ftx/fp13Lp1C56enjhy5Aj8/f1L9OHkyZPo3bs3jh49Cj8/PwDAn3/+if79+yMvLw9GRkbV/FMgoqrEMztEVKvExMQgNzcX//nPf2BmZqZ6bNmyBbGxsarjngxC1tbW8PT0xI0bNwAAN27cQNeuXdXq7dq1K6Kjo6FQKBAREQE9PT307Nmz3L60bt1a9f9OTk4AgNTU1Gd+jUT0fOnXdAeIiJ6UnZ0NANi/fz/q16+vtk8mk6kFnsoyNjbW6DgDAwPV/0skEgDF44mIqG7hmR0iqlWaN28OmUyG+Ph4NG7cWO3h6uqqOu78+fOq/3/48CFu3bqFZs2aAQCaNWuGs2fPqtV79uxZNG3aFHp6emjVqhWUSqXaGCAi0l08s0NEtYq5uTlmzZqFGTNmQKlUolu3bpDL5Th79iwsLCzg7u4OAPjiiy9gY2MDBwcHfPLJJ7C1tcXgwYMBAB988AE6duyIL7/8EiNHjkRISAhWrFiB77//HgDQoEEDjBs3DhMmTMDy5cvRpk0b3L17F6mpqRgxYkRNvXQiqiYMO0RU63z55Zews7NDcHAwbt++DSsrK3h7e+Pjjz9WXUZauHAhpk2bhujoaLRt2xZ//PEHDA0NAQDe3t7YsWMH5s2bhy+//BJOTk744osv8Oabb6raWLVqFT7++GO8++67ePDgAdzc3PDxxx/XxMslomrGu7GIqE55fKfUw4cPYWVlVdPdIaI6gGN2iIiISKcx7BAREZFO42UsIiIi0mk8s0NEREQ6jWGHiIiIdBrDDhEREek0hh0iIiLSaQw7REREpNMYdoiIiEinMewQERGRTmPYISIiIp3GsENEREQ67f8AmeGyZWBdfUUAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plt.plot(np.arange(len(epochs_x)), epochs_acc, marker = '.')\n", + "plt.xlabel('epoch')\n", + "plt.ylabel('test accuracy')\n", + "plt.ylim(0, 100)\n", + "plt.xticks(np.arange(len(epochs_x)))\n", + "for i, txt in enumerate(epochs_acc):\n", + " if i%5 ==0 or i == epochs-1:\n", + " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", + "plt.show()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "speck-rescnn", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.0" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tests/test_nonsequential/using_SumPool2d/models/ResCSNN3.py b/tests/test_nonsequential/using_SumPool2d/models/ResCSNN3.py new file mode 100644 index 00000000..fa6bfd46 --- /dev/null +++ b/tests/test_nonsequential/using_SumPool2d/models/ResCSNN3.py @@ -0,0 +1,122 @@ +import torch.nn as nn +import sinabs.layers as sl +from sinabs.exodus.layers import IAFSqueeze + +class ResCSNN3(nn.Module): + def __init__(self, input_size, nb_classes, batch_size, surrogate_fn, min_v_mem=-1.0, spk_thr=1.0) -> None: + super().__init__() + + self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False) + self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool1 = sl.SumPool2d(2,2) + self.pool1a = sl.SumPool2d(6,6) + + self.conv2 = nn.Conv2d(10, 10, 2, 1, bias=False) + self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool2 = sl.SumPool2d(3,3) + + self.conv3 = nn.Conv2d(10, 10, 3, 1, bias=False) + self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool3 = sl.SumPool2d(2,2) + + self.flat = nn.Flatten() + + flat_s = ResCSNN3.get_flatten_size(input_size) + + self.fc1 = nn.Linear(flat_s, 100, bias=False) + self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc2 = nn.Linear(100, 100, bias=False) + self.iaf5 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc3 = nn.Linear(100, 100, bias=False) + self.iaf6 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc4 = nn.Linear(100, nb_classes, bias=False) + self.iaf7 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.merge_fc = sl.Merge() + self.merge_conv = sl.Merge() + + @staticmethod + def get_flatten_size(input_size): + conv1_dims = ResCSNN3.conv2d_output_size(input_size, 10, (2, 2)) + pool1_dims = ResCSNN3.pool_output_size(conv1_dims, 10, (2, 2)) + + conv2_dims = ResCSNN3.conv2d_output_size(pool1_dims, 10, (2, 2)) + pool2_dims = ResCSNN3.pool_output_size(conv2_dims, 10, (3, 3)) + + conv3_dims = ResCSNN3.conv2d_output_size(pool2_dims, 10, (3, 3)) + pool3_dims = ResCSNN3.pool_output_size(conv3_dims, 10, (2, 2)) + + return pool3_dims[0]*pool3_dims[1]*pool3_dims[2] + + @staticmethod + def conv2d_output_size(input_size, out_channels, kernel_size, stride=1, padding=0, dilation=1): + input_height, input_width, input_channels = input_size + kernel_height, kernel_width = kernel_size + + output_height = ((input_height + 2 * padding - dilation * (kernel_height - 1) - 1) // stride) + 1 + output_width = ((input_width + 2 * padding - dilation * (kernel_width - 1) - 1) // stride) + 1 + + return (output_height, output_width, out_channels) + + @staticmethod + def pool_output_size(input_size, out_channels, kernel_size, stride=None, padding=0): + input_height, input_width, input_channels = input_size + kernel_height, kernel_width = kernel_size + + if stride is None: + stride = kernel_height + + output_height = ((input_height + 2 * padding - kernel_height) // stride) + 1 + output_width = ((input_width + 2 * padding - kernel_width) // stride) + 1 + + return (output_height, output_width, out_channels) + + def detach_neuron_states(self): + for name, layer in self.named_modules(): + if name != '': + if isinstance(layer, sl.StatefulLayer): + for name, buffer in layer.named_buffers(): + buffer.detach_() + + def init_weights(self): + for name, layer in self.named_modules(): + if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear): + nn.init.xavier_normal_(layer.weight.data) + + def forward(self, x): + + con1_out = self.conv1(x) + iaf1_out = self.iaf1(con1_out) + pool1_out = self.pool1(iaf1_out) + pool1a_out = self.pool1a(iaf1_out) + + conv2_out = self.conv2(pool1_out) + iaf2_out = self.iaf2(conv2_out) + pool2_out = self.pool2(iaf2_out) + + merged_conv_out = self.merge_conv(pool1a_out, pool2_out) + + conv3_out = self.conv3(merged_conv_out) + iaf3_out = self.iaf3(conv3_out) + pool3_out = self.pool3(iaf3_out) + + flat_out = self.flat(pool3_out) + + fc1_out = self.fc1(flat_out) + iaf4_out = self.iaf4(fc1_out) + + fc2_out = self.fc2(iaf4_out) + iaf5_out = self.iaf5(fc2_out) + + fc3_out = self.fc3(iaf5_out) + iaf6_out = self.iaf6(fc3_out) + + merge_fc_out = self.merge_fc(iaf4_out, iaf6_out) + + fc4_out = self.fc4(merge_fc_out) + iaf7_out = self.iaf7(fc4_out) + + return iaf7_out \ No newline at end of file diff --git a/tests/test_nonsequential/using_SumPool2d/models/SCNN.py b/tests/test_nonsequential/using_SumPool2d/models/SCNN.py new file mode 100644 index 00000000..5e3d689a --- /dev/null +++ b/tests/test_nonsequential/using_SumPool2d/models/SCNN.py @@ -0,0 +1,130 @@ +import torch.nn as nn +import sinabs.layers as sl +from sinabs.exodus.layers import IAFSqueeze + +class SCNN(nn.Module): + def __init__(self, input_size, nb_classes, batch_size, surrogate_fn, min_v_mem=-1.0, spk_thr=1.0) -> None: + super().__init__() + + self.conv1 = nn.Conv2d(2, 1, 2, 1, bias=False) + self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool1 = sl.SumPool2d(2,2) + + self.conv2 = nn.Conv2d(1, 8, 2, 1, bias=False) + self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool2 = sl.SumPool2d(2,2) + + self.conv3 = nn.Conv2d(8, 16, 2, 1, bias=False) + self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool3 = sl.SumPool2d(2,2) + + self.conv4 = nn.Conv2d(16, 16, 2, 1, bias=False) + self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool4 = sl.SumPool2d(2,2) + + self.flat = nn.Flatten() + + flat_s = SCNN.get_flatten_size(input_size) + + self.fc1 = nn.Linear(flat_s, 1024, bias=False) + self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc2 = nn.Linear(1024, 256, bias=False) + self.iaf5 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc3 = nn.Linear(256, 128, bias=False) + self.iaf6 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc4 = nn.Linear(128, 64, bias=False) + self.iaf7 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc5 = nn.Linear(64, nb_classes, bias=False) + self.iaf8 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + @staticmethod + def get_flatten_size(input_size): + conv1_dims = SCNN.conv2d_output_size(input_size, 1, (2, 2)) + pool1_dims = SCNN.pool_output_size(conv1_dims, 1, (2, 2)) + + conv2_dims = SCNN.conv2d_output_size(pool1_dims, 8, (2, 2)) + pool2_dims = SCNN.pool_output_size(conv2_dims, 8, (2, 2)) + + conv3_dims = SCNN.conv2d_output_size(pool2_dims, 16, (2, 2)) + pool3_dims = SCNN.pool_output_size(conv3_dims, 16, (2, 2)) + + conv4_dims = SCNN.conv2d_output_size(pool3_dims, 16, (2, 2)) + pool4_dims = SCNN.pool_output_size(conv4_dims, 16, (2, 2)) + + return pool4_dims[0]*pool4_dims[1]*pool4_dims[2] + + @staticmethod + def conv2d_output_size(input_size, out_channels, kernel_size, stride=1, padding=0, dilation=1): + input_height, input_width, input_channels = input_size + kernel_height, kernel_width = kernel_size + + output_height = ((input_height + 2 * padding - dilation * (kernel_height - 1) - 1) // stride) + 1 + output_width = ((input_width + 2 * padding - dilation * (kernel_width - 1) - 1) // stride) + 1 + + return (output_height, output_width, out_channels) + + @staticmethod + def pool_output_size(input_size, out_channels, kernel_size, stride=None, padding=0): + input_height, input_width, input_channels = input_size + kernel_height, kernel_width = kernel_size + + if stride is None: + stride = kernel_height + + output_height = ((input_height + 2 * padding - kernel_height) // stride) + 1 + output_width = ((input_width + 2 * padding - kernel_width) // stride) + 1 + + return (output_height, output_width, out_channels) + + def detach_neuron_states(self): + for name, layer in self.named_modules(): + if name != '': + if isinstance(layer, sl.StatefulLayer): + for name, buffer in layer.named_buffers(): + buffer.detach_() + + def init_weights(self): + for name, layer in self.named_modules(): + if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear): + nn.init.xavier_normal_(layer.weight.data) + + def forward(self, x): + + con1_out = self.conv1(x) + iaf1_out = self.iaf1(con1_out) + pool1_out = self.pool1(iaf1_out) + + conv2_out = self.conv2(pool1_out) + iaf2_out = self.iaf2(conv2_out) + pool2_out = self.pool2(iaf2_out) + + conv3_out = self.conv3(pool2_out) + iaf3_out = self.iaf3(conv3_out) + pool3_out = self.pool3(iaf3_out) + + conv4_out = self.conv4(pool3_out) + iaf4_out = self.iaf4(conv4_out) + pool4_out = self.pool4(iaf4_out) + + flat_out = self.flat(pool4_out) + + fc1_out = self.fc1(flat_out) + iaf4_out = self.iaf4(fc1_out) + + fc2_out = self.fc2(iaf4_out) + iaf5_out = self.iaf5(fc2_out) + + fc3_out = self.fc3(iaf5_out) + iaf6_out = self.iaf6(fc3_out) + + fc4_out = self.fc4(iaf6_out) + iaf7_out = self.iaf7(fc4_out) + + fc5_out = self.fc5(iaf7_out) + iaf8_out = self.iaf8(fc5_out) + + return iaf8_out \ No newline at end of file diff --git a/tests/test_nonsequential/utils/train_test_fn.py b/tests/test_nonsequential/utils/train_test_fn.py new file mode 100644 index 00000000..bdc5d1da --- /dev/null +++ b/tests/test_nonsequential/utils/train_test_fn.py @@ -0,0 +1,81 @@ +from tqdm.notebook import tqdm +import torch + +def training_loop(device, nb_time_steps, batch_size, feature_map_size, dataloader_train, model, loss_fn, optimizer, epochs, dataloader_test): + epochs_y = [] + epochs_x = [] + epochs_acc = [] + model.train() + + for e in range(epochs): + losses = [] + batches = [] + batch_count = 0 + train_p_bar = tqdm(dataloader_train) + + for X, y in train_p_bar: + # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width] + X = X.reshape(-1, feature_map_size[2], feature_map_size[0], feature_map_size[1]).to(dtype=torch.float, device=device) + y = y.to(dtype=torch.long, device=device) + + # forward + pred = model(X) + + # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes] + pred = pred.reshape(batch_size, nb_time_steps, -1) + + # accumulate all time-steps output for final prediction + pred = pred.sum(dim = 1) + loss = loss_fn(pred, y) + + # gradient update + optimizer.zero_grad() + loss.backward() + optimizer.step() + + # detach the neuron states and activations from current computation graph(necessary) + model.detach_neuron_states() + + train_p_bar.set_description(f"Epoch {e} - BPTT Training Loss: {round(loss.item(), 4)}") + + batch_count += 1 + losses.append(loss.item()) + batches.append(batch_count) + + epochs_y.append(losses) + epochs_x.append(batches) + + acc = test(device, nb_time_steps, batch_size, feature_map_size, dataloader_test, model) + print(f'Epoch {e} accuracy: {acc}') + epochs_acc.append(acc) + + return epochs_x, epochs_y, epochs_acc + +def test(device, nb_time_steps, batch_size, feature_map_size, dataloader_test, model): + correct_predictions = [] + with torch.no_grad(): + test_p_bar = tqdm(dataloader_test) + for X, y in test_p_bar: + # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width] + X = X.reshape(-1, feature_map_size[2], feature_map_size[0], feature_map_size[1]).to(dtype=torch.float, device=device) + y = y.to(dtype=torch.long, device=device) + + # forward + output = model(X) + + # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes] + output = output.reshape(batch_size, nb_time_steps, -1) + + # accumulate all time-steps output for final prediction + output = output.sum(dim=1) + + # calculate accuracy + pred = output.argmax(dim=1, keepdim=True) + + # compute the total correct predictions + correct_predictions.append(pred.eq(y.view_as(pred))) + + test_p_bar.set_description(f"Testing Model...") + + correct_predictions = torch.cat(correct_predictions) + return correct_predictions.sum().item()/(len(correct_predictions))*100 \ No newline at end of file From 97c2485fd4bdb69d19c41eaad6f978c323f9f72d Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Wed, 1 May 2024 17:42:28 +0200 Subject: [PATCH 066/379] trying different architectures --- .../ARCHITECTURES_SEARCH/Res-SCNN3.ipynb | 123 ++++++++++----- .../using_SumPool2d/models/ResSCNN1.py | 138 ++++++++++++++++ .../using_SumPool2d/models/ResSCNN2.py | 143 +++++++++++++++++ .../using_SumPool2d/models/ResSCNN3.py | 146 +++++++++++++++++ .../using_SumPool2d/models/ResSCNN4.py | 149 ++++++++++++++++++ 5 files changed, 660 insertions(+), 39 deletions(-) create mode 100644 tests/test_nonsequential/using_SumPool2d/models/ResSCNN1.py create mode 100644 tests/test_nonsequential/using_SumPool2d/models/ResSCNN2.py create mode 100644 tests/test_nonsequential/using_SumPool2d/models/ResSCNN3.py create mode 100644 tests/test_nonsequential/using_SumPool2d/models/ResSCNN4.py diff --git a/tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/Res-SCNN3.ipynb b/tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/Res-SCNN3.ipynb index d7817d15..e531875b 100644 --- a/tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/Res-SCNN3.ipynb +++ b/tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/Res-SCNN3.ipynb @@ -27,7 +27,9 @@ "\n", "from SCNN import SCNN\n", "# from CSNN3 import CSNN3\n", - "from train_test_fn import training_loop" + "from train_test_fn import training_loop\n", + "\n", + "from torch.utils.data import Subset" ] }, { @@ -35,13 +37,22 @@ "execution_count": 2, "metadata": {}, "outputs": [], + "source": [ + "rand_seed = 1" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], "source": [ "torch.backends.cudnn.enabled = False\n", "torch.backends.cudnn.deterministic = True\n", - "random.seed(1)\n", - "torch.manual_seed(1)\n", - "torch.cuda.manual_seed(1)\n", - "np.random.seed(1)" + "random.seed(rand_seed)\n", + "torch.manual_seed(rand_seed)\n", + "torch.cuda.manual_seed(rand_seed)\n", + "np.random.seed(rand_seed)" ] }, { @@ -53,7 +64,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -66,12 +77,14 @@ "v_min = -0.313\n", "\n", "grad_scale = 1.534\n", - "grad_width = 0.759" + "grad_width = 0.759\n", + "\n", + "validation_ratio = 0.2" ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -82,7 +95,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -93,9 +106,43 @@ "snn_test_dataset = DVSGesture(save_to=root_dir, train=False, transform=to_raster)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "getting validation dataset...." + ] + }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "validation samples: 215, training samples: 862, test samples: 264\n" + ] + } + ], + "source": [ + "num_samples = len(snn_train_dataset)\n", + "num_validation_samples = int(validation_ratio * num_samples)\n", + "print(f'validation samples: {num_validation_samples}, training samples: {num_samples-num_validation_samples}, test samples: {len(snn_test_dataset)}')\n", + "\n", + "np.random.seed(rand_seed)\n", + "\n", + "validation_indices = np.random.choice(np.arange(num_samples), size=num_validation_samples, replace=False)\n", + "training_indices = np.array(list(filter(lambda x: x not in validation_indices, np.arange(num_samples))))\n", + "\n", + "train_dataset = Subset(snn_train_dataset, training_indices)\n", + "validation_dataset = Subset(snn_train_dataset, validation_indices)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, "metadata": {}, "outputs": [ { @@ -107,7 +154,7 @@ } ], "source": [ - "sample_data, label = snn_train_dataset[0]\n", + "sample_data, label = train_dataset[0]\n", "print(f\"The transformed array is in shape [Time-Step, Channel, Height, Width] --> {sample_data.shape}\")" ] }, @@ -120,15 +167,20 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ "disk_cache_train = tonic.DiskCachedDataset(\n", - " dataset=snn_train_dataset,\n", + " dataset=train_dataset,\n", " cache_path='./cached_train'\n", ")\n", "\n", + "disk_cache_validation = tonic.DiskCachedDataset(\n", + " dataset=validation_dataset,\n", + " cache_path='./cached_validation'\n", + ")\n", + "\n", "disk_cache_test = tonic.DiskCachedDataset(\n", " dataset=snn_test_dataset,\n", " cache_path='./cached_test'\n", @@ -137,11 +189,12 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ "snn_train_dataloader = DataLoader(disk_cache_train, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True)\n", + "snn_validation_dataloader = DataLoader(disk_cache_validation, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True)\n", "snn_test_dataloader = DataLoader(disk_cache_test, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=False)" ] }, @@ -156,7 +209,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 11, "metadata": {}, "outputs": [ { @@ -177,24 +230,16 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 12, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "flat_s: 784\n" - ] - } - ], + "outputs": [], "source": [ "snn = SCNN(DVSGesture.sensor_size, 11, batch_size, PeriodicExponential(grad_scale=grad_scale, grad_width=grad_width), v_min, spk_thr).to(device)" ] }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 13, "metadata": {}, "outputs": [], "source": [ @@ -203,7 +248,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 14, "metadata": {}, "outputs": [], "source": [ @@ -227,18 +272,18 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 15, "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "e2d31de3202340d6898918b79f8f2f90", + "model_id": "ee0385e2a65240a7b5ca8de458950f9d", "version_major": 2, "version_minor": 0 }, "text/plain": [ - " 0%| | 0/134 [00:00 1\u001b[0m epochs_x, epochs_y, epochs_acc \u001b[38;5;241m=\u001b[39m \u001b[43mtraining_loop\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 2\u001b[0m \u001b[43m \u001b[49m\u001b[43mdevice\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[1;32m 3\u001b[0m \u001b[43m \u001b[49m\u001b[43mn_time_steps\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 4\u001b[0m \u001b[43m \u001b[49m\u001b[43mbatch_size\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 5\u001b[0m \u001b[43m \u001b[49m\u001b[43mDVSGesture\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msensor_size\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 6\u001b[0m \u001b[43m \u001b[49m\u001b[43msnn_train_dataloader\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[1;32m 7\u001b[0m \u001b[43m \u001b[49m\u001b[43msnn\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[1;32m 8\u001b[0m \u001b[43m \u001b[49m\u001b[43mloss_fn\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[1;32m 9\u001b[0m \u001b[43m \u001b[49m\u001b[43moptimizer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[1;32m 10\u001b[0m \u001b[43m \u001b[49m\u001b[43mepochs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[1;32m 11\u001b[0m \u001b[43m \u001b[49m\u001b[43msnn_test_dataloader\u001b[49m\u001b[43m)\u001b[49m\n", + "Cell \u001b[0;32mIn[15], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m epochs_x, epochs_y, epochs_acc \u001b[38;5;241m=\u001b[39m \u001b[43mtraining_loop\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 2\u001b[0m \u001b[43m \u001b[49m\u001b[43mdevice\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[1;32m 3\u001b[0m \u001b[43m \u001b[49m\u001b[43mn_time_steps\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 4\u001b[0m \u001b[43m \u001b[49m\u001b[43mbatch_size\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 5\u001b[0m \u001b[43m \u001b[49m\u001b[43mDVSGesture\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msensor_size\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 6\u001b[0m \u001b[43m \u001b[49m\u001b[43msnn_train_dataloader\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[1;32m 7\u001b[0m \u001b[43m \u001b[49m\u001b[43msnn\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[1;32m 8\u001b[0m \u001b[43m \u001b[49m\u001b[43mloss_fn\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[1;32m 9\u001b[0m \u001b[43m \u001b[49m\u001b[43moptimizer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[1;32m 10\u001b[0m \u001b[43m \u001b[49m\u001b[43mepochs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[1;32m 11\u001b[0m \u001b[43m \u001b[49m\u001b[43msnn_validation_dataloader\u001b[49m\u001b[43m)\u001b[49m\n", "File \u001b[0;32m~/Documents/github/sinabs/tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/../../utils/train_test_fn.py:33\u001b[0m, in \u001b[0;36mtraining_loop\u001b[0;34m(device, nb_time_steps, batch_size, feature_map_size, dataloader_train, model, loss_fn, optimizer, epochs, dataloader_test)\u001b[0m\n\u001b[1;32m 31\u001b[0m \u001b[38;5;66;03m# gradient update\u001b[39;00m\n\u001b[1;32m 32\u001b[0m optimizer\u001b[38;5;241m.\u001b[39mzero_grad()\n\u001b[0;32m---> 33\u001b[0m \u001b[43mloss\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbackward\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 34\u001b[0m optimizer\u001b[38;5;241m.\u001b[39mstep()\n\u001b[1;32m 36\u001b[0m \u001b[38;5;66;03m# detach the neuron states and activations from current computation graph(necessary)\u001b[39;00m\n", "File \u001b[0;32m~/.pyenv/versions/3.10.0/lib/python3.10/site-packages/torch/_tensor.py:525\u001b[0m, in \u001b[0;36mTensor.backward\u001b[0;34m(self, gradient, retain_graph, create_graph, inputs)\u001b[0m\n\u001b[1;32m 515\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m has_torch_function_unary(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m 516\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m handle_torch_function(\n\u001b[1;32m 517\u001b[0m Tensor\u001b[38;5;241m.\u001b[39mbackward,\n\u001b[1;32m 518\u001b[0m (\u001b[38;5;28mself\u001b[39m,),\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 523\u001b[0m inputs\u001b[38;5;241m=\u001b[39minputs,\n\u001b[1;32m 524\u001b[0m )\n\u001b[0;32m--> 525\u001b[0m \u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mautograd\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbackward\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 526\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mgradient\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mretain_graph\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcreate_graph\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minputs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43minputs\u001b[49m\n\u001b[1;32m 527\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n", "File \u001b[0;32m~/.pyenv/versions/3.10.0/lib/python3.10/site-packages/torch/autograd/__init__.py:267\u001b[0m, in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[0m\n\u001b[1;32m 262\u001b[0m retain_graph \u001b[38;5;241m=\u001b[39m create_graph\n\u001b[1;32m 264\u001b[0m \u001b[38;5;66;03m# The reason we repeat the same comment below is that\u001b[39;00m\n\u001b[1;32m 265\u001b[0m \u001b[38;5;66;03m# some Python versions print out the first line of a multi-line function\u001b[39;00m\n\u001b[1;32m 266\u001b[0m \u001b[38;5;66;03m# calls in the traceback and some print out the last line\u001b[39;00m\n\u001b[0;32m--> 267\u001b[0m \u001b[43m_engine_run_backward\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 268\u001b[0m \u001b[43m \u001b[49m\u001b[43mtensors\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 269\u001b[0m \u001b[43m \u001b[49m\u001b[43mgrad_tensors_\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 270\u001b[0m \u001b[43m \u001b[49m\u001b[43mretain_graph\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 271\u001b[0m \u001b[43m \u001b[49m\u001b[43mcreate_graph\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 272\u001b[0m \u001b[43m \u001b[49m\u001b[43minputs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 273\u001b[0m \u001b[43m \u001b[49m\u001b[43mallow_unreachable\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 274\u001b[0m \u001b[43m \u001b[49m\u001b[43maccumulate_grad\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 275\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n", @@ -306,7 +351,7 @@ " loss_fn, \n", " optimizer, \n", " epochs, \n", - " snn_test_dataloader)" + " snn_validation_dataloader)" ] }, { @@ -316,7 +361,7 @@ "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjIAAAGwCAYAAACzXI8XAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAAB4uElEQVR4nO3dd1hT1/8H8HcSIGErIkuG4MCNA0TQqlUErVWr/TmqrVatVot1te6qrVpRW61VW1fdto5+66qzTqzWhYJ74EQRN0NAVnJ+f2CuRFCIBiL4fj1PHuHmfu49gQhvzjn3XJkQQoCIiIioGJIbuwFEREREr4pBhoiIiIotBhkiIiIqthhkiIiIqNhikCEiIqJii0GGiIiIii0GGSIiIiq2TIzdgMKm0Whw+/ZtWFtbQyaTGbs5REREVABCCDx+/BguLi6Qy1/c71Lig8zt27fh5uZm7GYQERHRK7h58yZcXV1f+HyJDzLW1tYAsr8QNjY2Rm4NERERFURSUhLc3Nyk3+MvUuKDjHY4ycbGhkGGiIiomMlvWggn+xIREVGxxSBDRERExdZbGWTUajXGjh0LT09PmJubo0KFCpg4cSKevxH4+fPn0bZtW9ja2sLS0hJ+fn6IiYl54XEzMzMxYcIEVKhQASqVCj4+Pti+fXuu/WJjY/Hxxx+jTJkyMDc3R82aNREREWHw10lERFTSlfg5MnmZOnUq5s6di2XLlqF69eqIiIhAz549YWtri4EDBwIArly5gkaNGqF379747rvvYGNjg7Nnz0KlUr3wuN988w1WrlyJhQsXokqVKtixYwfat2+P//77D3Xq1AEAxMfHo2HDhnj33Xexbds2lC1bFtHR0ShdunSRvHYiIqKSRCae74YoYZKSkmBra4vExERpsu/7778PR0dHLFq0SNrvww8/hLm5OVauXAkA6NKlC0xNTbFixYoCn8vFxQVjxoxBaGjoC487cuRIHDx4EP/++68hXh4REVGJlNfv77y8lUNLgYGB2L17Ny5dugQAOHnyJA4cOIBWrVoByF5Eb8uWLahcuTJCQkLg4OAAf39/bNiw4aXHTU9Pz9VjY25ujgMHDkifb9q0Cb6+vujYsSMcHBxQp04dLFy40LAvkIiI6C3xVgaZkSNHokuXLqhSpQpMTU1Rp04dDB48GN26dQMA3Lt3D8nJyZgyZQpatmyJf/75B+3bt0eHDh0QHh7+wuOGhIRgxowZiI6Ohkajwc6dO7Fu3TrExcVJ+1y9ehVz585FpUqVsGPHDvTv3x8DBw7EsmXLCv11ExERlTiihEtMTBQARGJiorRt1apVwtXVVaxatUqcOnVKLF++XNjZ2YmlS5cKIYSIjY0VAMRHH32kc6w2bdqILl26vPBc9+7dE+3atRNyuVwoFApRuXJl8cUXXwiVSiXtY2pqKgICAnTqvvzyS9GgQQNDvFwiIqISIa/f33l5K3tkhg0bJvXK1KxZE5988gmGDBmCsLAwAIC9vT1MTExQrVo1nbqqVau+9KqlsmXLYsOGDUhJScGNGzdw4cIFWFlZwcvLS9rH2dlZ7+MSERFR3t7KIJOamprrBlQKhQIajQYAYGZmBj8/P1y8eFFnn0uXLsHDwyPf46tUKpQrVw5ZWVn466+/0K5dO+m5hg0bvvJxiYiISNdbefl1mzZt8P3338Pd3R3Vq1dHZGQkZsyYgV69ekn7DBs2DJ07d0bjxo3x7rvvYvv27fj777+xb98+aZ/u3bujXLlyUk/OkSNHEBsbi9q1ayM2NhbffvstNBoNhg8fLtUMGTIEgYGBmDx5Mjp16oSjR49iwYIFWLBgQZG9fiIiohKjiIa6jCavMbakpCQxaNAg4e7uLlQqlfDy8hJjxowR6enpOrWLFi0SFStWFCqVSvj4+IgNGzboPN+kSRPRo0cP6fN9+/aJqlWrCqVSKcqUKSM++eQTERsbm6tNf//9t6hRo4ZQKpWiSpUqYsGCBYZ90URERMVcQefIvJXryBhCXOITXHuQAk97SzjbmhvsuERERFTw399v5dDS61pzLAYj152GEIBMBkzpUBOd/dyN3SwiIqK3zls52fd1xCU+wainIQYAhABGrzuDuMQnxm0YERHRW4hBRk/XHqRA89xgnFoIXH+QapwGERERvcUYZPTkaW8JuUx3m0ImQ3l7C+M0iIiI6C3GIKMnZ1tzhHWoCdnTMCMDMLlDDU74JSIiMgIGmVfQ2c8do1pWAQAEVCjDib5ERERGwiDzilztsoeSstQl+up1IiKiNxqDzCuyUmZfuf44PcvILSEiInp7Mci8IitVdpBJTs80ckuIiIjeXgwyr8j6aY9Mchp7ZIiIiIyFQeYVPeuRyUIJv8sDERHRG4tB5hVp58hkqgXSszRGbg0REdHbiUHmFVmaPbtNVTIn/BIRERkFg8wrkstlUq8M58kQEREZB4PMa5CCDHtkiIiIjIJB5jVYP53w+5g9MkREREbBIPMacl65REREREWPQeY1PBta4qJ4RERExsAg8xq0Q0uc7EtERGQcDDKvgfdbIiIiMi4GmddgpTQFwB4ZIiIiY2GQeQ1WvGqJiIjIqBhkXoM115EhIiIyKgaZ18AeGSIiIuNikHkNvPyaiIjIuBhkXgMXxCMiIjIuowYZtVqNsWPHwtPTE+bm5qhQoQImTpwIIYS0jxAC48aNg7OzM8zNzREUFITo6GgjtvoZa940koiIyKiMGmSmTp2KuXPnYs6cOTh//jymTp2KadOmYfbs2dI+06ZNw6xZszBv3jwcOXIElpaWCAkJQVpamhFbno09MkRERMZlYsyT//fff2jXrh1at24NAChfvjxWrVqFo0ePAsjujZk5cya++eYbtGvXDgCwfPlyODo6YsOGDejSpYvR2g7kWBCPPTJERERGYdQemcDAQOzevRuXLl0CAJw8eRIHDhxAq1atAADXrl3DnTt3EBQUJNXY2trC398fhw4dyvOY6enpSEpK0nkUFuunC+KlZ2mQkaUptPMQERFR3ozaIzNy5EgkJSWhSpUqUCgUUKvV+P7779GtWzcAwJ07dwAAjo6OOnWOjo7Sc88LCwvDd999V7gNf8pSqZA+TknPgpmJWZGcl4iIiLIZtUdm7dq1+P333/HHH3/gxIkTWLZsGX788UcsW7bslY85atQoJCYmSo+bN28asMW6TBRymJtmhxnOkyEiIip6Ru2RGTZsGEaOHCnNdalZsyZu3LiBsLAw9OjRA05OTgCAu3fvwtnZWaq7e/cuateunecxlUollEplobddy1plgieZas6TISIiMgKj9sikpqZCLtdtgkKhgEaTPd/E09MTTk5O2L17t/R8UlISjhw5goCAgCJt64vwyiUiIiLjMWqPTJs2bfD999/D3d0d1atXR2RkJGbMmIFevXoBAGQyGQYPHoxJkyahUqVK8PT0xNixY+Hi4oIPPvjAmE2XWHN1XyIiIqMxapCZPXs2xo4diy+++AL37t2Di4sLPv/8c4wbN07aZ/jw4UhJSUHfvn2RkJCARo0aYfv27VCpVEZs+TO83xIREZHxyETOZXRLoKSkJNja2iIxMRE2NjYGP/7nKyKw4+xdfN++Brr5exj8+ERERG+jgv7+5r2WXpPV07VkeJsCIiKioscg85qsObRERERkNAwyr8lKyauWiIiIjIVB5jVxsi8REZHxMMi8Jitefk1ERGQ0DDKvyZoL4hERERkNg8xrknpkOLRERERU5BhkXpM2yDxmjwwREVGRY5B5TdK9ltgjQ0REVOQYZF6TtXZBPPbIEBERFTkGmdek7ZFJzVBDrSnRd3sgIiJ64zDIvCbtHBmAvTJERERFjUHmNZmZyKE0yf4yMsgQEREVLQYZA7DmhF8iIiKjYJAxAK7uS0REZBwMMgbA+y0REREZB4OMAfAO2ERERMbBIGMAVtq1ZNgjQ0REVKQYZAzAmkNLRERERsEgYwC83xIREZFxMMgYAO+3REREZBwMMgbAy6+JiIiMg0HGAKQF8Ti0REREVKQYZAxAmiPDoSUiIqIixSBjAFxHhoiIyDgYZAyAk32JiIiMg0HGAKy1C+KxR4aIiKhIMcgYAHtkiIiIjINBxgCkOTIZWdBohJFbQ0RE9PZgkDEA7eXXQgCpmWojt4aIiOjtwSBjAEoTOUwVMgAcXiIiIipKDDIGIJPJuLovERGRETDIGIgV74BNRERU5BhkDMSKl2ATEREVOQYZA7FW8hJsIiKiosYgYyAcWiIiIip6DDIGIt04kkNLRERERYZBxkC4ui8REVHRY5AxEGtefk1ERFTkGGQM5Nk6MuyRISIiKioMMgbCyb5ERERFj0HGQNgjQ0REVPQYZAzEmpN9iYiIihyDjIFwZV8iIqKixyBjIJwjQ0REVPQYZAyEc2SIiIiKHoOMgUhzZNKzIIQwcmuIiIjeDgwyBqINMmqNQFqmxsitISIiejswyBiIuakCcln2x4+5ui8REVGRYJAxEJlM9myeDCf8EhERFQkGGQOyVvESbCIioqLEIGNA7JEhIiIqWgwyBqRdSyaJQYaIiKhIMMgYENeSISIiKloMMgZkJd1viVctERERFQUGGQOyZo8MERFRkWKQMSDt0NJjBhkiIqIiwSBjQM+GlhhkiIiIigKDjAFxsi8REVHRYpAxIGv2yBARERUpBhkDslJmr+zLOTJERERFg0HGgDhHhoiIqGgxyBgQ58gQEREVLQYZA5LmyDDIEBERFQkGGQPiZF8iIqKixSBjQNqhpQy1BulZaiO3hoiIqORjkDEgSzMT6WP2yhARERU+BhkDkstlnPBLRERUhBhkDEy63xJ7ZIiIiAodg4yBadeSYZAhIiIqfEYPMrGxsfj4449RpkwZmJubo2bNmoiIiJCeF0Jg3LhxcHZ2hrm5OYKCghAdHW3EFr8ch5aIiIiKjlGDTHx8PBo2bAhTU1Ns27YN586dw/Tp01G6dGlpn2nTpmHWrFmYN28ejhw5AktLS4SEhCAtLc2ILX+xZ2vJZBq5JURERCWfSf67FJ6pU6fCzc0NS5YskbZ5enpKHwshMHPmTHzzzTdo164dAGD58uVwdHTEhg0b0KVLlyJvc36kHhkOLRERERU6o/bIbNq0Cb6+vujYsSMcHBxQp04dLFy4UHr+2rVruHPnDoKCgqRttra28Pf3x6FDh/I8Znp6OpKSknQeRUma7MuhJSIiokJn1CBz9epVzJ07F5UqVcKOHTvQv39/DBw4EMuWLQMA3LlzBwDg6OioU+fo6Cg997ywsDDY2tpKDzc3t8J9Ec/hjSOJiIiKjlGDjEajQd26dTF58mTUqVMHffv2RZ8+fTBv3rxXPuaoUaOQmJgoPW7evGnAFufPmpN9iYiIioxRg4yzszOqVaums61q1aqIiYkBADg5OQEA7t69q7PP3bt3peeep1QqYWNjo/MoSuyRISIiKjpGDTINGzbExYsXdbZdunQJHh4eALIn/jo5OWH37t3S80lJSThy5AgCAgKKtK0FZaU0BcA5MkREREXBqFctDRkyBIGBgZg8eTI6deqEo0ePYsGCBViwYAEAQCaTYfDgwZg0aRIqVaoET09PjB07Fi4uLvjggw+M2fQXYo8MERFR0TFqkPHz88P69esxatQoTJgwAZ6enpg5cya6desm7TN8+HCkpKSgb9++SEhIQKNGjbB9+3aoVCojtvzFOEeGiIio6MiEEMLYjShMSUlJsLW1RWJiYpHMlzl2/RE6zjsET3tL7P26aaGfj4iIqCQq6O9vo9+ioKSx5r2WiIiIigyDjIE9u9cSb1FARERU2BhkDMz66VVLaZkaZKo1Rm4NERFRycYgY2CWSoX0cQon/BIRERUqBhkDM1HIYW6aHWY4T4aIiKhwMcgUAitO+CUiIioSDDKFgGvJEBERFQ0GmUIgre7LK5eIiIgKFYNMIdBegs2hJSIiosKld5B58uQJUlNTpc9v3LiBmTNn4p9//jFow4ozKw4tERERFQm9g0y7du2wfPlyAEBCQgL8/f0xffp0tGvXDnPnzjV4A4sj3jiSiIioaOgdZE6cOIF33nkHAPC///0Pjo6OuHHjBpYvX45Zs2YZvIHFESf7EhERFQ29g0xqaiqsra0BAP/88w86dOgAuVyOBg0a4MaNGwZvYHHEy6+JiIiKht5BpmLFitiwYQNu3ryJHTt2IDg4GABw7969Irm7dHFg9fQ2BeyRISIiKlx6B5lx48bh66+/Rvny5eHv74+AgAAA2b0zderUMXgDiyPOkSEiIioaJvoW/N///R8aNWqEuLg4+Pj4SNubN2+O9u3bG7RxxRXnyBARERUNvYMMADg5OcHJyQkAkJSUhD179sDb2xtVqlQxaOOKK2kdGQYZIiKiQqX30FKnTp0wZ84cANlryvj6+qJTp06oVasW/vrrL4M3sDh6NrTElX2JiIgKk95BZv/+/dLl1+vXr4cQAgkJCZg1axYmTZpk8AYWR9YqDi0REREVBb2DTGJiIuzs7AAA27dvx4cffggLCwu0bt0a0dHRBm9gcWStvWqJk32JiIgKld5Bxs3NDYcOHUJKSgq2b98uXX4dHx8PlUpl8AYWR9qhpZQMNdQaYeTWEBERlVx6T/YdPHgwunXrBisrK3h4eKBp06YAsoecatasaej2FUuWSoX0cUpGFmxUpkZsDRERUcmld5D54osvUL9+fdy8eRMtWrSAXJ7dqePl5cU5Mk8pTRQwM5EjI0uDx2kMMkRERIXllS6/9vX1ha+vL4QQEEJAJpOhdevWhm5bsWatNMHDrAzOkyEiIipEes+RAYDly5ejZs2aMDc3h7m5OWrVqoUVK1YYum3FmnQJdjovwSYiIiosevfIzJgxA2PHjsWAAQPQsGFDAMCBAwfQr18/PHjwAEOGDDF4I4sjaVE89sgQEREVGr2DzOzZszF37lx0795d2ta2bVtUr14d3377LYPMU1a8TQEREVGh03toKS4uDoGBgbm2BwYGIi4uziCNKgmseeNIIiKiQqd3kKlYsSLWrl2ba/uaNWtQqVIlgzSqJGCPDBERUeHTe2jpu+++Q+fOnbF//35pjszBgwexe/fuPAPO20o72ZdzZIiIiAqP3j0yH374IY4cOQJ7e3ts2LABGzZsgL29PY4ePYr27dsXRhuLJSvtbQrYI0NERFRoXmkdmXr16mHlypWGbkuJwjkyREREha9AQSYpKanAB7SxsXnlxpQknCNDRERU+AoUZEqVKgWZTPbSfbQr/KrVaoM0rLiT1pFhkCEiIio0BQoye/fuLex2lDjSyr5pXNmXiIiosBQoyDRp0qSw21HiSHNk2CNDRERUaF7pXkuUP2vtVUuc7EtERFRoGGQKibSODHtkiIiICg2DTCHJedWSEMLIrSEiIiqZGGQKiXaOjBBASgav5CIiIioMrxRksrKysGvXLsyfPx+PHz8GANy+fRvJyckGbVxxpjSRw0Sefck658kQEREVDr1X9r1x4wZatmyJmJgYpKeno0WLFrC2tsbUqVORnp6OefPmFUY7ix2ZTAYrlQkSUjORnJ4JQGXsJhEREZU4evfIDBo0CL6+voiPj4e5ubm0vX379ti9e7dBG1fcSYvisUeGiIioUOjdI/Pvv//iv//+g5mZmc728uXLIzY21mANKwl4mwIiIqLCpXePjEajyfM2BLdu3YK1tbVBGlVS8MaRREREhUvvIBMcHIyZM2dKn8tkMiQnJ2P8+PF47733DNm2Yo/3WyIiIipceg8tTZ8+HSEhIahWrRrS0tLQtWtXREdHw97eHqtWrSqMNhZbViqu7ktERFSY9A4yrq6uOHnyJFavXo1Tp04hOTkZvXv3Rrdu3XQm/xLnyBARERU2vYMMAJiYmODjjz82dFtKHN44koiIqHDpHWQ2bdqU53aZTAaVSoWKFSvC09PztRtWEvDyayIiosKld5D54IMPIJPJct0/SLtNJpOhUaNG2LBhA0qXLm2whhZHHFoiIiIqXHpftbRz5074+flh586dSExMRGJiInbu3Al/f39s3rwZ+/fvx8OHD/H1118XRnuLFSvp8utMI7eEiIioZNK7R2bQoEFYsGABAgMDpW3NmzeHSqVC3759cfbsWcycORO9evUyaEOLIxvOkSEiIipUevfIXLlyBTY2Nrm229jY4OrVqwCASpUq4cGDB6/fumLOSpl9+TXnyBARERUOvYNMvXr1MGzYMNy/f1/adv/+fQwfPhx+fn4AgOjoaLi5uRmulcWUFXtkiIiICpXeQ0uLFi1Cu3bt4OrqKoWVmzdvwsvLCxs3bgQAJCcn45tvvjFsS4shTvYlIiIqXHoHGW9vb5w7dw7//PMPLl26JG1r0aIF5PLsDp4PPvjAoI0srrTryDxOy5Ku6CIiIiLDeaUF8eRyOVq2bImWLVsauj0lirZHRq0RSMvUwNxMYeQWERERlSyvFGRSUlIQHh6OmJgYZGRk6Dw3cOBAgzSsJLAwU0AmA4QAHqdnMsgQEREZmN5BJjIyEu+99x5SU1ORkpICOzs7PHjwABYWFnBwcGCQyUEmk8FKaYLHaVlITsuCg7WxW0RERFSy6H3V0pAhQ9CmTRvEx8fD3Nwchw8fxo0bN1CvXj38+OOPhdHGYs2aE36JiIgKjd5BJioqCl999RXkcjkUCgXS09Ph5uaGadOmYfTo0YXRxmLt2eq+DDJERESGpneQMTU1la5OcnBwQExMDADA1tYWN2/eNGzrSgDpxpHskSEiIjI4vefI1KlTB8eOHUOlSpXQpEkTjBs3Dg8ePMCKFStQo0aNwmhjsWalyl7dlz0yREREhqd3j8zkyZPh7OwMAPj+++9RunRp9O/fH/fv38eCBQsM3sDijnNkiIiICo9ePTJCCDg4OEg9Lw4ODti+fXuhNKyk4Oq+REREhUevHhkhBCpWrMi5MHqwyrG6LxERERmWXkFGLpejUqVKePjwYWG1p8R51iOTaeSWEBERlTx6z5GZMmUKhg0bhjNnzhRGe0oca15+TUREVGj0vmqpe/fuSE1NhY+PD8zMzGBubq7z/KNHjwzWuJJACjKcI0NERGRwegeZmTNnFkIzsnt6Ro0ahUGDBknnSEtLw1dffYXVq1cjPT0dISEh+PXXX+Ho6FgobSgMVsrsy685R4aIiMjw9A4yPXr0MHgjjh07hvnz56NWrVo624cMGYItW7bgzz//hK2tLQYMGIAOHTrg4MGDBm9DYbFijwwREVGh0XuODABcuXIF33zzDT766CPcu3cPALBt2zacPXtW72MlJyejW7duWLhwIUqXLi1tT0xMxKJFizBjxgw0a9YM9erVw5IlS/Dff//h8OHDr9Jso+Dl10RERIVH7yATHh6OmjVr4siRI1i3bh2Sk5MBACdPnsT48eP1bkBoaChat26NoKAgne3Hjx9HZmamzvYqVarA3d0dhw4deuHx0tPTkZSUpPMwJmtefk1ERFRo9A4yI0eOxKRJk7Bz506YmZlJ25s1a6Z3T8nq1atx4sQJhIWF5Xruzp07MDMzQ6lSpXS2Ozo64s6dOy88ZlhYGGxtbaWHm5ubXm0yNKlHhkGGiIjI4PQOMqdPn0b79u1zbXdwcMCDBw8KfJybN29i0KBB+P3336FSqfRtxguNGjUKiYmJ0sPYi/dp58hkqDVIz1IbtS1EREQljd5BplSpUoiLi8u1PTIyEuXKlSvwcY4fP4579+6hbt26MDExgYmJCcLDwzFr1iyYmJjA0dERGRkZSEhI0Km7e/cunJycXnhcpVIJGxsbnYcxWZo9m0/NXhkiIiLD0jvIdOnSBSNGjMCdO3cgk8mg0Whw8OBBfP311+jevXuBj9O8eXOcPn0aUVFR0sPX1xfdunWTPjY1NcXu3bulmosXLyImJgYBAQH6NttoFHIZLM0UADjhl4iIyND0vvx68uTJCA0NhZubG9RqNapVqwa1Wo2uXbvim2++KfBxrK2tpZtPallaWqJMmTLS9t69e2Po0KGws7ODjY0NvvzySwQEBKBBgwb6NtuorFQmSMlQc8IvERGRgekdZMzMzLBw4UKMHTsWZ86cQXJyMurUqYNKlSoZvHE//fQT5HI5PvzwQ50F8YobK6UJ7iKdPTJEREQGJhNCCH0KDhw4gEaNGhVWewwuKSkJtra2SExMNNp8mXa/HMTJmwn4rbsvgqoVn1WJiYiIjKWgv7/1niPTrFkzeHp6YvTo0Th37txrNfJtYc1F8YiIiAqF3kHm9u3b+OqrrxAeHo4aNWqgdu3a+OGHH3Dr1q3CaF+JoF1L5jGDDBERkUHpHWTs7e0xYMAAHDx4EFeuXEHHjh2xbNkylC9fHs2aNSuMNhZ70v2WONmXiIjIoF7pXktanp6eGDlyJKZMmYKaNWsiPDzcUO0qUZ7dbynTyC0hIiIqWV45yBw8eBBffPEFnJ2d0bVrV9SoUQNbtmwxZNtKDGv2yBARERUKvS+/HjVqFFavXo3bt2+jRYsW+Pnnn9GuXTtYWFgURvtKBM6RISIiKhx6B5n9+/dj2LBh6NSpE+zt7QujTSWOtcoUAHtkiIiIDE3vIHPw4MHCaEeJJk32ZY8MERGRQekdZLTOnTuHmJgYZGRk6Gxv27btazeqpOE6MkRERIVD7yBz9epVtG/fHqdPn4ZMJoN2YWCZTAYAUKvVhm1hCaDtkeG9loiIiAxL76uWBg0aBE9PT9y7dw8WFhY4e/Ys9u/fD19fX+zbt68Qmlj8SZN9GWSIiIgMSu8emUOHDmHPnj2wt7eHXC6HXC5Ho0aNEBYWhoEDByIyMrIw2lmscR0ZIiKiwqF3j4xarYa1tTWA7FV+b9++DQDw8PDAxYsXDdu6EkK7jkxapgaZao2RW0NERFRy6N0jU6NGDZw8eRKenp7w9/fHtGnTYGZmhgULFsDLy6sw2ljsWSqffZlT0rNQysLMiK0hIiIqOfQOMt988w1SUlIAABMmTMD777+Pd955B2XKlMGaNWsM3sCSwFQhh8pUjrRMDR6nMcgQEREZit5BJiQkRPq4YsWKuHDhAh49eoTSpUtLVy5RblZKU6RlpvMSbCIiIgN6rZtGatnZ2THE5MOai+IREREZnEGCDOVPunKJl2ATEREZDINMEeGNI4mIiAyPQaaISPdbYo8MERGRwTDIFBFrLopHRERkcAwyRYQ9MkRERIbHIFNEOEeGiIjI8Bhkioi1yhQAe2SIiIgMiUGmiFhxHRkiIiKDY5ApIs8m+zLIEBERGQqDTBHRzpFJ4tASERGRwTDIFJFnVy3x8msiIiJDYZApIlYcWiIiIjI4BpkiYs11ZIiIiAyOQaaIaHtkUjLUUGuEkVtDRERUMjDIFBHtHBkASMlgrwwREZEhMMgUEaWJAmaK7C83h5eIiIgMg0GmCHFRPCIiIsNikClC0v2W2CNDRERkEAwyRYiXYBMRERkWg0wRsuIl2ERERAbFIFOEnt1viav7EhERGQKDTBHS9shwjgwREZFhMMgUIc6RISIiMiwGmSJkrTIFwDkyREREhsIgU4SsuY4MERGRQTHIFCFpHRkGGSIiIoNgkClCXBCPiIjIsBhkitCzdWR4+TUREZEhMMgUIWtetURERGRQDDJFiCv7EhERGRaDTBHiZF8iIiLDYpApQlY5Lr8WQhi5NURERMUfg0wRslZmL4gnBJCaoTZya4iIiIo/BpkipDKVQyGXAeCEXyIiIkNgkClCMpmMa8kQEREZEINMEeONI4mIiAyHQaaIWfMSbCIiIoNhkCliz3pkuLovERHR62KQKWLaS7A5R4aIiOj1McgUMc6RISIiMhwGmSJmrcpeS4ZzZIiIiF4fg0wRs1axR4aIiMhQGGSKGO+3REREZDgMMkWMC+IREREZDoNMEZNuHJnGy6+JiIheF4NMEbPmVUtEREQGwyBTxLiODBERkeEwyBQxriNDRERkOAwyRYyXXxMRERkOg0wRs1I+WxBPCGHk1hARERVvDDJFTDtHJksjkJ6lMXJriIiIijcGmSJmYaqATJb9MSf8EhERvR4GmSIml8tgZcZ5MkRERIZg1CATFhYGPz8/WFtbw8HBAR988AEuXryos09aWhpCQ0NRpkwZWFlZ4cMPP8Tdu3eN1GLDeLYoHoMMERHR6zBqkAkPD0doaCgOHz6MnTt3IjMzE8HBwUhJSZH2GTJkCP7++2/8+eefCA8Px+3bt9GhQwcjtvr1PbvfElf3JSIieh0mxjz59u3bdT5funQpHBwccPz4cTRu3BiJiYlYtGgR/vjjDzRr1gwAsGTJElStWhWHDx9GgwYNjNHs18YeGSIiIsN4o+bIJCYmAgDs7OwAAMePH0dmZiaCgoKkfapUqQJ3d3ccOnQoz2Okp6cjKSlJ5/Gm4aJ4REREhvHGBBmNRoPBgwejYcOGqFGjBgDgzp07MDMzQ6lSpXT2dXR0xJ07d/I8TlhYGGxtbaWHm5ubwdu6f/9+tGnTBi4uLpDJZNiwYYPO8zKZLM/HDz/8AODFi+L98ssvKF++PFQqFfz9/XH06FGd5z///HNUqFAB5ubmKFu2LNq1a4cLFy4Y/PUREREVF29MkAkNDcWZM2ewevXq1zrOqFGjkJiYKD1u3rxpoBY+k5KSAh8fH/zyyy95Ph8XF6fzWLx4MWQyGT788EMAgPXTRfFyXn69Zs0aDB06FOPHj8eJEyfg4+ODkJAQ3Lt3T9qnXr16WLJkCc6fP48dO3ZACIHg4GCo1WqDv0YiIqLiwKhzZLQGDBiAzZs3Y//+/XB1dZW2Ozk5ISMjAwkJCTq9Mnfv3oWTk1Oex1IqlVAqlYXa3latWqFVq1YvfP75tm3cuBHvvvsuvLy8AOSYI5OjR2bGjBno06cPevbsCQCYN28etmzZgsWLF2PkyJEAgL59+0r7ly9fHpMmTYKPjw+uX7+OChUqGObFERERFSNG7ZERQmDAgAFYv3499uzZA09PT53n69WrB1NTU+zevVvadvHiRcTExCAgIKCom/tK7t69iy1btqB3797SNumqpbTsq5YyMjJw/PhxnblAcrkcQUFBL5wLlJKSgiVLlsDT07NQhs+IiIiKA6P2yISGhuKPP/7Axo0bYW1tLc17sbW1hbm5OWxtbdG7d28MHToUdnZ2sLGxwZdffomAgIBic8XSsmXLYG1trXPJuPVzVy09ePAAarUajo6OOrWOjo655sD8+uuvGD58OFJSUuDt7Y2dO3fCzMyskF8FERHRm8moPTJz585FYmIimjZtCmdnZ+mxZs0aaZ+ffvoJ77//Pj788EM0btwYTk5OWLdunRFbrZ/FixejW7duUKlU0rbXuWqpW7duiIyMRHh4OCpXroxOnTohLS3NYO0lIiIqTozaI1OQuz+rVCr88ssvL5xY+yb7999/cfHiRZ1gBjybI6Od7Gtvbw+FQpFrxeK85gJpr8aqVKkSGjRogNKlS2P9+vX46KOPCvGVEBERvZnemKuWSqJFixahXr168PHx0dn+fI+MmZkZ6tWrpzMXSKPRYPfu3S+dCySEgBAC6enphdB6IiKiN98bcdVScZOcnIzLly9Ln1+7dg1RUVGws7ODu7s7ACApKQl//vknpk+fnqveWmWCu6tHw6R2E2DgOwCAoUOHokePHvD19UX9+vUxc+ZMpKSkSFcxXb16FWvWrEFwcDDKli2LW7duYcqUKTA3N8d7771XBK+aiIjozcMg8woiIiLw7rvvSp8PHToUANCjRw8sXboUALB69WoIIfIc8rFSmiIz/g4eJzxCXOITONuao3Pnzrh//z7GjRuHO3fuoHbt2ti+fbs0AVilUuHff//FzJkzER8fD0dHRzRu3Bj//fcfHBwcCv9FExERvYFkoiATVYqxpKQk2NraIjExETY2NsZuDgBgXvgVTNmWfTWSXAaEdaiJzn7uRm4VERHRm6Ogv785R6aIxSU+wbTtzy6p1ghg9LoziEt8YsRWERERFU8MMkXs2oMUaJ7rA1MLgesPUo3TICIiomKMQaaIedpbQi7T3aaQyVDe3sI4DSIiIirGGGSKmLOtOcI61NQJM8NbesPZ1tx4jSIiIiqmGGSMoLOfOw6ObIaqTtYAsoeWiIiISH8MMkbibGuOng2zb5K57kRsgVY5JiIiIl0MMkbUqqYTlCZyXL6XjDOxScZuDhERUbHDIGNE1ipTtKiWveDdushbRm4NERFR8cMgY2Qd6pYDAPx98jYy1Rojt4aIiKh4YZAxsncqlYW9lRkeJGfg3+j7xm4OERFRscIgY2SmCjna+LgAyJ70S0RERAXHIPMG6FDHFQCw89xdJKVlGrk1RERExQeDzBugRjkbVHSwQnqWBttP3zF2c4iIiIoNBpk3gEwmkyb9/nWCVy8REREVFIPMG+KD2uUgkwFHrj3CrXjeQJKIiKggGGTeEC6lzNHAswwAYGPUbSO3hoiIqHhgkHmDtH86vLTuxC3esoCIiKgAGGTeIK1qOEFlKseV+yk4dSvR2M0hIiJ64zHIvEGsVaYIruYEAFgfyTVliIiI8sMg84bRDi9t4i0LiIiI8sUg84Z5p6I97K2UeJSSgfCLvGUBERHRyzDIvGFMFHK0q519ywIOLxEREb0cg8wbqH2d7OGlnefvIvEJb1lARET0Igwyb6DqLjao7GiFjCwNtp2OM3ZziIiI3lgMMm8gmUyG9k9vJMk7YhMREb0Yg8wb6oM6LpDJgKPXH+HmI96ygIiIKC8MMm8oZ1tzBFbIvmXBBk76JSIiyhODzBtMO7y0PjKWtywgIiLKA4PMG6zl01sWXH2QgqibCcZuDhER0RuHQeYNZqU0QcvqvGUBERHRizDIvOHa180eXvr75G1kZPGWBURERDkxyLwBYmNj8fHHH6NMmTIwNzdHzZo1ERERAQBoWKEMylorEZ+aifBLuW9ZcPDgQZiYmKB27do62x8/fozBgwfDw8MD5ubmCAwMxLFjx4ri5RARERUZBhkji4+PR8OGDWFqaopt27bh3LlzmD59OkqXLg3g6S0LfLJvWbDuxC2d2oSEBHTv3h3NmzfPddzPPvsMO3fuxIoVK3D69GkEBwcjKCgIsbEcoiIiopJDJkr45TBJSUmwtbVFYmIibGxsjN2cXEaOHImDBw/i33//feE+524n4b1Z/8JMIcexMUGwtTAFAHTp0gWVKlWCQqHAhg0bEBUVBQB48uQJrK2tsXHjRrRu3Vo6Tr169dCqVStMmjSpUF8TERHR6yro72/2yBjZpk2b4Ovri44dO8LBwQF16tTBwoULdfap5mKDKk7WyFBrsOXpLQuWLFmCq1evYvz48bmOmZWVBbVaDZVKpbPd3NwcBw4cKLwXQ0REVMQYZIzs6tWrmDt3LipVqoQdO3agf//+GDhwIJYtW6azn/ZGkutO3EJ0dDRGjhyJlStXwsTEJNcxra2tERAQgIkTJ+L27dtQq9VYuXIlDh06hLg43ruJiIhKDgYZI9NoNKhbty4mT56MOnXqoG/fvujTpw/mzZuns1+72uUgkwHHrj3A/3Xqgu+++w6VK1d+4XFXrFgBIQTKlSsHpVKJWbNm4aOPPoJczm85ERGVHPytZmTOzs6oVq2azraqVasiJiZGZ5uTrQqNKtpDZDzBqagTGDBgAExMTGBiYoIJEybg5MmTMDExwZ49ewAAFSpUQHh4OJKTk3Hz5k0cPXoUmZmZ8PLyKrLXRkREVNgYZIysYcOGuHjxos62S5cuwcPDI9e+7euUg0xpAd+hixAZGYmoqChERUWhX79+8Pb2RlRUFPz9/XVqLC0t4ezsjPj4eOzYsQPt2rUr1NdDRERUlHJPsKAiNWTIEAQGBmLy5Mno1KkTjh49igULFmDBggXSPqNGjUJsbCzmLlwMCzNT3Jc5IsPGFfU8si/RdnBwgEqlQo0aNaSaHTt2QAgBb29vXL58GcOGDUOVKlXQs2fPIn+NREREhYU9Mkbm5+eH9evXY9WqVahRowYmTpyImTNnolu3btI+cXFxiImJgaXSBC1rZN+yYOXh6/jvygPEJT7J87iJiYkIDQ1FlSpV0L17dzRq1Ag7duyAqalpkbwuIiKiosB1ZIqZf6Pv45NFR6XP5TIgrENNdPZzN2KriIiIDIvryJRQnvaWOp9rBDB63ZkX9swQERGVZAwyxUzMo1Sdzx9HbsXNRaGoWM4BNjY2CAgIwLZt215Y37RpU8hkslyPnCsAf/vtt6hSpQosLS1RunRpBAUF4ciRI4X2moiIiF4Vg0wx42lvCbns2ecK6zIo3aQH1mzbh4iICDRr1gzt2rXD2bNn86xft24d4uLipMeZM2egUCjQsWNHaZ/KlStjzpw5OH36NA4cOIDy5csjODgY9+/nvmklERGRMXGOTDG05lgMRq87DXWO71wZSzP82NEH71ZxgJ2dHX744Qf07t0732PNnDkT48aNQ1xcHCwtLfPcR/s13LVrV543qCQiIjI0zpEpwTr7uePAyGZY1acB/vjMH1WcrPEwJQOfLj6M/xv2A1JSUhAQEJDvccLCwjB69GikpaXB09MTH3zwQa41bTIyMrBgwQLY2trCx8cHQHb48fb2hrm5Odzc3DBkyBCkpaVJNXPnzkWtWrVgY2NToOEuIiKiV8V1ZIopZ1tzONuaAwAmNbZBYMOWyExPx00zc9TpMRGmZdzyPcamTZvw5MkTrFq1ClWrVsXo0aMRHByMc+fOYe/evejSpQtSU1Ph7OyMnTt3wt7eHn/88QdGjhyJxYsXIzAwEJcuXcKnn34KmUyGGTNmAABcXV0xZcoUVKpUCUIILFu2DO3atUNkZCSqV69eqF8XIiJ6y4gSLjExUQAQiYmJxm5KoUlPTxfR0dFi7v92CMd3Ogu5uY0o//k8sfLwdaHRaF5Y17dvX1GzZk3p83v37gkAIjw8XCQnJ4vo6Ghx6NAh0atXL1G+fHlx9+5dERoaKpo1ayaEEGLy5MnC19dXmJqaClNTU9GuXTtx4cKFPM9VunRp8dtvv4m//vpL1KtXT9ja2goLCwvh4+Mjli9f/sI2fv755wKA+Omnn17ti0NERMVSQX9/c2ipBDAzM0PFihXR78NgnNqyFA7lK+PB4Q0Ys/4MPl9xHPEpGblqUlJSsHr1ap15NImJiQAAOzs7WFpaomLFimjQoAEWLVoEExMTLFq0CIGBgTh+/DiOHj2K8PBwdOzYEa6urvj888+RmZmJ4OBgpKSkSMdUq9VYvXq1NNxlZ2eHMWPGoGfPnqhQoQIuXLiA7t27IzAwMNew1vr163H48GG4uLgAAM6ePYsPP/wQ5cuXh0wmw8yZM3O9rrCwMPj5+cHa2hoODg55DpcREVHJwSBTwjhYq1DVyRo1nS1gqpDhn3N30ernf/HflQc6+/35559IT0/Hxx9/DCD7LtyDBw9Gw4YNdW51oKXRaJCeno6uXbtiwoQJaNSoEXbv3o0RI0YgJCQEs2fPxtKlSxETE4Pjx4/j9OnTsLKyglKpRL9+/bB+/XpUq1YNTZs2Rfv27XH+/HkMHToUx48fh7e3N+7fv68TgmJjY/Hll1/i999/l1YjTk1NhZeXF6ZMmQI7OzssXLgQLi4ukMlk2LBhAwAgPDwcoaGhOHz4MHbu3KkTrn755RdUrVpVmttTp06dXPUJCQkIDQ2Fs7MzlEolKleujK1btwIAfv/9d/j4+MDCwgLOzs7o1asXHj58mOf3YfXq1ZDJZPjggw9e91tKREQvU0Q9REZT0oeWRo4cKcLDw8W1a9fEqVOnxMiRI4VMJhP//POPOH0rQTjWayFsGvyfKD9ys5i67bzIyFILIYRo1KiR6Ny5s3Scfv36CQ8PD3Hx4kUxatQocejQIXH9+nUREREhevbsKZRKpThz5ozYu3evcHR0FAsXLhSnTp0S69atE25ubmLChAkiOjpaABCnT5+WhrsiIiLEyJEjhb29vTh79qxO2zUajdi1a5ewsLAQa9eulYa11Gq1ePfdd8XMmTOFEEJ4eHjkGlpycHAQLVq0EOvWrRMAxPr16/P8+miHy4YMGSKsra3F6tWrxZUrV8SIESOEqampGDVqlFSfnp4ufH19xXvvvScOHDggrl27Jvbt2yeioqLEgQMHhFwuFz///LO4evWqmD17trCyshIqlSrX+a9duybKlSsn3nnnHdGuXTshhBA9evQQAPJ8aGvHjx+f6zlvb28hhBBpaWli9OjRwt3dXZiYmAhzc3Nha2sr1Z85c0Z06NBBeHh45BqK27t3b57njYuL0/la3bp1S3Tr1k3Y2dkJlUolatSoIY4dO1aQtyERkcEV9Pc3g0wx16tXL+Hh4SHMzMxE2bJlRfPmzcU///wjPf9O48aiRpO2wmPEZuExYrNoO+eA+HPXIQFArFr/txBCiNDQUOHq6iquXr0qnjx5Itq3by9cXFyEmZmZcHZ2Fm3bthVHjx4VQmQHoK+//lqnDStWrBAqlUq89957omHDhnm2s3nz5qJv375CCCESEhKEpaWlMDExEUqlUixatEgnBE2ePFm0aNFCmt+TV5DJue1lQUZ7XB8fn1ztHjp0qGjYsKFUP3fuXOHl5SUyMjJyHeeHH34QXl5e0udbt24VwcHBws7OTuf8WVlZIjAwUPz222+iR48eUpBJSEgQcXFxIi4uTqxcuVJ8+eWXwsrKKleQqV69urRfXFycuH//vhBCiLZt2wp/f3+xc+dOsWTJEtGjRw/x/fffS/VHjx4VX3/9tVi1apVwcnLKM8hcvHhRrFu3TrRo0UI4OjrqnPvRo0fCw8NDfPrpp+LIkSPi6tWrYseOHWLlypV5hqDn6+fMmSM8PDyEUqkU9evXF0eOHMn1NZwzZ44AIExMTHRqw8PDxfvvvy+cnZ1f+L3UBkkbG5tc+73o3H/99ZcICgoS9vb2wtraWjRo0EBs375d57hJSUli0KBBwt3dXahUKhEQECC914nIuDhH5i2xaNEiXL9+Henp6bh37x527dqFFi1aSM/vDw/H6X0b8UvXurBRmeDkzQR8vfMhPEZsxqjDQEinHli/fj327NkDT09PqFQqrFu3DrGxsUhPT8ft27exceNG+Pn5Acge3pHLdd82CoUCmZmZOHv2LFavXp1nO7VDUwBgbW2NqKgoHDt2DN9//z2GDBmCTz75BA0bNkR6ejp+/vlnLF26FDKZLM9jFVTO4TKFQgGVSqXzvLm5OY4efXbfqk2bNiEgIAChoaFwdHREjRo1MHnyZKjVagQEBODmzZvYunUrhBCoW7cu0tLS8OGHH+occ8KECXBwcMi1ho+trS2cnJzg5OSEbt26oVmzZjpzibRMTEyk/ZycnGBvb4/t27cjPDwcW7duRVBQED799FMsXboUo0ePlur8/Pzwww8/oEuXLlAqlXl+PbR3Sa9fvz7mzp2r89zUqVPh5uaGJUuWoH79+vD09ERwcDDKlSsHALh48SLi4uKwcuVKDBo0CL/88otUu2bNGgwdOhTjx4/HiRMn4OPjg5CQENy7d0/aJyEhAZMmTYKnpydcXV11zp2SkgIfHx+dY+aUkJCA7t27w8fHByqVqsDn3r9/P1q0aIGtW7di3rx5ePDgAVq2bKkzlPjZZ59h586dWLFiBU6fPo3g4GAEBQVh4cKFaNGiBcqWLQsbGxtUr14dDRo0yDUUqTVlyhTIZDIMHjwYQPaNXrt27YrKlStDJpPBy8srV+2LVtlu3ry5VCuXy9GxY0e0adNGp37hwoV45513ULp0aWn17Zzv5X379qFu3bpQKpWoWLEili5dmuvrGhsbi48//hhlypSBubk5atasiYiIiDy/B0RvtKLJVcZT0ntk9HHixiOpZ8ZjxGZhVec9IVNaivdHzROz/j4qwqMuiVuxt0VqaqpU88knn4iRI0dKn48fP15YW1uLVatWiatXr4p//vlH2NjYCHNzc3H16lUhxMuHu/JSpUoVoVKpxM2bN8VPP/0kZDKZUCgU0gOAkMvlwsPDQ6opSI+Mdrjs5s2bYtSoUcLJyUlEREQIjUYjjh07JvUqaOu9vb2FUqkUvXr1EhEREWL16tXCzs5OfPvtt0IIIdauXSusrKykHoU2bdqIjIwMqf7ff/8V5cqVk3pRcvbIPO/9998XLVq0yNUjY2FhIZydnYWnp6fo2rWruHHjhujfv79o3ry5GDFihHBxcRGVKlUSX331lUhNTc3ztT/fg6XtkfHw8BBOTk4iKChIHDhwQKe2atWqYvDgweL//u//RNmyZUXt2rXFggULpNr4+Phcr0FbX79+fREaGiptV6vVwsXFRYSFhUnbOnfuLL755hsxfvx44ePj88LvWV7bn6/V99xCZPegjRkzRri6ukq1qampQqFQiM2bN+vsW7duXeHn5yemTp0qjh49Ki5duiQ6deok5HK5+PHHH3O18ejRo6J8+fKiVq1aYtCgQUKI7OHFgQMHimXLlgkvLy/h5+eXaxj04cOHOr1vZ86cEQqFQvzwww9Sbe3atUW7du3EmDFjdOq7du0qfvnlFxEZGSnOnz8vPv30U2Fraytu3bolrl69KiwsLMTQoUPFuXPnxKBBgwQAnd7DF/XAXb58WXrP1KlTR5iZmQl7e3tRpkyZXD1eefWEZWRkiO+++054eXkJpVIpatWqJb744gud/T777LMXDqEWpH7evHkv7cH76aefROXKlYVKpRKlSpUS1tbWudr+omHUnLWurq5i8ODB4smTJ9KxC9J7SIbDoaWnGGSeOXj5vk6Qef6HifYR2HOsmL37kjh05YF4p3Fj0aNHD+kYmZmZ4quRY0Q59/JCqVQKKysrYWlpKSIiIqR98hvuyik0NFRYWFiI+vXrCyGEePDggTh9+rTOw8XFRYwYMULn0u78gkzO4TIhhEhNTRU9e/YUJiYmQqFQCBcXFzF8+HCdIFOpUiXh5uYmsrKypONMnz5dODk5ibNnzwpnZ2cxbdo0cfLkSbF9+3ZRs2ZN0atXLwFA/PHHH6J8+fJi69atUu2LgkxsbKxQKBRizZo1Om3funWrWLt2rXT8gIAA4e7uLpo3by6USqVo3bq1OHLkiNiyZYv0S6ggQebChQti3rx5IiIiQhw8eFD6OuSsVSqVQqlUilGjRokTJ06I+fPnC5VKJUaMGJFnCNJ+3deuXSsUCkWuNnTv3l20bdtWCCHE4sWLhZ+fn8jMzNQ7yORVq8+5c1Kr1cLNzU06R1JSkgAgdu3apbNfw4YNRZMmTXLVV6tWTXz33Xc6bXz8+LGoVKmS2Llzp2jSpIkUZHLKuf1lv/x++uknYW1tLZKTk/OsfVl9VlaWsLa2FsuWLRPDhw8X1atXl57bunWrqFatmqhdu7ZUP2LECNGoUaM825EzCE2fPl2YmJgImUwm5s+fL/r06SNKlSol5s+fL8zMzMTixYvF2bNnpe0DBgwQLi4uYsuWLeLKlSvS/4/x48dL+6lUKuHt7Z3nEOrw4cPzrbeyshKDBw/Oc37c77//LpRKpfj999/F7NmzhampqbC1tRXdu3eX2njx4sU8Q9yMGTOk2mvXrokdO3YIGxsbYWNjIwWhn376KVeozPm9yuvnqVwul0JUXnPlQkJCXlhrbm4ulEql8PPzE5988omoUaOG9MfOJ598ImJjY6Xzx8fHiy+++EI4OTkJhUIhTExMhKmpaYECXM5a7c9tBweHPIeKw8LCBADpfblkyZJc7VYqlTrvqRf9rpk2bVqe70Gtgv7+5oJ4bxHtfZo0IvtzjxGbIZcBPRuWx8U7yYiMiUdKhhqxAH785xIAwKThCNiVs8X3W86hnocdbsWnYh0CYPJRAMz++RXq6APYumUTypUrhzt37gAA5syZA3Pz7MX6unfvjnLlyknDXWFhYfD19YWXlxfGjRuHzZs3Iz09HX369AEAlClTBmXKlNFpt6mpKZycnODt7Z3vaxRC4Msvv8T69euxb98+eHp6AsgeRlq8eDHmz5+Pu3fvwtnZGQsWLIC1tTUeP34MAHB2doapqSkUCoV0vKpVq+LOnTv4/vvv0bBhQwwbNgwAUKtWLVhaWuKdd94BANy5cwfXr19HmzZtpFqNRpP9NTQxwcWLF1GhQgUAwLJly1CqVKlcVzS1atVK+rhWrVrw9/eHh4cH4uLiIJPJ8Pvvv8PW1hYAMGPGDPzf//1fvl8PAPD29tb52gUGBuLKlSvYv3+/Tlt9fX0xefJkAECdOnVw5swZbN++HfPmzYOvry/S09Px22+/oWnTptJNRB8/fgy1Wg1HR0edczo6OuLChQuIjo7GyJEj8e+//8LERL8fN/nV5nfu5/34449ITk6WPre2tkZAQAAmTpyIqlWrwtHREatWrcKhQ4dQsWJFnVqNRoPHjx/Dzs5OZ3toaChat26NoKAgTJo0Sa/X97xFixahS5cuL7xVyMukpqYiMzMTdnZ2OHToEIKCgqTnWrVqhTt37kjDXkD2MGpISAg6duyI8PBwlCtXDl988QX69OmDefPmwdPTE9OnT4e/vz8+//xzPHjwAOvWrcPWrVuxZcsWTJw4EX369EHPnj0BAPPmzcOWLVuwbNkyhIWF4b333gMAnDlzBl5eXrh8+TKqVauGefPmYfXq1UhMTISTk1Ou17FixQqMGTPmpfVbtmyBo6Mj2rdvn6v+v//+Q8OGDdG1a1f4+/ujb9++UCqVOHLkCPbv348tW7agb9++0jCqlqenJzZt2iTVAsCRI0eQkpICT09PbNy4ETNnzsR3332HixcvwsHBIde5161bh4yM7KUuNm7ciC+//BJZWVmYMGECbty4gZCQEAQHB6Nly5Y651YqlRBCSLUAsHLlSgwbNgwfffQRvvrqK0ybNg2///47fv31VzRu3Bjx8fEYNGgQ2rZti4iICGRkZKBFixZwcHBA//79pSH7+vXrY/v27QgJCcGRI0cQHByMd999F9u2bUPZsmURHR0NS0tLqfZ///sfIiMjMXToUIwZMwYdO3bEzJkzERISgosXL+LGjRuYP38+atWqpfPabWxsdJa5eH5aQFxcnM7n27ZtQ+/evXMNzb8qzpF5izjbmiOsQ00onr7JFDIZwjrUxNj3q2PlZ/44OT4Ym79shG/bVEPrWs5wtFEiSyMQdTMBC/+9hn4rj2PSlvNSEHocuRWpyUlo2rQpnJ2dpceaNWukc8bExOi8ie8+SkCvPp/D29sbq1atgqurK3755Re8//77uHPnDp48eSLt2717d4waNUr6PCMjA1FRUYiKikJGRgZiY2MRFRWl8xpDQ0OxcuVK/PHHH7C2tsadO3d0jmtqagpXV1coFAqsXr0a77//vlTbsGFDXL58WQogAHDp0iU4OzsjLS0tz7lBWuXKlcPp06el9kVFRaFt27Z49913ERUVBTe37JWWhRBYvHgxPvnkE5iZmb30+1WqVClpnkS5cuWkEANkByzxGrdJq1+/vs7nzs7OqFatms62qlWr4v79+/j8889Rr149BAYGSis6//TTT/meQwiBrl274rvvvkPlypX1ap9arX7l2rz88ccf+O6777B27Vqd7StWrIAQAuXKlYNSqcSsWbPw0Ucf5fpea0NQp06dpG2rV6/GiRMnEBYW9trtO3r0KM6cOYPPPvvslepHjBgBFxcXBAUF4c6dO3mGu6SkJOnzq1evYu7cuahUqRJ27NiB/v37Y+DAgVi2bJkUhDIyMnD8+HEEBQUhJCQEhw4dglwuR7NmzXDr1i2dsCSXyxEUFIS0tDRpLpq23tXVFQcOHJD28/Lywv379+Hi4gIvLy9069YNMTExAID09PR864OCgnDo0KE8vw7ada4OHjyI48ePo0aNGti6dSvee+89qfb48ePw9fVFx44d4eDggDp16mDhwoU6a2QB2X90WVtbo2fPnlKIsrCwwOLFi/M8t52dnTS3bfHixfD394eVlRUGDx4s1V6+fBlKpVJnHlzp0qV1ap2cnDB79myYmppi1qxZqFatGhYvXgwHBwc8fPgQ3t7eaNCgAebMmYPjx48jJiYGixcvxqNHj7BhwwZs2bIFffr0wZQpU9ChQwfp3DkDXM55cOHh4VJtw4YNsWLFCvTt2xfjx4/Xed1z585Ft27dsHDhQpQuXVrntctkMp32P//+y/mck5MTNm7ciHfffRdeXl4veVcXHHtk3jKd/dzRuHJZXH+QivL2FtJtDgDARCFHjXK2qFHOFp829IQQArfinyDixiMcux6Pfy/dx834Z0HDY8TmXMcvbWGKvx5b4Ojvx+FW2gK9w5bC1c4Cl+89xuGrj7DJpDEUXRpDPTU7QJw7dw79+vVDv379AABLlizBp59+CiA7BMnlcly/fh0AcP36ddSpU0c6148//ogff/wRAHDt2jVERUVJk1ibNm2q066wsDC4urrC398f8fHxmDp1KqKiojBixAisWrUK165dQ+PGjTFr1iwMGjQIGRkZiI6OxtmzZzFw4ECUK1cOffr0wdy5cxESEoK4uDgMHjwY9evXx9GjR2FmZpZr/Z1SpUoBgM728PBwXL58uUA39ExOTsaVK1cQHByMTZs2ITk5GVZWVgCyA5ZcLtcJXfp4PgA2bNgw18KBly5dgoeHR67a+vXrS79YrK2toVAocPfuXZ197t69C3t7e2zbtg2RkZEYMGAAgOyeDW0AO3Xq1AvX2Xn8+DEiIiLyrNX2zrzs3Dn/2l+9ejU+++wz/Pnnnzq/fAGgQoUKCA8PR0pKCpKSkuDs7IzOnTvr/IDVhqCNGzdKf4k/ePAAY8aMwc6dO3NNIn8VixYtQs2aNXMFzIKYMmUKVq9ejX379hW4LS/qgZs3bx4ePnwIR0dHPHjwQOrxMjMzQ1JSEp48eQJra2sAyDMsWVtbY8aMGWjcuDHMzc2hVqtx+PBhnf0qVqyIlJQU/PXXX4iLi8N3332Hd955B2fOnEFISEi+9S/qcQOArl274sGDB2jatCnUajX69++Pfv36SRPjHR0dkZKSgrlz52Lo0KEYPXo0jh07hoEDB2LevHnSGlkajQZqtRohISFSbX4hSksbwMqVK6fTwxYUFIS9e/fiypUrcHBwQOnSpdGsWTNMmjRJpxc6IyMDMTExaNGihVSb17kTExMhk8lQqlQp6UKFfv364ejRo4iLi4OLiwtGjBgBhUKBoKAgrFu3Dp999lmuXricFzls2LAB9+/fR6VKlaBWq6FQKKRz//bbb/i///u/PHsfk5OT4eHhAY1Gg7p162Ly5MkvvB3N3bt3pd47Q2GQeQvlvE/Ty8hkMrjZWcDNzgLt67giLvEJGk7ZI/XIaFVxssadpDQkpGYiPjUT8amJOB2b+NJja0PQezWd4GijgrXKFDYqE6iUJthyKg7WKhPMWL4eNioT3E1Kg5XSBIfvyeE5cjM0AkiPOYU7q55dtTN06FAAQI8ePbB06VJ8+umnuH79Ovbt2wcAOH/+PDp27oLL0dEwMzNFrZo1kZSUhNatW+vUv/feezh27BiOHTsGMzMzjB07VvphcOveI0yb8TOGfvUVbG1sULduXQwePBghISFSkLKzs4O7uztGjRqFf//9FzVr1tR53YsWLYKvry+ysrKkMKGt/fXXX9GtWzesWbMG165dQ1ZWFhQKBcLCwnDw4EH07NkT3333HWJiYjBw4EC0a9cO69evx7Vr13Ds2DE8ePAAzs7OePDgARYuXIimTZvCysoKmzdvhqenJ6pXr460tDT89ttv2LNnj067hgwZgsDAQEyePBmdOnXC0aNHsWDBAixYsCDX9y4qKgrOzs4Asnu46tWrh927d0uhRKPRYPfu3fjiiy8wbdo0ndpff/0Ve/bswcWLF1/a02JjY4PTp0/nWfu///0PNWvWfOm5teFn1apV6NWrF1avXi19r/NiaWkJS0tLxMfHY8eOHVK7XxSCrly5gnv37qFu3brSNrVajf3792POnDlIT0/X6bF7Ge0q2xMmTCjQ/jn9+OOPmDJlCnbt2iV19zs5OeUZ7mxsbKRemRf1wP3111+vNLSl5eHhAXd3d1SpUkXa9v7770uLSgKAl5cXbt68iVq1aukMoa5duxY///wz+vTp89L6l9m3bx8mT56MsLAwDBs2DGFhYfj1118xceJEjB07VtpP+8sWeBbipk6dikePHuHXX3+Fp6cngoKCEBkZqVP7shClpQ2AMTExOj1s2kC4fPlyeHp64sqVKxg9ejRatWqFQ4cOSe+XnTt3AgA6d+6sc9yc505LS8OIESPw0UcfwcbGBlevXsWePXuk4bY+ffpg+vTpyMzMxPjx418a4EqVKoX4+Hh069YNy5cvR6tWrbBp0yZMmjQJ48ePBwA8evQIDx8+zLP30dvbG4sXL0atWrWQmJiIH3/8EYGBgTh79myuKxSB7KF1a2trdOjQ4aVfR30wyFCBaYemRq87A7UQUMhkmNyhBjr7uQMAHqdl4lb8E9x8lIqb8U9wKz4VNx9l/3v9QQrSsnL3Hmw9feeV2qJ0rwWPEZvh42oLWwszmJvKYWFmAnMzBSZuPoeaXUfBz1SBJQevwdxUgaibmUh5bzKcAMhkwAfvVsRiHxcoTeRQmSqgNJFDaZL9r1ye+7LvNcdisDjeG+LD6XCWAR97JGNi/y7SXb2fD1JxcXFwc3OTLrWNS3yC09fi8L+//kLoF1/o9Cxpa8uXL4/Nmzfj7t27MDExQfv27XH48GF4enri93V/46shg1HP1xdWlpZ48OCB1B2vrc/p3LlzqFOnDpo0aYJGzVrgx4GD8fDeHZibm8PLywvz5s1D3759pRDl6OiI9evXo2fPnhg7diwqV66MmTNn4v79+9i4caMUgn799Vfs3r0bc+fOlUJUhw4dMG7cOPj6+uLAgQNSD0fv3r11/mpPTk7W6ZGJi4tDVFQUlEol0tPTMWvWLADZwe7UqVNSKNQqVaoUhBDIysqS9nvRuXv27Ik//vgDPXr0wM8//wx/f39pDldOO3bsgBAC3t7euHz5MoYNG4YqVaqgZ8+eLw1BtWrVyhW0evbsiSpVqkjBt6CeX2W7oKZNm4bvv/8eO3bsgK+vr7Q9ICAg1y/+nTt3IiAgADt27ADw8h44ExMTqUdN2+MVHx8PGxsbmJubS3PK8gpL2vd8Wloa4uLipBvH5uzher7HTDuEevnyZZQtW1bv+pzGjh2LTz75BAMHDsTIkSNRpUoVTJ48GX379sWYMWNw9+5dqFSqPEPc/PnzMXDgQHz22We4ffs2AKBfv34ICwvDmDFjcg035sfLyytXD1uZMmXQtm1bAEDNmjVRq1YtVKhQAfv27UPz5s0BZIdvALnaqJWZmYlOnTpBCCH1QGs0Gjg4OGDatGlYvXo1goKCYGlpiR9++EEKI0DeAW7hwoVwcHDAggULpO/pp59+innz5mH8+PG4efMm9uzZgwoVKuTZ4xcQEICAgADp88DAQOnrOXHixFz7L168GN26dTNIT6YWgwzp5WVDU9YqU1R1NkVVZ5tcdbcTUtFo6l6d3hyZDPi8sRdkMhkep2UiOS0Lj7WP9Cw8TsvE47QsJKdnQf18N9BTJ2+9vOcnL0IAc/Zcxpw9l/N83kwhzw42TwOOiRy48ejZkJpGAMuvW6H7osOwVpnCVCGHqUL29F85Jm89j+ofjYSPXI5f9l7G2duJ2Hb6DgQAp4F/wszfHTt7DYO5mQIq0+wApTLN/tjcVPuxAoqngWrNsRiMWhcLTaNhcH4HCOtQUwqP+cmuPQ2Tj+boBLC+ffsCyB3AWrdurdOTNW3aNAweMhS3b9+GpYUF3N3dIISQhgK19fXr18e4ceNw69YtWFpaYufOnVKIiUt8gmsPUhB3/jjmz58vtU1bGxISIv2Cfb5NYT/PxbUHKfC0t8Tt27dx6dIlKQTmd+4FCxYgKysLoaGhCA0N1fm6aAPclStXMH36dFy/fh1mZmbo3bs3vv/+e/z55586IejKlSu4du2atEbPnTt3ULVqVZ0euJs3b+rc4kPb45aYmIhLly5Jc8ee770bO3YsnJycdIYXtLXJycm4ffs21qxZIw2pXbt2DYMGDcLcuXOxatUq/P7775g+fTpmz54NKysr9OvXD3PmzMHw4cPRq1cv7NmzB2vXrsWWLVukr/PLeuBOnTqFrVu3wszMTOrxevjwIQICAqDRaLB37164urq+tCdMpVLB09MTdevWxe7du6Wv//P7aV/jlStX8Mknn0jb9KnPSbvOVc62BwYGAsjuMdu9ezcqV66cZ4gzNTWVwoo2xGlDmzZ8vyxEaWkvdHh+eDuvWi8vL9jb2+Py5cto3rw5UlJSsGnTJsjl8jyDooODAzp16oQbN25gz549sLHJ/lmrvVDB0dFRCp/aCxUyMjJeGuA0Gg0qV64MhUIhvW5bW1up9vjx40hNTcWZM2ek9+DLeh9NTU1Rp04dXL6c++frv//+i4sXL+rMozQEBhnSW0GHpnJyKWXx0t6clxFC4NqDFATNCNcJQnIZMOmDGlCZKpCaocaTDDWeZKqRmqFGWqYaqRlZeJKpQWx8Kk7EJOQ6ro3KBGqNQHqWBlk5Dpyh1iBDrcHj9KyXtiv80oOXPp/nawHwx5EY/HEkJt99TRUymCnkSMlQS9s0Ahjx12ksOnANKlMF5DIZTOQyyOUyKGQyKOTPHhlZGhy4/ECndsV1KwxcdQI2KlOYKLJrTRRymMhl+HlXNPx6fIMAuQy//XsVpgo5bru3ADpVhzMAGYD3/d3xg7eDTnAzUchgKpfD1EQGE7kcZk+33X+cjs2nbmPi5nPQiOzv1/L/rqOjb3Z3s/bCBhlkkMmyj6+92kEGYG3ETWkoUy4DwkIn5Lmw24vs27dPJ0T9X5uW0nM5w9KVK1ekocg5c+YAwEtD0PP12h64tLQ0nX1y9rpFRUXl2Xs3atQo3Lp1K9dVIDlrjx8/jj///DPXuQHoXLn2v//9D+PHj8e3336L5WvWYdTwr/Hzzz/DxcUF48aNk4LltWvXUL58ecyfPx8//fQTxo0bB0tLS8ycORPdunVDYGCgFIQ6d+6MESNGQK1WY/78+ejfvz9SUlIwZcoUDBo0CCdOnEC1atUgl8ulhQ3XrVuH2rVrIzY2FqmpqXj8+DHc3Nxw/vx5zJw5Ew8ePIC3tzeuX7+O/v3748KFC1AoFPjoo49w5MgRxMbGvrQ+JSUFfn5+UtibPXs2ypcvDzs7O7Rp0wYzZsxAnTp18PHHH+Orr77CmjVr0KRJEwwYMAApKSn47bff0KZNG9SpUwf+/v5o3LgxFixYgJCQEMydO1faXrFiRSxatAht2rSBQqGARqPBrl278OGHH+YaFs7Ze7hx40bIZDKdHpwXBbBbt27h4cOH0lDtn3/+iYyMDNSuXTvPoGhrawuZTIa9e/fqBN+GDRvijz/+gImJiRTgKlasCGdnZ5iYmLw0wJUtW1a6yEEbAMPDw+Hs7AwzMzO8++67cHBwwEcffSQNlb2s91GtVuP06dPSlWc5LVq0CPXq1YOPj0+u516HTLzOpQ/FQFJSEmxtbZGYmCilVzKeuMQnefbmFMSaYzGvFITymtujkMlwYOS7Uhuy1BqkZ2U/0jLVTz9WIy1Tg9vxqQhdFQnxXG/SsBBvWJgqkKkWyFBrkKnWIEstkPk0CGWqNbj16An2Xbqfq00V7C1hopAjLSs7dD3JUCMtS4OMPIbf6JkylmYwN3s2FGhmkt17ZpZjaDC7N02OGw9ScOjqIwhkB6NmVRxQ09UWCtnT4Pc0/MnlMshlgEIug/xpGDx+PR5/nbgl1X7cwB1NKjtkh7anwc/kaU+cifzpv0+3myrk2HzqNiZvPS+FsAntauCj+u6Qy3Jfmvo8bS+aFOD06IHLq17bC/e8F80ni0t8gr82/4N5U8ch+uIFaXLv48ePUbt2bcyaNQv+/v6YM2cOvv76a2RmZsLX1xezZs1CWloa+nzeDzeuXYOllRXeb/0evL29sWDBAty5cwe1a9eGjY0Nzp49K91w1cXFBbt27ZImXudX36NHjzzDZY8ePTBx+mx8N2ESdv/9P9yJuw2VSoWsrCwpHGjbvnnzZnTu3BlpaWmoXLkyhg4dip49e2Lk2G+x+o/f8fDeHVhaWiIhIQGzZ89G06ZNMXPmTPzxxx86l/A//7UEgHfeeQdqtRonTpzA/PnzUb9+fcycORNr167FRx99hO7du2PatGlQKpW4dOkSHj9+jNOnT0OpVOKdd95BuXLl0L59e/To0UOqnzFjBpYtWyZNoJ80aRKcnJwwZswY2NnZ4e7du6hevTp69OiBChUqYPjw4VCpVOjZsyfS0tKwdu1arFq1Cm3atEGNGjWkANenTx+EhYXhm2++QY8ePfDll19i0aJFmDZtGjp06IBJkyZJbb9w4QIcHR3RvXt37Nu3Dx06dMDMmTMxYcIENGjQABUrVkRCQgJ++OEHbNiwAcePH9fpAdJOpp8+fbrUo5ufgv7+ZpChYuVVg9CrhqDXrS9IiMpJ87SHKC0zu3fp5qNUdFl4WCdEyWXAjx19UMrCFFlqAY0QUGuALI1G+lit0eBRSiambb+AnP/BZQA+e8cT5qYKZGkE1BqBTLWAWqNBpkZArRbI1Gig1gjcSUzDkWuPcrWxQllLmJspkJmVvW/OAJf59N+sp+GOcpPLABO5HHI5cvWiAcCD5IxcNZUcrGBulj3caPJ0XxO5XPdzhQwZWQK7zusOScgAdPZzg625qbS/iUK31vTp55Ex8Vh3IlYKcF393fFOJfvsnj/Fs6CnDYGKp72BJk9D4K5zdzFrT7QUor4O9kbb2i5SSJTqZTLIcrx+7fY/I25i9PpXC3GGDoBhHWri/pFN+OGHH6QQpQ1CQPbQUfny5aUAE5f4BPuOnkTX4AD8888/uHjxok7ttGnTMHnyZERGRuLBgwewsLBA165dMXHiRDg6OuLfiJNo7Fcbq9b/jS4fvI85c+ZI9VWrVsXJkyfzbPfevXvhXccff/+zD/OnjcfZ06dgZWUFtVqN1NTUfANcnz598PfOfRg57GtcuXAG5cqVQ7Vq1XDq1KkXvu4LFy6gS5cumDlzJvp+8SU2bdyA+Af3ULp0adSrVw+TJk3S6VUEsns5Bw8ejLi4OJ2lJF6GQeYpBhnSep3eoNepN1aIet1afUPY8/KaFyWXAf8MaQJHm+y5JgLZc5YgAPE0cgkB3ElKQ+tZ/+aqXfppfVibmyD9ae/Vs3/VOh9fvpeMtRG3crUpqKoDylorodZkB77s4JcdBrUf33+cgRMx8blqK5S1hMpUkR3aNNlhLUudHQCztGFOk92GF0zpoldgbiqXgpdOD1qOnjWhEYjJsTSEVq1ytlCZKXKEPxkU8uyeM4Ui+3gmchnSs9TY8tyFBzIAnzTwQCkLUymsSbVPg6P2mBE34vG/iFu5AqBMpg1weNrupx/LsodTteFu74W7mLf/KoTI7u0d2KwSWtdyloaOFc8/ZM/avz4yFuM2njFogCuq8JifEhVkfvnlFymZ+vj4YPbs2QVeb4FBht4ExgpRr1v7Noaw1w1wedXLZcC2Qe+gjJUSGo2AWogcvWlPH0LgbmIaPl16LFcP3IxOtWFrbvq0F00j9aZlqZ/++3T7o5QMzNwVnasX7uMGHlCZynP1wmXlOMbdx2mIuJ47wFV2tIKFmYnUzufbrP34SYYaCU8yc9WbyrMnQGXX5vvlIwOQIXuoVPZ0KFMGSCFKJns2Jw0AktJyzwV0K20uXXBg8nT49Pneuyy1BgevPNSp0+f/SUGUmCCzZs0adO/eHfPmzYO/vz9mzpyJP//884XLRD+PQYbo9byNIay4BrjXqS+MAPd8vRACQkAKQDk/vpP4BK1+zt0Lt7pvA9hbKXMMm+YIU0JAoxG4l5SWax6bXAZ8/0ENWJubSmErK+e/6meBMOFJBubtu5orAHb0ddUZhtX9V/O0By89z4sJKjtawUppAo3Ift0agac9f9lDyNpewNR0NeKS0nLVW6tMIJfJnoXGHMHxTbaqTwMEVCiT/44FUGKCjL+/P/z8/KSrCTQaDdzc3PDll19i5MiR+dYzyBC9vYwVoorruY0Z4F63viQHwJy0oShLo8Ht+DQ0n7EvV/jbGNoQDjYqaJ4GR+2/0sfI/vdeUjq6/XY4V/2v3erC1twsu/dOo4FanR3esp6Gtyy1wIPkdEzZpjsHjz0yecjIyICFhQX+97//6Sxl3qNHDyQkJGDjxo25atLT05Geni59npiYCHd3d9y8eZNBhogoH3cSnyDm4RO4lzGH0yv8QjJmvbHO/dfxm5jw93kpBI1rUxUf1nMrkvrifO78JCUlwc3NDQkJCS+fIPzSe2MbWWxsrAAg/vvvP53tw4YNE/Xr18+zZvz48S+8ZTgffPDBBx988FG8Hjdv3nxpVihxC+KNGjVKZ8EojUaDR48eoUyZMvmu36APbVJ81Z4eY9bz3Dx3cannud+uc79uPc9d/M79MkIIPH78GC4uLi/d740OMjnv9ZHTy5aJViqV0jLiWtq7EBcGGxub1/rmGbOe5+a5i0s9z/12nft163nu4nfuFynImjP63QWriOW8X4aWdqnmnDepIiIiorfTG90jA2TfV6RHjx7w9fWVlnrW3t2WiIiI3m5vfJDp3Lkz7t+/j3HjxknLJW/fvl26AZqxKJVKjB8/PtcwVnGo57l57uJSz3O/Xed+3Xqeu/id2xDe6MuviYiIiF7mjZ4jQ0RERPQyDDJERERUbDHIEBERUbHFIENERETFFoPMK/rll19Qvnx5qFQq+Pv74+jRowWq279/P9q0aQMXFxfIZDJs2LChwOcMCwuDn58frK2t4eDggA8++AAXL14scP3cuXNRq1YtaeGigIAAbNu2rcD1OU2ZMgUymQyDBw8u0P7ffvvt09vHP3tUqVKlwOeLjY3Fxx9/jDJlysDc3Bw1a9ZEREREgWrLly+f69wymQyhoaH51qrVaowdOxaenp4wNzdHhQoVMHHiROgzR/7x48cYPHgwPDw8YG5ujsDAQBw7dizPffN7fwghMG7cODg7O8Pc3BxBQUGIjo4uUO26desQHBwsrXIdFRVV4HNnZmZixIgRqFmzJiwtLeHi4oLu3bvj9u3bBTr3t99+iypVqsDS0hKlS5dGUFAQjhw5UuDXnVO/fv0gk8kwc+bMAtV++umnub73LVu21Ovc58+fR9u2bWFrawtLS0v4+fkhJiamQPV5vfdkMhl++OGHfGuTk5MxYMAAuLq6wtzcHNWqVcO8efMK3Pa7d+/i008/hYuLCywsLNCyZUvp/VKQnydpaWkIDQ1FmTJlYGVlhQ8//BB3794tUO2CBQvQtGlT2NjYQCaTISEhQXouv/pHjx7hyy+/hLe3N8zNzeHu7o6BAwciMTGxQOf+/PPPUaFCBZibm6Ns2bJo164dLly4UODXrSWEQKtWraSvbUFqmzZtmut73a9fP73OfejQITRr1gyWlpawsbFB48aN8eTJk3zrr1+//sL3W9euXfM99507d/DJJ5/AyckJlpaWqFu3Lv76668Ct/3KlSto3749ypYtCxsbG3Tq1CnXgraFgUHmFaxZswZDhw7F+PHjceLECfj4+CAkJAT37t3LtzYlJQU+Pj745Zdf9D5veHg4QkNDcfjwYezcuROZmZkIDg5GSkpKgepdXV0xZcoUHD9+HBEREWjWrBnatWuHs2fP6tWOY8eOYf78+ahVq5ZeddWrV0dcXJz0OHDgQIHq4uPj0bBhQ5iammLbtm04d+4cpk+fjtKlSxe4vTnPu3PnTgBAx44d862dOnUq5s6dizlz5uD8+fOYOnUqpk2bhtmzZxfo3ADw2WefYefOnVixYgVOnz6N4OBgBAUFITY2Nte++b0/pk2bhlmzZmHevHk4cuQILC0tERISgrS0tHxrU1JS0KhRI0ydOvWFz7+oPjU1FSdOnMDYsWNx4sQJrFu3DhcvXkTbtm0L1O7KlStjzpw5OH36NA4cOIDy5csjODgY9+/fL1C91vr163H48GGdJcsLUtuyZUud98CqVasKXH/lyhU0atQIVapUwb59+3Dq1CmMHTsWKpWqQPU5zxsXF4fFixdDJpPhww8/zLd26NCh2L59O1auXInz589j8ODBGDBgADZt2pTvuYUQ+OCDD3D16lVs3LgRkZGR8PDwQFBQEFJSUgr082TIkCH4+++/8eeffyI8PBy3b99Ghw4dClSbmpqKli1bYvTo0bnall/97du3cfv2bfz44484c+YMli5diu3bt6N3794FOne9evWwZMkSnD9/Hjt27IAQAsHBwVCr1Xr9HJ05c6bOrW0KWtunTx+d7/m0adMKXH/o0CG0bNkSwcHBOHr0KI4dO4YBAwZALpfnW+/m5pbr/fbdd9/BysoK9+/fz/fc3bt3x8WLF7Fp0yacPn0aHTp0QKdOnRAZGZnvuVNSUhAcHAyZTIY9e/bg4MGDyMjIQJs2baDRaHJ9bQ3qte/s+BaqX7++CA0NlT5Xq9XCxcVFhIWF6XUcAGL9+vWv3I579+4JACI8PPyVj1G6dGnx22+/FXj/x48fi0qVKomdO3eKJk2aiEGDBhWobvz48cLHx+eV2jhixAjRqFGjV6rNy6BBg0SFChWERqPJd9/WrVuLXr166Wzr0KGD6NatW4HOlZqaKhQKhdi8ebPO9rp164oxY8a8tPb594dGoxFOTk7ihx9+kLYlJCQIpVIpVq1a9dLanK5duyYAiMjIyAKfOy9Hjx4VAMSNGzf0rk1MTBQAxK5duwp87lu3boly5cqJM2fOCA8PD/HTTz8VqLZHjx6iXbt2L23Py+o7d+4sPv7441euf167du1Es2bNClRbvXp1MWHCBJ1tL3rvPF9/8eJFAUCcOXNG2qZWq0XZsmXFwoULc9U///MkISFBmJqaij///FPa5/z58wKAOHTo0Etrc9q7d68AIOLj43M9V5B6rbVr1wozMzORmZmpd+3JkycFAHH58uUCnzsyMlKUK1dOxMXFvfD7mletPj8X86r39/cX33zzzSvXP6927dq5foa9qNbS0lIsX75cZz87O7sCvV927Ngh5HK5SExMlPZJSEgQMplM7Ny5s0Cv51WxR0ZPGRkZOH78OIKCgqRtcrkcQUFBOHToUJG2JTExEQBgZ2end61arcbq1auRkpKi1+0eQkND0bp1a53XX1DR0dFwcXGBl5cXunXrJnXN52fTpk3w9fVFx44d4eDggDp16mDhwoV6nx/I/v6tXLkSvXr1KtBNRAMDA7F7925cunQJAHDy5EkcOHAArVq1KtD5srKyoFarpb/etczNzQvcI6V17do13LlzR+drb2trC39//yJ/7wHZ7z+ZTKb3vcwyMjKwYMEC2NrawsfHp0A1Go0Gn3zyCYYNG4bq1avr3dZ9+/bBwcEB3t7e6N+/Px4+fFjg827ZsgWVK1dGSEgIHBwc4O/vr9eQcE53797Fli1b0Lt37wLtHxgYiE2bNiE2NhZCCOzduxeXLl1CcHBwvrXp6ekAoPPek8vlUCqVeb73nv95cvz4cWRmZuq836pUqQJ3d/dc77fX+VlU0PrExETY2NjAxMQk1/aX1aakpGDJkiXw9PSEm5tbgc6dmpqKrl274pdffnnhff1edu7ff/8d9vb2qFGjBkaNGoXU1NQC1d+7dw9HjhyBg4MDAgMD4ejoiCZNmrzwZ0V+r/348eOIiorK8/2WV21gYCDWrFmDR48eQaPRYPXq1UhLS0PTpk3zrU9PT4dMJtNZGE+lUkEul+v9s05vhRqTSqDY2FgBQPz3338624cNGybq16+v17HwGj0yarVatG7dWjRs2FCvulOnTglLS0uhUCiEra2t2LJlS4FrV61aJWrUqCGePHkihNDvL4+tW7eKtWvXipMnT4rt27eLgIAA4e7uLpKSkvKtVSqVQqlUilGjRokTJ06I+fPnC5VKJZYuXVrgtmutWbNGKBQKERsbW6D91Wq1GDFihJDJZMLExETIZDIxefJkvc4ZEBAgmjRpImJjY0VWVpZYsWKFkMvlonLlyi+te/79cfDgQQFA3L59W2e/jh07ik6dOr20NidD9Mg8efJE1K1bV3Tt2rXAtX///bewtLQUMplMuLi4iKNHjxb43JMnTxYtWrSQetH06ZFZtWqV2Lhxozh16pRYv369qFq1qvDz8xNZWVn51mv/GrewsBAzZswQkZGRIiwsTMhkMrFv374Cv3atqVOnitKlS0v/h/KrTUtLE927dxcAhImJiTAzMxPLli3L89jP12dkZAh3d3fRsWNH8ejRI5Geni6mTJkiAIjg4GCd2rx+nvz+++/CzMws13n8/PzE8OHDX1qbU349MgX5WXb//n3h7u4uRo8eXeDaX375RVhaWgoAwtvbO8/emBfV9+3bV/Tu3Vv6PK/vzYtq58+fL7Zv3y5OnTolVq5cKcqVKyfat29foHMfOnRIABB2dnZi8eLF4sSJE2Lw4MHCzMxMXLp0qcCvXat///6iatWqBX7d8fHxIjg4WHq/2djYiB07dhSo/t69e8LGxkYMGjRIpKSkiOTkZDFgwAABQPTt2/eFbTQEBhk9vSlBpl+/fsLDw0PcvHlTr7r09HQRHR0tIiIixMiRI4W9vb04e/ZsvnUxMTHCwcFBnDx5UtqmT5B5Xnx8vLCxsSnQsJapqakICAjQ2fbll1+KBg0a6H3e4OBg8f777xd4/1WrVglXV1exatUqcerUKbF8+XJhZ2enV4i6fPmyaNy4sQAgFAqF8PPzE926dRNVqlR5ad2bGmQyMjJEmzZtRJ06dXS6kfOrTU5OFtHR0eLQoUOiV69eonz58uLu3bv51kdERAhHR0ed8KlPkHnelStXCjyspf3//tFHH+ns16ZNG9GlSxe9z+/t7S0GDBiQ53N51f7www+icuXKYtOmTeLkyZNi9uzZwsrKKs+u+rzqIyIihI+Pj/TeCwkJEa1atRItW7bU2S+vnycFDTL5/SzKL8jkV5+YmCjq168vWrZsKTIyMgpcm5CQIC5duiTCw8NFmzZtRN26dXMFyLzqN27cKCpWrCgeP34sbcvra1vQn8G7d+/Oc1grr3rt//FRo0bp7FuzZk0xcuRIvc6fmpoqbG1txY8//pjruRfVDhgwQNSvX1/s2rVLREVFiW+//VbY2tqKU6dOFah+x44dwsvLS8hkMqFQKMTHH38s6tatK/r16/eCr45hMMjoKT09XSgUilxv6u7du4u2bdvqdaxXDTKhoaHC1dVVXL16Ve/a5zVv3rxAaXn9+vXSD0PtA4D0hs3rr9v8+Pr65vrPmRd3d3edv46EEOLXX38VLi4uep3v+vXrQi6Xiw0bNhS4xtXVVcyZM0dn28SJE4W3t7de5xYi+xe5NoR06tRJvPfeey/d//n3h/YX8PMBpHHjxmLgwIEvrc3pdYJMRkaG+OCDD0StWrXEgwcP9Kp9XsWKFfPs3Xq+/qeffpLeZznfe3K5XHh4eLzSue3t7cW8efPyPXd6erowMTEREydO1Nlv+PDhIjAwMN/6nPbv3y8AiKioqDyff742NTVVmJqa5ppf1bt3bxESEqLXuRMSEsS9e/eEENlz/L744gvpuRf9PNH+An4+gLi7u4sZM2a8tDanlwWZ/OqTkpJEQECAaN68ea4Qos/PwfT0dGFhYSH++OOPfOsHDRr0wvdbkyZN9D53cnKyACC2b9+e77mvXr0qAIgVK1bobO/UqZNO72dBzr98+XJhamoqfd/zq718+XKuOVVCZP+O+Pzzz/U69/3796Xvt6Ojo5g2bdoL9zUEzpHRk5mZGerVq4fdu3dL2zQaDXbv3q3XXJNXIYTAgAEDsH79euzZsweenp6vfUyNRiONpb9M8+bNcfr0aURFRUkPX19fdOvWDVFRUVAoFHqdNzk5GVeuXIGzs3O++zZs2DDXZX6XLl2Ch4eHXudcsmQJHBwc0Lp16wLXpKamQi7X/W+iUCheaRa+paUlnJ2dER8fjx07dqBdu3Z61Xt6esLJyUnnvZeUlIQjR44U+nsPyL4Eu1OnToiOjsauXbtQpkyZ1zpeQd97n3zyCU6dOqXz3nNxccGwYcOwY8cOvc9769YtPHz4sEDvPTMzM/j5+Rnk/bdo0SLUq1evwPOCMjMzkZmZaZD3n62tLcqWLYvo6GhERESgXbt2+f48qVevHkxNTXXebxcvXkRMTAwaNGjwWj+LCvKzLCkpCcHBwTAzM8OmTZukuT6v8nNQZP/RjvT09HzrR44cmev9BgA//fQTFi9erPe5tfXOzs75nrt8+fJwcXF54ftNn9e+aNEitG3bFmXLlpW+Bi+r1c7jedH7TZ9z29vbo1SpUtizZw/u3bsnXd1YaAo1JpVQq1evFkqlUixdulScO3dO9O3bV5QqVUrcuXMn39rHjx+LyMhIERkZKQBI4+7PX/mRl/79+wtbW1uxb98+ERcXJz1SU1ML1O6RI0eK8PBwce3aNXHq1CkxcuRIIZPJxD///FOg+ufpM7T01VdfiX379olr166JgwcPiqCgIGFvb5/rr4W8HD16VJiYmIjvv/9eREdHi99//11YWFiIlStXFritarVauLu7ixEjRhS4RojsK17KlSsnNm/eLK5duybWrVsn7O3tdbrW87N9+3axbds2cfXqVfHPP/8IHx8f4e/vn6ubXIj83x9TpkwRpUqVkuZ8tGvXTnh6eoonT57kW/vw4UMRGRkptmzZIgCI1atXi8jISBEXF5fvuTMyMkTbtm2Fq6uriIqK0nn/paenv7Q2OTlZjBo1Shw6dEhcv35dREREiJ49ewqlUin99afv/4ucQ0svq338+LH4+uuvxaFDh8S1a9fErl27RN26dUWlSpVEWlpagc69bt06YWpqKhYsWCCio6PF7NmzhUKhEP/++2+B256YmCgsLCzE3Llz9fp+N2nSRFSvXl3s3btXXL16VSxZskSoVCrx66+/Fqh+7dq1Yu/eveLKlStiw4YNwsPDQ3To0EEIUbCfJ/369RPu7u5iz549IiIiQgQEBIiAgIAC1cbFxYnIyEixcOFCAUDs379fREZGiocPH+Zbn5iYKPz9/UXNmjXF5cuXdfbp16/fS2uvXLkiJk+eLCIiIsSNGzfEwYMHRZs2bYSdnZ24e/fuK/0cxdPervxqL1++LCZMmCAiIiLEtWvXxMaNG4WXl5do3Lhxgb/mP/30k7CxsRF//vmniI6OFt98841QqVTi8uXLBW57dHS0kMlkYtu2bdK2/GozMjJExYoVxTvvvCOOHDkiLl++LH788Uchk8nEli1bCnTuxYsXi0OHDonLly+LFStWCDs7OzF06NAXfl0NhUHmFc2ePVu4u7sLMzMzUb9+fXH48OEC1Wm7WZ9/9OjRI9/avOoAiCVLlhTo3L169RIeHh7CzMxMlC1bVjRv3vyVQ4wQ+gWZzp07C2dnZ2FmZibKlSsnOnfunOfkuxf5+++/RY0aNYRSqRRVqlQRCxYs0KutO3bsEADExYsX9apLSkoSgwYNEu7u7kKlUgkvLy8xZswYkZ6eXuBjrFmzRnh5eQkzMzPh5OQkQkNDRUJCQp775vf+0Gg0YuzYscLR0VEolUrRvHlz6TXlV7tkyZI8nx8/fny+9drhqLwee/fufWntkydPRPv27YWLi4swMzMTzs7Oom3btjqTffX9f5EzyLysNjU1VQQHB4uyZcsKU1NT4eHhIfr06aPzR0dBzr1o0SJRsWJFoVKphI+Pj87wZEHq58+fL8zNzXN93/OrjYuLE59++qlwcXERKpVKeHt7i+nTp0uTnvOr//nnn4Wrq6swNTUV7u7u4ptvvpHeuwX5efLkyRPxxRdfiNKlSwsLCwvRvn17aQJ0frXjx49/4T751b/odb3soa2NjY0VrVq1Eg4ODsLU1FS4urqKrl27igsXLhT4dT9PG2Tyq42JiRGNGzcWdnZ2QqlUiooVK4phw4ZJc8kKeu6wsDDh6uoqLCwsREBAgBSaC1o/atQo4ebmJtRqtc5ryK/20qVLokOHDsLBwUFYWFiIWrVqSZdjF6R+xIgRwtHRUZiamopKlSrpvFcLk+xpA4mIiIiKHc6RISIiomKLQYaIiIiKLQYZIiIiKrYYZIiIiKjYYpAhIiKiYotBhoiIiIotBhkiIiIqthhkiIiIqNhikCGit86+ffsgk8mQkJBg7KYQ0WtikCEiIqJii0GGiIiIii0GGSIqchqNBmFhYfD09IS5uTl8fHzwv//9D8CzYZ8tW7agVq1aUKlUaNCgAc6cOaNzjL/++gvVq1eHUqlE+fLlMX36dJ3n09PTMWLECLi5uUGpVKJixYpYtGiRzj7Hjx+Hr68vLCwsEBgYiIsXLxbuCycig2OQIaIiFxYWhuXLl2PevHk4e/YshgwZgo8//hjh4eHSPsOGDcP06dNx7NgxlC1bFm3atEFmZiaA7ADSqVMndOnSBadPn8a3336LsWPHYunSpVJ99+7dsWrVKsyaNQvnz5/H/PnzYWVlpdOOMWPGYPr06YiIiICJiQl69epVJK+fiAyHd78moiKVnp4OOzs77Nq1CwEBAdL2zz77DKmpqejbty/effddrF69Gp07dwYAPHr0CK6urli6dCk6deqEbt264f79+/jnn3+k+uHDh2PLli04e/YsLl26BG9vb+zcuRNBQUG52rBv3z68++672LVrF5o3bw4A2Lp1K1q3bo0nT55ApVIV8leBiAyFPTJEVKQuX76M1NRUtGjRAlZWVtJj+fLluHLlirRfzpBjZ2cHb29vnD9/HgBw/vx5NGzYUOe4DRs2RHR0NNRqNaKioqBQKNCkSZOXtqVWrVrSx87OzgCAe/fuvfZrJKKiY2LsBhDR2yU5ORkAsGXLFpQrV07nOaVSqRNmXpW5uXmB9jM1NZU+lslkALLn7xBR8cEeGSIqUtWqVYNSqURMTAwqVqyo83Bzc5P2O3z4sPRxfHw8Ll26hKpVqwIAqlatioMHD+oc9+DBg6hcuTIUCgVq1qwJjUajM+eGiEom9sgQUZGytrbG119/jSFDhkCj0aBRo0ZITEzEwYMHYWNjAw8PDwDAhAkTUKZMGTg6OmLMmDGwt7fHBx98AAD46quv4Ofnh4kTJ6Jz5844dOgQ5syZg19//RUAUL58efTo0QO9evXCrFmz4OPjgxs3buDevXvo1KmTsV46ERUCBhkiKnITJ05E2bJlERYWhqtXr6JUqVKoW7cuRo8eLQ3tTJkyBYMGDUJ0dDRq166Nv//+G2ZmZgCAunXrYu3atRg3bhwmTpwIZ2dnTJgwAZ9++ql0jrlz52L06NH44osv8PDhQ7i7u2P06NHGeLlEVIh41RIRvVG0VxTFx8ejVKlSxm4OEb3hOEeGiIiIii0GGSIiIiq2OLRERERExRZ7ZIiIiKjYYpAhIiKiYotBhoiIiIotBhkiIiIqthhkiIiIqNhikCEiIqJii0GGiIiIii0GGSIiIiq2/h+5yQxUdZWhjwAAAABJRU5ErkJggg==", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAG+CAYAAABvfyUjAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABUAUlEQVR4nO3deVxU5f4H8M8ZlhlAARFZZRWX3HBDAs0lDSyvW/1S01KzTdObZVpapmX3RnveVrNreq2b2uKWluauGWooipopriiCuLDIrvD9/UEzl4FZWUJPn/frNS/lcL7zPPOc55zzmTMLiogIiIiIiFRC09AdICIiIqpLDDdERESkKgw3REREpCoMN0RERKQqDDdERESkKgw3REREpCoMN0RERKQqDDdERESkKgw3REREpCoMN0RERKQqDDdEtZSQkICoqCg0btwYPj4+GDp0KI4dO2Zz/bJly6AoCoYOHVrtd0ePHsXgwYPh4eEBNzc3REVFIS0trQ57/9fEbUb24Hy59TDcENXS9u3bMWnSJOzevRsbN27E9evXERcXh4KCAqu1Z86cwRNPPIFGjRph7dq1RgfOkydPomfPnmjTpg22bduGlJQUvPTSS9DpdEb3Ye7AOW7cOCiKYnQbMGCA0Tq38kG7Nn3fvn07unXrhvz8fLRr185om1kbtzNnzmDatGm44447qt2vLdustmM+cuRIKIoCR0fHavW2jHlttll99d2WuWqp7/U91y3NF2vjVtt9PCEhAS1atICiKNBqtdX6bW2b/WWDlRBV8tprr0m3bt2kUaNG0qxZMxkyZIj8/vvvNteGh4cLAHF2djaqHTt2rAAwusXHxxvVL126VADIkCFDjJbbUnszycrKEgCyfft2i+vduHFDYmNjpV27dtKjRw/p27evHDhwQO655x4JDg6W++67Tx588EGL93H69GkJDAyUO+64w+S4DRgwQDIyMgy3q1evGq0THx8vixYtksOHDxu1nZ+fb/VxWmr7xIkT4uXlJdOnT5f9+/fLiRMnZPXq1XLx4kWr92uruux75W1madz02+zee++Vpk2bioODg9F+MmLECKvbLD4+XiZMmCAApE+fPkb9tjbXT58+LVqtVlq1alVtvqSkpFgd8+nTp4uTk5M4ODhU20dt2WYdO3YUANXaHjVqlNV91FLfR40aZXWu3kxzvfJ8sTRudbGP9+rVSzw9PaVLly7V5ou1bdbQ+2hDYrgxo75O8pU98cQTAkDee+89o+UjRowQANUOnLbUipgPCbbU1+YAYGkntHbwqe2BqzbbS6R2Y1ZVamqqAJBDhw5ZXG/27NkydOhQEal4jPq29QdOFxcXmTt3rsTFxUmzZs2ke/fusnLlSkO9tRNt5fu09bGbC2ZVH7u+7X//+98m27HlJF/X7A2VlfteeZtZGjf9NouPjzd5snJzc7O4zURqH6yqjrm+vm/fvhbH/MaNG+Lp6SkPP/ywDB48uNo+am2bmdpH9W3Hx8db3Eet9T0+Pt7iXL1x44YEBwdLaGioODo6mnzyZG2uV93HTc0XU/u4tfliadzqah+v3Hblfltquy6CeG3PZXV5XLUXX5YyozYvNaxfvx5Xr15Fly5dEBsba7J25cqV2L17NwICAoxqz5w5g1WrVqFVq1bo1auXybbN1QLAc889hzFjxsDBwQE//vijyUuvlurHjRuHhx9+GC+++CIiIyOxePFipKWlYfLkyWjTpg3c3NzQpEkT9O/fH3v27DHUlZWV4caNG3j77bfRoUMHeHh4GGr37dsHANBqtfDz8zPcmjRpYqgdPXo0oqOj8dtvv1W7dGuptvL2MnfZ+OWXX7bYd0uXjW0ZM+B/l36HDBmCp59+Gj169ED79u2N1pkwYQIURcG8efPw888/Y+HChfjss8+q3Vdubi4AoKioCK+//joGDBiAn376CcOGDcO9996L7du3AwDmzp0LHx8fFBQUoE2bNtXmy/Xr17Ft2zb4+PigdevWmDhxIq5cuWLUVtWXWPRte3l5WXzs+razsrKwdu1ao3E7evQo1q1bh1atWiE+Ph4+Pj6Ijo7GqlWrzI6bqUvmpsbNElN9N0Xf90ceeQQAICLVtpmpcau8zdavX4+IiAi4u7sb7ScFBQUWt5l+rr/yyisIDw832W9zc71qv6s+7t27d1sc87lz56JPnz74/PPP0aRJE6N99Ndff7W4zUz1u3LbWq3W4j5qre9ardbiXJ07dy6Ki4sxZ84c3HPPPSaPq5bqTb2UWHXcze3jluZL27ZtzY5bXe7jlcdNX+vp6Wlxm1k7Pqxdu9bqPlrbc5m5l2+t1daJOo1KKlabZ4VVa8+fPy+BgYFy+PBhCQkJsfpsuHK9uVp9vaVnZpbaFjH9zEz/DOX111+XjRs3ysmTJ+Xw4cPyyCOPiLu7u2RlZYmI6WcoVZ8Ne3h4SLNmzaRVq1YyYcIEuXz5slGtuWfDo0aNMltrru+Vx+y///2v2b5bu2ycn59vccyqth0aGiohISFy7tw5o3VWrFghkZGREhAQIAkJCRIaGio//PCD4ff6MSsrK5OBAwdKVFSUAJAHHnjA6H4GDRokI0eOlJ07d0pgYKBcunTJqL7yfJk9e7asXr1aUlJSZOXKlXLbbbdJVFSU3Lhxw+R8Gzx4sAwcOFB69OhhaM/UY6/ctqltFhgYKADE1dVV3n33XUlOTpaEhARRFEW2bdtmuG9LL5GYGjdLzwr149ajRw+ZM2eOtG7dWlxdXcXT01P69esnu3fvFhExOW5Vt9nSpUurjVuXLl3MbrPK+4mlbSZSfT+pOubm9hNz27vqfDE35ubq9f3eunWrxXpT+3flMbe0f1vre48ePUyOuX6umquvvI9bqjd1XK3ctrl5bst8ycjIMDluAMTPz6/O9/HK88Vc24qiyPvvv2/1+GBtH62Pc1ll1o6rtcVwY6PavNRQubasrEz69u0r8+bNExEx2qjmLmHq6w8ePGi21lx95cloqW1bDgCV5ebmCgDZtGmT1Z1QxPQJIyoqSrZt21bnJ+mqY26p79YuG2/dutXimFduu3Xr1qLT6eTUqVNG7VXdiadNm2Z42VF/UxTFcAsMDJSTJ0+Ko6OjvPrqq0b39dxzz0l0dLRNJ9qqj/3kyZOGx21qvlQ9yZuaL5aCWeVxs3aSr+sgPmHCBEPfzYXZU6dOVeu7uW1matzMbTONRiN9+/aV2NhYs9ssNja2XoKV/nH/+uuvZsf8vvvuM1lfeR9NT083W9+vXz+T+2jlMTe3f2dnZ1vte9UnApXHfM2aNWbrLe3jlee6qX28ctvmjot5eXlW54u5cevVq1e97OOV54u5tu+++25xc3Or8yBeF+cyPUu1dcWxrq8EqVF5ebnZlxoq01+GPHDggGGZVLnknZCQAEdHRzz11FNWa6u2vW7dOpO1luorX3p94403zNZXvvy5c+dOAMCkSZNw+PBh/Pzzz0brlpaWYsGCBfDw8ECLFi3Qt29ffPbZZ/D29jask5KSAhEx1I4cOdLwuw4dOqBjx45o0aIFRo4cic8//9yotmrf77//fsO4V67dtm0b+vXrV63vVcfcXN+LiopMjlnltleuXGl2zPTj1qxZMyQnJyMtLQ09evRAWFiY4ffl5eV46KGHMH36dLRr1w4A4OPjg0OHDhndz6xZs5CUlIQbN25g8+bNCA8PR1RUVLWXFI8fP44mTZpgz549GDRokFE7AODg4IDevXubfOzh4eHw9vbGiRMnoNVqjR777t27kZmZid9++w3NmzcHAJPz5cqVKzhz5ozJth0dHbFhwwZDP9q2bWvU/m233WaYD5VfItF/0uWzzz6Dj48P9u3bh549e1YbN73KL5Hs3LkTO3bswL59+7Bjxw40b94co0aNMlr/3XffxcKFC/HTTz8Z+i4VT+wgIgCAli1b4tixY2jRogWqCg8PR9OmTfHkk09i+PDhRtvs2rVr8Pb2xp49e/Dzzz9j+PDhJrdZQEAAHnroIaP9xNSYm9tPAFQbc33ft27dio4dO8LR0dHkmG/YsMHkNtPX79y5E97e3ibrw8PDMX/+fKxcubLa/l15zM31e/ny5Rbb3rp1q+GxV21XP67m6levXo2uXbuaPCbr69evX4+lS5ca7eNV+27umHzy5Emr8yUoKMjkuHXr1g25ubn48ssvDctqu4+XlZUBADQaDUpKSsy2rX8pytLxITY2Fnv37jW7j9bHuawyS+eiusJwYwNzJ/nKrl27Vu3gBRif5Pft24d//etf2L9/PxRFMaxTXFxssrZy2x9//DHGjx9frdZS25UnY0lJicm2AdOTseoBAADWrl2LkSNHorCwEP7+/ti4cSNycnKqHXyq7oSmhIeHw9PTE5mZmXV6kq465npV+75q1So8/PDDJsdcHygjIyOxfPlyk2NWedzi4uLw5ZdfolevXgCAzMxMAICHhwfmzZsHR0dH7Nu3DzNnzgQAODk5GT0mEcHvv/+OK1euICUlBS1btgQATJ8+HSNGjECvXr3Qt29frF+/Ht9//z02bNiAt956y6gvpk60VZ0/fx5XrlyBh4eHYb40bdoUkydPrhbMzM1Vc8Hs2rVreO+99zBjxgz06NED5eXlJg/aISEhdRbEx48fj7feegsZGRlISUkxCpV6lcPswIEDDX3/xz/+gR9++AEdO3Y0/Ozk5ISioiK4uLgAAMaMGYPAwEBMmjQJV69eRbdu3Yy2m6enJ1JSUnD8+HHDfmJum3322Wf49ttv6yRYiQiGDBmC9PR0rFixArGxsXB2djZ7omzdurXRSRYABg8ejAsXLmDVqlXo3r272fqDBw+ipKTEsI+KiGEf1Wg0hv+b6re3tzeuX79uNF9M9d0U/Vzt1KmT2ScCIoKvv/7abP3ly5exZMkSLF68GN7e3hAR7N6922i+mJvnANCmTRur86WsrMwwbvr5kpCQgFOnTqFt27aG+VKbfVxE8Nprr2HVqlXo1KkTPvvsMwQFBZndZlevXsXdd9+NN99802jMahPEgdqfy/QsjXmdqtPrQCo0adIkad68ucXL1iIiycnJ1S5D4o9LfxqNRk6cOCHvvfeeKIpSbR1FUUxewtTXb9261WytRqMRf39/i/U7d+602LaDg4PhEmZ5ebnh0uvx48eNHmN+fr6kpqZKYmKijB8/XkJDQ+Xs2bNy6NAhOXTokKSkpMjIkSNFp9PJ7bffLocOHZKSkhKT43Xu3DkBIO+//76h/tChQzJkyBC58847Zfjw4WYvWZ87d04URZGlS5fa/DJD1b6bG7PKl43nzJljdsyDgoIMbevHueptzpw54uvrK+np6dK7d28ZO3asycuvEydOFCcnJ8Pr6PpbYWGhLFy4UCIiIkSj0YiPj4+sWrXK5HiOHTtWwsLCDHP12rVrMm3aNElMTJTTp0/Lpk2bpEuXLtKyZUvZs2eP4bFXnif6x5aYmChvvPGG4bHrx0T/+5CQkGptV73Uv2LFCnFycpIFCxZIamqqfPDBB+Lg4CAbNmyw+hJJUlKSYdz0TL3fx9K4ff/99+Lm5iYApFGjRrJ3716jPpvbZp988olh3KKjoyUuLs4wbsXFxYZ6S/uJfpvpdDqJjIyUVatWSVFRkWGOjxgxQho3biw9evSQHj16yNatW+Xs2bNSWFhouI+HHnpIZsyYYZjrq1evtjpfli5dahjzoUOHyl133SUODg6yc+dOo35PmjRJXFxcpF+/fkb9NrXNNBqN/Oc//zHqu6Ojo3Tu3NnQb/2YV+77pEmTqvXbXN9PnDghU6ZMMczV/v37i6+vb7Ux16u6j5ub68HBwUb7eOW5rp/nr7zyisl1qs5zc/Nl0aJFhnFr1aqVDBs2zDDXK497bfbxiRMnioeHh8THx0t8fLxRrb7t22+/XZ544gmTbYtUPz6Y294ODg7y+eef1/m5TL+upVpTx5baYLgxQ38QCAgIqHbwMqXywcvcSf7y5ctGJ/JDhw5JQECAPPvss7JmzRqj+rCwMHF2dpa1a9darH3++efl4MGD1X4XFhYmWq1WfvzxR4v148aNs3oAMHXwmjFjhkRERMhrr71mWG5uJ8zKyjIcfIYNGyYjR440ecIQqflJWqPRGPVdvyOZEx4eLk899VS1YBUYGCi+vr7y+++/WxzzlStX1tlObOnAqacPR+bmatUTbWFhoeHjpU5OThISEiKPPfaYZGZmGs1Vc21/8MEHhnW6desmgwcPNjz2qm/6NXXgFDF9kjf1JKCmQdzSuOnDbOfOnSUiIkJCQ0NNfn9H1fcDWBq3yvRzfdu2bdVOVnr6/aSq2gYrS49bP+aKokiTJk3sOlHqt1njxo3FwcHBsM1s6fuAAQMM+2jHjh3Fy8vL5P5trr5du3aGMddqtdKyZctqY24umJnbZmfOnDHad821/eqrrxrNdRcXF5Pz3NR80TM112153Lbs49ZqFy5cKDqdTjQajcm2axPEa3suq/yE1VqtuTGvKeWPwfvLKC8vx4ULF9C4cWOLl8SmTp2Kb7/9Fl999ZXhEiIAuLu7Gy5bP/HEE/D398fLL79ssrZbt24AgE8++cRs7XfffYeJEyfiySefNKpfsmQJunbtiiVLlpht+4cffsDMmTONakUE06dPx5dffono6GisXr3a7GPs0KEDHn30Udx1112GZTExMSbX/fjjjzF69GgAwMCBAxEcHIxffvkFI0eONLzc4uHhYbJ23rx5WLNmDVJSUnDlyhW4urri//7v/zBr1iz4+PgY9V3/DZmJiYlo0aIFioqKMGrUKKSkpCA3Nxf+/v7o27cvZs2aBXd3d5w6dQoA8NZbb+Gnn34yXAaeNWsWQkND4e3tbXJ7RUZGGvXdVNvmxmzixIkYP368oW29f/zjH7h27RreeOMNREREID8/3/ASld69996LESNG4MEHHzSaV7VRm7mqN3HiROTm5uKrr74y247+sZuab2vXrsW6devMjptecXFxtXEbOXIkMjIysHTpUvTq1cvsuPXt2xdfffUVHBwcDMsrv0SSlJRk9DHlyjp37owHH3wQzz77rMX+2crcXDe1n+j3/6qqjrmluV55P/kr9hto2Hl+K2vIcxlQt2MuIrh27RoCAgKg0Vj+Jpu/XLg5f/48goKCGrobREREVAPnzp0z+Ub0yv5ybyhu3LgxgIrBcXd3r7P7/W7fObzy/W8oF0CjAHMGtcV9XW0LUbWp1cvMLcLZK4WI62z6mWttnpkVFxfjkUcewb59+3DlyhV4eXmhS5cuePKpp3G4uCk+3n4SIoACYHi35ri9hTc0CqBRFGg0gKIoFf9XAA0UKH/8TlGAnamX8PmuMxABzs0bbrI/9dH3adOm4Qx8DONek7b1Yx7S1BV+Hi6YOHEisnNyMOvtT3DwXA4OnsvFwfQcpGcXG+7vwsJJaNz5HjTuMrBaW65aDQpLTL9B05pATx2CvdwQ4KmDv4cLAptU/Bvg6QJfdx0cNMZXKav2HQAKS2/gzOUCnL1SiDOXC3HmSn7F/68UIL+kzGS75sbtlTfexd8fHw8HjWJ23Ba9+wp+WLOqTp8VFhQU4O2330bf/nG4Uu6KvUfPYt03XyJt3xb4jnodTk0t71caBfBwcYKHqxM8dE7wdHWCu4sTnDQarExOR9Vngv4eWlzOv47rZTXbblQ7ni6O8HHXwbuxFj6NtPBprIOPuzO8G+twJD0PC38+ZTiuPhQTghbejXAhtxgZuUXIyClGRl4RMnNL7N5+jhoFLs4auDg5wtXZATpnB7g6/e/fgpIyJJ66Uq2ulW8jKIqCgpIbKCi5gfzSMly/0XBzp5HWAQGeFceJQE8XBHjqcO5qEZYnnYP8MW7PxrXC3zoGQAGAP47jChRAwf+O5RW/ggIFaw6m45/rjtbqfGZOXl4egoKCDOdxS/5yV27y8vLg4eGB3NzcOgs3GblF6PH6FpRXGkkFwMCOfnBxcoQAKBcBpOLfip8rLrEVlpZhy+9ZRvenAOh/my8a6RyhKICDPhxoKiaWg0YfFip+Tr14DTtSL0P+qB0e1Rzx7fzgrnOqOFC7VBygdU4OMCUjtwinLxcgzNsN/n+c6MrKBVnXinEhpwjpOcVIzy7ChZyiP36uuF0rvlEn41dV+wB3hHq7Vex0Hjr4/7Hj+Xvo4OXmbPRyoqm+6/t/paAEF3NLcDGvGJl5xbj4xy0zrwTp2YU4ean6t03f5u+OZo21f4ybo2H8PF2c4e7yv/HckXoJb67/HeV/hLperbxRVFqOlPQcFF83PlgpCtDSpxG6BDdBl+AmCPJyweh/7zGaLw6Kgp9n9EWzRlrkFl3H1YJSXC0oRXZhKa4WXP/j31Icz7yGnScu2zWejhoFfh46BHq6oHkTV+QUlWLL0SzDibpFMzcUlJQhM6/Y7H0oCuDnoUNGTvV1FKDaSR8AXJwccJt/Y7QNcEe7AA+09XfHofRczF59GOUCnH3jbybbWrRoEcaNGwcA6NOnD0JDQ7F48WLD7ytv85lTJiInJwef//dr/HrmKn45loEFrzyNS6eOoKwoFw4u7nD2awmP2BHQ+req9pjeG94J4c3c4OniDE83JzRydoSmShDUW/5rGl5YcRhlInBQFLx2b3uMiApGebngSkEpLuQUISO3Yn/JyClCRm4x0nOKcPZKAbILr5u8z8Y6R7jrnNBY51jl/xX/urs4oVwEb60/ZjTGGgVYNC4K/p4ucHLQwFGjwNmx4l8nRw2cNBo4OSjIulaMnm9sNZprGgX4fFwUFEVB9h/z7GpBKa4WliK7oBRXCir+zS4sxZX8UpPbVn8/Lk4OcHF2gM7JAa7ODnBxqvi/i7MDCktuYO+Z7Gp197T3Q2ATl4o/cAlUnCT/eNKj4H9PfPKLb2DxL2eqB0pPHa7kl6K0DkOBRgH83HUIbFJxrHHXOeGL3Werjfn6Kb0Q6u0GZ0fLL4mYOifo9/HKxykAKLlRhoKSMuQX30B+yQ2cvVKAJ7/aD6myzRLu7Qgfdy2c/9jejg6aiv87KHByqNjeTg4aXCkowZAPdxmfjxRgVHQwcgqv43x2EdKzC3E5v7QWI2Y7c4+7Juw5fzPc1IFfTl7GqM/2WF+xgWkdNUYnaA8XJ2QXluJAWo5hJw7zdsP1snJk5hbjRnnNpkYr30Zw0zoaAly5CMrL/wh2fwS8chEUlpQhw8IJ1dxjCPjjGUbx9XLsP5ttCHVtA9zh6KBBVl4xsq6VoKyG/a8td50jOv8RZLqEeCIyyBPuOiejdcydKK0xddDUKMD7Izuj6HpZRfDMLqo4gP0RRu3Zjl5uzgjzdkO4txvCmrkh3LsRwpu5IdjLFTonB5P9HhwZiGMXr+HIhVz8diEPRy7k4ffMvGohzxQFwEO3h8DPU4fG2oqTeiNtxYm+0R8n+0baiv87OWiw/Nc0zFxxyBAqu4d54WpBKVKz8qvdd6CnC7qHeSEq1Avdw5og6Uw2Xlxp/5hXlpFbhDOXCxHq7WrzwdrcNtv5XF8ENnG16T5qOl9qW5ueXYg73qwejrZO64NgL1eL71u05wRvb99FBLlF13Exr8TwpCXr2v/+fyIr3+STl47NPdAuwAOBnvog44oATx383HVwdNDY1LatGmqb2VpfVFqG9JxCnMv+3zHjwLls7D511eZ2bLX0sdsR06Jpre+H4caCP/PKzRO9w9FY52R4JqKp/Azlj6sueUXXMW9TqtEzBEUB/t43Ao11TigzhANBuVRckRCRP5YD564WYm1KRrU+hXm7oay84gCQV3wdNdnKDhrF6NlMgKfOcDAI9Kx45hX33vYaH7zMHfRfHdoeRaVluJBTcfn4Qk4RLuQW49I109+ZY45GAbwbaeHnoYNPYx38PLTwc9fBx10HZwcNpn59oFrbCfd2hINGQW7R9YqxK7qOnMJSw8+5Rddx6Vop8oqrPxN/olc47u/WHOHejcw++6/6+O09UQL2Hfj0V+D0B6/Ek1ewPOlctfXmDmmHwZEB8HR1rpN+l5ULTl8uqAg8GXn47UIeDpzLqdXVPq2jgpIb5idyS59GiArzQvQfgSbAs3rfajrmtVXbkxVQu77XprYhT9I17XtdBKuatl1X9Q3RtqVx83PXGZ6cCoCKFyMqnrDq/5+RU4y7anFOsIbhxoL6CDdAwx0AbNmJy8sF10puIK/SCTu36DqS07KxYOfpavf5yuC2iGvnB5/G1d+rUZd9t7e+5EYZLuaWID2nCDtSL+GTbSerrTP1rlbo1aoZ/Nx18G7kXO3ZWF30va4OnLVR0wNfQ/b9Qk5htZdIFAD3d2uOcoHhsvy14uu4VnID+cU3cK34Boqum36/j96zd7XC6NtD4OVmPZg1pIYKVnWhIU/SNVUXweqvqCGvWFnDcGNBfYUb4NZ7ZnUrP7tp6L7fygfOhux7Tdq+UVaOgpIynLh0Df83P9HoKuSfHSrp1nIrB8qG1JBXrCxhuLGgPsNNQ+JJ+s/v+6184GzIvjfUSyREdGtjuLFAreGmNniSplsJtznRX5M95++/3PfcUHX+Hi637EniVu471Qy3ORFZY/nD+kRERES3GIYbIiIiUhWGGyIiIlIVhhsiIiJSFYYbIiIiUhWGGyIiIlIVhhsiIiJSFYYbIiIiUhWGGyIiIlIVhhsiIiJSFYYbIiIiUhWGGyIiIlIVhhsiIiJSFYYbIiIiUhWGGyIiIlIVhhsiIiJSFYYbIiIiUhWGGyIiIlIVhhsiIiJSFYYbIiIiUhWGGyIiIlIVhhsiIiJSFYYbIiIiUhWGGyIiIlIVhhsiIiJSFYYbIiIiUhWGGyIiIlIVhhsiIiJSFYYbIiIiUhWGGyIiIlIVhhsiIiJSFYYbIiIiUhWGGyIiIlIVhhsiIiJSFYYbIiIiUhWGGyIiIlIVhhsiIiJSFYYbIiIiUhWGGyIiIlIVhhsiIiJSFYYbIiIiUhWGGyIiIlIVhhsiIiJSFYYbIiIiUhWGGyIiIlIVhhsiIiJSFYYbIiIiUhWGGyIiIlIVhhsiIiJSFYYbIiIiUhWGGyIiIlIVhhsiIiJSFYYbIiIiUhWGGyIiIlIVhhsiIiJSFYYbIiIiUpUGDTcJCQmIiopC48aN4ePjg6FDh+LYsWNW67755hu0adMGOp0OHTp0wA8//PAn9JaIiIhuBQ0abrZv345JkyZh9+7d2LhxI65fv464uDgUFBSYrfnll1/wwAMP4JFHHkFycjKGDh2KoUOH4vDhw39iz4mIiOhmpYiINHQn9C5dugQfHx9s374dvXr1MrnOiBEjUFBQgLVr1xqW3X777ejUqRPmz59vtY28vDx4eHggNzcX7u7uddZ3IiIiqj/2nL9vqvfc5ObmAgC8vLzMrpOYmIj+/fsbLYuPj0diYqLJ9UtKSpCXl2d0IyIiIvW6acJNeXk5nn76afTo0QPt27c3u15mZiZ8fX2Nlvn6+iIzM9Pk+gkJCfDw8DDcgoKC6rTfREREdHO5acLNpEmTcPjwYSxbtqxO73fmzJnIzc013M6dO1en909EREQ3F8eG7gAATJ48GWvXrsWOHTvQvHlzi+v6+fnh4sWLRssuXrwIPz8/k+trtVpotdo66ysRERHd3Br0yo2IYPLkyVi5ciW2bNmCsLAwqzUxMTHYvHmz0bKNGzciJiamvrpJREREt5AGvXIzadIkfPXVV1i9ejUaN25seN+Mh4cHXFxcAABjxoxBYGAgEhISAABTpkxB79698c4772DgwIFYtmwZkpKSsGDBggZ7HERERHTzaNArN5988glyc3PRp08f+Pv7G27Lly83rJOWloaMjAzDz7Gxsfjqq6+wYMECREZG4ttvv8WqVassvgmZiIiI/jpuqu+5+TPwe26IiIhuPbfs99wQERER1RbDDREREakKww0RERGpCsMNERERqQrDDREREakKww0RERGpCsMNERERqQrDDREREakKww0RERGpCsMNERERqQrDDREREakKww0RERGpCsMNERERqQrDDREREakKww0RERGpCsMNERERqQrDDREREakKww0RERGpCsMNERERqQrDDREREakKww0RERGpCsMNERERqQrDDREREakKww0RERGpCsMNERERqQrDDREREakKww0RERGpCsMNERERqQrDDREREakKww0RERGpCsMNERERqQrDDREREakKww0RERGpCsMNERERqQrDDREREakKww0RERGpCsMNERERqQrDDREREakKww0RERGpCsMNERERqQrDDREREakKww0RERGpCsMNERERqQrDDREREakKww0RERGpCsMNERERqQrDDREREakKww0RERGpCsMNERERqQrDDREREakKww0RERGpCsMNERERqQrDDREREakKww0RERGpCsMNERERqQrDDREREakKww0RERGpCsMNERERqQrDDREREakKww0RERGpCsMNERERqQrDDREREakKww0RERGpCsMNERERqUqDhpsdO3Zg0KBBCAgIgKIoWLVqlcX1t23bBkVRqt0yMzP/nA4TERHRTa9Bw01BQQEiIyPx0Ucf2VV37NgxZGRkGG4+Pj711EMiIiK61TjaW1BUVAQRgaurKwDg7NmzWLlyJdq2bYu4uDi77uvuu+/G3XffbW8X4OPjA09PT7vriIiISP3svnIzZMgQLFmyBACQk5OD6OhovPPOOxgyZAg++eSTOu+gKZ06dYK/vz/uuusu7Nq1y+K6JSUlyMvLM7oRERGRetkdbvbv34877rgDAPDtt9/C19cXZ8+exZIlS/D+++/XeQcr8/f3x/z58/Hdd9/hu+++Q1BQEPr06YP9+/ebrUlISICHh4fhFhQUVK99JCIiooaliIjYU+Dq6orff/8dwcHBGD58ONq1a4c5c+bg3LlzaN26NQoLC2vWEUXBypUrMXToULvqevfujeDgYHzxxRcmf19SUoKSkhLDz3l5eQgKCkJubi7c3d1r1FciIiL6c+Xl5cHDw8Om87fdV24iIiKwatUqnDt3Dhs2bDC8zyYrK6tBwkL37t1x4sQJs7/XarVwd3c3uhEREZF62R1uZs+ejWnTpiE0NBTR0dGIiYkBAPz000/o3LlznXfQmgMHDsDf3/9Pb5eIiIhuTnZ/Wur//u//0LNnT2RkZCAyMtKwvF+/fhg2bJhd95Wfn2901eX06dM4cOAAvLy8EBwcjJkzZyI9Pd3wBuZ58+YhLCwM7dq1Q3FxMf79739jy5Yt+Omnn+x9GERERKRSdocbAPDz84Ofnx+AitfAtmzZgtatW6NNmzZ23U9SUhL69u1r+Hnq1KkAgLFjx2Lx4sXIyMhAWlqa4felpaV49tlnkZ6eDldXV3Ts2BGbNm0yug8iIiL6a7P7DcXDhw9Hr169MHnyZBQVFSEyMhJnzpyBiGDZsmW477776quvdcKeNyQRERHRzaFe31C8Y8cOw0fBV65cCRFBTk4O3n//ffzjH/+oWY+JiIiI6ojd4SY3NxdeXl4AgPXr1+O+++6Dq6srBg4ciNTU1DrvIBEREZE97A43QUFBSExMREFBAdavX2/4KHh2djZ0Ol2dd5CIiIjIHna/ofjpp5/G6NGj0ahRI4SEhKBPnz4AKl6u6tChQ133j4iIiMgudoebJ598Et27d8e5c+dw1113QaOpuPgTHh7O99wQERFRg7P701KV6UsVRamzDtU3flqKiIjo1lOvn5YCgCVLlqBDhw5wcXGBi4sLOnbsaPZvOxERERH9mex+Werdd9/FSy+9hMmTJ6NHjx4AgJ9//hkTJkzA5cuX8cwzz9R5J4mIiIhsZffLUmFhYXjllVcwZswYo+X/+c9/8PLLL+P06dN12sG6xpeliIiIbj31+rJURkYGYmNjqy2PjY1FRkaGvXdHREREVKfsDjcRERH4+uuvqy1fvnw5WrZsWSedIiIiIqopu99z88orr2DEiBHYsWOH4T03u3btwubNm02GHiIiIqI/k91Xbu677z7s2bMH3t7eWLVqFVatWgVvb2/s3bsXw4YNq48+EhEREdmsVt9zcyviG4qJiIhuPfacv216WSovL8/mxhkYiIiIqCHZFG48PT2tfguxiEBRFJSVldVJx4iIiIhqwqZws3Xr1vruBxEREVGdsCnc9O7du777QURERFQnavS3pYiIiIhuVgw3REREpCoMN0RERKQqDDdERESkKjUKNzdu3MCmTZvw6aef4tq1awCACxcuID8/v047R0RERGQvu/+21NmzZzFgwACkpaWhpKQEd911Fxo3bow33ngDJSUlmD9/fn30k4iIiMgmdl+5mTJlCrp164bs7Gy4uLgYlg8bNgybN2+u084RERER2cvuKzc7d+7EL7/8AmdnZ6PloaGhSE9Pr7OOEREREdWE3VduysvLTf6JhfPnz6Nx48Z10ikiIiKimrI73MTFxWHevHmGnxVFQX5+PubMmYN77rmnLvtGREREZDdFRMSegvPnzyM+Ph4igtTUVHTr1g2pqanw9vbGjh074OPjU199rRP2/Ml0IiIiujnYc/62O9wAFR8FX7ZsGVJSUpCfn48uXbpg9OjRRm8wvlkx3BAREd167Dl/2/2GYgBwdHTEgw8+WKPOEREREdUnu8PNmjVrTC5XFAU6nQ4REREICwurdceIiIiIasLucDN06FAoioKqr2bplymKgp49e2LVqlVo0qRJnXWUiIiIyBZ2f1pq48aNiIqKwsaNG5Gbm4vc3Fxs3LgR0dHRWLt2LXbs2IErV65g2rRp9dFfIiIiIovsvnIzZcoULFiwALGxsYZl/fr1g06nw+OPP44jR45g3rx5GD9+fJ12lIiIiMgWdl+5OXnypMl3Kbu7u+PUqVMAgJYtW+Ly5cu17x0RERGRnewON127dsX06dNx6dIlw7JLly7hueeeQ1RUFAAgNTUVQUFBdddLIiIiIhvZ/bLUwoULMWTIEDRv3twQYM6dO4fw8HCsXr0aAJCfn49Zs2bVbU+JiIiIbFCjL/ErLy/HTz/9hOPHjwMAWrdujbvuugsajd0Xgv50/BI/IiKiW0+9f0PxrYzhhoiI6NZT799QXFBQgO3btyMtLQ2lpaVGv3vqqadqcpdEREREdcLucJOcnIx77rkHhYWFKCgogJeXFy5fvgxXV1f4+Pgw3BAREVGDsvtNMs888wwGDRqE7OxsuLi4YPfu3Th79iy6du2Kt99+uz76SERERGQzu8PNgQMH8Oyzz0Kj0cDBwQElJSUICgrCm2++iRdeeKE++khERERkM7vDjZOTk+FTUT4+PkhLSwMAeHh44Ny5c3XbOyIiIiI72f2em86dO+PXX39Fy5Yt0bt3b8yePRuXL1/GF198gfbt29dHH4mIiIhsZveVm9deew3+/v4AgH/+859o0qQJJk6ciEuXLmHBggV13kEiIiIie9h15UZE4OPjY7hC4+Pjg/Xr19dLx4iIiIhqwq4rNyKCiIgIvreGiIiIblp2hRuNRoOWLVviypUr9dUfIiIiolqx+z03r7/+OqZPn47Dhw/XR3+IiIiIasXuvy3VpEkTFBYW4saNG3B2doaLi4vR769evVqnHaxr/NtSREREt556/dtS8+bNq2m/iIiIiOqd3eFm7Nix9dEPIiIiojph93tuAODkyZOYNWsWHnjgAWRlZQEAfvzxRxw5cqROO0dERERkL7vDzfbt29GhQwfs2bMHK1asQH5+PgDg4MGDmDNnTp13kIiIiMgedoebGTNm4B//+Ac2btwIZ2dnw/I777wTu3fvrtPOEREREdnL7nBz6NAhDBs2rNpyHx8fXL58uU46RURERFRTdocbT09PZGRkVFuenJyMwMDAOukUERERUU3ZHW5GjhyJ559/HpmZmVAUBeXl5di1axemTZuGMWPG1EcfiYiIiGxWo78K3qZNGwQFBSE/Px9t27ZFr169EBsbi1mzZtVHH4mIiIhsZvc3FOulpaXh8OHDyM/PR+fOndGyZcu67lu94DcUExER3Xrq9RuKf/75Z/Ts2RPBwcEIDg6ucSeJiIiI6oPdL0vdeeedCAsLwwsvvIDffvutPvpEREREVGN2h5sLFy7g2Wefxfbt29G+fXt06tQJb731Fs6fP2934zt27MCgQYMQEBAARVGwatUqqzXbtm1Dly5doNVqERERgcWLF9vdLhEREamX3eHG29sbkydPxq5du3Dy5Encf//9+M9//oPQ0FDceeeddt1XQUEBIiMj8dFHH9m0/unTpzFw4ED07dsXBw4cwNNPP41HH30UGzZssPdhEBERkUrV+A3FemVlZfjxxx/x0ksvISUlBWVlZTXriKJg5cqVGDp0qNl1nn/+eaxbtw6HDx82LBs5ciRycnKwfv16m9rhG4qJiIhuPfacv2v0hzMBYNeuXXjyySfh7++PUaNGoX379li3bl1N784miYmJ6N+/v9Gy+Ph4JCYmmq0pKSlBXl6e0Y2IiIjUy+5wM3PmTISFheHOO+9EWloa/vWvfyEzMxNffPEFBgwYUB99NMjMzISvr6/RMl9fX+Tl5aGoqMhkTUJCAjw8PAy3oKCgeu0jERERNSy7Pwq+Y8cOTJ8+HcOHD4e3t3d99KlOzZw5E1OnTjX8nJeXx4BDRESkYnaHm127dtVHP2zi5+eHixcvGi27ePEi3N3d4eLiYrJGq9VCq9X+Gd0jIiKim4Dd4Ubvt99+Q1paGkpLS42WDx48uNadMicmJgY//PCD0bKNGzciJiam3tokIiKiW4vd4ebUqVMYNmwYDh06BEVRoP+wlaIoAGDXp6Xy8/Nx4sQJw8+nT5/GgQMH4OXlheDgYMycORPp6elYsmQJAGDChAn48MMP8dxzz2H8+PHYsmULvv7663p/IzMRERHdOux+Q/GUKVMQFhaGrKwsuLq64siRI9ixYwe6deuGbdu22XVfSUlJ6Ny5Mzp37gwAmDp1Kjp37ozZs2cDADIyMpCWlmZYPywsDOvWrcPGjRsRGRmJd955B//+978RHx9v78MgIiIilbL7e268vb2xZcsWdOzYER4eHti7dy9at26NLVu24Nlnn0VycnJ99bVO8HtuiIiIbj31+j03ZWVlaNy4MYCKoHPhwgUAQEhICI4dO1aD7hIRERHVHbvfc9O+fXscPHgQYWFhiI6OxptvvglnZ2csWLAA4eHh9dFHIiIiIpvZHW5mzZqFgoICAMDcuXPxt7/9DXfccQeaNm2K5cuX13kHiYiIiOxR678tBQBXr15FkyZNDJ+YupnxPTdERES3HnvO3zX+npvKvLy86uJuiIiIiGqtxn84k4iIiOhmxHBDREREqsJwQ0RERKrCcENERESqwnBDREREqsJwQ0RERKrCcENERESqwnBDREREqsJwQ0RERKrCcENERESqwnBDREREqsJwQ0RERKrCcENERESqwnBDREREqsJwQ0RERKrCcENERESqwnBDREREqsJwQ0RERKrCcENERESqwnBDREREqsJwQ0RERKrCcENERESqwnBDREREqsJwQ0RERKrCcENERESqwnBDREREqsJwQ0RERKrCcENERESqwnBDREREqsJwQ0RERKrCcENERESqwnBDREREqsJwQ0RERKrCcENERESqwnBDREREqsJwQ0RERKrCcENERESqwnBDREREqsJwQ0RERKrCcENERESqwnBDREREqsJwQ0RERKrCcENERESqwnBDREREqsJwQ0RERKrCcENERESqwnBDREREqsJwQ0RERKrCcENERESqwnBDREREqsJwQ0RERKrCcENERESqwnBDREREqsJwQ0RERKrCcENERESqwnBDREREqsJwQ0RERKrCcENERESqwnBDREREqnJThJuPPvoIoaGh0Ol0iI6Oxt69e82uu3jxYiiKYnTT6XR/Ym+JiIjoZtbg4Wb58uWYOnUq5syZg/379yMyMhLx8fHIysoyW+Pu7o6MjAzD7ezZs39ij4mIiOhm1uDh5t1338Vjjz2Ghx9+GG3btsX8+fPh6uqKzz//3GyNoijw8/Mz3Hx9ff/EHhMREdHNrEHDTWlpKfbt24f+/fsblmk0GvTv3x+JiYlm6/Lz8xESEoKgoCAMGTIER44cMbtuSUkJ8vLyjG5ERESkXg0abi5fvoyysrJqV158fX2RmZlpsqZ169b4/PPPsXr1anz55ZcoLy9HbGwszp8/b3L9hIQEeHh4GG5BQUF1/jiIiIjo5tHgL0vZKyYmBmPGjEGnTp3Qu3dvrFixAs2aNcOnn35qcv2ZM2ciNzfXcDt37tyf3GMiIiL6Mzk2ZOPe3t5wcHDAxYsXjZZfvHgRfn5+Nt2Hk5MTOnfujBMnTpj8vVarhVarrXVfiYiI6NbQoFdunJ2d0bVrV2zevNmwrLy8HJs3b0ZMTIxN91FWVoZDhw7B39+/vrpJREREt5AGvXIDAFOnTsXYsWPRrVs3dO/eHfPmzUNBQQEefvhhAMCYMWMQGBiIhIQEAMDcuXNx++23IyIiAjk5OXjrrbdw9uxZPProow35MIiIiOgm0eDhZsSIEbh06RJmz56NzMxMdOrUCevXrze8yTgtLQ0azf8uMGVnZ+Oxxx5DZmYmmjRpgq5du+KXX35B27ZtG+ohEBER0U1EERFp6E78mfLy8uDh4YHc3Fy4u7s3dHeIiIjIBvacv2+5T0sRERERWcJwQ0RERKrCcENERESqwnBDREREqsJwQ0RERKrCcENERESqwnBDREREqsJwQ0RERKrCcENERESqwnBDREREqsJwQ0RERKrCcENERESqwnBDREREqsJwQ0RERKrCcENERESqwnBDREREqsJwQ0RERKrCcENERESqwnBDREREqsJwQ0RERKrCcENERESqwnBDREREqsJwQ0RERKrCcENERESqwnBDREREqsJwQ0RERKrCcENERESqwnBDREREqsJwQ0RERKrCcENERESqwnBDREREqsJwQ0RERKrCcENERESqwnBDREREqsJwQ0RERKrCcENERESqwnBDREREqsJwQ0RERKrCcENERESqwnBDREREqsJwQ0RERKrCcENERESqwnBDREREqsJwQ0RERKrCcENERESqwnBDREREqsJwQ0RERKrCcENERESqwnBDREREqsJwQ0RERKrCcENERESqwnBDREREqsJwQ0RERKrCcENERESqwnBDREREqsJwQ0RERKrCcENERESqwnBDREREqsJwQ0RERKrCcENERESqwnBDREREqsJwQ0RERKrCcENERESqwnBDREREqnJThJuPPvoIoaGh0Ol0iI6Oxt69ey2u/80336BNmzbQ6XTo0KEDfvjhhz+pp0RERHSza/Bws3z5ckydOhVz5szB/v37ERkZifj4eGRlZZlc/5dffsEDDzyARx55BMnJyRg6dCiGDh2Kw4cP/8k9JyIiopuRIiLSkB2Ijo5GVFQUPvzwQwBAeXk5goKC8Pe//x0zZsyotv6IESNQUFCAtWvXGpbdfvvt6NSpE+bPn2+1vby8PHh4eCA3Nxfu7u5190CIiIio3thz/nb8k/pkUmlpKfbt24eZM2calmk0GvTv3x+JiYkmaxITEzF16lSjZfHx8Vi1apXJ9UtKSlBSUmL4OTc3F0DFIBEREdGtQX/etuWaTIOGm8uXL6OsrAy+vr5Gy319ffH777+brMnMzDS5fmZmpsn1ExIS8Morr1RbHhQUVMNeExERUUO5du0aPDw8LK7ToOHmzzBz5kyjKz3l5eW4evUqmjZtCkVR6rStvLw8BAUF4dy5c3a/5FWb2lu57drWs+2/Vtu1rWfbbPtWqf+rtm2JiODatWsICAiwum6Dhhtvb284ODjg4sWLRssvXrwIPz8/kzV+fn52ra/VaqHVao2WeXp61rzTNnB3d6/xBq1N7a3cdm3r2fZfq+3a1rNttn2r1P9V2zbH2hUbvQb9tJSzszO6du2KzZs3G5aVl5dj8+bNiImJMVkTExNjtD4AbNy40ez6RERE9NfS4C9LTZ06FWPHjkW3bt3QvXt3zJs3DwUFBXj44YcBAGPGjEFgYCASEhIAAFOmTEHv3r3xzjvvYODAgVi2bBmSkpKwYMGChnwYREREdJNo8HAzYsQIXLp0CbNnz0ZmZiY6deqE9evXG940nJaWBo3mfxeYYmNj8dVXX2HWrFl44YUX0LJlS6xatQrt27dvqIdgoNVqMWfOnGovg9V37a3cdm3r2fZfq+3a1rNttn2r1P9V264rDf49N0RERER1qcG/oZiIiIioLjHcEBERkaow3BAREZGqMNwQERGRqjDc1JGPPvoIoaGh0Ol0iI6Oxt69e22q27FjBwYNGoSAgAAoimL2b2SZk5CQgKioKDRu3Bg+Pj4YOnQojh07ZlPtJ598go4dOxq+aCkmJgY//vijXe3rvf7661AUBU8//bRN67/88stQFMXo1qZNG7vaTE9Px4MPPoimTZvCxcUFHTp0QFJSktW60NDQam0rioJJkybZ1G5ZWRleeuklhIWFwcXFBS1atMCrr75q0987ASq+Ovzpp59GSEgIXFxcEBsbi19//dXkutbmh4hg9uzZ8Pf3h4uLC/r374/U1FSb61esWIG4uDjDN3YfOHDAptrr16/j+eefR4cOHeDm5oaAgACMGTMGFy5csLntl19+GW3atIGbmxuaNGmC/v37Y8+ePTbVVjZhwgQoioJ58+bZ3Pa4ceOqbf8BAwbY3PbRo0cxePBgeHh4wM3NDVFRUUhLS7Op3tTcUxQFb731ltXa/Px8TJ48Gc2bN4eLiwvatm1r9AeDrdVfvHgR48aNQ0BAAFxdXTFgwADDfLHlWFJcXIxJkyahadOmaNSoEe677z7Dl6raUr9gwQL06dMH7u7uUBQFOTk5NtVevXoVf//739G6dWu4uLggODgYTz31lOFvBdrS9hNPPIEWLVrAxcUFzZo1w5AhQ/D777/bdQwVEdx9991GY2tLfZ8+fapt7wkTJtjcdmJiIu688064ubnB3d0dvXr1QlFRkdX6M2fOmJ1vo0aNstp2ZmYmHnroIfj5+cHNzQ1dunTBd999Z/PjPnnyJIYNG4ZmzZrB3d0dw4cPr/YlvPWF4aYOLF++HFOnTsWcOXOwf/9+REZGIj4+HllZWVZrCwoKEBkZiY8++qhGbW/fvh2TJk3C7t27sXHjRly/fh1xcXEoKCiwWtu8eXO8/vrr2LdvH5KSknDnnXdiyJAhOHLkiF19+PXXX/Hpp5+iY8eOdtW1a9cOGRkZhtvPP/9sc212djZ69OgBJycn/Pjjj/jtt9/wzjvvoEmTJjb1t3K7GzduBADcf//9NrX9xhtv4JNPPsGHH36Io0eP4o033sCbb76JDz74wKb6Rx99FBs3bsQXX3yBQ4cOIS4uDv3790d6enq1da3NjzfffBPvv/8+5s+fjz179sDNzQ3x8fEoLi62qb6goAA9e/bEG2+8YVfbhYWF2L9/P1566SXs378fK1aswLFjxzB48GCb+96qVSt8+OGHOHToEH7++WeEhoYiLi4Oly5dsnm/WLlyJXbv3l3t69htqR8wYIDRPFi6dKlNtSdPnkTPnj3Rpk0bbNu2DSkpKXjppZeg0+lsqq/cZkZGBj7//HMoioL77rvPau3UqVOxfv16fPnllzh69CiefvppTJ48GWvWrLHatohg6NChOHXqFFavXo3k5GSEhISgf//+KCgosOlY8swzz+D777/HN998g+3bt+PChQu49957Adh2LCosLMSAAQPwwgsvGPXNWu2FCxdw4cIFvP322zh8+DAWL16M9evX45FHHrG57a5du2LRokU4evQoNmzYABFBXFwctm3bZvMxdN68edX+bI+tx+DHHnvMaLu/+eabNtUmJiZiwIABiIuLw969e/Hrr79i8uTJ0Gg0VuuDgoKqzbdXXnkFjRo1wqVLl6y2PWbMGBw7dgxr1qzBoUOHcO+992L48OFITk622nZBQQHi4uKgKAq2bNmCXbt2obS0FIMGDUJ5eXm1sa1zQrXWvXt3mTRpkuHnsrIyCQgIkISEBLvuB4CsXLmyVn3JysoSALJ9+/Ya1Tdp0kT+/e9/27z+tWvXpGXLlrJx40bp3bu3TJkyxaa6OXPmSGRkZI36KCLy/PPPS8+ePWtcX9mUKVOkRYsWUl5ebtP6AwcOlPHjxxstu/fee2X06NFWawsLC8XBwUHWrl1rtLxLly7y4osvWqytOj/Ky8vFz89P3nrrLcOynJwc0Wq1snTpUqv1lZ0+fVoASHJysk1tm7J3714BIGfPnq1RfW5urgCQTZs22VR7/vx5CQwMlMOHD0tISIi89957Nvd97NixMmTIEIv9MVc7YsQIefDBB63WWup7ZUOGDJE777zTptp27drJ3LlzjZaZmztV648dOyYA5PDhw4ZlZWVl0qxZM/nss8+q1Vc9luTk5IiTk5N88803hnWOHj0qACQxMdFqfWVbt24VAJKdnV3td9Zq9b7++mtxdnaW69ev16j+4MGDAkBOnDhhU21ycrIEBgZKRkaGxe1qqt7WY6Op2ujoaJk1a5bVWkt9r6xTp07Vjl/mat3c3GTJkiVG63l5edk0XzZs2CAajUZyc3MN6+Tk5IiiKLJx40abHk9t8MpNLZWWlmLfvn3o37+/YZlGo0H//v2RmJj4p/dHf5nWy8vLrrqysjIsW7YMBQUFdv0pi0mTJmHgwIFGj99WqampCAgIQHh4OEaPHm24rG+LNWvWoFu3brj//vvh4+ODzp0747PPPrO7D6Wlpfjyyy8xfvx4m/+QamxsLDZv3ozjx48DAA4ePIiff/4Zd999t9XaGzduoKyszPAsX8/FxcWuK1cAcPr0aWRmZhqNvYeHB6Kjoxts7imKUqO/3VZaWooFCxbAw8MDkZGRVtcvLy/HQw89hOnTp6Ndu3Y16C2wbds2+Pj4oHXr1pg4cSKuXLliU7vr1q1Dq1atEB8fDx8fH0RHR9v9crLexYsXsW7dOsMVCGtiY2OxZs0apKenQ0SwdetWHD9+HHFxcVZrS0pKAMBo7mk0Gmi1WpNzr+qxZN++fbh+/brRfGvTpg2Cg4NNzreaHotsrc3NzYW7uzscHat/F621+oKCAixatAhhYWEICgqyWltYWIhRo0bho48+Mvt3DK21/d///hfe3t5o3749Zs6cicLCQqu1WVlZ2LNnD3x8fBAbGwtfX1/07t3b7LHC2uPet28fDhw4YHK+maqNjY3F8uXLcfXqVZSXl2PZsmUoLi5Gnz59rNaXlJRAURSjL/LT6XTQaDR2H+tqpN7jk8qlp6cLAPnll1+Mlk+fPl26d+9u132hllduysrKZODAgdKjRw+ba1JSUsTNzU0cHBzEw8ND1q1bZ3Pt0qVLpX379lJUVCQitj87ERH54Ycf5Ouvv5aDBw/K+vXrJSYmRoKDgyUvL8+meq1WK1qtVmbOnCn79++XTz/9VHQ6nSxevNjm/ouILF++XBwcHCQ9Pd3mmrKyMnn++edFURRxdHQURVHktddes7k+JiZGevfuLenp6XLjxg354osvRKPRSKtWrSzWVZ0fu3btEgBy4cIFo/Xuv/9+GT58uNX6ymp75aaoqEi6dOkio0aNsqv++++/Fzc3N1EURQICAmTv3r021b722mty1113Ga622XvlZunSpbJ69WpJSUmRlStXym233SZRUVFy48YNi7X6Z+2urq7y7rvvSnJysiQkJIiiKLJt2zabH7feG2+8IU2aNDHsQ9Zqi4uLZcyYMQJAHB0dxdnZWf7zn//Y9LhLS0slODhY7r//frl69aqUlJTI66+/LgAkLi7OqNbUseS///2vODs7V2snKipKnnvuOav1lVm6cmPLcezSpUsSHBwsL7zwgl31H330kbi5uQkAad26dbWrNuZqH3/8cXnkkUcMP5vbrubqP/30U1m/fr2kpKTIl19+KYGBgTJs2DCrtYmJiQJAvLy85PPPP5f9+/fL008/Lc7OznL8+HGbH7fexIkT5bbbbrO539nZ2RIXF2eYb+7u7rJhwwab6rOyssTd3V2mTJkiBQUFkp+fL5MnTxYA8vjjj5vtY11huKmlmyncTJgwQUJCQuTcuXM215SUlEhqaqokJSXJjBkzxNvbW44cOWK1Li0tTXx8fOTgwYOGZfaEm6qys7PF3d3d5pfEnJycJCYmxmjZ3//+d7n99tvtajcuLk7+9re/2VWzdOlSad68uSxdulRSUlJkyZIl4uXlZXOwOnHihPTq1UsAiIODg0RFRcno0aOlTZs2Futu1nBTWloqgwYNks6dOxtdgralPj8/X1JTUyUxMVHGjx8voaGhcvHiRYu1SUlJ4uvraxRI7Q03VZ08edKml8T0+/sDDzxgtN6gQYNk5MiRdrfdunVrmTx5ss39fuutt6RVq1ayZs0aOXjwoHzwwQfSqFEjk5f5TdUnJSVJZGSkYe7Fx8fL3XffLQMGDDBaz9SxxJ5wY+1YZCncWKvNzc2V7t27y4ABA6S0tNSu+pycHDl+/Lhs375dBg0aJF26dDEKlqZqV69eLREREXLt2jXDMnPb1dZj8ObNm6u9JGaqVr+Pz5w506i+Q4cOMmPGDLvaLiwsFA8PD3n77bdt7vfkyZOle/fusmnTJjlw4IC8/PLL4uHhISkpKTbVb9iwQcLDw0VRFHFwcJAHH3xQunTpIhMmTLAwOnWD4aaWSkpKxMHBodpEHzNmjAwePNiu+6pNuJk0aZI0b95cTp06VaN6vX79+tmUqleuXGk4QOpvAAyTuOozYFt069at2g5rTnBwsNEzKRGRjz/+WAICAmxu78yZM6LRaGTVqlV29bN58+by4YcfGi179dVXpXXr1nbdT35+viGYDB8+XO655x6L61edH/oTctVA0qtXL3nqqaes1ldW03BTWloqQ4cOlY4dO8rly5dt7rs5ERER1a6CVa197733DPOs8tzTaDQSEhJS47a9vb1l/vz5FmtLSkrE0dFRXn31VaP1nnvuOYmNjbWr7R07dggAOXDggMnfV60tLCwUJyenau/XeuSRRyQ+Pt6utnNyciQrK0tEKt4z+OSTTxp+Z+5Yoj8hVw0kwcHB8u6771qtr8xcuLFWm5eXJzExMdKvXz+TV7vsOQ6WlJSIq6urfPXVVxZrp0yZYna+9e7du0Zt5+fnCwBZv369xdpTp04JAPniiy+Mlg8fPtzoKqktbS9ZskScnJwM291a7YkTJ6q9R0uk4hzxxBNP2NX2pUuXDNva19dX3nzzTbPr1hW+56aWnJ2d0bVrV2zevNmwrLy8HJs3b7brvSs1JSKYPHkyVq5ciS1btiAsLKxW91deXm54bd6Sfv364dChQzhw4IDh1q1bN4wePRoHDhyAg4ODXe3m5+fj5MmT8Pf3t2n9Hj16VPvY4fHjxxESEmJzm4sWLYKPjw8GDhxoV18LCwuN/pgrADg4ONj9CQA3Nzf4+/sjOzsbGzZswJAhQ+yqDwsLg5+fn9Hcy8vLw549e/6UuXf9+nUMHz4cqamp2LRpE5o2bVrr+7Rl/j300ENISUkxmnsBAQGYPn06NmzYUKN2z58/jytXrlidf87OzoiKiqr13AOAhQsXomvXrja9xwioGO/r16/Xydzz8PBAs2bNkJqaiqSkJAwZMsTqsaRr165wcnIymm/Hjh1DWloaYmJianUssqU2Ly8PcXFxcHZ2xpo1a4zeO1STtqXiyT2Ki4st1s6YMaPafAOA9957D4sWLapR2/r78PPzs1gbGhqKgIAAs/PNnrYXLlyIwYMHo1mzZobHb6lW/54gc/PNnra9vb3h6emJLVu2ICsry+hTlfWm3uPTX8CyZctEq9XK4sWL5bfffpPHH39cPD09JTMz02rttWvXJDk5WZKTkwWA4XV8U584MWXixIni4eEh27Ztk4yMDMOtsLDQau2MGTNk+/btcvr0aUlJSZEZM2aIoijy008/2dR2Vfa8LPXss8/Ktm3b5PTp07Jr1y7p37+/eHt7V3tWYc7evXvF0dFR/vnPf0pqaqr897//FVdXV/nyyy9tqi8rK5Pg4GB5/vnnbVq/srFjx0pgYKCsXbtWTp8+LStWrBBvb+9ql+bNWb9+vfz4449y6tQp+emnnyQyMlKio6NNXmK3Nj9ef/118fT0NLx/ZMiQIRIWFmZ4Vmut/sqVK5KcnCzr1q0TALJs2TJJTk6WjIwMi7WlpaUyePBgad68uRw4cMBo7pWUlFhtOz8/X2bOnCmJiYly5swZSUpKkocffli0Wq0cPnzY7v2i6stSluqvXbsm06ZNk8TERDl9+rRs2rRJunTpIi1btpTi4mKrba9YsUKcnJxkwYIFkpqaKh988IE4ODjIzp07bRpzkYqXVlxdXeWTTz6xa3v37t1b2rVrJ1u3bpVTp07JokWLRKfTyccff2xT/ddffy1bt26VkydPyqpVqyQkJETuvfdeEbHtWDJhwgQJDg6WLVu2SFJSksTExBheHralPiMjQ5KTk+Wzzz4TALJjxw5JTk6Whx9+2GJtbm6uREdHS4cOHeTEiRNG69y4ccNq2ydPnpTXXntNkpKS5OzZs7Jr1y4ZNGiQeHl5ybhx4+w+hqLSVTFrbZ84cULmzp0rSUlJcvr0aVm9erWEh4dLr169bBqz9957T9zd3eWbb76R1NRUmTVrluh0Ojlx4oTNx//U1FRRFEV+/PFHwzJrtaWlpRIRESF33HGH7NmzR06cOCFvv/22KIoi69ats6ntzz//XBITE+XEiRPyxRdfiJeXl0ydOtXsuNYlhps68sEHH0hwcLA4OztL9+7dZffu3TbV6S/PVr2NHTvWpnpTtQBk0aJFVmvHjx8vISEh4uzsLM2aNZN+/frVONiI2BduRowYIf7+/uLs7CyBgYEyYsSIam/us+b777+X9u3bi1arlTZt2siCBQtsrt2wYYMAkGPHjtnVpkjFpfEpU6ZIcHCw6HQ6CQ8PlxdffNFwUrdm+fLlEh4eLs7OzuLn5yeTJk2SnJwck+tamx/l5eXy0ksvia+vr2i1WunXr5/RY7JWv2jRIpO/nzNnjsVa/ctYpm5bt2612nZRUZEMGzZMAgICxNnZWfz9/WXw4MGGNxTbu19UDTeW6gsLCyUuLk6aNWsmTk5OEhISIo899pjhyYgtbS9cuFAiIiJEp9NJZGSk0UubttR/+umn4uLiUm27W6vNyMiQcePGSUBAgOh0OmndurW88847hjdWW6v/17/+Jc2bNxcnJycJDg6WWbNmGeatLceSoqIiefLJJ6VJkybi6uoqw4YNk4yMDJvr58yZY3Y9S7XmHhcAi3NRX5+eni533323+Pj4iJOTkzRv3lxGjRolv//+e42OoZXDjbX6tLQ06dWrl3h5eYlWq5WIiAiZPn264asPbGk7ISFBmjdvLq6urhITE2MI0rbWz5w5U4KCgqSsrMzoMVirPX78uNx7773i4+Mjrq6u0rFjR8NHw22pf/7558XX11ecnJykZcuWRnO1vil/dJKIiIhIFfieGyIiIlIVhhsiIiJSFYYbIiIiUhWGGyIiIlIVhhsiIiJSFYYbIiIiUhWGGyIiIlIVhhsiIiJSFYYbIvrL27ZtGxRFQU5OTkN3hYjqAMMNERERqQrDDREREakKww0RNbjy8nIkJCQgLCwMLi4uiIyMxLfffgvgfy8ZrVu3Dh07doROp8Ptt9+Ow4cPG93Hd999h3bt2kGr1SI0NBTvvPOO0e9LSkrw/PPPIygoCFqtFhEREVi4cKHROvv27UO3bt3g6uqK2NhYHDt2rH4fOBHVC4YbImpwCQkJWLJkCebPn48jR47gmWeewYMPPojt27cb1pk+fTreeecd/Prrr2jWrBkGDRqE69evA6gIJcOHD8fIkSNx6NAhvPzyy3jppZewePFiQ/2YMWOwdOlSvP/++zh69Cg+/fRTNGrUyKgfL774It555x0kJSXB0dER48eP/1MePxHVLf5VcCJqUCUlJfDy8sKmTZsQExNjWP7oo4+isLAQjz/+OPr27Ytly5ZhxIgRAICrV6+iefPmWLx4MYYPH47Ro0fj0qVL+Omnnwz1zz33HNatW4cjR47g+PHjaN26NTZu3Ij+/ftX68O2bdvQt29fbNq0Cf369QMA/PDDDxg4cCCKioqg0+nqeRSIqC7xyg0RNagTJ06gsLAQd911Fxo1amS4LVmyBCdPnjSsVzn4eHl5oXXr1jh69CgA4OjRo+jRo4fR/fbo0QOpqakoKyvDgQMH4ODggN69e1vsS8eOHQ3/9/f3BwBkZWXV+jES0Z/LsaE7QER/bfn5+QCAdevWITAw0Oh3Wq3WKODUlIuLi03rOTk5Gf6vKAqAivcDEdGthVduiKhBtW3bFlqtFmlpaYiIiDC6BQUFGdbbvXu34f/Z2dk4fvw4brvtNgDAbbfdhl27dhnd765du9CqVSs4ODigQ4cOKC8vN3oPDxGpF6/cEFGDaty4MaZNm4ZnnnkG5eXl6NmzJ3Jzc7Fr1y64u7sjJCQEADB37lw0bdoUvr6+ePHFF+Ht7Y2hQ4cCAJ599llERUXh1VdfxYgRI5CYmIgPP/wQH3/8MQAgNDQUY8eOxfjx4/H+++8jMjISZ8+eRVZWFoYPH95QD52I6gnDDRE1uFdffRXNmjVDQkICTp06BU9PT3Tp0gUvvPCC4WWh119/HVOmTEFqaio6deqE77//Hs7OzgCALl264Ouvv8bs2bPx6quvwt/fH3PnzsW4ceMMbXzyySd44YUX8OSTT+LKlSsIDg7GCy+80BAPl4jqGT8tRUQ3Nf0nmbKzs+Hp6dnQ3SGiWwDfc0NERESqwnBDREREqsKXpYiIiEhVeOWGiIiIVIXhhoiIiFSF4YaIiIhUheGGiIiIVIXhhoiIiFSF4YaIiIhUheGGiIiIVIXhhoiIiFTl/wE2T0DXkMeh5gAAAABJRU5ErkJggg==", "text/plain": [ "
" ] @@ -347,7 +392,7 @@ "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAG2CAYAAACZEEfAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABmHUlEQVR4nO3deVxUVeMG8GeGZdhBdpBF3MBdcQX34EXN3FMzU1PTLCyXLKXSst5EzdLXNLdcW9xKzTT3NRUVFBRNERCFkEVEhn1x5vz+IOfnyDaDIDA+389nPjX33rMMDjMP9557jkQIIUBERESko6Q13QEiIiKi6sSwQ0RERDqNYYeIiIh0GsMOERER6TSGHSIiItJpDDtERESk0xh2iIiISKcx7BAREZFOY9ghIiIincawQ0RERDqtRsPO6dOnMWDAADg7O0MikWDPnj1q+4UQmDdvHpycnGBsbAx/f39ER0erHZOeno7Ro0fDwsICVlZWmDhxIrKzs5/jqyAiIqLarEbDTk5ODtq0aYOVK1eWun/x4sVYvnw5Vq9ejQsXLsDU1BR9+vRBfn6+6pjRo0fj+vXrOHLkCPbt24fTp09j8uTJz+slEBERUS0nqS0LgUokEuzevRuDBw8GUHxWx9nZGR988AFmzZoFAJDL5XBwcMCmTZvw2muv4caNG2jevDlCQ0PRoUMHAMDBgwfx8ssv459//oGzs3NNvRwiIiKqJfRrugNliYuLQ3JyMvz9/VXbLC0t0blzZ4SEhOC1115DSEgIrKysVEEHAPz9/SGVSnHhwgUMGTKk1LoLCgpQUFCgeq5UKpGeng4bGxtIJJLqe1FERERUZYQQyMrKgrOzM6TSsi9W1dqwk5ycDABwcHBQ2+7g4KDal5ycDHt7e7X9+vr6sLa2Vh1TmuDgYMyfP7+Ke0xEREQ1ISEhAS4uLmXur7VhpzoFBQVh5syZqudyuRxubm5ISEiAhYVFDfaMiIiINJWZmQlXV1eYm5uXe1ytDTuOjo4AgJSUFDg5Oam2p6SkoG3btqpjUlNT1co9evQI6enpqvKlkclkkMlkJbZbWFgw7BAREdUxFQ1BqbXz7Hh4eMDR0RHHjh1TbcvMzMSFCxfg4+MDAPDx8UFGRgYuXbqkOub48eNQKpXo3Lnzc+8zERER1T41emYnOzsbMTExqudxcXGIiIiAtbU13NzcMH36dPz3v/9FkyZN4OHhgblz58LZ2Vl1x1azZs3Qt29fTJo0CatXr0ZRURGmTp2K1157jXdiEREREYAaDjthYWHo3bu36vnjcTTjxo3Dpk2b8NFHHyEnJweTJ09GRkYGunXrhoMHD8LIyEhV5ueff8bUqVPh5+cHqVSKYcOGYfny5c/9tRAREVHtVGvm2alJmZmZsLS0hFwu55gdIiKiOkLT7+9aO2aHiIiIqCow7BAREZFOY9ghIiIincawQ0RERDqNYYeIiIh0GsMOERER6TSGHSIiItJpDDtERESk0xh2iIiISKcx7BAREZFOY9ghIiIincawQ0RERDqNYYeIiIh0GsMOERER6TSGHSIiItJpDDtERESk0xh2iIiISKcx7BAREZFOY9ghIiIincawQ0RERDqNYYeIiIh0GsMOERER6TSGHSIiItJpDDtERESk0xh2iIiISKcx7BAREZFOY9ghIiIincawQ0RERDqNYYeIiIh0GsMOERER6TSGHSIiItJpDDtERESk0xh2iIiISKcx7BAREZFOY9ghIiIincawQ0RERDqNYYeIiIh0GsMOERER6TSGHSIiItJpDDtERESk0xh2iIiISKcx7BAREZFOY9ghIiIincawQ0RERDqNYYeIiIh0GsMOERER6TSGHSIiItJpDDtERESk0xh2iIiISKcx7BAREZFOY9ghIiIincawQ0RERDqNYYeIiIh0GsMOERER6TSGHSIiItJpDDtERESk0xh2iIiISKcx7BAREZFOY9ghIiIincawQ0RERDqNYYeIiIh0GsMOERER6TSGHSIiItJptTrsKBQKzJ07Fx4eHjA2NkajRo3w5ZdfQgihOkYIgXnz5sHJyQnGxsbw9/dHdHR0DfaaiIiIapNaHXYWLVqEVatWYcWKFbhx4wYWLVqExYsX47vvvlMds3jxYixfvhyrV6/GhQsXYGpqij59+iA/P78Ge05ERES1hUQ8eZqklnnllVfg4OCA9evXq7YNGzYMxsbG+OmnnyCEgLOzMz744APMmjULACCXy+Hg4IBNmzbhtdde06idzMxMWFpaQi6Xw8LColpeCxEREVUtTb+/a/WZHV9fXxw7dgy3bt0CAFy5cgVnzpxBv379AABxcXFITk6Gv7+/qoylpSU6d+6MkJCQMustKChAZmam2oOIiIh0k35Nd6A8c+bMQWZmJry8vKCnpweFQoGvvvoKo0ePBgAkJycDABwcHNTKOTg4qPaVJjg4GPPnz6++jhMREVGtUavP7OzYsQM///wzfvnlF1y+fBmbN2/GkiVLsHnz5meqNygoCHK5XPVISEiooh4TERFRbVOrz+x8+OGHmDNnjmrsTatWrXD37l0EBwdj3LhxcHR0BACkpKTAyclJVS4lJQVt27Yts16ZTAaZTFatfSciIqLaoVaf2cnNzYVUqt5FPT09KJVKAICHhwccHR1x7Ngx1f7MzExcuHABPj4+z7WvREREVDvV6jM7AwYMwFdffQU3Nze0aNEC4eHh+PbbbzFhwgQAgEQiwfTp0/Hf//4XTZo0gYeHB+bOnQtnZ2cMHjy4ZjtPREREtUKtDjvfffcd5s6di3fffRepqalwdnbG22+/jXnz5qmO+eijj5CTk4PJkycjIyMD3bp1w8GDB2FkZFSDPSciIqLaolbPs/O8cJ4dIiKiukcn5tkhIiIielYMO0RERKTTGHaIiIhIpzHsEBERkU5j2CEiIiKdxrBDREREOo1hh4iIiHQaww4RERHpNIYdIiIi0mkMO0RERKTTGHaIiIhIpzHsEBERkU5j2CEiIqISGjRoAIlEUuIRGBgIAFi7di169eoFCwsLSCQSZGRkVFjn559/XqI+Ly8vtWNiY2MxZMgQ2NnZwcLCAiNGjEBKSsozvRaGHSIiIiohNDQUSUlJqseRI0cAAMOHDwcA5Obmom/fvvj444+1qrdFixZq9Z45c0a1LycnBwEBAZBIJDh+/DjOnj2LwsJCDBgwAEqlstKvRb/SJYmIiEhn2dnZqT1fuHAhGjVqhJ49ewIApk+fDgA4efKkVvXq6+vD0dGx1H1nz57FnTt3EB4eDgsLCwDA5s2bUa9ePRw/fhz+/v7avYh/8cwOERERlauwsBA//fQTJkyYAIlE8kx1RUdHw9nZGQ0bNsTo0aMRHx+v2ldQUACJRAKZTKbaZmRkBKlUqnYGSFsMO0RERFSuPXv2ICMjA2+++eYz1dO5c2ds2rQJBw8exKpVqxAXF4fu3bsjKysLANClSxeYmppi9uzZyM3NRU5ODmbNmgWFQoGkpKRKt8uwQ0REROVav349+vXrB2dn52eqp1+/fhg+fDhat26NPn364M8//0RGRgZ27NgBoPjS2c6dO/HHH3/AzMwMlpaWyMjIgLe3N6TSykcWjtkhIiKiMt29exdHjx7Frl27qrxuKysrNG3aFDExMaptAQEBiI2NRVpaGvT19WFlZQVHR0c0bNiw0u3wzA4RERGVaePGjbC3t0f//v2rvO7s7GzExsbCycmpxD5bW1tYWVnh+PHjSE1NxcCBAyvdDsMOERERlUqpVGLjxo0YN24c9PXVLwYlJycjIiJCdVYmMjISERERSE9PVx3j5+eHFStWqJ7PmjULp06dwp07d3Du3DkMGTIEenp6GDVqlOqYjRs34vz584iNjcVPP/2E4cOHY8aMGfD09Kz06+BlLCIiIirV0aNHER8fjwkTJpTYt3r1asyfP1/1vEePHgCKw8rjgcyxsbGI+ycJ52LT4GFrin/++QejRo3CgwcPYGdnh27duuH8+fNqt7lHRUUhKCgI6enpaNCgAT755BPMmDHjmV6HRAghnqkGHZCZmQlLS0vI5XLVff1ERET0bLaHxiNoVySUApBKgOChrTCyo1uV1a/p9zcvYxEREVUgMTERb7zxBmxsbGBsbIxWrVohLCxMtT87OxtTp06Fi4sLjI2N0bx5c6xevbrcOjdt2lRi6QQjI6MSx924cQMDBw6EpaUlTE1N0bFjR7W5aWqjgkcK/HElEXN+Kw46AKAUwMe7riFJnvfc+8PLWERE9MJIkuchLi0HHramcLI01qjMw4cP0bVrV/Tu3RsHDhyAnZ0doqOjUa9ePdUxM2fOxPHjx/HTTz+hQYMGOHz4MN599104OzuXO7DWwsICUVFRqudPT9gXGxuLbt26YeLEiZg/fz4sLCxw/fr1UkNRVb9ubQghEJWShTPRaTgdnYaLcQ+QX1RyeQeFELiTllstfSgPww4REb0QKntJZdGiRXB1dcXGjRtV2zw8PNSOOXfuHMaNG4devXoBACZPnow1a9bg4sWL5YYdiURS5tIJAPDJJ5/g5ZdfxuLFi1XbGjVqVGGfn1QVl5JKC0upmfn4KzoNZ2KKH/ezCtTKWJsaIj2nUG2bnkSCBrYmWrVdFRh2iIhI5yXJ8zDnt0g8HqT6+JJKj6Z2FZ5l2Lt3L/r06YPhw4fj1KlTqF+/Pt59911MmjRJdYyvry/27t2LCRMmwNnZGSdPnsStW7ewdOnScuvOzs6Gu7s7lEolvL29sWDBArRo0aK4j0ol9u/fj48++gh9+vRBeHg4PDw8EBQUhMGDB5dan0IpEJ+ei5tJmbiRlInwhAz8FZ2m2q8UwOzfIrHurzg4WRrBzkwGW3MZbM0MYWsm+/+HuSFsTGXQk0rUwpJEAnRtZIv7WQWISslSa9vIQIrOHjbo3sQW3ZrYwtPBHDvCEvDxrmtQCAE9iQQLhrZ87md1AA5QBsABykREuuyRQolp28KxPzK5xL6tk7rAp5FNueUfXzKaOXMmhg8fjtDQUEybNg2rV6/GuHHjABSv6TR58mRs2bIF+vr6kEqlWLduHcaOHVtmvSEhIYiOjkbr1q0hl8uxZMkSnD59GtevX4eLiwuSk5Ph5OQEExMTfPTJZ3Bv1RG3Lp3Bwi8+w4kTJ9Cmow9uJmfhZlImbiZn4UZyFm4lZyGvSPEMP63/J5UAlsYGeJhbVOp+iQRo6WyJbk1s0b2JLdq714NMX6/EcUnyPNxJy0UDW5MqDzqafn8z7IBhh4hIV8lzizB162W1sxtPmu7XBNP8m5S7uKWhoSE6dOiAc+fOqba9//77CA0NRUhICABgyZIlWLduHZYsWQJ3d3ecPn0aQUFB2L17t8YrdRcVFaFZs2YYNWoUvvzyS9y7dw/169eHb8BAJHlPLj6zAiBn3wIo9WQw6/dBqfUYGUjh6WAOL0cLOFsZYdmxaDz5TS+VAEuGt4FSAGnZBbifVYC07H8fWYVIyy5Aem4hyksH7/s1xpu+HrA2NdTotVUXTb+/eRmLiIh00q2ULEzaEoa7D3JhbKCHod71se1iAhRCQAJAAFh2LBoJD/Pw1ZCWMDIoeVYCAJycnNC8eXO1bc2aNcNvv/0GAMjLy8PHH3+M3bt3q2YZbt26NSIiIrBkyRKNw46BgQHatWunmqTP3Koe9PT0cT3PAlb/Bg8BoMDMCQX//A0zAC71jOHlaIFmTuaq/7rbmEJP+v/hzdHSqMSlpKHeLuX25ZFCifTcQty8l4Vxmy6qBR89iQSjOrnVeNDRBsMOERHpnMPXkzFjewRyChVwqWeMtWM6oLmzBaa+1Bh30nLhbmOMA9dS8NX+v/Hb5X8Qcz8ba8e0h4NFybucunbtqnbHFADcunUL7u7uAIrPyBQVFZVYqFJPTw9KZck7ksqiUCgQGRkJ764vIWjXVey/mgR9h8Z4lJ6odlxReiLaNmuMw58HwMLIoMJ6R3Z0Q4+mdlpdStLXk8Le3Aj2nkZYOLRVrRh38ywYdoiISGcolQLfHY/B0qO3AAA+DW2wcrS36iyEk6Wx6ot6YjcPeDqYI/CXy7iSkIFXvjuD1W+0R3v3emp1zpgxA76+vliwYAFGjBiBixcvYu3atVi7di2A4tvHe/bsiQ8//BDGxsZwd3fHqVOnsGXLFnz77beqesaOHYv69esjODgYAPDFF1+gS5cuaNy4MSJvJ+Kz/y5EdGwcMru1guHFBACAa6+RiNn2FWQuLWDk3hp5ty8hL+Yi5iw/rFHQeezJ162tyoSl2oZjdsAxO0REuiC74BE+2BGBQ9dTAABv+jbAJ/2bwUCv/Plz7z7IweQtlxCVkgVDPSn+O7glRnR0VTtm3759CAoKQnR0NDw8PDBz5ky1u7GSk5MRFBSEw4cPIz09He7u7pg8eTJmzJihGg/k260HrBycsX7DRjhZGuPtwPfw62+7kJF2HzAyhcyhMax6jIGNmyf6tXLE4Hb10cXDBu9+tgTrVy6DIisNBtb18f6HH+PrWW9V8U+vbuIAZS0w7BAR1W13H+Rg0pYw3ErJLg4sQ1piRAfXigv+K6fgET7YcQUHrxffsaVpUNKU2u3bABo7mCE2NVs1u7C+VIJenvYY0q4+/JrZlxg/VJ13NNVlDDtaYNghIqq7zkSnIfCXy5DnFcHeXIbVY9rD261exQWfolQKrDgRg2+PlH4JrLKuJDzE4JXnUNqXrbebFYa0q4/+rZ3r1IDf2oJhRwsMO0REdY8QAuvPxGHBnzegFEBbVyusKWOQsTbKGtysqeyCR7hw+wH+ik7DX9H3EXs/p9Tjlo5ogyEV3BVF5WPY0QLDDhFR3ZJfpMDHuyKxK7z4TqVX27vgv4PLvn1cW0/ftr5keBv0b+1U6rGPFEpcTZTjTHQazkSn4XL8QzxS/v9X6+Pb3J+kJ5HgzJzevCT1jDjPDhER6ZwkeR4u3X2IFcdjcDM5C3pSCT7t3wxv+jYod2JAbTV1MMfvgV3x3tZw/PXvZbKLce4IaO6IhvamKHykLF4XKjoNZ2PTkJX/SK28m7VJ8czCjW3h28gWB68n1fnbt+syntkBz+wQ0YsjMTERs2fPxoEDB5Cbm4vGjRtj48aN6NChQ4ljp0yZgjVr1mDp0qWYPn16mXWuWrUKq1atwp07dwAALVq0wLx589CvXz/VMbGxsZg1axbOnDmDgoIC9O3bF9999x0cHBw07vu2i/EI2h2pmuDO2FAP68d2gG9jW43r0NYjhRKLD0Vh7enb5R5nYaSPro1t/w04dnCzKbnYJQcZVz2e2SEiIjUPHz5E165d0bt3bxw4cAB2dnaIjo5GvXolB/Pu3r0b58+fh7Ozc4X1uri4YOHChWjSpAmEENi8eTMGDRqE8PBwtGjRAjk5OQgICECbNm1w/PhxAMDcuXMxYMAAnD9/vsRkfE8reKTA5nN3sODPm+rbixTwsDPV4iegPX09KcZ3bYB1p2+XuBTV1tUKfl726N7UDq3qW6rNWlyaZ5nrhp4Nww4R0Qti0aJFcHV1xcaNG1XbPDw8ShyXmJiI9957D4cOHVItf1CeAQMGqD3/6quvsGrVKpw/fx4tWrTA2bNncefOHYSHh6v++t68eTPq1auH48ePl7mcQkZuIX6+EI9N5+7gflZBif1KAdxJy632ABGXllPqnVSz+3pVuIgo1Q5VM4EAERHVenv37kWHDh0wfPhw2Nvbo127dli3bp3aMUqlEmPGjMGHH36IFi1aaN2GQqHAtm3bkJOTAx8fHwDFK4JLJBLIZDLVcUZGRpBKpThz5kyJOu4+yMG836/BJ/g4vj4UhftZBbAzk+Hp8yZ6Egka2Ja8XFTVPGxN8fRJm+fVNlUNhh0iohfE7du3sWrVKjRp0gSHDh3CO++8g/fffx+bN29WHbNo0SLo6+vj/fff16ruyMhImJmZQSaTYcqUKdi9e7dq8cwuXbrA1NQUs2fPRm5uLnJycjBr1iwoFAokJSWp6rh0Nx1TfryEXktOYkvIXeQVKdDcyQJLR7bB2TkvYeGwVtD7dxDy8xzk62RpjOChNdM2VQ1exiIiekEolUp06NABCxYsAAC0a9cO165dw+rVqzFu3DhcunQJ//vf/3D58mWt72zy9PREREQE5HI5fv31V4wbNw6nTp1C8+bNYWdnh507d+Kdd97B8uXLIZVKMWrUKHh7e0MikeBAZBLW/nUb4fEZqvp6edphcveG8Glko+pLTa7RpAvrQ73IeDcWeDcWEb0Y3N3d8Z///Ac//PCDatuqVavw3//+F4mJiVi2bBlmzpypNmBYoVBAKpXC1dVVdbeVJvz9/dGoUSOsWbNGbXtaWhrScopwL0+Kwb4tYeszFMpWAwEAhnpSDGlXH29190ATB/Nne7H0QuDdWEREpKZr166IiopS23br1i24u7sDAMaMGVNisHCfPn0wZswYjB8/Xqu2lEolCgrUBxUXKZRYH3ofq0/dRu7dK8h6+ADmzt6wMzHAmC7uGOPjDnvzZ5v9mKg0DDtERC+IGTNmwNfXFwsWLMCIESNw8eJFrF27FmvXrgUA2NjYwMZG/e4iAwMDODo6wtPTU7XNz88PQ4YMwdSpUwEAQUFB6NevH9zc3JCVlYVffvkFJ0+exMGDBxF7PxtnotOwfsMGJCjrodDAHAX3buLh0bUw7zgIhrYu2P1O12q/hZxebAw7REQviI4dO2L37t0ICgrCF198AQ8PDyxbtgyjR4/Wqp7Y2FikpaWpnqempmLs2LFISkqChYUl6jfyxPBPVuHzS3q4d/wUAODhtRvIvnYUyrxs6Fvaw9JnBMw7DoYQQHJmPsMOVSuO2QHH7BARaSNJnoe4tBx42JqinokhLt19WLx0Qsx9XL+XiSe/VQz1pOjQoB66NbFFM0cLTNwciieWjeIaUfRMOGaHiIiq3PbQeATtilQFFn09CR4p1P9m9nI0R7fGtuje1A6dGljD2PD/F+cMHtqKa0TRc8ewQ0REFUrLLsDP5+9i6dFote2PFAI2pobo6WmH7k1s0bWxbbmDjHkLN9UEhh0iIipVXqECh/9Oxp7wRJyOToNCWfqohxWvt4NPI80X4+QaUfS8MewQEZGKQikQEvsAu8MTcfBaEnIKFap9zRzNcTM5S22dqOJlEzi4mGo3hh0iohfQk4OMHS2McCMpC7vD/8HvEfeQ+sSim67WxhjStj4GtauPRnZm2B4azzE3VOcw7BARvWCeHGQsAWBvIUNK5v8HHCsTA7zS2glD2tWHt1s9taUjOOaG6iKGHSKiSnry7Mjz/tKvqG2FUuBhbiHSsguQlvXvf7MLEPcgBz+fj1cdJwCkZBbAQE+C/zR3wOC29dHL0x6G+mWvE80xN1TXMOwQEVXCk2dHpJLiW6pHdnR7Lm1vOheH+X/8DfHvmZluTWxhayZDWnYB7mcVIC27EOk5BShjPHGpVr3RHv7NHKqtz0Q1iWGHiEhLSfI8zNkVqZo8TymA2b9F4veIe2jragUvJws0czSHh60p9PXKPkNSEYVSIC4tBzeTM3EzKQs3kzNxLTETyZn5qmMEgL+i00otL5EA9UwMYWtmCFszGWzNZDA20MOOsIQSg4xbOHNCVdJdWoedEydOoHfv3tXRFyKiOuF87AOUNvf8udgHOBf7QPXcUF+KJvZmaOZkAS9Hc9V/bcxkANQvRRnp6+HGE6HmRlIWbqVkoeCRUqM+vd7JFe3drWFrLoOtmSHszGSwNjUsNWx5u1txkDG9ULReLkImk8HFxQXjx4/HuHHj4OrqWl19e264XAQRaSojtxADvzuD+Id5atulEmDmf5rinjwfN5MyEZWcpXbb9pPszGWwMjZAdGp2he0ZG+jB09EczZzM4eVoATszQ0zdGv7MSy4kyfM4yJjqvGpbLiIxMRE//vgjNm/ejPnz5+Oll17CxIkTMXjwYBgaGj5Tp4mIarPCR0q8/eMlxD/Mg5WxATLzi6AUUJ0deXLMjlIp8M/DPNXZmhtJmbiZnIm76bm4n1U8tuZpzlZGaFXfEl6OFqpw42ZtAqlUonZccMGjZz4zw0HG9CJ5poVAL1++jI0bN2Lr1q0AgNdffx0TJ05EmzZtqqyDzwPP7BBRRYQQmLXzKn67/A/MZPr47R1fWBjra312JKfgEX67/A/m/X69xL6tk7rAp5GNRvXwzAyR5t/flR85B8Db2xtBQUGYOnUqsrOzsWHDBrRv3x7du3fH9eslf5GJiOqqlSdi8Nvlf6AnlWDlaG94OprDydIYPo1stAobpjJ9/Ke5A546WfPvTMQmGtdTmbaJXlSVCjtFRUX49ddf8fLLL8Pd3R2HDh3CihUrkJKSgpiYGLi7u2P48OFV0sHExES88cYbsLGxgbGxMVq1aoWwsDDVfiEE5s2bBycnJxgbG8Pf3x/R0dHl1EhEuuDzzz+HRCJRe3h5eQEA7ty5U2Lf48fOnTvLrPPNN98scXzfvn3xx5V7WHL4FgDg7bbG+PbDt2BrawsLCwt069YNJ06c0KrvTpbGCB7aCnr/TtbHQcJE1UvrMTvvvfcetm7dCiEExowZg8WLF6Nly5aq/aampliyZAmcnZ2fuXMPHz5E165d0bt3bxw4cAB2dnaIjo5GvXr1VMcsXrwYy5cvx+bNm+Hh4YG5c+eiT58++Pvvv2FkVPbKu0RU97Vo0QJHjx5VPdfXL/5Ic3V1RVJSktqxa9euxddff41+/fqVW2ffvn2xceNG1fMbqXl4e8cVAMDEbh74YeZgNGnSBMePH4exsTGWLVuGV155BbGxsXB0dNS475yJmOj50Trs/P333/juu+8wdOhQyGSyUo+xtbXV+i+d0ixatAiurq5qHzweHh6q/xdCYNmyZfj0008xaNAgAMCWLVvg4OCAPXv24LXXXnvmPhBR7aWvr19qwNDT0yuxfffu3RgxYgTMzMzKrVMmk6nKxj/IxazfI1H4SAn/Zg6Y3MkO86KjsX79erRu3RoAsHDhQnz//fe4du2aVmEH4CBhoudF68tYx44dw6hRo8oMOkDxB1DPnj2fqWMAsHfvXnTo0AHDhw+Hvb092rVrh3Xr1qn2x8XFITk5Gf7+/qptlpaW6Ny5M0JCQsqst6CgAJmZmWoPIqp7oqOj4ezsjIYNG2L06NGIj48v9bhLly4hIiICEydOrLDOkydPwt7eHk2aNoXvgFFITUtDy/oWWD6qLeztbOHp6YktW7YgJycHjx49wpo1a2Bvb4/27dtX9csjoiqiddgJDg7Ghg0bSmzfsGEDFi1aVCWdeuz27dtYtWoVmjRpgkOHDuGdd97B+++/j82bNwMAkpOTAQAODupTnDs4OKj2lSY4OBiWlpaqhy7MFUT0ouncuTM2bdqEgwcPYtWqVYiLi0P37t2RlZVV4tj169ejWbNm8PX1LbfOvn37YsuWLTh4+Aic/jMRqbfC8fC3+Vj7hjdMDPUhkUhw9OhRhIeHw9zcHEZGRvj2229x8OBBtcvrRFS7aB121qxZoxoE+KQWLVpg9erVVdKpx5RKJby9vbFgwQK0a9cOkydPxqRJk565naCgIMjlctUjISGhinpMRM9Lv379MHz4cLRu3Rp9+vTBn3/+iYyMDOzYsUPtuLy8PPzyyy8andV57bXXMGDAAGyPkSDevCXcRs1HTmIUblw+D6D40nlgYCDs7e3x119/4eLFixg8eDAGDBhQYowQEdUeWoed5ORkODk5ldhuZ2dX5b/sTk5OaN68udq2Zs2aqU5VP74+npKSonZMSkpKudfOZTIZLCws1B5EVLdZWVmhadOmiImJUdv+66+/Ijc3F2PHjtWonjWnb2N7WAKkEmBt4MuwtbVV1Xn8+HHs27cP27ZtQ9euXeHt7Y3vv/8exsbGqjPORFT7aB12XF1dcfbs2RLbz549WyV3YD2pa9euiIqKUtt269YtuLu7AygerOzo6Ihjx46p9mdmZuLChQvw8fGp0r4QUe2WnZ2N2NjYEn+MrV+/HgMHDoSdnV2FdRyITMLCAzcBAPNeaY6mZkV48OCBqs7c3FwAgFSq/tEplUqhVGq2hhUR1QChpUWLFgkbGxuxYcMGcefOHXHnzh2xfv16YWNjIxYsWKBtdeW6ePGi0NfXF1999ZWIjo4WP//8szAxMRE//fST6piFCxcKKysr8fvvv4urV6+KQYMGCQ8PD5GXl6dxO3K5XAAQcrm8SvtPRNXngw8+ECdPnhRxcXHi7Nmzwt/fX9ja2orU1FTVMdHR0UIikYgDBw6UWoenp6fYtWuXEEKIszcSRL0uw4TjG0vE+2sOiqNHjwpvb2/RpEkTkZ+fL4QQ4v79+8LGxkYMHTpUREREiKioKDFr1ixhYGAgIiIiqv9FE5EaTb+/tQ47SqVSfPTRR8LIyEhIpVIhlUqFiYmJmD9/fqU7W54//vhDtGzZUshkMuHl5SXWrl1boj9z584VDg4OQiaTCT8/PxEVFaVVGww7RHXPyJEjhZOTkzA0NBT169cXI0eOFDExMWrHBAUFCVdXV6FQKEqtA4DYuHGjSEjPEe3m7RNGDdoJI/N6wsDAQLi7u4tJkyaJ5ORktTKhoaEiICBAWFtbC3Nzc9GlSxfx559/VtvrJKKyafr9Xem1sbKzs3Hjxg0YGxujSZMm5d6KXttxbSyiuitJnoe4tBx42JpqPWdNkjwP1+9lYsH+G7idlgMvR3P8+o4vzGRaT0FGRDWg2lY9f8zMzAwdO3asbHEiome2PTQeQbsioRSAVAJ81McTr7TRbOzgviv3sPhQFJT//rlnLtPHhjc7MugQ6aBK/VaHhYVhx44diI+PR2Fhodq+Xbt2VUnHiIjKExKbhtm/RaqeKwWw8GAUFh6MKqdU2XIKH0Eiqfg4Iqp7tL4ba9u2bfD19cWNGzewe/duFBUV4fr16zh+/DgsLS2ro49ERACK57k5F5uGCZtCMWrdhVKPMZBKINOXlvsweHrJcRSHpTtpudX9EoioBmh9ZmfBggVYunQpAgMDYW5ujv/973/w8PDA22+/Xer8O0REz6pIocSfkUlY99dtXEsse3kXPYkEp2f3rnDsTpI8D10XHlddwnpctoGtSVV1mYhqEa3P7MTGxqJ///4AAENDQ+Tk5EAikWDGjBlYu3ZtlXeQiF5cmflFWHs6Fj0Wn8C0bRG4lpgJIwMp3ujihhOzemHRsFbQ+/fak55EggVDW2o0SNnJ0hjBQytXlojqHq3P7NSrV0+19kz9+vVx7do1tGrVChkZGaoJt4iInkViRh42nonDttAEZBc8AgDYmskwzscdo7u4w9rUEADgYWuKHk3tcCctFw1sTbQKKyM7ulW6LBHVLVqHnR49euDIkSNo1aoVhg8fjmnTpuH48eM4cuQI/Pz8qqOPRKTDnrx1/H5WAdb9FYc/I5Og+PcaUxN7M7zV3QOD2taHkYFeifJOlsaVDirPUpaI6g6tw86KFSuQn58PAPjkk09gYGCAc+fOYdiwYfj000+rvINEpLuevHX8aV0b2+Ct7g3Rs4kdpKUMKCYi0pRWYefRo0fYt28f+vTpA6B4PZg5c+ZUS8eISLfdy8jFnF2ReHpa074tHPGeX2O0cObdnURUNbQaoKyvr48pU6aozuwQEVXG1X8yMGnLpRJBBwDG+TZg0CGiKqX1ZaxOnTohIiJCtfI4EZGm7qTlYMnhKOy7mlTqft7+TUTVQeuw8+6772LmzJlISEhA+/btYWpqqra/devWVdY5ItINadkFWH4sGr9ciMcjpYBEAgxpWx9NHMyw5NAtKITg7d9EVG20XghUKi155UsikUAIAYlEAoVCUWWde164EChR9cgueIQf/rqNdadvI6ew+LOhZ1M7zO7rhebOxb9rSfI83v5NRJVSbQuBxsXFPVPHiEj3FSmU2HYxHv87Fo207OL181q7WGJOXy/4NrZVO5a3fxNRddM67HCsDhE97fFcOQ1sTHA5PgNLDkXhzoPiSUbdbUzwYR9P9G/lBAlX2iSiGqB12NmyZUu5+8eOHVvpzhBR3VPWXDm2ZoaY5tcEr3Vyg4Ge1ivTEBFVGa3H7NSrV0/teVFREXJzc2FoaAgTExOkp6dXaQefB47ZIaqc0hbUBICJ3Rpg5n88YSrT+u8pIiKNafr9rfWfWw8fPlR7ZGdnIyoqCt26dcPWrVufqdNEVHcUPlJi2ZHoUmc/9m/myKBDRLVGlXwaNWnSBAsXLsQbb7yBmzdvVkWVRFSLXbr7EB/vikRUSlaJfZwrh4hqmyr700tfXx/37t2rquqIqBbKzC/C4oM38fOFeAgBWJsa4j/N7fFr2D9QCHCuHCKqlbQOO3v37lV7LoRAUlISVqxYga5du1ZZx4io9hBC4OC1ZHy29zpSswoAAK+2d8HHLzeDtakhpvs35Vw5RFRraR12Bg8erPZcIpHAzs4OL730Er755puq6hcR1RL3MvIw7/drOHojFQDgYWuKrwa3VJsvh3PlEFFtpnXYUSqV1dEPIqplFEqBzefuYMnhKOQWKmCgJ8GUno0Q2LsxjAz0arp7REQa4+QXRLXMwoULIZFIMH36dNW25ORkjBkzBo6OjjA1NYW3tzd+++23Z6qzvHqvJcox5Puz+GLf38gtVKC9ez3sf787PgjwZNAhojpH6zM7w4YNQ6dOnTB79my17YsXL0ZoaCh27txZZZ0jetGEhoZizZo1JRbUHTt2LDIyMrB3717Y2tril19+wYgRIxAWFoZ27dpVqs4n693wyw7kSEwQcngvho8YAedxy6Bv3xDmRvqY088Lozq6QSrl7MdEVDdpfWbn9OnTePnll0ts79evH06fPl0lnSJ6EWVnZ2P06NFYt25dick7z507h/feew+dOnVCw4YN8emnn8LKygqXLl2qdJ2P6+3Y/3VMPZKFDw+lYJfoDImhKfKSotG/lROOzeyJ0Z3dGXSIqE7TOuxkZ2fD0NCwxHYDAwNkZmZWSaeIXkSBgYHo378//P39S+zz9fXF9u3bkZ6eDqVSiW3btiE/Px+9evWqdJ0A0L5TF2z88RcU5WZBCCVy/j4FoSjE528Px8rR3rC3MKqKl0ZEVKO0vozVqlUrbN++HfPmzVPbvm3bNjRv3rzKOkb0Itm2bRsuhl3C9zsPIkmeV2L/jh07MHLkSNjY2EBfXx8mJibYvXs3GjduXG6dly9fRmhoaKn7UzPz4frqx7j49Sz8s3wUINWDRF8GuyGfoEMr/i4Tke7QOuzMnTsXQ4cORWxsLF566SUAwLFjx7B161aO1yGqhISEBEwJfA9mgz/D+C1XIJUARv/OZfPY3LlzkZGRgaNHj8LW1hZ79uzBiBEj8Ndff6FVq1al1jlt2jQcOXIERkbqZ2ey8ouw7vRtrPsrDol/roSyIAf2I/8LPRML5N46j7TfFyFvWl+gkU21vm4ioudF64VAAWD//v1YsGABIiIiYGxsjNatW+Ozzz5Dz549q6OP1Y4LgVJN2vDzdkx84zVA8sRVZaGERCKBVCpFVFQUGjdujGvXrqFFixaqQ/z9/dG4cWOsXr26RJ179uzBkCFDoKf3/3dOKRQKSCQSCIkEbh/sxiN5Ku6tnYT3v/8d++L1oRACehIJTI4tQFfvlqXWS0RUm2j6/V2p5SL69++P/v37V7pzRPT/Hlo2hdOEFWrbHvz5PxjZumDIm4E4F5UIAJBK1YfY6enplTnvlZ+fHyIjIwEASqXAyVup+GRGIISlMyw6D0MjewsM72KGd9cCU3o1wRznBqoZkN+8/D/Op0VEOkXrAcqhoaG4cOFCie0XLlxAWFhYlXSK6EUghMDKEzH47sw9GNo1UHtIDGQQMnOcvG+ET088hJFNfbw8fAx2HDiB2NhYfPPNNzhy5IjajOZ+fn5YsaI4NJmbm6Nly5Z4KHNA0MkMfBuWjyKJAUwtrPD15FdwaEYPvDWgOxo3boy3334bCVGRsEcGfvnh+xL1EhHVdVqHncDAQCQkJJTYnpiYiMDAwCrpFJGuyy18hKlbw/H1oSgAQJeG1nh8d7eeRIKGdmZ4uZUTRnVyhYWpEayHzkNqkQyjXh0Kz+YtsXTVD/jfqh/UpoGIjY1FWloaAOBaohxj1l/AmPUXcf1eJsxk+nC1NsGr7V0wurM7DPSkMDAwwJ9//gk7OzsMGDAArVu3xpYtW7B58+ZSp5cgIqqrtB6zY2ZmhqtXr6Jhw4Zq2+Pi4tC6dWtkZWVVaQefB47ZoecpIT0Xk3+8hBtJmTDQk2D+wJZ4vbMbkuR5pS6mmV+kwMmoVOy6nIgTUakoUhT/ykokgG8jGwxuWx99Wzoiu+ARLtxOx/6rSThyIwUAYKAnwRtd3DG1d2PYmMlq5PUSEVWXahuzI5PJkJKSUiLsJCUlQV+/UkOAiF4YIbEP8O7Pl/Awtwi2ZoZY9UZ7dGxgDaDsxTSNDPTQt6UT+rZ0QkZuIfZHJmFPeCJC7zzE2ZgHOBvzAEG7I/FIof53y+C2zvggwBOu1ibP5bUREdVWWp/ZGTVqFJKSkvD777/D0tISAJCRkYHBgwfD3t4eO3bsqJaOViee2aHqJkTxoppf7r8BhVKgVX1LrBnTHs5WlV8pPCE9F3vCE7EzLAHxD9Xn5pFKgLNzXuJK5ESk06rtzM6SJUvQo0cPuLu7q9bkiYiIgIODA3788cfK95hIRxU8UmDunmvYEfYPAGBIu/oIHtrqmRfUdLU2wXt+TdDevR5e/0H9pgGlAO6k5TLsEBGhEmGnfv36uHr1Kn7++WdcuXIFxsbGGD9+PEaNGgUDA4Pq6CNRnZWamY+3f7qE8PgMSCVAUL9meKu7BySSqltrysPOFFJJccB5TE8iQQNbXr4iIgIqcTcWAJiammLy5MlYuXIllixZgrFjxzLoUAkLFy6ERCLB9OnTS+wTQqBfv36QSCTYs2dPufV8/vnn8PLygqmpKerVqwd/f3+16Q/u3LmDiRMnwsPDA8bGxmjUqBE+++wzFBYWVvEr0k54/EMMWHEG4fEZsDDSx6bxnTCpR8MqDTpA8Vif4KGtoPdvvXoSCRYMbcmzOkRE/6r0iOK///4b8fHxJb5QBg4c+MydorovNDQUa9asQevWrUvdv2zZMo2/9Js2bYoVK1agYcOGyMvLw9KlSxEQEICYmBjY2dnh5s2bUCqVWLNmjWqm4UmTJiEnJwdLliypypelsZ1hCfhk9zUUKpRoYm+GdWM7oIGtabW1N7KjG3o0tSv1bi4iohed1gOUb9++jSFDhiAyMrJ46nnx+DbY4i8uhUJR9b2sZhygXLWys7Ph7e2N77//Hv/973/Rtm1bLFu2TLU/IiICr7zyCsLCwuDk5ITdu3drNYnd43+vo0ePws/Pr9Rjvv76a6xatQq3b99+xlejnYT0HHy1/wYOXi++9TuguQO+HdkWZjLeqUhEVNU0/f7W+jLWtGnT4OHhgdTUVJiYmOD69es4ffo0OnTogJMnTz5Ln0lHBAYGon///vD39y+xLzc3F6+//jpWrlwJR0dHresuLCzE2rVrYWlpiTZt2pR5nFwuh7W1tdb1P4s1p2LRffFJVdDxa2aP1W+0Z9AhIqphWn8Kh4SE4Pjx47C1tYVUKoVUKkW3bt0QHByM999/H+Hh4dXRT6ojtm3bhsuXLyM0NLTU/TNmzICvry8GDRqkVb379u3Da6+9htzcXDg5OeHIkSOwtbUt9diYmBh89913z+0Sljy3CIsP38TP5+PVtp+8eR8pWfm8pEREVMO0DjsKhQLm5uYAAFtbW9y7dw+enp5wd3dHVFRUlXeQ6o6EhARMmzYNR44cgZGRUYn9e/fuxfHjxysViHv37o2IiAikpaVh3bp1GDFiBC5cuAB7e3u14xITE9G3b18MHz4ckyZNqvRr0UR+kQKbz93ByhMxyMx/VGK/Qgje/k1EVAtofRmrZcuWuHLlCgCgc+fOWLx4Mc6ePYsvvviixKzK9GK5dOkSUlNT4e3tDX19fejr6+PUqVNYvnw59PX1ceTIEcTGxsLKykq1HwCGDRuGXr16lVu3qakpGjdujC5dumD9+vXQ19fH+vXr1Y65d+8eevfuDV9fX6xdu7a6XiYUSoGdYQl4aclJBB+4icz8R2hoa4qnx1vz9m8iotpB6zM7n376KXJycgAAX3zxBV555RV0794dNjY22L59e5V3kOoOPz8/REZGqm0bP348vLy8MHv2bNja2uLtt99W29+qVSssXboUAwYM0KotpVKJgoIC1fPExET07t0b7du3x8aNGyGVVmpWhXIJIXD8ZioWHbyJWynZAABnSyPMDPDEkHb18eulBHy86xoUQvD2byKiWkTrsNOnTx/V/zdu3Bg3b95Eeno66tWrV+Xzh1DdYm5ujpYtW6ptMzU1hY2NjWp7aYOS3dzc4OHhoXru5eWF4OBgDBkyBDk5Ofjqq68wcOBAODk5IS0tDStXrkRiYiKGDx8OoDjo9OrVC+7u7liyZAnu37+vqqsyg6BLczn+IRYeuImLcekAAEtjAwT2boSxPg1UMyHz9m8iotqpSm4Ted53vZBui4qKglwuBwDo6enh5s2b2Lx5M9LS0mBjY4OOHTvir7/+QosWLQAAR44cQUxMDGJiYuDi4qJWl5YzK5QQez8bXx+MwsHryQAAmb4U47t64J2ejWBpUnIizbIW8yQiopqj9Tw7uojz7NQuSfI8xKXlwMPW9LkHh8dtm8v0sTU0AdtDE6BQCkglwKvtXTDdv+kzLd5JRERVp9oWAiWqTttD4xG0KxJKUbxyd/DQVhjZ0e25t/0k/2b2+LCPFzwdzZ9LP4iIqGox7FCtkSTPw5xdkXh8rlEpgDm/RcLBwgg9mthBKq36MWEKpcC1RDkOXEvC6lMlZ1teNdob/Vo5VXm7RET0/Ggddk6fPg1fX1/VbcOPPXr0COfOnUOPHj2qrHP0Yvnt0j94+qKqAPDmxlBYmxrCt5ENujexRbcmdqj/DJeSEtJzcSYmDWei03A2Ng0ZuUVlHmtlYljpdoiIqHbQOuz07t0bSUlJJSZzk8vl6N27d51cG4tq3vbQeHx75Fap+0wM9ZCeU4h9V5Ow72oSAKChrSm6NbFFt8a28GlkA3Oj4sHCpY33kecVIST2Ac7E3MeZ6DTceZCrVr+5TB9t3axwJjoNT2YtzpNDRKQbtA47QohSbzF/8OABTE2rb1Vn0k1CCKw4HoNv/g06Hdzr4XL8QygFVHPVDPV2wZWEDJyOTsOZ6Pu48o8ct9NycDstB1tC7kJPKkE7VytYmRjg2M1UCAFIJMBLXvZ4mFOIK//IoXhiII6eVIK2rlbo3sQW3ZvYoo2LFfT1pNgeGs95coiIdJDGd2MNHToUAPD777+jb9++kMlkqn0KhQJXr16Fp6cnDh48WD09rUa8G6tmKJQCn++9jh/P3wUABPZuhFkBnkjOzC93rprM/H/P1ESn4a/o+yXO1JTmyTNBXRrZwMKo5G3jQPGZIc6TQ0RUN1T53ViWlpYAiv8SNzc3h7Hx/38RGBoaokuXLtW+FhHpjvwiBWZsj8CBa8mQSIDPB7TAON8GACqeq8bCyAB9WjiiT4viCQMT0nOxOeQOfvgrrsSxk3t4YJyvh8ZjfDhPDhGR7tE47GzcuBEA0KBBA8yaNYuXrKjS5HlFmLQlDBfj0mGoJ8XSkW3Rv3Xl73hytTbBxG4e2HAmTu22cT2JBOO7ejC8EBG94LReQOijjz5SG7Nz9+5dLFu2DIcPH67SjpFuSpbnY+SaEFyMS4e5TB+bJnR8pqDzmJOlMYKHtoLev+9NjrkhIqLHtB6gPGjQIAwdOhRTpkxBRkYGOnXqBENDQ6SlpeHbb7/FO++8Ux39JB0Qk5qNcRsuIjEjD/bmMmwa3wnNnatujBTXpiIiotJofWbn8uXL6N69OwDg119/haOjI+7evYstW7Zg+fLlVd5B0g2X7j7Eq6vPITEjDw1tTfHbO75VGnQec7I0hk8jGwYdIiJS0frMTm5uLszNi6fNP3z4MIYOHQqpVIouXbrg7t27Vd5BqvuO3UhB4C+XkV+kRBtXK2x8syOsTTlZHxERPR9an9lp3Lgx9uzZg4SEBBw6dAgBAQEAgNTUVN62TSXsCE3A5B8vIb9Iid6edtg6qTODDhERPVdan9mZN28eXn/9dcyYMQMvvfQSfHx8ABSf5WnXrl2Vd5DqniR5HuLu5+DUrVSsOV18O/ir7V0QPLQVDPS0ztdERETPROtvnldffRXx8fEICwvDoUOHVNv9/PywdOnSKu3c0xYuXAiJRILp06ertuXn5yMwMBA2NjYwMzPDsGHDkJKSUq39oLJtD41H14XH8foPF1RB591ejfD1q60ZdIiIqEZU6tvH0dER5ubmOHLkCPLy8gAAHTt2hJeXV5V27kmhoaFYs2YNWrdurbZ9xowZ+OOPP7Bz506cOnUK9+7dU832TM9XkjwPQbsi1ea6kUiAMT7upS4xQkRE9DxoHXYePHgAPz8/NG3aFC+//DKSkooXZpw4cSI++OCDKu8gAGRnZ2P06NFYt24d6tWrp9oul8uxfv16fPvtt3jppZfQvn17bNy4EefOncP58+erpS9UusJHSnxz+JZa0AEAIYA7aRUv50BERFRdtA47M2bMgIGBAeLj42Fi8v8rQo8cObLa1sUKDAxE//794e/vr7b90qVLKCoqUtvu5eUFNzc3hISElFlfQUEBMjMz1R5UeaF30vHy8r/w66V/SuzjyuFERFTTtB6gfPjwYRw6dAguLi5q25s0aVItt55v27YNly9fRmhoaIl9ycnJMDQ0hJWVldp2BwcHJCcnl1lncHAw5s+fX9VdfeHIc4uw8OBNbL0YDwCwNTOEfzMH7AxLgEJwFmMiIqodtA47OTk5amd0HktPT1dbCb0qJCQkYNq0aThy5AiMjIyqrN6goCDMnDlT9TwzMxOurq5VVr+uE0Jg39UkzP/jb6RlFwAAXuvoijn9vGBlYohp/k04izEREdUaWoed7t27Y8uWLfjyyy8BABKJBEqlEosXL0bv3r2rtHOXLl1CamoqvL29VdsUCgVOnz6NFStW4NChQygsLERGRoba2Z2UlBQ4OjqWWa9MJqvyYPai+OdhLubuuYYTUfcBAA3tTBE8pBU6N7RRHcOVw4mIqDbROuwsXrwYfn5+CAsLQ2FhIT766CNcv34d6enpOHv2bJV2zs/PD5GRkWrbxo8fDy8vL8yePRuurq4wMDDAsWPHMGzYMABAVFQU4uPjVfP/UNV4pFBi07k7+ObwLeQVKWCoJ8W7vRvhnV6NINPXq+nuERERlUnrsNOyZUvcunULK1asgLm5ObKzszF06FAEBgbCyenZV69+krm5OVq2bKm2zdTUFDY2NqrtEydOxMyZM2FtbQ0LCwu899578PHxQZcuXaq0Ly+yyH/kCNp9FdcSiwdyd/KwxoIhrdDY3qyGe0ZERFQxrcNOfHw8XF1d8cknn5S6z83NrUo6pqmlS5dCKpVi2LBhKCgoQJ8+ffD9998/1z7ooiR5Hm4kZeLw9RTsCEuAUgCWxgb4+GUvDG/vCqmU8+YQEVHdIBFCiIoP+396enpISkqCvb292vYHDx7A3t4eCoWiSjv4PGRmZsLS0hJyuZzre6F4FuQ5uyLx5DtjYBtnzH2lOezMOdaJiIhqB02/v7U+syOEKHU23Ozs7Cq9Y4pqRtz9HMz+TX2clFQCBL3sxaBDRER1ksZh5/Gt2hKJBHPnzlW7/VyhUODChQto27ZtlXeQnp+/72Xirc0l5zNS/jsLMu+wIiKiukjjsBMeHg6g+MxOZGQkDA0NVfsMDQ3Rpk0bzJo1q+p7SNVOCIGNZ+9g4YGbKFQoS+znLMhERFSXaRx2Tpw4AaD41u///e9/HNuiI+5nFeDDX6/g5L/z5vg3c4BvIxt8tf8GFEJwFmQiIqrztB6grIte1AHKp27dxwc7riAtuwAyfSk+faU53ujsBolEgiR5HmdBJiKiWq3aBihT3VfwSIGvD0bhhzNxAABPB3MsH9UOno7mqmM4CzIREekKhp0XTExqNt7fGo6/k4onCHzTtwHm9POCkQFnQSYiIt3EsPOCEEJge2gC5v/xN/KKFLA2NcTXr7aGXzOHmu4aERFRtWLYeQFk5BYiaFckDlxLBgB0b2KLb4a3gb0F50UiIiLdx7Cjo5LkeYhLy0FGTiG+3H8DSfJ8GOhJ8GEfT7zVrSGXeyAiohcGw44O2h4aj6BdkVA+cZ+dh60plr/WDq1cLGuuY0RERDWAYUfHJMnzSgQdCYAfxnZAI65STkRELyBpTXeAqlZUUpZa0AEAASA1q6BG+kNERFTTGHZ0SG7hIyw/Hl1iO5d7ICKiFxnDjo7ILXyECZtCcTk+AzJ9KR6PP+ZyD0RE9KLjmB0dkFPwCOM3heJiXDrMZfrYPLETnCyNuNwDERERGHbqvOyCRxi/8SJC7zyEuUwfWyZ2Qju3egDAkENERASGnTotK78Ib24MxaW7D2FupI+fJnZGG1ermu4WERFRrcKwU0dl5hfhzQ0XcTk+AxZG+vjprc5o7WJV090iIiKqdRh26qDM/CKMXX8REQkZsDQ2wM9vdUbL+pwskIiIqDQMO3WMPK8IYzdcxJWEDFiZGOCniQw6RERE5WHYqUPkuUUYs+ECrv4jRz0TA/z8Vhc0d7ao6W4RERHVagw7dURGbiHeWH8B1xIzYW1qiJ/f6oxmTgw6REREFWHYqQMe5hRi9A8X8HdSJmxMDfHLpC7wdDSv6W4RERHVCQw7tViSPA9X/5Hj60NRiEnNhq1ZcdBp6sCgQ0REpCmGnVpqe2i82urlZjJ9bJ3UBU0YdIiIiLTCtbFqoSR5nlrQAYrXvjIzYjYlIiLSFsNOLRSXlqMWdABAKYA7abk10yEiIqI6jGGnFnKxKrmmlZ5Egga2JjXQGyIiorqNYacWCrv7UO25nkSCBUNbcmFPIiKiSuAgkFpGCIEf/ooDAEzp2RA9m9qjga0Jgw4REVElMezUMiG3H+DvpEwYGUjxdo9GqGdqWNNdIiIiqtN4GauWWf/vWZ1X27sw6BAREVUBhp1a5Pb9bBy7mQoAGN/Vo4Z7Q0REpBsYdmqRDWeLz+r4edmjkZ1ZDfeGiIhINzDs1BIPcwrx66V/AAATu/OsDhERUVVh2KklfrkYj/wiJZo7WcCnoU1Nd4eIiEhnMOzUAoWPlNh87g4A4K3uHpBIJDXbISIiIh3CsFML7Lt6D6lZBbA3l+GV1s413R0iIiKdwrBTw4QQWH+meGDyON8GMNTnPwkREVFV4jdrDTt/Ox3X7xVPIvh6J7ea7g4REZHOYdipYevP3AbASQSJiIiqC8NODbp9PxtHb3ASQSIiourEsFODNp69A4CTCBIREVUnhp0akpFbiJ2XEgBwEkEiIqLqxLBTQ36+wEkEiYiIngeGnRpQ+EiJLSF3AAATu3ESQSIiourEsFMD9kfeQ0pm8SSCA9pwEkEiIqLqxLDznAkh8MNfnESQiIjoeeE37XPGSQSJiIieL4adUgQHB6Njx44wNzeHvb09Bg8ejKioKLVj1q5di169esHCwgISiQQZGRka1TvwP90Rv3Q4EpaPxvjRI6qkXiIiIiobw04pTp06hcDAQJw/fx5HjhxBUVERAgICkJOTozomNzcXffv2xccff6xxvQeOHIdo1geObyzB9j37q6xeIiIiKptECCFquhM1LTMzE5aWlpDL5bCwsCix//79+7C3t8epU6fQo0cPtX0nT55E79698fDhQ1hZWZXbztw91/Dj+bvw87LH+jc7Vlm9REREL6KKvr8f45kdDcjlcgCAtbV1pevIyC3Er5f+AfD/kwhWRb1ERERUPoadCiiVSkyfPh1du3ZFy5YtK13PLxfjkVekQLN/JxGsqnqJiIiofPo13YHaLjAwENeuXcOZM2cqXUfhIyU2n7sDAHjr30kE33333Weul4iIiCrGsFOOqVOnYt++fTh9+jRcXFwqXc+fkUlqkwhWVb1ERERUMYadUggh8N5772H37t04efIkPDwqv1CnEAI/nLkNABjr446Z09+vknqJiIhIMww7pQgMDMQvv/yC33//Hebm5khOTgYAWFpawtjYGACQnJyM5ORkxMTEAAAiIyNhbm4ONzc31YBjPz8/tO0egGv5LWFkIEX41iX4bef2Z66XiIiINMdbz1Hy1rWyFubcuHEj3nzzTQDA559/jvnz55d7jKubO8xa+SOv1VCM7uyGBUNbV0m9REREpPmt5ww70PyHpY3tofEI2hUJ5b8/3Q8CmuK9l5pUSd1ERETEeXZqVJI8Ty3oAMCyI9FIkufVXKeIiIheUAw71SAuLUct6ACAQgjcScutmQ4RERG9wGp12NFkQc78/HwEBgbCxsYGZmZmGDZsGFJSUmqox8U8bE0hfWrYj55Egga2JjXTISIiohdYrQ47mizIOWPGDPzxxx/YuXMnTp06hXv37mHo0KE12GvAydIYwUNbQe/fgc56EgkWDG0JJ0vjGu0XERHRi6hODVB+euFMuVwOOzs7/PLLL3j11VcBADdv3kSzZs0QEhKCLl26aFRvdQxQBorH7txJy0UDWxMGHSIioiqm6fd3nZpn5+mFMy9duoSioiL4+/urjvHy8oKbm1u5YaegoAAFBQWq55mZmdXSXydLY4YcIiKiGlarL2M9qbSFM5OTk2FoaAgrKyu1Yx0cHFQT9pUmODgYlpaWqoerq2t1dp2IiIhqUJ0JO48X5Ny2bdsz1xUUFAS5XK56JCQkVEEPiYiIqDaqE5exylo409HREYWFhcjIyFA7u5OSkgJHR8cy65PJZJDJZNXZZSIiIqolavWZHSEEpk6dit27d+P48eMlFs5s3749DAwMcOzYMdW2qKgoxMfHw8fH53l3l4iIiGqhWn1mp6IFOS0tLTFx4kTMnDkT1tbWsLCwwHvvvQcfHx+N78QiIiIi3Varbz3XZEHO/Px8fPDBB9i6dSsKCgrQp08ffP/99+Vexnpadd16TkRERNWHC4FqgWGHiIio7uFCoERERERg2CEiIiIdx7BDREREOo1hh4iIiHQaww4RERHpNIYdIiIi0mkMO0RERKTTGHaIiIhIpzHsEBERkU5j2CEiIiKdxrBDREREOo1hh4iIiHQaww4RERHpNIYdIiIi0mkMO0RERKTTGHaIiIhIpzHsEBERkU5j2CEiIiKdxrBDREREOo1hh4iIiHQaww4RERHpNIYdIiIi0mkMO0RERKTTGHaIiIhIpzHsEBERkU5j2CEiIiKdxrBDREREOo1hh4iIiHQaww4RERHpNIYdIiIi0mkMO0RERKTTGHaIiIhIpzHsEBERkU5j2CEiIiKdxrBDREREOo1hh4iIiHQaww4RERHpNIYdIiIi0mkMO0RERKTTGHaIiIhIpzHsEBERkU5j2CEiIiKdxrBDREREOo1hh4iIiHQaww4RERHpNIYdIiIi0mkMO0RERKTTGHaIiIhIpzHsEBERkU5j2CEiIiKdxrBDREREOo1hh4iIiHQaww4RERHpNIYdIiIi0mkMO0RERKTTGHaIiIhIpzHsEBERkU5j2CEiIiKdxrBDREREOo1hh4iIiHQaww4RERHpNIYdIiIi0mkMO0RERKTTdCbsrFy5Eg0aNICRkRE6d+6Mixcv1nSXiIiIqBbQibCzfft2zJw5E5999hkuX76MNm3aoE+fPkhNTa3prhEREVEN04mw8+2332LSpEkYP348mjdvjtWrV8PExAQbNmyo6a4RERFRDdOv6Q48q8LCQly6dAlBQUGqbVKpFP7+/ggJCSm1TEFBAQoKClTP5XI5ACAzM7N6O0tERERV5vH3thCi3OPqfNhJS0uDQqGAg4OD2nYHBwfcvHmz1DLBwcGYP39+ie2urq7V0kciIiKqPllZWbC0tCxzf50PO5URFBSEmTNnqp4rlUqkp6fDxsYGEomkytrJzMyEq6srEhISYGFh8VzLs+3n3/azlmfbL1bbz1qebbPtulL+WdsujxACWVlZcHZ2Lve4Oh92bG1toaenh5SUFLXtKSkpcHR0LLWMTCaDTCZT22ZlZVVdXYSFhcUz/QM/S3m2/fzbftbybPvFavtZy7Nttl1Xyj9r22Up74zOY3V+gLKhoSHat2+PY8eOqbYplUocO3YMPj4+NdgzIiIiqg3q/JkdAJg5cybGjRuHDh06oFOnTli2bBlycnIwfvz4mu4aERER1TCdCDsjR47E/fv3MW/ePCQnJ6Nt27Y4ePBgiUHLz5tMJsNnn31W4pLZ8yjPtp9/289anm2/WG0/a3m2zbbrSvlnbbsqSERF92sRERER1WF1fswOERERUXkYdoiIiEinMewQERGRTmPYISIiIp3GsFONVq5ciQYNGsDIyAidO3fGxYsXNSp3+vRpDBgwAM7OzpBIJNizZ4/GbQYHB6Njx44wNzeHvb09Bg8ejKioKI3Lr1q1Cq1bt1ZN/uTj44MDBw5oXP5JCxcuhEQiwfTp0zU6/vPPP4dEIlF7eHl5adxeYmIi3njjDdjY2MDY2BitWrVCWFiYRmUbNGhQom2JRILAwMAKyyoUCsydOxceHh4wNjZGo0aN8OWXX1a4VsuTsrKyMH36dLi7u8PY2Bi+vr4IDQ0tcVxF7w0hBObNmwcnJycYGxvD398f0dHRGpfftWsXAgICVLOJR0REaNx+UVERZs+ejVatWsHU1BTOzs4YO3Ys7t27p1Hbn3/+Oby8vGBqaop69erB398fFy5c0LjvT5oyZQokEgmWLVumUdk333yzxL993759tWr7xo0bGDhwICwtLWFqaoqOHTsiPj6+wrKlve8kEgm+/vprjdrOzs7G1KlT4eLiAmNjY9ViyJqUTUlJwZtvvglnZ2eYmJigb9++qveLJp8l+fn5CAwMhI2NDczMzDBs2DDVBK+alF+7di169eoFCwsLSCQSZGRkqPZVVD49PR3vvfcePD09YWxsDDc3N7z//vuQy+Uatf3222+jUaNGMDY2hp2dHQYNGqRaYkibz1EhBPr166f6+WpStlevXiX+vadMmaJV2yEhIXjppZdgamoKCwsL9OjRA1988UW5Ze/cuVPm+23nzp0atZ2cnIwxY8bA0dERpqam8Pb2xm+//aZR2djYWAwZMgR2dnawsLDAiBEjSkwIXF0YdqrJ9u3bMXPmTHz22We4fPky2rRpgz59+iA1NbXCsjk5OWjTpg1WrlypdbunTp1CYGAgzp8/jyNHjqCoqAgBAQHIycnRqLyLiwsWLlyIS5cuISwsDC+99BIGDRqE69eva9WP0NBQrFmzBq1bt9aqXIsWLZCUlKR6nDlzRqNyDx8+RNeuXWFgYIADBw7g77//xjfffIN69epp3N8n2z1y5AgAYPjw4RWWXbRoEVatWoUVK1bgxo0bWLRoERYvXozvvvtOo7YB4K233sKRI0fw448/IjIyEgEBAfD390diYqLacRW9NxYvXozly5dj9erVuHDhAkxNTdGnTx/k5+drVD4nJwfdunXDokWLytxfVvnc3FxcvnwZc+fOxeXLl7Fr1y5ERUVh4MCBGrXdtGlTrFixApGRkThz5gwaNGiAgIAA3L9/X6Pyj+3evRvnz59Xmz5ek7J9+/ZVew9s3bpV4/KxsbHo1q0bvLy8cPLkSVy9ehVz586FkZFRhWWfbDMpKQkbNmyARCLBsGHDNGp75syZOHjwIH766SfcuHED06dPx9SpU7F3795yywohMHjwYNy+fRu///47wsPD4e7uDn9/f+Tk5Gj0WTJjxgz88ccf2LlzJ06dOoV79+5h6NChADT7LMrNzUXfvn3x8ccfl+hfReXv3buHe/fuYcmSJbh27Ro2bdqEgwcPYuLEiRq13b59e2zcuBE3btzAoUOHIIRAQEAAFAqFVp+jy5YtU1tmSNOykyZNUvt3X7x4scblQ0JC0LdvXwQEBODixYsIDQ3F1KlTcebMmXLLurq6lni/zZ8/H2ZmZujXr59GbY8dOxZRUVHYu3cvIiMjMXToUIwYMQJ//PFHuWVzcnIQEBAAiUSC48eP4+zZsygsLMSAAQOgVCpL/FyrnKBq0alTJxEYGKh6rlAohLOzswgODtaqHgBi9+7dle5HamqqACBOnTpV6Trq1asnfvjhB42Pz8rKEk2aNBFHjhwRPXv2FNOmTdOo3GeffSbatGlTqT7Onj1bdOvWrVJlSzNt2jTRqFEjoVQqKzy2f//+YsKECWrbhg4dKkaPHq1RW7m5uUJPT0/s27dPbbu3t7f45JNPyiz39HtDqVQKR0dH8fXXX6u2ZWRkCJlMJrZu3Vph+SfFxcUJACI8PFzj9ktz8eJFAUDcvXtX67JyuVwAEEePHtW47X/++UfUr19fXLt2Tbi7u4ulS5dqVHbcuHFi0KBB5fanvPIjR44Ub7zxRqXKPm3QoEHipZde0rh8ixYtxBdffKG2rbT3ztNlo6KiBABx7do11TaFQiHs7OzEunXrSrT99GdJRkaGMDAwEDt37lQdc+PGDQFAhISEVFj+SSdOnBAAxMOHD0t93RWVf2zHjh3C0NBQFBUVaV32ypUrAoCIiYnRuO3w8HBRv359kZSUVOa/bWlltflcLK18586dxaefflqpsk9r27Ztic+v8sqbmpqKLVu2qB1nbW1d4j3zdNlDhw4JqVQq5HK56piMjAwhkUjEkSNHKnwtz4pndqpBYWEhLl26BH9/f9U2qVQKf39/hISEPNe+yOVyAIC1tbXWZRUKBbZt24acnBytlt4IDAxE//791V6/pqKjo+Hs7IyGDRti9OjRiI+P16jc3r170aFDBwwfPhz29vZo164d1q1bp3X7QPG/308//YQJEyZotDCsr68vjh07hlu3bgEArly5gjNnzqBfv34atffo0SMoFAoYGRmpbTc2Ntb4zBYAxMXFITk5We3nbmlpic6dOz/3991jcrkcEolE67XnCgsLsXbtWlhaWqJNmzYalVEqlRgzZgw+/PBDtGjRQuu+njx5Evb29vD09MQ777yDBw8eaNzu/v370bRpU/Tp0wf29vbo3LmzVpefH0tJScH+/fsxceJEjcv4+vpi7969SExMhBACJ06cwK1btxAQEFBuuYKCAgBQe99JpVLIZLJS33dPf5ZcunQJRUVFau83Ly8vuLm5lfp+e5bPIk3Ly+VyWFhYQF9fv8T28srm5ORg48aN8PDwgKurq0Zt5+bm4vXXX8fKlSvLXIexvLZ//vln2NraomXLlggKCkJubq5G5VNTU3HhwgXY29vD19cXDg4O6Nmzp0b/Zk+7dOkSIiIiyny/lVbe19cX27dvR3p6OpRKJbZt24b8/Hz06tWr3LIFBQWQSCRqEwsaGRlBKpVq9TlXadUep15AiYmJAoA4d+6c2vYPP/xQdOrUSau68AxndhQKhejfv7/o2rWrVuWuXr0qTE1NhZ6enrC0tBT79+/XuOzWrVtFy5YtRV5enhBCu79g/vzzT7Fjxw5x5coVcfDgQeHj4yPc3NxEZmZmhWVlMpmQyWQiKChIXL58WaxZs0YYGRmJTZs2adz3x7Zv3y709PREYmKiRscrFAoxe/ZsIZFIhL6+vpBIJGLBggVatenj4yN69uwpEhMTxaNHj8SPP/4opFKpaNq0aZllnn5vnD17VgAQ9+7dUztu+PDhYsSIERWWf1JVnNnJy8sT3t7e4vXXX9e47B9//CFMTU2FRCIRzs7O4uLFixq3vWDBAvGf//xHdTZOmzM7W7duFb///ru4evWq2L17t2jWrJno2LGjePToUYXlH/9Vb2JiIr799lsRHh4ugoODhUQiESdPntTodT+2aNEiUa9ePdXvjyZ9z8/PF2PHjhUAhL6+vjA0NBSbN2+usGxhYaFwc3MTw4cPF+np6aKgoEAsXLhQABABAQFqZUv7LPn555+FoaFhiXY6duwoPvroowrLP6miMzuafJbdv39fuLm5iY8//ljjsitXrhSmpqYCgPD09Cz1rE5Z5SdPniwmTpyoel7av01ZZdesWSMOHjworl69Kn766SdRv359MWTIEI3aDgkJEQCEtbW12LBhg7h8+bKYPn26MDQ0FLdu3dLodT/2zjvviGbNmpW6r6zyDx8+FAEBAar3m4WFhTh06FCFZVNTU4WFhYWYNm2ayMnJEdnZ2WLq1KkCgJg8eXKZfawqDDvVoLaEnSlTpgh3d3eRkJCgVbmCggIRHR0twsLCxJw5c4Stra24fv16heXi4+OFvb29uHLlimqbNmHnaQ8fPhQWFhYaXUIzMDAQPj4+atvee+890aVLF63bDQgIEK+88orGx2/dulW4uLiIrVu3iqtXr4otW7YIa2trrYJWTEyM6NGjhwAg9PT0RMeOHcXo0aOFl5dXmWVqc9gpLCwUAwYMEO3atVM7bV1R2ezsbBEdHS1CQkLEhAkTRIMGDURKSkqF5cPCwoSDg4NaQNUm7DwtNjZW40toj3/fR40apXbcgAEDxGuvvaZV256enmLq1Kll7i+t/Ndffy2aNm0q9u7dK65cuSK+++47YWZmVuLSQGllw8LCRJs2bVTvuz59+oh+/fqJvn37qh1X2meJNmGnos+iisJOReXlcrno1KmT6Nu3rygsLNS4bEZGhrh165Y4deqUGDBggPD29i4RNEsr//vvv4vGjRuLrKws1bbSfr6afgYfO3as1EtopZV//HseFBSkdmyrVq3EnDlzNG47NzdXWFpaiiVLlpS6v6zyU6dOFZ06dRJHjx4VERER4vPPPxeWlpbi6tWrFZY9dOiQaNiwoZBIJEJPT0+88cYbwtvbW0yZMqWcn07VYNipBgUFBUJPT6/EG3/s2LFi4MCBWtVV2bATGBgoXFxcxO3bt7Uu+zQ/Pz+Nkvfu3btVH5qPHwBUb+zS/kquSIcOHdR+gcvi5uam9leWEEJ8//33wtnZWav27ty5I6RSqdizZ4/GZVxcXMSKFSvUtn355ZfC09NTq7aFKP6yfxxWRowYIV5++eUyj336vfH4C/rpgNKjRw/x/vvvV1j+Sc8SdgoLC8XgwYNF69atRVpamlZln9a4ceNSz5I9XX7p0qWq99mT7z2pVCrc3d0r1batra1YvXp1hW0XFBQIfX198eWXX6od99FHHwlfX1+N2z59+rQAICIiIsrs09Plc3NzhYGBQYnxXhMnThR9+vTRuO2MjAyRmpoqhCgeb/juu++q9pX1WfL4C/rpgOLm5ia+/fbbCss/qbywU1H5zMxM4ePjI/z8/EoEFW0+BwsKCoSJiYn45ZdfKiw/bdq0Mt9vPXv21Lrt7OxsAUAcPHiwwrZv374tAIgff/xRbfuIESNUZ1E1aXvLli3CwMBA9e/+pLLKx8TElBjnJUTxd8Tbb7+tcdv3799X/Vs7ODiIxYsXl3lsVeGYnWpgaGiI9u3b49ixY6ptSqUSx44d02rsS2UIITB16lTs3r0bx48fh4eHxzPXqVQqVdf3y+Pn54fIyEhERESoHh06dMDo0aMREREBPT09rdrNzs5GbGwsnJycKjy2a9euJW5zvHXrFtzd3bVqc+PGjbC3t0f//v01LpObmwupVP1XSU9Pr1J3GJiamsLJyQkPHz7EoUOHMGjQII3Lenh4wNHRUe19l5mZiQsXLlT7++6xoqIijBgxAtHR0Th69ChsbGyeqT5N33tjxozB1atX1d57zs7O+PDDD3Ho0CGt2/3nn3/w4MEDjd57hoaG6Nix4zO//9avX4/27dtrPEYJKP55FxUVPfP7z9LSEnZ2doiOjkZYWBgGDRpU4WdJ+/btYWBgoPZ+i4qKQnx8PHx8fJ75s0iT8pmZmQgICIChoSH27t2rGn9UmbZF8R//KCgoqLD8nDlzSrzfAGDp0qXYsGGD1m0/Lu/k5FRh2w0aNICzs3Op7zc3NzeN216/fj0GDhwIOzs7tZ9BeeUfjysq7f2mUCg0btvW1hZWVlY4fvw4UlNTVXdsVqtqj1MvqG3btgmZTCY2bdok/v77bzF58mRhZWUlkpOTKyyblZUlwsPDRXh4uACgGgfw9B0tpXnnnXeEpaWlOHnypEhKSlI9cnNzNer3nDlzxKlTp0RcXJy4evWqmDNnjpBIJOLw4cMalX+aNpexPvjgA3Hy5EkRFxcnzp49K/z9/YWtrW2pf3k87eLFi0JfX1989dVXIjo6Wvz888/CxMRE/PTTTxr3VaFQCDc3NzF79myNywhRfCdP/fr1xb59+0RcXJzYtWuXsLW1LXEqvzwHDx4UBw4cELdv3xaHDx8Wbdq0EZ07dy5xSr6i98bChQuFlZWVavzJoEGDhIeHh+ov3orKP3jwQISHh4v9+/cLAGLbtm0iPDxcJCUlVVi+sLBQDBw4ULi4uIiIiAi1919BQUG5ZbOzs0VQUJAICQkRd+7cEWFhYWL8+PFCJpOp/orU9vfiyctY5ZXNysoSs2bNEiEhISIuLk4cPXpUeHt7iyZNmoj8/HyN2t61a5cwMDAQa9euFdHR0eK7774Tenp64q+//tKo33K5XJiYmIhVq1aVeB0Vle/Zs6do0aKFOHHihLh9+7bYuHGjMDIyEt9//32FZXfs2CFOnDghYmNjxZ49e4S7u7sYOnSoEEKzz5IpU6YINzc3cfz4cREWFiZ8fHxUl5M1KZ+UlCTCw8PFunXrBABx+vRpER4eLh48eFBheblcLjp37ixatWolYmJi1I6ZMmVKuWVjY2PFggULRFhYmLh79644e/asGDBggLC2thYpKSmV+hzFv2fOKiobExMjvvjiCxEWFibi4uLE77//Lho2bCh69Oih8c9t6dKlwsLCQuzcuVNER0eLTz/9VBgZGYnXX39do35HR0cLiUQiDhw4oLa9orYLCwtF48aNRffu3cWFCxdETEyMWLJkiZBIJOLll1+usO0NGzaIkJAQERMTI3788UdhbW0tZs6cWebPtCox7FSj7777Tri5uQlDQ0PRqVMncf78eY3KPT6l+/Rj3LhxFZYtrRwAsXHjRo3anjBhgnB3dxeGhobCzs5O+Pn5VTroCKFd2Bk5cqRwcnIShoaGon79+mLkyJGlDhgsyx9//CFatmwpZDKZ8PLyEmvXrtWqr4cOHRIARFRUlFblMjMzxbRp04Sbm5swMjISDRs2FJ988okoKCjQuI7t27eLhg0bCkNDQ+Ho6CgCAwNFRkZGieMqem8olUoxd+5c4eDgIGQymfDz81N7PRWV37hxY6n7P/vsswrLP770VdrjxIkT5ZbNy8sTQ4YMEc7OzsLQ0FA4OTmJgQMHqg1Q1vb34smwU17Z3NxcERAQIOzs7ISBgYFwd3cXkyZNUvvDRJO2169fLxo3biyMjIxEmzZtVJdCNSm7Zs0aYWxsXKl/86SkJPHmm28KZ2dnYWRkJDw9PcU333wjlEplhWX/97//CRcXF2FgYCDc3NzEp59+qnrfavJZkpeXJ959911Rr149YWJiIoYMGaIKxpqU/+yzz8o8pqLyZb228h6PyyYmJop+/foJe3t7YWBgIFxcXMTrr78ubt68qXHfn/Y47FRUNj4+XvTo0UNYW1sLmUwmGjduLD788EPV2DZN2w4ODhYuLi7CxMRE+Pj4iL/++kvjskFBQcLV1VUoFIoSr6Gi8rdu3RJDhw4V9vb2wsTERLRu3Vps2bJFo7KzZ88WDg4OwsDAQDRp0kT1Pn0eJP++QCIiIiKdxDE7REREpNMYdoiIiEinMewQERGRTmPYISIiIp3GsENEREQ6jWGHiIiIdBrDDhEREek0hh0ioqecPHkSEokEGRkZNd0VIqoCDDtERESk0xh2iIiISKcx7BBRraNUKhEcHAwPDw8YGxujTZs2+PXXXwH8/yWm/fv3o3Xr1jAyMkKXLl1w7do1tTp+++03tGjRAjKZDA0aNMA333yjtr+goACzZ8+Gq6srZDIZGjdujPXr16sdc+nSJXTo0AEmJibw9fUtsdI0EdUNDDtEVOsEBwdjy5YtWL16Na5fv44ZM2bgjTfewKlTp1THfPjhh/jmm28QGhoKOzs7DBgwAEVFRQCKQ8qIESPw2muvITIyEp9//jnmzp2LTZs2qcqPHTsWW7duxfLly3Hjxg2sWbMGZmZmav345JNP8M033yAsLAz6+vqYMGHCc3n9RFS1uBAoEdUqBQUFsLa2xtGjR+Hj46Pa/tZbbyE3NxeTJ09G7969sW3bNowcORIAkJ6eDhcXF2zatAkjRozA6NGjcf/+fRw+fFhV/qOPPsL+/ftx/fp13Lp1C56enjhy5Aj8/f1L9OHkyZPo3bs3jh49Cj8/PwDAn3/+if79+yMvLw9GRkbV/FMgoqrEMztEVKvExMQgNzcX//nPf2BmZqZ6bNmyBbGxsarjngxC1tbW8PT0xI0bNwAAN27cQNeuXdXq7dq1K6Kjo6FQKBAREQE9PT307Nmz3L60bt1a9f9OTk4AgNTU1Gd+jUT0fOnXdAeIiJ6UnZ0NANi/fz/q16+vtk8mk6kFnsoyNjbW6DgDAwPV/0skEgDF44mIqG7hmR0iqlWaN28OmUyG+Ph4NG7cWO3h6uqqOu78+fOq/3/48CFu3bqFZs2aAQCaNWuGs2fPqtV79uxZNG3aFHp6emjVqhWUSqXaGCAi0l08s0NEtYq5uTlmzZqFGTNmQKlUolu3bpDL5Th79iwsLCzg7u4OAPjiiy9gY2MDBwcHfPLJJ7C1tcXgwYMBAB988AE6duyIL7/8EiNHjkRISAhWrFiB77//HgDQoEEDjBs3DhMmTMDy5cvRpk0b3L17F6mpqRgxYkRNvXQiqiYMO0RU63z55Zews7NDcHAwbt++DSsrK3h7e+Pjjz9WXUZauHAhpk2bhujoaLRt2xZ//PEHDA0NAQDe3t7YsWMH5s2bhy+//BJOTk744osv8Oabb6raWLVqFT7++GO8++67ePDgAdzc3PDxxx/XxMslomrGu7GIqE55fKfUw4cPYWVlVdPdIaI6gGN2iIiISKcx7BAREZFO42UsIiIi0mk8s0NEREQ6jWGHiIiIdBrDDhEREek0hh0iIiLSaQw7REREpNMYdoiIiEinMewQERGRTmPYISIiIp3GsENEREQ67f8AmeGyZWBdfUUAAAAASUVORK5CYII=", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAG2CAYAAACZEEfAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABctklEQVR4nO3dd1gU18IG8HfpHaWzUq1YsSv2QrDFfi2JiRqNxoiJLRZMNIkpqDGaWGKLJSbGlmv3i71FgwUFKwIqiiJFBRbpuHu+Pwh7XWm7sChO3t/z7JPszJw5Z5Zx9t0zZ2ZkQggBIiIiIokyeNUNICIiIqpIDDtEREQkaQw7REREJGkMO0RERCRpDDtEREQkaQw7REREJGkMO0RERCRpDDtEREQkaQw7REREJGkMO0RERCRprzTsnDp1Cr1794ZcLodMJsOuXbs05gshMGfOHLi6usLc3Bz+/v6Ijo7WWCY5ORnDhg2DjY0NqlSpgtGjRyM9Pf0lbgURERFVZq807GRkZMDX1xfLly8vcv6CBQuwZMkSrFy5EufOnYOlpSW6deuG7Oxs9TLDhg3D9evXcfjwYezbtw+nTp3C2LFjX9YmEBERUSUnqywPApXJZNi5cyf69esHIL9XRy6XY+rUqfjkk08AAAqFAs7OztiwYQOGDh2KiIgI1KtXDxcuXEDz5s0BAAcOHEDPnj3x4MEDyOXyV7U5REREVEkYveoGFCcmJgYJCQnw9/dXT7O1tUWrVq0QEhKCoUOHIiQkBFWqVFEHHQDw9/eHgYEBzp07h/79+xe57pycHOTk5Kjfq1QqJCcnw97eHjKZrOI2ioiIiPRGCIGnT59CLpfDwKD4k1WVNuwkJCQAAJydnTWmOzs7q+clJCTAyclJY76RkRHs7OzUyxQlODgYX375pZ5bTERERK/C/fv34ebmVuz8Sht2KlJQUBCmTJmifq9QKODh4YH79+/DxsbmFbaMiIiItJWWlgZ3d3dYW1uXuFylDTsuLi4AgMTERLi6uqqnJyYmonHjxuplkpKSNMo9e/YMycnJ6vJFMTU1hampaaHpNjY2DDtERESvmdKGoFTa++x4e3vDxcUFR48eVU9LS0vDuXPn4OfnBwDw8/NDamoqLl68qF7m2LFjUKlUaNWq1UtvMxEREVU+r7RnJz09Hbdu3VK/j4mJQXh4OOzs7ODh4YFJkybh66+/Rq1ateDt7Y3Zs2dDLperr9iqW7cuunfvjjFjxmDlypXIy8vDhAkTMHToUF6JRURERABecdgJDQ1F586d1e8LxtGMGDECGzZswPTp05GRkYGxY8ciNTUV7dq1w4EDB2BmZqYus2nTJkyYMAFdu3aFgYEBBg4ciCVLlrz0bSEiIqLKqdLcZ+dVSktLg62tLRQKBcfsEBERvSa0/f6utGN2iIiIiPSBYYeIiIgkjWGHiIiIJI1hh4iIiCSNYYeIiIgkjWGHiIiIJI1hh4iIiCSNYYeIiIgkjWGHiIiIJI1hh4iIiCSNYYeIiIgkjWGHiIiIJI1hh4iIiCSNYYeIiIgkjWGHiIiIJI1hh4iIiCSNYYeIiIgkjWGHiIiIJI1hh4iIiCSNYYeIiIgkjWGHiIiIJI1hh4iIiCSNYYeIiIgkjWGHiIiIJI1hh4iIiCSNYYeIiIgkjWGHiIiIJI1hh4iIiCSNYYeIiIgkjWGHiIiIJI1hh4iIiCSNYYeIiIgkjWGHiIiIJI1hh4iIiCSNYYeIiIgkjWGHiIiIJI1hh4iIiCSNYYeIiIgkjWGHiIiIJI1hh4iIiCSNYYeIiIgkjWGHiIiIJI1hh4iIiCSNYYeIiIgkjWGHiIiIJI1hh4iIiCSNYYeIiIgkjWGHiIiIJI1hh4iIiCSNYYeIiIgkjWGHiIiIJI1hh4iIiCSNYYeIiIgkjWGHiIiIJI1hh4iIiCSNYYeIiIgkjWGHiIiIJI1hh4iIiCSNYYeIiIgkjWGHiIiIJI1hh4iIiCSNYYeIiIgkjWGHiIiIJK1Shx2lUonZs2fD29sb5ubmqFGjBr766isIIdTLCCEwZ84cuLq6wtzcHP7+/oiOjn6FrSYiIqLKpFKHnfnz52PFihVYtmwZIiIiMH/+fCxYsABLly5VL7NgwQIsWbIEK1euxLlz52BpaYlu3bohOzv7FbaciIiIKguZeL6bpJJ588034ezsjLVr16qnDRw4EObm5vjtt98ghIBcLsfUqVPxySefAAAUCgWcnZ2xYcMGDB06VKt60tLSYGtrC4VCARsbmwrZFiIiItIvbb+/K3XPTps2bXD06FFERUUBAC5fvozTp0+jR48eAICYmBgkJCTA399fXcbW1hatWrVCSEhIsevNyclBWlqaxouIiIikyehVN6AkM2fORFpaGnx8fGBoaAilUolvvvkGw4YNAwAkJCQAAJydnTXKOTs7q+cVJTg4GF9++WXFNZyIiIgqjUrds7Nt2zZs2rQJv//+Oy5duoRffvkFCxcuxC+//FKu9QYFBUGhUKhf9+/f11OLiYiIqLKp1D0706ZNw8yZM9Vjbxo2bIh79+4hODgYI0aMgIuLCwAgMTERrq6u6nKJiYlo3Lhxses1NTWFqalphbadiIiIKodK3bOTmZkJAwPNJhoaGkKlUgEAvL294eLigqNHj6rnp6Wl4dy5c/Dz83upbSUiIqLKqVL37PTu3RvffPMNPDw8UL9+fYSFhWHRokUYNWoUAEAmk2HSpEn4+uuvUatWLXh7e2P27NmQy+Xo16/fq208ERERVQqVOuwsXboUs2fPxvjx45GUlAS5XI4PPvgAc+bMUS8zffp0ZGRkYOzYsUhNTUW7du1w4MABmJmZvcKWExERUWVRqe+z87LwPjtERESvH0ncZ4eIiIiovBh2iIiISNIYdoiIiEjSGHaIiIhI0hh2iIiISNIYdoiIiEjSGHaIiIhI0hh2iIiISNIYdoiIiEjSGHaIiIhI0hh2iIiISNIYdoiIiEjSGHaIiIhI0hh2iIiISNIYdoiIiEjSGHaIiIhI0hh2iIiISNIYdoiIiEjSGHaIiIhI0hh2iIiISNIYdoiIiEjSGHaIiIhI0hh2iIiISNIYdoiIiEjSGHaIiIhI0hh2iIiISNIYdoiIiEjSGHaIiIhI0hh2iIiISNIYdoiIiEjSGHaIiIhI0hh2iIiISNIYdoiIiEjSGHaIiIhI0hh2iIiISNIYdoiIiEjSGHaIiIhI0hh2iIiISNIYdoiIiEjSGHaIiIhI0hh2iIiISNIYdoiIiEjSGHaIiIhI0hh2iIiISNIYdoiIiEjSGHaIiIhI0hh2iIiISNIYdoiIiEjSGHaIiIhI0hh2iIiISNIYdoiIiEjSGHaIiIhI0hh2iIiISNIYdoiIiEjSGHaIiIhI0hh2iIiISNIYdoiIiEjSGHaIiIhI0nQOO8ePH6+IdhARERFVCJ3DTvfu3VGjRg18/fXXuH//fkW0iYiIiEhvdA47cXFxmDBhAv744w9Ur14d3bp1w7Zt25Cbm1sR7SMiIiIqF53DjoODAyZPnozw8HCcO3cOtWvXxvjx4yGXy/Hxxx/j8uXLFdFOIiIiojIp1wDlpk2bIigoCBMmTEB6ejrWrVuHZs2aoX379rh+/bq+2khERERUZmUKO3l5efjjjz/Qs2dPeHp64uDBg1i2bBkSExNx69YteHp6YtCgQXppYFxcHN555x3Y29vD3NwcDRs2RGhoqHq+EAJz5syBq6srzM3N4e/vj+joaL3UTURERK8/ncPORx99BFdXV3zwwQeoXbs2wsLCEBISgvfffx+Wlpbw8vLCwoULcfPmzXI3LiUlBW3btoWxsTH+/PNP3LhxA99//z2qVq2qXmbBggVYsmQJVq5ciXPnzsHS0hLdunVDdnZ2uesnIiKi15+RrgVu3LiBpUuXYsCAATA1NS1yGQcHB71coj5//ny4u7tj/fr16mne3t7q/xdC4IcffsBnn32Gvn37AgA2btwIZ2dn7Nq1C0OHDi13G4iIiOj1pnPPztGjR/HWW28VG3QAwMjICB07dixXwwBgz549aN68OQYNGgQnJyc0adIEa9asUc+PiYlBQkIC/P391dNsbW3RqlUrhISEFLvenJwcpKWlabyIiIhImnQOO8HBwVi3bl2h6evWrcP8+fP10qgCd+7cwYoVK1CrVi0cPHgQH374IT7++GP88ssvAICEhAQAgLOzs0Y5Z2dn9byiBAcHw9bWVv1yd3fXa7uJiIio8tA57KxatQo+Pj6FptevXx8rV67US6MKqFQqNG3aFN9++y2aNGmCsWPHYsyYMeWuJygoCAqFQv3izRGJiIikS+ewk5CQAFdX10LTHR0dER8fr5dGFXB1dUW9evU0ptWtWxexsbEAABcXFwBAYmKixjKJiYnqeUUxNTWFjY2NxouIiIikSeew4+7ujjNnzhSafubMGcjlcr00qkDbtm0RGRmpMS0qKgqenp4A8gcru7i44OjRo+r5aWlpOHfuHPz8/PTaFiIiIno96Xw11pgxYzBp0iTk5eWhS5cuAPIHLU+fPh1Tp07Va+MmT56MNm3a4Ntvv8XgwYNx/vx5rF69GqtXrwYAyGQyTJo0CV9//TVq1aoFb29vzJ49G3K5HP369dNrW4iIiOj1pHPYmTZtGp48eYLx48ern4dlZmaGGTNmICgoSK+Na9GiBXbu3ImgoCDMnTsX3t7e+OGHHzBs2DD1MtOnT0dGRgbGjh2L1NRUtGvXDgcOHICZmZle20JERESvJ5kQQpSlYHp6OiIiImBubo5atWqVeCl6ZZeWlgZbW1soFAqO3yEiInpNaPv9rXPPTgErKyu0aNGirMWJiIiIXooyhZ3Q0FBs27YNsbGx6lNZBXbs2KGXhhERERHpg85XY23ZsgVt2rRBREQEdu7ciby8PFy/fh3Hjh2Dra1tRbSRiIiIqMx0DjvffvstFi9ejL1798LExAQ//vgjbt68icGDB8PDw6Mi2khERERUZjqHndu3b6NXr14AABMTE2RkZEAmk2Hy5MnqS8KJiIiIKgudw07VqlXx9OlTAEC1atVw7do1AEBqaioyMzP12zoiIiKictJ5gHKHDh1w+PBhNGzYEIMGDcLEiRNx7NgxHD58GF27dq2INhIRERGVmc5hZ9myZcjOzgYAfPrppzA2Nsbff/+NgQMH4rPPPtN7A4mIiIjKQ6ew8+zZM+zbtw/dunUDABgYGGDmzJkV0jAiIiIifdBpzI6RkRHGjRun7tkhIiIiqux0HqDcsmVLhIeHV0BTiIiIiPRP5zE748ePx5QpU3D//n00a9YMlpaWGvMbNWqkt8YRERERlZfODwI1MCjcGSSTySCEgEwmg1Kp1FvjXhY+CJSIiOj1U2EPAo2JiSlXw4iIiIheJp3DjqenZ0W0g4iIiKhC6Bx2Nm7cWOL84cOHl7kxRERERPqm85idqlWrarzPy8tDZmYmTExMYGFhgeTkZL028GXgmB0iIqLXj7bf3zpfep6SkqLxSk9PR2RkJNq1a4fNmzeXq9FERERE+qZz2ClKrVq1MG/ePEycOFEfqyMiIiLSG72EHSD/7soPHz7U1+qIiIiI9ELnAcp79uzReC+EQHx8PJYtW4a2bdvqrWFERERE+qBz2OnXr5/Ge5lMBkdHR3Tp0gXff/+9vtpFREREpBc6hx2VSlUR7SAiIiKqEHobs0NERERUGekcdgYOHIj58+cXmr5gwQIMGjRIL40iIiIi0hedw86pU6fQs2fPQtN79OiBU6dO6aVRRERERPqic9hJT0+HiYlJoenGxsZIS0vTS6OIiIiI9EXnsNOwYUNs3bq10PQtW7agXr16emkUERERkb7ofDXW7NmzMWDAANy+fRtdunQBABw9ehSbN2/G9u3b9d5AIiIiovLQOez07t0bu3btwrfffos//vgD5ubmaNSoEY4cOYKOHTtWRBuJiIiIykznp55LEZ96TkRE9PqpsKeeX7hwAefOnSs0/dy5cwgNDdV1dUREREQVSuewExgYiPv37xeaHhcXh8DAQL00ioiIiEhfdA47N27cQNOmTQtNb9KkCW7cuKGXRhERERHpi85hx9TUFImJiYWmx8fHw8hI5/HORERERBVK57ATEBCAoKAgKBQK9bTU1FTMmjULb7zxhl4bR0RERFReOnfFLFy4EB06dICnpyeaNGkCAAgPD4ezszN+/fVXvTeQiIiIqDx0DjvVqlXDlStXsGnTJly+fBnm5uZ477338NZbb8HY2Lgi2khERERUZmUaZGNpaYmxY8fquy1EREREelfmEcU3btxAbGwscnNzNab36dOn3I0iIiIi0hedw86dO3fQv39/XL16FTKZDAU3YJbJZAAApVKp3xYSERERlYPOV2NNnDgR3t7eSEpKgoWFBa5fv45Tp06hefPmOHHiRAU0kYiIiKjsdO7ZCQkJwbFjx+Dg4AADAwMYGBigXbt2CA4Oxscff4ywsLCKaCcRERFRmejcs6NUKmFtbQ0AcHBwwMOHDwEAnp6eiIyM1G/riIiIiMpJ556dBg0a4PLly/D29karVq2wYMECmJiYYPXq1ahevXpFtJGIiIiozHQOO5999hkyMjIAAHPnzsWbb76J9u3bw97eHlu3btV7A4mIiIjKQyYKLqcqh+TkZFStWlV9RdbrJi0tDba2tlAoFLCxsXnVzSEiIiItaPv9rZcnd9rZ2eljNURERER6p/MAZSIiIqLXCcMOERERSRrDDhEREUmazmHn1KlTePbsWaHpz549w6lTp/TSKCIiIiJ90TnsdO7cGcnJyYWmKxQKdO7cWS+NIiIiItIXncOOEKLIS8yfPHkCS0tLvTSKiIiISF+0vvR8wIABAPKfbj5y5EiYmpqq5ymVSly5cgVt2rTRfwuJiIiIykHrsGNrawsgv2fH2toa5ubm6nkmJiZo3bo1xowZo/8WEhEREZWD1mFn/fr1AAAvLy988sknPGVFRERErwWdx+xMnz5dY8zOvXv38MMPP+DQoUN6bRgRERGRPugcdvr27YuNGzcCAFJTU9GyZUt8//336Nu3L1asWKH3BhIRERGVh85h59KlS2jfvj0A4I8//oCLiwvu3buHjRs3YsmSJXpvIBEREVF56Bx2MjMzYW1tDQA4dOgQBgwYAAMDA7Ru3Rr37t3TewOJiIiIykPnsFOzZk3s2rUL9+/fx8GDBxEQEAAASEpKKvHx6kRERESvgs5hZ86cOfjkk0/g5eWFli1bws/PD0B+L0+TJk303kAiIiKi8tA57PznP/9BbGwsQkNDcfDgQfX0rl27YvHixXpt3IvmzZsHmUyGSZMmqadlZ2cjMDAQ9vb2sLKywsCBA5GYmFih7SAiIqLXR5meeu7i4gJra2scPnwYWVlZAIAWLVrAx8dHr4173oULF7Bq1So0atRIY/rkyZOxd+9ebN++HSdPnsTDhw/Vd3smIiIi0jnsPHnyBF27dkXt2rXRs2dPxMfHAwBGjx6NqVOn6r2BAJCeno5hw4ZhzZo1qFq1qnq6QqHA2rVrsWjRInTp0gXNmjXD+vXr8ffff+Ps2bMV0hYiIiJ6vegcdiZPngxjY2PExsbCwsJCPX3IkCE4cOCAXhtXIDAwEL169YK/v7/G9IsXLyIvL09juo+PDzw8PBASElLs+nJycpCWlqbxIiIiImnS+nERBQ4dOoSDBw/Czc1NY3qtWrUq5NLzLVu24NKlS7hw4UKheQkJCTAxMUGVKlU0pjs7OyMhIaHYdQYHB+PLL7/Ud1OJiIioEtK5ZycjI0OjR6dAcnKyxpPQ9eH+/fuYOHEiNm3aBDMzM72tNygoCAqFQv26f/++3tZNRERElYvOYad9+/bqx0UAgEwmg0qlwoIFC9C5c2e9Nu7ixYtISkpC06ZNYWRkBCMjI5w8eRJLliyBkZERnJ2dkZubi9TUVI1yiYmJcHFxKXa9pqamsLGx0XgRERGRNOl8GmvBggXo2rUrQkNDkZubi+nTp+P69etITk7GmTNn9Nq4rl274urVqxrT3nvvPfj4+GDGjBlwd3eHsbExjh49ioEDBwIAIiMjERsbq77/DxEREf276Rx2GjRogKioKCxbtgzW1tZIT0/HgAEDEBgYCFdXV702ztraGg0aNNCYZmlpCXt7e/X00aNHY8qUKbCzs4ONjQ0++ugj+Pn5oXXr1nptCxEREb2edA47sbGxcHd3x6efflrkPA8PD700TFuLFy+GgYEBBg4ciJycHHTr1g0//fTTS20DERERVV4yIYTQpYChoSHi4+Ph5OSkMf3JkydwcnKCUqnUawNfhrS0NNja2kKhUHD8DhER0WtC2+9vnQcoCyEgk8kKTU9PT9frFVNERERE+qD1aawpU6YAyL/6avbs2RqXnyuVSpw7dw6NGzfWewOJiIiIykPrsBMWFgYgv2fn6tWrMDExUc8zMTGBr68vPvnkE/23kIiIiKgctA47x48fB5B/6fePP/7IsS1ERET0WtD5aqz169dXRDuIiIiIKoTOA5SJiIiIXicMO0RERCRpDDtEREQkaQw7REREJGkMO0RERCRpDDtEREQkaQw7REREJGkMO0RERCRpDDtEREQkaQw7REREJGkMO0RERCRpDDtEREQkaQw7REREJGkMO0RERCRpDDtEREQkaQw7REREJGkMO0RERCRpDDtEREQkaQw7REREJGkMO0RERCRpDDtEREQkaQw7REREJGkMO0RERCRpDDtEREQkaQw7REREJGkMO0RERCRpDDtEREQkaQw7REREJGkMO0RERCRpDDtEREQkaQw7REREJGkMO0RERCRpDDtEREQkaQw7REREJGkMO0RERCRpDDtEREQkaQw7REREJGkMO0RERCRpDDtEREQkaQw7REREJGkMO0RERCRpDDtEREQkaQw7REREJGkMO0RERCRpDDtEREQkaQw7REREJGkMO0RERCRpDDtEREQkaQw7REREJGkMO0RERCRpDDtEREQkaQw7REREJGkMO0RERCRpDDtEREQkaQw7REREJGkMO0RE/yKnTp1C7969IZfLIZPJsGvXLo356enpmDBhAtzc3GBubo569eph5cqVJa5zw4YNkMlkGi8zM7NCy0VERKBPnz6wtbWFpaUlWrRogdjYWH1uHlGRjF51A4iI6OXJyMiAr68vRo0ahQEDBhSaP2XKFBw7dgy//fYbvLy8cOjQIYwfPx5yuRx9+vQpdr02NjaIjIxUv5fJZBrzb9++jXbt2mH06NH48ssvYWNjg+vXrxcZioj0jWGHiOhfpEePHujRo0ex8//++2+MGDECnTp1AgCMHTsWq1atwvnz50sMOzKZDC4uLsXO//TTT9GzZ08sWLBAPa1GjRq6bwBRGVTq01jBwcFo0aIFrK2t4eTkhH79+mn8cgCA7OxsBAYGwt7eHlZWVhg4cCASExNfUYuJiF5vbdq0wZ49exAXFwchBI4fP46oqCgEBASUWC49PR2enp5wd3dH3759cf36dfU8lUqF/fv3o3bt2ujWrRucnJzQqlWrQqfQiCpKpQ47J0+eRGBgIM6ePYvDhw8jLy8PAQEByMjIUC8zefJk7N27F9u3b8fJkyfx8OHDIrtmiYiodEuXLkW9evXg5uYGExMTdO/eHcuXL0eHDh2KLVOnTh2sW7cOu3fvxm+//QaVSoU2bdrgwYMHAICkpCSkp6dj3rx56N69Ow4dOoT+/ftjwIABOHny5MvaNPo3E6+RpKQkAUCcPHlSCCFEamqqMDY2Ftu3b1cvExERIQCIkJAQrderUCgEAKFQKPTeZiKiygqA2Llzp8a07777TtSuXVvs2bNHXL58WSxdulRYWVmJw4cPa73e3NxcUaNGDfHZZ58JIYSIi4sTAMRbb72lsVzv3r3F0KFDy70d9O+l7ff3azVmR6FQAADs7OwAABcvXkReXh78/f3Vy/j4+MDDwwMhISFo3bp1kevJyclBTk6O+n1aWloFtpqI6PWQlZWFWbNmYefOnejVqxcAoFGjRggPD8fChQs1jrUlMTY2RpMmTXDr1i0AgIODA4yMjFCvXj2N5erWrYvTp0/rdyOIilCpT2M9T6VSYdKkSWjbti0aNGgAAEhISICJiQmqVKmisayzszMSEhKKXVdwcDBsbW3VL3d394psOhHRayEvLw95eXkwMND8ajA0NIRKpdJ6PUqlElevXoWrqysAwMTEBC1atCg05jIqKgqenp7lbzhRKV6bnp3AwEBcu3ZNL78CgoKCMGXKFPX7tLQ0Bh4i+ldIT09X97gAQExMDMLDw2FnZwcPDw907NgR06ZNg7m5OTw9PXHy5Els3LgRixYtUpcZPnw4qlWrhuDgYADA3Llz0bp1a9SsWROpqan47rvvcO/ePbz//vvqMtOmTcOQIUPQoUMHdO7cGQcOHMDevXtx4sSJl7bt9C/2kk6rlUtgYKBwc3MTd+7c0Zh+9OhRAUCkpKRoTPfw8BCLFi3Sev0cs0NE/xbHjx8XAAq9RowYIYQQIj4+XowcOVLI5XJhZmYm6tSpI77//nuhUqnU6/Br2170GDBEPEzNFEIIMWnSJOHh4SFMTEyEs7Oz6Nmzp7h06VKhuteuXStq1qwpzMzMhK+vr9i1a9dL2WaSLm2/v2VCCPHqolbJhBD46KOPsHPnTpw4cQK1atXSmK9QKODo6IjNmzdj4MCBAIDIyEj4+PiUOGbnRWlpabC1tYVCoYCNjY3et4OISCq2XohF0I6rUAnAQAYED2iIIS08XnWz6F9K2+/vSn0aKzAwEL///jt2794Na2tr9TgcW1tbmJubw9bWFqNHj8aUKVNgZ2cHGxsbfPTRR/Dz89M66BARkXbiFVnqoAMAKgHM2nENHWo7wtXW/NU2jqgElTrsrFixAgDUd/IssH79eowcORIAsHjxYhgYGGDgwIHIyclBt27d8NNPP73klhIRSZtSJbDixG110FFPFwJ3H2cy7FClVqlPY70sPI1FRFS86w8VmLXjKi4/UBQ5f3hrT3zepz4MDWRFzieqKNp+f782l54TEdHLlZn7DMH/F4E+y87g8gMFrM2M0K+xHIb/ZJqCaLPx7D2M2RiKtOy8V9ZWopIw7BTj1KlT6N27N+RyOWQyWYnPcBk3bhxkMhl++OGHEte5YsUKNGrUCDY2NrCxsYGfnx/+/PNPjWVu376N/v37w9HRETY2Nhg8ePBr9awvfm5E0nAiMgkBi09h1ak7UKoEejV0xdEpHfHD0CY4PbMLNo9pjb+DuuCHIY1hamSAYzeT0G/5Gdx+lP6qm05UCMNOMTIyMuDr64vly5eXuNzOnTtx9uxZyOXyUtfp5uaGefPm4eLFiwgNDUWXLl00HpiXkZGBgIAAyGQyHDt2DGfOnEFubi569+6t0w29XqWMjAzU8KmHj2YHl7gcPzeiyunR0xx8vDkMI9dfwIOULMhtzbB2RHMsH9YUTjZmAABXW3P41bCHq605+jWphj/GtYGrrRnuPMpAv2VncPxm0iveCqIXVPxV8JVfadfpo4jnxwghxIMHD0S1atXEtWvXhKenp1i8eLHOdVetWlX8/PPPQgghDh48KAwMDDTakZqaKmQymU7PpXmVtpy/J7xn7hOeM/YJAGLqgtWFluHnRlT5qFQqseX8PdHoi4PCc8Y+4T1zn5i797pIz87TqnxSWrYY+NMZ4Tljn/CauU8sPx6tcW8eooqg7X122LNTRiqVCu+++y6mTZuG+vXr61xeqVRiy5YtyMjIgJ+fH4D8Z3bJZDKYmpqqlzMzM4OBgcFr8fyYFy9LBYBfz95DvCJL/Z6fG1HlcyspHUNWn8WM/16FIisP9eU22B3YDrPfrAdLU+0u2nW0NsXvY1rj7VYeEAJYcCASH20OQ1ausoJbT1Q6hp0ymj9/PoyMjPDxxx/rVO7q1auwsrKCqakpxo0bh507d6ofjte6dWtYWlpixowZyMzMREZGBj755BMolUrEx8dXxGboVczjjEKXpQoBbDhzF8p/ZvBzoxfFK7Lw9+3HGqGYKl68Igsno5Iwd+919PzxL5yPSYa5sSE+61UXuwPboqGbrc7rNDEywLf9G+Lrfg1gZCDDvivxGLjibzxIyayALSDSHsNOGVy8eBE//vgjNmzYAJlMt0st69Spg/DwcJw7dw4ffvghRowYgRs3bgAAHB0dsX37duzduxdWVlawtbVF9P1ENPRtXOjBfJXRvccZRU5fdeoOuv9wCj9tP/TSPrfU1FQ0bdr0tfjc/s22XohF23nH8Paac2g77xi2XojVumxJg+Hz8vIwY8YMNGzYEJaWlpDL5Rg+fDgePnxY4jqVSiVmz54Nb29vmJubo0aNGvjqq68gnrtDR3p6OiZMmAA3NzeYm5ujXr16WLlypc7b/iptvRCLNvOOYcS6C1h35i5ylSp0ruOIQ5M74P321WFkWL5/N++09sTvY1rD3tIEN+LT0GfZGZy980RPrSfSHb8JyuCvv/5CUlISPDw8YGRkBCMjI9y7dw9Tp06Fl5dXiWVNTExQs2ZNNGvWDMHBwfD19cWPP/6onh8QEIDbt29j1cFLqPbRJtz0GYEbt+7iqXHVCt6q8jlz6zE+35MfPp6PMc08q6KKhTGik9Lx2co/kFjBn1tSUhIeP36MX3/9FXFxcahevXoFbC3pw93HGZj5X8278c7871Vsu3Af6TnPSi1f0kUEmZmZuHTpEmbPno1Lly5hx44diIyMRJ8+fUpc5/z587FixQosW7YMERERmD9/PhYsWIClS5eql5kyZQoOHDiA3377DREREZg0aRImTJiAPXv26PYBvCLnY55gxn+v4vk7rBnIgG/6N4C7nYXe6mnpbYc9H7VDg2o2SM7IxTs/n8PGkLsawZHoZWHYKYN3330XV65cQXh4uPoll8sxbdo0HDx4UKd1qVQq5OTkqN8LIXD2zhN8ezQOMlMrZN27DGWGAiezvSptN//l+6kYuzEUuUoVutd3wanpnbB5TP7jOj7oWAMnp3XG+E41YN+4K1zfWwqnET+i39zfsOfo33r73Ao4ODigSpUqOHbsGJKSkkr9cqsMKqKH4osvvoBMJtN4+fj4aCyTnZ2NwMBA2Nvbw8rKCgMHDnwpl+s/Ts/B4sNR6L3sNF782hMApv/3Chp/eQiDVv6NH49E4+K9FDxTFr6qrkePHvj666/Rv3//QvNsbW1x+PBhDB48GHXq1EHr1q2xbNkyXLx4EbGxxfce/f333+jbty969eoFLy8v/Oc//0FAQADOnz+vscyIESPQqVMneHl5YezYsfD19dVYpjJ69DQHc3Zfw9DVZwvNUwng3hP9H1+qVTHH9g/aoG9jOZ6pBObsvo6Z/72Ke08yXtmpS5421Y1Ujk+V+nERr1J6ejpu3bqlfh8TE4Pw8HDY2dnBw8MD9vb2GssbGxvDxcUFderUUU/r2rUr+vfvjwkTJgAAgoKC0KNHD3h4eODp06f4/fffceLECWzftRd7Lj/E6ehH2LHlN2SYu8DAwhY5D28i5chqWLfoC0O7ajh7+wn6N3V7OR+Alm4lPcXI9eeRkatE25r2+KZ3Tdy/G42CG8fHxMTAK/I6htazw3C/vvjhSBS2hd5HaBpw6cAjpOcBllXsy/S5PR+Q1q9fj7p168LR0REhISGYOHEiJk+erLHeyqqgh2LUqFEYMGCAxrzneyh8fX2RkpKCiRMnok+fPggNDS1xvfXr18eRI0fU742MNP+5T548Gfv378f27dtha2uLCRMmYMCAAThz5oz+Nu45t5LSsfZ0DP576QFynxV9SwAZgGpVzPAgNRsX7qbgwt0ULD4SBWszI/hVt0f72o5oX9MBnvYW6lOhBV9ayRmFw+/zFAoFZDIZqlSpUuwybdq0werVqxEVFYXatWvj8uXLOH36NBYtWqSxzJ49ezBq1CjI5XKcOHECUVFRWLx4sW4fyEuSnvMMa07dwZq/7iCzmMHChjIZvBz016vzPHMTQ/wwpDHqy20w78+b2Bp6H1tD7wPI71H6rFc9jGzjBQMt774cr8hCzOMMeDtYav2Iiuw8JR6n52Dz+fv46cQtiH/hQ0zL8rkB0jk+MewUIzQ0FJ07d1a/nzJlCgBgxIgR2LBhg1bruH37Nh4/fqx+n5SUhOHDhyM+Ph4WVjaw96iJ5uMWYurfAP4OAwCkPLiLjGs/Q5mVDiNbJ9j6DYZ1i34AgMnbLuPPawkY06E6mntW1Xnci749SMnEOz+fR0pmHnzdbLHq3eYIDTld4uc2b2AjvN/eGwsOROLQjURk5DzD0mO3YNbkJj7oWAM2ZsYlfm62trZo1KgRDh48iDfeeEO9TGRkJIKCgpCcnAwvLy98+umnmDx58sv7MMqhR48e6NGjR5HzCnoonrds2TK0bNkSsbGx8PAo/kBtZGQEFxeXIucpFAqsXbsWv//+O7p06QLgf4Hx7NmzenuQrhAC52KSsebUHRx97t4rvm62GNOhOtKy8jB713UohYChTIZvBzTAkBYeuJ+cib+iH+P0rUc4c+sJFFl5OHQjEYdu5P+yc6tqjva1HGAgk2Hz+fyemqAdV2FZ26/IL6/s7GzMmDEDb731Vom3lJ85cybS0tLg4+MDQ0NDKJVKfPPNNxg2bJh6maVLl2Ls2LFwc3ODkZERDAwMsGbNGnTo0EEvn5m+5D5TYcuFWCw5Go3H6bkA8j/3mT3qIjY5A7N2XNP43Cvy2VYymQxjO9SAo7UZJm8NV09XCWDuvhv4Zv8NOFibwsHquZe1CRxfeH86+jG+/b8I9RPXp3Wrg1bV7fH4aQ4ep+ficXrO/15P898/Ss/B0+zCp0VVIn+f+Tc8xLQ8T6qXyvGJYacYnTp10unc8t27dwtNC7kcgZjHGXiYmonUzGdoNWIWctuMxYW7ycjOU+EZgILDv4+LNdrXckC7UT+hpZcd9lyOUx+MDGRAbWdr3Ex4qj7g+7pXwdj21dGtvnO5BxOWxeP0HLy79jwS0rJR08kKG95rCStTI60+t5pO1lg9vDku3ktGsOdOhN5LwfLjt7HpXCwmdK6JkxevIy41C/GKLLjammPt2rWltmfevHmYN2+evjavzMr666mgLKCfHgoAiI6Ohlwuh5mZGfz8/BAcHKw++Fy8eBF5eXnw9/dXL+/j4wMPDw+EhISUO+zkKVX4v6vx+PmvGFyNy3+ekkwG+Nd1xpj21dHC639hvbOPE+4+zoSXg4X6M3O3s8DbrTzwdisPKFUC1+IUOH3rMf6KfoSL91LwICULm8/f16hTCGBWEV9eeXl5GDx4MIQQ6ocLF2fbtm3YtGkTfv/9d9SvXx/h4eGYNGkS5HI5RowYASA/7Jw9exZ79uyBp6cnTp06hcDAQMjlco3PUxvl2V+Ko1IJ7L8aj4WHInHvSf5VUN4OlpjWrQ56NHCBTCaDXw17dKjtWOhzr2jONqZFTlcKIDEtB4lpJe/7z1MJYP6BSK2XNzQAXjwbqhLAyHXnMSWgDvzrOpf4bK9Tp07hu+++w8WLFxEfH4+dO3eiX79+6vk7duzAypUrcfHiRSQnJyMsLAyNGzcusU2dOnXCyZMnC03v2bMn9u/fX2j6uHHjsGrVKixevBiTJk0qdr1CCDxIycLNhKe4EPMEq/+K0djminxSfWU9PjHsVJDN52Mxa8fVQmMSCjhZm6JdLQe0r+WAtjUd4GRtpjF/SAuPQgejW0lP8fNfMdgRFofL91MR+PsluFU1x6i23hjcwh1WWt4Po7zSsvMwYt15xDzOQLUq5vh1dEtUtTTReT3NPO2wfZwfjkQkYf6Bm7iVlI6v90fg6/0RAPK/HGf3qodR7bz1vQkVYu1fd/D1/ggI5J+O6VTHEfXk2j1Y9sbDNJyIfARAPz0UrVq1woYNG1CnTh3Ex8fjyy+/RPv27XHt2jVYW1sjISEBJiYmhQ5Izs7OSEhI0HaT1Qq+tB2tTHEy6hHWnY7BQ0U2AMDM2AD/aeaGUW29Ud3RqlBZV1vzEg+6hgYy+LpXga97FQR2romMnGc4H5OMbRfu48/rmm1VCmDd6RhMDagDM2NDddC5d+8ejh07VuqDfqdNm4aZM2di6NChAICGDRvi3r17CA4OxogRI5CVlYVZs2Zh586d6NWrF+IVWWjaQ47e50OxcOHCYsOOSiWQmpX3T49Dfm/D4RuJ2H8lPn9/kQFf9W2Ad1p7lti+0vx96zHmHbiJK/88sNPByhQT/WthaAt3GL/wo6i0z70ieDtYwkAGjVtUGMiAXePbQiaTqXtinu+ZKXjFK7KL7KFxsDRBtarmcNToGTLR6ClytDJFRm4e2s0/Xuj2GJGJ6fjg14vwsrfA6Hbe+E8zd5ibGBaqp6TTOQXz27Vrh8GDB2PMmDFafR47duxAbm6u+v2TJ0/g6+uLQYMGFVp23W9bcPTUabi4umpMT895hsiENETEP8XNhDTcjH+KyISneFrCIH+lEIhMeKr3v39lPT4BDDsVIl6RhU93Fg46ravbwb+uMzrUdkQtJ6tST0O9eDCq6WSNeQMbYWpAHfx69h5+DbmLBylZmLvvBhYficKwVp4Y2cYLLrZmFfKLEcg/9/3+L6G4/jAN9pYm+HV0y3KtXyaT4Y16zuhcxxFrT8cg+M+b6nniny7u5cdvoUE1W/i4WqOuiw18XK1R3cEKJkaFe7QqartLsyssDl/9E9KA/IG2xyMf4fg/AUYX4p+rkmo6WaGZp516ui49FM93Ozdq1AitWrWCp6cntm3bhtGjRxdavuBzyytiIHBpfj93D5/tulboS8TBygTD/bzwTmtP2JUhDBfH0tQInX2c4ONqjYM3EgrVu+avGOwMi8Pbzavh6PIg3Iu5jePHjxcaZ1eUzMzMQrcrMDQ0VD92JC8vD3l5eTAwMNA4NfDkaiIcVAr8/Ned/C/rF76on6Tn4tmLDX2OEMBnu65h3ekYNHSzRV1XG/i4WKOuqw2crE2LPFY8v6+nZORh3oGbOBWVv79Zmhjig441MLqdt9Y3BXwZXG3NETygYaFTaI3cq5RaNl6Rhbbzjmn8vQ1lwN6P22n1b93WwrhQ3TN7+CAlMxe/nb2Hu08yMXv3dXx/OArvtvbEu36eGj9CSzqdA+RfuAIU3ctfHDs7O433W7ZsgYWFRaGws2L/eXw0fgKcBs3Foz++xJbzsbixMRQ3E54iNrnoexiZGBqgppMVPOwtcPBaQqHvo5k7rmLegIboVMdJ6/aWpCKPT/pQef4VSEhRN9cDgIlda8OvRukH3NI4Wptiyhu18WHHGvjvpQdYezoGMY8zsPLkbfz81x00crNF2P1UvQ/Cy1OqMOH3SzgfkwxrUyP8Mqplkb/Uy8LI0KDYm5g9ycjFyahHOBn1v+BgbChDDUcr9ZeCj6sNohKeIvjPiDKdly6r3GcqLDwUidWn7hQ5v3sD51IPxPGpWThwXfMqAwFg0MoQ9GzoijHtq6Oei6VOPRQvqlKlCmrXrq0edO/i4oLc3FykpqbiYHSa+kv7wa1YNOxgitxnKjzJeG7cw9PnfnGn5/4zRiIHSWnZUBTxazuohw9GtPGCmXHhX8j6Ym2oxAcNDLD0aP42KRWJaGWThttpwCOFBT77aBTyEm9jxBcrcCcxDUpl/uBcOzs7mJjkh68XB8P37t0b33zzDTw8PFC/fn2EhYVh0aJFGDVqFADAwtIKTVu1xchxH0PWdhQMbZyQc/8aMq4dg0mX99W9ksWxMTOCg7UpTAwNcDPhaaH5dx5n4M7jDOwO/9/VLFUtjOHzT8gvCPtXHigwZ3fhgGlsKMOwVp6Y0KUmHKyKPmX0qhXVa62N4oKSLj9qiqs7sHNNbA+9j7VnYnA/OQtLj93CqpN30K+JHO+3r47aztZl2lZdrV27FkOHDoWlpaV6WlxKBqaMHwOblgNg4pjf8xeVmI6EG/87ZrjYmMHH1Ro+Ljao+89/qztaqnvztl6I1RgWYWNmjARFNkauv4DevnLMebMeHK3Lvr/o2oP6opKOT8/37iQmJhY7zqc0DDsVoKiu2oq42sHcxBDvtPbE2y09cPRmEtacuoPzd5NxKTZVvYy+BuGpVAIz/riCIxFJMDUywM8jmqNBNd3vsFqS4rq4V77bDI+f5qq7aCMS0vA0+xluJjwt8gsDKDgvXbGDD+88SsfHW8JwLS6tyPmGMhk+712/9LCjyMKhG4mFvrhUAth3JR57w+7j2eFFMM5IxLnTp7TqoXhReno6bt++rf712axZMxgbG2PTjv1YGF0FQgB5Tx5AmfYIhx/bovZnf5ayxpI1cqtSoUEHyL+IYOY7PdXvk4/9jG3Hfsa7w4ej/ZDxGLvwHABg7dRBeH7U17Fjx9SD6F8cDL906VLMnj0b48ePR1JSEuRyOYa8+x583nwfH/wair9vP0Fq83HIOPkLsvcuhCo7HYY2TqjS/l1YNe4Bvxr28HGxVp86cbA2UZ9KsbcygalR/mdSVC+FgQyYP7ARkp7mICI+DTcTnuLOo3SkZOYh5M4ThJRyUz7/us6Y/WZdeNpblrhcZVDWU2hlDUql1W1paoSRbb3xrp8XDl1PwJq/7uBSbCq2hT7AttAH6FjbEWM7VIe3gwXuPqmYu0GfP38e165d0xijGPM4A33HTAUMDGHdTPM2GiP8PNG9gSt8XKxLHUbw4udmY2aMRYejsP5MDPZefoiTkUmY1bMuBjd31/qquAIFQSc6OlrrHtQXFXd8Onr0KAYOHAgg/yKU2NhY9WOCdMWwUwH08QtEFwYG+aeC3qjnjF9D7mL27usa81UCmL79CsZ0qI62NR1KHIRXFCEEvtp/AzvC4mBoIMPyt5uiVfXy91C9qLjPLaCeZpIXQiAuNQs3/zlHHZHwFGH3UtRjRAooBXDgWgLea6vfMT9CCGwPfYDP91xHVp4SVSyMsWBgI6Rk5pbpb15UD8WHDQ3QrLYH9kZn4ucvPkJOwm04/WcO+i8/jbdbuaNXQzmquTgW20PxySefoHfv3vD09MTDhw/x+eefw9DQEIOHDEX4/VT8FfUI7n5vYtLkybDvORkyUwukHF4JU7kPTKvl3+/CyEAGeyuTEq+QERAYse58hQf7opQ2GP59lQrnY5Kx5q87OBLxvyvBfrhujHS7h+jRwEV9EUHBYHhra2t88e0CvDFqBk7feoS/oh9jX0oW9u2PUpev6uCIrlO+wcnIRxqnBgxlMiwa7KvV37y4fX1Qc3eN5bLzlLiVlK4OPxHxabjyIBXpOYUvIR/dzvu1CDrlVZFjjQwNZOjR0BU9Grri4r1krDkVg4M3Egr1LAPA6ehH6KfHuteuXYuGDRuiZcuWSHqajSVHo7Fh9zHEH94K1xE/apzKNJAB4zrV0OlzePFzm/1mPfRrXA0zd1zB9YdpmLnjKnZcisO3AxqgptP/erJKug2Lq6sr/vOf/+DSpUvYt28flEqlekxNST2oxR2f3nrrLQD5V3mNHj0aU6ZMgZ2dHWxsbPDRRx/Bz8+vzBdPyARvZ4m0tDTY2tpCoVDo3P1WknhF1ku/2qGoX4zPc7Q2RV9fOfo1qYb6chutLl9fcjQaiw7nH+wXD/FF/yYVe6+fsnxuJW23f10nTO/uo5euaEVWHmbtvIr9V/KfudWmhj0WDW4MF1uzMrf9xIkTGpfrFxgxYgS++OILeHsXHdY+XPALvhw3BI7WpvDy8sLIkSPxxRdfAACGDh2KU6dO4cmTJ7Czd4BX/abw7v4+rqebQ5GVBwAQz3KRfGwtMiNOQijzYObdFPZvjIexdVX838ftUdvZWqtfec93kT9/+XhlcvvRP/f4ufgAOf/c46eKuTEUWXnqAcKdajsiOSMXV+IUGncXNjaUoalH1fyrJWs5omE1WxgayPSy3WXZXx6mZhYaaGsok+H0zM6Sv4T6Vbj3JANLj0bjj0tx/5s2/0049v8Uaz//ED0byjV+QN69exfe3t5aXY1VICMjA3K5HLNmfw7jRr2w5q8YZOUpkXZhN1KO/wwDmQFUBTulUEFmYAAPd3edxgcV55lShQ1/38X3h6KQlaeEsaEM4zvVxPjONWBqZFjm49Px48fRqVMnACjx+OTo6Ih27drhm2++QY0aNdTls7OzMXXqVGzevBk5OTno1q0bfvrpp0KnsbT9/mbYQcWFnVdF8yAMjO1YA+nZz7DvykOkZOapl6vlZIV+Taqhb2M53KoW/Uv8+Z6iz3vX03sviT69eF66hZcdQu+lQKnKfz+wqRsmv1Eb8ipl+0K4cDcZk7aEIy41C0YGMkwJqI0POtTQuaesrDJynmFb6H2s+2dcAZD/4MX+javh/fbesDIzQszjDDhYmeLOo3Scin6M09GPCw1gtDYzQpsa9mhXK/8GfWfvPMGnO1/+l/ar8CQ9B7+evYcNZ+4iNSuv2OVqOVmpr5Zs5W1f7CDfV7Xdr0PAlJK/bz/G22vOqd8XhB2L2n5wsTFD38b5PyDrutqUKez8vHYdPhz/IepN+R0Kkf/DqbF7FXzY2gnVTPMvx3/0NBtxqdmYNnowRgwfjvfee0+vN029n5yJObuvqS+qqO5oiW/7N0RrPfTiV+SFIww7OpBa2AGKPgjnPlPhZNQj7AqLw+GIRI272LbytkP/JtXQo6ErbM2NEa/Iwqaz97Ds+G0AwMdda2HKG7Vfybbo4sXtvv0oHQsPRuLPa/ldq6ZGBhjZ1gvjO9aErYWxVut8plRh6bFbWHosGioBeNpb4MehTdBYiytIKsIzpQoHrydizV93EH4/tdTljQzyeyba1XJAu1oOaFTNttC9mV6XsKIvJyKTMHL9hULTx3Wsob6isbL7t/3NXqV4RRb85u5HbnJ+j278ho9Rtcv7qFqzMXIMLWBk4wRl1lO4m2SgqQOwfOb72LJlC+rUqQMXFxd1b8Tw4cNRrVo1BAcHA8gfC7nvajze7dcdz8yqwLHvDFT/555I3f+5J9KLvLy8MGnSpBLvs1NWQuTfo+mLPTfwOD0/ZA1p7o6gnj7IylNqFVhyninx5LkbPP7flQT899IDCFTMhSMMOzqQYtgpTVp2Hg5cTcDOsDicjXmi7rY3MTRAbWcrXH+Yph6P0Ka6HTaNaf3K79hcHpdiUzDvz5s4H5MMIP+qmMDONUu9YuhBSiYmbQlH6L0UAMCAptUwt2+Dl3ZPo5IIIXDxXgqWHovGyajHheYPauaGbvVd0LqGfaVob2VS9GXMPBVExZuzciu++nBooen+fQajxn+mY9e2TUjcV/iRITM//QzBX38FAGjTrgOqOMuxdt163E7KwLwDEQi7cgMPfx6HWiPn4fNxQzG4eeF7Ij2vIsNOAUVm/q0MCu5ObmVqiIwcpfqUb/8m1eBtb6m+OvN/90bKQVoRV2c+T9//zhh2dPBvDDvPe5iahd3hD7Ez7AGiEtMLzTeUAadndnntvwSEEDgemYT5f0YiMjH/Ki65rRkmv1EbA5q6FTodtffyQ8zaeRVPs5/B2tQIX/dvgL6Nq72KppfoxS72ApvHtNbLrQ6kiqeCSFcl9aalZubi/64mYFdYHM7fTVZPNzUywBv1nOFgZYKNIfcKjSu0MjXCBx2qY3R7b1iYVK4fJRfuJmPa9ss6X4FmZCCDg5UpzEwMcPdx4bL6PDYx7Ojg3x52CgghsOX8fQTtvFponpS+OJUqgR2XHmDR4SjE/3MFVx1na8zoUQc+LvmP5fjvxQfYfzX/1FcTjypYMrQJ3O0q/gqjsmAvRdnxVBBVhPvJmdhz+SF2XHqA248yil1ucHM3zOjuA/tKek8kADgZlYQR6wqf8u1U2wH15Lb/XKWZf9fqgis1bc2NYWAgeynHJoYdHTDs/M+/6YszO0+JX/6+i+XHbxXZ9SoD8FGXmvi4a61X8vwxXbCXgqjyEULgWlwafjp+q9CjTYDX40dkeb8TKvrYxLCjA4YdTf+2L05FZh4WHLqJTWdjNaYbyIAzr9HpO/ZSEFVOr/uPyPJ+J1TksYlhRwcMO4X92744Oe6FiCrS6/4jsrJ+J2j7/V25RkNRpfEqnoj8Kr2sR3wQ0b+TPh518Sq97t8JlXsgAtFLUnD7fsN/Lq+v6Ed8ENG/j6utOfxq2PO48gqwZ4foH6/7Ly8iIioaww7Rc173rloiIiqMp7GIiIhI0hh2iIiISNIYdoiIiEjSGHaIiIhI0hh2iIiISNIYdoiIiEjSGHaIiIhI0hh2iIiISNIYdoiIiEjSGHaIiIhI0hh2iIiISNIYdoiIiEjSGHaIiIhI0hh2iIiISNIYdoiIiEjSGHaIiIhI0hh2iIiISNIYdoiIiEjSGHaIiIhI0hh2iIiISNIYdoiIiEjSGHaIiIhI0hh2iIiISNIYdoiIiEjSGHaIiIhI0hh2iIiISNIYdoiIiEjSGHaIiIhI0hh2iIiISNIYdoiIiEjSGHaIiIhI0hh2iIiISNIYdoiIiEjSJBN2li9fDi8vL5iZmaFVq1Y4f/78q24SERERVQKSCDtbt27FlClT8Pnnn+PSpUvw9fVFt27dkJSU9KqbRkRERK+YJMLOokWLMGbMGLz33nuoV68eVq5cCQsLC6xbt+5VN42IiIheMaNX3YDyys3NxcWLFxEUFKSeZmBgAH9/f4SEhBRZJicnBzk5Oer3CoUCAJCWllaxjSUiIiK9KfjeFkKUuNxrH3YeP34MpVIJZ2dnjenOzs64efNmkWWCg4Px5ZdfFpru7u5eIW0kIiKiivP06VPY2toWO/+1DztlERQUhClTpqjfq1QqJCcnw97eHjKZTG/1pKWlwd3dHffv34eNjc1LLc+6X37d5S3Puv9ddZe3POtm3a9L+fLWXRIhBJ4+fQq5XF7icq992HFwcIChoSESExM1picmJsLFxaXIMqampjA1NdWYVqVKlYpqImxsbMr1By5Pedb98usub3nW/e+qu7zlWTfrfl3Kl7fu4pTUo1PgtR+gbGJigmbNmuHo0aPqaSqVCkePHoWfn98rbBkRERFVBq99zw4ATJkyBSNGjEDz5s3RsmVL/PDDD8jIyMB77733qptGREREr5gkws6QIUPw6NEjzJkzBwkJCWjcuDEOHDhQaNDyy2ZqaorPP/+80Cmzl1Gedb/8ustbnnX/u+oub3nWzbpfl/LlrVsfZKK067WIiIiIXmOv/ZgdIiIiopIw7BAREZGkMewQERGRpDHsEBERkaQx7FSg5cuXw8vLC2ZmZmjVqhXOnz+vVblTp06hd+/ekMvlkMlk2LVrl9Z1BgcHo0WLFrC2toaTkxP69euHyMhIrcuvWLECjRo1Ut/8yc/PD3/++afW5Z83b948yGQyTJo0Savlv/jiC8hkMo2Xj4+P1vXFxcXhnXfegb29PczNzdGwYUOEhoZqVdbLy6tQ3TKZDIGBgaWWVSqVmD17Nry9vWFubo4aNWrgq6++KvVZLc97+vQpJk2aBE9PT5ibm6NNmza4cOFCoeVK2zeEEJgzZw5cXV1hbm4Of39/REdHa11+x44dCAgIUN9NPDw8XOv68/LyMGPGDDRs2BCWlpaQy+UYPnw4Hj58qFXdX3zxBXx8fGBpaYmqVavC398f586d07rtzxs3bhxkMhl++OEHrcqOHDmy0N++e/fuOtUdERGBPn36wNbWFpaWlmjRogViY2NLLVvUfieTyfDdd99pVXd6ejomTJgANzc3mJubqx+GrE3ZxMREjBw5EnK5HBYWFujevbt6f9HmWJKdnY3AwEDY29vDysoKAwcOVN/gVZvyq1evRqdOnWBjYwOZTIbU1FT1vNLKJycn46OPPkKdOnVgbm4ODw8PfPzxx1AoFFrV/cEHH6BGjRowNzeHo6Mj+vbtq37EkC7HUSEEevToof58tSnbqVOnQn/vcePG6VR3SEgIunTpAktLS9jY2KBDhw6YO3duiWXv3r1b7P62fft2repOSEjAu+++CxcXF1haWqJp06b473//q1XZ27dvo3///nB0dISNjQ0GDx5c6IbAFYVhp4Js3boVU6ZMweeff45Lly7B19cX3bp1Q1JSUqllMzIy4Ovri+XLl+tc78mTJxEYGIizZ8/i8OHDyMvLQ0BAADIyMrQq7+bmhnnz5uHixYsIDQ1Fly5d0LdvX1y/fl2ndly4cAGrVq1Co0aNdCpXv359xMfHq1+nT5/WqlxKSgratm0LY2Nj/Pnnn7hx4wa+//57VK1aVev2Pl/v4cOHAQCDBg0qtez8+fOxYsUKLFu2DBEREZg/fz4WLFiApUuXalU3ALz//vs4fPgwfv31V1y9ehUBAQHw9/dHXFycxnKl7RsLFizAkiVLsHLlSpw7dw6Wlpbo1q0bsrOztSqfkZGBdu3aYf78+cXOL658ZmYmLl26hNmzZ+PSpUvYsWMHIiMj0adPH63qrl27NpYtW4arV6/i9OnT8PLyQkBAAB49eqRV+QI7d+7E2bNnNW4fr03Z7t27a+wDmzdv1rr87du30a5dO/j4+ODEiRO4cuUKZs+eDTMzs1LLPl9nfHw81q1bB5lMhoEDB2pV95QpU3DgwAH89ttviIiIwKRJkzBhwgTs2bOnxLJCCPTr1w937tzB7t27ERYWBk9PT/j7+yMjI0OrY8nkyZOxd+9ebN++HSdPnsTDhw8xYMAAANodizIzM9G9e3fMmjWrUPtKK//w4UM8fPgQCxcuxLVr17BhwwYcOHAAo0eP1qruZs2aYf369YiIiMDBgwchhEBAQACUSqVOx9EffvhB4zFD2pYdM2aMxt99wYIFWpcPCQlB9+7dERAQgPPnz+PChQuYMGECTp8+XWJZd3f3Qvvbl19+CSsrK/To0UOruocPH47IyEjs2bMHV69exYABAzB48GDs3bu3xLIZGRkICAiATCbDsWPHcObMGeTm5qJ3795QqVSFPle9E1QhWrZsKQIDA9XvlUqlkMvlIjg4WKf1ABA7d+4sczuSkpIEAHHy5Mkyr6Nq1ari559/1nr5p0+filq1aonDhw+Ljh07iokTJ2pV7vPPPxe+vr5lauOMGTNEu3btylS2KBMnThQ1atQQKpWq1GV79eolRo0apTFtwIABYtiwYVrVlZmZKQwNDcW+ffs0pjdt2lR8+umnxZZ7cd9QqVTCxcVFfPfdd+ppqampwtTUVGzevLnU8s+LiYkRAERYWJjW9Rfl/PnzAoC4d++ezmUVCoUAII4cOaJ13Q8ePBDVqlUT165dE56enmLx4sValR0xYoTo27dvie0pqfyQIUPEO++8U6ayL+rbt6/o0qWL1uXr168v5s6dqzGtqH3nxbKRkZECgLh27Zp6mlKpFI6OjmLNmjWF6n7xWJKamiqMjY3F9u3b1ctEREQIACIkJKTU8s87fvy4ACBSUlKK3O7SyhfYtm2bMDExEXl5eTqXvXz5sgAgbt26pXXdYWFholq1aiI+Pr7Yv21RZXU5LhZVvlWrVuKzzz4rU9kXNW7cuNDxq6TylpaWYuPGjRrL2dnZFdpnXix78OBBYWBgIBQKhXqZ1NRUIZPJxOHDh0vdlvJiz04FyM3NxcWLF+Hv76+eZmBgAH9/f4SEhLzUtigUCgCAnZ2dzmWVSiW2bNmCjIwMnR69ERgYiF69emlsv7aio6Mhl8tRvXp1DBs2DLGxsVqV27NnD5o3b45BgwbByckJTZo0wZo1a3SuH8j/+/32228YNWqUVg+GbdOmDY4ePYqoqCgAwOXLl3H69Gn06NFDq/qePXsGpVIJMzMzjenm5uZa92wBQExMDBISEjQ+d1tbW7Rq1eql73cFFAoFZDKZzs+ey83NxerVq2FrawtfX1+tyqhUKrz77ruYNm0a6tevr3NbT5w4AScnJ9SpUwcffvghnjx5onW9+/fvR+3atdGtWzc4OTmhVatWOp1+LpCYmIj9+/dj9OjRWpdp06YN9uzZg7i4OAghcPz4cURFRSEgIKDEcjk5OQCgsd8ZGBjA1NS0yP3uxWPJxYsXkZeXp7G/+fj4wMPDo8j9rTzHIm3LKxQK2NjYwMjIqND0kspmZGRg/fr18Pb2hru7u1Z1Z2Zm4u2338by5cuLfQ5jSXVv2rQJDg4OaNCgAYKCgpCZmalV+aSkJJw7dw5OTk5o06YNnJ2d0bFjR63+Zi+6ePEiwsPDi93fiirfpk0bbN26FcnJyVCpVNiyZQuys7PRqVOnEsvm5ORAJpNp3FjQzMwMBgYGOh3nyqzC49S/UFxcnAAg/v77b43p06ZNEy1bttRpXShHz45SqRS9evUSbdu21anclStXhKWlpTA0NBS2trZi//79WpfdvHmzaNCggcjKyhJC6PYL5v/+7//Etm3bxOXLl8WBAweEn5+f8PDwEGlpaaWWNTU1FaampiIoKEhcunRJrFq1SpiZmYkNGzZo3fYCW7duFYaGhiIuLk6r5ZVKpZgxY4aQyWTCyMhIyGQy8e233+pUp5+fn+jYsaOIi4sTz549E7/++qswMDAQtWvXLrbMi/vGmTNnBADx8OFDjeUGDRokBg8eXGr55+mjZycrK0s0bdpUvP3221qX3bt3r7C0tBQymUzI5XJx/vx5rev+9ttvxRtvvKHujdOlZ2fz5s1i9+7d4sqVK2Lnzp2ibt26okWLFuLZs2elli/4VW9hYSEWLVokwsLCRHBwsJDJZOLEiRNabXeB+fPni6pVq6r//WjT9uzsbDF8+HABQBgZGQkTExPxyy+/lFo2NzdXeHh4iEGDBonk5GSRk5Mj5s2bJwCIgIAAjbJFHUs2bdokTExMCtXTokULMX369FLLP6+0nh1tjmWPHj0SHh4eYtasWVqXXb58ubC0tBQARJ06dYrs1Smu/NixY8Xo0aPV74v62xRXdtWqVeLAgQPiypUr4rfffhPVqlUT/fv316rukJAQAUDY2dmJdevWiUuXLolJkyYJExMTERUVpdV2F/jwww9F3bp1i5xXXPmUlBQREBCg3t9sbGzEwYMHSy2blJQkbGxsxMSJE0VGRoZIT08XEyZMEADE2LFji22jvjDsVIDKEnbGjRsnPD09xf3793Uql5OTI6Kjo0VoaKiYOXOmcHBwENevXy+1XGxsrHBychKXL19WT9Ml7LwoJSVF2NjYaHUKzdjYWPj5+WlM++ijj0Tr1q11rjcgIEC8+eabWi+/efNm4ebmJjZv3iyuXLkiNm7cKOzs7HQKWrdu3RIdOnQQAIShoaFo0aKFGDZsmPDx8Sm2TGUOO7m5uaJ3796iSZMmGt3WpZVNT08X0dHRIiQkRIwaNUp4eXmJxMTEUsuHhoYKZ2dnjYCqS9h50e3bt7U+hVbw7/2tt97SWK53795i6NChOtVdp04dMWHChGLnF1X+u+++E7Vr1xZ79uwRly9fFkuXLhVWVlaFTg0UVTY0NFT4+vqq97tu3bqJHj16iO7du2ssV9SxRJewU9qxqLSwU1p5hUIhWrZsKbp37y5yc3O1LpuamiqioqLEyZMnRe/evUXTpk0LBc2iyu/evVvUrFlTPH36VD2tqM9X22Pw0aNHizyFVlT5gn/nQUFBGss2bNhQzJw5U+u6MzMzha2trVi4cGGR84srP2HCBNGyZUtx5MgRER4eLr744gtha2srrly5UmrZgwcPiurVqwuZTCYMDQ3FO++8I5o2bSrGjRtXwqejHww7FSAnJ0cYGhoW2vGHDx8u+vTpo9O6yhp2AgMDhZubm7hz547OZV/UtWtXrZL3zp071QfNghcA9Y5d1K/k0jRv3lzjH3BxPDw8NH5lCSHETz/9JORyuU713b17VxgYGIhdu3ZpXcbNzU0sW7ZMY9pXX30l6tSpo1PdQuR/2ReElcGDB4uePXsWu+yL+0bBF/SLAaVDhw7i448/LrX888oTdnJzc0W/fv1Eo0aNxOPHj3Uq+6KaNWsW2Uv2YvnFixer97Pn9z0DAwPh6elZprodHBzEypUrS607JydHGBkZia+++kpjuenTp4s2bdpoXfepU6cEABEeHl5sm14sn5mZKYyNjQuN9xo9erTo1q2b1nWnpqaKpKQkIUT+eMPx48er5xV3LCn4gn4xoHh4eIhFixaVWv55JYWd0sqnpaUJPz8/0bVr10JBRZfjYE5OjrCwsBC///57qeUnTpxY7P7WsWNHnetOT08XAMSBAwdKrfvOnTsCgPj11181pg8ePFjdi6pN3Rs3bhTGxsbqv/vziit/69atQuO8hMj/jvjggw+0rvvRo0fqv7Wzs7NYsGBBscvqC8fsVAATExM0a9YMR48eVU9TqVQ4evSoTmNfykIIgQkTJmDnzp04duwYvL29y71OlUqlPr9fkq5du+Lq1asIDw9Xv5o3b45hw4YhPDwchoaGOtWbnp6O27dvw9XVtdRl27ZtW+gyx6ioKHh6eupU5/r16+Hk5IRevXppXSYzMxMGBpr/lAwNDct0hYGlpSVcXV2RkpKCgwcPom/fvlqX9fb2houLi8Z+l5aWhnPnzlX4flcgLy8PgwcPRnR0NI4cOQJ7e/tyrU/bfe/dd9/FlStXNPY9uVyOadOm4eDBgzrX++DBAzx58kSrfc/ExAQtWrQo9/63du1aNGvWTOsxSkD+552Xl1fu/c/W1haOjo6Ijo5GaGgo+vbtW+qxpFmzZjA2NtbY3yIjIxEbGws/P79yH4u0KZ+WloaAgACYmJhgz5496vFHZalb5P/4R05OTqnlZ86cWWh/A4DFixdj3bp1OtddUN7V1bXUur28vCCXy4vc3zw8PLSue+3atejTpw8cHR01PoOSyheMKypqf1MqlVrX7eDggCpVquDYsWNISkpSX7FZoSo8Tv1LbdmyRZiamooNGzaIGzduiLFjx4oqVaqIhISEUss+ffpUhIWFibCwMAFAPQ7gxStaivLhhx8KW1tbceLECREfH69+ZWZmatXumTNnipMnT4qYmBhx5coVMXPmTCGTycShQ4e0Kv8iXU5jTZ06VZw4cULExMSIM2fOCH9/f+Hg4FDkL48XnT9/XhgZGYlvvvlGREdHi02bNgkLCwvx22+/ad1WpVIpPDw8xIwZM7QuI0T+lTzVqlUT+/btEzExMWLHjh3CwcGhUFd+SQ4cOCD+/PNPcefOHXHo0CHh6+srWrVqVahLvrR9Y968eaJKlSrq8Sd9+/YV3t7e6l+8pZV/8uSJCAsLE/v37xcAxJYtW0RYWJiIj48vtXxubq7o06ePcHNzE+Hh4Rr7X05OToll09PTRVBQkAgJCRF3794VoaGh4r333hOmpqbqX5G6/rt4/jRWSWWfPn0qPvnkExESEiJiYmLEkSNHRNOmTUWtWrVEdna2VnXv2LFDGBsbi9WrV4vo6GixdOlSYWhoKP766y+t2q1QKISFhYVYsWJFoe0orXzHjh1F/fr1xfHjx8WdO3fE+vXrhZmZmfjpp59KLbtt2zZx/Phxcfv2bbFr1y7h6ekpBgwYIITQ7lgybtw44eHhIY4dOyZCQ0OFn5+f+nSyNuXj4+NFWFiYWLNmjQAgTp06JcLCwsSTJ09KLa9QKESrVq1Ew4YNxa1btzSWGTduXIllb9++Lb799lsRGhoq7t27J86cOSN69+4t7OzsRGJiYpmOo/in56y0srdu3RJz584VoaGhIiYmRuzevVtUr15ddOjQQevPbfHixcLGxkZs375dREdHi88++0yYmZmJt99+W6t2R0dHC5lMJv7880+N6aXVnZubK2rWrCnat28vzp07J27duiUWLlwoZDKZ6NmzZ6l1r1u3ToSEhIhbt26JX3/9VdjZ2YkpU6YU+5nqE8NOBVq6dKnw8PAQJiYmomXLluLs2bNalSvo0n3xNWLEiFLLFlUOgFi/fr1WdY8aNUp4enoKExMT4ejoKLp27VrmoCOEbmFnyJAhwtXVVZiYmIhq1aqJIUOGFDlgsDh79+4VDRo0EKampsLHx0esXr1ap7YePHhQABCRkZE6lUtLSxMTJ04UHh4ewszMTFSvXl18+umnIicnR+t1bN26VVSvXl2YmJgIFxcXERgYKFJTUwstV9q+oVKpxOzZs4Wzs7MwNTUVXbt21die0sqvX7++yPmff/55qeULTn0V9Tp+/HiJZbOyskT//v2FXC4XJiYmwtXVVfTp00djgLKu/y6eDzsllc3MzBQBAQHC0dFRGBsbC09PTzFmzBiNHyba1L127VpRs2ZNYWZmJnx9fdWnQrUpu2rVKmFubl6mv3l8fLwYOXKkkMvlwszMTNSpU0d8//33QqVSlVr2xx9/FG5ubsLY2Fh4eHiIzz77TL3fanMsycrKEuPHjxdVq1YVFhYWon///upgrE35zz//vNhlSitf3LaV9CooGxcXJ3r06CGcnJyEsbGxcHNzE2+//ba4efOm1m1/UUHYKa1sbGys6NChg7CzsxOmpqaiZs2aYtq0aeqxbdrWHRwcLNzc3ISFhYXw8/MTf/31l9Zlg4KChLu7u1AqlYW2obTyUVFRYsCAAcLJyUlYWFiIRo0aiY0bN2pVdsaMGcLZ2VkYGxuLWrVqqffTl0H2zwYSERERSRLH7BAREZGkMewQERGRpDHsEBERkaQx7BAREZGkMewQERGRpDHsEBERkaQx7BAREZGkMewQEb3gxIkTkMlkSE1NfdVNISI9YNghIiIiSWPYISIiIklj2CGiSkelUiE4OBje3t4wNzeHr68v/vjjDwD/O8W0f/9+NGrUCGZmZmjdujWuXbumsY7//ve/qF+/PkxNTeHl5YXvv/9eY35OTg5mzJgBd3d3mJqaombNmli7dq3GMhcvXkTz5s1hYWGBNm3aFHrSNBG9Hhh2iKjSCQ4OxsaNG7Fy5Upcv34dkydPxjvvvIOTJ0+ql5k2bRq+//57XLhwAY6Ojujduzfy8vIA5IeUwYMHY+jQobh69Sq++OILzJ49Gxs2bFCXHz58ODZv3owlS5YgIiICq1atgpWVlUY7Pv30U3z//fcIDQ2FkZERRo0a9VK2n4j0iw8CJaJKJScnB3Z2djhy5Aj8/PzU099//31kZmZi7Nix6Ny5M7Zs2YIhQ4YAAJKTk+Hm5oYNGzZg8ODBGDZsGB49eoRDhw6py0+fPh379+/H9evXERUVhTp16uDw4cPw9/cv1IYTJ06gc+fOOHLkCLp27QoA+L//+z/06tULWVlZMDMzq+BPgYj0iT07RFSp3Lp1C5mZmXjjjTdgZWWlfm3cuBG3b99WL/d8ELKzs0OdOnUQEREBAIiIiEDbtm011tu2bVtER0dDqVQiPDwchoaG6NixY4ltadSokfr/XV1dAQBJSUnl3kYiermMXnUDiIiel56eDgDYv38/qlWrpjHP1NRUI/CUlbm5uVbLGRsbq/9fJpMByB9PRESvF/bsEFGlUq9ePZiamiI2NhY1a9bUeLm7u6uXO3v2rPr/U1JSEBUVhbp16wIA6tatizNnzmis98yZM6hduzYMDQ3RsGFDqFQqjTFARCRd7NkhokrF2toan3zyCSZPngyVSoV27dpBoVDgzJkzsLGxgaenJwBg7ty5sLe3h7OzMz799FM4ODigX79+AICpU6eiRYsW+OqrrzBkyBCEhIRg2bJl+OmnnwAAXl5eGDFiBEaNGoUlS5bA19cX9+7dQ1JSEgYPHvyqNp2IKgjDDhFVOl999RUcHR0RHByMO3fuoEqVKmjatClmzZqlPo00b948TJw4EdHR0WjcuDH27t0LExMTAEDTpk2xbds2zJkzB1999RVcXV0xd+5cjBw5Ul3HihUrMGvWLIwfPx5PnjyBh4cHZs2a9So2l4gqGK/GIqLXSsGVUikpKahSpcqrbg4RvQY4ZoeIiIgkjWGHiIiIJI2nsYiIiEjS2LNDREREksawQ0RERJLGsENERESSxrBDREREksawQ0RERJLGsENERESSxrBDREREksawQ0RERJLGsENERESS9v/SZo2QKB7azgAAAABJRU5ErkJggg==", "text/plain": [ "
" ] diff --git a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN1.py b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN1.py new file mode 100644 index 00000000..e557f437 --- /dev/null +++ b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN1.py @@ -0,0 +1,138 @@ +import torch.nn as nn +import sinabs.layers as sl +from sinabs.exodus.layers import IAFSqueeze + +class SCNN(nn.Module): + def __init__(self, input_size, nb_classes, batch_size, surrogate_fn, min_v_mem=-0.313, spk_thr=2.0) -> None: + super().__init__() + + self.conv1 = nn.Conv2d(2, 8, 2, 1, bias=False) + self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool1 = sl.SumPool2d(2,2) + self.pool1a = sl.SumPool2d(4,4) + + self.conv2 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool2 = sl.SumPool2d(2,2) + + self.conv3 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool3 = sl.SumPool2d(2,2) + + self.conv4 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool4 = sl.SumPool2d(2,2) + + self.flat = nn.Flatten() + + flat_s = SCNN.get_flatten_size(input_size) + + self.fc1 = nn.Linear(flat_s, 100, bias=False) + self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc2 = nn.Linear(100, 100, bias=False) + self.iaf5 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc3 = nn.Linear(100, 100, bias=False) + self.iaf6 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc4 = nn.Linear(100, 64, bias=False) + self.iaf7 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc5 = nn.Linear(64, nb_classes, bias=False) + self.iaf8 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + # skip + + self.merge1 = sl.Merge() + + @staticmethod + def get_flatten_size(input_size): + conv1_dims = SCNN.conv2d_output_size(input_size, 8, (2, 2)) + pool1_dims = SCNN.pool_output_size(conv1_dims, 8, (2, 2)) + + conv2_dims = SCNN.conv2d_output_size(pool1_dims, 8, (2, 2)) + pool2_dims = SCNN.pool_output_size(conv2_dims, 8, (2, 2)) + + conv3_dims = SCNN.conv2d_output_size(pool2_dims, 8, (2, 2)) + pool3_dims = SCNN.pool_output_size(conv3_dims, 8, (2, 2)) + + conv4_dims = SCNN.conv2d_output_size(pool3_dims, 8, (2, 2)) + pool4_dims = SCNN.pool_output_size(conv4_dims, 8, (2, 2)) + + return pool4_dims[0]*pool4_dims[1]*pool4_dims[2] + + @staticmethod + def conv2d_output_size(input_size, out_channels, kernel_size, stride=1, padding=0, dilation=1): + input_height, input_width, input_channels = input_size + kernel_height, kernel_width = kernel_size + + output_height = ((input_height + 2 * padding - dilation * (kernel_height - 1) - 1) // stride) + 1 + output_width = ((input_width + 2 * padding - dilation * (kernel_width - 1) - 1) // stride) + 1 + + return (output_height, output_width, out_channels) + + @staticmethod + def pool_output_size(input_size, out_channels, kernel_size, stride=None, padding=0): + input_height, input_width, input_channels = input_size + kernel_height, kernel_width = kernel_size + + if stride is None: + stride = kernel_height + + output_height = ((input_height + 2 * padding - kernel_height) // stride) + 1 + output_width = ((input_width + 2 * padding - kernel_width) // stride) + 1 + + return (output_height, output_width, out_channels) + + def detach_neuron_states(self): + for name, layer in self.named_modules(): + if name != '': + if isinstance(layer, sl.StatefulLayer): + for name, buffer in layer.named_buffers(): + buffer.detach_() + + def init_weights(self): + for name, layer in self.named_modules(): + if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear): + nn.init.xavier_normal_(layer.weight.data) + + def forward(self, x): + + con1_out = self.conv1(x) + iaf1_out = self.iaf1(con1_out) + pool1_out = self.pool1(iaf1_out) + pool1a_out = self.pool1a(iaf1_out) + + conv2_out = self.conv2(pool1_out) + iaf2_out = self.iaf2(conv2_out) + pool2_out = self.pool2(iaf2_out) + + merge1_out = self.merge1(pool1a_out, pool2_out) + + conv3_out = self.conv3(merge1_out) + iaf3_out = self.iaf3(conv3_out) + pool3_out = self.pool3(iaf3_out) + + conv4_out = self.conv4(pool3_out) + iaf4_out = self.iaf4(conv4_out) + pool4_out = self.pool4(iaf4_out) + + flat_out = self.flat(pool4_out) + + fc1_out = self.fc1(flat_out) + iaf4_out = self.iaf4(fc1_out) + + fc2_out = self.fc2(iaf4_out) + iaf5_out = self.iaf5(fc2_out) + + fc3_out = self.fc3(iaf5_out) + iaf6_out = self.iaf6(fc3_out) + + fc4_out = self.fc4(iaf6_out) + iaf7_out = self.iaf7(fc4_out) + + fc5_out = self.fc5(iaf7_out) + iaf8_out = self.iaf8(fc5_out) + + return iaf8_out \ No newline at end of file diff --git a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN2.py b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN2.py new file mode 100644 index 00000000..ada4e6be --- /dev/null +++ b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN2.py @@ -0,0 +1,143 @@ +import torch.nn as nn +import sinabs.layers as sl +from sinabs.exodus.layers import IAFSqueeze + +class SCNN(nn.Module): + def __init__(self, input_size, nb_classes, batch_size, surrogate_fn, min_v_mem=-0.313, spk_thr=2.0) -> None: + super().__init__() + + self.conv1 = nn.Conv2d(2, 8, 2, 1, bias=False) + self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool1 = sl.SumPool2d(2,2) + self.pool1a = sl.SumPool2d(4,4) + + self.conv2 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool2 = sl.SumPool2d(2,2) + + self.conv3 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool3 = sl.SumPool2d(2,2) + self.pool3a = sl.SumPool2d(4,4) + + self.conv4 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool4 = sl.SumPool2d(2,2) + + self.flat = nn.Flatten() + + flat_s = SCNN.get_flatten_size(input_size) + + self.fc1 = nn.Linear(flat_s, 100, bias=False) + self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc2 = nn.Linear(100, 100, bias=False) + self.iaf5 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc3 = nn.Linear(100, 100, bias=False) + self.iaf6 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc4 = nn.Linear(100, 64, bias=False) + self.iaf7 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc5 = nn.Linear(64, nb_classes, bias=False) + self.iaf8 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + # skip + + self.merge1 = sl.Merge() + self.merge2 = sl.Merge() + + @staticmethod + def get_flatten_size(input_size): + conv1_dims = SCNN.conv2d_output_size(input_size, 8, (2, 2)) + pool1_dims = SCNN.pool_output_size(conv1_dims, 8, (2, 2)) + + conv2_dims = SCNN.conv2d_output_size(pool1_dims, 8, (2, 2)) + pool2_dims = SCNN.pool_output_size(conv2_dims, 8, (2, 2)) + + conv3_dims = SCNN.conv2d_output_size(pool2_dims, 8, (2, 2)) + pool3_dims = SCNN.pool_output_size(conv3_dims, 8, (2, 2)) + + conv4_dims = SCNN.conv2d_output_size(pool3_dims, 8, (2, 2)) + pool4_dims = SCNN.pool_output_size(conv4_dims, 8, (2, 2)) + + return pool4_dims[0]*pool4_dims[1]*pool4_dims[2] + + @staticmethod + def conv2d_output_size(input_size, out_channels, kernel_size, stride=1, padding=0, dilation=1): + input_height, input_width, input_channels = input_size + kernel_height, kernel_width = kernel_size + + output_height = ((input_height + 2 * padding - dilation * (kernel_height - 1) - 1) // stride) + 1 + output_width = ((input_width + 2 * padding - dilation * (kernel_width - 1) - 1) // stride) + 1 + + return (output_height, output_width, out_channels) + + @staticmethod + def pool_output_size(input_size, out_channels, kernel_size, stride=None, padding=0): + input_height, input_width, input_channels = input_size + kernel_height, kernel_width = kernel_size + + if stride is None: + stride = kernel_height + + output_height = ((input_height + 2 * padding - kernel_height) // stride) + 1 + output_width = ((input_width + 2 * padding - kernel_width) // stride) + 1 + + return (output_height, output_width, out_channels) + + def detach_neuron_states(self): + for name, layer in self.named_modules(): + if name != '': + if isinstance(layer, sl.StatefulLayer): + for name, buffer in layer.named_buffers(): + buffer.detach_() + + def init_weights(self): + for name, layer in self.named_modules(): + if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear): + nn.init.xavier_normal_(layer.weight.data) + + def forward(self, x): + + con1_out = self.conv1(x) + iaf1_out = self.iaf1(con1_out) + pool1_out = self.pool1(iaf1_out) + pool1a_out = self.pool1a(iaf1_out) + + conv2_out = self.conv2(pool1_out) + iaf2_out = self.iaf2(conv2_out) + pool2_out = self.pool2(iaf2_out) + + merge1_out = self.merge1(pool1a_out, pool2_out) + + conv3_out = self.conv3(merge1_out) + iaf3_out = self.iaf3(conv3_out) + pool3_out = self.pool3(iaf3_out) + pool3a_out = self.pool3a(iaf3_out) + + conv4_out = self.conv4(pool3_out) + iaf4_out = self.iaf4(conv4_out) + pool4_out = self.pool4(iaf4_out) + + merge2_out = self.merge2(pool3a_out, pool4_out) + + flat_out = self.flat(merge2_out) + + fc1_out = self.fc1(flat_out) + iaf4_out = self.iaf4(fc1_out) + + fc2_out = self.fc2(iaf4_out) + iaf5_out = self.iaf5(fc2_out) + + fc3_out = self.fc3(iaf5_out) + iaf6_out = self.iaf6(fc3_out) + + fc4_out = self.fc4(iaf6_out) + iaf7_out = self.iaf7(fc4_out) + + fc5_out = self.fc5(iaf7_out) + iaf8_out = self.iaf8(fc5_out) + + return iaf8_out \ No newline at end of file diff --git a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN3.py b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN3.py new file mode 100644 index 00000000..f861131c --- /dev/null +++ b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN3.py @@ -0,0 +1,146 @@ +import torch.nn as nn +import sinabs.layers as sl +from sinabs.exodus.layers import IAFSqueeze + +class SCNN(nn.Module): + def __init__(self, input_size, nb_classes, batch_size, surrogate_fn, min_v_mem=-0.313, spk_thr=2.0) -> None: + super().__init__() + + self.conv1 = nn.Conv2d(2, 8, 2, 1, bias=False) + self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool1 = sl.SumPool2d(2,2) + self.pool1a = sl.SumPool2d(4,4) + + self.conv2 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool2 = sl.SumPool2d(2,2) + + self.conv3 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool3 = sl.SumPool2d(2,2) + self.pool3a = sl.SumPool2d(4,4) + + self.conv4 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool4 = sl.SumPool2d(2,2) + + self.flat = nn.Flatten() + + flat_s = SCNN.get_flatten_size(input_size) + + self.fc1 = nn.Linear(flat_s, 100, bias=False) + self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc2 = nn.Linear(100, 100, bias=False) + self.iaf5 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc3 = nn.Linear(100, 100, bias=False) + self.iaf6 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc4 = nn.Linear(100, 64, bias=False) + self.iaf7 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc5 = nn.Linear(64, nb_classes, bias=False) + self.iaf8 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + # skip + + self.merge1 = sl.Merge() + self.merge2 = sl.Merge() + self.merge3 = sl.Merge() + + @staticmethod + def get_flatten_size(input_size): + conv1_dims = SCNN.conv2d_output_size(input_size, 8, (2, 2)) + pool1_dims = SCNN.pool_output_size(conv1_dims, 8, (2, 2)) + + conv2_dims = SCNN.conv2d_output_size(pool1_dims, 8, (2, 2)) + pool2_dims = SCNN.pool_output_size(conv2_dims, 8, (2, 2)) + + conv3_dims = SCNN.conv2d_output_size(pool2_dims, 8, (2, 2)) + pool3_dims = SCNN.pool_output_size(conv3_dims, 8, (2, 2)) + + conv4_dims = SCNN.conv2d_output_size(pool3_dims, 8, (2, 2)) + pool4_dims = SCNN.pool_output_size(conv4_dims, 8, (2, 2)) + + return pool4_dims[0]*pool4_dims[1]*pool4_dims[2] + + @staticmethod + def conv2d_output_size(input_size, out_channels, kernel_size, stride=1, padding=0, dilation=1): + input_height, input_width, input_channels = input_size + kernel_height, kernel_width = kernel_size + + output_height = ((input_height + 2 * padding - dilation * (kernel_height - 1) - 1) // stride) + 1 + output_width = ((input_width + 2 * padding - dilation * (kernel_width - 1) - 1) // stride) + 1 + + return (output_height, output_width, out_channels) + + @staticmethod + def pool_output_size(input_size, out_channels, kernel_size, stride=None, padding=0): + input_height, input_width, input_channels = input_size + kernel_height, kernel_width = kernel_size + + if stride is None: + stride = kernel_height + + output_height = ((input_height + 2 * padding - kernel_height) // stride) + 1 + output_width = ((input_width + 2 * padding - kernel_width) // stride) + 1 + + return (output_height, output_width, out_channels) + + def detach_neuron_states(self): + for name, layer in self.named_modules(): + if name != '': + if isinstance(layer, sl.StatefulLayer): + for name, buffer in layer.named_buffers(): + buffer.detach_() + + def init_weights(self): + for name, layer in self.named_modules(): + if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear): + nn.init.xavier_normal_(layer.weight.data) + + def forward(self, x): + + con1_out = self.conv1(x) + iaf1_out = self.iaf1(con1_out) + pool1_out = self.pool1(iaf1_out) + pool1a_out = self.pool1a(iaf1_out) + + conv2_out = self.conv2(pool1_out) + iaf2_out = self.iaf2(conv2_out) + pool2_out = self.pool2(iaf2_out) + + merge1_out = self.merge1(pool1a_out, pool2_out) + + conv3_out = self.conv3(merge1_out) + iaf3_out = self.iaf3(conv3_out) + pool3_out = self.pool3(iaf3_out) + pool3a_out = self.pool3a(iaf3_out) + + conv4_out = self.conv4(pool3_out) + iaf4_out = self.iaf4(conv4_out) + pool4_out = self.pool4(iaf4_out) + + merge2_out = self.merge2(pool3a_out, pool4_out) + + flat_out = self.flat(merge2_out) + + fc1_out = self.fc1(flat_out) + iaf4_out = self.iaf4(fc1_out) + + fc2_out = self.fc2(iaf4_out) + iaf5_out = self.iaf5(fc2_out) + + merge3_out = self.merge3(iaf4_out, iaf5_out) + + fc3_out = self.fc3(merge3_out) + iaf6_out = self.iaf6(fc3_out) + + fc4_out = self.fc4(iaf6_out) + iaf7_out = self.iaf7(fc4_out) + + fc5_out = self.fc5(iaf7_out) + iaf8_out = self.iaf8(fc5_out) + + return iaf8_out \ No newline at end of file diff --git a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN4.py b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN4.py new file mode 100644 index 00000000..620b146c --- /dev/null +++ b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN4.py @@ -0,0 +1,149 @@ +import torch.nn as nn +import sinabs.layers as sl +from sinabs.exodus.layers import IAFSqueeze + +class SCNN(nn.Module): + def __init__(self, input_size, nb_classes, batch_size, surrogate_fn, min_v_mem=-0.313, spk_thr=2.0) -> None: + super().__init__() + + self.conv1 = nn.Conv2d(2, 8, 2, 1, bias=False) + self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool1 = sl.SumPool2d(2,2) + self.pool1a = sl.SumPool2d(4,4) + + self.conv2 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool2 = sl.SumPool2d(2,2) + + self.conv3 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool3 = sl.SumPool2d(2,2) + self.pool3a = sl.SumPool2d(4,4) + + self.conv4 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool4 = sl.SumPool2d(2,2) + + self.flat = nn.Flatten() + + flat_s = SCNN.get_flatten_size(input_size) + + self.fc1 = nn.Linear(flat_s, 100, bias=False) + self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc2 = nn.Linear(100, 100, bias=False) + self.iaf5 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc3 = nn.Linear(100, 100, bias=False) + self.iaf6 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc4 = nn.Linear(100, 64, bias=False) + self.iaf7 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc5 = nn.Linear(64, nb_classes, bias=False) + self.iaf8 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + # skip + + self.merge1 = sl.Merge() + self.merge2 = sl.Merge() + self.merge3 = sl.Merge() + self.merge4 = sl.Merge() + + @staticmethod + def get_flatten_size(input_size): + conv1_dims = SCNN.conv2d_output_size(input_size, 8, (2, 2)) + pool1_dims = SCNN.pool_output_size(conv1_dims, 8, (2, 2)) + + conv2_dims = SCNN.conv2d_output_size(pool1_dims, 8, (2, 2)) + pool2_dims = SCNN.pool_output_size(conv2_dims, 8, (2, 2)) + + conv3_dims = SCNN.conv2d_output_size(pool2_dims, 8, (2, 2)) + pool3_dims = SCNN.pool_output_size(conv3_dims, 8, (2, 2)) + + conv4_dims = SCNN.conv2d_output_size(pool3_dims, 8, (2, 2)) + pool4_dims = SCNN.pool_output_size(conv4_dims, 8, (2, 2)) + + return pool4_dims[0]*pool4_dims[1]*pool4_dims[2] + + @staticmethod + def conv2d_output_size(input_size, out_channels, kernel_size, stride=1, padding=0, dilation=1): + input_height, input_width, input_channels = input_size + kernel_height, kernel_width = kernel_size + + output_height = ((input_height + 2 * padding - dilation * (kernel_height - 1) - 1) // stride) + 1 + output_width = ((input_width + 2 * padding - dilation * (kernel_width - 1) - 1) // stride) + 1 + + return (output_height, output_width, out_channels) + + @staticmethod + def pool_output_size(input_size, out_channels, kernel_size, stride=None, padding=0): + input_height, input_width, input_channels = input_size + kernel_height, kernel_width = kernel_size + + if stride is None: + stride = kernel_height + + output_height = ((input_height + 2 * padding - kernel_height) // stride) + 1 + output_width = ((input_width + 2 * padding - kernel_width) // stride) + 1 + + return (output_height, output_width, out_channels) + + def detach_neuron_states(self): + for name, layer in self.named_modules(): + if name != '': + if isinstance(layer, sl.StatefulLayer): + for name, buffer in layer.named_buffers(): + buffer.detach_() + + def init_weights(self): + for name, layer in self.named_modules(): + if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear): + nn.init.xavier_normal_(layer.weight.data) + + def forward(self, x): + + con1_out = self.conv1(x) + iaf1_out = self.iaf1(con1_out) + pool1_out = self.pool1(iaf1_out) + pool1a_out = self.pool1a(iaf1_out) + + conv2_out = self.conv2(pool1_out) + iaf2_out = self.iaf2(conv2_out) + pool2_out = self.pool2(iaf2_out) + + merge1_out = self.merge1(pool1a_out, pool2_out) + + conv3_out = self.conv3(merge1_out) + iaf3_out = self.iaf3(conv3_out) + pool3_out = self.pool3(iaf3_out) + pool3a_out = self.pool3a(iaf3_out) + + conv4_out = self.conv4(pool3_out) + iaf4_out = self.iaf4(conv4_out) + pool4_out = self.pool4(iaf4_out) + + merge2_out = self.merge2(pool3a_out, pool4_out) + + flat_out = self.flat(merge2_out) + + fc1_out = self.fc1(flat_out) + iaf4_out = self.iaf4(fc1_out) + + fc2_out = self.fc2(iaf4_out) + iaf5_out = self.iaf5(fc2_out) + + merge3_out = self.merge3(iaf4_out, iaf5_out) + + fc3_out = self.fc3(merge3_out) + iaf6_out = self.iaf6(fc3_out) + + fc4_out = self.fc4(iaf6_out) + iaf7_out = self.iaf7(fc4_out) + + merge4_out = self.merge4(iaf6_out, iaf7_out) + + fc5_out = self.fc5(merge4_out) + iaf8_out = self.iaf8(fc5_out) + + return iaf8_out \ No newline at end of file From 9303796926b5633952639e4eab81e06bf263db7b Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Wed, 1 May 2024 17:51:31 +0200 Subject: [PATCH 067/379] trying different architectures --- .../ARCHITECTURES_SEARCH/Res-SCNN3.ipynb | 1310 ++++++++++++++--- 1 file changed, 1126 insertions(+), 184 deletions(-) diff --git a/tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/Res-SCNN3.ipynb b/tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/Res-SCNN3.ipynb index e531875b..64926de1 100644 --- a/tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/Res-SCNN3.ipynb +++ b/tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/Res-SCNN3.ipynb @@ -8,16 +8,12 @@ "source": [ "import torch, random, sys\n", "\n", - "from tqdm.notebook import tqdm\n", - "\n", "import tonic\n", - "from tonic.datasets.dvsgesture import DVSGesture\n", - "from tonic.transforms import ToFrame\n", "from torch.utils.data import DataLoader\n", "from torch.nn import CrossEntropyLoss\n", "from torch.optim import Adam\n", "\n", - "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential, MultiGaussian\n", + "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential\n", "\n", "import matplotlib.pyplot as plt\n", "import numpy as np\n", @@ -25,17 +21,34 @@ "sys.path.append('../../utils')\n", "sys.path.append('../models')\n", "\n", - "from SCNN import SCNN\n", - "# from CSNN3 import CSNN3\n", - "from train_test_fn import training_loop\n", - "\n", - "from torch.utils.data import Subset" + "from train_test_fn import training_loop, load_dataset, split_train_validation, load_architecture" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "device: NVIDIA GeForce RTX 3070 Ti\n" + ] + } + ], + "source": [ + "if torch.cuda.is_available():\n", + " device = torch.device('cuda:0')\n", + " print('device: ', torch.cuda.get_device_name(0))\n", + "else:\n", + " device = torch.device('cpu')" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, "outputs": [], "source": [ "rand_seed = 1" @@ -43,7 +56,16 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "achitecture = 'ResSCNN3'" + ] + }, + { + "cell_type": "code", + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -55,16 +77,9 @@ "np.random.seed(rand_seed)" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Loading Data" - ] - }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -79,31 +94,24 @@ "grad_scale = 1.534\n", "grad_width = 0.759\n", "\n", - "validation_ratio = 0.2" + "validation_ratio = 0.2\n", + "n_time_steps = 50" ] }, { - "cell_type": "code", - "execution_count": 5, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "root_dir = \"../../DVSGESTURE\"\n", - "_ = DVSGesture(save_to=root_dir, train=True)\n", - "_ = DVSGesture(save_to=root_dir, train=False)" + "## Loading Data" ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ - "n_time_steps = 50\n", - "to_raster = ToFrame(sensor_size=DVSGesture.sensor_size, n_time_bins=n_time_steps)\n", - "\n", - "snn_train_dataset = DVSGesture(save_to=root_dir, train=True, transform=to_raster)\n", - "snn_test_dataset = DVSGesture(save_to=root_dir, train=False, transform=to_raster)" + "snn_train_dataset, snn_test_dataset, sensor_size = load_dataset('DVSGESTURE', n_time_steps)" ] }, { @@ -115,34 +123,16 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 8, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "validation samples: 215, training samples: 862, test samples: 264\n" - ] - } - ], + "outputs": [], "source": [ - "num_samples = len(snn_train_dataset)\n", - "num_validation_samples = int(validation_ratio * num_samples)\n", - "print(f'validation samples: {num_validation_samples}, training samples: {num_samples-num_validation_samples}, test samples: {len(snn_test_dataset)}')\n", - "\n", - "np.random.seed(rand_seed)\n", - "\n", - "validation_indices = np.random.choice(np.arange(num_samples), size=num_validation_samples, replace=False)\n", - "training_indices = np.array(list(filter(lambda x: x not in validation_indices, np.arange(num_samples))))\n", - "\n", - "train_dataset = Subset(snn_train_dataset, training_indices)\n", - "validation_dataset = Subset(snn_train_dataset, validation_indices)" + "train_dataset, validation_dataset = split_train_validation(validation_ratio, snn_train_dataset, rand_seed)" ] }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 9, "metadata": {}, "outputs": [ { @@ -167,7 +157,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ @@ -175,26 +165,18 @@ " dataset=train_dataset,\n", " cache_path='./cached_train'\n", ")\n", + "snn_train_dataloader = DataLoader(disk_cache_train, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True)\n", "\n", "disk_cache_validation = tonic.DiskCachedDataset(\n", " dataset=validation_dataset,\n", " cache_path='./cached_validation'\n", ")\n", + "snn_validation_dataloader = DataLoader(disk_cache_validation, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True)\n", "\n", "disk_cache_test = tonic.DiskCachedDataset(\n", " dataset=snn_test_dataset,\n", " cache_path='./cached_test'\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "snn_train_dataloader = DataLoader(disk_cache_train, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True)\n", - "snn_validation_dataloader = DataLoader(disk_cache_validation, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True)\n", + ")\n", "snn_test_dataloader = DataLoader(disk_cache_test, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=False)" ] }, @@ -202,53 +184,22 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Network Module\n", - "\n", - "We need to define a `nn.Module` implementing the network we want the chip to reproduce." + "## Network Module" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "device: NVIDIA GeForce RTX 3070 Ti\n" - ] - } - ], - "source": [ - "if torch.cuda.is_available():\n", - " device = torch.device('cuda:0')\n", - " print('device: ', torch.cuda.get_device_name(0))\n", - "else:\n", - " device = torch.device('cpu')" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "snn = SCNN(DVSGesture.sensor_size, 11, batch_size, PeriodicExponential(grad_scale=grad_scale, grad_width=grad_width), v_min, spk_thr).to(device)" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, "outputs": [], "source": [ + "snn = load_architecture(achitecture, sensor_size, 11, batch_size, PeriodicExponential(grad_scale=grad_scale, grad_width=grad_width), v_min, spk_thr).to(device)\n", "snn.init_weights()" ] }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 12, "metadata": {}, "outputs": [], "source": [ @@ -260,25 +211,53 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Define train and test" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Training loop (HPO)" + "## Training loop" ] }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 13, "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "ee0385e2a65240a7b5ca8de458950f9d", + "model_id": "5bf192a7696645b9a33b40575b296ed8", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/107 [00:00 1\u001b[0m epochs_x, epochs_y, epochs_acc \u001b[38;5;241m=\u001b[39m \u001b[43mtraining_loop\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 2\u001b[0m \u001b[43m \u001b[49m\u001b[43mdevice\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[1;32m 3\u001b[0m \u001b[43m \u001b[49m\u001b[43mn_time_steps\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 4\u001b[0m \u001b[43m \u001b[49m\u001b[43mbatch_size\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 5\u001b[0m \u001b[43m \u001b[49m\u001b[43mDVSGesture\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msensor_size\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 6\u001b[0m \u001b[43m \u001b[49m\u001b[43msnn_train_dataloader\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[1;32m 7\u001b[0m \u001b[43m \u001b[49m\u001b[43msnn\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[1;32m 8\u001b[0m \u001b[43m \u001b[49m\u001b[43mloss_fn\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[1;32m 9\u001b[0m \u001b[43m \u001b[49m\u001b[43moptimizer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[1;32m 10\u001b[0m \u001b[43m \u001b[49m\u001b[43mepochs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[1;32m 11\u001b[0m \u001b[43m \u001b[49m\u001b[43msnn_validation_dataloader\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/Documents/github/sinabs/tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/../../utils/train_test_fn.py:33\u001b[0m, in \u001b[0;36mtraining_loop\u001b[0;34m(device, nb_time_steps, batch_size, feature_map_size, dataloader_train, model, loss_fn, optimizer, epochs, dataloader_test)\u001b[0m\n\u001b[1;32m 31\u001b[0m \u001b[38;5;66;03m# gradient update\u001b[39;00m\n\u001b[1;32m 32\u001b[0m optimizer\u001b[38;5;241m.\u001b[39mzero_grad()\n\u001b[0;32m---> 33\u001b[0m \u001b[43mloss\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbackward\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 34\u001b[0m optimizer\u001b[38;5;241m.\u001b[39mstep()\n\u001b[1;32m 36\u001b[0m \u001b[38;5;66;03m# detach the neuron states and activations from current computation graph(necessary)\u001b[39;00m\n", - "File \u001b[0;32m~/.pyenv/versions/3.10.0/lib/python3.10/site-packages/torch/_tensor.py:525\u001b[0m, in \u001b[0;36mTensor.backward\u001b[0;34m(self, gradient, retain_graph, create_graph, inputs)\u001b[0m\n\u001b[1;32m 515\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m has_torch_function_unary(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m 516\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m handle_torch_function(\n\u001b[1;32m 517\u001b[0m Tensor\u001b[38;5;241m.\u001b[39mbackward,\n\u001b[1;32m 518\u001b[0m (\u001b[38;5;28mself\u001b[39m,),\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 523\u001b[0m inputs\u001b[38;5;241m=\u001b[39minputs,\n\u001b[1;32m 524\u001b[0m )\n\u001b[0;32m--> 525\u001b[0m \u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mautograd\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbackward\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 526\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mgradient\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mretain_graph\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcreate_graph\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minputs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43minputs\u001b[49m\n\u001b[1;32m 527\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/.pyenv/versions/3.10.0/lib/python3.10/site-packages/torch/autograd/__init__.py:267\u001b[0m, in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[0m\n\u001b[1;32m 262\u001b[0m retain_graph \u001b[38;5;241m=\u001b[39m create_graph\n\u001b[1;32m 264\u001b[0m \u001b[38;5;66;03m# The reason we repeat the same comment below is that\u001b[39;00m\n\u001b[1;32m 265\u001b[0m \u001b[38;5;66;03m# some Python versions print out the first line of a multi-line function\u001b[39;00m\n\u001b[1;32m 266\u001b[0m \u001b[38;5;66;03m# calls in the traceback and some print out the last line\u001b[39;00m\n\u001b[0;32m--> 267\u001b[0m \u001b[43m_engine_run_backward\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 268\u001b[0m \u001b[43m \u001b[49m\u001b[43mtensors\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 269\u001b[0m \u001b[43m \u001b[49m\u001b[43mgrad_tensors_\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 270\u001b[0m \u001b[43m \u001b[49m\u001b[43mretain_graph\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 271\u001b[0m \u001b[43m \u001b[49m\u001b[43mcreate_graph\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 272\u001b[0m \u001b[43m \u001b[49m\u001b[43minputs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 273\u001b[0m \u001b[43m \u001b[49m\u001b[43mallow_unreachable\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 274\u001b[0m \u001b[43m \u001b[49m\u001b[43maccumulate_grad\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 275\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/.pyenv/versions/3.10.0/lib/python3.10/site-packages/torch/autograd/graph.py:744\u001b[0m, in \u001b[0;36m_engine_run_backward\u001b[0;34m(t_outputs, *args, **kwargs)\u001b[0m\n\u001b[1;32m 742\u001b[0m unregister_hooks \u001b[38;5;241m=\u001b[39m _register_logging_hooks_on_whole_graph(t_outputs)\n\u001b[1;32m 743\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 744\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mVariable\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_execution_engine\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun_backward\u001b[49m\u001b[43m(\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;66;43;03m# Calls into the C++ engine to run the backward pass\u001b[39;49;00m\n\u001b[1;32m 745\u001b[0m \u001b[43m \u001b[49m\u001b[43mt_outputs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\n\u001b[1;32m 746\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;66;03m# Calls into the C++ engine to run the backward pass\u001b[39;00m\n\u001b[1;32m 747\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[1;32m 748\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m attach_logging_hooks:\n", - "\u001b[0;31mKeyboardInterrupt\u001b[0m: " + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "480bad2eec294f51bc53bc3b1efa5fd3", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/26 [00:00" + " 0%| | 0/26 [00:00" + " 0%| | 0/107 [00:00" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "y_avg = []\n", + "for y in epochs_y:\n", + " y_avg.append(np.mean(y))\n", + "\n", + "plt.plot(np.arange(len(epochs_x)), y_avg, marker = '.')\n", + "plt.xlabel('epoch')\n", + "plt.ylabel('average loss')\n", + "plt.ylim(0,)\n", + "plt.xticks(np.arange(len(epochs_x)))\n", + "for i, txt in enumerate(y_avg):\n", + " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAG2CAYAAACZEEfAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABl+0lEQVR4nO3deVhU1eMG8PcOOyiDyC6CuOGGGC6ImqYQaub+yzRTU9MsNfeFTK3sG2qaZpZauZZ7uVtuuKWhAoq7iIhisgnIIDvMnN8fxuTINoMgOL6f55mnuPeee84M48zLvWeRhBACRERERHpKVtkNICIiIqpIDDtERESk1xh2iIiISK8x7BAREZFeY9ghIiIivcawQ0RERHqNYYeIiIj0GsMOERER6TWGHSIiItJrDDtERESk1yo17Jw8eRI9e/aEk5MTJEnCrl27NPYLITBnzhw4OjrCzMwMfn5+iIyM1DgmJSUFgwcPhqWlJaysrDBy5Eikp6c/x2dBREREVVmlhp2MjAx4enri+++/L3L/woULsWzZMqxcuRJnz56FhYUFunbtiuzsbPUxgwcPxtWrV3H48GHs27cPJ0+exOjRo5/XUyAiIqIqTqoqC4FKkoSdO3eiT58+AB5f1XFycsKUKVMwdepUAIBCoYC9vT3WrVuHgQMH4vr162jSpAlCQkLQqlUrAMCBAwfwxhtv4J9//oGTk1NlPR0iIiKqIgwruwHFiY6ORnx8PPz8/NTb5HI5vL29ERwcjIEDByI4OBhWVlbqoAMAfn5+kMlkOHv2LPr27VvkuXNycpCTk6P+WaVSISUlBTVr1oQkSRX3pIiIiKjcCCHw6NEjODk5QSYr/mZVlQ078fHxAAB7e3uN7fb29up98fHxsLOz09hvaGgIa2tr9TFFCQwMxOeff17OLSYiIqLKcO/ePTg7Oxe7v8qGnYoUEBCAyZMnq39WKBRwcXHBvXv3YGlpWYktIyIiIm2lpaWhdu3aqF69eonHVdmw4+DgAABISEiAo6OjentCQgJatGihPiYxMVGjXH5+PlJSUtTli2JiYgITE5NC2y0tLRl2iIiIXjCldUGpsvPsuLm5wcHBAUFBQeptaWlpOHv2LHx8fAAAPj4+SE1NRVhYmPqYo0ePQqVSwdvb+7m3mYiIiKqeSr2yk56ejlu3bql/jo6ORnh4OKytreHi4oKJEyfiyy+/RIMGDeDm5obZs2fDyclJPWKrcePG6NatG0aNGoWVK1ciLy8P48aNw8CBAzkSi4iIiABUctgJDQ1F586d1T8X9KMZNmwY1q1bh+nTpyMjIwOjR49GamoqOnTogAMHDsDU1FRdZuPGjRg3bhx8fX0hk8nQv39/LFu27Lk/FyIiIqqaqsw8O5UpLS0NcrkcCoWCfXaIiIheENp+f1fZPjtERERE5YFhh4iIiPQaww4RERHpNYYdIiIi0msMO0RERKTXGHaIiIhIrzHsEBERkV5j2CEiIiK9xrBDREREeo1hh4iIiPQaww4RERHpNYYdIiIi0msMO0RERKTXGHaIiIhIrzHsEBERkV5j2CEiIiK9xrBDREREeo1hh4iIiPQaww4RERHpNYYdIiIi0msMO0RERKTXGHaIiIhIrzHsEBERkV5j2CEiIiK9xrBDREREeo1hh4iIiPQaww4RERHpNYYdIiIi0msMO0RERKTXGHaIiIhIrzHsEBERkV5j2CEiIiK9xrBDREREeo1hh4iIiPQaww4RERHpNYYdIiIi0msMO0RERKTXGHaIiIhIrzHsEBERkV5j2CEiIiK9xrBDREREeo1hh4iIiPQaww4RERHpNYYdIiIi0msMO0RERKTXGHaIiIhIrzHsEBERkV5j2CEiIiK9xrBDREREeo1hh4iIiPQaww4RERHpNYYdIiIi0msMO0RERKTXGHaIiIhIrzHsEBERkV5j2CEiIiK9xrBDREREeo1hh4iIiPQaww4RERHpNYYdIiIi0msMO0RERKTXGHaIiIhIrzHsEBERkV6r0mFHqVRi9uzZcHNzg5mZGerVq4d58+ZBCKE+RgiBOXPmwNHREWZmZvDz80NkZGQltpqIiIiqkioddhYsWIAVK1Zg+fLluH79OhYsWICFCxfiu+++Ux+zcOFCLFu2DCtXrsTZs2dhYWGBrl27Ijs7uxJbTkRERFWFJJ68TFLFvPnmm7C3t8fq1avV2/r37w8zMzP8+uuvEELAyckJU6ZMwdSpUwEACoUC9vb2WLduHQYOHKhVPWlpaZDL5VAoFLC0tKyQ50JERETlS9vv7yp9Zaddu3YICgrCzZs3AQAXL17EqVOn0L17dwBAdHQ04uPj4efnpy4jl8vh7e2N4ODgYs+bk5ODtLQ0jQcRERHpJ8PKbkBJZs6cibS0NDRq1AgGBgZQKpX43//+h8GDBwMA4uPjAQD29vYa5ezt7dX7ihIYGIjPP/+84hpOREREVUaVvrKzbds2bNy4EZs2bcL58+exfv16LFq0COvXr3+m8wYEBEChUKgf9+7dK6cWExERUVVTpa/sTJs2DTNnzlT3vfHw8MDdu3cRGBiIYcOGwcHBAQCQkJAAR0dHdbmEhAS0aNGi2POamJjAxMSkQttOREREVUOVvrKTmZkJmUyziQYGBlCpVAAANzc3ODg4ICgoSL0/LS0NZ8+ehY+Pz3NtKxEREVVNVfrKTs+ePfG///0PLi4uaNq0KS5cuIBvvvkGI0aMAABIkoSJEyfiyy+/RIMGDeDm5obZs2fDyckJffr0qdzGExERUZVQpcPOd999h9mzZ+Ojjz5CYmIinJyc8MEHH2DOnDnqY6ZPn46MjAyMHj0aqamp6NChAw4cOABTU9NKbDkRERFVFVV6np3nhfPsEBERvXj0Yp4dIiIiomfFsENERER6jWGHiIiI9BrDDhEREek1hh0iIiLSaww7REREpNcYdoiIiEivMewQERGRXmPYISIiIr3GsENERER6jWGHiIiI9BrDDhEREek1hh0iIiLSaww7REREpNcYdoiIiEivMewQERGRXmPYISIiIr3GsENERER6jWGHiIiI9BrDDhEREek1hh0iIiLSaww7REREpNcYdoiIiEivMewQERGRXmPYISIiIr3GsENERER6jWGHiIiI9BrDDhEREek1hh0iIiLSaww7REREpNcYdoiIiEivMewQERGRXmPYISIiIr3GsENERER6jWGHiIiI9BrDDhEREek1hh0iIiLSaww7REREpNcYdoiIiEivMewQERGRXmPYISIiIr3GsENERER6jWGHiIiI9BrDDhEREek1hh0iIiLSaww7REREpNcYdoiIiEivMewQERGRXmPYISIiIr3GsENERER6jWGHiIiI9BrDDhEREek1hh0iIiLSaww7REREpNcYdoiIiEivMewQERGRXmPYISIiIr3GsENERER6Teewc+zYsYpoBxEREVGF0DnsdOvWDfXq1cOXX36Je/fuVUSbiIiIiMqNzmHn/v37GDduHH777TfUrVsXXbt2xbZt25Cbm1sR7SMiIiJ6JjqHHRsbG0yaNAnh4eE4e/YsGjZsiI8++ghOTk74+OOPcfHixYpoJxEREVGZPFMHZS8vLwQEBGDcuHFIT0/HmjVr0LJlS7z66qu4evVqebWRiIiIqMzKFHby8vLw22+/4Y033oCrqysOHjyI5cuXIyEhAbdu3YKrqyveeuutcmng/fv38e6776JmzZowMzODh4cHQkND1fuFEJgzZw4cHR1hZmYGPz8/REZGlkvdRERE9OLTOeyMHz8ejo6O+OCDD9CwYUNcuHABwcHBeP/992FhYYE6depg0aJFuHHjxjM37uHDh2jfvj2MjIzw559/4tq1a1i8eDFq1KihPmbhwoVYtmwZVq5cibNnz8LCwgJdu3ZFdnb2M9dPRERELz5DXQtcu3YN3333Hfr16wcTE5Mij7GxsSmXIeoLFixA7dq1sXbtWvU2Nzc39f8LIbB06VJ8+umn6N27NwBgw4YNsLe3x65duzBw4MBnbgMRERG92HS+shMUFIRBgwYVG3QAwNDQEJ06dXqmhgHAnj170KpVK7z11luws7PDK6+8gp9++km9Pzo6GvHx8fDz81Nvk8vl8Pb2RnBwcLHnzcnJQVpamsaDiIiI9JPOYScwMBBr1qwptH3NmjVYsGBBuTSqwO3bt7FixQo0aNAABw8exIcffoiPP/4Y69evBwDEx8cDAOzt7TXK2dvbq/cVJTAwEHK5XP2oXbt2ubabiIiIqg6dw86qVavQqFGjQtubNm2KlStXlkujCqhUKnh5eeGrr77CK6+8gtGjR2PUqFHPXE9AQAAUCoX6wckRiYiI9JfOYSc+Ph6Ojo6Fttva2iIuLq5cGlXA0dERTZo00djWuHFjxMTEAAAcHBwAAAkJCRrHJCQkqPcVxcTEBJaWlhoPIiIi0k86h53atWvj9OnThbafPn0aTk5O5dKoAu3bt0dERITGtps3b8LV1RXA487KDg4OCAoKUu9PS0vD2bNn4ePjU65tISIioheTzqOxRo0ahYkTJyIvLw9dunQB8LjT8vTp0zFlypRybdykSZPQrl07fPXVVxgwYADOnTuHH3/8ET/++CMAQJIkTJw4EV9++SUaNGgANzc3zJ49G05OTujTp0+5toWIiIheTDqHnWnTpiE5ORkfffSRej0sU1NTzJgxAwEBAeXauNatW2Pnzp0ICAjAF198ATc3NyxduhSDBw9WHzN9+nRkZGRg9OjRSE1NRYcOHXDgwAGYmpqWa1uIiIjoxSQJIURZCqanp+P69eswMzNDgwYNShyKXtWlpaVBLpdDoVCw/w4REdELQtvvb52v7BSoVq0aWrduXdbiRERERM9FmcJOaGgotm3bhpiYGPWtrAI7duwol4YRERERlQedR2Nt2bIF7dq1w/Xr17Fz507k5eXh6tWrOHr0KORyeUW0kYiIiKjMdA47X331FZYsWYK9e/fC2NgY3377LW7cuIEBAwbAxcWlItpIREREVGY6h52oqCj06NEDAGBsbIyMjAxIkoRJkyaph4QTERGR/ohTZOHvqCTEKbIquyllonOfnRo1auDRo0cAgFq1auHKlSvw8PBAamoqMjMzy72BREREVDmUKoHVp6Ix/8/rUAlAJgGB/TzwdusX606OzmGnY8eOOHz4MDw8PPDWW29hwoQJOHr0KA4fPgxfX9+KaCMRERE9gzhFFqKTMuBmYwFHuRkAQAiBlIxcxCmyEZuapf5vrCIbcf/+HK/IgvKJCWpUAvhkxxV0bGirPs+LQOews3z5cmRnZwMAZs2aBSMjI/z999/o378/Pv3003JvIBEREZXdlnMxCNh5GQWz6tWztYBKALGpWcjJV+l8PqUQuJOU+UKFHZ367OTn52Pfvn0wMDB4XFgmw8yZM7Fnzx4sXrwYNWrUqJBGEhERke5O3nyAmTv+CzoAEPUgA9FJGeqgY1PNBJ7OcnRr6oDh7etg1huN8f07XuiacxJ3F7yp8bj/0xgAQF6mAuPHj4e7uzvMzMzg4uKCjz/+GAqFQuu2jRkzBpIkYenSpRrbU1JSMHjwYFhaWsLKygojR45Eenr6M70OOl3ZMTQ0xJgxY3D9+vVnqpSIiIgqTlJ6DhYfuokt52KK3D/7zcZ4vbED7OUmMDE0KPKYELkZmjZtivFfr8NXf9yASgCQPb5G8vHqY3CIjsGiRYvQpEkT3L17F2PGjEFsbCx+++23Utu3c+dOnDlzpsgFxAcPHoy4uDgcPnwYeXl5GD58OEaPHo1NmzZp/wI8RefbWG3atEF4eLh65XEiIiKqGnLylVh3+g6WH72FRzn5RR5jIEl4w8NRq9tQhoaG+KB7K/Rq1xR3kjJhaiTDlO0XcfuBA2Te4+DSojXq1bZCvXr18L///Q/vvvsu8vPzYWhYfLy4f/8+xo8fj4MHD6pHdxe4fv06Dhw4gJCQELRq1QoA8N133+GNN97AokWLigxH2tA57Hz00UeYPHky7t27h5YtW8LCwkJjf/PmzcvUECIiIiobIQQOXUvAV39cx93kxyOjPWrJMadnE9x+kI5PdlyBUggYSBK+6tdM6/42kZGRcHJygqmpKXx8fBAYGIjfxrTD8LXncPEfBQb9dAarhrTEqw1s1etTlRR0VCoVhgwZgmnTpqFp06aF9gcHB8PKykoddADAz88PMpkMZ8+eRd++fXV8ZR7TOewMHDgQAPDxxx+rt0mSBCEEJEmCUqksU0OIiIhId9fj0vDF3msIvp0MALCtboLpXd3R38sZMpmE1nWs0bGhLe4kZaKOjbnWQcfb2xvr1q2Du7s74uLi8Pnnn+PVV1/FlStXsGlUW4z5NQx/RSZhxLoQfObvgnnz5mH06NElnnPBggUwNDTUyBBPio+Ph52dncY2Q0NDWFtbIz4+Xqt2F0XnsBMdHV3myoiIiKh8FPTL2RoSA5UAjA1lGP1qXXz4Wj1YmGh+vTvKzXQePdW9e3f1/zdv3hze3t5wdXXFtm3bMHLkSPw8rBWmbLuIPSFRGDGoPxq5uuGzzz4r9nxhYWH49ttvcf78eUiSpFNbnpXOYYd9dYiIiJ6vJ+fJsbYwLtQvp0dzR8zs1gi1rc0rrA1WVlZo2LAhbt26BQAwMTTAlz3qY/cXIyAzNkNq+wlYfjwaE/0aFBlm/vrrLyQmJmosLaVUKjFlyhQsXboUd+7cgYODAxITEzXK5efnIyUlBQ4ODmVuu85hZ8OGDSXuHzp0aJkbQ0RERJq2hsQgYMdlqAQgAahhYYyUjFwA//XLaV3HusLbkZ6ejqioKAwZMgQAkJaWhu7duqKegxXGT/0WP5z6B98GRSIlIxef9WoKA5lm4BkyZAj8/Pw0tnXt2hVDhgzB8OHDAQA+Pj5ITU1FWFgYWrZsCQA4evQoVCoVvL29y954oSMrKyuNh4WFhZAkSZiYmIgaNWroeroqQaFQCABCoVBUdlOIiKiKmTt3rgCg8XB3d1fvz8rKEh999JGwtrYWFhYWol+/fiI+Pl7r83/wwQcCgFiyZInG9oiICOHfvYeQmVkKydhMmNRqIuwHfiVcZ+wTr3xxSGwLiRFKpaq8nmYhU6ZMEcePHxfR0dHi9OnTws/PT9jY2IjExEShUCiEt7e38PDwELdu3RJxcXFi2d5zwnncL8Jl2m7x0cYwkZ2XL9zd3cWOHTuKrcPV1bXQ8+7WrZt45ZVXxNmzZ8WpU6dEgwYNxKBBg4osr+33t85Xdh4+fFhoW2RkJD788ENMmzat7KmLiIioimratCmOHDmi/vnJEUeTJk3C/v37sX37dsjlcowbNw79+vXD6dOnSz1vSfPNvPnmm7BxcoX9wP9BMjRGWugeJP7+OWqN/hmL3/NH50Z2RZyx/Pzzzz8YNGgQkpOTYWtriw4dOuDMmTOwtbXF8ePHcfbsWQBA/fr1Ncq5frQG+y8ZQJGZh4iICJ0mGgSAjRs3Yty4cfD19YVMJkP//v2xbNmyZ3ouOoedojRo0ADz58/Hu+++ixs3bpTHKYmIiKoMQ0PDIvuMKBQKrF69Gps2bUKXLl0AAGvXrkXjxo1x5swZtG3btthzljTfTFJSEiIjI2H/5kQYGz/uK1uj0zCkX9iP/OS7aORYvRyfXdG2bNlS7L7XXnsN4slpmZ9wKjIJo38JxalbSej53V9o4++Bv6OSNNblKnDnzp1C5a2trZ9pAsGi6LRcREkMDQ0RGxtbXqcjIiKqMgrmm6lbty4GDx6MmJjHMxOHhYUhLy9Poy9Ko0aN4OLiguDg4GLPV9J8M0IIBEVnwrimM8KDdsNAmQMIJR6FH4CBuRW+fL9XlV6XqkMDG2we1RbWFsa49I8Cbyw7hXd+Oov2849ia0jRMzpXNJ2v7OzZs0fjZyEE4uLisHz5crRv377cGkZERMUrahVrqhglzTcTHx8PY2NjWFlZaZSxt7cvcV6Y4uabSXyUjU92XMaR64mwHfAlMvYHInrx/0Emk6FGTRv88ecf8O/oURFPs1x51rbCD4O9MPDHM+ptlblius5hp0+fPho/S5IEW1tbdOnSBYsXLy6vdhERUTGeHJ0jk4DAfh54u7VL6QWpTEqab8bMTPcv7eLmm7kem4auS07iYWYejGQS5Jd+gUeTuvh002qYmZnh559/xoh33kJISAgcHR3L5blVJFURt7kqa8V0ncOOSqX7cvBERFQ+4hRZ6qADVO5fy5WhKlzRenK+mddffx25ublITU3VuLqTkJBQ7Lwwxc038+Oiz2FQ3Qavf/EbBjg9xPsLjuGvhw9haWkJAPjhhx9w+PBhrF+/HjNnzqzQ51ge3GwsIJOgfq8Cj9flqmNTcXMBFafc+uwQEVHF+/tWksaXB/DfX8u6mj9/PiRJwsSJE9Xb4uPjMWTIEDg4OMDCwgJeXl74/fffSzzPZ599BkmSNB6NGjUqdFxwcDC6dOkCcwsLWFSrjrbtOiArK6vUdmbnKRGdlIH/7b+GdoFHK63/R5wiC39HJeHW/QeIioqCo6MjWrZsCSMjIwQFBamPi4iIQExMDHx8fIo8z5AhQ3Dp0iWEh4fj511H0WTsShhUs4bcux8mLV6H3WPbw8b08RUfmUzza1omk70wFx0c5WYI7OcBg3+vXum6Lld50vnKTv/+/dGmTRvMmDFDY/vChQsREhKC7du3l1vjiIjoP3eTMzD/z4hC2yUJOv+1HBISglWrVhVavHno0KFITU3Fnj17YGNjg02bNmHAgAEIDQ3FK6+8Uuz5ShqaDTwOOt26dUOPIR/Cuu7/QUgGuP0gGttC78HPozbiUrNwPzULcYpsxKVmIVaRjdh/fy6YQO9JKgHM+P0ykjNyMbC1C6wtjHV6/rqYOnUqLBq0wZrwdOQ9SoHi9EYYCQmDBg2CXC7HyJEjMXnyZFhbW8PS0hLjx4+Hj4+PxkisRo0aITAwEH379kXNmjVhVl2OwD9u4JczjwAzRxgaGWHk66/g6/cf3zLz8fFBjRo1MGzYMMyZMwdmZmb46aefEB0dXWjkVlX2dmuXMq3LVd50DjsnT54scu2L7t27s88OEVEF+edhJt756SwepOfArroJktJz1Fd4zI0MYGZkoPW50tPTMXjwYPz000/48ssvNfb9/fffWLFiBdq0aQMA+PTTT7FkyRKEhYWVGHaKG5pdYNKkSRgx+kPsNnwVhtUebzOq6Yy5+yMxd39kqW02MZQhJ7/wFY2FByKw+NBNtK9vg57NHeHf1AFyM6NSz6eL67eicXDVWiiz0mBgJoeJcxNYvr0Ah29nokl2KgI+D4QkSejfvz9ycnLQtWtX/PDDDxrniIiIwN24B/g7Kgnp2fn46o/ruPPv6uTDfFyxboupxlIPNjY2OHDgAGbNmoUuXbogLy8PTZs2xe7du+Hp6Vmuz6+ilWVdrvImieIGyhfDzMwM4eHhcHd319h+48YNvPLKK1pdkqxq0tLSIJfL1cvTExFVJXGKLLy96gxiUjJR18YCWz5oC6VK4FZCOmbvvoI7yZkY1KY2Avs1L/1kAIYNGwZra2ssWbIEr732Glq0aIGlS5cCAPz9/WFsbIwNGzbAyspKvejjxYsXC00eV+Czzz7D119/DblcDlNTU/j4+CAwMFDdJyUxMRH29vZo9+5UhB3dh7zUeBjVdIZVxyEwdW4KA9njL0QnKzM4yU3h+O9/nazM/t1uiszcfHRYcEzjFp4EoKFDNUTEp6u3GRvI0MndFm82d4RfY/tCC2KWRgiBmJRMhN19iNC7D3H+7kPciH9UajkjAwkOctPH7S1oe8HzkZvhbHQy5u27ptF+B0tTfP1Wc7zawFanNtJ/tP3+1vnKjoeHB7Zu3Yo5c+ZobN+yZQuaNGmie0uJiKhYiWnZGPzTWcSkZMLF2hybRrWFXXVTAI8DwtdveeKtlcHYfO4e+nk5l7pG0pYtW3D+/HmEhIQUuX/btm14++23UbNmTRgaGsLc3Bw7d+4sNugAJQ/NNjW3wHc7TwEAzvy2CjU6j4CxfV2kXzmKhC2z4Dzye5ye/y5q1Sj5NpyVuTEC+3ngkx1XoBRC3f/j7dYuiE7KwL6LsdhzMRaRiek4fC0Bh68lwNRIBt/G9ujZ3AmvudvC1MigUAfnnHwlrtxPw/m7DxF6NwVhd1ORlJ5TYluAx0GrqZMlkjNykZCWjTylwL2ULNxL0e4PfgnAhpFt0NC+4icHpDKEndmzZ6Nfv36IiopSzxYZFBSEzZs3s78OEVE5Sk7PweCfz+J2UgZqWZlh0yhvOMhNNY5pXccag9rUxuZz9/DJjsvY//GrMDYseuzJvXv3MGHCBBw+fBimpqZFHjN79mykpqbiyJEjsLGxwa5duzBgwAD89ddf8PAoen6X4oZmf7HsZ1ww88KV87cBAK7te2HER6Ox9vQdGNvXQ87di2iZcxG1aozW6vUorv+Hm40Fxvs2wHjfBoiIf4S9F2Ox71Is7iRnYv+lOOy/FIdqJoZoaF8NF+6lQvy7oKaLtTni0rKR+9TtMSMDCR615GjpWgMtXa3h5WqFYzcSiwxaAJCvVCHhUY5mX6Mn/j8mJROPsvM16hAAktNzAXutnjo9I51vYwHA/v378dVXXyE8PBxmZmZo3rw55s6di06dOlVEGyscb2MRUVWTmpmLgT+ewY34R3CwNMW2D3zgUrPoqx+pmbnw++YEktJzMa2rO8Z2LvoqzK5du9C3b18YGPzXv0epVEKSJMhkMkRERKB+/fq4cuWKxqy+fn5+qF+/PlauXKlV228lpqOtdxvkOTRFjU7vwSI3GdeWDMO69esxbOhQxCmycCcpE4HTPkB1MxNs3LhRh1dGO0IIXLmfhr2XYrHvYixiFdnFHmttYfxvsKmBVq410KyWHKZF9IEqaLeuHW3jFFloP/9ooSHYp2Z2rvS+LC+6CruNBQA9evR4oXqDExG9SBRZeRiy+hxuxD+CbXUTbBrlXWzQAR7f4pn9ZhNM2BKOZUGR6OHhiDo2FoWO8/X1xeXLlzW2DR8+HI0aNcKMGTOQmfm4w+zTw50NDAy0Gu6cmpmLpUcisf7kDTyMvwfrBh0x6lU3jO38OppuDcCtyMcdkQs6rN6LjtK4KlSeJEmCh7McHs5yzOzWCBvO3MFne64VOu6bAZ7o+0otjcn9ilPWjrYFQ7CfvjLEoPP86Bx2QkJCoFKp4O3trbH97NmzMDAwQKtWrcqtcUREL5v0nHy8t/YcLt9XoKaFMTa97426ttVKLdfL0wm/hf2DvyKTMHv3FWwY0abQF3j16tXRrFkzjW0WFhaoWbMmmjVrhry8PNSvXx8ffPABFi1ahJo1a2LXrl04fPgw9u3bpy7j6+uLvn37Yty4cQCAyZOnwLhua+y+lYuHSYlQnNoIYyND/LlsJlo1qgMAmDZtGubOnQtPT0+0aNEC69evx40bN/Dbb7894ytWOplMQtemDvhi77VCV1d86tXUKug8q6oyBPtlpfOkgmPHjsW9e/cKbb9//z7Gjh1bLo0iItJFUZPjAf9NYmdhYQFLS0t07NixxBGjJ0+eRM+ePeHk5ARJkrBr165Cx7z33nuFJtDr1q1buTyPzNx8DF97DhdiUmFlboRf3/dGAy07sEqShC/7NIOJoQx/RSZhz0XdF2Y2MjLCH3/8AVtbW/Ts2RPNmzfHhg0bsH79erzxxhvq425G3sKFmzGIU2TheEQiNh67gEUBY3Fj+Uik7luIdk3r4MqFUHXQAYCJEyciICAAkyZNgqenJ4KCgnD48GHUq1dP53aWRVWY4M5RbgafejUZdCqBzn12qlWrhkuXLqFu3boa26Ojo9G8eXM8elT6EL2qhn12iF5cISEhGDBgACwtLdG5c2f1EOqCSewCAgLQs2dPGBoa4uLFi+jduzdMTEyKPNeff/6J06dPo2XLlujXrx927txZaD3A9957DwkJCVi7dq16m4mJCWrUqPFMzyM7T4kR60Lwd1QyqpsaYtP7beHhLNf5PN8fu4WvD0bAppoxjkzuBCvz8p1s78l1uZ5kbWGMya83xMDWtWFoUHUn5y9rvxuqmiqsz46JiQkSEhIKhZ24uLhCM2YSEVWkkibHmzRpEj7++GONNYSenh/sad27d9eqD4mJiUmJE+jpKjtPidG/hOHvqGRYGBtg/Yg2ZQo6ADDq1brYdeE+IhPTseDADa3n3tHG0+tyFRjUpjZmdm9c7pP5VYSqMMEdPX86x29/f38EBARAoVCot6WmpuKTTz7B66+/Xq6NIyIqydixY9GjRw/4+flpbE9MTMTZs2dhZ2eHdu3awd7eHp06dcKpU6fKpd7jx4/Dzs4O7u7u+PDDD5GcnFzmc+XmqzB243mcvPkAZkYGWDeiDbxcyn6VyNhQhq/6PR4ivvncPYTcSSnzuZ52+R9FoaADAL08a70QQYdeXjqHnUWLFuHevXtwdXVF586d0blzZ7i5uSE+Pp7LRRDRc1MwOV5gYGChfbdvP57X5bPPPsOoUaNw4MABeHl5wdfXF5GRpS9NUJJu3bphw4YNCAoKwoIFC3DixAl0794dSqVS53PFpGRg8M9nEHQjESaGMqwe1qrUSQG1UTD3DgB8suNyoXlkyuJqrAJz9lwttL2yVrEm0oXO951q1aqFS5cuYePGjbh48SLMzMwwfPhwDBo0CEZGTPZEVPFKmxyvYJj0Bx98gOHDhwMAXnnlFQQFBWHNmjVFBiRtDRw4UP3/Hh4eaN68OerVq4fjx4/D19dX6/NsPHMXs3ZdUf88uK0r2tW3KXO7njajWyMcvpaAyMR0/PTX7WLn3tHG7vD7mPH7JWTnqWBtYYzUzFyoROWuYk2kizJ1srGwsMDo0drNeElEVN7CwsKQmJgILy8v9TalUomTJ09i+fLliIh4vDL400vYNG7cGDExMeXalrp168LGxga3bt0qNewoVQJnbidja0gM9lyM09i3/vQdjHrVrdyCg7Zz75QkX6nCggM38NNf0QCATg1tsWzgK8jMy2cnX3qhlLlH8bVr1xATE4Pc3FyN7b169XrmRhERleTJyfEePMrG/dQsLJo1ER5Nm2DGjBmoW7cunJyc1KGnwM2bN8t9Ert//vkHycnJcHR0LHK/SiVwPuYh9l6Mxf7L8cWuu6QUAneSMss1PGgz905xUjJyMX7zeZy+9bg/0kev1cMUf3cYyCTIYcSQQy8UncPO7du30bdvX1y+fBmSJKFg5HrBP6Cy3LcmItJFweR4W0NiELDnLlQCSEjKhVO+sXrSPG0msXt6crz09HTcunVLvT86Ohrh4eGwtraGi4sL0tPT8fnnn6N///5wcHBAVFQUpk+fjvr166Nr167qckIIXL6v+HeNpjjEPbFUgZW5ETo1tMGei3EQT01wV959Xwrm3vFfclI9907vFrVKLXc1VoHRG8JwPzUL5sYGWPSWJ97wKDrMEb0IdA47EyZMgJubG4KCguDm5oZz584hOTkZU6ZMwaJFiyqijUREhTw9DFoAOHUrCTsv3EczJ0u8/+E4ZGdnY9KkSUhJSYGnp2ehSeyenBzPUW6G0NBQdO7cWb1/8uTJAIBhw4Zh3bp1MDAwwKVLl7B+/XqkpqbC3sERXu06Yd68L2BiYqJehHLvpVjcTc5Un6eaiSH8m9qjp6cTOtS3gZGBDO3qxTyX5QNca1rgY98G+PpgBObtu4ZODW1LnHvnyf45rjXN8eOQVnB34Mrc9GLTeVJBGxsbHD16FM2bN4dcLse5c+fg7u6Oo0ePYsqUKbhw4UJFtbXCcFJBohfPjvP/YPK2iyUeY2lqCCcrMzjKTeFkZabx/+fvPsSiQxFQCUAmAYH9PNSrWGvjycn1JAC21U2Q+Oi/W1RmRgbwbWyHnp5O6NTQtlwXltRVbr4KPZb9hcjEdAxqU7vIuXfylSrM//MGfj6l2T9Hbs6BJ1R1VdikgkqlEtWrP075NjY2iI2Nhbu7O1xdXQvdHyciqgjHIhIxe/eVIvfVtbHAg/QcPMrOR1p2PtLiH+FGfMkzu6sEMOP3y1hwIAIGstL7tChVAikZ//VXFAASH+XASCahc6PHAce3sR3MjUv+iH1eE9wVzL3z1spgbD53D/28nDWGuJfUP4dIH+gcdpo1a4aLFy/Czc0N3t7eWLhwIYyNjfHjjz8WmlWZiKg8CSHww/EoLDoUASEAF2sz/PMwS2MYdMHVmfScfMSlZuF+ahbiFNmIS81CrCIbsalZuP0gA/Fp2YXO/2SAKYuVQ1rCt7H9M52johTMvbP53D18suMy9n/8KowNZeyfQy8FncPOp59+ioyMDADAF198gTfffBOvvvoqatasia1bt5Z7A4mIgMfhZeq2izhwNR4AMKiNCz7r1QQpGblF3gqqZmKIBvbVi1xIM06Rhfbzj2rMBiyTgHXD28C2etHrZj3pwaMcDFt7rlAH4yZOVfs2+JNz73xzOAKGMgk//XUbOfmC/XNIr+ncZ6coKSkpqFGjhtZDGqsa9tkhqtqikzIwekMoIhPTYWQg4fNezfCOt/b9a4qyNaRwB2Fd++w8S/nKsjv8PiZsCdfY1tC+GrZ/0I79c+iFU2F9dopibf3s05sTERXlWEQiPt58AY+y82FX3QQr3m2Jlq7PtsI4ALzd2gUdG9qWuYPws5avLK3rFH7tbiWmIzMvH3Iw7JB+4jLlRFQlPd0/x8vFCivebQl7y8LLQ5TVs3YQfhFX0L7zxJD4AiqBcp/QkKgqYdghoiqnuP45JoaFh2+TbtxsLCCToNFfiYt5kr7TedVzIqKKFJ2Ugb7fn8aBq/EwMpDwVV8PBPbzYNApJ45yMwT284DBv30suZgnvQx0vrJz8uRJtGvXDoaGmkXz8/Px999/o2PHjuXWOCJ6uVRU/xzS9KL2NyIqK51HYxkYGCAuLg52dnYa25OTk2FnZ/dCro3F0VhElSdOkYXoBxk4GfkAq07errD+OUSkfypsNJYQosgh5snJybCwsND1dET0EntyyYUC7J9DROVN67DTr18/AI9X0X3vvfdgYvLfxFtKpRKXLl1Cu3btyr+FRKSX7j/MxMzfL+PJS8uSBHzsW59Bh4jKldZhRy6XA3h8Zad69eowM/vvHq+xsTHatm2LUaNGlX8LiUjvnI95iGnbL+Lpe+iCQ6CJqAJoHXbWrl0LAKhTpw6mTp3KW1ZE5WTFihVYsWIF7ty5AwBo2rQp5syZg+7duwMAoqKiMHXqVJw6dQo5OTno1q0bvvvuO9jba7cG0/z58xEQEIAJEyZg6dKlAB7Pej537lwcOnQIMTExsLW1RZ8+fTBv3jz1HzYVITY1CwsO3MDu8Ngi93MINBFVBJ2Hnk+fPl2jz87du3exdOlSHDp0qFwbRvSycHZ2xvz58xEWFobQ0FB06dIFvXv3xtWrV5GRkQF/f39IkoSjR4/i9OnTyM3NRc+ePaFSqUo9d0hICFatWoXmzZtrbI+NjUVsbCwWLVqEK1euYN26dThw4ABGjhxZIc8xMzcf3xy+iS6Lj2N3eCwkCRjQyhmz3mjMIdBEVOF0Ho3l7++Pfv36YcyYMUhNTYW7uzuMjY2RlJSEb775Bh9++GFFtbXCcDQWVTXW1tb4+uuvUbt2bXTv3h0PHz5UvzcVCgVq1KiBQ4cOwc/Pr9hzpKenw8vLCz/88AO+/PJLtGjRQn1lpyjbt2/Hu+++i4yMjEJTS5SVSiWw++J9LPgzQr3KeJs61pj9ZhN4OD++ghSnyOIQaCIqE22/v3W+snP+/Hm8+uqrAIDffvsNDg4OuHv3LjZs2IBly5aVvcVEBKVSiS1btiAjIwM+Pj7IycmBJEkaAwJMTU0hk8lw6tSpEs81duxY9OjRo8RAVCBOkYWwyH9QrbpluQWd8zEP0W/F35i09SLi07LhXMMMPwz2wtYP2qqDDvB4kjufejUZdIiowuj8qZaZmYnq1asDAA4dOoR+/fpBJpOhbdu2uHv3brk3kOhlcPnyZfj4+CA7OxvVqlXDzp070aRJE9ja2sLCwgIzZszAV199BSEEZs6cCaVSibi4uGLPt2XLFpw/fx4hISGl1r01JAbTfz2F++sCUa1pF2wNiXmm1buf7pdjYWyAjzrXx8gObjA14igrInr+dA479evXx65du9C3b18cPHgQkyZNAgAkJibyFhBRGbm7uyM8PBwKhQK//fYbhg0bhhMnTqBJkybYvn07PvzwQyxbtgwymQyDBg2Cl5cXZLKiL8zeu3cPEyZMwOHDh2FqWvKkfJEJjzB901nEb/8cRjVdIG//Dmb8fhm/BN9FHRsLOFmZwVFuCke5GWpZmcHRyhQ1LYwLzbUVp8jCjbg0nIxMwuZzMcjOU0GSgLdaOmOqvzvsODkgEVUinfvs/Pbbb3jnnXegVCrRpUsXHD58GAAQGBiIkydP4s8//6yQhlYk9tmhqiBOkYXopAy42VhgSP+eqFevHlatWqXen5SUBENDQ1hZWcHBwQFTpkzBtGnTCp2n4I8RA4P/rqIolUpIkgSZTIacnBwkZeRhzelorD12DTEbP4VkZAK7/5sLydC41HYaG8rgKDeFk/xx+EnNyMOxiESNYeRP98shIqoIFTaD8v/93/+hQ4cOiIuLg6enp3q7r68v+vbtW7bWaqmoIbTZ2dmYMmUKtmzZgpycHHTt2hU//PCD1sNyiaqCJ2cSlkmAeWomnHNyNI6xsbEBABw9ehSJiYno1atXkefy9fXF5cuXNbYNHz4cjRo1wtsjx2HGjivYHX4fOZkZSNg2G5KBEWz7z1YHHZkEzOvTDFm5SsSmZiNOkYVYRTZiU7OQlJ6D3HwV7iZn4m5yZpH1yyRg6UBPOFlxCDkRVQ1l6ono4OCA9PR0HD58GB07doSZmRlat25d5DIS5aW4IbSTJk3C/v37sX37dsjlcowbNw79+vXD6dOnK6wtVHU9eXXkRenwOn7yNPyWYAMDS1uocrOQce04osPO4NNPPwHweI6rxo0bw9bWFsHBwZgwYQImTZoEd3d39TkK/tgYN24cqlevjmbNmqn3CSGglBkjLD4Pfx1IAgCocjKRsetz1Komw8eBP2DJsbtQ5mRBJgH/G9weg7xdi2xrbr4KCWmPg0+cIhvBUcnYGnpP4xiVAO4mZzHsEFGVoXPYSU5OxoABA3Ds2DFIkoTIyEjUrVsXI0eORI0aNbB48eJyb2R6ejoGDx6Mn376CV9++aV6u0KhwOrVq7Fp0yZ06dIFwH9fDGfOnEHbtm3LvS1UdT19dSSwn8czdbR9Xu7ci8WDQxugzEiBzMQCxrZ1YDfgCyy7bobz+Rdx83gIps+YCUXqQ9SpUwezZs1S95UrEBUVhaSkJI1t+UoVDlyNx48nb+NaXBqMlVmoKQHdmjrAyygWo5deRQqASf07apTzmRJdbFuNDWWobW2O2taPg4x3XWtsD7unsbYVJwYkoqpG5z47Q4cORWJiIn7++Wc0btwYFy9eRN26dXHw4EFMnjwZV69eLfdGDhs2DNbW1liyZAlee+019XwhR48eha+vLx4+fAgrKyv18a6urpg4cWKhL4QCOTk5yHniFkFaWhpq167NPjsvsDhFFtrPP6rxpSsBGPNaXbjbWz7uY2JlBge5KYwMip9x4XlfGVKpBD7dfQWbzsaUemwNcyN4udRAyzo10MrVGs2d5Rqjmwra7mBpir8ik/Dzqdu4l5IFADAxlOGtVs54v0Nd1LEp39nPt4bE4JMdV6AUQj0x4IsQMonoxVdhfXYOHTqEgwcPwtnZWWN7gwYNKmToeUlDaOPj42FsbKwRdADA3t4e8fHxxZ4zMDAQn3/+eXk3lSpRdFKGRtABAAFgxfHbGtskCbCtZgJHKzPUsno8yqggCF2NTcOK47ee25WhPKUKM3+/jN/P//O4bf+22UCSMKtHY7jWNEfo3YcIu/sQF++l4mFmHoJuJCLoRiIAwMhAQlMnOVq61kBOnhKbzsUUeg1qmBthqE8dDPVxRc1qJqgIb7d2QceGtpwYkIiqLJ3DTkZGBszNC1+iTklJ0Zj4rDzoMoRWFwEBAZg8ebL654IrO/TisjAuPH+LBKBrMwekZuYiTpGNOEU2cvNVSHyUg8RHObh4r/B5CqgE8MmOK+jY0LZCvryzcpUYt+k8gm4kwkAmYX4/D3RoYFMoMPg2ftzRPjdfhWtxaQi9k4LzMQ8ReuchEh/lIPxeKsLvpRZZx1T/hhjZoS7Minhtytvj0MiQQ0RVk85h59VXX8WGDRswb948AIAkSVCpVFi4cCE6d+5cro0LCwtDYmIivLy81NuUSiVOnjyJ5cuX4+DBg8jNzUVqaqrG1Z2EhAQ4ODgUe14TE5NyD2ZUedKy8zDjd83RR0XdThFCIDkjF7GpWepRRnH/jjKKiE9DZGKGxjmUQuBmwqNy/xJXZOZh5PoQhN59CBNDGb5/xwt+TR6HmuLqMjaUoUVtK7SobaV+Lv88zELY3YfYfykOh68nFCrT0tX6uQQdIqKqTuews3DhQvj6+iI0NBS5ubmYPn06rl69ipSUlHIfAVXSENoZM2agdu3aMDIyQlBQEPr37w8AiIiIQExMDHx8fMq1LVQ15ear8NGv53Ej/hFsq5tg1bteyMkXRd5OkSQJNtVMYFPNBM0178IW2ecHAGbvuoJlg7zUIeNZJaRlY+jqc4hIeITqpoZY815rtK5jrfN5JElSdxT2rmuNoBsJ7CRMRFQMncNOs2bNcPPmTSxfvhzVq1dHeno6+vXrh7Fjx8LR0bFcG/f0EFoAsLCwQM2aNdXbR44cicmTJ8Pa2hqWlpYYP348fHx8OBLrJSCEQMCOyzh1KwnmxgZY+15rNKtVtknsHOVmCOznoe5oK5MAC2NDxKRkof+KvzH2tXoY79ugxM7Npbn9IB1DVp/D/dQs2FU3wfoRbdDY8dk7xD/ddq4eTkSkSeewExMTg9q1a2PWrFlF7nNxeb6jMJYsWQKZTIb+/ftrTCpIz9+KFSuwYsUK3LlzBwDQtGlTzJkzB927dwcAvPbaazhx4oRGmQ8++AArV64s9pw7duzAypUrERYWhpSUFFy4cAEtWrQAACw9Eonfz/8DSZWHhlG/o1Pzwc80seTTHW3NjAwwZ/dV7LkYi2VHb+FoRCK+GdACDe2r63ReALj8jwLvrT2H5Ixc1Klpjl9GequHb5cHdhImIiqezkPPDQwMEBcXBzs7O43tycnJsLOzg1KpLNcGPg9cLqJ87N27FwYGBmjQoAGEEFi/fj2+/vprXLhwAU2bNsVrr72Ghg0b4osvvlCXMTc3L/E1/+WXXxAdHQ0nJyeMGjVKHXa2hd7D9N8uAQDq39yCm6EnsG7dOvXEkjKZrNxuq+69GIvZu68gNTMPxoYyTPN3x4gObjCQaTeJ5t+3kjBqQygycpVoVssS64a3gU0FjYwiInqZaPv9rXPYkclkSEhIgK2trcb2u3fvokmTJsjIyCimZNXFsFNxrK2t8fXXX2PkyJEacyTp6s6dO3Bzc8OFCxeQZl4LI9aFIF8lMKKNHb4c2B6bNm3C//3f/wEAbty4gcaNGyM4OLjcbmcmpGVjxu+XcDziAYDHaz8tHuBZ6tWZPy7HYeKWcOQqVWhXryZWDWmJ6qZG5dImIqKXXbnPs1MwVFuSJMyePVtj+LlSqcTZs2fVtxeIlEoltm/fjoyMDI3O4hs3bsSvv/4KBwcH9OzZs9B7qTRRiemYe/o88lUCvVs4ob08BXl5efDz81Mf06hRI7i4uJRr2LG3NMXa91pjS8g9fLnvGs7dSUG3pSfx6ZtNMLB17SKXSvn1zF3M3n0FQgBveDhgydstYGLI0VFERM+b1mHnwoULAB53Cr18+TKMjf9bHdnY2Bienp6YOnVq+beQXiiXL1+Gj48PsrOzUa1aNezcuRNNmjQBALzzzjtwdXWFk5MTLl26hBkzZiAiIgI7duzQ+vyf7bmK9GrOaFvXGgv/rzl+37a1TBNLloUkSRjUxgXt69lg6vaLOHcnBQE7LuPQ1Xgs6N8cdpaP54ISQuC7o7fwzeGbAIB3vF0wr3czrW97ERFR+dI67Bw7dgzA46Hf3377LW/3UJHc3d0RHh4OhUKB3377DcOGDcOJEyfQpEkTjB49Wn2ch4cHHB0d4evri6ioKNSrV6/E8z7KyQMAJGfkomndalj1bqtKu0riUtMcm0e3xepTt7Ho4E0ci3gA/6Un8WWfZnilthU+33sNh649nvfmY98GmOTXoEIXySUiopLpPBpr7dq1FdEO0hPGxsaoX78+AKBly5YICQnBt99+i1WrVhU61tvbGwBw69atEsNObr4Ks3deAQDUsDDC2uGtITd/3O/FwcGhTBNLPisDmYTRHeuhU0M7TN4WjquxaRi36YLGMb08nTD59YYV1gYiItJO2ScNIdKCSqXSWHT1SeHh4QBQ4vxMBXPphN59CAD4vFdTONf4r49Py5Yt1RNLFnieE0u6O1THzo/aY3i7OoX27b8UhzhFVoW3gYiISsawQ+UmICAAJ0+exJ07d3D58mUEBATg+PHjGDx4MKKiojBv3jyEhYXhzp072LNnD4YOHYqOHTuiefPm6nM0atQIO3fuVP/8vx0h2PznSSiTHy9kpXwYi/DwcHV/HLlcrp5Y8tixYwgLC8Pw4cOf68SSxoYyvN608Jw+SiFwJynzubSBiIiKp/NtLKLiJCYmYujQoYiLi4NcLkfz5s1x8OBBvP7667h37x6OHDmCpUuXIiMjA7Vr10b//v3x6aefapwjIiICCoUCALAt9B6WrtmM5D+WqvcPHDgQADB37lx89tlnAKrGxJJuNhaQSeCSDUREVZDO8+zoI86zU7XEKbKwOzwWXx+4AaUAxnWuj6ld3Su7WaXaGhJTaMmGJxciJSKi8lXu8+wQPQ9bQ2Iwc8dlFETwV1ysMMX/xejkyyUbiIiqJoYdqjIi4tMw8/fLePJS48V7qYhPy35hgoOj3OyFaSsR0cuCYYcq3b2UTKw+FY1NZ2Pw9D1VlQDuJGUyQBARUZkx7FClufyPAqtORuGPy3EaHXufxE6+RET0rBh26LkSQuD4zQf48cRtBN9OVm9/tYENPuhYD/88zMSsnZqdfHlVh4iIngXDDj0Xufkq7LkYi59O3kZEwiMAgKFMQk9PJ4x6tS6aOP3Xi76TOzv5EhFR+WHYoXIXp8hCdFIG3GwsYGFiiM1nY7D29B3Ep2UDACyMDTCojQtGdHCDk1XhMMNOvkREVJ4YdqhcbQ2JQcCOy1AJQMLj2YVz8lUAALvqJhje3g3veLtAbmZUuQ0lIqKXBsMOlZvbD9I1ho4LADn5KtSpaY6POtdH7xZOlbZSORERvbwYduiZZOcpceLmA+y9GItDVxMKDR0HgK/6eqBdfZvn3jYiIiKAYYfKIE+pwqlbSdh7MRaHrybgUU5+sccaSBLcbC2eY+uIiIg0MexQkZ7sZOwoN4NSJXD2djL2XorDn1fikJqZpz7WwdIUbzZ3RE9PJ1yPS+PQcSIiqlIYdqgQjU7GEuBTtyYiE9Px4FGO+hibasZ4w+NxwGnpUgMymQQA8KxtxaHjRERUpTDskIbY1EyNhTiFAP6Oejz5n9zMCN2bOeDN5k5oW9cahgayIs/BoeNERFSVMOwQACAi/hH2XYrFttB76qDzpGldG2LUq/VgbFh0wCEiIqqqGHZeYtFJGdh3MRZ7L8XiZkJ6sccZSBL6eTkz6BAR0QuJYecl88/DTOy/FIe9l2Jx5X6aeruRgYRODe3Q09MRisw8fL73GjsZExGRXmDY0VNPjqYykCTsvxyHfZfiEHb3ofoYA5mE9vVt8GZzR3Rt4gC5+X+zGr/e1J6djImISC8w7OihJ0dTPU2SgDZ1rNHT0wndmzmgZjWTIs/BTsZERKQvGHb0TJwiS2M0VYGmTpbo7+WMHs0dYW9pWjmNIyIiqgQMO3pmy7miR1N92qMJfOrVfP4NIiIiqmQMO3pCqRJYdCgCK45HFdpnIEmoY2NeCa0iIiKqfBxLrAdSM3MxfF2IOui82sAG/05ozNFURET00uOVnRfcjfg0jN4QhpiUTJgaybCgf3P0blELcYosjqYiIiICw84L7Y/LcZi6/SIyc5WoZWWGH4e2RFMnOQCOpiIiIirAsPMCUqoEFh+KwA//3rZqX78mvhvkBWsL40puGRERUdXDsPOCUWTm4eMtF3Di5gMAwKhX3TCjW6NiF+UkIiJ62THsvEAi4h9h9C+huJus2T+HiIiIisew84IoqX8OERERFY9hpwqLU2QhKjEDh67GY8OZuwDYP4eIiEhXDDtVVFHrW7F/DhERke74rVkFxSmyCgUdSQJGdHBj0CEiItIRvzmroOikjEIrlgsB3EnKrJwGERERvcAYdqogm2omhbZxfSsiIqKyYdipgo7dSNT4metbERERlR07KFcxufkqrDkdDQD45I1G8KhlxfWtiIiIngHDThWzO/w+EtJyYG9pgmHt6sDE0KCym0RERPRC422sKkSlEvjx5G0AwIj2bgw6RERE5YBhpwo5FpGIyMR0VDcxxCBvl8puDhERkV5g2KlCVp14fFXnnbYusDQ1quTWEBER6QeGnSoi7O5DnLuTAiMDCSPau1V2c4iIiPQGw04V8ePJKABAnxa1YG9pWsmtISIi0h8MO1VA1IN0HLqWAAAY3bFuJbeGiIhIvzDsVAE//3UbQgB+je3QwL56ZTeHiIhIrzDsVLLER9n4/fx9AMAHnepVcmuIiIj0D8NOJVv/9x3k5qvg5WKFVq41Krs5REREeodhpxKl5+Tjl+C7AB5f1ZEkqZJbREREpH8YdirRlnMxSMvOR10bC7ze2L6ym0NERKSXGHYqSZ5ShdWnHi/4ObpjXchkvKpDRERUERh2Ksnei7GIU2TDtroJ+rxSq7KbQ0REpLcYdiqBEEK9NMTw9nVgasQFP4mIiCoKw04lOH7zASISHsHC2ACDvV0ruzlERER6jWGnEqw68XhpiEFtXCA344KfREREFYlhpxgnT55Ez5494eTkBEmSsGvXLo39kiQV+fj666+LPWedOnUgSRK2ftAOdxe8idk9m0KSJIwdO1Z9TFRUFPr27QtbW1tYWlpiwIABSEhIqKinSUREpPeqdNgJDAxE69atUb16ddjZ2aFPnz6IiIjQOCY7Oxtjx45FzZo1Ua1aNfTv379cwkFGRgY8PT3x/fffF7k/Li5O47FmzRpIkoT+/fsXe86QkBC8t/wgnMf+gtGrDuPw4cMAgLfeektdp7+/PyRJwtGjR3H69Gnk5uaiZ8+eUKlUz/yciIiIXkaSEEJUdiOK061bNwwcOBCtW7dGfn4+PvnkE1y5cgXXrl2DhYUFAODDDz/E/v37sW7dOsjlcowbNw4ymQynT5/Wup60tDTI5XIoFApYWloW2i9JEnbu3Ik+ffoUe44+ffrg0aNHCAoKKvaYO0kZ6LL4OFQCODDxVaycPwf79u1DZGQkJEnCoUOH0L17dzx8+FDdDoVCgRo1auDQoUPw8/PT+jkRERHpu9K+vwsYPsc26ezAgQMaP69btw52dnYICwtDx44doVAosHr1amzatAldunQBAKxduxaNGzfGmTNn0LZt2+fSzoSEBOzfvx/r168v8bifT92GSgCd3W1R19oUv/76KyZPnqyeOTknJweSJMHExERdxtTUFDKZDKdOnWLYISIiKoMqfRvraQqFAgBgbW0NAAgLC0NeXp5GCGjUqBFcXFwQHBxc7HlycnKQlpam8XgW69evR/Xq1dGvX79ij0lKz8H20H8APF4aYteuXUhNTcV7772nPqZt27awsLDAjBkzkJmZiYyMDEydOhVKpRJxcXHP1EYiIqKX1QsTdlQqFSZOnIj27dujWbNmAID4+HgYGxvDyspK41h7e3vEx8cXe67AwEDI5XL1o3bt2s/UtjVr1mDw4MEwNTUt9pgNf99BTr4KnrWt4O1mjdWrV6N79+5wcnJSH2Nra4vt27dj7969qFatGuRyOVJTU+Hl5QWZ7IX5VREREVUpVfo21pPGjh2LK1eu4NSpU898roCAAEyePFn9c1paWpkDz19//YWIiAhs3bq12GMycvKx/t8FP8d0rIuYmBgcOXIEO3bsKHSsv78/oqKikJSUBENDQ1hZWcHBwQF169YtU/uIiIhedi9E2Bk3bhz27duHkydPwtnZWb3dwcEBubm5SE1N1bi6k5CQAAcHh2LPZ2JiotEv5lmsXr0aLVu2hKenZ7HHbAu9B0VWHurUNId/UwfM++Jz2NnZoUePHsWWsbGxAQAcPXoUiYmJ6NWrV7m0l4iI6GVTpcOOEALjx4/Hzp07cfz4cbi5uWnsb9myJYyMjBAUFKQe8h0REYGYmBj4+Pg8U93p6em4deuW+ufo6GiEh4fD2toaLi4uAB5fEdq+fTsWL15c5Dl8fX3Rq3cfbMtsCgAY1bEuJAisXbsWw4YNg6Fh4Ze/oIO1ra0tgoODMWHCBEyaNAnu7u7P9HyIiIheVlU67IwdOxabNm3C7t27Ub16dXU/HLlcDjMzM8jlcowcORKTJ0+GtbU1LC0tMX78ePj4+DzzSKzQ0FB07txZ/XPBba9hw4Zh3bp1AIAtW7ZACIFBgwYVeY6oqCgEX4vGfau6qGlhjP5ezjhy5AhiYmIwYsSIIstEREQgICAAKSkpqFOnDmbNmoVJkyY903MhIiJ6mVXpeXYKhmQ/be3atepRTNnZ2ZgyZQo2b96MnJwcdO3aFT/88EOJt7Gepu04fV3FpmZi0I9ncTclE1Neb4jxvg3K7dxEREQvO72YZ0ebHGZqaorvv/++2JmOK8vWkBjM3HEZBU+hmmmVfqmJiIj0FsczV4A4RRYCngg6APDlvuuIU2RVXqOIiIheUgw7FSA6KQOqpy5KKYXAnaTMymkQERHRS4xhpwK42VhA9lR3IwNJQh0b88ppEBER0UuMYacCOMrNENjPAwb/drA2kCR81a8ZHOVmldwyIiKilw97zVaQt1u7oGNDW9xJykQdG3MGHSIiokrCsFOBHOVmDDlERESVjLexiIiISK8x7BAREZFeY9ghIiIivcawQ0RERHqNYYeIiIj0GsMOERER6TWGHSIiItJrDDtERESk1xh2iIiISK8x7BAREZFeY9ghIiIivcawQ0RERHqNYYeIiIj0GsMOERER6TWGHSIiItJrDDtERESk1xh2iIiISK8x7BAREZFeY9ghIiIivcawQ0RERHqNYYeIiIj0GsMOERER6TWGHSIiItJrDDtERESk1xh2iIiISK8x7BAREZFeY9ghIiIivcawQ0RERHqNYYeIiIj0GsMOERER6TWGHSIiItJrDDtERESk1xh2iIiISK8x7BAREZFeY9ghIiIivcawQ0RERHqNYYeIiIj0GsMOERER6TWGHSIiItJrDDtERESk1xh2iIiISK8x7BAREZFeY9ghIiIivcawQ0RERHqNYYeIiIj0GsMOERER6TWGHSIiItJrDDtERESk1xh2iIiISK8x7BAREZFeY9ghIiIivcawQ0RERHqNYYeIiIj0GsMOERER6TWGHSIiItJrDDtERESk1xh2iIiISK/pTdj5/vvvUadOHZiamsLb2xvnzp2r7CYRERFRFaAXYWfr1q2YPHky5s6di/Pnz8PT0xNdu3ZFYmJiZTeNiIiIKplehJ1vvvkGo0aNwvDhw9GkSROsXLkS5ubmWLNmTWU3jYiIiCqZYWU34Fnl5uYiLCwMAQEB6m0ymQx+fn4IDg4uskxOTg5ycnLUPysUCgBAWlpaxTaWiIiIyk3B97YQosTjXviwk5SUBKVSCXt7e43t9vb2uHHjRpFlAgMD8fnnnxfaXrt27QppIxEREVWcR48eQS6XF7v/hQ87ZREQEIDJkyerf1apVEhJSUHNmjUhSVK51ZOWlobatWvj3r17sLS0fK7lWffzr/tZy7Pul6vuZy3Puln3i1L+WesuiRACjx49gpOTU4nHvfBhx8bGBgYGBkhISNDYnpCQAAcHhyLLmJiYwMTERGOblZVVRTURlpaWz/QLfpbyrPv51/2s5Vn3y1X3s5Zn3az7RSn/rHUXp6QrOgVe+A7KxsbGaNmyJYKCgtTbVCoVgoKC4OPjU4ktIyIioqrghb+yAwCTJ0/GsGHD0KpVK7Rp0wZLly5FRkYGhg8fXtlNIyIiokqmF2Hn7bffxoMHDzBnzhzEx8ejRYsWOHDgQKFOy8+biYkJ5s6dW+iW2fMoz7qff93PWp51v1x1P2t51s26X5Tyz1p3eZBEaeO1iIiIiF5gL3yfHSIiIqKSMOwQERGRXmPYISIiIr3GsENERER6jWGnAn3//feoU6cOTE1N4e3tjXPnzmlV7uTJk+jZsyecnJwgSRJ27dqldZ2BgYFo3bo1qlevDjs7O/Tp0wcRERFal1+xYgWaN2+unvzJx8cHf/75p9blnzR//nxIkoSJEydqdfxnn30GSZI0Ho0aNdK6vvv37+Pdd99FzZo1YWZmBg8PD4SGhmpVtk6dOoXqliQJY8eOLbWsUqnE7Nmz4ebmBjMzM9SrVw/z5s0rda2WJz169AgTJ06Eq6srzMzM0K5dO4SEhBQ6rrT3hhACc+bMgaOjI8zMzODn54fIyEity+/YsQP+/v7q2cTDw8O1rj8vLw8zZsyAh4cHLCws4OTkhKFDhyI2Nlaruj/77DM0atQIFhYWqFGjBvz8/HD27Fmt2/6kMWPGQJIkLF26VKuy7733XqHffbdu3XSq+/r16+jVqxfkcjksLCzQunVrxMTElFq2qPedJEn4+uuvtao7PT0d48aNg7OzM8zMzNSLIWtTNiEhAe+99x6cnJxgbm6Obt26qd8v2nyWZGdnY+zYsahZsyaqVauG/v37qyd41ab8jz/+iNdeew2WlpaQJAmpqanqfaWVT0lJwfjx4+Hu7g4zMzO4uLjg448/hkKh0KruDz74APXq1YOZmRlsbW3Ru3dv9RJDunyOCiHQvXt39eurTdnXXnut0O97zJgxOtUdHByMLl26wMLCApaWlujYsSO++OKLEsveuXOn2Pfb9u3btao7Pj4eQ4YMgYODAywsLODl5YXff/9dq7JRUVHo27cvbG1tYWlpiQEDBhSaELiiMOxUkK1bt2Ly5MmYO3cuzp8/D09PT3Tt2hWJiYmlls3IyICnpye+//57nes9ceIExo4dizNnzuDw4cPIy8uDv78/MjIytCrv7OyM+fPnIywsDKGhoejSpQt69+6Nq1ev6tSOkJAQrFq1Cs2bN9epXNOmTREXF6d+nDp1SqtyDx8+RPv27WFkZIQ///wT165dw+LFi1GjRg2t2/tkvYcPHwYAvPXWW6WWXbBgAVasWIHly5fj+vXrWLBgARYuXIjvvvtOq7oB4P3338fhw4fxyy+/4PLly/D394efnx/u37+vcVxp742FCxdi2bJlWLlyJc6ePQsLCwt07doV2dnZWpXPyMhAhw4dsGDBgmL3F1c+MzMT58+fx+zZs3H+/Hns2LEDERER6NWrl1Z1N2zYEMuXL8fly5dx6tQp1KlTB/7+/njw4IFW5Qvs3LkTZ86c0Zg+Xpuy3bp103gPbN68WevyUVFR6NChAxo1aoTjx4/j0qVLmD17NkxNTUst+2SdcXFxWLNmDSRJQv/+/bWqe/LkyThw4AB+/fVXXL9+HRMnTsS4ceOwZ8+eEssKIdCnTx/cvn0bu3fvxoULF+Dq6go/Pz9kZGRo9VkyadIk7N27F9u3b8eJEycQGxuLfv36AdDusygzMxPdunXDJ598Uqh9pZWPjY1FbGwsFi1ahCtXrmDdunU4cOAARo4cqVXdLVu2xNq1a3H9+nUcPHgQQgj4+/tDqVTq9Dm6dOlSjWWGtC07atQojd/7woULtS4fHByMbt26wd/fH+fOnUNISAjGjRuHU6dOlVi2du3ahd5vn3/+OapVq4bu3btrVffQoUMRERGBPXv24PLly+jXrx8GDBiAvXv3llg2IyMD/v7+kCQJR48exenTp5Gbm4uePXtCpVIVel3LnaAK0aZNGzF27Fj1z0qlUjg5OYnAwECdzgNA7Ny5s8ztSExMFADEiRMnynyOGjVqiJ9//lnr4x89eiQaNGggDh8+LDp16iQmTJigVbm5c+cKT0/PMrVxxowZokOHDmUqW5QJEyaIevXqCZVKVeqxPXr0ECNGjNDY1q9fPzF48GCt6srMzBQGBgZi3759Gtu9vLzErFmzii339HtDpVIJBwcH8fXXX6u3paamChMTE7F58+ZSyz8pOjpaABAXLlzQuv6inDt3TgAQd+/e1bmsQqEQAMSRI0e0rvuff/4RtWrVEleuXBGurq5iyZIlWpUdNmyY6N27d4ntKan822+/Ld59990ylX1a7969RZcuXbQu37RpU/HFF19obCvqvfN02YiICAFAXLlyRb1NqVQKW1tb8dNPPxWq++nPktTUVGFkZCS2b9+uPub69esCgAgODi61/JOOHTsmAIiHDx8W+bxLK19g27ZtwtjYWOTl5elc9uLFiwKAuHXrltZ1X7hwQdSqVUvExcUV+7stqqwun4tFlff29haffvppmco+rUWLFoU+v0oqb2FhITZs2KBxnLW1daH3zNNlDx48KGQymVAoFOpjUlNThSRJ4vDhw6U+l2fFKzsVIDc3F2FhYfDz81Nvk8lk8PPzQ3Bw8HNti0KhAABYW1vrXFapVGLLli3IyMjQaemNsWPHokePHhrPX1uRkZFwcnJC3bp1MXjwYMTExGhVbs+ePWjVqhXeeust2NnZ4ZVXXsFPP/2kc/3A49/fr7/+ihEjRmi1MGy7du0QFBSEmzdvAgAuXryIU6dOoXv37lrVl5+fD6VSCVNTU43tZmZmWl/ZAoDo6GjEx8drvO5yuRze3t7P/X1XQKFQQJIkndeey83NxY8//gi5XA5PT0+tyqhUKgwZMgTTpk1D06ZNdW7r8ePHYWdnB3d3d3z44YdITk7Wut79+/ejYcOG6Nq1K+zs7ODt7a3T7ecCCQkJ2L9/P0aOHKl1mXbt2mHPnj24f/8+hBA4duwYbt68CX9//xLL5eTkAIDG+04mk8HExKTI993TnyVhYWHIy8vTeL81atQILi4uRb7fnuWzSNvyCoUClpaWMDQ0LLS9pLIZGRlYu3Yt3NzcULt2ba3qzszMxDvvvIPvv/++2HUYS6p748aNsLGxQbNmzRAQEIDMzEytyicmJuLs2bOws7NDu3btYG9vj06dOmn1O3taWFgYwsPDi32/FVW+Xbt22Lp1K1JSUqBSqbBlyxZkZ2fjtddeK7FsTk4OJEnSmFjQ1NQUMplMp8+5MqvwOPUSun//vgAg/v77b43t06ZNE23atNHpXHiGKztKpVL06NFDtG/fXqdyly5dEhYWFsLAwEDI5XKxf/9+rctu3rxZNGvWTGRlZQkhdPsL5o8//hDbtm0TFy9eFAcOHBA+Pj7CxcVFpKWllVrWxMREmJiYiICAAHH+/HmxatUqYWpqKtatW6d12wts3bpVGBgYiPv372t1vFKpFDNmzBCSJAlDQ0MhSZL46quvdKrTx8dHdOrUSdy/f1/k5+eLX375RchkMtGwYcNiyzz93jh9+rQAIGJjYzWOe+utt8SAAQNKLf+k8riyk5WVJby8vMQ777yjddm9e/cKCwsLIUmScHJyEufOndO67q+++kq8/vrr6qtxulzZ2bx5s9i9e7e4dOmS2Llzp2jcuLFo3bq1yM/PL7V8wV/15ubm4ptvvhEXLlwQgYGBQpIkcfz4ca2ed4EFCxaIGjVqqP/9aNP27OxsMXToUAFAGBoaCmNjY7F+/fpSy+bm5goXFxfx1ltviZSUFJGTkyPmz58vAAh/f3+NskV9lmzcuFEYGxsXqqd169Zi+vTppZZ/UmlXdrT5LHvw4IFwcXERn3zyidZlv//+e2FhYSEACHd39yKv6hRXfvTo0WLkyJHqn4v63RRXdtWqVeLAgQPi0qVL4tdffxW1atUSffv21aru4OBgAUBYW1uLNWvWiPPnz4uJEycKY2NjcfPmTa2ed4EPP/xQNG7cuMh9xZV/+PCh8Pf3V7/fLC0txcGDB0stm5iYKCwtLcWECRNERkaGSE9PF+PGjRMAxOjRo4ttY3lh2KkAVSXsjBkzRri6uop79+7pVC4nJ0dERkaK0NBQMXPmTGFjYyOuXr1aarmYmBhhZ2cnLl68qN6mS9h52sOHD4WlpaVWt9CMjIyEj4+Pxrbx48eLtm3b6lyvv7+/ePPNN7U+fvPmzcLZ2Vls3rxZXLp0SWzYsEFYW1vrFLRu3bolOnbsKAAIAwMD0bp1azF48GDRqFGjYstU5bCTm5srevbsKV555RWNy9allU1PTxeRkZEiODhYjBgxQtSpU0ckJCSUWj40NFTY29trBFRdws7ToqKitL6FVvDvfdCgQRrH9ezZUwwcOFCnut3d3cW4ceOK3V9U+a+//lo0bNhQ7NmzR1y8eFF89913olq1aoVuDRRVNjQ0VHh6eqrfd127dhXdu3cX3bp10ziuqM8SXcJOaZ9FpYWd0sorFArRpk0b0a1bN5Gbm6t12dTUVHHz5k1x4sQJ0bNnT+Hl5VUoaBZVfvfu3aJ+/fri0aNH6m1Fvb7afgYHBQUVeQutqPIF/84DAgI0jvXw8BAzZ87Uuu7MzEwhl8vFokWLitxfXPlx48aJNm3aiCNHjojw8HDx2WefCblcLi5dulRq2YMHD4q6desKSZKEgYGBePfdd4WXl5cYM2ZMCa9O+WDYqQA5OTnCwMCg0Bt/6NCholevXjqdq6xhZ+zYscLZ2Vncvn1b57JP8/X11Sp579y5U/2hWfAAoH5jF/VXcmlatWql8Q+4OC4uLhp/ZQkhxA8//CCcnJx0qu/OnTtCJpOJXbt2aV3G2dlZLF++XGPbvHnzhLu7u051C/H4y74grAwYMEC88cYbxR779Huj4Av66YDSsWNH8fHHH5da/knPEnZyc3NFnz59RPPmzUVSUpJOZZ9Wv379Iq+SPV1+yZIl6vfZk+89mUwmXF1dy1S3jY2NWLlyZal15+TkCENDQzFv3jyN46ZPny7atWundd0nT54UAER4eHixbXq6fGZmpjAyMirU32vkyJGia9euWtedmpoqEhMThRCP+xt+9NFH6n3FfZYUfEE/HVBcXFzEN998U2r5J5UUdkorn5aWJnx8fISvr2+hoKLL52BOTo4wNzcXmzZtKrX8hAkTin2/derUSee609PTBQBx4MCBUuu+ffu2ACB++eUXje0DBgxQX0XVpu4NGzYIIyMj9e/9ScWVv3XrVqF+XkI8/o744IMPtK77wYMH6t+1vb29WLhwYbHHlhf22akAxsbGaNmyJYKCgtTbVCoVgoKCdOr7UhZCCIwbNw47d+7E0aNH4ebm9sznVKlU6vv7JfH19cXly5cRHh6ufrRq1QqDBw9GeHg4DAwMdKo3PT0dUVFRcHR0LPXY9u3bFxrmePPmTbi6uupU59q1a2FnZ4cePXpoXSYzMxMymeY/JQMDgzKNMLCwsICjoyMePnyIgwcPonfv3lqXdXNzg4ODg8b7Li0tDWfPnq3w912BvLw8DBgwAJGRkThy5Ahq1qz5TOfT9r03ZMgQXLp0SeO95+TkhGnTpuHgwYM61/vPP/8gOTlZq/eesbExWrdu/czvv9WrV6Nly5Za91ECHr/eeXl5z/z+k8vlsLW1RWRkJEJDQ9G7d+9SP0tatmwJIyMjjfdbREQEYmJi4OPj88yfRdqUT0tLg7+/P4yNjbFnzx51/6Oy1C0e//GPnJycUsvPnDmz0PsNAJYsWYI1a9boXHdBeUdHx1LrrlOnDpycnIp8v7m4uGhd9+rVq9GrVy/Y2tpqvAYllS/oV1TU+02pVGpdt42NDaysrHD06FEkJiaqR2xWqAqPUy+pLVu2CBMTE7Fu3Tpx7do1MXr0aGFlZSXi4+NLLfvo0SNx4cIFceHCBQFA3Q/g6REtRfnwww+FXC4Xx48fF3FxcepHZmamVu2eOXOmOHHihIiOjhaXLl0SM2fOFJIkiUOHDmlV/mm63MaaMmWKOH78uIiOjhanT58Wfn5+wsbGpsi/PJ527tw5YWhoKP73v/+JyMhIsXHjRmFubi5+/fVXrduqVCqFi4uLmDFjhtZlhHg8kqdWrVpi3759Ijo6WuzYsUPY2NgUupRfkgMHDog///xT3L59Wxw6dEh4enoKb2/vQpfkS3tvzJ8/X1hZWan7n/Tu3Vu4ubmp/+ItrXxycrK4cOGC2L9/vwAgtmzZIi5cuCDi4uJKLZ+bmyt69eolnJ2dRXh4uMb7Lycnp8Sy6enpIiAgQAQHB4s7d+6I0NBQMXz4cGFiYqL+K1LXfxdP3sYqqeyjR4/E1KlTRXBwsIiOjhZHjhwRXl5eokGDBiI7O1urunfs2CGMjIzEjz/+KCIjI8V3330nDAwMxF9//aVVuxUKhTA3NxcrVqwo9DxKK9+pUyfRtGlTcezYMXH79m2xdu1aYWpqKn744YdSy27btk0cO3ZMREVFiV27dglXV1fRr18/IYR2nyVjxowRLi4u4ujRoyI0NFT4+PiobydrUz4uLk5cuHBB/PTTTwKAOHnypLhw4YJITk4utbxCoRDe3t7Cw8ND3Lp1S+OYMWPGlFg2KipKfPXVVyI0NFTcvXtXnD59WvTs2VNYW1uLhISEMn2O4t8rZ6WVvXXrlvjiiy9EaGioiI6OFrt37xZ169YVHTt21Pp1W7JkibC0tBTbt28XkZGR4tNPPxWmpqbinXfe0ardkZGRQpIk8eeff2psL63u3NxcUb9+ffHqq6+Ks2fPilu3bolFixYJSZLEG2+8UWrda9asEcHBweLWrVvil19+EdbW1mLy5MnFvqbliWGnAn333XfCxcVFGBsbizZt2ogzZ85oVa7gku7Tj2HDhpVatqhyAMTatWu1qnvEiBHC1dVVGBsbC1tbW+Hr61vmoCOEbmHn7bffFo6OjsLY2FjUqlVLvP3220V2GCzO3r17RbNmzYSJiYlo1KiR+PHHH3Vq68GDBwUAERERoVO5tLQ0MWHCBOHi4iJMTU1F3bp1xaxZs0ROTo7W59i6dauoW7euMDY2Fg4ODmLs2LEiNTW10HGlvTdUKpWYPXu2sLe3FyYmJsLX11fj+ZRWfu3atUXunzt3bqnlC259FfU4duxYiWWzsrJE3759hZOTkzA2NhaOjo6iV69eGh2Udf138WTYKalsZmam8Pf3F7a2tsLIyEi4urqKUaNGafxhok3dq1evFvXr1xempqbC09NTfStUm7KrVq0SZmZmZfqdx8XFiffee084OTkJU1NT4e7uLhYvXixUKlWpZb/99lvh7OwsjIyMhIuLi/j000/V71ttPkuysrLERx99JGrUqCHMzc1F37591cFYm/Jz584t9pjSyhf33Ep6FJS9f/++6N69u7CzsxNGRkbC2dlZvPPOO+LGjRtat/1pBWGntLIxMTGiY8eOwtraWpiYmIj69euLadOmqfu2aVt3YGCgcHZ2Fubm5sLHx0f89ddfWpcNCAgQtWvXFkqlstBzKK38zZs3Rb9+/YSdnZ0wNzcXzZs3Fxs2bNCq7IwZM4S9vb0wMjISDRo0UL9Pnwfp3ydIREREpJfYZ4eIiIj0GsMOERER6TWGHSIiItJrDDtERESk1xh2iIiISK8x7BAREZFeY9ghIiIivcawQ0T0lOPHj0OSJKSmplZ2U4ioHDDsEBERkV5j2CEiIiK9xrBDRFWOSqVCYGAg3NzcYGZmBk9PT/z2228A/rvFtH//fjRv3hympqZo27Ytrly5onGO33//HU2bNoWJiQnq1KmDxYsXa+zPycnBjBkzULt2bZiYmKB+/fpYvXq1xjFhYWFo1aoVzM3N0a5du0IrTRPRi4Fhh4iqnMDAQGzYsAErV67E1atXMWnSJLz77rs4ceKE+php06Zh8eLFCAkJga2tLXr27Im8vDwAj0PKgAEDMHDgQFy+fBmfffYZZs+ejXXr1qnLDx06FJs3b8ayZctw/fp1rFq1CtWqVdNox6xZs7B48WKEhobC0NAQI0aMeC7Pn4jKFxcCJaIqJScnB9bW1jhy5Ah8fHzU299//31kZmZi9OjR6Ny5M7Zs2YK3334bAJCSkgJnZ2esW7cOAwYMwODBg/HgwQMcOnRIXX769OnYv38/rl69ips3b8Ld3R2HDx+Gn59foTYcP34cnTt3xpEjR+Dr6wsA+OOPP9CjRw9kZWXB1NS0gl8FIipPvLJDRFXKrVu3kJmZiddffx3VqlVTPzZs2ICoqCj1cU8GIWtra7i7u+P69esAgOvXr6N9+/Ya523fvj0iIyOhVCoRHh4OAwMDdOrUqcS2NG/eXP3/jo6OAIDExMRnfo5E9HwZVnYDiIielJ6eDgDYv38/atWqpbHPxMREI/CUlZmZmVbHGRkZqf9fkiQAj/sTEdGLhVd2iKhKadKkCUxMTBATE4P69etrPGrXrq0+7syZM+r/f/jwIW7evInGjRsDABo3bozTp09rnPf06dNo2LAhDAwM4OHhAZVKpdEHiIj0F6/sEFGVUr16dUydOhWTJk2CSqVChw4doFAocPr0aVhaWsLV1RUA8MUXX6BmzZqwt7fHrFmzYGNjgz59+gAApkyZgtatW2PevHl4++23ERwcjOXLl+OHH34AANSpUwfDhg3DiBEjsGzZMnh6euLu3btITEzEgAEDKuupE1EFYdghoipn3rx5sLW1RWBgIG7fvg0rKyt4eXnhk08+Ud9Gmj9/PiZMmIDIyEi0aNECe/fuhbGxMQDAy8sL27Ztw5w5czBv3jw4Ojriiy++wHvvvaeuY8WKFfjkk0/w0UcfITk5GS4uLvjkk08q4+kSUQXjaCwieqEUjJR6+PAhrKysKrs5RPQCYJ8dIiIi0msMO0RERKTXeBuLiIiI9Bqv7BAREZFeY9ghIiIivcawQ0RERHqNYYeIiIj0GsMOERER6TWGHSIiItJrDDtERESk1xh2iIiISK8x7BAREZFe+38aEvV9Ns7iMAAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plt.plot(np.arange(len(epochs_x)), epochs_acc, marker = '.')\n", + "plt.xlabel('epoch')\n", + "plt.ylabel('test accuracy')\n", + "plt.ylim(0, 100)\n", + "plt.xticks(np.arange(len(epochs_x)))\n", + "for i, txt in enumerate(epochs_acc):\n", + " if i%5 ==0 or i == epochs-1:\n", + " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "with open(f'./architectures_results/{achitecture}-Training_Validation-TM.npy', 'wb') as f:\n", + " np.save(f, np.array(epochs_x))\n", + " np.save(f, np.array(epochs_y))\n", + " np.save(f, np.array(epochs_acc))" ] } ], From 91963dbbd52c5957bba9e7280c2fdee1e449c640 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Wed, 1 May 2024 17:53:17 +0200 Subject: [PATCH 068/379] utils funcs --- .../test_nonsequential/utils/train_test_fn.py | 69 ++++++++++++++++++- 1 file changed, 68 insertions(+), 1 deletion(-) diff --git a/tests/test_nonsequential/utils/train_test_fn.py b/tests/test_nonsequential/utils/train_test_fn.py index bdc5d1da..3b896152 100644 --- a/tests/test_nonsequential/utils/train_test_fn.py +++ b/tests/test_nonsequential/utils/train_test_fn.py @@ -1,5 +1,10 @@ from tqdm.notebook import tqdm import torch +from tonic.datasets.dvsgesture import DVSGesture +from tonic.datasets.nmnist import NMNIST +from tonic.transforms import ToFrame +import numpy as np +from torch.utils.data import Subset def training_loop(device, nb_time_steps, batch_size, feature_map_size, dataloader_train, model, loss_fn, optimizer, epochs, dataloader_test): epochs_y = [] @@ -78,4 +83,66 @@ def test(device, nb_time_steps, batch_size, feature_map_size, dataloader_test, m test_p_bar.set_description(f"Testing Model...") correct_predictions = torch.cat(correct_predictions) - return correct_predictions.sum().item()/(len(correct_predictions))*100 \ No newline at end of file + return correct_predictions.sum().item()/(len(correct_predictions))*100 + +def load_dataset(dataset, n_time_steps): + if dataset == 'DVSGESTURE': + root_dir = "../../DVSGESTURE" + _ = DVSGesture(save_to=root_dir, train=True) + _ = DVSGesture(save_to=root_dir, train=False) + + to_raster = ToFrame(sensor_size=DVSGesture.sensor_size, n_time_bins=n_time_steps) + + snn_train_dataset = DVSGesture(save_to=root_dir, train=True, transform=to_raster) + snn_test_dataset = DVSGesture(save_to=root_dir, train=False, transform=to_raster) + + return snn_train_dataset, snn_test_dataset, DVSGesture.sensor_size + + elif dataset == 'NMNIST': + root_dir = "../../NMNIST" + _ = NMNIST(save_to=root_dir, train=True) + _ = NMNIST(save_to=root_dir, train=False) + + to_raster = ToFrame(sensor_size=NMNIST.sensor_size, n_time_bins=n_time_steps) + + snn_train_dataset = NMNIST(save_to=root_dir, train=True, transform=to_raster) + snn_test_dataset = NMNIST(save_to=root_dir, train=False, transform=to_raster) + + return snn_train_dataset, snn_test_dataset, NMNIST.sensor_size + + else: + + raise ValueError('no valid dataset') + +def split_train_validation(validation_ratio, snn_train_dataset, rand_seed): + num_samples = len(snn_train_dataset) + num_validation_samples = int(validation_ratio * num_samples) + + np.random.seed(rand_seed) + + validation_indices = np.random.choice(np.arange(num_samples), size=num_validation_samples, replace=False) + training_indices = np.array(list(filter(lambda x: x not in validation_indices, np.arange(num_samples)))) + + train_dataset = Subset(snn_train_dataset, training_indices) + validation_dataset = Subset(snn_train_dataset, validation_indices) + + return train_dataset, validation_dataset + +def load_architecture(architecture, input_size, nb_classes, batch_size, surrogate_fn, min_v_mem=-0.313, spk_thr=2.0): + import sys + sys.path.append('../models') + + if architecture == 'ResSCNN1': + from ResSCNN1 import SCNN + return SCNN(input_size, nb_classes, batch_size, surrogate_fn, min_v_mem, spk_thr) + elif architecture == 'ResSCNN2': + from ResSCNN2 import SCNN + return SCNN(input_size, nb_classes, batch_size, surrogate_fn, min_v_mem, spk_thr) + elif architecture == 'ResSCNN3': + from ResSCNN3 import SCNN + return SCNN(input_size, nb_classes, batch_size, surrogate_fn, min_v_mem, spk_thr) + elif architecture == 'ResSCNN4': + from ResSCNN4 import SCNN + return SCNN(input_size, nb_classes, batch_size, surrogate_fn, min_v_mem, spk_thr) + else: + return None \ No newline at end of file From af679b29ed3bda6c18d644ba2b4b9fd15906295a Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Wed, 1 May 2024 18:53:28 +0200 Subject: [PATCH 069/379] loading ResSCNN models from 1 to 8 --- tests/test_nonsequential/utils/train_test_fn.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_nonsequential/utils/train_test_fn.py b/tests/test_nonsequential/utils/train_test_fn.py index 3b896152..30b603e4 100644 --- a/tests/test_nonsequential/utils/train_test_fn.py +++ b/tests/test_nonsequential/utils/train_test_fn.py @@ -144,5 +144,17 @@ def load_architecture(architecture, input_size, nb_classes, batch_size, surrogat elif architecture == 'ResSCNN4': from ResSCNN4 import SCNN return SCNN(input_size, nb_classes, batch_size, surrogate_fn, min_v_mem, spk_thr) + elif architecture == 'ResSCNN5': + from ResSCNN5 import SCNN + return SCNN(input_size, nb_classes, batch_size, surrogate_fn, min_v_mem, spk_thr) + elif architecture == 'ResSCNN6': + from ResSCNN6 import SCNN + return SCNN(input_size, nb_classes, batch_size, surrogate_fn, min_v_mem, spk_thr) + elif architecture == 'ResSCNN7': + from ResSCNN7 import SCNN + return SCNN(input_size, nb_classes, batch_size, surrogate_fn, min_v_mem, spk_thr) + elif architecture == 'ResSCNN8': + from ResSCNN8 import SCNN + return SCNN(input_size, nb_classes, batch_size, surrogate_fn, min_v_mem, spk_thr) else: return None \ No newline at end of file From c4cec9945f5b668d3f6f7f00166e527e67d2ad0d Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Wed, 1 May 2024 18:54:22 +0200 Subject: [PATCH 070/379] Residual SCNNs architectures --- .../using_SumPool2d/models/ResSCNN5.py | 138 +++++++++++++++++ .../using_SumPool2d/models/ResSCNN6.py | 141 +++++++++++++++++ .../using_SumPool2d/models/ResSCNN7.py | 142 +++++++++++++++++ .../using_SumPool2d/models/ResSCNN8.py | 146 ++++++++++++++++++ 4 files changed, 567 insertions(+) create mode 100644 tests/test_nonsequential/using_SumPool2d/models/ResSCNN5.py create mode 100644 tests/test_nonsequential/using_SumPool2d/models/ResSCNN6.py create mode 100644 tests/test_nonsequential/using_SumPool2d/models/ResSCNN7.py create mode 100644 tests/test_nonsequential/using_SumPool2d/models/ResSCNN8.py diff --git a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN5.py b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN5.py new file mode 100644 index 00000000..51e82235 --- /dev/null +++ b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN5.py @@ -0,0 +1,138 @@ +import torch.nn as nn +import sinabs.layers as sl +from sinabs.exodus.layers import IAFSqueeze + +class SCNN(nn.Module): + def __init__(self, input_size, nb_classes, batch_size, surrogate_fn, min_v_mem=-0.313, spk_thr=2.0) -> None: + super().__init__() + + self.conv1 = nn.Conv2d(2, 8, 2, 1, bias=False) + self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool1 = sl.SumPool2d(2,2) + self.pool1a = sl.SumPool2d(8,8) + + self.conv2 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool2 = sl.SumPool2d(2,2) + + self.conv3 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool3 = sl.SumPool2d(2,2) + + self.conv4 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool4 = sl.SumPool2d(2,2) + + self.flat = nn.Flatten() + + flat_s = SCNN.get_flatten_size(input_size) + + self.fc1 = nn.Linear(flat_s, 100, bias=False) + self.iaf1_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc2 = nn.Linear(100, 100, bias=False) + self.iaf2_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc3 = nn.Linear(100, 100, bias=False) + self.iaf3_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc4 = nn.Linear(100, 100, bias=False) + self.iaf4_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc5 = nn.Linear(100, nb_classes, bias=False) + self.iaf5_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + # skip + + self.merge1 = sl.Merge() + + @staticmethod + def get_flatten_size(input_size): + conv1_dims = SCNN.conv2d_output_size(input_size, 8, (2, 2)) + pool1_dims = SCNN.pool_output_size(conv1_dims, 8, (2, 2)) + + conv2_dims = SCNN.conv2d_output_size(pool1_dims, 8, (2, 2)) + pool2_dims = SCNN.pool_output_size(conv2_dims, 8, (2, 2)) + + conv3_dims = SCNN.conv2d_output_size(pool2_dims, 8, (2, 2)) + pool3_dims = SCNN.pool_output_size(conv3_dims, 8, (2, 2)) + + conv4_dims = SCNN.conv2d_output_size(pool3_dims, 8, (2, 2)) + pool4_dims = SCNN.pool_output_size(conv4_dims, 8, (2, 2)) + + return pool4_dims[0]*pool4_dims[1]*pool4_dims[2] + + @staticmethod + def conv2d_output_size(input_size, out_channels, kernel_size, stride=1, padding=0, dilation=1): + input_height, input_width, input_channels = input_size + kernel_height, kernel_width = kernel_size + + output_height = ((input_height + 2 * padding - dilation * (kernel_height - 1) - 1) // stride) + 1 + output_width = ((input_width + 2 * padding - dilation * (kernel_width - 1) - 1) // stride) + 1 + + return (output_height, output_width, out_channels) + + @staticmethod + def pool_output_size(input_size, out_channels, kernel_size, stride=None, padding=0): + input_height, input_width, input_channels = input_size + kernel_height, kernel_width = kernel_size + + if stride is None: + stride = kernel_height + + output_height = ((input_height + 2 * padding - kernel_height) // stride) + 1 + output_width = ((input_width + 2 * padding - kernel_width) // stride) + 1 + + return (output_height, output_width, out_channels) + + def detach_neuron_states(self): + for name, layer in self.named_modules(): + if name != '': + if isinstance(layer, sl.StatefulLayer): + for name, buffer in layer.named_buffers(): + buffer.detach_() + + def init_weights(self): + for name, layer in self.named_modules(): + if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear): + nn.init.xavier_normal_(layer.weight.data) + + def forward(self, x): + # conv 1 + con1_out = self.conv1(x) + iaf1_out = self.iaf1(con1_out) + pool1_out = self.pool1(iaf1_out) + pool1a_out = self.pool1a(iaf1_out) + # conv 2 + conv2_out = self.conv2(pool1_out) + iaf2_out = self.iaf2(conv2_out) + pool2_out = self.pool2(iaf2_out) + # conv 3 + conv3_out = self.conv3(pool2_out) + iaf3_out = self.iaf3(conv3_out) + pool3_out = self.pool3(iaf3_out) + + merge1_out = self.merge1(pool1a_out, pool3_out) + # conv 4 + conv4_out = self.conv4(merge1_out) + iaf4_out = self.iaf4(conv4_out) + pool4_out = self.pool4(iaf4_out) + + flat_out = self.flat(pool4_out) + # fc 1 + fc1_out = self.fc1(flat_out) + iaf1_fc_out = self.iaf1_fc(fc1_out) + # fc 2 + fc2_out = self.fc2(iaf1_fc_out) + iaf2_fc_out = self.iaf2_fc(fc2_out) + # fc 3 + fc3_out = self.fc3(iaf2_fc_out) + iaf3_fc_out = self.iaf3_fc(fc3_out) + # fc 4 + fc4_out = self.fc4(iaf3_fc_out) + iaf4_fc_out = self.iaf4_fc(fc4_out) + # fc 5 + fc5_out = self.fc5(iaf4_fc_out) + iaf5_fc_out = self.iaf5_fc(fc5_out) + + return iaf5_fc_out \ No newline at end of file diff --git a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN6.py b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN6.py new file mode 100644 index 00000000..98221913 --- /dev/null +++ b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN6.py @@ -0,0 +1,141 @@ +import torch.nn as nn +import sinabs.layers as sl +from sinabs.exodus.layers import IAFSqueeze + +class SCNN(nn.Module): + def __init__(self, input_size, nb_classes, batch_size, surrogate_fn, min_v_mem=-0.313, spk_thr=2.0) -> None: + super().__init__() + + self.conv1 = nn.Conv2d(2, 8, 2, 1, bias=False) + self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool1 = sl.SumPool2d(2,2) + self.pool1a = sl.SumPool2d(8,8) + + self.conv2 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool2 = sl.SumPool2d(2,2) + + self.conv3 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool3 = sl.SumPool2d(2,2) + + self.conv4 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool4 = sl.SumPool2d(2,2) + + self.flat = nn.Flatten() + + flat_s = SCNN.get_flatten_size(input_size) + + self.fc1 = nn.Linear(flat_s, 100, bias=False) + self.iaf1_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc2 = nn.Linear(100, 100, bias=False) + self.iaf2_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc3 = nn.Linear(100, 100, bias=False) + self.iaf3_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc4 = nn.Linear(100, 100, bias=False) + self.iaf4_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc5 = nn.Linear(100, nb_classes, bias=False) + self.iaf5_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + # skip + + self.merge1 = sl.Merge() + self.merge2 = sl.Merge() + + @staticmethod + def get_flatten_size(input_size): + conv1_dims = SCNN.conv2d_output_size(input_size, 8, (2, 2)) + pool1_dims = SCNN.pool_output_size(conv1_dims, 8, (2, 2)) + + conv2_dims = SCNN.conv2d_output_size(pool1_dims, 8, (2, 2)) + pool2_dims = SCNN.pool_output_size(conv2_dims, 8, (2, 2)) + + conv3_dims = SCNN.conv2d_output_size(pool2_dims, 8, (2, 2)) + pool3_dims = SCNN.pool_output_size(conv3_dims, 8, (2, 2)) + + conv4_dims = SCNN.conv2d_output_size(pool3_dims, 8, (2, 2)) + pool4_dims = SCNN.pool_output_size(conv4_dims, 8, (2, 2)) + + return pool4_dims[0]*pool4_dims[1]*pool4_dims[2] + + @staticmethod + def conv2d_output_size(input_size, out_channels, kernel_size, stride=1, padding=0, dilation=1): + input_height, input_width, input_channels = input_size + kernel_height, kernel_width = kernel_size + + output_height = ((input_height + 2 * padding - dilation * (kernel_height - 1) - 1) // stride) + 1 + output_width = ((input_width + 2 * padding - dilation * (kernel_width - 1) - 1) // stride) + 1 + + return (output_height, output_width, out_channels) + + @staticmethod + def pool_output_size(input_size, out_channels, kernel_size, stride=None, padding=0): + input_height, input_width, input_channels = input_size + kernel_height, kernel_width = kernel_size + + if stride is None: + stride = kernel_height + + output_height = ((input_height + 2 * padding - kernel_height) // stride) + 1 + output_width = ((input_width + 2 * padding - kernel_width) // stride) + 1 + + return (output_height, output_width, out_channels) + + def detach_neuron_states(self): + for name, layer in self.named_modules(): + if name != '': + if isinstance(layer, sl.StatefulLayer): + for name, buffer in layer.named_buffers(): + buffer.detach_() + + def init_weights(self): + for name, layer in self.named_modules(): + if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear): + nn.init.xavier_normal_(layer.weight.data) + + def forward(self, x): + # conv 1 + con1_out = self.conv1(x) + iaf1_out = self.iaf1(con1_out) + pool1_out = self.pool1(iaf1_out) + pool1a_out = self.pool1a(iaf1_out) + # conv 2 + conv2_out = self.conv2(pool1_out) + iaf2_out = self.iaf2(conv2_out) + pool2_out = self.pool2(iaf2_out) + # conv 3 + conv3_out = self.conv3(pool2_out) + iaf3_out = self.iaf3(conv3_out) + pool3_out = self.pool3(iaf3_out) + + merge1_out = self.merge1(pool1a_out, pool3_out) + # conv 4 + conv4_out = self.conv4(merge1_out) + iaf4_out = self.iaf4(conv4_out) + pool4_out = self.pool4(iaf4_out) + + flat_out = self.flat(pool4_out) + # fc 1 + fc1_out = self.fc1(flat_out) + iaf1_fc_out = self.iaf1_fc(fc1_out) + # fc 2 + fc2_out = self.fc2(iaf1_fc_out) + iaf2_fc_out = self.iaf2_fc(fc2_out) + # fc 3 + fc3_out = self.fc3(iaf2_fc_out) + iaf3_fc_out = self.iaf3_fc(fc3_out) + + merge2_out = self.merge2(iaf1_fc_out, iaf3_fc_out) + # fc 4 + fc4_out = self.fc4(merge2_out) + iaf4_fc_out = self.iaf4_fc(fc4_out) + # fc 5 + fc5_out = self.fc5(iaf4_fc_out) + iaf5_fc_out = self.iaf5_fc(fc5_out) + + return iaf5_fc_out \ No newline at end of file diff --git a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN7.py b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN7.py new file mode 100644 index 00000000..084e9675 --- /dev/null +++ b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN7.py @@ -0,0 +1,142 @@ +import torch.nn as nn +import sinabs.layers as sl +from sinabs.exodus.layers import IAFSqueeze + +class SCNN(nn.Module): + def __init__(self, input_size, nb_classes, batch_size, surrogate_fn, min_v_mem=-0.313, spk_thr=2.0) -> None: + super().__init__() + + self.conv1 = nn.Conv2d(2, 8, 2, 1, bias=False) + self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool1 = sl.SumPool2d(2,2) + self.pool1a = sl.SumPool2d(4,4) + self.pool1b = sl.SumPool2d(8,8) + + self.conv2 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool2 = sl.SumPool2d(2,2) + + self.conv3 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool3 = sl.SumPool2d(2,2) + + self.conv4 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool4 = sl.SumPool2d(2,2) + + self.flat = nn.Flatten() + + flat_s = SCNN.get_flatten_size(input_size) + + self.fc1 = nn.Linear(flat_s, 100, bias=False) + self.iaf1_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc2 = nn.Linear(100, 100, bias=False) + self.iaf2_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc3 = nn.Linear(100, 100, bias=False) + self.iaf3_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc4 = nn.Linear(100, 100, bias=False) + self.iaf4_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc5 = nn.Linear(100, nb_classes, bias=False) + self.iaf5_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + # skip + + self.merge1a = sl.Merge() + self.merge1b = sl.Merge() + self.merge1c = sl.Merge() + + @staticmethod + def get_flatten_size(input_size): + conv1_dims = SCNN.conv2d_output_size(input_size, 8, (2, 2)) + pool1_dims = SCNN.pool_output_size(conv1_dims, 8, (2, 2)) + + conv2_dims = SCNN.conv2d_output_size(pool1_dims, 8, (2, 2)) + pool2_dims = SCNN.pool_output_size(conv2_dims, 8, (2, 2)) + + conv3_dims = SCNN.conv2d_output_size(pool2_dims, 8, (2, 2)) + pool3_dims = SCNN.pool_output_size(conv3_dims, 8, (2, 2)) + + conv4_dims = SCNN.conv2d_output_size(pool3_dims, 8, (2, 2)) + pool4_dims = SCNN.pool_output_size(conv4_dims, 8, (2, 2)) + + return pool4_dims[0]*pool4_dims[1]*pool4_dims[2] + + @staticmethod + def conv2d_output_size(input_size, out_channels, kernel_size, stride=1, padding=0, dilation=1): + input_height, input_width, input_channels = input_size + kernel_height, kernel_width = kernel_size + + output_height = ((input_height + 2 * padding - dilation * (kernel_height - 1) - 1) // stride) + 1 + output_width = ((input_width + 2 * padding - dilation * (kernel_width - 1) - 1) // stride) + 1 + + return (output_height, output_width, out_channels) + + @staticmethod + def pool_output_size(input_size, out_channels, kernel_size, stride=None, padding=0): + input_height, input_width, input_channels = input_size + kernel_height, kernel_width = kernel_size + + if stride is None: + stride = kernel_height + + output_height = ((input_height + 2 * padding - kernel_height) // stride) + 1 + output_width = ((input_width + 2 * padding - kernel_width) // stride) + 1 + + return (output_height, output_width, out_channels) + + def detach_neuron_states(self): + for name, layer in self.named_modules(): + if name != '': + if isinstance(layer, sl.StatefulLayer): + for name, buffer in layer.named_buffers(): + buffer.detach_() + + def init_weights(self): + for name, layer in self.named_modules(): + if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear): + nn.init.xavier_normal_(layer.weight.data) + + def forward(self, x): + # conv 1 + con1_out = self.conv1(x) + iaf1_out = self.iaf1(con1_out) + pool1_out = self.pool1(iaf1_out) + pool1a_out = self.pool1a(iaf1_out) + pool1b_out = self.pool1b(iaf1_out) + # conv 2 + conv2_out = self.conv2(pool1_out) + iaf2_out = self.iaf2(conv2_out) + pool2_out = self.pool2(iaf2_out) + # conv 3 + merge_1a_out = self.merge1a(pool1a_out, pool2_out) + conv3_out = self.conv3(merge_1a_out) + iaf3_out = self.iaf3(conv3_out) + pool3_out = self.pool3(iaf3_out) + # conv 4 + merge_1b_out = self.merge1b(pool1b_out, pool3_out) + conv4_out = self.conv4(merge_1b_out) + iaf4_out = self.iaf4(conv4_out) + pool4_out = self.pool4(iaf4_out) + + flat_out = self.flat(pool4_out) + # fc 1 + fc1_out = self.fc1(flat_out) + iaf1_fc_out = self.iaf1_fc(fc1_out) + # fc 2 + fc2_out = self.fc2(iaf1_fc_out) + iaf2_fc_out = self.iaf2_fc(fc2_out) + # fc 3 + fc3_out = self.fc3(iaf2_fc_out) + iaf3_fc_out = self.iaf3_fc(fc3_out) + # fc 4 + fc4_out = self.fc4(iaf3_fc_out) + iaf4_fc_out = self.iaf4_fc(fc4_out) + # fc 5 + fc5_out = self.fc5(iaf4_fc_out) + iaf5_fc_out = self.iaf5_fc(fc5_out) + + return iaf5_fc_out \ No newline at end of file diff --git a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN8.py b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN8.py new file mode 100644 index 00000000..c99af380 --- /dev/null +++ b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN8.py @@ -0,0 +1,146 @@ +import torch.nn as nn +import sinabs.layers as sl +from sinabs.exodus.layers import IAFSqueeze + +class SCNN(nn.Module): + def __init__(self, input_size, nb_classes, batch_size, surrogate_fn, min_v_mem=-0.313, spk_thr=2.0) -> None: + super().__init__() + + self.conv1 = nn.Conv2d(2, 8, 2, 1, bias=False) + self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool1 = sl.SumPool2d(2,2) + self.pool1a = sl.SumPool2d(4,4) + self.pool1b = sl.SumPool2d(8,8) + + self.conv2 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool2 = sl.SumPool2d(2,2) + + self.conv3 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool3 = sl.SumPool2d(2,2) + + self.conv4 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool4 = sl.SumPool2d(2,2) + + self.flat = nn.Flatten() + + flat_s = SCNN.get_flatten_size(input_size) + + self.fc1 = nn.Linear(flat_s, 100, bias=False) + self.iaf1_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc2 = nn.Linear(100, 100, bias=False) + self.iaf2_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc3 = nn.Linear(100, 100, bias=False) + self.iaf3_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc4 = nn.Linear(100, 100, bias=False) + self.iaf4_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc5 = nn.Linear(100, nb_classes, bias=False) + self.iaf5_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + # skip + + self.merge1a = sl.Merge() + self.merge1b = sl.Merge() + + self.merge_fc1a = sl.Merge() + self.merge_fc1b = sl.Merge() + + @staticmethod + def get_flatten_size(input_size): + conv1_dims = SCNN.conv2d_output_size(input_size, 8, (2, 2)) + pool1_dims = SCNN.pool_output_size(conv1_dims, 8, (2, 2)) + + conv2_dims = SCNN.conv2d_output_size(pool1_dims, 8, (2, 2)) + pool2_dims = SCNN.pool_output_size(conv2_dims, 8, (2, 2)) + + conv3_dims = SCNN.conv2d_output_size(pool2_dims, 8, (2, 2)) + pool3_dims = SCNN.pool_output_size(conv3_dims, 8, (2, 2)) + + conv4_dims = SCNN.conv2d_output_size(pool3_dims, 8, (2, 2)) + pool4_dims = SCNN.pool_output_size(conv4_dims, 8, (2, 2)) + + return pool4_dims[0]*pool4_dims[1]*pool4_dims[2] + + @staticmethod + def conv2d_output_size(input_size, out_channels, kernel_size, stride=1, padding=0, dilation=1): + input_height, input_width, input_channels = input_size + kernel_height, kernel_width = kernel_size + + output_height = ((input_height + 2 * padding - dilation * (kernel_height - 1) - 1) // stride) + 1 + output_width = ((input_width + 2 * padding - dilation * (kernel_width - 1) - 1) // stride) + 1 + + return (output_height, output_width, out_channels) + + @staticmethod + def pool_output_size(input_size, out_channels, kernel_size, stride=None, padding=0): + input_height, input_width, input_channels = input_size + kernel_height, kernel_width = kernel_size + + if stride is None: + stride = kernel_height + + output_height = ((input_height + 2 * padding - kernel_height) // stride) + 1 + output_width = ((input_width + 2 * padding - kernel_width) // stride) + 1 + + return (output_height, output_width, out_channels) + + def detach_neuron_states(self): + for name, layer in self.named_modules(): + if name != '': + if isinstance(layer, sl.StatefulLayer): + for name, buffer in layer.named_buffers(): + buffer.detach_() + + def init_weights(self): + for name, layer in self.named_modules(): + if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear): + nn.init.xavier_normal_(layer.weight.data) + + def forward(self, x): + # conv 1 + con1_out = self.conv1(x) + iaf1_out = self.iaf1(con1_out) + pool1_out = self.pool1(iaf1_out) + pool1a_out = self.pool1a(iaf1_out) + pool1b_out = self.pool1b(iaf1_out) + # conv 2 + conv2_out = self.conv2(pool1_out) + iaf2_out = self.iaf2(conv2_out) + pool2_out = self.pool2(iaf2_out) + # conv 3 + merge_1a_out = self.merge1a(pool1a_out, pool2_out) + conv3_out = self.conv3(merge_1a_out) + iaf3_out = self.iaf3(conv3_out) + pool3_out = self.pool3(iaf3_out) + # conv 4 + merge_1b_out = self.merge1b(pool1b_out, pool3_out) + conv4_out = self.conv4(merge_1b_out) + iaf4_out = self.iaf4(conv4_out) + pool4_out = self.pool4(iaf4_out) + + flat_out = self.flat(pool4_out) + # fc 1 + fc1_out = self.fc1(flat_out) + iaf1_fc_out = self.iaf1_fc(fc1_out) + # fc 2 + fc2_out = self.fc2(iaf1_fc_out) + iaf2_fc_out = self.iaf2_fc(fc2_out) + # fc 3 + merge_fc1a_out = self.merge_fc1a(iaf1_fc_out, iaf2_fc_out) + fc3_out = self.fc3(merge_fc1a_out) + iaf3_fc_out = self.iaf3_fc(fc3_out) + # fc 4 + merge_fc1b_out = self.merge_fc1b(iaf1_fc_out, iaf3_fc_out) + fc4_out = self.fc4(merge_fc1b_out) + iaf4_fc_out = self.iaf4_fc(fc4_out) + # fc 5 + fc5_out = self.fc5(iaf4_fc_out) + iaf5_fc_out = self.iaf5_fc(fc5_out) + + return iaf5_fc_out \ No newline at end of file From 3d5f6eeff041acde4669bcc188a30b81282fce68 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Thu, 2 May 2024 14:11:36 +0200 Subject: [PATCH 071/379] methods for weight initialization taking into account SumPool2d layers --- .../baseline-SCNN-example_3-SumPool.ipynb | 1539 +++++++++++++++++ .../exp_set_A/baseline-SCNN-example_3.ipynb | 1500 ++++++++++++++++ .../single_training.ipynb | 1401 +++++++++++++++ .../using_SumPool2d/models/ResSCNN1.py | 50 +- .../using_SumPool2d/models/ResSCNN2.py | 54 +- .../using_SumPool2d/models/ResSCNN3.py | 54 +- .../using_SumPool2d/models/ResSCNN4.py | 52 +- .../utils/weight_initialization.py | 25 + 8 files changed, 4577 insertions(+), 98 deletions(-) create mode 100644 tests/test_nonsequential/using_AvgPool2d/exp_set_A/baseline-SCNN-example_3-SumPool.ipynb create mode 100644 tests/test_nonsequential/using_AvgPool2d/exp_set_A/baseline-SCNN-example_3.ipynb create mode 100644 tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/single_training.ipynb create mode 100644 tests/test_nonsequential/utils/weight_initialization.py diff --git a/tests/test_nonsequential/using_AvgPool2d/exp_set_A/baseline-SCNN-example_3-SumPool.ipynb b/tests/test_nonsequential/using_AvgPool2d/exp_set_A/baseline-SCNN-example_3-SumPool.ipynb new file mode 100644 index 00000000..a25e3392 --- /dev/null +++ b/tests/test_nonsequential/using_AvgPool2d/exp_set_A/baseline-SCNN-example_3-SumPool.ipynb @@ -0,0 +1,1539 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import torch, random, sys\n", + "import torch.nn as nn\n", + "import sinabs.layers as sl\n", + "from tqdm.notebook import tqdm\n", + "\n", + "from tonic.datasets.dvsgesture import DVSGesture\n", + "from tonic.transforms import ToFrame\n", + "from torch.utils.data import DataLoader\n", + "from torch.nn import CrossEntropyLoss\n", + "from torch.optim import Adam\n", + "\n", + "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential\n", + "from sinabs.exodus.layers import IAFSqueeze\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "\n", + "sys.path.append('../../utils')\n", + "\n", + "from weight_initialization import rescale_method_1\n", + "import tonic" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "torch.backends.cudnn.enabled = False\n", + "torch.backends.cudnn.deterministic = True\n", + "random.seed(1)\n", + "torch.manual_seed(1)\n", + "torch.cuda.manual_seed(1)\n", + "np.random.seed(1)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Loading Data" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "batch_size = 3\n", + "num_workers = 1\n", + "epochs = 30\n", + "lr = 1e-3" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "root_dir = \"../../DVSGESTURE\"\n", + "_ = DVSGesture(save_to=root_dir, train=True)\n", + "_ = DVSGesture(save_to=root_dir, train=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "n_time_steps = 50\n", + "to_raster = ToFrame(sensor_size=DVSGesture.sensor_size, n_time_bins=n_time_steps)\n", + "\n", + "snn_train_dataset = DVSGesture(save_to=root_dir, train=True, transform=to_raster)\n", + "snn_test_dataset = DVSGesture(save_to=root_dir, train=False, transform=to_raster)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The transformed array is in shape [Time-Step, Channel, Height, Width] --> (50, 2, 128, 128)\n" + ] + } + ], + "source": [ + "sample_data, label = snn_train_dataset[0]\n", + "print(f\"The transformed array is in shape [Time-Step, Channel, Height, Width] --> {sample_data.shape}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "disk_cache_train = tonic.DiskCachedDataset(\n", + " dataset=snn_train_dataset,\n", + " cache_path='./cached_train'\n", + ")\n", + "snn_train_dataloader = DataLoader(disk_cache_train, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True)\n", + "\n", + "disk_cache_test = tonic.DiskCachedDataset(\n", + " dataset=snn_test_dataset,\n", + " cache_path='./cached_test'\n", + ")\n", + "snn_test_dataloader = DataLoader(disk_cache_test, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=False)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Network Module\n", + "\n", + "We need to define a `nn.Module` implementing the network we want the chip to reproduce." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "device: NVIDIA GeForce RTX 3070 Ti\n" + ] + } + ], + "source": [ + "if torch.cuda.is_available():\n", + " device = torch.device('cuda:0')\n", + " print('device: ', torch.cuda.get_device_name(0))\n", + "else:\n", + " device = torch.device('cpu')" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "class SNN(nn.Module):\n", + " def __init__(self) -> None:\n", + " super().__init__()\n", + "\n", + " self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False)\n", + " self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + " self.pool1 = sl.SumPool2d(2,2)\n", + "\n", + " self.conv2 = nn.Conv2d(10, 10, 2, 1, bias=False)\n", + " self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + " self.pool2 = sl.SumPool2d(3,3)\n", + "\n", + " self.conv3 = nn.Conv2d(10, 10, 3, 1, bias=False)\n", + " self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + " self.pool3 = sl.SumPool2d(2,2)\n", + "\n", + " self.flat = nn.Flatten()\n", + "\n", + " self.fc1 = nn.Linear(810, 100, bias=False)\n", + " self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + "\n", + " self.fc2 = nn.Linear(100, 100, bias=False)\n", + " self.iaf5 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + "\n", + " self.fc3 = nn.Linear(100, 100, bias=False)\n", + " self.iaf6 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + "\n", + " self.fc4 = nn.Linear(100, 11, bias=False)\n", + " self.iaf7 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + "\n", + " def detach_neuron_states(self):\n", + " for name, layer in self.named_modules():\n", + " if name != '':\n", + " if isinstance(layer, sl.StatefulLayer):\n", + " for name, buffer in layer.named_buffers():\n", + " buffer.detach_()\n", + "\n", + " def init_weights(self):\n", + " for name, layer in self.named_modules():\n", + " if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear):\n", + " nn.init.xavier_normal_(layer.weight.data)\n", + "\n", + " def rescale_conv_weights(self, rescale_fn, lambda_):\n", + " rescale_fn(self.conv2, [(2, 2)], lambda_)\n", + " rescale_fn(self.conv3, [(3, 3)], lambda_)\n", + "\n", + " def forward(self, x):\n", + " \n", + " con1_out = self.conv1(x)\n", + " iaf1_out = self.iaf1(con1_out)\n", + " pool1_out = self.pool1(iaf1_out)\n", + "\n", + " conv2_out = self.conv2(pool1_out)\n", + " iaf2_out = self.iaf2(conv2_out)\n", + " pool2_out = self.pool2(iaf2_out)\n", + "\n", + " conv3_out = self.conv3(pool2_out)\n", + " iaf3_out = self.iaf3(conv3_out)\n", + " pool3_out = self.pool3(iaf3_out)\n", + "\n", + " flat_out = self.flat(pool3_out)\n", + " \n", + " fc1_out = self.fc1(flat_out)\n", + " iaf4_out = self.iaf4(fc1_out)\n", + "\n", + " fc2_out = self.fc2(iaf4_out)\n", + " iaf5_out = self.iaf5(fc2_out)\n", + "\n", + " fc3_out = self.fc3(iaf5_out)\n", + " iaf6_out = self.iaf6(fc3_out)\n", + "\n", + " fc4_out = self.fc4(iaf6_out)\n", + " iaf7_out = self.iaf7(fc4_out)\n", + "\n", + " return iaf7_out" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "snn = SNN().to(device)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "snn.init_weights()" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "recaling factor: 2.0 (computed using 1 kernels and lambda 0.5)\n", + "recaling factor: 4.5 (computed using 1 kernels and lambda 0.5)\n" + ] + } + ], + "source": [ + "lambda_ = 0.5\n", + "snn.rescale_conv_weights(rescale_method_1, lambda_)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "optimizer = Adam(snn.parameters(), lr=lr, betas=(0.9, 0.999), eps=1e-8)\n", + "loss_fn = CrossEntropyLoss()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Define train and test" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "def train(dataloader, model, loss_fn, optimizer, epochs, test_func, dataloader_test):\n", + " epochs_y = []\n", + " epochs_x = []\n", + " epochs_acc = []\n", + " model.train()\n", + "\n", + " for e in range(epochs):\n", + " losses = []\n", + " batches = []\n", + " batch_count = 0\n", + " train_p_bar = tqdm(snn_train_dataloader)\n", + "\n", + " for X, y in train_p_bar:\n", + " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", + " X = X.reshape(-1, 2, 128, 128).to(dtype=torch.float, device=device)\n", + " y = y.to(dtype=torch.long, device=device)\n", + "\n", + " # forward\n", + " pred = model(X)\n", + "\n", + " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", + " pred = pred.reshape(batch_size, n_time_steps, -1)\n", + "\n", + " # accumulate all time-steps output for final prediction\n", + " pred = pred.sum(dim = 1)\n", + " loss = loss_fn(pred, y)\n", + "\n", + " # gradient update\n", + " optimizer.zero_grad()\n", + " loss.backward()\n", + " optimizer.step()\n", + "\n", + " # detach the neuron states and activations from current computation graph(necessary)\n", + " model.detach_neuron_states()\n", + "\n", + " train_p_bar.set_description(f\"Epoch {e} - BPTT Training Loss: {round(loss.item(), 4)}\")\n", + "\n", + " batch_count += 1\n", + " losses.append(loss.item())\n", + " batches.append(batch_count)\n", + "\n", + " epochs_y.append(losses)\n", + " epochs_x.append(batches)\n", + "\n", + " acc = test_func(dataloader_test, model)\n", + " print(f'Epoch {e} accuracy: {acc}')\n", + " epochs_acc.append(acc)\n", + "\n", + " return epochs_x, epochs_y, epochs_acc\n" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "def test(dataloader, model):\n", + " correct_predictions = []\n", + " with torch.no_grad():\n", + " test_p_bar = tqdm(dataloader)\n", + " for X, y in test_p_bar:\n", + " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", + " X = X.reshape(-1, 2, 128, 128).to(dtype=torch.float, device=device)\n", + " y = y.to(dtype=torch.long, device=device)\n", + "\n", + " # forward\n", + " output = model(X)\n", + "\n", + " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", + " output = output.reshape(batch_size, n_time_steps, -1)\n", + "\n", + " # accumulate all time-steps output for final prediction\n", + " output = output.sum(dim=1)\n", + "\n", + " # calculate accuracy\n", + " pred = output.argmax(dim=1, keepdim=True)\n", + "\n", + " # compute the total correct predictions\n", + " correct_predictions.append(pred.eq(y.view_as(pred)))\n", + "\n", + " test_p_bar.set_description(f\"Testing Model...\")\n", + " \n", + " correct_predictions = torch.cat(correct_predictions)\n", + " return correct_predictions.sum().item()/(len(correct_predictions))*100" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Training loop" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "d7307153ad334c27a70afe651a5daaf4", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/359 [00:00" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "y_avg = []\n", + "for y in epochs_y:\n", + " y_avg.append(np.mean(y))\n", + "\n", + "plt.plot(np.arange(len(epochs_x)), y_avg, marker = '.')\n", + "plt.xlabel('epoch')\n", + "plt.ylabel('average loss')\n", + "plt.ylim(0,)\n", + "plt.xticks(np.arange(len(epochs_x)))\n", + "for i, txt in enumerate(y_avg):\n", + " if i%2 == 0:\n", + " pass\n", + " else:\n", + " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAG2CAYAAACZEEfAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAAB8eUlEQVR4nO3dd1TV9f8H8OdlXPaQDTLdey/cJuLKnZqZWprmNyxHmSPN0hIzLcvM9ctVrjS3pbkHoYLgVgREUaaIXPa69/37g7h5Zd0rF4Hr83HOPUc+n/f6wJX74j0lQggBIiIiIh2lV9kNICIiIqpIDHaIiIhIpzHYISIiIp3GYIeIiIh0GoMdIiIi0mkMdoiIiEinMdghIiIincZgh4iIiHQagx0iIiLSaQx2iIiISKdVarBz9uxZDBgwAC4uLpBIJNi3b5/KfSEEPv/8czg7O8PExAQ+Pj4IDw9XSZOcnIzRo0fD0tIS1tbWmDBhAtLT01/iUxAREVFVVqnBTkZGBpo3b45Vq1YVe3/p0qX48ccfsWbNGly8eBFmZmbo3bs3srOzlWlGjx6Nmzdv4tixYzh06BDOnj2LSZMmvaxHICIioipOUlUOApVIJNi7dy8GDx4MoKBXx8XFBR9//DE++eQTAIBMJoOjoyM2bdqEN998E7dv30ajRo0QFBSENm3aAACOHDmCfv364dGjR3BxcamsxyEiIqIqwqCyG1CSqKgoxMfHw8fHR3nNysoK7du3R2BgIN58800EBgbC2tpaGegAgI+PD/T09HDx4kUMGTKk2LJzcnKQk5Oj/FqhUCA5ORm2traQSCQV91BERESkNUIIpKWlwcXFBXp6JQ9WVdlgJz4+HgDg6Oioct3R0VF5Lz4+Hg4ODir3DQwMYGNjo0xTHH9/f3z55ZdabjERERFVhocPH8LV1bXE+1U22KlIc+bMwYwZM5Rfy2QyuLu74+HDh7C0tKzElhEREZG6UlNT4ebmBgsLi1LTVdlgx8nJCQCQkJAAZ2dn5fWEhAS0aNFCmSYxMVElX35+PpKTk5X5i2NkZAQjI6Mi1y0tLRnsEBERVTNlTUGpsvvseHl5wcnJCSdOnFBeS01NxcWLF+Ht7Q0A8Pb2RkpKCi5fvqxMc/LkSSgUCrRv3/6lt5mIiIiqnkrt2UlPT0dERITy66ioKFy5cgU2NjZwd3fHtGnT8NVXX6Fu3brw8vLC/Pnz4eLiolyx1bBhQ/Tp0wcTJ07EmjVrkJeXhylTpuDNN9/kSiwiIiICUMnBTnBwMHr06KH8unAezbhx47Bp0yZ8+umnyMjIwKRJk5CSkoLOnTvjyJEjMDY2VubZunUrpkyZgp49e0JPTw/Dhg3Djz/++NKfhYiIiKqmKrPPTmVKTU2FlZUVZDIZ5+wQERFVE+p+flfZOTtERERE2sBgh4iIiHQagx0iIiLSaQx2iIiISKcx2CEiIiKdxmCHiIiIdBqDHSIiItJpDHaIiIhIpzHYISIiIp3GYIeIiIh0GoMdIiIi0mkMdoiIiEinMdghIiIincZgh4iIiHQagx0iIiLSaQx2iIiISKcx2CEiIiKdxmCHiIiIdBqDHSIiItJpDHaIiIhIpzHYISIiIp3GYIeIiIh0GoMdIiIi0mkMdoiIiEinMdghIiIincZgh4iIiHQagx0iIiLSaQx2iIiISKcx2CEiIiKdxmCHiIiIdBqDHSIiItJpDHaIiIhIpzHYISIiIp3GYIeIiIh0GoMdIiIi0mkMdoiIiEinMdghIiIincZgh4iIiHQagx0iIiLSaQx2iIiISKcx2CEiIiKdxmCHiIiIdBqDHSIiItJpDHaIiIhIpzHYISIiIp3GYIeIiIh0GoMdIiIi0mkMdoiIiEinMdghIiIincZgh4iIiHQagx0iIiLSaQx2iIiISKcx2CEiIiKdxmCHiIiIdBqDHSIiItJpDHaIiIhIpzHYISIiIp3GYIeIiIh0GoMdIqqWPD09IZFIirz8/PwAAJGRkRgyZAjs7e1haWmJESNGICEhodQyz549iwEDBsDFxQUSiQT79u0rkqa4OiUSCb799tuKeEwi0gIGO0RULQUFBSEuLk75OnbsGABg+PDhyMjIgK+vLyQSCU6ePImAgADk5uZiwIABUCgUJZaZkZGB5s2bY9WqVSWmebbOuLg4bNiwARKJBMOGDdP6MxKRdhhUdgOIiF6Evb29ytdLlixB7dq10a1bNxw7dgz3799HaGgoLC0tAQCbN29GjRo1cPLkSfj4+BRbZt++fdG3b99S63VyclL5ev/+/ejRowdq1apVjqchoorEnh0iqvZyc3Px22+/Yfz48ZBIJMjJyYFEIoGRkZEyjbGxMfT09HD+/Hmt1ZuQkIDDhw9jwoQJWiuTiLSPwQ4RVXv79u1DSkoK3nnnHQBAhw4dYGZmhlmzZiEzMxMZGRn45JNPIJfLERcXp7V6N2/eDAsLCwwdOlRrZVLxypqjFR8fjzFjxsDJyQlmZmZo1aoV/vjjjzLLXbVqFTw9PWFsbIz27dvj0qVLKvfff/99GBoavpJ1165dGyYmJrC3t8egQYNw586dMsutqqp0sCOXyzF//nx4eXnBxMQEtWvXxqJFiyCEUKYRQuDzzz+Hs7MzTExM4OPjg/Dw8EpsNRG9bL/88gv69u0LFxcXAAVDXLt27cLBgwdhbm4OKysrpKSkoFWrVtDT096vvQ0bNmD06NEwNjbWWplUvNLmaAHA2LFjERYWhgMHDuD69esYOnQoRowYgdDQ0BLL3LlzJ2bMmIEFCxYgJCQEzZs3R+/evZGYmKhM07p1a/zxxx+4dOkSjh49il69eimHUHW97o0bN+L27ds4evQohBDw9fWFXC4vsdwqTVRhX3/9tbC1tRWHDh0SUVFRYteuXcLc3Fz88MMPyjRLliwRVlZWYt++feLq1ati4MCBwsvLS2RlZaldj0wmEwCETCariMcgogp0//59oaenJ/bt21fs/cePH4unT58KIYRwdHQUS5cuVatcAGLv3r3F3vPw8BAAirw++OADERUVVew9AOL3338vtrzc3Fzx6aefiiZNmghTU1Ph7OwsxowZI2JiYpRpTp06VWK5ly5dUuuZdMnUqVNF7dq1hUKhEEIIYWZmJrZs2aKSxsbGRqxfv77EMtq1ayf8/PyUX8vlcuHi4iL8/f1LzHP16lUBQLi7u7+SdUdERJSYpjKo+/ldpYOd/v37i/Hjx6tcGzp0qBg9erQQQgiFQiGcnJzEt99+q7yfkpIijIyMxPbt29Wuh8EOUfW1YMEC4eTkJPLy8kpNd+LECSGRSMSdO3fUKre0YCcxMVGMGDFCNGvWTMTFxYljx44JAOLUqVMiPz9fxMXFqby+/PJLYW5uLtLS0ootLyUlRfj4+IidO3eKO3fuiMDAQNGuXTvRunVrZZqcnJwi5b733nvCy8tL+cH3qsjJyRG2trbi66+/Vl7r1auX6N+/v3jy5ImQy+Vi+/btwtTUVISHh5dYhr6+fpGf8dixY8XAgQOLzZOeni4+/PBDoaenJxYuXPhK1T1t2jTh5eUlcnJyik1TWXQi2Pn666+Fh4eHCAsLE0IIceXKFeHg4CB+++03IYQQkZGRAoAIDQ1Vyde1a1fx0UcflVhudna2kMlkytfDhw8Z7BBVQ3K5XLi7u4tZs2YVubdhwwYRGBgoIiIixK+//ipsbGzEjBkzVNK89tprYuXKlcqv09LSRGhoqAgNDRUAxHfffSdCQ0PFgwcPVPLJZDJhamoqVq9eLYQo2svwvBYtWhT5w60sly5dEgCK1F1ar1Khf/75R/To0UOYmpoKCwsL0aVLF5GZmVliXfn5+WLevHnC09NTGBsbi1q1aomFCxeqPE98fLwYN26ccHZ2FiYmJqJ3797i7t27Gj2TtuzcuVPo6+ur9Hw9ffpU+Pr6CgDCwMBAWFpaiqNHj5ZYRkxMjAAg/vnnH5XrM2fOFO3atVO5tmrVKmFmZiYACGdn51ey7vr161e5Xh0hdCTYkcvlYtasWUIikQgDAwMhkUjE4sWLlfcDAgIEABEbG6uSb/jw4WLEiBEllrtgwYJif1kw2CGqXo4ePSoAKP8getasWbOEo6OjMDQ0FHXr1hXLly8vEoy4urmL8R/OFLEpBYFASUNF48aNU8m3du1aYWJiIlJSUortZXhWcHCwACACAgI0erZjx44JiURS5PdSYmKiiIuLE+vXrxcSiURs27ZN2askREGgY2lpKfz9/cWNGzfEnTt3xM6dO0V2dnaJdZU1ZUChUIgOHTqILl26iEuXLok7d+6ISZMmCXd3d5Genq7Rc2mDr6+veP3111WuTZkyRbRr104cP35cXLlyRXzxxRfCyspKXLt2rdgyNPnQT0lJEXfv3hVnzpwR9vb2wtLSUmWqxKtQ94ABA0SrVq00miLyMuhEsLN9+3bh6uoqtm/fLq5duya2bNkibGxsxKZNm4QQLx7ssGeHiHZceiC8Zh8SHrMOCa/Zh8SOSw/KzlSM4noZnvW///1PNGzYUKMys7KyRKtWrcRbb71VYpq+ffuKvn37FulVat++vZg3b55G9ZU1ZSAsLEwAEDdu3FDel8vlwt7evtS5IRWhuDlaERERRdonhBA9e/YU77//frHlvMhwTmHdUqlUbNu27ZWpuzCfqampsu6qQt1gp0qvxpo5cyZmz56NN998E02bNsWYMWMwffp0+Pv7A/hvc6/nt4BPSEgosvHXs4yMjGBpaanyIqJXR5wsC3P2XIfi34WdCgHM3XMDcbIsjct6fiXYs7KysrBt2zaN9uHJy8vDiBEjIITA6tWri03z6NEjHD16FGPHjlXZXygxMREXL16Eg4MDOnbsCEdHR3Tr1q3MvYU6duyIEydO4O7duwCAq1ev4vz588oNFnNycgBAZdWZnp4ejIyMtLpvkTo2btwIBwcH9O/fX3ktMzNT2aZn6evrl7hjtlQqRevWrXHixAnlNYVCgRMnTsDb27vUuvX09JTfk1ehbqBg5bMQQll3tfMyIq8XZWNjI37++WeVa4sXLxZ169YVQvw3QXnZsmXK+zKZjBOUiahUARGPhcesQ0Ve58Mfa1ROWSvBtmzZIgwNDUViYqJa5eXm5orBgweLZs2aiaSkpBLTLVy4UNjb24utW7eq9CoFBgYKAMLGxkZs2LBBhISEiGnTpgmpVFrq/Jqypgzk5uYKd3d3MXz4cJGcnCxycnLEkiVLBADh6+ur1rNpQ0lztHJzc0WdOnVEly5dxMWLF0VERIRYtmyZkEgk4vDhw8p0z8/R2rFjhzAyMhKbNm0St27dEpMmTRLW1tYiPj5eCFEwL3Tx4sUiODhYREVFCUdHR1GnTh1hY2MjEhISXmrd9g4Ooqa7l6hR4+XW/eDBAxEQECAGDBig8txVhU4MY40bN07UrFlTOY68Z88eYWdnJz799FNlmiVLlghra2uxf/9+ce3aNTFo0CAuPSeiUsWmZArPYoKdEWv+ESkZuWqXU9ZKsG7duolhw4apVVZhoNO4ceNSgyOFQiG8vLzExx9/XGTuSuHQ/pw5c1TyNG3aVMyePbvEMsuaMiBEwdyj5s2bCwBCX19f9O7dW/Tt21f06dNHrefThtLmaN29e1cMHTpUODg4CFNTU9GsWbMiS7I9PDzEjFlzRUDEY+U8rZUrVwp3d3chlUpFu3btxIULF5TpY2JiRN++fYWDg4PQ19cXAMTrr79eZEVfRdetp1dQt3HttqLmxDUqQ64VXbehoaFwdXUVb731ltorGV8mdT+/JUI8s0NfFZOWlob58+dj7969SExMhIuLC0aNGoXPP/8cUqkUQEHX2oIFC7Bu3TqkpKSgc+fO+Pnnn1GvXj2160lNTYWVlRVkMhmHtIheAUIIdPA/gYTUgi55PQmgrydBnlzA09YU68e2QV1Hi1LLUCgU8PLywqhRo7BkyZIi9yMiIlCvXj38+eef6NOnT5H7DRo0gL+/P4YMGYK8vDy88cYbCAkJwaFDh+Do6KhMZ2Njo/x9BwAnTpyAj48Pjh8/Dl9fX+zZsweDBg0CAERFRaFWrVr49ddf8fbbbyvzjBw5EgYGBti6dWuxz+Lm5obZs2crd+UFgK+++gq//fZbkV1zZTIZcnNzYW9vj/bt26NNmzalHpxalewMilYOX+pJAP+hTTGyrXul1L14SFO83twFSWk5SEoveD1Oz0VSWg4ep+cor8enZiM2JbtIea7WxnCyMoGduRHsLKSwMzeCvYVRwdfmRrD/97qp1KBSn7uiqfv5XaUPArWwsMCKFSuwYsWKEtNIJBIsXLgQCxcufHkNIyIABVv4P3jwoMj1Dz74AKtWrUL37t1x5swZlXvvv/8+1qxZU2KZhX/ArF+/HikpKejUqRNWr16NunXrKtMkJyfjww8/xMGDB6Gnp4dhw4bhhx9+gLm5uVrtPh32GAmpOTAx1MPKUa3QuKYlkjNyMWnLZdx/konBqwKw4s2W6NXIscQyjh8/jujoaIwfP77Y+xs2bICrqyt8fX2LvR8WFgaZTAYAiImJwYEDBwAALVq0UEl36tQpdO/eXfn1L7/8go4dO+LcuXNF5q54enrCxcUFYWFhKmXcvXu31ANOMzMz1Z73YWVlBQAIDw9HcHAwFi1aVGK5VUlJ87S61rOHs5VJhdb9MDkDs/dch3im7tl7rmP2nusvXOajlGw8KiYIep6xoR6y8/77Ob7M565KqnSwQ0RVW1BQkMr28Tdu3ECvXr2UW9kDwMSJE1X+GDE1NS21zKVLl+LHH3/E5s2b4eXlhfnz56N37964deuWcoLs6NGjlUcG5OXl4d1338WkSZOwbds2tdq95kxkQTntPeDzb0DjbGWCA1M6wW9bCC7cS8bELcGY0asepvSoAz09SZEyfH19UVrH+OLFi7F48eIS7wshECfLwj+RSfCycyy1rGdt27ZN2as0btw4GBj892tcIpFg5syZWLBgAZo3b44WLVpg8+bNuHPnDnbv3q1M17NnTwwZMgRTpkwBAAwYMABff/013N3d0bhxY4SGhuK7775TCeR27doFe3t7uLu74/r165g6dSoGDx5cYjBX1UQlZSgDnUJyIbDyRDg+7FlX6x/8QgjciEnF3tAY7L78CCX9eE2l+v/2xkj/7aUp7Jkp+FoiAT7YGqLSdj0J8NOolhCQKHuFktJz8DgtV+Xr7DyFSqDz7HPfT8pksENEpI7Cc3oKLVmyBLVr10a3bt2U10xNTUtdHfksIQRWrFiBefPmKYdmtmzZAkdHR+zbtw9vvvkmbt++jSNHjiAoKAht2rQBAKxcuRL9+vXDsmXLil0V9azQ6Ke4GJUMAz0JJnTxUrlna26EXye0x9eHb2PTP/fx3bG7uB2XimXDm8PMSLu/LssztFBar9K0adOQnZ2N6dOnIzk5Gc2bN8exY8dQu3ZtZZrIyEgkJSUpv165ciXmz5+PDz74QDll4P3338fnn3+uTBMXF4cZM2YgISEBzs7OGDt2LObPn1+O78DLdTc+rdjr2y49xPagh/CuZYvBLWuibxMnWBgbvnA9D5MzceBqLPaGxiAiMb3EdHoS4Nj0bqjtUHZvpP/Qppi75wbkQkBfIsHioU3Qr1np73MhBDJy5bgVK8PIdRdUgi09CeBpV/ofHbqmSs/ZeVk4Z4eo/HJzc+Hi4oIZM2Zg7ty5AIDu3bvj5s2bEELAyckJAwYMwPz580vs3bl37x5q166N0NBQleGcbt26oUWLFvjhhx+wYcMGfPzxx3j69Knyfn5+PoyNjbFr1y4MGTKk1HZO/vUyjtyMx7BWrlg+onmJ6XYGRWPevhvIkws0cLLAujFt4G5b/g8IuULgz+ux+HD7FZXr+hIJzs/u8dL+2o6TZSEqKQNedmY6/xf+vcfpGLDyPDJy5ZCgYKdIPQkwuGVNPErOwqX7ycq0RgZ66NXIEUNa1kTXevYw1C97hxZZZh4OX4/DvtCYEstKSM3G/H03VQIWTebNxMmycD8pE552phr/vHYGRWPunuuQ//tpb2MmxemZ3WFZjqCuqtCJOTtEVH3s27cPKSkpeOedd5TX3nrrLXh4eMDFxQXXrl3DrFmzEBYWhj179hRbRnx8PACoTNAt/LrwXnx8PBwcHFTuGxgYwMbGRpmmJJGP03H0VkGa97vVKjXtyLbuqONggcm/Xcad+DQMXHUeq95qhU517ErNV5LbcanYFxqD/VdiEZ9adK7Fyxxa+OXcPXx1+LbyQ1+XJqw+LztPjinbQpGRK0eHWjZYNrw5HiZnqQQNhb0xe0IeIfJxBg5di8Oha3GwMZNiQDNnDG5ZEy3crBGfmq0MEG3MpDh15zH2hcbg5J1E5MoLhoskEsC7li2GtKyJPs/1EvVo4PDCAYuzlckLvzdGtnVH13r2uBmbinl7byA+NRtz9lzHT6NaQiIpOkSrixjsEJFWFLe53qRJk5T/btq0KZydndGzZ09ERkaqDKu8LP937h6EAHo2cEC9MlZbAUBrjxo4OKUz3v81GFcfyTB2wyXM7dcQ4zt5qvUhESfLwoErBUMad54ZRjE30kd6jrxI+ofJmfCubavZQ2lACIH15+5h8Z//rbDSZMJqWRPS33//fRw/fhyxsbEwNzdHx44d8c0336BBgwZqtW/y5MlYu3Ytvv/+e0ybNk15/e7du5g5cyYCAgKQm5uLZs2aYdGiRejRo0eZZfr/eRu34lJhYybFD2+2hKOlMVxrqPbQudmYwq9HHXzQvTZuxqZiT0gMDlyNRVJ6DjYHPsDmwAewNZMiOSMXhUMhJob6yMr772fYwMkCQ1rWxMAWLiV+H8sTsJRXYd02ZlKMWBOIw9fi0Km2Hd5qr5tB7vMY7BBRuT148ADHjx8vscemUPv27QEULMsuLth5dld0Z2dn5fWEhATlsJaTkxMSExNV8uXn5yM5ObnUuUGJadn443IMAGByd/UDLScrY+x83xuf7b2BP0IeYdGhW7gVm4qvhzSBsaF+kfRp2Xn460Y89oXGIPDeE+VcCam+Hl5r4IDBLWuiRwN77AuNUc7DKPTpH9dwI1aGuf0aFlt2edx7nI7P9t5A4L0nRe6p26tU1oT01q1bY/To0XB3d0dycjK++OIL+Pr6IioqCvr6pT/P3r17ceHChWLnXL3++uuoW7cuTp48CRMTE6xYsQKvv/46IiMjS/2ZH7kRh82BBcHZ8hHN4WhpXGJaoGCCd5OaVmhS0wpz+zVAQOQT7AuNwV/X4/AkI1clbVaeHHbmUgxr5YrBLWuioXP1mALRyr0GZvauD/+/7uDLgzfRysMaDZyqR9vLg8EOEZVbcVv4F+fKlSsAoBLIPMvLywtOTk44ceKEMrhJTU3FxYsX8b///Q8A4O3tjZSUFFy+fBmtW7cGAJw8eRIKhUIZTBXbxoD7yJUr0MrdGm08amj0fMaG+lg2vBkauVhi8Z+38UfII0Q8TseigY2RnpsP1xomCE9Ix97QGBy7lYCc/P9WwLTztMHgljXRv6kzrEz/G9IoHFq4n5QJF2tj/Br4AP93PgpbAh/g4r1k/DiqJeo7ld37VJbcfAXWnonEylMRyM1XwMhAgtx8gecna157mFJmr1JZE9Kf7cnz9PTEV199hebNm+P+/ful9uTFxMTgww8/xNGjR4u8h5KSkhAeHo5ffvkFzZo1U9b7888/48aNGyUGOw+TM/Hp7msAgPe71kKP+g7FpiuJgb4eutWzR7d69hjQ3BnjNwUXSfPDmy1feFizMk3sUguB957gdNhj+G0NwcEPO8NUqtvhQJU+G4uIqj6FQoGNGzcWWQYdGRmJRYsW4fLly7h//z4OHDiAsWPHomvXrsoPLaBgc729e/cCKPjLetq0afjqq69w4MABXL9+HWPHjoWLiwsGDx4MAGjYsCH69OmDiRMn4tKlSwgICMCUKVPw5ptvlrgSKy07D79dKPgLf3K32i80T0EikWBCZy9sGd8O1qaGuPowBQNXBeCt9RfRdelpTNgcjEPX4pCTr0BtezPM7F0f5z7tgd8ne+Ot9u4qgU4hZysTeNe2hYetGea93gibx7eDnbkRwhLSMOCn89gSeF/tJenFCb6fjP4/nsPyY3eRm69Al7p2ODa9O5YMawr9f78Hhd8J/yN3sPJEuNr15ebmqpzL9byMjAxs3LgRXl5ecHNzK7EchUKBMWPGYObMmWjcuHGR+7a2tqhfvz62bNmCjIwM5OfnY+3atXBwcFAGu8/Lkyvw0Y5QpGbno6W7NT7pXV+tZypJQ2dLPL/7gL5Eglr2ZuUqt7Lo6UmwfHhzOFoaIfJxBj7ff7PEtJ6eBUO2z7/8/PyU+13Vr18fJiYmcHd3x0cffaTcP6okxZUnkUjw7bffKtMMHDgQ7u7uMDY2hrOzM8aMGYPY2NgXf+iK2cC5euFxEUQvrqQt/KOjo0XXrl2FjY2NMDIyEnXq1BEzZ84s8v8MgNi4caPya4VCIebPny8cHR2FkZGR6NmzZ5Gynzx5IkaNGiXMzc2FpaWlePfdd0VaWlqJbVx7JkJ4zDokXlt2SsjlinI/c1DUk2LP1vp011Vx7WGK8gTyF/E4LVuM23BRWeaETZfEk/QcjcpIycwVc/ZcU5bRauHfYl/oI5V2xaZkin8ikkTM0wzx7ZE7yrSf7b0m8tX4HpV02vuqVauEmZmZACDq168vIiIiSi1n8eLFolevXsq2eXh4iO+//14lzcOHD0Xr1q2FRCIR+vr6wtnZWYSEhJRc5p+3hMesQ6LpgiMi+klGmc+ijh2XHohasw8Lj1mHRK3Zh1WObKiuAiOThNfsgp/7H5cfFpsmMTFRxMXFKV/Hjh0TAMSpU6fE9evXxdChQ8WBAwdERESEOHHihKhbt26ZR6Q8W15cXJzYsGGDkEgkIjIyUpnmu+++E4GBgeL+/fsiICBAeHt7C29v7yJl6cRxES8Ll54TVZ6KXgKdm69Al6UnkZCag6XDmmFE25J7GdT1T2QS3lp/scj17RM7aGWCsRACm/65D/8/7yBXroCDhRG+G9ECneuWPmQihMCf1+PxxcGbeJxWcBTGyDZumNOvAaxNpaXm3RJ4HwsO3IQQQJ/GTljxZotS5w317t0bUqkUBw8eVLkuk8mQmJiIuLg4LFu2DDExMQgICFA5Mb3Q5cuX0b9/f4SEhCh75Tw9PTFt2jTlBGUhBAYPHoy8vDx89tlnMDExwf/93//hwIEDCAoKKjIkeiosEe9uDAIArHm7Ffo0KX7I9EWUZ/l3VfXD8XB8f/wuTKX6OPhhZ9S2L33fn2nTpuHQoUMIDw8vtkdv165dePvtt5GRkaHS01uawYMHIy0tTeUk9ucdOHAAgwcPRk5ODgwN/+slVffzm8NYRFRpdgZFo9OSk3hr/UV0WnISO4OitV7HvisxSEjNgaOlEQa1LH0jNnV52ZkVO6yhrY3aJBIJ3u3khX1+nVDHwRyJaTl4+5eL8P/zNnLzi+6ICwCPnmZiwuZg+G0LweO0HNSyN8OOSR3wzRvNygx0AGCstydWvdUKUn09HLkZj7EbLkGWlVds2sIJ6e+9916Re1ZWVqhbty66du2K3bt3486dO8phyuedO3cOiYmJcHd3h4GBAQwMDPDgwQN8/PHH8PT0BFAwH+vQoUPYsWMHOnXqhFatWuHnn3+GiYkJNm/erFJevCwbH/9+FQAwzttDq4EO8N+wo64EOgAw5bU68K5li8zcgiX62XlFVwkWKmvoEoAy6FA30ElISMDhw4cxYcKEEtMkJydj69at6Nixo0qgowkGO0RUKeJkWZj9x/NnFV1HnCxLa3UoFALrzt4DAIzv5AUjA+2scHK2MoH/0P/mvRRuEqftD8FGLpY4OKWzcnnw2rP3MGz1P4hKylAeNfEwOQP/d+4efL8/i5N3EmGoL8HUnnXx19Qu6FBLs16mfk2dsXl8O1gYGeBSVDJGrAlEvKzonkDqTkgXQkAIgZycnGLvjxkzBteuXcOVK1eULxcXF8ycORNHjx4FUHBuF4AiZ3fp6empnN0lVwhM3RGK5IxcNHK2xJx+DTV69leVvp4EK95sAVszKW7HpeLrw7dLTFvcXlrPSkpKwqJFi1Qmqpdl8+bNsLCwwNChQ4vcmzVrFszMzGBra4vo6Gjs379f7XKfx2EscBiL6GWTKwSm77yCA1eLTjjU1lAQABy7lYCJW4JhYWSAgDmvaX3H2Jc5rHHkRjxm77mGlMw8SPULTmh//pd3O08bLB7aBHUcyreK61ZsKt7ZeAmJaTlwsTLGlgntlGWWdNr7vXv3sHPnTvj6+sLe3h6PHj3CkiVLEBAQgNu3bys3gnz2tPfiPD+MlZSUhAYNGqBbt274/PPPYWJigvXr1+OHH35AUFAQmjcv2AX7+2N38cOJcJj9OxxTq4zhGFJ1OiwR7/w7/Ld6dCv0bVq0V6ykoUug4HO0V69esLGxwYEDB9TugWnQoAF69eqFlStXFrmXlJSE5ORkPHjwAF9++SWsrKxw6NAhlV4lDmPRK6W0FQMAsG7dOnTv3h2WlpaQSCRISUkps0x/f3+0bdsWFhYWcHBwwODBg4ucJv0i5WpbdXt2WVYeJmwOKjbQAYCrD9UrRx1r/z3w860O7hWyNf7LHNbo08QJf03tgpbu1sgtJtCZ06cBdkzqUO5AByjoUfrjfx1Ry94MsbJsvLEmEJcfFBzPUdK5XMbGxjh37hz69euHOnXqYOTIkbCwsMA///yjsuP1s6e9q8POzg5HjhxBeno6XnvtNbRp0wbnz5/H/v37lYHOP5FJ+PFkOADg6yFNGei8gO71HTC5W8H2AJ/+cQ0PkzNV7pc2dJmWloY+ffrAwsICe/fuVTvQOXfuHMLCwootEyj42derVw+9evXCjh078Oeff+LChQsaPlkBBjukE4KCghAXF6d8HTt2DACUm51lZmaiT58+yjOb1HHmzBn4+fnhwoULytO1fX19kZGRoUzzIuVqW3V69ojEdAxZFYDTYY9hbKiHN9u6FVkC/e3fYTgfnlRyIWoKvp+M4AdPIdXXw/hOXmVnqAacrUzwca/il1E3c7Mu9nT2F+VmY4rdkzuihZs1UjLzMPr/LuDE7QTlae/16tVTSe/i4oI///wTCQkJyM3NxcOHD7F161bUr6/aXiEEeg8ZiX8ik4odsrx//77K7skA0KZNGxw9ehRPnjxBamoqAgMD0bdvXwBAUnoOpu24AiGAEW0KNvijF/Oxbz20crdGWnY+pmwPVZkfVtLQZWpqKnx9fSGVSnHgwIFiJ6KX5JdffkHr1q2VQWtpCocsSxoSLQuHscBhLF1U0oqB06dPo0ePHnj69Cmsra01KvPx48dwcHDAmTNn0LVrV5V75SlX26rqs5+4nYBpO64gLScfNa1NsHZMazSpaaUcCvKwNcE3R8Kw/0osLIwNsOd/HVFXjSMdSvLe5mAcv52AkW3c8M0bzcrOUE3EybLQaclJ5VwnoGIPEc3MzYff1hCcCnsMfT0J/Ic0LdeKtvKc9v48hULg3U1BOHP3Meo4mOPAlE46vzleRXv0NBP9fjiH1Ox8TOpaC3P7NSxx6LIw0MnMzMTevXthZvbfvkP29vbKXbOLG7pMTU2Fs7Mzli9fjsmTJ6u04eLFiwgKCkLnzp1Ro0YNREZGYv78+UhISMDNmzdhZGSkUg4PAqVXUuGKgRkzZmj1kLvCrncbGxutlaltVfHZhRD4+XQklv0dBiGAdl42+Hl0K9iZF/zCeva8oG+GNUPM0ywEP3iKdzcFYZ9fJ2U6TUQkpuH47QRIJMCkMg78rG4KJ0cXHjVRUZOjC5lKDbBubBvM2XMduy8/wqd/XEPk4zR0q+cAL3vV7QLy5QokZ+TicXoOktJzkZSWg6T0wlcuYp5m4tL9/06rVwhgzp7r6FTHrsh5VepYd+4eztx9DCMDPax6qxUDHS1wrWGKb4c3x/u/Xsa6s/fQoZYN8qOvFjt0GRISgosXC7ZgqFOnjsq9qKgo5Yq64oYud+zYASEERo0aVaQNpqam2LNnDxYsWICMjAw4OzujT58+mDdvnkqgown27IA9O7rm999/x1tvvYXo6OgiO+q+aO+GQqHAwIEDkZKSgvPnzxe5X1V6dqras2fm5mPmrms4fD0OADCmgwc+H9AIhvolj6AnZ+RiyM8BePAkEy3drbF9YgeNz4mauesqdl1+BN9Gjlg3to1GeauLl73nixACy/4Ow6pTkSrX6ziYQV+ih8fpOXiamYsX+UQxk+qjW317dK5jjy517eBmU3bgc/nBU4xYGwi5QsB/aFOMavdqHGj5snxx4CY2/XMfNUwN8efULuV6j1XkXlrs2aFXVnGnb5eXn58fbty4UeyHfVVSlZ79YXImJm4Jxp34NBjqS7BwUBO1PpBszKTY8E5bDP35H4RGp+DjXVex8s2Was9HiZdlY98VzQ/8rG5e9gnaEokEb3fwwM+nIlUmR0ckZqik05MANmZGsDOXwt7CCHbmBf+2MzeCgZ4EX/15u0hAlJErx5/X4/Hn9XgAgKetKTrXtUPnOvbwrm0LK5P/JrzGybJw41Eq5u+/AblCYEBzF7yphY0iSdWcfg0Q/CAZN2JSMXXHFSwf3gwPn2ZpHLDsuBSNuXu1M2xZHgx2SKeoe/q2JqZMmYJDhw7h7NmzcHV11Vq52laVnv2fyCT4bQ3B08w82JkbYc3brdDGU/0hsNr25ljzdmuM+eUiDl+Lg5etmdrnG20IiEKeXKCdpw1auWt24CeVLiopo8gqMACY268hutazg525EWqYSqFfSmBqbmygMgS3aHBj1HeyxPnwJJyPeIyQ6BTcf5KJ+0+i8duFaOhJgOZu1uhS1x65+XKsO3tPOV/JxkyKxUOaaHXIlgoYGehj5ahWeP3Hc7gUlYyuS09DoCBg+XpIU/g2ciwYqvx3mPJxWo7K10npOUiQ5eBx+n8Tigv20rqBrvXsX/rGjAx2SKeou9mZOoQQ+PDDD7F3716cPn0aXl5Ve0VPVXh2IQQ2/3Mfiw7fhlwh0MzVCmvHtH6hX2zetW3hP7QpZu6+hp9ORcDD1hTD25T+F7wsKw/bLhbswjy5u27N1akKCneOfn5y9IDmzmr/jJ897f3ZIbjWHjUw1acu0rLzcOFeMs6HP8a5iCTce5yB0OgUhEanFCkrJTMX6Tn5sKiAbQWo4Of9aZ/6WHDgljLILZxnNWfP9RcqUy4E7idlMtghelElnb4NAPHx8YiPj0dERAQA4Pr167CwsIC7u7ty0m3Pnj0xZMgQTJkyBUDB8M22bduwf/9+WFhYID6+oIvdysoKJiYmapf7MlSFZ//1z7M4fz8d/yToQd/EAkNa1oT/0KYaz7d51vA2brj/JAOrTkVi7t7rcK1hWuqGg1svPkB6Tj7qOZqjez2HEtPRi9HW5OjShuAsjA3Rq5EjejVyBADEpGQhIDwJe6/EIDDyiUpahUClfHC+SkpbEVnD1PDfYUoj2Fn8N1xpb24EO4uCI0re2xxcJDjW1rEqGin1mNBXBE891w0lnb4thBALFiwQAIq8nj1t28PDQyxYsED5dXHpn8+jTrkvQ1V6dtt+08T6s5HlOvn7WXK5Qnyw9bLwmHVINPviqIhILP5086zcfNHmq2PCY9YhsTu4+BOcSTsKT0yPTcl8qXUWntBd+Ko1+/BLbcOrqLjvu9fsQ+LBk3S18lf0afE89VwDXI1FhSr6BO6qqqTnlisEnmTkIClNdSy+cFnxo+eWEgMFY/oBs1/T6vcvO0+OUesvIDQ6BR62ptj7QSfYmKkebrnjUjRm77kOZytjnJnZA1ID7pmqa3YGRRfpVaqMya6vmvJ+3yty5aC6n98MdsBghwrsDCr4sBSVvGrgRakbqOUV7oXy7x4oh6/HYXfwI+WYfB0HcxjoSZCUnoMnGS+2lFib51sVSkrPweBVAXj0NAttPWvgt/faKw/2lCsEen13BveSMjCvf0O814XzdXTVy15yTwWq6vedS8+JNBAny8KcfwMdoGAuwOw/rqOxixWa1LSq3Map4dldaSUSYFhLV3jZm6n0whT2yjzNzCu1rIjEdJWvJRLAxlT677i89L8xenMjGOgBi/+6oxIQVdSYvJ25ETa+0xZDV/+DoPtP8enua1gxsgUkEgmO3UrAvaQMWBob4E3ut6LTXvaSeypQ3b/vDHaIULCkVvFcD4YAMHhVAIa2qon3utRCvXIcXVCR4mRZyh4pABAC2B3yqNQ8ehLA1twIJob6iH7uwD8AmNO3AbrUtYedhRQ2plIYlLIJoKWJ4UvbzbeuowVWj26NcRsvYf+VWHjammGaT12s+ffAz7HenjA34q81IlLF3wpEKOi5KE6+QuD34Ef4PfgRute3x8QutdCxtm2V2tcjLD6t2KGmrnXt0NDFsmBlhHLFhBT2/+6FoqcnKfGcpYEtXMq9lLiidK5rh68GN8GcPdfxw4lwxDzNxJWHKTDUl2BcR88KrZuIqicGO0QA/r6VoPJ1YQ9FHQcL/N+5ezh6Mx6nwx7jdNhjNHK2xHtdvPB6M5cqMQn29J3HRa7pSyT45o1mZQYeL2MpcUUY1c4d95MysPbsPewOKdgtOU8ucPJOQrWaZ0VELwcnKIMTlF91WblydPrmJJIzcvHloMao52BRpIfiwZMMbDgfhd+DHyErTw4AcLI0xjudPDGqnbvKdvYvU0RiGvr+cA55cqHc7K2qrZaoKDFPM9Hpm1Mq1yry9G8iqno4QZlITbsvP0RyRi7cbEwwup17sfNTPGzN8OWgJpjeqx62XozG5n/uIz41G0v+uoOVJ8Ixoq0bxnfygoG+5KUtXRdC4LO9N5AnF+jZwAGLBjfGgydZLxSwVMfJhw+KmWtUWbuzElHVxmCHXmn5cgXWnbsHAJjYpVapE3EBwNpUCr8edfBeFy8cuBKL/zsXhbCENGwMuI9NAfeVy7dfxtL1PSExuBiVDGNDPXwxsDFcrE3hYl0JO5NWkpKOLqiU3VmJqEqr/AkHRJXorxvxeJicBRszKYa3Vv/kZCMDfQxv44Yj07pgy/h2aOdZQ+WAxMID7+JkWdpvNArOBPr6z9sAgKk968HN5tX7gC+cb6T/72Txil4JRkTVF3t26JUlhFAuWR7n7QkTqeZnOEkkEnStZw8DfQneWn9R5V5FDql8c+QOkjNyUc/RHO91qdoHlFakl70SjIiqJwY79MoKiHiCm7GpMDHUx1hvj3KVVdyQigSokCGVyw+Ssf3SQwDAV4ObwrCMoTddVx3nGxHRy/Vq/5akV9raswW9OiPbuqGGWfH77Kjr+SEVoGBTwjvxaeUq93l5cgU+23sDADCijSvaeb28k9WJiKorBjv0SroRI8O58CTo60kwobN2hoFGtnXH+dk9sH1iBwxp4QIAmLo9FPeTMrRSPgBsDIjCnfg01DA1xOy+DbVWLhGRLmOwQ6+ktWcLVmANaOas1cm9zlYm8K5ti2/eaI5W7tZIzc7H5N8uIzM3v9xlx6Rk4ftj4QCAOf0aFjn1m4iIisdgh1450U8ycfhaLABgUtfaFVKH1EAPq99uDXsLI9yJT8Onu6+hvPt3fnHgJrLy5GjnaYM3WrlqqaVERLqPwQ69cv7v/D0oBNCtnj0auWhnx+yYmBi8/fbbsLW1hYmJCZo2bYqHd2/g59GtYKAnwaFrcfi/c1EAgMmTJ0MikWDFihVllrtq1Sp4enpCamSMLbNHIz/+Lr4a0gR6ehIkJyfjww8/RP369WFiYgJ3d3d89NFHkMlkWnkmIiJdwWCHXilP0nPwe3DBSqb3u9XSSplPnz5Fp06dYGhoiL/++gu3bt3C8uXLUaNGDbT1tMHnAxoBAPz/ug3/nzfjwoULcHFxKbPcnTt3YsaMGZg9dx4a/e9nSB288OSPL2AtKdi7JzY2FrGxsVi2bBlu3LiBTZs24ciRI5gwYYJWnouISFdw6Tm9UjYHPkB2ngLNXa3gXctWK2V+8803cHNzw8aNG5XXvLz+m/Q8poMHrj2SYcfpK1gw5xP89ddfmPDWG2WW+91332HixIl46toJKffuoemIT3B/1Ths2LABs2fPRpMmTfDHH38o09euXRtff/013n77beTn58PAgP+9iYgA9uzQKyQzNx9bAu8DAN7vVhuSZ5aJl8eBAwfQpk0bDB8+HA4ODmjZsiXWr1+vvC+RSLBwYCNkH/sR5m2H4PvLWShr9k5ubi4uX76MBq074v/OFwx/LRrSFL18fBAYGFhivsLD8BjoEBH9h8EOvTJ2Bj1ESmYePG1N0buxk9bKvXfvHlavXo26devi6NGj+N///oePPvoImzdvVqb54btlaOxaAx5d38CNmFSkZOaWOmE5KSkJcrkcf9xKh1wh0LuxI3o2dISjoyPi4+NLzLNo0SJMmjRJa89GRKQL+OcfvRLy5ArlBOGJXWtBX087vToAoFAo0KZNGyxevBgA0LJlS9y4cQNr1qzBuHHjcPnyZfzwww8ICQnB/Uwp3v7lIjJy5Ah+8LTMsu/Ep6KGlzsWDGhcarrU1FT0798fjRo1whdffKGNxyIi0hns2aFXwuFrcYhJyYKduRTDtLxs29nZGY0aNVK51rBhQ0RHRwMAzp07h8TERLi7u6NrAyc8+HYQ5KmJ2Pbj13BxLf5UdImxBaCnB3lGCmb0qgcX64LjEBISEuDkpNorlZaWhj59+sDCwgJ79+6FoaGhVp+PiKi6Y88O6bxnD/x8t5MXjA01P/CzNJ06dUJYWJjKtbt378LDo+C8rTFjxsDHx0elPd7dekK/XjfYduiHhNRsOFoaq+RffuIepI51YJJ0G+909ARQ0IN04sQJTJkyRZkuNTUVvXv3hpGREQ4cOABjY9VyiIiIwQ69As7cfYw78Wkwk+rj7fblO/CzONOnT0fHjh2xePFijBgxApcuXcK6deuwbt06AICtrS1sbVVXftlamsLE2Qlpxg7432+XsWOSN/r27oUhQ4agTb9R2H35EazaDUb8kR+w9bdf0a5dO6xYsQIZGRl49913ARQEOr6+vsjMzMRvv/2G1NRUpKamAgDs7e2hr6/doI6IqLpisEM6r7BXZ1Q7d1iZan+Ip23btti7dy/mzJmDhQsXwsvLCytWrMDo0aNLzCMBMKKNG/blGSAkOgVfHryJyMhIJCQ+xrx9BQd9vjfubbj4uOHzzz9HfHw8WrRogSNHjsDR0REAEBISgosXLwIA6tSpo1J+VFQUPD09tf6sRETVkUSUdw97HZCamgorKyvlsl3SHVcepmDwqgAY6ElwblYPOFuZVHaTVJwKS8T4TUEQApjbtwEiH2dgZ/BD2JpJcfLj7hUSnBER6Qp1P785QZl02tp/e3UGtahZ5QIdAOhR3wEzfOoBABb/dQc7/93d2aeRAwMdIiItYbBDOisqKQNHbhbsSaOtoyEqwtBWNYtc2x38CHGyrEpoDRGR7mGwQzpr/bl7EALo2cAB9RwtKrs5JXqQnFnkmlwA95OKXiciIs0x2CGdlJiWjd2XHwEoOBqiKvOyM8PzexzqSyTwtDOtnAYREekYBjukkzb/cx+5+Qq0crdGW88ald2cUjlbmcB/aFPo/3tWl75EgsVDm1TJOUZERNURl56TzknPycevgQ8AaPfAz4o0sq07utazx/2kTHjamTLQISLSIgY7pFPiZFlYdTICqdn5qGVvhl4NHSu7SWpztjJhkENEVAEY7JDO2BkUjTl7rkPx785RLd2soafFAz+JiKh60njOzqlTpyqiHUTlEifLUgl0AGBvaAyXbxMRkebBTp8+fVC7dm189dVXePjwYUW0iUhjUUkZKoEOACi4fJuIiPACwU5MTAymTJmC3bt3o1atWujduzd+//135ObmVkT7iNRiUMxwFZdvExER8ALBjp2dHaZPn44rV67g4sWLqFevHj744AO4uLjgo48+wtWrVyuinUQlSsnMxew/rqtc4/JtIiIqVO6DQGNjY7Fu3TosWbIEBgYGyM7Ohre3N9asWYPGjRtrq50VigeBVl+5+QqM3XARF+4lw8XKGGvHtEZ6jpzLt4mIXgEVehBoXl4edu/ejX79+sHDwwNHjx7FTz/9hISEBERERMDDwwPDhw9/4cY/KyYmBm+//TZsbW1hYmKCpk2bIjg4WHlfCIHPP/8czs7OMDExgY+PD8LDw7VSN1VtQgjM2XMdF+4lw9zIABvebYumrtbwrm3LQIeIiJQ0DnY+/PBDODs74/3330e9evUQGhqKwMBAvPfeezAzM4OnpyeWLVuGO3fulLtxT58+RadOnWBoaIi//voLt27dwvLly1Gjxn874i5duhQ//vgj1qxZg4sXL8LMzAy9e/dGdnZ2ueunqu3n05H4I+QR9PUk+OmtlmjgxF45IiIqSuNg59atW1i5ciViY2OxYsUKNGnSpEgaOzs7rSxR/+abb+Dm5oaNGzeiXbt28PLygq+vL2rXLjjrSAiBFStWYN68eRg0aBCaNWuGLVu2IDY2Fvv27St3/aSZL774AhKJROXVoEEDAMD9+/eL3Ct87dq1q9jy8vLyMGvWLDRt2hRmZmZwcXHB2LFjERsbi4NXY/Ht0TAAwCfdamL9wumwtLSEtbU1JkyYgPT09Jf23EREVLVpHOycOHECo0aNgpGRUYlpDAwM0K1bt3I1DAAOHDiANm3aYPjw4XBwcEDLli2xfv165f2oqCjEx8fDx8dHec3Kygrt27dHYGBgieXm5OQgNTVV5UXa0bhxY8TFxSlf58+fBwC4ubmpXI+Li8OXX34Jc3Nz9O3bt9iyMjMzERISgvnz5yMkJAR79uxBWFgYfPr0x8e7CibCj+/khQMrZuPmzZs4duwYDh06hLNnz2LSpEkv7ZmJiKhq03gHZX9/fzg6OmL8+PEq1zds2IDHjx9j1qxZWmvcvXv3sHr1asyYMQNz585FUFAQPvroI0ilUowbNw7x8fEAAEdH1SMBHB0dlfdKeoYvv/xSa+2k/xgYGMDJyanIdX19/SLX9+7dixEjRsDc3LzYsqysrHDs2DGVa/O/XoYBvbqiZud49GnfFENrAQuOHEFQUBDatGkDAFi5ciX69euHZcuWwcXFRUtPRkRE1ZXGPTtr165VDk08q3HjxlizZo1WGlVIoVCgVatWWLx4MVq2bIlJkyZh4sSJ5a5nzpw5kMlkyhc3R9Se8PBwuLi4oFatWhg9ejSio6OLTXf58mVcuXIFEyZMULtsWVYePt99CYAEjTyd8cObLXDp4gVYW1srAx0A8PHxgZ6eHi5evFjexyEiIh2gcbATHx8PZ2fnItft7e0RFxenlUYVcnZ2RqNGjVSuNWzYUPkBWthTkJCQoJImISGh2N6FQkZGRrC0tFR5Ufm1b98emzZtwpEjR7B69WpERUWhS5cuSEtLK5L2l19+QcOGDdGxY0e1ys6TKzB5UyBu7lsN2+Y9sOX97jAzMkB8fDwcHBxU0hoYGMDGxqbU3j0iInp1aBzsuLm5ISAgoMj1gIAArQ8ZdOrUCWFhYSrX7t69Cw8PDwCAl5cXnJyccOLECeX91NRUXLx4Ed7e3lptC5Wtb9++GD58OJo1a4bevXvjzz//REpKCn7//XeVdFlZWdi2bZvavTpCCHz2Ryj2fzcTEglweOcmOFkZV8QjEBGRDtJ4zs7EiRMxbdo05OXl4bXXXgNQMGn5008/xccff6zVxk2fPh0dO3bE4sWLMWLECFy6dAnr1q3DunXrAAASiQTTpk3DV199hbp168LLywvz58+Hi4sLBg8erNW2kOasra1Rr149REREqFzfvXs3MjMzMXbsWLXKWX0yDD/NmwK5LBHb9/+J9vXdlPecnJyQmJiokj4/Px/Jycml9u4REdErRGhIoVCITz/9VBgbGws9PT2hp6cnTE1NxZdffqlpUWo5ePCgaNKkiTAyMhINGjQQ69atK9Ke+fPnC0dHR2FkZCR69uwpwsLCNKpDJpMJAEImk2mz6a+8tLQ0UaNGDfHDDz+oXO/WrZsYNmyYWmUcDHkgTOt2EIZ27mLFgaAi92/duiUAiODgYOW1o0ePColEImJiYsr3AEREVKWp+/n9wsdFpKen4/bt2zAxMUHdunVLXYpe1fG4CO345JNPMGDAAHh4eCA2NhYLFizAlStXcOvWLdjb2wMAIiIiUK9ePfz555/o06dPkTIaNGgAf39/DBkyBMH3HqNH34HIiovA+IVr8OXIjpBICg78tLGxgVQqBVAwfJaQkIA1a9YgLy8P7777Ltq0aYNt27a9vIcnIqKXTt3Pb42HsQqZm5ujbdu2L5qddNCjR48watQoPHnyBPb29ujcuTMuXLigDHSAgi0KXF1d4evrW2wZYWFheBD3GAeuxmLO5hNIv3sBALB++htYP/2/dKdOnUL37t0BAFu3bsWUKVPQs2dP6OnpYdiwYfjxxx8r7DmJiKh6eaGeneDgYPz++++Ijo5Gbm6uyr09e/ZorXEvC3t2qo6dQdGYs+c6FP++K50sjXBsRjdYGBtWbsOIiKjKqbCDQHfs2IGOHTvi9u3b2Lt3L/Ly8nDz5k2cPHkSVlZW5Wo0vdriZFkqgQ4AJKblID0nv/IaRURE1Z7Gwc7ixYvx/fff4+DBg5BKpfjhhx9w584djBgxAu7u7hXRRnpF3IhJVQl0AEAhgPtJmZXTICIi0gkaBzuRkZHo378/AEAqlSIjIwMSiQTTp09XLgkn0lS8LBv+f94ucl1fIoGnnWkltIiIiHSFxsFOjRo1lDvi1qxZEzdu3AAApKSkIDOTf4GT5iIS0zD05wDcS8qAhZEB9AoWXEFfIsHioU3gbGVSuQ0kIqJqTePVWF27dsWxY8fQtGlTDB8+HFOnTsXJkydx7Ngx9OzZsyLaSDrs8oOnmLA5CCmZeahlb4bN77aDgb4E95My4WlnykCHiIjKTePVWMnJycjOzoaLiwsUCgWWLl2Kf/75B3Xr1sW8efNQo0aNimprheFqrMpx/FYCpmwPQXaeAi3crLHhnbawMZNWdrOIiKiaqJB9dvLz83Ho0CH07t0bAKCnp4fZs2eXr6X0StoZFI25e29ArhDoUd8eq0a3gqn0hbd9IiIiKpFGc3YMDAwwefJkZGdnV1R7SMcJIbDyRDhm/XEdcoXAG61dsW5sGwY6RERUYTSeoNyuXTtcuXKlAppCuk6uEPh8/00sP3YXAODXoza+faMZDPU1fhsSERGpTeM/pz/44APMmDEDDx8+ROvWrWFmZqZyv1mzZlprHOmO7Dw5pu+8gr9uxEMiAb4Y0BjjOnpWdrOIiOgVoPEEZT29on+FSyQSCCEgkUggl8u11riXhROUK5YsKw8TtwTjUlQypPp6+H5kC/Rv5lzZzSIiomquwg4CjYqKKlfD6NUSL8vGOxsv4U58GiyMDLB2bGt0rG1X2c0iIqJXiMbBjoeHR0W0g3RInCwLUUkZkAD4ZNc1xKRkwcHCCJvebYdGLuw5IyKil0vjYGfLli2l3h87duwLN4aqv+dPLQeAWnZm2Dy+HdxseOwDERG9fBrP2Xl+08C8vDxkZmZCKpXC1NQUycnJWm3gy8A5O9oRJ8tCpyUnixzm+dfULmjozO8rERFpl7qf3xqv+X369KnKKz09HWFhYejcuTO2b99erkZT9RaVlFEk0AGAlMy8l98YIiKif2llg5O6detiyZIlmDp1qjaKo2rKy85MeYhnIZ5aTkRElU1ru7kZGBggNjZWW8VRNeRsZYL5rzdSfq0nAU8tJyKiSqfxBOUDBw6ofC2EQFxcHH766Sd06tRJaw2j6qmxixUAwM5MioMfdWagQ0RElU7jYGfw4MEqX0skEtjb2+O1117D8uXLtdUuqqbuxKcCAJq7WTPQISKiKkHjYEehUFREO0hH3I5LAwDUd7Ko5JYQEREV4AmMpFVh//bsNOBScyIiqiI0DnaGDRuGb775psj1pUuXYvjw4VppFFVPCoVAWHxBz05D9uwQEVEVoXGwc/bsWfTr16/I9b59++Ls2bNaaRRVT4+eZiEjVw6pvh687MwquzlEREQAXiDYSU9Ph1QqLXLd0NAQqampWmkUVU+3/x3CqutoDgN9jpASEVHVoPEnUtOmTbFz584i13fs2IFGjRoVk4NeFXf+nZzcwInzdYiIqOrQeDXW/PnzMXToUERGRuK1114DAJw4cQLbt2/Hrl27tN5Aqj4Kl503dOZ8HSIiqjo0DnYGDBiAffv2YfHixdi9ezdMTEzQrFkzHD9+HN26dauINlI1cSeey86JiKjq0TjYAYD+/fujf//+2m4LVWNZuXLcf5IBgMNYRERUtWg8ZycoKAgXL14scv3ixYsIDg7WSqOo+rmbkAYhADtzKewtjCq7OUREREoaBzt+fn54+PBhkesxMTHw8/PTSqOo+imcr8NeHSIiqmo0DnZu3bqFVq1aFbnesmVL3Lp1SyuNourntnIlFufrEBFR1aJxsGNkZISEhIQi1+Pi4mBg8EJTgEgH3OExEUREVEVpHOz4+vpizpw5kMlkymspKSmYO3cuevXqpdXGUfUghFCuxGLPDhERVTUad8UsW7YMXbt2hYeHB1q2bAkAuHLlChwdHfHrr79qvYFU9SWk5iAlMw96EqCOg3llN4eIiEiFxsFOzZo1ce3aNWzduhVXr16FiYkJ3n33XYwaNQqGhoYV0Uaq4gqHsGrZm8PYUL+SW0NERKTqhSbZmJmZYdKkSdpuC1VTHMIiIqKq7IVnFN+6dQvR0dHIzc1VuT5w4MByN4qqlztxhcdEcHIyERFVPRoHO/fu3cOQIUNw/fp1SCQSCCEAABKJBAAgl8u120Kq8tizQ0REVZnGq7GmTp0KLy8vJCYmwtTUFDdv3sTZs2fRpk0bnD59ugKaSFVZbr4CEYnpALjsnIiIqiaNe3YCAwNx8uRJ2NnZQU9PD3p6eujcuTP8/f3x0UcfITQ0tCLaSVVU5ON05CsELIwN4GJlXNnNISIiKkLjnh25XA4Li4LhCjs7O8TGxgIAPDw8EBYWpt3WUZX33zERFsqhTCIioqpE456dJk2a4OrVq/Dy8kL79u2xdOlSSKVSrFu3DrVq1aqINlIV9t98HQ5hERFR1aRxsDNv3jxkZGQAABYuXIjXX38dXbp0ga2tLXbu3Kn1BlLVdqfwTCxnTk4mIqKqSeNgp3fv3sp/16lTB3fu3EFycjJq1KjBYYxXEE87JyKiqk4rJ3fa2NhooxiqZpIzcpGQmgMAqM9l50REVEVpPEGZqFBhr467jSnMjXjiPRERVU0MduiFFc7XYa8OERFVZQx26IUV9uw0ZLBDRERVmMbBztmzZ5Gfn1/ken5+Ps6ePauVRlH1EFa47Jw7JxMRURWmcbDTo0cPJCcnF7kuk8nQo0cPrTSKqj65QiAsgWdiERFR1adxsCOEKHaJ+ZMnT2BmZqaVRlHV9+BJBrLzFDA21IOHLX/uRERUdam9hGbo0KEACk43f+edd2BkZKS8J5fLce3aNXTs2FH7LaQqqXDn5PqOFtDX4/5KRERUdakd7FhZWQEo6NmxsLCAiYmJ8p5UKkWHDh0wceJE7beQqqQ7cdxMkIiIqge1g52NGzcCADw9PfHJJ59wyOoVdzuey86JiKh60HjOzqeffqoyZ+fBgwdYsWIF/v77b602jKo25TERPBOLiIiqOI2DnUGDBmHLli0AgJSUFLRr1w7Lly/HoEGDsHr1aq03kKqe9Jx8PEzOAsBhLCIiqvo0DnZCQkLQpUsXAMDu3bvh5OSEBw8eYMuWLfjxxx+13kCqegr313G0NIKNmbSSW0NERFQ6jYOdzMxMWFgUDF38/fffGDp0KPT09NChQwc8ePBA6w2kqocnnRMRUXWicbBTp04d7Nu3Dw8fPsTRo0fh6+sLAEhMTISlJT/8XgWFZ2Jxvg4REVUHGgc7n3/+OT755BN4enqiXbt28Pb2BlDQy9OyZUutN5Cqnv/OxGJwS0REVZ/Gwc4bb7yB6OhoBAcH4+jRo8rrPXv2xPfff6/Vxj1vyZIlkEgkmDZtmvJadnY2/Pz8YGtrC3NzcwwbNgwJCQkV2o5XmRCCp50TEVG18kKnnjs5OcHCwgLHjh1DVlbBqpy2bduiQYMGWm3cs4KCgrB27Vo0a9ZM5fr06dNx8OBB7Nq1C2fOnEFsbKxyt2fSvpiULKTl5MNAT4La9uaV3RwiIqIyaRzsPHnyBD179kS9evXQr18/xMXFAQAmTJiAjz/+WOsNBID09HSMHj0a69evR40aNZTXZTIZfvnlF3z33Xd47bXX0Lp1a2zcuBH//PMPLly4UCFtedUVrsSq42AOqcELxcpEREQvlcafVtOnT4ehoSGio6NhamqqvD5y5EgcOXJEq40r5Ofnh/79+8PHx0fl+uXLl5GXl6dyvUGDBnB3d0dgYGCJ5eXk5CA1NVXlReopPBOLJ50TEVF1ofZxEYX+/vtvHD16FK6urirX69atWyFLz3fs2IGQkBAEBQUVuRcfHw+pVApra2uV646OjoiPjy+xTH9/f3z55Zfabuor4XbhmVjOnJxMRETVg8Y9OxkZGSo9OoWSk5NVTkLXhocPH2Lq1KnYunUrjI2NtVbunDlzIJPJlK+HDx9qrWxdx54dIiKqbjQOdrp06aI8LgIAJBIJFAoFli5dih49emi1cZcvX0ZiYiJatWoFAwMDGBgY4MyZM/jxxx9hYGAAR0dH5ObmIiUlRSVfQkICnJycSizXyMgIlpaWKi8qW3aeHPcepwMAGrJnh4iIqgmNh7GWLl2Knj17Ijg4GLm5ufj0009x8+ZNJCcnIyAgQKuN69mzJ65fv65y7d1330WDBg0wa9YsuLm5wdDQECdOnMCwYcMAAGFhYYiOjlbu/0PaE5GYDoUArE0N4WCh3V48IiKiiqJxsNOkSRPcvXsXP/30EywsLJCeno6hQ4fCz88Pzs7OWm2chYUFmjRponLNzMwMtra2yusTJkzAjBkzYGNjA0tLS3z44Yfw9vZGhw4dtNoWema+jpMFJBJJJbeGiIhIPRoHO9HR0XBzc8Nnn31W7D13d3etNExd33//PfT09DBs2DDk5OSgd+/e+Pnnn19qG14VYcr5OhzCIiKi6kMihBCaZNDX10dcXBwcHBxUrj958gQODg6Qy+VabeDLkJqaCisrK8hkMs7fKcXb/3cR5yOS8M2wphjZ9uUGtURERM9T9/Nb4wnKQohihzDS09O1umKKqh6edk5ERNWR2sNYM2bMAFCw+mr+/Pkqy8/lcjkuXryIFi1aaL2BVDU8TstBUnouJBKgniOXnRMRUfWhdrATGhoKoKBn5/r165BKpcp7UqkUzZs3xyeffKL9FlKVUNir42lrBhOpfiW3hoiISH1qBzunTp0CULD0+4cffuDclldM4Unn3EyQiIiqG41XY23cuLEi2kFV3G3O1yEiomqKx1aTWpTLzp3Zs0NERNULgx0qU75cgfCEf4+JYM8OERFVMwx2qExRSRnIlStgJtWHaw2Tym4OERGRRhjsUJlu/zuEVd/JAnp6PCaCiIiqFwY7VKY7/56JVZ9DWEREVA0x2KEy3fm3Z6chJycTEVE1xGCHynQnjsvOiYio+mKwQ6WSZeUhVpYNoGDODhERUXXDYEdHxcmy8E9kEuJkWeUqp3B/nZrWJrAyMdRG04iIiF4qjXdQpqpvZ1A05uy5DoUA9CSA/9CmGNnW/YXK+u+kc/bqEBFR9cSeHR0TJ8tSBjoAoBDA3D03XriH53Ycd04mIqLqjcGOjolKylAGOoXkQuB+UuYLlVfYs8Nl50REVF0x2NExXnZmxV6/8jBF47IUCqGcs9OQw1hERFRNMdjRMcuXfoO4zdMR/f1wPFw5Gol7vkLek0f45sgdrDh+F0IIREZGYsiQIbC3t4elpSVGjBiBhISEImU9fJqJzFw5pPp6sDNSYNq0afDw8ICJiQk6duyIoKAgZdq8vDzMmjULTZs2hZmZGVxcXDB27FjExsa+zMcnIiIqgsGOjvn94N+waNUfr3/2Czbu3A9vTytkHlgIRW42VhwPx8wdl+Dr6wuJRIKTJ08iICAAubm5GDBgABQKhUpZhZsJ1nU0x+T3J+HYsWP49ddfcf36dfj6+sLHxwcxMTEAgMzMTISEhGD+/PkICQnBnj17EBYWhoEDB7707wEREdGzJEIIUXYy3ZaamgorKyvIZDJYWlbfuSlXH6Zg0KoASCTA0WldUc/RAo8fP4aDgwPmrtqBbQ/NkXkvBI93fYH4x0lwsK0BAJDJZKhRowb+/vtv+Pj4KMv74Xg4vj9+FwOb2OHndzph//796N+/v/J+69at0bdvX3z11VfFticoKAjt2rXDgwcP4O7+YqvBiIiISqLu5zd7dnTIt0fDAABDW7qinmPBHBuZTAYAGNW1MVaPbgV9IYcAMHnbVcgy8wAAxsbG0NPTw/nz51XKK5ycXM/eBHK5HMbGxir3TUxMiuR5lkwmg0QigbW1tTYej4iI6IUw2NER/0Qk4XxEEgz1JZjmUxcAoFAUzLPp1KkTmjRpgj5NnPHLzFHQkxrj+ObvMXTlSUTGJuGTTz6BXC5HXFycSpmFw1gtarnA29sbixYtQmxsLORyOX777TcEBgYWyVMoOzsbs2bNwqhRo6p1bxkREVV/DHZ0gBAC3/zbqzO6vQfcbEwBAH5+frhx4wZ27NihTNu3bX2s2/QbcqOCcHJ2X9Rxc0R0/GO0atUKenr/vR0yc/Nx/0kGgIJjIn799VcIIVCzZk0YGRnhxx9/xKhRo1TyFMrLy8OIESMghMDq1asr8tGJiIjKxGBHBxy9mYCrD1NgKtWHX486AIApU6bg0KFDOHXqFFxdXVXST3hzCCIiItD5y71w+3Ab7jcZj/vRj1CrVi1lmrsJ6RACsDOXwt7CCLVr18aZM2eQnp6Ohw8f4tKlS8jLy1PJA/wX6Dx48ADHjh1jrw4REVU6BjvVnFwhsOzvgl6dCZ29YGcuxZQpU7B3716cPHkSXl5exeZzrWGKAx/3RZt6roi/E4zkpMewbdRReb+kk87NzMzg7OyMp0+f4ujRoxg0aJDyXmGgEx4ejuPHj8PW1lbbj0tERKQxno1Vze0JeYSIxHRYmxpiYtda8PPzw7Zt27B//35YWFggPj4eAGBlZQUTExMAwMaNG9GwYUPY29vjddNI/HnwG1i0HYSvzqfAxD4ao9q5Y+7EEUh1aIEGnT8CABw9ehRCCNSvXx8RERGYOXMmGjRogHfffRdAQaDzxhtvICQkBIcOHYJcLlfWbWNjA6lUWgnfHSIiIgY71VpOvhwrjocDAD7oXhuWxobKOTLdu3dXSbtx40a88847AICwsDDMmTMHycnJ8PT0xKIvPkeiR0/suhyDOXuuIzE1B/GPHkBqUQtOVgUrsGQyGebMmYNHjx7BxsYGw4YNw9dffw1Dw4KT0GNiYnDgwAEAQIsWLVTqPnXqVJH2EBERvSzcZwfVd5+djQFR+PLgLThZGuP0zO4wNtR/4bKEEPju2F2sPBmhcl0iAZaU49R0IiKiisJ9dnRcek4+fvo3MJnqU7dcgQ4ASCQSfOxbH5/41lO5Lsp5ajoREVFlY7BTTW04H4UnGbnwsjPD8NauZWdQUyuPGkWulefUdCIiosrGYKcaSs7Ixfqz9wAAM3rVg4G+9n6MXnZm0JOoXtOXSOBpZ6q1OoiIiF4mBjvV0OrTEUjLyUdjF0v0b+qs1bKdrUzgP7Qp9CUFEY++RILFQ5vA2cpEq/UQERG9LAx2KoC/vz/atm0LCwsLODg4YPDgwQgLC1NJ0717d0gkEpXX5MmTSy23MN281xvjwTev48+pXaGvr4dvv/1WJd3hw4fRvn17mJiYoEaNGhg8eLBG7R/Z1h3nZ/fA9okdcH52D05OJiKiao1LzyvAmTNn4Ofnh7Zt2yI/Px9z586Fr68vbt26BTMzM2W6iRMnYuHChcqvTU1LHyqKi4vDV4duYf+VWLR0t8Zgu0S89957GDZsmDLNH3/8gYkTJ2Lx4sV47bXXkJ+fjxs3bmj8DM5WJuzNISIincBgpwIcOXJE5etNmzbBwcEBly9fRteuXZXXTU1N4eTkpHa5mfrm+OteNvTNa2DBSG8smjoePXr0UB7ZkJ+fj6lTp+Lbb7/FhAkTlPkaNWpUziciIiKqvjiM9RLIZDIABTsJP2vr1q2ws7NDkyZNMGfOHGRmlr7iafmxu5ArBHwaOsDVOA+HDx9WCWpCQkIQExMDPT09tGzZEs7Ozujbt+8L9ewQERHpCvbsVDCFQoFp06ahU6dOaNKkifL6W2+9BQ8PD7i4uODatWuYNWsWwsLCsGfPnmLLuREjw+FrcZBIgE9618fmzWtgYWGBoUOHKtPcu1ewQuuLL77Ad999B09PTyxfvhzdu3fH3bt3iwRbRERErwIGOxXMz88PN27cwPnz51WuT5o0Sfnvpk2bwtnZGT179kRkZCRq165dpJylRwsmOA9uURMNnCwxeMMGjB49GsbGxso0CoUCAPDZZ58p5/Fs3LgRrq6u2LVrF95//32tPx8REVFVx2GsCjRlyhQcOnQIp06dgqtr6Rv/tW/fHgAQERFR5F5g5BOcvfsYBnoSTPeph3PnziEsLAzvvfeeSjpn54Jl6M/O0TEyMkKtWrUQHR1d3schIiKqlhjsVAAhBKZMmYK9e/fi5MmT8PLyKjPPlStXAPwXsDxb1tKjdwAAb7V3h7utKX755Re0bt0azZs3V0nbunVrGBkZqSxzz8vLw/379+Hh4VHOpyIiIqqeOIxVAfz8/LBt2zbs378fFhYWiI+PBwBYWVnBxMQEkZGR2LZtG/r16wdbW1tcu3YN06dPR9euXdGsWTNlOQ0aNMDID2YhNNYBJob6mPJaHaSmpmLXrl1Yvnx5kXotLS0xefJkLFiwAG5ubvDw8FDuwTN8+PCX8/BERERVDIOdCrB69WoABRsHPmvjxo145513IJVKcfz4caxYsQIZGRlwc3PDsGHDMG/ePJX0YWFh+CMwDPBwwLudPOFgYYx167ZACIFRo0YVW/e3334LAwMDjBkzBllZWWjfvj1OnjyJGjWKnnlFRET0KpAIIURlN6KyqXtE/Mu24fw9LDx0GxbGBjg/6zVYmRhWdpOIiIiqDHU/vzlnp4r6NfABFh66DQBIz87HkRtxldwiIiKi6onBThV09u5jzN//30aAAsDcPTcQJ8uqvEYRERFVUwx2qhAhBDYFRGH8pqAi9+RC4H5S6TssExERUVGcoFxFPEnPwczd13DyTmKx9/UlEnjalX5QKBERERXFYKcKOHv3MT7edRWP03IgNdDDZ/0awshAD5/tvQG5ENCXSLB4aBOeQk5ERPQCGOxUotx8BZb9HYZ1ZwvOtKrnaI4fR7VEA6eCGeXd6tvjflImPO1MGegQERG9IAY7lSTycTqm7gjFjZhUAMCYDh74rH9DGBvqK9M4W5kwyCEiIionBjsvmRACu4IfYcGBm8jKk6OGqSGWvtEcvRo5VnbTiIiIdBKDnZdIlpmHuXuv4/D1gj1zOta2xXcjWsDJyriMnERERPSiGOy8JEH3kzFtxxXEpGTBQE+Cj33r4/2utaCnJ6nsphEREek0BjsVKE6WhYjEdJy8k4jN/9yHQgCetqb44c2WaO5mXdnNIyIieiUw2KkgO4OiMWfPdSieOXlsWCtXfDmoMcyN+G0nIiJ6WbiDcgWIk2UVCXQkEuCT3vUY6BAREb1kDHYqQFRShkqgAwBCgMc9EBERVQIGOxXAy84Mz8875nEPRERElYPBTgVwtjKB/9Cm0JcURDw87oGIiKjycAJJBRnZ1h1d6/G4ByIiosrGYKcC8bgHIiKiysdhLCIiItJpVTrY8ff3R9u2bWFhYQEHBwcMHjwYYWFhKmmys7Ph5+cHW1tbmJubY9iwYUhISKikFhMREVFVU6WDnTNnzsDPzw8XLlzAsWPHkJeXB19fX2RkZCjTTJ8+HQcPHsSuXbtw5swZxMbGYujQoZXYaiIiIqpKJEIIUXayquHx48dwcHDAmTNn0LVrV8hkMtjb22Pbtm144403AAB37txBw4YNERgYiA4dOqhVbmpqKqysrCCTyWBpaVmRj0BERERaou7nd5Xu2XmeTCYDANjY2AAALl++jLy8PPj4+CjTNGjQAO7u7ggMDCyxnJycHKSmpqq8iIiISDdVm2BHoVBg2rRp6NSpE5o0aQIAiI+Ph1QqhbW1tUpaR0dHxMfHl1iWv78/rKyslC83N7eKbDoRERFVomoT7Pj5+eHGjRvYsWNHucuaM2cOZDKZ8vXw4UMttJCIiIiqomqxz86UKVNw6NAhnD17Fq6ursrrTk5OyM3NRUpKikrvTkJCApycnEosz8jICEZGRhXZZCIiIqoiqnTPjhACU6ZMwd69e3Hy5El4eXmp3G/dujUMDQ1x4sQJ5bWwsDBER0fD29v7ZTeXiIiIqqAq3bPj5+eHbdu2Yf/+/bCwsFDOw7GysoKJiQmsrKwwYcIEzJgxAzY2NrC0tMSHH34Ib29vtVdiERERkW6r0kvPJRJJsdc3btyId955B0DBpoIff/wxtm/fjpycHPTu3Rs///xzqcNYz+PScyIioupH3c/vKh3svCwMdoiIiKofndxnh4iIiEhTDHaIiIhIpzHYISIiIp3GYIeIiIh0GoMdIiIi0mkMdoiIiEinMdghIiIincZgh4iIiHQagx0iIiLSaQx2iIiISKcx2CEiIiKdxmCHiIiIdBqDHSIiItJpDHaIiIhIpzHYISIiIp3GYIeIiIh0GoMdIiIi0mkMdoiIiEinMdghIiIincZgh4iIiHQagx0iIiLSaQx2iIiISKcx2CEiIiKdxmCHiIiIdBqDHSIiItJpDHaIiIhIpzHYISIiIp3GYIeIiIh0GoMdIiIi0mkMdoiIiEinMdghIiIincZgh4iIiHQagx0iIiLSaQx2iIiISKcx2CEiIiKdxmCHiIiIdBqDHSIiItJpDHaIiIhIpzHYISIiIp3GYIeIiIh0GoMdIiIi0mkMdoiIiEinMdghIiIincZgh4iIiHQagx0iIiLSaQx2iIiISKcx2CEiIiKdxmCHiIiIdBqDHSIiItJpDHaIiIhIpzHYISIiIp3GYIeIiIh0GoMdIiIi0mkMdoiIiEinMdghIiIincZgh4iIiHQagx0iIiLSaQx2iIiISKcx2CEiIiKdxmCHiIiIdBqDHSIiItJpDHaIiIhIpzHYISIiIp2mM8HOqlWr4OnpCWNjY7Rv3x6XLl2q7CYRERFRFaATwc7OnTsxY8YMLFiwACEhIWjevDl69+6NxMTEym4aERERVTKdCHa+++47TJw4Ee+++y4aNWqENWvWwNTUFBs2bKjsphEREVElM6jsBpRXbm4uLl++jDlz5iiv6enpwcfHB4GBgcXmycnJQU5OjvJrmUwGAEhNTa3YxhIREZHWFH5uCyFKTVftg52kpCTI5XI4OjqqXHd0dMSdO3eKzePv748vv/yyyHU3N7cKaSMRERFVnLS0NFhZWZV4v9oHOy9izpw5mDFjhvJrhUKB5ORk2NraQiKRaK2e1NRUuLm54eHDh7C0tHyp+Vn3y6+7vPlZ96tVd3nzs27WXV3yl7fu0gghkJaWBhcXl1LTVftgx87ODvr6+khISFC5npCQACcnp2LzGBkZwcjISOWatbV1RTURlpaW5foBlyc/6375dZc3P+t+teoub37WzbqrS/7y1l2S0np0ClX7CcpSqRStW7fGiRMnlNcUCgVOnDgBb2/vSmwZERERVQXVvmcHAGbMmIFx48ahTZs2aNeuHVasWIGMjAy8++67ld00IiIiqmQ6EeyMHDkSjx8/xueff474+Hi0aNECR44cKTJp+WUzMjLCggULigyZvYz8rPvl113e/Kz71aq7vPlZN+uuLvnLW7c2SERZ67WIiIiIqrFqP2eHiIiIqDQMdoiIiEinMdghIiIincZgh4iIiHQag50KtGrVKnh6esLY2Bjt27fHpUuX1Mp39uxZDBgwAC4uLpBIJNi3b5/adfr7+6Nt27awsLCAg4MDBg8ejLCwMLXzr169Gs2aNVNu/uTt7Y2//vpL7fzPWrJkCSQSCaZNm6ZW+i+++AISiUTl1aBBA7Xri4mJwdtvvw1bW1uYmJigadOmCA4OViuvp6dnkbolEgn8/PzKzCuXyzF//nx4eXnBxMQEtWvXxqJFi8o8q+VZaWlpmDZtGjw8PGBiYoKOHTsiKCioSLqy3htCCHz++edwdnaGiYkJfHx8EB4ernb+PXv2wNfXV7mb+JUrV9SuPy8vD7NmzULTpk1hZmYGFxcXjB07FrGxsWrV/cUXX6BBgwYwMzNDjRo14OPjg4sXL6rd9mdNnjwZEokEK1asUCvvO++8U+Rn36dPH43qvn37NgYOHAgrKyuYmZmhbdu2iI6OLjNvce87iUSCb7/9Vq2609PTMWXKFLi6usLExER5GLI6eRMSEvDOO+/AxcUFpqam6NOnj/L9os7vkuzsbPj5+cHW1hbm5uYYNmyYcoNXdfKvW7cO3bt3h6WlJSQSCVJSUpT3ysqfnJyMDz/8EPXr14eJiQnc3d3x0UcfQSaTqVX3+++/j9q1a8PExAT29vYYNGiQ8oghTX6PCiHQt29f5fdXnbzdu3cv8vOePHmyRnUHBgbitddeg5mZGSwtLdG1a1csXLiw1Lz3798v8f22a9cuteqOj4/HmDFj4OTkBDMzM7Rq1Qp//PGHWnkjIyMxZMgQ2Nvbw9LSEiNGjCiyIXBFYbBTQXbu3IkZM2ZgwYIFCAkJQfPmzdG7d28kJiaWmTcjIwPNmzfHqlWrNK73zJkz8PPzw4ULF3Ds2DHk5eXB19cXGRkZauV3dXXFkiVLcPnyZQQHB+O1117DoEGDcPPmTY3aERQUhLVr16JZs2Ya5WvcuDHi4uKUr/Pnz6uV7+nTp+jUqRMMDQ3x119/4datW1i+fDlq1KihdnufrffYsWMAgOHDh5eZ95tvvsHq1avx008/4fbt2/jmm2+wdOlSrFy5Uq26AeC9997DsWPH8Ouvv+L69evw9fWFj48PYmJiVNKV9d5YunQpfvzxR6xZswYXL16EmZkZevfujezsbLXyZ2RkoHPnzvjmm29KvF9S/szMTISEhGD+/PkICQnBnj17EBYWhoEDB6pVd7169fDTTz/h+vXrOH/+PDw9PeHr64vHjx+rlb/Q3r17ceHCBZXt49XJ26dPH5X3wPbt29XOHxkZic6dO6NBgwY4ffo0rl27hvnz58PY2LjMvM/WGRcXhw0bNkAikWDYsGFq1T1jxgwcOXIEv/32G27fvo1p06ZhypQpOHDgQKl5hRAYPHgw7t27h/379yM0NBQeHh7w8fFBRkaGWr9Lpk+fjoMHD2LXrl04c+YMYmNjMXToUADq/S7KzMxEnz59MHfu3CLtKyt/bGwsYmNjsWzZMty4cQObNm3CkSNHMGHCBLXqbt26NTZu3Ijbt2/j6NGjEELA19cXcrlco9+jK1asUDlmSN28EydOVPm5L126VO38gYGB6NOnD3x9fXHp0iUEBQVhypQpOH/+fKl53dzcirzfvvzyS5ibm6Nv375q1T127FiEhYXhwIEDuH79OoYOHYoRI0bg4MGDpebNyMiAr68vJBIJTp48iYCAAOTm5mLAgAFQKBRFvq9aJ6hCtGvXTvj5+Sm/lsvlwsXFRfj7+2tUDgCxd+/eF25HYmKiACDOnDnzwmXUqFFD/N///Z/a6dPS0kTdunXFsWPHRLdu3cTUqVPVyrdgwQLRvHnzF2rjrFmzROfOnV8ob3GmTp0qateuLRQKRZlp+/fvL8aPH69ybejQoWL06NFq1ZWZmSn09fXFoUOHVK63atVKfPbZZyXme/69oVAohJOTk/j222+V11JSUoSRkZHYvn17mfmfFRUVJQCI0NBQtesvzqVLlwQA8eDBA43zymQyAUAcP35c7bofPXokatasKW7cuCE8PDzE999/r1becePGiUGDBpXantLyjxw5Urz99tsvlPd5gwYNEq+99pra+Rs3biwWLlyocq24987zecPCwgQAcePGDeU1uVwu7O3txfr164vU/fzvkpSUFGFoaCh27dqlTHP79m0BQAQGBpaZ/1mnTp0SAMTTp0+Lfe6y8hf6/fffhVQqFXl5eRrnvXr1qgAgIiIi1K47NDRU1KxZU8TFxZX4sy0urya/F4vL3759ezFv3rwXyvu8Fi1aFPn9VVp+MzMzsWXLFpV0NjY2Rd4zz+c9evSo0NPTEzKZTJkmJSVFSCQScezYsTKfpbzYs1MBcnNzcfnyZfj4+Civ6enpwcfHB4GBgS+1LTKZDABgY2OjcV65XI4dO3YgIyNDo6M3/Pz80L9/f5XnV1d4eDhcXFxQq1YtjB49GtHR0WrlO3DgANq0aYPhw4fDwcEBLVu2xPr16zWuHyj4+f32228YP368WgfDduzYESdOnMDdu3cBAFevXsX58+fRt29fterLz8+HXC6HsbGxynUTExO1e7YAICoqCvHx8SrfdysrK7Rv3/6lv+8KyWQySCQSjc+ey83Nxbp162BlZYXmzZurlUehUGDMmDGYOXMmGjdurHFbT58+DQcHB9SvXx//+9//8OTJE7XrPXz4MOrVq4fevXvDwcEB7du312j4uVBCQgIOHz6MCRMmqJ2nY8eOOHDgAGJiYiCEwKlTp3D37l34+vqWmi8nJwcAVN53enp6MDIyKvZ99/zvksuXLyMvL0/l/dagQQO4u7sX+34rz+8idfPLZDJYWlrCwMCgyPXS8mZkZGDjxo3w8vKCm5ubWnVnZmbirbfewqpVq0o8h7G0urdu3Qo7Ozs0adIEc+bMQWZmplr5ExMTcfHiRTg4OKBjx45wdHREt27d1PqZPe/y5cu4cuVKie+34vJ37NgRO3fuRHJyMhQKBXbs2IHs7Gx079691Lw5OTmQSCQqGwsaGxtDT09Po99zL6zCw6lXUExMjAAg/vnnH5XrM2fOFO3atdOoLJSjZ0cul4v+/fuLTp06aZTv2rVrwszMTOjr6wsrKytx+PBhtfNu375dNGnSRGRlZQkhNPsL5s8//xS///67uHr1qjhy5Ijw9vYW7u7uIjU1tcy8RkZGwsjISMyZM0eEhISItWvXCmNjY7Fp0ya1215o586dQl9fX8TExKiVXi6Xi1mzZgmJRCIMDAyERCIRixcv1qhOb29v0a1bNxETEyPy8/PFr7/+KvT09ES9evVKzPP8eyMgIEAAELGxsSrphg8fLkaMGFFm/mdpo2cnKytLtGrVSrz11ltq5z148KAwMzMTEolEuLi4iEuXLqld9+LFi0WvXr2UvXGa9Oxs375d7N+/X1y7dk3s3btXNGzYULRt21bk5+eXmb/wr3pTU1Px3XffidDQUOHv7y8kEok4ffq0Ws9d6JtvvhE1atRQ/v9Rp+3Z2dli7NixAoAwMDAQUqlUbN68ucy8ubm5wt3dXQwfPlwkJyeLnJwcsWTJEgFA+Pr6quQt7nfJ1q1bhVQqLVJP27Ztxaefflpm/meV1bOjzu+yx48fC3d3dzF37ly1865atUqYmZkJAKJ+/frF9uqUlH/SpEliwoQJyq+L+9mUlHft2rXiyJEj4tq1a+K3334TNWvWFEOGDFGr7sDAQAFA2NjYiA0bNoiQkBAxbdo0IZVKxd27d9V67kL/+9//RMOGDYu9V1L+p0+fCl9fX+X7zdLSUhw9erTMvImJicLS0lJMnTpVZGRkiPT0dDFlyhQBQEyaNKnENmoLg50KUFWCncmTJwsPDw/x8OFDjfLl5OSI8PBwERwcLGbPni3s7OzEzZs3y8wXHR0tHBwcxNWrV5XXNAl2nvf06VNhaWmp1hCaoaGh8Pb2Vrn24Ycfig4dOmhcr6+vr3j99dfVTr99+3bh6uoqtm/fLq5duya2bNkibGxsNAq0IiIiRNeuXQUAoa+vL9q2bStGjx4tGjRoUGKeqhzs5ObmigEDBoiWLVuqdFuXlTc9PV2Eh4eLwMBAMX78eOHp6SkSEhLKzB8cHCwcHR1VAlRNgp3nRUZGqj2EVvj/fdSoUSrpBgwYIN58802N6q5fv76YMmVKifeLy//tt9+KevXqiQMHDoirV6+KlStXCnNz8yJDA8XlDQ4OFs2bN1e+73r37i369u0r+vTpo5KuuN8lmgQ7Zf0uKivYKSu/TCYT7dq1E3369BG5ublq501JSRF3794VZ86cEQMGDBCtWrUqEmgWl3///v2iTp06Ii0tTXmtuO+vur+DT5w4UewQWnH5C/+fz5kzRyVt06ZNxezZs9WuOzMzU1hZWYlly5YVe7+k/FOmTBHt2rUTx48fF1euXBFffPGFsLKyEteuXSsz79GjR0WtWrWERCIR+vr64u233xatWrUSkydPLuW7ox0MdipATk6O0NfXL/LGHzt2rBg4cKBGZb1osOPn5ydcXV3FvXv3NM77vJ49e6oVee/du1f5S7PwBUD5xi7ur+SytGnTRuU/cEnc3d1V/soSQoiff/5ZuLi4aFTf/fv3hZ6enti3b5/aeVxdXcVPP/2kcm3RokWifv36GtUtRMGHfWGwMmLECNGvX78S0z7/3ij8gH4+QOnatav46KOPysz/rPIEO7m5uWLw4MGiWbNmIikpSaO8z6tTp06xvWTP5//++++V77Nn33t6enrCw8Pjheq2s7MTa9asKbPunJwcYWBgIBYtWqSS7tNPPxUdO3ZUu+6zZ88KAOLKlSsltun5/JmZmcLQ0LDIfK8JEyaI3r17q113SkqKSExMFEIUzDf84IMPlPdK+l1S+AH9fIDi7u4uvvvuuzLzP6u0YKes/KmpqcLb21v07NmzSKCiye/BnJwcYWpqKrZt21Zm/qlTp5b4fuvWrZvGdaenpwsA4siRI2XWfe/ePQFA/PrrryrXR4wYoexFVafuLVu2CENDQ+XP/Vkl5Y+IiCgyz0uIgs+I999/X+26Hz9+rPxZOzo6iqVLl5aYVls4Z6cCSKVStG7dGidOnFBeUygUOHHihEZzX16EEAJTpkzB3r17cfLkSXh5eZW7TIVCoRzfL03Pnj1x/fp1XLlyRflq06YNRo8ejStXrkBfX1+jetPT0xEZGQlnZ+cy03bq1KnIMse7d+/Cw8NDozo3btwIBwcH9O/fX+08mZmZ0NNT/a+kr6//QisMzMzM4OzsjKdPn+Lo0aMYNGiQ2nm9vLzg5OSk8r5LTU3FxYsXK/x9VygvLw8jRoxAeHg4jh8/Dltb23KVp+57b8yYMbh27ZrKe8/FxQUzZ87E0aNHNa730aNHePLkiVrvPalUirZt25b7/ffLL7+gdevWas9RAgq+33l5eeV+/1lZWcHe3h7h4eEIDg7GoEGDyvxd0rp1axgaGqq838LCwhAdHQ1vb+9y/y5SJ39qaip8fX0hlUpx4MAB5fyjF6lbFPzxj5ycnDLzz549u8j7DQC+//57bNiwQeO6C/M7OzuXWbenpydcXFyKfb+5u7urXfcvv/yCgQMHwt7eXuV7UFr+wnlFxb3f5HK52nXb2dnB2toaJ0+eRGJionLFZoWq8HDqFbVjxw5hZGQkNm3aJG7duiUmTZokrK2tRXx8fJl509LSRGhoqAgNDRUAlPMAnl/RUpz//e9/wsrKSpw+fVrExcUpX5mZmWq1e/bs2eLMmTMiKipKXLt2TcyePVtIJBLx999/q5X/eZoMY3388cfi9OnTIioqSgQEBAgfHx9hZ2dX7F8ez7t06ZIwMDAQX3/9tQgPDxdbt24Vpqam4rffflO7rXK5XLi7u4tZs2apnUeIgpU8NWvWFIcOHRJRUVFiz549ws7OrkhXfmmOHDki/vrrL3Hv3j3x999/i+bNm4v27dsX6ZIv672xZMkSYW1trZx/MmjQIOHl5aX8i7es/E+ePBGhoaHi8OHDAoDYsWOHCA0NFXFxcWXmz83NFQMHDhSurq7iypUrKu+/nJycUvOmp6eLOXPmiMDAQHH//n0RHBws3n33XWFkZKT8K1LT/xfPDmOVljctLU188sknIjAwUERFRYnjx4+LVq1aibp164rs7Gy16t6zZ48wNDQU69atE+Hh4WLlypVCX19fnDt3Tq12y2QyYWpqKlavXl3kOcrK361bN9G4cWNx6tQpce/ePbFx40ZhbGwsfv755zLz/v777+LUqVMiMjJS7Nu3T3h4eIihQ4cKIdT7XTJ58mTh7u4uTp48KYKDg4W3t7dyOFmd/HFxcSI0NFSsX79eABBnz54VoaGh4smTJ2Xml8lkon379qJp06YiIiJCJc3kyZNLzRsZGSkWL14sgoODxYMHD0RAQIAYMGCAsLGxEQkJCS/0exT/9pyVlTciIkIsXLhQBAcHi6ioKLF//35Rq1Yt0bVrV7W/b99//72wtLQUu3btEuHh4WLevHnC2NhYvPXWW2q1Ozw8XEgkEvHXX3+pXC+r7tzcXFGnTh3RpUsXcfHiRRERESGWLVsmJBKJ6NevX5l1b9iwQQQGBoqIiAjx66+/ChsbGzFjxowSv6faxGCnAq1cuVK4u7sLqVQq2rVrJy5cuKBWvsIu3edf48aNKzNvcfkAiI0bN6pV9/jx44WHh4eQSqXC3t5e9OzZ84UDHSE0C3ZGjhwpnJ2dhVQqFTVr1hQjR44sdsJgSQ4ePCiaNGkijIyMRIMGDcS6des0auvRo0cFABEWFqZRvtTUVDF16lTh7u4ujI2NRa1atcRnn30mcnJy1C5j586dolatWkIqlQonJyfh5+cnUlJSiqQr672hUCjE/PnzhaOjozAyMhI9e/ZUeZ6y8m/cuLHY+wsWLCgzf+HQV3GvU6dOlZo3KytLDBkyRLi4uAipVCqcnZ3FwIEDVSYoa/r/4tlgp7S8mZmZwtfXV9jb2wtDQ0Ph4eEhJk6cqPKHiTp1//LLL6JOnTrC2NhYNG/eXDkUqk7etWvXChMTkxf6mcfFxYl33nlHuLi4CGNjY1G/fn2xfPlyoVAoysz7ww8/CFdXV2FoaCjc3d3FvHnzlO9bdX6XZGVliQ8++EDUqFFDmJqaiiFDhigDY3XyL1iwoMQ0ZeUv6dlKexXmjYmJEX379hUODg7C0NBQuLq6irfeekvcuXNH7bY/rzDYKStvdHS06Nq1q7CxsRFGRkaiTp06YubMmcq5berW7e/vL1xdXYWpqanw9vYW586dUzvvnDlzhJubm5DL5UWeoaz8d+/eFUOHDhUODg7C1NRUNGvWTGzZskWtvLNmzRKOjo7C0NBQ1K1bV/k+fRkk/z4gERERkU7inB0iIiLSaQx2iIiISKcx2CEiIiKdxmCHiIiIdBqDHSIiItJpDHaIiIhIpzHYISIiIp3GYIeI6DmnT5+GRCJBSkpKZTeFiLSAwQ4RERHpNAY7REREpNMY7BBRlaNQKODv7w8vLy+YmJigefPm2L17N4D/hpgOHz6MZs2awdjYGB06dMCNGzdUyvjjjz/QuHFjGBkZwdPTE8uXL1e5n5OTg1mzZsHNzQ1GRkaoU6cOfvnlF5U0ly9fRps2bWBqaoqOHTsWOWmaiKoHBjtEVOX4+/tjy5YtWLNmDW7evInp06fj7bffxpkzZ5RpZs6cieXLlyMoKAj29vYYMGAA8vLyABQEKSNGjMCbb76J69ev44svvsD8+fOxadMmZf6xY8di+/bt+PHHH3H79m2sXbsW5ubmKu347LPPsHz5cgQHB8PAwADjx49/Kc9PRNrFg0CJqErJycmBjY0Njh8/Dm9vb+X19957D5mZmZg0aRJ69OiBHTt2YOTIkQCA5ORkuLq6YtOmTRgxYgRGjx6Nx48f4++//1bm//TTT3H48GHcvHkTd+/eRf369XHs2DH4+PgUacPp06fRo0cPHD9+HD179gQA/Pnnn+jfvz+ysrJgbGxcwd8FItIm9uwQUZUSERGBzMxM9OrVC+bm5srXli1bEBkZqUz3bCBkY2OD+vXr4/bt2wCA27dvo1OnTirldurUCeHh4ZDL5bhy5Qr09fXRrVu3UtvSrFkz5b+dnZ0BAImJieV+RiJ6uQwquwFERM9KT08HABw+fBg1a9ZUuWdkZKQS8LwoExMTtdIZGhoq/y2RSAAUzCciouqFPTtEVKU0atQIRkZGiI6ORp06dVRebm5uynQXLlxQ/vvp06e4e/cuGjZsCABo2LAhAgICVMoNCAhAvXr1oK+vj6ZNm0KhUKjMASIi3cWeHSKqUiwsLPDJJ59g+vTpUCgU6Ny5M2QyGQICAmBpaQkPDw8AwMKFC2FrawtHR0d89tlnsLOzw+DBgwEAH3/8Mdq2bYtFixZh5MiRCAwMxE8//YSff/4ZAODp6Ylx48Zh/Pjx+PHHH9G8eXM8ePAAiYmJGDFiRGU9OhFVEAY7RFTlLFq0CPb29vD398e9e/dgbW2NVq1aYe7cucphpCVLlmDq1KkIDw9HixYtcPDgQUilUgBAq1at8Pvvv+Pzzz/HokWL4OzsjIULF+Kdd95R1rF69WrMnTsXH3zwAZ48eQJ3d3fMnTu3Mh6XiCoYV2MRUbVSuFLq6dOnsLa2ruzmEFE1wDk7REREpNMY7BAREZFO4zAWERER6TT27BAREZFOY7BDREREOo3BDhEREek0BjtERESk0xjsEBERkU5jsENEREQ6jcEOERER6TQGO0RERKTTGOwQERGRTvt/SlQpQMz5O+MAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plt.plot(np.arange(len(epochs_x)), epochs_acc, marker = '.')\n", + "plt.xlabel('epoch')\n", + "plt.ylabel('test accuracy')\n", + "plt.ylim(0, 100)\n", + "plt.xticks(np.arange(len(epochs_x)))\n", + "for i, txt in enumerate(epochs_acc):\n", + " if i%2 == 0:\n", + " pass\n", + " else:\n", + " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", + "plt.show()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "speck-rescnn", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.0" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tests/test_nonsequential/using_AvgPool2d/exp_set_A/baseline-SCNN-example_3.ipynb b/tests/test_nonsequential/using_AvgPool2d/exp_set_A/baseline-SCNN-example_3.ipynb new file mode 100644 index 00000000..dc53313a --- /dev/null +++ b/tests/test_nonsequential/using_AvgPool2d/exp_set_A/baseline-SCNN-example_3.ipynb @@ -0,0 +1,1500 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import torch, random\n", + "import torch.nn as nn\n", + "import sinabs.layers as sl\n", + "from tqdm.notebook import tqdm\n", + "\n", + "from tonic.datasets.dvsgesture import DVSGesture\n", + "from tonic.transforms import ToFrame\n", + "from torch.utils.data import DataLoader\n", + "from torch.nn import CrossEntropyLoss\n", + "from torch.optim import Adam\n", + "\n", + "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "torch.backends.cudnn.deterministic = True\n", + "random.seed(1)\n", + "torch.manual_seed(1)\n", + "torch.cuda.manual_seed(1)\n", + "np.random.seed(1)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Loading Data" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "batch_size = 3\n", + "num_workers = 1\n", + "epochs = 30\n", + "lr = 1e-3" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "root_dir = \"./DVSGESTURE\"\n", + "_ = DVSGesture(save_to=root_dir, train=True)\n", + "_ = DVSGesture(save_to=root_dir, train=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "n_time_steps = 50\n", + "to_raster = ToFrame(sensor_size=DVSGesture.sensor_size, n_time_bins=n_time_steps)\n", + "\n", + "snn_train_dataset = DVSGesture(save_to=root_dir, train=True, transform=to_raster)\n", + "snn_test_dataset = DVSGesture(save_to=root_dir, train=False, transform=to_raster)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The transformed array is in shape [Time-Step, Channel, Height, Width] --> (50, 2, 128, 128)\n" + ] + } + ], + "source": [ + "sample_data, label = snn_train_dataset[0]\n", + "print(f\"The transformed array is in shape [Time-Step, Channel, Height, Width] --> {sample_data.shape}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "snn_train_dataloader = DataLoader(snn_train_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True)\n", + "snn_test_dataloader = DataLoader(snn_test_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=False)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Network Module\n", + "\n", + "We need to define a `nn.Module` implementing the network we want the chip to reproduce." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "device: NVIDIA RTX A4000\n" + ] + } + ], + "source": [ + "if torch.cuda.is_available():\n", + " device = torch.device('cuda:0')\n", + " print('device: ', torch.cuda.get_device_name(0))\n", + "else:\n", + " device = torch.device('cpu')" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "class SNN(nn.Module):\n", + " def __init__(self) -> None:\n", + " super().__init__()\n", + "\n", + " self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False)\n", + " self.iaf1 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + " self.pool1 = nn.AvgPool2d(2,2)\n", + "\n", + " self.conv2 = nn.Conv2d(10, 10, 2, 1, bias=False)\n", + " self.iaf2 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + " self.pool2 = nn.AvgPool2d(3,3)\n", + "\n", + " self.conv3 = nn.Conv2d(10, 10, 3, 1, bias=False)\n", + " self.iaf3 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + " self.pool3 = nn.AvgPool2d(2,2)\n", + "\n", + " self.flat = nn.Flatten()\n", + "\n", + " self.fc1 = nn.Linear(810, 100, bias=False)\n", + " self.iaf4 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + "\n", + " self.fc2 = nn.Linear(100, 100, bias=False)\n", + " self.iaf5 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + "\n", + " self.fc3 = nn.Linear(100, 100, bias=False)\n", + " self.iaf6 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + "\n", + " self.fc4 = nn.Linear(100, 11, bias=False)\n", + " self.iaf7 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", + "\n", + " def detach_neuron_states(self):\n", + " for name, layer in self.named_modules():\n", + " if name != '':\n", + " if isinstance(layer, sl.StatefulLayer):\n", + " for name, buffer in layer.named_buffers():\n", + " buffer.detach_()\n", + "\n", + " def init_weights(self):\n", + " for name, layer in self.named_modules():\n", + " if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear):\n", + " nn.init.xavier_normal_(layer.weight.data)\n", + "\n", + " def forward(self, x):\n", + " \n", + " con1_out = self.conv1(x)\n", + " iaf1_out = self.iaf1(con1_out)\n", + " pool1_out = self.pool1(iaf1_out)\n", + "\n", + " conv2_out = self.conv2(pool1_out)\n", + " iaf2_out = self.iaf2(conv2_out)\n", + " pool2_out = self.pool2(iaf2_out)\n", + "\n", + " conv3_out = self.conv3(pool2_out)\n", + " iaf3_out = self.iaf3(conv3_out)\n", + " pool3_out = self.pool3(iaf3_out)\n", + "\n", + " flat_out = self.flat(pool3_out)\n", + " \n", + " fc1_out = self.fc1(flat_out)\n", + " iaf4_out = self.iaf4(fc1_out)\n", + "\n", + " fc2_out = self.fc2(iaf4_out)\n", + " iaf5_out = self.iaf5(fc2_out)\n", + "\n", + " fc3_out = self.fc3(iaf5_out)\n", + " iaf6_out = self.iaf6(fc3_out)\n", + "\n", + " fc4_out = self.fc4(iaf6_out)\n", + " iaf7_out = self.iaf7(fc4_out)\n", + "\n", + " return iaf7_out" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "snn = SNN().to(device)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "snn.init_weights()" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "optimizer = Adam(snn.parameters(), lr=lr, betas=(0.9, 0.999), eps=1e-8)\n", + "loss_fn = CrossEntropyLoss()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Define train and test" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "def train(dataloader, model, loss_fn, optimizer, epochs, test_func, dataloader_test):\n", + " epochs_y = []\n", + " epochs_x = []\n", + " epochs_acc = []\n", + " model.train()\n", + "\n", + " for e in range(epochs):\n", + " losses = []\n", + " batches = []\n", + " batch_count = 0\n", + " train_p_bar = tqdm(snn_train_dataloader)\n", + "\n", + " for X, y in train_p_bar:\n", + " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", + " X = X.reshape(-1, 2, 128, 128).to(dtype=torch.float, device=device)\n", + " y = y.to(dtype=torch.long, device=device)\n", + "\n", + " # forward\n", + " pred = model(X)\n", + "\n", + " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", + " pred = pred.reshape(batch_size, n_time_steps, -1)\n", + "\n", + " # accumulate all time-steps output for final prediction\n", + " pred = pred.sum(dim = 1)\n", + " loss = loss_fn(pred, y)\n", + "\n", + " # gradient update\n", + " optimizer.zero_grad()\n", + " loss.backward()\n", + " optimizer.step()\n", + "\n", + " # detach the neuron states and activations from current computation graph(necessary)\n", + " model.detach_neuron_states()\n", + "\n", + " train_p_bar.set_description(f\"Epoch {e} - BPTT Training Loss: {round(loss.item(), 4)}\")\n", + "\n", + " batch_count += 1\n", + " losses.append(loss.item())\n", + " batches.append(batch_count)\n", + "\n", + " epochs_y.append(losses)\n", + " epochs_x.append(batches)\n", + "\n", + " acc = test_func(dataloader_test, model)\n", + " print(f'Epoch {e} accuracy: {acc}')\n", + " epochs_acc.append(acc)\n", + "\n", + " return epochs_x, epochs_y, epochs_acc\n" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "def test(dataloader, model):\n", + " correct_predictions = []\n", + " with torch.no_grad():\n", + " test_p_bar = tqdm(dataloader)\n", + " for X, y in test_p_bar:\n", + " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", + " X = X.reshape(-1, 2, 128, 128).to(dtype=torch.float, device=device)\n", + " y = y.to(dtype=torch.long, device=device)\n", + "\n", + " # forward\n", + " output = model(X)\n", + "\n", + " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", + " output = output.reshape(batch_size, n_time_steps, -1)\n", + "\n", + " # accumulate all time-steps output for final prediction\n", + " output = output.sum(dim=1)\n", + "\n", + " # calculate accuracy\n", + " pred = output.argmax(dim=1, keepdim=True)\n", + "\n", + " # compute the total correct predictions\n", + " correct_predictions.append(pred.eq(y.view_as(pred)))\n", + "\n", + " test_p_bar.set_description(f\"Testing Model...\")\n", + " \n", + " correct_predictions = torch.cat(correct_predictions)\n", + " return correct_predictions.sum().item()/(len(correct_predictions))*100" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Training loop" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "ebf2bea3d0124365abf1f2aa248ceab6", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/359 [00:00" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "y_avg = []\n", + "for y in epochs_y:\n", + " y_avg.append(np.mean(y))\n", + "\n", + "plt.plot(np.arange(len(epochs_x)), y_avg, marker = '.')\n", + "plt.xlabel('epoch')\n", + "plt.ylabel('average loss')\n", + "plt.ylim(0,)\n", + "plt.xticks(np.arange(len(epochs_x)))\n", + "for i, txt in enumerate(y_avg):\n", + " if i%2 == 0:\n", + " pass\n", + " else:\n", + " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAG2CAYAAACZEEfAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAAB9HElEQVR4nO3dd1RUx98G8GfpRUB6kaqg2HvB3oIt9qiJJjHGWH5irLFgYkw0ETWJJsbE9ipqrDGxG3vBhgiKXSmKglQLLL3uvH8QNq60XYrg+nzO2XNk7p07c3evu987d4pECCFAREREpKY0qroCRERERJWJwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqbUqDXbOnTuH/v37w87ODhKJBPv27VPYLoTA119/DVtbW+jr66Nnz54ICwtT2OfFixcYNWoUjI2NUbNmTYwdOxapqamv8SyIiIioOqvSYCctLQ1NmzbFb7/9VuT2ZcuWYeXKlVizZg0CAgJgaGiIXr16ITMzU77PqFGjcOfOHZw4cQKHDh3CuXPnMH78+Nd1CkRERFTNSarLQqASiQR79+7FoEGDAOS36tjZ2WHmzJn44osvAABSqRTW1tbYtGkT3n//fdy7dw8NGjRAYGAgWrVqBQA4evQo+vbtiydPnsDOzq6qToeIiIiqCa2qrkBxIiIiEBcXh549e8rTTExM0LZtW/j7++P999+Hv78/atasKQ90AKBnz57Q0NBAQEAABg8eXOSxs7KykJWVJf9bJpPhxYsXMDc3h0QiqbyTIiIiogojhEBKSgrs7OygoVH8w6pqG+zExcUBAKytrRXSra2t5dvi4uJgZWWlsF1LSwtmZmbyfYri4+ODb7/9toJrTERERFUhKioK9vb2xW6vtsFOZfL29saMGTPkf0ulUjg6OiIqKgrGxsZVWDMiIiJSVnJyMhwcHGBkZFTiftU22LGxsQEAxMfHw9bWVp4eHx+PZs2ayfdJSEhQyJebm4sXL17I8xdFV1cXurq6hdKNjY0Z7BAREb1hSuuCUm3n2XFxcYGNjQ1OnTolT0tOTkZAQAA8PDwAAB4eHkhKSsLVq1fl+5w+fRoymQxt27Z97XUmIiKi6qdKW3ZSU1MRHh4u/zsiIgLXr1+HmZkZHB0dMW3aNHz33Xdwc3ODi4sL5s+fDzs7O/mIrfr166N3794YN24c1qxZg5ycHEyePBnvv/8+R2IRERERgCoOdoKCgtCtWzf53wX9aEaPHo1NmzZh9uzZSEtLw/jx45GUlISOHTvi6NGj0NPTk+fZtm0bJk+ejB49ekBDQwNDhw7FypUrX/u5EBERUfVUbebZqUrJyckwMTGBVCplnx0iIqI3hLK/39W2zw4RERFRRWCwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGptWod7OTl5WH+/PlwcXGBvr4+6tSpg0WLFkEIId9HCIGvv/4atra20NfXR8+ePREWFlaFtSYiIqLqpFoHO0uXLsXq1auxatUq3Lt3D0uXLsWyZcvw66+/yvdZtmwZVq5ciTVr1iAgIACGhobo1asXMjMzq7DmREREVF1IxMvNJNXMu+++C2tra2zYsEGeNnToUOjr62Pr1q0QQsDOzg4zZ87EF198AQCQSqWwtrbGpk2b8P777ytVTnJyMkxMTCCVSmFsbFwp50JEREQVS9nf72rdstO+fXucOnUKoaGhAIAbN27gwoUL6NOnDwAgIiICcXFx6NmzpzyPiYkJ2rZtC39//2KPm5WVheTkZIUXERERqSetqq5ASebOnYvk5GS4u7tDU1MTeXl5+P777zFq1CgAQFxcHADA2tpaIZ+1tbV8W1F8fHzw7bffVl7FiYiIqNqo1i07f/75J7Zt24bt27fj2rVr2Lx5M3788Uds3ry5XMf19vaGVCqVv6KioiqoxkRERFTdVOuWnVmzZmHu3LnyvjeNGzfG48eP4ePjg9GjR8PGxgYAEB8fD1tbW3m++Ph4NGvWrNjj6urqQldXt1LrTkRERNVDtW7ZSU9Ph4aGYhU1NTUhk8kAAC4uLrCxscGpU6fk25OTkxEQEAAPD4/XWlciqhqx0gxcevAMsdKMqq4KVSJnZ2dIJJJCLy8vLwBA165dC22bOHFiiccs6ngSiQQ//PCDwn6HDx9G27Ztoa+vD1NTUwwaNKiyTpMqSbVu2enfvz++//57ODo6omHDhggODsby5cvx6aefAsi/UKdNm4bvvvsObm5ucHFxwfz582FnZ8eLkegtsCswEt57bkEmAA0J4DOkMUa0dqzqalElCAwMRF5envzv27dv45133sGwYcPkaePGjcPChQvlfxsYGJR4zNjYWIW/jxw5grFjx2Lo0KHytL///hvjxo3D4sWL0b17d+Tm5uL27dvlPR16zap1sPPrr79i/vz5mDRpEhISEmBnZ4cJEybg66+/lu8ze/ZspKWlYfz48UhKSkLHjh1x9OhR6OnpVWHNiaiyxUozMHfPLRRMniETwLw9t9G5riVsTfSrtnJU4SwtLRX+XrJkCerUqYMuXbrI0wwMDOTdG5Tx6r779+9Ht27dULt2bQBAbm4upk6dih9++AFjx46V79egQYOynAJVoWr9GMvIyAg///wzHj9+jIyMDDx48ADfffcddHR05PtIJBIsXLgQcXFxyMzMxMmTJ1G3bt0qrDURvQ7XHifi1VnC8oTAo2fpVVMhem2ys7OxdetWfPrpp5BIJPL0bdu2wcLCAo0aNYK3tzfS05W/FuLj43H48GGFoObatWuIjo6GhoYGmjdvDltbW/Tp04ctO2+gat2yQ0RUlPTsXKw8XXhZGE0J4GxR8qMLevPt27cPSUlJ+OSTT+RpI0eOhJOTE+zs7HDz5k3MmTMHISEh2LNnj1LH3Lx5M4yMjDBkyBB52sOHDwEA33zzDZYvXw5nZ2f89NNP6Nq1K0JDQ2FmZlah50WVh8EOEb1R8mQCU3ZcR0hcKgx1NJGRkwfZvy08ozu48BHWW2DDhg3o06cP7Ozs5Gnjx4+X/7tx48awtbVFjx498ODBA9SpU6fUY27cuBGjRo1S6AJRMBjmyy+/lPfj8fX1hb29PXbv3o0JEyZU1ClRJavWj7GIiF7l8889nLwXDx0tDWwZ2xYX53ZH30b5fS9uRiWhGq+AQxXg8ePHOHnyJD777LMS92vbti0AIDw8vNRjnj9/HiEhIYWOWTClyct9dHR1dVG7dm1ERkaqWnWqQgx2iOiNsfXyY/zfhQgAwE/DmqKlkylsTfSxYEBD6GhqIOhxIgIiXlRxLaky+fr6wsrKCv369Stxv+vXrwOAwhxsxdmwYQNatmyJpk2bKqS3bNkSurq6CAkJkafl5OTg0aNHcHJyUr3yVGUY7BBRmZU298mECRNQp04d6Ovrw9LSEgMHDsT9+/eVPv7EiRMhkUjw888/wy/0KRYcuAMA+Li+Nv7v6//BwsICxsbGGNq3J1rrxgAAfjtT+p08vZlkMhl8fX0xevRoaGn91wvjwYMHWLRoEa5evYpHjx7hwIED+Pjjj9G5c2c0adJEvp+7uzv27t2rcMzk5GTs3r27yJYiY2NjTJw4ERMmTJBf2zo6OkhISMCECRMq5Tp/WWhoKAYOHCi/zjt27IgzZ84ofVz6D4MdIiqzwMBAxMbGyl8nTpwAAPncJy1btoSvry/u3buHY8eOQQgBT09PhflSirN3715cvnwZdnZ2SEjOhNe2a8iTCQxtYY+d309Gbm4uTp8+jatXr6Jp06bYu3QKRHoizoc9Q3BkYqWeN1WNkydPIjIyUj7XWgEdHR2cPHkSnp6ecHd3x8yZMzF06FAcPHhQYb+QkBBIpVKFtJ07d0IIgQ8++KDIMn/44QeMHz8e5ubmMDQ0RKdOnbB+/XoAFX+dv+rdd98tdJ2/++67Ja79SMUQJKRSqQAgpFJpVVeF6I02depUUadOHSGTyYrcfuPGDQFAhIeHl3icJ0+eiFq1aonbt28LB0dH4dxvknCac0gMX3NJRMfGCwDi3Llz8v2Tk5MFADH0yzXCac4hMXbTlQo9L8rn5OQkABR6TZo0SQghxPjx40Xt2rWFnp6esLCwEAMGDBD37t0r8Zh///23eOedd4SZmZkAIIKDgwvts3btWtGlSxdhZGQkAIjExMQyn0NMUrq4GP5UxCSll/kYlXGdOzk5iRUrVsi3PX36tNjr/MSJE2WuuyrehM9b2d9vtuwQUYUobu6TAmlpafD19YWLiwscHByKPY5MJsNHH32EWbNmobabO56lZkOakQMXC0Os/aglbK0tUa9ePWzZsgVpaWnIzc3F2rVrYWVlhXmj+0IiAU7eS8DdmOTKPN23UmW05KWlpaFjx45YunRpsfukp6ejd+/emDdvXrnqvyswEh2WnMbI9QHosOQ0dgWq3sm4Mq7zhg0bFtpubm6OevXqoW/fvvJHaMbGxgCAd955B15eXnjx4gU+//xz1KtXD/r6+nB0dMSUKVMKtV69SpllMgYMGICcnBzo6OjA0tISQ4cOxY4dOwC8OZ+3glICu7cCW3aIym/Xrl1CU1NTREdHK6T/9ttvwtDQUAAQ9erVK/Vud/HixeKdd94Rubl5YuIfQULT2ErY9Z4gHj5Nle8TFRUlWrZsKSQSidDU1BS2trbi2rVrQgghvLZdFU5zDolJ265W/EmSgopq4RBCiIiIiGLv9AucOXOmzC07MUnpwnnuIeE0579X7bmHVW7hqejrvOC9e7VlR4j867xJkyYCgNDQ0BBWVlZi9erVAoA4c+aMuHXrlhgyZIg4cOCACA8PF6dOnRJubm5i6NChJZYdGxur8Nq4caOQSCTiwYMH8n2WL18u/P39xaNHj8TFixeFh4eHsLGxqXafN1t2iN4SJXUSLuud3zfffAN3d3cYGhrC1NQUPXv2REBAQInljhgxAm5uboX6HowaNQrBwcHw8/ND3bp1MXz4cGRmZhZZ7tWrV/HLL79g06ZN+OF4KI7cjoNEAgxv5QgXC0MAgBACXl5esLKywvnz53HlyhUMGjQI/fv3R2xsLLy6uQIA/rkViwdPU8v6tlIpKqqF43WJeJZWITNuFzXHD1D267yo9w747zqvVasWLly4gMDAQAwdOhRffPEFnJ2d0aVLFzRq1Ah///03+vfvjzp16qB79+74/vvvcfDgQeTm5hZ7DjY2NgqvV5fJAIDp06ejXbt2cHJyQvv27TFz5kzExcVh9OjRb8TnXUip4ddbgC079CZLSEhQuEs7ceJEue/8tm3bJk6cOCEePHggbt++LcaOHSuMjY1FQkKCfB8nJyexcOFCERsbK65cuSI0NDTEzp07SzxuVlaWMDAwENu3by9y+4oVK4REIhEampoCEo381793tU5OTkIIIU6ePCk0NDQK/X91dXUVPj4+Qgghxm4KFE5zDokZu66X9vZRGVVUC0eBym7ZiU5MU2jVKUvLzqNHj4SGhobYt29fifspe51ramrKX8pc51lZWUJDQ0N4enoWW/b69euFhYWF0ucUFxcntLS0xLZt24rd5/nz58LDw0MAqHafN1t2iN4SlpaWCndphw4dki+QWNY7v5EjR6Jnz56oXbs2GjZsiOXLlyM5ORk3b95U2M/IyAg2NjY4fPgwrKysFFaLLooQAkIIZGVlFbn9o48+wuZDfrAbsxK2Y1biizX7YWdnh1mzZuHYsWMAIF/vSEND8etLQ0NDPuPt5O75rTv7rkcj6gXXyqoMFdHC8TrFJRe+5ub2cVdpxm1l5/hR5jq/efMmrl+/Ln8pc53v27cPMpkMzZs3L/K4z549w6JFixRmky5NUctkFJgzZw4MDQ1hbm6Ou3fvwtPT8435vF/FYIdIjZT2aAEApFIpjI2NFeYpKe2Y69atg4mJSaFJ15YsWQIzMzMsXrwY7u7uCtsePnwIHx8fXL16FZGRkbh06RKGDRsGfX199O3bV77fy3OfJObpYNmVdGiaO2HYO+2xbFw/aGtrw8bGBvXq1QMAeHh4wNTUFKNHj8aNGzcQGhqKWbNmISIiQv4j1MyhJjq5WSBPJrDG74Fybx4praRZjE1MTODm5obOnTvjr7/+wv379wvNbVMVDt2IBQB4NrBGPesaAIC07OID/lcVN8dPWa5zc3NzNGrUSOGlzHU+Z84cSCSSIofJJycno1+/fmjQoAG++eYbpc+rqGUyCsyaNQvBwcH4448/IJVK8fz580IzlFfXz/tVDHaI1EhRCyS+TJU7v0OHDqFGjRrQ09PDihUrcOLECVhYWMi3T5kyBTt37sTixYuRk5ODq1evYvbs2fLtenp6OH/+PPr27QtXV1eMGDECRkZGuHTpEqysrOT7Fcx98iw1C2M2BSIlMxetnEyxdGiTIgM2CwsLHD16FKmpqejevTtatWqFCxcuYP/+/QrB2OR/++7sDnqCOGn1u9N8k1VUC8frIpMJ/HMrP9gZ1soBk/69NnZeiUJunkypYxQ3x4+q17myXr3OW7RogUePHuGrr74qdNORkpKC3r17w8jICHv37oW2trZSZRS3TMbLdahbty7Cw8NhZWWFq1ev4vLly8Uer7p83kVS6uGammOfHVIXnp6e4t133y1ym1QqFW3atBG9e/cW2dnZpR4rNTVVhIWFCX9/f/Hpp58KZ2dnER8fX+z+GzZsEFpaWiIzM1OlOsckpYsz9+NFv1/OCac5h0SnpafFsxTVjlGc91ZfFE5zDomFB+9UyPFIiLy8POHo6CjmzJmjkP7gwQOxePFiERQUJB4/fiwuXrwo+vfvL8zMzBSum3r16ok9e/bI/37+/LkIDg4Whw8fFgDEzp07RXBwsIiNjZXvExsbK4KDg8X69evlc88EBweL58+fK1XnwIjnwmnOIdHo66MiIztXZObkihYLjwunOYfEkVuxpR+ggpRnjp8FCxYIGxsbkZOTo5AulUpFu3btRJcuXURaWppKxxw9erRo2bJlifsUfN7/+9//5H0Bhag+n7eyv98MdgSDHVIPJXWeTE5OFh4eHqJHjx4iIyOjTMd3dXUVixcvLnb77du3BQBx//59pY+588pj4fLScOB6X/4jwuJTylS/opwNScg/7lf/VFgA9bY7duyYACBCQkIU0qOjo0WfPn2ElZWV0NbWFvb29mLkyJGFrgcAwtfXV/63r69vkRPXLViwQL7PggULitzn5eOUZMH+28JpziExfWewPG3JkXvCac4h8eH/XVb1LSiTl691l7mHxM4rj5XOW1yAKZVKRdu2bUXjxo1FeHi4wkCF3Nxc+X6vBhwFeQ0MDMTq1asLlXf58mXx66+/iuDgYLFlyxYBQDRv3lzUqVNHfjNTXT5vBjsqYLBD6qAy7vxeVrt2bYUvpFdt3bpVaGhoiBcvXih1vIinqcL5ldExLnMPlWtm21fJZDLR/9fzwmnOIbH0SMkzu74pSpvVNiMjQ0yaNEmYmZkJQ0NDMWTIEBEXF6f08SdMmCAAFJrzpahyC0a/qaoiZjFWVl6eTLT+7oRwmnNInLz73/sQ+TxNPu/Oy3M4VYaYpHSFoF7VkWDFBZgFo5WKekVERMj3KypQWLt2rdDX1xdJSUmFyrt586bo1q2bMDMzE7q6usLZ2Vl8/OlnYt+Fm2X6zCrz81b291u5HopEVK3JZDL834aN6DlgGJ6m5cDWJP+/dnJyMjw9PZGeno6tW7ciOTkZycn5MwtbWlpCU1MTQH7nSR8fHwwePBhpaWn4/vvvMWDAANja2uLZs2f47bffEB0dLZ851d/fHwEBAejWrRuMjIzg7++P6dOn48MPP4SpqWkJ9RQIiHiBfcHROHAjGuLV7QJ49CxdpREyJZFIJPDq5ooJf1zFFv/HmNC5DkwMlOvPUF0FBgYqzFB7+/ZtvPPOO/LPZvr06Th8+DB2794NExMTTJ48GUOGDMHFixdLPXZJ6zQBwMKFCzFu3Dj530ZGRirXf3vAY3y59zYEAA0J4DOkMUa0dlT5OMoKepyIhJQsGOlpoaPbf33OHMwM0K2eFU7fT8C2y4/x1bsNKq0OEc/SICtmjh9lrnVPT89CHYMBoGvXrkWmv0oIgVhpBi49eAYXC0PYmuhj/PjxRfbdE0LAvnY9rN2xH09Ts/AsNRsn7sTh0M1Y+B2MhORgJLrWs0QDO+NSywWAuzHJOBvy9LV93sVhsEOkBr76bTuin0ThtGiEDktOy79Qrl27Jp8M0NXVVSFPREQEnJ2dAeR3ngwKe4J20gyY6mri/v372Lx5M549ewZzc3O0bt0a58+fl09rr6uri507d+Kbb75BVlYWXFxcMH36dMyYMaPI+oXGp2BvcDT2B0cjpoTOwpoSCZwtDJQ+7+joaMyZMwdHjhxBeno6XF1d4evri1atWgEA4uPjsX3ZXMTtO4Ts9FS0+6cNDmzfADc3t2KP2bVrV/j5+RVK79u3Lw4fPgwg/wdh+vTpWLN2LbIyM6GhoQFXV1ds27ZNoew5c+bg+PHjSEpKQufOnfHrr7+Wu2wLCwssWLAA69evR1JSEiwtLeHo6IguXbpAKpViw4YN2L59O7p37w4gvzNx/fr1cfnyZbRr167E9/Lzzz/HsWPHiu14XDDVQFnFSjPkgQ6QH9zO23MbnetaVliA+6pDN2MAAJ4NbKCrpamw7cN2jjh9PwG7rz7BF73qQU9bs6hDlJuFoW6hNA0JVLrWy2NXYCS899yCTAASCTCilQNcrWrkBzMp2XiWmiV/PU/NRu6rkdlLBIAzIU9xJuSpyvV4HZ93cRjsEL3hYpLSsSPaFE5zDgHI/0KZ8/ctbLr0GLVqGmLOXzdgUUMXFjV0YGmkB4saOrAw0oVpDV0IIfBnUBRc5h7CthfAjn8DpT179pRYZosWLUoclQEA8cmZOHA9BnuDo3E39r91qoz0tNCvsS0GNa+FiGdp+GrvbeQJAU2JBIuHNFL6SzAxMREdOnRAt27dcOTIEVhaWiIsLEzesiSEwKBBg6CtrY3vf9+C5WejkBB8AD169MS9e3dhaGhY5HH37NmD7Oxs+d/Pnz9H06ZN5S0nQP4M0ytXroSuQ2OYN3kHabdOIv5ZDAwMDAqVvX//fhgbG2P58uXo2bMn7t4tX9nLli3DypUrsXnzZtjb26N9+/YwNDREVlYWrl69ipycHPTs2VO+v7u7OxwdHeHv719ssFPaOk0FlixZgkWLFsHR0REjR47E9OnTlZ7CAADO3E8o1JqnSguHqvJkAv/cyl8h/N2mtoW2d6lrBXtTfTxJzMDBGzEY1qpyZv7dez26UJqbVQ3YGBce7l3RYqUZ8kAHAIQAdgZGlZrPWE8LFka60NHUwP24lELbezeyLvUzi03KwNE78Qpplfl5l4TBDlEFUKaFQdW7/D179mDx4sUIDw9HTk4O3NzcMHPmTHz00UcAgJycHMz1nocNO/Yg+Wk0NHQNoefUFDW7fAItI3Pci03GvdiSF8PU1pAg56W7OJkA5v59C7FJmahtVSM/QKqhC4sauqhpoF3s3D2x0gxEPEuDlZEebkQlYd/1aFwMfyb/gtXWlKBrPSsMbl4L3d2t5HfQ7Wqbo2s9Szx6lg5nCwOVvgCXLl0KBwcH+Pr6ytNcXFzk/w4LC8Ply5dx+/ZtuNdvgEOxfnhoNhGJ6z7Bjh07ih1ua2ZmpvD3zp07YWBgIA84hBD48ccfoWlsBesPFgMADFzb4MmvH8LvchAaNGigUHZB8LB69WrY2NiUu+yff/4ZX331FQYOHIg///wTeXl5SEtLk084p6Ojg5o1ayocx9raGnFxcSW+l1paWpgyZUqx+0yZMgUtWrSAmZkZLl26BG9vb8TGxmL58uXF5nlZTp4Mmy49KpSuamueKq5EvMCz1CyY6GujQx2LQts1NSQY2dYRy46GYGtAZKUEOwnJmfC9GAEAWDq0CTQ1AO+/byEkPhV7g6MxpIV9hZf5socJhR+hAYBHHXO42xjBooZu/v9xI51/b4p0YV5DR94KFivNQIclpxWOoSmRYEH/hqUHO9IMHL8bXyjv62rRehmDHaJyUqWFQZW7fDMzM3z55Zdwd3eHjo4ODh06hDFjxsDKygq9evVCeMxzbD54FtqthsHWygWyzFS8OLUOT/csQq1PfobP4MbIFaJQM/Wz1Gw8S8lCSlauQqBTQAD4+VRYoXQtDQnMa+R/IVoa6cq/GKNepOOfW7GF7tgBoKWTKQY3r4V+jW1haqhT5HnamuiX6S7vwIED6NWrF4YNGwY/Pz/UqlULkyZNkvcpKZjrQ09PD5oaEvyvax3M/usmMmWa8Dt3vtiA41UbNmzA+++/L/+cHj58iPT0dBg2aIOn+3yQGXUbmjXMoWlsiTPnLuB/n36sUHYBDQ0N6Orq4sKFC2UuOyIiAnFxcfKWm4JZjJOTk+Hv74+2bdsqddyXFazTdO3atWKDWQAKjyibNGkCHR0dTJgwAT4+PtDVLfyY5lXrzj1EaHwq9LU1kZWbJ/8BnOlZt9Lu8g/fyn+E1auhNXS0ip5WbngrB6w4EYobUUm49USKxvYmFVqHX0+HIzNHhhaONTG8lT0kEgnik7Pww7EQfHvwLjq6WcDKqPJaeC4+eFYoTVMiwfLhTZV6321N9OEzpDHm7VG9BbY8eSsagx2iclKlhUGVu/yuXbsq/D116lRs3rwZFy5cQE23Vpi07RZqDP4G5oY6eK+lPf7vfATwzkTEbZmBme3NMKJNyZ0AM3PycDdGiqFr/BUWSJQA6OZuhbSsXHlwJM3IQa5MID45C/FFTLv/qnGdXPBRO2c4mlfeHdzDhw+xevVqzJgxA/PmzUNgYCCmTJkCHR0djB49Wv74xtvbG2vXrkW/hpaYNX8fcpKf4npIhFJlXLlyBbdv38aGDRsAAJHP0+G14QwAIC3kIoxbD4K1x3BkxYbhxfHfccrvImKSMgqVbWhoiBUrVuDJkyeIjY0tU9kA5K0z1tbW8lmM9+zZg23btiEuLg42NjbIzs5GUlKSQutOfHx8sX1tzp8/j4SEBDg6/ne95OXlYebMmfj555/x6NGjIvO1bdsWubm5ePTokXzW3+KEJ6Tgl38D6O8GNUJ7V3NM+OMqbj6R4kVadol5yyo3T4ajt/Pfr35Niu5wDQAWNXTRt7Et9l+PwdbLj7H0vSYVVofHz9Ow40okAGB2b3d5MDm+c238cysWd2KSsWD/Haz+sGWFlfmyq48Tse7cQwD5fXWEQJkCjhGtHdG5btlaYMuTtyJxBmWicjpw4ABatWqFYcOGwcrKCs2bN8f69evl20u7y1eGEAKnTp1CSEgIsi3rYdT/BeB5WjYa2hnjwOcd4d23Pi7M7QbvHk6QSCT4qEvx/S4K6GlrooWTGZYMaQzNf7+ENSUSLBnaGBs/aY1dEzxwamZX3FjgidDv+sDfuzsOTO6AjZ+0wrKhTTCrVz30bmhd5LG7u1tXaqAD5PczadGiBRYvXozmzZtj/PjxGDduHNasWQMA0NbWxp49exAaGgozMzOYGNWASWIo9Gq3xJOkTGTnlj5z7oYNG9C4cWM0b9kKa/wewPNnP1yPTAIAWDi4wrzraOhY14Fxs97QqmmNpLgovLPcD1uvPMHuv/6Wl21gYIAzZ86gT58+hdb0Kq3sNm3aFLm9qFmMW7ZsCW1tbZw6dUqeFhISgsjISHh4eBR5HGXWaSrK9evXoaGhoTBLcFHyZAKz/7qJ7FwZutazxJAWtWBroo/pPesCAP4MikK6Css2KCv/EVY2ahpoo30d8xL3/bCdEwBg/41oSDNyKqwOK06EIlcm0LmuJdrV/q8O2poaWPZeE2hpSHDkdhyO3FIuAFaFND0HU3YEI1cm8G4TW1yc0w07xrXDhbndyjQaytZEHx51zMsUrJQnb0Vhyw5ROanawqDKXb5UKkWtWrWQlZUFTU1NdB/rjV0xpgAE+je1w7KhTaCvk/9s3VRXgs2/fI8PPvgAxsbKDQsFlLvz0tHSKPJxU1U+k7e1tUWDBorDhevXr4+///5b/nfLli1x/fp1SKVSZGdnw6imGSxqN0KuSR3sDX5S4pd+Wloadu7ciQnTvdH/1wvyTpqt67vgEICOzerh97nd5e+bxy59PElLRFp2Hr49eBdN7U2w7R8/2Bvmry9maWmJtm3byvtxlaSg7IULFyqkF7TOxMbGKkw1EB8fj2bNmsHExARjx47FjBkzYGZmBmNjY3z++efw8PBQ6Jz88lQD5ubmMDdXDAZeXaeprFMNAMDmS49wLTIJNXS1sHhwY3nrRpe6lnA2N8Cj5+nYGxyNUW2dSn1fVHHo3wCid0MbaGuWHGC2cjJFPWsjhMSnYM+1JxjTwaXE/ZVxLzYZ+2/kP0ab3atwy1dDOxNM7FIHq86EY/7+O2hX27zYR72qEkJg9t83EJ2UAUczA/gMaQwjPW3Y1Xz9fWWqC7bsEJWTqi0MqtzlGxkZ4fr16zh29gLq9huHoxt+QGbkTczt446V7zeTBzo5OTkYPnw4hBBYvXq1yudQ1juvgmfyL7cMva5n8h06dEBISIhCWmhoKJycCv9ompiYwNLSElGPHiI9JhQGbm2x+uyDEtdF+mP7TqRlZGJnkhPux6WgpoE2fnivCfZ7D4aenh5u3Lghf98MJTl4EvUYtZ0c8N2gRjDS1cKNJ1IMWHURv1+MgaGJKcLCwhAUFISBAweWem67d+9GVlYWPvzwQ4V0FxcX2NjYYNaiH+VTDXh8ewj+lwPkLTcrVqzAu+++i6FDh6Jz586wsbEpNLouJCQEj2Of4tKDZ4iVZpRan4KpBrp06YKGDRvi+++/x/Tp07Fu3boS80U+T8cPx/I/o7l93GFX87/rQkNDgo88nAHkB0TKzBejrJcfYb1bwiOsAhKJBB965F83Wy8/rpC6/HgsBEIA/ZrYolGtovsBfd7DFa5WNfAsNQuLDt0td5kFtvg/xrE78dDWlGDVyOYw0nuz55aqEBU+neEbiDMoq4cnT56IUaNGCTMzM6GnpycaNWokAgMD5dtRzEyjy5YtK/aYubm54quvvhLOzs5CT09P1K5dWyxcuFDIZDL5Po6OjqJZs2bCxsZG6OnpiR49eohvvvlG2NnZFTpeUlKSSEhIEEII0aZNG/mstyW5EZUo2n5/UjjNOSRMm/cSrTp0VdienZ0tBg0aJJo0aSKePXtW6vEqQ0xSurgU/uy1zIhb4MqVK0JLS0t8//33IiwsTGzbtk0YGBiIrVu3yvf5888/xZkzZ8SDBw/Evn37hJOTkxgwaLBo9u0x4TTnkNgX/ER89NFHYu7cuQrHPnY7Vhg5NRIG7p3kywy8vNyEl5eXACA+/PBDcejQIdGyZUshkUjExo0bhRBCxEkzhOfnS4T1B4uF3YT/E+4fLRTWdg5iyJAhCuUUVbYQQnTs2FGMGDFCIS0vTyZuPUkSvcfMEBq6hsJyyHxh++kqoe/WTmiZWIsuPsfEqPWXxdQd18Sig3fE6rPh4q+gKHE2JEHcjk4S8dIMkZObJ4Qo39IFypLJZOKDdf7Cac4hMWLtJZGXJyu0jzQjW9Sff0Q4zTkkLoY9rbCyz4XmLxPSfOFx+TmXJiUzRzQoqEt4+epSsBZXbe/D4kFCycufXH38Qj6T8+l7xa89p6xbT5KE27x/hNOcQ2LD+YflPl51xxmU6a1S2ogoAIUeGR05cgRjx47F0KFDiz3u0qVLsXr1amzevBkNGzZEUFAQxowZAxMTE/kwXVNTU9y6dQt///03XFxcMH/+fCxfvhz169cvdDwTk/w7vIK7/EWLFpV4XnuuPcHcPbeQnStDHUtDmLlb4mnMf3NkFLTohIWF4cyZM4UeR7wuZR1RVR6tW7fG3r174e3tjYULF8LFxQU///wzRo0aJd8nNjYWM2bMQHx8PGxtbfHxxx9j/vz5WHfhMX48HorfzoQjOzJS3sIWJ83EggO3cejcVaQ8vo1GY5fh97Ft0MnNUqHsX3/9Fc+ePcPOnTuxdetWGBgYYOHChRgzZgwAwNpYD/1c9RG8eykSniYg3tAUNRp2h82gGXiakgVLo/zRS5EvlV0gJCQEFy5cwPHjxxGdlIELYU9xPuwZLj14jhdp2RCW3WDUMgbPj/0KWWYa9OwbwGr4QjxKysGjpMIjb14mkQAmetpIeqlfSmVN9LYzMAqXHjyHnrYGlg5tAg2NwiO9jPW0MbSFPf64/BibLj1Ce9fCw8PL4vDNfx9hNbKBVimPsArU0NXCoOa1sC0gEtsuR6J9EUPVlSGEwLKj+a1Zw1vZo7ZljRL3b+Foik87uGDDhQjM23sLx6d3LnNLTGpWLiZvv4bsPBl61rfGmA7OZTqOOpIIUYFth2+o5ORkmJiYQCqVqtTXgaqPuXPn4uLFizh//rzSeQYNGoSUlBSFzpyvevfdd2Ftba0wImbo0KHQ19fH1q1bIYSApaUlEhMTsWjRIgwfPhxnz57FuHHj4OXlhVWrVgHIfyxRMNPtrVu3MHXqVLRs2VKhf8nHH3+MWrVqwcfHB7l5MvT5dCZuZppDy9QW7RyN0FLjERbM/xKrV6/GZ599hpycHLz33nu4du0aDh06BGvr/zoLm5mZQUenYp7/qyNpRg46LjmNlKxcTO9ZF0Nb1sLp+wlYdjQEqVm50NKQYFzn2pjS3U3+qLCs0rJy8dPxUGy6FAGZyJ+sbV7f+hjeygHxKZmIeJYmn8I/JTMHlx++yA9wwp/h4dM0hWMZ6miiuWNNXAx/rjDUX0MC/DS8GWQyoTjFQGoWnqbk//tFWlaR860U2DGuHTxK6cirrFhpBjyXn0NKVi6+6lcfn3WqXey+4Qkp6Ln8HDQkgN+sbnAwK1+/kpw8GVp/fxJJ6TnY/llblQKoe7HJ6PPLeWhpSHBpbndYlWHSvzMhCRjjGwgdLQ34zeqqVACZkZ2HXj+fQ+SLdIxq64jvBzdWuVwhBKbtuo7912NgZ6KHf6Z2Qk0D9f8OUPb3my07pBZKm3PlVfHx8Th8+DA2b95c4nHbt2+PdevWITQ0FHXr1sWNGzdw4cIF+URqEREReP78OX755ResX79e3sLg5uYmX3cKKL6F4WUFd/lJ6dn4fEcwroTFIv3+LiDtBc4YGiDW3R1bt27FiBEjAORPZHjgwAEAQLNmzRSOdebMmUJD1+k/JvraaO1shtMhCVhxMhQrTobKtzV3rAmfIY3hblMxNz6Gulr4un8DDGpuB+89t3AnJhlz99zCWr+HePwif8I3CQBHMwM8ScpA3ksRiYYEaOpQE51cLdCpriWaOdSEtqYGdgVGFpq7ZHDzWiXWI08mkJiejXuxyfh445VC0w04mVdMq44QAl/tvY2UrFw0c6hZamdfVysjdHKzwPmwZ9h6+TG8+xZuEVXFpQfPkZSeA4saOmjjYlZ6hpfUtzVGKydTBD1OxM7AKEzpUfykn0WRyf5r1fmkvbPSLWX6OppYMrQxRq4PwLaASLzbxE7lwHN30BPsvx4DTQ0JVn7Q/K0IdFTBlh2wZUcdFAzrnjFjBoYNG4bAwEBMnToVa9aswejRowvtv2zZMixZsgQxMTEKQ8JfJZPJMG/ePCxbtgyamprIy8vD999/D29vbwDApUuX0KFDB8TExMDW9r/p6IcPHw6JRIJdu3YpfQ6x0gycD3uGn0+GIiYpE/ramvhxWFP0a1J4mnsqn6JmhQWALzzr4n9dXaFZxCOXipCbJ4PvxUf46XgIMosZ+u5sboCObhbo6GoJjzrmMNEv+pFGrDSjzHOX5AdLt5D30vmPauuIhQMblfvc91+PxtSd16GjqYFDUzqirnXpi4WevBuPz7YEwURfG5e9e5SrNW32XzfwZ9ATfNjOEd8NUr2FZF9wNKbtug5bEz2cn91N6cdgAHDgRgym7AiGka4Wzs3upvLoqnl7b2F7QCSczA1wdGpnpd+HsPgU9F91AZk5MszqVQ9e3VxLz6Qm2LJDbxWZTIZWrVph8eL86fubN2+O27dvFxvsbNy4EaNGjSox0AGAP//8E9u2bcP27dvRsGFDXL9+HdOmTYOdnV2Rxy2rLZceYcGBO/JHE6YG2tj2WTulVxYm1RS1CjUAtHQyq7RABwC0NDUwrnNtWBnpYuqu64W2r3y/GQY0K7mFpkB5+km9PN1A0OMXWH4iFNsCIvEsNQu/vN+8zAtiPkvNwjcH7gAAPu/uqlSgA+RPYuloZoDIF+nYdz0aH5QyIWZxsnNlOPbvWkzKjMIqSp/GNlh4SAex0kycvp8Az4bKLXyakyfDT8fzW3XGda5dpmHk3n3cceZ+Ah4/T8dPx0OUWok9IzsPXtuvITNHhk5uFvhflzoql/s24NBzUgvFzbkSGRlZaN/z588jJCREqSn7Z82ahblz5+L9999H48aN8dFHH2H69Onw8fEB8N+8J/HxiovdlTRjbYHcPBn8Qp9iwpYgfP1SoAPk9ykxNeRw0criYmGIV2Oa17lmT5vaZkWW31rFxy7lUTBs/vPubvhtZAvoaGrg2J14fLzhCqTpZZtYb8GBO0hMz0F9W2NM7Kr8j66mhgQf/zv0uzzD0C+GP4M0IweWRrpo7Vy291JXSxPD/10j64/Lj5XO92dQFB4/T4e5oQ7GdizbPD1GetpY/G9/nY0XI3AtMrHUPAsP3UFofCosauhi+fBmRXYEJwY7pCZUmXNlw4YNaNmyJZo2bVrqcdPT0wuNltHU1IRMlv8IomDek5c7OScnJyMgIKDIGWuFELj1RIqFB++inc9pjN54BcfuxhfaTyaAR8/SS60flU1Vzg9UHcp/Vd/Gttj8aRsY6WrhyqMXGL7WX6n5d1527E4cDt+MhaaGBD+816TUifxeNayVA/S1NXE/LgWXH75QKW+BQ/+OwurbyKZcLXSj2jpCIgHOhz3Do2dppe6fkZ2Hlf8uhzG5uysMdcv+0KSbe/6CuTIBzP7rJrJy84rd98CNGOy4EgWJBPjl/WbyUX5UGB9jkVqYPn062rdvj8WLF2P48OG4cuUK1q1bV2jSs+TkZOzevRs//fRTkcfp0aMHBg8ejMmTJwMA+vfvj++//x6Ojo5o2LAhgoODsXz5cnz66acA8icjmzZtGr777ju4ubnJh57b2dlh0KBB8uNGvUjHgRsx2BscjfCEVHm6qYE2urtbYU9wtEKH0apaGfhtUtVr9lR1+a/yqGOOPyd6YPTGKwiJT8HQ3y9hy9g2cLUq/VGUND0HX+27DQCY0Ll2sZPolcREXxtDWuQP/d586ZHKHXSzcvNw/G7pa2Epw8HMAF3qWuJsyFNsvxKJeaV0mt7s/wjxyVmoVVMfI9uW7RHcy75+twHOhz1FeEIqVp0Ox0zPwjMwP3qWhnl7bgEAJndzRYcKGravrhjskFpQZs4VANi5cyeEEPjggw+KPE5oWDiCQyMRK82ArYk+fv31V8yfPx+TJk1CQkIC7OzsMGHCBHz99dfyPLNnz0ZaWhrGjx+PpKQkNG7ZFlt270OWTBN7r0Ri77VoXHn0352qrpYG3mlgjcHNa6FzXUtoa2qgjYtZtVgZ+G1TFfMDVafyX1Xf1hh7JrXHxxuv4OHTNAxd7Y+Nn7RCS6eSHwl9d/gunqZkoY6locojmF42ur0ztgVE4vjdOEQnZaBWTeXfmwthz5CSmQsrI120cip5CQtlfNTOCWdDnuLPoCjMeKdusf2YpBk5WH32AQBg+jt1oatVvqkKAMDUUAcLBzbCpG3XsPrsA/RpZKvQfy8rNw+Td1xDalYu2jibYWo53vO3BUdjgaOxKN+uwEh477kFmcgf8uszpLFKC+a9nB/Ib53J+/e/l0QCeNQ2x6DmtdC7kQ2Mi5g0rDyja4gq0ou0bHy6KRDXo5Kgq6WBVSNb4J0GRS/66hf6FKM3XoFEAvw10aPUwKg0I9dfxqUHzzGxSx3M7eOudL4Zu65jT3A0PmnvjG8GlL4QbmnyZAKdl51BdFIGfhrWFENb2he53w/H7uO3Mw/gZlUDR6d1rtAO7hP/uIqjd+LQ0M4Y+7w6yB8NfnvwDnwvPoKpgTb+mdrprf6+4GgsIiVl5uTh6O04zP37lryTsEwAc/6+BZ9/7ivV4U8mEwqz0gJAnhCoY2mI4a0cMKCZXalfSNXtLp/eXmaGOtg+ri28tl3DmZCnmPBHEBYPboz3XxkllZqVK3+UMtrDudyBDpA/P82lB8+xMzAS03q6KTUyLDMnDyf+7fvWv2nFTNWgqSHByLaO+OFYCLYGPC4y2ElIycTGC48AAF/0qlfhI/kWDmoI/4fPcScmGevOPYRXN1ccvxMH34v5Zf40vCm/M5TEYIfeOjKZwN3YZJwPe4YL4U8R+CgR2cXMefJqAKOq7wY1rrBZaYleJwMdLaz7uBW899zCX1fzly15mpKFyd1d5SuXLzt6H9FJGXAw08fs3oX7lZRFj/rWsDfVx5PEDBy4HoPhrR1KzXMu9ClSsnJha6KH5g7lf4RVYHgrB/x8MhTBkUm4HS0t1Bfpt9PhyMjJQzOHmvAspuWrPKyM9PD1uw0wc/cN/HwyFJoaEvx2JhwA8FlHF3R3r/gy1RWDHVIrsdIMhen3C8QkZeBC2DOcC3sqX2PoZVZGukhIyVJI05AAW8e2VWqEw9OULHy4IUBh7hZ2MqY3nbamBn54rwmsjXXx25kH+OlEKOJTMjGxSx0cvxOPLf75Q7OXDGkCA52K+TnR1JDgo3ZO8DlyH76XHmFYK3t5cFWcw7f+HYXV2LZCh15bGumidyNbHLwRg20Bj+EzpIl8W9SLdGy/kj+1xeze9UqtY1kNaVELa889QGh8KpYcuQ8AsDfVx+zeyj/iIwY7pEZe7XPzYTsnSIBi1xjyqGOOjq4W6OhmiTqWhvgzKKpQJ2Fl19VxszaCz5DG7GRMakcikWBWL3dY1tDFt4fuYuvlSGy9/N/8Va2dTSt8JNCI1g5YcTIU92KTEfgoscRlHzJz8nDy30dYlTHb+IdtHXHwRgz2BcfAu299eX+7FSdCkZMn0MnNosyLhiojLjlTYQQnkH/z9jwti98vKmCwQ2ohVpqh0DlYJiC/6wSKX2PoZeUdClzdhhITVaRPOrhAU1OC+fvuKKRffZwoH71YUWoa6GBw81rYcSUKmy89KjHYORvyFGnZeahVUx/NHWpWWB0KtHExQ13rGgiNT8Xea9EY3d4Z9+OSsfd6NABgVq+KeXxXnKJm+y6Yh4vfMcpjsENqobjp/3vWt8J7LR1KXGPoZeXtJMxOxqTO6ljWKJRWWT+8o9s7Y8eVKBy9E4eYpAzYFTMM/b9HWDaV8ihJIpHgw3ZO+Hr/HWy9/Bgfezjhx2OhECK/zCb2NSu8zJcVzPbNR+TlwxmUSS24WBji1a85TYkEiwY1Qu9GNkoFOkRUste5zIa7jTHa1TZDnkxgW0DRyzZkZOfh1L3yrYWljMHNa8FARxNhCalY7fcAJ+/FQ0MCzHinclt1gOo32/abisEOqYVnKdl4OdrhFwJRxXvdP7yftHcGAOy4EoXMnMLLJpwNSUB6dh7sTfXRxF71WZuVZaSnjUHN8xdoXXY0f1maYS0d4GpVuKWrMoxo7YgLc7thx7h2uDC3m0rzf1E+PsaiClfciKjKkp0rw6y/bkAIoIe7FT7rVJt9Zogqyevsm9azvjXsTPQQI83EwRsxGNZKcRh6wVpY/ZrYVtpoqAKWNRRHZda2NKzU8l7FR+Tlw5YdNRQdHY0PP/wQ5ubm0NfXR+PGjREUFAQAyMnJwZw5c9C4cWMYGhrCzs4OH3/8MWJiYko85rlz59C/f3/Y2dlBIpFg3759CtsLjuvo6o5alqbo3Kwe6nboh9X/BFbWacqt8XuA+3EpMDXQxrL3msCjjjm/FIgqUcGK6ZX9/0xLUwMfeTgDADa9shp6enYuTt3/9xFW48p7hAXk38D9ejpMIW3Z0RCVF0ulqsNgR80kJiaiQ4cO0NbWxpEjR3D37l389NNPMDXNn2grPT0d165dw/z583Ht2jXs2bMHISEhGDBgQInHTUtLQ9OmTfHbb78VuT09PR3+V4KQ2WgQbEf/AstB85D9IhrTxo6s1C+E0PgU+ZfQNwMawrwGV/0lUifvt3aArpYG7sQk4+rjRHn66fsJyMyRwdHMAI1qVe4yP0UNgMgTAo+epVdquVRx+BhLzSxduhQODg7w9fWVp7m4uMj/bWJighMnTijkWbVqFdq0aYPIyEg4Ohb9LLhPnz7o06dPseWamJjggwVrsfTf59kAYPbORMRtmYHLN0MxuFPTsp5SsfJkArP+uomcPIGe9a0woGnl3t0R0etnaqiDgc3s8GfQE2y69AitnPOHoR9+jY+wOCLqzceWHTVz4MABtGrVCsOGDYOVlRWaN2+O9evXl5hHKpVCIpGgZs2aZSozKzcPCw/eVQh0AECWlQ5AAoMalXPXtfFCBG5EJcFIVwvfDWpc6V94RFQ1Rv/bUfno7TjESTORlpWL0/cTAADvVsJEgq/iiKg3H1t21MzDhw+xevVqzJgxA/PmzUNgYCCmTJkCHR0djB49utD+mZmZmDNnDj744IMyrfgenpCCz3dcx73YZABA+zrmuPzwOfJyspF01hcGDTpj4bEItHC1U2rZBWU9epaGH4/nB1df9qsPGxO9Cjs2EVUvDe1M0MbZDFcevcC2gMdwszZCVq4MLhaGaGBbuY+wCnDS0Dcbgx01I5PJ0KpVKyxevBgA0Lx5c9y+fRtr1qwpFOzk5ORg+PDhEEJg9erVKpUjhMD2gEgsPHQHmTkymBvq4IdhTdDd3RqRz5IxasRwGFoawnzELEQlZeCzLUHYNb6dUisYl36OAnP+vomsXBk6uJpjhBILBRLRm210e2dcefQC2wMi5cPM+zWu/EdYL+OIqDcXH2OpGVtbWzRo0EAhrX79+oiMjFRIKwh0Hj9+jBMnTqjcqrPG7yHm7b2FzBwZOrlZ4MjUTujubo2cnBxMHTcayc9icf7saWyZ2BU1DbRxIyoJM/68DllR0xyraPuVSAREvIC+tiaWDGnCx1dEbwHPhtawNdHD87RsnAl5CqBy1sIi9cRgR8106NABISGKfWdCQ0Ph5OQk/7sg0AkLC8PJkydhbm6u9PH9HzwHAARHJkJbU4Iv+9bH5jFtYGWsh5ycHPTv3x9nzpxBVFQU7O3tMaBbO0xrogFtTQn+uRWHZcfu4+uvv4atrS309fXRs2dPhIWFlVimj48PWrduDSMjI1hYWmHS6A+Q8/wJZveuBwez/A6CEyZMQJ06daCvrw9LS0sMHDgQ9+/fV/q8iKh609bUKDRx4I2opKqpDL1xGOyomenTp+Py5ctYvHgxwsPDsX37dqxbtw5eXl4A8gOd9957D0FBQdi2bRvy8vIQFxeHuLg4ZGdny4/To0cPrFq1Sv53ojQZ03/fh6HfbwcAmOQmwqeLEXo5a0FDQ4KcnBwMGDAAJ0+eRJcuXbBt2zb4+fnhyy+/RNt6tbBkSBMAwNKly/DTil+wZs0aBAQEwNDQEL169UJmZmax5+Tn5wcvLy/4+/vDY/Jy5ObmIHHPArzXxFK+T8uWLeHr64t79+7h2LFjEELA09MTeXmFZ10lojdPrDQDJ/5d3bzAl3tvc64bUo4gIZVKBQAhlUqruioV4uDBg6JRo0ZCV1dXuLu7i3Xr1sm3RURECABFvs6cOSPfz8nJSSxYsEAIIcTjZ2miw5SVReYZPXq00sf98eg9oWloKsy7fyouhj0VQgiRlJQkdHV1xY4dO0o9r7+CooTTnEPCZdoOAUD4+fkVu++NGzcEABEeHq7am0dE1dLF8KfCac6hQq9L4c+qumpUhZT9/WYHZTX07rvv4t133y1ym7Ozs8IspMXxv3EPEc/SsOliBH48HopU/dpovOAofIY0KfI5ubOzM+rXr49evXrhyZMn8PPzQ61atTBp0iR07doVADDIVRtfpCVCx7EZJm69ij2TOsDVygRt27aFv78/3n///WLrk5CSiYWH7gIAPm5liW8BmJmZFblvWloafH194eLiAgcHdl4mUgec64bKg4+xqJBdgZHosOQ0Rq4PwDcH7yI1KxetnU1xZFrnEjsEFgx7d3Nzw7Fjx/C///0PU6ZMwebNmwEA8fH5TdDN6jkhOTMXYzZdwfPULFhbWyMuLq7EOn297w6kGTloaFsDgduXo0OHDmjUqJHCPr///jtq1KiBGjVq4MiRIzhx4gR0dHTK+W4QUXXAuW6oPCRCmdv8l5w5cwbdunWrrPpUieTkZJiYmEAqlZZprhl1IU3PweFbMZi397ZCugTAudld4WBW8sJ3Ojo6aNWqFS5duiRPmzJlCgIDA+Hv749Lly6hQ4cOuB0Wgf/tiUDki3S0dDIFTq2AlqYGdu3aVeRx/7kVi0nbrkFLQ4JWT/7G5XOncOHCBdjb2yvWXypFQkICYmNj8eOPPyI6OhoXL16Enh7n4CFSF7HSDM51Q3LK/n6r/Bird+/esLe3x5gxYzB69Gg+JniDZefKcC0yERfCnuF8+DPcepJUaP0XIL/jzZPEzFKDneKGvf/9998AABsbGwBATmoSNn7SGoN/v4irjxORc+chhr7TochjJqZl4+v9+cGX9e1tuBh8DufOnSsU6AD5S1aYmJjAzc0N7dq1g6mpKfbu3YsPPvigtLeCiN4QnOuGykLlx1jR0dGYPHky/vrrL9SuXRu9evXCn3/+qTCSh6perDQDlx48UxipIIRAaHwKNlyIwBjfK2i28DjeX3cZq86E40ZUfqDjbG6AV2etUfa5eGnD3l1cXGBjY4NTp07B1aoG1n7YEho5GYgJu4XEGs5FHnPhobt4mpKFvAv/h0dXz+L06dMKa30VRwgBIQSysrJK3ZeIiNSbyi07FhYWmD59OqZPn45r167B19cXkyZNwqRJkzBy5EiMHTsWTZtW/KKPpLxdgZHw3nMLMgFoSIDhrRyQkydwIfwp4pMVf/wtauigg6sFOrpaoKObBWxN9LErMBLz9txGnhAqPRefPn062rdvj8WLF2P48OG4cuUK1q1bh3Xr1gEAJBIJpk2bhu+++w5ubm5wcXGBedB6RNUww7lsF/x99QmGtrRHjx49MHjwYDTsORx7g6OReGI1xIMLOHTgAIyMjOT9e0xMTKCvr4+HDx9i165d8PT0hKWlJZ48eYIlS5ZAX18fffv2rfg3mIiI3igq99l5VUxMDNatW4clS5ZAS0sLmZmZ8PDwwJo1a9CwYcOKqmelUqc+O7HSDHRYcrrIx1EAoKulgTYuZujoaoFObpZwtzGChkbhGYjL+lz80KFD8Pb2RlhYGFxcXDBjxgyMGzdOvl0IgQULFmDdunVISkpCx44d0ez9mfgrXAZtTQm2jm2LEd1b4oMPP8JZw66IlWbi8dKiR5b5+vrik08+QUxMDD777DNcvXoViYmJsLa2RufOnfH111+jXr16StediIjeLMr+fpcp2MnJycH+/fuxceNGnDhxAq1atcLYsWPxwQcf4OnTp/jqq69w7do13L17t1wnAeQ/NpszZw6OHDmC9PR0uLq6wtfXF61atQLw34/n+vXrkZSUhA4dOshHBClLnYKdSw+eYeT6gELp/ZvaYkQrR7RyNq2Q9akqkkwmMHnHNfxzKw41DbSx7qOWWOP3EKfvJ8DJ3ABHp3aGvk71qjMREVW9Suug/Pnnn2PHjh0QQuCjjz7CsmXLFIYAGxoa4scff4SdnV3Zav6SxMREdOjQAd26dcORI0dgaWmJsLAwmJqayvdZtmwZVq5cic2bN8PFxQXz589Hr169cPfu3bdyFE5xc1HM61u/2nbq09CQYPnwZohOuowbUUkYvvayfFuvBjYMdIiIqFxUbtnp0aMHPvvsMwwZMgS6urpF7pObm4uLFy+iS5cu5arc3LlzcfHiRZw/f77I7UII2NnZYebMmfjiiy8A5A8/tra2xqZNm0qcpO5l6tSyAwDTdgZj3/UYAP/NRTGitWMV16p0t6OlePfXCwppmhIJLsztVm0DNSIiqjrK/n6rPBrr1KlT+OCDD4oNdABAS0ur3IEOABw4cACtWrXCsGHDYGVlhebNm2P9+vXy7REREYiLi0PPnj3laSYm/83IW5ysrCwkJycrvNRJSmYuAGBEawdcmNvtjQh0ACA5M6dQWp4QePQsvQpqQ0RE6kLlYMfHxwcbN24slL5x40YsXbq0QipVoLQZeQtG5VhbWyvkK21GXh8fH/mcLCYmJmo1V1BWbh4u/bsy+UftnN6oFpGCR3Av43TwRERUXioHO2vXroW7u3uh9IYNG2LNmjUVUqkCMpkMLVq0wOLFi9G8eXOMHz8e48aNK3c53t7ekEql8ldUVFQF1bjqBUYkIiMnD5ZGumho92Y9kuN08EREVBlU7qAcFxcHW9vC6yNZWloiNja2QipVQNkZeePj4xXqFB8fj2bNmhV7XF1d3RIfw73JzoYkAAC61LWERFJ4SHl1N6K1IzrXteR08EREVGFUbtlxcHDAxYsXC6VfvHixQkZgvUyVGXkLJCcnIyAgAB4eHhValzfF2dCnAICu9SyruCZlZ2uiD4865gx0iIioQqjcsjNu3DhMmzYNOTk56N69O4D8TsuzZ8/GzJkzK7RyZZmRd/78+bCzs8OgQYMqtC5vgieJ6QhPSIWGBOjk+uYGO0RERBVJ5WBn1qxZeP78OSZNmiRfD0tPTw9z5syBt7d3hVaudevW2Lt3L7y9vbFw4UK4uLjg559/xqhRo+T7zJ49G2lpaRg/frx8Rt6jR4++lXPsnA3Jb9Vp4WgKEwPtKq4NERFR9VDm5SJSU1Nx79496Ovrw83N7Y3uA6Mu8+x8tjkIJ+/F4wvPupjcXfkZpImIiN5ElTaDcoEaNWqgdevWZc1OFSx/yPkzAEDXelZVXBsiIqLqo0zBTlBQEP78809ERkbKH2UV2LNnT4VUjFQT9CgR6dl5sKihiwa2b27rFBERUUVTeTTWzp070b59e9y7dw979+5FTk4O7ty5g9OnT8PExKQy6khKeHnIeVGrmBMREb2tVA52Fi9ejBUrVuDgwYPQ0dHBL7/8gvv372P48OFwdHwzliVQRwWdk7u8wUPOiYiIKoPKwc6DBw/Qr18/AICOjg7S0tIgkUgwffp0+ZBwer2ikzIQ9u+Q885uFlVdHSIiompF5WDH1NQUKSkpAIBatWrh9u3bAICkpCSkp3PBxqrg92+rTjOHmqhpoFPFtSEiIqpeVO6g3LlzZ5w4cQKNGzfGsGHDMHXqVJw+fRonTpxAjx49KqOOVIqC/jochUVERFSYysHOqlWrkJmZCQD48ssvoa2tjUuXLmHo0KH46quvKryCVLLsXBkuhhcMOWd/HSIiolepFOzk5ubi0KFD6NWrFwBAQ0MDc+fOrZSKkXKCHr9AWnYeLGrooJEdR8MRERG9SqU+O1paWpg4caK8ZYeqXkF/nc5uHHJORERUFJU7KLdp0wbXr1+vhKpQWXDIORERUclU7rMzadIkzJgxA1FRUWjZsiUMDQ0Vtjdp0qTCKkcli0nKQEh8yr9DzhnsEBERFUXlYOf9998HAEyZMkWeJpFIIISARCJBXl5exdWOSuQXmt+q09ShJkwNOeSciIioKCoHOxEREZVRDyqDl5eIICIioqKpHOw4OTlVRj1IRTl5MlwMfw6A8+sQERGVROVgZ8uWLSVu//jjj8tcGVLe1ceJSM3KhZmhDprU4pBzIiKi4qgc7EydOlXh75ycHKSnp0NHRwcGBgYMdl6Ts/Ih5xYcck5ERFQClYeeJyYmKrxSU1MREhKCjh07YseOHZVRRyoCl4ggIiJSjsrBTlHc3NywZMmSQq0+VDnipJm4H5cCiQTozM7JREREJaqQYAfIn105Jiamog5HJfALzW/VaWJfE2Ycck5ERFQilfvsHDhwQOFvIQRiY2OxatUqdOjQocIqRsUr6K/Tla06REREpVI52Bk0aJDC3xKJBJaWlujevTt++umniqoXFSMnT4YLYVzlnIiISFkqBzsymawy6kFKuvY4ESlZuTA10EYT+5pVXR0iIqJqr8L67NDrcfbfJSI6uVlCk0POiYiISqVysDN06FAsXbq0UPqyZcswbNiwCqkUFc+voL8OH2EREREpReVg59y5c+jbt2+h9D59+uDcuXMVUikqWkJyJu7GJgPgkHMiIiJlqRzspKamQken8HBnbW1tJCcnV0ilqGgFj7Ca2JvAooZuFdeGiIjozaBysNO4cWPs2rWrUPrOnTvRoEGDCqkUFc2PQ86JiIhUpvJorPnz52PIkCF48OABunfvDgA4deoUduzYgd27d1d4BSlfbp4M58Pyg50uXCKCiIhIaSoHO/3798e+ffuwePFi/PXXX9DX10eTJk1w8uRJdOnSpTLqSACCo5KQnJmLmgbaaOZQs6qrQ0RE9MZQOdgBgH79+qFfv34VXRcqQcHCnxxyTkREpBqV++wEBgYiICCgUHpAQACCgoIqpFLqZsmSJZBIJJg2bZpCur+/P7p37w5DQ0MYGxujc+fOyMjIKPIYZ0OeQur/J4589wmMjIxgZWWFQYMGISQkRGG/devWoWvXrjA2NoZEIkFSUlIlnRUREdGbQeVgx8vLC1FRUYXSo6Oj4eXlVSGVUieBgYFYu3YtmjRpopDu7++P3r17w9PTE1euXEFgYCAmT54MDY3CH0lCSibuxCQjM+o2pn4+GZcvX8aJEyeQk5MDT09PpKWlyfdNT09H7969MW/evEo/NyIiojeByo+x7t69ixYtWhRKb968Oe7evVshlVIXqampGDVqFNavX4/vvvtOYdv06dMxZcoUzJ07V55Wr169Io9TMAqrx/Sf8fnETvL0TZs2wcrKClevXkXnzp0BQN56dPbs2Qo8EyIiojeXyi07urq6iI+PL5QeGxsLLa0ydQFSW15eXujXrx969uypkJ6QkICAgABYWVmhffv2sLa2RpcuXXDhwoUij1Mwv07XuoqjsKRSKQDAzMysEmpPRESkHlQOdjw9PeHt7S3/oQWApKQkzJs3D++8806FVu5NtnPnTly7dg0+Pj6Ftj18+BAA8M0332DcuHE4evQoWrRogR49eiAsLExh39xiVjmXyWSYNm0aOnTogEaNGlXimRAREb3ZVG6K+fHHH9G5c2c4OTmhefPmAIDr16/D2toaf/zxR4VX8E0UFRWFqVOn4sSJE9DT0yu0vWDl+AkTJmDMmDEA8h8Dnjp1Chs3blQIkG48SYI0IwfGeloKQ869vLxw+/btYluDiIiIKJ/KwU6tWrVw8+ZNbNu2DTdu3IC+vj7GjBmDDz74ANra2pVRxzfO1atXkZCQoNC3KS8vD+fOncOqVavkI6henXG6fv36iIyMVEg7+29/nU51LaGlmd8QN3nyZBw6dAjnzp2Dvb19ZZ4KERHRG69MnWwMDQ0xfvz4iq6L2ujRowdu3bqlkDZmzBi4u7tjzpw5qF27Nuzs7AoNGw8NDUWfPn0U0s6+tESEEAKff/459u7di7Nnz8LFxaVyT4SIiEgNlLlH8d27dxEZGYns7GyF9AEDBpS7Um86IyOjQv1oDA0NYW5uLk+fNWsWFixYgKZNm6JZs2bYvHkz7t+/j7/++kuep3PXbritXQ/GLfujSz1LeHl5Yfv27di/fz+MjIwQFxcHADAxMYG+vj4AIC4uDnFxcQgPDwcA3Lp1C0ZGRnB0dGRHZiIieiupHOw8fPgQgwcPxq1btyCRSCCEAABIJPmz+ubl5VVsDdXUtGnTkJmZienTp+PFixdo2rQpTpw4gTp16sj3uR8aDlltWzS0M4aVkR5Wr14NAOjatavCsXx9ffHJJ58AANasWYNvv/1Wvq1gSPrL+xAREb1NJKIgWlFS//79oampif/7v/+Di4sLrly5gufPn2PmzJn48ccf0alTp9IPUs0kJyfDxMQEUqkUxsbGVV0duc82B+HkvXiM9nDCtwM54oqIiOhlyv5+qzz03N/fHwsXLoSFhQU0NDSgoaGBjh07wsfHB1OmTClXpek/O65E4uS9/PmMtlx+jF2BkaXkICIioqKoHOzk5eXByMgIAGBhYYGYmBgAgJOTU6EOt1Q2sdIMzNv7XwdnIYB5e24jVlr0ullERERUPJX77DRq1Ag3btyAi4sL2rZti2XLlkFHRwfr1q1D7dq1K6OOb52IZ2l49eFinhB49Cwdtib6VVMpIiKiN5TKwc5XX30lX3hy4cKFePfdd9GpUyeYm5tj165dFV7Bt5Hmv529X01ztjCogtoQERG92VQOdnr16iX/t6urK+7fv48XL17A1NRUPiKLymff9RiFvzUlEiwe0oitOkRERGVQISt3cv6WipOQkom/rz0BAKwe1QI1DXTgbGHAQIeIiKiMuEx5NbPp4iNk58rQwrEmejeyYWsZERFROak8GosqT2pWLv64/BgAMKFLHQY6REREFYDBTjWyIyASKZm5qG1piHfqW1d1dYiIiNSCysHOuXPnkJubWyg9NzcX586dq5BKvY2yc2XYcCECADChc21oaLBVh4iIqCKoHOx069YNL168KJQulUrRrVu3CqnU22j/9WjEJWfCykgXg5rXqurqEBERqQ2Vgx0hRJF9SZ4/fw5DQ8MKqdTbRiYTWHfuIQDg044u0NXSrOIaERERqQ+lR2MNGTIEQP7q5p988gl0dXXl2/Ly8nDz5k20b9++4mv4Fjh9PwFhCakw0tXCyLaOVV0dIiIitaJ0sGNiYgIgv2XHyMgI+vr/zfuio6ODdu3aYdy4cRVfw7fA2nMPAAAj2znCWE+7imtDRESkXpQOdnx9fQEAzs7O+OKLL/jIqoJcffwCgY8SoaOpgU87uFR1dYiIiNSOyn12Zs+erdBn5/Hjx/j5559x/PjxCq3Y22KNX35fncHNa8HaWK+Ka0NERKR+VA52Bg4ciC1btgAAkpKS0KZNG/z0008YOHAgVq9eXeEVVGfhCak4cTceEgkwrjNXjCciIqoMKgc7165dQ6dOnQAAf/31F2xsbPD48WNs2bIFK1eurPAKqrN1//bVeae+NVytalRxbYiIiNSTysFOeno6jIyMAADHjx/HkCFDoKGhgXbt2uHx48cVXkF1FZ+cib3B0QDyl4YgIiKiyqFysOPq6op9+/YhKioKx44dg6enJwAgISEBxsbGFV5BdbXxQgRy8gTaOJuhpZNpVVeHiIhIbakc7Hz99df44osv4OzsjDZt2sDDwwNAfitP8+bNK7yC6ig5MwfbAiIBABO6sK8OERFRZVI52HnvvfcQGRmJoKAgHDt2TJ7eo0cPrFixokIr96olS5ZAIpFg2rRp8rTMzEx4eXnB3NwcNWrUwNChQxEfH1+p9SivbZcjkZqVCzerGuhWz6qqq0NERKTWyrTquY2NDYyMjHDixAlkZGQAAFq3bg13d/cKrdzLAgMDsXbtWjRp0kQhffr06Th48CB2794NPz8/xMTEyGd7ro6ycvOw8eK/C352qcMFP4mIiCqZysHO8+fP0aNHD9StWxd9+/ZFbGwsAGDs2LGYOXNmhVcQAFJTUzFq1CisX78epqb/9W+RSqXYsGEDli9fju7du6Nly5bw9fXFpUuXcPny5UqpS3ntC47G05Qs2JroYUBTu6quDhERkdpTOdiZPn06tLW1ERkZCQMDA3n6iBEjcPTo0QqtXAEvLy/069cPPXv2VEi/evUqcnJyFNLd3d3h6OgIf3//Yo+XlZWF5ORkhdfrIJMJrP13wc+xHV2go1WmhjUiIiJSgdLLRRQ4fvw4jh07Bnt7e4V0Nze3Shl6vnPnTly7dg2BgYGFtsXFxUFHRwc1a9ZUSLe2tkZcXFyxx/Tx8cG3335b0VUt1Yl78Xj4NA3Gelp4vw0X/CQiInodVG5aSEtLU2jRKfDixQuFldArQlRUFKZOnYpt27ZBT6/illLw9vaGVCqVv6Kioirs2MURQmCNX/4kgh95OKGGrspxJhEREZWBysFOp06d5MtFAIBEIoFMJsOyZcvQrVu3Cq3c1atXkZCQgBYtWkBLSwtaWlrw8/PDypUroaWlBWtra2RnZyMpKUkhX3x8PGxsbIo9rq6uLoyNjRVelS3wUSKCI5Ogo6WBT9pzwU8iIqLXReXmhWXLlqFHjx4ICgpCdnY2Zs+ejTt37uDFixe4ePFihVauR48euHXrlkLamDFj4O7ujjlz5sDBwQHa2to4deoUhg4dCgAICQlBZGSkfP6f6qKgVee9lvawNKrYFjAiIiIqnsrBTqNGjRAaGopVq1bByMgIqampGDJkCLy8vGBra1uhlTMyMkKjRo0U0gwNDWFubi5PHzt2LGbMmAEzMzMYGxvj888/h4eHB9q1a1ehdSmPkLgUnL6fkL/gZydOIkhERPQ6qRzsREZGwsHBAV9++WWR2xwdX2/H2xUrVkBDQwNDhw5FVlYWevXqhd9///211qE0a/9d8LN3Qxu4WBhWcW2IiIjeLhIhhFAlg6amJmJjY2FlpTjz7/Pnz2FlZYW8vLwKreDrkJycDBMTE0il0grvvxOTlIHOy84gVyaw36sDmjrUrNDjExERva2U/f1WuYOyEAISSeFZf1NTUyt0xJS62HghArkygXa1zRjoEBERVQGlH2PNmDEDQP7oq/nz5ysMP8/Ly0NAQACaNWtW4RV8k4XGpWDr5fy5hyZ2qVPFtSEiIno7KR3sBAcHA8hv2bl16xZ0dHTk23R0dNC0aVN88cUXFV/DN9SuwEjM/fsWCp4Rxkkzq7Q+REREbyulg50zZ84AyB/6/csvv7yWuWneVLHSDHjv+S/QAYAv995Gl3qWsDXRr7J6ERERvY1U7rPj6+vLQKcUEc/SIHul23eeEHj0LL1qKkRERPQW40qUlcDFwhAar/Th1pRI4GxReJkNIiIiqlwMdiqBrYk+fIY0hua/o9Y0JRIsHtKIj7CIiIiqAFejrCQjWjuic11LPHqWDmcLAwY6REREVYTBTiWyNdFnkENERFTF+BiLiIiI1BqDHSIiIlJrDHaIiIhIrTHYISIiIrXGYIeIiIjUGoMdIiIiUmsMdoiIiEitMdghIiIitcZgh4iIiNQagx0iIiJSawx2iIiISK0x2CEiIiK1xmCHiIiI1BqDHSIiIlJrDHaIiIhIrTHYISIiIrXGYIeIiIjUGoMdIiIiUmsMdoiIiEitMdghIiIitcZgh4iIiNQagx0iIiJSawx2iIiISK0x2CEiIiK1xmCHiIiI1BqDHSIiIlJrDHaIiIhIrTHYISIiIrXGYIeIiIjUGoMdIiIiUmsMdoiIiEitMdghIiIitcZgh4iIiNQagx0iIiJSawx2iIiISK0x2CEiIiK1xmCHiIiI1BqDHSIiIlJrDHaIiIhIrTHYISIiIrXGYIeIiIjUGoMdIiIiUmsMdoiIiEitMdghIiIitcZgh4iIiNQagx0iIiJSawx2iIiISK0x2CEiIiK1xmCHiIiI1BqDHSIiIlJrDHaIiIhIrTHYISIiIrXGYIeIiIjUGoMdIiIiUmvVOtjx8fFB69atYWRkBCsrKwwaNAghISEK+2RmZsLLywvm5uaoUaMGhg4divj4+CqqMREREVU31TrY8fPzg5eXFy5fvowTJ04gJycHnp6eSEtLk+8zffp0HDx4ELt374afnx9iYmIwZMiQKqw1ERERVScSIYSo6koo6+nTp7CysoKfnx86d+4MqVQKS0tLbN++He+99x4A4P79+6hfvz78/f3Rrl07pY6bnJwMExMTSKVSGBsbV+YpEBERUQVR9ve7WrfsvEoqlQIAzMzMAABXr15FTk4OevbsKd/H3d0djo6O8Pf3L/Y4WVlZSE5OVngRERGRenpjgh2ZTIZp06ahQ4cOaNSoEQAgLi4OOjo6qFmzpsK+1tbWiIuLK/ZYPj4+MDExkb8cHBwqs+pERERUhd6YYMfLywu3b9/Gzp07y30sb29vSKVS+SsqKqoCakhERETVkVZVV0AZkydPxqFDh3Du3DnY29vL021sbJCdnY2kpCSF1p34+HjY2NgUezxdXV3o6upWZpWJiIiomqjWLTtCCEyePBl79+7F6dOn4eLiorC9ZcuW0NbWxqlTp+RpISEhiIyMhIeHx+uuLhEREVVD1bplx8vLC9u3b8f+/fthZGQk74djYmICfX19mJiYYOzYsZgxYwbMzMxgbGyMzz//HB4eHkqPxCIiIiL1Vq2HnkskkiLTfX198cknnwDIn1Rw5syZ2LFjB7KystCrVy/8/vvvJT7GehWHnhMREb15lP39rtbBzuvCYIeIiOjNo5bz7BARERGpisEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTW1CbY+e233+Ds7Aw9PT20bdsWV65cqeoqERERUTWgFsHOrl27MGPGDCxYsADXrl1D06ZN0atXLyQkJFR11YiIiKiKqUWws3z5cowbNw5jxoxBgwYNsGbNGhgYGGDjxo1VXTUiIiKqYlpVXYHyys7OxtWrV+Ht7S1P09DQQM+ePeHv719knqysLGRlZcn/lkqlAIDk5OTKrSwRERFVmILfbSFEifu98cHOs2fPkJeXB2tra4V0a2tr3L9/v8g8Pj4++PbbbwulOzg4VEodiYiIqPKkpKTAxMSk2O1vfLBTFt7e3pgxY4b8b5lMhhcvXsDc3BwSiaTCyklOToaDgwOioqJgbGz8WvOz7Ndfdnnzs+y3q+zy5mfZLPtNyV/esksihEBKSgrs7OxK3O+ND3YsLCygqamJ+Ph4hfT4+HjY2NgUmUdXVxe6uroKaTVr1qysKsLY2LhcH3B58rPs1192efOz7Ler7PLmZ9ks+03JX96yi1NSi06BN76Dso6ODlq2bIlTp07J02QyGU6dOgUPD48qrBkRERFVB298yw4AzJgxA6NHj0arVq3Qpk0b/Pzzz0hLS8OYMWOqumpERERUxdQi2BkxYgSePn2Kr7/+GnFxcWjWrBmOHj1aqNPy66arq4sFCxYUemT2OvKz7Ndfdnnzs+y3q+zy5mfZLPtNyV/esiuCRJQ2XouIiIjoDfbG99khIiIiKgmDHSIiIlJrDHaIiIhIrTHYISIiIrXGYKcS/fbbb3B2doaenh7atm2LK1euKJXv3Llz6N+/P+zs7CCRSLBv3z6ly/Tx8UHr1q1hZGQEKysrDBo0CCEhIUrnX716NZo0aSKf/MnDwwNHjhxROv/LlixZAolEgmnTpim1/zfffAOJRKLwcnd3V7q86OhofPjhhzA3N4e+vj4aN26MoKAgpfI6OzsXKlsikcDLy6vUvHl5eZg/fz5cXFygr6+POnXqYNGiRaWu1fKylJQUTJs2DU5OTtDX10f79u0RGBhYaL/Srg0hBL7++mvY2tpCX18fPXv2RFhYmNL59+zZA09PT/ls4tevX1e6/JycHMyZMweNGzeGoaEh7Ozs8PHHHyMmJkapsr/55hu4u7vD0NAQpqam6NmzJwICApSu+8smTpwIiUSCn3/+Wam8n3zySaHPvnfv3iqVfe/ePQwYMAAmJiYwNDRE69atERkZWWreoq47iUSCH374QamyU1NTMXnyZNjb20NfX1++GLIyeePj4/HJJ5/Azs4OBgYG6N27t/x6Uea7JDMzE15eXjA3N0eNGjUwdOhQ+QSvyuRft24dunbtCmNjY0gkEiQlJcm3lZb/xYsX+Pzzz1GvXj3o6+vD0dERU6ZMgVQqVarsCRMmoE6dOtDX14elpSUGDhwoX2JIle9RIQT69Okjf3+Vydu1a9dCn/fEiRNVKtvf3x/du3eHoaEhjI2N0blzZyxcuLDEvI8ePSr2etu9e7dSZcfFxeGjjz6CjY0NDA0N0aJFC/z9999K5X3w4AEGDx4MS0tLGBsbY/jw4YUmBK4sDHYqya5duzBjxgwsWLAA165dQ9OmTdGrVy8kJCSUmjctLQ1NmzbFb7/9pnK5fn5+8PLywuXLl3HixAnk5OTA09MTaWlpSuW3t7fHkiVLcPXqVQQFBaF79+4YOHAg7ty5o1I9AgMDsXbtWjRp0kSlfA0bNkRsbKz8deHCBaXyJSYmokOHDtDW1saRI0dw9+5d/PTTTzA1NVW6vi+Xe+LECQDAsGHDSs27dOlSrF69GqtWrcK9e/ewdOlSLFu2DL/++qtSZQPAZ599hhMnTuCPP/7ArVu34OnpiZ49eyI6Olphv9KujWXLlmHlypVYs2YNAgICYGhoiF69eiEzM1Op/GlpaejYsSOWLl1a7Pbi8qenp+PatWuYP38+rl27hj179iAkJAQDBgxQquy6deti1apVuHXrFi5cuABnZ2d4enri6dOnSuUvsHfvXly+fFlh+nhl8vbu3VvhGtixY4fS+R88eICOHTvC3d0dZ8+exc2bNzF//nzo6emVmvflMmNjY7Fx40ZIJBIMHTpUqbJnzJiBo0ePYuvWrbh37x6mTZuGyZMn48CBAyXmFUJg0KBBePjwIfbv34/g4GA4OTmhZ8+eSEtLU+q7ZPr06Th48CB2794NPz8/xMTEYMiQIQCU+y5KT09H7969MW/evEL1Ky1/TEwMYmJi8OOPP+L27dvYtGkTjh49irFjxypVdsuWLeHr64t79+7h2LFjEELA09MTeXl5Kn2P/vzzzwrLDCmbd9y4cQqf+7Jly5TO7+/vj969e8PT0xNXrlxBYGAgJk+ejAsXLpSY18HBodD19u2336JGjRro06ePUmV//PHHCAkJwYEDB3Dr1i0MGTIEw4cPx8GDB0vMm5aWBk9PT0gkEpw+fRoXL15EdnY2+vfvD5lMVuh9rXCCKkWbNm2El5eX/O+8vDxhZ2cnfHx8VDoOALF3794y1yMhIUEAEH5+fmU+hqmpqfi///s/pfdPSUkRbm5u4sSJE6JLly5i6tSpSuVbsGCBaNq0aZnqOGfOHNGxY8cy5S3K1KlTRZ06dYRMJit13379+olPP/1UIW3IkCFi1KhRSpWVnp4uNDU1xaFDhxTSW7RoIb788sti8716bchkMmFjYyN++OEHeVpSUpLQ1dUVO3bsKDX/yyIiIgQAERwcrHT5Rbly5YoAIB4/fqxyXqlUKgCIkydPKl32kydPRK1atcTt27eFk5OTWLFihVJ5R48eLQYOHFhifUrKP2LECPHhhx+WKe+rBg4cKLp37650/oYNG4qFCxcqpBV17byaNyQkRAAQt2/flqfl5eUJS0tLsX79+kJlv/pdkpSUJLS1tcXu3bvl+9y7d08AEP7+/qXmf9mZM2cEAJGYmFjkeZeWv8Cff/4pdHR0RE5Ojsp5b9y4IQCI8PBwpcsODg4WtWrVErGxscV+tkXlVeV7saj8bdu2FV999VWZ8r6qWbNmhb6/SspvaGgotmzZorCfmZlZoWvm1bzHjh0TGhoaQiqVyvdJSkoSEolEnDhxotRzKS+27FSC7OxsXL16FT179pSnaWhooGfPnvD393+tdZFKpQAAMzMzlfPm5eVh586dSEtLU2npDS8vL/Tr10/h/JUVFhYGOzs71K5dG6NGjUJkZKRS+Q4cOIBWrVph2LBhsLKyQvPmzbF+/XqVywfyP7+tW7fi008/VWph2Pbt2+PUqVMIDQ0FANy4cQMXLlxAnz59lCovNzcXeXl50NPTU0jX19dXumULACIiIhAXF6fwvpuYmKBt27av/borIJVKIZFIVF57Ljs7G+vWrYOJiQmaNm2qVB6ZTIaPPvoIs2bNQsOGDVWu69mzZ2FlZYV69erhf//7H54/f650uYcPH0bdunXRq1cvWFlZoW3btio9fi4QHx+Pw4cPY+zYsUrnad++PQ4cOIDo6GgIIXDmzBmEhobC09OzxHxZWVkAoHDdaWhoQFdXt8jr7tXvkqtXryInJ0fhenN3d4ejo2OR11t5vouUzS+VSmFsbAwtLa1C6SXlTUtLg6+vL1xcXODg4KBU2enp6Rg5ciR+++23YtdhLKnsbdu2wcLCAo0aNYK3tzfS09OVyp+QkICAgABYWVmhffv2sLa2RpcuXZT6zF519epVXL9+vdjrraj87du3x65du/DixQvIZDLs3LkTmZmZ6Nq1a4l5s7KyIJFIFCYW1NPTg4aGhkrfc2VW6eHUWyg6OloAEJcuXVJInzVrlmjTpo1Kx0I5Wnby8vJEv379RIcOHVTKd/PmTWFoaCg0NTWFiYmJOHz4sNJ5d+zYIRo1aiQyMjKEEKrdwfzzzz/izz//FDdu3BBHjx4VHh4ewtHRUSQnJ5eaV1dXV+jq6gpvb29x7do1sXbtWqGnpyc2bdqkdN0L7Nq1S2hqaoro6Gil9s/LyxNz5swREolEaGlpCYlEIhYvXqxSmR4eHqJLly4iOjpa5Obmij/++ENoaGiIunXrFpvn1Wvj4sWLAoCIiYlR2G/YsGFi+PDhpeZ/WUW07GRkZIgWLVqIkSNHKp334MGDwtDQUEgkEmFnZyeuXLmidNmLFy8W77zzjrw1TpWWnR07doj9+/eLmzdvir1794r69euL1q1bi9zc3FLzF9zVGxgYiOXLl4vg4GDh4+MjJBKJOHv2rFLnXWDp0qXC1NRU/v9HmbpnZmaKjz/+WAAQWlpaQkdHR2zevLnUvNnZ2cLR0VEMGzZMvHjxQmRlZYklS5YIAMLT01Mhb1HfJdu2bRM6OjqFymndurWYPXt2qflfVlrLjjLfZU+fPhWOjo5i3rx5Suf97bffhKGhoQAg6tWrV2SrTnH5x48fL8aOHSv/u6jPpri8a9euFUePHhU3b94UW7duFbVq1RKDBw9Wqmx/f38BQJiZmYmNGzeKa9euiWnTpgkdHR0RGhqq1HkX+N///ifq169f5Lbi8icmJgpPT0/59WZsbCyOHTtWat6EhARhbGwspk6dKtLS0kRqaqqYPHmyACDGjx9fbB0rCoOdSlBdgp2JEycKJycnERUVpVK+rKwsERYWJoKCgsTcuXOFhYWFuHPnTqn5IiMjhZWVlbhx44Y8TZVg51WJiYnC2NhYqUdo2trawsPDQyHt888/F+3atVO5XE9PT/Huu+8qvf+OHTuEvb292LFjh7h586bYsmWLMDMzUynQCg8PF507dxYAhKampmjdurUYNWqUcHd3LzZPdQ52srOzRf/+/UXz5s0Vmq1Ly5uamirCwsKEv7+/+PTTT4Wzs7OIj48vNX9QUJCwtrZWCFBVCXZe9eDBA6UfoRX8f//ggw8U9uvfv794//33VSq7Xr16YvLkycVuLyr/Dz/8IOrWrSsOHDggbty4IX799VdRo0aNQo8GisobFBQkmjZtKr/uevXqJfr06SN69+6tsF9R3yWqBDulfReVFuyUll8qlYo2bdqI3r17i+zsbKXzJiUlidDQUOHn5yf69+8vWrRoUSjQLCr//v37haurq0hJSZGnFfX+KvsdfOrUqSIfoRWVv+D/ube3t8K+jRs3FnPnzlW67PT0dGFiYiJ+/PHHIrcXl3/y5MmiTZs24uTJk+L69evim2++ESYmJuLmzZul5j127JioXbu2kEgkQlNTU3z44YeiRYsWYuLEiSW8OxWDwU4lyMrKEpqamoUu/I8//lgMGDBApWOVNdjx8vIS9vb24uHDhyrnfVWPHj2Uirz37t0r/9IseAGQX9hF3SWXplWrVgr/gYvj6OiocJclhBC///67sLOzU6m8R48eCQ0NDbFv3z6l89jb24tVq1YppC1atEjUq1dPpbKFyP+xLwhWhg8fLvr27Vvsvq9eGwU/0K8GKJ07dxZTpkwpNf/LyhPsZGdni0GDBokmTZqIZ8+eqZT3Va6urkW2kr2af8WKFfLr7OVrT0NDQzg5OZWpbAsLC7FmzZpSy87KyhJaWlpi0aJFCvvNnj1btG/fXumyz507JwCI69evF1unV/Onp6cLbW3tQv29xo4dK3r16qV02UlJSSIhIUEIkd/fcNKkSfJtxX2XFPxAvxqgODo6iuXLl5ea/2UlBTul5U9OThYeHh6iR48ehQIVVb4Hs7KyhIGBgdi+fXup+adOnVrs9dalSxeVy05NTRUAxNGjR0st++HDhwKA+OOPPxTShw8fLm9FVabsLVu2CG1tbfnn/rLi8oeHhxfq5yVE/m/EhAkTlC776dOn8s/a2tpaLFu2rNh9Kwr77FQCHR0dtGzZEqdOnZKnyWQynDp1SqW+L2UhhMDkyZOxd+9enD59Gi4uLuU+pkwmkz/fL0mPHj1w69YtXL9+Xf5q1aoVRo0ahevXr0NTU1OlclNTU/HgwQPY2tqWum+HDh0KDXMMDQ2Fk5OTSmX6+vrCysoK/fr1UzpPeno6NDQU/ytpamqWaYSBoaEhbG1tkZiYiGPHjmHgwIFK53VxcYGNjY3CdZecnIyAgIBKv+4K5OTkYPjw4QgLC8PJkydhbm5eruMpe+199NFHuHnzpsK1Z2dnh1mzZuHYsWMql/vkyRM8f/5cqWtPR0cHrVu3Lvf1t2HDBrRs2VLpPkpA/vudk5NT7uvPxMQElpaWCAsLQ1BQEAYOHFjqd0nLli2hra2tcL2FhIQgMjISHh4e5f4uUiZ/cnIyPD09oaOjgwMHDsj7H5WlbJF/84+srKxS88+dO7fQ9QYAK1aswMaNG1UuuyC/ra1tqWU7OzvDzs6uyOvN0dFR6bI3bNiAAQMGwNLSUuE9KCl/Qb+ioq63vLw8pcu2sLBAzZo1cfr0aSQkJMhHbFaqSg+n3lI7d+4Uurq6YtOmTeLu3bti/PjxombNmiIuLq7UvCkpKSI4OFgEBwcLAPJ+AK+OaCnK//73P2FiYiLOnj0rYmNj5a/09HSl6j137lzh5+cnIiIixM2bN8XcuXOFRCIRx48fVyr/q1R5jDVz5kxx9uxZERERIS5evCh69uwpLCwsirzzeNWVK1eElpaW+P7770VYWJjYtm2bMDAwEFu3blW6rnl5ecLR0VHMmTNH6TxC5I/kqVWrljh06JCIiIgQe/bsERYWFoWa8kty9OhRceTIEfHw4UNx/Phx0bRpU9G2bdtCTfKlXRtLliwRNWvWlPc/GThwoHBxcZHf8ZaW//nz5yI4OFgcPnxYABA7d+4UwcHBIjY2ttT82dnZYsCAAcLe3l5cv35d4frLysoqMW9qaqrw9vYW/v7+4tGjRyIoKEiMGTNG6Orqyu8iVf1/8fJjrJLypqSkiC+++EL4+/uLiIgIcfLkSdGiRQvh5uYmMjMzlSp7z549QltbW6xbt06EhYWJX3/9VWhqaorz588rVW+pVCoMDAzE6tWrC51Hafm7dOkiGjZsKM6cOSMePnwofH19hZ6envj9999Lzfvnn3+KM2fOiAcPHoh9+/YJJycnMWTIECGEct8lEydOFI6OjuL06dMiKChIeHh4yB8nK5M/NjZWBAcHi/Xr1wsA4ty5cyI4OFg8f/681PxSqVS0bdtWNG7cWISHhyvsM3HixBLzPnjwQCxevFgEBQWJx48fi4sXL4r+/fsLMzMzER8fX6bvUfzbclZa3vDwcLFw4UIRFBQkIiIixP79+0Xt2rVF586dlX7fVqxYIYyNjcXu3btFWFiY+Oqrr4Senp4YOXKkUvUOCwsTEolEHDlyRCG9tLKzs7OFq6ur6NSpkwgICBDh4eHixx9/FBKJRPTt27fUsjdu3Cj8/f1FeHi4+OOPP4SZmZmYMWNGse9pRWKwU4l+/fVX4ejoKHR0dESbNm3E5cuXlcpX0KT76mv06NGl5i0qHwDh6+urVNmffvqpcHJyEjo6OsLS0lL06NGjzIGOEKoFOyNGjBC2trZCR0dH1KpVS4wYMaLIDoPFOXjwoGjUqJHQ1dUV7u7uYt26dSrV9dixYwKACAkJUSlfcnKymDp1qnB0dBR6enqidu3a4ssvvxRZWVlKH2PXrl2idu3aQkdHR9jY2AgvLy+RlJRUaL/Srg2ZTCbmz58vrK2tha6urujRo4fC+ZSW39fXt8jtCxYsKDV/waOvol5nzpwpMW9GRoYYPHiwsLOzEzo6OsLW1lYMGDBAoYOyqv8vXg52Ssqbnp4uPD09haWlpdDW1hZOTk5i3LhxCjcmypS9YcMG4erqKvT09ETTpk3lj0KVybt27Vqhr69fps88NjZWfPLJJ8LOzk7o6emJevXqiZ9++knIZLJS8/7yyy/C3t5eaGtrC0dHR/HVV1/Jr1tlvksyMjLEpEmThKmpqTAwMBCDBw+WB8bK5F+wYEGx+5SWv7hzK+lVkDc6Olr06dNHWFlZCW1tbWFvby9Gjhwp7t+/r3TdX1UQ7JSWNzIyUnTu3FmYmZkJXV1d4erqKmbNmiXv26Zs2T4+PsLe3l4YGBgIDw8Pcf78eaXzent7CwcHB5GXl1foHErLHxoaKoYMGSKsrKyEgYGBaNKkidiyZYtSeefMmSOsra2Ftra2cHNzk1+nr4Pk3xMkIiIiUkvss0NERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BARveLs2bOQSCRISkqq6qoQUQVgsENERERqjcEOERERqTUGO0RU7chkMvj4+MDFxQX6+vpo2rQp/vrrLwD/PWI6fPgwmjRpAj09PbRr1w63b99WOMbff/+Nhg0bQldXF87Ozvjpp58UtmdlZWHOnDlwcHCArq4uXF1dsWHDBoV9rl69ilatWsHAwADt27cvtNI0Eb0ZGOwQUbXj4+ODLVu2YM2aNbhz5w6mT5+ODz/8EH5+fvJ9Zs2ahZ9++gmBgYGwtLRE//79kZOTAyA/SBk+fDjef/993Lp1C9988w3mz5+PTZs2yfN//PHH2LFjB1auXIl79+5h7dq1qFGjhkI9vvzyS/z0008ICgqClpYWPv3009dy/kRUsbgQKBFVK1lZWTAzM8PJkyfh4eEhT//ss8+Qnp6O8ePHo1u3bti5cydGjBgBAHjx4gXs7e2xadMmDB8+HKNGjcLTp09x/Phxef7Zs2fj8OHDuHPnDkJDQ1GvXj2cOHECPXv2LFSHs2fPolu3bjh58iR69OgBAPjnn3/Qr18/ZGRkQE9Pr5LfBSKqSGzZIaJqJTw8HOnp6XjnnXdQo0YN+WvLli148OCBfL+XAyEzMzPUq1cP9+7dAwDcu3cPHTp0UDhuhw4dEBYWhry8PFy/fh2ampro0qVLiXVp0qSJ/N+2trYAgISEhHKfIxG9XlpVXQEiopelpqYCAA4fPoxatWopbNPV1VUIeMpKX19fqf20tbXl/5ZIJADy+xMR0ZuFLTtEVK00aNAAurq6iIyMhKurq8LLwcFBvt/ly5fl/05MTERoaCjq168PAKhfvz4uXryocNyLFy+ibt260NTUROPGjSGTyRT6ABGR+mLLDhFVK0ZGRvjiiy8wffp0yGQydOzYEVKpFBcvXoSxsTGcnJwAAAsXLoS5uTmsra3x5ZdfwsLCAoMGDQIAzJw5E61bt8aiRYswYsQI+Pv7Y9WqVfj9998BAM7Ozhg9ejQ+/fRTrFy5Ek2bNsXjx4+RkJCA4cOHV9WpE1ElYbBDRNXOokWLYGlpCR8fHzx8+BA1a9ZEixYtMG/ePPljpCVLlmDq1KkICwtDs2bNcPDgQejo6AAAWrRogT///BNff/01Fi1aBFtbWyxcuBCffPKJvIzVq1dj3rx5mDRpEp4/fw5HR0fMmzevKk6XiCoZR2MR0RulYKRUYmIiatasWdXVIaI3APvsEBERkVpjsENERERqjY+xiIiISK2xZYeIiIjUGoMdIiIiUmsMdoiIiEitMdghIiIitcZgh4iIiNQagx0iIiJSawx2iIiISK0x2CEiIiK1xmCHiIiI1Nr/A45Ofu2hDDlOAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plt.plot(np.arange(len(epochs_x)), epochs_acc, marker = '.')\n", + "plt.xlabel('epoch')\n", + "plt.ylabel('test accuracy')\n", + "plt.ylim(0, 100)\n", + "plt.xticks(np.arange(len(epochs_x)))\n", + "for i, txt in enumerate(epochs_acc):\n", + " if i%2 == 0:\n", + " pass\n", + " else:\n", + " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", + "plt.show()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "speck-rescnn", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/single_training.ipynb b/tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/single_training.ipynb new file mode 100644 index 00000000..f5e3cfe9 --- /dev/null +++ b/tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/single_training.ipynb @@ -0,0 +1,1401 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import torch, random, sys\n", + "\n", + "import tonic\n", + "from torch.utils.data import DataLoader\n", + "from torch.nn import CrossEntropyLoss\n", + "from torch.optim import Adam\n", + "\n", + "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "\n", + "sys.path.append('../../utils')\n", + "sys.path.append('../models')\n", + "\n", + "from train_test_fn import training_loop, load_dataset, split_train_validation, load_architecture\n", + "from weight_initialization import rescale_method_1" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "device: NVIDIA GeForce RTX 3070 Ti\n" + ] + } + ], + "source": [ + "if torch.cuda.is_available():\n", + " device = torch.device('cuda:0')\n", + " print('device: ', torch.cuda.get_device_name(0))\n", + "else:\n", + " device = torch.device('cpu')" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "rand_seed = 1" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "achitecture = 'ResSCNN1'" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "torch.backends.cudnn.enabled = False\n", + "torch.backends.cudnn.deterministic = True\n", + "random.seed(rand_seed)\n", + "torch.manual_seed(rand_seed)\n", + "torch.cuda.manual_seed(rand_seed)\n", + "np.random.seed(rand_seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "batch_size = 8\n", + "num_workers = 4\n", + "epochs = 30\n", + "lr = 5e-5\n", + "\n", + "spk_thr = 2.0\n", + "v_min = -0.313\n", + "\n", + "grad_scale = 1.534\n", + "grad_width = 0.759\n", + "\n", + "validation_ratio = 0.2\n", + "n_time_steps = 50" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Loading Data" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "snn_train_dataset, snn_test_dataset, sensor_size = load_dataset('DVSGESTURE', n_time_steps)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "getting validation dataset...." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "train_dataset, validation_dataset = split_train_validation(validation_ratio, snn_train_dataset, rand_seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The transformed array is in shape [Time-Step, Channel, Height, Width] --> (50, 2, 128, 128)\n" + ] + } + ], + "source": [ + "sample_data, label = train_dataset[0]\n", + "print(f\"The transformed array is in shape [Time-Step, Channel, Height, Width] --> {sample_data.shape}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "disk caching samples..." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "disk_cache_train = tonic.DiskCachedDataset(\n", + " dataset=train_dataset,\n", + " cache_path='./cached_train'\n", + ")\n", + "snn_train_dataloader = DataLoader(disk_cache_train, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True)\n", + "\n", + "disk_cache_validation = tonic.DiskCachedDataset(\n", + " dataset=validation_dataset,\n", + " cache_path='./cached_validation'\n", + ")\n", + "snn_validation_dataloader = DataLoader(disk_cache_validation, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True)\n", + "\n", + "disk_cache_test = tonic.DiskCachedDataset(\n", + " dataset=snn_test_dataset,\n", + " cache_path='./cached_test'\n", + ")\n", + "snn_test_dataloader = DataLoader(disk_cache_test, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=False)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Network Module" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "snn = load_architecture(achitecture, sensor_size, 11, batch_size, PeriodicExponential(grad_scale=grad_scale, grad_width=grad_width), v_min, spk_thr).to(device)\n", + "snn.init_weights()" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "recaling factor: 3.2 (computed using 1 kernels and lambda 0.8)\n", + "recaling factor: 8.0 (computed using 2 kernels and lambda 0.8)\n", + "recaling factor: 3.2 (computed using 1 kernels and lambda 0.8)\n" + ] + } + ], + "source": [ + "lambda_ = 0.8\n", + "snn.rescale_conv_weights(rescale_method_1, lambda_)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "optimizer = Adam(snn.parameters(), lr=lr, betas=(0.9, 0.999), eps=1e-8)\n", + "loss_fn = CrossEntropyLoss()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Training loop" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "3eb61008049e42e4bc72738b0d76ad4b", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/107 [00:00" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "y_avg = []\n", + "for y in epochs_y:\n", + " y_avg.append(np.mean(y))\n", + "\n", + "plt.plot(np.arange(len(epochs_x)), y_avg, marker = '.')\n", + "plt.xlabel('epoch')\n", + "plt.ylabel('average loss')\n", + "plt.ylim(0,)\n", + "plt.xticks(np.arange(len(epochs_x)))\n", + "for i, txt in enumerate(y_avg):\n", + " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAG2CAYAAACZEEfAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABUOklEQVR4nO3deVhUZf8G8HtYZlhkEZFllE1FccXdUNMSAs3MLZeysvTNTDD3BUstLVHTLPfs55Jlbr1qLoUpbmmIouKKCIqispnAIDvC8/vDmNeRbQYGgdP9ua65inPO9zzPHI7DPc95zoxMCCFAREREJFEG1d0BIiIioqrEsENERESSxrBDREREksawQ0RERJLGsENERESSxrBDREREksawQ0RERJLGsENERESSxrBDREREksawQ0RERJJWrWHnxIkT6NevH5RKJWQyGfbs2aOxXgiBOXPmwNHREaampvDx8UF0dLTGNikpKRgxYgQsLS1hbW2N0aNHIyMj4zk+CyIiIqrJqjXsZGZmwtPTE6tWrSpx/eLFi7F8+XKsXbsWYWFhMDc3h5+fH3JyctTbjBgxAlevXsWhQ4ewf/9+nDhxAmPGjHleT4GIiIhqOFlN+SJQmUyG3bt3Y8CAAQCejOoolUpMmTIFU6dOBQCoVCrY29tj06ZNGD58OCIjI9GiRQucPXsWHTt2BAAEBwfj1Vdfxb1796BUKqvr6RAREVENYVTdHShNbGwsEhMT4ePjo15mZWWFLl26IDQ0FMOHD0doaCisra3VQQcAfHx8YGBggLCwMAwcOLDEfefm5iI3N1f9c2FhIVJSUlCvXj3IZLKqe1JERESkN0IIPHr0CEqlEgYGpV+sqrFhJzExEQBgb2+vsdze3l69LjExEXZ2dhrrjYyMYGNjo96mJEFBQfj888/13GMiIiKqDnfv3kXDhg1LXV9jw05VCgwMxOTJk9U/q1QqODs74+7du7C0tKzGnhEREZG20tPT4eTkBAsLizK3q7Fhx8HBAQCQlJQER0dH9fKkpCS0bdtWvU1ycrJG3ePHj5GSkqKuL4lCoYBCoSi23NLSkmGHiIiolilvCkqN/ZwdNzc3ODg4ICQkRL0sPT0dYWFh8PLyAgB4eXkhLS0N586dU29z5MgRFBYWokuXLs+9z0RERFTzVOvITkZGBmJiYtQ/x8bGIiIiAjY2NnB2dsbEiRPxxRdfwN3dHW5ubpg9ezaUSqX6jq3mzZujd+/e+OCDD7B27Vrk5+cjICAAw4cP551YREREBKCaw054eDhefvll9c9F82hGjhyJTZs2Yfr06cjMzMSYMWOQlpaG7t27Izg4GCYmJuqaLVu2ICAgAN7e3jAwMMDgwYOxfPny5/5ciIiIqGaqMZ+zU53S09NhZWUFlUrFOTtERES1hLZ/v2vsnB0iIiIifWDYISIiIklj2CEiIiJJY9ghIiIiSWPYISIiIklj2CEiIiJJY9ghIiIiSWPYISIiIklj2CEiIiJJY9ghIiIiSWPYISIiIklj2CEiIiJJY9ghIiIiSWPYISIiIklj2CEiIiJJY9ghIiIiSWPYISIiIklj2CEiIiJJY9ghIiIiSWPYISIiIklj2CEiIiJJY9ghIiIiSWPYISIiIklj2CEiIiJJY9ghIiIiSWPYISIiIklj2CEiIiJJY9ghIiIiSWPYISIiIklj2CEiIiJJY9ghIiIiSWPYISIiIklj2CEiIiJJY9ghIiIiSWPYISIiIklj2CEiIiJJY9ghIiIiSWPYISIiIklj2CEiIiJJY9ghIiIiSWPYISIiIklj2CEiIiJJY9ghIiIiSWPYISIiIklj2CEiIiJJY9ghIiIiSWPYISIiIklj2CEiIiJJY9ghIiIiSWPYISIiIklj2CEiIiJJY9ghIiIiSWPYISIiIklj2CEiIiJJY9ghIiIiSWPYISIiIklj2CEiIiJJY9ghIiIiSWPYISIiIklj2CEiIiJJY9ghIiIiSWPYISIiIklj2CEiIiJJq9Fhp6CgALNnz4abmxtMTU3RuHFjzJ8/H0II9TZCCMyZMweOjo4wNTWFj48PoqOjq7HXREREVJPU6LCzaNEirFmzBitXrkRkZCQWLVqExYsXY8WKFeptFi9ejOXLl2Pt2rUICwuDubk5/Pz8kJOTU409JyIioppCJp4eJqlhXnvtNdjb22P9+vXqZYMHD4apqSl++uknCCGgVCoxZcoUTJ06FQCgUqlgb2+PTZs2Yfjw4Vq1k56eDisrK6hUKlhaWlbJcyEiIiL90vbvd40e2enatStCQkJw48YNAMDFixdx8uRJ9OnTBwAQGxuLxMRE+Pj4qGusrKzQpUsXhIaGlrrf3NxcpKenazyIiIhImoyquwNlmTlzJtLT0+Hh4QFDQ0MUFBTgyy+/xIgRIwAAiYmJAAB7e3uNOnt7e/W6kgQFBeHzzz+vuo4TERFRjVGjR3Z27NiBLVu24Oeff8b58+fxww8/YMmSJfjhhx8qtd/AwECoVCr14+7du3rqMREREdU0NXpkZ9q0aZg5c6Z67k3r1q1x584dBAUFYeTIkXBwcAAAJCUlwdHRUV2XlJSEtm3blrpfhUIBhUJRpX0nIiKimqFGj+xkZWXBwECzi4aGhigsLAQAuLm5wcHBASEhIer16enpCAsLg5eX13PtKxEREdVMNXpkp1+/fvjyyy/h7OyMli1b4sKFC/j6668xatQoAIBMJsPEiRPxxRdfwN3dHW5ubpg9ezaUSiUGDBhQvZ0nIiKiGqFGh50VK1Zg9uzZGDduHJKTk6FUKvHhhx9izpw56m2mT5+OzMxMjBkzBmlpaejevTuCg4NhYmJSjT0nIiKimqJGf87O88LP2SEiIqp9JPE5O0RERESVxbBDREREksawQ0RERJLGsENERESSxrBDREREksawQ0RERJLGsENERESSxrBDREREksawQ0RERJLGsENERESSxrBDREREksawQ0RERJLGsENERESSxrBDREREksawQ0RERJLGsENERESSxrBDREREksawQ0RERJLGsENERESSxrBDREREksawQ0RERJLGsENERESSxrBDREREksawQ0RERJLGsENERESSxrBDREREksawQ0RERJLGsENERESSxrBDREREksawQ0RERJLGsENERESSxrBDREREksawQ0RERJLGsENERESSxrBDREREksawQ0RERJLGsENERESSxrBDREREksawQ0RERJLGsENERESSxrBDREREksawQ0RERJLGsENERESSxrBDREREksawQ0RERJLGsENERESSxrBDREREksawQ0RERJLGsENERESSxrBDREREksawQ0RERJLGsENERESSxrBDREREksawQ0RERJLGsENERESSxrBDREREksawQ0RERJLGsENERESSpnPYOXr0aFX0g4iIiKhK6Bx2evfujcaNG+OLL77A3bt3q6JPRERERHqjc9i5f/8+AgIC8Msvv6BRo0bw8/PDjh07kJeXVxX9IyIiIqoUncOOra0tJk2ahIiICISFhaFp06YYN24clEolPv74Y1y8eLEq+klERERUIZWaoNy+fXsEBgYiICAAGRkZ2LBhAzp06IAXX3wRV69e1VcfiYiIiCqsQmEnPz8fv/zyC1599VW4uLjg4MGDWLlyJZKSkhATEwMXFxcMGTJELx28f/8+3n77bdSrVw+mpqZo3bo1wsPD1euFEJgzZw4cHR1hamoKHx8fREdH66VtIiIiqv10Djvjx4+Ho6MjPvzwQzRt2hQXLlxAaGgo/vOf/8Dc3Byurq5YsmQJrl+/XunOpaamolu3bjA2Nsbvv/+Oa9euYenSpahbt656m8WLF2P58uVYu3YtwsLCYG5uDj8/P+Tk5FS6fSIiIqr9jHQtuHbtGlasWIFBgwZBoVCUuI2tra1eblFftGgRnJycsHHjRvUyNzc39f8LIfDNN9/g008/Rf/+/QEAmzdvhr29Pfbs2YPhw4dXug9ERERUu+k8shMSEoI333yz1KADAEZGRujZs2elOgYAe/fuRceOHTFkyBDY2dmhXbt2+P7779XrY2NjkZiYCB8fH/UyKysrdOnSBaGhoaXuNzc3F+np6RoPIiIikiadw05QUBA2bNhQbPmGDRuwaNEivXSqyK1bt7BmzRq4u7vj4MGD+Oijj/Dxxx/jhx9+AAAkJiYCAOzt7TXq7O3t1etKEhQUBCsrK/XDyclJr/0mIiKimkPnsPPdd9/Bw8Oj2PKWLVti7dq1eulUkcLCQrRv3x4LFixAu3btMGbMGHzwwQeVbicwMBAqlUr94IcjEhERSZfOYScxMRGOjo7FltevXx8JCQl66VQRR0dHtGjRQmNZ8+bNERcXBwBwcHAAACQlJWlsk5SUpF5XEoVCAUtLS40HERERSZPOYcfJyQmnTp0qtvzUqVNQKpV66VSRbt26ISoqSmPZjRs34OLiAuDJZGUHBweEhISo16enpyMsLAxeXl567QsRERHVTjrfjfXBBx9g4sSJyM/PR69evQA8mbQ8ffp0TJkyRa+dmzRpErp27YoFCxZg6NChOHPmDNatW4d169YBAGQyGSZOnIgvvvgC7u7ucHNzw+zZs6FUKjFgwAC99oWIiIhqJ53DzrRp0/Dw4UOMGzdO/X1YJiYmmDFjBgIDA/XauU6dOmH37t0IDAzEvHnz4Obmhm+++QYjRoxQbzN9+nRkZmZizJgxSEtLQ/fu3REcHAwTExO99oWIiIhqJ5kQQlSkMCMjA5GRkTA1NYW7u3uZt6LXdOnp6bCysoJKpeL8HSIiolpC27/fOo/sFKlTpw46depU0XIiIiKi56JCYSc8PBw7duxAXFyc+lJWkV27dumlY0RERET6oPPdWNu2bUPXrl0RGRmJ3bt3Iz8/H1evXsWRI0dgZWVVFX0kIiIiqjCdw86CBQuwbNky7Nu3D3K5HN9++y2uX7+OoUOHwtnZuSr6SERERFRhOoedmzdvom/fvgAAuVyOzMxMyGQyTJo0SX1LOBEREVFNoXPYqVu3Lh49egQAaNCgAa5cuQIASEtLQ1ZWln57R0RERFRJOk9Q7tGjBw4dOoTWrVtjyJAhmDBhAo4cOYJDhw7B29u7KvpIREREVGE6h52VK1ciJycHAPDJJ5/A2NgYf/31FwYPHoxPP/1U7x0kIiIiqgydws7jx4+xf/9++Pn5AQAMDAwwc+bMKukYERERkT7oNGfHyMgIY8eOVY/sEBEREdV0Ok9Q7ty5MyIiIqqgK0RERET6p/OcnXHjxmHy5Mm4e/cuOnToAHNzc431bdq00VvniIiIiCpL5y8CNTAoPhgkk8kghIBMJkNBQYHeOve88ItAiYiIap8q+yLQ2NjYSnWMiIiI6HnSOey4uLhURT+IiIiIqoTOYWfz5s1lrn/33Xcr3BkiIiIifdN5zk7dunU1fs7Pz0dWVhbkcjnMzMyQkpKi1w4+D5yzQ0REVPto+/db51vPU1NTNR4ZGRmIiopC9+7dsXXr1kp1moiIiEjfdA47JXF3d8fChQsxYcIEfeyOiIiISG/0EnaAJ5+uHB8fr6/dEREREemFzhOU9+7dq/GzEAIJCQlYuXIlunXrpreOEREREemDzmFnwIABGj/LZDLUr18fvXr1wtKlS/XVLyIiIiK90DnsFBYWVkU/iIiIiKqE3ubsEBEREdVEOoedwYMHY9GiRcWWL168GEOGDNFLp4iIiIj0Reewc+LECbz66qvFlvfp0wcnTpzQS6eIiIiI9EXnsJORkQG5XF5subGxMdLT0/XSKSIiIiJ90TnstG7dGtu3by+2fNu2bWjRooVeOkVERESkLzrfjTV79mwMGjQIN2/eRK9evQAAISEh2Lp1K3bu3Kn3DhIRERFVhs5hp1+/ftizZw8WLFiAX375BaampmjTpg0OHz6Mnj17VkUfiYiIiCpM5289lyJ+6zkREVHtU2Xfen727FmEhYUVWx4WFobw8HBdd0dERERUpXQOO/7+/rh7926x5ffv34e/v79eOkVERESkLzqHnWvXrqF9+/bFlrdr1w7Xrl3TS6eIiIiI9EXnsKNQKJCUlFRseUJCAoyMdJ7vTERERFSldA47vr6+CAwMhEqlUi9LS0vDrFmz8Morr+i1c0RERESVpfNQzJIlS9CjRw+4uLigXbt2AICIiAjY29vjxx9/1HsHiYiIiCpD57DToEEDXLp0CVu2bMHFixdhamqK999/H2+++SaMjY2roo9EREREFVahSTbm5uYYM2aMvvtCREREpHcVnlF87do1xMXFIS8vT2P566+/XulOEREREemLzmHn1q1bGDhwIC5fvgyZTIaiD2CWyWQAgIKCAv32kIiIiKgSdL4ba8KECXBzc0NycjLMzMxw9epVnDhxAh07dsSxY8eqoItEREREFafzyE5oaCiOHDkCW1tbGBgYwMDAAN27d0dQUBA+/vhjXLhwoSr6SURERFQhOo/sFBQUwMLCAgBga2uL+Ph4AICLiwuioqL02zsiIiKiStJ5ZKdVq1a4ePEi3Nzc0KVLFyxevBhyuRzr1q1Do0aNqqKPRERERBWmc9j59NNPkZmZCQCYN28eXnvtNbz44ouoV68etm/frvcOEhEREVWGTBTdTlUJKSkpqFu3rvqOrNomPT0dVlZWUKlUsLS0rO7uEBERkRa0/futl2/utLGx0cduiIiIiPRO5wnKRERERLUJww4RERFJGsMOERERSZrOYefEiRN4/PhxseWPHz/GiRMn9NIpIiIiIn3ROey8/PLLSElJKbZcpVLh5Zdf1kuniIiIiPRF57AjhCjxFvOHDx/C3NxcL50iIiIi0hetbz0fNGgQgCffbv7ee+9BoVCo1xUUFODSpUvo2rWr/ntIREREVAlahx0rKysAT0Z2LCwsYGpqql4nl8vxwgsv4IMPPtB/D4mIiIgqQeuws3HjRgCAq6srpk6dyktWREREVCvoPGdn+vTpGnN27ty5g2+++QZ//PGHXjtGREREpA86h53+/ftj8+bNAIC0tDR07twZS5cuRf/+/bFmzRq9d5CIiIioMnQOO+fPn8eLL74IAPjll1/g4OCAO3fuYPPmzVi+fLneO0hERERUGTqHnaysLFhYWAAA/vjjDwwaNAgGBgZ44YUXcOfOHb13kIiIiKgydA47TZo0wZ49e3D37l0cPHgQvr6+AIDk5OQyv16diIiIqDroHHbmzJmDqVOnwtXVFZ07d4aXlxeAJ6M87dq103sHiYiIiCpD57DzxhtvIC4uDuHh4Th48KB6ube3N5YtW6bXzj1r4cKFkMlkmDhxonpZTk4O/P39Ua9ePdSpUweDBw9GUlJSlfaDiIiIao8Kfeu5g4MDLCwscOjQIWRnZwMAOnXqBA8PD7127mlnz57Fd999hzZt2mgsnzRpEvbt24edO3fi+PHjiI+PV3/aMxEREZHOYefhw4fw9vZG06ZN8eqrryIhIQEAMHr0aEyZMkXvHQSAjIwMjBgxAt9//z3q1q2rXq5SqbB+/Xp8/fXX6NWrFzp06ICNGzfir7/+wunTp6ukL0RERFS76Bx2Jk2aBGNjY8TFxcHMzEy9fNiwYQgODtZr54r4+/ujb9++8PHx0Vh+7tw55Ofnayz38PCAs7MzQkNDS91fbm4u0tPTNR5EREQkTVp/XUSRP/74AwcPHkTDhg01lru7u1fJrefbtm3D+fPncfbs2WLrEhMTIZfLYW1trbHc3t4eiYmJpe4zKCgIn3/+ub67SkRERDWQziM7mZmZGiM6RVJSUjS+CV0f7t69iwkTJmDLli0wMTHR234DAwOhUqnUj7t37+pt30RERFSz6Bx2XnzxRfXXRQCATCZDYWEhFi9ejJdfflmvnTt37hySk5PRvn17GBkZwcjICMePH8fy5cthZGQEe3t75OXlIS0tTaMuKSkJDg4Ope5XoVDA0tJS40FERETSpPNlrMWLF8Pb2xvh4eHIy8vD9OnTcfXqVaSkpODUqVN67Zy3tzcuX76ssez999+Hh4cHZsyYAScnJxgbGyMkJASDBw8GAERFRSEuLk79+T9ERET076Zz2GnVqhVu3LiBlStXwsLCAhkZGRg0aBD8/f3h6Oio185ZWFigVatWGsvMzc1Rr1499fLRo0dj8uTJsLGxgaWlJcaPHw8vLy+88MILeu0LERER1U46h524uDg4OTnhk08+KXGds7OzXjqmrWXLlsHAwACDBw9Gbm4u/Pz8sHr16ufaByIiIqq5ZEIIoUuBoaEhEhISYGdnp7H84cOHsLOzQ0FBgV47+Dykp6fDysoKKpWK83eIiIhqCW3/fus8QVkIAZlMVmx5RkaGXu+YIiIiItIHrS9jTZ48GcCTu69mz56tcft5QUEBwsLC0LZtW713kIiIiKgytA47Fy5cAPBkZOfy5cuQy+XqdXK5HJ6enpg6dar+e0hERERUCVqHnaNHjwJ4cuv3t99+y7ktREREVCvofDfWxo0bq6IfRERERFVC5wnKRERERLUJww4RERFJGsMOERERSRrDDhEREUkaww4RERFJGsMOERERSRrDDhEREUkaww4RERFJGsMOERERSRrDDhEREUkaww4RERFJGsMOERERSRrDDhEREUkaww4RERFJGsMOERERSRrDDhEREUkaww4RERFJGsMOERERSRrDDhEREUkaww4RERFJGsMOERERSRrDDhEREUkaww4RERFJGsMOERERSRrDDhEREUkaww4RERFJGsMOERERSRrDDhEREUkaww4RERFJGsMOERERSRrDDhEREUkaww4RERFJGsMOERERSRrDDhEREUkaww4RERFJGsMOERERSRrDDhEREUkaww4RERFJGsMOERERSRrDDhEREUkaww4RERFJGsMOERERSRrDDhEREUkaww4RERFJGsMOERERSRrDDhEREUkaww4RERFJGsMOERERSRrDDhEREUkaww4RERFJGsMOERERSRrDDhEREUkaww4RERFJGsMOERERSRrDDhEREUkaww4RERFJGsMOERERSRrDDhEREUlajQ47QUFB6NSpEywsLGBnZ4cBAwYgKipKY5ucnBz4+/ujXr16qFOnDgYPHoykpKRq6jERERHVNDU67Bw/fhz+/v44ffo0Dh06hPz8fPj6+iIzM1O9zaRJk7Bv3z7s3LkTx48fR3x8PAYNGlSNvSYiIqKaRCaEENXdCW09ePAAdnZ2OH78OHr06AGVSoX69evj559/xhtvvAEAuH79Opo3b47Q0FC88MILWu03PT0dVlZWUKlUsLS0rMqnQERERHqi7d/vGj2y8yyVSgUAsLGxAQCcO3cO+fn58PHxUW/j4eEBZ2dnhIaGlrqf3NxcpKenazyIiIhImmpN2CksLMTEiRPRrVs3tGrVCgCQmJgIuVwOa2trjW3t7e2RmJhY6r6CgoJgZWWlfjg5OVVl14mIiKga1Zqw4+/vjytXrmDbtm2V3ldgYCBUKpX6cffuXT30kIiIiGoio+rugDYCAgKwf/9+nDhxAg0bNlQvd3BwQF5eHtLS0jRGd5KSkuDg4FDq/hQKBRQKRVV2mYiIiGqIGj2yI4RAQEAAdu/ejSNHjsDNzU1jfYcOHWBsbIyQkBD1sqioKMTFxcHLy+t5d5eIiIhqoBo9suPv74+ff/4Zv/76KywsLNTzcKysrGBqagorKyuMHj0akydPho2NDSwtLTF+/Hh4eXlpfScWERERSVuNvvVcJpOVuHzjxo147733ADz5UMEpU6Zg69atyM3NhZ+fH1avXl3mZaxn8dZzIiKi2kfbv981Ouw8Lww7REREtY8kP2eHiIiISFcMO0RERCRpDDtEREQkaQw7REREJGkMO0RERCRpDDtEREQkaQw7REREJGkMO0RERCRpDDulOHHiBPr16welUgmZTIY9e/ao1+Xn52PGjBlo3bo1zM3NoVQq8e677yI+Pr7MfX722WeQyWQaDw8PD41tcnJy4O/vj3r16qFOnToYPHgwkpKSquIpEhER/Ssw7JQiMzMTnp6eWLVqVbF1WVlZOH/+PGbPno3z589j165diIqKwuuvv17uflu2bImEhAT14+TJkxrrJ02ahH379mHnzp04fvw44uPjMWjQIL09LyIion8dQUKlUgkAQqVSlbgegNi9e3eZ+zhz5owAIO7cuVPqNnPnzhWenp6lrk9LSxPGxsZi586d6mWRkZECgAgNDS2z/Zri+PHj4rXXXhOOjo7FjlteXp6YPn26aNWqlTAzMxOOjo7inXfeEffv3y93vytXrhQuLi5CoVCIzp07i7CwMPW6hw8fioCAANG0aVNhYmIinJycxPjx40VaWlpVPEUiIqohyvv7XYQjO3qiUqkgk8lgbW1d5nbR0dFQKpVo1KgRRowYgbi4OPW6c+fOIT8/Hz4+PuplHh4ecHZ2RmhoaFV1Xa+qYkRs+/btmDx5MubOnYvz58/D09MTfn5+SE5OBgDEx8cjPj4eS5YswZUrV7Bp0yYEBwdj9OjRVfIciYiolnlO4atGq+zITnZ2tmjfvr146623ymznt99+Ezt27BAXL14UwcHBwsvLSzg7O4v09HQhhBBbtmwRcrm8WF2nTp3E9OnTtX9CNUR5x00I7UbEOnfuLPz9/dU/FxQUCKVSKYKCgkqt2bFjh5DL5SI/P1/nfhMRUe2g7ciOUbUmLQnIz8/H0KFDIYTAmjVryty2T58+6v9v06YNunTpAhcXF+zYseNfOwpR3ohYXl4ezp07h8DAQPUyAwMD+Pj4lDnaVfQNuEZGPMWJiP7teBmrEoqCzp07d3Do0KEyv16+JNbW1mjatCliYmIAAA4ODsjLy0NaWprGdklJSXBwcNBXt2uMnJwczJgxA2+++Wapx+7vv/9GQUEB7O3tNZbb29sjMTGx1Jr58+djzJgxeu8zERHVPgw7FVQUdKKjo3H48GHUq1dP531kZGTg5s2bcHR0BAB06NABxsbGCAkJUW8TFRWFuLg4eHl56a3vNYEuI2K6SE9PR9++fdGiRQt89tlnetsvERHVXhzjL0VGRoZ6xAUAYmNjERERARsbGzg6OuKNN97A+fPnsX//fhQUFKhHGWxsbCCXywEA3t7eGDhwIAICAgAAU6dORb9+/eDi4oL4+HjMnTsXhoaGePPNNwEAVlZWGD16NCZPngwbGxtYWlpi/Pjx8PLywgsvvPCcj0DFJaiyEft3Zqnrnx4RO3LkSJkjYra2tjA0NCz2WUMljXY9evQIvV7xhczIBKs3/QxjY+PKPZHnrOi4udmaw9HKtLq7Q0QkHc9lBlENV9IEp6NHjwoAxR4jR44UsbGxJa4DII4ePareh4uLi5g7d67652HDhglHR0chl8tFgwYNxLBhw0RMTIxGX7Kzs8W4ceNE3bp1hZmZmRg4cKBISEio6kOgN9vO3BFuM/cLlxn7BQAxZfE6jfV5eXliwIABomXLliI5OVmrfXbu3FkEBASofy4oKBANGjTQmKCsUqmEe6t2wsSplXCa/ItwnblfrP/zligsLNTPE6tCx48fF+27ewvDOjYCgLAb9InYduZ/E7YLCwvF7NmzhYODgzAxMRHe3t7ixo0b5e6ztI8AqMx+a5LyniOPW8l43EgXNf184a3nlfTSSy9BCFHssWnTJri6upa4TgiBl156Sb2P0IuR8H0nAAmqbADAtm3bEB8fj9zcXNy7dw/btm1D48aNNdo1MTHBqlWrkJKSgszMTOzatavWzNdJUGVjxrYzyEm8hbykWwCADcFnMGzBz1i1LxSR91PxxhtvIDw8HFu2bFGPiCUmJiIvL0+9H29vb6xcuVL98+TJk/H999/jhx9+QGRkJD766CNkZmbi3ZHv4Wq8CusOX4Z7+264nZQKmz4TIHKz8fhRKuZuO4Vmn+xHryXHMOL/TmPazov4+tANbDsTh+M3HiAm+REycx8Xew5/3fxb/Tt7Hv66fh8xBbaweWUsAEAIIHDXZXUfFi9ejOXLl2Pt2rUICwuDubk5/Pz8kJOTU+o+y/oIgCIV2W9VqOgxL+858riV7N9+3CqjOl4fqptkzpcyo9C/hLbJUBfbztwRrjOejHC4zdyv8U5dqlYdjRb2by4occTLvJW3aDB2vVYjYg2dnMWo8dNEfFqWetmKFSuEk5OzMDaWC+dmbYRf4P+JFrN/Fy4z9pfaJgDRYOx64fLP76G0R+u5wcJv2XHht+y4ellV/84KCwvFketJYvh3oRp9ASDqD/xEuMzYLzadejIy5eDgIL766it1bVpamlAoFGLr1q1atYVS3o1Vdr/6sO3MHeE6s/LH/NnnKPXjtuHkLb28vvzbjltl6Otcra3i07IEALH+p23qZTXhfOGt59UoQZWNwF2XIf75uVAAM/97GU3s6qCDi0219q0qCCGw+thNfHUwCibObeAyY796nYEMeLuLC6KSHuHivTSNdQBgZCBDywZWOPGoLrIuJ+B+ajaMR6xGiACOLjyCoR2dYGQoQ/jjNjAcsRrKfw7q9UIAeQWoozDCi9690PStK9hw6jaEgEbb2z98AY8LgPi0bCSoshGvykFCWjbi03IQr8rGo5zHSM95jPTERxr9KhTAjP9eRmpmPoZ1ckJdc7lejlXe40LsvRiP70/cQlTSkzYNZUCBKL7tZ3uv4WpUDBITEzU+aNLKygpdunRBaGgohg8fXqF+xMbGVsl+dXE3JRMz/6v57yRw12X0aFq/0nOWqur5Vedxy8h9jEPXErEz/B7+uvlQvZzHrepdvpeGGf+9rP65UAAztobh8IbF+OO3fUhOTka7du3w7bffolOnTiXuIyEhAVOmTEF4eDhiYmLw8ccf45tvvtHYZteuXViwYAFiYmKQn58Pd3d3TJkyBe+8805VPr1ybT8bh8BdT55/4K7LMG/qhWGdnGvV+cKwUwVi/85E4TN/vASAN9aE4tU2jviwRyO0aWhdHV3Tu4zcx5i64yKCrz6ZoN3FzQZnb6egUACGMhkWDGqFYZ2cAQD5BYW4Fp+O8DupOH8nFeF3UpCUnouLd9Nw8W4aNpyK1dh3oQC2nb2rsczJxhQdnOuig6sNOjjXRTMHCxgayAAATe0tMGvXFRQIoW67k2vZd8ll5D5GQlo2Qq4nY+Hv14utXxh8HUv+iMKL7rZ4rY0Svi3tYWGi+8Tn9Jx8bA2Lw8ZTt5GY/mQY1lxuiDc7O2NUdzf8Gf0As3ZdAQDIZEB7Z2ucj0vD5pCLT+phprG/sm6910ZRrS639OtTTHIGPtwcjmczXqEAPtt7Fd8MawdTuWGF919Vz+95H7fsvAIcjUrGvovxOHI9GbmPC0vcrlAAG07GYtarzSGTySrcnlSOm77k5Bdg/clYrAiJLrbuwe8rsD87Adt//BFKpRI//fQTfHx8cO3aNTRo0KDY9rm5uahfvz4+/fRTLFu2rMT2bGxs8Mknn8DDwwNyuRz79+/H+++/Dzs7O/j5+en9+WkjPi0LM3ddVr+RFAKYtesKejStX6vOF4adKuBmaw4DGUoMPAcuJeDApQS80MgGH/ZojJea1a/Ui1N1iv07E2M2hyM6OQNyQwPM698Swzs7I0GVjdt/Z8HV1kzjnaaxoQE8nazh6WSN0d3dIITAvdRsnI9LRfjtVJy48QB3UrKKtfNqKwf081Sig0td2FmalNqfYZ2c0aNp/RLbLk0dhRHc7S1Qx8QIi4Ova/zOZACa2NVBdHIGjkY9wNGoB5DvNsDLzeqjn6cS3h725f5BTlBlY+Op2/g5LA4Z/8wPsrNQ4P1ubnirizOsTI01+q5cCAQNao1RI7oh+EoiPv42GkkARm48g2kDDfBhj8bqcFcbFRYK/BB6Gwt/v17qH+6DV5PQd/mf+HpYW7R1sn6+HawBch8X4M8bf2PfpXgcvpaEzLwC9bpGtuZ4qVl9bPrrdrHXl+//jMXNB5lYOKh1mf9OqHxCCPx2ORFBv0fiXmrx+TmF+bnIijoF88GzEZ5jj4mNGuOzzz7Dvn37sGbNGnzxxRfFalxdXfHtt98CADZs2FBiu0/P+QSACRMm4IcffsDJkyerJexEJT7C1J0RGiPmAFAgBC7dTYPFc+9RxTHsVAFHK1MEDWpdbJShTUNrfH/iFvZejMfpWyk4fSsFTe3r4IMXG6F/2waQG9We+eJHo5Lx8dYLeJTzGHYWCqx9pwPaO9cF8OT5axM0ZDIZnGzM4GRjhv5tGyBBlY1uC49ovIgbymSY3a+F1sPz2rZdUl1Jv7NhnZwRk5yB/Zfise9iPG4+yMTBq0k4eDUJZnJD+DS3Rz9PJXo0tYXCyFB9+3hBocDuC/exNyIej/95Qu52dfBBj0bo31YJhVHxkFTUbxtzBQCgdysHbBnvh87/B+Smp2JxcBRCIpOxdIgnkpKS0LZtW52fZ5GiSe9JSUnqz3kq+rky+y3LvdQsTNt5CaG3nlyCedHdFi+618ei36+rj/m7XV3w2+UE3Po7E4PX/IVxLzXG+F7uOv/bqKrnp+/9Fp0vTnVNEft3FvZfikfwlUSk5/xv4nwDa1P081Sin6cjWjhaQiaToZnD/0YxDWRAn1aOOHQtCUeuJ8P3mxP4YkArvNZGWe3Pr6r3WxWu3Fdh3r5rOHM7BQDgYGmCmX08kJ1fgE93/3PMRQEgCiEzNMbyIzG4Gp+OZcPbwtTUFCdPntRLP4QQOHLkCKKiorBo0aLn+tEUKZl5+PpQFH4OiysWqouM33oBPg2fvPGqDecLw04VKW2U4ethbTHVrxk2nIzF1jNxuJGUgWm/XMKSP6LU7/YtK3CZ5Hkpmp+z5I8oCAF0cKmLNSPa6+WdZGmB43l95kxpv7MmdnUw0acpJni743riI+y7GI99l+JxNyUbey/GY+/FeFiYGKGpvQXO30ktdmmmi5sNPuzZCC81tYOBjqMyHVt7wMHBAT5WybigaIpzd1LhuzgYt0+fxtixYyv8XN3c3ODg4ICQkBD1i0d6ejrCwsLw0UcfVXi/JRFC4Jdz9/D5vmvIyH0MU2NDzOrbHG93cYZMJkM/T0eNYz7B2x1zfr2KvRfjseJIDI5cT8bXQ9uimYP27yOr6vnpa79CCGw4GYsvfoss9q4ZeDL691obJV7zdEQ7J+tio78lnas3kh5h0vYIXI1PR8DPF/DH1STM698S1mbazzer6cetKiWn5+Crg1H45fw9CAGYGD8ZSf2wZyOYyZ/8qXyp2f+O+eDTXlBF70e2vQsOXyvAC+/vQVRoKJo0aVKpfqhUKjRo0AC5ubkwNDTE6tWrkWLdTP1G0ED2ZPS3aHqAPuU9LsTm0Nv4NiQaj/4J3L1bOqBNQyss/eMGgCeX2RvWNcW91GwcuF0IQ/O6+GjxD1i10AXtnevW2POFYacKlTbKoLQ2xaevtcB4b3f8HBaHjadikZSei4W/X8fKIzF4q4sz3u/mCgAVTvJV8S7g2fk5I7o4Y26/lnodkarIpSh9KmtkSCaTobmjJZo7WmKaXzNcvKfCvovxOHApAYnpOTh3J7VYzf+N7Aif5vYl7O1/yvoAS2dnZ0ycOBELFy7E16vaYGdULo5tXQVhWhf7VA3wkiobjlamxT7Asrx9ymQyTJw4EV988QXc3d3h5uaG2bNnQ6lUYsCAARU4ciV78CgXgbsu43Dkkw+F7OBSF0uHeMLV1ly9zbPH3NpMjuVvtoNfSwd8sucyrsano9+Kk5jq1xSjuzdSX8bT5riV9/wqe9zq2Dris7lzYe/gqLHfRzn5SFDlIP6fyfAJqv/9N0GVg/upWcgrYVb6wLZKDOvsjE6uNuVernz2uDW1t8Ducd2w8kg0Vh27ib0X4xEW+xCL3/BEz6b11dvVhONWVedbRRTNy1l9NEZ9yXBAWyWm9/aA0lrzteDpY/7jjz9i1KhROLH8HcDAAHL7xrBo2RPZGXeLtaELCwsLREREICMjAyEhIZg4aTLM+s6AiXMbAP+74cW2jgK9POz0Mg1CCIGQyGR8+Vuk+gNhmztaYs5rLdDaXoGYmBi4v2oD34XAuA6WeN23Lu5k1sPemDzs7tgfZ3Z9j965Fmjbwh0Zf/1cI88X2T+3fv2rpaenw8rKSv3lkc9b7uMC/Brx5A6d6OQMANCY8yMD0L+tEh1ctbuT69ztFPwaEQ/xz36+HNgab3au3LuA0ubn0JN5KJv+uo15+68VW7f1gxfg1bjsSdLHjh3Dyy+/XGz5yJEjsWnTJgghMHfuXKxbtw5paWlo1KoDsju9B2GlhKWJEeb1b4WJA7th8PARGPzBJLjZmiPqQliZ+wRQbL/du3fH6tWr0bRp04odiGcEX0nArN1XkJKZB2NDGSa/0gxjejTSac5RcnoOZu66jCPXkwEAnV1tsGSIJ5zrmel83Ep6fk7OLvAdMAxfzP8cjlam5e4TALLzHmPGrE+xcf3/IeNROkwatoCN7zi0at4MgAzxadl49MznN2lLm/NFGxF30zB5RwRuPXjyh2tEF2fMerU5zBVG1Xbcnt1v6w5dsGz5CnTv0Ean51bZN3JF9a71zHAhToUFv0XiftqTeTltnawxp18L9SV5bWRmZiI24QHmhyRg/zczIPKy8cWanzDRp2mpI7kvvfQS2rZtW+xurKelZubhp9N3MHfaeGSnJsN+2Pxi2zjbmOG1No7o56mEh4NFhYJPVOIjfHHgGv6M/hsAYFtHjqm+zTCkoxMMDWTl/m6jEtPxXsA0nA3egYKcTJg0bIG2w6dgwqCeGNCuAUyMDfVyvpT2+qTt32+GHVR/2ClSWChw7EYylofEIOJuml73/UIjG3RvYov2LnXR1slaPSyrjaPXk/HxtpLn59ATpc03Ojnz5SoZnYpJzsCUHRG4eE8FAGjdwBJX49OrfJhbG6rsfHy+9yp2XbgPAPBwsMCyYW3R3LFi/7aEENgRfhfz9l1DZl4BzOSG+LRvC7zZ2alS72qLbqd9+pgNbt8QyY9yNUZi4tNy/vnogic//52RV/7OAViZGsPRygRKa1P1f5XWJnC0MoWxoQxD1oZW6fmSnVeAxQevY+Op2wCe/GH8eqgnOmr5pqk0JR03Xc61ytTrs+2nFc3Led1TqfOl5iLJfz+Es6sbzLuPhEXb3vD2sMOy4W1LnJZQVti5m5KF9Sdjsf3sXWTnF+Dv377B47REOLy1UL2NDIDCyAA5T03yb2JXB/3+ufTZuH6dcvv77LwcuaEBRnV3g//LjSt0x2miKgcb/4rFz6fj1GHfto4CHVyscehaUpW9NjHs6KCmhJ0if938G299H1ZseWfXuqhXR1Fm7cOMXJy5XfxyytMMDWRo4WiJDi510cGlLjq61i32Apugykbsg0wcv5GMdX/G6n1+jhRtPxtX4gTnqvK4oBCrj93Et4dvFPucHgMZ8P27HdG6oRVszRXlvoDr693yg0dPLscmqHJgIAPG9myMCT7uJU7I1tXdlCxM2XkRZ2KfTBx9qVl9LBrcBoVCaNV3IQRSMvOQoMrBlfsqjc/CKlLSXZQlMTaUIb+Ey1CBfTzg3dwOjlamMFeU/YbieZ0vf8X8jWm/XML9tGzIZMCYHo0w+ZWmSMnM0+q4PS4oVAfAq/HpmPvrVY3jJgPg19JBq48KyM4rUF8G17W+MrWl1QPAqG6umOrXTKc3gABw8OBBCCHQrFkzxMTEYNq0aTAxMcGkb7di9r7rSArZANM8FU7+/l80sXsy3ywiIgIA8J///AfNmjXDtGnTIJfL0aJFC1y+p8KYKZ/gtswBBtaOwON8WP59GdEH1uH9aV/giGitca7081TiyPUnH0twNOoB8p4KPi0cLdHPU4nX2jjCyebJx1YU/RttaG2KP64lFZuXE/iqB1zqmaOyHuXkY9uZu9hwKhYJquKfdqzvUM+wo4OaFnYqM0pQUq2BDJjg7Y4byRk4dztV/TkvT1Namfzz2TXWSMnMw8qjMRr7qIr5OVJU2m33VennsDuYtftKqevlhgawt1JAaWWqHmVwtDaF8p/RhtO3HmL+/mt6fbfsWs8MS4e2RQcX/Y4AFhYKbDgVi8UHo5D3uBCmxobIyS9QX7Kd4OMOz4bWJcyXeTIyU9rt7k8zMpDB3tIESuuiUZknIzJKK1M4/vPf7PzH6L7oaKVHZp7X+ZKek495+67hl3P3AAD2lgokP8qF+Od3HvByE7RsYIWEf45T/D/HLyEtG0mPclGgTQKspSp66XDHjh0IDAzEvXv3YGNjg8GDB+PLL7+ElZUVLt9Twbv/UKQ/iEeT95dg6VBP+LV0KHEk0l7phJfn7kDorYdIPfEjsq6fgMhIgZmZKVq1aI4JEyZg2LBhZZ4r6Tn5OHQ1CfsvxePP6L/Vd4ACQDtnazSwNsVvlxOKBfmieTn6uHT6rPyCQnx96AbWHLtZbJ2+LtcCDDs6qWlhB6jcu77yau+nZePcnVScu52Cc3GpuPbP5Y/SyGTAXzN78Zu4a6iSAi4A1K+jwMPMXK1GKp7VwNoERoblB9vHBYW4n6YZnmUAQqb0RCMthtIrKjrpEQK2nkdUYobOtfUtFLA1lyPymU/NNpABe/y7oaXSSqt5Rc97JE8f/riaiOm/XEJadr5OdUUB0LaOXH3ptIgMT8KSpWn5lz7Ss/Ox8mhMsZEhbeorU1tafVVeav47Ixf+W84j7J+RyI97NcGwTk64k5KFhtamCItNwfd/3sKNpCfnsJGBDP08lfjgxUZooaz436HUzDwEX03EvovxOH3rYan//gN7e+A/Os6h09XzuLzPsKODmhh2gMq969OlNjP3MS7eTUP4nVQcjkzCpWdezAD9JnHSv9L+8OYXFCIpPUc90vHsiEdcSpZ6KFufnsf58mf0A7yz/kyx5U51TeFub1FsnozSyhT2Vgr1JTV9hJXqGMmrrOArCRj70/liyxvbmsPd3kLjmDlam6CBtSls6yjUfxQre9yq8o1cVdfrKr+gEF8eiMSmv26Xus3Tn6T+7N1flZX8KAerj94ssf3n9Zpe1cecYUcHNTXsVIfnPdGW9Kcif3hLu+y59u325c4PA57MEfvwp/ManxXzvM4XfZyrtTGsVFZNOG7P641cVdRXxPo/b2H+gchiyz96qTHG9mys/iT1qlATXtOr8phr+/ebn7NDGqr7g/2o4iry6dGl/b59WzqWX/yPhdV0vujjXK3oJ27XZjXhuFWmvjrbrqjmpVyW6uFev0qDDlAzXtNrwr8zjuyAIzsl+Te+4/03q43vlmtC27UZj9vzI/XRlerEy1g6YNghIqKqVBsntNcGvIxFRERUQ1T3V+H82zHsEBERPQc1Ye7KvxU/IY6IiIgkjWGHiIiIJI1hh4iIiCSNYYeIiIgkjWGHiIiIJI1hh4iIiCSNYYeIiIgkjWGHiIiIJI1hh4iIiCSNYYeIiIgkjWGHiIiIJI1hh4iIiCSNYYeIiIgkjWGHiIiIJI1hh4iIiCSNYYeIiIgkjWGHiIiIJI1hh4iIiCSNYYeIiIgkjWGHiIiIJI1hh4iIiCSNYYeIiIgkjWGHiIiIJI1hh4iIiCSNYYeIiIgkTTJhZ9WqVXB1dYWJiQm6dOmCM2fOVHeXiIiIqAaQRNjZvn07Jk+ejLlz5+L8+fPw9PSEn58fkpOTq7trREREVM0kEXa+/vprfPDBB3j//ffRokULrF27FmZmZtiwYUN1d42IiIiqmVF1d6Cy8vLycO7cOQQGBqqXGRgYwMfHB6GhoSXW5ObmIjc3V/2zSqUCAKSnp1dtZ4mIiEhviv5uCyHK3K7Wh52///4bBQUFsLe311hub2+P69evl1gTFBSEzz//vNhyJyenKukjERERVZ1Hjx7Bysqq1PW1PuxURGBgICZPnqz+ubCwECkpKahXrx5kMpne2klPT4eTkxPu3r0LS0vL51rPtp9/25WtZ9v/rrYrW8+22XZtqa9s22URQuDRo0dQKpVlblfrw46trS0MDQ2RlJSksTwpKQkODg4l1igUCigUCo1l1tbWVdVFWFpaVuoXXJl6tv38265sPdv+d7Vd2Xq2zbZrS31l2y5NWSM6RWr9BGW5XI4OHTogJCREvaywsBAhISHw8vKqxp4RERFRTVDrR3YAYPLkyRg5ciQ6duyIzp0745tvvkFmZibef//96u4aERERVTNJhJ1hw4bhwYMHmDNnDhITE9G2bVsEBwcXm7T8vCkUCsydO7fYJbPnUc+2n3/bla1n2/+utitbz7bZdm2pr2zb+iAT5d2vRURERFSL1fo5O0RERERlYdghIiIiSWPYISIiIklj2CEiIiJJY9ipQqtWrYKrqytMTEzQpUsXnDlzRqu6EydOoF+/flAqlZDJZNizZ4/WbQYFBaFTp06wsLCAnZ0dBgwYgKioKK3r16xZgzZt2qg//MnLywu///671vVPW7hwIWQyGSZOnKjV9p999hlkMpnGw8PDQ+v27t+/j7fffhv16tWDqakpWrdujfDwcK1qXV1di7Utk8ng7+9fbm1BQQFmz54NNzc3mJqaonHjxpg/f36539XytEePHmHixIlwcXGBqakpunbtirNnzxbbrrxzQwiBOXPmwNHREaampvDx8UF0dLTW9bt27YKvr6/608QjIiK0bj8/Px8zZsxA69atYW5uDqVSiXfffRfx8fFatf3ZZ5/Bw8MD5ubmqFu3Lnx8fBAWFqZ13582duxYyGQyfPPNN1rVvvfee8V+971799ap7cjISLz++uuwsrKCubk5OnXqhLi4uHJrSzrvZDIZvvrqK63azsjIQEBAABo2bAhTU1P1lyFrU5uUlIT33nsPSqUSZmZm6N27t/p80ea1JCcnB/7+/qhXrx7q1KmDwYMHqz/gVZv6devW4aWXXoKlpSVkMhnS0tLU68qrT0lJwfjx49GsWTOYmprC2dkZH3/8MVQqlVZtf/jhh2jcuDFMTU1Rv3599O/fX/0VQ7q8jgoh0KdPH/Xx1ab2pZdeKvb7Hjt2rE5th4aGolevXjA3N4elpSV69OiBefPmlVl7+/btUs+3nTt3atV2YmIi3nnnHTg4OMDc3Bzt27fHf//7X61qb968iYEDB6J+/fqwtLTE0KFDi30gcFVh2Kki27dvx+TJkzF37lycP38enp6e8PPzQ3Jycrm1mZmZ8PT0xKpVq3Ru9/jx4/D398fp06dx6NAh5Ofnw9fXF5mZmVrVN2zYEAsXLsS5c+cQHh6OXr16oX///rh69apO/Th79iy+++47tGnTRqe6li1bIiEhQf04efKkVnWpqano1q0bjI2N8fvvv+PatWtYunQp6tatq3V/n2730KFDAIAhQ4aUW7to0SKsWbMGK1euRGRkJBYtWoTFixdjxYoVWrUNAP/5z39w6NAh/Pjjj7h8+TJ8fX3h4+OD+/fva2xX3rmxePFiLF++HGvXrkVYWBjMzc3h5+eHnJwcreozMzPRvXt3LFq0qNT1pdVnZWXh/PnzmD17Ns6fP49du3YhKioKr7/+ulZtN23aFCtXrsTly5dx8uRJuLq6wtfXFw8ePNCqvsju3btx+vRpjY+P16a2d+/eGufA1q1bta6/efMmunfvDg8PDxw7dgyXLl3C7NmzYWJiUm7t020mJCRgw4YNkMlkGDx4sFZtT548GcHBwfjpp58QGRmJiRMnIiAgAHv37i2zVgiBAQMG4NatW/j1119x4cIFuLi4wMfHB5mZmVq9lkyaNAn79u3Dzp07cfz4ccTHx2PQoEEAtHstysrKQu/evTFr1qxi/SuvPj4+HvHx8ViyZAmuXLmCTZs2ITg4GKNHj9aq7Q4dOmDjxo2IjIzEwYMHIYSAr68vCgoKdHod/eabbzS+Zkjb2g8++EDj97548WKt60NDQ9G7d2/4+vrizJkzOHv2LAICAnDy5Mkya52cnIqdb59//jnq1KmDPn36aNX2u+++i6ioKOzduxeXL1/GoEGDMHToUOzbt6/M2szMTPj6+kImk+HIkSM4deoU8vLy0K9fPxQWFhY7rnonqEp07txZ+Pv7q38uKCgQSqVSBAUF6bQfAGL37t0V7kdycrIAII4fP17hfdStW1f83//9n9bbP3r0SLi7u4tDhw6Jnj17igkTJmhVN3fuXOHp6VmhPs6YMUN07969QrUlmTBhgmjcuLEoLCwsd9u+ffuKUaNGaSwbNGiQGDFihFZtZWVlCUNDQ7F//36N5e3btxeffPJJqXXPnhuFhYXCwcFBfPXVV+plaWlpQqFQiK1bt5Zb/7TY2FgBQFy4cEHr9kty5swZAUDcuXNH51qVSiUAiMOHD2vd9r1790SDBg3ElStXhIuLi1i2bJlWtSNHjhT9+/cvsz9l1Q8bNky8/fbbFap9Vv/+/UWvXr20rm/ZsqWYN2+exrKSzp1na6OiogQAceXKFfWygoICUb9+ffH9998Xa/vZ15K0tDRhbGwsdu7cqd4mMjJSABChoaHl1j/t6NGjAoBITU0t8XmXV19kx44dQi6Xi/z8fJ1rL168KACImJgYrdu+cOGCaNCggUhISCj1d1tSrS6viyXVd+nSRXz66acVqn1W27Zti71+lVVvbm4uNm/erLGdjY1NsXPm2dqDBw8KAwMDoVKp1NukpaUJmUwmDh06VO5zqSyO7FSBvLw8nDt3Dj4+PuplBgYG8PHxQWho6HPti0qlAgDY2NjoXFtQUIBt27YhMzNTp6/e8Pf3R9++fTWev7aio6OhVCrRqFEjjBgxAnFxcVrV7d27Fx07dsSQIUNgZ2eHdu3a4fvvv9e5feDJ7++nn37CqFGjtPpi2K5duyIkJAQ3btwAAFy8eBEnT55Enz59tGrv8ePHKCgogImJicZyU1NTrUe2ACA2NhaJiYkax93KygpdunR57uddEZVKBZlMpvN3z+Xl5WHdunWwsrKCp6enVjWFhYV45513MG3aNLRs2VLnvh47dgx2dnZo1qwZPvroIzx8+FDrdg8cOICmTZvCz88PdnZ26NKli06Xn4skJSXhwIEDGD16tNY1Xbt2xd69e3H//n0IIXD06FHcuHEDvr6+Zdbl5uYCgMZ5Z2BgAIVCUeJ59+xryblz55Cfn69xvnl4eMDZ2bnE860yr0Xa1qtUKlhaWsLIyKjY8rJqMzMzsXHjRri5ucHJyUmrtrOysvDWW29h1apVpX4PY1ltb9myBba2tmjVqhUCAwORlZWlVX1ycjLCwsJgZ2eHrl27wt7eHj179tTqd/asc+fOISIiotTzraT6rl27Yvv27UhJSUFhYSG2bduGnJwcvPTSS2XW5ubmQiaTaXywoImJCQwMDHR6nauwKo9T/0L3798XAMRff/2lsXzatGmic+fOOu0LlRjZKSgoEH379hXdunXTqe7SpUvC3NxcGBoaCisrK3HgwAGta7du3SpatWolsrOzhRC6vYP57bffxI4dO8TFixdFcHCw8PLyEs7OziI9Pb3cWoVCIRQKhQgMDBTnz58X3333nTAxMRGbNm3Suu9Ftm/fLgwNDcX9+/e12r6goEDMmDFDyGQyYWRkJGQymViwYIFObXp5eYmePXuK+/fvi8ePH4sff/xRGBgYiKZNm5Za8+y5cerUKQFAxMfHa2w3ZMgQMXTo0HLrn6aPkZ3s7GzRvn178dZbb2ldu2/fPmFubi5kMplQKpXizJkzWre9YMEC8corr6hH43QZ2dm6dav49ddfxaVLl8Tu3btF8+bNRadOncTjx4/LrS96V29mZia+/vprceHCBREUFCRkMpk4duyYVs+7yKJFi0TdunXV/3606XtOTo549913BQBhZGQk5HK5+OGHH8qtzcvLE87OzmLIkCEiJSVF5ObmioULFwoAwtfXV6O2pNeSLVu2CLlcXqydTp06ienTp5db/7TyRna0eS178OCBcHZ2FrNmzdK6dtWqVcLc3FwAEM2aNStxVKe0+jFjxojRo0erfy7pd1Na7XfffSeCg4PFpUuXxE8//SQaNGggBg4cqFXboaGhAoCwsbERGzZsEOfPnxcTJ04Ucrlc3LhxQ6vnXeSjjz4SzZs3L3FdafWpqanC19dXfb5ZWlqKgwcPllubnJwsLC0txYQJE0RmZqbIyMgQAQEBAoAYM2ZMqX3UF4adKlBTws7YsWOFi4uLuHv3rk51ubm5Ijo6WoSHh4uZM2cKW1tbcfXq1XLr4uLihJ2dnbh48aJ6mS5h51mpqanC0tJSq0toxsbGwsvLS2PZ+PHjxQsvvKBzu76+vuK1117TevutW7eKhg0biq1bt4pLly6JzZs3CxsbG52CVkxMjOjRo4cAIAwNDUWnTp3EiBEjhIeHR6k1NTns5OXliX79+ol27dppDFuXV5uRkSGio6NFaGioGDVqlHB1dRVJSUnl1oeHhwt7e3uNgKpL2HnWzZs3tb6EVvTv/c0339TYrl+/fmL48OE6td2sWTMREBBQ6vqS6r/66ivRtGlTsXfvXnHx4kWxYsUKUadOnWKXBkqqDQ8PF56enurzzs/PT/Tp00f07t1bY7uSXkt0CTvlvRaVF3bKq1epVKJz586id+/eIi8vT+vatLQ0cePGDXH8+HHRr18/0b59+2JBs6T6X3/9VTRp0kQ8evRIvayk46vta3BISEiJl9BKqi/6dx4YGKixbevWrcXMmTO1bjsrK0tYWVmJJUuWlLi+tPqAgADRuXNncfjwYRERESE+++wzYWVlJS5dulRu7cGDB0WjRo2ETCYThoaG4u233xbt27cXY8eOLePo6AfDThXIzc0VhoaGxU78d999V7z++us67auiYcff3180bNhQ3Lp1S+faZ3l7e2uVvHfv3q1+0Sx6AFCf2CW9Sy5Px44dNf4Bl8bZ2VnjXZYQQqxevVoolUqd2rt9+7YwMDAQe/bs0bqmYcOGYuXKlRrL5s+fL5o1a6ZT20I8+WNfFFaGDh0qXn311VK3ffbcKPoD/WxA6dGjh/j444/LrX9aZcJOXl6eGDBggGjTpo34+++/dap9VpMmTUocJXu2ftmyZerz7Olzz8DAQLi4uFSobVtbW7F27dpy287NzRVGRkZi/vz5GttNnz5ddO3aVeu2T5w4IQCIiIiIUvv0bH1WVpYwNjYuNt9r9OjRws/PT+u209LSRHJyshDiyXzDcePGqdeV9lpS9Af62YDi7Owsvv7663Lrn1ZW2CmvPj09XXh5eQlvb+9iQUWX18Hc3FxhZmYmfv7553LrJ0yYUOr51rNnT53bzsjIEABEcHBwuW3funVLABA//vijxvKhQ4eqR1G1aXvz5s3C2NhY/Xt/Wmn1MTExxeZ5CfHkb8SHH36oddsPHjxQ/67t7e3F4sWLS91WXzhnpwrI5XJ06NABISEh6mWFhYUICQnRae5LRQghEBAQgN27d+PIkSNwc3Or9D4LCwvV1/fL4u3tjcuXLyMiIkL96NixI0aMGIGIiAgYGhrq1G5GRgZu3rwJR0fHcrft1q1bsdscb9y4ARcXF53a3LhxI+zs7NC3b1+ta7KysmBgoPlPydDQsEJ3GJibm8PR0RGpqak4ePAg+vfvr3Wtm5sbHBwcNM679PR0hIWFVfl5VyQ/Px9Dhw5FdHQ0Dh8+jHr16lVqf9qee++88w4uXbqkce4plUpMmzYNBw8e1Lnde/fu4eHDh1qde3K5HJ06dar0+bd+/Xp06NBB6zlKwJPjnZ+fX+nzz8rKCvXr10d0dDTCw8PRv3//cl9LOnToAGNjY43zLSoqCnFxcfDy8qr0a5E29enp6fD19YVcLsfevXvV848q0rZ48uYfubm55dbPnDmz2PkGAMuWLcOGDRt0bruo3tHRsdy2XV1doVQqSzzfnJ2dtW57/fr1eP3111G/fn2NY1BWfdG8opLOt4KCAq3btrW1hbW1NY4cOYLk5GT1HZtVqsrj1L/Utm3bhEKhEJs2bRLXrl0TY8aMEdbW1iIxMbHc2kePHokLFy6ICxcuCADqeQDP3tFSko8++khYWVmJY8eOiYSEBPUjKytLq37PnDlTHD9+XMTGxopLly6JmTNnCplMJv744w+t6p+ly2WsKVOmiGPHjonY2Fhx6tQp4ePjI2xtbUt85/GsM2fOCCMjI/Hll1+K6OhosWXLFmFmZiZ++uknrftaUFAgnJ2dxYwZM7SuEeLJnTwNGjQQ+/fvF7GxsWLXrl3C1ta22FB+WYKDg8Xvv/8ubt26Jf744w/h6ekpunTpUmxIvrxzY+HChcLa2lo9/6R///7Czc1N/Y63vPqHDx+KCxcuiAMHDggAYtu2beLChQsiISGh3Pq8vDzx+uuvi4YNG4qIiAiN8y83N7fM2oyMDBEYGChCQ0PF7du3RXh4uHj//feFQqFQv4vU9d/F05exyqp99OiRmDp1qggNDRWxsbHi8OHDon379sLd3V3k5ORo1fauXbuEsbGxWLdunYiOjhYrVqwQhoaG4s8//9Sq3yqVSpiZmYk1a9YUex7l1ffs2VO0bNlSHD16VNy6dUts3LhRmJiYiNWrV5dbu2PHDnH06FFx8+ZNsWfPHuHi4iIGDRokhNDutWTs2LHC2dlZHDlyRISHhwsvLy/15WRt6hMSEsSFCxfE999/LwCIEydOiAsXLoiHDx+WW69SqUSXLl1E69atRUxMjMY2Y8eOLbP25s2bYsGCBSI8PFzcuXNHnDp1SvTr10/Y2NiIpKSkCr2O4p+Rs/JqY2JixLx580R4eLiIjY0Vv/76q2jUqJHo0aOH1sdt2bJlwtLSUuzcuVNER0eLTz/9VJiYmIi33npLq35HR0cLmUwmfv/9d43l5bWdl5cnmjRpIl588UURFhYmYmJixJIlS4RMJhOvvvpquW1v2LBBhIaGipiYGPHjjz8KGxsbMXny5FKPqT4x7FShFStWCGdnZyGXy0Xnzp3F6dOntaorGtJ99jFy5Mhya0uqAyA2btyoVdujRo0SLi4uQi6Xi/r16wtvb+8KBx0hdAs7w4YNE46OjkIul4sGDRqIYcOGlThhsDT79u0TrVq1EgqFQnh4eIh169bp1NeDBw8KACIqKkqnuvT0dDFhwgTh7OwsTExMRKNGjcQnn3wicnNztd7H9u3bRaNGjYRcLhcODg7C399fpKWlFduuvHOjsLBQzJ49W9jb2wuFQiG8vb01nk959Rs3bixx/dy5c8utL7r0VdLj6NGjZdZmZ2eLgQMHCqVSKeRyuXB0dBSvv/66xgRlXf9dPB12yqrNysoSvr6+on79+sLY2Fi4uLiIDz74QOONiTZtr1+/XjRp0kSYmJgIT09P9aVQbWq/++47YWpqWqHfeUJCgnjvvfeEUqkUJiYmolmzZmLp0qWisLCw3Npvv/1WNGzYUBgbGwtnZ2fx6aefqs9bbV5LsrOzxbhx40TdunWFmZmZGDhwoDoYa1M/d+7cUrcpr76051bWo6j2/v37ok+fPsLOzk4YGxuLhg0birfeektcv35d674/qyjslFcbFxcnevToIWxsbIRCoRBNmjQR06ZNU89t07btoKAg0bBhQ2FmZia8vLzEn3/+qXVtYGCgcHJyEgUFBcWeQ3n1N27cEIMGDRJ2dnbCzMxMtGnTRmzevFmr2hkzZgh7e3thbGws3N3d1efp8yD75wkSERERSRLn7BAREZGkMewQERGRpDHsEBERkaQx7BAREZGkMewQERGRpDHsEBERkaQx7BAREZGkMewQET3j2LFjkMlkSEtLq+6uEJEeMOwQERGRpDHsEBERkaQx7BBRjVNYWIigoCC4ubnB1NQUnp6e+OWXXwD87xLTgQMH0KZNG5iYmOCFF17AlStXNPbx3//+Fy1btoRCoYCrqyuWLl2qsT43NxczZsyAk5MTFAoFmjRpgvXr12tsc+7cOXTs2BFmZmbo2rVrsW+aJqLagWGHiGqcoKAgbN68GWvXrsXVq1cxadIkvP322zh+/Lh6m2nTpmHp0qU4e/Ys6tevj379+iE/Px/Ak5AydOhQDB8+HJcvX8Znn32G2bNnY9OmTer6d999F1u3bsXy5csRGRmJ7777DnXq1NHoxyeffIKlS5ciPDwcRkZGGDVq1HN5/kSkX/wiUCKqUXJzc2FjY4PDhw/Dy8tLvfw///kPsrKyMGbMGLz88svYtm0bhg0bBgBISUlBw4YNsWnTJgwdOhQjRozAgwcP8Mcff6jrp0+fjgMHDuDq1au4ceMGmjVrhkOHDsHHx6dYH44dO4aXX34Zhw8fhre3NwDgt99+Q9++fZGdnQ0TE5MqPgpEpE8c2SGiGiUmJgZZWVl45ZVXUKdOHfVj8+bNuHnzpnq7p4OQjY0NmjVrhsjISABAZGQkunXrprHfbt26ITo6GgUFBYiIiIChoSF69uxZZl/atGmj/n9HR0cAQHJycqWfIxE9X0bV3QEioqdlZGQAAA4cOIAGDRporFMoFBqBp6JMTU212s7Y2Fj9/zKZDMCT+UREVLtwZIeIapQWLVpAoVAgLi4OTZo00Xg4OTmptzt9+rT6/1NTU3Hjxg00b94cANC8eXOcOnVKY7+nTp1C06ZNYWhoiNatW6OwsFBjDhARSRdHdoioRrGwsMDUqVMxadIkFBYWonv37lCpVDh16hQsLS3h4uICAJg3bx7q1asHe3t7fPLJJ7C1tcWAAQMAAFOmTEGnTp0wf/58DBs2DKGhoVi5ciVWr14NAHB1dcXIkSMxatQoLF++HJ6enrhz5w6Sk5MxdOjQ6nrqRFRFGHaIqMaZP38+6tevj6CgINy6dQvW1tZo3749Zs2apb6MtHDhQkyYMAHR0dFo27Yt9u3bB7lcDgBo3749duzYgTlz5mD+/PlwdHTEvHnz8N5776nbWLNmDWbNmoVx48bh4cOHcHZ2xqxZs6rj6RJRFePdWERUqxTdKZWamgpra+vq7g4R1QKcs0NERESSxrBDREREksbLWERERCRpHNkhIiIiSWPYISIiIklj2CEiIiJJY9ghIiIiSWPYISIiIklj2CEiIiJJY9ghIiIiSWPYISIiIklj2CEiIiJJ+3+XrUfWLLtZNwAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plt.plot(np.arange(len(epochs_x)), epochs_acc, marker = '.')\n", + "plt.xlabel('epoch')\n", + "plt.ylabel('test accuracy')\n", + "plt.ylim(0, 100)\n", + "plt.xticks(np.arange(len(epochs_x)))\n", + "for i, txt in enumerate(epochs_acc):\n", + " if i%5 ==0 or i == epochs-1:\n", + " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "# with open(f'./architectures_results/{achitecture}-Training_Validation-TM.npy', 'wb') as f:\n", + "# np.save(f, np.array(epochs_x))\n", + "# np.save(f, np.array(epochs_y))\n", + "# np.save(f, np.array(epochs_acc))" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "speck-rescnn", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.0" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN1.py b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN1.py index e557f437..b8f5b838 100644 --- a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN1.py +++ b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN1.py @@ -28,19 +28,19 @@ def __init__(self, input_size, nb_classes, batch_size, surrogate_fn, min_v_mem=- flat_s = SCNN.get_flatten_size(input_size) self.fc1 = nn.Linear(flat_s, 100, bias=False) - self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.iaf1_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) self.fc2 = nn.Linear(100, 100, bias=False) - self.iaf5 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.iaf2_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) self.fc3 = nn.Linear(100, 100, bias=False) - self.iaf6 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.iaf3_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.fc4 = nn.Linear(100, 64, bias=False) - self.iaf7 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.fc4 = nn.Linear(100, 100, bias=False) + self.iaf4_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.fc5 = nn.Linear(64, nb_classes, bias=False) - self.iaf8 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.fc5 = nn.Linear(100, nb_classes, bias=False) + self.iaf5_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) # skip @@ -97,42 +97,56 @@ def init_weights(self): if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear): nn.init.xavier_normal_(layer.weight.data) + def rescale_conv_weights(self, rescale_fn, lambda_): + rescale_fn(self.conv2, [(2, 2)], lambda_) + rescale_fn(self.conv3, [(2, 2), (4, 4)], lambda_) + rescale_fn(self.conv4, [(2, 2)], lambda_) + + def forward(self, x): - + # conv 1 con1_out = self.conv1(x) iaf1_out = self.iaf1(con1_out) pool1_out = self.pool1(iaf1_out) pool1a_out = self.pool1a(iaf1_out) + # conv 2 conv2_out = self.conv2(pool1_out) iaf2_out = self.iaf2(conv2_out) pool2_out = self.pool2(iaf2_out) merge1_out = self.merge1(pool1a_out, pool2_out) + # conv 3 conv3_out = self.conv3(merge1_out) iaf3_out = self.iaf3(conv3_out) pool3_out = self.pool3(iaf3_out) + # conv 4 conv4_out = self.conv4(pool3_out) iaf4_out = self.iaf4(conv4_out) pool4_out = self.pool4(iaf4_out) flat_out = self.flat(pool4_out) + # fc 1 fc1_out = self.fc1(flat_out) - iaf4_out = self.iaf4(fc1_out) + iaf1_fc_out = self.iaf1_fc(fc1_out) - fc2_out = self.fc2(iaf4_out) - iaf5_out = self.iaf5(fc2_out) + # fc 2 + fc2_out = self.fc2(iaf1_fc_out) + iaf2_fc_out = self.iaf2_fc(fc2_out) - fc3_out = self.fc3(iaf5_out) - iaf6_out = self.iaf6(fc3_out) + # fc 3 + fc3_out = self.fc3(iaf2_fc_out) + iaf3_fc_out = self.iaf3_fc(fc3_out) - fc4_out = self.fc4(iaf6_out) - iaf7_out = self.iaf7(fc4_out) + # fc 4 + fc4_out = self.fc4(iaf3_fc_out) + iaf4_fc_out = self.iaf4_fc(fc4_out) - fc5_out = self.fc5(iaf7_out) - iaf8_out = self.iaf8(fc5_out) + # fc 5 + fc5_out = self.fc5(iaf4_fc_out) + iaf5_fc_out = self.iaf5_fc(fc5_out) - return iaf8_out \ No newline at end of file + return iaf5_fc_out \ No newline at end of file diff --git a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN2.py b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN2.py index ada4e6be..21ada5a4 100644 --- a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN2.py +++ b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN2.py @@ -29,19 +29,19 @@ def __init__(self, input_size, nb_classes, batch_size, surrogate_fn, min_v_mem=- flat_s = SCNN.get_flatten_size(input_size) self.fc1 = nn.Linear(flat_s, 100, bias=False) - self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.iaf1_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) self.fc2 = nn.Linear(100, 100, bias=False) - self.iaf5 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.iaf2_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) self.fc3 = nn.Linear(100, 100, bias=False) - self.iaf6 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.iaf3_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.fc4 = nn.Linear(100, 64, bias=False) - self.iaf7 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.fc4 = nn.Linear(100, 100, bias=False) + self.iaf4_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.fc5 = nn.Linear(64, nb_classes, bias=False) - self.iaf8 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.fc5 = nn.Linear(100, nb_classes, bias=False) + self.iaf5_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) # skip @@ -100,23 +100,23 @@ def init_weights(self): nn.init.xavier_normal_(layer.weight.data) def forward(self, x): - + # conv 1 con1_out = self.conv1(x) iaf1_out = self.iaf1(con1_out) pool1_out = self.pool1(iaf1_out) pool1a_out = self.pool1a(iaf1_out) - + # conv 2 conv2_out = self.conv2(pool1_out) iaf2_out = self.iaf2(conv2_out) pool2_out = self.pool2(iaf2_out) merge1_out = self.merge1(pool1a_out, pool2_out) - + # conv 3 conv3_out = self.conv3(merge1_out) iaf3_out = self.iaf3(conv3_out) pool3_out = self.pool3(iaf3_out) pool3a_out = self.pool3a(iaf3_out) - + # conv 4 conv4_out = self.conv4(pool3_out) iaf4_out = self.iaf4(conv4_out) pool4_out = self.pool4(iaf4_out) @@ -124,20 +124,20 @@ def forward(self, x): merge2_out = self.merge2(pool3a_out, pool4_out) flat_out = self.flat(merge2_out) - + # fc 1 fc1_out = self.fc1(flat_out) - iaf4_out = self.iaf4(fc1_out) - - fc2_out = self.fc2(iaf4_out) - iaf5_out = self.iaf5(fc2_out) - - fc3_out = self.fc3(iaf5_out) - iaf6_out = self.iaf6(fc3_out) - - fc4_out = self.fc4(iaf6_out) - iaf7_out = self.iaf7(fc4_out) - - fc5_out = self.fc5(iaf7_out) - iaf8_out = self.iaf8(fc5_out) - - return iaf8_out \ No newline at end of file + iaf1_fc_out = self.iaf1_fc(fc1_out) + # fc 2 + fc2_out = self.fc2(iaf1_fc_out) + iaf2_fc_out = self.iaf2_fc(fc2_out) + # fc 3 + fc3_out = self.fc3(iaf2_fc_out) + iaf3_fc_out = self.iaf3_fc(fc3_out) + # fc 4 + fc4_out = self.fc4(iaf3_fc_out) + iaf4_fc_out = self.iaf4_fc(fc4_out) + # fc 5 + fc5_out = self.fc5(iaf4_fc_out) + iaf5_fc_out = self.iaf5_fc(fc5_out) + + return iaf5_fc_out \ No newline at end of file diff --git a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN3.py b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN3.py index f861131c..81479e94 100644 --- a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN3.py +++ b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN3.py @@ -29,19 +29,19 @@ def __init__(self, input_size, nb_classes, batch_size, surrogate_fn, min_v_mem=- flat_s = SCNN.get_flatten_size(input_size) self.fc1 = nn.Linear(flat_s, 100, bias=False) - self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.iaf1_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) self.fc2 = nn.Linear(100, 100, bias=False) - self.iaf5 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.iaf2_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) self.fc3 = nn.Linear(100, 100, bias=False) - self.iaf6 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.iaf3_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.fc4 = nn.Linear(100, 64, bias=False) - self.iaf7 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.fc4 = nn.Linear(100, 100, bias=False) + self.iaf4_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.fc5 = nn.Linear(64, nb_classes, bias=False) - self.iaf8 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.fc5 = nn.Linear(100, nb_classes, bias=False) + self.iaf5_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) # skip @@ -101,23 +101,23 @@ def init_weights(self): nn.init.xavier_normal_(layer.weight.data) def forward(self, x): - + # conv 1 con1_out = self.conv1(x) iaf1_out = self.iaf1(con1_out) pool1_out = self.pool1(iaf1_out) pool1a_out = self.pool1a(iaf1_out) - + # conv 2 conv2_out = self.conv2(pool1_out) iaf2_out = self.iaf2(conv2_out) pool2_out = self.pool2(iaf2_out) merge1_out = self.merge1(pool1a_out, pool2_out) - + # conv 3 conv3_out = self.conv3(merge1_out) iaf3_out = self.iaf3(conv3_out) pool3_out = self.pool3(iaf3_out) pool3a_out = self.pool3a(iaf3_out) - + # conv 4 conv4_out = self.conv4(pool3_out) iaf4_out = self.iaf4(conv4_out) pool4_out = self.pool4(iaf4_out) @@ -125,22 +125,22 @@ def forward(self, x): merge2_out = self.merge2(pool3a_out, pool4_out) flat_out = self.flat(merge2_out) - + # fc 1 fc1_out = self.fc1(flat_out) - iaf4_out = self.iaf4(fc1_out) - - fc2_out = self.fc2(iaf4_out) - iaf5_out = self.iaf5(fc2_out) - - merge3_out = self.merge3(iaf4_out, iaf5_out) + iaf1_fc_out = self.iaf1_fc(fc1_out) + # fc 2 + fc2_out = self.fc2(iaf1_fc_out) + iaf2_fc_out = self.iaf2_fc(fc2_out) + merge3_out = self.merge3(iaf1_fc_out, iaf2_fc_out) + # fc 3 fc3_out = self.fc3(merge3_out) - iaf6_out = self.iaf6(fc3_out) - - fc4_out = self.fc4(iaf6_out) - iaf7_out = self.iaf7(fc4_out) - - fc5_out = self.fc5(iaf7_out) - iaf8_out = self.iaf8(fc5_out) - - return iaf8_out \ No newline at end of file + iaf3_fc_out = self.iaf3_fc(fc3_out) + # fc 4 + fc4_out = self.fc4(iaf3_fc_out) + iaf4_fc_out = self.iaf4_fc(fc4_out) + # fc 5 + fc5_out = self.fc5(iaf4_fc_out) + iaf5_fc_out = self.iaf5_fc(fc5_out) + + return iaf5_fc_out \ No newline at end of file diff --git a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN4.py b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN4.py index 620b146c..535c04f8 100644 --- a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN4.py +++ b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN4.py @@ -29,19 +29,19 @@ def __init__(self, input_size, nb_classes, batch_size, surrogate_fn, min_v_mem=- flat_s = SCNN.get_flatten_size(input_size) self.fc1 = nn.Linear(flat_s, 100, bias=False) - self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.iaf1_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) self.fc2 = nn.Linear(100, 100, bias=False) - self.iaf5 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.iaf2_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) self.fc3 = nn.Linear(100, 100, bias=False) - self.iaf6 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.iaf3_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.fc4 = nn.Linear(100, 64, bias=False) - self.iaf7 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.fc4 = nn.Linear(100, 100, bias=False) + self.iaf4_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.fc5 = nn.Linear(64, nb_classes, bias=False) - self.iaf8 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.fc5 = nn.Linear(100, nb_classes, bias=False) + self.iaf5_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) # skip @@ -102,23 +102,23 @@ def init_weights(self): nn.init.xavier_normal_(layer.weight.data) def forward(self, x): - + # conv 1 con1_out = self.conv1(x) iaf1_out = self.iaf1(con1_out) pool1_out = self.pool1(iaf1_out) pool1a_out = self.pool1a(iaf1_out) - + # conv 2 conv2_out = self.conv2(pool1_out) iaf2_out = self.iaf2(conv2_out) pool2_out = self.pool2(iaf2_out) merge1_out = self.merge1(pool1a_out, pool2_out) - + # conv 3 conv3_out = self.conv3(merge1_out) iaf3_out = self.iaf3(conv3_out) pool3_out = self.pool3(iaf3_out) pool3a_out = self.pool3a(iaf3_out) - + # conv 4 conv4_out = self.conv4(pool3_out) iaf4_out = self.iaf4(conv4_out) pool4_out = self.pool4(iaf4_out) @@ -126,24 +126,24 @@ def forward(self, x): merge2_out = self.merge2(pool3a_out, pool4_out) flat_out = self.flat(merge2_out) - + # fc 1 fc1_out = self.fc1(flat_out) - iaf4_out = self.iaf4(fc1_out) - - fc2_out = self.fc2(iaf4_out) - iaf5_out = self.iaf5(fc2_out) - - merge3_out = self.merge3(iaf4_out, iaf5_out) + iaf1_fc_out = self.iaf1_fc(fc1_out) + # fc 2 + fc2_out = self.fc2(iaf1_fc_out) + iaf2_fc_out = self.iaf2_fc(fc2_out) + merge3_out = self.merge3(iaf1_fc_out, iaf2_fc_out) + # fc 3 fc3_out = self.fc3(merge3_out) - iaf6_out = self.iaf6(fc3_out) - - fc4_out = self.fc4(iaf6_out) - iaf7_out = self.iaf7(fc4_out) - - merge4_out = self.merge4(iaf6_out, iaf7_out) + iaf3_fc_out = self.iaf3_fc(fc3_out) + # fc 4 + fc4_out = self.fc4(iaf3_fc_out) + iaf4_fc_out = self.iaf4_fc(fc4_out) + merge4_out = self.merge4(iaf3_fc_out, iaf4_fc_out) + # fc 5 fc5_out = self.fc5(merge4_out) - iaf8_out = self.iaf8(fc5_out) + iaf5_fc_out = self.iaf5_fc(fc5_out) - return iaf8_out \ No newline at end of file + return iaf5_fc_out \ No newline at end of file diff --git a/tests/test_nonsequential/utils/weight_initialization.py b/tests/test_nonsequential/utils/weight_initialization.py new file mode 100644 index 00000000..8c69f2a3 --- /dev/null +++ b/tests/test_nonsequential/utils/weight_initialization.py @@ -0,0 +1,25 @@ +import torch.nn as nn +import numpy as np + +def rescale_method_1(conv_layer: nn.Conv2d, input_pool_kernel: list, lambda_: float = 1): + """ + The `method 1` will use the average of the computed rescaling factor for each pooling layer + feeding into `conv_layer` (if there are more than one) to rescale its weights. + + Arguments + --------- + input_pool_kernel (list): the kernels of all pooling layers feeding input to `conv_layer`. + lambda_ (float): scales the computed re-scaling factor. If the outputs of the pooling are too small + the rescaling might lead to vanishing gradients, so we can try to control that by scaling it by + lambda. + """ + rescaling_factors = [] + + for kernel in input_pool_kernel: + rescaling_factors.append(kernel[0]*kernel[1]) + + rescaling_factor = np.mean(rescaling_factors)*lambda_ + + print(f'recaling factor: {rescaling_factor} (computed using {len(input_pool_kernel)} kernels and lambda {lambda_})') + + conv_layer.weight.data /= rescaling_factor \ No newline at end of file From 711fe53b57cdd099094e28eab09ecc57b96a797e Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Fri, 3 May 2024 17:38:09 +0200 Subject: [PATCH 072/379] architectures to be tested --- tests/test_nonsequential/using_SumPool2d/models/ResSCNN7.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN7.py b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN7.py index 084e9675..749e55eb 100644 --- a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN7.py +++ b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN7.py @@ -47,7 +47,6 @@ def __init__(self, input_size, nb_classes, batch_size, surrogate_fn, min_v_mem=- self.merge1a = sl.Merge() self.merge1b = sl.Merge() - self.merge1c = sl.Merge() @staticmethod def get_flatten_size(input_size): From 18aa911a83fba86f7d145c63a38630b9174a8657 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Fri, 3 May 2024 23:39:00 +0200 Subject: [PATCH 073/379] simple Gaussian Search for training HPO --- .../HPO_GAUSSIAN_SEARCH/GS_utils.py | 54 +++++ .../HPO_GAUSSIAN_SEARCH/main.py | 202 ++++++++++++++++++ .../HPO_GAUSSIAN_SEARCH/network.py | 83 +++++++ 3 files changed, 339 insertions(+) create mode 100644 tests/test_nonsequential/using_SumPool2d/HPO_GAUSSIAN_SEARCH/GS_utils.py create mode 100644 tests/test_nonsequential/using_SumPool2d/HPO_GAUSSIAN_SEARCH/main.py create mode 100644 tests/test_nonsequential/using_SumPool2d/HPO_GAUSSIAN_SEARCH/network.py diff --git a/tests/test_nonsequential/using_SumPool2d/HPO_GAUSSIAN_SEARCH/GS_utils.py b/tests/test_nonsequential/using_SumPool2d/HPO_GAUSSIAN_SEARCH/GS_utils.py new file mode 100644 index 00000000..2ef3c4b2 --- /dev/null +++ b/tests/test_nonsequential/using_SumPool2d/HPO_GAUSSIAN_SEARCH/GS_utils.py @@ -0,0 +1,54 @@ +import numpy as np + +def define_probabilistic_model(hyperparams): + prob_model = {} + + for key, data in hyperparams.items(): + prob_model[key] = {'mu': data['value'], 'sigma': data['sigma']} + + return prob_model + +def update_probabilistic_model(max_iter, cur_iter, prob_model, hyperparams_new): + + sigma_scale = update_sigma_scale(max_iter, cur_iter) + + for key, value in hyperparams_new.items(): + prob_model[key]['mu'] = value + prob_model[key]['sigma'] = prob_model[key]['sigma']*sigma_scale + +def update_sigma_scale(max_iter, cur_iter): + _ = np.round(1-(cur_iter/max_iter), 2) + + if _ > 0: + return _ + else: + return 0.01 + +def sample_values_to_eval(iteration, prob_model, hyperparams, nb_samples: int = 5): + new_values = {} + np.random.seed(iteration) + + for hp, data in prob_model.items(): + sampled_values = np.round(np.random.normal(data['mu'], data['sigma'], nb_samples), hyperparams[hp]['precision']) + + fixed_sampled_values = [] + + for val in sampled_values: + if val < hyperparams[hp]['min']: + fixed_sampled_values.append(hyperparams[hp]['min']) + elif val > hyperparams[hp]['max']: + fixed_sampled_values.append(hyperparams[hp]['max']) + else: + fixed_sampled_values.append(val) + + new_values[hp] = fixed_sampled_values + + return new_values + +def get_sampled_set(sampled_values, i): + sampled_set = {} + + for key, val in sampled_values.items(): + sampled_set[key] = val[i] + + return sampled_set \ No newline at end of file diff --git a/tests/test_nonsequential/using_SumPool2d/HPO_GAUSSIAN_SEARCH/main.py b/tests/test_nonsequential/using_SumPool2d/HPO_GAUSSIAN_SEARCH/main.py new file mode 100644 index 00000000..124b86b0 --- /dev/null +++ b/tests/test_nonsequential/using_SumPool2d/HPO_GAUSSIAN_SEARCH/main.py @@ -0,0 +1,202 @@ +from GS_utils import * +import torch, tonic, sys, random +import numpy as np +from torch.utils.data import DataLoader +from sinabs.activation.surrogate_gradient_fn import PeriodicExponential +from torch.nn import CrossEntropyLoss +from torch.optim import Adam +import pandas as pd +from tqdm import tqdm + +sys.path.append('../../utils') +sys.path.append('../models') + +from network import SCNN_GS +from train_test_fn import training_loop_no_tqdm, load_dataset, split_train_validation +from weight_initialization import rescale_method_1 + +if torch.cuda.is_available(): + device = torch.device('cuda:0') + print('device: ', torch.cuda.get_device_name(0)) +else: + device = torch.device('cpu') + +torch.backends.cudnn.enabled = False +torch.backends.cudnn.deterministic = True +random.seed(1) +torch.manual_seed(1) +torch.cuda.manual_seed(1) + +### Initialization #################################################### + +max_iter = 100 +nb_samples = 5 +batch_size = 8 +num_workers = 8 +validation_ratio = 0.2 +n_time_steps = 50 +epochs = 40 +validation_rand_seed = 1 +output_csv = 'gaussian_search_history.csv' +params_set_history = {} + +loss_fn = CrossEntropyLoss() + +hyperparams = { + 'learning_rate': {'value': 0.001, 'min': 0.00008, 'max': 0.08, 'precision': 6, 'sigma': 0.0001}, + 'spike_threshold': {'value': 2.75, 'min': 0.5, 'max': 5.0, 'precision': 2, 'sigma': 1.0}, + 'mem_v_min': {'value': -2.5, 'min': -5.0, 'max': 0.0, 'precision': 2, 'sigma': 1.0}, + 'grad_scale': {'value': 1.55, 'min': 0.1, 'max': 3.0, 'precision': 2, 'sigma': 0.5}, + 'grad_width': {'value': 1.55, 'min': 0.1, 'max': 3.0, 'precision': 2, 'sigma': 0.5}, + 'w_rescale_lambda': {'value': 0.5, 'min': 0.1, 'max': 1.0, 'precision': 3, 'sigma': 0.1667}, +} + +prob_model = define_probabilistic_model(hyperparams) + +with open('fixed_parameters.txt', 'w') as file: + file.write(f'max_iter: {max_iter}\n') + file.write(f'nb_samples: {nb_samples}\n') + file.write(f'batch_size: {batch_size}\n') + file.write(f'num_workers: {num_workers}\n') + file.write(f'validation_ratio: {validation_ratio}\n') + file.write(f'n_time_steps: {n_time_steps}\n') + file.write(f'epochs: {epochs}\n') + file.write(f'validation_rand_seed: {validation_rand_seed}\n') + +### Data Loading ##################################################### + +snn_train_dataset, snn_test_dataset, sensor_size = load_dataset('DVSGESTURE', n_time_steps) +train_dataset, validation_dataset = split_train_validation(validation_ratio, snn_train_dataset, validation_rand_seed) + +disk_cache_train = tonic.DiskCachedDataset( + dataset=train_dataset, + cache_path='./cached_train' +) +snn_train_dataloader = DataLoader(disk_cache_train, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True) + +disk_cache_validation = tonic.DiskCachedDataset( + dataset=validation_dataset, + cache_path='./cached_validation' +) +snn_validation_dataloader = DataLoader(disk_cache_validation, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True) + +disk_cache_test = tonic.DiskCachedDataset( + dataset=snn_test_dataset, + cache_path='./cached_test' +) +snn_test_dataloader = DataLoader(disk_cache_test, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=False) + +### Baseline Accuracy ################################################ + +# instantiate model. +csnn = SCNN_GS( + batch_size = batch_size, + surrogate_fn = PeriodicExponential(grad_scale = hyperparams['grad_scale']['value'], grad_width = hyperparams['grad_width']['value']), + min_v_mem = hyperparams['mem_v_min']['value'], + spk_thr = hyperparams['spike_threshold']['value'], + rescale_fn = rescale_method_1, + rescale_lambda = hyperparams['w_rescale_lambda']['value'] + ).to(device) + +# instantiate optimizer. +optimizer = Adam(csnn.parameters(), lr = hyperparams['learning_rate']['value'], betas = (0.9, 0.999), eps = 1e-8) + +# train/test model. +best_acc = training_loop_no_tqdm( + device, + n_time_steps, + batch_size, + sensor_size, + snn_train_dataloader, + csnn, + loss_fn, + optimizer, + epochs, + snn_validation_dataloader) + +# initialize parameters history. +best_param_set = { + 'learning_rate': [hyperparams['learning_rate']['value']], + 'spike_threshold': [hyperparams['spike_threshold']['value']], + 'mem_v_min': [hyperparams['mem_v_min']['value']], + 'grad_scale': [hyperparams['grad_scale']['value']], + 'grad_width': [hyperparams['grad_width']['value']], + 'w_rescale_lambda': [hyperparams['w_rescale_lambda']['value']], + 'accuracy': [best_acc] +} + +df = pd.DataFrame(best_param_set) +df.to_csv(output_csv, index=True) + +print(f'> initial accuracy: {best_acc}\n') + +### HPO Loop ########################################################## + +train_p_bar = tqdm(range(1, max_iter+1)) +counter = 1 + +for iter in train_p_bar: + + # sample values to be tested. + sampled_values = sample_values_to_eval(iter, prob_model, hyperparams, nb_samples) + + # test each sampled set. + acc = [] + for i in range(nb_samples): + sampled_set = get_sampled_set(sampled_values, i) + + # instantiate model. + csnn = SCNN_GS( + batch_size = batch_size, + surrogate_fn = PeriodicExponential(grad_scale = sampled_set['grad_scale'], grad_width = sampled_set['grad_width']), + min_v_mem = sampled_set['mem_v_min'], + spk_thr = sampled_set['spike_threshold'], + rescale_fn = rescale_method_1, + rescale_lambda = sampled_set['w_rescale_lambda'] + ).to(device) + + # instantiate optimizer. + optimizer = Adam(csnn.parameters(), lr = sampled_set['learning_rate'], betas = (0.9, 0.999), eps = 1e-8) + + # train/test model. + ith_acc = training_loop_no_tqdm( + device, + n_time_steps, + batch_size, + sensor_size, + snn_train_dataloader, + csnn, + loss_fn, + optimizer, + epochs, + snn_validation_dataloader) + + acc.append(ith_acc) + + # update progress bar + train_p_bar.set_description(f'model {counter}/{max_iter*nb_samples} - best acc.: {np.round(best_acc, 2)}') + counter += 1 + + # get best parameters set. + highest_acc_index = acc.index(np.max(acc)) + best_param_set = get_sampled_set(sampled_values, highest_acc_index) + + # update model. + if acc[highest_acc_index] > best_acc: + best_acc = acc[highest_acc_index] + + update_probabilistic_model(max_iter, iter, prob_model, best_param_set) + + # save to history. + best_param_set['accuracy'] = best_acc + + else: + + best_param_set = {} + for key, val in prob_model.items(): + best_param_set[key] = val['mu'] + best_param_set['accuracy'] = best_acc + + # update history + df = pd.DataFrame([best_param_set], index=[iter]) + df.to_csv(output_csv, mode='a', header=False) \ No newline at end of file diff --git a/tests/test_nonsequential/using_SumPool2d/HPO_GAUSSIAN_SEARCH/network.py b/tests/test_nonsequential/using_SumPool2d/HPO_GAUSSIAN_SEARCH/network.py new file mode 100644 index 00000000..53a9feab --- /dev/null +++ b/tests/test_nonsequential/using_SumPool2d/HPO_GAUSSIAN_SEARCH/network.py @@ -0,0 +1,83 @@ +import torch.nn as nn +from sinabs.activation.surrogate_gradient_fn import PeriodicExponential +from sinabs.exodus.layers import IAFSqueeze +import sinabs.layers as sl + +class SCNN_GS(nn.Module): + def __init__(self, batch_size, surrogate_fn, min_v_mem, spk_thr, rescale_fn, rescale_lambda): + super().__init__() + + self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False) + self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool1 = sl.SumPool2d(2,2) + + self.conv2 = nn.Conv2d(10, 10, 2, 1, bias=False) + self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool2 = sl.SumPool2d(3,3) + + self.conv3 = nn.Conv2d(10, 10, 3, 1, bias=False) + self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool3 = sl.SumPool2d(2,2) + + self.flat = nn.Flatten() + + self.fc1 = nn.Linear(810, 100, bias=False) + self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc2 = nn.Linear(100, 100, bias=False) + self.iaf5 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc3 = nn.Linear(100, 100, bias=False) + self.iaf6 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc4 = nn.Linear(100, 11, bias=False) + self.iaf7 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.init_weights() + self.rescale_conv_weights(rescale_fn, rescale_lambda) + + def detach_neuron_states(self): + for name, layer in self.named_modules(): + if name != '': + if isinstance(layer, sl.StatefulLayer): + for name, buffer in layer.named_buffers(): + buffer.detach_() + + def init_weights(self): + for name, layer in self.named_modules(): + if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear): + nn.init.xavier_normal_(layer.weight.data) + + def rescale_conv_weights(self, rescale_fn, lambda_): + rescale_fn(self.conv2, [(2, 2)], lambda_) + rescale_fn(self.conv3, [(3, 3)], lambda_) + + def forward(self, x): + + con1_out = self.conv1(x) + iaf1_out = self.iaf1(con1_out) + pool1_out = self.pool1(iaf1_out) + + conv2_out = self.conv2(pool1_out) + iaf2_out = self.iaf2(conv2_out) + pool2_out = self.pool2(iaf2_out) + + conv3_out = self.conv3(pool2_out) + iaf3_out = self.iaf3(conv3_out) + pool3_out = self.pool3(iaf3_out) + + flat_out = self.flat(pool3_out) + + fc1_out = self.fc1(flat_out) + iaf4_out = self.iaf4(fc1_out) + + fc2_out = self.fc2(iaf4_out) + iaf5_out = self.iaf5(fc2_out) + + fc3_out = self.fc3(iaf5_out) + iaf6_out = self.iaf6(fc3_out) + + fc4_out = self.fc4(iaf6_out) + iaf7_out = self.iaf7(fc4_out) + + return iaf7_out \ No newline at end of file From f9153ed29a74f01e6084cb0109908bdfb63bded9 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Fri, 3 May 2024 23:40:14 +0200 Subject: [PATCH 074/379] experimenting with different residual architectures --- .../architectures_results.ipynb | 306 ++ .../ARCHITECTURES_SEARCH/model_training.py | 92 + .../single_training.ipynb | 3529 ++++++++++++++++- .../ARCHITECTURES_SEARCH/train_all.py | 4 + 4 files changed, 3824 insertions(+), 107 deletions(-) create mode 100644 tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/architectures_results.ipynb create mode 100644 tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/model_training.py create mode 100644 tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/train_all.py diff --git a/tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/architectures_results.ipynb b/tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/architectures_results.ipynb new file mode 100644 index 00000000..f6df49bf --- /dev/null +++ b/tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/architectures_results.ipynb @@ -0,0 +1,306 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import matplotlib.pyplot as plt" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "top_n = 3" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "nb_skip_conns = {\n", + " 'ResSCNN1': {'nb_skip_conns': 1, 'nb_jumped_layer': 1},\n", + " 'ResSCNN2': {'nb_skip_conns': 2, 'nb_jumped_layer': 1},\n", + " 'ResSCNN3': {'nb_skip_conns': 3, 'nb_jumped_layer': 1},\n", + " 'ResSCNN4': {'nb_skip_conns': 4, 'nb_jumped_layer': 1},\n", + " 'ResSCNN5': {'nb_skip_conns': 1, 'nb_jumped_layer': 2},\n", + " 'ResSCNN6': {'nb_skip_conns': 2, 'nb_jumped_layer': 2},\n", + " 'ResSCNN7': {'nb_skip_conns': 2, 'nb_jumped_layer': (1, 2)},\n", + " 'ResSCNN8': {'nb_skip_conns': 4, 'nb_jumped_layer': (1, 2)},\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "architectures_TM = {}\n", + "acc = []\n", + "architecture = []\n", + "\n", + "for i in range(1, 9):\n", + " with open(f'architectures_results_set_1/ResSCNN{i}-Training_Validation-TM.npy', 'rb') as f:\n", + " epochs_x = np.load(f)\n", + " epochs_y = np.load(f)\n", + " epochs_acc = np.load(f)\n", + "\n", + " architectures_TM[f'ResSCNN{i}'] = {\n", + " 'epochs_x': epochs_x,\n", + " 'epochs_y': epochs_y,\n", + " 'epochs_acc': epochs_acc,\n", + " }\n", + "\n", + " acc.append(epochs_acc[-1])\n", + " architecture.append(f'ResSCNN{i}')" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "combined = list(zip(acc, architecture))\n", + "sorted_combined = sorted(combined, key=lambda x: x[0], reverse=True)\n", + "sorted_values, sorted_labels = zip(*sorted_combined)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "top_indices = np.argsort(acc)[-top_n:]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAA2gAAAG2CAYAAAAKvaVLAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAAEAAElEQVR4nOzddXyV5fvA8c+pdXfBigWjGR2SAhISooAdiAoGdmL7xRbBAkWRFpEG6YbRDAYb6+7u7dTz++PZzhjbCEWdP+/363VebE8/57DtXOe67utWSJIkIQiCIAiCIAiCIPzjlP/0BQiCIAiCIAiCIAgyEaAJgiAIgiAIgiC0EiJAEwRBEARBEARBaCVEgCYIgiAIgiAIgtBKiABNEARBEARBEAShlRABmiAIgiAIgiAIQishAjRBEARBEARBEIRWQgRogiAIgiAIgiAIrYQI0ARBEARBEARBEFoJEaAJgiAIgiAIgiC0Ev9ogHbw4EHGjRuHl5cXCoWCDRs2NFovSRJvvvkmnp6eWFpaMnz4cOLj4xttU1RUxD333IOdnR0ODg488sgjVFRU/I13IQiCIAiCIAiCcHP8owFaZWUlXbp04euvv252/ccff8z8+fP57rvvOH78ONbW1owcOZKamhrTNvfccw8XL15k165dbNmyhYMHDzJjxoy/6xYEQRAEQRAEQRBuGoUkSdI/fREACoWC9evXM2HCBEDOnnl5efH888/zwgsvAFBaWoq7uztLlixh6tSpxMTEEBYWxsmTJ+nRowcA27dvZ/To0WRkZODl5fVP3Y4gCIIgCIIgCMINU//TF9CS5ORkcnJyGD58uGmZvb09vXv3JiIigqlTpxIREYGDg4MpOAMYPnw4SqWS48ePM3HixGaPXVtbS21trel7o9FIUVERzs7OKBSKv+6mBEEQBKEVkCSJ8vJyvLy8UCrFcHRBEITWpNUGaDk5OQC4u7s3Wu7u7m5al5OTg5ubW6P1arUaJycn0zbNmTt3Lu+8885NvmJBEARB+HdJT0/Hx8fnn74MQRAE4TKtNkD7K7366qs899xzpu9LS0tp27YtycnJ2Nra/oNXJgiCIAh/vfLycvz9/cXfPEEQhFao1QZoHh4eAOTm5uLp6WlanpubS9euXU3b5OXlNdpPr9dTVFRk2r855ubmmJubN1nu5OSEnZ3dTbh6QRAEQWi9NBoNgCjrFwRBaIVabeG5v78/Hh4e7Nmzx7SsrKyM48eP07dvXwD69u1LSUkJp0+fNm2zd+9ejEYjvXv3/tuvWRAEQRAEQRAE4c/4RzNoFRUVJCQkmL5PTk4mMjISJycn2rZty+zZs3n//fcJCgrC39+fOXPm4OXlZer02L59e0aNGsWjjz7Kd999h06n48knn2Tq1Kmig6MgCIIgCIIgCP86/2iAdurUKYYMGWL6vn5c2AMPPMCSJUt46aWXqKysZMaMGZSUlDBgwAC2b9+OhYWFaZ8VK1bw5JNPMmzYMJRKJXfccQfz58//2+9FEARBEARBEAThz2o186D9k8rKyrC3t6e0tFSMQRMEQRD+3xN/9wRBEFqvVjsGTRAEQRAEQRAE4b9GBGiCIAiCIAiCIAithAjQBEEQBEEQBEEQWgkRoAmCIAiCIAiCILQSIkATBEEQBEEQBEFoJUSAJgiCIAiCIAiC0EqIAE0QBEEQBEEQBKGVEAGaIAiCIAiCIAhCKyECNEEQBEEQBEEQhFZCBGiCIAiCIAiCIAithAjQBEEQBEEQBEEQWgkRoAmCIAiCIAiCILQSIkATBEEQBEEQBEFoJUSAJgiCIAiCIAiC0EqIAE0QBEEQBEEQBKGVEAGaIAiCIAiCIAhCKyECNEEQBEEQBEEQhFZCBGiCIAiCIAiCIAithAjQBEEQBEEQBEEQWgkRoAmCIAiCIAiCILQSIkATBEEQBEEQBEFoJUSAJgiCIAiCIAiC0EqIAE0QBEEQBEEQBKGVEAGaIAiCIAiCIAhCKyECNEEQBEEQBEEQhFZCBGiCIAiCIAiCIAitRKsP0MrLy5k9eza+vr5YWlrSr18/Tp48aVovSRJvvvkmnp6eWFpaMnz4cOLj4//BKxYEQRAEQRAEQfhjWn2ANn36dHbt2sWyZcuIiopixIgRDB8+nMzMTAA+/vhj5s+fz3fffcfx48extrZm5MiR1NTU/MNXLgiCIAiCIAiCcGMUkiRJ//RFtKS6uhpbW1s2btzImDFjTMvDw8O57bbbeO+99/Dy8uL555/nhRdeAKC0tBR3d3eWLFnC1KlTr+s8ZWVl2NvbU1paip2d3V9yL4IgCILQWoi/e4IgCK2X+p++gKvR6/UYDAYsLCwaLbe0tOTw4cMkJyeTk5PD8OHDTevs7e3p3bs3ERERLQZotbW11NbWmr4vKysDQKfTodPp/oI7EQRBEITWQ/ytEwRBaL1adYBma2tL3759ee+992jfvj3u7u6sWrWKiIgI2rVrR05ODgDu7u6N9nN3dzeta87cuXN55513mizfuXMnVlZWN/cmBEEQBKGVqaqq+qcvQRAEQWhBqw7QAJYtW8bDDz+Mt7c3KpWK7t27M23aNE6fPv2Hj/nqq6/y3HPPmb4vKyujTZs2jBgxQpR6CIIgCP/v1VeOCIIgCK1Pqw/QAgMDOXDgAJWVlZSVleHp6cmUKVMICAjAw8MDgNzcXDw9PU375Obm0rVr1xaPaW5ujrm5eZPlGo0GjUZz0+9BEARBEFoT8bdOEASh9Wr1XRzrWVtb4+npSXFxMTt27GD8+PH4+/vj4eHBnj17TNuVlZVx/Phx+vbt+w9erSAIgtDa7YvNI3TO76w5mf5PX4ogCIIgmLT6AG3Hjh1s376d5ORkdu3axZAhQwgNDeWhhx5CoVAwe/Zs3n//fTZt2kRUVBT3338/Xl5eTJgw4Z++dEEQBKEVm7crjhqdkS92x6E3GP/py6G8RkdURilGY8vNlfPLa4nPLb/qcYortaQWVt7syxMEQRD+Jq2+xLG0tJRXX32VjIwMnJycuOOOO/jggw9M5RkvvfQSlZWVzJgxg5KSEgYMGMD27dubdH4UBEEQhHrn0ks4l1EKQHZpDbtjchnV0fMae/11LmSW8tiy02SWVOPrbMXUnm2ZHO6Dq605BqPEofh8Vp1IY3dMHgajxBODA3lhRAgqpaLRcfbF5vHMqrOU1+p5cWQITwwKRKFQtHBWQRAEoTVq1fOg/V3EfDCCIAj/v2j1RtacSmdYezc87S2brH9+zTl+O5OBhUZJjc5Iv0BnVj7a5x+4Ulh3JoNX10VRq2+cxVMrFQwKduVSTjmZJdVN9rsl2JX5U7viYGWGJEl8sz+RT3fGcvlf9dGdPPhkcheszRt/Hiv+7gmCILRerb7EURAEQRBu1Pw98byx4QIP/XQS3RXli0WVWjafzwLg0zu7oFTA0cRCEvKuXjp4s+kMRt7edJHn1pyjVm9kSIgrx14dxseTO9OtrQN6o8SeS3lkllRjb6nhof5+7Hz2Fr6c2hULjZKDcfmM++owp1KKeGL5GT7ZIQdn03q15b3xHdCoFGyLymHiN0dILhAlj4IgCP8WIoOG+CRREITrk1pYyVd7E3h6WBBtnP74nImH4wvYEJnJG2Pa42BldhOv8N/nSEIBm89l8cbYMGzMW6i6j94IZ5aBdFmgpVRBz0cheESTzYsrtQz4aC+VWgMAr94WymODAk3rv92fyEfbL9HR247NTw5gxrLT7IrO5YG+vrwzvuM1r3lx1GJszWy5K+SuG7vZOvUliwv2JnA6tRiAp4cFMXtYEMrLShYv5ZSx82IubZwsua2jJxYalWlddFYZjy0/RXpRQ2bNTKXk3fEdmNqrLQCnU4t5Yvlp8sprsbVQ8+XUrgwNlecNFX/3BEEQWq9WPwZNEAShtVh4MIlfT2dQpTPw9d3d/9AxzmeUMH3pSWp0RkI9bJk+MOAmX+W/R5VWzzOrz1JQocXfxbpREGVi0MGWZ6GqsOm63IvwzHlQNf5T9v2hJCq1Bmwt1JTX6PlidxyjO3nSxskKg1Fi+bFUAO7v64dCoeD+vr7sis7ltzOZvDgqtOVAEUguTWbemXkADG4zGDcrt+u+35zSGn49lc7qk+mmkkUbczWf39WFER08mmwf6mFHqEfzwVOYlxxcPrXqLIfiC3C3M+e7e8Pp1tbRtE24ryNbnhrAEyvOcDq1mIUHkhgS4ibGpAmCILRyIkATBEG4TjHZ8uS+u6NzKavRYWdxY3NJ5ZTW8OjSU9To5ExQZHrJzb7Ef0xhRS1PrDiDnYWGD+/ohItN07kmr7T8WCoFFVoAdkXnNh+gJe6VgzNrVxjxfsPyHa9DWSbEboOw202Liyq1/Hw0BZDLF386ksyxpCLe2HCBJQ/1ZF9dyaCDlYbbu3gB0D/QhQAXa5IKKll/NpP7+vi2eM0XCi6Yvt6fvr/ZLFpplY57Fx8n5YqywkqtnvoGjfaWGiZ19+ahfv60da7Lxu7/EC6uh8k/gnuHFq+hnoOVGUse6sXRxAI6eds3m411s7Ng1aN9mLc7jof6+4vgTBAE4V9AjEETBEG4DkajRGyOPEapVm9k+4WcG9q/Wmtg+tKT5JbVYmchfzZ2LqPkT12TzmCkolb/p45xM9TqDTy+/DQnkovYHZPLuAWHOZdeApIE5bnN7lOl1bPwQBJq9NhTwem0YvLLa5tueP4X+Ryhk6DL1IZH+APy+hOLGm1enz3r4GXHiDB3PpjYCTOVkgNx+Ww7m8LKiHgA7urRxlQyqFQquLcuKFsWkcLllf+SJJFXXmP6PqogyvT1nrSGOTgvN/f3GKIySymv1Td6GCXo5e/EF1O6cPy1Ybw1rkNDcJZ+Qg7Q8i/ByilQkdfCs92YSqlgYJBr4+BMVwM1paZvzdRKXhoViqvttYNmQRAE4Z8nAjRBEITrkF5cRVXdmCaADWczr3tfo1Hi+V8juZBZhpO1GatmyN0C04uqKaxoJii5Tk+vOkvvD3aTlF/xh4/xZ0mSxBvrL3AypRhbczUBLtZkl9Zw18IjpH5/D3wWDAtvgVM/Qk2Zab+New/zSO1Sjls8zWmLJ+ijuMjumCuCuZoyuLQVgGnHfTmVUtSwLvwhUCgh5RDkXQLkLF599mz28GAUCgWBrjbMGtIONXoCNk1kQdokxqoiuLd34yzZHeE+WGpUxOVWcDy5iIKKWhYeSGToZwfo9cEePt0RCzTOoJ3IPkGZtqzRcU4kF7G6buLr7+/vwf4XBpseJ14fxprH+jKxm0+j8WQYdLD5GUCS76k0HX65Vw60bpTRCEvGwEd+cqAX+zsY/vkgXhAEQbh+IkATBEG4DjHZcvbMw06eYzEiqZDs0qatz5vzxe44tkXloFEpWHhfOB287AlwtQbgfGbpNfZuXo3OwO6YXCq1Btacymhxu5SCSk5eHtj8QRcySzmWVNhkEuVFdePylAr46p7ubHyyP8PbuzOTtfhmycEV2efkcWSfhcKGWRiW3M60Y+OZqd6EMyWoMfCyejU7LmQ3PumlLaCvIdHoyRm9L6+si6JWXxckO7SBkNHy1yd/AOD7Q8lUaQ109LZjePuGsWGPDw7gCfvjtCcZa0UtX2kW0Pb03EaBi72lhondvQF49pdI+s7dw9zfL5m6H361L4FfTiZzqUgOBh3MHdBLeg5lHDIdo1Zv4NV15wF4sWMFt1Zvxy91renhlrGr+WDp6HzIiwYrZ3hoO1jYQ/px2Pw0XNnHK/s8pBxp+YVK3k967lmiNSqI2w6rpsK8TrDvf1CS3vJ+giAIQqshAjRBEITrcClHzpQMDHKhl58TkgSbIrOuuV9MdhkL9iYAMHdSZ3r6OQHQ1ccBkCdM/iPOZ5SiM8hv3jdGZjYJnEAOGKYsiuDO7yJYd6blIO5aDscXMP7rI0xddIxBn+7j630J5JXXsCs6lw+3ywHLm2PDGBTsiq2FhkXdknlGvQ6At3X3s8RmBnqnINBVQuRyVCkHMEoKjqu6Yxj/LUa1JV2ViWiS9lBeo7vsJuXyxvWGAYCChLwKFh5Ialjfc7r877nVFBYVsjQiBYDZw4IbjbUyx8CT6g0AnDYGyQuPzoflk6CyofnI/X3lrFp2aQ06g0SXNg58dEcnHhskN3KZ8/sudEYddmZ2TA6eDMDetL2m/b/bn0RifiX3WR1jVsIMOcC6/PHLvbB8IlQWNNxDURIc+Fj+euT/oG1vuPNnUKjk+z/8OdSWw6mfYOEgWDgQloyG+N3NvlbrIj5mvI8Xd3t7cqnn/XLQV54FBz6C36Zf5VUWBEEQWgsRoAmCICC3Zv9sZyzpRVXNrr9Ul0EL8bBlQjc507L+Osoc64OG0Z08mBzuY1re2ccekAOtP+JUakNWLLu0huPJTbNkv0flkFsml1C+8ltU4xLB65SYX8HMFacxGCVUSgXpRdV8seMib334ESmrn2eg4hz39fbhgX5+8g4Zp1BunAVAauh0flOP5e2CwfQrm0vsbWvQdX+YRYo7uUU7j/TRy1B1uxtlr0cBeFL5K/sv1Y29KstGSjoAwG/mfoR3OQXKGr7al9BQ0hkwGJyDQFvO8fXfUKU10NnHnmHtr+iseG4l5pWZVJs5E9H/R4yTl4DGGpIPwKLBcoYPuWvi2+PCmD7An61PD2DjrP5M6dmWl0eGcltHDyQzOQMVZB/G8LbDATiceZhaQy2J+RV8vS+B7oo43uZb+bxtekPwbXWPUWBmA8kHYdFgdBknWXLhJ05sfgL0NeA/CDpPkfcLHAKjP5G/3vMufBoCW2ZDdiRHLSxY4GBP3P53GmXXtAYt7x54ibcMmegUCgzAKkcneC5GbjrifwuEP3jDr78gCILw9xMBmiAIAvDO5oss2JvAJ3Vjja5Un0Fr72nHmE6emKmUXMopNy1vTmm1jg1n5SzbA339Gq3r0sYBkDNof2Q6yjN182dZm8ljmZobE1cfHDpZm6E1GHls2ekWA9DmlFRpmf7zKcpq9IT7OnL2ySB+77iPk1bP8K3mcx5Vbmap2Ue8m3ovikOfQsZpWDUNDLUQMhrfuz5m45P9CXKzIa9Cy9hNBh4pmMb/qieidvJlQle5iyL9n0GrtKCLMonMkxvlZRfWokBipyKIyjYbidOuxSX4O/TKHF5ff0F+zhQK9D3krFBQ6ipAYvbwoMadCvVaOPgpAJZDXuDJkZ1RdpwI03eDoz+UpsHiEXBOztY92N+fN8aG0cHL3nQIpVLBZ3d1wdlZbgwTm+ZAG+sg3KzcqNJXcSzrGK+vj8LVkMsSy3mojDoIHSuXK969uu7xi3xOp0Ck0nTe3TiNz05/zmNSJietbGDsF3D5dfd8BHo9Jn+tqwTnIDKHvMJsnzYscrTnDrMS7lk3jvXx60krS+PhHQ/za8rvKCSJ25HLZ7clbaPUUAMd74AHNsvNVQRBEIRWTwRogiD85yXkVbDpnBxIHU0sbBIwVdbqSa0LbEI9bLG30jA4xBXAFIA1Z+3pDKp1BkI9bOnl79RoXXtPO9RKBYWVWjKKm45lW382g0eWnKS4UttknSRJpgmOnxoml+xti8qmRtfQxORCZiln0krQqBRsmNmfDl52FFbKAVejMsIW6AxGZq44Q3pBKffYRrLK8iPsFvWgfcL3OBqL0Vu6kOJ5G5KFPYqSNNj7PvwwFCrzwL0jTFoEShUBrjasn9Wf2zp6oDNIHIzLl697aBBqVd2fIGsXijrIXRkHZPxArU6PVFfe+KmzPZJSbpZRq8jByv9rTuYd5LczmeSV1/DwmUAqJXOClJl83rOcISFXZM8iV0BpOpKNO5+rynl+//NU6arAPQxm7IN2t8oZrPUz4PdX5IYdzbAyU+PiLDcxyStwY/jnBykvCgXg+a0riErK5CfzT7EzllDu0ZHn3Vz46NQn1Ojla684cID0OV+iv2MtPwf2YIONJQB6hYJnPT1I1zQzYfnI/8HEhfDgNqRZJ/hAl061oRZXpQVqSeJ8RSpvHn2TMevHcC7/HLZGia9z83m/x6sEOwZTY6hhQ8KGhuOJFvuCIAj/CiJAEwThP2/B3njT/FQFFbUk5jeevyoutxxJAldbc5zr5veaWFfm2NL4L6NRYlldBuv+Pj4orgj6LDQq2nvKkxCfzyiVy9XqttEZjLy7OZo9l/JYc6ppY4ekgkqKq3SYq5U81N8PL3sLymv17L2UJ3fxoyF7NqqjJ22drfjhgR642ZoTm1vOM6sjKa3SUVZT96jWUlZS2OixYM3vDEj9mmPmT/GB7mPMUvbLJw8cCnf+jPr5GPweW43i+Vg5iGjbV15v7QbTVoG5rel6bczVfHNPd14cGYJCIRHqacv4+uxZHbcRL1KFBR0ViWRsmYsiJ4rdltZk2+ahUqhYeOtCerj3QKGsxbLNMt49/Bljv9rFwcwy1ir6U65QcLtua9Ps2aHPANjd5XZ+ilnOztSdvHLoFYySESwd5czWwBfk7Y9/C0snQEma3Ka+/qGtolJXSVq5/JyaG3zJK6+lMDcYgBrNeb7QLCBYkY7exp0XfYPYmb6X5THLuf/3+8mqyKLgu4VUHDjAmVU/8LlRDlKfLSqmo1FFqVHLrL2zKNeWN36hVWo56+XXnx1pOzmUeQiNUsPi4d+xK7uIZ4pK8DF3BqCdhSurM7MZqHFGETqGqaFytuyX2F/kexUEQRD+NUSAJghCq/HR9ksEv/E7M1ec5lB8frOBz82WkFduyp75OMpZjYikwkbbXKqb/yzUoyHoGBLqhq2FusXxX4cSCrAuiuYj85+YtncgLBokN4S4TJc2chldddRGucPhpicBOBiXT3GVnMnZcbHpfGv12bMuPg6Yq1WMrwsW44+sg4/90K66j92RcmOS+sYXnvaWfH9/D8zVSvZeyqPLuzvp/PZO+r+9gfi5/bCbF9Do8Vzs3cxUb8JFUQo27jDgOXg6Eu5bDx0mgLou46OxlIOIh7fDM+dg1nFwaNvkmhUKBZN72dK2y6d4hSzFQONMldLWlZOudwAQeO5TqhQK3nWWs5T3h91PP69+LBqxiGmh9wAgOeym2us1bEPe5lP/BPr5teG2ytPErX8IMk/Lwe7ZZVCaTrmtB3OLT5vOtS99H1+e+bLuxCoYNgemLJfHiKUelrsefti24fE/T6JXTkBCwsPKnUPPj2fVo31YOvl2rBXmKNSVuFjFIKkt+LTbWI7kncZCZYGDuQMxRTFM3XQXVbExAFyM2IyExF3Bd/HQQxHMn7QJNys3kkuTefHAi+iNTbs8lmnL+OjERwA82ulR/D3DcekxnemlZWwtk1g79ldWl0FbvR56PAQqNWP8x2CrsSW9PJ0jmVfp+igIgiC0OiJAEwThhhVU1LI/Nq/R42RKEYarBFS1egPxueUtrk8rrGLRwSS0eiPbonK4b/EJU8fA0qprl+RdlSRBViTompYSzt+TgCTBrWHu3NWjDQDHrgzQshvGn9Wz0KgY08kTuGL8l0EHp5fQdu1otpq/xhTFLhTaCsg5D4uGQEJD973O3nY8p17D5PiXoSIHzi6H/NhGzUfOppeQV9Z4PqzTKXKA1t3XEZCzeaGKNB7JfhdqSjGL3cQa5RsMdyujR902II97mzelq2mibBUGvtZ8SbgyvsnzYpAUZLr0lwOXZy/C8LfAyb+FJ7iOox9YObW4+vuo7ymqLeRk7nHePvp2k1JS9cBnqJDkaQy+crSnWGPE1dKTx7s8DoBGqeG13q8wM2wOGC2bHD9HrebpwmMULh4udzs8IAc1X7YLJ7+6AF87X97t9y4AP174sXH5X/tx8OhecO/U7LVHFckBVqeiTJyPvkffS/+j/8ZbGFIuB+d7bGz4tf8jrMjYBcDcgXP5ZewvtHdqjzq/BEWV/Bq2ydbT26M3r/R+BYVDG1zt2/LV0K+wVFtyJOsIH5/8uEnGa97peRRUF+Bn58cjnR6RF/Z7GjTWKLMiCYnagHnmaVBqoLtcKmqlsWJ8u/EArI5d3eJrIgiCILQ+6n/6AgRB+He5kFnKlIURVF42aXO9/u2cWTCtO07WjcfTpBRU8tiy08TmljNnbBiPDGj6Rv+rffEYjBI9/RwJ87Rj3dlM0ouq+WRHLJvPZbHpyQGYqf/gZ0rRG+DXB8ElBKauABd53FZ8bjmbz8vZs9nDg6isle/peJI8Dq2+XC6mmQwawIRu3qw+mc76yEz6tXNmfFdv2DgLzv+CP6CVVNQGj8U2fAoc+hwyT8HyyTDsTejxEKOjZmOjltu0SzbuKCpy0UYsZFf0rQC42JhRUKFlV0wu91w2sfLpNDlAqw++gq2rWWr5GTbGGvIcukBpBu2UWXxb9TyKWHsIHW3a97ZOntwa5o5BklDteAX1ySgkjRW6+zYhuXUwbadSqvA2M/9jz3czcipzWBcvt95XKpRsSdpCoEMg0zs1tH7vGRbEz7+Nop/ZVlbYyc/1u/3fxEpj1ehYT/S8i0e7T8JIQyBTVlvGA1umkEYez7q78UN2FGZApKMXa0rkyaXn9JlDb8/eZFZksvD8Qt6JeIe2tm3p7t5dPohrCDx+CAxXjPsrz+bCrplQnU6HqnI4usC0apjCjy0Y2ejiQUX6NgCe7vY0w33lLo9Lb1vKT98+ARwDwLMYPun1HhqlxnSM9s7t+d+A//Hs/mdZdWkVBzMOMjl4MhPaTSCjPINf434F4M2+b2KmqvvZsnaBXo/CkXmwf668rMMEsGkYgzc1dCrLY5ZzKOMQGeUZ+Ng2dBEVBEEQWi+RQRME4brlldXw6NJTVGoNeNlb0NHbzvSw1Kg4klDIuAWHuXDZ5Mv7YvO4/avDxNZlzz7d0bSVfWphJb+dkbNGr45uzzvjO3LiteF8emcXnKzNuJRTzveHGpcH1tPqjaw6kdbQer055+U3uBTEwvdDIfZ3AL7cE48kwcgO7nTwsqdLG3ssNEoKKrQk5MnHkyTJlEEL9bBrdNhefk4Mb++OVm/kmdWRrFq5GM7/ghElc3XTeMZrFbb3LIXQMfDQtrrshgR73oEvOmKTtpcaScNs7UwyBs8DQHF+NWp9JYGu1jzUXw5kd17MNZ2zpKrh2pwci9kctw7d6rtxM+aTZPRgcukzjK5+n1O0R6OvhNXTYPfbUNVQhqlWKTE/+xPqk4vkc05ciFnbnphbWJke6psYnAF8f/57dEYdvTx68Vqv1+Tn/8yX7E5tyCiaqZVcaDeZ55zbYlQo6OwwmAHeA5o9nlqlxkxlZnq4WLmwYOT32GpsOWthxjtdbkUXMJR3vNogIXF74O309uwNwMyuM7nV91b0Rj2z983mq7Nf8XXk1/Lj3DfszToCavOGh6MfF5Ry6WGnfi9D2ATodBfcv5F+jxzBXGVOqbYcg2RgbMDYRkGnhdqCycqepu+VEpgnXzEhNzDcdzhz+szBVmNLZkUmX575klt/vZWn9j4FwMR2E+np0bPxTnVZNJNeMxqt9rXzpZ9XPyQk1sSuucYrJAiCILQWIoMmCMJ1qdEZeHTZabJLawh0tWbdzP7YWzZkAWJzypmx7BSphVXc8e1R5k7qRHZpDZ/ujEWSoHtbBxQKBadTi3lz4wV+fLCnKUO1YG8CBqPEoGBXureVs0KWZiomh/ugUsKzv5zjyz3xjOnkiZ9LwxtSSZJ4bX0Ua09nEOBqza5nB6FSXtGpTlsJiXvkr906QN5FWDWVgh7PsS2qO6DkmWFyswdztYpwX0eOJBRyLKmQIHdbcspqKKvRo1YqCHSzbnRopVLBwvvC+WxnLD/uj6b/pbmghJXcxkLDOBYO6NKwsdocbp8PXt1g24ugrQD7tryleYkNGU70kzrSxjkITWE8E1WH8eg2i5EdPPhkRyxHEwsoq9FhZ6HhTF32zM/NyNP7p1OqLWWdvoaPLR14tPQF0rQWgAXbuy+ih2YFHP8ODn8BEV9D2Hh5LiyDFra9JF/XsDch7PY/95/jGrIrslmXIGfPnujyBD08epBYmsiqS6t47fBreFp7UlRTxNq4tew37sdoYQSjJZ8Om3ND5wmwD+DTwZ8yc/dMNpXFkuLSmYSCBBzMHXihxwum7ZQKJR8M+IDMikyiC6NZeH5hk2P9POpnU2atoLqA7MpsFCgI6/YQ9H7KtJ0V0NerL/vT99PFtQtv93u7caMSoDYurtH3NRejserevck57wq5i3GB49iZspO1cWuJzI+kpLYEJwsnnu/xfNMbtnaG3jPk19ejM/j0bLLJtNBpHM06yrqEdczsOhMLtcXVnkJBEAShFRAZNEEQrkmSJF749Rzn0ktwsNKw+IGejYIzkCdw3jRrAENCXKnVG3luzTk+2SEHZ3f3bsuqGX34eHJnzFRK9sXmszVKziKkFFSaxlzNHh7U+MS1FUzI/JzZXtFo9UZe3xDVaNzSooNJrD2dAUBSfiWbzjUzcXTiXrmNuoMvzNhvyjK4nPqcb9RfcluYK2FeDZmxPv5yV7xjSXLGqX6C6kBXG8zVqiaHVykVvDQqlK2dj9JWmU+m5Mz/au7Ay96CYaFuTbanx0PwyE4Y8gbM2I9DQA8AzmWUUtZJHj90v2on47t40c7NhkBXa3QGif2xcue/U3Xjz8zdtlKqlTOVpywtmNa2LXaBDSWY9/QLhNs+gjuXyG/eDVqI+hWWjIFlE0EyyBMjD3iu6TVeh4isCF4++DLpZU27TF7p+6jv0RvlsVc9POT7fannS/Tz6ke1vpqpW6cyc89M9qbvxYgRb4sw3ujxKZ42zTx/19DPqx+v9HoFgPMF5wF4occLOFo4NtrOUm3Jt8O/5bHOjzE1ZKrp0d1NDpzejXgXXV3L/YsFFwHwt/fHxsymyTlf6vkST3Z9kq+GfoW5qmnmsTZWnlvPsoscsNfExLR4/ZZqS8a3G8+y0ctYf/t6ZnWdxdfDvsbe3L75HQa9DMPegjt+aLaN/kDvgXhZe1FaW8rvyb+3eF5BEASh9RABmiAI1/Tlnni2nM9GrVTw3b3hjbJYl7OvC96eHtoOADOVkg8ndeJ/EzthrlYR6GrDzCGBALyzOZrSap0pezY4xJVubRu/iWbPuyhOLeapqm+wUkscSShkQ6QchO2KzuXD7ZcA6NbWAYAFexLQG65oKR6zRf43dCyozSga9AELHZ+nVtIwSnWS14JSG23eN7A+QJPHocXUTUQd6tl4/FkjORcIjP8RgK8tH6cKCx4e4N8wz9eVvLvDoBfB2pnOPvK1n8soYZ3xFiolc4KVmbQpOwPAiA4eAOys6+Z4OrUYK6uLZBmOoJAkPsgvxM/MkVxtCSmaj1Hbn2JYqBv+9a9Rh4nyuKoZ++XsWX2A4dMLxs2/4bmxJElicdRiHt/9ONuSt7H4wuKrbp9VkcX6hPUAPNH1CdNytVLNJ4M+IcA+AAA7MzvubX8vG8ZvYPuUX5jS6ZYbuq7LTQ2Vgy2A3p69uT2w+Qyhk4UTT3Z7ktf7vG56zB86HycLJxJLE/np4k8ARBVEAdDRpWOzx2lj24bHujyGg4VDk3XG6mq0qfL/MfuJEwGoiY6+rvto59iOx7s83uJ5AbmL5sDn5PFzzVApVdwVchcAa+PXXtd5BUEQhH+WCNAE4V/g56MpjJl/qEk3v7/D/tg85u2Wu/x9MLEjfQKcr7q9UqnguREhbHlqADuevYWpvRq3XH9icCABrtbkl9fy3C+RpoBr9vDgxgfKOA0n5DFSqpoi3g+Xx629tyWGowkFPLP6LJIE9/Zpy7JHeuNopSGpoNLUMh+QOyrGbZe/bj+WC5mljFtwmLnZ4SxnFABt4lc0Om1nHwcsNEoKK7XE51WYMmhXjj8zMRpg8zNyRqr97bz+7HOsfbxvs41QmlPfav9SdjkrIktYb6gbc1V37yPC3AHYH5tPZa2eoswovD2XAjC1vIrbB73Lyju2MrjNYPSSDkuvtfToEtn0RF7dYNyX8PwluPtXuH8DaG6s3K1KV8XzB55n3pl5pk6D+9L3YTA2bRhTz5Q98+xNuHt4o3V2ZnYsvW0p3w7/lr137eXlXi8T6BB4Q9fUkld7v8rCWxcyb/C8JiWHV2Nvbs9LPeXyz4XnFpJalsqFQrnJSCeX5js8Xk1tQgJIEipnZ2wGDjAtM2qbTkBeuHgxSRMmostsJhP8J0wKmsRT3Z7iyyFf3tTjCoIgCH8NEaAJwr/AooNJXMwqY0d07rU3vsl+PSWXEE7r1YYpPZvOb9WSjt72DVmcy5irVfxvovxGd8+lPAxGiSEhrnRt49CwkUEvBz1IoJSHyo63OEuwuw1FlVru/uE4VVoDA9q58Na4DtiYq3n0FjkTs2DvZVm01KNQUwJWzqwv9OaOb4+SWVKNr7MVQ+55FVBA0j4oSDCd2kytpIev3Cr+UEIW0TlyaeGVHRxNTv0od2c0s4XbPsLaXE0PP6dGQUG5tpxqfdMW/wDeDpY4W5uhN0rE51WwUhopr7i0FUoz6eLjgLudORW1enZvXMYIp4/IMVPgZpB4esxi6DkdWzNbvhzyJY91fgyAn6K/p6im6dxs8gtgC8EjwKz5LGhLUstSuWfbPexK3YVaqebVXq9iq7GlqKbIVEp4pcyKTDbEbwBgZpeZzW5jb27PAO8BzZYG/hlKhZJ+Xv2aLUm8ltH+o+nn1Q+tUct7Ee9xoUAO0K6ayWpBfXmjRUgwai8vVPb2oNdTG9d4agNJp6Pgu4XUXrpE4U9Lbvg8V+No4ciMzjNwsXS5qccVBEEQ/hoiQBOEVi67tJrMEvnN/dXmEfsr6A1GDicUADA5vM1NO26fAGfu6tHQ8rtJ9uzYN5AbBZaOMOYzAFSxW5g7seENcoCLNV/f3R1NXRnh/X39cLTSkFxQycbIuizaJbm88axlX55dc4FavZEhIa5smjWAgOAOEFwXDJ384YrrcwIMfBc/i2zbd0BZ1XyJY34s7H5H/nr4W2Dn1WSTrIosRv02ikG/DOKto29xPv98o3F0CoWCLpcFp94h4eDbX87InV6C0lDDix5nWWP2Dh3iX2Slgzz/12v93sImYIhpP6VCyayus+jg3IFqfTVLLixper03yCgZiciK4Pn9zzNh4wQSShJwtXTlp5E/cXf7u7mljVyGuDdtb7P7/xj1I3pJTx/PPg2t7P8FFAoFb/R+A3OVOcdzjlNaW4pGqSHYMfjaO1+hJlZuEGIeHIJCocA8rL28PKZxmWPliRMYy+Wf79INGzBWVv7JuxAEQRD+rUSAJgit3OnUYtPXcX9zgHYuo5TSah32lhq6+LTQpOAPem10e/oEODF9gH+jAIXi1IZ5nW59T25nrraEkjTCLbJ47tZgOvvY88MDPbC3amhUYmOuZsYtcnncgr3x6PUGDHXjzxZkhwLw1NB2coOT+v16Pir/G7kCahva9PcNdEZpkUWNIhelpgxb96N42F1RDlhVBCungLZcDqh6PNzsfS46v4gybRnV+mrWxa/jnm33MHnzZFZfWo3OKDeh6HzZczuxmzf0rGvTfvw7+CyEyWnv00MZy9suTugVCvwsejA0ZHKTcykUCmZ2lTNVq2NXU1hd2GSb61Glq+KHqB8Ys24MM3bNYGfqTlOTj1/G/kJXt64ADG0zFIA9aXuaTDpdpi1jc9JmAGZ0btz+/d+gjV0b0wTZACGOIQ1zkN2A+gyaeYg8RsyifZi8/IpGIRV79pi+NlZUULp58w2fSxAEQfj/QQRogtDKXR6gxedeZa6vv8DBOLm8b0A7l5YbXvxBDlZmrJ7RlzfGhjUslCTY+jzoqsB3AHS7F8ysoN0weX3MFp4eFsSmJwcQ4Nq0dO3+vr44WZuRUljF0nUbUJVnUSmZE6npysL7wnl+RAjKy9vwBw4FpwCoLYOohnmiOnk7YGGbYvpeaX+YMm1Zw356LfxyHxQny90h71oKyqYdHjPKM9iYsBGAN3q/we2Bt2OuMieuOI4Pjn/AozsfpbC60FTeaWuuZmioG7QfBzYe8nXVlCLZt+Ely5GcszBHMpjxROeXWhxXNdB7IJ1cOlGtr+anCz9d41Voqr6U8cszX5JRkYGNxoapIVNZO24tP4z8AVcrV9O2A7wHYKY0I708nYSShEbH2ZSwiWp9Ne0c2tHDvccNX0dr8ECHB2jnIDe8+SPljZIkNSpxBLAIk/+/11xsyKBJRiPle+QspPUAeZxa8YqVTYJeQRAE4b9BBGiC0MpdHqAVVmoprKhtdrujiQV8viuOilr9TTv3gboAbVCwa+MVp5fIY69uRN6lJhMmN3FxHSTsApUZjP2iocNg6Bj530tbr3oKa3M1M+rGolWe3wTAKU04a2YNZmRdN8RGlMqGbNWJH+QAEXkcmqNTGgCSpMCoqOHniz9TtwC2Pgeph5HMbPl1wKNsyz3R7PX8EPUDeklPX8++TAmdwgcDPmDPnXt4uefLWGusOZ17milbpuDkmMtTQ9vxxZSuWGhUoNLApIVyVu7e31A8c54LXnL5pL6sO0MCWy61UygUPNFF7pb4S+wvFFQXNFqfVZHFJyc/4ffk39EaGjeqOJhxkGlbpplKGd/t9y577tzD631eJ8SpaZdAK40Vfb36Ao3LHI2SkV9ifwFgasjUG2rS0ZpolBq+GPwFdwXfxUMdH7rh/fV5eRhKS0GlwixQzu5a1Jc4xsYiGeTmKjUXLqDPzUVpZYXXh3NRWFpSGx9P9alTN+9mBEEQhH8NEaAJQitWpdVzMUvO3NiYy80y4vOaz6K98lsU8/fEM+HrIyTm//lMW3GllvMZJQAMDL6suUDcTrmBx5Zn4fii6zuY0Qi/PSJPqLtldvPbVBfD7/L8VQx4DlwvC0KCR4FCJY9LK0656qnu7+uLi40ZI5UnAeh52320c7tKi/yud8sllHkXIS0CAL1RT7VKzghpC+QyvhUxKyipKZEnfD67DBRKFve9h3cvLuLlQy+zPn59o8Nenj2rLzsEuSnGvWH3snLMSvzt/cmtyuWhHQ8S4B/N8LqOjQAEDJaD1HbDQamkRiOXxPlZdcfSrGm27nIDvAfQ2aUzNYaaRlm049nHmbJlCkujl/LSwZcY/utwPjv1GUmlSXx37jue3PMk5bpyurp25ZexvzAxaCJWGqurnmto24Yyx3rHso6RUpaCtcaasYFjr7p/a+dn78ecvnPwsmk6vvBa6rNnZv5+KM3lJihmvr4orKyQamrQJicDUL5bfu6sB92C2sUF+3HjAChasfJm3IIgCILwL9OqAzSDwcCcOXPw9/fH0tKSwMBA3nvvvUZlH5Ik8eabb+Lp6YmlpSXDhw8nPj7+KkcVhH+Pc+mlGIwSHnYW9PaXOws21yikoKKWtCK5DX1CXgUTvjrC7j/Z8fFwQgFGCULcbfG0lxtToK2USxDrbX8ZEvY0f4DLXdoMuXInPKI3Quz2ptvsfgcq88A5SJ7X6XJWTuDbr+5YV8+iWZmpWT/FjWBlJpJSjVXYbSSXJvPQ9od4/9j7xBbFNt7B0hE63yl/feJ7AGIKY9BJ1UgGS7QFw/C1CaJKX8XSo+/Bzjfky+37CF+mbTMd5t1j73Iy56Tp+++jvkcv6enn1c80ZutyAfYBrBy9kqFthqI1annz6Jt8Hfl1s/eUVpZGYW0WKoWaLyc0HXt2JYVCYZpzbE3sGgqqC/j54s/M2DWDktoSAu0DcbNyo7i2mCUXlzB+w3i+jvwaCYkpIVP4ceSPjUoZr2Zwm8EoFUpiimLIrpAnH18VuwqA8YHjsdbcWLfI/09q4+QGIRbBDdlHhUqFRd14tPoJq8vrxp/ZDhsOgOM9d8vLd+9Gl5v3t12vIAiC0Dq06gDto48+4ttvv+Wrr74iJiaGjz76iI8//pgFCxaYtvn444+ZP38+3333HcePH8fa2pqRI0dSU/P3zxclCDfbmTS5vDHcz5EgdzkLFNfMOLT6TJePoyU9/Rwpr9Uzfekp5u2Ow2i8wXEsdR+A1I8/u+Xy7Nn+uVCaBvZt5OYdkhF+fUjuZtgSoxH2fyR/bVfXuXHbC42acpB2DE7XZXrGzQN1My3XQ+syMdcI0ADa5O4DQOE3EKOFPW8cfoNTuaf4JfYXJm+ezN1b72Zd/DqqdHJQa2oWErMJynM4mSsHWhpdO9ztLHmym5wBW5G+m2KlgpiO43gt/wAgl/CN9BuJ3qjnuf3PkV6WTnp5uil7Vl9u2BwbMxu+GPIFs7rOAuSuh+XapgH4kawjAHR370aI2/UFTv29+tPZVc6iTd0ylU9PfYpRMnJ74O2sHruaHXfsYMHQBQzyGYRSoUSj1PBOv3d4o88baFSaa5+gjpOFE11duwKwN30vWRVZHMw4CMCU0CnXfZz/j0wdHEMal4dePg6tNikZbWIiaDTYDJK7YlqEhGAZHg56PSVr1nAzVJ0+jaS/eeXPgiAIwl+nVQdoR48eZfz48YwZMwY/Pz8mT57MiBEjOHFCHu8hSRLz5s3jjTfeYPz48XTu3JmlS5eSlZXFhg0b/tmLF4Sb4FSKPF4rvK0jwe5yU4zmOjmeSy8FoJefEyum9+GBvr4AzNsdz4xlpyir0V3fCY1GWDoe6csulMXKAcigYDd5XfZ5iPhG/nrMZzD+K2jbF2pL5W6GLY0ti9kklw+a28EjO8G+LZSmN3Rq1Gvr5jxDbgriN6D549SPQ0uLgMqC5rcBKM2AMz+b9lkTu4bzBeex1lhzq++tqJVqogqieOvoW0zaNInS2lLw7AxteoNRD2eWciJH/h0zs88Itj9zCyP9h9HeMZQqjHzh5MCT+jSq9TX08+rHy71e5v3+79PRuSMltSU8ufdJ5p2eh0Ey0N+rf7PZs8spFUoe6/wY7RzaoTVq2Z26u8k2RzLlAK2/V/+rHutyCoWCWV3kwC+3Khe1Qp677P3+72OhtkCtVDO4zWC+GvYVuyfvZtukbUwKmnTdx79cfZnj3rS9rIldg1Ey0tuzNwH2AX/oeP9fNHRwbDxm0DQOLSaG8j3y623duzcq24ZSXMe7pwFQsmYNku46f36bIRmN5H/9Nan33EveF1/84eMIgiAIfx/1P30BV9OvXz8WLVpEXFwcwcHBnDt3jsOHD/P5558DkJycTE5ODsOHDzftY29vT+/evYmIiGDq1KnNHre2tpba2oZGC2Vl8hgfnU6H7k/8IRSEK9XqjcTmlNPByw6V8sYaJRiNkimD1tXHFgXy/nG55U3+n0bWbdfRyxaFZOCN0SGEedowZ1MMu2PyGL/gMN/c3ZV2bleftFeRuAd18gEUwFfS23xodh9dvYaiq61BtekplJIBY/vxGPyHggRM+gn1kpEoipMxrr4Hw91r5QYf9SQj6v0fogAMPWdgtHJDMepj1L9MRTr2Dfr2E1Em7kGVfwnJygX9kLegpZ9Baw/UHp1R5JxHH70Fqes9Ta8/9TCqddNRVBUgWbmQ3aYvX+6Ts2OzOs9iashUimqK2Jy0meWXlpNZkclvsb9xX/v7UHS9D3X6cbQX13HWXs4i9vPsiY2ZAr1ezwzPwTxbfIn1tjZQW4yfnR9z+81FMkioUPHZwM+4b8d9JJUmkVSaBMCMjjOu+3fKbb63saBkAZsSNzHWr2HcltagNQWMvd1739DvqB6uPRjlO4pLxZd4vdfrhLuFo28mi+KgcQD4w7//bvG6hU/5lNO5p00lpHe2u/M/8/tU0mrRJidjFhxsaogi6XTUJsn/D1SBgY2eC3WwHLDVREdjqJazuFZDhjTaxnLIEFQuLujz8ynevgPbUSNv+LoM5eXkvfY6lfv3y99XVaPValEoFP+Z10YQBOHfqFUHaK+88gplZWWEhoaiUqkwGAx88MEH3HOP/MYsJycHAHd390b7ubu7m9Y1Z+7cubzzzjtNlu/cuRMrq6sPiBeEG7E+Rcn+bCWh9kbuDzJiff2VY+RUQWm1GjOlRMrZIxgkUKCiuErHLxu3YVt3LEmCU8kqQEF56gW2FcljvSyAp9rD4lgVyYVVTPj6CPe0M9LFueWSx96Jn+MBlCodsDeWMEexhLTFCZRbeNMhOxKdyoo9qmHUbmsYe2Xr8RgDy95Fk3aUwq+Gc8pvJlqNHQBexSfomR+DTmXFztJA9HX7hTv0xqfkONUr78e6Vh5jc8Z1Mhn7Iq76nAQr2tGe8+QfWsKJLMeGFZJEQP4OOmSuRoGREsu2nPB7hh/3vU+FrgIflQ82CTZsS5TP74orAxQD2MhGfj73M45JjpgblIxCyaXSRKqsPbBUWBJ3NI4EhdwsJCQrgvYqLTHmZlgqLJkoTeTQ7kONrm+yajLf8z06dASrg0k/kU466Ve9p3rmRrms83TuaVZuWYmD0gGARF0i1fpqbBQ2xB+NJ1GReF3HqzeAAQxQDiD3VC7b2HbtHf4gD6UHOcYcSrWl2CvsqTxfybaov+58rYnz9u0479tP8YAB5I+Tg2uz7Gz89HoMFhbsOn26oSMpgF5PkEqFsbyc2vNRABwzGjBsa/x8OXfpgvOePWS89x5ZaanU+vhwvcxy8/Batgyz/HyMajV5EycQ17UL/P47AFVVVX/yrgVBEIS/SqsO0NasWcOKFStYuXIlHTp0IDIyktmzZ+Pl5cUDDzzwh4/76quv8txzDU0IysrKaNOmDSNGjMDOzu5mXLogoNUbeevjA4COS6VKvk205pu7uxLq0VDGJEkSx5OLSciv4M7u3phrGrrzrTmVAeei6ebrxLixPQGYH3+I9OJqfDv1oU+A3DQkvbiKymOH0agUPDJpZKNjANxZqeWZX85xPLmYH+NUTO7ujZttwxgvjUrBuC6e+CrzUZ89B8A7Lp/imLmP1zWraFt02LSt8tZ3GBZ+d5N7VSQFI619ANeKaEalfYh+8s/g0Qn19/+T9+s3ixG33NmwQ0U40nd9savJBMDoP5jO097DqTKTEzknuD3gdtTKZn495fnB9+vwqIxmrFUk1GUVFfkxKDPlN7fGjpOxHv05qrxTXDxwEZVCxacjPiXYsXGZ2RD9EPas30ORrgjHbo709xoL5as4USw/B328+zD2loZMlurHz3mroIj5IX15tO+bdHHt0vT6gPZZ7VkVu4oXwl/Az86v2W1asm/3Pk7nnUYboGV02GgAvjz7JcTAYL/BjO3bejsipp5P5fsLcpOVezrdw7iO4/7hK/r7ZG7YSDXgePgwwUOHYn/nZMo3byEXsA4LY/SYMU32SV++wjRZtXnnzoxspuLD0K8fGYmJkJKC38JFuM6Zg92E8Y220aWnU7FzF8bLAi5Jp6N0zRqkykrUHh54zPuC4A4dGu1XXzkiCIIgtD6tOkB78cUXeeWVV0ylip06dSI1NZW5c+fywAMP4OEhz2uUm5uLp6enab/c3Fy6du3a4nHNzc0xN2/ahECj0aDR3ECKQxCuYn98LiXVOlxszLA0U5FeVM1di07w8eTO9At0Zu3pDFafTCe5oBKA1KIa3r694U1UZIb8BqqHn5Pp/2WIhy3pxdUkF1UzMEReFp0j79/e0w4bK4sm1+HhoGHF9D7M/f0Siw8ns/ZMZpNtfjySyubQnfghYfAfwpY4G7SGMTxyxzi8ds2E6iLw6YWq13RUymaGrobcCo/ugdX3oChKRLN0DHScDPmXwNweVb8nUV3+s+XoA7e+I7fqV1ugHPcFCo2GV468QnRhNBX6Ch7p9EjT83h1BqcAFEVJqI583nidQgUjP0DZ+3Fq9NV8dEpuTHJ/2P10cOvQ5FAajYaJQRNZFr2MNfFrGOw7GNrfzqkzlwDo7dW74fdBZQFkn6MDEgtHLALbZuZUqzPYd7B8rD9gXOA4Tued5vfU33m0i1yaGZEjZxUH+Axo1b+fRvqP5PsL36NRargz9M5Wfa03myE72/R1/v/+h2VAALpEOfNqGRra7HNh0SHMFKDZj7i12W00rq74r/2VrJdepmLvXvLmzEEXE4PbC89TceAAxWvWUBVxrMXrsurZE+95X6B2dm567P/Q6yMIgvBv06oDtKqqKpRXvBlUqVQYjUYA/P398fDwYM+ePaaArKysjOPHj/PEEy13ThOEK608nkZ8Xjmv3BaKufrqc0xdrw1n5UBoYjdvZg5ux9Orz3IovoCnVp1FrVSgr+uuaGWmokpr4OeIFCZ286ZLGwegYYLqHr5OpmMGuduyOyaP5Kw82PgpBN3KufQgADr72Ld4LWqVkjljw+gX6Myh+MYNNiLTS4hJz8P+0mpQQJT3FLQxRnwcLfHsNgQCDsKFtdDlbnli55a4tYdH98K6GRC/AyKXy8v7zgRLh6bbd39Q7gLpFAhOAZzLiyS6MBqAJReXMDV0atMW7QoFTPoBon6V962nVEGHidCmF0bJyCenPiG7MhtvG28e7/J4i5c8JWQKy6KXcTjzMOnl6XgEj+BM9JcA9LALbNgwcR8ggXvHqwZnf9atfrfywfEPiC+OJ7YoFkcLR+KK41CgME0I3VqFOIXw4cAPcTR3xMXS5do7/D8hSRK6ugDNqlcvqk6cIOOZZ9DUld5f2cGxnkVYGKX8BoDNsGEtHl9lY4PPVwso+PZbChZ8RfHKlRT/+mvDWE2FAuv+/THz82u0n5mvL45Tp6AQgZggCMK/TqsO0MaNG8cHH3xA27Zt6dChA2fPnuXzzz/n4YcfBuQuZbNnz+b9998nKCgIf39/5syZg5eXFxMmTPhnL17416jRGXh700W0BiOl1To+u7OLaaD/H1VWo2NXjDwP2YRu3jham7HkoV58siOW7w4kojdKdGnjwLSebRjXxYvX10exITKLV9dFsenJ/pTV6Emqy6x1a+tgOm59J8e2qb9B2TKI+pUcx28AC7r4OHAtw9q7M6x94zGbOoORrUs/xTG1ggzJhQcOOwBGBgW7ys+DQxsY8Oz13bilA0xbDQc+hAMfgZUL9G4hQFIqoed007erY1ebvi6pLWHVpVVM7zS96X4+4fKjGeXacl47/Br70/cD8EafN6460bKvnS/9vfpzJOsIa2LXMKztMKqVShwMBoKyL4F3H3nDhLrOiu1afiN9M9iZ2THIZxC703azNWkrAQ5yF8Qw5zCcLJyusfc/b0xA01K+/+8MRUVItbWgUODzzdekPfIINefOU1sqd1a1uKKDYz3rnj1BqcSiQwfM/f2veg6FUonrrFlYhIWR9eJLGCsqULu54TD5DhzuuAONt/dNvy9BEAThn9OqA7QFCxYwZ84cZs6cSV5eHl5eXjz22GO8+eabpm1eeuklKisrmTFjBiUlJQwYMIDt27djYdG01EsQmhOVWYrWIGdj1p3JJMjNlicGB15jr6vbHpWDVm/kEadIwpbNAn0tKuAV4EUbiQq/kdhP/R7UcsfDN8aGsT8un+jsMn48koy/ixyIBbnZ4GDV0BUxyE0ev9azbJe8QF/DtLwv2MwrpszbjdIoFUzQynOLrTLeSqlOfi4GBV/ffFtNKJUw5DXoeAeYWTefPbtCYXUhO1N2AnBv+3tZHrOcJReXMC102nVPdJxUksQz+54hpSwFM6UZc/rOYYB3Cy37LzM1dCpHso6wPmE95iq59LlHTS3K2G3Q40F56oHEvfLGgX9tgAYwNmCsHKAlbzXNL9bf+/rb6wt/L11WFgBqV1dUNja0+eorku+agr4uq2YeFNTsfuZBQfj/tha16/X/nNkOGULAls1oU1Kw6tEDhbpV/wkXBEEQ/qBWPQ+ara0t8+bNIzU1lerqahITE3n//fcxM2t4w6pQKHj33XfJycmhpqaG3bt3Exzc/CeWgtCcUylyKaGLjfz/6uMdl9hxsWkX0LIa3XXPJ7a+rrzxfvPDKKqLQVdleqj01dgnbJAna66bFNrFxpzXbpPnRvpiVzwbI+X9w30dGx030NWGAGU2HUlEUqgwqszpp4jiLrMIAl2v3kK/RZmnITsSVOaMeeBlvB0scbczp3+7hjK1vKo8JOkGJ7x2DQH76+s6ty5+HTqjjs4unXmhh9xco7S2lJUxK5tsqzfqTSWA9Y8tSVuYtnUaKWUpuFu5s/S2pUxoN+G6zj3QeyDeNt6U1pby80V5/rQeNTWQtB9qyyH3AlTmgcYa2va53rv/wwb6DMTWzJa8qjx2p8mZuxuZ/0z4e+ky5QBN4+UFyIFam2+/QeXggFXfPiitW/6AwaJ9e9QuN1YOqvHwwLpPHxGcCYIg/D8mfsML/3n1Y70eHxRIamEVy46lMnt1JGuf6EuYpx0RSYWsOpHOjgs5WJmrWPt4X9q52bZ4vKySao4lFwISPjVx8sKpq8C9rlFF5ilY+4g8mbJrqDxGC7izhw9rz2RwIrmILeflT9+vDNAszVTcb30CdFDiOZAMm050ilvA6+plqGqeB6s/UAZ3Qu68R8dJhLXzZ/+LvhiMEhYaFVW6Kt6OeJvfk3/nufDneKjjQzd+/GvQG/WsiVsDyNkslVLFY10e49VDr5qyaDZmcvCZWpbK7H2zSShJaPZYPdx78OmgT3G2bNoUoSUqpYq7Qu7ii9NfUGOoAaCXmRuUJcmljUXJ8ob+A0HdtLnQzWamMmOk30jWxq3FKBmx0djQybXTX35e4Y+pH39WH6ABWISG0m7/PhSXfZgoCIIgCNerVWfQBOGvJkkNk0F393XkrXFhDAxyoVpn4OElJxn62QHu/v44m89loTUYKanS8cjPpyiu1LZ4zE3nspAkGNkWVNUFoFBCwGBw9JUfHe+AEe/LG+98HeLk0j6FQsH/JnbCTNXwY3llgIYkMVo6CMB5p5GsMZ9ErNEHe2Mp7JrTeNvqEojZDPmxLT8BpZlwcZ38dU+5a6BGpcRCoyK9PJ37fr+P35PleZN+if3lxrNo1+FA+gFyKnNwNHdkhN8IAG7zuw1/e3/KtGWsvLTStN20LdNIKEnAUm2Ji6WL6eFm5cZDHR9i0YhFNxSc1ZvYbiJmSvnNtKO5I4FBdWOpLm1tKG9sN/zP3+x1GhvQ0E6/j2cfNErR6KG1qi9x1Hh5NlqutLBAcbWmOoIgCILQApFBE/7TkgsqKarUYqZW0tHLHrVKyVd3d2fiN0dIyq8EarExVzO+qxdjOnny0m/nSS2s4vHlp1n2SG/M1E3fgNV3b7y7bQnkAS4hYHZFo4q+s6AgFs4shbUPwyM7wT2Mdm42PDE4kC/3xONsbYa/yxXlURkncdNnUyFZsFfRk7NZlUTrHuE383fg7HLoMg1UZnB6CVxYB/pqeb+2fSH8QQgbD2oLSDksbxOzCQxa8OrWqPHGkcwjvHTwJcq0ZThZOFGtryazIpNz+efo6tb15jz5dVbFrgJgUtAk0xgwlVLF450f5+VDL/PzxZ+p0dfwfZSc6evm1o3PB39+UzsFOlo4Msp/FJsSN9HDowfKgHEQsQDidsilqQCBQ2/a+a6lm1s3vKy9yKrMop93v7/tvMKN02XXjUG7LIMmCIIgCH+GCNCE/7T68sYuPvamYMveUsPPD/Xim/0JdG3jwNjOXlibyz8qPz7Yk0nfHOV4chFzNlzgwzs6Ner4GJNdxqWccsxUSnpbpMsLPTs3PbFCAaM/g8IkSD0MKyaD/y0APG2UuLVNOdr2k5p2kzz/CwA7jD2JzK7lUnY5eimEio73YXNhGSydAMbLxsk5tJWzZGkR8uP3l8DKGYqSGrbx7AK3f2X6dmXMSj46+RFGyUgnl058PvhzFpxdwKbETWxJ2vKHA7T44njWJ6xnoPdAenv2RqlQklSaxPHs4ygVSu4KuavR9iP9RrLw/EKSSpNMwdnUkKm81PMlNKqbn1F6NvxZrNRW3N3+brD1BWs3eewZgKM/OP+5xjE3QqlQ8v6A9zmYcZDxgeOvvYPwj2nIoIkATRAEQbg5RIAm/KfVB2jhvo3HbrVxsmLupKaBVbC7LQvu7sYjS07yy6l0gtxtmD4wwLS+Pns2NNQNi4IL8kLPLs2fXG0GU5bB90OhOBnOyZkkFdARoGA7+Do1ZG70WjkrBmww9OdchtzG28XGDOsx70HKTqjIBbUldJwkZ8x8ekJ5NpxdIWfrStOgphTMbKDTnRD+gJw9q1Opq+STU59glIzcEXQHr/V+DTOVGWMCxrApcRM7Unbwcs+XbzhAqtZX8/Tep8moyGBZ9DJ8bHy4I/gOkkvl8V2DfAbhZdP4Da5KqeKJLk/w4sEXMVOa8UafN5gYNPGGznsjXCxdeL3P6w0LQkfLWUb4y9vrN6enR096evT8288r3Bh9fZMQTxGgCYIgCDeHCNCE/7SGAM3xGls2GBLixutjwnhvSzQfbIthxfE007qsErmkcEI3b9h1Xl7o0UwGrZ6VEzyyCy78BobahuWpERD3O6x5EKbvBtdgSNwD1UVI1u4cq+1g2rSzjwMKS0d4cCtknoHgkY1b29t5waAXYeBzkHwAqoshaCSYN+36eDbvLHqjHm8bb97q+5Ypg9fbozculi4UVBdwJOsIg9sMvu7nC2DhuYVkVGRgb26PwWggoyKDL898aVo/NXRqs/uN8h+FlcYKH1sfAuwDmt3mLxM67rIA7e8bfyb8exgrKzHUzXem8RYBmiAIgnBziABN+M8qqdISn1cBQPfLJoO+Hg/39yO5oILlx9JIrptQup6HnQVDfNVytgrA4xod+Gxcoc8Vkzn3fhx+vh3Sj8HKu+DRvabyRkWnyfhcsDOd1zRBtUuQ/GiJUnXNcVQnc04Ccvbm8vJKlVLFbf63sSx6GVuSttxQgBZXHGdqX/9uv3fp49mHnak7WRu3lnP552jv1J4+ni23r7/F55brPtdN5T9QLhHVa8Fv4D9zDUKrVt/BUWlri8rmD05zIQiCIAhXEAGa8J91Nq0EgABnK5w33A2FCfDYQbCwv+a+CoWC9yd04t4+vpTX6ButC3S1wTz3qPyNo991TdTchNocpixvKH9cfQ9knZHXdb6Ldnn6hgCtzbWv93qdyjkF0Gxp3diAsSyLXsb+9P2Ua8uxNWt5qoF6RsnIuxHvopf0DGs7jKFt5QBxQrsJTGg3geyKbOzM7VAqWmG3O7W5/P9BkprNNgpCcy32BUEQBOHPEgGa8J91KrUIgLHuBfJ8VyD/2/GO6z5GqIdd8yuy68obWxp/dg1xxXFkV2TD8Bdh15uQfwYnpZFOLiHg2YVg91h2RecCconjzVCpq+Ri4UUAero3DdDaO7UnwD6ApNIkdqfuvq7xYL/G/sq5/HNYa6x5pdcrTdZ72ng2s1crYnn9pa/Cf8+Vk1QLgiAIws0gAjThP6t+/Fn9vGIAJOy9oQCtRdnn5H+vNv6sBUklSdy5+U6MklFe4GwLyNmqHzwG0luhINhd/r6NkyVO1jdnMtwzuWcwSAZ8bHyaDZwUCgVjA8Yy/+x8tiZtvWaAlleVx7wz8wB4qttTeFh73JTrFITWwtTB0bOVf9AgCIIg/KuIAE34f+9sWjHxuRVMDvdBqZTHVekMRiLTS1BipF3ujoaNE/fIJW1XtrcHSNoP2iq5u9+15PzxDNqhzEMYJSOO5o742PoAkFuaQp6unN3W1vQGRnbw4I7uPtwa5nbDx2/JydyG8WctGR0wmvln53Mi5wQ5lTlXDbo+PPEhFboKOjp3ZGpI801ABOHfzFTi+Dc2CMlPTaYwI43Q/oNa3CYrvpi0i0VNlru0saVd+M37nSEIgiD8NUSAJvy/lltWw/2LT1BeqyelsJKXRoUC8nxlNTojIyxiUVflyqVsuhq5JX1eNLh3aHyginxYPlmeY+yBzaY5y5pVWwEF8fLXfyBAq2/U8XDHh3mw44MA7Enbw+x9szmadxoAC42Kz+76Y+WTLZ43+9oBmreNN93dunMm7wy/J//OQx0fana7A+kH2JW6C5VCxVv93kKlVN3UaxWE1uDvzqBJksSGT96jLD8PGydnfNp3bLJNdbmWzV+dR19raLIupI+HCNAEQRD+BUSAJvy/9s7mi5TXyk08vtmfSDs3GyZ19zGVN95vfQwqgQ6ToCQNEnZBwp6mAdrFdQ0TQG+eDU8cBY1F8yfNvQhIYOMBNjf2ZshgNHA6Vw7Ceno2BEq9PXqjVqhJK08jvSydNnZtbui411KhrSC6KFo+7zXm3hobOJYzeWfYlLiJBzo80KTBR5Wuig+OfwDAfWH3EeoUelOv9b+qtlrP4TVx+HZ0EW+y/0IF3y2kMiKi0TKlpSVuL76AeWDjycr/7kmqc5MSKMuXJ09Pj45qNkA7uysNfa0BezdLfDs6N1rn5tvCmFlBEAShVWmFrdME4ebYE5PLtqgcVEoFt3eR30C98lsUp1OLOJVajAW19Ko5Im/ceUrDZMT1DUMud35Nw9dFiXDos5ZPXD/+7A9kzy4VX6JCV4GNxoZQx4bAxsbMhi5u8vGOZB254eNey5m8MxglI21s21xzrNgI3xFYqCxIKElgwdkFTdZ/Hfk12ZXZeFl78USXJ276tf5XndmewqWIHPYsiaY0v/qfvpz/l/QFBeTPm0fV8eONHhX791P400+NtpX0evS5cqMe9d8UoCWcbAgcs+IuNVlfVaYlan8GAAPuDGLgXcGNHiG9xThQQRCEfwMRoAn/L1XW6nlzo9yRcPoAf+ZN6cqIMHe0BiMzlp7mWGIhtypPY2aoAgdfaNOrYTLitAjQXja3WWEiZJ4ChQpGfyovO/wF5DV9gwRATn2AduMNQurLDMPdw5uUBQ7wHgBcf4C2M2Unt2+4nV8u/XLt89aVVfby6HXNbe3N7Xmz75sA/BD1A5sTN5vWRRdGszxmOQBv9HkDK43VdV2rcHXV5VrO788EQK8zcmBVLJIk/cNX9f9PdVQUABofH7w//wzvzz/DZeZMAKpOnGy0rT43F4xGFBoNaheXP31undbA3qUxLJsTQeqFwma3STh5zPR1dtwlJKOx0frIXWnotUbcfG2bZM8EQRCEfw8RoAn/KgUVtdTomo6tuNIXu+LILKnGx9GSZ4b4oqwt5YspXQnztKOwUkthpZaJ6vrs2V1yUxDndmDfFgxaSDnccLD67FngEOg5HYJHyeWOW2bDFW+QgD+VQbtao45+Xv0AOJF9Ap1B1+IxDEYD807P4/kDz5Ncmsy8M/Oo0lVd/bx1AVoPjx7XdZ3jAsfxaKdHAXjr6FtE5kWiN+p5J+IdjJKRUX6jGOjz35jcWVujR9fMeJ/L1VTorrnN1dSXrTm4W6FSK0mPLiL+ZO4fPp7QvJqoKPQqC1Td+2I3ejR2o0fj9PBDoFSiS0szNQWpqdRRmSqXN6o9PVEo/9yf0rKCan77+DQxR7Mpy69my9fnOLUtpVEQXpSVSWFGGkqVCrW5ObVVlRRmpJnWV5VpiTogZ896jvVvNNG8IAiC8O8iAjThX+N0ahH9PtzLfYuPYzC2nD24kFnKj0eSAXj/9hCsVtwOnwRiffYHfrg/HFdbc5woY5CyLpDqdJf8r0JxWZnjHvlfSYLzdRmozlPkbUZ/ChprOdN2dmnjk+u1DZm1G2yxrzfqOZMrT0bdXIAW6hSKk4UTVfoqIvMjmz1GaW0ps/bMYvGFxQBYqi2p0FWwJWlLi+ct15YTUxQjn7eZ+c9a8mS3Jxnedjg6o45n9j3Dl2e+JLowGluNLS/3evm6j/NvlngmjyUvH2HFW8coL6ppdpuc5FJ+fv0ov354Cr32xoO06vKGsrX+d7Sjx2hfAA7/Gk9NZcuBunDjUqMKONrnXX4vG8Tv30WRerEQhZU1Fh06IKEgbstptn5znh9fOMSq5WVcaP8Qpd5d/1Q2Mz26iDVzT1KYUYGlrYagnu4gwfFNSWxfeAFtjTyGtr68sU2HzngFhQCQGRtjOs7Znaly9szPTmTPBEEQ/uVEgCb8K2j1Rl75LQqt3sjJlGKWRaQ0u53eYOTVdVEYJRjb2ZPBhb9Cxkkw6mH7y3jte5Yf7+nIY87nUGEEr27gGtxwgPoALbEuQMs4BcXJckAWOkZe5tAGhryGFth38F0qipIa9s+PkbNrFg7g0PaG7jG2KJYKXQW2GltCHEOarFcqlKYs2pHMpmWOccVxTNkyhSNZR7BQWfDxLR/zZNcnAVgdu7rFN5FncuXxZ752vrhbu1/39SoVSj4Y8AHtndpTVFPEkotLAJgdPhsXyz9W8lVdriX5fME/Ur5n0BlJOJ2Htlp/zW2NRomIDYlsX3QBXa2BypJatn5z3vRmul55UQ3bvo1CX2ugOLuSU9tSbvi6zu68rGytkzPdRvji6GlNdbmOo+sSbvh4/xUGg5HEs3nUVFw7iJUkiVPbkjmuvAW9xhoJBUmR+WxZcI5lbxzlkv8dHO3zHvtPW5FyvgBJAqNRQZ57D46Zj2TFW8c4syOVqjLtdV+fTmvg1O8pbF4QSW2lHjc/O+56rScjHunAkHtDUarla1j74SlKcqtIOCWXN7br2RevkDAAsuLkAK2qTMuFA3IJbK9msme62hp2//A1lSXF1319giAIwj9HBGjCv8Kig4nE51WgrpvH7JMdsWSXNm2U8MnOWKIyS7GzUPP2Ldaw/0N5Rfvb5TFk51fTacddPGZdNzl15ymND+B/CyjVUJgAxSkN2bP2Y8HMumG73o/zRttAnnayZtqG8SQm7pKXZ182/uwGS4xO5JwAmh9/Vq+/d3+g6Ti0Sl0lM3fPJLMiE28bb5aPXs5t/rcxvt14LFQWxBfHcybvTLPHNJU3ul9feePlrDRWzB86H1dLVwC6uXVjcvDkGz4OyG+St317nm3fnCfuxN9bvicZJXb8cIEd319g3/IWxhbWqanUsfWrc5zZngpAx0HeWNqZUZhRwe6fopHqsrvaGj1bvzlPdZkWK3t5MvGzO9MozKy47utqrmxNpVYy+B45gI85kk1WvHjTfSVJkti//BLbF15g548Xr7qttkbP9kUXOL4pGRRKvHOOcNdLXek81AdzKzUVRbUklblSa+GIxlBF1+FtuPvt3gxxPI135iHUSgOledVErE/k51ePsH3RBdJjikz/D65UkFHOwVWxLHn5CMc3JiFJ0L6fJxOf74aNo9wZNmyAFxOf7461gznFOVWs//QQ2XVNQQJ79MIruD0AWXUZtDM7U9HrjLj729G2g1Pj58JoZPs38zi363fWf/SOGLsoCILwLyDa7AutXnJBJfP3JmCGjg2dTnA0U8+CgnDe3nSRhfc1BBW/nkpn4QE5m/X+hI647H8a9NXgNxDuWgoph+DXBxuCKIUKOt7R+GQW9uDTC9KOQtwOuPCbvLy+DLLO4Zxj/K6SP5lPUcHdB5/lg+x7GF5aN7j/T8x/drU29309+wJwqegSBdUFpkzVgrMLyK3KxcfGh9VjV2Nvbg/IDT3GBIzht/jfWH1pNeHu4U3Pex0TVF+Nh7UH3936HasurWJ6p+lN2u5fr9QLheQklQFwKSL7b+04d2xTEsnnCgBIOJ1Hj9EVOHvbNNmuJLeKzQsiKSuoQa1RMvT+9gT1dCektwcbPj9L8rkCjm1MpM/4QHb/FG0qW7vjpXAOr4kn+VwB+1fEMumF7iiU1w7gz9Y3fbiibM2rnQNhA72I2nuMzV+eZcLzj+Du73Bd91pZWsvZnWm4trUlsLsras3NmaNOkiQuHMhEY64ipI/HXzoGqqpMS+TuNNqEOtEmzKnJ+rM707gUkQPIJYTZCSV4tnNosl15UQ2b50dSnFOFUikRHLOSANcKXAOccA1wou+EQBLP5pMVU4Biyce45p4l+J0daDysKc+PIyT+MP3v7Ui+T18uHswkL7WcxDN5xJ+MRq1KwKWtPw6XNQsqya0mL6XM9L2diwXht/nRvp8n2uoqLuw7QHlhPr0n3ImHvz13vtqDjV+cJT9Vzp55BAajKVBhn2WHUqmmJDebnORsLtZlz5obe3Z07Srijh1GqVIz+L7pYmyaIAjCv4DIoAmtmiRJvL5eLm38yG0XYbFfM71iISfMZzEy7i1O7N8CksSJ5CJeWy93YHt6aDtuV0XI7fJVZjB2npzN8r8FZhwAz67ywdsNa36esvoyxwMfQ3URWLtCwGDT6mp9Ne8fex+ASX6j6SmZU6VU8GziSuYnb8QA4HFjAZreqDdluK4WKDlbOhPmLJc3Hc06CsCFggusjFkJwJy+c0zBWb2poVMB2J26m/yq/EbryrRlXCq6dM3zXkuwYzBv9X0LbxvvP7S/JEmc3JJs+j4jtpiK4to/fD034tKxbFM2zMFd7jp5cmtyk+2MRoldP16krKAGOxcL7ng5XB4vBHgE2DP0fnlahDM70tj4ZSTJ5wpQqZWMfqIzds6WDJwSjMZcRU5SKRcPZ13zuqrKtFyoG3vWXNlaj9t80FdtoSJ/H7+8t4Y1/zvJhYOZVy3R1NUa2PLVOc7tSWf3T9EsefkIh9fEU5RV2eI+1yslqpCDq+PY83MMO3+42KTc82bJTSnj17knObszjU0LIjm5NblRtiopMp+IDYlAw+t5YkvT11OSJPYujaE4pwprezMGuUXjlX0Uy44Nc4upzVSE9PZgyIMd8XWvRSnpqTwhZ7rr50CzautFWH8vJjzXie4jqlApfkNbtpSq4qOknVvJpSMXiDueS9zxXPJSylAqFQR2d+X2p7tyzzt9cHAtY8d3X/Ld4/ezZ/E3nNjwK1vnf4LRaMDa3pzRMzuDQb4fiQCKfoml6mAOA9pMAmDjZ9sasmdXBKsxRw5w7LdVAAx/dCY+YU3nTRMEQRBaH5FBE/4ROy/m8PGOWHSGxl0QQz1smdqzLbcEu6JSKlh/NpOjiYW0V2czoaKu3NApAPOiJCapDsP+w9RGtuNQaT9sDP0I7+jKRf3/eP3ICSabm9G1z/MoXNo1nMChDTy8HaI3QcCg5i+u3TDY+x5UyRkVOk4GVcOPyrfnviWzIhMPaw9e7vcWZv3e4fMNU1hWlcT3thakKF34zKMzN/I5dUxhDJW6SmzNbAl2DL7qtv29+hNdGM2RzCOM9h/NOxHvICExJmCMaYxao+fUKZSurl2JzI9kbdxanujaMDfZ7tTdGCUjfnZ+uFn9c5Mfp14oJC+1HLWZEntXKwozK4g/mUu3ETc2ju9GZSeUmEoaw0f5EtTTndXvnyDxTD4FGRW4+DRk0aL2Z5CXWo6ZpZpJL4Rj7WDe6FjBvTwozqni1LYUMmPlssMh94XiESAHzLZOFvQeH8DhNfFErE/Ev4sL1vaNjwFy4JCTWMrJrcktlq0BpEQeRTLKjUkkQyr5ae04sDKWI2vjCb/Nj/CRvo2ydJJRYveSaArSK7Cw1qA2V1JRVMu5vemc25uOZzt7wgZ40a67G2qzG8uqXRlgJ5zOoyi7ktse74SD282baiH6SBYHV8Vh0BuxtNVQXa7jxOZk8lLLGf5QGGUF1ez6KRok6HiLN91GtmXFnGNkXComK74EryAH07HijueQcakYlUbJhOe7U/Lsd1QDFp07NXtu6169qDl3nqoTJ7EfP97U0VHt6cnZHVs4snoZtVVyoKtUqTC3dqC6rBALi0OEj3sBhVKJxlxFQFdXrOzMqCguYtWc58lJjDedw8m7DWV5uSSdOcmhlT8z6N6HsbSRMOrlbo3K4jYYjXLm3lPpT6h9b+LL0nBv15ERj3RoFMRnx8ey49t5APQYN4lOQ0bcrJdBEARB+IuJAE3420mSxKc7Y0nJK0aPptG61MIqdlzMxdvBksnhPiw7looCI4udl6Mo1UHQCLh7DbVpp9i59COG6g9iXZLA8yTwtMVK3qEzG/MLwFLNJksPAksjmBy9nHGB4xoySxpL6DKlmSur49EFrFwaArTODeWNsUWxLL0od258vffrpnm+XrpzIx32v8kbKevYZW1FnNJA0zYfLasvM+zh3qPF8Wf1+nv35/uo74nIimBp9FIuFV3CzsyOF3u82OI+00KnEZkfya9xvzK983TUCjU/RP1gmmh6SJshN3C1N9flb+47DfLBztWSAytjiTuZ85cGaGUF1fy+MAqjXiKgmyu9bw9AoVTQrrsbCafzOLk1mdsek9+slxfVcHyjXD7bd2Jgk+CsXq+x/pTkVRF/MpueYwOblGl2GuxD3PEc8lLLOfRLHIOmNfwv0euMJJ3N5+KhTIpz5GkRFAroPT6gSfZMkiTO7mjozGlpk03PCYHEHMmmOKeK4xuTyEspY/iDYZhZyr/mj29OIulsPkq1gtse64BHO0fSo4u4eCiTlKhCshNKyU4o5fCaeIJ7exDa1wPbujFR9SysNc2WZqacLyA/rRy1uYoRD4exf0UsRVmVrP3wFMMfCsOv0401jdHV6tDVGk33LUlwcksyFw7KpXz+XVwY/mAYiWfzOLAyjpTzBfw69yR6rR59rQGfUEcGTAlCpVIS2t+T6ENZnNiSzIRnuwFQXaHl8Fq5wUrPMX7YO5mRc1Eeq2bZqWmAJhmNWPXqReH3P1B1/DiGkhKk6moMCgV7Nq0l5sh+AOzdPeg0ZAQdBg8HSeKn556gLD8FhRRFt1vHNdyftpaNn75PTmI8ao0ZwX0H0HnYKLxC2hN79CBb53/Cqc3rcPZug9rMDKPBgLWjB151v8NqlQrMjRJdnAZTW36IsS+FY2bR8Oe8rCCfDZ+8h0GnIyC8FwPuug9tejlmbWxv6HUQBEEQ/hkiQBP+djHZ5fjpP6YoJJU3XO+kTaenANDqJXZF5/LbmQwyS6r5co/8yfLTjsfxKj0LGiu5xb1CgblvTxymfkvvxfsZrzrKfWb7sFalsUWfDwoFt1ZWcdjWgcTSJD46+RELzi7gvf7vMcLvOj5FViohcChErQHnILnTI/L8Yu9GvItBMnCr760MbjO40W5jBr/Lnt1F7Mo8wJaU3wlxCbvu5+R6xp/V6+zaGWuNNcW1xcw/Mx+A53s8j7Nly621b/W9lY9Pfkx+dT5bErdwMOMgu9N2AzA5eDJPdnvyuq/1ZkuJqsuemavoNqItCqWCQ7/EUZBeQWFm82PB/qzinEq5gUe5Dte2tgx/MMwUePQY40fCmTySzuZTkFGOi48th36JQ1drwCPAjg4DvFo8rkKpwMUrkeiKn7C2mQUENFqvVCoYfE8ov354isQz+SSeyW/2OGozJUE93Ok4yBs3X7sm67PiLpGfkoRaY4YkGSkvyMO3g5quw3sTczSbA6tiST5XwK8fnmL0E53ISy3n9O+pSJKEd2A8v77/Fbfc/SDdRo3Dt6MzlaW1xBzJJvpIFuWFNUTu2MvJdVvRWI1EZd7w/9jZ25rbn+mGlZ2ZaZkkSaYSws6DffDv4oqbrx3bF0WRk1TG1m/OM/zBsOseU1hwNp6138RjUFk0XamQg+Aet/mhUCpo388LJy8bfv/2LPmJP2PUZ2NhF0anQVNR1r+et/lx6Wg2mbHFZMUX4xXkyNF1idRU6HDysqbrrW2pjY9DqqlBaW2Nmb8/IHc+jI04zPk928mOu4R3UCguTnZ4ZGVRdfIk1Ro1Z9r5UHpkPwqlklvueYjw0eMbzYk2cNoD7PnxWw6vXkq7Xn2xdXJBkiR2fDOPnIQ4LGxsufuDz3D0aPg/Fdp/EIWZGRz7bRW7vv8aZ582AHQYOACfWDPQGjhXrsNFpSXAworu1n0w5JaDryMARoOBTZ/9j6rSElza+jHmqReojS6maHUsFmHOuNx//b+XBEEQhH+GGIMm/O0ijuylyi6eWqWCI2nLCCeWcF8n+gY68+a4MI6/Nox5U7rSy9+JEJsanjbUzTU25DVw9DUdZ2CQK5P7hbHXZiy6Rw6wqPdUDAoF/WsNfB50L3um7OeN3m8Q5BhElb6K5w88z7zT8zAYr2Muqp7TwcYdBr1s6sa4Jm4N5wvOY6Ox4ZVerzS725jgiQBsS9rW4nlSy1JJL083fX+t+c+upFFq6OPZBwCDZCDcPZyJ7SZefR+VhjuC5YYobx59k91pu9EoNbzV9y3e6vsWZiqzq+7/V7k8e9Z5sDeWtmZYWGtMDTGup5uj0WAkK6GE2qrrmxMsKTKfXz88RWleNTaO5ox+ohMa84aspbOXDe3C5XLPk1tSSIrMJ/lcgSm4ulpzD8lo5My2jRj1enZ9/xVpF8432ca1rS09x/g12+TTpY0Ng6YF8+BHAxh6f/tmgzOAyLrsWXCP3ni2lQOKlHOnUSgUhPX3YtLz4dg4mlOSW8WvH55i77IYJGMNVlY7iT+2GX1tLUfXrkKvldvCW9ub02O0H/e915exT3VGKZ0AJPS1kY3OW5hZye/fnUd/2WTxyecKKEivQGOuwq+TivKiAqwdzJnwbHfa9/MECfYtu0ROUmmLz9vlzi4/2mxwZmVvxpgnOtNzjH+j18DN1xZHt2MY9RmAgZqyKNZ/+DpLnp/J6a0bsbBR0L6/HACd2JxMZmwxl47K5YlD7g1FpVJSHSWPX7Xo1ImC9FT2/PgtCx9/gB3fzjN1T8yMv8S5Nq7sDfNlz4ofORLsQ6mZCktbOya//j49xk5sMmF1l1tvw7NdCNrqavb+uBCAiLWriI04hFKl4vbnX2sUnNXrN3kawX0HYjToyU+Vfz4C2/VAozVgVECVrRl+j4STp0tHrdRQ+HM0hroW/2e2bSQ3KR4LaxsmvvQmZpZWVETI92vmZd3kXIJwPQoLC3FzcyMlJeWfvhThXyQlJQWFQkFkZORNPe727dvp2rUrRqPx2hv/S4kATfhbGfR6eke/Q5yZXNp4yNIc7S/3yC3t61hoVEzo5s2ax/qyo/3vqLWl8qTPvZ9ocry3b+9AxKvDsLcrZXO23DRj5sTVcOs72JrZMiV0CmvGruGBsAcAWHxhMbP2zKK09hpvFtv2hhfioPOdAGxI2MCnJz8F4Jnuz7Q4Xmug90DszOzIq84zlS1eLrEkkYkbJzJ63Wge3vEw25K2EZkXSZW+Cjszu2uOP6tXP9ZMrVTzZt83r6sz253Bd6JSyIGIm6UbP4366Q+3xL9ZLi+N63prQzljcC852xJ3IqfFduVlhdUc35TE0teOsv7TM6x+7wR5qWXNbgvyGKzjm5L4/bsodDUGvIIcuPPVnqbW5pfrOdofFHIwVz9GreuIttfM5mXGRlOWnwfImYzNn/+P4uzMpscf488T3wxh5reNH1Ne70XHQT6YW7Zc3FBZUkzcMXmaBed1m7E5LHf4SznXMI2Cu78dd77aE68gB3Q1BvS1BUjaXyjOuohaY4aFrR015WXERhxqdGyFUoFCyqK2Sr4HyZDDfR90Zua3Q7j77d6YW6nJSSpj3/JLSJIkB9h1DVXahZvxy9vPsvSFJynKykSlUTLk3lD8u7hg0BvZ9u15ygqbTo1xOX1ZOalF8nPc8eIPDL/wFo992J2Z3w7hwQ/749e5aank8XW/kHBCDnhuffRJOg65FbW5OUWZ6exf+j07vplH+ChflCoFmXEl7PjhAgAdbvE2jQ+siIwk3cmW/cpalr70FJE7tlJbVYm9mzsDpt7PfR/Np/9d92JtZo5OrSKppgKtWoWj2px7P5xH247NT0qvUCq5dcaTKFUqEk5GsGvRV0SslRv6DJ8+izZhzY93UyiVjHriGdwDggCwcXTCukx+XqzaO3Pv//rTtpMLWe6plGkLocpI8bp4yvLzOPLrCgBuufdh7Fzd0GZVoE0tA6UC616eV33+/wsefPBBFAoFCoUCjUaDv78/L730EjU1zU80/0ccOHCAoUOH4uTkhJWVFUFBQTzwwANotQ3z5EmSxKJFi+jduzc2NjY4ODjQo0cP5s2bR1WVXOL89ttvo1AoePzxxxsdPzIyEoVCYQqW6t8Eu7m5UV5e3mjbrl278vbbb5u+f/vttwkNDcXa2hpHR0eGDx/O8ePHr3lPH3zwAePHj8fPz8+07OTJkwwbNgwHBwccHR0ZOXIk586da3Su+uf68oe19dU/KGhun9WrV5vWnz17lm7dumFjY8O4ceMoKioyrdPr9YSHh3OirpnPzZSfn4+ZmRmVlZXodDqsra1JS0u76ee5EX/09byaMWPGsGjRIgBmzJjBu+++ezMu9YY8/fTThIeHY25uTteuXZusHzVqFBqNhhUrVvzt1/Z3afUBmp+fX7M/rLNmzQKgpqaGWbNm4ezsjI2NDXfccQe5uX/vHEr/dfti89gXm3dd26Zun4eVOo3quk+aK5VKjkuVsHIq1Fz25tpogMhVEPUrKJQw7stGjTqutOj8IgySgQHeA+js2vjNklqp5oWeL/DRwI+wUFlwJOsIU7ZMIbEk8ZrXqzPo+ODYB8w5MgetUcuwtsO4M/jOFrc3U5kx0m8kAFsStzRZ/+25b9HVDfI/mXOSlw+9zCM7HwHk8WfX26J+VJvR3Kl/lDkh7xFgH3DtHZDb4b/W+zXuCLqDX8b9QhfXG58K4Ga6sjTO0qYhi+fX2RkzCxUVxbVkJZQ02i87sZTNC86x7I0ITm1LobJUCwqoKK5l3SdniKnLjlyuOKeSrd+eN00U3XmoD7fP7tqoVO9yTl7WBPWQOzTWVOiwc7Gg52i/a95TzKH9AIT0uwXPdiHUVFaw/uP3qKloOvdZc7/XrkfU3h0YDXocqmqwzSvEpVx+M5cedc6UEQOwsjPj9tldCehSgb5yNdrqQmxdXJn67sf0GDMBaMjEXS5yZ+NlSaeOo1AocPSwZuSjHVEoFcQdz+XMjtSG7JmFisrCwxh0OmoqK9jw8bvUVFSgTUmmm+IETq5yQ49tzUzmfbmEZdupNXdEbazB07YCY0EB+Z9/0eLzE3fsMEfWLAdg2MNP0Hn4KEY+/gyPf7eUIQ8+BgoFsRGHqC7LIqwui1ZdrsPKzoy+EwKorapk9+JvWRd/jqg2bhRUlqFUqQju3Z87Xn+PR778nt4T78LNL4A+d0zl7unP0DMpC8/icvzySxjZqSd2LldvruPq60/4WDnDfX7PdgDCx06k09Crl1trzC2Y8OIbhPQdyOAHHqUmWn4TahnWUMrs3j6Ew3nrkJCouVTEsUXL0dfW4tO+Ix2H3ApAZV32zLKjM6oW/r//14waNYrs7GySkpL44osvWLhwIW+99dZNOXZ0dDSjRo2iR48eHDx4kKioKBYsWICZmRkGQ0Pm+b777mP27NmMHz+effv2ERkZyZw5c9i4cSM7d+40bWdhYcHixYuJj49v7nSNlJeX8+mnn151m+DgYL766iuioqI4fPgwfn5+jBgxgvz85sutAaqqqli8eDGPPPKIaVlFRQWjRo2ibdu2HD9+nMOHD2Nra8vIkSPR6eS/cS+88ALZ2dmNHmFhYdx5Z8t/Q+v99NNPjfabMGGCad306dMZOnQoZ86cobS0lP/973+mdZ999hn9+/enV69e1zzHjYqIiKBLly5YW1tz5swZnJycaNv2r21kdS1/5PW8GkmSOHbsGP37y3OuHjp0yPT13+3hhx9mypSW+wU8+OCDzJ8//2+8or9Xqw/QTp482eiHdNcueULg+h/wZ599ls2bN/Prr79y4MABsrKymDRp0j95yf8pcbnlPLzkJI8sOUlq4TVadZdm4HXmU2LMGr9J2GvvDPkx8NsjUJoBBz6BL7vChrpPDXvNAO/uLR42rSyNLUnym8qZXWa2uN3ogNEsH70cbxtvMisymb1vNlqDtsXtC6oLmL5zOqtj5U/uZnadyeeDP79mE4+xAWMB2J22mxp9w6ey8cXx7EyR//B+M+wbZnaZibuVO0ZJTtH39ux91ePWKy+qYceXMTif7EjJWgdqKq+vtA/grpC7eLvf26b50/5Jl5fGdb21TaN1ao2KwLoyw7jj8nxWkiRxbm866z87Q9rFQpDAJ9SREdM78PDHA/DrLGdq9i6N4eDqOLTVemKP57Du09OsfPs4qVGFqDRKhj8UxsC7glGprv7rr8doP+pbcQ6+O/SanQ31Oh2xx+SMVOdhIxn/4hvYOrtSnJXB5nkfYtD/+bbz+ooKzqyV/z/6FpRiP348XgMGYq7To9frSDl8sNH2tZXlxB9bhtFQS5sOnbl37jzcA9rRadhIVGo1OYnxZCfEmrYvLywg4aSckQu7ZSgACaeOmda3ae/ELVPkrM6xDUkcWCXv2y5cQ/xx+d6t7B0ozs5k/RsvkDj5TkrmfUboppcxN1ZRmFnJzu+jMDaTFZUkidgIOdvo62XE5505AJSsWUPV6dNNts9JjOf3r78AoPvo8XQePsq0ztzKmu63jSOkzwBALivsPsoXpVp+QQdOCcbcSsPO7+ZzbudW9AqwqtXRb+wkZnyzhHHPvYpf525NShZteoTjWq2jW1oeYVmFWHr7tPRSNdL3jqnYu8kBf0D3ntxyz4PXtZ+NkzNjZ79MYEhPdNmVoACL9g0dPb2C21OuKyK9Rn4dHHKdUKrUDH90FgqFAmOVjqpI+cMzm74tj538rzE3N8fDw4M2bdowYcIEhg8fbnpvAWA0Gpk7dy7+/v5YWlrSpUsX1q5da1pfXFzMPffcg6urK5aWlgQFBfHTTz8BsHPnTjw8PPj444/p2LEjgYGBjBo1iu+//x5LS0sA1qxZw4oVK1i1ahWvvfYaPXv2xM/Pj/Hjx7N3716GDGlo2hQSEsKQIUN4/fXXr3lfTz31FJ9//jl5eS1/YHr33XczfPhwAgIC6NChA59//jllZWWcP9+0HLvetm3bMDc3p0+fPqZlly5doqioiHfffZeQkBA6dOjAW2+9RW5uLqmp8rQlNjY2eHh4mB65ublER0c3CvRa4uDg0GhfC4uGSoeYmBgeffRRgoODmTZtGjEx8oTtSUlJLF68mA8++OCax/8jjh49agpWDh8+fF2By/79++nVqxfW1tY4ODjQv39/0/MDsHnzZnr27ImFhQUuLi5MnHj14QpX+iOv59XExsYiSRJhYWEUFBSQkJBA795Xf29ytZ+HP2r+/PnMmjWLgICWP4AeN24cp06dIjHx2h+2/xu1+iYhrq6ujb7/8MMPCQwMZNCgQZSWlrJ48WJWrlzJ0KHym4mffvqJ9u3bc+zYsUa/TIS/xpd74pEkkIDlx1J5fUzLA9ANW1/EwljNfjN/wIC/vT/Jpcnss3fijYJCVPE74YsODTtYOEC3e2HoG1e9hoXnF2KQDAz0Hkgn1+ZLhuqFOIWwcsxKJm2cREpZCosvLOaJLk1LJ+OL43l81+PkVedho7Hhw4EfMqhNC235r9DVraspCNyfsZ9RfvIbx+/OfYeExK2+tzLQZyADfQYyo/MMjmQdIaU05aqZuXqZscVs//4CNRVyUFZdpiViQyJD7gm9rms7tyedpMjGn6yp1Ao63uJDQDfXZvfJii/h/L4MwgZ40jas5UYkN+Ly0rhOQxpnz+qF9PIg5kg2CWfy6TupHYfWxBF3XM6OB/Vwo9ftAY1auI9+vBMnt6VwcksyUfszuHAw01QeqVAq8OvkTM+x/rheZyc7J09rRj/eCb3O2OxkyFdKPnuS2spKbJyc8QnriFKpYsJLc1j15oukRUVycPmPDHlwBgDl+/ZRum4dNkOGYnfbKJR1b9zqadPTKfl1LdWRkXILwzqphblU22ow0xvo/PhTuN5/H1J1Ne4PTCUNIxe/mod/rz6obORyuFNb1qOrqcbNL5DJr7+HUqVCm5FJ4fvv421UkgYcmfMqPdU2mAcFkRDgjWQ04tO+I30mTSH64F7SL56npqICi7pjdhzkQ1F2FVH7M6gq1crZs4LDSJKRgPBe9L/zHla99hxZ2RmoHa3o6uqCRUYmHc9+zdlus0m9WMz+9zYw9K3Gb0TKjkSQYyFPidFhYnesunhhP/kOsjZtYvf7b1HdpSOXz11RkJ6GXluLf9dwevceSO7cuRgqK3F/5VVUNnIJVd/J04g9dpiEkxH0nZzFmJmdqSrVEtjdlcTTJ4g7fgSFQkn3pEw8za0Ivvehq2YyldbWWHbsKL8ugMb7+oIejbkFE19+i8TTJ+g6cgzKa3zIc6Xq6EIAzP3tUVk3dL51D2iHSq3mfP4BfHyC8bIKZMCt9+DsLX/gUXk6F0lnRONhjZlf8+MZbxZJkqjWXcf43pvMUqP6U5NvX7hwgaNHj+Lr2zC+ee7cuSxfvpzvvvuOoKAgDh48yL333ourqyuDBg1izpw5REdH8/vvv+Pi4kJCQgLV1XL5roeHB9nZ2Rw8eJBbbrml2XOuWLGCkJAQxo8f32SdQqHA3r7xXJYffvghPXv25NSpU/To0aPFe5k2bRq7du3i3Xff5auvvrrmvWu1WhYtWoS9vT1durRcUXHo0CHCw8MbLQsJCcHZ2ZnFixfz2muvYTAYWLx4Me3bt29UBnm5H374geDgYAYOHHjNa5s1axbTp08nICCAxx9/nIceavjZ7NKlC7t27aJdu3bs2bOHzp3lqpnHH3+cjz/+GFvbm9etNC0tzXT8qqoqVCoVS5Ysobq6GoVCgYODA3fffTfffPNNk331ej0TJkzg0UcfZdWqVWi1Wk6cOGG6j61btzJx4kRef/11li5dilarZdu2bab93377bZYsWXLd4/6u9/VsztixYzl8+DB6vZ7q6mocHR0xGAwYDAZ8fOQPokpKSprd92o/D83x8/PjwQcfbFR6+0e0bdsWd3d3Dh06RGBg4J86VmvU6gO0y2m1WpYvX85zzz2HQqHg9OnT6HQ6hg8fbtomNDSUtm3bEhER0WKAVltbS21twyS4ZWVyaZ1OpzOl5oVri8stZ1tUQznZmlPpPDU4AMtmMg2KS1tRx21DJ6k4bOEFpDMteBrzI+dTqC3l7NBX6LHzbQCMbfti7HofUug4uSU+QAuvy+XZs0c7PHpdr5+typYXur/Aq0df5fvz3zPcZzh+dn6m9YXVhczcPZO86jz87fz5/JbP8bXzvaH/G6N8R7H44mI2J2xmmPcwEkoS2JkqZ8+md5je6Fh93fvS170vGDGVP15JkiQu7M/i2IYkJCM4+1jTeagP+5bGEn0oi3bhLngE2je7b72cpFIO/9p8mUx6TDFdhvvQc5yfqfudJElcPJhFxDp5IuDEs3n0HONL1xFt/tQbIoCUy7JnHQd7NvvcuvpZY+1gRmWJlpVvH6O6XIdCCX0mBNBxsBcKhaLJft1G+uDkZcnepbHoagzYOJkT2s+DkD7upjnHbuR19AlzuO59Lh7cC0Bw34EYDEYMBiOO3m0Y+cSzbJ33IWe3b6HLyLGoM7LIemY2klZL+a7d5M6di+2YMdhOnIA+PYPS336jOiKi2XMkBXgCGtr3GYjj3dPQ6/Wg0RBy34OkrfiRHH0tGc8+h+eC+VRXVhC5Xf7Z6DVpCgajEYPRSN6CBVTs34+PlTlpQT6k62sIOh+L+tRJIjsGgFJBx2Gj/o+98w6Povr+8Ls1vfcGaSSEEggdQm+hCoIgIAoWEAS/ClYsIIiIiF2qIIoCiiKC9N57CSWNJISEhPRedrNtfn8MWVhSSGiWX97n2edJZu7cuTM7MzvnnnM+B2tnVxy9fchLvU78mRM0Dr81QdF+qC/5maWkxuQT0ErOhW1HAGjTewBlH86jRWIqZ33dSXG2w/ep52jWuj1Fmzah2bmFSx6PE3PDBu/v/sBv/C3p+Zi1B9DJO2AhVeMcYEfUoX1ckpSTFnIzhCjmcqXzYW9rT7PIOJJ/vGXs6XJycf/yCyQyGbZuHgS1D78ZCrmGQdNEcZ+ykmL2rlwCQJOGAbhFxmPeva14Pu+CeZvWRgNN4uJS6+vJ1s2DsAHiC3ldf2fKLouTKsrG9pW2dfULJD0+lmsll/G3CaWBLgitVotgECg5LhbTNm/nWunYHvRvnUqrp8nMnQ+0z9oQPScCS2XdXmW2bNmCtbU1Op2O8vJypFKp0aApLy9n3rx57Nmzh44dOwLg7+/PkSNHWLZsGd26dSMlJYWwsDCjsXS7QTJixAh27txJt27dcHd3p0OHDvTq1YtnnnkGW1vRSI6Pjyc4uPaFWFq1asXIkSN566232Lt3b7XtJBIJ8+fPZ/DgwUybNq3al9YtW7YwatQoysrK8PDwYPfu3Tg7Vx9VkZycjKen6WSEjY0NBw4cYOjQoXz44YcANGrUiJ07dyKXV/4+1Go1a9as4e23qxbYup05c+bQs2dPLC0t2bVrFy+99BIlJSX873//A0RD76WXXmLhwoWEh4czY8YMfvrpJywtLWnbti0REREkJiYyatQo5s6de9f91YSnpyeRkZEUFRXRpk0bTp48iZWVFS1btmTr1q00aNAAa+uqc5OLioooLCxk0KBBxu8iJCTEuP6jjz5i1KhRzJ4927jsdsPK2dm5VoZHXb/PqlixYgVqtZpJkybRoUMHxo8fz8yZM7G3t2f69Ok1blvT/VAVAQEBdR5fdXh6epp4JP9L/KsMtD///JOCggLGjx8PQEZGBkqlEnt7e5N2bm5uZGRkVNvPxx9/bHJDVLBr1y4sLR9cUdX/OquuSBEEKaGOBlJLJeSpdHy8dhcdXE3Dl5TaIrrHvY8cWKofiMpMDMcpiCnAH38ucpGVKVcoDv4Qg1RJibkHXAeu7zfpJ0ufxS+lv5BryDUuExAwYCBYHkzKqRRSqF3CriAINJI3Il4Xz2s7XuM5q+fEF35By/cl35Ohz8BJ6sRoRhN1JIoooup0bqz04gz+kbQj/LblNzarNgPQTNGM+GPxxHP3fILbKYg1oyRJ9DJZemoxb5JBfHYGlt5mlKUq2bbiPG7hZVSXwiYYIPOoJSDDwk2LhcetlzVNvoySZCUX9qQSG3kNxxYqpDLIjzKnLE2crVfY6NEWyzi9JZnLpxNwCFUjlYvOnfIcGaXXFZQXyLAPKcfSo+aXXEGArJtjMfcuY9/B3dW2lToqocAMVbEWqdKAY0s111UXuL79QrXbADh1kKBXSVE6FJOhzyHjaOWX+8zjB9AWF+HZoz9ShaKKXu6ORKvFe/l3SNPTudrYB6QSLL5cROxXS8nt25fC9u1AIsHCzQNVZjqbv/mc1rsOItdoUPn4ICstRZmXR+Evv1B4WxK8IJFQ1qgRxaGhGMzE772srITcq1GAhAKvBiYzrfpy8WW7xMKM3BPHuP7aa8S52KEtV2Pm6ExMehax27YhLS3Ff+tWpEB5z95YZF9HpS4luns4TtGxYn0trZ4bh46QkFeIYOcEqdc5vWoF+llzQQJFrVpT1LoVgo81zjYyEi5vQhAM2CstKXjxJeTFxbgoFHi6NeBG1nUOrV1FfNoNrLx8YLwHDjuvkS/x5djBUpKKv0Ud4I88v4CSLAtwBqltFstffg5d2a2QaZeiUjwKSk1i8qUGA87FV9EbBASplNLgYCzj4yk9cIDTr7xKzoD+AGicRMGZq2dPsvHn1Zg5OpNz7gTFudnIrayxvSYaMSlKJZG3ndPqsBSgIrBx/+XLGK5ereNVUzfkWgmh1+yRIOFYxgU0286brFfJxesjuuAYfjbN0SYWcfDXXch0Ehrl2aCTGTiUcRbDHYdWIUTx/5EePXqwZMkSSktL+eKLL5DL5QwfLqrcJiQkUFZWRp8+fUy20Wg0hIWJJVcmT57M8OHDOXfuHH379mXo0KF06iQKN8lkMlatWsXcuXPZt28fJ0+eZN68eXzyySecOnUKDw8PBKFq4aOamDt3LiEhIezatQtX1+rzHiMiIujcuTPvv/8+a9eurfb4IyMjycnJ4bvvvmPkyJGcPHmy2n5VKpVJiGHFsueff57w8HDWrVuHXq9n4cKFDBw4kNOnTxvDOSvYuHEjxcXFjBs37q7H+v777xv/DgsLo7S0lE8//dRooDVt2pSDBw8a2+Tm5jJr1iwOHTrEyy+/TKdOnfjjjz9o27Yt7du3Z/DgwZX2MWnSJH7++Wfj/yVV5AkDyOVyfH19Wb9+PW3btiU0NJSjR4/i5uZWrYe0AkdHR8aPH09ERAR9+vShd+/ejBw5Eg8PUawnMjKSCRMmVLv91KlTmTr17iVw6vp9VoW7uztarZYTJ07wzTff4Ovry/Hjx1m1atVdDa6a7oeqqGmSoa5YWFj8Z59l/yoDbeXKlfTv37/STE5dmTFjhsmMQFFRET4+PvTt29c4w1VPzVzJLCby5iz/vNHhHIzP4dNd8Vwos2d2/w63PCy6cmRrhiHV5nNV8OBbSQ+UkkjkUjnjBo6jYVpDLh65yDXFNcIHf1WtZyZfnc+4XePIMlSOrVdKlczqPYvGjrUL86ugZUlLRmwdQZIuCV1jHY/5P8b7x9/neuF1bBQ2rIhYQUPbhnfvqBp2bd9FbH4s8a7xRCVEIUHCzD4z8ZY3RKaQmki710TG1SI23zRIOg675T0CUHfTsn7uWdQl4KFoRquIqhOWz+1MIa0kGQsbBSOmdcDcytQgSTybzcG1VyjPkVNy3hkzKwVlaSVIpGKh5OY9PIk9lsHR3xNRZSowu2SLf5gL8acyKc675Y0uvGxJ116huPraoi4pRiKVYmZpqtiVdCGHtOIYFOYyHp/QrdJYbqewjYoNn5zDwcOKPs81xtqxitpY90Buagpr1n4HQAMrM5P8pbpQsGYtOSkpXHe0QZBKsFGVY1cm5h26bdxIIwk4v/MOV50d2Pb1AooTYpGUlqAMDsZ/9Y9IzM1RnThJ4e+/k3/oIGa2dtgPHYrtsMdReN/KbyrNz+PXWW8C0KhDOP1HVE6c/jXyBJmJ8eTYWOJ24gQlTcRrt++zE/ELE8s35H//Pbk6HWYhIXSaNw+HQ/vY8923FKPF0D4MkhLxyS3E+6c12I19iuwb2ewBikqLkGdlIRMEXLZtw2XXLqx79ULTvCl/Jovx/00vxyFXaZB7e+Px5Rc0Cgpi73ffEn1oH7knD9Hzg/k4eflQ1rWc9e8eptTaCw5cos/QoeT8voktjk0RBAGFPBZdWSlWDo407d6HJl17UjpnLqVJByods9zbG7vhw7EZOgS5szPFW7eS+fYMHA8epHHv3tgOFT1WO/OziDt2CHlWKm06d+bXdSsBGPDSq0hmfoIWCB0+HMsaXioqMPRQcX3vXmQODkQMH37f3uS7oTqbRdGZq8g9LOn9eOWokMzgRvw2ZwaNenXBwtoV9blsQtU+IJOgoQDbdp70G1D5uCoiRx4UFgoZ0XMiHmiftd1vXbGysiIwUAyn/f7772nRooVRBKPiRX3r1q14eXmZbGdmJnrh+/fvT3JyMtu2bWP37t306tWLKVOmmAh0eHl58fTTT/P000/z4YcfEhQUxNKlS5k9ezZBQUHExsbWacwBAQFMmDCBt99+m5UrV9bYdv78+XTs2JE33nijxuMPDAykQ4cONGrUiJUrVzJjxowq2zs7O5Ofn2+ybO3atVy7do3jx48jvZmruXbtWhwcHNi0aROjRo0yab9ixQoGDRqEm5tbbQ/ZSPv27fnwww8pLy83fge3M336dF599VW8vb05cOAAc+fOxcrKioEDB3LgwIEqDbQ5c+bw+uuv33XfTZs2JTk5Ga1Wi8FgMHpedTod1tbWNGzYkKio6idwV61axf/+9z927NjBr7/+ynvvvcfu3bvp0KFDJSP2Xqnr93kn8+bNY968eQiCQFlZmXEiorS0lIiICCQSCdu3b682NLU298PDIi8vr1Iq1H+Ff42BlpyczJ49e/jjjz+My9zd3dFoNBQUFJh40TIzM3F3r74oqpmZWZU3uUKhQHGPM+n/31h0UMwfGtjcg2Y+jng6WvPVvkSi04u5lF5K64YOoqtky8uQehKN3IYJpdPxalBKNtDIvhGW5pZ0a9ANpVRJakkqKaUpBDoEVtqXVq/lzaNvklqSipe1F4t7L8ZSfsvTaaO0wUpR9/o+vg6+ovDH2c/5MvJLkkuS2XZtGzKJjM97fE6gU+Wx1IXBAYOJPRPL7wm/IzVIGaJ4mvh15eyLOYGTpzUj321rDCesDr3OwJFfEwBo3MmDVn19TdYr7BV0HtGIPauiOb/jOsFtPbB3M/UCF2SVcX6HWHet84hG2NhX9hI37uCJi4+tKIWeo6Y4rxxzawURLzTFu7GYfxXavQGuDezYsewSBRkqzm0XvZVmlnKC2rtTmKUiJSqXnd/FEPGCDxvnv4muvJxG7cMJ7RWBd5PmIMC57eJYQnt4VzmW23H2UvD8wi7IFNIH+iIcf+KI8e9Le7bTqt+gOvdvUKspuPmilBXaBPJzCH1yDIG9+lP4119kf/ElRX9sRJOQiP9nC7GQylAZ9GR5e9BlyWIUN/NMlN26UublxoGCNKzs7Hls+FAsff2M+9Fqytn65XxK8nJx9PSm78SpVT6n/Fq2ITMxnjwfD0qzctBpNLgHNKJR245IJBIEvZ6i9b8B4Dj2KZRKJU269uDIuh8pzsmmOCcbqUxG8x59Kf/lVwp/XoMCMA9piFopxzB5Al7uXhSs/w315cuU7NzJ+Zjz4GCDW2EJHs1aYD9yJDZ9+yC9+Xzt++LLFGZlkhYbxZbP5zFm7mfYOdkRProJ+9clctWtJ56vvku24IrgFYq5LJqM+MvIFUpGffAJ9u7iTLPj4kXosrNN8vGQSJG7OJsIeTgOHYo+JYWcxUvImjMHC38/LFu3puMTo7ly/AhJ506Tl3YdQTAQ3LEL7qWuFIZOR+4ejWVQ09o9/xUKArZtrSQg8rAojCsAwLKZS5Xj827chJd//B2ZXI4+v5yMyBw0CbfKiNh08qpyuwf9WyeRSOocavhPQCqV8s477zB9+nTGjBlDkyZNMDMzIyUlhW7dqs87dnFxYdy4cYwbN44uXbrwxhtvVPtC6uDggIeHB6Wlold4zJgxjBo1ik2bNlXKQxMEgaKiokp5aAAzZ84kICDARHK+Ktq1a8ewYcNqFU4IoijK7WkfdxIWFmbibQLRAyuVmj6XK/6/szZVUlIS+/fvZ/PmzbUaz51ERkbi4OBQ5Xvb3r17iYmJMYpS6PV6Y/huTWG8rq6utfIwbdu2Da1WS69evViwYAGtW7dm1KhRjB8/3ij1fjfCwsIICwtjxowZdOzYkbVr19KhQwdCQ0PZu3cvzz777F37qAt3+z7vZNKkSYwcOZLFixeTmprKvHnz+O2339i1axfffSdOZN45WXEndbkfHhRqtZrExESjQflf4x+v4ljBqlWrcHV1ZeDAgcZlrVu3RqFQmLhL4+LiSElJMcaO1/PgiUkvYtulDCQS+F8vUdHN0UrJYy1Ez+ZPx6+JDY9+CRfWgUTGx9ZvkSh44edZAGD0dlkprOjgKc4K702p7PYWBIEPT3zI2cyzWCus+bbnt/jb+eNu5W783ItxVsHYJmMJdgimsLyQH6J+AOCd9u/Q1rUdCWezapQE12r0xJ/ORKupOjG+v19/zPSWtEsexNhzs3E70orr0XkgQG5aCSmXc6vc7nbO704h70Yp5tYKwofdMhgFQSDx7CmKcrIJaueGTxNH9DoDB9bGmoxZEAQOro1DrxOFLhq1rX720snLmhEz2uIdrMPRPZthb7QwGmcVuPvbMeKdtvg2d8IryJ5e40MYNz+crk8GETGhKU5e1pQVlrPxk88pLy1Fr9MRe/Qg6+e8w6ppL7Jt0UoyE4+A/gIyyWXO79xS6RNz9CDCbT/wcuX9iQDciWAwEHv0tvCY1BRSq8hvuhsF69ejy85G6+1FZn4OSCQ0iRiEwt0d5wkT8Fm+HKmdHeqLF0kaOAjvNDGP6EbzYBS3RQHodTp2L/sGnaacwuxM1r3/BjE3xycIAjsXf0lGYjzm1jYMfWsm5lZV5zv4thCVTrOVcpKdxJe7dn0HGs9dyaFDaNPSkNnZYXvzOapQmhnl2AEC23XC/4MP8PxkPmaNGuE4ZgyNOosvqelKGQ4jR+L3+2/4/bEBybAhpNuLyfjd35tDw59WYzd4kNE4A5DJFTz22jvYubpRmJnBX59/jF6nJaRrAzz9rDDIlFw278QN66YIBhVlxaLh3GH4KKNxBmJdMIWbGwp391sfN9cqjSTnqVOxiYgArZbUqS+T99PPSA4cws9TFM4ozMxAqVDQ0sqZwh3iRJPctQk5PyWhuVF1mNOdPCrjzFCuRx0vei4smlYv0CNXKJBIJMgdzbFqfeseN2tkj8KlPmz/bowYMQKZTMaiRYuwsbHh9ddfZ9q0afz4448kJiZy7tw5vvnmG3788UdANJQ2bdpEQkICUVFRbNmyxZhbtGzZMiZPnsyuXbtITEwkKiqKt956i6ioKKMnZ+TIkTz55JOMHj2aefPmcebMGZKTk9myZQu9e/dm//79VY7Tzc2N6dOn10pa/KOPPmLfvn3Exd1SaS0tLeWdd97hxIkTJCcnc/bsWZ577jnS0tJqlL6PiIggKirKxIvWp08f8vPzmTJlCjExMURFRfHss88il8tNVChB9FJ6eHjQv3//Sn1v3LiRxo1vRb/89ddfrFixgsuXL5OQkMCSJUuYN28eL7/8cqVt1Wo1U6dOZfny5UYvXnh4OIsWLeLChQts2LDhviXiGzZsiLW1NZmZmQwZMgQfHx+ioqIYPnw4gYGBJuIyd5KUlMSMGTM4fvw4ycnJ7Nq1i/j4eOO1MmvWLNatW8esWbOIiYnh0qVLfPLJJ8btv/32W3r16lVt//f6fd6Jo6MjgYGBREdH079/fwIDA4mPj6dv375Gz1xN3r6a7oeq6NWr111FbBISEoiMjCQjIwOVSkVkZCSRkZEmtQRPnDiBmZnZf/Z9/19hoBkMBlatWsW4ceNMkk/t7Ox4/vnnmT59Ovv37+fs2bM8++yzdOzYsV7B8SHy9V4xf2pAcw+C3W+pJT3TUXxQbbuUQdH5jbBHzPPb4vUqqzL8kUpAYibme4Q43bp5ezUQH0D7ru+rtK/V0avZmLARqUTKgq4LqvSw3Q8KqYJZHWchuSkP91TIU4wMHsmFPdfZ+d1ldiy/XG2+wL7VMexaGcW+1TFVrne2cOaJa6/Q6kYfLLW2WNoqad2vIY07it7dSwdTaxxbQVaZsW5X5xGNMLe+NVMXe/Qgfy6Yw/rZb6PTlNNtdBAyhZS0uAJ+ePsoB9bGkZ1SzJWTGaTG5iNTSOk2Ouiuhk5eWgJJZxZzI+Ynfnn/JQ6t/YH8jBsmbazszBg4pQVDp7eicQcPFDdFYZTmcga81By5LIHykkQkEjmPTX+HFn36o7SwID/9BrFH/kSn2oe6aC+H1nzHvu+XVvps+/pTTm36varhPRAqikkrLSyMUvIVYhq1xaBWk3NzZjGvm/jj4NOkObbOt0ItrDuH4/f7b5gFByNoNDTIK0IqlZKZdp3MqwnGdme3/kl2yjXMrW1oGBqGTlPOtq8/5cDqFRz7bQ1xx8UizI+99g4O7tWHd3sEBmFmZYVGU45eJsWuVI3NsVvFWvPXrgPAbvhwpLflk7TsOwBuXhdhfUXDzW7IEPz/2oz7zPdpHDEAgMSzJzEYxMmIQnMlhwozQQKBbTvi3bn6XAxLWzsef2sWSgtLUmMus2eFqHbWY3xzpFLIc2xCoV0AOtUhtOUlOHk3oM3guklN345EKsVz/seYN22KPj+fzI8+IvPDuXjvPWz0wAVdTUP9ZyQISgxleSCUos9Tk73kglGW/p9A8b4U0AnIHM2Ru9XO0LLpIeZCQr20fm2Ry+VMnTqVBQsWUFpayocffsj777/Pxx9/TEhICP369WPr1q34+YmebaVSyYwZMwgNDaVr167IZDKjV6tdu3aUlJQwadIkmjZtSrdu3Thx4gR//vmn0SMnkUhYu3Ytn3/+uXF5aGgoH3zwAUOGDCEiovpQ0ddff71aUYrbCQoK4rnnnjMpwC2TyYiNjWX48OEEBQUxePBgcnNzOXz4ME2bNq22r+bNm9OqVSvWr19vXNa4cWP++usvLl68SMeOHenSpQs3btxgx44dxhwrEN/ffvjhB8aPH49MVjkctbCw0MSIVCgULFq0iI4dO9KyZUuWLVvG559/XmWdutmzZzNw4ECTQsZff/01kZGRdO3alcGDBxtzC++HAwcOGKXwT506hbe3t8kxVoelpaXJ+Z44cSJTpkzhxRdfBKB79+789ttvbN68mZYtW9KzZ0+TAts5OTk1SsjX9vvs3r27UbuhOnQ6HUePHjXm1R08ePCuOXYV1HQ/VEViYiI5OTk19vnCCy8QFhbGsmXLuHLlitELeePGrfeRdevW8dRTT/1ntSMkwr1kqz5idu3aRUREBHFxcQQFBZmsU6vVvPbaa6xbt47y8nIiIiJYvHhxjSGOd1IRTlBYWFifg1YDOr2BbZcz+N+680gksPPVrgS5mcrZDv/2AF43dvGZ+fcoDCp2WA5mUt5oAN7u35jfMieSpcrip/4/0dK1JSCqJvZY3wMBgV3Dd+Fh7UGptpQ/4v/g09OfIiDwVtu3GNtk7EM7ts2Jm0ktTmVi6ETkUjnr550mO6UYgD7PNSGonen1dO1SDlsX3aozMmhqCxo2M53hjjuZwZ5V0QgyA+FjfQlt549MJqUwu4yfZ54AAZ6a08FEKr4CQRDY/FUkqbH5eDd24LFXWhqNK4Nezw+vTSY/XXxQtR3yBF3HjOdqZDbHNiRQmH1L3lYilSAYBDoM9ad1P18Sz54iJ+UarQY8hsLMNJ+rKDuLNe9Op6ywQAyXuk31rUGzUHo+Oxknb9N6ZXeiKinm+1deRF1ShNy8E75h/bG0VaLXlZOTcp7sa9FIZdCwmRMyeeX5IY2qjGsXziFXKBm3cJGJF8V47i+c43r0JToOH41cWVme/27sXv4tF/fuoGn33rQeOJTVb0xFIpUyYdH32DiaKkvlr1+PoawMx7Fjkdw2OZT3449kfjyfch8vzjbxoyAjnb6T/kfzHpWLDxvKyshdsRKlb0OOXIsj9uhBmvXoQ8SkVyjMyuCH16ag05QTMflVmnTtwbH1azi5cb1JH31f/N9dCxsD/PX5x1w5eRSANlfTcdPoCdi9C6GsjMR+/UEiIWDXTpQ+pt9j1MG9qEuKaTVgSCUjXq/TsXTiWNSlJTz5wXyKsrPYvfxbdFoN9u4ePPHuXGONr5pIijzLxvmzEQQDrQcOpe1jw4k+WsCpv5LQa6+jLRHDL0fN+RSv4OpnX2uLLjubnKXL0OXe8lRfKyuizKAjxMoZiXk/kJgjES7i+ExfSk6WU35F9BBYd/XGrr/vQ80vM6h1FB9MReFhhWVo5RyK0rOZ5P92BQDH0cFYtqh9wn/ZhWx0eWpsunkjqSaMuv53r566sHXrVt544w0uX75s9FbV8++gYcOGzJ49+65G2r+JnJwcgoODOXPmjHHi5L/GvyJgvG/fvtV6MczNzVm0aBGLFi16xKP6/8P1vDJ+PX2d9Weuk1UsxjUPCvU0Nc5yEuDcD6wt+hkzZT4Y4ISkJVPyRmJtJueLJ1sS5idj0fosJEgIcrhlaDtZOBHmGsa5rHP8EPUD5fpytiVtQ6UTDY0ngp7gqZCnHuoxPhbwmPHv4jy10TgDOPJbPA2aOhnFLLTleg6tE1+crOzNKC0o5+C6OEbPbG8U/lCXajn6u+hp7DAwkLCOvsb+7FwsadDEiZSoXC4fTKPziEaVxnPlVOYtz9eYYJMXxdijB8lPv4FcoUSn1XDmrz8ICe+Gf0s//EKdSYsvIPpwGonnszHoBRw9rWjZpwHFuTn89fk89DodcccPM+T1d7FzFQ1PjaqMjQvmUFZYgIuvPyNnzuN69CUu7dlB0oVzpFy+yIZ5M3lq3udY2TtUex4Pr1mFuqQIa0cPtIa2pMbenljuh9LajzYDfWk/uOrik4Ig8Pvc90i5fIE9Kxcz/J05JseeFhvNnwvmoNfpMLO0ot2QJ6odS1XcXky6SZceuDTwxatxU9Jio7i4ZyfhI29dZ3lr1pD5oSjRXLJvP15ffoHc0RFNYSEXfv6RawGe5FmbQ0Y6CnMLgtpXHUojtbTE5X9ieE7L2Ghijx4k9shBuo59jj0rl6DTiEWkm3brhUQiofOoZ3D1C2DHoi/QlqtpPejxWhlnAAFt2nPl5FE8GgXTQGKDOjKS3BUrkNyU97Tq2qWScQbQtFv1YTQyuRz/1u2IPrSPHYu/oDBLrEXnF9aGAS+/Xm3I5Z34tWxN93EvsP+H5Zzd+ifnd/xFQOv2WFg3oCBtDwAt+vR/IMYZgNzFBff3TWsoVsiuFB9KpXBbEjJHc9xfm4xEJsU8RKBodzLF+69TcigVqaUc2+41T0jcK9qsMnJ/ikZ3czKl/Goh9oP8kdyctCi/Vkj+H+Lzw6aHT52MMwDLFv/NpPl6/j4GDhxIfHw8aWlp+FTxDKnnn0lUVBR2dnY888wzf/dQHijXrl1j8eLF/1njDP4lBlo9fw+CIPD4LzOIy72G6sZIMJjhZKXkiTbevNzzplFRmgObpsCVHQCYAZk48ouuO8t0g/B1sWX5M20IcLHmSJqYX+Jr54ulwtRr1LNBT85lnWNt7C1ZYF9bX0YGj2RU41EPXSntdpIuiLlCbn62aNR68tNLOfZHAj2fFl8cT21JojhPjbWjGSPebstvH5+mOFfN6S1JdBouhmAe25CAqliLo6cVYX0rKys27+5FSlQuscfTaT/E3xgmCFBaWG6sV9ZmgK+Jh82g13N8gxiq1uGJ0WQmxhN/6hi7l3/L6A8/RSKV4h3sgHewA6piDSnReXg3dkAmk3Lyz9+MXrHs5CR+njGNga+8SYNmoWz9+lNyUq5hZe/A0Dfex9zKmkZtO9KobUcKszLZ8PEs8m+ksunTuYyYNQ+FsnKydmrMZS7tE2u9DXzlVaQyL7KSTZXilBZygttX792WSCT0fuElfnxjKskXzxN75AAhXcR8hsKsTDZ99pHxGM789QctIwaiNK+9EtadxaQBWkYMJC02ikt7d9Bh2EhkcgUlR46SOe9jcSOFgrJTp7g4cgR5A/sSe/Yk5Y6WN8crxS+sNR2GjaqkVlkVnsEhuDT0Izs5ib8+E41gmVxO7xemmFzjQe3DcfMLICs5iYDW7Wp9fCGduyNTKPFp0gzD5SiuP/8CBb/8iuRmXpjjmDG17ut2Att2IPrQPqNx1mH4KDo9MabOuVhh/QZjYWvH+W2bSU+II/7UMeAYAFb2DnQefXcJ7vvFoNFTfDO82LaHDxKZeAwSqQS7CF9k1goK/rpK0c5rKFwssGj6YOr1VKCKyiFv/RWEcj1SKzmGMh2lJ9LRppfiNDYEQWsg96do0AtYNHPCts+9K8nWU8+D5NVXX/27h1BPHWnatCkXL168e8N/GW3atKmxaPt/gXoDrZ5q2R4dR6JmK3Ib8Gq0gRmtP6FvE0+UFaFpN87DL2OhKBUkUmjUF1qP588b/nyxM4G+Tdz4bGQLbMxFz1NMrpirVZUcfoRvBEsuLEGr19K7YW+eCHqCNm5tHqlhVsHVSNFAC2ztiquvLRsXniPmaDqNO7ijMJdzYa+oQthtVDCWtkq6jg5m2+KLRO69TqN2bmhUOmKOiQW8u48JBvTcme7ZoKkTts7mFOWoiT+VSZPOYq6ITqNn+9JLqEtuGnd9TI27mCMHKMhIx9zGlrCIgZSryki+dJ70hDgu7N5Oy4hbIjoWNkqjMVSUk83lfWIR2QFTX+Pcjr/ISLjCH/Nm4RXShNRoUTlvyBvvmeRRAdi5uvH4m++z9t3XSE+IY9fSrxnw8usm341Oq2X3cjHpt3mvCLwbi/Hvno3s63z+HTy86DBsFEd//Yn9q1fgG9YGqVTGnwvmoCoqxNU3AI26jIKMdCJ3bq2TFy3m8AEAGod3QyoVjeJG7Tpi5eBIaX4e8aeO4+fuTdq0aaDXYzV4MAXtW3Fu3U/kyiVw0/tmrtER0rINbaa8Uul81YREIqFlxEB2L/+W69GXAGj/+JM4elZWyLJzdTd6OGvdv1RKcMfOAAidOmHRqhWqc+cQNBoUPj5YVSOTfDd8Q1thaWePTlNOvynTadT23pKyJRIJIeHdCAnvRta1q1zat5PoQ/vRqFX0fG5Srb1x90PpiXQMpVpkjuZYtqrsmbIO90KbraL0RDp5v8bhMskcpef9j0swCBTtSaZ4n/j8UPrZ4fRUYzSpJeT9EosmuYjMr88jNZdhKNWh8LLGYWRwtSGK9dRTTz31/HepN9DqqRJBEPjy2Ba4meJTJL1InOZXBslfExecXwNbpoG+HBwDYNQacBU9TC8Gw9DWDXG1MTN5iY/JEw20Jo5NKu3P3cqdbcO2IZfKsVX+ffkQ6hItN+JFiWq/Fi7YuVjQpIsn0YdvsP/nOJTmMgSDQEArF3xDxZl1v1BnAsJcSDyfzYGfY9GoRSGFpl08Sbm8m7Xv/ohP01Ca94qgUbtOyBUKpFIJzbp6c+yPBC4dTCUkXMy12vdTLJlJRZhZyun/YnOTPC2DXs+JDWLibdvBw1BaWKK0sKTzqGfYt2oZh9f9SGDbDlg7VlZ7O3XTe+bTpDkhXXrQqH04e79fwuX9u0mNFhUMIya/gkdgcJXnxcHDi8HT32HDvPeJPXoQRy9vOg4fjUatIvboIS7s3kbejVQs7ezpOub+JYPbPjaM2KMHyU1N4eDqlaiKC8m5noyVgyND33yflMsX2LH4C07/9Qct+w5AaXH3JGF1SQlXz4kJ2E263FIZk8kVhPaK4Pjv6zi/dRPy05cp1KhJbxnC9dwUyn+LBrkoI+NSWIpPbhFedo4EvjMLyT1IlYeEd+fQz6soLyvF0dObtnUM06wtEokEl5enkvLscwA4jBp1z+qDCnNzxn+2WCy6bW1z9w1qgauvP72em0zXp55FVVyErXPdwvjuBRPvWc9b3rM7sR/sjy5HRXlCAbk/RuM6tSUym7rnO1YgCAL5v1+h7JwoQGId7ondAD8kMikWjR1xnRpG7upodFllGIpBaqPE6ZkmSJV1r/FVTz311FPPv5/6TM96quRYYi4pqvMAhDg0B+CHqB/YGPcbbHsDNr0kGmdB/Uh4ciWpFqYvbW625pW8X0YPmlPVBaUdzR3/VuMM4NrlHASDgJOXNXYuYuhcx6EBWNgqKcgsIyu5GKW5jC4jTcVqujwZhNJcRlZyMQWZZVjYKmnURs6x9WLtmOtRF9n29acsmzyOA6tXEH/qGGYWKQiGRDITIzm/8yintyQQfzoTqVRCvxebV6pnFn14PwWZ6VjY2Jp4ylr0HYBHYDAaVRn7f1he6ZiKcrKMoYcdR4ghbnKlkr4v/o/eL7yEtYMjXZ96lsbh1df8AVEopNfzkwE4tn4Nmz+fx9IXn2H38m/ISkpEJpfTZ+LLmNdCYexuyOQKek+YAkDUwT1cPXcauULJ0Nffw8bJmZDO3XHw8ERdXMT5WiowXjl5BL1Oh3MDX1wamsath/bqh1Qm40ZCHAfM9Bxu3IAEQUN5aQk2zi50GvkULyxaRb/BT+BlY4f7e+/ek3EGorHT8YnRWDs6ETH5VeQPsfaiZYcO2A4ejFnjxtg/cX9qZhY2tg/MOLsdhZn5IzHOAEqP3xC9Z07mWIZVL2wikUlxGtMYuYsF+sJycldHI2irLqdRG4oPXBeNMyk4jAzCfnCAiXGocLbAdUoLLMNckTma4/xME+R2lcOI66mnnnrq+f/Bv0LF8WFTr2ZliiAIjFh2lFjFa0jkZazuv5pjN46x9MJS5Ej4Lj2DkHIN21sN5zdKiM6LxkJuwYbHNuBjU3XycJGmiPB1oojCkVFHsDOrXITzn8C2JRdJupBTScgi/kwmu1ZEAdBtdBDNunlX2vbSgVQO/SKKh/R5PoSzmz/nxpUY/Fq2xj0wiEv7dlGSV0PtM4kFMmVTOj3xGO0ea2WySq/TsWr6JAozM+j61LO0fcz0ZTs7OYmf3n4FwWCgZcRAuj/zAjK5+OK/+7tvubhnBz5NQxk5c949nZfbObD6O85u3WT8397dg9Be/WjarReWdvb33f/tVCguAgx69S2CO94K0Ys+vJ/t336GubUNE75deVcv2q+z3yY1+jJdxoyvMizy98nPkZwnejgkUikBrdsT2rsfDUNbGsMh6/n3YijXk7HgFIZSHQ5PBGHV5u7Kk9ocFVmLIhFUOmz7+d6TaEjZpRzy1oiTU/aPB2Ld/u7y3I+C+t+9euqpp55/LvUhjvVU4mhCLucyLmHlV4aVwprmzs1p4dKCpISd7CxN4n9uLujl5pTl3arXodKp+OjERyzpvaTKvLG4PLHOiaeV5z/WONNq9GIhacC/pWleUWBrV3LTStCW62napXK+EECzrl4UZqmQm0kpyz/PjSsxKMwt6D1hKrbOLnQYNoqkyLNEHdxDSb64H125npzUEgRDkViLqfwMh9ec4VpkKG7+gcZzWZSTTWFmBha2drTsO7DSvl0a+tFt7PMcWP0dkTu3kp2cxOBpM9BrtVzeLyrkdXri3gQi7qTrWDFkTlVcTNNuvfFp2vyh5Qp2eWo8GrUK75BmJsYZQONOXTmx4Rfy09M4v2ML7R8fWW0/RTlZYiinRFKlp1AddwXfE+dRu9nh07MPrSdMxtrBsYqe6qlAX6Sh7HwWVm3dkFo+PC/gg6Lk+A0MpTrkTuZYhtXOY6dwtsC2ZwMKt15Fc4fgTW3QpJWQv1589ll38vzHGGf11FNPPfX8s6mVgbZ58+Y6d9ynT58aK4/X889EEAS+2HMFubXoCerk2RG5VA4xW/gw6jCpHi5EmZmBQYOvrS9PBD1BC5cWPL/zeY7eOMr2pO0M8B9Qqd+aBEL+KVyPzkOnNWDjaI6zt2mYnkQiocOQgBq3l0gldB7ZiJL8PH6Y/iMAnUc9bRSRkMpkBLRuV0mV7/dPzpBxtQAnj2zMzOJIunCW61EXuR5VWXmp7WPDUZibV1oO0HrgEOzd3dn2zWekxUbz89uv4NzAF4NeR4NmoUbVwvtFKpXR/ZkJD6Svu2FuZc3A/71R9ThkMjoOH8W2bz+7qeg4CLNqClZW1BXzCWlWSdRDMBjImDULK5WaboHh+Lz57oM9iP8o+RuuoI7LRxWVg8uEUCSKf27EvKFcR8khMffMpmcDJLLaTygobz4LtDdK6rRPfVE5OT9GIWgNmAU5YDew6tIS9dRTTz311HMntTLQhg4dWqdOJRIJ8fHx+PvX/yD92ziSkMPZ5HysfEWZ93DPcEi/AH9MwEIwsNgjgg0NQ2np2tJEZXFC6AQWRS7ik9OfEO4VXslLViEQEuL0YGocPQySbqo3+rV0rpVHKCPhCic2rsenSTOadO2JhY0YJrT/x+8oLyvFzb+RSa5YdfQe34Skizk07dIdpbmcopwsYo8eoqyo0KSdpa0drfo/Vk0vIgGt2/PUvC/YtHAueWnXjZ66jg/Ie/ZPIzi8K8f/+JX8G6mc3fonnUZUPs602Ggu7hHDJEN9Kj+TCtavRxUZidTKCvf33qu0vp7KlKcUoY4Ta9xpUorJ23AFxyeD/xbV1dpQcjwdQ5kOubMFli3rlu+m8BDLJ+gLNehLNMis7y4WIugM5KyOxlCkQe5qgdOYxnUyCuupp5566vn/Ta2nPDMyMjAYDLX6WFYzi13PPxtBEPhi9xWQliG1SAEg3K4RrBsN2jLw747jgC+YEDqBtu5tTV7Gnm/2PP52/uSp8/ji7BeV+o7NiwUgxPGfaaAZ9AaSLuUAlcMbq6IwK4M/5n9A4pkTHFi9gmWTnmHr159y+q8/uHL8MBKplD4Tp9Yqd8nezZKwPg1QmovzJbbOrrQb8gTdn37e5NNuyBPI5HefU3H09OKpjz6jUbtOgFgg2DvkwXjP/mlIpTI6PjEagBN//ELyxUiT9Xqdlt3fifL/3nlFCAs+J+OjeQhaLQDarCyyPvscAJdXX0XhXjdZ+/+vFO0Rnw/KBjYglaCKzDbKx//TMPWe+dTZUJKay5E7i9Eg2hultdqm+MB1tKklSC3lOI9ritS8Ppugnn83ubm5uLq6cu3atb97KPX8x7h27RoSiYTIyMg6b/v+++8zceLEBz+ou7Bjxw5atmyJwWAwLtNoNPj6+nLmzJkHso9aGWjjxo2rU7ji2LFj65OO/4X8dCKZcykFmNteBQT87fxx/+s1KEoDp0Yw4keQVf2ioZApmNlxJgAb4jdwNvMsABmlGSy5sISrhVeBR+9BEwSBPSsW8cNrLxnl5KviRkIh5aU6zK0UeATUnCNXXlbGxk/moCouwsm7Aa6+Aeh1OmKPHuTQz98D0GrAENz8ag6JfJgoLSwZPH0GT837gsGvvfPQ95f/63qudAon7bXXKT15itpoD2lS07g69HFimjU3+ST07EXZuXO13nfjTl1p0qUHgsHAX19+TN6NVOO6M39tJDc1BaVOT+MbokBL/k8/kfLsc+hycsic9zGG4mLMmzfHYczoOh+3oBfIXnFJDGUz/P/QWypPLqL8Sj5IJTg+GYz9zdDfot3JlN2c5LgfdLkqMj47Q8H2pDptl78xnoyFZyi/Zup5Ljl2m/esxb2pRSo8RS+aphZhjtrsMor2i8aq/ZAA5E71of7/JMaPH49EIkEikaBQKPDz8+PNN99ErVY/sH0cPHiQnj174ujoiKWlJY0aNWLcuHFoNBpjG0EQWL58Oe3bt8fa2hp7e3vatGnDl19+SVlZGQAffPABEomESZMmmfQfGRmJRCIxGksVL7iurq4UFxebtG3ZsiUffPCB8f+KY7/z8+mnn9Z4TB999BFDhgzB19fXuOz06dP06tULe3t7HBwciIiI4MKFCybbCYLAwoULCQoKwszMDC8vLz766KMa95WXl8dTTz2Fra0t9vb2PP/885SUmN57d+v3/PnzhIWFYW1tzeDBg8nLyzOu0+l0tG7dmlOnTvGgyc7ORqlUUlpailarxcrKipSUlAe+n9qi1Wp56623aN68OVZWVnh6evLMM89w48aN++p34MCBLF8uqkZPnDiROXPmPIjh1omMjAy++uor3n33VlrCoUOHGDx4MJ6enkgkEv78888693vt2jWef/55/Pz8sLCwICAggFmzZpncv/369UOhULBmzRrjMqVSyeuvv85bb711X8dVQa0MtFWrVmFjU3t55SVLluDs7HzPg6rn/tEatOSr82tso9FryFPnUa7T887GS8zcJKoUhviLN2641AZST4GZHYz5FSzsa+yvtVtrhjcS1QU/OPYBL+99mYgNESyOXIxBMBDiGIKLRe2L+j4I4k8e5cLu7eSmpvDb3Hc5v+OvKo2HivBG3xbOSKupjQRgMOjZ+vUCclNTsHJwZPg7c3j6k68Y+/GXhPbqh8LcAifvBoSPeOqBHoeg1aLNzKzTNhKJBPeARiiUD1euu+TwYTJmz0afl0fR1q2kjBvH1f4DyF35Pbr8qq9BfUkJqZMnUx4bCzqdyUd74wapU6aiSU2tcts7kUgk9Hnxf3gGhVBeWsrGT2ajKikmP+MGxzesAyAkLQfH7j3w/vYbpFZWlJ05w9XBj1G8YwfIZHjMmY1EVnelRm1mKeUJBahj8lDVwTgRdAZ0BQ/uhfBu+9KkFpt+bpQg6O/NoCzakwyAZStX5E4WWLf3wDpcLLSevz4OTWpxTZujL9Ui6A1VrhMEgfyNCeiyVZQcTkVfWF6rMelLNJSeykCXoyJ7+SVKjt9AEAQMah0lh296z3rVLffsdhSetctDEwSBgo0JoBcwC3LAIvTRPu/qqR39+vUjPT2dq1ev8sUXX7Bs2TJmzZr1QPqOjo6mX79+tGnThkOHDnHp0iW++eYblEolev2tUg1PP/00r776KkOGDGH//v1ERkby/vvvs2nTJnbt2mVsZ25uzsqVK4mPj7/rvouLi1m4cGGNbdLT000+33//PRKJhOHDqy/DUVZWxsqVK3n++eeNy0pKSujXrx8NGjTg5MmTHDlyBBsbGyIiItDejFAAeOWVV1ixYgULFy4kNjaWzZs3065du6p2Y+Spp54iKiqK3bt3s2XLFg4dOlTJS3K3fl944QV69uzJuXPnKCwsZN68WwrGn332GeHh4Xcdx71w/PhxWrRogZWVFefOncPR0ZEGDRo88P3UlrKyMs6dO8f777/PuXPn+OOPP4iLi+Oxx2pOl6gJQRA4ceIE4eGiMvfhw4eNfz9KVqxYQadOnWjYsKFxWWlpKS1atGDRokX33G9sbCwGg4Fly5YRFRXFF198wdKlS3nnHdPJ7vHjx/P111+bLHvqqac4cuQIUVFR97x/I8J9oNFohMuXLwsXLlwQ1Gr1/XT1t1JYWCgAQmFh4d89lAeCTq8Txm0fJzT/obmw/MJywWAwVGpzIeuC0HN9T6HV6tZCxJKVQsO3tgi+b28Rvtl7Rei5vqfQ7IdmwpGFDQVhlq0gnF5Z630XqAuErr90FZr90Mz4Gb99vLAlcYug1j3aa0RdWiIsefFpYeHIgcLKVyYIC0cOFBaOHChs+/YzQVN+aywlBWph5euHhG9f3Ctcjcyqsc/9Py4XFo4cKHz51ONCenxcpfV6nU7Q63QP/FhSX39DiA5uLGQu/EwwPIT+7xV1fLwQ27qNEB3cWEidNk24MXOWEBvWSogObixEBzcW4jp0FEqOnzDZxqDTCSkTXxSigxsLVzp3EVRxcYImI1P8pKYKV4cNF6KDGwuJgwYJuuLiWo+ltCBfWD7lWWHhyIHCr7NnCOvnvCMsHDlQ+KF/DyEquLGgiooSx5yYKCT0628cY8YnC+75+EvOZgjX3zokXH/rkJD+2WnBoK98r92OJqtUyN+SKKTNPiZcf+uQUHw07Z73XRt0xeVC+qenjWO8/ZO34Uqd+1NfKxS3n3FY0OaqjMsNOoOQtfKScP2tQ0LaRycEXWHV97oqIV9Ife+IkP7ZaUFfqqm0/vbzef2tQ0LBrmu1GlfJqXRxm3cOG7fNXR8nFOxMEr+bhXf/bmpCFZcn9vPp6ZrHcVocf+p7R0zOzz+R/9rvXm0ZN26cMGTIEJNlw4YNE8LCwoz/6/V6Yd68eYKvr69gbm4uhIaGCr/99ptxfV5enjBmzBjB2dlZMDc3FwIDA4Xvv/9eEARB+OKLLwRfX98ax/Drr78KgPDnn39WWmcwGISCggJBEARh1qxZQosWLYQ+ffoII0aMMLY5f/68AAhJSUmCIAhCUlKSAAhvvPGGYG1tLWRmZhrbtmjRQpg1a1a1YxkyZIjQs2fPGsf722+/CS4uLibLTp8+LQBCSkqKcdnFixcFQIiPjxcEQRCio6MFuVwuxMbG1tj/7URHRwuAcPr0rXtt+/btgkQiEdLS0mrdr4WFhRATEyMIgiAsXrxYGDBggCAIgpCYmCg0atRIKCoqqvWY6sJbb70lvPLKK4IgCMLChQuFJ5988q7b7N+/X2jbtq1gaWkp2NnZCZ06dRKuXbv17Nu8ebPQpk0bwczMTHBychKGDh16X2M8deqUAAjJycn3tH1MTIzg4OAgGAwGITs7W5DL5ULxXX6ra7pnKq7f8+fP12kcTZs2Fb799ttq1wPCxo0b69RndSxYsEDw8/MzWZacnCwAQkJCgsnyHj16CO+999597/OeZbcOHz6Mr68vPXr0oHv37vj4+LBjx477tRfreQD8EvcLZzPPIiDw9fmvmX5gOqXaW7kTG65sYPyO8WSVZaExlJOqWIaNdQHfj2tLvzAJWWVZmCGhdUkB+LSHVuNrvW87MzvmdZ5HiGMI45qMY/PQzazqt4qB/gMxkz3awquH1/5IaX4eDh5ePLPgW7o9/TwSiZToQ/v4ZeabnNu+mTNbN/HrnBUUZ5/C3CIZj8Cqiywb9HrObt1krP/Vb8o03AODKrWTymRI78ETUxMGjYbiPaJUfu5333F94ovoCwoe6D7uBV1+PtcnTcZQUoJlmzZ4zp+Px+wPaHT4EO4fzkEZGIA+P5+U558nd9UPRs9l1qcLKTl4EImZGd6LF2EeFITCzVX8eHnhvXgRchcXyuMTSJs+HUFfuwLBlnb2DH1zJgpzC65HXSTl8gVkSGiamo1N716YN2kCgJm/P76/rcfuieHY9OmNy9Qp93wOtBm37itdlgrVxewq26micshadpHMz85ScjgNQ5kOgIK/ElHH5VW5TW3Q5ahQReVWGV4p6Azk/hyDLkeFRClDZm9m/ACUnsqg/GpBlf0aNHrKzmehyzP18lV4z6xauyF3vKUmKpFJxMLOrpYYijSiQIbG9HvT5qjI/TkGQWtAl3Xz79s8afpSLYVbxVBopa/tzTGmI+iq9rbdjipaDF+17eGDXX8/kEDZ2UxjXpxtrwZIpPcu0lER4qjLUWFQ66psoy/RULhNHL9t7wYm5+f/BYIAmtJH/7nPUq6XL1/m2LFjKJW3xF8+/vhjVq9ezdKlS4mKimLatGmMHTuWgwcPAmLeS3R0NNu3bycmJsYkasjd3Z309HQOHTpU7T7XrFlDcHAwQ4YMqbROIpFgZ2caZj9//nw2bNhw19yW0aNHExgYWOtws8zMTLZu3WriGauKw4cP07p1a5NlwcHBODk5sXLlSjQaDSqVipUrVxISEmIMg/zrr7/w9/dny5Yt+Pn54evrywsvvGASbngnx48fN4Z7VtC7d2+kUiknT56sdb8tWrRg9+7d6HQ69u7dS2hoKACTJk1iwYIFdYoKuxspKSnY29tjb2/P559/zrJly7C3t+edd97hzz//xN7enpdeeqnKbXU6HUOHDqVbt25cvHiR48ePM3HiRGOO/9atW3n88ccZMGAA58+fZ+/evSaevw8++MAk7LQ2FBYWIpFIsLe3r9N2gwYNMn43hYWFODg44Ofnh16vx9vbu8b+arpnqsLX19ckNPdO8vLyiI6ONrlOHiaFhYU4OpqW3mnQoAFubm4cPnzYZHm7du0qLbsXap25bDAYkEpv2XOvvvoqa9asoXv37gAsX76cyZMnk5RUt7yBeh4sGaUZfH1OdLn2adiHA9cPsCdlD1e3XuXTbp+yLnYdv1/5XWxc2gy9tBCZxXW8G6+jtf9gNsZvB6BNWRnmEhkM+hKkdbPjw73CCfd69O7u27lxJYYLe8Rj6f3CFORKJW0GPY6rrz9bvvyErKREspISTbYpUMHK/+2iabeeNO/ZDydvH4pysrm8fxeX9u+mJFcMYes04qlKNbkeJqqzZxFUKqTW1gg6HaVHj5L0xAi8v/0G88Z/T9kCg0ZD6ssvo01NReHjg9c3XyO5+YIjtbLCYcQI7AYPJmPWBxRu2kTWJ5+gvnwZi5YtyfvhBwA853+MRfPmlfpWuLnhvXgxyWPHUnroMFkLPsVtxtu1GpdLA18G/u8N/vz0QxAEAtJzsdLocJliaoTJrK3xnDv3/k4CoE0XDTS5swW6HBVFe1OwCHUxMQZKT2eQv+FmeJIEzBs7YtXOHdXlXMrOZpK7NhbXKS1RuNZNXEkQBHJWR6HLUmHexAnHkUFGMQrhZqig5loREjMZri+1QOFmZdw2f2M8pSczyN+YgNsrrZDIb93jgl4g96doyuMLQAJmgfZYtfNAaikXl0kl2PSoXLBZai7HeVwTshZHok0tIf+3KziOboxEKsFQpiX3hygElQ6FhxW6XDXlVwsp2JSI/eNivb/CbUlinTI3S5yfa0bGp6cxFGtRReVi2aL6UEFDuR51vBhKa9HMGYW7FQpPK/LWxYq5Zy4W9x1qKLNWIrNToi/UoE0vxcyvcp5q4dYkDGU6FO5WWHeuulbifxptGczzfPT7fecGKK3u3u42tmzZgrW1NTqdjvLycqRSKd9+K4oJlZeXM2/ePPbs2UPHjh0B8Pf358iRIyxbtoxu3bqRkpJCWFiY8eXw9hfkESNGsHPnTrp164a7uzsdOnSgV69ePPPMM8bc/Pj4eIKDg2s93latWjFy5Ejeeust9u7dW207iUTC/PnzGTx4MNOmTSMgoOY86B9//BEbGxuGDRtWY7vk5GQ8PU2/WxsbGw4cOMDQoUP58MMPAWjUqBE7d+5EflPM6urVqyQnJ/Pbb7+xevVq9Ho906ZN44knnmDfvn1V7isjIwNXV9NcUblcjqOjIxkZGbXud8WKFbz00kssXLiQ8PBwZsyYwU8//YSlpSVt27YlIiKCxMRERo0axdz7/C3w9PQkMjKSoqIi2rRpw8mTJ7GysqJly5Zs3bqVBg0aYG1d9eRvUVERhYWFDBo0yPh9hYTcytX/6KOPGDVqFLNnzzYua9GihfFvZ2fnu37Pt6NWq3nrrbcYPXp0nbUiVqxYgVqtZtKkSXTo0IHx48czc+ZM7O3tmT59eo3b1nTPVEVAQECNBlxKSgqCIFS6Lh8GCQkJfPPNN1WGD3t6epKcnHzXZfdCrd+827dvz7nbEvc1Go1JXG2DBg0eaJJtPbVHpzfwxe4rnLyay8cnP6ZMV0YLlxYs7LaQVf1W4WrhytXCqwzfPJzfr/yOBAn+sicoThmDt+YlXC3cSC1N5o2Db3AoRXy4havUEP4KuDX5247rxpVYDvy0Eo1aVaft9Dodu5d/C4JAcKce5N6w5+iGBDKvFeHTNJSxH39Jiz4DcPFthVQRjEwZjE/Tdtg4uaAuKebs1k388NpkVk2fzIqpz3P893WU5OZgbmNLpxFP0WH4qId0xFVTcvgIADa9e+P7yzoU3t5oU1O5Nmo0JTdndO8VXW4uGfPmUfDHRgyq2p1nQRDI+GA2qjNnkVpb47NkMXIHh0rtpObmeMz/GLd33wWZjKKtW8m8mcTt/PJUbPv3r3YfFs2b4fnJfADyfvyRrK++qjaf7U4CWrdj0Ctv0sTGEf+sfGz69MY85OGI01R40OwfC0BiIUeXbepFK79aQP6fCQBYtXfH/e12OI9rikWIEw6PB6L0tUUo15PzQxT6Um2V+6gOXVYZuizxO1NH55K1KBJttiguUHIojbKzmSABp6dCTIwzALt+fkhtFOiyVRQfMFVfLNiSKBpiMgkIUB5fQN6aGHK+uyQeRxu3ar1DcicLnMY2AZkE1aUcivamIOgN5K6NRZejQmZnhvNzzXAcFQwS0YtXcuwG6sQC43gdhjVCqpRh1U4s6lxyvOZkdvWVfNAJyJzMkbuJRq55Iwdcp4Zh3dULp6dC7st7VkFFHpomrXIemjohn7LzWeL4hzdCUkMeaz1/Pz169CAyMpKTJ08ybtw4nn32WWMOVkJCAmVlZfTp0wdra2vjZ/Xq1SQmipN6kydP5pdffqFly5a8+eabHDt2zNi3TCZj1apVpKamsmDBAry8vJg3bx5NmzYlPT0doFYiSncyd+5cDh8+bJKfVhURERF07tyZ999//659fv/99zz11FOYV1NbswKVSlWpjUql4vnnnyc8PJwTJ05w9OhRmjVrxsCBA1Hd/C0xGAyUl5ezevVqunTpQvfu3Vm5ciX79+8nLi7uruOrjtr027RpUw4ePEhycjJr165Fq9Uya9Ysvv32W15++WU6derEhQsX+OOPP/jrr7+q3M+kSZNMroHqkMvl+Pr6EhsbS9u2bQkNDSUjIwM3Nze6du2Kr69vtcaGo6Mj48ePJyIigsGDB/PVV18ZrxMQBWF69epV7b6nTp1ao9F+O1qtlpEjRyIIAkuWLKnVNrfj7u6Ol5cXJ06c4KmnnsLX15fjx48zcuRIfH19azS6arpnqmLv3r1MnTq12vUV19jdrt37JS0tjX79+jFixAgmTKhc/9XCwsIo6lPTsnuh1h60b7/9lhdeeIFu3boxd+5cZs2aRevWrQkODkar1RIbG8s333xz3wOqp+7sjMrkq73x/HxpKxqnfcglcmZ1nIVUIqWFSwt+Hfwrrx14jXNZ57BR2vC0/zt8slGCVAKfPh6O0jKAcTvGcezGrRsmXOkMXasuEPwoUJUUs2nhXMoKC1Cam9OpDqIbZ/76g5zrycgUVqTEhpAcI74gR+5OwdnHmqadPfFq8jhxZ2JQWkO3McE06+qFwaDn2oVzXNyzk6vnTpGXJr64+jQNJbRXBIHtOiFXKB7K8dZE6RHRQLPq0hnzxo3x+/03US3x6FEyP56Pdbdu99SvIAikv/seJQcOAJD58cfYDR6M/cgRNXrm8r5fReEff4BUitcXn2MWGFhtW4lEguPTYzELDiJt2nT0ubnYDhiAczXhHrdj268f5f+7Ss7X35C7ZCl5K1Zi07cv9iNHYtmubY01t3yd3TEcFcOBnKfcewhjTehLNBiKRaNK2dAWmy5eFO1KNnrR9Plqcn+OAb2ARagz9kMCTQwFiVyK09NNyFoUiT5PTe7P0bg839zEm1UTqigxrE/haYWhRIsuW0XWt5FYd/Sg+KAojGE/yB/zoCqMZws59oMDyFsbS9H+61iEuqBwtaTk+A1Kj6eLht2YxijcrSg9k0npmQzxWGVVe89ux8zPDofHG5H/+xWK96ZQfrUATVIREqUUp3FNkNkosWjihF1/Pwq3JVG45SrSm7XFrNp7YNZQnNW1bu9O8f7raK4VoUkvRelRtZdEfTO80aKJk8k1IXc0x37Ag6vFqfC0Rh2TV6VQSPEB8XxbdfBA6fPgQqf+VSgsRW/W37HfOmJlZUXgzefW999/T4sWLYwiGBVqgVu3bsXLy9QTamYmhgf379+f5ORktm3bxu7du+nVqxdTpkwxmWH38vLi6aef5umnn+bDDz8kKCiIpUuXMnv2bIKCgoiNja3TmAMCApgwYQJvv/02K1eurLHt/Pnz6dixI2+8Uf1v+OHDh4mLi+PXX3+9676dnZ3Jv2OCbO3atVy7do3jx48bo6vWrl2Lg4MDmzZtYtSoUXh4eCCXywkKupUOUOEdSklJqdKL6O7uTlZWlskynU5HXl4e7jfLoNxLv9OnT+fVV1/F29ubAwcOMHfuXKysrBg4cCAHDhxg8ODBlbaZM2cOr7/++l3PT9OmTUlOTkar1WIwGIzeWZ1Oh7W1NQ0bNqxRNGLVqlX873//Y8eOHfz666+899577N69mw4dOtRJQb0mKoyz5ORk9u3bV2fv2bx585g3bx6CIFBWVkZYWBgginJEREQgkUjYvn07XbpUHV1Um3umLlQYvPn5+bi4PBwxphs3btCjRw86depkVKy8k7y8vEr7r2rZvVBrA619+/acPn2aBQsW0Lp1axYsWEBcXBwnT55Er9fTtm3bSg+zeh4NsVfiWKGcywxb8WVxfLPxNHJoZFzvbOHMir4r2Ht9L8F2zRj3XTxQxqqGu2jxuxg69rECXr35XuGu0+HX7ytQ/H3y0IfXrKKssACAi3t30v7xJ6usAXZmy0Yid24RFdsMAtpyPeUl4nZSRRcEwRw3P1tsncy5GplDzvUSDq67Ytw+tKc3zbqK161UKsM/rC3+YW0pycvletRF3AKCcPT8+65rbWYm5VeugESCVSexrpnM3h6vLz7nSoeOaK5dQ5uZhcKtsny4vqic3HWxWLfzwDKs8vriXbtF40yhQOHmhjY1lfy1a8lfuxbLtm3xXPAJCg8P02327SPr5gPV7e23sa7mYXwnVu3a4f/nRsrOncemR/daFzR2njwZhbsH+WvWoI6KomjrVoq2bkUZEIDn/PlYNK9c382gVpM5/xMQBGz69KnW2CzcnoQuXy0WWL4Hj0eF90zmZI7UTIZ1J09KjqShy1ZRejKdkuM3xJA3b2scRwRV6cWRWSluhgVeQJNURN76OBxHBCNR3H08FQaadUdPzBs7krsmBs21IhNjwapT9eEfFs2dMQ92QB2XT/7GBGx7+FDwl+ghsI3wxaKp+ANoF+GLbe8GqOMLkFkrkDvcfcbSqo0b2qwySg6lokkqAgk4jmqM0vPWLLR1Fy+0WWWUncnEUKxBaqPArp/vrXNja4ZFUydUl3IoPX4D5bBGlfYj6A2oYsS8E4umTncd1/2g9KpayVFfWE55YgEANl28H+oY/tFIJHUONfwnIJVKeeedd5g+fTpjxoyhSZMmmJmZkZKSQrcaJr9cXFwYN24c48aNo0uXLrzxxhvVvmw6ODjg4eFBaan4zBgzZgyjRo1i06ZNlfLQBEGgqKioUh4awMyZMwkICOCXX36p8ZjatWvHsGHDePvt6kPDV65cSevWrU3C5aojLCyMn3/+2WRZWVkZUqnU5Fle8X9Fbajw8HB0Oh2JiYnGMLwrV8Tf39uV926nY8eOFBQUcPbsWWPe2759+zAYDLRv3/6e+t27dy8xMTGsWrUKAL1eb1SavF1x8k5cXV0rhVtWxbZt29BqtfTq1cv4jjxq1CjGjx9vlGO/G2FhYYSFhTFjxgw6duzI2rVr6dChA6Ghoezdu5dnn332rn1UR4VxFh8fz/79+3FyqvuzctKkSYwcOZLFixeTmprKvHnz+O2339i1axffffcdwF1tgLrcM3cjICAAW1tboqOjTQz1B0VaWho9evSgdevWrFq1yiTFqwK1Wk1iYqLRWK3g8uXLlZbdC3V6K5HJZMyYMYOtW7fyzTffMHnyZFq3bs3QoUPrjbNHTK4ql+yybLLLsmkS/xEnXNJRK8qxkLjwYuiLldorZAr6+fZj/ckSUvLKaGSjo2vmT1CYAoUp9MpJYVqeOEPW39IXSWDPR31IRlJjLnNpnxjGobSwpDQ/j4TTxyu1K8rJ5si6HynMyqQoO4uS3GzKS/IAAzIzX1r06c2T77Xlibfa0PeFZoz/JJzOIxrh4C7OuDZs5kT48Kq9P9aOToR06fG3GmcApUeOAmDevLlJGKHM1tYYtldWTS2X0rNZaJKKKNiSiKA1FVrQFxeTeTPu3umF5wnYtZMG36/Epl8/kCsoO3OOpOFPUHryVt/quDjSXn8DBAH7UU/i8PTYOh2L3MUF24i+xly12iCRSLAf9jh+G37H9/ffsX/ySaSWlmgSE0l+6ikKNv5p0l6blsa1MWMoPXoU5HKcqwmR0GaUUnwwFdXFHDTXa5aFrw5tuhjCoHAXX0ql5nKsu4jXS8GmRHRZKmS2SpyfaYJEUb1wjMLNCqcxjUEKqos5ZC27gK6gZnl5XUE52rQSMactxBGZjRKXCc2xvmmQmQU5YD/Yv0ZDWCKRiF49hRRNUiE5P0aBQZTPt+lmamhIZFIsGjui9K69d8iun6/RaLLr74dFE9OXAolEgsPQQMz87cTQwCGBlQo6W3cUJwjKzmdhUFUW5yi/Woig1iG1VqBs8HBrbxql9rPKTO6nsgvZIIjCJv/vhEH+I4wYMQKZTMaiRYuwsbHh9ddfZ9q0afz4448kJiZy7tw5vvnmG3788UdANJQ2bdpEQkICUVFRbNmyxejBWbZsGZMnT2bXrl0kJiYSFRXFW2+9RVRUlNFLM3LkSJ588klGjx7NvHnzOHPmDMnJyWzZsoXevXuzf//+Ksfp5ubG9OnTK0l7V8VHH33Evn37qgwlLCoq4rfffuOFF16o1fmJiIggKirKxIvWp08f8vPzmTJlCjExMURFRfHss88il8vp0aMHIIp7tGrViueee47z589z9uxZXnzxRfr06WN8qT516hSNGzcmLS0NED1h/fr1Y8KECZw6dYqjR48ydepURo0aZcw3qk2/FajVaqZOncry5cuNL9nh4eEsWrSICxcusGHDhvuWiG/YsCHW1tZkZmYyZMgQfHx8iIqKYvjw4QQGBlZrjAIkJSUxY8YMjh8/TnJyMrt27SI+Pt54Pc2aNYt169Yxa9YsYmJiuHTpEp988olx+2+//bbGEEitVssTTzzBmTNnWLNmDXq9noyMDDIyMkzqet0NR0dHAgMDiY6Opn///gQGBhIfH0/fvn0JDAwkMDCwRm9fTfdMVfTq1cuYF1oVUqmU3r17c+RmhFEFJSUlREZGGoteJyUlERkZWadadGlpaXTv3p0GDRqwcOFCsrOzjefsdk6cOIGZmZkxV7WCw4cP07dv31rvr1rqIvl4+fJl4ffffxfi4kR58R9++EHw9/cXFi1adN9ykn8n/ya54ZyyHOGFnS+YyNjf/nni41cFlaZqGfaY9EIhYMZWoeFbW4QLW5eIEvrftBWE62eMn/SrewWd5u8rmaDVaITvX31RWDhyoLBz2dfCkV9/FhaOHCj8MuutSm13f/etsHDkQOGzMROFr5//Wfj6+Z+FNTM3Csf+OCaUFVcvcW0wGIT8zFJBfx+y24+K66++KkQHNxayvvqq0rqM+Z8I0cGNhRvvvV/lttmrLhslx0vOZpisS589W4gObiwk9I0Q9LeVyDDoDUL65yeFlOlbhZiwzkJ0k6ZC7o8/CtqsLOFKjx5CdHBj4dr48YJBU1ki/VGhKywUUl6cZJTJT5/zoWDQaISS48eFuPYdqpX3v528P64Yz03Roev3NI7c9XGVpOD1aq1RQj/1vSNCeWrtywSoruQZt02bc1xQJ+ZX27b4SKpw/a1DQuaSyErrdAXqOknKFx28bjwXmYsjBYNWX+tt74bBYBB0ReU1t9Hpq5WkNxgMQvrnZ8Tv6XBqpfV5G+PFkgG/171kQF0xGAxC2hzx+ylPuSXRnfHlWbFkwokbD30MD5J/0+/eg6QqmX1BEISPP/5YcHFxEUpKSgSDwSB8+eWXQnBwsKBQKAQXFxchIiJCOHjwoCAIgvDhhx8KISEhgoWFheDo6CgMGTJEuHr1qiAIgnDu3Dlh7Nixgp+fn1EWvWvXrsLmzZtN9qfX64UlS5YY5dVtbW2F1q1bC1999ZVQVlYmCMItmf3bKSwsFJydnauU2b9TpnzixIkCUElmf9myZYKFhYVRzr82tGvXTli6dKnJsl27dgnh4eGCnZ2d4ODgIPTs2VM4fvy4SZu0tDRh2LBhgrW1teDm5iaMHz9eyM3NNa7fv3+/ybEIgiDk5uYKo0ePFqytrQVbW1vh2WefrSTjfrd+K3j77beF1157zWRZfHy80LZtW8HW1laYPHmyoNff/zNv3bp1QufOnQVBEIRDhw4JgYGBtdouIyNDGDp0qODh4SEolUqhYcOGwsyZM03GtGHDBqFly5aCUqkUnJ2dhWHDhhnXzZo1S2jYsGG1/VdcG1V99u/fb2zXrVs3Ydy4cTWOVavVCtbW1kZZeX9/f+HIkSO1Os6a7pmqrt+GDRvWWB5CEARh27ZtgpeXl8m5qrie7vzcfmx3O2erVq2q9pzdzsSJE4UXX3zRZNmxY8cEe3t74z18P0gEoXbZqp9//jnvvfceoaGhxMfHM3/+fCZMmEBOTg7Tp08nPj6e5cuX07wKVbZ/OhXhBIWFhXWOy32UROVE8eqBV8koFa14qUQKBlHKWgI8WVSCZ1YbnEZ+zaBQ09Amg0Fg+NJjnE8poG8TN5abfQkxf0HXN6Hnu/xTOL5hHcfWr8HSzp5nP1+KVqPmuynPIRgMPPPpt7g08AWgKCeLFS9PQDDoMbMbSWivDjTt7ImTV/WJvP82BL2eK53CMRQW0nDtWixbmbrMi/fvJ3XySygbNiRgp2mJC0EQSP/whFHOXeljg+uUlgCoIiO5NnoMCAINfliFVYcOxu3KkwrJXnbxZifFFG99D3QqZHZ26AsLUfr64vvrL8iqCL95lAgGAzmLFpNzsxileWhX1DGnQavCvEkTvL/9BkU16k4GtY70eScRNKIXxKKFC06j666GmfnNebRpJTiNDcGi2a0E8NLTGRTuuobD0EZ1DrvT5anJ/SlaVIeUgt0Af6zDPSt5wrK/u0h5YiF2A/3uO6xO0Avk/BiFoVSL87NNkVnX3sP5KCg5kU7BnwnInS1wm97aGCoqGAQy5p9CX6TBaXxTLBo73qWn+yd75SXK4wuwfzwQ6/YeaDNKyfzyHMgkeL7bHqnlo89RvVf+Lb979fwz2Lp1K2+88QaXL1+uMtyrnn83DRs2ZPbs2YwfP/7vHkqtEQSB9u3bM23aNEaPHl3r7caNG4dEIuGHm4rS90JOTg7BwcGcOXMGPz8/4/Inn3ySFi1aVCpqfS/U+i5bsGABW7du5cSJE5w7d47PP/8cEBP1Vq9ezZw5cxg5cuR9D6ieqvkz4U+e2f4MGaUZNLRtyJ9D/uSCcwQXrl1n01UNU8ufZ0ZePn1lZ9l07nql7decSuF8SgHWZnJmD/SHhJuqP40HPuIjqZ68G2mc3LgegO7jJmBubY2NozON2oru48idW4xtj/++HsGgRyr3oe2gznR9Mug/ZZwBqC9dwlBYiNTGBovQyhMflm3agFSKJjkZbWamyTpdtko0zuQSkEnQXC9Gk1qMoNWSPnMWCAJ2Q4aYGGcghpIZkdhg9+THIFegLyxEameH95LFf7txBiCRSnF5eapYM80nDIX/WCy7vIntkOE0XLumWuMMxPpYgsYAN8U47iXEUdALaDNNQxwrsGrrjue7He4pJ0ruaI7L5BZYtnQBAxRuuUrZOdOEeUOZlvKkQoBKYYP3gkQmweW5Zri9HPaPM84ALMNckZjJ0OWoKNiYYKyLpk0rQV+kQaKUYR5g/0jGUpFDV5GHVnG/mAc7/quMs3rqqSsDBw5k4sSJxlDEev47REVFYWdnxzPPPPN3D6VOSCQSli9fjk5XdW3KqhAEgQMHDhhLQ9wr165dY/HixSbGmUajoXnz5kybNu2++q6g1gaaIAjGWROZTFZJJrZPnz6cP3/+gQyqnlsIgsD8U/N5/+j7aAwaunt3Z93AdQSUFMApUVXmXd1zKEP6Y1BY4SHJIyf+FHmlt2KLM4vULNguKka93jcIj+wTYs0aOx/wuHuC8KNAEAT2rFiEXqvFt0UrGnfqalzXMkI0ImMOH6C8rJSi7CyiDuwGwN6zO637Vx/f/W+m5Gb+mVWnTkiqEEiR2dgYiy/fmYemSS4CRM+ZRXPRu1NyPJ3cH36g/MoVZPb2uL79lsk2gs5A2UWx1pvdAD8kCimGUkucX1mKTb9++CxdgtltD6N/AjY9e+IwVlTZktl6oWw8Comy+oLogiBQckKUMLbtKaoR6vPU1Urcq6/ko4rJrbRcl6sCnQGJQorsAecdSZUyHJ4Mxqa7OL7CrVdNxqeKyQODaBjKnf4+IZ9HhdRMht0Asfh06ekMspddRF9YbhRJMQ92qJWoyoPAKLV/oxTBIFAWKRpolmEPR0Wsnnr+Sbz66qv4+NSs4lrPv4+mTZty8eLFf6VntGXLljz99NO1bi+RSEhOTr7v67hNmzY8+eSTJsuUSiXvvffeA1PerPW38cYbbzBgwAA6depEy5YtqyxK97DrEfx/ZOe1nayJWQPASy1e4queX2Ejs4C/XgEE/qILRw3NaenvjrRRHwB6SU6z9eItuePZf0VRXK6jhY89T3f0hdit4orGA0XlrX8AyRfPcz3qInKlGb1feMkkpMu7SXOcvBugLVcTdXAv+1f/jCCI3rM+L/RFXoMAw7+Z0puV6K06V5/AbNmuHVDZQCu/aaCZNbTFuqPoTSo7n0n2ku8BcH3rrUq1y9RxeQhqHTJbJdadvXB8UpQqLk/UYz/qTSwfgCrRg0YQBLQZt2bPyuPyKdyWVG378oQCdNkqJEpRdVHuLD5ItamVvWiGMi05P0aRuzpaNMhuo0LBUeFu9UBqbN2JRCLBtk8DFO6WGMp0Jsekuikrb/6QVQv/SVi398D52WZILORorheT+c15ys6LXuOHrd54O4oKJcf0UsoTC9AXapCYybBo/P/nu6innnrqqefhU2sD7fXXX+fEiRNMmzaNI0eOMHHixIc5rnoAvUHP4gtiMcEuzmOY3HKymHd2cglkXkJvZs8H6jFYKGSEeNhC40EAREjPsPG8GIawNyaTbZcykEklzHu8GTJBD3HbxB38jeGNNxIK2LbkIpcPpaFR6YjcJRqNzXv2xc7VnaIcFSf+TGT7skvEHk+neU+xqPGZLRtJOH0AAP82g/AOrlzj6b+AvrAQ1SWxMLB1587VtrNqLxpot6stwm0etIa2KBvYIHdWil4Xz3bYDuiP3dAhlfqqCNeyaOmKRCrBopkzthGid7Jgc4LRY/Eo0Zdqyfs1jtJzmVWu1+Wo0OeXg0yCwwhRvavkSBolp9KrbF9yXFxu2doVqbkcpfdNj0gVYY7qhALQCyDcEfqJ+IIOoKimNteDQCKTYj+sEUjEsEx1YgEGjZ7yK6KS2oMIb/w3YR7kgNvUlijcxbpv+kINyCSYP4LcswrkjuZIzGSgM1C0OxkQyxU8Kg9ePfXUU089/z+odR00gObNm/8rRUD+rexK3kVS4VUEvTl7TgST260cJ7kaDn0KwJmgaeSetqODjx0KmRSC+iJIFTQijYLr0cSkN2fmJrE44vOd/WjqaQfXjoAqDywcoEGnv+W4BEHg4No48m6UknQhh8O/nqEsRzQwHL07sPnrSK7H5ImaOcDV89kolOZI5WYU52QDIDdrSMSEByBj+g+l9PhxMBhQBgZUqkV2OxatW4NUijYlBW16OgoPD/SlYtFiAGUDWwwlJajO/4nCZwBmwX1w/7BXJdEJg0pnrCdl2fJWuJZNdx90WSrKzmeR+3M0tr0aYNOzwUPxGt2JoDOQ+3MMmqRCVDG5WDZ3qfQirL5prJj52WHV2g19QTlFu5Mp+DMRuZOFSW6SrkCN+ma4onUH8ZwqfGwgMhtNauXiwxV9A5RFZmPTq4HxvN3uQXuYmDWwxaq9B6Un0inYmIBtn4YIWgMyezMUnv++mlP3i9zJApeXWpC/IR7VhWzMgxwqSfM/TCRSCQoPK7F4dopo1FdVY7Ceeuqpp5567odaTftNnz7dWGCxNsyYMYO8vLx7HlQ9ovdsyU3vmSa3CxqtGb+euQ4nloK6EJyD+V0v5mm1aXhzBtncDomfWDi4j/Qs474/RVqBCi97C17tfbPIa0V4Y1B/kD26F5vbuRFfQN6NUuRKKQ7ulpQXRwICUnkDjm7I5Xq0aJz5NHGkzQBfbJ3N0WpkSGS3ama0HTIKc+v/blJ+yc3wRuvONReClllbY960KXArzFGTInrP5C4WSM0kpE2bjjpyG4KuDInCDk1yWaV+VJdyQC8gd7M08QpJJBIchjfCqoMHCFC0J4Xcn2MwqGtOytXcKCHj87NkfHGW4sNp1eZ4VYcgCOT/mYDmphiGoNajiq38TKnwJpk3Ej2pNj19sGjhAgaB3NXRFGxLQpstHm/pyQwQwMzfDoWbeIxKH7Gulya12CSvVhAEY98geuq0txlxRg/aQzbQQKwnJrVRostRkb8xHhDD+mpb7Pu/hlQpw3FUMC4vtTCG4T5Kbi+2LbNTYub394vm1FNPPfXU89+iVgbaV199RVlZ5Ze66li0aBEFBQX3OiYT0tLSGDt2LE5OTlhYWNC8eXPOnDljXC8IAjNnzsTDwwMLCwt69+5NfHz8A9n338mOaztIKkxCJlihyRdzkDYdj0E4IcqK0+1Nztx8EW/d8LYwv4owR9lpsorFYrdzH2+GpVIOggAxN5UQQwY9mgOpgksHxPDL4PbujJgRhkwWA4DCKgxLWyWt+zVk7Icdeex/LWn/mD9j53TksVdb4t+6N0jMsHVpTsdh1Yf9/dsRBMFYoNqqhvDGCoxhjhUG2m3hjZnzP6H0yBEkShmWoaIhXxHmdzul5yvEDlwrvfhL5FIchgbi8EQjkEtQR+eStSgSbWbVkzZlkVlkL7mALqsMXWYZhVuvkj7vJLm/xFJ+taCSwFBVlBy5QdmZTJCA8uYL8J1hhoLWQPlV0YAzvxnqKpFIcHyiEUpfW4RyPSWHUsn87CzZyy9SelI87opizgBKD2uQSm6GzN0qDq3LLBMVAhVSY45Txf4Nah36m4WkFTeLnj9MpOZy7B/zB0RDFcD8/1l4451IJBLMGtg+Uu9ZBYrb1GItb4YD11NPPfXUU8+DpFa/boIgEBQUVOsZ27p422oiPz+f8PBwevTowfbt23FxcSE+Ph6H28QNFixYwNdff82PP/6In58f77//PhEREURHR/9rRUv0Bj1LLywV/ynsCgZzZFIJ/Uo2IlEUgktjchsOIClnPwCtGtxmoAUPgK3TaSVNwIV82oc2oUfwzRCcjEtQmAJyC/Dv8YiPSqQkv5yrkWKYYvPu3lw5cQRNWQk2Ti4899WzyGSySi88EqkEn8aO+DTujvalcGQS6X/ae6A6dw5dZiYSc3Ms27a5a3vLdu3IXbGSslOngVsCIbqMGPJ//hkAzwWfYNmqGaqYM5Rfyafk2A2sOnogkUjQFaiNnirLltWHa1m1cUfhZkXuz9HoslVkfnkO88aOWLVzxzxINP4KtydRckQ0wM2CHLBo4kjpqQy0N0pRRWajisxG7myBVTt3LFu7IbOq7AVVxeZRuO0qAHYD/TEPtCfzy3Oo4/IwlGmNcubl1woRtAaktkrkbrcMJYlChsuEUNRxeZSeykAdl2c05GR2SsxDnG5rK0XhYYU2rQTN9WLk9uIzoyK8Uelnh2U7d1RRuZRdyMZuoJ8xvFFmp3xk0uoWzZwxb+yIOjYPqaUcM996r83fheI2D1p9eGM99dRTTz0Pg1oZaKtWrapzx25ubnXe5k4++eQTfHx8TPZ/e80BQRD48ssvee+99xgyRBQ9WL16NW5ubvz555+MGjXqvsfwyFEXsv30V1wruoaNwpYbme2RSyW80MaR5y9sF9t0e4uz18WX8Eau1tjd/pJo6wFebSDtDC95xDFo8Ihb62Jves8Ce4Hy4c/8V0XUkTQEg4BHoB1OXtbsWCyGXLbo0x+54u6Xo0Lx7w9rLE8qRNALmPnbVTJGBa2WjA/nI/dshWX7pkjNqpeMr8CiVWuQydBev47mehqa62IoXv5PXwPgMm0atn1EhU+rDh6UHk+nYHMimtRiHB4PpOymwWzmb4fcvub9KX1scH05jPzfrqCOy0cdk4c6Jk80VmyUxjBAmx4+2PZpiEQqwbqDJ5rUYkpPZVAWmY0uR0XhtiQKd17DoqkTyga3Fck1CBTtTQEBrNq5G4s0Kzys0KaXUnYpB+v2Yv6Y+rbwxkpeP5kEiyZOWDRxQldQTtmZDNRx+Vh39UIiM22r9La+aaCVYNncxbTvIAfMAx2QWikwlGpRxxegz1cDjya80Xg8Egn2jweSvz4Oi+YulY6hnkeHws0Sy9ZuSM1lj/QaqKeeeuqp5/8PtTLQxo0b97DHUSWbN28mIiKCESNGcPDgQby8vHjppZeYMGECAElJSWRkZNC7d2/jNnZ2drRv357jx49Xa6CVl5dTXn4rnKmoSDR2tFotWm3dcmUeOOtGs0x3FZQKOtv2Zr3BnMaeNryo3I6tpIw4gzcSx+6cPieGa7VqYFdpzNKgAcjSzjDO4TJ6c6lxvTxmCxJA16g/wt9wnHqdgajDonelSRcPUuNiyEi4glQup3GXHn//uX8E6PPV5Hx3EQwgczDDorUr5q1ckNko0d4opeDngygCJqJUmIO5jPJCFVLLu9ymZkrMmjah/OIlCvedAZ0jgrYUQ+ENbAYPwvbZ8cZza9W/ARI7JSU7kyk7l4UmoxShXAybMwt1qt13YCbBbmwwVtkqVGezUJ3PRl+oESXHlVJshwVi3tQRnV4HYtdI3MyxHuyLZV8f1JdyUZ3JRJdWiupiDqqbtdduR+Fni1X/BsYClGahTmjTSyk9l4lZK7GumzpOzElTBNjUPG4rKRbdPLHoJoY23tlW5iFOVpRfL0Kr1SJo9MZC0HJ/G3QGHWbNnVCdyKD0bAYSc7Gsg9TV4tFes5ZS7MeHVHkM9TxabIaKE4X/5u/h3zz2eh49ubm5hISEcOrUKXx9ff/u4dTzH+LatWv4+flx/vx5WrZs+cD6Xbp0KVu3buWvv/56YH0+Sv4elYhacvXqVZYsWcL06dN55513OH36NP/73/9QKpWMGzeOjIwMoLK3zs3NzbiuKj7++GNmz55dafmuXbuwtPx7PEsAzsXR5OWc55qrM3Z6PdPOfU+yxAZbnTfW55YB8JVuGKXrj5JSKgEkyPJT2LYt2aQfa7UVvQCSDpK5SJTSl2DAqyAKA1J2XZOgTd32aA8OKEuXoyqyQGpmIPbGabL+PAiApbcvB44ee+Tj+TtwzFbiZxBDpPT55ZTsuU7xnhTKzQ2Yq2WAIxIFCBhABTErj5AccPf8T2dHRxyBrEOXsHPphj43AVXDhsS3b4+wfXul9jYhcvyvWMMNMVzPIBE4lHYWfebd88PuRNIc7POUWBfJyXZXo04+Acl32agBWDjJcMo2Q6Ex9QZpzAxkOOWj33XNuExRLqE59miTi9nzxw4EiUBolgMCAgevnkZ/ve7jrsC8TEZT7FAnF7Jt6zZsCxQ00ttQbqZn16n9IAHLEhkh2FEWlYPaXI8lci5nXCF/2+V73m899fyd1CWv/L/E+PHj+fHHHwGQy+V4e3szYsQI5syZ88DSIg4ePMjs2bOJjIxErVbj5eVFp06d+O6771AqlYAYAfTdd9+xcuVKoqKikMvlBAYGMnbsWCZOnIilpSUffPABs2fP5sUXX2Tp0qXG/iMjIwkLCyMpKQlfX1/jC66LiwuJiYnY2NgY27Zs2ZKhQ4fywQcfVDr+CiIiItixY0eNx/TRRx8xZMgQE+Ps9OnTvP3225w9exaJREK7du1YsGABLVq0ACAuLo5JkyYRHR1NYWEhnp6ejBkzhlmzZtUYDfO///2Po0ePcvnyZUJCQoiMjDRZX3Fe7sTS0tKYYrN7926mTJlCRkYGQ4YMYeXKlcZzX1hYSNu2bdm9ezcNGzas8bjryunTpxkyZAg3btzgxo0bBAQEUFhYaNz338Hy5ctZu3Yt586do7i4mPz8fOzt7e+rz4EDBzJkyBAmTpzIxIkT8fb2ZubMmQ9mwLXgwoULzJ8/nyNHjpCTk4Ovry+TJk3ilVdeMbZ57rnn+PDDDzl8+DBdutQsuPZP5B9toBkMBtq0acO8efMACAsL4/LlyyxduvS+vHozZswwKbRdVFSEj48Pffv2xdbWtoYtHyKCgPTnxYy0E/f/jN4SD0Maa5QfkSdricKgotguiO2Z7bAuUFKuMwAGnh3cFV+nymE2wnerkWZF4V1w0nSFfzf6PDbyERxQZTZ/cQEoomVPX5p2ceD7DT8A0H/c83g0avy3jOlRU7T1GqqEDCzauKJoaIPqTBba5GLM1TIEQY8u9QxSi2yc3nqTgpXROGeZ02hwK5S+NV+Xpba2pB84iKVGNP4EbTYhP/5Ac6fqxST0BeUUrI1Dl16GRVNnIh7r+ECP9X5oUcWy/IJoNFeL6ODYHJmNgqJzV1F62xAx5P7GLRgEsmNOI9MY6Nu2O2Wns1CRgX1zDwYMFAV6BEEgN/0C5KqxLBMfm20jOpnkvtVTz7+JisiR/4/069ePVatWodVqOXv2LOPGjUMikfDJJ5/cd9/R0dH069ePl19+ma+//hoLCwvi4+PZsGEDer3e2O7pp5/mjz/+4L333uPbb7/FxcWFCxcu8OWXX+Lr68vQoUMBMDc3Z+XKlbz22ms0atSoxn0XFxezcOHCKo2Xqo6/ArO7hNKXlZWxcuVKdu7caVxWUlJCv379eOyxx1i8eDE6nY5Zs2YRERHB9evXUSgUKBQKnnnmGVq1aoW9vT0XLlxgwoQJGAwG43tddTz33HOcPHmSixcvVlr3+uuvM2nSJJNlvXr1om3btoD47jhmzBhmzJhBREQETzzxBMuXL2fq1KkAvP3220yaNOmBG2cAx48fJzxc/N04fPgwbdq0+VuNMxC/v379+tGvXz9mzJhx3/0JgsCJEydYsGABIB7nt99+e9/91oWzZ8/i6urKzz//jI+PD8eOHWPixInIZDLj96xUKhkzZgxff/11vYH2oPHw8KBJkyYmy0JCQtiwYQMA7u7uAGRmZuJxW62ozMzMGt2kZmZmVT6QKh4ofwtXD3Iu8zxXPN0wl5kxYtRf/LXwBQZLjuCafw4Aq77v0nCbNddyxZlPJyslgW52VQtmjFoDCXtAMNxaJpUhDR6I9AEeo06r58DPcaTdJkkOIJNL8Q11pklnTxw9rMhJLSHjahFSqYTQbj5c3PMHeq0WV98AfEKaPVTRj+LDqZRfLcRxVGOkZrKHtp/aoL/psbIIdMCypSu2bT3RZpZSuO0IOV+8B2jw2/QnZn5OaNu5U3oqg+LNSbi90gqJvHrRVdt27UiXyZA5imp/zhNHYHHz/qgOhYsCt5daoorOwyzQHtk/PL/PqpU7mqtFlF/MMeb+mAc7PpB7VuFljSapCEO6Gm3CTcGUxk4mfVuFuVK0J0X8RybB3MMGiay+QHE9/07+C/m894qZmZnx/cHHx4fevXuze/duo4FmMBj45JNPWL58ORkZGQQFBfH+++/zxBNPAKKA2dSpU9m1axclJSV4e3vzzjvv8Oyzz7Jr1y7c3d2NL68AAQEB9OvXz/j/+vXrWbNmDX/++acxfx7A19eXxx57zMR4Dg4OxtXVlXfffZf169fXeFwvv/wyn3/+OVOmTMHVtXoBm9uPvzZs27YNMzMzOnToYFwWGxtLXl4ec+bMwcfHB4BZs2YRGhpKcnIygYGB+Pv74+/vb9ymYcOGHDhwgMM3y8hUx9dfi/nT2dnZVRpo1tbWWFvfEuu5cOEC0dHRRi9jTk4OOTk5vPTSS5ibm/PYY48REyOqRR87dozTp08/NIPi2LFjRgPtyJEjxr9r4vfff2f27NkkJCRgaWlJWFgYmzZtwspK/J37/vvv+eyzz0hISMDR0ZHhw4fXafyvvvoqAAcOHKjz8VRFXFwcgiDQpEkTcnJySEhIoH379jVuU9M9cy8899xzJv/7+/tz/Phx/vjjD6OBBjB48GD69OmDSqXCwsLinvb1d/GPfrsIDw8nLi7OZNmVK1eMsx5+fn64u7uzd+9e4/qioiJOnjxJx47/HG/AXREEOPAxv9iKD5wB/gPJLrPm5fLJfGx4BkEiA58OSEMeY2yHWzM+rRpWFkcw4ugH7SZA+xdvfdq+IIqIPLBhC+z/KZbYEzcozlNTkl9u/BRmq7iw9zrrZp/kj4VnOfKbWPrAr6UL2SnRHN+wDoDWA4c8VONMEASK9l0XxSyicx/afoz70xsQDFWH2wk6A5obooiG0vtWCIrUXEfeig8RNCU4TXoRs5tCOHb9fJFaK9Blqyg+cL3G/UqtrLDu1g+phQNIBKw7hdTYvgKJQoZlC5cq1RT/aVg0cwK5FF22CtXN79I8yOEuW9WOiu9DdUkUMUEKZoH2Jm1uV+xTuFrWG2f11HMbgiBQpi175J/alO2oicuXL3Ps2DETL8fHH3/M6tWrWbp0KVFRUUybNo2xY8dy8KAYlv/+++8THR3N9u3biYmJYcmSJTg7i7mx7u7upKenc+jQoWr3uWbNGoKDg02MswokEgl2dqYqrfPnz2fDhg0mJYaqYvTo0QQGBjJnzpwa2x04cABXV1eCg4OZPHkyubk1/zYePnyY1q1bmywLDg7GycmJlStXotFoUKlUrFy5kpCQkGpz1BISEtixYwfdunWrcX91ZcWKFQQFBRm9JC4uLnh4eLBr1y7Kyso4fPgwoaGhaLVaJk+ezLJly5DJHtxk7ZEjR7C3t8fe3p7ff/+dd999F3t7e5YuXcrXX3+Nvb098+fPr3Lb9PR0Ro8ezXPPPUdMTAwHDhxg2LBhxut6yZIlTJkyhYkTJ3Lp0iU2b95MYGCgcfvx48fTvXv3B3YsNTFo0CDs7e1p06YNhYWFODg44Ofnh16vx9vbu8awyZrumarw9fU1huXWlsLCQhwdHU2WtWnTBp1Ox8mTJ6vZ6p9LnT1ohYWF6PX6SichLy8PuVz+QEMEp02bRqdOnZg3bx4jR47k1KlTLF++nOXLlwPig+zVV19l7ty5NGrUyCiz7+npaQwP+FeQdJDstJPs9vECYFTwKCKTCgAJkV5jkDzzIcjNQSplRGsfFu6KQ601mNY/+xs4uyOZuJNpaIrXYW5loOvYqbj6ioVjS/LLiTmWTvKlHNJveiQAGjaVsOWL+QgGA0269iSky8OV+zeUahFUotCE+kr+Q5XF1pdqyVoUidRMhuvLYZUUGrWZZaATkFjIkTmZo8vPp3DjnxT8+iv63FyUAQE4vfCCsb3UUoH94ADy1sVStP86Fi1cULhUH1Ln8Nx0Cn5PROFti0Tx93oKHwZSczkWTRxFURG9gMRcbmLo3g8VBavVcTfl9auosSV3skDZwAZNSrFJMe966qkHVDoV7dfWPIv+MDg55iSWirqFGm/ZsgVra2t0Oh3l5eVIpVKjR6K8vJx58+axZ88e40Svv78/R44cYdmyZXTr1o2UlBTCwsJo00Ysg3K7QTJixAh27txJt27dcHd3p0OHDvTq1YtnnnnG+H4UHx9PcHDti6y3atWKkSNH8tZbb5lMSN+JRCJh/vz5DB48mGnTphEQEFCpTb9+/Rg2bBh+fn4kJibyzjvv0L9/f44fP16t0ZKcnIynp6fJMhsbGw4cOMDQoUP58MMPAWjUqBE7d+5ELjd9dnbq1Ilz585RXl7OxIkT72pA1gW1Ws2aNWt4++23jcskEgnr169n2rRpvPLKKwwYMIDnnnuO+fPn06NHD8zNzQkPDycnJ4eXX37ZxONyL7Rp04bIyEhiY2MZM2YMZ8+eJS8vz3jc5ubm1Rov6enp6HQ6hg0bZnQ+NG/e3Lh+7ty5vPbaaya5VRWhnCBGmhkMt0VKPURWrFiBWq1m0qRJdOjQgfHjxzNz5kzs7e1N0oaqoqZ7pioCAgJqNODu5NixY/z6669s3brVZLmlpSV2dnYkJ98tMf6fR50NtFGjRjF48GBeeuklk+Xr169n8+bNbNv24MQn2rZty8aNG5kxYwZz5szBz8+PL7/8kqeeesrY5s0336S0tJSJEydSUFBA586d2bFjx7+nBpogwP6P+d3GGp1EQkuXloQ4hfDzoUsAtPSxBwt7Y3M7SwWv9g5i/enrDAp9cN6wupJ4PouTm65i0FxB0GejKoLdyz6ix/iJtOjTH9eGtvi3dKEkX03MsXSunMrEzgWOrf+S8rJSPIOb0Gfiyw+9npkuS2X8Wx2fj2AQHlph2cJtSejz1OgB7Y2SSsaD5noxAHJ7uPHa6xTv3m1U05Ta2OD50Vykd8SqW4Q6Y37OAXVcPvm/x2Pd2fRH8nbUUaJxYdbwb8qjfARYhrkaVR/NG9k/MLn5O7+r6jxztr0bkv9HPJat7r+MSD311PP30KNHD5YsWUJpaSlffPEFcrmc4cOHA6KXp6ysjD43S5NUoNFoCAsLA2Dy5MkMHz6cc+fO0bdvX4YOHUqnTp0AkMlkrFq1irlz57Jv3z5OnjzJvHnz+OSTTzh16hQeHh735PWbO3cuISEh7Nq1q8bwxYiICDp37sz777/P2rVrK62/Xd26efPmhIaGEhAQwIEDB+jVq1eVfapUqkrvVCqViueff57w8HDWrVuHXq9n4cKFDBw4kNOnT5uEk/36668UFxdz4cIF3njjDRYuXMibb75Z11NQJRs3bqS4uLiSLkHnzp05ffq08f8rV66wevVqzp8/T9euXXnllVfo378/zZo1o2vXroSGhlbqu3///sZwzIYNGxIVFVXlGMzNzfH19WX9+vX0798fPz8/jh07RpcuXWjcuOb8+hYtWtCrVy+aN29OREQEffv25YknnsDBwYGsrCxu3LhR7fcCorf3UeHu7o5Wq+XEiRN88803+Pr6cvz4cVatWnVXg6ume6YqapqIuJPLly8zZMgQZs2aRd++fSutt7Cw+FeKItXZQDt58iSff/55peXdu3fn3XfffSCDup1BgwYxaNCgatdLJBLmzJnzQGdkHilX96O9foLfK7xnjcWH58VU0esU6m1faZNJ3QKY1K3yzNijIjulmD2rogFQmkWjLQNbF1eKsrPYu3IxmVfj6fXcZORKJdYO5rQd6EerCG82fDSTgox0bF1cGfLaO8gfQQ6ENvvWTWko0aJNL0XpZV3DFveGOrGAsrOZt/6Py6/WQCs9spXyS+JEhnnTptiPHIntwIHIrCt7ZSQSCfZDAsn84iya5CLyku+e2K9s+GC8Sv9EzIMckFrKMZTpMG/04DzIMgczpFZyDKU6436q27/H2+0e2H7rqee/goXcgpNjHn0YkYW87nklVlZWxjCx77//nhYtWrBy5Uqef/55SkrEMPStW7fi5eVlsl1F7nr//v1JTk5m27Zt7N69m169ejFlyhQWLlxobOvl5cXTTz/N008/zYcffkhQUBBLly5l9uzZBAUFERsbW6cxBwQEMGHCBN5++21WrlxZY9v58+fTsWNH3njjjbv26+/vj7OzMwkJCdUaAs7OzuTnm+aZr127lmvXrnH8+HGkUqlxmYODA5s2bTIxBCty1Jo0aYJer2fixIm89tprDyTMcMWKFQwaNOiutXdffPFFPvvsMwwGA+fPn2fEiBFYWlrSrVs3Dh48WKWBtmLFClQqcZK3ppzNiny4Cm/spk2b0Gg0CIKAtbU1Xbp0YXsVasogGvS7d+/m2LFj7Nq1i2+++YZ3332XkydP1smD9LCZN28e8+bNE0OZy8qMkxWlpaVEREQgkUjYvn17tWIctbln7oXo6Gh69erFxIkTee+996psk5eXh4uLy33t5++gzgZaeXm5sTbR7Wi1WuOFXE8tEQQ4MJ/9lhZkyWU4mjvSp2Ef1Fo9Menii3gLH7u7dPJoKS0oZ+vii+g0Blx9VKRcTEEmlzNm7mdEH9rH4bU/cnn/brKuXcWnyS03fW5qCtejL6Ewt2DomzOxtLN/JOPVZZnOmqiv5N+zgaaOz8dQpsMi1NnE8ydoDRRsTABAZmeGvrAcdXw+tr0amGyvSRUNNF12AsrAADznf4JFs6Z33a/c0RyHEUGUHE8Xr5ma2jqYY9G4euXGfzsSmRSHYY1Qx+Vj0fLBPXAlEglKbxvUcflIrRQoPB+8EV9PPf9lJBJJnUMN/wlIpVLeeecdpk+fzpgxY2jSpAlmZmakpKTUmCvl4uLCuHHjGDduHF26dDF6hqrCwcEBDw8PowT8mDFjGDVqFJs2baqUhyYIAkVFRZXy0ABmzpxJQEAAv/zyS43H1K5dO4YNG2YS9lcdqamp5Obmmgit3UlYWBg///yzybKysjKkUqnJb2HF/zWF3BkMBrRaLQaD4b4NtKSkJPbv38/mzZtrbLdy5UocHR157LHHjIZmRR1ArVZroq55O3ca6NURGRmJTqejZcuW7NmzB3d3d7p06cLixYtp3rz5XcUpJBIJ4eHhhIeHM3PmTBo2bMjGjRuZPn06vr6+7N27lx49Hm46yN2YNGkSI0eOZPHixaSmpjJv3jx+++03du3axXfffQfc/XzV5Z6pDVFRUfTs2ZNx48bx0UcfVdkmMTERtVptNCj/TdTZQGvXrh3Lly/nm2++MVm+dOnSSkmk9dyFlONw/SS/eIhqSk8EPYFSpuR8Wj46g4CTlRIv+3+O6oxWo2fbkouUFpTj4G6JpW0kAEEdOmNl70Dbx4bj4uvP1q8WkJWUSFZSomkHEgmDXnkTlwa+j27M2Tdnvzys0KaXor6Sh20Pnzr3oytQk7PqMhjA4qITDiODkJqJt0/R/hR0OSqkNkqcxjUh6+vzaFKKMKh1xjwmQ7neaCwa8q/hOOrZWhlnFViGumAZ+u+bAXoYWDRzxqLZg59ZVPraoY7LxzzY4aGFwdZTTz3/PEaMGMEbb7zBokWLeP3113n99deZNm0aBoOBzp07U1hYyNGjR7G1tWXcuHHMnDmT1q1b07RpU8rLy9myZQshIaIw07Jly4iMjOTxxx8nICAAtVrN6tWriYqKMr43jRw5ko0bNzJ69Gjee+89+vbti4uLC5cuXeKLL77g5ZdfrjKP3s3NjenTp/Ppp5/e9Zg++ugjmjZtapIPVlJSwuzZsxk+fDju7u4kJiby5ptvEhgYSERERLV9RUREMGPGDPLz83FwEKML+vTpwxtvvMGUKVN4+eWXMRgMzJ8/H7lcbjQm1qxZg0KhoHnz5piZmXHmzBlmzJjBk08+afRIVaSx3O5RTEhIoKSkhIyMDFQqlbEOWpMmTUzEXL7/P/bOOyyq4/vD7wILS0eaYAMsVBV7V2IDjRqNxhoRo4nRqLFHTWI3akyMJsYajcafJmqK0VhiLyh2wAJWpFhQRJDOsmV+f6xs3FAExZbvfZ9nn2Tnzsw9c9l177nnzOf8+COurq507NixSNuTkpKYPXs2x44dA3TOso+PD4sWLSIwBvj+yQAA3jpJREFUMJD9+/c/c/ZX9erVOXHiBOXLl6dFixYkJCSQkZFBly5dCuzH+zcnT55k//79BAYG4uzszMmTJ7l//77+8zR9+nSGDh2Ks7MzHTt2JCMjg2PHjjFy5EhAVzbq9u3brFu3rshz3L17l7t373L9uu5B8oULF7C2tqZKlSoF9CSKwt7eHnt7e6Kjo+nduzfVq1fn2rVrBAYGGoiWFEVx35nCaNu2LW+//XaR+wMvXrxImzZtCAoKYuzYsfrax8bGxgbRstDQUKpWrVrofsxXnVI7aLNnz6Zdu3acO3dOHw7fv38/p0+fZs+ePWVu4H8ZceoHrsvlnFaYYiwzpqdnTwDO3XwIgH9lu+e+R6ukCK1g/9pLJMVnoLCU0zbEg41TdDLCdYI66fu5165L8LxvuXhoH2pVnsEc7rXrUaVmwTSC54n6UYqjVfOKpP52lbz4DAPHqaRknbwLjx4K5kQ9QLXkHA7BPiAg4/AtAOzeqoppBStMHM1RJ+egvP5Q70iobmeCAG1uKkKZhmXLFmW3SIkywbpFBYxMjTCv8/yEZCQkJF49TExMGDFiBPPnz2fYsGHMmjULJycn5s6dy40bN7Czs6NevXp8+umngK6+0uTJk4mLi8Pc3JyWLVvqo1qNGjXi6NGjDB06lDt37mBlZYWfnx9//vmnPiInk8n4+eefWblyJT/++CNffPEFJiYm1KhRgwEDBhTrLI0fP55ly5aRm5tb7Jo8PT0ZNGiQXlQNdDev58+f56effuLhw4dUqFCBwMBAZs2aVWwttFq1alGvXj02b97Mhx9+CIC3tzd//fUXM2bMoGnTphgZGVG3bl3+/vtvfTTOxMSEL7/8kqtXryKEwM3NjREjRjBmzBj93GlpaQXUut9//329Yiagj37kF+YGXSRu7dq1DBw4sNhI3KhRoxg3bpyByMnatWsJCQnhu+++Y8KECQaiG0/LoUOHaNWqFaArVN60adMnOmcANjY2HDlyhEWLFpGeno6bmxsLFizQO50hISHk5uaycOFCxo8fj6Ojo77cA+hERhISEoo9R35qbT75dq5Zs4aBAwcCum1K7u7urF27tsh51Go1x44dY8mSJfp1Dh48+IlrhOK/M4URExNDcnJykcd/++037t+/z/r16w2iu25ubsTFxenf//LLL3zwwQclsvFVQyaeYrdqZGQkX331FZGRkZibm1O7dm0mT578xCKKryr56QRpaWkvrlB1xl3EN358YW/NJhtrqlk04c+eujDx2E2R/BFxmzHtPBnV7tW4pif/usGZHXEYGcvoOroOty8d4MiGNTi7V6P/vEWvjCP5ONo8DXemhYEA188bc3/ZOdQPcnEI9sXcr+RpgEKtJXHuKbRZKqzfqEzW2XtoM/KQmRljbGeG+l42Cm97HEJ8kclkPNwWQ2bYHSwbuVCuu+7vl3HkFmk7Y1HdCUd1fTM1joYiM5Jk2iUkJF4OL+V3T+K1ZceOHUyYMIGLFy/q95xJ/Hdwc3NjxowZeoftv0B+CuTVq1cLTRl+1XmqQtV16tRhw4YNZW3L/xZnfyIbDVutdIIOFy/XZF/0Pdr5lify1kMAapfh/rMHdzLZ/v05agVUol6Q25MHPMbV03c5syMOgIB+XrhUs2HndzqRizodOr2Szhmgq2clwMjCBGMrU8w8y6E+nkju1ZRSOWg5F5J1cv3abO4vHIpNp7cR1vVQ3clFfS8bmakRdt2q6a+DmWc5MsPukHs1FSEEMplMLxCiTY3DskVzyTmTkJCQkHht6NSpE9euXeP27dt60Q+J/wZRUVHY2toyYMCAl21KmZKYmMi6deteS+cMnsJB27lzJ8bGxgVC8Lt370ar1RabCyzxCI0Kzq5hn6UFuUaAygl1VjVGbYxgzXuNuHFft5HYvxAFx6LITM1FYSnHxLTwUH/43/Fkpig5szOOmq0qYmpe+J8+5U4Webn/iMA8vJfKgXUXAQvqtK+Cb/MK3Ag/TVrSPRSWVng3a1ViG180+emNJo9qhyk8y5F1PNHAccpHqLRo8zSFFmzOPH4HgLwre1HFx/Ng6SIwlmMV+DEyc09sO1fFxO4fCWKzqrZgIkPzUIn6fg5yZwu9QIgmNRarFoOe15IlJCQkJCSeC6NHj37ZJkg8B/z8/Dh//vzLNqPMadeu3cs24ZkotYM2adKkQiuiCyGYNGmS5KCVhMvbISORP110OdFtK3UkSTgSFvOAAT/qZIor25tjb2la3Cx6bkan8Nf356jkZUeXj+sUiGhlp+dxPTwJAJVSw+UTd6ndulJBs04ksn/tJf17IVTkpa9DaNOwsKuGvUsPNGo3IndvB8CvdXvkZq9uvTnVoxpoJk46oRWzqnZgLEOTqkSdnKMv+qxJzyNpaSTaLBWO79cyqCOWdzuTvIQMkAlUcaGYenhg4uRE9qlTZO5aADIjjM16Ydlwqv66G5kaY+Zui/L6Q3KvpmJkYYImVYkQWjQPE7Bs3vzFXggJCQkJCQkJCYnXhlLnWV27dg1fX98C7d7e3nqFGIkncGoVScbGnHkkVNHbtytL362Hh6MluSqdEkVJo2eqPA2Hfr6M0ApuXkrl1pXUAn2ij91BqxYYmegciIuHbxUolKlRazm1LRYAS1tTbBwVyE0uIbS6emzZD2PY+d18VgwbSOy5cJDJqNP+zada/osiP4Imd9Y5YkZmxpi565wv5VXddRIqDcnrotA8VCJUWh6si0ad8s/m6/zoGepbCGU6Np074bbuJ6ru3In9wIGA4OEvG0ldb5jym19HK/dqKnm3dHV1tJn3UHh6YOLw35XBl5CQkJCQkJCQeDZK7aDZ2tpy48aNAu3Xr1/H0rJgoV2Jf3EvGuKPstPKCmQgct1pXKkGdhamrA5pgM0jp61OZbsSTXdmRyzpyf84FKf/ijVwvrQaLVFHbgPQvEcN5GbGpN7N5va/HLnLxxPJSMnFwtaU/rOa0mdqfYT6LADNer1Lkx59sCpnT056GgiBh3897FyKrpvyKqC+bxhBA0PHSQhByq9XUd3KxMjCBLmLBdosFck/RaFVqtFmq8iOvK/rf3EHAJaNdEWKzap6UH7SRJzHjwfg3ty5ZIaGFjiP8kYaylidk6tNjcWyhaTeKCEhISEhISEhUTSldtC6du3K6NGjiYn5p8bV9evXGTduHG+99VaZGvef5PQqAP6000l5uxo1x+hRzaWqTlZseL8JH7T0oHfDJ2/CTb6VScTem4BOvMNYbkRiTBq3Lv/jfMWdf0BmqhKFlRzfFq54NdHVXLtw6La+j0at5cyuOADqBblhYmrM+b1/k/UwFRsnZxp1fYfmvfrzwZI1dJ0whXpvdqXt4I+e/Vo8R4RW/FMDzfmf4qlmnrqaH8obaaTviSfnfDIYybB/1weH92piZC1HfS+blF+ukHX6Lqi1GDvIUcWGIzMzQ+Hvb3Ae+0HvYdu9O2i13B4zFuWjKLJJeQuMbUxBrSXrlK4+hyY1HitJXl9CQkJCQkJCQqIYSu2gzZ8/H0tLS7y9vfHw8MDDwwMfHx8cHByeqSL4/wS56XB+E9flcmKMlAhhTEPnNwy61Kpky2edfLFWFBSreBytVnBogy61sWpdJ2q2qohfS92etlOPRdEuPKrR5du8AiZyY2oG6Cq9x567T8ajVL5LYYlkpiixsDXFr0UFVHlKTm/7DYDGb/fC2ERni5GxMdUbNKZ1yAfYOpcvm2vynNA8VIJaC8YyjMv9s09O7mKBkbUpQqUl46DOuS33dnUU1ewwsTXDcYAfmBiRezmFtN1xABib6/bvmdeti5Gp4b5AmUyG6/RpWDRogDYzk5vDPkKdmopMJsPsURRN5OhEV0TuXczr1HnOK5eQkJCQkJCQkHideaoUx7CwMHbs2MFHH33EuHHj2L9/PwcOHMDOzu45mPgfInID5GWyw0kXHVNnelGvUkGxjpIQdeQ292LTkSuMadnLE9BFv4zlRty9kcatS6mk3s3i1uVUZDLwa6Vz3hwqWFHR0w4hICr0NhqVlrOPomf1O+RHz3Y9ip6Vp1JmNZLXRSNUmmdf/wtEla/g6GiOzOgf0RSZTKZPPwSwalERy4Yu+vemla2x76m7nmhBpjAmL+YIABaNCi9mKTM1peLi75BXrozq5k1uDfsIdUqKwXmEVo3CrxIyefGOt4SEhISEhISExP82T1WMSSaTERgYyIQJExgxYoS+KrlEMWTeh0Pz0AI7LHV7otRpdfB1sUar0ZZuqlQlx//UpZg26VoNq3JmAFjamlGzpS5Cdmr7DS4c1qUxutVyxMbhn31YNQN0TmH00TtcPHKbzFQllram+LaogEqZy6mtuuhZkzd7kX38LrnRD0jff/Pp1/4MiFJem3zUSQXTG/Ox8HcCQOHrgO2bHoUet2mvqxVn1bQC2adOAP/sPysMk3LlqLxsKUbW1uRERhL7zjsI9V145Btq025h1aLZU61FQkJCQkJCQkLif4enKlSdlZXF4cOHSUhIIC8vz+DYxx9/XCaG/efYPRlyHxJewZdEdQZCY4Z5th8nv7vAJQcFb4+vr9+LVhRZD5VcCksk6uhtVLkaynvY6FMW86kbVIWLobe5eyOdpDhd7a1abxj28ajjiKWtKVlpeYT9rtszVa+DOyZyY85s30Z22kNsncvj4erPQ64BkHHkFhZ1nJC7vBghGE2WipRNV8iLS9NJ31exefKgx/inBpp5gWMKz3K4Tm6EkbWpQXTtcWzaVsGiQXnUSQloHjxAplCgqF272HOaVa+O+88buDViJHnx8dx8Lxib7l+jzTFDkxqLpVT/TEJCQkLiNeTBgwf4+Phw6tQp3N3dX7Y5Ev8hDh06ROvWrUlNTS3TTLzly5ezY8cO/vrrrzKb80VS6ghaREQE1atXp2/fvowYMYLZs2czevRoPv30UxYtWvQcTPwPcH0fXPgVZEZsr1ofAFVGLVoobMlOy+PujXSun7lX5PDbV1LZuew8P30axsltN8hMUWJuLad1f+8CTp2lrRk1W+kcMq1WYFfegsre9gZ9jI2N8Husj6WdGb4tXFEpczm97XcAGnfvjeqRg4eRDLSC1D+uIbSG8vwlRXUvy0C+vjjybmeStDgC5dVURN4/e8VKdb58iX2nghE0AGNbsyKds3xMbM3IPn0aAPO6dQrsPysMsxo1cP91M1YBAQilkqxDP6LNug+qWEyfMp1VQkJCQqJsGDhwIDKZDJlMhlwux8PDg08++YTc3JL9PpWEw4cP06ZNG+zt7bGwsKBGjRqEhIQYPNAWQrBy5UoaN26MlZUVdnZ2NGjQgEWLFpGdrfv9mj59OjKZjKFDhxrMHxkZiUwmIy4uDoC4uDhkMhnOzs5kZGQY9K1Tpw7Tp08v1M6hQ4cik8lKdO/2xRdf0LVrVwPn7PTp07Rt2xY7OzvKlStHUFAQ586dMxh3/vx5WrZsiUKhoHLlysyfP7/Y85w7d46+fftSuXJlzM3N8fHx4dtvvy3Qb8mSJfj4+GBubo6Xlxfr1q0zOL537148PT2xsbEhODjY4NqnpaXh6elJfHz8E9ddWk6fPk2FCrotJXfu3MHc3LxAIONlIYSgY8eOyGQy/vzzz2eay8/Pjz179gAQGBhY4Po/b3Jzcxk4cCC1atXCxMSEbt26FegzaNAgwsPDCX1MYft1otQO2pgxY+jSpQupqamYm5tz4sQJ4uPjqV+/viQSUhh52bB9LADKhoPZkxQOgDqtLtXyjPXdTu+IQ1uI85MQ9YA/F0YQey4ZoRW4VrOl7UAfgr9ohkNFq0JPWTewCiZy3Z+2ZquKhToivi0qYGSsa6/fwQ0TuTFHNqzVRc/Ku+Dbsg3KGzp5eLsuVZGZGZOXkKFXJCwNuVdTufdtOEnLIhHq4lMWs8LvkbTsHJqHSowfpW7mXk4psXOXT36Ko0khKY6lIfuUzkErLr3x3xjb2FBp2VIcP/oIdWIEWXs/w6J+9WeyQ0JCQkKibOjQoQOJiYncuHGDhQsXsmLFCqZNm1Ymc0dHR9OhQwcaNGjAkSNHuHDhAosXL8bU1BSN5p+93MHBwYwePZquXbty8OBBIiMjmTJlClu3btXf+AIoFApWr17NtWvXnnjujIyMEt+HbdmyhRMnTuidieLIzs5m9erVDB48WN+WmZlJhw4dqFKlCidPnuTo0aNYW1sTFBSESqUCID09ncDAQNzc3Dh79ixfffUV06dPZ+XKlUWe6+zZszg7O7N+/XqioqL47LPPmDx5Mt9//72+z7Jly5g8eTLTp08nKiqKGTNmMHz4cH2kRKvV0q9fP4YOHcrx48c5c+aMwTknTZrE0KFDcXNzK9G1Kg3Hjx+nefPmAISGhtKgQQNMS/Bw90WwaNEiZLLiH0yXhIcPH3L16lWaNGmCRqMxWPOLQqPRYG5uzscff0y7du0K7WNqakq/fv347rvvXqhtZUWpHbTIyEjGjRuHkZERxsbGKJVK/VORTz/99HnY+HpzZD48jAebioR6BpChykAuyiHL8sA8RfePmLHciIf3srl22jCKpsrTcPiXKwB4+DvSZ2ojuk+oj3cTV+SmxgVOlY+lrRmtg73xbVEB35aF/+NraWtGQF8var1RCd/mFYjcvYPI3dsBaB3yAWRrdXXEZLo9WbaBun/I0v6ORZNe8qdBqqRsHvx8CbSgzVDpnb5/I4Tg4fYbpG6+CmotCm97yn9cD7PqdiAg62Riic+pyVKhzdJdWxPHgimOJUUIQfapUwBYNG5cqrEyIyOcPh5JpWVLse7QAfuBIU9th4SEhIRE2WFmZoaLiwuVK1emW7dutGvXjr179+qPa7Va5s6di4eHB+bm5vj7+/Pbb7/pj6empvLuu+/i5OSEubk5NWrUYM2aNQDs2bMHFxcX5s+fT82aNalWrRodOnTghx9+wNxc93u0efNmNmzYwC+//MKnn35Kw4YNcXd3p2vXrhw4cIDWrVvrz+Xl5UXr1q357LPPnriukSNH8s0335CUlFRsv9u3bzNy5Eg2bNiAvATCVTt37sTMzIwmTZro2y5fvkxKSgozZ87Ey8sLPz8/pk2bxr179/SRqQ0bNpCXl8ePP/6In58fffr04eOPP+abb74p8lyDBg3i22+/JSAggKpVq9K/f3/ee+89/vjjD32f//u//+PDDz+kd+/eVK1alT59+jBkyBC+/PJLAJKTk0lOTuajjz7Cz8+Pt956i0uXLgEQFhbG6dOnGTVq1BPX/TSEhYXpnZWjR4+WyHH57bffqFWrFubm5jg4ONCuXTuysrL0x/Ovn5mZGa6urowYMaLUdkVGRrJgwQJ+/PHHUo/9NydOnMDPzw8bGxsiIyOxtLSkWrVqxY6Jj4+nS5culCtXDktLS/z8/Ni5c+dT22BpacmyZcv44IMPcHFxKbJfly5d2LZtGzk5OU99rpdFqR00uVyOkZFumLOzMwkJCYBO3fHmzZcjJPHKci8KwhYDkNp+OgvOLQVAk1EHD5UJaATWDgoavOkOwJmdcQaCIflFqK3KmdHuPV8cKhQeMSsMz0YutO7vXawj59uiAq36eHLz0jkOrF0BQIu+IVSr3xhl7EMA5K6WGFnIsWxaAXklK0Suhod/xRQ55+NoHhV9Frka/SctJ/pBoX2VN9LIPKoTNbFuUxmHAb4YmZtg1VRXDDvr9F2EqmSCIepk3RfR2NYMI7Oi1/8k8q5fR5OSgkyhwLxmzaeaw7p1ayotWiilN0pISPynEUKgzc5+4a/8kjJPy8WLFwkLCzOIcsydO5d169axfPlyoqKiGDNmDP379+fw4cMATJkyhejoaHbt2sWlS5dYtmwZjo6OALi4uJCYmMiRI0eKPOeGDRvw8vKia9euBY7JZDJsbW0N2ubNm8fvv//OmTNnil1L3759qV69OjNnziyyj1arJTg4mAkTJuDn51fsfPmEhoZSv359gzYvLy8cHBxYvXo1eXl55OTksHr1anx8fPRpkMePH6dVq1YG1zYoKIgrV66QmppKSUlLS8Pe/p+tGkqlEoVCYdDH3NycU6dOoVKpcHJywtXVlT179pCdnU1oaCi1a9dGpVIxbNgwVqxYgbHx098b/JujR49iZ2eHnZ0dv/32G5999hl2dnYsX76c7777Djs7O+bNm1fo2MTERPr27cugQYO4dOkShw4donv37vrP9bJlyxg+fDhDhgzhwoULbNu2jerV/8nIGThwIG+88Uax9mVnZ9OvXz+WLFlSrDPzJGrXro2dnR3du3cnKioKOzs7WrVqRXJyMnZ2dtQuZp/+8OHDUSqV+qjyl19+iZVV0fe0MpmMtWvXPrWt+TRo0AC1Ws3Jkyefea4XTalFQurWrcvp06epUaMGAQEBTJ06leTkZP7v//6Pmk95E/ufRAj4azRo1ai83mTM7Z3cyryFi0UFrl9tQXO17tJX9XeidutKnNt3Ux9F82rialCEulUfT0wVT6Xnoic7PY39Py7H1smZWm0CKeeq24P24PZNti+ch9Bq8W3VhkZd3wHQR7rMqtoBIDOSUa57DZK+jyDnQjI5l1Mw/9feNoPlq7U8WH8JzYNcjMuZYdPOjdRfr5IT/QC7t6oVSLvMjtA98bNoUB7bQHd9u8LbAWNbMzRpSrLP38eyvmH9tazweyivPcT2TQ+MrXU/AuqkRwIhzk8fPQPIyo+e1auL7BVJUZCQkJB4FRE5OVypV//JHcsYr/CzyCxKl8q+fft2rKysUKvVKJVKjIyM9Cl0SqWSOXPmsG/fPpo2bQpA1apVOXr0KCtWrCAgIICEhATq1q1LgwYNAAz2ZfXs2ZPdu3cTEBCAi4sLTZo0oW3btgwYMAAbG53Y1bVr1/Dy8iqxvfXq1aNXr15MnDiR/fv3F9lPJpMxb948unTpwpgxYwqNanz55ZeYmJiUStAtPj6+QCqktbU1hw4dolu3bsyaNQuAGjVqsHv3bkxMdPcrd+/excPDUCm5fPny+mPlypXjSYSFhbFp0yZ27NihbwsKCmLVqlV069aNevXqcfbsWVatWoVKpSI5ORlXV1c2b97MmDFjGDVqFG+++SaDBg1i3rx5tG7dGoVCQfPmzUlOTmbkyJFPFZF6nAYNGhAZGcnly5fp168fZ8+eJSUlhWbNmhEeHo5CoShS/CIxMRG1Wk337t31KZe1atXSH589ezbjxo0ziPg1bPhPyR9XV1e02uIfXo8ZM4ZmzZoV+kCgNOzcuRO1Wk3nzp0ZPXo07dq1Y9CgQXTo0IFevXrp/+6FkZCQQI8ePfRrq1q1arHn8vLyKvCg4mmwsLDA1tb2uew3fN6UOoI2Z84cXF11UY0vvviCcuXKMWzYMO7fv19sXvH/HPei4NYphImCWa4VOXvvLFZyKwZWm4VMbUkNte7pTdW6jpgqTKjTXlcb7fTOODRqrUERao9HsvDPQsSubVw9Hsrpbb/z4+gP+XXWp0QfOcCf82eizM6igpcv7YeM1OcnK2PyHbR/viCmFaywaqFz7B7+eR1tXuG10YQQPNwaQ15sGjIzYxxD/LDwd0JmZow2PY+8W4abmIVKQ86FZAAs6xk6YDJjGZZNdE98Mk8YpjnmXEwmdfNVsiOSeLAuWh9hU91/JLFfhEBISck++chBa1S69EYJCQkJiVeX1q1bExkZycmTJwkJCeG9996jR48eAFy/fp3s7Gzat2+PlZWV/rVu3TpiYnTZI8OGDWPjxo3UqVOHTz75hLCwMP3cxsbGrFmzhlu3bjF//nwqVqzInDlz8PPzIzFR9xv2NFG/2bNnExoaarA/rTCCgoJo0aIFU6ZMKXDs7NmzfPvtt6xdu7ZUe5FycnIKRKxycnIYPHgwzZs358SJExw7doyaNWvSqVOnMksnu3jxIl27dmXatGkEBgbq26dMmULHjh1p0qQJcrmcrl27EhKi20aQn+HVokULTp8+TWxsLEuWLCE2NpZ169Yxe/ZsgoODGTJkCKGhocycOZPz588Xev6OHTvq//7FRRsVCgXu7u6cP3+ejh074uHhweXLl2nZsiXe3t64u7sX6aD5+/vTtm1batWqRc+ePfnhhx/00cWkpCTu3LlD27Ztizx3frS3KLZt28aBAwfKRMSvUqVKKBQKYmJi6NOnD66urpw+fZq+ffvi7u5OpWIyhT7++GNmz55N8+bNmTZtWpHXPJ/Lly/z9ttvP7PNoIuu5gvvvE6UOiyT/8QIdCmOf//9d5ka9J8hRveUa52bH1vid2MkM+KrgK84e6kcldQPMNWCwkqOSzU7AGq9UYnIvTdJS8ph+/fnChShfhaEEFw6eggAJzcP7ifEkXDxPAkXdV8QGydnuo77FJNHueiadKUuTVAGZu6G8vY27dzIOZ+M5qGS9L3x2HUq+BQk8+gdsk7raoDZ9/XWS/MrvMqRcz6Z3OgHBrL5OZdSEEoNxnZmmLoXlNO3bOhC+r4EVDczyLuZgWlla/JuZ5KySbc/Dxnk3cwg5ber2PfxKpMImtBq9QqOFqUQCJGQkJD4X0Rmbo5X+NmXct7SYmlpqU8T+/HHH/H399eLYGRmZgKwY8cOKlY0LFFjZqYTrurYsSPx8fHs3LmTvXv30rZtW4YPH24g0FGxYkWCg4MJDg5m1qxZeHp6snz5cmbMmIGnpyeXL18ulc3VqlXjgw8+YNKkSaxevbrYvvPmzaNp06ZMmDDBoD00NJSkpCSqVKmib9NoNIwbN45FixbpVSH/jaOjY4GUxJ9//pm4uDiOHz+ud4p+/vlnypUrx9atW+nTpw8uLi7cu2e4tz7//ZNS7aKjo2nbti1Dhgzh888/Nzhmbm7Ojz/+yIoVK7h37x6urq6sXLkSa2trnJwKf6D94YcfsmDBArRaLREREfTs2RMLCwsCAgI4fPhwoel5q1at0jubxe3Vy0/Vy4/Gbt26lby8PIQQWFlZ0bJlS3bt2lXoWGNjY/bu3UtYWBh79uxh8eLFfPbZZ5w8eVKfNvssHDhwgJiYmAIOYo8ePWjZsiWHDh0q0TxDhw5l/fr1aLValEolLi4uun362dn4+PgAur/Z45+tx3n//fcJCgpix44d7Nmzh7lz57JgwQJGjhz5LMsrESkpKUV+Ll5lnqpQtUQJuL6PQ+bmLNDeB+CThp/QomILou6kUV2li5551HbUy+SbKkyoG6j7YN+6rPuHsGm3f4pQPwt3rl4mLekecoU5fWd9xQffr6bpO32xsnfA3NqGbp9MxcLWTt8/P71RXsEKIwvDf5SMTI2xe1v3w5Z57DZ5tzMNjudcTiFt5w0AbN+sapAGae7noOsTZbgPTZ/eWMe5UMVJYytTLGrrvlyZx++gSc/jwboohEqLWQ07HAfVBCMZOefuk3Hg5mM10J4+gqa8fh1Naioyc3PMa0mpuxISEhLFIZPJMLKweOGvZ1WlMzIy4tNPP+Xzzz8nJycHX19fzMzMSEhIoHr16gavypUr68c5OTkREhLC+vXrWbRoUbEZROXKlcPV1VUv/NCvXz+uXr3K1q1bC/QVQpCWVriY1tSpU7l69SobN24sdk2NGjWie/fuTJo0yaA9ODiY8+fPExkZqX9VqFCBCRMmsHv37iLnq1u3LtHR0QZt2dnZGBkZGVz//Pf5KXdNmzblyJEjelVH0Mnfe3l5FZveGBUVRevWrQkJCeGLL74osp9cLqdSpUoYGxuzceNGOnfurHcWH2f16tXY29vz1ltv6ZU0821SqVQG6pqPU7FiRf3fvjjFx8jISM6cOYOxsTH79+8nMjISBwcHNm/eTGRkJKtWrSpyLOi+O82bN2fGjBlERERgamrKli1bsLa2xt3dvdi01icxadKkAn9zgIULF+qFbUrCzJkziYyM1Kc3RkZGMnDgQPr372/wWSqOypUrM3ToUP744w/GjRvHDz/88NTrKikxMTHk5uZSt27d536uskZy0J4Hykyyb57kUycHBNDTsyf9vPsBEHU7nRoq3WX3qGPo0dcMqIjCSucQlfew0dcqe1YuhR4EoEajpsjNFNg4OtOs57sMWbqWoSv+D6cq7obm5+8/8yg8/9fcyx7z2o6ghdQt/9RGU93NIuXnyyDAspELVi0Mv6wKL3swlqG+n4PqUZRLk6Ui94rOIbWoW/QTDstHYiHZ5++T/FMUmrQ8TJzMcejng6JGOey66XLt0/fG6yX5nyXFMe13XT04i3r1kJVA5UpCQkJC4vWkZ8+eGBsbs2TJEqytrRk/fjxjxozhp59+IiYmhvDwcBYvXsxPP/0E6BylrVu3cv36daKioti+fbs+irBixQqGDRvGnj17iImJISoqiokTJxIVFUWXLl0A6NWrF71796Zv377MmTOHM2fOEB8fz/bt22nXrh0HDx4s1M7y5cszduzYEsmGf/HFFxw4cIArV67o2xwcHKhZs6bBSy6X4+LiUuyeuKCgIKKiogyiaO3btyc1NZXhw4dz6dIloqKieO+99zAxMdGrUPbr1w9TU1MGDx5MVFQUmzZt4ttvv2Xs2LH6ebZs2YK3t7f+/cWLF2ndujWBgYGMHTuWu3fvcvfuXe7fv6/vc/XqVdavX8+1a9c4deoUffr04eLFi8yZM6eA7UlJScyePZvFi3WCbeXKlcPHx4dFixZx/Phx9u/f/8wS8dWrV+fhw4eUL1+eFi1aYGpqSkZGBl26dKF69eoFIrGPc/LkSf1nICEhgT/++IP79+/rP0/Tp09nwYIFfPfdd1y7dk3/Wcxn8uTJDBgwoMj5XVxcCvzNAapUqVJgf2BxODs7U716dc6fP69f16VLl3jzzTf1Tmxxe9BGjx7N7t27iY2NJTw8nIMHD+rXWBje3t5s2bKlWJuio6OJjIwkJSWFtLQ0Awc0n9DQUKpWrfpElclXEclBex7EHSVCLiPD2IjyFuWZ3HgyMpmM1Kw81A9ysRFGmJgaUdnb8AmSqcKEgL5euFa3pW2IT4Ei1EWh1Wi4djKM9OSC0roatYorx3VF+nxatjY4JpPJMCpEyegfgZCiN2jadamGTGGM6lYmmWF30GTmkbw2CpGnwayqrU4I5F9PNo0UJpg9SunMV3PMuXAftAK5qyXy8pZFns+0sjXyilagFqhuZ2JkYYLjQD+MzHX/IFg1csWq+SOHUIDMzBgj66dzrB7++ScpP+lyuu1693qqOSQkJCQkXg9MTEwYMWIE8+fPJysri1mzZjFlyhTmzp2Lj48PHTp0YMeOHfobWlNTUyZPnkzt2rVp1aqVPoIDuuhVZmYmQ4cOxc/Pj4CAAE6cOMGff/5JQEAAoPvt/fnnn/nmm2/07bVr12b69Ol07dqVoKCgIm0dP358sep3+Xh6ejJo0KAyKcBdq1Yt6tWrx+bNm/Vt3t7e/PXXX5w/f56mTZvSsmVL7ty5w99//63XKbC1tWXPnj3ExsZSv359xo0bx9SpUxkyZIh+nrS0NAMn8rfffuP+/fusX78eV1dX/etxYQyNRsOCBQvw9/enffv25ObmEhYWZiDWks+oUaMYN26cQXRn7dq1+ojbhAkTDOZ+Wg4dOkSrVq0AXaHypk2bFuuw5GNjY8ORI0d488038fT05PPPP2fBggV07NgRgJCQEBYtWsTSpUvx8/Ojc+fOBjXxEhMT9Wrqz4K7u3uRBc3zuXv3LrGxsTRp0oS8vDxOnDihX/OT0Gg0DB8+XP998vT0ZOnSpUX2v3LlSpGR5HzefPNN6taty19//cWhQ4eoW7dugUjZL7/8wgcffFAiG181ZOJZNWr/A6Snp2Nra0taWppeZemZ2DmBRdc2sdrOlreqvcUXLXQh+qPXkln2/VmaKeVUq+tEhw9rPWGiJ5OTkc72b+eTcCESawcnBi5Ygqn5P5Gj62dOsvWrWVjalWPIsrUYGRUvLatJU5I49xTIoMLUpnoHqDAyTybycMt1ZKbGmDibo7qViYmDAqeP6mBsWbhzlD/GtLI1zsPrkLTsHHnx6di+6YF1q+Kl6LNO3yX192tgJMPp/Zp6hcl8hEbwYF0UuVdSMa1ijfNHdYqdrzCyw8NJCBmIUKlw+PBDnMeMLvUcEhISEq86Zf67J/GfZseOHUyYMIGLFy8WmkYo8fqSnZ2Ng4MDu3bteqJk/+tEVFQUbdq04erVq2WiCPmieTbtdonCub6f048Ujxq5/CMwEXkzlRr5+8/qlGzDolar4cTvGzE2keMb0AZr+382jd6LjWHbgjmk39dtus14cJ9jmzfoCk0/Ij+90bt5wBOdMwBl7GP7z4pxzkAn3pEdnkRefDqqW5nIFMY4hPgV6ZwBmPs68PDP6+TdzEAZl0ZefLquGHYJrodFvfKoHyoxrWJdwDkDneKjfV9vMo7cQlFMCYCiyLt1m1sjRiJUKqzbt8dpVMlliCUkJCQkJP6rdOrUiWvXrnH79m2DvXgSrz8HDx6kTZs2/ynnDHTRxXXr1r2Wzhk8pYO2f/9+9u/fT1JSUoH6C2VRpTyf6dOnM2PGDIM2Ly8vvfpRbm4u48aNY+PGjSiVSoKCgli6dKm+zsZLISWWrNQbRLnpokENXBoghOCnsDjW7r7Oe1ozkIFbTYcSTXcj/AzHf/sFgGOb1lO1fkNqt+1AbmYGe1d+j1qVh115V+p26MzBn34gYtdf+LZsTfmq1VFmZxFzVlec79/pjUVRmLx+Uehqo1Xn3ncRIAQO7/ogdy5+35extalOhTEhg9Rfr+rOVc0OY5sni6HIjGXYti96oy7o0igfr6NWUjSZmdwaNgxNSgpmvj5U+HIeMukpoYSEhISEBKDbRyTx36NTp0506tTpZZtR5rRr1+5lm/BMlNpBmzFjBjNnzqRBgwa4uro+s4LSk/Dz82Pfvn3694/n9I4ZM4YdO3bw66+/Ymtry4gRI+jevTvHjh17rjYVS8x+IhRmaGQyKlpVxMHMhXG/nuOPs7dpnauLLFXwtENhKScnM4Md387HvmIl2gz8sNDprp86DoC5jS056WnEnDlJzJl/KqJ71KlPh4GjydyWQAv/3hw9t4k9Kxfz7hffcO1kGBqVCvuKlXF2L74oYD7KGw+BkjloAPLyljgP8weh2ydWEsz9HMhLyED9QJcbb1HXuUTjnhdCCO5MnITy2jVMnJyovHQpRqUsfCohISEhISEhISFRFpTaQVu+fDlr164lODj4edhTABMTk0LrZaSlpbF69Wp+/vln2rRpA8CaNWvw8fHhxIkTNGnS5IXY9zj3UrOxjt7DaYUuGlTDpg7vLA/j8q10uuSY4p2nSzH0beqKRq3mr2/mcjPqPPHnI6jboQvlXAxVD7UaDTHhumLJXUZPxMKuHBf27ybq8H5yMzNo/HZvmvXqR9rWG+TFplPJrCoKS2uSYmOI+Psvbjwa69uydYkcaXWaUuc0yYpWcCwM00olc8zyUfg5krYrDgCZ3Egvv/+ySN+xk8z9+5HJ5VRaugT5E+qzSEhISEhISEhISDwvSu2g5eXl0axZs+dhS6Fcu3aNChUqoFAoaNq0KXPnzqVKlSqcPXsWlUplEML09vamSpUqHD9+vFgHTalUolQq9e/T09MBXT2Mx+t1lIaff71M5pH72Jt04KaLDaZW59h1xgLLlAwG5ChwUMuQGclo2t0D97r27Fu1lJtR/1RSjzpygMZv9zaY81b0RXIz0lFYWeFczRMjY2Oa9w2h8Tv9yM3MwKqcPXkPcsg6o9uDJpRaAtqHsPvP7zm66f9Q5+UBUL1x8xKtKyviLgAmFSzRGAs0T3ktnoitCcZO5mju52DqVe75nusJaNLSuPdImrfckCGYeHs/9WdAQkJC4nVB+ndOQkJC4tWl1A7a+++/z88//8yUKVOehz0GNG7cmLVr1+Ll5UViYiIzZsygZcuWXLx4kbt372JqalqgOnr58uW5e/dusfPOnTu3wN42gD179mDxlKltt45bYIcxKWp3qt1yp8qdHlyXa6imMsNUK8PIVItD3VwSss5xftEGksN1qYvW7tXJiLtO+J6dJJtaGUS67p/V9ZE7ufJ3EUUkq8RY4KRR6N/n3TVG4VSe3EfCIQonF46ePvNE+63STPC8ZI0MGTfkSZzcufOprkNJKWdrSoUMc6JlN8jZee3JA54Tzr//gV1KCkpnZ05UcIXnvG4JCQmJV4Hs7OyXbYKEhISERBGU2kHLzc1l5cqV7Nu3j9q1ayP/VxHfb775psyMy68DAVC7dm0aN26Mm5sbmzdvxtzc/KnnnTx5skGhxPT0dCpXrkxgYOBTyQ3fjkvn1q5zgAaXcpuIzm2DfY4LPo+CdE5u1rQf7INVOTPizp3lr19WA9Ci30Bqtglk1fD3UGWkU9+7Bi7VPAHdvqi1e/4EoFW3HlSr37jAeTUPlSSfjAQElq0qkHXkDi7Z1vQY9ym/fDYOrUZNsy5vU7NNYLH2qx/kkLLiIkJoUNR2oME7jZ/73sJ8ipf8eL7knD3L7VO6NNCq87/Er379l2iNhISExIsjP3NEQkJCQuLVo9QO2vnz56lTpw6gq/j+OM/7pt7Ozg5PT0+uX79O+/btycvL4+HDhwZRtHv37hW6Z+1xzMzMMDMrqBool8sLOJwl4fjh2wA4mUUT77KXzbYn6WU7EO9LzuRlJ+LgbEPEznAQEHV4H0Joqdm6PY3e6oFMJqN6gyZcPnaYa8ePUtnbT7eO2Bgyku9jYmpGtboNCrUrMzQOtAKz6nbYBXqQffIe2gwVDsZOdBg+hltRF6jZul2xa9Jmq3iw/ioiR4NpZWscenojk//31Qu1eXncnzUbALue72DzEvYsSkhISLwsnua3TkJCQkLixVBqB+3gwYPPw44SkZmZSUxMDMHBwdSvXx+5XM7+/fvp0aMHoKs8npCQQNOmTV+YTUIruH8xBWOgnvkutpjrZPRrKBQknNsEwN1/ZfBV8qlJu/c/0ju0vi1bc/nYYS6HHSEgeDDGJiZcP30CAHf/esjNFPwbdUqufu+ZTbsqyEyMUHiVI+d8MrnRD/DpEIBP84DibddoefDzZdTJORjbmuEwwPd/wjkDeLBqFXkxMRg7OOA8fvzLNkdCQkJCQkJCQkICeMZC1bdu3QKgUqVKZWLMvxk/fjxdunTBzc2NO3fuMG3aNIyNjenbty+2trYMHjyYsWPHYm9vj42NDSNHjqRp06YvRMFx2bllPMh5QLDdhxjnatHKVDgpzhJt6oKxRkba9tOATgbf2aOafpzCyppabQIxNpEjNFpSt1ynnLWDXkY//kIEVes2JOa0bv9Z9YaFryXj4E199MzMXae4aO7nSM75ZHKiHmDbwcOgvzpNServ19Bm/7MxXCg1qO/nIDM1wiHEF2Nr0zK9Ri8DTVoaiZ9/jnW7dth27VpoH2VMDA+WrwCg/OTJGL+mRQwlJCQkJCReBHl5efj6+rJu3boXKhQn8XoTFxeHh4cHERER+uy7kjJlyhTu3bvHypUrn49xRfD3338zadIkwsPDMXpUDzcvLw9PT09+++03GjRo8ELsKHW4RKvVMnPmTGxtbXFzc8PNzQ07OztmzZpVoGj1s3Lr1i369u2Ll5cXvXr1wsHBgRMnTuDk5ATAwoUL6dy5Mz169KBVq1a4uLjwxx9/lKkNhZGck8zSyKVsurKJn7ftAqC6WRjnzI3QyqBlfGUyk5OxdnCi85hJtOgzQP9q0PltzCwsAcg6e4/sM/fIPHiLpl5vA3Ap9BAP793lfkIcMiMjqtZvVOD86pRcss4+ip49VrhZ4VUOjGWo7+egSvpnA7gQgod/XEN5NRXVrUz9S30/B2Rg39sL0wpWz+tyvVDStm4lY+8+7kyaTMahQwWOax4+5OZHHyHy8rBs0QKbTm++eCMlJCQkJF4KAwcORCaTIZPJkMvleHh48Mknn5Cbm1tm5zh8+DBt2rTB3t4eCwsLatSoQUhICHmPlJVB97u8cuVKGjdujJWVFXZ2djRo0IBFixbpBVymT5+OTCZj6NChBvNHRkYik8mIi4sDdDfBMpkMZ2dnMjIyDPrWqVOH6dOn699nZmYyYsQIKlWqhLm5Ob6+vixfvvyJa1q+fDkeHh565+zQoUP66/jv1+nTpw3s+vfrxIkTxZ4rISGBTp06YWFhgbOzMxMmTECtVuuPR0REULduXaysrOjSpQspKSn6Y2q1mvr163Pq0f7ysuT+/fuYmpqSlZWFSqXC0tKShISEMj/P0zJ06FBkMhmLFi16pnk6deqkd4qGDBnCzJkzy8C60nH37l2+/fZbPvvsM33bkSNH6NKlCxUqVEAmk/Hnn3+Wet64uDgGDx6Mh4cH5ubmVKtWjWnTphl8Nzt06IBcLmfDhg36NlNTU8aPH8/EiROfaV2lodQO2meffcb333/PvHnziIiIICIigjlz5rB48eIyV3bcuHEjd+7cQalUcuvWLTZu3Ei1ao9FoxQKlixZQkpKCllZWfzxxx9P3H9WFpy7fw4AY60JZvH2ANQy38dpRzfKpctxu6JLXWw7eCimisLFTIRaS8aBm/r3TsnlqWhRg+unTxB95AAAlX1rYm5lWGNMq1ST+ud1XfSshh1mbv+ImhgpTDCrZgdATtQDfXvOhWRyr6SCsQz7Pl44DPTTv1zGN8Dcz/EZr8irQ2boUd3/CMGdsePIvXpVf0yoVNwaNRpVfALyChWo8OW8FyaGIiEhISHxatChQwcSExO5ceMGCxcuZMWKFUybNq1M5o6OjqZDhw40aNCAI0eOcOHCBRYvXoypqSkajUbfLzg4mNGjR9O1a1cOHjxIZGQkU6ZMYevWrezZs0ffT6FQsHr1aq5de7LacUZGBl9//XWxfcaOHcvff//N+vXruXTpEqNHj2bEiBFs27atyDFCCL7//nsGDx6sb2vWrBmJiYkGr/fffx8PD48CEYZ9+/YZ9KtfjCCXRqOhU6dO5OXlERYWxk8//cTatWuZOnWqvs/7779PmzZtCA8PJy0tjTmPSuUALFiwgObNm9OoUcGH28/K8ePH8ff3x9LSkvDwcOzt7alSpUqZn+dp2LJlCydOnKBChQpP7lwMQghOnDhB8+bNAQgNDdX//4tk1apVNGvWDDe3f4IQWVlZ+Pv7s2TJkqee9/Lly2i1WlasWEFUVBQLFy5k+fLlfPrppwb9Bg4cyHfffWfQ9u6773L06FGioqKe+vylodQO2k8//cSqVasYNmwYtWvXpnbt2nz00Uf88MMPrF279jmY+OoRmRQJQO3sZphpLMg0TeWC4wNO21ei2UUHZAJqNGpWqPJiPlln76F5qMTIWo5lI51T2dT5Layw5dTWXwGo1sBwL50qOYekJedQXtU5W7ZB7gXmzS/6nBOtc9C0OWoe/hUDgPUblbGo44y5t73+ZeLw9GqYrxra3FyyHz25M6tRA212NreGfYT6wQOEENydOYvskycxsrCg0rJlmDi83ALZEhISEhIvHjMzM1xcXKhcuTLdunWjXbt27N27V39cq9Uyd+5c/VN2f39/fvvtN/3x1NRU3n33XZycnDA3N6dGjRqsWbMG0JXrcXFxYf78+dSsWZNq1arRoUMHfvjhB7369ObNm9mwYQO//PILn376KQ0bNsTd3Z2uXbty4MABWrdurT+Xl5cXrVu3NogkFMXIkSP55ptvSEpKKrJPWFgYISEhvPHGG7i7uzNkyBD8/f2LjTidPXuWmJgYOnXqpG8zNTXFxcVF/3JwcGDr1q289957BR58Ojg4GPQtTqBmz549REdHs379eurUqUPHjh2ZNWsWS5Ys0Uc5Ll26xAcffICnpyd9+/bl0qVLANy4cYPVq1fzxRdfPPFaPQ1hYWF6Z+Xo0aMlclwOHTpEo0aNsLS0xM7OjubNmxMfH68//tdff9GwYUMUCgWOjo68/fbbpbbr9u3bjBw5kg0bNjyz+M+VK1cQQuDr60tycjLXr1+nceOi72Wh+O/D07Jx40a6dOli0NaxY0dmz579VNconw4dOrBmzRoCAwOpWrUqb731FuPHjy+QfdelSxfOnDlDTEyMvq1cuXI0b96cjRs3PvX5S0OpHbSUlBS8vb0LtHt7exuEmf/L5Dtore7VA+Ca41mmOluhvXAPp4dmmCgUtH5vSJHjhVqr20MGWAdUxq5rdcxq2GEsM6Fl+R7Itbq9YI/vP8u5nELS9xGok7IxsjbFaUhtTCtZF5jb3NcBZKC6mYEmTUna37FoM1SYOJpj80blsroEryTZZ84icnMxKV+eKut+Qu5WBdXt29wa+TEpq1fz8NdfQSajwoKvUXh5vmxzJSQkJP4zCCFQKTUv/CWEeCa7L168SFhYGKam/+zBnjt3LuvWrWP58uVERUUxZswY+vfvz+HDhwHd3pjo6Gh27drFpUuXWLZsGY6OukwUFxcXEhMTOXLkSJHn3LBhA15eXnQtZJ+0TCbD9l/7oufNm8fvv//OmTPF1zTt27cv1atXLzYlrVmzZmzbto3bt28jhODgwYNcvXqVwMCiy/GEhobi6emJtXXBe458tm3bxoMHD3jvvfcKHHvrrbdwdnamRYsWxUbqQBelqlWrFuXLl9e3BQUFkZ6ero9c+Pv7s3fvXtRqNfv376d27dqALsVv/vz5xdpZWhISErCzs8POzo5vvvmGFStWYGdnx6effsqff/6JnZ0dH330UaFj1Wo13bp1IyAggPPnz3P8+HGGDBmid2B37NjB22+/zZtvvklERAT79+83iPxNnz4dd3f3Yu3TarUEBwczYcIE/Pz8nnqdnTt31qfZpqWlUa5cOTw8PNBoNFSqVKlAzeHHKe77UBju7u4Gabf/JiUlhejo6Be21ystLQ17e3uDtipVqlC+fHlCQ0MN2hs1alSg7XlRapEQf39/vv/++wKhv++//x5/f/8yM+xVJU+TR9SDKMxUFqiSdOIoSTY3MMpWU/eKMwCt+g3E2r7oD+c/0TNTrBq7IDOW4dDPh8TvzmCRakPL8j1INUtGFqUkg1uoH+aSdSIRBJi62eDwrg/GNoULehhbm2Ja2Zq8hAzS/o4jO0L3JM3u7er/eYXGrKO69EbLli0wKVeOysuWEde7Dznh4eSEhwPgPGEC1o89nZSQkJCQeHbUeVpWjjr8ws875NsA5GbGpRqzfft2rKysUKvVKJVKjIyM+P777wFQKpXMmTOHffv26RWhq1atytGjR1mxYgUBAQEkJCRQt25d/Q3k4zfRPXv2ZPfu3QQEBODi4kKTJk1o27YtAwYM0NdZvXbtGl5eXiW2t169evTq1YuJEyeyf//+IvvJZDLmzZtHly5dGDNmjMGWkHwWL17MkCFDqFSpEiYmJhgZGfHDDz/QqlWrIueNj49/Yurc6tWrCQoKMhCNs7Ky0qccGhkZ8fvvv9OtWzf+/PNP3nrrrULnuXv3roFzBujf3717F9Clv3300Ud8/fXXNG/enMmTJ/N///d/WFhY0LBhQ4KCgoiJiaFPnz7Mnj27WLufRIUKFYiMjCQ9PZ0GDRpw8uRJLC0tqVOnDjt27KBKlSpYWRW+hz89PZ20tDQ6d+6s/1v4+Pjoj3/xxRf06dOHGTNm6Nsev492dHQs9G/4OF9++SUmJiZ8/PHHz7JMVq1aRW5uLkOHDqVJkyYMHDiQqVOnYmdnZ1A3uDCK+z4URrVq1Yp14BISEhBCPHO6Zkm4fv06ixcvLjQ1uEKFCgbRzqLanheldtDmz59Pp06dDP7xOn78ODdv3mTnzp1lbuCrRvSDaFRaFQ2S66NFjsY4leqKd2h0cg2mahkaF0v823cscvzje8+s36iETK77YTEyN6H8YH9ufh2GvZkr9riStivWYKxlE1fsOldFZlK8o2Xu50heQobeObOoXx7Fo71p/2Uyj+qeali1aAGAWdWqVFy0kJtDPgSNBtt3emD/3sCXaKGEhISExMumdevWLFu2jKysLBYuXIiJiYm+XM/169fJzs6mffv2BmPy8vKoW7cuAMOGDaNHjx6Eh4cTGBhIt27d9OIZxsbGrFmzhtmzZ3PgwAFOnjzJnDlz+PLLLzl16hSurq5PFfWbPXs2Pj4+7NmzB2dn5yL7BQUF0aJFC6ZMmcLPP/9c4PjixYs5ceIE27Ztw83NjSNHjjB8+HAqVKhAu3btCp0zJycHhaJguZ98bt26xe7du9m8ebNBu6Ojo8HNfcOGDblz5w5fffVVkQ5aSfDz89NHMwEePHjAtGnTOHLkCCNHjqRZs2b88ccfNGzYkMaNGxdIlQNdtG39+vX695mZmYWey8TEBHd3dzZv3kzDhg2pXbs2x44do3z58sU6tQD29vYMHDiQoKAg2rdvT7t27ejVqxeurq6ATuzlgw8+KHL8iBEjGDFiRJHHz549y7fffkt4ePgz76d3cXFBpVJx4sQJFi9ejLu7O8ePH2fNmjVPdLiK+z4URnEPGUD3eQOK/cyVBbdv36ZDhw707Nmz0L+Dubm5XrCnuLbnRakdtICAAK5evcqSJUu4fPkyAN27d+ejjz56Id7uyyY/vdH3vm6T6yXVLWqEHkKoZWBnztujPsXIqOineVln7qFJexQ9a2QoaGLiaI5VHzeS917FqZI7RsaP5pHJUHiVw6K2U4lsVPg56J07I0sTbN/0eMKI1x9VYiJ512PAyAjLx+rgWTVvTuUVK8i9FI1DSIgkCiIhISHxHDAxNWLIt8XX3nxe5y0tlpaWVK9eHYAff/wRf39/Vq9ezeDBg/U36jt27KBixYoG48zMzADdXpj4+Hh27tzJ3r17adu2LcOHDzd4Cl+xYkWCg4MJDg5m1qxZeHp6snz5cmbMmIGnp6f+/qmkVKtWjQ8++IBJkyaxevXqYvvOmzePpk2bMmHCBIP2nJwcPv30U7Zs2aLfT1a7dm0iIyP5+uuvi3TQHB0duXDhQpHnW7NmDQ4ODiVyuho3bmyw3+/fuLi4FNgPd+/ePf2xwhg7diyjR4+mUqVKHDp0iNmzZ2NpaUmnTp04dOhQoQ7azJkzGV+CGqh+fn7Ex8ejUqnQarX6yKtarcbKygo3N7diRSPWrFnDxx9/zN9//82mTZv4/PPP2bt3L02aNNHvSXxaQkNDSUpKMhAq0Wg0jBs3jkWLFulVPp/EnDlzmDNnDkIIsrOz9Q8isrKyCAoKQiaTsWvXLlq2bFno+JJ8H0pDfnQtNTVVr9pe1ty5c4fWrVvTrFmzImX8U1JSCpy/sLbnxVPVQatQocJz24T5qhORFIFCZYV5ZjVUOYeorgxHAB51G/DmiPEoigh1g+HeM5vHomeP41SnOk51qj+TjXJHc+QVrVDdzsS2czWMLZ9t0+jrQOaj9Ebz2rUL1DWzatEcqxYvXoVIQkJC4n8FmUxW6lTDVwEjIyM+/fRTxo4dS79+/fD19cXMzIyEhAQCAop2OJ2cnAgJCSEkJISWLVsyYcKEIm9Iy5Urh6urK1lZWQD069ePPn36sHXr1gL70IQQpKenF9iHBjB16lSqVav2RJGCRo0a0b17dyZNmmTQrlKpUKlU+tpO+RgbGxdbJqlu3bosW7YMIUSBh5xCCNasWcOAAQNKJFARGRmpjyAVRtOmTfniiy9ISkrSRwr37t2LjY0Nvr6+Bfrv37+fS5cu6UUpNBoNKpVKv96icHZ2LjYSmc/OnTtRqVS0bduW+fPnU79+ffr06cPAgQP1cuxPom7dutStW5fJkyfTtGlTfv75Z5o0aULt2rXZv39/ofv2SkJwcHABpzooKIjg4OBSzTl06FB69erF0qVLuXXrFnPmzOHXX39lz549/PDDDwAFHlb8m9J8H55EtWrVsLGxITo6Gk/PstcLuH37Nq1bt6Z+/fqsWbOmwPcBIDc3l5iYGL2zms/FixcLtD0vSuSgnT9/npo1a2JkZMT58+eL7Zu/WfO/iBCCyPuRVE71QpW5Da1aF6Vq0qMPzd7ph6yQP/LjZJ19FD2zMcWyUdH/QJUFDsG+qJNzUFS3e67neVXIOnoMAMtH6Y0SEhISEhIloWfPnkyYMIElS5Ywfvx4xo8fz5gxY9BqtbRo0YK0tDSOHTuGjY0NISEhTJ06lfr16+Pn54dSqWT79u36vUUrVqwgMjKSt99+m2rVqpGbm8u6deuIiopi8eLFAPTq1YstW7bQt29fPv/8cwIDA3FycuLChQssXLiQkSNH0q1btwJ2li9fnrFjx/LVV189cU1ffPEFfn5+mJj8c5tnY2NDQEAAEyZMwNzcHDc3Nw4fPsy6dev45ptvipyrdevWZGZmEhUVRc2aNQ2OHThwgNjYWN5///0C43766SdMTU31N7R//PEHP/74I6tWrdL32bJlC5MnT9ZHFAMDA/H19SU4OJj58+dz9+5dPv/8c4YPH66PYOaTm5vLiBEj+OWXX/Q32c2bN2fJkiUMHz6c33//vdh1lQQ3Nzfu3r3LvXv36Nq1KzKZjKioKHr06FGsowkQGxvLypUreeutt6hQoQJXrlzh2rVrDBgwAIBp06bRtm1bqlWrRp8+fVCr1ezcuVNfa+v7779ny5YtRaYEOjg44PAvNWq5XI6Li0up9jja29tjb29PdHQ0vXv3pnr16ly7do3AwEB9pLk4ivs+FEbbtm15++23i0zfNDIyol27dhw9etTge5CZmcn169f172NjY4mMjCxVuYPbt2/zxhtv4Obmxtdff839+/f1xx6P0J44cQIzMzP9Vq58QkNDmTVrVonO9cyIEiCTycS9e/f0/29kZCRkMlmBl5GRUUmme+VIS0sTgEhLSyu2X3xavKi5tqaYNGqC+LpXJzG/V1cxf8WvJT7P/TUXxc2JR0T6oYRnNVniMbQqlbjcoKGI9vIW2ZGRL9scCQkJiVeekv7u/dcICQkRXbt2LdA+d+5c4eTkJDIzM4VWqxWLFi0SXl5eQi6XCycnJxEUFCQOHz4shBBi1qxZwsfHR5ibmwt7e3vRtWtXcePGDSGEEOHh4aJ///7Cw8NDmJmZCQcHB9GqVSuxbds2g/NpNBqxbNky0bBhQ2FhYSFsbGxE/fr1xbfffiuys7OFEEJMmzZN+Pv7G4xLS0sTjo6OAhCxsbFCCCFiY2MFICIiIgz6DhkyRABi2rRp+rbExEQxcOBAUaFCBaFQKISXl5dYsGCB0Gq1xV63Xr16iUmTJhVo79u3r2jWrFmhY9auXSt8fHz062vUqJH49VfDe6Y1a9aIf9+KxsXFiY4dOwpzc3Ph6Ogoxo0bJ1QqVYH5J02aJMaNG2fQdu3aNdGwYUNhY2Mjhg0bJjQaTbHrKgm//PKLaNGihRBCiCNHjojq1auXaNzdu3dFt27dhKurqzA1NRVubm5i6tSpBjb9/vvvok6dOsLU1FQ4OjqK7t27649NmzZNuLm5lcpWNzc3sXDhQoO2gIAAERISUuw4lUolrKysxPXr14UQQlStWlUcPXq0ROcs7vtQ2GfTzc3N4DNZGDt37hQVK1Y0uFYHDx4UQIHX42t70jXL/7wV9nqcIUOGiA8//NCgLSwsTNjZ2em/n88bmRBP3q0aHx9PlSpVkMlkT1Qvebyo3OtCfjpBWlqaXmWpMLbFbOOzI58zcF9dUD/ginU9Bn86isZVn1xPSwhB4uyTaLNUOH3kj1mVos8jUTqywyOI79cPY1tbaoQdQ2b8+qXZSEhISLxISvq7JyEBukyq9u3bExMTU6RqocSriZubGzNmzGDgwIEv25QSI4SgcePGjBkzhr59+5Z4XMgjrYFnqcucnJyMl5cXZ86cwcPjHw2H3r174+/vX6Co9fOiRCmOjztd8fHxNGvWzCBsDrp6D2FhYa+lg1ZSIu9F4JdQEdQPADnHyvmzuIpdicZqUpVos1RgJMPUVfrHrSzJeqTeaNGsqeScSUhISEhIlDG1a9fmyy+/JDY2llq1ar1scyRKSFRUFLa2tvq0ytcFmUzGypUrixWn+TdCCA4dOsTRR5oET0tcXBxLly41cM7y8vKoVasWY8aMeaa5S0OJImiPY2xsTGJiYoHNlQ8ePMDZ2RmNRlOmBr4ISvok8e1fO1B/uyXmuUoyLBqw16sVxya1KdE5ss/fJ+Xny8grWlF+5IvZYPi/Qmyv3uSeP4/rF19g16P7yzZHQkJC4pVHiqBJSEhIvLqUWp9WFKLiAzoHzdLSskyMehVJz0tHdSMF81wlYMpFG3/cHS1KPD7vVgYAppXLrsL964zq9m0erF6NNjf3meZRp6aS++gJi6Wk1CghISEhISEhIfGaU2KZ/e7ddZEJmUzGwIEDDdR0NBoN58+fL7Yw3evOubuR1LlaDgBjRV2umZrRyaHkDmneTV1tFdNKUnojQNLCRaRv3442Nxen4cOfep6ssDAQAjNPT+Tly5ehhRISEhISEhISEhIvnhI7aPk1OYQQWFtbGxTYMzU1pUmTJsVWRH/dCT/wO+Uy5YApwrYeacYCd4eSRdCEVqC6LUXQHif30iUAMvbuezYH7Yhu/5kkry8hISEhISEhIfFfoMQOWn4RQHd3d8aPH/+fTmf8N0KrJfPYNRQYYayoS6KVJWizqGJfsmugTspG5GmRmRph4lTytMj/KiIvj7xHaqDKy5fJu3UL00qVSj2P8vp10nbuBMC69RtlaKGEhISEhISEhITEy6HUe9CmTZv2P+WcAVw7cwJFuhFCJsfErD7RQledvqR70PL3n8krWiMzKrh/73+NvPh4UKv17zP27Sv1HEKrJXHqNFCpsGrdGvMGDcrSRAkJCQkJCQkJCYmXQokjaI/z22+/sXnzZhISEsjLyzM4Fh4eXiaGvUpcPL0HABNTX4zlCi5rc0AGVexL6KDdlNIbH0f5WCV4gMx9+3EopD6HUKtR37uHvGLFAsce/vobOeHhyCwscJnyeaHCNRISEhISEhISEhKvG6WOoH333Xe89957lC9fnoiICBo1aoSDgwM3btygY8eOz8PGl05iXDQAMiMHrCtbo5KBs7UZFqYl82/zbkkCIY+jvKZz0CybNQUgOzwcdUpKgX53Jk7iett2JE6dhvaxBwHq+/dJWrAAAKePRyKvUOEFWC0hISEhISEhISHx/Cm1g7Z06VJWrlzJ4sWLMTU15ZNPPmHv3r18/PHHpKWlPQ8bXzq5qTkAyIzLQXmdeqV7CRUchUqLKjELkCJo+eRH0CxbtkLh6wtaLZkHDxr0ybkYRfqOHQA83LyZhOABqO4lAXBv7jy06ekofH2x79//xRovISEhISHxP0ZeXh7Vq1cnLCzsZZsi8R/j0KFDyGQyHj58WKbzTpo0iZEjR5bpnC+SUjtoCQkJejl9c3NzMjJ06XvBwcH88ssvZWvdK4BWo0FkagEwMi5Hio0xAG4lVHDMS8wErcDIUo6xndmTB/wPkO+gmVWvjlW7tgBk7Ntv0Cf5++8BMK9bFyMbG3LOnSO2Rw+Sly8nfedOMDLCZdZMZCZPlaUrISEhIfE/yMCBA5HJZMhkMuRyOR4eHnzyySfkPmNNzsc5fPgwbdq0wd7eHgsLC2rUqEFISIjBlhAhBCtXrqRx48ZYWVlhZ2dHgwYNWLRoEdnZ2QBMnz4dmUzG0KFDDeaPjIxEJpMRFxcHQFxcHDKZDGdnZ/09WT516tRh+vTpBm2XLl3irbfewtbWFktLSxo2bEhCQkKxa1q+fDkeHh76+7/8m+rCXqdPnwbgypUrtG7dmvLly6NQKKhatSqff/45KpWq2HMlJCTQqVMnLCwscHZ2ZsKECagf27cOsGHDBvz9/bGwsMDV1ZVBgwbx4MED/fG9e/fi6emJjY0NwcHBBtc+LS0NT09P4h+JlZUlp0+fpsKjrJ47d+5gbm5eYCvQiyQlJYWRI0fi5eWFubk5VapUKZOAip+fH3v26Lb/BAYGsm7durIwt8QcOnSIrl274urqiqWlJXXq1GHDhg0GfcaPH89PP/3EjRs3XqhtZUWpHTQXFxdSHqWjValShRMnTgAQGxuLEKJsrXsFSL+fhEzIAGOMzOXEqXVfNHfHkkXQVI/tP/tf2SeljI0lceo0VHfvFjimfUzB0axGdazbtgMg69gxtFm6SGPOhYtkHjoERka4zvkCj99+xczTE01yMvcXfQuAfXAw5n5+L2ZBEhISEhL/GTp06EBiYiI3btxg4cKFrFixgmnTppXJ3NHR0XTo0IEGDRpw5MgRLly4oM840mg0+n7BwcGMHj2arl27cvDgQSIjI5kyZQpbt27V3/gCKBQKVq9ezbVr15547oyMDL7++uti+8TExNCiRQu8vb05dOgQ58+fZ8qUKSgUiiLHCCH4/vvvGTx4sL6tWbNmJCYmGrzef/99PDw8aPBItEsulzNgwAD27NnDlStXWLRoET/88EOx11qj0dCpUyfy8vIICwvjp59+Yu3atUydOlXf59ixYwwYMIDBgwcTFRXFr7/+yqlTp/SlnrRaLf369WPo0KEcP36cM2fOsHLlSv34SZMmMXToUNzc3Iq/oE/B8ePHad68OQChoaE0aNAAU1PTMj9PSblz5w537tzh66+/5uLFi6xdu5a///7b4G9ZWh4+fMjVq1dp0qQJGo3GYM0virCwMGrXrs3vv//O+fPnee+99xgwYADbt2/X93F0dCQoKIhly5a9UNvKDFFKBg8eLKZPny6EEOL7778X5ubmol27dsLOzk4MGjSotNO9EqSlpQlApKWlFTh2I/y0+LpXJ7Ggb4hY//WvosfSY8Jt4naxLfJ2ieZ+sPGyuDnxiEjbG1fWZr+SaLVaEdu7j4j28haJM2YUOJ5z+YqI9vIWl+s3EFqtVmi1WnGtfaCI9vIWaX/vFkIIkTDkQxHt5S1uf/KJfpwmK0vcHD1aRHt5i6utWwtNZuYLW5OEhITEf43ifveeBq1WK/Jycl74S6vVlsrOkJAQ0bVrV4O27t27i7p16+rfazQaMWfOHOHu7i4UCoWoXbu2+PXXX/XHU1JSRL9+/YSjo6NQKBSievXq4scffxRCCLFw4ULh7u5erA2bNm0SgPjzzz8LvY4PHz4UQggxbdo04e/vL9q3by969uyp7xMRESEAERsbK4QQIjY2VgBiwoQJwsrKSty7d0/f19/fX0ybNk3/vnfv3qJ///7FX6R/cfr0aWFkZCTS09OL7JOXlyecnJzEzJkzi51rzJgxokWLFkUe37lzpzAyMhJ3797Vty1btkzY2NgIpVIphBDiq6++ElWrVjUY991334mKFSsKIYS4d++eAEROTo4QQohPPvlEfPTRR0IIIY4dOybq168v1Gp1sXY+Lb179xYLFy4UQggxYsQIMXHixCeO+fXXX0XNmjWFQqEQ9vb2om3btiLzsXuc1atXC19fX2FqaipcXFzE8OHDn8nGzZs3C1NTU6FSqZ5q/K5du4S/v78QQogzZ86I8uXLP3FMXFyc6Ny5s7CzsxMWFhbC19dX7NixQwghxMGDBwUgUlNTn8qefN58803x3nvvGbT99NNPolKlSs8078ui1PlhK1euRKvVpfwNHz4cBwcHwsLCeOutt/jwww/LznN8RUiKvwmAzMiOKnVMiDuuSz0o6R40vcT+/8j+s6xjYeRERur+/9SpAsfzYv5Jb8yPKFq3bUvKmjVk7NuHvIIrmYcPg7ExjsOG6ccZWVhQ8ZtvyO7dB7NqVTH6Hyv1ICEhIfEqo1Yq+S7knRd+3o9/+g15MdGfJ3Hx4kXCwsIMoilz585l/fr1LF++nBo1anDkyBH69++Pk5MTAQEBTJkyhejoaHbt2oWjoyPXr18nJ0e3V93FxYXExESOHDlCq1atCj3nhg0b8PLyomvXrgWOyWQybG1tDdrmzZtHw4YNOXPmjD46VRh9+/Zl7969zJw5k+8fbRN4HK1Wy44dO/jkk08ICgoiIiICDw8PJk+eTLdu3YqcNzQ0FE9PT6yti76P2bZtGw8ePOC9994rss/169f5+++/6d69e5F9jh8/Tq1atShfvry+LSgoiGHDhhEVFUXdunVp2rQpn376KTt37qRjx44kJSXx22+/8eabbwLg5OSEq6sre/bsoV27doSGhhISEoJKpWLYsGH8+OOPGBsbF2lDaTl69CidO3cGIDMzk7/++ovp06eTlZWFXC5n+fLlTJo0iUmTJhUYm5iYSN++fZk/fz5vv/02GRkZhIaG6jPSli1bxtixY5k3bx4dO3YkLS2NY8eO6ccPHDiQuLg4Dh06VGJ709LSsLGxwaSUW0Rq166tV29XqVTY2dmhUqlQKpXY2dlRpUoVzp8/X+jY4cOHk5eXx5EjR7C0tCQ6Ohorq6KF82QyGWvWrGFgIQrfxa3Lx8fHoK1Ro0bcunWLuLg43N3dSzzXq0CpHTQjIyOMjP7JjOzTpw99+vQpU6NeJWIjrwKQa2aCc/UKJO+9D0CVEuxB0+aoUd/X/aNtWum/76AJIUhevFj/Pu96DOoHDzBxcNC36fef1aiub7Nup3PQMg8dQvMofda2SxdM//VlkslkWDZp/BxXICEhISHxX2f79u1YWVmhVqtRKpUYGRnpHRqlUsmcOXPYt28fTZvqlIarVq3K0aNHWbFiBQEBASQkJFC3bl29s/T4jV/Pnj3ZvXs3AQEBuLi40KRJE9q2bcuAAQOwsbEB4Nq1a3h5eZXY3nr16tGrVy8mTpzI/v37i+wnk8mYN28eXbp0YcyYMVSrVs3geFJSEpmZmcybN4/Zs2fz5Zdf6h2mgwcPEhAQUOi88fHx+n1VRbF69WqCgoKoVKlSgWPNmjUjPDwcpVLJkCFDmDlzZpHz3L1718A5A/Tv7z7aNtG8eXM2bNhA7969yc3NRa1W06VLF5YsWaK/Dps3b2bMmDGMGjWKN998k0GDBjFv3jxat26NQqGgefPmJCcnM3LkSEaMGFHs2p5EgwYNiIyM5PLly/Tr14+zZ8+SkpKiX7dCocDOzq7QsYmJiajVarp3765/SFCrVi398dmzZzNu3DhGjRqlb2vYsKH+/11dXfVBk5KQnJzMrFmzGDJkSClXCTt37kStVtO5c2dGjx5Nu3btGDRoEB06dKBXr17FOnwJCQn06NFDv7aqVasWey4vL68CDyqKY/PmzZw+fZoVK1YYtOd/buPj4/+bDlpRHnFh1K5d+6mNedUQWkFSnC6CdtPhLpnCFbiPvaUptubyJ47Pj54Z2yswtnxy/9edrKNHyTl3DplCgYmDA6rbt8k+fRqbDh30ffIl9s2q/+Ogmdepg7GDA5oHD8g6duxR9GxogfklJCQkJF5NTMzM+Pin317KeUtL69atWbZsGVlZWSxcuBATExN69OgB6KI82dnZtG/f3mBMXl4edevWBWDYsGH06NGD8PBwAgMD6datm148w9jYmDVr1jB79mwOHDjAyZMnmTNnDl9++SWnTp3C1dX1qfbrz549Gx8fH/bs2YOzs3OR/YKCgmjRogVTpkzh559/NjiWfyPftWtXxowZA+hERMLCwli+fHmRDlpOTk6xe9Ru3brF7t272bx5c6HHN23aREZGBufOnWPChAl8/fXXfPLJJ8Wutziio6MZNWoUU6dOJSgoiMTERCZMmMDQoUNZvXo1AC1atNCLlQBcvXqVdevWERERQatWrRg1ahQdO3akZs2atGrVqtB7144dOxIaGgqAm5sbUVFRhdqjUChwd3dn8+bNdOzYEQ8PD8LCwmjZsiXe3t7FrsXf35+2bdtSq1YtgoKCCAwM5J133qFcuXIkJSVx584d2rZtW+T4uXPnPvF65ZOenk6nTp3w9fUtIBxTEipVqsTdu3eJiYmhT58+yOVyTp8+zU8//UTlypWLHfvxxx8zbNgwfVSzR48exfoLly9fLrFdBw8e5L333uOHH37A71/aBObm5gB64Z3XiRI5aHXq1EEmkyGEeKLQxeObYF934qMeoFLqIjo3na+TlKG7XCUuUP0/VP9MCMH9xbonkOX69EFoNKT+3/+RdfKkoYP2KIJm+piDJjM2xrpNax7+qvtxt33rLUyfw+ZdCQkJCYnng0wme6ZUwxeJpaUl1R/9Bv3444/4+/uzevVqBg8eTGam7nd7x44dVKxY0WCc2SNnsGPHjsTHx7Nz50727t1L27ZtGT58uIFAR8WKFQkODiY4OJhZs2bh6enJ8uXLmTFjBp6enqW6AQWoVq0aH3zwAZMmTdI7IUUxb948mjZtyoQJEwzaHR0dMTExwdfX16Ddx8eHo0ePFjmfo6MjFy5cKPL4mjVrcHBw4K233ir0eP7Nu6+vLxqNhiFDhjBu3LhC0wxdXFw49a/tEffu3dMfA51T0rx5c/36ateujaWlJS1btmT27Nm4uroWmPfDDz9kwYIFaLVaIiIi6NmzJxYWFgQEBHD48OFCnYVVq1bpU1fl8qIfsuen6uVHY7du3UpeXh5CCKysrGjZsiW7du0qdKyxsTF79+4lLCyMPXv2sHjxYj777DNOnjyJo6NjkecsLRkZGXTo0AFra2u2bNlS7HoKY+jQoaxfvx6tVotSqcTFxQUhBNnZ2fq0wujoaKpUqVLo+Pfff5+goCB27NjBnj17mDt3LgsWLHhmGfzDhw/TpUsXFi5cyIABAwoczxc1dHJyeqbzvAxKpOIYGxvLjRs3iI2N5ffff8fDw4OlS5cSERFBREQES5cupVq1avz+++/P1dh58+Yhk8kYPXq0vi03N1e/F87KyooePXrov8zPyvkDcaBN172xziXuQf7+sxI6aI8pOP7XyQoNJff8eWQKBQ7vD8aikS4En33qnydY2rw88h5J+T4eQQOwyn9CJEXPJCQkJCReEEZGRnz66ad8/vnn5OTk4Ovri5mZGQkJCVSvXt3g9XiUwMnJiZCQENavX8+iRYsMVAL/Tbly5XB1dSXrkVJxv379uHr1Klu3bi3QVwhRpAT61KlTuXr1Khs3bix2TY0aNaJ79+4F9jyZmprSsGFDrly5YtB+9erVYhUN69aty+XLlwuN/AkhWLNmDQMGDCjRTb9Wq0WlUhWZlte0aVMuXLhAUlKSvm3v3r3Y2NjoHcvs7GyDrTaA3tkrzMbVq1djb2/PW2+9pQ8i5Ev9q1SqIgMLFStW1P/ti7s+kZGRnDlzBmNjY/bv309kZCQODg5s3ryZyMhIVq1aVeRY0D3caN68OTNmzCAiIgJTU1O2bNmCtbU17u7uxaa1loT09HQCAwMxNTVl27ZtxUZDi2LmzJlERkbq0xsjIyMZOHAg/fv3JzIyksjIyCemwVauXJmhQ4fyxx9/MG7cOH744YenXRKgk9rv1KkTX375ZZEpmxcvXkQulxeIrL0WlFZVpGHDhnrllcfZsWOHqFev3tPLlTyBU6dOCXd3d1G7dm0xatQoffvQoUNF5cqVxf79+8WZM2dEkyZNRLNmzUo1d2FqVg+TssV3728SX/fqJOb0e1P0XVNfTPztnHCbuF18s+dKiea988UJcXPiEZF742Gp7Hnd0Gq14sY7PUW0l7e4O+9LIYQQ6tRUEe3tI6K9vIXq/n0hhBA5ly/rFBwbNCygvKVVq8Xd+fNF6m+/vXD7JSQkJP7XKGsVx9eFwlQcVSqVqFixovjqq6+EEEJ89tlnwsHBQaxdu1Zcv35dnD17Vnz33Xdi7dq1QgghpkyZIv78809x7do1cfHiRdG5c2fRqFEjIYQQy5cvF0OHDhW7d+8W169fFxcvXhSffPKJMDIyEocOHRJC6H4ze/fuLczNzcUXX3whTp8+LeLi4sRff/0l2rRpI7Zs2SKE+EfF8XGmTJkiFApFoSqOERER+n5XrlwRJiYmQqFQGKg4/vHHH0Iul4uVK1eKa9euicWLFwtjY2MRGhpa5DVLTk4WcrlcXLhwocCxffv2CUBcunSpwLH169eLTZs2iejoaBETEyM2bdokKlSoIN59910De7y8vPTv1Wq1qFmzpggMDBSRkZHi77//Fk5OTmLy5Mn6PmvWrBEmJiZi6dKlIiYmRhw9elQ0aNBA/zd4nHv37gl3d3dx+/Y/yts+Pj5i+vTpIiwsTFhZWYlTp04VufaScvz4cVGlShUhhBDx8fHC0tKyRCqJJ06c0H8G4uPj9QqLO3fuFEIIsXbtWqFQKMS3334rrl69qv8s5jNp0iQRHBxc5PxpaWmicePGolatWuL69esiMTFR/3oaJUsvLy9x4MABIYQQrVu3Fj///HOJxo0aNUr8/fff4saNG+Ls2bOicePGolevXkKIwlUcvby8xB9//FHkfAcOHBAWFhZi8uTJBmt68OCBQb9p06aJNm3alHKVrwaldtAUCoWIjo4u0B4dHS0UCkWZGPVvMjIyRI0aNcTevXtFQECA3kF7+PChkMvlBvK3ly5dEoA4fvx4iecv7Ifq6G/XxLfvLRdf9+okJn7QTgz/v5ai94ow4TZxu/gj/OaTbT52W9yceETc/DRUaJTPR871VSH94EER7eUtLvnXEarkZH17TNduOvn8Rw79w7+2i2gvbxHbp+/LMlVCQkJCQkgO2r+ZO3eucHJyEpmZmUKr1YpFixYJLy8vIZfLhZOTkwgKChKHDx8WQggxa9Ys4ePjI8zNzYW9vb3o2rWruHHjhhBCiPDwcNG/f3/h4eEhzMzMhIODg2jVqpXYtm2bwfk0Go1YtmyZaNiwobCwsBA2Njaifv364ttvvxXZ2dlCiMIdtLS0NOHo6PhEB00IIYYMGSIAAwdNCJ1se/Xq1YVCoRD+/v6Fyv3/m169eolJkyYVaO/bt2+RD8U3btwo6tWrJ6ysrISlpaXw9fUVc+bM0cvfC6Fztv4dK4iLixMdO3YU5ubmwtHRUYwbN66As/Pdd98JX19fYW5uLlxdXcW7774rbt26VcCGPn36iMWLFxu0nTx5Unh7ewt7e3sxo5ByQE/D3Llz9eUL1q1bJ9q1a1eicdHR0SIoKEg4OTkJMzMz4enpWcDe5cuX6z+Lrq6uYuTIkfpjISEhIiAgoMj5852fwl75nx8hhHBzcyvwOfk3iYmJwtTUVGRnZwulUinMzc0LveaFMWLECFGtWjVhZmYmnJycRHBwsEh+dL9YmIMGiDVr1hQ5X0hISKFr+ve18PLyEr/88kuJbHzVkAlRut2q9erVo2bNmqxatUpffC8vL4/333+fixcvEh4e/gzxvMIJCQnB3t6ehQsX8sYbb1CnTh0WLVrEgQMHaNu2LampqQYKOW5ubowePVq/CfbfKJVKlEql/n16ejqVK1cmOTkZGxsb1HkaNkw5RVZKGOqco8RUyMShiSX7bowlMS2XzR80om4Vu0LnBlBef8jDdZdBgFVQFSxbFB/2fVVQ373L/S/mYDsgGIvHVIKKQ5ORwe33BpF35Qp2A0NwHDdOf+z+l1+Stn4DNr164jxlCg8Wf0/qypXY9OiB8/SyKQoqISEhIVF60tPTcXR01EtuS0gUx/nz52nfvj0xMTHFyqNLvH5kZ2fj4ODArl27eOONN162OWXGrl27GDduHOfPny91SYFXgVJbvHz5crp06UKlSpX0myrPnz+PTCbjr7/+KnMDN27cSHh4uIEaTz53797F1NS0gHxp+fLl9XKshTF37lxmzJhRoH3Pnj1YWFiQddMEZbY5yFIBSLdUY5upITEtF4BrEWEkXix8bkW2EV4XbTARRiQ7KTmbFgk7I0u01peN4/Yd2IeGkhwby82Phj2xv+m9JCr83/9hev8+GjMzzlaujGbnTv1xSyMjKgL3Dx7iTP36uB47hjVwPU/Jmcf6SUhISEi8WF5HVTOJl0ft2rX58ssviY2NNZCBl3j9OXjwIG3atPlPOWcAWVlZrFmz5rV0zuApHLRGjRpx48YNNmzYoFch6t27N/369cOyjIsH37x5k1GjRrF3796n2tRYFJMnT2bs2LH69/kRtMDAQGxsbNjydSSQgYVVDunZkG6pooJTDbgGVmYm9HyrfaFqltpsFSkrLqLRKJG7WeM7sBF+JiXSYXklSPhhFXmA+a1bBLVogXExT1Uz9x/g3vLliOxsTMqXp9KihXjVrGnQR9O8ObH/tx6z+/cJbNiQ28uWoQLqdHkLi6ZNnu9iJCQkJCSKJD09/WWbIPGaUZqiwRKvD506daJTp04v24wy55133nnZJjwTT+VWWlpaPlWRu9Jy9uxZkpKSqFevnr5No9Fw5MgRvv/+e3bv3k1eXh4PHz40iKLdu3dPL8daGGZmZnq53MeRy+WYmJiQcluntJSX8wDQRdBkRjq5UzcHC31q5+MItZb7Gy+hSVFibK/AMdgXY/OC/V5VVPfukfdIAh+NhrwzZ7EJCizQT2i13F+8mAfLlgNg0bAhFRctNChGnY/c0REzb2+Uly6hPHESVYKuppyFt1epJV4lJCQkJMoO6d9gCQkJiVeXEjlo27Zto2PHjsjlcrZt21Zs36LqYDwNbdu2LVB747333sPb25uJEydSuXJl5HI5+/fv1xeZvHLlCgkJCTRt2vSpzqnO06JRaxHaXJQZOqnbdAsV2Rqdg+buUDBKqMlSkbLxMnmxacjMjHEM8cXY6vVxzkBXZNrwfWihDlrq+g1658w+ZADO48cjK+aH3rJRQ5SXLvFw0ybQajGytcXkNaxHISEhISEhISEhIfEiKJGD1q1bN+7evYuzszPdunUrsp9MJivTQtXW1tbU/FfanKWlJQ4ODvr2wYMHM3bsWOzt7bGxsWHkyJE0bdqUJk2eLoUuJzNP9z8ynXOWa6pGJRfcV/4TQXucvDuZPPi/aDSpSmRyIxze9UFevmxTPV8EmY8cNIV/bXLPnSfz6LEChcmFEDz8dTMATmPG4Pjhk6OoFo0bk/LTOnLOnQN09c+eVOxcQkJCQkJCQkJC4n+VEjlojxcULKq44Mti4cKFGBkZ0aNHD5RKJUFBQSxduvSp51NmqQEwMUlHCTy01L2/lW4LaAwiaNkRSaT+cQ2h0mLsoEtrlLu8fs6Z0GjICjsOgPPo0dz8cCjqxETyYmIMCkorr1xBee06Mrmccn37lGhui/r1QSaDR2KhZtWqlf0CJCQkJCQkJCQkJP4jvHbSJocOHTJ4r1AoWLJkCUuWLCmT+fMjaEaPImjpVipkQnA71QTQUOVRBC1tbzwZ+xN0NniVw763F0YWr2dOf+6FC2jT0jCyscGiYUMsGjYk69gxMo8eNXDQ0rbpVDqt3nijWAGRxzG2tcXMxxtl9CUAg/kkJCQkJCQkJCQkJAwpkYP23XfflXjCjz/++KmNeRXIzVIBILSPJPYt1NhhxO1UnePm7mCJJjNP75xZt66MTXs3ZEavb9peZqguvdGyaVNkJiZYtmhB1rFjZB09hsMj1Sah0ZC+YwcANm91KdX8lo0a/+Og1ZAcNAkJCQkJCQkJCYmiKJGDtnDhwhJNJpPJXn8HLVPnoKnzUgCdxL4dchK0AoXcCGdrM3LO3QdAXtEK2yD3l2VqmZEvEGLVsoX+v0lffkn26dNoc3MxUijIPn0a9b17GNnYYBUQUKr5LRo1ImXtWkCKoElISEhISEhISEgUR4kctNjY2OdtxytDbqYKIQSqnGQA0izVuMt0Ndjc7C0xMpKhvKJz3hQ1yr00O8sKzcOH5DxSyrRsoXPQTKtVw8TFBfXdu2SfPoNVyxb69EabDh0wKqTMQHFYNGqIiasrJo6OGDs6lu0CJCQkJCQkJJ4rDx48wMfHh1OnTuHu7v6yzZH4D3Ho0CFat25NamqqQcmsZ2X58uXs2LGDv/76q8zmfJG8PlWUXxC5mSoQ2WjVSgAyLFQo0Al/VHGwQGgFudceAqDwtHtJVpYdWcePg1aLWY3qyB/VjpPJZPpoWtbRULS5uWTs3g2AbSnTGwGMrayotmsn7hvWSwqOEhISEhIvjYEDByKTyZDJZMjlcjw8PPjkk0/Izc0ts3McPnyYNm3aYG9vj4WFBTVq1CAkJIS8vDx9HyEEK1eupHHjxlhZWWFnZ0eDBg1YtGgR2dnZAEyfPh2ZTMbQoUMN5o+MjEQmkxEXFwdAXFwcMpkMZ2dnMjIyDPrWqVOH6dOnA6BSqZg4cSK1atXC0tKSChUqMGDAAO7cufPENX3xxRd07dpV75ytXbtWfx3//UpKSgIgMTGRfv364enpiZGREaNHjy7xNVy7di21a9dGoVDg7OzM8OHD9ceuXLlC69atKV++PAqFgqpVq/L555+jUqn0ffbu3Yunpyc2NjYEBwcbXPu0tDQ8PT2Jj48vsT0l5fTp01SoUAGAO3fuYG5ubnDuF01KSgojR47Ey8sLc3NzqlSpwscff0xaWtozzevn58eePXsACAwMZN26dWVhbonJzc1l4MCB1KpVCxMTk0IV5gcNGkR4eDihoaEv1Lay4qkctFu3brF06VImTZrE2LFjDV6vO7lZKoRGt/8MCy1aY5BpdA5aNScrVHcy0WapkJkZY1qlZEIZrzL58vqWzVsYtOe/zzx6jMyDB9FmZSGvUAHzx4qGlwYjhQJZKSNvEhISEhISZU2HDh1ITEzkxo0bLFy4kBUrVjBt2rQymTs6OpoOHTrQoEEDjhw5woULF1i8eDGmpqYGZYiCg4MZPXo0Xbt25eDBg0RGRjJlyhS2bt2qv/EFnRDa6tWruXbt2hPPnZGRwddff13k8ezsbMLDw5kyZQrh4eH88ccfXLly5Yn1a7Ozs1m9ejWDBw/Wt/Xu3ZvExESDV1BQEAEBATg7OwOgVCpxcnLi888/x9/f/4n25/PNN9/w2WefMWnSJKKioti3bx9BQUH643K5nAEDBrBnzx6uXLnCokWL+OGHH/R/Q61WS79+/Rg6dCjHjx/nzJkzrFy5Uj9+0qRJDB06FDc3txLbVFKOHz9O8+bNAQgNDaVBgwaYvsR7nzt37nDnzh2+/vprLl68yNq1a/n7778N/pal5eHDh1y9epUmTZqg0WgM1vyi0Gg0mJub8/HHH9OuXbtC+5iamtKvX79S6Wi8UohSsm/fPmFhYSFq1qwpTExMRJ06dYSdnZ2wtbUVrVu3Lu10rwRpaWkCEGlpaeLPheFi0cBF4utencS8Ye1FzbU1Rf+l7wq3idvFnxG3RNqBeHFz4hFx/6eol232M6PVasXVlq1EtJe3yDh61OCYOi1NRPv6iWgvb3Gjew8R7eUt7i345iVZKiEhISFRljz+u1cWaLVaoVGqX/hLq9WWys6QkBDRtWtXg7bu3buLunXr6t9rNBoxZ84c4e7uLhQKhahdu7b49ddf9cdTUlJEv379hKOjo1AoFKJ69erixx9/FEIIsXDhQuHu7l6sDZs2bRKA+PPPPwu9jg8fPhRCCDFt2jTh7+8v2rdvL3r27KnvExERIQARGxsrhBAiNjZWAGLChAnCyspK3Lt3T9/X399fTJs2rUhbTp06JQARHx9fZJ9ff/1VODk5FbumpKQkIZfLxbp16wo9HhAQIEaNGlXsHELorq25ubnYt2/fE/s+zpgxY0SLFi2EEELcu3dPACInJ0cIIcQnn3wiPvroIyGEEMeOHRP169cXarW6VPOXlN69e4uFCxcKIYQYMWKEmDhx4hPH/Prrr6JmzZpCoVAIe3t70bZtW5GZmak/vnr1auHr6ytMTU2Fi4uLGD58+DPZuHnzZmFqaipUKtVTjd+1a5fw9/cXQghx5swZUb58+SeOiYuLE507dxZ2dnbCwsJC+Pr6ih07dgghhDh48KAARGpq6lPZU9h3Op/Dhw8LU1NTkZ2d/VRzv0xKLbM/efJkxo8fz4wZM7C2tub333/H2dmZd999lw4dOpSl7/hSeDyClm2hS3lIzrIGwNvFhtwTNwBQeL7++8+U166hTkpCplBg0aCBwTFjGxvMa9cmJyKC3Kgo4OnSGyUkJCQk/vsIlZY7U8Ne+HkrzGyGzNT4qcdfvHiRsLAwg2jK3LlzWb9+PcuXL6dGjRocOXKE/v374+TkREBAAFOmTCE6Oppdu3bh6OjI9evXycnJAcDFxYXExESOHDlCq1atCj3nhg0b8PLyomvXrgWOyWQybG1tDdrmzZtHw4YNOXPmDA3+9Vv9OH379mXv3r3MnDmT77//vkTrT0tLQyaTFbv3JzQ0lPr16xc7z7p167CwsOCdd94p0XmLYu/evWi1Wm7fvo2Pjw8ZGRk0a9aMBQsWULly5ULHXL9+nb///pvu3bsD4OTkhKurK3v27KFdu3aEhoYSEhKCSqVi2LBh/PjjjxgbP/1n5t8cPXqUzp07A5CZmclff/3F9OnTycrKQi6Xs3z5ciZNmsSkSZMKjE1MTKRv377Mnz+ft99+m4yMDEJDQxGPascuW7aMsWPHMm/ePDp27EhaWhrHjh3Tjx84cCBxcXEFSlAVR1paGjY2NpiYlM4FqF27NgkJCeTl5aFSqbCzs0OlUqFUKrGzs6NKlSqcP3++0LHDhw8nLy+PI0eOYGlpSXR0NFZWVkWeSyaTsWbNGgY+UhJ/Who0aIBarebkyZO88cYbzzTXi6bUDtqlS5f45ZdfdINNTMjJycHKyoqZM2fStWtXhg0bVuZGvkhyM1V6if00c90+tIdKO+TGMtytzbifoMvv/i84aFmP5PUtGjbEyMyswHHLFs3JiYgAwMzXR1JglJCQkJB47dm+fTtWVlao1WqUSiVGRkZ6h0apVDJnzhz27dtH06ZNAahatSpHjx5lxYoVBAQEkJCQQN26dfXO0uOiGT179mT37t0EBATg4uJCkyZNaNu2LQMGDMDmUf3Qa9eu4eXlVWJ769WrR69evZg4cSL79+8vsp9MJmPevHl06dKFMWPGUK1atWLnzc3NZeLEifTt21dvW2HEx8fr91UVxerVq+nXrx/m5ubFL+YJ3LhxA61Wy5w5c/j222+xtbXl888/p3379pw/f94gXbBZs2aEh4ejVCoZMmQIM2fOBHTXYfPmzYwZM4ZRo0bx5ptvMmjQIObNm0fr1q1RKBQ0b96c5ORkRo4cyYgRI57J5gYNGhAZGcnly5fp168fZ8+eJSUlRW+fQqEo0gFOTExErVbTvXt3/UOCWrVq6Y/Pnj2bcePGMWrUKH1bw4YN9f/v6uqKVqstsa3JycnMmjWLIUOGlHKVsHPnTtRqNZ07d2b06NG0a9eOQYMG0aFDB3r16lWsw5eQkECPHj30a6tatWqx5/Ly8irwoOJpsLCwwNbW9rnsN3zelNpBs7S01G94dHV1JSYmBj8/P0D3h3/d0UXQHgJw31K34TRd7Ux1Z2u0semgFZg4mmNir3iJVhaP0GhIXrqMnIhwbDp1wqZjR4wsLP45LgQ5Z8+S9ucW4B95/X9j1bIlyYt1P1q2XYrPUZeQkJCQ+N9FJjeiwsxmL+W8paV169YsW7aMrKwsFi5ciImJCT169AB00Zjs7Gzat29vMCYvL4+6desCMGzYMHr06EF4eDiBgYF069aNZs10azc2NmbNmjXMnj2bAwcOcPLkSebMmcOXX37JqVOncHV11UdHSsPs2bPx8fFhz549+j1ehREUFESLFi2YMmUKP//8c5H9VCoVvXr1QgjBsmXLij13Tk4OCkXR9zzHjx/n0qVL/N///d+TF/IEtFotKpWK7777jsDAQAB++eUXXFxcOHjwoMFetE2bNpGRkcG5c+eYMGECX3/9NZ988gkALVq04PTp0/q+V69eZd26dURERNCqVStGjRpFx44dqVmzJq1ataJ27doFbOnYsaNeYMLNzY2oR9lE/0ahUODu7s7mzZvp2LEjHh4ehIWF0bJlS7y9vYtdr7+/P23btqVWrVoEBQURGBjIO++8Q7ly5UhKSuLOnTu0bdu2yPFz584tdv7HSU9Pp1OnTvj6+uqFY0pDpUqVuHv3LjExMfTp0we5XM7p06f56aefioxu5vPxxx8zbNgwfVSzR48ehV7zfC5fvlxq+4rC3NxcL7zzOlFqB61JkyYcPXoUHx8f3nzzTcaNG8eFCxf4448/aNKkyfOw8YWhztOgUmoQWp26TaKVbkNvtrocPi7W5F7TRdZe5eiZOjWVO+PGkxWmSzXJCjvOvTlzsXmrC7ZdupBz/jwPN/9K3g1dqqbM3ByrNm0KnUvh54fcrQqaBynYdHrzha1BQkJCQuL1QiaTPVOq4YvE0tKS6o8yQn788Uf8/f31IhiZmZkA7Nixg4oVKxqMM3uUadKxY0fi4+PZuXMne/fupW3btgwfPtxAoKNixYoEBwcTHBzMrFmz8PT0ZPny5cyYMQNPT89S34BWq1aNDz74gEmTJrF69epi+86bN4+mTZsyYcKEQo/nO2fx8fEcOHCg2OgZgKOjI6mpqUUeX7VqFXXq1HliGmRJcHV1BcDX11ff5uTkhKOjIwkJCQZ9850CX19fNBoNQ4YMYdy4cYWmL3744YcsWLAArVZLREQEPXv2xMLCgoCAAA4fPlyos7Bq1Sp96qpcLi/S5vxUvfxo7NatW8nLy0MIgZWVFS1btmTXrl2FjjU2Nmbv3r2EhYWxZ88eFi9ezGeffcbJkydxLMOyRBkZGXTo0AFra2u2bNlS7HoKY+jQoaxfvx6tVotSqcTFxQUhBNnZ2fj4+AA6gZwqVaoUOv79998nKCiIHTt2sGfPHubOncuCBQsYOXLkM6/tSaSkpODk5PTcz1PWlPrR0zfffEPjxo0BmDFjBm3btmXTpk24u7s/8R+NV53cLBUIJaAGIMVC95RLaKzxdrEi96ruHyizV9RBy718mbievcgKC0Nmbo59yADkVaqgzcri4S8bie/3LknzviTvxg1k5ubYvtMD919+xrRSpULnkxkb475xI1V3bEdezBM7CQkJCQmJ1xEjIyM+/fRTPv/8c3JycvD19cXMzIyEhASqV69u8Ho8SuDk5ERISAjr169n0aJFBiqB/6ZcuXK4urqSlZUFQL9+/bh69Spbt24t0FcIUaQE+tSpU7l69SobN24sdk2NGjWie/fuhe55ynfOrl27xr59+3BwcCh2LoC6desSHR1d6LHMzEw2b978TKqAj5OvBnjlyhV9W0pKCsnJycWqLuZH3gpL91u9ejX29va89dZbeiXNfEl+lUploK75OBUrVtT/7Ys7d2RkJGfOnMHY2Jj9+/cTGRmJg4MDmzdvJjIyklWrVhW7ZplMRvPmzZkxYwYRERGYmpqyZcsWrK2tcXd3LzattSSkp6cTGBiIqakp27ZtKzYaWhQzZ84kMjJSn94YGRnJwIED6d+/P5GRkURGRj4xDbZy5coMHTqUP/74g3HjxvHDDz887ZJKTExMDLm5ufro9+tEqSNoj+eNWlpasnz58jI16GWSm6VCCN0/oHIzORpjgVwLaM2oZaFAk6oEYxlmVZ89L7asSdu+g8TPP0fk5iKvXJlK3y9G4eWF88SJZJ86xcPNm8k4cBDTqh6U69ULm86dMS5mg2Y+JuVeTWdUQkJCQkKiLOjZsycTJkxgyZIljB8/nvHjxzNmzBi0Wi0tWrTQCzPY2NgQEhLC1KlTqV+/Pn5+fiiVSrZv366PIqxYsYLIyEjefvttqlWrRm5uLuvWrSMqKorFixcD0KtXL7Zs2ULfvn35/PPPCQwMxMnJiQsXLrBw4UJGjhxZaF2n8uXLM3bsWL766qsnrumLL77Az8/PYF+QSqXinXfeITw8nO3bt6PRaLh79y4A9vb2RcrBBwUFMXnyZFJTUyn3r3uCTZs2oVar6d+/f6FjIyMjAZ0jd//+fSIjIzE1NdVHyLZs2cLkyZP1EUVPT0+6du3KqFGjWLlyJTY2NkyePBlvb29at24N6ERW5HI5tWrVwszMjDNnzjB58mR69+5dIDKUlJTE7Nmz9cIa5cqVw8fHh0WLFhEYGMj+/fv57LPPnng9i6N69eqcOHGC8uXL06JFCxISEsjIyKBLly5PFOI4efIk+/fvJzAwEGdnZ06ePMn9+/f1n6fp06czdOhQnJ2d6dixIxkZGRw7dkwfeZo8eTK3b98usg5ZvnOWnZ3N+vXrSU9PJz09HdA9ZCipWIqzszPOzs6cP3+eYcOGUb16dS5dusQHH3ygj0YXx+jRo+nYsSOenp6kpqZy8OBB/RoLw9vbm7lz5/L2228X2Sc6Opq8vDxSUlLIyMjQf9bq1Kmj7xMaGkrVqlWfuB/zlaS0so+DBw8WBw8eLEslyZdOvtxw1OlY8e2gNeLrXp3Esg97iJpra4oWP9QWbhO3izv748TNiUdE0g/nX7a5BUg/eFBEe3mLaC9vET/4faF+SqlSCQkJCYn/DcpaZv91oShJ7rlz5wonJyeRmZkptFqtWLRokfDy8hJyuVw4OTmJoKAgcfjwYSGEELNmzRI+Pj7C3Nxc2Nvbi65du4obN24IIYQIDw8X/fv3Fx4eHsLMzEw4ODiIVq1aiW3bthmcT6PRiGXLlomGDRsKCwsLYWNjI+rXry++/fZbvSR4vsz+46SlpQlHR8dCZfYjIiIM+g4ZMkQAepn9/H6FvZ50X9eoUSOxfPnyAu1NmzYV/fr1K3JcYedyc3PTH1+zZo34961oWlqaGDRokLCzsxP29vbi7bffFgkJCfrjGzduFPXq1RNWVlbC0tJS+Pr6ijlz5uhl9R+nT58+YvHixQZtJ0+eFN7e3sLe3l7MmDGj2HWXlLlz54r+/fsLIYRYt26daNeuXYnGRUdHi6CgIOHk5CTMzMyEp6dnAXuXL1+u/yy6urqKkSNH6o+FhISIgICAIufPl7Av7JX/+RFCCDc3t2LLMQghRGJiol6yXqlUCnNzc3Hr1q0SrXPEiBGiWrVqwszMTDg5OYng4GCRnJxsYOPjMvuAWLNmTbFzurm5FbquxwkMDBRz584tkY2vGjIhSrdbtWvXruzevRsnJyf69OlD//79S1WA8FUkPT0dW1tbwg9eI/Snv1Fl/421ixmL613FJdeMlKS5/F3RFeXVVGzf9MC6VeEpgS8DbVYWMV26oL6TiF3PnrhMn4asDOVjJSQkJCT+e+T/7uVLbktIFMeOHTuYMGECFy9exMio9MIsEq8u2dnZODg4sGvXrtdOir44oqKiaNOmDVevXi0TRcgXTam/ZVu3biUxMZEpU6Zw+vRp6tWrh5+fH3PmzCEuLu45mPjieDzFUZjq8pNN1ApqlrciL1aXE/6qCYTc/24x/9/encdFVa9/AP/MDDMw7PsugiKiooi45FJqLqhJbuFakVm2WL/S22a3sm6ldu8tW665lXm9am5l5VZuKLkjiuICgoAogiD7Ogwz398fNCdGQEF2+7xfL14vOOfMOc85IM7D8/0+34ob6VB6eMDl7flMzoiIiKhRPfLII5g9ezbS0tJaOhRqZBEREXj44Yfvq+QMqFzCYO3atW0yOQOAelfQbnf9+nV8//33WL16NRISElBRUdFYsTUbw18SD2w+i7Pbt0CnOQNVRy1Wdr4Bz3x3PGb3CUIuFEJurYLb/L6QyWQtHTIAoPT8BaRMngzo9Wi3aiUsH3ywpUMiIqI2gBU0IqLWq0F1aq1Wi1OnTuHEiRNISUmBi4tLY8XVIjRFFRD6ygpaqUlla1VdhRW66isfk5mvbatJzkRFBTLeew/Q62E9ZgyTMyIiIiKi+8A9JWgRERF49tln4eLigqeeegrW1tbYsWMHrl+/3tjxNavSYi2EvnINlEJlGQBAU2ELl8rRjlC6WrRUaNXkrFuHsosXIbe2hsvb81s6HCIiIiIiagT1brPv4eGBnJwcjBo1CitXrkRoaKi0eGNbV16iBf6Yg5b3R4JWqnOAulALHQATJ3ULRvcn7Y0byPqysl2v82t/g0kjLmZIREREREQtp94J2vvvv4+wsDDY2to2QTgtq7SoXKqg3VLrAMhhofaE7lZlsqZ0Nm/B6CoJIZDx4UcQJSVQ9+oF28cea+mQiIiIiIiokdR7iOOzzz57XyZnAFBaWASgckX5m38UywJtvIAKPaCQQWFX/9XX60tXUICi33+HqGVl+8I9e1EUEQEolXD74H3I2O6WiIiIiOi+wXf3VZQWVbbSV5mZIVtV2Qwk2MwZAGDiqIZM3vQNQtJeew3Xnp2Nmx9/XG2frrAQNz/6CADg8MwsmHbq1OTxEBERERFR82GCVkVFWSEAQG2ugv6Pbo1dFJXth5tjeGPJmTMojvwdAJC74XvkrF9vtD9ryeeoyMqCsr0XHJ97rsnjISIiIiKi5sUErao/Wuwr/xjeqNbJ4KqrfETN0SDk1lf/qby+pycA4ObCRSg6fAQAUBoTg9zvvwcAuL3/PuRmTT/ckoiIiKi8vBy+vr44evRoS4dCbYy3tzc+//zzRj3nrVu34Ozs3Oa7x98JE7QqxB8dHHUm5QAAtU4FVX7l501dQSs5fRrFR48CJibwWvMdbMaPB3Q6pM2di7L4y0h/bwEgBGzGjYNF//5NGgsREdH96KmnnoJMJoNMJoNSqYSPjw/eeOMNlJWVNdo1Dh06hIcffhj29vYwNzdHp06dEB4ejvLycukYIQRWrlyJfv36wdLSEra2tujduzc+//xzlJSUAKhsyiaTyfD8888bnT8mJgYymQwpKSkAgJSUFMhkMjg7O6OwsNDo2J49e+L999+Xvv7xxx8xcuRIODg4QCaTISYmpk73tHz5cvj4+GDAgAHSNWfNmgUfHx+o1Wp07NgRCxYsMLrHgwcPYty4cXBzc4OFhQV69uyJ9beNDKrNmjVr0KNHD5iZmcHZ2Rlz5syR9qWkpOChhx6ChYUFHnroIek5GIwdOxY//PBDna5TH3q9HtbW1rh8+TIAwM/PD5GRkY1+nfqIjIxEaGgo3N3dIZPJ8NNPPzX4nP/6178wffp0AMCGDRvw8MMPN/ic9bVy5UoMGTIE1tbWkMlkyMvLM9rv6OiIJ598EgsWLGj22JoLE7QqhL7yl6JGXvmL2gJqVGRVLlht4tS0Cdqt/1RWz2wnTIDK0xOu//gA6uBg6AsLkTJlCjSXL0Nhawvnt95s0jiIiIjuZ6NGjUJ6ejqSkpKwZMkSrFixotHe6F28eBGjRo1C7969ERkZidjYWHz11VdQqVTQVWn+9cQTT+DVV1/FuHHjEBERgZiYGLz77rv4+eefsWfPHuk4MzMzfPvtt0hISLjrtQsLC/Hvf//7jscUFxdj0KBB+OSTT+p8T0II/Oc//8GsWbOkbXFxcdDr9VixYgUuXLiAJUuWYPny5Xj77belY44ePYoePXrghx9+wLlz5zBz5kw8+eST2LFjxx2v99lnn+Hvf/873nrrLVy4cAH79u1DSEiItP9vf/sbPDw8EBMTAzc3N7z22mvSvk2bNkEul2PSpEl1vr+6On/+PMzMzODn54ebN2/i6tWr6NOnT6Nfpz6Ki4sRGBiIpUuXNto5jx07hoEDBwIAfv/9d+nz5lRSUoJRo0YZ/TzdbubMmVi/fj1ycnKaMbJmJFqxr7/+WnTv3l1YWVkJKysr8cADD4hdu3ZJ+0tLS8WLL74o7O3thYWFhZg4caLIyMio93Xy8/MFALFw8gvi35MfEZ/NHyoC1gSIp1ZNFNfejBTX3owUurKKxrw1I8WnTomLnf3FxW4BQnPturRdm50tEh4eVrmvs7/I/XFbk8VARER/HYb/9/Lz8xvlfHq9Xmg0mmb/0Ov19YozPDxcjBs3zmjbxIkTRVBQkPS1TqcTCxcuFN7e3sLMzEz06NFDbNmyRdqfk5Mjpk+fLhwdHYWZmZnw9fUVq1evFkIIsWTJEuHt7X3HGDZt2iQAiJ9++qnG55iXlyeEEGLBggUiMDBQjBgxQoSFhUnHnDlzRgAQycnJQgghkpOTBQDx+uuvC0tLS3Hz5k3p2MDAQLFgwYJq1zG85syZM3eMVQghoqKihFwuFwUFBXc87p///Kfw8fG54zFjxowRM2fOrHV/Tk6OUKvVYt++fbUe06VLF7F7924hhBC7du0SXbt2FUIIkZubK3x9fUVqauodY7hXy5Ytk352tm7dKvr163fX18TExIghQ4YIS0tLYWVlJXr16iWioqKk/YcPHxaDBw8WarVa2NraipEjR4qcnJx7ig+A2LZt2z29tipXV1fp5yIgIEB61rXR6/ViwYIFol27dkKlUgk3Nzfx8ssvS/vbt28vlixZck+xRERECAAiNze3xv0+Pj7im2++uadzt3b1XgetOXl6emLx4sXo1KkThBD473//i3HjxuHMmTPo1q0b5s6di507d2LLli2wsbHBSy+9hIkTJ+LIkSP3dD2hLwHkQKmycqhjR1kHAIDCxhRyU0Wj3dftsgzVs4kTofL0kLab2Nuj3fJlSH16FtQ9A2EzflyTxUBERHSvtFotFi5c2OzXffvtt6FSqe759efPn8fRo0fRvn17aduiRYuwbt06LF++HJ06dUJkZCQef/xxODk5YfDgwXj33Xdx8eJF7N69G46OjkhMTERpaeVoG1dXV6SnpyMyMhIPPfRQjddcv349OnfujHHjqv+fLpPJYGNjY7Rt8eLF6NOnD06dOoXevXvXei/Tpk3D3r178Y9//AP/+eN9RWP4/fff4efnBysrqzsel5+fD3t7+7se06VLl1r37927F3q9HmlpaejSpQsKCwsxYMAAfPrpp2jXrh0AIDAwEPv27cPIkSOxZ88e9OjRAwDw+uuvY86cOdJxjcWwtFRZWRmEELC1tYVGo4FOp4OtrS0GDRpUa1VwxowZCAoKwrJly6BQKBATEwOlUgmgcqjqsGHD8PTTT+OLL76AiYkJIiIipErrmjVrMHPmTAghGvV+arJ48WIsXrwYQOX3aPDgwZDJZMjPz8fkyZMhl8uxY8cODBo0qNprf/jhByxZsgQbN25Et27dkJGRgbNnz9Z6raeeegopKSk4ePBgg+Pu27cvfv/9d6Pq7v2iVSdooaGhRl9//PHHWLZsGY4fPw5PT098++23RuNjv/vuO3Tp0gXHjx/HAw88UP8L/jEHrVRVOcSxo8wbAGDi3HQNQkpOnULJseOAUgnH52ZX22/aqRN8Iw9BJmv6Fv9ERET3ux07dsDS0hIVFRXQaDSQy+VSQqPRaLBw4ULs27cP/f+Y792hQwccPnwYK1aswODBg5GamoqgoCApWfL29pbOHRYWht9++w2DBw+Gq6srHnjgAQwbNgxPPvkkrK0ru0InJCSgc+fOdY63V69emDx5Mt58803s37+/1uNkMhkWL16M0NBQzJ07Fx07dqzvo6nR1atX4e7ufsdjEhMT8dVXX91xiOXmzZsRFRWFFStW1HpMUlIS9Ho9Fi5ciC+++AI2NjZ45513MGLECJw7dw4qlQr//ve/8dxzz8Hb2xs9evTAihUrEBkZiZiYGHzyySeYPHkyTp06hZEjR+LLL79sUAIPVCZSQggEBwdjw4YN8Pf3x8iRI/H+++9jwIABMLtD07bU1FS8/vrr8Pf3BwB0qrI80j//+U/07t0bX3/9tbStW7du0uc2Njb1+jlpiOeffx5Tp07FmjVrcPz4cSxfvhy7du3CmjVrsHnzZgCVf3yoSWpqKlxdXTF8+HAolUp4eXmhb9++tV7Lzc0Ner2+UeJ2d3fHmTNnGuVcrU2rTtCq0ul02LJlC4qLi9G/f39ER0dDq9Vi+PDh0jH+/v7w8vLCsWPH7pigaTQaaDQa6euCggIAgNAXAwo5itTlAEzgoav8YZQ7mEGr1TbJfWV+9RUAwHr8eMDZucmuQ0REZNDY/9colco7zhdpKoZqRH0MHToUy5YtQ3FxMZYsWQITExNpzlJiYiJKSkowYsQIo9eUl5cjKCgIAPDCCy9g0qRJOH36NEaOHInx48dLzTMUCgW+++47fPTRRzhw4ABOnDiBhQsX4pNPPsHJkyfh5uZ2TxWRjz76CF26dMGePXvg7Oxc63EhISEYNGgQ3n33XWzYsKHe16lJaWnpHZOQtLQ0jBo1CmFhYXj22WdrPCYiIgIzZ87EqlWrjJKQ2+n1emi1Wnz55ZcYOXIkAOD777+Hq6srIiIiEBISAg8PD6OKlUajQUhICP773//io48+gpWVFeLj4zFq1CisWLECL7/8crXrLFy40Kjie/HiRXh5edUYk7e3N06ePAlzc3OMGjUK169fx40bNzBp0iSYmprWei8AMG/ePDzzzDP43//+h+HDhyMsLExKnGNiYhAWFlbraydMmIAJEybc8fyNxdbWFra2tjh58iQmTZoEb29vnDlzBo8++qjRHyBqEhYWhs8//xwdOnTAqFGjMGbMGISGhsLEpOYUY9GiRY0Wt1qtlprq3G9afYIWGxuL/v37o6ysDJaWlti2bRu6du2KmJgYqFQqqfRs4OLigoyMjDuec9GiRfjggw9q2KMDIMcts8rM3iSrsodK3M0ruLXrYiPcjTGLixfhceIkhEKBmI4dUbFrV6Nfg4iI6HaN/aZGJpM1uFLRXCwsLODr6wsAWL16NQIDA/Htt99i1qxZKCoqAgDs3LkTHh4eRq8zvBkfPXo0rl69il27dmHv3r0YNmwY5syZY1Q98vDwwBNPPIEnnngCH374Ifz8/LB8+XJ88MEH8PPzQ1xcXL1i7tixI5599lm89dZb+Pbbb+947OLFi9G/f3+8/vrr9bpGbRwdHREbG1vjvhs3bmDo0KEYMGAAVq5cWeMxhw4dQmhoKJYsWYInn3zyjtdyc3MDAHTt2lXa5uTkBEdHR6Smptb4moULF2LkyJEIDg7Gs88+i48++ghKpRITJ07EgQMHakzQnn/+eUyePFn6urYK4ejRo/H777+joqICFRUVsLS0hE6ng0ajgYODAwBIPzM1ef/99zF9+nTs3LkTu3fvxoIFC7Bx40ZMmDABanXTL99UF7///jtGjx4NoPL3wsGDBzF37lyUlpZCqVRi8eLFePvtt2v9A0y7du0QHx+Pffv2Ye/evXjxxRfxr3/9C4cOHbqnP6DUR05ODpycnJr0Gi2l1SdonTt3RkxMDPLz87F161aEh4fj0KFDDTrn/PnzMW/ePOnrgoICacyyiUqNbFXlcEJ3mRMAgZ5DekPVwaamU9VICAHNhQso/PkXKJydYPfMM9WGKOpLSpC65HNUALB78kl0mjG9QfdERERUV4aRI391crkcb7/9NubNm4fp06eja9euMDU1RWpqKgYPHlzr65ycnBAeHo7w8HA8+OCDeP3112sd3mdnZwc3NzcUF1dOo5g+fTqmTp2Kn3/+udo8NCEECgoKqs1DA4D33nsPHTt2xMaNG+94T3379sXEiRPx1ltv3e3268Qwh0oIYfReJi0tDUOHDkVwcDC+++47yOXVG4MfPHgQY8eOxSeffILZs6tP47idoWNgfHw8PP9YEzYnJwe3bt0ymidocOnSJWzYsEFaLkCn00nVYa1Wa9Q5syp7e/u7zpcDgG+++QalpaUIDw/HxIkTMW7cOLz22mvw9/fHM888c9fXA5Xt+P38/DB37lxMmzYN3333HSZMmIAePXpg//79tRQMmk/v3r0RExOD6OhovPHGG9i/fz9SU1Px6KOP4vTp05DL5Xd9Vmq1GqGhoQgNDcWcOXPg7++P2NhY9OrVq0ljP3/+PIYMGdKk12gprT5BU6lU0l+6goODERUVhS+++AJTpkxBeXk58vLyjKpoN2/erHWcrIGpqWmtZWml2hw5CgWUehMoCiqHIZi5WUNRh78C6AoLUbBjB3I3b4Hm0iVpu0JhUm1+2c2vl6EiIwNKT0+4/N/LkDfxXxmIiIgMmvov221JWFgYXn/9dSxduhSvvfYaXnvtNcydOxd6vR6DBg1Cfn4+jhw5Amtra4SHh+O9995DcHAwunXrBo1Ggx07dkiNL1asWIGYmBhMmDABHTt2RFlZGdauXYsLFy7gqz+mNEyePBnbtm3DtGnT8M4772DkyJFwcnJCbGwslixZgpdffhnjx4+vFqeLiwvmzZuHf/3rX3e9p48//hjdunWrNswsJycHqampuHHjBoDKRAionF9U23unoUOHoqioCBcuXEBAQACAyuRsyJAhaN++Pf79738jKytLOt5wnoiICIwdOxavvPIKJk2aJI1uUqlU0hv+bdu2Yf78+VJF0c/PD+PGjcMrr7yClStXwtraGvPnz4e/vz+GDh1qFJcQArNnz8aSJUtgYWEBoDLBW7VqFfz8/LB27VpMmzbtrs/qTjw8PFBRUYFz585h3bp18PHxwblz5/Dmm29K701rU1paitdffx2PPfYYfHx8cP36dURFRUnDaefPn4/u3bvjxRdfxPPPPw+VSoWIiAiEhYXB0dGx2rOpSVFRERITE6Wvk5OTERMTA3t7+1qHbN5OrVbD19cXW7duxZAhQ6QFyQcOHAg/P7+7vn7NmjXQ6XTo168fzM3NsW7dOqjV6hoTasN9p6WlYe3atbWeMyMjAxkZGdK9xcbGwsrKCl5eXtLPTklJCaKjo1ukOVGzaLH+kfdo6NChIjw8XOTl5QmlUim2bt0q7YuLixMAxLFjx+p1TkO74Y8mjBQrX3pGBKwJEI8sHyGuvRkpri84Uqc2vvm7fxWXgnpJLfEvde8hUp56Svo6/7ffpGNLYs+Li126ioud/UVhZGS9YiUiImqoxm6z31bU1GZfCCEWLVoknJycRFFRkdDr9eLzzz8XnTt3FkqlUjg5OYmQkBBx6NAhIYQQH374oejSpYtQq9XC3t5ejBs3TiQlJQkhhDh9+rR4/PHHhY+PjzA1NRUODg7ioYceEr/88ovR9XQ6nVi2bJno06ePMDc3F9bW1iI4OFh88cUXoqSkRAjxZ5v9qvLz84Wjo2ONbfZvb5k/e/ZsAcCozf53330nAFT7qKkVf1WTJ08Wb7311l3PU/VtZXh4eI37Bw8eXO08t9/j008/LWxtbYW9vb2YMGFCja3zly9fLiZNmmS07ebNm2LYsGHCyspKhIWFieLi4jveV10cO3ZMeHp6CiGEuHbtmjA3Nxfl5eV3fZ1GoxFTp06V2s+7u7uLl156SZSWlkrHHDx4UAwYMECYmpoKW1tbERISIrWUr+nZ3M7Qhv72j/DwcOmYBQsWiPbt29813pCQEKll/dNPPy0++uiju75GCCG2bdsm+vXrJ6ytrYWFhYV44IEHjJZJuL3Nfnh4uNHPQE0WLFhQ431999130jEbNmwQnTt3rlOMbZFMiGbo33mP5s+fj9GjR8PLywuFhYXYsGEDPvnkE/z2228YMWIEXnjhBanLjLW1tTTO+OjRo/W6jmE4wUcTRsLFxwtf9DiOB/OD8PaNZ6FqZwXnOT3v+PrSs2dx9YknIcrLofLtCLvJk2Hz6KNQ2Noi48OPkLt+PWRqNbzXr4Opnx9SJk9B2cWLsB4zBh6ffXqvj4eIiOieGP7fy8/Pl7oLEtXm3LlzGDFiBK5cuQJLS8uWDofqITw8HDKZDGvWrGnpUBrVAw88gP/7v//D9On35xShVj3EMTMzE08++STS09NhY2ODHj16SMkZACxZskRaMd7Qxadqu9J7IVTlAICO5ZUTVU2c7jyJU3vjBq7NeQmivByWw4bB86svIasyDttl/lsoT0lB8ZEjuPbCi7B59FGUXbwIubU1XN6e36BYiYiIiJpajx498MknnyA5ORndu3dv6XCojoQQOHjwIA4fPtzSoTSqW7duYeLEiQ0ewtqateoKWnOpWkGzD7DC1x3j8d71WehfGAzrUd6wHlLzoof64mKkzHgcmrg4mPr7w3v9Osj/GAddla6gAClTp6E8KUna5vqPD2BXpYMQERE1raysLFy7ds1om0KhgL+//13bZTeFa9euwcbGpkUqWKygERG1Xq26gtYSKkwqWw97lFdOclXWUkETej3S3ngTmrg4KBwc0O7rpTUmZwCgsLZGu+XLkBI2Gbr8fKh79YLtY481zQ0QEZERIQSio6Oxa9euGhdI9ff3x9SpU5s1ptTUVKxevRpubm547rnnmvXaRETUujFBu02ZshgQgIu2cl0FE2fzGo/LWvI5ivbvh0ylQrul/4GyljU0DFReXmj3zSrkrlsPx5dfMhoGSURETUOr1WLXrl04c+YMgMqubObmf/5ev3LlCuLi4nDp0iWpE19zMLQFT09PR2FhIaysrJrt2kRE1LoxQbtNmbIEjhV2MNWrALkMJvZm1Y4pv34d2d98AwBw+/hjqHv2rNO51d27Q/3J4sYMl4iIapGfn4/NmzcjLS0NMpkMw4YNw8CBA43Wctq3bx8OHz6MXbt2wcfHB2Zm1X/nNzatVosLFy5IX1+9elVqX05ERMQyzm2KVCVop3EBAJg4mEGmqP6I8jZuBISAxYD+sAkd29whEhHRXaSkpGDlypVIS0uDmZkZZsyYgUGDBhklZwAwePBg2NnZobCwEAcOHGiW2BISEqDRaKSvk5OTm+W6RETUNjBBq0KhNEexSQXa/TH/zMSp+vBGvUaDvK0/AADsZsxo1viIiNqa5u5DJYTAiRMnsHbtWhQXF8PFxQWzZ8+udVFZpVKJsWMr/9B28uRJpKWlNXmM586dAwA4OzsDqEwmiYiIDJigVaFSW6NQoYdneWUFTelcvUFIwa7d0OXlwcTdDZZDhjRzhEREbUdkZCQWLlxYrXNiVREREVi8eDFu3LjR4OtptVr89NNP2L17N/R6PQICAjBr1izY29vf8XUdO3ZEjx49AADbt2+HTqdrcCy1KSkpQUJCAgDgkUceAQBkZ2ejsLCw0a6xefNmfPXVVygtLW20cxIRUfNhglaFqdoSeXLFn0Mca6ig5W7YAACwmzIVMoWiWeMjImor0tPTERERAa1Wi6ioqBqP0el0OHHiBMrKyrBv3756nT8zMxOJiYnSx+XLl7F69WqcPXsWMpkMISEhmDRpElQqVZ3OFxISArVajYyMDBw/frxesdTHxYsXodPp4OLigvbt28PVtXLERmNV0QoKCnDx4kVkZ2cjLi6uUc5JRETNi01CqlCpzZGrkMNTGuJoXEErPXcOZbGxkCmVsA1jm3wiopro9Xps375dGt54+fJl6HQ6KG77o9bVq1dRVlYGAEhKSkJqaiq8vLzuev7jx4/j119/rXGfubk5wsLC4OPjU6+YLSwsMGLECPzyyy84ePAggoODm6RhiGF4o6Fi5+3tjYyMDKSkpDTKAsCJiYnS53FxcQgKCmrwOYmIqHmxglaFytQERVDBscIWAKp1cMxdX1k9sxo9CiZ3GTJDRG1LWVkZTp06haKiopYOpVVKSkpCUlJSnY49efIkbty4AVNTU6jVapSVleHq1avVjjNUeOR/LDsSERFx13NfvnwZv/32GwDAyckJrq6u0oe/vz9mz55d7+TMICgoCA4ODtBqtdIwxMaUm5uL1NRUAJCSMUOsjVVBu3LlitHn5eXljXJeIiJqPqygVaUScNDZVn5uIoPcQintqsjNRcHu3QAA++nTWyA4ImoqOp0OGzduREpKCq5fv47x48e3dEitSlFREdatWwchBJ555hl4eHjUemx+fr7UDXH48OG4ceMGzpw5g7i4OHTo0EE6TgghJWijRo3Cr7/+iuTkZFy9ehXt27ev8dyZmZnYunUrhBDo1asXQkNDq3VlbAiZTAZ/f38cOXIEcXFxjVLRqio2NhZAZVJmbW0NAFLFMDs7GwUFBdL2e6HX66UETaFQoKKiAleuXGnW9d2IiKjhWEGrQpiUwUlbWRkzsTUz+o8/b+tWiPJymHXrBrPAwJYKkYgamRACO3fulCoYCQkJ0Ov1LRtUDcrLy7Fz584mqezczZUrV6DX6yGEuGsTjd27d6O8vByenp4IDg6Gv78/gMpqWdWOjjdu3EBBQQGUSiWCgoKkoXgHDx6s8bzFxcXYsGEDysvL0b59e4wZM6ZRkzMDQzKTkJCAioqKRjuvEKLa8EYAUKvVcHNzA9DwKlpaWhrKyspgZmaG3r17AwDnoRERtUFM0KooVxbAscIOAKCw+XNiudDpkPf9RgCA3fTpTfKmgIhaxokTJ3D69GkAlVWH4uJi3Lx5s4Wjqu7kyZOIiorCzp07m711fdV5TRkZGThx4kSNx126dAlxcXGQy+UIDQ2FXC5Hhw4doFQqUVBQgPT0dOlYQ+LQqVMnKJVKPPjgg5DL5UhOTq6WqFRUVGDTpk3Iy8uDnZ0dpkyZAhOTphkA4u7uDktLS5SXlzfq+mTp6em4desWTExMqlW0vL29ATQ8QTN8nzp06ICuXbsCAOLj45u0KyURETU+JmhVaBS5cNIaEjRTaXvBr79Ce+MGFDY2sH5kTEuFR0SNLCEhQZrPNHLkSHTs2BGAcULSHMrKyvDNN99g7969Ne7X6/VSJ8S8vDxkZ2c32rW1Wi02btyItWvX1lgxqjpsLvCP0QMRERHIy8szOi4rKwu7du0CAAwYMAAuLn8sV6JUSmuQXbp0STrekKAZKmy2trbo1asXgD+raBUVFbhw4QL++9//IjU1Faamppg+fTrMzat32G0scrncqOrXGHQ6HQ4fPgwA6Ny5c7XmI42VoBm+Tx07dkS7du1gbm5e6/w/IiJqvZigVVFikg9nQ4JmW5mglV2+jIx33wMA2E6bCnkTdPUiouaXmZmJLVu2QAiBoKAg9O/fX0okqjZaaA6JiYm4fv06jhw5goyMjGr7L1++jPz8fOnrxorPMGQxLi4OSUlJNQ6fzMjIQElJCVQqFUJDQ+Hl5QWtVmtUybt48SJWrVqFwsJCODg4YPDgwUbnMFSMDAlPdnY2srKyIJfL0alTJ+k4QxUtJSUF27Ztw2effYYtW7bg2rVrUCgUCAsLg5OTU6Pc+51UTdAaOty1qKgIa9euxcWLFwFAGnpYlZeXF2QyGXJyclBQUHBP1ykpKZEW2fb19YVcLkfnzp0BcJgjEVFbwwStimJR9OcQR1tTVGRn4/rzL0BfUgLzvn3h9OKLLRwhETUGnU6HTZs2SfOZHnnkEchkMilBS01NhUajabZ4DG+sgZrnYBmqZxYWFgAar8J3+PBhaV4UAKPPDQzX8vHxgYmJiTR0MSEhARcuXMD+/fuxefNmlJeXw9vbGzNnzoRSqTQ6R6dOnSCXy5GVlWW0Ppe3tzfU6j+XM7GxsZGqaGfPnkVJSQksLS3x0EMP4eWXX5a+P03N29sbpqamKC4uxvXr1+/5PNevX8eKFStw9epVqFQqTJ06tcYOk2q1usHroSUlJUEIAScnJ9jY2ABArfP/iIiodWMXxypKRKk0xFFuocD1l16G9sYNKNt7weOLzyGr44KnRNS6nTt3DtnZ2TA3N8fkyZOl+Uz29vawt7dHTk4OkpOTpTe4d5KWlgaFQiG9wb4XVRO0uLg4pKenS40jbt26JVXMQkNDpW6TWq22WiJUHxcvXsT+/fsBAH369EFUVBQuX76M0tJSo6TJkKAZkiMnJyc8+OCDOHToEH744Qfpjf8DDzyAESNGVFvrDKhMQLy9vZGUlIS4uDhpqGNNz3fw4MHIyMiAWq1GcHAwOnXqVOM5m5KJiQn8/PwQGxuLuLi4u67NlpeXh+TkZKNqW1FRESIjI6HT6eDg4ICpU6fesfrn4+OD9PR0pKSkGDURqelaOTk58PHxMZoPffv3CUC1+X/u7u53vXciImp5TNCqKEK5lKDlrl2J0jNnILeyQrtly2BiZ9fC0RFRY9DpdDh06BAAYNCgQVJVyqBjx47IyclBYmLiXRO0/Px8rF69Gnq9HiNGjED//v3r3URIp9NJzTPc3d1x48YNHDx4ENOmTQMAnDp1CkBlFapz586wtLREUVERUlNTpTlz9XXjxg1s27YNANC3b1+MHj0aKSkpyMrKwqVLl6QqVllZmVRBqvrGf9CgQTh//jyys7NhYmKCRx999I5JBVCZjCUlJeH06dPSHLqanq+VlRWeeeaZe7qvxuTv7y8laCNGjKj2fdXpdLh8+TKio6PvWNHs3LkzJkyYcNdFr729vXH06NE7NiYpKyvD6tWrUVBQgNGjR6Nfv34AKoeqVp1/ZmCY/3fp0iVcunSJCRoRURvBIY5VaGECK33lm7WCnT8ACgU8Pl8C0ypr9xBR23b27Fnk5eXBwsKixvlAhkQkMTHxrsPCzp8/D51OByEE9uzZgx9++KHawsC3bt3CoUOHam2Pf+vWLWi1WqhUKowfPx4ymQzx8fFIT09HeXk5zpw5A6Aykao6DPNe56EVFRXh+++/h1arRceOHRESEgKZTCYlWFWHORqqQg4ODrCr8kcqpVKJKVOmoHfv3pg1a9ZdkzPgz2TMkJx5eHg0aM2vpubr6wuFQoGcnBxkZWVJ23U6HSIjI7FkyRJs2rRJSs48PT3h5+dn9DFq1ChMmTLlrskZ8Oc8tNzcXKP5hlVFRERIc9R+/fVX6WcqMzMThYWFMDExqbaG3O3z/4iIqPVjBa0Kub7yDYheWwJUlMHlnXdgOXBgC0dFRI3F8OYaAAYOHAhVDcOWvb29IZfLpaFkDg4OtZ7PkMx06tQJV65cwfnz55GZmYmwsDBkZGQgOjpamlOkVCrx5ptvVmsPbxje6O7uDmdnZwQEBCA2NhYHDx5Ep06doNFoYGdnJ1VGfH19ERMTg8TERIwcObLez+DEiRMoLCyEo6MjwsLCpOGD3bt3x/79+5GSkoK8vDzY2tpKyUdNlTpnZ2eMHTu2zte1traGh4eHdL91GT7akkxNTdGhQwckJCQgLi4Ozs7OKC4uxtatW6Uql4WFBXr27IlevXrd8eekLszMzODh4YHr169j9+7dmDx5MuTyP/+GmpaWJi1v4OXlhdTUVGzduhWzZs2Svk/e3t53nf/X0DiJiKjpsYJWhUJULlItSnIBALZhj7VkOETUyGJiYu5YPQMq35gbqhB3Grp28+ZN3Lx5EwqFAhMnTkR4eDgsLCyQmZmJpUuX4ocffkBKSgpkMhkUCgW0Wi1SU1OrnceQsHh4eAConINlqKIZhmL26dNHerPe4Y+KfmZm5j11/DPM/3rooYeMKju2trbSfZ8/fx5CiBrnNTVE1aSstSdowJ8xXrp0CTdu3MDKlSuRnJwMpVKJcePGYe7cuRgxYkSjJT0hISFQKBSIi4vDgQMHpO06nQ7bt28HULnI9ZNPPgkvLy9oNBp8//33UofImr5Phvl/AKtoRERtBRO0Ksz0fyRopTkwcXGB3NT0Lq8gotro9XqUlZUZfWi12ia71t1UVFRI1bNBgwbVWD0zqMt6aFWrZ2q1Gu3bt8dzzz0nJVrW1tYYPHgwXnnlFQQEBNR6vqoVNABwdHRE9+7dAUAathYUFCQdb25uLl2jvt0cb926hVu3bkEul8PPz6/afsNQxbNnzyI7Oxv5+flQKBTSG/yG6tatG5RKJTw8PJqlXX5DGdrUp6enY/Xq1cjPz4e9vT2effZZBAUFNfpi2e3atcOjjz4KoLLD5tmzZwFUVj0NjVNCQkJgYmKCKVOmwNbWFrm5udLPUG1zEg2J5uXLlxs1XiIiahoc4liF3R8t9vWluVB6erZwNERtV3l5OZYvX46cnByj7XK5HI8//rhUBWoMSUlJ2LhxIwIDA/HII4/UetzZs2eRn58PS0vLWqtnBr6+vti3bx9SUlJQUVFR7Y24Xq9HbGwsABjNv7K2tsbTTz+NzMxMODs7S8MHfX19cfbs2WrDErVaLTIzMwH8WUEDKqtbsbGxEEKge/fuRl0VDedLS0vDlStXpIYedWGooPj4+NQ4L6pr167YtWsXsrKycOTIEQBA+/bt75jM1oe9vT1efvnlRjtfU7O0tJSGE1ZUVMDPzw8TJkyo9v1oTIGBgcjKysLhw4fxyy+/QCaTISIiAgAwYsQIqamNhYUFpk+fjm+//RYajQY2NjZwdHSs8Zxdu3aFpaXlPTeVISKi5sUKWhWOWlsAlRU0FRM0ug9kZWU1WdXqTq5evVotOQMqE5vff/+90a5z69YtaQ2uqKioWhtn3F49u1t7ehcXF1haWtY6LPHq1asoKCiAmZmZ0ULLAKBQKODm5mbUGt7wxvj2YYkZGRnQ6/WwsLCQ1q4CKqtoAwYMgKWlJQbWMA+2aqOQ+iykbEjQahteqFarpcqaoTlJY7+pt7a2rlPTjNaib9++MDU1xeDBgzF16tQmTc4MHn74Yfj7+0On0+HHH3+EVqtF+/btjSqpQOU8wLCwMJibm0tNZGpiaWmJrl27wpSjQoiI2gQmaFU4VNgCAPSlOVC2a9eywRA1UFJSEpYuXYodO3Y0+7UNjTECAwPxzjvv4J133sHLL78MmUyG5ORko65496qkpAQbNmxAWVmZlHDt2LGjxoQ0MjJSqp4FBwff9dxVuyXWNIzQMLyxa9eudVqLrOqwxKpJZNX5Z7e/uR4xYgRee+21Gqsi7u7uMDMzQ1lZmdEaandSWFgotcw3DN2rye0dGZtrcejWKiAgAG+99RaGDh1q1LSjKcnlckycOFFaW0+hUCA0NLTGBMzX1xevv/56jYk8ERG1TUzQqnD8I0ETpblQtWMFjdo2QwvuCxcuVGv93tQMCZqPjw9MTExgYmICBwcHqToTFRXVoPPrdDps3rwZOTk5sLGxwQsvvAArKyvk5uZKlTKDc+fOSdtGjhxZ58Wdq85Dq9puX6vVSk0Z6tJevqbzGdzeIKSuFAqFNEy0rvPQ4uPjpWvdqb19p06dpAqXlZUVnJ2d6xXb/ai+a9s1BpVKhWnTpqFz584YO3ZsrcMXgZaJj4iImg4TtCoctIY5aKygUdt348YNAJXD++51zax7UVZWJl379uYSffr0AVDZTVGj0dzT+YUQ2LlzJ1JSUqBSqTB9+nTY29tjzJgxAIAjR47g5s2bAIBr167h559/BlDZVr++CZVMJkNmZiZ++uknqTJ3+fJlaDQaWFtbw8vLq87nq2lY4u0NQuqjvuuhGbo3GtbFqo2JiQm6desmXYNv/luOjY0Npk2bVm1oIxER3d9adYK2aNEi9OnTR/or7vjx46W/AhuUlZVhzpw5cHBwgKWlJSZNmiS9OasvM1E5cV2wSQi1MomJidi+fTvKysrqdLxOp5OSJKB522tfu3YNQgjY2dnB1tbWaF+HDh1gb2+P8vJyowWR6+P48eM4ffo0AGDSpElwcXEBUJl4dO7cGXq9Hjt27EBeXh42btwInU6Hzp07Y9iwYfW6jrm5OUaPHg2ZTIazZ89i9erVyMvLk+Lu0aNHvYa8eXh4wNTUVEpgS0tLpXl69a2gAX9W5NLS0rB27VrpY/369dWqamVlZdLaXXVpbz98+HAMHTq03s+MiIiIGq5VJ2iHDh3CnDlzcPz4cezduxdarRYjR45EcXGxdMzcuXOxfft2bNmyBYcOHcKNGzcwceLEe76mvqwAMqUCJm2gBTT9dfz666+Ijo6uNnyvNrdu3TKaixUfHw+dTtdU4RkxDG+sqTW7XC5H3759AQAnT540GjpYF5cvX8aePXsAVA5XvH0u1ZgxY6BSqXDt2jWsWLECxcXFcHFxwcSJE+9p/lDfvn3xxBNPQK1WIz09HStXrpSGjtanGgdUH5ZoSKDt7Oxgbm5e79hsbGzg7u4OIQSSkpKkj4SEBHz//fe4du2adGxCQgL0ej0cHR3vOFTOQK1WY/DgwbC0tKx3XERERNQwrTpB+/XXX/HUU0+hW7duCAwMxJo1a5Camoro6GgAQH5+Pr799lt89tlnePjhhxEcHIzvvvsOR48exfHjx+/pmqI0B0pPTw7roXqrqKjAunXrsHXr1nonHndSWFiIW7duAahMaoqKiu76GsPQOS8vL5ibm6OsrAxXr15tlHg0Go1Uqampg6ChUlPb2lmBgYFQKpXIysoyiqmiogK7d+/Gl19+WWPFLzMzU3q2QUFB6N+/f7VjbGxs8PDDDwMASktLYWFhgWnTpjWoe12HDh3w3HPPwdXVFSUlJdDr9XB1db2nuVlVG4/c6/yzqqZOnYqJEycaffj6+kKn02Hjxo3Iy8sDcPfujURERNR6tKl10PLz8wFUrqUDANHR0dBqtRg+fLh0jL+/P7y8vHDs2DE88MADNZ5Ho9EYzX+p2vZaX5oLEw+PFmlNTm1bdHS0NLRs4MCBdapU1EVSUpL0eUVFBX7//Xejn/maGKon7u7usLOzw9mzZ3Hx4kW0q8PcSq1WC71eX2NSo9frsWXLFimmuLg4ozbzGo0G6enpAABPT88a/x2ZmJggICAAZ86cwYkTJ+Dh4YHCwkL88MMPUtKyceNGPPjgg3jwwQchk8lQXFyMDRs2oLy8HF5eXggJCUFFRUWN8QcFBeHChQu4efMmHnvsMVhYWDT437OFhQWefPJJ7N69G7GxsQgODr6ncxqS1rS0NKmi5+rqes/xqdXqanPKOnbsiP/+97/IzMzEhg0bMGPGDKnq5+vry99tBAD8OSAiasVkojH/1N+E9Ho9Hn30UeTl5eHw4cMAgA0bNmDmzJnVmg307dsXQ4cOxSeffFLjud5//3188MEH1bZffHU3TK8fR4bVNWSNe7Txb4LuW3q9HhcvXpTe9Hh4eDRa97vU1FRkZ2dDrVajtLQUMpkM3bp1u2M3wri4OJSWlsLb2xtyuRxJSUlQKpXo1q3bHavDRUVFSE5Ohl6vh5eXF+zs7Iz2p6WlSQsrA5Vd/qq2Yc/Pz0dSUhJUKpXUaKImpaWlUlXH29sb169fR0VFBRQKBaytrZGbmwsAUiOO5ORkFBcXQ6VSoXPnztUWjr6dEAJ6vd5oLbLGotPpGnTeS5cuGc0l7NSpU6MPJSwvL0d8fDwqKipgamoKjUZTp+8//XWUlJRg+vTpyM/Pv2NXTyIian5tpoI2Z84cnD9/XkrOGmL+/PmYN2+e9HVBQYFUWdCX5qLT6IHo80dHOKK6OHXqFM6ePSt9bWpqKnUVbKjly5cDAB555BEcPXoUN27cgIWFRa1VtIqKCimWsWPHwtzcHEuWLIFWq0WvXr3g5uZW7TVCCOkeDMMWU1JS4ObmhiFDhkAulyMmJkZavHjo0KGIiIhAYWEhHnjgAamqvX//fiQlJaFLly53vf+1a9fi2rVr0pw1JycnPPbYY7C3t8e5c+ewe/duFBQUIC4uTko0nnrqqUarTLYUpVKJkydPAqhsjz5+/HioVKpGv05aWhr+97//SX/A6tGjB0aNGtXo16G2qerIESIial3aRIL20ksvYceOHYiMjIRnle6Krq6uKC8vR15enlG3uJs3b0oLfNbE1NS01jkpojQHZt7edV4riaiiogJHjx4FUNlGPioqCqmpqQDQ4J+jwsJCZGdnA6gcumZmZoZ169YhOjoagwYNgpWVVbXXZGRkQK/Xw8LCAg4ODtKiy5cuXUJiYmK11vBarRY7d+6UkrqAgABYWVnh2LFjOHbsGDIzM9G7d2/s3r0bADBkyBAMHjwY165dQ2JiIs6cOSO98Tfcd8eOHe9673379pWGYnbr1g2PPvqo9O8yODgYbm5u2LRpE/Lz8yGTyRAWFlZjctnW+Pn5SQmas7MzLCwsmuQ63t7eGDduHH788UcAdV9Um/4a+LNARNR6teoETQiBl19+Gdu2bcPBgwfh4+NjtD84OBhKpRL79+/HpEmTAFR2q0tNTa2xgUBd6EtzoPRgi/3mptVqcfToUbi6ulbrzNfanT59GoWFhbCyssLIkSMRHx+PgoICXL161Wj4370wVJdcXV2hVqvRsWNHeHp64vr16zhy5EiNFZGqzScMw9n8/f1x6dIlxMXFSU00ACAvLw+bNm1Ceno6ZDIZRowYgf79+0Mmk8Hd3R0///wzrly5Iq211a1bNwwePBhAZYKVmJiImJgYPPzww9Dr9dL8s/bt29/13rp164acnBxYWVkhKCio2tA7d3d3zJ49G5GRkfD29m7ws2wt2rdvDxMTE1RUVDSoQUhd9OjRAzqdDrm5udV+fxIREVHr1KoTtDlz5mDDhg34+eefYWVlhYyMDACVndrUajVsbGwwa9YszJs3D/b29rC2tsbLL7+M/v3719og5G5EaS5Unk37pomMVU0STE1N8cYbbzTJ3KGmoNVq8fvvvwMAHnzwQSiVSnTs2BFnzpxBYmJioyVohuYSMpkMQ4YMwbp163Dq1CkMHDiwWhWtpsWP/fz8IJfLkZmZiezsbDg4OCA5ORlbtmxBSUkJ1Go1wsLCpDbwANC9e3c4OTlJ3QDd3d0xfvx4KZHy9fWFnZ0dcnNzERsbCysrKwghYG9vDxsbm7vem1wul5K92lhYWGD06NF3PVdbYvgZiY+Pr1Mi21Bc5JiIiKhtadUJ2rJlywBUDqmq6rvvvsNTTz0FAFiyZAnkcjkmTZoEjUaDkJAQfP311/d0PSH0UKhNIG+iIUdUXdUkAajsAnjt2rVaW7TfTVFREbZv3w5PT08MHDjwnta+qoler8euXbuQn5+PwMBA+Pv7w8TERKqeWVtbo1evXgAqE5czZ85IVaeGMCRoVasfVatohw8frpbAGNbXqlqdUavV8Pb2RlJSEuLi4iCXy7Fnzx4IIeDq6oopU6ZUawgCVFbunnvuOVy+fBl+fn5Gw6Lkcjl69+6NvXv3IioqSorxXr93fyWhoaHo3r07unbt2tKhEBERUSvTqhO0ujSYNDMzw9KlS7F06dKGX68sH6p27nc/kBpMCIFjx45h7969UpJgbm6OpKQkXLly5Z7f5B89ehTx8fHSUNeJEydCrVY3ON6YmBicOnUKQOWiv+bm5ujZsydiY2MBVFbPDJ0FO3ToAJlMhqysLOTn59epmlSTgoICaf5Z1XljMpkMQ4cOxf/+9z+pimbowlZaWiq95vbhc/7+/khKSkJERITUor5Hjx4IDQ2943wUtVqNwMDAGvcFBQUhIiICGRkZ0ppbTNDuztLSEgEBAS0dBhEREbVCrXqh6uYmSnOhrMM6UdRwERERUgUnMDAQs2bNkpIAw1pi9aXVaqUugzKZDAkJCVi1ahVu3rzZoFiLioqwZ88eAJVDBa2srFBSUoKjR49K1bOqw8jUarXUzOZe7wX4s3rm5uZWLcns0KED2rVrB51OhyNHjkjbDdUzOzs7mJubG73GMLevoqICMpkMo0ePxoQJExrULMDc3FxKNAyt45mgEREREd07JmhV6MvyoGzHBiFNLSMjQ5q3FRISgvHjx0vzcgAgPT0dRUVF9T7v+fPnUVpaChsbGzzzzDOwsbFBTk4OvvnmG1y4cOGe4/3tt99QVlYmDQV89dVXMW3aNPj5+cHU1BSjRo2qti6X4V4aI0GrKeExzEUDKlv8G1pmV20QcjsbGxsEBATAxsYG4eHh6NevX6OsidW3b1/pc8NcUCIiIiK6N0zQqqhsEMIKWlPS6/XYvn07hBDo2rWr1DEQqBz2ZVgeob7zt4QQUuvyPn36wMPDA7Nnz4aPjw+0Wi22bNkiVZfq48qVK4iNjYVMJkNoaCgUCgUUCgU6d+6M6dOnY/78+TXOIzI0B0lKSoJOp6v3dYE7J2hAZRXNy8sLOp1OWh/QcI9VG4RU9dhjj+HVV19t1CqXu7u7VDFkp0AiIiKihmGCVoW+NA9KT1bQmlJUVBTS0tKkytPtDIlNfRO0tLQ0pKenQ6FQSMMNLSws8Pjjj0tD+wwJXF2Vl5djx44dACqrRPVpie7u7g61Wg2NRiNVtYDKBPXs2bNISkq64+sLCgqQk5MDmUxWa6e/qlW06OhoFBQU3LGCVvV1jW306NHw8/O75+UtiIiIiKgSE7SqynKh4hDHJlNQUID9+/cDAIYNG1bjUDhDgpaYmAi9Xl/ncxuSr4CAAKOFfxUKBQYNGgQAiI2NlbpF1kVkZCRyc3NhbW1ttHZYXcjlcqllvWGYY1lZGTZt2oRt27Zh7dq12LdvX633WHX+mZmZWa3X8fHxkapou3btQmFhIWQyWbMv6Ozh4YHp06fD0dGxWa9LREREdL9hglZFhSYfJi4uLR3GfWvXrl0oLy+Hp6cnevfuXeMxnp6eUKlUKCkpkda9M9BoNNi5cyeioqKMOnwWFRVJc8yqzoeqek43NzfodDqcPn36rnFqNBqcOnUKR48eBQCMGTMGpqamdb5Pg6rVwKysLKxatQrx8fFS6//Dhw9j/fr1NSaNdxveaGDo6AgAcXFxAABnZ2eoVKp6x0tERERELY8JWhXFphWQtZEFktuauLg4af2t0NDQWtcnMzExkeYx3d5g48CBA4iKisLOnTuxdetWaDQaAMCZM2eg0+ng7u5e49A+mUyGPn36AKhsqFFb1erGjRvYvn07Pv30U+zYsQN6vR5dunSBv7//Pd2zoVFIWloaVq1ahezsbFhbW2PWrFmYOHEiTExMcOXKFaxatapaMpqcnAygbh0Rvb29jYZB1mcoJhERERG1Lq16HbTmprGzbOkQ7juFhYWIiYnBsWPHAAD9+/eHy12qlL6+voiPj8eVK1fw0EMPAahMck6cOAGgcvjghQsXkJWVhcmTJ0vrk9VUPTMICAjAnj17kJeXh4SEBGleGlDZdn7Lli2Ij4+Xttnb2yM4OPiO57wba2truLi44ObNmygvL0f79u0RFhYGS0tLeHh4wNnZGRs3bkRubi5WrFhh1AlSq9VCJpMZrX9WG8NctP/+978AmKARERERtWVM0KqQuXH+WWMQQuDKlSuIjo5GfHy8VLFycnLC4MGD7/p6Q+Xp2rVrKCsrg1KpxPbt2wEA3bt3R58+fbB582ZkZmZi2bJl0Ol0UKvV6NatW63nVKlUCAoKwrFjxxAVFSUlaEII7Nq1Sxp62LVrVwQHB8Pb27tRmmkEBgZiz5496NevH0aOHAlFlQqtq6srZs+ejR9//BGJiYnQarVGr+3UqdMd559V5ePjgy5duuDKlSvS8yMiIiKitocJWhVmHt4tHcJ94eTJk9i9e7f0taenJ4KDg9GtW7c6zY2yt7eHvb09cnJykJycjNzcXGRkZECtViMkJASWlpaYPXs2Nm/ejOvXrwMAevXqddcFl/v06YNjx44hMTER2dnZcHBwwPHjx3H69GnIZDJMnToVfn5+Dbv52wwYMADBwcG1zmEzNzfH448/joKCAqN2/DKZrN7riYWFhUGv11dbk42IiIiI2g6+k6vCsg7DydqK/Px85OTktMi6VNHR0QAqq12DBg2665DGmvj6+uLkyZM4ffq01DBjxIgRsLSsHIZqbW2Np556Cvv370dGRgYeeOCBu57T3t4enTp1QkJCAqKiotChQwfs2bMHADBy5MhGT84M6tJgpDEWd5bL5bXO7SMiIiKitoEJWhWW3vdHgqbX67Fu3TpkZWXh8ccfl7oJNofs7GxkZmZCLpdjzJgxUKvV93Sejh074uTJk0hISAAAtG/fXlrfzMDExAQhISH1Om+fPn2QkJCAM2fO4PTp0xBCoFevXnVK8IiIiIiImhr/3F6FbYf7I0FLSkpCVlYWAOD48ePNem1Dq3dvb+97Ts4MrzfM11IoFBg7dmyjzAnz9fWFnZ0dNBqN1LhjzJgxTbJ4MxERERFRfTFBq8LM1qalQ2gUUVFR0ueJiYnIyclptmsbErR7bU1vYGpqKg3PHDRoEJycnBocG1A5DNDQct/Ozg5TpkzhnC0iIiIiajX4zvQ+k5eXh8uXLwOoXLA4MzMTUVFR9R4KWJuUlBTs27cPDz/8MDp06GC0r7CwENeuXQMAozb29+rRRx9Famoqunbt2uBzVdWvXz+Ym5ujY8eOMDc3b9RzExERERE1BCtorYwQotaFlOvi1KlTEELAx8cHw4cPB1C5kHN5eXmdXl+1k+Dtbt26hY0bN+L69evYsWNHtWMNiaG7uztsbBpejbS2tkZAQECjN75QKBTo2bMnrKysGvW8REREREQNxQStlSgrK8PJkyexfPlyfPzxx0hMTKz3ObRaLU6fPg2gctFmX19f2NraoqysDOfPn7/r65OTk7Fw4UL88MMP1RK6kpISbNiwAWVlZQCAnJycaue8dOkSAKBLly71jp2IiIiIiJigtbj09HT89NNP+Pe//41du3bh5s2b0Ol0+OWXX6DRaOp1rosXL6KkpATW1tbw8/Mzmm918uRJCCHu+Pro6GjodDrExsbim2++keau6XQ6bNmyBTk5ObCxsUH//v0BAIcOHZKqaGVlZUhOTgbQ8PlnRERERER/VUzQWtCNGzewatUqxMTEoKKiAk5OTggJCYGtrS0KCgoQERFRr/OdPHkSANC7d2+pA2JQUBBMTEyQkZEhLepck4qKCqmlvUqlQmZmJlauXImEhATs2rULycnJUKlUmDZtGoYMGQK1Wo2cnBzExsYCqGxGotPp4ODgAEdHx3t5HEREREREf3lM0FqIXq/H9u3bodfr0b59ezz99NN48cUX0b9/f4wdOxYAcOLECdy4caNO50tLS0NaWhoUCgV69eolbTc3N0dAQACAPxO4mqSkpECj0cDS0hJz5syBh4cHysrKsH79emnh6UmTJsHV1RWmpqYYOHAgACAyMhI6nc6oeyNb1hMRERER3RsmaC3k5MmTSE9Ph5mZGR577DF4eXlJiY2vry8CAgIghMD27dvv2LjDwNBav2vXrrC0tDTa17dvXwDAhQsXUFRUVOPrDfPHOnfuDBsbG8ycORPBwcHS/hEjRhh1ZuzTpw/Mzc2Rk5ODM2fOSA1COLyRiIiIiOjeMUGrwZEjR7Bt2zYkJiY2qKNibfLz83HgwAEAwPDhw2vsJjhq1CiYmZkhPT39jpUvoLKBh6FhhyEZq8rd3R0eHh7Q6/VSNawqvV6P+Ph4AH8mWCYmJggNDcWUKVMwfvx4DBgwwOg1Vatov/76K8rLy2FpaQkPD4+73T4REREREdWCCdptTp48ib179+Ls2bNYt24dvvzyS0RGRqKwsLBRzi+EwK5du1BeXo527doZDUesytLSEiNGjAAAHDhwAHl5ebWe88yZM6ioqICrqys8PT1rPMaQuJ04caJah8a0tDQUFRVBpVJJi0MbdOnSBT179qxx2KKhilZRUQGgMrlr7Jb4RERERER/JXw3XUVSUhJ2794NAOjYsSNMTU2Rl5eHAwcO4LPPPpNa2DfEpUuXEB8fD7lcjtDQ0DsmNEFBQfDy8oJWq8Wvv/5a4zF6vV4a3ti3b99a538FBATAzs4OJSUl0vEGhvljfn5+MDGp+9rlKpVKqqIBHN5IRERERNRQTNCq2LZtG4QQCAwMxOOPP46//e1vGD9+PDw9PSGEwO7du5Gbm1vr6/Py8u44JLKsrExKAAcOHAhnZ+c7xiOXyzF27FjIZDLExcXV2IUxISEBeXl5MDMzk5qB1EShUOChhx4CUDmE01BFE0JI88/uJcHq06cPHB0d4eDgAG9v73q/noiIiIiI/sQErQrDsMPQ0FDIZDKoVCr07NkTs2bNQvv27aHVarFr164a1xP77bff8Pnnn9c4JDIvLw8RERH4+uuvUVhYCHt7eylZuhtnZ2f06NEDQOW6Y7czVMOCgoKgUqnueK4ePXpIVTTDvLZbt24hJycHCoUCvr6+dYqpKpVKhRdeeAFz5sypV/WNiIiIiIiqa/UJWmRkJEJDQ+Hu7g6ZTIaffvrJaL8QAu+99x7c3NygVqsxfPhwaT2v+rKxscGUKVOqJRoymQxjx46FXC5HQkICLl68aLT/1KlTOHbsGAAYDYncuHEj1q9fjy+++AKHDh1CQUEBzM3NMX78eCiVyjrH9dBDD0EmkyEhIcGoipadnY3ExEQAkBakvhOFQoHBgwcDAI4ePQqNRiNVz3x8fGBmZlbnmG4/L+eeERERERE1XKt/V11cXIzAwEAsXbq0xv3//Oc/8eWXX2L58uU4ceIELCwsEBISgrKysnpf67HHHqvWot7AyckJDz74IABg9+7dKC0tBQAkJydj165dACoTqfHjx6Ndu3YQQiAuLg4JCQkQQsDb2xuTJk3CvHnz4OXlVa+4HBwcEBgYCAA4ePCgtN1QPfP19YW9vX2dztW9e3fY29tLc9Gqrl9GREREREQtq9WPSRs9ejRGjx5d4z4hBD7//HO88847GDduHABg7dq1cHFxwU8//YSpU6fW61p3mxM2aNAgnD9/HtnZ2di/fz/69++PTZs2Qa/XIyAgAEOHDoVMJkPPnj2RmZmJs2fPQi6XIzAwEI6OjvWK5XYPPfQQzp49i8TERFy7dg0uLi6IiYkBUHNr/doY5qL99NNP+P3336HRaADAaI0zIiIiIiJqGa0+QbuT5ORkZGRkYPjw4dI2Gxsb9OvXD8eOHas1QdNoNFJiAgAFBQUAAK1WC61We8drjh49GuvWrcOpU6dw+fJllJWVwd3dHWPGjJHazQOAnZ0dhgwZIn19t/PejZWVFXr06IGzZ88iIiIC/v7+KCsrg62tLby9vet1/i5duiAyMhI5OTkAAE9PT5iZmTU4RiIiahv4+56IqPVq0wlaRkYGAMDFxcVou4uLi7SvJosWLcIHH3xQbfuePXtgbm5+1+va29sjJycHBQUFUCqVsLe3x969e+sZff0Z/kNNSkpCamoqAMDc3FzqDFkfVlZWUoKm1+ulYZpERHT/KykpaekQiIioFm06QbtX8+fPx7x586SvCwoK0K5dO4wcORLW1tZ3fX1JSQlWrVqF8vJyPPHEE3B1dW3KcI0olUqcPXsWFRUVMDExwdSpU6FWq+t9Hr1ej9WrVyMnJwcTJkyAra1t4wdLREStkmHkCBERtT5tOkEzJEY3b96Em5ubtP3mzZvo2bNnra8zNTWFqalpte1KpbJO3RVtbGzw4osvQq/X19pUpKkMGTIEsbGx0Ov16N69e50Syto8/fTT0Gq1zX4PRETUsurTSZiIiJpXq+/ieCc+Pj5wdXXF/v37pW0FBQU4ceIE+vfv36TXNjc3b5HExs7ODg8++CCsra0xcODABp3L1NSUyRkRERERUSvS6itoRUVF0lpfQGVjkJiYGNjb28PLywuvvvoqPvroI3Tq1Ak+Pj5499134e7ujvHjx7dc0E1s6NChGDp0aEuHQUREREREjazVJ2inTp0ySkYMc8fCw8OxZs0avPHGGyguLsbs2bORl5eHQYMG4ddff73nRZeJiIiIiIhaikwIIVo6iJZWUFAAGxsb5OfnN2hOFxERUVvA//eIiFqvNj0HjYiIiIiI6H7CBI2IiIiIiKiVYIJGRERERETUSjBBIyIiIiIiaiWYoBEREREREbUSTNCIiIiIiIhaCSZoRERERERErQQTNCIiIiIiolaCCRoREREREVErwQSNiIiIiIiolWCCRkRERERE1EowQSMiIiIiImolmKARERERERG1EkzQiIiIiIiIWgkmaERERERERK0EEzQiIiIiIqJWggkaERERERFRK8EEjYiIiIiIqJVggkZERERERNRKMEEjIiIiIiJqJZigERERERERtRJM0IiIiIiIiFoJJmhEREREREStBBM0IiIiIiKiVoIJGhERERERUSvBBI2IiIiIiKiVuG8StKVLl8Lb2xtmZmbo168fTp482dIhERERERER1ct9kaBt2rQJ8+bNw4IFC3D69GkEBgYiJCQEmZmZLR0aERERERFRnd0XCdpnn32GZ599FjNnzkTXrl2xfPlymJubY/Xq1S0dGhERERERUZ2ZtHQADVVeXo7o6GjMnz9f2iaXyzF8+HAcO3asxtdoNBpoNBrp6/z8fABATk4OtFpt0wZMRETUwgoLCwEAQogWjoSIiG7X5hO0W7duQafTwcXFxWi7i4sL4uLianzNokWL8MEHH1Tb7uPj0yQxEhERtUbZ2dmwsbFp6TCIiKiKNp+g3Yv58+dj3rx50td5eXlo3749UlNT+R9VEykoKEC7du1w7do1WFtbt3Q49yU+46bHZ9w8+JybXn5+Pry8vGBvb9/SoRAR0W3afILm6OgIhUKBmzdvGm2/efMmXF1da3yNqakpTE1Nq223sbHhm4EmZm1tzWfcxPiMmx6fcfPgc256cvl9MRWdiOi+0uZ/M6tUKgQHB2P//v3SNr1ej/3796N///4tGBkREREREVH9tPkKGgDMmzcP4eHh6N27N/r27YvPP/8cxcXFmDlzZkuHRkREREREVGf3RYI2ZcoUZGVl4b333kNGRgZ69uyJX3/9tVrjkNqYmppiwYIFNQ57pMbBZ9z0+IybHp9x8+Bzbnp8xkRErZdMsMcuERERERFRq9Dm56ARERERERHdL5igERERERERtRJM0IiIiIiIiFoJJmhEREREREStxF8+QVu6dCm8vb1hZmaGfv364eTJky0dUpu1aNEi9OnTB1ZWVnB2dsb48eMRHx9vdExZWRnmzJkDBwcHWFpaYtKkSdUWGae6W7x4MWQyGV599VVpG59x40hLS8Pjjz8OBwcHqNVqdO/eHadOnZL2CyHw3nvvwc3NDWq1GsOHD0dCQkILRty26HQ6vPvuu/Dx8YFarUbHjh3x4YcfomrfKj7j+omMjERoaCjc3d0hk8nw008/Ge2vy/PMycnBjBkzYG1tDVtbW8yaNQtFRUXNeBdERPSXTtA2bdqEefPmYcGCBTh9+jQCAwMREhKCzMzMlg6tTTp06BDmzJmD48ePY+/evdBqtRg5ciSKi4ulY+bOnYvt27djy5YtOHToEG7cuIGJEye2YNRtV1RUFFasWIEePXoYbeczbrjc3FwMHDgQSqUSu3fvxsWLF/Hpp5/Czs5OOuaf//wnvvzySyxfvhwnTpyAhYUFQkJCUFZW1oKRtx2ffPIJli1bhv/85z+4dOkSPvnkE/zzn//EV199JR3DZ1w/xcXFCAwMxNKlS2vcX5fnOWPGDFy4cAF79+7Fjh07EBkZidmzZzfXLRAREQCIv7C+ffuKOXPmSF/rdDrh7u4uFi1a1IJR3T8yMzMFAHHo0CEhhBB5eXlCqVSKLVu2SMdcunRJABDHjh1rqTDbpMLCQtGpUyexd+9eMXjwYPHKK68IIfiMG8ubb74pBg0aVOt+vV4vXF1dxb/+9S9pW15enjA1NRXff/99c4TY5j3yyCPi6aefNto2ceJEMWPGDCEEn3FDARDbtm2Tvq7L87x48aIAIKKioqRjdu/eLWQymUhLS2u22ImI/ur+shW08vJyREdHY/jw4dI2uVyO4cOH49ixYy0Y2f0jPz8fAGBvbw8AiI6OhlarNXrm/v7+8PLy4jOvpzlz5uCRRx4xepYAn3Fj+eWXX9C7d2+EhYXB2dkZQUFBWLVqlbQ/OTkZGRkZRs/ZxsYG/fr143OuowEDBmD//v24fPkyAODs2bM4fPgwRo8eDYDPuLHV5XkeO3YMtra26N27t3TM8OHDIZfLceLEiWaPmYjor8qkpQNoKbdu3YJOp4OLi4vRdhcXF8TFxbVQVPcPvV6PV199FQMHDkRAQAAAICMjAyqVCra2tkbHuri4ICMjowWibJs2btyI06dPIyoqqto+PuPGkZSUhGXLlmHevHl4++23ERUVhf/7v/+DSqVCeHi49Cxr+v3B51w3b731FgoKCuDv7w+FQgGdToePP/4YM2bMAAA+40ZWl+eZkZEBZ2dno/0mJiawt7fnMyciakZ/2QSNmtacOXNw/vx5HD58uKVDua9cu3YNr7zyCvbu3QszM7OWDue+pdfr0bt3byxcuBAAEBQUhPPnz2P58uUIDw9v4ejuD5s3b8b69euxYcMGdOvWDTExMXj11Vfh7u7OZ0xERH9pf9khjo6OjlAoFNW62928eROurq4tFNX94aWXXsKOHTsQEREBT09PaburqyvKy8uRl5dndDyfed1FR0cjMzMTvXr1gomJCUxMTHDo0CF8+eWXMDExgYuLC59xI3Bzc0PXrl2NtnXp0gWpqakAID1L/v64d6+//jreeustTJ06Fd27d8cTTzyBuXPnYtGiRQD4jBtbXZ6nq6trtSZZFRUVyMnJ4TMnImpGf9kETaVSITg4GPv375e26fV67N+/H/3792/ByNouIQReeuklbNu2DQcOHICPj4/R/uDgYCiVSqNnHh8fj9TUVD7zOho2bBhiY2MRExMjffTu3RszZsyQPuczbriBAwdWWyLi8uXLaN++PQDAx8cHrq6uRs+5oKAAJ06c4HOuo5KSEsjlxv8FKRQK6PV6AHzGja0uz7N///7Iy8tDdHS0dMyBAweg1+vRr1+/Zo+ZiOgvq6W7lLSkjRs3ClNTU7FmzRpx8eJFMXv2bGFraysyMjJaOrQ26YUXXhA2Njbi4MGDIj09XfooKSmRjnn++eeFl5eXOHDggDh16pTo37+/6N+/fwtG3fZV7eIoBJ9xYzh58qQwMTERH3/8sUhISBDr168X5ubmYt26ddIxixcvFra2tuLnn38W586dE+PGjRM+Pj6itLS0BSNvO8LDw4WHh4fYsWOHSE5OFj/++KNwdHQUb7zxhnQMn3H9FBYWijNnzogzZ84IAOKzzz4TZ86cEVevXhVC1O15jho1SgQFBYkTJ06Iw4cPi06dOolp06a11C0REf0l/aUTNCGE+Oqrr4SXl5dQqVSib9++4vjx4y0dUpsFoMaP7777TjqmtLRUvPjii8LOzk6Ym5uLCRMmiPT09JYL+j5we4LGZ9w4tm/fLgICAoSpqanw9/cXK1euNNqv1+vFu+++K1xcXISpqakYNmyYiI+Pb6Fo256CggLxyiuvCC8vL2FmZiY6dOgg/v73vwuNRiMdw2dcPxERETX+Dg4PDxdC1O15Zmdni2nTpglLS0thbW0tZs6cKQoLC1vgboiI/rpkQgjRMrU7IiIiIiIiquovOweNiIiIiIiotWGCRkRERERE1EowQSMiIiIiImolmKARERERERG1EkzQiIiIiIiIWgkmaERERERERK0EEzQiIiIiIqJWggkaEbUpBw8ehEwmQ15eXkuHQkRERNTomKARERERERG1EkzQiIiIiIiIWgkmaERUL3q9HosWLYKPjw/UajUCAwOxdetWAH8OP9y5cyd69OgBMzMzPPDAAzh//rzROX744Qd069YNpqam8Pb2xqeffmq0X6PR4M0330S7du1gamoKX19ffPvtt0bHREdHo3fv3jA3N8eAAQMQHx/ftDdORERE1AyYoBFRvSxatAhr167F8uXLceHCBcydOxePP/44Dh06JB3z+uuv49NPP0VUVBScnJwQGhoKrVYLoDKxmjx5MqZOnYrY2Fi8//77ePfdd7FmzRrp9U8++SS+//57fPnll7h06RJWrFgBS0tLozj+/ve/49NPP8WpU6dgYmKCp59+ulnun4iIiKgpyYQQoqWDIKK2QaPRwN7eHvv27UP//v2l7c888wxKSkowe/ZsDB06FBs3bsSUKVMAADk5OfD09MSaNWswefJkzJgxA1lZWdizZ4/0+jfeeAM7d+7EhQsXcPnyZXTu3Bl79+7F8OHDq8Vw8OBBDB06FPv27cOwYcMAALt27cIjjzyC0tJSmJmZNfFTICIiImo6rKARUZ0lJiaipKQEI0aMgKWlpfSxdu1aXLlyRTquavJmb2+Pzp0749KlSwCAS5cuYeDAgUbnHThwIBISEqDT6RATEwOFQoHBgwffMZYePXpIn7u5uQEAMjMzG3yPRERERC3JpKUDIKK2o6ioCACwc+dOeHh4GO0zNTU1StLulVqtrtNxSqVS+lwmkwGonB9HRERE1JaxgkZEdda1a1eYmpoiNTUVvr6+Rh/t2rWTjjt+/Lj0eW5uLi5fvowuXboAALp06YIjR44YnffIkSPw8/ODQqFA9+7dodfrjea0EREREf1VsIJGRHVmZWWF1157DXPnzoVer8egQYOQn5+PI0eOwNraGu3btwcA/OMf/4CDgwNcXFzw97//HY6Ojhg/fjwA4G9/+xv69OmDDz/8EFOmTMGxY8fwn//8B19//TUAwNvbG+Hh4Xj66afx5ZdfIjAwEFevXkVmZiYmT57cUrdORERE1CyYoBFRvXz44YdwcnLCokWLkJSUBFtbW/Tq1Qtvv/22NMRw8eLFeOWVV5CQkICePXti+/btUKlUAIBevXph8+bNeO+99/Dhhx/Czc0N//jHP/DUU09J11i2bBnefvttvPjii8jOzoaXlxfefvvtlrhdIiIiombFLo5E1GgMHRZzc3Nha2vb0uEQERERtTmcg0ZERERERNRKMEEjIiIiIiJqJTjEkYiIiIiIqJVgBY2IiIiIiKiVYIJGRERERETUSjBBIyIiIiIiaiWYoBEREREREbUSTNCIiIiIiIhaCSZoRERERERErQQTNCIiIiIiolaCCRoREREREVErwQSNiIiIiIiolfh/nsuUycA/vjoAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "fig, ax = plt.subplots()\n", + "\n", + "for label in list(sorted_labels):\n", + " _ = np.round(architectures_TM[label]['epochs_acc'][-1], 2)\n", + " nb_skip = nb_skip_conns[label]['nb_skip_conns']\n", + " nb_skiped_l = nb_skip_conns[label]['nb_jumped_layer']\n", + " txt = f'{label} ({_}% - # sc: {nb_skip}, # sl: {nb_skiped_l})'\n", + " ax.plot(np.arange(len(architectures_TM[label]['epochs_x'])), architectures_TM[label]['epochs_acc'], label=txt)\n", + "\n", + "ax.set_ylim(0, 100)\n", + "ax.set_yticks(np.arange(0, 110, 10))\n", + "ax.set_ylabel('validation acc [%]')\n", + "ax.set_xlim(0, 100)\n", + "ax.set_xlabel('epoch')\n", + "\n", + "pos = ax.get_position()\n", + "ax.set_position([pos.x0, pos.y0, pos.width * 0.9, pos.height])\n", + "ax.legend(loc='center right', bbox_to_anchor=(1.8, 0.5), framealpha=0)\n", + "ax.grid(axis='y')\n", + "\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAArUAAAG2CAYAAABh3H5yAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAADUb0lEQVR4nOzdeVzU1f748dewgygu7K4ogoqACC6YSa5oZpimpmlYlunX6rpvv3BJRbOuUuoVLTS9WS6ZSy6J5p6KKyqgIoiYCpgbu6yf3x/EXEZ2HATr/Xw8Pg+bc87nnPMZ7THvOXMWlaIoCkIIIYQQQrzAdKq6A0IIIYQQQjwrCWqFEEIIIcQLT4JaIYQQQgjxwpOgVgghhBBCvPAkqBVCCCGEEC88CWqFEEIIIcQLT4JaIYQQQgjxwpOgVgghhBBCvPAkqBVCCCGEEC88CWqFEEIIIcQLr0qD2qNHj9KvXz9sbW1RqVRs375dI19RFGbNmoWNjQ3Gxsb06NGD69eva5R5+PAhb7/9NrVq1aJ27dqMGjWKlJSU5/gUQgghhBCiqlVpUJuamoqrqysrVqwoMn/x4sV8/fXXBAYGEhISQo0aNfD29ubJkyfqMm+//Tbh4eHs37+fXbt2cfToUUaPHv28HkEIIYQQQlQDKkVRlKruBIBKpWLbtm30798fyBultbW1ZdKkSUyePBmAxMRErKys+O6773jrrbe4cuUKrVq14syZM3h4eADw66+/8uqrr3L79m1sbW2r6nGEEEIIIcRzpFfVHShOTEwM8fHx9OjRQ51mZmZGhw4dOHnyJG+99RYnT56kdu3a6oAWoEePHujo6BASEsIbb7xRZN0ZGRlkZGSoX+fm5vLw4UPq1auHSqWqvIcSQgghqgFFUUhOTsbW1hYdHVleI/4eqm1QGx8fD4CVlZVGupWVlTovPj4eS0tLjXw9PT3q1q2rLlOUhQsXMnfuXC33WAghhHix/PHHHzRo0KCquyGEVlTboLYyzZgxg4kTJ6pfJyYm0qhRI2JiYqhZs2YV9kwIIYSofMnJydjZ2clnnvhbqbZBrbW1NQAJCQnY2Nio0xMSEmjTpo26zL179zTuy87O5uHDh+r7i2JoaIihoWGh9Lp161KrVi0t9F4IIYSovvT19QFkyp34W6m2E2ns7Oywtrbmt99+U6clJSUREhKCp6cnAJ6enjx+/Jhz586pyxw8eJDc3Fw6dOjw3PsshBBCCCGqRpWO1KakpBAVFaV+HRMTQ2hoKHXr1qVRo0aMHz+e+fPn07x5c+zs7PDz88PW1la9Q0LLli3p3bs3H3zwAYGBgWRlZfHRRx/x1ltvyc4HQgghhBD/IFUa1J49e5auXbuqX+fPc/X19eW7775j6tSppKamMnr0aB4/fkznzp359ddfMTIyUt+zYcMGPvroI7p3746Ojg4DBw7k66+/fu7PIoQQQgghqk612ae2KiUlJWFmZkZiYqLMqRVCCPG3J5974u+o2s6pFUIIIYQQoqwkqBVCCCGEEC88CWqFEEIIIcQLT4JaIYQQQgjxwpOgVgghhBBCvPAkqBVCCCGEEC88CWqFEEIIIcQLT4JaIYQQQgjxwpOgVgghhBBCvPAkqBVCCCGEEC88CWqFEEIIIcQLT4JaIYQQQgjxwpOgVgghhBBCvPAkqBVCCCGEEC88CWqFEEIIIcQLT4JaIYQQQgjxwpOgVgghhBBCvPAkqBVCCCGEEC88CWqFEEIIIcQLT4JaIYQQQgjxwpOgVgghhBBCvPAkqBVCCCGEEC88CWqFEEIIIcQLT4JaIYQQQgjxwpOgVgghhBBCvPAkqBVCCCGEEC+8ah/UJicnM378eBo3boyxsTGdOnXizJkz6nxFUZg1axY2NjYYGxvTo0cPrl+/XoU9FkIIIYQQz1u1D2rff/999u/fz3//+18uX75Mr1696NGjB3fu3AFg8eLFfP311wQGBhISEkKNGjXw9vbmyZMnVdxzIYQQQgjxvKgURVGquhPFSU9Pp2bNmuzYsYO+ffuq093d3enTpw/z5s3D1taWSZMmMXnyZAASExOxsrLiu+++46233ipTO0lJSZiZmZGYmEitWrUq5VmEEEKI6kI+98TfkV5Vd6Ak2dnZ5OTkYGRkpJFubGzM8ePHiYmJIT4+nh49eqjzzMzM6NChAydPniw2qM3IyCAjI0P9OikpCYCsrCyysrIq4UmEEEKI6kM+68TfUbUOamvWrImnpyfz5s2jZcuWWFlZ8eOPP3Ly5Ens7e2Jj48HwMrKSuM+KysrdV5RFi5cyNy5cwul79q1i9q1a2v1GYQQQojqJi0traq7IITWVeugFuC///0v7733HvXr10dXV5e2bdsydOhQzp07V+E6Z8yYwcSJE9Wvk5KSaNiwIVevXmXAgAG0adNGCz0XQgghqqf8XyiF+Dup9kFts2bNOHLkCKmpqSQlJWFjY8OQIUNo2rQp1tbWACQkJGBjY6O+JyEhocTA1NDQEENDwyLz9uzZg4ODA2ZmZlp9DiGEEKK60NfXr+ouCKF11X73g3w1atTAxsaGR48esW/fPnx8fLCzs8Pa2prffvtNXS4pKYmQkBA8PT0r1I6iKDx8+FBb3RZCCCGEEM9BtR+p3bdvH4qi4OjoSFRUFFOmTKFFixa8++67qFQqxo8fz/z582nevDl2dnb4+flha2tL//79K9SeSqWibt262n0IIYQQQghRqap9UJuYmMiMGTO4ffs2devWZeDAgSxYsED908nUqVNJTU1l9OjRPH78mM6dO/Prr78W2jGhrPr16ydTD4QQQgghXjDVep/a5yV/v77p06fz6aefUqNGjarukhBCCFFpZJ9a8Xf0wsypfV7CwsKqugtCCCGEEKKcJKh9ysWLF6u6C0IIIYQQopwkqH3K3bt3uX//flV3QwghhBBClIMEtQU0bdoUgEuXLlVxT4QQQgghRHlIUFtA69atgbygNjc3t4p7I4QQQgghykqC2gIcHBwwMDDg8ePH/PHHH1XdHSGEEEIIUUYS1Bagr69Pq1atAJmCIIQQQgjxIpGg9ikuLi4AhIeHk5WVVcW9EUIIIYQQZSFB7VOaNGlCrVq1ePLkCdevX6/q7gghhBBCiDKQoPYpOjo6ODs7AzIFQQghhBDiRSFBbRHypyBERkaSlpZWxb0RQgghhBClkaC2CFZWVlhbW5Obm0t4eHhVd0cIIYQQQpRCgtpi5I/WyhQEIYQQQojqT4LaYjg7O6NSqfjjjz94+PBhVXdHCCGEEEKUQILaYtSsWVOOzRVCCCGEeEFIUFuCglMQFEWp4t4IIYQQQojiSFBbghYtWqCvr8/Dhw+5fft2VXdHCCGEEEIUQ4LaEhgaGtKyZUtApiAIIYQQQlRnEtSWIn8KQlhYGNnZ2VXcGyGEEEIIURQJakthZ2eHqakp6enpREVFVXV3hBBCCCFEESSoLYWurq4cmyuEEEIIUc1JUFsG+VMQrl27Rnp6ehX3RgghhBBCPE2C2jKwtrbGwsKCnJwcIiIiqro7QgghhBDiKRLUloFKpcLV1RWQKQhCCCGEENWRBLVllD+vNjY2lsePH1dtZ4QQQgghhAYJasvIzMyMJk2aADJaK4QQQghR3VTroDYnJwc/Pz/s7OwwNjamWbNmzJs3T+PIWkVRmDVrFjY2NhgbG9OjRw+uX79eKf0pOAVBjs0VQgghhKg+qnVQ+/nnn7Ny5UqWL1/OlStX+Pzzz1m8eDHLli1Tl1m8eDFff/01gYGBhISEUKNGDby9vXny5InW+9OyZUv09PS4f/8+cXFxWq9fCCGEEEJUTLUOak+cOIGPjw99+/alSZMmvPnmm/Tq1YvTp08DeaO0AQEBfPrpp/j4+ODi4sL69eu5e/cu27dv13p/jIyMcHR0BODixYtar18IIYQQQlSMXlV3oCSdOnVi9erVREZG4uDgwMWLFzl+/DhLliwBICYmhvj4eHr06KG+x8zMjA4dOnDy5EneeuutIuvNyMggIyND/TopKQmArKwssrKySuyTk5MT4eHhhIWF0a1bN3R0qvX3AiGEEKKQ0j7rhHgRVeugdvr06SQlJdGiRQt0dXXJyclhwYIFvP322wDEx8cDYGVlpXGflZWVOq8oCxcuZO7cuYXSg4ODMTExKbFPiqKgp6dHamoqmzZtwszMrLyPJYQQQlSptLS0qu6CEFpXrYPazZs3s2HDBn744QecnJwIDQ1l/Pjx2Nra4uvrW+F6Z8yYwcSJE9Wvk5KSaNiwIb169aJWrVql3q+np8fZs2cxNjbm1VdfrXA/hBBCiKqQ/wulEH8n1TqonTJlCtOnT1dPI3B2diY2NpaFCxfi6+uLtbU1AAkJCdjY2KjvS0hIoE2bNsXWa2hoiKGhYaF0fX199PX1S+2Xm5sbZ8+e5dq1a+Tm5hZZlxBCCFFdleWzTogXTbWeEJqWllZozqquri65ubkA2NnZYW1tzW+//abOT0pKIiQkBE9Pz0rrl62tLfXq1SM7O5srV65UWjtCCCGEEKJsqnVQ269fPxYsWMDu3bu5efMm27ZtY8mSJbzxxhtA3vG148ePZ/78+ezcuZPLly/zzjvvYGtrS//+/SutXyqVChcXF0B2QRBCCCGEqA6q9fSDZcuW4efnx//93/9x7949bG1t+fDDD5k1a5a6zNSpU0lNTWX06NE8fvyYzp078+uvv2JkZFSpfXNxceHQoUPExMSQmJgoC8aEEEIIIaqQSpGjsUhKSsLMzIzExMQyLRTLt2bNGm7dukXPnj156aWXKrGHQgghhPZU9HNPiOqsWk8/qO5kCoIQQgghRPUgQe0zcHJyQldXl3v37pW4L64QQgghhKhcEtQ+A2NjYxwcHAC4dOlSFfdGCCGEEOKfS4LaZ5Q/BeHy5cvqrcaEEEIIIcTzJUHtM2revDnGxsYkJycTExNT1d0RQgghhPhHkqD2Genp6eHk5ATIFAQhhBBCiKoiQa0W5E9BiIiIIDMzs4p7I4QQQgjxzyNBrRY0bNiQOnXqkJWVxdWrV6u6O0IIIYQQ/zgS1GpBwWNzZQqCEEIIIcTzJ0GtluQHtdHR0SQnJ1dxb4QQQggh/lkkqNWSevXq0aBBAxRFISwsrKq7I4QQQgjxjyJBrRbJFAQhhBBCiKohQa0WOTk5oaOjQ1xcHPfu3avq7gghhBBC/GNIUKtFNWrUoHnz5oCM1gohhBBCPE8S1GpZwSkIcmyuEEIIIcTzIUGtljk4OGBoaEhSUhKxsbFV3R0hhBBCiH8ECWq1TF9fX47NFUIIIYR4ziSorQQFj83Nysqq4t4IIYQQQvz9SVBbCRo1aoSZmRkZGRlcu3atqrsjhBBCCPG3J0FtJdDR0ZE9a4UQQgghniMJaitJflAbFRVFampqFfdGCCGEEOLvTYLaSmJhYYGtrS25ublybK4QQgghRCWToLYSyRQEIYQQQojnQ4LaApIfPtBqfa1bt0alUnHnzh3u37+v1bqFEEIIIcT/SFBbwLpJY7l8MFhr9ZmammJvbw/IaK0QQgghql5mZib29vacOHGiqrtCYGAg/fr101p9EtQWoCgK+79ZTvID7Y2qFpyCoCiK1uoVQggh/slGjhyJSqVCpVKhr6+PnZ0dU6dO5cmTJ1pr48iRI3Tr1o26detiYmJC8+bN8fX1JTMzU11GURRWr15Nhw4dMDU1pXbt2nh4eBAQEEBaWhoAc+bMQaVSMWbMGI36Q0NDUalU3Lx5E4CbN2+iUqmwtLQkOTlZo2ybNm2YM2eO+vXPP/9Mr169qFevHiqVitDQ0DI9U2BgIHZ2dnTq1AmAw4cPq9/Hp68zZ85o9Ovp69SpUyW2devWLfr27YuJiQmWlpZMmTKF7Oxsdf57773H+fPnOXbsWJn6XppqH9Q2adKkyDdy3LhxADx58oRx48ZRr149TE1NGThwIAkJCRVuT8nN5XH8XW11H0dHRwwMDHj8+DF//PGH1uoVQggh/ul69+5NXFwcN27cYOnSpaxatYrZs2drpe6IiAh69+6Nh4cHR48e5fLlyyxbtgwDAwNycnLU5UaMGMH48ePx8fHh0KFDhIaG4ufnx44dOwgO/t+vv0ZGRgQFBXH9+vVS205OTubLL78ssUxqaiqdO3fm888/L/MzKYrC8uXLGTVqlDqtU6dOxMXFaVzvv/8+dnZ2eHh4aNx/4MABjXLu7u7FtpWTk0Pfvn3JzMzkxIkTrFu3ju+++45Zs2apyxgYGDBs2DC+/vrrMj9DaQ9Yrd27d0+Ji4tTX/v371cA5dChQ4qiKMqYMWOUhg0bKr/99pty9uxZpWPHjkqnTp3K1UZiYqICKPPf6KX8+61+StL9P7X6DNu2bVNmz56t7Ny5U6v1CiGEEBWR/7mXmJio1XrvPk5Tfo/6U7n7OE2r9RbF19dX8fHx0UgbMGCA4ubmpn6dk5Oj+Pv7K02aNFGMjIwUFxcXZcuWLer8hw8fKsOGDVPMzc0VIyMjxd7eXlmzZo2iKIqydOlSpUmTJiX2YdOmTQqgbN++vVBebm6u8vjxY0VRFGX27NmKq6ur0rNnT2XQoEHqMhcuXFAAJSYmRlEURYmJiVEAZcqUKYqpqamSkJCgLuvq6qrMnj27UDv591y4cKHEviqKopw5c0bR0dFRkpKSii2TmZmpWFhYKJ999lmF2si3Z88eRUdHR4mPj1enrVy5UqlVq5aSkZGhTjty5IhiYGCgpKU9+7+Zaj9Sa2FhgbW1tfratWsXzZo1w8vLi8TERIKCgliyZAndunXD3d2dtWvXcuLEiVKHxIvT7d0PqVnPXKvPkD8FITw8XGPYXQghhKiOFEUhLTO7XNd/T97kpUUHGfZNCC8tOsh/T94sdx3KM0zTCwsL48SJExgYGKjTFi5cyPr16wkMDCQ8PJwJEyYwfPhwjhw5AoCfnx8RERHs3buXK1eusHLlSszN82IAa2tr4uLiOHr0aLFtbtiwAUdHR3x8fArlqVQqzMzMNNIWLVrE1q1bOXv2bInPMnToUOzt7fnss8/K/PxlcezYMRwcHKhZs2axZXbu3MmDBw949913C+W9/vrrWFpa0rlzZ3bu3FliWydPnsTZ2RkrKyt1mre3N0lJSYSHh6vTPDw8yM7OJiQkpAJPpEnvmWt4jjIzM/n++++ZOHEiKpWKc+fOkZWVRY8ePdRlWrRoQaNGjTh58iQdO3Yssp6MjAwyMjLUr5OSktT/nZWZSVZWllb7Xb9+fWrWrElycjJXrlyhRYsWWq1fCCGEKI/SPufSs3JoNWtfhevPVcBvRzh+O8JLL1xAxGfemBiUPTTZtWsXpqamZGdnk5GRgY6ODsuXLwfyPuv9/f05cOAAnp6eADRt2pTjx4+zatUqvLy8uHXrFm5ubuqf2Zs0aaKue9CgQezbtw8vLy+sra3p2LEj3bt355133qFWrVoAXL9+HUdHxzL3t23btgwePJhp06bx22+/FVtOpVKxaNEi+vXrx4QJE2jWrFmZ2yhJbGwstra2JZYJCgrC29ubBg0aqNNMTU3597//zUsvvYSOjg5bt26lf//+bN++nddff73IeuLj4zUCWkD9Oj4+Xp1mYmKCmZkZsbGxFX0stRcqqN2+fTuPHz9m5MiRQN6bYmBgQO3atTXKWVlZabxhT1u4cCFz584tMu/E1o3cyVRQ6epqq9sAGBsbk5yczIEDB7hx44ZW6xZCCCHKI38B04uua9eurFy5ktTUVJYuXYqenh4DBw4E8k70TEtLo2fPnhr3ZGZm4ubmBsDYsWMZOHAg58+fp1evXvTv31+9gEpXV5e1a9cyf/58Dh48SEhICP7+/nz++eecPn0aGxubCo0sz58/n5YtWxIcHIylpWWx5by9vencuTN+fn788MMP5W6nKOnp6RgZGRWbf/v2bfbt28fmzZs10s3NzZk4caL6dbt27bh79y5ffPFFsUFteRgbG2vl3+QLFdQGBQXRp0+fUr9llGbGjBkafzlJSUk0bNgQ41q1yU5Lxa6WCa28uj9rdzXcu3ePb775huTkZLp27YqxsbFW6xdCCCHKquAvlEUx1tcl4jPvMtcXn/iEHkuOkFsgxtNRwYGJXlibFR9EFdVuedSoUUO9deaaNWtwdXUlKCiIUaNGkZKSAsDu3bupX7++xn2GhoYA9OnTh9jYWPbs2cP+/fvp3r0748aN01ikVb9+fUaMGMGIESOYN28eDg4OBAYGMnfuXBwcHLh69Wq5+tysWTM++OADpk+fTlBQUIllFy1ahKenJ1OmTClXG8UxNzfn8uXLxeavXbuWevXqlSlQ7dChA/v37y8239ramtOnT2uk5S/kt7a21kh/+PAhFhYWpbZZmhcmqI2NjeXAgQP8/PPP6jRra2syMzN5/PixxmhtQkJCoTesIENDQ/U/6ILc+vTj/M8bObdrG87deqKjo73R2vr162NlZUVCQgKRkZGFVhQKIYQQz4u+vn6J+SqVqlzTAJpamLJwgDMzfw4jR1HQVanwH9Caphamz9rVMtPR0WHmzJlMnDiRYcOG0apVKwwNDbl16xZeXl7F3mdhYYGvry++vr68/PLLTJkypdidB+rUqYONjQ2pqakADBs2jLfeeosdO3YUmlerKApJSUmF5tUCzJo1i2bNmrFx48YSn6l9+/YMGDCA6dOnl/b4ZeLm5sbKlStRFAWVSlWov2vXruWdd94p9d8H5G1HZmNjU2y+p6cnCxYs4N69e+oR6f3791OrVi1atWqlLhcdHc2TJ0/Uo+fPotovFMu3du1aLC0t6du3rzrN3d0dfX19jXkp165d49atW+r5M+Xh/EoPjGqY8ijuDtdDTmql3wW5uroCcPHiRa3XLYQQQlSlIe0acXx6V378oCPHp3dlSLtGz70PgwYNQldXlxUrVlCzZk0mT57MhAkTWLduHdHR0Zw/f55ly5axbt06IC+43LFjB1FRUYSHh7Nr1y5atmwJwKpVqxg7dizBwcFER0cTHh7OtGnTCA8PVx8YMHjwYIYMGcLQoUPx9/fn7NmzxMbGsmvXLnr06MGhQ4eK7KeVlRUTJ04s01ZWCxYs4ODBg1y7dk0j/eHDh4SGhhIREQHkxT+hoaElTr/s2rUrKSkpGgu18h08eJCYmBjef//9Qnnr1q3jxx9/5OrVq1y9ehV/f3/WrFnDxx9/rC6zbds2jTVDvXr1olWrVowYMYKLFy+yb98+Pv30U8aNG6cxsHjs2DGaNm2qnXnDz7x/wnOQk5OjNGrUSJk2bVqhvDFjxiiNGjVSDh48qJw9e1bx9PRUPD09y1V/wa1Njm/6XvlycF9l/dRPlNzcXG09grqdOXPmKLNnz1YePHig1bqFEEKIsqqsLb2ep6K29FIURVm4cKFiYWGhpKSkKLm5uUpAQIDi6Oio6OvrKxYWFoq3t7dy5MgRRVEUZd68eUrLli0VY2NjpW7duoqPj49y48YNRVEU5fz588rw4cMVOzs7xdDQUKlXr57SpUuXQttz5uTkKCtXrlTatWunmJiYKLVq1VLc3d2Vr776Sr1NVf6WXgUlJiYq5ubmRW7p9fTWWaNHj1YAjS291q5dqwCFrqK2/Spo8ODByvTp0wulDx06tNgtUb/77julZcuW6udr3769xtZoBftT0M2bN5U+ffooxsbGirm5uTJp0iQlKytLo0yvXr2UhQsXltjnslIpSvU/5io4OBhvb2+uXbuGg4ODRt6TJ0+YNGkSP/74IxkZGXh7e/Of//ynxOkHT8v/eSAxMRF9Fawe9y7ZGRkMmDEXuzbFbyxcEevXr+fGjRt07dq1xJ9DhBBCiMpS8HMvfyW/+Ge4dOkSPXv2JDo6GlPT5zc9pCjh4eF069aNyMjIIqdplNcLMf2gV69eKIpSKKCFvBM6VqxYwcOHD0lNTeXnn38uV0D7NOOatXDt0RuA09u3VLie4hScgvACfJ8QQgghxN+Ii4sLn3/+OTExMVXdFeLi4li/fr1WAlp4QYLa5839tTfQ0dXj9pUwbl8t3x57pWnRogX6+vo8fPiQO3fuaLVuIYQQQojSjBw5Emdn56ruBj169MDbu+y7bJRGgtoi1KxrjpNXN0D7o7WGhobqidSXLl3Sat1CCCGEEP9UEtQWo93rA1GpdIi5cJZ7N7V7WEL+FISwsDBycnK0WrcQQgghxD+RBLXFqGNTH4eOLwHaH621s7OjRo0apKWlERUVpdW6hRBCCCH+iSSoLUH7/oMAiDz1O4/itDf/VVdXVz2XRaYgCCGEEEI8OwlqS2DZpClN27ZDUXI5s3OrVuvOn4Jw9epVnjx5otW6hRBCCCH+aSSoLUV7n7zR2vAjB0l+cF9r9VpbW2NhYUFOTo76NBAhhBBCCFExEtSWon6LVjRo2ZrcnGzO7d6mtXpVKhUuLi6AHJsrhBBCCPGsJKgtgw5/za29eOBX0pIStVZvflAbGxvL48ePtVavEEIIIURRHjx4gKWlJTdv3qzqrjB9+nQ+/vhjrdUnQW0ZNHZti2WTZmRnZHDh11+0Vq+ZmRlNmjQB4PLly1qrVwghhPi7GzlyJCqVCpVKhb6+PnZ2dkydOlWr61SOHDlCt27dqFu3LiYmJjRv3hxfX18yMzPVZRRFYfXq1XTo0AFTU1Nq166Nh4cHAQEBpKWlATBnzhxUKhVjxozRqD80NBSVSqUOMG/evIlKpcLS0pLk5GSNsm3atGHOnDkAZGVlMW3aNJydnalRowa2tra888473L17t9RnWrBgAT4+Pur447vvvlO/j09f9+7dA+Dw4cNF5sfHx5fY1qVLl3j55ZcxMjKiYcOGLF68WCN/8uTJrFu3jhs3tLN1qgS1ZaBSqejwRt5o7YVffyEzPU1rdRecgiDH5gohhBBl17t3b+Li4rhx4wZLly5l1apVzJ49Wyt1R0RE0Lt3bzw8PDh69CiXL19m2bJlGBgYaOwxP2LECMaPH4+Pjw+HDh0iNDQUPz8/duzYQXBwsLqckZERQUFBXL9+vdS2k5OT+fLLL4vNT0tL4/z58/j5+XH+/Hl+/vlnrl27xuuvv15ivWlpaQQFBTFq1Ch12pAhQ4iLi9O4vL298fLywtLSUuP+a9euaZR7Or+gpKQkevXqRePGjTl37hxffPEFc+bMYfXq1eoy5ubmeHt7s3LlytLekrJRhJKYmKgASmJiYrFlcnKylaB/jVa+HNxXOb3jJ621nZ6ersybN0+ZPXu2cufOHa3VK4QQQhSnLJ97FfL4tqLcOJL3ZyXz9fVVfHx8NNIGDBiguLm5qV/n5OQo/v7+SpMmTRQjIyPFxcVF2bJlizr/4cOHyrBhwxRzc3PFyMhIsbe3V9asWaMoiqIsXbpUadKkSYl92LRpkwIo27dvL5SXm5urPH78WFEURZk9e7bi6uqq9OzZUxk0aJC6zIULFxRAiYmJURRFUWJiYhRAmTJlimJqaqokJCSoy7q6uiqzZ88uti+nT59WACU2NrbYMlu2bFEsLCxKfKZ79+4p+vr6yvr169Vphw4dUgDl0aNHJd5b0H/+8x+lTp06SkZGhjpt2rRpiqOjo0a5devWKQ0aNChzvSWRkdoy0tHRpb3PmwCc272d7AI/PTwLIyMjHB0dAdmzVgghRDWhKJCZWr7r9DcQ0BrW9cv78/Q35a/jGX6xDAsL48SJExgYGKjTFi5cyPr16wkMDCQ8PJwJEyYwfPhwjhw5AoCfnx8RERHs3buXK1eusHLlSszNzYG8XYri4uI4evRosW1u2LABR0dHfHx8CuWpVCrMzMw00hYtWsTWrVs5e/Zsic8ydOhQ7O3t+eyzz8r8/ImJiahUKmrXrl1smWPHjuHu7l5iPevXr8fExIQ333yzUF6bNm2wsbGhZ8+e/P777yXWc/LkSbp06aLx9+Ht7c21a9d49OiROq19+/bcvn1bK3N89Z65hn+Qli+/woktP5D84E/CjxzAteerWqnXxcWF8PBwLl++TM+ePdHV1dVKvUIIIUSFZKWBv23F71dyYc/kvKs8Zt4FgxplLr5r1y5MTU3Jzs4mIyMDHR0dli9fDkBGRgb+/v4cOHAAT09PAJo2bcrx48dZtWoVXl5e3Lp1Czc3Nzw8PADU80wBBg0axL59+/Dy8sLa2pqOHTvSvXt33nnnHWrVqgXA9evX1QNTZdG2bVsGDx7MtGnT+O2334otp1KpWLRoEf369WPChAk0a9asxHqfPHnCtGnTGDp0qLpvRYmNjcXWtuS/16CgIIYNG4axsbE6zcbGhsDAQDw8PMjIyODbb7/llVdeISQkhLZt2xZZT3x8PHZ2dhppVlZW6rw6deoAqPsTGxur8f5XhIzUloOunj4e/d4A4MzOreQWmFPzLOzt7TExMSE1NVVrk6WFEEKIv7uuXbsSGhpKSEgIvr6+vPvuuwwcOBCAqKgo0tLS6NmzJ6ampupr/fr1REdHAzB27Fg2btxImzZtmDp1KidOnFDXraury9q1a7l9+zaLFy+mfv36+Pv74+TkRFxcHECF1sLMnz+fY8eOacy3LYq3tzedO3fGz8+vxHJZWVkMHjwYRVFKnZuanp6OkZFRsfknT57kypUrGnNuARwdHfnwww9xd3enU6dOrFmzhk6dOrF06dIS2yuL/OA5f1Hds5CR2nJy7taLU1s3kngvgWsnjtLy5a7PXKeuri6tW7fm9OnTXLp0iebNm2uhp0IIIUQF6ZvkjZqWVdJdWNE+b4Q2n0oXxoVArXKM+OqblL0sUKNGDezt7QFYs2YNrq6u6oVQKSkpAOzevZv69etr3GdoaAhAnz59iI2NZc+ePezfv5/u3bszbtw4jUVa9evXZ8SIEYwYMYJ58+bh4OBAYGAgc+fOxcHBgatXr5arz82aNeODDz5g+vTpBAUFlVh20aJFeHp6MmXKlCLz8wPa2NhYDh48WOIoLeQtzCr40//Tvv32W9q0aVPqFAXImzZw/PjxYvOtra1JSEjQSMt/bW1trU57+PAhABYWFqW2WRoZqS0nfUMj3Pv2ByBk+xaU3NySbyij/F0Qrly5QkZGhlbqFEIIISpEpcqbBlDWy7w59PsqL5CFvD/7BeSll6celarCXdbR0WHmzJl8+umnpKen06pVKwwNDbl16xb29vYaV8OGDdX3WVhY4Ovry/fff09AQIDG6vyn1alTBxsbG1JTUwEYNmwYkZGR7Nixo1BZRVFITCx6b/tZs2YRGRnJxo0bS3ym9u3bM2DAAKZPn14oLz+gvX79OgcOHKBevXol1gXg5uZW7CmmKSkpbN68udAobXFCQ0OxsbEpNt/T05OjR4+SlZWlTtu/fz+Ojo7qqQeQNxdaX18fJyenMrVbEglqK8C116sYGBvz4PYtos+f0Uqd9evXp169emRnZ3PlyhWt1CmEEEI8N23fgfGXwXdX3p9t33nuXRg0aBC6urqsWLGCmjVrMnnyZCZMmMC6deuIjo7m/PnzLFu2jHXr1gF5weWOHTuIiooiPDycXbt20bJlSwBWrVrF2LFjCQ4OJjo6mvDwcKZNm0Z4eDj9+vUDYPDgwQwZMoShQ4fi7+/P2bNniY2NZdeuXfTo0YNDhw4V2U8rKysmTpzI119/XeozLViwgIMHD3Lt2jV1WlZWFm+++SZnz55lw4YN5OTkEB8fT3x8vMYeuk/z9vYmPDy8yNHaTZs2kZ2dzfDhwwvlBQQEqN+nsLAwxo8fz8GDBxk3bpy6zPLly+nevbv69bBhwzAwMGDUqFGEh4ezadMmvvrqKyZOnKhR97Fjx3j55Zc15vBWlAS1FWBUw5Q2vfoCELJtk1b2ly14bK7sgiCEEOKFZFYf7F7O+7MK6Onp8dFHH7F48WJSU1OZN28efn5+LFy4kJYtW9K7d292796tXsBkYGDAjBkzcHFxoUuXLujq6qpHT9u3b09KSgpjxozByckJLy8vTp06xfbt2/Hy8gLyPrt/+OEHlixZok53cXFhzpw5+Pj44O3tXWxfJ0+ejKmpaanP5ODgwHvvvadxqMSdO3fYuXMnt2/fVu9IkH8VnBf8NGdnZ9q2bcvmzZsL5QUFBTFgwIAid0/IzMxk0qRJODs74+XlxcWLFzlw4IBGEHv//n31XGXIO2AqODiYmJgY3N3dmTRpErNmzWL06NEadW/cuJEPPvig1PehLFSKNiKyF1xSUhJmZmYkJiaWOh8lX+rjR3z70SiyszIZ5LeARq1dn7kfjx494quvvgJg4sSJZe6LEEIIUR4V+dwTfw+7d+9mypQphIWFoaNTtWObe/fuZdKkSVy6dAk9vWdf5iUjtRVUo3YdWnfrCeTNrdWGOnXq0KhRI0COzRVCCCGE9vXt25fRo0dz586dqu4KqamprF27VisBLUhQ+0za9RuIjq4uty6HEhd1rfQbykCmIAghhBCiMo0fP15jsVxVefPNN+nQoYPW6pOg9hnUsrCkxUt582pOa2m01snJCV1dXRISEoiPj9dKnUIIIYQQf3cS1D6j9j6DQKUi6swpHty+9cz1GRsb4+DgAMhorRBCCCFEWUlQ+4zqNWhI83Z5x+9pa7Q2fwrC5cuXydXSPrhCCCGEEH9nEtRqQfv+gwC48vsREu8llFK6dM2bN8fIyIjk5GRiYmKeuT4hhBBCiL+7ah/U3rlzh+HDh1OvXj2MjY1xdnbm7Nmz6nxFUZg1axY2NjYYGxvTo0cPrl+//lz7aN2sOY1d3FBycznzy8/PXJ+enh6tW7cGZAqCEEIIIURZVOug9tGjR7z00kvo6+uzd+9eIiIi+Pe//61xvNrixYv5+uuvCQwMJCQkhBo1auDt7a2xSfHz0N4nb7Q27FAwqY+LP1e5rAoem1vS6SBCCCGEEKKaB7Wff/45DRs2ZO3atbRv3x47Ozt69epFs2bNgLxR2oCAAD799FN8fHxwcXFh/fr13L17l+3btz/XvjZ0csamuSM5WVmc21P4DOhy19ewIXXq1CEzM5OrV69qoYdCCCGEEH9f2tnttpLs3LkTb29vBg0axJEjR6hfvz7/93//pz5OLSYmhvj4eHr06KG+x8zMjA4dOnDy5EneeuutIuvNyMggIyND/TopKQnIO0s5Kyurwv117zeQXUv8Cd23G7dXfTCqUfrxdyVxcnLi+PHjXLx4UX0WtRBCCPGsnuWzTrzYHjx4QMuWLTl9+jRNmjSp0r5Mnz6d1NRUli1bppX6qnVQe+PGDVauXMnEiROZOXMmZ86c4ZNPPsHAwABfX1/1Pq5WVlYa91lZWZW4x+vChQuZO3duofTg4GBMTEwq3F9FUTAwq0Nm4iN+WhFA3dZtK1wXoJ5CER0dzY4dO9DX13+m+oQQQgiAtLS0qu7CMxs5ciTr1q0D8taiNGjQgEGDBvHZZ59hZGSklTaOHDnC3LlzCQ0N5cmTJ9SvX59OnTrxzTffYGBgAOR99n/zzTcEBQURHh6Onp4e9vb2DB8+nNGjR2NiYsKcOXOYO3cuH374IYGBger6Q0NDcXNzIyYmhiZNmnDz5k3s7OywsLAgOjqamjVrqsu2adOG/v37M2fOHADmzJnDxo0b+eOPPzAwMMDd3Z0FCxaUepjBggUL8PHxUQe03333He+++26RZRMSErC0tOTw4cN07dq1UH5cXBzW1tbFtnXp0iXGjRvHmTNnsLCw4OOPP2bq1Knq/MmTJ9O0aVMmTJhA06ZNS+x3WVTroDY3NxcPDw/8/f0BcHNzIywsjMDAQHx9fStc74wZM5g4caL6dVJSEg0bNqRXr17PfAb2tTo12fefpaTFXGfIJ5PRf8b/sdauXcvdu3exsbGhffv2z1SXEEIIAf/7hfJF17t3b9auXUtWVhbnzp3D19cXlUrF559//sx1R0RE0Lt3bz7++GO+/vprjI2NuX79Olu3biUnJ0ddbsSIEfz88898+umnLF++HAsLCy5evEhAQABNmjShf//+ABgZGREUFMSkSZNo3rx5iW0nJyfz5ZdfFjkAl8/BwYHly5fTtGlT0tPTWbp0Kb169SIqKgoLC4si70lLSyMoKIh9+/ap04YMGULv3r01yo0cOZInT55gaWmpkX7t2jWNOOnp/IKSkpLo1asXPXr0IDAwkMuXL/Pee+9Ru3ZtRo8eDYC5uTne3t6sXLmSL774ovg3pKyUaqxRo0bKqFGjNNL+85//KLa2toqiKEp0dLQCKBcuXNAo06VLF+WTTz4pczuJiYkKoCQmJj5zn3Oys5VvPh6lfDm4r3Ju9/Znri8kJESZPXu2EhgY+Mx1CSGEEIqi3c+9guJS4pSQuyFKXEqcVustiq+vr+Lj46ORNmDAAMXNzU39OicnR/H391eaNGmiGBkZKS4uLsqWLVvU+Q8fPlSGDRummJubK0ZGRoq9vb2yZs0aRVEUZenSpUqTJk1K7MOmTZsUQNm+vfDnfW5urvL48WNFURRl9uzZiqurq9KzZ09l0KBB6jIXLlxQACUmJkZRFEWJiYlRAGXKlCmKqampkpCQoC7r6uqqzJ49u9i+5P+dHjhwoNgyW7ZsUSwsLEp8pnv37in6+vrK+vXr1WmHDh1SAOXRo0cl3lvQf/7zH6VOnTpKRkaGOm3atGmKo6OjRrl169YpDRo0KHO9JanWC8Veeuklrl27ppEWGRlJ48aNAbCzs8Pa2prffvtNnZ+UlERISAienp7Pta/5dHR1addvIABndm0jJ/vZ5i05OTmho6NDXFwcf/75pza6KIQQQpRIURTSstLKdW28uhHvn7wZFTwK75+82Xh1Y7nrUBSlwn0OCwvjxIkT6mkBkDfdcP369QQGBhIeHs6ECRMYPnw4R44cAcDPz4+IiAj27t3LlStXWLlyJebm5gBYW1sTFxfH0aNHi21zw4YNODo64uPjUyhPpVJhZmamkbZo0SK2bt2qsTVpUYYOHYq9vT2fffZZmZ49MzOT1atXY2Zmhqura7Hljh07hru7e4l1rV+/HhMTE958881CeW3atMHGxoaePXvy+++/l1jPyZMn6dKli8bfh7e3N9euXePRo//tEtW+fXtu377NzZs3S6yvLKr19IMJEybQqVMn/P39GTx4MKdPn2b16tWsXr0ayPsHM378eObPn0/z5s2xs7PDz88PW1tb9XB/VXDy6s7JrT+S8uA+EccO4dy1V4XrqlGjBvb29kRGRnLx4kWNRXFCCCFEZUjPTqfDDyXPzSxJLrksCFnAgpAF5bovZFgIJvplX9uya9cuTE1Nyc7OJiMjAx0dHZYvXw7kLQr39/fnwIED6oGupk2bcvz4cVatWoWXlxe3bt3Czc0NDw8PAI2FU4MGDWLfvn14eXlhbW1Nx44d6d69O++88476J/jr16/j6OhY5v62bduWwYMHM23aNI0BuaepVCoWLVpEv379mDBhgnrXp6Ke/6233iItLQ0bGxv279+vDsqLEhsbi62tbYl9DAoKYtiwYRgbG6vTbGxsCAwMxMPDg4yMDL799lteeeUVQkJCaNu26PVD8fHx2NnZaaTlr4GKj49Xb8+a35/Y2NhnXrhWrUdq27Vrx7Zt2/jxxx9p3bo18+bNIyAggLfffltdZurUqXz88ceMHj2adu3akZKSwq+//qq1SeIVoWdggHvf/gCc2fETubk5Jd9QivxvXXJsrhBCCPE/Xbt2JTQ0lJCQEHx9fXn33XcZODDv19KoqCjS0tLo2bMnpqam6mv9+vVER0cDMHbsWDZu3EibNm2YOnUqJ06cUNetq6vL2rVruX37NosXL6Z+/fr4+/vj5OREXFwcQIVGlufPn8+xY8cIDg4usZy3tzedO3fGz8+v1Oc/ceIEvXv3ZvDgwdy7d6/Y8unp6SXGRydPnuTKlSuMGjVKI93R0ZEPP/wQd3d3OnXqxJo1a+jUqRNLly4t8RnKIj941sbixWo9Ugvw2muv8dprrxWbr1Kp+Oyzz8o8RP+8uPbozeltm3kUd5frISdx9Oxc4bocHBwwNDQkMTGRW7duVfkWHEIIIf7ejPWMCRkWUubyCWkJ9N/en1z+N/Cio9Jhu892rEysSrizcLvlkf9rJsCaNWtwdXUlKCiIUaNGkZKSAsDu3bupX7++xn2GhoYA9OnTh9jYWPbs2cP+/fvp3r0748aN48svv1SXrV+/PiNGjGDEiBHMmzcPBwcHAgMDmTt3Lg4ODuXeS75Zs2Z88MEHTJ8+naCgoBLLLlq0CE9PT6ZMmVLi89vb29OxY0eaN29OUFAQM2bMKLK8ubm5xk//T/v2229p06ZNqVMUIG/awPHjx4vNt7a2JiEhQSMt/3XBHRMePnwIUOzitvKo1iO1LzIDYxPc+vQDIGT75meaJ6Svr0+rVq0AuHjxolb6J4QQQhRHpVJhom9S5svOzI7ZnWajo8oLK3RUOsz2nI2dmV256lGpVBXus46ODjNnzuTTTz8lPT2dVq1aYWhoyK1bt9SBX/7VsGFD9X0WFhb4+vry/fffExAQoJ7iWJQ6depgY2NDamoqAMOGDSMyMpIdOwofuqQoComJiUXWM2vWLCIjI9m4cWOJz9S+fXsGDBjA9OnTy/IWkJubq7EP/9Pc3NyIiIgoMi8lJYXNmzcXGqUtTmhoKDY2NsXme3p6cvToUY09kffv34+jo6PGybBhYWHo6+vj5ORUpnZLIkFtJXLr3Q99QyP+vHmDmxfPP1Nd+VMQIiIiZNNsIYQQ1c6A5gPYN3Afa7zXsG/gPgY0H/Dc+zBo0CB0dXVZsWIFNWvWZPLkyUyYMIF169YRHR3N+fPnWbZsmXp/21mzZrFjxw6ioqIIDw9n165d6sOOVq1axdixYwkODiY6Oprw8HCmTZtGeHg4/frlDVoNHjyYIUOGMHToUPz9/Tl79iyxsbHs2rWLHj16cOjQoSL7aWVlxcSJE/n6669LfaYFCxZw8OBBjYXzqampzJw5k1OnThEbG8u5c+d47733uHPnDoMGDSq2Lm9vb8LDw4scrd20aRPZ2dkMHz68UF5AQID6fQoLC2P8+PEcPHiQcePGqcssX76c7t27q18PGzYMAwMDRo0aRXh4OJs2beKrr77S2FIV8havvfzyyxpzeCtKgtpKZFyzFi498vZ+C9m2+ZnqatSoEWZmZmRkZBAZGamN7gkhhBBaZV3DmnbW7bCuUfyG/JVJT0+Pjz76iMWLF5Oamsq8efPw8/Nj4cKFtGzZkt69e7N79271AiYDAwNmzJiBi4sLXbp0QVdXVz162r59e1JSUhgzZgxOTk54eXlx6tQptm/fjpeXF5A3ov3DDz+wZMkSdbqLiwtz5szBx8cHb2/vYvs6efJkTE1LP3nUwcGB9957T30gE+TN97169SoDBw7EwcGBfv368eDBA44dO1biiKezszNt27Zl8+bCMUlQUBADBgygdu3ahfIyMzOZNGkSzs7OeHl5cfHiRQ4cOKARxN6/f189VxnyTngNDg4mJiYGd3d3Jk2axKxZs9R71ObbuHGj+qTYZ6VSnuV38b+JpKQkzMzMSExMfObDF56W/PA+3370Prk52QyZ+zkNWlR8eP3AgQMcP34cBwcHhg0bpsVeCiGE+CepzM89Ub3t3r2bKVOmEBYWho5O1Y5t7t27l0mTJnHp0iX09J59mZeM1FaymnXNcXol75vM6e1bnqmu/CkIUVFR6vk8QgghhBBl1bdvX0aPHs2dO3equiukpqaydu1arQS0IEHtc9Hu9YGoVDrEXDjLvZs3KlyPhYUFNjY25ObmEh4ersUeCiGEEOKfYvz48RqL5arKm2++SYcOFd8P+WkS1D4HdaxtcfhrS6+QZxytdXFxAWQXBCGEEEKIgiSofU469M9bjRh56jgP71Z8yN/Z2RmVSsWdO3d48OCBtronhBBCCPFCk6C2gKynNgnWJovGdjRt2w4UhTM7t1a4HlNTU/VxeZcuXdJW94QQQgghXmgS1BZw47V+PP7pp0qrv33/wQBEHD1I8oP7Fa4nfwrCpUuXnulQByGEEEKIvwsJagvKzSVu1myy4uMrpfr6ji1p0Ko1uTnZnN21rcL1tGjRAgMDAx49esQff/yhxR4KIYQQQryYJKh9Wm4umbG3Kq36Dj55c2sv/fYraUlFH59XGgMDA/WJJzIFQQghhBBCgtrCdHQwaNyo0qpv7NoWS7tmZGdkcOHXXypcT/4UhLCwMLKzs7XVPSGEEEKIF5IEtU+x+Ne/0LeuvOP9VCoVHd7Im1t74ddfyEhLq1A9dnZ21KxZkydPnnD9+nVtdlEIIYQQf1MPHjzA0tKSmzdvVnVXmD59Oh9//LHW6pOg9ikZkZGV3kbzdp7UsW1ARmoqF/fvqVAdOjo6ODs7AzIFQQghxD/PyJEjUalUqFQq9PX1sbOzY+rUqTx58kRrbRw5coRu3bpRt25dTExMaN68Ob6+vmRmZqrLKIrC6tWr6dChA6amptSuXRsPDw8CAgJI+2vgas6cOahUKsaMGaNRf2hoKCqVSh1g3rx5E5VKhaWlJcnJyRpl27Rpw5w5c4rs55gxY1CpVAQEBJT6TAsWLMDHx4cmTZoA8N1336nfx6eve/fuAXD48OEi8+NLWYN06dIlXn75ZYyMjGjYsCGLFy/WyJ88eTLr1q3jxo2KH0xVkAS1T0navZv0Sj6tS6WjQ3ufNwE4t3s72QX+5yiP/CkIkZGRpKena61/QgghxIugd+/exMXFcePGDZYuXcqqVauYPXu2VuqOiIigd+/eeHh4cPToUS5fvsyyZcswMDAgJydHXW7EiBGMHz8eHx8fDh06RGhoKH5+fuzYsYPg4GB1OSMjI4KCgsr062pycjJffvllmfq5bds2Tp06ha2tball09LSCAoKYtSoUeq0IUOGEBcXp3F5e3vj5eWFpaWlxv3Xrl3TKPd0fkFJSUn06tWLxo0bc+7cOb744gvmzJnD6tWr1WXMzc3x9vZm5cqVZXrW0khQW0DNPn0A+PPfSyq9rZadX6GmuQVpiY8JO3ygQnVYW1tjZWVFTk6OHJsrhBCiymXFx5N6KqTSdhF6mqGhIdbW1jRs2JD+/fvTo0cP9u/fr87Pzc1l4cKF2NnZYWxsjKurKz8V2Lrz0aNHvP3221hYWGBsbEzz5s1Zu3YtAMHBwVhbW7N48WJat25Ns2bN6N27N9988w3GxsYAbN68mQ0bNvDjjz8yc+ZM2rVrR5MmTfDx8eHgwYN07dpV3ZajoyNdu3bl//2//1fqc3388ccsWbJEPVJanDt37vDxxx+zYcMG9PX1S613z549GBoa0rFjR3WasbEx1tbW6ktXV5eDBw9qBL75LC0tNcrq6BQfRm7YsIHMzEzWrFmDk5MTb731Fp988glLlmjGWP369WPjxo2l9r0sJKgtwHzsGNDXJ/XECVJ+/71S29LV08PjtQEAnNm5ldwC3/rKo+CetUIIIYQ2KIpCblpaua6HP/xAVLfu3Bo5kqhu3Xn4ww/lruNZ9l4PCwvjxIkTGBgYqNMWLlzI+vXrCQwMJDw8nAkTJjB8+HCOHDkCgJ+fHxEREezdu5crV66wcuVKzM3NgbyBo7i4OI4ePVpsmxs2bMDR0REfH59CeSqVCjMzM420RYsWsXXrVs6ePVviswwdOhR7e3s+++yzYsvk5uYyYsQIpkyZgpOTU4n15Tt27Bju7u4lllm/fj0mJia8+eabhfLatGmDjY0NPXv25PdS4qSTJ0/SpUsXjb8Pb29vrl27xqNHj9Rp7du35/bt21qZ46v3zDX8jRjUr0+doW/xaP1/+fPfS6jh6YmqhG8hz8q5W09O/byRpD8TuHriKK1e7lr6TU/X4ezM/v37uXXrFo8ePaJOnTqV0FMhhBD/JEp6Otfalhz8lCg3l4TP5pHw2bxy3eZ4/hwqE5Myl9+1axempqZkZ2eTkZGBjo4Oy5cvByAjIwN/f38OHDiAp6cnAE2bNuX48eOsWrUKLy8vbt26hZubGx4eHgDqeaYAgwYNYt++fXh5eWFtbU3Hjh3p3r0777zzDrVq1QLg+vXrODo6lrm/bdu2ZfDgwUybNo3ffvut2HIqlYpFixbRr18/JkyYoD5JtKDPP/8cPT09PvnkkzK3HxsbW+o0haCgIIYNG6YejQawsbEhMDAQDw8PMjIy+Pbbb3nllVcICQmhbdu2RdYTHx+PnZ2dRpqVlZU6Lz9eye9PbGysxvtfETJS+xTzMWPQqVGDJxERJO3ZW6lt6Rsa4f5q3re709u3oOTmlruOWrVq0bRpU0BGa4UQQvyzdO3aldDQUEJCQvD19eXdd99l4MCBAERFRZGWlkbPnj0xNTVVX+vXryc6OhqAsWPHsnHjRtq0acPUqVM5ceKEum5dXV3Wrl3L7du3Wbx4MfXr18ff3x8nJyfi4uIAKjSyPH/+fI4dO6Yx37Yo3t7edO7cGT8/v0J5586d46uvvlIv8iqr9PR0jIyMis0/efIkV65cKTT1wNHRkQ8//BB3d3c6derEmjVr6NSpE0uXLi1z28XJD57TKrgbVEEyUvsUvbp1qff+KP786mv+DAigVq+eqAoMnWuba69XOb3jJx7cvkX0udPYt+tY+k1PcXFx4caNG1y6dIkuXbqU6x+4EEII8TSVsTGO58+VuXxWQgI3+r4GBQdndHRounsX+n+NzpW13fKoUaMG9vb2AKxZswZXV1f1QqiUlBQAdu/eTf369TXuMzQ0BKBPnz7ExsayZ88e9u/fT/fu3Rk3bpzGIq369eszYsQIRowYwbx583BwcCAwMJC5c+fi4ODA1atXy9XnZs2a8cEHHzB9+nSCgoJKLLto0SI8PT2ZMmWKRvqxY8e4d+8ejRr9b1/9nJwcJk2aREBAQLE/5Zubm2v89P+0b7/9ljZt2pQ6RQHypg0cP3682Hxra2sSEhI00vJfWxfYOvXhw4cAWFhYlNpmaSo0Urtu3Tp2796tfj116lRq165Np06diI2NfeZOVbW6vr7oWpiTdfs2jzZtrtS2jGqY0sa7LwAh2zdX6Ftfy5Yt0dPT48GDB9y5c0fbXRRCCPEPo1Kp0DExKfNlaGeHzWdzIX/Kno4ONp/NxdDOrlz1PMugjI6ODjNnzuTTTz8lPT2dVq1aYWhoyK1bt7C3t9e4GjZsqL7PwsICX19fvv/+ewICAjRW5z+tTp062NjYkJqaCsCwYcOIjIxkx44dhcoqikJiYtEnh86aNYvIyMhSF0i1b9+eAQMGMH36dI30ESNGcOnSJUJDQ9WXra0tU6ZMYd++fcXW5+bmRkRERJF5KSkpbN68ucgFYkUJDQ3Fxsam2HxPT0+OHj1KVlaWOm3//v04OjpqTJUMCwtDX1+/zPOCS1KhoNbf3189XHzy5ElWrFjB4sWLMTc3Z8KECc/cqaqmY2KCxbiPALj/n/+Q89e3vcri/qoPevoGxEdF8kd4+acQGBoayrG5QgghqlTtN9/E/uBvNFq3DvuDv1G7iIVGlW3QoEHo6uqyYsUKatasyeTJk5kwYQLr1q0jOjqa8+fPs2zZMtatWwfkBZc7duwgKiqK8PBwdu3apf48XbVqFWPHjiU4OJjo6GjCw8OZNm0a4eHh9OvXD4DBgwczZMgQhg4dir+/P2fPniU2NpZdu3bRo0cPDh06VGQ/raysmDhxIl9//XWpz7RgwQIOHjzItWvX1Gn16tWjdevWGpe+vj7W1tYlzvH19vYmPDy8yNHaTZs2kZ2dzfDhwwvlBQQEqN+nsLAwxo8fz8GDBxk3bpy6zPLly+nevbv69bBhwzAwMGDUqFGEh4ezadMmvvrqKyZOnKhR97Fjx3j55Zc15vBWVIWC2j/++EM93L99+3YGDhzI6NGjWbhwIceOHXvmTlUHtQcOwKBJE3IePeLhmrWV2paJWW1ad+sFQMi2io0MFzw2N6eCOykIIYQQz0Lf2poaHdpX6smcJdHT0+Ojjz5i8eLFpKamMm/ePPz8/Fi4cCEtW7akd+/e7N69W72AycDAgBkzZuDi4kKXLl3Q1dVVj562b9+elJQUxowZg5OTE15eXpw6dYrt27fj5eUF5I1o//DDDyxZskSd7uLiwpw5c/Dx8cHb27vYvk6ePBlTU9NSn8nBwYH33ntPK4dKODs707ZtWzZvLhxrBAUFMWDAAGrXrl0oLzMzk0mTJuHs7IyXlxcXL17kwIEDGkHs/fv31XOVAczMzAgODiYmJgZ3d3cmTZrErFmzGD16tEbdGzdu5IMPPnjmZwNQKRX4vdvS0pJ9+/bh5uaGm5sbEydOZMSIEURHR+Pq6qqex/KiSEpKwszMjMTERPWKRoCkfcHc+de/UJmYYL/vV/S0MN+j2D78eY+gf31Abk4Owxb8Gxv7sq+mhLy5NEuWLCE1NZWhQ4eWazWmEEKIf5biPvfE39/u3buZMmUKYWFhJe4z+zzs3buXSZMmcenSJfT0nn2ZV4WepmfPnrz//vu8//77REZG8uqrrwIQHh7+zNsxVCc1e/XEyMUFJS2N+1o67aI4tSwsadn5FSBvJ4Ty0tXVlWNzhRBCCFGivn37Mnr06GqxBic1NZW1a9dqJaCFCga1K1aswNPTkz///JOtW7dSr149IG+LiaFDh2qlY/C/s5ILXi1atFDnP3nyhHHjxlGvXj1MTU0ZOHBgoZV2z0KlUmE5eRIAjzZvIVMLGwOXpN3rb4JKRdSZU9z/o/wL7vKnIFy7dk2rZ18LIYQQ4u9j/PjxGovlqsqbb75Jhw4dtFZfhYLa2rVrs3z5cnbs2EHv3r3V6XPnzi3T8W/lkb8fXP5VcPuICRMm8Msvv7BlyxaOHDnC3bt3GTBggFbbr9G+PTW8ukB2NvcCvtJq3U+r16AhzdvlbRB9esdPpZQuzMbGBnNzc7Kzs4td3SiEEEII8XdUoaD2119/1QguV6xYQZs2bRg2bFiJ+59VhJ6ensY5w/nH1yUmJhIUFMSSJUvo1q0b7u7urF27lhMnTnDq1Cmt9sFy4kRQqUj+9VfSK/mn/fb9BwFw9fcjJN4r39nZKpUKV1dXQKYgCCGEEOKfpUKTGKZMmcLnn38OwOXLl5k0aRITJ07k0KFDTJw4kbVrtbdbwPXr17G1tcXIyAhPT08WLlxIo0aNOHfuHFlZWfTo0UNdtkWLFjRq1IiTJ0/SsWPxhxhkZGSQkZGhfp2UlARAVlaWxn5q+XSbNqVmv34k79xJwhdfYhv0baUdcFCvURMaObfh1uVQQrb/RNd3PyzX/S1btuS3337j5s2bPHjwQBYACCGEKKSozzohXnQVCmpjYmJo1aoVAFu3buW1117D39+f8+fPqxeNaUOHDh347rvvcHR0JC4ujrlz5/Lyyy8TFhZGfHw8BgYGhbaesLKyIj6+5BHOhQsXMnfu3ELpwcHBmBRz5rReq1Y02bOH9DNnOBIQQFol7i6QbVkfCCXs0H6Sa9VDz7js52ADmJqakpKSwk8//aRxaocQQggB2jmSVIjqpkJBrYGBgfp/iAMHDvDOO+8AULduXfWopzb06dNH/d8uLi506NCBxo0bs3nz5mfapHfGjBkam/8mJSXRsGFDevXqVeLI5v24uzxet56mx3+n4b/+haqStsJQFIUtt6KIv36NellpvDSwfBtYh4aGsnv3brKysujTp48cmyuEEEKDNj+rhaguKhTUdu7cmYkTJ/LSSy9x+vRpNm3aBEBkZCQNGjTQagcLql27Ng4ODkRFRdGzZ08yMzN5/PixxmhtQkJCqaOThoaG6nOfC9LX10dfX7/Y+yzGjCHp521kRkaSvm8fZq+/XuFnKU3HNwazffE8Lh34lY5vDMGoDBs053N2dubXX3/l/v37PHjwoMRj7IQQQvzzlPRZJ8SLqkJDjcuXL0dPT4+ffvqJlStXUr9+fSBvE92CuyFoW0pKCtHR0djY2ODu7o6+vj6//fabOv/atWvcunULT0/PSmlfr04d6v116sWfAV+Rm5lZKe0ANHVrh3mjJmQ9SSd0365y3WtkZKQ+fOHixYuV0T0hhBBCiGqlQkFto0aN2LVrFxcvXmTUqFHq9KVLl5bpHOOymjx5MkeOHOHmzZucOHGCN954A11dXYYOHYqZmRmjRo1SL1A7d+4c7777Lp6eniUuEntWdUcMR8/Skqy7d3n844+V1o5KR0e9E8K5vTvJKue+s/m7IFy+fFmOzRVCCCEEAA8ePMDS0pKblbz3fllMnz6djz/+WGv1VXhSaE5ODlu3bmX+/PnMnz+fbdu2aT14un37tvrI18GDB1OvXj1OnTqFxV/H1S5dupTXXnuNgQMH0qVLF6ytrfn555+12oen6RgbY/7xRwDcXxlITnJypbXl2LEzZlbWPElO4tJv+8p1r729PcbGxqSmphITE1NJPRRCCCGqxsiRI9UHM+nr62NnZ8fUqVO1evjQkSNH6NatG3Xr1sXExITmzZvj6+tLZoFfahVFYfXq1XTo0AFTU1Nq166Nh4cHAQEB6vVH+YdJjRkzRqP+0NBQVCqVOsC8efNm3sFPlpYkPxVftGnThjlz5hT5/PlXWX4tX7BgAT4+PuoTYL/77rtC9eRf9+7dA+Dw4cNF5pe2MP/SpUu8/PLLGBkZ0bBhQxYvXqyRP3nyZNatW8eNGzdK7XdZVCiojYqKomXLlrzzzjv8/PPP/PzzzwwfPhwnJyeio6O10jGAjRs3cvfuXTIyMrh9+zYbN26kWbNm6nwjIyNWrFjBw4cPSU1N5eeff34uq/1rv/EGBk2bkvP4MQ++Daq0dnR0dWn/et4isbO7fiYnu+xbsOjq6tK6dWtApiAIIYT4e+rduzdxcXHcuHGDpUuXsmrVKmbPnq2VuiMiIujduzceHh4cPXqUy5cvs2zZMgwMDDQG8UaMGMH48ePx8fHh0KFDhIaG4ufnx44dOwgODlaXMzIyIigoiOvXr5fadnJyMl9++WWp5fKfP//6sZRfkNPS0ggKCtL4lX3IkCEadcTFxeHt7Y2XlxeWlpYa91+7dk2j3NP5BSUlJdGrVy8aN27MuXPn+OKLL5gzZw6rV69WlzE3N8fb25uVK1eW+qxlUaGg9pNPPqFZs2b88ccfnD9/nvPnz3Pr1i3s7Oz45JNPtNKx6kylp4flpLzdEx6uW0dWwr1Ka6uVV3dq1KlLysMHRBw9VK5786cgXL16VWNfXiGEEKIypDx6wu1rj0h59HyOajc0NMTa2pqGDRvSv39/evTowf79+9X5ubm5LFy4EDs7O4yNjXF1deWnn/53YuejR494++23sbCwwNjYmObNm6v32g8ODsba2prFixfTunVrmjVrRu/evfnmm2/UOzBt3ryZDRs28OOPPzJz5kzatWtHkyZN8PHx4eDBg3Tt2lXdlqOjI127di3Tyasff/wxS5YsUY+Ulvb8+VedOnVKLL9nzx4MDQ01pmkaGxtr1KGrq8vBgwc1At98lpaWGmV1StgFasOGDWRmZrJmzRqcnJx46623+OSTT1iyZIlGuX79+rFx48YS+11WFQpqjxw5wuLFi6lbt646rV69eixatIgjR45opWPVnWm3bhi7uaE8ecL9FSsqrR09fX08+vYH4MzOn8jNLfsUj/r161O3bl2ysrK4evVqJfVQCCHE342iKGRl5JTrunz4NutnnmDH0gusn3mCy4dvl7sORVEq3OewsDBOnDiBgYGBOm3hwoWsX7+ewMBAwsPDmTBhAsOHD1fHKn5+fkRERLB3716uXLnCypUr1SeXWltbExcXx9GjR4ttc8OGDTg6OuLj41MoT6VSYWZmppG2aNEitm7dytmzZ0t8lqFDh2Jvb89nn31WYrnDhw9jaWmJo6MjY8eO5cGDByWWP3bsGO7u7iWWWb9+PSYmJrz5ZuHtRNu0aYONjQ09e/bk999/L7GekydP0qVLF42/D29vb65du6Zx+mz79u25ffu2Vub4VmhLL0NDw0JzPSBvd4KCnf87U6lUWE6eROzbw3m8dSt1R/pi2LRppbTl0rMPIds28yjuLtdDTuDo+XKZ++ji4sLhw4e5ePGieuRWCCGEKEl2Zi6r/1XxQSpFgaMbIzm6MbJc943+ygt9Q90yl9+1axempqZkZ2eTkZGBjo4Oy5cvB/JOD/X39+fAgQPqXZGaNm3K8ePHWbVqFV5eXty6dQs3Nzc8PDwA1PNMAQYNGsS+ffvw8vLC2tqajh070r17d9555x31nvbXr19X7zZUFm3btmXw4MFMmzZNY/emp6lUKhYtWkS/fv2YMGGCxtTLfL1792bAgAHY2dkRHR3NzJkz6dOnDydPnkRXt+j3MDY2Fltb2xL7GBQUxLBhwzTOA7CxsSEwMBAPDw8yMjL49ttveeWVVwgJCaFt27ZF1hMfH4+dnZ1GmpWVlTovf1Q5vz+xsbEa739FVGik9rXXXmP06NGEhISgKAqKonDq1CnGjBnD65W4d2t1Y+Lujmm3bpCTw59LAyqtHQMjY9z69AMgZNvmcn2TdXFxAfJOgZPNtoUQQvyddO3aldDQUEJCQvD19eXdd99l4MCBQN76n7S0NHr27Impqan6Wr9+vXr9z9ixY9m4cSNt2rRh6tSpnDhxQl23rq4ua9eu5fbt2yxevJj69evj7++Pk5MTcXFxABUaWZ4/fz7Hjh3TmG9bFG9vbzp37oyfn1+R+W+99Ravv/46zs7O9O/fn127dnHmzBkOHz5cbJ3p6ekYGRkVm3/y5EmuXLlSaOqBo6MjH374Ie7u7nTq1Ik1a9bQqVMnli5dWuIzlEV+8KyNU+4qNFL79ddf4+vri6enp3oD56ysLHx8fAgICHjmTr1ILCeMJ+XwYZL37yc9NBTjNm0qpR233v04+8s2/oyN4WboOezcPMp0X926dWnYsCF//PEHYWFhdOrUqVL6J4QQ4u9Dz0CH0V95lbl8yuMMfpxzioIxnkoFQ+d0xLR24cOOSmq3PGrUqIG9vT0Aa9aswdXVVb0QKiUlBYDdu3er99PPl38AU58+fYiNjWXPnj3s37+f7t27M27cOI1FWvXr12fEiBGMGDGCefPm4eDgQGBgIHPnzsXBwaHc0/uaNWvGBx98wPTp0wkKKnmx+aJFi/D09GTKlCml1tu0aVPMzc2Jioqie/fuRZYxNzfX+On/ad9++y1t2rQpdYoC5E0bOH78eLH51tbWJCQkaKTlvy64qP/hw4cA6p2tnkWFRmpr167Njh07iIyM5KeffuKnn34iMjKSbdu2aZzu9U9g2Lw5Zm/0ByDhyy+faT5QSYxr1sKlZ96xwSHbN5fr3vzRWtkFQQghRFmoVCr0DXXLfNWxMuGV4S1Q/RVVqHTgleEtqGNlUq56nuVYdx0dHWbOnMmnn35Keno6rVq1wtDQkFu3bmFvb69xNWzYUH2fhYUFvr6+fP/99wQEBGiszn9anTp1sLGxITU1FYBhw4YRGRnJjh07CpVVFIXExMQi65k1axaRkZGlLpBq3749AwYMYPr06aU+/+3bt0s9RdTNzY2IiIgi81JSUti8eXORC8SKEhoaWmJbnp6eHD16lKys/+3ctH//fhwdHTUWtIWFhaGvr4+Tk1OZ2i1JmUdqJ06cWGL+oUP/W5n/9Mq2vzuLjz4iaddu0s+eI+XwYWoWWO2oTR59+xP66y/cuRrB7SthNGjZukz3OTk58euvv5KQkEBCQoJ6TosQQgihLa1esqVRq7ok3kvHzNIY0zrF/8xdWQYNGsSUKVNYsWIFkydPZvLkyUyYMIHc3Fw6d+5MYmIiv//+O7Vq1cLX15dZs2bh7u6Ok5MTGRkZ7Nq1i5YtWwKwatUqQkNDeeONN2jWrBlPnjxh/fr1hIeHs2zZMgAGDx7Mtm3bGDp0KJ9++im9evXCwsKCy5cvs3TpUj7++GP69+9fqJ9WVlZMnDiRL774otRnWrBgAU5OTujp/S9kS0lJYe7cuQwcOBBra2uio6OZOnUq9vb2eHt7F1uXt7c3M2bM4NGjR4V2Sti0aRPZ2dkMHz680H0BAQHY2dnh5OTEkydP+Pbbbzl48KDGFIrly5ezbds29VzhYcOGMXfuXEaNGsW0adMICwvjq6++KjRl4dixY7z88ssac3grqsxB7YULF8pU7lm+Zb2o9G1sqDtiOA++DeLPJUsw7dIFVTGTtJ+Fad16OHn14NJvv3J6+5YyB7X5G0ZfvXqVS5cu0bNnT633TQghhDCtY1QlwWw+PT09PvroIxYvXszYsWOZN28eFhYWLFy4kBs3blC7dm3atm3LzJkzATAwMGDGjBncvHkTY2NjXn75ZfXoaf7P62PGjOHu3buYmpri5OTE9u3b8fLKm5qhUqn44YcfWL16NWvWrGHBggXo6enRvHlz3nnnnRIDzMmTJ7Ny5cpSD4twcHDgvffe0xhB1tXV5dKlS6xbt47Hjx9ja2tLr169mDdvnnpqRVGcnZ1p27Ytmzdv5sMPP9TICwoKYsCAAUX+4p6ZmcmkSZO4c+cOJiYmuLi4cODAAY0ty+7fv69xVoGZmRnBwcGMGzcOd3d3zM3NmTVrFqNHj9aoe+PGjRqHSjwLlVJZv5e/QJKSkjAzMyMxMVG9orG8chITierlTW5iIjb+/tQe8IaWe5nncXwca8Z/iKLkMnzRV1jZFV4RWZQrV66wadMmatasyYQJE0rcW04IIcTfmzY+98SLaffu3UyZMoWwsLAqjwX27t3LpEmTuHTpksZIdEVJZKMlumZmmP/17ePPr78mV4vH9BVU29oGx055W3qd3vFTKaX/p3nz5hgZGZGcnFwtznsWQgghxPPXt29fRo8ezZ07d6q6K6SmprJ27VqtBLQgQa1W1Rn+Nno2NmTHx/Noww+V1k57n7wNkSNPHefh3bL9o9TT01NPwr506VKl9U0IIYQQ1dv48eM1FstVlTfffJMOHTporT4JarVIx9AQi48/BuD+6tXkFLPq8VlZNLajadt2oCic2Vn20dr8wxciIiLIzMyslL4JIYQQQlQFCWq1zMzndQyb25ObmMiDb7+ttHY6vDEYgIijh0i6/2eZ7mnYsCG1a9cmMzOTa9euVVrfhBBCCCGeNwlqtUylq4vFX9ufPVz/X7Li4yulHVuHljRs5UxuTjbndm0rW9/+OjYXZAqCEEIIIf5eJKitBKavvIKxhztKRgZ//rWXXWVo338QAJcO7iMtqWxTHfKD2qioKPVpK0IIIYQQLzoJaiuBSqXCctIkABK3bSfj+vVKaaexixtWTe3Jzsjgwt6dZbrH3Nyc+vXroygKYWFhldIvIYQQQojnTYLaSmLi5kbNnj0hN5d7SwMqpQ2VSkWH/nlzay/8uouMtLQy3SdTEIQQQgjxdyNBbSWymDABdHVJOXiQtHPnKqUN+3YdqWvbgIy0VC7u31Ome1q3bo2Ojg53797lzz/LtshMCCGEEKI6k6C2Ehk2taP2wIEA3Pvy31TG4W0qHR313Npzu7eTlZlR6j01atTA3t4ekNFaIYQQ4p8kMzMTe3t7Tpw4UdVdITAwkH79+mmtPglqK5n5uHGojIxIv3CBlIMHK6WNFi95UdPcgrTEx4QfOlCmewpOQcjNza2UfgkhhBCVZeTIkahUKlQqFfr6+tjZ2TF16lSeaPFEzyNHjtCtWzfq1q2LiYkJzZs3x9fXV2Ovd0VRWL16NR06dMDU1JTatWvj4eFBQEAAaX9NC5wzZw4qlYoxY8Zo1B8aGopKpVKf9Hnz5s28dTmWliQnJ2uUbdOmDXPmzNFIu3LlCq+//jpmZmbUqFGDdu3acevWrRKfKTAwEDs7Ozp16gTA4cOH1e/j09eZM2c0+vX0derUqRLbunXrFn379sXExARLS0umTJlCdna2Ov+9997j/PnzHDt2rMR6ykqC2kqmb2VJXV9fAO4tWYpS4C9TW3T19GjXbwAAZ375mZwytOHo6IihoSGJiYml/g8ghBBClEXyg/vcCrtE8oP7z6W93r17ExcXx40bN1i6dCmrVq1i9uzZWqk7IiKC3r174+HhwdGjR7l8+TLLli3DwMCAnJwcdbkRI0Ywfvx4fHx8OHToEKGhofj5+bFjxw6Cg4PV5YyMjAgKCuJ6GRaPJycn8+WXX5ZYJjo6ms6dO9OiRQsOHz7MpUuX8PPzw8jIqNh7FEVh+fLljBo1Sp3WqVMn4uLiNK73338fOzs7PDw8NO4/cOCARjl3d/di28rJyaFv375kZmZy4sQJ1q1bx3fffcesWbPUZQwMDBg2bBhff/11aW9J2ShCSUxMVAAlMTGxUurPTkpSrrXvoEQ4tlAebdlSKW1kZjxR/vPB28qXg/sq4Ud+K9M927dvV2bPnq3s2LGjUvokhBCieirtcy83N1fJTE8v13Xh113Kv4e8pnw5uK/y7yGvKRd+3VXuOnJzc8v8DL6+voqPj49G2oABAxQ3Nzf165ycHMXf319p0qSJYmRkpLi4uChbCnwOP3z4UBk2bJhibm6uGBkZKfb29sqaNWsURVGUpUuXKk2aNCmxD5s2bVIAZfv27UW+h48fP1YURVFmz56tuLq6Kj179lQGDRqkLnPhwgUFUGJiYhRFUZSYmBgFUKZMmaKYmpoqCQkJ6rKurq7K7Nmz1a+HDBmiDB8+vOQ36SlnzpxRdHR0lKSkpGLLZGZmKhYWFspnn32mTsvv14ULF8rc1p49exQdHR0lPj5enbZy5UqlVq1aSkZGhjrtyJEjioGBgZKWllauZymKnnZCY1ES3Zo1qTd2DPcWfc6fXy+jVt++6Bgba7UNfQND2vZ5neMb13N6x0+07PwKKp2SB+JdXFy4cOEC4eHh9OnTB319fa32SQghxIspOyODr33frPD9iqLw25qV/LZmZbnu+2TdT+iXMNJYkrCwME6cOEHjxo3VaQsXLuT7778nMDCQ5s2bc/ToUYYPH46FhQVeXl74+fkRERHB3r17MTc3JyoqivT0dACsra2Ji4vj6NGjdOnSpcg2N2zYgKOjIz4+PoXyVCoVZmZmGmmLFi2iXbt2nD17ttAoaEFDhw5l//79fPbZZyxfvrxQfm5uLrt372bq1Kl4e3tz4cIF7OzsmDFjBv379y+23mPHjuHg4EDNmjWLLbNz504ePHjAu+++Wyjv9ddf58mTJzg4ODB16lRef/31Yus5efIkzs7OWFlZqdO8vb0ZO3Ys4eHhuLm5AeDh4UF2djYhISG88sorxdZXFjL94DmpM2wY+ra2ZN+7x8P/fl8pbbTx7ouBsQkPbt8i6lxIqeUbN25MrVq1yMjIIDIyslL6JIQQQlSWXbt2YWpqipGREc7Ozty7d48pU6YAkJGRgb+/P2vWrMHb25umTZsycuRIhg8fzqpVq4C8OZ9ubm54eHjQpEkTevTooV64NGjQIIYOHYqXlxc2Nja88cYbLF++nKSkJHX7169fx9HRscz9bdu2LYMHD2batGklllOpVCxatIjVq1cTHR1dKP/evXukpKSwaNEievfuTXBwMG+88QYDBgzgyJEjxdYbGxuLra1tiW0HBQXh7e1NgwYN1Gmmpqb8+9//ZsuWLezevZvOnTvTv39/du4sfo/8+Ph4jYAWUL+OL3DaqomJCWZmZsTGxpbYr7KQkdrnRMfAAIvx/+Lu1Gk8+OYbag96E706dbTahqFJDdp49+X09i2c3rYZe4+OqFSq4vuko4OLiwvHjx/n0qVLODk5abU/QgghXkx6hoZ8su6nMpdPfviA7yaO0djlR6Wjw8h/r6Rm3Xrlarc8unbtysqVK0lNTWXp0qXo6ekx8K9dh6KiokhLS6Nnz54a92RmZqpHCceOHcvAgQM5f/48vXr1on///uoFVLq6uqxdu5b58+dz8OBBQkJC8Pf35/PPP+f06dPY2NhUaFej+fPn07JlS4KDg7G0tCy2nLe3N507d8bPz48ffvhBIy9/gbePjw8TJkwA8haSnThxgsDAQLy8vIqsMz09vcQ5t7dv32bfvn1s3rxZI93c3JyJEyeqX7dr1467d+/yxRdflDhaW1bGxsbqRXXPQkZqn6Nar72GYYsW5CYn82D1N5XShvurPujpGxAffZ1bYRdLLZ+/C8L169dJTU2tlD4JIYR4sahUKvSNjMp81bWtT8/RH6unval0dOj5wUfUta1frnpKGogpSv4Wla6urqxZs4aQkBCCgoIA1EfB7969m9DQUPUVERHBTz/lBex9+vQhNjaWCRMmcPfuXbp3787kyZM12qhfvz4jRoxg+fLlhIeH8+TJEwIDAwFwcHDg6tWr5epzs2bN+OCDD5g+fXqpQfGiRYvYtGkTFy5c0Eg3NzdHT0+PVq1aaaS3bNmyxMXf5ubmPHr0qNj8tWvXUq9evTIFqh06dCAqKqrYfGtraxISEjTS8l9bW1trpD98+BALC4tS2yzNCxXULlq0CJVKxfjx49VpT548Ydy4cdSrVw9TU1MGDhxY6E2sLlQ6OlhOyvum8+j778m6c0frbZiY1ca5uzcAp7dvLqU0WFpaYmNjQ25uLuHh4VrvjxBCiH8G5269+GD5GgbP8ueD5Wtw7tbrubavo6PDzJkz+fTTT0lPT6dVq1YYGhpy69Yt7O3tNa6GDRuq77OwsMDX15fvv/+egIAAVq9eXWwbderUwcbGRj0INGzYMCIjI9mxY0ehsoqikJiYWGQ9s2bNIjIyko0bN5b4TO3bt2fAgAFMnz5dI93AwIB27dpx7do1jfTIyEiNOcVPc3Nz4+rVq0UG04qisHbtWt55550yrbEJDQ3Fxsam2HxPT08uX77MvXv31Gn79++nVq1aGsF4dHQ0T548UY+eP4sXJqg9c+YMq1atUo8s5pswYQK//PILW7Zs4ciRI9y9e5cBAwZUUS9LV6NzZ0w6dEDJyuLPZYUnf2uDR7830NHV5VbYJeKuXyu1vBybK4QQQhtq1jOnoZMLNeuZV0n7gwYNQldXlxUrVlCzZk0mT57MhAkTWLduHdHR0Zw/f55ly5axbt06IC+43LFjB1FRUYSHh7Nr1y5atmwJwKpVqxg7dizBwcFER0cTHh7OtGnTCA8PV8+7HTx4MEOGDGHo0KH4+/tz9uxZYmNj2bVrFz169ODQoUNF9tPKyoqJEyeWaSurBQsWcPDgwUIB7JQpU9i0aRPffPMNUVFRLF++nF9++YX/+7//K7aurl27kpKSUuQg1sGDB4mJieH9998vlLdu3Tp+/PFHrl69ytWrV9VzlT/++GN1mW3bttGiRQv16169etGqVStGjBjBxYsX2bdvH59++injxo3DsMA0k2PHjtG0aVOaNWtW6ntRqmfeP+E5SE5OVpo3b67s379f8fLyUv71r38piqIojx8/VvT19TW257hy5YoCKCdPnixz/ZW9pdfT0i5dUiIcWygRLVoq6VevVUobe1csVb4c3FfZtnheqWWTkpKUOXPmKLNnz1bu379fKf0RQghRfTzvz73KUNSWXoqiKAsXLlQsLCyUlJQUJTc3VwkICFAcHR0VfX19xcLCQvH29laOHDmiKIqizJs3T2nZsqVibGys1K1bV/Hx8VFu3LihKIqinD9/Xhk+fLhiZ2enGBoaKvXq1VO6dOmi7Ny5U6O9nJwcZeXKlUq7du0UExMTpVatWoq7u7vy1Vdfqbepyt/Sq6DExETF3Ny8yC29nt46a/To0QqgsaWXoihKUFCQYm9vrxgZGSmurq5Fbi32tMGDByvTp08vlD506FClU6dORd7z3XffKS1btlQ/X/v27TViL0VRlLVr1ypPh5U3b95U+vTpoxgbGyvm5ubKpEmTlKysLI0yvXr1UhYuXFhqv8tCpSiVcHarlvn6+lK3bl2WLl3KK6+8Qps2bQgICODgwYN0796dR48eUbt2bXX5xo0bM378ePXk6adlZGSQkfG/42STkpJo2LAh9+/fp1atWpX9OADET5pMSnAwJl26YLtC+yO2D+/e5vtpn4Ci8PbCAOo1LP7nCIAff/yRGzdu8PLLLxe7dYkQQoi/h6SkJMzNzUlMTHxun3uierh06RI9e/YkOjoaU1PTKu1LeHg43bp1IzIystD2ZxVR7Xc/2LhxI+fPn1cf1VZQfHw8BgYGGgEt5A3rF9wu4mkLFy5k7ty5hdKDg4MxMTF55j6Xhb6rC00OHCDt6FEOLV9OetOmWm+jRoMmpP4Rw85Vy7Hq1LXEsvmno4SEhJCcnFzuyfpCCCFeHNpYaS5eTC4uLnz++efExMTg7OxcpX2Ji4tj/fr1WglooZoHtX/88Qf/+te/2L9/f4lbUJTXjBkzNLamyB+p7dWr13P9xnrv1i2SNm3G/sQJGowbp/VA8l5LRzb6TSbl1g0GfDIJM0vrYstmZmYSEBBAZmYmrq6uGvvTCSGE+HspuNeq+OcZOXJkVXcBgB49emi1vmod1J47d4579+7Rtm1bdVpOTg5Hjx5l+fLl7Nu3j8zMTB4/fqwxWpuQkFBou4iCDA0NNSYp59PX13+up2pZffQRyb/sIuNyGE8OHaaWt3ZXitZ3aEFjFzdiL10gdO9Oerw/rtiy+vr6tGrViosXLxIeHo6dnZ1W+yKEEKL6kBMkxd9Rtd79oHv37ly+fFljfzkPDw/efvtt9X/r6+vz22+/qe+5du0at27dwtPTswp7XjZ6FhbU++vb0p9Ll6JkZWm9jQ5vDAYg7PABUh49LLFs/i4I4eHhZGdna70vQgghhBCVpVoHtTVr1qR169YaV40aNahXrx6tW7fGzMyMUaNGMXHiRA4dOsS5c+d499138fT0pGPHjlXd/TKp+9676NatS+bNmzze+rPW62/QsjW2Di3Jycri3O7tJZa1s7OjZs2apKenl7ihshBCCCFEdVOtg9qyWLp0Ka+99hoDBw6kS5cuWFtb8/PP2g8OK4uuqSnmY8cC8OeK5eRqefK+SqWiff9BAFzcv5cnf52wUhQdHR31pPGLF0s/jUwIIYQQorp44YLaw4cPExAQoH5tZGTEihUrePjwIampqfz8888lzqetjuoMGYx+gwbk/Hmfh+vXa73+pm3bYdGoCVlP0rmw75cSy+ZPQYiMjCQ9PV3rfRFCCCGEqAwvXFD7d6QyMMDir6N/H3zzLdklnMtcofoLjNae3/sLWU+eFFvW2toaS0tLcnJyiIiI0Go/hBBCCCEqiwS11UStV/tg2Koluamp3F+5Uuv1O3TsTG0rG54kJ3Hpt30llnV1dQVkCoIQQgghXhwS1FYTKh0dLCdNAuDRjxvJvH1bq/Xr6OrSzmcgAGd3/Ux2CTsttG7dGoBbt27xSMujxkIIIYSoOg8ePMDS0pKbN29WdVeYPn06H3/8sdbqk6C2GjF96SVqdOoEWVn8+dXXWq+/VZfumNapS8rDB1w5dqjYcmZmZup9ai9fvqz1fgghhBDPauTIkahUKlQqFfr6+tjZ2TF16lSelDDFrryOHDlCt27dqFu3LiYmJjRv3hxfX18yMzPVZRRFYfXq1XTo0AFTU1Nq166Nh4cHAQEB6pPb5syZg0qlYsyYMRr1h4aGolKp1AHmzZs3UalUWFpakpycrFG2TZs2zJkzR/06/9mfvr744osSn2nBggX4+PjQpEkTAL777rti67p37x6Qt56pqPySTm+FvCN5X375ZYyMjGjYsCGLFy/WyJ88eTLr1q3jxo0bJdZTVhLUVjMWk/JOOkv65ReeaHlOq56+Pu6vvQHA6R1byM3NKbZswSkIiqJotR9CCCH+nrITM3gS/ZjsxIzn0l7v3r2Ji4vjxo0bLF26lFWrVjF79myt1B0REUHv3r3x8PDg6NGjXL58mWXLlmFgYKA+Wh5gxIgRjB8/Hh8fHw4dOkRoaCh+fn7s2LGD4OBgdTkjIyOCgoK4fv16qW0nJyfz5ZdfllgmLi5O41qzZg0qlYqBAwcWe09aWhpBQUGMGjVKnTZkyJBCdXl7e+Pl5YWlpaXG/deuXdMo93R+QUlJSfTq1YvGjRtz7tw5vvjiC+bMmcPq1avVZczNzfH29mallqZdSlBbzRg7OVGrb18A7i1ZqvX6XXr0xsi0Jo/j44g89Xux5Vq2bImenh4PHjzg7t27Wu+HEEKI6ktRFHIzc8p1JZ+8S/yi09z/5jLxi06TfPJuueso7yCKoaEh1tbWNGzYkP79+9OjRw/279+vzs/NzWXhwoXY2dlhbGyMq6srP/30kzr/0aNHvP3221hYWGBsbEzz5s1Zu3YtAMHBwVhbW7N48WJat25Ns2bN6N27N9988w3GxsYAbN68mQ0bNvDjjz8yc+ZM2rVrR5MmTfDx8eHgwYN07dpV3ZajoyNdu3bl//2//1fqc3388ccsWbJEPVJaFGtra41rx44ddO3alaZNmxZ7z549ezA0NNTYy9/Y2FijHl1dXQ4ePKgR+OaztLTUKKujU3wYuWHDBjIzM1mzZg1OTk689dZbfPLJJyxZskSjXL9+/di4cWNJb0eZVetjcv+pLMb/i6TgYFKPHyf15ElqaPF0NAMjY9x69+PkTz9wevsWHD1fRqVSFSpnaGhIixYtCAsL49KlS9SvX19rfRBCCFG9KVm53J114hkqgMQd0STuiC7XbbafdUJloFuhJsPCwjhx4gSNGzdWpy1cuJDvv/+ewMBAmjdvztGjRxk+fDgWFhZ4eXnh5+dHREQEe/fuxdzcnKioKPV2ltbW1sTFxXH06FG6dOlSZJsbNmzA0dERHx+fQnkqlQozMzONtEWLFtGuXTvOnj2Lh4dHsc8ydOhQ9u/fz2effcby5ctLffaEhAR2797NunXrSix37Ngx3N3dSyyzfv16TExMePPNNwvltWnThoyMDFq3bs2cOXN46aWXiq3n5MmTdOnSBQMDA3Wat7c3n3/+OY8ePaJOnToAtG/fntu3b3Pz5k31lIiKkpHaasigYUPqDBkCwL0v/42Sm6vV+t369EPf0Ig/Y2OICT1bbLn8KQiXL1/W+KlFCCGEqA527dqFqakpRkZGODs7c+/ePaZMmQJARkYG/v7+rFmzBm9vb5o2bcrIkSMZPnw4q1atAvIWRLu5ueHh4UGTJk3o0aMH/fr1A2DQoEEMHToULy8vbGxseOONN1i+fDlJSUnq9q9fv46jo2OZ+9u2bVsGDx7MtGnTSiynUqlYtGgRq1evJjq69C8G69ato2bNmgwYMKDEcrGxsdja2pZYJigoiGHDhqlHowFsbGwIDAxk69atbN26lYYNG/LKK69w/vz5YuuJj4/HyspKIy3/dcG5uPn9iY2NLbFfZSEjtdWU+dgxJP78M0/Cw0n+9Vdqvfqq1uo2Nq2Ja69XOfvLz4Rs20JTt3ZFlmvatCk1atQgNTWV6OhoHBwctNYHIYQQ1ZdKXwfbzzqVuXxOYgYJS85BwdkDKrCa6I6umWG52i2Prl27snLlSlJTU1m6dCl6enrqOaVRUVGkpaXRs2dPjXsyMzNxc3MDYOzYsQwcOJDz58/Tq1cv+vfvT6dOec+tq6vL2rVrmT9/PgcPHiQkJAR/f38+//xzTp8+jY2NTYXWnMyfP5+WLVsSHBxc4pxUb29vOnfujJ+fHz/88EOJda5Zs4a3334bIyOjEsulp6eXWObkyZNcuXKF//73vxrpjo6OGsF7p06diI6OZunSpYXKlld+8JymhRNVZaS2mtKrV4+6o94D4F7AVygFVlpqg/urPujq6XH3WgS3r4QVWUZXV1e9vdelS5e02r4QQojqS6VSoWOgW+ZL38KEOgOaQ/5sNhXUGdAcfQuTctVT1HS4ktSoUQN7e3tcXV1Zs2YNISEhBAUFAZDy17Hwu3fvJjQ0VH1FRESo59X26dOH2NhYJkyYwN27d+nevTuTJ0/WaKN+/fqMGDGC5cuXEx4ezpMnTwgMDATAwcGBq1evlqvPzZo144MPPmD69OmlBsWLFi1i06ZNXLhwodgyx44d49q1a7z//vultm1ubl7iVp3ffvstbdq0KXWKAuRNG4iKiio239ramoSEBI20/NcFT359+PAhABYWFqW2WRoJaquxeiNHomtuTtatWzzaskWrdZvWrYfTKz0ACNlefN35UxCuXr2q1W1ShBBC/L3UaGeN9fT2mH/gjPX09tRo93yPrNfR0WHmzJl8+umnpKen06pVKwwNDbl16xb29vYaV8OGDdX3WVhY4Ovry/fff09AQIDG6vyn1alTBxsbG1JTUwEYNmwYkZGR7Nixo1BZRVFITEwssp5Zs2YRGRlZ6gKp9u3bM2DAAKZPn15smaCgINzd3dWf1yVxc3Mr9rTQlJQUNm/eXOQCsaKEhoZiY2NTbL6npydHjx4lq8C++Pv378fR0VE9nxby5kLr6+vj5ORUpnZLIkFtNaZTowYW4/4PgPv/WUlOSqpW62/XbyAqlQ43Q8+REFP0nB0bGxvMzc3Jzs7mypUrWm1fCCHE34uemSFGzWqjV44pB9o0aNAgdHV1WbFiBTVr1mTy5MlMmDCBdevWER0dzfnz51m2bJl6QdWsWbPYsWMHUVFRhIeHs2vXLlq2bAnAqlWrGDt2LMHBwURHRxMeHs60adMIDw9Xz7sdPHgwQ4YMYejQofj7+3P27FliY2PZtWsXPXr04NChoveEt7KyYuLEiXz9del70i9YsICDBw9y7dq1QnlJSUls2bKlTKO0kDelITw8vMjR2k2bNpGdnc3w4cML5QUEBKjfp7CwMMaPH8/BgwcZN26cuszy5cvp3r27+vWwYcMwMDBg1KhRhIeHs2nTJr766ismTpyoUfexY8d4+eWXNebwVpQEtdVc7TffRL9xI3IePODhd99pt25rGxw7vQzA6WJGa1UqFS4uLoBMQRBCCFG96enp8dFHH7F48WJSU1OZN28efn5+LFy4kJYtW9K7d292796tPmDIwMCAGTNm4OLiQpcuXdDV1VWPnrZv356UlBTGjBmDk5MTXl5enDp1iu3bt+Pl5QXkfUb+8MMPLFmyRJ3u4uLCnDlz8PHxwdvbu9i+Tp48GVNT01KfycHBgffee6/IX0s3btyIoigMHTq0TO+Ps7Mzbdu2ZfPmzYXygoKCGDBgALVr1y6Ul5mZyaRJk3B2dsbLy4uLFy9y4MABjSD2/v37GovazMzMCA4OJiYmBnd3dyZNmsSsWbMYPXp0oWf44IMPytT/0qgU2VmfpKQkzMzMSExMpFatWlXdnUKSfv2VO+MnoGNiQrPgfeiZm2ut7j9v3WT9lI9ApeLdJSupa9ugUJnHjx8TEBAAwIQJEwptUSKEEOLFUt0/90Tl2b17N1OmTCEsLKzEfWafh7179zJp0iQuXbqEnt6z710gI7UvgJre3hg5O5Oblsb9lYFarduiUROaurcHReHMzq1Flqldu7Z63z85NlcIIYR4cfXt25fRo0dz586dqu4KqamprF27VisBLUhQ+0JQqVRYTpoEwKNNm8i8dUur9XfoPxiAiKMHSbpf9Okl+VMQ5NhcIYQQ4sU2fvx4jcVyVeXNN9+kQ4cOWqtPgtoXRI2OHajx8suQnc2ff00F0BZbhxY0bOVMbk4OZ3dtK7JMq1at0NXV5c8//9TYNFkIIYQQojqQoPYFYjlpIqhUJO3ZS/rloveWraj2b+SN1l7+LZi0pMJbkBgbG6s3XpYFY0IIIYSobiSofYEYtWhBrX6vAXDv3//W6jSAxs5tsGranOzMDM7v2VlkmfwpCHJsrhBCCCGqGwlqXzAWn/wLlb4+aadOkfr7Ca3Vq1Kp6NB/EACh+3aRUcRxdfb29hgbG5OSkkJMTIzW2hZCCCGEeFYS1L5gDBrUp86wYcBfo7W5uVqr275dR+raNiAjLZWL+/cUytfT05Njc4UQQghRLUlQ+wKqN+ZDdExNybhyhaTdhYPPilLp6ND+r9Hac7u3k5WZUahM/hSEK1eukJFROF8IIYQQoipIUPsC0qtTh3p/HYn3Z0AAuZmZWqu7xUte1LKwJC3xMeGHDhTKb9CgAXXr1iUrK4urV69qrV0hhBBCiGchQe0Lqu47I9CzsCDrzh0eb9yktXp19fTw6DcAgDO/bCUnO1sjX47NFUIIIV5cmZmZ2Nvbc+KE9tblVFRgYCD9+vXTWn0S1L6gdExMMP/oIwDur1xJTkqK1upu3bUnJma1SfrzHtdOHC2Unx/U3rhxg+TkZK21K4QQQpTVyJEjUalUqFQq9PX1sbOzY+rUqTx58kRrbRw5coRu3bpRt25dTExMaN68Ob6+vmQW+IVUURRWr15Nhw4dMDU1pXbt2nh4eBAQEEDaX4uu58yZg0qlYsyYMRr1h4aGolKpuHnzJgA3b97MO3DJ0rLQ52ubNm2YM2eO+nVKSgofffQRDRo0wNjYmFatWhEYWPqpo4GBgdjZ2dGpUycADh8+rH4fn77OnDmj0a+nr1OnTpXY1q1bt+jbty8mJiZYWloyZcoUsgsMlr333nucP3+eY8eOldrvspCg9gVWe+AADJo0IefRIx4EBWmtXn0DQ9q+6gNAyPYthRaj1a1bl4YNG6IoihybK4QQQi0xMZGYmBgSEwvvd14ZevfuTVxcHDdu3GDp0qWsWrWK2bNna6XuiIgIevfujYeHB0ePHuXy5cssW7YMAwMDjW0tR4wYwfjx4/Hx8eHQoUOEhobi5+fHjh07CA4OVpczMjIiKCiI69evl9p2cnIyX375ZYllJk6cyK+//sr333/PlStXGD9+PB999BE7dxa9LSfkBeDLly9n1KhR6rROnToRFxencb3//vvY2dnh4eGhcf+BAwc0yrm7uxfbVk5ODn379iUzM5MTJ06wbt06vvvuO2bNmqUuY2BgwLBhw/j6669Le0vKRqnG/vOf/yjOzs5KzZo1lZo1ayodO3ZU9uzZo85PT09X/u///k+pW7euUqNGDWXAgAFKfHx8udtJTExUACUxMVGb3X8uEvftUyIcWyhX2rgpmQkJWqv3SWqKsmzkYOXLwX2VyJDfC+WfPn1amT17trJy5UqttSmEEOL5KO1zLzc3V8nIyCjXFRISosyZM0eZPXu2MmfOHCUkJKTcdeTm5pb5GXx9fRUfHx+NtAEDBihubm7q1zk5OYq/v7/SpEkTxcjISHFxcVG2bNmizn/48KEybNgwxdzcXDEyMlLs7e2VNWvWKIqiKEuXLlWaNGlSYh82bdqkAMr27duLfA8fP36sKIqizJ49W3F1dVV69uypDBo0SF3mwoULCqDExMQoiqIoMTExCqBMmTJFMTU1VRIKfK67uroqs2fPVr92cnJSPvvsM40227Ztq/y///f/iu3vmTNnFB0dHSUpKanYMpmZmYqFhYVG3fn9unDhQrH3PW3Pnj2Kjo6ORly2cuVKpVatWkpGRoY67ciRI4qBgYGSlpZW5rqLo6ed0LhyNGjQgEWLFtG8eXMURWHdunX4+Phw4cIFnJycmDBhArt372bLli2YmZnx0UcfMWDAAH7//feq7vpzU7NnT4xdXUm/eJH7//kPNgV+mngWhiY1aOPdl5Btmzm9fQv27TxRqVTqfCcnJ/bu3Ut8fDwJCQlYWVlppV0hhBBVLysrC39//wrfrygKe/bsYc+e8u3QM3PmTAwMDCrUZlhYGCdOnKBx48bqtIULF/L9998TGBhI8+bNOXr0KMOHD8fCwgIvLy/8/PyIiIhg7969mJubExUVRXp6OgDW1tbExcVx9OhRunTpUmSbGzZswNHRER8fn0J5KpUKMzMzjbRFixbRrl07zp49W2gUtKChQ4eyf/9+PvvsM5YvX15kmU6dOrFz507ee+89bG1tOXz4MJGRkSxdurTYeo8dO4aDgwM1a9YstszOnTt58OAB7777bqG8119/nSdPnuDg4MDUqVN5/fXXi63n5MmTODs7a8QH3t7ejB07lvDwcNzc3ADw8PAgOzubkJAQXnnllWLrK4tqPf2gX79+vPrqqzRv3hwHBwcWLFiAqakpp06dIjExkaCgIJYsWUK3bt1wd3dn7dq1nDhxotQ5Hn8nKpUKy8mTAHi85ScytHgoQts+r6NnYEh89HVuXb6okWdiYoKDgwMgC8aEEEJUjV27dmFqaoqRkRHOzs7cu3ePKVOmAJCRkYG/vz9r1qzB29ubpk2bMnLkSIYPH86qVauAvDmfbm5ueHh40KRJE3r06KFeuDRo0CCGDh2Kl5cXNjY2vPHGGyxfvpykpCR1+9evX1cfIV8Wbdu2ZfDgwUybNq3EciqVikWLFrF69Wqio6OLLLNs2TJatWpFgwYNMDAwoHfv3qxYsaLYABwgNjYWW1vbEtsOCgrC29ubBg0aqNNMTU3597//zZYtW9i9ezedO3emf//+JU51iI+PLzTglf86Pj5enWZiYoKZmRmxsbEl9qssqvVIbUE5OTls2bKF1NRUPD09OXfuHFlZWfTo0UNdpkWLFjRq1IiTJ0/SsWPHYuvKyMjQ2GM1/x9oVlYWWVlZlfcQlUS/TRtMvLqQduQoCUuWYrPk39qp16QGTq/04GLwbk5t24htSyeNfCcnJ65evcqlS5fw8vLSGMkVQghRfZX2Waevr8/MmTPLXF9SUhIrVqzQOL5dpVIxbtw4atWqVeZ69PX1y1wWoGvXrqxcuZLU1FSWLl2Knp4eAwcOBCAqKoq0tDR69uypcU9mZqZ6lHDs2LEMHDiQ8+fP06tXL/r3769eQKWrq8vatWuZP38+Bw8eJCQkBH9/fz7//HNOnz6NjY1NhY6rnz9/Pi1btiQ4OBhLS8tiy3l7e9O5c2f8/Pz44YcfCuUvW7aMU6dOsXPnTho3bszRo0cZN24ctra2GrFRQenp6RgZGRXb5u3bt9m3bx+bN2/WSDc3N2fixInq1+3atePu3bt88cUXJY7WlpWxsbF6Ud2zqPZB7eXLl/H09OTJkyeYmpqybds2WrVqRWhoKAYGBtSuXVujvJWVlcY3gKIsXLiQuXPnFkoPDg7GxMREm91/bgzatqXx0WOk7t/PwcBAnjRqpJV6s4xrgUrF7Ygwfl6/FiPz/33rys3NRVdXl+TkZDZv3lzizxlCCCGqj9ICCJVKVa5pAObm5vTr149ffvkFRVFQqVT069cPc3PzZ+1qiWrUqIG9vT0Aa9aswdXVlaCgIEaNGkXKX7sC7d69m/r162vcZ2hoCECfPn2IjY1lz5497N+/n+7duzNu3DiNRVr169dnxIgRjBgxgnnz5uHg4EBgYCBz587FwcGh3Hu2N2vWjA8++IDp06cTVMoi70WLFuHp6akefc6Xnp7OzJkz2bZtG3379gXydiYKDQ3lyy+/LDaoNTc3L3GB99q1a6lXr16ZAtUOHTqwf//+YvOtra05ffq0RlpCQoI6r6CHDx9iYWFRapulqfZBraOjI6GhoSQmJvLTTz/h6+vLkSNHnqnOGTNmaHzjSEpKomHDhvTq1atc3yirm4QbMSTv2IFDyGnqf/ih1kZODzxKIOLoQfTvx/HqO5pzbHR0dLhw4QImJia8+uqrWmnv/7d33/FxVPfexz8z27SrVbW6LUtyr7JxxbhgY2xDKCEkoYTEDgSSG8oNNcEkgRAuMQ4PNfQkYEhCIIUOxgYbd+PecS+Si2RJVtf2mXn+2NVKsiQ39fXvzWvZ2aln1x7vd86eOUcIIUTbqv8TemsZMWIEvXv3prS0lMTExEbtSduaqqo89NBD3HvvvfzgBz9g0KBB2Gw28vPzufjii5vdLjk5mVmzZjFr1iwmTpzIAw880GzPAwkJCaSnp1NTUwPAD37wA2644QY+/PDDRu1qDcOgsrKyyc/h4Ycfpnfv3rzzzjunfE9jxozh2muv5cEHH2wwv/aXZVVt2IrUZDKhn9RjUX0XXHABL7/8cvjC4+TyvvHGG8ycOfOMasw3b95Menp6s8vHjRvH448/TlFRUbhG+osvviA2NpZBgwaF19u/fz8ejydce94SnT7UWq3W8FXYyJEjWbduHc899xzXX389Pp+P8vLyBrW1x48fb3QFcDKbzRa+SqvPYrGc9U8fnUnqL/6X6vnz8axfj2/1apynOInPxtjvXMc3y7/i4MZ1VBQcJalndnjZ8OHD2bRpE7t27eLKK6885wb+Qggh2k9bfdfFxcW1e5it7/vf/z4PPPAAL774Ivfffz/3338/99xzD7quM2HCBCoqKli5ciWxsbHMmjWLhx9+mJEjRzJ48GC8Xi+ffPIJAwcOBODVV19l8+bNfOc736F37954PB7eeustduzYwZ/+9CcArrvuOt5//31uvPFGfvOb3zB9+nSSk5PZtm0bzzzzDHfddRfXXHNNo3KmpqZy77338uSTT572PT3++OMMHjwYs7kussXGxnLxxRfzwAMPYLfbycrKYunSpbz11ls8/fTTze5rypQpVFdXs2PHDoYMGdJg2eLFizl48CC3hkYsre/NN9/EarWGg+d7773H66+/zl/+8pfwOu+//z6zZ88O11xPnz6dQYMG8aMf/Yg//vGPFBYW8pvf/IY77rijQQZbvnw5vXr1onfv3qf9LE6nU98o1hRd1/F6vYwcORKLxcKiRYvCy3bv3k1+fj7jxo3rwBJ2HEtGBgk//CEARU89jVGvH72WSMzoQb8xwTZGaz/8T4NlmZmZxMfH4/P52L17d6scTwghhDgXZrOZO++8kz/+8Y/U1NTw2GOP8dvf/pY5c+YwcOBALrvsMj799FNycnKAYMXZ7Nmzyc3NZdKkSZhMpnDt6ZgxY6iuruZ//ud/GDx4MBdffDFff/01H3zwQbjmV1EU3n77bZ5++unw/NzcXH73u9/x7W9/mxkzZjRb1vvvvx+n03na99SvXz9uueWWRoNKvPPOO4wePZqbbrqJQYMG8cQTT/D44483GuChvm7duvGd73yHf/zjH42W/fWvf+Wiiy5iwIABTW772GOPMXLkSMaOHcuHH37Iu+++26CHhIqKigY5wGQy8cknn2AymRg3bhw//OEPmTlzJr///e8b7Pef//wnt91222k/hzOhGOfSyrmdzJ49m8svv5yePXtSVVXF22+/zdy5c1mwYAHTpk3j5z//OZ999hnz5s0jNjaWu+66C+Csh36r/XmgoqKiSzc/ANDKy9k3fQZ6ZSXpT8whvokrxHNx/MA+/j77bhRF5ZbnXiM+ta42fPHixSxbtoy+ffty0003tcrxhBBCtJ1I+t4TZ2fr1q1MmzaN/fv3n1Gobks7duzgkksuYc+ePa1Sw9+pa2qLioqYOXMm/fv3Z+rUqaxbty4caAGeeeYZrrzySr773e8yadIk0tLSeO+99zq41B3LFB9P0k+DVzzFzz+PXq+Xh5ZI7dWH7GEjMAyd9R//t8Gy2mFz9+3bF26YL4QQQojOJzc3l7lz53KwFbsAPVcFBQW89dZbrdZkpVPX1LaXSLti1T0e9s+4jMDx46T86ld0u/nHrbLfI99s591HH8RkNnPrC6/jTEgML3vttdc4duwYl1122Sm7UxNCCNHxIu17Twjo5DW14tyoUVEk/2+wKcaJV15Ba6W7XLsPHExG/0FogQAbPv2gwbJhw4YBMhCDEEIIITqGhNoIFfftb2Pt0xutooITfzl1P3hnSlEUxl7zfQC2fDEfd3VVeNmQIUNQFIVjx45RUlLSKscTQgghhDhTEmojlGI2kxLqi7f0rbfwhzo8bqmcC0aR3DMbv8fN5s8/Cc+v3wH2li1bmttcCCGEEKJNSKiNYM4pU7CPGIHh8VDywgutsk9FURgTqq3dOP8jfB53eFltE4Rt27adsvNnIYQQQojWJqE2gimKQsr99wFQ/t/38O7f3yr77TduAvFp6Xiqq9i2aEF4fv/+/bFarZSXl3P48OFWOZYQQgghxJmQUBvhHCNG4Jw6FXSdomeeaZV9qqqJ0Vd/D4D1H79HwO8HgiPU1A59J00QhBBCCNGeJNSeB1LuuRtUleovF+HauKlV9jlo0iU4ExKpLivlm2WLw/NrmyDs2LEDfyjsCiGEEEK0NQm15wFbnz7Ef/daAIqeeorW6JrYbLEw6qrgPtd99B90PTgkb1ZWFrGxsXi9Xvbu3dvi4wghhBBCnAkJteeJpDvvRLHZcG/YQPVXS1pln0OnziDKGUN5YQF7vl4JgKqqDB06FJAmCEIIIYRoPxJqzxOW1FQSZ84EoOjppzA0rcX7tEbZGXH51QCsff9f4Rrg2iYIe/fuxeVytfg4QgghhBCnI6H2PNLttltR4+Lw7dtPxQcftso+h192JZYoO8X5hzi4aT0AKSkppKWloes6O3bsaJXjCCGEEEKcioTa84gpNpakn/0MgOI//Qnd42nxPu3OGIZNuxyANfVqa3NzcwFpgiCEEEKI9iGh9jyTcNMPMKenEygspOzvf2+VfY684hpMZjPH9uzk6M5gzezQoUNRFIUjR45QWlraKscRQgghhGiOhNrzjGqzkfy//wtAyWt/Risvb/E+nQmJDJkyDYA1H/wLgJiYGHr16gXA1q1bW3wMIYQQQohTkVB7Hoq7+ips/fqhV1ZS8uc/t8o+R131XRRF5dCWjRw/sA9o2AShNboRE0IIIYRojoTa85BiMpFy370AlP3t7/gLClq8z/jUNAaMnwTA2g/+DcDAgQOxWCyUlZVx5MiRFh9DCCGEEKI5EmrPU9GTJuEYPRrD56P4Ty+0yj7HfDs4dO6etasoPXYEq9XKwIEDAWmCIIQQQoi2JaH2PKUoCin33wdAxQcf4Nmzp8X7TOqZTe9RY8EwWPvhf4C6Jgjbt28nEAi0+BhCCCGEEE2RUHsesw8bRsz06aDrFD/zbKvsc8y3vw/AzuVfUVlSRK9evXA6nbjdblauXElFRUWrHEcIIYQQoj4Jtee55LvvBpOJ6q++wrV+fYv3l9FvAJmDc9E1jfWfvI+qqqSkpADw1Vdf8eyzz7Jx48YWH0cIIYQQoj4Jtec5W68c4r8XbAtb9OT/a5VeCsZecx0A2xYt5PiRwxw8eDC8zDAMPv74Y6mxFUIIIUSrklBbz/Ga4x1dhA6RdMftKHY77i1bqPryyxbvr+fQYaT17kvA52XNgs8aBWXDMPjb3/7WIOwKIYQQQrSEhNp6vvPRd3hv73sdXYx2Z0lJIXHWTACKn34Go4U3dCmKwphrgm1rD65ehqIojdYpKSnhzTff5K233uLo0aMtOp4QQgghhITaenRD59HVj1JYU9jRRWl33W69FVN8PL6DByl/r+XBvs+oC0nsnkmgqoLBPdLDwVZRFKZPn87o0aNRVZUDBw7w5z//mXfffZfi4uIWH1cIIYQQ5yfFkKGeqKysJC4ujoEvD8RkN/GTIT/hFyN+0WQNYyQrfestjv9hDubkZHovXIBqt7dofzuWLuLzl57BERfPdX94hsqqahITE4mLiwOgrKyMJUuWsGXLFiAYeIcNG8bkyZOJj49v6dsRQgjRjNrvvYqKCmJjYzu6OEK0CqmpbcJft/+VH3/+Y3aX7u7oorSr+BtuwNK9O4HiYkrf+luL9zdg/MXEJqfgqignf8MacnJywoEWICEhge985zvcfvvtDBgwAMMw2Lx5M88//zyfffYZ1dXVLS6DEEIIIc4PnTrUzpkzh9GjRxMTE0NKSgrXXHMNu3c3DJoej4c77riDbt264XQ6+e53v8vx4+d2w5eqqEzLmobdbGdj0Uau/+R65q6dS5WvqjXeTqenWq0k3/0LAE78+c8EyspatD+T2czoq74LwJr3/8WhLZuoOlHSaL2UlBRuuOEGbr31VnJyctB1nbVr1/Lcc8+xaNEi3G53i8ohhBBCiMjXqZsfXHbZZdxwww2MHj2aQCDAQw89xPbt2/nmm2+Ijo4G4Oc//zmffvop8+bNIy4ujjvvvBNVVVm5cuUZH6f2Z5g9x/bQN70vhTWFPLnuSRbmLQSgW1Q37ht1H1f2ujLimyQYus7Ba7+Ld9cuEmfNInX2gy3an9/n5ZWf/hBfKJgqisK0n97F0EumN7vNgQMHWLRoUfgGsqioKCZMmMCYMWOwWq0tKo8QQghpfiAiU6cOtScrLi4mJSWFpUuXMmnSJCoqKkhOTubtt9/me6G+Vnft2sXAgQNZvXo1F1544Rntt7mTe9WxVcxZM4dDlYcAGJEygl9f+Gv6JfRr9ffWmVQvX8Hh225DsVjoNX8+1h7dz3lfVSdKeO32m4G6v2aKqnLbC68T0y2p2e0Mw2DXrl0sXrw4fAOZ0+nk4osv5oILLsBsNp9zmYQQ4nwnoVZEoi4Vavft20ffvn3Ztm0bQ4YMYfHixUydOpWysrIGNxZlZWVx9913c8899zS5H6/Xi9frDb+urKwkMzOTkpKSRie3T/Pxj13/4M/b/4xH82BSTNzQ7wZ+lvsznBZnm7zPjmYYBsduuw33mrXEXHUlqX/4wznv6/A323j/Dw83mp8zYjTjb5hJYkaPU26v6zo7duxg6dKl4QEb4uPjmTRpEoMHD0ZVO3ULGiGE6JQqKytJSkqSUCsiSpep7tJ1nbvvvpvx48czZMgQAAoLC7FarY3ulE9NTaWwsPluuebMmcOjjz7aaP7ChQtxOByN5qeSyp3RdzLfPZ8d/h38Y/c/+HDPh1xuv5xcS25ENkmwjRlD1pq1VH7yKdt69cKXkXFO+wm4qgGF+jW1AAc3ruPgxnU40nsQ138IjvQep/wcs7OzOXHiBIWFhZSXl/PRRx+xcOFC0tPTiYuLi8g/AyGEaCsul6ujiyBEq+syofaOO+5g+/btrFixosX7mj17Nvfee2/4dW1N7fTp0095xfoDfsCqY6v444Y/kl+Vz79d/+ZAygEeHPUgveN7t7hcnU3h3n1Uf/45gzZsJOPWW895PzuSEln8+ssYuo6iqgy/7ErKCws4uGk9roIjuAqOEJ+WwbDpVzBw4hSsp+hKzOfzsX79elavXo3H4+HgwYN0796dyZMnk52dfc5lFEKI80llZWVHF0GIVtclmh/ceeedfPjhhyxbtoycnJzw/HNtfnCys21b5NN8vLnjTV7b+hoezYNZMXPTwJv4+fCfE22JPuv311n58vLYf8WVEAjQc948oi8ce877qjpRQnnhMeLTMsJtacuPF7J5wcdsW/wFPnew1sBqdzD0kmkMn3EV8alpze7P7XazcuVK1qxZg9/vB6BXr15MnTqV7t3PvQ2wEEKcD6RNrYhEnTrUGobBXXfdxfvvv8+SJUvo27dvg+W1N4r985//5LvfDXYdtXv3bgYMGNAqN4qdzrHqY/xx3R9ZlL8IgBR7CvePvp/Lsi+LmJ/DC3//GGVvv03U0KFk/+vdNnlfPreLHUsXsenzTygrCA2Zqyj0HjmGEZdfTebg5pt4VFVVsXz5ctavX4+u6wAMHDiQKVOmkJKS0uplFUKISCChVkSiTh1qb7/9dt5++20+/PBD+vfvH54fFxeHPfQT9c9//nM+++wz5s2bR2xsLHfddRcAq1atOuPjtPTkXn5kOXPWzuFw1WEAxqSN4aGxD0VEk4RASQn7ps/AcLno/uyzxF42o82OZeg6h7ZsZOP8jzi0ZWN4flJmFhdcfhUDJ0zGYotqctumRifLzc1l8uTJJCQktFmZhRCiK5JQKyJRpw61zdXOvfHGG/z4xz8GgoMv3Hffffzzn//E6/UyY8YMXnrpJdLSmv/p+mStcXJ7NS9vbH+Dv2z7C17Ni1kx86NBP+J/hv0PDkvjm8+6kuI/vUDJiy9izcqi1ycfo1gsbX7ME0cPs+nzT/hm6SL8Xg8AUc4YcqfOYNj0K4hNSm5yu6KiIhYvXsyuXbsAUFWVUaNGMWnSJJzOyOytQgghzpaEWhGJOnWobS+teXIfqTrC3HVzWXJ4CQApjhQeGP0AM7JmdNkmCVp1DfunT0crLSXtkYdJuPHGdju2p6aa7YsXsmnBp1QWB0eKU1SVvmMuYsTlV5PRf2CTn+uRI0dYvHgxBw4cAMBisXDhhRdy0UUXhWv5hRDifCWhVkQiCbW0zcm99PBSnlj7BEeqjwBwYfqFzB47m15xvVpl/+2t9O//4Pj//R+mpCT6LPgcNbp9b4jTdY39G9ay6bOPOPzNtvD81F59GHH51fQbNxFzEzXITY1ONn78eMaOHSujkwkhzlsSakUkklBL253cXs3L69te5y/b/oJP92FWzcwcNJOf5f6syzVJMHw+9l95Ff78fJL+9y6Sb7+9w8pSdOgAmz7/hJ0rvkIL9XzgiItn2LTLGTbtW0THN2xDaxgGu3fvZtGiRQ1GJ5s0aRIjRoyQ0cmEEOcdCbUiEkmope1P7sNVh5m7di5LjywFIC06jQdGPcC0rGldqklC5WefcfTe+1AdDnp/+QXmxMQOLY+rsoJtixaweeGnVJeeAEA1mel/0URGXH41ab0b9pah6zrbtm3jq6++ory8HAiOTjZlyhSGDh0qo5MJIc4bEmpFJJJQS/ud3EsOL+GJtU9wtDr4U/hFGRcxe8xssuOy2+yYrcnQdQ59/zo8O3aQ8KMfkfbrhzq6SABogQB7165i4/yPKNizKzw/o99ARnzravqMHoepXm1sIBBg48aNLFu2jOrqagCSk5OZOnUq/fv371IXGkIIcS4k1IpIJKGW9j25PQEPf93+V17f9nq4ScLNg2/m1qG3dokmCTWrV5N/8y1gsdD7s0+xZmZ2dJEaKNy3h42ff8zuVcvRtQAAzm5JDJ/2LYZOnYEjNi68rs/nY82aNaxcuRKPJ9jDQvfu3Zk6dSq9enXNts9CCHEmJNSKSCShlrqT+0R+EYmZTXcV1dryK/OZs3YOK44Gh/1Nj07nV6N/xSU9L+n0NYX5P7mVmpUrib3ySrr/vyc7ujhNqi4rZcsX89n65XxcFeUAmC1WBkyYzIhvXU1yz+zwum63m1WrVvH11183GJ3skksuoUePHh1QeiGEaFsSakUkklBL3cn9zT3z6XnjcKJHn3kfty1hGAZfHf6KuWvncqzmGADju49n9pjZZMVmtUsZzoXnm284eG1wBLfs//4H++DBHVyi5gX8fnavWsbG+R9RdHB/eH7m4FxGXH41vUaORlVNQNOjkw0YMIBLLrlERicTQkQUCbUiEkmopV6ovXs+MVHRpD04BnOcrd2O7w64+cu2v/DG9jfw634sqoWbhwSbJNjNnbNP1aP3P0DlJ58QfdFF9Hz9rx1dnNMyDIOju79h0/yP2bt2FUYotMalpDJ8xpUMmTKNqOjg4Ay1o5Nt3boVwzBkdDIhRMSRUCsikYRaTgq1tmgSfzgQx5Ckdi9HXmUec9bMYeWxlQBkRGfwqzG/YkrmlE7XJMF3+DD7v3UF+P1k/vUvOMeP7+ginbHKkiI2L/yMbYsW4KmuAsBii2Lw5KlccNlVJGYEmxwUFRXx1VdfsXPnTqBudLKJEycSExPTYeUXQoiWklArIpGEWhqHWswKzjHpOCf1wBzffjW2EKxRXJy/mLnr5lJQUwDAxO4TmT1mNpmxneymrMf/QNnf/kbUoEFk/+ffKF2sSyy/18POFUvY+NlHnDiSH56fPXwkIy6/muzcC1BUlaNHj7Jo0aIGo5ONHTuW8ePHy+hkQoguSUKtiEQSaqk7uXf84jNMFgfRplA4Myk4LkghZnImlqT2DS8uvyvYJGHHGwT0AFbVyi1Db+EnQ35ClDmqXcvSnEBpKfunTUevqSHjqf9H3BVXdHSRzolhGBzesZWN8z9i/4a1EDolEjJ6cMFlVzL44qlYo+wyOpkQImJIqBWRSEItdSf3n275CMUSzdB+8fS1qWiHgz9No4B9aBIxU3piTW/f4WEPVhxkzpo5rC5YDUB3Z3ceHPMgkzMnt2s5mlPy8ssUP/c8lsxMen/6CUoXD3flhQVsWvAJ27/6Ap/bBYDNEc2QKdMYPuNK4lJSZXQyIUSXJ6FWRCIJtdSd3E/e/BF2azC0KqrC0EEJ9DYp6HmV4XWjBiQSc0kmtp7t94+AYRh8mf8lc9fO5bjrOAAX97iYX435FZkxHdskQXe52Dd9BlpJCam/+Q2JP7ypQ8vTWnxuFzuWLmLT5x9TVhDsmQJFoffIsYy4/Gq6DxzM9u3bG41ONnnyZHJzc2V0MiFEpyahVkQiCbXUndz/7ycfceFlAyg9VkP+N6Xh5f36xjEg2oySVwmhT8vWKy4YbnvHt9tNXC6/i9e2vsab37wZbpJw69BbuXnIzR3aJKHsnXco/N2jmBIT6b1wISZn+9ZmtyVD1zm4ZQMbP/uIvK2bwvOTemZzwWVX0e+iiWzdtr3R6GSXXHIJAwYM6HQ3+AkhBEioFZFJQi11J/fRQ8fJyAr2R1p8uIpNC/PZt6EIQw9+RJndo8lNjsKcXwWheZbMGGInZxI1MBFFbZ8Ac6DiAH9Y8wfWFKwBoIezB7PHzmZSj0ntcvyTGX4/B668Cl9eHkm3307y/97VIeVoayeOHGbT5x+zY9kiAl4vAFExseROncHgKdP5Zt9+VqxYIaOTCSE6PQm1IhJJqOXUJ3dliZvNiw6zc8UxAv5g36bJ3WyM7OHEdqwaQvPMqQ5ip2RiH5qMYmr7cGsYBgvyFvDkuicpchUBMDlzMr8a/St6xLT/KFiVny/g6N13ozgc9FnwOebk9hmZrSN4qqvZ/tVCNi34lMriYHMQRVXpO3Y8g6dexoHjxaxZsyY8OllOTg5Tp06V0cmEEJ2GhFoRiSTUcmYnt7vax7YlR9n21RE8NcGwEhdjYVR2DDFFLgyvBoCpWxQxF/cgekQqirnt21W6/C5e2fIKf/vmbwSMADaTLdwkwWZqv+7IDMPg0PU34Nm6lYQf3Ejaww+327E7iq5r7F+/hk3zP+bwN9vC81N79WXQ1BkUeAJs2LhRRicTQnQ6EmpFJJJQy9md3H6fxs6VBWz+Mp+qE8GfmaNsJsb0iiWx3IPhDgBgirXinNSD6DFpqFZTm7+H/eX7+cOaP7C2cC0AmTGZzB4zm4k9Jrb5sWvVrFlL/qxZYDbT+5OPsWZnt9uxO1rRoQNs+vxjdq5YghaqoXXExdPv4mmUWaLYsXMXtadabm4uU6ZMkdHJhBAdRkKtiEQSajm3k1vXdPZtLGLTwnxKDgdvEDKrCiN7x5Lm8kOoNleNNuMc3x3nuAxUe9t292QYBp8f+pwn1z1JsTvY3dQlmZfwyzG/pLuze5seu1b+z35GzdJlxFx+GT2eeaZdjtmZuCor2LZoAZsXfEJ1WfBmQ5PZTOboi6iO7cbB/MNAcHSykSNHMmnSJBmdTAjR7iTUikgkoZaWndyGYXBkZxkbF+ZxZFcZACowLDuGnpoGVcFwq9hMOMdl4JyQgcnZtn251vhreHnzy/x959/RDI0oUxS35d7Gjwf/GKupbY/t2b2bg9d8BwyD7H//C/vQoW16vM5KCwTYu2YlGz//mII9u8LzE/oPwZecQeGJYOA1m81ceOGFMjqZEKJdSagVkUhCLa13chflVbJpYT77NxZhGKAAAzMc9DYrqJU+ABSLSvTotHYZgndv2V7+sOYPrD++HoCs2Cxmj5nN+O7j2/S4x371IBUffohj7Fh6znvjvO/WqnDfHjbO/4jdq1ega8HmKZa0Hmjdc6hwuQEZnUwI0b4k1IpIJKGW1j+5K4pdbP7iMDtXF6CFekfolWRjYLQFc0WwK6j2GoLXMAw+O/gZ/2/9/6PEXQLApT0v5Zejf0m6M71Njuk/epT9l12O4feT+efXcE5sv3a9nVl1WSlbvpjP1i/n46ooxwCMuCSMnr1xBYI3GkZHRzNp0iRGjhwpo5MJIdqMhFoRiSTU0nYnt6vSx7YlR9i25AheV7CGrnushdxEG9byULhVwJ6bTMzkzDYdgrfaV81LW17i7Z1voxkadrOdn+b+lJmDZrZJk4TjT8yldN48bP37k/P+eygywlZYwO9n96plbPzsI4oO7ccAArGJaN1z8BOs1ZbRyYQQbUlCrYhEEmpp+5Pb5wmEe0yoLguG2WS7ieEpdhy1Nbe0zxC8e8r28PjXj7OxaCMA2bHZzB47m4syLmrV4wTKytg/bTp6dTUZf5xL3NVXt+r+I4FhGBzd/Q2bPvuIvetWo+sG/vgk/Kk90NVgjxkyOpkQoi1IqBWRSEIt7Xdya5rOvvVFbFqYx4mjNQDEWxQuSHcQG2pzC20/BK9hGHxy4BOeWv8UJzwnAJiWNY1fjv4ladFprXackldfo/iZZ7BkZNDr8/mo0la0WZUlRWxe+Bnbvvwct8uFLyEFf1IahinYBCEjI4OpU6fSu3fvDi6pECISSKgVkUhCLe1/chuGQf6OUjYtzOPonnIAolW4IN1BN5cfQn8ilswYYqdkEjWgbYbgrfJV8dLml3h719voho7dbOdnuT9j5qCZWEyWFu9fd7vZP+MyAkVFpM5+kMRZs1qh1JHN7/Wwc/kSNs7/iJJjR/F1S8WXmAqhmtuc7GymXnqpjE4mhGgRCbUiEnX6ULts2TKefPJJNmzYQEFBAe+//z7XXHNNeLlhGDzyyCP8+c9/pry8nPHjx/Pyyy/Tt2/fMz5GR57chQcr2Lwwn/2bi8EAuwK5KXZSAxqKFvyjaesheHeX7ubxNY+zqWgTADlxOTw09iEuTL+wxfsu+/e/Kfztw6ixsWTMfYKogQOxpLVebXCkMgyD/O1b2Dj/I/Zt2YSvWxr++GQIta/t17cvUy+9lNTU1A4uqRCiK5JQKyJRpw+18+fPZ+XKlYwcOZJrr722UaidO3cuc+bM4c033yQnJ4ff/va3bNu2jW+++YaoqKgzOkZnOLnLj7vY9GU+u1cXogV0rAoMTrSRqYASCPagYOoWRezFmThGpLT6ELyGYfDxgY95av1TlHqCfajOyJ7BA6MeIDX63IOTEQiwd/IUtJJgzwuoKum/f5T4732vNYp9XigrPMbmBZ+yZdlXVMckEojrBqFmKQP79WP65ZeTkJBARUUFpaWlJCYmEhcX18GlFkJ0Zp3he0+I1tbpQ219iqI0CLWGYZCRkcF9993H/fffD0BFRQWpqanMmzePG2644Yz225lO7poKL1u/OsL2pUfxuQOYFegfa6GXVUUNdQ/WlkPwVvoqeXHTi7yz+51wk4SfD/s5Pxz4w3NqkuAvLGTflEvgpL9mibfdRtwV38LWv7/cAHWGfG4X25csYt0X8ynGTCA2EQj2h5yWnERhyQkMw0BRFK666ipGjBjRsQUWQnRanel7T4jW0qVD7YEDB+jduzebNm1i+PDh4fUuvvhihg8fznPPPdfkfrxeL15vXa8DlZWVZGZmUlJS0mlObp8nwK6VhWxbcpSach8moFe0iX4OM+ZQuFUcZhzj0nGMTW31IXh3l+1mzro5bC3ZCkBObA4Pjn6Q0amjz2o/rrVrOfaTW5tdbkpKwjF+PI4J43GMG4dJahhPy9B18rZuYvXnn5BXUYPmbPozmzVrJt2795CLBiFEI5WVlSQlJUmoFRGlS/fuXlhYCNCoXWFqamp4WVPmzJnDo48+2mj+woULcTgcrVvIFoofC9ZjZqoOWtlbDftrNDKtCv2iVRyuADWLDlO5JI+iNC9F6R4Clta7Rvme8T1623uzwLOAg5UH+dmin5FryeUy+2XEqmf2j6C5vIIcRUGpd+1kKAqu3r2w5+VDSQlVH35I1YcfYigKnsxMavr3w9W/P57u3cNtSEVjcbmjGVBRRv6unVRZGw/g8eabb4GhY9J1LIqCxWLGFmXDHu3EERtHVLRT+sAV4jzlcrk6ughCtLouXVO7atUqxo8fz7Fjx0hPrxsd67rrrkNRFN59990m99MVampPZujBHhO2fHmEwgOVKECGVWFwnBV7qOYWi4p9ZArRE9IxxbXeELyVvkpe3PIi/933X3RDx2F28D9D/4fr+1+PRT19k4TK996j6NHfg66DqpLyyMPEXnsths+He8NGXCtX4Fq5Et++/Q22UxMScIwbF6rFvQhzUrdWe0+RZNeGtfx3/sJwO1sg1NzDAOXUodVk6NgtZpwOBwnx8SSlppLWvQfpPbNwOp1SyytEhJKaWhGJunSoPdfmByfram2LCvZXsGlhHge3BG++SjMrDI634gzdUNZWQ/B+c+IbHv/68XCThD7xfXho7EOMTjt9kwR/YSG+vHysWT2b7f3AX1BA9fLl1CxfQc3q1ejV1Q2WRw0eTPTECTgnTsQ+bBiKDCMLQNWJEl586H48aVnBYGsYRB3P54f3PEBFeQWFRw5TXFxEeVk51W43noBGwGQG06k/P8XQsakq0VE24mJj6ZacTFr37vTIzqFbcooM4ytEF9bVvveEOBNdOtTW3ih2//33c9999wHBEzUlJaXL3ih2NsoKa9i0MJ/dawrRNYNks8KgWAvxeuiPtA2G4NUNnQ/2fcAzG56h3FsOwBW9ruC+kfeR7EhulWMAGH4/7i1bqF62nOoVy/F+s7PBcjUmhuiLLsI5cQLREyac992EbVu8kAVvvIpmtmIK+Jhx888Yesn0Ztf31FRz/HA+x/IOcbyggNLSE1RVVePy+fChYJitDWt+T2YYmDFwWC3EREeTmJhIclo63bOySM/s2ema8QghGuqq33tCnEqnD7XV1dXs27cPgAsuuICnn36aKVOmkJiYSM+ePZk7dy5PPPFEgy69tm7d2uW69GqJmnIvWxYfZseyo/g8GgkmhYExZupHzKiBicRMab0heCu8FTy/8Xn+veffGBhEW6K5Y/gd3DjgRsxq69fgBYqLqV65kpply6lZuRKtoqLBclu/fnW1uCNGnJejl1WdKKG88BjxaRnEdEs65/3omkZZUSFHDxyk8OhhSoqLKa+ooNrtxqvpaGZLeDCI5qiGjs2k4rTbiYuNIyklmbTuPeiRnUNiUpK05RWig3X17z0hmtLpQ+2SJUuYMmVKo/mzZs1i3rx54cEXXnvtNcrLy5kwYQIvvfQS/fr1O+NjRMrJ7XUH2LHsKFsWH8ZV4SNWhQHRZtJMCrV1brbeccFw20pD8O4o2cH/ff1/bD+xHYC+CX359dhfMzJ1ZIv33RxD0/Bs30718hVUL1+GZ+u2Bl2GKQ4H0RdeGKzFnTgRq4y+1WoMw8BdXUVh3iGO5edRVFBAWVkZldXVuP1+/IoJw3yadtaGgUUxcFitxDqdJHbrRkpaOt2zs0nvkYnN1nrtwYUQTYuU7z0h6uv0obY9RNrJrfl1dq8tZNPCfMqPu4hWoZ/dRKZFDYfb1hyCVzd03tv7Hs9ufJYKb7AG9apeV3HvqHtJsp97jeGZCpSVUbNqFTXLllO9cmXdQA8h1uxsoidNxDlxIo7Ro1HPsAZfnD0t4OfEsaMcPXSQwmPHKCkupqKykhqPB69moJstp+3RwmTo2EwmnA47CXHxJKemkNajBz1yehEbFy+1vEK0gkj73hMCJNQCkXtyG7rBwa0lbFyQx/GDldgV6BOlkh1lQg39qbfmELzlnnKe2/Qc/93zXwwMnBYnd15wJ9f3v54Sdwn5lfn0jO1JWnTbtX81dB3vrl3hWlz3ps2gaeHlis2GY8yYUFvciVhzsuUO/3ZiGAY1FeUUHDrIsfw8io8fp7S8jOoaF25/gIBiwjjdzWeGjk0Bh81GbEwM3bp1IzU9g4ycHNIyumOxnP0AIUKcjyL1e0+c3yTUEvknt2EYFOwL9phwaNsJrAr0tqn0tpswhf70W3MI3m3F23h8zePsOLEDgFRHKkWuIgwMVEXlkXGPcG3fa1v6ts6IVlVFzerV1CxfQfXy5QRO6r/Y0qNHuC1u9NixqNGtc0OdOHsBn4+iI/kczTtE0bFjlJSUUFlVRY3Hi88gWMt7mgsQk6FjN5uCXZQlJJCcmkp6Zk96ZOfgjImRCxghQiL9e0+cnyTUUu/kzt9JbOaAji5OmzpxtJrNX+SzZ+1xVMOgl1Wlj91Ebf1Waw3Bq+ka/937X57Z8AzV/oZdcykoPDX5KUaljiIhKqEF7+bsGIaBb98+qpevoGbFclzr1mP4/XUrWCw4Ro4M1+La+vWVENRJGIZBVekJjh48SMHhPIqLiigvL6fK5Qp2UaaawXTqv6+KoWNTFKKjbMTGxpKUlERqRgY9cnqRnJaO6TTbCxFJJNSKSCShlnon94NxxH7/eRgxs6OL1OaqSj1sWXyYb5YfQ/dqZNlU+tlN1N6io0abcU7ojvPCjBYNwbsobxF3L7m72eUJtgRy4nLIicuhV1wvesX3oldcL9Ki01BPM3BAS+k1NdSsXRuuxfUfPtxguTk1NViLO2Ei0ReNwyT/8HdaPo+b4/n5HM0P1vKeOHGCyupqXF4fPgh2UXYqoS7K7BZzqIuyBJJT08no2ZPu2dk4HMEa/GN5hzi8fz+ZvXuTkZXd5u9LiLYioVZEIgm11A+1McTaVLjkt9BjFCT1hZj00/7k2ZV5avxsX3aUrYsP463yB4fgtZtwhN6zYjPhHJeBc0IGJufZd5NVWFPIjP/OQDf0BvNT7akcdx9vdju72U52bHajsNszpicWU9u0m/QdOhRsi7tiOa41azE8nrqFJhP2YcNwTppI9ISJRA0aiCI3LHUJhq5TXlLEkQMHKDx6hJJQLW+124NH09BMp795TdV1FHQ0xRQe4CLV6eCCUaOITUggLrEbCUlJ2O0Oqd0XXYKEWhGJJNRycqg96QvJEg3dekO3PsGQ261P6HVfiIqcfwgCfo1dqwvZ/EU+lcVuMiwK/e0mYkI9IygWlejRaTgn9cAcf3ZdLr239z0eXf0ouqE3aFPr8rs4VHmIAxUHOFB+IDhdfoC8qjwCeqDJfZkUE5kxmY3Cbk5cDtGW1msPq3u9uNatp2b5cqpXrMC3v+EQvqbERKInjMc5cRLRE8ZjTmi/ZhSidXldNRTkHeJofh5FBccoLS2jqroal8+PH+X0XZTVZxioho5ZAbOqYrNYsFmtREVF4XA4cDqdOGNjiY2PJy4hkfikJGLjE6Tpg2h3EmpFJJJQSxM1tTkXQ8VhKDsEhtb8htEpoaAbCrm1wTc+C073c2cnpesGBzYVs2lhHkV5VaSZFfpFqSTU3jwWGoI3dnIm5rMYgrewppDDVYfJjMk8be8Hft3PkaojHKg4wMGKgxysOMiB8gMcqDiAK+BqdrsUR0ow6NY+4oNht1tUtxbXnvmPHq2rxV21Gt1VrxyKQtSQIcGbzSZOwJ6biyIhJSLoukZpYQHLFnzO1rwjjZarAV9wPdV02gEpmmUYKIaOCQOzomI1m7BZLERF2bDb7TiincTExBATF0dsfALx3bqRkJyMLar1hsAW5x8JtSISSailfqiNJ/b7z9W1qdX8wWB7Yh+U7A0+1z6qm//pHMUECdmhWt0+kBR67tYXYtK6RHMGwzA4tqecjQvzyN9RSrJZoa9NJdkSCrdtMATvmZTpuOt4w7AbquU94TnR7Hax1ti6mt16YTcjOgPTOQQRw+fDtXlzsBZ3+Qq8u3Y1WK7GxRF90bhwLa4lJeWsjyE6l2N5h3jt9TcanruGwU9vuZmMrGy0QIDq8jLKT5ygvPQEVeXlVFVVUFNVjcvlwu3x4PX68Ab8BDSdgAGaop725rZT0jVMhhGsFTap2MwWbDZrMAjbHThjamuFE0K1wsnExMWhygWXQEKtiEwSajnH3g88FXBif+ixt17w3Q/+mua3szrrmjN0q9+coU+nbc5QcqSaTQvz2Lu+iHgF+kWppFnq2iC29hC856LCW9Ew6IbC7tHqoxg0/VfcZrKRFZsVDrs58cHgmxWbhc105k0s/MeLqFm5kurly6hZuQq9srLhcQYMCI9u5rjgAhTpS7VL+ugff2Pjnn3hNrUj+vXh6pt+dM77MwwDr8tF+YliyktLqSgrpbqikqqqSlw1NbjdbjxeL16fH5+mEdANNMBQTed+YazrqLqGSQGzomC1mEO1wlE47A4c0dHExMYQExdsHhHXLZH4xCQsUVHSVjjCSKgVkUhCLa18chsGVBU0DLkn9gWDb1neqZszOFNDQbd3vfa7fSEhC9ro5qizUXnCzZYvD/PNymM4Ajp9o0x0t6ptNgRva/AEPORV5jUMuxUHyKvIw6f7mtxGVVS6O7vXhd24nHDtbqz11H8/jEAA97Zt4R4VPNu3NxjCV42OxjHuQpwTJuKcOAFL9+6t+n5F2zqWd4gjB/bTo1fH9X6gaxoVZaWUlZRQWVZKZXkZ1ZWV1FTX4HK7cLs9eH1evIEAAc0gYBjoqgrn2puIYaBoAUwYmBQFiynUVthmxRFlx+FwEB0TQ0xsHLEJ8cQndCOuWzccsXGYTjeYRjOkl4m2J6FWRCIJtbTjyR3w1TVnCNfuhpoz1BQ1v11tc4Zw0O1T137XmdruzRk81X62LT3C1q+OYHL56WszkWlVqR1t15oZQ0wrDcHbVjRd42j10UZh92D5Qar8Vc1ul2RPqgu6tU0ZYnNIcaQ0GeQDpaXUrFxFzYpgUwWttLTBcmvv3jgnTCB60kQco0ah2s7uJjwhzoRhGLhdLspLSigvLaGyrIzKigpqqqupqanG7fbg8Xrx+etqhXUUjHPt4cMwQNdQdQ0zwZvmrGYTNpuVKFvwprloZzQxMbGhWuEE4pKSccbFsfCD99i4Z3+r1YiLpkmoFZFIQi11J/fu/EL6ZaZ2TCHc5VC6vy7khoPvfvA3f3MU1phmemfoA7aYNi2y36exa1UBm7/Mx3/CQ58olSyriikU7ixpDmImZ2LPTUar8hEocWNOsmOO67zBzTAMStwlTYbdInfzFx5Oi7Nhf7uhwNvd2R2zGqytMnQdzzc7gwF32XLcmzeDXtfVmRIVhWPsGJwTJ+GcOAFrVlZbv10hTsnv91NZXkZ5SQkVZaVUlpcHa4VranC5XHg8Hjw+H76ARkAPthU+5yAMwSGtVbVR22WnEcBmsWA2mzGbTFisFixmC1arBavNhtVmw2a1YbPbsUVFEWV3YHc4iIqOxm53YHM4MFttWGw2aVMcIqFWRCIJtdSd3Fn3/Is5N4zlxjE9O7pIdQwDKo81DLm1N62V58FJ/b824ExrpneGnq3anEHXdPZvKmbjgjwqj1TT26aSY1OxhL6Y1Ggzek2oiy4FEq7tS/ToU/eA0BlV+ao4VHGoYditOMjhqsON+uGtZVEtZMVmNQq7WbFZWGt81Kz+muoVy6lZvoLA8YY3H1p69gzX4kaPGYPqcLTH2xSiRTRNw+WqofzECSpKS4NNJCorqamqCt40V9tW2O/Hr+nB5hEobfuLk66DoaPoOoqhowAqYFKCzY1MqoLZZMJkMmE2m7CYLVgsZiwWazA4W4PBOSoqKhyabQ47drsDe3Q0NrsDa1QUFlsUZqu1S/RhLaFWRCIJtdSd3Jl3/wvV5kBVwGJSsZpUzKZgG7LgQ8Fcb9piUjGrClZz8PmU65mU4P5UFYtZwaLWrdfwOLXr1003exzDh7XqMNby/ZjL96OW7kepreWtKW7+DatmSMipq9Wt337XmXLOXy6GYXBkVxmbFuZRsKuMXlaVXjYV20lNEAzAkmLHFGtDdZhRHZbgc7QFU+10vXmKzdRp2ug2xaf5yK/MbxR2D1UcwqN5mtxGQSHDmVEXdmNz6F1qIWnLYbSvN+DasAHqDeGrWCw4Ro8iesJEnJMmYu3du1N/JkKcDV3X8Xq97N/1Df/54KNGNbV9u6djtpjx+/z4/H78fj+BQICApqFpGgFNRzMM9NoHYLR1UK7PMCAUmIPPBqoSDM6qAiZVxaQG/103m8x1wdlqwWqxYrFasdXWOIeCs91ux1avxtlmtwdDs82G2WJt8fm/e/s2BgzNlVArIoqEWhqH2q6sNggnqDX0MR0nhwKylQKyOUamfozu+jGi8Da7vVuNpsSWSWlUJqVRWVQ4sqhw9KQqOhus0eHAbW0m4FvMKhZVxVfspmhdMaa9FYxznvswuwCYFFR7KOhGB58bhN/oeiHYYUGNtqDazR3enlc3dApqCsJ97NZv0lDhrWh2u8SoRPpF9WTMUTv9dtfQbUs+psKSBuuY09ODtbgTJxB90UWYnE62bl3H3s1r6Tt8DLm5o9v67QnRJlqrlwnDMAgEAvj9fnw+Hx6XC7fbhafGhcftwuN24/V48HrceL1efF4vPr+PQCg4BwIB/IFAKDRraLqBZujoOui0Q+3yyerVNqPrqBjh0FxX26yGa5wtZjMWiwWLxYLVasVitYWD84EDB9hfUsYTc+dKqBURRUItDUOtOcrBB3eMJzHail8zCGg6Pk0noBn4NR1/6Dmg6/gCBgFdbzj/5PU0HV9oP35Nx68b+AM6Ad0I7bdu3Sa31w18AT10nIbrBfSz/6NT0EmjjBy1gF5K3SNHKaCHUoxJaX6fhUYCB/R0DhrpHKj3OGIko9F0O7VxXjN/tNob1CoYhsFGV7AXCKuiYFXBqoSmFbAoCjYVLKFuh86FDrgVcJlqHwpus4LbpOAxBx9ei4LHrOKzKHgtKn6LimJRURUFs6qgqgomRcFkCj2roYcSXGZW681TlUbbmU3BefW3q512aRUUefI57s6nwJVHoTuPAlc+JZ4m+j82DNJLYfQhM2PyLPQ64MLsr9fcwWymIsFJbHE5CqArsOlb47n+l4+jRkWh2O0oFovU7LaCggo3B0tqyEmKJj1OBj9oK1t37Gbnzt0MHNif3MH9O7o4zdI0DX+o5tjn9eJxu3DXhmaXOxiYPR48Xk8wNHu9+Hx+/H5fg9rmYI2zjmboaLqBbhCqbabNgrPX6+WJJ56QUCsiioRa6kJt9j3/5okbx3D96E7UpvYUdD0YbGtDri8Utv0BA7+uN5xfLxA3GdZ1A83nwVGTj7P6ILE1ecTW5JHgziPRk090oLzZcgQwcdyUzlFTD46oGeQrGRwig4NkUOpy8MsyK8PsJlRFQTcMtrg13lK9GChEGRBl1D3bDAW7AbbQPDPBoBsMvkqD8NtwXv0wfO5fAm7DoBKDCgzKlbrp4ENv8LoSg3IM3Od8tCYoXlRbMaq1GNVWhGotCr0uQVGCQdbiNxh02GD4foPhBwy6l55mn4CuKHhNZnxmC16zFZ/Zgs9kw2+y4jfb8Fms+M210xYCZhv+8DwrAbOVgNVKwGLDb7ERsFjRQs8Biw3NGnyNKVhDrioKigKqoqAqoISeg68bLgu+rl2v4euG24fmqU1szxmsc9I+OdUxGpQxOG/V/hP8/es8jOCmzBqXxcX9UkAhfHyFum0VCC2r278SXrdu/brn4LbULkep93nUbUf9/Z20rqI03l/9ctSfV7sdCqfeH3XHD5elDS+Q3l2Xz+z3tqEbwXLNuXZol/k3ubWdXNvs83mDNc41NXjc7nCNs88T6sbN4w010QiFZr8ff/1mGrqOL6ChKaqEWhGRJNTSSXo/6OxcpVB6oN7IavX64A003W4UQLdEs7PiQtZU/RyHyYxLCzA25hV6Th6MMzEdwxSFrlrRzVY01YauBp811YKmWvH4Lbh9ZrweM26fgtdrwuMBn0fH7w7gcwcIuDX87gABj0bAE0D3aJh8el0APqkm+OR5llAoVs/xi1oD3CZwqwo1ZqgxQY2qUKNCtQpVikG1ApWKQZUCFQpUGaEbZAwI6MGfNAO6jqaDbgQvOnSD4E+euoFmBDDMJaGgW1wv7BYxfpeHuz9qfKOapoCpnc9uTVHwmix4TFa8Jitek6Xu2WzFq1rqpust85y0rs9kwdPEel6TFY/JEhyWtgMkucvJqC7hmDOJEnt8h5ShMzllSKZ+GG96ujYkE1pX1w1O1DTuPzotNgqzSWkUspUGob3eaxoGcjgptJ+0n9qV6t5H45Bfd4zQ6yb21eg4TZSJBuVvfAFR/700V6b6FynNlam543iLj2I/vAqvzyehVkScFjZ2jCxp8nNi8xyJwUePUQ3n6zpUHm0YcsO9M+Sj+msY7FhElm0zFYE04syFOE0nYO2XAOG7kM+aYgKzLfSIArsVYqLAbAVzFLoahU+JwWvE4jWceHUnHt2OV4vGq0XhDURRHbDi8Vvx+ix4fWYCPhOGT0XR1HDtr1WtqwmuqyGum2dSFEyAUwOnZpAcvrfr1GnSQAWrihJlRrGbMTktmJ1WzDGhNsEOS7idsCk6+KzYzRgmtV7QDYbfVRu/QP/4fgKxWQQSemMu24+pMo8/zR6FN8GK5nahe1wYbjeGxw1eL3g9WHwa1gDY/NQ9+8EWMOqm/WAL1E4bWGun660f5Qc19HZNhoEj4MMRaHpgi9aimUxoFhta/Rpjq41A7bTZGqxRNlsJ1NY4W6z4zMGaZZ/Zgq+2ZtpsJWAOhm2fuW6Z12RBU1R0AypcPrLWLOIXm/+DGmpP+dzw77Fn1BSirWYMI/gnbhhGaNpoOI/gvUQNpjHCPbrVn6/Xmya0H725fZy8PwM46fh6qExtwTBAa7Dz1jnQyRcPhZXNXziLszfJH0V3S/P3VgjRVUlNLdK1SZvxe+DwGnjr2zT8slNg4LeD/VFqvmBNb8Abenjqzau3TPOCHmiXYuuGiteIxqvH4NGjQ6E4+NprROPRneGg7Ddi0YhFN2LAcGBSLM02j6itIW5J8whdJbhjmxrqHcKKOTaKo2u3k2RKRlGUYC8UWj7Db7kcxaSASQl2MWRWgjfPmVQUk0IADY/hwWN48Rge3IYHt+7BrbuDz5obt9+NO+DGFXDhDrgbPFwBV3C534XP60Zzu9A8bnS3C9WnhQKxUReO64Xn4LTRKCDXzj95fWsAbL5zvABqAcViQbHb0c1mjNJS1Kh4VGcqevVxdE85piFDiYp2gKqgqCZQ1eBnrarBz1kJTium0IhetfNMpuA2tfNUFVTTSftRGs+r3U+96Qb7Vk+aVk0N9hOsWjVhKAqoKkZoXSO0T0NRwtsbtYMv1FtmhLap3R5FwTCZwjdNGaoJQwFDMYVeh9ZXFQxFDa5Xf98o4dBtYFBc5eWfv3uB/91Ud/Hwpwu+x3d/ezvdnLYGQb+pYB9aGgr2NHuBEVqxblkTFyC1xwmt2uS+OGl+3et6x2q0HeGhu2vn0eyFSt1xTj7WyWWiUZkar49hUOUJMG/VIbLK97D01Xvle09EFAm1SKhtcxvfgo/vDg4RrJjgqmdhxMyz34+u1QXf2qAb8J4UiE+ap9VbVhuSm5x3criu3dbT+BinqI3SDDNePRqPERMKwsEAHAzCodd6DJoRj2bEouMEolGwY1bMzTaZsLSgecS5MtCDX75KsPrPCD2jNHwoKqDooIKiGKF+jAx0VUdXdHRVR1N0AqqGpur4VY2AEsCvBvCrGl7Fj0/x41EDePHjwY/H8OHGjxsvbny4DA9uzYtHc6MHfCiahqppmAIaakDDrAUw+TVMmobJr2Hxa5gDGla/gc1vNArUlkCw5rmuFrouPEf5m/48LFnjsQ3/EYqiYhg63s1/w5+3sv3+QCJRKBzXXggYioLh8eCPq/vVwVKRhyk+HsVsDv7srqjh7Wp/hg8P2KAqDV8roQu5ptapF8wVpeHrJo9TG/gVJXyBEF6ndr8nrRNsQlDvOCeXRSHUp23dcRrsN3zsUx+n7j2oTa9Te+zQvD1Lv8a06ivG7tsr33siokioRUJtu6g4GmyTm9gL4rp3dGnOnWEEa4ybDMlN1TafQQ10aF7A68frAY9XxetT8XpNeH1mPH4zXr8Nf8BBQHOiaU4MojGMaBTFTrRqJ9PauI1ppaajG6GbighmTUUJZc560wrtH5jbi4GBXu8/LfRaU3Q0RSMQmg6g41c0AqGHZmjoBB+GoWELwCBjYIMbpAzDYGvga2pMHkDHMEIP9NCgKAaGoYGhB2vQwss0QA9VowWX1z0boefgdP1+T+svVwwj2LVT7Tqh17XrBbt9ql2mo4ama7uDUuqtbwotU3UD1TBQdQXVCP290INNS9TQNUyDZ51gf6y11zcnvT5XNUNnktprfPhXh+MHVhK97a0W/k0QJyt1dmfChkXyvSciioRaJNSKLsgwwiG5YsdeKv/lahC4dMPAP9ZDVEoMekBD1/S6Z81AC+ihaR09oKNroAd0DM1AC4ChGcGcFTAwdCWcuXQN0JVQtlJC/Q4pGIYCuhr8GTT4+zMYCkrtM6F5BOeFewrg9KE7OK3UC+A0mK7dNlKDeXsLt9+l6WcdI/jHW2++HnrV8DXhuaeeV/efRTeRpSQ2unjYrx/HY/LXbV3bPCC8R07ac/35Ohigo4fW0sNrUztt6OG1qTdd7x2HmgjUNlqu/2mAYtR7d0btOnqDTy10cmAYBkrowkcJHa92fbXetsrJz+HtjODFC3r4WQ21oVCMehc2ofWV8HE0FMPA5tJIir4KW0p/Bj/3LfneExFFbhQToitSlPCNcnEjR1G0+hNsh2PD3aZ5Myvpe+2VHV3KZhmGga7VPvTwtFZvuuEyHa2J+YHaZQEjGM5DD8OvYwQC6H4NI6BjBDQMfzDEGwEdND0U3HXQjOB0beegWjAEoQN6sDGi6tfJMZsbha18n4Ze7+52jNCd50a9O8/rPYBG86mdVpqYV/8O/ZP2dar5jebBGXfDVXdx0AEXCU00mlYUhT6mk4bVPtuiyfVOI1Xemo4ughCtTkKtEBGg751XUrljH5W7jxLfvzuxgyd1dJFOSVEUTGYFkxloZuCOzqS6zMOSR1aTW6+/5a1ujcmPjsOZENXkNkbo7pxwjwjh142XNb1e06/D03qoJvF0+9Frb1gKXhAE5+mgBZ+DrSAM0EPbBYfMCm6v14X94DENlPB6ode100ZoWb1ta+9WClZGht68XlfW4OvQOgZ4K8pJLTEaXTwcj1ewRjsJN2evfW/U2294WWh/BJtB1Nbq1lay1k0bDdYP7zc8r9664cIE/6eEZitGeGaT69VOK6FXSv1jhOYq9VY9OXs3tyzcfdfJ69LwoqapdRSCPbgIEYkk1AoRIWIH9yF2cJ+OLkZEciZE0ev6/nz5j104VAWXbnDhTQOaDbRQe2NRXQARZ+bLp/5Gv6Ks8MXDnpQ8Lr3v7IfJPV80uOiBUGuLuguP4DqELxyqDlfieWNHRxZZiDYTMW1qX3zxRZ588kkKCwsZNmwYf/rTnxgzZswZbSttaoUQZ6K6zENFkZu4FPspA61omQMb11O48xBpA7PpNWLU6TcQZ2XvP3fhW3OQIdKmVkSYiAi17777LjNnzuSVV15h7NixPPvss/z73/9m9+7dpKSknHZ7CbVCCCHOJ0e2HyFzaKZ874mI0t59mbeJp59+mttuu42bb76ZQYMG8corr+BwOHj99dc7umhCCCFEpxPbU4KsiDxdvk2tz+djw4YNzJ49OzxPVVUuvfRSVq9e3eQ2Xq8Xr7duiMCKigoASktL8fub6XldCCGEiBBVVVUARMCPtUKEdflQW1JSgqZppKamNpifmprKrl27mtxmzpw5PProo43m5+TktEkZhRBCiM7oxIkTxMXFdXQxhGgVXT7UnovZs2dz7733hl+Xl5eTlZVFfn6+nNxtpLKykszMTA4fPiztt9qIfMZtTz7j9iGfc9urqKigZ8+eJCYmdnRRhGg1XT7UJiUlYTKZOH78eIP5x48fJy0trcltbDYbNput0fy4uDj5B7SNxcbGymfcxuQzbnvyGbcP+ZzbnqpGxK01QgARcKOY1Wpl5MiRLFq0KDxP13UWLVrEuHHjOrBkQgghhBCivXT5mlqAe++9l1mzZjFq1CjGjBnDs88+S01NDTfffHNHF00IIYQQQrSDiAi1119/PcXFxTz88MMUFhYyfPhwPv/880Y3jzXHZrPxyCOPNNkkQbQO+YzbnnzGbU8+4/Yhn3Pbk89YRKKIGHxBCCGEEEKc37p8m1ohhBBCCCEk1AohhBBCiC5PQq0QQgghhOjyJNQKIYQQQogu77wPtS+++CLZ2dlERUUxduxY1q5d29FFiijLli3jqquuIiMjA0VR+OCDDzq6SBFnzpw5jB49mpiYGFJSUrjmmmvYvXt3Rxcrorz88svk5uaGBwMYN24c8+fP7+hiRbQnnngCRVG4++67O7ooEeV3v/sdiqI0eAwYMKCjiyVEqzivQ+27777LvffeyyOPPMLGjRsZNmwYM2bMoKioqKOLFjFqamoYNmwYL774YkcXJWItXbqUO+64g6+//povvvgCv9/P9OnTqamp6eiiRYwePXrwxBNPsGHDBtavX88ll1zCt7/9bXbs2NHRRYtI69at49VXXyU3N7ejixKRBg8eTEFBQfixYsWKji6SEK3ivO7Sa+zYsYwePZoXXngBCI5ElpmZyV133cWDDz7YwaWLPIqi8P7773PNNdd0dFEiWnFxMSkpKSxdupRJkyZ1dHEiVmJiIk8++SQ/+clPOrooEaW6upoRI0bw0ksv8X//938MHz6cZ599tqOLFTF+97vf8cEHH7B58+aOLooQre68ran1+Xxs2LCBSy+9NDxPVVUuvfRSVq9e3YElE6JlKioqgGDoEq1P0zTeeecdampqZCjuNnDHHXdwxRVXNPi3WbSuvXv3kpGRQa9evbjpppvIz8/v6CIJ0SoiYkSxc1FSUoKmaY1GHUtNTWXXrl0dVCohWkbXde6++27Gjx/PkCFDOro4EWXbtm2MGzcOj8eD0+nk/fffZ9CgQR1drIjyzjvvsHHjRtatW9fRRYlYY8eOZd68efTv35+CggIeffRRJk6cyPbt24mJieno4gnRIudtqBUiEt1xxx1s375d2si1gf79+7N582YqKir4z3/+w6xZs1i6dKkE21Zy+PBhfvGLX/DFF18QFRXV0cWJWJdffnl4Ojc3l7Fjx5KVlcW//vUvaUojurzzNtQmJSVhMpk4fvx4g/nHjx8nLS2tg0olxLm78847+eSTT1i2bBk9evTo6OJEHKvVSp8+fQAYOXIk69at47nnnuPVV1/t4JJFhg0bNlBUVMSIESPC8zRNY9myZbzwwgt4vV5MJlMHljAyxcfH069fP/bt29fRRRGixc7bNrVWq5WRI0eyaNGi8Dxd11m0aJG0kxNdimEY3Hnnnbz//vssXryYnJycji7SeUHXdbxeb0cXI2JMnTqVbdu2sXnz5vBj1KhR3HTTTWzevFkCbRuprq5m//79pKend3RRhGix87amFuDee+9l1qxZjBo1ijFjxvDss89SU1PDzTff3NFFixjV1dUNagAOHjzI5s2bSUxMpGfPnh1Ysshxxx138Pbbb/Phhx8SExNDYWEhAHFxcdjt9g4uXWSYPXs2l19+OT179qSqqoq3336bJUuWsGDBgo4uWsSIiYlp1A48Ojqabt26SfvwVnT//fdz1VVXkZWVxbFjx3jkkUcwmUzceOONHV00IVrsvA61119/PcXFxTz88MMUFhYyfPhwPv/880Y3j4lzt379eqZMmRJ+fe+99wIwa9Ys5s2b10Gliiwvv/wyAJMnT24w/4033uDHP/5x+xcoAhUVFTFz5kwKCgqIi4sjNzeXBQsWMG3atI4umhBn5ciRI9x4442cOHGC5ORkJkyYwNdff01ycnJHF02IFjuv+6kVQgghhBCR4bxtUyuEEEIIISKHhFohhBBCCNHlSagVQgghhBBdnoRaIYQQQgjR5UmoFUIIIYQQXZ6EWiGEEEII0eVJqBVCCCGEEF2ehFohRJeyZMkSFEWhvLy8o4sihBCiE5FQK4QQQgghujwJtUIIIYQQosuTUCuEOCu6rjNnzhxycnKw2+0MGzaM//znP0Bd04BPP/2U3NxcoqKiuPDCC9m+fXuDffz3v/9l8ODB2Gw2srOzeeqppxos93q9/OpXvyIzMxObzUafPn3461//2mCdDRs2MGrUKBwOBxdddBG7d+9u2zcuhBCiU5NQK4Q4K3PmzOGtt97ilVdeYceOHdxzzz388Ic/ZOnSpeF1HnjgAZ566inWrVtHcnIyV111FX6/HwiG0euuu44bbriBbdu28bvf/Y7f/va3zJs3L7z9zJkz+ec//8nzzz/Pzp07efXVV3E6nQ3K8etf/5qnnnqK9evXYzabueWWW9rl/QshhOicFMMwjI4uhBCia/B6vSQmJvLll18ybty48Pxbb70Vl8vFT3/6U6ZMmcI777zD9ddfD0BpaSk9evRg3rx5XHfdddx0000UFxezcOHC8Pa//OUv+fTTT9mxYwd79uyhf//+fPHFF1x66aWNyrBkyRKmTJnCl19+ydSpUwH47LPPuOKKK3C73URFRbXxpyCEEKIzkppaIcQZ27dvHy6Xi2nTpuF0OsOPt956i/3794fXqx94ExMT6d+/Pzt37gRg586djB8/vsF+x48fz969e9E0jc2bN2Mymbj44otPWZbc3NzwdHp6OgBFRUUtfo9CCCG6JnNHF0AI0XVUV1cD8Omnn9K9e/cGy2w2W4Nge67sdvsZrWexWMLTiqIAwfa+Qgghzk9SUyuEOGODBg3CZrORn59Pnz59GjwyMzPD63399dfh6bKyMvbs2cPAgQMBGDhwICtXrmyw35UrV9KvXz9MJhNDhw5F1/UGbXSFEEKI05GaWiHEGYuJieH+++/nnnvuQdd1JkyYQEVFBStXriQ2NpasrCwAfv/739OtWzdSU1P59a9/TVJSEtdccw0A9913H6NHj+axxx7j+uuvZ/Xq1bzwwgu89NJLAGRnZzNr1ixuueUWnn/+eYYNG0ZeXh5FRUVcd911HfXWhRBCdHISaoUQZ+Wxxx4jOTmZOXPmcODAAeLj4xkxYgQPPfRQ+Of/J554gl/84hfs3buX4cOH8/HHH2O1WgEYMWIE//rXv3j44Yd57LHHSE9P5/e//z0//vGPw8d4+eWXeeihh7j99ts5ceIEPXv25KGHHuqItyuEEKKLkN4PhBCtprZngrKyMuLj4zu6OEIIIc4j0qZWCCGEEEJ0eRJqhRBCCCFElyfND4QQQgghRJcnNbVCCCGEEKLLk1ArhBBCCCG6PAm1QgghhBCiy5NQK4QQQgghujwJtUIIIYQQosuTUCuEEEIIIbo8CbVCCCGEEKLLk1ArhBBCCCG6PAm1QgghhBCiy/v/itrxIleeIsYAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "fig, ax = plt.subplots()\n", + "epoch_lim = 6\n", + "\n", + "for key, val in architectures_TM.items():\n", + " y_avg = []\n", + " for y in val['epochs_y']:\n", + " y_avg.append(np.mean(y))\n", + " _ - np.round(y_avg[-1], 2)\n", + " ax.plot(np.arange(len(val['epochs_x']))[:epoch_lim], y_avg[:epoch_lim], label=f'{key} ({_})', marker='.')\n", + "\n", + "ax.set_ylim(0, 100)\n", + "ax.set_yticks(np.arange(0, 110, 10))\n", + "ax.set_ylabel('loss')\n", + "ax.set_xlim(0, epoch_lim-1)\n", + "ax.set_xlabel('epoch')\n", + "\n", + "pos = ax.get_position()\n", + "ax.set_position([pos.x0, pos.y0, pos.width * 0.9, pos.height])\n", + "ax.legend(loc='center right', bbox_to_anchor=(1.4, 0.5), framealpha=0)\n", + "ax.grid(axis='y')\n", + "\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAssAAAHHCAYAAABJIhU9AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAADGYklEQVR4nOzdd1iV5RvA8e85h8PeGxRkO1HcO/feq7RMK8uGlWY/28tRplmZLdOc5Ww4UjPNbSrurSjbATjY+6zfHy8cJEDBLLXuz3WdS3jH8z4vqNw8537vW2UymUwIIYQQQgghylDf7QkIIYQQQghxr5JgWQghhBBCiApIsCyEEEIIIUQFJFgWQgghhBCiAhIsCyGEEEIIUQEJloUQQgghhKiABMtCCCGEEEJUQIJlIYQQQgghKiDBshBCCCGEEBWQYFkIIYQQQogKSLAshKi0PXv28N5775Genn7X5vDSSy/RqFEjXF1dsbW1pXbt2rz33ntkZ2fftTkJIYT491KZTCbT3Z6EEOL+MGPGDCZMmEBcXBwBAQF3ZQ5t2rShcePGhISEYG1tzZEjR5g/fz5NmjRh586dqNWyBiCEEOLOsbjbExBCiKrYvXt3mW3BwcH873//Y//+/bRo0eIuzEoIIcS/lSzBCCEq5b333mPChAkABAYGolKpUKlUxMfHA6DX65k8eTLBwcFYWVkREBDAG2+8QUFBQalxAgIC6N27N5s2bSIiIgJra2vq1KnDzz//fNtzK17lvpvpIUIIIf6dJA1DCFEpx48f58MPP2TZsmV8+umnuLu7AzBgwADs7Ox47LHHWLRoEYMHD6ZDhw5ERkayePFi+vfvz6pVq8zjBAQEYGVlxZUrV3jmmWfw9PRkwYIFnDp1io0bN9KlS5dbzkWv15Oenk5hYSEnT55k7NixXLp0ifj4eFxdXf+2r4EQQoj/HgmWhRCVVlHO8rFjx4iIiODJJ59k7ty55u0TJkxgxowZbN26lQ4dOgBKsJyQkMBPP/3EwIEDAcjMzKRWrVp4e3tz+PDhW85j3759tGzZ0vx5zZo1mT17Nu3bt78zNyqEEEIUkTQMIcRftmHDBgDGjx9favvLL78MwPr160tt9/X1ZcCAAebPHR0dGTFiBEeOHCE5OfmW16tTpw6bN29m9erVvPLKK9jZ2Uk1DCGEEH8LecBPCPGXJSQkoFarCQkJKbXd29sbZ2dnEhISSm0PCQlBpVKV2hYWFgZAfHw83t7eN72eo6MjnTt3BqBfv34sXbqUfv36cfjwYRo0aPBXb0cIIYQwk5VlIcQd8+cA+J9SnM6xfPnyu3J9IYQQ/14SLAshKq2iYLhGjRoYjUbOnz9fantKSgrp6enUqFGj1Pbo6Gj+/LjEuXPnAG6rfnNBQQFGo5GMjIwqnyuEEELcjATLQohKs7OzA8qWaOvZsycAM2fOLLX9k08+AaBXr16ltl++fLlUhYzMzEwWL15MRETETVMw0tPT0el0ZbZ/++23ADRp0qRyNyKEEEJUkuQsCyEqrXHjxgC8+eabDB06FK1WS58+fWjQoAEjR45kzpw5pKen065dO/bv38+iRYvo37+/uRJGsbCwMEaNGsWBAwfw8vJi/vz5pKSksGDBgptef/v27bz44osMHjyY0NBQCgsL2bVrFz///DNNmjRh+PDhf9u9CyGE+G+S0nFCiCqZMmUKs2fPJikpCaPRaC4jp9fr+eCDD1i4cCEXL17E29ub4cOH8+6772JlZWU+PyAggHr16vHiiy8yYcIEoqKiCAwMZPLkyQwePPim146JiWHSpEns3r2bpKQkTCYTwcHBDB48mAkTJphXvoUQQog7RYJlIcQ/qjhYXrdu3d2eihBCCHFLkrMshBBCCCFEBSRYFkIIIYQQogISLAshhBBCCFGBuxos79y5kz59+uDr64tKpWL16tWl9ptMJt555x18fHywsbGhc+fOZeq4pqam8sgjj+Do6IizszOjRo2StrdC3MPi4+MlX1kIIcR9464Gyzk5OTRo0IAvv/yy3P3Tp09n1qxZzJ49m8jISOzs7OjWrRv5+fnmYx555BFOnTrF5s2bWbduHTt37mT06NH/1C0IIYQQQoh/sXumGoZKpWLVqlX0798fUFaVfX19efnll/nf//4HQEZGBl5eXixcuJChQ4dy5swZ6tSpw4EDB8zNCDZu3EjPnj25ePEivr6+d+t2hBBCCCHEv8A925QkLi6O5ORkOnfubN7m5ORE8+bN2bt3L0OHDmXv3r04OzuX6trVuXNn1Go1kZGRDBgwoNyxCwoKKCgoMH9uNBpJTU3Fzc2twna+QgghxL+FyWQiKysLX19f1Gp5fEmIm7lng+Xk5GQAvLy8Sm338vIy70tOTsbT07PUfgsLC1xdXc3HlGfq1KlMnDjxDs9YCCGEuL9cuHCB6tWr3+1pCHFPu2eD5b/T66+/zvjx482fZ2Rk4O/vT1xcHA4ODndxZkIIIcTfLysri8DAQPmZJ0Ql3LPBsre3NwApKSn4+PiYt6ekpBAREWE+5sqVK6XO0+v1pKamms8vj5WVVan2u8VcXV1xdHS8A7MXQggh7l1arRZAUg+FqIR7NlEpMDAQb29vtmzZYt6WmZlJZGQkLVu2BKBly5akp6dz6NAh8zFbt27FaDTSvHnzf3zOQgghhBDi3+WurixnZ2cTHR1t/jwuLo6jR4/i6uqKv78/48aNY8qUKYSGhhIYGMjbb7+Nr6+vuWJG7dq16d69O0899RSzZ89Gp9Px/PPPM3ToUKmEIYQQQggh/rK7GiwfPHiQDh06mD8vziMeOXIkCxcu5JVXXiEnJ4fRo0eTnp5OmzZt2LhxI9bW1uZzlixZwvPPP0+nTp1Qq9UMGjSIWbNm/eP3IoQQQggh/n3umTrLd1NmZiZOTk5kZGRIzrIQQoh/Pfm5J0Tl3bM5y0IIIYQQQtxtEiwLIYQQQghRAQmWhRBCCCGEqIAEy0IIIYQQQlRAgmUhhBBCCCEqIMGyEEIIIYQQFZBgWQghhBBCiApIsCyEEEIIIUQFJFgWQgghhBCiAhIsCyGEEEIIUQEJloUQQgghhKiABMtCCCGEEEJUQIJlIYQQQgghKiDBshBCCCGEEBWQYFkIIYQQQogKSLAshBBCCCFEBSRYFkIIIYQQogISLAshhBBCCFEBCZaFEEIIIYSogATLQgghhBBCVECCZSGEEEIIISogwbIQQgghhBAVkGBZCCGEEEKICkiwLIQQQgghRAUkWBZCCCGEEKICEiwLIYQQQghRgXs+WM7KymLcuHHUqFEDGxsbWrVqxYEDB8z7TSYT77zzDj4+PtjY2NC5c2fOnz9/F2cshBBCCCH+Le75YPnJJ59k8+bNfPfdd5w4cYKuXbvSuXNnLl26BMD06dOZNWsWs2fPJjIyEjs7O7p160Z+fv5dnrkQQgghhLjfqUwmk+luT6IieXl5ODg4sGbNGnr16mXe3rhxY3r06MHkyZPx9fXl5Zdf5n//+x8AGRkZeHl5sXDhQoYOHVqp62RmZuLk5ERGRgaOjo5/y70IIYQQ9wr5uSdE5Vnc7QncjF6vx2AwYG1tXWq7jY0Nu3fvJi4ujuTkZDp37mze5+TkRPPmzdm7d2+FwXJBQQEFBQXmzzMzMwHQ6XTodLq/4U6EEEKIe4f8rBOi8u7pYNnBwYGWLVsyefJkateujZeXF8uWLWPv3r2EhISQnJwMgJeXV6nzvLy8zPvKM3XqVCZOnFhm+6ZNm7C1tb2zNyGEEELcY3Jzc+/2FIS4b9zTwTLAd999xxNPPEG1atXQaDQ0atSIYcOGcejQodse8/XXX2f8+PHmzzMzM/Hz86Nr167ydpQQQoh/veJ3VIUQt3bPB8vBwcHs2LGDnJwcMjMz8fHx4aGHHiIoKAhvb28AUlJS8PHxMZ+TkpJCREREhWNaWVlhZWVVZrtWq0Wr1d7xexBCCCHuJfKzTojKu+erYRSzs7PDx8eHtLQ0fvvtN/r160dgYCDe3t5s2bLFfFxmZiaRkZG0bNnyLs5WCCHEvW7XxV00/b4pq86vuttTEULcw+75YPm3335j48aNxMXFsXnzZjp06ECtWrV4/PHHUalUjBs3jilTprB27VpOnDjBiBEj8PX1pX///nd76kIIIe5hXx39inxDPl8e/RK9UX+3p0N2YTanrp/CaDJWeMy1vGvEpMfcdJz0/HQuZF6409MT4j/rnk/DyMjI4PXXX+fixYu4uroyaNAg3n//ffNbSK+88go5OTmMHj2a9PR02rRpw8aNG8tU0BBCCCGKnbx2kpPXTwKQkpvC9gvb6Vyj881P+hudvn6acdvGkZSThJ+DH4NCB9EvpB/uNu4YjAb2Ju3lx3M/sv3CdgwmA6PqjeKFhi+gUWtKjbPr4i5e3fUq2YXZvNjoRUbVG4VKpbo7NyXEv8Q9XWf5nyL1JoUQ4t9FZ9CxKnoVD1R/AG877zL739z9Jmtj1mKtsSbfkE9z7+Z82+3buzBT+CXmFybunUiBoaDUdguVBa2rteZ82nku51wuc15r39ZMe2AaTlZOmEwmvj3xLZ8f+RwTJT/Wu9TowpTWU7DVlq70JD/3hKi8ez4NQwghhKiqr499zeR9kxmzZQw6Y+mawmn5aWyM2wjA5DaTUavURCZHEpse+4/OUWfU8eH+D3lj9xsUGApoW60tvw/+nUmtJlHfoz56k54dF3dwOecyjpaODK89nFV9VzGt7TSsNdb8cfkPHlr3EEeuHGH89vHMOjILEyYGhw3mzeZvYqG2YHPCZh7Z8AgJmQn/6L0J8W8iK8vIb9hCiMq5kHmBOSfm8HT9p6nuUP22x9l7eS/rY9czoekEnKyc7uAM7z/7kvaxMW4jE5pOwE5rV/5Bp9fA4e/gxlxetQaaPgVhXcscnp6fTrefupGrV2oJj288nsfrPW7eP+/EPGYenklt19qs6L2CsdvGsu3CNobVGsYbzd+45Zy/3h6Dg7UFw1vUqNrNFilOq/jm2DccvXoUgGcaPMOzDZ5FrSpZwzqXdo6tiVupZl+NLjW6YG1Rkl4YlRrF2G1juZR9ybxNq9byZvM3GRQ2CICjV44yfvt4ruZdxUHrwIcPfMgD1R8A5OeeEFVxz+csCyHEvWLBqQWsjl5Nnj6PGe1m3NYYp66d4sWtL5JvyCfUJZSRdUfe4VneP3J1uby681VS81Op4VijVEBrZtDBupcg93rZfSmnYOxx0JT+Ubbo9CJy9bk4aB3I0mXx1dGv6FKjC9UdqmMwGlgZtRKAYbWGoVKpGFprKNsubGNtzFrGNhpbcdAOxFzNZtrGswB0qeOFl2Pln49JyUlhdfRqfj7/szmtwk5rxwdtPqCjf8cyx4e5hBHmElbuWDVda7Ki9wpe2fkKey7vwdPGk087fEp9j/rmYyI8I1jRewXjt4/n6NWjLDi5gLbV2koOsxBVJMGyEEJUUlRaFADbL2wnqzALB0uHKp2fkpNiDpRBecjs3yI1P5Xx28fjYOnAey3fw83G7ZbnrIxaSWp+KgDbLmwrP1iO2aoEynYe0HVKyfbf3oTMSxC1Aer0NW9Oy09j6ZmlgJJiseTMEg4kH2BK5BS+7vQ1uy7t4nLOZZysnOgR2AOAFj4tCHAMID4znnUx63io1kMVzvnYhXTzx5tPp5S7upxRkMHozaNJzEwstT1Xn2uudOFo6Ujf4L48XPth/Bz8lAO2fwinVsHg+eBVt+IvXBEnKye+6vQVkcmR1HWrW+67FB62HszvNp+vjn3FI7UfkUBZiNsgOctCCFEJRpOR82nnASgwFPB7wu9VOj9Pn8cLW1/gSt4Vc5B94tqJvzQnnVFHji7nL41xJxQaCnlp20scSjnE9gvbeWjdQ8ovAiYTZKWUe06uLpcFpxaYPz965SjX8q6VPfD4CnRARt1+0GBoyatx0Yr8/jmlDl90SllVru1am45+HXm7xdto1Vr+uPQHv8X8wvKiQHpAyABzWoNapeahmkqAvDxqOTdmJ5pMplLzujFY/u1Ucrn39umhTzl9/TTZuuxSL6PJSGOvxnzQ5gO2DNnCq81eLQmUL+xXguWrZ2HpQ5B9pdyx/0yj1tDKt1XpQFmXD/kZ5k+1Gi1jG43F3ca9UmMKIUqTYFkIISrhUtYl8vR55s/Xx66v9LlGk5E3d7/JmdQzuFi5ML/bfGXM7EvmldXb8erOV+m4siPxGfG3PcZfZTKZmLxvMoevHMZea0+AYwApuSmM/HUkq1b0h4/D4JsH4OB8yC9psbziyFek5qdS3WCkdkEhJkxsv7C99OD5mXB2Pa94utPp+jaOXDlSsq/x46BSQ/wuuKKkRaTmp7L0rBIMP9vgWVQqFYFOgTxV/ykApux+iz+S9qICHqz5YKlL9Q3pi42FDdHp0RxMOcj1vOssOLmAPqv70GFlB2YdngXA0YslQejemOtk5JV+ePBQyiF+Ov8TALM6zGL9gPXm17YHt7Gw+0L6BPcplX+MQQe/jAVMyj1lXIAVw5Wgt6qMRljYC6YFKEF31K9guPs1pIW4n0mwLIQQlXAu7RwAnraeAOxP3k9yTvkri3/25dEv2ZywGQu1BTM7zKSWay0CHAMAJYf5duTr89l2YRu5+lxWRVfcgS4xM5HDKYdv6xo3On39NAeSD5RpmLHw1EJWR69GrVIzo90MlvVaRnu/9hQaC3mnIJYpbi7oko4peccf14LVY8hd1IcFJ5VfGJ5OTaNrjvIg3pbELaUvenYdsSo9v9vZUmDU8d6e9yg0FCr7nP2gZk/l4wNKybdFpxaRp8+jtmtt2vu1Nw8zqt4oAq1cyVApK8ZtcvPw2zunVBDpaOlI76DeALy+63U6/9iZTw59Yq4iMffEXH46t5ozl5WA38VWi95oYntUyQpwoaGQiXsnAjDYtz0dribif36b+eUev7f8wHXPLLhyGmzd4PGNYO0EFyLhlxeV1fkbJR2H+D8q/D4Rtx0uHVQehjy3EZYNhZnhsO0DSJdGJULcDgmWhRCiEoqD5Va+rWjk2QgTJn6N+/WW50WlRjHnuJIq8F7L92jk1QiAcPdw4Pbzlk9dP2XuOrc+dn25Xd8KDYU8vvFxRm4cyS8xv9zWdUCp3vHw+od54rcn6PlzT7498S3X8q6xLXEbnx76FIBXmr5C62qtsbe05zP3BxiTlo7KZGKFowOjajfjmkco6HLg6PcsTz1CmkaDH1p6d55Bx0IlIIy8vJfswuySCx9fwXKHkrzw2IxY5hcF2QA0fVL589hyUjMSWXZ2GQDPRTxXKjfX0gTvXC1ZwR+WmaUEqN8PhJySBweH1hoKKE1K9EY94e7hTGw10ZxLPXnfRAyWsTjZaBnWzB+ATadK0kzmnZxHXEYcbhZ2jNv7nRLs3vhaMRy+HwA5N6SbpMbCjunKx90+AP/mMGQRqDRwfAXs/gQKsuDgAvimHXzTFhb2hPPlpwFd3KSsgP9iaMm1+k8rAXjWZdgxDX56soLvsBDiZiRYFkIIlHJjnx/5vFQprhsVB8uhzqH0DlZWINfFrrvluMUBXJcaXegX0s+8va678gBXcRe5qroxJSElN4VDKYfKHLMpYRNX8pSVz3f3vFs6jaGS4jLieHnHyxhMBjQqDZeyL/HZ4c/o8kMnJmwbhwkTD4U9yMO1HlZOuHgQ9ZoxPJOeyeeuLbHX2nMkP5kHPRw5OvALcps8xkJ3pUnI063fxSLiYYIaPk5AoQ6dycDui7uUcTKTyInbyVoHpTJFLTulu97c43NL0k6C2oNbKBRmMXfnm+Tp86jrVpd21duVvoljS2lyPZF3MnU8V+8pWvf6CrR2ELcD5rSHpGOAUn3itWavMaLOCH7o8wNLey1lYOhAxjUaR5caXTCY9FhX/45afnq611PuYXvUFfJ1BuIy4ph7fC4AryZfwsloAr/mENaj6NUdLO0hbifMaY/uwmHm7Igm/YcXQJ8Pge2gftGDhcEdoOdHysdbJsGMmrBuHCQdNd9S3qZJpVadC/QGpq/YjE/ydgBm6gfykXE4jD+jPDAY+AA0fqzK338hhATLQggBwIcHPmTO8Tl8dvizcvcXB8thrmF0rdEVrVrLubRz5u3lySzMZEPcBoCSYLLIjSvLt1Pu/tgVJcCztVA6s5UXuC8/uxwAFysXdEYd47aNq/CXgfJkFGTwwtYXyCrMIsIjgh3dvmOyS1Ma6IzoTUYKMNI8L59XI1ei2jUDLh6CZcPAUAA1e9Ku92yW9VpGsFMwV/Ou8vixTxhvVUCasQB/B396BfVSLtR6LB3zldzfraeVnGNO/sg6e1ty1Goo9ODAwY5oC2tTaCxk8r7JytdMpULX9AmmuLnwfepRoOyqMvpC2KmU+RvS5EWebfwi6nqD4MnfwSUQMhJhXlc4tgKAR2o/woSmE6jlWss8hFqlZkrrKTiqAlBb5HDZ6ksCPTR4O1qTU2jgj+irTN43GZ1RR+sCA92zMqFWbyWl4uHlRa8VyjVdg5Wc5PndcN48Huek3RjVltD7U7hx3k1HQbOnlY91OeAWSlqbd+lt+JhckxU2V48xeeZnrDx4gfhrOQybsw/7E4vRqEycs21EjKkaa45dIqNQDfUGwchflAcjhRBVJsGyEOI/LzYj1pxSsT9pf5ngNVeXy4UsJd8zzCUMJysn2lZrC9z8Qb810WvI0+cR6hJKY6/GpfbVdK2JhcqC1PzUclsZ/xLzC89veZ70/PQy+0wmk7mZxdMNlIBqU/ymUu2ST18/zbGrx7BQW7Ck1xJqu9YmNT+V57c8XzrVoQI6o46Xt79MQmYCPpZOzLyWjtPXbeh/+Ce+v3iRn67n845lDT7LKECbnghbp8C3HSHnCnjVg4FzQK0hwCmAJb2W0KVGF/RGPX9c/sM8bwt1UfVSO3c6Bill3HZeO0ahvgDT8eUsc7QHID+1BaAmLbE3JqOW/cn7WRuzlmt51xh1fTcrHJVUjTE1epu/L2ZHl0DGBUz2Xnx0vSVjlhwmp0APXnVg9DYI6aKs7K4aDb++pjxsVw5brS1WqaMw6hxI1yfSb00/1H4fYRv0CW8cGM6B5ANYm+CtK8kYPcMZr3uOievPkK8zlAziWRue2kqCW1u0pkIetNgBwFemgSRQtiU33T6AAd/AYxswjdnP+AttOKnz4SdNdwD6pi3ilR+P0X7Gdk4lXmGoxTYAQnuPo5a3A/k6Iz8cuiFPWcrGCXFbJFgWQvznfXPsG3PO7/X868RlxpXaH50ejQkT7jbuuFq7AphTMSrKFzaajOaV3aFhD6H6UwBupbEizFVpOGEus1Z0jM6oY/qB6ey4uKPch/fiM+NJL0jHSmPF8NrD8bbzJluXzY4LO5RqCJSsKnfx74Kfgx+zOs7Cw8aD6PRoXt31KhkFGWQVZimvgkyyMi+Vek3d8RqRyZHYGk18HnsWt9idysWDO8KQRYS9eIohw9ZhNz5KCej8Wyr77Txh2DKwKsk1ttPa8XG7jxnbaCwqVNR0qUnPwJ6l7im83Tt4GIzkqEzs3/oGB9PPE2NpCUYtxqzGfDeqGc2qh1J4rRMAk/d8yJBfHuTItRPYoeGL5CuMTr5YdlV518cAnAkaxZe7L7P+RBLjVhzFaDSBjYuy4tv2f8rxkV/D4v6QnqiUXit+FeaSXaAnLkVL3sURWGtsuJp3lSzjJTRWV8g1KXnLz6emUc3anZctXuPnU+ks+COewbP3cDEt1zyl3+MK6HD5aT7TDwQgRhPMZ3k9eWLhATLz/xSoayyU1eCA1qw/mcy2qKtYatS0HTkJk4UNDdSxPOh0BoBRLsdwJQscq6Gq2YsRLQMA+G5fgnKvQojbJk1JhBD3jJmHZvLd6e9o59eOwWGDaeHTolT7379DbHrJqnI1+2pcyr7EgaQDBDkFmY8xp2Dc0E3tgeoP4KB1MOcLN/VuWmrcvZf3kpiViL1KQ+9VL4PLF/DgInAtGTfcPZzT109zMmoN3X58EUI7Q78v2XNpD+kF6YBSIeLPzTqOXjkKQF23ulhqLOkV2It5J+ex7vh8ui5/kozAtmzQKeXUih9a87bzZlbHWTy28TF2XtxJm+Vtbvm1UZlMTLtyjZpWrtD8EWg0AlwDSx+ktSmpfZwWD1aOYOtadiyVit7+w5m9zhWtzg29QYXFDd9atb0nHRyCWZkbx5aon8goWi0uzGjEk63q0DbUgxZBbkxZb8sPSUcpsE6mIC8bQ4Enthe70E49FcPZdaQvfwbntk+BbyM48h1kXMBo78XTp+uZr7X5dArTf4vitR61lLbZnd4G3whY9Qwk7FaqR/yJ3rsVPVVNOGHZmtWDNxKTHoO+IJdFy76nnXE/9blMHaOab4PfZ/UxFdZaNTZaDScvZdLn8918+XAjXOwsGbv8CEaTmiuNx2PqMAkHgx1u3xwh5moOzy89wvyRTbDQlP47n5GnY+IvpwF4rkMwATUCoPlo+OMzprmt57ERo6m1/mPIA5o8DhoL+jf0ZeqvZ0i4nsuO81fpUNPzlt9vIUT5ZGVZCFFl1/Ous/vS7lKvwymHMRgNFZ5TaCgkJj2mwv0Xsi6w8NRCCo2FbE7YzNObn6bnzz2Ze3wuGQUZFZ5XKSYTXD4Kurwyu2Yfn40JEx38OtA/pD8AB1IOlDqmvGDZSmNF14CuwJ/yhQ06OLSQ5b+9AEC/9HRsC7Ig+TjM6QDRJVUM6rrWAeBE3GbIToYj38PVqFLjHb96nKu5V0vNpzgFI8IzAsBc8mxX6kkyCrNYfWkHBYZCajoG0NCzofm8eu71mNp2aqU6D1objbyh9qJ937nw0ino/G7ZQPnPXALKDZSLfbktmmsZVuyNzub1n0+USXfp2OhZADbb2bLV1gYAV0N7xnYOBUCrUTOxbwPG1HsD9E7oMiLIjR9DbGE4PxvaoMGI89llMLejUjVixzQANjgN40I2BLrbMX2w0g569o4Yfjh4Q4pC7T7w1FbwKhsoAzgn7+FLy1msNzyN685PaHpoOS2XDmd2/kqGFcZTU2diV70pvH9MySGf+VAEv7zQhnrVHEnL1TF8XiTDv40kp9BAq2A33utbF5WzH55urnw7sgk2Wg07z11l8rrTZVaCp208y9WsAoI87Hi2fbCysdWLoLVDdfkIdc5/jfryIVBroZHSrMXW0oIhjZWGJ9/tTbjpt00IcXOysiyEqJIz18/w2MbHyNXnltnX3Kc5Hz3wES7WLqW2J2YmMnbbWKLTo3ml6Ss8WufRMufOPT4Xg8lAI89G1HStybqYdVzKvsSsI7PYGL+R5b2Wo9Vob2/Sp1fDD4+Be00YugTcleArJj2GjXEbAeXBsOJueAeSD2Aymcxv6ZcXLAP0CurFT+d/Yl3MOpp7N6dnUE9YM4ZLp39kR3VfQMVD3i2h63DY9YlS//b7wdDpHWjyOOH7FyrTs7LEYO+FJjuF7H1fsy1jNwCu1q6k5qey7cK2Uk00ileWiwPhEK0jtfQmzlqo2Fi9DstRfrkYmnAKVdSvUKsk5aFLjS508OugpI5sfF2pUay1hRGrwLOkxbJabYGF1ub2vt7luJyex4oDSnCqVsGqI5cI8bRnTIcQ8zHNAjpj/4cFGRrlc2NODab27oqtZekfVWNad+DpFjsx3BBsZ+Z24oWvvqVDzq/01kRimax0Ryy09eLl2AgA3u9fj1Yh7lxMzWXW1mjeWHWCAHc7mgYUBfgeNeGZXVBcy7lYVhIbvptBo+vr8DakwZ7Pzbty7Pz5PL0Vmyw7kbDfHjAxoVtNutfzAeDHZ1rxxqoT/Hz4EtdzCgl0t+OrRxqhvWH1uF41Jz59qAHPfH+YRXsT2Bp1haFN/RnSpDoXUnNZGqm0zf5gQDhWFkVfHDt3aPYU/DETtk9VttXtD/YlK8iPtqzB/D/i2BZ1hQupufi52lbqeyWEKE1WloUQlXY19yovbH2BXH0u3nbe1HatbX7ZWNgQmRTJ0HVDOXP9jPmcXRd3MXT9UKLTowHKLc92IfMCa2PWAjC+yXjeaP4GWx7cwpTWU3CxcuFc2jkWnlpY7px0Bh0/nvvx5l3sjv+g/HktSll1jFLSLmYfU1aVO/l3opZrLcLdw7HWWJOan0psRiygPExXUbDc2KuxuQHHq7teZcbmF9AfX8EKR0dMKhUtPBsT+OAyqNULHt9QtOpngi0T4dN6BEbvwMZoJE+tJrbruwBsiVlLgaGAQKdA8y8VWy9sNV8zoyDDPDcHgll9IAbj8ofplZGufH1t4KJGhYNJRc+MVFg+DH5/D3JL6gxbqC2wPLwYywPfYglYDvgGS78WWFo5mF93MlAGZVW50GCkZZAbE/spKREf/RbFxpNJ5mO0Gi3NvEse0Kvn2Jv2FaQPWGjUWFlozC8PR2vGPvEY72pepGn+l6zyegFjUCcm8iwFJksGNapOqxCl3fO4zmH0DPdGZzDx9HeH+GRTFJ9sPqe8fj/Ppqg0sLAqebkE8H7uQFoXzOJM+2+gTn8IfxBGrEH14iEWqPoTm2eHwWhiQMNqPFe8+gtYazV8PKQBHwwIp0sdL+Y/1hRnW8sy99O9ng9T+tfDwdqCC6l5fPRbFK2mbuXJRQcBeLBJdVoEuZU+qWh12azZ6FK7A93taBvqjskE3++T1WUhbpcEy0KISsnX5zN221hSclMIdArkp74/sbLPSvNrSc8l+Dn4cTnnMo/++ii/xPzC3ONzGbNlDFmFWTTwaECERwR5+jze3/d+qbfgvzn+DQaTgdbVWtPAowEANhY29Avpx4SmEwAlsE3MTCw1J5PJxMS9E5m4dyIvbH2h/DSQwhyIKeoM51kXCjJh2VCiN73Gb/G/AUprZABLjSUNPJXrH0hWUjFSclPIKszCQmVBoFPpNAS1Ss3M9jN5Mlxp9rDo8nae8fZklbOyUjm07g0r6BZW0HcW9J6pvF1emI3GyZ+6bkoqxkkbW3ALZZ218t9y76DedPTvCEBkUiRZhVkAHLuqlIzzs6/BqPmnMa59EfXFA3TTaVChIqNQ6TDXr/YwbItLj+3+FD6uqTSliN8NMVthwyvKvk7vQJ2+5X/T75BL6XmsLEp5GNc5lEdb1GBkyxoAvLTiGMcvprMt6gqjFx/k14O+ykl6Rz7tM7xK1wnxtOerRxqRrXbgpYSWDMx6mSWpYbjYanmzV23zcWq1io+HRBBezYnUnEJmbY1m1pbz5tfo7w5xIL7kl4urWQVcSs/DqNJQvcUgJfd80FwIao+tlSVtQz0AaOTvzNSB4aUfMkTJ1364uT9zRzQh0N2OigxvUYP9b3RmxpAGNK7hgt5oIi1Xh5udJW/0rF32BDs3JXcZwLs+VG9a5pCRRQ/6rTh4oXRlDiFEpUmwLIS4JZPJxNt/vM2JaydwsnLii45f4GjpWOqYUJdQlvVaRttqbSkwFPDG7jeYdWQWJkwMCRvC/G7zmdR6Elq1ll2XdvFbghKoJmYmmnN0n2vwXOkLF2TT+/RWmjsGUWgsZNK+SaWC7IWnFrImZg2gVIgormlcSsxWpTSYcw0YvR2ajUYPfBb9AyZMdPbvRE3XmubDm3opAUdxsFy8qhzgFIClpuyKoEatYWyjsXzi0hwbo5FIG2vSTTq87bzLNscA5QGsUZugw1swejvhvkoViZPXTnKl4TAira0B6BnQgyCnIAKdAtEb9ey+pKRmFDcWMeT684RuKQM1u9Gb1EzVj6WOSyPzZYbWfhh6TIMhC5VAylAIJ36Ahb3guwFgMihNMNqMLzvHSth9/hpjlx8h4XrOLY/9cls0OoOJVsFuNC9aHX27dx3ahrqTpzPQ94s/eHzBATadTqEwsy6ehQ/xVtPp+DrZV3lebUM9eK+P8gvI0QvpALzZqw6udqW/dzaWGhY+3pQXO4YwomUN86tZUUrGGz+foFCvVBY5flEZJ9jDHgfrsqlA7/Suw8tdwpg3sinWWk2V5/zneQ1uXJ2fnm3FppceYHyXsApXowFo9yp0ehcGfVtuabgOtTyp5mxDeq6OtcfKligUQtyaBMtCiFuafWw2G+M3YqGy4NP2n+Lv6F/ucU5WTnzR6Querq+saGrVWt5r+R7vtHwHS40lgU6BPBX+FADT9k8jszDTvKrcplob6nvULz3glkmoDs3nndiTWGmsiEyKNAfWN7ZaLj5vzvE55hbQZmeKHpar1RssLEnr+AbPhLdlu50tapOJZ5zqlTq8mU8zAA6mHLxpCkYpySfpcuRnll5OoYa18lb/8NrDS+oI/1m1RtBuAti5mTv5nbh2gl/t7TCpVDTMz6f69XgAOvopq8tbE5VUjOJ85RbJx3nRYjUAX1iPZm1WTY6eVoL+dtXbUcNRWbml7gAlD3f0dqWDm2VRAFq9GfSZVeXauyaTia+3xzBifiRrjl5m9o6KH9oEuJiWa36Qblznkq+hhUbNFw83IsRTmY+TjZYnWgey+aX2bHnqLR6q37pK87rRoy0DGFG0ct06xI1BjaqVe5ybvRXju9ZkUr965tecEY1xt7fk/JVs5uxU7u1YUdDdoLpzueP4u9nyQqdQXOwqCGhvU5iXAy92CqWBX/nXBZRqJG3HK/nW5dCoVQxvoXwtlu1PLPcYIcTNyQN+QtwHlp5Zyuro1XzZ6Us8bD3+0WvvvrSbr459BcDbLd8uUyLtz9QqNc83fJ6O/h2x09qVBG1FRoWPYkPcBuIz43lj1xvmFdMyq8oXD8H+OQD4Z13j6SbDmBW/ho8OfISnrSev7npVabVc8yFeavwS3X/qTnxmPL/G/Uqf4D7KGAYdnFMe4KN2b85cP8O4beO4nHMZG5WGKSnJ1Dy5Dho9ab5sPbd65rzlmPSYWwfLRgP8MhZMBkJCerFy4DdEpUUR4RFxqy8tUNLJ73zaeVYVPVjWOztXuffAtnT078i8k/PYdWkXubpcTlw9DsDIwjPoVRZY9JzGE+EjObniGL+fCUdXMJpQ/3JWtH0bKq+uUyBhLwS0Bq11peZYLKdAz4Qfj7HhRLJ52+bTKUzpb0KjLj/o/nJbDDqDidYhbjQLLF0pw8lGy4/PtOTohXRaBLn95VXZG73Xpy5d6ngR4edcJi3iZpxtLXm7dx3GLj/KrK3R9Krvy7GLygOTEX5Od2x+/6SHmvphMBp5qGn5v+QKIW5OVpaFuA8sPLWQM6lnzKuL/6RV55WmGINCBzEwdGClz6vjVqdMoAxKXvA7Ld8BYMfFHRhMBtpWa0u4xw0luwx6JQDFBEWrs4/l6glxDiGtII0nNz1Jnj6PFj4teLXZq9hp7RhZVymZ9c3xb0pWlxP2QH462Lrxi+4aj/76KJdzLuPn4MeSdp/RNTcfYrfBtWjzpbUarbkk257LkUSlKsFyqEto+Td6cL5S5cLSAXpMw1ZrS0PPhqUCtMx8HXmF5eeL+tj54Grtit6kJyYjBguVhq45uXB2PWRcop57PTxtPMnR5bBo5zsUGAtxNBiw19lT+MgaaPokjtZa5jzamBc7hmLIC2L21stczy4o93pYOUBYV7CsOHe2PHHXchjw1R9sOJGMVqNiYt+6OFhbcC27kCOJaeWecyG1/FXlGznbWtK+pucdDZRByUtuG+pRbtrErfRt4EvbUHcK9UbeXHWCY0VpGPUrWFm+17naWfJ8x1A8HKzu9lSEuC9JsCzEPS45J5mkHKViQHFFiX+K3qhnX9I+AHMN4juhqXdTBoQMMH/+XMSfVpX3fQUpJ5QOa72UDmzaqPW82+Id8yEBjgHMaDcDrVoJhobVGoazlTMJmQkluctn16EDpvmF8cYfb1JgKKBttbYs67WM0BrtIKybctyBb8vMD+CzP9YRm6508yt3ZflqFPw+Ufm487vg6FvmkItpubSdto1Gkzfz6o/HOZKYVirvWqVSUc+9JBWkbfUHcPZrqeQUH1qIWl9AB9vqACxMVKp4BOZrOdVrLbYhJY1F1GoVL3UJo351J/J0BubsjC073yoyGk3sPn+NMUsO0/XTHZxLycbTwYrlo1syslUAnWoplSo2nU4p9/zZO2LQG020CXEvKc92H1CpVEzpXw8rCzV7Yq6TnqvDUqOmls+t61MLIf59JFgW4h5X3IACICbj5vmhd9rJayfJLMzE0dKxVEB3J7zc5GWaejdlRJ0RpcdOSyipG9tlslKiy8IG0hOJMFkwJmIMdd3q8nnHz3GyKnlb3E5rx2N1HwOU9tV6g47rUesZ7e3J94VKqbrR9UfzRacvSs5rquRPc3QJFGSbx2rmreQtF1ieApURS5U9XrZepW8gNxWWPgSFWVCjNTR5otz7/HJbNBl5OvJ0BlYcvMCAr/bQ47NdfLc3Hp1BeYCsnlvJ/fcO6g1Ni9JCImfDxzXpeEpJJclVK/9lp9n2oX2TP+V3owR544oaeCzem8C1ilaXbyGnQM9X26NpP2M7w+dFsv5EkvkBvXUvtKFxDaWOdte63gD8diq5TIORjDwdPx9Wvu431lK+X9RwszM3QwGo7eNQUuNYCPGfIsGyEPe4Y1eOmT++WQe8v8Oey3sAaOHTouKH1W6Tk5UT87vNN5eGA5ROe+tfBl0u1GgDDYeDpS2EdFL2n1nHMw2eYXnv5QQ4BZQZc1itYbhYuZCYlcjMnW/wkKOJgzbW2FnYMbPDTF5o+ELp9tnBHZX20wWZcGKleXNdt7pYqKxQqZQAsCDXk4w8Xcl5+kJY8SikxSlVNh5crLRN/hMlDeEiAJP712Ngo2pYWag5m5zF22tO8ci3kVzLLjCnoNhr7Wnn107pJmfvrcwrP4Omlp7YUZJO8EKbHhXm4Xao6UkDP2fydAa+ucXDd+UpTreYvjGKxNRcHKwsGNGyBhtebMvSp1rg6ViS59wuzANLCzUJ13M5l5JdapyfDl0kT2cgzMueFkH3z6ryjZ5qG0RNL2U1+aYP2Qkh/tUkWBbiHldcKgwgNT+V1PzUco/bn7SfL49+ae5Cdyf8cekPANpUa1N6x6GFSq5uVVw5W6Y5RhmnfobozaCxhN6fllRqqNVL+fPs+ptewlZry2P1HgNgUeJGUiwsCFBZsbTXUjr5dyp7glpdsoq7/1slWEfJW7Y1ljSW0OV58+0uJR1DCejHQ8JuTJYOrK83kzXny1/B/XJbNHqjibah7jzaogafPBjB/jc6807vOthbWbA/LpU+n+/GzlCH0fVHM7XtVKw0VqDRwsBvlNXq4T+hHXsMF62y2q1CTbuARuVeD0qvLn+3L4GrWaXndjEtlynrTrP22GUK9KXzqLeeTaHvF7vN6RbTB9cn8s1OTOpXjzq+pUsFAthZWdC2qNHHplMlD/0ZjSZzE4xHWwZU6QG7e4lWo+br4Y14pLk/ox8IutvTEULcJRIsC3EPy9Xlcjb1LKCkGUDFq8vv7nmX2cdm8/D6h4nLiPvL107PT+fk9ZMAtCyqBQzAuU3Kw3frXoLIOZUbzGiEn0YpzTHWjSv/mLw0+PU15eM248HjhhzhsO6g0ih5zGnxN73U0JpDcbVWVjLb5+SyNPxFgpxvEuhEPKykeVw5BYl7AdAbjGSklTycaCzwZsEfcaTlFMLeL+HId6BSsy7sfcb8nsfY5UfNTTeKXUjN5cdDyqryuBvezney1fJEm0BWj2lNsIcdSRn5PDQnEi99f9r7tS8ZIKi98gtDSGdQq0m/qjSlCLCvg43FzbvrtQ/zIMLPmXydsdTq8p7oa/T5fDff7o7jxWVHaDl1Kx9sOEP0lWxmbTnPqEUHycrX07iGC+teaMODTfzKtJr+s651lfSU306XBMu7o68Rey0HeysLBjQsv2zb/SLIw573B4RT3UVaRQvxX3VPB8sGg4G3336bwMBAbGxsCA4OZvLkyaVy40wmE++88w4+Pj7Y2NjQuXNnzp8/fxdnLcSdc+r6KQwmA562njTxagKUHyxfz7vOxWwlMIvNiOXh9Q+z/cL2v3TtfUn7MJqMhDiH4G2n5KZSmKOkSRTb+CpEb7n1YGd/gRQl8Ob0GojaWPaY3ydCzhVwC1Xqxt7I1hVqtCoa69ary4uaT2JWylU+u5aOQ60+xFzN5qFv9vL26pOcvpxZ+gQbF6g/RPl4/1wATl7OJDczwHyIv30IOYUGNq1fCZveAuBM/Vd54aC7+Zg3V51gX+x18+dfbC1ZVW5co2waQoinPavHtKZrHS8K9UZe+ek4n2w+V+49xV/LISkpmMJLI/i4/bSb3j+UXl3+PjKBK1n5zN0Zy/B5kaTl6gj1tMfb0ZrUnELm7Iyl8yc7+GTzOUwmGN7Cn2V/Sre4mc61vVCr4OSlTC6l5wFKvjTA4MbVsbeSCqVCiPvbPR0sT5s2ja+//povvviCM2fOMG3aNKZPn87nn39uPmb69OnMmjWL2bNnExkZiZ2dHd26dSM/P/8uzlyIO6O4AUVDz4YEOytpAeVVxDh1/RQA1eyr0cizEdm6bF7Y+gJfH/0ao8lYtYsW/TL6x2UlBaO17w3NIbZPhYxEcPJTHrwzGeGHx5WqEBUxGmF7UYDnqFR1YMP/Sj1QR+I+OLRA+bjPTKU19J/V6q38eYtgGSDg4hE65OahDmiL0cqZl1ceIzIule/2JdBz1i76ffkHKw4kklNQVGKu+EG/M2shK5l9sdcx5lXHCg88bTx56YG2ALieWgSYSAsdxMDDSlvsES1r0Ku+DzqDiWe/P0TC9RwSr+fy4+HiVeWKm5k4WGuZPbwx47sox8zeEUNmvq7McTvPXwVUNHRvQ6hb5WrltgvzoKG/srrc74s/eH/DGYwmGNioGr+80Ibdr3bg2xFN6FTLE7UKLDVqpg0KZ0r/cCwtKv+jwc3eiiZFvwxsOpXMxbRctp5VqmMUN8MQQoj72T0dLO/Zs4d+/frRq1cvAgICGDx4MF27dmX//v2Asqo8c+ZM3nrrLfr160f9+vVZvHgxly9fZvXq1Xd38kLcAcX5yhEeEYQ4KxUFyltZPnlNWbVt5NmIb7t+y7BawwD46thXjN06lqzCrMpd0GiExf0wfVafPYnbAWhdrShYTjoOe5XmJPT6GPp9Af4toSBDqQpRUS7ymbVKioOVo9Lm2ckfMi6UVLzQFxbVVEZ5oC+gTfnjFOctJ+6FnGsV30PGRTi8yHzOksgEjl5Ix97Kgp7h3mg1Ko5dSOfVn07Q/bOdZOTqwKc++DUHox4OL2ZvzHVAwxMBM/mp70/0Cq9BfR9bmnMCgJfjmpGnM9I21J13etfh4yENaFDdibRcHaMWHWTaxrMYjCYeCPMwV46oiFqt4oWOIYR52VOoN7LxhoYfxXZEXQWgXZjnTce6kUql4qWiQD0pIx8LtVIb+eMhDbDWarDQqOlcx4t5jzVl3+ud2PFK+9tuWlGcirHpVApLIhMxmpTOecXd+YQQ4n52T78/1qpVK+bMmcO5c+cICwvj2LFj7N69m08++QSAuLg4kpOT6dy5s/kcJycnmjdvzt69exk6dGi54xYUFFBQUPLQS2am8rasTqdDpyu7qiPE7So0FHI+/Ty1XGqhKadaws0YTUaOXVUqYYS7hkPRM1LR6dFl/p4ev6J0davtUhuMMKHRBGo51+L9/e+z/eJ2hq0bxscPfEyQ080fUlLFbMEibgfntVquOJuwVlkQ7lIPXUE+mrUvoDYZMNbuhyGwI5iAgQuwWNgNVVocxuWPYHj4R+XhvGImIxbbP0QFGJqOxmjriar7dCxWDMW07yv0tQegjtmC5upZTLbu6Du8CxX9G7TzxsK7Pqrk4+hPr8MU8UjZ+SfsRvPzk6hyr2GydSfJtwvT5iqr3uM7h/BoC3+uZxfw89HLzP8jgQupeSzbH8+o1gGoIh7F4kIkxpM/czBFKcvWJrA6dho79Ho9b4Rn4bgzjzSTPduzqxHkbsfMIeGYjAY0wJfDGjDom0iir2QTfUVZNX+hfWCl/0/pW9+HGZvP8/PhCwyI8DZvL9Ab2VuU3tEqyLlK/0e1CHCid7g3p5MymdyvDs0CXNHr9WWOc7FR/m7e7v9/HcLcmLIe9senciZJ+f/04abV5f/Te5h8b4SovHs6WH7ttdfIzMykVq1aaDQaDAYD77//Po88ovyQTE5WVmC8vErXP/Xy8jLvK8/UqVOZOHFime2bNm3C1lYe4hB3zoa8Dewp2EOoRShDbIdgq678368rhitkFmaiRUvMvhgMGFChIr0gnZXrVmKvVlbtTCYTRzKVFeiMqAw2xCgNOTRoGGU7iqU5S0nISuDh9Q8z2HYwdSzrVHjN5jGf4A3ssFdWQ5vmZJE6dyhZ1tWom3QUncaWLZpOFGzYYD7Hwftp2mZOQpu4h+tfdOZgwHMUapXKCb5p+2l69Qw6jS2bMoLRF53X2Lk51dMjyVs6AruCKwAc9hjMxW17b/o1CVOFUJvjXN21kP2Xb1ixNZkIuvobdS8tR4WRdBt/9geM5aulJ8guUFPD3oTL9ZNs2KCswFcDOnmqWJGt4dttUXiln8bKoKY7atRXz+Cmu4zJwpOYw7uIKy7IcflHAHYZw7G2UPFw9Qx2b9tcan6P1oDPTmnQGVXUdjZy+cQeLp+46S2Z2RUAWBAZl8qSVRtwKcpEOZehIrdQg4PWROzh3cRXsbBEF3voEgrXTu9jw+mqnVsVvrYaLudCep4OZ0sTBXGH2BD/911P/DW5ubl3ewpC3Dfu6WB55cqVLFmyhKVLl1K3bl2OHj3KuHHj8PX1ZeTIkbc97uuvv8748SUPEGVmZuLn50fXrl1xdCxbHkmI26Ez6Pho1UcAnNefZ7FxMTNazyjVCc5kMnHwykFiM2LpH9xfKRtWZFX0KtgPDTwb0KdzHwDmr53PxeyLBDYNpKmX0mXuUvYlctfmYqG24LFej5UaA2Bw/mBe3f0qB68cZGnuUvp598PDxsO8X6vW0j2gO34GIxZHlJXsfUFNIfUErfIK8M/cbT5W3WUinRo/XOZeVbFhmH4ciUf2abonfoh+8CLwDsdi7gfKea3G0PWBISUnZDfGNLsljvlK0wpjYHvqD5uMc1oee2NTGdTQFwtNOVliVwJg7s9455ymt+1RipfbVVfPoL6kBOLGeoOx6/kJptgcjh4+gkat4vMRLan9p+5r7Qv1bPhoJ9fy9TiENqVdmAdkLYP4XXRVHyQh9HF694owH6+Zr7yjlezRigW9mtPI37ns/IDa56+xaG8Cb/aoRZBH1VpK/5p6gP3xaeS41+aRtoEAnPztHBBP57q+9O4VfvMB7qLzVtF8sV3pGvh421D6tJdSa/ey4ndUhRC3dk8HyxMmTOC1114zp1OEh4eTkJDA1KlTGTlyJN7eyluVKSkp+Pj4mM9LSUkhIiKiwnGtrKywsir7AJFWq0Wr1ZZzhhBVtztpNxmFGbhau2JjYcPF7Is8vvlxJrWaRDOfZqyJXsNP538iIVOpHHAx5yKvNXvNfP6JVGVJsqFXQ/PfyxDnEC5mXyQhO4FW1ZXqEGczlNJyNV1qYm9dNkfUU+vJ3G5z+eTQJ3x3+jvWxK4pc8x3Z79jqn1d2mMiN6g9R9KVMdt0/RjWvwp5qVC9GZpmT6JRlxPE1uwCT22B5Y+gSo1Bu7gX1BsMV8+ClROaVs+jufHflkt16DJRKT9nYY26z6eotFrGrdzPiUsZZBUYebZ9cNnr+NYH1yBUqbFo/vik9D6VBrq9j7r5M+QVGpi07hAAT7YJpL5/2WoUTlotDzbxY97uOJbsv0jnur5Qu68SLGsOcipkQsn/BznXIEn5RWL046PBwaPMeMU61fGhUx2fCvffzMBG1dkfn8Yvx5MZ01H5pWpXtJKC0b6W1z39/1PP+tX4Ynsslho1D7cIuKfnKpDvjxBVcE8Hy7m5uaj/9INZo9FgNCpP9wcGBuLt7c2WLVvMwXFmZiaRkZE8++yz//R0xX3sh3M/EJsey0uNX8Lyxpzbv2Bd7DoA+gT14cnwJ3ll5yvsTdrLhJ0TsFBZoDcpuaM2Fjbk6fNYemYpvYN6m1s/F1fCiPCMMI8Z7BzM9ovbibl+FtY8D6FdOJmjlEq8WTtqC7UFrzR9hebezc1d+YqdvHaS49eO80JaJM86O1En7AF0UQupZl+NGnWHQPWWcPJHaPCw0sSjIp614amt8PNoOP8bHP1e2d7yObBxLnt8o8eUahquweAaxOGENE5cygBgzs4YHm1Zo2zZMZUKBn4LJ35Qzi2m1kDdAeDXDKPRxJT1Z7iUnkd1F5tSLYv/bHiLGszbHcf2c1dJvJ6LT2h3tL9OoInqHM4+N7RvjtkGmMCrHjh4VzjeX9Uj3Id31pzibHIWZ5IycbWz5GxyFioVtAlxv/UAd1EdX0c+GxqBi60lHg7lVDMRQoj71D0dLPfp04f3338ff39/6taty5EjR/jkk0944okngKJaouPGMWXKFEJDQwkMDOTtt9/G19eX/v37393Ji/tGvj6fqZFT0Rl1ZBZmMqX1lL/ccSyrMMtc57h3cG+crZ35uvPXzDoyi/kn56M36Ql3D2dQ6CB6BPZg0r5JrI9dz8S9E1nWaxlZhVnEZ8YD0MCjgXlcc/m4S/vg9D448QMnG3QAbh4sF2vn105pp3wDnVHHjA1PsfT6Ib52ccIhVsnNbe3bWvk6OPtBm5cqd+M2zjBsOez4EHZMA1t3aP5M+cfe2D0P+G5vvPnjtFwdi/bEM6ZDSNnzqjdWXuXIzNcxfsUxfj+jlC6b0r/eTZtqBLrb8UCYBzvPXeX7yAS61fVGawykvjqOkNRdEFSUShD9u/JnSDldAO8gJxstHWt5svFUMquPXDJXkwiv5oSb/b0fgPaLuL8bkAghRHnu6WD5888/5+233+a5557jypUr+Pr68vTTT/POO++Yj3nllVfIyclh9OjRpKen06ZNGzZu3Ii1deUK6gtx+vppdEblyfC1MWsJcgpiVPiovzTm7wm/U2gsJMTak5pzu4O+AA3wEtBFa4FlQFvCui0EC2UVe0KTCey+tJuzqWf5/vT31HBU6tMGOwXjZOVkHtdcPi43CRNg1OdzOlV5aque262D5fJoVRa8fimOOlnXmeTpSZZOqeRgLhlXVWo1dHgD6g0CS7vyV5X/5Fp2ARuKSqY90TqQ+X/EMXdXLCNbBVS6qUX0lSxGf3eI2Ks5WFqoeb9/PdrXvHWptREtarDz3FVWHryAtYUag6Ep9dVxqKPWQ5ORSjm9mK3KwcF/b7AM0L9hNTaeSmbN0cvmsnPtwipO+xBCCPH3uqfrLDs4ODBz5kwSEhLIy8sjJiaGKVOmYGlZ8ja5SqVi0qRJJCcnk5+fz++//05YWMVNAIT4s+JaxsUtkj87/BlbEst2pcsqzKp0veLiFIxeBUZUeWmgyzW/6uVmEnZ6vdKYo6gBiJuNGy83VjrjfXXsKzbEKQ+r3ZiCARDgFKBUxFCZuK7REmttSx4mbNWWBDoFVv3mAS4dgqSj9MvTs7jjV/ja+eJp40kLnxbmQ1Iy80t1zqwUj5rgVL1Sh644cIFCg5EIP2fe7FWbIHc70otWl/9MbzASVZSmUPxafeQS/b74g9irOfg4WfPjMy0Z0sSvUtfuUMuT6i42pOfqmLMrlk1GpVMisduhIEvpPJhzBbR24N/ipmPdCR1qeeBobUFyZj6/nkwC4AEJloUQ4q65p1eWhfgnHL16FIAn6j3BhawLrIhaweu7Xmdxj8XUdKnJgeQD/HjuR35P/B1brS2Luy8myLniJ/2Tc5I5kHwAgJ5XlS5uDF0GXnWVjy8dhB9HKY0zPGopOb1A/5D+rIlZw6GUQ2yMV9pB/zlYtrGwobqFHRf02cQENOOya3W4tpc6+Xlo8jOUttBVVdTimXoDqVu9FesGrsNoMmKlsSK3UM9rP51g7bHLvN6jFk+3K+ehu79IbzCyZJ/ykOOIljXQqFW82CmUcSuOMmdnLCNa1sDBWnkYKe5aDk9/d5BzKdnljtU80JUvH2mEexVSFjRqFcNb1ODDX8+SrzNynmoUOgVimRGnpF+kxikHBrYtv7PgHWZloaFXfV+W7VeaezhYWRDh5/y3X1cIIUT57umVZSH+biaTiWNXlCoHEZ4RvNbsNVr6tCRPn8eY38fQZ3UfRm0axa/xv6Iz6sgoyOD5rc+Tnp9e4Zgb4jZgwkRjt3B8s66ASg1B7cGlhvKqNwi6TlEO3vQmnNsEKO+SvNPyHbTqkqfUIzwi/jxhgvOUQDHaty4nnH0BCM/Ngc1vlz42Lx3O/HLzVtQZl+DUz8rHRS2ftWotVhorEq/nMvCrPaw9dhmA7yMTqr66XAm/n7nC5Yx8XO0s6RmuVJHo08CXYA87MvJKVpe3nEmh7xe7OZeSjY1Wg4eDlfnl7WjN0+2C+P7J5lUKlIs92MTP3OLZ1c4KbR2lVB9n15ekYIR0ruDsO29Aw5Lc39Yh7mjLK6MnhBDiHyEry+I/LSEzgbSCNCzVltRxrYOF2oIZ7WfwyPpHlAfs8sBOa0evwF50DejKu3ve5ULWBV7a/hJzusxBqylbfqk4BaO3U01gPbjXBMs/NSNpOQauRcHhxfDjE0obaK86BDkF8WT4k3x97GtcrV3NuctmFw8QkpPBdmcnYqxtOFmcr1xQAEe+hwbDlA56hxbCyZ9Bn6ec598SGj8GdfqBhTXE71aOObMWDIXg27DUQ3M7zl3lxWVHyMjT4W5vSW6hgQupeRxOTKNxjdtYvb6J7/bFA/BQUz+stUonueLV5bHLjzJ3Vxx5OgNfblPafDep4cJXwxvh6XDnnktwtbOkT31ffjp8keaBrqhq94a9s+Dcb0r6DEBwxzt2vVtpUsOFas42XErPkxQMIYS4yyRYFv9pxSkY9dzrmQNfR0tHZneZzdzjc6nvUZ/uAd2x1SrB7hcdv2D4r8M5mHKQKZFTeK/le6UqZ0SlRnE+7TxatZYuuqLVQJ/6ZS+sUkHPj+F6LCTshiWDIfABAJ40GdFb+hEe0LVsVY7jKwguVB5GPJN2nvNpRWXjwvrB0eWwuD8Yb2hj6+yvrB4n7lVev74Ctm6QGltyjE8D6PuF+dNFe+KZ+MspjCZo4OfM7OGN+Oi3KH4+fInVRy7fdrAclZzFyoMX6FDTk1bBbqjVKqKvZPNH9HXUKnikuX+p43vX9+XzrdFEX8k2B8ojWtbgrV51zKvAd9JrPWphZ6VhZKsAcLMFO08lVxnAJRDc7nwKSkXUahUfP9iArWevMKixVJgQQoi7SYJl8Z9WXi1jgGr21Xiv1Xtljg9xCeGjBz7i+a3P8/P5nwlyCmJk3ZJukutj1wPQrno7nFKUxh74NCgzDqBUwnjoO5jbEdLi4NgyACyBFwHO7QP3+iUrmvpCOPkzIQYlGD55XWnd7Grtik/XDyF6G2SngIUN1BuorCRXbwpZSXBkibKKnZEI+RlgaQ/hQ6DxSGVVuUh2gZ4p609jNMHQpn5M7FcXKwsNAxpW4+fDl1h3/DJv9656sJpXaOCpxQdJTM1l3u44/F1teaipH7FXcwDoVNuL6i6lV981ahVjO4XywrIjWFqomdK/Hg9W8qG92+HhYMWkfjdUFKnVU1l9h7+9ZFx5WgS50SLI7R+/rhBCiNIkWBb/aeZg+c+5wTfRtnpb/tfkf0w/MJ2PD37MD+d+MO9LzlHKn/UO6g1HxykbvctZWS5m6wqjNsPJn8BQULI9YS+c+xVWPgZP/g4eYRCzBfJSCbT3Qq1SYyxqylHPvR4qWxd4bD1cOgxh3UqXa3P0hXYToO14iNsBeWkQ2g2synb7Oxifis5gorqLDVMHhptXtlsFu+PhYMXVrAJ2nrtK5zpelf56Aczaep7E1FycbbUYDCYSU3P56LeSXOoRLWuUe16fBr7YWWnwd7Uz1xz+x9Tqc0Ow/M/lKwshhLi3SLAs/rMyCjKIyVDe3m/gWcHqbwWG1x5OQmYCK6JWmNtVF/O09aSta11lFRfAO/zmg9l7QIs/Ne5o/gws6gsX9sHSB5XOeMdXAGBVbzB+ecfN1zU3I3EPVV4VUWtumXe7LzYVgJZBbqVSQDRqFX0b+DJvdxyrjl6qUrB8NjmTuTuVtI/pg+rTJtSd9ceTWLY/kcOJ6dSr5kjr4Iq703WsVbXA/I4JbKuksegLIaDt3ZmDEEKIu06CZfGfdeyqUgUjwLEGrj8+Bdej4emdYO10izOVyhVvtXiLB2s+SHZh6TJmgU6BWF5WxsYloFJNOcqwsIKHvi9J0Vj+CFw+rOyr/yBBZ7PNwXK4+y2C8SrYF3sdoNy3/wc0rMa83XH8fjqFzHwdjtZlH278M6PRxBs/n0BvNNGtrhdd6yqtooc08WNIEz8upefhZKNFrf5rHRP/FhZWyt8Hk6ncVXghhBD/DVKPSPxnFadgNLD3V+rppsWXtDWupDCXMBp5NSr1crF2gaTjygEV5SvfwtnkTLZcMLKvxZfoLewgcQ/o85XKGj4NzJ38AOq61b2ta/xZdoGeE5cyAGgRXDZYruvrSIinPQV6IxtPJldqzCVFq8f2Vha817fsPKs521S6Q99dYeNye7WrhRBC/GtIsCz+s4orYTTMzizZGL31zgyeVLSyfLN85QpEX8mi52e7GLXoIEPXZPFU7nMYTcrKa0L13qBSmYPlavbVlOD8DjgQn4rBaMLf1ZZqzjZl9qtUKnP939VHLt1yvJTMfKb/qjzk+L+uYfg4lR1TCCGEuNdJsCz+9Y5fPc6q86vMD8QB6Iw6Tlw9AUBE/IGSg2O2mFtQlxG7Hc5uqNxFk29/ZXl71FWMJqX2bwM/Z1KrdWCSxQtsNDRlqUF50Kyjf0f6Bvflf03+V+XxK1KSglHxSmrfBkoTlL2x10nKyLvpeBN/OUVWgZ4G1Z14tGXAHZunEEII8U+6h9//FOKvu5J7hac3P022LpvErETGNhoLwLnUc+Qb8nG0sCUwI1F5u12Xr5RZu3K6pDV1seyr8P1gpYbxyF/MNZHLVZAN15T6x7cTLO+NUYLWZ9oFMfoBpbbvb6eCefq7FgTEFvA6YG1hzftt3q/y2Dezr+i6LctJwSjm52pLswBX9sensvbo5QrbX285k8KGE8lo1Co+GBiO5l7MSRZCCCEqQVaWxb/ah/s/JFunPID37Ylv+SXmF6AkBaMBVso/groDIaCNclL0lrIDnfq5pNnHL+OUwLoiKacAE9h7g71nleZrMJrYH1dckaKkQkSrYDcs1Crir+eScD2nSmNWRla+zpyv3Dzw5rV9+xelYvx8+BJGY9lV+JwCPe+sOQXAqDaB1PW99QOTQgghxL1KgmXxr7Xjwg42J2xGo9LQI7AHAO/ueZejV45y5MoRABpeL8q9rf9QSeOJ8h7yO76y5OPUGNj1ccUXLs5Xvo1V5dOXM8kq0ONgZUEdX0fzdgdrLY1qKLnJO89drfK4t3IwPg2jCWq42eJbTr7yjXqF+2CtVROVksWMTVFl9n+6+RyX0vOo5mzDuM43KWUnhBBC3AckWBb/Srm6XN6PVNIURtQZwYdtP6SjX0d0Rh1jt43lQLKSpxyRmwXONcCvWUnjicS9UHjD6u31GLh0EFQa6DlD2bb7U7hytvyLJxcHy1V/uG9v7DUAmgW6lkldaBfmAcCOc9cqNdaGE0l0+ng73+1LuOWxe4vylVtWomOck62WqQOVcnVfbY/h58MXzftOXspg/h9xAEwZUA9bS8n0EkIIcX+TYFncV67nXafgxk53Ffjy6Jck5SRRzb4az9QbhTo/k6ltp1LLtRap+amk5qeiAeoVFEL9B0GlArcQcPIHQyHE7y4ZrHhVObgDNH0SwrorKRnrxoHRWPbif2FlubgpSHl1jouD5b0x1yjUl3PdIgajiWkbz/LcksPEXM1h+q9nySnQ3+K6FddXLs+AhtUZ00HJV37tpxMcSkhFbzDy+s8nMJqgd30fOtSsWgqKEEIIcS+SYFncN45eOUrXH7syetNoDEZDhcedvn6a7898D8CbTV/D9vtB8FEwtocW83mHWbjbKLnAtQoKsTGZIPxB5USV6oZUjKK8ZZPJ3DmP+g8px/ScAVo7ZQX6yOLSF9cXlqw4V7FsnN5g5EBxvnI5D9nV8XHE3d6SnEIDhxLSyh0jPbeQxxce4OvtSmdCW0sNWQV6Vh+tuNRbZr6Ok8X1lSsZLAO83KUm3et6U2gwMnrxIab/FsWJSxk4WFvwTp86lR5HCCGEuJdJsCzuCzqDjvf2vEehsZDDVw6zPGp5ucfpjXom7p2I0WSke0B32iYehYsHwKiHja/ivek9vnhgBvWsvRiRkQm+DcEjrGSA4mA5pihYvnhQ6aCntYNavZRtzn7Q4Q1lXhvfJvtaSRoCV88oq87Wzkqr5Co4nVSUr2xtQW0fxzL71WoVbUOV1eWd58vmLZ9NzqTPF7vZee4q1lo1s4Y1ZHwX5d6+25uAqYKSeAfiUjGaINDdDm8n60rPV61W8clDDajr68j1nELmFLW0fq1HLTwdKj+OEEIIcS+TYFncFxacWkBMRgwWKiUHdtbhWSTnlO0iN+vILE5fP42DpQOvhj0M2z9UdtTuq+QcH19O3TUvsex6Lj1zcpXV4hsFPgBqC6X1dVp8yapy7d5gaVdyXPNnuGAVhlaXSdqXXUg4c0jZnnRDvrKqauXSikvGNS8nX7mYOW85qnSwnF2g5/EFB7iQmoefqw0/P9uavg18GdLYD2utmrPJWRyIL381ujL1lStia2nBtyOb4OlgBUCTGi4Ma1q1XxKEEEKIe5kEy+Kel5CZwDfHvgFgkmtT6lt7kavP5cP9H5Y6bnX0ahacXADA283fwn3LFNDnQUBbeHAxjFgNtm5KQHvllBI81xtU+mLWTlC9mfLxud/g5E/Kx8WpGkW2R6fyWOZTXDK54We6jPvynhz5bdFfanNdmbzhNqFKCsnppEyuZJWUr5vxWxRJGfn4u9ryy/NtzJU0nGy19I9QSr0t3htfwXUrzpOuDB8nGxaPasawZv58+lAEaqmpLIQQ4l9EgmVxTzOZTEzeO5lCYyEtrb3pfXAF70YfxcJkYkviFrYe+gpMJg6lHGLi3okAPF3/aXrk5Col4DSW0Humssob+ACM3gE+EcrgIZ3Kr4NcnIqxYzrkpYKdBwS1N+/OKzTw1uqTxJiqMa/2Ak5Z1sdOlU/DvS9ScHipcpB31YJlvcFoXvm9WdDqbm9FeDWlbvGuoqoYxy6ks6goEH5/QD2cbS1LnfNoyxoAbDyZzJXM0vWhM/J0nLpc9XzlP6vl7cjUgeH4udre9hhCCCHEvUjqOom7YmviVj47/Bm64kYfRcJcwhgYOpDWvq3RqDWsi11HZHIkVmpL3o45jgoIc/BjZMZ15jk78cHRz6kWOY9xDmr0Rj0P+Hbm6P5Qsq48jQNgavsyKveQkgs4+8ETG+H0WghqV/7kQjrB1smQW1Sird5g0JT8U5m55RwX0/LwdbLm5QGtsVJvYd+3L9AiZTlWBqXknMmnPlVZXz15OZPsAj2OFeQr3+iBMHdOXMpg5/mr9Ivw5fWfT2AyQf8IX3NO843q+jrRuIYLhxLSWLo/kXGdS3K0N55MwmiCIHc7vBwlz1gIIYT4MwmWxT/OZDLx+ZHPic2ILbPvQtYFtiRuwcfOh/4h/Vl+VnmQ7xm9NX6F+RDaFR5eyTMX9vHbjhe5aJHPME0BOqOKOgU6Hj0ZT3rqRzho0og2+vLioSYMtohjYKNqJSuuWhto8FCZa5t5NwBb95JguX5JCsbpy5l8u0upIzypXz3srJR/Qi2e/YaDvzSi7sG3SMWBjAJPqlIPojgFo3mQ2y1bQ7cL8+TLbTHsOn+Nb3fHcTopEycbLW/1rviKI1rWUILlyETGdAjBQq3iq+0x5qYiXep4VWG2QgghxH+HBMviH3cu7RzR6dFYGk287TmYgKb9AdAZdWxN3MramLUk5STx9bGvAQixcmfk2cOgtVXKtqlUWPu35O1On/H05qfRqVR4mlTMSrmClyEJNMp13jON5vTVAiatO83Hm6L4aEgDeob73HqCajUEd4QTK8EtVKmYgVK/+I1VJzAYTfSo503nPwWYTfo8zSupIWw8c42hx5OpU92l0l+TqtQ5bujvjL2VBak5hXz0mxLsvtGzFu72VhWe06OeD5Ptz3Alq4BVRy6x9cwVNp5SHpAc1syf8V3DKjxXCCGE+C+TnGXxj1t3fD4A7fLy6H3gcyLy8ojwjKCpd1NebfYqWx/cytS2U2ns1RgPazcmX4pHC0q5Npca5nFa+bbikdqP4GXrxaw+y1hWYzZL9R3JVDtD63F89ebzTO5fj1reDuQUGnhuyWGmbTyLwVh+CbVSmj4J9l7Q7lVzVYslkQkcvZCOg5UF7/WtW+5pnZrUJRN71hy9VOF14q7lkHg91/z5jfWVK1ORQqtR0zpECaoNRhPNAl15sInfTc+xtFAzrJlyzCs/HmfjqWQsNWqmDgxn6sBwrCw0t7yuEEII8V+kMlVUfPU/JDMzEycnJzIyMnB0vHm+qPhrDPpCun7fhCsqEx8mp9MrLxOTrRuqp7aCS0DZE356Ek78oDT4eGpbqdzhG8Vfy6HTJzswGE2seq4VDf1LVnX1BiPTNp5lblH6xANhHswaGlHmQbib+eHgBd5cfZJCvZHJ/eryaMty5goU6A00e38LGXk6ljzZnNYh7qX2n0/Jotes3RQajLQIcmVYM3+8Ha15aM4+nGy0HHm7S6WqSSyJTODNVSfRalT8OvYBQjztb3lOUkYebaZtw2A04eVoxdfDG9PIv/Kr30KIfw/5uSdE5d3zK8sBAQGoVKoyrzFjxgCQn5/PmDFjcHNzw97enkGDBpGSknKXZ/3fsuviLnZd3FWpYw/smMgVlQkHg5GpGW9zwhiAKvc6LB0K+ZklBxoNcHSZEiir1NDnswoDZYAvtkVjMJpoX9OjVKAMYKFR82avOnw2NAJrrZqd567S54vdnE/JuuV8C/VG3llzkgk/HqdQb6RbXS8ebl6jwuOtLDT0qq+keqw6UrZr3swt5yk0KK2q98WmMnb5UYbN3Qco9ZUrW3atf0Q1+jTwZfrg+pUKlEEp8TapX12GNfPjlxfaSKAshBBCVMI9HywfOHCApKQk82vz5s0ADBkyBICXXnqJX375hR9++IEdO3Zw+fJlBg4ceDen/J8SnRbNmC1jeH7r81zIvHDzgzMusi7qBwBcs/y4aKzGk4X/I9PCTel899MoyLgIOz6CzyJg9TPKec1GQ7VGFQ4bfy3HHJjeWOnhz/pFVOPnZ1vj52rDhdQ8nv7+EAX6ittmX8nK55Fv97F4bwIAL3UO4+tHGt/yAbwBDZW6xhtPJpOvKxk/KjmLDSeSAFjweFPGdQ7Fx8ma4myNP69C34ydlQWfD2vIgIbVK30OwCPNazB1YH3psCeEEEJU0j0fLHt4eODt7W1+rVu3juDgYNq1a0dGRgbz5s3jk08+oWPHjjRu3JgFCxawZ88e9u3bd7en/p8w+/hsTJgwmoysiFpx02Pz1v+P3220AJxJ70Wwhx0puDLG9AomC2s4vwk+rQvbpkBGotIyuuXz0Pm9m477+VZlVblDTQ8i/JxvemwdX0dWP9cad3srYq/mMHt72YocoAS2fT7fzYH4NBysLJg3sgljO4dWauW3sb8L1V1syC7Q8/uZknc5Zm05j8kEPcO96VDTk3Gdw9j9akfmP9aEt3rVZlgz6XwnhBBC3Gvuq2oYhYWFfP/994wfPx6VSsWhQ4fQ6XR07tzZfEytWrXw9/dn7969tGjRotxxCgoKKCgoMH+emam8/a/T6dDpdOWeI8qKTo9mU/wm8+erolcxut5obCxsyhyrOrueHZd2kOPpjo3Riay8AEZ09mfG5vPsyvEjuut0Qne+CIDRvyXGiEcx1eqjlHkDqOD7En89h1VHLgIwpn1Qpb5/jlZq3uwRxks/nOCLbefpXseDII+SVtbXsgt4fMF+UjILCPaw4+uHIwh0t6vS340+9b35ekccPx+6SLfaHpxLyWJ90arycw8ElhqrbbArbYNdwWRAp6t4pVsIIe4U+VknROXdV8Hy6tWrSU9P57HHHgMgOTkZS0tLnJ2dSx3n5eVFcnJyheNMnTqViRMnltm+adMmbG2lA1llLc9ZjgkTdbR1uGy4THphOjN+mUFjq8aljrPUZdI+6m3Wuyhf25y0RoCa7IQThNmrOZSvZuoJVx6tORmj2pJsax+4AFzYVmqc5FxYcE7D1Rua0JlMYERFHWcjl47/waXjlZu7ygS1ndWcSVczZuEunq9jRKUCnRG+OKXhcrYKD2sTo2pkcGb/Ds5U8WvjnAtgwfZzV1i5ZgM/xKoBNRFuRmIO7yKmiuMJIcSdlJube+uDhBDAfRYsz5s3jx49euDr6/uXxnn99dcZP368+fPMzEz8/Pzo2rWrPBVcSdHp0ZzccBKAd7u8yx+X/2DW0VmcsT7DW93fQlVUbg19AZolA8kwZLDbVsmvzUtviFaj4rEB3fE7e4VDK44TXWBH60HPlJz3J6k5hQz+JpLkvLwy+ywt1HwwrAV1fav2vavfKpeen+8hOhPyfMIZ1NCXCT+dJD47CUdrC5Y83ZxAd7tbD1SBtVf3cupyFidVARxNvYhKBe8Pa02Yl8NtjymEEHdC8TuqQohbu2+C5YSEBH7//Xd+/vln8zZvb28KCwtJT08vtbqckpKCt7d3hWNZWVlhZVW2gYNWq0Wr1d7Ref9bzT01F4CuNbpSx6MOPg4+zD4+m7NpZzmdfpoIzwhl2XfdC3Axkt9cPNCroJptKGcLPant64CdjRUd6/hgaXGSxNQ84tMKyg0kC/VGXlhxnAtpefi52rDgsWbYWZXUBXaw1mJvVfW/ykGeTrzUOYypv55l2m/niLuex5pjSWjUKr4e3pgwH+fb/fIAMKBhdU5dPsOyA0qaSM9wH+pWv3UdZSGE+LvJzzohKu+ef8Cv2IIFC/D09KRXr17mbY0bN0ar1bJlyxbztqioKBITE2nZsuXdmOZ/QlRqFJsTNqNCxTMNlIoVLtYu9AjsAcDyKKVFNX/MhGPLQKVhnV9tAKppWgGYV4HtrSxoU1QF4reTZVNnTCYTb60+wf641KIH7ZoS4mmPj5ON+XU7gXKxJ9oEUtvHkfRcHXN2Kg/7TepXt0qVKSrSt4Evxc8DqlQwtlPoXx5TCCGEEP+s+yJYNhqNLFiwgJEjR2JhURIYOTk5MWrUKMaPH8+2bds4dOgQjz/+OC1btqzw4T7x131z/BsAugZ0JdSlJAAcVmsYAJviN3H9+HL4fSI64IOGPTiWFY9apaYgswEA9ao5mc/rVldpG73pdNn62N/uimPlwYuoVTDr4YZ3PIVBW9TFrjj747FWATxykzrKVeHpaG0OunuF+0j6hRBCCHEfui/SMH7//XcSExN54oknyuz79NNPUavVDBo0iIKCArp168ZXX311F2b576c36tmcsLlkVbn+M6X213WvS7hbXU5cP8XP299kgFrFy8H1OZymPHU3ttFYvlmrRKU35hd3qu2FSnWCE5cyuJSeRzVnpezaigMX+OBX5dG6t3rVoUNNz7/lviL8nJkxuAGJqbm80DHkjo79bp+6fL8vgec6BN/RcYUQQgjxz5B210jbz1u5lH2Jn879xOro1VzNuwpAj4AeTG83veSga9FweCFrz67kTScr3PUG1BaWXMGAndaOqW2mUs+lFU3f/x2VCk6+1w27G9Inhszew4H4NB5rFUCB3sCao5fJLVTKqA1r5s8HA+pV+PCfEEKIqpGfe0JU3n2xsizuDpPJxMCVLxOd/zug/E7lau1Kv5B+PF3/aeWgnGuwZgyc2whANxXMcPDjmoUGMBDoFMhnHT4j0CmQ7VFXAAhytysVKAN0q+vNgfg0Fu6JN28LcrfjkRY1GNGyhgTKQgghhLgrJFgWFfotdjfR+Up7cUdTXd5u9zid/Dui1RQ9RX35CCwfDpkXQaWG0K5YNX6Mxwou8emRz+jo15H327yPvaU9AKcuK6WK6vo6lblWz3AfZv5+nkKDkR71vBnWzJ/mga4SJAshhBDirpJgWZTLZDLx+eEvAShMbcmllH4c9vSne2BRoHxkCax7CQwF4BoMQ5eAp1Lx4gmgd0hfPGw8SgW7py5nAFCvWtm3/HydbdgxoT0WGjVONlLSSAghhBD3BgmWRbkikyNJzD2FyWhBmHVfTgBzdsYS6mbFkOtfw/45yoFh3Ylu8wlWFi743XC+p23Zh/FOXqp4ZRnAzb5s7WshhBBCiLvpvigdJ/5ZJpOJr44qFUV06c14t0crXuwUCpjQrH/RHCifCHmWvtfG0Pmro3SbuZPE6xW3T83I05GYquyvaqc9IYQQQoi7RYJlUca+pH0cuXIEk9ECy+xORPg5M65TKDOr72Cgehd61Iw1jqfPybYcv5wFQG6hgbfWnKSi4iqni/KVqznb4Gxr+Y/dixBCCCHEX1GpNIy1a9dWeeAuXbpgY2NT5fPE3fXnVeWOgcFYaNRwZh39riktrifqRrDG0IQgdzuGNfOnUQ1nhs2NZOe5q6w9dpl+EdXKjFucryyrykIIIYS4n1QqWO7fv3+VBlWpVJw/f56goKDbmZO4i/Ym7eXo1aOoTFoKr7enXWsPSDoGPz+FChP5EY/j7fQ8y2u4lKpW8XyHED7ZfI7J607TPswTJ9vSD+kVV8K4sXOfEEIIIcS9rtJpGMnJyRiNxkq9bG1t/845i7/JjavKhWnNMOkdaV/NCMuGgS4Xgtpj3WcGYzqE0CLIrVSli2faBRPiac+17EI+3HimzNiysiyEEEKI+1GlguWRI0dWKaVi+PDh0hHoPrQ8ajnHrh7DQmVJwfV2hHnY4P3rU5B5CdxCYcgi0JT/ZoSlhZoPBoQDsGz/BfbHpQKQlJHHZ7+fJ/pKNiAry0IIIYS4v1QqDWPBggVVGvTrr7++rcmIO0dnMJKVr8fVruKH6Qr0BrLz9TjYqPhw/4f8cO4HAGpY9CBN78jLzjvgwn6wcoKHV4CN802v2SzQlaFN/Vh+4AKv/XScIA87tp69grHomb961RzxdJDycEIIIYS4f/ylOss6nY5z585hMBioWbMmVlYSCN0LDEYTj3wbyYH4VP7XtSbPtQ8u0wnvSGIaz35/mLSCq9RqsJrYrFOoUPF8w+eZty4QH5LolKQ80EeX98AtuFLXfr1HbX4/k0LstRxir+UA0DzQlYeb+9Otrrd05BNCCCHEfeW2g+Vdu3YxdOhQdDoder0eCwsLFi9eTPfu3e/k/MRt+G5vvDkN4qPfojhxMYMZDzbA3kr5di/fn8g7a06h18Zj4/8dsVlZ2FnYM73dNLy1DfkgcwfzLBdioc8Bv+bQ6LFKX9vJVsvHD0YwfeNZWgW7MbSZP8Ee9n/HbQohhBBC/O0qHSwbjUbU6pIU53HjxrFkyRLat28PwJw5c3j22WeJi4u745MUlZeUkcdHv0UB0DPcm99PX2HjqWSiv8zmi4cbsmhPAsv2J6LSZOMUuAgDORjyvbDKfZoGbi1ZeeAC3dQH6aQ+BGoL6D0T1FUrx90uzIN2YR5/w90JIYQQQvyzKh0FNW/enMOHD5s/LywsxN/f3/y5v78/+fn5d3Z2olL0Rj1fHf2Kg8kHeXfNKXIKDTTyd+aLYY1Y/nQLvBytiL6STfeZu5RAWQURETswkEOwUyiOaeNJSLHl+aWH2Xs6jve0i5SBW48Frzp39+aEEEIIIe6iSgfLX3zxBU8++SQvvfQSOTk5vPvuuzRu3JgWLVrQuHFjBg0axPvvv/93zlVUYEviFr4+9jVjt/6PTaeTsFCrmDqwPmq1ikb+LvzyQhuaBrgA4GhtwYT+KqJzd6FWqZnSZhLfPtoaG62GXeev8cDF2fioUil0CoAHJtzdGxNCCCGEuMsqnYbRvHlzDhw4wPTp02ncuDHTp08nKiqKyMhIDAYDTZs2pVq1sp3bxN/vyMXdAGTqrqOxjWV0s27U9HYw7/d0sGbJky3YdDqZ2r42PL/jYQCG2QVT77thABx2MHA9pwBfrgOg7TsTtNKBUQghhBD/bSqTyWSq6kkxMTE888wzODo68vnnn+Pr6/t3zO0fk5mZiZOTExkZGfdNfehr2QUYi2qyPb+qLWdQUmCscpuy+8m5WGs15Z4389BM5p2ch6eNB2vPHsPOqC9zzHH3XtR/funfN3khhBB31f34c0+Iu6VK1TBOnTrF2bNnCQ8PZ/PmzSxatIi2bdvy8ssv89xzz/1dcxQ3uJZdwLjlR9kdfQ2ATpq9nAvNg6KSbGq746DSAWWD5XNp51h0SslHfsOrHXanD4F7TehfUhf7Sq6BukEN//4bEUIIIYS4D1Q6Z/mTTz6hadOmfPTRR7Rs2ZK5c+cycuRIIiMj2bdvHy1btuTEiRN/51z/845fTKfv57vNgbKjKpfB9ssxqFR46A1U0+nJMxWw/eL2MucaTUYm7p2I3qSno19HOiWfV3bU6QfVG5tfnmHN0Fho/8G7EkIIIYS4d1U6WJ4+fTrr169n3759HD58mE8++QQAd3d3Fi9ezKRJk3jwwQf/ton+1/1w8AKDZ+/lckY+ge52bH7pAY63PcAF6wIAGnk0oGeO0gRkfcy6sudH/cDxq8ex09rxeuOXIHqLsqNWr3/sHoQQQggh7jeVDpZNJpO5zrJGo+HPqc5dunThyJEjd3Z2ApPJxHtrTzHhx+MU6o10ru3JmudbE6o7B/vncNRa6ZrYMKgbvQuU78nuS7tJy08zj3El9wozD88E4IWGL+CddBp0ueDkBz4N/vF7EkIIIYS4X1Q6WJ4wYQI9e/akVatWREREMH78+DLHWFtb39HJCVh3PImFe+IBGNc5lDmPNsFRq4JfxmLExFFbpTtehHdTggI6UrugEL3JwG/xv5nH+HD/h2Trsgl3D2dozaFwdr2yo1Yvc66zEEIIIYQoq9LB8v/+9z/27dvHSy+9xO7duxk9evTfOS8BGIwmZm45i5X3arq3TGRc5zDUahVEfg0pJ4izdyMLAzYWNoS5hkGt3vTOVlIx1sUqqRg7Luxgc8JmNCoN77R8B43JBFEblAtICoYQQgghxE1VqY9xeHg4Q4YMoVatWn/XfMQN1p9IIj7nKJYu+9ifOZfU/FTIz4CdHwFwJGIAAPXc66FVayGsKz3yClGbTBy7eoyo1Cjej1QaxTxa51FqudaCC/sgLxVsXMC/1V27NyGEEEKI+0GlguXx48eTU/TwWGW8/vrrpKam3vakhLKq/Nnv59DYXABAZ9Tx8/mfYd9sJWB2r8nRorSXCI8I5SRrJzz8W9MiT6m5/Ozvz5KUk4SvnS/PNnhWOaY4BSOsB2iqVDlQCCGEEOI/p1LB8meffUZubm6lB/3yyy9JT0+/3TmVcunSJYYPH46bmxs2NjaEh4dz8OBB836TycQ777yDj48PNjY2dO7cmfPnz9+Ra99N645fJuZqDlZ2l8zbfji7AsO+L5VP2r3C0avHAIjwjCg5sVZvehf9YnM17yoAb7V4C1utLZhMcKaoUkbt3n/7PQghhBBC3O8qFSybTCbCwsJwdXWt1Ksqq9A3k5aWRuvWrdFqtfz666+cPn2ajz/+GBcXF/Mx06dPZ9asWcyePZvIyEjs7Ozo1q0b+fn5d2QOd4PBaOKzLecBE9b2SrCsUWm4nJvMTnUheNQiNbgdCZkJADTwuKGiRc2edMrJw8ZoBKB7QHfaVm+r7Es+ARmJYGEDQR3+yVsSQgghhLgvVep9+AULFlR5YC8vryqf82fTpk3Dz8+v1PUDAwPNH5tMJmbOnMlbb71Fv379AFi8eDFeXl6sXr2aoUOH/uU5/OPyMzi4ZRWXrjrh5FBAvjETC5UFD4X0Z8n5H1nmaE+HNq9y9JrSACbYKRgnK6eS8x19sPVtzPNpUez2b8CrzV4t2Xe2aFU5pBNY2v6DNyWEEEIIcX+qVLA8cuTIv3se5Vq7di3dunVjyJAh7Nixg2rVqvHcc8/x1FNPARAXF0dycjKdO3c2n+Pk5ETz5s3Zu3dvhcFyQUEBBQUF5s8zMzMB0Ol06HS6v/GObk297GGaJ+xmhWUQy2uPZH0WhLqE8nBWHktNJvba2BDtXY/DsWsBqO9ev8yc1WE9GbHtIMNzLTFYOJn3W5xZhwrQh/bAdJfvUwghxN1zt3/WCXE/uaef8IqNjeXrr79m/PjxvPHGGxw4cIAXX3wRS0tLRo4cSXJyMlB2FdvLy8u8rzxTp05l4sSJZbZv2rQJW9u7t+LqnnWa1gm7AYhQx/LbpS/B0RqnTGt8YhbygIc9O2xtmL55BpcMSnqG6rKKDRs2lBrHPt+OTgBxO0j5UikPp8JItfRTGFGzKV6F7mLpc4QQQvx3VOU5JCH+6+7pYNloNNKkSRM++OADABo2bMjJkyeZPXv2X1rtfv3110s1VcnMzMTPz4+uXbvi6Oj4l+d9W0wmNN99DcA6QwuaO6ZyVpsOQI+COLTGPB5S+7GDfE6YTlBoKgRgRKcR+Dv6lx1u7mLUV05RPT2y9I6gdnTpK23JhRDiv6z4HVUhxK3d08Gyj48PderUKbWtdu3a/PTTTwB4e3sDkJKSgo+Pj/mYlJQUIiIiKhzXysoKKyurMtu1Wi1arfYOzPw2xO6AC3spMFkwnRGsfqoHp9Z0AIzUTzoDQOs2b+Af9S2JWYkAuFq7EuQahKq8LnxDl0D072AylmxTa1DX7IX6bt2jEEKIe8Jd+1knxH3ong6WW7duTVRUVKlt586do0aNGoDysJ+3tzdbtmwxB8eZmZlERkby7LPP/tPTvX0mE2yfCsAyQ0daRoSTakolDyM2KgsC9Ubwa4G6dj8eIouPDipNSRp4NCg/UAZwDYRmT/1TdyCEEEII8a9UpQ5+ABkZGeU2HElNTb3jb+u89NJL7Nu3jw8++IDo6GiWLl3KnDlzGDNmDAAqlYpx48YxZcoU1q5dy4kTJxgxYgS+vr7079//js7lbxW3AxL3UmDS8rW+L4+2rMGJomoXdT0j0LwSCyPWgFpNv5B+WGuKmpHcWF9ZCCGEEELccVUOlocOHcry5cvLbF+5cuUdL9XWtGlTVq1axbJly6hXrx6TJ09m5syZPPLII+ZjXnnlFV544QVGjx5N06ZNyc7OZuPGjVgXdbe755lMsE1ZVV5q6Ej1GsHUq+bEyWsnAQh3DwcbZ9Aq9+Nk5cSzEc8S4BhA94Dud2vWQgghhBD/CSqTyWSqygmurq788ccf1K5du9T2s2fP0rp1a65fv35HJ/hPyMzMxMnJiYyMjH/+Ab+YrfDdAPKx5IH8T3lzaAf6RVTjoXUPcfr6aWa0m0G3gG7/7JyEEEL8q93Vn3tC3GeqvLJcUFCAXq8vs12n05GXl3dHJvWfYTLB9g8BWKrviNHei+71vCkwFHAu9RxQtLIshBBCCCHuiioHy82aNWPOnDllts+ePZvGjRvfkUn9ZyTuhQuRFGLJ1/o+DGvmj5WFhqjUKPQmPa7WrvjY+dx6HCGEEEII8beocjWMKVOm0LlzZ44dO0anTp0A2LJlCwcOHGDTpk13fIL/ZsbIOUx1cyGpsAaphS483Fypl1z8cF8993oVV7sQQgghhBB/uyqvLLdu3Zq9e/fi5+fHypUr+eWXXwgJCeH48eO0bdv275jjv1NWMmdiN7Lc0YEd7qnUqXUEHycbAE5dOwUowbIQQgghxJ10/fp1PD09iY+Pv9tT+dvMnj2bPn363JGxqhwsA0RERLBkyRJOnTrFwYMHmT9/PqGhoXdkQv8ZhxZxWlvy5U8wrWT7he3ADSvLbhIsCyGEEBV57LHHUKlUqFQqtFotgYGBvPLKK+Tn59+xa+zYsYOOHTvi6uqKra0toaGhjBw5ksLCQvMxJpOJOXPm0Lx5c+zt7XF2dqZJkybMnDnT3Fr8vffeQ6VS8cwzz5Qa/+jRo6hUKnPgGh8fj0qlwtPTk6ysrFLHRkRE8N5775V7/8Wv7t1vXSnr/fffp1+/fgQEBJi3HThwgE6dOuHs7IyLiwvdunXj2LFj5v1RUVF06NABLy8vrK2tCQoK4q233kKn0930Wi+++CKNGzfGysqq3IZxxV+XP7/s7OzMx2zevJmwsDAcHR159NFHS33tMzIyCAsLIyEhodS4TzzxBIcPH2bXrl23/HrcSpWD5Q0bNvDbb7+V2f7bb7/x66+//uUJ/ScYdHBoAWcsLQFQGW0xYeLVna9yKOUQ8ZnxgKwsCyGEELfSvXt3kpKSiI2N5dNPP+Wbb77h3XffvSNjnz59mu7du9OkSRN27tzJiRMn+Pzzz7G0tMRgMJiPe/TRRxk3bhz9+vVj27ZtHD16lLfffps1a9aUSlG1trZm3rx5nD9//pbXzsrKYsaMGbc8rvj+i1/Lli276fG5ubnMmzePUaNGmbdlZ2fTvXt3/P39iYyMZPfu3Tg4ONCtWzdzMKzVahkxYgSbNm0iKiqKmTNnMnfu3Ep9rZ944gkeeuihcvf973//KzX/pKQk6tSpw5AhQwAwGo08/PDDPPPMM+zdu5eDBw+Wenbutdde45lnnjE3rCtmaWnJww8/zKxZs245v1upcs7ya6+9xocfflhmu8lk4rXXXqNHjx5/eVL/emfXQVYSJ32rAdDe/SlyLfcRmRzJM5uV3zir2VfDxdrlbs5SCCHEf5TJZCJPZ7j1gX8DG62mSs/rWFlZ4e3tDYCfnx+dO3dm8+bNTJs2DVCCrWnTpjFnzhySk5MJCwvj7bffZvDgwQCkpaXx/PPPs2nTJrKzs6levTpvvPEGjz/+OJs2bcLb25vp06ebrxccHFxq9XblypUsWbKE1atX069fP/P2gIAA+vbtW6phW82aNfH09OTNN99k5cqVN72vF154gU8++YQxY8bg6elZqfuvjA0bNmBlZUWLFi3M286ePUtqaiqTJk3Cz88PgHfffZf69euTkJBASEgIQUFBBAUFmc+pUaMG27dvv+XKbXGwevXqVY4fP15mv729Pfb29ubPjx07xunTp5k9ezYA165d49q1azz33HNYW1vTt29fzpw5A8CePXs4cOAAX3zxRbnX7tOnD126dCEvLw8bG5vKfHnKVeVg+fz589SpU6fM9lq1ahEdHX3bE/lP2f8teuCc1gIwMax+G+p4D+GRDY+QkKm8jSAl44QQQtwteToDdd4p+y7yP+H0pG7YWlY5PAHg5MmT7Nmzp9Qq49SpU/n++++ZPXs2oaGh7Ny5k+HDh+Ph4UG7du14++23OX36NL/++ivu7u5ER0ebS+F6e3uTlJTEzp07eeCBB8q95pIlS6hZs2apQLmYSqXCycmp1LYPP/yQpk2bcvDgQZo0aVLhvQwbNozNmzczadKkCoNBgO3bt+Pp6YmLiwsdO3ZkypQpuLm5VXj8rl27ylQvq1mzJm5ubsybN4833ngDg8HAvHnzqF27dqlUjRtFR0ezceNGBg4cWOG1bse3335LWFiY+Tk4Dw8PfHx82LRpE507d2bXrl2MHDkSnU7Hs88+y/z589FoNOWO1aRJE/R6PZGRkbRv3/6251TlNAwnJydiY2PLbI+Oji6VXyIqkHIaEnYTY2mFQW1CbbKmuX8YTlZOfNHxCxwsHQAJloUQQojKWLduHfb29lhbWxMeHs6VK1eYMGECoPSG+OCDD5g/fz7dunUjKCiIxx57jOHDh/PNN98AkJiYSMOGDWnSpAkBAQF07tzZ/GDYkCFDGDZsGO3atcPHx4cBAwbwxRdflFotPn/+PDVr1qz0fBs1asSDDz7Iq6++etPjVCoVH374IXPmzCEmJqbcY7p3787ixYvZsmUL06ZNY8eOHfTo0aNUisifJSQk4OvrW2qbg4MD27dv5/vvv8fGxgZ7e3s2btzIr7/+ioVF6V9cWrVqhbW1NaGhobRt25ZJkyZV8s5vLT8/nyVLlpRKEVGpVKxcuZLJkydTt25dGjZsyBNPPMGHH35Ihw4dsLa2pnXr1tSsWbPMLxW2trY4OTmVyWeuqir/6tavXz/GjRvHqlWrCA4OBpRA+eWXX6Zv375/aTL/CQe+BWCdTS0gDV/bYNQq5XeWAKcA5nady6+xvzIw9M7+piaEEEJUlo1Ww+lJd6d7rI22/FXCinTo0IGvv/6anJwcPv30UywsLBg0aBCgxCe5ubl06dKl1DmFhYU0bNgQgGeffZZBgwZx+PBhunbtSv/+/WnVqhUAGo2GBQsWMGXKFLZu3UpkZCQffPAB06ZNY//+/fj4+FDFRsiAUoa3du3abNq06aYpFt26daNNmza8/fbbLF26tMz+oUOHmj8ODw+nfv36BAcHs337dnN53z/Ly8vD2tq6zLZRo0bRunVrli1bhsFgYMaMGfTq1YsDBw6USmFYsWIFWVlZHDt2jAkTJjBjxgxeeeWVqn4JyrVq1SqysrIYOXJkqe1t2rThwIED5s/PnTvH4sWLOXLkCA888ABjx46lR48e1KtXjwceeID69eubj7WxsTE/ZHm7qhwsT58+ne7du1OrVi2qV68OwMWLF2nbtm2lEtH/0/Iz4fgKAH7BC0ijRbX6pQ6p61aXum5178LkhBBCCIVKpbrtVIh/mp2dHSEhIQDMnz+fBg0amB9gy87OBmD9+vVUq1at1HlWVlYA9OjRg4SEBDZs2MDmzZvp1KkTY8aMKRXTVKtWjUcffZRHH32UyZMnExYWxuzZs5k4cSJhYWGcPXu2SnMODg7mqaee4rXXXmPevHk3PfbDDz+kZcuW5tXymwkKCjKnklQULLu7u5OWllZq29KlS4mPj2fv3r2o1WrzNhcXF9asWVMqKC/Oaa5Tpw4Gg4HRo0fz8ssvV5gKURXffvstvXv3xsvL66bHPf3003z88ccYjUaOHDnCkCFDsLW1pV27duzYsaNUsJyamoqHh8dfmtdtpWHs2bOH9evX89xzz/Hyyy+zZcsWtm7dirOz81+azL/e0SVQmM01mwBSrAoAiPCSwFgIIYS4E9RqNW+88QZvvfUWeXl51KlTBysrKxITEwkJCSn1Kg76QMmLHTlyJN9//z0zZ84st1NxMRcXF3x8fMjJyQHg4Ycf5ty5c6xZs6bMsSaTiYyMjHLHeeeddzh37hzLly+/6T01a9aMgQMH8tprr93y/i9evMj169fx8am4+2/Dhg05ffp0qW25ubmo1epSD1YWf240Giscy2g0otPpbnpMZcXFxbFt27ZSKRjlmTdvHq6urvTt29ecblJcsUOn05VKQYmJiSE/P9/8LsLtuq06yyqViq5duzJhwgSef/75CpPexQ2yr8J2pYrIfF1XNNaXAajtVvtuzkoIIYT4VxkyZAgajYYvv/wSBwcH/ve///HSSy+xaNEiYmJiOHz4MJ9//jmLFi0ClKB1zZo1REdHc+rUKdatW0ft2srP5m+++YZnn32WTf9v797Doqr2N4C/MzDMcL/fVFBUBMxEvCHSySOiaGZ69OctKzTNo2EnxSy1Q2peMLvosVT0hNpFu1iZqZUSKmQhAkoqoqIimAlekDsMyKzfH+Q+TDAICs6A7+d55nmYtdfs/d3Lcl43a6+9fz8uXLiAtLQ0vPbaa0hLS5PmNY8bNw7jx4/HxIkTsWLFCiQnJyMrKwt79uxBcHAwDh48WGedzs7OCA8Pb9DSZsuXL8eBAwdw9uxZqa24uBjz5s3DkSNHcOnSJcTGxmLkyJHo3LkzQkJ0T6EJCQlBWlqa1tXlwYMH49atWwgLC0N6ejrS0tIwZcoUGBsbY+DAgQCqb2T88ssvkZ6ejosXL+LLL7/EggULMH78eCgUCgDV0yi8vb21jnf+/HmkpqYiJycHZWVlSE1NRWpqqtZayUD1bwVcXV3rXVXt2rVrWLZsGd5//30A1f9w8fHxwZo1a5CQkIDY2FgEBgZK/X/++Wd07NhRmjZ8r+7pdywlJSWIi4tDdnZ2rZP917/+dV8FtVr7FgDl+SixewSbrneFqdGPMJGbwMPaQ9+VERERtRrGxsaYNWsWVq1ahZkzZ2Lp0qVwdHREZGQkLl68CBsbG/Ts2RMLFy4EUL0e74IFC3Dp0iWYmprib3/7m3S1t2/fvjh8+DBmzJiBP/74AxYWFnjkkUfw7bffYsCAAQCqLyBu374dmzZtwubNm7F8+XIYGxvD09MTzz33XL3B9ZVXXsGGDRvu+hCVLl264Pnnn9e64m1kZIQTJ07go48+Qn5+Ptq0aYMhQ4Zg6dKl0hSTujz66KPo2bMnvvzyS/zzn/8EUL2i2e7du7FkyRIEBARALpfDz88PP/74o3SV2tjYGG+99RbOnTsHIQTat2+PWbNmYc6cOdK+CwoKtAI9AEybNg1xcXHS+ztXeTMzM6WVNjQaDbZu3YrJkyfXO53j5Zdfxty5c7VuUNy6dStCQ0Oxdu1azJs3D3369JG2ffbZZ3jhhRd07q+hZKKRM9OPHz+OJ554AqWlpSgpKYGdnR1u3LgBMzMzODk51blShqErLCyEtbU1CgoKYGVl1fQHOP8T8OkYQCbHB502Yk3WBZi2245u9t3w2ZP1Lx5ORETU1Jr9e48M2t69ezFv3jycOnVKmqPc2qSlpSEoKAjnzp2rtXxfYzV6hObMmYMRI0bg1q1bMDU1xZEjR5CVlYVevXrxBr+6VJQCe8IBALd7T8PG89aQ/zkFw9veu75PEhERETW54cOHY/r06bhy5Yq+S2k2V69exccff3zfQRm4h2kYqamp2LhxI+RyOYyMjKBWq9GxY0esWrUKoaGhTb44dYsXvwrIzwKs2uJgm+koKj8HG9ccVAHwseN8ZSIiInrwZs+ere8SmlVwcHCT7avRV5YVCoV0yd7JyQnZ2dkAqlfJuHz5cpMV1irkpgG/Vk9CLwqKxNKY3wEIGJv+eXMfwzIRERGRQWv0lWU/Pz8kJSXB09MTAwYMwBtvvIEbN27gk08+Qbdu3ZqjxpZJCGD3bEBzGxqvJzE10QnZeXlo61CBQlEII5kRPG099V0lEREREdWj0VeWV6xYId0ZuXz5ctja2mLmzJm4fv16vesSPnRy04Dfj0IYq7ACU3A0Mw+WSmPMCjEDAHhYe0BlrLrLToiIiIhInxp9Zbl3797Sz05OTvjxxx+btKBW40IsAOCydW98+Jsachnw/tN+SC//GgCnYBARERG1BK1zvRBDcP4nAMDmnOqFsCOe7Iq/eznhzM3qR2LyYSREREREho9huTmoiyGyjwAA4jTd8bS/Oyb37wAASM9LBwB423HZOCIiIiJDx7DcHC4dhqyqAtkaR5RbdsCSpx6BTCZDfnk+rpZcBcCwTERERNQSMCw3hz/nK8druqN/Z0cojKqH+c5VZTdLN1iaWOqtPCIiInp43bx5E05OTrh06ZK+S2k2P/74I3r06AGNRnPf+2JYbg7nq8NynMYXAZ3speaTN04C4FVlIiKipjB58mTIZDLIZDIoFAp4eHjg1VdfRXl5eZMdIy4uDkFBQbCzs4OZmRk8PT0RGhqKiooKqY8QAps2bYK/vz8sLCxgY2OD3r17Y82aNSgtLQUALF68GDKZDDNmzNDaf2pqKmQymRRcL126BJlMBicnJxQVFWn17dGjBxYvXiy9v3Puf329/fbb9Z7T8uXLMXLkSHTo0EFqS0pKwqBBg2BjYwNbW1uEhITgt99+0/qcEALvvPMOunTpAqVSibZt22L58uX1HisvLw+TJk2ClZUVbGxsMHXqVBQXFzdqv8ePH4efnx8sLCwwYsQI5OXlSdtu376NXr164ejRo1r7HDp0KBQKBbZt21ZvfQ1xT2E5NjYWCxcuxLRp0/D8889rvZrSnf+war68vf8XNMvLyxEWFgZ7e3tYWFhgzJgxyM3NbdIaGi0vE8i7gEphhARNV/h72EEIgW3p27AhdQMAwM/JT781EhERtRJDhw7F1atXcfHiRaxevRobN27EokWLmmTfp0+fxtChQ9G7d2/Ex8fj5MmTeP/992FiYoKqqiqp37PPPovZs2dj5MiROHjwIFJTUxEREYFdu3Zh//79Uj+VSoXo6GhkZGTc9dhFRUV455136u1z9epVrdfmzZshk8kwZswYnZ8pLS1FdHQ0pk6dKrUVFxdj6NChcHd3R2JiIg4fPgxLS0uEhISgsrJS6vfyyy/jww8/xDvvvIMzZ87gu+++Q9++feutcdKkSUhLS0NMTAz27NmD+Ph4TJ8+XavP3fY7bdo0BAUF4dixYygoKMCKFSukbe+++y4CAwPrrGPy5MlYu3ZtvfU1iGikxYsXC7lcLvr27StGjhwpRo0apfVqSosWLRKPPPKIuHr1qvS6fv26tH3GjBnCzc1NxMbGiuTkZNGvXz/Rv3//Rh+noKBAABAFBQX3X/TR/wqxyEociegrAlfGirLKMrHw54Wi29ZuotvWbmJe3DxRfrv8/o9DRER0j+76vafRCKEu1s9Lo2nweYSGhoqRI0dqtY0ePVr4+flJ76uqqsSKFStEhw4dhEqlEt27dxc7duyQtufl5Ymnn35aODg4CJVKJTp37iw2b94shBBi9erVokOHDvXW8MUXXwgA4ttvv61jGDUiPz9fCFGdaXx9fcXgwYPF2LFjpT7Hjx8XAERmZqYQQojMzEwBQMybN09YWFiI3Nxcqa+vr69YtGiRzlpGjhwpgoKC6q13x44dwtHRUastKSlJABDZ2dlS24kTJwQAkZGRIYQQ4vTp08LY2FicOXOm3v3XdPr0aQFAJCUlSW0//PCDkMlk4sqVKw3er6mpqUhPTxdCCLF+/XrxxBNPCCGEuHDhgvD09BSFhYV1fi4rK0sAEOfPn29wzXVp9DrLUVFR2Lp1K5599tn7T+oNYGxsDBcXl1rtBQUFiI6Oxvbt2xEUFAQA2LJlC3x8fHDkyBH069fvgdRXU155HsozfgCMjbC30huPdCjGcz88h/S8dBjJjBDeKxzPdn0WMpnsgddGRETUYJWlwIo2+jn2wj8AE/N7+uipU6fw66+/on379lJbZGQkPv30U0RFRcHT0xPx8fF45pln4OjoiAEDBiAiIgKnT5/GDz/8AAcHB5w/fx5lZWUAABcXF1y9ehXx8fF4/PHH6zzmtm3b4OXlhZEjR9baJpPJYG1trdW2cuVK9OnTB8nJyVrPrviriRMnIiYmBm+++SY++OCDu557bm4u9u7di48++qjefj///DN69eql1ebl5QV7e3tER0dj4cKFqKqqQnR0NHx8fKSpGrt370bHjh2xZ88eDB06FEIIBAcHY9WqVbCzs6vzWAkJCdKUlDuCg4Mhl8uRmJiIf/zjHw3ar6+vL2JiYtC5c2fExsaie/fuAIAZM2Zg1apVsLSs+z4wd3d3ODs74+eff0anTp3uOoa6NDosV1RUoH///vd8wMbKyMhAmzZtoFKpEBAQgMjISLi7uyMlJQWVlZUIDg6W+np7e8Pd3R0JCQn1hmW1Wg21Wi29LywsBABUVlZq/bqhMXZf3I1FR/78tY9bWwDHgPJjQDlgo7TBysCV6OvSF7dv376n/RMRETWVe/2uM0R79uyBhYUFbt++DbVaDblcLoVLtVqNFStW4KeffkJAQAAAoGPHjjh8+DA2btyIAQMGIDs7G35+flKgqzmPd+zYsdi3bx8GDBgAFxcX9OvXD4MGDcJzzz0HKysrANU5xcvLq8H19uzZE+PGjcNrr72G2NhYnf1kMhlWrlyJESNGYM6cOXcNex999BEsLS0xevToevtlZWWhTRvtfwhZWlri0KFDGDVqFJYuXQoA8PT0xL59+2BsXB0VL168iKysLOzYsQMff/wxqqqqMGfOHPzf//0fDhw4UOexcnJy4OTkpNVmbGwMOzs75OTkNHi/H374IV588UW88847CAwMxIIFC/DJJ5/AzMwMffr0QUhICC5cuIAJEyZg2bJlWsdr06YNsrKy6h2Tu2l0WJ42bRq2b9+OiIiI+zpwQ/j7+2Pr1q3w8vLC1atXsWTJEvztb3/DqVOnkJOTAxMTE9jY2Gh9xtnZWfoD0CUyMhJLliyp1b5//36YmZndU60fFVf/S85YCMgFoIYCxnLAzagdxijH4MaxG/ge39/TvomIiJrSnZvOdFKYVV/h1QdF476HBw4ciA0bNqCkpASrV6+GsbGxNGf3/PnzKC0txeDBg7U+U1FRAT+/6vuHZs6ciTFjxuDYsWMYMmQIRo0aJV0UNDIywpYtW7Bs2TIcOHAAiYmJWLFiBd566y0cPXoUrq6uEEI0+hSXLVsGHx8f7N+/v1aYrCkkJASPPfYYIiIisH379nr3uXnzZkyaNAkqlarefmVlZbX6lJWVYerUqQgMDMRnn32GqqoqvPPOOxg+fDiSkpJgamoKjUYDtVqNjz/+GF26dAEAREdHo1evXjh79myj/sFQU0P2+8gjjyAuLk76zM2bN7Fo0SLEx8fjpZdeQv/+/fHNN9+gT58+8Pf3x4gRI6S+pqamd//v/S4aHZbLy8uxadMm/PTTT+jevTsUCoXW9vfee+++Cqpp2LBh0s/du3eHv78/2rdvjy+//BKmpqb3vN8FCxYgPDxcel9YWAg3NzcMGTJE+pdiY5RWlmLx14sBAN9cuYrj5f2w2mIuDoT/7Z5rJCIiai53fqOqk0x2z1MhHjRzc3N07twZQHVg9PX1lW5gu7Pqwt69e9G2bVutzymVSgDVWSMrKwvff/89YmJiMGjQIISFhWndXNe2bVs8++yzePbZZ7F06VJ06dIFUVFRWLJkCbp06YIzZ840quZOnTrhhRdewPz58xEdHV1v35UrVyIgIADz5s3T2efnn3/G2bNn8cUXX9z12A4ODrh165ZW2/bt23Hp0iUkJCRALpdLbba2tti1axcmTJgAV1dXGBsbS4EWAHx8qp9GnJ2dXWdYdnFxwbVr17Tabt++jby8PGmK7b3sNzw8HLNnz0a7du1w6NAhLFu2DObm5hg+fDgOHTqkFZbz8vLg6Oh413GpT6PD8okTJ9CjRw8A1XODamruubg2Njbo0qULzp8/j8GDB6OiogL5+flaV5dzc3PrnONck1KplP4nqUmhUNQK/w2RmpOK25rbaKuRoUPlbayp6o6ATvb3tC8iIqLm1lq/n+RyORYuXIjw8HA8/fTT6Nq1K5RKJbKzszFgwACdn3N0dERoaChCQ0Pxt7/9DfPmzdO5EoWtrS1cXV1RUlICAHj66acxYcIE7Nq1q9a8ZSEECgsLa81bBoA33ngDnTp1wueff17vOfXt2xejR4/G/Pnzdfa5cyXW19e33n0BgJ+fHz799FOtttLSUsjlcq0cd+f9nXWKAwMDcfv2bVy4cEGaEnLu3DkA0JojXlNAQADy8/ORkpIizZM+cOAANBoN/P3972m/sbGxSE9Px5YtWwAAVVVV0rSiv04vKi8vx4ULF6TfItyz+7o98AErKioStra24j//+Y/Iz88XCoVCfPXVV9L2M2fOCAAiISGhUfu939Uwlh9ZLrpt7SaWrG0vxCIr0fO1beLrlMv3tC8iIqLm1qSrQOlRXathVFZWirZt24q3335bCCHE66+/Luzt7cXWrVvF+fPnRUpKili7dq3YunWrEEKIiIgI8e2334qMjAxx6tQp8eSTT4q+ffsKIYSIiooSM2bMEPv27RPnz58Xp06dEq+++qqQy+Xi0KFDQojqFS/Gjx8vTE1NxfLly0VSUpK4dOmS2L17twgKChI7d+4UQvxvNYyaIiIihEqlqnM1jOPHj0v9zp49K4yNjYVKpaq1GkZBQYEwMzMTGzZsaNCYnThxQhgbG4u8vDypLT09XSiVSjFz5kxx+vRpcerUKfHMM88Ia2tr8ccffwghqlcV6dmzp3j88cfFsWPHRHJysvD39xeDBw+W9pOYmCi8vLzE77//LrUNHTpU+Pn5icTERHH48GHh6ekpJk6cKG1vyH7vKCsrE97e3lpjM2zYMPHCCy+I1NRU0a5dO/Hll19K2w4ePCgsLCxESUlJg8ZGl/sKy5cvXxaXLzdfKJw7d644dOiQyMzMFL/88osIDg4WDg4O4tq1a0KI6qXj3N3dxYEDB0RycrIICAgQAQEBjT7OvfylsSbmnHh95wmhrqwST3z9hOi2tZv46S1ncSLCV7R/bY+4cqu00XUQERE9CK05LAshRGRkpHB0dBTFxcVCo9GINWvWCC8vL6FQKISjo6MICQkRcXFxQgghli5dKnx8fISpqamws7MTI0eOFBcvXhRCCHHs2DHxzDPPCA8PD6FUKoW9vb14/PHHxXfffad1vKqqKrFhwwbRp08fYWZmJqysrESvXr3Ef/7zH1FaWp0H6grLBQUFwsHB4a5hWQghpk+fLgDUCssbN24Upqam0hJ1DdG3b18RFRWl1bZ//34RGBgorK2tha2trQgKCqp18fHKlSti9OjRwsLCQjg7O4vJkyeLmzdvStsPHjyodS5CCHHz5k0xceJEYWFhIaysrMSUKVNEUVFRo/Z7x/z588XcuXO12jIyMkSfPn2ElZWVmDlzpqiqqpK2TZ8+Xfzzn/9s8LjoIhOicTPTNRoNli1bhnfffVeaC2RpaYm5c+fi9ddfl+a6NIUJEyYgPj4eN2/ehKOjIx577DEsX75cukxfXl6OuXPn4rPPPoNarUZISAjWr19/12kYf3XnVyQFBQUNmrN8vUiNPst/AgA82dMEcWXhMBYCP2f9jk0VY7DL5hnEzRvY+BMmIiJ6ABr7vUety969ezFv3jycOnWqSXObIblx4wa8vLyQnJwMDw+P+9pXo+csv/7664iOjsbKlSsRGBgIADh8+DAWL16M8vLyuz72sDHuNo9HpVJh3bp1WLduXZMdsyGOZf9vYnxcZgzgAviWq1Fo5oONZU9ilId9PZ8mIiIi0p/hw4cjIyMDV65cgZubm77LaRaXLl3C+vXr7zsoA/cQlj/66CN8+OGHeOqpp6S27t27o23btnjxxRebNCwbqpSs6rDsZqWAm0UMTgLwv22MBcoFUMMIAZ0YlomIiMhwzZ49W98lNKvevXvX+9CXxmj0tfe8vDx4e3vXavf29kZeXl6TFGXo7oTljW2+xgWzCgDAnvyJ+PmqEQDAv2PdT7IhIiIiopal0WHZ19e3zscufvDBBw1asqSlU9+uwsnfC/CMUQxKcnaiVC6HSqNCWrEfNALoYG8GV+t7XwOaiIiIiAxHo6dhrFq1CsOHD9d6dGRCQgIuX76M779v/U+oO3WlACZVxfi3ahui/nwwyuMdgpBywwKXbpYioJODniskIiIioqbS6CvLAwYMwLlz5/CPf/wD+fn5yM/Px+jRo3H27Fn87W+t/4l1KVm3MEB+AipU4BfL6kXG/+7+GD6Z6o8ZAzrhpaDOeq6QiIiIiJpKo68sA0CbNm0eihv56pJ86RaeNErCDbkcZ6qnKKN/m/6wNzXD/GG153ITERERUcvVoLB84sQJdOvWDXK5HCdOnKi3b/fu3ZukMEMkhMDJrGt4V56KQ2YqAICPnQ/sTbn6BREREVFr1KCw3KNHD+Tk5MDJyQk9evSATCZDXc8ykclkqKqqavIiDUXWzVJ0KUuFpUkZDlu2BQA81vYxPVdFRERERM2lQWE5MzMTjo6O0s8Pq5SsWxgiT4YAcMRUBYhK9G/TX99lERERETXYzZs34ePjg6NHj6JDhw76LqfRfvzxR8yfPx/Hjh17IE8gbNAR2rdvD5lMBgDIyspC27Zt0b59e61X27ZtkZWV1azF6tuxrJsYbJSCm0Zy5IlKyGVydHdsvdNOiIiIDNnkyZMhk8kgk8mgUCjg4eGBV199FeXl5U12jLi4OAQFBcHOzg5mZmbw9PREaGgoKioqpD5CCGzatAn+/v6wsLCAjY0NevfujTVr1qC0tBQAsHjxYshkMsyYMUNr/6mpqZDJZLh06RKA6ifPyWQyODk5oaioSKtvjx49sHjxYun94sWL4e3tDXNzc9ja2iI4OBiJiYl3Pafly5dj5MiRWkE5KSkJgwYNgo2NDWxtbRESEoLffvtN61h3xrrmy9zcvN5j1fWZmk9oPn78OPz8/GBhYYERI0ZoPbPj9u3b6NWrF44ePaq1z6FDh0KhUGDbtm13Pdem0Og4PnDgwDofPlJQUICBAwc2SVGGquRCIpxk+chUWgEAXMxcYGJkoueqiIiIHl5Dhw7F1atXcfHiRaxevRobN27EokWLmmTfp0+fxtChQ9G7d2/Ex8fj5MmTeP/992FiYqI17fTZZ5/F7NmzMXLkSBw8eBCpqamIiIjArl27sH//fqmfSqVCdHQ0MjIy7nrsoqIivPPOO/X26dKlCz744AOcPHkShw8fRocOHTBkyBBcv35d52dKS0sRHR2NqVOnSm3FxcUYOnQo3N3dkZiYiMOHD8PS0hIhISGorKwEALzyyiu4evWq1qtr164YO3bsXc9ly5YtWp8bNWqUtG3atGkICgrCsWPHUFBQgBUrVkjb3n33XQQGBqJv37619jl58mSsXbv2rsduEqKRZDKZuHbtWq32s2fPCktLy8buziAUFBQIAKKgoEBnn/zSCrHh9UlCLLISOz59UnTb2k1M3Tf1AVZJRETUNO72vafRaERJRYleXhqNpsHnERoaKkaOHKnVNnr0aOHn5ye9r6qqEitWrBAdOnQQKpVKdO/eXezYsUPanpeXJ55++mnh4OAgVCqV6Ny5s9i8ebMQQojVq1eLDh061FvDF198IQCIb7/9ts5xzM/PF0IIsWjRIuHr6ysGDx4sxo4dK/U5fvy4ACAyMzOFEEJkZmYKAGLevHnCwsJC5ObmSn19fX3FokWLdNZy58/1p59+0tlnx44dwtHRUastKSlJABDZ2dlS24kTJwQAkZGRUed+UlNTBQARHx+v81hCCAFA7Ny5U+d2U1NTkZ6eLoQQYv369eKJJ54QQghx4cIF4enpKQoLC+v8XFZWlgAgzp8/X+/xm0KDl44bPXo0gOrL6ZMnT4ZSqZS2VVVV4cSJE+jfv/XO3z2elYch8iQAwFUHdyDnEtwt3fVcFRERUdMru10G/+3+ejl24tOJMFOY3dNnT506hV9//RXt27eX2iIjI/Hpp58iKioKnp6eiI+PxzPPPANHR0cMGDAAEREROH36NH744Qc4ODjg/PnzKCsrAwC4uLjg6tWriI+Px+OPP17nMbdt2wYvLy+MHDmy1jaZTAZra2uttpUrV6JPnz5ITk5G7969dZ7LxIkTERMTgzfffLPOJyf/VUVFBTZt2gRra+t6n6j8888/o1evXlptXl5esLe3R3R0NBYuXIiqqipER0fDx8dH55zmDz/8EF26dGnQMzbCwsIwbdo0dOzYETNmzMCUKVOk6b2+vr6IiYlB586dERsbK62qNmPGDKxatQqWlpZ17tPd3R3Ozs74+eef0alTp7vWcD8aHJbv/GELIWBpaQlT0/890tnExAT9+vXDCy+80PQVGohLZ47h7/Ic3JYpcNmkeuoFwzIREZF+7dmzBxYWFrh9+zbUajXkcrkULtVqNVasWKH11OGOHTvi8OHD2LhxIwYMGIDs7Gz4+flJwbVmOBw7diz27duHAQMGwMXFBf369cOgQYPw3HPPwcqqekpmRkYGvLy8Glxvz549MW7cOLz22muIjY3V2U8mk2HlypUYMWIE5syZozMQ7tmzBxMmTEBpaSlcXV0RExMDBwfdTxPOyspCmzZttNosLS1x6NAhjBo1CkuXLgUAeHp6Yt++fTA2rh0Vy8vLsW3bNsyfP/+u5/vmm28iKCgIZmZm2L9/P1588UUUFxfjX//6F4Dq0P3iiy/inXfeQWBgIBYsWIBPPvkEZmZm6NOnD0JCQnDhwgVMmDABy5Yt09p3mzZtHsj9cg0Oy1u2bAFQ/R/RK6+8ctcJ3a2N6cUfAQC5Dv1wuSQHAOBm6abPkoiIiJqFqbEpEp+++41izXXsxhg4cCA2bNiAkpISrF69GsbGxhgzZgwA4Pz58ygtLcXgwYO1PlNRUQE/Pz8AwMyZMzFmzBgcO3YMQ4YMwahRo6TflBsZGWHLli1YtmwZDhw4gMTERKxYsQJvvfUWjh49CldX1zqX0r2bZcuWwcfHB/v374eTk5POfiEhIXjssccQERGB7du36zz/1NRU3LhxA//9738xbtw4JCYm6txvWVkZVCpVrbapU6ciMDAQn332GaqqqvDOO+9g+PDhSEpK0rpACgA7d+5EUVERQkND73quERER0s9+fn4oKSnB22+/LYXlRx55BHFxcVKfmzdvYtGiRYiPj8dLL72E/v3745tvvkGfPn3g7++PESNGSH1NTU2lGyibU6Nv8Fu0aNFDF5RvV2nQtSAeACD3eRLZRdkAADcrhmUiImp9ZDIZzBRmennd+fV8Q5mbm6Nz587w9fXF5s2bkZiYiOjoaADVN64BwN69e5Gamiq9Tp8+ja+++goAMGzYMGRlZWHOnDn4448/MGjQILzyyitax2jbti2effZZfPDBB0hLS0N5eTmioqIAVN9kd+bMmUbV3KlTJ7zwwguYP3/+XcP2ypUr8cUXX+D48eP1nn+/fv0QHR0NY2Nj6fzr4uDggFu3bmm1bd++HZcuXcKWLVvQp08f9OvXD9u3b0dmZiZ27dpVax8ffvghnnzySTg7OzfgbLX5+/vj999/h1qtrnN7eHg4Zs+ejXbt2uHQoUMYO3YszM3NMXz4cBw6dEirb15enrS0cXO6p8XpvvrqK4wbNw79+vVDz549tV6t0YXzZ/Go7CI0QgZV90EorCgEALSzaKfnyoiIiOgOuVyOhQsX4t///jfKysrQtWtXKJVKZGdno3PnzlovN7f/XfBydHREaGgoPv30U6xZswabNm3SeQxbW1u4urqipKQEAPD000/j3LlzdYZKIQQKCgrq3M8bb7yBc+fOaS2jVpe+ffti9OjRDZryAAAajUZnEAWqr+6ePn1aq620tBRyuVzrHyp33ms0Gq2+mZmZOHjwoNZqGo2RmpoKW1tbrXvf7oiNjUV6ejpmzZoFoPqeuDurcVRWVmqtQFJeXo4LFy5IvyFoTo0Oy2vXrsWUKVPg7OyM48ePo2/fvrC3t8fFixcxbNiw5qhR7279thcAcF7VFb+L6rUbHU0d7/kGBCIiImoeY8eOhZGREdatWwdLS0u88sormDNnDj766CNcuHABx44dw/vvv4+PPvoIQHVo3bVrF86fP4+0tDTs2bMHPj4+AICNGzdi5syZ2L9/Py5cuIC0tDS89tprSEtLk6YDjBs3DuPHj8fEiROxYsUKJCcnIysrC3v27EFwcDAOHjxYZ53Ozs4IDw9v0PJny5cvx4EDB3D27FmpraSkBAsXLsSRI0eQlZWFlJQUPP/887hy5Uq9y7mFhIQgLS1N6+ry4MGDcevWLYSFhSE9PR1paWmYMmUKjI2Nay0LvHnzZri6utaZ+Xbu3Alvb2/p/e7du/Hhhx/i1KlTOH/+PDZs2IAVK1bgpZdeqvXZ8vJyzJo1C5s2bZIeNBIYGIh169bht99+w9dff43AwECp/5EjR6BUKqW56M2qsctneHl5ie3btwshhLCwsBAXLlwQQggREREhwsLCmm6djgfobkvoJESFCbHISiR8MFXsvbBXdNvaTTz3/XMPuEoiIqKm0ZAlU1uCupaOE0KIyMhI4ejoKIqLi4VGoxFr1qwRXl5eQqFQCEdHRxESEiLi4uKEEEIsXbpU+Pj4CFNTU2FnZydGjhwpLl68KIQQ4tixY+KZZ54RHh4eQqlUCnt7e/H444+L7777Tut4VVVVYsOGDaJPnz7CzMxMWFlZiV69eon//Oc/orS0VAjxv6XjaiooKBAODg51Lh13/Phxrb7Tp08XAKSl48rKysQ//vEP0aZNG2FiYiJcXV3FU089JY4ePXrXcevbt6+IiorSatu/f78IDAwU1tbWwtbWVgQFBYmEhIRa59muXTuxcOHCOve7ZcsWUTNa/vDDD6JHjx7CwsJCmJubC19fXxEVFSWqqqpqfXb+/Pli7ty5Wm0ZGRmiT58+wsrKSsycOVPrc9OnTxf//Oc/73quTUEmRONmppuZmSE9PR3t27eHk5MTYmJi4Ovri4yMDPTr1w83b95s+kTfzAoLC2FtbY2CggLp7taakteMR+/8H5HgMQu/9XDDutR1GNV5FJYGLtVDtURERPfnbt971Lrt3bsX8+bNw6lTpx7I46Kb2o0bN+Dl5YXk5GR4eHg0+/EaPUIuLi7SE/zc3d1x5MgRANVzWBqZu1sMZfkNAIDc0hmXiy4D4LJxRERE1DINHz4c06dPx5UrV/Rdyj25dOkS1q9f/0CCMtCIpePuCAoKwnfffQc/Pz9MmTIFc+bMwVdffYXk5GTpwSWtjXll9dVyhbUrLhclA+CycURERNRyzZ49W98l3LPevXvX+0CXptbosLxp0ybpzsiwsDDY29vj119/xVNPPYV//vOfTV6gIbCuqp4Eb2rniuxzXDaOiIiI6GHR6LAsl8u15rdMmDABEyZMaNKiDImoug0bUQDIACNrW9wsr77KzCvLRERERK1fg8LyiRMnGrzDO8/0bi1KC67BXCagETKUqG4DAGyVtrAy4Q0RRERERK1dg8Jyjx49IJPJIIS465N1ai4Y3RoUXrsCcwB5sMKNymsAeFWZiIiI6GHRoNUwMjMzcfHiRWRmZuLrr7+Gh4cH1q9fj+PHj+P48eNYv349OnXqhK+//rpZi125ciVkMpnWpPTy8nJp7rSFhQXGjBmD3NzcJjtmcV71naL5cltkF3K+MhEREdHDpEFXltu3by/9PHbsWKxduxZPPPGE1Na9e3e4ubkhIiICo0aNavIiASApKQkbN26sNc1jzpw52Lt3L3bs2AFra2vMmjULo0ePxi+//NIkx1XfugoAKDK247JxRERERA+ZRq+zfPLkyTrXtfPw8Kj1rPGmUlxcjEmTJuG///0vbG1tpfaCggJER0fjvffeQ1BQEHr16oUtW7bg119/ldZ/vl+3C3MAAGUmdsgu+vPKMqdhEBERET0UGr0aho+PDyIjI/Hhhx/CxMQEAFBRUYHIyEjpWepNLSwsDMOHD0dwcDCWLVsmtaekpKCyshLBwcFSm7e3N9zd3ZGQkIB+/frVuT+1Wg21Wi29LywsBABUVlaisrJSq6/4MyyrVQ64XHgRAOBq6lqrHxERUUvB7zCihmt0WI6KisKIESPQrl07aUrEiRMnIJPJsHv37iYv8PPPP8exY8eQlJRUa1tOTg5MTExgY2Oj1e7s7IycnByd+4yMjMSSJUtqte/fvx9mZmZabU43sgAAV0oFckqr93ku8RyuyFvmU2+IiIhKS0v1XQJRi9HosNy3b19cvHgR27Ztw5kzZwAA48ePx9NPPw1zc/MmLe7y5ct4+eWXERMTA5VK1WT7XbBgAcLDw6X3hYWFcHNzw5AhQ2Blpb0k3MUz7wGVgKx9O0ANWCgs8H/D/++uq4IQEREZqju/USWiu2t0WAYAc3NzTJ8+valrqSUlJQXXrl1Dz549pbaqqirEx8fjgw8+wL59+1BRUYH8/Hytq8u5ublwcXHRuV+lUgmlUlmrXaFQQKFQaLWZV+YBAArNjQF19XzlO9NPiIiIWqK/ftcRkW4NCsvfffcdhg0bBoVCge+++67evk899VSTFAYAgwYNwsmTJ7XapkyZAm9vb7z22mtwc3ODQqFAbGwsxowZAwA4e/YssrOzERAQ0CQ1WFVVh+VbiuoHkvDmPiIiIqKHR4PC8qhRo5CTkwMnJ6d6l4aTyWRN+lASS0tLdOvWTavN3Nwc9vb2UvvUqVMRHh4OOzs7WFlZ4aWXXkJAQIDOm/sa5bYaVqIYAHBLXj2/y92Ky8YRERERPSwaFJY1Gk2dPxuC1atXQy6XY8yYMVCr1QgJCcH69eubZN9VRbkwAlAhjHCj6gYArrFMRERE9DC5pznL+nTo0CGt9yqVCuvWrcO6deua/FhFN/6ADYAbsEZu6R8AgHaW7Zr8OERERERkmBoUlteuXdvgHf7rX/+652IMTfHNK7ABcF1mgz9KqsMyrywTERERPTwaFJZXr17doJ3JZLJWFZbL/3zU9SWVNarETaiMVHA0c9RzVURERET0oDQoLGdmZjZ3HQapsqD6ISS/q8wA3EQ7y3aQyxr9hHAiIiIiaqGY/OpTnAsAuKqqXleZy8YRERERPVzu6Qa/33//Hd999x2ys7NRUVGhte29995rksIMgbz0OgAgx6R6BRAPaw99lkNERERED1ijw3JsbCyeeuopdOzYEWfOnEG3bt1w6dIlCCG0nrTXGijLq5eLyzEuAwTQxbaLnisiIiIiogep0dMwFixYgFdeeQUnT56ESqXC119/jcuXL2PAgAEYO3Zsc9SoN2YVN6EBkINbABiWiYiIiB42jQ7L6enpeO655wAAxsbGKCsrg4WFBd5880289dZbTV6gPlnezsMfxkZQCzWM5cboYN1B3yURERER0QPU6LBsbm4uzVN2dXXFhQsXpG03btxousr0TV0MU5Qjw6T65r5O1p2gkCv0XBQRERERPUiNnrPcr18/HD58GD4+PnjiiScwd+5cnDx5Et988w369evXHDXqx58rYaQpTAFwCgYRERHRw6jRYfm9995DcXExAGDJkiUoLi7GF198AU9Pz1a1EkZ5fg5UANJMGJaJiIiIHlaNDssdO3aUfjY3N0dUVFSTFmQoim9egQrABRNjAIJhmYiIiOgh1Og5y9OmTcOhQ4eaoRTDUpZ3FWUyGXIUAgDQxY5hmYiIiOhh0+iwfP36dQwdOhRubm6YN28efvvtt+aoS+8qC3JwUaGAkAF2KjvYq+z1XRIRERERPWCNDsu7du3C1atXERERgaSkJPTs2ROPPPIIVqxYgUuXLjVDifohinNxzqR69QtPW0/IZDI9V0RERERED1qjwzIA2NraYvr06Th06BCysrIwefJkfPLJJ+jcuXNT16c38tLrUljmfGUiIiKih9M9heU7KisrkZycjMTERFy6dAnOzs5NVZfemZRdx7k/11hmWCYiIiJ6ON1TWD548CBeeOEFODs7Y/LkybCyssKePXvw+++/N3V9eqOquMkry0REREQPuUYvHde2bVvk5eVh6NCh2LRpE0aMGAGlUtkctemPEFCLfOQbuUAGGTrZdNJ3RURERESkB40Oy4sXL8bYsWNhY2PTDOUYiLJbuGhSfdHdSdUOSqNW9o8BIiIiImqQRoflF154oTnqMCzF16QpGB7WnnouhoiIiIj05b5u8GutNEW50s19PvZeeq6GiIiIiPSFYbkOJXl/SFeWuzt567kaIiIiItIXhuU6FOb9jkxFdVjuas+wTERERPSwYliuQ2b+BdyWyaDUyOFq7qrvcoiIiIhITxiW63CxpHq9aMcqSz7mmoiIiOghZtBhecOGDejevTusrKxgZWWFgIAA/PDDD9L28vJyhIWFwd7eHhYWFhgzZgxyc3Pv+7iXbt8EADjA8b73RUREREQtl0GH5Xbt2mHlypVISUlBcnIygoKCMHLkSKSlpQEA5syZg927d2PHjh2Ii4vDH3/8gdGjR9/3cS+hBADgqHC/730RERERUcvV6HWWH6QRI0ZovV++fDk2bNiAI0eOoF27doiOjsb27dsRFBQEANiyZQt8fHxw5MgR9OvX756Pm2VUBUAOF3OusUxERET0MDPosFxTVVUVduzYgZKSEgQEBCAlJQWVlZUIDg6W+nh7e8Pd3R0JCQn1hmW1Wg21Wi29LywsBABUVlaiQl2OPKPqecr2lh1RWVnZTGdERESkH/xuI2o4gw/LJ0+eREBAAMrLy2FhYYGdO3eia9euSE1NhYmJSa3Hbjs7OyMnJ6fefUZGRmLJkiW12vfv3w+lohy3/7yp7/ffC/D999832bkQEREZgtLSUn2XQNRiGHxY9vLyQmpqKgoKCvDVV18hNDQUcXFx97XPBQsWIDw8XHpfWFgINzc3DBkyBMWFaUA8YKIRGDJgEHq3t73fUyAiIjIod36jSkR3Z/Bh2cTEBJ07dwYA9OrVC0lJSfjPf/6D8ePHo6KiAvn5+VpXl3Nzc+Hi4lLvPpVKJZRKZa12hUKBkrLq1TQsNYCTtRkUfz6chIiIqLXgdxtRwxn0ahh10Wg0UKvV6NWrFxQKBWJjY6VtZ8+eRXZ2NgICAu55/zcLrgAAzKpkcDCvHaiJiIiI6OFh0FeWFyxYgGHDhsHd3R1FRUXYvn07Dh06hH379sHa2hpTp05FeHg47OzsYGVlhZdeegkBAQH3tRLGtcKrAAClxhiWKoMeHiIiIiJqZgadBq9du4bnnnsOV69ehbW1Nbp37459+/Zh8ODBAIDVq1dDLpdjzJgxUKvVCAkJwfr16+/rmDeLrwMAlBoTyOV8eh8RERHRw8ygw3J0dHS921UqFdatW4d169Y12THzy28BAJQwbbJ9EhEREVHL1OLmLDe3worqO4SVMnM9V0JERERE+saw/BfFVdVrT6qMrfVcCRERERHpG8PyXxSLcgCAqYLrKxMRERE97BiW/6JYVD8C1EzlqOdKiIiIiEjfGJb/oliuAQBYW7TRcyVEREREpG8MyzVV3UbRnyNiY9VOv7UQERERkd4xLNcgSm+hQF49JE527nquhoiIiIj0jWG5huKi31Elq34QSVtbZz1XQ0RERET6xrBcQ37hFQCAiUbA1cpSz9UQERERkb4xLNeQe+t3AIC5RgZbMxM9V0NERERE+sawXENeUQ4AwLTKGAojDg0RERHRw46JsIa84hsAAJVQ6LkSIiIiIjIEDMs1FKhvAQBUUOm5EiIiIiIyBAzLNRRXFgEAVHJzPVdCRERERIaAYbmGkqoSAIDKyErPlRARERGRIWBYrqFEUw4AMDOx03MlRERERGQIGJZrKEUlAMBC5aDnSoiIiIjIEDAs11AiqwIA2Ji76LkSIiIiIjIEDMs1FP85GrbW7fRbCBEREREZBIblGgr+fBCJsy3DMhERERExLGspklcPh5uNo54rISIiIiJDwLBcg0YmAwC0Z1gmIiIiIjAs16LUANampvoug4iIiIgMAMPyX5hqjPRdAhEREREZCIblvzAVCn2XQEREREQGgmH5L0yh1HcJRERERGQgDDosR0ZGok+fPrC0tISTkxNGjRqFs2fPavUpLy9HWFgY7O3tYWFhgTFjxiA3N/eej2kqM7/fsomIiIiolTDosBwXF4ewsDAcOXIEMTExqKysxJAhQ1BSUiL1mTNnDnbv3o0dO3YgLi4Of/zxB0aPHn3PxzQ1tmyK0omIiIioFTDWdwH1+fHHH7Xeb926FU5OTkhJScHjjz+OgoICREdHY/v27QgKCgIAbNmyBT4+Pjhy5Aj69evX6GOaK2yaonQiIiIiagUMOiz/VUFBAQDAzs4OAJCSkoLKykoEBwdLfby9veHu7o6EhASdYVmtVkOtVkvvCwsLpZ8tlPaorKxsjvKJiIgMAr/niBquxYRljUaD2bNnIzAwEN26dQMA5OTkwMTEBDY2Nlp9nZ2dkZOTo3NfkZGRWLJkSZ3bivJv4/vvv2+yuomIiAxNaWmpvksgajFaTFgOCwvDqVOncPjw4fve14IFCxAeHi69LywshJubGwCgu08fPNHnifs+BhERkaGq+RtVIqpfiwjLs2bNwp49exAfH4927dpJ7S4uLqioqEB+fr7W1eXc3Fy4uLjo3J9SqYRSWfcScW3s2kKh4FrLRETUevF7jqjhDHo1DCEEZs2ahZ07d+LAgQPw8PDQ2t6rVy8oFArExsZKbWfPnkV2djYCAgLu6ZhuNk73VTMRERERtR4GfWU5LCwM27dvx65du2BpaSnNQ7a2toapqSmsra0xdepUhIeHw87ODlZWVnjppZcQEBBwTythAEAbS7umPAUiIiIiasEMOixv2LABAPD3v/9dq33Lli2YPHkyAGD16tWQy+UYM2YM1Go1QkJCsH79+ns+pq3K5p4/S0RERESti0wIIfRdhL4VFhbC2toaPdZ1xfEX0/RdDhERUbO6871XUFAAKysrfZdDZNAMes7yg6YSBn2hnYiIiIgeMIblGsxgou8SiIiIiMiAMCzXoJKZ6bsEIiIiIjIgDMs1mBmb67sEIiIiIjIgDMs1mBtb67sEIiIiIjIgDMs1WJja67sEIiIiIjIgDMs1WJs66rsEIiIiIjIgDMs12Fu30XcJRERERGRAGJZrcLHiNAwiIiIi+h+G5RpcLRmWiYiIiOh/GJZrsFPZ6LsEIiIiIjIgDMs1WCmt9F0CERERERkQhuUaGJaJiIiIqCaG5RoUcoW+SyAiIiIiA8KwTERERESkA8MyEREREZEODMtERERERDowLBMRERER6cCwTERERESkA8MyEREREZEODMtERERERDowLBMRERER6cCwTERERESkA8MyEREREZEODMtERERERDoYfFiOj4/HiBEj0KZNG8hkMnz77bda24UQeOONN+Dq6gpTU1MEBwcjIyNDP8USERERUati8GG5pKQEvr6+WLduXZ3bV61ahbVr1yIqKgqJiYkwNzdHSEgIysvLH3ClRERERNTaGOu7gLsZNmwYhg0bVuc2IQTWrFmDf//73xg5ciQA4OOPP4azszO+/fZbTJgw4UGWSkREREStjMFfWa5PZmYmcnJyEBwcLLVZW1vD398fCQkJeqyMiIiIiFoDg7+yXJ+cnBwAgLOzs1a7s7OztK0uarUaarVael9YWAgAqKysRGVlZTNUSkREZDj4XUfUcC06LN+ryMhILFmypFb7/v37YWZmpoeKiIiIHpzS0lJ9l0DUYrTosOzi4gIAyM3Nhaurq9Sem5uLHj166PzcggULEB4eLr0vLCyEm5sbhgwZAisrq2arl4iIyBDc+Y0qEd1diw7LHh4ecHFxQWxsrBSOCwsLkZiYiJkzZ+r8nFKphFKprNWuUCigUCiaq1wiIiKDwO86ooYz+LBcXFyM8+fPS+8zMzORmpoKOzs7uLu7Y/bs2Vi2bBk8PT3h4eGBiIgItGnTBqNGjdJf0URERETUKhh8WE5OTsbAgQOl93emT4SGhmLr1q149dVXUVJSgunTpyM/Px+PPfYYfvzxR6hUKn2VTERERESthEwIIfRdhL4VFhbC2toaBQUFnLNMREStHr/3iBquRa+zTERERETUnBiWiYiIiIh0YFgmIiIiItKBYZmIiIiISAeGZSIiIiIiHRiWiYiIiIh0YFgmIiIiItKBYZmIiIiISAeGZSIiIiIiHRiWiYiIiIh0YFgmIiIiItKBYZmIiIiISAeGZSIiIiIiHRiWiYiIiIh0YFgmIiIiItKBYZmIiIiISAeGZSIiIiIiHRiWiYiIiIh0YFgmIiIiItKBYZmIiIiISAeGZSIiIiIiHRiWiYiIiIh0YFgmIiIiItKBYZmIiIiISAeGZSIiIiIiHRiWiYiIiIh0aDVhed26dejQoQNUKhX8/f1x9OhRfZdERERERC1cqwjLX3zxBcLDw7Fo0SIcO3YMvr6+CAkJwbVr1/RdGhERERG1YK0iLL/33nt44YUXMGXKFHTt2hVRUVEwMzPD5s2b9V0aEREREbVgLT4sV1RUICUlBcHBwVKbXC5HcHAwEhIS9FgZEREREbV0xvou4H7duHEDVVVVcHZ21mp3dnbGmTNn6vyMWq2GWq2W3hcUFAAA8vLyUFlZ2XzFEhERGYCioiIAgBBCz5UQGb4WH5bvRWRkJJYsWVKr3cPDQw/VEBER6cfNmzdhbW2t7zKIDFqLD8sODg4wMjJCbm6uVntubi5cXFzq/MyCBQsQHh4uvc/Pz0f79u2RnZ3NvzSaSWFhIdzc3HD58mVYWVnpu5xWiWPc/DjGDwbHufkVFBTA3d0ddnZ2+i6FyOC1+LBsYmKCXr16ITY2FqNGjQIAaDQaxMbGYtasWXV+RqlUQqlU1mq3trbmX8zNzMrKimPczDjGzY9j/GBwnJufXN7ib10ianYtPiwDQHh4OEJDQ9G7d2/07dsXa9asQUlJCaZMmaLv0oiIiIioBWsVYXn8+PG4fv063njjDeTk5KBHjx748ccfa930R0RERETUGK0iLAPArFmzdE67uBulUolFixbVOTWDmgbHuPlxjJsfx/jB4Dg3P44xUcPJBNeNISIiIiKqE2f2ExERERHpwLBMRERERKQDwzIRERERkQ4My0REREREOjz0YXndunXo0KEDVCoV/P39cfToUX2X1GJFRkaiT58+sLS0hJOTE0aNGoWzZ89q9SkvL0dYWBjs7e1hYWGBMWPG1Hr6IjXcypUrIZPJMHv2bKmNY9w0rly5gmeeeQb29vYwNTXFo48+iuTkZGm7EAJvvPEGXF1dYWpqiuDgYGRkZOix4palqqoKERER8PDwgKmpKTp16oSlS5ei5j3nHOPGiY+Px4gRI9CmTRvIZDJ8++23WtsbMp55eXmYNGkSrKysYGNjg6lTp6K4uPgBngWR4Xmow/IXX3yB8PBwLFq0CMeOHYOvry9CQkJw7do1fZfWIsXFxSEsLAxHjhxBTEwMKisrMWTIEJSUlEh95syZg927d2PHjh2Ii4vDH3/8gdGjR+ux6pYrKSkJGzduRPfu3bXaOcb379atWwgMDIRCocAPP/yA06dP491334Wtra3UZ9WqVVi7di2ioqKQmJgIc3NzhISEoLy8XI+VtxxvvfUWNmzYgA8++ADp6el46623sGrVKrz//vtSH45x45SUlMDX1xfr1q2rc3tDxnPSpElIS0tDTEwM9uzZg/j4eEyfPv1BnQKRYRIPsb59+4qwsDDpfVVVlWjTpo2IjIzUY1Wtx7Vr1wQAERcXJ4QQIj8/XygUCrFjxw6pT3p6ugAgEhIS9FVmi1RUVCQ8PT1FTEyMGDBggHj55ZeFEBzjpvLaa6+Jxx57TOd2jUYjXFxcxNtvvy215efnC6VSKT777LMHUWKLN3z4cPH8889rtY0ePVpMmjRJCMExvl8AxM6dO6X3DRnP06dPCwAiKSlJ6vPDDz8ImUwmrly58sBqJzI0D+2V5YqKCqSkpCA4OFhqk8vlCA4ORkJCgh4raz0KCgoAAHZ2dgCAlJQUVFZWao25t7c33N3dOeaNFBYWhuHDh2uNJcAxbirfffcdevfujbFjx8LJyQl+fn7473//K23PzMxETk6O1jhbW1vD39+f49xA/fv3R2xsLM6dOwcA+O2333D48GEMGzYMAMe4qTVkPBMSEmBjY4PevXtLfYKDgyGXy5GYmPjAayYyFK3mCX6NdePGDVRVVdV6JLazszPOnDmjp6paD41Gg9mzZyMwMBDdunUDAOTk5MDExAQ2NjZafZ2dnZGTk6OHKlumzz//HMeOHUNSUlKtbRzjpnHx4kVs2LAB4eHhWLhwIZKSkvCvf/0LJiYmCA0Nlcayrr8/OM4NM3/+fBQWFsLb2xtGRkaoqqrC8uXLMWnSJADgGDexhoxnTk4OnJyctLYbGxvDzs6OY04PtYc2LFPzCgsLw6lTp3D48GF9l9KqXL58GS+//DJiYmKgUqn0XU6rpdFo0Lt3b6xYsQIA4Ofnh1OnTiEqKgqhoaF6rq51+PLLL7Ft2zZs374djzzyCFJTUzF79my0adOGY0xEBuWhnYbh4OAAIyOjWqsE5ObmwsXFRU9VtQ6zZs3Cnj17cPDgQbRr105qd3FxQUVFBfLz87X6c8wbLiUlBdeuXUPPnj1hbGwMY2NjxMXFYe3atTA2NoazszPHuAm4urqia9euWm0+Pj7Izs4GAGks+ffHvZs3bx7mz5+PCRMm4NFHH8Wzzz6LOXPmIDIyEgDHuKk1ZDxdXFxq3eB++/Zt5OXlcczpofbQhmUTExP06tULsbGxUptGo0FsbCwCAgL0WFnLJYTArFmzsHPnThw4cAAeHh5a23v16gWFQqE15mfPnkV2djbHvIEGDRqEkydPIjU1VXr17t0bkyZNkn7mGN+/wMDAWssenjt3Du3btwcAeHh4wMXFRWucCwsLkZiYyHFuoNLSUsjl2l9BRkZG0Gg0ADjGTa0h4xkQEID8/HykpKRIfQ4cOACNRgN/f/8HXjORwdD3HYb69PnnnwulUim2bt0qTp8+LaZPny5sbGxETk6OvktrkWbOnCmsra3FoUOHxNWrV6VXaWmp1GfGjBnC3d1dHDhwQCQnJ4uAgAAREBCgx6pbvpqrYQjBMW4KR48eFcbGxmL58uUiIyNDbNu2TZiZmYlPP/1U6rNy5UphY2Mjdu3aJU6cOCFGjhwpPDw8RFlZmR4rbzlCQ0NF27ZtxZ49e0RmZqb45ptvhIODg3j11VelPhzjxikqKhLHjx8Xx48fFwDEe++9J44fPy6ysrKEEA0bz6FDhwo/Pz+RmJgoDh8+LDw9PcXEiRP1dUpEBuGhDstCCPH+++8Ld3d3YWJiIvr27SuOHDmi75JaLAB1vrZs2SL1KSsrEy+++KKwtbUVZmZm4h//+Ie4evWq/opuBf4aljnGTWP37t2iW7duQqlUCm9vb7Fp0yat7RqNRkRERAhnZ2ehVCrFoEGDxNmzZ/VUbctTWFgoXn75ZeHu7i5UKpXo2LGjeP3114VarZb6cIwb5+DBg3X+HRwaGiqEaNh43rx5U0ycOFFYWFgIKysrMWXKFFFUVKSHsyEyHDIhajwuiYiIiIiIJA/tnGUiIiIiorthWCYiIiIi0oFhmYiIiIhIB4ZlIiIiIiIdGJaJiIiIiHRgWCYiIiIi0oFhmYiIiIhIB4ZlImpRDh06BJlMhvz8fH2XQkREDwGGZSIiIiIiHRiWiYiIiIh0YFgmokbRaDSIjIyEh4cHTE1N4evri6+++grA/6ZI7N27F927d4dKpUK/fv1w6tQprX18/fXXeOSRR6BUKtGhQwe8++67WtvVajVee+01uLm5QalUonPnzoiOjtbqk5KSgt69e8PMzAz9+/fH2bNnm/fEiYjoocSwTESNEhkZiY8//hhRUVFIS0vDnDlz8MwzzyAuLk7qM2/ePLz77rtISkqCo6MjRowYgcrKSgDVIXfcuHGYMGECTp48icWLFyMiIgJbt26VPv/cc8/hs88+w9q1a5Geno6NGzfCwsJCq47XX38d7777LpKTk2FsbIznn3/+gZw/ERE9XGRCCKHvIoioZVCr1bCzs8NPP/2EgIAAqX3atGkoLS3F9OnTMXDgQHz++ecYP348ACAvLw/t2rXD1q1bMW7cOEyaNAnXr1/H/v37pc+/+uqr2Lt3L9LS0nDu3Dl4eXkhJiYGwcHBtWo4dOgQBg4ciJ9++gmDBg0CAHz//fcYPnw4ysrKoFKpmnkUiIjoYcIry0TUYOfPn0dpaSkGDx4MCwsL6fXxxx/jwoULUr+aQdrOzg5eXl5IT08HAKSnpyMwMFBrv4GBgcjIyEBVVRVSU1NhZGSEAQMG1FtL9+7dpZ9dXV0BANeuXbvvcyQiIqrJWN8FEFHLUVxcDADYu3cv2rZtq7VNqVRqBeZ7ZWpq2qB+CoVC+lkmkwGonk9NRETUlHhlmYgarGvXrlAqlcjOzkbnzp21Xm5ublK/I0eOSD/funUL586dg4+PDwDAx8cHv/zyi9Z+f/nlF3Tp0gVGRkZ49NFHodFotOZAExER6QuvLBNRg1laWuKVV17BnDlzoNFo8Nhjj6GgoAC//PILrKys0L59ewDAm2++CXt7ezg7O+P111+Hg4MDRo0aBQCYO3cu+vTpg6VLl2L8+PFISEjABx98gPXr1wMAOnTogNDQUDz//PNYu3YtfH19kZWVhWvXrmHcuHH6OnUiInpIMSwTUaMsXboUjo6OiIyMxMWLF2FjY4OePXti4cKF0jSIlStX4uWXX0ZGRgZ69OiB3bt3w8TEBADQs2dPfPnll3jjjTewdOlSuLq64s0338TkyZOlY2zYsAELFy7Eiy++iJs3b8Ld3R0LFy7Ux+kSEdFDjqthEFGTubNSxa1bt2BjY6PvcoiIiO4b5ywTEREREenAsExEREREpAOnYRARERER6cAry0REREREOjAsExERERHpwLBMRERERKQDwzIRERERkQ4My0REREREOjAsExERERHpwLBMRERERKQDwzIRERERkQ4My0REREREOvw/1veEVSQvxd8AAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "fig, ax = plt.subplots()\n", + "\n", + "for idx in top_indices:\n", + " _ = np.round(architectures_TM[f'ResSCNN{idx+1}']['epochs_acc'][-1], 2)\n", + " ax.plot(np.arange(len(architectures_TM[f'ResSCNN{idx+1}']['epochs_x'])), architectures_TM[f'ResSCNN{idx+1}']['epochs_acc'], label=f'ResSCNN{idx+1} ({_}%)')\n", + "\n", + "ax.set_ylim(0, 100)\n", + "ax.set_yticks(np.arange(0, 110, 10))\n", + "ax.set_ylabel('validation acc [%]')\n", + "ax.set_xlim(0, 100)\n", + "ax.set_xlabel('epoch')\n", + "plt.title(f'top {top_n}')\n", + "\n", + "pos = ax.get_position()\n", + "ax.set_position([pos.x0, pos.y0, pos.width * 0.9, pos.height])\n", + "ax.legend(loc='center right', bbox_to_anchor=(1.45, 0.5), framealpha=0)\n", + "ax.grid(axis='y')\n", + "\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsMAAAG2CAYAAACai4utAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABkJklEQVR4nO3deVhUZf8G8Htm2HfZ98UFXAFFUdQ0d1vcM7NyqbRfpfYq2aJmZrllb1ZumeVbaWpaaqaV+66IiqK4soMLCIgM+zrn98fIKII6AzPMwLk/18V1yTBzzpfDCPc88zzfRyIIggAiIiIiIhGS6rsAIiIiIiJ9YRgmIiIiItFiGCYiIiIi0WIYJiIiIiLRYhgmIiIiItFiGCYiIiIi0WIYJiIiIiLRYhgmIiIiItFiGCYiIiIi0WIYJiIiIiLR0msY/u677xAYGAgbGxvY2NggLCwM//77r+rrxcXFmDRpEhwcHGBlZYURI0bg9u3beqyYiIiIiBoTiSAIgr5OvmPHDshkMrRo0QKCIOCXX37Bl19+iXPnzqFNmzZ4++238ffff+Pnn3+Gra0tJk+eDKlUiuPHj+urZCIiIiJqRPQahmtib2+PL7/8Ei+88AKcnJywYcMGvPDCCwCAq1evolWrVoiIiECXLl30XCkRERERNXRG+i6gUkVFBX7//XcUFBQgLCwMUVFRKCsrQ9++fVX3admyJby9vR8bhktKSlBSUqL6XKFQIDs7Gw4ODpBIJDr/PoiIiPRJEATk5eXB3d0dUimXBhE9id7DcExMDMLCwlBcXAwrKyts27YNrVu3RnR0NExMTGBnZ1fl/i4uLkhPT3/k8RYuXIi5c+fquGoiIiLDdv36dXh6euq7DCKDp/cwHBAQgOjoaMjlcvzxxx8YN24cDh8+XOvjzZgxA+Hh4arP5XI5vL29kZSUBGtra22UTEREZLDy8vLg5+fHv3lEatJ7GDYxMUHz5s0BACEhITh9+jS+/fZbjBo1CqWlpcjJyakyOnz79m24uro+8nimpqYwNTWtdru9vT1sbGy0Xj8REZEhMTY2BgBODSRSk8FNJlIoFCgpKUFISAiMjY2xf/9+1deuXbuG1NRUhIWF6bFCIiIiImos9DoyPGPGDDzzzDPw9vZGXl4eNmzYgEOHDmH37t2wtbXFG2+8gfDwcNWo7pQpUxAWFsZOEkRERESkFXoNwxkZGRg7dizS0tJga2uLwMBA7N69G/369QMAfP3115BKpRgxYgRKSkowYMAArFy5Up8lExEREVEjYnB9hrUtNzcXtra2kMvlnDNMRESNHv/uEWnG4OYMExERERHVF4ZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLb2G4YULF6JTp06wtraGs7Mzhg4dimvXrlW5T3FxMSZNmgQHBwdYWVlhxIgRuH37tp4qJiIiIqLGRK9h+PDhw5g0aRJOnjyJvXv3oqysDP3790dBQYHqPtOmTcOOHTvw+++/4/Dhw7h16xaGDx+ux6qJiIiIqLGQCIIg6LuISpmZmXB2dsbhw4fRo0cPyOVyODk5YcOGDXjhhRcAAFevXkWrVq0QERGBLl26PPGYubm5sLW1hVwuh42Nja6/BSIiIr3i3z0izRjpu4AHyeVyAIC9vT0AICoqCmVlZejbt6/qPi1btoS3t/cjw3BJSQlKSkpUn+fm5gIAysrKUFZWpsvyiYiI9I5/64g0YzBhWKFQYOrUqejWrRvatm0LAEhPT4eJiQns7Oyq3NfFxQXp6ek1HmfhwoWYO3dutdv37NkDCwsLrddNRERkSAoLC/VdAlGDolYY/uuvvzQ+cL9+/WBubq72/SdNmoSLFy/i2LFjGp/rQTNmzEB4eLjq89zcXHh5eaF///58u4iIiBq9yndEiUg9aoXhoUOHanRQiUSCuLg4NG3aVK37T548GTt37sSRI0fg6emput3V1RWlpaXIycmpMjp8+/ZtuLq61ngsU1NTmJqaVrvd2NgYxsbGGn0fREREDQ3/1hFpRu1uEunp6VAoFGp9qDsdQRAETJ48Gdu2bcOBAwfg5+dX5eshISEwNjbG/v37Vbddu3YNqampCAsLU7d0IiIiIqIaqTUyPG7cOI2mPLz66qtqTUmYNGkSNmzYgO3bt8Pa2lo1D9jW1hbm5uawtbXFG2+8gfDwcNjb28PGxgZTpkxBWFiYWp0kiIiIiIgeR6+t1SQSSY23//TTTxg/fjwA5aYb7733HjZu3IiSkhIMGDAAK1eufOQ0iYexxQwREYkJ/+4RaaZOYbisrAyxsbGoqKhAQEBAjXN19Y2/FIiISEz4d49IM7Xege7o0aPw9fVFr1698PTTT8PLywu7du3SZm1ERERERDqldhhWKBRVPp86dSrWr1+PjIwMZGdnY968eXj77be1XiARERERka6oHYY7d+6Ms2fPqj4vLS2Ft7e36nNvb28UFxdrtzoiIiIiIh1Sewe65cuXY8KECejZsyfmzZuHOXPmICQkBAEBASgrK8PVq1exbNkyXdZKRERERKRVaofhzp074/Tp01i8eDFCQkKwePFiXLt2DZGRkaioqECnTp3g4eGhy1qJiIiIiLSqVt0kEhIS8NZbb8HGxgbLli2Du7u7LmrTCq6qJSIiMeHfPSLNaNRN4tKlS9iyZQsqKiqwd+9eDB48GE899RRWrlypq/qIiEgH0uRFOJGQhTR5kb5LISLSK7XD8JIlS9CpUyd8+eWXCAsLww8//IBx48YhMjISJ0+eRFhYGGJiYnRZKxERacGm06notugAXv4hEt0WHcCm06n6LomISG/Unibh6uqKjRs3olevXkhJScHAgQNx5coV1df37t2Ld999t8pthoBvFxER3ZcmL0LXRQfw4G9+mUSCYx/1gputuf4KI63h3z0izag9MiwIAqRS5d1lMhkeztD9+vXDuXPntFsdERFpjSAIWH04EQ8PgVQIApKzCvVTFBGRnqndTeL999/Hs88+i6CgIMTGxmLBggXV7mNmZqbV4oiISDuKyyow+8+L+D3qRrWvySQS+Dpa6KEqIiL9UzsMT58+HQMGDMDVq1fRrl07tGzZUpd1ERGRltzKKcJbv0bhwg05pBJgYBtX7LqUDsW9EeL3B/hzigQRiZbaYRgA2rVrh3bt2umqFiIi0rKIhDuYvOEs7hSUws7CGMtHd0D3Fo5Ikxdh3P9OIfZ2PmzMTfRdps6lyYuQlFUAP0dLBn8iqkKtOcPh4eEoKChQ+6AzZsxAdnZ2rYsiIqK6EQQB/zuWhFfXROJOQSlau9lgx+Tu6N7CEQDgZmuO5wOVPeKPxGbqs1Sd23Q6FV3ZPYOIHkGtMPztt9+isFD9xRUrVqxATk5ObWsiIqI6KCqtQPjm8/hs52VUKAQMDXbHlre7wsu+6rzgnv5OAIDj8Vkoq1Doo1SdS5MX4aOtMapFgwoBmLn1IvsrE5GKWtMkBEGAv78/JBKJWgfVZBSZiIi053p2Id76NQqXbuVCJpVg5rOt8Ho33xp/f7f1sEUTC2PcLSxD9PUcdPK110PFupWUVVBj94zEzAJOlyAiAGqG4Z9++knjA7u4uGj8GCIiqr1jcVmYsvEs7haWwd7SBCte7oCwZg6PvL9MKsFTLZzw1/lbOHwts1GGYWNZzYM4S/fHoY27DewsGv98aSJ6PLXC8Lhx43RdBxER1ZIgCPjhaCIW/XsVCgFo52GLVWNC4GH35JHPHv7KMHwkLhPTBwTUQ7X1a8+l21U+l0iULwIik7IxePlxfD8mBK3cuDEFkZhp1E2CiIgMR5q8CFfTcvHryVTsv5oBAHghxBPzhraFmbFMrWP0uLegLuamHHfyS+BgZaqzeutbUWkFNp9R9lX+8oVAeDaxgK+jBe4WlOH/fj2D1OxCDF95AotfCMSgIHc9V0tE+sIwTETUAG06nYoZW2NUvYKlEmDu4DZ4tYuP2us7AMDZxgyt3GxwJS0Xx+KzMCTYQ0cV178d529BXlQGzybmGN7BEzKp8rq42Zpjx+TumLLxHI7GZWHKxnOIuSnHBwMCYCRTe2NWImok+L+eiKiBSZMXVQnClfq2dtEoCFeq7CpxuBG1WBMEAb9EJAMAXu3iowrClewsTPDza6F4++lmAIDVRxIx7qdTyC4ore9SiUjPGIaJiBqYpKyCakFYIQDJWeq3wHxQD3/lVIkjsVlQPHzgBupsag4u3cqFqZEUozp61XgfmVSCDwe2xIqXO8DCRIbj8XcwaNkxXLwpr+dqiUifNA7Dcrm8xg01srOzkZubq5WiiIjo0SxqmA8sk0jg62hRw72frKOPPSxMZMjKL8GV9Mbxe3zdvVHhQUHuaGL5+I4RzwW6Yds73eDjYIGbOUUY8d0JbD17ox6qJCJDoHEYfumll/Dbb79Vu33z5s146aWXtFIUERHVTBAELD+YUOU2mUSCBcPb1rpvromRFF3vtWBrDFMlsvJL8E9MOgBgbJiPWo8JcLXGX5O6o1eAE0rKFQjffB5zd1xCWYUCafIinEjI4kYdRI2UxgvoIiMjsWTJkmq3P/3005g1a5ZWiiIioprtvpSOfVduw1gmwc+vhUJ6b0S4rhtI9PR3wr4rGTgSm4l3nm6upWr1Y9Pp6yitUCDIyw6BnnZqP87WwhhrxnXCN/tisfRAPH46noxD1zKRckc5LUUqARYOb4dRnbx1VzwR1TuNR4ZLSkpQXl5e7faysjIUFfFVMxGRruQWl2HOX5cAAP/Xoxm6NXdEWDMHreyk1uPeIrozyXeRX1L9d3xDUV6hwK8nUwAAY7uoNyr8IKlUgvD+Afh+TAgsTGRV5mdzK2eixknjMBwaGorVq1dXu33VqlUICQnRSlFERFTdf3dfw+3cEvg6WGByb+2O3vo4WMLHwQLlCgERCXe0euz6tO9KBtLkxbC3NMFzgW61Ps6ANq74bHCbardXCEKtFyoSkWHSeJrEvHnz0LdvX5w/fx59+vQBAOzfvx+nT5/Gnj17tF4gEREB51LvYt29Ec/5w9qpvamGJnr6O2FtRAoOx2agX2sXrR+/Pqw7mQwAGNXJq87XqFsLR0glqNK5oy4LFYnIMGk8MtytWzdERETAy8sLmzdvxo4dO9C8eXNcuHABTz31lC5qJCIStbIKBWZsjYEgAMPbe6Bbc0ednKdHi/v9hgWh4bVYi8/Ix/H4O5BKgFc6131er5utORYOb4fKDsUSoE4LFYnIMNVqB7rg4GCsX79e27UQEVEN1hxLwtX0PNhZGGPWc610dp6wZg4wlklwPbsIyXcK4edoqbNz6ULlXOHeLV3g2UQ7o7ejOnnDwcoUE345AyOpBH1bNcwRcyJ6NI1Hhv/55x/s3r272u27d+/Gv//+q5WiiIhI6Xp2Ib7ZFwsAmPlsKzhYmersXJamRujoYw8AONLAWqzll5RjS5SyN7C67dTU1beVCwI9bVGmEPDb6etaPTYR6Z/GYfijjz5CRUVFtdsFQcBHH32klaKIiEj5e3XWnxdRXKZAl6b2GBniqfNz9gxQTpVoaGF427mbyCsph5+jJbrrYBrJmHudKTZEpqKikezSR0RKGofhuLg4tG7dutrtLVu2RHx8vFaKIiIiYMeFNByJzYSJTIr5w9pBIpE8+UF1VDlv+ETCHZSUVx/4MESCIKh2nBvTxQdSqfav06AgdzSxMMbNnCLsv3Jb68cnIv3ROAzb2toiMTGx2u3x8fGwtGxY88uIiAyVvLAMn+1Q9hSe1Ks5mjlZ1ct5W7lZw8naFEVlFYhKvlsv56yryKRsxN7Oh7mxDCN0NHpuZizDi528AEDV1YOIGgeNw/CQIUMwdepUJCTc3w40Pj4e7733HgYPHqzV4oiIxGrRrivIyi9FMydLvPV003o7r0QiqdJVoiFYF6EMp0Pbe8DW3Fhn53m1sw8kEuBoXBYSMvN1dh4iql8ah+HFixfD0tISLVu2hJ+fH/z8/NCqVSs4ODjgv//9ry5qJCISlVNJ2dh4SrlQa8GwdjA10n5P4cepnDesizCcJi/CiYQsre3idju3GLsvpQPQ/sK5h3nZW6B3gDOA+wGciBo+jVur2dra4sSJE9i7dy/Onz8Pc3NzBAYGokePHrqoj4hIVErKKzBzWwwAYFRHL3Ru6lDvNTzV3BESCXA1PQ+3c4vhYmOmleNuOp2KGVtjoBAAqQRYOLwdRnWqWz/gDZGpKFcI6OTbBK3cbLRS5+OMCfPB/qsZ2BJ1A+8PCIClaa06lBKRAanV/2KJRIL+/fujf//+2q6HiEjUVh9ORHxGPhytTDDj2ZZ6qaGJpQkCPWxx/oYcR2IzMbKjV52PmSYvUgVhQLmr28ytF9HD36nWm1iUliuw4VQqAGBsmG+da1RHjxZO8HWwQPKdQvwZfROvdNbtaDQR6V6twnBBQQEOHz6M1NRUlJaWVvnau+++q5XCiIjEJjEzH8sOKrvyzH6+NewsTPRWS09/J5y/IcdhLYXhpKwCPNyRrEIQkJxVWOswvPtSOjLzSuBkbYoBbVzrXKM6pFIJXu3ig3l/X8G6iBS8HOpdL10+iEh3NA7D586dw7PPPovCwkIUFBTA3t4eWVlZsLCwgLOzM8MwEVEtCIKAWdsuorRcgadaOGJwkLte6+nh74SlB+JxLD4LFQoBsjq2K3N6xGYh2QUltT5m5bzd0aHeMDHSeAlMrY0M8cJ/91zD1fQ8nE6+i1A/+3o7NxFpn8a/PaZNm4ZBgwbh7t27MDc3x8mTJ5GSkoKQkBAuoCMiqqUtZ28iIvEOTI2kmDe0rd5HG4O97GBtZoScwjJcuJFT5+Otj0yt8fZ3f4vGT8eTIAiabWRxNT0Xp5KzIZNK8HJo3eYda8rWwhhDgz0AAGvv9TcmooZL4zAcHR2N9957D1KpFDKZDCUlJfDy8sLixYsxc+ZMXdRIRNSoXUnLxad/XQQA/KdvC/g46L9nu5FMqtrJ7UhsVp2Odf56Dn65Fxq/fSkYGyd2wcH3nsbQYHdUKATM3XEZ720+j+Iy9Tf5WHtvVHhAGxe42mpngZ8mxtzrXLHrYjoycou1fnxtd90gokfTOAwbGxtDKlU+zNnZGampylf7tra2uH6de7YTEWli0+lUPPPtUeSXKIOgnQ775Gqqp/+9rZnjat9irbxCgRlbYyAIwLD2HhgS7IGwZg7wc7LE16OCMfv51pBJJdh67iZGfHcCN+4WPvGY8qIybDt7EwAwpotvrWurizbutgjxaYJyhaBaxKctm06notuiA3j5h0h0W3QAm05r9/hEVJXGYbh9+/Y4ffo0AKBnz5745JNPsH79ekydOhVt27bVeoFERI3VxZtyfLQlpspts/+8ZDCjgT3uheFzqXchLyyr1TH+dzwJl9NyYWdhjFnPtaryNYlEgje6+2HdG6GwtzTBpVu5GLTsGE7EP34kekvUDRSVVcDfxQpdmupvvm5lX+MNkakoq1Bo5ZiP6rphKM8JosZI4zC8YMECuLm5AQDmz5+PJk2a4O2330ZmZiZWr16t9QKJiBoThULA0bhMvLM+CoOXH8PDM2UrOywYAnc7c7RwtoJCAI4naD5V4np2Ib7eGwcAmPlMKzg+YhFd12aO2DGlO9p52OJuYRleXROJH44k1jiPWKEQ8Ou97ZDHhPnqdW71M23d4Ghlioy8Euy5dFsrx3xc1w0i0g2Nu0l07NhR9W9nZ2fs2rVLqwURETVGGbnF+D3qBn47nYrr2Y8e5ZNJJPB1tKjHyh6vh78T4jLycfhaJp5t56b24wRBwOztF1FUVoHOfvYY2dHzsff3sDPH72+FYda2i9hy9gbm/3MFF27K8cWIdrAwuf+n6nhCFhKzCmBlaoRh7T1q/X1pg4mRFKNDvbDsQDzWRiTjuUD1r8+jWJtV/7NsaM8Josam/nrREBGJTIVCwKFrGfi/dWcQtugAvtx9Ddezi2BtZoRxYT749z9P4YsR7SC7N7opk0iwYHjbWvfd1YUH5w1r0vHh75g0HLqWCROZFAuGt1NrBNfMWIb/jgzEZ0PawEgqwY7ztzB85Qmk3rk/Klq5cG5EBw9YGcDuby939oZMKkFkUjaupufW6VjFZRWY/eelKrcZ4nOCqLHR/28SIqJGIE1ehKSsAvg5WkIqkWDz6ev47fR13My5Pwoc4tMEo0O98Vw7N5ibyAAArdxs0MPfCclZhfB1tDC40BPqZw9TIynS5MWIy8iHv4v1Ex8jLyrD3B2XAQDv9GqGZk5Wap9PIpFgbJgvWrra4J31UbianodBy49h2ej2sDKVYd9l5XSEym4O+uZma45+rVyw61I61kWkYP6wdrU6jiAI+OCPC4i+ngNbc2P8MDYEFQoY5HOCqLFhGCYiqqNNp1OrLHqSSIDKQVQbMyOMCPHE6FDvRwZJN1tzgw08ZsYydGnqgMOxmTgSm6lWGP5i11Vk5pWgqZMl3n66Wa3OG+pnjx1TuuOtX8/i/PUcjP3fqSpfj0q5i+bOT66lPozt6oNdl9Kx7dxNfPhMS9iYad4RZNmBePx1/haMpBKsejUEoX4OOqiUiGrCaRJERHXw8Op/QBmEgz3t8PWoIJya1RdzBrVRK0QaqsquEodjn9xi7UxyNjbc22BjwbB2MDWS1fq8brbm2Px/XTA4qPpcXEPqsBDW1AEtnK1QWFqBrVE3NH783xfSsGRvLABg3tC2CGvGIEy1V1paiubNm+PEiRP6LkVndu3aheDgYCgU2uniwjBMRFQHNa3+B4APn2mJYe09YWZc+zBoKHr6KzffiEzKRlHpozfGKC1X9hQGgBc7eqJL07qHOlMjGV6qYYc5Q+qwIJFIVNM21p1M0Whu9YUbOXjv92gAwBvd/Wr8XsVq/PjxkEgkkEgkMDY2hp+fHz744AMUF2tvk5PDhw+jd+/esLe3h4WFBVq0aIFx48ahtLRUdR9BELB69Wp07twZVlZWsLOzQ8eOHfHNN9+gsFD5HPz0008hkUjw1ltvVTl+dHQ0JBIJkpOTAQDJycmQSCRwdnZGXl5elfsGBwfj008/VX2+detW9O/fHw4ODpBIJIiOjlbre1q1ahX8/PzQtWtX1W3z589H165dYWFhATs7uxofV3mtH/z47bffHnuu2NhYDBkyBI6OjrCxsUH37t1x8OBB1dd//vnnGo8rkUiQkZEBADh37hzat28PKysrDBo0CNnZ2arHl5eXIyQkBKdOVX1naODAgTA2Nsb69evVuiZPUqswvH//fsycORMTJkzA66+/XuWDiEhMfB2qr/JvbKv/mzlZwcPOHKXlCpxMuvPI+60+koC4jHw4WJpg5rOtHnk/TSnnYVe9zdCu8bD2HrA0kSEhswAnEh59jR6ULi/GhF/OoLhMgV4BTlq9ZrpS3zvjDRw4EGlpaUhMTMTXX3+N77//HnPmzNHKsS9fvoyBAweiY8eOOHLkCGJiYrBs2TKYmJigouL+i74xY8Zg6tSpGDJkCA4ePIjo6GjMnj0b27dvx549e1T3MzMzw5o1axAXF/fEc+fl5eG///3vY+9TUFCA7t2744svvlD7exIEAcuXL8cbb7xR5fbS0lKMHDkSb7/99mMf/9NPPyEtLU31MXTo0Mfe//nnn0d5eTkOHDiAqKgoBAUF4fnnn0d6ejoAYNSoUVWOl5aWhgEDBqBnz55wdnYGAEyYMAG9e/fG2bNnIZfLsWDBAtXxv/rqK3Tr1g2hoaHVzj1+/HgsXbpUncvyRBrPGZ47dy4+++wzdOzYEW5ubnrt8UhEpG938qtuRtEYV/9LJBL08HfExlPXcSQ2E70CnKvdJzmrAEsPxAMAZj/fGnYWJlo7v5utORYOb4eZWy+iQhAM8hpbmxljeAdPrDuZgl9OJKPbva2sH6WotAIT1p5GRl4J/F2ssHR0e8geTvw6IggCijTY+rrSlqgbmPPXJSgEQCoB5g5ugxEhj2+Z9zBzY5lGucHU1BSurq4AAC8vL/Tt2xd79+5VBUSFQoEvvvgCq1evRnp6Ovz9/TF79my88MILAIC7d+9i8uTJ2LNnD/Lz8+Hp6YmZM2fitddew549e+Dq6orFixerztesWTMMHDhQ9fnmzZuxfv16/PnnnxgyZIjqdl9fXwwePBi5ufc7iAQEBMDZ2RmzZs3C5s2bH/t9TZkyBUuWLMGkSZNUofBhY8aMAQDVqLI6oqKikJCQgOeee67K7XPnzgWgHKl9HDs7O9X1fpKsrCzExcVhzZo1CAwMBAAsWrQIK1euxMWLF+Hq6gpzc3OYm9//f5qZmYkDBw5gzZo1qtuuXLmC9evXw9/fH6NHj8bOnTsBAImJiVizZg2ioqJqPP+gQYMwefJkJCQkoFmz2q1NqKRxGF61ahV+/vln1Q+JiHTvwU4FhhQACNhzWTkC8nSAE/6vR7NGu/q/p7+TKgw/TBAEzPozBqXlCjzVwhFDgt21fv5RnbwNuusGoNyRbt3JFOy7chs3c4rgYVdzjQqFgPDN0bh4Mxf2liZYM64TrGux6K62isoq0PqT3XU6hkIAZm+/hNnbLz35zg+4/NmAKn2jNXHx4kWcOHECPj73O4ksXLgQv/76K1atWoUWLVrgyJEjePXVV+Hk5ISePXti9uzZuHz5Mv799184OjoiPj4eRUXKUW1XV1ekpaXhyJEj6NGjR43nXL9+PQICAqoE4UoSiQS2trZVblu0aBE6deqEM2fOVNmX4WGjR4/G3r178dlnn2H58uW1uRw1Onr0KPz9/WFtXbs1CpMmTcKECRPQtGlTvPXWW3jttdce+eLFwcEBAQEBWLt2LTp06ABTU1N8//33cHZ2RkhISI2PWbt2LSwsLFQvVgAgKCgIe/fuRfPmzbF//35VsH7rrbewePHiR34v3t7ecHFxwdGjR+s/DJeWllaZh0JEuvVgpwKpBFg4vB1GdeK8QkOx+5IyDA8Jdm/UC5+6NneETCpBQmYBbtwthGeT+1MUtp27iePxd2BqJMW8oW119o6hIXfdAIAWLtYIa+qAiMQ72BCZgvcHtKzxfl/vi8W/F9NhIpPi+zEh8LI3nOkehmbnzp2wsrJCeXk5SkpKIJVKVeGxpKQECxYswL59+xAWFgYAaNq0KY4dO4bvv/8ePXv2RGpqKtq3b68Kpr6+vqpjjxw5Ert370bPnj3h6uqKLl26oE+fPhg7dixsbGwAAHFxcQgICFC73g4dOuDFF1/Ehx9+iP379z/yfhKJBIsWLcKgQYMwbdq0Ooe5SikpKXB3r92L0c8++wy9e/eGhYUF9uzZg3feeQf5+fl49913a7y/RCLBvn37MHToUFhbW0Mqlao2Y2vSpEmNj1mzZg1efvnlKqPFP/74I9555x3897//Rbdu3TBjxgysW7cOFhYW6NSpEwYMGICEhAS89NJLmDdvXpXjubu7IyUlpVbf74M0DsMTJkzAhg0bMHv27DqfnIge7+FOBQpBuYq+h7+TQYcCsUjKKkDs7XwYSSXoHeCi73J0ysbMGB287XA6+S6OxGbh5c7KF2TZBaWY9/cVAMC7fVrAx8FSn2Xq3dgwH0Qk3sFvp67j3T4tqnXT2B59E8vuTSdZMLwdOvna13uN5sYyXP5sgEaPSZcXo++Sw1UWi0olwL7wnnC1NdPo3Jro1asXvvvuOxQUFODrr7+GkZERRowYAQCIj49HYWEh+vXrV+UxpaWlaN++PQDg7bffxogRI3D27Fn0798fQ4cOVQ3oyWQy/PTTT5g3bx4OHDiAyMhILFiwAF988QVOnToFNzc3jRZDVpo3bx5atWqFPXv2PHIKBAAMGDAA3bt3x+zZs7FhwwaNz1OToqIimJmp//N40IO5rn379igoKMCXX375yDAsCIJqmsfRo0dhbm6OH3/8EYMGDcLp06fh5la1C0xERASuXLmCdevWVbm9TZs2OHz4sOrzO3fuYM6cOThy5AimTJmCrl27YuvWrejUqRM6d+6MQYMGqe5rbm6uWsRYFxovoCsuLsaSJUvQs2dPTJkyBeHh4VU+iEh7aupUYEir6MVuz71R4S5NHWBrUX9vc+tLjxaVLdYyVLct+OcKsgtKEeBijTd7NNVXaQajX2sXuNqY4U5BKf6NSa/ytaiUu3j/jwsAgLd6NsMLGs631RaJRAILEyONPpo6WWHh8Kq7JS4c3g5Nnaw0Oo6m7xpYWlqiefPmCAoKwv/+9z9ERkaq5pvm5+cDAP7++29ER0erPi5fvow//vgDAPDMM88gJSUF06ZNw61bt9CnTx9Mnz69yjk8PDwwZswYLF++HJcuXUJxcTFWrVoFAPD398fVq1c1qrlZs2aYOHEiPvrooyeG6UWLFmHTpk04d+6cRud4FEdHR9y9e1crx+rcuTNu3LiBkpKSGr9+4MAB7Ny5E7/99hu6deuGDh06YOXKlTA3N8cvv/xS7f4//vgjgoODHzmFolJ4eDimTp0KT09PHDp0CCNHjoSlpSWee+45HDp0qMp9s7Oz4eTkVOvvsZLGYfjChQsIDg6GVCrFxYsXce7cOdWHum0/iEg9fo6WePhvh1QCg1pFL2Z77u2G1r9N4x4VrtQzQPlH50T8HZRVKBCRcAd/RN2ARKIc5TSWsVunkUyqGjX/JSJZdfuNu4X4v3VnUFquQP/WLvhggPpvvRuKUZ28ceyjXtg4sQuOfdSr3qdrSaVSzJw5Ex9//DGKiorQunVrmJqaIjU1Fc2bN6/y4eXlpXqck5MTxo0bh19//RXffPMNVq9e/chzNGnSBG5ubigoKAAAvPzyy4iNjcX27dur3VcQBMjl8hqP88knnyA2NvaJrclCQ0MxfPhwfPTRR+pcgidq3749rl69WqsR7YdFR0ejSZMmMDU1rfHrlSOyUmnV//dSqbRa/9/8/Hxs3ry5WpeLh+3fvx9XrlzB5MmTAQAVFRUoK1MuUi4rK6vS5aO4uBgJCQmqdwHqQuNpEg/2jyMi3XKzNUcbNxtcvHV/xXInX3tOkTAAGXnFOJuqHIHp11ocYbituy3sLU2QXVCKyMRsfLL9IgDglc7eCPGpeY6gGL0U6oVlB+JwLjUHF2/K4edoiQm/nEFWfilaudng61HBkNZT5wht0/e87ZEjR+L999/HihUrMH36dEyfPh3Tpk2DQqFA9+7dIZfLcfz4cdjY2GDcuHH45JNPEBISgjZt2qCkpAQ7d+5Eq1bKFnbff/89oqOjMWzYMDRr1gzFxcVYu3YtLl26hGXLlgEAXnzxRWzbtg2jR4/Gxx9/jP79+8PJyQkxMTH4+uuvMWXKlBrbj7m4uCA8PBxffvnlE7+n+fPno02bNjAyqhrJsrOzkZqailu3bgEArl27BkC58O9RHR969eqF/Px8XLp0CW3btlXdnpqaqjpeRUWFavCyefPmsLKywo4dO3D79m106dIFZmZm2Lt3LxYsWFBlFP3UqVMYO3Ys9u/fDw8PD4SFhaFJkyaq62xubo4ffvgBSUlJ1bpZbNq0CeXl5Xj11VcfeR2Ki4sxefJkbNy4URWwu3XrhhUrVmDSpEnYsmULlixZorr/yZMnYWpqqpovXhd1ehl/48YN3Lih+W47RKSeNHkRrqQrG7OP7+oLAIi5KUducdljHkX1Yd/lDAgCEORpK5oXJ1KpBE+1ULYMm7zhLBKzCuBkbfrIhWJi5WxthmfaKudLrjwUjzFrTuFqeh4crUzx47iOsDStXTcFAoyMjDB58mQsXrwYBQUF+PzzzzF79mwsXLgQrVq1wsCBA/H333/Dz88PAGBiYoIZM2YgMDAQPXr0gEwmU43WhoaGIj8/H2+99RbatGmDnj174uTJk/jzzz/Rs2dPAMopJRs2bMCSJUtUtwcGBuLTTz/FkCFDMGDAo+deT58+HVZWVk/8nvz9/fH6669X20zkr7/+Qvv27VXB8qWXXkL79u1VUzhq4uDggGHDhlXbjOKTTz5B+/btMWfOHOTn56N9+/Zo3749zpw5AwAwNjbGihUrEBYWhuDgYHz//fdYsmRJlZ7OhYWFuHbtmmqk1tHREbt27UJ+fj569+6Njh074tixY9i+fTuCgoKqnH/NmjUYPnz4Izf8AJTt35577jkEBwerblu6dCmio6PRo0cPDBo0SDVfHAA2btyIV155BRYWdX+nVCJoOJauUCgwb948fPXVV6r5OtbW1njvvfcwa9asasPl+pabmwtbW1vI5XLV6lCihuKrPdew7EA8OvvZ47c3u2DAN0cQezsfcwa1xmvd/PRdnqiN/+kUDl3LxPsDAjCpV3N9l1NvPvjjPDafuT8I8kpnb8wf1k6PFRmmM8nZeGFVRJXbJj3dDO8P1P0LB/7dE7cLFy6gX79+SEhIUCuMN0RZWVkICAjAmTNnVC986kLj5Dpr1iwsX74cixYtUs0VXrBgAZYtW8YOE0RaVFquwMZT1wEAY8N8lVu+dqndlq+kXXnFZTgRr9xlbIBI5gsDyncq/oiq+m7gb6dS6203sobE3a76iv5VhxN5rUjnAgMD8cUXXyApKUnfpehMcnIyVq5cqZUgDNRizvAvv/yCH3/8EYMHD1bdFhgYCA8PD7zzzjuYP3++VgojErt/L6YhK78ELjamqgVawzp44otd15CYWYDj8XfQvcXjd7ki3Th0LROlFQo0dbREM6fGOfJSk5q7mwDJWYWimSqiruQ71Tu+VHaC4bUiXRs/fry+S9Cpjh07PnZTE01pPDKcnZ2Nli2rv83TsmVLZGdna6UoIgLWRigbiY8O9Vat0rcyNcLwDh4Aqq5Up/p1v4uEq6i2pPdztMTD675kEgm7m9SA14qo4dA4DAcFBdW4deDy5curTZgmotq5dEuOqJS7MJJK8HJo1fZFY8OUUyX239vylepXSXkFDl5V9tkVS0u1Sm625tV6zS4Y3pYjnTXgtSJqODSeJrF48WI899xzVbY/jIiIwPXr1/HPP/9ovUAiMVp3b1R4YFtXONtUnXvY3NkaXZs54ETCHaw/mYIP6mFBTkOSJi9CUlYB/BwtdRI8IhLuIL+kHM7Wpgj2tNP68Q3dqE7e6OHvhOSsQvg6WjDcPQavFVHDoPHIcM+ePREbG4thw4YhJycHOTk5GD58OK5du4annnpKFzUSiYq8sAx/Rt8EoFw4V5PK0eFNp6+jpLyixvuI0abTqei26ABe/iES3RYdwKbTqVo/R+UUiX6tXRpsr9i6crM1R1gzB4Y7NfBaERm+WjU7dHd350I5Ih35Peo6issUaOlqjU6+NW9k0LeVC9xszZAmL8Y/MWkY1l4/27oakjR5ET7aGoPKJhsKAZi59SJ6+DtpLYgoFAL2PjBfmIiIGj61wvCFCxfQtm1bSKVSXLhw4bH3DQwM1EphRGKkUAj49aRyisSYMJ9HLs4ykknxcqg3vtobi19OpDAMAzh/PQcPd5vT9ur9c9dzkJlXAmtTI4Q1ddDKMYmISL/UCsPBwcFIT0+Hs7MzgoODIZFIauxxKpFIquwbTUSaORKXieQ7hbA2NcLQYI/H3velUG8sPRCH6Os5iLkhRztP23qq0vAUl1VgxcGEarfLJNDq6v09l9IBAL1aOsPEyLA2GCIiotpRKwwnJSXByclJ9W8i0o3KhXMvdPR84patTtameLadG7ZH38LaiGR8OVKc3VwEQcDMrTGIuSmHqZEUZRUKVS/cVzr7aG1UWBAE7L4XhsXWRYKIqDFTa2jDx+f+27UpKSnw8PCAj49PlQ8PDw+kpKTotFiiB6XJi3AiIavR7Oh0PbsQB64pW3ZV7jT3JJUL6f46fwt3C0p1Vltt1NfP57vDCdh67iZkUgl+HNcRxz/qjSFB7gCAMyl3tbZTX3xGPpLvFMJEJsXTAc5aOSYREemfxu/z9erVq8bNNeRyOXr16qWVooiepD66BtS3XyNTIAjAUy0c0VTNXc06eDdBazcblJQr8HvUdR1XqL76+vnsupiOxbuuAQA+HdQaT7VQLpb7dHAbWJrIcDktV9X9oa4qR4W7NXeA1RNG7YmIqOHQOAwLglDjop47d+7A0tJSK0URPYpCIeDPczfx4ZYY1VvhlV0DGvIIcXFZBTafVoZZdUeFAeU8/crR4XUnU1Dx8F65epAmL8KMrbr/+Vy8Kce0TdEAgHFhPhjzQBu6JpYmGN9N+fk3++Kg0MJ1qQzVA9hFgoioUVF7eGP48OEAlH98x48fD1NTU9XXKioqcOHCBXTt2lX7FRIByMgtxu9RN7DxVCpu3K0eqrTdNaC+7Th/C3cLy+BhZ44+rTSbjzok2AML/rmC69lFOBybgd4t9TufNSmrAA9nzwpBQEJGvtZ+Phm5xZi49gyKyirwVAtHzH6+dbX7THyqKX45kYIr90aHB7atfYi9lVOECzfkkEig8c+HiIgMm9ph2NZWuVJdEARYW1vD3Pz+HzUTExN06dIFEydO1H6FJFoVCgFH4jKxMTIV+69mqEY9rUxlKCipwIN5S6rlrgH1bd29dmqvdPGGTMONHMxNZHixoxd+PJaEtREpeg/Dfo6WkAB4eCz2632xaO1uC3tLkzodv7isAhPXRSFNXoxmTpZY/nIHGMmqv8llZ2GC17r5YtmBeHyzLxb967BJRmVv4RDvJnCyNn3CvYmIqCFROwz/9NNPAABfX19Mnz6dUyJIZ9Llxdh85jo2nb6Omzn3R4E7+TbB6FDvex0UbmLm1ououLc4ysXGDE5WDTOkRF/PwYUbcpjIpBjV0atWx3i1iw9+PJaEw7GZSLlTAB8H/f3/dLM1h4uNKdJzSwAoX6gYSSWISsnBoGXH8P2YELT1qF0bOEEQMP338zh/PQd2FsZYM64TbM2NH3n/N7r74efjybianoc9l9MxsK1brc6757JyvjCnSBARNT4arwKZM2eOLuogkUqTFyEpqwDe9haIvZ2HDZHXceDqbdXb7LbmxhjRwROjQ73QwsVa9bhRnbzRw98J56/n4P0/LiBNXoyfjidjYo+mevpOam9tRDIA4PlANzjUMtD7Olqip78TDsdm4teTKZj1XPVpA/Ul9U4h0nNLIJUA378agraetsgrLseba88g+U4hRnx3AguHt8PwDppvFPLt/jjsvJAGI6kE370SAl/Hx4f+ytHhpQfi8c2+OPRv7arx6HBOYSlOJioXDfdrzSkSRESNTa2WRP/xxx/YvHkzUlNTUVpatZ3T2bNntVIYNX6bTqdWWWj1oFA/e7wc6o2BbV1hZiyr8fFutuZwszWHvKgMH26JwZK9sRjY1hVe9g1nukR2QSl2XkgDoNxxri7GhvngcGwmNp+5gfB+ATA3qfm66VrlKGpnPwf0uzeS6mYLbJ/cHVN/O4eD1zIRvvk8LtyQY9ZzrWBcwxSHmuw4fwvf7IsDAMwf1hZhzdTbAe6N7k3x073R4V2X0vFsO81Ghw/cm6IT4GL9xPBNREQNj8bdJJYuXYrXXnsNLi4uOHfuHEJDQ+Hg4IDExEQ888wzuqiRGqGHOw5UGt3JC/vCe2Lz/4VhaHuPRwbhB73Y0QuhfvYoKqvAJ9svaq2vbH3YdPo6SssVaOdhi2Avuzod6+kAZ3g2Ub44+Ov8Te0UWAt7LlV2Xag6imprrpzW8G6fFgCAn08k45UfI5GZV/LEY56/noPpv58HAEzo7odRnbzVrsfWwhivdfcDAHxbi84Sj/p+iIiocdA4DK9cuRKrV6/GsmXLYGJigg8++AB79+7Fu+++C7lcrosaqRGqqeMAAAwO9kBzZ/V67FaSSCRYMKwdTGRSHLyWiX9i0rVUpW5VKAT8em/h3JgwnxpbFmpCJpWo2rKtjUjRy4uCrPwSnEm5N6Wghvm1UqkE4f38sXpMCKxMjXAqKRuDlh1D9PWcRx4zTV6EiWvPoKRcgd4tnTHj2VYa1/VGdz9Ymxnh2u08/HtR/edHcVkFDsdmAgD6c74wEVGjpHEYTk1NVbVQMzc3R15eHgBgzJgx2Lhxo3aro0arpt3SZBJJrTtCNHe2wttPNwMAfLrjEuRFZXWqrz4cuJqBmzlFsLMwxuB7O6bV1YsdvWBqJMWlW7k4m5qjlWNqYv8V5Xzvdh628LB7dBu1/m1c8eekbmjqZIn03GK8uCpC1Wf5QYWl5Zjwyxlk5JUgwMUa374UrHG3DUA5Kv16t3ujw/tj1R4dPhqXhaKyCnjYmaONu43G5yUiIsOncRh2dXVV7UDn7e2NkydPAgCSkpIa1NvTpD8l5RX4am8sAKAy1sgkEiwY3rZOfWjffroZmjpaIjOvBIt3XdVCpbpVuXBuVEcvtaaDqKOJpQkG3QvW6+4dvz5VTinor8ZCs+bOVtg+qRv6tXZBaYUCH2y5gI//jEFpuQKAcoOV8E3ncelWLhwsTfDjuI6wNnt054gnef3e6HDs7Xz8czFNrcdU7jrXr7VLnUfuiYjIMGkchnv37o2//voLAPDaa69h2rRp6NevH0aNGoVhw4ZpvUBqfL47lIDEzAI4Wpli99Qe2DixC4591EujeaA1MTOWYf6wdgCA9ZGpiEq5q41ydSIxMx9H47IgkSjbomlT5Y50/8SkqzUfV1vyS8pxND4LgPpTCqzNjPH9qyEI7+cPiQT49WQqRv9wEjE3cjBtczR2XUqHiUyK78eE1HlhpK25MSZ0V3Yb+XZf3BN36yuvUGD/lXvhnvOFiYgaLY3D8OrVqzFr1iwAwKRJk/C///0PrVq1wmeffYbvvvtO6wVS4xKfkY+VBxMAAHMGtYa/qzXCmjlobWeysGYOGBmibNk1c2sMyioUWjmutv16MhUA0CvAWevdLwI97RDkZYfSCgU2nU7V6rEf50hsJkrLFfB1sIC/i/rzvqVSCd7t0wJrxnWEtZkRolLuYtDy49gefQsAMLS9Ozr62mulxte6+8LGzAhxGfn4J+bxo8NnUu7ibmEZ7CyMEaql8xMRkeHROAxLpVIYGd3vyPbSSy9h6dKlmDJlCkxM6razFDVugiBg1rYYlFYo0NPfCc8H1m4DhCeZ+Wwr2Fua4NrtPKw+kqiTc9RFYWk5fo9Szo+tazu1Rxl377jrI1NRXk8vCPbcm1LQv41rraYU9G7pgjXjOla7fUvUTaTJq2/BXRs2ZsaY8NS90eH9jx8drpwi0aelS4073BERUeOgVp/hCxcuqH3AwMDAWhdDjdvvUTcQmZQNM2Mp5g1tq7M5mE0sTfDxc60Qvvk8lu6Pw/OBbnrdke1hf567hbzicvg4WKBnCyednOPZdm6Y9/cVpMmLse9KBga21W0nhNJyBfZfzQBQtxZk5TWE0wpBQHJWodbePRjfzRdrjiUhPiMfOy/cwpBgj2r3EQSBLdWIiERCrTAcHBwMiUQCQRCeGGAqKiq0Uhg1LnfyS7DgnysAgGl9/XW+Mcaw9h7YcvYGjsffwcd/XsTa10MNYgGUIAiqhXNjuvhovBuausyMZRjVyQvfHUrAupPJOg/DkUl3kFdcDkcrUwR7Nan1cfwcLSGVoErbvbp0GamJjZkxJnT3w1d7Y++9WHKv1qHi0q1c3MwpgpmxFE/p6AULEREZBrXe+0tKSkJiYiKSkpKwZcsW+Pn5YeXKlTh37hzOnTuHlStXolmzZtiyZYuu66UGat7fV5BTWIZWbjZ4/d4GCLokkUgwf2g7mBpJcTQuSzX/VN/OpNzF1fQ8mBlLMTLES6fneqWzN6QS4Hj8HcRn5On0XJWjqP1aO9eq9VklN1tzLBzeDrJ7L1y00WWkJuO7+cLW3BgJmQXYeaH6c2PPZeX306OFk9528iMiovqh1siwj8/9eY0jR47E0qVL8eyzz6puCwwMhJeXF2bPno2hQ4dqvUh9SZMXISmrAH6Ollr/YywmR+Myse3cTUgkwMLh7dTefreufB0t8W6fFvhy9zV8vvMyng5wgp2Ffue1f39YOYe5XysX2FrUvk2YOjybWKB3Sxfsu3Ibi3ddw9whbXTyPFYoBNUWzNrYmGJUJ2/08HdCclYhfB0tdFKztZkxJj7lh//uicW3NYwOV85/HsCNNoiIGj2NU0lMTAz8/KqP7Pn5+eHy5ctaKcoQ/HYqFV0XHcDLP0Si26ID9boqvzEpLqvAx39eBACM7eJT5y2HNTXxqabwd7HCnYJSLPxHv72HfziSiH33WnXtjEmrl+eUj4NyesGey7d19jy+cFOO27klsDI1QtdmDlo5pputuVa7jNRkXFdf2FkYIzGzADvO3x8dTr1TiKvpeZBJJejTylln5yciIsOgcRhu1aoVFi5ciNLS+zuIlZaWYuHChWjVSvNtUg1RmrwIM7bGoHIPEYUAfLQ1BvEZ+fotrAFadiAOKXcK4WpjhukDAur9/CZGUiy413t405nriEy8U+81AMrnVOWcaQAQBGDm1ota65LwqHP+dDxJ9blCR+es7LrwdIATTI0azpQC5eiwsrPE0v1xqq4blaPcnf3s9f5OAhER6Z7GYXjVqlXYvXs3PD090bdvX/Tt2xeenp7YvXs3Vq1apYsa690fUTfw8Jp2QQCeW3oUM7ZewIUbOfooq8G5lp6nmhbw6eA2ddo9rC46+trj5c7KDT1mbItBSXn9L/KMz8iv9pyq7JKgK0lZBXi4OYMuzvlgS7WGZlxXXzSxMEZiVgF23Js7rMkuekRE1PBpHIZDQ0ORmJiIefPmITAwEIGBgZg/fz4SExMRGhqqixrr1eVbuVhxML7Gr5WUK7Dx1HUMXn4czy09il9PpiCvuKyeK2wYFAoBM7fFoFwhoF9rF513M3iSDwe2hKOVKRIzC/DdoYR6P39qdvUAqu0uCQ+r7MzwIIkEWj1nfEY+EjILYCyT4OmAhtd1wcrUCBN7VI4OxyMjtxinU5TbzfdrgOGeiIg0p9YCuodZWlrizTff1HYtepeRV4wJv5xGcZkCzZ0tkZipHFmTSSSYP6wt/BwtsfFUKv65mI5Lt3Lx8Z8XMf/vKxgc5I7Rnb0R5GlrEO27DMGGU8rtkC1NZJg7uI2+y4GtuTHmDGqNKRvPYeXBBAwKckczJ/V3SauryjmpEgACdNcl4UGVnRlmbr2IintzfixNjGBnrr23/iunFHRt5ggbPY3819XYMF/8cCQRSVkFmLopGoIAtPOwhYcdF80SEYmBWmH4r7/+wjPPPANjY2P89ddfj73v4MGDtVJYfSsuq8D/rYvCLXkxmjpaYstb3VBYVl5tRXvnpg6YU1CKreduYuOpVMRn5GPTmevYdOY6WrnZ4OVQLwxp74GCknLRdqLIyC3GF7uUi9Xe6x8AdwMJFc8HumHL2Rs4dC0T038/j/f7B8DPSfc/n9jbeTiZmA2pBNj6dlcUlSl01iXhYZWdGRIy8jH9jwtIlxdjfWSKahe2ulJNKWjAG1NYmRrhzR7N8MWuqziRoJxTHtaU2y8TEYmFRBCER+9Heo9UKkV6ejqcnZ0hlT56ZoVEIjG4TTdyc3Nha2sLuVwOGxubGu8jCAKmborG9uhbsDU3xp+TusHP8ck7lgmCgDMpd7ExMhU7Y9JQWq5cgGMkk6C8QnlZpffaiY3q5K29b8rATdpwFn9fSEOgpy22vdOtTn1nte16diF6f3UIZfX485n950WsO5mCAW1c8P2Y6tsN15dNp1Px4ZYYOFqZ4OgHvevcPzddXowuC/dDIgEiZ/aBs7WZliqtfwUl5Qidvw8FpcrfX2L8f0uNhzp/94joPrXmDCsUCjg7O6v+/agPQwvC6lpxMB7bo2/BSCrBd690UCsIA8rw38nXHktGBePUzD6YM6g1/BwtVEEY0N0KfkN18GoG/r6QBqkEWDCsnUEFYaDqCxVA9z+fvOIybD17AwAwLsxXJ+dQ1/AOnvCyN0dWfil+PZlS5+Ptvdcmrr2XXYMOwgCQW1yGwtL7v7/E9v+WiEjM6mf3AwP2T0wa/rsnFgAwd0gbdG3uWKvj2FmY4LVufpg/tF21r+m6a4ChKCwtV/UUfr2bH9p62Oq5ouqSsgrqtavD1rM3UVBagebOVgjTUg/e2jKWSTGlVwsAwPdHElBYWl6n4zXkLhIPq+/nBRERGQ615gwvXbpU7QO+++67tS6mvsXckCN8czQA4LVuvnils8/jH6AGPyflCv4HW1pJtbyC31B9vTcWN3OK4GFnjmn9/PVdTo0qOyzUx89HEASsuzcCO6aLj0EsrhzWwQPLD8YjNbsQv55MwZs9mtXqOPKiMkTcm1/bGHZpq+l5oetuH0REZBjUCsNff/21WgeTSCQNJgzfzi3GhLXKzhFPBzhh1rPa2TDk/gr+GFS+G9/UyQquNrp/G1lf20enyYtw8GoG1hxTbvDw+dA2sDStVaMSnav8+czYGqMKPgPauOrkekUk3EF8Rj4sTWQY3sFD68evDWOZFJN7N8cHf1zA94cT8WoXH1iYaP6zOnQtA+UKAS2crdSeVmTIHu68UR/dPoiIyDCo9VcwKSnpyXdqQIpKKzDhlzO4nVuCFs5WWDq6PYxk2psxUrmCPzLxDj744wLiM/LxZ/RNDGvvqbVzPGzT6VRVwKvPxT8PnhdQtqTq3dKwOwtU/nx+OZGCVYcTEJmUjYKScq0H+LURylHhYR089LbhSE2Gt/fAioPxSLlTiHURKfi/npqPDld2kWgMo8KVKp8XD3eQISKixk10c4YVCgHv/R6NmJty2FuaYM24Tjrpj+pma46h7T3xn77K6QKf77yCuwWlT3hU7VRuH10ZSJWLf2J0vvjn4fMCwKVb8gax6MjN1hzT+/vD18EC2QWlquCqLbdyilQLzMbqeeHcw4xkUkzpXTl3OBEFJZrNHS4uq8ChaxkAGnZLtZq42ZojrJkDgzARkYjUKgzfuHEDK1euxEcffYTw8PAqH4bum/1x+CcmHcYyCVa9GgJvB93OCZz4VFP4u1ghu6AUC/65opNz1LztLvDDkUSd7ZAXezsPn/51qdp5FQIazKKjB0Ph6iMJyNcwFD7OhshUVCgEdGlqD38Xa60dV1uGBrvX+oXAiYQsFJRWwM3WDO0McJEkERGRJjR+X3j//v0YPHgwmjZtiqtXr6Jt27ZITk6GIAjo0KGDLmrUmu3RN7F0fxwAZduvUD/dN9Y3MZJi4fB2GPFdBH6PuoHhHTy13lXgUTtl/e94snL7aC3tkFdUWoG/Y9Kw8d7ucjVpaIuOhgS7Y/nBeCRlFeCXE8mY1Kt5nY9ZUl6B306nAjC8UeFKlS8E3vv9PFYfScDYMB+1p4nsvnhvo43WLgaxKJCIiKguNB4ZnjFjBqZPn46YmBiYmZlhy5YtuH79Onr27ImRI0fqokatOH/9Lt7/4wIA4P96NMXIjl71du4QH3u80lk5f3fWnzEoKddeP2ZBELD8QHyV26QS5W5rzZ2tUFRWgU1nrmPoiuN45tujWBuRDHmRZqPFV9NzMWf7RYQu2Ifpv59HVMpdyKQSDGjjgte6+aKylXBDXHSkDIXKAPzD0UStjA7vupiOrPxSuNiYol9rw51GMCTYHX6OlrhbWIZfIpLVekyFQsC+K5W7zjWe+cJERCReau1A9yBra2tER0ejWbNmaNKkCY4dO4Y2bdrg/PnzGDJkCJKTk3VUau1U7sTTbsY25CqM0beVC74fE1Lvm0HIi8rQd8lhZOaV4D99Wmit9dj3hxOw8N+rkEqAr14MhquNmWrxz6N2yDMzluL5QHeMDvVGB287SCSSap0oCkvLsfOCchT4XGqO6nxe9uZ4qZM3RoZ4wvleh4w0eVGDXnRUXqFA/6+PIDGrAO8PCKjz6PCI704gKuUupvX1x3/6ttBSlbqx9ewNhG8+DzsLYxz7sDesnjA6fDo5GyNXRcDW3BhnPu4LYy0uPCUi7eAOdESa0XiahKWlJUpLlQvB3NzckJCQgDZt2gAAsrKytFudFuUUlcHdyQrfvhSsl13RbM2NMWdQa0zecA7fHUrAoCB3NHe2qtMx916+jUW7rgIAPnm+NYa1r9q+q3KHvE6+9vhkUGtsO3cTG0+lIvZ2Pv6IuoE/om4gwMUaAa5W2HkhDQoBkEiAzn72uHQzF3n3RkmNpBL0b+OC0aHe6NbMEdKHrp+brXmDDMGVjGRSvNunBaZuisYPRxMxNsyn1t0fLt6UIyrlLoykEowOrb93H2prcJA7lh+IR6Ka00R2X1RutNGnpTODMBERNQoa/zXr0qULjh07BgB49tln8d5772H+/Pl4/fXX0aVLF60XqE0ZeSXI1dGCMnU8184NvQKcUFqhwMxtMVA8vPpMA1fScvGf385BEIBXu3hjXFffx96/coe83VN7YMvbYRjRwROmRlJcu52Hv86nqRbCCQJwMjEbeSXl8HGwwIcDWyJiRh+sfCUET7VwqhaEG4tBQe5o6mSJnMIy/HIiudbHWXdvMdrAtq6qkXNDZiSTYkqf+9NEHrfgUhAE7LlcOUXCcKd/EBERaULjMLxkyRJ07twZADB37lz06dMHmzZtgq+vL9asWaP1ArVJ350OJBIJPhvSFubGMpxKysYfUTdqdZzMvBJM+OUMCksr0K25A+YMaqP2QiaJRIIQH3t89WIQTs3qi/GPCNGznm2Fg+89jbefbgYna9Na1dmQyKQS/KePckrDD0eTavWiSV5Yhu3nbwLAE1+cGJLBQR5qvRC4mp6H1OxCmBpJ0cPfqf4KJCIi0iGNw3DTpk0RGBgIQDllYtWqVbhw4QK2bNkCH5+6b2esS4bQ6cDL3gLh9+YLz//nCrLySzR6fHFZBd5cdwY3c4rQ1NESK18OqfXb1bbmxvi/nk3x8GCvTCLB80FujXYU+FGeD3RHMydLyIvK8MvxZI0f/3vUdRSXKdDS1RodfZpov0AdefiFwKNGhys32niqhVOtdq0jIiIyRBqnqAkTJuDQoUM6KEW3DKnTwWvdfNHazQbyojLM23lZ7ccJgoAPt1zAudQc2JobY834TrC1qNuGIZXb0MrujSwb0nWqbzKpBO+qQmGiRqPDCoWAdSeVUyTGhvk2uJZjD74Q+PkRLwT2XFbOF+YUCSIiakw0DsOZmZkYOHAgvLy88P777+P8+fO1PvmRI0cwaNAguLu7QyKR4M8//6zydUEQ8Mknn8DNzQ3m5ubo27cv4uLianWu3dOeqpftidVhJFP2HpZKgD+jb+FoXKZaj1txMB7bo2/BSCrBd690gJ+jpVbqGdXJG8c+6oWNE7vg2Ee9DOY66cPzgcqFjbnF5fjpWLLajzsSl4mUO4WwNjPC0PbuuitQRx58IfDjserTRK5nF+LSrVxIJUDfVgzDRETUeGgchrdv3460tDTMnj0bp0+fRocOHdCmTRssWLBA47ZqBQUFCAoKwooVK2r8+uLFi7F06VKsWrUKkZGRsLS0xIABA1BcXKxp2XA1sJHOIC871YYMs7ZdRFHp43sP/xuThv/uiQUAzB3SBl2bO2q1Hm5Dq/TglIE1xxLV7slcuXDuhRDPBjuFoPKFQE2jw3vvLZzr5GsPe0sTPVRHRESkG7WabNqkSRO8+eabOHToEFJSUjB+/HisW7cOzZtr1p/1mWeewbx58zBs2LBqXxMEAd988w0+/vhjDBkyBIGBgVi7di1u3bpVbQS5oZo+IAButmZIzS7E0gOPHvGOuSHHtM3RAJRTLF7pbNhzsxu6Z9u5oUXl6PDxpCfe/3p2IQ5cywAAjOnScH82VUaHj1Z9IVA5RWIAN9ogIqJGpk6NQsvKynDmzBlERkYiOTkZLi7ae/s0KSkJ6enp6Nu3r+o2W1tbdO7cGREREVo7jz5ZmRph7mBlj+YfjiTianputfvczi3GhLWnUVymwNMBTpj1bKv6LlN0ZFKJarOMNceSnjg6/OvJFAgC8FQLRzR1qlvvaH177oEXApWjw9kFpTiVlA0ABr2jHhERUW3U6v3cgwcPYsOGDdiyZQsUCgWGDx+OnTt3onfv3lorLD1dORL1cMB2cXFRfa0mJSUlKCm536EhN1cZMMvKylBWpr8ew4/Sy98B/Vo5Y++VDMzYcgG/TQhVdXEoKq3AhF9O43ZuCZo7WWLJC20hKCpQptDeds5Us34BjmjhbIm4jAL8eCQe7/au+V2P4rIKbDp9HQDwSidPg3yOaWry003xn80X8OOxRLwa6oG9VzKgEIBWrtZwtTZuFN8jUWPG/6NEmtE4DHt4eCA7OxsDBw7E6tWrMWjQIJiaGk4f2oULF2Lu3LnVbt+zZw8sLPTbVu1RupsDR2QynLsux8c/70J3VwEKAfglToqYO1JYGgkY7SnH0QN79V2qqHSzkyAuQ4YfjiTAPT8WFjX8b4nMkCCnSAZ7UwFFiWfwz5NnVRg8hQC4msuQXlSOWWv340YBAEjha5SDf/75R9/lEdETFBbqr58+UUOkcRj+9NNPMXLkSNjZ2emgnPtcXZVzE2/fvg03NzfV7bdv30ZwcPAjHzdjxgyEh4erPs/NzYWXlxf69+9v0Hu0l7ul4vO/r+LfW6YYPSAEayNSEX0nDcYyCX4Y1wmdfBtO39rGYqBCwIkVEYjNyMdNS3/8p0/V0WFBEPDDqkgAuXi9hz+e7+Gnn0J1QOaTjnc3XcCRDGOUlSsACHhnSHe0dLXWd2lE9ASV74gSkXo0DsMTJ07URR3V+Pn5wdXVFfv371eF39zcXERGRuLtt99+5ONMTU1rHKk2NjaGsXHdevLq0vhuTfHX+TScvyHH8FWRqtuHBnugawtnPVYmblP7+eOd9WfxS0QqJvZoXqWv87nUu7h4KxcmMilGd/Yx6OeXpp4P8sT8f6/hdu79KUeX0vLRzstej1URkToa0+8iovpQpwV0dZWfn4/o6GhER0cDUC6ai46ORmpqKiQSCaZOnYp58+bhr7/+QkxMDMaOHQt3d3cMHTpUn2XrhEwqUe1M96CtZ28iTV6kh4oIAAa2cUVLV2vklZTjx2OJVb5W2U7t+SA3OFgZzlQhbbidV4yM3Kq7I87adpHPRSIianT0GobPnDmD9u3bo3379gCA8PBwtG/fHp988gkA4IMPPsCUKVPw5ptvolOnTsjPz8euXbtgZmamz7J1xtio+o+jQhCQnMX5X/oilUow9V5niZ+OJyOnsBQAcCe/BDsvpAGAql90Y5KUVQDhodv4XCQiosZIr7sDPP300xCEh//k3ieRSPDZZ5/hs88+q8eq9MfP0RJSiXIBUyWZRAJfR8Nc+CcW/VsrR4evpufhx6NJmD4gAJvOXEdphQKBnrYI9rLTd4lax+ciERGJhV5HhqkqN1tzLBzeDjKJsrWaTCLBguFtRb8rnL4pR4eVU1h+Op6ErPwSrD+ZCqBhb7LxOHwuEhGRWEiExw3NNgK5ubmwtbWFXC436G4SD0qTFyE5qxC+jhYMHwZCEAQ8t/QYLqflorWbDS6n5cLGzAinZvWFmbFM3+XpDJ+LRA1PQ/y7R6RPHBk2QG625ghr5sDwYUAkkvu70l1OU7Ytyisux/bom/osS+f4XCQiosaOYZhITe08qo6wCABmbmWHBSIiooaMYZhITcl3qndSYIcFIiKiho1hmEhNlR0WHsQOC0RERA0bwzCRmthhgYiIqPHRa59hooZmVCdv9PB3YocFIiKiRoJhmEhDbrbmDMFERESNBKdJEBEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWg1iDC8YsUK+Pr6wszMDJ07d8apU6f0XRIRERERNQIGH4Y3bdqE8PBwzJkzB2fPnkVQUBAGDBiAjIwMfZdGRERERA2cwYfhJUuWYOLEiXjttdfQunVrrFq1ChYWFvjf//6n79KIiIiIqIEz0ncBj1NaWoqoqCjMmDFDdZtUKkXfvn0RERFR42NKSkpQUlKi+lwulwMAsrOzUVZWptuCiYiI9CwvLw8AIAiCnishahgMOgxnZWWhoqICLi4uVW53cXHB1atXa3zMwoULMXfu3Gq3+/n56aRGIiIiQ3Tnzh3Y2trquwwig2fQYbg2ZsyYgfDwcNXnOTk58PHxQWpqKn8pPEZubi68vLxw/fp12NjY6Lscg8ZrpR5eJ/XxWqmH10k9crkc3t7esLe313cpRA2CQYdhR0dHyGQy3L59u8rtt2/fhqura42PMTU1hampabXbbW1t+ctTDTY2NrxOauK1Ug+vk/p4rdTD66QeqdTglwURGQSD/p9iYmKCkJAQ7N+/X3WbQqHA/v37ERYWpsfKiIiIiKgxMOiRYQAIDw/HuHHj0LFjR4SGhuKbb75BQUEBXnvtNX2XRkREREQNnMGH4VGjRiEzMxOffPIJ0tPTERwcjF27dlVbVPcopqammDNnTo1TJ+g+Xif18Vqph9dJfbxW6uF1Ug+vE5FmJAJ7rxARERGRSBn0nGEiIiIiIl1iGCYiIiIi0WIYJiIiIiLRYhgmIiIiItFq1GF4xYoV8PX1hZmZGTp37oxTp07puySD8+mnn0IikVT5aNmypb7L0rsjR45g0KBBcHd3h0QiwZ9//lnl64Ig4JNPPoGbmxvMzc3Rt29fxMXF6adYPXvStRo/fny159jAgQP1U6weLVy4EJ06dYK1tTWcnZ0xdOhQXLt2rcp9iouLMWnSJDg4OMDKygojRoyotulQY6fOdXr66aerPafeeustPVWsP9999x0CAwNVm5CEhYXh33//VX2dzyci9TTaMLxp0yaEh4djzpw5OHv2LIKCgjBgwABkZGTouzSD06ZNG6Slpak+jh07pu+S9K6goABBQUFYsWJFjV9fvHgxli5dilWrViEyMhKWlpYYMGAAiouL67lS/XvStQKAgQMHVnmObdy4sR4rNAyHDx/GpEmTcPLkSezduxdlZWXo378/CgoKVPeZNm0aduzYgd9//x2HDx/GrVu3MHz4cD1WXf/UuU4AMHHixCrPqcWLF+upYv3x9PTEokWLEBUVhTNnzqB3794YMmQILl26BIDPJyK1CY1UaGioMGnSJNXnFRUVgru7u7Bw4UI9VmV45syZIwQFBem7DIMGQNi2bZvqc4VCIbi6ugpffvml6racnBzB1NRU2Lhxox4qNBwPXytBEIRx48YJQ4YM0Us9hiwjI0MAIBw+fFgQBOVzyNjYWPj9999V97ly5YoAQIiIiNBXmXr38HUSBEHo2bOn8J///Ed/RRmwJk2aCD/++COfT0QaaJQjw6WlpYiKikLfvn1Vt0mlUvTt2xcRERF6rMwwxcXFwd3dHU2bNsUrr7yC1NRUfZdk0JKSkpCenl7l+WVra4vOnTvz+fUIhw4dgrOzMwICAvD222/jzp07+i5J7+RyOQDA3t4eABAVFYWysrIqz6uWLVvC29tb1M+rh69TpfXr18PR0RFt27bFjBkzUFhYqI/yDEZFRQV+++03FBQUICwsjM8nIg0Y/A50tZGVlYWKiopqu9S5uLjg6tWreqrKMHXu3Bk///wzAgICkJaWhrlz5+Kpp57CxYsXYW1tre/yDFJ6ejoA1Pj8qvwa3Tdw4EAMHz4cfn5+SEhIwMyZM/HMM88gIiICMplM3+XphUKhwNSpU9GtWze0bdsWgPJ5ZWJiAjs7uyr3FfPzqqbrBAAvv/wyfHx84O7ujgsXLuDDDz/EtWvXsHXrVj1Wqx8xMTEICwtDcXExrKyssG3bNrRu3RrR0dF8PhGpqVGGYVLfM888o/p3YGAgOnfuDB8fH2zevBlvvPGGHiujxuKll15S/btdu3YIDAxEs2bNcOjQIfTp00ePlenPpEmTcPHiRc7Pf4JHXac333xT9e927drBzc0Nffr0QUJCApo1a1bfZepVQEAAoqOjIZfL8ccff2DcuHE4fPiwvssialAa5TQJR0dHyGSyaqtmb9++DVdXVz1V1TDY2dnB398f8fHx+i7FYFU+h/j8qp2mTZvC0dFRtM+xyZMnY+fOnTh48CA8PT1Vt7u6uqK0tBQ5OTlV7i/W59WjrlNNOnfuDACifE6ZmJigefPmCAkJwcKFCxEUFIRvv/2WzyciDTTKMGxiYoKQkBDs379fdZtCocD+/fsRFhamx8oMX35+PhISEuDm5qbvUgyWn58fXF1dqzy/cnNzERkZyeeXGm7cuIE7d+6I7jkmCAImT56Mbdu24cCBA/Dz86vy9ZCQEBgbG1d5Xl27dg2pqamiel496TrVJDo6GgBE95yqiUKhQElJCZ9PRBpotNMkwsPDMW7cOHTs2BGhoaH45ptvUFBQgNdee03fpRmU6dOnY9CgQfDx8cGtW7cwZ84cyGQyjB49Wt+l6VV+fn6VUaakpCRER0fD3t4e3t7emDp1KubNm4cWLVrAz88Ps2fPhru7O4YOHaq/ovXkcdfK3t4ec+fOxYgRI+Dq6oqEhAR88MEHaN68OQYMGKDHquvfpEmTsGHDBmzfvh3W1taqeZu2trYwNzeHra0t3njjDYSHh8Pe3h42NjaYMmUKwsLC0KVLFz1XX3+edJ0SEhKwYcMGPPvss3BwcMCFCxcwbdo09OjRA4GBgXquvn7NmDEDzzzzDLy9vZGXl4cNGzbg0KFD2L17N59PRJrQdzsLXVq2bJng7e0tmJiYCKGhocLJkyf1XZLBGTVqlODm5iaYmJgIHh4ewqhRo4T4+Hh9l6V3Bw8eFABU+xg3bpwgCMr2arNnzxZcXFwEU1NToU+fPsK1a9f0W7SePO5aFRYWCv379xecnJwEY2NjwcfHR5g4caKQnp6u77LrXU3XCIDw008/qe5TVFQkvPPOO0KTJk0ECwsLYdiwYUJaWpr+itaDJ12n1NRUoUePHoK9vb1gamoqNG/eXHj//fcFuVyu38L14PXXXxd8fHwEExMTwcnJSejTp4+wZ88e1df5fCJSj0QQBKE+wzcRERERkaFolHOGiYiIiIjUwTBMRERERKLFMExEREREosUwTERERESixTBMRERERKLFMExEREREosUwTERERESixTBMRAbl0KFDkEgkyMnJ0XcpREQkAgzDRERERCRaDMNEREREJFoMw0RUhUKhwMKFC+Hn5wdzc3MEBQXhjz/+AHB/CsPff/+NwMBAmJmZoUuXLrh48WKVY2zZsgVt2rSBqakpfH198dVXX1X5eklJCT788EN4eXnB1NQUzZs3x5o1a6rcJyoqCh07doSFhQW6du2Ka9eu6fYbJyIiUWIYJqIqFi5ciLVr12LVqlW4dOkSpk2bhldffRWHDx9W3ef999/HV199hdOnT8PJyQmDBg1CWVkZAGWIffHFF/HSSy8hJiYGn376KWbPno2ff/5Z9fixY8di48aNWLp0Ka5cuYLvv/8eVlZWVeqYNWsWvvrqK5w5cwZGRkZ4/fXX6+X7JyIicZEIgiDouwgiMgwlJSWwt7fHvn37EBYWprp9woQJKCwsxJtvvolevXrht99+w6hRowAA2dnZ8PT0xM8//4wXX3wRr7zyCjIzM7Fnzx7V4z/44AP8/fffuHTpEmJjYxEQEIC9e/eib9++1Wo4dOgQevXqhX379qFPnz4AgH/++QfPPfccioqKYGZmpuOrQEREYsKRYSJSiY+PR2FhIfr16wcrKyvVx9q1a5GQkKC634NB2d7eHgEBAbhy5QoA4MqVK+jWrVuV43br1g1xcXGoqKhAdHQ0ZDIZevbs+dhaAgMDVf92c3MDAGRkZNT5eyQiInqQkb4LICLDkZ+fDwD4+++/4eHhUeVrpqamVQJxbZmbm6t1P2NjY9W/JRIJAOV8ZiIiIm3iyDARqbRu3RqmpqZITU1F8+bNq3x4eXmp7nfy5EnVv+/evYvY2Fi0atUKANCqVSscP368ynGPHz8Of39/yGQytGvXDgqFosocZCIiIn3hyDARqVhbW2P69OmYNm0aFAoFunfvDrlcjuPHj8PGxgY+Pj4AgM8++wwODg5wcXHBrFmz4OjoiKFDhwIA3nvvPXTq1Amff/45Ro0ahYiICCxfvhwrV64EAPj6+mLcuHF4/fXXsXTpUgQFBSElJQUZGRl48cUX9fWtExGRSDEME1EVn3/+OZycnLBw4UIkJibCzs4OHTp0wMyZM1XTFBYtWoT//Oc/iIuLQ3BwMHbs2AETExMAQIcOHbB582Z88skn+Pzzz+Hm5obPPvsM48ePV53ju+++w8yZM/HOO+/gzp078Pb2xsyZM/Xx7RIRkcixmwQRqa2y08Pdu3dhZ2en73KIiIjqjHOGiYiIiEi0GIaJiIiISLQ4TYKIiIiIRIsjw0REREQkWgzDRERERCRaDMNEREREJFoMw0REREQkWgzDRERERCRaDMNEREREJFoMw0REREQkWgzDRERERCRaDMNEREREJFr/D8QavWgHj7xaAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "fig, ax = plt.subplots()\n", + "\n", + "key = 'ResSCNN1'\n", + "max_epochs = 30\n", + "max_y = 30\n", + "\n", + "_ = np.round(architectures_TM[key]['epochs_acc'][max_y-1], 2)\n", + "ax.plot(np.arange(len(architectures_TM[key]['epochs_x'][:max_epochs])), architectures_TM[key]['epochs_acc'][:max_epochs], label=f'{key} ({_}%)', marker='.')\n", + "\n", + "ax.set_ylim(0, max_y)\n", + "ax.set_yticks(np.arange(0, max_y+10, 10))\n", + "ax.set_ylabel('validation acc [%]')\n", + "ax.set_xlim(0, max_epochs)\n", + "ax.set_xlabel('epoch')\n", + "\n", + "pos = ax.get_position()\n", + "ax.set_position([pos.x0, pos.y0, pos.width * 0.9, pos.height])\n", + "ax.legend(loc='center right', bbox_to_anchor=(1.45, 0.5), framealpha=0)\n", + "ax.grid(axis='y')\n", + "\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAqwAAAG2CAYAAAC6Z9RQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAAA2VElEQVR4nO3deXhU5d3/8c/MZDLZyIQkQBIIENlkR9lEqEhBQa0F8VFUtFGrPlZcAIUKFhBRg7Qil0pF/Vmpj4paFahSLYgKVRYRjAKyCwYkLAGSCVknmfP7IzA1JkASQubO5P26rrkyc+acme8cJ5yPd77nPjbLsiwBAAAAhrIHugAAAADgdAisAAAAMBqBFQAAAEYjsAIAAMBoBFYAAAAYjcAKAAAAoxFYAQAAYDQCKwAAAIxGYAUAAIDRCKwAAAAwWkAD68qVK3X11VcrKSlJNptNixYtKve8ZVmaOnWqEhMTFR4eriFDhmjHjh2BKRYAAAABEdDAmpeXp+7du2vu3LmVPj9r1iw9++yzmjdvntauXavIyEgNHTpUhYWFdVwpAAAAAsVmWZYV6CIkyWazaeHChRoxYoSkstHVpKQkPfjgg3rooYckSTk5OWrWrJnmz5+vG264IYDVAgAAoK6EBLqAU9m9e7cOHDigIUOG+Je53W717dtXq1evPmVgLSoqUlFRkf+xz+fT0aNHFRcXJ5vNds7rBgAgkCzLUm5urpKSkmS3c6oKgoOxgfXAgQOSpGbNmpVb3qxZM/9zlUlLS9P06dPPaW0AAJhu7969atGiRaDLAGqFsYG1piZNmqTx48f7H+fk5Khly5bavXu3GjVqFMDKAAA493Jzc5WSksIxD0HF2MCakJAgSTp48KASExP9yw8ePKgePXqccjuXyyWXy1VheWxsrKKjo2u9TgAATOJ0OiWJNjgEFWObW1JSUpSQkKDly5f7l3k8Hq1du1b9+vULYGUAAACoSwEdYT1+/Lh27tzpf7x7926lp6crNjZWLVu21NixY/X444+rXbt2SklJ0ZQpU5SUlOSfSQAAAADBL6CB9euvv9agQYP8j0/2nqampmr+/PmaOHGi8vLydNdddyk7O1sDBgzQxx9/rLCwsECVDAAAgDpmzDys54rH45Hb7VZOTg49rACAoMdxD8HI2B5WAAAAQCKwAgAAwHAEVgAAABiNwAoAAACjEVgBAABgNAIrAAAAjEZgBQAAgNEIrAAAADAagRUAAABGI7ACAADAaARWAAAAGI3ACgAAAKMRWAEAAGA0AisAAACMRmAFAACA0QisAAAAMBqBFQAAAEYjsAIAAMBoBFYAAAAYjcAKAAAAoxFYAQAAYDQCKwAAAIxGYAUAAIDRCKwAAAAwGoEVAAAARiOwAgAAwGgEVgAAABiNwAoAAACjEVgBAABgNAIrAAAAjEZgBQAAgNEIrAAAADAagRUAAABGI7ACAADAaARWAAAAGI3ACgAAAKMRWAEAAGA0AisAAACMRmAFAACA0QisAAAAMBqBFQAAAEYjsAIAAMBoBFYAAAAYjcAKAAAAoxFYAQAAYDQCKwAAAIxGYAUAAIDRCKwAAAAwGoEVAAAARiOwAgAAwGgEVgAAABiNwAoAAACjEVgBAABgNAIrAAAAjEZgBQAAgNEIrAAAADAagRUAAABGI7ACAADAaARWAAAAGI3ACgAAAKMRWAEAAGA0AisAAACMRmAFAACA0YwOrKWlpZoyZYpSUlIUHh6uNm3aaMaMGbIsK9ClAQAAoI6EBLqA03nqqaf0wgsv6O9//7s6d+6sr7/+Wrfddpvcbrfuv//+QJcHAACAOmB0YF21apWGDx+uq666SpLUunVrLViwQF999VWAKwMAAEBdMTqwXnzxxXrppZe0fft2tW/fXt9++62++OILzZ49+5TbFBUVqaioyP/Y4/FIkrxer7xe7zmvGQCAQOJYh2BkdGB9+OGH5fF4dP7558vhcKi0tFRPPPGERo8efcpt0tLSNH369ArLly5dqoiIiHNZLgAAAZefnx/oEoBaZ7MMPoPprbfe0oQJE/TnP/9ZnTt3Vnp6usaOHavZs2crNTW10m0qG2FNTk5WVlaWoqOj66p0AAACwuPxKD4+Xjk5ORz3EDSMDqzJycl6+OGHNWbMGP+yxx9/XK+//rq2bt1apdfweDxyu9384gIAGgSOewhGRk9rlZ+fL7u9fIkOh0M+ny9AFQEAAKCuGd3DevXVV+uJJ55Qy5Yt1blzZ33zzTeaPXu2br/99kCXBgAAgDpidEtAbm6upkyZooULF+rQoUNKSkrSjTfeqKlTpyo0NLRKr8GfRgAADQnHPQQjowNrbeAXFwDQkHDcQzAyuocVAAAAILACAADAaARWAAAAGI3ACgAAAKMRWAEAAGA0AisAAACMRmAFAACA0QisAAAAMBqBFQAAAEYjsAIAAMBoBFYAAAAYjcAKAAAAoxFYAQAAYDQCKwAAAIxGYAUAAIDRCKwAAAAwGoEVAAAARiOwAgAAwGgEVgAAABiNwAoAAACjEVgBAABgNAIrAAAAjEZgBQAAgNEIrAAAADAagRUAAABGI7ACAADAaARWAAAAGI3ACgAAAKMRWAEAAGA0AisAAACMRmAFAACA0QisAAAAMBqBFQAAAEYjsAIAAMBoBFYAAAAYjcAKAAAAoxFYAQAAYDQCKwAAAIxGYAUAAIDRCKwAAAAwGoEVAAAARiOwAgAAwGgEVgAAABiNwAoAAACjEVgBAABgNAIrAAAAjEZgBQAAgNEIrAAAADAagRUAAABGI7ACAADAaARWAAAAGI3ACgAAAKMRWAEAAGA0AisAAACMRmAFAACA0QisAAAAMBqBFQAAAEYjsAIAAMBoBFYAAAAYjcAKAAAAoxFYAQAAYDQCKwAAAIxGYAUAAIDRjA+sP/30k26++WbFxcUpPDxcXbt21ddffx3osgAAAFBHQgJdwOkcO3ZM/fv316BBg/TRRx+pSZMm2rFjhxo3bhzo0gAAAFBHjA6sTz31lJKTk/Xqq6/6l6WkpASwIgAAANQ1owPrP//5Tw0dOlTXXXedVqxYoebNm+uee+7RnXfeecptioqKVFRU5H/s8XgkSV6vV16v95zXDABAIHGsQzAyOrD+8MMPeuGFFzR+/HhNnjxZ69at0/3336/Q0FClpqZWuk1aWpqmT59eYfnSpUsVERFxrksGACCg8vPzA10CUOtslmVZgS7iVEJDQ9WrVy+tWrXKv+z+++/XunXrtHr16kq3qWyENTk5WVlZWYqOjj7nNQMAEEgej0fx8fHKycnhuIegYfQIa2Jiojp16lRuWceOHfXee++dchuXyyWXy1VhudPplNPprPUaAQAwCcc6BCOjp7Xq37+/tm3bVm7Z9u3b1apVqwBVBAAAgLpmdGAdN26c1qxZoyeffFI7d+7Um2++qZdeekljxowJdGkAAACoI0YH1t69e2vhwoVasGCBunTpohkzZmjOnDkaPXp0oEsDAABAHTH6pKva4PF45Ha7aT4HADQIHPcQjIweYQUAAAAIrAAAADAagRUAAABGI7ACAADAaARWAAAAGI3ACgAAAKMRWAEAAGA0AisAAACMRmAFAACA0QisAAAAMBqBFQAAAEYjsAIAAMBoBFYAAAAYjcAKAAAAoxFYAQAAYDQCKwAAAIxGYAUAAIDRCKwAAAAwGoEVAAAARiOwAgAAwGgEVgAAABitRoH173//u5YsWeJ/PHHiRMXExOjiiy/Wjz/+WGvFAQAAADUKrE8++aTCw8MlSatXr9bcuXM1a9YsxcfHa9y4cbVaIAAAABq2kJpstHfvXrVt21aStGjRIl177bW666671L9/f1166aW1WR8AAAAauBqNsEZFRenIkSOSpKVLl+qyyy6TJIWFhamgoKD2qgMAAKhHbr31VtlsNtlsNjmdTqWkpGjixIkqLCyslddfsWKFfv3rXys2NlYRERFq166dUlNTVVxc7F/Hsiy99NJL6tu3r6KiohQTE6NevXppzpw5ys/PlyQ9+uijstlsuvvuu8u9fnp6umw2m/bs2SNJ2rNnj2w2m5o2barc3Nxy6/bo0UOPPvqo//H777+vyy+/XHFxcbLZbEpPT6+VzyzVMLBedtlluuOOO3THHXdo+/btuvLKKyVJmzdvVuvWrWutOAAAgPpm2LBhyszM1A8//KBnnnlGL774oqZNm3bWr/v9999r2LBh6tWrl1auXKmNGzfqueeeU2hoqEpLS/3r3XLLLRo7dqyGDx+uzz77TOnp6ZoyZYoWL16spUuX+tcLCwvTK6+8oh07dpzxvXNzc/WXv/zltOvk5eVpwIABeuqpp2r+IU+hRoF17ty56tevnw4fPqz33ntPcXFxkqT169frxhtvrNUCAQAAzkZmToFW7cpSZk7d/BXY5XIpISFBycnJGjFihIYMGaJly5ZJknw+n9LS0pSSkqLw8HB1795d7777rn/bY8eOafTo0WrSpInCw8PVrl07vfrqq5LK/qqdkJCgWbNmqUuXLmrTpo2GDRuml19+2X9u0TvvvKM33nhDCxYs0OTJk9W7d2+1bt1aw4cP16effqpBgwb536tDhw4aNGiQHnnkkTN+pvvuu0+zZ8/WoUOHTrnOLbfcoqlTp2rIkCE12m+nU6Me1piYGD3//PMVlk+fPv2sCwIAAPgly7JU4C0984q/8N76fZr2z83yWZLdJk3/bWdd27NFtV4j3OmQzWar9ntL0qZNm7Rq1Sq1atVKkpSWlqbXX39d8+bNU7t27bRy5UrdfPPNatKkiQYOHKgpU6bo+++/10cffaT4+Hjt3LnT326ZkJCgzMxMrVy5Updcckml7/fGG2+oQ4cOGj58eIXnbDab3G53uWUzZ85U79699fXXX6tXr16n/Bw33nijli1bpscee6zSDHiu1Siwfvzxx4qKitKAAQMklY24vvzyy+rUqZPmzp2rxo0b12qRAACgYSvwlqrT1H+f1Wv4LGnK4s2asnhztbb7/rGhigitemT68MMPFRUVpZKSEhUVFclut+v5559XUVGRnnzySX3yySfq16+fJOm8887TF198oRdffFEDBw5URkaGLrjgAn94/Hmr5XXXXad///vfGjhwoBISEnTRRRdp8ODB+t3vfqfo6GhJ0o4dO9ShQ4cq13rhhRfq+uuv1x//+EctX778lOvZbDbNnDlTV199tcaNG6c2bdpU+T1qQ41aAiZMmCCPxyNJ2rhxox588EFdeeWV2r17t8aPH1+rBQIAANQngwYNUnp6utauXavU1FTddtttuvbaa7Vz507l5+frsssuU1RUlP/22muvadeuXZKkP/zhD3rrrbfUo0cPTZw4UatWrfK/rsPh0Kuvvqp9+/Zp1qxZat68uZ588kl17txZmZmZkspGoqvr8ccf13/+859y/a2VGTp0qAYMGKApU6ZU+z3OVo1GWHfv3q1OnTpJkt577z395je/0ZNPPqkNGzb4T8ACAACoLeFOh75/bGi1tjmQU6ghs1fI97MMZ7dJn4wfqAR3WLXeuzoiIyP903/+7W9/U/fu3fXKK6+oS5cukqQlS5aoefPm5bZxuVySpCuuuEI//vij/vWvf2nZsmUaPHiwxowZU+6Ep+bNm+uWW27RLbfcohkzZqh9+/aaN2+epk+frvbt22vr1q3VqrdNmza688479fDDD+uVV1457bozZ85Uv379NGHChGq9x9mq0QhraGiof1qETz75RJdffrkkKTY21j/yCgAAUFtsNpsiQkOqdTuvSZTSRnaV40T/qcNmU9rIrjqvSVS1Xqem/auSZLfbNXnyZP3pT39Sp06d5HK5lJGRobZt25a7JScn+7dp0qSJUlNT9frrr2vOnDl66aWXTvn6jRs3VmJiovLy8iRJN910k7Zv367FixdXWNeyLOXk5FT6OlOnTtX27dv11ltvnfbz9OnTRyNHjtTDDz9clY9fa2o0wjpgwACNHz9e/fv311dffaW3335bkrR9+3a1aFG9RmYAAIBzZVTvlrqkfRPtycpX6/gIJbrD67yG6667ThMmTNCLL76ohx56SOPGjZPP59OAAQOUk5OjL7/8UtHR0UpNTdXUqVPVs2dPde7cWUVFRfrwww/VsWNHSdKLL76o9PR0XXPNNWrTpo0KCwv12muvafPmzXruueckSddff70WLlyoG2+8UX/60590+eWXq0mTJtq4caOeeeYZ3XfffRoxYkSFGps1a6bx48frz3/+8xk/zxNPPKHOnTsrJKR8jDx69KgyMjK0f/9+SdK2bdsklZ0slpCQcDa7sGYjrM8//7xCQkL07rvv6oUXXvAPa3/00UcaNmzYWRUEAABQmxLd4erXJi4gYVWSQkJCdO+992rWrFmaNGmSpkyZorS0NHXs2FHDhg3TkiVLlJKSIqnsr9iTJk1St27ddMkll8jhcPhHPfv06aPjx4/r7rvvVufOnTVw4ECtWbNGixYt0sCBAyWVjUS/+eabmj17tn95t27d9Oijj2r48OEaOvTUbRUPPfSQoqKizvh52rdvr9tvv73CxRD++c9/6oILLtBVV10lSbrhhht0wQUXaN68eTXabz9ns2rSnVuPeDweud1u5eTk+M+gAwAgWHHcQzCqUUuAJJWWlmrRokXasmWLJKlz58767W9/K4ejeo3JAAAAwOnUKLDu3LlTV155pX766Sf/XF9paWlKTk7WkiVL6nxuLgAAAASvGvWw3n///WrTpo327t2rDRs2aMOGDcrIyFBKSoruv//+2q4RAAAADViNRlhXrFihNWvWKDY21r8sLi5OM2fOVP/+/WutOAAAAKBGI6wul0u5ubkVlh8/flyhoaFnXRQAAABwUo0C629+8xvdddddWrt2rSzLkmVZWrNmje6++2799re/re0aAQAA0IDVKLA+++yzatOmjfr166ewsDCFhYXp4osvVtu2bTVnzpxaLhEAAAANWY16WGNiYrR48WLt3LnTP61Vx44d/dfNBQAAAGpLlQPr+PHjT/v8Z5995r8/e/bsmlcEAAAA/EyVA+s333xTpfVsNluNiwEAAAB+qcqB9ecjqAAAAEBdqdFJVwAAAEBdIbACAADAaARWAAAAGI3ACgAAAKMRWAEAAGA0AisAAACMRmAFAACA0QisAAAAMFqDCawHcgoCXQIAAABqoMEE1sufWam312UEugwAAABUU4MJrD5Lmvz+JmUy0goAAFCvNJjAKkmllqU9WfmBLgMAAADV0KACq90mtY6PCHQZAAAAqIYGFVjDnQ75rEBXAQAAgOpoMIG1bdNI5RWX6n//72sVeksDXQ4AAACqqMEE1rk3XajYyFBt+smjye9vlGUx1AoAAFAfNJjA2rxxhJ6/6QI57Da9/81PevXLPYEuCQAAAFVQrwLrzJkzZbPZNHbs2Bptf3GbeD1yZUdJ0hP/2qJVu7JqsToAAACcC/UmsK5bt04vvviiunXrdlavc1v/1hp5YXOV+izd++Y32neMaa4AAABMVi8C6/HjxzV69Gi9/PLLaty48Vm9ls1m05PXdFXX5m4dzSvW//7fehUUcxIWAACAqUICXUBVjBkzRldddZWGDBmixx9//LTrFhUVqaioyP/Y4/FIkrxer7xeryTJIen5G7rpmnlrtHm/RxPfTdfT/9NVNpvtnH0GAADqwsljHRBMjA+sb731ljZs2KB169ZVaf20tDRNnz69wvKlS5cqIqL8RQNGt5LmbnHog+8OyJHzkwYlMXMAAKB+y8+n1Q3Bx2YZPL/T3r171atXLy1btszfu3rppZeqR48emjNnTqXbVDbCmpycrKysLEVHR1dY/7U1GZqxZKvsNunV1J66uE3cOfksAADUBY/Ho/j4eOXk5FR63APqI6MD66JFi3TNNdfI4XD4l5WWlspms8lut6uoqKjcc5XxeDxyu92n/MW1LEsT3v1O767fp8YRTv3z3gFKjuXyrQCA+ulMxz2gPjL6pKvBgwdr48aNSk9P99969eql0aNHKz09/YxhtSpsNpseH9FF3Vu4dSzfq7s4CQsAAMAoRgfWRo0aqUuXLuVukZGRiouLU5cuXWrtfcKcDs27pafio0K1JdOjie99x5WwAAAADGF0YK1Lie5w/XV0T4XYbfrg2/16+T8/BLokAAAAyPAe1tpQ3V6e/1u9R1MWb5bdJj0zqoeaNHIpJT5Sie7wOqgWAICzQw8rgpHx01rVtZsvaqWNP+Xona/36YG30iVJdpuUNrKrRvVuGdjiAAAAGiBaAn7BZrPpnkvbllvms6TJ729SZk5BgKoCAABouAisldhfSTAttSztyWIyZgAAgLpGYK1ESnyk7JVcpXV9xjFmDwAAAKhjBNZKJLrDlTayqxy28qn1L//eprFvpyuvqCRAlQEAADQ8zBJwGpk5BdqTla9WceFa8t0Bzfx4q0p9lto2jdILoy9Uu2aNzlHVAADUDLMEIBgRWKth3Z6juvfNDTroKVK406EnR3bRNRe0qKVKAQA4ewRWBCNaAqqhd+tYLbn/VxrQNl4F3lKNe/tbTXp/owq9XMoVAADgXCGwVlN8lEt/v72PHhjcTjabtOCrDF37wir9eCQv0KUBAAAEJQJrDTjsNo27rL3+flsfxUaGavN+j37z3Bf69+YDgS4NAAAg6BBYz8Il7Ztoyf0D1LNVY+UWluh//2+9nljyvbylvkCXBgAAEDQIrGcp0R2ut+66SHcMSJEkvfyf3brxpTU6kFOozJwCrdqVxRWyAAAAzgKzBNSijzdlasI/vlNuUYkiQx3K95bKsiS7TUob2VWjerc8p+8PAACzBCAYMcJai4Z1SdQH9w1Qu6ZRyisuC6uS5LOkye9vYqQVAACgBgistax1fKQeuapjheWllqXF6fu5tCsAAEA1EVjPgQ4JjWS3VVw+86OtuuyZlXpj7Y8qKGbuVgAAgKogsJ4Die5wpY3sKoetLLXabVL/tnGKcoVo56HjemThJl2UtlxPfbyVNgEAAIAz4KSrcygzp0B7svLVOj5Cie5w5RZ69Y+v92n+qj3KOJovqWxO1yu7Juq2/q11YcvGdVofACD4cNIVghGBNQBKfZaWbzmov325W2t+OOpf3iM5RrcPSNEVXRKUdbxIu7PylBIfqUR3eACrBQDUJyYe94CzRWANsO/3e/Tql7u1OH2/ik9ccCA6LES5hSWyxJRYAIDqMf24B9QEgdUQh3OL9ObaDM1ftUfH8ovLPWeTNO/mC/Xrjs3kdNB2DAA4tfpy3AOqg8BqmBXbDyn1b+sqfS7KFaJ+beL0q3bx+lW7JmodFyGbrZLpCAAADVZ9O+4BVRES6AJQXvtmZVNi+X72vxE2lbUJ5BSWaNn3B7Xs+4OSpOYx4f7w2r9tnGIiQiWVnexF/ysAAAgWjLAa6O11GZr8/iaVWpYcNpueHNlF1/VM1ub9Hv1n52H9Z3uW1v94zN/zKkk2m9StuVuxkaH6fPthLgkLAA1UfTzuAWdCYDXUL6fE+qX84hJ9tfuo/rMjS1/syNK2g7mVvo5N0qPDO+vS9k3UMpYWAgAIdvX1uAecDoE1SBz0FGr+l3v0wopdp1wnJsKprs3d6t4iRt1auNU9OUbNosMqrEdLAQDUXw3luIeGhcAaRDJzCtR/5qcV+l/PT4zWrkPHy7UQnNQs2qVuLWLUvYVb3VrEaNfh45rx4ffy0VIAAPVSQzruoeEgsAaZyvpfR/VuqeISn7YdyNW3+7L17d5sfbcvRzsO5ZYLt5Wx26RPH7xUreMj6+YDAADOSkM77qFhILAGoTP1v56UX1yiTT959N2+bH27L0df7T6ig56iCuvZbVLnJLd6JMeoe3KMeiTH6Lz4SNnt9MMCgGka4nEPwY/ACr/KWgpOpVFYiLq3iCkXYps0ctH/CgABxnEPwYjAinJ+2VLwxDVd1L9tvL7dl630jGx9uy9bG3/KUaG3Yj9sTLhT2QVeSWXTbM0Y3kU3X9Sqrj8CADRoHPcQjAisqOBMLQXeUp+2H8xV+t6yftj0vdnafvB4pa91fkIj9W4dqwtaxuiClo25OhcAnGMc9xCMCKyoFZ9uPajb5399xvViIpzqkRyjC5Ib64KWZe0E7nCn/3laCgDg7HDcQzDi0qyoFR0ToytcUtZukx4b3kV7svL0zd6yVoLsfK8+33ZYn2877F+vTZNIXdCysUp9Pi1K389VugAAQDmMsKLWnGpKrZOKS3zakulR+t5sfZNxTN/szdaPR/JP+5r928QppUnZaGtSTFjZT3e4mrldcoU4yq3L6CwAcNxDcCKwolZVdUqtk44cL9K3+7L1wbf7tfCb/dV6r/go14kQG6bcwhKt3nVElspO+Jp8RUf9fkAKU28BaHA47iEYEVhhhMqm1LLbpAlDz1d+cYn2ZxcqM6dAmTmF2p9doKKSirMU/JLTYVPzmHA1bxxe9jMmwn+/ReNwJbjD5HTY/e/P6CyAYMBxD8GIHlYYIdEdrrSRXU/bUnCSZVk6lu/V/uyyAPvlzizNX7WnwnreUkt7juRrzynaDuw2qVl0mEIddv14tGwdm6RbLmql/+nVQgnRYYqLcslRhVFaAi8AAOcOI6wwSnVbCk5uU9no7D/u7idvqaWfjhXop+yC//48cSuuwiitw25T00YuNYsOU0J0mBLcYWX33S4lRJeN0n6xI0vT/rlJPk4WA2AAjnsIRoywwiiJ7vBqj1CeanS2Z6vYU27j81nKyivSx5sOaOrizRWebxzhVE6BV6U+S5k5hcrMKaxSLT5Levj9jYqPcql/23iFOR1n3ggAAJwWI6wIGrU1Ouuw2fTFw4PUJMqlrOPFOuAp1IGcQh3IKdABT5EOnnh80FOon07TT+uw29SuaZQ6JUWrc5JbnZOi1SkpWtFhznLr0U4AoDZx3EMwIrCiwTvTdFynsz87XwOe+qxc4JXKX6b2l1rFRajziRCbdbxIf1+156zaCc4m8BKWgeDDcQ/BiMAKqGajsydVFniv75WsA55Cbf7Jo837Pdq0P0ff7/fop+yCM75emyaRahTmVJjTrjCnQ2Ehjv/edzrkctpPLHNoS2aOPvg2s2w6L0mpF7fSZZ0S5AqxyxVStq7/foj9xGOHHHab3l6XoUnvb6xxWD7bsBvIsExQRzDjuIdgRGAFakFVA++xvGJ9n+nR5v05+nzbYa3adaQOq/yvELtUWSfDgLZxatIoTI3CQk7cnOV+Rp+4/9m2Q3rqo63+sPvY8C4acUFzlZT65C21VOLzqaTUUonP8i8r9Vnynli+7PsD+n9f7JZllc2bO3FoB93Yp6WiXCEKOTHV2OmcTeAMdFAHzjWOewhGBFYgQE41u8Ezo3ooIjREhd7SsluJT4XFJ++XqtDrU6G3VBlH8/WfHVkVXje5cbgcdpuKSnxlN2+pikp8Kvll34Khwp0ORf08MLvK7ke5yh7/eDRPn2455B9VvqJrgjonuVVSaslb6vOHYu/J8Fxa9tmLS306XujViu3l95lN0pVdExQTESqno2xEOjTErlBH2U/niZ+hIXZt+PGY3l6313+BivGXtdcNvVsqOjykwpXXKkPYRV3guIdgRGAFAuhs+mdPd8JYZWGopNSn4lKfirw+ZRzN0zV/XVUhLE8cer7sdim3sES5hSXyFHpP3Pf6Hx/NK1ZeUelpawux2xTisMlptyvEYVOIwy6n3SaHw6aSEkuZnqrNulCfuELsig53yh3uVHRYiKLDnYoOO/E4PES7D+fpo00H/GH33kFtdV3PZLnDy0avq3JVtobYr1xf6w4kjnsIRgRWIMBqu3+2qoG3ptueamT40wcvVYsTo7s226nD16mC9mcTBqqRy6njRf8NyscLS5Rb5PUH6K2Zufrgu4qX8L20fbySGkfIaT8Rjh12OR02hZwIzKGOsp95xaV6+t/b9PN/9Gw26e6B58kV4lBxia/sVuqTt7RshPrkskO5hUrfm1OVXVttNpvUyBUid0RZwP357WQI3nEwV4u+2e8PvPf/up1GXthcka6y0WdXiP2U+/1s2yCkwITlQNddX3HcQzAisAL13NkE3ppuezZB+Wy2r+6ocl2898qJlyoqzClPgVeeQq88BWWBO6fAe2JZibYd8Ojfmw9WeM1Qh13FpWe+gEVVOOw2RYQ6FBkaokiXQ5GuEEWGhshhl77YWb5X2ibp1v6t1TgitGwE3P7fcO+wl42MO06MkofY7VrzwxG9vvZHf8/xPQPb6KpuSQoNKd9C4XLaT/zPwX/7kH8ZOqde3UmDOjTV0bxiHcsv1rE8r47lF5947NWxvGIdzS/W4dxC7c6qeJW6Nk0iFR/lUkyEUzHhoYqJPPEzwqmYcKfcEU41jih7/Mn3h87qoh719cRAjnsIRgRWADVyNkH5bLY/27AciPc+XdCOjQyVp6BEOQX/Dbk5v7jtOJSrldsr9iu7QuynnAc4kOw2+ft/cwtLAl1OOZ0So9U40qlwZ1mwjwh1KCI0RJGhDoWfCPvhTofS92brza8yZJ0Iu5Ou7Kjf9WtVpV7lk2oaOt/6KkOTF9Z8ZHn73oPq0DKB4x6CCoEVQL1ztmE5EO99rvqVmzUKU763VPlFJTpeVKL84lIdLypRXlGJ8opLtf9YgZ76eGv5NghJ11zQXC6nQ6U/n9Gh3H1LpT6fjhwv1tYDuRVqcoeXXQCjuMSnopLSCnMRn44rxK74KJcaR5aNhsZGhqpxROiJ+041jgyVz7L0wFvpsio5KdFusym7wKuc/GJl53t1LN+rnIKT94uVU1DWa30uzjMMdzrkDncqJqKsXSPmZ+0bMSdbOiJClb43W69+eWImDEn/07OFOiVFn2hzKfH3hh8/cf94Ydl/v+yCynvEW8aGKzbSVaFP2n2iVzo6PETucKfW/nBUz//7O/34zPUc9xBUCKwAUEfqW7/yyZqr0oZx8qS+4pL/9v7uO1agm/7fmgqh88uHf12lz1/bF/Ww26S0a7rK5XQov7hU+cVlAT+vuEQFxaXKKypVgbdE+44V6Lt956ZfuS74ivK1dw6BFcGFwAoA9UQg+pWlwIXlQNVdeUiXPnrgEoU5HcouKBvFzc4v37qRfWJ098cj+ZWOSl90Xqxax0WemKatbHaIqLCQE1O3ORUVVjad3U0vr6kQtJ+78QI5HXZ5Ckt+1h/t9beTeAq9yswu0N5jBQRWBCUCKwDgjAIVls+WaS0c53Jk+eT7lhQSWBF8CKwAAFQiUKPSZ/Peb6/L0MMLvtKeZ67juIegQmAFAOAcCNTIMrMEIBiFBLoAAACCUaI7PCAXK0hoIBdIQMNiP/MqAAAAQOAQWAEAAGA0AisAAACMRmAFAACA0QisAAAAMBqBFQAAAEYjsAIAAMBoRgfWtLQ09e7dW40aNVLTpk01YsQIbdu2LdBlAQAAoA4ZHVhXrFihMWPGaM2aNVq2bJm8Xq8uv/xy5eXlBbo0AAAA1JF6dWnWw4cPq2nTplqxYoUuueSSKm3DpVkBAA0Jxz0Eo3p1adacnBxJUmxs7CnXKSoqUlFRkf+xx+ORJHm9Xnm93nNbIAAAAcaxDsGo3gRWn8+nsWPHqn///urSpcsp10tLS9P06dMrLF+6dKkiIiLOZYkAAARcfn5+oEsAal29aQn4wx/+oI8++khffPGFWrRoccr1KhthTU5OVlZWFn8aAQAEPY/Ho/j4eFoCEFTqxQjrvffeqw8//FArV648bViVJJfLJZfLVWG50+mU0+k8VyUCAGAEjnUIRkYHVsuydN9992nhwoX6/PPPlZKSEuiSAAAAUMeMDqxjxozRm2++qcWLF6tRo0Y6cOCAJMntdis8PDzA1QEAAKAuGN3DarPZKl3+6quv6tZbb63SazC9BwCgIeG4h2Bk9AirwVkaAAAAdcToK10BAAAABFYAAAAYjcAKAAAAoxFYAQAAYDQCKwAAAIxGYAUAAIDRCKwAAAAwGoEVAAAARiOwAgAAwGgEVgAAABiNwAoAAACjEVgBAABgNAIrAAAAjEZgBQAAgNEIrAAAADAagRUAAABGI7ACAADAaARWAAAAGI3ACgAAAKMRWAEAAGA0AisAAACMRmAFAACA0QisAAAAMBqBFQAAAEYjsAIAAMBoBFYAAAAYjcAKAAAAoxFYAQAAYDQCKwAAAIxGYAUAAIDRCKwAAAAwGoEVAAAARiOwAgAAwGgEVgAAABiNwAoAAACjEVgBAABgNAIrAAAAjEZgBQAAgNEIrAAAADAagRUAAABGI7ACAADAaARWAAAAGI3ACgAAAKMRWAEAAGA0AisAAACMRmAFAACA0QisAAAAMBqBFQAAAEYjsAIAAMBoBFYAAAAYjcAKAAAAoxFYAQAAYDQCKwAAAIxGYAUAAIDRCKwAAAAwGoEVAAAARiOwAgAAwGgEVgAAABiNwAoAAACjEVgBAABgNAIrAAAAjEZgBQAAgNEIrAAAADBavQisc+fOVevWrRUWFqa+ffvqq6++CnRJAAAAqCPGB9a3335b48eP17Rp07RhwwZ1795dQ4cO1aFDhwJdGgAAAOqA8YF19uzZuvPOO3XbbbepU6dOmjdvniIiIvS3v/0t0KUBAACgDoQEuoDTKS4u1vr16zVp0iT/MrvdriFDhmj16tWVblNUVKSioiL/45ycHEnS0aNH5fV6z23BAAAEWG5uriTJsqwAVwLUHqMDa1ZWlkpLS9WsWbNyy5s1a6atW7dWuk1aWpqmT59eYXlKSso5qREAABPl5ubK7XYHugygVhgdWGti0qRJGj9+vP9xdna2WrVqpYyMDH5xq8jj8Sg5OVl79+5VdHR0oMupF9hn1cc+qz72WfU1xH1mWZZyc3OVlJQU6FKAWmN0YI2Pj5fD4dDBgwfLLT948KASEhIq3cblcsnlclVY7na7G8w/VrUlOjqafVZN7LPqY59VH/us+hraPmOABsHG6JOuQkND1bNnTy1fvty/zOfzafny5erXr18AKwMAAEBdMXqEVZLGjx+v1NRU9erVS3369NGcOXOUl5en2267LdClAQAAoA4YH1hHjRqlw4cPa+rUqTpw4IB69Oihjz/+uMKJWKficrk0bdq0StsEUDn2WfWxz6qPfVZ97LPqY58BwcFmMe8FAAAADGZ0DysAAABAYAUAAIDRCKwAAAAwGoEVAAAARgvqwDp37ly1bt1aYWFh6tu3r7766qtAl2S0Rx99VDabrdzt/PPPD3RZRlm5cqWuvvpqJSUlyWazadGiReWetyxLU6dOVWJiosLDwzVkyBDt2LEjMMUa4kz77NZbb63wvRs2bFhgijVAWlqaevfurUaNGqlp06YaMWKEtm3bVm6dwsJCjRkzRnFxcYqKitK1115b4QIrDUlV9tmll15a4Xt29913B6hiANUVtIH17bff1vjx4zVt2jRt2LBB3bt319ChQ3Xo0KFAl2a0zp07KzMz03/74osvAl2SUfLy8tS9e3fNnTu30udnzZqlZ599VvPmzdPatWsVGRmpoUOHqrCwsI4rNceZ9pkkDRs2rNz3bsGCBXVYoVlWrFihMWPGaM2aNVq2bJm8Xq8uv/xy5eXl+dcZN26cPvjgA/3jH//QihUrtH//fo0cOTKAVQdWVfaZJN15553lvmezZs0KUMUAqs0KUn369LHGjBnjf1xaWmolJSVZaWlpAazKbNOmTbO6d+8e6DLqDUnWwoUL/Y99Pp+VkJBg/fnPf/Yvy87Otlwul7VgwYIAVGieX+4zy7Ks1NRUa/jw4QGppz44dOiQJclasWKFZVll3ymn02n94x//8K+zZcsWS5K1evXqQJVplF/uM8uyrIEDB1oPPPBA4IoCcFaCcoS1uLhY69ev15AhQ/zL7Ha7hgwZotWrVwewMvPt2LFDSUlJOu+88zR69GhlZGQEuqR6Y/fu3Tpw4EC5753b7Vbfvn353p3B559/rqZNm6pDhw76wx/+oCNHjgS6JGPk5ORIkmJjYyVJ69evl9frLfc9O//889WyZUu+Zyf8cp+d9MYbbyg+Pl5dunTRpEmTlJ+fH4jyANSA8Ve6qomsrCyVlpZWuBpWs2bNtHXr1gBVZb6+fftq/vz56tChgzIzMzV9+nT96le/0qZNm9SoUaNAl2e8AwcOSFKl37uTz6GiYcOGaeTIkUpJSdGuXbs0efJkXXHFFVq9erUcDkegywson8+nsWPHqn///urSpYuksu9ZaGioYmJiyq3L96xMZftMkm666Sa1atVKSUlJ+u677/THP/5R27Zt0/vvvx/AagFUVVAGVtTMFVdc4b/frVs39e3bV61atdI777yj3//+9wGsDMHshhtu8N/v2rWrunXrpjZt2ujzzz/X4MGDA1hZ4I0ZM0abNm2il7waTrXP7rrrLv/9rl27KjExUYMHD9auXbvUpk2bui4TQDUFZUtAfHy8HA5HhbNmDx48qISEhABVVf/ExMSoffv22rlzZ6BLqRdOfrf43p2d8847T/Hx8Q3+e3fvvffqww8/1GeffaYWLVr4lyckJKi4uFjZ2dnl1ud7dup9Vpm+fftKUoP/ngH1RVAG1tDQUPXs2VPLly/3L/P5fFq+fLn69esXwMrql+PHj2vXrl1KTEwMdCn1QkpKihISEsp97zwej9auXcv3rhr27dunI0eONNjvnWVZuvfee7Vw4UJ9+umnSklJKfd8z5495XQ6y33Ptm3bpoyMjAb7PTvTPqtMenq6JDXY7xlQ3wRtS8D48eOVmpqqXr16qU+fPpozZ47y8vJ02223Bbo0Yz300EO6+uqr1apVK+3fv1/Tpk2Tw+HQjTfeGOjSjHH8+PFyIzK7d+9Wenq6YmNj1bJlS40dO1aPP/642rVrp5SUFE2ZMkVJSUkaMWJE4IoOsNPts9jYWE2fPl3XXnutEhIStGvXLk2cOFFt27bV0KFDA1h14IwZM0ZvvvmmFi9erEaNGvn7Ut1ut8LDw+V2u/X73/9e48ePV2xsrKKjo3XfffepX79+uuiiiwJcfWCcaZ/t2rVLb775pq688krFxcXpu+++07hx43TJJZeoW7duAa4eQJUEepqCc+m5556zWrZsaYWGhlp9+vSx1qxZE+iSjDZq1CgrMTHRCg0NtZo3b26NGjXK2rlzZ6DLMspnn31mSapwS01NtSyrbGqrKVOmWM2aNbNcLpc1ePBga9u2bYEtOsBOt8/y8/Otyy+/3GrSpInldDqtVq1aWXfeead14MCBQJcdMJXtK0nWq6++6l+noKDAuueee6zGjRtbERER1jXXXGNlZmYGrugAO9M+y8jIsC655BIrNjbWcrlcVtu2ba0JEyZYOTk5gS0cQJXZLMuy6jIgAwAAANURlD2sAAAACB4EVgAAABiNwAoAAACjEVgBAABgNAIrAAAAjEZgBQAAgNEIrAAAADAagRWAUT7//HPZbDZlZ2cHuhQAgCEIrAAAADAagRUAAABGI7ACKMfn8yktLU0pKSkKDw9X9+7d9e6770r675/rlyxZom7duiksLEwXXXSRNm3aVO413nvvPXXu3Fkul0utW7fW008/Xe75oqIi/fGPf1RycrJcLpfatm2rV155pdw669evV69evRQREaGLL75Y27ZtO7cfHABgLAIrgHLS0tL02muvad68edq8ebPGjRunm2++WStWrPCvM2HCBD399NNat26dmjRpoquvvlper1dSWdC8/vrrdcMNN2jjxo169NFHNWXKFM2fP9+//e9+9zstWLBAzz77rLZs2aIXX3xRUVFR5ep45JFH9PTTT+vrr79WSEiIbr/99jr5/AAA89gsy7ICXQQAMxQVFSk2NlaffPKJ+vXr519+xx13KD8/X3fddZcGDRqkt956S6NGjZIkHT16VC1atND8+fN1/fXXa/To0Tp8+LCWLl3q337ixIlasmSJNm/erO3bt6tDhw5atmyZhgwZUqGGzz//XIMGDdInn3yiwYMHS5L+9a9/6aqrrlJBQYHCwsLO8V4AAJiGEVYAfjt37lR+fr4uu+wyRUVF+W+vvfaadu3a5V/v52E2NjZWHTp00JYtWyRJW7ZsUf/+/cu9bv/+/bVjxw6VlpYqPT1dDodDAwcOPG0t3bp1899PTEyUJB06dOisPyMAoP4JCXQBAMxx/PhxSdKSJUvUvHnzcs+5XK5yobWmwsPDq7Se0+n037fZbJLK+msBAA0PI6wA/Dp16iSXy6WMjAy1bdu23C05Odm/3po1a/z3jx07pu3bt6tjx46SpI4dO+rLL78s97pffvml2rdvL4fDoa5du8rn85XriQUA4HQYYQXg16hRIz300EMaN26cfD6fBgwYoJycHH355ZeKjo5Wq1atJEmPPfaY4uLi1KxZMz3yyCOKj4/XiBEjJEkPPvigevfurRkzZmjUqFFavXq1nn/+ef31r3+VJLVu3Vqpqam6/fbb9eyzz6p79+768ccfdejQIV1//fWB+ugAAIMRWAGUM2PGDDVp0kRpaWn64YcfFBMTowsvvFCTJ0/2/0l+5syZeuCBB7Rjxw716NFDH3zwgUJDQyVJF154od555x1NnTpVM2bMUGJioh577DHdeuut/vd44YUXNHnyZN1zzz06cuSIWrZsqcmTJwfi4wIA6gFmCQBQZSfP4D927JhiYmICXQ4AoIGghxUAAABGI7ACAADAaLQEAAAAwGiMsAIAAMBoBFYAAAAYjcAKAAAAoxFYAQAAYDQCKwAAAIxGYAUAAIDRCKwAAAAwGoEVAAAARiOwAgAAwGj/H5mlPsKfRujAAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "fig, ax = plt.subplots()\n", + "\n", + "val = architectures_TM[key]\n", + "y_avg = []\n", + "for y in val['epochs_y']:\n", + " y_avg.append(np.mean(y))\n", + "_ - np.round(y_avg[-1], 2)\n", + "ax.plot(np.arange(len(val['epochs_x']))[:max_epochs], y_avg[:max_epochs], label=f'{key}', marker='.')\n", + "\n", + "ax.set_ylim(0, 10)\n", + "ax.set_ylabel('loss')\n", + "ax.set_xlim(0, max_epochs-1)\n", + "ax.set_xlabel('epoch')\n", + "\n", + "pos = ax.get_position()\n", + "ax.set_position([pos.x0, pos.y0, pos.width * 0.9, pos.height])\n", + "ax.legend(loc='center right', bbox_to_anchor=(1.4, 0.5), framealpha=0)\n", + "ax.grid(axis='y')\n", + "\n", + "plt.show()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.0" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/model_training.py b/tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/model_training.py new file mode 100644 index 00000000..84c4144f --- /dev/null +++ b/tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/model_training.py @@ -0,0 +1,92 @@ +import torch, random, sys + +import tonic +from torch.utils.data import DataLoader +from torch.nn import CrossEntropyLoss +from torch.optim import Adam + +from sinabs.activation.surrogate_gradient_fn import PeriodicExponential + +import matplotlib.pyplot as plt +import numpy as np + +sys.path.append('../../utils') +sys.path.append('../models') + +from train_test_fn import training_loop, load_dataset, split_train_validation, load_architecture + +if torch.cuda.is_available(): + device = torch.device('cuda:0') + print('device: ', torch.cuda.get_device_name(0)) +else: + device = torch.device('cpu') + +rand_seed = 1 + +achitecture = sys.argv[1] + +torch.backends.cudnn.enabled = False +torch.backends.cudnn.deterministic = True +random.seed(rand_seed) +torch.manual_seed(rand_seed) +torch.cuda.manual_seed(rand_seed) +np.random.seed(rand_seed) + +batch_size = 8 +num_workers = 4 +epochs = 100 +lr = 5e-5 + +spk_thr = 2.0 +v_min = -0.313 + +grad_scale = 1.534 +grad_width = 0.759 + +validation_ratio = 0.2 +n_time_steps = 50 + +snn_train_dataset, snn_test_dataset, sensor_size = load_dataset('DVSGESTURE', n_time_steps) + +train_dataset, validation_dataset = split_train_validation(validation_ratio, snn_train_dataset, rand_seed) + +disk_cache_train = tonic.DiskCachedDataset( + dataset=train_dataset, + cache_path='./cached_train' +) +snn_train_dataloader = DataLoader(disk_cache_train, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True) + +disk_cache_validation = tonic.DiskCachedDataset( + dataset=validation_dataset, + cache_path='./cached_validation' +) +snn_validation_dataloader = DataLoader(disk_cache_validation, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True) + +disk_cache_test = tonic.DiskCachedDataset( + dataset=snn_test_dataset, + cache_path='./cached_test' +) +snn_test_dataloader = DataLoader(disk_cache_test, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=False) + +snn = load_architecture(achitecture, sensor_size, 11, batch_size, PeriodicExponential(grad_scale=grad_scale, grad_width=grad_width), v_min, spk_thr).to(device) +snn.init_weights() + +optimizer = Adam(snn.parameters(), lr=lr, betas=(0.9, 0.999), eps=1e-8) +loss_fn = CrossEntropyLoss() + +epochs_x, epochs_y, epochs_acc = training_loop( + device, + n_time_steps, + batch_size, + sensor_size, + snn_train_dataloader, + snn, + loss_fn, + optimizer, + epochs, + snn_validation_dataloader) + +with open(f'./architectures_results/{achitecture}-Training_Validation-TM.npy', 'wb') as f: + np.save(f, np.array(epochs_x)) + np.save(f, np.array(epochs_y)) + np.save(f, np.array(epochs_acc)) \ No newline at end of file diff --git a/tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/single_training.ipynb b/tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/single_training.ipynb index f5e3cfe9..33bac3a0 100644 --- a/tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/single_training.ipynb +++ b/tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/single_training.ipynb @@ -61,7 +61,7 @@ "metadata": {}, "outputs": [], "source": [ - "achitecture = 'ResSCNN1'" + "achitecture = 'ResSCNN4'" ] }, { @@ -86,7 +86,7 @@ "source": [ "batch_size = 8\n", "num_workers = 4\n", - "epochs = 30\n", + "epochs = 125\n", "lr = 5e-5\n", "\n", "spk_thr = 2.0\n", @@ -202,20 +202,10 @@ "cell_type": "code", "execution_count": 12, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "recaling factor: 3.2 (computed using 1 kernels and lambda 0.8)\n", - "recaling factor: 8.0 (computed using 2 kernels and lambda 0.8)\n", - "recaling factor: 3.2 (computed using 1 kernels and lambda 0.8)\n" - ] - } - ], + "outputs": [], "source": [ - "lambda_ = 0.8\n", - "snn.rescale_conv_weights(rescale_method_1, lambda_)" + "# lambda_ = 0.8\n", + "# snn.rescale_conv_weights(rescale_method_1, lambda_)" ] }, { @@ -243,7 +233,3332 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "3eb61008049e42e4bc72738b0d76ad4b", + "model_id": "7f01930878c74d7eb18bb72c0921d870", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/107 [00:00" ] @@ -1343,7 +4658,7 @@ "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAG2CAYAAACZEEfAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABUOklEQVR4nO3deVhUZf8G8HtYZlhkEZFllE1FccXdUNMSAs3MLZeysvTNTDD3BUstLVHTLPfs55Jlbr1qLoUpbmmIouKKCIqispnAIDvC8/vDmNeRbQYGgdP9ua65inPO9zzPHI7DPc95zoxMCCFAREREJFEG1d0BIiIioqrEsENERESSxrBDREREksawQ0RERJLGsENERESSxrBDREREksawQ0RERJLGsENERESSxrBDREREksawQ0RERJJWrWHnxIkT6NevH5RKJWQyGfbs2aOxXgiBOXPmwNHREaampvDx8UF0dLTGNikpKRgxYgQsLS1hbW2N0aNHIyMj4zk+CyIiIqrJqjXsZGZmwtPTE6tWrSpx/eLFi7F8+XKsXbsWYWFhMDc3h5+fH3JyctTbjBgxAlevXsWhQ4ewf/9+nDhxAmPGjHleT4GIiIhqOFlN+SJQmUyG3bt3Y8CAAQCejOoolUpMmTIFU6dOBQCoVCrY29tj06ZNGD58OCIjI9GiRQucPXsWHTt2BAAEBwfj1Vdfxb1796BUKqvr6RAREVENYVTdHShNbGwsEhMT4ePjo15mZWWFLl26IDQ0FMOHD0doaCisra3VQQcAfHx8YGBggLCwMAwcOLDEfefm5iI3N1f9c2FhIVJSUlCvXj3IZLKqe1JERESkN0IIPHr0CEqlEgYGpV+sqrFhJzExEQBgb2+vsdze3l69LjExEXZ2dhrrjYyMYGNjo96mJEFBQfj888/13GMiIiKqDnfv3kXDhg1LXV9jw05VCgwMxOTJk9U/q1QqODs74+7du7C0tKzGnhEREZG20tPT4eTkBAsLizK3q7Fhx8HBAQCQlJQER0dH9fKkpCS0bdtWvU1ycrJG3ePHj5GSkqKuL4lCoYBCoSi23NLSkmGHiIiolilvCkqN/ZwdNzc3ODg4ICQkRL0sPT0dYWFh8PLyAgB4eXkhLS0N586dU29z5MgRFBYWokuXLs+9z0RERFTzVOvITkZGBmJiYtQ/x8bGIiIiAjY2NnB2dsbEiRPxxRdfwN3dHW5ubpg9ezaUSqX6jq3mzZujd+/e+OCDD7B27Vrk5+cjICAAw4cP551YREREBKCaw054eDhefvll9c9F82hGjhyJTZs2Yfr06cjMzMSYMWOQlpaG7t27Izg4GCYmJuqaLVu2ICAgAN7e3jAwMMDgwYOxfPny5/5ciIiIqGaqMZ+zU53S09NhZWUFlUrFOTtERES1hLZ/v2vsnB0iIiIifWDYISIiIklj2CEiIiJJY9ghIiIiSWPYISIiIklj2CEiIiJJY9ghIiIiSWPYISIiIklj2CEiIiJJY9ghIiIiSWPYISIiIklj2CEiIiJJY9ghIiIiSWPYISIiIklj2CEiIiJJY9ghIiIiSWPYISIiIklj2CEiIiJJY9ghIiIiSWPYISIiIklj2CEiIiJJY9ghIiIiSWPYISIiIklj2CEiIiJJY9ghIiIiSWPYISIiIklj2CEiIiJJY9ghIiIiSWPYISIiIklj2CEiIiJJY9ghIiIiSWPYISIiIklj2CEiIiJJY9ghIiIiSWPYISIiIklj2CEiIiJJY9ghIiIiSWPYISIiIklj2CEiIiJJY9ghIiIiSWPYISIiIklj2CEiIiJJY9ghIiIiSWPYISIiIklj2CEiIiJJY9ghIiIiSWPYISIiIklj2CEiIiJJY9ghIiIiSWPYISIiIklj2CEiIiJJY9ghIiIiSWPYISIiIklj2CEiIiJJY9ghIiIiSWPYISIiIklj2CEiIiJJY9ghIiIiSWPYISIiIklj2CEiIiJJY9ghIiIiSWPYISIiIklj2CEiIiJJq9Fhp6CgALNnz4abmxtMTU3RuHFjzJ8/H0II9TZCCMyZMweOjo4wNTWFj48PoqOjq7HXREREVJPU6LCzaNEirFmzBitXrkRkZCQWLVqExYsXY8WKFeptFi9ejOXLl2Pt2rUICwuDubk5/Pz8kJOTU409JyIioppCJp4eJqlhXnvtNdjb22P9+vXqZYMHD4apqSl++uknCCGgVCoxZcoUTJ06FQCgUqlgb2+PTZs2Yfjw4Vq1k56eDisrK6hUKlhaWlbJcyEiIiL90vbvd40e2enatStCQkJw48YNAMDFixdx8uRJ9OnTBwAQGxuLxMRE+Pj4qGusrKzQpUsXhIaGlrrf3NxcpKenazyIiIhImoyquwNlmTlzJtLT0+Hh4QFDQ0MUFBTgyy+/xIgRIwAAiYmJAAB7e3uNOnt7e/W6kgQFBeHzzz+vuo4TERFRjVGjR3Z27NiBLVu24Oeff8b58+fxww8/YMmSJfjhhx8qtd/AwECoVCr14+7du3rqMREREdU0NXpkZ9q0aZg5c6Z67k3r1q1x584dBAUFYeTIkXBwcAAAJCUlwdHRUV2XlJSEtm3blrpfhUIBhUJRpX0nIiKimqFGj+xkZWXBwECzi4aGhigsLAQAuLm5wcHBASEhIer16enpCAsLg5eX13PtKxEREdVMNXpkp1+/fvjyyy/h7OyMli1b4sKFC/j6668xatQoAIBMJsPEiRPxxRdfwN3dHW5ubpg9ezaUSiUGDBhQvZ0nIiKiGqFGh50VK1Zg9uzZGDduHJKTk6FUKvHhhx9izpw56m2mT5+OzMxMjBkzBmlpaejevTuCg4NhYmJSjT0nIiKimqJGf87O88LP2SEiIqp9JPE5O0RERESVxbBDREREksawQ0RERJLGsENERESSxrBDREREksawQ0RERJLGsENERESSxrBDREREksawQ0RERJLGsENERESSxrBDREREksawQ0RERJLGsENERESSxrBDREREksawQ0RERJLGsENERESSxrBDREREksawQ0RERJLGsENERESSxrBDREREksawQ0RERJLGsENERESSxrBDREREksawQ0RERJLGsENERESSxrBDREREksawQ0RERJLGsENERESSxrBDREREksawQ0RERJLGsENERESSxrBDREREksawQ0RERJLGsENERESSxrBDREREksawQ0RERJLGsENERESSxrBDREREksawQ0RERJLGsENERESSxrBDREREksawQ0RERJLGsENERESSxrBDREREksawQ0RERJLGsENERESSxrBDREREksawQ0RERJLGsENERESSxrBDREREksawQ0RERJLGsENERESSxrBDREREksawQ0RERJLGsENERESSxrBDREREksawQ0RERJLGsENERESSpnPYOXr0aFX0g4iIiKhK6Bx2evfujcaNG+OLL77A3bt3q6JPRERERHqjc9i5f/8+AgIC8Msvv6BRo0bw8/PDjh07kJeXVxX9IyIiIqoUncOOra0tJk2ahIiICISFhaFp06YYN24clEolPv74Y1y8eLEq+klERERUIZWaoNy+fXsEBgYiICAAGRkZ2LBhAzp06IAXX3wRV69e1VcfiYiIiCqsQmEnPz8fv/zyC1599VW4uLjg4MGDWLlyJZKSkhATEwMXFxcMGTJELx28f/8+3n77bdSrVw+mpqZo3bo1wsPD1euFEJgzZw4cHR1hamoKHx8fREdH66VtIiIiqv10Djvjx4+Ho6MjPvzwQzRt2hQXLlxAaGgo/vOf/8Dc3Byurq5YsmQJrl+/XunOpaamolu3bjA2Nsbvv/+Oa9euYenSpahbt656m8WLF2P58uVYu3YtwsLCYG5uDj8/P+Tk5FS6fSIiIqr9jHQtuHbtGlasWIFBgwZBoVCUuI2tra1eblFftGgRnJycsHHjRvUyNzc39f8LIfDNN9/g008/Rf/+/QEAmzdvhr29Pfbs2YPhw4dXug9ERERUu+k8shMSEoI333yz1KADAEZGRujZs2elOgYAe/fuRceOHTFkyBDY2dmhXbt2+P7779XrY2NjkZiYCB8fH/UyKysrdOnSBaGhoaXuNzc3F+np6RoPIiIikiadw05QUBA2bNhQbPmGDRuwaNEivXSqyK1bt7BmzRq4u7vj4MGD+Oijj/Dxxx/jhx9+AAAkJiYCAOzt7TXq7O3t1etKEhQUBCsrK/XDyclJr/0mIiKimkPnsPPdd9/Bw8Oj2PKWLVti7dq1eulUkcLCQrRv3x4LFixAu3btMGbMGHzwwQeVbicwMBAqlUr94IcjEhERSZfOYScxMRGOjo7FltevXx8JCQl66VQRR0dHtGjRQmNZ8+bNERcXBwBwcHAAACQlJWlsk5SUpF5XEoVCAUtLS40HERERSZPOYcfJyQmnTp0qtvzUqVNQKpV66VSRbt26ISoqSmPZjRs34OLiAuDJZGUHBweEhISo16enpyMsLAxeXl567QsRERHVTjrfjfXBBx9g4sSJyM/PR69evQA8mbQ8ffp0TJkyRa+dmzRpErp27YoFCxZg6NChOHPmDNatW4d169YBAGQyGSZOnIgvvvgC7u7ucHNzw+zZs6FUKjFgwAC99oWIiIhqJ53DzrRp0/Dw4UOMGzdO/X1YJiYmmDFjBgIDA/XauU6dOmH37t0IDAzEvHnz4Obmhm+++QYjRoxQbzN9+nRkZmZizJgxSEtLQ/fu3REcHAwTExO99oWIiIhqJ5kQQlSkMCMjA5GRkTA1NYW7u3uZt6LXdOnp6bCysoJKpeL8HSIiolpC27/fOo/sFKlTpw46depU0XIiIiKi56JCYSc8PBw7duxAXFyc+lJWkV27dumlY0RERET6oPPdWNu2bUPXrl0RGRmJ3bt3Iz8/H1evXsWRI0dgZWVVFX0kIiIiqjCdw86CBQuwbNky7Nu3D3K5HN9++y2uX7+OoUOHwtnZuSr6SERERFRhOoedmzdvom/fvgAAuVyOzMxMyGQyTJo0SX1LOBEREVFNoXPYqVu3Lh49egQAaNCgAa5cuQIASEtLQ1ZWln57R0RERFRJOk9Q7tGjBw4dOoTWrVtjyJAhmDBhAo4cOYJDhw7B29u7KvpIREREVGE6h52VK1ciJycHAPDJJ5/A2NgYf/31FwYPHoxPP/1U7x0kIiIiqgydws7jx4+xf/9++Pn5AQAMDAwwc+bMKukYERERkT7oNGfHyMgIY8eOVY/sEBEREdV0Ok9Q7ty5MyIiIqqgK0RERET6p/OcnXHjxmHy5Mm4e/cuOnToAHNzc431bdq00VvniIiIiCpL5y8CNTAoPhgkk8kghIBMJkNBQYHeOve88ItAiYiIap8q+yLQ2NjYSnWMiIiI6HnSOey4uLhURT+IiIiIqoTOYWfz5s1lrn/33Xcr3BkiIiIifdN5zk7dunU1fs7Pz0dWVhbkcjnMzMyQkpKi1w4+D5yzQ0REVPto+/db51vPU1NTNR4ZGRmIiopC9+7dsXXr1kp1moiIiEjfdA47JXF3d8fChQsxYcIEfeyOiIiISG/0EnaAJ5+uHB8fr6/dEREREemFzhOU9+7dq/GzEAIJCQlYuXIlunXrpreOEREREemDzmFnwIABGj/LZDLUr18fvXr1wtKlS/XVLyIiIiK90DnsFBYWVkU/iIiIiKqE3ubsEBEREdVEOoedwYMHY9GiRcWWL168GEOGDNFLp4iIiIj0Reewc+LECbz66qvFlvfp0wcnTpzQS6eIiIiI9EXnsJORkQG5XF5subGxMdLT0/XSKSIiIiJ90TnstG7dGtu3by+2fNu2bWjRooVeOkVERESkLzrfjTV79mwMGjQIN2/eRK9evQAAISEh2Lp1K3bu3Kn3DhIRERFVhs5hp1+/ftizZw8WLFiAX375BaampmjTpg0OHz6Mnj17VkUfiYiIiCpM5289lyJ+6zkREVHtU2Xfen727FmEhYUVWx4WFobw8HBdd0dERERUpXQOO/7+/rh7926x5ffv34e/v79eOkVERESkLzqHnWvXrqF9+/bFlrdr1w7Xrl3TS6eIiIiI9EXnsKNQKJCUlFRseUJCAoyMdJ7vTERERFSldA47vr6+CAwMhEqlUi9LS0vDrFmz8Morr+i1c0RERESVpfNQzJIlS9CjRw+4uLigXbt2AICIiAjY29vjxx9/1HsHiYiIiCpD57DToEEDXLp0CVu2bMHFixdhamqK999/H2+++SaMjY2roo9EREREFVahSTbm5uYYM2aMvvtCREREpHcVnlF87do1xMXFIS8vT2P566+/XulOEREREemLzmHn1q1bGDhwIC5fvgyZTIaiD2CWyWQAgIKCAv32kIiIiKgSdL4ba8KECXBzc0NycjLMzMxw9epVnDhxAh07dsSxY8eqoItEREREFafzyE5oaCiOHDkCW1tbGBgYwMDAAN27d0dQUBA+/vhjXLhwoSr6SURERFQhOo/sFBQUwMLCAgBga2uL+Ph4AICLiwuioqL02zsiIiKiStJ5ZKdVq1a4ePEi3Nzc0KVLFyxevBhyuRzr1q1Do0aNqqKPRERERBWmc9j59NNPkZmZCQCYN28eXnvtNbz44ouoV68etm/frvcOEhEREVWGTBTdTlUJKSkpqFu3rvqOrNomPT0dVlZWUKlUsLS0rO7uEBERkRa0/futl2/utLGx0cduiIiIiPRO5wnKRERERLUJww4RERFJGsMOERERSZrOYefEiRN4/PhxseWPHz/GiRMn9NIpIiIiIn3ROey8/PLLSElJKbZcpVLh5Zdf1kuniIiIiPRF57AjhCjxFvOHDx/C3NxcL50iIiIi0hetbz0fNGgQgCffbv7ee+9BoVCo1xUUFODSpUvo2rWr/ntIREREVAlahx0rKysAT0Z2LCwsYGpqql4nl8vxwgsv4IMPPtB/D4mIiIgqQeuws3HjRgCAq6srpk6dyktWREREVCvoPGdn+vTpGnN27ty5g2+++QZ//PGHXjtGREREpA86h53+/ftj8+bNAIC0tDR07twZS5cuRf/+/bFmzRq9d5CIiIioMnQOO+fPn8eLL74IAPjll1/g4OCAO3fuYPPmzVi+fLneO0hERERUGTqHnaysLFhYWAAA/vjjDwwaNAgGBgZ44YUXcOfOHb13kIiIiKgydA47TZo0wZ49e3D37l0cPHgQvr6+AIDk5OQyv16diIiIqDroHHbmzJmDqVOnwtXVFZ07d4aXlxeAJ6M87dq103sHiYiIiCpD57DzxhtvIC4uDuHh4Th48KB6ube3N5YtW6bXzj1r4cKFkMlkmDhxonpZTk4O/P39Ua9ePdSpUweDBw9GUlJSlfaDiIiIao8Kfeu5g4MDLCwscOjQIWRnZwMAOnXqBA8PD7127mlnz57Fd999hzZt2mgsnzRpEvbt24edO3fi+PHjiI+PV3/aMxEREZHOYefhw4fw9vZG06ZN8eqrryIhIQEAMHr0aEyZMkXvHQSAjIwMjBgxAt9//z3q1q2rXq5SqbB+/Xp8/fXX6NWrFzp06ICNGzfir7/+wunTp6ukL0RERFS76Bx2Jk2aBGNjY8TFxcHMzEy9fNiwYQgODtZr54r4+/ujb9++8PHx0Vh+7tw55Ofnayz38PCAs7MzQkNDS91fbm4u0tPTNR5EREQkTVp/XUSRP/74AwcPHkTDhg01lru7u1fJrefbtm3D+fPncfbs2WLrEhMTIZfLYW1trbHc3t4eiYmJpe4zKCgIn3/+ub67SkRERDWQziM7mZmZGiM6RVJSUjS+CV0f7t69iwkTJmDLli0wMTHR234DAwOhUqnUj7t37+pt30RERFSz6Bx2XnzxRfXXRQCATCZDYWEhFi9ejJdfflmvnTt37hySk5PRvn17GBkZwcjICMePH8fy5cthZGQEe3t75OXlIS0tTaMuKSkJDg4Ope5XoVDA0tJS40FERETSpPNlrMWLF8Pb2xvh4eHIy8vD9OnTcfXqVaSkpODUqVN67Zy3tzcuX76ssez999+Hh4cHZsyYAScnJxgbGyMkJASDBw8GAERFRSEuLk79+T9ERET076Zz2GnVqhVu3LiBlStXwsLCAhkZGRg0aBD8/f3h6Oio185ZWFigVatWGsvMzc1Rr1499fLRo0dj8uTJsLGxgaWlJcaPHw8vLy+88MILeu0LERER1U46h524uDg4OTnhk08+KXGds7OzXjqmrWXLlsHAwACDBw9Gbm4u/Pz8sHr16ufaByIiIqq5ZEIIoUuBoaEhEhISYGdnp7H84cOHsLOzQ0FBgV47+Dykp6fDysoKKpWK83eIiIhqCW3/fus8QVkIAZlMVmx5RkaGXu+YIiIiItIHrS9jTZ48GcCTu69mz56tcft5QUEBwsLC0LZtW713kIiIiKgytA47Fy5cAPBkZOfy5cuQy+XqdXK5HJ6enpg6dar+e0hERERUCVqHnaNHjwJ4cuv3t99+y7ktREREVCvofDfWxo0bq6IfRERERFVC5wnKRERERLUJww4RERFJGsMOERERSRrDDhEREUkaww4RERFJGsMOERERSRrDDhEREUkaww4RERFJGsMOERERSRrDDhEREUkaww4RERFJGsMOERERSRrDDhEREUkaww4RERFJGsMOERERSRrDDhEREUkaww4RERFJGsMOERERSRrDDhEREUkaww4RERFJGsMOERERSRrDDhEREUkaww4RERFJGsMOERERSRrDDhEREUkaww4RERFJGsMOERERSRrDDhEREUkaww4RERFJGsMOERERSRrDDhEREUkaww4RERFJGsMOERERSRrDDhEREUkaww4RERFJGsMOERERSRrDDhEREUkaww4RERFJGsMOERERSRrDDhEREUkaww4RERFJGsMOERERSRrDDhEREUkaww4RERFJGsMOERERSRrDDhEREUkaww4RERFJGsMOERERSRrDDhEREUkaww4RERFJGsMOERERSRrDDhEREUkaww4RERFJGsMOERERSRrDDhEREUkaww4RERFJGsMOERERSRrDDhEREUlajQ47QUFB6NSpEywsLGBnZ4cBAwYgKipKY5ucnBz4+/ujXr16qFOnDgYPHoykpKRq6jERERHVNDU67Bw/fhz+/v44ffo0Dh06hPz8fPj6+iIzM1O9zaRJk7Bv3z7s3LkTx48fR3x8PAYNGlSNvSYiIqKaRCaEENXdCW09ePAAdnZ2OH78OHr06AGVSoX69evj559/xhtvvAEAuH79Opo3b47Q0FC88MILWu03PT0dVlZWUKlUsLS0rMqnQERERHqi7d/vGj2y8yyVSgUAsLGxAQCcO3cO+fn58PHxUW/j4eEBZ2dnhIaGlrqf3NxcpKenazyIiIhImmpN2CksLMTEiRPRrVs3tGrVCgCQmJgIuVwOa2trjW3t7e2RmJhY6r6CgoJgZWWlfjg5OVVl14mIiKga1Zqw4+/vjytXrmDbtm2V3ldgYCBUKpX6cffuXT30kIiIiGoio+rugDYCAgKwf/9+nDhxAg0bNlQvd3BwQF5eHtLS0jRGd5KSkuDg4FDq/hQKBRQKRVV2mYiIiGqIGj2yI4RAQEAAdu/ejSNHjsDNzU1jfYcOHWBsbIyQkBD1sqioKMTFxcHLy+t5d5eIiIhqoBo9suPv74+ff/4Zv/76KywsLNTzcKysrGBqagorKyuMHj0akydPho2NDSwtLTF+/Hh4eXlpfScWERERSVuNvvVcJpOVuHzjxo147733ADz5UMEpU6Zg69atyM3NhZ+fH1avXl3mZaxn8dZzIiKi2kfbv981Ouw8Lww7REREtY8kP2eHiIiISFcMO0RERCRpDDtEREQkaQw7REREJGkMO0RERCRpDDtEREQkaQw7REREJGkMO0RERCRpDDulOHHiBPr16welUgmZTIY9e/ao1+Xn52PGjBlo3bo1zM3NoVQq8e677yI+Pr7MfX722WeQyWQaDw8PD41tcnJy4O/vj3r16qFOnToYPHgwkpKSquIpEhER/Ssw7JQiMzMTnp6eWLVqVbF1WVlZOH/+PGbPno3z589j165diIqKwuuvv17uflu2bImEhAT14+TJkxrrJ02ahH379mHnzp04fvw44uPjMWjQIL09LyIion8dQUKlUgkAQqVSlbgegNi9e3eZ+zhz5owAIO7cuVPqNnPnzhWenp6lrk9LSxPGxsZi586d6mWRkZECgAgNDS2z/Zri+PHj4rXXXhOOjo7FjlteXp6YPn26aNWqlTAzMxOOjo7inXfeEffv3y93vytXrhQuLi5CoVCIzp07i7CwMPW6hw8fioCAANG0aVNhYmIinJycxPjx40VaWlpVPEUiIqohyvv7XYQjO3qiUqkgk8lgbW1d5nbR0dFQKpVo1KgRRowYgbi4OPW6c+fOIT8/Hz4+PuplHh4ecHZ2RmhoaFV1Xa+qYkRs+/btmDx5MubOnYvz58/D09MTfn5+SE5OBgDEx8cjPj4eS5YswZUrV7Bp0yYEBwdj9OjRVfIciYiolnlO4atGq+zITnZ2tmjfvr146623ymznt99+Ezt27BAXL14UwcHBwsvLSzg7O4v09HQhhBBbtmwRcrm8WF2nTp3E9OnTtX9CNUR5x00I7UbEOnfuLPz9/dU/FxQUCKVSKYKCgkqt2bFjh5DL5SI/P1/nfhMRUe2g7ciOUbUmLQnIz8/H0KFDIYTAmjVryty2T58+6v9v06YNunTpAhcXF+zYseNfOwpR3ohYXl4ezp07h8DAQPUyAwMD+Pj4lDnaVfQNuEZGPMWJiP7teBmrEoqCzp07d3Do0KEyv16+JNbW1mjatCliYmIAAA4ODsjLy0NaWprGdklJSXBwcNBXt2uMnJwczJgxA2+++Wapx+7vv/9GQUEB7O3tNZbb29sjMTGx1Jr58+djzJgxeu8zERHVPgw7FVQUdKKjo3H48GHUq1dP531kZGTg5s2bcHR0BAB06NABxsbGCAkJUW8TFRWFuLg4eHl56a3vNYEuI2K6SE9PR9++fdGiRQt89tlnetsvERHVXhzjL0VGRoZ6xAUAYmNjERERARsbGzg6OuKNN97A+fPnsX//fhQUFKhHGWxsbCCXywEA3t7eGDhwIAICAgAAU6dORb9+/eDi4oL4+HjMnTsXhoaGePPNNwEAVlZWGD16NCZPngwbGxtYWlpi/Pjx8PLywgsvvPCcj0DFJaiyEft3Zqnrnx4RO3LkSJkjYra2tjA0NCz2WUMljXY9evQIvV7xhczIBKs3/QxjY+PKPZHnrOi4udmaw9HKtLq7Q0QkHc9lBlENV9IEp6NHjwoAxR4jR44UsbGxJa4DII4ePareh4uLi5g7d67652HDhglHR0chl8tFgwYNxLBhw0RMTIxGX7Kzs8W4ceNE3bp1hZmZmRg4cKBISEio6kOgN9vO3BFuM/cLlxn7BQAxZfE6jfV5eXliwIABomXLliI5OVmrfXbu3FkEBASofy4oKBANGjTQmKCsUqmEe6t2wsSplXCa/ItwnblfrP/zligsLNTPE6tCx48fF+27ewvDOjYCgLAb9InYduZ/E7YLCwvF7NmzhYODgzAxMRHe3t7ixo0b5e6ztI8AqMx+a5LyniOPW8l43EgXNf184a3nlfTSSy9BCFHssWnTJri6upa4TgiBl156Sb2P0IuR8H0nAAmqbADAtm3bEB8fj9zcXNy7dw/btm1D48aNNdo1MTHBqlWrkJKSgszMTOzatavWzNdJUGVjxrYzyEm8hbykWwCADcFnMGzBz1i1LxSR91PxxhtvIDw8HFu2bFGPiCUmJiIvL0+9H29vb6xcuVL98+TJk/H999/jhx9+QGRkJD766CNkZmbi3ZHv4Wq8CusOX4Z7+264nZQKmz4TIHKz8fhRKuZuO4Vmn+xHryXHMOL/TmPazov4+tANbDsTh+M3HiAm+REycx8Xew5/3fxb/Tt7Hv66fh8xBbaweWUsAEAIIHDXZXUfFi9ejOXLl2Pt2rUICwuDubk5/Pz8kJOTU+o+y/oIgCIV2W9VqOgxL+858riV7N9+3CqjOl4fqptkzpcyo9C/hLbJUBfbztwRrjOejHC4zdyv8U5dqlYdjRb2by4occTLvJW3aDB2vVYjYg2dnMWo8dNEfFqWetmKFSuEk5OzMDaWC+dmbYRf4P+JFrN/Fy4z9pfaJgDRYOx64fLP76G0R+u5wcJv2XHht+y4ellV/84KCwvFketJYvh3oRp9ASDqD/xEuMzYLzadejIy5eDgIL766it1bVpamlAoFGLr1q1atYVS3o1Vdr/6sO3MHeE6s/LH/NnnKPXjtuHkLb28vvzbjltl6Otcra3i07IEALH+p23qZTXhfOGt59UoQZWNwF2XIf75uVAAM/97GU3s6qCDi0219q0qCCGw+thNfHUwCibObeAyY796nYEMeLuLC6KSHuHivTSNdQBgZCBDywZWOPGoLrIuJ+B+ajaMR6xGiACOLjyCoR2dYGQoQ/jjNjAcsRrKfw7q9UIAeQWoozDCi9690PStK9hw6jaEgEbb2z98AY8LgPi0bCSoshGvykFCWjbi03IQr8rGo5zHSM95jPTERxr9KhTAjP9eRmpmPoZ1ckJdc7lejlXe40LsvRiP70/cQlTSkzYNZUCBKL7tZ3uv4WpUDBITEzU+aNLKygpdunRBaGgohg8fXqF+xMbGVsl+dXE3JRMz/6v57yRw12X0aFq/0nOWqur5Vedxy8h9jEPXErEz/B7+uvlQvZzHrepdvpeGGf+9rP65UAAztobh8IbF+OO3fUhOTka7du3w7bffolOnTiXuIyEhAVOmTEF4eDhiYmLw8ccf45tvvtHYZteuXViwYAFiYmKQn58Pd3d3TJkyBe+8805VPr1ybT8bh8BdT55/4K7LMG/qhWGdnGvV+cKwUwVi/85E4TN/vASAN9aE4tU2jviwRyO0aWhdHV3Tu4zcx5i64yKCrz6ZoN3FzQZnb6egUACGMhkWDGqFYZ2cAQD5BYW4Fp+O8DupOH8nFeF3UpCUnouLd9Nw8W4aNpyK1dh3oQC2nb2rsczJxhQdnOuig6sNOjjXRTMHCxgayAAATe0tMGvXFRQIoW67k2vZd8ll5D5GQlo2Qq4nY+Hv14utXxh8HUv+iMKL7rZ4rY0Svi3tYWGi+8Tn9Jx8bA2Lw8ZTt5GY/mQY1lxuiDc7O2NUdzf8Gf0As3ZdAQDIZEB7Z2ucj0vD5pCLT+phprG/sm6910ZRrS639OtTTHIGPtwcjmczXqEAPtt7Fd8MawdTuWGF919Vz+95H7fsvAIcjUrGvovxOHI9GbmPC0vcrlAAG07GYtarzSGTySrcnlSOm77k5Bdg/clYrAiJLrbuwe8rsD87Adt//BFKpRI//fQTfHx8cO3aNTRo0KDY9rm5uahfvz4+/fRTLFu2rMT2bGxs8Mknn8DDwwNyuRz79+/H+++/Dzs7O/j5+en9+WkjPi0LM3ddVr+RFAKYtesKejStX6vOF4adKuBmaw4DGUoMPAcuJeDApQS80MgGH/ZojJea1a/Ui1N1iv07E2M2hyM6OQNyQwPM698Swzs7I0GVjdt/Z8HV1kzjnaaxoQE8nazh6WSN0d3dIITAvdRsnI9LRfjtVJy48QB3UrKKtfNqKwf081Sig0td2FmalNqfYZ2c0aNp/RLbLk0dhRHc7S1Qx8QIi4Ova/zOZACa2NVBdHIGjkY9wNGoB5DvNsDLzeqjn6cS3h725f5BTlBlY+Op2/g5LA4Z/8wPsrNQ4P1ubnirizOsTI01+q5cCAQNao1RI7oh+EoiPv42GkkARm48g2kDDfBhj8bqcFcbFRYK/BB6Gwt/v17qH+6DV5PQd/mf+HpYW7R1sn6+HawBch8X4M8bf2PfpXgcvpaEzLwC9bpGtuZ4qVl9bPrrdrHXl+//jMXNB5lYOKh1mf9OqHxCCPx2ORFBv0fiXmrx+TmF+bnIijoF88GzEZ5jj4mNGuOzzz7Dvn37sGbNGnzxxRfFalxdXfHtt98CADZs2FBiu0/P+QSACRMm4IcffsDJkyerJexEJT7C1J0RGiPmAFAgBC7dTYPFc+9RxTHsVAFHK1MEDWpdbJShTUNrfH/iFvZejMfpWyk4fSsFTe3r4IMXG6F/2waQG9We+eJHo5Lx8dYLeJTzGHYWCqx9pwPaO9cF8OT5axM0ZDIZnGzM4GRjhv5tGyBBlY1uC49ovIgbymSY3a+F1sPz2rZdUl1Jv7NhnZwRk5yB/Zfise9iPG4+yMTBq0k4eDUJZnJD+DS3Rz9PJXo0tYXCyFB9+3hBocDuC/exNyIej/95Qu52dfBBj0bo31YJhVHxkFTUbxtzBQCgdysHbBnvh87/B+Smp2JxcBRCIpOxdIgnkpKS0LZtW52fZ5GiSe9JSUnqz3kq+rky+y3LvdQsTNt5CaG3nlyCedHdFi+618ei36+rj/m7XV3w2+UE3Po7E4PX/IVxLzXG+F7uOv/bqKrnp+/9Fp0vTnVNEft3FvZfikfwlUSk5/xv4nwDa1P081Sin6cjWjhaQiaToZnD/0YxDWRAn1aOOHQtCUeuJ8P3mxP4YkArvNZGWe3Pr6r3WxWu3Fdh3r5rOHM7BQDgYGmCmX08kJ1fgE93/3PMRQEgCiEzNMbyIzG4Gp+OZcPbwtTUFCdPntRLP4QQOHLkCKKiorBo0aLn+tEUKZl5+PpQFH4OiysWqouM33oBPg2fvPGqDecLw04VKW2U4ethbTHVrxk2nIzF1jNxuJGUgWm/XMKSP6LU7/YtK3CZ5Hkpmp+z5I8oCAF0cKmLNSPa6+WdZGmB43l95kxpv7MmdnUw0acpJni743riI+y7GI99l+JxNyUbey/GY+/FeFiYGKGpvQXO30ktdmmmi5sNPuzZCC81tYOBjqMyHVt7wMHBAT5WybigaIpzd1LhuzgYt0+fxtixYyv8XN3c3ODg4ICQkBD1i0d6ejrCwsLw0UcfVXi/JRFC4Jdz9/D5vmvIyH0MU2NDzOrbHG93cYZMJkM/T0eNYz7B2x1zfr2KvRfjseJIDI5cT8bXQ9uimYP27yOr6vnpa79CCGw4GYsvfoss9q4ZeDL691obJV7zdEQ7J+tio78lnas3kh5h0vYIXI1PR8DPF/DH1STM698S1mbazzer6cetKiWn5+Crg1H45fw9CAGYGD8ZSf2wZyOYyZ/8qXyp2f+O+eDTXlBF70e2vQsOXyvAC+/vQVRoKJo0aVKpfqhUKjRo0AC5ubkwNDTE6tWrkWLdTP1G0ED2ZPS3aHqAPuU9LsTm0Nv4NiQaj/4J3L1bOqBNQyss/eMGgCeX2RvWNcW91GwcuF0IQ/O6+GjxD1i10AXtnevW2POFYacKlTbKoLQ2xaevtcB4b3f8HBaHjadikZSei4W/X8fKIzF4q4sz3u/mCgAVTvJV8S7g2fk5I7o4Y26/lnodkarIpSh9KmtkSCaTobmjJZo7WmKaXzNcvKfCvovxOHApAYnpOTh3J7VYzf+N7Aif5vYl7O1/yvoAS2dnZ0ycOBELFy7E16vaYGdULo5tXQVhWhf7VA3wkiobjlamxT7Asrx9ymQyTJw4EV988QXc3d3h5uaG2bNnQ6lUYsCAARU4ciV78CgXgbsu43Dkkw+F7OBSF0uHeMLV1ly9zbPH3NpMjuVvtoNfSwd8sucyrsano9+Kk5jq1xSjuzdSX8bT5riV9/wqe9zq2Dris7lzYe/gqLHfRzn5SFDlIP6fyfAJqv/9N0GVg/upWcgrYVb6wLZKDOvsjE6uNuVernz2uDW1t8Ducd2w8kg0Vh27ib0X4xEW+xCL3/BEz6b11dvVhONWVedbRRTNy1l9NEZ9yXBAWyWm9/aA0lrzteDpY/7jjz9i1KhROLH8HcDAAHL7xrBo2RPZGXeLtaELCwsLREREICMjAyEhIZg4aTLM+s6AiXMbAP+74cW2jgK9POz0Mg1CCIGQyGR8+Vuk+gNhmztaYs5rLdDaXoGYmBi4v2oD34XAuA6WeN23Lu5k1sPemDzs7tgfZ3Z9j965Fmjbwh0Zf/1cI88X2T+3fv2rpaenw8rKSv3lkc9b7uMC/Brx5A6d6OQMANCY8yMD0L+tEh1ctbuT69ztFPwaEQ/xz36+HNgab3au3LuA0ubn0JN5KJv+uo15+68VW7f1gxfg1bjsSdLHjh3Dyy+/XGz5yJEjsWnTJgghMHfuXKxbtw5paWlo1KoDsju9B2GlhKWJEeb1b4WJA7th8PARGPzBJLjZmiPqQliZ+wRQbL/du3fH6tWr0bRp04odiGcEX0nArN1XkJKZB2NDGSa/0gxjejTSac5RcnoOZu66jCPXkwEAnV1tsGSIJ5zrmel83Ep6fk7OLvAdMAxfzP8cjlam5e4TALLzHmPGrE+xcf3/IeNROkwatoCN7zi0at4MgAzxadl49MznN2lLm/NFGxF30zB5RwRuPXjyh2tEF2fMerU5zBVG1Xbcnt1v6w5dsGz5CnTv0Ean51bZN3JF9a71zHAhToUFv0XiftqTeTltnawxp18L9SV5bWRmZiI24QHmhyRg/zczIPKy8cWanzDRp2mpI7kvvfQS2rZtW+xurKelZubhp9N3MHfaeGSnJsN+2Pxi2zjbmOG1No7o56mEh4NFhYJPVOIjfHHgGv6M/hsAYFtHjqm+zTCkoxMMDWTl/m6jEtPxXsA0nA3egYKcTJg0bIG2w6dgwqCeGNCuAUyMDfVyvpT2+qTt32+GHVR/2ClSWChw7EYylofEIOJuml73/UIjG3RvYov2LnXR1slaPSyrjaPXk/HxtpLn59ATpc03Ojnz5SoZnYpJzsCUHRG4eE8FAGjdwBJX49OrfJhbG6rsfHy+9yp2XbgPAPBwsMCyYW3R3LFi/7aEENgRfhfz9l1DZl4BzOSG+LRvC7zZ2alS72qLbqd9+pgNbt8QyY9yNUZi4tNy/vnogic//52RV/7OAViZGsPRygRKa1P1f5XWJnC0MoWxoQxD1oZW6fmSnVeAxQevY+Op2wCe/GH8eqgnOmr5pqk0JR03Xc61ytTrs+2nFc3Led1TqfOl5iLJfz+Es6sbzLuPhEXb3vD2sMOy4W1LnJZQVti5m5KF9Sdjsf3sXWTnF+Dv377B47REOLy1UL2NDIDCyAA5T03yb2JXB/3+ufTZuH6dcvv77LwcuaEBRnV3g//LjSt0x2miKgcb/4rFz6fj1GHfto4CHVyscehaUpW9NjHs6KCmhJ0if938G299H1ZseWfXuqhXR1Fm7cOMXJy5XfxyytMMDWRo4WiJDi510cGlLjq61i32Apugykbsg0wcv5GMdX/G6n1+jhRtPxtX4gTnqvK4oBCrj93Et4dvFPucHgMZ8P27HdG6oRVszRXlvoDr693yg0dPLscmqHJgIAPG9myMCT7uJU7I1tXdlCxM2XkRZ2KfTBx9qVl9LBrcBoVCaNV3IQRSMvOQoMrBlfsqjc/CKlLSXZQlMTaUIb+Ey1CBfTzg3dwOjlamMFeU/YbieZ0vf8X8jWm/XML9tGzIZMCYHo0w+ZWmSMnM0+q4PS4oVAfAq/HpmPvrVY3jJgPg19JBq48KyM4rUF8G17W+MrWl1QPAqG6umOrXTKc3gABw8OBBCCHQrFkzxMTEYNq0aTAxMcGkb7di9r7rSArZANM8FU7+/l80sXsy3ywiIgIA8J///AfNmjXDtGnTIJfL0aJFC1y+p8KYKZ/gtswBBtaOwON8WP59GdEH1uH9aV/giGitca7081TiyPUnH0twNOoB8p4KPi0cLdHPU4nX2jjCyebJx1YU/RttaG2KP64lFZuXE/iqB1zqmaOyHuXkY9uZu9hwKhYJquKfdqzvUM+wo4OaFnYqM0pQUq2BDJjg7Y4byRk4dztV/TkvT1Namfzz2TXWSMnMw8qjMRr7qIr5OVJU2m33VennsDuYtftKqevlhgawt1JAaWWqHmVwtDaF8p/RhtO3HmL+/mt6fbfsWs8MS4e2RQcX/Y4AFhYKbDgVi8UHo5D3uBCmxobIyS9QX7Kd4OMOz4bWJcyXeTIyU9rt7k8zMpDB3tIESuuiUZknIzJKK1M4/vPf7PzH6L7oaKVHZp7X+ZKek495+67hl3P3AAD2lgokP8qF+Od3HvByE7RsYIWEf45T/D/HLyEtG0mPclGgTQKspSp66XDHjh0IDAzEvXv3YGNjg8GDB+PLL7+ElZUVLt9Twbv/UKQ/iEeT95dg6VBP+LV0KHEk0l7phJfn7kDorYdIPfEjsq6fgMhIgZmZKVq1aI4JEyZg2LBhZZ4r6Tn5OHQ1CfsvxePP6L/Vd4ACQDtnazSwNsVvlxOKBfmieTn6uHT6rPyCQnx96AbWHLtZbJ2+LtcCDDs6qWlhB6jcu77yau+nZePcnVScu52Cc3GpuPbP5Y/SyGTAXzN78Zu4a6iSAi4A1K+jwMPMXK1GKp7VwNoERoblB9vHBYW4n6YZnmUAQqb0RCMthtIrKjrpEQK2nkdUYobOtfUtFLA1lyPymU/NNpABe/y7oaXSSqt5Rc97JE8f/riaiOm/XEJadr5OdUUB0LaOXH3ptIgMT8KSpWn5lz7Ss/Ox8mhMsZEhbeorU1tafVVeav47Ixf+W84j7J+RyI97NcGwTk64k5KFhtamCItNwfd/3sKNpCfnsJGBDP08lfjgxUZooaz436HUzDwEX03EvovxOH3rYan//gN7e+A/Os6h09XzuLzPsKODmhh2gMq969OlNjP3MS7eTUP4nVQcjkzCpWdezAD9JnHSv9L+8OYXFCIpPUc90vHsiEdcSpZ6KFufnsf58mf0A7yz/kyx5U51TeFub1FsnozSyhT2Vgr1JTV9hJXqGMmrrOArCRj70/liyxvbmsPd3kLjmDlam6CBtSls6yjUfxQre9yq8o1cVdfrKr+gEF8eiMSmv26Xus3Tn6T+7N1flZX8KAerj94ssf3n9Zpe1cecYUcHNTXsVIfnPdGW9Kcif3hLu+y59u325c4PA57MEfvwp/ManxXzvM4XfZyrtTGsVFZNOG7P641cVdRXxPo/b2H+gchiyz96qTHG9mys/iT1qlATXtOr8phr+/ebn7NDGqr7g/2o4iry6dGl/b59WzqWX/yPhdV0vujjXK3oJ27XZjXhuFWmvjrbrqjmpVyW6uFev0qDDlAzXtNrwr8zjuyAIzsl+Te+4/03q43vlmtC27UZj9vzI/XRlerEy1g6YNghIqKqVBsntNcGvIxFRERUQ1T3V+H82zHsEBERPQc1Ye7KvxU/IY6IiIgkjWGHiIiIJI1hh4iIiCSNYYeIiIgkjWGHiIiIJI1hh4iIiCSNYYeIiIgkjWGHiIiIJI1hh4iIiCSNYYeIiIgkjWGHiIiIJI1hh4iIiCSNYYeIiIgkjWGHiIiIJI1hh4iIiCSNYYeIiIgkjWGHiIiIJI1hh4iIiCSNYYeIiIgkjWGHiIiIJI1hh4iIiCSNYYeIiIgkjWGHiIiIJI1hh4iIiCSNYYeIiIgkTTJhZ9WqVXB1dYWJiQm6dOmCM2fOVHeXiIiIqAaQRNjZvn07Jk+ejLlz5+L8+fPw9PSEn58fkpOTq7trREREVM0kEXa+/vprfPDBB3j//ffRokULrF27FmZmZtiwYUN1d42IiIiqmVF1d6Cy8vLycO7cOQQGBqqXGRgYwMfHB6GhoSXW5ObmIjc3V/2zSqUCAKSnp1dtZ4mIiEhviv5uCyHK3K7Wh52///4bBQUFsLe311hub2+P69evl1gTFBSEzz//vNhyJyenKukjERERVZ1Hjx7Bysqq1PW1PuxURGBgICZPnqz+ubCwECkpKahXrx5kMpne2klPT4eTkxPu3r0LS0vL51rPtp9/25WtZ9v/rrYrW8+22XZtqa9s22URQuDRo0dQKpVlblfrw46trS0MDQ2RlJSksTwpKQkODg4l1igUCigUCo1l1tbWVdVFWFpaVuoXXJl6tv38265sPdv+d7Vd2Xq2zbZrS31l2y5NWSM6RWr9BGW5XI4OHTogJCREvaywsBAhISHw8vKqxp4RERFRTVDrR3YAYPLkyRg5ciQ6duyIzp0745tvvkFmZibef//96u4aERERVTNJhJ1hw4bhwYMHmDNnDhITE9G2bVsEBwcXm7T8vCkUCsydO7fYJbPnUc+2n3/bla1n2/+utitbz7bZdm2pr2zb+iAT5d2vRURERFSL1fo5O0RERERlYdghIiIiSWPYISIiIklj2CEiIiJJY9ipQqtWrYKrqytMTEzQpUsXnDlzRqu6EydOoF+/flAqlZDJZNizZ4/WbQYFBaFTp06wsLCAnZ0dBgwYgKioKK3r16xZgzZt2qg//MnLywu///671vVPW7hwIWQyGSZOnKjV9p999hlkMpnGw8PDQ+v27t+/j7fffhv16tWDqakpWrdujfDwcK1qXV1di7Utk8ng7+9fbm1BQQFmz54NNzc3mJqaonHjxpg/f36539XytEePHmHixIlwcXGBqakpunbtirNnzxbbrrxzQwiBOXPmwNHREaampvDx8UF0dLTW9bt27YKvr6/608QjIiK0bj8/Px8zZsxA69atYW5uDqVSiXfffRfx8fFatf3ZZ5/Bw8MD5ubmqFu3Lnx8fBAWFqZ13582duxYyGQyfPPNN1rVvvfee8V+971799ap7cjISLz++uuwsrKCubk5OnXqhLi4uHJrSzrvZDIZvvrqK63azsjIQEBAABo2bAhTU1P1lyFrU5uUlIT33nsPSqUSZmZm6N27t/p80ea1JCcnB/7+/qhXrx7q1KmDwYMHqz/gVZv6devW4aWXXoKlpSVkMhnS0tLU68qrT0lJwfjx49GsWTOYmprC2dkZH3/8MVQqlVZtf/jhh2jcuDFMTU1Rv3599O/fX/0VQ7q8jgoh0KdPH/Xx1ab2pZdeKvb7Hjt2rE5th4aGolevXjA3N4elpSV69OiBefPmlVl7+/btUs+3nTt3atV2YmIi3nnnHTg4OMDc3Bzt27fHf//7X61qb968iYEDB6J+/fqwtLTE0KFDi30gcFVh2Kki27dvx+TJkzF37lycP38enp6e8PPzQ3Jycrm1mZmZ8PT0xKpVq3Ru9/jx4/D398fp06dx6NAh5Ofnw9fXF5mZmVrVN2zYEAsXLsS5c+cQHh6OXr16oX///rh69apO/Th79iy+++47tGnTRqe6li1bIiEhQf04efKkVnWpqano1q0bjI2N8fvvv+PatWtYunQp6tatq3V/n2730KFDAIAhQ4aUW7to0SKsWbMGK1euRGRkJBYtWoTFixdjxYoVWrUNAP/5z39w6NAh/Pjjj7h8+TJ8fX3h4+OD+/fva2xX3rmxePFiLF++HGvXrkVYWBjMzc3h5+eHnJwcreozMzPRvXt3LFq0qNT1pdVnZWXh/PnzmD17Ns6fP49du3YhKioKr7/+ulZtN23aFCtXrsTly5dx8uRJuLq6wtfXFw8ePNCqvsju3btx+vRpjY+P16a2d+/eGufA1q1bta6/efMmunfvDg8PDxw7dgyXLl3C7NmzYWJiUm7t020mJCRgw4YNkMlkGDx4sFZtT548GcHBwfjpp58QGRmJiRMnIiAgAHv37i2zVgiBAQMG4NatW/j1119x4cIFuLi4wMfHB5mZmVq9lkyaNAn79u3Dzp07cfz4ccTHx2PQoEEAtHstysrKQu/evTFr1qxi/SuvPj4+HvHx8ViyZAmuXLmCTZs2ITg4GKNHj9aq7Q4dOmDjxo2IjIzEwYMHIYSAr68vCgoKdHod/eabbzS+Zkjb2g8++EDj97548WKt60NDQ9G7d2/4+vrizJkzOHv2LAICAnDy5Mkya52cnIqdb59//jnq1KmDPn36aNX2u+++i6ioKOzduxeXL1/GoEGDMHToUOzbt6/M2szMTPj6+kImk+HIkSM4deoU8vLy0K9fPxQWFhY7rnonqEp07txZ+Pv7q38uKCgQSqVSBAUF6bQfAGL37t0V7kdycrIAII4fP17hfdStW1f83//9n9bbP3r0SLi7u4tDhw6Jnj17igkTJmhVN3fuXOHp6VmhPs6YMUN07969QrUlmTBhgmjcuLEoLCwsd9u+ffuKUaNGaSwbNGiQGDFihFZtZWVlCUNDQ7F//36N5e3btxeffPJJqXXPnhuFhYXCwcFBfPXVV+plaWlpQqFQiK1bt5Zb/7TY2FgBQFy4cEHr9kty5swZAUDcuXNH51qVSiUAiMOHD2vd9r1790SDBg3ElStXhIuLi1i2bJlWtSNHjhT9+/cvsz9l1Q8bNky8/fbbFap9Vv/+/UWvXr20rm/ZsqWYN2+exrKSzp1na6OiogQAceXKFfWygoICUb9+ffH9998Xa/vZ15K0tDRhbGwsdu7cqd4mMjJSABChoaHl1j/t6NGjAoBITU0t8XmXV19kx44dQi6Xi/z8fJ1rL168KACImJgYrdu+cOGCaNCggUhISCj1d1tSrS6viyXVd+nSRXz66acVqn1W27Zti71+lVVvbm4uNm/erLGdjY1NsXPm2dqDBw8KAwMDoVKp1NukpaUJmUwmDh06VO5zqSyO7FSBvLw8nDt3Dj4+PuplBgYG8PHxQWho6HPti0qlAgDY2NjoXFtQUIBt27YhMzNTp6/e8Pf3R9++fTWev7aio6OhVCrRqFEjjBgxAnFxcVrV7d27Fx07dsSQIUNgZ2eHdu3a4fvvv9e5feDJ7++nn37CqFGjtPpi2K5duyIkJAQ3btwAAFy8eBEnT55Enz59tGrv8ePHKCgogImJicZyU1NTrUe2ACA2NhaJiYkax93KygpdunR57uddEZVKBZlMpvN3z+Xl5WHdunWwsrKCp6enVjWFhYV45513MG3aNLRs2VLnvh47dgx2dnZo1qwZPvroIzx8+FDrdg8cOICmTZvCz88PdnZ26NKli06Xn4skJSXhwIEDGD16tNY1Xbt2xd69e3H//n0IIXD06FHcuHEDvr6+Zdbl5uYCgMZ5Z2BgAIVCUeJ59+xryblz55Cfn69xvnl4eMDZ2bnE860yr0Xa1qtUKlhaWsLIyKjY8rJqMzMzsXHjRri5ucHJyUmrtrOysvDWW29h1apVpX4PY1ltb9myBba2tmjVqhUCAwORlZWlVX1ycjLCwsJgZ2eHrl27wt7eHj179tTqd/asc+fOISIiotTzraT6rl27Yvv27UhJSUFhYSG2bduGnJwcvPTSS2XW5ubmQiaTaXywoImJCQwMDHR6nauwKo9T/0L3798XAMRff/2lsXzatGmic+fOOu0LlRjZKSgoEH379hXdunXTqe7SpUvC3NxcGBoaCisrK3HgwAGta7du3SpatWolsrOzhRC6vYP57bffxI4dO8TFixdFcHCw8PLyEs7OziI9Pb3cWoVCIRQKhQgMDBTnz58X3333nTAxMRGbNm3Suu9Ftm/fLgwNDcX9+/e12r6goEDMmDFDyGQyYWRkJGQymViwYIFObXp5eYmePXuK+/fvi8ePH4sff/xRGBgYiKZNm5Za8+y5cerUKQFAxMfHa2w3ZMgQMXTo0HLrn6aPkZ3s7GzRvn178dZbb2ldu2/fPmFubi5kMplQKpXizJkzWre9YMEC8corr6hH43QZ2dm6dav49ddfxaVLl8Tu3btF8+bNRadOncTjx4/LrS96V29mZia+/vprceHCBREUFCRkMpk4duyYVs+7yKJFi0TdunXV/3606XtOTo549913BQBhZGQk5HK5+OGHH8qtzcvLE87OzmLIkCEiJSVF5ObmioULFwoAwtfXV6O2pNeSLVu2CLlcXqydTp06ienTp5db/7TyRna0eS178OCBcHZ2FrNmzdK6dtWqVcLc3FwAEM2aNStxVKe0+jFjxojRo0erfy7pd1Na7XfffSeCg4PFpUuXxE8//SQaNGggBg4cqFXboaGhAoCwsbERGzZsEOfPnxcTJ04Ucrlc3LhxQ6vnXeSjjz4SzZs3L3FdafWpqanC19dXfb5ZWlqKgwcPllubnJwsLC0txYQJE0RmZqbIyMgQAQEBAoAYM2ZMqX3UF4adKlBTws7YsWOFi4uLuHv3rk51ubm5Ijo6WoSHh4uZM2cKW1tbcfXq1XLr4uLihJ2dnbh48aJ6mS5h51mpqanC0tJSq0toxsbGwsvLS2PZ+PHjxQsvvKBzu76+vuK1117TevutW7eKhg0biq1bt4pLly6JzZs3CxsbG52CVkxMjOjRo4cAIAwNDUWnTp3EiBEjhIeHR6k1NTns5OXliX79+ol27dppDFuXV5uRkSGio6NFaGioGDVqlHB1dRVJSUnl1oeHhwt7e3uNgKpL2HnWzZs3tb6EVvTv/c0339TYrl+/fmL48OE6td2sWTMREBBQ6vqS6r/66ivRtGlTsXfvXnHx4kWxYsUKUadOnWKXBkqqDQ8PF56enurzzs/PT/Tp00f07t1bY7uSXkt0CTvlvRaVF3bKq1epVKJz586id+/eIi8vT+vatLQ0cePGDXH8+HHRr18/0b59+2JBs6T6X3/9VTRp0kQ8evRIvayk46vta3BISEiJl9BKqi/6dx4YGKixbevWrcXMmTO1bjsrK0tYWVmJJUuWlLi+tPqAgADRuXNncfjwYRERESE+++wzYWVlJS5dulRu7cGDB0WjRo2ETCYThoaG4u233xbt27cXY8eOLePo6AfDThXIzc0VhoaGxU78d999V7z++us67auiYcff3180bNhQ3Lp1S+faZ3l7e2uVvHfv3q1+0Sx6AFCf2CW9Sy5Px44dNf4Bl8bZ2VnjXZYQQqxevVoolUqd2rt9+7YwMDAQe/bs0bqmYcOGYuXKlRrL5s+fL5o1a6ZT20I8+WNfFFaGDh0qXn311VK3ffbcKPoD/WxA6dGjh/j444/LrX9aZcJOXl6eGDBggGjTpo34+++/dap9VpMmTUocJXu2ftmyZerz7Olzz8DAQLi4uFSobVtbW7F27dpy287NzRVGRkZi/vz5GttNnz5ddO3aVeu2T5w4IQCIiIiIUvv0bH1WVpYwNjYuNt9r9OjRws/PT+u209LSRHJyshDiyXzDcePGqdeV9lpS9Af62YDi7Owsvv7663Lrn1ZW2CmvPj09XXh5eQlvb+9iQUWX18Hc3FxhZmYmfv7553LrJ0yYUOr51rNnT53bzsjIEABEcHBwuW3funVLABA//vijxvKhQ4eqR1G1aXvz5s3C2NhY/Xt/Wmn1MTExxeZ5CfHkb8SHH36oddsPHjxQ/67t7e3F4sWLS91WXzhnpwrI5XJ06NABISEh6mWFhYUICQnRae5LRQghEBAQgN27d+PIkSNwc3Or9D4LCwvV1/fL4u3tjcuXLyMiIkL96NixI0aMGIGIiAgYGhrq1G5GRgZu3rwJR0fHcrft1q1bsdscb9y4ARcXF53a3LhxI+zs7NC3b1+ta7KysmBgoPlPydDQsEJ3GJibm8PR0RGpqak4ePAg+vfvr3Wtm5sbHBwcNM679PR0hIWFVfl5VyQ/Px9Dhw5FdHQ0Dh8+jHr16lVqf9qee++88w4uXbqkce4plUpMmzYNBw8e1Lnde/fu4eHDh1qde3K5HJ06dar0+bd+/Xp06NBB6zlKwJPjnZ+fX+nzz8rKCvXr10d0dDTCw8PRv3//cl9LOnToAGNjY43zLSoqCnFxcfDy8qr0a5E29enp6fD19YVcLsfevXvV848q0rZ48uYfubm55dbPnDmz2PkGAMuWLcOGDRt0bruo3tHRsdy2XV1doVQqSzzfnJ2dtW57/fr1eP3111G/fn2NY1BWfdG8opLOt4KCAq3btrW1hbW1NY4cOYLk5GT1HZtVqsrj1L/Utm3bhEKhEJs2bRLXrl0TY8aMEdbW1iIxMbHc2kePHokLFy6ICxcuCADqeQDP3tFSko8++khYWVmJY8eOiYSEBPUjKytLq37PnDlTHD9+XMTGxopLly6JmTNnCplMJv744w+t6p+ly2WsKVOmiGPHjonY2Fhx6tQp4ePjI2xtbUt85/GsM2fOCCMjI/Hll1+K6OhosWXLFmFmZiZ++uknrftaUFAgnJ2dxYwZM7SuEeLJnTwNGjQQ+/fvF7GxsWLXrl3C1ta22FB+WYKDg8Xvv/8ubt26Jf744w/h6ekpunTpUmxIvrxzY+HChcLa2lo9/6R///7Czc1N/Y63vPqHDx+KCxcuiAMHDggAYtu2beLChQsiISGh3Pq8vDzx+uuvi4YNG4qIiAiN8y83N7fM2oyMDBEYGChCQ0PF7du3RXh4uHj//feFQqFQv4vU9d/F05exyqp99OiRmDp1qggNDRWxsbHi8OHDon379sLd3V3k5ORo1fauXbuEsbGxWLdunYiOjhYrVqwQhoaG4s8//9Sq3yqVSpiZmYk1a9YUex7l1ffs2VO0bNlSHD16VNy6dUts3LhRmJiYiNWrV5dbu2PHDnH06FFx8+ZNsWfPHuHi4iIGDRokhNDutWTs2LHC2dlZHDlyRISHhwsvLy/15WRt6hMSEsSFCxfE999/LwCIEydOiAsXLoiHDx+WW69SqUSXLl1E69atRUxMjMY2Y8eOLbP25s2bYsGCBSI8PFzcuXNHnDp1SvTr10/Y2NiIpKSkCr2O4p+Rs/JqY2JixLx580R4eLiIjY0Vv/76q2jUqJHo0aOH1sdt2bJlwtLSUuzcuVNER0eLTz/9VJiYmIi33npLq35HR0cLmUwmfv/9d43l5bWdl5cnmjRpIl588UURFhYmYmJixJIlS4RMJhOvvvpquW1v2LBBhIaGipiYGPHjjz8KGxsbMXny5FKPqT4x7FShFStWCGdnZyGXy0Xnzp3F6dOntaorGtJ99jFy5Mhya0uqAyA2btyoVdujRo0SLi4uQi6Xi/r16wtvb+8KBx0hdAs7w4YNE46OjkIul4sGDRqIYcOGlThhsDT79u0TrVq1EgqFQnh4eIh169bp1NeDBw8KACIqKkqnuvT0dDFhwgTh7OwsTExMRKNGjcQnn3wicnNztd7H9u3bRaNGjYRcLhcODg7C399fpKWlFduuvHOjsLBQzJ49W9jb2wuFQiG8vb01nk959Rs3bixx/dy5c8utL7r0VdLj6NGjZdZmZ2eLgQMHCqVSKeRyuXB0dBSvv/66xgRlXf9dPB12yqrNysoSvr6+on79+sLY2Fi4uLiIDz74QOONiTZtr1+/XjRp0kSYmJgIT09P9aVQbWq/++47YWpqWqHfeUJCgnjvvfeEUqkUJiYmolmzZmLp0qWisLCw3Npvv/1WNGzYUBgbGwtnZ2fx6aefqs9bbV5LsrOzxbhx40TdunWFmZmZGDhwoDoYa1M/d+7cUrcpr76051bWo6j2/v37ok+fPsLOzk4YGxuLhg0birfeektcv35d674/qyjslFcbFxcnevToIWxsbIRCoRBNmjQR06ZNU89t07btoKAg0bBhQ2FmZia8vLzEn3/+qXVtYGCgcHJyEgUFBcWeQ3n1N27cEIMGDRJ2dnbCzMxMtGnTRmzevFmr2hkzZgh7e3thbGws3N3d1efp8yD75wkSERERSRLn7BAREZGkMewQERGRpDHsEBERkaQx7BAREZGkMewQERGRpDHsEBERkaQx7BAREZGkMewQET3j2LFjkMlkSEtLq+6uEJEeMOwQERGRpDHsEBERkaQx7BBRjVNYWIigoCC4ubnB1NQUnp6e+OWXXwD87xLTgQMH0KZNG5iYmOCFF17AlStXNPbx3//+Fy1btoRCoYCrqyuWLl2qsT43NxczZsyAk5MTFAoFmjRpgvXr12tsc+7cOXTs2BFmZmbo2rVrsW+aJqLagWGHiGqcoKAgbN68GWvXrsXVq1cxadIkvP322zh+/Lh6m2nTpmHp0qU4e/Ys6tevj379+iE/Px/Ak5AydOhQDB8+HJcvX8Znn32G2bNnY9OmTer6d999F1u3bsXy5csRGRmJ7777DnXq1NHoxyeffIKlS5ciPDwcRkZGGDVq1HN5/kSkX/wiUCKqUXJzc2FjY4PDhw/Dy8tLvfw///kPsrKyMGbMGLz88svYtm0bhg0bBgBISUlBw4YNsWnTJgwdOhQjRozAgwcP8Mcff6jrp0+fjgMHDuDq1au4ceMGmjVrhkOHDsHHx6dYH44dO4aXX34Zhw8fhre3NwDgt99+Q9++fZGdnQ0TE5MqPgpEpE8c2SGiGiUmJgZZWVl45ZVXUKdOHfVj8+bNuHnzpnq7p4OQjY0NmjVrhsjISABAZGQkunXrprHfbt26ITo6GgUFBYiIiIChoSF69uxZZl/atGmj/n9HR0cAQHJycqWfIxE9X0bV3QEioqdlZGQAAA4cOIAGDRporFMoFBqBp6JMTU212s7Y2Fj9/zKZDMCT+UREVLtwZIeIapQWLVpAoVAgLi4OTZo00Xg4OTmptzt9+rT6/1NTU3Hjxg00b94cANC8eXOcOnVKY7+nTp1C06ZNYWhoiNatW6OwsFBjDhARSRdHdoioRrGwsMDUqVMxadIkFBYWonv37lCpVDh16hQsLS3h4uICAJg3bx7q1asHe3t7fPLJJ7C1tcWAAQMAAFOmTEGnTp0wf/58DBs2DKGhoVi5ciVWr14NAHB1dcXIkSMxatQoLF++HJ6enrhz5w6Sk5MxdOjQ6nrqRFRFGHaIqMaZP38+6tevj6CgINy6dQvW1tZo3749Zs2apb6MtHDhQkyYMAHR0dFo27Yt9u3bB7lcDgBo3749duzYgTlz5mD+/PlwdHTEvHnz8N5776nbWLNmDWbNmoVx48bh4cOHcHZ2xqxZs6rj6RJRFePdWERUqxTdKZWamgpra+vq7g4R1QKcs0NERESSxrBDREREksbLWERERCRpHNkhIiIiSWPYISIiIklj2CEiIiJJY9ghIiIiSWPYISIiIklj2CEiIiJJY9ghIiIiSWPYISIiIklj2CEiIiJJ+3+XrUfWLLtZNwAAAABJRU5ErkJggg==", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAG2CAYAAACZEEfAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAACvJ0lEQVR4nOzdd3gUVffA8e+k901vkE6v0oQAghgMINKliYhifQWlCEJQsIC0nwoqioKIKFJEAQGlSZUWQokE6REIhBRCkg3pITu/P5YsWZJAoikQzud59nnZmdm7dwZ1z3vvufcoqqqqCCGEEEJUUyZV3QEhhBBCiIokwY4QQgghqjUJdoQQQghRrUmwI4QQQohqTYIdIYQQQlRrEuwIIYQQolqTYEcIIYQQ1ZoEO0IIIYSo1iTYEUIIIUS1JsGOEEIIIaq1Kg12du/eTY8ePfD29kZRFNauXWt0XlVVpkyZgpeXF9bW1nTu3JmzZ88aXZOcnMyQIUNwcHDA0dGRF154gfT09Eq8CyGEEELcy6o02MnIyKBp06Z88cUXxZ6fPXs2n332GV999RXh4eHY2trSpUsXsrOzDdcMGTKEv//+m61bt7JhwwZ2797Nyy+/XFm3IIQQQoh7nHKvFAJVFIU1a9bQu3dvQD+q4+3tzZtvvsm4ceMA0Gq1eHh48N133zFo0CBOnjxJgwYNiIiIoGXLlgBs2rSJJ554gsuXL+Pt7V1VtyOEEEKIe4RZVXegJOfPnyc+Pp7OnTsbjmk0Glq3bs3+/fsZNGgQ+/fvx9HR0RDoAHTu3BkTExPCw8Pp06dPsW3n5OSQk5NjeK/T6UhOTsbFxQVFUSrupoQQQghRblRV5fr163h7e2NiUvJk1T0b7MTHxwPg4eFhdNzDw8NwLj4+Hnd3d6PzZmZmODs7G64pzowZM3j//ffLucdCCCGEqAqXLl2iZs2aJZ6/Z4OdihQWFsbYsWMN77VaLb6+vly6dAkHB4cq7JkQQgghSistLQ0fHx/s7e3veN09G+x4enoCkJCQgJeXl+F4QkICDz30kOGaxMREo8/duHGD5ORkw+eLY2lpiaWlZZHjDg4OEuwIIYQQ95m7paDcs/vsBAQE4OnpybZt2wzH0tLSCA8PJzg4GIDg4GBSU1M5fPiw4Zrt27ej0+lo3bp1pfdZCCGEEPeeKh3ZSU9P59y5c4b358+fJzIyEmdnZ3x9fRk9ejTTpk2jdu3aBAQEMHnyZLy9vQ0rturXr0/Xrl156aWX+Oqrr8jLy2PkyJEMGjRIVmIJIYQQAqjiYOfQoUN06tTJ8L4gj2bYsGF89913vPXWW2RkZPDyyy+TmppK+/bt2bRpE1ZWVobP/Pjjj4wcOZKQkBBMTEzo168fn332WaXfixBCCCHuTffMPjtVKS0tDY1Gg1arlZwdIYQQ4j5R2t/vezZnRwghhBCiPEiwI4QQQohqTYIdIYQQQlRrEuwIIYQQolqTYEcIIYQQ1ZoEO0IIIYSo1iTYEUIIIUS1JsGOEEIIIao1CXaEEEIIUa1JsCOEEEKIak2CHSGEEEJUaxLsCCGEEKJak2BHCCGEENWaBDtCCCGEqNYk2BFCCCFEtSbBjhBCCCGqNQl2hBBCCFGtSbAjhBBCiGpNgh0hhBBCVGsS7AghhBCiWpNgRwghhBDVmgQ7QgghhKjWJNgRQgghRLUmwY4QQgghqjUJdoQQQghRrUmwI4QQQohqTYIdIYQQQlRrEuwIIYQQolqTYEcIIYQQ1ZoEO0IIIYSo1iTYEUIIIUS1JsGOEEIIIao1CXaEEEIIUa1JsCOEEEKIak2CHSGEEEJUaxLsCCGEEKJak2BHCCHEAy8/P5/JkycTEBCAtbU1QUFBTJ06FVVVDdc899xzKIpi9Oratesd2929ezc9evTA29sbRVFYu3at4VycNot90UlcSc1kypQpeHh6YmllzSOPPsbZs2cr6lYfSBLsCCGEeODNmjWL+fPnM2/ePE6ePMmsWbOYPXs2n3/+udF1Xbt2JS4uzvBavny50fnbg6aBAweSlpbGvHnzjK5bGRFDu5nb6fLUUGo42TJ95ixo9xIuT/8fR2Izaf9oCNnZ2UafiY2N5ZlnnsHFxQVra2saN27MoUOHAMjLy2PChAk0btwYW1tbvL29efbZZ7ly5UoFPK37jwQ7QgghHnj79u2jV69edO/eHX9/f5566ilCQ0M5ePCg0XWWlpZ4enoaXk5OTkbnbw+aPv/8c44cOcLly5cN18RpswhbHUX66X3kXDkNioK5bzOsa7fBwj0AlyfHcjUhnsXLfjJ85uTFK7RsHUyeqrBx40ZOnDjBxx9/bPj+zMxMjhw5wuTJkzly5AirV6/m9OnT9OzZswKf2v1Dgh0hhBD3rdJMPxX26quvoigKc+fONTretm1btm3bxpkzZwD49ttv2bBhA7/99pvR9NPOnTtxd3enbt26tGzZktq1a2Nra4uTkxOdO3fm999/v2vQdD4pg9y0JJK3fo1jx+dAVbFw9jacN7G0xcK7Ljt27wX0o0DBg0aRotgT4TuI84onAQEBhIaGEhQUBIBGo2Hr1q0MGDCAunXr0qZNG+bNm8fhw4c5f/58uTyj291piq7Ae++9R7169YyeUXh4+B3brQgS7AghhLhvlXb6CWDNmjUcOHAAb2/vIucmTpzIoEGDqFevHubm5rz44ou0bduWb7/91nBN165d+f7779m2bRuzZs0iLi4OU1NTIiMj2bNnD/7+/hw6dIitW7cagqa//vqLPXv20K1bN0M7fs7WXNvwCQ6t+2JiYQOAYmlt1B8zW0ey064ZRoEyz4Zj4VmbhDUzePrRJgTVa8RHn31xx2ej1WpRFIXFixeXyzO6XUZGBk2bNuWLL0ruR506dZg3bx5RUVGGZxQaGsrVq1fv2n55MqvUbxNCCCHKUeHpJwB/f3+WL19eZPopNjaW119/nc2bNxuuLeynn37ixx9/ZNmyZTRs2JDIyEhGjx5NWlqa4ZpBgwYZ/ty4cWOaNGlCUFAQMTExhISE8Mknn7Bo0SLatm1LvXr1MDU1JT8/nw8//JAhQ4bwzDPPAPD9V5/h6mCNSYue5MSeAqCBlwMF2TWKAo1raLA2N+V8UgY6FfJS48k7+jsOrXrjETyA1LizvPXmWP65lsOX748tcj/Z2dlMmDCBwYMHc+TIkXJ5Rrfr1q2bURBXnKefftrofcEzOnbsGCEhIXf9jvIiIztCCCHuW7dPPxU3kqLT6Rg6dCjjx4+nYcOGxbYzfvx4w+hO48aNGTp0KGPGjGHGjBklfndgYCCurq6cO3eO3NxcFixYgLW1NX/++SfLli3jyJEjLFmyhI8++oglS5YAEB0dzaeffkrv0R+iKApPPFwXgHZ+dox4VD8l9ZCPIxZ51/H09MTPWT/yg6pi6RGEU8dhWHgEYf9QV+yaduHbbxYSp80y6ldeXh4DBgxAVVXmz59fbs/ovyp4RhqNhqZNm1bId5RIFapWq1UBVavVVnVXhBBClEF+fr46YcIEVVEU1czMTFUURZ0+fbrRNdOnT1cff/xxVafTqaqqqn5+fuqcOXOMrnF2dla//PLLIp+rXbu2Cqhr1qwp8t2XLl1SFUVRLS0tVUVRVG9vb9Xd3V2dN2+e0XVTp05V69atqwLq8OHDVUVRVMXEVEUxUU1MTVVAVRRFrenjq/pN2KD6jvlJtbS0VH/88Ud16P/GqqYaD/01Ftaqpv0Q1fet9arfhA2qc+hrqmJupfoG1lJtbGxUR0dH9bHHHlM7dOigNmnSRE1KSirxGQXVrqN6eXkZ7u32ZwQU+woNDTVco6qq+u677xruzdbWVg0JCVEPHDhgdP/r169XbW1tDc/oq6++Up988kmj779dSd8/e/Zso+tK+/stIztCCCHuW4Wnn4obSTl8+DCffvop3333HYqilNhOjx49+PDDD/ntt9+4cOECa9as4ZNPPqFPnz4AZGVlMX78eA4cOMCFCxfYtm0bvXr1IigoiMOHD7Nv3z66du1KYmIiW7ZsMbSbnp5OQkICWVn60ZeAgAB++ukn6g77EK/nP2P5xl04ODhgYWFB2MQJBJpe4+qGT3BwdufcuXP89MMinB9/lcAWHajhXYO0g6u5fng9AHnJsZjaOTH7k7lERUWxY8cOoqOj2bNnDytWrMDFxcXoGb0+9TM8np2LXcve/HP+Ak0e6QLcGm0q/Ixq1qxJSEgITk5O/PDDD8yePRuAvXv3GuX6FOTkAEyfPr3YnJxOnToRGRlpeEZTpkyhVq1ad8z1Kby8Py4ujm+//RZFUejXr19p/rEo6o6h0ANCRnaEEOLecSU1U9177qp6JTXzrtfWrFmzxJEUVVXVOXPmqIqiqKampoYXoJqYmKh+fn6Gz6SlpamjRo1SfX19VSsrKzUwMFB9++231ZycHBVQV6xYoYaGhqpubm6qubm56lnDRx0y7Hk1Pj7e6LsVRVHNLSzU71f+op4/f1794IMPih2hsG0UovpN2KAmp+eofn5+amhoqOrh4aGaWVioVn5NVeeAhmrdevVVuyaPq34TNqg/b9yhmpmZqTUDaqs2tVqrrj3GqYq5pfrQ0HdUVVXV3NxctWfPnqq3t7cKqD/99JMaFxenxsXFqTVr1lQ//L9P1ICJG1S/CfqX5pFnVHPnmkajTbc/I26O1qiqqvbq1Ut97LHH1L59+6pDhgwp8vfAzRGagt/TP/74o8S/s1q1ahlG3yhhZOd2Bd9/u9L+fkuCshBCiHvGyogYwlZHoVPBRIEZfRszsJVviddnZmZiYmI8SWFqaopOpwNg6NChdO7c2eh8ly5dGDp0KM8//7zhmL29PXPnzi1xubWlpSWbN2826t8+BXbG5DDQ41bfTRzcUGwcef6lVzHJScOnRg3efvttpkyZgoWFBQAn49Lo9umfONqY42SrP9atWzc2b95MvDab4JnbuPTlcGx9/Mi68BcNbNLp17U7n3zyCWPGjEFRzuN4Iwmnx17ies1gEtOyyUyOZ926dYb+DhgwwPBnBwcHUjJvoCu00lxRTFBV/TPq2LEjY8aMKfKMatWqxblz59i3bx+//fYbH3zwAXPnzuWTTz4p9hnl5eWVKidHp9ORk5NT4vnbJSQk8NtvvxlG6/4NCXaEEELcEwqWWRf8KOtUmLT6OB3quOGlsS72MwXTT76+vjRs2JCjR4/yySefMHz4cABcXFwM0zkFzM3N8fT0pG7duoZjISEh9OnTh5EjRxKnzeLviwmo2njc7K0AOH/+PFv/PMD45ScxcXBHl5tNyv6VjI1tQ+D47iQnJ/Hi6++Rn56Ce793sXDzw1RR2DOxU5G+X0jKAMDfxVb//sIFwzlPjRWt/J1R//ctqqrDdtf3bHpvMOZTnzGs7AoLCwPgqfn7OHQxhXc+/55lM8aiKApeXl6sXbuWVq1aGdp87rnn+PHrueS3fQlzV19yE6JJi1iLfZPH0Yb/goODA40aNSryjHr16kViYiLt27dHVVUmTZpEjx49GDJkCKCfojt37hy7d+8GYODAgbi6uvLpV99wRgva3CQWzfuEnj174uXlRVJSEl988QWxsbH079//Lv803LJkyRLs7e3p27dvqT9zOwl2hBBC3BMKllkXlq+qXEjKLDHY+fzzz5k8eTKvvfYaiYmJeHt788orrzBlypQyfXd0dDRJSUmGkZvMi8dIWD7JcH7sWP3ybttGIbh2H4NiYkJe8mUS1nxIuxVhODg5ke/gj+eQWVi4+d2x7//cDHYCXG2L7Yu3Rh9gZZ78k4wTO3nt/U95pfejhuXw3t7eDBs2jGfa+HHoYgqRN7w5dPgIqSnJLFy4kAEDBhAeHo67u7vhGYW9/Q5fL5lPfmYqpnbO2Dfrxhf/N41n2v1S4jOJjIxkx44deHp60rFjR7Zu3coff/zBkiVLGDZsGIcOHaJTp06G61VV5erVqzz/7FBqvPINplY2uB/S51ElJSXh4uJCq1at+PPPP8u04uvbb79lyJAhWFlZlfozt1Nuzpk90NLS0tBoNGi1WhwcHKq6O0II8UCK02bRdsZ2Cv8olTQ6UhHffehCMm+siKTwr6KJAmtea8vphHQ+3HASbXae0edMFNg78THOX83g6W+MdwYuqe/jV/3FqsOXGft4Hd4IqV2kH+1mbkenwuUvn0PT5ikcW/QwtDNt2jSWLl3KqVOnyLmRT/CM7SRn5PJmaB2ealETL401tWvXZvjw4YYRIICfD19m3Kq/DO97P+TN3EHNUBSFNWvW0Lt37yLPxMfHh6eeeoq5c+cSGRmJu38d3n3/A7ZvWM25M6eLfYbtZm7n0tcvYdf4cTTBA+7693en7wf4888/6dChA5GRkcVOjZX291tWYwkhhLgneGmsaeqjMbw3UWB630YVHuisjIih7cztvL7cONAB/VRary/28dbPx9Bm56GxNsOk0KKuNoEuuNpZMmPjqSLtTutTfN/PF0xjFTOyU3h0S83LAcXEMEIExvlIlmamNPbW/8B/vOUM7WZuZ2VETLE5MUsPXASgvpf++sspxnvzFCczM5P9+/fTokULTuY40XbmdlYdvsLFpHRWRsSU3HdVRc3XB4WF+/5vLFq0iBYtWvznfXkk2BFCCHHPKDyN9WHvOycnl4eCPKHSznFcz77BmtfaMrFbPQD2RV9j8IIDRMVq0Vib88v/grE21/+0BpYwTXXhWkaJ5wNcbQ3BlHWth9HuW0lOdASmGVeLLIePvpLEr4s+Jif2FDe0iWTFnePFF14skhPTpn1Hdq/9AXNThUlP1EOXm0XkX5EcPXoU0OcjRUZGEhNjHMB07dqVgwcP0rhZC8Yt/oOM0/tIi1iLTZ1gJq0+TvSVJCZNmsSBAwe4ePEi1y+f5trvc7lx/Ro2ddsb2hk/vJ9R1ff09HQiIyOJjIy84/enpaWxatUqXnzxxTv/pZSC5OwIIYS4J6iqahj1AEjOzK3w7ywuT6iACaC77ZhOhcxcHa92DGLHqUTCzydz6GIKAN0aedLCz5nuTbz5+fBlNhyLo3WgcXJ0WnYeSen6+ypuZMdLY82Mvo2ZtPo4zp1fQbvnR27s+YaOrWcVyUeKSc0m99pl0qO2kZ+Vhqm1AxaetZm+aDVaKw/Dzsp/nzqLab0adGnoycMBztxIOMelZWE0/1r/nQX5SMOGDeO7774z9KV169asXLmS3zds4GrSd5jaOWP3UDcc2w0iX1W5lJrDqVOnjHJynD1rY1cobwngxOmzHD0TQ5w2Cy+NdZFcn5K+f8WKFaiqyuDBg4v/CyoDydlBcnaEEOJekJSeQ8tpfxje921Wg08GPlSh3xmnzSJ4xnajYybA5083o6aTNX2+3GcUDBXkoACG3Jpb52DPxMc4HX+d5xZH4GpnwYGwEMxMb02iHLucSs95e3G1s+TQO8ZL4m/v14WkTPxdbUqcxiuc31OYgn6TnILZtoLTLz8SyKTu9en8yS7OJaazZPjDdKzjVvLDueng+WsM+PqA0bGScnEe/2QXZxPTebVjEI7W5szcdGt6rzRbCZSV5OwIIYS4r1woNKoDEH01vcg1cdos9kUnFakH9W+521thaXbrp9BUUZjRrzHdm3jT1MeJGX0bY3pzV2FTRTHkEBW/cgwuJGXSrpYrTjbmJKXncuCfZKNrCkauSpriKuClsSY4yOWO+UoFo0Cmt+0MrRb638JdXLTnH+K0WdTxsAPgTPx1o8+V9GwjLqQU+e4xj9cu0rfT8dc5m5iOhakJr3UKoudDXkbnC7YSKK+/u7KQYEcIIUSp+fv7oyhKkdeIESMA/RLuPn364ObmhoODAwMGDCAhIaFU7bQKcOHi7J7EL3yJa38sYMe8twzttG3blvot2lDT04N2tdxoOeabIkmy+fn5TJ48mYCAAKytrQkKCmLq1KkUnsBYvXo1oaGhuLi4oCgKq7fuIeeGDltLU5a+8DB7JnYyGnkY2MqXPRM7sfylNkbnCufWFDBVFPxdbTA3NaFrI/0P/YZjV4yuuZWcbPMv/waMFe7f54MfuuO1BcFYHQ97AE4n3Ap2VkbE0G7mdp5eGG5IdC6w/i/9PYR1q0dzH0cAziYWDUQLrutQxw0HK3MuXCuamPxfE5b/LQl2hBBClFpERIRRzaKtW7cC0L9/fzIyMggNDUVRFLZv387evXvJzc2lR48ehhVExbUzceJE7O31P8CuT75J5yEjST+8nuzkOH76dSN79+4lPTOLs2fOoek4DAC1mFGCWbNmMX/+fObNm8fJkyeZNWsWs2fPNqrllJGRQfv27Zk1axYAf11KBSA40IX2tYvfvLC4UZbbR1UKj/oA9GiiD3Z+j4rjz7NXDf28YNhjx+7fPP5iFfSvpb9zkQCssIJgrCDYOXsz2ClpM8c4bRbnEq9zKv465qYKA1v58EFv/eaDv0Ze4adDlwz3paqqIbDr0dTr5j2WHBBWNklQFkIIUWpubsY5HjNnziQoKMiw6dyFCxc4evSoIX9iyZIlODk5sX37dqOyDYXbiYqKwsfHh8tJadjU70Adj2R+Q8XcuSYW7v40ruXKR9/9QpdmtTAxtzR87vZN+/bt20evXr3o3r07oB89Wr58OQcPHjR8ZujQocCtXYuPxWoBJ4KDXMv8LAa28qVDHbdic2taB7pgZ2lGWvYNhi46aMhXOW8Idsr/B79wcnO+qupzdhR9YFg4GMvwyAfgTEI6Op16x80cD/xzDYBHarvhaGOBo40FjbwdOH4ljbd+Pma4r4beGi5cy8TK3ITO9T2K7c/tAWFlkmBHCCHEv5Kbm8vSpUsZO1ZfqiAnJwdFUbC0vBWQWFlZYWJiwp49e4rUqCrw8MMPs3HjRnwe6QeKQt61S/rPBrYg+mo67Wq5UsfbGRSF3Lizhs/dPkrQtm1bFixYwJkzZ6hTpw5//fUXe/bsKbGWE8DxWC04OxF826qp0vLSWBf74514PZuMnBuG9/rRkiisLUyB4ldilYfbAzCgSDDm72KDhakJWXn5XE7JIsDVFuVmUFRAAfxcrHlnrX605smbI1Vx2ixOxKUZ3VfY6ih6NfUGIDjIBVvLW6HFnQLCyiTBjhBCiH9l7dq1pKam8txzzwHQpk0bbG1tmTBhAtOnT0dVVSZOnEh+fj5xcXEltlOnTh1UVeXirlXw52rm6fIxM7cgL/E8py5dJaOJGx9PnQyqjvxMreFzU3s3NPrxnDhxImlpadSrVw9TU1NDLamCWk7Fyc7T4WFjTj1P+//+QAo5n5TB7Uud81VIz9GPqhTUxaoItwdgtwcYZqYmBLrZcir+OmcSrtO5gQcP+ThyNCb11kUKrI+MI/pqBuamCo830I/WFDcKpFNhTaQ+KNp56iorI2KM8p5KCggrk+TsCCGE+FcWLVpEt27d8PbW/796Nzc3Vq1axfr167Gzs0Oj0ZCamkrz5s2LVCYvbMaMGVhYWuLaYzw1n/+Mb7/7DjNzczJO7mLGoNZoNBpSUlKx9qqFUmjlkbej8Q/oTz/9xI8//siyZcs4ckRfk+mjjz66a7XsNoEumNwp2eVfKC5fpeC9t8YKK3PTcv2+sqrreStJOTsvn7MJ+oTj93s24PH6HqgqzLi5bDwvX+X3KH2wWtx9FaZSdSuu7kSCHSGEEGV28eJF/vjjjyK724aGhhIdHU1iYiJJSUn88MMPxMbGEhgYWGI7x44do0uv/tg26Eiteg14ftgwnvvfKEys7Gke9jNJSUmEzZ5HTloSVk7uhs9uOGY8WjR+/HgmTpzIoEGDaNy4MUOHDmXMmDHMmDHjjvcSHPTvprDupCBfpXBgYH7zjbfTvy9oWV4KkpTPJFxn5+mrpOfcwFtjxdA2/ozrUqfI9QUBzO2J2cUFEVW14upOJNgRQghRZosXL8bd3d2QDHw7V1dXHB0d2b59O4mJifTs2bPEdhRFwbFmEKDPJwFwt7cGVce1fEvMrO349qf16DK0tAjuYPjs5r/jybmRb3ifmZlZZASpcC2pwgp/rm0FBDugz1fZO/ExPnqqCaYK5OTr538OXUgttrZUZboV7KQbVlE92dQbExOFaxlFd64uHMAUXu6+ZkTbe2bF1Z3c08FOafZMUFWVKVOm4OXlhbW1NZ07d+bs2bN3aFUIIcR/odPpWLx4McOGDcPMzDj1c/HixRw4cIDo6GiWLl1K//79GTNmDHXr1jVcExISwrx58wztNGzYkLXfzSczOgLHfK2+BtRHs7D3qU9eShyffbWIeW+/hm3DR6njph8VsclK4MyCN3j1zcmGdnv06MGHH37Ib7/9xoULF4rUkgJITk4mMjKS5Zv3AWCZEc/12HPEx8dXyLPy0ljTrrZrkTyXqp7qqXsz2IlOTGfbyUTgVhJyaZaMFyx3v9PGi/cU9R724Ycfqi4uLuqGDRvU8+fPq6tWrVLt7OzUTz/91HDNzJkzVY1Go65du1b966+/1J49e6oBAQFqVlZWqb9Hq9WqgKrVaiviNoQQolrZvHmzCqinT58ucm7ChAmqh4eHam5urtauXVv9+OOPVZ1OZ3SNn5+f+u677xraOXz4sNrw8UGqqYObam5hqQYGBqrBwcGqpb2ziomZ6uUboDo07FiwIbDRy7t+c0O7aWlp6qhRo1RfX1/VyspKDQwMVN9++201JyfHcM3ixYuLbefdd9+tsOe199xV1W/ChiKvfeeSKuw77yY/X6fWe2ejoS+PzNpu9Pe04uBFNXDib6rfhA1q4MTf1BUHL96xvSupmeq+c0nqldTMiu66kdL+ft/TtbGefPJJPDw8WLRokeFYv379sLa2ZunSpaiqire3N2+++Sbjxo0DQKvV4uHhwXfffcegQYNK9T1SG0sIISpPnDaL80kZBLjaGkYAQj7eSfTVDJa+0Jr2tfV73ry9Joofw2NoX8uVPeeScLAy4+iUUExNFI7EpND3y31Ym5swf0gL6nrZl2o0obh6UiXVeSrP+63s7yyNnvP2cOyyfnXbs238DBsGFihNfa6qVi1qY7Vt25Zt27Zx5swZAMOeCd26dQP0ZeHj4+ON9m7QaDS0bt2a/fv3l9huTk4OaWlpRi8hhBAVr7iyBDfydcQk6/NBCk+VBLnpdxnecy4J0K+aMr05v9LMxxFHa3Oy8nQ8911EkRIHJfn7SlqJG+hVlLvttlxVzAsVKP0h/GKR51ea+lz3i3t6n5277ZlQMMfq4eFh9DkPD487zr/OmDGD999/v+I6LoQQooiSyhIEudmRl69iYWaCd6Ef1iB345IKhVdNxadlo83KM7wvaKtDnVslH24fQToTf50P1v1dpF+VkVB7r2yuVyBOm8WRi7cKfKrFPL/q5J4OdgrvmdCwYUMiIyMZPXo03t7eDBs27F+3GxYWxtixYw3v09LS8PHxKY8uCyGEKEFJZQkiLugrg/u72Bjtd1PrDsFO8Zv2qfx2LI7uTbzYdjKRyWuPo6LfDbiupz2nClX5VtAn61TmKMu9sLlegZKeX+HyG9XJPR3sFN4zAaBx48ZcvHiRGTNmMGzYMDw9PQFISEjAy+tWKfmEhAQeeuihEtu1tLQ02s5cCCFExStY5XN7wFMwQnP7rsK7z1w1en80JpV6ng53bGvabyeZ9ttJo2MqGAU6oA925j3djOZ+TtXyx/1uint+9+KS8fJyT+fs3G3PhICAADw9Pdm2bZvhfFpaGuHh4QQHB1dqX4UQQtxZQe7K7RbtOQ+Am8Ot/xMap83i7TVRRte9s+bWcu3b82DKSgc421o+kIEO3Lt5RBXlnh7ZKdgzwdfXl4YNG3L06FE++eQThg8fDoCiKIwePZpp06ZRu3ZtAgICmDx5Mt7e3vTu3btqOy+EEKKIga18eX/9CTJz85nYtR4zN50i7+Zme8sOxNCkhoaBrXzvWIm74Ae5IA/mt2NxRUZz7qY6j2KU1r2WR1SR7umRnc8//5ynnnqK1157jfr16zNu3DheeeUVpk6darjmrbfe4vXXX+fll1+mVatWpKens2nTJqysqn47biGEEMYyc2+QmavfvTikvjuFx2UK11UqzcZ2oB+h6N7Eq8i1CrdqUZkqCv2a13hgRjHKojqtuLqTe3qfncoi++wIIUTluJCUwaMf7cTa3JRFw1ry9DfhRa5Z/lIbgoNcWBkRw6TVx8lXVUOAUriadmHFXXv7qMX9sG+MKJvS/n7f09NYQgghqpfE6zkAuDtYEuB25yTZskyzlHRt4c/cS6uhROW6p6exhBBCVC+J17MBcLOzLFWSbFmmWR6UKRlRdjKyI4QQotIkpt0a2YEHK0lWVB0JdoQQQlQawzSW/a1FJDK9JCqaTGMJIYSoNIZpLHvZ2FVUHgl2hBBCVJqrhpEdCXZE5ZFgRwghRKUxBDsOsheaqDwS7AghhKg0iTKyI6qABDtCCCEqRe4NHckZuYAEO6JySbAjhBCiUiSl60d1zEwUnGwsqrg34kEiS8+FEEKUizhtFueTMghwtQUw/LlgWXnBFJabvSUmtxezEqICyciOEELco/z9/VEUpchrxIgRACxYsIBHH30UBwcHFEUhNTW1VO3GxsbyzDPP4OLigrW1NY0bN+bQoUOG8++99x716tXD1tYWJycnOnfuTHh40RpWha2MiKHdzO08vTCctjO203aG/s/tZm5nZUQMAIlp+mXnMoUlKpsEO0IIcY+KiIggLi7O8Nq6dSsA/fv3ByAzM5OuXbsyadKkUreZkpJCu3btMDc3Z+PGjZw4cYKPP/4YJycnwzV16tRh3rx5REVFsWfPHvz9/QkNDeXq1avFthmnzSJsdZShxpV68wX6ulcFlcxvjezISixRuWQaSwgh7lFubm5G72fOnElQUBAdO3YEYPTo0QDs3Lmz1G3OmjULHx8fFi9ebDgWEBBgdM3TTz9t9P6TTz5h0aJFHDt2jJCQkCJtnk/KMCrmebt8VeVCUqZREVAhKpOM7AghxH0gNzeXpUuXMnz4cBTl3+e7rFu3jpYtW9K/f3/c3d1p1qwZCxcuvOP3LliwAI1GQ9OmTYu9piBHpyQFlcyvXpdpLFE1JNgRQoj7wNq1a0lNTeW55577T+38888/zJ8/n9q1a7N582b+97//8cYbb7BkyRKj6zZs2ICdnR1WVlbMmTOHrVu34urqWmybXhprnGzMDe+Vm68CBZXMDUVAZRpLVDKZxhJCiPvAokWL6NatG97e3v+pHZ1OR8uWLZk+fToAzZo14/jx43z11VcMGzbMcF2nTp2IjIwkKSmJhQsXMmDAAMLDw3F3dy/SZnrODVIy8wBYOLQFjWpqOB6r5aXvD+Nobc6Alj6AbCgoqo6M7AghxD3u4sWL/PHHH7z44ov/uS0vLy8aNGhgdKx+/frExMQYHbO1taVWrVq0adOGRYsWYWZmxqJFi4pt82zCdUC/pPzxhp54aax5pLYbZiYKqVl5xKZmAbeKgErOjqhsEuwIIaqFuy2nTkhI4LnnnsPb2xsbGxu6du3K2bNn79jmo48+WuzS7+7duxuuWb16NaGhobi4uKAoCpGRkeV+b4sXL8bd3d3oe/+tdu3acfr0aaNjZ86cwc/P746f0+l05OTkFHvubEI6AHU97A3HrMxNqeupfx91WUu+TiUpvWD3ZJnGEpVLgh0hxH3vbsupVVWld+/e/PPPP/z6668cPXoUPz8/OnfuTEZGRontrl692mjp9/HjxzE1NTUs/QbIyMigffv2zJo1q0LuTafTsXjxYoYNG4aZmXHmQXx8PJGRkZw7dw6AqKgoIiMjSU5ONlwTEhLCvHnzDO/HjBnDgQMHmD59OufOnWPZsmUsWLDAsHdPRkYGkyZN4sCBA1y8eJHDhw8zfPhwYmNjje67sNM3R3bqFAp2AJrUdATgWKyW5Ixc8nUqigKudrJ7sqhkqlC1Wq0KqFqttqq7IoT4FyZMmKC2b9++xPOnT59WAfX48eOGY/n5+aqbm5u6cOHCUn/PnDlzVHt7ezU9Pb3IufPnz6uAevToUVVVVfXy5cvqkCFDVGdnZ9XKykpt1KiRGhERYbg+Pj5eHTZsmOrl5aVaW1urXbp0Uc+cOVOk3c2bN6uAevr0aXXBggVq+/btVUdHR9XR0VENCAgo2NLG6OXp6ana2Niojo6OqpWVlfrCCy8Ytbl+/Xq1UaNGqqWlpVqvXj11wYIFhnNZWVlqnz59VG9vb9XCwkL18vJSe/bsqR48eLDE5/LMNwdUvwkb1OXhF42OLwu/qPpN2KA+vXC/ejw2VfWbsEFtMXVLqZ61EKVR2t9vSVAWQtz31q1bR5cuXejfvz+7du2iRo0avPbaa7z00ksAhukXK6tb0ycmJiZYWlqyZ8+eUufCLFq0iEGDBmFre+el1gUjTZ06dWLjxo24ublx9uzZIiNN5ubm/Prrrzg4OPDJJ5/QuXNnTpw4YdR+aGgoqqrfxOb9999n8ODBtG3bFisrK2bNmkVycjJ///03NWrUAGDZsmW4u7sTGBhIVlYWc+bM4adVq+jz8jia1/XDS2PNk08+yZNPPlls362srFi9enWpnkeBMwUjO57GIzuNa2gAOHZZKxsKiqpVKaHXPU5GdoS4v1laWqqWlpZqWFiYeuTIEfXrr79Wrays1O+++05VVVXNzc1VfX191f79+6vJyclqTk6OOnPmTBVQQ0NDS/Ud4eHhKqCGh4cXe77wyE5ljTTduHFDtbe3V5csWVLiNd9uP64CqkuPt1Tbho+qdg6OxY40Xb9+XR0xYoRao0YN1crKSq1fv746f/78O35/x44dix1ZeuKJJ1RVVdXcG/mqfZPORc536dKl1PcoxJ2U9vdbcnaEEPc9nU5H8+bNmT59Os2aNePll1/mpZde4quvvgLA3Nyc1atXc+bMGZydnbGxsWHHjh1069YNE5PS/Wdw0aJFNG7cmIcffpg4bRb7opOI02YVe+3dNu6720hTaWVmZpKXl4ezs3Ox5y9e1TJu6idgYUPqzm9BMcOh9xR2HjhSpETE2LFj2bRpE0uXLuXkyZOMHj2akSNHsm7duhK/f/Xq1fwefoKaI36g5aRVRXKazE1NcLQ2xyqgBR3fW03NET/wvwV/sHz58lLfoxDlQYIdIcR9rzTLqVu0aEFkZCSpqanExcWxadMmrl27RmBg4F3bz8jIYMWKFbzwwgtGBS8LF7ks7G4b99WrVw9fX1/CwsJISUkhNzeXWbNmcfnyZeLi4kp93xMmTMDb25vOnTsbHS/YEDDAwwltxFpsa7fBTOOOa/fRmHvVQWfnTmhoKEFBQYbP7Nu3j2HDhvHoo4/i7+/Pyy+/TNOmTTl48GCJ3+/s7Mw1nQ2mdk40quXL1q1bsbGxMUpkdrIxRzEz50KWBaZ2TgT41jAKsoSoDBLsCCHue2VZTq3RaAw5NIcOHaJXr153bX/VqlXk5OTQuWc/o4KXhYtcFlYZI02T3pvK0mXLWfD9cqMRIri1IeC6LTuwDmhBxsldmLv4cnXtDC59PoSX+jxWpERE27ZtWbduHbGxsaiqyo4dOzhz5gyhoaF37MeZQiuxistpcrK1IDsmikufDyF24Sts/Goa165dK9U9ClFeJEFZCHHfGzNmDG3btmX69OkMGDCAgwcPsmDBAhYsWGC4ZtWqVbi5ueHr60tUVBSjRo2ia/ce2AU1J06bhZfGmmeffZYaNWowY8YM4rRZnE/KIMDVlkWLFtGlew92XsgqUvAyNzONLbvDcTPNBOD06dO4uLgYimsWtOPtH0TML7/cardWAyIjI9FqteTm5uLm5kbr1q1p2bLlXe/3mVFvs+zrT/EYOI2XNlxlhkUMHeq4GfrrpdFvCBgUFETt/rkcjdpK+rEtOLTux+NPvUhf31zeeOMNLCwsDLsmf/7557z88svUrFkTUzMzTE1MWLhwIR06dLhjX07H64Md5eo5jh8/XmTjwV5PPsEJi/qYOXpwIyWOc3/9RLdu3di/fz+mpqZ3vVchyoOiquodatU+GNLS0tBoNGi1WhwcHKq6O0I8UGJjY5kwYQIbN24kMzOTWrVqsXjxYsOPfklFL2fPns348eMN7zds2EBYWBhnz57F0dERMzMz0tLSAGjYsCFNmzblt99+IyEhAS8vL5xrBnEyJoHcxPOouVks2naM7z94HTc3N+LTb7Bv5zbUGzmY2rtyI+UKngOnYunfjIuzil/FdDtfX19m//wnYauj0B75ndQ/l6LmpIOZJeauvji1G8yn459nYCtfAKZOncqUKVOwtrYmKyuLlJQUHB0di7T7zgcfMv3D6XgM+ADLGvUMxxUFVBVMFJjRtzEDW/ly7HIqPeft5eKsJ3F080Qz/BssTE3YMf5RZk2ZQEREBPv37wfgo48+4pPP56NrNQQTB3dyLx8nY+9S1v+6tsg0WQFVVWk+dSspmXm0uvwzp48d5tixY0bX5OtUGr+3mczcfADmPuFFn44t+OOPP4qtoC5EWZT291tGdoQQVeZuS7SBIjksGzdu5IUXXqBfv35GxwuWU8dps/hh5Wq8nWxo/VAjEtKyWLhoMd9+8wVHjx6lYcOGxGmzaNRvFNZBvlgHtSJ11xI+WH+SHz5fzFNdOpDpWg/3/u9hYqPhRsoVzBy9MHfyAqDmiB+Mvjfrn0Nc2/QZP207SPvmDfHSWBMREUHbtm3537jJWNdrT17KFXS5WTh2fA6bWq1JP76NhJ8/YGRGKn7TXyLhwlnmzJlD3foN6NS9L199NA3AaKQJ4O33pzFr2ge4PDkOM40H+ekpACgWVphYWKPLzSZl/0rGxrYhcHx3Fm8/RtLv+tGtxx5pi0WgC/v/ucbsTaewdPXhwsWfidNmcepyEmGTJuHSaxJWQa0AsHAPIDfhPB/OnFVisJOUnktKZh5qXjZb1q/mgw8+KHKNqYlCI28NBy/oNzqs6euPq6sr586dk2BHVBoJdoQQVWbWrFn4+PiwePFiw7GC6Z8Cnp6eRu9//fVXOnXqVGxi8cqImJs5NY6YKNBHzWDN0Vh0Dp3JN/2Gz5b/xtfTGnI+KQP7lvpcnewY/UiETlV58qW3yDF1wPOJ0YY2zR2Nv9/UTh+ITe5eH29Ha/r1+wIr38aM35yAyZaEm6MqrZj+5RLenjSJlL3LMdN44Nz5Fewf6gqAU4dnuR6xluTt3/BIqy/w9vLi0T7PcNTlMdZcPgnA6iOXiImJMeTwrIyIYdYnn5N/I4+ktTOM+qRpNxjH9kNQTEzIS75MwpoPabciDKzsMfOoRfO2HUlMTOST0Dr0/2o/v0ZeIXnbXnIVDW1nbCc/J5MbeXmoGI+iqYoJGdm5Jf79FdTEsr58kIScHJ555plir7Mwu5WH1Pf/1nPt2jW8vLxKbFeI8ibBjhCiytxtM8DbJSQk8NtvvxlWNRUWp80qkjz8y5FYVF0+maf2oMvL5vcEB+K0WeTd0BXbfta5cKwCmnN17QyyLx3H1M4F+2ZPGIKUAqaKwhNNvLiamEBWdASu3ccYvnPS6uN0qOPGMwP6MP8fTZEcn4L+qKoOzyGzWfv2ABLSsnljeSSFL31//QnW/bCGjNx8/jgRz8Rfoqj5v2+L9NmEWxvYKGYWuPd5GxMFvnqmOS//cAQ7SzM+7aqhU4dHWP3t5+SleJMbd4b0vzbh3GUkKmBiaYOlTyNSdn6LYm6BqYM7OZeOk/H3dvrOmm34rttHmgrKRKRGbqF37964uLgY9S09PZ3xkyazLaUmpnZO5KXEkbpzMWaOXjRpc+dcICHKkwQ7QogqU7BEe+zYsUyaNImIiIgiibOFLVmyBHt7e/r27Vvk3PmkDKPAIvfqBeJ/GId6IxfFwhr3Pm9j6uLD2YTrzNh4qtj+5KXGk3f0dxxa9cYjeAA5cWdJ2baAVkFunHVoSb6qYqooTO/bCC+NNdOmL8bEwhqbOm0NbeSrKheSMgkOcuHxBh5s/jtBf/zqRRJ+HEdebo6hPxauvgz4+kCxfVFV6PXFvjs+v8nd6/NEEy92n7lqFOi19HdiX7R+2ujxBh60D36INWvWMOrNt7gSfQ4zjQdOj72EXcNOhrbcek4gZdcSktZ/hC47HVMHd/y7DGfcqNcN1xQeaQL9Sqy8a5dJOBPJC/NuBUUFTE1NOXI0ksQji9FlZ2Bq54x1QDMcH3mGuOv5+Lvf8faEKDcS7AghqoxOp6Nly5ZMnz4dgGbNmnH8+HG++uqrYoOdb7/9liFDhhRZag0Q4GpcwsHcuQZez3+GLieTzNN7SPptDh5Pz2TJfndOxV/HxdaC7194mJ07bjByuT7BF1XF0rMWTh2HYQIsHPMUq7/I58SRjezZNJ4LSZn4u9rgpbEGYPOaFdg1eBTF7FZhS1MF/F1tALh4Tb9C6+UOAQxp1Z68sFCiYxP4cfkq1q78HFPNNCxcff/VsysYXfLSWDOwlS8d6rjx+7E4pv52koPnU/jrkhaAHk3100VPPvkkLR4Jod3M7UVGm0A/Pefx5BhWvxZMTHIWE345RmZuPh9vPcMjtV0JcLVl+a8bOZ+UYVi9djw2DXOXmny37zyPB/sXadPa2pq1G34v8p2mimJ4RkJUBgl2hBBVpqTNAH/55Zci1/7555+cPn2alStXGi0LLwg88m/7BTczs+CpkFasPXoFS89a5Mad5fqhdWy7GVw80diLht4art6s3/Rujwa88q0T5q6+htGb7k28iXmoMZs2/IqXxtrwXQX9iT57hllLP+Wr4/mGH/OXOwTipbHmXOJ1TsVfx9xU4bVHa+FoYwFuGmrVqkWXju1oFXWUU4fW4dJ1ZKmfl4minyorPLpkeJYaa154JJAdp6+y51wSOTen6uJSs42umdG3MZNWHydfvZmhc3MVV0GbTX2caOrjRFZePm/9fIz5O6OZvzPaqB8K0NRHQ1SsPqB6f93fWJmZGFaWFXb7dxbXdyEqmgQ7QjxA7rbM+7nnniuSD9OlSxc2bdpUYpvz589n/vz5XLhwAdAv854yZQrdunUzXBMfH8/48ePZunUr169fp27durz99ttl2gxw0aJFtGjRglO5TvS5OVJQeJn18oP6nYxb+jvy5uP1DCMw47rU5fCFFPovV1Hz8wztLQuP4bVOt3YQ7tvch/WhnTh/IYaNEzsZfozv1p+3hnRlqDaLib8cY9eZJOLT9KUg1v+lX0X2SG03faBzG0tTBQr1B/T5N6NCajOpYKSpEFNFYfVrwWTm6oxGlworKGNR2JRf/+ax+u6G6wtGgQpGqYAiI1YA7WsZ598UpgKRN0eOwDhXqbh+3f6dEuiIyiY7KAvxgChY5m1ubs7GjRs5ceJEkfpIAF27diUuLs7wulsdo5o1azJz5kwOHz7MoUOHeOyxx+jVqxd///234Zpnn32W06dPs27dOqKioujbty8DBgzgiSee4MCBA0yfPp1z586xbNkyFixYwIgRI4y+Iy0tjVWrVtF/yLPF7mDcrsOjfP75PACGtwtk3TcfcfavCC5cuEBSzDlWzp9FdkwUtg0eNbSZez2ZLbvDOXfuHABRUVE81bsnf0ceYvEXc0rVn4Jq6V4aa8Y+XheA347F0eHRTnz15RcAPNnEi7CwMHbv3s2FCxeIiooiLCyMfXt288Yrz2N6M6pRM1J4pbEJbmoqAM/XN+VG4j/kZ103GnUJDnIpMVi4PW8JbuUQFealsTa0U/jPhV24ZvyZuynue0r6TiEqm4zsCPGAKM0ybwBLS8siy73vpEePHkbvP/zwQ+bPn8+BAwdo2LAhoK+7NH/+fB5++GEA3nnnHebMmUN2djZr1qwhLCyMDz74gICAAObOncuQIUOM2lyxYgWqqtKs05PoVpw0Opevqpw8cxZdLW/q2FvyeAMPfklM5NlnnyUuLg6NRkPdBo3wHPgBlv7NDJ9Lj9zIc18sM7wv2Cl41KhRLF++vFT9GTx4sOFYUx9HGtfQT+1E/n0ak7peuDc04fEGHvx2W3+aNGnC5s2befzxxxmpzeJCUiY/L/yEic9MN7T3/iv6+lJvz/qcEa/0KlWQEOBqa5jqKvBv82OKa+tOJA9H3MtkB2VkB2XxYGjQoAFdunTh8uXLJS7zfu6551i7di0WFhY4OTnx2GOPMW3atCJLikuSn5/PqlWrGDZsGEePHjXk44SGhmJhYcH333+Po6MjP/30Ey+88AJ//fUXtWrVKvU9rDkay5iVkUWO13K35VxiBm88VouxoXWL/ezKiJgieSPF5Zj8FysOxjBxdZThfcc6riwZ3rpcv+NuyvM+C7d1e35P72berD16pUKfpxB3U9rfbwl2kGBHPBgKVjCNHTuW/v37ExERwahRo4xWPq1YsQIbGxsCAgKIjo5m0qRJ2NnZ3bWOUVRUFMHBwWRnZ2NnZ8eyZct44oknDOdTU1MZOHAgW7ZswczMDBsbG1atWnXXIpOFaTPz6DJ3N/Fp2ShAcf/hCutWj1c6BhVzRi/u5ihKReWNZObeoNkHWw3JwQows1/jSg8CyvM+C7cFxvk9Ff08hbgbCXbKQIId8SCwsLCgZcuW7Nt3a++WN954w6g+0u3++ecfgoKC7lrHKDc3l5iYGLRaLT///DPffPMNu3btMozsvP766xw8eJDp06fj6urK2rVrmTNnDn/++SeNGze+a9/jtFmMX3WMPeeSCHS1ZdGwVvx9JZWRyyONrjNVFPYUSi6ubHHaLNrO2G4UiFV1n4Sozkr7+y0JykI8IEpa5h0TE1PiZwIDAw11jO7EwsKCWrVq0aJFC2bMmEHTpk359NNPAYiOjmbevHl8++23hISE0LRpU959911atmzJF198cdd+r4yIoe2M7ew5p19l1LWRJwFutjjbWRa59m5JshXtfFJGkRGnqu6TEEKCHSEeGGVZ5l3g8uXL/6qOkU6nIydHvwQ7M1P/Q194513Q766r0xVftqFAQQmIwgHE17v+IU6bZUigNWqzipNk78U+CSEk2BHigTFmzJg7LvNOT09n/PjxHDhwgAsXLrBt2zZ69eql3wSvSxdDOyEhIUybNo1nnnkGFxcXzM3NCQwMNCwrDwsLY8eOHWzbtg1bW1s6dOiAtbU1gwcP5uDBg0RHR/Pxxx+zdetWevfubWi3YI+YOG0WM2fORFEURo0eXWQ1UOblEzzZNZRa3q7Efz6IhB8noMvLuSc2qyvYQK9gOfm90CchhOTsAJKzIx4cGzZsICwsjLNnzxIQEMDYsWMNq7GysrLo3bs3R48eJTU1FW9vb0JDQ5k6dSoeHh6GNnx8fLh+/Tp9+vThf//7Hx999BG7du0iJSUFR0dHmjRpQqtWrQgJCSEwMJCsrCzee+891q5di6Ojo2Ezw3HjxjF06FCgcLVyyIs/Q96WT/Byc6ZZ6/bsdL6V6JwTe5KEVe8yceJEhvTvi5mZGTv3R1Dv4ceoU8PpngkqJHFXiMohCcplIMGOeNAVV36hJCPHjOPPPXvY9McOw4qcu3224N+x4hKd47RZhtpJutws4r4bhWuX1wi4tBkLj0Au1O5vuDb+hzd5omsoqxd99t9vWghx35MEZSGqkdjYWMO0kbW1NY0bN+bQoUMA5OXlMWHCBBo3boytrS3e3t48++yzXLly5Y5tvvfeeyiKgqIoeDva0K6WG36BdVgZcSthOTs7mxEjRuDi4oKdnR2tH+vG10uWcx5ParV+HBuNM/51GtFjxPu0m7nd6LMFcnNzWbBgARqNhqZNmxY5X3jX3+St87EOaoWl30Nk5N7geEHtpZ4N+KJvEDlXTtPpodq0bdsWDw8POnbsyJ49e/7tYxVCPCBkB2UhqtjdRkYKyjx06tSJjRs34ubmxtmzZw1lHjIzMzly5AiTJ0+madOmpKSkMGrUKHr27GkIiEpSt34DMkPCMIzvmpgQtjqKep72ZOTm8/X0MHZs3cyqVavIM7Wk19MvcCMljutHf8ehVW9sgweQE3eWlG0LUEzNmYRiqI+0YcMGBg0aRGZmJl5eXmzduhVXV9cifSioVp5xYhe58dF4DZsDwNXrOeTa6qjjYcfQNv4cPBgO6IO0jz76iIceeojvv/+ekJAQjh8/Tu3atf/tX4EQopqTYEeIKlQ4V6VwUcvC7lbmQaPRsHXrVqPPzJs3j4cffpiYmBh8fUve0C4fBRNb49pYOhV6fbEPXU4Gl777jtEffsZjjz3GvugkXJ4YzZVv/oe5kzdOHfUbEVp4BJGXdJHrkb9j1ziEC0mZeGms6dSpE5GRkSQlJbFw4UIGDBhAeHg47u7uRt939XoON9KukrxtIR4Dp6KY6YtmJmfkYmELD/s7Y2KiGFZuvfLKKzz//PMANGvWjG3btvHtt98yY8aMUj1zIcSDR6axhKgiBcuqjYtaRhGnzTK6bt26dbRs2ZL+/fvj7u5Os2bNWLhw4R3b1mq1KIqCo6PjHa+7fOE8l794ltivXuDq+v/jRlqi4VxO/DnQ3WBNggtx2ixSM3Ixd/EBxQTF3HiPG3MXH/LTrmKiYFhmbWtrS61atWjTpg2LFi3CzMyMRYsWFenDjwdiyI0/hy4zlYTvR3Nxdk8uzu5JzqXjXD+8ng/7PcTl5HTD8vey7hUkhBAysiNEFSm+QjUcvpCMs52lYVrrn3/+Yf78+YwdO5ZJkyYRERHBG2+8gYWFhaHMQ4E4bRanYq8xdtx4Bg8eXGLCXpw2C9ua9ajT/y2umrqQn56Mdu9y4n+cgPfwLzCxtEGXkQKmZmBpy+ojsSwNvwiAiaUN+enXDPWR1hyNJS85FjMHdx7yceR8UgZAkSm5wnvvFNBm5fHrX7FY+TVl+aY93NDpeOvnYwBc+/1TzF1q4tC6H5dScmgT6I+3t3exewV169atbA9fCPFAkWBHiCpSUlXpghIIBdNaOp2Oli1bMn26viJ2s2bNOH78uFFNK9BPiU1cdZSENdPJv36dMXMmFfu9t6bO7MGrJRoLU356uQ1xV5+hX8eHyDq9B9smxjWr/m+zPsBwtDbHzLcG/5w9ywDzCF5oPoTAtKuMnbsF+86vcSQmlUFf7CLtwErGvfg0Q0OakZSUxBdffEFsbCz9+99aWRUSEoJX0w5kW7Skvq8HA0PbEp+WjdXuNHQqKOaWmFjZY+0egL+rDYqiMH78eN59912aNm3KQw89xJIlSzh16hQ///zzf/3rEEJUYzKNJUQV8dJY82xwybsX66e1juPu4XnXqZs4bZY+0Fk7kxvaRNwHTmXq5gtsOHbFaFrs9qkzgOy8fFztLQltFki9unVp534DU+VmLk/+DXTZ6YZr07LzyM7MZPjw59mw5mcaNWrEvI9n8eGs/8OuYScAFBMTcq9d5r1RL1K7Th26PNGd2PhE/vzzTxo2bGho68zZc+w6Fg3AM218URSlyKZ8ym2b8o0ePZqwsDDGjBlD06ZN2bZtG1u3biUoqOTin0IIISM7QlQhGwv9v4KP1nHlqZY+jFx21Oh8vqrSoFmru5Z5OBuXqg90Uq7gMXgGptYO6ICRy44aJT4XN3WmqvpK1vam+URHRzN06FDmD+vEqn01GPXTu2Rd/Avbuu0AyEm6zJXLl3jxxRf55ptvDG3si05i3kL9ainFzAL3Pm/r/4y+OvlZBf7Bg1Y3r18ZEYPZkC8Nq8B0hbb7GtjKlw513Ljw4s5iN+WbOHEiEydOLO0jFkIIGdkRoiodvpgCQLfGXrTwcyq2rtKYMaPvWOYhLy+PSSOeJzf+HK49xoFOR356CvnpKaj5eYYRokc6dmLH6u8p+IqU7YvIjolCp03kavQx+vTpg6mpKYMHD8ZLY03/tnWwb/o4Kdu/IfviMXLiz3Ft41xaPNyaNm3aGPWzuJpQgKGmVUEf4rRZt+pdFQq6Plh/0mgEyktjTXCQi+w+LIQoFzKyI0QVycvX8dflVABa+DkZpnAm/qIvfKkA0/s2olsrX9asWUNYWBgffPABAQEBzJ07lyFDhgD6DQf3bt8MQNziN4y+w2PwdKx8m5CvqpyLjub85XhU67oA3LieRNL6/0PJSee19W60b9+eAwcO4ObmBugDji8+m8vIUWO5unY6an4eLds9yrrl3xW5l4K+T1p9nHxVNYzoFFZQ/TtfpxaTmK0alqwLIUR5k2BHiCpy4koa2Xk6NNbmBLraAfopnKvXc/hoyxlaBzgb9tx58sknefLJJ4ttR2vqiN+EDZgosOylNiRdz+GNFUeLBBSf/7qXWZtOwfVcejTx4umX1ty1dtPQ9nXovH1Vqeo8GaafkjKxsTChz5f7ivRBp6rM+eN0kc9KZXAhREWSYEeIKlIwhdXCzwmTQnNA7Wu78dGWM5xJTEdVVRSlmPmhm+K0WUz+9TgAvZvVoE2gCwAZuTcMoywFxq06ZvhzS38ngoNcStVPL411qUdcCl9beKSnwJBvwg1/Lhj9kcrgQoiKJsGOEFXkcMytYKew+l72mJsqJGfkEpuaRU2n4kc8VkbEMLFQ7kstNzvDucKjLHn5Op799qDRZz9Yf5LQhp4VGmAU7kPMtQwmrI4yOq8A855uRnO/e6dauRCieipzgvKOHTsqoh9CVDuFC20WvOrVq2c4v+/I3ySunsY7/drg4ODAgAEDSEhIwNLMlLqe9gAcu6w1anP37t306NEDTy8vBj3sR8bp/YZzH285Q5w2i/fee4969epRy9uVJ1rW4s3nnyLnivHUUUGOTEUrSDT2cSkasOkAZ1tLCXSEEBWuzMFO165dCQoKYtq0aVy6dKki+iREtdGwYUPi4uIMr4IK3edikzi+6C0URWHzH1vZu3cvubm59OjRA51OR+MajkDRYCcjI4OmTZsyasrMIt9VEMDUqVOHefPmERUVxZ49e6gdFEDCT5PJz7zVVmXnyBS3WkvydIQQlaXMwU5sbCwjR47k559/JjAwkC5duvDTTz+Rm5tbEf0T4r5mZmaGp6en4VVQ9XvZuq3c0CbS4cV3ebh5Mxo3bsySJUs4dOgQ27dvp2lNDQDHbq7WKtCtWzemTZvGkz16FfmuguDh6aefpnPnzgQGBtKwYUO+mvcZak4m+VcvGK6r7ByZ2zcLlDwdIURlKnOw4+rqypgxY4iMjCQ8PJw6derw2muv4e3tzRtvvMFff/1VEf0UotLcbfopPj6eoUOH4unpia2tLc2bN+eXX34ptq2zZ8/i7e2Nt7c3NWrUwMPDA0VR2LpdPx3cIvBWBXBHR0dUVeXxxx9ncGs/Ls56kuUvBzN79myjNlVVZc4fZ42OlRQ85ObmsmDBAjQaDVs+HMryl9qwZ2KnIpXVK8PAVr7smdipSvsghHgw/adNBZs3b05YWBgjR44kPT2db7/9lhYtWvDII4/w999/l1cfhah0JU0/ATz77LOcPn2adevWERUVRd++fRkwYABHjxrvfty6dWu+++47Nm3axGuvvYaiKOh0OgBSzV1RzK2I+OlzMjMzycjIYPjw4QA888wzxFyOxePJ0YDCw51uFbmM02Yx4/eTbD+lr04+5vE6xQYPGzZswM7ODisrK+bMmcPWrVtpFOhT5Rv1yWaBQoiq8K+Cnby8PH7++WeeeOIJ/Pz82Lx5M/PmzSMhIYFz587h5+dnVPDvv4iNjeWZZ57BxcUFa2trGjduzKFDhwznVVVlypQpeHl5YW1tTefOnTl79uwdWhTi7kqafgLYt28fr7/+Og8//DCBgYG88847ODo6cvjwYaM2unXrRv/+/WnSpAnvvPMOx48fN0z3JmSb4NZ7Itu3bMTOzg6NRkNubi7NmzfHzs4OnxreKDGHsfJrjNbcGdCvvmo3czsL/jxv+I76Xg7FBg+dOnUiMjKSffv20bVrVwYMGEBiYmJFPS4hhLinlTnYef311/Hy8uKVV16hTp06HD16lP379/Piiy9ia2uLv78/H330EadOnfrPnUtJSaFdu3aYm5uzceNGTpw4wccff4yT062lurNnz+azzz7jq6++Ijw8HFtbW7p06UJ2dvZ//n7x4CqYfgoMDGTIkCFGRTfbtm3LypUrSU5ORqfTsWLFCrKzs3n00Ufv2KajoyMBQbUM760DmlPjlW/wff1Hjkdf4ocffiA2NpbAwEASEhJI/Hs/dk1COXZZW2wBT4DkjJxiv8vW1pZatWrRpk0bFi1ahJmZGYsWLfrXz0MIIe5nZd5n58SJE3z++ef07dsXS0vLYq9xdXUtlyXqs2bNwsfHh8WLFxuOBQQEGP6sqipz587lnXfeoVcvfcLm999/j4eHB2vXrmXQoEH/uQ/iwVMw/VS3bl3i4uJ4//33eeSRRzh+/Dj29vb89NNPDBw4EBcXF8zMzLCxsWHNmjXUqlXrju2mp6fzzz//FD1h7UDKDQu2b99OYmIiPXv2ZMmSJdjY2mJTpy3HLqcWW8ATIPF66RYG6HQ6cnKKD4yEEKK6K/PIzrZt2xg8eHCJgQ7opwA6duz4nzoGsG7dOlq2bEn//v1xd3enWbNmLFy40HD+/PnzxMfH07lzZ8MxjUZD69at2b9/f3FNApCTk0NaWprRS4gChaefunTpwu+//05qaio//fQTAJMnTyY1NZU//viDQ4cOMXbsWAYMGEBUlPGmeePGjWPXrl1cuHCBffv20adPHxTl1r9y6ce2khN7Cl1qPIe3/Ur//v0ZM2YMdevW5dtvv6X3UwNJ/Pk9/ly71FBoSpebRW7CP+Qm6IOmnOQ4IiMjDSNPGRkZTJo0iQMHDnDx4kUOHz7M8OHDiY2NLbepZSGEuO+oZTR9+nR10aJFRY4vWrRInTlzZlmbuyNLS0vV0tJSDQsLU48cOaJ+/fXXqpWVlfrdd9+pqqqqe/fuVQH1ypUrRp/r37+/OmDAgBLbfffdd1X0Px9GL61WW679F9VHy5Yt1YkTJ6rnzp1TAfX48eNG50NCQtRXXnnF6NjAgQNVLy8v1cLCQq1Ro4Y6cOBA9bEpK1RAdevzturQ+inV1NZRNTUzV2vXrq1+/PHHqk6nU3fv3q0C6uEjR1VzjbuqaTdYbTF1i+o3YYPqMXh6sf/sDhs2TFVVVc3KylL79Omjent7qxYWFqqXl5fas2dP9eDBg5X1qIQQotJotdpS/X6XeRrr66+/ZtmyZUWON2zYkEGDBjFhwoR/F3UVQ6fT0bJlS6ZPnw5As2bNOH78OF999RXDhg371+2GhYUxduxYw/u0tDR8fHz+c39F9ZSenk50dDRDhw4lM1O/67CJifGgqKmpqWGlVYEVK1YYvd/ydzwv/6BPYp78ZH1adhxTbHHNRYsW0aJFC5o3e4h276zkfFIGSen66ao3h/Wh/dSXSyzKaWVlxerVq//bDQshRDVT5mms+Ph4vLy8ihx3c3MjLi6uXDpVwMvLiwYNGhgdq1+/vmHI3tPTE4CEhASjaxISEgznimNpaYmDg4PRS4gCxU0/mZqaMnjwYH0Zhlq1eOWVVzh48CDR0dF8/PHHbN26ld69exvaCAkJYd68eYb3Zy4lMmHBOsP00w1tItbXL5GnvWr03WlpaaxatYoXX3yROG0WF5IyjM5/tfOfu1YfF0IIYazMwY6Pjw979+4tcnzv3r14e3uXS6cKtGvXjtOnjWv6nDlzBj8/P0CfrOzp6cm2bdsM59PS0ggPDyc4OLhc+yIeHJcvX2bw4MHUrVuXAQMG4OLiwoEDB3Bzc8Pc3Jzff/8dNzc3evToQZMmTfj+++9ZsmQJTzzxhKGN6OhokpKSAP2S8Q4TFhH52SvEffcGAGPHjqVZs2ZMmTLF6LtXrFiBqqoMHjyY80kZ3J6TXFk1rYQQojop8zTWSy+9xOjRo8nLy+Oxxx4D9EnLb731Fm+++Wa5dm7MmDG0bduW6dOnM2DAAA4ePMiCBQtYsGABAIqiMHr0aKZNm0bt2rUJCAhg8uTJeHt7G/2/bCHK4vbpp9vVrl27xB2TC+z/6yTnkzI4+M81Jv4ShZVvE/wmbAD0ux3vmdip2NGZl19+mZdffhmAALIwUTBahSX1pIQQouzKHOyMHz+ea9eu8dprrxk2SLOysmLChAmEhYWVa+datWrFmjVrCAsL44MPPiAgIIC5c+cyZMgQwzVvvfUWGRkZvPzyy6SmptK+fXs2bdqElZVVufZFiNJaGRFT7J44BQpGZ+42FVVQT2rS6uPkq6rUkxJCiH9JUVW1hP8k31l6ejonT57E2tqa2rVr33Ep+r0uLS0NjUaDVquV/B3xn8Rps2g3c3uJgQ7ceWSnpDYvJGVKro4QQtymtL/fZR7ZKWBnZ0erVq3+7ceFqJZK2vyvYDrq34zOeGmsJcgRQoj/4F8FO4cOHeKnn34iJibGMJVVQJa9igdZgKttsXk2q18LJjNXJ6MzQghRBcq8GmvFihW0bduWkydPsmbNGvLy8vj777/Zvn07Go2mIvooRJWJ02axLzqJOG1Wqa730ljzSodAw3tTBab3bURTHyep9i2EEFWkzCM706dPZ86cOYwYMQJ7e3s+/fRTAgICeOWVV4rdf0eI+1XhRGMTBWb0bczAVr53/ZzGxgKAVv5OfDa4mQQ4QghRxco8shMdHU337t0BsLCwICMjA0VRGDNmjGFJuBD3u9urjOtUCFsdxV+XUu460nP4YgoAoQ08JdARQoh7QJlHdpycnLh+/ToANWrU4Pjx4zRu3JjU1FTDVvpC3O+KSzTWqdD7i32olDzSo6oqR24GO839nCqpt0IIIe6kzCM7HTp0YOvWrQD079+fUaNG8dJLLzF48GBCQkLKvYNCVIUAV1uUYo4XxD86FSatPl5khOfitUyuZeRiYWZCoxqyjYEQQtwLyjyyM2/ePLKzswF4++23MTc3Z9++ffTr14933nmn3DsoRFXw0ljT1EdD5CUtAAqUWLqh8FTVoZujOk1qaLA0M62k3gohhLiTMgU7N27cYMOGDXTp0gXQV36eOHFihXRMiKqWlnUDgAld69I2yIU+X+4zmtoyUShSuqEgX6eFTGEJIcQ9o0zTWGZmZrz66quGkR0hqqu07Dz+uVlxfGArX5r6ODGjb2NMCs1t1fWw53xShtFUluTrCCHEvafMOTsPP/wwkZGRFdAVIf6b9957D0VRjF716tUznF+wYAGPPvooDg4OKIpCampqiW0dj9VPX3naqHzw9lv4+fnxXIe6OG+fyvN18lGAk/HXGfzVXup3GYpvrXrY2tqybUofkjZ8jLd56fblEUIIUfHKnLPz2muvMXbsWC5dukSLFi2wtbU1Ot+kSZNy65wQZdWwYUP++OMPw3szs1v/iGdmZtK1a1e6du1616K1xy7rg52k3z9j6/VYfvjhB7y9vVm6dCkfjxmK5pnPMLV3Rb2RQ058NDkP9WHm7K5MX3OIjF3f8PzT/Tl06FDF3KQQQogyKXMhUBOTooNBiqKgqiqKopCfn19unassUgi0enjvvfdYu3btXUced+7cSadOnUhJScHR0bHYa0b8eIT1Ry4QO3cA69b9athbCqBeo6bEOdTHqcNQo8/0bebN6qNXaOuQyvK3n+HixYv4+t59E0IhhBD/ToUVAj1//vx/6pgQFens2bN4e3tjZWVFcHAwM2bM+FcBx7HYVNDlo9PlY2VlZXTOwd6Wi5f/LvKZiAvJALha3kBRlBIDKSGEEJWrzMGOn59fRfRDiP+sdevWfPfdd9StW5e4uDjef/99HnnkEY4fP469vX2p20nOyOVSchYmljY83LoNU6dOpX79+nh4eLB8+XIOHwzHo6Y/popCfqGB0Usp2ag3cvn6/6bSNrSnjBIKIcQ9oszBzvfff3/H888+++y/7owQ/0W3bt0Mf27SpAmtW7fGz8+Pn376iRdeeKHU7UTdTE4OdLVl4Y9LGT58ODVq1MDU1JTmzZszePBgDh8+zPaJnbiQlMmNfB1Dvz2Imn+Dq7/OBCC2wTPEabOkXIQQQtwDyhzsjBo1yuh9Xl4emZmZWFhYYGNjI8GOuGc4OjpSp04dzp07V6bPHbuUCkCTmhqCgoLYtWsXGRkZpKWl4eXlxcCBAwkMDMRLY42Xxpp90UmGQOeGNhGPwdNRLayLbDgohBCiapR56XlKSorRKz09ndOnT9O+fXuWL19eEX0U4l9JT08nOjoaLy+vMn3u2M2RncY1HQ3HbG1t8fLyIiUlhc2bN9OrVy/DuZoaC5J+ncmNlCt4DPoQU2sHTBWlyIaDQgghqkaZR3aKU7t2bWbOnMkzzzzDqVOnyqNJIcps3Lhx9OjRAz8/P65cucK7776LqakpgwcPBiA+Pp74+HjDSE9UVBT29vb4+vri7OwMQEhICNE2DaBhV5rU1LB582ZUVaVu3bqcO3eO8ePHU69ePZ5//nlAP7L5+gtDsdJexKnHJNDpUDNSeOuJerhYS7kIIYS4F5RLsAP6/UyuXLlSXs0JUWaXL19m8ODBXLt2DTc3N9q3b8+BAwdwc3MD4KuvvuL99983XN+hQwcAFi9ezHPPPQfA2XPnSPPxwFmBht4ObDioJSwsjMuXL+Ps7Ey/fv348MMPMTc3ByA2NpZ169bpG1z8uqHtV+dB3R07ePTRRyv+xoUQQtxRmffZMfyH/SZVVYmLi2PevHn4+PiwcePGcu1gZZB9dkSBnyIu8dYvxwh0tWX7uEerujtCCCHuoML22endu7fRe0VRcHNz47HHHuPjjz8uc0eFuFesjIhh4i9RAPyTlMHKiBgGtpJNAYUQ4n5X5mBHp9NVRD+EqFJx2izCVkdReJhz0urjdKjjJiuqhBDiPlfm1VhCVEfnkzLQ3Tahm6+qXEjKrJoOCSGEKDdlDnb69evHrFmzihyfPXs2/fv3L5dOCVHZlGKOyfJxIYSoHsoc7OzevZsnnniiyPFu3bqxe/fucumUEJXpUnIG76w5bnTMVFGY3reRTGEJIUQ1UOacnfT0dCwsLIocNzc3Jy0trVw6JURlKUhKLpjBGtmpFu1queLvaiOBjhBCVBNlHtlp3LgxK1euLHJ8xYoVNGjQoFw6JURliNNmMfG2pOT5O6Ml0BFCiGqmzCM7kydPpm/fvkRHR/PYY48BsG3bNpYvX86qVavKvYNCVJTzSRncvstUQVKyBDtCCFF9lDnY6dGjB2vXrmX69On8/PPPWFtb06RJE/744w86duxYEX0UokKk59wockySkoUQovr5V+UiunfvTvfu3cu7L0JUqh/2XwT0K7FUJClZCCGqqzIHOxEREeh0Olq3bm10PDw8HFNTU1q2bFlunROiohw8n8yfZ5MwM1FY+Uobcm+okqsjhBDVVJkTlEeMGMGlS5eKHI+NjWXEiBHl0ikh/qs4bRb7opOI02YVOXclNZMpv+qXmvdv6UMLP2eCg1wk0BFCiGqqzCM7J06coHnz5kWON2vWjBMnTpRLp4T4L1ZGxBC2OgqdCiYKzOjb2FDjamVEjH4F1s3EZH8Xyc8RQojqrswjO5aWliQkJBQ5HhcXh5nZv0oBEqLcFNS4Kij9oFP1Na7itFm36l8VWoE1e9PpYkd/hBBCVB9lDnZCQ0MJCwtDq9UajqWmpjJp0iQef/zxcu2cEGV1pxpXUv9KCCEeTGUeivnoo4/o0KEDfn5+NGvWDIDIyEg8PDz44Ycfyr2DQpRFgKttkWMmCvi72nApuWhQI0vNhRCi+itzsFOjRg2OHTvGjz/+yF9//YW1tTXPP/88gwcPxtzcvCL6KESpeWmscbOz4Gp6ruHYI7Vdcbe3YsSPR4yulaXmQgjxYFBU9fY9ZB88aWlpaDQatFotDg4OVd0d8R+kZOTSbOpWAF7pEMjXu/8BoG2QC/uir2Fnacb3w1uRI0vNhRDivlfa3+8y5+wUOHHiBJs2bWLdunVGLyH+i5kzZ6IoCqNHjy5yTlVVunXrhqIorF27ttjPH72Ugpp/AzV8KcsmDODynKe4/MWzrPt0EjeuX6NrI0+a31xq/srQgfj6+mJlZYWXlxdDhw7lypUrFXuDQgghKl2Zp7H++ecf+vTpQ1RUFIqiUDAwpCgKAPn5+eXbQ/HAiIiI4Ouvv6ZJkybFnp87d67hn7OSHL6YgnojB5Nr5xn55gSm7ssgPyud5G0LuLp6Kmsc5vJmaB28NNZ06tSJSZMm4eXlRWxsLOPGjeOpp55i3759FXF7QgghqkiZR3ZGjRpFQEAAiYmJ2NjY8Pfff7N7925atmzJzp07K6CL4kGQnp7OkCFDWLhwIU5OTkXOR0ZG8vHHH/Ptt9/esZ3DF1MwsbRl5jc/0fiRrpg518SyRj2cH3+V3Phz5GgTDauvxowZQ5s2bfDz86Nt27ZMnDiRAwcOkJeXVyH3KIQQomqUOdjZv38/H3zwAa6urpiYmGBiYkL79u2ZMWMGb7zxRkX0UTwARowYQffu3encuXORc5mZmTz99NN88cUXeHp6lthGXr6OyEupALTwcyLA1RaTmwNBupxMQMHcyr7Y1VfJycn8+OOPtG3bVhLthRCimilzsJOfn4+9vT0Arq6uhhwHPz8/Tp8+Xb69Ew+EFStWcOTIEWbMmFHs+TFjxtC2bVt69ep1x3ZOxqWRnafDwcqMIDc7vDTWzOjbGCU/j9Sdi7Fr0JGZgx82SkqeMGECtra2uLi4EBMTw6+//lqu9yaEEKLqlTnYadSoEX/99RcArVu3Zvbs2ezdu5cPPviAwMDAcu+gqN4uXbrEqFGj+PHHH7Gysipyft26dWzfvp25c+feta3DF1MAaO7nhMnNIZ2+D3lR+/hC/F1sOLxphaFsRIHx48dz9OhRtmzZgqmpKc8++yyyQFEIIaqXMicov/POO2RkZADwwQcf8OSTT/LII4/g4uLCypUry72Dono7fPgwiYmJRvXW8vPz2b17N/PmzeN///sf0dHRODo6Gn2uX79+PPLII0Z5YgXBTks/fc5PXl4eAwYMIOHKZfbt3oGLi0uR73d1dcXV1ZU6depQv359fHx8OHDgAMHBweV/s0IIIapEmYOdLl26GP5cq1YtTp06RXJyMk5OTnddKSPE7UJCQoiKijK8v3o9m9deeYmG9evz3uRJuLq68sorrxh9pnHjxsyZM4cePXoYHT9SaGSnINA5e/YsO3YUH+jcTqfTAZCTk/Nfb0sIIcQ9pFwqdzo7O5dHM+IBZG9vT6NGjYCb1crXXeRK6g2u/JNB/ywHBnp6FpuU7OvrS0BAgOF9rTp1SWv0FPb12tHAw5annnqKI0eOsGHDBvLz84mPjwf0/6xaWFgQHh5OREQE7du3x8nJiejoaCZPnkxQUJCM6gghRDXzrzcVFKI83V6tXC1Urbw0os+eQZeTSS13W1KTEli3bh2XL1/moYcewsvLy/Aq2EPHxsaG1atXExISQt26dXnhhRdo0qQJu3btwtLSsqJuUwghRBUol5EdIf6rwhXJPZ+eCdyqSH57SYfbE4hXRsTgP2EDKnAmPp3wqyZ3TTJu3Lgx27dvL7f+CyGEuHfJyI64JxRXrbw0FckLRoQKQhuVso0ICSGEqP7KHOzs3r2bGzduFDl+48YNdu/eXS6dEg8eL401fs63AhsThVJVJC88IlSgYERICCGEgH8R7HTq1Ink5OQix7VaLZ06dSqXTokHU0burbpqY0PrFtkTJ06bxb7oJKNRm387IiSEEOLBUeacHVVVi11ifu3aNWxti/7wCFEaGTk3SEq/teT7xBWt0fmVETGGBGYTBWb0bczAVr5cSc02us5UUUo1IiSEEOLBUepgp2/fvoC+uvlzzz1ntGIlPz+fY8eO0bZt2/LvoXggxCQbTzsd+CcZnU7FxEQpslJLd3OlVoc6bszZegaAJ5t4MqS1P/6uNhLoCCGEMFLqaSyNRoNGo0FVVezt7Q3vNRoNnp6evPzyyyxdurQi+yrKwfz582nSpAkODg44ODgQHBzMxo0bDeejo6Pp06cPbm5uODg46HcgTki4Y5vXr19n9OjR+Pn5YW1tTdu2bYmIiDC6RlVVpkyZgoenJ5ZW1jzy6GOcPXvWcP7iNX2wU9/LARsLU5IzcjmdcB0oOS/nyx3n2HMuCXNThQld6xMc5CKBjhBCiCJKPbKzePFiAPz9/Rk3bpxMWd2natasycyZM6lduzaqqrJkyRJ69erF0aNH8ff3JzQ0lKZNmxqWZU+ePJkePXpw4MABTEyKj41ffPFFjh8/zg8//IC3tzdLly6lc+fOnDhxgho1agAwe/ZsPp7zKXahb+Ci8eDIn0tp/2gIF6PPYGVlRUyyvgRJkJstbvaW7D5zlf3R16jv5YCHfdGaWQA/HIgBoLmvEz7OkqMjhBCieIpaxqqHWVlZqKqKjY3+x+XixYusWbOGBg0aEBoaWiGdrGhpaWloNBq0Wi0ODg5V3Z1K5+zszP/93//h4+NDt27dSElJMTwHrVaLk5MTW7ZsoXPnzkU+m5WVhb29Pb/++ivdu3c3HG/RogXdunVj2rRpqKqKp5cXNxp0x/5h/XSoLieDy58/wxcLvuF/w5/l7TVR/Bgew4hOQdhbmTNz4yk61/fgm2EtWRYew6Q1UUW+u4CJAnsnPiajOkII8YAp7e93mVdj9erVi++//x6A1NRUHn74YT7++GN69erF/Pnz/32PRaXLz89nxYoVZGRkEBwcTE5ODoqiGOVjWVlZYWJiwp49e4pt48aNG+Tn5xepWG5tbW34zPnz50lMSMDS7yHDeRNLWyy867Jj917gVs6On7MtwYH6Olbh56+Rr1NZeuAiAG88Vot3utcv0gediiw1F0IIUaIyBztHjhzhkUceAeDnn3/G09OTixcv8v333/PZZ5+VewdF+YuKisLOzg5LS0teffVVw8hcmzZtsLW1ZcKECWRmZpKRkcG4cePIz88nLi6u2Lbs7e0JDg5m6tSpXLlyhfz8fJYuXcr+/fsNnymoS2Vq62j0WTNbR7LTrgG3cnZ8XWxo6O2AvZUZ17Nv8GP4RU7EpWFpZsLw9gF0b+KFyW2LAWWpuRBCiDspc7CTmZmJvb09AFu2bKFv376YmJjQpk0bLl68WO4dFOWvbt26REZGEh4ezv/+9z+GDRvGiRMncHNzY9WqVaxfvx47Ozs0Gg2pqak0b968xHwdgB9++AFVValRowaWlpZ89tlnDB48uMhnAtyM87wa19BgbW5KXr6O2FT93jl+LjaYmZrQOkBfXHbmxlMAPNnEG0cbC7w01szo2xjTm9sfyFJzIYQQd1PmfXZq1arF2rVr6dOnD5s3b2bMmDEAJCYmPpD5LvcjCwsLatWqBehzayIiIvj000/5+uuvCQ0NJTo6mqSkJMzMzHB0dMTT05PAwMAS2wsKCmLXrl1kZGSQlpaGl5cXAwcONHymoGr5uQuxWHjoj5mbKJjnXsfTM4grqVnk61QszEwMycjBQa78cTKRzJsbDQ5pc2uDwYGtfOlQx40LSZmy1FwIIcRdlXlkZ8qUKYwbNw5/f38efvhhgoODAf0oT7Nmzcq9g6Li6XQ6cnJyjI65urri6OjI9u3bSUxMpGfPnndtx9bWFi8vL1JSUti8eTO9evUCICAgAHsnV7IvRtKhjhuONubkZGUQfjCc4OBgQ76Or7MNJjfnqNKy8ozaPhN/3ei9l8ZalpoLIYQolTIHO0899RQxMTEcOnSIzZs3G46HhIQwZ86ccu3c7WbOnImiKIwePdpwLDs7mxEjRuDi4oKdnR39+vW7674wD7KwsDB2797NhQsXiIqKIiwsjJ07dzJkyBBAv8XAgQMHiI6OZunSpfTv358xY8ZQt25dQxshISEMHDjQsF+Pra0t9evXZ/HixWzdupV27dqh1Wp55ZVXUBQFExMTrqckkbLjW7L3LqGmepWk3z5B4+JO7969Dfk6F1f/H4qioCgKY0PrcnHWkyT8NAWAt9foi3smJyczZMgQHBwccHR05IUXXiA9Pb3yH6QQQoj7RpmnsUA/LZGens7WrVvp0KED1tbWtGrVqtgyEuUlIiKCr7/+miZNmhgdHzNmDL/99hurVq1Co9EwcuRI+vbty969eyusL/ezxMREnn32WeLi4tBoNDRp0oTNmzfz+OOPA3D69GnCwsJITk7G39+ft99+2zBVWSA6Ohp7JxeeHRVGcLNGROzexvvvv8/w4cNxcXFhwIABrF271jCtufJgDO/O/pS0Az+xc/0qUlK/wcy7Pr0mzLu5x44+2LG1NKNr16689u5HvLb0qP7LzMyBW8U9Pxj5DHFxcWzdupW8vDyef/55Xn75ZZYtW1ZJT1AIIcT9psz77Fy7do0BAwawY8cOFEXh7NmzBAYGMnz4cJycnPj444/LvZPp6ek0b96cL7/8kmnTpvHQQw8xd+5ctFotbm5uLFu2jKeeegqAU6dOUb9+ffbv30+bNm1K1f6Dvs9OWZVUp6pgv54XXnjB6NoJv0RxZfEbWHoE8c2ib3C0seCVHw5T38uBjaMe4ZUfDrH57wRqHluMq8UN5i9ZTruZ2412TTZVFBb38aJj6+ZERETQsmVLADZt2sQTTzzB5cuX8fb2ruxHIYQQogpV2D47Y8aMwdzcnJiYGMPGggADBw5k06ZN/663dzFixAi6d+9eZFO7w4cPk5eXZ3S8Xr16+Pr6sn///hLby8nJIS0tzeglSqe4OlVhP//F/G+/N+zXU/jaiaujyIk/R17iP9g2CWXS6uN4Ouj38TmTcJ3svHzDNJadpRk7d+6kaW0/spe9QfKWL8nPSjOsuDp3/CiOjo6GQAegc+fOmJiYEB4eXnkPQQghxH2lzNNYW7ZsYfPmzdSsWdPoeO3atStk6fmKFSs4cuRIkVpLoN+/xcLCAkdHR6PjHh4ehr1dijNjxgzef//98u7qA6FwnarcqxeI/2Ec6o1c3rKzM+zXU2DHqURUFdKPbcHcxQermvXJV1Uyc/Nxs7fk6vUc/r6SZpjG6tatK689/zQBAQFER0fz1sQwzHb+H1t27Kamsx3Tt8bj7u5u1B8zMzOcnZ3v+PcthBDiwVbmYCcjI8NoRKdAcnKy0c675eHSpUuMGjWKrVu3Ftmh978ICwtj7NixhvdpaWn4+PiUW/vVWYDrrb1yzJ1r4PX8Zyi5mfRxusywYcPYtWsXTjUC2HYygRkbT6HLyyHjxC4c2w4ECjYAtKVJDQ3bTiWy45R+ebmiwKvDh2JpZgpA48aNadKkCUFBQZw+Gk7NkJAquV8hhBD3vzJPYz3yyCOGchEAiqKg0+mYPXs2nTp1KtfOHT58mMTERJo3b46ZmRlmZmbs2rWLzz77DDMzMzw8PMjNzSU1NdXocwkJCYa9XYpjaWlpqPpd8BKl46WxxtXOAgDF1BxLZ28+fq0Pn3/yfzRt2pQ33vmQtjO3887av8nIycfkwgHUvBxsG4UYbQDYuKYGgA3HrgDgrbE2BDoFAgMDcXV15dy5c4A+MT4xMdHomhs3bpCcnHzHv28hhBAPtjKP7MyePZuQkBAOHTpEbm4ub731Fn///TfJycnlvgIqJCSEqCjjApDPP/889erVY8KECfj4+GBubs62bdvo168foF9NFBMTY5Q7IspPckYuSem5hvcDW/owsJV+w7/s3Bv8dTYBlzq3rk+I2EjnLt34YFQXow0Am9Z0BODCtVt77Nzu8uXLXLt2DS8vLwCCg4NJTU3l8OHDtGjRAoDt27ej0+lo3bp1ud+rEEKI6qHMwU6jRo04c+YM8+bNw97envT0dPr27cuIESMMP0rlxd7enkaNGhkds7W1xcXFxXD8hRdeYOzYsTg7O+Pg4MDrr79OcHBwqVdiibI5GpMCQMqu77AObMlh2yyi6igsW7aMfXt249b/A8O1eSlXyL70N0+8N5ngIBejdhrX1BC78FWcOj6LTZ22eNnA+PHj6devH56envqcnbfeolatWnTp0gWA+vXr07VrV1566SW++uor8vLyGDlyJIMGDZKVWEIIIUpU5mAnJiYGHx8f3n777WLP+fr6FvOpijNnzhxMTEzo168fOTk5dOnShS+//LJS+/AgOXxRH+y4mGYTs+ETtq5MJuRzJ5o0acKy1esIO6BQsGI8/dhWTB1cGdCne5F2XO0suZF8GV3OzZEdNzs2HjvGkiVLSE1Nxdvbm9DQUKZOnWqUC/bjjz8ycuRIQkJCDH/vUoBWCCHEnZR5nx1TU1Pi4uKKrIq5du0a7u7u5Ofnl2sHK4Pss1N6A7/eT/j5ZGb2bcxHW86QlJ7DqleDaeXvjKqqtJj2B8kZ+mmughydgmmu2736w2E2/a1fRTW1d0OGtvGvrNsQQghRDZT297vMIzuqqha7U3J6enq5rpgS9568fB1/XU4FoKW/Ey39nNj0dzyHL6bQyt+Z6KsZJGfkYm6isODZltTzsr9j7SqVW3H2lF//xsLUpMTASAghhPi3Sh3sFCzVVhSFyZMnGy0/z8/PJzw8nIceeqjcOyjuHSeupJGdp8PRxpxAVzta3Ax2Dl1IgY76fXUA2gS50Kme+x3bitNmseXErRpmqgqTVh+nQx03Ke4phBCiXJU62Dl6VF+rSFVVoqKisLCwMJyzsLCgadOmjBs3rvx7KO4ZBfk6zX2dMDFRaO7nBMCRmBRUVWX7zWCnU907Bzqg35zw9gnUgvpXEuwIIYQoT6UOdnbs2AHol35/+umnktvyADp8cyVWi5tBTqMaDliYmZCckcvx2DQiLiQD8NhdRnVAvzmhiUKR+lf+rkWXoAshhBD/RZk3FVy8eLEEOg+oI4VGdgAszUxpUkO/OeCn285wQ6cS6GqLf6FdlkvipbFmRt/GmN7M/yq84aAQQghRnsqcoCweTFdSs4jTZmNqotDUR2M43sLPiUMXU/jj5M0prFKM6hQY2MqXDnXcuJCUabThoBBCCFGeJNgRBnHaLM4nZRDgalsk8Nh6Up9MXNvdDhuLW//YFOTtFCjNFFZhXhprCXKEEEJUqDJPY4mqMX/+fJo0aWKo5RUcHMzGjRsBfRHW119/nbp162JtbY2vry9vvPEGWq32ru2ePHmSnj17YmPnQA03Jzq1b0vrsBWsjIgBIDo6mocf7crwkKbEzOnPn1+9zdebDhs+f+lmxfICF5IyyvGuhRBCiP9ORnbuEzVr1mTmzJnUrl0bVVVZsmQJvXr14ujRo6iqypUrV/joo49o0KABFy9e5NVXX+XKlSv8/PPPJbYZHR1N+/btGfjMMJz6dwYLG/KSYlBNLQj7JQrT/FzGDHycZCsvPAZPByD1z6WMGv40T/59BBMTE6b/ftKozSm//s1j9d1ltEYIIcQ9o8w7KFdH9+sOys7Ozvzf//0fL7zwQpFzq1at4plnniEjIwMzs+Jj2kGDBmFubs7g8bN4bdnRIuezzh8hcdV7+IxagYmlfpWULieDS3MH8emSn2nZrgNPLwwv8rnlL7UpUgtLCCGEKG+l/f2Waaz7UH5+PitWrCAjI6PE6u4Ff/ElBTo6nY4Nv/2G6uDJ00/15NLnQ4j7fiyZZ/YbrlHz8wBQTM0NxxRTC1AULp44bFg+XpgsHxdCCHGvkWDnPhIVFYWdnR2Wlpa8+uqrrFmzhgYNGhS5LikpialTp/Lyyy+X2NbCzUfISE9n2cLPMfFphu/TH2JbJ5ira6aTHRMFgKV3PRRzK1J2LoYb2ehys0ndsQhUHekpSbJ8XAghxH1BcnbuI3Xr1iUyMhKtVsvPP//MsGHD2LVrl1HAk5aWRvfu3WnQoAHvvfdese3EabP48LcTAFjXaoNDq94oCuyY8jQvD73C35EbsfJtjKmNBrfeE0ne8iUxR9ZjYmJC734DOE9zTEz0cbIsHxdCCHGvk2DnPmJhYUGtWrUAaNGiBREREXz66ad8/fXXAFy/fp2uXbtib2/PmjVrMDc3L7ad80kZKNYOYGKKuasPoN/JODNXx6Otm5G6aRs3FIV8VcUusAWfbQknJMAGMzMzHB0d8fT0JDAw0NCeLB8XQghxL5Ng5z6m0+nIyckB9CM6Xbp0wdLSknXr1t2xAr2PkzWKqTmWnrW5kRwL3Mq1OXPmDC0a1uHjiZ2KHa3Zvn07iYmJ9OzZs2JvTgghhCgnEuzcJ8LCwujWrRu+vr5cv36dZcuWsXPnTjZv3kxaWhqhoaFkZmaydOlS0tLSSEtLA8DNzQ1TU1MA6tWrx4wZM9D5tgLAoXVfrv46G2ufRrzz0lP88sMi1q9fz86dOw2jNYsXL6Z+/fq4ubmxf/9+Ro0axZgxY6hbt26VPQshhBCiLCTYuU8kJiby7LPPEhcXh0ajoUmTJmzevJnHH3+cnTt3Eh6uXwJeMM1V4Pz58/j7+wNw+vRpklNSWHL2LABvvjyUtIc9WL7wMyYOWUjdunX55ZdfaN++veHzp0+fJiwsjOTkZPz9/Xn77bcZM2ZM5dy0EEIIUQ5knx3u3312/o2vd0YzY9MpnGzM2TvxMaPSD0IIIcT9RPbZEUUsPXCRGZtOAZCamcf6v65UcY+EEEKIiifBTjUTp81iX3QScdqsIscnrz1ueK8Ck1YfL3KdEEIIUd1IsFPBZsyYQatWrbC3t8fd3Z3evXtz+vRpo2uio6Pp06cPbm5uODg4MGDAABISEu7YbnGFQSd+uoR2M7fz9MJw2s3cTue+QwgKCsLa2pr6AT4k/DKVvGuXDG3kqyoXkjLv8C1CCCHE/U+CnQq2a9cuRowYwYEDB9i6dSt5eXmEhoaSkaGvDp6RkUFoaCiKorB9+3b27t1Lbm4uPXr0QKfTldhuQWHQw4cPc+jQIR5u9wizxr5IduJFQL9vzl9Zznz0+VecPHmSF6YtBFQSVk5B1eUDUtpBCCHEg0ESlKncBOWrV6/i7u7Orl276NChA1u2bKFbt26kpKQYvlur1eLk5MSWLVvo3LlzqdrdF53EI40CcHx0OPZNQw3Hl7/UhqY+GjrM3sGVf04Tt/h1vF9eiJWzN9P7NmJgK98KuU8hhBCiopX291uW4lQyrVYL6CuWA+Tk5KAoCpaWloZrrKysMDExYc+ePaUKdvLz8/lr1+/o8rKxrFHP6FxNJyu+33+RxOQ0lLM78fHzZ+no7tT2dpRdj4UQQjwQJNipRDqdjtGjR9OuXTsaNWoEQJs2bbC1tWXChAlMnz4dVVWZOHEi+fn5xMXF3bG9qKgogoODyc7Oxs7ODu9+72DuajxS81LYDLYu/hhdXjbefkHs2PYHQUFeFXaPQgghxL1GcnYq0YgRIzh+/DgrVqwwHHNzc2PVqlWsX78eOzs7NBoNqampNG9+q9hmSQoKg4aHh/NIz6eJX/8JLnkJLH+pNe/2qA/ACZsmeD73KR5Pz8TTJ4ABAwaQnZ1dofcphBBC3EtkZKeSjBw5kg0bNrB7925q1qxpdC40NJTo6GiSkpJKLLZZnILCoKqqktc8A4vd+7A79wfBQcPxc7Hh/fUnMbG0xcTSFnPnGqTUqEf8F0+zZs0aBg8eXJG3K4QQQtwzJNipYKqq8vrrr7NmzRp27txJQEBAide6uroCZS+2eSQmhVPx1zFBpYa9/q/0wrWiS8rzdSo6nWooHiqEEEI8CCTYqWAjRoxg2bJl/Prrr9jb2xMfHw+ARqPB2lqfIFyaYpshISH06dOHkSNHAsaFQWcuCydl189kxkTx/LCPADBJTyTtwE9Y+jfH1MaBG2nXuB6+Chtra5544olKfgpCCCFE1ZFgp4LNnz8fgEcffdTo+OLFi3nuueeAuxfbjNNmceL0WR66fCthuaAwaOyVOHTm1li4+eMx4AOSHfUBkq+bI355MUStWkd+djqmto60aduORRu+x93dvWJvWgghhLiHyD473NuFQFdGxBC2OgqdCiYKzOjb2LA3Tpw2i7YztlP4L9BUUdgzsZNhWXmcNosLSZn4u9rIUnMhhBDVihQCvc/FabNY/1esIdAB/a7IhetZHb+cxu2R6u0lILw01gQHuUigI4QQ4oEl01j3oMKjObcrCGa8NNbsPZdU5LyUgBBCCCGMycjOPSZOm1VioAOgKODvakNyRi6rDuuLepoo+nOmisL0vo1kFEcIIYQoREZ27jHnEtNLDHRAH9Bk5OTz3d4LZOTm06iGAwuGtuDitSzJyxFCCCGKIcHOPaa4qSkT4LPBD/HDgRjCzyfz6tLDXLymr5o+9vE6eDva4O0oU1dCCCFEcWQa6x4Rp81ixcEYvvnzH0A/XQX6kZwZ/RrzZNMafDqoGVZmJpxLTCcvXz/8k5gmGwQKIYQQdyIjO/eA2xOSG3g58M2wolNTKio5N3RGn317zXE61nWT6SshhBCiBDKyU8WKS0g+FZ+GoihFloyfT8q461JzIYQQQhiTYKeKnU/KKJKQrFMpNoAJcLU1rLwqIEvNhRBCiDuTYKeKBbjaclv8UmIA46WxZkbfxpjeTOiRpeZCCCHE3UnOThXz0lhTz9Oek/HXgbsHMANb+dKhjpuUgBBCCCFKSYKdKqbNzOPc1XQAPnqqCe1qu941gPHSWEuQI4QQQpSSBDtVbPPf8eTlq9TztOeplj5V3R0hhBCi2pGcnSq2/tgVAJ5s4lXFPRFCCCGqJwl2qtC19Bz2RV8D4Mkm3lXcGyGEEKJ6kmCnCm08Hk++TqVxDQ3+rrZV3R0hhBCiWpJgp4rEabP4MfwiIFNYQgghREWSBOUqcHt5CFW9Q5lzIYQQQvwnMrJTyYorD/F/m88Qp82quk4JIYQQ1ZgEO5WsuPIQUt9KCCGEqDgS7FQyP+eiZSCkvpUQQghRcSTYqURx2iy+3v2P0TGpbyWEEEJULElQriS3JyV3aeDBc+0CpL6VEEIIUcFkZKcSFJeU/MfJBAl0hBBCiEogwU4lKD4pGUlKFkIIISqBBDuVIMDVFuW2Y5KULIQQQlQOCXYqgZfGmu6FdkmWpGQhhBCi8kiCciXxdtQHNk808mRyjwYS6AghhBCVREZ2KkmcNhuA5n5OEugIIYQQlUiCnUoSf7MchKfGqop7IoQQQjxYJNipJPFp+pEdLwl2hBBCiEolwU4lUFWVBG0OAB4OEuwIIYQQlemeDnZmzJhBq1atsLe3x93dnd69e3P69Gmja7KzsxkxYgQuLi7Y2dnRr18/EhISqqjHxUvOyCU3X4eigLu9BDtCCCFEZbqng51du3YxYsQIDhw4wNatW8nLyyM0NJSMjAzDNWPGjGH9+vWsWrWKXbt2ceXKFfr27VuFvS6qIDnZxdYSC7N7+pELIYQQ1c49vfR806ZNRu+/++473N3dOXz4MB06dECr1bJo0SKWLVvGY489BsDixYupX78+Bw4coE2bNlXR7SLitZKvI4QQQlSV+2qYQavVAuDs7AzA4cOHycvLo3PnzoZr6tWrh6+vL/v37y+xnZycHNLS0oxeFakgOVlWYgkhhBCV774JdnQ6HaNHj6Zdu3Y0atQIgPj4eCwsLHB0dDS61sPDg/j4+BLbmjFjBhqNxvDy8fGpyK4bRnY8JTlZCCGEqHT3TbAzYsQIjh8/zooVK/5zW2FhYWi1WsPr0qVL5dDDksnIjhBCCFF17umcnQIjR45kw4YN7N69m5o1axqOe3p6kpubS2pqqtHoTkJCAp6eniW2Z2lpiaWlZUV22YiM7AghhBBV554e2VFVlZEjR7JmzRq2b99OQECA0fkWLVpgbm7Otm3bDMdOnz5NTEwMwcHBld3dEsmGgkIIIUTVuadHdkaMGMGyZcv49ddfsbe3N+ThaDQarK2t0Wg0vPDCC4wdOxZnZ2ccHBx4/fXXCQ4OvmdWYsGtkR0PCXaEEEKISndPBzvz588H4NFHHzU6vnjxYp577jkA5syZg4mJCf369SMnJ4cuXbrw5ZdfVnJPS3Y9O4/0nBuATGMJIYQQVeGeDnZUVb3rNVZWVnzxxRd88cUXldCjsku4OYXlYGWGreU9/biFEEKIaumeztmpDgp2T5aVWEIIIUTVkGCnghlWYmmsq7gnQgghxINJgp0KdmvZeeUtdRdCCCHELRLslGD37t306NEDb29vFEVh7dq1Ruffe+896tWrh62tLU5OTnTu3Jnw8PAi7cSl3RrZuX79OqNHj8bPzw9ra2vatm1LREREiX149dVXURSFuXPnluetCSGEEA8UCXZKkJGRQdOmTUtMfK5Tpw7z5s0jKiqKPXv24O/vT2hoKFevXjW6LqFQEdAXX3yRrVu38sMPPxAVFUVoaCidO3cmNja2SPtr1qzhwIEDeHt7l//NCSGEEA8QCXZK0K1bN6ZNm0afPn2KPf/000/TuXNnAgMDadiwIZ988glpaWkcO3bM6LqCBGUnC5VffvmF2bNn06FDB2rVqsV7771HrVq1DEvsC8TGxvL666/z448/Ym5uXjE3KIQQQjwgJNgpB7m5uSxYsACNRkPTpk2NzhUsPXe1NSM/Px8rK+NVWdbW1uzZs8fwXqfTMXToUMaPH0/Dhg0rvvNCCCFENSfBzn+wYcMG7OzssLKyYs6cOWzduhVXV1fD+ey8fK5l5AIQ5O1GcHAwU6dO5cqVK+Tn57N06VL2799PXFyc4TOzZs3CzMyMN954o9LvRwghhKiOJNj5Dzp16kRkZCT79u2ja9euDBgwgMTERMP5xLQcACzNTHC0MeeHH35AVVVq1KiBpaUln332GYMHD8bERP/XcPjwYT799FO+++47FEWpknsSQgghqhsJdv4DW1tbatWqRZs2bVi0aBFmZmYsWrTIcL5wAVBFUQgKCmLXrl2kp6dz6dIlDh48SF5eHoGBgQD8+eefJCYm4uvri5mZGWZmZly8eJE333wTf3//qrhFIYQQ4r4n9QvKkU6nIycnx/A+TpsFgMdtNbFsbW2xtbUlJSWFzZs3M3v2bACGDh1K586dja7t0qULQ4cO5fnnn6/g3gshhBDVkwQ7JUhPT+fcuXOG9+fPnycyMhJnZ2dcXFz48MMP6dmzJ15eXiQlJfHFF18QGxtL//79DZ+Z8EJ/0pyb4PXQKwBs3rwZVVWpW7cu586dY/z48dSrV88QyLi4uODi4mLUD3Nzczw9Palbt24l3LUQQghR/UiwU4JDhw7RqVMnw/uxY8cCMGzYML766itOnTrFkiVLSEpKwsXFhVatWvHnn38araCKu3QBxdofWyv9Y9ZqtYSFhXH58mWcnZ3p168fH374oSwvF0IIISqQopamtHg1l5aWhkajQavV4uDgUC5troyIYcIvUQAowMx+jRnYyrdc2hZCCCFE6X+/JUG5AsRpswhbHWV4rwKTVh835PAIIYQQovJIsFMBzidloLttvCxfVbmQlFk1HRJCCCEeYBLsVIAAV1tMbtsmx1RR8He1qZoOCSGEEA8wCXYqgJfGmhl9G/9/e3ceHkWV7g/8W70v6U4n6c7adBJDQgLZSAIBEmQJmySIsstO1IFhEdkDqAhcB1xgvIiioMLgXKIOEu+IiCIqesEFRMRxGGBw444bMAEGgizJ9/dHbh27WRxnRgbs3/t5nnpIdVVXnT5VdeqtsxQw/t+LAY2ahl/1yUZCpP0Kp0wIIYT4/4+MxrpMBrYK4NoMHz47XIcUr0MCHSGEEOIKkWDnMkqItEuQI4QQQlxh0owlhBBCiLAmwY4QQgghwpoEO0IIIYQIaxLsCCGEECKsSbAjhBBCiLAmwY4QQgghwpoEO0IIIYQIaxLsCCGEECKsSbAjhBBCiLAmwY4QQgghwpoEO0IIIYQIaxLsCCGEECKsSbAjhBBCiLAmwY4QQgghwpoEO0IIIYQIaxLsCCGEECKsSbAjhBBCiLAmwY4QQgghwpoEO0IIIYQIaxLsCCGEECKsSbAjhBBCiLAmwY4QQgghwpoEO0IIIYQIaxLsCCGEECKsSbAjhBBCiLAmwY4QQgghwpoEO0IIIYQIaxLsCCGEECKsSbAjhBBCiLAmwY4QQgghwpoEO0IIIYQIaxLsCCGEECKsSbAjhBBCiLAmwY4QQgghwpoEO0IIIYQIaxLsCCGEECKsSbAjhBBCiLAmwY4QQgghwpoEO0IIIYQIaxLsCCGEECKsSbAjhBBCiLAmwY4QQgghwpoEO0IIIYQIa2ET7Dz88MNISUmBzWZDcXEx3nvvvSudJCGEEEJcBcIi2HnmmWcwefJkzJkzBzt37kReXh66d++Ob7/99konTQghhBBXWFgEO4sXL8att96KUaNGoXnz5nj00UfhcDjw5JNPXumkCSGEEOIKM13pBPyrzpw5g/fffx8zZ85UnxkMBnTp0gVvv/32Rb9z+vRpnD59Ws0fO3YMAHD8+PHLm1ghhBBC/GT0+zbJH1zvZx/sHD58GPX19YiLiwv5PC4uDn/6058u+p0FCxZg7ty5F3zepEmTy5JGIYQQQlw+f/vb3xAZGXnJ5WHRjPWPmjlzJo4dO6am2tpaHDhwAEePHg35/F+dDh48CAA4ePDgBfM/tOxqWFfSJ+mT9F1d+5T0SfrCJX0/5XT06FEcPHgQiYmJ+CE/+5odr9cLo9GIb775JuTzb775BvHx8Rf9jtVqhdVqDfnM4/FcriTC7XbD7XaHzP/QsqthXUmfpE/Sd3XtU9In6Qu39P1UfqhGR/ezr9mxWCwoLCzE5s2b1WcNDQ3YvHkz2rZtewVTJoQQQoirwc++ZgcAJk+ejBEjRqCoqAitW7fGgw8+iJMnT2LUqFFXOmlCCCGEuMLCItgZOHAgDh06hLvuugtff/018vPzsXHjxgs6Lf+7Wa1WzJkzRzWZnT//Q8uuhnUlfZI+Sd/VtU9Jn6QvHNJ3JWj8e+O1hBBCCCF+xn72fXaEEEIIIX6IBDtCCCGECGsS7AghhBAirEmwI4QQQojwRnHZLF26lMnJyTSbzfR4PPR6vQTApk2bMiIigj6fj7m5uczIyKDL5aLL5WKbNm24YcMGkmRZWRkBhEzXXHMNhwwZQoPBcMEyAHS5XLRYLLTb7TSbzQRAo9HIqKgotf/58+czPT1dbcNms6l1AdDv99NkMhEADQYDNU0jAFqtVnq9XrWdmJiYC/avb8disVw0ffpksVhCtm0wGGi1WtVnBoOBJpNJLQ/+V1+uaRqNRiONRiOjo6OZlZV10XyxWCx0OBxqG5ea9O3q2w5eP3i7RqNRLTMajTSbzRes63Q6abPZLrmfH0pHZGSkyn+LxaLyctSoURw7diztdrs6bvqy7OxsZmRkhHzPZrPR4XAQAIcNG8Zf/epXl9xnYmKi+u7PaQo+FhEREezUqZPKn/PzOikpiUajUX2u/61vRz9mHo/nnzpuV+PkcDhCrkWv10u3263mg893PQ8vlc82m41Wq5WpqakheRy8LbvdHpKvl9qWXk5omsb4+PiQ/Qbnc2pqasi6wWWCxWJhTEwMbTYbfT7fP3XMLlZe6J8ZjUba7fYffdwvtZ7RaLxoWflDafhn1j1//z807/f7f/LzWT82wWV4cB4YDAYGAgFmZGSwSZMmtFqtjI+P59ChQ/nJJ58wLy+PAPjBBx9clvux1OxcJs888wwmT56MOXPm4KGHHkJKSgrq6uoAANdddx3eeecdbNq0CWazGbW1tXjrrbewY8cOdO7cGb1798bTTz+NHTt2wGq14pZbbsFXX32FPXv24OzZszCbzdiwYQPeffddVFdX4+2331bvFJo5cyYee+wxREdH49y5cwCAMWPGIDs7GydPngQAvP766/jiiy8wffp0AI3DAs1ms0p7Q0MDJk6cCADIzMxEQkICAGDkyJFISkpS2zlx4gSMRiMGDRoEAOjUqRMsFgsAYPjw4Rg5ciQmTJgAAMjIyEB0dDSAxrddnj17Fg0NDQgEAmjWrBk0TcOZM2cQEREBl8uF2NhYeL1eNVRx3rx5KCgoQGpqKgAgNzcXUVFRcDqdaNu2LcxmM7744gv1n8GlpaUhKysLUVFRMJka37DgdruxfPly3HHHHcjJyYHJZILP54PJZEJZWRlIoqGhAa1btwZJmEwmpKamIj09HQ0NDfB6vQCA2NhYGAwGLF++HCUlJTh79iyaN2+O+fPno2vXrmhoaFC/EwDy8/PxP//zPygpKQEADBo0CLNnz0bnzp1hMBjw5JNP4rbbbgMAaJqGuXPngiSsVivOnDmDyMhIuN1uPP3001i7di0iIyPh9XqhaRrq6+sBACdPnsTo0aMRHx8Pr9cLi8WCli1bwuPxwO12Y+3atbj//vvhcrkAAK1bt0aTJk2QkpICs9mMnj17IjY2FmlpaSgqKoLL5YLVaoXL5YKmabjxxhthMpmQnp4OAAgEAtA0DWlpaRg8eDBMJhPi4+Mxa9YsAIDT6cRvfvMbvP322ygoKIDJZEKnTp0wdOhQOJ1OxMXF4a677kJKSgoA4Nprr8WYMWOQlJQEu92OKVOmYPjw4QgEArjvvvvw3nvvoaioCAaDAUajEUuWLFG/f9KkSXjmmWdw9uxZvPPOO9A0DTExMUhLS0NCQoI6DnV1dZg3bx5qamrgcDhgt9uRnJyM2NhYxMXFoaGhAVVVVThx4gQMBgOSk5OxadMmREVFwWw24/rrr0dsbCxat26N8ePHo7q6GsuWLUN+fj4AqGOov/KiWbNm6rybP38+ACAmJga33HKLutaWLl2Kxx9/XL1VdtmyZbj33nvVunfddRecTicAYOzYsXj99dfVdTtkyBA88MADAIDRo0fjl7/8JYxGIwDAbDajW7duqKurw5kzZ/DQQw/huuuuw+HDh3H8+HH07t0bDocDJGE2m1FYWAgAsNvtqKiogNPphMlkQkxMDCIjI1FfX4/k5GSYzWbExsbi1KlTMBgMaNeuHSIiIlQZkpycjPr6evTu3Rv3338/Fi1aBE3TYDA03mqKi4tBEmfPnsW4ceOQlpaGr7/+GidOnEB+fj4cDofKMwD49NNPMWHCBLz88svwer04d+4c/H4/2rdvD4PBgCNHjmDSpEnqP4NMTEzEtm3bVDlRWFgIv9+vtnv99dejuLhYpScxMVH9rZ+3+vWbn5+P7777TuWpvg392ASXmbm5uQimHzN9Xb3M1M8LTdNgs9nUsdKVlpbC5/NB0zT1WatWrUL207VrVzWfnp6OvLw8AFDpDP5eRkZGyH6Tk5MBAGfPng15NUvbtm3Rvn17NZ+VlYUbbrgBQGM5NmbMGLXMZDKF5Fnw24sdDgfi4uIQEREBm82m8mHRokXIycmB2WzGl19+idLSUuzduxfPPfccDhw4gHbt2v3d/+7hX3ZZQijB1q1bc9y4cWq+vr6eiYmJBMCamhr1+bfffksA3LJli/rM4/EwLi6Ow4YNo9Pp5MSJE0mSM2bMYGlp6UX3l5KSQpfLxYaGBtbV1dFoNLJt27Yh+ysoKCAAejwe3n///SRJAPztb39Lq9WqonA9sta/+9577xEAly9fzmPHjqn1WrduTbvdzl//+tcEwJKSEg4dOvSC3wiATZo0YWZmJjt37sy3336bAJicnMzZs2fzz3/+M/F/T6GLFy8mAK5evZoAuHDhQgLg+vXrQ/LLZDLxyy+/JAA++eSTBBDydFdTU6PWHTZs2AX5rM/HxsaysrKSL7/8skrDf/7nfxIAKysrqWkaN23axMjISLXt2tpaRkVF8fHHHydJOp1OGo1Gnj17Vs0DUL8zNzeXJFlcXEybzaa+RzJkOwCYlpbGpKQkaprGtm3bUtM0Tpo0iW3atCEARkdHc9OmTSwtLaXL5VJpuvnmm5mens5NmzaxQ4cO7NatGwFw48aNbNmypTru+jnhdDq5adMmulwuFhYWqu+WlpbS4/Gocw8Ar732WnVcjx07Ro/HQ03TGAgEqGkahw0bxszMTALgww8/TKCxpkk3Z84cJicn02Kx8M4772SzZs0IgHFxcZwyZQoBsKKignPmzKHD4eAdd9yhvpeXl6e2YzababVaWVlZyQEDBhAAo6KiuGLFCs6YMUPlUV5enrpOjh49qp4w9fNav47087pVq1bqvLbb7SwoKKDT6WR2djZnzJhBr9fLoUOHXvL6c7lcTE1N5fDhwwmAH330kXpKjYuLo9frZVlZGQOBQEgepaamsqGhgSTZs2dPAuCZM2c4ceJENmnSRH0/Pj6eALhu3TqSZHx8PKOiotjQ0MCJEycyLS2NDQ0NLC8vV7Ufo0aNYuvWrQk01oCsWLGC5eXlKi+Ki4tZWVnJXr160WAwsEWLFgTAnj17sry8nJWVlezTpw979OhBoLFWTM9zs9msanCaN2/OyspK+v1+pqSksLy8nMnJyRwyZAhJsq6uTuVrhw4dVK2PzWZjmzZt1BO/yWSiwWDg+vXrmZeXp2od/H4/Z8+erco0v9+vagqeffZZtS29hs/n87Guro6apjEiIoLTp0+n0Wjk+vXrCYD9+vUjSZUPwbVcqamp6lrWy4IJEyao5ZGRkSG1VsG1UR6Ph/Pnz1e1o+evG/y3y+Xi3XfffdHt1tTU8MCBAyE1JnfccYf6e9myZTx69Kia/+///m9++umnF61pCS7HAXD79u3csGGDSvtXX30Vsm5xcXHIvL7dxYsXq+00b96cS5YsCcmD0aNHq/mlS5cSAH/zm99Q0zTefvvtBMDPP/9cXW8zZ85kamqqun7uuusuAuCuXbsua82OBDuXwenTp2k0GkNu+CRVYRj8+f79+1UBee7cOVZXV9NgMHDYsGGcM2eOahJJTU2l2+3mzTffzH79+tHn8zE/P5/Lly/n6dOnabfb6fF4uHfvXh4/flxdgMH7KykpuWRAc+21115y2aZNm9QJfO+996oLs1OnTqpQ0S/oqqoqVYi0bt2aNTU1BMAOHToQAB966CG+8MILqpDasmWLCmx8Ph9vvPFGlR+BQIBDhgwhAG7dujUkvzwej/p70KBBNJvNIRehfgMCwClTpjA/P58AWFhYyLfeekt9Vw+k9JuuxWLh2LFjVWBkMBjYt29fWiwWRkdHEwAff/xxWiwW7t69m9XV1TSZTPR4PDx37hwfe+wxdRN+6qmnVGChHwtN05iTk0Ofz8fMzEyazWbu3r2b99xzj/pejx49aDQa6Xa7qWka9+/fz5ycHAJgUVERSTIuLo75+fmMjY0lAGZkZPD2228nSXbo0IHx8fG02Ww8ceKEanL43e9+p46DxWJhVFSUKtTj4+MZGxtLs9nM9PR0de4Bjc2X+k0muIlHb6azWq3qpqXfPGw2GxMSEpiamsrs7GyazWYaDAZGRkaqG1lsbCyzs7NVvujbtVgsIduMiYlhamqqOu9KS0tV2jRNY3R0NK1Wqwp29JteQUEBfT6f2naXLl3o8/lotVpZVlbG9u3bE2hsjk1KSiLQGESXlZVd0BSj55WmaXQ4HHS73czPz+cDDzxATdPU9QmATz31lHp4uOOOO5iUlESr1cp77rmHgUBA3Rz0oO706dOMiIigw+Hg6dOnGR0dzeLiYprNZq5Zs0bdQF0uFzMyMgiA1113HYuLi6lpGlNSUvjWW29x3rx5Kr0vvfSSCqBMJhP379+vyh8A7Ny5MxMSEhgdHc1mzZqpYHXSpEm855571DL9ejYYDOzZsyeNRmPITdFisai86tWrlwrS/H4/a2pquHXrVrXu/fffr/42GAzqnNbTCICvvvoqS0pKVJmSlpbGDh06qDItOTlZfefWW28lABVE6k3P+nVqs9nUQ4oeYLdo0YJff/31RYMDvblPfygEwDlz5oScC/qy4Ob34HJTTwsAFfDq3w9uJg5eL3jZgw8+yIaGhpCmx6lTp6q/t2zZws2bN6v5uXPnqqBEfzi5WDkOgNu2beOwYcMIgMOHDw9ZFvzgpJ/verC8ePFi7tixQ+XjzJkzQ/KssrJS5cmwYcOoaRqPHDlCo9HI3r17E2h8SNq0aRM1TePUqVNZWFhIktyzZw/tdjtzc3PV75Bg52fkL3/5izq5gk2bNo3A98FHfX09y8vLmZ+fr54o7HY7k5OTeerUKW7YsIHNmzfn4MGDuXHjRnWBTZkyhTt37uRjjz1Gm83GsWPH0mAwcNy4cerJDoC62T/33HN86qmnQi7ML7/8kuT3AU3//v0vepE888wzTEtLC7ko9MJu8uTJ9Pl86ncFX/yLFy/mggULVJqHDBlyQT+eVq1a8fDhw6pQABr7FJSUlJAki4qKVG1YbW0t6+vr2bVrV1qtVnbr1k0VIpGRkezXrx/Ly8tVWhcvXsz09HRVkOTm5jI/P5+33347zWYzO3bsqArF4EnTNPXUGtz/6OGHH1a/Re/boQckUVFRF9wcjUajKmj1G31wHhmNRnWcjEajCj7O7zPRs2dPnjhxQhVklZWVrK6uptPp5Lhx41QhHh0dzVOnTpEkMzIyQs4D/aZBkllZWQTAWbNmqWAMAPPz87lgwQLV7r5ixQpGRUXR5XKFFP4mk4l9+vRRNxOgMQDT+wvpAXV0dDS3bdvGjRs3Mj09PaSPjMFgoM1m47p169RNdtSoUezbt6/aT2pqKlNSUmiz2WgymfjEE09c9AYFgLNnzw75rXpNgdFo5Ny5c9Vxa9++PXfu3KnWPb/fSVJSklpmNpvZu3fvix7X4OvQZDLR4XCwtLQ0ZP8dO3ZU+ayfV3/5y1/YqlUrtmvXjgD4xz/+kSS5YsUKAo0PBHqQ5HK52L9/fz7zzDPq3HnwwQfZr18/dRzGjBlDo9HIW265hRaLRdWKXmy6VH+s8/umBR+ni/Xp0D/Tj9vF1ikoKAjpYxcc4AZPei2mPh8ZGcnMzEwVUOqfX3PNNTx37lxIORS87/T0dDZt2lTlm/5wph8v/eECaAxsBw4ceNG8uP7660N+GwDOmDEj5JqsqKggAJW+4HzdunXrRfNZf8DQ0wg0PqAFr6MH3i6Xi0eOHAmppQ7Ou/fee4/Lly9X8+PHj1dBwvn9lj744AMeOnQo5DM9qAmurQHA//iP/whZr7q6mqNGjSIAVlVV8Ze//CWBxlrq4PWmTZvGhIQENe/1ejl48GCSpNfrZXR0NM1mM0+dOsWCggL26tWLbreb3bt3V9ef3+/n4cOHJdj5Ofqxwc6YMWOYnJzMAwcOcP/+/Vy/fj0dDgc9Hg8//vhjko1P6XozltlsptFoDGkGmTBhAiMjI9myZUv6/X5WV1dz9+7dvP/++0M6Gbdq1UrVkgA/PtgpKipSJ/iECRNot9vVU35NTQ2Tk5NVM1bwpP/GXr16EWis6fF4POzRowcTExM5ZsyYCzrbNWnShDabjQcPHiRJ+nw+VajU1taysrKSFouFHTt25MiRI5mUlMQZM2bQ6XQyJiaGgUCABw8eJAB2795dVf8DjTcyfbvR0dF0u900mUyMj4/nI488whEjRlwQjHXq1ImZmZls2bJlyNP/hAkTGBUVxVWrVjEhIYEmk4lPP/00q6qq6HQ6aTKZmJKSop7I7rzzTnWz1vfx/PPPc9q0aTQYDKyoqFCfN23alGvWrKHL5QoJDvQCb+jQoYyNjWVhYSEnTpzIa665hgDYv39/kuTHH39MTdMYFxfHV199lU6nk5qm0ev1ct++fSqAq6mp4bp16wh8/7QcExPDwsJCxsTEcMqUKTQajUxISGDr1q35/PPPh3R4BhoDgkAgwDFjxrC2tpZGo5GtWrVSherjjz/OY8eOsaCggEajkfPmzeOvfvUrOhwOGgwGLlu2jLW1tQTAcePGqVoAPY07d+6k2+1mUlKSqnkDvg8y9I7y+qTnocFgYNu2bTlhwgS2adNG5V1OTo66jqxWKx0OB00mE9PS0lTAaTabGR8fT4vFwl//+tcqLXrNTvPmzdXxHzRoED0ej6qVadWqFaurq0Nu4N27d6fX66XP5yNJtmrVSp2XtbW1PHbsGN1uN30+Hw8dOsTS0lJmZmbS6XQyLy+PXbp0UTfYmpoa9bQNNNZwVVRUkCRzcnJCBg4sX75cNTudP+k1R9HR0XQ6nWzRooWqaejXrx/vvfdeFcDHxcWxf//+KgiKi4vjyJEjaTAYGBERwYqKipDaPr1ZbMyYMezQoYN6MNKvdYPBEDIY4vzp/BoG/YHEaDSGXBPnB085OTkhHYuDzwdN00K2a7PZQs6dRx55JGRbwcv+kWAnOTlZPWAGX1d6bVznzp3VMr255/ztulyuCwYSrFmzRv2tPyjp87fddtslg5233nor5HwZP368CjDuvfdeVcYD4MqVK0O++/vf/15tt0OHDurBzeFwsGnTply0aNEFx87hcNDr9fLYsWM8c+aMqg13uVzs1asXs7OzmZqayptvvpmHDh3i7Nmz2aJFC7Zr1449e/bkJ598QkCCnZ+VH9OMNW7cOPr9fn7yySdqud7koz9tnT9yJBAI0Ov1sqqqSn1n/vz5qoBbunRpyP70ZU888QRJhhR+f68Z68yZM+qCPXz4MIHv+7AEX3jn/6sXAvpvnz59ulq/ffv2Ib85KyuLVqtVFTZms5lDhw4lSY4bN45Go1H1FdGDkXbt2nH06NFqOxMnTlTbP3+ETXCN0apVq9R27XY7mzdvTgB88803VX6VlZUxMTGRXbt2VfuMi4vjfffdx7KyMlWTUVtby44dOzI2NpZlZWXs1KkTf/GLX3DixImXHMGlFzp6zcrGjRtJkrGxsYyNjQ1plvlHRmf8Oyaj0chz586xqKiIVVVVqvmhZcuWdDgcvO+++0iSFotF9Q9q2bIlJ0+ezLZt27KsrIwFBQWsqqq6II+Cj5eeRykpKSqPioqKVMAJND79V1VVMRAI8Oabb2azZs1UAK7XQLlcLt5888185JFHmJiYqG56Pp+PZ86cod1uV013gUAgpFkmeAq+sbhcLhoMBpaUlPCRRx5hRESEaoLT133++edJkp999pn6/MMPPyTQ2OxEkomJier3f/HFFyrNzz77LD/77DMV/F7sPDr/Bh+8z/LycvW5fk77/X5WVFTQ4XBw+PDhjIuLU9+LjIzk0qVLOX/+fJUfet77/X7m5uYyLi6OTZs2pd/vV9fLxaaoqCgVkEdFRbGiooLNmjXj9OnT1fdiY2N54sQJ9ZCVnJzMqKgo1Y/D6XRyxIgRJMk+ffqo4KRbt27s2rUre/TowdzcXPbu3ZvdunXj3XfffcnzSK+hy87OpslkCrmxBwdAweVd8Pn3zzZjlZeXh5Sjwf38zp/0Jtfzt5uenh7SvBecPqCxb57e/wgA77vvvks2Y+Xm5oaM6B08ePAF17X+t95nS58efvhhtd3k5GS1bmJiIg8fPqzuD1arlUVFRSov5syZwzNnzqjmq1GjRtFkMjErK4vXXHMNhw0bxvr6epJk7969Q66x4H/1ZrafkozGugwsFgsKCwuxefNm9VlDQ4OaX758OWpqavDaa6+pUQMAUFZWho8++ghFRUXo1asXdu3ahaKiIgwZMgS7du1CcXExjh49qkZHAcD69ethNptBMqSHPPB97/zo6GjU1tbi5ZdfBgB4PJ6QtNXV1eHdd99V82fPnsWAAQMAAHPnzkVMTAwAoGPHjti9ezf8fj8AYPHixUhMTMS0adMAAKmpqWrEmW7fvn3q7z/+8Y947bXXkJKSgvHjx+Pzzz9HQUEBPvjgA7XfoUOHYvz48fjd736H+vp6tGnTBgCwZs0a5ObmIicnBy+88ILazvHjx+Hz+WA2mzF37lz0798fADBr1izk5uaqPMnPz8f48eNRU1ODlJQU/PWvfwUANcJKP0ZA4wgNTdOwa9cufPvtt7j++utx4sQJfPfddwCA48ePY+fOndA0Db///e8BAKdPn0ZVVRV2796tRg3NnTsXQOPIp+rqaiQmJqK2thYA1DGsq6vDyZMnce211wJoHNG2bds2rFu3Di1atIDX60XHjh3RpEkTAMCwYcPUso4dO4acOxkZGWjZsiWysrJQUVGB//qv/8KTTz6Ja665BpqmoV+/fkhLSwMAVFZWYu3atTCZTCHHs0WLFnC73YiJiVEjYtatW4dTp07hwIEDSEhIUCPB9uzZg7q6Olx//fXYuXMnzpw5o7b/ySef4LnnnoPFYsGaNWvw6aefIiEhAVVVVXjzzTcBAD179sS2bdsAAH369EF1dTXi4+Px5ZdfAmgc5XHgwAH87W9/w1dffQWgcQRgQkICSkpKsHfvXpw6dQoulwvt27fH559/DgDw+/3Yu3cv9u3bB7/fr0YlxsfHY8CAAbBYLHC73UhNTUVJSQk0TcPQoUNhsVjQtGlTdT5NmzYNPXr0gNlshtlsRkNDA2pra7Fv3z5YLBacPHkSPp9PpbW8vBwAsHLlStjtdgBAVVUVAGDChAnYu3cvvvzySzUirm/fvjhy5Aji4uJw4403YuXKlYiNjcWyZcuwY8cOGI1GRERE4P333wcAdO/eHT6fT12P0dHRap87duxQ54J+TtfV1anz32q1qpGc+nluMBhw9uxZdV4DQH19Pb755hscOnQIt9xyCzRNQ11dHZKTk5GcnAy/3w+r1YpOnTrBaDTCZrPhtttuQ3x8PEji3LlzMBgMaGhowL59+9QoJJPJBKfTiYSEBNTW1uKrr75CfHw80tLSoGkaTp48icGDB6O2thabN29W5ci2bdtw6NAhHDx4EGvXrsUbb7yBvn37Yvz48XjrrbegaRo6dOiALVu2QNM0dO/eHZs3b0bLli2xd+9enDt3DpmZmeq6s1qtyM7OVqM2g8vCyMhIWCwWfPjhhwCAiIgIHD16VI14jIyMVMtIwuVyqbw0m80oLi5W5WhkZCROnDih8tXtdqtj4XA4UFxcHLJM3+4333yDTz/9FJdis9lQWlqq5vXyEUDIiC89TXr5BACjRo3CCy+8AADo0aMHdu3apZYtXLgwZDRUcLl49OhRNZJ2/PjxcLvd6v6QlpamrteGhgb069cPAwYMUNvevn07Ghoa8N1336F169ZYuXKlyoclS5bgww8/xEsvvQQAWLBgAYDGkcz33HPPJfPgn/aTh0+CJPn000/TarVy1apV3L59O/v06aOqQ202G1esWMHt27dz/PjxXLduHffs2cPdu3ezqqqKmqbxlVde4ZQpU5iXl8dRo0Zx69atqolg1qxZ3L9/P5966ilqmsaKigqOGDGCSUlJXL9+Pf/whz/w1ltvDennob/fAIBqL9V7waekpKhqeqCxv4ReNZ2fn69qZ/Sqab3qtrS0lJGRkbzpppsIhI5smDp1KkeOHKnmNU1jQkICH330Ufbt21c1yeXn59Nut9NqtdJqtbJ79+50Op1s2rQps7Ky1NNWfHw8O3fuTIfDwT59+nDVqlXs27cvnU4ni4uL6fF4WF5ervo8tG3blna7nREREdQ0jR06dGBERARvuummkKYJvR1Zb24DoJqGALBr166qDV+v3tZ/58yZM9XT/bRp09i/f39VK+RwONSym266iS+99JKqZk5ISGC/fv1Cnqb0vhjZ2dl89913uXDhQtUcEtxsEwgE+Nprr6lmCz2t0dHRTE9P5/PPP89AIMAbbriB27dv55tvvsmYmBharVZ+8803Kj8feOABfvrpp6pJJSsri/v372cgEFBPq6mpqXS5XMzLy1MdnvX+DnoelpWVsVu3brTb7TQYDGr0isFgoN/v58KFC+l2u+l0Orl69WqWlZWpJ97f/va3qk/WxIkT2atXL/U+Er/fz8TERFqtVnW+6f+OHDmSS5cuVU/X/fv3V/19YmJiVDOZ3kSlXwcej4cxMTGqObe0tJT33nuveteM0Wjk+PHj1b5GjBjByZMnq2PkdrtDnkTNZjPNZrM6Nq+//jqXLFmi1tNr8fx+P3fs2MGCggKazWZ26dJF5a/P5+OAAQNYVVXF2NhYjh49mlu3bmVFRQUNBgOHDx+uOh5HR0ezT58+qmmnd+/e3L9/P2fPnq3SqI+sqqmpCak9WLhwoUqP3synjwIKPlc9Hg+tVitdLhfNZjN79OjBwsJCapqmRtQVFhaqp/rMzEy6XK6Q/jOapjE/P5+apqn+ai6Xi2vWrOHUqVNVrc3QoUMZFxen8nTYsGFMSEgI6dBvtVoZExPDefPm0e120+/3c+3atRwzZowqT5ctW8aUlBSVX6tXr1bfj4qK4uDBg9U5p9c+6bW0OTk5Ie/xCW6mLSkpCelbF1y+BQKBkJqU5s2bU9M0Vf4EN+253e6QpjG9X5I+H9xMdrH3k+lNXHp5pJcxQGPTmN4R/fz3enXt2jXkHAgeEdixY8eQDus33XRTSI1SRUVFSA2X3vyVlJTEpk2bhuxLT7PD4WBOTg49Hg/j4+Pp9XpVHuXl5bG6uporVqzg3LlzuWnTJn722WfcvHkz27Vrx7S0NP7pT38iIM1YP0sPPfQQA4HAj3pZm8lkos/nY1lZGV955RWS5MCBA9Voh6SkJA4cOJDLly9ndnY2rVarGvmgj8CaOHEiA4HAD7aJyxQ6XerFW3pH1H/mxVv6iBB9pJQ+BXd61tfTm63Kysq4cuVK9unTh7Gxseqc0TSN6enpTEtL47hx4zh27NiQwPTvTQkJCYyNjVXVwnrhFxERQavVSpvNxqSkJCYlJalRRpmZmbTb7erFjzab7YKmNZPJdMFv+bFp0vMhKSlJBTsXyxO9k7V+M+zZsydbtmx50b5MZrOZJSUlzM7OVqO4fmx69N9osViYmZnJ0aNHX3AzCn4ppd75WdM01XHzxhtvZGJiYkiTrh7cut1uOhwOdb3+0GQymej3+9mpUycC4BtvvBHSNBIIBFRgHBcXR4fDoYIY/TgHd7wOfgmfyWRikyZNLnip4I89ZvpDSXJycsg29KAmOF/0hxlN09ilS5dLvjTQarWqoOsfvdb0czA+Pl4FkPqkvwDxH712Zfr3TVarlSkpKRwzZgz/93//97J3UNbI/6vPE0IIIYQIQ9JnRwghhBBhTYIdIYQQQoQ1CXaEEEIIEdYk2BFCCCFEWJNgRwghhBBhTYIdIYQQQoQ1CXaEEEIIEdYk2BFCiPO88cYb0DQNR48evdJJEUL8BCTYEUIIIURYk2BHCCGEEGFNgh0hxFWnoaEBCxYsQGpqKux2O/Ly8rB27VoA3zcxvfjii8jNzYXNZkObNm3whz/8IWQbzz33HFq0aAGr1YqUlBQsWrQoZPnp06cxY8YMNGnSBFarFU2bNsUTTzwRss7777+PoqIiOBwOtGvXDnv37r28P1wIcVlIsCOEuOosWLAAq1evxqOPPoqPP/4YkyZNwtChQ7Flyxa1zrRp07Bo0SJs374dPp8PvXr1wtmzZwE0BikDBgzAoEGD8NFHH+Huu+/GnXfeiVWrVqnvDx8+HNXV1ViyZAn27NmDxx57DBERESHpmD17NhYtWoQdO3bAZDKhsrLy3/L7hRA/LfmPQIUQV5XTp08jOjoar776Ktq2bas+v+WWW1BXV4df/OIX6NSpE55++mkMHDgQAPDXv/4Vfr8fq1atwoABAzBkyBAcOnQIr7zyivr+9OnT8eKLL+Ljjz/Gvn370KxZM2zatAldunS5IA1vvPEGOnXqhFdffRVlZWUAgA0bNqC8vBynTp2CzWa7zLkghPgpSc2OEOKq8uc//xl1dXXo2rUrIiIi1LR69WocOHBArRccCEVHR6NZs2bYs2cPAGDPnj0oKSkJ2W5JSQn279+P+vp67Nq1C0ajER06dPjBtOTm5qq/ExISAADffvvtv/wbhRD/XqYrnQAhhAh24sQJAMCLL76IpKSkkGVWqzUk4Pln2e32H7We2WxWf2uaBqCxP5EQ4udFanaEEFeV5s2bw2q14osvvkDTpk1DpiZNmqj13nnnHfV3bW0t9u3bh6ysLABAVlYWtm7dGrLdrVu3IiMjA0ajETk5OWhoaAjpAySECF9SsyOEuKq4XC5MnToVkyZNQkNDA0pLS3Hs2DFs3boVbrcbycnJAIB58+YhJiYGcXFxmD17NrxeL2644QYAwJQpU9CqVSvMnz8fAwcOxNtvv42lS5fikUceAQCkpKRgxIgRqKysxJIlS5CXl4fPP/8c3377LQYMGHClfroQ4jKRYEcIcdWZP38+fD4fFixYgE8++QQejwcFBQWYNWuWakZauHAhJk6ciP379yM/Px8vvPACLBYLAKCgoADPPvss7rrrLsyfPx8JCQmYN28eRo4cqfaxbNkyzJo1C2PHjsWRI0cQCAQwa9asK/FzhRCXmYzGEkL8rOgjpWpra+HxeK50coQQPwPSZ0cIIYQQYU2CHSGEEEKENWnGEkIIIURYk5odIYQQQoQ1CXaEEEIIEdYk2BFCCCFEWJNgRwghhBBhTYIdIYQQQoQ1CXaEEEIIEdYk2BFCCCFEWJNgRwghhBBhTYIdIYQQQoS1/weNWD5T+dKXeAAAAABJRU5ErkJggg==", "text/plain": [ "
" ] diff --git a/tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/train_all.py b/tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/train_all.py new file mode 100644 index 00000000..9af56038 --- /dev/null +++ b/tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/train_all.py @@ -0,0 +1,4 @@ +import os + +for i in range(1, 9): + os.system(f'python model_training.py ResSCNN{i}') \ No newline at end of file From 40b7091b231f98207b7ad293c9920e42db5da2ee Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Fri, 3 May 2024 23:45:38 +0200 Subject: [PATCH 075/379] helper functions for HPO scripts --- .../test_nonsequential/utils/train_test_fn.py | 58 +++++++++++++++++++ .../utils/weight_initialization.py | 26 ++++++++- 2 files changed, 83 insertions(+), 1 deletion(-) diff --git a/tests/test_nonsequential/utils/train_test_fn.py b/tests/test_nonsequential/utils/train_test_fn.py index 30b603e4..dd19b816 100644 --- a/tests/test_nonsequential/utils/train_test_fn.py +++ b/tests/test_nonsequential/utils/train_test_fn.py @@ -56,6 +56,64 @@ def training_loop(device, nb_time_steps, batch_size, feature_map_size, dataloade return epochs_x, epochs_y, epochs_acc +def training_loop_no_tqdm(device, nb_time_steps, batch_size, feature_map_size, dataloader_train, model, loss_fn, optimizer, epochs, dataloader_test): + model.train() + + for e in range(epochs): + for X, y in dataloader_train: + # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width] + X = X.reshape(-1, feature_map_size[2], feature_map_size[0], feature_map_size[1]).to(dtype=torch.float, device=device) + y = y.to(dtype=torch.long, device=device) + + # forward + pred = model(X) + + # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes] + pred = pred.reshape(batch_size, nb_time_steps, -1) + + # accumulate all time-steps output for final prediction + pred = pred.sum(dim = 1) + loss = loss_fn(pred, y) + + # gradient update + optimizer.zero_grad() + loss.backward() + optimizer.step() + + # detach the neuron states and activations from current computation graph(necessary) + model.detach_neuron_states() + + acc = test_no_tqdm(device, nb_time_steps, batch_size, feature_map_size, dataloader_test, model) + + return acc + +def test_no_tqdm(device, nb_time_steps, batch_size, feature_map_size, dataloader_test, model): + correct_predictions = [] + + with torch.no_grad(): + for X, y in dataloader_test: + # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width] + X = X.reshape(-1, feature_map_size[2], feature_map_size[0], feature_map_size[1]).to(dtype=torch.float, device=device) + y = y.to(dtype=torch.long, device=device) + + # forward + output = model(X) + + # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes] + output = output.reshape(batch_size, nb_time_steps, -1) + + # accumulate all time-steps output for final prediction + output = output.sum(dim=1) + + # calculate accuracy + pred = output.argmax(dim=1, keepdim=True) + + # compute the total correct predictions + correct_predictions.append(pred.eq(y.view_as(pred))) + + correct_predictions = torch.cat(correct_predictions) + return correct_predictions.sum().item()/(len(correct_predictions))*100 + def test(device, nb_time_steps, batch_size, feature_map_size, dataloader_test, model): correct_predictions = [] with torch.no_grad(): diff --git a/tests/test_nonsequential/utils/weight_initialization.py b/tests/test_nonsequential/utils/weight_initialization.py index 8c69f2a3..53d2ddb6 100644 --- a/tests/test_nonsequential/utils/weight_initialization.py +++ b/tests/test_nonsequential/utils/weight_initialization.py @@ -1,5 +1,6 @@ import torch.nn as nn import numpy as np +import statistics def rescale_method_1(conv_layer: nn.Conv2d, input_pool_kernel: list, lambda_: float = 1): """ @@ -20,6 +21,29 @@ def rescale_method_1(conv_layer: nn.Conv2d, input_pool_kernel: list, lambda_: fl rescaling_factor = np.mean(rescaling_factors)*lambda_ - print(f'recaling factor: {rescaling_factor} (computed using {len(input_pool_kernel)} kernels and lambda {lambda_})') + # print(f'method 1 - recaling factor: {rescaling_factor} (computed using {len(input_pool_kernel)} kernels and lambda {lambda_})') + + conv_layer.weight.data /= rescaling_factor + +def rescale_method_2(conv_layer: nn.Conv2d, input_pool_kernel: list, lambda_: float = 1): + """ + The `method 2` will use the harmonic mean of the computed rescaling factor for each pooling layer + feeding into `conv_layer` (if there are more than one) to rescale its weights. + + Arguments + --------- + input_pool_kernel (list): the kernels of all pooling layers feeding input to `conv_layer`. + lambda_ (float): scales the computed re-scaling factor. If the outputs of the pooling are too small + the rescaling might lead to vanishing gradients, so we can try to control that by scaling it by + lambda. + """ + rescaling_factors = [] + + for kernel in input_pool_kernel: + rescaling_factors.append(kernel[0]*kernel[1]) + + rescaling_factor = statistics.harmonic_mean(rescaling_factors)*lambda_ + + # print(f'method 2 - recaling factor: {rescaling_factor} (computed using {len(input_pool_kernel)} kernels and lambda {lambda_})') conv_layer.weight.data /= rescaling_factor \ No newline at end of file From e76a70026ada9a6864e4d365f3feed63fb68f3d3 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Mon, 6 May 2024 15:22:21 +0200 Subject: [PATCH 076/379] architecture search scripts --- .../ARCHITECTURES_SEARCH/main.py | 136 ++++++++++++++++ .../using_SumPool2d/models/ResSCNN_1.py | 152 +++++++++++++++++ .../using_SumPool2d/models/ResSCNN_10.py | 121 ++++++++++++++ .../using_SumPool2d/models/ResSCNN_11.py | 112 +++++++++++++ .../using_SumPool2d/models/ResSCNN_2.py | 148 +++++++++++++++++ .../using_SumPool2d/models/ResSCNN_3.py | 151 +++++++++++++++++ .../using_SumPool2d/models/ResSCNN_4.py | 154 ++++++++++++++++++ .../using_SumPool2d/models/ResSCNN_5.py | 143 ++++++++++++++++ .../using_SumPool2d/models/ResSCNN_6.py | 146 +++++++++++++++++ .../using_SumPool2d/models/ResSCNN_7.py | 146 +++++++++++++++++ .../using_SumPool2d/models/ResSCNN_8.py | 151 +++++++++++++++++ .../using_SumPool2d/models/ResSCNN_9.py | 124 ++++++++++++++ .../test_nonsequential/utils/train_test_fn.py | 94 ++++++++--- 13 files changed, 1759 insertions(+), 19 deletions(-) create mode 100644 tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/main.py create mode 100644 tests/test_nonsequential/using_SumPool2d/models/ResSCNN_1.py create mode 100644 tests/test_nonsequential/using_SumPool2d/models/ResSCNN_10.py create mode 100644 tests/test_nonsequential/using_SumPool2d/models/ResSCNN_11.py create mode 100644 tests/test_nonsequential/using_SumPool2d/models/ResSCNN_2.py create mode 100644 tests/test_nonsequential/using_SumPool2d/models/ResSCNN_3.py create mode 100644 tests/test_nonsequential/using_SumPool2d/models/ResSCNN_4.py create mode 100644 tests/test_nonsequential/using_SumPool2d/models/ResSCNN_5.py create mode 100644 tests/test_nonsequential/using_SumPool2d/models/ResSCNN_6.py create mode 100644 tests/test_nonsequential/using_SumPool2d/models/ResSCNN_7.py create mode 100644 tests/test_nonsequential/using_SumPool2d/models/ResSCNN_8.py create mode 100644 tests/test_nonsequential/using_SumPool2d/models/ResSCNN_9.py diff --git a/tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/main.py b/tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/main.py new file mode 100644 index 00000000..976aded5 --- /dev/null +++ b/tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/main.py @@ -0,0 +1,136 @@ +import torch, tonic, sys, random +import numpy as np +from torch.utils.data import DataLoader +from sinabs.activation.surrogate_gradient_fn import PeriodicExponential +from torch.nn import CrossEntropyLoss +from torch.optim import Adam +import os +from tqdm import tqdm + +sys.path.append('../../utils') +sys.path.append('../models') + +from train_test_fn import training_loop_no_tqdm, load_dataset, split_train_validation_used_seed, load_architecture +from weight_initialization import rescale_method_1 + +if torch.cuda.is_available(): + device = torch.device('cuda:0') + print('device: ', torch.cuda.get_device_name(0)) +else: + device = torch.device('cpu') + +torch.backends.cudnn.enabled = False +torch.backends.cudnn.deterministic = True +random.seed(1) +torch.manual_seed(1) +torch.cuda.manual_seed(1) + +### Initialization #################################################### + +total_architectures = 11 + +lr = 5e-5 +batch_size = 8 +num_workers = 4 +n_time_steps = 50 +epochs = 25 +w_rescale_lambda = 0.8 + +spk_thr = 2.0 +v_min = -0.313 +grad_scale = 1.534 +grad_width = 0.759 + +validation_ratio = 0.2 +prev_used_seed = 1 + +loss_fn = CrossEntropyLoss() + +directory = f'./architectures_results_2' + +if not os.path.exists(directory): + os.makedirs(directory) + +with open(f'./architectures_results_2/fixed_parameters.txt', 'w') as file: + file.write(f'lr: {lr}\n') + file.write(f'batch_size: {batch_size}\n') + file.write(f'num_workers: {num_workers}\n') + file.write(f'n_time_steps: {n_time_steps}\n') + file.write(f'epochs: {epochs}\n') + file.write(f'w_rescale_lambda: {w_rescale_lambda}\n') + file.write(f'spk_thr: {spk_thr}\n') + file.write(f'v_min: {v_min}\n') + file.write(f'grad_scale: {grad_scale}\n') + file.write(f'grad_width: {grad_width}\n') + file.write(f'validation_ratio: {validation_ratio}\n') + file.write(f'prev_used_seed: {prev_used_seed}\n') + +### Data Loading ##################################################### + +snn_train_dataset, snn_test_dataset, sensor_size = load_dataset('DVSGESTURE', n_time_steps) +train_dataset, validation_dataset = split_train_validation_used_seed(validation_ratio, snn_train_dataset, prev_used_seed) + +disk_cache_train = tonic.DiskCachedDataset( + dataset=train_dataset, + cache_path='./cached_train' +) +snn_train_dataloader = DataLoader(disk_cache_train, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True) + +disk_cache_validation = tonic.DiskCachedDataset( + dataset=validation_dataset, + cache_path='./cached_validation' +) +snn_validation_dataloader = DataLoader(disk_cache_validation, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True) + +disk_cache_test = tonic.DiskCachedDataset( + dataset=snn_test_dataset, + cache_path='./cached_test' +) +snn_test_dataloader = DataLoader(disk_cache_test, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=False) + +### Training Loop ########################################################## + +train_p_bar = tqdm(range(9, total_architectures+1)) + +for iter in train_p_bar: + achitecture = f'ResSCNN_{iter}' + + # instantiate model. + csnn = load_architecture( + achitecture, + sensor_size, + 11, + batch_size, + PeriodicExponential(grad_scale=grad_scale, grad_width=grad_width), + v_min, + spk_thr + ).to(device) + + csnn.init_weights() + csnn.rescale_conv_weights(rescale_method_1, w_rescale_lambda) + + # instantiate optimizer. + optimizer = Adam(csnn.parameters(), lr = lr, betas = (0.9, 0.999), eps = 1e-8) + + # train/test model. + epochs_x, epochs_y, epochs_acc = training_loop_no_tqdm( + device, + n_time_steps, + batch_size, + sensor_size, + snn_train_dataloader, + csnn, + loss_fn, + optimizer, + epochs, + snn_validation_dataloader, + True) + + # export model data. + with open(f'./architectures_results_2/{achitecture}-training_metrics.npy', 'wb') as f: + np.save(f, np.array(epochs_x)) + np.save(f, np.array(epochs_y)) + np.save(f, np.array(epochs_acc)) + + # update progress bar + train_p_bar.set_description(f'{iter}/{total_architectures} - model {achitecture} - acc.: {np.round(epochs_acc[-1], 2)}') \ No newline at end of file diff --git a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_1.py b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_1.py new file mode 100644 index 00000000..b8f5b838 --- /dev/null +++ b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_1.py @@ -0,0 +1,152 @@ +import torch.nn as nn +import sinabs.layers as sl +from sinabs.exodus.layers import IAFSqueeze + +class SCNN(nn.Module): + def __init__(self, input_size, nb_classes, batch_size, surrogate_fn, min_v_mem=-0.313, spk_thr=2.0) -> None: + super().__init__() + + self.conv1 = nn.Conv2d(2, 8, 2, 1, bias=False) + self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool1 = sl.SumPool2d(2,2) + self.pool1a = sl.SumPool2d(4,4) + + self.conv2 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool2 = sl.SumPool2d(2,2) + + self.conv3 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool3 = sl.SumPool2d(2,2) + + self.conv4 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool4 = sl.SumPool2d(2,2) + + self.flat = nn.Flatten() + + flat_s = SCNN.get_flatten_size(input_size) + + self.fc1 = nn.Linear(flat_s, 100, bias=False) + self.iaf1_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc2 = nn.Linear(100, 100, bias=False) + self.iaf2_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc3 = nn.Linear(100, 100, bias=False) + self.iaf3_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc4 = nn.Linear(100, 100, bias=False) + self.iaf4_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc5 = nn.Linear(100, nb_classes, bias=False) + self.iaf5_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + # skip + + self.merge1 = sl.Merge() + + @staticmethod + def get_flatten_size(input_size): + conv1_dims = SCNN.conv2d_output_size(input_size, 8, (2, 2)) + pool1_dims = SCNN.pool_output_size(conv1_dims, 8, (2, 2)) + + conv2_dims = SCNN.conv2d_output_size(pool1_dims, 8, (2, 2)) + pool2_dims = SCNN.pool_output_size(conv2_dims, 8, (2, 2)) + + conv3_dims = SCNN.conv2d_output_size(pool2_dims, 8, (2, 2)) + pool3_dims = SCNN.pool_output_size(conv3_dims, 8, (2, 2)) + + conv4_dims = SCNN.conv2d_output_size(pool3_dims, 8, (2, 2)) + pool4_dims = SCNN.pool_output_size(conv4_dims, 8, (2, 2)) + + return pool4_dims[0]*pool4_dims[1]*pool4_dims[2] + + @staticmethod + def conv2d_output_size(input_size, out_channels, kernel_size, stride=1, padding=0, dilation=1): + input_height, input_width, input_channels = input_size + kernel_height, kernel_width = kernel_size + + output_height = ((input_height + 2 * padding - dilation * (kernel_height - 1) - 1) // stride) + 1 + output_width = ((input_width + 2 * padding - dilation * (kernel_width - 1) - 1) // stride) + 1 + + return (output_height, output_width, out_channels) + + @staticmethod + def pool_output_size(input_size, out_channels, kernel_size, stride=None, padding=0): + input_height, input_width, input_channels = input_size + kernel_height, kernel_width = kernel_size + + if stride is None: + stride = kernel_height + + output_height = ((input_height + 2 * padding - kernel_height) // stride) + 1 + output_width = ((input_width + 2 * padding - kernel_width) // stride) + 1 + + return (output_height, output_width, out_channels) + + def detach_neuron_states(self): + for name, layer in self.named_modules(): + if name != '': + if isinstance(layer, sl.StatefulLayer): + for name, buffer in layer.named_buffers(): + buffer.detach_() + + def init_weights(self): + for name, layer in self.named_modules(): + if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear): + nn.init.xavier_normal_(layer.weight.data) + + def rescale_conv_weights(self, rescale_fn, lambda_): + rescale_fn(self.conv2, [(2, 2)], lambda_) + rescale_fn(self.conv3, [(2, 2), (4, 4)], lambda_) + rescale_fn(self.conv4, [(2, 2)], lambda_) + + + def forward(self, x): + # conv 1 + con1_out = self.conv1(x) + iaf1_out = self.iaf1(con1_out) + pool1_out = self.pool1(iaf1_out) + pool1a_out = self.pool1a(iaf1_out) + + # conv 2 + conv2_out = self.conv2(pool1_out) + iaf2_out = self.iaf2(conv2_out) + pool2_out = self.pool2(iaf2_out) + + merge1_out = self.merge1(pool1a_out, pool2_out) + + # conv 3 + conv3_out = self.conv3(merge1_out) + iaf3_out = self.iaf3(conv3_out) + pool3_out = self.pool3(iaf3_out) + + # conv 4 + conv4_out = self.conv4(pool3_out) + iaf4_out = self.iaf4(conv4_out) + pool4_out = self.pool4(iaf4_out) + + flat_out = self.flat(pool4_out) + + # fc 1 + fc1_out = self.fc1(flat_out) + iaf1_fc_out = self.iaf1_fc(fc1_out) + + # fc 2 + fc2_out = self.fc2(iaf1_fc_out) + iaf2_fc_out = self.iaf2_fc(fc2_out) + + # fc 3 + fc3_out = self.fc3(iaf2_fc_out) + iaf3_fc_out = self.iaf3_fc(fc3_out) + + # fc 4 + fc4_out = self.fc4(iaf3_fc_out) + iaf4_fc_out = self.iaf4_fc(fc4_out) + + # fc 5 + fc5_out = self.fc5(iaf4_fc_out) + iaf5_fc_out = self.iaf5_fc(fc5_out) + + return iaf5_fc_out \ No newline at end of file diff --git a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_10.py b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_10.py new file mode 100644 index 00000000..9e631bd4 --- /dev/null +++ b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_10.py @@ -0,0 +1,121 @@ +import torch.nn as nn +import sinabs.layers as sl +from sinabs.exodus.layers import IAFSqueeze + +class SCNN(nn.Module): + def __init__(self, input_size, nb_classes, batch_size, surrogate_fn, min_v_mem=-0.313, spk_thr=2.0) -> None: + super().__init__() + + self.conv1 = nn.Conv2d(2, 8, 2, 1, bias=False) + self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.conv2 = nn.Conv2d(8, 8, 3, 1, 1, bias=False) + self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool2a = sl.SumPool2d(2,2) + + self.conv3 = nn.Conv2d(8, 8, 3, 1, 1, bias=False) + self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool3a = sl.SumPool2d(4,4) + + self.conv4 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool4 = sl.SumPool2d(2,2) + self.pool4a = sl.SumPool2d(4,4) + + self.conv5 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf5 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool5 = sl.SumPool2d(2,2) + self.pool5a = sl.SumPool2d(4,4) + + self.conv6 = nn.Conv2d(8, 8, 3, 1, 1, bias=False) + self.iaf6 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.conv7 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf7 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool7 = sl.SumPool2d(2,2) + + self.conv8 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf8 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool8 = sl.SumPool2d(2,2) + + self.flat = nn.Flatten() + + self.fc_out = nn.Linear(392, nb_classes, bias=False) + self.iaf_fc_out = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + # skip + + self.merge1 = sl.Merge() + self.merge2 = sl.Merge() + self.merge3 = sl.Merge() + self.merge4 = sl.Merge() + self.merge5 = sl.Merge() + + def detach_neuron_states(self): + for name, layer in self.named_modules(): + if name != '': + if isinstance(layer, sl.StatefulLayer): + for name, buffer in layer.named_buffers(): + buffer.detach_() + + def init_weights(self): + for name, layer in self.named_modules(): + if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear): + nn.init.xavier_normal_(layer.weight.data) + + def rescale_conv_weights(self, rescale_fn, lambda_): + rescale_fn(self.conv5, [(2,2), (2,2)], lambda_) + rescale_fn(self.conv6, [(4,4), (2,2)], lambda_) + rescale_fn(self.conv7, [(4,4)], lambda_) + rescale_fn(self.conv8, [(4,4), (2,2)], lambda_) + + def forward(self, x): + # -- conv block 1 --- + con1_out = self.conv1(x) + iaf1_out = self.iaf1(con1_out) # to CONV 4 + # -- conv block 2 --- + conv2_out = self.conv2(iaf1_out) + iaf2_out = self.iaf2(conv2_out) + pool2a_out = self.pool2a(iaf2_out) # to CONV 5 + # -- conv block 3 --- + conv3_out = self.conv3(iaf2_out) + iaf3_out = self.iaf3(conv3_out) + pool3a_out = self.pool3a(iaf3_out) # to CONV 6 + # -- conv block 4 --- + #print(iaf1_out.shape, iaf3_out.shape) + merge1_out = self.merge1(iaf1_out, iaf3_out) + conv4_out = self.conv4(merge1_out) + iaf4_out = self.iaf4(conv4_out) + pool4_out = self.pool4(iaf4_out) + pool4a_out = self.pool4a(iaf4_out) # to CONV 7 + # -- conv block 5 --- + #print(pool2a_out.shape, pool4_out.shape) + merge2_out = self.merge2(pool2a_out, pool4_out) + conv5_out = self.conv5(merge2_out) + iaf5_out = self.iaf5(conv5_out) + pool5_out = self.pool5(iaf5_out) + pool5a_out = self.pool5a(iaf5_out) # to CONV 8 + # -- conv block 6 --- + #print(pool3a_out.shape, pool5_out.shape) + merge3_out = self.merge3(pool3a_out, pool5_out) + conv6_out = self.conv6(merge3_out) + iaf6_out = self.iaf6(conv6_out) + # -- conv block 7 --- + #print(pool4a_out.shape, iaf6_out.shape) + merge4_out = self.merge4(pool4a_out, iaf6_out) + conv7_out = self.conv7(merge4_out) + iaf7_out = self.iaf7(conv7_out) + pool7_out = self.pool7(iaf7_out) + # -- conv block 8 --- + #print(pool5a_out.shape, pool7_out.shape) + merge5_out = self.merge5(pool5a_out, pool7_out) + conv8_out = self.conv8(merge5_out) + iaf8_out = self.iaf8(conv8_out) + pool8_out = self.pool8(iaf8_out) + # -- output -- + flat = self.flat(pool8_out) + #print(flat.shape) + fc_out = self.fc_out(flat) + iaf_fc_out = self.iaf_fc_out(fc_out) + + return iaf_fc_out \ No newline at end of file diff --git a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_11.py b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_11.py new file mode 100644 index 00000000..209f00bd --- /dev/null +++ b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_11.py @@ -0,0 +1,112 @@ +import torch.nn as nn +import sinabs.layers as sl +from sinabs.exodus.layers import IAFSqueeze + +class SCNN(nn.Module): + def __init__(self, input_size, nb_classes, batch_size, surrogate_fn, min_v_mem=-0.313, spk_thr=2.0) -> None: + super().__init__() + + self.conv1 = nn.Conv2d(2, 8, 2, 1, bias=False) + self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.conv2 = nn.Conv2d(8, 8, 3, 1, 1, bias=False) + self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.conv3 = nn.Conv2d(8, 8, 3, 1, 1, bias=False) + self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool3a = sl.SumPool2d(4,4) + + self.conv4 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool4 = sl.SumPool2d(2,2) + + self.conv5 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf5 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool5 = sl.SumPool2d(2,2) + self.pool5a = sl.SumPool2d(4,4) + + self.conv6 = nn.Conv2d(8, 8, 3, 1, 1, bias=False) + self.iaf6 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.conv7 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf7 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool7 = sl.SumPool2d(2,2) + + self.conv8 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf8 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool8 = sl.SumPool2d(2,2) + + self.flat = nn.Flatten() + + self.fc_out = nn.Linear(392, nb_classes, bias=False) + self.iaf_fc_out = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + # skip + + self.merge1 = sl.Merge() + self.merge2 = sl.Merge() + self.merge3 = sl.Merge() + self.merge4 = sl.Merge() + + def detach_neuron_states(self): + for name, layer in self.named_modules(): + if name != '': + if isinstance(layer, sl.StatefulLayer): + for name, buffer in layer.named_buffers(): + buffer.detach_() + + def init_weights(self): + for name, layer in self.named_modules(): + if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear): + nn.init.xavier_normal_(layer.weight.data) + + def rescale_conv_weights(self, rescale_fn, lambda_): + rescale_fn(self.conv5, [(2,2)], lambda_) + rescale_fn(self.conv6, [(4,4), (2,2)], lambda_) + rescale_fn(self.conv8, [(4,4), (2,2)], lambda_) + + def forward(self, x): + # -- conv block 1 --- + con1_out = self.conv1(x) + iaf1_out = self.iaf1(con1_out) # to CONV 4 + # -- conv block 2 --- + conv2_out = self.conv2(iaf1_out) + iaf2_out = self.iaf2(conv2_out) + # -- conv block 3 --- + conv3_out = self.conv3(iaf2_out) + iaf3_out = self.iaf3(conv3_out) + pool3a_out = self.pool3a(iaf3_out) # to CONV 6 + # -- conv block 4 --- + #print(iaf1_out.shape, iaf3_out.shape) + merge1_out = self.merge1(iaf1_out, iaf3_out) + conv4_out = self.conv4(merge1_out) + iaf4_out = self.iaf4(conv4_out) + pool4_out = self.pool4(iaf4_out) + # -- conv block 5 --- + conv5_out = self.conv5(pool4_out) + iaf5_out = self.iaf5(conv5_out) + pool5_out = self.pool5(iaf5_out) + pool5a_out = self.pool5a(iaf5_out) # to CONV 8 + # -- conv block 6 --- + #print(pool3a_out.shape, pool5_out.shape) + merge3_out = self.merge3(pool3a_out, pool5_out) + conv6_out = self.conv6(merge3_out) + iaf6_out = self.iaf6(conv6_out) + # -- conv block 7 --- + conv7_out = self.conv7(iaf6_out) + iaf7_out = self.iaf7(conv7_out) + pool7_out = self.pool7(iaf7_out) + # -- conv block 8 --- + #print(pool5a_out.shape, pool7_out.shape) + merge4_out = self.merge4(pool5a_out, pool7_out) + conv8_out = self.conv8(merge4_out) + iaf8_out = self.iaf8(conv8_out) + pool8_out = self.pool8(iaf8_out) + # -- output -- + flat = self.flat(pool8_out) + #print(flat.shape) + fc_out = self.fc_out(flat) + iaf_fc_out = self.iaf_fc_out(fc_out) + + + return iaf_fc_out \ No newline at end of file diff --git a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_2.py b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_2.py new file mode 100644 index 00000000..bb9877df --- /dev/null +++ b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_2.py @@ -0,0 +1,148 @@ +import torch.nn as nn +import sinabs.layers as sl +from sinabs.exodus.layers import IAFSqueeze + +class SCNN(nn.Module): + def __init__(self, input_size, nb_classes, batch_size, surrogate_fn, min_v_mem=-0.313, spk_thr=2.0) -> None: + super().__init__() + + self.conv1 = nn.Conv2d(2, 8, 2, 1, bias=False) + self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool1 = sl.SumPool2d(2,2) + self.pool1a = sl.SumPool2d(4,4) + + self.conv2 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool2 = sl.SumPool2d(2,2) + + self.conv3 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool3 = sl.SumPool2d(2,2) + self.pool3a = sl.SumPool2d(4,4) + + self.conv4 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool4 = sl.SumPool2d(2,2) + + self.flat = nn.Flatten() + + flat_s = SCNN.get_flatten_size(input_size) + + self.fc1 = nn.Linear(flat_s, 100, bias=False) + self.iaf1_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc2 = nn.Linear(100, 100, bias=False) + self.iaf2_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc3 = nn.Linear(100, 100, bias=False) + self.iaf3_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc4 = nn.Linear(100, 100, bias=False) + self.iaf4_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc5 = nn.Linear(100, nb_classes, bias=False) + self.iaf5_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + # skip + + self.merge1 = sl.Merge() + self.merge2 = sl.Merge() + + @staticmethod + def get_flatten_size(input_size): + conv1_dims = SCNN.conv2d_output_size(input_size, 8, (2, 2)) + pool1_dims = SCNN.pool_output_size(conv1_dims, 8, (2, 2)) + + conv2_dims = SCNN.conv2d_output_size(pool1_dims, 8, (2, 2)) + pool2_dims = SCNN.pool_output_size(conv2_dims, 8, (2, 2)) + + conv3_dims = SCNN.conv2d_output_size(pool2_dims, 8, (2, 2)) + pool3_dims = SCNN.pool_output_size(conv3_dims, 8, (2, 2)) + + conv4_dims = SCNN.conv2d_output_size(pool3_dims, 8, (2, 2)) + pool4_dims = SCNN.pool_output_size(conv4_dims, 8, (2, 2)) + + return pool4_dims[0]*pool4_dims[1]*pool4_dims[2] + + @staticmethod + def conv2d_output_size(input_size, out_channels, kernel_size, stride=1, padding=0, dilation=1): + input_height, input_width, input_channels = input_size + kernel_height, kernel_width = kernel_size + + output_height = ((input_height + 2 * padding - dilation * (kernel_height - 1) - 1) // stride) + 1 + output_width = ((input_width + 2 * padding - dilation * (kernel_width - 1) - 1) // stride) + 1 + + return (output_height, output_width, out_channels) + + @staticmethod + def pool_output_size(input_size, out_channels, kernel_size, stride=None, padding=0): + input_height, input_width, input_channels = input_size + kernel_height, kernel_width = kernel_size + + if stride is None: + stride = kernel_height + + output_height = ((input_height + 2 * padding - kernel_height) // stride) + 1 + output_width = ((input_width + 2 * padding - kernel_width) // stride) + 1 + + return (output_height, output_width, out_channels) + + def detach_neuron_states(self): + for name, layer in self.named_modules(): + if name != '': + if isinstance(layer, sl.StatefulLayer): + for name, buffer in layer.named_buffers(): + buffer.detach_() + + def init_weights(self): + for name, layer in self.named_modules(): + if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear): + nn.init.xavier_normal_(layer.weight.data) + + def rescale_conv_weights(self, rescale_fn, lambda_): + rescale_fn(self.conv2, [(2,2)], lambda_) + rescale_fn(self.conv3, [(4,4), (2,2)], lambda_) + rescale_fn(self.conv4, [(2,2)], lambda_) + + def forward(self, x): + # conv 1 + con1_out = self.conv1(x) + iaf1_out = self.iaf1(con1_out) + pool1_out = self.pool1(iaf1_out) + pool1a_out = self.pool1a(iaf1_out) + # conv 2 + conv2_out = self.conv2(pool1_out) + iaf2_out = self.iaf2(conv2_out) + pool2_out = self.pool2(iaf2_out) + + merge1_out = self.merge1(pool1a_out, pool2_out) + # conv 3 + conv3_out = self.conv3(merge1_out) + iaf3_out = self.iaf3(conv3_out) + pool3_out = self.pool3(iaf3_out) + pool3a_out = self.pool3a(iaf3_out) + # conv 4 + conv4_out = self.conv4(pool3_out) + iaf4_out = self.iaf4(conv4_out) + pool4_out = self.pool4(iaf4_out) + + merge2_out = self.merge2(pool3a_out, pool4_out) + + flat_out = self.flat(merge2_out) + # fc 1 + fc1_out = self.fc1(flat_out) + iaf1_fc_out = self.iaf1_fc(fc1_out) + # fc 2 + fc2_out = self.fc2(iaf1_fc_out) + iaf2_fc_out = self.iaf2_fc(fc2_out) + # fc 3 + fc3_out = self.fc3(iaf2_fc_out) + iaf3_fc_out = self.iaf3_fc(fc3_out) + # fc 4 + fc4_out = self.fc4(iaf3_fc_out) + iaf4_fc_out = self.iaf4_fc(fc4_out) + # fc 5 + fc5_out = self.fc5(iaf4_fc_out) + iaf5_fc_out = self.iaf5_fc(fc5_out) + + return iaf5_fc_out \ No newline at end of file diff --git a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_3.py b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_3.py new file mode 100644 index 00000000..0975a51a --- /dev/null +++ b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_3.py @@ -0,0 +1,151 @@ +import torch.nn as nn +import sinabs.layers as sl +from sinabs.exodus.layers import IAFSqueeze + +class SCNN(nn.Module): + def __init__(self, input_size, nb_classes, batch_size, surrogate_fn, min_v_mem=-0.313, spk_thr=2.0) -> None: + super().__init__() + + self.conv1 = nn.Conv2d(2, 8, 2, 1, bias=False) + self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool1 = sl.SumPool2d(2,2) + self.pool1a = sl.SumPool2d(4,4) + + self.conv2 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool2 = sl.SumPool2d(2,2) + + self.conv3 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool3 = sl.SumPool2d(2,2) + self.pool3a = sl.SumPool2d(4,4) + + self.conv4 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool4 = sl.SumPool2d(2,2) + + self.flat = nn.Flatten() + + flat_s = SCNN.get_flatten_size(input_size) + + self.fc1 = nn.Linear(flat_s, 100, bias=False) + self.iaf1_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc2 = nn.Linear(100, 100, bias=False) + self.iaf2_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc3 = nn.Linear(100, 100, bias=False) + self.iaf3_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc4 = nn.Linear(100, 100, bias=False) + self.iaf4_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc5 = nn.Linear(100, nb_classes, bias=False) + self.iaf5_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + # skip + + self.merge1 = sl.Merge() + self.merge2 = sl.Merge() + self.merge3 = sl.Merge() + + @staticmethod + def get_flatten_size(input_size): + conv1_dims = SCNN.conv2d_output_size(input_size, 8, (2, 2)) + pool1_dims = SCNN.pool_output_size(conv1_dims, 8, (2, 2)) + + conv2_dims = SCNN.conv2d_output_size(pool1_dims, 8, (2, 2)) + pool2_dims = SCNN.pool_output_size(conv2_dims, 8, (2, 2)) + + conv3_dims = SCNN.conv2d_output_size(pool2_dims, 8, (2, 2)) + pool3_dims = SCNN.pool_output_size(conv3_dims, 8, (2, 2)) + + conv4_dims = SCNN.conv2d_output_size(pool3_dims, 8, (2, 2)) + pool4_dims = SCNN.pool_output_size(conv4_dims, 8, (2, 2)) + + return pool4_dims[0]*pool4_dims[1]*pool4_dims[2] + + @staticmethod + def conv2d_output_size(input_size, out_channels, kernel_size, stride=1, padding=0, dilation=1): + input_height, input_width, input_channels = input_size + kernel_height, kernel_width = kernel_size + + output_height = ((input_height + 2 * padding - dilation * (kernel_height - 1) - 1) // stride) + 1 + output_width = ((input_width + 2 * padding - dilation * (kernel_width - 1) - 1) // stride) + 1 + + return (output_height, output_width, out_channels) + + @staticmethod + def pool_output_size(input_size, out_channels, kernel_size, stride=None, padding=0): + input_height, input_width, input_channels = input_size + kernel_height, kernel_width = kernel_size + + if stride is None: + stride = kernel_height + + output_height = ((input_height + 2 * padding - kernel_height) // stride) + 1 + output_width = ((input_width + 2 * padding - kernel_width) // stride) + 1 + + return (output_height, output_width, out_channels) + + def detach_neuron_states(self): + for name, layer in self.named_modules(): + if name != '': + if isinstance(layer, sl.StatefulLayer): + for name, buffer in layer.named_buffers(): + buffer.detach_() + + def init_weights(self): + for name, layer in self.named_modules(): + if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear): + nn.init.xavier_normal_(layer.weight.data) + + def rescale_conv_weights(self, rescale_fn, lambda_): + rescale_fn(self.conv2, [(2,2)], lambda_) + rescale_fn(self.conv3, [(4,4), (2,2)], lambda_) + rescale_fn(self.conv4, [(2,2)], lambda_) + + def forward(self, x): + # conv 1 + con1_out = self.conv1(x) + iaf1_out = self.iaf1(con1_out) + pool1_out = self.pool1(iaf1_out) + pool1a_out = self.pool1a(iaf1_out) + # conv 2 + conv2_out = self.conv2(pool1_out) + iaf2_out = self.iaf2(conv2_out) + pool2_out = self.pool2(iaf2_out) + + merge1_out = self.merge1(pool1a_out, pool2_out) + # conv 3 + conv3_out = self.conv3(merge1_out) + iaf3_out = self.iaf3(conv3_out) + pool3_out = self.pool3(iaf3_out) + pool3a_out = self.pool3a(iaf3_out) + # conv 4 + conv4_out = self.conv4(pool3_out) + iaf4_out = self.iaf4(conv4_out) + pool4_out = self.pool4(iaf4_out) + + merge2_out = self.merge2(pool3a_out, pool4_out) + + flat_out = self.flat(merge2_out) + # fc 1 + fc1_out = self.fc1(flat_out) + iaf1_fc_out = self.iaf1_fc(fc1_out) + # fc 2 + fc2_out = self.fc2(iaf1_fc_out) + iaf2_fc_out = self.iaf2_fc(fc2_out) + + merge3_out = self.merge3(iaf1_fc_out, iaf2_fc_out) + # fc 3 + fc3_out = self.fc3(merge3_out) + iaf3_fc_out = self.iaf3_fc(fc3_out) + # fc 4 + fc4_out = self.fc4(iaf3_fc_out) + iaf4_fc_out = self.iaf4_fc(fc4_out) + # fc 5 + fc5_out = self.fc5(iaf4_fc_out) + iaf5_fc_out = self.iaf5_fc(fc5_out) + + return iaf5_fc_out \ No newline at end of file diff --git a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_4.py b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_4.py new file mode 100644 index 00000000..4d07ff37 --- /dev/null +++ b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_4.py @@ -0,0 +1,154 @@ +import torch.nn as nn +import sinabs.layers as sl +from sinabs.exodus.layers import IAFSqueeze + +class SCNN(nn.Module): + def __init__(self, input_size, nb_classes, batch_size, surrogate_fn, min_v_mem=-0.313, spk_thr=2.0) -> None: + super().__init__() + + self.conv1 = nn.Conv2d(2, 8, 2, 1, bias=False) + self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool1 = sl.SumPool2d(2,2) + self.pool1a = sl.SumPool2d(4,4) + + self.conv2 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool2 = sl.SumPool2d(2,2) + + self.conv3 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool3 = sl.SumPool2d(2,2) + self.pool3a = sl.SumPool2d(4,4) + + self.conv4 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool4 = sl.SumPool2d(2,2) + + self.flat = nn.Flatten() + + flat_s = SCNN.get_flatten_size(input_size) + + self.fc1 = nn.Linear(flat_s, 100, bias=False) + self.iaf1_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc2 = nn.Linear(100, 100, bias=False) + self.iaf2_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc3 = nn.Linear(100, 100, bias=False) + self.iaf3_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc4 = nn.Linear(100, 100, bias=False) + self.iaf4_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc5 = nn.Linear(100, nb_classes, bias=False) + self.iaf5_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + # skip + + self.merge1 = sl.Merge() + self.merge2 = sl.Merge() + self.merge3 = sl.Merge() + self.merge4 = sl.Merge() + + @staticmethod + def get_flatten_size(input_size): + conv1_dims = SCNN.conv2d_output_size(input_size, 8, (2, 2)) + pool1_dims = SCNN.pool_output_size(conv1_dims, 8, (2, 2)) + + conv2_dims = SCNN.conv2d_output_size(pool1_dims, 8, (2, 2)) + pool2_dims = SCNN.pool_output_size(conv2_dims, 8, (2, 2)) + + conv3_dims = SCNN.conv2d_output_size(pool2_dims, 8, (2, 2)) + pool3_dims = SCNN.pool_output_size(conv3_dims, 8, (2, 2)) + + conv4_dims = SCNN.conv2d_output_size(pool3_dims, 8, (2, 2)) + pool4_dims = SCNN.pool_output_size(conv4_dims, 8, (2, 2)) + + return pool4_dims[0]*pool4_dims[1]*pool4_dims[2] + + @staticmethod + def conv2d_output_size(input_size, out_channels, kernel_size, stride=1, padding=0, dilation=1): + input_height, input_width, input_channels = input_size + kernel_height, kernel_width = kernel_size + + output_height = ((input_height + 2 * padding - dilation * (kernel_height - 1) - 1) // stride) + 1 + output_width = ((input_width + 2 * padding - dilation * (kernel_width - 1) - 1) // stride) + 1 + + return (output_height, output_width, out_channels) + + @staticmethod + def pool_output_size(input_size, out_channels, kernel_size, stride=None, padding=0): + input_height, input_width, input_channels = input_size + kernel_height, kernel_width = kernel_size + + if stride is None: + stride = kernel_height + + output_height = ((input_height + 2 * padding - kernel_height) // stride) + 1 + output_width = ((input_width + 2 * padding - kernel_width) // stride) + 1 + + return (output_height, output_width, out_channels) + + def detach_neuron_states(self): + for name, layer in self.named_modules(): + if name != '': + if isinstance(layer, sl.StatefulLayer): + for name, buffer in layer.named_buffers(): + buffer.detach_() + + def init_weights(self): + for name, layer in self.named_modules(): + if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear): + nn.init.xavier_normal_(layer.weight.data) + + def rescale_conv_weights(self, rescale_fn, lambda_): + rescale_fn(self.conv2, [(2,2)], lambda_) + rescale_fn(self.conv3, [(4,4), (2,2)], lambda_) + rescale_fn(self.conv4, [(2,2)], lambda_) + + def forward(self, x): + # conv 1 + con1_out = self.conv1(x) + iaf1_out = self.iaf1(con1_out) + pool1_out = self.pool1(iaf1_out) + pool1a_out = self.pool1a(iaf1_out) + # conv 2 + conv2_out = self.conv2(pool1_out) + iaf2_out = self.iaf2(conv2_out) + pool2_out = self.pool2(iaf2_out) + + merge1_out = self.merge1(pool1a_out, pool2_out) + # conv 3 + conv3_out = self.conv3(merge1_out) + iaf3_out = self.iaf3(conv3_out) + pool3_out = self.pool3(iaf3_out) + pool3a_out = self.pool3a(iaf3_out) + # conv 4 + conv4_out = self.conv4(pool3_out) + iaf4_out = self.iaf4(conv4_out) + pool4_out = self.pool4(iaf4_out) + + merge2_out = self.merge2(pool3a_out, pool4_out) + + flat_out = self.flat(merge2_out) + # fc 1 + fc1_out = self.fc1(flat_out) + iaf1_fc_out = self.iaf1_fc(fc1_out) + # fc 2 + fc2_out = self.fc2(iaf1_fc_out) + iaf2_fc_out = self.iaf2_fc(fc2_out) + + merge3_out = self.merge3(iaf1_fc_out, iaf2_fc_out) + # fc 3 + fc3_out = self.fc3(merge3_out) + iaf3_fc_out = self.iaf3_fc(fc3_out) + # fc 4 + fc4_out = self.fc4(iaf3_fc_out) + iaf4_fc_out = self.iaf4_fc(fc4_out) + + merge4_out = self.merge4(iaf3_fc_out, iaf4_fc_out) + # fc 5 + fc5_out = self.fc5(merge4_out) + iaf5_fc_out = self.iaf5_fc(fc5_out) + + return iaf5_fc_out \ No newline at end of file diff --git a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_5.py b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_5.py new file mode 100644 index 00000000..8ba45649 --- /dev/null +++ b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_5.py @@ -0,0 +1,143 @@ +import torch.nn as nn +import sinabs.layers as sl +from sinabs.exodus.layers import IAFSqueeze + +class SCNN(nn.Module): + def __init__(self, input_size, nb_classes, batch_size, surrogate_fn, min_v_mem=-0.313, spk_thr=2.0) -> None: + super().__init__() + + self.conv1 = nn.Conv2d(2, 8, 2, 1, bias=False) + self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool1 = sl.SumPool2d(2,2) + self.pool1a = sl.SumPool2d(8,8) + + self.conv2 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool2 = sl.SumPool2d(2,2) + + self.conv3 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool3 = sl.SumPool2d(2,2) + + self.conv4 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool4 = sl.SumPool2d(2,2) + + self.flat = nn.Flatten() + + flat_s = SCNN.get_flatten_size(input_size) + + self.fc1 = nn.Linear(flat_s, 100, bias=False) + self.iaf1_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc2 = nn.Linear(100, 100, bias=False) + self.iaf2_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc3 = nn.Linear(100, 100, bias=False) + self.iaf3_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc4 = nn.Linear(100, 100, bias=False) + self.iaf4_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc5 = nn.Linear(100, nb_classes, bias=False) + self.iaf5_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + # skip + + self.merge1 = sl.Merge() + + @staticmethod + def get_flatten_size(input_size): + conv1_dims = SCNN.conv2d_output_size(input_size, 8, (2, 2)) + pool1_dims = SCNN.pool_output_size(conv1_dims, 8, (2, 2)) + + conv2_dims = SCNN.conv2d_output_size(pool1_dims, 8, (2, 2)) + pool2_dims = SCNN.pool_output_size(conv2_dims, 8, (2, 2)) + + conv3_dims = SCNN.conv2d_output_size(pool2_dims, 8, (2, 2)) + pool3_dims = SCNN.pool_output_size(conv3_dims, 8, (2, 2)) + + conv4_dims = SCNN.conv2d_output_size(pool3_dims, 8, (2, 2)) + pool4_dims = SCNN.pool_output_size(conv4_dims, 8, (2, 2)) + + return pool4_dims[0]*pool4_dims[1]*pool4_dims[2] + + @staticmethod + def conv2d_output_size(input_size, out_channels, kernel_size, stride=1, padding=0, dilation=1): + input_height, input_width, input_channels = input_size + kernel_height, kernel_width = kernel_size + + output_height = ((input_height + 2 * padding - dilation * (kernel_height - 1) - 1) // stride) + 1 + output_width = ((input_width + 2 * padding - dilation * (kernel_width - 1) - 1) // stride) + 1 + + return (output_height, output_width, out_channels) + + @staticmethod + def pool_output_size(input_size, out_channels, kernel_size, stride=None, padding=0): + input_height, input_width, input_channels = input_size + kernel_height, kernel_width = kernel_size + + if stride is None: + stride = kernel_height + + output_height = ((input_height + 2 * padding - kernel_height) // stride) + 1 + output_width = ((input_width + 2 * padding - kernel_width) // stride) + 1 + + return (output_height, output_width, out_channels) + + def detach_neuron_states(self): + for name, layer in self.named_modules(): + if name != '': + if isinstance(layer, sl.StatefulLayer): + for name, buffer in layer.named_buffers(): + buffer.detach_() + + def init_weights(self): + for name, layer in self.named_modules(): + if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear): + nn.init.xavier_normal_(layer.weight.data) + + def rescale_conv_weights(self, rescale_fn, lambda_): + rescale_fn(self.conv2, [(2,2)], lambda_) + rescale_fn(self.conv3, [(2,2)], lambda_) + rescale_fn(self.conv4, [(8,8), (2,2)], lambda_) + + def forward(self, x): + # conv 1 + con1_out = self.conv1(x) + iaf1_out = self.iaf1(con1_out) + pool1_out = self.pool1(iaf1_out) + pool1a_out = self.pool1a(iaf1_out) + # conv 2 + conv2_out = self.conv2(pool1_out) + iaf2_out = self.iaf2(conv2_out) + pool2_out = self.pool2(iaf2_out) + # conv 3 + conv3_out = self.conv3(pool2_out) + iaf3_out = self.iaf3(conv3_out) + pool3_out = self.pool3(iaf3_out) + + merge1_out = self.merge1(pool1a_out, pool3_out) + # conv 4 + conv4_out = self.conv4(merge1_out) + iaf4_out = self.iaf4(conv4_out) + pool4_out = self.pool4(iaf4_out) + + flat_out = self.flat(pool4_out) + # fc 1 + fc1_out = self.fc1(flat_out) + iaf1_fc_out = self.iaf1_fc(fc1_out) + # fc 2 + fc2_out = self.fc2(iaf1_fc_out) + iaf2_fc_out = self.iaf2_fc(fc2_out) + # fc 3 + fc3_out = self.fc3(iaf2_fc_out) + iaf3_fc_out = self.iaf3_fc(fc3_out) + # fc 4 + fc4_out = self.fc4(iaf3_fc_out) + iaf4_fc_out = self.iaf4_fc(fc4_out) + # fc 5 + fc5_out = self.fc5(iaf4_fc_out) + iaf5_fc_out = self.iaf5_fc(fc5_out) + + return iaf5_fc_out \ No newline at end of file diff --git a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_6.py b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_6.py new file mode 100644 index 00000000..5f761619 --- /dev/null +++ b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_6.py @@ -0,0 +1,146 @@ +import torch.nn as nn +import sinabs.layers as sl +from sinabs.exodus.layers import IAFSqueeze + +class SCNN(nn.Module): + def __init__(self, input_size, nb_classes, batch_size, surrogate_fn, min_v_mem=-0.313, spk_thr=2.0) -> None: + super().__init__() + + self.conv1 = nn.Conv2d(2, 8, 2, 1, bias=False) + self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool1 = sl.SumPool2d(2,2) + self.pool1a = sl.SumPool2d(8,8) + + self.conv2 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool2 = sl.SumPool2d(2,2) + + self.conv3 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool3 = sl.SumPool2d(2,2) + + self.conv4 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool4 = sl.SumPool2d(2,2) + + self.flat = nn.Flatten() + + flat_s = SCNN.get_flatten_size(input_size) + + self.fc1 = nn.Linear(flat_s, 100, bias=False) + self.iaf1_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc2 = nn.Linear(100, 100, bias=False) + self.iaf2_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc3 = nn.Linear(100, 100, bias=False) + self.iaf3_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc4 = nn.Linear(100, 100, bias=False) + self.iaf4_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc5 = nn.Linear(100, nb_classes, bias=False) + self.iaf5_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + # skip + + self.merge1 = sl.Merge() + self.merge2 = sl.Merge() + + @staticmethod + def get_flatten_size(input_size): + conv1_dims = SCNN.conv2d_output_size(input_size, 8, (2, 2)) + pool1_dims = SCNN.pool_output_size(conv1_dims, 8, (2, 2)) + + conv2_dims = SCNN.conv2d_output_size(pool1_dims, 8, (2, 2)) + pool2_dims = SCNN.pool_output_size(conv2_dims, 8, (2, 2)) + + conv3_dims = SCNN.conv2d_output_size(pool2_dims, 8, (2, 2)) + pool3_dims = SCNN.pool_output_size(conv3_dims, 8, (2, 2)) + + conv4_dims = SCNN.conv2d_output_size(pool3_dims, 8, (2, 2)) + pool4_dims = SCNN.pool_output_size(conv4_dims, 8, (2, 2)) + + return pool4_dims[0]*pool4_dims[1]*pool4_dims[2] + + @staticmethod + def conv2d_output_size(input_size, out_channels, kernel_size, stride=1, padding=0, dilation=1): + input_height, input_width, input_channels = input_size + kernel_height, kernel_width = kernel_size + + output_height = ((input_height + 2 * padding - dilation * (kernel_height - 1) - 1) // stride) + 1 + output_width = ((input_width + 2 * padding - dilation * (kernel_width - 1) - 1) // stride) + 1 + + return (output_height, output_width, out_channels) + + @staticmethod + def pool_output_size(input_size, out_channels, kernel_size, stride=None, padding=0): + input_height, input_width, input_channels = input_size + kernel_height, kernel_width = kernel_size + + if stride is None: + stride = kernel_height + + output_height = ((input_height + 2 * padding - kernel_height) // stride) + 1 + output_width = ((input_width + 2 * padding - kernel_width) // stride) + 1 + + return (output_height, output_width, out_channels) + + def detach_neuron_states(self): + for name, layer in self.named_modules(): + if name != '': + if isinstance(layer, sl.StatefulLayer): + for name, buffer in layer.named_buffers(): + buffer.detach_() + + def init_weights(self): + for name, layer in self.named_modules(): + if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear): + nn.init.xavier_normal_(layer.weight.data) + + def rescale_conv_weights(self, rescale_fn, lambda_): + rescale_fn(self.conv2, [(2,2)], lambda_) + rescale_fn(self.conv3, [(2,2)], lambda_) + rescale_fn(self.conv4, [(8,8), (2,2)], lambda_) + + def forward(self, x): + # conv 1 + con1_out = self.conv1(x) + iaf1_out = self.iaf1(con1_out) + pool1_out = self.pool1(iaf1_out) + pool1a_out = self.pool1a(iaf1_out) + # conv 2 + conv2_out = self.conv2(pool1_out) + iaf2_out = self.iaf2(conv2_out) + pool2_out = self.pool2(iaf2_out) + # conv 3 + conv3_out = self.conv3(pool2_out) + iaf3_out = self.iaf3(conv3_out) + pool3_out = self.pool3(iaf3_out) + + merge1_out = self.merge1(pool1a_out, pool3_out) + # conv 4 + conv4_out = self.conv4(merge1_out) + iaf4_out = self.iaf4(conv4_out) + pool4_out = self.pool4(iaf4_out) + + flat_out = self.flat(pool4_out) + # fc 1 + fc1_out = self.fc1(flat_out) + iaf1_fc_out = self.iaf1_fc(fc1_out) + # fc 2 + fc2_out = self.fc2(iaf1_fc_out) + iaf2_fc_out = self.iaf2_fc(fc2_out) + # fc 3 + fc3_out = self.fc3(iaf2_fc_out) + iaf3_fc_out = self.iaf3_fc(fc3_out) + + merge2_out = self.merge2(iaf1_fc_out, iaf3_fc_out) + # fc 4 + fc4_out = self.fc4(merge2_out) + iaf4_fc_out = self.iaf4_fc(fc4_out) + # fc 5 + fc5_out = self.fc5(iaf4_fc_out) + iaf5_fc_out = self.iaf5_fc(fc5_out) + + return iaf5_fc_out \ No newline at end of file diff --git a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_7.py b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_7.py new file mode 100644 index 00000000..21039b28 --- /dev/null +++ b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_7.py @@ -0,0 +1,146 @@ +import torch.nn as nn +import sinabs.layers as sl +from sinabs.exodus.layers import IAFSqueeze + +class SCNN(nn.Module): + def __init__(self, input_size, nb_classes, batch_size, surrogate_fn, min_v_mem=-0.313, spk_thr=2.0) -> None: + super().__init__() + + self.conv1 = nn.Conv2d(2, 8, 2, 1, bias=False) + self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool1 = sl.SumPool2d(2,2) + self.pool1a = sl.SumPool2d(4,4) + self.pool1b = sl.SumPool2d(8,8) + + self.conv2 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool2 = sl.SumPool2d(2,2) + + self.conv3 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool3 = sl.SumPool2d(2,2) + + self.conv4 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool4 = sl.SumPool2d(2,2) + + self.flat = nn.Flatten() + + flat_s = SCNN.get_flatten_size(input_size) + + self.fc1 = nn.Linear(flat_s, 100, bias=False) + self.iaf1_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc2 = nn.Linear(100, 100, bias=False) + self.iaf2_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc3 = nn.Linear(100, 100, bias=False) + self.iaf3_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc4 = nn.Linear(100, 100, bias=False) + self.iaf4_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc5 = nn.Linear(100, nb_classes, bias=False) + self.iaf5_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + # skip + + self.merge1a = sl.Merge() + self.merge1b = sl.Merge() + + @staticmethod + def get_flatten_size(input_size): + conv1_dims = SCNN.conv2d_output_size(input_size, 8, (2, 2)) + pool1_dims = SCNN.pool_output_size(conv1_dims, 8, (2, 2)) + + conv2_dims = SCNN.conv2d_output_size(pool1_dims, 8, (2, 2)) + pool2_dims = SCNN.pool_output_size(conv2_dims, 8, (2, 2)) + + conv3_dims = SCNN.conv2d_output_size(pool2_dims, 8, (2, 2)) + pool3_dims = SCNN.pool_output_size(conv3_dims, 8, (2, 2)) + + conv4_dims = SCNN.conv2d_output_size(pool3_dims, 8, (2, 2)) + pool4_dims = SCNN.pool_output_size(conv4_dims, 8, (2, 2)) + + return pool4_dims[0]*pool4_dims[1]*pool4_dims[2] + + @staticmethod + def conv2d_output_size(input_size, out_channels, kernel_size, stride=1, padding=0, dilation=1): + input_height, input_width, input_channels = input_size + kernel_height, kernel_width = kernel_size + + output_height = ((input_height + 2 * padding - dilation * (kernel_height - 1) - 1) // stride) + 1 + output_width = ((input_width + 2 * padding - dilation * (kernel_width - 1) - 1) // stride) + 1 + + return (output_height, output_width, out_channels) + + @staticmethod + def pool_output_size(input_size, out_channels, kernel_size, stride=None, padding=0): + input_height, input_width, input_channels = input_size + kernel_height, kernel_width = kernel_size + + if stride is None: + stride = kernel_height + + output_height = ((input_height + 2 * padding - kernel_height) // stride) + 1 + output_width = ((input_width + 2 * padding - kernel_width) // stride) + 1 + + return (output_height, output_width, out_channels) + + def detach_neuron_states(self): + for name, layer in self.named_modules(): + if name != '': + if isinstance(layer, sl.StatefulLayer): + for name, buffer in layer.named_buffers(): + buffer.detach_() + + def init_weights(self): + for name, layer in self.named_modules(): + if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear): + nn.init.xavier_normal_(layer.weight.data) + + def rescale_conv_weights(self, rescale_fn, lambda_): + rescale_fn(self.conv2, [(2,2)], lambda_) + rescale_fn(self.conv3, [(4,4), (2,2)], lambda_) + rescale_fn(self.conv4, [(8,8), (2,2)], lambda_) + + def forward(self, x): + # conv 1 + con1_out = self.conv1(x) + iaf1_out = self.iaf1(con1_out) + pool1_out = self.pool1(iaf1_out) + pool1a_out = self.pool1a(iaf1_out) + pool1b_out = self.pool1b(iaf1_out) + # conv 2 + conv2_out = self.conv2(pool1_out) + iaf2_out = self.iaf2(conv2_out) + pool2_out = self.pool2(iaf2_out) + # conv 3 + merge_1a_out = self.merge1a(pool1a_out, pool2_out) + conv3_out = self.conv3(merge_1a_out) + iaf3_out = self.iaf3(conv3_out) + pool3_out = self.pool3(iaf3_out) + # conv 4 + merge_1b_out = self.merge1b(pool1b_out, pool3_out) + conv4_out = self.conv4(merge_1b_out) + iaf4_out = self.iaf4(conv4_out) + pool4_out = self.pool4(iaf4_out) + + flat_out = self.flat(pool4_out) + # fc 1 + fc1_out = self.fc1(flat_out) + iaf1_fc_out = self.iaf1_fc(fc1_out) + # fc 2 + fc2_out = self.fc2(iaf1_fc_out) + iaf2_fc_out = self.iaf2_fc(fc2_out) + # fc 3 + fc3_out = self.fc3(iaf2_fc_out) + iaf3_fc_out = self.iaf3_fc(fc3_out) + # fc 4 + fc4_out = self.fc4(iaf3_fc_out) + iaf4_fc_out = self.iaf4_fc(fc4_out) + # fc 5 + fc5_out = self.fc5(iaf4_fc_out) + iaf5_fc_out = self.iaf5_fc(fc5_out) + + return iaf5_fc_out \ No newline at end of file diff --git a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_8.py b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_8.py new file mode 100644 index 00000000..9086a3e3 --- /dev/null +++ b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_8.py @@ -0,0 +1,151 @@ +import torch.nn as nn +import sinabs.layers as sl +from sinabs.exodus.layers import IAFSqueeze + +class SCNN(nn.Module): + def __init__(self, input_size, nb_classes, batch_size, surrogate_fn, min_v_mem=-0.313, spk_thr=2.0) -> None: + super().__init__() + + self.conv1 = nn.Conv2d(2, 8, 2, 1, bias=False) + self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool1 = sl.SumPool2d(2,2) + self.pool1a = sl.SumPool2d(4,4) + self.pool1b = sl.SumPool2d(8,8) + + self.conv2 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool2 = sl.SumPool2d(2,2) + + self.conv3 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool3 = sl.SumPool2d(2,2) + + self.conv4 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool4 = sl.SumPool2d(2,2) + + self.flat = nn.Flatten() + + flat_s = SCNN.get_flatten_size(input_size) + + self.fc1 = nn.Linear(flat_s, 100, bias=False) + self.iaf1_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc2 = nn.Linear(100, 100, bias=False) + self.iaf2_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc3 = nn.Linear(100, 100, bias=False) + self.iaf3_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc4 = nn.Linear(100, 100, bias=False) + self.iaf4_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc5 = nn.Linear(100, nb_classes, bias=False) + self.iaf5_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + # skip + + self.merge1a = sl.Merge() + self.merge1b = sl.Merge() + + self.merge_fc1a = sl.Merge() + self.merge_fc1b = sl.Merge() + + @staticmethod + def get_flatten_size(input_size): + conv1_dims = SCNN.conv2d_output_size(input_size, 8, (2, 2)) + pool1_dims = SCNN.pool_output_size(conv1_dims, 8, (2, 2)) + + conv2_dims = SCNN.conv2d_output_size(pool1_dims, 8, (2, 2)) + pool2_dims = SCNN.pool_output_size(conv2_dims, 8, (2, 2)) + + conv3_dims = SCNN.conv2d_output_size(pool2_dims, 8, (2, 2)) + pool3_dims = SCNN.pool_output_size(conv3_dims, 8, (2, 2)) + + conv4_dims = SCNN.conv2d_output_size(pool3_dims, 8, (2, 2)) + pool4_dims = SCNN.pool_output_size(conv4_dims, 8, (2, 2)) + + return pool4_dims[0]*pool4_dims[1]*pool4_dims[2] + + @staticmethod + def conv2d_output_size(input_size, out_channels, kernel_size, stride=1, padding=0, dilation=1): + input_height, input_width, input_channels = input_size + kernel_height, kernel_width = kernel_size + + output_height = ((input_height + 2 * padding - dilation * (kernel_height - 1) - 1) // stride) + 1 + output_width = ((input_width + 2 * padding - dilation * (kernel_width - 1) - 1) // stride) + 1 + + return (output_height, output_width, out_channels) + + @staticmethod + def pool_output_size(input_size, out_channels, kernel_size, stride=None, padding=0): + input_height, input_width, input_channels = input_size + kernel_height, kernel_width = kernel_size + + if stride is None: + stride = kernel_height + + output_height = ((input_height + 2 * padding - kernel_height) // stride) + 1 + output_width = ((input_width + 2 * padding - kernel_width) // stride) + 1 + + return (output_height, output_width, out_channels) + + def detach_neuron_states(self): + for name, layer in self.named_modules(): + if name != '': + if isinstance(layer, sl.StatefulLayer): + for name, buffer in layer.named_buffers(): + buffer.detach_() + + def init_weights(self): + for name, layer in self.named_modules(): + if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear): + nn.init.xavier_normal_(layer.weight.data) + + def rescale_conv_weights(self, rescale_fn, lambda_): + rescale_fn(self.conv2, [(2,2)], lambda_) + rescale_fn(self.conv3, [(4,4), (2,2)], lambda_) + rescale_fn(self.conv4, [(8,8), (2,2)], lambda_) + + def forward(self, x): + # conv 1 + con1_out = self.conv1(x) + iaf1_out = self.iaf1(con1_out) + pool1_out = self.pool1(iaf1_out) + pool1a_out = self.pool1a(iaf1_out) + pool1b_out = self.pool1b(iaf1_out) + # conv 2 + conv2_out = self.conv2(pool1_out) + iaf2_out = self.iaf2(conv2_out) + pool2_out = self.pool2(iaf2_out) + # conv 3 + merge_1a_out = self.merge1a(pool1a_out, pool2_out) + conv3_out = self.conv3(merge_1a_out) + iaf3_out = self.iaf3(conv3_out) + pool3_out = self.pool3(iaf3_out) + # conv 4 + merge_1b_out = self.merge1b(pool1b_out, pool3_out) + conv4_out = self.conv4(merge_1b_out) + iaf4_out = self.iaf4(conv4_out) + pool4_out = self.pool4(iaf4_out) + + flat_out = self.flat(pool4_out) + # fc 1 + fc1_out = self.fc1(flat_out) + iaf1_fc_out = self.iaf1_fc(fc1_out) + # fc 2 + fc2_out = self.fc2(iaf1_fc_out) + iaf2_fc_out = self.iaf2_fc(fc2_out) + # fc 3 + merge_fc1a_out = self.merge_fc1a(iaf1_fc_out, iaf2_fc_out) + fc3_out = self.fc3(merge_fc1a_out) + iaf3_fc_out = self.iaf3_fc(fc3_out) + # fc 4 + merge_fc1b_out = self.merge_fc1b(iaf1_fc_out, iaf3_fc_out) + fc4_out = self.fc4(merge_fc1b_out) + iaf4_fc_out = self.iaf4_fc(fc4_out) + # fc 5 + fc5_out = self.fc5(iaf4_fc_out) + iaf5_fc_out = self.iaf5_fc(fc5_out) + + return iaf5_fc_out \ No newline at end of file diff --git a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_9.py b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_9.py new file mode 100644 index 00000000..7ed5ca4d --- /dev/null +++ b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_9.py @@ -0,0 +1,124 @@ +import torch.nn as nn +import sinabs.layers as sl +from sinabs.exodus.layers import IAFSqueeze + +class SCNN(nn.Module): + def __init__(self, input_size, nb_classes, batch_size, surrogate_fn, min_v_mem=-0.313, spk_thr=2.0): + super().__init__() + + self.conv1 = nn.Conv2d(2, 8, 2, 1, bias=False) + self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.conv2 = nn.Conv2d(8, 8, 3, 1, 1, bias=False) + self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.conv3 = nn.Conv2d(8, 8, 3, 1, 1, bias=False) + self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool3a = sl.SumPool2d(2,2) + + self.conv4 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool4 = sl.SumPool2d(2,2) + self.pool4a = sl.SumPool2d(4,4) + + self.conv5 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf5 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool5 = sl.SumPool2d(2,2) + self.pool5a = sl.SumPool2d(2,2) + + self.conv6 = nn.Conv2d(8, 8, 3, 1, 1, bias=False) + self.iaf6 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool6a = sl.SumPool2d(2,2) + + self.conv7 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf7 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool7 = sl.SumPool2d(2,2) + + self.conv8 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf8 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool8 = sl.SumPool2d(2,2) + + self.flat = nn.Flatten() + + self.fc_out = nn.Linear(392, nb_classes, bias=False) + self.iaf_fc_out = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + # skip + + self.merge1 = sl.Merge() + self.merge2 = sl.Merge() + self.merge3 = sl.Merge() + self.merge4 = sl.Merge() + self.merge5 = sl.Merge() + self.merge6 = sl.Merge() + + def detach_neuron_states(self): + for name, layer in self.named_modules(): + if name != '': + if isinstance(layer, sl.StatefulLayer): + for name, buffer in layer.named_buffers(): + buffer.detach_() + + def init_weights(self): + for name, layer in self.named_modules(): + if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear): + nn.init.xavier_normal_(layer.weight.data) + + def rescale_conv_weights(self, rescale_fn, lambda_): + rescale_fn(self.conv5, [(2,2), (2,2)], lambda_) + rescale_fn(self.conv6, [(4,4), (2,2)], lambda_) + rescale_fn(self.conv7, [(2,2)], lambda_) + rescale_fn(self.conv8, [(2,2), (2,2)], lambda_) + + def forward(self, x): + # -- conv block 1 --- + con1_out = self.conv1(x) + iaf1_out = self.iaf1(con1_out) + # -- conv block 2 --- + conv2_out = self.conv2(iaf1_out) + iaf2_out = self.iaf2(conv2_out) # to CONV 4 + # -- conv block 3 --- + #print(iaf1_out.shape, iaf2_out.shape) + merge1_out = self.merge1(iaf1_out, iaf2_out) + conv3_out = self.conv3(merge1_out) + iaf3_out = self.iaf3(conv3_out) + pool3a_out = self.pool3a(iaf3_out) # to CONV 5 + # -- conv block 4 --- + #print(iaf2_out.shape, iaf3_out.shape) + merge2_out = self.merge2(iaf2_out, iaf3_out) + conv4_out = self.conv4(merge2_out) + iaf4_out = self.iaf4(conv4_out) + pool4_out = self.pool4(iaf4_out) + pool4a_out = self.pool4a(iaf4_out) # to CONV 6 + # -- conv block 5 --- + #print(pool3a_out.shape, pool4_out.shape) + merge3_out = self.merge3(pool3a_out, pool4_out) + conv5_out = self.conv5(merge3_out) + iaf5_out = self.iaf5(conv5_out) + pool5_out = self.pool5(iaf5_out) + pool5a_out = self.pool5a(iaf5_out) # to CONV 7 + # -- conv block 6 --- + #print(pool4a_out.shape, pool5_out.shape) + merge4_out = self.merge4(pool4a_out, pool5_out) + conv6_out = self.conv6(merge4_out) + iaf6_out = self.iaf6(conv6_out) + pool6a_out = self.pool6a(iaf6_out) # to CONV 8 + # -- conv block 7 --- + #print(pool5a_out.shape, iaf6_out.shape) + merge5_out = self.merge5(pool5a_out, iaf6_out) + conv7_out = self.conv7(merge5_out) + iaf7_out = self.iaf7(conv7_out) + pool7_out = self.pool7(iaf7_out) + # -- conv block 8 --- + #print(pool6a_out.shape, pool7_out.shape) + merge6_out = self.merge6(pool6a_out, pool7_out) + conv8_out = self.conv8(merge6_out) + iaf8_out = self.iaf8(conv8_out) + pool8_out = self.pool8(iaf8_out) + # -- output -- + flat = self.flat(pool8_out) + #print(flat.shape) + fc_out = self.fc_out(flat) + iaf_fc_out = self.iaf_fc_out(fc_out) + + return iaf_fc_out \ No newline at end of file diff --git a/tests/test_nonsequential/utils/train_test_fn.py b/tests/test_nonsequential/utils/train_test_fn.py index dd19b816..d166fdff 100644 --- a/tests/test_nonsequential/utils/train_test_fn.py +++ b/tests/test_nonsequential/utils/train_test_fn.py @@ -56,10 +56,18 @@ def training_loop(device, nb_time_steps, batch_size, feature_map_size, dataloade return epochs_x, epochs_y, epochs_acc -def training_loop_no_tqdm(device, nb_time_steps, batch_size, feature_map_size, dataloader_train, model, loss_fn, optimizer, epochs, dataloader_test): +def training_loop_no_tqdm(device, nb_time_steps, batch_size, feature_map_size, dataloader_train, model, loss_fn, optimizer, epochs, dataloader_test, record_data = False): + epochs_y = [] + epochs_x = [] + epochs_acc = [] + model.train() for e in range(epochs): + losses = [] + batches = [] + batch_count = 0 + for X, y in dataloader_train: # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width] X = X.reshape(-1, feature_map_size[2], feature_map_size[0], feature_map_size[1]).to(dtype=torch.float, device=device) @@ -83,9 +91,23 @@ def training_loop_no_tqdm(device, nb_time_steps, batch_size, feature_map_size, d # detach the neuron states and activations from current computation graph(necessary) model.detach_neuron_states() + if record_data: + batch_count += 1 + losses.append(loss.item()) + batches.append(batch_count) + + if record_data: + epochs_y.append(losses) + epochs_x.append(batches) + acc = test_no_tqdm(device, nb_time_steps, batch_size, feature_map_size, dataloader_test, model) + if record_data: + epochs_acc.append(acc) - return acc + if record_data: + return epochs_x, epochs_y, epochs_acc + else: + return acc def test_no_tqdm(device, nb_time_steps, batch_size, feature_map_size, dataloader_test, model): correct_predictions = [] @@ -186,33 +208,67 @@ def split_train_validation(validation_ratio, snn_train_dataset, rand_seed): return train_dataset, validation_dataset -def load_architecture(architecture, input_size, nb_classes, batch_size, surrogate_fn, min_v_mem=-0.313, spk_thr=2.0): +def split_train_validation_used_seed(validation_ratio, snn_train_dataset, used_seed): + """ Will generate a validation dataset in which the random indices do not overlap + with the ones that are generated using the random seed `used_seed`. + """ + num_samples = len(snn_train_dataset) + num_validation_samples = int(validation_ratio * num_samples) + + np.random.seed(used_seed) + + used_validation_indices = np.random.choice(np.arange(num_samples), size=num_validation_samples, replace=False) + + validation_indices = np.random.choice(np.setdiff1d(np.arange(num_samples), used_validation_indices), size=len(used_validation_indices), replace=False) + + training_indices = np.array(list(filter(lambda x: x not in validation_indices, np.arange(num_samples)))) + + if len(np.intersect1d(used_validation_indices, validation_indices)) != 0: + raise ValueError(f'data leakage: generated validation set overlaps with previously generated indices') + + train_dataset = Subset(snn_train_dataset, training_indices) + validation_dataset = Subset(snn_train_dataset, validation_indices) + + return train_dataset, validation_dataset + +def load_architecture( + architecture, input_size, nb_classes, batch_size, surrogate_fn, + min_v_mem=-0.313, spk_thr=2.0, hetero_init = False, hetero_seed = 1): import sys sys.path.append('../models') - if architecture == 'ResSCNN1': - from ResSCNN1 import SCNN + if architecture == 'ResSCNN_1': + from ResSCNN_1 import SCNN + return SCNN(input_size, nb_classes, batch_size, surrogate_fn, min_v_mem, spk_thr) + elif architecture == 'ResSCNN_2': + from ResSCNN_2 import SCNN + return SCNN(input_size, nb_classes, batch_size, surrogate_fn, min_v_mem, spk_thr) + elif architecture == 'ResSCNN_3': + from ResSCNN_3 import SCNN return SCNN(input_size, nb_classes, batch_size, surrogate_fn, min_v_mem, spk_thr) - elif architecture == 'ResSCNN2': - from ResSCNN2 import SCNN + elif architecture == 'ResSCNN_4': + from ResSCNN_4 import SCNN return SCNN(input_size, nb_classes, batch_size, surrogate_fn, min_v_mem, spk_thr) - elif architecture == 'ResSCNN3': - from ResSCNN3 import SCNN + elif architecture == 'ResSCNN_5': + from ResSCNN_5 import SCNN return SCNN(input_size, nb_classes, batch_size, surrogate_fn, min_v_mem, spk_thr) - elif architecture == 'ResSCNN4': - from ResSCNN4 import SCNN + elif architecture == 'ResSCNN_6': + from ResSCNN_6 import SCNN return SCNN(input_size, nb_classes, batch_size, surrogate_fn, min_v_mem, spk_thr) - elif architecture == 'ResSCNN5': - from ResSCNN5 import SCNN + elif architecture == 'ResSCNN_7': + from ResSCNN_7 import SCNN return SCNN(input_size, nb_classes, batch_size, surrogate_fn, min_v_mem, spk_thr) - elif architecture == 'ResSCNN6': - from ResSCNN6 import SCNN + elif architecture == 'ResSCNN_8': + from ResSCNN_8 import SCNN return SCNN(input_size, nb_classes, batch_size, surrogate_fn, min_v_mem, spk_thr) - elif architecture == 'ResSCNN7': - from ResSCNN7 import SCNN + elif architecture == 'ResSCNN_9': + from ResSCNN_9 import SCNN + return SCNN(input_size, nb_classes, batch_size, surrogate_fn, min_v_mem, spk_thr, hetero_init, hetero_seed) + elif architecture == 'ResSCNN_10': + from ResSCNN_10 import SCNN return SCNN(input_size, nb_classes, batch_size, surrogate_fn, min_v_mem, spk_thr) - elif architecture == 'ResSCNN8': - from ResSCNN8 import SCNN + elif architecture == 'ResSCNN_11': + from ResSCNN_11 import SCNN return SCNN(input_size, nb_classes, batch_size, surrogate_fn, min_v_mem, spk_thr) else: return None \ No newline at end of file From 65d3e08e1f667b89feeed92df1958b6044d4753f Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Mon, 6 May 2024 15:29:16 +0200 Subject: [PATCH 077/379] HPO history from Gaussian Search --- .../gaussian_search_history.csv | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 tests/test_nonsequential/using_SumPool2d/HPO_GAUSSIAN_SEARCH/gaussian_search_history.csv diff --git a/tests/test_nonsequential/using_SumPool2d/HPO_GAUSSIAN_SEARCH/gaussian_search_history.csv b/tests/test_nonsequential/using_SumPool2d/HPO_GAUSSIAN_SEARCH/gaussian_search_history.csv new file mode 100644 index 00000000..e87da0e0 --- /dev/null +++ b/tests/test_nonsequential/using_SumPool2d/HPO_GAUSSIAN_SEARCH/gaussian_search_history.csv @@ -0,0 +1,57 @@ +,learning_rate,spike_threshold,mem_v_min,grad_scale,grad_width,w_rescale_lambda,accuracy +0,0.001,2.75,-2.5,1.55,1.55,0.5,10.576923076923077 +1,0.000947,1.99,-2.82,1.11,2.0,0.344,86.0576923076923 +2,0.000769,1.09,-2.29,1.11,1.83,0.319,87.98076923076923 +3,0.000811,1.01,-1.43,0.85,1.73,0.42,88.9423076923077 +4,0.000811,1.01,-1.43,0.85,1.73,0.42,88.9423076923077 +5,0.000811,1.01,-1.43,0.85,1.73,0.42,88.9423076923077 +6,0.000782,1.87,0.0,0.91,1.9,0.235,92.3076923076923 +7,0.000741,1.87,-0.15,1.16,1.97,0.175,95.67307692307693 +8,0.000741,1.87,-0.15,1.16,1.97,0.175,95.67307692307693 +9,0.000741,1.87,-0.15,1.16,1.97,0.175,95.67307692307693 +10,0.000741,1.87,-0.15,1.16,1.97,0.175,95.67307692307693 +11,0.000741,1.87,-0.15,1.16,1.97,0.175,95.67307692307693 +12,0.000741,1.87,-0.15,1.16,1.97,0.175,95.67307692307693 +13,0.000741,1.87,-0.15,1.16,1.97,0.175,95.67307692307693 +14,0.000741,1.87,-0.15,1.16,1.97,0.175,95.67307692307693 +15,0.000741,1.87,-0.15,1.16,1.97,0.175,95.67307692307693 +16,0.000741,1.87,-0.15,1.16,1.97,0.175,95.67307692307693 +17,0.000826,1.51,0.0,1.06,1.64,0.223,96.15384615384616 +18,0.000826,1.51,0.0,1.06,1.64,0.223,96.15384615384616 +19,0.000826,1.51,0.0,1.06,1.64,0.223,96.15384615384616 +20,0.000826,1.51,0.0,1.06,1.64,0.223,96.15384615384616 +21,0.000826,1.51,0.0,1.06,1.64,0.223,96.15384615384616 +22,0.000826,1.51,0.0,1.06,1.64,0.223,96.15384615384616 +23,0.000826,1.51,0.0,1.06,1.64,0.223,96.15384615384616 +24,0.000826,1.51,0.0,1.06,1.64,0.223,96.15384615384616 +25,0.000826,1.51,0.0,1.06,1.64,0.223,96.15384615384616 +26,0.000826,1.51,0.0,1.06,1.64,0.223,96.15384615384616 +27,0.000826,1.51,0.0,1.06,1.64,0.223,96.15384615384616 +28,0.000826,1.51,0.0,1.06,1.64,0.223,96.15384615384616 +29,0.000797,1.83,-0.09,0.67,1.56,0.121,97.59615384615384 +30,0.000797,1.83,-0.09,0.67,1.56,0.121,97.59615384615384 +31,0.000797,1.83,-0.09,0.67,1.56,0.121,97.59615384615384 +32,0.000797,1.83,-0.09,0.67,1.56,0.121,97.59615384615384 +33,0.000797,1.83,-0.09,0.67,1.56,0.121,97.59615384615384 +34,0.000797,1.83,-0.09,0.67,1.56,0.121,97.59615384615384 +35,0.000797,1.83,-0.09,0.67,1.56,0.121,97.59615384615384 +36,0.000797,1.83,-0.09,0.67,1.56,0.121,97.59615384615384 +37,0.000797,1.83,-0.09,0.67,1.56,0.121,97.59615384615384 +38,0.000797,1.83,-0.09,0.67,1.56,0.121,97.59615384615384 +39,0.000797,1.83,-0.09,0.67,1.56,0.121,97.59615384615384 +40,0.000797,1.83,-0.09,0.67,1.56,0.121,97.59615384615384 +41,0.000797,1.83,-0.09,0.67,1.56,0.121,97.59615384615384 +42,0.000797,1.83,-0.09,0.67,1.56,0.121,97.59615384615384 +43,0.000797,1.83,-0.09,0.67,1.56,0.121,97.59615384615384 +44,0.000797,1.83,-0.09,0.67,1.56,0.121,97.59615384615384 +45,0.000797,1.83,-0.09,0.67,1.56,0.121,97.59615384615384 +46,0.000797,1.83,-0.09,0.67,1.56,0.121,97.59615384615384 +47,0.000797,1.83,-0.09,0.67,1.56,0.121,97.59615384615384 +48,0.000797,1.83,-0.09,0.67,1.56,0.121,97.59615384615384 +49,0.000797,1.83,-0.09,0.67,1.56,0.121,97.59615384615384 +50,0.000797,1.83,-0.09,0.67,1.56,0.121,97.59615384615384 +51,0.000797,1.83,-0.09,0.67,1.56,0.121,97.59615384615384 +52,0.000797,1.83,-0.09,0.67,1.56,0.121,97.59615384615384 +53,0.000797,1.83,-0.09,0.67,1.56,0.121,97.59615384615384 +54,0.000797,1.83,-0.09,0.67,1.56,0.121,97.59615384615384 +55,0.000797,1.83,-0.09,0.67,1.56,0.121,97.59615384615384 From 395596788cdfefdb953bd2be7dc35e67ee714653 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Tue, 7 May 2024 14:21:08 +0200 Subject: [PATCH 078/379] validation of edges accounting for cases with sl.SumPool2d too --- .../backend/dynapcnn/sinabs_edges_handler.py | 4 ++-- sinabs/backend/dynapcnn/sinabs_edges_utils.py | 19 ++++++++++++------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index a57439ca..41e9be2c 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -67,10 +67,10 @@ def update_dynapcnnlayer_mapper(edge_type: int, edge: Tuple[int, int], mapper: d if edge_type in [0, 6]: init_xor_complete_new_dynapcnnlayer_blk(mapper, edge, layers) - elif edge_type == 1: + elif edge_type in [1, 7]: add_pool_to_dynapcnnlayer_blk(mapper, edge, layers) - elif edge_type in [2, 3, 4, 5]: + elif edge_type in [2, 3, 4, 5, 8, 9]: connect_dynapcnnlayer_blks(mapper, edge, layers) else: diff --git a/sinabs/backend/dynapcnn/sinabs_edges_utils.py b/sinabs/backend/dynapcnn/sinabs_edges_utils.py index 29566153..4431cd38 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_utils.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_utils.py @@ -12,20 +12,25 @@ # Constraints. # @TODO constraints are ideally device-dependent. VALID_SINABS_EDGES = { - 0: (nn.Conv2d, sl.iaf.IAFSqueeze), # 'nn.Conv2d' is always followed by a 'sl.iaf'. + 0: (nn.Conv2d, sl.iaf.IAFSqueeze), # convoluion is always followed by a neuron layer. 1: (sl.iaf.IAFSqueeze, nn.AvgPool2d), 2: (sl.iaf.IAFSqueeze, nn.Conv2d), - 3: (sl.iaf.IAFSqueeze, nn.Linear), # same case as '2' since 'nn.Linear' layers are converted into 'nn.Conv2d' by 'DynapcnnLayer'. - 4: (nn.AvgPool2d, nn.Conv2d), # 'nn.Pool2d' is always "ending" a DynapcnnLayer sequence of modules (comes after a 'sl.iaf'). - 5: (nn.AvgPool2d, nn.Linear), # same as case '4' since 'nn.Linear' layers are converted into 'nn.Conv2d' by 'DynapcnnLayer'. - 6: (nn.Linear, sl.iaf.IAFSqueeze), # same as case '0' since 'nn.Linear' layers are converted into 'nn.Conv2d' by 'DynapcnnLayer'. + 3: (sl.iaf.IAFSqueeze, nn.Linear), # same case as `2` since `nn.Linear` layers are converted into `nn.Conv2d` by `DynapcnnLayer`. + 4: (nn.AvgPool2d, nn.Conv2d), # `nn.Pool2d` is always "ending" a DynapcnnLayer sequence of modules (comes after a `sl.iaf`). + 5: (nn.AvgPool2d, nn.Linear), # same as case `4` since `nn.Linear` layers are converted into `nn.Conv2d` by `DynapcnnLayer`. + 6: (nn.Linear, sl.iaf.IAFSqueeze), # same as case `0` since `nn.Linear` layers are converted into `nn.Conv2d` by `DynapcnnLayer`. + 7: (sl.iaf.IAFSqueeze, sl.SumPool2d), # same as key `1` but with `sl.SumPool2d` instead. + 8: (sl.SumPool2d, nn.Conv2d), # same as key `4` but with `sl.SumPool2d` instead. + 9: (sl.SumPool2d, nn.Linear), # same as key `5` but with `sl.SumPool2d` instead. } VALID_DYNAPCNNLAYER_EDGES = [ (sl.iaf.IAFSqueeze, nn.Conv2d), - (sl.iaf.IAFSqueeze, nn.Linear), # 'nn.Linear' layers are converted into 'nn.Conv2d' by 'DynapcnnLayer'. + (sl.iaf.IAFSqueeze, nn.Linear), # `nn.Linear` layers are converted into `nn.Conv2d` by `DynapcnnLayer`. (nn.AvgPool2d, nn.Conv2d), - (nn.AvgPool2d, nn.Linear), # 'nn.Linear' layers are converted into 'nn.Conv2d' by 'DynapcnnLayer'. + (sl.SumPool2d, nn.Conv2d), + (nn.AvgPool2d, nn.Linear), # `nn.Linear` layers are converted into `nn.Conv2d` by `DynapcnnLayer`. + (sl.SumPool2d, nn.Linear), ] VALID_SINABS_NODE_FAN_IN = [] From 19779ebd912c1410f294eeff42f2fc495ee304db Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Tue, 7 May 2024 14:47:08 +0200 Subject: [PATCH 079/379] creation of DynapcnnLayer accounting for direct use of SumPool2d (i.e. not a conversion from a AvgPool2d) --- sinabs/backend/dynapcnn/dynapcnn_layer_new.py | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer_new.py b/sinabs/backend/dynapcnn/dynapcnn_layer_new.py index 1a07827d..03372ca1 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer_new.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer_new.py @@ -112,7 +112,9 @@ def __init__( if len(pool) != 0: # the 1st pooling targets the 1st destination in `dcnnl_data['destinations']`, the 2nd pooling targets the 2nd destination... for plyr in pool: - if plyr.kernel_size[0] != plyr.kernel_size[1]: + # @TODO POSSIBLE INCONSISTENCY: if the `SumPool2d` is the result of a conversion from `AvgPool2d` then `SumPool2d.kernel_size` + # is of type tuple, otherwise it is an int. + if isinstance(plyr.kernel_size, tuple) and plyr.kernel_size[0] != plyr.kernel_size[1]: raise ValueError("Only square kernels are supported") self.pool_layer.append(deepcopy(plyr)) @@ -223,10 +225,22 @@ def get_conv_output_shape(self, conv_layer: nn.Conv2d, input_shape: tuple): def summary(self) -> dict: # TODO I can't see pooling being used in checking memory constraints by the builder so I'm ignoring for now the fact that multiple pooling could exist. + + _pool = None + + # @TODO POSSIBLE INCONSISTENCY: if the `SumPool2d` is the result of a conversion from `AvgPool2d` then `SumPool2d.kernel_size` + # is of type tuple, otherwise it is an int. + if len(self.pool_layer) != 0: + # @TODO ignoring for now that there could be multiple poolings (just use the first one). + if isinstance(self.pool_layer[0].kernel_size, tuple): + _pool = list(self.pool_layer[0].kernel_size) + elif isinstance(self.pool_layer[0].kernel_size, int): + _pool = [self.pool_layer[0].kernel_size, self.pool_layer[0].kernel_size] + else: + raise ValueError('Type of `self.pool_layer[0].kernel_size` not understood.') + return { - "pool": ( # ignoring for now that there could be multiple poolings (just use the first one). - None if len(self.pool_layer) == 0 else list(self.pool_layer[0].kernel_size) - ), + "pool": (_pool), "kernel": list(self.conv_layer.weight.data.shape), "neuron": self.conv_out_shape, # neuron layer output has the same shape as the convolution layer ouput. } @@ -318,7 +332,7 @@ def get_layer_config_dict(self) -> dict: dest_config = { 'layer': self.dynapcnnlayer_destination[i],# TODO this destination index is not the core index yet, just the index of the DynapcnnLayers themselves. 'enable': True, - 'pooling': self.pool_layer[i].kernel_size[0] # TODO make sure the kernel is a square. + 'pooling': self.pool_layer[i].kernel_size[0] if isinstance(self.pool_layer[i].kernel_size, tuple) else self.pool_layer[i].kernel_size # TODO make sure the kernel is a square. } config_dict['destinations'].append(dest_config) From d1766445d55d03deafe930f59c6f44ba66f7d75b Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Tue, 7 May 2024 20:15:51 +0200 Subject: [PATCH 080/379] unit tests for new DynapcnnNetwork --- .../architectures_samples.py | 292 +++++++++++++++ .../conftest_dynapcnnnetwork.py | 349 ++++++++++++++++++ .../test_dynapcnnnetwork.py | 84 +++++ 3 files changed, 725 insertions(+) create mode 100644 tests/test_dynapcnnnetwork/architectures_samples.py create mode 100644 tests/test_dynapcnnnetwork/conftest_dynapcnnnetwork.py create mode 100644 tests/test_dynapcnnnetwork/test_dynapcnnnetwork.py diff --git a/tests/test_dynapcnnnetwork/architectures_samples.py b/tests/test_dynapcnnnetwork/architectures_samples.py new file mode 100644 index 00000000..5adde8e0 --- /dev/null +++ b/tests/test_dynapcnnnetwork/architectures_samples.py @@ -0,0 +1,292 @@ +import torch +import torch.nn as nn +from sinabs.layers import Merge, IAFSqueeze + +class EXAMPLE_1(nn.Module): + def __init__(self) -> None: + super().__init__() + + self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False) + self.iaf1 = IAFSqueeze(batch_size=1) + self.pool1 = nn.AvgPool2d(3,3) + self.pool1a = nn.AvgPool2d(4,4) + + self.conv2 = nn.Conv2d(10, 10, 4, 1, bias=False) + self.iaf2 = IAFSqueeze(batch_size=1) + + self.conv3 = nn.Conv2d(10, 1, 2, 1, bias=False) + self.iaf3 = IAFSqueeze(batch_size=1) + + self.flat = nn.Flatten() + + self.fc1 = nn.Linear(49, 500, bias=False) + self.iaf4 = IAFSqueeze(batch_size=1) + + self.fc2 = nn.Linear(500, 10, bias=False) + self.iaf5 = IAFSqueeze(batch_size=1) + + self.adder = Merge() + + def forward(self, x): + + con1_out = self.conv1(x) + iaf1_out = self.iaf1(con1_out) + pool1_out = self.pool1(iaf1_out) + pool1a_out = self.pool1a(iaf1_out) + + conv2_out = self.conv2(pool1_out) + iaf2_out = self.iaf2(conv2_out) + + conv3_out = self.conv3(self.adder(pool1a_out, iaf2_out)) + iaf3_out = self.iaf3(conv3_out) + + flat_out = self.flat(iaf3_out) + + fc1_out = self.fc1(flat_out) + iaf4_out = self.iaf4(fc1_out) + fc2_out = self.fc2(iaf4_out) + iaf5_out = self.iaf5(fc2_out) + + return iaf5_out + +class EXAMPLE_2(nn.Module): + def __init__(self) -> None: + super().__init__() + + self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False) + self.conv1_iaf = IAFSqueeze(batch_size=1) + self.pool1 = nn.AvgPool2d(3,3) + self.pool1a = nn.AvgPool2d(4,4) + + self.conv2 = nn.Conv2d(10, 10, 4, 1, bias=False) + self.conv2_iaf = IAFSqueeze(batch_size=1) + + self.conv3 = nn.Conv2d(10, 1, 2, 1, bias=False) + self.conv3_iaf = IAFSqueeze(batch_size=1) + + self.flat = nn.Flatten() + + self.fc1 = nn.Linear(49, 100, bias=False) + self.fc1_iaf = IAFSqueeze(batch_size=1) + + self.fc2 = nn.Linear(100, 100, bias=False) + self.fc2_iaf = IAFSqueeze(batch_size=1) + + self.fc3 = nn.Linear(100, 10, bias=False) + self.fc3_iaf = IAFSqueeze(batch_size=1) + + self.merge1 = Merge() + + def forward(self, x): + # -- conv. block 1 -- + con1_out = self.conv1(x) + conv1_iaf_out = self.conv1_iaf(con1_out) + pool1_out = self.pool1(conv1_iaf_out) + pool1a_out = self.pool1a(conv1_iaf_out) + # -- conv. block 2 -- + conv2_out = self.conv2(pool1_out) + conv2_iaf_out = self.conv2_iaf(conv2_out) + # -- conv. block 3 -- + merge1_out = self.merge1(pool1a_out, conv2_iaf_out) + conv3_out = self.conv3(merge1_out) + conv3_iaf_out = self.conv3_iaf(conv3_out) + flat_out = self.flat(conv3_iaf_out) + # -- fc clock 1 -- + fc1_out = self.fc1(flat_out) + fc1_iaf_out = self.fc1_iaf(fc1_out) + # -- fc clock 2 -- + fc2_out = self.fc2(fc1_iaf_out) + fc2_iaf_out = self.fc2_iaf(fc2_out) + # -- fc clock 3 -- + fc3_out = self.fc3(fc2_iaf_out) + fc3_iaf_out = self.fc3_iaf(fc3_out) + + return fc3_iaf_out + +class EXAMPLE_3(nn.Module): + def __init__(self) -> None: + super().__init__() + + self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False) + self.conv1_iaf = IAFSqueeze(batch_size=1) + self.pool1 = nn.AvgPool2d(3,3) + self.pool1a = nn.AvgPool2d(4,4) + + self.conv2 = nn.Conv2d(10, 10, 4, 1, bias=False) + self.conv2_iaf = IAFSqueeze(batch_size=1) + + self.conv3 = nn.Conv2d(10, 1, 2, 1, bias=False) + self.conv3_iaf = IAFSqueeze(batch_size=1) + + self.flat = nn.Flatten() + + self.fc1 = nn.Linear(49, 100, bias=False) + self.fc1_iaf = IAFSqueeze(batch_size=1) + + self.fc2 = nn.Linear(100, 100, bias=False) + self.fc2_iaf = IAFSqueeze(batch_size=1) + + self.fc3 = nn.Linear(100, 10, bias=False) + self.fc3_iaf = IAFSqueeze(batch_size=1) + + self.merge1 = Merge() + self.merge2 = Merge() + + def forward(self, x): + # -- conv. block 0 -- + con1_out = self.conv1(x) + conv1_iaf_out = self.conv1_iaf(con1_out) + pool1_out = self.pool1(conv1_iaf_out) + pool1a_out = self.pool1a(conv1_iaf_out) + # -- conv. block 1 -- + conv2_out = self.conv2(pool1_out) + conv2_iaf_out = self.conv2_iaf(conv2_out) + # -- conv. block 2 -- + merge1_out = self.merge1(pool1a_out, conv2_iaf_out) + conv3_out = self.conv3(merge1_out) + conv3_iaf_out = self.conv3_iaf(conv3_out) + flat_out = self.flat(conv3_iaf_out) + # -- fc clock 3 -- + fc1_out = self.fc1(flat_out) + fc1_iaf_out = self.fc1_iaf(fc1_out) + # -- fc clock 4 -- + fc2_out = self.fc2(fc1_iaf_out) + fc2_iaf_out = self.fc2_iaf(fc2_out) + # -- fc clock 5 -- + merge2_out = self.merge2(fc1_iaf_out, fc2_iaf_out) + fc3_out = self.fc3(merge2_out) + fc3_iaf_out = self.fc3_iaf(fc3_out) + + return fc3_iaf_out + +class EXAMPLE_5(nn.Module): + def __init__(self) -> None: + super().__init__() + + self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False) + self.conv1_iaf = IAFSqueeze(batch_size=1) + self.pool1 = nn.AvgPool2d(3,3) + self.pool1a = nn.AvgPool2d(4,4) + + self.conv2 = nn.Conv2d(10, 10, 4, 1, bias=False) + self.conv2_iaf = IAFSqueeze(batch_size=1) + + self.conv3 = nn.Conv2d(10, 10, 2, 1, bias=False) + self.conv3_iaf = IAFSqueeze(batch_size=1) + + self.conv4 = nn.Conv2d(10, 1, 2, 1, bias=False) + self.conv4_iaf = IAFSqueeze(batch_size=1) + + self.flat = nn.Flatten() + + self.fc1 = nn.Linear(36, 100, bias=False) + self.fc1_iaf = IAFSqueeze(batch_size=1) + + self.fc2 = nn.Linear(100, 100, bias=False) + self.fc2_iaf = IAFSqueeze(batch_size=1) + + self.fc3 = nn.Linear(100, 100, bias=False) + self.fc3_iaf = IAFSqueeze(batch_size=1) + + self.fc4 = nn.Linear(100, 10, bias=False) + self.fc4_iaf = IAFSqueeze(batch_size=1) + + self.merge1 = Merge() + self.merge2 = Merge() + self.merge3 = Merge() + + def forward(self, x): + # -- conv. block 0 -- + con1_out = self.conv1(x) + conv1_iaf_out = self.conv1_iaf(con1_out) + pool1_out = self.pool1(conv1_iaf_out) + pool1a_out = self.pool1a(conv1_iaf_out) + # -- conv. block 1 -- + conv2_out = self.conv2(pool1_out) + conv2_iaf_out = self.conv2_iaf(conv2_out) + # -- conv. block 2 -- + merge1_out = self.merge1(pool1a_out, conv2_iaf_out) + conv3_out = self.conv3(merge1_out) + conv3_iaf_out = self.conv3_iaf(conv3_out) + # -- conv. block 3 -- + conv4_out = self.conv4(conv3_iaf_out) + conv4_iaf_out = self.conv4_iaf(conv4_out) + flat_out = self.flat(conv4_iaf_out) + # -- fc clock 4 -- + fc1_out = self.fc1(flat_out) + fc1_iaf_out = self.fc1_iaf(fc1_out) + # -- fc clock 5 -- + fc2_out = self.fc2(fc1_iaf_out) + fc2_iaf_out = self.fc2_iaf(fc2_out) + # -- fc clock 6 -- + merge2_out = self.merge2(fc1_iaf_out, fc2_iaf_out) + fc3_out = self.fc3(merge2_out) + fc3_iaf_out = self.fc3_iaf(fc3_out) + # -- fc clock 7 -- + merge3_out = self.merge3(fc2_iaf_out, fc3_iaf_out) + fc4_out = self.fc4(merge3_out) + fc4_iaf_out = self.fc4_iaf(fc4_out) + + return fc4_iaf_out + +class EXAMPLE_4(nn.Module): + def __init__(self) -> None: + super().__init__() + + self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False) + self.conv1_iaf = IAFSqueeze(batch_size=1) + self.pool1 = nn.AvgPool2d(3,3) + self.pool1a = nn.AvgPool2d(4,4) + + self.conv2 = nn.Conv2d(10, 10, 4, 1, bias=False) + self.conv2_iaf = IAFSqueeze(batch_size=1) + + self.conv3 = nn.Conv2d(10, 10, 2, 1, bias=False) + self.conv3_iaf = IAFSqueeze(batch_size=1) + + self.conv4 = nn.Conv2d(10, 1, 2, 1, bias=False) + self.conv4_iaf = IAFSqueeze(batch_size=1) + + self.flat = nn.Flatten() + + self.fc1 = nn.Linear(36, 100, bias=False) + self.fc1_iaf = IAFSqueeze(batch_size=1) + + self.fc2 = nn.Linear(100, 100, bias=False) + self.fc2_iaf = IAFSqueeze(batch_size=1) + + self.fc3 = nn.Linear(100, 10, bias=False) + self.fc3_iaf = IAFSqueeze(batch_size=1) + + self.merge1 = Merge() + self.merge2 = Merge() + + def forward(self, x): + # -- conv. block 0 -- + con1_out = self.conv1(x) + conv1_iaf_out = self.conv1_iaf(con1_out) + pool1_out = self.pool1(conv1_iaf_out) + pool1a_out = self.pool1a(conv1_iaf_out) + # -- conv. block 1 -- + conv2_out = self.conv2(pool1_out) + conv2_iaf_out = self.conv2_iaf(conv2_out) + # -- conv. block 2 -- + merge1_out = self.merge1(pool1a_out, conv2_iaf_out) + conv3_out = self.conv3(merge1_out) + conv3_iaf_out = self.conv3_iaf(conv3_out) + # -- conv. block 3 -- + conv4_out = self.conv4(conv3_iaf_out) + conv4_iaf_out = self.conv4_iaf(conv4_out) + flat_out = self.flat(conv4_iaf_out) + # -- fc clock 4 -- + fc1_out = self.fc1(flat_out) + fc1_iaf_out = self.fc1_iaf(fc1_out) + # -- fc clock 5 -- + fc2_out = self.fc2(fc1_iaf_out) + fc2_iaf_out = self.fc2_iaf(fc2_out) + # -- fc clock 6 -- + merge2_out = self.merge2(fc1_iaf_out, fc2_iaf_out) + fc3_out = self.fc3(merge2_out) + fc3_iaf_out = self.fc3_iaf(fc3_out) + + return fc3_iaf_out \ No newline at end of file diff --git a/tests/test_dynapcnnnetwork/conftest_dynapcnnnetwork.py b/tests/test_dynapcnnnetwork/conftest_dynapcnnnetwork.py new file mode 100644 index 00000000..300f7455 --- /dev/null +++ b/tests/test_dynapcnnnetwork/conftest_dynapcnnnetwork.py @@ -0,0 +1,349 @@ +from architectures_samples import * + +# --- test_NIRtoDynapcnnNetwork_edges_list(snn, edges_list) --- + +edges_list_1 = [ + (0, 1), + (1, 2), + (1, 3), + (2, 4), + (3, 5), + (4, 6), + (6, 5), + (7, 8), + (8, 9), + (9, 10), + (10, 11), + (11, 12), + (12, 13), + (5, 7), +] + +edges_list_2 = [ + (0, 1), + (1, 2), + (1, 3), + (2, 4), + (3, 5), + (4, 6), + (6, 5), + (7, 8), + (8, 9), + (9, 10), + (10, 11), + (11, 12), + (12, 13), + (13, 14), + (14, 15), + (5, 7), +] + +edges_list_3 = [ + (0, 1), + (1, 2), + (1, 3), + (2, 4), + (3, 5), + (4, 6), + (6, 5), + (7, 8), + (8, 9), + (9, 10), + (10, 11), + (11, 12), + (11, 13), + (12, 14), + (14, 13), + (15, 16), + (5, 7), + (13, 15), +] + +edges_list_5 = [ + (0, 1), + (1, 2), + (1, 3), + (2, 4), + (3, 5), + (4, 6), + (6, 5), + (7, 8), + (8, 9), + (9, 10), + (10, 11), + (11, 12), + (12, 13), + (13, 14), + (13, 15), + (14, 16), + (16, 15), + (16, 17), + (18, 19), + (19, 17), + (20, 21), + (5, 7), + (15, 18), + (17, 20), +] + +edges_list_4 = [ + (0, 1), + (1, 2), + (1, 3), + (2, 4), + (3, 5), + (4, 6), + (6, 5), + (7, 8), + (8, 9), + (9, 10), + (10, 11), + (11, 12), + (12, 13), + (13, 14), + (13, 15), + (14, 16), + (16, 15), + (17, 18), + (5, 7), + (15, 17), +] + +args_NIRtoDynapcnnNetwork_edges_list = [ + (EXAMPLE_1(), edges_list_1), + (EXAMPLE_2(), edges_list_2), + (EXAMPLE_3(), edges_list_3), + (EXAMPLE_4(), edges_list_4), + (EXAMPLE_5(), edges_list_5), + ] + +# --- test_NIRtoDynapcnnNetwork_IO(snn, io_dict) --- + +nodes_IO_1 = { + 0: {'in': torch.Size([1, 2, 34, 34]), 'out': torch.Size([1, 10, 33, 33])}, + 1: {'in': torch.Size([1, 10, 33, 33]), 'out': torch.Size([1, 10, 33, 33])}, + 2: {'in': torch.Size([1, 10, 33, 33]), 'out': torch.Size([1, 10, 11, 11])}, + 3: {'in': torch.Size([1, 10, 33, 33]), 'out': torch.Size([1, 10, 8, 8])}, + 4: {'in': torch.Size([1, 10, 11, 11]), 'out': torch.Size([1, 10, 8, 8])}, + 6: {'in': torch.Size([1, 10, 8, 8]), 'out': torch.Size([1, 10, 8, 8])}, + 7: {'in': torch.Size([1, 10, 8, 8]), 'out': torch.Size([1, 1, 7, 7])}, + 8: {'in': torch.Size([1, 1, 7, 7]), 'out': torch.Size([1, 1, 7, 7])}, + 10: {'in': torch.Size([1, 49]), 'out': torch.Size([1, 500])}, + 11: {'in': torch.Size([1, 500]), 'out': torch.Size([1, 500])}, + 12: {'in': torch.Size([1, 500]), 'out': torch.Size([1, 10])}, + 13: {'in': torch.Size([1, 10]), 'out': torch.Size([1, 10])}, +} + +nodes_IO_2 = { + 0: {'in': torch.Size([1, 2, 34, 34]), 'out': torch.Size([1, 10, 33, 33])}, + 1: {'in': torch.Size([1, 10, 33, 33]), 'out': torch.Size([1, 10, 33, 33])}, + 2: {'in': torch.Size([1, 10, 33, 33]), 'out': torch.Size([1, 10, 11, 11])}, + 3: {'in': torch.Size([1, 10, 33, 33]), 'out': torch.Size([1, 10, 8, 8])}, + 4: {'in': torch.Size([1, 10, 11, 11]), 'out': torch.Size([1, 10, 8, 8])}, + 6: {'in': torch.Size([1, 10, 8, 8]), 'out': torch.Size([1, 10, 8, 8])}, + 7: {'in': torch.Size([1, 10, 8, 8]), 'out': torch.Size([1, 1, 7, 7])}, + 8: {'in': torch.Size([1, 1, 7, 7]), 'out': torch.Size([1, 1, 7, 7])}, + 10: {'in': torch.Size([1, 49]), 'out': torch.Size([1, 100])}, + 11: {'in': torch.Size([1, 100]), 'out': torch.Size([1, 100])}, + 12: {'in': torch.Size([1, 100]), 'out': torch.Size([1, 100])}, + 13: {'in': torch.Size([1, 100]), 'out': torch.Size([1, 100])}, + 14: {'in': torch.Size([1, 100]), 'out': torch.Size([1, 10])}, + 15: {'in': torch.Size([1, 10]), 'out': torch.Size([1, 10])}, +} + +nodes_IO_3 = { + 0: {'in': torch.Size([1, 2, 34, 34]), 'out': torch.Size([1, 10, 33, 33])}, + 1: {'in': torch.Size([1, 10, 33, 33]), 'out': torch.Size([1, 10, 33, 33])}, + 2: {'in': torch.Size([1, 10, 33, 33]), 'out': torch.Size([1, 10, 11, 11])}, + 3: {'in': torch.Size([1, 10, 33, 33]), 'out': torch.Size([1, 10, 8, 8])}, + 4: {'in': torch.Size([1, 10, 11, 11]), 'out': torch.Size([1, 10, 8, 8])}, + 6: {'in': torch.Size([1, 10, 8, 8]), 'out': torch.Size([1, 10, 8, 8])}, + 7: {'in': torch.Size([1, 10, 8, 8]), 'out': torch.Size([1, 1, 7, 7])}, + 8: {'in': torch.Size([1, 1, 7, 7]), 'out': torch.Size([1, 1, 7, 7])}, + 10: {'in': torch.Size([1, 49]), 'out': torch.Size([1, 100])}, + 11: {'in': torch.Size([1, 100]), 'out': torch.Size([1, 100])}, + 12: {'in': torch.Size([1, 100]), 'out': torch.Size([1, 100])}, + 14: {'in': torch.Size([1, 100]), 'out': torch.Size([1, 100])}, + 15: {'in': torch.Size([1, 100]), 'out': torch.Size([1, 10])}, + 16: {'in': torch.Size([1, 10]), 'out': torch.Size([1, 10])}, +} + +nodes_IO_4 = { + 0: {'in': torch.Size([1, 2, 34, 34]), 'out': torch.Size([1, 10, 33, 33])}, + 1: {'in': torch.Size([1, 10, 33, 33]), 'out': torch.Size([1, 10, 33, 33])}, + 2: {'in': torch.Size([1, 10, 33, 33]), 'out': torch.Size([1, 10, 11, 11])}, + 3: {'in': torch.Size([1, 10, 33, 33]), 'out': torch.Size([1, 10, 8, 8])}, + 4: {'in': torch.Size([1, 10, 11, 11]), 'out': torch.Size([1, 10, 8, 8])}, + 6: {'in': torch.Size([1, 10, 8, 8]), 'out': torch.Size([1, 10, 8, 8])}, + 7: {'in': torch.Size([1, 10, 8, 8]), 'out': torch.Size([1, 10, 7, 7])}, + 8: {'in': torch.Size([1, 10, 7, 7]), 'out': torch.Size([1, 10, 7, 7])}, + 9: {'in': torch.Size([1, 10, 7, 7]), 'out': torch.Size([1, 1, 6, 6])}, + 10: {'in': torch.Size([1, 1, 6, 6]), 'out': torch.Size([1, 1, 6, 6])}, + 12: {'in': torch.Size([1, 36]), 'out': torch.Size([1, 100])}, + 13: {'in': torch.Size([1, 100]), 'out': torch.Size([1, 100])}, + 14: {'in': torch.Size([1, 100]), 'out': torch.Size([1, 100])}, + 16: {'in': torch.Size([1, 100]), 'out': torch.Size([1, 100])}, + 17: {'in': torch.Size([1, 100]), 'out': torch.Size([1, 10])}, + 18: {'in': torch.Size([1, 10]), 'out': torch.Size([1, 10])}, +} + +nodes_IO_5 = { + 0: {'in': torch.Size([1, 2, 34, 34]), 'out': torch.Size([1, 10, 33, 33])}, + 1: {'in': torch.Size([1, 10, 33, 33]), 'out': torch.Size([1, 10, 33, 33])}, + 2: {'in': torch.Size([1, 10, 33, 33]), 'out': torch.Size([1, 10, 11, 11])}, + 3: {'in': torch.Size([1, 10, 33, 33]), 'out': torch.Size([1, 10, 8, 8])}, + 4: {'in': torch.Size([1, 10, 11, 11]), 'out': torch.Size([1, 10, 8, 8])}, + 6: {'in': torch.Size([1, 10, 8, 8]), 'out': torch.Size([1, 10, 8, 8])}, + 7: {'in': torch.Size([1, 10, 8, 8]), 'out': torch.Size([1, 10, 7, 7])}, + 8: {'in': torch.Size([1, 10, 7, 7]), 'out': torch.Size([1, 10, 7, 7])}, + 9: {'in': torch.Size([1, 10, 7, 7]), 'out': torch.Size([1, 1, 6, 6])}, + 10: {'in': torch.Size([1, 1, 6, 6]), 'out': torch.Size([1, 1, 6, 6])}, + 12: {'in': torch.Size([1, 36]), 'out': torch.Size([1, 100])}, + 13: {'in': torch.Size([1, 100]), 'out': torch.Size([1, 100])}, + 14: {'in': torch.Size([1, 100]), 'out': torch.Size([1, 100])}, + 16: {'in': torch.Size([1, 100]), 'out': torch.Size([1, 100])}, + 18: {'in': torch.Size([1, 100]), 'out': torch.Size([1, 100])}, + 19: {'in': torch.Size([1, 100]), 'out': torch.Size([1, 100])}, + 20: {'in': torch.Size([1, 100]), 'out': torch.Size([1, 10])}, + 21: {'in': torch.Size([1, 10]), 'out': torch.Size([1, 10])}, +} + +args_test_NIRtoDynapcnnNetwork_IO = [ + (EXAMPLE_1(), nodes_IO_1), + (EXAMPLE_2(), nodes_IO_2), + (EXAMPLE_3(), nodes_IO_3), + (EXAMPLE_4(), nodes_IO_4), + (EXAMPLE_5(), nodes_IO_5), +] + +# --- test_DynapcnnLyers_edges_list(snn, edges_list) --- + +dcnnl_edges_list_1 = [ + (0, 1), + (0, 2), + (1, 2), + (2, 3), + (3, 4), +] + +dcnnl_edges_list_2 = [ + (0, 1), + (0, 2), + (1, 2), + (2, 3), + (3, 4), + (4, 5), +] + +dcnnl_edges_list_3 = [ + (0, 1), + (0, 2), + (1, 2), + (2, 3), + (3, 4), + (3, 5), + (4, 5), +] + +dcnnl_edges_list_4 = [ + (0, 1), + (0, 2), + (1, 2), + (2, 3), + (3, 4), + (4, 5), + (4, 6), + (5, 6), +] + +dcnnl_edges_list_5 = [ + (0, 1), + (0, 2), + (1, 2), + (2, 3), + (3, 4), + (4, 5), + (4, 6), + (5, 6), + (5, 7), + (6, 7), +] + +args_DynapcnnLyers_edges_list = [ + (EXAMPLE_1(), dcnnl_edges_list_1), + (EXAMPLE_2(), dcnnl_edges_list_2), + (EXAMPLE_3(), dcnnl_edges_list_3), + (EXAMPLE_4(), dcnnl_edges_list_4), + (EXAMPLE_5(), dcnnl_edges_list_5), +] + +# --- test_DynapcnnNetwork_forward_edges(snn, forward_edges_list) --- + +forward_edges_list_1 = [ + (0, '0_pool0'), + (0, '0_pool1'), + ('0_pool0', 1), + (('0_pool1', 1), 'merge_0'), + ('merge_0', 2), + (2, 3), + (3, 4), +] + +forward_edges_list_2 = [ + (0, '0_pool0'), + (0, '0_pool1'), + ('0_pool0', 1), + (('0_pool1', 1), 'merge_0'), + ('merge_0', 2), + (2, 3), + (3, 4), + (4, 5), +] + +forward_edges_list_3 = [ + (0, '0_pool0'), + (0, '0_pool1'), + ('0_pool0', 1), + (('0_pool1', 1), 'merge_0'), + ('merge_0', 2), + (2, 3), + (3, 4), + ((3, 4), 'merge_1'), + ('merge_1', 5), +] + +forward_edges_list_4 = [ + (0, '0_pool0'), + (0, '0_pool1'), + ('0_pool0', 1), + (('0_pool1', 1), 'merge_0'), + ('merge_0', 2), + (2, 3), + (3, 4), + (4, 5), + ((4, 5), 'merge_1'), + ('merge_1', 6), +] + +forward_edges_list_5 = [ + (0, '0_pool0'), + (0, '0_pool1'), + ('0_pool0', 1), + (('0_pool1', 1), 'merge_0'), + ('merge_0', 2), + (2, 3), + (3, 4), + (4, 5), + ((4, 5), 'merge_1'), + ('merge_1', 6), + ((5, 6), 'merge_2'), + ('merge_2', 7), +] + +args_DynapcnnNetwork_forward_edges = [ + (EXAMPLE_1(), forward_edges_list_1), + (EXAMPLE_2(), forward_edges_list_2), + (EXAMPLE_3(), forward_edges_list_3), + (EXAMPLE_4(), forward_edges_list_4), + (EXAMPLE_5(), forward_edges_list_5), +] \ No newline at end of file diff --git a/tests/test_dynapcnnnetwork/test_dynapcnnnetwork.py b/tests/test_dynapcnnnetwork/test_dynapcnnnetwork.py new file mode 100644 index 00000000..6ee0de51 --- /dev/null +++ b/tests/test_dynapcnnnetwork/test_dynapcnnnetwork.py @@ -0,0 +1,84 @@ +import pytest, sys +import torch +from conftest_dynapcnnnetwork import args_NIRtoDynapcnnNetwork_edges_list, args_test_NIRtoDynapcnnNetwork_IO, args_DynapcnnLyers_edges_list, args_DynapcnnNetwork_forward_edges + +sys.path.append('../../sinabs/backend/dynapcnn') + +@pytest.mark.parametrize("snn, edges_list", args_NIRtoDynapcnnNetwork_edges_list) +def test_NIRtoDynapcnnNetwork_edges_list(snn, edges_list): + from NIRGraphExtractor import NIRtoDynapcnnNetworkGraph + + batch_size = 1 + channels = 2 + height = 34 + width = 34 + + input_shape = (batch_size, channels, height, width) + dummy_input = torch.randn(input_shape) + + graph_tracer = NIRtoDynapcnnNetworkGraph(spiking_model = snn, dummy_input = dummy_input) + + assert graph_tracer.get_edges_list() == edges_list + +@pytest.mark.parametrize("snn, io_dict", args_test_NIRtoDynapcnnNetwork_IO) +def test_NIRtoDynapcnnNetwork_IO(snn, io_dict): + from NIRGraphExtractor import NIRtoDynapcnnNetworkGraph + + batch_size = 1 + channels = 2 + height = 34 + width = 34 + + input_shape = (batch_size, channels, height, width) + dummy_input = torch.randn(input_shape) + + graph_tracer = NIRtoDynapcnnNetworkGraph(spiking_model = snn, dummy_input = dummy_input) + + computed_IOs = {} + + for node, IO in io_dict.items(): + _in, _out = graph_tracer.get_node_io_shapes(node) + + computed_IOs[node] = {'in': _in, 'out': _out} + + assert computed_IOs == io_dict + +@pytest.mark.parametrize("snn, dcnnl_edges_list", args_DynapcnnLyers_edges_list) +def test_DynapcnnLyers_edges_list(snn, dcnnl_edges_list): + from sinabs.backend.dynapcnn import DynapcnnNetworkGraph + + channels = 2 + height = 34 + width = 34 + + input_shape = (channels, height, width) + + hw_model = DynapcnnNetworkGraph( + snn, + discretize=True, + input_shape=input_shape + ) + + computed_edges_list = hw_model.get_dynapcnnlayers_edges() + + assert computed_edges_list == dcnnl_edges_list + +@pytest.mark.parametrize("snn, forward_edges_list", args_DynapcnnNetwork_forward_edges) +def test_DynapcnnNetwork_forward_edges(snn, forward_edges_list): + from sinabs.backend.dynapcnn import DynapcnnNetworkGraph + + channels = 2 + height = 34 + width = 34 + + input_shape = (channels, height, width) + + hw_model = DynapcnnNetworkGraph( + snn, + discretize=True, + input_shape=input_shape + ) + + computed_forward_edges_list = hw_model.get_network_module().get_forward_edges() + + assert computed_forward_edges_list == forward_edges_list \ No newline at end of file From 85577eb214f51f645baa380170e96013367e5138 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Wed, 8 May 2024 11:48:36 +0200 Subject: [PATCH 081/379] some public methods for unit testing --- .../dynapcnn/dynapcnn_network_graph.py | 25 +++++++++++-------- .../dynapcnn/dynapcnnnetwork_module.py | 3 +++ 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py index ed1bdff2..d2c9fc6f 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py @@ -277,6 +277,19 @@ def make_config( else: raise ValueError(f"Generated config is not valid for {device}") + def get_network_module(self): + return self._get_network_module() + + def get_dynapcnnlayers_edges(self) -> List[Tuple[int, int]]: + """ Create edges representing connections between `DynapcnnLayer` instances. """ + dcnnl_edges = [] + + for dcnnl_idx, layer_data in self.dynapcnn_layers.items(): + for dest in layer_data['destinations']: + dcnnl_edges.append((dcnnl_idx, dest)) + + return dcnnl_edges + ### Private Methods ### def _get_network_module(self) -> nn.Module: @@ -285,19 +298,9 @@ def _get_network_module(self) -> nn.Module: """ # get connections between `DynapcnnLayer`s. - dcnnl_edges = self._get_dynapcnnlayers_edges() + dcnnl_edges = self.get_dynapcnnlayers_edges() return DynapcnnNetworkModule(dcnnl_edges, self.dynapcnn_layers) - - def _get_dynapcnnlayers_edges(self) -> List[Tuple[int, int]]: - """ Create edges representing connections between `DynapcnnLayer` instances. """ - dcnnl_edges = [] - - for dcnnl_idx, layer_data in self.dynapcnn_layers.items(): - for dest in layer_data['destinations']: - dcnnl_edges.append((dcnnl_idx, dest)) - - return dcnnl_edges def _make_config( self, diff --git a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py index 869bf7eb..f027c3e8 100644 --- a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py +++ b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py @@ -27,6 +27,9 @@ def __init__(self, dcnnl_edges: List[Tuple[int, int]], dynapcnn_layers: Dict) -> self._forward_edges, self._forward_map = self._build_module_forward_from_graph(dcnnl_edges, dynapcnn_layers) + def get_forward_edges(self): + return self._forward_edges + def _build_module_forward_from_graph(self, dcnnl_edges: list, dynapcnn_layers: dict) -> Union[list, dict]: """ TODO use copy.deepcopy for create the `forward_map`. From 0268bf167e9556a90642e01d90509e4c37844c6c Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Wed, 8 May 2024 11:50:01 +0200 Subject: [PATCH 082/379] HPO params. validation --- .../TOP_2_ARCHITECTURES/single_training.ipynb | 3837 +++++++++++++++++ 1 file changed, 3837 insertions(+) create mode 100644 tests/test_nonsequential/using_SumPool2d/TOP_2_ARCHITECTURES/single_training.ipynb diff --git a/tests/test_nonsequential/using_SumPool2d/TOP_2_ARCHITECTURES/single_training.ipynb b/tests/test_nonsequential/using_SumPool2d/TOP_2_ARCHITECTURES/single_training.ipynb new file mode 100644 index 00000000..30637cad --- /dev/null +++ b/tests/test_nonsequential/using_SumPool2d/TOP_2_ARCHITECTURES/single_training.ipynb @@ -0,0 +1,3837 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import torch, random, sys\n", + "\n", + "import tonic\n", + "from torch.utils.data import DataLoader\n", + "from torch.nn import CrossEntropyLoss\n", + "from torch.optim import Adam\n", + "\n", + "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "\n", + "sys.path.append('../../utils')\n", + "sys.path.append('../models')\n", + "\n", + "from train_test_fn import training_loop, load_dataset, load_architecture\n", + "from weight_initialization import rescale_method_1" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "metadata": {} + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "device: NVIDIA GeForce RTX 3070 Ti\n" + ] + } + ], + "source": [ + "if torch.cuda.is_available():\n", + " device = torch.device('cuda:0')\n", + " print('device: ', torch.cuda.get_device_name(0))\n", + "else:\n", + " device = torch.device('cpu')" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "rand_seed = 1" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "achitecture = 'ResSCNN_5'" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "torch.backends.cudnn.enabled = False\n", + "torch.backends.cudnn.deterministic = True\n", + "random.seed(rand_seed)\n", + "torch.manual_seed(rand_seed)\n", + "torch.cuda.manual_seed(rand_seed)\n", + "np.random.seed(rand_seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "batch_size = 8\n", + "num_workers = 4\n", + "epochs = 50\n", + "n_time_steps = 50\n", + "\n", + "lr = 5e-5\n", + "spk_thr = 2.0\n", + "v_min = -0.5\n", + "grad_scale = 1.75\n", + "grad_width = 0.5\n", + "w_rescale_lambda = 0.6" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Loading Data" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "snn_train_dataset, snn_test_dataset, sensor_size = load_dataset('DVSGESTURE', n_time_steps)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "getting validation dataset...." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "disk caching samples..." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "metadata": {} + }, + "outputs": [], + "source": [ + "disk_cache_train = tonic.DiskCachedDataset(\n", + " dataset=snn_train_dataset,\n", + " cache_path='./cached_train'\n", + ")\n", + "snn_train_dataloader = DataLoader(disk_cache_train, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True)\n", + "\n", + "disk_cache_test = tonic.DiskCachedDataset(\n", + " dataset=snn_test_dataset,\n", + " cache_path='./cached_test'\n", + ")\n", + "snn_test_dataloader = DataLoader(disk_cache_test, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=False)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Network Module" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "metadata": {} + }, + "outputs": [ + { + "ename": "ModuleNotFoundError", + "evalue": "No module named 'sinabs.exodus'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[9], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m snn \u001b[38;5;241m=\u001b[39m \u001b[43mload_architecture\u001b[49m\u001b[43m(\u001b[49m\u001b[43machitecture\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43msensor_size\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m11\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbatch_size\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mPeriodicExponential\u001b[49m\u001b[43m(\u001b[49m\u001b[43mgrad_scale\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgrad_scale\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mgrad_width\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgrad_width\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mv_min\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mspk_thr\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241m.\u001b[39mto(device)\n\u001b[1;32m 2\u001b[0m snn\u001b[38;5;241m.\u001b[39minit_weights()\n", + "File \u001b[0;32m~/Documents/github/sinabs/tests/test_nonsequential/using_SumPool2d/TOP_2_ARCHITECTURES/../../utils/train_test_fn.py:281\u001b[0m, in \u001b[0;36mload_architecture\u001b[0;34m(architecture, input_size, nb_classes, batch_size, surrogate_fn, min_v_mem, spk_thr)\u001b[0m\n\u001b[1;32m 279\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m SCNN(input_size, nb_classes, batch_size, surrogate_fn, min_v_mem, spk_thr)\n\u001b[1;32m 280\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m architecture \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mResSCNN_5\u001b[39m\u001b[38;5;124m'\u001b[39m:\n\u001b[0;32m--> 281\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mResSCNN_5\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m SCNN\n\u001b[1;32m 282\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m SCNN(input_size, nb_classes, batch_size, surrogate_fn, min_v_mem, spk_thr)\n\u001b[1;32m 283\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m architecture \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mResSCNN_6\u001b[39m\u001b[38;5;124m'\u001b[39m:\n", + "File \u001b[0;32m~/Documents/github/sinabs/tests/test_nonsequential/using_SumPool2d/TOP_2_ARCHITECTURES/../models/ResSCNN_5.py:3\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mtorch\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mnn\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mnn\u001b[39;00m\n\u001b[1;32m 2\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01msinabs\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mlayers\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01msl\u001b[39;00m\n\u001b[0;32m----> 3\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01msinabs\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mexodus\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mlayers\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m IAFSqueeze\n\u001b[1;32m 5\u001b[0m \u001b[38;5;28;01mclass\u001b[39;00m \u001b[38;5;21;01mSCNN\u001b[39;00m(nn\u001b[38;5;241m.\u001b[39mModule):\n\u001b[1;32m 6\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m, input_size, nb_classes, batch_size, surrogate_fn, min_v_mem\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m0.313\u001b[39m, spk_thr\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m2.0\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m:\n", + "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'sinabs.exodus'" + ] + } + ], + "source": [ + "snn = load_architecture(achitecture, sensor_size, 11, batch_size, PeriodicExponential(grad_scale=grad_scale, grad_width=grad_width), v_min, spk_thr).to(device)\n", + "snn.init_weights()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "metadata": {} + }, + "outputs": [], + "source": [ + "snn.rescale_conv_weights(rescale_method_1, w_rescale_lambda)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "metadata": {} + }, + "outputs": [], + "source": [ + "optimizer = Adam(snn.parameters(), lr=lr, betas=(0.9, 0.999), eps=1e-8)\n", + "loss_fn = CrossEntropyLoss()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Training loop" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "metadata": {} + }, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "87b59082674c47e5a466ddd2ca5097ba", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/134 [00:00" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "y_avg = []\n", + "for y in epochs_y:\n", + " y_avg.append(np.mean(y))\n", + "\n", + "plt.plot(np.arange(len(epochs_x)), y_avg, marker = '.')\n", + "plt.xlabel('epoch')\n", + "plt.ylabel('average loss')\n", + "plt.ylim(0,)\n", + "plt.xticks(np.arange(len(epochs_x)))\n", + "for i, txt in enumerate(y_avg):\n", + " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "metadata": {} + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAG2CAYAAACZEEfAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAACoJElEQVR4nOzddXhT1xsH8O9N0qSpu3uhLe5SKFJgFBn+QzbGsME2YNhwH2wU28aADTaGDmfoOmS4FisUKFKspVCllrol5/dHyKVpU4Ma4f08T56H3Nx7cu5NyH17znvO4RhjDIQQQgghWkpQ1RUghBBCCKlIFOwQQgghRKtRsEMIIYQQrUbBDiGEEEK0GgU7hBBCCNFqFOwQQgghRKtRsEMIIYQQrUbBDiGEEEK0GgU7hBBCCNFqFOwQQgghRKtVabBz/vx59OjRA3Z2duA4DgcPHlR7nTGGefPmwdbWFlKpFJ06dcLjx4/V9klMTMTgwYNhZGQEExMTjBw5EmlpaZV4FoQQQgipzqo02ElPT0eDBg3w66+/anx92bJlWLVqFdatW4erV69CX18ffn5+yMrK4vcZPHgw7t27hxMnTiAgIADnz5/H6NGjK+sUCCGEEFLNcdVlIVCO43DgwAH07t0bgLJVx87ODt9++y2mTJkCAJDJZLC2tsbmzZsxaNAgPHjwALVr18b169fRtGlTAMCxY8fQrVs3vHz5EnZ2dlV1OoQQQgipJkRVXYGihIWFISYmBp06deK3GRsbo0WLFggMDMSgQYMQGBgIExMTPtABgE6dOkEgEODq1avo06ePxrKzs7ORnZ3NP1coFEhMTIS5uTk4jqu4kyKEEEJIuWGMITU1FXZ2dhAIiu6sqrbBTkxMDADA2tpabbu1tTX/WkxMDKysrNReF4lEMDMz4/fRxN/fH999910515gQQgghVeHFixdwcHAo8vVqG+xUpJkzZ2Ly5Mn8c5lMBicnJ7x48QJGRkZVWDNCCCGElFZKSgocHR1haGhY7H7VNtixsbEBAMTGxsLW1pbfHhsbi4YNG/L7xMXFqR2Xl5eHxMRE/nhNJBIJJBJJoe1GRkYU7BBCCCHvmZJSUKrtPDuurq6wsbHBqVOn+G0pKSm4evUqvL29AQDe3t5ITk5GUFAQv8/p06ehUCjQokWLSq8zIYQQQqqfKm3ZSUtLw5MnT/jnYWFhCA4OhpmZGZycnDBx4kR8//33qFmzJlxdXTF37lzY2dnxI7Zq1aqFLl26YNSoUVi3bh1yc3Mxbtw4DBo0iEZiEUIIIQRAFQc7N27cgK+vL/9clUczdOhQbN68GdOmTUN6ejpGjx6N5ORk+Pj44NixY9DV1eWP2b59O8aNG4eOHTtCIBCgX79+WLVqVaWfCyGEEEKqp2ozz05VSklJgbGxMWQyGeXsEEIIIe+J0t6/q23ODiGEEEJIeaBghxBCCCFajYIdQgghhGg1CnYIIYQQotUo2CGEEEKIVqNghxBCCCFajYIdQgghhGg1CnYIIYQQotUo2CGEEEKIVqNghxBCCCFajYIdQgghhGg1CnYIIYQQotUo2CGEEEKIVqNghxBCCCFajYIdQgghhGg1CnYIIYQQotUo2CGEEEKIVqNghxBCCCFajYIdQgghhGg1CnYIIYQQotUo2CGEEEKIVqNghxBCCCFajYIdQgghhGg1CnYIIYQQotUo2CGEEEKIVqNghxBCCCFajYIdQgghhGg1CnYIIYQQotUo2CGEEEKIVqNghxBCCCFajYIdQgghhGg1CnYIIYQQotUo2CGEEEKIVqNghxBCCCFajYIdQgghhGg1CnYIIYQQotUo2CGEEEKIVqNghxBCCCFajYIdQgghhGg1CnYIIYQQotUo2CGEEEKIVqNghxBCCCFajYIdQgghhGg1CnYIIYQQotUo2CGEEEKIVqNghxBCCCFajYIdQgghhGg1CnYIIYQQotUo2CGEEEJIIXK5HHPnzoWrqyukUinc3d2xaNEiMMb4ffbv34/OnTvD3NwcHMchODi4VGWvXLkSnp6ekEqlcHR0xKRJk5CVlcW/7u/vj2bNmsHQ0BBWVlbo3bs3QkND3/pcKNghhBBCSCFLly7F2rVrsWbNGjx48ABLly7FsmXLsHr1an6f9PR0+Pj4YOnSpaUud8eOHZgxYwbmz5+PBw8eYMOGDdi9ezdmzZrF73Pu3DmMHTsWV65cwYkTJ5Cbm4vOnTsjPT39rc5F9FZHEUIIIUSrXb58Gb169UL37t0BAC4uLti5cyeuXbvG7zNkyBAAQHh4eJnKbd26NT799FO+3E8++QRXr17l9zl27JjaMZs3b4aVlRWCgoLQtm3bMp8LtewQQgghpJBWrVrh1KlTePToEQDg9u3buHjxIrp27frO5QYFBfFB07Nnz3DkyBF069atyGNkMhkAwMzM7K3ek1p2CCGEEFLIjBkzkJKSAi8vLwiFQsjlcvzwww8YPHjwO5X76aefIj4+Hj4+PmCMIS8vD1999ZVaN1Z+CoUCEydOROvWrVG3bt23ek9q2SGEEEJIIXv27MH27duxY8cO3Lx5E1u2bMGKFSuwZcuWdyr37NmzWLx4MX777TfcvHkT+/fvx7///otFixZp3H/s2LEICQnBrl273vo9qWWHEEIIIYVMnToVM2bMwKBBgwAA9erVw/Pnz+Hv74+hQ4e+dblz587FkCFD8MUXX/DlpqenY/To0Zg9ezYEgjftMOPGjUNAQADOnz8PBweHt35PCnYIIYQQUkhGRoZa4AEAQqEQCoWiQsoFwA9rZ4zhm2++wYEDB3D27Fm4urq+03tSsEMIIYSQQnr06IEffvgBTk5OqFOnDm7duoWffvoJI0aM4PdJTExEREQEoqKiAICfC8fGxgY2NjYAgM8//xz29vbw9/fny/3pp5/QqFEjtGjRAk+ePMHcuXPRo0cPPugZO3YsduzYgUOHDsHQ0BAxMTEAAGNjY0il0rKfDCNMJpMxAEwmk1V1VQghhJBqISUlhU2YMIE5OTkxXV1d5ubmxmbPns2ys7P5fX7+9XcGoNBj/vz5/D7erduwrn0HsqjkDMYYY7m5uWzBggXM3d2d6erqMkdHRzZmzBiWlJTEH6OpTABs06ZNanUs7f2be13oBy0lJQXGxsaQyWQwMjKq6uoQQggh1d7u6xGYuf8uFAwQcIB/33oY2MypzPu8i9Lev2k0FiGEEELKJFqWyQcxAKBgwKz9IYiWZZZpn8pCwQ4hhBBCShQty8Tlp/GISs7AlsvP+SBGRc4YwuMz+OdP4tJK3KeyUIIyIYQQQoqVvzuqOOYGYgBAVq4cf154Vuh1IcfBxUKvIqpYLGrZIYQQQkiRCnZHqbT3tISAU982cVcwroUloO9vl3HuUTyEAo7fR8hxWNy3LmyN32I01Tuq1sFOaZaXZ4xh3rx5sLW1hVQqRadOnfD48eMqrDUhhBCiPcLi0zW26HzZ1h2XZnTAzlEtsWV4M1gYiHE/OgUDfr+C+9EpAIARrV34fS7O8C3X5OSyqNbBTmmWl1+2bBlWrVqFdevW4erVq9DX14efnx+ysrKqsOaEEEK0gYuLCziOK/QYO3YswsPDNb7GcRz27t2rsbzc3FxMnz4d9erVg76+Puzs7PD555/z89QAyuUUiir3+vXrlXXqPFcL/UItOKruKFtjKbzdzdHO0wq/DW5c6NiNF8MBAN7u5lXSoqNSrYOd/MvLu7i44H//+x86d+7Mr5TKGMPKlSsxZ84c9OrVC/Xr18fWrVsRFRWFgwcPVm3lCSGEvPeuX7+O6Oho/nHixAkAQP/+/eHo6Kj2WnR0NL777jsYGBgUuTJ4RkYGbt68iblz5/LrQoWGhqJnz578Pq1atSpU7hdffAFXV1c0bdq0Us47P1tjKcZ3qMk/F3DQ2B2Vp6H5p6oSkguq1sFOScvLh4WFISYmBp06deKPMTY2RosWLRAYGFhkudnZ2UhJSVF7EEIIIQVZWlryswHb2NggICAA7u7uaNeuHYRCodprNjY2OHDgAAYMGAADAwON5RkbG+PEiROYNm0avLy84O3tjWvXriEoKIhvMRKLxbCxsUFYWBg+/fRTuLu7Y8OGDVAoFMX2WpQm9SM2NhbDhg2DnZ0d9PT00KVLl1KlfqgSj+vZG+HSjA4au6OKawGqatU62FEtQObl5QUdHR00atQIEydO5JeXV00fbW1trXactbU1/5om/v7+MDY25h+Ojo4VdxKEEEIqVHFdTSqBgYHo0KED9PX1YWRkhLZt2yIzs+j5XhYsWFCoPE9PT2zbtg0jRoxAUlISvvnmG3h6ekIqlcLJyQmDBg1CcHAweg38DJefxhc7n0zAqfM4HBiC4NBn2L17N7+9f//+fH39unSBZ5PWmDZ7AQBg+vTphdaUyq+k1A/GGHr37o1nz57h0KFDuHXrFpydndGpUyekp6cXe41vPE8CAHSqZVNkd5StsRT+fetByCkjnqpMSC6oWg89z7+8fJ06dRAcHIyJEyfCzs7unVZcnTlzJiZPnsw/T0lJoYCHEELeU9evX4dcLuefh4SE4KOPPlILHLp06YKZM2di9erVEIlEuH37drGBAwDUqVMHJ0+e5J8HBATgq6++wrBhwxAVFYWoqCisWLECtWvXxvPnz9G3b1/o6ulj4pkMKE5fLdWswpDnQBCwCJ6ensjLy0O7du0AAJ+PHgth3a44KmyJuA3z0cC7Hb7++uti65s/9QNQBoE7d+7kUz8eP36MK1euICQkBHXq1AEArF27FjY2Nti5cye/CrkmN8KVwU5TF9Ni6zCwmRPaelgiPD6Dz+mpDqp1sFPS8vKqRcZiY2Nha2vLHxcbG4uGDRsWWa5EIoFEIqnQuhNCCKkclpaWas+XLFnCdzUBwKRJkzB+/HjMmDGD38fT07PEchknwLN0EVwt9GFrLMXevXvRtWtX2NnZwc7ODvv27eP3tbOzQ05OLrKysiCXy8EJhPyMwW09LPmbfv5h3Eyeh1cHlyAvJQP6ucmYOuVbcByHu0+e40nILZh2ao7ozRORE/sE92SOOHjsFHp36VhkfVu1aoU//vgDjx49goeHB5/68dNPPwFQpnAAgK6uLn+MQCCARCLBxYsXiwx2opIzEZmcCaGAQ0NHkxKvm62xtNoEOSrVuhurpOXlXV1dYWNjg1OnTvGvp6Sk4OrVq/D29q7UuhJCCClaSV1N7du3L/TaV199VWyZRY1YWr9+PUaMGAGO4xAXF4erV69i69atEAgEEAgEsLCwwMWLF4stOyRShgehj9C2oSecnF3RtF1nnDx5En0/GaKxi+rvv/9Gdk42OIkeOIGQ314wQVc1jJvJ8/Dq0BLkyeJg2PhjpKbIMGzYMABA4K0HAADZxR0QGlmA0zWArktDDOjZrdj8GlXjgKenJziOQ8OGDREXF4fPPvsMHMfht99+g5OTE7y9vdWu18uXL9XuowXdeJ6E50s/xjP/7jDQ1VE7dvny5Wr7/vvvv2jRogWkUilMTU3Ru3fvYq9zZanWLTslLS/PcRwmTpyI77//HjVr1oSrqyvmzp0LOzu7anOBCSGElNzVBACjRo3CwoUL+ed6esUntkZHR6s9P3r0KEaMGIHs7Gw+cNiwYQMAICEhAT/88AM8PDywceNGdOzYESEhIahZs2bBYhEty8RFmQnMu02Cjpk95GmJCD66CgoGzL+mgOBO4S6qdevWgROIYNCom1pZAg5qCbqJaTlvAp2kKFh/4o/4gBXw7dQZdnZ2AIDbL5RdRgYNuyDjwXkY1vsIFh2/gCQzDBs3boS/v7/G66FK/Vi6cjWk5vZIiQrDCv9FGDVqFJYvX46BAwfiiy++4Fu8BAIB2rZtyweBRQkKT4TD2L8wsLkjpnT25K/1yJEj0a9fP36/ffv2YdSoUVi8eDE6dOiAvLw8hISEFFluZarWwc7q1asxd+5cjBkzBnFxcbCzs8OXX36JefPm8ftMmzYN6enpGD16NJKTk+Hj44Njx46pNdMRQgipWiV1NQHK4EaVnlAaBfc9dOgQzMzM0KpVK9jZ2SEvL4/vwhk/fjxmzpwJAOjXrx/q169fZOAQFp8OXbc3Q7yZpTPAcQAnQHpoIAwbdFbroop59hCXL1+GjnUNmPsMBjjwk/C1cDWDb4tG8Pf3R5fuPbH06D28OuiPnNinsPrfPOQlxyErPBhtxq5BTk4OLoUl49Dj1y1BjCFPFguDBp2xuG9d7H9WGxEREUVej6lTp+KjT0ZjXYwrFNGAgPNE54Ej8eeff/LXmuM4NG3aFLVq1cLChQthaWmJFi1aFDuk/Xp4EoQGpvBt6MFf80OHDsHX1xdubm4AgLy8PEyYMAHLly/HyJEj+WNr165dZLmVqVp3YxkaGmLlypV4/vw5MjMz8fTpU3z//fcQi8X8PhzHYeHChYiJiUFWVhZOnjwJDw+PKqw1IeR9URFdK6WZ1T0xMRGDBw+GkZERTExMMHLkSKSlpVXYeZZWZV2Pe/fu8aOaOI5DYmIiHjx4gFWrVkEgEMDU1BTffvstMjJKPz9LbGwsAgICkJSUxOee3Lx5E/Hx8QCA7du3w9bWFl27dkVISAhq1apVZOAg4NTHT2eFB0OeGg8dM3vkJb+Z/E/OGE4Gh8P3o66AUAc2g5dgSf9GuDSjA2Z3qwUACHyWiNDQUMhkMvx0IhRhz18g88lVyFPjEb1pPGL+mgQwhvlTxmL2ur8xbsdNCI2sYWBmCd3oYEjsa+HjNk0xsJkTHj16BGdn5yKvQXp6Bg7fjlFbZfy/+7GQyWT8tVb5+++/UatWLXh4eODatWvw8/PTWGZqVi4exiinZ1ElJ8fGxuLff/9VC2pu3ryJyMhICAQCNGrUSO1aVwfVOtghhJCKVNyEcSqjRo1S22fZsmXFllmaWd0HDx6Me/fu4cSJEwgICMD58+cxevToijnJMqis6+Hr64vk5GS+q2nw4MEQi8X4+eefsWnTJkgkEqxduxafffZZqeu+ZcsW6OjowNramh+N9OyZciFKoVCIZs2aISAgAKampmjfvj0ePHhQZOCw6VKY2nMDtyZYeeQ25GkJEOqb8dsV2Rn44tM+yNKzhOOEnfDxsseAZo6wNZZiVFs3DGyqHOXrs+QU5DXa4s8LYRAZW+PUgxgwxiCXy+Ho6ASbNgPhPD0Ae6OMkZ4th4u5PubPmom0xFgYNu2Fh48eY+7cuXj48KFagNGxY0esWbOGf16vVQckX96NjKfXkSeLRcajy5Bd2QfGGH+t9+7diyZNmmD58uWYO3cuEhMToauri82bN2u8FrcikqFggKOZFNZGuvy1NjQ0RN++ffn9VNd6wYIFmDNnjtq1TkxMLPHzq3CMMJlMxgAwmUxW1VUhhFShCRMmMHd3d6ZQKBhjjLVr145NmDCh1McrFApmY2PDli9fzm9LTk5mEomE7dy5kzHG2P379xkAdv36dX6fo0ePMo7jWGRkZPmcSDmpqOvBcRxr3LgxY6zo6wGAAWBPnjwp1Xt5eHgwQ0NDNn36dH7b9u3bGQDWv39/ZmRkxPbu3cvu3bvHpFIp09HRUSu7Q4cObPXq1exGeAJznh7AjJr3YWu2HWT7zt5kh46fZp06dWKGJmbMefwO5jw9gDlN3MPEtp5Mx9KF2Y1ezxzG/sUcx/3FgkOfsby8POW5ZuSwuvOOMefpAfyj15oL/HseP36cAWAOo39X28d1RgCLSs5g0+d+x4SGFozTkbCWLb3ZhQtvjmWMMWdnZzZ//nzGGGM3whNZrRn7mWGTnkxoZMk4kZiJTGyYyMSGdezsxx/zyy+/MAcHB6ajo8OcnJzYnDlz2LFjx4q81j/+F8qcpwewSbtu8ds8PT3ZuHHj1PZTXevff/+d35aVlcUsLCzYunXrSvUZvo3S3r+rdc4OIYRUlpycHGzbtg2TJ09Wa+7fvn07tm3bBhsbG/To0QNz584tMnG2pFndBw0ahMDAQJiYmKjlSHTq1AkCgQBXr15Fnz59Ku4ky6CirkdycjIYY/x0IcVdD4VCgSdPnsDd3b3Yul64cIGfaV81gAUA/x6fjvgS5s6eGD9xImRJSRAKhfjf//6nVu7Tp0/x6tUr+B95CACwE2Xgh6lfIyEhAZaWlvDx8cGtG9egZ2GH8PgMnD17BrNWhgIAov4YxZfTcI3yvF1cXJCRk4e07Dy1ut55KUO0LBO2xlJ07twZl568wqfrr6rto2BAeHwGFi+Yi0OsOTJz5dj2bTu4W6rPyhweHo5oWSbWnXuKlSceIYuJ0XDABEQkKlsJ5bI4RP7xBb4Z82Z+nvHjx2P8+PGIlmUiLD4drhb6MBIpRzhrutY3wpWtMk1ed2FduHABoaGhahMh5r/W+XN0JBIJ3Nzcis0zqiwU7BBCCICDBw+qda0AwKeffgpnZ2fY2dnhzp07mD59OkJDQ7F//36NZZRmVveYmBhYWVmpvS4SiWBmZlbszO+VraKux6ZNm5QDSERiXH4aj8fhLzReDyMjIyQnJ6vNoVaUDRs2oEmTJrhx44ba9iZNmkBHLMGoVYehX78zJENaYu3HnpjatzXqNWmOy0/j+Tl0wsPDsfv6C2zZdwcSEYczRw7CxljzQBdbYylcLLpj/ZMAtdXAhRyHizN8+TlmwuLTUXC1KFUgo9pHtcRCwXJcLPQgEHDwsDbA7ZcyPIpJLRTs7L4egRn77vLvUdPaAIfGtsaRu9GYsvcOch+ehrWVFd+tp7LzWgRm7VceJ+CAYe7KLtaC1zpXrkDwi2QAQDMXM7Vr3aBBg0LXWiKRIDQ0FD4+Psrjc3MRHh5ebJ5RZaFghxBCoPwRV00Yp5I/j6ZevXqwtbVFx44d8fTp0xJbG9535X09omWZeBqXij83bISZtT3OhsYjZP1VpAQ+gyg1E4sWLUK3bt1gbm6OO3fuICUlBTVq1ED9+vX5Mry8vODv76/W+pWSkoK9e/fixx9/VHuvsPh0iAQcdOv5IenCdggMLSAyssLYMauQk5GL1WGW+HW9cgj5ND9PMABLjylbanLyGM49itO4/pOKammEWftDIGdM49IIxQUypS3Hw9oQt1/KEBqbiq713gQjqskJ8wdTT+PSIMvMRa+G9lgccB93bh5H/4EDIRK9udUHBt/DmG/9oeveDEKpIXLiwrFo3Xq0bOVT6Fp/+e0cZOSYwkhXhBqWBhqvtYqRkRG++uorzJ8/H46OjnB2dubn4Mmf81VVKNghhHzwnj9/jpMnTxbZQqHSokULAJqb+wGUalZ3GxsbxMXFqR2Xl5eHxMTEMg27rkjlfT22ng7G7yEKpD+7ibiXL6BjUxO6FiYAAIG+CeIT4nHk2H9YuXIl0tPT4ejoCMYYFixYoFaealRTfrt27QJjDJ988gmAAksxADD1HQFOIER8wE9gedmQ2HrCfMD3EOgqW0kUDFjyOshRYSg887EmJS2NUJqAqKRyPG0MAQCPYlPVjlFNTphf/lajOniOWymvIPJSn3E5Iikbmc9vI+XGYShysyAysoCeRyss+lU9gAkNDcXdsGgApmjqYgaBgCt0rQtavnw5RCIRhgwZgszMTLRo0QKnT5+GqWnxS0xUBgp2CCEfvE2bNsFKQ3N/QcHBwQAKN/er5J/VXRXcqGZ1V61r5O3tjeTkZAQFBaFJkyYAgNOnT0OhUPDBQ1Urz+thZW2NZRv+hmHzvpC6NobjxD14sXowjJv1BgBI7GqB5WRi9LQFGN5LeWP+77//0KVLF/j6+qqVxxhDtCxTrftp9OjRfItT/qUYVDihCKYdRsK0g3IUEwcU6lrSRDXzcUnLHpS0NEJp14oqqhwPa2WwExqjHuy4WugXOpf8rUZTRgzAhUx7XE8SIC07DwYS5e3+WaYubD5dolaWkAPquKp3vTLGMHzzNeDhK3hYKwPD/NdaEx0dHaxYsQIrVqwocp+qQkPPCSEfNIVCgU2bNmHo0KFqzf1Pnz7FokWLEBQUhPDwcBw+fBiff/452rZtW6i5/8CBAwDUZ3U/fPgw7t69i88//1xtVvdatWqhS5cuGDVqFK5du4ZLly5h3LhxGDRokFqXUVUp7+vRtu9Q5XDox1eR8yoc8f/+BJGBGfQ8lEv66Fg4QurWBD/N+7bE67H7egRaLTmNT9dfReslp7H7unriq6bWDkCZlwIog4EZXb345/zr+fZRKdjd9C5sjaXwdjd/q/WiVMFOeEIGsvPezEBtayzlW32AwiuM13cwhpuFPrJyFTgWosydSkjLxvarLwAo50hU+bSFc6G67boWgTMPXwEAfj//rNC1ft9Qyw4h5IN28uRJREREqI3iAQCxWIyTJ0+qda3069cPc+bMUduvYNdKaWZ13759O8aNG4eOHTtCIBCgX79+WLVqVcWeaCmV5/U4ExqH26ZtYdgkAgnHV0ORlQ5dh9qwHrAQQh0xH5j0/3Ypci9uKPZ6RMsy1ZJxNS2yWVRrx/4x3sjIUfAtKyZ6OoW6lgCU2N1UFayNJDDSFSElKw/PXqWjlq0RACBPrkBkknJ9rsV96sLXy0qtvhzHoU8je/x44hEO3orE/5o4YPXpJ0jLzkNdeyP8/lkTLDsWikO3oxASJQNjjB91Fy3LxMwDd/mymIZr/b7hGGOladHTaikpKTA2NoZMJoORkVFVV4cQ8p7JP4z3fb0ZlKdoWSa2X3mOtWefQs4AD2sDPIlLg4K9aYFo62GJbVee49czT2GuL8alGR2gqyMsssytgeGYd+heoe07R7WEt7s5//zjVRcQEqWc8Vf1XpoSjaNlmYW6ljRtqw76r7uM6+FJ+GVQQ/RqaA8AuPMyGT3XXIKRrgi35nWGsGDTFIAXiRlos+wMOA7YPdobg/+8glw5w7aRLeBT0wKvUrPReulp5OQpsHt0S7RwU17Hv4NeYMreO4XKK3itq4PS3r+pZYcQQt5BwWG8+ReH/BAVHA7dwMEYe79qhYT07EKBxKROHjh4KwqRyZnYG/QSQ1pqHqIcm5KFX04WXu27YFcTYwwxKdkAgAU9asOvrk2ZcmRKyr+pKh7WhrgenqSWtxP4NAEA0NzVXGOgAwCOZnpo5mKK6+FJGLbxKnLlDG1qWsCnpgUAwNJQgv81ccCOqxFYd+4pWriZQ5aZi1WluNbvG8rZIYSQtxQty+QDHeBN10q0LLNK61VVNA2HDomUISE9W2PeikgowKg2rgCA9eefIU+uKFTe2dA4DN90HQnpObA2kkB1W+eAQl1NEYkZiE/LhlgowKDmTtUycHkbmkZkXXmmDHZauplpPEbF0VQZoGTkKq9tIycTtddHt3GDgAPOhL7CjqvP8dVfNxCRlAljqUgt16m6dOu9LWrZIYSQtxT2qvCkcaUdxaONNCUIywtMolfQgGaO+OXUY0QkZuBoSAx6NFAmJRccQi7VEWLPl944//gV5h68B3crg0ItaNfDkwAA9RyMi+0Se9/wI7JeBzt5cgV/ri3diu5WipZl4mBwpNq2X08/xSf5AkEXC33UsTPG3UgZZh1QLtopEnDY/kVLmBuIq2W33tuglh1CCHlLadm5hba9783978LVQr/QtpKuh55YhKGtXAAAq08/xuUn8bj5PBEzCgwhz86TQywSoEsd5TD3p6/SkJyRo1ZW0HPl0gZNnat+XpfypAp2XiRmIj07D/eiUpCWnQcjXRGfsKyJ5uBTGYyrRMsyERIlK7SPuYH4nUaRVTcU7BBCyFs6GhKr9lxT18qHxMZIF6Z6Ovzz0nZ/fO7tApGQw6PYNHz651X0XRuIgkNnVBPmWRpKUMPKAIwBV8PUV9NWtXY0dSm+a+d9Y6YvhoWBBADwOC6N78IqLl8HeDODc34Fg8+w+PRC15q9vtbahIIdQgh5C4npOfj3bjQAYPjrlgk7E90POjn5eUIGkjJyIRIAm4Y1w8UZvqW6Htl5csjlxQ8Mzn+TVuWpqG76AJCUnoMncWkAgCZa1rIDAJ42yon9HsWmljpfRzWDs/D1kPLilrTITxtbJynYIYSQ11Sz85YmwXhf0Evk5ClQ194I3/p5QiTgEJmchYgE5V/ELi4u4Diu0GPs2LEAgKysLIwdOxbm5uYwMDBAv379EBsbW9xb8oorW9NrdevWLXXZAPDVV1+B4zisXLmyxPddsuTNbLwXnsQDAJo4mxWa96U4mhbMBFBkgqwqT+XKszctO0HPla067pb6MNMXl+p93yeqrqz7USmlytdRGdjMCRdn+GLnqJYag8/SBETagBKUCSEEr4dM778LxkoeQq5QMOy4ppxRdnALZxhIRGjsZIpr4Ym48OQVBps74/r165DL38x4GxISgo8++ohfFHHSpEn4999/sXfvXhgbG2PcuHHo27cvLl26VGJdFyxYgOXLlyMhIQFjxoyBZ4MmGNT7Y3To2hP//vsvmjVrhsjISPzyyy8wNDTEvHnzSl32gQMHcOXKlSJnc164cCFGjRrFPzc0fDOL78XHyhl327we2lxaRS2YWXAyQJUWrsqb/MOYFCRn5MBET4wbr4Odps7a1YWl4vk62Am4E12qfJ38ymtJi/cZtewQQj54/JDp1zfbkoaQBz5LQFh8OgwkIvR8PXpINXfJxcfK1g1LS0vY2Njwj4CAALi7u6Ndu3aQyWTYsGEDfvrpJ3To0AFNmjTBpk2bcPnyZVy5cqXYukZGRmLOnDnYs2cPdHV18TRZjtHf/wGRiS2mXMxFalYubt68iV9++QX/+9//4Ofnhy1btpS67G+++Qbbt2+Hjo4OUjJzC7V0GRoaqp2Xvr4yKTlPrsDlJwmvr4VlCVdcXVGtCw0cTTUmyGrK27kR/jo52UX7urAAwOP18PP4NOU8Qi3cis/XKSttSkbWhIIdQsgHrzSjVvLbfvU5AKBPI3vov15gUdWacelJPOQFCsvJycG2bdswYsQIcByHoKAg5ObmolOnTvw+Xl5ecHJyQmBgYJH1VCgUGDJkCKZOnYo6depArmA4HPwSaffOwqD+R2DgkJyaAblcjlGjRqFRo0ZYvnw5atSoUeay03Pk+OXU40LrUC1ZsgTm5uZ82Xl5eQCA2y9lSM3Og7FUB/XsjYt8n6KU1N1SkCpfJfBpArJy5bjzUjmiSNuSk1VqWhmoPS9NFxZ5g7qxCCEfvJJWkM7v7stkfmHFwS3f3JDrO5jwaxjdeZmMRk5vWhgOHjyI5ORkDBs2DAAQExMDsVgMExMTtbLNLCwR9DAM0bJMjX9hL126FCKRCOPHjwcA5CkYcuPCochKg35d5YrhUpeGyH50EefOncPly5cxc+ZMREdHw9raGjExMUVeg/xlR8sykZyeA8MCLV3DRn2N9q1bwMzMTK3sn376iW/Ral3j7VscyjKDsbebBbZdicCVZwkIiZQhR66AhYEYLubalVirYqirA3sTKSKTla1sNSwLD/MnRaNghxDywbM1lqJ1DQtcfJ1gy3Gah5AXXArh9otkeNko8yaEAg6t3C1w7F4MLj6OVwt2NmzYgK5duxa7qvnu6xG4H52CZ7ejcHnJ6UI5Q0FBQfjll19w8+ZNfsFGkYBDdnQopG5NIDJU/qWv794MuU8uo379+qhfvz7EYjG+/PJL1K5bH5HJmRoDqYJla0oYljMGv0Ff8Gsj5S/b398fF/h8nbJ1Yb2tFq9bdh7GpOLEA2XydRNnU/7aaCMDyZuJEodvvv7BL01SFtSNRQghABT5JhsZ1NSx0E1E01IIBfN62ngou7IuvG7lAIDnz5/j5MmT+OKLL/htNjY2yMnJQXJyslrZ8vRkCPVNNeYMXbhwAXFxcXBycoJIJIJIJMLLFxHIS4xEdtQjfr/RXRqpld2iRQvk5eXh3pNwHHmSqdYlVVTZbT1tIE+JQ9KZDXi5Vrn6OQfA2Vw9SOLLDn2MWy+U7+dTo2zJyW/LwkDCd+1sv6I8H21NTgaU35FHsWn88w99aZKyomCHvJfKMkSYkJIwxvAgOoV/npKVV2if0uT1tKmhbNW4GZGEtGxlGZs2bYKVlRW6d+/O79ekSRPo6Ojg1KlTfNnZ8S8hT3kFiZ2XxrKHDBmCO3fuIDg4mH9I9fTB6eii1hcr+GHavT9qo1b22cvXAHCQpyVCYuel8SZZsOxzl69DaGAOo+Z9YT1gofIaAThwK0rt/IODgyEQCBCergO5gsHVQh+OZpXXjaTKW1Fda21NTgY0D88vLq+MqKNgR8tERkbis88+g7m5OaRSKerVq4cbN27wr6elpWHcuHFwcHCAVCpF7dq1sW7dumLLvHfvHvr168fPsVFw7g0AkMvlmDt3LlxdXSGVSuHu7o5FixaBFZyasxzsuhaBVv6nCyVOfsgo+Hs3canZSMp4s/SDKi8iv9JMvuZkrgdncz3kKRiuPkuAQqHApk2bMHToULxKfzOyydjYGCNHjsTkyZNx5swZ3LwRhIQjKyGx84LE3osvb0jXVjhw4AAAwNzcHHXr1uUftWvXRlZWFsS2nujl2wIf1bZGduQDzF66Br1798b48eMxe/ZszJgyGQKpoVrZcsbQsnH9Iss+EycBBEJY21hj7/R+mNbFE9mRDzD3h2VYvuM49p65gV/Xb8KkSZPw2WefIThOee0qq1VHJX+SrljIoY5d2ROj3xcfyuR/FYVydrRIUlISWrduDV9fXxw9ehSWlpZ4/PgxTE3f/LUzefJknD59Gtu2bYOLiwv+++8/jBkzBnZ2dujZs6fGcjMyMuDm5ob+/ftj0qRJGvdZunQp1q5diy1btqBOnTq4ceMGhg8fDmNjYz6ZsjwU7EpQ/ZXa1sNSa4dMliT/goklzQ9DNFO16ogEHPIUDFEagh1bYylGtHbFnxfDABQ9+ZpPDQs8T4jAnhsvEPfwGiIiImDdtAtaLzmt9hn9/PPPEAgE6NevH1LSMyF2aQTzj8aolfX08SPcePwSLTXk2Zw8eRJMIYfEzgPNXMxgpi/G4VM6OH/sEHRSo5Ceno4lS5aAEwggcWkCc79xasdHPHuisewXiRnYGqgcbda5tg1a1bBAqxoWCLrhiG2n/sT0ETsAeS5ExtboN3A4vps/E5/8GaQ89zLOr/Ou8gf3OXKGA7deau13XzU8f9b+EMgZ09rJ/yoKxyriT+/3TEpKCoyNjSGTyWBkVLpJmqqjGTNm4NKlS7hw4UKR+9StWxcDBw7E3Llz+W1NmjRB165d8f3335f4Hi4uLpg4cSImTpyotv3jjz+GtbU1NmzYwG/r168fpFIptm3bVvaTKcLlp/H4dP3VQtt3jmrJJ05WlsjISEyfPh1Hjx5FRkYGatSogU2bNqFp06YAUGSi5LJlyzB16lSNr8nlcixYsADbtm1DTEwM7OzsMGzYMMyZM4cvjzGG+fPnY/369cq8DGtPmHUeAx0zewDKm/DFGb70I1gG6849xZKjD9HK3RyXnyaA44DQRV0hFqk3fu+8pgwsGzoaY+1nTTRe47kHQ/DXFWWwIOCAth6WOBv6Sm2f/J/R8Xsx+PKvIOjqCLDnS2+kZ8vxKDYF8w/f5/fXFMRm5shRb8Fx5CkYLkzzhb2JFO1XnEVEYgaW9auPAc0cER6fji6/nEdWrgIch0JrIGkqe9LuYBy4FYnWNcyxbWQL/nsXkZCOtsvPFjo+f7kLetbGsFauJV/wchAty+QDSJUP4bsfLcvU6sn/yqq092/qxtIihw8fRtOmTdG/f39YWVmhUaNGWL9+vdo+rVq1wuHDhxEZGQnGGM6cOYNHjx6hc+fO7/TerVq1wqlTp/DokTJR8vbt27h48SK6du36TuUW9DarKlcEVSuajo4Ojh49ivv37+PHH39Ua0WLjo5We2zcuBEcx6Ffv35FlqtqIVuzZg0ePHiApUuXYtmyZVi9ejW/z7Jly7Bq1SqsW7cOv/99DJyOLuL2zAPLU64ATf34Zadq2Wnlbg6JSADGgNiUrEL7RSQqr2t9BxONN5poWSY/Bw+gbHksGOgAbz6jPLkCy449BAB84eOG+g4m8HY3R+c6NsgfKmvKswl+kYw8BYONkS4cTKUQCDh82kIZsGy7+hwKBcOM/XeQlatA6xrmuDRdOYfND73rqNVFwYCZ++8iWpaJc6FxOHArEgAwo0sttYD9pYbWLkA9gFr0z4NK60ot69xI2kLbJ/+rKNSNpUWePXuGtWvXYvLkyZg1axauX7+O8ePHQywWY+jQoQCA1atXY/To0XBwcIBIJIJAIMD69evRtm3bd3rvGTNmICUlBV5eXhAKhZDL5fjhhx8wePDg8jg1nkigHp8LihgiXNGWLl0KR0dHbNq0id/m6qr+F62NjY3a80OHDsHX1xdubm5Flnv58mX06tWLT2Z1cXHBzp07ce3aNQDKVp2VK1dizpw56NWrF6JlmbD8eDIiVn+GjEeB0K/djvrx38LD6FQAQG07I9iZSBEWn47I5MxCybYvXgc7TkUk4Wq6AQMoNIcPALxKy4L/0Yd4+iodZvpifNnuzfeiuGRU1Xc96LlyxuAmLm+GW/dv4oAf/wvFnZcyjNp6A1eeJUKqI4R/n/qwM9GDnYkemIZVqBQM6LXmIuJSc/ht96NlqOfwJgdG05IOBRWsY0UqaokJ+u4TTahlR4soFAo0btwYixcvRqNGjTB69GiMGjVKLQF59erVuHLlCg4fPoygoCD8+OOPGDt2LE6ePPlO771nzx5s374dO3bswM2bN7FlyxasWLECW7ZsedfTUnM1LEHt+ei2blXSR1+aVrT8YmNj8e+//2LkyJHFlltSC1lYWBhiYmL4mXdtjaXo19IDEjtPZEcpWwioH79ssvPkePpKOaTXy8YIdia6AKAxb+dFknKbg6nmG2pRSaQzunnxSyGojN8ZjA2v839a1zCHoa5OseUIOKjdyFWLQTbNt8K3uYEEtV+vl3TqYRwAoGMtKzjlm2hPU9kA1AIdoHBLUsElHQQAChZTmcHGh7KAJSkf1LKjRWxtbVG7dm21bbVq1cK+ffsAAJmZmZg1axYOHDjAtxzUr18fwcHBWLFihdrU9WU1depUzJgxA4MGDQIA1KtXD8+fP4e/vz/fqlQerjxTBjsSkQDZeQpEywp3NVSG0rSi5bdlyxYYGhqib9++xZZbUguZagZca2tr/hhTfTGEeiaQpydDTyzE/5o4luOZar+ncenIUzAY6Ypga6wLu9c3S03fLVXLjqOZ5htqUUmkA5s5oWcDO4THZ0CuUOCzDdfUjvv3TjRmdXuTKFywHADoVMuaf12uYLgZoQx2muVbHiFalok7kTK1so/cjVabSFBTHQc2d8COqy/UjtPUSlNwwcjzj15VacLsh7CAJSkfFOxokdatWyM0NFRt26NHj+Ds7AwAyM3NRW5uLgQFuoKEQiEUCsU7vXdGRkaFlFvQlWfKpvveDe2x+8YLtUm2KpNCoUDTpk2xePFiAECjRo0QEhKCdevWaQx2Nm7ciMGDB0NXV7fYcvO3kNWpUwfBwcGYOHEi7OzsigwaVa0SAJCRI8fDmBStHoJb3lT5Ol62RuA4DrYmyhtmweHn6dl5SExXtn4UN5dMUTdg1VIIl5/GFzpGwVBkYLHtynP8euYpQiJlkCsYhAIOj2JTkZqVBz2xEF42b1YdD4tPL5SEXFzZqjoCwK5rL0rVJZR/SYfqEGyUZYkJ8uGibiwtMmnSJFy5cgWLFy/GkydPsGPHDvzxxx8YO3YsAMDIyAjt2rXD1KlTcfbsWYSFhWHz5s3YunUr+vTpw5fz+eefY+bMmfzznJwcfrKxnJwcREZGIjg4GE+ePOH36dGjB3744Qf8+++/CA8Px4EDB/DTTz+plfuuXqVm40lcGjgOGOKtDOCexqUhT16+AVVpFNWKFhFReM6fCxcuIDQ0VG0G3aLkbyGrV68ehgwZgkmTJsHf3x/Amzyg2NhY/phnr9Ihz0iG1Ej5F37Q86S3Pq/qqqT5o4YNGwaO49QeXbp0KbbM1NRUTJw4ESO7NkfEj30R+PPXuH79OuzzdWPt378fnTt3hrm5OQx0dZAT+wwmejowytflpElxSaRlmS/F1liKbzrUhKmeDqJkWTgbquyauvH6M27sZAqR8M3PeFnLVtXxXbqEKGGWvA8o2NEizZo1w4EDB7Bz507UrVsXixYtwsqVK9WShHft2oVmzZph8ODBqF27NpYsWYIffvgBX331Fb9PREQEoqOj+edRUVFo1KgRGjVqhOjoaKxYsQKNGjVSu3mvXr0a//vf/zBmzBjUqlULU6ZMwZdffolFixaV2/mp8nW8bIxQ29YIemIhcuQKhCdU/uiLklrR8tuwYQOaNGmCBg0alFhuSS1krq6usLGx4WfHzcqV43lMPLKjQtG2TWsAb3I5tEVpRr4BQJcuXdRGv+3cubPYcr/44gucOHECzYfPg+2INWjRxhedOnWCTlYyAGWwk56eDh8fHyxdupQ/zrGIfJ3SKmtgoasjRP+myq7J7VeVwfSN8NfJyc7q1+BdgpayrjpOyHuFESaTyRgAJpPJqroq1UJUcga79OQVi0rOqOqqqJl94A5znh7AFhwOYYwx1nPNReY8PYD9eyeqyGNevnzJBg8ezMzMzJiuri6rW7cuu379Ov/60KFDGZQDZfiHn59fsfWYP39+oWNsbW2Znp4e27ZtG2OMsdGjRzM3NzcmkUgYAFa/fn324MGDQmV16NCBrV69Wq0+9vb2bPPmzaxXr17MwMCAAWAWFhZ8vZcsWcJ0dHQK1aFZ63bMeXoA8158stT19vT0VNtHVW9dXV1mYWHBevbsqbHeJSnP79D06dOZj49PsfsM+OQz1qZT11K/X0ZGBhMKhSwgIIA1WXSCOU8PYLcikljjxo3Z1xOnMufpAazuvGP8/mFhYcrPedgq9vW2G+90PipRyRns8pP4UtX52as05jw9gLnMCGAvEtNZK/9TzHl6ALvw6NU7l03I+6y0929q2SFqdl+PQKsl1XMpBlW+jmqKeE9r5SKAoTGpGvevqBYBAKhTpw62bt0KLy8vSCQSGBgYqLWiNWnSBJs2bcK8efMgkUhgZ2eHzp07Qy6Xq5Xz6PET3HoUwY96Wb16NT7++GOMHDkSAQEBMDIywtixY7F582a+3tOmTUOtWrUgFosh0hFD4lAbPpN+w4F9eyEUcIiSZWlc7kBV7/znevHiRbXXVfV+8OABjh8/DsaYxnoXZ/f1CLQux+9QSSPfdl+PwJG70bh44Twc7Gxh5+yOr7/+GgkJCUWWmZeXB7lcjiyFAPFp2eA4wNPaEFKpFCE3lZNWpmbnISUrt9Cx5bX2U1m6f1wt9OFTwwKMAT+feIzI5EwIOKChk8k7l03Ih4ASlAkvWpaJGfvv8gmO1Wkphvz5Oi1clbkpHtbKxMxHsZqDndLMhQMAEomk0Jw4JRGJRBgyZAiGDBmi8fXRo0cDANq2bYtZs2bhzp07aNCgAcLDw+Hu7g5AeZPWGfwbTjHgzJLT/Cy2JiYm8Pb2LnImbI7j0KhRI7i6uqLjN8vx44lHaN7YAfbWlqhrZ4TbL2W4EZ4I+4b2Gutd3Lmq6g0o5/j5/vvvC9W7OKrlPBTl+B0qbuRbvfY9MH3fXei6NobUoxVEJtZQJMfg1Jm/EdS1KwIDAyEUCguVaWhoCG9vb3z//ffIa/olajrbY9+enQgMDESNGjVgqqeDpIxcRCVnwshGPT/nXbux3tbgFk64+CQe+26+BKCcE8hAQj/hhJQGtewQnqaRHEXNSFrZC0/mz9cx0RMDADxtig92SjsXztmzZ2FlZQVPT88SWwRUHj9+DDs7O7i5uWHw4MEaE5NV0tPTsWnTJri6usLRUZl7UVRQEC3LLFO9p/dtgcj1X+LatqVISEhAE2dlIHijiLydd613SSpiVtuC80f1GDgE3ft/hnlLfsagP5StMPq120GvZguILV2gW7MlFq/dhuvXr+Ps2bNFlvvXX38hPTsPkb8NxbmZfli1ahU++eQTCAQC2L0ekaVprp3KXNU7v061rWFpKOGf17J5f5e2IaSyUbBDeLoizV+HR3EpaoHN6lOPK33VcdX8Oi3d3swp4vm6ZSc8IQNZuYW7WVQtAjVr1sTx48fx9ddfY/z48WoTHXbp0gVbt27FqVOnsHTpUpw7dw5du3YtttumRYsW2Lx5M44dO4a1a9ciLCwMbdq0QWqqetD122+/wcDAAAYGBjh69ChOnDgBsVgZqBUXFJSl3s3H/gzTdsPw/N4NdO3aFY0dlTfAGxpGZJVHvUviaqGPgkuCFZwMr6zyj3xTdZGdiRUj8uUL5GgYiSfkOLRuXBsWFhZqIwYLcnd3x8ezfofjpL/x3c5zuHbtGnJzc+Hm5sa3QkUlK+faYfn+Cihq9uSKpiMUoJ79mykF/g56Wa26mQmpzijYIbwzGtbwAYD5h+7j0/VX0cr/NBot/A8/nnhUaNXxd23hKWlocf58na+++gocx2H7hrUwlupArmB49iq9UJkKhQJ2dnbYsWMHvL29sWHDBvTs2ZOfUToxMRGXLl3C1KlT0bx5c4wfPx7NmjUrsUWga9eu6N+/P+rXrw8/Pz8cOXIEycnJ2LNnj9p+gwcPxq1bt3Du3Dl4eHhgwIAByMpS3jxdLfQ1zD6rDApKMxP2oEGD0KNHD7wSWUPPwxubd/6N69evI+vFXQDAw5iUQvkm5VHv4kRGRmLq2FGIXv0pIn7si6gNY5Ed/RifNncq1IWl+gxXrlxZYrmWlpbYtm0bdHV1MbTXR8iMDEVuYiRERlZQZKXC9dFuRK7/EhE/9sXL34bD49leJEWGISEhAba2tsWW/SA6FQKxLlrWrYGkpCQcP34cvXr1Uht+DoCfXwcc+BmWK1u0LJMfeg4oM8zL4/8eIR8CCnYIACBXrsCu68oZVL/vXRc7R7XE3q9aqu3DACRlFE7YfNduipISieNSs/h8nVd3L+DKlSuws7MDx3F8646mrixjY2M8f/4c8+fPx82bN9GgQQP8888/CA8PB6AcUh8VFYUVK1YgJCQEmzdvRmBgIMRicbEtAgWZmJjAw8Oj0DHGxsaoWbMm2rZti7///hsPHz7EgQMHACgTSD1eJ1irfNrCWTnnSSnn8IlLzUZ6jhxCAQefxnVhYWGBhKgIOJvrgTHgVkRyude7KPk/w3pfLIXtyN/g3O1LCHQNIC/QgnXgwAH+MyzJ7t27cfv2beTl5aHnoKEQGlshducspAUfg0Hj7shLTUBeyiu0b9EIFh9Phkm7obh14T+0bt0aNWrUgJ+fH19Wx44dsWbNGv75v0eO4nbgGeQmxyDuwTX4+vrCy8sLw4cP57uxnr2MQXBwMC7duA0A0M+MxYOQu/xM1pXpQ134kpDyQMEOAQCcvB+LV6nZsDCQYEBTR3i7myO34F3qtYLdFO+6Hk7+ROLmzZvD1dUVnTt35hNij4UobywO4kzMmDIJ27dvh46OMmnUw+b1iCwNwY5CoYC1tTWGDx+O2rVrY926dRAIBHyXTN26dbFv3z706NED7u7u6NChAyZPnoycnBxYWVmVuv5paWl4+vRpsa0IjDEwxpCdnc0/j09TthbUs1d2PV1+Go88uaLUc/g8jVPOnOxkpoe4mCi+JUM194pqLpaCVK1oZmZmuHHjBrZs2aLWirZgwQJ4eXlBX18f1tbWyMrKwv3794u9BkuXLoW1tTUSUzNw8/dpiN4wBjmXt0CRlYaLT960GEZGRuKbb76Bh4cHoqKicO7cuWLL/emnn/Dll1/i4MGDuHv9EjIeXwWT50LXrTEM6vhCauWKLdu2QyDPQcqp35FwZCXSM7KQkpKCM2fOQCJ5k+Py9OlTxMe/mb046HEkXh1fi+g/v8KUcaPh4+OD48ePQ0dHhw92bl44iUaNGmHC8IEAgCe7fkCjRo3UWtkqS1kmDCSEqKNghwB4M1nZwGYOEL/O3SlyUcOuXmpdMO+6Hk5xCbm7r0dg3qF7YEyBG5sXoePAL1CnTh3+WL5lp8Dw85ycHCQnJyMuLo6fUXrXrl3Izs6GhYUFAGWQMnXqVFy5cgXh4eE4deoU/P39IRAI0K1bN76sgi0CU6ZMwblz5xAeHo7Lly+jT58+EAqF+OSTTwAoc4X8/f0RFBSEiIgIXL58Gf3794dUKuXLjZJlISE9ByIBh43DmsNUTwdPX6Vjb9DLEmfCVtX7+JkLyJPFQvrqHnr16sW3ZKjWSlo2fnChegcEBKBFixZISkpCjRo1YGxiiq9nLEKuSMrXW0dHB/PmzcPmzZvRsGFD6Ojo4JdffsGrV5q7OQFla82DBw9w8eIlgDEYWdmjRxc/SPQN8SIxE88T0qFQKDBkyBD4+fnh0aNHGkdJFfwMg4KC0KlTJ3z88cd4cC8E3x0Mhn6d9oBCwU+Y52ZjhuPHj2PjiWA4Tz0It65fwMzMDPb26qPRAm8/QOch4xAty8Tu6xHYFG0L+y//hNOUg1h5+CrWrFkDY2NlToyqq0pcuwMYY1h96hGcpwdg8u5gMMawYMGCYuteEWjhS0LeHo1bJAiLT8fFJ/HgOGBQvllTi1vU0NVcD6P/ugkTqQ76v+PCk0UNLc7IA36JUHZ1pFz5GxAIcUGnmVqOgmr4ecGWnfj4eCgUCixfvhxbtmzBwoUL4erqig4dOiA5ORmAcmbiO3fuYMuWLUhOToaNjQ0SExMxbty4YlsEXr58iU8++QQJCQmwtLSEj48Prly5AktLSwCArq4uLly4gJUrVyIpKQnW1tZo27YtLl++zLcY3X2ZzNff0lCCbzrUxMKA+/j5xCOcndoeBw4cwMyZM/l655/DR1Xvi+s3IiM1BekWVujfqzsWLVoEiUTCr4KdGPMCsXGv1Oo9aNAgZGQouz2cajeG4cDR2BBlg027wzG1jRUuXLiAoKAgtXpfunQJTZs2xZ07d9CxY0eNn+HTp0+hUCjg2qIXJA7N8LFtBjb8/B3qD5yCWCMbnH8cj5dndkAul+PYsWP477//SpxROj4+HnK5XG3RU3sTKYR6JhCnxeDiDF+1G33nOtaYueMi7h3ZhK9HDlcra/f1CLXRbwUVHB6vatmJkWVBrmB4kaj8zhW1AGhlqQ5rURHyPqJgh2DnNWWrTnsPy0LDaov6cW3vaQ2pjhDJmbl4HJfGDwN/G0UtqvnHH79D4fcdsmOeICXoMGyH/gIFoJajoAp2XiZlIj07D/oF5h1p3bo1pkyZwj+fNm0a33UilUpx/PhxAEBKSgo++ugj1KlTBytWrFArI/D2A4TFp/MrR+/atavY87Gzs8ORI0eK3efOS+XK1A0clS0Jg1s6YdPlMLxIzMSCw/cw6aOOuHv3rsZjVfUesuEqLjyOx9J+9dSm9ne3NICJng7w1UZ4dK2jVu/atWvDz88PT8Ke48iJ0xDevgfDRt1g2LALVlx4hYs796ndQHNycrBq1SoYGxsXG5zI5XJYWdsg5kUYsi7/gys1XNCqVSs8uxYA9GyJA/+dx+XffoG7uzumTZum1jpXFk/ilEGtgURU6EYvVmQj7fAP0DF3Qp0eI/ntBYf5a6x/gRW+rQx1IRRwyFMwvErNxouk16udV9EcO/nRwpeElB11Y33gwuPTsePqcwDA4BaF13UCNM/GKhYJ0NRF2YKgGhb+tjQl5Hp5eeH5c2UQlv3iHhTpMkSuHY7ny3qijac1nj9/jm+//RaN6njwc488jnuz+reFhQWEQqHagpmAcgHNgpPqpaamokuXLjA0NMSBAwf4fCAA2HUtokKG2d+NVAY79exNAAASkRCt3ZXda3tuvCzVe6lGoLlbqic6CwQcbI2U3TBzD91TK0vVimZq6wTrAQth2Kgbkk79gbS7p9SSXQMCAmBgYABdXV38/PPPOHHiBN/9V5RXcXEQmdrB79tVGDd2DM6fP4+EyDAAQOClS4iLi0NgYCC+/fZbiEQiyOVyHDp0CC4uLhrL0/QZPolLgzwjGbZFfIY2Fqaw6jsbJx6++U4+jSuc2FtQwdwXoYCDzetrGJmciYjE161h5lUf7BBCyo6CnQ/Y7usR8F1xFmnZyjllXqVll+l41bIN7xrsFEzIjZZlYuuxK8iTmoMDYFjPF7YjVsNhxGos33EMwcHBsLOzU+atHD+uMW9HLBajSZMm/IKZgLIF6dSpU/D29ua3paSkoHPnzhCLxTh8+DB0dXX5CRP/vROlnFFadTwDZu6/i2hZZqFJFcsyySJjjG/Zqe9gzB+/58aLN3XN916aZObI+SUh3AoEO9GyTDzMdy3yTw+gGtb+2TczILZ2h2HDLjBo4IfU4CNq8+H4+voiODgYly9fRpcuXTBgwADExcWhKBzHQSjWhWm7ofi0WzuMHj0adevWBVPIYSzVgcLUCUbGpjh58iSCg4MRHBwMoVAIX19fvnWtoIKfIWMModEpyAq/jVatNH+GAf8cBicSI+h5EmJkyuHyR+9FFyqbA/h8tKJyX1R5Oy+TMhD9uqzq0LJDCCk76sb6QKma9vP/wTvnQAjae5Z+Wn/VBH9XwxKhUDAICmYzl9KkSZPQqlUrLF68GPpePvh+8z9IOLYHZn7j0K2eLeZ83KFQN5qOjg5sbGzg6ekJj0f3cfFJPGZ80R9xoz7DuHHjAACTJ0/G0KFD0bRpUzRv3hwrV65Eeno6hg9X5nOobpIZGRnYtm0bUlJS8Ne5+/jhyENwUiNwgsIJtAoGjNh8HaExqVAw5Q2zTyN7HLgVyT9XLftQlBeJmZBl5kIsFPDdcJqGFSsYcD0sET01LPvwLF7ZimWqpwMzffUJ/8Li01GwIYPvprG1Ra1atfDn+TD+NR1zR2SEXoKvpxV/ffX19VGjRg3UqFEDLVu2RM2aNbFhwwbMnDlT4zlZWFriVdwryAL3oOYgF+zYsQP379+Hnp4eWtcwx67z4UiRJaFz585v6iSX48yZM/Dz8+OnA+jYsSP69Omj8TOsUachwg79ApabhYljlMtaFPwMpchBHRMF7ryU4eidSJgZ6mL7FWWrFscBjL0JbkrKfVHm7SThRngS5AoGsUgAq3wzGBNC3h8U7Hygipuzo7TBTj17E0h1hEhMz3mnvJ1mzZrhwIEDmDp9BkJDF0BobA3TDqNgUMcXx0KiMefjWvB2Ny/yeM/Xw8/jIp+rJRIPHDgQr169wrx58xATE4M69epj2Z+7oNBVDvW+efMmrl5VLjdQo0YNtTLtv9oAkbE1NHkQrd5qsu9mpNrzktaCuhOZDACoZWtYaORbwc/ku3/uw1RfDKGAg6uFPl/m0yK6sIoqS9Vq07p1a1wLvodUy0RIRALs+bIlxk/ci6tGVjj36BXuR6Wgtl3hZQgUCgU/bF4ThzrNkSS/Dfnji2jXcjdcXV3Rpk0bZGRkoE1NS/xT1xdNWrXFzwMb8sc0bNgQ3q3bYMwcfz6vqGAyeP7PMDo6BpyFC+qPWgZnB2XienGf4SrpNqSLlV2tY9q7Y4i3c6HgprjvuipJWdVy6WAqfeuAnhBStagb6wOlaVr/kubsKDjLcZNGDeAK5Rw4V54lYP/+/ejcuTPMzc3BcRyCg4NLrMf+/fvRtGlTfPbZZwgPC4PI3AHG3gNg2LALAEDOlAnJDx48QM+ePWFsbAx9fX1YWlqib9++AN4kKdf4Zgu+nDRdrfxx48bh+fPn2HrxMZI/WoAfruehlf9pzD14F6eSzeEyPQDOGh75A5383R0f1yt+Rl5lnYuf6O3u6y6seg5vpv4vOKxYwAE2RhIkpOdgyIZrhXKGnr1Stuy4WeoXKv9NWW+22ZtIYWWoi2/GT8C94BuQBe5BD1cB7l04ihvH9qJ1z8HIUzCM/ysQQ76aiICT5/D8+XMEBQVhxIgRiIyMRP/+/fny8g/H3309AtGOHZCXFAVhTR8s3fEf5s6di8DAQIwdOxY+NSwglBrhmdwMzjU8UbduXdStWxemlja4p1MTM08l8OeWf3h4wc9ww7lQ2H7+Exo3aca/1r59e34OI9Vj7ZnHcJ4egCShCXLkDF42hvi2s2eZVwK3M1Z2Y6lywagLi5D3F7XsfKBsjaVoW9MS5x4phyaXNGeHaoZcX19fHD16FJaWlnj8+DEuxwlxPyQLV54loBVLh4+PDwYMGIBRo0aVqh5mZmaYPXs2vLy8sO9WNH7auAsJR1ZCqGcMqVsT5c0/NQY+Hdti5MiR+O6772BkZIR79+5BV1d5M1LlvyRn5qJ1vtXDVQqOxmEA/rpSukRjIcdh/xhvZOQo+EDwSEh0iQmvYlHRLQB8vs7r5GSVgiPfUjPz0Hnlef71/K1GxbXs5C/r5vNkTPv7Nl4kZWLz5XAYSGxh0Wc2Ui9sxZqxu/lh7b0GDkHbZWfw+FUGLp+5hp3b/wKXnQZLC3M0a9YMFy5cUBtBpWqBuR8lw/R9dyGx9YBln9lIPrcFkwbshLu7m9pweVcLfYTFp+PKs0R8VNsa/92PQVJGLozyLYQ6fd9dvqtJU3egKuioYa35nAHlZ73seIEJGWNTEZeaVeYRTKqWHZWqHnZOCHl7FOx8wGSZyqUfvm7njs9bORd7M8g/y7GKq6srzJ8nYnNIIK6GJeLX2Z9BIOD4/IvSaN++PQAgNiULu3dHwKhpL6SHnEb2y/swcG+KxX3r4pcl09CtWzcsW7aMP041u3K0LBPf/XOP364MCO6qdSNde5ZYYnCiMrqNGzZcDFObV6iBo6naPgXnHurdyA4Hb0VBnm+xyK+33cSPAxoU6n5SKBhCIgu37KjkH1Yclq87R0XVaqSaPbmoYEdVVvf6UqRk5WLm/rtYduwBdHVE0KvRHIsnDMUXbdz4faNlmcjOU4ATiWHVZzYAZaBXcC4b1b7bT9xASKQM/X8P5Lfr1WgOvRrNAQBbR7VU63r0qWGBsPh0bLkcjoA7UTgUHAWHrzcWqjPLF/wU7A588vqca1oV3V1aVO5TWbpnVQoGO1W1ACgh5N1RsPOBSsvO44c/D25ZeKHGgg4fPgw/Pz/0798f586dg729PcaMGYOhw0fyeTuP4lLhZVM436MkjDHMORiClMxc2Gc8QVxqNL77sj8G9PaFtaEEX/z7L6ZNmwY/Pz/cunULrq6umDlzJnr37l1E7hFw6UkCWtcwx/lH8fA/8qDQewoAoEBei5DjMNzHBcN9XIpNXNU099AUP0+Ex2fAQCLE1L/v4GFMKoZsuKZ8r3ytFOEJ6UjNzoNEJEBNq6IDFaDoPB57U12ExStbdjR1YxU0qJkj1p9/hmfx6cjOUwa4ujrqydfFJjXnuwYlTc4HFN8devFJ4QCuKAXfn2/ZKea6abpmb7ukQqGWHerGIuS9RTk7H6gb4YmQKxgczaRwKMWPuGp+lpo1a+L48eP4+uuvMX78eOzc/teb+Xaeln0IerQsE4sPBWHDF20QsaI3gtbPwJrVqzFhaD/YGksRFxeHtLQ0LFmyBF26dMF///2HPn36oG/fvjh37pzGJS0AYNrft9HK/zSm77uD5MxcGOmK1HJv/PvVK3Lq/dLkdhTcR/W8noMJVg1qpLZv/qHfqi6sOnZGEAmL/+9XMI9HZfnxR8jMlUMoAETCkhNmY1KyEJ6gvir8/EP31PJiNF3H/EPRAc2T83EcML2LZ7FLGETLMrH99VxO+cue2c3rTY4SoGEV+DdBiiwjF69SlQnSxQU75bmkgpGuCPriN0FhwQk3CSHvD2rZ+UBdeaZcJLKla9GjnPIrapbjdevW4bPFf+HC43hceZaIYa1dS10HVSuBXKGA7fBVaGavhzb6MZg8eTLc3NzQvn17KBQKAECvXr0wadIkAMpRPJcvX8a6deuwc2c7tW4lAae8catyWlTSsvNwYEwrPvdGdfOriKn349MLj1pStVK8mV/HpFRl5W9FevoqDXMOhuCf21HKMhVA++VnSxzqXpqRdwWXBgEAZ3N9fmK9osphDGjoaIqLM3yLvI5FdS3VtzdRO+78o1dqwdSi3nXedGG9Uo6AszPWhYGk+J+t8lpSgeM42JlIKUGZEC1Awc4HSjWcVjUxYEk0zXJcq1Yt7Nu3jy/j4pNXiEwqehRSfvlbCThOAB1TO9zJBNZO6IsHDx7A398f7du3h4WFBUQikcb3vnjxIoDCN7ewV+n49M+ravsrGJCRoyg0hL0ipt4vbuj33dfDzuvZF87XKUr+1qancWnYdDmcf600Q91L27Wjuo43wpMweU8wwuLTcTb0FXy9lOt5vY471ajKKe46Fvf++Y8b2MwJrWtYwG/leaRny9XykR7Hvs5RKqHrT6W8PldVsKOnI0BGbh6MoVPyQYSQaoe6sT5A+fN1WhYzf01+BWc5BoBHjx7B2dkZoTEpr8uVw2fZGfxzO1JTEWqKSyTNP6eLWCxGs2bNinxvlfzdSq6Wmldrf5u8jbehqfvJWE8HBhIRQiKV16q+huTk0uhUq/DcPyUNdS9L146tsRQ9GthhxOsWuiVHH0KuYMiTK7D8+EO1fUvbRVSW93cw1UPn2sqlIE4/fDNjc2mSkytCZm4eACAjV1Guy4UQQioXtex8gFT5Ok5merA3Kd1fv/lnOR4wYACuXbuGP/74A8tWrsGcgyEAAHlmKuQpr7B4u7KLTBWg2NjY8OtRff7557C3t0frQd8AAGSBeyC2qQmRqS0E8lwc23kXf/31F9auXcu/99SpUzFw4EC0bdsWvr6+OHbsGP755x+cPXtWY12LWq29MhdPVLWSPIhOxaz9dxGTkoVvdt5CZq4cemJhoSUeSsvN6u0ScMvatTOmfQ3svBaB0NhU7L/5EskZubj9UgZDXRG2f9EC6dnyMnURleX9fb2scOBWJE4/jMPMbrUAvElOrlnMsPPyFi3LxPWwJP55aVrRCCHVU5mDnTNnzsDX17ci6kIqCZ+v83q5h9JQzXI8c+ZMLFy4kJ+fpU6HHlA8VXYZZT65ioQjK/ljBg0aBACYP38+FixYAACIiIhAZq4Cxw4pAySWm43EE79BnpoAPT0pTtSpjW3btmHgwIF8OX369MG6devg7++P8ePHw9PTE/v27YOPj0+R9S2vvI13oepKWdKvHoZtuo6zoco5jTysDCF8y5l43yWQK0vXjrGeDsb61oD/0YdYcvQhUrKUo7jmdK9V6nyjt33/djUtIRRweByXhoiEDDiZ6/EtO8UlJ5e30o5QI4RUf2UOdrp06QIHBwcMHz4cQ4cOhaOjY0XUi5SDaFkmwuLT1eZ5Acqer6Py8ccf4+OPPy70HqqWBoN6nWBQrxOEHHBxRgeNN4SNewMwdOM1JCVlop69MVYf3oBoWVaJAcmIESMwYsSIMtW3IvJx3kZ7Tys0djLBzYhkAMDtl8nYfT2i2KTi4lRWIDe0lQt+O/sECek5/DZWzJDz8mKsp4Omzqa4GpaI0w9j0b+pI7/oaY23bBF7G+U5jJ0QUrXKnLMTGRmJcePG4e+//4abmxv8/PywZ88e5OTklHwwqTS7r0eg9ZLThZYZyJ+v06KMwY4mqpaG/A0VU/w8Nd6Ad12LQIcfz+FFkvLG9XEDW7hY6JdpCv/3UbQsE8EvkvnnDG+Gor+tsi598DaSMnKQkpmntm32gXerd2l1rKVMij71MA5PXy+NYWEggWmBRU8rUnkOYyeEVK0yBzsWFhaYNGkSgoODcfXqVXh4eGDMmDGws7PD+PHjcfv27YqoJymDqOQMzNj3Zghv/nlerr9Fvk5JBjZzwqUZHeD+eoI7TTcD1eir/JYdDa2UG2dVK27od3VWXDdORevwegTY1WeJuP16uH4Nq5InUCxvA5s54eIMX+wc1RIXZ/i+dWscIaRqvdNorMaNG2PmzJkYN24c0tLSsHHjRjRp0gRt2rTBvXv3Si6AlKtoWSYO3HqJkZtvFHmTetOFVfp8ndKwNZaivafyBnU9PLHQ61V546xqmibsex+6Q6qy3u6WBnAy00OOXIEtr4faV/ZILJXKaEUjhFSstwp2cnNz8ffff6Nbt25wdnbG8ePHsWbNGsTGxuLJkydwdnZWWyH5XRRcabtevXq4ceMG/zpjDPPmzYOtrS2kUik6deqEx48fl8t7V4QFCxaA4zi1h5eXFwAgPDy80Guqx969e4ssc9iwYa8nQNND38aOODapHWL3zFPbJzcxEmOHDcL8/t6I+Lk/DiwciTNnzpTruTV7PZNy0POkQq9ZGEgKbXsfbvjl4X3tDqnKenMcx7fuPKmCkViEEO1S5gTlb775Bjt37gRjDEOGDMGyZctQt25d/nV9fX2sWLECdnZ271y5olbaNjV9szDjsmXLsGrVKmzZsgWurq6YO3cu/Pz8cP/+fX5V7OqmTp06OHnyJP9cJFJ+DI6OjoiOjlbb948//sDy5cvRtWvXIsvLzJFD6toE5t0m8ts4kQ6fXMkBiPv7OySZ2sFiwPfgRGLEBh1G127dER72jB8W/q6aOCtbi0JjUyHLzIWx9M0EbKpJ4VTelxt+eakOo8PeRlXWu4OXFTbnm0CxMpOTCSHapczBzv3797F69Wr07dsXEknhv9YBZV5PebQaFLXStgpjDCtXrsScOXPQq1cvAMDWrVthbW2NgwcP8kOfqxuRSKQxwBAKhYW2HzhwAAMGDICBQdE/9ClZuYBIB0ID9dW5Vw1qBHMDCQxYOuovjYJ51/EQWymvn0nboXhx81+cuxqEgb26l8NZAZaGEriY6yE8IQM3I5Lg+7pbCwBOPYwFAHzawgk96tu9Vzf88lJdRoeVVVXVu4WbGfTEQmTkyAEAhlKaFowQ8nbK3I116tQpfPLJJ0UGOoDyZt6uXbt3qhigXGm7adOm6N+/P6ysrNCoUSOsX7+efz0sLAwxMTHo1KkTv83Y2BgtWrRAYGBgkeVmZ2cjJSVF7VGZHj9+DDs7O7i5uWHw4MGIiNA8K2tQUBCCg4MxcuTIYstLz85DVsRdvFg9GJHrv0TC8V+BrFQ0cTGFt7s56ro7wsmtBtJDTkORkwWmkCM1+BgEeiYwcfQq13NTte7cyJe3I1cwfo6ZHvXtKP+BlIpEJISrxZuk5F5rLtEMxoSQt1LmYMff3x8bN24stH3jxo1YunRpuVRKpaiVtrds2QIAiImJAQBYW6tPoW9tbc2/pom/vz+MjY35R2XOFdSiRQts3rwZx44dw9q1axEWFoY2bdogNTW10L4bNmxArVq10KpVq2LLVNg3hEX3ybAZ9ANM2w1D9osQiE4sgZWBcpgux3HY/88R5MQ9w4uf+yNiRR+k3jgI2wELUd/93bsb81Pl7dwIf5O3c/tlMhLTc2CoK+JXSCekJNGyTNyPevOHSP5RhYQQUhZlDnZ+//13PqE2vzp16mDdunXlUikVhUKBxo0bY/HixWjUqBFGjx6NUaNGvfP7zJw5EzKZjH+8ePGinGpcsq5du6J///6oX78+/Pz8cOTIESQnJ2PPnj1q+2VmZmLHjh0ltuqExafjpVkj6Hu0wOF5n+DQ8kk4f+oYnt6/zS+nwBjDwplTUNvNAXafLYPN5z9Bv2ZLZB5ZDGQkl+v5qYKZ4BfJyMlTrhx5+oFyjaN2HpbQEdJybKR0PuQRfISQ8lXmO09MTAxsbW0Lbbe0tCyUXPuuilppW9Xto8pviY2NVdsnNja22KRbiUQCIyMjtUdVMTExgYeHB548eaK2/e+//0ZGRgY+//zzYo/feU15Ldp7WKKJsxm83c3RskFtWFhY8GWePn0aAQEBOHf0EG6sGYv9cz/DwzN/w8zIgG8lKy/ulgYw1dNBdp4C96KU86Ocer2go2qiOEJK430dsk8IqX7KHOw4Ojri0qVLhbZfunSpXEZg5VfcStuAMlnZxsYGp06d4l9PSUnB1atX4e3tXa51qShpaWl4+vRpoQByw4YN6NmzJywtLYs8NitXjr03lK1Sg1u8WQH85cuXSEhI4MvMyFD+JSwQCNTmDBEIBFAoFOV6PhzHoYnzm66saFkmHkSngOOAdh4U7JDSe1+H7BNCqp8yD28YNWoUJk6ciNzcXHTo0AGAMml52rRp+Pbbb8u1ckWttP3HH38AUN5YJ06ciO+//x41a9bkh57b2dmhd+/e5VqX8jJlyhT06NEDzs7OiIqKwvz58yEUCvHJJ5/w+zx58gTnz5/HkSNHNJbh5eUFf39/cC7NkZCcgrwbeyDtaYbw8Ew8ffoU06ZNQ40aNeDn5wcA8Pb2hqmpKYYOHYp58+ZBKpVi/fr1CAsLQ/fu5TMSK7+mLmY4+SAON54nQk8iBAA0djKFWSVO9U+0w/s6ZJ8QUr2UOdiZOnUqEhISMGbMGH49LF1dXUyfPh0zZ84s18oVtdL24MGD+X2mTZuG9PR0jB49GsnJyfDx8cGxY8eq7Rw7L1++xCeffIKEhARYWlrCx8cHV65cUWvB2bhxIxwcHNC5c2eNZYSGhkImk+Hfq88BTgDjjGj06d0LycnJsLOzQ+fOnbFo0SJ+xJyFhQWOHTuG2bNno0OHDsjNzUWdOnVw6NAhNGjQoNzPsWm+lp08uTLrQjVBHCFl9b4O2SeEVB8cY2+3jnFaWhoePHgAqVSKmjVrFjsUvbpLSUmBsbExZDJZlebvlMXFx/H4bMNVCDggcGZHWBtVn+AuO0+Oegv+Q06egp/Y8NjENvCyeT+uLSGEkPdDae/fbz00xsDAAM2aNUPdunXf60DnfbT7egSGbLgKQBlInA2Nq+IaqZOIhKhvbwxAWT9rQwk8ratmXSNCCCHkraYkvXHjBvbs2YOIiAi+K0tl//795VIxoplq9fD8zXGz9oegrYdltWrq15e8+WrFpmZjz40XtGI0IYSQKlHmlp1du3ahVatWePDgAQ4cOIDc3Fzcu3cPp0+fhrGxcUXUkeQTGp0KRYGOx+o290i0LBPnH79S20aTwRFCCKkqZQ52Fi9ejJ9//hn//PMPxGIxfvnlFzx8+BADBgyAkxP95V6RGGP468rzQtur29wjYfHpKJgJVt0CMkIIIR+OMgc7T58+5Ycri8VipKeng+M4TJo0iR8STirG2nNPcephHAQc+MnWquPcIzQZHCGEkOqkzDk7pqam/DpO9vb2CAkJQb169ZCcnMxPXkfKV7QsE/uDXmL5f48AAIt610UHL6tqO/eIajK4WftDIGesWgZkhBBCPhxlDnbatm2LEydOoF69eujfvz8mTJiA06dP48SJE+jYsWNF1PGDtvt6BGbuv8vn6bRwNeNnS67OwQNNBkcIIaS6KHOws2bNGmRlZQEAZs+eDR0dHVy+fBn9+vXDnDlzyr2CHzLVyKv8CcnXwxMRLct8L4IHmgyOEEJIdVCmYCcvLw8BAQH8MgQCgQAzZsyokIoRZaJvwZFXCgaEx2dQEEEIIYSUUpkSlEUiEb766iu+ZYdULFcL/ULbKNGXEEIIKZsyj8Zq3rw5goODK6AqpKCcPAXyD2qiRF9CCCGk7MqcszNmzBhMnjwZL168QJMmTaCvr976UL9+/XKr3Idu/YVnYFAmJU/s5EGJvoQQQshbKPNCoAJB4cYgjuPAGAPHcZDL5eVWucpSHRcCjU/LRuslp5Gdp8DOUS3h7W5e1VUihBBCqpXS3r/L3LITFhb2ThUjpbP5Ujiy8xRo4GiClm5mVV0dQggh5L1V5mDH2dm5IupB8nkSl4aNF58BAL5u5waO40o4ghBCCCFFKXOws3Xr1mJf//zzz9+6MkQ5ieCMfW9WNU/KyK3S+hBCCCHvuzLn7Jiamqo9z83NRUZGBsRiMfT09JCYmFiuFawM1SVnJ1qWidZLTqvNrSPkOFyc4UuJyYQQQkgBpb1/l3noeVJSktojLS0NoaGh8PHxwc6dO9+p0h86TZMI0mrhhBBCyLspc7CjSc2aNbFkyRJMmDChPIr7YLla6KNgdg5NIkgIIYS8m3IJdgDl7MpRUVHlVdwHydZYiibOb7oJaRJBQggh5N2VOUH58OHDas8ZY4iOjsaaNWvQunXrcqvYhyojRzlP0eSPPNC/qQMFOoQQQsg7KnOw07t3b7XnHMfB0tISHTp0wI8//lhe9fog5eQp8DguFQDQt7E9BTqEEEJIOShzsKNQKCqiHgTAo9hU5MoZjKU6sDehQIcQQggpD+WWs0Pe3f2oFABAHTsjmkiQEEIIKSdlDnb69euHpUuXFtq+bNky9O/fv1wq9aG6FyUDoAx2CCGEEFI+yhzsnD9/Ht26dSu0vWvXrjh//ny5VOpDdY9v2TGu4poQQggh2qPMwU5aWhrEYnGh7To6OkhJSSmXSn2IFAqGB9FvurEIIYQQUj7KHOzUq1cPu3fvLrR9165dqF27drlU6kMUnpCO9Bw5dHUEcLM0qOrqEEIIIVqjzKOx5s6di759++Lp06fo0KEDAODUqVPYuXMn9u7dW+4V/FCourC8bIwgFFByMiGEEFJeyhzs9OjRAwcPHsTixYvx999/QyqVon79+jh58iTatWtXEXX8INyLoi4sQgghpCKUOdgBgO7du6N79+7lXZcP2puRWJScTAghhJSnMufsXL9+HVevXi20/erVq7hx40a5VOpDwxhTm2OHEEIIIeWnzMHO2LFj8eLFi0LbIyMjMXbs2HKp1IcmNiUbCek5EAo4eNoYVnV1CCGEEK1S5mDn/v37aNy4caHtjRo1wv3798ulUh8aVRdWDUsD6OoIq7g2hBBCiHYpc7AjkUgQGxtbaHt0dDREordKAfrgqZKTa1MXFiGEEFLuyhzsdO7cGTNnzoRMJuO3JScnY9asWfjoo4/KtXIfClomghBCCKk4ZW6KWbFiBdq2bQtnZ2c0atQIABAcHAxra2v89ddf5V7BDwG17BBCCCEVp8zBjr29Pe7cuYPt27fj9u3bkEqlGD58OD755BPo6OhURB21miwjFy+TMgEAdWxp2DkhhBBS3t4qyUZfXx+jR48u77p8kM4/jgMA2BjrwliPgkVCCCGkvL11RvH9+/cRERGBnJwcte09e/Z850p9KHZfj8CMfXcBADGyLOy+HoGBzZyquFaEEEKIdilzsPPs2TP06dMHd+/eBcdxYIwBADhOuZ6TXC4v3xpqqWhZJmbuvwuWb9us/SFo62EJW2NpldWLEEII0TZlHo01YcIEuLq6Ii4uDnp6erh37x7Onz+Ppk2b4uzZsxVQRe0UFp8OBVPfJmcM4fEZVVMhQgghREuVuWUnMDAQp0+fhoWFBQQCAQQCAXx8fODv74/x48fj1q1bFVFPreNqoQ8BB7WAR8hxcLHQq7pKEUIIIVqozC07crkchobKJQ0sLCwQFRUFAHB2dkZoaGj51k6L2RpLsbhPPf65gAMW961LXViEEEJIOStzsFO3bl3cvn0bANCiRQssW7YMly5dwsKFC+Hm5lbuFdRm3evb8v8+ObkdJScTQgghFaDM3Vhz5sxBeno6AGDhwoX4+OOP0aZNG5ibm2P37t3lXkFt9io1GwBgIBHBzdKgimtDCCGEaKcyBzt+fn78v2vUqIGHDx8iMTERpqam/IgsUjqqYMfSUFLFNSGEEEK0V7ms3GlmZlYexXxwXqW9DnYMKNghhBBCKkqZc3ZI+Ymnlh1CCCGkwlGwU4VULTsWBuIqrgkhhBCivSjYqUKUs0MIIYRUvDIHO+fPn0deXl6h7Xl5eTh//ny5VOpDQcEOIYQQUvHKHOz4+voiMTGx0HaZTAZfX99yqdSHgk9QpmCHEEIIqTBlDnYYYxqHmCckJEBfX79cKvWhiE9VrhhvaaBbxTUhhBBCtFeph5737dsXgHJ182HDhkEiedMaIZfLcefOHbRq1ar8a6ilFAqGeFWCsiElKBNCCCEVpdTBjrGxMQBly46hoSGk0jdrOInFYrRs2RKjRo0q/xpqqeTMXOS9XgXUXJ+6sQghhJCKUupgZ9OmTQAAFxcXTJkyhbqs3pEqOdlUTwdiEQ2KI4QQQipKme+y06ZNU8vZef78OVauXIn//vuvXCum7WgkFiGEEFI5yhzs9OrVC1u3bgUAJCcno3nz5vjxxx/Rq1cvrF27ttwrqK3iaSQWIYQQUinKHOzcvHkTbdq0AQD8/fffsLGxwfPnz7F161asWrWq3CuorVQtOxa0LhYhhBBSococ7GRkZMDQ0BAA8N9//6Fv374QCARo2bIlnj9/Xu4V1Fa0CCghhBBSOcoc7NSoUQMHDx7EixcvcPz4cXTu3BkAEBcXByMjo3KvoLainB1CCCGkcpQ52Jk3bx6mTJkCFxcXNG/eHN7e3gCUrTyNGjUq9wpqKwp2CCGEkMpR5mDnf//7HyIiInDjxg0cP36c396xY0f8/PPP5Vq5gpYsWQKO4zBx4kR+W1ZWFsaOHQtzc3MYGBigX79+iI2NrdB6lAdKUCaEEEIqx1tN8GJjYwNDQ0OcOHECmZmZAIBmzZrBy8urXCuX3/Xr1/H777+jfv36atsnTZqEf/75B3v37sW5c+cQFRXFz/ZcnVGCMiGEEFI5yhzsJCQkoGPHjvDw8EC3bt0QHR0NABg5ciS+/fbbcq8gAKSlpWHw4MFYv349TE1N+e0ymQwbNmzATz/9hA4dOqBJkybYtGkTLl++jCtXrlRIXcpDrlyBxIzX62JRyw4hhBBSococ7EyaNAk6OjqIiIiAnp4ev33gwIE4duxYuVZOZezYsejevTs6deqktj0oKAi5ublq2728vODk5ITAwMAiy8vOzkZKSoraozIlpueAMUAo4GCqR+tiEUIIIRWp1MtFqPz33384fvw4HBwc1LbXrFmzQoae79q1Czdv3sT169cLvRYTEwOxWAwTExO17dbW1oiJiSmyTH9/f3z33XflXdVSU3VhmeuLIRQUXkGeEEIIIeWnzC076enpai06KomJiWoroZeHFy9eYMKECdi+fTt0dXXLrdyZM2dCJpPxjxcvXpRb2aXxipKTCSGEkEpT5mCnTZs2/HIRAMBxHBQKBZYtWwZfX99yrVxQUBDi4uLQuHFjiEQiiEQinDt3DqtWrYJIJIK1tTVycnKQnJysdlxsbCxsbGyKLFcikcDIyEjtUZkoOZkQQgipPGXuxlq2bBk6duyIGzduICcnB9OmTcO9e/eQmJiIS5culWvlOnbsiLt376ptGz58OLy8vDB9+nQ4OjpCR0cHp06dQr9+/QAAoaGhiIiI4Of/qY5ojh1CCCGk8pQ52Klbty4ePXqENWvWwNDQEGlpaejbty/Gjh0LW1vbcq2coaEh6tatq7ZNX18f5ubm/PaRI0di8uTJMDMzg5GREb755ht4e3ujZcuW5VqX8kTBDiGEEFJ5yhzsREREwNHREbNnz9b4mpOTU7lUrLR+/vlnCAQC9OvXD9nZ2fDz88Nvv/1WqXUoq3haF4sQQgipNGUOdlxdXREdHQ0rKyu17QkJCXB1dYVcLi+3ymly9uxZtee6urr49ddf8euvv1bo+5YnatkhhBBCKk+ZE5QZY+C4wsOl09LSynXElDZTjcaiBGVCCCGk4pW6ZWfy5MkAlKOv5s6dqzb8XC6X4+rVq2jYsGG5V1AbUcsOIYQQUnlKHezcunULgLJl5+7duxCL38z8KxaL0aBBA0yZMqX8a6hlsnLlSM3KA0DBDiGEEFIZSh3snDlzBoBy6Pcvv/xS6XPTaAtVcrJYJICRbplTpgghhBBSRmW+227atKki6vHB4LuwDCQac58IIYQQUr7KnKBM3g0/ezJ1YRFCCCGVgoKdSvaK5tghhBBCKhUFO5WMRmIRQgghlYuCnUoWTyueE0IIIZWKgp1KRi07hBBCSOWiYKeSvRmNJS5hT0IIIYSUBwp2Ktkr6sYihBBCKhUFO5WIMZavZYfWESOEEEIqAwU7lSg9R46sXAUAIE+hqOLaEEIIIR8GCnYq0ebL4fy/O/10DruvR1RdZQghhJAPBAU7lSRalokf/wvlnysYMGt/CKJlmVVYK0IIIUT7UbBTScLi08GY+jY5YwiPz6iaChFCCCEfCAp2KomrhT4KLvsp5Di4WOhVSX0IIYSQDwUFO5XE1liKTrWt+edCjsPivnVhayytwloRQggh2k9U1RX4kNibKAOb3g3tML2rFwU6hBBCSCWglp1KlJCeAwCoa29MgQ4hhBBSSSjYqUSJ6coJBc1pqQhCCCGk0lCwU4kS0pQtO2b6tFQEIYQQUlko2KlESRnKYMdcn1p2CCGEkMpCwU4lYYwhMV3VskPBDiGEEFJZKNipJKnZeciVK2cVpGCHEEIIqTwU7FSSxNf5OnpiIXR1hFVcG0IIIeTDQcFOJUmgLixCCCGkSlCwU0lU+TqUnEwIIYRULgp2KkkStewQQgghVYKCnUqi6sYypWCHEEIIqVQU7FQSfvZkCnYIIYSQSkXBTiV5k6BMsycTQgghlYmCnUpCCcqEEEJI1aBgp5LQ7MmEEEJI1aBgp5LwwQ6teE4IIYRUKgp2Kgkf7OhRsEMIIYRUJgp2KkFWrhwZOXIA1LJDCCGEVDYKdiqBaiSWjpCDoURUxbUhhBBCPiwU7FQC1SKgZvpicBxXxbUhhBBCPiwU7FSChNcTCtIcO4QQQkjlo2CnEiRlqFp2dKq4JoQQQsiHh4KdSpCQRrMnE0IIIVWFgp1KQLMnE0IIIVWHgp1KQLMnE0IIIVWHgp1KkEDBDiGEEFJlKNipBEkU7BBCCCFVhoKdSkDdWIQQQkjVoWCnEiRQgjIhhBBSZSjYqWC5cgVkmbkAqGWHEEIIqQoU7FQw1YSCHAeY0IrnhBBCSKWjYKeCqfJ1TPXEEApoXSxCCCGkslGwU8HeBDu0VAQhhBBSFSjYqWBvZk+mpSIIIYSQqkDBTgWjYeeEEEJI1aJgp4Lxi4AaULBDCCGEVAUKdioYLQJKCCGEVC0KdipY/tFYhBBCCKl8FOxUML5lh7qxCCGEkCpBwU4FowRlQgghpGpRsFOE8+fPo0ePHrCzswPHcTh48KDa62lpaRg3bhwcHBwglUpRu3ZtrFu3rlA5CfmCnc2bN4PjOLWHrq5uoWMePHiAnj17wtjYGPr6+mjWrBkiIiIq5DwJIYQQbSeq6gpUV+np6WjQoAFGjBiBvn37Fnp98uTJOH36NLZt2wYXFxf8999/GDNmDOzs7NCzZ08AgELB+OUiVPPsGBkZITQ0lC+H49RnVX769Cl8fHwwcuRIfPfddzAyMsK9e/c0BkWEEEIIKRkFO0Xo2rUrunbtWuTrly9fxtChQ9G+fXsAwOjRo/H777/j2rVrfLCTkpULuYIBAEz1lTMocxwHGxubIsudPXs2unXrhmXLlvHb3N3d3/V0CCGEkA9Wte7G8vf3R7NmzWBoaAgrKyv07t1brVUEALKysjB27FiYm5vDwMAA/fr1Q2xsbIXXrVWrVjh8+DAiIyPBGMOZM2fw6NEjdO7cmd9H1YVlIBFBIhICUHZ/OTs7w9HREb169cK9e/f4/RUKBf799194eHjAz88PVlZWaNGiRaEuNEIIIYSUXrUOds6dO4exY8fiypUrOHHiBHJzc9G5c2ekp6fz+0yaNAn//PMP9u7di3PnziEqKkpjt1N5W716NWrXrg0HBweIxWJ06dIFv/76K9q2bcvvk1QgOdnT0xMbN27EoUOHsG3bNigUCrRq1QovX74EAMTFxSEtLQ1LlixBly5d8N9//6FPnz7o27cvzp07V+HnRAghhGgl9h6Ji4tjANi5c+cYY4wlJyczHR0dtnfvXn6fBw8eMAAsMDCw1OXKZDIGgMlkMo2vA2AHDhxQ27Z8+XLm4eHBDh8+zG7fvs1Wr17NDAwM2IkTJ/h9joVEM+fpAazXmosay83JyWHu7u5szpw5jDHGIiMjGQD2ySefqO3Xo0cPNmjQoFKfDyGEEPIhKOn+rfJe5ezIZDIAgJmZGQAgKCgIubm56NSpE7+Pl5cXnJycEBgYiJYtW2osJzs7G9nZ2fzzlJSUMtUjMzMTs2bNwoEDB9C9e3cAQP369REcHIwVK1bw9Slp9mQdHR00atQIT548AQBYWFhAJBKhdu3aavvVqlULFy9eLFMdCSGEEKJUrbux8lMoFJg4cSJat26NunXrAgBiYmIgFothYmKitq+1tTViYmKKLMvf3x/Gxsb8w9HRsUx1yc3NRW5uLgQC9csnFAqhUCj45yXNsSOXy3H37l3Y2toCAMRiMZo1a1YoL+nRo0dwdnYuUx0JIYQQovTetOyMHTsWISEh5dLCMXPmTEyePJl/npKSUijgSUtL41tcACAsLAzBwcEwMzODk5MT2rVrh6lTp0IqlcLZ2Rnnzp3D1q1b8dNPP/HHrF/0LZIyxTBruxAAsHDhQrRs2RI1atRAcnIyli9fjufPn+OLL77gj5k6dSoGDhyItm3bwtfXF8eOHcM///yDs2fPvvN5E0IIIR+i9yLYGTduHAICAnD+/Hk4ODjw221sbJCTk4Pk5GS11p3Y2Nhih3dLJBJIJJJi3/PGjRvw9fXln6uCo6FDh2Lz5s3YtWsXZs6cicGDByMxMRHOzs744Ycf8NVXX/HHxEW/hFxkCpFAOZdOUlISRo0ahZiYGJiamqJJkya4fPmyWrdVnz59sG7dOvj7+2P8+PHw9PTEvn374OPjU7qLRQghhBA1HGOMVXUlisIYwzfffIMDBw7g7NmzqFmzptrrMpkMlpaW2LlzJ/r16wcACA0NhZeXV7E5OwWlpKTA2NgYMpkMRkZG5VL33dcjMH3fXQAAB2BJv3oY2MypXMomhBBCSOnv39W6ZWfs2LHYsWMHDh06BENDQz4Px9jYGFKpFMbGxhg5ciQmT54MMzMzGBkZ4ZtvvoG3t3epA52KEC3LxMz9d/nnDMCs/SFo62EJW2NpldWLEEII+RBV62Bn7dq1AMDPUqyyadMmDBs2DADw888/QyAQoF+/fsjOzoafnx9+++23Sq6purD4dCgKtJfJGUN4fAYFO4QQQkglq9bBTml62HR1dfHrr7/i119/rYQalY6rhT4EHNQCHiHHwcVCr+oqRQghhHyg3puh5+8TW2Mp/PvWg/D1Ip9CjsPivnWpVYcQQgipAtW6Zed9NrCZE9p6WCI8PgMuFnoU6BBCCCFVhIKdCmRrLKUghxBCCKli1I1FCCGEEK1GwQ4hhBBCtBoFO4QQQgjRahTsEEIIIUSrUbBDCCGEEK1GwQ4hhBBCtBoFO4QQQgjRahTsEEIIIUSrUbBDCCGEEK1GwQ4hhBBCtBoFO4QQQgjRahTsEEIIIUSrUbBDCCGEEK1GwQ4hhBBCtBoFO4QQQgjRahTsEEIIIUSrUbBDCCGEEK1GwQ4hhBBCtBoFO4QQQgjRahTsEEIIIUSrUbBDCCGEEK1GwQ4hhBBCtBoFO4QQQgjRahTsEEIIIUSrUbBDCCGEEK1GwQ4hhBBCtBoFO4QQQgjRahTsEEIIIUSrUbBDCCGEEK1GwQ4hhBBCtBoFO4QQQgjRahTsEEIIIUSrUbBDCCGEEK1GwQ4hhBBCtBoFO4QQQgjRahTsEEIIIUSrUbBDCCGEEK1GwQ4hhBBCtBoFO4QQQgjRahTsEEIIIUSrUbBDCCGEEK1GwQ4hhBBCtBoFO4QQQgjRahTsEEIIIUSrUbBDCCGEEK1GwQ4hhBBCtBoFO4QQQgjRahTsEEIIIUSrUbBDCCGEEK1GwQ4hhBBCtBoFO4QQQgjRahTsEEIIIUSrUbBDCCGEEK1GwQ4hhBBCtBoFO4QQQgjRahTsEEIIIUSrUbBDCCGEEK1GwQ4hhBBCtBoFO4QQQgjRahTsEEIIIUSraU2w8+uvv8LFxQW6urpo0aIFrl27VtVVIoQQQkg1oBXBzu7duzF58mTMnz8fN2/eRIMGDeDn54e4uLiqrhohhBBCqphWBDs//fQTRo0aheHDh6N27dpYt24d9PT0sHHjxqquGiGEEEKqmKiqK/CucnJyEBQUhJkzZ/LbBAIBOnXqhMDAQI3HZGdnIzs7m38uk8kAACkpKRVbWUIIIYSUG9V9mzFW7H7vfbATHx8PuVwOa2trte3W1tZ4+PChxmP8/f3x3XffFdru6OhYIXUkhBBCSMVJTU2FsbFxka+/98HO25g5cyYmT57MP1coFEhMTIS5uTk4jiu390lJSYGjoyNevHgBIyMjjdvKa5+KLJvqWL32qer3/5DOo6rfn+pIdaxO71+edSwvjDGkpqbCzs6u2P3e+2DHwsICQqEQsbGxattjY2NhY2Oj8RiJRAKJRKK2zcTEpKKqCCMjo0IfcMFt5bVPRZZNdaxe+1T1+39I51HV7091pDpWp/cvzzqWh+JadFTe+wRlsViMJk2a4NSpU/w2hUKBU6dOwdvbuwprRgghhJDq4L1v2QGAyZMnY+jQoWjatCmaN2+OlStXIj09HcOHD6/qqhFCCCGkimlFsDNw4EC8evUK8+bNQ0xMDBo2bIhjx44VSlqubBKJBPPnz1frMiu4rbz2qciyqY7Va5+qfv8P6Tyq+v2pjlTH6vT+5VnHysaxksZrEUIIIYS8x977nB1CCCGEkOJQsEMIIYQQrUbBDiGEEEK0GgU7hBBCCNFujFSYNWvWMGdnZyaRSJiXlxfz8fFhtra2DAD77LPPWNOmTZmBgQGztLRkvXr1YvPnz2f16tVjhoaGzNDQkLVs2ZIdOXKEL8/f358BKPRwc3NjgwcPZmZmZkxXV5fp6Oho3M/Q0JDp6uoyNzc3NmLECNalSxemr6/PADAdHR1Wp04dtTr279+fmZmZMYFAwMRiMQPA3N3d1crv06cPc3V1ZQKBgAkEAgaAP4bjOAaALV26lJ07d459/PHHTE9PT2PdGjZsyLZu3cqsra354zQ9nJyc+PcXCoVMV1eXiUQiBoCJxWLWsWNHNn78eGZpackfI5FImFQq5Z9/9NFHrEGDBkwikTAA/PmpyuU4jtnY2LChQ4fyn5Hq/As+6tWrx7y8vJhQKCyyzgCYnp4ef15SqZRZWVnxx4hEImZgYMBfP5FIxJycnJitrS2/TUdHhz9PjuOYjo4Os7S0ZEZGRozjOMZxHBOJRMze3p65uLjwZec/b9XD1NSU2dvbF3udAbA6deowa2trpqenx6ysrBgApqurywwNDVn//v3ZtGnT+Ourr6/P6tSpwwAwe3t7ZmhoyACwZs2a8Z+Vo6Mj++abb1jjxo35bRYWFqxWrVoMAJswYQJTKBSsS5cuRdZpwIABzNfXl+nq6hZb9/yPkj4b1TUvaR/VZ5H/YW5uXuJxRkZGJe4jlUqZhYUFEwqFTCgUMo7jmEAgYEKhkInFYqarq8vq1q3LBg0axJ87x3Fq/98lEgmzsLBgzs7OfBmq74rq/6NAIGDu7u7s33//ZYMHD+b/D2g6V1dXV2ZpaVni9+R9fIhEIv76iMVi/rlAIGDOzs5s/vz5zMfHh//u6OjoMCMjI/47YG9vz1q3bs0sLCz46yMWi5mJiQkTiUT876GdnR1r166d2v9JOzs7jd8R1f/V4q636neI4zjm5OTE/Pz8+N8WgUDArKys+O+H6jNs0qQJ/zmrfjNV3zOhUMhsbW2Zg4OD2jWoXbs2GzRoEDMwMGAA+PtE/oehoSHz9vZmYrG42DrXq1eP2dvbM11dXVazZk3WsmVLZmtry6RSKfPz82OPHj2qlPsxBTsVZNeuXUwsFrONGzeye/fusS5dujCJRMI2bdrEALBGjRqxTZs2sZCQEBYcHMy6devGLCws2L59+9ijR49YaGgomzVrFtPR0WEhISHs2rVrzMXFhVlZWTFzc3MWHR3NoqOj2YMHD5ijoyMbNmwYu3r1Knv27BnbvXs3CwwM5PcZPnw4A8AWL17MwsLC2N69e5muri6ztrZmDg4ODAD77bff2KeffsrEYjFbv349A5Q36Llz57JRo0bxNy0fHx/WrVs3/otcv359NmPGDPbVV1/xwZipqSkbM2YMW7lyJf9DvnPnTva///2Pubi48DeJCRMmsA0bNjAAbMWKFczAwIC1bNmSrVixggFgPXr0YKNGjeL3AZQ32+nTp7Njx47xPxpSqZTt3buX9evXj/+PbGJiwjZs2MB27NjB/9CPHTuWAWDe3t5MIBCwMWPGsCNHjrA//viDDxYOHTrEdu7cyd/cf/vtNzZt2jTWunVr/kfy6NGj7OTJk6xjx478jWjixIlsz549zNjYmOnr67NDhw6xkydPspo1azJAGZDu3buX/fPPP/yP0ciRI9k///zDBx4CgYCtXLmSNWjQgBkbGzOBQMB69erFADBfX1/WunVrJhQK2eLFi5mzszOztrZmAoGALVq0iB04cID5+voyqVTKBAIBa9KkCQPA/xgvWbKEXbp0ia1bt46JxWImEAjYnDlz2JEjR9jChQtZt27dmFAoZNu2bWPffPMNfyPYuXMn27lzJ/8j37VrV3bnzh3Wtm1bJhAImI6ODuvbty/bsmULk0gkTEdHh7Vq1Yr/LkilUubi4sKGDRvGTp06xRwdHZlYLGbu7u5s+PDhbOvWrUwqlTIdHR02fvx49tNPPzFvb28GgBkYGLAvvviCHT16lDk6OjI3NzcmFouZv78/2759O/8jPXz4cHb06FFmYmLC/6AfOXKEnTlzhhkbGzMAzMHBgZ0+fZo5ODgwgUDAHB0d2bFjx1jjxo2Zg4MDa9OmDbO2tmaAMhju3bs3/5m5urrygYaHhwc7fvw4++ijj5idnR1r3749f0Pw8PBg9erVYzVq1GC3b99mt2/fZrt372Y6OjrM2tqanTp1ih08eJB16tSJubu7s9u3b7MpU6YwgUDA6tSpw2xtbVmvXr34QNHY2Jh16tSJtW3bltWvX599+umnjOM41qFDB7Zt2zbm6enJOI5jDRs2ZADYrl27WL169ZiZmRnr06cP+/PPP5m9vT0TCoXMz8+P7dy5kw0YMIBJpVJma2vLfH19Wc2aNZlEImH29vbMxsaGDRw4kB05coTt2LGDcRzHatasybZu3crOnDnDxo0bx6ysrPh9VO+7YMECdu3aNTZhwgT+Wh85coT16tWL/65///33/LX+9NNP+f/Phw8fZj4+PgwAW7NmDdu7dy8zNzfnA+Bly5axjz76iP/jZdiwYaxDhw78TRQA+/PPP9nt27fZ0qVLGQA2ePBg/lqPHj2a30d1rVU3719//VUtEP3f//7HPvroI+bu7s4HJqr/I6rvlqquCxcuZBzHMYlEwtatW8f279/P/2ZIpVL266+/sgEDBvBB9EcffcRq1arFTE1NGcdxTCgUsr/++otdu3aNffTRR/xvaUBAAFu9ejUTiURMKpXy+6iuh7GxMdu1axdbuHAhX28nJyf2119/sfr16/OB0IoVK1iHDh343wPVd7tBgwb8/4HFixez+vXrMxMTEyYUCpmLiwvbvHkz69+/P/+b6ezszKysrJidnR3jOI65ubmx/fv3sz179jBdXV0mEAiYk5MT27FjB1u6dClzd3dnAoGArV+/nv8N4DiO7dmzhz179oy5urryv/cPHz5ko0ePZk5OTiwtLa3C78kU7FSQ5s2bs7Fjx/LP5XI5s7Oz478ABw4cUNs/Li6OAWDnzp1T225qasrWrFnDatasyU6cOMGcnZ2ZhYUF//r06dOZj49PsXVxcXFhhoaGTKFQ8Nt69erFOI5jAQEBavVp3LgxmzVrFgPAhg4dyu+fnJzMALDJkyezsLAw/j9awfNQbX/+/Lna8127djF7e3sWEhLC/wfNf0zr1q3ZZ599prYtf9mA8i/XhQsXMsYYCw0NZQD4YOLcuXNMLpczU1NTBoB99913/LGXLl1iANjEiRMZAPb48eNC1/rPP/9kANjp06cZY4ydOXOGDxDt7e1ZdHQ0fy6q41Sf2ZAhQ/hyCn6OdevWZQDYrFmzGGOMHT9+nP8h+fzzz1lycjL/l7mhoSH7888/2YMHD/jnU6ZMYQBYUlISY0z5ffjzzz/Znj17mFgs5p8zxtjt27f5H5cff/yRAcrWGYlEwu/TokULNmfOHLXjVFTbGjZsyEQiEdPX12dr1qzhgzGRSMQ6duzIUlNT+R8tZ2dn9vXXX7OaNWvyQemAAQPYkSNH+M+wXbt2bMKECSw1NZXZ2toykUjE2rZtyx/3+++/88Gtra0tc3NzY4CyhVC1z4kTJ5ihoSFr3rw5S01N5be1a9eO38fd3Z2/bvmvta6uLnNwcFC71m5ubowxxl/rkSNH8scnJSWx+fPnswYNGjDGGH+t586dy29TXev8LVp16tRhLVq04PdRXe82bdqobctftupad+/enfn4+PB1lkgkrEaNGoyxN//3dHV1maurK1/OxIkTGcdxbNWqVQwAu3XrVqHfg88//1zt/6NMJmMAmKenJ///UV9fn1lYWKgd5+XlxSwtLdW+HwXLNjQ0VKuP6r1GjRrFX+uuXbsygUDA/vjjD/5aDxgwgP9DJSkpiU2YMIG5u7vzv0979uzhW6AUCgV/rW1sbJidnR3/f9HV1VXtOBsbG2Zqaqr2O5e/7IYNG/ItOcOHD+evtY6ODjM3N2eDBw/mrzXHcczMzIwvx8/PjwFgTZs25a+1jY0Nc3d35/e5du2a2m+b6lpbWFgwqVTKQkJCmLOzM9PR0WF6enr8cba2tkxfX58NHjyY31awbFVA3b59e8bYm98/1fVUKBR8QGNpaclmz57NkpOT+Zag7t27MwAsMDCQCYVC5u7uzmbPns3XWVUOY4zFxMQwQNkyNHr0aObs7Mz/8aTaZ+DAgWzQoEFq2zIyMtTK7tWrF9PX12e2trZs9uzZfJ1r1arFZs+ezRhT3hctLS3Z+vXrWUWjnJ0KkJOTg6CgIHTq1InfJhAI0KlTJwQGBmo8RiaTAQDMzMwAAHK5HLt27UJ6ejr+++8/dO/enS8vOTkZdnZ2cHNzw9q1a+Hp6Yn+/fvDysoKjRo1wvr169XqEhsbC6FQiMePHwMAbt++jUuXLoExBl1dXbV6SKVSnDx5EgDQoEEDfrtq7ZHQ0NBSXQMTExPk5OTwZa5ZswZTp05FnTp1AADR0dGwsrKCp6cnAODGjRvw8PCAn58frKysAABXr14FAH7ds5o1a+Lw4cOIjIxEVlYWAODFixf8dRMI3nyd8197VXnR0dEAlIvU5b/W+d/D0tIS6enp2LJlCwBg/fr1+PXXX9XWWevZsyfq1q2Lb7/9FgBgb2+PVq1awdraGt27d+fLDgoKQkhICADg4sWLSExMRFpaGpjyjwwMGjQIQUFByM3NBcdxyMjIgLe3N2rWrAlzc3NkZGTw1yv/98Hb2xtJSUmQSCT88/T0dKxbtw5CoRBCoRBdunQBoPxeZWdn49tvv4WXlxeuXr2KsLAwJCcnY9q0aWjXrh3OnTvHl21kZITg4GA4OTkhMzMTAQEBaNy4MTiOg0KhgIODA8aOHct/NxQKBc6ePYvu3btjxIgRkEgkiI6OxsqVKwEA7du356/b2LFjUadOHZiYmIDjOJw9exadO3fGgwcPIJFIcPHiRXh5eaFnz5583X///XckJCTg8OHDSE1NhVQqhZubG16+fIlFixZBJpPh7NmzaNq0KZ4+fQqxWIy0tDTY2Njg888/h0KhQG5uLhISEuDu7o7c3FwAQEREBOzs7NCtWzfo6enh8ePHiIyMBAA0bNgQ+/btw6NHj2BnZ4cxY/7f3rmHVVXl//997pwL5xzO4XATDqBclIuSgnKJCE1GUbyOaGYXsdTQssZIJCvFSp+Z1Hkcs5HyrkOWgZYYGd4mL1mEaDUqeMseLS9FIkl44f37g/b6cuz7m2l+v68zDd/1ep79wNln7bXX/uy11v7s9XmvdfKhUqlw5coV1NfXIzAwEOnp6VCpVMjIyMCpU6cAAHV1daiursahQ4eg0+lgs9lw4MABqNVqfPbZZ9BoNDAajVi7di3q6urgcrlQW1sLq9WKnTt3Ii4uDnPmzEFraytaWlrg5+eHUaNGITIyEgDw448/IjU1VbT1V199FTabDQsWLAAAjBkzBmvWrEFiYqJIU1paCgB4/PHH4efnh27duolr8fX1RWZmJpqbm3Hp0iV8+umn8Pb2htFoxNGjR+Ht7Y2AgACo1WqYzWaUlJSIvH19fXHlyhVcuXIFZ8+eRUtLC8rKygAAmZmZol4fOHAAFosFe/fuRdeuXRESEoKKigoMHDgQQFv/tG7dOuTl5YkfYf72229BEnl5ebh69Spef/11qNVq3LhxA0uXLhVt8fTp0zh37hzi4+MxZcoUfPPNN0hLS0NaWhr8/f2Rnp6OVatWIS8vDzU1NaitrYVGo4HVakVVVRWOHz8u6nBDQwMSEhLg5eUFlUoFkmhtbUVdXZ3oewCAPy1Ld+zYMVy5cgWXL18WaWpqagAAP/zwAz7//HOUlJRAp9Ph0qVL6N+/P2JjY3Ht2jXcvHkTzc3NcDgciIqKwvnz53H16lV88skn8PPzQ3x8PBoaGkTe58+fR1NTEwAgNjYWJPHhhx+KdnX06FGcOnVK9GHXr1/Hnj17YLPZkJiYCAA4ceKE6Edu3rwJLy8v7NmzRzx3lHyuXbuGkpISAG2/O6n098p1v/3224iKikJZWRkCAwPFvj59+mDTpk0i7x07dqCiogK9e/fGlStXsH37dtFnf/nll8jKygLQ9lxU2v5t57a7U/8LOXv2LAFw3759HvsLCgrYu3fvn41a3Lx5k4MGDWJaWhoPHz5Ms9lMjUZDm83Gp59+mnFxcWxubibZNlKQnZ3NQ4cOsbKyUsRKp0+fzpqaGi5btoxeXl5ctWoVSXLDhg1Uq9WcMmWKeDtXqVR86aWXmJKSwoyMDALg22+/zbVr14ohfgBcvny5R/nx0wjM/21kp7m5Wbzhms1mUbYBAwawf//+4o0LAPv168fDhw+zvLxc5GUymbhw4UIePHhQ7Nu1a5cYnl63bp14e1SGmW02G/v06cOWlha+9NJL4rhbbWuxWMSbSFZWFtPS0kSa8+fP08vLi3a7XcSmlbfdCRMmiHwAMCQkhIcPH+aaNWvEW5PD4eCKFStYXV3N8PBwqlQq1tXVcfLkybRYLOzTp48IzShbcnIyDxw44KGZSEtLE/deo9Fw5MiRYrREqQ+vvPKK0D0ZDAY++uijHrocnU7HV155RVyHXq9ncXExN27c6KG1euKJJ0SIBT+Nhrzyyiti2N1qtYqQozL073Q62b9/f8bExIhQgt1up9PpFPVTedtXRn4aGhqYkZHBAQMGsFu3bgwJCWFRUREDAgJEWaKjo+lyuRgcHCzqOn56I7bZbFyxYoUIYyi6pP3794uROrvdzkceeYTdunVjTEwMfX19Pcqt1+tZUlLCOXPmiDrp5eXFffv2sbKykmazmUajkYWFhQTAjRs3smvXrvT19eXWrVvpcrnYqVMnD+2WyWSi0+mky+Xi6tWrxdtzSEgI7XY7n3/+eRFCU6lUnDBhAt944w0OGzaMQFuod/jw4QwJCREhYiVt+78DBgzg6NGjxfdarZYzZ85kTU2NuFdK+GTWrFkeafbv3y+OU2zRvkyFhYWsqakRowZarZYjR44U7R8AY2JiWFpayuHDh4t6OHPmTD7++OM0Go2iHSr5+/j4cNSoUSwpKaFGoxH7s7KySJJdunShSqXiW2+9RQBcsWIFNRoNz549S5K8ePEinU4nVSqVqNeBgYFUqVQcM2aMR1+kUqlYVVXFdevWidCf3W7nihUrWFNTI8LtH374IR999FER0ktPT/doi0rbU0LJyr72faZSH9vfpxdeeIEzZszwSBMWFvYzzaTdbvdI06NHD44YMUKEs9rnqfTPAHjvvfd6lElp9xqNRuj8lLq8ceNGj3N27tyZN27cEM8bJZxYXV3NiIgI4qeRsTvuuONnfZPFYhH9UufOnel2u0UY32AwiFFjpZwGg4EzZ84k0KZlUuqB3W7nsmXLRDrlWpOSkvjdd9+xpaWF8+fP96gftxPp7NwG/llnZ/LkyQwNDeVXX33FlpYW1tfXs7q6mvn5+VSpVCwrKxNplZCAgk6no0aj8QhJPPbYY0xOTiZJZmVl8Y477mBwcDBLS0vFg9rhcPD3v/8977rrLtFZJSUl8b777qPb7f6nnZ1r164xJydHHFdfXy86W5VKxUOHDnnkk5eX5/EZaNOltN+XmJjIMWPGMDo6mkBbWC0qKorvvPMODx06xB49eohjFQGs0tHcatsePXoIZyckJIRfffUVybahZkXUd+jQIdbV1XHIkCHU6/XU6/W8ePGiyAeAGG6dPHkyHQ4HAXDy5Mke5+ratSunT58uwkwPPfQQExMTuXr1av7lL38RYY/2nZy/vz8NBgO3bt3K6upqBgQE0Gg0Cv3Uzp07WVhYSIfDwaioKKakpLCgoIBOp5ObN2/m9OnT6eXlJTRdShhpxIgR9PX15cGDB7lhwwZxTh8fHx48eJD19fWMiIhgcnIynU4nTSYTn3jiCRF+mz17Nmtrazlu3DiPB6fS+anVao4dO1bY22w2U6vVirBgQ0MD+/TpQ6PRyLi4OA4YMIAnTpygTqdjTk4Od+/eLR72RqNR1BPlnip5K6EarVYr0pw5c4YA2LVrV9psNr788ssMCgqiv78/q6qqWFtby9zcXA87q9Vqdu/enXq9XrSZO+64g3q93iNk2NDQQKvVyvDwcA4YMIAXLlyg1Wrliy++yJdeeolms5lxcXG0Wq3iWsvLy8Vxr7/+uihz+3ORZExMDPV6PY1GI19++WU+8sgj4qF+q62BtskMUVFRBNqc8PZtX3HSgbbQilqtpsViEe1RpVLRz8+PCQkJ3L9/P4cMGSIenAqhoaFUqVS0WCwkyX379olyK/2Ick+0Wi1JMjo6mqmpqbTb7YyOjmZKSgrj4uJoNBpFGBJoC6kMHDiQAwYMIElarVZ26dJFhIn79u3LwYMHi7bYu3dvOp1OZmVlsa6ujrt37xbCYKUttu8fFLp27erRFsm2vs/b25vTp0+nzWZjVFQUw8LC2KVLF7pcLs6fP1+Ed5Rt0KBBwhG32+2iz8zPzycA0TfOnTuXZrOZDoeDa9euZUZGBkNDQ2k2m2mz2VhcXMy7775b1LvJkyfz8OHDdDqd1Ov1dDgcLC0tZVVVlTh3Tk6O6J8VYXppaSnDwsIYERFBlUolnAklFH7rhA+z2Swcco1GQx8fHwYGBgpnR6PRMD4+3mMCR8+ePTlo0CAP0b9Wq/Vw7txuN4cMGfKzCQ8pKSkewmqbzSacb5VKxZCQEHp7ezM4OJiHDh1iQUGBcN4ULVn7+nE7kc7ObaClpYUajeZnepYHHnhAdDbKd1OmTGFwcDBPnjz5s3yUUQ/lDUDxjJXPN27coNvtpq+vLwsLC8VxS5cuZVBQEE+fPk21Wk2n08klS5Z45D137lxGR0eTpIdjk5ubKxrpwoULPY5ROoNbnZ1r165x2LBhQiB3q9ZGaZy3XkNoaKhHmiFDhngcN3z4cPHQVRrHli1bPOyWm5vLvn37Mi8vj8HBwaKjbWho8LCt2+0WedXW1pIkGxsbhZNx5MgRj3yVEaT2s8qUz0FBQQwODmZlZSUBsKCgwONcubm5In6u3MPPP//cw5Y+Pj6MjIzk5s2bxUMhIiKCEydOJEm63W5GRkZy8ODB4noaGxtps9kYFBQkRlL69evHiRMnctq0aR7lVP5Xq9W02+2cOHEiT548Kb5PSEgQ58rNzeXYsWMZExNDtVrNAwcOeOTT/r61t0P7z+3f3P7evvZv+7eW9e9tvySf/659kG2jTXFxcR62Dg4OFm3G7XYzJCRECGcVW5vNZoaGhgpbJyYmsrCw8O/aOiMjQ6RT9GkBAQEe7TM3N5dms9nD1jqdjt27dxdpvLy86HK5GBgYSJJi1peXl5dI43a7abVaxQPv4MGDdDgcNJlMoj0GBwczNTWVQUFBJMlFixZ52OvWkQWyrf8C2hxi5TilTGq1mn/961/FAzE/P59+fn5Uq9XctGkTJ0yYwN/85jfCSV+/fj179+7N/Px8nj59mkDbi47i7CjHNTY2MiUlhampqWIfSZ4+fdrDvu3vu0qlYkZGhujnlLaoHKc45SkpKWKGFAD6+/t79IeKQDsyMtLD1u01O8HBwXQ4HELQfPDgQVqtVvr5+QlbX7p0Sey71da31mG1Wu1ha0VErKDMDm1v63HjxjE6OppNTU08d+4cJ0yYwICAAGZlZXH37t0E2kbVu3TpwqysLJ47d4533XUXIyMjmZmZSQDctm0br127JkYqlX4pNzeX2dnZYqTl1k2tVtNkMjE7O5vfffcdNRoNTSYTe/XqxezsbDY1NTE/P5+pqaniXFu2bKFOp2N6ejqzs7PFtU2YMIF9+/blhQsXSFLUj9uN1OzcBvR6PXr16oXt27eLfa2trdi+fTtSUlIAACQxdepUlJeXY8eOHQgPD/9ZPv369UNSUhJycnJQW1uL2tpaJCYm4r777hPx5z59+uD7778X8VOgTTsQGhqKlStXws/PDyQ99CwAoNFo0NraKj47HA40NDTg/fffx5gxYwAAhw8fFt8rOhdFY6Nw48YN5Obmor6+Xmh9bsXpdGLSpEniGgBg2LBheP/99z3SKedQOHv2LH744Qf06tULQFu8WaVSedjNZrOhvr4elZWVWLFiBU6dOgWNRoN7771XpGlpacGZM2eEvic0NBSXL19GZGQkGhoa8PHHHyM6Otoj3+LiYuj1eiQlJcHlcmHLli0AgLS0NNy8eRM7duyA2WwGAFRUVIjjwsLCsGPHDly6dAlZWVno0qULAPzM/goZGRnQarW4ePEinE4nWlpacOzYMZw5cwZWq1VoTBobG5GVlQWVSoXMzEyhtVL0HYWFhaiurobBYEB4eDhycnIAAIsWLUK3bt3Q0tKCsLAw+Pr6AgB0Oh1aWloA/Fd9OXfuHEJCQmA0GgEA8fHxHnXP29sbISEhKCsrE5ocf39/DB48GGVlZSgvLxc2Ki4uBgC89957MJlMcDgcKC0tRW1tLfbt24fY2FhxXGVlJQAgKioKixYtEtoPf39/9O3bF2VlZVixYgUAiPMr+RgMBnh5eSE5ORlvvvkmAGDw4MGifTQ1NaG5uRne3t4etr5y5QoCAwOFrb///nuh4WpsbES/fv3Q0tKCqVOnwsvLC01NTThx4gQCAwOFrfV6PYxGI2bMmCFsvWTJEpGuoaEBANDQ0ODRPo8cOYLm5mbEx8cLW5MUugygTfP2ww8/iPr07bffQqPRCO0EAPTu3RuNjY1Ckwa06SyuXbsm2mN6ejpOnz6N0NBQAMD9998PrVYLnU4n7mtQUBAcDgdMJhMA4MKFCwDatCfKcQCg1WqhVquxfPlyJCQk4MaNGzh37hz0ej38/PwwaNAg0a8oepiLFy+iuroaQ4cOxR/+8AcAwPjx40WeLpcL6enpyMrKgl6vx9133y3yAoCVK1fC19cXer0ec+bMQW1tLSZNmgQAWLBgAVauXImVK1fCbrcDaNMzKsf5+fmhqakJZ8+eRWRkpLi/P/74o0d7bG1thVbb9pvYiq0BiPah2ELRySlcv34d33//vbC10+nE9evXxT26//77oVarERYWhtDQUGFrrVYryqvX64XNNBqNyPvmzZs/s7WSzmw2IzAwEDdu3MClS5cwcuRIpKenw+Vy4YsvvsA333yDkSNHwmw246OPPsLZs2eFhtFut2PYsGH4+uuvMXv2bMTGxop+f+jQoXj44Yfx2WefISgoCHq9Hs899xyCgoLw2GOPQaPRYOjQofDx8UFCQgKuXr2Ko0ePYujQoTCbzTh37hwCAwOxZ88euN1upKen4/r166ipqcHQoUPFtSm6QpfLhfr6elE/bju33Z36X8obb7xBg8HAVatW8W9/+xvHjx9Pi8Uihi1TU1NpsVi4YcMGMUV86tSp3LZtG0+dOsXDhw+zsLCQKpWK27ZtE/kGBwdz5MiRPHXqFPfu3StCAEVFRayvr+f69etpMpm4Zs0aut1uzpgxgw8++CA7derELVu28NSpUywrKxP6iyVLlhAAJ02axIiICEZFRQmFvpeXF+fPn8+lS5eK0Z5HH31UrK8CtK274+vryyVLlvDdd98l0BZuWrhwoRjGV6ZZbt26VcyMSk1N5eLFi/niiy8SaNOMaDQaPvbYY+INHD+90ShTxrt06SLeZEtLS/nQQw+J6Z/PP/88g4ODmZ2dzaioKKpUKi5atIiVlZV0Op1Uq9VC47F+/Xra7Xaq1WquX7+eFRUVjI+Pp8lk4urVq/nOO++wf//+1Ov19Pb25q5du0RIzmAwcPXq1Vy1ahXdbreIm8+ePZv79u0To1vKec6cOUObzcbY2FiWl5ezoqJCDLmPGjWKCxcuFOugAG2zyLp37y6G0keNGkWgLXauDDMvXryY69atE7H2WbNmcePGjczIyBBvj2vWrCEAMWW1qKiIJSUltNls4k25sLCQU6dOpV6vF+meeuopfvrppyI0snjxYh4/flyEg+655x6uXbuWDoeD3bt3p8Fg4IgRI1hdXc2UlBRaLBaOHTtWvNnHxMTQaDTy3nvv5ddff82PPvqIM2fOZM+ePZmXl8e9e/cyJyeHWq2WjzzyCEny+PHj4prHjx/PzZs3i+vX6/V86623WF9fz1mzZolree+993jt2jVqNBo6nU6Wl5fzzTffFCNsU6ZMEbZWqVQ0m81cu3Yt4+Li6O3tTYPBIEIVYWFhQhNRWVnJkpIShoWF0WAw8LXXXuPGjRuZlpZGrVZLLy8vbtq0SdgmICCA3t7eXLBgAYOCgsQQf0FBAXfu3MkHH3xQ1I/Fixdz9+7dQvui0Wj4u9/9jgMHDhRpevfuTZvNxoEDB4p7O27cOG7ZskXMRExNTSUAEfYGwKlTp3L79u3s06cPgbYlA8rLyz3C1i+++CJra2vF+dVqNfPy8hgTEyPqda9evVhVVSXKrcxemjNnjqjHVquVkyZN4sqVK6nT6Tht2jQGBQUxPDxcaGQ+/vhj6vV6+vv78+DBg2IG3qhRoxgfH8+YmBjW1dUxICCAycnJrKys5PHjx+nv78+IiAg6HA6eP39ezGoF2qYvnzhxgi6XizabjZ07d6bVauWGDRsYFBTE1NRUEdLx8/PjU089xYiICPr5+YkZQEoYS6VSMTExkTabTdRHAHz22Wf53nvviZC10uaU0T1lNtzu3buZnZ1NnU5Hs9nMP/7xj/ztb38r0ih9tqJHMplMXL58OdetWyfadUJCAnfu3MmHH36YQFtY12Aw8IUXXmBsbKwYOdy9e7cIufr4+PC1117jsmXLhE6vU6dOfOedd5iSkkKdTifWDQLaZq+q1WoGBwdzz549XL16NX19fel2u7l8+XJu3ryZWVlZBNo0PceOHaPL5aLFYmFoaChff/11rl27VvRP/v7+XLZsGZ9//nmq1WoGBARQpVJx9uzZ3LZtm9DDffDBBzx58iTz8/Op0+lYXFzMTZs2MTQ0lCNGjPiXPJOls3Mb+dOf/kS32029Xi86hn+0KTFdl8vFfv36eTg6JOlyuURctlOnThw9ejRLSkoYFxcnFi8sKSnh+++/TwA8duwYGxsbOW3aNLrdbrGo4K26ALnd3q39ooSKzkrpZM1ms8eig3q9XmgD/tHWXgxpMBjocrno7+8vzmc2mz3SmEwmhoeH02AwiI7YbreLRQ2VupeSksK0tDT6+fnRZDLR5XKJB2JkZCQXLFjAq1evMigoiAaDgSaTicOHDxfrNv2jTQkrKOuuJCUlCS2aosVRwhYREREsKCjgnXfeydTUVAYHB9NkMjElJYV+fn60WCy8efMmSdLhcHjY1mAweCzGaLVaabFYPITKiibsn7G10WhkdnY2Bw0aJBbhbB8qAdqEniNGjGB0dLSHWNpkMglbd+rUidnZ2czKyvJwRJVF6ZQF77p27cply5Zx9OjRHoto/rP10GKxsKioiO+++67oMxQ7aTQaUTcjIyPZt29fcQ7FGVREu8rU+JiYGFGW6OhoDhs2TGg9AgIC2KtXL/r4+AjBq7Lm1y/ZlPs4ePBgHj16lCRFv6Zci3JPJkyYwMuXL3PevHni/AkJCRw3bpzQpxw7dox1dXXMycmh0Wj8mRhcq9VSq9Wyc+fOnD59OpOSkjwW+vyl5VbCqGq1moGBgUxKShJ9r6Kjab8wpMViYUhIiMd9HTZsGDMzM8XCkaGhoXS73cImGo2G4eHhYqkNpa0rC7wq+SQnJ/+iMrdfNNBgMDA5OZlhYWFiTS6lj2hf/9PS0sSikyqVij4+PoyJiRHT+wMCAjh+/HiOHTuWQUFBQlNos9mo0+nodrs5a9YstrS0/Euexyqy3bioRCKRSCQSSQdDanYkEolEIpF0aKSzI5FIJBKJpEMjnR2JRCKRSCQdGunsSCQSiUQi6dBIZ0cikUgkEkmHRjo7EolEIpFIOjTS2ZFIJBKJRNKhkc6ORCKR3MKuXbugUqnEzw9IJJL/bKSzI5FIJBKJpEMjnR2JRCKRSCQdGunsSCSSXx2tra2YN28ewsPDYTQa0aNHD2zcuBHAf4WYKioq0L17d/Gr559//rlHHm+//TZiY2NhMBgQFhaGBQsWeHzf0tKCGTNmICQkBAaDAREREVi+fLlHmk8//RSJiYkwmUxITU3FsWPHbu+FSySS24J0diQSya+OefPmYc2aNfjzn/+ML774Ak8++STGjRuH3bt3izQFBQVYsGABPvnkE7hcLuTk5OD69esA2pyU3NxcjBkzBp999hlmz56NZ599FqtWrRLHP/DAAygtLcXixYtx5MgRLFu2DBaLxaMczzzzDBYsWIDq6mpotVrk5eX9S65fIpH8zyJ/CFQikfyqaGlpgcPhQFVVFVJSUsT+hx9+GFevXsXEiRORmZmJN954A6NHjwYAfPfddwgODsaqVauQm5uL++67DxcvXsS2bdvE8U8//TQqKirwxRdfoK6uDtHR0fjggw9wzz33/KwMu3btQmZmJqqqqtCvXz8AwNatWzFo0CA0NzfDy8vrNltBIpH8TyJHdiQSya+K48eP4+rVq+jfvz8sFovY1qxZgxMnToh07R0hh8OB6OhoHDlyBABw5MgRpKWleeSblpaG+vp63Lx5E7W1tdBoNMjIyPi7Zenevbv4PzAwEABw4cKF/+9rlEgk/1q0/+4CSCQSSXuampoAABUVFejUqZPHdwaDwcPh+X/FaDT+onQ6nU78r1KpALTpiSQSyX8WcmRHIpH8qoiJiYHBYMCZM2cQERHhsYWEhIh0H330kfi/oaEBdXV16NatGwCgW7du2Lt3r0e+e/fuRVRUFDQaDeLj49Ha2uqhAZJIJB0XObIjkUh+VXh7e+Opp57Ck08+idbWVtx55524fPky9u7dC6vVitDQUABAcXExnE4n/P398cwzz8DX1xfDhg0DAEyfPh1JSUmYO3cuRo8ejf3792PJkiVYunQpACAsLAwPPvgg8vLysHjxYvTo0QNffvklLly4gNzc3H/XpUskktuEdHYkEsmvjrlz58LlcmHevHk4efIk7HY7evbsiaKiIhFGmj9/PqZNm4b6+nokJCTg3XffhV6vBwD07NkTb775Jp577jnMnTsXgYGBKC4uxkMPPSTO8eqrr6KoqAj5+fn49ttv4Xa7UVRU9O+4XIlEcpuRs7EkEsl/FMpMqYaGBtjt9n93cSQSyX8AUrMjkUgkEomkQyOdHYlEIpFIJB0aGcaSSCQSiUTSoZEjOxKJRCKRSDo00tmRSCQSiUTSoZHOjkQikUgkkg6NdHYkEolEIpF0aKSzI5FIJBKJpEMjnR2JRCKRSCQdGunsSCQSiUQi6dBIZ0cikUgkEkmHRjo7EolEIpFIOjT/BwCNszJEZkJGAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plt.plot(np.arange(len(epochs_x)), epochs_acc, marker = '.')\n", + "plt.xlabel('epoch')\n", + "plt.ylabel('test accuracy')\n", + "plt.ylim(0, 100)\n", + "plt.xticks(np.arange(len(epochs_x)))\n", + "for i, txt in enumerate(epochs_acc):\n", + " if i%5 ==0 or i == epochs-1:\n", + " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "metadata": {} + }, + "outputs": [], + "source": [ + "# with open(f'{achitecture}-Training_Test-TM.npy', 'wb') as f:\n", + "# np.save(f, np.array(epochs_x))\n", + "# np.save(f, np.array(epochs_y))\n", + "# np.save(f, np.array(epochs_acc))" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "speck-rescnn", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.0" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 41f144f86be5c3489d319b2e060f0227395f3395 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Mon, 13 May 2024 12:33:21 +0200 Subject: [PATCH 083/379] more test cases for new DynapcnnNetwork --- sinabs/backend/dynapcnn/NIRGraphExtractor.py | 3 + .../DynapcnnNetwork-example_1.ipynb | 402 +++++++++++++++++ .../DynapcnnNetwork-example_2.ipynb | 396 ++++++++++++++++ .../DynapcnnNetwork-example_3.ipynb | 402 +++++++++++++++++ .../DynapcnnNetwork-example_4.ipynb | 380 ++++++++++++++++ .../DynapcnnNetwork-example_5.ipynb | 400 +++++++++++++++++ .../DynapcnnNetwork-example_5a.ipynb | 390 ++++++++++++++++ .../DynapcnnNetwork-example_6.ipynb | 422 ++++++++++++++++++ .../complex_network_structure.ipynb | 367 +++++++++++++++ .../using_SumPool2d/models/ResCSNN3.py | 122 ----- .../using_SumPool2d/models/ResSCNN1.py | 152 ------- .../using_SumPool2d/models/ResSCNN2.py | 143 ------ .../using_SumPool2d/models/ResSCNN3.py | 146 ------ .../using_SumPool2d/models/ResSCNN4.py | 149 ------- .../using_SumPool2d/models/ResSCNN5.py | 138 ------ .../using_SumPool2d/models/ResSCNN6.py | 141 ------ .../using_SumPool2d/models/ResSCNN7.py | 141 ------ .../using_SumPool2d/models/ResSCNN8.py | 146 ------ 18 files changed, 3162 insertions(+), 1278 deletions(-) create mode 100644 tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_1.ipynb create mode 100644 tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_2.ipynb create mode 100644 tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_3.ipynb create mode 100644 tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_4.ipynb create mode 100644 tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_5.ipynb create mode 100644 tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_5a.ipynb create mode 100644 tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_6.ipynb create mode 100644 tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/complex_network_structure.ipynb delete mode 100644 tests/test_nonsequential/using_SumPool2d/models/ResCSNN3.py delete mode 100644 tests/test_nonsequential/using_SumPool2d/models/ResSCNN1.py delete mode 100644 tests/test_nonsequential/using_SumPool2d/models/ResSCNN2.py delete mode 100644 tests/test_nonsequential/using_SumPool2d/models/ResSCNN3.py delete mode 100644 tests/test_nonsequential/using_SumPool2d/models/ResSCNN4.py delete mode 100644 tests/test_nonsequential/using_SumPool2d/models/ResSCNN5.py delete mode 100644 tests/test_nonsequential/using_SumPool2d/models/ResSCNN6.py delete mode 100644 tests/test_nonsequential/using_SumPool2d/models/ResSCNN7.py delete mode 100644 tests/test_nonsequential/using_SumPool2d/models/ResSCNN8.py diff --git a/sinabs/backend/dynapcnn/NIRGraphExtractor.py b/sinabs/backend/dynapcnn/NIRGraphExtractor.py index b2af6b41..750885cc 100644 --- a/sinabs/backend/dynapcnn/NIRGraphExtractor.py +++ b/sinabs/backend/dynapcnn/NIRGraphExtractor.py @@ -33,6 +33,9 @@ def __init__(self, spiking_model: nn.Module, dummy_input: torch.tensor): ### Publich Methods ### + def get_edges_list(self): + return self._edges_list + def remove_ignored_nodes(self, default_ignored_nodes): """ Recreates the edges list based on layers that 'DynapcnnNetwork' will ignore. This is done by setting the source (target) node of an edge where the source (target) node diff --git a/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_1.ipynb b/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_1.ipynb new file mode 100644 index 00000000..8b085ac9 --- /dev/null +++ b/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_1.ipynb @@ -0,0 +1,402 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "metadata": {} + }, + "outputs": [], + "source": [ + "import torch\n", + "import torch.nn as nn\n", + "from sinabs.backend.dynapcnn import DynapcnnNetworkGraph\n", + "from sinabs.layers import Merge, IAFSqueeze" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "metadata": {} + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "torch.manual_seed(0)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "metadata": {} + }, + "outputs": [], + "source": [ + "channels = 2\n", + "height = 34\n", + "width = 34\n", + "\n", + "input_shape = (channels, height, width)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Network Module\n", + "\n", + "We need to define a `nn.Module` implementing the network we want the chip to reproduce." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "metadata": {} + }, + "outputs": [], + "source": [ + "class SNN(nn.Module):\n", + " def __init__(self) -> None:\n", + " super().__init__()\n", + "\n", + " self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False) # node 0\n", + " self.iaf1 = IAFSqueeze(batch_size=1) # node 1\n", + " self.pool1 = nn.AvgPool2d(3,3) # node 2\n", + " self.pool1a = nn.AvgPool2d(4,4) # node 3\n", + "\n", + " self.conv2 = nn.Conv2d(10, 10, 4, 1, bias=False)# node 4\n", + " self.iaf2 = IAFSqueeze(batch_size=1) # node 6\n", + "\n", + " self.conv3 = nn.Conv2d(10, 1, 2, 1, bias=False) # node 8\n", + " self.iaf3 = IAFSqueeze(batch_size=1) # node 9\n", + "\n", + " self.flat = nn.Flatten()\n", + "\n", + " self.fc1 = nn.Linear(49, 500, bias=False) # node 10\n", + " self.iaf4 = IAFSqueeze(batch_size=1) # node 11\n", + " \n", + " self.fc2 = nn.Linear(500, 10, bias=False) # node 12\n", + " self.iaf5 = IAFSqueeze(batch_size=1) # node 13\n", + "\n", + " self.adder = Merge()\n", + "\n", + " def forward(self, x):\n", + " \n", + " con1_out = self.conv1(x)\n", + " iaf1_out = self.iaf1(con1_out)\n", + " pool1_out = self.pool1(iaf1_out)\n", + " pool1a_out = self.pool1a(iaf1_out)\n", + "\n", + " conv2_out = self.conv2(pool1_out)\n", + " iaf2_out = self.iaf2(conv2_out)\n", + "\n", + " conv3_out = self.conv3(self.adder(pool1a_out, iaf2_out))\n", + " iaf3_out = self.iaf3(conv3_out)\n", + "\n", + " flat_out = self.flat(iaf3_out)\n", + " \n", + " fc1_out = self.fc1(flat_out)\n", + " iaf4_out = self.iaf4(fc1_out)\n", + " fc2_out = self.fc2(iaf4_out)\n", + " iaf5_out = self.iaf5(fc2_out)\n", + "\n", + " return iaf5_out" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "metadata": {} + }, + "outputs": [], + "source": [ + "snn = SNN()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The following exemplifies how each of the layers in the SNN should be grouped together to form a `DynapcnnLayer` instance. Let's check the shapes of the tensors each of the (original) layers in the model inputs and outputs by feeding a fake input through the model:" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "metadata": {} + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "DynapcnnLayer 0: ... [1, 2, 34, 34]\n", + " conv1: [1, 10, 33, 33]\n", + " iaf1: [1, 10, 33, 33]\n", + " pool1: [1, 10, 11, 11]\n", + " pool1a: [1, 10, 8, 8]\n", + "\n", + "DynapcnnLayer 1: ... [1, 10, 11, 11]\n", + " conv2: [1, 10, 8, 8]\n", + " iaf2: [1, 10, 8, 8]\n", + "\n", + "DynapcnnLayer 2: ... [1, 10, 8, 8] [ Merge(pool1a, iaf2_out) ]\n", + " conv3: [1, 1, 7, 7]\n", + " iaf3: [1, 1, 7, 7]\n", + "\n", + "DynapcnnLayer 3: ... [1, 49]\n", + " fc1: [1, 500]\n", + " iaf4: [1, 500]\n", + "\n", + "DynapcnnLayer 4: ... [1, 500]\n", + " fc2: [1, 10]\n", + " iaf5: [1, 10]\n", + "\n" + ] + } + ], + "source": [ + "x = torch.randn((1, *input_shape))\n", + "\n", + "print(f'DynapcnnLayer 0: ... {list(x.shape)}')\n", + "con1_out = snn.conv1(x)\n", + "print(f' conv1: {list(con1_out.shape)}')\n", + "iaf1_out = snn.iaf1(con1_out)\n", + "print(f' iaf1: {list(iaf1_out.shape)}')\n", + "pool1_out = snn.pool1(iaf1_out)\n", + "print(f' pool1: {list(pool1_out.shape)}')\n", + "pool1a_out = snn.pool1a(iaf1_out)\n", + "print(f' pool1a: {list(pool1a_out.shape)}\\n')\n", + "\n", + "print(f'DynapcnnLayer 1: ... {list(pool1_out.shape)}')\n", + "conv2_out = snn.conv2(pool1_out)\n", + "print(f' conv2: {list(conv2_out.shape)}')\n", + "iaf2_out = snn.iaf2(conv2_out)\n", + "print(f' iaf2: {list(iaf2_out.shape)}\\n')\n", + "# pool2_out = snn.pool2(iaf2_out)\n", + "# print(f' pool2: {list(pool2_out.shape)}\\n')\n", + "\n", + "added = snn.adder(pool1a_out, iaf2_out)\n", + "\n", + "print(f'DynapcnnLayer 2: ... {list(added.shape)} [ Merge(pool1a, iaf2_out) ]')\n", + "conv3_out = snn.conv3(added)\n", + "print(f' conv3: {list(conv3_out.shape)}')\n", + "iaf3_out = snn.iaf3(conv3_out)\n", + "print(f' iaf3: {list(iaf3_out.shape)}\\n')\n", + "\n", + "flat_out = snn.flat(iaf3_out)\n", + "\n", + "print(f'DynapcnnLayer 3: ... {list(flat_out.shape)}')\n", + "fc1_out = snn.fc1(flat_out)\n", + "print(f' fc1: {list(fc1_out.shape)}')\n", + "iaf4_out = snn.iaf4(fc1_out)\n", + "print(f' iaf4: {list(iaf4_out.shape)}\\n')\n", + "\n", + "\n", + "print(f'DynapcnnLayer 4: ... {list(iaf4_out.shape)}')\n", + "fc2_out = snn.fc2(iaf4_out)\n", + "print(f' fc2: {list(fc2_out.shape)}')\n", + "iaf5_out = snn.iaf5(fc2_out)\n", + "print(f' iaf5: {list(iaf5_out.shape)}\\n')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## DynapcnnNetwork Class\n", + "\n", + "In the constructor of `DynapcnnNetworkGraph` the SNN passed as argument (defined as a `nn.Module`) will be parsed such that each layer is represented in a computational graph (using `nirtorch.extract_torch_graph`). \n", + "\n", + "The layers are the `nodes` of the graph, while their connectivity (how the outputs from a layer are sent to other layers) is represented as `edges`, represented in a `list` of `tuples`.\n", + "\n", + "Once the constructor finishes its initialization, the `hw_model.dynapcnn_layers` property is a dictionary where each entry represents the ID of a `DynapcnnLayer` instance (an `int` from `0` to `L`), with this entry containing a `DynapcnnLayer` instance where a subset of the layers in the original SNN has been incorporated into, the core such instance has been assigned to, and the list of `DynapcnnLayer` instances (their IDs) the layer targets." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "metadata": {} + }, + "outputs": [], + "source": [ + "hw_model = DynapcnnNetworkGraph(\n", + " snn,\n", + " discretize=True,\n", + " input_shape=input_shape\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The `hw_model.to()` call will figure out into which core eac `DynapcnnLayer` instance will be assigned to. Once this assingment is made the instance itself is used to configure the `CNNLayerConfig` instance representing the core's configuration.\n", + "\n", + "If the cores' configuration is valid, each `DynapcnnLayer` instance and their respective destinations will be used to create a computational graph that encodes how the `forward` method of `hw_model.network` (a `nn.Module` using the `DynapcnnLayer` instances) propagates that through the network." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "metadata": {} + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Network is valid\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "hw_model.to(device=\"speck2fmodule:0\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The layers comprising our `hw_model` and their respective metadata can be inspected by calling `print` on a `DynapcnnNetworkGraph` instance." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "metadata": {} + }, + "outputs": [ + { + "ename": "AttributeError", + "evalue": "'DynapcnnNetworkGraph' object has no attribute 'get_forward_edges'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[9], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[38;5;28mprint\u001b[39m(hw_model\u001b[38;5;241m.\u001b[39mget_forward_edges())\n", + "\u001b[0;31mAttributeError\u001b[0m: 'DynapcnnNetworkGraph' object has no attribute 'get_forward_edges'" + ] + } + ], + "source": [ + "print(hw_model.get_network_module().get_forward_edges())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "metadata": {} + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "---- DynapcnnLayer 0 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 0): Conv2d(2, 10, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + "(node 1): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "(node 2): SumPool2d(norm_type=1, kernel_size=(3, 3), stride=None, ceil_mode=False)\n", + "(node 3): SumPool2d(norm_type=1, kernel_size=(4, 4), stride=None, ceil_mode=False)\n", + "> layer destinations: [1, 2]\n", + "> assigned core: 0\n", + "\n", + "---- DynapcnnLayer 1 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 4): Conv2d(10, 10, kernel_size=(4, 4), stride=(1, 1), bias=False)\n", + "(node 6): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [2]\n", + "> assigned core: 1\n", + "\n", + "---- DynapcnnLayer 2 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 7): Conv2d(10, 1, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + "(node 8): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [3]\n", + "> assigned core: 2\n", + "\n", + "---- DynapcnnLayer 3 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 9): Conv2d(1, 500, kernel_size=(7, 7), stride=(1, 1), bias=False)\n", + "(node 10): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [4]\n", + "> assigned core: 3\n", + "\n", + "---- DynapcnnLayer 4 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 11): Conv2d(500, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + "(node 12): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: []\n", + "> assigned core: 4\n", + "\n", + "\n" + ] + } + ], + "source": [ + "print(hw_model)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "speck-rescnn", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_2.ipynb b/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_2.ipynb new file mode 100644 index 00000000..abdda89e --- /dev/null +++ b/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_2.ipynb @@ -0,0 +1,396 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "metadata": {} + }, + "outputs": [], + "source": [ + "import torch\n", + "import torch.nn as nn\n", + "from sinabs.backend.dynapcnn import DynapcnnNetworkGraph\n", + "from sinabs.layers import Merge, IAFSqueeze" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "metadata": {} + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "torch.manual_seed(0)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "metadata": {} + }, + "outputs": [], + "source": [ + "channels = 2\n", + "height = 34\n", + "width = 34\n", + "\n", + "input_shape = (channels, height, width)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Network Module\n", + "\n", + "We need to define a `nn.Module` implementing the network we want the chip to reproduce." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "metadata": {} + }, + "outputs": [], + "source": [ + "class SNN(nn.Module):\n", + " def __init__(self) -> None:\n", + " super().__init__()\n", + "\n", + " self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False)\n", + " self.conv1_iaf = IAFSqueeze(batch_size=1)\n", + " self.pool1 = nn.AvgPool2d(3,3)\n", + " self.pool1a = nn.AvgPool2d(4,4)\n", + "\n", + " self.conv2 = nn.Conv2d(10, 10, 4, 1, bias=False)\n", + " self.conv2_iaf = IAFSqueeze(batch_size=1)\n", + "\n", + " self.conv3 = nn.Conv2d(10, 1, 2, 1, bias=False)\n", + " self.conv3_iaf = IAFSqueeze(batch_size=1)\n", + "\n", + " self.flat = nn.Flatten()\n", + "\n", + " self.fc1 = nn.Linear(49, 100, bias=False)\n", + " self.fc1_iaf = IAFSqueeze(batch_size=1)\n", + " \n", + " self.fc2 = nn.Linear(100, 100, bias=False)\n", + " self.fc2_iaf = IAFSqueeze(batch_size=1)\n", + "\n", + " self.fc3 = nn.Linear(100, 10, bias=False)\n", + " self.fc3_iaf = IAFSqueeze(batch_size=1)\n", + "\n", + " self.merge1 = Merge()\n", + "\n", + " def forward(self, x):\n", + " # -- conv. block 1 --\n", + " con1_out = self.conv1(x)\n", + " conv1_iaf_out = self.conv1_iaf(con1_out)\n", + " pool1_out = self.pool1(conv1_iaf_out)\n", + " pool1a_out = self.pool1a(conv1_iaf_out)\n", + " # -- conv. block 2 --\n", + " conv2_out = self.conv2(pool1_out)\n", + " conv2_iaf_out = self.conv2_iaf(conv2_out)\n", + " # -- conv. block 3 --\n", + " merge1_out = self.merge1(pool1a_out, conv2_iaf_out)\n", + " conv3_out = self.conv3(merge1_out)\n", + " conv3_iaf_out = self.conv3_iaf(conv3_out)\n", + " flat_out = self.flat(conv3_iaf_out)\n", + " # -- fc clock 1 --\n", + " fc1_out = self.fc1(flat_out)\n", + " fc1_iaf_out = self.fc1_iaf(fc1_out)\n", + " # -- fc clock 2 --\n", + " fc2_out = self.fc2(fc1_iaf_out)\n", + " fc2_iaf_out = self.fc2_iaf(fc2_out)\n", + " # -- fc clock 3 --\n", + " fc3_out = self.fc3(fc2_iaf_out)\n", + " fc3_iaf_out = self.fc3_iaf(fc3_out)\n", + "\n", + " return fc3_iaf_out" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "metadata": {} + }, + "outputs": [], + "source": [ + "snn = SNN()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The following exemplifies how each of the layers in the SNN should be grouped together to form a `DynapcnnLayer` instance. Let's check the shapes of the tensors each of the (original) layers in the model inputs and outputs by feeding a fake input through the model:" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "metadata": {} + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "torch.Size([1, 10, 33, 33])\n", + "torch.Size([1, 10, 33, 33])\n", + "torch.Size([1, 10, 11, 11])\n", + "torch.Size([1, 10, 8, 8])\n", + "torch.Size([1, 10, 8, 8])\n", + "torch.Size([1, 10, 8, 8])\n", + "torch.Size([1, 10, 8, 8])\n", + "torch.Size([1, 1, 7, 7])\n", + "torch.Size([1, 1, 7, 7])\n", + "torch.Size([1, 49])\n", + "torch.Size([1, 100])\n", + "torch.Size([1, 100])\n", + "torch.Size([1, 100])\n", + "torch.Size([1, 100])\n", + "torch.Size([1, 10])\n", + "torch.Size([1, 10])\n" + ] + } + ], + "source": [ + "x = torch.randn((1, *input_shape))\n", + "\n", + "# -- conv. block 1 --\n", + "con1_out = snn.conv1(x)\n", + "print(con1_out.shape)\n", + "conv1_iaf_out = snn.conv1_iaf(con1_out)\n", + "print(conv1_iaf_out.shape)\n", + "pool1_out = snn.pool1(conv1_iaf_out)\n", + "print(pool1_out.shape)\n", + "pool1a_out = snn.pool1a(conv1_iaf_out)\n", + "print(pool1a_out.shape)\n", + "# -- conv. block 2 --\n", + "conv2_out = snn.conv2(pool1_out)\n", + "print(conv2_out.shape)\n", + "conv2_iaf_out = snn.conv2_iaf(conv2_out)\n", + "print(conv2_iaf_out.shape)\n", + "# -- conv. block 3 --\n", + "merge1_out = snn.merge1(pool1a_out, conv2_iaf_out)\n", + "print(merge1_out.shape)\n", + "conv3_out = snn.conv3(merge1_out)\n", + "print(conv3_out.shape)\n", + "conv3_iaf_out = snn.conv3_iaf(conv3_out)\n", + "print(conv3_iaf_out.shape)\n", + "flat_out = snn.flat(conv3_iaf_out)\n", + "print(flat_out.shape)\n", + "# -- fc clock 1 --\n", + "fc1_out = snn.fc1(flat_out)\n", + "print(fc1_out.shape)\n", + "fc1_iaf_out = snn.fc1_iaf(fc1_out)\n", + "print(fc1_iaf_out.shape)\n", + "# -- fc clock 2 --\n", + "fc2_out = snn.fc2(fc1_iaf_out)\n", + "print(fc2_out.shape)\n", + "fc2_iaf_out = snn.fc2_iaf(fc2_out)\n", + "print(fc2_iaf_out.shape)\n", + "# -- fc clock 3 --\n", + "fc3_out = snn.fc3(fc2_iaf_out)\n", + "print(fc3_out.shape)\n", + "fc3_iaf_out = snn.fc3_iaf(fc3_out)\n", + "print(fc3_iaf_out.shape)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## DynapcnnNetwork Class\n", + "\n", + "In the constructor of `DynapcnnNetworkGraph` the SNN passed as argument (defined as a `nn.Module`) will be parsed such that each layer is represented in a computational graph (using `nirtorch.extract_torch_graph`). \n", + "\n", + "The layers are the `nodes` of the graph, while their connectivity (how the outputs from a layer are sent to other layers) is represented as `edges`, represented in a `list` of `tuples`.\n", + "\n", + "Once the constructor finishes its initialization, the `hw_model.dynapcnn_layers` property is a dictionary where each entry represents the ID of a `DynapcnnLayer` instance (an `int` from `0` to `L`), with this entry containing a `DynapcnnLayer` instance where a subset of the layers in the original SNN has been incorporated into, the core such instance has been assigned to, and the list of `DynapcnnLayer` instances (their IDs) the layer targets." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "metadata": {} + }, + "outputs": [], + "source": [ + "hw_model = DynapcnnNetworkGraph(\n", + " snn,\n", + " discretize=True,\n", + " input_shape=input_shape\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The `hw_model.to()` call will figure out into which core eac `DynapcnnLayer` instance will be assigned to. Once this assingment is made the instance itself is used to configure the `CNNLayerConfig` instance representing the core's configuration.\n", + "\n", + "If the cores' configuration is valid, each `DynapcnnLayer` instance and their respective destinations will be used to create a computational graph that encodes how the `forward` method of `hw_model.network` (a `nn.Module` using the `DynapcnnLayer` instances) propagates that through the network." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "metadata": {} + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Network is valid\n", + "(0, 1)\n", + "(0, 2)\n", + "(1, 2)\n", + "(2, 3)\n", + "(3, 4)\n", + "(4, 5)\n" + ] + }, + { + "ename": "RuntimeError", + "evalue": "Device is already opened!", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[8], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m hw_model\u001b[38;5;241m.\u001b[39mto(device\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mspeck2fmodule:0\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", + "File \u001b[0;32m~/Documents/github/sinabs/sinabs/backend/dynapcnn/dynapcnn_network_graph.py:172\u001b[0m, in \u001b[0;36mDynapcnnNetworkGraph.to\u001b[0;34m(self, device, chip_layers_ordering, monitor_layers, config_modifier, slow_clk_frequency)\u001b[0m\n\u001b[1;32m 163\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m device_name \u001b[38;5;129;01min\u001b[39;00m ChipFactory\u001b[38;5;241m.\u001b[39msupported_devices: \u001b[38;5;66;03m# pragma: no cover\u001b[39;00m\n\u001b[1;32m 165\u001b[0m config \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmake_config( \u001b[38;5;66;03m# generate config.\u001b[39;00m\n\u001b[1;32m 166\u001b[0m chip_layers_ordering\u001b[38;5;241m=\u001b[39mchip_layers_ordering,\n\u001b[1;32m 167\u001b[0m device\u001b[38;5;241m=\u001b[39mdevice,\n\u001b[1;32m 168\u001b[0m monitor_layers\u001b[38;5;241m=\u001b[39mmonitor_layers,\n\u001b[1;32m 169\u001b[0m config_modifier\u001b[38;5;241m=\u001b[39mconfig_modifier,\n\u001b[1;32m 170\u001b[0m )\n\u001b[0;32m--> 172\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msamna_device \u001b[38;5;241m=\u001b[39m open_device(device) \u001b[38;5;66;03m# apply configuration to device.\u001b[39;00m\n\u001b[1;32m 173\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msamna_device\u001b[38;5;241m.\u001b[39mget_model()\u001b[38;5;241m.\u001b[39mapply_configuration(config)\n\u001b[1;32m 174\u001b[0m time\u001b[38;5;241m.\u001b[39msleep(\u001b[38;5;241m1\u001b[39m)\n", + "File \u001b[0;32m~/Documents/github/sinabs/sinabs/backend/dynapcnn/io.py:256\u001b[0m, in \u001b[0;36mopen_device\u001b[0;34m(device_id)\u001b[0m\n\u001b[1;32m 254\u001b[0m device_map \u001b[38;5;241m=\u001b[39m get_device_map()\n\u001b[1;32m 255\u001b[0m device_info \u001b[38;5;241m=\u001b[39m device_map[device_id]\n\u001b[0;32m--> 256\u001b[0m device_handle \u001b[38;5;241m=\u001b[39m samna\u001b[38;5;241m.\u001b[39mdevice\u001b[38;5;241m.\u001b[39mopen_device(device_info)\n\u001b[1;32m 258\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m device_handle \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 259\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m device_handle\n", + "\u001b[0;31mRuntimeError\u001b[0m: Device is already opened!" + ] + } + ], + "source": [ + "hw_model.to(device=\"speck2fmodule:0\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The layers comprising our `hw_model` and their respective metadata can be inspected by calling `print` on a `DynapcnnNetworkGraph` instance." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "metadata": {} + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "---- DynapcnnLayer 0 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 0): Conv2d(2, 10, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + "(node 1): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "(node 2): SumPool2d(norm_type=1, kernel_size=(3, 3), stride=None, ceil_mode=False)\n", + "(node 3): SumPool2d(norm_type=1, kernel_size=(4, 4), stride=None, ceil_mode=False)\n", + "> layer destinations: [1, 2]\n", + "> assigned core: 0\n", + "\n", + "---- DynapcnnLayer 1 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 4): Conv2d(10, 10, kernel_size=(4, 4), stride=(1, 1), bias=False)\n", + "(node 6): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [2]\n", + "> assigned core: 1\n", + "\n", + "---- DynapcnnLayer 2 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 7): Conv2d(10, 1, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + "(node 8): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [3]\n", + "> assigned core: 2\n", + "\n", + "---- DynapcnnLayer 3 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 9): Conv2d(1, 100, kernel_size=(7, 7), stride=(1, 1), bias=False)\n", + "(node 10): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [4]\n", + "> assigned core: 3\n", + "\n", + "---- DynapcnnLayer 4 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 11): Conv2d(100, 100, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + "(node 12): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [5]\n", + "> assigned core: 4\n", + "\n", + "---- DynapcnnLayer 5 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 13): Conv2d(100, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + "(node 14): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: []\n", + "> assigned core: 5\n", + "\n", + "\n" + ] + } + ], + "source": [ + "print(hw_model)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "speck-rescnn", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_3.ipynb b/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_3.ipynb new file mode 100644 index 00000000..5b8e7216 --- /dev/null +++ b/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_3.ipynb @@ -0,0 +1,402 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "metadata": {} + }, + "outputs": [], + "source": [ + "import torch\n", + "import torch.nn as nn\n", + "from sinabs.backend.dynapcnn import DynapcnnNetworkGraph\n", + "from sinabs.layers import Merge, IAFSqueeze" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "metadata": {} + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "torch.manual_seed(0)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "metadata": {} + }, + "outputs": [], + "source": [ + "channels = 2\n", + "height = 34\n", + "width = 34\n", + "\n", + "input_shape = (channels, height, width)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Network Module\n", + "\n", + "We need to define a `nn.Module` implementing the network we want the chip to reproduce." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "metadata": {} + }, + "outputs": [], + "source": [ + "class SNN(nn.Module):\n", + " def __init__(self) -> None:\n", + " super().__init__()\n", + "\n", + " self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False)\n", + " self.conv1_iaf = IAFSqueeze(batch_size=1)\n", + " self.pool1 = nn.AvgPool2d(3,3)\n", + " self.pool1a = nn.AvgPool2d(4,4)\n", + "\n", + " self.conv2 = nn.Conv2d(10, 10, 4, 1, bias=False)\n", + " self.conv2_iaf = IAFSqueeze(batch_size=1)\n", + "\n", + " self.conv3 = nn.Conv2d(10, 1, 2, 1, bias=False)\n", + " self.conv3_iaf = IAFSqueeze(batch_size=1)\n", + "\n", + " self.flat = nn.Flatten()\n", + "\n", + " self.fc1 = nn.Linear(49, 100, bias=False)\n", + " self.fc1_iaf = IAFSqueeze(batch_size=1)\n", + " \n", + " self.fc2 = nn.Linear(100, 100, bias=False)\n", + " self.fc2_iaf = IAFSqueeze(batch_size=1)\n", + "\n", + " self.fc3 = nn.Linear(100, 10, bias=False)\n", + " self.fc3_iaf = IAFSqueeze(batch_size=1)\n", + "\n", + " self.merge1 = Merge()\n", + " self.merge2 = Merge()\n", + "\n", + " def forward(self, x):\n", + " # -- conv. block 0 --\n", + " con1_out = self.conv1(x)\n", + " conv1_iaf_out = self.conv1_iaf(con1_out)\n", + " pool1_out = self.pool1(conv1_iaf_out)\n", + " pool1a_out = self.pool1a(conv1_iaf_out)\n", + " # -- conv. block 1 --\n", + " conv2_out = self.conv2(pool1_out)\n", + " conv2_iaf_out = self.conv2_iaf(conv2_out)\n", + " # -- conv. block 2 --\n", + " merge1_out = self.merge1(pool1a_out, conv2_iaf_out)\n", + " conv3_out = self.conv3(merge1_out)\n", + " conv3_iaf_out = self.conv3_iaf(conv3_out)\n", + " flat_out = self.flat(conv3_iaf_out)\n", + " # -- fc clock 3 --\n", + " fc1_out = self.fc1(flat_out)\n", + " fc1_iaf_out = self.fc1_iaf(fc1_out)\n", + " # -- fc clock 4 --\n", + " fc2_out = self.fc2(fc1_iaf_out)\n", + " fc2_iaf_out = self.fc2_iaf(fc2_out)\n", + " # -- fc clock 5 --\n", + " merge2_out = self.merge2(fc1_iaf_out, fc2_iaf_out)\n", + " fc3_out = self.fc3(merge2_out)\n", + " fc3_iaf_out = self.fc3_iaf(fc3_out)\n", + "\n", + " return fc3_iaf_out" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "metadata": {} + }, + "outputs": [], + "source": [ + "snn = SNN()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The following exemplifies how each of the layers in the SNN should be grouped together to form a `DynapcnnLayer` instance. Let's check the shapes of the tensors each of the (original) layers in the model inputs and outputs by feeding a fake input through the model:" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "metadata": {} + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "torch.Size([1, 10, 33, 33])\n", + "torch.Size([1, 10, 33, 33])\n", + "torch.Size([1, 10, 11, 11])\n", + "torch.Size([1, 10, 8, 8])\n", + "torch.Size([1, 10, 8, 8])\n", + "torch.Size([1, 10, 8, 8])\n", + "merge1: torch.Size([1, 10, 8, 8])\n", + "torch.Size([1, 1, 7, 7])\n", + "torch.Size([1, 1, 7, 7])\n", + "torch.Size([1, 49])\n", + "torch.Size([1, 100])\n", + "torch.Size([1, 100])\n", + "torch.Size([1, 100])\n", + "torch.Size([1, 100])\n", + "merge2: torch.Size([1, 100])\n", + "torch.Size([1, 10])\n", + "torch.Size([1, 10])\n" + ] + } + ], + "source": [ + "x = torch.randn((1, *input_shape))\n", + "\n", + "# -- conv. block 0 --\n", + "con1_out = snn.conv1(x)\n", + "print(con1_out.shape)\n", + "conv1_iaf_out = snn.conv1_iaf(con1_out)\n", + "print(conv1_iaf_out.shape)\n", + "pool1_out = snn.pool1(conv1_iaf_out)\n", + "print(pool1_out.shape)\n", + "pool1a_out = snn.pool1a(conv1_iaf_out)\n", + "print(pool1a_out.shape)\n", + "# -- conv. block 1 --\n", + "conv2_out = snn.conv2(pool1_out)\n", + "print(conv2_out.shape)\n", + "conv2_iaf_out = snn.conv2_iaf(conv2_out)\n", + "print(conv2_iaf_out.shape)\n", + "# -- conv. block 2 --\n", + "merge1_out = snn.merge1(pool1a_out, conv2_iaf_out)\n", + "print(f'merge1: {merge1_out.shape}')\n", + "conv3_out = snn.conv3(merge1_out)\n", + "print(conv3_out.shape)\n", + "conv3_iaf_out = snn.conv3_iaf(conv3_out)\n", + "print(conv3_iaf_out.shape)\n", + "flat_out = snn.flat(conv3_iaf_out)\n", + "print(flat_out.shape)\n", + "# -- fc clock 3 --\n", + "fc1_out = snn.fc1(flat_out)\n", + "print(fc1_out.shape)\n", + "fc1_iaf_out = snn.fc1_iaf(fc1_out)\n", + "print(fc1_iaf_out.shape)\n", + "# -- fc clock 4 --\n", + "fc2_out = snn.fc2(fc1_iaf_out)\n", + "print(fc2_out.shape)\n", + "fc2_iaf_out = snn.fc2_iaf(fc2_out)\n", + "print(fc2_iaf_out.shape)\n", + "# -- fc clock 5 --\n", + "merge2_out = snn.merge2(fc1_iaf_out, fc2_iaf_out)\n", + "print(f'merge2: {merge2_out.shape}')\n", + "fc3_out = snn.fc3(merge2_out)\n", + "print(fc3_out.shape)\n", + "fc3_iaf_out = snn.fc3_iaf(fc3_out)\n", + "print(fc3_iaf_out.shape)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## DynapcnnNetwork Class\n", + "\n", + "In the constructor of `DynapcnnNetworkGraph` the SNN passed as argument (defined as a `nn.Module`) will be parsed such that each layer is represented in a computational graph (using `nirtorch.extract_torch_graph`). \n", + "\n", + "The layers are the `nodes` of the graph, while their connectivity (how the outputs from a layer are sent to other layers) is represented as `edges`, represented in a `list` of `tuples`.\n", + "\n", + "Once the constructor finishes its initialization, the `hw_model.dynapcnn_layers` property is a dictionary where each entry represents the ID of a `DynapcnnLayer` instance (an `int` from `0` to `L`), with this entry containing a `DynapcnnLayer` instance where a subset of the layers in the original SNN has been incorporated into, the core such instance has been assigned to, and the list of `DynapcnnLayer` instances (their IDs) the layer targets." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "metadata": {} + }, + "outputs": [], + "source": [ + "hw_model = DynapcnnNetworkGraph(\n", + " snn,\n", + " discretize=True,\n", + " input_shape=input_shape\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The `hw_model.to()` call will figure out into which core eac `DynapcnnLayer` instance will be assigned to. Once this assingment is made the instance itself is used to configure the `CNNLayerConfig` instance representing the core's configuration.\n", + "\n", + "If the cores' configuration is valid, each `DynapcnnLayer` instance and their respective destinations will be used to create a computational graph that encodes how the `forward` method of `hw_model.network` (a `nn.Module` using the `DynapcnnLayer` instances) propagates that through the network." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "metadata": {} + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Network is valid\n", + "(0, 1)\n", + "(0, 2)\n", + "(1, 2)\n", + "(2, 3)\n", + "(3, 4)\n", + "(3, 5)\n", + "(4, 5)\n" + ] + }, + { + "ename": "RuntimeError", + "evalue": "Device is already opened!", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[8], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m hw_model\u001b[38;5;241m.\u001b[39mto(device\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mspeck2fmodule:0\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", + "File \u001b[0;32m~/Documents/github/sinabs/sinabs/backend/dynapcnn/dynapcnn_network_graph.py:172\u001b[0m, in \u001b[0;36mDynapcnnNetworkGraph.to\u001b[0;34m(self, device, chip_layers_ordering, monitor_layers, config_modifier, slow_clk_frequency)\u001b[0m\n\u001b[1;32m 163\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m device_name \u001b[38;5;129;01min\u001b[39;00m ChipFactory\u001b[38;5;241m.\u001b[39msupported_devices: \u001b[38;5;66;03m# pragma: no cover\u001b[39;00m\n\u001b[1;32m 165\u001b[0m config \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmake_config( \u001b[38;5;66;03m# generate config.\u001b[39;00m\n\u001b[1;32m 166\u001b[0m chip_layers_ordering\u001b[38;5;241m=\u001b[39mchip_layers_ordering,\n\u001b[1;32m 167\u001b[0m device\u001b[38;5;241m=\u001b[39mdevice,\n\u001b[1;32m 168\u001b[0m monitor_layers\u001b[38;5;241m=\u001b[39mmonitor_layers,\n\u001b[1;32m 169\u001b[0m config_modifier\u001b[38;5;241m=\u001b[39mconfig_modifier,\n\u001b[1;32m 170\u001b[0m )\n\u001b[0;32m--> 172\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msamna_device \u001b[38;5;241m=\u001b[39m open_device(device) \u001b[38;5;66;03m# apply configuration to device.\u001b[39;00m\n\u001b[1;32m 173\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msamna_device\u001b[38;5;241m.\u001b[39mget_model()\u001b[38;5;241m.\u001b[39mapply_configuration(config)\n\u001b[1;32m 174\u001b[0m time\u001b[38;5;241m.\u001b[39msleep(\u001b[38;5;241m1\u001b[39m)\n", + "File \u001b[0;32m~/Documents/github/sinabs/sinabs/backend/dynapcnn/io.py:256\u001b[0m, in \u001b[0;36mopen_device\u001b[0;34m(device_id)\u001b[0m\n\u001b[1;32m 254\u001b[0m device_map \u001b[38;5;241m=\u001b[39m get_device_map()\n\u001b[1;32m 255\u001b[0m device_info \u001b[38;5;241m=\u001b[39m device_map[device_id]\n\u001b[0;32m--> 256\u001b[0m device_handle \u001b[38;5;241m=\u001b[39m samna\u001b[38;5;241m.\u001b[39mdevice\u001b[38;5;241m.\u001b[39mopen_device(device_info)\n\u001b[1;32m 258\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m device_handle \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 259\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m device_handle\n", + "\u001b[0;31mRuntimeError\u001b[0m: Device is already opened!" + ] + } + ], + "source": [ + "hw_model.to(device=\"speck2fmodule:0\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The layers comprising our `hw_model` and their respective metadata can be inspected by calling `print` on a `DynapcnnNetworkGraph` instance." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "metadata": {} + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "---- DynapcnnLayer 0 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 0): Conv2d(2, 10, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + "(node 1): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "(node 2): SumPool2d(norm_type=1, kernel_size=(3, 3), stride=None, ceil_mode=False)\n", + "(node 3): SumPool2d(norm_type=1, kernel_size=(4, 4), stride=None, ceil_mode=False)\n", + "> layer destinations: [1, 2]\n", + "> assigned core: 0\n", + "\n", + "---- DynapcnnLayer 1 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 4): Conv2d(10, 10, kernel_size=(4, 4), stride=(1, 1), bias=False)\n", + "(node 6): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [2]\n", + "> assigned core: 1\n", + "\n", + "---- DynapcnnLayer 2 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 7): Conv2d(10, 1, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + "(node 8): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [3]\n", + "> assigned core: 2\n", + "\n", + "---- DynapcnnLayer 3 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 9): Conv2d(1, 100, kernel_size=(7, 7), stride=(1, 1), bias=False)\n", + "(node 10): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [4, 5]\n", + "> assigned core: 3\n", + "\n", + "---- DynapcnnLayer 4 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 11): Conv2d(100, 100, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + "(node 13): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [5]\n", + "> assigned core: 4\n", + "\n", + "---- DynapcnnLayer 5 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 14): Conv2d(100, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + "(node 15): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: []\n", + "> assigned core: 5\n", + "\n", + "\n" + ] + } + ], + "source": [ + "print(hw_model)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "speck-rescnn", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_4.ipynb b/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_4.ipynb new file mode 100644 index 00000000..1f190961 --- /dev/null +++ b/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_4.ipynb @@ -0,0 +1,380 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "metadata": {} + }, + "outputs": [], + "source": [ + "import torch\n", + "import torch.nn as nn\n", + "from sinabs.backend.dynapcnn import DynapcnnNetworkGraph\n", + "from sinabs.layers import Merge, IAFSqueeze" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "metadata": {} + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "torch.manual_seed(0)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "metadata": {} + }, + "outputs": [], + "source": [ + "channels = 2\n", + "height = 34\n", + "width = 34\n", + "\n", + "input_shape = (channels, height, width)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Network Module\n", + "\n", + "We need to define a `nn.Module` implementing the network we want the chip to reproduce." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "metadata": {} + }, + "outputs": [], + "source": [ + "class SNN(nn.Module):\n", + " def __init__(self) -> None:\n", + " super().__init__()\n", + "\n", + " self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False)\n", + " self.conv1_iaf = IAFSqueeze(batch_size=1)\n", + " self.pool1 = nn.AvgPool2d(3,3)\n", + " self.pool1a = nn.AvgPool2d(4,4)\n", + "\n", + " self.conv2 = nn.Conv2d(10, 10, 4, 1, bias=False)\n", + " self.conv2_iaf = IAFSqueeze(batch_size=1)\n", + "\n", + " self.conv3 = nn.Conv2d(10, 10, 2, 1, bias=False)\n", + " self.conv3_iaf = IAFSqueeze(batch_size=1)\n", + "\n", + " self.conv4 = nn.Conv2d(10, 1, 2, 1, bias=False)\n", + " self.conv4_iaf = IAFSqueeze(batch_size=1)\n", + "\n", + " self.flat = nn.Flatten()\n", + "\n", + " self.fc1 = nn.Linear(36, 100, bias=False)\n", + " self.fc1_iaf = IAFSqueeze(batch_size=1)\n", + " \n", + " self.fc2 = nn.Linear(100, 100, bias=False)\n", + " self.fc2_iaf = IAFSqueeze(batch_size=1)\n", + "\n", + " self.fc3 = nn.Linear(100, 10, bias=False)\n", + " self.fc3_iaf = IAFSqueeze(batch_size=1)\n", + "\n", + " self.merge1 = Merge()\n", + " self.merge2 = Merge()\n", + "\n", + " def forward(self, x):\n", + " # -- conv. block 0 --\n", + " con1_out = self.conv1(x)\n", + " conv1_iaf_out = self.conv1_iaf(con1_out)\n", + " pool1_out = self.pool1(conv1_iaf_out)\n", + " pool1a_out = self.pool1a(conv1_iaf_out)\n", + " # -- conv. block 1 --\n", + " conv2_out = self.conv2(pool1_out)\n", + " conv2_iaf_out = self.conv2_iaf(conv2_out)\n", + " # -- conv. block 2 --\n", + " merge1_out = self.merge1(pool1a_out, conv2_iaf_out)\n", + " conv3_out = self.conv3(merge1_out)\n", + " conv3_iaf_out = self.conv3_iaf(conv3_out)\n", + " # -- conv. block 3 --\n", + " conv4_out = self.conv4(conv3_iaf_out)\n", + " conv4_iaf_out = self.conv4_iaf(conv4_out)\n", + " flat_out = self.flat(conv4_iaf_out)\n", + " # -- fc clock 4 --\n", + " fc1_out = self.fc1(flat_out)\n", + " fc1_iaf_out = self.fc1_iaf(fc1_out)\n", + " # -- fc clock 5 --\n", + " fc2_out = self.fc2(fc1_iaf_out)\n", + " fc2_iaf_out = self.fc2_iaf(fc2_out)\n", + " # -- fc clock 6 --\n", + " merge2_out = self.merge2(fc1_iaf_out, fc2_iaf_out)\n", + " fc3_out = self.fc3(merge2_out)\n", + " fc3_iaf_out = self.fc3_iaf(fc3_out)\n", + "\n", + " return fc3_iaf_out" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "metadata": {} + }, + "outputs": [], + "source": [ + "snn = SNN()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The following exemplifies how each of the layers in the SNN should be grouped together to form a `DynapcnnLayer` instance. Let's check the shapes of the tensors each of the (original) layers in the model inputs and outputs by feeding a fake input through the model:" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "metadata": {} + }, + "outputs": [], + "source": [ + "x = torch.randn((1, *input_shape))\n", + "\n", + "# -- conv. block 0 --\n", + "con1_out = snn.conv1(x)\n", + "conv1_iaf_out = snn.conv1_iaf(con1_out)\n", + "pool1_out = snn.pool1(conv1_iaf_out)\n", + "pool1a_out = snn.pool1a(conv1_iaf_out)\n", + "# -- conv. block 1 --\n", + "conv2_out = snn.conv2(pool1_out)\n", + "conv2_iaf_out = snn.conv2_iaf(conv2_out)\n", + "# -- conv. block 2 --\n", + "merge1_out = snn.merge1(pool1a_out, conv2_iaf_out)\n", + "conv3_out = snn.conv3(merge1_out)\n", + "conv3_iaf_out = snn.conv3_iaf(conv3_out)\n", + "# -- conv. block 3 --\n", + "conv4_out = snn.conv4(conv3_iaf_out)\n", + "conv4_iaf_out = snn.conv4_iaf(conv4_out)\n", + "flat_out = snn.flat(conv4_iaf_out)\n", + "# -- fc clock 4 --\n", + "fc1_out = snn.fc1(flat_out)\n", + "fc1_iaf_out = snn.fc1_iaf(fc1_out)\n", + "# -- fc clock 5 --\n", + "fc2_out = snn.fc2(fc1_iaf_out)\n", + "fc2_iaf_out = snn.fc2_iaf(fc2_out)\n", + "# -- fc clock 6 --\n", + "merge2_out = snn.merge2(fc1_iaf_out, fc2_iaf_out)\n", + "fc3_out = snn.fc3(merge2_out)\n", + "fc3_iaf_out = snn.fc3_iaf(fc3_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## DynapcnnNetwork Class\n", + "\n", + "In the constructor of `DynapcnnNetworkGraph` the SNN passed as argument (defined as a `nn.Module`) will be parsed such that each layer is represented in a computational graph (using `nirtorch.extract_torch_graph`). \n", + "\n", + "The layers are the `nodes` of the graph, while their connectivity (how the outputs from a layer are sent to other layers) is represented as `edges`, represented in a `list` of `tuples`.\n", + "\n", + "Once the constructor finishes its initialization, the `hw_model.dynapcnn_layers` property is a dictionary where each entry represents the ID of a `DynapcnnLayer` instance (an `int` from `0` to `L`), with this entry containing a `DynapcnnLayer` instance where a subset of the layers in the original SNN has been incorporated into, the core such instance has been assigned to, and the list of `DynapcnnLayer` instances (their IDs) the layer targets." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "metadata": {} + }, + "outputs": [], + "source": [ + "hw_model = DynapcnnNetworkGraph(\n", + " snn,\n", + " discretize=True,\n", + " input_shape=input_shape\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The `hw_model.to()` call will figure out into which core eac `DynapcnnLayer` instance will be assigned to. Once this assingment is made the instance itself is used to configure the `CNNLayerConfig` instance representing the core's configuration.\n", + "\n", + "If the cores' configuration is valid, each `DynapcnnLayer` instance and their respective destinations will be used to create a computational graph that encodes how the `forward` method of `hw_model.network` (a `nn.Module` using the `DynapcnnLayer` instances) propagates that through the network." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "metadata": {} + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Network is valid\n", + "(0, 1)\n", + "(0, 2)\n", + "(1, 2)\n", + "(2, 3)\n", + "(3, 4)\n", + "(4, 5)\n", + "(4, 6)\n", + "(5, 6)\n" + ] + }, + { + "ename": "RuntimeError", + "evalue": "Device is already opened!", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[8], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m hw_model\u001b[38;5;241m.\u001b[39mto(device\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mspeck2fmodule:0\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", + "File \u001b[0;32m~/Documents/github/sinabs/sinabs/backend/dynapcnn/dynapcnn_network_graph.py:172\u001b[0m, in \u001b[0;36mDynapcnnNetworkGraph.to\u001b[0;34m(self, device, chip_layers_ordering, monitor_layers, config_modifier, slow_clk_frequency)\u001b[0m\n\u001b[1;32m 163\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m device_name \u001b[38;5;129;01min\u001b[39;00m ChipFactory\u001b[38;5;241m.\u001b[39msupported_devices: \u001b[38;5;66;03m# pragma: no cover\u001b[39;00m\n\u001b[1;32m 165\u001b[0m config \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmake_config( \u001b[38;5;66;03m# generate config.\u001b[39;00m\n\u001b[1;32m 166\u001b[0m chip_layers_ordering\u001b[38;5;241m=\u001b[39mchip_layers_ordering,\n\u001b[1;32m 167\u001b[0m device\u001b[38;5;241m=\u001b[39mdevice,\n\u001b[1;32m 168\u001b[0m monitor_layers\u001b[38;5;241m=\u001b[39mmonitor_layers,\n\u001b[1;32m 169\u001b[0m config_modifier\u001b[38;5;241m=\u001b[39mconfig_modifier,\n\u001b[1;32m 170\u001b[0m )\n\u001b[0;32m--> 172\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msamna_device \u001b[38;5;241m=\u001b[39m open_device(device) \u001b[38;5;66;03m# apply configuration to device.\u001b[39;00m\n\u001b[1;32m 173\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msamna_device\u001b[38;5;241m.\u001b[39mget_model()\u001b[38;5;241m.\u001b[39mapply_configuration(config)\n\u001b[1;32m 174\u001b[0m time\u001b[38;5;241m.\u001b[39msleep(\u001b[38;5;241m1\u001b[39m)\n", + "File \u001b[0;32m~/Documents/github/sinabs/sinabs/backend/dynapcnn/io.py:256\u001b[0m, in \u001b[0;36mopen_device\u001b[0;34m(device_id)\u001b[0m\n\u001b[1;32m 254\u001b[0m device_map \u001b[38;5;241m=\u001b[39m get_device_map()\n\u001b[1;32m 255\u001b[0m device_info \u001b[38;5;241m=\u001b[39m device_map[device_id]\n\u001b[0;32m--> 256\u001b[0m device_handle \u001b[38;5;241m=\u001b[39m samna\u001b[38;5;241m.\u001b[39mdevice\u001b[38;5;241m.\u001b[39mopen_device(device_info)\n\u001b[1;32m 258\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m device_handle \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 259\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m device_handle\n", + "\u001b[0;31mRuntimeError\u001b[0m: Device is already opened!" + ] + } + ], + "source": [ + "hw_model.to(device=\"speck2fmodule:0\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The layers comprising our `hw_model` and their respective metadata can be inspected by calling `print` on a `DynapcnnNetworkGraph` instance." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "metadata": {} + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "---- DynapcnnLayer 0 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 0): Conv2d(2, 10, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + "(node 1): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "(node 2): SumPool2d(norm_type=1, kernel_size=(3, 3), stride=None, ceil_mode=False)\n", + "(node 3): SumPool2d(norm_type=1, kernel_size=(4, 4), stride=None, ceil_mode=False)\n", + "> layer destinations: [1, 2]\n", + "> assigned core: 0\n", + "\n", + "---- DynapcnnLayer 1 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 4): Conv2d(10, 10, kernel_size=(4, 4), stride=(1, 1), bias=False)\n", + "(node 6): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [2]\n", + "> assigned core: 1\n", + "\n", + "---- DynapcnnLayer 2 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 7): Conv2d(10, 10, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + "(node 8): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [3]\n", + "> assigned core: 2\n", + "\n", + "---- DynapcnnLayer 3 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 9): Conv2d(10, 1, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + "(node 10): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [4]\n", + "> assigned core: 3\n", + "\n", + "---- DynapcnnLayer 4 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 11): Conv2d(1, 100, kernel_size=(6, 6), stride=(1, 1), bias=False)\n", + "(node 12): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [5, 6]\n", + "> assigned core: 4\n", + "\n", + "---- DynapcnnLayer 5 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 13): Conv2d(100, 100, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + "(node 15): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [6]\n", + "> assigned core: 5\n", + "\n", + "---- DynapcnnLayer 6 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 16): Conv2d(100, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + "(node 17): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: []\n", + "> assigned core: 6\n", + "\n", + "\n" + ] + } + ], + "source": [ + "print(hw_model)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "speck-rescnn", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_5.ipynb b/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_5.ipynb new file mode 100644 index 00000000..4b3fcddb --- /dev/null +++ b/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_5.ipynb @@ -0,0 +1,400 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "metadata": {} + }, + "outputs": [], + "source": [ + "import torch\n", + "import torch.nn as nn\n", + "from sinabs.backend.dynapcnn import DynapcnnNetworkGraph\n", + "from sinabs.layers import Merge, IAFSqueeze" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "metadata": {} + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "torch.manual_seed(0)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "metadata": {} + }, + "outputs": [], + "source": [ + "channels = 2\n", + "height = 34\n", + "width = 34\n", + "\n", + "input_shape = (channels, height, width)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Network Module\n", + "\n", + "We need to define a `nn.Module` implementing the network we want the chip to reproduce." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "metadata": {} + }, + "outputs": [], + "source": [ + "class SNN(nn.Module):\n", + " def __init__(self) -> None:\n", + " super().__init__()\n", + "\n", + " self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False)\n", + " self.conv1_iaf = IAFSqueeze(batch_size=1)\n", + " self.pool1 = nn.AvgPool2d(3,3)\n", + " self.pool1a = nn.AvgPool2d(4,4)\n", + "\n", + " self.conv2 = nn.Conv2d(10, 10, 4, 1, bias=False)\n", + " self.conv2_iaf = IAFSqueeze(batch_size=1)\n", + "\n", + " self.conv3 = nn.Conv2d(10, 10, 2, 1, bias=False)\n", + " self.conv3_iaf = IAFSqueeze(batch_size=1)\n", + "\n", + " self.conv4 = nn.Conv2d(10, 1, 2, 1, bias=False)\n", + " self.conv4_iaf = IAFSqueeze(batch_size=1)\n", + "\n", + " self.flat = nn.Flatten()\n", + "\n", + " self.fc1 = nn.Linear(36, 100, bias=False)\n", + " self.fc1_iaf = IAFSqueeze(batch_size=1)\n", + " \n", + " self.fc2 = nn.Linear(100, 100, bias=False)\n", + " self.fc2_iaf = IAFSqueeze(batch_size=1)\n", + "\n", + " self.fc3 = nn.Linear(100, 100, bias=False)\n", + " self.fc3_iaf = IAFSqueeze(batch_size=1)\n", + "\n", + " self.fc4 = nn.Linear(100, 10, bias=False)\n", + " self.fc4_iaf = IAFSqueeze(batch_size=1)\n", + "\n", + " self.merge1 = Merge()\n", + " self.merge2 = Merge()\n", + " self.merge3 = Merge()\n", + "\n", + " def forward(self, x):\n", + " # -- conv. block 0 --\n", + " con1_out = self.conv1(x)\n", + " conv1_iaf_out = self.conv1_iaf(con1_out)\n", + " pool1_out = self.pool1(conv1_iaf_out)\n", + " pool1a_out = self.pool1a(conv1_iaf_out)\n", + " # -- conv. block 1 --\n", + " conv2_out = self.conv2(pool1_out)\n", + " conv2_iaf_out = self.conv2_iaf(conv2_out)\n", + " # -- conv. block 2 --\n", + " merge1_out = self.merge1(pool1a_out, conv2_iaf_out)\n", + " conv3_out = self.conv3(merge1_out)\n", + " conv3_iaf_out = self.conv3_iaf(conv3_out)\n", + " # -- conv. block 3 --\n", + " conv4_out = self.conv4(conv3_iaf_out)\n", + " conv4_iaf_out = self.conv4_iaf(conv4_out)\n", + " flat_out = self.flat(conv4_iaf_out)\n", + " # -- fc clock 4 --\n", + " fc1_out = self.fc1(flat_out)\n", + " fc1_iaf_out = self.fc1_iaf(fc1_out)\n", + " # -- fc clock 5 --\n", + " fc2_out = self.fc2(fc1_iaf_out)\n", + " fc2_iaf_out = self.fc2_iaf(fc2_out)\n", + " # -- fc clock 6 --\n", + " merge2_out = self.merge2(fc1_iaf_out, fc2_iaf_out)\n", + " fc3_out = self.fc3(merge2_out)\n", + " fc3_iaf_out = self.fc3_iaf(fc3_out)\n", + " # -- fc clock 7 --\n", + " merge3_out = self.merge3(fc2_iaf_out, fc3_iaf_out)\n", + " fc4_out = self.fc4(merge3_out)\n", + " fc4_iaf_out = self.fc4_iaf(fc4_out)\n", + "\n", + " return fc4_iaf_out" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "metadata": {} + }, + "outputs": [], + "source": [ + "snn = SNN()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The following exemplifies how each of the layers in the SNN should be grouped together to form a `DynapcnnLayer` instance. Let's check the shapes of the tensors each of the (original) layers in the model inputs and outputs by feeding a fake input through the model:" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "metadata": {} + }, + "outputs": [], + "source": [ + "x = torch.randn((1, *input_shape))\n", + "\n", + "# -- conv. block 0 --\n", + "con1_out = snn.conv1(x)\n", + "conv1_iaf_out = snn.conv1_iaf(con1_out)\n", + "pool1_out = snn.pool1(conv1_iaf_out)\n", + "pool1a_out = snn.pool1a(conv1_iaf_out)\n", + "# -- conv. block 1 --\n", + "conv2_out = snn.conv2(pool1_out)\n", + "conv2_iaf_out = snn.conv2_iaf(conv2_out)\n", + "# -- conv. block 2 --\n", + "merge1_out = snn.merge1(pool1a_out, conv2_iaf_out)\n", + "conv3_out = snn.conv3(merge1_out)\n", + "conv3_iaf_out = snn.conv3_iaf(conv3_out)\n", + "# -- conv. block 3 --\n", + "conv4_out = snn.conv4(conv3_iaf_out)\n", + "conv4_iaf_out = snn.conv4_iaf(conv4_out)\n", + "flat_out = snn.flat(conv4_iaf_out)\n", + "# -- fc clock 4 --\n", + "fc1_out = snn.fc1(flat_out)\n", + "fc1_iaf_out = snn.fc1_iaf(fc1_out)\n", + "# -- fc clock 5 --\n", + "fc2_out = snn.fc2(fc1_iaf_out)\n", + "fc2_iaf_out = snn.fc2_iaf(fc2_out)\n", + "# -- fc clock 6 --\n", + "merge2_out = snn.merge2(fc1_iaf_out, fc2_iaf_out)\n", + "fc3_out = snn.fc3(merge2_out)\n", + "fc3_iaf_out = snn.fc3_iaf(fc3_out)\n", + "# -- fc clock 7 --\n", + "merge3_out = snn.merge3(fc2_iaf_out, fc3_iaf_out)\n", + "fc4_out = snn.fc4(merge3_out)\n", + "fc4_iaf_out = snn.fc4_iaf(fc4_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## DynapcnnNetwork Class\n", + "\n", + "In the constructor of `DynapcnnNetworkGraph` the SNN passed as argument (defined as a `nn.Module`) will be parsed such that each layer is represented in a computational graph (using `nirtorch.extract_torch_graph`). \n", + "\n", + "The layers are the `nodes` of the graph, while their connectivity (how the outputs from a layer are sent to other layers) is represented as `edges`, represented in a `list` of `tuples`.\n", + "\n", + "Once the constructor finishes its initialization, the `hw_model.dynapcnn_layers` property is a dictionary where each entry represents the ID of a `DynapcnnLayer` instance (an `int` from `0` to `L`), with this entry containing a `DynapcnnLayer` instance where a subset of the layers in the original SNN has been incorporated into, the core such instance has been assigned to, and the list of `DynapcnnLayer` instances (their IDs) the layer targets." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "metadata": {} + }, + "outputs": [], + "source": [ + "hw_model = DynapcnnNetworkGraph(\n", + " snn,\n", + " discretize=True,\n", + " input_shape=input_shape\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The `hw_model.to()` call will figure out into which core eac `DynapcnnLayer` instance will be assigned to. Once this assingment is made the instance itself is used to configure the `CNNLayerConfig` instance representing the core's configuration.\n", + "\n", + "If the cores' configuration is valid, each `DynapcnnLayer` instance and their respective destinations will be used to create a computational graph that encodes how the `forward` method of `hw_model.network` (a `nn.Module` using the `DynapcnnLayer` instances) propagates that through the network." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "metadata": {} + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Network is valid\n", + "(0, 1)\n", + "(0, 2)\n", + "(1, 2)\n", + "(2, 3)\n", + "(3, 4)\n", + "(4, 5)\n", + "(4, 6)\n", + "(5, 6)\n", + "(5, 7)\n", + "(6, 7)\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "hw_model.to(device=\"speck2fmodule:0\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The layers comprising our `hw_model` and their respective metadata can be inspected by calling `print` on a `DynapcnnNetworkGraph` instance." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "metadata": {} + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "---- DynapcnnLayer 0 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 0): Conv2d(2, 10, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + "(node 1): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "(node 2): SumPool2d(norm_type=1, kernel_size=(3, 3), stride=None, ceil_mode=False)\n", + "(node 3): SumPool2d(norm_type=1, kernel_size=(4, 4), stride=None, ceil_mode=False)\n", + "> layer destinations: [1, 2]\n", + "> assigned core: 0\n", + "\n", + "---- DynapcnnLayer 1 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 4): Conv2d(10, 10, kernel_size=(4, 4), stride=(1, 1), bias=False)\n", + "(node 6): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [2]\n", + "> assigned core: 1\n", + "\n", + "---- DynapcnnLayer 2 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 7): Conv2d(10, 10, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + "(node 8): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [3]\n", + "> assigned core: 2\n", + "\n", + "---- DynapcnnLayer 3 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 9): Conv2d(10, 1, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + "(node 10): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [4]\n", + "> assigned core: 3\n", + "\n", + "---- DynapcnnLayer 4 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 11): Conv2d(1, 100, kernel_size=(6, 6), stride=(1, 1), bias=False)\n", + "(node 12): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [5, 6]\n", + "> assigned core: 4\n", + "\n", + "---- DynapcnnLayer 5 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 13): Conv2d(100, 100, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + "(node 15): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [6, 7]\n", + "> assigned core: 5\n", + "\n", + "---- DynapcnnLayer 6 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 17): Conv2d(100, 100, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + "(node 18): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [7]\n", + "> assigned core: 6\n", + "\n", + "---- DynapcnnLayer 7 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 19): Conv2d(100, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + "(node 20): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: []\n", + "> assigned core: 7\n", + "\n", + "\n" + ] + } + ], + "source": [ + "print(hw_model)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "speck-rescnn", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_5a.ipynb b/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_5a.ipynb new file mode 100644 index 00000000..991e48fa --- /dev/null +++ b/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_5a.ipynb @@ -0,0 +1,390 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "metadata": {} + }, + "outputs": [], + "source": [ + "import torch\n", + "import torch.nn as nn\n", + "from sinabs.backend.dynapcnn import DynapcnnNetworkGraph\n", + "from sinabs.layers import Merge, IAFSqueeze, SumPool2d" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "metadata": {} + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "torch.manual_seed(0)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "metadata": {} + }, + "outputs": [], + "source": [ + "channels = 2\n", + "height = 34\n", + "width = 34\n", + "\n", + "input_shape = (channels, height, width)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Network Module\n", + "\n", + "We need to define a `nn.Module` implementing the network we want the chip to reproduce." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "metadata": {} + }, + "outputs": [], + "source": [ + "class SNN(nn.Module):\n", + " def __init__(self) -> None:\n", + " super().__init__()\n", + "\n", + " self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False)\n", + " self.conv1_iaf = IAFSqueeze(batch_size=1)\n", + " self.pool1 = SumPool2d(3,3)\n", + " self.pool1a = SumPool2d(4,4)\n", + "\n", + " self.conv2 = nn.Conv2d(10, 10, 4, 1, bias=False)\n", + " self.conv2_iaf = IAFSqueeze(batch_size=1)\n", + "\n", + " self.conv3 = nn.Conv2d(10, 10, 2, 1, bias=False)\n", + " self.conv3_iaf = IAFSqueeze(batch_size=1)\n", + "\n", + " self.conv4 = nn.Conv2d(10, 1, 2, 1, bias=False)\n", + " self.conv4_iaf = IAFSqueeze(batch_size=1)\n", + "\n", + " self.flat = nn.Flatten()\n", + "\n", + " self.fc1 = nn.Linear(36, 100, bias=False)\n", + " self.fc1_iaf = IAFSqueeze(batch_size=1)\n", + " \n", + " self.fc2 = nn.Linear(100, 100, bias=False)\n", + " self.fc2_iaf = IAFSqueeze(batch_size=1)\n", + "\n", + " self.fc3 = nn.Linear(100, 100, bias=False)\n", + " self.fc3_iaf = IAFSqueeze(batch_size=1)\n", + "\n", + " self.fc4 = nn.Linear(100, 10, bias=False)\n", + " self.fc4_iaf = IAFSqueeze(batch_size=1)\n", + "\n", + " self.merge1 = Merge()\n", + " self.merge2 = Merge()\n", + " self.merge3 = Merge()\n", + "\n", + " def forward(self, x):\n", + " # -- conv. block 0 --\n", + " con1_out = self.conv1(x)\n", + " conv1_iaf_out = self.conv1_iaf(con1_out)\n", + " pool1_out = self.pool1(conv1_iaf_out)\n", + " pool1a_out = self.pool1a(conv1_iaf_out)\n", + " # -- conv. block 1 --\n", + " conv2_out = self.conv2(pool1_out)\n", + " conv2_iaf_out = self.conv2_iaf(conv2_out)\n", + " # -- conv. block 2 --\n", + " merge1_out = self.merge1(pool1a_out, conv2_iaf_out)\n", + " conv3_out = self.conv3(merge1_out)\n", + " conv3_iaf_out = self.conv3_iaf(conv3_out)\n", + " # -- conv. block 3 --\n", + " conv4_out = self.conv4(conv3_iaf_out)\n", + " conv4_iaf_out = self.conv4_iaf(conv4_out)\n", + " flat_out = self.flat(conv4_iaf_out)\n", + " # -- fc clock 4 --\n", + " fc1_out = self.fc1(flat_out)\n", + " fc1_iaf_out = self.fc1_iaf(fc1_out)\n", + " # -- fc clock 5 --\n", + " fc2_out = self.fc2(fc1_iaf_out)\n", + " fc2_iaf_out = self.fc2_iaf(fc2_out)\n", + " # -- fc clock 6 --\n", + " merge2_out = self.merge2(fc1_iaf_out, fc2_iaf_out)\n", + " fc3_out = self.fc3(merge2_out)\n", + " fc3_iaf_out = self.fc3_iaf(fc3_out)\n", + " # -- fc clock 7 --\n", + " merge3_out = self.merge3(fc2_iaf_out, fc3_iaf_out)\n", + " fc4_out = self.fc4(merge3_out)\n", + " fc4_iaf_out = self.fc4_iaf(fc4_out)\n", + "\n", + " return fc4_iaf_out" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "metadata": {} + }, + "outputs": [], + "source": [ + "snn = SNN()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The following exemplifies how each of the layers in the SNN should be grouped together to form a `DynapcnnLayer` instance. Let's check the shapes of the tensors each of the (original) layers in the model inputs and outputs by feeding a fake input through the model:" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "metadata": {} + }, + "outputs": [], + "source": [ + "x = torch.randn((1, *input_shape))\n", + "\n", + "# -- conv. block 0 --\n", + "con1_out = snn.conv1(x)\n", + "conv1_iaf_out = snn.conv1_iaf(con1_out)\n", + "pool1_out = snn.pool1(conv1_iaf_out)\n", + "pool1a_out = snn.pool1a(conv1_iaf_out)\n", + "# -- conv. block 1 --\n", + "conv2_out = snn.conv2(pool1_out)\n", + "conv2_iaf_out = snn.conv2_iaf(conv2_out)\n", + "# -- conv. block 2 --\n", + "merge1_out = snn.merge1(pool1a_out, conv2_iaf_out)\n", + "conv3_out = snn.conv3(merge1_out)\n", + "conv3_iaf_out = snn.conv3_iaf(conv3_out)\n", + "# -- conv. block 3 --\n", + "conv4_out = snn.conv4(conv3_iaf_out)\n", + "conv4_iaf_out = snn.conv4_iaf(conv4_out)\n", + "flat_out = snn.flat(conv4_iaf_out)\n", + "# -- fc clock 4 --\n", + "fc1_out = snn.fc1(flat_out)\n", + "fc1_iaf_out = snn.fc1_iaf(fc1_out)\n", + "# -- fc clock 5 --\n", + "fc2_out = snn.fc2(fc1_iaf_out)\n", + "fc2_iaf_out = snn.fc2_iaf(fc2_out)\n", + "# -- fc clock 6 --\n", + "merge2_out = snn.merge2(fc1_iaf_out, fc2_iaf_out)\n", + "fc3_out = snn.fc3(merge2_out)\n", + "fc3_iaf_out = snn.fc3_iaf(fc3_out)\n", + "# -- fc clock 7 --\n", + "merge3_out = snn.merge3(fc2_iaf_out, fc3_iaf_out)\n", + "fc4_out = snn.fc4(merge3_out)\n", + "fc4_iaf_out = snn.fc4_iaf(fc4_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## DynapcnnNetwork Class\n", + "\n", + "In the constructor of `DynapcnnNetworkGraph` the SNN passed as argument (defined as a `nn.Module`) will be parsed such that each layer is represented in a computational graph (using `nirtorch.extract_torch_graph`). \n", + "\n", + "The layers are the `nodes` of the graph, while their connectivity (how the outputs from a layer are sent to other layers) is represented as `edges`, represented in a `list` of `tuples`.\n", + "\n", + "Once the constructor finishes its initialization, the `hw_model.dynapcnn_layers` property is a dictionary where each entry represents the ID of a `DynapcnnLayer` instance (an `int` from `0` to `L`), with this entry containing a `DynapcnnLayer` instance where a subset of the layers in the original SNN has been incorporated into, the core such instance has been assigned to, and the list of `DynapcnnLayer` instances (their IDs) the layer targets." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "metadata": {} + }, + "outputs": [], + "source": [ + "hw_model = DynapcnnNetworkGraph(\n", + " snn,\n", + " discretize=True,\n", + " input_shape=input_shape\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The `hw_model.to()` call will figure out into which core eac `DynapcnnLayer` instance will be assigned to. Once this assingment is made the instance itself is used to configure the `CNNLayerConfig` instance representing the core's configuration.\n", + "\n", + "If the cores' configuration is valid, each `DynapcnnLayer` instance and their respective destinations will be used to create a computational graph that encodes how the `forward` method of `hw_model.network` (a `nn.Module` using the `DynapcnnLayer` instances) propagates that through the network." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "metadata": {} + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Network is valid\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "hw_model.to(device=\"speck2fmodule:0\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The layers comprising our `hw_model` and their respective metadata can be inspected by calling `print` on a `DynapcnnNetworkGraph` instance." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "metadata": {} + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "---- DynapcnnLayer 0 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 0): Conv2d(2, 10, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + "(node 1): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "(node 2): SumPool2d(norm_type=1, kernel_size=3, stride=3, ceil_mode=False)\n", + "(node 3): SumPool2d(norm_type=1, kernel_size=4, stride=4, ceil_mode=False)\n", + "> layer destinations: [1, 2]\n", + "> assigned core: 0\n", + "\n", + "---- DynapcnnLayer 1 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 4): Conv2d(10, 10, kernel_size=(4, 4), stride=(1, 1), bias=False)\n", + "(node 6): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [2]\n", + "> assigned core: 1\n", + "\n", + "---- DynapcnnLayer 2 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 7): Conv2d(10, 10, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + "(node 8): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [3]\n", + "> assigned core: 2\n", + "\n", + "---- DynapcnnLayer 3 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 9): Conv2d(10, 1, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + "(node 10): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [4]\n", + "> assigned core: 3\n", + "\n", + "---- DynapcnnLayer 4 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 11): Conv2d(1, 100, kernel_size=(6, 6), stride=(1, 1), bias=False)\n", + "(node 12): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [5, 6]\n", + "> assigned core: 4\n", + "\n", + "---- DynapcnnLayer 5 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 13): Conv2d(100, 100, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + "(node 15): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [6, 7]\n", + "> assigned core: 5\n", + "\n", + "---- DynapcnnLayer 6 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 17): Conv2d(100, 100, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + "(node 18): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [7]\n", + "> assigned core: 6\n", + "\n", + "---- DynapcnnLayer 7 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 19): Conv2d(100, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + "(node 20): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: []\n", + "> assigned core: 7\n", + "\n", + "\n" + ] + } + ], + "source": [ + "print(hw_model)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "speck-rescnn", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_6.ipynb b/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_6.ipynb new file mode 100644 index 00000000..9a8dc655 --- /dev/null +++ b/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_6.ipynb @@ -0,0 +1,422 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "metadata": {} + }, + "outputs": [], + "source": [ + "import torch\n", + "import torch.nn as nn\n", + "from sinabs.backend.dynapcnn import DynapcnnNetworkGraph\n", + "from sinabs.layers import Merge, IAFSqueeze, SumPool2d\n", + "import sinabs.layers as sl\n", + "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "metadata": {} + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "torch.manual_seed(0)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "metadata": {} + }, + "outputs": [], + "source": [ + "channels = 2\n", + "height = 34\n", + "width = 34\n", + "\n", + "input_shape = (channels, height, width)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Network Module\n", + "\n", + "We need to define a `nn.Module` implementing the network we want the chip to reproduce." + ] + }, + { + "cell_type": "code", + "execution_count": 67, + "metadata": { + "metadata": {} + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "torch.Size([8, 128])\n" + ] + }, + { + "data": { + "text/plain": [ + "tensor([[0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]], grad_fn=)" + ] + }, + "execution_count": 67, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "class SNN(nn.Module):\n", + " def __init__(self, nb_classes, batch_size, surrogate_fn, min_v_mem=-0.313, spk_thr=2.0) -> None:\n", + " super().__init__()\n", + "\n", + " self.conv1 = nn.Conv2d(2, 1, 2, 1, bias=False)\n", + " self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", + " self.pool1 = sl.SumPool2d(2,2)\n", + "\n", + " self.conv2 = nn.Conv2d(1, 8, 2, 1, bias=False)\n", + " self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", + " self.pool2 = sl.SumPool2d(2,2)\n", + "\n", + " self.conv3 = nn.Conv2d(8, 16, 2, 1, bias=False)\n", + " self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", + " self.pool3 = sl.SumPool2d(2,2)\n", + "\n", + " self.conv4 = nn.Conv2d(16, 32, 2, 1, bias=False)\n", + " self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", + "\n", + " self.flat = nn.Flatten()\n", + "\n", + " self.fc1 = nn.Linear(128, 1024, bias=False)\n", + " self.iaf1_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", + "\n", + " self.fc2 = nn.Linear(1024, 512, bias=False)\n", + " self.iaf2_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", + "\n", + " self.fc3 = nn.Linear(512, 256, bias=False)\n", + " self.iaf3_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", + "\n", + " self.fc4 = nn.Linear(256, 128, bias=False)\n", + " self.iaf4_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", + "\n", + " self.fc5 = nn.Linear(128, nb_classes, bias=False)\n", + " self.iaf5_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", + "\n", + " def forward(self, x):\n", + " # conv 1\n", + " con1_out = self.conv1(x)\n", + " iaf1_out = self.iaf1(con1_out)\n", + " pool1_out = self.pool1(iaf1_out)\n", + "\n", + " # conv 2\n", + " conv2_out = self.conv2(pool1_out)\n", + " iaf2_out = self.iaf2(conv2_out)\n", + " pool2_out = self.pool2(iaf2_out)\n", + "\n", + " # conv 3\n", + " conv3_out = self.conv3(pool2_out)\n", + " iaf3_out = self.iaf3(conv3_out)\n", + " pool3_out = self.pool3(iaf3_out)\n", + "\n", + " # conv 4\n", + " conv4_out = self.conv4(pool3_out)\n", + " iaf4_out = self.iaf4(conv4_out)\n", + "\n", + " flat_out = self.flat(iaf4_out)\n", + " \n", + " # fc 1\n", + " print(flat_out.shape)\n", + " fc1_out = self.fc1(flat_out)\n", + " iaf1_fc_out = self.iaf1_fc(fc1_out)\n", + "\n", + " # fc 2\n", + " fc2_out = self.fc2(iaf1_fc_out)\n", + " iaf2_fc_out = self.iaf2_fc(fc2_out)\n", + "\n", + " # fc 3\n", + " fc3_out = self.fc3(iaf2_fc_out)\n", + " iaf3_fc_out = self.iaf3_fc(fc3_out)\n", + "\n", + " # fc 4\n", + " fc4_out = self.fc4(iaf3_fc_out)\n", + " iaf4_fc_out = self.iaf4_fc(fc4_out)\n", + "\n", + " # fc 5\n", + " fc5_out = self.fc5(iaf4_fc_out)\n", + " iaf5_fc_out = self.iaf5_fc(fc5_out)\n", + "\n", + " return iaf5_fc_out\n", + " \n", + "snn = SNN(11, 8, PeriodicExponential())\n", + "\n", + "x = torch.randn((8, *input_shape))\n", + "\n", + "snn(x)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The following exemplifies how each of the layers in the SNN should be grouped together to form a `DynapcnnLayer` instance. Let's check the shapes of the tensors each of the (original) layers in the model inputs and outputs by feeding a fake input through the model:" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": { + "metadata": {} + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "torch.Size([8, 8, 17, 17])\n", + "torch.Size([8, 8, 16, 16])\n", + "torch.Size([8, 8, 1, 1])\n", + "torch.Size([8, 8])\n" + ] + }, + { + "ename": "RuntimeError", + "evalue": "mat1 and mat2 shapes cannot be multiplied (8x8 and 800x11)", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[30], line 3\u001b[0m\n\u001b[1;32m 1\u001b[0m x \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mrandn((\u001b[38;5;241m8\u001b[39m, \u001b[38;5;241m*\u001b[39minput_shape))\n\u001b[0;32m----> 3\u001b[0m snn(x)\n", + "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/nn/modules/module.py:1511\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1509\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs) \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m 1510\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1511\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n", + "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/nn/modules/module.py:1520\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1515\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1516\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1517\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m 1518\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1519\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1520\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m forward_call(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m 1522\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 1523\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n", + "Cell \u001b[0;32mIn[28], line 74\u001b[0m, in \u001b[0;36mSNN.forward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m 72\u001b[0m flat \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mflat(iaf8_out)\n\u001b[1;32m 73\u001b[0m \u001b[38;5;28mprint\u001b[39m(flat\u001b[38;5;241m.\u001b[39mshape)\n\u001b[0;32m---> 74\u001b[0m fc_out \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfc_out(flat)\n\u001b[1;32m 75\u001b[0m iaf_fc_out \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39miaf_fc_out(fc_out)\n\u001b[1;32m 78\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m iaf_fc_out\n", + "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/nn/modules/module.py:1511\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1509\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs) \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m 1510\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1511\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n", + "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/nn/modules/module.py:1520\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1515\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1516\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1517\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m 1518\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1519\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1520\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m forward_call(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m 1522\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 1523\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n", + "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/nn/modules/linear.py:116\u001b[0m, in \u001b[0;36mLinear.forward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m 115\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;28minput\u001b[39m: Tensor) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tensor:\n\u001b[0;32m--> 116\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m F\u001b[38;5;241m.\u001b[39mlinear(\u001b[38;5;28minput\u001b[39m, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mweight, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbias)\n", + "\u001b[0;31mRuntimeError\u001b[0m: mat1 and mat2 shapes cannot be multiplied (8x8 and 800x11)" + ] + } + ], + "source": [ + "x = torch.randn((8, *input_shape))\n", + "\n", + "snn(x)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## DynapcnnNetwork Class\n", + "\n", + "In the constructor of `DynapcnnNetworkGraph` the SNN passed as argument (defined as a `nn.Module`) will be parsed such that each layer is represented in a computational graph (using `nirtorch.extract_torch_graph`). \n", + "\n", + "The layers are the `nodes` of the graph, while their connectivity (how the outputs from a layer are sent to other layers) is represented as `edges`, represented in a `list` of `tuples`.\n", + "\n", + "Once the constructor finishes its initialization, the `hw_model.dynapcnn_layers` property is a dictionary where each entry represents the ID of a `DynapcnnLayer` instance (an `int` from `0` to `L`), with this entry containing a `DynapcnnLayer` instance where a subset of the layers in the original SNN has been incorporated into, the core such instance has been assigned to, and the list of `DynapcnnLayer` instances (their IDs) the layer targets." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "metadata": {} + }, + "outputs": [], + "source": [ + "hw_model = DynapcnnNetworkGraph(\n", + " snn,\n", + " discretize=True,\n", + " input_shape=input_shape\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The `hw_model.to()` call will figure out into which core eac `DynapcnnLayer` instance will be assigned to. Once this assingment is made the instance itself is used to configure the `CNNLayerConfig` instance representing the core's configuration.\n", + "\n", + "If the cores' configuration is valid, each `DynapcnnLayer` instance and their respective destinations will be used to create a computational graph that encodes how the `forward` method of `hw_model.network` (a `nn.Module` using the `DynapcnnLayer` instances) propagates that through the network." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "metadata": {} + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Network is valid\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "hw_model.to(device=\"speck2fmodule:0\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The layers comprising our `hw_model` and their respective metadata can be inspected by calling `print` on a `DynapcnnNetworkGraph` instance." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "metadata": {} + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "---- DynapcnnLayer 0 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 0): Conv2d(2, 10, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + "(node 1): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "(node 2): SumPool2d(norm_type=1, kernel_size=3, stride=3, ceil_mode=False)\n", + "(node 3): SumPool2d(norm_type=1, kernel_size=4, stride=4, ceil_mode=False)\n", + "> layer destinations: [1, 2]\n", + "> assigned core: 0\n", + "\n", + "---- DynapcnnLayer 1 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 4): Conv2d(10, 10, kernel_size=(4, 4), stride=(1, 1), bias=False)\n", + "(node 6): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [2]\n", + "> assigned core: 1\n", + "\n", + "---- DynapcnnLayer 2 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 7): Conv2d(10, 10, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + "(node 8): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [3]\n", + "> assigned core: 2\n", + "\n", + "---- DynapcnnLayer 3 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 9): Conv2d(10, 1, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + "(node 10): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [4]\n", + "> assigned core: 3\n", + "\n", + "---- DynapcnnLayer 4 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 11): Conv2d(1, 100, kernel_size=(6, 6), stride=(1, 1), bias=False)\n", + "(node 12): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [5, 6]\n", + "> assigned core: 4\n", + "\n", + "---- DynapcnnLayer 5 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 13): Conv2d(100, 100, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + "(node 15): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [6, 7]\n", + "> assigned core: 5\n", + "\n", + "---- DynapcnnLayer 6 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 17): Conv2d(100, 100, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + "(node 18): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [7]\n", + "> assigned core: 6\n", + "\n", + "---- DynapcnnLayer 7 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 19): Conv2d(100, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + "(node 20): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: []\n", + "> assigned core: 7\n", + "\n", + "\n" + ] + } + ], + "source": [ + "print(hw_model)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "speck-rescnn", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/complex_network_structure.ipynb b/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/complex_network_structure.ipynb new file mode 100644 index 00000000..560fa2fe --- /dev/null +++ b/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/complex_network_structure.ipynb @@ -0,0 +1,367 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "metadata": {} + }, + "outputs": [], + "source": [ + "import torch\n", + "import torch.nn as nn\n", + "from sinabs.backend.dynapcnn import DynapcnnNetworkGraph\n", + "from sinabs.layers import Merge, IAFSqueeze, SumPool2d\n", + "import sinabs.layers as sl\n", + "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "metadata": {} + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "torch.manual_seed(0)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "metadata": {} + }, + "outputs": [], + "source": [ + "channels = 2\n", + "height = 34\n", + "width = 34\n", + "\n", + "input_shape = (channels, height, width)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Network Module\n", + "\n", + "We need to define a `nn.Module` implementing the network we want the chip to reproduce." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "```mermaid\n", + "stateDiagram\n", + " [*] --> A\n", + " A --> B\n", + " A --> C\n", + " C --> D\n", + " C --> E\n", + " B --> D\n", + " D --> F\n", + " E --> F\n", + " F --> [*]\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "metadata": {} + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "torch.Size([8, 10, 16, 16]) torch.Size([8, 10, 16, 16])\n", + "torch.Size([8, 250])\n", + "torch.Size([8, 250]) torch.Size([8, 250])\n" + ] + } + ], + "source": [ + "class SNN(nn.Module):\n", + " def __init__(self, nb_classes, batch_size, surrogate_fn, min_v_mem=-0.313, spk_thr=2.0) -> None:\n", + " super().__init__()\n", + "\n", + " self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False)\n", + " self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", + "\n", + " self.conv2 = nn.Conv2d(10, 10, 2, 1, bias=False)\n", + " self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", + " self.pool2 = sl.SumPool2d(2,2)\n", + "\n", + " self.conv3 = nn.Conv2d(10, 10, 2, 1, bias=False)\n", + " self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", + " self.pool3 = sl.SumPool2d(2,2)\n", + " self.pool3a = sl.SumPool2d(6,6)\n", + "\n", + " self.conv4 = nn.Conv2d(10, 10, 2, 1, bias=False)\n", + " self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", + " self.pool4 = sl.SumPool2d(3,3)\n", + "\n", + " self.flat = nn.Flatten()\n", + " self.flat_a = nn.Flatten()\n", + "\n", + " self.fc1 = nn.Linear(250, 250, bias=False)\n", + " self.iaf1_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", + "\n", + " self.fc2 = nn.Linear(250, nb_classes, bias=False)\n", + " self.iaf2_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", + "\n", + " # -- merges --\n", + " self.merge1 = Merge()\n", + " self.merge2 = Merge()\n", + "\n", + " def forward(self, x):\n", + " # conv 1 - A\n", + " con1_out = self.conv1(x)\n", + " iaf1_out = self.iaf1(con1_out)\n", + "\n", + " # conv 2 - B\n", + " conv2_out = self.conv2(iaf1_out)\n", + " iaf2_out = self.iaf2(conv2_out)\n", + " pool2_out = self.pool2(iaf2_out)\n", + "\n", + " # conv 3 - C\n", + " conv3_out = self.conv3(iaf1_out)\n", + " iaf3_out = self.iaf3(conv3_out)\n", + " pool3_out = self.pool3(iaf3_out)\n", + " pool3a_out = self.pool3a(iaf3_out)\n", + "\n", + " # conv 4 - D\n", + " print(pool2_out.shape, pool3_out.shape)\n", + " merge1_out = self.merge1(pool2_out, pool3_out)\n", + " conv4_out = self.conv4(merge1_out)\n", + " iaf4_out = self.iaf4(conv4_out)\n", + " pool4_out = self.pool4(iaf4_out)\n", + " flat_out = self.flat(pool4_out)\n", + " \n", + " # fc 1 - E\n", + " flat_a_out = self.flat_a(pool3a_out)\n", + " print(flat_a_out.shape)\n", + " fc1_out = self.fc1(flat_a_out)\n", + " iaf1_fc_out = self.iaf1_fc(fc1_out)\n", + "\n", + " # fc 2 - F\n", + " print(iaf1_fc_out.shape, flat_out.shape)\n", + " merge2_out = self.merge2(iaf1_fc_out, flat_out)\n", + " fc2_out = self.fc2(merge2_out)\n", + " iaf2_fc_out = self.iaf2_fc(fc2_out)\n", + "\n", + " return iaf2_fc_out\n", + " \n", + "snn = SNN(11, 8, PeriodicExponential())\n", + "\n", + "x = torch.randn((8, *input_shape))\n", + "\n", + "out = snn(x)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## DynapcnnNetwork Class\n", + "\n", + "In the constructor of `DynapcnnNetworkGraph` the SNN passed as argument (defined as a `nn.Module`) will be parsed such that each layer is represented in a computational graph (using `nirtorch.extract_torch_graph`). \n", + "\n", + "The layers are the `nodes` of the graph, while their connectivity (how the outputs from a layer are sent to other layers) is represented as `edges`, represented in a `list` of `tuples`.\n", + "\n", + "Once the constructor finishes its initialization, the `hw_model.dynapcnn_layers` property is a dictionary where each entry represents the ID of a `DynapcnnLayer` instance (an `int` from `0` to `L`), with this entry containing a `DynapcnnLayer` instance where a subset of the layers in the original SNN has been incorporated into, the core such instance has been assigned to, and the list of `DynapcnnLayer` instances (their IDs) the layer targets." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "metadata": {} + }, + "outputs": [], + "source": [ + "hw_model = DynapcnnNetworkGraph(\n", + " snn,\n", + " discretize=True,\n", + " input_shape=input_shape\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The `hw_model.to()` call will figure out into which core eac `DynapcnnLayer` instance will be assigned to. Once this assingment is made the instance itself is used to configure the `CNNLayerConfig` instance representing the core's configuration.\n", + "\n", + "If the cores' configuration is valid, each `DynapcnnLayer` instance and their respective destinations will be used to create a computational graph that encodes how the `forward` method of `hw_model.network` (a `nn.Module` using the `DynapcnnLayer` instances) propagates that through the network." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "metadata": {} + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Network is valid\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "hw_model.to(device=\"speck2fmodule:0\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The layers comprising our `hw_model` and their respective metadata can be inspected by calling `print` on a `DynapcnnNetworkGraph` instance." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "metadata": {} + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "---- DynapcnnLayer 0 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 0): Conv2d(2, 10, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + "(node 1): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "(node 2): SumPool2d(norm_type=1, kernel_size=3, stride=3, ceil_mode=False)\n", + "(node 3): SumPool2d(norm_type=1, kernel_size=4, stride=4, ceil_mode=False)\n", + "> layer destinations: [1, 2]\n", + "> assigned core: 0\n", + "\n", + "---- DynapcnnLayer 1 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 4): Conv2d(10, 10, kernel_size=(4, 4), stride=(1, 1), bias=False)\n", + "(node 6): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [2]\n", + "> assigned core: 1\n", + "\n", + "---- DynapcnnLayer 2 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 7): Conv2d(10, 10, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + "(node 8): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [3]\n", + "> assigned core: 2\n", + "\n", + "---- DynapcnnLayer 3 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 9): Conv2d(10, 1, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + "(node 10): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [4]\n", + "> assigned core: 3\n", + "\n", + "---- DynapcnnLayer 4 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 11): Conv2d(1, 100, kernel_size=(6, 6), stride=(1, 1), bias=False)\n", + "(node 12): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [5, 6]\n", + "> assigned core: 4\n", + "\n", + "---- DynapcnnLayer 5 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 13): Conv2d(100, 100, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + "(node 15): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [6, 7]\n", + "> assigned core: 5\n", + "\n", + "---- DynapcnnLayer 6 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 17): Conv2d(100, 100, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + "(node 18): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [7]\n", + "> assigned core: 6\n", + "\n", + "---- DynapcnnLayer 7 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 19): Conv2d(100, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + "(node 20): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: []\n", + "> assigned core: 7\n", + "\n", + "\n" + ] + } + ], + "source": [ + "print(hw_model)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "speck-rescnn", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tests/test_nonsequential/using_SumPool2d/models/ResCSNN3.py b/tests/test_nonsequential/using_SumPool2d/models/ResCSNN3.py deleted file mode 100644 index fa6bfd46..00000000 --- a/tests/test_nonsequential/using_SumPool2d/models/ResCSNN3.py +++ /dev/null @@ -1,122 +0,0 @@ -import torch.nn as nn -import sinabs.layers as sl -from sinabs.exodus.layers import IAFSqueeze - -class ResCSNN3(nn.Module): - def __init__(self, input_size, nb_classes, batch_size, surrogate_fn, min_v_mem=-1.0, spk_thr=1.0) -> None: - super().__init__() - - self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False) - self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool1 = sl.SumPool2d(2,2) - self.pool1a = sl.SumPool2d(6,6) - - self.conv2 = nn.Conv2d(10, 10, 2, 1, bias=False) - self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool2 = sl.SumPool2d(3,3) - - self.conv3 = nn.Conv2d(10, 10, 3, 1, bias=False) - self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool3 = sl.SumPool2d(2,2) - - self.flat = nn.Flatten() - - flat_s = ResCSNN3.get_flatten_size(input_size) - - self.fc1 = nn.Linear(flat_s, 100, bias=False) - self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc2 = nn.Linear(100, 100, bias=False) - self.iaf5 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc3 = nn.Linear(100, 100, bias=False) - self.iaf6 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc4 = nn.Linear(100, nb_classes, bias=False) - self.iaf7 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.merge_fc = sl.Merge() - self.merge_conv = sl.Merge() - - @staticmethod - def get_flatten_size(input_size): - conv1_dims = ResCSNN3.conv2d_output_size(input_size, 10, (2, 2)) - pool1_dims = ResCSNN3.pool_output_size(conv1_dims, 10, (2, 2)) - - conv2_dims = ResCSNN3.conv2d_output_size(pool1_dims, 10, (2, 2)) - pool2_dims = ResCSNN3.pool_output_size(conv2_dims, 10, (3, 3)) - - conv3_dims = ResCSNN3.conv2d_output_size(pool2_dims, 10, (3, 3)) - pool3_dims = ResCSNN3.pool_output_size(conv3_dims, 10, (2, 2)) - - return pool3_dims[0]*pool3_dims[1]*pool3_dims[2] - - @staticmethod - def conv2d_output_size(input_size, out_channels, kernel_size, stride=1, padding=0, dilation=1): - input_height, input_width, input_channels = input_size - kernel_height, kernel_width = kernel_size - - output_height = ((input_height + 2 * padding - dilation * (kernel_height - 1) - 1) // stride) + 1 - output_width = ((input_width + 2 * padding - dilation * (kernel_width - 1) - 1) // stride) + 1 - - return (output_height, output_width, out_channels) - - @staticmethod - def pool_output_size(input_size, out_channels, kernel_size, stride=None, padding=0): - input_height, input_width, input_channels = input_size - kernel_height, kernel_width = kernel_size - - if stride is None: - stride = kernel_height - - output_height = ((input_height + 2 * padding - kernel_height) // stride) + 1 - output_width = ((input_width + 2 * padding - kernel_width) // stride) + 1 - - return (output_height, output_width, out_channels) - - def detach_neuron_states(self): - for name, layer in self.named_modules(): - if name != '': - if isinstance(layer, sl.StatefulLayer): - for name, buffer in layer.named_buffers(): - buffer.detach_() - - def init_weights(self): - for name, layer in self.named_modules(): - if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear): - nn.init.xavier_normal_(layer.weight.data) - - def forward(self, x): - - con1_out = self.conv1(x) - iaf1_out = self.iaf1(con1_out) - pool1_out = self.pool1(iaf1_out) - pool1a_out = self.pool1a(iaf1_out) - - conv2_out = self.conv2(pool1_out) - iaf2_out = self.iaf2(conv2_out) - pool2_out = self.pool2(iaf2_out) - - merged_conv_out = self.merge_conv(pool1a_out, pool2_out) - - conv3_out = self.conv3(merged_conv_out) - iaf3_out = self.iaf3(conv3_out) - pool3_out = self.pool3(iaf3_out) - - flat_out = self.flat(pool3_out) - - fc1_out = self.fc1(flat_out) - iaf4_out = self.iaf4(fc1_out) - - fc2_out = self.fc2(iaf4_out) - iaf5_out = self.iaf5(fc2_out) - - fc3_out = self.fc3(iaf5_out) - iaf6_out = self.iaf6(fc3_out) - - merge_fc_out = self.merge_fc(iaf4_out, iaf6_out) - - fc4_out = self.fc4(merge_fc_out) - iaf7_out = self.iaf7(fc4_out) - - return iaf7_out \ No newline at end of file diff --git a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN1.py b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN1.py deleted file mode 100644 index b8f5b838..00000000 --- a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN1.py +++ /dev/null @@ -1,152 +0,0 @@ -import torch.nn as nn -import sinabs.layers as sl -from sinabs.exodus.layers import IAFSqueeze - -class SCNN(nn.Module): - def __init__(self, input_size, nb_classes, batch_size, surrogate_fn, min_v_mem=-0.313, spk_thr=2.0) -> None: - super().__init__() - - self.conv1 = nn.Conv2d(2, 8, 2, 1, bias=False) - self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool1 = sl.SumPool2d(2,2) - self.pool1a = sl.SumPool2d(4,4) - - self.conv2 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool2 = sl.SumPool2d(2,2) - - self.conv3 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool3 = sl.SumPool2d(2,2) - - self.conv4 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool4 = sl.SumPool2d(2,2) - - self.flat = nn.Flatten() - - flat_s = SCNN.get_flatten_size(input_size) - - self.fc1 = nn.Linear(flat_s, 100, bias=False) - self.iaf1_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc2 = nn.Linear(100, 100, bias=False) - self.iaf2_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc3 = nn.Linear(100, 100, bias=False) - self.iaf3_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc4 = nn.Linear(100, 100, bias=False) - self.iaf4_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc5 = nn.Linear(100, nb_classes, bias=False) - self.iaf5_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - # skip - - self.merge1 = sl.Merge() - - @staticmethod - def get_flatten_size(input_size): - conv1_dims = SCNN.conv2d_output_size(input_size, 8, (2, 2)) - pool1_dims = SCNN.pool_output_size(conv1_dims, 8, (2, 2)) - - conv2_dims = SCNN.conv2d_output_size(pool1_dims, 8, (2, 2)) - pool2_dims = SCNN.pool_output_size(conv2_dims, 8, (2, 2)) - - conv3_dims = SCNN.conv2d_output_size(pool2_dims, 8, (2, 2)) - pool3_dims = SCNN.pool_output_size(conv3_dims, 8, (2, 2)) - - conv4_dims = SCNN.conv2d_output_size(pool3_dims, 8, (2, 2)) - pool4_dims = SCNN.pool_output_size(conv4_dims, 8, (2, 2)) - - return pool4_dims[0]*pool4_dims[1]*pool4_dims[2] - - @staticmethod - def conv2d_output_size(input_size, out_channels, kernel_size, stride=1, padding=0, dilation=1): - input_height, input_width, input_channels = input_size - kernel_height, kernel_width = kernel_size - - output_height = ((input_height + 2 * padding - dilation * (kernel_height - 1) - 1) // stride) + 1 - output_width = ((input_width + 2 * padding - dilation * (kernel_width - 1) - 1) // stride) + 1 - - return (output_height, output_width, out_channels) - - @staticmethod - def pool_output_size(input_size, out_channels, kernel_size, stride=None, padding=0): - input_height, input_width, input_channels = input_size - kernel_height, kernel_width = kernel_size - - if stride is None: - stride = kernel_height - - output_height = ((input_height + 2 * padding - kernel_height) // stride) + 1 - output_width = ((input_width + 2 * padding - kernel_width) // stride) + 1 - - return (output_height, output_width, out_channels) - - def detach_neuron_states(self): - for name, layer in self.named_modules(): - if name != '': - if isinstance(layer, sl.StatefulLayer): - for name, buffer in layer.named_buffers(): - buffer.detach_() - - def init_weights(self): - for name, layer in self.named_modules(): - if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear): - nn.init.xavier_normal_(layer.weight.data) - - def rescale_conv_weights(self, rescale_fn, lambda_): - rescale_fn(self.conv2, [(2, 2)], lambda_) - rescale_fn(self.conv3, [(2, 2), (4, 4)], lambda_) - rescale_fn(self.conv4, [(2, 2)], lambda_) - - - def forward(self, x): - # conv 1 - con1_out = self.conv1(x) - iaf1_out = self.iaf1(con1_out) - pool1_out = self.pool1(iaf1_out) - pool1a_out = self.pool1a(iaf1_out) - - # conv 2 - conv2_out = self.conv2(pool1_out) - iaf2_out = self.iaf2(conv2_out) - pool2_out = self.pool2(iaf2_out) - - merge1_out = self.merge1(pool1a_out, pool2_out) - - # conv 3 - conv3_out = self.conv3(merge1_out) - iaf3_out = self.iaf3(conv3_out) - pool3_out = self.pool3(iaf3_out) - - # conv 4 - conv4_out = self.conv4(pool3_out) - iaf4_out = self.iaf4(conv4_out) - pool4_out = self.pool4(iaf4_out) - - flat_out = self.flat(pool4_out) - - # fc 1 - fc1_out = self.fc1(flat_out) - iaf1_fc_out = self.iaf1_fc(fc1_out) - - # fc 2 - fc2_out = self.fc2(iaf1_fc_out) - iaf2_fc_out = self.iaf2_fc(fc2_out) - - # fc 3 - fc3_out = self.fc3(iaf2_fc_out) - iaf3_fc_out = self.iaf3_fc(fc3_out) - - # fc 4 - fc4_out = self.fc4(iaf3_fc_out) - iaf4_fc_out = self.iaf4_fc(fc4_out) - - # fc 5 - fc5_out = self.fc5(iaf4_fc_out) - iaf5_fc_out = self.iaf5_fc(fc5_out) - - return iaf5_fc_out \ No newline at end of file diff --git a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN2.py b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN2.py deleted file mode 100644 index 21ada5a4..00000000 --- a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN2.py +++ /dev/null @@ -1,143 +0,0 @@ -import torch.nn as nn -import sinabs.layers as sl -from sinabs.exodus.layers import IAFSqueeze - -class SCNN(nn.Module): - def __init__(self, input_size, nb_classes, batch_size, surrogate_fn, min_v_mem=-0.313, spk_thr=2.0) -> None: - super().__init__() - - self.conv1 = nn.Conv2d(2, 8, 2, 1, bias=False) - self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool1 = sl.SumPool2d(2,2) - self.pool1a = sl.SumPool2d(4,4) - - self.conv2 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool2 = sl.SumPool2d(2,2) - - self.conv3 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool3 = sl.SumPool2d(2,2) - self.pool3a = sl.SumPool2d(4,4) - - self.conv4 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool4 = sl.SumPool2d(2,2) - - self.flat = nn.Flatten() - - flat_s = SCNN.get_flatten_size(input_size) - - self.fc1 = nn.Linear(flat_s, 100, bias=False) - self.iaf1_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc2 = nn.Linear(100, 100, bias=False) - self.iaf2_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc3 = nn.Linear(100, 100, bias=False) - self.iaf3_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc4 = nn.Linear(100, 100, bias=False) - self.iaf4_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc5 = nn.Linear(100, nb_classes, bias=False) - self.iaf5_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - # skip - - self.merge1 = sl.Merge() - self.merge2 = sl.Merge() - - @staticmethod - def get_flatten_size(input_size): - conv1_dims = SCNN.conv2d_output_size(input_size, 8, (2, 2)) - pool1_dims = SCNN.pool_output_size(conv1_dims, 8, (2, 2)) - - conv2_dims = SCNN.conv2d_output_size(pool1_dims, 8, (2, 2)) - pool2_dims = SCNN.pool_output_size(conv2_dims, 8, (2, 2)) - - conv3_dims = SCNN.conv2d_output_size(pool2_dims, 8, (2, 2)) - pool3_dims = SCNN.pool_output_size(conv3_dims, 8, (2, 2)) - - conv4_dims = SCNN.conv2d_output_size(pool3_dims, 8, (2, 2)) - pool4_dims = SCNN.pool_output_size(conv4_dims, 8, (2, 2)) - - return pool4_dims[0]*pool4_dims[1]*pool4_dims[2] - - @staticmethod - def conv2d_output_size(input_size, out_channels, kernel_size, stride=1, padding=0, dilation=1): - input_height, input_width, input_channels = input_size - kernel_height, kernel_width = kernel_size - - output_height = ((input_height + 2 * padding - dilation * (kernel_height - 1) - 1) // stride) + 1 - output_width = ((input_width + 2 * padding - dilation * (kernel_width - 1) - 1) // stride) + 1 - - return (output_height, output_width, out_channels) - - @staticmethod - def pool_output_size(input_size, out_channels, kernel_size, stride=None, padding=0): - input_height, input_width, input_channels = input_size - kernel_height, kernel_width = kernel_size - - if stride is None: - stride = kernel_height - - output_height = ((input_height + 2 * padding - kernel_height) // stride) + 1 - output_width = ((input_width + 2 * padding - kernel_width) // stride) + 1 - - return (output_height, output_width, out_channels) - - def detach_neuron_states(self): - for name, layer in self.named_modules(): - if name != '': - if isinstance(layer, sl.StatefulLayer): - for name, buffer in layer.named_buffers(): - buffer.detach_() - - def init_weights(self): - for name, layer in self.named_modules(): - if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear): - nn.init.xavier_normal_(layer.weight.data) - - def forward(self, x): - # conv 1 - con1_out = self.conv1(x) - iaf1_out = self.iaf1(con1_out) - pool1_out = self.pool1(iaf1_out) - pool1a_out = self.pool1a(iaf1_out) - # conv 2 - conv2_out = self.conv2(pool1_out) - iaf2_out = self.iaf2(conv2_out) - pool2_out = self.pool2(iaf2_out) - - merge1_out = self.merge1(pool1a_out, pool2_out) - # conv 3 - conv3_out = self.conv3(merge1_out) - iaf3_out = self.iaf3(conv3_out) - pool3_out = self.pool3(iaf3_out) - pool3a_out = self.pool3a(iaf3_out) - # conv 4 - conv4_out = self.conv4(pool3_out) - iaf4_out = self.iaf4(conv4_out) - pool4_out = self.pool4(iaf4_out) - - merge2_out = self.merge2(pool3a_out, pool4_out) - - flat_out = self.flat(merge2_out) - # fc 1 - fc1_out = self.fc1(flat_out) - iaf1_fc_out = self.iaf1_fc(fc1_out) - # fc 2 - fc2_out = self.fc2(iaf1_fc_out) - iaf2_fc_out = self.iaf2_fc(fc2_out) - # fc 3 - fc3_out = self.fc3(iaf2_fc_out) - iaf3_fc_out = self.iaf3_fc(fc3_out) - # fc 4 - fc4_out = self.fc4(iaf3_fc_out) - iaf4_fc_out = self.iaf4_fc(fc4_out) - # fc 5 - fc5_out = self.fc5(iaf4_fc_out) - iaf5_fc_out = self.iaf5_fc(fc5_out) - - return iaf5_fc_out \ No newline at end of file diff --git a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN3.py b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN3.py deleted file mode 100644 index 81479e94..00000000 --- a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN3.py +++ /dev/null @@ -1,146 +0,0 @@ -import torch.nn as nn -import sinabs.layers as sl -from sinabs.exodus.layers import IAFSqueeze - -class SCNN(nn.Module): - def __init__(self, input_size, nb_classes, batch_size, surrogate_fn, min_v_mem=-0.313, spk_thr=2.0) -> None: - super().__init__() - - self.conv1 = nn.Conv2d(2, 8, 2, 1, bias=False) - self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool1 = sl.SumPool2d(2,2) - self.pool1a = sl.SumPool2d(4,4) - - self.conv2 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool2 = sl.SumPool2d(2,2) - - self.conv3 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool3 = sl.SumPool2d(2,2) - self.pool3a = sl.SumPool2d(4,4) - - self.conv4 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool4 = sl.SumPool2d(2,2) - - self.flat = nn.Flatten() - - flat_s = SCNN.get_flatten_size(input_size) - - self.fc1 = nn.Linear(flat_s, 100, bias=False) - self.iaf1_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc2 = nn.Linear(100, 100, bias=False) - self.iaf2_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc3 = nn.Linear(100, 100, bias=False) - self.iaf3_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc4 = nn.Linear(100, 100, bias=False) - self.iaf4_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc5 = nn.Linear(100, nb_classes, bias=False) - self.iaf5_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - # skip - - self.merge1 = sl.Merge() - self.merge2 = sl.Merge() - self.merge3 = sl.Merge() - - @staticmethod - def get_flatten_size(input_size): - conv1_dims = SCNN.conv2d_output_size(input_size, 8, (2, 2)) - pool1_dims = SCNN.pool_output_size(conv1_dims, 8, (2, 2)) - - conv2_dims = SCNN.conv2d_output_size(pool1_dims, 8, (2, 2)) - pool2_dims = SCNN.pool_output_size(conv2_dims, 8, (2, 2)) - - conv3_dims = SCNN.conv2d_output_size(pool2_dims, 8, (2, 2)) - pool3_dims = SCNN.pool_output_size(conv3_dims, 8, (2, 2)) - - conv4_dims = SCNN.conv2d_output_size(pool3_dims, 8, (2, 2)) - pool4_dims = SCNN.pool_output_size(conv4_dims, 8, (2, 2)) - - return pool4_dims[0]*pool4_dims[1]*pool4_dims[2] - - @staticmethod - def conv2d_output_size(input_size, out_channels, kernel_size, stride=1, padding=0, dilation=1): - input_height, input_width, input_channels = input_size - kernel_height, kernel_width = kernel_size - - output_height = ((input_height + 2 * padding - dilation * (kernel_height - 1) - 1) // stride) + 1 - output_width = ((input_width + 2 * padding - dilation * (kernel_width - 1) - 1) // stride) + 1 - - return (output_height, output_width, out_channels) - - @staticmethod - def pool_output_size(input_size, out_channels, kernel_size, stride=None, padding=0): - input_height, input_width, input_channels = input_size - kernel_height, kernel_width = kernel_size - - if stride is None: - stride = kernel_height - - output_height = ((input_height + 2 * padding - kernel_height) // stride) + 1 - output_width = ((input_width + 2 * padding - kernel_width) // stride) + 1 - - return (output_height, output_width, out_channels) - - def detach_neuron_states(self): - for name, layer in self.named_modules(): - if name != '': - if isinstance(layer, sl.StatefulLayer): - for name, buffer in layer.named_buffers(): - buffer.detach_() - - def init_weights(self): - for name, layer in self.named_modules(): - if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear): - nn.init.xavier_normal_(layer.weight.data) - - def forward(self, x): - # conv 1 - con1_out = self.conv1(x) - iaf1_out = self.iaf1(con1_out) - pool1_out = self.pool1(iaf1_out) - pool1a_out = self.pool1a(iaf1_out) - # conv 2 - conv2_out = self.conv2(pool1_out) - iaf2_out = self.iaf2(conv2_out) - pool2_out = self.pool2(iaf2_out) - - merge1_out = self.merge1(pool1a_out, pool2_out) - # conv 3 - conv3_out = self.conv3(merge1_out) - iaf3_out = self.iaf3(conv3_out) - pool3_out = self.pool3(iaf3_out) - pool3a_out = self.pool3a(iaf3_out) - # conv 4 - conv4_out = self.conv4(pool3_out) - iaf4_out = self.iaf4(conv4_out) - pool4_out = self.pool4(iaf4_out) - - merge2_out = self.merge2(pool3a_out, pool4_out) - - flat_out = self.flat(merge2_out) - # fc 1 - fc1_out = self.fc1(flat_out) - iaf1_fc_out = self.iaf1_fc(fc1_out) - # fc 2 - fc2_out = self.fc2(iaf1_fc_out) - iaf2_fc_out = self.iaf2_fc(fc2_out) - - merge3_out = self.merge3(iaf1_fc_out, iaf2_fc_out) - # fc 3 - fc3_out = self.fc3(merge3_out) - iaf3_fc_out = self.iaf3_fc(fc3_out) - # fc 4 - fc4_out = self.fc4(iaf3_fc_out) - iaf4_fc_out = self.iaf4_fc(fc4_out) - # fc 5 - fc5_out = self.fc5(iaf4_fc_out) - iaf5_fc_out = self.iaf5_fc(fc5_out) - - return iaf5_fc_out \ No newline at end of file diff --git a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN4.py b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN4.py deleted file mode 100644 index 535c04f8..00000000 --- a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN4.py +++ /dev/null @@ -1,149 +0,0 @@ -import torch.nn as nn -import sinabs.layers as sl -from sinabs.exodus.layers import IAFSqueeze - -class SCNN(nn.Module): - def __init__(self, input_size, nb_classes, batch_size, surrogate_fn, min_v_mem=-0.313, spk_thr=2.0) -> None: - super().__init__() - - self.conv1 = nn.Conv2d(2, 8, 2, 1, bias=False) - self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool1 = sl.SumPool2d(2,2) - self.pool1a = sl.SumPool2d(4,4) - - self.conv2 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool2 = sl.SumPool2d(2,2) - - self.conv3 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool3 = sl.SumPool2d(2,2) - self.pool3a = sl.SumPool2d(4,4) - - self.conv4 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool4 = sl.SumPool2d(2,2) - - self.flat = nn.Flatten() - - flat_s = SCNN.get_flatten_size(input_size) - - self.fc1 = nn.Linear(flat_s, 100, bias=False) - self.iaf1_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc2 = nn.Linear(100, 100, bias=False) - self.iaf2_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc3 = nn.Linear(100, 100, bias=False) - self.iaf3_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc4 = nn.Linear(100, 100, bias=False) - self.iaf4_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc5 = nn.Linear(100, nb_classes, bias=False) - self.iaf5_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - # skip - - self.merge1 = sl.Merge() - self.merge2 = sl.Merge() - self.merge3 = sl.Merge() - self.merge4 = sl.Merge() - - @staticmethod - def get_flatten_size(input_size): - conv1_dims = SCNN.conv2d_output_size(input_size, 8, (2, 2)) - pool1_dims = SCNN.pool_output_size(conv1_dims, 8, (2, 2)) - - conv2_dims = SCNN.conv2d_output_size(pool1_dims, 8, (2, 2)) - pool2_dims = SCNN.pool_output_size(conv2_dims, 8, (2, 2)) - - conv3_dims = SCNN.conv2d_output_size(pool2_dims, 8, (2, 2)) - pool3_dims = SCNN.pool_output_size(conv3_dims, 8, (2, 2)) - - conv4_dims = SCNN.conv2d_output_size(pool3_dims, 8, (2, 2)) - pool4_dims = SCNN.pool_output_size(conv4_dims, 8, (2, 2)) - - return pool4_dims[0]*pool4_dims[1]*pool4_dims[2] - - @staticmethod - def conv2d_output_size(input_size, out_channels, kernel_size, stride=1, padding=0, dilation=1): - input_height, input_width, input_channels = input_size - kernel_height, kernel_width = kernel_size - - output_height = ((input_height + 2 * padding - dilation * (kernel_height - 1) - 1) // stride) + 1 - output_width = ((input_width + 2 * padding - dilation * (kernel_width - 1) - 1) // stride) + 1 - - return (output_height, output_width, out_channels) - - @staticmethod - def pool_output_size(input_size, out_channels, kernel_size, stride=None, padding=0): - input_height, input_width, input_channels = input_size - kernel_height, kernel_width = kernel_size - - if stride is None: - stride = kernel_height - - output_height = ((input_height + 2 * padding - kernel_height) // stride) + 1 - output_width = ((input_width + 2 * padding - kernel_width) // stride) + 1 - - return (output_height, output_width, out_channels) - - def detach_neuron_states(self): - for name, layer in self.named_modules(): - if name != '': - if isinstance(layer, sl.StatefulLayer): - for name, buffer in layer.named_buffers(): - buffer.detach_() - - def init_weights(self): - for name, layer in self.named_modules(): - if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear): - nn.init.xavier_normal_(layer.weight.data) - - def forward(self, x): - # conv 1 - con1_out = self.conv1(x) - iaf1_out = self.iaf1(con1_out) - pool1_out = self.pool1(iaf1_out) - pool1a_out = self.pool1a(iaf1_out) - # conv 2 - conv2_out = self.conv2(pool1_out) - iaf2_out = self.iaf2(conv2_out) - pool2_out = self.pool2(iaf2_out) - - merge1_out = self.merge1(pool1a_out, pool2_out) - # conv 3 - conv3_out = self.conv3(merge1_out) - iaf3_out = self.iaf3(conv3_out) - pool3_out = self.pool3(iaf3_out) - pool3a_out = self.pool3a(iaf3_out) - # conv 4 - conv4_out = self.conv4(pool3_out) - iaf4_out = self.iaf4(conv4_out) - pool4_out = self.pool4(iaf4_out) - - merge2_out = self.merge2(pool3a_out, pool4_out) - - flat_out = self.flat(merge2_out) - # fc 1 - fc1_out = self.fc1(flat_out) - iaf1_fc_out = self.iaf1_fc(fc1_out) - # fc 2 - fc2_out = self.fc2(iaf1_fc_out) - iaf2_fc_out = self.iaf2_fc(fc2_out) - - merge3_out = self.merge3(iaf1_fc_out, iaf2_fc_out) - # fc 3 - fc3_out = self.fc3(merge3_out) - iaf3_fc_out = self.iaf3_fc(fc3_out) - # fc 4 - fc4_out = self.fc4(iaf3_fc_out) - iaf4_fc_out = self.iaf4_fc(fc4_out) - - merge4_out = self.merge4(iaf3_fc_out, iaf4_fc_out) - # fc 5 - fc5_out = self.fc5(merge4_out) - iaf5_fc_out = self.iaf5_fc(fc5_out) - - return iaf5_fc_out \ No newline at end of file diff --git a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN5.py b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN5.py deleted file mode 100644 index 51e82235..00000000 --- a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN5.py +++ /dev/null @@ -1,138 +0,0 @@ -import torch.nn as nn -import sinabs.layers as sl -from sinabs.exodus.layers import IAFSqueeze - -class SCNN(nn.Module): - def __init__(self, input_size, nb_classes, batch_size, surrogate_fn, min_v_mem=-0.313, spk_thr=2.0) -> None: - super().__init__() - - self.conv1 = nn.Conv2d(2, 8, 2, 1, bias=False) - self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool1 = sl.SumPool2d(2,2) - self.pool1a = sl.SumPool2d(8,8) - - self.conv2 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool2 = sl.SumPool2d(2,2) - - self.conv3 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool3 = sl.SumPool2d(2,2) - - self.conv4 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool4 = sl.SumPool2d(2,2) - - self.flat = nn.Flatten() - - flat_s = SCNN.get_flatten_size(input_size) - - self.fc1 = nn.Linear(flat_s, 100, bias=False) - self.iaf1_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc2 = nn.Linear(100, 100, bias=False) - self.iaf2_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc3 = nn.Linear(100, 100, bias=False) - self.iaf3_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc4 = nn.Linear(100, 100, bias=False) - self.iaf4_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc5 = nn.Linear(100, nb_classes, bias=False) - self.iaf5_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - # skip - - self.merge1 = sl.Merge() - - @staticmethod - def get_flatten_size(input_size): - conv1_dims = SCNN.conv2d_output_size(input_size, 8, (2, 2)) - pool1_dims = SCNN.pool_output_size(conv1_dims, 8, (2, 2)) - - conv2_dims = SCNN.conv2d_output_size(pool1_dims, 8, (2, 2)) - pool2_dims = SCNN.pool_output_size(conv2_dims, 8, (2, 2)) - - conv3_dims = SCNN.conv2d_output_size(pool2_dims, 8, (2, 2)) - pool3_dims = SCNN.pool_output_size(conv3_dims, 8, (2, 2)) - - conv4_dims = SCNN.conv2d_output_size(pool3_dims, 8, (2, 2)) - pool4_dims = SCNN.pool_output_size(conv4_dims, 8, (2, 2)) - - return pool4_dims[0]*pool4_dims[1]*pool4_dims[2] - - @staticmethod - def conv2d_output_size(input_size, out_channels, kernel_size, stride=1, padding=0, dilation=1): - input_height, input_width, input_channels = input_size - kernel_height, kernel_width = kernel_size - - output_height = ((input_height + 2 * padding - dilation * (kernel_height - 1) - 1) // stride) + 1 - output_width = ((input_width + 2 * padding - dilation * (kernel_width - 1) - 1) // stride) + 1 - - return (output_height, output_width, out_channels) - - @staticmethod - def pool_output_size(input_size, out_channels, kernel_size, stride=None, padding=0): - input_height, input_width, input_channels = input_size - kernel_height, kernel_width = kernel_size - - if stride is None: - stride = kernel_height - - output_height = ((input_height + 2 * padding - kernel_height) // stride) + 1 - output_width = ((input_width + 2 * padding - kernel_width) // stride) + 1 - - return (output_height, output_width, out_channels) - - def detach_neuron_states(self): - for name, layer in self.named_modules(): - if name != '': - if isinstance(layer, sl.StatefulLayer): - for name, buffer in layer.named_buffers(): - buffer.detach_() - - def init_weights(self): - for name, layer in self.named_modules(): - if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear): - nn.init.xavier_normal_(layer.weight.data) - - def forward(self, x): - # conv 1 - con1_out = self.conv1(x) - iaf1_out = self.iaf1(con1_out) - pool1_out = self.pool1(iaf1_out) - pool1a_out = self.pool1a(iaf1_out) - # conv 2 - conv2_out = self.conv2(pool1_out) - iaf2_out = self.iaf2(conv2_out) - pool2_out = self.pool2(iaf2_out) - # conv 3 - conv3_out = self.conv3(pool2_out) - iaf3_out = self.iaf3(conv3_out) - pool3_out = self.pool3(iaf3_out) - - merge1_out = self.merge1(pool1a_out, pool3_out) - # conv 4 - conv4_out = self.conv4(merge1_out) - iaf4_out = self.iaf4(conv4_out) - pool4_out = self.pool4(iaf4_out) - - flat_out = self.flat(pool4_out) - # fc 1 - fc1_out = self.fc1(flat_out) - iaf1_fc_out = self.iaf1_fc(fc1_out) - # fc 2 - fc2_out = self.fc2(iaf1_fc_out) - iaf2_fc_out = self.iaf2_fc(fc2_out) - # fc 3 - fc3_out = self.fc3(iaf2_fc_out) - iaf3_fc_out = self.iaf3_fc(fc3_out) - # fc 4 - fc4_out = self.fc4(iaf3_fc_out) - iaf4_fc_out = self.iaf4_fc(fc4_out) - # fc 5 - fc5_out = self.fc5(iaf4_fc_out) - iaf5_fc_out = self.iaf5_fc(fc5_out) - - return iaf5_fc_out \ No newline at end of file diff --git a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN6.py b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN6.py deleted file mode 100644 index 98221913..00000000 --- a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN6.py +++ /dev/null @@ -1,141 +0,0 @@ -import torch.nn as nn -import sinabs.layers as sl -from sinabs.exodus.layers import IAFSqueeze - -class SCNN(nn.Module): - def __init__(self, input_size, nb_classes, batch_size, surrogate_fn, min_v_mem=-0.313, spk_thr=2.0) -> None: - super().__init__() - - self.conv1 = nn.Conv2d(2, 8, 2, 1, bias=False) - self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool1 = sl.SumPool2d(2,2) - self.pool1a = sl.SumPool2d(8,8) - - self.conv2 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool2 = sl.SumPool2d(2,2) - - self.conv3 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool3 = sl.SumPool2d(2,2) - - self.conv4 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool4 = sl.SumPool2d(2,2) - - self.flat = nn.Flatten() - - flat_s = SCNN.get_flatten_size(input_size) - - self.fc1 = nn.Linear(flat_s, 100, bias=False) - self.iaf1_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc2 = nn.Linear(100, 100, bias=False) - self.iaf2_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc3 = nn.Linear(100, 100, bias=False) - self.iaf3_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc4 = nn.Linear(100, 100, bias=False) - self.iaf4_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc5 = nn.Linear(100, nb_classes, bias=False) - self.iaf5_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - # skip - - self.merge1 = sl.Merge() - self.merge2 = sl.Merge() - - @staticmethod - def get_flatten_size(input_size): - conv1_dims = SCNN.conv2d_output_size(input_size, 8, (2, 2)) - pool1_dims = SCNN.pool_output_size(conv1_dims, 8, (2, 2)) - - conv2_dims = SCNN.conv2d_output_size(pool1_dims, 8, (2, 2)) - pool2_dims = SCNN.pool_output_size(conv2_dims, 8, (2, 2)) - - conv3_dims = SCNN.conv2d_output_size(pool2_dims, 8, (2, 2)) - pool3_dims = SCNN.pool_output_size(conv3_dims, 8, (2, 2)) - - conv4_dims = SCNN.conv2d_output_size(pool3_dims, 8, (2, 2)) - pool4_dims = SCNN.pool_output_size(conv4_dims, 8, (2, 2)) - - return pool4_dims[0]*pool4_dims[1]*pool4_dims[2] - - @staticmethod - def conv2d_output_size(input_size, out_channels, kernel_size, stride=1, padding=0, dilation=1): - input_height, input_width, input_channels = input_size - kernel_height, kernel_width = kernel_size - - output_height = ((input_height + 2 * padding - dilation * (kernel_height - 1) - 1) // stride) + 1 - output_width = ((input_width + 2 * padding - dilation * (kernel_width - 1) - 1) // stride) + 1 - - return (output_height, output_width, out_channels) - - @staticmethod - def pool_output_size(input_size, out_channels, kernel_size, stride=None, padding=0): - input_height, input_width, input_channels = input_size - kernel_height, kernel_width = kernel_size - - if stride is None: - stride = kernel_height - - output_height = ((input_height + 2 * padding - kernel_height) // stride) + 1 - output_width = ((input_width + 2 * padding - kernel_width) // stride) + 1 - - return (output_height, output_width, out_channels) - - def detach_neuron_states(self): - for name, layer in self.named_modules(): - if name != '': - if isinstance(layer, sl.StatefulLayer): - for name, buffer in layer.named_buffers(): - buffer.detach_() - - def init_weights(self): - for name, layer in self.named_modules(): - if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear): - nn.init.xavier_normal_(layer.weight.data) - - def forward(self, x): - # conv 1 - con1_out = self.conv1(x) - iaf1_out = self.iaf1(con1_out) - pool1_out = self.pool1(iaf1_out) - pool1a_out = self.pool1a(iaf1_out) - # conv 2 - conv2_out = self.conv2(pool1_out) - iaf2_out = self.iaf2(conv2_out) - pool2_out = self.pool2(iaf2_out) - # conv 3 - conv3_out = self.conv3(pool2_out) - iaf3_out = self.iaf3(conv3_out) - pool3_out = self.pool3(iaf3_out) - - merge1_out = self.merge1(pool1a_out, pool3_out) - # conv 4 - conv4_out = self.conv4(merge1_out) - iaf4_out = self.iaf4(conv4_out) - pool4_out = self.pool4(iaf4_out) - - flat_out = self.flat(pool4_out) - # fc 1 - fc1_out = self.fc1(flat_out) - iaf1_fc_out = self.iaf1_fc(fc1_out) - # fc 2 - fc2_out = self.fc2(iaf1_fc_out) - iaf2_fc_out = self.iaf2_fc(fc2_out) - # fc 3 - fc3_out = self.fc3(iaf2_fc_out) - iaf3_fc_out = self.iaf3_fc(fc3_out) - - merge2_out = self.merge2(iaf1_fc_out, iaf3_fc_out) - # fc 4 - fc4_out = self.fc4(merge2_out) - iaf4_fc_out = self.iaf4_fc(fc4_out) - # fc 5 - fc5_out = self.fc5(iaf4_fc_out) - iaf5_fc_out = self.iaf5_fc(fc5_out) - - return iaf5_fc_out \ No newline at end of file diff --git a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN7.py b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN7.py deleted file mode 100644 index 749e55eb..00000000 --- a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN7.py +++ /dev/null @@ -1,141 +0,0 @@ -import torch.nn as nn -import sinabs.layers as sl -from sinabs.exodus.layers import IAFSqueeze - -class SCNN(nn.Module): - def __init__(self, input_size, nb_classes, batch_size, surrogate_fn, min_v_mem=-0.313, spk_thr=2.0) -> None: - super().__init__() - - self.conv1 = nn.Conv2d(2, 8, 2, 1, bias=False) - self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool1 = sl.SumPool2d(2,2) - self.pool1a = sl.SumPool2d(4,4) - self.pool1b = sl.SumPool2d(8,8) - - self.conv2 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool2 = sl.SumPool2d(2,2) - - self.conv3 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool3 = sl.SumPool2d(2,2) - - self.conv4 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool4 = sl.SumPool2d(2,2) - - self.flat = nn.Flatten() - - flat_s = SCNN.get_flatten_size(input_size) - - self.fc1 = nn.Linear(flat_s, 100, bias=False) - self.iaf1_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc2 = nn.Linear(100, 100, bias=False) - self.iaf2_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc3 = nn.Linear(100, 100, bias=False) - self.iaf3_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc4 = nn.Linear(100, 100, bias=False) - self.iaf4_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc5 = nn.Linear(100, nb_classes, bias=False) - self.iaf5_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - # skip - - self.merge1a = sl.Merge() - self.merge1b = sl.Merge() - - @staticmethod - def get_flatten_size(input_size): - conv1_dims = SCNN.conv2d_output_size(input_size, 8, (2, 2)) - pool1_dims = SCNN.pool_output_size(conv1_dims, 8, (2, 2)) - - conv2_dims = SCNN.conv2d_output_size(pool1_dims, 8, (2, 2)) - pool2_dims = SCNN.pool_output_size(conv2_dims, 8, (2, 2)) - - conv3_dims = SCNN.conv2d_output_size(pool2_dims, 8, (2, 2)) - pool3_dims = SCNN.pool_output_size(conv3_dims, 8, (2, 2)) - - conv4_dims = SCNN.conv2d_output_size(pool3_dims, 8, (2, 2)) - pool4_dims = SCNN.pool_output_size(conv4_dims, 8, (2, 2)) - - return pool4_dims[0]*pool4_dims[1]*pool4_dims[2] - - @staticmethod - def conv2d_output_size(input_size, out_channels, kernel_size, stride=1, padding=0, dilation=1): - input_height, input_width, input_channels = input_size - kernel_height, kernel_width = kernel_size - - output_height = ((input_height + 2 * padding - dilation * (kernel_height - 1) - 1) // stride) + 1 - output_width = ((input_width + 2 * padding - dilation * (kernel_width - 1) - 1) // stride) + 1 - - return (output_height, output_width, out_channels) - - @staticmethod - def pool_output_size(input_size, out_channels, kernel_size, stride=None, padding=0): - input_height, input_width, input_channels = input_size - kernel_height, kernel_width = kernel_size - - if stride is None: - stride = kernel_height - - output_height = ((input_height + 2 * padding - kernel_height) // stride) + 1 - output_width = ((input_width + 2 * padding - kernel_width) // stride) + 1 - - return (output_height, output_width, out_channels) - - def detach_neuron_states(self): - for name, layer in self.named_modules(): - if name != '': - if isinstance(layer, sl.StatefulLayer): - for name, buffer in layer.named_buffers(): - buffer.detach_() - - def init_weights(self): - for name, layer in self.named_modules(): - if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear): - nn.init.xavier_normal_(layer.weight.data) - - def forward(self, x): - # conv 1 - con1_out = self.conv1(x) - iaf1_out = self.iaf1(con1_out) - pool1_out = self.pool1(iaf1_out) - pool1a_out = self.pool1a(iaf1_out) - pool1b_out = self.pool1b(iaf1_out) - # conv 2 - conv2_out = self.conv2(pool1_out) - iaf2_out = self.iaf2(conv2_out) - pool2_out = self.pool2(iaf2_out) - # conv 3 - merge_1a_out = self.merge1a(pool1a_out, pool2_out) - conv3_out = self.conv3(merge_1a_out) - iaf3_out = self.iaf3(conv3_out) - pool3_out = self.pool3(iaf3_out) - # conv 4 - merge_1b_out = self.merge1b(pool1b_out, pool3_out) - conv4_out = self.conv4(merge_1b_out) - iaf4_out = self.iaf4(conv4_out) - pool4_out = self.pool4(iaf4_out) - - flat_out = self.flat(pool4_out) - # fc 1 - fc1_out = self.fc1(flat_out) - iaf1_fc_out = self.iaf1_fc(fc1_out) - # fc 2 - fc2_out = self.fc2(iaf1_fc_out) - iaf2_fc_out = self.iaf2_fc(fc2_out) - # fc 3 - fc3_out = self.fc3(iaf2_fc_out) - iaf3_fc_out = self.iaf3_fc(fc3_out) - # fc 4 - fc4_out = self.fc4(iaf3_fc_out) - iaf4_fc_out = self.iaf4_fc(fc4_out) - # fc 5 - fc5_out = self.fc5(iaf4_fc_out) - iaf5_fc_out = self.iaf5_fc(fc5_out) - - return iaf5_fc_out \ No newline at end of file diff --git a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN8.py b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN8.py deleted file mode 100644 index c99af380..00000000 --- a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN8.py +++ /dev/null @@ -1,146 +0,0 @@ -import torch.nn as nn -import sinabs.layers as sl -from sinabs.exodus.layers import IAFSqueeze - -class SCNN(nn.Module): - def __init__(self, input_size, nb_classes, batch_size, surrogate_fn, min_v_mem=-0.313, spk_thr=2.0) -> None: - super().__init__() - - self.conv1 = nn.Conv2d(2, 8, 2, 1, bias=False) - self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool1 = sl.SumPool2d(2,2) - self.pool1a = sl.SumPool2d(4,4) - self.pool1b = sl.SumPool2d(8,8) - - self.conv2 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool2 = sl.SumPool2d(2,2) - - self.conv3 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool3 = sl.SumPool2d(2,2) - - self.conv4 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool4 = sl.SumPool2d(2,2) - - self.flat = nn.Flatten() - - flat_s = SCNN.get_flatten_size(input_size) - - self.fc1 = nn.Linear(flat_s, 100, bias=False) - self.iaf1_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc2 = nn.Linear(100, 100, bias=False) - self.iaf2_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc3 = nn.Linear(100, 100, bias=False) - self.iaf3_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc4 = nn.Linear(100, 100, bias=False) - self.iaf4_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc5 = nn.Linear(100, nb_classes, bias=False) - self.iaf5_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - # skip - - self.merge1a = sl.Merge() - self.merge1b = sl.Merge() - - self.merge_fc1a = sl.Merge() - self.merge_fc1b = sl.Merge() - - @staticmethod - def get_flatten_size(input_size): - conv1_dims = SCNN.conv2d_output_size(input_size, 8, (2, 2)) - pool1_dims = SCNN.pool_output_size(conv1_dims, 8, (2, 2)) - - conv2_dims = SCNN.conv2d_output_size(pool1_dims, 8, (2, 2)) - pool2_dims = SCNN.pool_output_size(conv2_dims, 8, (2, 2)) - - conv3_dims = SCNN.conv2d_output_size(pool2_dims, 8, (2, 2)) - pool3_dims = SCNN.pool_output_size(conv3_dims, 8, (2, 2)) - - conv4_dims = SCNN.conv2d_output_size(pool3_dims, 8, (2, 2)) - pool4_dims = SCNN.pool_output_size(conv4_dims, 8, (2, 2)) - - return pool4_dims[0]*pool4_dims[1]*pool4_dims[2] - - @staticmethod - def conv2d_output_size(input_size, out_channels, kernel_size, stride=1, padding=0, dilation=1): - input_height, input_width, input_channels = input_size - kernel_height, kernel_width = kernel_size - - output_height = ((input_height + 2 * padding - dilation * (kernel_height - 1) - 1) // stride) + 1 - output_width = ((input_width + 2 * padding - dilation * (kernel_width - 1) - 1) // stride) + 1 - - return (output_height, output_width, out_channels) - - @staticmethod - def pool_output_size(input_size, out_channels, kernel_size, stride=None, padding=0): - input_height, input_width, input_channels = input_size - kernel_height, kernel_width = kernel_size - - if stride is None: - stride = kernel_height - - output_height = ((input_height + 2 * padding - kernel_height) // stride) + 1 - output_width = ((input_width + 2 * padding - kernel_width) // stride) + 1 - - return (output_height, output_width, out_channels) - - def detach_neuron_states(self): - for name, layer in self.named_modules(): - if name != '': - if isinstance(layer, sl.StatefulLayer): - for name, buffer in layer.named_buffers(): - buffer.detach_() - - def init_weights(self): - for name, layer in self.named_modules(): - if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear): - nn.init.xavier_normal_(layer.weight.data) - - def forward(self, x): - # conv 1 - con1_out = self.conv1(x) - iaf1_out = self.iaf1(con1_out) - pool1_out = self.pool1(iaf1_out) - pool1a_out = self.pool1a(iaf1_out) - pool1b_out = self.pool1b(iaf1_out) - # conv 2 - conv2_out = self.conv2(pool1_out) - iaf2_out = self.iaf2(conv2_out) - pool2_out = self.pool2(iaf2_out) - # conv 3 - merge_1a_out = self.merge1a(pool1a_out, pool2_out) - conv3_out = self.conv3(merge_1a_out) - iaf3_out = self.iaf3(conv3_out) - pool3_out = self.pool3(iaf3_out) - # conv 4 - merge_1b_out = self.merge1b(pool1b_out, pool3_out) - conv4_out = self.conv4(merge_1b_out) - iaf4_out = self.iaf4(conv4_out) - pool4_out = self.pool4(iaf4_out) - - flat_out = self.flat(pool4_out) - # fc 1 - fc1_out = self.fc1(flat_out) - iaf1_fc_out = self.iaf1_fc(fc1_out) - # fc 2 - fc2_out = self.fc2(iaf1_fc_out) - iaf2_fc_out = self.iaf2_fc(fc2_out) - # fc 3 - merge_fc1a_out = self.merge_fc1a(iaf1_fc_out, iaf2_fc_out) - fc3_out = self.fc3(merge_fc1a_out) - iaf3_fc_out = self.iaf3_fc(fc3_out) - # fc 4 - merge_fc1b_out = self.merge_fc1b(iaf1_fc_out, iaf3_fc_out) - fc4_out = self.fc4(merge_fc1b_out) - iaf4_fc_out = self.iaf4_fc(fc4_out) - # fc 5 - fc5_out = self.fc5(iaf4_fc_out) - iaf5_fc_out = self.iaf5_fc(fc5_out) - - return iaf5_fc_out \ No newline at end of file From b2a91b0d69177a526c74b81649669244feea52ae Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Mon, 13 May 2024 14:08:21 +0200 Subject: [PATCH 084/379] complext networks beyond sequential test cases --- .../split_and_merge.ipynb | 368 +++++++++++++++++ .../two_networks_merging_outputs.ipynb | 380 ++++++++++++++++++ 2 files changed, 748 insertions(+) create mode 100644 tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/split_and_merge.ipynb create mode 100644 tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/two_networks_merging_outputs.ipynb diff --git a/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/split_and_merge.ipynb b/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/split_and_merge.ipynb new file mode 100644 index 00000000..e0af08dc --- /dev/null +++ b/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/split_and_merge.ipynb @@ -0,0 +1,368 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "metadata": {} + }, + "outputs": [], + "source": [ + "import torch\n", + "import torch.nn as nn\n", + "from sinabs.backend.dynapcnn import DynapcnnNetworkGraph\n", + "from sinabs.layers import Merge, IAFSqueeze, SumPool2d\n", + "import sinabs.layers as sl\n", + "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "metadata": {} + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "torch.manual_seed(0)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "metadata": {} + }, + "outputs": [], + "source": [ + "channels = 2\n", + "height = 34\n", + "width = 34\n", + "\n", + "input_shape = (channels, height, width)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Network Module\n", + "\n", + "We need to define a `nn.Module` implementing the network we want the chip to reproduce." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "```mermaid\n", + "stateDiagram\n", + " [*] --> A\n", + " A --> B\n", + " B --> C\n", + " C --> D\n", + " B --> E\n", + " E --> F\n", + " D --> G\n", + " F --> G\n", + " G --> [*]\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "metadata": {} + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "torch.Size([8, 360])\n", + "torch.Size([8, 360]) torch.Size([8, 360])\n" + ] + } + ], + "source": [ + "class SNN(nn.Module):\n", + " def __init__(self, nb_classes, batch_size, surrogate_fn, min_v_mem=-0.313, spk_thr=2.0) -> None:\n", + " super().__init__()\n", + "\n", + " self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False)\n", + " self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", + "\n", + " self.conv2 = nn.Conv2d(10, 10, 2, 1, bias=False)\n", + " self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", + " self.pool2 = sl.SumPool2d(2,2)\n", + " self.pool2a = sl.SumPool2d(5,5)\n", + "\n", + " self.conv3 = nn.Conv2d(10, 10, 2, 1, bias=False)\n", + " self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", + " self.pool3 = sl.SumPool2d(2,2)\n", + "\n", + " self.conv4 = nn.Conv2d(10, 10, 2, 1, bias=False)\n", + " self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", + "\n", + " self.flat = nn.Flatten()\n", + " self.flat_a = nn.Flatten()\n", + "\n", + " self.fc1 = nn.Linear(360, 360, bias=False)\n", + " self.iaf1_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", + "\n", + " self.fc2 = nn.Linear(360, 360, bias=False)\n", + " self.iaf2_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", + "\n", + " self.fc3 = nn.Linear(360, nb_classes, bias=False)\n", + " self.iaf3_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", + "\n", + " # -- merges --\n", + " self.merge1 = Merge()\n", + "\n", + " def forward(self, x):\n", + " # conv 1 - A\n", + " con1_out = self.conv1(x)\n", + " iaf1_out = self.iaf1(con1_out)\n", + "\n", + " # conv 2 - B\n", + " conv2_out = self.conv2(iaf1_out)\n", + " iaf2_out = self.iaf2(conv2_out)\n", + " pool2_out = self.pool2(iaf2_out)\n", + " pool2a_out = self.pool2a(iaf2_out)\n", + "\n", + " # conv 3 - C\n", + " conv3_out = self.conv3(pool2_out)\n", + " iaf3_out = self.iaf3(conv3_out)\n", + " pool3_out = self.pool3(iaf3_out)\n", + "\n", + " # conv 4 - D\n", + " conv4_out = self.conv4(pool3_out)\n", + " iaf4_out = self.iaf4(conv4_out)\n", + " flat_out = self.flat(iaf4_out)\n", + " \n", + " # fc 1 - E\n", + " flat_a_out = self.flat_a(pool2a_out)\n", + " print(flat_a_out.shape)\n", + " fc1_out = self.fc1(flat_a_out)\n", + " iaf1_fc_out = self.iaf1_fc(fc1_out)\n", + "\n", + " # fc 2 - F\n", + " fc2_out = self.fc2(iaf1_fc_out)\n", + " iaf2_fc_out = self.iaf2_fc(fc2_out)\n", + "\n", + " # fc 2 - G\n", + " print(flat_out.shape, iaf2_fc_out.shape)\n", + " merge1_out = self.merge1(flat_out, iaf2_fc_out)\n", + " fc3_out = self.fc3(merge1_out)\n", + " iaf3_fc_out = self.iaf3_fc(fc3_out)\n", + "\n", + " return iaf3_fc_out\n", + " \n", + "snn = SNN(11, 8, PeriodicExponential())\n", + "\n", + "x = torch.randn((8, *input_shape))\n", + "\n", + "out = snn(x)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## DynapcnnNetwork Class\n", + "\n", + "In the constructor of `DynapcnnNetworkGraph` the SNN passed as argument (defined as a `nn.Module`) will be parsed such that each layer is represented in a computational graph (using `nirtorch.extract_torch_graph`). \n", + "\n", + "The layers are the `nodes` of the graph, while their connectivity (how the outputs from a layer are sent to other layers) is represented as `edges`, represented in a `list` of `tuples`.\n", + "\n", + "Once the constructor finishes its initialization, the `hw_model.dynapcnn_layers` property is a dictionary where each entry represents the ID of a `DynapcnnLayer` instance (an `int` from `0` to `L`), with this entry containing a `DynapcnnLayer` instance where a subset of the layers in the original SNN has been incorporated into, the core such instance has been assigned to, and the list of `DynapcnnLayer` instances (their IDs) the layer targets." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "metadata": {} + }, + "outputs": [], + "source": [ + "hw_model = DynapcnnNetworkGraph(\n", + " snn,\n", + " discretize=True,\n", + " input_shape=input_shape\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The `hw_model.to()` call will figure out into which core eac `DynapcnnLayer` instance will be assigned to. Once this assingment is made the instance itself is used to configure the `CNNLayerConfig` instance representing the core's configuration.\n", + "\n", + "If the cores' configuration is valid, each `DynapcnnLayer` instance and their respective destinations will be used to create a computational graph that encodes how the `forward` method of `hw_model.network` (a `nn.Module` using the `DynapcnnLayer` instances) propagates that through the network." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "metadata": {} + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Network is valid\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "hw_model.to(device=\"speck2fmodule:0\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The layers comprising our `hw_model` and their respective metadata can be inspected by calling `print` on a `DynapcnnNetworkGraph` instance." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "metadata": {} + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "---- DynapcnnLayer 0 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 0): Conv2d(2, 10, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + "(node 1): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "(node 2): SumPool2d(norm_type=1, kernel_size=3, stride=3, ceil_mode=False)\n", + "(node 3): SumPool2d(norm_type=1, kernel_size=4, stride=4, ceil_mode=False)\n", + "> layer destinations: [1, 2]\n", + "> assigned core: 0\n", + "\n", + "---- DynapcnnLayer 1 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 4): Conv2d(10, 10, kernel_size=(4, 4), stride=(1, 1), bias=False)\n", + "(node 6): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [2]\n", + "> assigned core: 1\n", + "\n", + "---- DynapcnnLayer 2 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 7): Conv2d(10, 10, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + "(node 8): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [3]\n", + "> assigned core: 2\n", + "\n", + "---- DynapcnnLayer 3 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 9): Conv2d(10, 1, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + "(node 10): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [4]\n", + "> assigned core: 3\n", + "\n", + "---- DynapcnnLayer 4 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 11): Conv2d(1, 100, kernel_size=(6, 6), stride=(1, 1), bias=False)\n", + "(node 12): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [5, 6]\n", + "> assigned core: 4\n", + "\n", + "---- DynapcnnLayer 5 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 13): Conv2d(100, 100, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + "(node 15): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [6, 7]\n", + "> assigned core: 5\n", + "\n", + "---- DynapcnnLayer 6 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 17): Conv2d(100, 100, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + "(node 18): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [7]\n", + "> assigned core: 6\n", + "\n", + "---- DynapcnnLayer 7 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 19): Conv2d(100, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + "(node 20): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: []\n", + "> assigned core: 7\n", + "\n", + "\n" + ] + } + ], + "source": [ + "print(hw_model)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "speck-rescnn", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/two_networks_merging_outputs.ipynb b/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/two_networks_merging_outputs.ipynb new file mode 100644 index 00000000..9e0ee4b4 --- /dev/null +++ b/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/two_networks_merging_outputs.ipynb @@ -0,0 +1,380 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "metadata": {} + }, + "outputs": [], + "source": [ + "import torch\n", + "import torch.nn as nn\n", + "from sinabs.backend.dynapcnn import DynapcnnNetworkGraph\n", + "from sinabs.layers import Merge, IAFSqueeze, SumPool2d\n", + "import sinabs.layers as sl\n", + "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "metadata": {} + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "torch.manual_seed(0)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "metadata": {} + }, + "outputs": [], + "source": [ + "channels = 2\n", + "height = 34\n", + "width = 34\n", + "\n", + "input_shape = (channels, height, width)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Network Module\n", + "\n", + "We need to define a `nn.Module` implementing the network we want the chip to reproduce." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "```mermaid\n", + "stateDiagram\n", + " [*] --> A\n", + " A --> B\n", + " B --> C\n", + " D --> E\n", + " E --> F\n", + " C --> G\n", + " F --> G\n", + " G --> H\n", + " H --> I\n", + " I --> [*]\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "metadata": {} + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "torch.Size([8, 490]) torch.Size([8, 490])\n" + ] + } + ], + "source": [ + "class SNN(nn.Module):\n", + " def __init__(self, nb_classes, batch_size, surrogate_fn, min_v_mem=-0.313, spk_thr=2.0) -> None:\n", + " super().__init__()\n", + "\n", + " self.conv_A = nn.Conv2d(2, 10, 2, 1, bias=False)\n", + " self.iaf_A = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", + "\n", + " self.conv_B = nn.Conv2d(10, 10, 2, 1, bias=False)\n", + " self.iaf_B = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", + " self.pool_B = sl.SumPool2d(2,2)\n", + "\n", + " self.conv_C = nn.Conv2d(10, 10, 2, 1, bias=False)\n", + " self.iaf_C = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", + " self.pool_C = sl.SumPool2d(2,2)\n", + "\n", + " self.conv_D = nn.Conv2d(2, 10, 2, 1, bias=False)\n", + " self.iaf_D = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", + "\n", + " self.conv_E = nn.Conv2d(10, 10, 2, 1, bias=False)\n", + " self.iaf_E = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", + " self.pool_E = sl.SumPool2d(2,2)\n", + "\n", + " self.conv_F = nn.Conv2d(10, 10, 2, 1, bias=False)\n", + " self.iaf_F = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", + " self.pool_F = sl.SumPool2d(2,2)\n", + "\n", + " self.flat_brach1 = nn.Flatten()\n", + " self.flat_brach2 = nn.Flatten()\n", + " self.merge = Merge()\n", + "\n", + " self.fc1 = nn.Linear(490, 200, bias=False)\n", + " self.iaf1_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", + "\n", + " self.fc2 = nn.Linear(200, 200, bias=False)\n", + " self.iaf2_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", + "\n", + " self.fc3 = nn.Linear(200, nb_classes, bias=False)\n", + " self.iaf3_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", + "\n", + " def forward(self, x):\n", + " # conv 1 - A\n", + " conv_A_out = self.conv_A(x)\n", + " iaf_A_out = self.iaf_A(conv_A_out)\n", + " # conv 2 - B\n", + " conv_B_out = self.conv_B(iaf_A_out)\n", + " iaf_B_out = self.iaf_B(conv_B_out)\n", + " pool_B_out = self.pool_B(iaf_B_out)\n", + " # conv 3 - C\n", + " conv_C_out = self.conv_C(pool_B_out)\n", + " iaf_C_out = self.iaf_C(conv_C_out)\n", + " pool_C_out = self.pool_C(iaf_C_out)\n", + "\n", + " # ---\n", + "\n", + " # conv 4 - D\n", + " conv_D_out = self.conv_D(x)\n", + " iaf_D_out = self.iaf_D(conv_D_out)\n", + " # conv 5 - E\n", + " conv_E_out = self.conv_E(iaf_D_out)\n", + " iaf_E_out = self.iaf_E(conv_E_out)\n", + " pool_E_out = self.pool_E(iaf_E_out)\n", + " # conv 6 - F\n", + " conv_F_out = self.conv_F(pool_E_out)\n", + " iaf_F_out = self.iaf_F(conv_F_out)\n", + " pool_F_out = self.pool_F(iaf_F_out)\n", + "\n", + " # ---\n", + "\n", + " flat_brach1_out = self.flat_brach1(pool_C_out)\n", + " flat_brach2_out = self.flat_brach2(pool_F_out)\n", + " merge_out = self.merge(flat_brach1_out, flat_brach2_out)\n", + " print(flat_brach1_out.shape, flat_brach2_out.shape)\n", + "\n", + " # FC 7 - G\n", + " fc1_out = self.fc1(merge_out)\n", + " iaf1_fc_out = self.iaf1_fc(fc1_out)\n", + " # FC 8 - H\n", + " fc2_out = self.fc2(iaf1_fc_out)\n", + " iaf2_fc_out = self.iaf2_fc(fc2_out)\n", + " # FC 9 - I\n", + " fc3_out = self.fc3(iaf2_fc_out)\n", + " iaf3_fc_out = self.iaf3_fc(fc3_out)\n", + "\n", + " return iaf3_fc_out\n", + " \n", + "snn = SNN(11, 8, PeriodicExponential())\n", + "\n", + "x = torch.randn((8, *input_shape))\n", + "\n", + "out = snn(x)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## DynapcnnNetwork Class\n", + "\n", + "In the constructor of `DynapcnnNetworkGraph` the SNN passed as argument (defined as a `nn.Module`) will be parsed such that each layer is represented in a computational graph (using `nirtorch.extract_torch_graph`). \n", + "\n", + "The layers are the `nodes` of the graph, while their connectivity (how the outputs from a layer are sent to other layers) is represented as `edges`, represented in a `list` of `tuples`.\n", + "\n", + "Once the constructor finishes its initialization, the `hw_model.dynapcnn_layers` property is a dictionary where each entry represents the ID of a `DynapcnnLayer` instance (an `int` from `0` to `L`), with this entry containing a `DynapcnnLayer` instance where a subset of the layers in the original SNN has been incorporated into, the core such instance has been assigned to, and the list of `DynapcnnLayer` instances (their IDs) the layer targets." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "metadata": {} + }, + "outputs": [], + "source": [ + "hw_model = DynapcnnNetworkGraph(\n", + " snn,\n", + " discretize=True,\n", + " input_shape=input_shape\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The `hw_model.to()` call will figure out into which core eac `DynapcnnLayer` instance will be assigned to. Once this assingment is made the instance itself is used to configure the `CNNLayerConfig` instance representing the core's configuration.\n", + "\n", + "If the cores' configuration is valid, each `DynapcnnLayer` instance and their respective destinations will be used to create a computational graph that encodes how the `forward` method of `hw_model.network` (a `nn.Module` using the `DynapcnnLayer` instances) propagates that through the network." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "metadata": {} + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Network is valid\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "hw_model.to(device=\"speck2fmodule:0\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The layers comprising our `hw_model` and their respective metadata can be inspected by calling `print` on a `DynapcnnNetworkGraph` instance." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "metadata": {} + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "---- DynapcnnLayer 0 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 0): Conv2d(2, 10, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + "(node 1): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "(node 2): SumPool2d(norm_type=1, kernel_size=3, stride=3, ceil_mode=False)\n", + "(node 3): SumPool2d(norm_type=1, kernel_size=4, stride=4, ceil_mode=False)\n", + "> layer destinations: [1, 2]\n", + "> assigned core: 0\n", + "\n", + "---- DynapcnnLayer 1 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 4): Conv2d(10, 10, kernel_size=(4, 4), stride=(1, 1), bias=False)\n", + "(node 6): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [2]\n", + "> assigned core: 1\n", + "\n", + "---- DynapcnnLayer 2 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 7): Conv2d(10, 10, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + "(node 8): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [3]\n", + "> assigned core: 2\n", + "\n", + "---- DynapcnnLayer 3 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 9): Conv2d(10, 1, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + "(node 10): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [4]\n", + "> assigned core: 3\n", + "\n", + "---- DynapcnnLayer 4 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 11): Conv2d(1, 100, kernel_size=(6, 6), stride=(1, 1), bias=False)\n", + "(node 12): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [5, 6]\n", + "> assigned core: 4\n", + "\n", + "---- DynapcnnLayer 5 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 13): Conv2d(100, 100, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + "(node 15): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [6, 7]\n", + "> assigned core: 5\n", + "\n", + "---- DynapcnnLayer 6 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 17): Conv2d(100, 100, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + "(node 18): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [7]\n", + "> assigned core: 6\n", + "\n", + "---- DynapcnnLayer 7 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 19): Conv2d(100, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + "(node 20): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: []\n", + "> assigned core: 7\n", + "\n", + "\n" + ] + } + ], + "source": [ + "print(hw_model)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "speck-rescnn", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 2ac878597eeea9a670c3c02872137edd491433e6 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Tue, 14 May 2024 10:52:07 +0200 Subject: [PATCH 085/379] architectures variations --- .../using_SumPool2d/models/ResSCNN_12.py | 91 ++++++++++++++++++ .../using_SumPool2d/models/ResSCNN_13.py | 96 +++++++++++++++++++ 2 files changed, 187 insertions(+) create mode 100644 tests/test_nonsequential/using_SumPool2d/models/ResSCNN_12.py create mode 100644 tests/test_nonsequential/using_SumPool2d/models/ResSCNN_13.py diff --git a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_12.py b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_12.py new file mode 100644 index 00000000..adfa33cd --- /dev/null +++ b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_12.py @@ -0,0 +1,91 @@ +import torch.nn as nn +import sinabs.layers as sl +from sinabs.exodus.layers import IAFSqueeze + +class SCNN(nn.Module): + def __init__(self, input_size, nb_classes, batch_size, surrogate_fn, min_v_mem=-0.313, spk_thr=2.0) -> None: + super().__init__() + + self.conv1 = nn.Conv2d(2, 1, 3, 1, bias=False) + self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool1 = sl.SumPool2d(3,3) + + self.conv2 = nn.Conv2d(1, 8, 3, 1, bias=False) + self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool2 = sl.SumPool2d(3,3) + + self.conv3 = nn.Conv2d(8, 16, 3, 1, bias=False) + self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool3 = sl.SumPool2d(3,3) + + self.conv4 = nn.Conv2d(16, 32, 3, 1, bias=False) + self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.flat = nn.Flatten() + + self.fc1 = nn.Linear(32, 1024, bias=False) + self.iaf1_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc2 = nn.Linear(1024, 512, bias=False) + self.iaf2_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc3 = nn.Linear(512, 256, bias=False) + self.iaf3_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc4 = nn.Linear(256, 128, bias=False) + self.iaf4_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc5 = nn.Linear(128, nb_classes, bias=False) + self.iaf5_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + def detach_neuron_states(self): + for name, layer in self.named_modules(): + if name != '': + if isinstance(layer, sl.StatefulLayer): + for name, buffer in layer.named_buffers(): + buffer.detach_() + + def init_weights(self): + for name, layer in self.named_modules(): + if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear): + nn.init.xavier_normal_(layer.weight.data) + + def rescale_conv_weights(self, rescale_fn, lambda_): + rescale_fn(self.conv2, [(3, 3)], lambda_) + rescale_fn(self.conv3, [(3, 3)], lambda_) + rescale_fn(self.conv4, [(3, 3)], lambda_) + + def forward(self, x): + # conv 1 + con1_out = self.conv1(x) + iaf1_out = self.iaf1(con1_out) + pool1_out = self.pool1(iaf1_out) + # conv 2 + conv2_out = self.conv2(pool1_out) + iaf2_out = self.iaf2(conv2_out) + pool2_out = self.pool2(iaf2_out) + # conv 3 + conv3_out = self.conv3(pool2_out) + iaf3_out = self.iaf3(conv3_out) + pool3_out = self.pool3(iaf3_out) + # conv 4 + conv4_out = self.conv4(pool3_out) + iaf4_out = self.iaf4(conv4_out) + flat_out = self.flat(iaf4_out) + # fc 1 + fc1_out = self.fc1(flat_out) + iaf1_fc_out = self.iaf1_fc(fc1_out) + # fc 2 + fc2_out = self.fc2(iaf1_fc_out) + iaf2_fc_out = self.iaf2_fc(fc2_out) + # fc 3 + fc3_out = self.fc3(iaf2_fc_out) + iaf3_fc_out = self.iaf3_fc(fc3_out) + # fc 4 + fc4_out = self.fc4(iaf3_fc_out) + iaf4_fc_out = self.iaf4_fc(fc4_out) + # fc 5 + fc5_out = self.fc5(iaf4_fc_out) + iaf5_fc_out = self.iaf5_fc(fc5_out) + + return iaf5_fc_out \ No newline at end of file diff --git a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_13.py b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_13.py new file mode 100644 index 00000000..8e473dcd --- /dev/null +++ b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_13.py @@ -0,0 +1,96 @@ +import torch.nn as nn +import sinabs.layers as sl +from sinabs.exodus.layers import IAFSqueeze + +class SCNN(nn.Module): + def __init__(self, nb_classes, batch_size, surrogate_fn, min_v_mem=-0.313, spk_thr=2.0) -> None: + super().__init__() + + self.conv1 = nn.Conv2d(2, 16, 3, 1, bias=False) + self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool1 = sl.SumPool2d(2,2) + + self.conv2 = nn.Conv2d(16, 16, 3, 1, bias=False) + self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool2 = sl.SumPool2d(2,2) + + self.conv3 = nn.Conv2d(16, 16, 3, 1, bias=False) + self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool3 = sl.SumPool2d(2,2) + + self.conv4 = nn.Conv2d(16, 16, 3, 1, bias=False) + self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool4 = sl.SumPool2d(2,2) + + self.conv5 = nn.Conv2d(16, 16, 3, 1, bias=False) + self.iaf5 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + self.pool5 = sl.SumPool2d(2,2) + + self.flat = nn.Flatten() + + self.fc1 = nn.Linear(64, 200, bias=False) + self.iaf1_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc2 = nn.Linear(200, 200, bias=False) + self.iaf2_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc3 = nn.Linear(200, 200, bias=False) + self.iaf3_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + self.fc4 = nn.Linear(200, nb_classes, bias=False) + self.iaf4_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) + + def detach_neuron_states(self): + for name, layer in self.named_modules(): + if name != '': + if isinstance(layer, sl.StatefulLayer): + for name, buffer in layer.named_buffers(): + buffer.detach_() + + def init_weights(self): + for name, layer in self.named_modules(): + if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear): + nn.init.xavier_normal_(layer.weight.data) + + def rescale_conv_weights(self, rescale_fn, lambda_): + rescale_fn(self.conv2, [(2, 2)], lambda_) + rescale_fn(self.conv3, [(2, 2)], lambda_) + rescale_fn(self.conv4, [(2, 2)], lambda_) + rescale_fn(self.conv5, [(2, 2)], lambda_) + + def forward(self, x): + # conv 1 + con1_out = self.conv1(x) + iaf1_out = self.iaf1(con1_out) + pool1_out = self.pool1(iaf1_out) + # conv 2 + conv2_out = self.conv2(pool1_out) + iaf2_out = self.iaf2(conv2_out) + pool2_out = self.pool2(iaf2_out) + # conv 3 + conv3_out = self.conv3(pool2_out) + iaf3_out = self.iaf3(conv3_out) + pool3_out = self.pool3(iaf3_out) + # conv 4 + conv4_out = self.conv4(pool3_out) + iaf4_out = self.iaf4(conv4_out) + pool4_out = self.pool4(iaf4_out) + # conv 5 + conv5_out = self.conv5(pool4_out) + iaf5_out = self.iaf5(conv5_out) + pool5_out = self.pool5(iaf5_out) + # fc 1 + flat_out = self.flat(pool5_out) + fc1_out = self.fc1(flat_out) + iaf1_fc_out = self.iaf1_fc(fc1_out) + # fc 2 + fc2_out = self.fc2(iaf1_fc_out) + iaf2_fc_out = self.iaf2_fc(fc2_out) + # fc 3 + fc3_out = self.fc3(iaf2_fc_out) + iaf3_fc_out = self.iaf3_fc(fc3_out) + # fc 4 + fc4_out = self.fc4(iaf3_fc_out) + iaf4_fc_out = self.iaf4_fc(fc4_out) + + return iaf4_fc_out \ No newline at end of file From 187e0aa779018d7fb83d21901b5c0879692f3809 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Tue, 14 May 2024 18:08:43 +0200 Subject: [PATCH 086/379] unit testing network architecture examples from https://github.com/synsense/sinabs/issues/181 --- .../architectures_samples.py | 260 ++++++++++++++++-- .../conftest_dynapcnnnetwork.py | 232 +++++++++++++++- .../complex_network_structure.ipynb | 142 ++++------ .../split_and_merge.ipynb | 167 ++++++----- .../two_networks_merging_outputs.ipynb | 138 +++++----- 5 files changed, 669 insertions(+), 270 deletions(-) diff --git a/tests/test_dynapcnnnetwork/architectures_samples.py b/tests/test_dynapcnnnetwork/architectures_samples.py index 5adde8e0..e5bc2302 100644 --- a/tests/test_dynapcnnnetwork/architectures_samples.py +++ b/tests/test_dynapcnnnetwork/architectures_samples.py @@ -1,6 +1,6 @@ import torch import torch.nn as nn -from sinabs.layers import Merge, IAFSqueeze +from sinabs.layers import Merge, IAFSqueeze, SumPool2d class EXAMPLE_1(nn.Module): def __init__(self) -> None: @@ -158,8 +158,8 @@ def forward(self, x): fc3_iaf_out = self.fc3_iaf(fc3_out) return fc3_iaf_out - -class EXAMPLE_5(nn.Module): + +class EXAMPLE_4(nn.Module): def __init__(self) -> None: super().__init__() @@ -185,15 +185,11 @@ def __init__(self) -> None: self.fc2 = nn.Linear(100, 100, bias=False) self.fc2_iaf = IAFSqueeze(batch_size=1) - self.fc3 = nn.Linear(100, 100, bias=False) + self.fc3 = nn.Linear(100, 10, bias=False) self.fc3_iaf = IAFSqueeze(batch_size=1) - self.fc4 = nn.Linear(100, 10, bias=False) - self.fc4_iaf = IAFSqueeze(batch_size=1) - self.merge1 = Merge() self.merge2 = Merge() - self.merge3 = Merge() def forward(self, x): # -- conv. block 0 -- @@ -222,14 +218,10 @@ def forward(self, x): merge2_out = self.merge2(fc1_iaf_out, fc2_iaf_out) fc3_out = self.fc3(merge2_out) fc3_iaf_out = self.fc3_iaf(fc3_out) - # -- fc clock 7 -- - merge3_out = self.merge3(fc2_iaf_out, fc3_iaf_out) - fc4_out = self.fc4(merge3_out) - fc4_iaf_out = self.fc4_iaf(fc4_out) - return fc4_iaf_out - -class EXAMPLE_4(nn.Module): + return fc3_iaf_out + +class EXAMPLE_5(nn.Module): def __init__(self) -> None: super().__init__() @@ -255,11 +247,15 @@ def __init__(self) -> None: self.fc2 = nn.Linear(100, 100, bias=False) self.fc2_iaf = IAFSqueeze(batch_size=1) - self.fc3 = nn.Linear(100, 10, bias=False) + self.fc3 = nn.Linear(100, 100, bias=False) self.fc3_iaf = IAFSqueeze(batch_size=1) + self.fc4 = nn.Linear(100, 10, bias=False) + self.fc4_iaf = IAFSqueeze(batch_size=1) + self.merge1 = Merge() self.merge2 = Merge() + self.merge3 = Merge() def forward(self, x): # -- conv. block 0 -- @@ -288,5 +284,235 @@ def forward(self, x): merge2_out = self.merge2(fc1_iaf_out, fc2_iaf_out) fc3_out = self.fc3(merge2_out) fc3_iaf_out = self.fc3_iaf(fc3_out) + # -- fc clock 7 -- + merge3_out = self.merge3(fc2_iaf_out, fc3_iaf_out) + fc4_out = self.fc4(merge3_out) + fc4_iaf_out = self.fc4_iaf(fc4_out) + + return fc4_iaf_out + +class EXAMPLE_6(nn.Module): + """ This is the 'two networks with merging outputs' example in https://github.com/synsense/sinabs/issues/181 . """ + def __init__(self) -> None: + super().__init__() + + self.conv_A = nn.Conv2d(2, 4, 2, 1, bias=False) + self.iaf_A = IAFSqueeze(batch_size=1) + + self.conv_B = nn.Conv2d(4, 4, 2, 1, bias=False) + self.iaf_B = IAFSqueeze(batch_size=1) + self.pool_B = SumPool2d(2,2) + + self.conv_C = nn.Conv2d(4, 4, 2, 1, bias=False) + self.iaf_C = IAFSqueeze(batch_size=1) + self.pool_C = SumPool2d(2,2) + + self.conv_D = nn.Conv2d(2, 4, 2, 1, bias=False) + self.iaf_D = IAFSqueeze(batch_size=1) + + self.conv_E = nn.Conv2d(4, 4, 2, 1, bias=False) + self.iaf_E = IAFSqueeze(batch_size=1) + self.pool_E = SumPool2d(2,2) + + self.conv_F = nn.Conv2d(4, 4, 2, 1, bias=False) + self.iaf_F = IAFSqueeze(batch_size=1) + self.pool_F = SumPool2d(2,2) + + self.flat_brach1 = nn.Flatten() + self.flat_brach2 = nn.Flatten() + self.merge = Merge() + + self.fc1 = nn.Linear(196, 200, bias=False) + self.iaf1_fc = IAFSqueeze(batch_size=1) + + self.fc2 = nn.Linear(200, 200, bias=False) + self.iaf2_fc = IAFSqueeze(batch_size=1) + + self.fc3 = nn.Linear(200, 10, bias=False) + self.iaf3_fc = IAFSqueeze(batch_size=1) + + def forward(self, x): + # conv 1 - A + conv_A_out = self.conv_A(x) + iaf_A_out = self.iaf_A(conv_A_out) + # conv 2 - B + conv_B_out = self.conv_B(iaf_A_out) + iaf_B_out = self.iaf_B(conv_B_out) + pool_B_out = self.pool_B(iaf_B_out) + # conv 3 - C + conv_C_out = self.conv_C(pool_B_out) + iaf_C_out = self.iaf_C(conv_C_out) + pool_C_out = self.pool_C(iaf_C_out) + + # --- + + # conv 4 - D + conv_D_out = self.conv_D(x) + iaf_D_out = self.iaf_D(conv_D_out) + # conv 5 - E + conv_E_out = self.conv_E(iaf_D_out) + iaf_E_out = self.iaf_E(conv_E_out) + pool_E_out = self.pool_E(iaf_E_out) + # conv 6 - F + conv_F_out = self.conv_F(pool_E_out) + iaf_F_out = self.iaf_F(conv_F_out) + pool_F_out = self.pool_F(iaf_F_out) + + # --- + + flat_brach1_out = self.flat_brach1(pool_C_out) + flat_brach2_out = self.flat_brach2(pool_F_out) + merge_out = self.merge(flat_brach1_out, flat_brach2_out) + + # FC 7 - G + fc1_out = self.fc1(merge_out) + iaf1_fc_out = self.iaf1_fc(fc1_out) + # FC 8 - H + fc2_out = self.fc2(iaf1_fc_out) + iaf2_fc_out = self.iaf2_fc(fc2_out) + # FC 9 - I + fc3_out = self.fc3(iaf2_fc_out) + iaf3_fc_out = self.iaf3_fc(fc3_out) + + return iaf3_fc_out + +class EXAMPLE_7(nn.Module): + """ This is the 'a network with a merge and a split' example in https://github.com/synsense/sinabs/issues/181 . """ + def __init__(self) -> None: + super().__init__() + + self.conv1 = nn.Conv2d(2, 4, 2, 1, bias=False) + self.iaf1 = IAFSqueeze(batch_size=1) + + self.conv2 = nn.Conv2d(4, 4, 2, 1, bias=False) + self.iaf2 = IAFSqueeze(batch_size=1) + self.pool2 = SumPool2d(2,2) + self.pool2a = SumPool2d(5,5) + + self.conv3 = nn.Conv2d(4, 4, 2, 1, bias=False) + self.iaf3 = IAFSqueeze(batch_size=1) + self.pool3 = SumPool2d(2,2) + + self.conv4 = nn.Conv2d(4, 4, 2, 1, bias=False) + self.iaf4 = IAFSqueeze(batch_size=1) + + self.flat = nn.Flatten() + self.flat_a = nn.Flatten() + + self.fc1 = nn.Linear(144, 144, bias=False) + self.iaf1_fc = IAFSqueeze(batch_size=1) + + self.fc2 = nn.Linear(144, 144, bias=False) + self.iaf2_fc = IAFSqueeze(batch_size=1) + + self.fc3 = nn.Linear(144, 10, bias=False) + self.iaf3_fc = IAFSqueeze(batch_size=1) + + # -- merges -- + self.merge1 = Merge() + + def forward(self, x): + # conv 1 - A + con1_out = self.conv1(x) + iaf1_out = self.iaf1(con1_out) + + # conv 2 - B + conv2_out = self.conv2(iaf1_out) + iaf2_out = self.iaf2(conv2_out) + pool2_out = self.pool2(iaf2_out) + pool2a_out = self.pool2a(iaf2_out) + + # conv 3 - C + conv3_out = self.conv3(pool2_out) + iaf3_out = self.iaf3(conv3_out) + pool3_out = self.pool3(iaf3_out) + + # conv 4 - D + conv4_out = self.conv4(pool3_out) + iaf4_out = self.iaf4(conv4_out) + flat_out = self.flat(iaf4_out) + + # fc 1 - E + flat_a_out = self.flat_a(pool2a_out) + fc1_out = self.fc1(flat_a_out) + iaf1_fc_out = self.iaf1_fc(fc1_out) + + # fc 2 - F + fc2_out = self.fc2(iaf1_fc_out) + iaf2_fc_out = self.iaf2_fc(fc2_out) + + # fc 2 - G + merge1_out = self.merge1(flat_out, iaf2_fc_out) + fc3_out = self.fc3(merge1_out) + iaf3_fc_out = self.iaf3_fc(fc3_out) + + return iaf3_fc_out + +class EXAMPLE_8(nn.Module): + """ This is the 'a complex network structure' example in https://github.com/synsense/sinabs/issues/181 . """ + def __init__(self) -> None: + super().__init__() + + self.conv1 = nn.Conv2d(2, 8, 2, 1, bias=False) + self.iaf1 = IAFSqueeze(batch_size=1) + + self.conv2 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf2 = IAFSqueeze(batch_size=1) + self.pool2 = SumPool2d(2,2) + + self.conv3 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf3 = IAFSqueeze(batch_size=1) + self.pool3 = SumPool2d(2,2) + self.pool3a = SumPool2d(6,6) + + self.conv4 = nn.Conv2d(8, 8, 2, 1, bias=False) + self.iaf4 = IAFSqueeze(batch_size=1) + self.pool4 = SumPool2d(3,3) + + self.flat = nn.Flatten() + self.flat_a = nn.Flatten() + + self.fc1 = nn.Linear(200, 200, bias=False) + self.iaf1_fc = IAFSqueeze(batch_size=1) + + self.fc2 = nn.Linear(200, 10, bias=False) + self.iaf2_fc = IAFSqueeze(batch_size=1) + + # -- merges -- + self.merge1 = Merge() + self.merge2 = Merge() + + def forward(self, x): + # conv 1 - A + con1_out = self.conv1(x) + iaf1_out = self.iaf1(con1_out) + + # conv 2 - B + conv2_out = self.conv2(iaf1_out) + iaf2_out = self.iaf2(conv2_out) + pool2_out = self.pool2(iaf2_out) + + # conv 3 - C + conv3_out = self.conv3(iaf1_out) + iaf3_out = self.iaf3(conv3_out) + pool3_out = self.pool3(iaf3_out) + pool3a_out = self.pool3a(iaf3_out) + + # conv 4 - D + merge1_out = self.merge1(pool2_out, pool3_out) + conv4_out = self.conv4(merge1_out) + iaf4_out = self.iaf4(conv4_out) + pool4_out = self.pool4(iaf4_out) + flat_out = self.flat(pool4_out) + + # fc 1 - E + flat_a_out = self.flat_a(pool3a_out) + fc1_out = self.fc1(flat_a_out) + iaf1_fc_out = self.iaf1_fc(fc1_out) + + # fc 2 - F + merge2_out = self.merge2(iaf1_fc_out, flat_out) + fc2_out = self.fc2(merge2_out) + iaf2_fc_out = self.iaf2_fc(fc2_out) - return fc3_iaf_out \ No newline at end of file + return iaf2_fc_out \ No newline at end of file diff --git a/tests/test_dynapcnnnetwork/conftest_dynapcnnnetwork.py b/tests/test_dynapcnnnetwork/conftest_dynapcnnnetwork.py index 300f7455..01087fad 100644 --- a/tests/test_dynapcnnnetwork/conftest_dynapcnnnetwork.py +++ b/tests/test_dynapcnnnetwork/conftest_dynapcnnnetwork.py @@ -59,7 +59,7 @@ (13, 15), ] -edges_list_5 = [ +edges_list_4 = [ (0, 1), (1, 2), (1, 3), @@ -77,16 +77,12 @@ (13, 15), (14, 16), (16, 15), - (16, 17), - (18, 19), - (19, 17), - (20, 21), + (17, 18), (5, 7), - (15, 18), - (17, 20), + (15, 17), ] -edges_list_4 = [ +edges_list_5 = [ (0, 1), (1, 2), (1, 3), @@ -104,9 +100,87 @@ (13, 15), (14, 16), (16, 15), + (16, 17), + (18, 19), + (19, 17), + (20, 21), + (5, 7), + (15, 18), + (17, 20), +] + +edges_list_6 = [ + (0, 1), + (1, 2), + (2, 3), + (3, 4), + (4, 5), + (5, 6), + (6, 7), + (7, 8), + (9, 10), + (10, 11), + (11, 12), + (12, 13), + (13, 14), + (14, 15), + (15, 16), + (16, 17), + (8, 18), (17, 18), + (18, 19), + (19, 20), + (20, 21), + (21, 22), + (22, 23), + (23, 24), +] + +edges_list_7 = [ + (0, 1), + (1, 2), + (2, 3), + (3, 4), + (3, 5), + (4, 6), (5, 7), - (15, 17), + (6, 8), + (8, 9), + (9, 10), + (10, 11), + (11, 12), + (12, 13), + (7, 14), + (14, 15), + (15, 16), + (16, 17), + (17, 13), + (18, 19), + (13, 18), +] + +edges_list_8 = [ + (0, 1), + (1, 2), + (1, 3), + (2, 4), + (4, 5), + (5, 6), + (3, 7), + (7, 8), + (7, 9), + (8, 6), + (9, 10), + (11, 12), + (12, 13), + (13, 14), + (14, 15), + (10, 16), + (16, 17), + (17, 15), + (18, 19), + (6, 11), + (15, 18), ] args_NIRtoDynapcnnNetwork_edges_list = [ @@ -115,6 +189,9 @@ (EXAMPLE_3(), edges_list_3), (EXAMPLE_4(), edges_list_4), (EXAMPLE_5(), edges_list_5), + (EXAMPLE_6(), edges_list_6), + (EXAMPLE_7(), edges_list_7), + (EXAMPLE_8(), edges_list_8), ] # --- test_NIRtoDynapcnnNetwork_IO(snn, io_dict) --- @@ -208,12 +285,79 @@ 21: {'in': torch.Size([1, 10]), 'out': torch.Size([1, 10])}, } +nodes_IO_6 = { + 0: {'in': torch.Size([1, 2, 34, 34]), 'out': torch.Size([1, 4, 33, 33])}, + 1: {'in': torch.Size([1, 4, 33, 33]), 'out': torch.Size([1, 4, 33, 33])}, + 2: {'in': torch.Size([1, 4, 33, 33]), 'out': torch.Size([1, 4, 32, 32])}, + 3: {'in': torch.Size([1, 4, 32, 32]), 'out': torch.Size([1, 4, 32, 32])}, + 4: {'in': torch.Size([1, 4, 32, 32]), 'out': torch.Size([1, 4, 16, 16])}, + 5: {'in': torch.Size([1, 4, 16, 16]), 'out': torch.Size([1, 4, 15, 15])}, + 6: {'in': torch.Size([1, 4, 15, 15]), 'out': torch.Size([1, 4, 15, 15])}, + 7: {'in': torch.Size([1, 4, 15, 15]), 'out': torch.Size([1, 4, 7, 7])}, + 17: {'in': torch.Size([1, 196]), 'out': torch.Size([1, 200])}, + 18: {'in': torch.Size([1, 200]), 'out': torch.Size([1, 200])}, + 8: {'in': torch.Size([1, 2, 34, 34]), 'out': torch.Size([1, 4, 33, 33])}, + 9: {'in': torch.Size([1, 4, 33, 33]), 'out': torch.Size([1, 4, 33, 33])}, + 10: {'in': torch.Size([1, 4, 33, 33]), 'out': torch.Size([1, 4, 32, 32])}, + 11: {'in': torch.Size([1, 4, 32, 32]), 'out': torch.Size([1, 4, 32, 32])}, + 12: {'in': torch.Size([1, 4, 32, 32]), 'out': torch.Size([1, 4, 16, 16])}, + 13: {'in': torch.Size([1, 4, 16, 16]), 'out': torch.Size([1, 4, 15, 15])}, + 14: {'in': torch.Size([1, 4, 15, 15]), 'out': torch.Size([1, 4, 15, 15])}, + 15: {'in': torch.Size([1, 4, 15, 15]), 'out': torch.Size([1, 4, 7, 7])}, + 19: {'in': torch.Size([1, 200]), 'out': torch.Size([1, 200])}, + 20: {'in': torch.Size([1, 200]), 'out': torch.Size([1, 200])}, + 21: {'in': torch.Size([1, 200]), 'out': torch.Size([1, 11])}, + 22: {'in': torch.Size([1, 11]), 'out': torch.Size([1, 11])}, +} + +nodes_IO_7 = { + 0: {'in': torch.Size([1, 2, 34, 34]), 'out': torch.Size([1, 4, 33, 33])}, + 1: {'in': torch.Size([1, 4, 33, 33]), 'out': torch.Size([1, 4, 33, 33])}, + 2: {'in': torch.Size([1, 4, 33, 33]), 'out': torch.Size([1, 4, 32, 32])}, + 3: {'in': torch.Size([1, 4, 32, 32]), 'out': torch.Size([1, 4, 32, 32])}, + 4: {'in': torch.Size([1, 4, 32, 32]), 'out': torch.Size([1, 4, 16, 16])}, + 5: {'in': torch.Size([1, 4, 32, 32]), 'out': torch.Size([1, 4, 6, 6])}, + 6: {'in': torch.Size([1, 4, 16, 16]), 'out': torch.Size([1, 4, 15, 15])}, + 7: {'in': torch.Size([1, 4, 15, 15]), 'out': torch.Size([1, 4, 15, 15])}, + 8: {'in': torch.Size([1, 4, 15, 15]), 'out': torch.Size([1, 4, 7, 7])}, + 12: {'in': torch.Size([1, 144]), 'out': torch.Size([1, 144])}, + 13: {'in': torch.Size([1, 144]), 'out': torch.Size([1, 144])}, + 9: {'in': torch.Size([1, 4, 7, 7]), 'out': torch.Size([1, 4, 6, 6])}, + 10: {'in': torch.Size([1, 4, 6, 6]), 'out': torch.Size([1, 4, 6, 6])}, + 16: {'in': torch.Size([1, 144]), 'out': torch.Size([1, 11])}, + 17: {'in': torch.Size([1, 11]), 'out': torch.Size([1, 11])}, + 14: {'in': torch.Size([1, 144]), 'out': torch.Size([1, 144])}, + 15: {'in': torch.Size([1, 144]), 'out': torch.Size([1, 144])}, +} + +nodes_IO_8 = { + 0: {'in': torch.Size([1, 2, 34, 34]), 'out': torch.Size([1, 8, 33, 33])}, + 1: {'in': torch.Size([1, 8, 33, 33]), 'out': torch.Size([1, 8, 33, 33])}, + 2: {'in': torch.Size([1, 8, 33, 33]), 'out': torch.Size([1, 8, 32, 32])}, + 4: {'in': torch.Size([1, 8, 32, 32]), 'out': torch.Size([1, 8, 32, 32])}, + 5: {'in': torch.Size([1, 8, 32, 32]), 'out': torch.Size([1, 8, 16, 16])}, + 3: {'in': torch.Size([1, 8, 33, 33]), 'out': torch.Size([1, 8, 32, 32])}, + 7: {'in': torch.Size([1, 8, 32, 32]), 'out': torch.Size([1, 8, 32, 32])}, + 8: {'in': torch.Size([1, 8, 32, 32]), 'out': torch.Size([1, 8, 16, 16])}, + 9: {'in': torch.Size([1, 8, 32, 32]), 'out': torch.Size([1, 8, 5, 5])}, + 10: {'in': torch.Size([1, 8, 16, 16]), 'out': torch.Size([1, 8, 15, 15])}, + 11: {'in': torch.Size([1, 8, 15, 15]), 'out': torch.Size([1, 8, 15, 15])}, + 12: {'in': torch.Size([1, 8, 15, 15]), 'out': torch.Size([1, 8, 5, 5])}, + 14: {'in': torch.Size([1, 200]), 'out': torch.Size([1, 200])}, + 15: {'in': torch.Size([1, 200]), 'out': torch.Size([1, 200])}, + 16: {'in': torch.Size([1, 200]), 'out': torch.Size([1, 11])}, + 17: {'in': torch.Size([1, 11]), 'out': torch.Size([1, 11])}, +} + args_test_NIRtoDynapcnnNetwork_IO = [ (EXAMPLE_1(), nodes_IO_1), (EXAMPLE_2(), nodes_IO_2), (EXAMPLE_3(), nodes_IO_3), (EXAMPLE_4(), nodes_IO_4), (EXAMPLE_5(), nodes_IO_5), + (EXAMPLE_6(), nodes_IO_6), + (EXAMPLE_7(), nodes_IO_7), + (EXAMPLE_8(), nodes_IO_8), ] # --- test_DynapcnnLyers_edges_list(snn, edges_list) --- @@ -269,12 +413,46 @@ (6, 7), ] +dcnnl_edges_list_6 = [ + (0, 1), + (1, 2), + (2, 3), + (3, 7), + (4, 5), + (5, 6), + (6, 3), + (7, 8), +] + +dcnnl_edges_list_7 = [ + (0, 1), + (1, 2), + (1, 3), + (2, 4), + (3, 6), + (4, 5), + (6, 5), +] + +dcnnl_edges_list_8 = [ + (0, 1), + (0, 2), + (1, 3), + (2, 3), + (2, 4), + (3, 5), + (4, 5), +] + args_DynapcnnLyers_edges_list = [ (EXAMPLE_1(), dcnnl_edges_list_1), (EXAMPLE_2(), dcnnl_edges_list_2), (EXAMPLE_3(), dcnnl_edges_list_3), (EXAMPLE_4(), dcnnl_edges_list_4), (EXAMPLE_5(), dcnnl_edges_list_5), + (EXAMPLE_6(), dcnnl_edges_list_6), + (EXAMPLE_7(), dcnnl_edges_list_7), + (EXAMPLE_8(), dcnnl_edges_list_8), ] # --- test_DynapcnnNetwork_forward_edges(snn, forward_edges_list) --- @@ -340,10 +518,46 @@ ('merge_2', 7), ] +forward_edges_list_6 = [ + (0, 1), + (1, 2), + ((2, 6), 'merge_0'), + ('merge_0', 3), + (3, 7), + (4, 5), + (5, 6), + (7, 8), +] + +forward_edges_list_7 = [ + (1, '1_pool0'), + (1, '1_pool1'), + ('1_pool0', 2), + ('1_pool1', 3), + (2, 4), + (3, 6), + ((4, 6), 'merge_0'), + ('merge_0', 5), +] + +forward_edges_list_8 = [ + (0, 1), + (2, '2_pool0'), + (2, '2_pool1'), + (('2_pool0', 1), 'merge_0'), + ('merge_0', 3), + ('2_pool1', 4), + ((3, 4), 'merge_1'), + ('merge_1', 5), +] + args_DynapcnnNetwork_forward_edges = [ (EXAMPLE_1(), forward_edges_list_1), (EXAMPLE_2(), forward_edges_list_2), (EXAMPLE_3(), forward_edges_list_3), (EXAMPLE_4(), forward_edges_list_4), (EXAMPLE_5(), forward_edges_list_5), + (EXAMPLE_6(), forward_edges_list_6), + (EXAMPLE_7(), forward_edges_list_7), + (EXAMPLE_8(), forward_edges_list_8), ] \ No newline at end of file diff --git a/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/complex_network_structure.ipynb b/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/complex_network_structure.ipynb index 560fa2fe..4494924e 100644 --- a/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/complex_network_structure.ipynb +++ b/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/complex_network_structure.ipynb @@ -26,7 +26,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 2, @@ -82,49 +82,39 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 4, "metadata": { "metadata": {} }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "torch.Size([8, 10, 16, 16]) torch.Size([8, 10, 16, 16])\n", - "torch.Size([8, 250])\n", - "torch.Size([8, 250]) torch.Size([8, 250])\n" - ] - } - ], + "outputs": [], "source": [ "class SNN(nn.Module):\n", " def __init__(self, nb_classes, batch_size, surrogate_fn, min_v_mem=-0.313, spk_thr=2.0) -> None:\n", " super().__init__()\n", "\n", - " self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False)\n", + " self.conv1 = nn.Conv2d(2, 8, 2, 1, bias=False)\n", " self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", "\n", - " self.conv2 = nn.Conv2d(10, 10, 2, 1, bias=False)\n", + " self.conv2 = nn.Conv2d(8, 8, 2, 1, bias=False)\n", " self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", " self.pool2 = sl.SumPool2d(2,2)\n", "\n", - " self.conv3 = nn.Conv2d(10, 10, 2, 1, bias=False)\n", + " self.conv3 = nn.Conv2d(8, 8, 2, 1, bias=False)\n", " self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", " self.pool3 = sl.SumPool2d(2,2)\n", " self.pool3a = sl.SumPool2d(6,6)\n", "\n", - " self.conv4 = nn.Conv2d(10, 10, 2, 1, bias=False)\n", + " self.conv4 = nn.Conv2d(8, 8, 2, 1, bias=False)\n", " self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", " self.pool4 = sl.SumPool2d(3,3)\n", "\n", " self.flat = nn.Flatten()\n", " self.flat_a = nn.Flatten()\n", "\n", - " self.fc1 = nn.Linear(250, 250, bias=False)\n", + " self.fc1 = nn.Linear(200, 200, bias=False)\n", " self.iaf1_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", "\n", - " self.fc2 = nn.Linear(250, nb_classes, bias=False)\n", + " self.fc2 = nn.Linear(200, nb_classes, bias=False)\n", " self.iaf2_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", "\n", " # -- merges --\n", @@ -148,7 +138,6 @@ " pool3a_out = self.pool3a(iaf3_out)\n", "\n", " # conv 4 - D\n", - " print(pool2_out.shape, pool3_out.shape)\n", " merge1_out = self.merge1(pool2_out, pool3_out)\n", " conv4_out = self.conv4(merge1_out)\n", " iaf4_out = self.iaf4(conv4_out)\n", @@ -157,23 +146,17 @@ " \n", " # fc 1 - E\n", " flat_a_out = self.flat_a(pool3a_out)\n", - " print(flat_a_out.shape)\n", " fc1_out = self.fc1(flat_a_out)\n", " iaf1_fc_out = self.iaf1_fc(fc1_out)\n", "\n", " # fc 2 - F\n", - " print(iaf1_fc_out.shape, flat_out.shape)\n", " merge2_out = self.merge2(iaf1_fc_out, flat_out)\n", " fc2_out = self.fc2(merge2_out)\n", " iaf2_fc_out = self.iaf2_fc(fc2_out)\n", "\n", " return iaf2_fc_out\n", " \n", - "snn = SNN(11, 8, PeriodicExponential())\n", - "\n", - "x = torch.randn((8, *input_shape))\n", - "\n", - "out = snn(x)" + "snn = SNN(11, 1, PeriodicExponential())" ] }, { @@ -191,7 +174,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 5, "metadata": { "metadata": {} }, @@ -215,7 +198,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 6, "metadata": { "metadata": {} }, @@ -228,14 +211,17 @@ ] }, { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" + "ename": "RuntimeError", + "evalue": "Device is already opened!", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[6], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m hw_model\u001b[38;5;241m.\u001b[39mto(device\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mspeck2fmodule:0\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", + "File \u001b[0;32m~/Documents/github/sinabs/sinabs/backend/dynapcnn/dynapcnn_network_graph.py:172\u001b[0m, in \u001b[0;36mDynapcnnNetworkGraph.to\u001b[0;34m(self, device, chip_layers_ordering, monitor_layers, config_modifier, slow_clk_frequency)\u001b[0m\n\u001b[1;32m 163\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m device_name \u001b[38;5;129;01min\u001b[39;00m ChipFactory\u001b[38;5;241m.\u001b[39msupported_devices: \u001b[38;5;66;03m# pragma: no cover\u001b[39;00m\n\u001b[1;32m 165\u001b[0m config \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmake_config( \u001b[38;5;66;03m# generate config.\u001b[39;00m\n\u001b[1;32m 166\u001b[0m chip_layers_ordering\u001b[38;5;241m=\u001b[39mchip_layers_ordering,\n\u001b[1;32m 167\u001b[0m device\u001b[38;5;241m=\u001b[39mdevice,\n\u001b[1;32m 168\u001b[0m monitor_layers\u001b[38;5;241m=\u001b[39mmonitor_layers,\n\u001b[1;32m 169\u001b[0m config_modifier\u001b[38;5;241m=\u001b[39mconfig_modifier,\n\u001b[1;32m 170\u001b[0m )\n\u001b[0;32m--> 172\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msamna_device \u001b[38;5;241m=\u001b[39m open_device(device) \u001b[38;5;66;03m# apply configuration to device.\u001b[39;00m\n\u001b[1;32m 173\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msamna_device\u001b[38;5;241m.\u001b[39mget_model()\u001b[38;5;241m.\u001b[39mapply_configuration(config)\n\u001b[1;32m 174\u001b[0m time\u001b[38;5;241m.\u001b[39msleep(\u001b[38;5;241m1\u001b[39m)\n", + "File \u001b[0;32m~/Documents/github/sinabs/sinabs/backend/dynapcnn/io.py:256\u001b[0m, in \u001b[0;36mopen_device\u001b[0;34m(device_id)\u001b[0m\n\u001b[1;32m 254\u001b[0m device_map \u001b[38;5;241m=\u001b[39m get_device_map()\n\u001b[1;32m 255\u001b[0m device_info \u001b[38;5;241m=\u001b[39m device_map[device_id]\n\u001b[0;32m--> 256\u001b[0m device_handle \u001b[38;5;241m=\u001b[39m samna\u001b[38;5;241m.\u001b[39mdevice\u001b[38;5;241m.\u001b[39mopen_device(device_info)\n\u001b[1;32m 258\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m device_handle \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 259\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m device_handle\n", + "\u001b[0;31mRuntimeError\u001b[0m: Device is already opened!" + ] } ], "source": [ @@ -251,7 +237,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": { "metadata": {} }, @@ -262,77 +248,61 @@ "text": [ "---- DynapcnnLayer 0 ----------------------------------------------------------\n", "> layer modules: \n", - "(node 0): Conv2d(2, 10, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + "(node 0): Conv2d(2, 8, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", "(node 1): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "(node 2): SumPool2d(norm_type=1, kernel_size=3, stride=3, ceil_mode=False)\n", - "(node 3): SumPool2d(norm_type=1, kernel_size=4, stride=4, ceil_mode=False)\n", + "tensor(723.), min_v_mem=Parameter containing:\n", + "tensor(-113.), batch_size=1, num_timesteps=-1)\n", "> layer destinations: [1, 2]\n", "> assigned core: 0\n", "\n", "---- DynapcnnLayer 1 ----------------------------------------------------------\n", "> layer modules: \n", - "(node 4): Conv2d(10, 10, kernel_size=(4, 4), stride=(1, 1), bias=False)\n", - "(node 6): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [2]\n", + "(node 2): Conv2d(8, 8, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + "(node 4): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1452.), min_v_mem=Parameter containing:\n", + "tensor(-227.), batch_size=1, num_timesteps=-1)\n", + "(node 5): SumPool2d(norm_type=1, kernel_size=2, stride=2, ceil_mode=False)\n", + "> layer destinations: [3]\n", "> assigned core: 1\n", "\n", "---- DynapcnnLayer 2 ----------------------------------------------------------\n", "> layer modules: \n", - "(node 7): Conv2d(10, 10, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", - "(node 8): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [3]\n", + "(node 3): Conv2d(8, 8, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + "(node 7): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1437.), min_v_mem=Parameter containing:\n", + "tensor(-225.), batch_size=1, num_timesteps=-1)\n", + "(node 8): SumPool2d(norm_type=1, kernel_size=2, stride=2, ceil_mode=False)\n", + "(node 9): SumPool2d(norm_type=1, kernel_size=6, stride=6, ceil_mode=False)\n", + "> layer destinations: [3, 4]\n", "> assigned core: 2\n", "\n", "---- DynapcnnLayer 3 ----------------------------------------------------------\n", "> layer modules: \n", - "(node 9): Conv2d(10, 1, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", - "(node 10): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [4]\n", + "(node 10): Conv2d(8, 8, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + "(node 11): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1439.), min_v_mem=Parameter containing:\n", + "tensor(-225.), batch_size=1, num_timesteps=-1)\n", + "(node 12): SumPool2d(norm_type=1, kernel_size=3, stride=3, ceil_mode=False)\n", + "> layer destinations: [5]\n", "> assigned core: 3\n", "\n", "---- DynapcnnLayer 4 ----------------------------------------------------------\n", "> layer modules: \n", - "(node 11): Conv2d(1, 100, kernel_size=(6, 6), stride=(1, 1), bias=False)\n", - "(node 12): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [5, 6]\n", - "> assigned core: 4\n", - "\n", - "---- DynapcnnLayer 5 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 13): Conv2d(100, 100, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + "(node 14): Conv2d(8, 200, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", "(node 15): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [6, 7]\n", + "tensor(3592.), min_v_mem=Parameter containing:\n", + "tensor(-562.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [5]\n", "> assigned core: 5\n", "\n", - "---- DynapcnnLayer 6 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 17): Conv2d(100, 100, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - "(node 18): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [7]\n", - "> assigned core: 6\n", - "\n", - "---- DynapcnnLayer 7 ----------------------------------------------------------\n", + "---- DynapcnnLayer 5 ----------------------------------------------------------\n", "> layer modules: \n", - "(node 19): Conv2d(100, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - "(node 20): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "(node 16): Conv2d(200, 11, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + "(node 17): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(3592.), min_v_mem=Parameter containing:\n", + "tensor(-562.), batch_size=1, num_timesteps=-1)\n", "> layer destinations: []\n", - "> assigned core: 7\n", + "> assigned core: 4\n", "\n", "\n" ] diff --git a/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/split_and_merge.ipynb b/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/split_and_merge.ipynb index e0af08dc..4ce4ffc8 100644 --- a/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/split_and_merge.ipynb +++ b/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/split_and_merge.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "metadata": { "metadata": {} }, @@ -18,7 +18,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "metadata": { "metadata": {} }, @@ -26,10 +26,10 @@ { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 3, + "execution_count": 2, "metadata": {}, "output_type": "execute_result" } @@ -40,7 +40,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "metadata": { "metadata": {} }, @@ -82,50 +82,41 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 4, "metadata": { "metadata": {} }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "torch.Size([8, 360])\n", - "torch.Size([8, 360]) torch.Size([8, 360])\n" - ] - } - ], + "outputs": [], "source": [ "class SNN(nn.Module):\n", " def __init__(self, nb_classes, batch_size, surrogate_fn, min_v_mem=-0.313, spk_thr=2.0) -> None:\n", " super().__init__()\n", "\n", - " self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False)\n", + " self.conv1 = nn.Conv2d(2, 4, 2, 1, bias=False)\n", " self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", "\n", - " self.conv2 = nn.Conv2d(10, 10, 2, 1, bias=False)\n", + " self.conv2 = nn.Conv2d(4, 4, 2, 1, bias=False)\n", " self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", " self.pool2 = sl.SumPool2d(2,2)\n", " self.pool2a = sl.SumPool2d(5,5)\n", "\n", - " self.conv3 = nn.Conv2d(10, 10, 2, 1, bias=False)\n", + " self.conv3 = nn.Conv2d(4, 4, 2, 1, bias=False)\n", " self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", " self.pool3 = sl.SumPool2d(2,2)\n", "\n", - " self.conv4 = nn.Conv2d(10, 10, 2, 1, bias=False)\n", + " self.conv4 = nn.Conv2d(4, 4, 2, 1, bias=False)\n", " self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", "\n", " self.flat = nn.Flatten()\n", " self.flat_a = nn.Flatten()\n", "\n", - " self.fc1 = nn.Linear(360, 360, bias=False)\n", + " self.fc1 = nn.Linear(144, 144, bias=False)\n", " self.iaf1_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", "\n", - " self.fc2 = nn.Linear(360, 360, bias=False)\n", + " self.fc2 = nn.Linear(144, 144, bias=False)\n", " self.iaf2_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", "\n", - " self.fc3 = nn.Linear(360, nb_classes, bias=False)\n", + " self.fc3 = nn.Linear(144, nb_classes, bias=False)\n", " self.iaf3_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", "\n", " # -- merges --\n", @@ -170,11 +161,7 @@ "\n", " return iaf3_fc_out\n", " \n", - "snn = SNN(11, 8, PeriodicExponential())\n", - "\n", - "x = torch.randn((8, *input_shape))\n", - "\n", - "out = snn(x)" + "snn = SNN(11, 1, PeriodicExponential())" ] }, { @@ -192,11 +179,20 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 5, "metadata": { "metadata": {} }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "torch.Size([1, 144])\n", + "torch.Size([1, 144]) torch.Size([1, 144])\n" + ] + } + ], "source": [ "hw_model = DynapcnnNetworkGraph(\n", " snn,\n", @@ -216,7 +212,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 6, "metadata": { "metadata": {} }, @@ -229,14 +225,17 @@ ] }, { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" + "ename": "RuntimeError", + "evalue": "Device is already opened!", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[6], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m hw_model\u001b[38;5;241m.\u001b[39mto(device\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mspeck2fmodule:0\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", + "File \u001b[0;32m~/Documents/github/sinabs/sinabs/backend/dynapcnn/dynapcnn_network_graph.py:172\u001b[0m, in \u001b[0;36mDynapcnnNetworkGraph.to\u001b[0;34m(self, device, chip_layers_ordering, monitor_layers, config_modifier, slow_clk_frequency)\u001b[0m\n\u001b[1;32m 163\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m device_name \u001b[38;5;129;01min\u001b[39;00m ChipFactory\u001b[38;5;241m.\u001b[39msupported_devices: \u001b[38;5;66;03m# pragma: no cover\u001b[39;00m\n\u001b[1;32m 165\u001b[0m config \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmake_config( \u001b[38;5;66;03m# generate config.\u001b[39;00m\n\u001b[1;32m 166\u001b[0m chip_layers_ordering\u001b[38;5;241m=\u001b[39mchip_layers_ordering,\n\u001b[1;32m 167\u001b[0m device\u001b[38;5;241m=\u001b[39mdevice,\n\u001b[1;32m 168\u001b[0m monitor_layers\u001b[38;5;241m=\u001b[39mmonitor_layers,\n\u001b[1;32m 169\u001b[0m config_modifier\u001b[38;5;241m=\u001b[39mconfig_modifier,\n\u001b[1;32m 170\u001b[0m )\n\u001b[0;32m--> 172\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msamna_device \u001b[38;5;241m=\u001b[39m open_device(device) \u001b[38;5;66;03m# apply configuration to device.\u001b[39;00m\n\u001b[1;32m 173\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msamna_device\u001b[38;5;241m.\u001b[39mget_model()\u001b[38;5;241m.\u001b[39mapply_configuration(config)\n\u001b[1;32m 174\u001b[0m time\u001b[38;5;241m.\u001b[39msleep(\u001b[38;5;241m1\u001b[39m)\n", + "File \u001b[0;32m~/Documents/github/sinabs/sinabs/backend/dynapcnn/io.py:256\u001b[0m, in \u001b[0;36mopen_device\u001b[0;34m(device_id)\u001b[0m\n\u001b[1;32m 254\u001b[0m device_map \u001b[38;5;241m=\u001b[39m get_device_map()\n\u001b[1;32m 255\u001b[0m device_info \u001b[38;5;241m=\u001b[39m device_map[device_id]\n\u001b[0;32m--> 256\u001b[0m device_handle \u001b[38;5;241m=\u001b[39m samna\u001b[38;5;241m.\u001b[39mdevice\u001b[38;5;241m.\u001b[39mopen_device(device_info)\n\u001b[1;32m 258\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m device_handle \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 259\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m device_handle\n", + "\u001b[0;31mRuntimeError\u001b[0m: Device is already opened!" + ] } ], "source": [ @@ -252,7 +251,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": { "metadata": {} }, @@ -263,78 +262,70 @@ "text": [ "---- DynapcnnLayer 0 ----------------------------------------------------------\n", "> layer modules: \n", - "(node 0): Conv2d(2, 10, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + "(node 0): Conv2d(2, 4, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", "(node 1): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "(node 2): SumPool2d(norm_type=1, kernel_size=3, stride=3, ceil_mode=False)\n", - "(node 3): SumPool2d(norm_type=1, kernel_size=4, stride=4, ceil_mode=False)\n", - "> layer destinations: [1, 2]\n", + "tensor(758.), min_v_mem=Parameter containing:\n", + "tensor(-119.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [1]\n", "> assigned core: 0\n", "\n", "---- DynapcnnLayer 1 ----------------------------------------------------------\n", "> layer modules: \n", - "(node 4): Conv2d(10, 10, kernel_size=(4, 4), stride=(1, 1), bias=False)\n", - "(node 6): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [2]\n", + "(node 2): Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + "(node 3): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1022.), min_v_mem=Parameter containing:\n", + "tensor(-160.), batch_size=1, num_timesteps=-1)\n", + "(node 4): SumPool2d(norm_type=1, kernel_size=2, stride=2, ceil_mode=False)\n", + "(node 5): SumPool2d(norm_type=1, kernel_size=5, stride=5, ceil_mode=False)\n", + "> layer destinations: [2, 3]\n", "> assigned core: 1\n", "\n", "---- DynapcnnLayer 2 ----------------------------------------------------------\n", "> layer modules: \n", - "(node 7): Conv2d(10, 10, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", - "(node 8): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [3]\n", + "(node 6): Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + "(node 7): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1062.), min_v_mem=Parameter containing:\n", + "tensor(-166.), batch_size=1, num_timesteps=-1)\n", + "(node 8): SumPool2d(norm_type=1, kernel_size=2, stride=2, ceil_mode=False)\n", + "> layer destinations: [4]\n", "> assigned core: 2\n", "\n", "---- DynapcnnLayer 3 ----------------------------------------------------------\n", "> layer modules: \n", - "(node 9): Conv2d(10, 1, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", - "(node 10): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [4]\n", - "> assigned core: 3\n", + "(node 12): Conv2d(4, 144, kernel_size=(6, 6), stride=(1, 1), bias=False)\n", + "(node 13): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(3048.), min_v_mem=Parameter containing:\n", + "tensor(-477.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [6]\n", + "> assigned core: 5\n", "\n", "---- DynapcnnLayer 4 ----------------------------------------------------------\n", "> layer modules: \n", - "(node 11): Conv2d(1, 100, kernel_size=(6, 6), stride=(1, 1), bias=False)\n", - "(node 12): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [5, 6]\n", - "> assigned core: 4\n", + "(node 9): Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + "(node 10): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1027.), min_v_mem=Parameter containing:\n", + "tensor(-161.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [5]\n", + "> assigned core: 3\n", "\n", "---- DynapcnnLayer 5 ----------------------------------------------------------\n", "> layer modules: \n", - "(node 13): Conv2d(100, 100, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - "(node 15): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [6, 7]\n", - "> assigned core: 5\n", + "(node 16): Conv2d(4, 11, kernel_size=(6, 6), stride=(1, 1), bias=False)\n", + "(node 17): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(3054.), min_v_mem=Parameter containing:\n", + "tensor(-478.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: []\n", + "> assigned core: 4\n", "\n", "---- DynapcnnLayer 6 ----------------------------------------------------------\n", "> layer modules: \n", - "(node 17): Conv2d(100, 100, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - "(node 18): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [7]\n", + "(node 14): Conv2d(144, 144, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + "(node 15): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(3048.), min_v_mem=Parameter containing:\n", + "tensor(-477.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [5]\n", "> assigned core: 6\n", "\n", - "---- DynapcnnLayer 7 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 19): Conv2d(100, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - "(node 20): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: []\n", - "> assigned core: 7\n", - "\n", "\n" ] } diff --git a/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/two_networks_merging_outputs.ipynb b/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/two_networks_merging_outputs.ipynb index 9e0ee4b4..8f1e2be3 100644 --- a/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/two_networks_merging_outputs.ipynb +++ b/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/two_networks_merging_outputs.ipynb @@ -26,7 +26,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 2, @@ -83,43 +83,35 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 4, "metadata": { "metadata": {} }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "torch.Size([8, 490]) torch.Size([8, 490])\n" - ] - } - ], + "outputs": [], "source": [ "class SNN(nn.Module):\n", " def __init__(self, nb_classes, batch_size, surrogate_fn, min_v_mem=-0.313, spk_thr=2.0) -> None:\n", " super().__init__()\n", "\n", - " self.conv_A = nn.Conv2d(2, 10, 2, 1, bias=False)\n", + " self.conv_A = nn.Conv2d(2, 4, 2, 1, bias=False)\n", " self.iaf_A = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", "\n", - " self.conv_B = nn.Conv2d(10, 10, 2, 1, bias=False)\n", + " self.conv_B = nn.Conv2d(4, 4, 2, 1, bias=False)\n", " self.iaf_B = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", " self.pool_B = sl.SumPool2d(2,2)\n", "\n", - " self.conv_C = nn.Conv2d(10, 10, 2, 1, bias=False)\n", + " self.conv_C = nn.Conv2d(4, 4, 2, 1, bias=False)\n", " self.iaf_C = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", " self.pool_C = sl.SumPool2d(2,2)\n", "\n", - " self.conv_D = nn.Conv2d(2, 10, 2, 1, bias=False)\n", + " self.conv_D = nn.Conv2d(2, 4, 2, 1, bias=False)\n", " self.iaf_D = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", "\n", - " self.conv_E = nn.Conv2d(10, 10, 2, 1, bias=False)\n", + " self.conv_E = nn.Conv2d(4, 4, 2, 1, bias=False)\n", " self.iaf_E = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", " self.pool_E = sl.SumPool2d(2,2)\n", "\n", - " self.conv_F = nn.Conv2d(10, 10, 2, 1, bias=False)\n", + " self.conv_F = nn.Conv2d(4, 4, 2, 1, bias=False)\n", " self.iaf_F = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", " self.pool_F = sl.SumPool2d(2,2)\n", "\n", @@ -127,7 +119,7 @@ " self.flat_brach2 = nn.Flatten()\n", " self.merge = Merge()\n", "\n", - " self.fc1 = nn.Linear(490, 200, bias=False)\n", + " self.fc1 = nn.Linear(196, 200, bias=False)\n", " self.iaf1_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", "\n", " self.fc2 = nn.Linear(200, 200, bias=False)\n", @@ -168,7 +160,6 @@ " flat_brach1_out = self.flat_brach1(pool_C_out)\n", " flat_brach2_out = self.flat_brach2(pool_F_out)\n", " merge_out = self.merge(flat_brach1_out, flat_brach2_out)\n", - " print(flat_brach1_out.shape, flat_brach2_out.shape)\n", "\n", " # FC 7 - G\n", " fc1_out = self.fc1(merge_out)\n", @@ -182,11 +173,7 @@ "\n", " return iaf3_fc_out\n", " \n", - "snn = SNN(11, 8, PeriodicExponential())\n", - "\n", - "x = torch.randn((8, *input_shape))\n", - "\n", - "out = snn(x)" + "snn = SNN(11, 1, PeriodicExponential())" ] }, { @@ -204,7 +191,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 5, "metadata": { "metadata": {} }, @@ -228,7 +215,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 6, "metadata": { "metadata": {} }, @@ -243,10 +230,10 @@ { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 8, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" } @@ -264,7 +251,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 7, "metadata": { "metadata": {} }, @@ -275,75 +262,86 @@ "text": [ "---- DynapcnnLayer 0 ----------------------------------------------------------\n", "> layer modules: \n", - "(node 0): Conv2d(2, 10, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + "(node 0): Conv2d(2, 4, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", "(node 1): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "(node 2): SumPool2d(norm_type=1, kernel_size=3, stride=3, ceil_mode=False)\n", - "(node 3): SumPool2d(norm_type=1, kernel_size=4, stride=4, ceil_mode=False)\n", - "> layer destinations: [1, 2]\n", + "tensor(758.), min_v_mem=Parameter containing:\n", + "tensor(-119.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [1]\n", "> assigned core: 0\n", "\n", "---- DynapcnnLayer 1 ----------------------------------------------------------\n", "> layer modules: \n", - "(node 4): Conv2d(10, 10, kernel_size=(4, 4), stride=(1, 1), bias=False)\n", - "(node 6): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "(node 2): Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + "(node 3): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1022.), min_v_mem=Parameter containing:\n", + "tensor(-160.), batch_size=1, num_timesteps=-1)\n", + "(node 4): SumPool2d(norm_type=1, kernel_size=2, stride=2, ceil_mode=False)\n", "> layer destinations: [2]\n", "> assigned core: 1\n", "\n", "---- DynapcnnLayer 2 ----------------------------------------------------------\n", "> layer modules: \n", - "(node 7): Conv2d(10, 10, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", - "(node 8): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "(node 5): Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + "(node 6): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1062.), min_v_mem=Parameter containing:\n", + "tensor(-166.), batch_size=1, num_timesteps=-1)\n", + "(node 7): SumPool2d(norm_type=1, kernel_size=2, stride=2, ceil_mode=False)\n", "> layer destinations: [3]\n", "> assigned core: 2\n", "\n", "---- DynapcnnLayer 3 ----------------------------------------------------------\n", "> layer modules: \n", - "(node 9): Conv2d(10, 1, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", - "(node 10): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [4]\n", - "> assigned core: 3\n", + "(node 17): Conv2d(4, 200, kernel_size=(7, 7), stride=(1, 1), bias=False)\n", + "(node 18): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(3556.), min_v_mem=Parameter containing:\n", + "tensor(-557.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [7]\n", + "> assigned core: 5\n", "\n", "---- DynapcnnLayer 4 ----------------------------------------------------------\n", "> layer modules: \n", - "(node 11): Conv2d(1, 100, kernel_size=(6, 6), stride=(1, 1), bias=False)\n", - "(node 12): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [5, 6]\n", - "> assigned core: 4\n", + "(node 8): Conv2d(2, 4, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + "(node 9): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(796.), min_v_mem=Parameter containing:\n", + "tensor(-125.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [5]\n", + "> assigned core: 3\n", "\n", "---- DynapcnnLayer 5 ----------------------------------------------------------\n", "> layer modules: \n", - "(node 13): Conv2d(100, 100, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - "(node 15): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [6, 7]\n", - "> assigned core: 5\n", + "(node 10): Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + "(node 11): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1027.), min_v_mem=Parameter containing:\n", + "tensor(-161.), batch_size=1, num_timesteps=-1)\n", + "(node 12): SumPool2d(norm_type=1, kernel_size=2, stride=2, ceil_mode=False)\n", + "> layer destinations: [6]\n", + "> assigned core: 4\n", "\n", "---- DynapcnnLayer 6 ----------------------------------------------------------\n", "> layer modules: \n", - "(node 17): Conv2d(100, 100, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - "(node 18): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [7]\n", - "> assigned core: 6\n", + "(node 13): Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + "(node 14): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1033.), min_v_mem=Parameter containing:\n", + "tensor(-162.), batch_size=1, num_timesteps=-1)\n", + "(node 15): SumPool2d(norm_type=1, kernel_size=2, stride=2, ceil_mode=False)\n", + "> layer destinations: [3]\n", + "> assigned core: 8\n", "\n", "---- DynapcnnLayer 7 ----------------------------------------------------------\n", "> layer modules: \n", - "(node 19): Conv2d(100, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + "(node 19): Conv2d(200, 200, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", "(node 20): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "tensor(3592.), min_v_mem=Parameter containing:\n", + "tensor(-562.), batch_size=1, num_timesteps=-1)\n", + "> layer destinations: [8]\n", + "> assigned core: 6\n", + "\n", + "---- DynapcnnLayer 8 ----------------------------------------------------------\n", + "> layer modules: \n", + "(node 21): Conv2d(200, 11, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + "(node 22): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(3592.), min_v_mem=Parameter containing:\n", + "tensor(-562.), batch_size=1, num_timesteps=-1)\n", "> layer destinations: []\n", "> assigned core: 7\n", "\n", From 6e6a61ccc011de82d1b9beecab9bb3cb72688d05 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Tue, 14 May 2024 18:10:49 +0200 Subject: [PATCH 087/379] small refactor to allow architecture cases listed in https://github.com/synsense/sinabs/issues/181 (e.g. nets with two input nodes merging along the comp. graph) --- sinabs/backend/dynapcnn/NIRGraphExtractor.py | 41 ++++++++++++++++--- .../dynapcnn/dynapcnn_network_graph.py | 30 +++++++++++--- 2 files changed, 60 insertions(+), 11 deletions(-) diff --git a/sinabs/backend/dynapcnn/NIRGraphExtractor.py b/sinabs/backend/dynapcnn/NIRGraphExtractor.py index 750885cc..de94ca23 100644 --- a/sinabs/backend/dynapcnn/NIRGraphExtractor.py +++ b/sinabs/backend/dynapcnn/NIRGraphExtractor.py @@ -223,7 +223,13 @@ def _get_nodes_io_shapes(self, input_dummy: torch.tensor) -> Dict[int, Dict[str, else: # find node generating the input to be used. inp_node = self._find_input_to_node(src) - _input = nodes_io_map[inp_node]['output'] + + if inp_node == -1: + # `src` is receiving external (not from another layer) input. This will be the case when two + # parallel branches (two independent "input nodes" in the graph) merge at some point in the graph. + _input = input_dummy + else: + _input = nodes_io_map[inp_node]['output'] # forward input through the node. _output = self.modules_map[src](_input) @@ -242,7 +248,13 @@ def _get_nodes_io_shapes(self, input_dummy: torch.tensor) -> Dict[int, Dict[str, else: # find node generating the input to be used. inp_node = self._find_input_to_node(src) - _input = nodes_io_map[inp_node]['output'] + + if inp_node == -1: + # `src` is receiving external (not from another layer) input. This will be the case when two + # parallel branches (two independent "input nodes" in the graph) merge at some point in the graph. + _input = input_dummy + else: + _input = nodes_io_map[inp_node]['output'] # forward input through the node. _output = self.modules_map[src](_input) @@ -256,7 +268,13 @@ def _get_nodes_io_shapes(self, input_dummy: torch.tensor) -> Dict[int, Dict[str, # find node generating the input to be used. inp_node = self._find_input_to_node(trg) - _input = nodes_io_map[inp_node]['output'] + + if inp_node == -1: + # `src` is receiving external (not from another layer) input. This will be the case when two + # parallel branches (two independent "input nodes" in the graph) merge at some point in the graph. + _input = input_dummy + else: + _input = nodes_io_map[inp_node]['output'] # forward input through the node. _output = self.modules_map[trg](_input) @@ -272,11 +290,24 @@ def _get_nodes_io_shapes(self, input_dummy: torch.tensor) -> Dict[int, Dict[str, return nodes_io_map def _find_input_to_node(self, node: int) -> int: - """ Finds the first edge `(X, node)` returns `X`. """ + """ Finds the first edge `(X, node)` returns `X`. + + Parameters + ---------- + node (int): the node in the computational graph for which we whish to find the input source (either another node in the + graph or the original input itself to the network). + + Returns + ---------- + input source (int): this indicates the node in the computational graph providing the input to `node`. If `node` is + receiving outside input (i.e., it is a starting node) the return will be -1. For example, this will be the case + when a network with two independent branches (each starts from a different "input node") merge along the computational graph. + """ for edge in self._edges_list: if edge[1] == node: return edge[0] - raise ValueError(f'Node {node} is not the target node of any edge in the graph.') + + return -1 def _find_merge_arguments(self, merge_node: int) -> list: """ A `Merge` layer receives two inputs. Return the two inputs to `merge_node` representing a `Merge` layer. """ diff --git a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py index d2c9fc6f..1c582b88 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py @@ -437,15 +437,27 @@ def find_original_node_name(name_mapper: dict, node: int): return orig_name raise ValueError(f'Node {node} could not be found within the name remapping done by self._get_sinabs_edges_and_modules().') - def find_my_input(edges_list: list, node: int): - """ Returns the node `X` in the first edge `(X, node)`.""" + def find_my_input(edges_list: list, node: int) -> int: + """ Returns the node `X` in the first edge `(X, node)`. + + Parameters + ---------- + node (int): the node in the computational graph for which we whish to find the input source (either another node in the + graph or the original input itself to the network). + + Returns + ---------- + input source (int): this indicates the node in the computational graph providing the input to `node`. If `node` is + receiving outside input (i.e., it is a starting node) the return will be -1. For example, this will be the case + when a network with two independent branches (each starts from a different "input node") merge along the computational graph. + """ for edge in edges_list: if edge[1] == node: # TODO nodes originally receiving input from merge will appear twice in the list of edges, one # edge per input to the merge layer. For now both inputs to a `Merge` have the same dimensions # necessarily so this works for now but later will have to be revised. return edge[0] - raise ValueError(f'Node {node} is not receiving input from any other node in the graph.') + return -1 # access the I/O shapes for each node in `self.sinabs_edges` from the original graph in `self.graph_tracer`. for dcnnl_idx, dcnnl_data in self.nodes_to_dcnnl_map.items(): @@ -462,9 +474,15 @@ def find_my_input(edges_list: list, node: int): # necessary cuz if a node originally receives input from a `nn.Flatten` for instance, when mapped into # a `DynapcnnLayer` it will be receiving the input from a privious `sl.SumPool2d`. input_node = find_my_input(self.sinabs_edges, node) - input_node_orig_name = find_original_node_name(self.nodes_name_remap, input_node) - _, _input_source_shape = self.graph_tracer.get_node_io_shapes(input_node_orig_name) - node_data['input_shape'] = tuple(list(_input_source_shape)[1:]) + + if input_node == -1: + # node does not have an input source within the graph (it consumes the original input to the model). + node_data['input_shape'] = tuple(list(_in)[1:]) + else: + # input comes from another node in the graph. + input_node_orig_name = find_original_node_name(self.nodes_name_remap, input_node) + _, _input_source_shape = self.graph_tracer.get_node_io_shapes(input_node_orig_name) + node_data['input_shape'] = tuple(list(_input_source_shape)[1:]) else: # first node does not have an input source within the graph. node_data['input_shape'] = tuple(list(_in)[1:]) From f42f2c4672654e42f58dc39d3072407ff2bab9a9 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Thu, 16 May 2024 19:31:38 +0200 Subject: [PATCH 088/379] Refactor DynapcnnLayer forward forward method of a DynapcnnLayer now returns as many tensors as there are destinations (unit tests need to be updated) --- sinabs/backend/dynapcnn/dynapcnn_layer_new.py | 111 ++++++++- .../dynapcnn/dynapcnn_network_graph.py | 8 +- .../dynapcnn/dynapcnnnetwork_module.py | 229 ++++++++---------- sinabs/backend/dynapcnn/utils.py | 3 +- 4 files changed, 211 insertions(+), 140 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer_new.py b/sinabs/backend/dynapcnn/dynapcnn_layer_new.py index 03372ca1..6edc144b 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer_new.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer_new.py @@ -18,7 +18,8 @@ class DynapcnnLayer(nn.Module): def __init__( self, dcnnl_data: dict, - discretize: bool + discretize: bool, + sinabs_edges: list ): super().__init__() """ @@ -118,6 +119,50 @@ def __init__( raise ValueError("Only square kernels are supported") self.pool_layer.append(deepcopy(plyr)) + # map destination nodes for each layer in this instance. + self.nodes_destinations = self._get_destinations_input_source(sinabs_edges) + + def get_destination_dcnnl_index(self, dcnnl_id: int) -> int: + """ The `forward` method will return as many tensors as there are elements in `self.dynapcnnlayer_destination`. Since the i-th returned tensor is to be + fed to the i-th destionation in `self.dynapcnnlayer_destination`, the return of this method can be used to index a tensor returned in the `forward` method. + + Parameters + ---------- + dcnnl_id (int): this should be one of the values listed within `self.dynapcnnlayer_destination`. + + Returns + ---------- + The index of `dcnnl_id` within `self.dynapcnnlayer_destination`. + """ + return self.dynapcnnlayer_destination.index(dcnnl_id) + + def _get_destinations_input_source(self, sinabs_edges: list) -> dict: + """ Creates a mapping between each layer in this `DynapcnnLayer` instance and its targe nodes that are part of different + `DynapcnnLayer` instances. This mapping is used to figure out how many tensors the `forward` method needs to return. + """ + destinations_input_source = {} + + # check whether spiking layer projects outside this DynapcnnLayer (i.e. to one of the destinations without passing through a pooling). + spk_destinations = [] + for edge in sinabs_edges: + if edge[0] == self.spk_node_id and edge[1] not in self.pool_node_id: + # spiking layer projects to a node outside this DynapcnnLayer. + spk_destinations.append(edge[1]) + if len(spk_destinations) > 0: + destinations_input_source[self.spk_node_id] = [] + for node_id in spk_destinations: + destinations_input_source[self.spk_node_id].append(node_id) + + # get `pooling->destination` mapping. The pooling outputs will be arranged sequentially since the pooling layers are added sequentially + # to `self.pool_layer` (i.e., as they appear in the computational graph of the original `nn.Module`). + for id in self.pool_node_id: + destinations_input_source[id] = [] + for edge in sinabs_edges: + if edge[0] == id: + destinations_input_source[id].append(edge[1]) + + return destinations_input_source + def __str__(self): pretty_print = '\n' @@ -127,19 +172,73 @@ def __str__(self): for idx, lyr in enumerate(self.pool_layer): pretty_print += f'\n(node {self.pool_node_id[idx]}): {lyr}' + for node, destinations in self.nodes_destinations.items(): + pretty_print += f'\n> node {node} destination nodes: {destinations}' + return pretty_print def forward(self, x): - """Torch forward pass.""" + """Torch forward pass. + + Returns + ---------- + This method will return as many tensors as there are destinations associated with this instance. The returned tensors always follows the + sequence `return spiking_layer_output, pooling_layer_1, pooling_layer_2, ...`. + + Example + ---------- + With `self.nodes_destinations = {1: [5, 8], 2: [4], 3: [7]}` this method will return 4 tensors (each to be sent to a different DynapcnnLayer): the 1st + and 2nd are the outputs of the spiking layer (`node 1`); the 3rd is the output of the first pooling layer (`node 2`, receiving the output of node 1) appearing + right after the spiking layer as defined in the original `nn.Module` being converted to a `DynapcnnNetwork`; the 4th is the output of the second pooling + layer (`node 3`) to appear after the spiking layer. Thus, the return will be `return node_1_out, node_1_out, node_2_out, node_3_out`. This means the edges + in the computational graph involved in this mapping were: + + 1 --> 2 # `2` is one of the pooling layers of this DynapcnnLayer. + 1 --> 3 # `3` is one of the pooling layers of this DynapcnnLayer. + 1 --> 5 # `5` is a conv layer belonging to another DynapcnnLayer U. + 1 --> 8 # `8` is a conv layer belonging to another DynapcnnLayer V. + 2 --> 4 # `4` is a conv layer belonging to another DynapcnnLayer X. + 3 --> 7 # `7` is a conv layer belonging to another DynapcnnLayer Y. + """ + + returns = [] x = self.conv_layer(x) x = self.spk_layer(x) - if len(self.pool_layer) == 1: - # single pooling layer (not a divergent node). - x = self.pool_layer[0](x) + # pooling layers are sequentially added to `self.pool_layer` so we'll pass data to them sequentially. + pooling_indexer = 0 + + # building return set of all layers as they appear in `self.nodes_destinations`. + for node_id, destination_node_list in self.nodes_destinations.items(): + if node_id == self.spk_node_id: + # spiking output for each node outside this DynapcnnLayer receiving from its spiking layer. + for _ in destination_node_list: + returns.append(x) + else: + # returns of each pooling layer are arranged sequenatially. + for _ in destination_node_list: + ith_pool_output = self.pool_layer[pooling_indexer](x) + returns.append(ith_pool_output) + + # forward through next pooling layer in `self.pool_layer` in the next iteration. + pooling_indexer += 1 + + if len(returns) != len(self.dynapcnnlayer_destination): + raise ValueError(f'Number of returned tensors ({len(returns)}) differ from the number of destinations ({len(self.dynapcnnlayer_destination)}).') + + if len(returns) == 0 and len(self.pool_layer) == 0: + # this is the output layer and there's no pooling after the neurons. + returns.append(x) + elif len(returns) == 0 and len(self.pool_layer) == 1: + # this is the output layer and there's 1 pooling after the neurons. + returns.append(self.pool_layer[0](x)) + elif len(returns) == 0 and len(self.pool_layer) > 1: + raise ValueError(f'Output DynapcnnLayer starting with node {self.conv_node_id} has {len(self.pool_layer)} pooling layers: it should have either 1 or none.') + else: + pass - return x + return tuple(returns) def _convert_linear_to_conv(self, lin: nn.Linear, layer_data: dict) -> nn.Conv2d: """Convert Linear layer to Conv2d. diff --git a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py index 1c582b88..70ae6344 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py @@ -67,6 +67,7 @@ def __init__( assert len(self.input_shape) == 3, "infer_input_shape did not return 3-tuple" # computational graph from original PyTorch module. + # TODO - bacth size must be passed as argument. self.graph_tracer = NIRtoDynapcnnNetworkGraph( snn, torch.randn((1, *self.input_shape))) # needs the batch dimension. @@ -84,6 +85,9 @@ def __init__( self._populate_nodes_io() # build `DynapcnnLayer` instances from graph edges and mapper. + # for edge in self.sinabs_edges: + # print(edge) + # print('\n') self.dynapcnn_layers = build_from_graph( discretize=discretize, edges=self.sinabs_edges, @@ -104,9 +108,9 @@ def __str__(self): if 'core_destinations' in layer_data: core_dest = layer_data['destinations'] - pretty_print += f'\n> layer modules: {layer}\n> layer destinations: {dest}\n> core destinations: {core_dest}\n> assigned core: {core}\n\n' + pretty_print += f'\n> layer modules: {layer}\n> destination DynapcnnLayers: {dest}\n> core destinations: {core_dest}\n> assigned core: {core}\n\n' else: - pretty_print += f'\n> layer modules: {layer}\n> layer destinations: {dest}\n> assigned core: {core}\n\n' + pretty_print += f'\n> layer modules: {layer}\n> destination DynapcnnLayers: {dest}\n> assigned core: {core}\n\n' return pretty_print def to( diff --git a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py index f027c3e8..75179186 100644 --- a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py +++ b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py @@ -25,144 +25,139 @@ class DynapcnnNetworkModule(nn.Module): def __init__(self, dcnnl_edges: List[Tuple[int, int]], dynapcnn_layers: Dict) -> nn.Module: super().__init__() - self._forward_edges, self._forward_map = self._build_module_forward_from_graph(dcnnl_edges, dynapcnn_layers) + self.dcnnl_edges = dcnnl_edges - def get_forward_edges(self): - return self._forward_edges + self.forward_map, self.merge_points = self._build_module_forward_from_graph_v2(dcnnl_edges, dynapcnn_layers) + + def _spot_merging_points(self, dcnnl_edges: list) -> dict: + """ . """ + + nodes_with_merge_input = {} - def _build_module_forward_from_graph(self, dcnnl_edges: list, dynapcnn_layers: dict) -> Union[list, dict]: - """ - TODO use copy.deepcopy for create the `forward_map`. - """ - forward_map = {} - new_edges_set = [] - divergent_nodes = [] - for edge in dcnnl_edges: - source_dcnnl = edge[0] - target_dcnnl = edge[1] + trg_node = edge[1] + fan_in = 0 + src_nodes = [] + + for edge_inner in dcnnl_edges: + if edge_inner[1] == trg_node: + fan_in += 1 + src_nodes.append(edge_inner[0]) + + if fan_in == 2 and trg_node not in nodes_with_merge_input: + nodes_with_merge_input[trg_node] = {'sources': tuple(src_nodes), 'merge': sl.Merge()} + + if fan_in > 2: + raise ValueError(f'Node {trg_node} is the has fan-in of {fan_in}: only fan-in of 2 is currently handled.') + + return nodes_with_merge_input - new_edge_2_append = [] + + def _build_module_forward_from_graph_v2(self, dcnnl_edges: list, dynapcnn_layers: dict) -> Union[dict, dict]: + """ .""" - # processing the source `DynapcnnLayer`. + # mapper to flag nodes that need input from a `Merge` layer. + merge_points = self._spot_merging_points(dcnnl_edges) - if source_dcnnl not in forward_map: - forward_map[source_dcnnl] = copy.deepcopy(dynapcnn_layers[source_dcnnl]['layer']) + # this dict. will be used to call the `forward` methods of each `DynapcnnLayer`. + forward_map = {} - if len(forward_map[source_dcnnl].pool_layer) > 1: - # this `DynapcnnLayer` is a divergent point in the graph. - divergent_nodes.append(source_dcnnl) - for i in range(len(forward_map[source_dcnnl].pool_layer)): - - # create edge representing forward through the i-th pooling layer. - pool_name = f'{source_dcnnl}_pool{i}' - new_edges_set.append((source_dcnnl, pool_name)) + for edge in dcnnl_edges: + src_dcnnl = edge[0] # source layer + trg_dcnnl = edge[1] # target layer - # create forward 'node' for the i-th pooling layer. - if pool_name not in forward_map: - forward_map[pool_name] = copy.deepcopy(forward_map[source_dcnnl].pool_layer[i]) + if src_dcnnl not in forward_map: + forward_map[src_dcnnl] = copy.deepcopy(dynapcnn_layers[src_dcnnl]['layer']) + + if trg_dcnnl not in forward_map: + forward_map[trg_dcnnl] = copy.deepcopy(dynapcnn_layers[trg_dcnnl]['layer']) - # create edge from i-th pooling to its target `DynapcnnLayer`. - new_edge_2_append.append((pool_name, dynapcnn_layers[source_dcnnl]['destinations'][i])) + return forward_map, merge_points + + def forward(self, x): + """ .""" - # processing the target `DynapcnnLayer`. + layers_outputs = {} - if target_dcnnl not in forward_map: - forward_map[target_dcnnl] = copy.deepcopy(dynapcnn_layers[target_dcnnl]['layer']) + # TODO - currently `node 0` (this 1st node in the 1st edge of `self.dcnnl_edges`) is always taken to be the + # input node of the network. This won't work in cases where there are more the one input nodes to the network + # so this functionality needs some refactoring. + self.forward_map[self.dcnnl_edges[0][0]](x) - if len(forward_map[target_dcnnl].pool_layer) > 1: - # this `DynapcnnLayer` is a divergent point in the graph. - divergent_nodes.append(target_dcnnl) - for i in range(len(forward_map[target_dcnnl].pool_layer)): - - # create edge representing forward through the i-th pooling layer. - pool_name = f'{target_dcnnl}_pool{i}' - new_edges_set.append((target_dcnnl, pool_name)) + # forward the input `x` through the input `DynapcnnLayer` in the `DynapcnnNetwork`s graph (1st node in the 1st edge in `self.dcnnl_edges`). + layers_outputs[self.dcnnl_edges[0][0]] = self.forward_map[self.dcnnl_edges[0][0]](x) - # create forward 'node' for the i-th pooling layer. - if pool_name not in forward_map: - forward_map[pool_name] = copy.deepcopy(forward_map[target_dcnnl].pool_layer[i]) + # propagate outputs in `layers_outputs` through the rest of the nodes of `self.dcnnl_edges`. + for edge in self.dcnnl_edges: + + # target DynapcnnLayer (will consume tensors from `layers_outputs`). + trg_dcnnl = edge[1] - # create edge from i-th pooling to its target `DynapcnnLayer`. - new_edge_2_append.append((pool_name, dynapcnn_layers[target_dcnnl]['destinations'][i])) + if trg_dcnnl in self.merge_points and trg_dcnnl not in layers_outputs: + # by this points the arguments of the `Merge` associated with `trg_dcnnl` should have been computed. + arg1, arg2 = self.merge_points[trg_dcnnl]['sources'] - if source_dcnnl not in divergent_nodes and target_dcnnl not in divergent_nodes: - # save original edge. - new_edges_set.append(edge) + # find which returned tensor from the `forward` call of DynapcnnLayers `arg1` and `arg2` are to be fed + # to the target DynapcnnLayer `trg_dcnnl`. + return_index_arg1 = self.forward_map[arg1].get_destination_dcnnl_index(trg_dcnnl) + return_index_arg2 = self.forward_map[arg2].get_destination_dcnnl_index(trg_dcnnl) - if len(new_edge_2_append) != 0: - new_edges_set.extend(new_edge_2_append) + # retrieve input tensors to `Merge`. + _arg1 = layers_outputs[arg1][return_index_arg1] + _arg2 = layers_outputs[arg2][return_index_arg2] - forward_edges = self._find_merging_nodes(new_edges_set, forward_map) + # merge tensors. + merge_output = self.merge_points[trg_dcnnl]['merge'](_arg1, _arg2) - return forward_edges, forward_map - - def _find_merging_nodes(self, edges_list: list, forward_map: dict) -> list: - """ Loops through the edges and see if a node appeards in more than one edge. If so, this is a node - that requires a `Merge` layer. For instance, edges `(A, X)` and `(B, X)` will be replace by two new - edges `((A, B), Merge_X)` and `(Merge_X, X)`, where `A` and `B` are the inputs to a `Merge` feeding into `X`. - """ - merge_mapping = {} + # call the forward. + layers_outputs[trg_dcnnl] = self.forward_map[trg_dcnnl](merge_output) - for edge in edges_list: - src = edge[0] - trg = edge[1] + elif trg_dcnnl not in layers_outputs: + # input source for `trg_dcnnl`. + src_dcnnl = edge[0] - if trg in merge_mapping: - # node needs to receive input from a `Merge` layer. - merge_arguments = ( - merge_mapping[trg]['src'], # merge_arguments[0] = source (from 1st edge containing `trg`). - src) # merge_arguments[1] = `src` (the source of the 2nd edge containing `trg`). + # find which returned tensor from the `forward` call of the source DynapcnnLayer `src_dcnnl` is to be fed + # to the target DynapcnnLayer `trg_dcnnl`. + return_index = self.forward_map[src_dcnnl].get_destination_dcnnl_index(trg_dcnnl) - merge_mapping[trg] = {'src': merge_arguments} + # call the forward. + layers_outputs[trg_dcnnl] = self.forward_map[trg_dcnnl](layers_outputs[src_dcnnl][return_index]) else: - merge_mapping[trg] = {'src': src} - - final_edges = [] - merge_idx = 0 - - # create edges `((A, B), Merge_X)` and `(Merge_X, X)`. - for trg, src in merge_mapping.items(): - _ = src['src'] - - if isinstance(_, tuple): - # `trg` receives from a `Merge` layer. - merge_node = f'merge_{merge_idx}' - forward_map[merge_node] = Merge() - - new_edge = (_, merge_node) - final_edges.append(new_edge) - new_edge = (merge_node, trg) - final_edges.append(new_edge) - - merge_idx += 1 - - else: - final_edges.append((_, trg)) + pass - return final_edges + # TODO - this assumes the network has a single output node. + # last computed is the output layer. + return layers_outputs[trg_dcnnl][0] def parameters(self) -> list: - """ .""" + """ Gathers all the parameters of the network in a list. This is done by accessing the convolutional layer in each `DynapcnnLayer`, calling + its `.parameters` method and saving it to a list. + + Note: the method assumes no biases are used. + + Returns + ---------- + parameters (list): a list of parameters of all convolutional layers in the `DynapcnnNetwok`. + """ parameters = [] - for layer in self._forward_map.values(): + for layer in self.forward_map.values(): if isinstance(layer, sinabs.backend.dynapcnn.dynapcnn_layer_new.DynapcnnLayer): parameters.extend(layer.conv_layer.parameters()) return parameters - def init_weights(self): - """ .""" - for layer in self._forward_map.values(): + def init_weights(self, init_fn: nn.init = nn.init.xavier_normal_) -> None: + """ Call the weight initialization method `init_fn` on each `DynapcnnLayer.conv_layer.weight.data` in the `DynapcnnNetwork` instance.""" + for layer in self.forward_map.values(): if isinstance(layer, sinabs.backend.dynapcnn.dynapcnn_layer_new.DynapcnnLayer): - nn.init.xavier_normal_(layer.conv_layer.weight.data) + init_fn(layer.conv_layer.weight.data) - def to(self, device): + def to(self, device) -> None: """ .""" - for layer in self._forward_map.values(): + for layer in self.forward_map.values(): if isinstance(layer, sinabs.backend.dynapcnn.dynapcnn_layer_new.DynapcnnLayer): layer.conv_layer.to(device) layer.spk_layer.to(device) @@ -177,36 +172,8 @@ def to(self, device): def detach_neuron_states(self) -> None: """ Detach the neuron states and activations from current computation graph (necessary). """ - for module in self._forward_map.values(): + for module in self.forward_map.values(): if isinstance(module, sinabs.backend.dynapcnn.dynapcnn_layer_new.DynapcnnLayer): if isinstance(module.spk_layer, sl.StatefulLayer): for name, buffer in module.spk_layer.named_buffers(): - buffer.detach_() - - def forward(self, x): - """ The torch forward uses `self._forward_edges` to feed data throguh the - layers in `self._forward_map`. - """ - - layers_outputs = {} - - # input node has to be `0`. - layers_outputs[0] = self._forward_map[0](x) - - for edge in self._forward_edges: - src = edge[0] - trg = edge[1] - - # gets the input to the target node (must have been computed already). - if isinstance(src, tuple): - # `trg` is a Merge layer. - arg1 = layers_outputs[src[0]] - arg2 = layers_outputs[src[1]] - - layers_outputs[trg] = self._forward_map[trg](arg1, arg2) - - else: - x = layers_outputs[src] - layers_outputs[trg] = self._forward_map[trg](x) - - return layers_outputs[trg] \ No newline at end of file + buffer.detach_() \ No newline at end of file diff --git a/sinabs/backend/dynapcnn/utils.py b/sinabs/backend/dynapcnn/utils.py index 9768d6da..cc16d9af 100644 --- a/sinabs/backend/dynapcnn/utils.py +++ b/sinabs/backend/dynapcnn/utils.py @@ -693,7 +693,8 @@ def construct_dynapcnnlayer( # instantiate a DynapcnnLayer from the data in 'dcnnl_data'. dynapcnnlayer = DynapcnnLayer( dcnnl_data = dcnnl_data, - discretize = discretize + discretize = discretize, + sinabs_edges = edges ) return dynapcnnlayer From 67f15bbd74540c4c4c6c561940c18318b08c32ed Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Thu, 16 May 2024 22:47:08 +0200 Subject: [PATCH 089/379] Refactor DynapcnnNetwork the class now is inheriting from 'nn.Module', implements '.forward()', '.to()' behaviour is device-dependent and it frees memory after sucessful call --- sinabs/backend/dynapcnn/dynapcnn_layer_new.py | 45 ++- .../dynapcnn/dynapcnn_network_graph.py | 320 ++++++++++++------ .../dynapcnn/dynapcnnnetwork_module.py | 119 +------ sinabs/backend/dynapcnn/utils.py | 4 +- 4 files changed, 264 insertions(+), 224 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer_new.py b/sinabs/backend/dynapcnn/dynapcnn_layer_new.py index 6edc144b..356ab44f 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer_new.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer_new.py @@ -17,6 +17,7 @@ class DynapcnnLayer(nn.Module): def __init__( self, + dpcnnl_index: int, dcnnl_data: dict, discretize: bool, sinabs_edges: list @@ -27,9 +28,17 @@ def __init__( Parameters ---------- + dpcnnl_index (int): ... dcnnl_data (dict): ... discretize (bool): ... + sinabs_edges (list): ... """ + self.dpcnnl_index = dpcnnl_index + self.assigned_core = None + + if 'core_idx' in dcnnl_data: + self.assigned_core = dcnnl_data['core_idx'] + self.lin_to_conv_conversion = False conv = None @@ -162,20 +171,6 @@ def _get_destinations_input_source(self, sinabs_edges: list) -> dict: destinations_input_source[id].append(edge[1]) return destinations_input_source - - def __str__(self): - pretty_print = '\n' - - pretty_print += f'(node {self.conv_node_id}): {self.conv_layer}\n' - pretty_print += f'(node {self.spk_node_id}): {self.spk_layer}' - if len(self.pool_layer) != 0: - for idx, lyr in enumerate(self.pool_layer): - pretty_print += f'\n(node {self.pool_node_id[idx]}): {lyr}' - - for node, destinations in self.nodes_destinations.items(): - pretty_print += f'\n> node {node} destination nodes: {destinations}' - - return pretty_print def forward(self, x): """Torch forward pass. @@ -466,4 +461,24 @@ def memory_summary(self): "neuron": f * pow(2, np.ceil(np.log2(neuron_height)) + np.ceil(np.log2(neuron_width))), "bias": 0 if self.conv_layer.bias is None else len(self.conv_layer.bias), - } \ No newline at end of file + } + + def __str__(self): + pretty_print = '\n' + + pretty_print += 'COMPUTATIONAL NODES:\n\n' + + pretty_print += f'(node {self.conv_node_id}): {self.conv_layer}\n' + pretty_print += f'(node {self.spk_node_id}): {self.spk_layer}' + if len(self.pool_layer) != 0: + for idx, lyr in enumerate(self.pool_layer): + pretty_print += f'\n(node {self.pool_node_id[idx]}): {lyr}' + + pretty_print += '\n\nMETADATA:\n' + pretty_print += f'\n> assigned core index: {self.assigned_core}' + pretty_print += f'\n> destination DynapcnnLayers: {self.dynapcnnlayer_destination}' + + for node, destinations in self.nodes_destinations.items(): + pretty_print += f'\n> node {node} feeds input to nodes {destinations}' + + return pretty_print \ No newline at end of file diff --git a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py index 70ae6344..adc82016 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py @@ -3,11 +3,10 @@ # contact : williansoaresgirao@gmail.com import time -from subprocess import CalledProcessError from typing import List, Optional, Sequence, Tuple, Union, Dict import samna -import sinabs.layers +import sinabs.layers as sl import torch import torch.nn as nn @@ -15,8 +14,7 @@ from .chip_factory import ChipFactory from .dvs_layer import DVSLayer -from .dynapcnn_layer import DynapcnnLayer -from .io import disable_timestamps, enable_timestamps, open_device, reset_timestamps +from .io import open_device from .utils import ( DEFAULT_IGNORED_LAYER_TYPES, build_from_graph, @@ -24,23 +22,14 @@ parse_device_id, ) -from .graph_tracer import GraphTracer -from .exceptions import InvalidTorchModel -from warnings import warn - from .NIRGraphExtractor import NIRtoDynapcnnNetworkGraph from .sinabs_edges_handler import merge_handler from .dynapcnnnetwork_module import DynapcnnNetworkModule -class DynapcnnNetworkGraph(): - """Given a sinabs spiking network, prepare a dynapcnn-compatible network. This can be used to - test the network will be equivalent once on DYNAPCNN. This class also provides utilities to - make the dynapcnn configuration and upload it to DYNAPCNN. - - TODO `make_config` and `_make_config` should be merged into a single method. - """ +# TODO `make_config` and `_make_config` should be merged into a single method. +class DynapcnnNetworkGraph(nn.Module): def __init__( self, snn: nn.Module, @@ -49,6 +38,21 @@ def __init__( discretize: bool = True ): """ + Given a sinabs spiking network, prepare a dynapcnn-compatible network. This can be used to + test the network will be equivalent once on DYNAPCNN. This class also provides utilities to + make the dynapcnn configuration and upload it to DYNAPCNN. + + Some of the properties defined within the class constructor are meant to be temporary data structures handling the conversion + of the `snn` (the original `nn.Module`) into a set of `DynapcnnLayer`s composing a `DynapcnnNetwork` instance. Thus, the following + private properties are delted once a successfull call to `self.to(device='speck...')` is made: + + - self._graph_tracer + - self._sinabs_edges + - self._sinabs_modules_map + - self._nodes_name_remap + - self._nodes_to_dcnnl_map + - self._dynapcnn_layers + Parameters ---------- snn : a `nn.Module` implementing a spiking network. @@ -57,6 +61,7 @@ def __init__( discretize: If `True`, discretize the parameters and thresholds. This is needed for uploading weights to dynapcnn. Set to `False` only for testing purposes. """ + super().__init__() # TODO for now the graph part is not taking into consideration DVS inputs. # check if dvs input is expected. @@ -68,50 +73,129 @@ def __init__( # computational graph from original PyTorch module. # TODO - bacth size must be passed as argument. - self.graph_tracer = NIRtoDynapcnnNetworkGraph( + self._graph_tracer = NIRtoDynapcnnNetworkGraph( snn, torch.randn((1, *self.input_shape))) # needs the batch dimension. - self.sinabs_edges, \ - self.sinabs_modules_map, \ - self.nodes_name_remap = self._get_sinabs_edges_and_modules() + self._sinabs_edges, \ + self._sinabs_modules_map, \ + self._nodes_name_remap = self._get_sinabs_edges_and_modules() # create a dict holding the data necessary to instantiate a `DynapcnnLayer`. - self.nodes_to_dcnnl_map = build_nodes_to_dcnnl_map( - layers=self.sinabs_modules_map, - edges=self.sinabs_edges) + self._nodes_to_dcnnl_map = build_nodes_to_dcnnl_map( + layers=self._sinabs_modules_map, + edges=self._sinabs_edges) - # updates 'self.nodes_to_dcnnl_map' to include the I/O shape for each node. + # updates 'self._nodes_to_dcnnl_map' to include the I/O shape for each node. self._populate_nodes_io() # build `DynapcnnLayer` instances from graph edges and mapper. - # for edge in self.sinabs_edges: - # print(edge) - # print('\n') - self.dynapcnn_layers = build_from_graph( + self._dynapcnn_layers = build_from_graph( discretize=discretize, - edges=self.sinabs_edges, - nodes_to_dcnnl_map=self.nodes_to_dcnnl_map) + edges=self._sinabs_edges, + nodes_to_dcnnl_map=self._nodes_to_dcnnl_map) - # the trainable network (a `nn.Module`) instance: set at the end of the `.make_config()` call if configuration is valid. - self.network = None + # these gather all data necessay to implement the forward method for this class. + self._dcnnl_edges, self._forward_map, self._merge_points = self._get_network_module() - ### Public Methods ### + ####################################################### Public Methods ####################################################### + + @property + def dynapcnn_layers(self): + if hasattr(self, '_dynapcnn_layers'): + return self._dynapcnn_layers + else: + return None + + def forward(self, x): + """ .""" + + layers_outputs = {} + + # TODO - currently `node 0` (this 1st node in the 1st edge of `self._dcnnl_edges`) is always taken to be the + # input node of the network. This won't work in cases where there are more the one input nodes to the network + # so this functionality needs some refactoring. + self._forward_map[self._dcnnl_edges[0][0]](x) + + # forward the input `x` through the input `DynapcnnLayer` in the `DynapcnnNetwork`s graph (1st node in the 1st edge in `self._dcnnl_edges`). + layers_outputs[self._dcnnl_edges[0][0]] = self._forward_map[self._dcnnl_edges[0][0]](x) + + # propagate outputs in `layers_outputs` through the rest of the nodes of `self._dcnnl_edges`. + for edge in self._dcnnl_edges: + + # target DynapcnnLayer (will consume tensors from `layers_outputs`). + trg_dcnnl = edge[1] + + if trg_dcnnl in self._merge_points and trg_dcnnl not in layers_outputs: + # by this points the arguments of the `Merge` associated with `trg_dcnnl` should have been computed. + arg1, arg2 = self._merge_points[trg_dcnnl]['sources'] + + # find which returned tensor from the `forward` call of DynapcnnLayers `arg1` and `arg2` are to be fed + # to the target DynapcnnLayer `trg_dcnnl`. + return_index_arg1 = self._forward_map[arg1].get_destination_dcnnl_index(trg_dcnnl) + return_index_arg2 = self._forward_map[arg2].get_destination_dcnnl_index(trg_dcnnl) + + # retrieve input tensors to `Merge`. + _arg1 = layers_outputs[arg1][return_index_arg1] + _arg2 = layers_outputs[arg2][return_index_arg2] + + # merge tensors. + merge_output = self._merge_points[trg_dcnnl]['merge'](_arg1, _arg2) + + # call the forward. + layers_outputs[trg_dcnnl] = self._forward_map[trg_dcnnl](merge_output) + + elif trg_dcnnl not in layers_outputs: + # input source for `trg_dcnnl`. + src_dcnnl = edge[0] + + # find which returned tensor from the `forward` call of the source DynapcnnLayer `src_dcnnl` is to be fed + # to the target DynapcnnLayer `trg_dcnnl`. + return_index = self._forward_map[src_dcnnl].get_destination_dcnnl_index(trg_dcnnl) + + # call the forward. + layers_outputs[trg_dcnnl] = self._forward_map[trg_dcnnl](layers_outputs[src_dcnnl][return_index]) - def __str__(self): - pretty_print = '' - for idx, layer_data in self.dynapcnn_layers.items(): - pretty_print += f'---- DynapcnnLayer {idx} ----------------------------------------------------------' - layer = layer_data['layer'] - dest = layer_data['destinations'] - core = layer_data['core_idx'] - - if 'core_destinations' in layer_data: - core_dest = layer_data['destinations'] - pretty_print += f'\n> layer modules: {layer}\n> destination DynapcnnLayers: {dest}\n> core destinations: {core_dest}\n> assigned core: {core}\n\n' else: - pretty_print += f'\n> layer modules: {layer}\n> destination DynapcnnLayers: {dest}\n> assigned core: {core}\n\n' - return pretty_print + + pass + + # TODO - this assumes the network has a single output node. + # last computed is the output layer. + return layers_outputs[trg_dcnnl][0] + + def parameters(self) -> list: + """ Gathers all the parameters of the network in a list. This is done by accessing the convolutional layer in each `DynapcnnLayer`, calling + its `.parameters` method and saving it to a list. + + Note: the method assumes no biases are used. + + Returns + ---------- + parameters (list): a list of parameters of all convolutional layers in the `DynapcnnNetwok`. + """ + parameters = [] + + for layer in self._forward_map.values(): + if isinstance(layer, sinabs.backend.dynapcnn.dynapcnn_layer_new.DynapcnnLayer): + parameters.extend(layer.conv_layer.parameters()) + + return parameters + + def init_weights(self, init_fn: nn.init = nn.init.xavier_normal_) -> None: + """ Call the weight initialization method `init_fn` on each `DynapcnnLayer.conv_layer.weight.data` in the `DynapcnnNetwork` instance.""" + for layer in self._forward_map.values(): + if isinstance(layer, sinabs.backend.dynapcnn.dynapcnn_layer_new.DynapcnnLayer): + init_fn(layer.conv_layer.weight.data) + + def detach_neuron_states(self) -> None: + """ Detach the neuron states and activations from current computation graph (necessary). """ + + for module in self._forward_map.values(): + if isinstance(module, sinabs.backend.dynapcnn.dynapcnn_layer_new.DynapcnnLayer): + if isinstance(module.spk_layer, sl.StatefulLayer): + for name, buffer in module.spk_layer.named_buffers(): + buffer.detach_() def to( self, @@ -159,35 +243,42 @@ def to( self.device = device if isinstance(device, torch.device): - return super().to(device) + self._to_device(device) elif isinstance(device, str): device_name, _ = parse_device_id(device) - if device_name in ChipFactory.supported_devices: # pragma: no cover + if device_name in ChipFactory.supported_devices: - config = self.make_config( # generate config. + # generate config. + config = self.make_config( chip_layers_ordering=chip_layers_ordering, device=device, monitor_layers=monitor_layers, config_modifier=config_modifier, ) - self.samna_device = open_device(device) # apply configuration to device. + # apply configuration to device. + self.samna_device = open_device(device) self.samna_device.get_model().apply_configuration(config) time.sleep(1) - if slow_clk_frequency is not None: # set external slow-clock if needed. + # set external slow-clock if needed. + if slow_clk_frequency is not None: dk_io = self.samna_device.get_io_module() dk_io.set_slow_clk(True) - dk_io.set_slow_clk_rate(slow_clk_frequency) # Hz + dk_io.set_slow_clk_rate(slow_clk_frequency) # Hz builder = ChipFactory(device).get_config_builder() - self.samna_input_buffer = builder.get_input_buffer() # create input source node. - self.samna_output_buffer = builder.get_output_buffer() # create output sink node node. + # create input source node. + self.samna_input_buffer = builder.get_input_buffer() + + # create output sink node node. + self.samna_output_buffer = builder.get_output_buffer() - self.device_input_graph = samna.graph.EventFilterGraph() # connect source node to device sink. + # connect source node to device sink. + self.device_input_graph = samna.graph.EventFilterGraph() self.device_input_graph.sequential( [ self.samna_input_buffer, @@ -195,7 +286,8 @@ def to( ] ) - self.device_output_graph = samna.graph.EventFilterGraph() # connect sink node to device. + # connect sink node to device. + self.device_output_graph = samna.graph.EventFilterGraph() self.device_output_graph.sequential( [ self.samna_device.get_model().get_source_node(), @@ -207,10 +299,10 @@ def to( self.device_output_graph.start() self.samna_config = config - return self + return print(self) else: - return super().to(device) + self._to_device(device) else: raise Exception("Unknown device description.") @@ -272,40 +364,52 @@ def make_config( if is_compatible: # validate config. - print("Network is valid") - - # constructs a `nn.Module` class combining the `DynapcnnLayer` uploaded to the chip. - self.network = self._get_network_module() + print("Network is valid: \n") + + # DynapcnnLayer have been configured: removing intermediary data structures no longer necessary. + del self._graph_tracer + del self._sinabs_edges + del self._sinabs_modules_map + del self._nodes_name_remap + del self._nodes_to_dcnnl_map + del self._dynapcnn_layers return config else: raise ValueError(f"Generated config is not valid for {device}") - def get_network_module(self): - return self._get_network_module() - - def get_dynapcnnlayers_edges(self) -> List[Tuple[int, int]]: + ####################################################### Private Methods ####################################################### + + def _get_network_module(self) -> Union[list, dict, dict]: + """ Uses the `DynapcnnLayer` instances in `self._dynapcnn_layers` and the connectivity between them to create three data structures + that guide the data forwarding between the layer during the forward pass. + + Note: the property `DynapcnnLayer.assigned_core` is only set after `self.to(device='speck...')` is called. + + Returns + ---------- + dcnnl_edges (list): edges, represented as tuples of `DynapcnnLayer` indices, used to guide the data forwarding through each `DynapcnnLayer` in forward method. + forward_map (dict): have all the `DynapcnnLayer` (`value`), each being accessible via its index (`key`). Used to call `DynapcnnLayer.forward` in forward method. + merge_points (dict): used to compose the inputs to a `DynapcnnLayer` that requires an input from a `Merge` layer. + """ + + # get connections between `DynapcnnLayer`s. + dcnnl_edges = self._get_dynapcnnlayers_edges() + + dcnnnet_module = DynapcnnNetworkModule(dcnnl_edges, self._dynapcnn_layers) + + return dcnnnet_module.dcnnl_edges, dcnnnet_module.forward_map, dcnnnet_module.merge_points + + def _get_dynapcnnlayers_edges(self) -> List[Tuple[int, int]]: """ Create edges representing connections between `DynapcnnLayer` instances. """ dcnnl_edges = [] - for dcnnl_idx, layer_data in self.dynapcnn_layers.items(): + for dcnnl_idx, layer_data in self._dynapcnn_layers.items(): for dest in layer_data['destinations']: dcnnl_edges.append((dcnnl_idx, dest)) return dcnnl_edges - ### Private Methods ### - - def _get_network_module(self) -> nn.Module: - """ Uses the `DynapcnnLayer` instances in `self.dynapcnn_layers` and the connectivity between the cores - to craete a `nn.Module` with a forward method that incorporates each `DynapcnnLayer` into a trainable network. - """ - - # get connections between `DynapcnnLayer`s. - dcnnl_edges = self.get_dynapcnnlayers_edges() - - return DynapcnnNetworkModule(dcnnl_edges, self.dynapcnn_layers) - def _make_config( self, chip_layers_ordering: Union[Sequence[int], str] = "auto", @@ -357,14 +461,18 @@ def _make_config( """ config_builder = ChipFactory(device).get_config_builder() - has_dvs_layer = isinstance(self.dynapcnn_layers[0]['layer'], DVSLayer) + has_dvs_layer = isinstance(self._dynapcnn_layers[0]['layer'], DVSLayer) if chip_layers_ordering == "auto": # figure out mapping of each DynapcnnLayer into one core. chip_layers_ordering = config_builder.get_valid_mapping(self) + # update the `assigned_core` property of each DynapcnnLayer instance. + for dcnnl_index, dcnnl_data in self._dynapcnn_layers.items(): + self._forward_map[dcnnl_index].assigned_core = dcnnl_data['core_idx'] + else: - # mapping from each DynapcnnLayer into cores has been provided. + # TODO - mapping from each DynapcnnLayer into cores has been provided by the user: NOT IMPLEMENTED YET. if has_dvs_layer: pass # TODO not handling DVSLayer yet. @@ -378,12 +486,12 @@ def _make_config( monitor_chip_layers = [] if monitor_layers is None: # check if any monitoring is enabled (if not, enable monitoring for the last layer). - for _, dcnnl_data in self.dynapcnn_layers.items(): + for _, dcnnl_data in self._dynapcnn_layers.items(): if len(dcnnl_data['destinations']) == 0: monitor_chip_layers.append(dcnnl_data['core_idx']) break elif monitor_layers == "all": - for _, dcnnl_data in self.dynapcnn_layers.items(): + for _, dcnnl_data in self._dynapcnn_layers.items(): # monitor each chip core (if not a DVSLayer). if not isinstance(dcnnl_data['layer'], DVSLayer): monitor_chip_layers.append(dcnnl_data['core_idx']) @@ -412,18 +520,18 @@ def _get_sinabs_edges_and_modules(self) -> Tuple[List[Tuple[int, int]], Dict[int remapped to connect the nodes involved in the merging directly. sinabs_modules_map (dict): a dict containing the nodes of the graph (described now by `edges_without_merge`) as `key` and their associated module as `value`. - remapped_nodes (dict): a dict where `key` is the original node name (as extracted by `self.graph_tracer`) and `value` is + remapped_nodes (dict): a dict where `key` is the original node name (as extracted by `self._graph_tracer`) and `value` is the new node name (after ignored layers have been dropped and `Merge` layers have be processed before being removed). """ # remap `(A, X)` and `(X, B)` into `(A, B)` if `X` is a layer in the original `snn` to be ignored. - sinabs_edges, remapped_nodes = self.graph_tracer.remove_ignored_nodes( + sinabs_edges, remapped_nodes = self._graph_tracer.remove_ignored_nodes( DEFAULT_IGNORED_LAYER_TYPES) # nodes (layers' "names") need remapping in case some layers have been removed (e.g. a `nn.Flattern` is ignored). sinabs_modules_map = {} for orig_name, new_name in remapped_nodes.items(): - sinabs_modules_map[new_name] = self.graph_tracer.modules_map[orig_name] + sinabs_modules_map[new_name] = self._graph_tracer.modules_map[orig_name] # bypass merging layers to connect the nodes involved in them directly to the node where the merge happens. edges_without_merge = merge_handler(sinabs_edges, sinabs_modules_map) @@ -432,10 +540,10 @@ def _get_sinabs_edges_and_modules(self) -> Tuple[List[Tuple[int, int]], Dict[int def _populate_nodes_io(self): """ Loops through the nodes in the original graph to retrieve their I/O tensor shapes and add them to their respective - representations in `self.nodes_to_dcnnl_map`.""" + representations in `self._nodes_to_dcnnl_map`.""" def find_original_node_name(name_mapper: dict, node: int): - """ Find what a node is originally named when built in `self.graph_tracer`. """ + """ Find what a node is originally named when built in `self._graph_tracer`. """ for orig_name, new_name in name_mapper.items(): if new_name == node: return orig_name @@ -463,32 +571,54 @@ def find_my_input(edges_list: list, node: int) -> int: return edge[0] return -1 - # access the I/O shapes for each node in `self.sinabs_edges` from the original graph in `self.graph_tracer`. - for dcnnl_idx, dcnnl_data in self.nodes_to_dcnnl_map.items(): + # access the I/O shapes for each node in `self._sinabs_edges` from the original graph in `self._graph_tracer`. + for dcnnl_idx, dcnnl_data in self._nodes_to_dcnnl_map.items(): for node, node_data in dcnnl_data.items(): # node dictionary with layer data. if isinstance(node, int): # some nodes might have been renamed (e.g. after droppping a `nn.Flatten`), so find how node was originally named. - orig_name = find_original_node_name(self.nodes_name_remap, node) - _in, _out = self.graph_tracer.get_node_io_shapes(orig_name) + orig_name = find_original_node_name(self._nodes_name_remap, node) + _in, _out = self._graph_tracer.get_node_io_shapes(orig_name) # update node I/O shape in the mapper (drop batch dimension). if node != 0: # Find node outputing into the current node being processed (this will be the input shape). This is # necessary cuz if a node originally receives input from a `nn.Flatten` for instance, when mapped into # a `DynapcnnLayer` it will be receiving the input from a privious `sl.SumPool2d`. - input_node = find_my_input(self.sinabs_edges, node) + input_node = find_my_input(self._sinabs_edges, node) if input_node == -1: # node does not have an input source within the graph (it consumes the original input to the model). node_data['input_shape'] = tuple(list(_in)[1:]) else: # input comes from another node in the graph. - input_node_orig_name = find_original_node_name(self.nodes_name_remap, input_node) - _, _input_source_shape = self.graph_tracer.get_node_io_shapes(input_node_orig_name) + input_node_orig_name = find_original_node_name(self._nodes_name_remap, input_node) + _, _input_source_shape = self._graph_tracer.get_node_io_shapes(input_node_orig_name) node_data['input_shape'] = tuple(list(_input_source_shape)[1:]) else: # first node does not have an input source within the graph. node_data['input_shape'] = tuple(list(_in)[1:]) - node_data['output_shape'] = tuple(list(_out)[1:]) \ No newline at end of file + node_data['output_shape'] = tuple(list(_out)[1:]) + + def _to_device(self, device: torch.device) -> None: + """ .""" + for layer in self._forward_map.values(): + if isinstance(layer, sinabs.backend.dynapcnn.dynapcnn_layer_new.DynapcnnLayer): + layer.conv_layer.to(device) + layer.spk_layer.to(device) + + # if there's more than one pooling each of them becomes a node that is catched by the `else` statement. + if len(layer.pool_layer) == 1: + layer.pool_layer[0].to(device) + else: + # this nodes are created from `DynapcnnLayer`s that have multiple poolings (each pooling becomes a new node). + layer.to(device) + + def __str__(self): + pretty_print = '' + for idx, layer_data in self._forward_map.items(): + pretty_print += f'----------------------- [ DynapcnnLayer {idx} ] -----------------------\n' + pretty_print += f'{layer_data}\n\n' + + return pretty_print \ No newline at end of file diff --git a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py index 75179186..5d3d471a 100644 --- a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py +++ b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py @@ -9,10 +9,9 @@ import sinabs import sinabs.layers as sl -class DynapcnnNetworkModule(nn.Module): +class DynapcnnNetworkModule(): """ - Uses the set of `DynapcnnLayer` instances that have been configured to the chip and how they address each other - to define what the `forward` method of the model should do. + Uses the set of `DynapcnnLayer` instances and how they address each other to define what the `forward` method of the model should do. Parameters ---------- @@ -22,12 +21,11 @@ class DynapcnnNetworkModule(nn.Module): destination layers, etc.). """ - def __init__(self, dcnnl_edges: List[Tuple[int, int]], dynapcnn_layers: Dict) -> nn.Module: - super().__init__() + def __init__(self, dcnnl_edges: List[Tuple[int, int]], dynapcnn_layers: Dict): self.dcnnl_edges = dcnnl_edges - self.forward_map, self.merge_points = self._build_module_forward_from_graph_v2(dcnnl_edges, dynapcnn_layers) + self.forward_map, self.merge_points = self._build_module_forward_from_graph(dcnnl_edges, dynapcnn_layers) def _spot_merging_points(self, dcnnl_edges: list) -> dict: """ . """ @@ -51,9 +49,8 @@ def _spot_merging_points(self, dcnnl_edges: list) -> dict: raise ValueError(f'Node {trg_node} is the has fan-in of {fan_in}: only fan-in of 2 is currently handled.') return nodes_with_merge_input - - def _build_module_forward_from_graph_v2(self, dcnnl_edges: list, dynapcnn_layers: dict) -> Union[dict, dict]: + def _build_module_forward_from_graph(self, dcnnl_edges: list, dynapcnn_layers: dict) -> Union[dict, dict]: """ .""" # mapper to flag nodes that need input from a `Merge` layer. @@ -72,108 +69,4 @@ def _build_module_forward_from_graph_v2(self, dcnnl_edges: list, dynapcnn_layers if trg_dcnnl not in forward_map: forward_map[trg_dcnnl] = copy.deepcopy(dynapcnn_layers[trg_dcnnl]['layer']) - return forward_map, merge_points - - def forward(self, x): - """ .""" - - layers_outputs = {} - - # TODO - currently `node 0` (this 1st node in the 1st edge of `self.dcnnl_edges`) is always taken to be the - # input node of the network. This won't work in cases where there are more the one input nodes to the network - # so this functionality needs some refactoring. - self.forward_map[self.dcnnl_edges[0][0]](x) - - # forward the input `x` through the input `DynapcnnLayer` in the `DynapcnnNetwork`s graph (1st node in the 1st edge in `self.dcnnl_edges`). - layers_outputs[self.dcnnl_edges[0][0]] = self.forward_map[self.dcnnl_edges[0][0]](x) - - # propagate outputs in `layers_outputs` through the rest of the nodes of `self.dcnnl_edges`. - for edge in self.dcnnl_edges: - - # target DynapcnnLayer (will consume tensors from `layers_outputs`). - trg_dcnnl = edge[1] - - if trg_dcnnl in self.merge_points and trg_dcnnl not in layers_outputs: - # by this points the arguments of the `Merge` associated with `trg_dcnnl` should have been computed. - arg1, arg2 = self.merge_points[trg_dcnnl]['sources'] - - # find which returned tensor from the `forward` call of DynapcnnLayers `arg1` and `arg2` are to be fed - # to the target DynapcnnLayer `trg_dcnnl`. - return_index_arg1 = self.forward_map[arg1].get_destination_dcnnl_index(trg_dcnnl) - return_index_arg2 = self.forward_map[arg2].get_destination_dcnnl_index(trg_dcnnl) - - # retrieve input tensors to `Merge`. - _arg1 = layers_outputs[arg1][return_index_arg1] - _arg2 = layers_outputs[arg2][return_index_arg2] - - # merge tensors. - merge_output = self.merge_points[trg_dcnnl]['merge'](_arg1, _arg2) - - # call the forward. - layers_outputs[trg_dcnnl] = self.forward_map[trg_dcnnl](merge_output) - - elif trg_dcnnl not in layers_outputs: - # input source for `trg_dcnnl`. - src_dcnnl = edge[0] - - # find which returned tensor from the `forward` call of the source DynapcnnLayer `src_dcnnl` is to be fed - # to the target DynapcnnLayer `trg_dcnnl`. - return_index = self.forward_map[src_dcnnl].get_destination_dcnnl_index(trg_dcnnl) - - # call the forward. - layers_outputs[trg_dcnnl] = self.forward_map[trg_dcnnl](layers_outputs[src_dcnnl][return_index]) - - else: - - pass - - # TODO - this assumes the network has a single output node. - # last computed is the output layer. - return layers_outputs[trg_dcnnl][0] - - def parameters(self) -> list: - """ Gathers all the parameters of the network in a list. This is done by accessing the convolutional layer in each `DynapcnnLayer`, calling - its `.parameters` method and saving it to a list. - - Note: the method assumes no biases are used. - - Returns - ---------- - parameters (list): a list of parameters of all convolutional layers in the `DynapcnnNetwok`. - """ - parameters = [] - - for layer in self.forward_map.values(): - if isinstance(layer, sinabs.backend.dynapcnn.dynapcnn_layer_new.DynapcnnLayer): - parameters.extend(layer.conv_layer.parameters()) - - return parameters - - def init_weights(self, init_fn: nn.init = nn.init.xavier_normal_) -> None: - """ Call the weight initialization method `init_fn` on each `DynapcnnLayer.conv_layer.weight.data` in the `DynapcnnNetwork` instance.""" - for layer in self.forward_map.values(): - if isinstance(layer, sinabs.backend.dynapcnn.dynapcnn_layer_new.DynapcnnLayer): - init_fn(layer.conv_layer.weight.data) - - def to(self, device) -> None: - """ .""" - for layer in self.forward_map.values(): - if isinstance(layer, sinabs.backend.dynapcnn.dynapcnn_layer_new.DynapcnnLayer): - layer.conv_layer.to(device) - layer.spk_layer.to(device) - - # if there's more than one pooling each of them becomes a node that is catched by the `else` statement. - if len(layer.pool_layer) == 1: - layer.pool_layer[0].to(device) - else: - # this nodes are created from `DynapcnnLayer`s that have multiple poolings (each pooling becomes a new node). - layer.to(device) - - def detach_neuron_states(self) -> None: - """ Detach the neuron states and activations from current computation graph (necessary). """ - - for module in self.forward_map.values(): - if isinstance(module, sinabs.backend.dynapcnn.dynapcnn_layer_new.DynapcnnLayer): - if isinstance(module.spk_layer, sl.StatefulLayer): - for name, buffer in module.spk_layer.named_buffers(): - buffer.detach_() \ No newline at end of file + return forward_map, merge_points \ No newline at end of file diff --git a/sinabs/backend/dynapcnn/utils.py b/sinabs/backend/dynapcnn/utils.py index cc16d9af..47f79f0a 100644 --- a/sinabs/backend/dynapcnn/utils.py +++ b/sinabs/backend/dynapcnn/utils.py @@ -645,7 +645,7 @@ def construct_dynapcnnlayers_from_mapper( for dpcnnl_idx, dcnnl_data in nodes_to_dcnnl_map.items(): # create a `DynapcnnLayer` from the set of layers in `dcnnl_data`. dynapcnnlayer = construct_dynapcnnlayer( - discretize, dcnnl_data, edges, nodes_to_dcnnl_map) + dpcnnl_idx, discretize, dcnnl_data, edges, nodes_to_dcnnl_map) dynapcnn_layers[dpcnnl_idx] = { 'layer': dynapcnnlayer, @@ -675,6 +675,7 @@ def update_nodes_io(updated_node: int, output_shape: tuple, nodes_to_dcnnl_map: val['input_shape'] = output_shape def construct_dynapcnnlayer( + dpcnnl_idx: int, discretize: bool, dcnnl_data: dict, edges: List[Tuple[int, int]], @@ -692,6 +693,7 @@ def construct_dynapcnnlayer( # instantiate a DynapcnnLayer from the data in 'dcnnl_data'. dynapcnnlayer = DynapcnnLayer( + dpcnnl_index = dpcnnl_idx, dcnnl_data = dcnnl_data, discretize = discretize, sinabs_edges = edges From 7a150c034a8d89bf5c821e140f59ebfc115b19f4 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Fri, 17 May 2024 14:30:57 +0200 Subject: [PATCH 090/379] Refacor DynapcnnNetwork duplicated func. collapsed into a single methods + data for DynapcnnNetwork functionality gathered in the last three properties created in the constructor (temp. private properties freed) --- sinabs/backend/dynapcnn/chips/dynapcnn.py | 52 ++--- sinabs/backend/dynapcnn/config_builder.py | 9 +- sinabs/backend/dynapcnn/dynapcnn_layer_new.py | 180 ++++++++------- .../dynapcnn/dynapcnn_network_graph.py | 215 +++++++----------- sinabs/backend/dynapcnn/mapping.py | 14 +- 5 files changed, 213 insertions(+), 257 deletions(-) diff --git a/sinabs/backend/dynapcnn/chips/dynapcnn.py b/sinabs/backend/dynapcnn/chips/dynapcnn.py index 179337a5..3559ee8b 100644 --- a/sinabs/backend/dynapcnn/chips/dynapcnn.py +++ b/sinabs/backend/dynapcnn/chips/dynapcnn.py @@ -207,30 +207,28 @@ def write_dynapcnn_layer_config( raise TypeError(f"Unexpected parameter {param} or value. {e}") @classmethod - def write_dynapcnn_layer_config_graph(cls, dcnnl_data: dict, chip_layer: "CNNLayerConfig", dynapcnn_layers: dict): - """ Uses the data in `dcnnl_data` to configure a `CNNLayerConfig` to be deployed on chip. + def write_dynapcnn_layer_config_graph(cls, dcnnl: DynapcnnLayer, chip_layer: "CNNLayerConfig", forward_map: dict) -> None: + """ Uses the data in `dcnnl` to configure a `CNNLayerConfig` to be deployed on chip. Parameters ---------- - dcnnl_data: - contains the DynapcnnLayer (`dcnnl_data['layer']`), is list of destination DynapcnnLayer indexes - (`dcnnl_data['destinations']`), and the core ID it is to be mapped to (`dcnnl_data['core_idx']`). - chip_layer: - a `CNNLayerConfig` (indexed by `dcnnl_data['core_idx']`) used to represent the DynapcnnLayer - in `dcnnl_data['layer']`. - dynapcnn_layers: - a dictionary with keys being the ID of each DynapcnnLayer and values being the dictionary with the - `dcnnl_data` structure described above. This is used to retrieve the `core_idx` for each of the - layers in `dcnnl_data['destinations']` such that `chip_layer.destinations` can be configured. + dcnnl (DynapcnnLayer): the layer for which the condiguration will be written. + chip_layer (CNNLayerConfig): used to represent/configure `dcnnl` onto the chip. + forward_map (dict): a dictionary with keys being the ID of each DynapcnnLayer and values being the layer + itself. This is used to retrieve the `.assigned_core` for each of the layers in `.dynapcnnlayer_destination` + such that `chip_layer.destinations` can be configured. """ # extracting from a DynapcnnLayer the config. variables for its CNNLayerConfig. - config_dict = dcnnl_data['layer'].get_layer_config_dict() + config_dict = dcnnl.get_layer_config_dict() # use core indexing instead of DynapcnnLayer indexing for destinations. for dest_config in config_dict['destinations']: dcnnl_idx = dest_config['layer'] - dcnnl_core_idx = dynapcnn_layers[dcnnl_idx]['core_idx'] # get the core the destination DynapcnnLayer is using. + + # get the core the destination DynapcnnLayer is using. + dcnnl_core_idx = forward_map[dcnnl_idx].assigned_core + dest_config['layer'] = dcnnl_core_idx # set the destinations configuration. @@ -299,25 +297,27 @@ def build_config(cls, model: Union["DynapcnnNetwork", "DynapcnnNetworkGraph"], c config.dvs_layer.pass_sensor_events = False elif type(model) == sinabs.backend.dynapcnn.dynapcnn_network_graph.DynapcnnNetworkGraph: - """ loops through `DynapcnnNetworkGraph.dynapcnn_layers`, where each represented layer representation constains their - core ID to be loaded onto and their target destinations. Each `layer_data` has all the info. necessary to config. + """ Loops through `DynapcnnNetworkGraph._forward_map`, containing all `DynapcnnLayer`s in the model, their + core ID to be loaded onto and their target destinations. Each `ith_dcnnl` has all the info. necessary to config. their respective `CNNLayerConfig` object. """ - has_dvs_layer = False # TODO DVSLayer not supported yet. + has_dvs_layer = False # TODO DVSLayer not supported yet. - for _, layer_data in model.dynapcnn_layers.items(): - if isinstance(layer_data['layer'], DVSLayer): - pass # TODO DVSLayer not supported yet. + for layer_index, ith_dcnnl in model.forward_map.items(): + if isinstance(ith_dcnnl, DVSLayer): + # TODO DVSLayer not supported yet. + pass - elif isinstance(layer_data['layer'], DynapcnnLayer): - chip_layer = config.cnn_layers[layer_data['core_idx']] - cls.write_dynapcnn_layer_config_graph(layer_data, chip_layer, model.dynapcnn_layers) + elif isinstance(ith_dcnnl, DynapcnnLayer): + chip_layer = config.cnn_layers[ith_dcnnl.assigned_core] + cls.write_dynapcnn_layer_config_graph(ith_dcnnl, chip_layer, model.forward_map) else: - print('[error] ', layer_data['layer']) - raise TypeError("Unexpected layer in the model.") # shouldn't happen since type checks are made previously. + # shouldn't happen since type checks are made previously. + raise TypeError(f"Layer (index {layer_index}) is unexpected in the model: \n{ith_dcnnl}") - if not has_dvs_layer: # TODO DVSLayer not supported yet. + if not has_dvs_layer: + # TODO DVSLayer not supported yet. config.dvs_layer.pass_sensor_events = False else: config.dvs_layer.pass_sensor_events = False diff --git a/sinabs/backend/dynapcnn/config_builder.py b/sinabs/backend/dynapcnn/config_builder.py index a42d1994..0b1954fc 100644 --- a/sinabs/backend/dynapcnn/config_builder.py +++ b/sinabs/backend/dynapcnn/config_builder.py @@ -95,11 +95,14 @@ def get_valid_mapping(cls, model: "DynapcnnNetwork") -> List[int]: elif type(model) == sinabs.backend.dynapcnn.dynapcnn_network_graph.DynapcnnNetworkGraph: mapping = get_valid_mapping(model, cls.get_constraints()) - if isinstance(model.dynapcnn_layers[0]['layer'], DVSLayer): - pass # TODO not handling DVSLayer yet. + if isinstance(model.forward_map[0], DVSLayer): + # TODO not handling DVSLayer yet. + # TODO if the architecture has more than one `DynapcnnLayer`s acting as input node of the model + # thi check will be wrong since it assumes the network has a single input node `model.forward_map[0]`. + pass for (dcnnl, core_idx) in mapping: - model.dynapcnn_layers[dcnnl]['core_idx'] = core_idx + model.forward_map[dcnnl].assigned_core = core_idx else: raise InvalidModel(model) diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer_new.py b/sinabs/backend/dynapcnn/dynapcnn_layer_new.py index 356ab44f..50a1a64b 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer_new.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer_new.py @@ -131,6 +131,8 @@ def __init__( # map destination nodes for each layer in this instance. self.nodes_destinations = self._get_destinations_input_source(sinabs_edges) + ####################################################### Public Methods ####################################################### + def get_destination_dcnnl_index(self, dcnnl_id: int) -> int: """ The `forward` method will return as many tensors as there are elements in `self.dynapcnnlayer_destination`. Since the i-th returned tensor is to be fed to the i-th destionation in `self.dynapcnnlayer_destination`, the return of this method can be used to index a tensor returned in the `forward` method. @@ -144,33 +146,6 @@ def get_destination_dcnnl_index(self, dcnnl_id: int) -> int: The index of `dcnnl_id` within `self.dynapcnnlayer_destination`. """ return self.dynapcnnlayer_destination.index(dcnnl_id) - - def _get_destinations_input_source(self, sinabs_edges: list) -> dict: - """ Creates a mapping between each layer in this `DynapcnnLayer` instance and its targe nodes that are part of different - `DynapcnnLayer` instances. This mapping is used to figure out how many tensors the `forward` method needs to return. - """ - destinations_input_source = {} - - # check whether spiking layer projects outside this DynapcnnLayer (i.e. to one of the destinations without passing through a pooling). - spk_destinations = [] - for edge in sinabs_edges: - if edge[0] == self.spk_node_id and edge[1] not in self.pool_node_id: - # spiking layer projects to a node outside this DynapcnnLayer. - spk_destinations.append(edge[1]) - if len(spk_destinations) > 0: - destinations_input_source[self.spk_node_id] = [] - for node_id in spk_destinations: - destinations_input_source[self.spk_node_id].append(node_id) - - # get `pooling->destination` mapping. The pooling outputs will be arranged sequentially since the pooling layers are added sequentially - # to `self.pool_layer` (i.e., as they appear in the computational graph of the original `nn.Module`). - for id in self.pool_node_id: - destinations_input_source[id] = [] - for edge in sinabs_edges: - if edge[0] == id: - destinations_input_source[id].append(edge[1]) - - return destinations_input_source def forward(self, x): """Torch forward pass. @@ -234,64 +209,6 @@ def forward(self, x): pass return tuple(returns) - - def _convert_linear_to_conv(self, lin: nn.Linear, layer_data: dict) -> nn.Conv2d: - """Convert Linear layer to Conv2d. - - Parameters - ---------- - lin: nn.Linear - Linear layer to be converted - - Returns - ------- - nn.Conv2d - Convolutional layer equivalent to `lin`. - """ - self.lin_to_conv_conversion = True - - input_shape = layer_data['input_shape'] - - in_chan, in_h, in_w = input_shape - - if lin.in_features != in_chan * in_h * in_w: - raise ValueError("Shapes don't match.") - - layer = nn.Conv2d( - in_channels=in_chan, - kernel_size=(in_h, in_w), - out_channels=lin.out_features, - padding=0, - bias=lin.bias is not None, - ) - - if lin.bias is not None: - layer.bias.data = lin.bias.data.clone().detach() - - layer.weight.data = ( - lin.weight.data.clone() - .detach() - .reshape((lin.out_features, in_chan, in_h, in_w)) - ) - - return layer, input_shape - - def _update_conv_node_output_shape(self, conv_layer: nn.Conv2d, layer_data: dict, input_shape: tuple) -> Tuple: - """ The input shapes to nodes are extracted using a list of edges by finding the output shape of the 1st element - in the edge and setting it as the input shape to the 2nd element in the edge. If a node used to be a `nn.Linear` - and it became a `nn.Conv2d`, output shape in the mapper needs to be updated, otherwise there will be a mismatch - between its output and the input it provides to another node. - """ - layer_data['output_shape'] = self.get_conv_output_shape(conv_layer, input_shape) - - return layer_data['output_shape'] - - def _update_neuron_node_output_shape(self, layer_data: dict, input_shape: tuple) -> None: - """ Following the conversion of a `nn.Linear` into a `nn.Conv2d` the neuron layer in the - sequence also needs its I/O shapes uodated. - """ - layer_data['input_shape'] = input_shape - layer_data['output_shape'] = layer_data['input_shape'] def get_modified_node_it(self, dcnnl_data: dict) -> Union[Tuple[int, tuple], Tuple[None, None]]: """ .""" @@ -340,7 +257,9 @@ def summary(self) -> dict: } def get_layer_config_dict(self) -> dict: - """ Returns a dict containing the properties required to configure a `CNNLayerConfig` instance.""" + """ Returns a dict containing the properties required to configure a `CNNLayerConfig` instance that + will map this DynapcnnLayer onto the chip. + """ config_dict = {} # configures `CNNLayerConfig.dimensions` (instance of `CNNLayerDimensions`). @@ -481,4 +400,91 @@ def __str__(self): for node, destinations in self.nodes_destinations.items(): pretty_print += f'\n> node {node} feeds input to nodes {destinations}' - return pretty_print \ No newline at end of file + return pretty_print + + ####################################################### Private Methods ####################################################### + + def _update_neuron_node_output_shape(self, layer_data: dict, input_shape: tuple) -> None: + """ Following the conversion of a `nn.Linear` into a `nn.Conv2d` the neuron layer in the + sequence also needs its I/O shapes uodated. + """ + layer_data['input_shape'] = input_shape + layer_data['output_shape'] = layer_data['input_shape'] + + def _update_conv_node_output_shape(self, conv_layer: nn.Conv2d, layer_data: dict, input_shape: tuple) -> Tuple: + """ The input shapes to nodes are extracted using a list of edges by finding the output shape of the 1st element + in the edge and setting it as the input shape to the 2nd element in the edge. If a node used to be a `nn.Linear` + and it became a `nn.Conv2d`, output shape in the mapper needs to be updated, otherwise there will be a mismatch + between its output and the input it provides to another node. + """ + layer_data['output_shape'] = self.get_conv_output_shape(conv_layer, input_shape) + + return layer_data['output_shape'] + + def _convert_linear_to_conv(self, lin: nn.Linear, layer_data: dict) -> nn.Conv2d: + """Convert Linear layer to Conv2d. + + Parameters + ---------- + lin: nn.Linear + Linear layer to be converted + + Returns + ------- + nn.Conv2d + Convolutional layer equivalent to `lin`. + """ + self.lin_to_conv_conversion = True + + input_shape = layer_data['input_shape'] + + in_chan, in_h, in_w = input_shape + + if lin.in_features != in_chan * in_h * in_w: + raise ValueError("Shapes don't match.") + + layer = nn.Conv2d( + in_channels=in_chan, + kernel_size=(in_h, in_w), + out_channels=lin.out_features, + padding=0, + bias=lin.bias is not None, + ) + + if lin.bias is not None: + layer.bias.data = lin.bias.data.clone().detach() + + layer.weight.data = ( + lin.weight.data.clone() + .detach() + .reshape((lin.out_features, in_chan, in_h, in_w)) + ) + + return layer, input_shape + + def _get_destinations_input_source(self, sinabs_edges: list) -> dict: + """ Creates a mapping between each layer in this `DynapcnnLayer` instance and its targe nodes that are part of different + `DynapcnnLayer` instances. This mapping is used to figure out how many tensors the `forward` method needs to return. + """ + destinations_input_source = {} + + # check whether spiking layer projects outside this DynapcnnLayer (i.e. to one of the destinations without passing through a pooling). + spk_destinations = [] + for edge in sinabs_edges: + if edge[0] == self.spk_node_id and edge[1] not in self.pool_node_id: + # spiking layer projects to a node outside this DynapcnnLayer. + spk_destinations.append(edge[1]) + if len(spk_destinations) > 0: + destinations_input_source[self.spk_node_id] = [] + for node_id in spk_destinations: + destinations_input_source[self.spk_node_id].append(node_id) + + # get `pooling->destination` mapping. The pooling outputs will be arranged sequentially since the pooling layers are added sequentially + # to `self.pool_layer` (i.e., as they appear in the computational graph of the original `nn.Module`). + for id in self.pool_node_id: + destinations_input_source[id] = [] + for edge in sinabs_edges: + if edge[0] == id: + destinations_input_source[id].append(edge[1]) + + return destinations_input_source \ No newline at end of file diff --git a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py index adc82016..5ec49c62 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py @@ -1,6 +1,9 @@ -# functionality : ... -# author : Willian Soares Girao -# contact : williansoaresgirao@gmail.com +""" +functionality : extracts the computational graph of a network defined as a `nn.Module` and converts it into a set of `DynapcnnLayer`s + that implement a network ()`DynapcnnNetwork`) instance that can be deployed to a Speck chip. +author : Willian Soares Girao +contact : williansoaresgirao@gmail.com +""" import time from typing import List, Optional, Sequence, Tuple, Union, Dict @@ -27,8 +30,6 @@ from .dynapcnnnetwork_module import DynapcnnNetworkModule -# TODO `make_config` and `_make_config` should be merged into a single method. - class DynapcnnNetworkGraph(nn.Module): def __init__( self, @@ -43,8 +44,10 @@ def __init__( make the dynapcnn configuration and upload it to DYNAPCNN. Some of the properties defined within the class constructor are meant to be temporary data structures handling the conversion - of the `snn` (the original `nn.Module`) into a set of `DynapcnnLayer`s composing a `DynapcnnNetwork` instance. Thus, the following - private properties are delted once a successfull call to `self.to(device='speck...')` is made: + of the `snn` (the original `nn.Module`) into a set of `DynapcnnLayer`s composing a `DynapcnnNetwork` instance. Once their role + in preprocessing `snn` is finished, all required data to train/deploy the `DynapcnnNetwork` instance is within `self._dcnnl_edges` + (the connectivity between each `DynapcnnLayer`/core), `self._forward_map` (every `DynapcnnLayer` in the network) and `self._merge_points` + (the `DynapcnnLayer`s that need a `Merge` input). Thus, the following private properties are delted as last step of the constructor: - self._graph_tracer - self._sinabs_edges @@ -55,10 +58,10 @@ def __init__( Parameters ---------- - snn : a `nn.Module` implementing a spiking network. - input_shape: a description of the input dimensions (features, height, width). - dvs_input: wether or not dynapcnn receive input from its DVS camera. - discretize: If `True`, discretize the parameters and thresholds. This is needed for uploading + snn (nn.Module): a implementing a spiking network. + input_shape (tuple): a description of the input dimensions as `(features, height, width)`. + dvs_input (bool): wether or not dynapcnn receive input from its DVS camera. + discretize (bool): If `True`, discretize the parameters and thresholds. This is needed for uploading weights to dynapcnn. Set to `False` only for testing purposes. """ super().__init__() @@ -98,14 +101,20 @@ def __init__( # these gather all data necessay to implement the forward method for this class. self._dcnnl_edges, self._forward_map, self._merge_points = self._get_network_module() - ####################################################### Public Methods ####################################################### + # all necessary `DynapcnnLayer` data held in `self._forward_map`: removing intermediary data structures no longer necessary. + del self._graph_tracer + del self._sinabs_edges + del self._sinabs_modules_map + del self._nodes_name_remap + del self._nodes_to_dcnnl_map + del self._dynapcnn_layers + ####################################################### Public Methods ####################################################### + @property - def dynapcnn_layers(self): - if hasattr(self, '_dynapcnn_layers'): - return self._dynapcnn_layers - else: - return None + def forward_map(self) -> dict: + """ This dictionary contains each `DynapcnnLayer` in the model indexed by their ID (layer index). """ + return self._forward_map def forward(self, x): """ .""" @@ -251,7 +260,7 @@ def to( if device_name in ChipFactory.supported_devices: # generate config. - config = self.make_config( + config = self._make_config( chip_layers_ordering=chip_layers_ordering, device=device, monitor_layers=monitor_layers, @@ -307,7 +316,9 @@ def to( else: raise Exception("Unknown device description.") - def make_config( + ####################################################### Private Methods ####################################################### + + def _make_config( self, chip_layers_ordering: Union[Sequence[int], str] = "auto", device="dynapcnndevkit:0", @@ -355,30 +366,61 @@ def make_config( ValueError If the generated configuration is not valid for the specified device. """ - config, is_compatible = self._make_config( - chip_layers_ordering=chip_layers_ordering, - device=device, - monitor_layers=monitor_layers, - config_modifier=config_modifier, - ) - - if is_compatible: + config_builder = ChipFactory(device).get_config_builder() + + # TODO not handling DVSLayer yet. + has_dvs_layer = isinstance(self._forward_map[0], DVSLayer) + + if chip_layers_ordering == "auto": + # figure out mapping of each DynapcnnLayer into one core. + chip_layers_ordering = config_builder.get_valid_mapping(self) + + else: + # TODO - mapping from each DynapcnnLayer into cores has been provided by the user: NOT IMPLEMENTED YET. + if has_dvs_layer: + # TODO not handling DVSLayer yet. + pass + + # update config. + config = config_builder.build_config(self, None) + + # TODO not handling DVSLayer yet (this is from the old implementation, should be revised). + if self.input_shape and self.input_shape[0] == 1: + config.dvs_layer.merge = True + + # TODO all this monitoring part needs validation still. + monitor_chip_layers = [] + if monitor_layers is None: + # check if any monitoring is enabled (if not, enable monitoring for the last layer). + for dcnnl_index, ith_dcnnl in self._forward_map.items(): + if len(ith_dcnnl.dynapcnnlayer_destination) == 0: + monitor_chip_layers.append(ith_dcnnl.assigned_core) + break + elif monitor_layers == "all": + for dcnnl_index, ith_dcnnl in self._forward_map.items(): + # TODO not handling DVSLayer yet + # monitor each chip core (if not a DVSLayer). + if not isinstance(ith_dcnnl, DVSLayer): + monitor_chip_layers.append(ith_dcnnl.assigned_core) + + if monitor_layers: + if "dvs" in monitor_layers: + monitor_chip_layers.append("dvs") + + # enable monitors on the specified layers. + config_builder.monitor_layers(config, monitor_chip_layers) + + if config_modifier is not None: + # apply user config modifier. + config = config_modifier(config) + + if config_builder.validate_configuration(config): # validate config. print("Network is valid: \n") - - # DynapcnnLayer have been configured: removing intermediary data structures no longer necessary. - del self._graph_tracer - del self._sinabs_edges - del self._sinabs_modules_map - del self._nodes_name_remap - del self._nodes_to_dcnnl_map - del self._dynapcnn_layers return config else: raise ValueError(f"Generated config is not valid for {device}") - - ####################################################### Private Methods ####################################################### def _get_network_module(self) -> Union[list, dict, dict]: """ Uses the `DynapcnnLayer` instances in `self._dynapcnn_layers` and the connectivity between them to create three data structures @@ -409,105 +451,6 @@ def _get_dynapcnnlayers_edges(self) -> List[Tuple[int, int]]: dcnnl_edges.append((dcnnl_idx, dest)) return dcnnl_edges - - def _make_config( - self, - chip_layers_ordering: Union[Sequence[int], str] = "auto", - device="dynapcnndevkit:0", - monitor_layers: Optional[Union[List, str]] = None, - config_modifier=None, - ) -> Tuple["SamnaConfiguration", bool]: - """Prepare and output the `samna` configuration for this network. - - Parameters - ---------- - - chip_layers_ordering: sequence of integers or `auto` - The order in which the dynapcnn layers will be used. If `auto`, - an automated procedure will be used to find a valid ordering. - A list of layers on the device where you want each of the model's DynapcnnLayers to be placed. - The index of the core on chip to which the i-th layer in the model is mapped is the value of the i-th entry in the list. - Note: This list should be the same length as the number of dynapcnn layers in your model. - - device: String - dynapcnndevkit, speck2b or speck2devkit - - monitor_layers: None/List/Str - A list of all layers in the module that you want to monitor. Indexing starts with the first non-dvs layer. - If you want to monitor the dvs layer for eg. - :: - - monitor_layers = ["dvs"] # If you want to monitor the output of the pre-processing layer - monitor_layers = ["dvs", 8] # If you want to monitor preprocessing and layer 8 - monitor_layers = "all" # If you want to monitor all the layers - - If this value is left as None, by default the last layer of the model is monitored. - - config_modifier: - A user configuration modifier method. - This function can be used to make any custom changes you want to make to the configuration object. - - Returns - ------- - Configuration object - Object defining the configuration for the device - Bool - True if the configuration is valid for the given device. - - Raises - ------ - ImportError - If samna is not available. - """ - config_builder = ChipFactory(device).get_config_builder() - - has_dvs_layer = isinstance(self._dynapcnn_layers[0]['layer'], DVSLayer) - - if chip_layers_ordering == "auto": - # figure out mapping of each DynapcnnLayer into one core. - chip_layers_ordering = config_builder.get_valid_mapping(self) - - # update the `assigned_core` property of each DynapcnnLayer instance. - for dcnnl_index, dcnnl_data in self._dynapcnn_layers.items(): - self._forward_map[dcnnl_index].assigned_core = dcnnl_data['core_idx'] - - else: - # TODO - mapping from each DynapcnnLayer into cores has been provided by the user: NOT IMPLEMENTED YET. - if has_dvs_layer: - pass # TODO not handling DVSLayer yet. - - # update config. - config = config_builder.build_config(self, None) - - if self.input_shape and self.input_shape[0] == 1: # TODO not handling DVSLayer yet (this is from the old implementation, should be revised). - config.dvs_layer.merge = True - - # TODO all this monitoring part needs validation still. - monitor_chip_layers = [] - if monitor_layers is None: - # check if any monitoring is enabled (if not, enable monitoring for the last layer). - for _, dcnnl_data in self._dynapcnn_layers.items(): - if len(dcnnl_data['destinations']) == 0: - monitor_chip_layers.append(dcnnl_data['core_idx']) - break - elif monitor_layers == "all": - for _, dcnnl_data in self._dynapcnn_layers.items(): - # monitor each chip core (if not a DVSLayer). - if not isinstance(dcnnl_data['layer'], DVSLayer): - monitor_chip_layers.append(dcnnl_data['core_idx']) - - if monitor_layers: - if "dvs" in monitor_layers: - monitor_chip_layers.append("dvs") - - # enable monitors on the specified layers. - config_builder.monitor_layers(config, monitor_chip_layers) - - if config_modifier is not None: - # apply user config modifier. - config = config_modifier(config) - - return config, config_builder.validate_configuration(config) def _get_sinabs_edges_and_modules(self) -> Tuple[List[Tuple[int, int]], Dict[int, nn.Module], Dict[int, int]]: """ The computational graph extracted from `snn` might contain layers that are ignored (e.g. a `nn.Flatten` will be diff --git a/sinabs/backend/dynapcnn/mapping.py b/sinabs/backend/dynapcnn/mapping.py index 1015e80c..97ed0f5b 100644 --- a/sinabs/backend/dynapcnn/mapping.py +++ b/sinabs/backend/dynapcnn/mapping.py @@ -71,18 +71,22 @@ def get_valid_mapping( graph = make_flow_graph(layer_mapping, len(constraints)) - new_graph = edmonds(graph, 0, len(graph) - 1) # use graph algorithm to find suitable cores for each DynapcnnLayer. + # use graph algorithm to find suitable cores for each DynapcnnLayer. + new_graph = edmonds(graph, 0, len(graph) - 1) netmap = recover_mapping(new_graph, layer_mapping) elif type(model) == sinabs.backend.dynapcnn.dynapcnn_network_graph.DynapcnnNetworkGraph: - for _, layer_data in model.dynapcnn_layers.items(): - if isinstance(layer_data['layer'], DynapcnnLayer): - layer_mapping.append(find_chip_layers(layer_data['layer'], constraints)) + for dcnnl_index, ith_dcnnl in model.forward_map.items(): + if isinstance(ith_dcnnl, DynapcnnLayer): + layer_mapping.append(find_chip_layers(ith_dcnnl, constraints)) + else: + raise ValueError(f'Layer {dcnnl_index} is not an instance of `DynapcnnLayer`.') graph = make_flow_graph(layer_mapping, len(constraints)) - new_graph = edmonds(graph, 0, len(graph) - 1) # use graph algorithm to find suitable cores for each DynapcnnLayer. + # use graph algorithm to find suitable cores for each DynapcnnLayer. + new_graph = edmonds(graph, 0, len(graph) - 1) netmap = recover_mapping(new_graph, layer_mapping) From 3dd7f065a6660782b5999645e036c5fc8a7e9674 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Fri, 17 May 2024 14:55:47 +0200 Subject: [PATCH 091/379] Refactor DynapcnnNetwork batch size passed as argument to the class constructor --- sinabs/backend/dynapcnn/dynapcnn_network_graph.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py index 5ec49c62..a9ef5f02 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py @@ -35,6 +35,7 @@ def __init__( self, snn: nn.Module, input_shape: Tuple[int, int, int], + batch_size: int, dvs_input: bool = False, discretize: bool = True ): @@ -78,7 +79,7 @@ def __init__( # TODO - bacth size must be passed as argument. self._graph_tracer = NIRtoDynapcnnNetworkGraph( snn, - torch.randn((1, *self.input_shape))) # needs the batch dimension. + torch.randn((batch_size, *self.input_shape))) # needs the batch dimension. self._sinabs_edges, \ self._sinabs_modules_map, \ From 5c32778dc491245f365b32f29b85127442895c04 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Fri, 17 May 2024 15:06:21 +0200 Subject: [PATCH 092/379] Refactor deplyment example #1 update how the forward call to the method is made --- .../DynapcnnNetwork-example_1.ipynb | 344 ++++++++++-------- 1 file changed, 183 insertions(+), 161 deletions(-) diff --git a/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_1.ipynb b/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_1.ipynb index 8b085ac9..6811c8bf 100644 --- a/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_1.ipynb +++ b/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_1.ipynb @@ -11,7 +11,8 @@ "import torch\n", "import torch.nn as nn\n", "from sinabs.backend.dynapcnn import DynapcnnNetworkGraph\n", - "from sinabs.layers import Merge, IAFSqueeze" + "from sinabs.layers import Merge, IAFSqueeze\n", + "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential" ] }, { @@ -24,7 +25,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 2, @@ -47,6 +48,7 @@ "channels = 2\n", "height = 34\n", "width = 34\n", + "batch_size = 3\n", "\n", "input_shape = (channels, height, width)" ] @@ -73,23 +75,23 @@ " super().__init__()\n", "\n", " self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False) # node 0\n", - " self.iaf1 = IAFSqueeze(batch_size=1) # node 1\n", + " self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) # node 1\n", " self.pool1 = nn.AvgPool2d(3,3) # node 2\n", " self.pool1a = nn.AvgPool2d(4,4) # node 3\n", "\n", " self.conv2 = nn.Conv2d(10, 10, 4, 1, bias=False)# node 4\n", - " self.iaf2 = IAFSqueeze(batch_size=1) # node 6\n", + " self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) # node 6\n", "\n", " self.conv3 = nn.Conv2d(10, 1, 2, 1, bias=False) # node 8\n", - " self.iaf3 = IAFSqueeze(batch_size=1) # node 9\n", + " self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) # node 9\n", "\n", " self.flat = nn.Flatten()\n", "\n", " self.fc1 = nn.Linear(49, 500, bias=False) # node 10\n", - " self.iaf4 = IAFSqueeze(batch_size=1) # node 11\n", + " self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) # node 11\n", " \n", " self.fc2 = nn.Linear(500, 10, bias=False) # node 12\n", - " self.iaf5 = IAFSqueeze(batch_size=1) # node 13\n", + " self.iaf5 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) # node 13\n", "\n", " self.adder = Merge()\n", "\n", @@ -127,94 +129,6 @@ "snn = SNN()" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The following exemplifies how each of the layers in the SNN should be grouped together to form a `DynapcnnLayer` instance. Let's check the shapes of the tensors each of the (original) layers in the model inputs and outputs by feeding a fake input through the model:" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "metadata": {} - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "DynapcnnLayer 0: ... [1, 2, 34, 34]\n", - " conv1: [1, 10, 33, 33]\n", - " iaf1: [1, 10, 33, 33]\n", - " pool1: [1, 10, 11, 11]\n", - " pool1a: [1, 10, 8, 8]\n", - "\n", - "DynapcnnLayer 1: ... [1, 10, 11, 11]\n", - " conv2: [1, 10, 8, 8]\n", - " iaf2: [1, 10, 8, 8]\n", - "\n", - "DynapcnnLayer 2: ... [1, 10, 8, 8] [ Merge(pool1a, iaf2_out) ]\n", - " conv3: [1, 1, 7, 7]\n", - " iaf3: [1, 1, 7, 7]\n", - "\n", - "DynapcnnLayer 3: ... [1, 49]\n", - " fc1: [1, 500]\n", - " iaf4: [1, 500]\n", - "\n", - "DynapcnnLayer 4: ... [1, 500]\n", - " fc2: [1, 10]\n", - " iaf5: [1, 10]\n", - "\n" - ] - } - ], - "source": [ - "x = torch.randn((1, *input_shape))\n", - "\n", - "print(f'DynapcnnLayer 0: ... {list(x.shape)}')\n", - "con1_out = snn.conv1(x)\n", - "print(f' conv1: {list(con1_out.shape)}')\n", - "iaf1_out = snn.iaf1(con1_out)\n", - "print(f' iaf1: {list(iaf1_out.shape)}')\n", - "pool1_out = snn.pool1(iaf1_out)\n", - "print(f' pool1: {list(pool1_out.shape)}')\n", - "pool1a_out = snn.pool1a(iaf1_out)\n", - "print(f' pool1a: {list(pool1a_out.shape)}\\n')\n", - "\n", - "print(f'DynapcnnLayer 1: ... {list(pool1_out.shape)}')\n", - "conv2_out = snn.conv2(pool1_out)\n", - "print(f' conv2: {list(conv2_out.shape)}')\n", - "iaf2_out = snn.iaf2(conv2_out)\n", - "print(f' iaf2: {list(iaf2_out.shape)}\\n')\n", - "# pool2_out = snn.pool2(iaf2_out)\n", - "# print(f' pool2: {list(pool2_out.shape)}\\n')\n", - "\n", - "added = snn.adder(pool1a_out, iaf2_out)\n", - "\n", - "print(f'DynapcnnLayer 2: ... {list(added.shape)} [ Merge(pool1a, iaf2_out) ]')\n", - "conv3_out = snn.conv3(added)\n", - "print(f' conv3: {list(conv3_out.shape)}')\n", - "iaf3_out = snn.iaf3(conv3_out)\n", - "print(f' iaf3: {list(iaf3_out.shape)}\\n')\n", - "\n", - "flat_out = snn.flat(iaf3_out)\n", - "\n", - "print(f'DynapcnnLayer 3: ... {list(flat_out.shape)}')\n", - "fc1_out = snn.fc1(flat_out)\n", - "print(f' fc1: {list(fc1_out.shape)}')\n", - "iaf4_out = snn.iaf4(fc1_out)\n", - "print(f' iaf4: {list(iaf4_out.shape)}\\n')\n", - "\n", - "\n", - "print(f'DynapcnnLayer 4: ... {list(iaf4_out.shape)}')\n", - "fc2_out = snn.fc2(iaf4_out)\n", - "print(f' fc2: {list(fc2_out.shape)}')\n", - "iaf5_out = snn.iaf5(fc2_out)\n", - "print(f' iaf5: {list(iaf5_out.shape)}\\n')" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -237,9 +151,9 @@ "outputs": [], "source": [ "hw_model = DynapcnnNetworkGraph(\n", - " snn,\n", - " discretize=True,\n", - " input_shape=input_shape\n", + " snn=snn,\n", + " input_shape=input_shape,\n", + " batch_size=batch_size\n", ")" ] }, @@ -247,73 +161,142 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The `hw_model.to()` call will figure out into which core eac `DynapcnnLayer` instance will be assigned to. Once this assingment is made the instance itself is used to configure the `CNNLayerConfig` instance representing the core's configuration.\n", - "\n", - "If the cores' configuration is valid, each `DynapcnnLayer` instance and their respective destinations will be used to create a computational graph that encodes how the `forward` method of `hw_model.network` (a `nn.Module` using the `DynapcnnLayer` instances) propagates that through the network." + "Notice in the model bellow how the property DynapcnnLayer in the model has yet to be assigned to a core. This is only done once\n", + "DynapcnnNetworkGraph.to() is called." ] }, { "cell_type": "code", "execution_count": 8, - "metadata": { - "metadata": {} - }, + "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Network is valid\n" + "----------------------- [ DynapcnnLayer 0 ] -----------------------\n", + "\n", + "COMPUTATIONAL NODES:\n", + "\n", + "(node 0): Conv2d(2, 10, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + "(node 1): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(361.), min_v_mem=Parameter containing:\n", + "tensor(-361.), batch_size=3, num_timesteps=-1)\n", + "(node 2): SumPool2d(norm_type=1, kernel_size=(3, 3), stride=None, ceil_mode=False)\n", + "(node 3): SumPool2d(norm_type=1, kernel_size=(4, 4), stride=None, ceil_mode=False)\n", + "\n", + "METADATA:\n", + "\n", + "> assigned core index: None\n", + "> destination DynapcnnLayers: [1, 2]\n", + "> node 2 feeds input to nodes [4]\n", + "> node 3 feeds input to nodes [7]\n", + "\n", + "----------------------- [ DynapcnnLayer 1 ] -----------------------\n", + "\n", + "COMPUTATIONAL NODES:\n", + "\n", + "(node 4): Conv2d(10, 10, kernel_size=(4, 4), stride=(1, 1), bias=False)\n", + "(node 6): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(14463.), min_v_mem=Parameter containing:\n", + "tensor(-14463.), batch_size=3, num_timesteps=-1)\n", + "\n", + "METADATA:\n", + "\n", + "> assigned core index: None\n", + "> destination DynapcnnLayers: [2]\n", + "> node 6 feeds input to nodes [7]\n", + "\n", + "----------------------- [ DynapcnnLayer 2 ] -----------------------\n", + "\n", + "COMPUTATIONAL NODES:\n", + "\n", + "(node 7): Conv2d(10, 1, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + "(node 8): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(13042.), min_v_mem=Parameter containing:\n", + "tensor(-13042.), batch_size=3, num_timesteps=-1)\n", + "\n", + "METADATA:\n", + "\n", + "> assigned core index: None\n", + "> destination DynapcnnLayers: [3]\n", + "> node 8 feeds input to nodes [9]\n", + "\n", + "----------------------- [ DynapcnnLayer 3 ] -----------------------\n", + "\n", + "COMPUTATIONAL NODES:\n", + "\n", + "(node 9): Conv2d(1, 500, kernel_size=(7, 7), stride=(1, 1), bias=False)\n", + "(node 10): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(889.), min_v_mem=Parameter containing:\n", + "tensor(-889.), batch_size=3, num_timesteps=-1)\n", + "\n", + "METADATA:\n", + "\n", + "> assigned core index: None\n", + "> destination DynapcnnLayers: [4]\n", + "> node 10 feeds input to nodes [11]\n", + "\n", + "----------------------- [ DynapcnnLayer 4 ] -----------------------\n", + "\n", + "COMPUTATIONAL NODES:\n", + "\n", + "(node 11): Conv2d(500, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + "(node 12): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(2842.), min_v_mem=Parameter containing:\n", + "tensor(-2842.), batch_size=3, num_timesteps=-1)\n", + "\n", + "METADATA:\n", + "\n", + "> assigned core index: None\n", + "> destination DynapcnnLayers: []\n", + "\n", + "\n" ] - }, - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" } ], "source": [ - "hw_model.to(device=\"speck2fmodule:0\")" + "print(hw_model)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "The layers comprising our `hw_model` and their respective metadata can be inspected by calling `print` on a `DynapcnnNetworkGraph` instance." + "Lets forward a sample data through our DynapcnnNetwork instance to see if the produces the correct output:" ] }, { "cell_type": "code", "execution_count": 9, - "metadata": { - "metadata": {} - }, + "metadata": {}, "outputs": [ { - "ename": "AttributeError", - "evalue": "'DynapcnnNetworkGraph' object has no attribute 'get_forward_edges'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[9], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[38;5;28mprint\u001b[39m(hw_model\u001b[38;5;241m.\u001b[39mget_forward_edges())\n", - "\u001b[0;31mAttributeError\u001b[0m: 'DynapcnnNetworkGraph' object has no attribute 'get_forward_edges'" + "name": "stdout", + "output_type": "stream", + "text": [ + "torch.Size([3, 10, 1, 1])\n" ] } ], "source": [ - "print(hw_model.get_network_module().get_forward_edges())" + "x = torch.randn((batch_size, *input_shape))\n", + "out = hw_model(x)\n", + "print(out.shape)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The `hw_model.to()` call will figure out into which core each `DynapcnnLayer` instance will be assigned to. Once this assingment is made the instance itself is used to configure the `CNNLayerConfig` instance representing the core's configuration assigned to it.\n", + "\n", + "If the call is sucessfull, the layers comprising the network and their associated metadata will be printed." ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "metadata": { "metadata": {} }, @@ -322,59 +305,98 @@ "name": "stdout", "output_type": "stream", "text": [ - "---- DynapcnnLayer 0 ----------------------------------------------------------\n", - "> layer modules: \n", + "Network is valid: \n", + "\n", + "----------------------- [ DynapcnnLayer 0 ] -----------------------\n", + "\n", + "COMPUTATIONAL NODES:\n", + "\n", "(node 0): Conv2d(2, 10, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", "(node 1): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", + "tensor(361.), min_v_mem=Parameter containing:\n", + "tensor(-361.), batch_size=3, num_timesteps=-1)\n", "(node 2): SumPool2d(norm_type=1, kernel_size=(3, 3), stride=None, ceil_mode=False)\n", "(node 3): SumPool2d(norm_type=1, kernel_size=(4, 4), stride=None, ceil_mode=False)\n", - "> layer destinations: [1, 2]\n", - "> assigned core: 0\n", "\n", - "---- DynapcnnLayer 1 ----------------------------------------------------------\n", - "> layer modules: \n", + "METADATA:\n", + "\n", + "> assigned core index: 0\n", + "> destination DynapcnnLayers: [1, 2]\n", + "> node 2 feeds input to nodes [4]\n", + "> node 3 feeds input to nodes [7]\n", + "\n", + "----------------------- [ DynapcnnLayer 1 ] -----------------------\n", + "\n", + "COMPUTATIONAL NODES:\n", + "\n", "(node 4): Conv2d(10, 10, kernel_size=(4, 4), stride=(1, 1), bias=False)\n", "(node 6): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [2]\n", - "> assigned core: 1\n", + "tensor(14463.), min_v_mem=Parameter containing:\n", + "tensor(-14463.), batch_size=3, num_timesteps=-1)\n", + "\n", + "METADATA:\n", + "\n", + "> assigned core index: 1\n", + "> destination DynapcnnLayers: [2]\n", + "> node 6 feeds input to nodes [7]\n", + "\n", + "----------------------- [ DynapcnnLayer 2 ] -----------------------\n", + "\n", + "COMPUTATIONAL NODES:\n", "\n", - "---- DynapcnnLayer 2 ----------------------------------------------------------\n", - "> layer modules: \n", "(node 7): Conv2d(10, 1, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", "(node 8): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [3]\n", - "> assigned core: 2\n", + "tensor(13042.), min_v_mem=Parameter containing:\n", + "tensor(-13042.), batch_size=3, num_timesteps=-1)\n", + "\n", + "METADATA:\n", + "\n", + "> assigned core index: 2\n", + "> destination DynapcnnLayers: [3]\n", + "> node 8 feeds input to nodes [9]\n", + "\n", + "----------------------- [ DynapcnnLayer 3 ] -----------------------\n", + "\n", + "COMPUTATIONAL NODES:\n", "\n", - "---- DynapcnnLayer 3 ----------------------------------------------------------\n", - "> layer modules: \n", "(node 9): Conv2d(1, 500, kernel_size=(7, 7), stride=(1, 1), bias=False)\n", "(node 10): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [4]\n", - "> assigned core: 3\n", + "tensor(889.), min_v_mem=Parameter containing:\n", + "tensor(-889.), batch_size=3, num_timesteps=-1)\n", + "\n", + "METADATA:\n", + "\n", + "> assigned core index: 3\n", + "> destination DynapcnnLayers: [4]\n", + "> node 10 feeds input to nodes [11]\n", + "\n", + "----------------------- [ DynapcnnLayer 4 ] -----------------------\n", + "\n", + "COMPUTATIONAL NODES:\n", "\n", - "---- DynapcnnLayer 4 ----------------------------------------------------------\n", - "> layer modules: \n", "(node 11): Conv2d(500, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", "(node 12): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: []\n", - "> assigned core: 4\n", + "tensor(2842.), min_v_mem=Parameter containing:\n", + "tensor(-2842.), batch_size=3, num_timesteps=-1)\n", + "\n", + "METADATA:\n", + "\n", + "> assigned core index: 4\n", + "> destination DynapcnnLayers: []\n", "\n", "\n" ] } ], "source": [ - "print(hw_model)" + "hw_model.to(device=\"speck2fmodule:0\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Notice above now how the layers of the model have been assigned to a chip core." ] } ], From d0bbd2db6265e428578568b5dfa9040539359b91 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Mon, 20 May 2024 13:24:11 +0200 Subject: [PATCH 093/379] Refactor scripts imports and class naming - New dynapcnn_network.py being imported via __init__ (old 'dynapcnn_network.py' becomes 'dynapcnn_network_deprecated.py'). - New dynapcnn_layer.py replaces old implementation (old 'dynapcnn_layer.py' becomes 'dynapcnn_layer_deprecated.py'). - Support functionality no longer checks for either type DynapcnnNetwork or DynapcnnNetworkGraph (DynapcnnNetworkGraph becomes the new DynapcnnNetwork). --- sinabs/backend/dynapcnn/chips/dynapcnn.py | 39 +- sinabs/backend/dynapcnn/config_builder.py | 11 - sinabs/backend/dynapcnn/dynapcnn_layer.py | 528 +++++++++++--- .../dynapcnn/dynapcnn_layer_deprecated.py | 204 ++++++ sinabs/backend/dynapcnn/dynapcnn_network.py | 688 ++++++++++-------- .../dynapcnn/dynapcnn_network_deprecated.py | 508 +++++++++++++ sinabs/backend/dynapcnn/mapping.py | 17 +- sinabs/backend/dynapcnn/utils.py | 3 +- .../DynapcnnNetwork-example_1.ipynb | 14 +- 9 files changed, 1505 insertions(+), 507 deletions(-) create mode 100644 sinabs/backend/dynapcnn/dynapcnn_layer_deprecated.py create mode 100644 sinabs/backend/dynapcnn/dynapcnn_network_deprecated.py diff --git a/sinabs/backend/dynapcnn/chips/dynapcnn.py b/sinabs/backend/dynapcnn/chips/dynapcnn.py index 3559ee8b..1039e8bf 100644 --- a/sinabs/backend/dynapcnn/chips/dynapcnn.py +++ b/sinabs/backend/dynapcnn/chips/dynapcnn.py @@ -9,8 +9,7 @@ import sinabs from sinabs.backend.dynapcnn.config_builder import ConfigBuilder from sinabs.backend.dynapcnn.dvs_layer import DVSLayer, expand_to_pair -# from sinabs.backend.dynapcnn.dynapcnn_layer import DynapcnnLayer -from sinabs.backend.dynapcnn.dynapcnn_layer_new import DynapcnnLayer +from sinabs.backend.dynapcnn.dynapcnn_layer import DynapcnnLayer from sinabs.backend.dynapcnn.mapping import LayerConstraints import sinabs @@ -244,7 +243,7 @@ def write_dynapcnn_layer_config_graph(cls, dcnnl: DynapcnnLayer, chip_layer: "CN raise TypeError(f"Unexpected parameter {param} or value. {e}") @classmethod - def build_config(cls, model: Union["DynapcnnNetwork", "DynapcnnNetworkGraph"], chip_layers: Union[List[int], None]) -> DynapcnnConfiguration: + def build_config(cls, model: Union["DynapcnnNetwork"], chip_layers: Union[List[int], None]) -> DynapcnnConfiguration: """ Uses `DynapcnnLayer` objects to configure their equivalent chip core via a `CNNLayerConfig` object that is built using using the `DynapcnnLayer` properties. @@ -263,40 +262,6 @@ def build_config(cls, model: Union["DynapcnnNetwork", "DynapcnnNetworkGraph"], c config = cls.get_default_config() if type(model) == sinabs.backend.dynapcnn.dynapcnn_network.DynapcnnNetwork: - """ loops through `DynapcnnNetwork.sequence`, sequentially using the core IDs in `chip_layers` to configure their - respective `CNNLayerConfig`. - """ - layers = model.sequence - - has_dvs_layer = False - i_cnn_layer = 0 # Instantiate an iterator for the cnn cores - _prev_idx = 0 - for i, chip_equivalent_layer in enumerate(layers): - if isinstance(chip_equivalent_layer, DVSLayer): - chip_layer = config.dvs_layer - cls.write_dvs_layer_config(chip_equivalent_layer, chip_layer) - has_dvs_layer = True - elif isinstance(chip_equivalent_layer, DynapcnnLayer): - chip_layer = config.cnn_layers[chip_layers[i_cnn_layer]] - cls.write_dynapcnn_layer_config(chip_equivalent_layer, chip_layer) - i_cnn_layer += 1 - else: - # in our generated network there is a spurious layer... - # should never happen - raise TypeError("Unexpected layer in the model") - - if i == len(layers) - 1: - # last layer - chip_layer.destinations[0].enable = False - else: - # Set destination layer - chip_layer.destinations[0].layer = chip_layers[i_cnn_layer] - chip_layer.destinations[0].enable = True - - if not has_dvs_layer: - config.dvs_layer.pass_sensor_events = False - - elif type(model) == sinabs.backend.dynapcnn.dynapcnn_network_graph.DynapcnnNetworkGraph: """ Loops through `DynapcnnNetworkGraph._forward_map`, containing all `DynapcnnLayer`s in the model, their core ID to be loaded onto and their target destinations. Each `ith_dcnnl` has all the info. necessary to config. their respective `CNNLayerConfig` object. diff --git a/sinabs/backend/dynapcnn/config_builder.py b/sinabs/backend/dynapcnn/config_builder.py index 0b1954fc..3843a6a8 100644 --- a/sinabs/backend/dynapcnn/config_builder.py +++ b/sinabs/backend/dynapcnn/config_builder.py @@ -83,17 +83,6 @@ def get_valid_mapping(cls, model: "DynapcnnNetwork") -> List[int]: if type(model) == sinabs.backend.dynapcnn.dynapcnn_network.DynapcnnNetwork: mapping = get_valid_mapping(model, cls.get_constraints()) - # turn the mapping into a dict - mapping = {m[0]: m[1] for m in mapping} - # Check if there is a dvs layer in the model - num_dynapcnn_cores = len(model.sequence) - if isinstance(model.sequence[0], DVSLayer): - num_dynapcnn_cores -= 1 - # apply the mapping - chip_layers_ordering = [mapping[i] for i in range(num_dynapcnn_cores)] - - elif type(model) == sinabs.backend.dynapcnn.dynapcnn_network_graph.DynapcnnNetworkGraph: - mapping = get_valid_mapping(model, cls.get_constraints()) if isinstance(model.forward_map[0], DVSLayer): # TODO not handling DVSLayer yet. diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer.py b/sinabs/backend/dynapcnn/dynapcnn_layer.py index a56454c8..50a1a64b 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer.py @@ -12,158 +12,348 @@ from .discretize import discretize_conv_spike_ from .dvs_layer import expand_to_pair - class DynapcnnLayer(nn.Module): - """Create a DynapcnnLayer object representing a dynapcnn layer. - - Requires a convolutional layer, a sinabs spiking layer and an optional - pooling value. The layers are used in the order conv -> spike -> pool. - - Parameters - ---------- - conv: torch.nn.Conv2d or torch.nn.Linear - Convolutional or linear layer (linear will be converted to convolutional) - spk: sinabs.layers.IAFSqueeze - Sinabs IAF layer - in_shape: tuple of int - The input shape, needed to create dynapcnn configs if the network does not - contain an input layer. Convention: (features, height, width) - pool: int or None - Integer representing the sum pooling kernel and stride. If `None`, no - pooling will be applied. - discretize: bool - Whether to discretize parameters. - rescale_weights: int - Layer weights will be divided by this value. - """ + """Create a DynapcnnLayer object representing a dynapcnn layer. """ def __init__( self, - conv: nn.Conv2d, - spk: sl.IAFSqueeze, - in_shape: Tuple[int, int, int], - pool: Optional[sl.SumPool2d] = None, - discretize: bool = True, - rescale_weights: int = 1, + dpcnnl_index: int, + dcnnl_data: dict, + discretize: bool, + sinabs_edges: list ): super().__init__() + """ + ... - self.input_shape = in_shape - + Parameters + ---------- + dpcnnl_index (int): ... + dcnnl_data (dict): ... + discretize (bool): ... + sinabs_edges (list): ... + """ + self.dpcnnl_index = dpcnnl_index + self.assigned_core = None + + if 'core_idx' in dcnnl_data: + self.assigned_core = dcnnl_data['core_idx'] + + self.lin_to_conv_conversion = False + + conv = None + self.conv_node_id = None + self.conv_in_shape = None + self.conv_out_shape = None + + spk = None + self.spk_node_id = None + + pool = [] + self.pool_node_id = [] + + self.dynapcnnlayer_destination = dcnnl_data['destinations'] + + for key, value in dcnnl_data.items(): + if isinstance(key, int): + # value has data pertaining a node (torch/sinabs layer). + if isinstance(value['layer'], sl.IAFSqueeze): + spk = value['layer'] + self.spk_node_id = key + elif isinstance(value['layer'], nn.Linear) or isinstance(value['layer'], nn.Conv2d): + conv = value['layer'] + self.conv_node_id = key + elif isinstance(value['layer'], sl.SumPool2d): + pool.append(value['layer']) + self.pool_node_id.append(key) + else: + raise ValueError(f'Node {key} has not valid layer associated with it.') + + if not conv: + raise ValueError(f'Convolution layer not present.') + + if not spk: + raise ValueError(f'Spiking layer not present.') + spk = deepcopy(spk) + if spk.is_state_initialised(): + # TODO this line bellow is causing an exception on `.v_men.shape` to be raised in `.get_layer_config_dict()`. Find out why. + # spk.v_mem = spk.v_mem.data.unsqueeze(-1).unsqueeze(-1) # expand dims. + + # TODO hacky stuff: make it better (THIS SEEMS TO BE FIXING THE PROBLEM ABOVE THO). + if len(list(spk.v_mem.shape)) != 4: + spk.v_mem = spk.v_mem.data.unsqueeze(-1).unsqueeze(-1) # expand dims. + if isinstance(conv, nn.Linear): - conv = self._convert_linear_to_conv(conv) - if spk.is_state_initialised(): - # Expand dims - spk.v_mem = spk.v_mem.data.unsqueeze(-1).unsqueeze(-1) + conv, conv_in_shape = self._convert_linear_to_conv(conv, dcnnl_data[self.conv_node_id]) + + # the original `nn.Linear` output shape becomes the equivalent `nn.Conv2d` shape. + self.conv_out_shape = self._update_conv_node_output_shape( + conv_layer=conv, layer_data=dcnnl_data[self.conv_node_id], input_shape=conv_in_shape) + + # the I/O shapes for neuron layer following the new conv need also to be updated. + self._update_neuron_node_output_shape(layer_data=dcnnl_data[self.spk_node_id], input_shape=self.conv_out_shape) + else: + self.conv_out_shape = dcnnl_data[self.conv_node_id]['output_shape'] conv = deepcopy(conv) - if rescale_weights != 1: + # check if convolution kernel is a square. + if conv.kernel_size[0] != conv.kernel_size[1]: + raise ValueError('The kernel of a `nn.Conv2d` must have the same height and width.') + + # input shape of conv layer. + self.conv_in_shape = dcnnl_data[self.conv_node_id]['input_shape'] + + # this weight rescale comes from the node projecting into this 'conv' node. + if dcnnl_data['conv_rescale_factor'] != 1: # this has to be done after copying but before discretizing - conv.weight.data = (conv.weight / rescale_weights).clone().detach() + conv.weight.data = (conv.weight / dcnnl_data['conv_rescale_factor']).clone().detach() - self.discretize = discretize + # int conversion is done while writing the config. if discretize: - # int conversion is done while writing the config. conv, spk = discretize_conv_spike_(conv, spk, to_int=False) + # consolidate layers. self.conv_layer = conv self.spk_layer = spk - if pool is not None: - if pool.kernel_size[0] != pool.kernel_size[1]: - raise ValueError("Only square kernels are supported") - self.pool_layer = deepcopy(pool) - else: - self.pool_layer = None + self.pool_layer = [] - def _convert_linear_to_conv(self, lin: nn.Linear) -> nn.Conv2d: - """Convert Linear layer to Conv2d. + if len(pool) != 0: + # the 1st pooling targets the 1st destination in `dcnnl_data['destinations']`, the 2nd pooling targets the 2nd destination... + for plyr in pool: + # @TODO POSSIBLE INCONSISTENCY: if the `SumPool2d` is the result of a conversion from `AvgPool2d` then `SumPool2d.kernel_size` + # is of type tuple, otherwise it is an int. + if isinstance(plyr.kernel_size, tuple) and plyr.kernel_size[0] != plyr.kernel_size[1]: + raise ValueError("Only square kernels are supported") + self.pool_layer.append(deepcopy(plyr)) + + # map destination nodes for each layer in this instance. + self.nodes_destinations = self._get_destinations_input_source(sinabs_edges) + + ####################################################### Public Methods ####################################################### + + def get_destination_dcnnl_index(self, dcnnl_id: int) -> int: + """ The `forward` method will return as many tensors as there are elements in `self.dynapcnnlayer_destination`. Since the i-th returned tensor is to be + fed to the i-th destionation in `self.dynapcnnlayer_destination`, the return of this method can be used to index a tensor returned in the `forward` method. Parameters ---------- - lin: nn.Linear - Linear layer to be converted + dcnnl_id (int): this should be one of the values listed within `self.dynapcnnlayer_destination`. Returns - ------- - nn.Conv2d - Convolutional layer equivalent to `lin`. + ---------- + The index of `dcnnl_id` within `self.dynapcnnlayer_destination`. """ + return self.dynapcnnlayer_destination.index(dcnnl_id) + + def forward(self, x): + """Torch forward pass. - in_chan, in_h, in_w = self.input_shape + Returns + ---------- + This method will return as many tensors as there are destinations associated with this instance. The returned tensors always follows the + sequence `return spiking_layer_output, pooling_layer_1, pooling_layer_2, ...`. + + Example + ---------- + With `self.nodes_destinations = {1: [5, 8], 2: [4], 3: [7]}` this method will return 4 tensors (each to be sent to a different DynapcnnLayer): the 1st + and 2nd are the outputs of the spiking layer (`node 1`); the 3rd is the output of the first pooling layer (`node 2`, receiving the output of node 1) appearing + right after the spiking layer as defined in the original `nn.Module` being converted to a `DynapcnnNetwork`; the 4th is the output of the second pooling + layer (`node 3`) to appear after the spiking layer. Thus, the return will be `return node_1_out, node_1_out, node_2_out, node_3_out`. This means the edges + in the computational graph involved in this mapping were: + + 1 --> 2 # `2` is one of the pooling layers of this DynapcnnLayer. + 1 --> 3 # `3` is one of the pooling layers of this DynapcnnLayer. + 1 --> 5 # `5` is a conv layer belonging to another DynapcnnLayer U. + 1 --> 8 # `8` is a conv layer belonging to another DynapcnnLayer V. + 2 --> 4 # `4` is a conv layer belonging to another DynapcnnLayer X. + 3 --> 7 # `7` is a conv layer belonging to another DynapcnnLayer Y. + """ - if lin.in_features != in_chan * in_h * in_w: - raise ValueError("Shapes don't match.") + returns = [] + + x = self.conv_layer(x) + x = self.spk_layer(x) - layer = nn.Conv2d( - in_channels=in_chan, - kernel_size=(in_h, in_w), - out_channels=lin.out_features, - padding=0, - bias=lin.bias is not None, - ) + # pooling layers are sequentially added to `self.pool_layer` so we'll pass data to them sequentially. + pooling_indexer = 0 + + # building return set of all layers as they appear in `self.nodes_destinations`. + for node_id, destination_node_list in self.nodes_destinations.items(): + if node_id == self.spk_node_id: + # spiking output for each node outside this DynapcnnLayer receiving from its spiking layer. + for _ in destination_node_list: + returns.append(x) + else: + # returns of each pooling layer are arranged sequenatially. + for _ in destination_node_list: + ith_pool_output = self.pool_layer[pooling_indexer](x) + returns.append(ith_pool_output) + + # forward through next pooling layer in `self.pool_layer` in the next iteration. + pooling_indexer += 1 + + if len(returns) != len(self.dynapcnnlayer_destination): + raise ValueError(f'Number of returned tensors ({len(returns)}) differ from the number of destinations ({len(self.dynapcnnlayer_destination)}).') + + if len(returns) == 0 and len(self.pool_layer) == 0: + # this is the output layer and there's no pooling after the neurons. + returns.append(x) + elif len(returns) == 0 and len(self.pool_layer) == 1: + # this is the output layer and there's 1 pooling after the neurons. + returns.append(self.pool_layer[0](x)) + elif len(returns) == 0 and len(self.pool_layer) > 1: + raise ValueError(f'Output DynapcnnLayer starting with node {self.conv_node_id} has {len(self.pool_layer)} pooling layers: it should have either 1 or none.') + else: + pass - if lin.bias is not None: - layer.bias.data = lin.bias.data.clone().detach() + return tuple(returns) - layer.weight.data = ( - lin.weight.data.clone() - .detach() - .reshape((lin.out_features, in_chan, in_h, in_w)) - ) + def get_modified_node_it(self, dcnnl_data: dict) -> Union[Tuple[int, tuple], Tuple[None, None]]: + """ .""" + if self.lin_to_conv_conversion: + return self.spk_node_id, dcnnl_data[self.spk_node_id]['output_shape'] + return None, None + + def zero_grad(self, set_to_none: bool = False) -> None: + return self.spk_layer.zero_grad(set_to_none) + + def get_conv_output_shape(self, conv_layer: nn.Conv2d, input_shape: tuple): + """ .""" + # get the layer's parameters. + out_channels = conv_layer.out_channels + kernel_size = conv_layer.kernel_size + stride = conv_layer.stride + padding = conv_layer.padding + dilation = conv_layer.dilation + + # compute the output height and width. + out_height = ((input_shape[1] + 2 * padding[0] - dilation[0] * (kernel_size[0] - 1) - 1) // stride[0]) + 1 + out_width = ((input_shape[2] + 2 * padding[1] - dilation[1] * (kernel_size[1] - 1) - 1) // stride[1]) + 1 + + return (out_channels, out_height, out_width) - return layer + def summary(self) -> dict: + # TODO I can't see pooling being used in checking memory constraints by the builder so I'm ignoring for now the fact that multiple pooling could exist. + + _pool = None + + # @TODO POSSIBLE INCONSISTENCY: if the `SumPool2d` is the result of a conversion from `AvgPool2d` then `SumPool2d.kernel_size` + # is of type tuple, otherwise it is an int. + if len(self.pool_layer) != 0: + # @TODO ignoring for now that there could be multiple poolings (just use the first one). + if isinstance(self.pool_layer[0].kernel_size, tuple): + _pool = list(self.pool_layer[0].kernel_size) + elif isinstance(self.pool_layer[0].kernel_size, int): + _pool = [self.pool_layer[0].kernel_size, self.pool_layer[0].kernel_size] + else: + raise ValueError('Type of `self.pool_layer[0].kernel_size` not understood.') - def get_neuron_shape(self) -> Tuple[int, int, int]: - """Return the output shape of the neuron layer. + return { + "pool": (_pool), + "kernel": list(self.conv_layer.weight.data.shape), + "neuron": self.conv_out_shape, # neuron layer output has the same shape as the convolution layer ouput. + } - Returns - ------- - features, height, width + def get_layer_config_dict(self) -> dict: + """ Returns a dict containing the properties required to configure a `CNNLayerConfig` instance that + will map this DynapcnnLayer onto the chip. """ + config_dict = {} + + # configures `CNNLayerConfig.dimensions` (instance of `CNNLayerDimensions`). + dimensions = {} + + # input shape of convolution. + dimensions['input_shape'] = { + 'size': {'x': self.conv_in_shape[2], 'y': self.conv_in_shape[1]}, + 'feature_count': self.conv_in_shape[0] + } + + # ouput shape of convolution. + dimensions['output_shape'] = { + 'size': {'x': self.conv_out_shape[2], 'y': self.conv_out_shape[1]}, + 'feature_count': self.conv_out_shape[0] + } + + # convolution padding, stride and kernel sizes. + dimensions['padding'] = {'x': self.conv_layer.padding[1], 'y': self.conv_layer.padding[0]} + dimensions['stride'] = {'x': self.conv_layer.stride[1], 'y': self.conv_layer.stride[0]} + dimensions['kernel_size'] = self.conv_layer.kernel_size[0] + + config_dict['dimensions'] = dimensions # update config dict. + + # update parameters from convolution. + if self.conv_layer.bias is not None: + (weights, biases) = self.conv_layer.parameters() + else: + (weights,) = self.conv_layer.parameters() + biases = torch.zeros(self.conv_layer.out_channels) - def get_shape_after_conv(layer: nn.Conv2d, input_shape): - (ch_in, h_in, w_in) = input_shape - (kh, kw) = expand_to_pair(layer.kernel_size) - (pad_h, pad_w) = expand_to_pair(layer.padding) - (stride_h, stride_w) = expand_to_pair(layer.stride) + # parameters of the convolution in the DynapcnnLayer. - def out_len(in_len, k, s, p): - return (in_len - k + 2 * p) // s + 1 + weights = weights.transpose(2, 3) # need this to match samna convention. + config_dict['weights'] = weights.int().tolist() # 4-D list of lists representing kernel parameters. + config_dict['biases'] = biases.int().tolist() + config_dict['leak_enable'] = biases.bool().any() - out_h = out_len(h_in, kh, stride_h, pad_h) - out_w = out_len(w_in, kw, stride_w, pad_w) - ch_out = layer.out_channels - return ch_out, out_h, out_w + # parameters of the neurons in the DynapcnnLayer. - conv_out_shape = get_shape_after_conv( - self.conv_layer, input_shape=self.input_shape - ) - return conv_out_shape - - def get_output_shape(self) -> Tuple[int, int, int]: - neuron_shape = self.get_neuron_shape() - # this is the actual output shape, including pooling - if self.pool_layer is not None: - pool = expand_to_pair(self.pool_layer.kernel_size) - return ( - neuron_shape[0], - neuron_shape[1] // pool[0], - neuron_shape[2] // pool[1], - ) - else: - return neuron_shape + # set neuron states. # TODO coppied from the old implementation. + if not self.spk_layer.is_state_initialised(): + # then we assign no initial neuron state to DYNAP-CNN. + f, h, w = self.conv_out_shape # same as the convolution layer. + neurons_state = torch.zeros(f, w, h) - def summary(self) -> dict: - return { - "pool": ( - None if self.pool_layer is None else list(self.pool_layer.kernel_size) - ), - "kernel": list(self.conv_layer.weight.data.shape), - "neuron": self.get_neuron_shape(), - } + elif self.spk_layer.v_mem.dim() == 4: + # 4-D states should be the norm when there is a batch dim. + neurons_state = self.spk_layer.v_mem.transpose(2, 3)[0] + else: + raise ValueError(f"Current v_mem (shape: {self.spk_layer.v_mem.shape}) of spiking layer not understood.") + # TODO error here: find where `self.spk_layer.v_mem` is being initialized. + + # resetting vs returning to 0. # TODO coppied from the old implementation. + if isinstance(self.spk_layer.reset_fn, sinabs.activation.MembraneReset): + return_to_zero = True # neurons in this layer will return to 0 when firing. + elif isinstance(self.spk_layer.reset_fn, sinabs.activation.MembraneSubtract): + return_to_zero = False # threshold will be subtracted from the value their membrane potential reached before firing. + else: + raise Exception("Unknown reset mechanism. Only MembraneReset and MembraneSubtract are currently understood.") + + if self.spk_layer.min_v_mem is None: + min_v_mem = -(2**15) + else: + min_v_mem = int(self.spk_layer.min_v_mem) + + # set neuron configuration for this DynapcnnLayer. + config_dict.update( + { + "return_to_zero": return_to_zero, + "threshold_high": int(self.spk_layer.spike_threshold), + "threshold_low": min_v_mem, + "monitor_enable": False, + "neurons_initial_value": neurons_state.int().tolist() + } + ) + + # set pooling configuration for each destinaition. This configures a `CNNLayerConfig.destinations` (instance of `CNNLayerDimensions`). + config_dict['destinations'] = [] + if len(self.pool_layer) != 0: + for i in range(len(self.pool_layer)): + dest_config = { + 'layer': self.dynapcnnlayer_destination[i],# TODO this destination index is not the core index yet, just the index of the DynapcnnLayers themselves. + 'enable': True, + 'pooling': self.pool_layer[i].kernel_size[0] if isinstance(self.pool_layer[i].kernel_size, tuple) else self.pool_layer[i].kernel_size # TODO make sure the kernel is a square. + } + + config_dict['destinations'].append(dest_config) + + # setting of the kill bits need to be done outside this method. + + return config_dict + def memory_summary(self): """Computes the amount of memory required for each of the components. Note that this is not necessarily the same as the number of parameters due to some architecture design @@ -183,7 +373,7 @@ def memory_summary(self): """ summary = self.summary() f, c, h, w = summary["kernel"] - f, neuron_height, neuron_width = self.get_neuron_shape() + f, neuron_height, neuron_width = self.conv_out_shape # neuron layer output has the same shape as the convolution layer ouput. return { "kernel": c * pow(2, np.ceil(np.log2(h * w)) + np.ceil(np.log2(f))), @@ -191,14 +381,110 @@ def memory_summary(self): * pow(2, np.ceil(np.log2(neuron_height)) + np.ceil(np.log2(neuron_width))), "bias": 0 if self.conv_layer.bias is None else len(self.conv_layer.bias), } + + def __str__(self): + pretty_print = '\n' - def forward(self, x): - """Torch forward pass.""" - x = self.conv_layer(x) - x = self.spk_layer(x) - if self.pool_layer is not None: - x = self.pool_layer(x) - return x + pretty_print += 'COMPUTATIONAL NODES:\n\n' - def zero_grad(self, set_to_none: bool = False) -> None: - return self.spk_layer.zero_grad(set_to_none) + pretty_print += f'(node {self.conv_node_id}): {self.conv_layer}\n' + pretty_print += f'(node {self.spk_node_id}): {self.spk_layer}' + if len(self.pool_layer) != 0: + for idx, lyr in enumerate(self.pool_layer): + pretty_print += f'\n(node {self.pool_node_id[idx]}): {lyr}' + + pretty_print += '\n\nMETADATA:\n' + pretty_print += f'\n> assigned core index: {self.assigned_core}' + pretty_print += f'\n> destination DynapcnnLayers: {self.dynapcnnlayer_destination}' + + for node, destinations in self.nodes_destinations.items(): + pretty_print += f'\n> node {node} feeds input to nodes {destinations}' + + return pretty_print + + ####################################################### Private Methods ####################################################### + + def _update_neuron_node_output_shape(self, layer_data: dict, input_shape: tuple) -> None: + """ Following the conversion of a `nn.Linear` into a `nn.Conv2d` the neuron layer in the + sequence also needs its I/O shapes uodated. + """ + layer_data['input_shape'] = input_shape + layer_data['output_shape'] = layer_data['input_shape'] + + def _update_conv_node_output_shape(self, conv_layer: nn.Conv2d, layer_data: dict, input_shape: tuple) -> Tuple: + """ The input shapes to nodes are extracted using a list of edges by finding the output shape of the 1st element + in the edge and setting it as the input shape to the 2nd element in the edge. If a node used to be a `nn.Linear` + and it became a `nn.Conv2d`, output shape in the mapper needs to be updated, otherwise there will be a mismatch + between its output and the input it provides to another node. + """ + layer_data['output_shape'] = self.get_conv_output_shape(conv_layer, input_shape) + + return layer_data['output_shape'] + + def _convert_linear_to_conv(self, lin: nn.Linear, layer_data: dict) -> nn.Conv2d: + """Convert Linear layer to Conv2d. + + Parameters + ---------- + lin: nn.Linear + Linear layer to be converted + + Returns + ------- + nn.Conv2d + Convolutional layer equivalent to `lin`. + """ + self.lin_to_conv_conversion = True + + input_shape = layer_data['input_shape'] + + in_chan, in_h, in_w = input_shape + + if lin.in_features != in_chan * in_h * in_w: + raise ValueError("Shapes don't match.") + + layer = nn.Conv2d( + in_channels=in_chan, + kernel_size=(in_h, in_w), + out_channels=lin.out_features, + padding=0, + bias=lin.bias is not None, + ) + + if lin.bias is not None: + layer.bias.data = lin.bias.data.clone().detach() + + layer.weight.data = ( + lin.weight.data.clone() + .detach() + .reshape((lin.out_features, in_chan, in_h, in_w)) + ) + + return layer, input_shape + + def _get_destinations_input_source(self, sinabs_edges: list) -> dict: + """ Creates a mapping between each layer in this `DynapcnnLayer` instance and its targe nodes that are part of different + `DynapcnnLayer` instances. This mapping is used to figure out how many tensors the `forward` method needs to return. + """ + destinations_input_source = {} + + # check whether spiking layer projects outside this DynapcnnLayer (i.e. to one of the destinations without passing through a pooling). + spk_destinations = [] + for edge in sinabs_edges: + if edge[0] == self.spk_node_id and edge[1] not in self.pool_node_id: + # spiking layer projects to a node outside this DynapcnnLayer. + spk_destinations.append(edge[1]) + if len(spk_destinations) > 0: + destinations_input_source[self.spk_node_id] = [] + for node_id in spk_destinations: + destinations_input_source[self.spk_node_id].append(node_id) + + # get `pooling->destination` mapping. The pooling outputs will be arranged sequentially since the pooling layers are added sequentially + # to `self.pool_layer` (i.e., as they appear in the computational graph of the original `nn.Module`). + for id in self.pool_node_id: + destinations_input_source[id] = [] + for edge in sinabs_edges: + if edge[0] == id: + destinations_input_source[id].append(edge[1]) + + return destinations_input_source \ No newline at end of file diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer_deprecated.py b/sinabs/backend/dynapcnn/dynapcnn_layer_deprecated.py new file mode 100644 index 00000000..a56454c8 --- /dev/null +++ b/sinabs/backend/dynapcnn/dynapcnn_layer_deprecated.py @@ -0,0 +1,204 @@ +from copy import deepcopy +from typing import Dict, Optional, Tuple, Union +from warnings import warn + +import numpy as np +import torch +from torch import nn + +import sinabs.activation +import sinabs.layers as sl + +from .discretize import discretize_conv_spike_ +from .dvs_layer import expand_to_pair + + +class DynapcnnLayer(nn.Module): + """Create a DynapcnnLayer object representing a dynapcnn layer. + + Requires a convolutional layer, a sinabs spiking layer and an optional + pooling value. The layers are used in the order conv -> spike -> pool. + + Parameters + ---------- + conv: torch.nn.Conv2d or torch.nn.Linear + Convolutional or linear layer (linear will be converted to convolutional) + spk: sinabs.layers.IAFSqueeze + Sinabs IAF layer + in_shape: tuple of int + The input shape, needed to create dynapcnn configs if the network does not + contain an input layer. Convention: (features, height, width) + pool: int or None + Integer representing the sum pooling kernel and stride. If `None`, no + pooling will be applied. + discretize: bool + Whether to discretize parameters. + rescale_weights: int + Layer weights will be divided by this value. + """ + + def __init__( + self, + conv: nn.Conv2d, + spk: sl.IAFSqueeze, + in_shape: Tuple[int, int, int], + pool: Optional[sl.SumPool2d] = None, + discretize: bool = True, + rescale_weights: int = 1, + ): + super().__init__() + + self.input_shape = in_shape + + spk = deepcopy(spk) + if isinstance(conv, nn.Linear): + conv = self._convert_linear_to_conv(conv) + if spk.is_state_initialised(): + # Expand dims + spk.v_mem = spk.v_mem.data.unsqueeze(-1).unsqueeze(-1) + else: + conv = deepcopy(conv) + + if rescale_weights != 1: + # this has to be done after copying but before discretizing + conv.weight.data = (conv.weight / rescale_weights).clone().detach() + + self.discretize = discretize + if discretize: + # int conversion is done while writing the config. + conv, spk = discretize_conv_spike_(conv, spk, to_int=False) + + self.conv_layer = conv + self.spk_layer = spk + if pool is not None: + if pool.kernel_size[0] != pool.kernel_size[1]: + raise ValueError("Only square kernels are supported") + self.pool_layer = deepcopy(pool) + else: + self.pool_layer = None + + def _convert_linear_to_conv(self, lin: nn.Linear) -> nn.Conv2d: + """Convert Linear layer to Conv2d. + + Parameters + ---------- + lin: nn.Linear + Linear layer to be converted + + Returns + ------- + nn.Conv2d + Convolutional layer equivalent to `lin`. + """ + + in_chan, in_h, in_w = self.input_shape + + if lin.in_features != in_chan * in_h * in_w: + raise ValueError("Shapes don't match.") + + layer = nn.Conv2d( + in_channels=in_chan, + kernel_size=(in_h, in_w), + out_channels=lin.out_features, + padding=0, + bias=lin.bias is not None, + ) + + if lin.bias is not None: + layer.bias.data = lin.bias.data.clone().detach() + + layer.weight.data = ( + lin.weight.data.clone() + .detach() + .reshape((lin.out_features, in_chan, in_h, in_w)) + ) + + return layer + + def get_neuron_shape(self) -> Tuple[int, int, int]: + """Return the output shape of the neuron layer. + + Returns + ------- + features, height, width + """ + + def get_shape_after_conv(layer: nn.Conv2d, input_shape): + (ch_in, h_in, w_in) = input_shape + (kh, kw) = expand_to_pair(layer.kernel_size) + (pad_h, pad_w) = expand_to_pair(layer.padding) + (stride_h, stride_w) = expand_to_pair(layer.stride) + + def out_len(in_len, k, s, p): + return (in_len - k + 2 * p) // s + 1 + + out_h = out_len(h_in, kh, stride_h, pad_h) + out_w = out_len(w_in, kw, stride_w, pad_w) + ch_out = layer.out_channels + return ch_out, out_h, out_w + + conv_out_shape = get_shape_after_conv( + self.conv_layer, input_shape=self.input_shape + ) + return conv_out_shape + + def get_output_shape(self) -> Tuple[int, int, int]: + neuron_shape = self.get_neuron_shape() + # this is the actual output shape, including pooling + if self.pool_layer is not None: + pool = expand_to_pair(self.pool_layer.kernel_size) + return ( + neuron_shape[0], + neuron_shape[1] // pool[0], + neuron_shape[2] // pool[1], + ) + else: + return neuron_shape + + def summary(self) -> dict: + return { + "pool": ( + None if self.pool_layer is None else list(self.pool_layer.kernel_size) + ), + "kernel": list(self.conv_layer.weight.data.shape), + "neuron": self.get_neuron_shape(), + } + + def memory_summary(self): + """Computes the amount of memory required for each of the components. Note that this is not + necessarily the same as the number of parameters due to some architecture design + constraints. + + .. math:: + + K_{MT} = c \\cdot 2^{\\lceil \\log_2\\left(k_xk_y\\right) \\rceil + \\lceil \\log_2\\left(f\\right) \\rceil} + + .. math:: + + N_{MT} = f \\cdot 2^{ \\lceil \\log_2\\left(f_y\\right) \\rceil + \\lceil \\log_2\\left(f_x\\right) \\rceil } + + Returns + ------- + A dictionary with keys kernel, neuron and bias and the corresponding memory sizes + """ + summary = self.summary() + f, c, h, w = summary["kernel"] + f, neuron_height, neuron_width = self.get_neuron_shape() + + return { + "kernel": c * pow(2, np.ceil(np.log2(h * w)) + np.ceil(np.log2(f))), + "neuron": f + * pow(2, np.ceil(np.log2(neuron_height)) + np.ceil(np.log2(neuron_width))), + "bias": 0 if self.conv_layer.bias is None else len(self.conv_layer.bias), + } + + def forward(self, x): + """Torch forward pass.""" + x = self.conv_layer(x) + x = self.spk_layer(x) + if self.pool_layer is not None: + x = self.pool_layer(x) + return x + + def zero_grad(self, set_to_none: bool = False) -> None: + return self.spk_layer.zero_grad(set_to_none) diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index 9d59236f..ad471c85 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -1,8 +1,15 @@ +""" +functionality : extracts the computational graph of a network defined as a `nn.Module` and converts it into a set of `DynapcnnLayer`s + that implement a network ()`DynapcnnNetwork`) instance that can be deployed to a Speck chip. +author : Willian Soares Girao +contact : williansoaresgirao@gmail.com +""" + import time -from subprocess import CalledProcessError -from typing import List, Optional, Sequence, Tuple, Union +from typing import List, Optional, Sequence, Tuple, Union, Dict import samna +import sinabs.layers as sl import torch import torch.nn as nn @@ -10,90 +17,196 @@ from .chip_factory import ChipFactory from .dvs_layer import DVSLayer -from .dynapcnn_layer import DynapcnnLayer -from .io import disable_timestamps, enable_timestamps, open_device, reset_timestamps +from .io import open_device from .utils import ( DEFAULT_IGNORED_LAYER_TYPES, - build_from_list, - convert_model_to_layer_list, - infer_input_shape, + build_from_graph, + build_nodes_to_dcnnl_map, parse_device_id, ) +from .NIRGraphExtractor import NIRtoDynapcnnNetworkGraph +from .sinabs_edges_handler import merge_handler -class DynapcnnNetwork(nn.Module): - """Given a sinabs spiking network, prepare a dynapcnn-compatible network. This can be used to - test the network will be equivalent once on DYNAPCNN. This class also provides utilities to - make the dynapcnn configuration and upload it to DYNAPCNN. - - The following operations are done when converting to dynapcnn-compatible: - - * multiple avg pooling layers in a row are consolidated into one and \ - turned into sum pooling layers; - * checks are performed on layer hyperparameter compatibility with dynapcnn \ - (kernel sizes, strides, padding) - * checks are performed on network structure compatibility with dynapcnn \ - (certain layers can only be followed by other layers) - * linear layers are turned into convolutional layers - * dropout layers are ignored - * weights, biases and thresholds are discretized according to dynapcnn requirements - - Note that the model parameters are only ever transferred to the device - on the `to` call, so changing a threshold or weight of a model that - is deployed will have no effect on the model on chip until `to` is called again. - """ +from .dynapcnnnetwork_module import DynapcnnNetworkModule +class DynapcnnNetwork(nn.Module): def __init__( self, - snn: Union[nn.Sequential, sinabs.Network], - input_shape: Optional[Tuple[int, int, int]] = None, + snn: nn.Module, + input_shape: Tuple[int, int, int], + batch_size: int, dvs_input: bool = False, - discretize: bool = True, + discretize: bool = True ): """ - DynapcnnNetwork: a class turning sinabs networks into dynapcnn - compatible networks, and making dynapcnn configurations. + Given a sinabs spiking network, prepare a dynapcnn-compatible network. This can be used to + test the network will be equivalent once on DYNAPCNN. This class also provides utilities to + make the dynapcnn configuration and upload it to DYNAPCNN. + + Some of the properties defined within the class constructor are meant to be temporary data structures handling the conversion + of the `snn` (the original `nn.Module`) into a set of `DynapcnnLayer`s composing a `DynapcnnNetwork` instance. Once their role + in preprocessing `snn` is finished, all required data to train/deploy the `DynapcnnNetwork` instance is within `self._dcnnl_edges` + (the connectivity between each `DynapcnnLayer`/core), `self._forward_map` (every `DynapcnnLayer` in the network) and `self._merge_points` + (the `DynapcnnLayer`s that need a `Merge` input). Thus, the following private properties are delted as last step of the constructor: + + - self._graph_tracer + - self._sinabs_edges + - self._sinabs_modules_map + - self._nodes_name_remap + - self._nodes_to_dcnnl_map + - self._dynapcnn_layers Parameters ---------- - snn: sinabs.Network - SNN that determines the structure of the `DynapcnnNetwork` - input_shape: None or tuple of ints - Shape of the input, convention: (features, height, width) - If None, `snn` needs an InputLayer - dvs_input: bool - Does dynapcnn receive input from its DVS camera? - discretize: bool - If True, discretize the parameters and thresholds. - This is needed for uploading weights to dynapcnn. Set to False only for - testing purposes. + snn (nn.Module): a implementing a spiking network. + input_shape (tuple): a description of the input dimensions as `(features, height, width)`. + dvs_input (bool): wether or not dynapcnn receive input from its DVS camera. + discretize (bool): If `True`, discretize the parameters and thresholds. This is needed for uploading + weights to dynapcnn. Set to `False` only for testing purposes. """ super().__init__() - # This attribute stores the location/core-id of each of the DynapcnnLayers upon placement on chip - self.chip_layers_ordering = [] + # TODO for now the graph part is not taking into consideration DVS inputs. + # check if dvs input is expected. + dvs_input = False + self.dvs_input = dvs_input + self.input_shape = input_shape + + assert len(self.input_shape) == 3, "infer_input_shape did not return 3-tuple" + + # computational graph from original PyTorch module. + # TODO - bacth size must be passed as argument. + self._graph_tracer = NIRtoDynapcnnNetworkGraph( + snn, + torch.randn((batch_size, *self.input_shape))) # needs the batch dimension. + + self._sinabs_edges, \ + self._sinabs_modules_map, \ + self._nodes_name_remap = self._get_sinabs_edges_and_modules() + + # create a dict holding the data necessary to instantiate a `DynapcnnLayer`. + self._nodes_to_dcnnl_map = build_nodes_to_dcnnl_map( + layers=self._sinabs_modules_map, + edges=self._sinabs_edges) + + # updates 'self._nodes_to_dcnnl_map' to include the I/O shape for each node. + self._populate_nodes_io() + + # build `DynapcnnLayer` instances from graph edges and mapper. + self._dynapcnn_layers = build_from_graph( + discretize=discretize, + edges=self._sinabs_edges, + nodes_to_dcnnl_map=self._nodes_to_dcnnl_map) + + # these gather all data necessay to implement the forward method for this class. + self._dcnnl_edges, self._forward_map, self._merge_points = self._get_network_module() + + # all necessary `DynapcnnLayer` data held in `self._forward_map`: removing intermediary data structures no longer necessary. + del self._graph_tracer + del self._sinabs_edges + del self._sinabs_modules_map + del self._nodes_name_remap + del self._nodes_to_dcnnl_map + del self._dynapcnn_layers + + ####################################################### Public Methods ####################################################### + + @property + def forward_map(self) -> dict: + """ This dictionary contains each `DynapcnnLayer` in the model indexed by their ID (layer index). """ + return self._forward_map + + def forward(self, x): + """ .""" - self.input_shape = input_shape # Convert models to sequential - layers = convert_model_to_layer_list( - model=snn, ignore=DEFAULT_IGNORED_LAYER_TYPES - ) - # Check if dvs input is expected - if dvs_input: - self.dvs_input = True - else: - self.dvs_input = False + layers_outputs = {} - input_shape = infer_input_shape(layers, input_shape=input_shape) - assert len(input_shape) == 3, "infer_input_shape did not return 3-tuple" + # TODO - currently `node 0` (this 1st node in the 1st edge of `self._dcnnl_edges`) is always taken to be the + # input node of the network. This won't work in cases where there are more the one input nodes to the network + # so this functionality needs some refactoring. + self._forward_map[self._dcnnl_edges[0][0]](x) - # Build model from layers - self.sequence = build_from_list( - layers, - in_shape=input_shape, - discretize=discretize, - dvs_input=self.dvs_input, - ) + # forward the input `x` through the input `DynapcnnLayer` in the `DynapcnnNetwork`s graph (1st node in the 1st edge in `self._dcnnl_edges`). + layers_outputs[self._dcnnl_edges[0][0]] = self._forward_map[self._dcnnl_edges[0][0]](x) + + # propagate outputs in `layers_outputs` through the rest of the nodes of `self._dcnnl_edges`. + for edge in self._dcnnl_edges: + + # target DynapcnnLayer (will consume tensors from `layers_outputs`). + trg_dcnnl = edge[1] + + if trg_dcnnl in self._merge_points and trg_dcnnl not in layers_outputs: + # by this points the arguments of the `Merge` associated with `trg_dcnnl` should have been computed. + arg1, arg2 = self._merge_points[trg_dcnnl]['sources'] + + # find which returned tensor from the `forward` call of DynapcnnLayers `arg1` and `arg2` are to be fed + # to the target DynapcnnLayer `trg_dcnnl`. + return_index_arg1 = self._forward_map[arg1].get_destination_dcnnl_index(trg_dcnnl) + return_index_arg2 = self._forward_map[arg2].get_destination_dcnnl_index(trg_dcnnl) + + # retrieve input tensors to `Merge`. + _arg1 = layers_outputs[arg1][return_index_arg1] + _arg2 = layers_outputs[arg2][return_index_arg2] + + # merge tensors. + merge_output = self._merge_points[trg_dcnnl]['merge'](_arg1, _arg2) + + # call the forward. + layers_outputs[trg_dcnnl] = self._forward_map[trg_dcnnl](merge_output) + + elif trg_dcnnl not in layers_outputs: + # input source for `trg_dcnnl`. + src_dcnnl = edge[0] + # find which returned tensor from the `forward` call of the source DynapcnnLayer `src_dcnnl` is to be fed + # to the target DynapcnnLayer `trg_dcnnl`. + return_index = self._forward_map[src_dcnnl].get_destination_dcnnl_index(trg_dcnnl) + + # call the forward. + layers_outputs[trg_dcnnl] = self._forward_map[trg_dcnnl](layers_outputs[src_dcnnl][return_index]) + + else: + + pass + + # TODO - this assumes the network has a single output node. + # last computed is the output layer. + return layers_outputs[trg_dcnnl][0] + + def parameters(self) -> list: + """ Gathers all the parameters of the network in a list. This is done by accessing the convolutional layer in each `DynapcnnLayer`, calling + its `.parameters` method and saving it to a list. + + Note: the method assumes no biases are used. + + Returns + ---------- + parameters (list): a list of parameters of all convolutional layers in the `DynapcnnNetwok`. + """ + parameters = [] + + for layer in self._forward_map.values(): + if isinstance(layer, sinabs.backend.dynapcnn.dynapcnn_layer_new.DynapcnnLayer): + parameters.extend(layer.conv_layer.parameters()) + + return parameters + + def init_weights(self, init_fn: nn.init = nn.init.xavier_normal_) -> None: + """ Call the weight initialization method `init_fn` on each `DynapcnnLayer.conv_layer.weight.data` in the `DynapcnnNetwork` instance.""" + for layer in self._forward_map.values(): + if isinstance(layer, sinabs.backend.dynapcnn.dynapcnn_layer_new.DynapcnnLayer): + init_fn(layer.conv_layer.weight.data) + + def detach_neuron_states(self) -> None: + """ Detach the neuron states and activations from current computation graph (necessary). """ + + for module in self._forward_map.values(): + if isinstance(module, sinabs.backend.dynapcnn.dynapcnn_layer_new.DynapcnnLayer): + if isinstance(module.spk_layer, sl.StatefulLayer): + for name, buffer in module.spk_layer.named_buffers(): + buffer.detach_() + def to( self, device="cpu", @@ -138,37 +251,43 @@ def to( For GPU or CPU usage these options are ignored. """ self.device = device + if isinstance(device, torch.device): - return super().to(device) + self._to_device(device) + elif isinstance(device, str): device_name, _ = parse_device_id(device) - if device_name in ChipFactory.supported_devices: # pragma: no cover - # Generate config - config = self.make_config( + + if device_name in ChipFactory.supported_devices: + + # generate config. + config = self._make_config( chip_layers_ordering=chip_layers_ordering, device=device, monitor_layers=monitor_layers, config_modifier=config_modifier, ) - # Apply configuration to device + # apply configuration to device. self.samna_device = open_device(device) self.samna_device.get_model().apply_configuration(config) time.sleep(1) - # Set external slow-clock if need + # set external slow-clock if needed. if slow_clk_frequency is not None: dk_io = self.samna_device.get_io_module() dk_io.set_slow_clk(True) - dk_io.set_slow_clk_rate(slow_clk_frequency) # Hz + dk_io.set_slow_clk_rate(slow_clk_frequency) # Hz builder = ChipFactory(device).get_config_builder() - # Create input source node + + # create input source node. self.samna_input_buffer = builder.get_input_buffer() - # Create output sink node node + + # create output sink node node. self.samna_output_buffer = builder.get_output_buffer() - # Connect source node to device sink + # connect source node to device sink. self.device_input_graph = samna.graph.EventFilterGraph() self.device_input_graph.sequential( [ @@ -177,7 +296,7 @@ def to( ] ) - # Connect sink node to device + # connect sink node to device. self.device_output_graph = samna.graph.EventFilterGraph() self.device_output_graph.sequential( [ @@ -185,14 +304,20 @@ def to( self.samna_output_buffer, ] ) + self.device_input_graph.start() self.device_output_graph.start() self.samna_config = config - return self + + return print(self) + else: - return super().to(device) + self._to_device(device) + else: raise Exception("Unknown device description.") + + ####################################################### Private Methods ####################################################### def _make_config( self, @@ -200,8 +325,8 @@ def _make_config( device="dynapcnndevkit:0", monitor_layers: Optional[Union[List, str]] = None, config_modifier=None, - ) -> Tuple["SamnaConfiguration", bool]: - """Prepare and output the `samna` configuration for this network. + ): + """Prepare and output the `samna` DYNAPCNN configuration for this network. Parameters ---------- @@ -210,7 +335,6 @@ def _make_config( The order in which the dynapcnn layers will be used. If `auto`, an automated procedure will be used to find a valid ordering. A list of layers on the device where you want each of the model's DynapcnnLayers to be placed. - The index of the core on chip to which the i-th layer in the model is mapped is the value of the i-th entry in the list. Note: This list should be the same length as the number of dynapcnn layers in your model. device: String @@ -235,274 +359,210 @@ def _make_config( ------- Configuration object Object defining the configuration for the device - Bool - True if the configuration is valid for the given device. Raises ------ ImportError If samna is not available. + ValueError + If the generated configuration is not valid for the specified device. """ config_builder = ChipFactory(device).get_config_builder() - has_dvs_layer = isinstance(self.sequence[0], DVSLayer) + # TODO not handling DVSLayer yet. + has_dvs_layer = isinstance(self._forward_map[0], DVSLayer) - # Figure out layer ordering if chip_layers_ordering == "auto": + # figure out mapping of each DynapcnnLayer into one core. chip_layers_ordering = config_builder.get_valid_mapping(self) + else: - # Truncate chip_layers_ordering just in case a longer list is passed + # TODO - mapping from each DynapcnnLayer into cores has been provided by the user: NOT IMPLEMENTED YET. if has_dvs_layer: - chip_layers_ordering = chip_layers_ordering[: len(self.sequence) - 1] - chip_layers_ordering = chip_layers_ordering[: len(self.sequence)] + # TODO not handling DVSLayer yet. + pass - # Save the chip layers - self.chip_layers_ordering = chip_layers_ordering - # Update config - config = config_builder.build_config(self, chip_layers_ordering) + # update config. + config = config_builder.build_config(self, None) + + # TODO not handling DVSLayer yet (this is from the old implementation, should be revised). if self.input_shape and self.input_shape[0] == 1: config.dvs_layer.merge = True - # Check if any monitoring is enabled and if not, enable monitoring for the last layer + + # TODO all this monitoring part needs validation still. + monitor_chip_layers = [] if monitor_layers is None: - monitor_layers = [-1] + # check if any monitoring is enabled (if not, enable monitoring for the last layer). + for dcnnl_index, ith_dcnnl in self._forward_map.items(): + if len(ith_dcnnl.dynapcnnlayer_destination) == 0: + monitor_chip_layers.append(ith_dcnnl.assigned_core) + break elif monitor_layers == "all": - num_cnn_layers = len(self.sequence) - int(has_dvs_layer) - monitor_layers = list(range(num_cnn_layers)) - - # Enable monitors on the specified layers - # Find layers corresponding to the chip - monitor_chip_layers = [ - self.find_chip_layer(lyr) for lyr in monitor_layers if lyr != "dvs" - ] - if "dvs" in monitor_layers: - monitor_chip_layers.append("dvs") - + for dcnnl_index, ith_dcnnl in self._forward_map.items(): + # TODO not handling DVSLayer yet + # monitor each chip core (if not a DVSLayer). + if not isinstance(ith_dcnnl, DVSLayer): + monitor_chip_layers.append(ith_dcnnl.assigned_core) + + if monitor_layers: + if "dvs" in monitor_layers: + monitor_chip_layers.append("dvs") + + # enable monitors on the specified layers. config_builder.monitor_layers(config, monitor_chip_layers) - # Fix default factory setting to not return input events (UGLY!! Ideally this should happen in samna) - # config.factory_settings.monitor_input_enable = False - - # Apply user config modifier if config_modifier is not None: + # apply user config modifier. config = config_modifier(config) - # Validate config - return config, config_builder.validate_configuration(config) - - def make_config( - self, - chip_layers_ordering: Union[Sequence[int], str] = "auto", - device="dynapcnndevkit:0", - monitor_layers: Optional[Union[List, str]] = None, - config_modifier=None, - ): - """Prepare and output the `samna` DYNAPCNN configuration for this network. - - Parameters - ---------- - - chip_layers_ordering: sequence of integers or `auto` - The order in which the dynapcnn layers will be used. If `auto`, - an automated procedure will be used to find a valid ordering. - A list of layers on the device where you want each of the model's DynapcnnLayers to be placed. - Note: This list should be the same length as the number of dynapcnn layers in your model. - - device: String - dynapcnndevkit, speck2b or speck2devkit - - monitor_layers: None/List/Str - A list of all layers in the module that you want to monitor. Indexing starts with the first non-dvs layer. - If you want to monitor the dvs layer for eg. - :: - - monitor_layers = ["dvs"] # If you want to monitor the output of the pre-processing layer - monitor_layers = ["dvs", 8] # If you want to monitor preprocessing and layer 8 - monitor_layers = "all" # If you want to monitor all the layers - - If this value is left as None, by default the last layer of the model is monitored. - - config_modifier: - A user configuration modifier method. - This function can be used to make any custom changes you want to make to the configuration object. - - Returns - ------- - Configuration object - Object defining the configuration for the device - - Raises - ------ - ImportError - If samna is not available. - ValueError - If the generated configuration is not valid for the specified device. - """ - config, is_compatible = self._make_config( - chip_layers_ordering=chip_layers_ordering, - device=device, - monitor_layers=monitor_layers, - config_modifier=config_modifier, - ) - # Validate config - if is_compatible: - print("Network is valid") + if config_builder.validate_configuration(config): + # validate config. + print("Network is valid: \n") + return config else: raise ValueError(f"Generated config is not valid for {device}") - def is_compatible_with(self, device_type: str) -> bool: - """Check if the current model is compatible with a given device. + def _get_network_module(self) -> Union[list, dict, dict]: + """ Uses the `DynapcnnLayer` instances in `self._dynapcnn_layers` and the connectivity between them to create three data structures + that guide the data forwarding between the layer during the forward pass. - Args: - device_type (str): Device type ie speck2b, speck2fmodule - - Returns: - bool: True if compatible - """ - try: - _, is_compatible = self._make_config(device=device_type) - except ValueError as e: - # Catch "No valid mapping found" error - if e.args[0] == ("No valid mapping found"): - return False - else: - raise e - return is_compatible - - def reset_states(self, randomize=False): - """Reset the states of the network.""" - if hasattr(self, "device") and isinstance(self.device, str): # pragma: no cover - device_name, _ = parse_device_id(self.device) - if device_name in ChipFactory.supported_devices: - config_builder = ChipFactory(self.device).get_config_builder() - # Set all the vmem states in the samna config to zero - config_builder.reset_states(self.samna_config, randomize=randomize) - self.samna_device.get_model().apply_configuration(self.samna_config) - # wait for the config to be written - time.sleep(1) - # Note: The below shouldn't be necessary ideally - # Erase all vmem memory - if not randomize: - if hasattr(self, "samna_input_graph"): - self.samna_input_graph.stop() - for lyr_idx in self.chip_layers_ordering: - config_builder.set_all_v_mem_to_zeros( - self.samna_device, lyr_idx - ) - time.sleep(0.1) - self.samna_input_graph.start() - return - for layer in self.sequence: - if isinstance(layer, DynapcnnLayer): - layer.spk_layer.reset_states(randomize=randomize) - - def find_chip_layer(self, layer_idx): - """Given an index of a layer in the model, find the corresponding cnn core id where it is - placed. - - > Note that the layer index does not include the DVSLayer. - > For instance your model comprises two layers [DVSLayer, DynapcnnLayer], - > then the index of DynapcnnLayer is 0 and not 1. - - Parameters - ---------- - layer_idx: int - Index of a layer + Note: the property `DynapcnnLayer.assigned_core` is only set after `self.to(device='speck...')` is called. Returns - ------- - chip_lyr_idx: int - Index of the layer on the chip where the model layer is placed. + ---------- + dcnnl_edges (list): edges, represented as tuples of `DynapcnnLayer` indices, used to guide the data forwarding through each `DynapcnnLayer` in forward method. + forward_map (dict): have all the `DynapcnnLayer` (`value`), each being accessible via its index (`key`). Used to call `DynapcnnLayer.forward` in forward method. + merge_points (dict): used to compose the inputs to a `DynapcnnLayer` that requires an input from a `Merge` layer. """ - # Compute the expected number of cores - num_cores_required = len(self.sequence) - if isinstance(self.sequence[0], DVSLayer): - num_cores_required -= 1 - if len(self.chip_layers_ordering) != num_cores_required: - raise Exception( - f"Number of layers specified in chip_layers_ordering {self.chip_layers_ordering} does not correspond to the number of cores required for this model {num_cores_required}" - ) - return self.chip_layers_ordering[layer_idx] + # get connections between `DynapcnnLayer`s. + dcnnl_edges = self._get_dynapcnnlayers_edges() - def forward(self, x): - if ( - hasattr(self, "device") - and parse_device_id(self.device)[0] in ChipFactory.supported_devices - ): # pragma: no cover - _ = self.samna_output_buffer.get_events() # Flush buffer - # NOTE: The code to start and stop time stamping is device specific - reset_timestamps(self.device) - enable_timestamps(self.device) - # Send input - self.samna_input_buffer.write(x) - received_evts = [] - # Record at least until the last event has been replayed - min_duration = max(event.timestamp for event in x) * 1e-6 - time.sleep(min_duration) - # Keep recording if more events are being registered - while True: - prev_length = len(received_evts) - time.sleep(0.1) - received_evts.extend(self.samna_output_buffer.get_events()) - if prev_length == len(received_evts): - break - # Disable timestamp - disable_timestamps(self.device) - return received_evts - else: - """Torch's forward pass.""" - return self.sequence(x) + dcnnnet_module = DynapcnnNetworkModule(dcnnl_edges, self._dynapcnn_layers) + + return dcnnnet_module.dcnnl_edges, dcnnnet_module.forward_map, dcnnnet_module.merge_points + + def _get_dynapcnnlayers_edges(self) -> List[Tuple[int, int]]: + """ Create edges representing connections between `DynapcnnLayer` instances. """ + dcnnl_edges = [] - def memory_summary(self): - """Get a summary of the network's memory requirements. + for dcnnl_idx, layer_data in self._dynapcnn_layers.items(): + for dest in layer_data['destinations']: + dcnnl_edges.append((dcnnl_idx, dest)) + + return dcnnl_edges + + def _get_sinabs_edges_and_modules(self) -> Tuple[List[Tuple[int, int]], Dict[int, nn.Module], Dict[int, int]]: + """ The computational graph extracted from `snn` might contain layers that are ignored (e.g. a `nn.Flatten` will be + ignored when creating a `DynapcnnLayer` instance). Thus the list of edges from such model need to be rebuilt such that if there are + edges `(A, X)` and `(X, B)`, and `X` is an ignored layer, an edge `(A, B)` is created. Returns - ------- - dict: - A dictionary with keys kernel, neuron, bias. - The values are a list of the corresponding number per layer in the same order as the model + ---------- + edges_without_merge (list): a list of edges based on `sinabs_edges` but where edges involving a `Merge` layer have been + remapped to connect the nodes involved in the merging directly. + sinabs_modules_map (dict): a dict containing the nodes of the graph (described now by `edges_without_merge`) as `key` and + their associated module as `value`. + remapped_nodes (dict): a dict where `key` is the original node name (as extracted by `self._graph_tracer`) and `value` is + the new node name (after ignored layers have been dropped and `Merge` layers have be processed before being removed). """ - summary = {} - - dynapcnn_layers = [ - lyr for lyr in self.sequence if isinstance(lyr, DynapcnnLayer) - ] - summary.update({k: list() for k in dynapcnn_layers[0].memory_summary().keys()}) - for lyr in dynapcnn_layers: - lyr_summary = lyr.memory_summary() - for k, v in lyr_summary.items(): - summary[k].append(v) - return summary - - def zero_grad(self, set_to_none: bool = False) -> None: - for lyr in self.sequence: - lyr.zero_grad(set_to_none) - - def __del__(self): - # Stop the input graph - if hasattr(self, "device_input_graph") and self.device_input_graph: - self.device_input_graph.stop() - - # Stop the output graph. - if hasattr(self, "device_output_graph") and self.device_output_graph: - self.device_output_graph.stop() - - -class DynapcnnCompatibleNetwork(DynapcnnNetwork): - """Deprecated class, use DynapcnnNetwork instead.""" - - def __init__( - self, - snn: Union[nn.Sequential, sinabs.Network], - input_shape: Optional[Tuple[int, int, int]] = None, - dvs_input: bool = False, - discretize: bool = True, - ): - from warnings import warn - - warn( - "DynapcnnCompatibleNetwork has been renamed to DynapcnnNetwork " - + "and will be removed in a future release." - ) - super().__init__(snn, input_shape, dvs_input, discretize) + + # remap `(A, X)` and `(X, B)` into `(A, B)` if `X` is a layer in the original `snn` to be ignored. + sinabs_edges, remapped_nodes = self._graph_tracer.remove_ignored_nodes( + DEFAULT_IGNORED_LAYER_TYPES) + + # nodes (layers' "names") need remapping in case some layers have been removed (e.g. a `nn.Flattern` is ignored). + sinabs_modules_map = {} + for orig_name, new_name in remapped_nodes.items(): + sinabs_modules_map[new_name] = self._graph_tracer.modules_map[orig_name] + + # bypass merging layers to connect the nodes involved in them directly to the node where the merge happens. + edges_without_merge = merge_handler(sinabs_edges, sinabs_modules_map) + + return edges_without_merge, sinabs_modules_map, remapped_nodes + + def _populate_nodes_io(self): + """ Loops through the nodes in the original graph to retrieve their I/O tensor shapes and add them to their respective + representations in `self._nodes_to_dcnnl_map`.""" + + def find_original_node_name(name_mapper: dict, node: int): + """ Find what a node is originally named when built in `self._graph_tracer`. """ + for orig_name, new_name in name_mapper.items(): + if new_name == node: + return orig_name + raise ValueError(f'Node {node} could not be found within the name remapping done by self._get_sinabs_edges_and_modules().') + + def find_my_input(edges_list: list, node: int) -> int: + """ Returns the node `X` in the first edge `(X, node)`. + + Parameters + ---------- + node (int): the node in the computational graph for which we whish to find the input source (either another node in the + graph or the original input itself to the network). + + Returns + ---------- + input source (int): this indicates the node in the computational graph providing the input to `node`. If `node` is + receiving outside input (i.e., it is a starting node) the return will be -1. For example, this will be the case + when a network with two independent branches (each starts from a different "input node") merge along the computational graph. + """ + for edge in edges_list: + if edge[1] == node: + # TODO nodes originally receiving input from merge will appear twice in the list of edges, one + # edge per input to the merge layer. For now both inputs to a `Merge` have the same dimensions + # necessarily so this works for now but later will have to be revised. + return edge[0] + return -1 + + # access the I/O shapes for each node in `self._sinabs_edges` from the original graph in `self._graph_tracer`. + for dcnnl_idx, dcnnl_data in self._nodes_to_dcnnl_map.items(): + for node, node_data in dcnnl_data.items(): + # node dictionary with layer data. + if isinstance(node, int): + # some nodes might have been renamed (e.g. after droppping a `nn.Flatten`), so find how node was originally named. + orig_name = find_original_node_name(self._nodes_name_remap, node) + _in, _out = self._graph_tracer.get_node_io_shapes(orig_name) + + # update node I/O shape in the mapper (drop batch dimension). + if node != 0: + # Find node outputing into the current node being processed (this will be the input shape). This is + # necessary cuz if a node originally receives input from a `nn.Flatten` for instance, when mapped into + # a `DynapcnnLayer` it will be receiving the input from a privious `sl.SumPool2d`. + input_node = find_my_input(self._sinabs_edges, node) + + if input_node == -1: + # node does not have an input source within the graph (it consumes the original input to the model). + node_data['input_shape'] = tuple(list(_in)[1:]) + else: + # input comes from another node in the graph. + input_node_orig_name = find_original_node_name(self._nodes_name_remap, input_node) + _, _input_source_shape = self._graph_tracer.get_node_io_shapes(input_node_orig_name) + node_data['input_shape'] = tuple(list(_input_source_shape)[1:]) + else: + # first node does not have an input source within the graph. + node_data['input_shape'] = tuple(list(_in)[1:]) + + node_data['output_shape'] = tuple(list(_out)[1:]) + + def _to_device(self, device: torch.device) -> None: + """ .""" + for layer in self._forward_map.values(): + if isinstance(layer, sinabs.backend.dynapcnn.dynapcnn_layer_new.DynapcnnLayer): + layer.conv_layer.to(device) + layer.spk_layer.to(device) + + # if there's more than one pooling each of them becomes a node that is catched by the `else` statement. + if len(layer.pool_layer) == 1: + layer.pool_layer[0].to(device) + else: + # this nodes are created from `DynapcnnLayer`s that have multiple poolings (each pooling becomes a new node). + layer.to(device) + + def __str__(self): + pretty_print = '' + for idx, layer_data in self._forward_map.items(): + pretty_print += f'----------------------- [ DynapcnnLayer {idx} ] -----------------------\n' + pretty_print += f'{layer_data}\n\n' + + return pretty_print \ No newline at end of file diff --git a/sinabs/backend/dynapcnn/dynapcnn_network_deprecated.py b/sinabs/backend/dynapcnn/dynapcnn_network_deprecated.py new file mode 100644 index 00000000..9d59236f --- /dev/null +++ b/sinabs/backend/dynapcnn/dynapcnn_network_deprecated.py @@ -0,0 +1,508 @@ +import time +from subprocess import CalledProcessError +from typing import List, Optional, Sequence, Tuple, Union + +import samna +import torch +import torch.nn as nn + +import sinabs + +from .chip_factory import ChipFactory +from .dvs_layer import DVSLayer +from .dynapcnn_layer import DynapcnnLayer +from .io import disable_timestamps, enable_timestamps, open_device, reset_timestamps +from .utils import ( + DEFAULT_IGNORED_LAYER_TYPES, + build_from_list, + convert_model_to_layer_list, + infer_input_shape, + parse_device_id, +) + + +class DynapcnnNetwork(nn.Module): + """Given a sinabs spiking network, prepare a dynapcnn-compatible network. This can be used to + test the network will be equivalent once on DYNAPCNN. This class also provides utilities to + make the dynapcnn configuration and upload it to DYNAPCNN. + + The following operations are done when converting to dynapcnn-compatible: + + * multiple avg pooling layers in a row are consolidated into one and \ + turned into sum pooling layers; + * checks are performed on layer hyperparameter compatibility with dynapcnn \ + (kernel sizes, strides, padding) + * checks are performed on network structure compatibility with dynapcnn \ + (certain layers can only be followed by other layers) + * linear layers are turned into convolutional layers + * dropout layers are ignored + * weights, biases and thresholds are discretized according to dynapcnn requirements + + Note that the model parameters are only ever transferred to the device + on the `to` call, so changing a threshold or weight of a model that + is deployed will have no effect on the model on chip until `to` is called again. + """ + + def __init__( + self, + snn: Union[nn.Sequential, sinabs.Network], + input_shape: Optional[Tuple[int, int, int]] = None, + dvs_input: bool = False, + discretize: bool = True, + ): + """ + DynapcnnNetwork: a class turning sinabs networks into dynapcnn + compatible networks, and making dynapcnn configurations. + + Parameters + ---------- + snn: sinabs.Network + SNN that determines the structure of the `DynapcnnNetwork` + input_shape: None or tuple of ints + Shape of the input, convention: (features, height, width) + If None, `snn` needs an InputLayer + dvs_input: bool + Does dynapcnn receive input from its DVS camera? + discretize: bool + If True, discretize the parameters and thresholds. + This is needed for uploading weights to dynapcnn. Set to False only for + testing purposes. + """ + super().__init__() + + # This attribute stores the location/core-id of each of the DynapcnnLayers upon placement on chip + self.chip_layers_ordering = [] + + self.input_shape = input_shape # Convert models to sequential + layers = convert_model_to_layer_list( + model=snn, ignore=DEFAULT_IGNORED_LAYER_TYPES + ) + # Check if dvs input is expected + if dvs_input: + self.dvs_input = True + else: + self.dvs_input = False + + input_shape = infer_input_shape(layers, input_shape=input_shape) + assert len(input_shape) == 3, "infer_input_shape did not return 3-tuple" + + # Build model from layers + self.sequence = build_from_list( + layers, + in_shape=input_shape, + discretize=discretize, + dvs_input=self.dvs_input, + ) + + def to( + self, + device="cpu", + chip_layers_ordering="auto", + monitor_layers: Optional[Union[List, str]] = None, + config_modifier=None, + slow_clk_frequency: int = None, + ): + """Note that the model parameters are only ever transferred to the device on the `to` call, + so changing a threshold or weight of a model that is deployed will have no effect on the + model on chip until `to` is called again. + + Parameters + ---------- + + device: String + cpu:0, cuda:0, dynapcnndevkit, speck2devkit + + chip_layers_ordering: sequence of integers or `auto` + The order in which the dynapcnn layers will be used. If `auto`, + an automated procedure will be used to find a valid ordering. + A list of layers on the device where you want each of the model's DynapcnnLayers to be placed. + The index of the core on chip to which the i-th layer in the model is mapped is the value of the i-th entry in the list. + Note: This list should be the same length as the number of dynapcnn layers in your model. + + monitor_layers: None/List + A list of all layers in the module that you want to monitor. Indexing starts with the first non-dvs layer. + If you want to monitor the dvs layer for eg. + :: + + monitor_layers = ["dvs"] # If you want to monitor the output of the pre-processing layer + monitor_layers = ["dvs", 8] # If you want to monitor preprocessing and layer 8 + monitor_layers = "all" # If you want to monitor all the layers + + config_modifier: + A user configuration modifier method. + This function can be used to make any custom changes you want to make to the configuration object. + + Note + ---- + chip_layers_ordering and monitor_layers are used only when using synsense devices. + For GPU or CPU usage these options are ignored. + """ + self.device = device + if isinstance(device, torch.device): + return super().to(device) + elif isinstance(device, str): + device_name, _ = parse_device_id(device) + if device_name in ChipFactory.supported_devices: # pragma: no cover + # Generate config + config = self.make_config( + chip_layers_ordering=chip_layers_ordering, + device=device, + monitor_layers=monitor_layers, + config_modifier=config_modifier, + ) + + # Apply configuration to device + self.samna_device = open_device(device) + self.samna_device.get_model().apply_configuration(config) + time.sleep(1) + + # Set external slow-clock if need + if slow_clk_frequency is not None: + dk_io = self.samna_device.get_io_module() + dk_io.set_slow_clk(True) + dk_io.set_slow_clk_rate(slow_clk_frequency) # Hz + + builder = ChipFactory(device).get_config_builder() + # Create input source node + self.samna_input_buffer = builder.get_input_buffer() + # Create output sink node node + self.samna_output_buffer = builder.get_output_buffer() + + # Connect source node to device sink + self.device_input_graph = samna.graph.EventFilterGraph() + self.device_input_graph.sequential( + [ + self.samna_input_buffer, + self.samna_device.get_model().get_sink_node(), + ] + ) + + # Connect sink node to device + self.device_output_graph = samna.graph.EventFilterGraph() + self.device_output_graph.sequential( + [ + self.samna_device.get_model().get_source_node(), + self.samna_output_buffer, + ] + ) + self.device_input_graph.start() + self.device_output_graph.start() + self.samna_config = config + return self + else: + return super().to(device) + else: + raise Exception("Unknown device description.") + + def _make_config( + self, + chip_layers_ordering: Union[Sequence[int], str] = "auto", + device="dynapcnndevkit:0", + monitor_layers: Optional[Union[List, str]] = None, + config_modifier=None, + ) -> Tuple["SamnaConfiguration", bool]: + """Prepare and output the `samna` configuration for this network. + + Parameters + ---------- + + chip_layers_ordering: sequence of integers or `auto` + The order in which the dynapcnn layers will be used. If `auto`, + an automated procedure will be used to find a valid ordering. + A list of layers on the device where you want each of the model's DynapcnnLayers to be placed. + The index of the core on chip to which the i-th layer in the model is mapped is the value of the i-th entry in the list. + Note: This list should be the same length as the number of dynapcnn layers in your model. + + device: String + dynapcnndevkit, speck2b or speck2devkit + + monitor_layers: None/List/Str + A list of all layers in the module that you want to monitor. Indexing starts with the first non-dvs layer. + If you want to monitor the dvs layer for eg. + :: + + monitor_layers = ["dvs"] # If you want to monitor the output of the pre-processing layer + monitor_layers = ["dvs", 8] # If you want to monitor preprocessing and layer 8 + monitor_layers = "all" # If you want to monitor all the layers + + If this value is left as None, by default the last layer of the model is monitored. + + config_modifier: + A user configuration modifier method. + This function can be used to make any custom changes you want to make to the configuration object. + + Returns + ------- + Configuration object + Object defining the configuration for the device + Bool + True if the configuration is valid for the given device. + + Raises + ------ + ImportError + If samna is not available. + """ + config_builder = ChipFactory(device).get_config_builder() + + has_dvs_layer = isinstance(self.sequence[0], DVSLayer) + + # Figure out layer ordering + if chip_layers_ordering == "auto": + chip_layers_ordering = config_builder.get_valid_mapping(self) + else: + # Truncate chip_layers_ordering just in case a longer list is passed + if has_dvs_layer: + chip_layers_ordering = chip_layers_ordering[: len(self.sequence) - 1] + chip_layers_ordering = chip_layers_ordering[: len(self.sequence)] + + # Save the chip layers + self.chip_layers_ordering = chip_layers_ordering + # Update config + config = config_builder.build_config(self, chip_layers_ordering) + if self.input_shape and self.input_shape[0] == 1: + config.dvs_layer.merge = True + # Check if any monitoring is enabled and if not, enable monitoring for the last layer + if monitor_layers is None: + monitor_layers = [-1] + elif monitor_layers == "all": + num_cnn_layers = len(self.sequence) - int(has_dvs_layer) + monitor_layers = list(range(num_cnn_layers)) + + # Enable monitors on the specified layers + # Find layers corresponding to the chip + monitor_chip_layers = [ + self.find_chip_layer(lyr) for lyr in monitor_layers if lyr != "dvs" + ] + if "dvs" in monitor_layers: + monitor_chip_layers.append("dvs") + + config_builder.monitor_layers(config, monitor_chip_layers) + + # Fix default factory setting to not return input events (UGLY!! Ideally this should happen in samna) + # config.factory_settings.monitor_input_enable = False + + # Apply user config modifier + if config_modifier is not None: + config = config_modifier(config) + + # Validate config + return config, config_builder.validate_configuration(config) + + def make_config( + self, + chip_layers_ordering: Union[Sequence[int], str] = "auto", + device="dynapcnndevkit:0", + monitor_layers: Optional[Union[List, str]] = None, + config_modifier=None, + ): + """Prepare and output the `samna` DYNAPCNN configuration for this network. + + Parameters + ---------- + + chip_layers_ordering: sequence of integers or `auto` + The order in which the dynapcnn layers will be used. If `auto`, + an automated procedure will be used to find a valid ordering. + A list of layers on the device where you want each of the model's DynapcnnLayers to be placed. + Note: This list should be the same length as the number of dynapcnn layers in your model. + + device: String + dynapcnndevkit, speck2b or speck2devkit + + monitor_layers: None/List/Str + A list of all layers in the module that you want to monitor. Indexing starts with the first non-dvs layer. + If you want to monitor the dvs layer for eg. + :: + + monitor_layers = ["dvs"] # If you want to monitor the output of the pre-processing layer + monitor_layers = ["dvs", 8] # If you want to monitor preprocessing and layer 8 + monitor_layers = "all" # If you want to monitor all the layers + + If this value is left as None, by default the last layer of the model is monitored. + + config_modifier: + A user configuration modifier method. + This function can be used to make any custom changes you want to make to the configuration object. + + Returns + ------- + Configuration object + Object defining the configuration for the device + + Raises + ------ + ImportError + If samna is not available. + ValueError + If the generated configuration is not valid for the specified device. + """ + config, is_compatible = self._make_config( + chip_layers_ordering=chip_layers_ordering, + device=device, + monitor_layers=monitor_layers, + config_modifier=config_modifier, + ) + # Validate config + if is_compatible: + print("Network is valid") + return config + else: + raise ValueError(f"Generated config is not valid for {device}") + + def is_compatible_with(self, device_type: str) -> bool: + """Check if the current model is compatible with a given device. + + Args: + device_type (str): Device type ie speck2b, speck2fmodule + + Returns: + bool: True if compatible + """ + try: + _, is_compatible = self._make_config(device=device_type) + except ValueError as e: + # Catch "No valid mapping found" error + if e.args[0] == ("No valid mapping found"): + return False + else: + raise e + return is_compatible + + def reset_states(self, randomize=False): + """Reset the states of the network.""" + if hasattr(self, "device") and isinstance(self.device, str): # pragma: no cover + device_name, _ = parse_device_id(self.device) + if device_name in ChipFactory.supported_devices: + config_builder = ChipFactory(self.device).get_config_builder() + # Set all the vmem states in the samna config to zero + config_builder.reset_states(self.samna_config, randomize=randomize) + self.samna_device.get_model().apply_configuration(self.samna_config) + # wait for the config to be written + time.sleep(1) + # Note: The below shouldn't be necessary ideally + # Erase all vmem memory + if not randomize: + if hasattr(self, "samna_input_graph"): + self.samna_input_graph.stop() + for lyr_idx in self.chip_layers_ordering: + config_builder.set_all_v_mem_to_zeros( + self.samna_device, lyr_idx + ) + time.sleep(0.1) + self.samna_input_graph.start() + return + for layer in self.sequence: + if isinstance(layer, DynapcnnLayer): + layer.spk_layer.reset_states(randomize=randomize) + + def find_chip_layer(self, layer_idx): + """Given an index of a layer in the model, find the corresponding cnn core id where it is + placed. + + > Note that the layer index does not include the DVSLayer. + > For instance your model comprises two layers [DVSLayer, DynapcnnLayer], + > then the index of DynapcnnLayer is 0 and not 1. + + Parameters + ---------- + layer_idx: int + Index of a layer + + Returns + ------- + chip_lyr_idx: int + Index of the layer on the chip where the model layer is placed. + """ + # Compute the expected number of cores + num_cores_required = len(self.sequence) + if isinstance(self.sequence[0], DVSLayer): + num_cores_required -= 1 + if len(self.chip_layers_ordering) != num_cores_required: + raise Exception( + f"Number of layers specified in chip_layers_ordering {self.chip_layers_ordering} does not correspond to the number of cores required for this model {num_cores_required}" + ) + + return self.chip_layers_ordering[layer_idx] + + def forward(self, x): + if ( + hasattr(self, "device") + and parse_device_id(self.device)[0] in ChipFactory.supported_devices + ): # pragma: no cover + _ = self.samna_output_buffer.get_events() # Flush buffer + # NOTE: The code to start and stop time stamping is device specific + reset_timestamps(self.device) + enable_timestamps(self.device) + # Send input + self.samna_input_buffer.write(x) + received_evts = [] + # Record at least until the last event has been replayed + min_duration = max(event.timestamp for event in x) * 1e-6 + time.sleep(min_duration) + # Keep recording if more events are being registered + while True: + prev_length = len(received_evts) + time.sleep(0.1) + received_evts.extend(self.samna_output_buffer.get_events()) + if prev_length == len(received_evts): + break + # Disable timestamp + disable_timestamps(self.device) + return received_evts + else: + """Torch's forward pass.""" + return self.sequence(x) + + def memory_summary(self): + """Get a summary of the network's memory requirements. + + Returns + ------- + dict: + A dictionary with keys kernel, neuron, bias. + The values are a list of the corresponding number per layer in the same order as the model + """ + summary = {} + + dynapcnn_layers = [ + lyr for lyr in self.sequence if isinstance(lyr, DynapcnnLayer) + ] + summary.update({k: list() for k in dynapcnn_layers[0].memory_summary().keys()}) + for lyr in dynapcnn_layers: + lyr_summary = lyr.memory_summary() + for k, v in lyr_summary.items(): + summary[k].append(v) + return summary + + def zero_grad(self, set_to_none: bool = False) -> None: + for lyr in self.sequence: + lyr.zero_grad(set_to_none) + + def __del__(self): + # Stop the input graph + if hasattr(self, "device_input_graph") and self.device_input_graph: + self.device_input_graph.stop() + + # Stop the output graph. + if hasattr(self, "device_output_graph") and self.device_output_graph: + self.device_output_graph.stop() + + +class DynapcnnCompatibleNetwork(DynapcnnNetwork): + """Deprecated class, use DynapcnnNetwork instead.""" + + def __init__( + self, + snn: Union[nn.Sequential, sinabs.Network], + input_shape: Optional[Tuple[int, int, int]] = None, + dvs_input: bool = False, + discretize: bool = True, + ): + from warnings import warn + + warn( + "DynapcnnCompatibleNetwork has been renamed to DynapcnnNetwork " + + "and will be removed in a future release." + ) + super().__init__(snn, input_shape, dvs_input, discretize) diff --git a/sinabs/backend/dynapcnn/mapping.py b/sinabs/backend/dynapcnn/mapping.py index 97ed0f5b..c93c6b3e 100644 --- a/sinabs/backend/dynapcnn/mapping.py +++ b/sinabs/backend/dynapcnn/mapping.py @@ -4,8 +4,7 @@ from typing import List, Optional, Tuple, Union from .dvs_layer import DVSLayer -# from .dynapcnn_layer import DynapcnnLayer -from .dynapcnn_layer_new import DynapcnnLayer +from .dynapcnn_layer import DynapcnnLayer import sinabs from .exceptions import InvalidModel @@ -48,7 +47,7 @@ def find_chip_layers( def get_valid_mapping( - model: Union["DynapcnnNetwork", "DynapcnnNetworkGraph"], constraints: List[LayerConstraints] + model: Union["DynapcnnNetwork"], constraints: List[LayerConstraints] ) -> List[Tuple[int, int]]: """Given a model, find a valid layer ordering for its placement within the constraints provided. @@ -65,18 +64,6 @@ def get_valid_mapping( layer_mapping = [] if type(model) == sinabs.backend.dynapcnn.dynapcnn_network.DynapcnnNetwork: - for layer in model.sequence: - if isinstance(layer, DynapcnnLayer): - layer_mapping.append(find_chip_layers(layer, constraints)) - - graph = make_flow_graph(layer_mapping, len(constraints)) - - # use graph algorithm to find suitable cores for each DynapcnnLayer. - new_graph = edmonds(graph, 0, len(graph) - 1) - - netmap = recover_mapping(new_graph, layer_mapping) - - elif type(model) == sinabs.backend.dynapcnn.dynapcnn_network_graph.DynapcnnNetworkGraph: for dcnnl_index, ith_dcnnl in model.forward_map.items(): if isinstance(ith_dcnnl, DynapcnnLayer): layer_mapping.append(find_chip_layers(ith_dcnnl, constraints)) diff --git a/sinabs/backend/dynapcnn/utils.py b/sinabs/backend/dynapcnn/utils.py index 47f79f0a..658aa1b9 100644 --- a/sinabs/backend/dynapcnn/utils.py +++ b/sinabs/backend/dynapcnn/utils.py @@ -9,8 +9,7 @@ from .crop2d import Crop2d from .dvs_layer import DVSLayer, expand_to_pair -#from .dynapcnn_layer import DynapcnnLayer -from .dynapcnn_layer_new import DynapcnnLayer +from .dynapcnn_layer import DynapcnnLayer from .exceptions import InputConfigurationError, MissingLayer, UnexpectedLayer, WrongModuleCount, WrongPoolingModule from .flipdims import FlipDims diff --git a/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_1.ipynb b/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_1.ipynb index 6811c8bf..52e549d1 100644 --- a/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_1.ipynb +++ b/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_1.ipynb @@ -10,7 +10,7 @@ "source": [ "import torch\n", "import torch.nn as nn\n", - "from sinabs.backend.dynapcnn import DynapcnnNetworkGraph\n", + "from sinabs.backend.dynapcnn import DynapcnnNetwork\n", "from sinabs.layers import Merge, IAFSqueeze\n", "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential" ] @@ -25,7 +25,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 2, @@ -144,13 +144,13 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 6, "metadata": { "metadata": {} }, "outputs": [], "source": [ - "hw_model = DynapcnnNetworkGraph(\n", + "hw_model = DynapcnnNetwork(\n", " snn=snn,\n", " input_shape=input_shape,\n", " batch_size=batch_size\n", @@ -167,7 +167,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 7, "metadata": {}, "outputs": [ { @@ -268,7 +268,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 8, "metadata": {}, "outputs": [ { @@ -296,7 +296,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 9, "metadata": { "metadata": {} }, From f8f2ec4159e9579005c27cc99b2747d698766477 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Mon, 20 May 2024 13:43:35 +0200 Subject: [PATCH 094/379] Cleaning scripts - Removed 'dynapcnn_layer_new.py' (it is now 'dynapcnn_layer.py'). - Removed 'dynapcnn_network_graph.py' (it is now 'dynapcnn_network.py'). --- sinabs/backend/dynapcnn/dynapcnn_layer_new.py | 490 --------------- .../dynapcnn/dynapcnn_network_graph.py | 568 ------------------ 2 files changed, 1058 deletions(-) delete mode 100644 sinabs/backend/dynapcnn/dynapcnn_layer_new.py delete mode 100644 sinabs/backend/dynapcnn/dynapcnn_network_graph.py diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer_new.py b/sinabs/backend/dynapcnn/dynapcnn_layer_new.py deleted file mode 100644 index 50a1a64b..00000000 --- a/sinabs/backend/dynapcnn/dynapcnn_layer_new.py +++ /dev/null @@ -1,490 +0,0 @@ -from copy import deepcopy -from typing import Dict, Optional, Tuple, Union -from warnings import warn - -import numpy as np -import torch -from torch import nn - -import sinabs.activation -import sinabs.layers as sl - -from .discretize import discretize_conv_spike_ -from .dvs_layer import expand_to_pair - -class DynapcnnLayer(nn.Module): - """Create a DynapcnnLayer object representing a dynapcnn layer. """ - - def __init__( - self, - dpcnnl_index: int, - dcnnl_data: dict, - discretize: bool, - sinabs_edges: list - ): - super().__init__() - """ - ... - - Parameters - ---------- - dpcnnl_index (int): ... - dcnnl_data (dict): ... - discretize (bool): ... - sinabs_edges (list): ... - """ - self.dpcnnl_index = dpcnnl_index - self.assigned_core = None - - if 'core_idx' in dcnnl_data: - self.assigned_core = dcnnl_data['core_idx'] - - self.lin_to_conv_conversion = False - - conv = None - self.conv_node_id = None - self.conv_in_shape = None - self.conv_out_shape = None - - spk = None - self.spk_node_id = None - - pool = [] - self.pool_node_id = [] - - self.dynapcnnlayer_destination = dcnnl_data['destinations'] - - for key, value in dcnnl_data.items(): - if isinstance(key, int): - # value has data pertaining a node (torch/sinabs layer). - if isinstance(value['layer'], sl.IAFSqueeze): - spk = value['layer'] - self.spk_node_id = key - elif isinstance(value['layer'], nn.Linear) or isinstance(value['layer'], nn.Conv2d): - conv = value['layer'] - self.conv_node_id = key - elif isinstance(value['layer'], sl.SumPool2d): - pool.append(value['layer']) - self.pool_node_id.append(key) - else: - raise ValueError(f'Node {key} has not valid layer associated with it.') - - if not conv: - raise ValueError(f'Convolution layer not present.') - - if not spk: - raise ValueError(f'Spiking layer not present.') - - spk = deepcopy(spk) - if spk.is_state_initialised(): - # TODO this line bellow is causing an exception on `.v_men.shape` to be raised in `.get_layer_config_dict()`. Find out why. - # spk.v_mem = spk.v_mem.data.unsqueeze(-1).unsqueeze(-1) # expand dims. - - # TODO hacky stuff: make it better (THIS SEEMS TO BE FIXING THE PROBLEM ABOVE THO). - if len(list(spk.v_mem.shape)) != 4: - spk.v_mem = spk.v_mem.data.unsqueeze(-1).unsqueeze(-1) # expand dims. - - if isinstance(conv, nn.Linear): - conv, conv_in_shape = self._convert_linear_to_conv(conv, dcnnl_data[self.conv_node_id]) - - # the original `nn.Linear` output shape becomes the equivalent `nn.Conv2d` shape. - self.conv_out_shape = self._update_conv_node_output_shape( - conv_layer=conv, layer_data=dcnnl_data[self.conv_node_id], input_shape=conv_in_shape) - - # the I/O shapes for neuron layer following the new conv need also to be updated. - self._update_neuron_node_output_shape(layer_data=dcnnl_data[self.spk_node_id], input_shape=self.conv_out_shape) - - else: - self.conv_out_shape = dcnnl_data[self.conv_node_id]['output_shape'] - conv = deepcopy(conv) - - # check if convolution kernel is a square. - if conv.kernel_size[0] != conv.kernel_size[1]: - raise ValueError('The kernel of a `nn.Conv2d` must have the same height and width.') - - # input shape of conv layer. - self.conv_in_shape = dcnnl_data[self.conv_node_id]['input_shape'] - - # this weight rescale comes from the node projecting into this 'conv' node. - if dcnnl_data['conv_rescale_factor'] != 1: - # this has to be done after copying but before discretizing - conv.weight.data = (conv.weight / dcnnl_data['conv_rescale_factor']).clone().detach() - - # int conversion is done while writing the config. - if discretize: - conv, spk = discretize_conv_spike_(conv, spk, to_int=False) - - # consolidate layers. - self.conv_layer = conv - self.spk_layer = spk - self.pool_layer = [] - - if len(pool) != 0: - # the 1st pooling targets the 1st destination in `dcnnl_data['destinations']`, the 2nd pooling targets the 2nd destination... - for plyr in pool: - # @TODO POSSIBLE INCONSISTENCY: if the `SumPool2d` is the result of a conversion from `AvgPool2d` then `SumPool2d.kernel_size` - # is of type tuple, otherwise it is an int. - if isinstance(plyr.kernel_size, tuple) and plyr.kernel_size[0] != plyr.kernel_size[1]: - raise ValueError("Only square kernels are supported") - self.pool_layer.append(deepcopy(plyr)) - - # map destination nodes for each layer in this instance. - self.nodes_destinations = self._get_destinations_input_source(sinabs_edges) - - ####################################################### Public Methods ####################################################### - - def get_destination_dcnnl_index(self, dcnnl_id: int) -> int: - """ The `forward` method will return as many tensors as there are elements in `self.dynapcnnlayer_destination`. Since the i-th returned tensor is to be - fed to the i-th destionation in `self.dynapcnnlayer_destination`, the return of this method can be used to index a tensor returned in the `forward` method. - - Parameters - ---------- - dcnnl_id (int): this should be one of the values listed within `self.dynapcnnlayer_destination`. - - Returns - ---------- - The index of `dcnnl_id` within `self.dynapcnnlayer_destination`. - """ - return self.dynapcnnlayer_destination.index(dcnnl_id) - - def forward(self, x): - """Torch forward pass. - - Returns - ---------- - This method will return as many tensors as there are destinations associated with this instance. The returned tensors always follows the - sequence `return spiking_layer_output, pooling_layer_1, pooling_layer_2, ...`. - - Example - ---------- - With `self.nodes_destinations = {1: [5, 8], 2: [4], 3: [7]}` this method will return 4 tensors (each to be sent to a different DynapcnnLayer): the 1st - and 2nd are the outputs of the spiking layer (`node 1`); the 3rd is the output of the first pooling layer (`node 2`, receiving the output of node 1) appearing - right after the spiking layer as defined in the original `nn.Module` being converted to a `DynapcnnNetwork`; the 4th is the output of the second pooling - layer (`node 3`) to appear after the spiking layer. Thus, the return will be `return node_1_out, node_1_out, node_2_out, node_3_out`. This means the edges - in the computational graph involved in this mapping were: - - 1 --> 2 # `2` is one of the pooling layers of this DynapcnnLayer. - 1 --> 3 # `3` is one of the pooling layers of this DynapcnnLayer. - 1 --> 5 # `5` is a conv layer belonging to another DynapcnnLayer U. - 1 --> 8 # `8` is a conv layer belonging to another DynapcnnLayer V. - 2 --> 4 # `4` is a conv layer belonging to another DynapcnnLayer X. - 3 --> 7 # `7` is a conv layer belonging to another DynapcnnLayer Y. - """ - - returns = [] - - x = self.conv_layer(x) - x = self.spk_layer(x) - - # pooling layers are sequentially added to `self.pool_layer` so we'll pass data to them sequentially. - pooling_indexer = 0 - - # building return set of all layers as they appear in `self.nodes_destinations`. - for node_id, destination_node_list in self.nodes_destinations.items(): - if node_id == self.spk_node_id: - # spiking output for each node outside this DynapcnnLayer receiving from its spiking layer. - for _ in destination_node_list: - returns.append(x) - else: - # returns of each pooling layer are arranged sequenatially. - for _ in destination_node_list: - ith_pool_output = self.pool_layer[pooling_indexer](x) - returns.append(ith_pool_output) - - # forward through next pooling layer in `self.pool_layer` in the next iteration. - pooling_indexer += 1 - - if len(returns) != len(self.dynapcnnlayer_destination): - raise ValueError(f'Number of returned tensors ({len(returns)}) differ from the number of destinations ({len(self.dynapcnnlayer_destination)}).') - - if len(returns) == 0 and len(self.pool_layer) == 0: - # this is the output layer and there's no pooling after the neurons. - returns.append(x) - elif len(returns) == 0 and len(self.pool_layer) == 1: - # this is the output layer and there's 1 pooling after the neurons. - returns.append(self.pool_layer[0](x)) - elif len(returns) == 0 and len(self.pool_layer) > 1: - raise ValueError(f'Output DynapcnnLayer starting with node {self.conv_node_id} has {len(self.pool_layer)} pooling layers: it should have either 1 or none.') - else: - pass - - return tuple(returns) - - def get_modified_node_it(self, dcnnl_data: dict) -> Union[Tuple[int, tuple], Tuple[None, None]]: - """ .""" - if self.lin_to_conv_conversion: - return self.spk_node_id, dcnnl_data[self.spk_node_id]['output_shape'] - return None, None - - def zero_grad(self, set_to_none: bool = False) -> None: - return self.spk_layer.zero_grad(set_to_none) - - def get_conv_output_shape(self, conv_layer: nn.Conv2d, input_shape: tuple): - """ .""" - # get the layer's parameters. - out_channels = conv_layer.out_channels - kernel_size = conv_layer.kernel_size - stride = conv_layer.stride - padding = conv_layer.padding - dilation = conv_layer.dilation - - # compute the output height and width. - out_height = ((input_shape[1] + 2 * padding[0] - dilation[0] * (kernel_size[0] - 1) - 1) // stride[0]) + 1 - out_width = ((input_shape[2] + 2 * padding[1] - dilation[1] * (kernel_size[1] - 1) - 1) // stride[1]) + 1 - - return (out_channels, out_height, out_width) - - def summary(self) -> dict: - # TODO I can't see pooling being used in checking memory constraints by the builder so I'm ignoring for now the fact that multiple pooling could exist. - - _pool = None - - # @TODO POSSIBLE INCONSISTENCY: if the `SumPool2d` is the result of a conversion from `AvgPool2d` then `SumPool2d.kernel_size` - # is of type tuple, otherwise it is an int. - if len(self.pool_layer) != 0: - # @TODO ignoring for now that there could be multiple poolings (just use the first one). - if isinstance(self.pool_layer[0].kernel_size, tuple): - _pool = list(self.pool_layer[0].kernel_size) - elif isinstance(self.pool_layer[0].kernel_size, int): - _pool = [self.pool_layer[0].kernel_size, self.pool_layer[0].kernel_size] - else: - raise ValueError('Type of `self.pool_layer[0].kernel_size` not understood.') - - return { - "pool": (_pool), - "kernel": list(self.conv_layer.weight.data.shape), - "neuron": self.conv_out_shape, # neuron layer output has the same shape as the convolution layer ouput. - } - - def get_layer_config_dict(self) -> dict: - """ Returns a dict containing the properties required to configure a `CNNLayerConfig` instance that - will map this DynapcnnLayer onto the chip. - """ - config_dict = {} - - # configures `CNNLayerConfig.dimensions` (instance of `CNNLayerDimensions`). - dimensions = {} - - # input shape of convolution. - dimensions['input_shape'] = { - 'size': {'x': self.conv_in_shape[2], 'y': self.conv_in_shape[1]}, - 'feature_count': self.conv_in_shape[0] - } - - # ouput shape of convolution. - dimensions['output_shape'] = { - 'size': {'x': self.conv_out_shape[2], 'y': self.conv_out_shape[1]}, - 'feature_count': self.conv_out_shape[0] - } - - # convolution padding, stride and kernel sizes. - dimensions['padding'] = {'x': self.conv_layer.padding[1], 'y': self.conv_layer.padding[0]} - dimensions['stride'] = {'x': self.conv_layer.stride[1], 'y': self.conv_layer.stride[0]} - dimensions['kernel_size'] = self.conv_layer.kernel_size[0] - - config_dict['dimensions'] = dimensions # update config dict. - - # update parameters from convolution. - if self.conv_layer.bias is not None: - (weights, biases) = self.conv_layer.parameters() - else: - (weights,) = self.conv_layer.parameters() - biases = torch.zeros(self.conv_layer.out_channels) - - # parameters of the convolution in the DynapcnnLayer. - - weights = weights.transpose(2, 3) # need this to match samna convention. - config_dict['weights'] = weights.int().tolist() # 4-D list of lists representing kernel parameters. - config_dict['biases'] = biases.int().tolist() - config_dict['leak_enable'] = biases.bool().any() - - # parameters of the neurons in the DynapcnnLayer. - - # set neuron states. # TODO coppied from the old implementation. - if not self.spk_layer.is_state_initialised(): - # then we assign no initial neuron state to DYNAP-CNN. - f, h, w = self.conv_out_shape # same as the convolution layer. - neurons_state = torch.zeros(f, w, h) - - elif self.spk_layer.v_mem.dim() == 4: - # 4-D states should be the norm when there is a batch dim. - neurons_state = self.spk_layer.v_mem.transpose(2, 3)[0] - - else: - raise ValueError(f"Current v_mem (shape: {self.spk_layer.v_mem.shape}) of spiking layer not understood.") - # TODO error here: find where `self.spk_layer.v_mem` is being initialized. - - # resetting vs returning to 0. # TODO coppied from the old implementation. - if isinstance(self.spk_layer.reset_fn, sinabs.activation.MembraneReset): - return_to_zero = True # neurons in this layer will return to 0 when firing. - elif isinstance(self.spk_layer.reset_fn, sinabs.activation.MembraneSubtract): - return_to_zero = False # threshold will be subtracted from the value their membrane potential reached before firing. - else: - raise Exception("Unknown reset mechanism. Only MembraneReset and MembraneSubtract are currently understood.") - - if self.spk_layer.min_v_mem is None: - min_v_mem = -(2**15) - else: - min_v_mem = int(self.spk_layer.min_v_mem) - - # set neuron configuration for this DynapcnnLayer. - config_dict.update( - { - "return_to_zero": return_to_zero, - "threshold_high": int(self.spk_layer.spike_threshold), - "threshold_low": min_v_mem, - "monitor_enable": False, - "neurons_initial_value": neurons_state.int().tolist() - } - ) - - # set pooling configuration for each destinaition. This configures a `CNNLayerConfig.destinations` (instance of `CNNLayerDimensions`). - config_dict['destinations'] = [] - if len(self.pool_layer) != 0: - for i in range(len(self.pool_layer)): - dest_config = { - 'layer': self.dynapcnnlayer_destination[i],# TODO this destination index is not the core index yet, just the index of the DynapcnnLayers themselves. - 'enable': True, - 'pooling': self.pool_layer[i].kernel_size[0] if isinstance(self.pool_layer[i].kernel_size, tuple) else self.pool_layer[i].kernel_size # TODO make sure the kernel is a square. - } - - config_dict['destinations'].append(dest_config) - - # setting of the kill bits need to be done outside this method. - - return config_dict - - def memory_summary(self): - """Computes the amount of memory required for each of the components. Note that this is not - necessarily the same as the number of parameters due to some architecture design - constraints. - - .. math:: - - K_{MT} = c \\cdot 2^{\\lceil \\log_2\\left(k_xk_y\\right) \\rceil + \\lceil \\log_2\\left(f\\right) \\rceil} - - .. math:: - - N_{MT} = f \\cdot 2^{ \\lceil \\log_2\\left(f_y\\right) \\rceil + \\lceil \\log_2\\left(f_x\\right) \\rceil } - - Returns - ------- - A dictionary with keys kernel, neuron and bias and the corresponding memory sizes - """ - summary = self.summary() - f, c, h, w = summary["kernel"] - f, neuron_height, neuron_width = self.conv_out_shape # neuron layer output has the same shape as the convolution layer ouput. - - return { - "kernel": c * pow(2, np.ceil(np.log2(h * w)) + np.ceil(np.log2(f))), - "neuron": f - * pow(2, np.ceil(np.log2(neuron_height)) + np.ceil(np.log2(neuron_width))), - "bias": 0 if self.conv_layer.bias is None else len(self.conv_layer.bias), - } - - def __str__(self): - pretty_print = '\n' - - pretty_print += 'COMPUTATIONAL NODES:\n\n' - - pretty_print += f'(node {self.conv_node_id}): {self.conv_layer}\n' - pretty_print += f'(node {self.spk_node_id}): {self.spk_layer}' - if len(self.pool_layer) != 0: - for idx, lyr in enumerate(self.pool_layer): - pretty_print += f'\n(node {self.pool_node_id[idx]}): {lyr}' - - pretty_print += '\n\nMETADATA:\n' - pretty_print += f'\n> assigned core index: {self.assigned_core}' - pretty_print += f'\n> destination DynapcnnLayers: {self.dynapcnnlayer_destination}' - - for node, destinations in self.nodes_destinations.items(): - pretty_print += f'\n> node {node} feeds input to nodes {destinations}' - - return pretty_print - - ####################################################### Private Methods ####################################################### - - def _update_neuron_node_output_shape(self, layer_data: dict, input_shape: tuple) -> None: - """ Following the conversion of a `nn.Linear` into a `nn.Conv2d` the neuron layer in the - sequence also needs its I/O shapes uodated. - """ - layer_data['input_shape'] = input_shape - layer_data['output_shape'] = layer_data['input_shape'] - - def _update_conv_node_output_shape(self, conv_layer: nn.Conv2d, layer_data: dict, input_shape: tuple) -> Tuple: - """ The input shapes to nodes are extracted using a list of edges by finding the output shape of the 1st element - in the edge and setting it as the input shape to the 2nd element in the edge. If a node used to be a `nn.Linear` - and it became a `nn.Conv2d`, output shape in the mapper needs to be updated, otherwise there will be a mismatch - between its output and the input it provides to another node. - """ - layer_data['output_shape'] = self.get_conv_output_shape(conv_layer, input_shape) - - return layer_data['output_shape'] - - def _convert_linear_to_conv(self, lin: nn.Linear, layer_data: dict) -> nn.Conv2d: - """Convert Linear layer to Conv2d. - - Parameters - ---------- - lin: nn.Linear - Linear layer to be converted - - Returns - ------- - nn.Conv2d - Convolutional layer equivalent to `lin`. - """ - self.lin_to_conv_conversion = True - - input_shape = layer_data['input_shape'] - - in_chan, in_h, in_w = input_shape - - if lin.in_features != in_chan * in_h * in_w: - raise ValueError("Shapes don't match.") - - layer = nn.Conv2d( - in_channels=in_chan, - kernel_size=(in_h, in_w), - out_channels=lin.out_features, - padding=0, - bias=lin.bias is not None, - ) - - if lin.bias is not None: - layer.bias.data = lin.bias.data.clone().detach() - - layer.weight.data = ( - lin.weight.data.clone() - .detach() - .reshape((lin.out_features, in_chan, in_h, in_w)) - ) - - return layer, input_shape - - def _get_destinations_input_source(self, sinabs_edges: list) -> dict: - """ Creates a mapping between each layer in this `DynapcnnLayer` instance and its targe nodes that are part of different - `DynapcnnLayer` instances. This mapping is used to figure out how many tensors the `forward` method needs to return. - """ - destinations_input_source = {} - - # check whether spiking layer projects outside this DynapcnnLayer (i.e. to one of the destinations without passing through a pooling). - spk_destinations = [] - for edge in sinabs_edges: - if edge[0] == self.spk_node_id and edge[1] not in self.pool_node_id: - # spiking layer projects to a node outside this DynapcnnLayer. - spk_destinations.append(edge[1]) - if len(spk_destinations) > 0: - destinations_input_source[self.spk_node_id] = [] - for node_id in spk_destinations: - destinations_input_source[self.spk_node_id].append(node_id) - - # get `pooling->destination` mapping. The pooling outputs will be arranged sequentially since the pooling layers are added sequentially - # to `self.pool_layer` (i.e., as they appear in the computational graph of the original `nn.Module`). - for id in self.pool_node_id: - destinations_input_source[id] = [] - for edge in sinabs_edges: - if edge[0] == id: - destinations_input_source[id].append(edge[1]) - - return destinations_input_source \ No newline at end of file diff --git a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py b/sinabs/backend/dynapcnn/dynapcnn_network_graph.py deleted file mode 100644 index a9ef5f02..00000000 --- a/sinabs/backend/dynapcnn/dynapcnn_network_graph.py +++ /dev/null @@ -1,568 +0,0 @@ -""" -functionality : extracts the computational graph of a network defined as a `nn.Module` and converts it into a set of `DynapcnnLayer`s - that implement a network ()`DynapcnnNetwork`) instance that can be deployed to a Speck chip. -author : Willian Soares Girao -contact : williansoaresgirao@gmail.com -""" - -import time -from typing import List, Optional, Sequence, Tuple, Union, Dict - -import samna -import sinabs.layers as sl -import torch -import torch.nn as nn - -import sinabs - -from .chip_factory import ChipFactory -from .dvs_layer import DVSLayer -from .io import open_device -from .utils import ( - DEFAULT_IGNORED_LAYER_TYPES, - build_from_graph, - build_nodes_to_dcnnl_map, - parse_device_id, -) - -from .NIRGraphExtractor import NIRtoDynapcnnNetworkGraph -from .sinabs_edges_handler import merge_handler - -from .dynapcnnnetwork_module import DynapcnnNetworkModule - -class DynapcnnNetworkGraph(nn.Module): - def __init__( - self, - snn: nn.Module, - input_shape: Tuple[int, int, int], - batch_size: int, - dvs_input: bool = False, - discretize: bool = True - ): - """ - Given a sinabs spiking network, prepare a dynapcnn-compatible network. This can be used to - test the network will be equivalent once on DYNAPCNN. This class also provides utilities to - make the dynapcnn configuration and upload it to DYNAPCNN. - - Some of the properties defined within the class constructor are meant to be temporary data structures handling the conversion - of the `snn` (the original `nn.Module`) into a set of `DynapcnnLayer`s composing a `DynapcnnNetwork` instance. Once their role - in preprocessing `snn` is finished, all required data to train/deploy the `DynapcnnNetwork` instance is within `self._dcnnl_edges` - (the connectivity between each `DynapcnnLayer`/core), `self._forward_map` (every `DynapcnnLayer` in the network) and `self._merge_points` - (the `DynapcnnLayer`s that need a `Merge` input). Thus, the following private properties are delted as last step of the constructor: - - - self._graph_tracer - - self._sinabs_edges - - self._sinabs_modules_map - - self._nodes_name_remap - - self._nodes_to_dcnnl_map - - self._dynapcnn_layers - - Parameters - ---------- - snn (nn.Module): a implementing a spiking network. - input_shape (tuple): a description of the input dimensions as `(features, height, width)`. - dvs_input (bool): wether or not dynapcnn receive input from its DVS camera. - discretize (bool): If `True`, discretize the parameters and thresholds. This is needed for uploading - weights to dynapcnn. Set to `False` only for testing purposes. - """ - super().__init__() - - # TODO for now the graph part is not taking into consideration DVS inputs. - # check if dvs input is expected. - dvs_input = False - self.dvs_input = dvs_input - self.input_shape = input_shape - - assert len(self.input_shape) == 3, "infer_input_shape did not return 3-tuple" - - # computational graph from original PyTorch module. - # TODO - bacth size must be passed as argument. - self._graph_tracer = NIRtoDynapcnnNetworkGraph( - snn, - torch.randn((batch_size, *self.input_shape))) # needs the batch dimension. - - self._sinabs_edges, \ - self._sinabs_modules_map, \ - self._nodes_name_remap = self._get_sinabs_edges_and_modules() - - # create a dict holding the data necessary to instantiate a `DynapcnnLayer`. - self._nodes_to_dcnnl_map = build_nodes_to_dcnnl_map( - layers=self._sinabs_modules_map, - edges=self._sinabs_edges) - - # updates 'self._nodes_to_dcnnl_map' to include the I/O shape for each node. - self._populate_nodes_io() - - # build `DynapcnnLayer` instances from graph edges and mapper. - self._dynapcnn_layers = build_from_graph( - discretize=discretize, - edges=self._sinabs_edges, - nodes_to_dcnnl_map=self._nodes_to_dcnnl_map) - - # these gather all data necessay to implement the forward method for this class. - self._dcnnl_edges, self._forward_map, self._merge_points = self._get_network_module() - - # all necessary `DynapcnnLayer` data held in `self._forward_map`: removing intermediary data structures no longer necessary. - del self._graph_tracer - del self._sinabs_edges - del self._sinabs_modules_map - del self._nodes_name_remap - del self._nodes_to_dcnnl_map - del self._dynapcnn_layers - - ####################################################### Public Methods ####################################################### - - @property - def forward_map(self) -> dict: - """ This dictionary contains each `DynapcnnLayer` in the model indexed by their ID (layer index). """ - return self._forward_map - - def forward(self, x): - """ .""" - - layers_outputs = {} - - # TODO - currently `node 0` (this 1st node in the 1st edge of `self._dcnnl_edges`) is always taken to be the - # input node of the network. This won't work in cases where there are more the one input nodes to the network - # so this functionality needs some refactoring. - self._forward_map[self._dcnnl_edges[0][0]](x) - - # forward the input `x` through the input `DynapcnnLayer` in the `DynapcnnNetwork`s graph (1st node in the 1st edge in `self._dcnnl_edges`). - layers_outputs[self._dcnnl_edges[0][0]] = self._forward_map[self._dcnnl_edges[0][0]](x) - - # propagate outputs in `layers_outputs` through the rest of the nodes of `self._dcnnl_edges`. - for edge in self._dcnnl_edges: - - # target DynapcnnLayer (will consume tensors from `layers_outputs`). - trg_dcnnl = edge[1] - - if trg_dcnnl in self._merge_points and trg_dcnnl not in layers_outputs: - # by this points the arguments of the `Merge` associated with `trg_dcnnl` should have been computed. - arg1, arg2 = self._merge_points[trg_dcnnl]['sources'] - - # find which returned tensor from the `forward` call of DynapcnnLayers `arg1` and `arg2` are to be fed - # to the target DynapcnnLayer `trg_dcnnl`. - return_index_arg1 = self._forward_map[arg1].get_destination_dcnnl_index(trg_dcnnl) - return_index_arg2 = self._forward_map[arg2].get_destination_dcnnl_index(trg_dcnnl) - - # retrieve input tensors to `Merge`. - _arg1 = layers_outputs[arg1][return_index_arg1] - _arg2 = layers_outputs[arg2][return_index_arg2] - - # merge tensors. - merge_output = self._merge_points[trg_dcnnl]['merge'](_arg1, _arg2) - - # call the forward. - layers_outputs[trg_dcnnl] = self._forward_map[trg_dcnnl](merge_output) - - elif trg_dcnnl not in layers_outputs: - # input source for `trg_dcnnl`. - src_dcnnl = edge[0] - - # find which returned tensor from the `forward` call of the source DynapcnnLayer `src_dcnnl` is to be fed - # to the target DynapcnnLayer `trg_dcnnl`. - return_index = self._forward_map[src_dcnnl].get_destination_dcnnl_index(trg_dcnnl) - - # call the forward. - layers_outputs[trg_dcnnl] = self._forward_map[trg_dcnnl](layers_outputs[src_dcnnl][return_index]) - - else: - - pass - - # TODO - this assumes the network has a single output node. - # last computed is the output layer. - return layers_outputs[trg_dcnnl][0] - - def parameters(self) -> list: - """ Gathers all the parameters of the network in a list. This is done by accessing the convolutional layer in each `DynapcnnLayer`, calling - its `.parameters` method and saving it to a list. - - Note: the method assumes no biases are used. - - Returns - ---------- - parameters (list): a list of parameters of all convolutional layers in the `DynapcnnNetwok`. - """ - parameters = [] - - for layer in self._forward_map.values(): - if isinstance(layer, sinabs.backend.dynapcnn.dynapcnn_layer_new.DynapcnnLayer): - parameters.extend(layer.conv_layer.parameters()) - - return parameters - - def init_weights(self, init_fn: nn.init = nn.init.xavier_normal_) -> None: - """ Call the weight initialization method `init_fn` on each `DynapcnnLayer.conv_layer.weight.data` in the `DynapcnnNetwork` instance.""" - for layer in self._forward_map.values(): - if isinstance(layer, sinabs.backend.dynapcnn.dynapcnn_layer_new.DynapcnnLayer): - init_fn(layer.conv_layer.weight.data) - - def detach_neuron_states(self) -> None: - """ Detach the neuron states and activations from current computation graph (necessary). """ - - for module in self._forward_map.values(): - if isinstance(module, sinabs.backend.dynapcnn.dynapcnn_layer_new.DynapcnnLayer): - if isinstance(module.spk_layer, sl.StatefulLayer): - for name, buffer in module.spk_layer.named_buffers(): - buffer.detach_() - - def to( - self, - device="cpu", - chip_layers_ordering="auto", - monitor_layers: Optional[Union[List, str]] = None, - config_modifier=None, - slow_clk_frequency: int = None, - ): - """Note that the model parameters are only ever transferred to the device on the `to` call, - so changing a threshold or weight of a model that is deployed will have no effect on the - model on chip until `to` is called again. - - Parameters - ---------- - - device: String - cpu:0, cuda:0, dynapcnndevkit, speck2devkit - - chip_layers_ordering: sequence of integers or `auto` - The order in which the dynapcnn layers will be used. If `auto`, - an automated procedure will be used to find a valid ordering. - A list of layers on the device where you want each of the model's DynapcnnLayers to be placed. - The index of the core on chip to which the i-th layer in the model is mapped is the value of the i-th entry in the list. - Note: This list should be the same length as the number of dynapcnn layers in your model. - - monitor_layers: None/List - A list of all layers in the module that you want to monitor. Indexing starts with the first non-dvs layer. - If you want to monitor the dvs layer for eg. - :: - - monitor_layers = ["dvs"] # If you want to monitor the output of the pre-processing layer - monitor_layers = ["dvs", 8] # If you want to monitor preprocessing and layer 8 - monitor_layers = "all" # If you want to monitor all the layers - - config_modifier: - A user configuration modifier method. - This function can be used to make any custom changes you want to make to the configuration object. - - Note - ---- - chip_layers_ordering and monitor_layers are used only when using synsense devices. - For GPU or CPU usage these options are ignored. - """ - self.device = device - - if isinstance(device, torch.device): - self._to_device(device) - - elif isinstance(device, str): - device_name, _ = parse_device_id(device) - - if device_name in ChipFactory.supported_devices: - - # generate config. - config = self._make_config( - chip_layers_ordering=chip_layers_ordering, - device=device, - monitor_layers=monitor_layers, - config_modifier=config_modifier, - ) - - # apply configuration to device. - self.samna_device = open_device(device) - self.samna_device.get_model().apply_configuration(config) - time.sleep(1) - - # set external slow-clock if needed. - if slow_clk_frequency is not None: - dk_io = self.samna_device.get_io_module() - dk_io.set_slow_clk(True) - dk_io.set_slow_clk_rate(slow_clk_frequency) # Hz - - builder = ChipFactory(device).get_config_builder() - - # create input source node. - self.samna_input_buffer = builder.get_input_buffer() - - # create output sink node node. - self.samna_output_buffer = builder.get_output_buffer() - - # connect source node to device sink. - self.device_input_graph = samna.graph.EventFilterGraph() - self.device_input_graph.sequential( - [ - self.samna_input_buffer, - self.samna_device.get_model().get_sink_node(), - ] - ) - - # connect sink node to device. - self.device_output_graph = samna.graph.EventFilterGraph() - self.device_output_graph.sequential( - [ - self.samna_device.get_model().get_source_node(), - self.samna_output_buffer, - ] - ) - - self.device_input_graph.start() - self.device_output_graph.start() - self.samna_config = config - - return print(self) - - else: - self._to_device(device) - - else: - raise Exception("Unknown device description.") - - ####################################################### Private Methods ####################################################### - - def _make_config( - self, - chip_layers_ordering: Union[Sequence[int], str] = "auto", - device="dynapcnndevkit:0", - monitor_layers: Optional[Union[List, str]] = None, - config_modifier=None, - ): - """Prepare and output the `samna` DYNAPCNN configuration for this network. - - Parameters - ---------- - - chip_layers_ordering: sequence of integers or `auto` - The order in which the dynapcnn layers will be used. If `auto`, - an automated procedure will be used to find a valid ordering. - A list of layers on the device where you want each of the model's DynapcnnLayers to be placed. - Note: This list should be the same length as the number of dynapcnn layers in your model. - - device: String - dynapcnndevkit, speck2b or speck2devkit - - monitor_layers: None/List/Str - A list of all layers in the module that you want to monitor. Indexing starts with the first non-dvs layer. - If you want to monitor the dvs layer for eg. - :: - - monitor_layers = ["dvs"] # If you want to monitor the output of the pre-processing layer - monitor_layers = ["dvs", 8] # If you want to monitor preprocessing and layer 8 - monitor_layers = "all" # If you want to monitor all the layers - - If this value is left as None, by default the last layer of the model is monitored. - - config_modifier: - A user configuration modifier method. - This function can be used to make any custom changes you want to make to the configuration object. - - Returns - ------- - Configuration object - Object defining the configuration for the device - - Raises - ------ - ImportError - If samna is not available. - ValueError - If the generated configuration is not valid for the specified device. - """ - config_builder = ChipFactory(device).get_config_builder() - - # TODO not handling DVSLayer yet. - has_dvs_layer = isinstance(self._forward_map[0], DVSLayer) - - if chip_layers_ordering == "auto": - # figure out mapping of each DynapcnnLayer into one core. - chip_layers_ordering = config_builder.get_valid_mapping(self) - - else: - # TODO - mapping from each DynapcnnLayer into cores has been provided by the user: NOT IMPLEMENTED YET. - if has_dvs_layer: - # TODO not handling DVSLayer yet. - pass - - # update config. - config = config_builder.build_config(self, None) - - # TODO not handling DVSLayer yet (this is from the old implementation, should be revised). - if self.input_shape and self.input_shape[0] == 1: - config.dvs_layer.merge = True - - # TODO all this monitoring part needs validation still. - monitor_chip_layers = [] - if monitor_layers is None: - # check if any monitoring is enabled (if not, enable monitoring for the last layer). - for dcnnl_index, ith_dcnnl in self._forward_map.items(): - if len(ith_dcnnl.dynapcnnlayer_destination) == 0: - monitor_chip_layers.append(ith_dcnnl.assigned_core) - break - elif monitor_layers == "all": - for dcnnl_index, ith_dcnnl in self._forward_map.items(): - # TODO not handling DVSLayer yet - # monitor each chip core (if not a DVSLayer). - if not isinstance(ith_dcnnl, DVSLayer): - monitor_chip_layers.append(ith_dcnnl.assigned_core) - - if monitor_layers: - if "dvs" in monitor_layers: - monitor_chip_layers.append("dvs") - - # enable monitors on the specified layers. - config_builder.monitor_layers(config, monitor_chip_layers) - - if config_modifier is not None: - # apply user config modifier. - config = config_modifier(config) - - if config_builder.validate_configuration(config): - # validate config. - print("Network is valid: \n") - - return config - else: - raise ValueError(f"Generated config is not valid for {device}") - - def _get_network_module(self) -> Union[list, dict, dict]: - """ Uses the `DynapcnnLayer` instances in `self._dynapcnn_layers` and the connectivity between them to create three data structures - that guide the data forwarding between the layer during the forward pass. - - Note: the property `DynapcnnLayer.assigned_core` is only set after `self.to(device='speck...')` is called. - - Returns - ---------- - dcnnl_edges (list): edges, represented as tuples of `DynapcnnLayer` indices, used to guide the data forwarding through each `DynapcnnLayer` in forward method. - forward_map (dict): have all the `DynapcnnLayer` (`value`), each being accessible via its index (`key`). Used to call `DynapcnnLayer.forward` in forward method. - merge_points (dict): used to compose the inputs to a `DynapcnnLayer` that requires an input from a `Merge` layer. - """ - - # get connections between `DynapcnnLayer`s. - dcnnl_edges = self._get_dynapcnnlayers_edges() - - dcnnnet_module = DynapcnnNetworkModule(dcnnl_edges, self._dynapcnn_layers) - - return dcnnnet_module.dcnnl_edges, dcnnnet_module.forward_map, dcnnnet_module.merge_points - - def _get_dynapcnnlayers_edges(self) -> List[Tuple[int, int]]: - """ Create edges representing connections between `DynapcnnLayer` instances. """ - dcnnl_edges = [] - - for dcnnl_idx, layer_data in self._dynapcnn_layers.items(): - for dest in layer_data['destinations']: - dcnnl_edges.append((dcnnl_idx, dest)) - - return dcnnl_edges - - def _get_sinabs_edges_and_modules(self) -> Tuple[List[Tuple[int, int]], Dict[int, nn.Module], Dict[int, int]]: - """ The computational graph extracted from `snn` might contain layers that are ignored (e.g. a `nn.Flatten` will be - ignored when creating a `DynapcnnLayer` instance). Thus the list of edges from such model need to be rebuilt such that if there are - edges `(A, X)` and `(X, B)`, and `X` is an ignored layer, an edge `(A, B)` is created. - - Returns - ---------- - edges_without_merge (list): a list of edges based on `sinabs_edges` but where edges involving a `Merge` layer have been - remapped to connect the nodes involved in the merging directly. - sinabs_modules_map (dict): a dict containing the nodes of the graph (described now by `edges_without_merge`) as `key` and - their associated module as `value`. - remapped_nodes (dict): a dict where `key` is the original node name (as extracted by `self._graph_tracer`) and `value` is - the new node name (after ignored layers have been dropped and `Merge` layers have be processed before being removed). - """ - - # remap `(A, X)` and `(X, B)` into `(A, B)` if `X` is a layer in the original `snn` to be ignored. - sinabs_edges, remapped_nodes = self._graph_tracer.remove_ignored_nodes( - DEFAULT_IGNORED_LAYER_TYPES) - - # nodes (layers' "names") need remapping in case some layers have been removed (e.g. a `nn.Flattern` is ignored). - sinabs_modules_map = {} - for orig_name, new_name in remapped_nodes.items(): - sinabs_modules_map[new_name] = self._graph_tracer.modules_map[orig_name] - - # bypass merging layers to connect the nodes involved in them directly to the node where the merge happens. - edges_without_merge = merge_handler(sinabs_edges, sinabs_modules_map) - - return edges_without_merge, sinabs_modules_map, remapped_nodes - - def _populate_nodes_io(self): - """ Loops through the nodes in the original graph to retrieve their I/O tensor shapes and add them to their respective - representations in `self._nodes_to_dcnnl_map`.""" - - def find_original_node_name(name_mapper: dict, node: int): - """ Find what a node is originally named when built in `self._graph_tracer`. """ - for orig_name, new_name in name_mapper.items(): - if new_name == node: - return orig_name - raise ValueError(f'Node {node} could not be found within the name remapping done by self._get_sinabs_edges_and_modules().') - - def find_my_input(edges_list: list, node: int) -> int: - """ Returns the node `X` in the first edge `(X, node)`. - - Parameters - ---------- - node (int): the node in the computational graph for which we whish to find the input source (either another node in the - graph or the original input itself to the network). - - Returns - ---------- - input source (int): this indicates the node in the computational graph providing the input to `node`. If `node` is - receiving outside input (i.e., it is a starting node) the return will be -1. For example, this will be the case - when a network with two independent branches (each starts from a different "input node") merge along the computational graph. - """ - for edge in edges_list: - if edge[1] == node: - # TODO nodes originally receiving input from merge will appear twice in the list of edges, one - # edge per input to the merge layer. For now both inputs to a `Merge` have the same dimensions - # necessarily so this works for now but later will have to be revised. - return edge[0] - return -1 - - # access the I/O shapes for each node in `self._sinabs_edges` from the original graph in `self._graph_tracer`. - for dcnnl_idx, dcnnl_data in self._nodes_to_dcnnl_map.items(): - for node, node_data in dcnnl_data.items(): - # node dictionary with layer data. - if isinstance(node, int): - # some nodes might have been renamed (e.g. after droppping a `nn.Flatten`), so find how node was originally named. - orig_name = find_original_node_name(self._nodes_name_remap, node) - _in, _out = self._graph_tracer.get_node_io_shapes(orig_name) - - # update node I/O shape in the mapper (drop batch dimension). - if node != 0: - # Find node outputing into the current node being processed (this will be the input shape). This is - # necessary cuz if a node originally receives input from a `nn.Flatten` for instance, when mapped into - # a `DynapcnnLayer` it will be receiving the input from a privious `sl.SumPool2d`. - input_node = find_my_input(self._sinabs_edges, node) - - if input_node == -1: - # node does not have an input source within the graph (it consumes the original input to the model). - node_data['input_shape'] = tuple(list(_in)[1:]) - else: - # input comes from another node in the graph. - input_node_orig_name = find_original_node_name(self._nodes_name_remap, input_node) - _, _input_source_shape = self._graph_tracer.get_node_io_shapes(input_node_orig_name) - node_data['input_shape'] = tuple(list(_input_source_shape)[1:]) - else: - # first node does not have an input source within the graph. - node_data['input_shape'] = tuple(list(_in)[1:]) - - node_data['output_shape'] = tuple(list(_out)[1:]) - - def _to_device(self, device: torch.device) -> None: - """ .""" - for layer in self._forward_map.values(): - if isinstance(layer, sinabs.backend.dynapcnn.dynapcnn_layer_new.DynapcnnLayer): - layer.conv_layer.to(device) - layer.spk_layer.to(device) - - # if there's more than one pooling each of them becomes a node that is catched by the `else` statement. - if len(layer.pool_layer) == 1: - layer.pool_layer[0].to(device) - else: - # this nodes are created from `DynapcnnLayer`s that have multiple poolings (each pooling becomes a new node). - layer.to(device) - - def __str__(self): - pretty_print = '' - for idx, layer_data in self._forward_map.items(): - pretty_print += f'----------------------- [ DynapcnnLayer {idx} ] -----------------------\n' - pretty_print += f'{layer_data}\n\n' - - return pretty_print \ No newline at end of file From 38395793d360d25de6b22dacad732bafe45cafc5 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Mon, 20 May 2024 15:50:03 +0200 Subject: [PATCH 095/379] Refactor - DynapcnnNetwork receives a method to handle multiple converging re-scaling factors onto a single convolutional layer. - mapper combining nodes into DynapcnnLayers has now a list holding the re-scaling factors computed for each future DynapcnnLayer to use. - DynapcnnLayer receives the weight re-scaling handler passed down frow the DynapcnnNetwork's constructor. --- sinabs/backend/dynapcnn/dynapcnn_layer.py | 40 ++++---- sinabs/backend/dynapcnn/dynapcnn_network.py | 33 ++++--- .../backend/dynapcnn/sinabs_edges_handler.py | 7 +- sinabs/backend/dynapcnn/utils.py | 92 ++++++++++++------- .../dynapcnn/weight_rescaling_methods.py | 33 +++++++ 5 files changed, 138 insertions(+), 67 deletions(-) create mode 100644 sinabs/backend/dynapcnn/weight_rescaling_methods.py diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer.py b/sinabs/backend/dynapcnn/dynapcnn_layer.py index 50a1a64b..b5a499d1 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer.py @@ -1,5 +1,5 @@ from copy import deepcopy -from typing import Dict, Optional, Tuple, Union +from typing import Dict, Callable, Tuple, Union from warnings import warn import numpy as np @@ -10,29 +10,31 @@ import sinabs.layers as sl from .discretize import discretize_conv_spike_ -from .dvs_layer import expand_to_pair class DynapcnnLayer(nn.Module): - """Create a DynapcnnLayer object representing a dynapcnn layer. """ + """ + Create a DynapcnnLayer object representing a dynapcnn layer. + + Parameters + ---------- + - dpcnnl_index (int): ... + - dcnnl_data (dict): ... + - discretize (bool): ... + - sinabs_edges (list): ... + - weight_rescaling_fn (callable): a method that handles how the re-scaling factor for one or more `SumPool2d` projecting to + the same convolutional layer are combined/re-scaled before applying them. + """ def __init__( self, dpcnnl_index: int, dcnnl_data: dict, discretize: bool, - sinabs_edges: list + sinabs_edges: list, + weight_rescaling_fn: Callable ): super().__init__() - """ - ... - Parameters - ---------- - dpcnnl_index (int): ... - dcnnl_data (dict): ... - discretize (bool): ... - sinabs_edges (list): ... - """ self.dpcnnl_index = dpcnnl_index self.assigned_core = None @@ -51,6 +53,7 @@ def __init__( pool = [] self.pool_node_id = [] + self.conv_rescaling_factor = None self.dynapcnnlayer_destination = dcnnl_data['destinations'] @@ -106,9 +109,13 @@ def __init__( self.conv_in_shape = dcnnl_data[self.conv_node_id]['input_shape'] # this weight rescale comes from the node projecting into this 'conv' node. - if dcnnl_data['conv_rescale_factor'] != 1: - # this has to be done after copying but before discretizing - conv.weight.data = (conv.weight / dcnnl_data['conv_rescale_factor']).clone().detach() + if len(dcnnl_data['conv_rescale_factor']): + # this means an `AvgPool2d` has been converted into a `SumPool2d`. + self.conv_rescaling_factor = weight_rescaling_fn(dcnnl_data['conv_rescale_factor']) + conv.weight.data = (conv.weight.data / self.conv_rescaling_factor).clone().detach() + else: + # this means `SumPool2d` have been used from the start. + conv.weight.data = (conv.weight.data).clone().detach() # int conversion is done while writing the config. if discretize: @@ -394,6 +401,7 @@ def __str__(self): pretty_print += f'\n(node {self.pool_node_id[idx]}): {lyr}' pretty_print += '\n\nMETADATA:\n' + pretty_print += f'\n> convolution\'s weight re-scaling factor: {self.conv_rescaling_factor}' pretty_print += f'\n> assigned core index: {self.assigned_core}' pretty_print += f'\n> destination DynapcnnLayers: {self.dynapcnnlayer_destination}' diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index ad471c85..872f1c8b 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -6,7 +6,7 @@ """ import time -from typing import List, Optional, Sequence, Tuple, Union, Dict +from typing import List, Optional, Sequence, Tuple, Union, Dict, Callable import samna import sinabs.layers as sl @@ -29,6 +29,7 @@ from .sinabs_edges_handler import merge_handler from .dynapcnnnetwork_module import DynapcnnNetworkModule +from .weight_rescaling_methods import rescale_method_1, rescale_method_2 class DynapcnnNetwork(nn.Module): def __init__( @@ -37,13 +38,26 @@ def __init__( input_shape: Tuple[int, int, int], batch_size: int, dvs_input: bool = False, - discretize: bool = True + discretize: bool = True, + weight_rescaling_fn: Callable = rescale_method_1 ): """ Given a sinabs spiking network, prepare a dynapcnn-compatible network. This can be used to test the network will be equivalent once on DYNAPCNN. This class also provides utilities to make the dynapcnn configuration and upload it to DYNAPCNN. + Parameters + ---------- + - snn (nn.Module): a implementing a spiking network. + - input_shape (tuple): a description of the input dimensions as `(features, height, width)`. + - dvs_input (bool): wether or not dynapcnn receive input from its DVS camera. + - discretize (bool): If `True`, discretize the parameters and thresholds. This is needed for uploading + weights to dynapcnn. Set to `False` only for testing purposes. + - weight_rescaling_fn (callable): a method that handles how the re-scaling factor for one or more `SumPool2d` projecting to + the same convolutional layer are combined/re-scaled before applying them. + + Notes + ---------- Some of the properties defined within the class constructor are meant to be temporary data structures handling the conversion of the `snn` (the original `nn.Module`) into a set of `DynapcnnLayer`s composing a `DynapcnnNetwork` instance. Once their role in preprocessing `snn` is finished, all required data to train/deploy the `DynapcnnNetwork` instance is within `self._dcnnl_edges` @@ -56,14 +70,6 @@ def __init__( - self._nodes_name_remap - self._nodes_to_dcnnl_map - self._dynapcnn_layers - - Parameters - ---------- - snn (nn.Module): a implementing a spiking network. - input_shape (tuple): a description of the input dimensions as `(features, height, width)`. - dvs_input (bool): wether or not dynapcnn receive input from its DVS camera. - discretize (bool): If `True`, discretize the parameters and thresholds. This is needed for uploading - weights to dynapcnn. Set to `False` only for testing purposes. """ super().__init__() @@ -95,9 +101,10 @@ def __init__( # build `DynapcnnLayer` instances from graph edges and mapper. self._dynapcnn_layers = build_from_graph( - discretize=discretize, - edges=self._sinabs_edges, - nodes_to_dcnnl_map=self._nodes_to_dcnnl_map) + discretize = discretize, + edges = self._sinabs_edges, + nodes_to_dcnnl_map = self._nodes_to_dcnnl_map, + weight_rescaling_fn = weight_rescaling_fn) # these gather all data necessay to implement the forward method for this class. self._dcnnl_edges, self._forward_map, self._merge_points = self._get_network_module() diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index 41e9be2c..bf02f334 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -202,9 +202,10 @@ def get_dynapcnnlayers_destinations(layers: Dict[int, nn.Module], edges: List[Tu else: raise InvalidLayerLoop(source_layer, destination_layer) - for dcnnl_idx, destinations in dynapcnnlayers_destinations_map.items(): # TODO document the 'rescale_factor' better. - mapper[dcnnl_idx]['destinations'] = destinations - mapper[dcnnl_idx]['conv_rescale_factor'] = 1 + for dcnnl_idx, destinations in dynapcnnlayers_destinations_map.items(): + # TODO document the 'rescale_factor' better. + mapper[dcnnl_idx]['destinations'] = destinations + mapper[dcnnl_idx]['conv_rescale_factor'] = [] def get_dynapcnnlayer_index(node: int, mapper: dict) -> int: """ Returns the DynapcnnLayer index to which 'node' belongs to. """ diff --git a/sinabs/backend/dynapcnn/utils.py b/sinabs/backend/dynapcnn/utils.py index 658aa1b9..c7da2f53 100644 --- a/sinabs/backend/dynapcnn/utils.py +++ b/sinabs/backend/dynapcnn/utils.py @@ -1,5 +1,5 @@ from copy import deepcopy -from typing import TYPE_CHECKING, List, Optional, Tuple, Type, Union, Dict +from typing import TYPE_CHECKING, List, Optional, Tuple, Type, Union, Dict, Callable import torch import torch.nn as nn @@ -10,7 +10,7 @@ from .crop2d import Crop2d from .dvs_layer import DVSLayer, expand_to_pair from .dynapcnn_layer import DynapcnnLayer -from .exceptions import InputConfigurationError, MissingLayer, UnexpectedLayer, WrongModuleCount, WrongPoolingModule +from .exceptions import InputConfigurationError, MissingLayer, UnexpectedLayer, WrongPoolingModule from .flipdims import FlipDims from .sinabs_edges_handler import process_edge, get_dynapcnnlayers_destinations @@ -20,7 +20,6 @@ DEFAULT_IGNORED_LAYER_TYPES = (nn.Identity, nn.Dropout, nn.Dropout2d, nn.Flatten) - def infer_input_shape( layers: List[nn.Module], input_shape: Optional[Tuple[int, int, int]] = None ) -> Tuple[int, int, int]: @@ -593,25 +592,28 @@ def build_nodes_to_dcnnl_map(layers: Dict[int, nn.Module], edges: List[Tuple[int def build_from_graph( discretize: bool, edges: List[Tuple[int, int]], - nodes_to_dcnnl_map: dict) -> dict: - """ Parses each edge of a 'sinabs_mode.spiking_model' computational graph. Each node (layer) is assigned to a - DynapcnnLayer object. The target destination of each DynapcnnLayer is computed via edges connecting nodes in - different DynapcnnLayer objects. + nodes_to_dcnnl_map: dict, + weight_rescaling_fn: Callable) -> dict: + """ Parses each edge in `edges`, where each node is a set of layer that will compose a `DynapcnnLayer`. The + target destination of each `DynapcnnLayer` is computed via edges connecting nodes in different `DynapcnnLayer` + instances. Parameters ---------- - discretize (bool): ... - edges (list): edges describing how nodes connect to each other. - nodes_to_dcnnl_map (dict): each entry represents the gathered data necessary to instantiate a `DynapcnnLayer` object (e.g. nodes, - their I/O shapes, the list of `DynapcnnLayer` that are to be targeted, etc). + - discretize (bool): if `True` the weights of all convolutional layers are discretized. + - edges (list): edges describing how nodes connect to each other. + - nodes_to_dcnnl_map (dict): each entry represents the gathered data necessary to instantiate a `DynapcnnLayer` object (e.g. nodes, + their I/O shapes, the list of `DynapcnnLayer` that are to be targeted, etc). + - weight_rescaling_fn (callable): a method that handles how the re-scaling factor for one or more `SumPool2d` projecting to + the same convolutional layer are combined/re-scaled before applying them. Returns ---------- - dynapcnn_layers (dict): `DynapcnnLayer` instances, each created from an entry in `nodes_to_dcnnl_map`. + - dynapcnn_layers (dict): `DynapcnnLayer` instances, each created from an entry in `nodes_to_dcnnl_map`. """ # turn each entry in `nodes_to_dcnnl_map` into a `DynapcnnLayer` instance. - dynapcnn_layers = construct_dynapcnnlayers_from_mapper(discretize, nodes_to_dcnnl_map, edges) + dynapcnn_layers = construct_dynapcnnlayers_from_mapper(discretize, nodes_to_dcnnl_map, edges, weight_rescaling_fn) # initialize attribute holding to which core a `DynapcnnLayer` instance in `dynapcnn_layers` will be mapped to. for idx, layer_data in dynapcnn_layers.items(): @@ -624,19 +626,22 @@ def build_from_graph( def construct_dynapcnnlayers_from_mapper( discretize: bool, nodes_to_dcnnl_map: dict, - edges: List[Tuple[int, int]]) -> Dict[int, Dict[DynapcnnLayer, List]]: + edges: List[Tuple[int, int]], + weight_rescaling_fn: Callable) -> Dict[int, Dict[DynapcnnLayer, List]]: """ Consumes a dictionaries containing sets of layers to be used to populate a DynapcnnLayer object. Parameters ---------- - discretize (bool): ... - nodes_to_dcnnl_map (dict): each entry represents the gathered data necessary to instantiate a `DynapcnnLayer` object (e.g. nodes, - their I/O shapes, the list of `DynapcnnLayer` that are to be targeted, etc). - edges (list): edges describing how nodes connect to each other. + - discretize (bool): if `True` the weights of all convolutional layers are discretized. + - nodes_to_dcnnl_map (dict): each entry represents the gathered data necessary to instantiate a `DynapcnnLayer` object (e.g. nodes, + their I/O shapes, the list of `DynapcnnLayer` that are to be targeted, etc). + - edges (list): edges describing how nodes connect to each other. + - weight_rescaling_fn (callable): a method that handles how the re-scaling factor for one or more `SumPool2d` projecting to + the same convolutional layer are combined/re-scaled before applying them. Returns ---------- - dynapcnn_layers (dict): `DynapcnnLayer` instances, each created from an entry in `nodes_to_dcnnl_map`. + - dynapcnn_layers (dict): `DynapcnnLayer` instances, each created from an entry in `nodes_to_dcnnl_map`. """ dynapcnn_layers = {} @@ -644,7 +649,7 @@ def construct_dynapcnnlayers_from_mapper( for dpcnnl_idx, dcnnl_data in nodes_to_dcnnl_map.items(): # create a `DynapcnnLayer` from the set of layers in `dcnnl_data`. dynapcnnlayer = construct_dynapcnnlayer( - dpcnnl_idx, discretize, dcnnl_data, edges, nodes_to_dcnnl_map) + dpcnnl_idx, discretize, dcnnl_data, edges, nodes_to_dcnnl_map, weight_rescaling_fn) dynapcnn_layers[dpcnnl_idx] = { 'layer': dynapcnnlayer, @@ -678,13 +683,20 @@ def construct_dynapcnnlayer( discretize: bool, dcnnl_data: dict, edges: List[Tuple[int, int]], - nodes_to_dcnnl_map: dict) -> DynapcnnLayer: + nodes_to_dcnnl_map: dict, + weight_rescaling_fn: Callable) -> DynapcnnLayer: """ Extract the modules (layers) in a dictionary and uses them to instantiate a DynapcnnLayer object. Parameters ---------- - dcnnl_data: contains the nodes to be merged into a DynapcnnLayer, their I/O shapes and the index of the other DynapcnnLayers to - be set as destinations. + - dcnnl_data (dict): contains the nodes to be merged into a DynapcnnLayer, their I/O shapes and the index of the other DynapcnnLayers to + be set as destinations. + - weight_rescaling_fn (callable): a method that handles how the re-scaling factor for one or more `SumPool2d` projecting to + the same convolutional layer are combined/re-scaled before applying them. + + Returns + ---------- + - """ # convert all AvgPool2d in 'dcnnl_data' into SumPool2d. @@ -692,10 +704,11 @@ def construct_dynapcnnlayer( # instantiate a DynapcnnLayer from the data in 'dcnnl_data'. dynapcnnlayer = DynapcnnLayer( - dpcnnl_index = dpcnnl_idx, - dcnnl_data = dcnnl_data, - discretize = discretize, - sinabs_edges = edges + dpcnnl_index = dpcnnl_idx, + dcnnl_data = dcnnl_data, + discretize = discretize, + sinabs_edges = edges, + weight_rescaling_fn = weight_rescaling_fn ) return dynapcnnlayer @@ -724,14 +737,11 @@ def convert_Avg_to_Sum_pooling(dcnnl_data: dict, edges: list, nodes_to_dcnnl_map # find which node `key` will target. for edge in edges: if edge[0] == key: - # find index of DynapcnnLayer where the target of `edge[0]` is. + # find index of `DynapcnnLayer` where the target of `edge[0]` is. trg_dcnnl_idx = find_nodes_dcnnl_idx(edge[1], nodes_to_dcnnl_map) # update the rescale factor for the target of node `key`. - if rescale_factor > nodes_to_dcnnl_map[trg_dcnnl_idx]['conv_rescale_factor']: - # If more than one DynapcnnLayers target `trg_dcnnl_idx` with different rescale - # factors, the highest amongst them is used. - nodes_to_dcnnl_map[trg_dcnnl_idx]['conv_rescale_factor'] = rescale_factor + nodes_to_dcnnl_map[trg_dcnnl_idx]['conv_rescale_factor'].append(rescale_factor) def find_nodes_dcnnl_idx(node, nodes_to_dcnnl_map): """ .""" @@ -747,7 +757,17 @@ def find_nodes_dcnnl_idx(node, nodes_to_dcnnl_map): raise ValueError(f'Node {node} is not part of any dictionary mapping into a DynapcnnLayer.') def build_SumPool2d(module: nn.AvgPool2d) -> Tuple[sl.SumPool2d, int]: - """ Converts a 'nn.AvgPool2d' into a 'sl.SumPool2d' layer. """ + """ Converts a `nn.AvgPool2d` into a `sl.SumPool2d` layer. + + Parameters + ---------- + module (torch.nn.AvgPool2d): the average pooling layer being converted into a sum pooling layer. + + Returns + ---------- + lyr_pool (sinabs.layers.SumPool2d): the equivalent sum pooling layer. + rescale_factor (int): the weight re-scaling computed for the weights of the convolution layer targeted by the pooling. + """ if isinstance(module, nn.AvgPool2d): if module.padding != 0: @@ -768,12 +788,14 @@ def build_SumPool2d(module: nn.AvgPool2d) -> Tuple[sl.SumPool2d, int]: f"Stride length {module.stride} should be the same as pooling kernel size {module.kernel_size}" ) - cumulative_pooling = ( # compute cumulative pooling. + # compute cumulative pooling. + cumulative_pooling = ( cumulative_pooling[0] * pooling[0], cumulative_pooling[1] * pooling[1], ) - if isinstance(module, nn.AvgPool2d): # update rescaling factor. + if isinstance(module, nn.AvgPool2d): + # update rescaling factor. rescale_factor *= pooling[0] * pooling[1] lyr_pool = sl.SumPool2d(cumulative_pooling) diff --git a/sinabs/backend/dynapcnn/weight_rescaling_methods.py b/sinabs/backend/dynapcnn/weight_rescaling_methods.py new file mode 100644 index 00000000..2853f7fd --- /dev/null +++ b/sinabs/backend/dynapcnn/weight_rescaling_methods.py @@ -0,0 +1,33 @@ +import numpy as np +import statistics + +def rescale_method_1(rescaling_from_sumpool: list, lambda_: float = 0.5) -> float: + """ + The `method 1` will use the average of the computed rescaling factor for each pooling layer + feeding into a convolutional layer (if there are more than one)... + + Arguments + --------- + """ + + if len(rescaling_from_sumpool): + return np.round(np.mean(rescaling_from_sumpool)*lambda_, 2) + else: + return 1 + +def rescale_method_2(rescaling_from_sumpool: list, lambda_: float = 0.5) -> float: + """ + The `method 2` will use the harmonic mean of the computed rescaling factor for each pooling layer + feeding into `conv_layer` (if there are more than one) ... + + Note: since the harmonic mean is less sensitive to outliers it **could be** that this is a better method + for weight re-scaling when multiple pooling with big differentces in kernel sizes are being considered. + + Arguments + --------- + """ + + if len(rescaling_from_sumpool): + return np.round(statistics.harmonic_mean(rescaling_from_sumpool)*lambda_, 2) + else: + return 1 \ No newline at end of file From d7eda57c7d3071f16044367e476cd5eacc77aecb Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Mon, 20 May 2024 16:09:19 +0200 Subject: [PATCH 096/379] Documentation/Cleaning - Functions strictly related to functionality specific of old implementation of DynapcnnNetwork have been commented out. - Functions related to functionality yet to be added (e.g., DVS input, NB layer handling) in documented section. - Functions related to functionality specific of new implementation of DynapcnnNetowkr in documented section. --- sinabs/backend/dynapcnn/utils.py | 1041 +++++++++++++++--------------- 1 file changed, 523 insertions(+), 518 deletions(-) diff --git a/sinabs/backend/dynapcnn/utils.py b/sinabs/backend/dynapcnn/utils.py index c7da2f53..c099c163 100644 --- a/sinabs/backend/dynapcnn/utils.py +++ b/sinabs/backend/dynapcnn/utils.py @@ -20,50 +20,315 @@ DEFAULT_IGNORED_LAYER_TYPES = (nn.Identity, nn.Dropout, nn.Dropout2d, nn.Flatten) -def infer_input_shape( - layers: List[nn.Module], input_shape: Optional[Tuple[int, int, int]] = None -) -> Tuple[int, int, int]: - """Checks if the input_shape is specified. If either of them are specified, then it checks if - the information is consistent and returns the input shape. +####################################################### Device Related ####################################################### + +def parse_device_id(device_id: str) -> Tuple[str, int]: + """Parse device id into device type and device index. + + Args: + device_id (str): Device id typically of the form `device_type:index`. + In case no index is specified, the default index of zero is returned. + + Returns: + Tuple[str, int]: (device_type, index) Returns a tuple with the index and device type. + """ + parts = device_id.split(sep=":") + if len(parts) == 1: + device_type = parts[0] + index = 0 + elif len(parts) == 2: + device_type, index = parts + else: + raise Exception( + "Device id not understood. A string of form `device_type:index` expected." + ) + + return device_type, int(index) + +def get_device_id(device_type: str, index: int) -> str: + """Generate a device id string given a device type and its index. + + Args: + device_type (str): Device type + index (int): Device index + + Returns: + str: A string of the form `device_type:index` + """ + return f"{device_type}:{index}" + +def standardize_device_id(device_id: str) -> str: + """Standardize device id string. + + Args: + device_id (str): Device id string. Could be of the form `device_type` or `device_type:index` + + Returns: + str: Returns a sanitized device id of the form `device_type:index` + """ + device_type, index = parse_device_id(device_id=device_id) + return get_device_id(device_type=device_type, index=index) + +####################################################### DynapcnnNetwork Related ####################################################### + +def build_nodes_to_dcnnl_map(layers: Dict[int, nn.Module], edges: List[Tuple[int, int]]) -> dict: + """ Initializes and populates a `dict` that will map data into a future `DynapcnnLayer` instance. The call + to `process_edge()` initializes a `key` (the index of a `DynapcnnLayer`) and assigns to it a dict containing the + nodes (layers in a `nn.Module`) that should belong to the same `DynapcnnLayer`. The call to `get_dynapcnnlayers_destinations()` + further incorporates to each "DynapcnnLayer dictionary" a `destinations` attribute, which is a list of integers indicating the + the target destinations of a `DynapcnnLayer` instance. + + Parameters + --------- + layers (dict): constains the nodes of a graph as `key` and their associated module as `value`. + edges (list): edges describing how nodes connect to each other. + + Returns + --------- + nodes_to_dcnnl_map (dict): each entry represents the gathered data necessary to instantiate a `DynapcnnLayer` object (e.g. nodes, + their I/O shapes, the list of `DynapcnnLayer` that are to be targeted, etc). + """ + # @TODO the graph extraction is not yet considering DVS input. + + # dvs_layer, lyr_indx_next, rescale_factor = construct_dvs_layer( + # layers, + # input_shape=in_shape, + # idx_start=0, + # dvs_input=False) + + dvs_layer = None + + # mapper from nodes to sets of layers that populate a DynapcnnLayer. + nodes_to_dcnnl_map = {} + + if dvs_layer is not None: + # TODO the graph extraction is not yet considering DVS input. + pass + else: + for edge in edges: + # Figure out to which (future) DynapcnnLayer each node will belong to. + process_edge(layers, edge, nodes_to_dcnnl_map) + + # look for edges between connecting nodes in different (future) DynapcnnLayer. + get_dynapcnnlayers_destinations(layers, edges, nodes_to_dcnnl_map) + + return nodes_to_dcnnl_map + +def build_from_graph( + discretize: bool, + edges: List[Tuple[int, int]], + nodes_to_dcnnl_map: dict, + weight_rescaling_fn: Callable) -> dict: + """ Parses each edge in `edges`, where each node is a set of layer that will compose a `DynapcnnLayer`. The + target destination of each `DynapcnnLayer` is computed via edges connecting nodes in different `DynapcnnLayer` + instances. Parameters ---------- - layers: - List of modules - input_shape : - (channels, height, width) + - discretize (bool): if `True` the weights of all convolutional layers are discretized. + - edges (list): edges describing how nodes connect to each other. + - nodes_to_dcnnl_map (dict): each entry represents the gathered data necessary to instantiate a `DynapcnnLayer` object (e.g. nodes, + their I/O shapes, the list of `DynapcnnLayer` that are to be targeted, etc). + - weight_rescaling_fn (callable): a method that handles how the re-scaling factor for one or more `SumPool2d` projecting to + the same convolutional layer are combined/re-scaled before applying them. Returns - ------- - Output shape: - (channels, height, width) + ---------- + - dynapcnn_layers (dict): `DynapcnnLayer` instances, each created from an entry in `nodes_to_dcnnl_map`. """ - if input_shape is not None and len(input_shape) != 3: - raise InputConfigurationError( - f"input_shape expected to have length 3 or None but input_shape={input_shape} given." - ) - input_shape_from_layer = None - if layers and isinstance(layers[0], DVSLayer): - input_shape_from_layer = layers[0].input_shape - if len(input_shape_from_layer) != 3: - raise InputConfigurationError( - f"input_shape of layer {layers[0]} expected to have length 3 or None but input_shape={input_shape_from_layer} found." - ) - if (input_shape is not None) and (input_shape_from_layer is not None): - if input_shape == input_shape_from_layer: - return input_shape - else: - raise InputConfigurationError( - f"Input shape from the layer {input_shape_from_layer} does not match the specified input_shape {input_shape}" - ) - elif input_shape_from_layer is not None: - return input_shape_from_layer - elif input_shape is not None: - return input_shape + # turn each entry in `nodes_to_dcnnl_map` into a `DynapcnnLayer` instance. + dynapcnn_layers = construct_dynapcnnlayers_from_mapper(discretize, nodes_to_dcnnl_map, edges, weight_rescaling_fn) + + # initialize attribute holding to which core a `DynapcnnLayer` instance in `dynapcnn_layers` will be mapped to. + for idx, layer_data in dynapcnn_layers.items(): + if 'core_idx' not in layer_data: + # a `DynapcnnLayer` gets assigned a core index when `DynapcnnNetworkGraph.to()`` is called. + layer_data['core_idx'] = -1 + + return dynapcnn_layers + +def construct_dynapcnnlayers_from_mapper( + discretize: bool, + nodes_to_dcnnl_map: dict, + edges: List[Tuple[int, int]], + weight_rescaling_fn: Callable) -> Dict[int, Dict[DynapcnnLayer, List]]: + """ Consumes a dictionaries containing sets of layers to be used to populate a DynapcnnLayer object. + + Parameters + ---------- + - discretize (bool): if `True` the weights of all convolutional layers are discretized. + - nodes_to_dcnnl_map (dict): each entry represents the gathered data necessary to instantiate a `DynapcnnLayer` object (e.g. nodes, + their I/O shapes, the list of `DynapcnnLayer` that are to be targeted, etc). + - edges (list): edges describing how nodes connect to each other. + - weight_rescaling_fn (callable): a method that handles how the re-scaling factor for one or more `SumPool2d` projecting to + the same convolutional layer are combined/re-scaled before applying them. + + Returns + ---------- + - dynapcnn_layers (dict): `DynapcnnLayer` instances, each created from an entry in `nodes_to_dcnnl_map`. + """ + + dynapcnn_layers = {} + + for dpcnnl_idx, dcnnl_data in nodes_to_dcnnl_map.items(): + # create a `DynapcnnLayer` from the set of layers in `dcnnl_data`. + dynapcnnlayer = construct_dynapcnnlayer( + dpcnnl_idx, discretize, dcnnl_data, edges, nodes_to_dcnnl_map, weight_rescaling_fn) + + dynapcnn_layers[dpcnnl_idx] = { + 'layer': dynapcnnlayer, + 'destinations': nodes_to_dcnnl_map[dpcnnl_idx]['destinations'] + } + + # check if any node (layer) in `dynapcnnlayer` has been modified (e.g. `nn.Linear` turned `nn.Conv2d`). + node, output_shape = dynapcnnlayer.get_modified_node_it(dcnnl_data) + + # one of the layers in `dynapcnnlayer` had its type modified (update input shape of nodes receiving from it). + if isinstance(node, int) and isinstance(output_shape, tuple): + update_nodes_io(node, output_shape, nodes_to_dcnnl_map, edges) + + return dynapcnn_layers + +def update_nodes_io(updated_node: int, output_shape: tuple, nodes_to_dcnnl_map: dict, edges: List[Tuple[int, int]]) -> None: + """ .""" + for edge in edges: + if edge[0] == updated_node: + # found source node where output shape has been modified. + for dcnnl_idx, dcnnl_data in nodes_to_dcnnl_map.items(): + for key, val in dcnnl_data.items(): + if isinstance(key, int): + # accessing node data (layer, input_shape, output_shape). + if key == edge[1]: + # accessing node targeted by `updated_node` (its input shape becomes `updated_node.output_shape`). + val['input_shape'] = output_shape + +def construct_dynapcnnlayer( + dpcnnl_idx: int, + discretize: bool, + dcnnl_data: dict, + edges: List[Tuple[int, int]], + nodes_to_dcnnl_map: dict, + weight_rescaling_fn: Callable) -> DynapcnnLayer: + """ Extract the modules (layers) in a dictionary and uses them to instantiate a DynapcnnLayer object. + + Parameters + ---------- + - dcnnl_data (dict): contains the nodes to be merged into a DynapcnnLayer, their I/O shapes and the index of the other DynapcnnLayers to + be set as destinations. + - weight_rescaling_fn (callable): a method that handles how the re-scaling factor for one or more `SumPool2d` projecting to + the same convolutional layer are combined/re-scaled before applying them. + + Returns + ---------- + - + """ + + # convert all AvgPool2d in 'dcnnl_data' into SumPool2d. + convert_Avg_to_Sum_pooling(dcnnl_data, edges, nodes_to_dcnnl_map) + + # instantiate a DynapcnnLayer from the data in 'dcnnl_data'. + dynapcnnlayer = DynapcnnLayer( + dpcnnl_index = dpcnnl_idx, + dcnnl_data = dcnnl_data, + discretize = discretize, + sinabs_edges = edges, + weight_rescaling_fn = weight_rescaling_fn + ) + + return dynapcnnlayer + +def convert_Avg_to_Sum_pooling(dcnnl_data: dict, edges: list, nodes_to_dcnnl_map: dict): + """ Converts every `AvgPool2d` node within `dcnnl_data` into a `SumPool2d` and update their respective `rescale_factor` (to + be used when creating the `DynapcnnLayer` instance for this layer's destinations). + + Parameters + ---------- + dcnnl_data: ... + edges: ... + nodes_to_dcnnl_map: ... + """ + for key, value in dcnnl_data.items(): + if isinstance(key, int): + # accessing the node `key` dictionary. + + if isinstance(value['layer'], nn.AvgPool2d): + # convert AvgPool2d into SumPool2d. + lyr_pool, rescale_factor = build_SumPool2d(value['layer']) + + # turn avg into sum pool. + value['layer'] = lyr_pool + + # find which node `key` will target. + for edge in edges: + if edge[0] == key: + # find index of `DynapcnnLayer` where the target of `edge[0]` is. + trg_dcnnl_idx = find_nodes_dcnnl_idx(edge[1], nodes_to_dcnnl_map) + + # update the rescale factor for the target of node `key`. + nodes_to_dcnnl_map[trg_dcnnl_idx]['conv_rescale_factor'].append(rescale_factor) + +def find_nodes_dcnnl_idx(node, nodes_to_dcnnl_map): + """ .""" + for dcnnl_idx, dcnnl_data in nodes_to_dcnnl_map.items(): + for key, value in dcnnl_data.items(): + if isinstance(key, int): + # `key` is a node. + if key == node: + # node belongs to DynapcnnLayer index `dcnnl_idx`. + return dcnnl_idx + + # this exception should never happen. + raise ValueError(f'Node {node} is not part of any dictionary mapping into a DynapcnnLayer.') + +def build_SumPool2d(module: nn.AvgPool2d) -> Tuple[sl.SumPool2d, int]: + """ Converts a `nn.AvgPool2d` into a `sl.SumPool2d` layer. + + Parameters + ---------- + module (torch.nn.AvgPool2d): the average pooling layer being converted into a sum pooling layer. + + Returns + ---------- + lyr_pool (sinabs.layers.SumPool2d): the equivalent sum pooling layer. + rescale_factor (int): the weight re-scaling computed for the weights of the convolution layer targeted by the pooling. + """ + + if isinstance(module, nn.AvgPool2d): + if module.padding != 0: + raise ValueError("Padding is not supported for the pooling layers.") + elif isinstance(module, sl.SumPool2d): + pass else: - raise InputConfigurationError("No input shape could be inferred") + raise WrongPoolingModule(type(module)) + + rescale_factor = 1 + cumulative_pooling = expand_to_pair(1) + pooling = expand_to_pair(module.kernel_size) + + if module.stride is not None: + stride = expand_to_pair(module.stride) + if pooling != stride: + raise ValueError( + f"Stride length {module.stride} should be the same as pooling kernel size {module.kernel_size}" + ) + + # compute cumulative pooling. + cumulative_pooling = ( + cumulative_pooling[0] * pooling[0], + cumulative_pooling[1] * pooling[1], + ) + + if isinstance(module, nn.AvgPool2d): + # update rescaling factor. + rescale_factor *= pooling[0] * pooling[1] + lyr_pool = sl.SumPool2d(cumulative_pooling) + + return lyr_pool, rescale_factor + +####################################################### MISSING FUNCTIONALITY ####################################################### +# TODO: these methods are currently not used by the new implementation of DynapcnnNetwork (but should). def convert_cropping2dlayer_to_crop2d( layer: sl.Cropping2dLayer, input_shape: Tuple[int, int] @@ -184,6 +449,8 @@ def construct_dvs_layer( def merge_conv_bn(conv, bn): """Merge a convolutional layer with subsequent batch normalization. + # TODO: new implementation of 'DynapcnnLayer' is not handling BN layers yet. + Parameters ---------- conv: torch.nn.Conv2d @@ -215,7 +482,6 @@ def merge_conv_bn(conv, bn): return conv - def construct_next_pooling_layer( layers: List[nn.Module], idx_start: int ) -> Tuple[Optional[sl.SumPool2d], int, float]: @@ -273,238 +539,13 @@ def construct_next_pooling_layer( if isinstance(lyr, nn.AvgPool2d): rescale_factor *= pooling[0] * pooling[1] - # If there are no layers - if cumulative_pooling == (1, 1): - return None, idx_next, 1 - else: - lyr_pool = sl.SumPool2d(cumulative_pooling) - return lyr_pool, idx_next, rescale_factor - - -def construct_next_dynapcnn_layer( - layers: List[nn.Module], - idx_start: int, - in_shape: Tuple[int, int, int], - discretize: bool, - rescale_factor: float = 1, -) -> Tuple[DynapcnnLayer, int, float]: - """Generate a DynapcnnLayer from a Conv2d layer and its subsequent spiking and pooling layers. - - Parameters - ---------- - - layers: sequence of layer objects - First object must be Conv2d, next must be an IAF layer. All pooling - layers that follow immediately are consolidated. Layers after this - will be ignored. - idx_start: - Layer index to start construction from - in_shape: tuple of integers - Shape of the input to the first layer in `layers`. Convention: - (input features, height, width) - discretize: bool - Discretize weights and thresholds if True - rescale_factor: float - Weights of Conv2d layer are scaled down by this factor. Can be - used to account for preceding average pooling that gets converted - to sum pooling. - - Returns - ------- - dynapcnn_layer: DynapcnnLayer - DynapcnnLayer - layer_idx_next: int - Index of the next layer after this layer is constructed - rescale_factor: float - rescaling factor to account for average pooling - """ - layer_idx_next = idx_start # Keep track of layer indices - - # Check that the first layer is Conv2d, or Linear - if not isinstance(layers[layer_idx_next], (nn.Conv2d, nn.Linear)): - raise UnexpectedLayer(nn.Conv2d, layers[layer_idx_next]) - - # Identify and consolidate conv layer - lyr_conv = layers[layer_idx_next] - layer_idx_next += 1 - if layer_idx_next >= len(layers): - raise MissingLayer(layer_idx_next) - # Check and consolidate batch norm - if isinstance(layers[layer_idx_next], nn.BatchNorm2d): - lyr_conv = merge_conv_bn(lyr_conv, layers[layer_idx_next]) - layer_idx_next += 1 - - # Check next layer exists - try: - lyr_spk = layers[layer_idx_next] - layer_idx_next += 1 - except IndexError: - raise MissingLayer(layer_idx_next) - - # Check that the next layer is spiking - # TODO: Check that the next layer is an IAF layer - if not isinstance(lyr_spk, sl.IAF): - raise TypeError( - f"Convolution must be followed by IAF spiking layer, found {type(lyr_spk)}" - ) - - # Check for next pooling layer - lyr_pool, i_next, rescale_factor_after_pooling = construct_next_pooling_layer( - layers, layer_idx_next - ) - # Increment layer index to after the pooling layers - layer_idx_next = i_next - - # Compose DynapcnnLayer - dynapcnn_layer = DynapcnnLayer( - conv=lyr_conv, - spk=lyr_spk, - pool=lyr_pool, - in_shape=in_shape, - discretize=discretize, - rescale_weights=rescale_factor, - ) - - return dynapcnn_layer, layer_idx_next, rescale_factor_after_pooling - - -def build_from_list( - layers: List[nn.Module], - in_shape, - discretize=True, - dvs_input=False, -) -> nn.Sequential: - """Build a sequential model of DVSLayer and DynapcnnLayer(s) given a list of layers comprising - a spiking CNN. - - Parameters - ---------- - - layers: sequence of layer objects - in_shape: tuple of integers - Shape of the input to the first layer in `layers`. Convention: - (channels, height, width) - discretize: bool - Discretize weights and thresholds if True - dvs_input: bool - Whether model should receive DVS input. If `True`, the returned model - will begin with a DVSLayer with `disable_pixel_array` set to False. - Otherwise, the model starts with a DVSLayer only if the first element - in `layers` is a pooling, cropping or flipping layer. - - Returns - ------- - nn.Sequential - """ - compatible_layers = [] - lyr_indx_next = 0 - # Find and populate dvs layer (NOTE: We are ignoring the channel information here and could lead to problems) - dvs_layer, lyr_indx_next, rescale_factor = construct_dvs_layer( - layers, input_shape=in_shape, idx_start=lyr_indx_next, dvs_input=dvs_input - ) - - if dvs_layer is not None: - compatible_layers.append(dvs_layer) - in_shape = dvs_layer.get_output_shape() - # Find and populate dynapcnn layers - while lyr_indx_next < len(layers): - if isinstance(layers[lyr_indx_next], DEFAULT_IGNORED_LAYER_TYPES): - # - Ignore identity, dropout and flatten layers - lyr_indx_next += 1 - continue - dynapcnn_layer, lyr_indx_next, rescale_factor = construct_next_dynapcnn_layer( - layers, - lyr_indx_next, - in_shape=in_shape, - discretize=discretize, - rescale_factor=rescale_factor, - ) - in_shape = dynapcnn_layer.get_output_shape() - compatible_layers.append(dynapcnn_layer) - - return nn.Sequential(*compatible_layers) - - -def convert_model_to_layer_list( - model: Union[nn.Sequential, sinabs.Network, nn.Module], - ignore: Union[Type, Tuple[Type, ...]] = (), -) -> List[nn.Module]: - """Convert a model to a list of layers. - - Parameters - ---------- - model: nn.Sequential, nn.Module or sinabs.Network. - ignore: type or tuple of types of modules to be ignored. - - Returns - ------- - List[nn.Module] - """ - if isinstance(model, sinabs.Network): - return convert_model_to_layer_list(model.spiking_model) - - elif isinstance(model, nn.Sequential): - layers = [layer for layer in model if not isinstance(layer, ignore)] - - elif isinstance(model, nn.Module): - layers = [layer for _, layer in model.named_children() if not isinstance(layer, ignore)] - - else: - raise TypeError("Expected torch.nn.Sequential or sinabs.Network") - - return layers - - -def parse_device_id(device_id: str) -> Tuple[str, int]: - """Parse device id into device type and device index. - - Args: - device_id (str): Device id typically of the form `device_type:index`. - In case no index is specified, the default index of zero is returned. - - Returns: - Tuple[str, int]: (device_type, index) Returns a tuple with the index and device type. - """ - parts = device_id.split(sep=":") - if len(parts) == 1: - device_type = parts[0] - index = 0 - elif len(parts) == 2: - device_type, index = parts - else: - raise Exception( - "Device id not understood. A string of form `device_type:index` expected." - ) - - return device_type, int(index) - - -def get_device_id(device_type: str, index: int) -> str: - """Generate a device id string given a device type and its index. - - Args: - device_type (str): Device type - index (int): Device index - - Returns: - str: A string of the form `device_type:index` - """ - return f"{device_type}:{index}" - - -def standardize_device_id(device_id: str) -> str: - """Standardize device id string. - - Args: - device_id (str): Device id string. Could be of the form `device_type` or `device_type:index` - - Returns: - str: Returns a sanitized device id of the form `device_type:index` - """ - device_type, index = parse_device_id(device_id=device_id) - return get_device_id(device_type=device_type, index=index) - - + # If there are no layers + if cumulative_pooling == (1, 1): + return None, idx_next, 1 + else: + lyr_pool = sl.SumPool2d(cumulative_pooling) + return lyr_pool, idx_next, rescale_factor + def extend_readout_layer(model: "DynapcnnNetwork") -> "DynapcnnNetwork": """Return a copied and extended model with the readout layer extended to 4 times the number of output channels. For Speck 2E and 2F, to get readout with correct output index, we need to @@ -546,258 +587,222 @@ def extend_readout_layer(model: "DynapcnnNetwork") -> "DynapcnnNetwork": ) # run a forward pass to initialize the new weights and last IAF return model -def build_nodes_to_dcnnl_map(layers: Dict[int, nn.Module], edges: List[Tuple[int, int]]) -> dict: - """ Initializes and populates a `dict` that will map data into a future `DynapcnnLayer` instance. The call - to `process_edge()` initializes a `key` (the index of a `DynapcnnLayer`) and assigns to it a dict containing the - nodes (layers in a `nn.Module`) that should belong to the same `DynapcnnLayer`. The call to `get_dynapcnnlayers_destinations()` - further incorporates to each "DynapcnnLayer dictionary" a `destinations` attribute, which is a list of integers indicating the - the target destinations of a `DynapcnnLayer` instance. - - Parameters - --------- - layers (dict): constains the nodes of a graph as `key` and their associated module as `value`. - edges (list): edges describing how nodes connect to each other. - - Returns - --------- - nodes_to_dcnnl_map (dict): each entry represents the gathered data necessary to instantiate a `DynapcnnLayer` object (e.g. nodes, - their I/O shapes, the list of `DynapcnnLayer` that are to be targeted, etc). - """ - # @TODO the graph extraction is not yet considering DVS input. - - # dvs_layer, lyr_indx_next, rescale_factor = construct_dvs_layer( - # layers, - # input_shape=in_shape, - # idx_start=0, - # dvs_input=False) - - dvs_layer = None - - # mapper from nodes to sets of layers that populate a DynapcnnLayer. - nodes_to_dcnnl_map = {} - - if dvs_layer is not None: - # TODO the graph extraction is not yet considering DVS input. - pass - else: - for edge in edges: - # Figure out to which (future) DynapcnnLayer each node will belong to. - process_edge(layers, edge, nodes_to_dcnnl_map) - - # look for edges between connecting nodes in different (future) DynapcnnLayer. - get_dynapcnnlayers_destinations(layers, edges, nodes_to_dcnnl_map) - - return nodes_to_dcnnl_map - -def build_from_graph( - discretize: bool, - edges: List[Tuple[int, int]], - nodes_to_dcnnl_map: dict, - weight_rescaling_fn: Callable) -> dict: - """ Parses each edge in `edges`, where each node is a set of layer that will compose a `DynapcnnLayer`. The - target destination of each `DynapcnnLayer` is computed via edges connecting nodes in different `DynapcnnLayer` - instances. - - Parameters - ---------- - - discretize (bool): if `True` the weights of all convolutional layers are discretized. - - edges (list): edges describing how nodes connect to each other. - - nodes_to_dcnnl_map (dict): each entry represents the gathered data necessary to instantiate a `DynapcnnLayer` object (e.g. nodes, - their I/O shapes, the list of `DynapcnnLayer` that are to be targeted, etc). - - weight_rescaling_fn (callable): a method that handles how the re-scaling factor for one or more `SumPool2d` projecting to - the same convolutional layer are combined/re-scaled before applying them. - - Returns - ---------- - - dynapcnn_layers (dict): `DynapcnnLayer` instances, each created from an entry in `nodes_to_dcnnl_map`. - """ - - # turn each entry in `nodes_to_dcnnl_map` into a `DynapcnnLayer` instance. - dynapcnn_layers = construct_dynapcnnlayers_from_mapper(discretize, nodes_to_dcnnl_map, edges, weight_rescaling_fn) - - # initialize attribute holding to which core a `DynapcnnLayer` instance in `dynapcnn_layers` will be mapped to. - for idx, layer_data in dynapcnn_layers.items(): - if 'core_idx' not in layer_data: - # a `DynapcnnLayer` gets assigned a core index when `DynapcnnNetworkGraph.to()`` is called. - layer_data['core_idx'] = -1 - - return dynapcnn_layers - -def construct_dynapcnnlayers_from_mapper( - discretize: bool, - nodes_to_dcnnl_map: dict, - edges: List[Tuple[int, int]], - weight_rescaling_fn: Callable) -> Dict[int, Dict[DynapcnnLayer, List]]: - """ Consumes a dictionaries containing sets of layers to be used to populate a DynapcnnLayer object. - - Parameters - ---------- - - discretize (bool): if `True` the weights of all convolutional layers are discretized. - - nodes_to_dcnnl_map (dict): each entry represents the gathered data necessary to instantiate a `DynapcnnLayer` object (e.g. nodes, - their I/O shapes, the list of `DynapcnnLayer` that are to be targeted, etc). - - edges (list): edges describing how nodes connect to each other. - - weight_rescaling_fn (callable): a method that handles how the re-scaling factor for one or more `SumPool2d` projecting to - the same convolutional layer are combined/re-scaled before applying them. - - Returns - ---------- - - dynapcnn_layers (dict): `DynapcnnLayer` instances, each created from an entry in `nodes_to_dcnnl_map`. - """ - - dynapcnn_layers = {} +####################################################### DEPRECATED METHODS ####################################################### +# TODO: these methods were used by the old implementation of DynapcnnNetwork - delete all. + +# def infer_input_shape( +# layers: List[nn.Module], input_shape: Optional[Tuple[int, int, int]] = None +# ) -> Tuple[int, int, int]: +# """Checks if the input_shape is specified. If either of them are specified, then it checks if +# the information is consistent and returns the input shape. + +# Parameters +# ---------- +# layers: +# List of modules +# input_shape : +# (channels, height, width) + +# Returns +# ------- +# Output shape: +# (channels, height, width) +# """ +# if input_shape is not None and len(input_shape) != 3: +# raise InputConfigurationError( +# f"input_shape expected to have length 3 or None but input_shape={input_shape} given." +# ) + +# input_shape_from_layer = None +# if layers and isinstance(layers[0], DVSLayer): +# input_shape_from_layer = layers[0].input_shape +# if len(input_shape_from_layer) != 3: +# raise InputConfigurationError( +# f"input_shape of layer {layers[0]} expected to have length 3 or None but input_shape={input_shape_from_layer} found." +# ) +# if (input_shape is not None) and (input_shape_from_layer is not None): +# if input_shape == input_shape_from_layer: +# return input_shape +# else: +# raise InputConfigurationError( +# f"Input shape from the layer {input_shape_from_layer} does not match the specified input_shape {input_shape}" +# ) +# elif input_shape_from_layer is not None: +# return input_shape_from_layer +# elif input_shape is not None: +# return input_shape +# else: +# raise InputConfigurationError("No input shape could be inferred") + +# def construct_next_dynapcnn_layer( +# layers: List[nn.Module], +# idx_start: int, +# in_shape: Tuple[int, int, int], +# discretize: bool, +# rescale_factor: float = 1, +# ) -> Tuple[DynapcnnLayer, int, float]: +# """Generate a DynapcnnLayer from a Conv2d layer and its subsequent spiking and pooling layers. + +# Parameters +# ---------- + +# layers: sequence of layer objects +# First object must be Conv2d, next must be an IAF layer. All pooling +# layers that follow immediately are consolidated. Layers after this +# will be ignored. +# idx_start: +# Layer index to start construction from +# in_shape: tuple of integers +# Shape of the input to the first layer in `layers`. Convention: +# (input features, height, width) +# discretize: bool +# Discretize weights and thresholds if True +# rescale_factor: float +# Weights of Conv2d layer are scaled down by this factor. Can be +# used to account for preceding average pooling that gets converted +# to sum pooling. + +# Returns +# ------- +# dynapcnn_layer: DynapcnnLayer +# DynapcnnLayer +# layer_idx_next: int +# Index of the next layer after this layer is constructed +# rescale_factor: float +# rescaling factor to account for average pooling +# """ +# layer_idx_next = idx_start # Keep track of layer indices + +# # Check that the first layer is Conv2d, or Linear +# if not isinstance(layers[layer_idx_next], (nn.Conv2d, nn.Linear)): +# raise UnexpectedLayer(nn.Conv2d, layers[layer_idx_next]) + +# # Identify and consolidate conv layer +# lyr_conv = layers[layer_idx_next] +# layer_idx_next += 1 +# if layer_idx_next >= len(layers): +# raise MissingLayer(layer_idx_next) +# # Check and consolidate batch norm +# if isinstance(layers[layer_idx_next], nn.BatchNorm2d): +# lyr_conv = merge_conv_bn(lyr_conv, layers[layer_idx_next]) +# layer_idx_next += 1 + +# # Check next layer exists +# try: +# lyr_spk = layers[layer_idx_next] +# layer_idx_next += 1 +# except IndexError: +# raise MissingLayer(layer_idx_next) + +# # Check that the next layer is spiking +# # TODO: Check that the next layer is an IAF layer +# if not isinstance(lyr_spk, sl.IAF): +# raise TypeError( +# f"Convolution must be followed by IAF spiking layer, found {type(lyr_spk)}" +# ) + +# # Check for next pooling layer +# lyr_pool, i_next, rescale_factor_after_pooling = construct_next_pooling_layer( +# layers, layer_idx_next +# ) +# # Increment layer index to after the pooling layers +# layer_idx_next = i_next + +# # Compose DynapcnnLayer +# dynapcnn_layer = DynapcnnLayer( +# conv=lyr_conv, +# spk=lyr_spk, +# pool=lyr_pool, +# in_shape=in_shape, +# discretize=discretize, +# rescale_weights=rescale_factor, +# ) + +# return dynapcnn_layer, layer_idx_next, rescale_factor_after_pooling + + +# def build_from_list( +# layers: List[nn.Module], +# in_shape, +# discretize=True, +# dvs_input=False, +# ) -> nn.Sequential: +# """Build a sequential model of DVSLayer and DynapcnnLayer(s) given a list of layers comprising +# a spiking CNN. + +# Parameters +# ---------- + +# layers: sequence of layer objects +# in_shape: tuple of integers +# Shape of the input to the first layer in `layers`. Convention: +# (channels, height, width) +# discretize: bool +# Discretize weights and thresholds if True +# dvs_input: bool +# Whether model should receive DVS input. If `True`, the returned model +# will begin with a DVSLayer with `disable_pixel_array` set to False. +# Otherwise, the model starts with a DVSLayer only if the first element +# in `layers` is a pooling, cropping or flipping layer. + +# Returns +# ------- +# nn.Sequential +# """ +# compatible_layers = [] +# lyr_indx_next = 0 +# # Find and populate dvs layer (NOTE: We are ignoring the channel information here and could lead to problems) +# dvs_layer, lyr_indx_next, rescale_factor = construct_dvs_layer( +# layers, input_shape=in_shape, idx_start=lyr_indx_next, dvs_input=dvs_input +# ) - for dpcnnl_idx, dcnnl_data in nodes_to_dcnnl_map.items(): - # create a `DynapcnnLayer` from the set of layers in `dcnnl_data`. - dynapcnnlayer = construct_dynapcnnlayer( - dpcnnl_idx, discretize, dcnnl_data, edges, nodes_to_dcnnl_map, weight_rescaling_fn) - - dynapcnn_layers[dpcnnl_idx] = { - 'layer': dynapcnnlayer, - 'destinations': nodes_to_dcnnl_map[dpcnnl_idx]['destinations'] - } - - # check if any node (layer) in `dynapcnnlayer` has been modified (e.g. `nn.Linear` turned `nn.Conv2d`). - node, output_shape = dynapcnnlayer.get_modified_node_it(dcnnl_data) - - # one of the layers in `dynapcnnlayer` had its type modified (update input shape of nodes receiving from it). - if isinstance(node, int) and isinstance(output_shape, tuple): - update_nodes_io(node, output_shape, nodes_to_dcnnl_map, edges) - - return dynapcnn_layers - -def update_nodes_io(updated_node: int, output_shape: tuple, nodes_to_dcnnl_map: dict, edges: List[Tuple[int, int]]) -> None: - """ .""" - for edge in edges: - if edge[0] == updated_node: - # found source node where output shape has been modified. - for dcnnl_idx, dcnnl_data in nodes_to_dcnnl_map.items(): - for key, val in dcnnl_data.items(): - if isinstance(key, int): - # accessing node data (layer, input_shape, output_shape). - if key == edge[1]: - # accessing node targeted by `updated_node` (its input shape becomes `updated_node.output_shape`). - val['input_shape'] = output_shape - -def construct_dynapcnnlayer( - dpcnnl_idx: int, - discretize: bool, - dcnnl_data: dict, - edges: List[Tuple[int, int]], - nodes_to_dcnnl_map: dict, - weight_rescaling_fn: Callable) -> DynapcnnLayer: - """ Extract the modules (layers) in a dictionary and uses them to instantiate a DynapcnnLayer object. - - Parameters - ---------- - - dcnnl_data (dict): contains the nodes to be merged into a DynapcnnLayer, their I/O shapes and the index of the other DynapcnnLayers to - be set as destinations. - - weight_rescaling_fn (callable): a method that handles how the re-scaling factor for one or more `SumPool2d` projecting to - the same convolutional layer are combined/re-scaled before applying them. +# if dvs_layer is not None: +# compatible_layers.append(dvs_layer) +# in_shape = dvs_layer.get_output_shape() +# # Find and populate dynapcnn layers +# while lyr_indx_next < len(layers): +# if isinstance(layers[lyr_indx_next], DEFAULT_IGNORED_LAYER_TYPES): +# # - Ignore identity, dropout and flatten layers +# lyr_indx_next += 1 +# continue +# dynapcnn_layer, lyr_indx_next, rescale_factor = construct_next_dynapcnn_layer( +# layers, +# lyr_indx_next, +# in_shape=in_shape, +# discretize=discretize, +# rescale_factor=rescale_factor, +# ) +# in_shape = dynapcnn_layer.get_output_shape() +# compatible_layers.append(dynapcnn_layer) + +# return nn.Sequential(*compatible_layers) + + +# def convert_model_to_layer_list( +# model: Union[nn.Sequential, sinabs.Network, nn.Module], +# ignore: Union[Type, Tuple[Type, ...]] = (), +# ) -> List[nn.Module]: +# """Convert a model to a list of layers. + +# Parameters +# ---------- +# model: nn.Sequential, nn.Module or sinabs.Network. +# ignore: type or tuple of types of modules to be ignored. + +# Returns +# ------- +# List[nn.Module] +# """ +# if isinstance(model, sinabs.Network): +# return convert_model_to_layer_list(model.spiking_model) - Returns - ---------- - - - """ - - # convert all AvgPool2d in 'dcnnl_data' into SumPool2d. - convert_Avg_to_Sum_pooling(dcnnl_data, edges, nodes_to_dcnnl_map) - - # instantiate a DynapcnnLayer from the data in 'dcnnl_data'. - dynapcnnlayer = DynapcnnLayer( - dpcnnl_index = dpcnnl_idx, - dcnnl_data = dcnnl_data, - discretize = discretize, - sinabs_edges = edges, - weight_rescaling_fn = weight_rescaling_fn - ) - - return dynapcnnlayer - -def convert_Avg_to_Sum_pooling(dcnnl_data: dict, edges: list, nodes_to_dcnnl_map: dict): - """ Converts every `AvgPool2d` node within `dcnnl_data` into a `SumPool2d` and update their respective `rescale_factor` (to - be used when creating the `DynapcnnLayer` instance for this layer's destinations). - - Parameters - ---------- - dcnnl_data: ... - edges: ... - nodes_to_dcnnl_map: ... - """ - for key, value in dcnnl_data.items(): - if isinstance(key, int): - # accessing the node `key` dictionary. - - if isinstance(value['layer'], nn.AvgPool2d): - # convert AvgPool2d into SumPool2d. - lyr_pool, rescale_factor = build_SumPool2d(value['layer']) - - # turn avg into sum pool. - value['layer'] = lyr_pool +# elif isinstance(model, nn.Sequential): +# layers = [layer for layer in model if not isinstance(layer, ignore)] - # find which node `key` will target. - for edge in edges: - if edge[0] == key: - # find index of `DynapcnnLayer` where the target of `edge[0]` is. - trg_dcnnl_idx = find_nodes_dcnnl_idx(edge[1], nodes_to_dcnnl_map) - - # update the rescale factor for the target of node `key`. - nodes_to_dcnnl_map[trg_dcnnl_idx]['conv_rescale_factor'].append(rescale_factor) - -def find_nodes_dcnnl_idx(node, nodes_to_dcnnl_map): - """ .""" - for dcnnl_idx, dcnnl_data in nodes_to_dcnnl_map.items(): - for key, value in dcnnl_data.items(): - if isinstance(key, int): - # `key` is a node. - if key == node: - # node belongs to DynapcnnLayer index `dcnnl_idx`. - return dcnnl_idx - - # this exception should never happen. - raise ValueError(f'Node {node} is not part of any dictionary mapping into a DynapcnnLayer.') - -def build_SumPool2d(module: nn.AvgPool2d) -> Tuple[sl.SumPool2d, int]: - """ Converts a `nn.AvgPool2d` into a `sl.SumPool2d` layer. - - Parameters - ---------- - module (torch.nn.AvgPool2d): the average pooling layer being converted into a sum pooling layer. +# elif isinstance(model, nn.Module): +# layers = [layer for _, layer in model.named_children() if not isinstance(layer, ignore)] - Returns - ---------- - lyr_pool (sinabs.layers.SumPool2d): the equivalent sum pooling layer. - rescale_factor (int): the weight re-scaling computed for the weights of the convolution layer targeted by the pooling. - """ - - if isinstance(module, nn.AvgPool2d): - if module.padding != 0: - raise ValueError("Padding is not supported for the pooling layers.") - elif isinstance(module, sl.SumPool2d): - pass - else: - raise WrongPoolingModule(type(module)) +# else: +# raise TypeError("Expected torch.nn.Sequential or sinabs.Network") - rescale_factor = 1 - cumulative_pooling = expand_to_pair(1) - pooling = expand_to_pair(module.kernel_size) - - if module.stride is not None: - stride = expand_to_pair(module.stride) - if pooling != stride: - raise ValueError( - f"Stride length {module.stride} should be the same as pooling kernel size {module.kernel_size}" - ) - - # compute cumulative pooling. - cumulative_pooling = ( - cumulative_pooling[0] * pooling[0], - cumulative_pooling[1] * pooling[1], - ) - - if isinstance(module, nn.AvgPool2d): - # update rescaling factor. - rescale_factor *= pooling[0] * pooling[1] - - lyr_pool = sl.SumPool2d(cumulative_pooling) - - return lyr_pool, rescale_factor \ No newline at end of file +# return layers \ No newline at end of file From 71c6c92765901a1a0addd502cbc3597f9b832062 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Mon, 20 May 2024 17:24:13 +0200 Subject: [PATCH 097/379] Documentation - Code in-line documentation updated for some methods/functions. - Type hint for the return of some methods/functions updated. --- sinabs/backend/dynapcnn/dynapcnn_network.py | 49 ++++++++++++----- .../dynapcnn/dynapcnnnetwork_module.py | 54 ++++++++++++++----- .../dynapcnn/weight_rescaling_methods.py | 37 +++++++++---- 3 files changed, 106 insertions(+), 34 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index 872f1c8b..4f3c5441 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -31,6 +31,8 @@ from .dynapcnnnetwork_module import DynapcnnNetworkModule from .weight_rescaling_methods import rescale_method_1, rescale_method_2 +from .dynapcnn_layer import DynapcnnLayer + class DynapcnnNetwork(nn.Module): def __init__( self, @@ -120,12 +122,28 @@ def __init__( ####################################################### Public Methods ####################################################### @property - def forward_map(self) -> dict: - """ This dictionary contains each `DynapcnnLayer` in the model indexed by their ID (layer index). """ + def forward_map(self) -> Dict[int, DynapcnnLayer]: + """ This dictionary contains each `DynapcnnLayer` in the model indexed by their ID (layer index). + + Returns + ---------- + - self._forward_map (dict): a mapper used to forward data through the `DynapcnnNetwork` instance when `self.forward` is called. + """ return self._forward_map def forward(self, x): - """ .""" + """ Forwards data through the `DynapcnnNetwork` instance. This method relies on three main data structures created to represent + the `DynapcnnLayer`s in the network and the data propagation through them during the forward pass: + + - `self._dcnnl_edges` (list): this is used to guide the sequence in which the `DynapcnnLayer`s in `self._forward_map` are to be called + to generate the input tensors to be propagated through the network during the forward pass. This list of edges represent the graph + describing the interactions between each `DynapcnnLayer` (the nodes in the edges are the indices of these layers). + - `self._forward_map` (dict): a mapper used to forward data through the `DynapcnnNetwork` instances. Each `key` is the indice associated + with a `DynapcnnLayer` instance. The edges in `self._dcnnl_edges` are accessed sequentially and each node in an edge is used to index + a forward call via `self._forward_map`. + - `self._merge_points` (dict): this mapper has a "support" role. It indexes wich convolutional layers in the set of `DynapcnnLayer`s + composing the network require two sources of input (because their input tensor is the output of a `Merge` layer). + """ layers_outputs = {} @@ -182,14 +200,14 @@ def forward(self, x): return layers_outputs[trg_dcnnl][0] def parameters(self) -> list: - """ Gathers all the parameters of the network in a list. This is done by accessing the convolutional layer in each `DynapcnnLayer`, calling - its `.parameters` method and saving it to a list. + """ Gathers all the parameters of the network in a list. This is done by accessing the convolutional layer in each `DynapcnnLayer`, + calling its `.parameters` method and saving it to a list. Note: the method assumes no biases are used. Returns ---------- - parameters (list): a list of parameters of all convolutional layers in the `DynapcnnNetwok`. + - parameters (list): a list of parameters of all convolutional layers in the `DynapcnnNetwok`. """ parameters = [] @@ -200,7 +218,12 @@ def parameters(self) -> list: return parameters def init_weights(self, init_fn: nn.init = nn.init.xavier_normal_) -> None: - """ Call the weight initialization method `init_fn` on each `DynapcnnLayer.conv_layer.weight.data` in the `DynapcnnNetwork` instance.""" + """ Call the weight initialization method `init_fn` on each `DynapcnnLayer.conv_layer.weight.data` in the `DynapcnnNetwork` instance. + + Parameters + ---------- + - init_fn (torch.nn.init): the weight initialization method to be used. + """ for layer in self._forward_map.values(): if isinstance(layer, sinabs.backend.dynapcnn.dynapcnn_layer_new.DynapcnnLayer): init_fn(layer.conv_layer.weight.data) @@ -434,13 +457,15 @@ def _get_network_module(self) -> Union[list, dict, dict]: """ Uses the `DynapcnnLayer` instances in `self._dynapcnn_layers` and the connectivity between them to create three data structures that guide the data forwarding between the layer during the forward pass. - Note: the property `DynapcnnLayer.assigned_core` is only set after `self.to(device='speck...')` is called. - Returns ---------- - dcnnl_edges (list): edges, represented as tuples of `DynapcnnLayer` indices, used to guide the data forwarding through each `DynapcnnLayer` in forward method. - forward_map (dict): have all the `DynapcnnLayer` (`value`), each being accessible via its index (`key`). Used to call `DynapcnnLayer.forward` in forward method. - merge_points (dict): used to compose the inputs to a `DynapcnnLayer` that requires an input from a `Merge` layer. + - dcnnl_edges (list): edges, represented as tuples of `DynapcnnLayer` indices, used to guide the data forwarding through each `DynapcnnLayer` in forward method. + - forward_map (dict): have all the `DynapcnnLayer` (`value`), each being accessible via its index (`key`). Used to call `DynapcnnLayer.forward` in forward method. + - merge_points (dict): used to compose the inputs to a `DynapcnnLayer` that requires an input from a `Merge` layer. + + Notes + ---------- + - the property `DynapcnnLayer.assigned_core` is only set after `self.to(device='speck...')` is called. """ # get connections between `DynapcnnLayer`s. diff --git a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py index 5d3d471a..263bb60c 100644 --- a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py +++ b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py @@ -1,13 +1,11 @@ -# functionality : ... -# author : Willian Soares Girao -# contact : williansoaresgirao@gmail.com +# author : Willian Soares Girao +# contact : williansoaresgirao@gmail.com import torch.nn as nn -from sinabs.layers import Merge from typing import List, Tuple, Dict, Union import copy -import sinabs import sinabs.layers as sl +from .dynapcnn_layer import DynapcnnLayer class DynapcnnNetworkModule(): """ @@ -15,10 +13,10 @@ class DynapcnnNetworkModule(): Parameters ---------- - dcnnl_edges (list): tuples representing the output->input mapping between `DynapcnnLayer` instances - that have been used as configuration for each core `CNNLayerConifg`. - dynapcnn_layers (dict): the `DynapcnnLayer` instances along with their supporting metadata (e.g. assigned core, - destination layers, etc.). + - dcnnl_edges (list): tuples representing the output->input mapping between `DynapcnnLayer` instances + that have been used as configuration for each core `CNNLayerConifg`. + - dynapcnn_layers (dict): a mapper containing `DynapcnnLayer` instances along with their supporting metadata (e.g. assigned core, + destination layers, etc.). """ def __init__(self, dcnnl_edges: List[Tuple[int, int]], dynapcnn_layers: Dict): @@ -27,8 +25,15 @@ def __init__(self, dcnnl_edges: List[Tuple[int, int]], dynapcnn_layers: Dict): self.forward_map, self.merge_points = self._build_module_forward_from_graph(dcnnl_edges, dynapcnn_layers) - def _spot_merging_points(self, dcnnl_edges: list) -> dict: - """ . """ + def _spot_merging_points(self, dcnnl_edges: list) -> Dict[int, Dict[Tuple, sl.Merge]]: + """ Loops throught the edges of the computational graph from a `DynapcnnNetwork` to flag with nodes need + input from a `Merge` layer and what the arguments of this layer should be. + + Parameters + ---------- + - dcnnl_edges (list): tuples representing the output->input mapping between `DynapcnnLayer` instances + that have been used as configuration for each core `CNNLayerConifg`. + """ nodes_with_merge_input = {} @@ -37,12 +42,15 @@ def _spot_merging_points(self, dcnnl_edges: list) -> dict: fan_in = 0 src_nodes = [] + # counts the fan-in for each target node `trg_node`. for edge_inner in dcnnl_edges: if edge_inner[1] == trg_node: + # fan-in update. fan_in += 1 src_nodes.append(edge_inner[0]) if fan_in == 2 and trg_node not in nodes_with_merge_input: + # node needs input from a `Merge` layer: instantiate `Merge` and its arguments. nodes_with_merge_input[trg_node] = {'sources': tuple(src_nodes), 'merge': sl.Merge()} if fan_in > 2: @@ -50,8 +58,28 @@ def _spot_merging_points(self, dcnnl_edges: list) -> dict: return nodes_with_merge_input - def _build_module_forward_from_graph(self, dcnnl_edges: list, dynapcnn_layers: dict) -> Union[dict, dict]: - """ .""" + def _build_module_forward_from_graph( + self, + dcnnl_edges: list, + dynapcnn_layers: dict) -> Union[Dict[int, DynapcnnLayer], Dict[Tuple, sl.Merge]]: + """ Creates two mappers, one indexing each `DynapcnnLayer` by its index (a node in `dcnnl_edges`) a another + indexing the `DynapcnnLayer` instances (also by the index) that need their input being the output of a + `Merge` layer (i.e., they are nodes in the graph where two different layer outputs converge to). + + Parameters + ---------- + - dcnnl_edges (list): tuples representing the output->input mapping between `DynapcnnLayer` instances + that have been used as configuration for each core `CNNLayerConifg`. + - dynapcnn_layers (dict): a mapper containing `DynapcnnLayer` instances along with their supporting metadata (e.g. assigned core, + destination layers, etc.). + + Returns + ---------- + - forward_map (dict): a mapper where each `key` is the layer index (`DynapcnnLayer.dpcnnl_index`) and the `value` the layer instance itself. + - merge_points (dict): a mapper where each `key` is the layer index and the `value` is a dictionary with a `Merge` layer (`merge_points[key]['merge'] = Merge()`, + computing the input tensor to layer `key`) and its arguments (`merge_points[key]['sources'] = (int A, int B)`, where `A` and `B` are the `DynapcnnLayer` + instances for which the ouput is to be used as the `Merge` arguments). + """ # mapper to flag nodes that need input from a `Merge` layer. merge_points = self._spot_merging_points(dcnnl_edges) diff --git a/sinabs/backend/dynapcnn/weight_rescaling_methods.py b/sinabs/backend/dynapcnn/weight_rescaling_methods.py index 2853f7fd..f41a08c2 100644 --- a/sinabs/backend/dynapcnn/weight_rescaling_methods.py +++ b/sinabs/backend/dynapcnn/weight_rescaling_methods.py @@ -1,33 +1,52 @@ +# author : Willian Soares Girao +# contact : williansoaresgirao@gmail.com + import numpy as np import statistics def rescale_method_1(rescaling_from_sumpool: list, lambda_: float = 0.5) -> float: """ - The `method 1` will use the average of the computed rescaling factor for each pooling layer - feeding into a convolutional layer (if there are more than one)... + This method will use the average (scaled by `lambda_`) of the computed re-scaling factor + for the pooling layer(s) feeding into a convolutional layer. Arguments --------- + - rescaling_from_sumpool (list): the list of re-scaling factors computed by each `SumPool2d` layer targeting a + single `Conv2d` layer within a `DynapcnnLayer` instance. + - lambda_ (float): a scaling variable that multiplies the computed average re-scaling factor of the pooling layers. + + Returns + --------- + - the averaged re-scaling factor multiplied by `lambda_` if `len(rescaling_from_sumpool) > 0`, else `1` is returned. """ if len(rescaling_from_sumpool): return np.round(np.mean(rescaling_from_sumpool)*lambda_, 2) else: - return 1 + return 1.0 def rescale_method_2(rescaling_from_sumpool: list, lambda_: float = 0.5) -> float: """ - The `method 2` will use the harmonic mean of the computed rescaling factor for each pooling layer - feeding into `conv_layer` (if there are more than one) ... - - Note: since the harmonic mean is less sensitive to outliers it **could be** that this is a better method - for weight re-scaling when multiple pooling with big differentces in kernel sizes are being considered. + This method will use the harmonic mean (scaled by `lambda_`) of the computed re-scaling factor + for the pooling layer(s) feeding into a convolutional layer. Arguments --------- + - rescaling_from_sumpool (list): the list of re-scaling factors computed by each `SumPool2d` layer targeting a + single `Conv2d` layer within a `DynapcnnLayer` instance. + - lambda_ (float): a scaling variable that multiplies the computed average re-scaling factor of the pooling layers. + + Returns + --------- + - the averaged re-scaling factor multiplied by `lambda_` if `len(rescaling_from_sumpool) > 0`, else `1` is returned. + + Note + --------- + - since the harmonic mean is less sensitive to outliers it **could be** that this is a better method + for weight re-scaling when multiple poolings with big differentces in kernel sizes are being considered. """ if len(rescaling_from_sumpool): return np.round(statistics.harmonic_mean(rescaling_from_sumpool)*lambda_, 2) else: - return 1 \ No newline at end of file + return 1.0 \ No newline at end of file From 1b338d73e3fca8526640632399d5ca2d5edcbc65 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Wed, 22 May 2024 15:30:01 +0200 Subject: [PATCH 098/379] Documentation/Refactor - Updated functions headers. - Refactored argument cascading between function (redundancies removed). --- sinabs/backend/dynapcnn/dynapcnn_layer.py | 25 +++++++++---- sinabs/backend/dynapcnn/utils.py | 45 ++++++++++++++++------- 2 files changed, 48 insertions(+), 22 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer.py b/sinabs/backend/dynapcnn/dynapcnn_layer.py index b5a499d1..4c92de94 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer.py @@ -1,5 +1,5 @@ from copy import deepcopy -from typing import Dict, Callable, Tuple, Union +from typing import Dict, Callable, Tuple, Union, List from warnings import warn import numpy as np @@ -13,14 +13,19 @@ class DynapcnnLayer(nn.Module): """ - Create a DynapcnnLayer object representing a dynapcnn layer. + Create a `DynapcnnLayer` object representing a layer on a Speck device. Parameters ---------- - - dpcnnl_index (int): ... - - dcnnl_data (dict): ... - - discretize (bool): ... - - sinabs_edges (list): ... + - dpcnnl_index (int): the index/ID that will be associated with a `DynapcnnLayer` instance. This integer indexes a `dict` within `dcnnl_data` + that comprises a set of layers (`nn.Module`) and their respective I/O tensor shapes. + - dcnnl_data (dict): contains the nodes to be merged into this `DynapcnnLayer`, their I/O shapes and the index of the other `DynapcnnLayer`s to + be set as destinations. The `int` keys correspond to the nodes IDs associated `nn.Module`s (a single layer in the original network) becoming + part of this `DynapcnnLayer` instance, while the `str` keys correspond to this instance's destinations and re-scaling factors. + - discretize (bool): whether or not the weights/neuron parameters of the model will be quantized. + - sinabs_edges (list): each `nn.Module` within `dcnnl_data` is a node in the original computational graph describing a spiking network. An edge + `(A, B)` describes how modules forward data amongst themselves. This list is used by a `DynapcnnLayer` to figure out the number and + sequence of output tesnors its forward method needs to return. - weight_rescaling_fn (callable): a method that handles how the re-scaling factor for one or more `SumPool2d` projecting to the same convolutional layer are combined/re-scaled before applying them. """ @@ -28,9 +33,9 @@ class DynapcnnLayer(nn.Module): def __init__( self, dpcnnl_index: int, - dcnnl_data: dict, + dcnnl_data: Dict[Union[int, str], Union[Dict[str, Union[nn.Module, Tuple[int, int, int], Tuple[int, int, int]]], List[int]]], discretize: bool, - sinabs_edges: list, + sinabs_edges: List[Tuple[int, int]], weight_rescaling_fn: Callable ): super().__init__() @@ -473,6 +478,10 @@ def _convert_linear_to_conv(self, lin: nn.Linear, layer_data: dict) -> nn.Conv2d def _get_destinations_input_source(self, sinabs_edges: list) -> dict: """ Creates a mapping between each layer in this `DynapcnnLayer` instance and its targe nodes that are part of different `DynapcnnLayer` instances. This mapping is used to figure out how many tensors the `forward` method needs to return. + + Returns + ---------- + - destinations_input_source (dict): maps a `nn.Module` within this `DynapcnnLayer` to the nodes it provides the input to. """ destinations_input_source = {} diff --git a/sinabs/backend/dynapcnn/utils.py b/sinabs/backend/dynapcnn/utils.py index c099c163..7e842305 100644 --- a/sinabs/backend/dynapcnn/utils.py +++ b/sinabs/backend/dynapcnn/utils.py @@ -174,7 +174,7 @@ def construct_dynapcnnlayers_from_mapper( for dpcnnl_idx, dcnnl_data in nodes_to_dcnnl_map.items(): # create a `DynapcnnLayer` from the set of layers in `dcnnl_data`. dynapcnnlayer = construct_dynapcnnlayer( - dpcnnl_idx, discretize, dcnnl_data, edges, nodes_to_dcnnl_map, weight_rescaling_fn) + dpcnnl_idx, discretize, edges, nodes_to_dcnnl_map, weight_rescaling_fn) dynapcnn_layers[dpcnnl_idx] = { 'layer': dynapcnnlayer, @@ -206,31 +206,38 @@ def update_nodes_io(updated_node: int, output_shape: tuple, nodes_to_dcnnl_map: def construct_dynapcnnlayer( dpcnnl_idx: int, discretize: bool, - dcnnl_data: dict, edges: List[Tuple[int, int]], - nodes_to_dcnnl_map: dict, + nodes_to_dcnnl_map: Dict[int, Dict[Union[int, str], Union[Dict[str, Union[nn.Module, Tuple[int, int, int]]], List[int]]]], weight_rescaling_fn: Callable) -> DynapcnnLayer: - """ Extract the modules (layers) in a dictionary and uses them to instantiate a DynapcnnLayer object. + """ Extract the modules (layers) in a dictionary and uses them to instantiate a `DynapcnnLayer` object. Parameters ---------- - - dcnnl_data (dict): contains the nodes to be merged into a DynapcnnLayer, their I/O shapes and the index of the other DynapcnnLayers to - be set as destinations. + - dpcnnl_idx (int): the index/ID that will be associated with a `DynapcnnLayer` instance. This integer indexes a `dict` within `nodes_to_dcnnl_map` + containing the data required to create the instance returned by this function. + - discretize (bool): whether or not the weights/neuron parameters of the model will be quantized. + - edges (list): each `nn.Module` within `nodes_to_dcnnl_map[dpcnnl_idx]` is a node in the original computational graph describing a spiking network + being converted to a `DynapcnnNetwork`. An edge `(A, B)` describes how modules forward data amongst themselves. This list is used by a `DynapcnnLayer` + to figure out the number and sequence of output tesnors its forward method needs to return. + - nodes_to_dcnnl_map (dict): contains all layers (`nn.Module`) in the original spiking network grouped into dictionaries gathering the data necessary + to instantiate a `DynapcnnLayer`. A `nodes_to_dcnnl_map[dpcnnl_idx]` will contain `int` keys (whose value corresponds to a `dict` with a `nn.Module` + instance and its associated I/O shapes, i.e., one layer within the `DynapcnnLayer` instance) or `str` keys (whose values correspond to a list of + integers corresponding to either destinations IDs or re-scaling factors). - weight_rescaling_fn (callable): a method that handles how the re-scaling factor for one or more `SumPool2d` projecting to - the same convolutional layer are combined/re-scaled before applying them. + the same convolutional layer are combined/re-scaled before being applied. Returns ---------- - - + - dynapcnnlayer (DynapcnnLayer): the a `DynapcnnLayer` instance made up by all the layers (`nn.Module`) in `dcnnl_data`. """ # convert all AvgPool2d in 'dcnnl_data' into SumPool2d. - convert_Avg_to_Sum_pooling(dcnnl_data, edges, nodes_to_dcnnl_map) + convert_Avg_to_Sum_pooling(nodes_to_dcnnl_map[dpcnnl_idx], edges, nodes_to_dcnnl_map) # instantiate a DynapcnnLayer from the data in 'dcnnl_data'. dynapcnnlayer = DynapcnnLayer( dpcnnl_index = dpcnnl_idx, - dcnnl_data = dcnnl_data, + dcnnl_data = nodes_to_dcnnl_map[dpcnnl_idx], discretize = discretize, sinabs_edges = edges, weight_rescaling_fn = weight_rescaling_fn @@ -238,15 +245,25 @@ def construct_dynapcnnlayer( return dynapcnnlayer -def convert_Avg_to_Sum_pooling(dcnnl_data: dict, edges: list, nodes_to_dcnnl_map: dict): +def convert_Avg_to_Sum_pooling( + dcnnl_data: Dict[Union[int, str], Union[Dict[str, Union[nn.Module, Tuple[int, int, int]]], List[int]]], + edges: List[Tuple[int, int]], + nodes_to_dcnnl_map: Dict[int, Dict[Union[int, str], Union[Dict[str, Union[nn.Module, Tuple[int, int, int]]], List[int]]]]) -> None: """ Converts every `AvgPool2d` node within `dcnnl_data` into a `SumPool2d` and update their respective `rescale_factor` (to be used when creating the `DynapcnnLayer` instance for this layer's destinations). Parameters ---------- - dcnnl_data: ... - edges: ... - nodes_to_dcnnl_map: ... + - dcnnl_data (dict): contains the nodes to be merged into a `DynapcnnLayer`, their I/O shapes and the index of the other `DynapcnnLayer`s to + be set as destinations. The `int` keys correspond to the nodes IDs associated `nn.Module`s (a single layer in the original network) becoming + part of a single `DynapcnnLayer` instance, while the `str` keys correspond to the instance's destinations and re-scaling factors. + - edges (list): each node is a `nn.Module` in the original computational graph describing a spiking network being converted to a `DynapcnnNetwork`. The + list is used to find the targets of a `SumPool2d` (part of the `DynapcnnLayer` instance being created) and update the re-scaling factor they will + require. + - nodes_to_dcnnl_map (dict): contains all layers (`nn.Module`) in the original spiking network grouped into dictionaries gathering the data necessary + to instantiate a `DynapcnnLayer`. A `nodes_to_dcnnl_map[dpcnnl_idx]` will contain `int` keys (whose value corresponds to a `dict` with a `nn.Module` + instance and its associated I/O shapes, i.e., one layer within the `DynapcnnLayer` instance) or `str` keys (whose values correspond to a list of + integers corresponding to either destinations IDs or re-scaling factors). """ for key, value in dcnnl_data.items(): if isinstance(key, int): From 4b1468a3e15600eb6390d90e9abddccb032c696a Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Wed, 22 May 2024 17:34:32 +0200 Subject: [PATCH 099/379] (WIP) Unit tests for DynapcnnLayer --- .../conftest_dynapcnnlayer.py | 167 ++++++++++++++++++ .../test_dynapcnnlayer/test_dynapcnnlayer.py | 36 ++++ 2 files changed, 203 insertions(+) create mode 100644 tests/test_dynapcnnlayer/conftest_dynapcnnlayer.py create mode 100644 tests/test_dynapcnnlayer/test_dynapcnnlayer.py diff --git a/tests/test_dynapcnnlayer/conftest_dynapcnnlayer.py b/tests/test_dynapcnnlayer/conftest_dynapcnnlayer.py new file mode 100644 index 00000000..575c0073 --- /dev/null +++ b/tests/test_dynapcnnlayer/conftest_dynapcnnlayer.py @@ -0,0 +1,167 @@ +import torch.nn as nn +from sinabs.layers import IAFSqueeze, SumPool2d +from sinabs.activation.surrogate_gradient_fn import PeriodicExponential + +nodes_to_dcnnl_map = { + 0: { + 0: { + 'layer': nn.Conv2d(2, 10, kernel_size=(2, 2), stride=(1, 1), bias=False), + 'input_shape': (2, 34, 34), + 'output_shape': (10, 33, 33) + }, + 1: { + 'layer': IAFSqueeze(batch_size=3, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), + 'input_shape': (10, 33, 33), + 'output_shape': (10, 33, 33) + }, + 2: { + 'layer': nn.AvgPool2d(kernel_size=3, stride=3, padding=0), + 'input_shape': (10, 33, 33), + 'output_shape': (10, 11, 11) + }, + 3: { + 'layer': nn.AvgPool2d(kernel_size=4, stride=4, padding=0), + 'input_shape': (10, 33, 33), + 'output_shape': (10, 8, 8) + }, + 'destinations': [1, 2], + 'conv_rescale_factor': [] + }, + 1: { + 4: { + 'layer': nn.Conv2d(10, 10, kernel_size=(4, 4), stride=(1, 1), bias=False), + 'input_shape': (10, 11, 11), + 'output_shape': (10, 8, 8) + }, + 6: { + 'layer': IAFSqueeze(batch_size=3, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), + 'input_shape': (10, 8, 8), + 'output_shape': (10, 8, 8) + }, + 'destinations': [2], + 'conv_rescale_factor': [9] + }, + 2: { + 7: { + 'layer': nn.Conv2d(10, 1, kernel_size=(2, 2), stride=(1, 1), bias=False), + 'input_shape': (10, 8, 8), + 'output_shape': (1, 7, 7) + }, + 8: { + 'layer': IAFSqueeze(batch_size=3, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), + 'input_shape': (1, 7, 7), + 'output_shape': (1, 7, 7) + }, + 'destinations': [3], + 'conv_rescale_factor': [16] + }, + 3: { + 9: { + 'layer': nn.Linear(in_features=49, out_features=500, bias=False), + 'input_shape': (1, 7, 7), + 'output_shape': (500,) + }, + 10: { + 'layer': IAFSqueeze(batch_size=3, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), + 'input_shape': (500,), + 'output_shape': (500,) + }, + 'destinations': [4], + 'conv_rescale_factor': [] + }, + 4: { + 11: { + 'layer': nn.Linear(in_features=500, out_features=10, bias=False), + 'input_shape': (500,), + 'output_shape': (10,) + }, + 12: { + 'layer': IAFSqueeze(batch_size=3, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), + 'input_shape': (10,), + 'output_shape': (10,) + }, + 'destinations': [], + 'conv_rescale_factor': [] + } +} + +sinabs_edges = [ + (0, 1), + (1, 2), + (1, 3), + (2, 4), + (3, 7), + (4, 6), + (6, 7), + (7, 8), + (8, 9), + (9, 10), + (10, 11), + (11, 12), +] + +expected_output = { + 0: { + 'dpcnnl_index': 0, + 'conv_node_id': 0, + 'conv_in_shape': (2, 34, 34), + 'conv_out_shape': (10, 33, 33), + 'spk_node_id': 1, + 'pool_node_id': [2, 3], + 'conv_rescaling_factor': None, + 'dynapcnnlayer_destination': [1, 2], + 'nodes_destination': {2: [4], 3: [7]}, + }, + 1: { + 'dpcnnl_index': 1, + 'conv_node_id': 4, + 'conv_in_shape': (10, 11, 11), + 'conv_out_shape': (10, 8, 8), + 'spk_node_id': 6, + 'pool_node_id': [], + 'conv_rescaling_factor': 4.5, + 'dynapcnnlayer_destination': [2], + 'nodes_destination': {6: [7]}, + }, + 2: { + 'dpcnnl_index': 2, + 'conv_node_id': 7, + 'conv_in_shape': (10, 8, 8), + 'conv_out_shape': (1, 7, 7), + 'spk_node_id': 8, + 'pool_node_id': [], + 'conv_rescaling_factor': 8.0, + 'dynapcnnlayer_destination': [3], + 'nodes_destination': {8: [9]}, + }, + 3: { + 'dpcnnl_index': 3, + 'conv_node_id': 9, + 'conv_in_shape': (1, 7, 7), + 'conv_out_shape': (500, 1, 1), + 'spk_node_id': 10, + 'pool_node_id': [], + 'conv_rescaling_factor': None, + 'dynapcnnlayer_destination': [4], + 'nodes_destination': {10: [11]}, + }, + 4: { + 'dpcnnl_index': 4, + 'conv_node_id': 11, + 'conv_in_shape': (500, 1, 1), + 'conv_out_shape': (10, 1, 1), + 'spk_node_id': 12, + 'pool_node_id': [], + 'conv_rescaling_factor': None, + 'dynapcnnlayer_destination': [], + 'nodes_destination': {}, + }, +} + +args_DynapcnnLayer = [ + (nodes_to_dcnnl_map, 0, sinabs_edges, expected_output), + (nodes_to_dcnnl_map, 1, sinabs_edges, expected_output), + (nodes_to_dcnnl_map, 2, sinabs_edges, expected_output), + (nodes_to_dcnnl_map, 3, sinabs_edges, expected_output), + (nodes_to_dcnnl_map, 4, sinabs_edges, expected_output), +] \ No newline at end of file diff --git a/tests/test_dynapcnnlayer/test_dynapcnnlayer.py b/tests/test_dynapcnnlayer/test_dynapcnnlayer.py new file mode 100644 index 00000000..a440dbe2 --- /dev/null +++ b/tests/test_dynapcnnlayer/test_dynapcnnlayer.py @@ -0,0 +1,36 @@ +import pytest +from sinabs.backend.dynapcnn import DynapcnnLayer +from conftest_dynapcnnlayer import args_DynapcnnLayer +import sys + +sys.path.append('../../sinabs/backend/dynapcnn') + +from weight_rescaling_methods import rescale_method_1 +from utils import convert_Avg_to_Sum_pooling + +@pytest.mark.parametrize("nodes_to_dcnnl_map, dpcnnl_idx, sinabs_edges, expected_output", args_DynapcnnLayer) +def test_DynapcnnLayer(nodes_to_dcnnl_map, dpcnnl_idx, sinabs_edges, expected_output): + + convert_Avg_to_Sum_pooling(nodes_to_dcnnl_map[dpcnnl_idx], sinabs_edges, nodes_to_dcnnl_map) + + dynapcnnlayer = DynapcnnLayer( + dpcnnl_index = dpcnnl_idx, + dcnnl_data = nodes_to_dcnnl_map[dpcnnl_idx], + discretize = True, + sinabs_edges = sinabs_edges, + weight_rescaling_fn = rescale_method_1 + ) + + config = { + 'dpcnnl_index': dynapcnnlayer.dpcnnl_index, + 'conv_node_id': dynapcnnlayer.conv_node_id, + 'conv_in_shape': dynapcnnlayer.conv_in_shape, + 'conv_out_shape': dynapcnnlayer.conv_out_shape, + 'spk_node_id': dynapcnnlayer.spk_node_id, + 'pool_node_id': dynapcnnlayer.pool_node_id, + 'conv_rescaling_factor': dynapcnnlayer.conv_rescaling_factor, + 'dynapcnnlayer_destination': dynapcnnlayer.dynapcnnlayer_destination, + 'nodes_destinations': dynapcnnlayer.nodes_destinations, + } + + assert config == expected_output[dpcnnl_idx] \ No newline at end of file From 76c6c5fba8e84d1a0bc8325783504db2cd1a7582 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Mon, 27 May 2024 15:37:18 +0200 Subject: [PATCH 100/379] Unit testing + Documentation - Unit testing the instantiation of DynapcnnLayers. - Updated methods headers of DynapcnnLayer class. - Updated in-line documentation/headers of some utils.py functions. --- sinabs/backend/dynapcnn/dynapcnn_layer.py | 45 +- .../dynapcnn/dynapcnn_layer_deprecated.py | 204 ------- .../dynapcnn/dynapcnn_network_deprecated.py | 508 ------------------ sinabs/backend/dynapcnn/utils.py | 25 +- .../conftest_dynapcnnlayer.py | 12 +- .../test_dynapcnnlayer/test_dynapcnnlayer.py | 69 ++- 6 files changed, 99 insertions(+), 764 deletions(-) delete mode 100644 sinabs/backend/dynapcnn/dynapcnn_layer_deprecated.py delete mode 100644 sinabs/backend/dynapcnn/dynapcnn_network_deprecated.py diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer.py b/sinabs/backend/dynapcnn/dynapcnn_layer.py index 4c92de94..2e3c1ed5 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer.py @@ -93,6 +93,9 @@ def __init__( spk.v_mem = spk.v_mem.data.unsqueeze(-1).unsqueeze(-1) # expand dims. if isinstance(conv, nn.Linear): + # A `nn.Linear` needs to be converted into `nn.Conv2d`. The I/O shapes of the spiking layer are updated + # accordingly following the conversion. + conv, conv_in_shape = self._convert_linear_to_conv(conv, dcnnl_data[self.conv_node_id]) # the original `nn.Linear` output shape becomes the equivalent `nn.Conv2d` shape. @@ -100,7 +103,7 @@ def __init__( conv_layer=conv, layer_data=dcnnl_data[self.conv_node_id], input_shape=conv_in_shape) # the I/O shapes for neuron layer following the new conv need also to be updated. - self._update_neuron_node_output_shape(layer_data=dcnnl_data[self.spk_node_id], input_shape=self.conv_out_shape) + self._update_neuron_node_output_shape(spiking_layer_data=dcnnl_data[self.spk_node_id], conv_out_shape=self.conv_out_shape) else: self.conv_out_shape = dcnnl_data[self.conv_node_id]['output_shape'] @@ -222,8 +225,19 @@ def forward(self, x): return tuple(returns) - def get_modified_node_it(self, dcnnl_data: dict) -> Union[Tuple[int, tuple], Tuple[None, None]]: - """ .""" + def get_modified_node_io(self, dcnnl_data: dict) -> Union[Tuple[int, tuple], Tuple[None, None]]: + """ Follwing a conversion, the I/O shapes of the spiking layer have been updated to match the convolution's + output. Thus, all nodes receiving input from this spiking layer need their input shapes updated. + + Parameters + ---------- + - dcnnl_data (dict): the set of layers grouped together to comprise this instance of a `DynapcnnLayer`. + + Returns + ---------- + - node ID (int): the ID of the spiking layer consuming the tunerd layer's output (`None` if there was no conversion). + - output shape (tuple): the new output shape following a converstion from `nn.Linear` to `nn.Conv2d` (`None` if there was no conversion). + """ if self.lin_to_conv_conversion: return self.spk_node_id, dcnnl_data[self.spk_node_id]['output_shape'] return None, None @@ -417,12 +431,18 @@ def __str__(self): ####################################################### Private Methods ####################################################### - def _update_neuron_node_output_shape(self, layer_data: dict, input_shape: tuple) -> None: - """ Following the conversion of a `nn.Linear` into a `nn.Conv2d` the neuron layer in the - sequence also needs its I/O shapes uodated. + def _update_neuron_node_output_shape(self, spiking_layer_data: dict, conv_out_shape: tuple) -> None: + """ Updates the spiking layer's I/O shapes after the conversion of a `nn.Linear` into a `nn.Conv2d` (to match the convolution's output). + + Parameters + ---------- + - spiking_layer_data (dict): the dictionary containing all data regarding the spiking layer. + - conv_out_shape (tuple): the output shape of the convolution layer preceeding the spiking layer. """ - layer_data['input_shape'] = input_shape - layer_data['output_shape'] = layer_data['input_shape'] + # spiking layer consumes the tensor coming out of the conv. layer. + spiking_layer_data['input_shape'] = conv_out_shape + # spiking layer outputs the same shape as the conv. layer. + spiking_layer_data['output_shape'] = spiking_layer_data['input_shape'] def _update_conv_node_output_shape(self, conv_layer: nn.Conv2d, layer_data: dict, input_shape: tuple) -> Tuple: """ The input shapes to nodes are extracted using a list of edges by finding the output shape of the 1st element @@ -435,18 +455,17 @@ def _update_conv_node_output_shape(self, conv_layer: nn.Conv2d, layer_data: dict return layer_data['output_shape'] def _convert_linear_to_conv(self, lin: nn.Linear, layer_data: dict) -> nn.Conv2d: - """Convert Linear layer to Conv2d. + """ Convert Linear layer to Conv2d. Parameters ---------- - lin: nn.Linear - Linear layer to be converted + - lin (nn.Linear): linear layer to be converted. Returns ------- - nn.Conv2d - Convolutional layer equivalent to `lin`. + - nn.Conv2d: convolutional layer equivalent to `lin`. """ + # this flags the necessity to update the I/O shape pre-computed for each of the original layers being compressed within a `DynapcnnLayer` instance. self.lin_to_conv_conversion = True input_shape = layer_data['input_shape'] diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer_deprecated.py b/sinabs/backend/dynapcnn/dynapcnn_layer_deprecated.py deleted file mode 100644 index a56454c8..00000000 --- a/sinabs/backend/dynapcnn/dynapcnn_layer_deprecated.py +++ /dev/null @@ -1,204 +0,0 @@ -from copy import deepcopy -from typing import Dict, Optional, Tuple, Union -from warnings import warn - -import numpy as np -import torch -from torch import nn - -import sinabs.activation -import sinabs.layers as sl - -from .discretize import discretize_conv_spike_ -from .dvs_layer import expand_to_pair - - -class DynapcnnLayer(nn.Module): - """Create a DynapcnnLayer object representing a dynapcnn layer. - - Requires a convolutional layer, a sinabs spiking layer and an optional - pooling value. The layers are used in the order conv -> spike -> pool. - - Parameters - ---------- - conv: torch.nn.Conv2d or torch.nn.Linear - Convolutional or linear layer (linear will be converted to convolutional) - spk: sinabs.layers.IAFSqueeze - Sinabs IAF layer - in_shape: tuple of int - The input shape, needed to create dynapcnn configs if the network does not - contain an input layer. Convention: (features, height, width) - pool: int or None - Integer representing the sum pooling kernel and stride. If `None`, no - pooling will be applied. - discretize: bool - Whether to discretize parameters. - rescale_weights: int - Layer weights will be divided by this value. - """ - - def __init__( - self, - conv: nn.Conv2d, - spk: sl.IAFSqueeze, - in_shape: Tuple[int, int, int], - pool: Optional[sl.SumPool2d] = None, - discretize: bool = True, - rescale_weights: int = 1, - ): - super().__init__() - - self.input_shape = in_shape - - spk = deepcopy(spk) - if isinstance(conv, nn.Linear): - conv = self._convert_linear_to_conv(conv) - if spk.is_state_initialised(): - # Expand dims - spk.v_mem = spk.v_mem.data.unsqueeze(-1).unsqueeze(-1) - else: - conv = deepcopy(conv) - - if rescale_weights != 1: - # this has to be done after copying but before discretizing - conv.weight.data = (conv.weight / rescale_weights).clone().detach() - - self.discretize = discretize - if discretize: - # int conversion is done while writing the config. - conv, spk = discretize_conv_spike_(conv, spk, to_int=False) - - self.conv_layer = conv - self.spk_layer = spk - if pool is not None: - if pool.kernel_size[0] != pool.kernel_size[1]: - raise ValueError("Only square kernels are supported") - self.pool_layer = deepcopy(pool) - else: - self.pool_layer = None - - def _convert_linear_to_conv(self, lin: nn.Linear) -> nn.Conv2d: - """Convert Linear layer to Conv2d. - - Parameters - ---------- - lin: nn.Linear - Linear layer to be converted - - Returns - ------- - nn.Conv2d - Convolutional layer equivalent to `lin`. - """ - - in_chan, in_h, in_w = self.input_shape - - if lin.in_features != in_chan * in_h * in_w: - raise ValueError("Shapes don't match.") - - layer = nn.Conv2d( - in_channels=in_chan, - kernel_size=(in_h, in_w), - out_channels=lin.out_features, - padding=0, - bias=lin.bias is not None, - ) - - if lin.bias is not None: - layer.bias.data = lin.bias.data.clone().detach() - - layer.weight.data = ( - lin.weight.data.clone() - .detach() - .reshape((lin.out_features, in_chan, in_h, in_w)) - ) - - return layer - - def get_neuron_shape(self) -> Tuple[int, int, int]: - """Return the output shape of the neuron layer. - - Returns - ------- - features, height, width - """ - - def get_shape_after_conv(layer: nn.Conv2d, input_shape): - (ch_in, h_in, w_in) = input_shape - (kh, kw) = expand_to_pair(layer.kernel_size) - (pad_h, pad_w) = expand_to_pair(layer.padding) - (stride_h, stride_w) = expand_to_pair(layer.stride) - - def out_len(in_len, k, s, p): - return (in_len - k + 2 * p) // s + 1 - - out_h = out_len(h_in, kh, stride_h, pad_h) - out_w = out_len(w_in, kw, stride_w, pad_w) - ch_out = layer.out_channels - return ch_out, out_h, out_w - - conv_out_shape = get_shape_after_conv( - self.conv_layer, input_shape=self.input_shape - ) - return conv_out_shape - - def get_output_shape(self) -> Tuple[int, int, int]: - neuron_shape = self.get_neuron_shape() - # this is the actual output shape, including pooling - if self.pool_layer is not None: - pool = expand_to_pair(self.pool_layer.kernel_size) - return ( - neuron_shape[0], - neuron_shape[1] // pool[0], - neuron_shape[2] // pool[1], - ) - else: - return neuron_shape - - def summary(self) -> dict: - return { - "pool": ( - None if self.pool_layer is None else list(self.pool_layer.kernel_size) - ), - "kernel": list(self.conv_layer.weight.data.shape), - "neuron": self.get_neuron_shape(), - } - - def memory_summary(self): - """Computes the amount of memory required for each of the components. Note that this is not - necessarily the same as the number of parameters due to some architecture design - constraints. - - .. math:: - - K_{MT} = c \\cdot 2^{\\lceil \\log_2\\left(k_xk_y\\right) \\rceil + \\lceil \\log_2\\left(f\\right) \\rceil} - - .. math:: - - N_{MT} = f \\cdot 2^{ \\lceil \\log_2\\left(f_y\\right) \\rceil + \\lceil \\log_2\\left(f_x\\right) \\rceil } - - Returns - ------- - A dictionary with keys kernel, neuron and bias and the corresponding memory sizes - """ - summary = self.summary() - f, c, h, w = summary["kernel"] - f, neuron_height, neuron_width = self.get_neuron_shape() - - return { - "kernel": c * pow(2, np.ceil(np.log2(h * w)) + np.ceil(np.log2(f))), - "neuron": f - * pow(2, np.ceil(np.log2(neuron_height)) + np.ceil(np.log2(neuron_width))), - "bias": 0 if self.conv_layer.bias is None else len(self.conv_layer.bias), - } - - def forward(self, x): - """Torch forward pass.""" - x = self.conv_layer(x) - x = self.spk_layer(x) - if self.pool_layer is not None: - x = self.pool_layer(x) - return x - - def zero_grad(self, set_to_none: bool = False) -> None: - return self.spk_layer.zero_grad(set_to_none) diff --git a/sinabs/backend/dynapcnn/dynapcnn_network_deprecated.py b/sinabs/backend/dynapcnn/dynapcnn_network_deprecated.py deleted file mode 100644 index 9d59236f..00000000 --- a/sinabs/backend/dynapcnn/dynapcnn_network_deprecated.py +++ /dev/null @@ -1,508 +0,0 @@ -import time -from subprocess import CalledProcessError -from typing import List, Optional, Sequence, Tuple, Union - -import samna -import torch -import torch.nn as nn - -import sinabs - -from .chip_factory import ChipFactory -from .dvs_layer import DVSLayer -from .dynapcnn_layer import DynapcnnLayer -from .io import disable_timestamps, enable_timestamps, open_device, reset_timestamps -from .utils import ( - DEFAULT_IGNORED_LAYER_TYPES, - build_from_list, - convert_model_to_layer_list, - infer_input_shape, - parse_device_id, -) - - -class DynapcnnNetwork(nn.Module): - """Given a sinabs spiking network, prepare a dynapcnn-compatible network. This can be used to - test the network will be equivalent once on DYNAPCNN. This class also provides utilities to - make the dynapcnn configuration and upload it to DYNAPCNN. - - The following operations are done when converting to dynapcnn-compatible: - - * multiple avg pooling layers in a row are consolidated into one and \ - turned into sum pooling layers; - * checks are performed on layer hyperparameter compatibility with dynapcnn \ - (kernel sizes, strides, padding) - * checks are performed on network structure compatibility with dynapcnn \ - (certain layers can only be followed by other layers) - * linear layers are turned into convolutional layers - * dropout layers are ignored - * weights, biases and thresholds are discretized according to dynapcnn requirements - - Note that the model parameters are only ever transferred to the device - on the `to` call, so changing a threshold or weight of a model that - is deployed will have no effect on the model on chip until `to` is called again. - """ - - def __init__( - self, - snn: Union[nn.Sequential, sinabs.Network], - input_shape: Optional[Tuple[int, int, int]] = None, - dvs_input: bool = False, - discretize: bool = True, - ): - """ - DynapcnnNetwork: a class turning sinabs networks into dynapcnn - compatible networks, and making dynapcnn configurations. - - Parameters - ---------- - snn: sinabs.Network - SNN that determines the structure of the `DynapcnnNetwork` - input_shape: None or tuple of ints - Shape of the input, convention: (features, height, width) - If None, `snn` needs an InputLayer - dvs_input: bool - Does dynapcnn receive input from its DVS camera? - discretize: bool - If True, discretize the parameters and thresholds. - This is needed for uploading weights to dynapcnn. Set to False only for - testing purposes. - """ - super().__init__() - - # This attribute stores the location/core-id of each of the DynapcnnLayers upon placement on chip - self.chip_layers_ordering = [] - - self.input_shape = input_shape # Convert models to sequential - layers = convert_model_to_layer_list( - model=snn, ignore=DEFAULT_IGNORED_LAYER_TYPES - ) - # Check if dvs input is expected - if dvs_input: - self.dvs_input = True - else: - self.dvs_input = False - - input_shape = infer_input_shape(layers, input_shape=input_shape) - assert len(input_shape) == 3, "infer_input_shape did not return 3-tuple" - - # Build model from layers - self.sequence = build_from_list( - layers, - in_shape=input_shape, - discretize=discretize, - dvs_input=self.dvs_input, - ) - - def to( - self, - device="cpu", - chip_layers_ordering="auto", - monitor_layers: Optional[Union[List, str]] = None, - config_modifier=None, - slow_clk_frequency: int = None, - ): - """Note that the model parameters are only ever transferred to the device on the `to` call, - so changing a threshold or weight of a model that is deployed will have no effect on the - model on chip until `to` is called again. - - Parameters - ---------- - - device: String - cpu:0, cuda:0, dynapcnndevkit, speck2devkit - - chip_layers_ordering: sequence of integers or `auto` - The order in which the dynapcnn layers will be used. If `auto`, - an automated procedure will be used to find a valid ordering. - A list of layers on the device where you want each of the model's DynapcnnLayers to be placed. - The index of the core on chip to which the i-th layer in the model is mapped is the value of the i-th entry in the list. - Note: This list should be the same length as the number of dynapcnn layers in your model. - - monitor_layers: None/List - A list of all layers in the module that you want to monitor. Indexing starts with the first non-dvs layer. - If you want to monitor the dvs layer for eg. - :: - - monitor_layers = ["dvs"] # If you want to monitor the output of the pre-processing layer - monitor_layers = ["dvs", 8] # If you want to monitor preprocessing and layer 8 - monitor_layers = "all" # If you want to monitor all the layers - - config_modifier: - A user configuration modifier method. - This function can be used to make any custom changes you want to make to the configuration object. - - Note - ---- - chip_layers_ordering and monitor_layers are used only when using synsense devices. - For GPU or CPU usage these options are ignored. - """ - self.device = device - if isinstance(device, torch.device): - return super().to(device) - elif isinstance(device, str): - device_name, _ = parse_device_id(device) - if device_name in ChipFactory.supported_devices: # pragma: no cover - # Generate config - config = self.make_config( - chip_layers_ordering=chip_layers_ordering, - device=device, - monitor_layers=monitor_layers, - config_modifier=config_modifier, - ) - - # Apply configuration to device - self.samna_device = open_device(device) - self.samna_device.get_model().apply_configuration(config) - time.sleep(1) - - # Set external slow-clock if need - if slow_clk_frequency is not None: - dk_io = self.samna_device.get_io_module() - dk_io.set_slow_clk(True) - dk_io.set_slow_clk_rate(slow_clk_frequency) # Hz - - builder = ChipFactory(device).get_config_builder() - # Create input source node - self.samna_input_buffer = builder.get_input_buffer() - # Create output sink node node - self.samna_output_buffer = builder.get_output_buffer() - - # Connect source node to device sink - self.device_input_graph = samna.graph.EventFilterGraph() - self.device_input_graph.sequential( - [ - self.samna_input_buffer, - self.samna_device.get_model().get_sink_node(), - ] - ) - - # Connect sink node to device - self.device_output_graph = samna.graph.EventFilterGraph() - self.device_output_graph.sequential( - [ - self.samna_device.get_model().get_source_node(), - self.samna_output_buffer, - ] - ) - self.device_input_graph.start() - self.device_output_graph.start() - self.samna_config = config - return self - else: - return super().to(device) - else: - raise Exception("Unknown device description.") - - def _make_config( - self, - chip_layers_ordering: Union[Sequence[int], str] = "auto", - device="dynapcnndevkit:0", - monitor_layers: Optional[Union[List, str]] = None, - config_modifier=None, - ) -> Tuple["SamnaConfiguration", bool]: - """Prepare and output the `samna` configuration for this network. - - Parameters - ---------- - - chip_layers_ordering: sequence of integers or `auto` - The order in which the dynapcnn layers will be used. If `auto`, - an automated procedure will be used to find a valid ordering. - A list of layers on the device where you want each of the model's DynapcnnLayers to be placed. - The index of the core on chip to which the i-th layer in the model is mapped is the value of the i-th entry in the list. - Note: This list should be the same length as the number of dynapcnn layers in your model. - - device: String - dynapcnndevkit, speck2b or speck2devkit - - monitor_layers: None/List/Str - A list of all layers in the module that you want to monitor. Indexing starts with the first non-dvs layer. - If you want to monitor the dvs layer for eg. - :: - - monitor_layers = ["dvs"] # If you want to monitor the output of the pre-processing layer - monitor_layers = ["dvs", 8] # If you want to monitor preprocessing and layer 8 - monitor_layers = "all" # If you want to monitor all the layers - - If this value is left as None, by default the last layer of the model is monitored. - - config_modifier: - A user configuration modifier method. - This function can be used to make any custom changes you want to make to the configuration object. - - Returns - ------- - Configuration object - Object defining the configuration for the device - Bool - True if the configuration is valid for the given device. - - Raises - ------ - ImportError - If samna is not available. - """ - config_builder = ChipFactory(device).get_config_builder() - - has_dvs_layer = isinstance(self.sequence[0], DVSLayer) - - # Figure out layer ordering - if chip_layers_ordering == "auto": - chip_layers_ordering = config_builder.get_valid_mapping(self) - else: - # Truncate chip_layers_ordering just in case a longer list is passed - if has_dvs_layer: - chip_layers_ordering = chip_layers_ordering[: len(self.sequence) - 1] - chip_layers_ordering = chip_layers_ordering[: len(self.sequence)] - - # Save the chip layers - self.chip_layers_ordering = chip_layers_ordering - # Update config - config = config_builder.build_config(self, chip_layers_ordering) - if self.input_shape and self.input_shape[0] == 1: - config.dvs_layer.merge = True - # Check if any monitoring is enabled and if not, enable monitoring for the last layer - if monitor_layers is None: - monitor_layers = [-1] - elif monitor_layers == "all": - num_cnn_layers = len(self.sequence) - int(has_dvs_layer) - monitor_layers = list(range(num_cnn_layers)) - - # Enable monitors on the specified layers - # Find layers corresponding to the chip - monitor_chip_layers = [ - self.find_chip_layer(lyr) for lyr in monitor_layers if lyr != "dvs" - ] - if "dvs" in monitor_layers: - monitor_chip_layers.append("dvs") - - config_builder.monitor_layers(config, monitor_chip_layers) - - # Fix default factory setting to not return input events (UGLY!! Ideally this should happen in samna) - # config.factory_settings.monitor_input_enable = False - - # Apply user config modifier - if config_modifier is not None: - config = config_modifier(config) - - # Validate config - return config, config_builder.validate_configuration(config) - - def make_config( - self, - chip_layers_ordering: Union[Sequence[int], str] = "auto", - device="dynapcnndevkit:0", - monitor_layers: Optional[Union[List, str]] = None, - config_modifier=None, - ): - """Prepare and output the `samna` DYNAPCNN configuration for this network. - - Parameters - ---------- - - chip_layers_ordering: sequence of integers or `auto` - The order in which the dynapcnn layers will be used. If `auto`, - an automated procedure will be used to find a valid ordering. - A list of layers on the device where you want each of the model's DynapcnnLayers to be placed. - Note: This list should be the same length as the number of dynapcnn layers in your model. - - device: String - dynapcnndevkit, speck2b or speck2devkit - - monitor_layers: None/List/Str - A list of all layers in the module that you want to monitor. Indexing starts with the first non-dvs layer. - If you want to monitor the dvs layer for eg. - :: - - monitor_layers = ["dvs"] # If you want to monitor the output of the pre-processing layer - monitor_layers = ["dvs", 8] # If you want to monitor preprocessing and layer 8 - monitor_layers = "all" # If you want to monitor all the layers - - If this value is left as None, by default the last layer of the model is monitored. - - config_modifier: - A user configuration modifier method. - This function can be used to make any custom changes you want to make to the configuration object. - - Returns - ------- - Configuration object - Object defining the configuration for the device - - Raises - ------ - ImportError - If samna is not available. - ValueError - If the generated configuration is not valid for the specified device. - """ - config, is_compatible = self._make_config( - chip_layers_ordering=chip_layers_ordering, - device=device, - monitor_layers=monitor_layers, - config_modifier=config_modifier, - ) - # Validate config - if is_compatible: - print("Network is valid") - return config - else: - raise ValueError(f"Generated config is not valid for {device}") - - def is_compatible_with(self, device_type: str) -> bool: - """Check if the current model is compatible with a given device. - - Args: - device_type (str): Device type ie speck2b, speck2fmodule - - Returns: - bool: True if compatible - """ - try: - _, is_compatible = self._make_config(device=device_type) - except ValueError as e: - # Catch "No valid mapping found" error - if e.args[0] == ("No valid mapping found"): - return False - else: - raise e - return is_compatible - - def reset_states(self, randomize=False): - """Reset the states of the network.""" - if hasattr(self, "device") and isinstance(self.device, str): # pragma: no cover - device_name, _ = parse_device_id(self.device) - if device_name in ChipFactory.supported_devices: - config_builder = ChipFactory(self.device).get_config_builder() - # Set all the vmem states in the samna config to zero - config_builder.reset_states(self.samna_config, randomize=randomize) - self.samna_device.get_model().apply_configuration(self.samna_config) - # wait for the config to be written - time.sleep(1) - # Note: The below shouldn't be necessary ideally - # Erase all vmem memory - if not randomize: - if hasattr(self, "samna_input_graph"): - self.samna_input_graph.stop() - for lyr_idx in self.chip_layers_ordering: - config_builder.set_all_v_mem_to_zeros( - self.samna_device, lyr_idx - ) - time.sleep(0.1) - self.samna_input_graph.start() - return - for layer in self.sequence: - if isinstance(layer, DynapcnnLayer): - layer.spk_layer.reset_states(randomize=randomize) - - def find_chip_layer(self, layer_idx): - """Given an index of a layer in the model, find the corresponding cnn core id where it is - placed. - - > Note that the layer index does not include the DVSLayer. - > For instance your model comprises two layers [DVSLayer, DynapcnnLayer], - > then the index of DynapcnnLayer is 0 and not 1. - - Parameters - ---------- - layer_idx: int - Index of a layer - - Returns - ------- - chip_lyr_idx: int - Index of the layer on the chip where the model layer is placed. - """ - # Compute the expected number of cores - num_cores_required = len(self.sequence) - if isinstance(self.sequence[0], DVSLayer): - num_cores_required -= 1 - if len(self.chip_layers_ordering) != num_cores_required: - raise Exception( - f"Number of layers specified in chip_layers_ordering {self.chip_layers_ordering} does not correspond to the number of cores required for this model {num_cores_required}" - ) - - return self.chip_layers_ordering[layer_idx] - - def forward(self, x): - if ( - hasattr(self, "device") - and parse_device_id(self.device)[0] in ChipFactory.supported_devices - ): # pragma: no cover - _ = self.samna_output_buffer.get_events() # Flush buffer - # NOTE: The code to start and stop time stamping is device specific - reset_timestamps(self.device) - enable_timestamps(self.device) - # Send input - self.samna_input_buffer.write(x) - received_evts = [] - # Record at least until the last event has been replayed - min_duration = max(event.timestamp for event in x) * 1e-6 - time.sleep(min_duration) - # Keep recording if more events are being registered - while True: - prev_length = len(received_evts) - time.sleep(0.1) - received_evts.extend(self.samna_output_buffer.get_events()) - if prev_length == len(received_evts): - break - # Disable timestamp - disable_timestamps(self.device) - return received_evts - else: - """Torch's forward pass.""" - return self.sequence(x) - - def memory_summary(self): - """Get a summary of the network's memory requirements. - - Returns - ------- - dict: - A dictionary with keys kernel, neuron, bias. - The values are a list of the corresponding number per layer in the same order as the model - """ - summary = {} - - dynapcnn_layers = [ - lyr for lyr in self.sequence if isinstance(lyr, DynapcnnLayer) - ] - summary.update({k: list() for k in dynapcnn_layers[0].memory_summary().keys()}) - for lyr in dynapcnn_layers: - lyr_summary = lyr.memory_summary() - for k, v in lyr_summary.items(): - summary[k].append(v) - return summary - - def zero_grad(self, set_to_none: bool = False) -> None: - for lyr in self.sequence: - lyr.zero_grad(set_to_none) - - def __del__(self): - # Stop the input graph - if hasattr(self, "device_input_graph") and self.device_input_graph: - self.device_input_graph.stop() - - # Stop the output graph. - if hasattr(self, "device_output_graph") and self.device_output_graph: - self.device_output_graph.stop() - - -class DynapcnnCompatibleNetwork(DynapcnnNetwork): - """Deprecated class, use DynapcnnNetwork instead.""" - - def __init__( - self, - snn: Union[nn.Sequential, sinabs.Network], - input_shape: Optional[Tuple[int, int, int]] = None, - dvs_input: bool = False, - discretize: bool = True, - ): - from warnings import warn - - warn( - "DynapcnnCompatibleNetwork has been renamed to DynapcnnNetwork " - + "and will be removed in a future release." - ) - super().__init__(snn, input_shape, dvs_input, discretize) diff --git a/sinabs/backend/dynapcnn/utils.py b/sinabs/backend/dynapcnn/utils.py index 7e842305..3d04b143 100644 --- a/sinabs/backend/dynapcnn/utils.py +++ b/sinabs/backend/dynapcnn/utils.py @@ -181,24 +181,37 @@ def construct_dynapcnnlayers_from_mapper( 'destinations': nodes_to_dcnnl_map[dpcnnl_idx]['destinations'] } - # check if any node (layer) in `dynapcnnlayer` has been modified (e.g. `nn.Linear` turned `nn.Conv2d`). - node, output_shape = dynapcnnlayer.get_modified_node_it(dcnnl_data) + # check if a `nn.Linear` in `dynapcnnlayer` has been turned into a `nn.Conv2d`. + node, output_shape = dynapcnnlayer.get_modified_node_io(dcnnl_data) - # one of the layers in `dynapcnnlayer` had its type modified (update input shape of nodes receiving from it). if isinstance(node, int) and isinstance(output_shape, tuple): + # a `nn.Linear` has been converted into a `nn.Conv2d`: update input shape of nodes receiving from the spiking layer after it. update_nodes_io(node, output_shape, nodes_to_dcnnl_map, edges) return dynapcnn_layers def update_nodes_io(updated_node: int, output_shape: tuple, nodes_to_dcnnl_map: dict, edges: List[Tuple[int, int]]) -> None: - """ .""" + """ Updates the `input_shape` entries of each node in `nodes_to_dcnnl_map` receiving as input the output of the spiking + layer `updated_node` that had its I/O shapes updated following a `nn.Linear` to `nn.Conv2d` conversion. + + Parameters + ---------- + - updated_node (int): the ID of the spiking layer that had its I/O shapes updated following a `nn.Linear` to `nn.Conv2d` conversion. + - output_shape (tuple): the updated shape of the spiking layer with node ID `updated_node`. + - nodes_to_dcnnl_map (dict): each entry represents the gathered data necessary to instantiate a `DynapcnnLayer` object (e.g. nodes, + their I/O shapes, the list of `DynapcnnLayer` that are to be targeted, etc). + - edges (list): edges describing how nodes connect to each other. + """ + for edge in edges: if edge[0] == updated_node: # found source node where output shape has been modified. - for dcnnl_idx, dcnnl_data in nodes_to_dcnnl_map.items(): + + # accessing every single node ID within the set of layers composing each `DynapcnnLayer` instance. + for _, dcnnl_data in nodes_to_dcnnl_map.items(): for key, val in dcnnl_data.items(): if isinstance(key, int): - # accessing node data (layer, input_shape, output_shape). + # accessing node data (`layer`, `input_shape` and `output_shape`). if key == edge[1]: # accessing node targeted by `updated_node` (its input shape becomes `updated_node.output_shape`). val['input_shape'] = output_shape diff --git a/tests/test_dynapcnnlayer/conftest_dynapcnnlayer.py b/tests/test_dynapcnnlayer/conftest_dynapcnnlayer.py index 575c0073..b467c268 100644 --- a/tests/test_dynapcnnlayer/conftest_dynapcnnlayer.py +++ b/tests/test_dynapcnnlayer/conftest_dynapcnnlayer.py @@ -1,5 +1,5 @@ import torch.nn as nn -from sinabs.layers import IAFSqueeze, SumPool2d +from sinabs.layers import IAFSqueeze from sinabs.activation.surrogate_gradient_fn import PeriodicExponential nodes_to_dcnnl_map = { @@ -110,7 +110,7 @@ 'pool_node_id': [2, 3], 'conv_rescaling_factor': None, 'dynapcnnlayer_destination': [1, 2], - 'nodes_destination': {2: [4], 3: [7]}, + 'nodes_destinations': {2: [4], 3: [7]}, }, 1: { 'dpcnnl_index': 1, @@ -121,7 +121,7 @@ 'pool_node_id': [], 'conv_rescaling_factor': 4.5, 'dynapcnnlayer_destination': [2], - 'nodes_destination': {6: [7]}, + 'nodes_destinations': {6: [7]}, }, 2: { 'dpcnnl_index': 2, @@ -132,7 +132,7 @@ 'pool_node_id': [], 'conv_rescaling_factor': 8.0, 'dynapcnnlayer_destination': [3], - 'nodes_destination': {8: [9]}, + 'nodes_destinations': {8: [9]}, }, 3: { 'dpcnnl_index': 3, @@ -143,7 +143,7 @@ 'pool_node_id': [], 'conv_rescaling_factor': None, 'dynapcnnlayer_destination': [4], - 'nodes_destination': {10: [11]}, + 'nodes_destinations': {10: [11]}, }, 4: { 'dpcnnl_index': 4, @@ -154,7 +154,7 @@ 'pool_node_id': [], 'conv_rescaling_factor': None, 'dynapcnnlayer_destination': [], - 'nodes_destination': {}, + 'nodes_destinations': {}, }, } diff --git a/tests/test_dynapcnnlayer/test_dynapcnnlayer.py b/tests/test_dynapcnnlayer/test_dynapcnnlayer.py index a440dbe2..cb3f810a 100644 --- a/tests/test_dynapcnnlayer/test_dynapcnnlayer.py +++ b/tests/test_dynapcnnlayer/test_dynapcnnlayer.py @@ -1,36 +1,51 @@ import pytest -from sinabs.backend.dynapcnn import DynapcnnLayer -from conftest_dynapcnnlayer import args_DynapcnnLayer -import sys - -sys.path.append('../../sinabs/backend/dynapcnn') +from sinabs.backend.dynapcnn.utils import construct_dynapcnnlayer, update_nodes_io +from sinabs.backend.dynapcnn.weight_rescaling_methods import rescale_method_1 -from weight_rescaling_methods import rescale_method_1 -from utils import convert_Avg_to_Sum_pooling +from conftest_dynapcnnlayer import args_DynapcnnLayer @pytest.mark.parametrize("nodes_to_dcnnl_map, dpcnnl_idx, sinabs_edges, expected_output", args_DynapcnnLayer) def test_DynapcnnLayer(nodes_to_dcnnl_map, dpcnnl_idx, sinabs_edges, expected_output): + """ Tests the instantiation of a set of `DynapcnnLayer` belonging to the same SNN and the data computed + within their constructors and shared among the differntly interacting instances (according to the graph + described by `sinabs_edges`). + """ + + # create a `DynapcnnLayer` from the set of layers in `nodes_to_dcnnl_map[dpcnnl_idx]`. + dynapcnnlayer = construct_dynapcnnlayer(dpcnnl_idx, True, sinabs_edges, nodes_to_dcnnl_map, rescale_method_1) - convert_Avg_to_Sum_pooling(nodes_to_dcnnl_map[dpcnnl_idx], sinabs_edges, nodes_to_dcnnl_map) + # check if any node (layer) in `dynapcnnlayer` has been modified (e.g. `nn.Linear` turned `nn.Conv2d`). + node, output_shape = dynapcnnlayer.get_modified_node_io(nodes_to_dcnnl_map[dpcnnl_idx]) - dynapcnnlayer = DynapcnnLayer( - dpcnnl_index = dpcnnl_idx, - dcnnl_data = nodes_to_dcnnl_map[dpcnnl_idx], - discretize = True, - sinabs_edges = sinabs_edges, - weight_rescaling_fn = rescale_method_1 - ) + # one of the layers in `dynapcnnlayer` had its type modified (update input shape of nodes receiving from it). + if isinstance(node, int) and isinstance(output_shape, tuple): + update_nodes_io(node, output_shape, nodes_to_dcnnl_map, sinabs_edges) - config = { - 'dpcnnl_index': dynapcnnlayer.dpcnnl_index, - 'conv_node_id': dynapcnnlayer.conv_node_id, - 'conv_in_shape': dynapcnnlayer.conv_in_shape, - 'conv_out_shape': dynapcnnlayer.conv_out_shape, - 'spk_node_id': dynapcnnlayer.spk_node_id, - 'pool_node_id': dynapcnnlayer.pool_node_id, - 'conv_rescaling_factor': dynapcnnlayer.conv_rescaling_factor, - 'dynapcnnlayer_destination': dynapcnnlayer.dynapcnnlayer_destination, - 'nodes_destinations': dynapcnnlayer.nodes_destinations, - } + dpcnnl_index = expected_output[dpcnnl_idx]['dpcnnl_index'] + conv_node_id = expected_output[dpcnnl_idx]['conv_node_id'] + conv_in_shape = expected_output[dpcnnl_idx]['conv_in_shape'] + conv_out_shape = expected_output[dpcnnl_idx]['conv_out_shape'] + spk_node_id = expected_output[dpcnnl_idx]['spk_node_id'] + pool_node_id = expected_output[dpcnnl_idx]['pool_node_id'] + conv_rescaling_factor = expected_output[dpcnnl_idx]['conv_rescaling_factor'] + dynapcnnlayer_destination = expected_output[dpcnnl_idx]['dynapcnnlayer_destination'] + nodes_destinations = expected_output[dpcnnl_idx]['nodes_destinations'] - assert config == expected_output[dpcnnl_idx] \ No newline at end of file + assert dynapcnnlayer.dpcnnl_index == expected_output[dpcnnl_idx]['dpcnnl_index'], \ + f'wrong \'DynapcnnLayer.dpcnnl_index\': ID of the instance should be {dpcnnl_index}.' + assert dynapcnnlayer.conv_node_id == expected_output[dpcnnl_idx]['conv_node_id'], \ + f'wrong \'DynapcnnLayer.conv_node_id\': convolution layer should be node {conv_node_id}.' + assert dynapcnnlayer.conv_in_shape == expected_output[dpcnnl_idx]['conv_in_shape'], \ + f'wrong \'DynapcnnLayer.conv_in_shape\': input tensor shape of convolution should be {conv_in_shape}.' + assert dynapcnnlayer.conv_out_shape == expected_output[dpcnnl_idx]['conv_out_shape'], \ + f'wrong \'DynapcnnLayer.conv_out_shape\': output tensor shape of convolution should be {conv_out_shape}.' + assert dynapcnnlayer.spk_node_id == expected_output[dpcnnl_idx]['spk_node_id'], \ + f'wrong \'DynapcnnLayer.spk_node_id\': spiking layer should be node {spk_node_id}.' + assert dynapcnnlayer.pool_node_id == expected_output[dpcnnl_idx]['pool_node_id'], \ + f'wrong \'DynapcnnLayer.pool_node_id\': pooling layer node(s) should be {pool_node_id}.' + assert dynapcnnlayer.conv_rescaling_factor == expected_output[dpcnnl_idx]['conv_rescaling_factor'], \ + f'wrong \'DynapcnnLayer.conv_rescaling_factor\': computed re-scaling factor should be {conv_rescaling_factor}.' + assert dynapcnnlayer.dynapcnnlayer_destination == expected_output[dpcnnl_idx]['dynapcnnlayer_destination'], \ + f'wrong \'DynapcnnLayer.dynapcnnlayer_destination\': the DynapcnnLayer(s) set as destination(s) should be {dynapcnnlayer_destination}.' + assert dynapcnnlayer.nodes_destinations == expected_output[dpcnnl_idx]['nodes_destinations'], \ + f'wrong \'DynapcnnLayer.nodes_destinations\': the targeted nodes within other DynapcnnLayer instance(s) should be {nodes_destinations}.' \ No newline at end of file From dd33ac231a33854348322fb4aa3046fae91f1c5a Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Mon, 27 May 2024 16:15:27 +0200 Subject: [PATCH 101/379] finished documenting methods' headers --- sinabs/backend/dynapcnn/dynapcnn_layer.py | 72 +++++++++++++++++------ 1 file changed, 53 insertions(+), 19 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer.py b/sinabs/backend/dynapcnn/dynapcnn_layer.py index 2e3c1ed5..03c220c7 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer.py @@ -46,7 +46,7 @@ def __init__( if 'core_idx' in dcnnl_data: self.assigned_core = dcnnl_data['core_idx'] - self.lin_to_conv_conversion = False + self._lin_to_conv_conversion = False conv = None self.conv_node_id = None @@ -154,11 +154,11 @@ def get_destination_dcnnl_index(self, dcnnl_id: int) -> int: Parameters ---------- - dcnnl_id (int): this should be one of the values listed within `self.dynapcnnlayer_destination`. + - dcnnl_id (int): this should be one of the values listed within `self.dynapcnnlayer_destination`. Returns ---------- - The index of `dcnnl_id` within `self.dynapcnnlayer_destination`. + - The index of `dcnnl_id` within `self.dynapcnnlayer_destination`. """ return self.dynapcnnlayer_destination.index(dcnnl_id) @@ -167,23 +167,23 @@ def forward(self, x): Returns ---------- - This method will return as many tensors as there are destinations associated with this instance. The returned tensors always follows the - sequence `return spiking_layer_output, pooling_layer_1, pooling_layer_2, ...`. + - forward output (tuple): returns as many tensors as there are destinations associated with this instance. The returned + tensors always follows the sequence `return spiking_layer_output, pooling_layer_1, pooling_layer_2, ...`. Example ---------- - With `self.nodes_destinations = {1: [5, 8], 2: [4], 3: [7]}` this method will return 4 tensors (each to be sent to a different DynapcnnLayer): the 1st + - With `self.nodes_destinations = {1: [5, 8], 2: [4], 3: [7]}` this method will return 4 tensors (each to be sent to a different DynapcnnLayer): the 1st and 2nd are the outputs of the spiking layer (`node 1`); the 3rd is the output of the first pooling layer (`node 2`, receiving the output of node 1) appearing right after the spiking layer as defined in the original `nn.Module` being converted to a `DynapcnnNetwork`; the 4th is the output of the second pooling layer (`node 3`) to appear after the spiking layer. Thus, the return will be `return node_1_out, node_1_out, node_2_out, node_3_out`. This means the edges in the computational graph involved in this mapping were: - 1 --> 2 # `2` is one of the pooling layers of this DynapcnnLayer. - 1 --> 3 # `3` is one of the pooling layers of this DynapcnnLayer. - 1 --> 5 # `5` is a conv layer belonging to another DynapcnnLayer U. - 1 --> 8 # `8` is a conv layer belonging to another DynapcnnLayer V. - 2 --> 4 # `4` is a conv layer belonging to another DynapcnnLayer X. - 3 --> 7 # `7` is a conv layer belonging to another DynapcnnLayer Y. + - 1 --> 2 # `2` is one of the pooling layers of this DynapcnnLayer. + - 1 --> 3 # `3` is one of the pooling layers of this DynapcnnLayer. + - 1 --> 5 # `5` is a conv layer belonging to another DynapcnnLayer U. + - 1 --> 8 # `8` is a conv layer belonging to another DynapcnnLayer V. + - 2 --> 4 # `4` is a conv layer belonging to another DynapcnnLayer X. + - 3 --> 7 # `7` is a conv layer belonging to another DynapcnnLayer Y. """ returns = [] @@ -238,15 +238,25 @@ def get_modified_node_io(self, dcnnl_data: dict) -> Union[Tuple[int, tuple], Tup - node ID (int): the ID of the spiking layer consuming the tunerd layer's output (`None` if there was no conversion). - output shape (tuple): the new output shape following a converstion from `nn.Linear` to `nn.Conv2d` (`None` if there was no conversion). """ - if self.lin_to_conv_conversion: + if self._lin_to_conv_conversion: return self.spk_node_id, dcnnl_data[self.spk_node_id]['output_shape'] return None, None def zero_grad(self, set_to_none: bool = False) -> None: return self.spk_layer.zero_grad(set_to_none) - def get_conv_output_shape(self, conv_layer: nn.Conv2d, input_shape: tuple): - """ .""" + def get_conv_output_shape(self, conv_layer: nn.Conv2d, input_shape: Tuple[int, int, int]) -> Tuple[int, int, int]: + """ Computes the output dimensions of `conv_layer`. + + Parameters + ---------- + - conv_layer (nn.Conv2d): conv. layer whose output will be computed for. + - input_shape (tuple): the shape for the input tensor the layer will process. + + Returns + ---------- + - output dimensions (tuple): a tuple describing `(output channels, height, width)`. + """ # get the layer's parameters. out_channels = conv_layer.out_channels kernel_size = conv_layer.kernel_size @@ -261,6 +271,7 @@ def get_conv_output_shape(self, conv_layer: nn.Conv2d, input_shape: tuple): return (out_channels, out_height, out_width) def summary(self) -> dict: + """ Returns a summary of the convolution's/pooling's kernel sizes and the output shape of the spiking layer.""" # TODO I can't see pooling being used in checking memory constraints by the builder so I'm ignoring for now the fact that multiple pooling could exist. _pool = None @@ -285,6 +296,10 @@ def summary(self) -> dict: def get_layer_config_dict(self) -> dict: """ Returns a dict containing the properties required to configure a `CNNLayerConfig` instance that will map this DynapcnnLayer onto the chip. + + Returns + ---------- + - config_dict (dict): a nested dictionary containing of the variables necessary to configure a `CNNLayerConfig` instance. """ config_dict = {} @@ -444,17 +459,29 @@ def _update_neuron_node_output_shape(self, spiking_layer_data: dict, conv_out_sh # spiking layer outputs the same shape as the conv. layer. spiking_layer_data['output_shape'] = spiking_layer_data['input_shape'] - def _update_conv_node_output_shape(self, conv_layer: nn.Conv2d, layer_data: dict, input_shape: tuple) -> Tuple: - """ The input shapes to nodes are extracted using a list of edges by finding the output shape of the 1st element + def _update_conv_node_output_shape(self, conv_layer: nn.Conv2d, layer_data: dict, input_shape: Tuple[int, int, int]) -> Tuple[int, int, int]: + """ Updates the shape of the output tensor of a node that used to be a `nn.Linear` and became a `nn.Conv2d`. + + The input shapes to nodes are extracted using a list of edges by finding the output shape of the 1st element in the edge and setting it as the input shape to the 2nd element in the edge. If a node used to be a `nn.Linear` and it became a `nn.Conv2d`, output shape in the mapper needs to be updated, otherwise there will be a mismatch between its output and the input it provides to another node. + + Parameters + ---------- + - conv_layer (nn.Module): the `nn.Conv2d` created from a `nn.Linear`. + - layer_data (dict): the dictionary containing the data associated with the original `nn.Linear` converted into `nn.Conv2d`. + - input_shape (tuple): the input shape the layer expects. + + Returns + ---------- + - output_shape (tuple): the tensor shape produced by the `nn.Conv2d` created from a `nn.Linear`. """ layer_data['output_shape'] = self.get_conv_output_shape(conv_layer, input_shape) return layer_data['output_shape'] - def _convert_linear_to_conv(self, lin: nn.Linear, layer_data: dict) -> nn.Conv2d: + def _convert_linear_to_conv(self, lin: nn.Linear, layer_data: dict) -> Tuple[nn.Conv2d, Tuple[int, int, int]]: """ Convert Linear layer to Conv2d. Parameters @@ -464,9 +491,10 @@ def _convert_linear_to_conv(self, lin: nn.Linear, layer_data: dict) -> nn.Conv2d Returns ------- - nn.Conv2d: convolutional layer equivalent to `lin`. + - input_shape (tuple): the tensor shape the layer expects. """ # this flags the necessity to update the I/O shape pre-computed for each of the original layers being compressed within a `DynapcnnLayer` instance. - self.lin_to_conv_conversion = True + self._lin_to_conv_conversion = True input_shape = layer_data['input_shape'] @@ -498,6 +526,12 @@ def _get_destinations_input_source(self, sinabs_edges: list) -> dict: """ Creates a mapping between each layer in this `DynapcnnLayer` instance and its targe nodes that are part of different `DynapcnnLayer` instances. This mapping is used to figure out how many tensors the `forward` method needs to return. + Parameters + ---------- + - sinabs_edges (list): each `nn.Module` within `dcnnl_data` is a node in the original computational graph describing a spiking + network. An edge `(A, B)` describes how modules forward data amongst themselves. This list is used by a `DynapcnnLayer` to + figure out the number and sequence of output tesnors its forward method needs to return. + Returns ---------- - destinations_input_source (dict): maps a `nn.Module` within this `DynapcnnLayer` to the nodes it provides the input to. From a3551138d243e5ddfc5038526c03d866f35f985d Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Mon, 27 May 2024 18:27:56 +0200 Subject: [PATCH 102/379] Refactor Method mapping nodes to their I/O shapes now handles the case where a Merge layer appears as producing input but its I/O shapes have yet to be computed. --- sinabs/backend/dynapcnn/NIRGraphExtractor.py | 73 +++++++++++++++++--- 1 file changed, 63 insertions(+), 10 deletions(-) diff --git a/sinabs/backend/dynapcnn/NIRGraphExtractor.py b/sinabs/backend/dynapcnn/NIRGraphExtractor.py index de94ca23..35fe8987 100644 --- a/sinabs/backend/dynapcnn/NIRGraphExtractor.py +++ b/sinabs/backend/dynapcnn/NIRGraphExtractor.py @@ -24,6 +24,13 @@ def __init__(self, spiking_model: nn.Module, dummy_input: torch.tensor): # converts the NIR representation into a list of edges with nodes represented as integers. self._edges_list, self._name_2_indx_map = self._get_edges_from_nir(nir_graph) + + # for key, val in self._name_2_indx_map.items(): + # print(key, val) + # print('---------------------------------------------------') + # for edge in self._edges_list: + # print(edge) + # print('---------------------------------------------------') # recovers the associated `nn.Module` (layer) of each node. self.modules_map = self._get_named_modules(spiking_model) @@ -174,7 +181,7 @@ def _get_nodes_io_shapes(self, input_dummy: torch.tensor) -> Dict[int, Dict[str, nodes_io_map[trg] = {'input': None, 'output': None} # find node generating the input to be used. - inp_node = self._find_input_to_node(trg) + inp_node = self._find_source_of_input_to(trg) _input = nodes_io_map[inp_node]['output'] # forward input through the node. @@ -222,13 +229,18 @@ def _get_nodes_io_shapes(self, input_dummy: torch.tensor) -> Dict[int, Dict[str, else: # find node generating the input to be used. - inp_node = self._find_input_to_node(src) + inp_node = self._find_source_of_input_to(src) if inp_node == -1: # `src` is receiving external (not from another layer) input. This will be the case when two # parallel branches (two independent "input nodes" in the graph) merge at some point in the graph. _input = input_dummy else: + if isinstance(self.modules_map[inp_node], sinabs.layers.merge.Merge): + # source of input is a `Merge` layer that might still need to have its I/O shapes computed. + self._handle_merge_source(inp_node, nodes_io_map) + + # record what the input shape for `src` should be. _input = nodes_io_map[inp_node]['output'] # forward input through the node. @@ -247,13 +259,18 @@ def _get_nodes_io_shapes(self, input_dummy: torch.tensor) -> Dict[int, Dict[str, _input = input_dummy else: # find node generating the input to be used. - inp_node = self._find_input_to_node(src) + inp_node = self._find_source_of_input_to(src) if inp_node == -1: # `src` is receiving external (not from another layer) input. This will be the case when two # parallel branches (two independent "input nodes" in the graph) merge at some point in the graph. _input = input_dummy else: + if isinstance(self.modules_map[inp_node], sinabs.layers.merge.Merge): + # source of input is a `Merge` layer that might still need to have its I/O shapes computed. + self._handle_merge_source(inp_node, nodes_io_map) + + # record what the input shape for `src` should be. _input = nodes_io_map[inp_node]['output'] # forward input through the node. @@ -267,13 +284,18 @@ def _get_nodes_io_shapes(self, input_dummy: torch.tensor) -> Dict[int, Dict[str, nodes_io_map[trg] = {'input': None, 'output': None} # find node generating the input to be used. - inp_node = self._find_input_to_node(trg) + inp_node = self._find_source_of_input_to(trg) if inp_node == -1: # `src` is receiving external (not from another layer) input. This will be the case when two # parallel branches (two independent "input nodes" in the graph) merge at some point in the graph. _input = input_dummy else: + if isinstance(self.modules_map[inp_node], sinabs.layers.merge.Merge): + # source of input is a `Merge` layer that might still need to have its I/O shapes computed. + self._handle_merge_source(inp_node, nodes_io_map) + + # record what the input shape for `trg` should be. _input = nodes_io_map[inp_node]['output'] # forward input through the node. @@ -288,20 +310,51 @@ def _get_nodes_io_shapes(self, input_dummy: torch.tensor) -> Dict[int, Dict[str, nodes_io_map[node]['output'] = io['output'].shape return nodes_io_map + + def _handle_merge_source(self, merge_node_id: int, nodes_io_map: dict) -> None: + """ This method finds the I/O shapes for node `merge_node_id` if they haven't been computed yet. When `self._find_source_of_input_to()` is + called the returned node might be a `Merge` layer for which the I/O shapes have yet to be computed. + + NOTE: In the current implemente both arguments to a `Merge` layer need to have the same output shapes. + + Parameters + ---------- + - merge_node_id (int): the ID of the node representing a `Merge` layer. + - nodes_io_map (dict): a dictionary mapping nodes to their I/O shapes. + """ + + if merge_node_id in nodes_io_map: + # I/O shapes have been computed already. + return None + + # finding nodes serving as argument to the `Merge` node... + for edge in self._edges_list: + + if edge[1] == merge_node_id: + # node `edge[0]` is one of the arguments for the `Merge` layer. + if edge[0] in nodes_io_map: + # I/O shapes of one of the arguments for the `Merge` node has been computed. + + # both arguments to `Merge` have the same I/O shape and merge outputs the same shape: updating I/O shape of `merge_node_id`. + nodes_io_map[merge_node_id] = {'input': nodes_io_map[edge[0]]['output'], 'output': nodes_io_map[edge[0]]['output']} + + return None + + raise ValueError(f'Node {merge_node_id} is a \'Merge\' layer and I/O shape for none of its arguments have been computed yet.') - def _find_input_to_node(self, node: int) -> int: + def _find_source_of_input_to(self, node: int) -> int: """ Finds the first edge `(X, node)` returns `X`. Parameters ---------- - node (int): the node in the computational graph for which we whish to find the input source (either another node in the - graph or the original input itself to the network). + - node (int): the node in the computational graph for which we whish to find the input source (either another node in the + graph or the original input itself to the network). Returns ---------- - input source (int): this indicates the node in the computational graph providing the input to `node`. If `node` is - receiving outside input (i.e., it is a starting node) the return will be -1. For example, this will be the case - when a network with two independent branches (each starts from a different "input node") merge along the computational graph. + - input source (int): this indicates the node in the computational graph providing the input to `node`. If `node` is + receiving outside input (i.e., it is a starting node) the return will be -1. For example, this will be the case + when a network with two independent branches (each starts from a different "input node") merge along the computational graph. """ for edge in self._edges_list: if edge[1] == node: From 55b6996a2584cdce5d88ddb02024dabb2d8709bc Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Mon, 27 May 2024 19:13:15 +0200 Subject: [PATCH 103/379] Unit testing Unit test example 'A network with a merge and a split' --- .../conftest_dynapcnnlayer.py | 224 +++++++++++++++++- 1 file changed, 215 insertions(+), 9 deletions(-) diff --git a/tests/test_dynapcnnlayer/conftest_dynapcnnlayer.py b/tests/test_dynapcnnlayer/conftest_dynapcnnlayer.py index b467c268..4fd0682c 100644 --- a/tests/test_dynapcnnlayer/conftest_dynapcnnlayer.py +++ b/tests/test_dynapcnnlayer/conftest_dynapcnnlayer.py @@ -1,8 +1,8 @@ import torch.nn as nn -from sinabs.layers import IAFSqueeze +from sinabs.layers import IAFSqueeze, SumPool2d from sinabs.activation.surrogate_gradient_fn import PeriodicExponential -nodes_to_dcnnl_map = { +nodes_to_dcnnl_map_1 = { 0: { 0: { 'layer': nn.Conv2d(2, 10, kernel_size=(2, 2), stride=(1, 1), bias=False), @@ -85,7 +85,7 @@ } } -sinabs_edges = [ +sinabs_edges_1 = [ (0, 1), (1, 2), (1, 3), @@ -100,7 +100,7 @@ (11, 12), ] -expected_output = { +expected_output_1 = { 0: { 'dpcnnl_index': 0, 'conv_node_id': 0, @@ -158,10 +158,216 @@ }, } +nodes_to_dcnnl_map_2 = { + 0: { + 0: {'layer': nn.Conv2d(2, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), + 'input_shape': (2, 34, 34), + 'output_shape': (4, 33, 33) + }, + 1: {'layer': IAFSqueeze(batch_size=8, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), + 'input_shape': (4, 33, 33), + 'output_shape': (4, 33, 33) + }, + 'destinations': [1], + 'conv_rescale_factor': [] + }, + 1: { + 2: {'layer': nn.Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), + 'input_shape': (4, 33, 33), + 'output_shape': (4, 32, 32) + }, + 3: {'layer': IAFSqueeze(batch_size=8, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), + 'input_shape': (4, 32, 32), + 'output_shape': (4, 32, 32) + }, + 4: {'layer': SumPool2d(kernel_size=2, stride=2, ceil_mode=False), + 'input_shape': (4, 32, 32), + 'output_shape': (4, 16, 16) + }, + 'destinations': [2, 3], + 'conv_rescale_factor': [] + }, + 2: { + 5: {'layer': nn.Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), + 'input_shape': (4, 16, 16), + 'output_shape': (4, 15, 15) + }, + 7: {'layer': IAFSqueeze(batch_size=8, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), + 'input_shape': (4, 15, 15), + 'output_shape': (4, 15, 15) + }, + 8: {'layer': SumPool2d(kernel_size=2, stride=2, ceil_mode=False), + 'input_shape': (4, 15, 15), + 'output_shape': (4, 7, 7) + }, + 'destinations': [4], + 'conv_rescale_factor': [] + }, + 3: { + 6: {'layer': nn.Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), + 'input_shape': (4, 16, 16), + 'output_shape': (4, 15, 15) + }, + 11: {'layer': IAFSqueeze(batch_size=8, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), + 'input_shape': (4, 15, 15), + 'output_shape': (4, 15, 15) + }, + 12: {'layer': SumPool2d(kernel_size=2, stride=2, ceil_mode=False), + 'input_shape': (4, 15, 15), + 'output_shape': (4, 7, 7) + }, + 'destinations': [6], + 'conv_rescale_factor': [] + }, + 4: { + 9: {'layer': nn.Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), + 'input_shape': (4, 7, 7), + 'output_shape': (4, 6, 6) + }, + 10: {'layer': IAFSqueeze(batch_size=8, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), + 'input_shape': (4, 6, 6), + 'output_shape': (4, 6, 6) + }, + 'destinations': [5], + 'conv_rescale_factor': [] + }, + 5 :{ + 15: {'layer': nn.Linear(in_features=144, out_features=10, bias=False), + 'input_shape': (4, 6, 6), + 'output_shape': (10,) + }, + 16: {'layer': IAFSqueeze(batch_size=8, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), + 'input_shape': (10,), + 'output_shape': (10,) + }, + 'destinations': [], + 'conv_rescale_factor': [] + }, + 6: { + 13: {'layer': nn.Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), + 'input_shape': (4, 7, 7), + 'output_shape': (4, 6, 6) + }, + 14: {'layer': IAFSqueeze(batch_size=8, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), + 'input_shape': (4, 6, 6), + 'output_shape': (4, 6, 6) + }, + 'destinations': [5], + 'conv_rescale_factor': [] + } +} + +sinabs_edges_2 = [ + (0, 1), + (1, 2), + (2, 3), + (3, 4), + (4, 5), + (4, 6), + (5, 7), + (7, 8), + (8, 9), + (9, 10), + (10, 15), + (6, 11), + (11, 12), + (12, 13), + (13, 14), + (14, 15), + (15, 16), +] + +expected_output_2 = { + 0: { + 'dpcnnl_index': 0, + 'conv_node_id': 0, + 'conv_in_shape': (2, 34, 34), + 'conv_out_shape': (4, 33, 33), + 'spk_node_id': 1, + 'pool_node_id': [], + 'conv_rescaling_factor': None, + 'dynapcnnlayer_destination': [1], + 'nodes_destinations': {1: [2]}, + }, + 1: { + 'dpcnnl_index': 1, + 'conv_node_id': 2, + 'conv_in_shape': (4, 33, 33), + 'conv_out_shape': (4, 32, 32), + 'spk_node_id': 3, + 'pool_node_id': [4], + 'conv_rescaling_factor': None, + 'dynapcnnlayer_destination': [2, 3], + 'nodes_destinations': {4: [5, 6]}, + }, + 2: { + 'dpcnnl_index': 2, + 'conv_node_id': 5, + 'conv_in_shape': (4, 16, 16), + 'conv_out_shape': (4, 15, 15), + 'spk_node_id': 7, + 'pool_node_id': [8], + 'conv_rescaling_factor': None, + 'dynapcnnlayer_destination': [4], + 'nodes_destinations': {8: [9]}, + }, + 3: { + 'dpcnnl_index': 3, + 'conv_node_id': 6, + 'conv_in_shape': (4, 16, 16), + 'conv_out_shape': (4, 15, 15), + 'spk_node_id': 11, + 'pool_node_id': [12], + 'conv_rescaling_factor': None, + 'dynapcnnlayer_destination': [6], + 'nodes_destinations': {12: [13]}, + }, + 4: { + 'dpcnnl_index': 4, + 'conv_node_id': 9, + 'conv_in_shape': (4, 7, 7), + 'conv_out_shape': (4, 6, 6), + 'spk_node_id': 10, + 'pool_node_id': [], + 'conv_rescaling_factor': None, + 'dynapcnnlayer_destination': [5], + 'nodes_destinations': {10: [15]}, + }, + 5: { + 'dpcnnl_index': 5, + 'conv_node_id': 15, + 'conv_in_shape': (4, 6, 6), + 'conv_out_shape': (10, 1, 1), + 'spk_node_id': 16, + 'pool_node_id': [], + 'conv_rescaling_factor': None, + 'dynapcnnlayer_destination': [], + 'nodes_destinations': {}, + }, + 6: { + 'dpcnnl_index': 6, + 'conv_node_id': 13, + 'conv_in_shape': (4, 7, 7), + 'conv_out_shape': (4, 6, 6), + 'spk_node_id': 14, + 'pool_node_id': [], + 'conv_rescaling_factor': None, + 'dynapcnnlayer_destination': [5], + 'nodes_destinations': {14: [15]}, + }, +} + args_DynapcnnLayer = [ - (nodes_to_dcnnl_map, 0, sinabs_edges, expected_output), - (nodes_to_dcnnl_map, 1, sinabs_edges, expected_output), - (nodes_to_dcnnl_map, 2, sinabs_edges, expected_output), - (nodes_to_dcnnl_map, 3, sinabs_edges, expected_output), - (nodes_to_dcnnl_map, 4, sinabs_edges, expected_output), + (nodes_to_dcnnl_map_1, 0, sinabs_edges_1, expected_output_1), + (nodes_to_dcnnl_map_1, 1, sinabs_edges_1, expected_output_1), + (nodes_to_dcnnl_map_1, 2, sinabs_edges_1, expected_output_1), + (nodes_to_dcnnl_map_1, 3, sinabs_edges_1, expected_output_1), + (nodes_to_dcnnl_map_1, 4, sinabs_edges_1, expected_output_1), + (nodes_to_dcnnl_map_2, 0, sinabs_edges_2, expected_output_2), + (nodes_to_dcnnl_map_2, 1, sinabs_edges_2, expected_output_2), + (nodes_to_dcnnl_map_2, 2, sinabs_edges_2, expected_output_2), + (nodes_to_dcnnl_map_2, 3, sinabs_edges_2, expected_output_2), + (nodes_to_dcnnl_map_2, 4, sinabs_edges_2, expected_output_2), + (nodes_to_dcnnl_map_2, 5, sinabs_edges_2, expected_output_2), + (nodes_to_dcnnl_map_2, 6, sinabs_edges_2, expected_output_2), ] \ No newline at end of file From c8794532c8ce856a46183ded88ff358d4256adbd Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Mon, 27 May 2024 22:58:06 +0200 Subject: [PATCH 104/379] Refactor + Unit testing - DynapcnnNetworkModule now computes a topological sorting on the DynapcnnLayer edges to implement the forward method. - (WIP) nodes I/O mapping method needs to use topological sorting. --- sinabs/backend/dynapcnn/NIRGraphExtractor.py | 59 +++++++-- sinabs/backend/dynapcnn/dynapcnn_layer.py | 21 ++- sinabs/backend/dynapcnn/dynapcnn_network.py | 123 +++++++++++------- .../dynapcnn/dynapcnnnetwork_module.py | 53 ++++++++ sinabs/backend/dynapcnn/utils.py | 40 ++++-- 5 files changed, 230 insertions(+), 66 deletions(-) diff --git a/sinabs/backend/dynapcnn/NIRGraphExtractor.py b/sinabs/backend/dynapcnn/NIRGraphExtractor.py index 35fe8987..4297824e 100644 --- a/sinabs/backend/dynapcnn/NIRGraphExtractor.py +++ b/sinabs/backend/dynapcnn/NIRGraphExtractor.py @@ -25,21 +25,25 @@ def __init__(self, spiking_model: nn.Module, dummy_input: torch.tensor): # converts the NIR representation into a list of edges with nodes represented as integers. self._edges_list, self._name_2_indx_map = self._get_edges_from_nir(nir_graph) - # for key, val in self._name_2_indx_map.items(): - # print(key, val) - # print('---------------------------------------------------') - # for edge in self._edges_list: - # print(edge) - # print('---------------------------------------------------') + for key, val in self._name_2_indx_map.items(): + print(key, val) + print('---------------------------------------------------') + for edge in self._edges_list: + print(edge) + print('---------------------------------------------------') # recovers the associated `nn.Module` (layer) of each node. self.modules_map = self._get_named_modules(spiking_model) # retrieves what the I/O shape for each node's module is. - self._nodes_io_shapes = self._get_nodes_io_shapes(dummy_input) + self._nodes_io_shapes, self._flagged_input_nodes = self._get_nodes_io_shapes(dummy_input) ### Publich Methods ### + @property + def flagged_input_nodes(self) -> List[int]: + return self._flagged_input_nodes + def get_edges_list(self): return self._edges_list @@ -161,18 +165,31 @@ def _get_named_modules(self, model: nn.Module) -> Dict[int, nn.Module]: return modules_map - def _get_nodes_io_shapes(self, input_dummy: torch.tensor) -> Dict[int, Dict[str, torch.Size]]: + # TODO - THIS ALSO NEEDS TOPOLOGICAL SORTING TO CORRECTLY GET I/O SHAPES UNDER ALL CIRCUNSTANCES. + def _get_nodes_io_shapes(self, input_dummy: torch.tensor) -> Tuple[Dict[int, Dict[str, torch.Size]], List]: """ Loops through the graph represented in `self._edges_list` and propagates the inputs through the nodes, starting from `node 0` fed `input_dummy`. + + Parameters + ---------- + - input_dummy (torch.tensor): a sample (random) tensor of the sort of input being fed to the network. + + Returns + ---------- + - nodes_io_map (dict): a dictionary mapping nodes to their I/O shapes. + - flagged_input_nodes (list): IDs of nodes that are receiving as input `input_dummy` (i.e., input nodes of the network). """ nodes_io_map = {} flagged_merge_nodes = {} + flagged_input_nodes = [] # propagate inputs through the nodes. for edge in self._edges_list: src = edge[0] trg = edge[1] + print('> ', edge) + if isinstance(self.modules_map[src], sinabs.layers.merge.Merge): # At this point the output of Merge has to have been calculated. @@ -226,6 +243,10 @@ def _get_nodes_io_shapes(self, input_dummy: torch.tensor) -> Dict[int, Dict[str, if src == 0: # first node in the graph. _input = input_dummy + + # flag node being an input node of the network. + if src not in flagged_input_nodes: + flagged_input_nodes.append(src) else: # find node generating the input to be used. @@ -235,11 +256,17 @@ def _get_nodes_io_shapes(self, input_dummy: torch.tensor) -> Dict[int, Dict[str, # `src` is receiving external (not from another layer) input. This will be the case when two # parallel branches (two independent "input nodes" in the graph) merge at some point in the graph. _input = input_dummy + + # flag node being an input node of the network. + if src not in flagged_input_nodes: + flagged_input_nodes.append(src) else: if isinstance(self.modules_map[inp_node], sinabs.layers.merge.Merge): # source of input is a `Merge` layer that might still need to have its I/O shapes computed. self._handle_merge_source(inp_node, nodes_io_map) + print(f'accessing node {inp_node} cuz it is the input to node {src}....') + # record what the input shape for `src` should be. _input = nodes_io_map[inp_node]['output'] @@ -257,6 +284,10 @@ def _get_nodes_io_shapes(self, input_dummy: torch.tensor) -> Dict[int, Dict[str, if src == 0: # first node in the graph. _input = input_dummy + + # flag node being an input node of the network. + if src not in flagged_input_nodes: + flagged_input_nodes.append(src) else: # find node generating the input to be used. inp_node = self._find_source_of_input_to(src) @@ -265,6 +296,10 @@ def _get_nodes_io_shapes(self, input_dummy: torch.tensor) -> Dict[int, Dict[str, # `src` is receiving external (not from another layer) input. This will be the case when two # parallel branches (two independent "input nodes" in the graph) merge at some point in the graph. _input = input_dummy + + # flag node being an input node of the network. + if src not in flagged_input_nodes: + flagged_input_nodes.append(src) else: if isinstance(self.modules_map[inp_node], sinabs.layers.merge.Merge): # source of input is a `Merge` layer that might still need to have its I/O shapes computed. @@ -290,6 +325,10 @@ def _get_nodes_io_shapes(self, input_dummy: torch.tensor) -> Dict[int, Dict[str, # `src` is receiving external (not from another layer) input. This will be the case when two # parallel branches (two independent "input nodes" in the graph) merge at some point in the graph. _input = input_dummy + + # flag node being an input node of the network. + if trg not in flagged_input_nodes: + flagged_input_nodes.append(trg) else: if isinstance(self.modules_map[inp_node], sinabs.layers.merge.Merge): # source of input is a `Merge` layer that might still need to have its I/O shapes computed. @@ -309,7 +348,7 @@ def _get_nodes_io_shapes(self, input_dummy: torch.tensor) -> Dict[int, Dict[str, nodes_io_map[node]['input'] = io['input'].shape nodes_io_map[node]['output'] = io['output'].shape - return nodes_io_map + return nodes_io_map, flagged_input_nodes def _handle_merge_source(self, merge_node_id: int, nodes_io_map: dict) -> None: """ This method finds the I/O shapes for node `merge_node_id` if they haven't been computed yet. When `self._find_source_of_input_to()` is @@ -352,7 +391,7 @@ def _find_source_of_input_to(self, node: int) -> int: Returns ---------- - - input source (int): this indicates the node in the computational graph providing the input to `node`. If `node` is + - input source (int): ID of the node in the computational graph providing the input to `node`. If `node` is receiving outside input (i.e., it is a starting node) the return will be -1. For example, this will be the case when a network with two independent branches (each starts from a different "input node") merge along the computational graph. """ diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer.py b/sinabs/backend/dynapcnn/dynapcnn_layer.py index 03c220c7..2fffd045 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer.py @@ -28,6 +28,7 @@ class DynapcnnLayer(nn.Module): sequence of output tesnors its forward method needs to return. - weight_rescaling_fn (callable): a method that handles how the re-scaling factor for one or more `SumPool2d` projecting to the same convolutional layer are combined/re-scaled before applying them. + - flagged_input_nodes (list): node IDs corresponding to layers in the original network that are input nodes (i.e., a "point of entry" for the external data). """ def __init__( @@ -36,12 +37,14 @@ def __init__( dcnnl_data: Dict[Union[int, str], Union[Dict[str, Union[nn.Module, Tuple[int, int, int], Tuple[int, int, int]]], List[int]]], discretize: bool, sinabs_edges: List[Tuple[int, int]], - weight_rescaling_fn: Callable + weight_rescaling_fn: Callable, + flagged_input_nodes: List[int] ): super().__init__() self.dpcnnl_index = dpcnnl_index self.assigned_core = None + self.entry_point = False if 'core_idx' in dcnnl_data: self.assigned_core = dcnnl_data['core_idx'] @@ -92,6 +95,9 @@ def __init__( if len(list(spk.v_mem.shape)) != 4: spk.v_mem = spk.v_mem.data.unsqueeze(-1).unsqueeze(-1) # expand dims. + if self.dpcnnl_index == 5: + print(' **************88888888888888888888888 ', spk.v_mem.shape) + if isinstance(conv, nn.Linear): # A `nn.Linear` needs to be converted into `nn.Conv2d`. The I/O shapes of the spiking layer are updated # accordingly following the conversion. @@ -146,6 +152,10 @@ def __init__( # map destination nodes for each layer in this instance. self.nodes_destinations = self._get_destinations_input_source(sinabs_edges) + # flag if the instance is an entry point (i.e., an input node of the network). + if self.conv_node_id in flagged_input_nodes: + self.entry_point = True + ####################################################### Public Methods ####################################################### def get_destination_dcnnl_index(self, dcnnl_id: int) -> int: @@ -191,6 +201,9 @@ def forward(self, x): x = self.conv_layer(x) x = self.spk_layer(x) + if self.dpcnnl_index == 5: + print('NEURON OUTPUT: ', x.shape) + # pooling layers are sequentially added to `self.pool_layer` so we'll pass data to them sequentially. pooling_indexer = 0 @@ -435,6 +448,7 @@ def __str__(self): pretty_print += f'\n(node {self.pool_node_id[idx]}): {lyr}' pretty_print += '\n\nMETADATA:\n' + pretty_print += f'\n> network\'s entry point: {self.entry_point}' pretty_print += f'\n> convolution\'s weight re-scaling factor: {self.conv_rescaling_factor}' pretty_print += f'\n> assigned core index: {self.assigned_core}' pretty_print += f'\n> destination DynapcnnLayers: {self.dynapcnnlayer_destination}' @@ -454,11 +468,16 @@ def _update_neuron_node_output_shape(self, spiking_layer_data: dict, conv_out_sh - spiking_layer_data (dict): the dictionary containing all data regarding the spiking layer. - conv_out_shape (tuple): the output shape of the convolution layer preceeding the spiking layer. """ + # spiking layer consumes the tensor coming out of the conv. layer. spiking_layer_data['input_shape'] = conv_out_shape # spiking layer outputs the same shape as the conv. layer. spiking_layer_data['output_shape'] = spiking_layer_data['input_shape'] + + if self.dpcnnl_index == 5: + print('>>>>> ', spiking_layer_data['output_shape']) + def _update_conv_node_output_shape(self, conv_layer: nn.Conv2d, layer_data: dict, input_shape: Tuple[int, int, int]) -> Tuple[int, int, int]: """ Updates the shape of the output tensor of a node that used to be a `nn.Linear` and became a `nn.Conv2d`. diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index 4f3c5441..bf4c0aad 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -5,7 +5,7 @@ contact : williansoaresgirao@gmail.com """ -import time +import time, copy from typing import List, Optional, Sequence, Tuple, Union, Dict, Callable import samna @@ -72,6 +72,7 @@ def __init__( - self._nodes_name_remap - self._nodes_to_dcnnl_map - self._dynapcnn_layers + - self._flagged_input_nodes """ super().__init__() @@ -87,7 +88,9 @@ def __init__( # TODO - bacth size must be passed as argument. self._graph_tracer = NIRtoDynapcnnNetworkGraph( snn, - torch.randn((batch_size, *self.input_shape))) # needs the batch dimension. + torch.randn((batch_size, *self.input_shape))) # needs the batch dimension. + + self._flagged_input_nodes = copy.deepcopy(self._graph_tracer.flagged_input_nodes) self._sinabs_edges, \ self._sinabs_modules_map, \ @@ -97,19 +100,33 @@ def __init__( self._nodes_to_dcnnl_map = build_nodes_to_dcnnl_map( layers=self._sinabs_modules_map, edges=self._sinabs_edges) + + # print('-----------------------------------------------------------------') + # for edge in self._sinabs_edges: + # print(edge) + # print('-----------------------------------------------------------------') # updates 'self._nodes_to_dcnnl_map' to include the I/O shape for each node. self._populate_nodes_io() + # print('-----------------------------------------------------------------') + # for key, val in self._nodes_to_dcnnl_map.items(): + # print(key, val) + # print('-----------------------------------------------------------------') + # build `DynapcnnLayer` instances from graph edges and mapper. self._dynapcnn_layers = build_from_graph( discretize = discretize, edges = self._sinabs_edges, nodes_to_dcnnl_map = self._nodes_to_dcnnl_map, - weight_rescaling_fn = weight_rescaling_fn) + weight_rescaling_fn = weight_rescaling_fn, + flagged_input_nodes = self._flagged_input_nodes) # these gather all data necessay to implement the forward method for this class. - self._dcnnl_edges, self._forward_map, self._merge_points = self._get_network_module() + self._dcnnl_edges, self._forward_map, self._merge_points, self._topological_order = self._get_network_module() + + # for edge in self._dcnnl_edges: + # print(edge) # all necessary `DynapcnnLayer` data held in `self._forward_map`: removing intermediary data structures no longer necessary. del self._graph_tracer @@ -118,6 +135,7 @@ def __init__( del self._nodes_name_remap del self._nodes_to_dcnnl_map del self._dynapcnn_layers + del self._flagged_input_nodes ####################################################### Public Methods ####################################################### @@ -135,9 +153,11 @@ def forward(self, x): """ Forwards data through the `DynapcnnNetwork` instance. This method relies on three main data structures created to represent the `DynapcnnLayer`s in the network and the data propagation through them during the forward pass: - - `self._dcnnl_edges` (list): this is used to guide the sequence in which the `DynapcnnLayer`s in `self._forward_map` are to be called - to generate the input tensors to be propagated through the network during the forward pass. This list of edges represent the graph - describing the interactions between each `DynapcnnLayer` (the nodes in the edges are the indices of these layers). + - `self._topological_order` (list): this is used to guide the sequence in which the `DynapcnnLayer`s in `self._forward_map` are to be called + to generate the input tensors to be propagated through the network during the forward pass. + - `self._dcnnl_edges` (list): this list of edges represent the graph describing the interactions between each `DynapcnnLayer` (the nodes in + the edges are the indices of these layers). An `edge` is used to index a mapper (using `edge[0]`) in order to retrieve the output to be fed + as input to a `DynapcnnLayer` instance (indexed by `edge[1]`). - `self._forward_map` (dict): a mapper used to forward data through the `DynapcnnNetwork` instances. Each `key` is the indice associated with a `DynapcnnLayer` instance. The edges in `self._dcnnl_edges` are accessed sequentially and each node in an edge is used to index a forward call via `self._forward_map`. @@ -147,57 +167,57 @@ def forward(self, x): layers_outputs = {} - # TODO - currently `node 0` (this 1st node in the 1st edge of `self._dcnnl_edges`) is always taken to be the - # input node of the network. This won't work in cases where there are more the one input nodes to the network - # so this functionality needs some refactoring. - self._forward_map[self._dcnnl_edges[0][0]](x) + print(self._topological_order) - # forward the input `x` through the input `DynapcnnLayer` in the `DynapcnnNetwork`s graph (1st node in the 1st edge in `self._dcnnl_edges`). - layers_outputs[self._dcnnl_edges[0][0]] = self._forward_map[self._dcnnl_edges[0][0]](x) + for i in self._topological_order: - # propagate outputs in `layers_outputs` through the rest of the nodes of `self._dcnnl_edges`. - for edge in self._dcnnl_edges: - - # target DynapcnnLayer (will consume tensors from `layers_outputs`). - trg_dcnnl = edge[1] + if self._forward_map[i].entry_point: + # `DynapcnnLayer i` is an entry point of the network. + layers_outputs[i] = self._forward_map[i](x) - if trg_dcnnl in self._merge_points and trg_dcnnl not in layers_outputs: - # by this points the arguments of the `Merge` associated with `trg_dcnnl` should have been computed. - arg1, arg2 = self._merge_points[trg_dcnnl]['sources'] + else: + # input to `DynapcnnLayer i` is the output of another instance. - # find which returned tensor from the `forward` call of DynapcnnLayers `arg1` and `arg2` are to be fed - # to the target DynapcnnLayer `trg_dcnnl`. - return_index_arg1 = self._forward_map[arg1].get_destination_dcnnl_index(trg_dcnnl) - return_index_arg2 = self._forward_map[arg2].get_destination_dcnnl_index(trg_dcnnl) + if i in self._merge_points and i not in layers_outputs: + # there are two sources of input for `DynapcnnLayer i`. - # retrieve input tensors to `Merge`. - _arg1 = layers_outputs[arg1][return_index_arg1] - _arg2 = layers_outputs[arg2][return_index_arg2] + # by this points the arguments of the `Merge` associated with `i` should have been computed due to the topological sorting. + arg1, arg2 = self._merge_points[i]['sources'] - # merge tensors. - merge_output = self._merge_points[trg_dcnnl]['merge'](_arg1, _arg2) + # find which returned tensor from the `forward` call of DynapcnnLayers `arg1` and `arg2` are to be fed + # to the target DynapcnnLayer `i`. + return_index_arg1 = self._forward_map[arg1].get_destination_dcnnl_index(i) + return_index_arg2 = self._forward_map[arg2].get_destination_dcnnl_index(i) - # call the forward. - layers_outputs[trg_dcnnl] = self._forward_map[trg_dcnnl](merge_output) + # retrieve input tensors to `Merge`. + _arg1 = layers_outputs[arg1][return_index_arg1] + _arg2 = layers_outputs[arg2][return_index_arg2] - elif trg_dcnnl not in layers_outputs: - # input source for `trg_dcnnl`. - src_dcnnl = edge[0] + # merge tensors. + merge_output = self._merge_points[i]['merge'](_arg1, _arg2) - # find which returned tensor from the `forward` call of the source DynapcnnLayer `src_dcnnl` is to be fed - # to the target DynapcnnLayer `trg_dcnnl`. - return_index = self._forward_map[src_dcnnl].get_destination_dcnnl_index(trg_dcnnl) + # call the forward. + layers_outputs[i] = self._forward_map[i](merge_output) - # call the forward. - layers_outputs[trg_dcnnl] = self._forward_map[trg_dcnnl](layers_outputs[src_dcnnl][return_index]) + elif i not in layers_outputs: + # there's a single source of input for `DynapcnnLayer i`. - else: + # input source for `i`. + src_dcnnl = self._get_input_to_dcnnl(i) - pass + # find which returned tensor from the `forward` call of the source DynapcnnLayer `src_dcnnl` is to be fed + # to the target DynapcnnLayer `i`. + return_index = self._forward_map[src_dcnnl].get_destination_dcnnl_index(i) + + # call the forward. + layers_outputs[i] = self._forward_map[i](layers_outputs[src_dcnnl][return_index]) + + else: + + pass # TODO - this assumes the network has a single output node. - # last computed is the output layer. - return layers_outputs[trg_dcnnl][0] + return layers_outputs[self._topological_order[-1]][0] def parameters(self) -> list: """ Gathers all the parameters of the network in a list. This is done by accessing the convolutional layer in each `DynapcnnLayer`, @@ -349,6 +369,13 @@ def to( ####################################################### Private Methods ####################################################### + def _get_input_to_dcnnl(self, dcnnl_ID) -> int: + """ . """ + for edge in self._dcnnl_edges: + if edge[1] == dcnnl_ID: + return edge[0] + raise ValueError(f'DynapcnnLayer {dcnnl_ID} has no source of input.') + def _make_config( self, chip_layers_ordering: Union[Sequence[int], str] = "auto", @@ -473,7 +500,7 @@ def _get_network_module(self) -> Union[list, dict, dict]: dcnnnet_module = DynapcnnNetworkModule(dcnnl_edges, self._dynapcnn_layers) - return dcnnnet_module.dcnnl_edges, dcnnnet_module.forward_map, dcnnnet_module.merge_points + return dcnnnet_module.dcnnl_edges, dcnnnet_module.forward_map, dcnnnet_module.merge_points, dcnnnet_module.get_topological_sort() def _get_dynapcnnlayers_edges(self) -> List[Tuple[int, int]]: """ Create edges representing connections between `DynapcnnLayer` instances. """ @@ -512,6 +539,12 @@ def _get_sinabs_edges_and_modules(self) -> Tuple[List[Tuple[int, int]], Dict[int # bypass merging layers to connect the nodes involved in them directly to the node where the merge happens. edges_without_merge = merge_handler(sinabs_edges, sinabs_modules_map) + # original nodes flagged as input nodes (i.e., entry points of the network) might have been renamed: update list flaggind them. + temp = [] + for i in range(len(self._flagged_input_nodes)): + temp.append(remapped_nodes[self._flagged_input_nodes[i]]) + self._flagged_input_nodes = temp + return edges_without_merge, sinabs_modules_map, remapped_nodes def _populate_nodes_io(self): diff --git a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py index 263bb60c..6c7ea596 100644 --- a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py +++ b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py @@ -5,6 +5,7 @@ from typing import List, Tuple, Dict, Union import copy import sinabs.layers as sl +from collections import defaultdict, deque from .dynapcnn_layer import DynapcnnLayer class DynapcnnNetworkModule(): @@ -24,6 +25,58 @@ def __init__(self, dcnnl_edges: List[Tuple[int, int]], dynapcnn_layers: Dict): self.dcnnl_edges = dcnnl_edges self.forward_map, self.merge_points = self._build_module_forward_from_graph(dcnnl_edges, dynapcnn_layers) + + self._add_entry_points_edges(dynapcnn_layers) + + def get_topological_sort(self) -> List[int]: + """ Performs a topological sorting (using Kahn's algorithm) of the edges describing the connectivity between + the `DynapcnnLayer` instances to organize the squencen they should be called during the `DynapcnnNetwork.forward` call. + + Parameters + ---------- + - topological_order (list): order in which each `DynapcnnLayer.forward` should be called. + """ + + graph = defaultdict(list) + in_degree = defaultdict(int) + + # initialize the graph and in-degrees. + for u, v in self.dcnnl_edges: + if u != 'input': + graph[u].append(v) + in_degree[v] += 1 + else: + if v not in in_degree: + in_degree[v] = 0 + if v not in in_degree: + in_degree[v] = 0 + + # find all nodes with zero in-degrees. + zero_in_degree_nodes = deque([node for node, degree in in_degree.items() if degree == 0]) + + # process nodes and create the topological order. + topological_order = [] + + while zero_in_degree_nodes: + node = zero_in_degree_nodes.popleft() + topological_order.append(node) + + for neighbor in graph[node]: + in_degree[neighbor] -= 1 + if in_degree[neighbor] == 0: + zero_in_degree_nodes.append(neighbor) + + # check if all nodes are processed (to handle cycles). + if len(topological_order) == len(in_degree): + return topological_order + + raise ValueError('The graph has a cycle and cannot be topologically sorted.') + + def _add_entry_points_edges(self, dynapcnn_layers: dict) -> None: + """ Addes an extra edge `('input', X)` to `self.dcnnl_edges` if `X` is an entry point of the `DynapcnnNetwork` (i.e., `dynapcnn_layers[X]['layer'].entry_point = True`).""" + for indx, dcnnl_data in dynapcnn_layers.items(): + if dcnnl_data['layer'].entry_point: + self.dcnnl_edges.append(('input', indx)) def _spot_merging_points(self, dcnnl_edges: list) -> Dict[int, Dict[Tuple, sl.Merge]]: """ Loops throught the edges of the computational graph from a `DynapcnnNetwork` to flag with nodes need diff --git a/sinabs/backend/dynapcnn/utils.py b/sinabs/backend/dynapcnn/utils.py index 3d04b143..b646a168 100644 --- a/sinabs/backend/dynapcnn/utils.py +++ b/sinabs/backend/dynapcnn/utils.py @@ -80,13 +80,13 @@ def build_nodes_to_dcnnl_map(layers: Dict[int, nn.Module], edges: List[Tuple[int Parameters --------- - layers (dict): constains the nodes of a graph as `key` and their associated module as `value`. - edges (list): edges describing how nodes connect to each other. + - layers (dict): constains the nodes of a graph as `key` and their associated module as `value`. + - edges (list): edges describing how nodes connect to each other. Returns --------- - nodes_to_dcnnl_map (dict): each entry represents the gathered data necessary to instantiate a `DynapcnnLayer` object (e.g. nodes, - their I/O shapes, the list of `DynapcnnLayer` that are to be targeted, etc). + - nodes_to_dcnnl_map (dict): each entry represents the gathered data necessary to instantiate a `DynapcnnLayer` object (e.g. nodes, + their I/O shapes, the list of `DynapcnnLayer` that are to be targeted, etc). """ # @TODO the graph extraction is not yet considering DVS input. @@ -118,7 +118,8 @@ def build_from_graph( discretize: bool, edges: List[Tuple[int, int]], nodes_to_dcnnl_map: dict, - weight_rescaling_fn: Callable) -> dict: + weight_rescaling_fn: Callable, + flagged_input_nodes: List[int]) -> dict: """ Parses each edge in `edges`, where each node is a set of layer that will compose a `DynapcnnLayer`. The target destination of each `DynapcnnLayer` is computed via edges connecting nodes in different `DynapcnnLayer` instances. @@ -131,6 +132,7 @@ def build_from_graph( their I/O shapes, the list of `DynapcnnLayer` that are to be targeted, etc). - weight_rescaling_fn (callable): a method that handles how the re-scaling factor for one or more `SumPool2d` projecting to the same convolutional layer are combined/re-scaled before applying them. + - flagged_input_nodes (list): node IDs corresponding to layers in the original network that are input nodes (i.e., a "point of entry" for the external data). Returns ---------- @@ -138,7 +140,7 @@ def build_from_graph( """ # turn each entry in `nodes_to_dcnnl_map` into a `DynapcnnLayer` instance. - dynapcnn_layers = construct_dynapcnnlayers_from_mapper(discretize, nodes_to_dcnnl_map, edges, weight_rescaling_fn) + dynapcnn_layers = construct_dynapcnnlayers_from_mapper(discretize, nodes_to_dcnnl_map, edges, weight_rescaling_fn, flagged_input_nodes) # initialize attribute holding to which core a `DynapcnnLayer` instance in `dynapcnn_layers` will be mapped to. for idx, layer_data in dynapcnn_layers.items(): @@ -152,7 +154,8 @@ def construct_dynapcnnlayers_from_mapper( discretize: bool, nodes_to_dcnnl_map: dict, edges: List[Tuple[int, int]], - weight_rescaling_fn: Callable) -> Dict[int, Dict[DynapcnnLayer, List]]: + weight_rescaling_fn: Callable, + flagged_input_nodes: List[int]) -> Dict[int, Dict[DynapcnnLayer, List]]: """ Consumes a dictionaries containing sets of layers to be used to populate a DynapcnnLayer object. Parameters @@ -163,6 +166,7 @@ def construct_dynapcnnlayers_from_mapper( - edges (list): edges describing how nodes connect to each other. - weight_rescaling_fn (callable): a method that handles how the re-scaling factor for one or more `SumPool2d` projecting to the same convolutional layer are combined/re-scaled before applying them. + - flagged_input_nodes (list): node IDs corresponding to layers in the original network that are input nodes (i.e., a "point of entry" for the external data). Returns ---------- @@ -174,7 +178,20 @@ def construct_dynapcnnlayers_from_mapper( for dpcnnl_idx, dcnnl_data in nodes_to_dcnnl_map.items(): # create a `DynapcnnLayer` from the set of layers in `dcnnl_data`. dynapcnnlayer = construct_dynapcnnlayer( - dpcnnl_idx, discretize, edges, nodes_to_dcnnl_map, weight_rescaling_fn) + dpcnnl_idx, discretize, edges, nodes_to_dcnnl_map, weight_rescaling_fn, flagged_input_nodes) + + # print('-----------------------------------------------------------------') + # print('dpcnnl_index: ', dynapcnnlayer.dpcnnl_index) + # print('conv_node_id: ', dynapcnnlayer.conv_node_id) + # print('conv_in_shape: ', dynapcnnlayer.conv_in_shape) + # print('conv_out_shape: ', dynapcnnlayer.conv_out_shape) + # print('spk_node_id: ', dynapcnnlayer.spk_node_id) + # print('pool_node_id: ', dynapcnnlayer.pool_node_id) + # print('conv_rescaling_factor: ', dynapcnnlayer.conv_rescaling_factor) + # print('dynapcnnlayer_destination: ', dynapcnnlayer.dynapcnnlayer_destination) + # print('nodes_destinations: ', dynapcnnlayer.nodes_destinations) + # print('entry_point: ', dynapcnnlayer.entry_point) + # print('-----------------------------------------------------------------') dynapcnn_layers[dpcnnl_idx] = { 'layer': dynapcnnlayer, @@ -221,7 +238,8 @@ def construct_dynapcnnlayer( discretize: bool, edges: List[Tuple[int, int]], nodes_to_dcnnl_map: Dict[int, Dict[Union[int, str], Union[Dict[str, Union[nn.Module, Tuple[int, int, int]]], List[int]]]], - weight_rescaling_fn: Callable) -> DynapcnnLayer: + weight_rescaling_fn: Callable, + flagged_input_nodes: List[int]) -> DynapcnnLayer: """ Extract the modules (layers) in a dictionary and uses them to instantiate a `DynapcnnLayer` object. Parameters @@ -238,6 +256,7 @@ def construct_dynapcnnlayer( integers corresponding to either destinations IDs or re-scaling factors). - weight_rescaling_fn (callable): a method that handles how the re-scaling factor for one or more `SumPool2d` projecting to the same convolutional layer are combined/re-scaled before being applied. + - flagged_input_nodes (list): node IDs corresponding to layers in the original network that are input nodes (i.e., a "point of entry" for the external data). Returns ---------- @@ -253,7 +272,8 @@ def construct_dynapcnnlayer( dcnnl_data = nodes_to_dcnnl_map[dpcnnl_idx], discretize = discretize, sinabs_edges = edges, - weight_rescaling_fn = weight_rescaling_fn + weight_rescaling_fn = weight_rescaling_fn, + flagged_input_nodes = flagged_input_nodes ) return dynapcnnlayer From 2ec4f5bed4dca4ea908260e4a051486c2650c09b Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Tue, 28 May 2024 12:11:13 +0200 Subject: [PATCH 105/379] Refactor - DynapcnnNetwork class using topological sorting of nodes (DynapcnnLayers) to guide its forward method. - NIRGraphExtractor class using topoligcal sorting of node (layer in the original network) to compute their respective I/O shapes. - Updated in-line documentation. --- sinabs/backend/dynapcnn/NIRGraphExtractor.py | 270 ++++++------------ sinabs/backend/dynapcnn/dynapcnn_layer.py | 16 +- sinabs/backend/dynapcnn/dynapcnn_network.py | 34 +-- .../dynapcnn/dynapcnnnetwork_module.py | 54 +--- sinabs/backend/dynapcnn/utils.py | 72 ++++- 5 files changed, 179 insertions(+), 267 deletions(-) diff --git a/sinabs/backend/dynapcnn/NIRGraphExtractor.py b/sinabs/backend/dynapcnn/NIRGraphExtractor.py index 4297824e..7ec5a0d7 100644 --- a/sinabs/backend/dynapcnn/NIRGraphExtractor.py +++ b/sinabs/backend/dynapcnn/NIRGraphExtractor.py @@ -1,9 +1,10 @@ -import torch +# author : Willian Soares Girao +# contact : wsoaresgirao@gmail.com + +import torch, sinabs, nirtorch, copy import torch.nn as nn -import nirtorch -import copy -import sinabs from typing import Tuple, Dict, List, Union +from .utils import topological_sorting class NIRtoDynapcnnNetworkGraph(): def __init__(self, spiking_model: nn.Module, dummy_input: torch.tensor): @@ -23,27 +24,30 @@ def __init__(self, spiking_model: nn.Module, dummy_input: torch.tensor): nir_graph = nirtorch.extract_torch_graph(spiking_model, dummy_input, model_name=None).ignore_tensors() # converts the NIR representation into a list of edges with nodes represented as integers. - self._edges_list, self._name_2_indx_map = self._get_edges_from_nir(nir_graph) + self._edges_list, self._name_2_indx_map, self._entry_nodes = self._get_edges_from_nir(nir_graph) - for key, val in self._name_2_indx_map.items(): - print(key, val) - print('---------------------------------------------------') - for edge in self._edges_list: - print(edge) - print('---------------------------------------------------') + # print('self._entry_nodes: ', self._entry_nodes) + + # for key, val in self._name_2_indx_map.items(): + # print(key, val) + # print('---------------------------------------------------') + # for edge in self._edges_list: + # print(edge) + # print('---------------------------------------------------') # recovers the associated `nn.Module` (layer) of each node. self.modules_map = self._get_named_modules(spiking_model) # retrieves what the I/O shape for each node's module is. - self._nodes_io_shapes, self._flagged_input_nodes = self._get_nodes_io_shapes(dummy_input) + self._nodes_io_shapes = self._get_nodes_io_shapes(dummy_input) - ### Publich Methods ### + ####################################################### Publich Methods ####################################################### @property - def flagged_input_nodes(self) -> List[int]: - return self._flagged_input_nodes + def entry_nodes(self) -> List[int]: + return self._entry_nodes + @property def get_edges_list(self): return self._edges_list @@ -97,43 +101,59 @@ def remove_ignored_nodes(self, default_ignored_nodes): return remapped_edges, remapped_nodes + # TODO - it would be good if I/O shapes were returned by the NIR graph. def get_node_io_shapes(self, node: int) -> Tuple[torch.Size, torch.Size]: """ Returns the I/O tensors' shapes of `node`. """ return self._nodes_io_shapes[node]['input'], self._nodes_io_shapes[node]['output'] - ### Pivate Methods ### + ####################################################### Pivate Methods ####################################################### - def _get_edges_from_nir(self, nir_graph: nirtorch.graph.Graph) -> Union[List[Tuple[int, int]], Dict[str, int]]: + def _get_edges_from_nir(self, nir_graph: nirtorch.graph.Graph) -> Tuple[List[Tuple[int, int]], Dict[str, int], List[int]]: """ Standardize the representation of `nirtorch.graph.Graph` into a list of edges (`Tuple[int, int]`) where each node in `nir_graph` is represented by an interger (with the source node starting as `0`). Parameters ---------- - nir_graph (nirtorch.graph.Graph): a NIR graph representation of `spiking_model`. + - nir_graph (nirtorch.graph.Graph): a NIR graph representation of `spiking_model`. Returns ---------- - edges_list (list): tuples describing the connections between layers in `spiking_model`. - name_2_indx_map (dict): `key` is the original variable name for a layer in `spiking_model` and `value - is an integer representing the layer in a standard format. + - edges_list (list): tuples describing the connections between layers in `spiking_model`. + - name_2_indx_map (dict): `key` is the original variable name for a layer in `spiking_model` and `value + is an integer representing the layer in a standard format. + - entry_nodes (list): IDs of nodes acting as entry points for the network (i.e., receiving external input). """ edges_list = [] name_2_indx_map = {} idx_counter = 0 # TODO maybe make sure the input node from nir always gets assined `0`. - for src_node in nir_graph.node_list: # source node. + nodes_IDs = [0] + + for src_node in nir_graph.node_list: + # source node. if src_node.name not in name_2_indx_map: name_2_indx_map[src_node.name] = idx_counter idx_counter += 1 - for trg_node in src_node.outgoing_nodes: # target node. + nodes_IDs.append(idx_counter) + + for trg_node in src_node.outgoing_nodes: + # target node. if trg_node.name not in name_2_indx_map: name_2_indx_map[trg_node.name] = idx_counter idx_counter += 1 + nodes_IDs.append(idx_counter) + edges_list.append((name_2_indx_map[src_node.name], name_2_indx_map[trg_node.name])) - return edges_list, name_2_indx_map + # finding entry/exits nodes of the graph. + all_sources = [x[0] for x in edges_list] + all_targets = [x[1] for x in edges_list] + + entry_nodes = list(set(all_sources) - set(all_targets)) + + return edges_list, name_2_indx_map, entry_nodes def _get_named_modules(self, model: nn.Module) -> Dict[int, nn.Module]: """ Find for each node in the graph what its associated layer in `model` is. @@ -165,10 +185,10 @@ def _get_named_modules(self, model: nn.Module) -> Dict[int, nn.Module]: return modules_map - # TODO - THIS ALSO NEEDS TOPOLOGICAL SORTING TO CORRECTLY GET I/O SHAPES UNDER ALL CIRCUNSTANCES. - def _get_nodes_io_shapes(self, input_dummy: torch.tensor) -> Tuple[Dict[int, Dict[str, torch.Size]], List]: - """ Loops through the graph represented in `self._edges_list` and propagates the inputs through the nodes, starting from - `node 0` fed `input_dummy`. + # TODO - it would be good if I/O shapes were returned by the NIR graph. + def _get_nodes_io_shapes(self, input_dummy: torch.tensor) -> Dict[int, Dict[str, torch.Size]]: + """ Iteratively calls the forward method of each `nn.Module` (i.e., a layer/node in the graph) using the topologically + sorted nodes extracted from the computational graph of the model being parsed. Parameters ---------- @@ -177,178 +197,63 @@ def _get_nodes_io_shapes(self, input_dummy: torch.tensor) -> Tuple[Dict[int, Dic Returns ---------- - nodes_io_map (dict): a dictionary mapping nodes to their I/O shapes. - - flagged_input_nodes (list): IDs of nodes that are receiving as input `input_dummy` (i.e., input nodes of the network). """ nodes_io_map = {} - flagged_merge_nodes = {} - flagged_input_nodes = [] - - # propagate inputs through the nodes. - for edge in self._edges_list: - src = edge[0] - trg = edge[1] - print('> ', edge) + # topological sorting of the graph. + temp_edges_list = copy.deepcopy(self._edges_list) + for node in self._entry_nodes: + temp_edges_list.append(('input', node)) + sorted_nodes = topological_sorting(temp_edges_list) - if isinstance(self.modules_map[src], sinabs.layers.merge.Merge): - # At this point the output of Merge has to have been calculated. - - # pass input through target. - if trg not in nodes_io_map: - nodes_io_map[trg] = {'input': None, 'output': None} + # propagate inputs through the nodes. + for node in sorted_nodes: - # find node generating the input to be used. - inp_node = self._find_source_of_input_to(trg) - _input = nodes_io_map[inp_node]['output'] + if isinstance(self.modules_map[node], sinabs.layers.merge.Merge): + # find `Merge` arguments (at this point the output of Merge has to have been calculated). + arg1, arg2 = self._find_merge_arguments(node) - # forward input through the node. - _output = self.modules_map[trg](_input) + # retrieve arguments output tensors. + arg1_out = nodes_io_map[arg1]['output'] + arg2_out = nodes_io_map[arg2]['output'] - # save node's input/output. - nodes_io_map[trg] = {'input': _input, 'output': _output} + # TODO - this is currently a limitation inpused by the validation checks done by Speck once a configuration: it wants two + # different input sources to a core to have the same output shapes. + if arg1_out.shape != arg2_out.shape: + raise ValueError(f'Layer `sinabs.layers.merge.Merge` (node {node}) require two input tensors with the same shape: arg1.shape {arg1_out.shape} differs from arg2.shape {arg2_out.shape}.') - elif isinstance(self.modules_map[trg], sinabs.layers.merge.Merge): - # Merge requires two inputs: need to check if both of its inputs have been calculated. - if trg not in flagged_merge_nodes: - flagged_merge_nodes[trg] = {} - - args = self._find_merge_arguments(trg) - - for arg in args: - if arg in nodes_io_map: - # one input to Merge has been computed. - flagged_merge_nodes[trg][arg] = nodes_io_map[arg] - - if len(flagged_merge_nodes[trg]) == 2: - # both arguments to Merge have been computed. - if trg not in nodes_io_map: - nodes_io_map[trg] = {'input': None, 'output': None} - - _output = self.modules_map[trg]( - nodes_io_map[args[0]]['output'], - nodes_io_map[args[1]]['output']) - - # Merge expands each input dim. into the max of that dim. between input tensors. - _input = torch.max(torch.stack([ - nodes_io_map[args[0]]['output'], - nodes_io_map[args[1]]['output']]), dim=0) - - nodes_io_map[trg]['input'] = _input.values - nodes_io_map[trg]['output'] = _output - - # pass input through source. - if src not in nodes_io_map: - nodes_io_map[src] = {'input': None, 'output': None} - - if src == 0: - # first node in the graph. - _input = input_dummy - - # flag node being an input node of the network. - if src not in flagged_input_nodes: - flagged_input_nodes.append(src) - - else: - # find node generating the input to be used. - inp_node = self._find_source_of_input_to(src) - - if inp_node == -1: - # `src` is receiving external (not from another layer) input. This will be the case when two - # parallel branches (two independent "input nodes" in the graph) merge at some point in the graph. - _input = input_dummy - - # flag node being an input node of the network. - if src not in flagged_input_nodes: - flagged_input_nodes.append(src) - else: - if isinstance(self.modules_map[inp_node], sinabs.layers.merge.Merge): - # source of input is a `Merge` layer that might still need to have its I/O shapes computed. - self._handle_merge_source(inp_node, nodes_io_map) - - print(f'accessing node {inp_node} cuz it is the input to node {src}....') - - # record what the input shape for `src` should be. - _input = nodes_io_map[inp_node]['output'] - - # forward input through the node. - _output = self.modules_map[src](_input) + # forward input through the node. + _output = self.modules_map[node](arg1_out, arg2_out) - # save node's input/output. - nodes_io_map[src] = {'input': _input, 'output': _output} + # save node's I/O tensors. + nodes_io_map[node] = {'input': arg1_out, 'output': _output} else: - # pass input through source. - if src not in nodes_io_map: - nodes_io_map[src] = {'input': None, 'output': None} - - if src == 0: - # first node in the graph. - _input = input_dummy - - # flag node being an input node of the network. - if src not in flagged_input_nodes: - flagged_input_nodes.append(src) - else: - # find node generating the input to be used. - inp_node = self._find_source_of_input_to(src) - - if inp_node == -1: - # `src` is receiving external (not from another layer) input. This will be the case when two - # parallel branches (two independent "input nodes" in the graph) merge at some point in the graph. - _input = input_dummy - - # flag node being an input node of the network. - if src not in flagged_input_nodes: - flagged_input_nodes.append(src) - else: - if isinstance(self.modules_map[inp_node], sinabs.layers.merge.Merge): - # source of input is a `Merge` layer that might still need to have its I/O shapes computed. - self._handle_merge_source(inp_node, nodes_io_map) - - # record what the input shape for `src` should be. - _input = nodes_io_map[inp_node]['output'] - - # forward input through the node. - _output = self.modules_map[src](_input) - # save node's input/output. - nodes_io_map[src] = {'input': _input, 'output': _output} + if node in self._entry_nodes: + # forward input dummy through node. + _output = self.modules_map[node](input_dummy) - # pass input through target. - if trg not in nodes_io_map: - nodes_io_map[trg] = {'input': None, 'output': None} + # save node's I/O tensors. + nodes_io_map[node] = {'input': input_dummy, 'output': _output} + else: # find node generating the input to be used. - inp_node = self._find_source_of_input_to(trg) - - if inp_node == -1: - # `src` is receiving external (not from another layer) input. This will be the case when two - # parallel branches (two independent "input nodes" in the graph) merge at some point in the graph. - _input = input_dummy - - # flag node being an input node of the network. - if trg not in flagged_input_nodes: - flagged_input_nodes.append(trg) - else: - if isinstance(self.modules_map[inp_node], sinabs.layers.merge.Merge): - # source of input is a `Merge` layer that might still need to have its I/O shapes computed. - self._handle_merge_source(inp_node, nodes_io_map) - - # record what the input shape for `trg` should be. - _input = nodes_io_map[inp_node]['output'] + input_node = self._find_source_of_input_to(node) + _input = nodes_io_map[input_node]['output'] # forward input through the node. - _output = self.modules_map[trg](_input) - - # save node's input/output. - nodes_io_map[trg] = {'input': _input, 'output': _output} + _output = self.modules_map[node](_input) + + # save node's I/O tensors. + nodes_io_map[node] = {'input': _input, 'output': _output} # replace the I/O tensor information by its shape information. for node, io in nodes_io_map.items(): nodes_io_map[node]['input'] = io['input'].shape nodes_io_map[node]['output'] = io['output'].shape - return nodes_io_map, flagged_input_nodes + return nodes_io_map def _handle_merge_source(self, merge_node_id: int, nodes_io_map: dict) -> None: """ This method finds the I/O shapes for node `merge_node_id` if they haven't been computed yet. When `self._find_source_of_input_to()` is @@ -401,12 +306,15 @@ def _find_source_of_input_to(self, node: int) -> int: return -1 - def _find_merge_arguments(self, merge_node: int) -> list: + def _find_merge_arguments(self, merge_node: int) -> Tuple[int, int]: """ A `Merge` layer receives two inputs. Return the two inputs to `merge_node` representing a `Merge` layer. """ args = [] + for edge in self._edges_list: if edge[1] == merge_node: args.append(edge[0]) - if len(args) == 2: - break - return args \ No newline at end of file + + if len(args) == 2: + return tuple(args) + else: + raise ValueError(f'Number of arguments found for `Merge` node {merge_node} is {len(args)} (should be 2).') \ No newline at end of file diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer.py b/sinabs/backend/dynapcnn/dynapcnn_layer.py index 2fffd045..891dbc94 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer.py @@ -28,7 +28,7 @@ class DynapcnnLayer(nn.Module): sequence of output tesnors its forward method needs to return. - weight_rescaling_fn (callable): a method that handles how the re-scaling factor for one or more `SumPool2d` projecting to the same convolutional layer are combined/re-scaled before applying them. - - flagged_input_nodes (list): node IDs corresponding to layers in the original network that are input nodes (i.e., a "point of entry" for the external data). + - entry_nodes (list): node IDs corresponding to layers in the original network that are input nodes (i.e., a "point of entry" for the external data). """ def __init__( @@ -38,7 +38,7 @@ def __init__( discretize: bool, sinabs_edges: List[Tuple[int, int]], weight_rescaling_fn: Callable, - flagged_input_nodes: List[int] + entry_nodes: List[int] ): super().__init__() @@ -95,9 +95,6 @@ def __init__( if len(list(spk.v_mem.shape)) != 4: spk.v_mem = spk.v_mem.data.unsqueeze(-1).unsqueeze(-1) # expand dims. - if self.dpcnnl_index == 5: - print(' **************88888888888888888888888 ', spk.v_mem.shape) - if isinstance(conv, nn.Linear): # A `nn.Linear` needs to be converted into `nn.Conv2d`. The I/O shapes of the spiking layer are updated # accordingly following the conversion. @@ -153,7 +150,7 @@ def __init__( self.nodes_destinations = self._get_destinations_input_source(sinabs_edges) # flag if the instance is an entry point (i.e., an input node of the network). - if self.conv_node_id in flagged_input_nodes: + if self.conv_node_id in entry_nodes: self.entry_point = True ####################################################### Public Methods ####################################################### @@ -201,9 +198,6 @@ def forward(self, x): x = self.conv_layer(x) x = self.spk_layer(x) - if self.dpcnnl_index == 5: - print('NEURON OUTPUT: ', x.shape) - # pooling layers are sequentially added to `self.pool_layer` so we'll pass data to them sequentially. pooling_indexer = 0 @@ -474,10 +468,6 @@ def _update_neuron_node_output_shape(self, spiking_layer_data: dict, conv_out_sh # spiking layer outputs the same shape as the conv. layer. spiking_layer_data['output_shape'] = spiking_layer_data['input_shape'] - - if self.dpcnnl_index == 5: - print('>>>>> ', spiking_layer_data['output_shape']) - def _update_conv_node_output_shape(self, conv_layer: nn.Conv2d, layer_data: dict, input_shape: Tuple[int, int, int]) -> Tuple[int, int, int]: """ Updates the shape of the output tensor of a node that used to be a `nn.Linear` and became a `nn.Conv2d`. diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index bf4c0aad..18ce6b16 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -33,6 +33,8 @@ from .dynapcnn_layer import DynapcnnLayer +from .utils import topological_sorting + class DynapcnnNetwork(nn.Module): def __init__( self, @@ -72,7 +74,7 @@ def __init__( - self._nodes_name_remap - self._nodes_to_dcnnl_map - self._dynapcnn_layers - - self._flagged_input_nodes + - self._entry_nodes """ super().__init__() @@ -90,8 +92,10 @@ def __init__( snn, torch.randn((batch_size, *self.input_shape))) # needs the batch dimension. - self._flagged_input_nodes = copy.deepcopy(self._graph_tracer.flagged_input_nodes) + # get list of nodes from graph tracer that act as entry points to the network. + self._entry_nodes = copy.deepcopy(self._graph_tracer.entry_nodes) + # pre-process and group original nodes/edges from the graph tracer into data structures used to later create `DynapcnnLayer` instance. self._sinabs_edges, \ self._sinabs_modules_map, \ self._nodes_name_remap = self._get_sinabs_edges_and_modules() @@ -120,7 +124,7 @@ def __init__( edges = self._sinabs_edges, nodes_to_dcnnl_map = self._nodes_to_dcnnl_map, weight_rescaling_fn = weight_rescaling_fn, - flagged_input_nodes = self._flagged_input_nodes) + entry_nodes = self._entry_nodes) # these gather all data necessay to implement the forward method for this class. self._dcnnl_edges, self._forward_map, self._merge_points, self._topological_order = self._get_network_module() @@ -135,7 +139,7 @@ def __init__( del self._nodes_name_remap del self._nodes_to_dcnnl_map del self._dynapcnn_layers - del self._flagged_input_nodes + del self._entry_nodes ####################################################### Public Methods ####################################################### @@ -167,8 +171,6 @@ def forward(self, x): layers_outputs = {} - print(self._topological_order) - for i in self._topological_order: if self._forward_map[i].entry_point: @@ -500,7 +502,7 @@ def _get_network_module(self) -> Union[list, dict, dict]: dcnnnet_module = DynapcnnNetworkModule(dcnnl_edges, self._dynapcnn_layers) - return dcnnnet_module.dcnnl_edges, dcnnnet_module.forward_map, dcnnnet_module.merge_points, dcnnnet_module.get_topological_sort() + return dcnnnet_module.dcnnl_edges, dcnnnet_module.forward_map, dcnnnet_module.merge_points, topological_sorting(dcnnl_edges) def _get_dynapcnnlayers_edges(self) -> List[Tuple[int, int]]: """ Create edges representing connections between `DynapcnnLayer` instances. """ @@ -519,12 +521,12 @@ def _get_sinabs_edges_and_modules(self) -> Tuple[List[Tuple[int, int]], Dict[int Returns ---------- - edges_without_merge (list): a list of edges based on `sinabs_edges` but where edges involving a `Merge` layer have been - remapped to connect the nodes involved in the merging directly. - sinabs_modules_map (dict): a dict containing the nodes of the graph (described now by `edges_without_merge`) as `key` and - their associated module as `value`. - remapped_nodes (dict): a dict where `key` is the original node name (as extracted by `self._graph_tracer`) and `value` is - the new node name (after ignored layers have been dropped and `Merge` layers have be processed before being removed). + - edges_without_merge (list): a list of edges based on `sinabs_edges` but where edges involving a `Merge` layer have been + remapped to connect the nodes involved in the merging directly. + - sinabs_modules_map (dict): a dict containing the nodes of the graph (described now by `edges_without_merge`) as `key` and + their associated module as `value`. + - remapped_nodes (dict): a dict where `key` is the original node name (as extracted by `self._graph_tracer`) and `value` is + the new node name (after ignored layers have been dropped and `Merge` layers have be processed before being removed). """ # remap `(A, X)` and `(X, B)` into `(A, B)` if `X` is a layer in the original `snn` to be ignored. @@ -541,9 +543,9 @@ def _get_sinabs_edges_and_modules(self) -> Tuple[List[Tuple[int, int]], Dict[int # original nodes flagged as input nodes (i.e., entry points of the network) might have been renamed: update list flaggind them. temp = [] - for i in range(len(self._flagged_input_nodes)): - temp.append(remapped_nodes[self._flagged_input_nodes[i]]) - self._flagged_input_nodes = temp + for i in range(len(self._entry_nodes)): + temp.append(remapped_nodes[self._entry_nodes[i]]) + self._entry_nodes = temp return edges_without_merge, sinabs_modules_map, remapped_nodes diff --git a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py index 6c7ea596..69df934c 100644 --- a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py +++ b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py @@ -1,11 +1,10 @@ # author : Willian Soares Girao -# contact : williansoaresgirao@gmail.com +# contact : wsoaresgirao@gmail.com import torch.nn as nn from typing import List, Tuple, Dict, Union import copy import sinabs.layers as sl -from collections import defaultdict, deque from .dynapcnn_layer import DynapcnnLayer class DynapcnnNetworkModule(): @@ -24,56 +23,21 @@ def __init__(self, dcnnl_edges: List[Tuple[int, int]], dynapcnn_layers: Dict): self.dcnnl_edges = dcnnl_edges + # create mappers to handle `DynapcnnLayer` instances' forward calling. self.forward_map, self.merge_points = self._build_module_forward_from_graph(dcnnl_edges, dynapcnn_layers) + # add extra edges marking which nodes are input to the network. self._add_entry_points_edges(dynapcnn_layers) - def get_topological_sort(self) -> List[int]: - """ Performs a topological sorting (using Kahn's algorithm) of the edges describing the connectivity between - the `DynapcnnLayer` instances to organize the squencen they should be called during the `DynapcnnNetwork.forward` call. + def _add_entry_points_edges(self, dynapcnn_layers: dict) -> None: + """ Addes an extra edge `('input', X)` to `self.dcnnl_edges` if `X` is an entry point of the `DynapcnnNetwork` + (i.e., `dynapcnn_layers[X]['layer'].entry_point = True`). Parameters ---------- - - topological_order (list): order in which each `DynapcnnLayer.forward` should be called. + - dynapcnn_layers (dict): a mapper containing `DynapcnnLayer` instances along with their supporting metadata (e.g. assigned core, + destination layers, etc.). """ - - graph = defaultdict(list) - in_degree = defaultdict(int) - - # initialize the graph and in-degrees. - for u, v in self.dcnnl_edges: - if u != 'input': - graph[u].append(v) - in_degree[v] += 1 - else: - if v not in in_degree: - in_degree[v] = 0 - if v not in in_degree: - in_degree[v] = 0 - - # find all nodes with zero in-degrees. - zero_in_degree_nodes = deque([node for node, degree in in_degree.items() if degree == 0]) - - # process nodes and create the topological order. - topological_order = [] - - while zero_in_degree_nodes: - node = zero_in_degree_nodes.popleft() - topological_order.append(node) - - for neighbor in graph[node]: - in_degree[neighbor] -= 1 - if in_degree[neighbor] == 0: - zero_in_degree_nodes.append(neighbor) - - # check if all nodes are processed (to handle cycles). - if len(topological_order) == len(in_degree): - return topological_order - - raise ValueError('The graph has a cycle and cannot be topologically sorted.') - - def _add_entry_points_edges(self, dynapcnn_layers: dict) -> None: - """ Addes an extra edge `('input', X)` to `self.dcnnl_edges` if `X` is an entry point of the `DynapcnnNetwork` (i.e., `dynapcnn_layers[X]['layer'].entry_point = True`).""" for indx, dcnnl_data in dynapcnn_layers.items(): if dcnnl_data['layer'].entry_point: self.dcnnl_edges.append(('input', indx)) @@ -115,7 +79,7 @@ def _build_module_forward_from_graph( self, dcnnl_edges: list, dynapcnn_layers: dict) -> Union[Dict[int, DynapcnnLayer], Dict[Tuple, sl.Merge]]: - """ Creates two mappers, one indexing each `DynapcnnLayer` by its index (a node in `dcnnl_edges`) a another + """ Creates two mappers, one indexing each `DynapcnnLayer` by its index (a node in `dcnnl_edges`) and another indexing the `DynapcnnLayer` instances (also by the index) that need their input being the output of a `Merge` layer (i.e., they are nodes in the graph where two different layer outputs converge to). diff --git a/sinabs/backend/dynapcnn/utils.py b/sinabs/backend/dynapcnn/utils.py index b646a168..75a2ee58 100644 --- a/sinabs/backend/dynapcnn/utils.py +++ b/sinabs/backend/dynapcnn/utils.py @@ -1,16 +1,16 @@ from copy import deepcopy -from typing import TYPE_CHECKING, List, Optional, Tuple, Type, Union, Dict, Callable +from typing import TYPE_CHECKING, List, Optional, Tuple, Union, Dict, Callable import torch import torch.nn as nn -import sinabs +from collections import defaultdict, deque import sinabs.layers as sl from .crop2d import Crop2d from .dvs_layer import DVSLayer, expand_to_pair from .dynapcnn_layer import DynapcnnLayer -from .exceptions import InputConfigurationError, MissingLayer, UnexpectedLayer, WrongPoolingModule +from .exceptions import WrongPoolingModule from .flipdims import FlipDims from .sinabs_edges_handler import process_edge, get_dynapcnnlayers_destinations @@ -119,7 +119,7 @@ def build_from_graph( edges: List[Tuple[int, int]], nodes_to_dcnnl_map: dict, weight_rescaling_fn: Callable, - flagged_input_nodes: List[int]) -> dict: + entry_nodes: List[int]) -> dict: """ Parses each edge in `edges`, where each node is a set of layer that will compose a `DynapcnnLayer`. The target destination of each `DynapcnnLayer` is computed via edges connecting nodes in different `DynapcnnLayer` instances. @@ -132,7 +132,7 @@ def build_from_graph( their I/O shapes, the list of `DynapcnnLayer` that are to be targeted, etc). - weight_rescaling_fn (callable): a method that handles how the re-scaling factor for one or more `SumPool2d` projecting to the same convolutional layer are combined/re-scaled before applying them. - - flagged_input_nodes (list): node IDs corresponding to layers in the original network that are input nodes (i.e., a "point of entry" for the external data). + - entry_nodes (list): node IDs corresponding to layers in the original network that are input nodes (i.e., a "point of entry" for the external data). Returns ---------- @@ -140,7 +140,7 @@ def build_from_graph( """ # turn each entry in `nodes_to_dcnnl_map` into a `DynapcnnLayer` instance. - dynapcnn_layers = construct_dynapcnnlayers_from_mapper(discretize, nodes_to_dcnnl_map, edges, weight_rescaling_fn, flagged_input_nodes) + dynapcnn_layers = construct_dynapcnnlayers_from_mapper(discretize, nodes_to_dcnnl_map, edges, weight_rescaling_fn, entry_nodes) # initialize attribute holding to which core a `DynapcnnLayer` instance in `dynapcnn_layers` will be mapped to. for idx, layer_data in dynapcnn_layers.items(): @@ -155,7 +155,7 @@ def construct_dynapcnnlayers_from_mapper( nodes_to_dcnnl_map: dict, edges: List[Tuple[int, int]], weight_rescaling_fn: Callable, - flagged_input_nodes: List[int]) -> Dict[int, Dict[DynapcnnLayer, List]]: + entry_nodes: List[int]) -> Dict[int, Dict[DynapcnnLayer, List]]: """ Consumes a dictionaries containing sets of layers to be used to populate a DynapcnnLayer object. Parameters @@ -166,7 +166,7 @@ def construct_dynapcnnlayers_from_mapper( - edges (list): edges describing how nodes connect to each other. - weight_rescaling_fn (callable): a method that handles how the re-scaling factor for one or more `SumPool2d` projecting to the same convolutional layer are combined/re-scaled before applying them. - - flagged_input_nodes (list): node IDs corresponding to layers in the original network that are input nodes (i.e., a "point of entry" for the external data). + - entry_nodes (list): node IDs corresponding to layers in the original network that are input nodes (i.e., a "point of entry" for the external data). Returns ---------- @@ -178,7 +178,7 @@ def construct_dynapcnnlayers_from_mapper( for dpcnnl_idx, dcnnl_data in nodes_to_dcnnl_map.items(): # create a `DynapcnnLayer` from the set of layers in `dcnnl_data`. dynapcnnlayer = construct_dynapcnnlayer( - dpcnnl_idx, discretize, edges, nodes_to_dcnnl_map, weight_rescaling_fn, flagged_input_nodes) + dpcnnl_idx, discretize, edges, nodes_to_dcnnl_map, weight_rescaling_fn, entry_nodes) # print('-----------------------------------------------------------------') # print('dpcnnl_index: ', dynapcnnlayer.dpcnnl_index) @@ -239,7 +239,7 @@ def construct_dynapcnnlayer( edges: List[Tuple[int, int]], nodes_to_dcnnl_map: Dict[int, Dict[Union[int, str], Union[Dict[str, Union[nn.Module, Tuple[int, int, int]]], List[int]]]], weight_rescaling_fn: Callable, - flagged_input_nodes: List[int]) -> DynapcnnLayer: + entry_nodes: List[int]) -> DynapcnnLayer: """ Extract the modules (layers) in a dictionary and uses them to instantiate a `DynapcnnLayer` object. Parameters @@ -256,7 +256,7 @@ def construct_dynapcnnlayer( integers corresponding to either destinations IDs or re-scaling factors). - weight_rescaling_fn (callable): a method that handles how the re-scaling factor for one or more `SumPool2d` projecting to the same convolutional layer are combined/re-scaled before being applied. - - flagged_input_nodes (list): node IDs corresponding to layers in the original network that are input nodes (i.e., a "point of entry" for the external data). + - entry_nodes (list): node IDs corresponding to layers in the original network that are input nodes (i.e., a "point of entry" for the external data). Returns ---------- @@ -273,7 +273,7 @@ def construct_dynapcnnlayer( discretize = discretize, sinabs_edges = edges, weight_rescaling_fn = weight_rescaling_fn, - flagged_input_nodes = flagged_input_nodes + entry_nodes = entry_nodes ) return dynapcnnlayer @@ -377,6 +377,54 @@ def build_SumPool2d(module: nn.AvgPool2d) -> Tuple[sl.SumPool2d, int]: return lyr_pool, rescale_factor +def topological_sorting(edges: List[Tuple[int, int]]) -> List[int]: + """ Performs a topological sorting (using Kahn's algorithm) of a graph descrobed by a list edges. An entry node `X` + of the graph have to be flagged inside `edges` by a tuple `('input', X)`. + + Parameters + ---------- + edges (list): the edges describing the *acyclic* graph. + + Returns + ---------- + - topological_order (list): the nodes sorted by the graph's topology. + """ + + graph = defaultdict(list) + in_degree = defaultdict(int) + + # initialize the graph and in-degrees. + for u, v in edges: + if u != 'input': + graph[u].append(v) + in_degree[v] += 1 + else: + if v not in in_degree: + in_degree[v] = 0 + if v not in in_degree: + in_degree[v] = 0 + + # find all nodes with zero in-degrees. + zero_in_degree_nodes = deque([node for node, degree in in_degree.items() if degree == 0]) + + # process nodes and create the topological order. + topological_order = [] + + while zero_in_degree_nodes: + node = zero_in_degree_nodes.popleft() + topological_order.append(node) + + for neighbor in graph[node]: + in_degree[neighbor] -= 1 + if in_degree[neighbor] == 0: + zero_in_degree_nodes.append(neighbor) + + # check if all nodes are processed (to handle cycles). + if len(topological_order) == len(in_degree): + return topological_order + + raise ValueError('The graph has a cycle and cannot be topologically sorted.') + ####################################################### MISSING FUNCTIONALITY ####################################################### # TODO: these methods are currently not used by the new implementation of DynapcnnNetwork (but should). From f2a56d7e36302016db4941a2aef79597f51b938d Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Tue, 28 May 2024 13:27:12 +0200 Subject: [PATCH 106/379] Unit testing - Validating gathering of individual layers into sets of layer to comprise DynapcnnLayer instances. - Validating I/O shapes for each individual layer. - Validating network's entry points. - Validating edges between layer comprising each DynpacnnLayer instance. - Valdating DynapcnnLayer instances and their respective configuration from sets of original layers. --- .../conftest_dynapcnnlayer.py | 401 ++---------------- tests/test_dynapcnnlayer/model_dummy_1.py | 166 ++++++++ tests/test_dynapcnnlayer/model_dummy_2.py | 211 +++++++++ tests/test_dynapcnnlayer/model_dummy_3.py | 287 +++++++++++++ tests/test_dynapcnnlayer/model_dummy_4.py | 207 +++++++++ .../test_dynapcnnlayer/test_dynapcnnlayer.py | 11 +- 6 files changed, 909 insertions(+), 374 deletions(-) create mode 100644 tests/test_dynapcnnlayer/model_dummy_1.py create mode 100644 tests/test_dynapcnnlayer/model_dummy_2.py create mode 100644 tests/test_dynapcnnlayer/model_dummy_3.py create mode 100644 tests/test_dynapcnnlayer/model_dummy_4.py diff --git a/tests/test_dynapcnnlayer/conftest_dynapcnnlayer.py b/tests/test_dynapcnnlayer/conftest_dynapcnnlayer.py index 4fd0682c..8e74e63d 100644 --- a/tests/test_dynapcnnlayer/conftest_dynapcnnlayer.py +++ b/tests/test_dynapcnnlayer/conftest_dynapcnnlayer.py @@ -1,373 +1,34 @@ -import torch.nn as nn -from sinabs.layers import IAFSqueeze, SumPool2d -from sinabs.activation.surrogate_gradient_fn import PeriodicExponential - -nodes_to_dcnnl_map_1 = { - 0: { - 0: { - 'layer': nn.Conv2d(2, 10, kernel_size=(2, 2), stride=(1, 1), bias=False), - 'input_shape': (2, 34, 34), - 'output_shape': (10, 33, 33) - }, - 1: { - 'layer': IAFSqueeze(batch_size=3, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), - 'input_shape': (10, 33, 33), - 'output_shape': (10, 33, 33) - }, - 2: { - 'layer': nn.AvgPool2d(kernel_size=3, stride=3, padding=0), - 'input_shape': (10, 33, 33), - 'output_shape': (10, 11, 11) - }, - 3: { - 'layer': nn.AvgPool2d(kernel_size=4, stride=4, padding=0), - 'input_shape': (10, 33, 33), - 'output_shape': (10, 8, 8) - }, - 'destinations': [1, 2], - 'conv_rescale_factor': [] - }, - 1: { - 4: { - 'layer': nn.Conv2d(10, 10, kernel_size=(4, 4), stride=(1, 1), bias=False), - 'input_shape': (10, 11, 11), - 'output_shape': (10, 8, 8) - }, - 6: { - 'layer': IAFSqueeze(batch_size=3, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), - 'input_shape': (10, 8, 8), - 'output_shape': (10, 8, 8) - }, - 'destinations': [2], - 'conv_rescale_factor': [9] - }, - 2: { - 7: { - 'layer': nn.Conv2d(10, 1, kernel_size=(2, 2), stride=(1, 1), bias=False), - 'input_shape': (10, 8, 8), - 'output_shape': (1, 7, 7) - }, - 8: { - 'layer': IAFSqueeze(batch_size=3, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), - 'input_shape': (1, 7, 7), - 'output_shape': (1, 7, 7) - }, - 'destinations': [3], - 'conv_rescale_factor': [16] - }, - 3: { - 9: { - 'layer': nn.Linear(in_features=49, out_features=500, bias=False), - 'input_shape': (1, 7, 7), - 'output_shape': (500,) - }, - 10: { - 'layer': IAFSqueeze(batch_size=3, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), - 'input_shape': (500,), - 'output_shape': (500,) - }, - 'destinations': [4], - 'conv_rescale_factor': [] - }, - 4: { - 11: { - 'layer': nn.Linear(in_features=500, out_features=10, bias=False), - 'input_shape': (500,), - 'output_shape': (10,) - }, - 12: { - 'layer': IAFSqueeze(batch_size=3, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), - 'input_shape': (10,), - 'output_shape': (10,) - }, - 'destinations': [], - 'conv_rescale_factor': [] - } -} - -sinabs_edges_1 = [ - (0, 1), - (1, 2), - (1, 3), - (2, 4), - (3, 7), - (4, 6), - (6, 7), - (7, 8), - (8, 9), - (9, 10), - (10, 11), - (11, 12), -] - -expected_output_1 = { - 0: { - 'dpcnnl_index': 0, - 'conv_node_id': 0, - 'conv_in_shape': (2, 34, 34), - 'conv_out_shape': (10, 33, 33), - 'spk_node_id': 1, - 'pool_node_id': [2, 3], - 'conv_rescaling_factor': None, - 'dynapcnnlayer_destination': [1, 2], - 'nodes_destinations': {2: [4], 3: [7]}, - }, - 1: { - 'dpcnnl_index': 1, - 'conv_node_id': 4, - 'conv_in_shape': (10, 11, 11), - 'conv_out_shape': (10, 8, 8), - 'spk_node_id': 6, - 'pool_node_id': [], - 'conv_rescaling_factor': 4.5, - 'dynapcnnlayer_destination': [2], - 'nodes_destinations': {6: [7]}, - }, - 2: { - 'dpcnnl_index': 2, - 'conv_node_id': 7, - 'conv_in_shape': (10, 8, 8), - 'conv_out_shape': (1, 7, 7), - 'spk_node_id': 8, - 'pool_node_id': [], - 'conv_rescaling_factor': 8.0, - 'dynapcnnlayer_destination': [3], - 'nodes_destinations': {8: [9]}, - }, - 3: { - 'dpcnnl_index': 3, - 'conv_node_id': 9, - 'conv_in_shape': (1, 7, 7), - 'conv_out_shape': (500, 1, 1), - 'spk_node_id': 10, - 'pool_node_id': [], - 'conv_rescaling_factor': None, - 'dynapcnnlayer_destination': [4], - 'nodes_destinations': {10: [11]}, - }, - 4: { - 'dpcnnl_index': 4, - 'conv_node_id': 11, - 'conv_in_shape': (500, 1, 1), - 'conv_out_shape': (10, 1, 1), - 'spk_node_id': 12, - 'pool_node_id': [], - 'conv_rescaling_factor': None, - 'dynapcnnlayer_destination': [], - 'nodes_destinations': {}, - }, -} - -nodes_to_dcnnl_map_2 = { - 0: { - 0: {'layer': nn.Conv2d(2, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), - 'input_shape': (2, 34, 34), - 'output_shape': (4, 33, 33) - }, - 1: {'layer': IAFSqueeze(batch_size=8, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), - 'input_shape': (4, 33, 33), - 'output_shape': (4, 33, 33) - }, - 'destinations': [1], - 'conv_rescale_factor': [] - }, - 1: { - 2: {'layer': nn.Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), - 'input_shape': (4, 33, 33), - 'output_shape': (4, 32, 32) - }, - 3: {'layer': IAFSqueeze(batch_size=8, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), - 'input_shape': (4, 32, 32), - 'output_shape': (4, 32, 32) - }, - 4: {'layer': SumPool2d(kernel_size=2, stride=2, ceil_mode=False), - 'input_shape': (4, 32, 32), - 'output_shape': (4, 16, 16) - }, - 'destinations': [2, 3], - 'conv_rescale_factor': [] - }, - 2: { - 5: {'layer': nn.Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), - 'input_shape': (4, 16, 16), - 'output_shape': (4, 15, 15) - }, - 7: {'layer': IAFSqueeze(batch_size=8, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), - 'input_shape': (4, 15, 15), - 'output_shape': (4, 15, 15) - }, - 8: {'layer': SumPool2d(kernel_size=2, stride=2, ceil_mode=False), - 'input_shape': (4, 15, 15), - 'output_shape': (4, 7, 7) - }, - 'destinations': [4], - 'conv_rescale_factor': [] - }, - 3: { - 6: {'layer': nn.Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), - 'input_shape': (4, 16, 16), - 'output_shape': (4, 15, 15) - }, - 11: {'layer': IAFSqueeze(batch_size=8, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), - 'input_shape': (4, 15, 15), - 'output_shape': (4, 15, 15) - }, - 12: {'layer': SumPool2d(kernel_size=2, stride=2, ceil_mode=False), - 'input_shape': (4, 15, 15), - 'output_shape': (4, 7, 7) - }, - 'destinations': [6], - 'conv_rescale_factor': [] - }, - 4: { - 9: {'layer': nn.Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), - 'input_shape': (4, 7, 7), - 'output_shape': (4, 6, 6) - }, - 10: {'layer': IAFSqueeze(batch_size=8, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), - 'input_shape': (4, 6, 6), - 'output_shape': (4, 6, 6) - }, - 'destinations': [5], - 'conv_rescale_factor': [] - }, - 5 :{ - 15: {'layer': nn.Linear(in_features=144, out_features=10, bias=False), - 'input_shape': (4, 6, 6), - 'output_shape': (10,) - }, - 16: {'layer': IAFSqueeze(batch_size=8, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), - 'input_shape': (10,), - 'output_shape': (10,) - }, - 'destinations': [], - 'conv_rescale_factor': [] - }, - 6: { - 13: {'layer': nn.Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), - 'input_shape': (4, 7, 7), - 'output_shape': (4, 6, 6) - }, - 14: {'layer': IAFSqueeze(batch_size=8, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), - 'input_shape': (4, 6, 6), - 'output_shape': (4, 6, 6) - }, - 'destinations': [5], - 'conv_rescale_factor': [] - } -} - -sinabs_edges_2 = [ - (0, 1), - (1, 2), - (2, 3), - (3, 4), - (4, 5), - (4, 6), - (5, 7), - (7, 8), - (8, 9), - (9, 10), - (10, 15), - (6, 11), - (11, 12), - (12, 13), - (13, 14), - (14, 15), - (15, 16), -] - -expected_output_2 = { - 0: { - 'dpcnnl_index': 0, - 'conv_node_id': 0, - 'conv_in_shape': (2, 34, 34), - 'conv_out_shape': (4, 33, 33), - 'spk_node_id': 1, - 'pool_node_id': [], - 'conv_rescaling_factor': None, - 'dynapcnnlayer_destination': [1], - 'nodes_destinations': {1: [2]}, - }, - 1: { - 'dpcnnl_index': 1, - 'conv_node_id': 2, - 'conv_in_shape': (4, 33, 33), - 'conv_out_shape': (4, 32, 32), - 'spk_node_id': 3, - 'pool_node_id': [4], - 'conv_rescaling_factor': None, - 'dynapcnnlayer_destination': [2, 3], - 'nodes_destinations': {4: [5, 6]}, - }, - 2: { - 'dpcnnl_index': 2, - 'conv_node_id': 5, - 'conv_in_shape': (4, 16, 16), - 'conv_out_shape': (4, 15, 15), - 'spk_node_id': 7, - 'pool_node_id': [8], - 'conv_rescaling_factor': None, - 'dynapcnnlayer_destination': [4], - 'nodes_destinations': {8: [9]}, - }, - 3: { - 'dpcnnl_index': 3, - 'conv_node_id': 6, - 'conv_in_shape': (4, 16, 16), - 'conv_out_shape': (4, 15, 15), - 'spk_node_id': 11, - 'pool_node_id': [12], - 'conv_rescaling_factor': None, - 'dynapcnnlayer_destination': [6], - 'nodes_destinations': {12: [13]}, - }, - 4: { - 'dpcnnl_index': 4, - 'conv_node_id': 9, - 'conv_in_shape': (4, 7, 7), - 'conv_out_shape': (4, 6, 6), - 'spk_node_id': 10, - 'pool_node_id': [], - 'conv_rescaling_factor': None, - 'dynapcnnlayer_destination': [5], - 'nodes_destinations': {10: [15]}, - }, - 5: { - 'dpcnnl_index': 5, - 'conv_node_id': 15, - 'conv_in_shape': (4, 6, 6), - 'conv_out_shape': (10, 1, 1), - 'spk_node_id': 16, - 'pool_node_id': [], - 'conv_rescaling_factor': None, - 'dynapcnnlayer_destination': [], - 'nodes_destinations': {}, - }, - 6: { - 'dpcnnl_index': 6, - 'conv_node_id': 13, - 'conv_in_shape': (4, 7, 7), - 'conv_out_shape': (4, 6, 6), - 'spk_node_id': 14, - 'pool_node_id': [], - 'conv_rescaling_factor': None, - 'dynapcnnlayer_destination': [5], - 'nodes_destinations': {14: [15]}, - }, -} +from model_dummy_1 import nodes_to_dcnnl_map_1, sinabs_edges_1, expected_output_1 +from model_dummy_2 import nodes_to_dcnnl_map_2, sinabs_edges_2, expected_output_2 +from model_dummy_3 import nodes_to_dcnnl_map_3, sinabs_edges_3, expected_output_3 +from model_dummy_4 import nodes_to_dcnnl_map_4, sinabs_edges_4, expected_output_4 args_DynapcnnLayer = [ - (nodes_to_dcnnl_map_1, 0, sinabs_edges_1, expected_output_1), - (nodes_to_dcnnl_map_1, 1, sinabs_edges_1, expected_output_1), - (nodes_to_dcnnl_map_1, 2, sinabs_edges_1, expected_output_1), - (nodes_to_dcnnl_map_1, 3, sinabs_edges_1, expected_output_1), - (nodes_to_dcnnl_map_1, 4, sinabs_edges_1, expected_output_1), - (nodes_to_dcnnl_map_2, 0, sinabs_edges_2, expected_output_2), - (nodes_to_dcnnl_map_2, 1, sinabs_edges_2, expected_output_2), - (nodes_to_dcnnl_map_2, 2, sinabs_edges_2, expected_output_2), - (nodes_to_dcnnl_map_2, 3, sinabs_edges_2, expected_output_2), - (nodes_to_dcnnl_map_2, 4, sinabs_edges_2, expected_output_2), - (nodes_to_dcnnl_map_2, 5, sinabs_edges_2, expected_output_2), - (nodes_to_dcnnl_map_2, 6, sinabs_edges_2, expected_output_2), + (nodes_to_dcnnl_map_1, 0, sinabs_edges_1, [0], expected_output_1), + (nodes_to_dcnnl_map_1, 1, sinabs_edges_1, [0], expected_output_1), + (nodes_to_dcnnl_map_1, 2, sinabs_edges_1, [0], expected_output_1), + (nodes_to_dcnnl_map_1, 3, sinabs_edges_1, [0], expected_output_1), + (nodes_to_dcnnl_map_1, 4, sinabs_edges_1, [0], expected_output_1), + (nodes_to_dcnnl_map_2, 0, sinabs_edges_2, [0], expected_output_2), + (nodes_to_dcnnl_map_2, 1, sinabs_edges_2, [0], expected_output_2), + (nodes_to_dcnnl_map_2, 2, sinabs_edges_2, [0], expected_output_2), + (nodes_to_dcnnl_map_2, 3, sinabs_edges_2, [0], expected_output_2), + (nodes_to_dcnnl_map_2, 4, sinabs_edges_2, [0], expected_output_2), + (nodes_to_dcnnl_map_2, 5, sinabs_edges_2, [0], expected_output_2), + (nodes_to_dcnnl_map_2, 6, sinabs_edges_2, [0], expected_output_2), + (nodes_to_dcnnl_map_3, 0, sinabs_edges_3, [0, 8], expected_output_3), + (nodes_to_dcnnl_map_3, 1, sinabs_edges_3, [0, 8], expected_output_3), + (nodes_to_dcnnl_map_3, 2, sinabs_edges_3, [0, 8], expected_output_3), + (nodes_to_dcnnl_map_3, 3, sinabs_edges_3, [0, 8], expected_output_3), + (nodes_to_dcnnl_map_3, 4, sinabs_edges_3, [0, 8], expected_output_3), + (nodes_to_dcnnl_map_3, 5, sinabs_edges_3, [0, 8], expected_output_3), + (nodes_to_dcnnl_map_3, 6, sinabs_edges_3, [0, 8], expected_output_3), + (nodes_to_dcnnl_map_3, 7, sinabs_edges_3, [0, 8], expected_output_3), + (nodes_to_dcnnl_map_3, 8, sinabs_edges_3, [0, 8], expected_output_3), + (nodes_to_dcnnl_map_4, 0, sinabs_edges_4, [0], expected_output_4), + (nodes_to_dcnnl_map_4, 1, sinabs_edges_4, [0], expected_output_4), + (nodes_to_dcnnl_map_4, 2, sinabs_edges_4, [0], expected_output_4), + (nodes_to_dcnnl_map_4, 3, sinabs_edges_4, [0], expected_output_4), + (nodes_to_dcnnl_map_4, 4, sinabs_edges_4, [0], expected_output_4), + (nodes_to_dcnnl_map_4, 5, sinabs_edges_4, [0], expected_output_4), ] \ No newline at end of file diff --git a/tests/test_dynapcnnlayer/model_dummy_1.py b/tests/test_dynapcnnlayer/model_dummy_1.py new file mode 100644 index 00000000..65989250 --- /dev/null +++ b/tests/test_dynapcnnlayer/model_dummy_1.py @@ -0,0 +1,166 @@ +# implementing "a network with residual connections" example in https://github.com/synsense/sinabs/issues/181 . """ + +import torch.nn as nn +from sinabs.layers import IAFSqueeze, SumPool2d +from sinabs.activation.surrogate_gradient_fn import PeriodicExponential + +nodes_to_dcnnl_map_1 = { + 0: { + 0: { + 'layer': nn.Conv2d(2, 10, kernel_size=(2, 2), stride=(1, 1), bias=False), + 'input_shape': (2, 34, 34), + 'output_shape': (10, 33, 33) + }, + 1: { + 'layer': IAFSqueeze(batch_size=3, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), + 'input_shape': (10, 33, 33), + 'output_shape': (10, 33, 33) + }, + 2: { + 'layer': nn.AvgPool2d(kernel_size=3, stride=3, padding=0), + 'input_shape': (10, 33, 33), + 'output_shape': (10, 11, 11) + }, + 3: { + 'layer': nn.AvgPool2d(kernel_size=4, stride=4, padding=0), + 'input_shape': (10, 33, 33), + 'output_shape': (10, 8, 8) + }, + 'destinations': [1, 2], + 'conv_rescale_factor': [] + }, + 1: { + 4: { + 'layer': nn.Conv2d(10, 10, kernel_size=(4, 4), stride=(1, 1), bias=False), + 'input_shape': (10, 11, 11), + 'output_shape': (10, 8, 8) + }, + 6: { + 'layer': IAFSqueeze(batch_size=3, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), + 'input_shape': (10, 8, 8), + 'output_shape': (10, 8, 8) + }, + 'destinations': [2], + 'conv_rescale_factor': [9] + }, + 2: { + 7: { + 'layer': nn.Conv2d(10, 1, kernel_size=(2, 2), stride=(1, 1), bias=False), + 'input_shape': (10, 8, 8), + 'output_shape': (1, 7, 7) + }, + 8: { + 'layer': IAFSqueeze(batch_size=3, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), + 'input_shape': (1, 7, 7), + 'output_shape': (1, 7, 7) + }, + 'destinations': [3], + 'conv_rescale_factor': [16] + }, + 3: { + 9: { + 'layer': nn.Linear(in_features=49, out_features=500, bias=False), + 'input_shape': (1, 7, 7), + 'output_shape': (500,) + }, + 10: { + 'layer': IAFSqueeze(batch_size=3, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), + 'input_shape': (500,), + 'output_shape': (500,) + }, + 'destinations': [4], + 'conv_rescale_factor': [] + }, + 4: { + 11: { + 'layer': nn.Linear(in_features=500, out_features=10, bias=False), + 'input_shape': (500,), + 'output_shape': (10,) + }, + 12: { + 'layer': IAFSqueeze(batch_size=3, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), + 'input_shape': (10,), + 'output_shape': (10,) + }, + 'destinations': [], + 'conv_rescale_factor': [] + } +} + +sinabs_edges_1 = [ + (0, 1), + (1, 2), + (1, 3), + (2, 4), + (3, 7), + (4, 6), + (6, 7), + (7, 8), + (8, 9), + (9, 10), + (10, 11), + (11, 12), +] + +expected_output_1 = { + 0: { + 'dpcnnl_index': 0, + 'conv_node_id': 0, + 'conv_in_shape': (2, 34, 34), + 'conv_out_shape': (10, 33, 33), + 'spk_node_id': 1, + 'pool_node_id': [2, 3], + 'conv_rescaling_factor': None, + 'dynapcnnlayer_destination': [1, 2], + 'nodes_destinations': {2: [4], 3: [7]}, + 'entry_point': True, + }, + 1: { + 'dpcnnl_index': 1, + 'conv_node_id': 4, + 'conv_in_shape': (10, 11, 11), + 'conv_out_shape': (10, 8, 8), + 'spk_node_id': 6, + 'pool_node_id': [], + 'conv_rescaling_factor': 4.5, + 'dynapcnnlayer_destination': [2], + 'nodes_destinations': {6: [7]}, + 'entry_point': False, + }, + 2: { + 'dpcnnl_index': 2, + 'conv_node_id': 7, + 'conv_in_shape': (10, 8, 8), + 'conv_out_shape': (1, 7, 7), + 'spk_node_id': 8, + 'pool_node_id': [], + 'conv_rescaling_factor': 8.0, + 'dynapcnnlayer_destination': [3], + 'nodes_destinations': {8: [9]}, + 'entry_point': False, + }, + 3: { + 'dpcnnl_index': 3, + 'conv_node_id': 9, + 'conv_in_shape': (1, 7, 7), + 'conv_out_shape': (500, 1, 1), + 'spk_node_id': 10, + 'pool_node_id': [], + 'conv_rescaling_factor': None, + 'dynapcnnlayer_destination': [4], + 'nodes_destinations': {10: [11]}, + 'entry_point': False, + }, + 4: { + 'dpcnnl_index': 4, + 'conv_node_id': 11, + 'conv_in_shape': (500, 1, 1), + 'conv_out_shape': (10, 1, 1), + 'spk_node_id': 12, + 'pool_node_id': [], + 'conv_rescaling_factor': None, + 'dynapcnnlayer_destination': [], + 'nodes_destinations': {}, + 'entry_point': False, + }, +} \ No newline at end of file diff --git a/tests/test_dynapcnnlayer/model_dummy_2.py b/tests/test_dynapcnnlayer/model_dummy_2.py new file mode 100644 index 00000000..9570c2f1 --- /dev/null +++ b/tests/test_dynapcnnlayer/model_dummy_2.py @@ -0,0 +1,211 @@ +# implementing "a network with a merge and a split" in https://github.com/synsense/sinabs/issues/181 + +import torch.nn as nn +from sinabs.layers import IAFSqueeze, SumPool2d +from sinabs.activation.surrogate_gradient_fn import PeriodicExponential + +nodes_to_dcnnl_map_2 = { + 0: { + 0: {'layer': nn.Conv2d(2, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), + 'input_shape': (2, 34, 34), + 'output_shape': (4, 33, 33) + }, + 1: {'layer': IAFSqueeze(batch_size=8, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), + 'input_shape': (4, 33, 33), + 'output_shape': (4, 33, 33) + }, + 'destinations': [1], + 'conv_rescale_factor': [] + }, + 1: { + 2: {'layer': nn.Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), + 'input_shape': (4, 33, 33), + 'output_shape': (4, 32, 32) + }, + 3: {'layer': IAFSqueeze(batch_size=8, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), + 'input_shape': (4, 32, 32), + 'output_shape': (4, 32, 32) + }, + 4: {'layer': SumPool2d(kernel_size=2, stride=2, ceil_mode=False), + 'input_shape': (4, 32, 32), + 'output_shape': (4, 16, 16) + }, + 'destinations': [2, 3], + 'conv_rescale_factor': [] + }, + 2: { + 5: {'layer': nn.Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), + 'input_shape': (4, 16, 16), + 'output_shape': (4, 15, 15) + }, + 7: {'layer': IAFSqueeze(batch_size=8, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), + 'input_shape': (4, 15, 15), + 'output_shape': (4, 15, 15) + }, + 8: {'layer': SumPool2d(kernel_size=2, stride=2, ceil_mode=False), + 'input_shape': (4, 15, 15), + 'output_shape': (4, 7, 7) + }, + 'destinations': [4], + 'conv_rescale_factor': [] + }, + 3: { + 6: {'layer': nn.Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), + 'input_shape': (4, 16, 16), + 'output_shape': (4, 15, 15) + }, + 11: {'layer': IAFSqueeze(batch_size=8, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), + 'input_shape': (4, 15, 15), + 'output_shape': (4, 15, 15) + }, + 12: {'layer': SumPool2d(kernel_size=2, stride=2, ceil_mode=False), + 'input_shape': (4, 15, 15), + 'output_shape': (4, 7, 7) + }, + 'destinations': [6], + 'conv_rescale_factor': [] + }, + 4: { + 9: {'layer': nn.Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), + 'input_shape': (4, 7, 7), + 'output_shape': (4, 6, 6) + }, + 10: {'layer': IAFSqueeze(batch_size=8, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), + 'input_shape': (4, 6, 6), + 'output_shape': (4, 6, 6) + }, + 'destinations': [5], + 'conv_rescale_factor': [] + }, + 5 :{ + 15: {'layer': nn.Linear(in_features=144, out_features=10, bias=False), + 'input_shape': (4, 6, 6), + 'output_shape': (10,) + }, + 16: {'layer': IAFSqueeze(batch_size=8, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), + 'input_shape': (10,), + 'output_shape': (10,) + }, + 'destinations': [], + 'conv_rescale_factor': [] + }, + 6: { + 13: {'layer': nn.Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), + 'input_shape': (4, 7, 7), + 'output_shape': (4, 6, 6) + }, + 14: {'layer': IAFSqueeze(batch_size=8, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), + 'input_shape': (4, 6, 6), + 'output_shape': (4, 6, 6) + }, + 'destinations': [5], + 'conv_rescale_factor': [] + } +} + +sinabs_edges_2 = [ + (0, 1), + (1, 2), + (2, 3), + (3, 4), + (4, 5), + (4, 6), + (5, 7), + (7, 8), + (8, 9), + (9, 10), + (10, 15), + (6, 11), + (11, 12), + (12, 13), + (13, 14), + (14, 15), + (15, 16), +] + +expected_output_2 = { + 0: { + 'dpcnnl_index': 0, + 'conv_node_id': 0, + 'conv_in_shape': (2, 34, 34), + 'conv_out_shape': (4, 33, 33), + 'spk_node_id': 1, + 'pool_node_id': [], + 'conv_rescaling_factor': None, + 'dynapcnnlayer_destination': [1], + 'nodes_destinations': {1: [2]}, + 'entry_point': True, + }, + 1: { + 'dpcnnl_index': 1, + 'conv_node_id': 2, + 'conv_in_shape': (4, 33, 33), + 'conv_out_shape': (4, 32, 32), + 'spk_node_id': 3, + 'pool_node_id': [4], + 'conv_rescaling_factor': None, + 'dynapcnnlayer_destination': [2, 3], + 'nodes_destinations': {4: [5, 6]}, + 'entry_point': False, + }, + 2: { + 'dpcnnl_index': 2, + 'conv_node_id': 5, + 'conv_in_shape': (4, 16, 16), + 'conv_out_shape': (4, 15, 15), + 'spk_node_id': 7, + 'pool_node_id': [8], + 'conv_rescaling_factor': None, + 'dynapcnnlayer_destination': [4], + 'nodes_destinations': {8: [9]}, + 'entry_point': False, + }, + 3: { + 'dpcnnl_index': 3, + 'conv_node_id': 6, + 'conv_in_shape': (4, 16, 16), + 'conv_out_shape': (4, 15, 15), + 'spk_node_id': 11, + 'pool_node_id': [12], + 'conv_rescaling_factor': None, + 'dynapcnnlayer_destination': [6], + 'nodes_destinations': {12: [13]}, + 'entry_point': False, + }, + 4: { + 'dpcnnl_index': 4, + 'conv_node_id': 9, + 'conv_in_shape': (4, 7, 7), + 'conv_out_shape': (4, 6, 6), + 'spk_node_id': 10, + 'pool_node_id': [], + 'conv_rescaling_factor': None, + 'dynapcnnlayer_destination': [5], + 'nodes_destinations': {10: [15]}, + 'entry_point': False, + }, + 5: { + 'dpcnnl_index': 5, + 'conv_node_id': 15, + 'conv_in_shape': (4, 6, 6), + 'conv_out_shape': (10, 1, 1), + 'spk_node_id': 16, + 'pool_node_id': [], + 'conv_rescaling_factor': None, + 'dynapcnnlayer_destination': [], + 'nodes_destinations': {}, + 'entry_point': False, + }, + 6: { + 'dpcnnl_index': 6, + 'conv_node_id': 13, + 'conv_in_shape': (4, 7, 7), + 'conv_out_shape': (4, 6, 6), + 'spk_node_id': 14, + 'pool_node_id': [], + 'conv_rescaling_factor': None, + 'dynapcnnlayer_destination': [5], + 'nodes_destinations': {14: [15]}, + 'entry_point': False, + }, +} \ No newline at end of file diff --git a/tests/test_dynapcnnlayer/model_dummy_3.py b/tests/test_dynapcnnlayer/model_dummy_3.py new file mode 100644 index 00000000..2d541751 --- /dev/null +++ b/tests/test_dynapcnnlayer/model_dummy_3.py @@ -0,0 +1,287 @@ +# implementing "two networks with merging outputs" in https://github.com/synsense/sinabs/issues/181 + +import torch.nn as nn +from sinabs.layers import IAFSqueeze, SumPool2d +from sinabs.activation.surrogate_gradient_fn import PeriodicExponential + +nodes_to_dcnnl_map_3 = { + 0: { + 0: { + 'layer': nn.Conv2d(2, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), + 'input_shape': (2, 34, 34), + 'output_shape': (4, 33, 33) + }, + 1: { + 'layer': IAFSqueeze(batch_size=2, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), + 'input_shape': (4, 33, 33), + 'output_shape': (4, 33, 33) + }, + 'destinations': [1], + 'conv_rescale_factor': [] + }, + 1: { + 2: { + 'layer': nn.Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), + 'input_shape': (4, 33, 33), + 'output_shape': (4, 32, 32) + }, + 3: { + 'layer': IAFSqueeze(batch_size=2, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), + 'input_shape': (4, 32, 32), + 'output_shape': (4, 32, 32) + }, + 4: { + 'layer': SumPool2d(kernel_size=2, stride=2, ceil_mode=False), + 'input_shape': (4, 32, 32), + 'output_shape': (4, 16, 16) + }, + 'destinations': [2], + 'conv_rescale_factor': [] + }, + 2: { + 5: { + 'layer': nn.Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), + 'input_shape': (4, 16, 16), + 'output_shape': (4, 15, 15)}, + 6: { + 'layer': IAFSqueeze(batch_size=2, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), + 'input_shape': (4, 15, 15), + 'output_shape': (4, 15, 15) + }, + 7: { + 'layer': SumPool2d(kernel_size=2, stride=2, ceil_mode=False), + 'input_shape': (4, 15, 15), + 'output_shape': (4, 7, 7) + }, + 'destinations': [3], + 'conv_rescale_factor': [] + }, + 3: { + 17: { + 'layer': nn.Linear(in_features=196, out_features=100, bias=False), + 'input_shape': (4, 7, 7), + 'output_shape': (100,) + }, + 18: { + 'layer': IAFSqueeze(batch_size=2, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), + 'input_shape': (100,), + 'output_shape': (100,) + }, + 'destinations': [7], + 'conv_rescale_factor': [] + }, + 4: { + 8: { + 'layer': nn.Conv2d(2, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), + 'input_shape': (2, 34, 34), + 'output_shape': (4, 33, 33)}, + 9: { + 'layer': IAFSqueeze(batch_size=2, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), + 'input_shape': (4, 33, 33), + 'output_shape': (4, 33, 33) + }, + 'destinations': [5], + 'conv_rescale_factor': [] + }, + 5: { + 10: { + 'layer': nn.Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), + 'input_shape': (4, 33, 33), + 'output_shape': (4, 32, 32) + }, + 11: { + 'layer': IAFSqueeze(batch_size=2, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), + 'input_shape': (4, 32, 32), + 'output_shape': (4, 32, 32) + }, + 12: { + 'layer': SumPool2d(kernel_size=2, stride=2, ceil_mode=False), + 'input_shape': (4, 32, 32), + 'output_shape': (4, 16, 16) + }, + 'destinations': [6], + 'conv_rescale_factor': [] + }, + 6: { + 13: { + 'layer': nn.Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), + 'input_shape': (4, 16, 16), + 'output_shape': (4, 15, 15) + }, + 14: { + 'layer': IAFSqueeze(batch_size=2, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), + 'input_shape': (4, 15, 15), + 'output_shape': (4, 15, 15) + }, + 15: { + 'layer': SumPool2d(kernel_size=2, stride=2, ceil_mode=False), + 'input_shape': (4, 15, 15), + 'output_shape': (4, 7, 7) + }, + 'destinations': [3], + 'conv_rescale_factor': [] + }, + 7: { + 19: { + 'layer': nn.Linear(in_features=100, out_features=100, bias=False), + 'input_shape': (100,), + 'output_shape': (100,) + }, + 20: { + 'layer': IAFSqueeze(batch_size=2, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), + 'input_shape': (100,), + 'output_shape': (100,) + }, + 'destinations': [8], + 'conv_rescale_factor': [] + }, + 8: { + 21: { + 'layer': nn.Linear(in_features=100, out_features=10, bias=False), + 'input_shape': (100,), + 'output_shape': (10,) + }, + 22: { + 'layer': IAFSqueeze(batch_size=2, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), + 'input_shape': (10,), + 'output_shape': (10,) + }, + 'destinations': [], + 'conv_rescale_factor': [] + } +} + +sinabs_edges_3 = [ + (0, 1), + (1, 2), + (2, 3), + (3, 4), + (4, 5), + (5, 6), + (6, 7), + (7, 17), + (8, 9), + (9, 10), + (10, 11), + (11, 12), + (12, 13), + (13, 14), + (14, 15), + (15, 17), + (17, 18), + (18, 19), + (19, 20), + (20, 21), + (21, 22), +] + +expected_output_3 = { + 0: { + 'dpcnnl_index': 0, + 'conv_node_id': 0, + 'conv_in_shape': (2, 34, 34), + 'conv_out_shape': (4, 33, 33), + 'spk_node_id': 1, + 'pool_node_id': [], + 'conv_rescaling_factor': None, + 'dynapcnnlayer_destination': [1], + 'nodes_destinations': {1: [2]}, + 'entry_point': True, + }, + 1: { + 'dpcnnl_index': 1, + 'conv_node_id': 2, + 'conv_in_shape': (4, 33, 33), + 'conv_out_shape': (4, 32, 32), + 'spk_node_id': 3, + 'pool_node_id': [4], + 'conv_rescaling_factor': None, + 'dynapcnnlayer_destination': [2], + 'nodes_destinations': {4: [5]}, + 'entry_point': False, + }, + 2: { + 'dpcnnl_index': 2, + 'conv_node_id': 5, + 'conv_in_shape': (4, 16, 16), + 'conv_out_shape': (4, 15, 15), + 'spk_node_id': 6, + 'pool_node_id': [7], + 'conv_rescaling_factor': None, + 'dynapcnnlayer_destination': [3], + 'nodes_destinations': {7: [17]}, + 'entry_point': False, + }, + 3: { + 'dpcnnl_index': 3, + 'conv_node_id': 17, + 'conv_in_shape': (4, 7, 7), + 'conv_out_shape': (100, 1, 1), + 'spk_node_id': 18, + 'pool_node_id': [], + 'conv_rescaling_factor': None, + 'dynapcnnlayer_destination': [7], + 'nodes_destinations': {18: [19]}, + 'entry_point': False, + }, + 4: { + 'dpcnnl_index': 4, + 'conv_node_id': 8, + 'conv_in_shape': (2, 34, 34), + 'conv_out_shape': (4, 33, 33), + 'spk_node_id': 9, + 'pool_node_id': [], + 'conv_rescaling_factor': None, + 'dynapcnnlayer_destination': [5], + 'nodes_destinations': {9: [10]}, + 'entry_point': True, + }, + 5: { + 'dpcnnl_index': 5, + 'conv_node_id': 10, + 'conv_in_shape': (4, 33, 33), + 'conv_out_shape': (4, 32, 32), + 'spk_node_id': 11, + 'pool_node_id': [12], + 'conv_rescaling_factor': None, + 'dynapcnnlayer_destination': [6], + 'nodes_destinations': {12: [13]}, + 'entry_point': False, + }, + 6: { + 'dpcnnl_index': 6, + 'conv_node_id': 13, + 'conv_in_shape': (4, 16, 16), + 'conv_out_shape': (4, 15, 15), + 'spk_node_id': 14, + 'pool_node_id': [15], + 'conv_rescaling_factor': None, + 'dynapcnnlayer_destination': [3], + 'nodes_destinations': {15: [17]}, + 'entry_point': False, + }, + 7: { + 'dpcnnl_index': 7, + 'conv_node_id': 19, + 'conv_in_shape': (100, 1, 1), + 'conv_out_shape': (100, 1, 1), + 'spk_node_id': 20, + 'pool_node_id': [], + 'conv_rescaling_factor': None, + 'dynapcnnlayer_destination': [8], + 'nodes_destinations': {20: [21]}, + 'entry_point': False, + }, + 8: { + 'dpcnnl_index': 8, + 'conv_node_id': 21, + 'conv_in_shape': (100, 1, 1), + 'conv_out_shape': (10, 1, 1), + 'spk_node_id': 22, + 'pool_node_id': [], + 'conv_rescaling_factor': None, + 'dynapcnnlayer_destination': [], + 'nodes_destinations': {}, + 'entry_point': False, + }, +} \ No newline at end of file diff --git a/tests/test_dynapcnnlayer/model_dummy_4.py b/tests/test_dynapcnnlayer/model_dummy_4.py new file mode 100644 index 00000000..8e08ccb7 --- /dev/null +++ b/tests/test_dynapcnnlayer/model_dummy_4.py @@ -0,0 +1,207 @@ +# implementing "a complex network structure" example in https://github.com/synsense/sinabs/issues/181 . """ + +import torch.nn as nn +from sinabs.layers import IAFSqueeze, SumPool2d +from sinabs.activation.surrogate_gradient_fn import PeriodicExponential + +nodes_to_dcnnl_map_4 = { + 0: { + 0: { + 'layer': nn.Conv2d(2, 1, kernel_size=(2, 2), stride=(1, 1), bias=False), + 'input_shape': (2, 34, 34), + 'output_shape': (1, 33, 33) + }, + 1: { + 'layer': IAFSqueeze(batch_size=2, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), + 'input_shape': (1, 33, 33), + 'output_shape': (1, 33, 33) + }, + 'destinations': [1, 2], + 'conv_rescale_factor': [] + }, + 1: { + 2: { + 'layer': nn.Conv2d(1, 1, kernel_size=(2, 2), stride=(1, 1), bias=False), + 'input_shape': (1, 33, 33), + 'output_shape': (1, 32, 32) + }, + 4: { + 'layer': IAFSqueeze(batch_size=2, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), + 'input_shape': (1, 32, 32), + 'output_shape': (1, 32, 32) + }, + 5: { + 'layer': SumPool2d(kernel_size=2, stride=2, ceil_mode=False), + 'input_shape': (1, 32, 32), + 'output_shape': (1, 16, 16) + }, + 'destinations': [3], + 'conv_rescale_factor': [] + }, + 2: { + 3: { + 'layer': nn.Conv2d(1, 1, kernel_size=(2, 2), stride=(1, 1), bias=False), + 'input_shape': (1, 33, 33), + 'output_shape': (1, 32, 32) + }, + 7: { + 'layer': IAFSqueeze(batch_size=2, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), + 'input_shape': (1, 32, 32), + 'output_shape': (1, 32, 32) + }, + 8: { + 'layer': SumPool2d(kernel_size=2, stride=2, ceil_mode=False), + 'input_shape': (1, 32, 32), + 'output_shape': (1, 16, 16) + }, + 9: { + 'layer': SumPool2d(kernel_size=5, stride=5, ceil_mode=False), + 'input_shape': (1, 32, 32), + 'output_shape': (1, 6, 6) + }, + 'destinations': [3, 4], + 'conv_rescale_factor': [] + }, + 3: { + 11: { + 'layer': nn.Conv2d(1, 1, kernel_size=(2, 2), stride=(1, 1), bias=False), + 'input_shape': (1, 16, 16), + 'output_shape': (1, 15, 15) + }, + 12: { + 'layer': IAFSqueeze(batch_size=2, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), + 'input_shape': (1, 15, 15), + 'output_shape': (1, 15, 15) + }, + 13: { + 'layer': SumPool2d(kernel_size=3, stride=3, ceil_mode=False), + 'input_shape': (1, 15, 15), + 'output_shape': (1, 5, 5) + }, + 'destinations': [5], + 'conv_rescale_factor': [] + }, + 4: { + 10: { + 'layer': nn.Conv2d(1, 1, kernel_size=(2, 2), stride=(1, 1), bias=False), + 'input_shape': (1, 6, 6), + 'output_shape': (1, 5, 5) + }, + 15: { + 'layer': IAFSqueeze(batch_size=2, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), + 'input_shape': (1, 5, 5), + 'output_shape': (1, 5, 5) + }, + 'destinations': [5], + 'conv_rescale_factor': [] + }, + 5: { + 16: { + 'layer': nn.Linear(in_features=25, out_features=10, bias=False), + 'input_shape': (1, 5, 5), + 'output_shape': (10,) + }, + 17: { + 'layer': IAFSqueeze(batch_size=2, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), + 'input_shape': (10,), + 'output_shape': (10,) + }, + 'destinations': [], + 'conv_rescale_factor': [] + } +} + +sinabs_edges_4 = [ + (0, 1), + (1, 2), + (1, 3), + (2, 4), + (4, 5), + (5, 11), + (3, 7), + (7, 8), + (7, 9), + (8, 11), + (9, 10), + (11, 12), + (12, 13), + (13, 16), + (15, 16), + (10, 15), + (16, 17), +] + +expected_output_4 = { + 0: { + 'dpcnnl_index': 0, + 'conv_node_id': 0, + 'conv_in_shape': (2, 34, 34), + 'conv_out_shape': (1, 33, 33), + 'spk_node_id': 1, + 'pool_node_id': [], + 'conv_rescaling_factor': None, + 'dynapcnnlayer_destination': [1, 2], + 'nodes_destinations': {1: [2, 3]}, + 'entry_point': True, + }, + 1: { + 'dpcnnl_index': 1, + 'conv_node_id': 2, + 'conv_in_shape': (1, 33, 33), + 'conv_out_shape': (1, 32, 32), + 'spk_node_id': 4, + 'pool_node_id': [5], + 'conv_rescaling_factor': None, + 'dynapcnnlayer_destination': [3], + 'nodes_destinations': {5: [11]}, + 'entry_point': False, + }, + 2: { + 'dpcnnl_index': 2, + 'conv_node_id': 3, + 'conv_in_shape': (1, 33, 33), + 'conv_out_shape': (1, 32, 32), + 'spk_node_id': 7, + 'pool_node_id': [8, 9], + 'conv_rescaling_factor': None, + 'dynapcnnlayer_destination': [3, 4], + 'nodes_destinations': {8: [11], 9: [10]}, + 'entry_point': False, + }, + 3: { + 'dpcnnl_index': 3, + 'conv_node_id': 11, + 'conv_in_shape': (1, 16, 16), + 'conv_out_shape': (1, 15, 15), + 'spk_node_id': 12, + 'pool_node_id': [13], + 'conv_rescaling_factor': None, + 'dynapcnnlayer_destination': [5], + 'nodes_destinations': {13: [16]}, + 'entry_point': False, + }, + 4: { + 'dpcnnl_index': 4, + 'conv_node_id': 10, + 'conv_in_shape': (1, 6, 6), + 'conv_out_shape': (1, 5, 5), + 'spk_node_id': 15, + 'pool_node_id': [], + 'conv_rescaling_factor': None, + 'dynapcnnlayer_destination': [5], + 'nodes_destinations': {15: [16]}, + 'entry_point': False, + }, + 5: { + 'dpcnnl_index': 5, + 'conv_node_id': 16, + 'conv_in_shape': (1, 5, 5), + 'conv_out_shape': (10, 1, 1), + 'spk_node_id': 17, + 'pool_node_id': [], + 'conv_rescaling_factor': None, + 'dynapcnnlayer_destination': [], + 'nodes_destinations': {}, + 'entry_point': False, + }, +} \ No newline at end of file diff --git a/tests/test_dynapcnnlayer/test_dynapcnnlayer.py b/tests/test_dynapcnnlayer/test_dynapcnnlayer.py index cb3f810a..f6bf6f26 100644 --- a/tests/test_dynapcnnlayer/test_dynapcnnlayer.py +++ b/tests/test_dynapcnnlayer/test_dynapcnnlayer.py @@ -4,15 +4,15 @@ from conftest_dynapcnnlayer import args_DynapcnnLayer -@pytest.mark.parametrize("nodes_to_dcnnl_map, dpcnnl_idx, sinabs_edges, expected_output", args_DynapcnnLayer) -def test_DynapcnnLayer(nodes_to_dcnnl_map, dpcnnl_idx, sinabs_edges, expected_output): +@pytest.mark.parametrize("nodes_to_dcnnl_map, dpcnnl_idx, sinabs_edges, entry_point, expected_output", args_DynapcnnLayer) +def test_DynapcnnLayer(nodes_to_dcnnl_map, dpcnnl_idx, sinabs_edges, entry_point, expected_output): """ Tests the instantiation of a set of `DynapcnnLayer` belonging to the same SNN and the data computed within their constructors and shared among the differntly interacting instances (according to the graph described by `sinabs_edges`). """ # create a `DynapcnnLayer` from the set of layers in `nodes_to_dcnnl_map[dpcnnl_idx]`. - dynapcnnlayer = construct_dynapcnnlayer(dpcnnl_idx, True, sinabs_edges, nodes_to_dcnnl_map, rescale_method_1) + dynapcnnlayer = construct_dynapcnnlayer(dpcnnl_idx, True, sinabs_edges, nodes_to_dcnnl_map, rescale_method_1, entry_point) # check if any node (layer) in `dynapcnnlayer` has been modified (e.g. `nn.Linear` turned `nn.Conv2d`). node, output_shape = dynapcnnlayer.get_modified_node_io(nodes_to_dcnnl_map[dpcnnl_idx]) @@ -30,6 +30,7 @@ def test_DynapcnnLayer(nodes_to_dcnnl_map, dpcnnl_idx, sinabs_edges, expected_ou conv_rescaling_factor = expected_output[dpcnnl_idx]['conv_rescaling_factor'] dynapcnnlayer_destination = expected_output[dpcnnl_idx]['dynapcnnlayer_destination'] nodes_destinations = expected_output[dpcnnl_idx]['nodes_destinations'] + entry_point = expected_output[dpcnnl_idx]['entry_point'] assert dynapcnnlayer.dpcnnl_index == expected_output[dpcnnl_idx]['dpcnnl_index'], \ f'wrong \'DynapcnnLayer.dpcnnl_index\': ID of the instance should be {dpcnnl_index}.' @@ -48,4 +49,6 @@ def test_DynapcnnLayer(nodes_to_dcnnl_map, dpcnnl_idx, sinabs_edges, expected_ou assert dynapcnnlayer.dynapcnnlayer_destination == expected_output[dpcnnl_idx]['dynapcnnlayer_destination'], \ f'wrong \'DynapcnnLayer.dynapcnnlayer_destination\': the DynapcnnLayer(s) set as destination(s) should be {dynapcnnlayer_destination}.' assert dynapcnnlayer.nodes_destinations == expected_output[dpcnnl_idx]['nodes_destinations'], \ - f'wrong \'DynapcnnLayer.nodes_destinations\': the targeted nodes within other DynapcnnLayer instance(s) should be {nodes_destinations}.' \ No newline at end of file + f'wrong \'DynapcnnLayer.nodes_destinations\': the targeted nodes within other DynapcnnLayer instance(s) should be {nodes_destinations}.' + assert dynapcnnlayer.entry_point == expected_output[dpcnnl_idx]['entry_point'], \ + f'wrong \'DynapcnnLayer.entry_point\': its value should be {entry_point}.' \ No newline at end of file From 8d5523a6355b45861b445a7a1f9404bfb90092d5 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Tue, 28 May 2024 14:48:30 +0200 Subject: [PATCH 107/379] Unit testing Unit testing the initial graph extraction steps from the original SNN being parsed. --- .../conftest_graph_extractor.py | 14 ++ tests/test_graph_extractor/model_dummy_1.py | 118 +++++++++++ tests/test_graph_extractor/model_dummy_2.py | 162 +++++++++++++++ tests/test_graph_extractor/model_dummy_3.py | 188 ++++++++++++++++++ tests/test_graph_extractor/model_dummy_4.py | 159 +++++++++++++++ .../test_graph_extractor.py | 22 ++ 6 files changed, 663 insertions(+) create mode 100644 tests/test_graph_extractor/conftest_graph_extractor.py create mode 100644 tests/test_graph_extractor/model_dummy_1.py create mode 100644 tests/test_graph_extractor/model_dummy_2.py create mode 100644 tests/test_graph_extractor/model_dummy_3.py create mode 100644 tests/test_graph_extractor/model_dummy_4.py create mode 100644 tests/test_graph_extractor/test_graph_extractor.py diff --git a/tests/test_graph_extractor/conftest_graph_extractor.py b/tests/test_graph_extractor/conftest_graph_extractor.py new file mode 100644 index 00000000..c531d6ce --- /dev/null +++ b/tests/test_graph_extractor/conftest_graph_extractor.py @@ -0,0 +1,14 @@ +# author : Willian Soares Girao +# contact : wsoaresgirao@gmail.com + +from model_dummy_1 import snn as snn_1, input_dummy as input_dummy_1, expected_output as expected_output_1 +from model_dummy_2 import snn as snn_2, input_dummy as input_dummy_2, expected_output as expected_output_2 +from model_dummy_3 import snn as snn_3, input_dummy as input_dummy_3, expected_output as expected_output_3 +from model_dummy_4 import snn as snn_4, input_dummy as input_dummy_4, expected_output as expected_output_4 + +args_GraphExtractor = [ + (snn_1, input_dummy_1, expected_output_1), + (snn_2, input_dummy_2, expected_output_2), + (snn_3, input_dummy_3, expected_output_3), + (snn_4, input_dummy_4, expected_output_4), +] \ No newline at end of file diff --git a/tests/test_graph_extractor/model_dummy_1.py b/tests/test_graph_extractor/model_dummy_1.py new file mode 100644 index 00000000..19f64a86 --- /dev/null +++ b/tests/test_graph_extractor/model_dummy_1.py @@ -0,0 +1,118 @@ +# author : Willian Soares Girao +# contact : wsoaresgirao@gmail.com +# implementing "a network with residual connections" example in https://github.com/synsense/sinabs/issues/181 + +import torch +import torch.nn as nn +from sinabs.layers import Merge, IAFSqueeze +from sinabs.activation.surrogate_gradient_fn import PeriodicExponential + +class SNN(nn.Module): + def __init__(self, batch_size) -> None: + super().__init__() + + self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False) # node 0 + self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) # node 1 + self.pool1 = nn.AvgPool2d(3,3) # node 2 + self.pool1a = nn.AvgPool2d(4,4) # node 3 + + self.conv2 = nn.Conv2d(10, 10, 4, 1, bias=False)# node 4 + self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) # node 6 + + self.conv3 = nn.Conv2d(10, 1, 2, 1, bias=False) # node 8 + self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) # node 9 + + self.flat = nn.Flatten() + + self.fc1 = nn.Linear(49, 500, bias=False) # node 10 + self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) # node 11 + + self.fc2 = nn.Linear(500, 10, bias=False) # node 12 + self.iaf5 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) # node 13 + + self.adder = Merge() + + def forward(self, x): + + con1_out = self.conv1(x) + iaf1_out = self.iaf1(con1_out) + pool1_out = self.pool1(iaf1_out) + pool1a_out = self.pool1a(iaf1_out) + + conv2_out = self.conv2(pool1_out) + iaf2_out = self.iaf2(conv2_out) + + conv3_out = self.conv3(self.adder(pool1a_out, iaf2_out)) + iaf3_out = self.iaf3(conv3_out) + + flat_out = self.flat(iaf3_out) + + fc1_out = self.fc1(flat_out) + iaf4_out = self.iaf4(fc1_out) + fc2_out = self.fc2(iaf4_out) + iaf5_out = self.iaf5(fc2_out) + + return iaf5_out + +channels = 2 +height = 34 +width = 34 +batch_size = 3 +input_shape = (batch_size, channels, height, width) + +torch.manual_seed(0) +input_dummy = torch.randn(input_shape) + +expected_output = { + 'edges_list': [ + (0, 1), + (1, 2), + (1, 3), + (2, 4), + (3, 5), + (4, 6), + (6, 5), + (7, 8), + (8, 9), + (9, 10), + (10, 11), + (11, 12), + (12, 13), + (5, 7), + ], + 'name_2_indx_map': { + 'conv1': 0, + 'iaf1': 1, + 'pool1': 2, + 'pool1a': 3, + 'conv2': 4, + 'adder': 5, + 'iaf2': 6, + 'conv3': 7, + 'iaf3': 8, + 'flat': 9, + 'fc1': 10, + 'iaf4': 11, + 'fc2': 12, + 'iaf5': 13, + }, + 'entry_nodes': [0], + 'nodes_io_shapes': { + 0: {'input': torch.Size([3, 2, 34, 34]), 'output': torch.Size([3, 10, 33, 33])}, + 1: {'input': torch.Size([3, 10, 33, 33]), 'output': torch.Size([3, 10, 33, 33])}, + 2: {'input': torch.Size([3, 10, 33, 33]), 'output': torch.Size([3, 10, 11, 11])}, + 3: {'input': torch.Size([3, 10, 33, 33]), 'output': torch.Size([3, 10, 8, 8])}, + 4: {'input': torch.Size([3, 10, 11, 11]), 'output': torch.Size([3, 10, 8, 8])}, + 6: {'input': torch.Size([3, 10, 8, 8]), 'output': torch.Size([3, 10, 8, 8])}, + 5: {'input': torch.Size([3, 10, 8, 8]), 'output': torch.Size([3, 10, 8, 8])}, + 7: {'input': torch.Size([3, 10, 8, 8]), 'output': torch.Size([3, 1, 7, 7])}, + 8: {'input': torch.Size([3, 1, 7, 7]), 'output': torch.Size([3, 1, 7, 7])}, + 9: {'input': torch.Size([3, 1, 7, 7]), 'output': torch.Size([3, 49])}, + 10: {'input': torch.Size([3, 49]), 'output': torch.Size([3, 500])}, + 11: {'input': torch.Size([3, 500]), 'output': torch.Size([3, 500])}, + 12: {'input': torch.Size([3, 500]), 'output': torch.Size([3, 10])}, + 13: {'input': torch.Size([3, 10]), 'output': torch.Size([3, 10])}, + }, +} + +snn = SNN(batch_size) \ No newline at end of file diff --git a/tests/test_graph_extractor/model_dummy_2.py b/tests/test_graph_extractor/model_dummy_2.py new file mode 100644 index 00000000..8d657b5d --- /dev/null +++ b/tests/test_graph_extractor/model_dummy_2.py @@ -0,0 +1,162 @@ +# author : Willian Soares Girao +# contact : wsoaresgirao@gmail.com +# implementing "a network with a merge and a split" in https://github.com/synsense/sinabs/issues/181 + +import torch +import torch.nn as nn +from sinabs.layers import Merge, IAFSqueeze, SumPool2d +from sinabs.activation.surrogate_gradient_fn import PeriodicExponential + +class SNN(nn.Module): + def __init__(self, batch_size) -> None: + super().__init__() + # -- graph node A -- + self.conv_A = nn.Conv2d(2, 4, 2, 1, bias=False) + self.iaf_A = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + # -- graph node B -- + self.conv_B = nn.Conv2d(4, 4, 2, 1, bias=False) + self.iaf2_B = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + self.pool_B = SumPool2d(2,2) + # -- graph node C -- + self.conv_C = nn.Conv2d(4, 4, 2, 1, bias=False) + self.iaf_C = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + self.pool_C = SumPool2d(2,2) + # -- graph node D -- + self.conv_D = nn.Conv2d(4, 4, 2, 1, bias=False) + self.iaf_D = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + # -- graph node E -- + self.conv_E = nn.Conv2d(4, 4, 2, 1, bias=False) + self.iaf3_E = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + self.pool_E = SumPool2d(2,2) + # -- graph node F -- + self.conv_F = nn.Conv2d(4, 4, 2, 1, bias=False) + self.iaf_F = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + # -- graph node G -- + self.fc3 = nn.Linear(144, 10, bias=False) + self.iaf3_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + + # -- merges -- + self.merge1 = Merge() + + # -- falts -- + self.flat_D = nn.Flatten() + self.flat_F = nn.Flatten() + + def forward(self, x): + # conv 1 - A/0 + convA_out = self.conv_A(x) # node 0 + iaf_A_out = self.iaf_A(convA_out) # node 1 + + # conv 2 - B/1 + conv_B_out = self.conv_B(iaf_A_out) # node 2 + iaf_B_out = self.iaf2_B(conv_B_out) # node 3 + pool_B_out = self.pool_B(iaf_B_out) # node 4 + + # conv 3 - C/2 + conv_C_out = self.conv_C(pool_B_out) # node 5 + iaf_C_out = self.iaf_C(conv_C_out) # node 7 + pool_C_out = self.pool_C(iaf_C_out) # node 8 + + # conv 4 - D/4 + conv_D_out = self.conv_D(pool_C_out) # node 9 + iaf_D_out = self.iaf_D(conv_D_out) # node 10 + + # fc 1 - E/3 + conv_E_out = self.conv_E(pool_B_out) # node 6 + iaf3_E_out = self.iaf3_E(conv_E_out) # node 12 + pool_E_out = self.pool_E(iaf3_E_out) # node 13 + + # fc 2 - F/6 + conv_F_out = self.conv_F(pool_E_out) # node 14 + iaf_F_out = self.iaf_F(conv_F_out) # node 15 + + # fc 2 - G/5 + flat_D_out = self.flat_D(iaf_D_out) # node 11 + flat_F_out = self.flat_F(iaf_F_out) # node 16 + + merge1_out = self.merge1(flat_D_out, flat_F_out) # node 19 + fc3_out = self.fc3(merge1_out) # node 17 + iaf3_fc_out = self.iaf3_fc(fc3_out) # node 18 + + return iaf3_fc_out + +channels = 2 +height = 34 +width = 34 +batch_size = 8 +input_shape = (batch_size, channels, height, width) + +torch.manual_seed(0) +input_dummy = torch.randn(input_shape) + +expected_output = { + 'edges_list': [ + (0, 1), + (1, 2), + (2, 3), + (3, 4), + (4, 5), + (4, 6), + (5, 7), + (7, 8), + (8, 9), + (9, 10), + (10, 11), + (6, 12), + (12, 13), + (13, 14), + (14, 15), + (15, 16), + (17, 18), + (19, 17), + (11, 19), + (16, 19), + ], + 'name_2_indx_map': { + 'conv_A': 0, + 'iaf_A': 1, + 'conv_B': 2, + 'iaf2_B': 3, + 'pool_B': 4, + 'conv_C': 5, + 'conv_E': 6, + 'iaf_C': 7, + 'pool_C': 8, + 'conv_D': 9, + 'iaf_D': 10, + 'flat_D': 11, + 'iaf3_E': 12, + 'pool_E': 13, + 'conv_F': 14, + 'iaf_F': 15, + 'flat_F': 16, + 'fc3': 17, + 'iaf3_fc': 18, + 'merge1': 19, + }, + 'entry_nodes': [0], + 'nodes_io_shapes': { + 0: {'input': torch.Size([8, 2, 34, 34]), 'output': torch.Size([8, 4, 33, 33])}, + 1: {'input': torch.Size([8, 4, 33, 33]), 'output': torch.Size([8, 4, 33, 33])}, + 2: {'input': torch.Size([8, 4, 33, 33]), 'output': torch.Size([8, 4, 32, 32])}, + 3: {'input': torch.Size([8, 4, 32, 32]), 'output': torch.Size([8, 4, 32, 32])}, + 4: {'input': torch.Size([8, 4, 32, 32]), 'output': torch.Size([8, 4, 16, 16])}, + 5: {'input': torch.Size([8, 4, 16, 16]), 'output': torch.Size([8, 4, 15, 15])}, + 6: {'input': torch.Size([8, 4, 16, 16]), 'output': torch.Size([8, 4, 15, 15])}, + 7: {'input': torch.Size([8, 4, 15, 15]), 'output': torch.Size([8, 4, 15, 15])}, + 12: {'input': torch.Size([8, 4, 15, 15]), 'output': torch.Size([8, 4, 15, 15])}, + 8: {'input': torch.Size([8, 4, 15, 15]), 'output': torch.Size([8, 4, 7, 7])}, + 13: {'input': torch.Size([8, 4, 15, 15]), 'output': torch.Size([8, 4, 7, 7])}, + 9: {'input': torch.Size([8, 4, 7, 7]), 'output': torch.Size([8, 4, 6, 6])}, + 14: {'input': torch.Size([8, 4, 7, 7]), 'output': torch.Size([8, 4, 6, 6])}, + 10: {'input': torch.Size([8, 4, 6, 6]), 'output': torch.Size([8, 4, 6, 6])}, + 15: {'input': torch.Size([8, 4, 6, 6]), 'output': torch.Size([8, 4, 6, 6])}, + 11: {'input': torch.Size([8, 4, 6, 6]), 'output': torch.Size([8, 144])}, + 16: {'input': torch.Size([8, 4, 6, 6]), 'output': torch.Size([8, 144])}, + 19: {'input': torch.Size([8, 144]), 'output': torch.Size([8, 144])}, + 17: {'input': torch.Size([8, 144]), 'output': torch.Size([8, 10])}, + 18: {'input': torch.Size([8, 10]), 'output': torch.Size([8, 10])}, + }, +} + +snn = SNN(batch_size) \ No newline at end of file diff --git a/tests/test_graph_extractor/model_dummy_3.py b/tests/test_graph_extractor/model_dummy_3.py new file mode 100644 index 00000000..39bd6e39 --- /dev/null +++ b/tests/test_graph_extractor/model_dummy_3.py @@ -0,0 +1,188 @@ + +# author : Willian Soares Girao +# contact : wsoaresgirao@gmail.com +# implementing "two networks with merging outputs" in https://github.com/synsense/sinabs/issues/181 + +import torch +import torch.nn as nn +from sinabs.layers import IAFSqueeze, SumPool2d, Merge +from sinabs.activation.surrogate_gradient_fn import PeriodicExponential + +class SNN(nn.Module): + def __init__(self, batch_size) -> None: + super().__init__() + + self.conv_A = nn.Conv2d(2, 4, 2, 1, bias=False) + self.iaf_A = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + + self.conv_B = nn.Conv2d(4, 4, 2, 1, bias=False) + self.iaf_B = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + self.pool_B = SumPool2d(2,2) + + self.conv_C = nn.Conv2d(4, 4, 2, 1, bias=False) + self.iaf_C = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + self.pool_C = SumPool2d(2,2) + + self.conv_D = nn.Conv2d(2, 4, 2, 1, bias=False) + self.iaf_D = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + + self.conv_E = nn.Conv2d(4, 4, 2, 1, bias=False) + self.iaf_E = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + self.pool_E = SumPool2d(2,2) + + self.conv_F = nn.Conv2d(4, 4, 2, 1, bias=False) + self.iaf_F = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + self.pool_F = SumPool2d(2,2) + + self.flat_brach1 = nn.Flatten() + self.flat_brach2 = nn.Flatten() + self.merge = Merge() + + self.fc1 = nn.Linear(196, 100, bias=False) + self.iaf1_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + + self.fc2 = nn.Linear(100, 100, bias=False) + self.iaf2_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + + self.fc3 = nn.Linear(100, 10, bias=False) + self.iaf3_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + + def forward(self, x): + # conv 1 - A + conv_A_out = self.conv_A(x) + iaf_A_out = self.iaf_A(conv_A_out) + # conv 2 - B + conv_B_out = self.conv_B(iaf_A_out) + iaf_B_out = self.iaf_B(conv_B_out) + pool_B_out = self.pool_B(iaf_B_out) + # conv 3 - C + conv_C_out = self.conv_C(pool_B_out) + iaf_C_out = self.iaf_C(conv_C_out) + pool_C_out = self.pool_C(iaf_C_out) + + # --- + + # conv 4 - D + conv_D_out = self.conv_D(x) + iaf_D_out = self.iaf_D(conv_D_out) + # conv 5 - E + conv_E_out = self.conv_E(iaf_D_out) + iaf_E_out = self.iaf_E(conv_E_out) + pool_E_out = self.pool_E(iaf_E_out) + # conv 6 - F + conv_F_out = self.conv_F(pool_E_out) + iaf_F_out = self.iaf_F(conv_F_out) + pool_F_out = self.pool_F(iaf_F_out) + + # --- + + flat_brach1_out = self.flat_brach1(pool_C_out) + flat_brach2_out = self.flat_brach2(pool_F_out) + merge_out = self.merge(flat_brach1_out, flat_brach2_out) + + # FC 7 - G + fc1_out = self.fc1(merge_out) + iaf1_fc_out = self.iaf1_fc(fc1_out) + # FC 8 - H + fc2_out = self.fc2(iaf1_fc_out) + iaf2_fc_out = self.iaf2_fc(fc2_out) + # FC 9 - I + fc3_out = self.fc3(iaf2_fc_out) + iaf3_fc_out = self.iaf3_fc(fc3_out) + + return iaf3_fc_out + +channels = 2 +height = 34 +width = 34 +batch_size = 2 +input_shape = (batch_size, channels, height, width) + +torch.manual_seed(0) +input_dummy = torch.randn(input_shape) + +expected_output = { + 'edges_list': [ + (0, 1), + (1, 2), + (2, 3), + (3, 4), + (4, 5), + (5, 6), + (6, 7), + (7, 8), + (9, 10), + (10, 11), + (11, 12), + (12, 13), + (13, 14), + (14, 15), + (15, 16), + (16, 17), + (8, 18), + (17, 18), + (18, 19), + (19, 20), + (20, 21), + (21, 22), + (22, 23), + (23, 24), + ], + 'name_2_indx_map': { + 'conv_A': 0, + 'iaf_A': 1, + 'conv_B': 2, + 'iaf_B': 3, + 'pool_B': 4, + 'conv_C': 5, + 'iaf_C': 6, + 'pool_C': 7, + 'flat_brach1': 8, + 'conv_D': 9, + 'iaf_D': 10, + 'conv_E': 11, + 'iaf_E': 12, + 'pool_E': 13, + 'conv_F': 14, + 'iaf_F': 15, + 'pool_F': 16, + 'flat_brach2': 17, + 'merge': 18, + 'fc1': 19, + 'iaf1_fc': 20, + 'fc2': 21, + 'iaf2_fc': 22, + 'fc3': 23, + 'iaf3_fc': 24, + }, + 'entry_nodes': [0, 9], + 'nodes_io_shapes': { + 0: {'input': torch.Size([2, 2, 34, 34]), 'output': torch.Size([2, 4, 33, 33])}, + 9: {'input': torch.Size([2, 2, 34, 34]), 'output': torch.Size([2, 4, 33, 33])}, + 1: {'input': torch.Size([2, 4, 33, 33]), 'output': torch.Size([2, 4, 33, 33])}, + 10: {'input': torch.Size([2, 4, 33, 33]), 'output': torch.Size([2, 4, 33, 33])}, + 2: {'input': torch.Size([2, 4, 33, 33]), 'output': torch.Size([2, 4, 32, 32])}, + 11: {'input': torch.Size([2, 4, 33, 33]), 'output': torch.Size([2, 4, 32, 32])}, + 3: {'input': torch.Size([2, 4, 32, 32]), 'output': torch.Size([2, 4, 32, 32])}, + 12: {'input': torch.Size([2, 4, 32, 32]), 'output': torch.Size([2, 4, 32, 32])}, + 4: {'input': torch.Size([2, 4, 32, 32]), 'output': torch.Size([2, 4, 16, 16])}, + 13: {'input': torch.Size([2, 4, 32, 32]), 'output': torch.Size([2, 4, 16, 16])}, + 5: {'input': torch.Size([2, 4, 16, 16]), 'output': torch.Size([2, 4, 15, 15])}, + 14: {'input': torch.Size([2, 4, 16, 16]), 'output': torch.Size([2, 4, 15, 15])}, + 6: {'input': torch.Size([2, 4, 15, 15]), 'output': torch.Size([2, 4, 15, 15])}, + 15: {'input': torch.Size([2, 4, 15, 15]), 'output': torch.Size([2, 4, 15, 15])}, + 7: {'input': torch.Size([2, 4, 15, 15]), 'output': torch.Size([2, 4, 7, 7])}, + 16: {'input': torch.Size([2, 4, 15, 15]), 'output': torch.Size([2, 4, 7, 7])}, + 8: {'input': torch.Size([2, 4, 7, 7]), 'output': torch.Size([2, 196])}, + 17: {'input': torch.Size([2, 4, 7, 7]), 'output': torch.Size([2, 196])}, + 18: {'input': torch.Size([2, 196]), 'output': torch.Size([2, 196])}, + 19: {'input': torch.Size([2, 196]), 'output': torch.Size([2, 100])}, + 20: {'input': torch.Size([2, 100]), 'output': torch.Size([2, 100])}, + 21: {'input': torch.Size([2, 100]), 'output': torch.Size([2, 100])}, + 22: {'input': torch.Size([2, 100]), 'output': torch.Size([2, 100])}, + 23: {'input': torch.Size([2, 100]), 'output': torch.Size([2, 10])}, + 24: {'input': torch.Size([2, 10]), 'output': torch.Size([2, 10])}, + }, +} + +snn = SNN(batch_size) \ No newline at end of file diff --git a/tests/test_graph_extractor/model_dummy_4.py b/tests/test_graph_extractor/model_dummy_4.py new file mode 100644 index 00000000..0acad595 --- /dev/null +++ b/tests/test_graph_extractor/model_dummy_4.py @@ -0,0 +1,159 @@ +# author : Willian Soares Girao +# contact : wsoaresgirao@gmail.com +# implementing "a complex network structure" example in https://github.com/synsense/sinabs/issues/181 . """ + +import torch +import torch.nn as nn +from sinabs.layers import IAFSqueeze, SumPool2d, Merge +from sinabs.activation.surrogate_gradient_fn import PeriodicExponential + +class SNN(nn.Module): + def __init__(self, batch_size) -> None: + super().__init__() + + self.conv1 = nn.Conv2d(2, 1, 2, 1, bias=False) + self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + + self.conv2 = nn.Conv2d(1, 1, 2, 1, bias=False) + self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + self.pool2 = SumPool2d(2,2) + + self.conv3 = nn.Conv2d(1, 1, 2, 1, bias=False) + self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + self.pool3 = SumPool2d(2,2) + self.pool3a = SumPool2d(5,5) + + self.conv4 = nn.Conv2d(1, 1, 2, 1, bias=False) + self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + self.pool4 = SumPool2d(3,3) + + self.flat1 = nn.Flatten() + self.flat2 = nn.Flatten() + + self.conv5 = nn.Conv2d(1, 1, 2, 1, bias=False) + self.iaf5 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + + self.fc2 = nn.Linear(25, 10, bias=False) + self.iaf2_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + + # -- merges -- + self.merge1 = Merge() + self.merge2 = Merge() + + def forward(self, x): + # conv 1 - A/0 + con1_out = self.conv1(x) + iaf1_out = self.iaf1(con1_out) + + # conv 2 - B/1 + conv2_out = self.conv2(iaf1_out) + iaf2_out = self.iaf2(conv2_out) + pool2_out = self.pool2(iaf2_out) + + # conv 3 - C/2 + conv3_out = self.conv3(iaf1_out) + iaf3_out = self.iaf3(conv3_out) + pool3_out = self.pool3(iaf3_out) + pool3a_out = self.pool3a(iaf3_out) + + # conv 4 - D/3 + merge1_out = self.merge1(pool2_out, pool3_out) + conv4_out = self.conv4(merge1_out) + iaf4_out = self.iaf4(conv4_out) + pool4_out = self.pool4(iaf4_out) + flat1_out = self.flat1(pool4_out) + + # conv 5 - E/4 + conv5_out = self.conv5(pool3a_out) + iaf5_out = self.iaf5(conv5_out) + flat2_out = self.flat2(iaf5_out) + + # fc 2 - F/5 + merge2_out = self.merge2(flat2_out, flat1_out) + + fc2_out = self.fc2(merge2_out) + iaf2_fc_out = self.iaf2_fc(fc2_out) + + return iaf2_fc_out + +channels = 2 +height = 34 +width = 34 +batch_size = 2 +input_shape = (batch_size, channels, height, width) + +torch.manual_seed(0) +input_dummy = torch.randn(input_shape) + +expected_output = { + 'edges_list': [ + (0, 1), + (1, 2), + (1, 3), + (2, 4), + (4, 5), + (5, 6), + (3, 7), + (7, 8), + (7, 9), + (8, 6), + (9, 10), + (11, 12), + (12, 13), + (13, 14), + (14, 15), + (16, 15), + (10, 17), + (17, 16), + (18, 19), + (6, 11), + (15, 18), + ], + 'name_2_indx_map': { + 'conv1': 0, + 'iaf1': 1, + 'conv2': 2, + 'conv3': 3, + 'iaf2': 4, + 'pool2': 5, + 'merge1': 6, + 'iaf3': 7, + 'pool3': 8, + 'pool3a': 9, + 'conv5': 10, + 'conv4': 11, + 'iaf4': 12, + 'pool4': 13, + 'flat1': 14, + 'merge2': 15, + 'flat2': 16, + 'iaf5': 17, + 'fc2': 18, + 'iaf2_fc': 19, + }, + 'entry_nodes': [0], + 'nodes_io_shapes': { + 0: {'input': torch.Size([2, 2, 34, 34]), 'output': torch.Size([2, 1, 33, 33])}, + 1: {'input': torch.Size([2, 1, 33, 33]), 'output': torch.Size([2, 1, 33, 33])}, + 2: {'input': torch.Size([2, 1, 33, 33]), 'output': torch.Size([2, 1, 32, 32])}, + 3: {'input': torch.Size([2, 1, 33, 33]), 'output': torch.Size([2, 1, 32, 32])}, + 4: {'input': torch.Size([2, 1, 32, 32]), 'output': torch.Size([2, 1, 32, 32])}, + 7: {'input': torch.Size([2, 1, 32, 32]), 'output': torch.Size([2, 1, 32, 32])}, + 5: {'input': torch.Size([2, 1, 32, 32]), 'output': torch.Size([2, 1, 16, 16])}, + 8: {'input': torch.Size([2, 1, 32, 32]), 'output': torch.Size([2, 1, 16, 16])}, + 9: {'input': torch.Size([2, 1, 32, 32]), 'output': torch.Size([2, 1, 6, 6])}, + 6: {'input': torch.Size([2, 1, 16, 16]), 'output': torch.Size([2, 1, 16, 16])}, + 10: {'input': torch.Size([2, 1, 6, 6]), 'output': torch.Size([2, 1, 5, 5])}, + 11: {'input': torch.Size([2, 1, 16, 16]), 'output': torch.Size([2, 1, 15, 15])}, + 17: {'input': torch.Size([2, 1, 5, 5]), 'output': torch.Size([2, 1, 5, 5])}, + 12: {'input': torch.Size([2, 1, 15, 15]), 'output': torch.Size([2, 1, 15, 15])}, + 16: {'input': torch.Size([2, 1, 5, 5]), 'output': torch.Size([2, 25])}, + 13: {'input': torch.Size([2, 1, 15, 15]), 'output': torch.Size([2, 1, 5, 5])}, + 14: {'input': torch.Size([2, 1, 5, 5]), 'output': torch.Size([2, 25])}, + 15: {'input': torch.Size([2, 25]), 'output': torch.Size([2, 25])}, + 18: {'input': torch.Size([2, 25]), 'output': torch.Size([2, 10])}, + 19: {'input': torch.Size([2, 10]), 'output': torch.Size([2, 10])}, + }, +} + +snn = SNN(batch_size) \ No newline at end of file diff --git a/tests/test_graph_extractor/test_graph_extractor.py b/tests/test_graph_extractor/test_graph_extractor.py new file mode 100644 index 00000000..6b914f0c --- /dev/null +++ b/tests/test_graph_extractor/test_graph_extractor.py @@ -0,0 +1,22 @@ +# author : Willian Soares Girao +# contact : wsoaresgirao@gmail.com + +import pytest +from sinabs.backend.dynapcnn.NIRGraphExtractor import NIRtoDynapcnnNetworkGraph + +from conftest_graph_extractor import args_GraphExtractor + +@pytest.mark.parametrize("snn, input_dummy, expected_output", args_GraphExtractor) +def test_GraphExtractor(snn, input_dummy, expected_output): + """ Tests the graph extraction from the original SNN being turned into a DynapcnnNetwork.""" + + graph_tracer = NIRtoDynapcnnNetworkGraph(snn, input_dummy) + + assert expected_output['edges_list'] == graph_tracer.get_edges_list, \ + f'wrong list of edges extracted from the SNN.' + assert expected_output['name_2_indx_map'] == graph_tracer.name_2_indx_map, \ + f'wrong mapping from layer variable name to node ID.' + assert expected_output['entry_nodes'] == graph_tracer.entry_nodes, \ + f'wrong list with entry node\'s IDs (i.e., layers serving as input to the SNN).' + assert expected_output['nodes_io_shapes'] == graph_tracer.nodes_io_shapes, \ + f'wrong I/O shapes computed for one or more nodes.' \ No newline at end of file From 15f852c48215835add31420c88ecb9b7adf9148b Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Tue, 28 May 2024 14:50:05 +0200 Subject: [PATCH 108/379] removed deprecated DynapcnnNetwork unit tests --- .../architectures_samples.py | 518 ---------------- .../conftest_dynapcnnnetwork.py | 563 ------------------ .../test_dynapcnnnetwork.py | 84 --- 3 files changed, 1165 deletions(-) delete mode 100644 tests/test_dynapcnnnetwork/architectures_samples.py delete mode 100644 tests/test_dynapcnnnetwork/conftest_dynapcnnnetwork.py delete mode 100644 tests/test_dynapcnnnetwork/test_dynapcnnnetwork.py diff --git a/tests/test_dynapcnnnetwork/architectures_samples.py b/tests/test_dynapcnnnetwork/architectures_samples.py deleted file mode 100644 index e5bc2302..00000000 --- a/tests/test_dynapcnnnetwork/architectures_samples.py +++ /dev/null @@ -1,518 +0,0 @@ -import torch -import torch.nn as nn -from sinabs.layers import Merge, IAFSqueeze, SumPool2d - -class EXAMPLE_1(nn.Module): - def __init__(self) -> None: - super().__init__() - - self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False) - self.iaf1 = IAFSqueeze(batch_size=1) - self.pool1 = nn.AvgPool2d(3,3) - self.pool1a = nn.AvgPool2d(4,4) - - self.conv2 = nn.Conv2d(10, 10, 4, 1, bias=False) - self.iaf2 = IAFSqueeze(batch_size=1) - - self.conv3 = nn.Conv2d(10, 1, 2, 1, bias=False) - self.iaf3 = IAFSqueeze(batch_size=1) - - self.flat = nn.Flatten() - - self.fc1 = nn.Linear(49, 500, bias=False) - self.iaf4 = IAFSqueeze(batch_size=1) - - self.fc2 = nn.Linear(500, 10, bias=False) - self.iaf5 = IAFSqueeze(batch_size=1) - - self.adder = Merge() - - def forward(self, x): - - con1_out = self.conv1(x) - iaf1_out = self.iaf1(con1_out) - pool1_out = self.pool1(iaf1_out) - pool1a_out = self.pool1a(iaf1_out) - - conv2_out = self.conv2(pool1_out) - iaf2_out = self.iaf2(conv2_out) - - conv3_out = self.conv3(self.adder(pool1a_out, iaf2_out)) - iaf3_out = self.iaf3(conv3_out) - - flat_out = self.flat(iaf3_out) - - fc1_out = self.fc1(flat_out) - iaf4_out = self.iaf4(fc1_out) - fc2_out = self.fc2(iaf4_out) - iaf5_out = self.iaf5(fc2_out) - - return iaf5_out - -class EXAMPLE_2(nn.Module): - def __init__(self) -> None: - super().__init__() - - self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False) - self.conv1_iaf = IAFSqueeze(batch_size=1) - self.pool1 = nn.AvgPool2d(3,3) - self.pool1a = nn.AvgPool2d(4,4) - - self.conv2 = nn.Conv2d(10, 10, 4, 1, bias=False) - self.conv2_iaf = IAFSqueeze(batch_size=1) - - self.conv3 = nn.Conv2d(10, 1, 2, 1, bias=False) - self.conv3_iaf = IAFSqueeze(batch_size=1) - - self.flat = nn.Flatten() - - self.fc1 = nn.Linear(49, 100, bias=False) - self.fc1_iaf = IAFSqueeze(batch_size=1) - - self.fc2 = nn.Linear(100, 100, bias=False) - self.fc2_iaf = IAFSqueeze(batch_size=1) - - self.fc3 = nn.Linear(100, 10, bias=False) - self.fc3_iaf = IAFSqueeze(batch_size=1) - - self.merge1 = Merge() - - def forward(self, x): - # -- conv. block 1 -- - con1_out = self.conv1(x) - conv1_iaf_out = self.conv1_iaf(con1_out) - pool1_out = self.pool1(conv1_iaf_out) - pool1a_out = self.pool1a(conv1_iaf_out) - # -- conv. block 2 -- - conv2_out = self.conv2(pool1_out) - conv2_iaf_out = self.conv2_iaf(conv2_out) - # -- conv. block 3 -- - merge1_out = self.merge1(pool1a_out, conv2_iaf_out) - conv3_out = self.conv3(merge1_out) - conv3_iaf_out = self.conv3_iaf(conv3_out) - flat_out = self.flat(conv3_iaf_out) - # -- fc clock 1 -- - fc1_out = self.fc1(flat_out) - fc1_iaf_out = self.fc1_iaf(fc1_out) - # -- fc clock 2 -- - fc2_out = self.fc2(fc1_iaf_out) - fc2_iaf_out = self.fc2_iaf(fc2_out) - # -- fc clock 3 -- - fc3_out = self.fc3(fc2_iaf_out) - fc3_iaf_out = self.fc3_iaf(fc3_out) - - return fc3_iaf_out - -class EXAMPLE_3(nn.Module): - def __init__(self) -> None: - super().__init__() - - self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False) - self.conv1_iaf = IAFSqueeze(batch_size=1) - self.pool1 = nn.AvgPool2d(3,3) - self.pool1a = nn.AvgPool2d(4,4) - - self.conv2 = nn.Conv2d(10, 10, 4, 1, bias=False) - self.conv2_iaf = IAFSqueeze(batch_size=1) - - self.conv3 = nn.Conv2d(10, 1, 2, 1, bias=False) - self.conv3_iaf = IAFSqueeze(batch_size=1) - - self.flat = nn.Flatten() - - self.fc1 = nn.Linear(49, 100, bias=False) - self.fc1_iaf = IAFSqueeze(batch_size=1) - - self.fc2 = nn.Linear(100, 100, bias=False) - self.fc2_iaf = IAFSqueeze(batch_size=1) - - self.fc3 = nn.Linear(100, 10, bias=False) - self.fc3_iaf = IAFSqueeze(batch_size=1) - - self.merge1 = Merge() - self.merge2 = Merge() - - def forward(self, x): - # -- conv. block 0 -- - con1_out = self.conv1(x) - conv1_iaf_out = self.conv1_iaf(con1_out) - pool1_out = self.pool1(conv1_iaf_out) - pool1a_out = self.pool1a(conv1_iaf_out) - # -- conv. block 1 -- - conv2_out = self.conv2(pool1_out) - conv2_iaf_out = self.conv2_iaf(conv2_out) - # -- conv. block 2 -- - merge1_out = self.merge1(pool1a_out, conv2_iaf_out) - conv3_out = self.conv3(merge1_out) - conv3_iaf_out = self.conv3_iaf(conv3_out) - flat_out = self.flat(conv3_iaf_out) - # -- fc clock 3 -- - fc1_out = self.fc1(flat_out) - fc1_iaf_out = self.fc1_iaf(fc1_out) - # -- fc clock 4 -- - fc2_out = self.fc2(fc1_iaf_out) - fc2_iaf_out = self.fc2_iaf(fc2_out) - # -- fc clock 5 -- - merge2_out = self.merge2(fc1_iaf_out, fc2_iaf_out) - fc3_out = self.fc3(merge2_out) - fc3_iaf_out = self.fc3_iaf(fc3_out) - - return fc3_iaf_out - -class EXAMPLE_4(nn.Module): - def __init__(self) -> None: - super().__init__() - - self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False) - self.conv1_iaf = IAFSqueeze(batch_size=1) - self.pool1 = nn.AvgPool2d(3,3) - self.pool1a = nn.AvgPool2d(4,4) - - self.conv2 = nn.Conv2d(10, 10, 4, 1, bias=False) - self.conv2_iaf = IAFSqueeze(batch_size=1) - - self.conv3 = nn.Conv2d(10, 10, 2, 1, bias=False) - self.conv3_iaf = IAFSqueeze(batch_size=1) - - self.conv4 = nn.Conv2d(10, 1, 2, 1, bias=False) - self.conv4_iaf = IAFSqueeze(batch_size=1) - - self.flat = nn.Flatten() - - self.fc1 = nn.Linear(36, 100, bias=False) - self.fc1_iaf = IAFSqueeze(batch_size=1) - - self.fc2 = nn.Linear(100, 100, bias=False) - self.fc2_iaf = IAFSqueeze(batch_size=1) - - self.fc3 = nn.Linear(100, 10, bias=False) - self.fc3_iaf = IAFSqueeze(batch_size=1) - - self.merge1 = Merge() - self.merge2 = Merge() - - def forward(self, x): - # -- conv. block 0 -- - con1_out = self.conv1(x) - conv1_iaf_out = self.conv1_iaf(con1_out) - pool1_out = self.pool1(conv1_iaf_out) - pool1a_out = self.pool1a(conv1_iaf_out) - # -- conv. block 1 -- - conv2_out = self.conv2(pool1_out) - conv2_iaf_out = self.conv2_iaf(conv2_out) - # -- conv. block 2 -- - merge1_out = self.merge1(pool1a_out, conv2_iaf_out) - conv3_out = self.conv3(merge1_out) - conv3_iaf_out = self.conv3_iaf(conv3_out) - # -- conv. block 3 -- - conv4_out = self.conv4(conv3_iaf_out) - conv4_iaf_out = self.conv4_iaf(conv4_out) - flat_out = self.flat(conv4_iaf_out) - # -- fc clock 4 -- - fc1_out = self.fc1(flat_out) - fc1_iaf_out = self.fc1_iaf(fc1_out) - # -- fc clock 5 -- - fc2_out = self.fc2(fc1_iaf_out) - fc2_iaf_out = self.fc2_iaf(fc2_out) - # -- fc clock 6 -- - merge2_out = self.merge2(fc1_iaf_out, fc2_iaf_out) - fc3_out = self.fc3(merge2_out) - fc3_iaf_out = self.fc3_iaf(fc3_out) - - return fc3_iaf_out - -class EXAMPLE_5(nn.Module): - def __init__(self) -> None: - super().__init__() - - self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False) - self.conv1_iaf = IAFSqueeze(batch_size=1) - self.pool1 = nn.AvgPool2d(3,3) - self.pool1a = nn.AvgPool2d(4,4) - - self.conv2 = nn.Conv2d(10, 10, 4, 1, bias=False) - self.conv2_iaf = IAFSqueeze(batch_size=1) - - self.conv3 = nn.Conv2d(10, 10, 2, 1, bias=False) - self.conv3_iaf = IAFSqueeze(batch_size=1) - - self.conv4 = nn.Conv2d(10, 1, 2, 1, bias=False) - self.conv4_iaf = IAFSqueeze(batch_size=1) - - self.flat = nn.Flatten() - - self.fc1 = nn.Linear(36, 100, bias=False) - self.fc1_iaf = IAFSqueeze(batch_size=1) - - self.fc2 = nn.Linear(100, 100, bias=False) - self.fc2_iaf = IAFSqueeze(batch_size=1) - - self.fc3 = nn.Linear(100, 100, bias=False) - self.fc3_iaf = IAFSqueeze(batch_size=1) - - self.fc4 = nn.Linear(100, 10, bias=False) - self.fc4_iaf = IAFSqueeze(batch_size=1) - - self.merge1 = Merge() - self.merge2 = Merge() - self.merge3 = Merge() - - def forward(self, x): - # -- conv. block 0 -- - con1_out = self.conv1(x) - conv1_iaf_out = self.conv1_iaf(con1_out) - pool1_out = self.pool1(conv1_iaf_out) - pool1a_out = self.pool1a(conv1_iaf_out) - # -- conv. block 1 -- - conv2_out = self.conv2(pool1_out) - conv2_iaf_out = self.conv2_iaf(conv2_out) - # -- conv. block 2 -- - merge1_out = self.merge1(pool1a_out, conv2_iaf_out) - conv3_out = self.conv3(merge1_out) - conv3_iaf_out = self.conv3_iaf(conv3_out) - # -- conv. block 3 -- - conv4_out = self.conv4(conv3_iaf_out) - conv4_iaf_out = self.conv4_iaf(conv4_out) - flat_out = self.flat(conv4_iaf_out) - # -- fc clock 4 -- - fc1_out = self.fc1(flat_out) - fc1_iaf_out = self.fc1_iaf(fc1_out) - # -- fc clock 5 -- - fc2_out = self.fc2(fc1_iaf_out) - fc2_iaf_out = self.fc2_iaf(fc2_out) - # -- fc clock 6 -- - merge2_out = self.merge2(fc1_iaf_out, fc2_iaf_out) - fc3_out = self.fc3(merge2_out) - fc3_iaf_out = self.fc3_iaf(fc3_out) - # -- fc clock 7 -- - merge3_out = self.merge3(fc2_iaf_out, fc3_iaf_out) - fc4_out = self.fc4(merge3_out) - fc4_iaf_out = self.fc4_iaf(fc4_out) - - return fc4_iaf_out - -class EXAMPLE_6(nn.Module): - """ This is the 'two networks with merging outputs' example in https://github.com/synsense/sinabs/issues/181 . """ - def __init__(self) -> None: - super().__init__() - - self.conv_A = nn.Conv2d(2, 4, 2, 1, bias=False) - self.iaf_A = IAFSqueeze(batch_size=1) - - self.conv_B = nn.Conv2d(4, 4, 2, 1, bias=False) - self.iaf_B = IAFSqueeze(batch_size=1) - self.pool_B = SumPool2d(2,2) - - self.conv_C = nn.Conv2d(4, 4, 2, 1, bias=False) - self.iaf_C = IAFSqueeze(batch_size=1) - self.pool_C = SumPool2d(2,2) - - self.conv_D = nn.Conv2d(2, 4, 2, 1, bias=False) - self.iaf_D = IAFSqueeze(batch_size=1) - - self.conv_E = nn.Conv2d(4, 4, 2, 1, bias=False) - self.iaf_E = IAFSqueeze(batch_size=1) - self.pool_E = SumPool2d(2,2) - - self.conv_F = nn.Conv2d(4, 4, 2, 1, bias=False) - self.iaf_F = IAFSqueeze(batch_size=1) - self.pool_F = SumPool2d(2,2) - - self.flat_brach1 = nn.Flatten() - self.flat_brach2 = nn.Flatten() - self.merge = Merge() - - self.fc1 = nn.Linear(196, 200, bias=False) - self.iaf1_fc = IAFSqueeze(batch_size=1) - - self.fc2 = nn.Linear(200, 200, bias=False) - self.iaf2_fc = IAFSqueeze(batch_size=1) - - self.fc3 = nn.Linear(200, 10, bias=False) - self.iaf3_fc = IAFSqueeze(batch_size=1) - - def forward(self, x): - # conv 1 - A - conv_A_out = self.conv_A(x) - iaf_A_out = self.iaf_A(conv_A_out) - # conv 2 - B - conv_B_out = self.conv_B(iaf_A_out) - iaf_B_out = self.iaf_B(conv_B_out) - pool_B_out = self.pool_B(iaf_B_out) - # conv 3 - C - conv_C_out = self.conv_C(pool_B_out) - iaf_C_out = self.iaf_C(conv_C_out) - pool_C_out = self.pool_C(iaf_C_out) - - # --- - - # conv 4 - D - conv_D_out = self.conv_D(x) - iaf_D_out = self.iaf_D(conv_D_out) - # conv 5 - E - conv_E_out = self.conv_E(iaf_D_out) - iaf_E_out = self.iaf_E(conv_E_out) - pool_E_out = self.pool_E(iaf_E_out) - # conv 6 - F - conv_F_out = self.conv_F(pool_E_out) - iaf_F_out = self.iaf_F(conv_F_out) - pool_F_out = self.pool_F(iaf_F_out) - - # --- - - flat_brach1_out = self.flat_brach1(pool_C_out) - flat_brach2_out = self.flat_brach2(pool_F_out) - merge_out = self.merge(flat_brach1_out, flat_brach2_out) - - # FC 7 - G - fc1_out = self.fc1(merge_out) - iaf1_fc_out = self.iaf1_fc(fc1_out) - # FC 8 - H - fc2_out = self.fc2(iaf1_fc_out) - iaf2_fc_out = self.iaf2_fc(fc2_out) - # FC 9 - I - fc3_out = self.fc3(iaf2_fc_out) - iaf3_fc_out = self.iaf3_fc(fc3_out) - - return iaf3_fc_out - -class EXAMPLE_7(nn.Module): - """ This is the 'a network with a merge and a split' example in https://github.com/synsense/sinabs/issues/181 . """ - def __init__(self) -> None: - super().__init__() - - self.conv1 = nn.Conv2d(2, 4, 2, 1, bias=False) - self.iaf1 = IAFSqueeze(batch_size=1) - - self.conv2 = nn.Conv2d(4, 4, 2, 1, bias=False) - self.iaf2 = IAFSqueeze(batch_size=1) - self.pool2 = SumPool2d(2,2) - self.pool2a = SumPool2d(5,5) - - self.conv3 = nn.Conv2d(4, 4, 2, 1, bias=False) - self.iaf3 = IAFSqueeze(batch_size=1) - self.pool3 = SumPool2d(2,2) - - self.conv4 = nn.Conv2d(4, 4, 2, 1, bias=False) - self.iaf4 = IAFSqueeze(batch_size=1) - - self.flat = nn.Flatten() - self.flat_a = nn.Flatten() - - self.fc1 = nn.Linear(144, 144, bias=False) - self.iaf1_fc = IAFSqueeze(batch_size=1) - - self.fc2 = nn.Linear(144, 144, bias=False) - self.iaf2_fc = IAFSqueeze(batch_size=1) - - self.fc3 = nn.Linear(144, 10, bias=False) - self.iaf3_fc = IAFSqueeze(batch_size=1) - - # -- merges -- - self.merge1 = Merge() - - def forward(self, x): - # conv 1 - A - con1_out = self.conv1(x) - iaf1_out = self.iaf1(con1_out) - - # conv 2 - B - conv2_out = self.conv2(iaf1_out) - iaf2_out = self.iaf2(conv2_out) - pool2_out = self.pool2(iaf2_out) - pool2a_out = self.pool2a(iaf2_out) - - # conv 3 - C - conv3_out = self.conv3(pool2_out) - iaf3_out = self.iaf3(conv3_out) - pool3_out = self.pool3(iaf3_out) - - # conv 4 - D - conv4_out = self.conv4(pool3_out) - iaf4_out = self.iaf4(conv4_out) - flat_out = self.flat(iaf4_out) - - # fc 1 - E - flat_a_out = self.flat_a(pool2a_out) - fc1_out = self.fc1(flat_a_out) - iaf1_fc_out = self.iaf1_fc(fc1_out) - - # fc 2 - F - fc2_out = self.fc2(iaf1_fc_out) - iaf2_fc_out = self.iaf2_fc(fc2_out) - - # fc 2 - G - merge1_out = self.merge1(flat_out, iaf2_fc_out) - fc3_out = self.fc3(merge1_out) - iaf3_fc_out = self.iaf3_fc(fc3_out) - - return iaf3_fc_out - -class EXAMPLE_8(nn.Module): - """ This is the 'a complex network structure' example in https://github.com/synsense/sinabs/issues/181 . """ - def __init__(self) -> None: - super().__init__() - - self.conv1 = nn.Conv2d(2, 8, 2, 1, bias=False) - self.iaf1 = IAFSqueeze(batch_size=1) - - self.conv2 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf2 = IAFSqueeze(batch_size=1) - self.pool2 = SumPool2d(2,2) - - self.conv3 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf3 = IAFSqueeze(batch_size=1) - self.pool3 = SumPool2d(2,2) - self.pool3a = SumPool2d(6,6) - - self.conv4 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf4 = IAFSqueeze(batch_size=1) - self.pool4 = SumPool2d(3,3) - - self.flat = nn.Flatten() - self.flat_a = nn.Flatten() - - self.fc1 = nn.Linear(200, 200, bias=False) - self.iaf1_fc = IAFSqueeze(batch_size=1) - - self.fc2 = nn.Linear(200, 10, bias=False) - self.iaf2_fc = IAFSqueeze(batch_size=1) - - # -- merges -- - self.merge1 = Merge() - self.merge2 = Merge() - - def forward(self, x): - # conv 1 - A - con1_out = self.conv1(x) - iaf1_out = self.iaf1(con1_out) - - # conv 2 - B - conv2_out = self.conv2(iaf1_out) - iaf2_out = self.iaf2(conv2_out) - pool2_out = self.pool2(iaf2_out) - - # conv 3 - C - conv3_out = self.conv3(iaf1_out) - iaf3_out = self.iaf3(conv3_out) - pool3_out = self.pool3(iaf3_out) - pool3a_out = self.pool3a(iaf3_out) - - # conv 4 - D - merge1_out = self.merge1(pool2_out, pool3_out) - conv4_out = self.conv4(merge1_out) - iaf4_out = self.iaf4(conv4_out) - pool4_out = self.pool4(iaf4_out) - flat_out = self.flat(pool4_out) - - # fc 1 - E - flat_a_out = self.flat_a(pool3a_out) - fc1_out = self.fc1(flat_a_out) - iaf1_fc_out = self.iaf1_fc(fc1_out) - - # fc 2 - F - merge2_out = self.merge2(iaf1_fc_out, flat_out) - fc2_out = self.fc2(merge2_out) - iaf2_fc_out = self.iaf2_fc(fc2_out) - - return iaf2_fc_out \ No newline at end of file diff --git a/tests/test_dynapcnnnetwork/conftest_dynapcnnnetwork.py b/tests/test_dynapcnnnetwork/conftest_dynapcnnnetwork.py deleted file mode 100644 index 01087fad..00000000 --- a/tests/test_dynapcnnnetwork/conftest_dynapcnnnetwork.py +++ /dev/null @@ -1,563 +0,0 @@ -from architectures_samples import * - -# --- test_NIRtoDynapcnnNetwork_edges_list(snn, edges_list) --- - -edges_list_1 = [ - (0, 1), - (1, 2), - (1, 3), - (2, 4), - (3, 5), - (4, 6), - (6, 5), - (7, 8), - (8, 9), - (9, 10), - (10, 11), - (11, 12), - (12, 13), - (5, 7), -] - -edges_list_2 = [ - (0, 1), - (1, 2), - (1, 3), - (2, 4), - (3, 5), - (4, 6), - (6, 5), - (7, 8), - (8, 9), - (9, 10), - (10, 11), - (11, 12), - (12, 13), - (13, 14), - (14, 15), - (5, 7), -] - -edges_list_3 = [ - (0, 1), - (1, 2), - (1, 3), - (2, 4), - (3, 5), - (4, 6), - (6, 5), - (7, 8), - (8, 9), - (9, 10), - (10, 11), - (11, 12), - (11, 13), - (12, 14), - (14, 13), - (15, 16), - (5, 7), - (13, 15), -] - -edges_list_4 = [ - (0, 1), - (1, 2), - (1, 3), - (2, 4), - (3, 5), - (4, 6), - (6, 5), - (7, 8), - (8, 9), - (9, 10), - (10, 11), - (11, 12), - (12, 13), - (13, 14), - (13, 15), - (14, 16), - (16, 15), - (17, 18), - (5, 7), - (15, 17), -] - -edges_list_5 = [ - (0, 1), - (1, 2), - (1, 3), - (2, 4), - (3, 5), - (4, 6), - (6, 5), - (7, 8), - (8, 9), - (9, 10), - (10, 11), - (11, 12), - (12, 13), - (13, 14), - (13, 15), - (14, 16), - (16, 15), - (16, 17), - (18, 19), - (19, 17), - (20, 21), - (5, 7), - (15, 18), - (17, 20), -] - -edges_list_6 = [ - (0, 1), - (1, 2), - (2, 3), - (3, 4), - (4, 5), - (5, 6), - (6, 7), - (7, 8), - (9, 10), - (10, 11), - (11, 12), - (12, 13), - (13, 14), - (14, 15), - (15, 16), - (16, 17), - (8, 18), - (17, 18), - (18, 19), - (19, 20), - (20, 21), - (21, 22), - (22, 23), - (23, 24), -] - -edges_list_7 = [ - (0, 1), - (1, 2), - (2, 3), - (3, 4), - (3, 5), - (4, 6), - (5, 7), - (6, 8), - (8, 9), - (9, 10), - (10, 11), - (11, 12), - (12, 13), - (7, 14), - (14, 15), - (15, 16), - (16, 17), - (17, 13), - (18, 19), - (13, 18), -] - -edges_list_8 = [ - (0, 1), - (1, 2), - (1, 3), - (2, 4), - (4, 5), - (5, 6), - (3, 7), - (7, 8), - (7, 9), - (8, 6), - (9, 10), - (11, 12), - (12, 13), - (13, 14), - (14, 15), - (10, 16), - (16, 17), - (17, 15), - (18, 19), - (6, 11), - (15, 18), -] - -args_NIRtoDynapcnnNetwork_edges_list = [ - (EXAMPLE_1(), edges_list_1), - (EXAMPLE_2(), edges_list_2), - (EXAMPLE_3(), edges_list_3), - (EXAMPLE_4(), edges_list_4), - (EXAMPLE_5(), edges_list_5), - (EXAMPLE_6(), edges_list_6), - (EXAMPLE_7(), edges_list_7), - (EXAMPLE_8(), edges_list_8), - ] - -# --- test_NIRtoDynapcnnNetwork_IO(snn, io_dict) --- - -nodes_IO_1 = { - 0: {'in': torch.Size([1, 2, 34, 34]), 'out': torch.Size([1, 10, 33, 33])}, - 1: {'in': torch.Size([1, 10, 33, 33]), 'out': torch.Size([1, 10, 33, 33])}, - 2: {'in': torch.Size([1, 10, 33, 33]), 'out': torch.Size([1, 10, 11, 11])}, - 3: {'in': torch.Size([1, 10, 33, 33]), 'out': torch.Size([1, 10, 8, 8])}, - 4: {'in': torch.Size([1, 10, 11, 11]), 'out': torch.Size([1, 10, 8, 8])}, - 6: {'in': torch.Size([1, 10, 8, 8]), 'out': torch.Size([1, 10, 8, 8])}, - 7: {'in': torch.Size([1, 10, 8, 8]), 'out': torch.Size([1, 1, 7, 7])}, - 8: {'in': torch.Size([1, 1, 7, 7]), 'out': torch.Size([1, 1, 7, 7])}, - 10: {'in': torch.Size([1, 49]), 'out': torch.Size([1, 500])}, - 11: {'in': torch.Size([1, 500]), 'out': torch.Size([1, 500])}, - 12: {'in': torch.Size([1, 500]), 'out': torch.Size([1, 10])}, - 13: {'in': torch.Size([1, 10]), 'out': torch.Size([1, 10])}, -} - -nodes_IO_2 = { - 0: {'in': torch.Size([1, 2, 34, 34]), 'out': torch.Size([1, 10, 33, 33])}, - 1: {'in': torch.Size([1, 10, 33, 33]), 'out': torch.Size([1, 10, 33, 33])}, - 2: {'in': torch.Size([1, 10, 33, 33]), 'out': torch.Size([1, 10, 11, 11])}, - 3: {'in': torch.Size([1, 10, 33, 33]), 'out': torch.Size([1, 10, 8, 8])}, - 4: {'in': torch.Size([1, 10, 11, 11]), 'out': torch.Size([1, 10, 8, 8])}, - 6: {'in': torch.Size([1, 10, 8, 8]), 'out': torch.Size([1, 10, 8, 8])}, - 7: {'in': torch.Size([1, 10, 8, 8]), 'out': torch.Size([1, 1, 7, 7])}, - 8: {'in': torch.Size([1, 1, 7, 7]), 'out': torch.Size([1, 1, 7, 7])}, - 10: {'in': torch.Size([1, 49]), 'out': torch.Size([1, 100])}, - 11: {'in': torch.Size([1, 100]), 'out': torch.Size([1, 100])}, - 12: {'in': torch.Size([1, 100]), 'out': torch.Size([1, 100])}, - 13: {'in': torch.Size([1, 100]), 'out': torch.Size([1, 100])}, - 14: {'in': torch.Size([1, 100]), 'out': torch.Size([1, 10])}, - 15: {'in': torch.Size([1, 10]), 'out': torch.Size([1, 10])}, -} - -nodes_IO_3 = { - 0: {'in': torch.Size([1, 2, 34, 34]), 'out': torch.Size([1, 10, 33, 33])}, - 1: {'in': torch.Size([1, 10, 33, 33]), 'out': torch.Size([1, 10, 33, 33])}, - 2: {'in': torch.Size([1, 10, 33, 33]), 'out': torch.Size([1, 10, 11, 11])}, - 3: {'in': torch.Size([1, 10, 33, 33]), 'out': torch.Size([1, 10, 8, 8])}, - 4: {'in': torch.Size([1, 10, 11, 11]), 'out': torch.Size([1, 10, 8, 8])}, - 6: {'in': torch.Size([1, 10, 8, 8]), 'out': torch.Size([1, 10, 8, 8])}, - 7: {'in': torch.Size([1, 10, 8, 8]), 'out': torch.Size([1, 1, 7, 7])}, - 8: {'in': torch.Size([1, 1, 7, 7]), 'out': torch.Size([1, 1, 7, 7])}, - 10: {'in': torch.Size([1, 49]), 'out': torch.Size([1, 100])}, - 11: {'in': torch.Size([1, 100]), 'out': torch.Size([1, 100])}, - 12: {'in': torch.Size([1, 100]), 'out': torch.Size([1, 100])}, - 14: {'in': torch.Size([1, 100]), 'out': torch.Size([1, 100])}, - 15: {'in': torch.Size([1, 100]), 'out': torch.Size([1, 10])}, - 16: {'in': torch.Size([1, 10]), 'out': torch.Size([1, 10])}, -} - -nodes_IO_4 = { - 0: {'in': torch.Size([1, 2, 34, 34]), 'out': torch.Size([1, 10, 33, 33])}, - 1: {'in': torch.Size([1, 10, 33, 33]), 'out': torch.Size([1, 10, 33, 33])}, - 2: {'in': torch.Size([1, 10, 33, 33]), 'out': torch.Size([1, 10, 11, 11])}, - 3: {'in': torch.Size([1, 10, 33, 33]), 'out': torch.Size([1, 10, 8, 8])}, - 4: {'in': torch.Size([1, 10, 11, 11]), 'out': torch.Size([1, 10, 8, 8])}, - 6: {'in': torch.Size([1, 10, 8, 8]), 'out': torch.Size([1, 10, 8, 8])}, - 7: {'in': torch.Size([1, 10, 8, 8]), 'out': torch.Size([1, 10, 7, 7])}, - 8: {'in': torch.Size([1, 10, 7, 7]), 'out': torch.Size([1, 10, 7, 7])}, - 9: {'in': torch.Size([1, 10, 7, 7]), 'out': torch.Size([1, 1, 6, 6])}, - 10: {'in': torch.Size([1, 1, 6, 6]), 'out': torch.Size([1, 1, 6, 6])}, - 12: {'in': torch.Size([1, 36]), 'out': torch.Size([1, 100])}, - 13: {'in': torch.Size([1, 100]), 'out': torch.Size([1, 100])}, - 14: {'in': torch.Size([1, 100]), 'out': torch.Size([1, 100])}, - 16: {'in': torch.Size([1, 100]), 'out': torch.Size([1, 100])}, - 17: {'in': torch.Size([1, 100]), 'out': torch.Size([1, 10])}, - 18: {'in': torch.Size([1, 10]), 'out': torch.Size([1, 10])}, -} - -nodes_IO_5 = { - 0: {'in': torch.Size([1, 2, 34, 34]), 'out': torch.Size([1, 10, 33, 33])}, - 1: {'in': torch.Size([1, 10, 33, 33]), 'out': torch.Size([1, 10, 33, 33])}, - 2: {'in': torch.Size([1, 10, 33, 33]), 'out': torch.Size([1, 10, 11, 11])}, - 3: {'in': torch.Size([1, 10, 33, 33]), 'out': torch.Size([1, 10, 8, 8])}, - 4: {'in': torch.Size([1, 10, 11, 11]), 'out': torch.Size([1, 10, 8, 8])}, - 6: {'in': torch.Size([1, 10, 8, 8]), 'out': torch.Size([1, 10, 8, 8])}, - 7: {'in': torch.Size([1, 10, 8, 8]), 'out': torch.Size([1, 10, 7, 7])}, - 8: {'in': torch.Size([1, 10, 7, 7]), 'out': torch.Size([1, 10, 7, 7])}, - 9: {'in': torch.Size([1, 10, 7, 7]), 'out': torch.Size([1, 1, 6, 6])}, - 10: {'in': torch.Size([1, 1, 6, 6]), 'out': torch.Size([1, 1, 6, 6])}, - 12: {'in': torch.Size([1, 36]), 'out': torch.Size([1, 100])}, - 13: {'in': torch.Size([1, 100]), 'out': torch.Size([1, 100])}, - 14: {'in': torch.Size([1, 100]), 'out': torch.Size([1, 100])}, - 16: {'in': torch.Size([1, 100]), 'out': torch.Size([1, 100])}, - 18: {'in': torch.Size([1, 100]), 'out': torch.Size([1, 100])}, - 19: {'in': torch.Size([1, 100]), 'out': torch.Size([1, 100])}, - 20: {'in': torch.Size([1, 100]), 'out': torch.Size([1, 10])}, - 21: {'in': torch.Size([1, 10]), 'out': torch.Size([1, 10])}, -} - -nodes_IO_6 = { - 0: {'in': torch.Size([1, 2, 34, 34]), 'out': torch.Size([1, 4, 33, 33])}, - 1: {'in': torch.Size([1, 4, 33, 33]), 'out': torch.Size([1, 4, 33, 33])}, - 2: {'in': torch.Size([1, 4, 33, 33]), 'out': torch.Size([1, 4, 32, 32])}, - 3: {'in': torch.Size([1, 4, 32, 32]), 'out': torch.Size([1, 4, 32, 32])}, - 4: {'in': torch.Size([1, 4, 32, 32]), 'out': torch.Size([1, 4, 16, 16])}, - 5: {'in': torch.Size([1, 4, 16, 16]), 'out': torch.Size([1, 4, 15, 15])}, - 6: {'in': torch.Size([1, 4, 15, 15]), 'out': torch.Size([1, 4, 15, 15])}, - 7: {'in': torch.Size([1, 4, 15, 15]), 'out': torch.Size([1, 4, 7, 7])}, - 17: {'in': torch.Size([1, 196]), 'out': torch.Size([1, 200])}, - 18: {'in': torch.Size([1, 200]), 'out': torch.Size([1, 200])}, - 8: {'in': torch.Size([1, 2, 34, 34]), 'out': torch.Size([1, 4, 33, 33])}, - 9: {'in': torch.Size([1, 4, 33, 33]), 'out': torch.Size([1, 4, 33, 33])}, - 10: {'in': torch.Size([1, 4, 33, 33]), 'out': torch.Size([1, 4, 32, 32])}, - 11: {'in': torch.Size([1, 4, 32, 32]), 'out': torch.Size([1, 4, 32, 32])}, - 12: {'in': torch.Size([1, 4, 32, 32]), 'out': torch.Size([1, 4, 16, 16])}, - 13: {'in': torch.Size([1, 4, 16, 16]), 'out': torch.Size([1, 4, 15, 15])}, - 14: {'in': torch.Size([1, 4, 15, 15]), 'out': torch.Size([1, 4, 15, 15])}, - 15: {'in': torch.Size([1, 4, 15, 15]), 'out': torch.Size([1, 4, 7, 7])}, - 19: {'in': torch.Size([1, 200]), 'out': torch.Size([1, 200])}, - 20: {'in': torch.Size([1, 200]), 'out': torch.Size([1, 200])}, - 21: {'in': torch.Size([1, 200]), 'out': torch.Size([1, 11])}, - 22: {'in': torch.Size([1, 11]), 'out': torch.Size([1, 11])}, -} - -nodes_IO_7 = { - 0: {'in': torch.Size([1, 2, 34, 34]), 'out': torch.Size([1, 4, 33, 33])}, - 1: {'in': torch.Size([1, 4, 33, 33]), 'out': torch.Size([1, 4, 33, 33])}, - 2: {'in': torch.Size([1, 4, 33, 33]), 'out': torch.Size([1, 4, 32, 32])}, - 3: {'in': torch.Size([1, 4, 32, 32]), 'out': torch.Size([1, 4, 32, 32])}, - 4: {'in': torch.Size([1, 4, 32, 32]), 'out': torch.Size([1, 4, 16, 16])}, - 5: {'in': torch.Size([1, 4, 32, 32]), 'out': torch.Size([1, 4, 6, 6])}, - 6: {'in': torch.Size([1, 4, 16, 16]), 'out': torch.Size([1, 4, 15, 15])}, - 7: {'in': torch.Size([1, 4, 15, 15]), 'out': torch.Size([1, 4, 15, 15])}, - 8: {'in': torch.Size([1, 4, 15, 15]), 'out': torch.Size([1, 4, 7, 7])}, - 12: {'in': torch.Size([1, 144]), 'out': torch.Size([1, 144])}, - 13: {'in': torch.Size([1, 144]), 'out': torch.Size([1, 144])}, - 9: {'in': torch.Size([1, 4, 7, 7]), 'out': torch.Size([1, 4, 6, 6])}, - 10: {'in': torch.Size([1, 4, 6, 6]), 'out': torch.Size([1, 4, 6, 6])}, - 16: {'in': torch.Size([1, 144]), 'out': torch.Size([1, 11])}, - 17: {'in': torch.Size([1, 11]), 'out': torch.Size([1, 11])}, - 14: {'in': torch.Size([1, 144]), 'out': torch.Size([1, 144])}, - 15: {'in': torch.Size([1, 144]), 'out': torch.Size([1, 144])}, -} - -nodes_IO_8 = { - 0: {'in': torch.Size([1, 2, 34, 34]), 'out': torch.Size([1, 8, 33, 33])}, - 1: {'in': torch.Size([1, 8, 33, 33]), 'out': torch.Size([1, 8, 33, 33])}, - 2: {'in': torch.Size([1, 8, 33, 33]), 'out': torch.Size([1, 8, 32, 32])}, - 4: {'in': torch.Size([1, 8, 32, 32]), 'out': torch.Size([1, 8, 32, 32])}, - 5: {'in': torch.Size([1, 8, 32, 32]), 'out': torch.Size([1, 8, 16, 16])}, - 3: {'in': torch.Size([1, 8, 33, 33]), 'out': torch.Size([1, 8, 32, 32])}, - 7: {'in': torch.Size([1, 8, 32, 32]), 'out': torch.Size([1, 8, 32, 32])}, - 8: {'in': torch.Size([1, 8, 32, 32]), 'out': torch.Size([1, 8, 16, 16])}, - 9: {'in': torch.Size([1, 8, 32, 32]), 'out': torch.Size([1, 8, 5, 5])}, - 10: {'in': torch.Size([1, 8, 16, 16]), 'out': torch.Size([1, 8, 15, 15])}, - 11: {'in': torch.Size([1, 8, 15, 15]), 'out': torch.Size([1, 8, 15, 15])}, - 12: {'in': torch.Size([1, 8, 15, 15]), 'out': torch.Size([1, 8, 5, 5])}, - 14: {'in': torch.Size([1, 200]), 'out': torch.Size([1, 200])}, - 15: {'in': torch.Size([1, 200]), 'out': torch.Size([1, 200])}, - 16: {'in': torch.Size([1, 200]), 'out': torch.Size([1, 11])}, - 17: {'in': torch.Size([1, 11]), 'out': torch.Size([1, 11])}, -} - -args_test_NIRtoDynapcnnNetwork_IO = [ - (EXAMPLE_1(), nodes_IO_1), - (EXAMPLE_2(), nodes_IO_2), - (EXAMPLE_3(), nodes_IO_3), - (EXAMPLE_4(), nodes_IO_4), - (EXAMPLE_5(), nodes_IO_5), - (EXAMPLE_6(), nodes_IO_6), - (EXAMPLE_7(), nodes_IO_7), - (EXAMPLE_8(), nodes_IO_8), -] - -# --- test_DynapcnnLyers_edges_list(snn, edges_list) --- - -dcnnl_edges_list_1 = [ - (0, 1), - (0, 2), - (1, 2), - (2, 3), - (3, 4), -] - -dcnnl_edges_list_2 = [ - (0, 1), - (0, 2), - (1, 2), - (2, 3), - (3, 4), - (4, 5), -] - -dcnnl_edges_list_3 = [ - (0, 1), - (0, 2), - (1, 2), - (2, 3), - (3, 4), - (3, 5), - (4, 5), -] - -dcnnl_edges_list_4 = [ - (0, 1), - (0, 2), - (1, 2), - (2, 3), - (3, 4), - (4, 5), - (4, 6), - (5, 6), -] - -dcnnl_edges_list_5 = [ - (0, 1), - (0, 2), - (1, 2), - (2, 3), - (3, 4), - (4, 5), - (4, 6), - (5, 6), - (5, 7), - (6, 7), -] - -dcnnl_edges_list_6 = [ - (0, 1), - (1, 2), - (2, 3), - (3, 7), - (4, 5), - (5, 6), - (6, 3), - (7, 8), -] - -dcnnl_edges_list_7 = [ - (0, 1), - (1, 2), - (1, 3), - (2, 4), - (3, 6), - (4, 5), - (6, 5), -] - -dcnnl_edges_list_8 = [ - (0, 1), - (0, 2), - (1, 3), - (2, 3), - (2, 4), - (3, 5), - (4, 5), -] - -args_DynapcnnLyers_edges_list = [ - (EXAMPLE_1(), dcnnl_edges_list_1), - (EXAMPLE_2(), dcnnl_edges_list_2), - (EXAMPLE_3(), dcnnl_edges_list_3), - (EXAMPLE_4(), dcnnl_edges_list_4), - (EXAMPLE_5(), dcnnl_edges_list_5), - (EXAMPLE_6(), dcnnl_edges_list_6), - (EXAMPLE_7(), dcnnl_edges_list_7), - (EXAMPLE_8(), dcnnl_edges_list_8), -] - -# --- test_DynapcnnNetwork_forward_edges(snn, forward_edges_list) --- - -forward_edges_list_1 = [ - (0, '0_pool0'), - (0, '0_pool1'), - ('0_pool0', 1), - (('0_pool1', 1), 'merge_0'), - ('merge_0', 2), - (2, 3), - (3, 4), -] - -forward_edges_list_2 = [ - (0, '0_pool0'), - (0, '0_pool1'), - ('0_pool0', 1), - (('0_pool1', 1), 'merge_0'), - ('merge_0', 2), - (2, 3), - (3, 4), - (4, 5), -] - -forward_edges_list_3 = [ - (0, '0_pool0'), - (0, '0_pool1'), - ('0_pool0', 1), - (('0_pool1', 1), 'merge_0'), - ('merge_0', 2), - (2, 3), - (3, 4), - ((3, 4), 'merge_1'), - ('merge_1', 5), -] - -forward_edges_list_4 = [ - (0, '0_pool0'), - (0, '0_pool1'), - ('0_pool0', 1), - (('0_pool1', 1), 'merge_0'), - ('merge_0', 2), - (2, 3), - (3, 4), - (4, 5), - ((4, 5), 'merge_1'), - ('merge_1', 6), -] - -forward_edges_list_5 = [ - (0, '0_pool0'), - (0, '0_pool1'), - ('0_pool0', 1), - (('0_pool1', 1), 'merge_0'), - ('merge_0', 2), - (2, 3), - (3, 4), - (4, 5), - ((4, 5), 'merge_1'), - ('merge_1', 6), - ((5, 6), 'merge_2'), - ('merge_2', 7), -] - -forward_edges_list_6 = [ - (0, 1), - (1, 2), - ((2, 6), 'merge_0'), - ('merge_0', 3), - (3, 7), - (4, 5), - (5, 6), - (7, 8), -] - -forward_edges_list_7 = [ - (1, '1_pool0'), - (1, '1_pool1'), - ('1_pool0', 2), - ('1_pool1', 3), - (2, 4), - (3, 6), - ((4, 6), 'merge_0'), - ('merge_0', 5), -] - -forward_edges_list_8 = [ - (0, 1), - (2, '2_pool0'), - (2, '2_pool1'), - (('2_pool0', 1), 'merge_0'), - ('merge_0', 3), - ('2_pool1', 4), - ((3, 4), 'merge_1'), - ('merge_1', 5), -] - -args_DynapcnnNetwork_forward_edges = [ - (EXAMPLE_1(), forward_edges_list_1), - (EXAMPLE_2(), forward_edges_list_2), - (EXAMPLE_3(), forward_edges_list_3), - (EXAMPLE_4(), forward_edges_list_4), - (EXAMPLE_5(), forward_edges_list_5), - (EXAMPLE_6(), forward_edges_list_6), - (EXAMPLE_7(), forward_edges_list_7), - (EXAMPLE_8(), forward_edges_list_8), -] \ No newline at end of file diff --git a/tests/test_dynapcnnnetwork/test_dynapcnnnetwork.py b/tests/test_dynapcnnnetwork/test_dynapcnnnetwork.py deleted file mode 100644 index 6ee0de51..00000000 --- a/tests/test_dynapcnnnetwork/test_dynapcnnnetwork.py +++ /dev/null @@ -1,84 +0,0 @@ -import pytest, sys -import torch -from conftest_dynapcnnnetwork import args_NIRtoDynapcnnNetwork_edges_list, args_test_NIRtoDynapcnnNetwork_IO, args_DynapcnnLyers_edges_list, args_DynapcnnNetwork_forward_edges - -sys.path.append('../../sinabs/backend/dynapcnn') - -@pytest.mark.parametrize("snn, edges_list", args_NIRtoDynapcnnNetwork_edges_list) -def test_NIRtoDynapcnnNetwork_edges_list(snn, edges_list): - from NIRGraphExtractor import NIRtoDynapcnnNetworkGraph - - batch_size = 1 - channels = 2 - height = 34 - width = 34 - - input_shape = (batch_size, channels, height, width) - dummy_input = torch.randn(input_shape) - - graph_tracer = NIRtoDynapcnnNetworkGraph(spiking_model = snn, dummy_input = dummy_input) - - assert graph_tracer.get_edges_list() == edges_list - -@pytest.mark.parametrize("snn, io_dict", args_test_NIRtoDynapcnnNetwork_IO) -def test_NIRtoDynapcnnNetwork_IO(snn, io_dict): - from NIRGraphExtractor import NIRtoDynapcnnNetworkGraph - - batch_size = 1 - channels = 2 - height = 34 - width = 34 - - input_shape = (batch_size, channels, height, width) - dummy_input = torch.randn(input_shape) - - graph_tracer = NIRtoDynapcnnNetworkGraph(spiking_model = snn, dummy_input = dummy_input) - - computed_IOs = {} - - for node, IO in io_dict.items(): - _in, _out = graph_tracer.get_node_io_shapes(node) - - computed_IOs[node] = {'in': _in, 'out': _out} - - assert computed_IOs == io_dict - -@pytest.mark.parametrize("snn, dcnnl_edges_list", args_DynapcnnLyers_edges_list) -def test_DynapcnnLyers_edges_list(snn, dcnnl_edges_list): - from sinabs.backend.dynapcnn import DynapcnnNetworkGraph - - channels = 2 - height = 34 - width = 34 - - input_shape = (channels, height, width) - - hw_model = DynapcnnNetworkGraph( - snn, - discretize=True, - input_shape=input_shape - ) - - computed_edges_list = hw_model.get_dynapcnnlayers_edges() - - assert computed_edges_list == dcnnl_edges_list - -@pytest.mark.parametrize("snn, forward_edges_list", args_DynapcnnNetwork_forward_edges) -def test_DynapcnnNetwork_forward_edges(snn, forward_edges_list): - from sinabs.backend.dynapcnn import DynapcnnNetworkGraph - - channels = 2 - height = 34 - width = 34 - - input_shape = (channels, height, width) - - hw_model = DynapcnnNetworkGraph( - snn, - discretize=True, - input_shape=input_shape - ) - - computed_forward_edges_list = hw_model.get_network_module().get_forward_edges() - - assert computed_forward_edges_list == forward_edges_list \ No newline at end of file From 34caea980ab74435b75940556e51ff99a9c9e687 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Tue, 28 May 2024 15:44:16 +0200 Subject: [PATCH 109/379] Unit tests (complete) - Testing 1st step in SNN to DynapcnnNetwork conversion: graph extracted from original SNN (via NIRGraphExtractor class). - Testing 2nd step in SNN to DynapcnnNetwork conversion: using graph extracted from original SNN to create sets of layers to be combined into DynapcnnLayer instances. - Testing 3rd step in SNN to DynapcnnNetwork conversion: using DynapcnnLayer instances and their connectivity to compose the forward method of the DynapcnnNetwork. --- .../conftest_dynapcnnlayer.py | 3 + tests/test_dynapcnnlayer/model_dummy_1.py | 4 +- tests/test_dynapcnnlayer/model_dummy_2.py | 2 + tests/test_dynapcnnlayer/model_dummy_3.py | 2 + tests/test_dynapcnnlayer/model_dummy_4.py | 2 + .../test_dynapcnnlayer/test_dynapcnnlayer.py | 3 + .../conftest_dynapcnnnetwork.py | 14 ++ tests/test_dynapcnnnetwork/model_dummy_1.py | 78 ++++++++++++ tests/test_dynapcnnnetwork/model_dummy_2.py | 106 ++++++++++++++++ tests/test_dynapcnnnetwork/model_dummy_3.py | 120 ++++++++++++++++++ tests/test_dynapcnnnetwork/model_dummy_4.py | 105 +++++++++++++++ .../test_dynapcnnnetwork.py | 38 ++++++ .../test_graph_extractor.py | 5 +- 13 files changed, 480 insertions(+), 2 deletions(-) create mode 100644 tests/test_dynapcnnnetwork/conftest_dynapcnnnetwork.py create mode 100644 tests/test_dynapcnnnetwork/model_dummy_1.py create mode 100644 tests/test_dynapcnnnetwork/model_dummy_2.py create mode 100644 tests/test_dynapcnnnetwork/model_dummy_3.py create mode 100644 tests/test_dynapcnnnetwork/model_dummy_4.py create mode 100644 tests/test_dynapcnnnetwork/test_dynapcnnnetwork.py diff --git a/tests/test_dynapcnnlayer/conftest_dynapcnnlayer.py b/tests/test_dynapcnnlayer/conftest_dynapcnnlayer.py index 8e74e63d..a55c6fbf 100644 --- a/tests/test_dynapcnnlayer/conftest_dynapcnnlayer.py +++ b/tests/test_dynapcnnlayer/conftest_dynapcnnlayer.py @@ -1,3 +1,6 @@ +# author : Willian Soares Girao +# contact : wsoaresgirao@gmail.com + from model_dummy_1 import nodes_to_dcnnl_map_1, sinabs_edges_1, expected_output_1 from model_dummy_2 import nodes_to_dcnnl_map_2, sinabs_edges_2, expected_output_2 from model_dummy_3 import nodes_to_dcnnl_map_3, sinabs_edges_3, expected_output_3 diff --git a/tests/test_dynapcnnlayer/model_dummy_1.py b/tests/test_dynapcnnlayer/model_dummy_1.py index 65989250..dd779af7 100644 --- a/tests/test_dynapcnnlayer/model_dummy_1.py +++ b/tests/test_dynapcnnlayer/model_dummy_1.py @@ -1,4 +1,6 @@ -# implementing "a network with residual connections" example in https://github.com/synsense/sinabs/issues/181 . """ +# author : Willian Soares Girao +# contact : wsoaresgirao@gmail.com +# implementing "a network with residual connections" example in https://github.com/synsense/sinabs/issues/181 import torch.nn as nn from sinabs.layers import IAFSqueeze, SumPool2d diff --git a/tests/test_dynapcnnlayer/model_dummy_2.py b/tests/test_dynapcnnlayer/model_dummy_2.py index 9570c2f1..b15a4cb7 100644 --- a/tests/test_dynapcnnlayer/model_dummy_2.py +++ b/tests/test_dynapcnnlayer/model_dummy_2.py @@ -1,3 +1,5 @@ +# author : Willian Soares Girao +# contact : wsoaresgirao@gmail.com # implementing "a network with a merge and a split" in https://github.com/synsense/sinabs/issues/181 import torch.nn as nn diff --git a/tests/test_dynapcnnlayer/model_dummy_3.py b/tests/test_dynapcnnlayer/model_dummy_3.py index 2d541751..487d6d93 100644 --- a/tests/test_dynapcnnlayer/model_dummy_3.py +++ b/tests/test_dynapcnnlayer/model_dummy_3.py @@ -1,3 +1,5 @@ +# author : Willian Soares Girao +# contact : wsoaresgirao@gmail.com # implementing "two networks with merging outputs" in https://github.com/synsense/sinabs/issues/181 import torch.nn as nn diff --git a/tests/test_dynapcnnlayer/model_dummy_4.py b/tests/test_dynapcnnlayer/model_dummy_4.py index 8e08ccb7..3dca038c 100644 --- a/tests/test_dynapcnnlayer/model_dummy_4.py +++ b/tests/test_dynapcnnlayer/model_dummy_4.py @@ -1,3 +1,5 @@ +# author : Willian Soares Girao +# contact : wsoaresgirao@gmail.com # implementing "a complex network structure" example in https://github.com/synsense/sinabs/issues/181 . """ import torch.nn as nn diff --git a/tests/test_dynapcnnlayer/test_dynapcnnlayer.py b/tests/test_dynapcnnlayer/test_dynapcnnlayer.py index f6bf6f26..43cdba70 100644 --- a/tests/test_dynapcnnlayer/test_dynapcnnlayer.py +++ b/tests/test_dynapcnnlayer/test_dynapcnnlayer.py @@ -1,3 +1,6 @@ +# author : Willian Soares Girao +# contact : wsoaresgirao@gmail.com + import pytest from sinabs.backend.dynapcnn.utils import construct_dynapcnnlayer, update_nodes_io from sinabs.backend.dynapcnn.weight_rescaling_methods import rescale_method_1 diff --git a/tests/test_dynapcnnnetwork/conftest_dynapcnnnetwork.py b/tests/test_dynapcnnnetwork/conftest_dynapcnnnetwork.py new file mode 100644 index 00000000..92f528ce --- /dev/null +++ b/tests/test_dynapcnnnetwork/conftest_dynapcnnnetwork.py @@ -0,0 +1,14 @@ +# author : Willian Soares Girao +# contact : wsoaresgirao@gmail.com + +from model_dummy_1 import snn as snn_1, input_shape as input_shape_1, batch_size as batch_size_1, expected_output as expected_output_1 +from model_dummy_2 import snn as snn_2, input_shape as input_shape_2, batch_size as batch_size_2, expected_output as expected_output_2 +from model_dummy_3 import snn as snn_3, input_shape as input_shape_3, batch_size as batch_size_3, expected_output as expected_output_3 +from model_dummy_4 import snn as snn_4, input_shape as input_shape_4, batch_size as batch_size_4, expected_output as expected_output_4 + +args_DynapcnnNetworkTest = [ + (snn_1, input_shape_1, batch_size_1, expected_output_1), + (snn_2, input_shape_2, batch_size_2, expected_output_2), + (snn_3, input_shape_3, batch_size_3, expected_output_3), + (snn_4, input_shape_4, batch_size_4, expected_output_4), +] \ No newline at end of file diff --git a/tests/test_dynapcnnnetwork/model_dummy_1.py b/tests/test_dynapcnnnetwork/model_dummy_1.py new file mode 100644 index 00000000..348f2299 --- /dev/null +++ b/tests/test_dynapcnnnetwork/model_dummy_1.py @@ -0,0 +1,78 @@ +# author : Willian Soares Girao +# contact : wsoaresgirao@gmail.com +# implementing "a network with residual connections" example in https://github.com/synsense/sinabs/issues/181 + +import torch +import torch.nn as nn +from sinabs.layers import Merge, IAFSqueeze +from sinabs.activation.surrogate_gradient_fn import PeriodicExponential + +class SNN(nn.Module): + def __init__(self, batch_size) -> None: + super().__init__() + + self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False) # node 0 + self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) # node 1 + self.pool1 = nn.AvgPool2d(3,3) # node 2 + self.pool1a = nn.AvgPool2d(4,4) # node 3 + + self.conv2 = nn.Conv2d(10, 10, 4, 1, bias=False)# node 4 + self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) # node 6 + + self.conv3 = nn.Conv2d(10, 1, 2, 1, bias=False) # node 8 + self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) # node 9 + + self.flat = nn.Flatten() + + self.fc1 = nn.Linear(49, 500, bias=False) # node 10 + self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) # node 11 + + self.fc2 = nn.Linear(500, 10, bias=False) # node 12 + self.iaf5 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) # node 13 + + self.adder = Merge() + + def forward(self, x): + + con1_out = self.conv1(x) + iaf1_out = self.iaf1(con1_out) + pool1_out = self.pool1(iaf1_out) + pool1a_out = self.pool1a(iaf1_out) + + conv2_out = self.conv2(pool1_out) + iaf2_out = self.iaf2(conv2_out) + + conv3_out = self.conv3(self.adder(pool1a_out, iaf2_out)) + iaf3_out = self.iaf3(conv3_out) + + flat_out = self.flat(iaf3_out) + + fc1_out = self.fc1(flat_out) + iaf4_out = self.iaf4(fc1_out) + fc2_out = self.fc2(iaf4_out) + iaf5_out = self.iaf5(fc2_out) + + return iaf5_out + +channels = 2 +height = 34 +width = 34 +batch_size = 3 +input_shape = (channels, height, width) + +snn = SNN(batch_size) + +expected_output = { + 'dcnnl_edges': [ + (0, 1), + (0, 2), + (1, 2), + (2, 3), + (3, 4), + ('input', 0), + ], + 'merge_points': {2: {'sources': (0, 1), 'merge': Merge()}}, + 'topological_order': [0, 1, 2, 3, 4], + 'output_shape': torch.Size([3, 10, 1, 1]), + 'entry_point': [0], +} \ No newline at end of file diff --git a/tests/test_dynapcnnnetwork/model_dummy_2.py b/tests/test_dynapcnnnetwork/model_dummy_2.py new file mode 100644 index 00000000..a83b54a2 --- /dev/null +++ b/tests/test_dynapcnnnetwork/model_dummy_2.py @@ -0,0 +1,106 @@ +# author : Willian Soares Girao +# contact : wsoaresgirao@gmail.com +# implementing "a network with a merge and a split" in https://github.com/synsense/sinabs/issues/181 + +import torch +import torch.nn as nn +from sinabs.layers import Merge, IAFSqueeze, SumPool2d +from sinabs.activation.surrogate_gradient_fn import PeriodicExponential + +class SNN(nn.Module): + def __init__(self, batch_size) -> None: + super().__init__() + # -- graph node A -- + self.conv_A = nn.Conv2d(2, 4, 2, 1, bias=False) + self.iaf_A = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + # -- graph node B -- + self.conv_B = nn.Conv2d(4, 4, 2, 1, bias=False) + self.iaf2_B = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + self.pool_B = SumPool2d(2,2) + # -- graph node C -- + self.conv_C = nn.Conv2d(4, 4, 2, 1, bias=False) + self.iaf_C = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + self.pool_C = SumPool2d(2,2) + # -- graph node D -- + self.conv_D = nn.Conv2d(4, 4, 2, 1, bias=False) + self.iaf_D = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + # -- graph node E -- + self.conv_E = nn.Conv2d(4, 4, 2, 1, bias=False) + self.iaf3_E = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + self.pool_E = SumPool2d(2,2) + # -- graph node F -- + self.conv_F = nn.Conv2d(4, 4, 2, 1, bias=False) + self.iaf_F = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + # -- graph node G -- + self.fc3 = nn.Linear(144, 10, bias=False) + self.iaf3_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + + # -- merges -- + self.merge1 = Merge() + + # -- falts -- + self.flat_D = nn.Flatten() + self.flat_F = nn.Flatten() + + def forward(self, x): + # conv 1 - A/0 + convA_out = self.conv_A(x) # node 0 + iaf_A_out = self.iaf_A(convA_out) # node 1 + + # conv 2 - B/1 + conv_B_out = self.conv_B(iaf_A_out) # node 2 + iaf_B_out = self.iaf2_B(conv_B_out) # node 3 + pool_B_out = self.pool_B(iaf_B_out) # node 4 + + # conv 3 - C/2 + conv_C_out = self.conv_C(pool_B_out) # node 5 + iaf_C_out = self.iaf_C(conv_C_out) # node 7 + pool_C_out = self.pool_C(iaf_C_out) # node 8 + + # conv 4 - D/4 + conv_D_out = self.conv_D(pool_C_out) # node 9 + iaf_D_out = self.iaf_D(conv_D_out) # node 10 + + # fc 1 - E/3 + conv_E_out = self.conv_E(pool_B_out) # node 6 + iaf3_E_out = self.iaf3_E(conv_E_out) # node 12 + pool_E_out = self.pool_E(iaf3_E_out) # node 13 + + # fc 2 - F/6 + conv_F_out = self.conv_F(pool_E_out) # node 14 + iaf_F_out = self.iaf_F(conv_F_out) # node 15 + + # fc 2 - G/5 + flat_D_out = self.flat_D(iaf_D_out) # node 11 + flat_F_out = self.flat_F(iaf_F_out) # node 16 + + merge1_out = self.merge1(flat_D_out, flat_F_out) # node 19 + fc3_out = self.fc3(merge1_out) # node 17 + iaf3_fc_out = self.iaf3_fc(fc3_out) # node 18 + + return iaf3_fc_out + +channels = 2 +height = 34 +width = 34 +batch_size = 8 +input_shape = (channels, height, width) + +snn = SNN(batch_size) + +expected_output = { + 'dcnnl_edges': [ + (0, 1), + (1, 2), + (1, 3), + (2, 4), + (3, 6), + (4, 5), + (6, 5), + ('input', 0), + ], + 'merge_points': {5: {'sources': (4, 6), 'merge': Merge()}}, + 'topological_order': [0, 1, 2, 3, 4, 6, 5], + 'output_shape': torch.Size([8, 10, 1, 1]), + 'entry_point': [0], +} \ No newline at end of file diff --git a/tests/test_dynapcnnnetwork/model_dummy_3.py b/tests/test_dynapcnnnetwork/model_dummy_3.py new file mode 100644 index 00000000..a5eb1ca1 --- /dev/null +++ b/tests/test_dynapcnnnetwork/model_dummy_3.py @@ -0,0 +1,120 @@ + +# author : Willian Soares Girao +# contact : wsoaresgirao@gmail.com +# implementing "two networks with merging outputs" in https://github.com/synsense/sinabs/issues/181 + +import torch +import torch.nn as nn +from sinabs.layers import IAFSqueeze, SumPool2d, Merge +from sinabs.activation.surrogate_gradient_fn import PeriodicExponential + +class SNN(nn.Module): + def __init__(self, batch_size) -> None: + super().__init__() + + self.conv_A = nn.Conv2d(2, 4, 2, 1, bias=False) + self.iaf_A = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + + self.conv_B = nn.Conv2d(4, 4, 2, 1, bias=False) + self.iaf_B = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + self.pool_B = SumPool2d(2,2) + + self.conv_C = nn.Conv2d(4, 4, 2, 1, bias=False) + self.iaf_C = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + self.pool_C = SumPool2d(2,2) + + self.conv_D = nn.Conv2d(2, 4, 2, 1, bias=False) + self.iaf_D = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + + self.conv_E = nn.Conv2d(4, 4, 2, 1, bias=False) + self.iaf_E = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + self.pool_E = SumPool2d(2,2) + + self.conv_F = nn.Conv2d(4, 4, 2, 1, bias=False) + self.iaf_F = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + self.pool_F = SumPool2d(2,2) + + self.flat_brach1 = nn.Flatten() + self.flat_brach2 = nn.Flatten() + self.merge = Merge() + + self.fc1 = nn.Linear(196, 100, bias=False) + self.iaf1_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + + self.fc2 = nn.Linear(100, 100, bias=False) + self.iaf2_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + + self.fc3 = nn.Linear(100, 10, bias=False) + self.iaf3_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + + def forward(self, x): + # conv 1 - A + conv_A_out = self.conv_A(x) + iaf_A_out = self.iaf_A(conv_A_out) + # conv 2 - B + conv_B_out = self.conv_B(iaf_A_out) + iaf_B_out = self.iaf_B(conv_B_out) + pool_B_out = self.pool_B(iaf_B_out) + # conv 3 - C + conv_C_out = self.conv_C(pool_B_out) + iaf_C_out = self.iaf_C(conv_C_out) + pool_C_out = self.pool_C(iaf_C_out) + + # --- + + # conv 4 - D + conv_D_out = self.conv_D(x) + iaf_D_out = self.iaf_D(conv_D_out) + # conv 5 - E + conv_E_out = self.conv_E(iaf_D_out) + iaf_E_out = self.iaf_E(conv_E_out) + pool_E_out = self.pool_E(iaf_E_out) + # conv 6 - F + conv_F_out = self.conv_F(pool_E_out) + iaf_F_out = self.iaf_F(conv_F_out) + pool_F_out = self.pool_F(iaf_F_out) + + # --- + + flat_brach1_out = self.flat_brach1(pool_C_out) + flat_brach2_out = self.flat_brach2(pool_F_out) + merge_out = self.merge(flat_brach1_out, flat_brach2_out) + + # FC 7 - G + fc1_out = self.fc1(merge_out) + iaf1_fc_out = self.iaf1_fc(fc1_out) + # FC 8 - H + fc2_out = self.fc2(iaf1_fc_out) + iaf2_fc_out = self.iaf2_fc(fc2_out) + # FC 9 - I + fc3_out = self.fc3(iaf2_fc_out) + iaf3_fc_out = self.iaf3_fc(fc3_out) + + return iaf3_fc_out + +channels = 2 +height = 34 +width = 34 +batch_size = 2 +input_shape = (channels, height, width) + +snn = SNN(batch_size) + +expected_output = { + 'dcnnl_edges': [ + (0, 1), + (1, 2), + (2, 3), + (3, 7), + (4, 5), + (5, 6), + (6, 3), + (7, 8), + ('input', 0), + ('input', 4), + ], + 'merge_points': {3: {'sources': (2, 6), 'merge': Merge()}}, + 'topological_order': [0, 4, 1, 5, 2, 6, 3, 7, 8], + 'output_shape': torch.Size([2, 10, 1, 1]), + 'entry_point': [0, 4], +} \ No newline at end of file diff --git a/tests/test_dynapcnnnetwork/model_dummy_4.py b/tests/test_dynapcnnnetwork/model_dummy_4.py new file mode 100644 index 00000000..845301db --- /dev/null +++ b/tests/test_dynapcnnnetwork/model_dummy_4.py @@ -0,0 +1,105 @@ +# author : Willian Soares Girao +# contact : wsoaresgirao@gmail.com +# implementing "a complex network structure" example in https://github.com/synsense/sinabs/issues/181 . """ + +import torch +import torch.nn as nn +from sinabs.layers import IAFSqueeze, SumPool2d, Merge +from sinabs.activation.surrogate_gradient_fn import PeriodicExponential + +class SNN(nn.Module): + def __init__(self, batch_size) -> None: + super().__init__() + + self.conv1 = nn.Conv2d(2, 1, 2, 1, bias=False) + self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + + self.conv2 = nn.Conv2d(1, 1, 2, 1, bias=False) + self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + self.pool2 = SumPool2d(2,2) + + self.conv3 = nn.Conv2d(1, 1, 2, 1, bias=False) + self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + self.pool3 = SumPool2d(2,2) + self.pool3a = SumPool2d(5,5) + + self.conv4 = nn.Conv2d(1, 1, 2, 1, bias=False) + self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + self.pool4 = SumPool2d(3,3) + + self.flat1 = nn.Flatten() + self.flat2 = nn.Flatten() + + self.conv5 = nn.Conv2d(1, 1, 2, 1, bias=False) + self.iaf5 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + + self.fc2 = nn.Linear(25, 10, bias=False) + self.iaf2_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + + # -- merges -- + self.merge1 = Merge() + self.merge2 = Merge() + + def forward(self, x): + # conv 1 - A/0 + con1_out = self.conv1(x) + iaf1_out = self.iaf1(con1_out) + + # conv 2 - B/1 + conv2_out = self.conv2(iaf1_out) + iaf2_out = self.iaf2(conv2_out) + pool2_out = self.pool2(iaf2_out) + + # conv 3 - C/2 + conv3_out = self.conv3(iaf1_out) + iaf3_out = self.iaf3(conv3_out) + pool3_out = self.pool3(iaf3_out) + pool3a_out = self.pool3a(iaf3_out) + + # conv 4 - D/3 + merge1_out = self.merge1(pool2_out, pool3_out) + conv4_out = self.conv4(merge1_out) + iaf4_out = self.iaf4(conv4_out) + pool4_out = self.pool4(iaf4_out) + flat1_out = self.flat1(pool4_out) + + # conv 5 - E/4 + conv5_out = self.conv5(pool3a_out) + iaf5_out = self.iaf5(conv5_out) + flat2_out = self.flat2(iaf5_out) + + # fc 2 - F/5 + merge2_out = self.merge2(flat2_out, flat1_out) + + fc2_out = self.fc2(merge2_out) + iaf2_fc_out = self.iaf2_fc(fc2_out) + + return iaf2_fc_out + +channels = 2 +height = 34 +width = 34 +batch_size = 2 +input_shape = (channels, height, width) + +snn = SNN(batch_size) + +expected_output = { + 'dcnnl_edges': [ + (0, 1), + (0, 2), + (1, 3), + (2, 3), + (2, 4), + (3, 5), + (4, 5), + ('input', 0), + ], + 'merge_points': { + 3: {'sources': (1, 2), 'merge': Merge()}, + 5: {'sources': (3, 4), 'merge': Merge()}, + }, + 'topological_order': [0, 1, 2, 3, 4, 5], + 'output_shape': torch.Size([2, 10, 1, 1]), + 'entry_point': [0], +} \ No newline at end of file diff --git a/tests/test_dynapcnnnetwork/test_dynapcnnnetwork.py b/tests/test_dynapcnnnetwork/test_dynapcnnnetwork.py new file mode 100644 index 00000000..4f8f3bd7 --- /dev/null +++ b/tests/test_dynapcnnnetwork/test_dynapcnnnetwork.py @@ -0,0 +1,38 @@ +# author : Willian Soares Girao +# contact : wsoaresgirao@gmail.com + +import pytest, torch +from sinabs.backend.dynapcnn.dynapcnn_network import DynapcnnNetwork +from conftest_dynapcnnnetwork import args_DynapcnnNetworkTest + +@pytest.mark.parametrize("snn, input_shape, batch_size, expected_output", args_DynapcnnNetworkTest) +def test_DynapcnnNetwork(snn, input_shape, batch_size, expected_output): + """ Tests if the correct graph representing the connections between each DynapcnnLayer within a DynapcnnNetwork + is created; if the DynapcnnLayer instances requiring input from a `Merge` are correctly flagged (along with what + their arguments should be); if the correct topological order of the DynapcnnLayers (i.e., the order in which their + forward methods should be called) is computed; if the output of the model matches what is expected.""" + + dcnnnet = DynapcnnNetwork(snn, input_shape, batch_size) + + torch.manual_seed(0) + x = torch.randn((batch_size, *input_shape)) + output = dcnnnet(x) + + assert expected_output['dcnnl_edges'] == dcnnnet.dcnnl_edges, \ + f'wrong list of edges describing DynapcnnLayer connectivity.' + + for node, args in dcnnnet.merge_points.items(): + + assert node in expected_output['merge_points'], \ + f'DynapcnnLayer {node} is not a merge point.' + assert args['sources'] == expected_output['merge_points'][node]['sources'], \ + f'DynapcnnLayer {node} has wrong input sources ({args}).' + + for entry_point in expected_output['entry_point']: + assert dcnnnet.forward_map[entry_point].entry_point, \ + f'DynapcnnLayer {entry_point} should be an entry point.' + + assert expected_output['topological_order'] == dcnnnet.topological_order, \ + f'wrong topological ordering between DynapcnnLayers.' + assert expected_output['output_shape'] == output.shape, \ + f'wrong model output tensor shape.' \ No newline at end of file diff --git a/tests/test_graph_extractor/test_graph_extractor.py b/tests/test_graph_extractor/test_graph_extractor.py index 6b914f0c..a7693a69 100644 --- a/tests/test_graph_extractor/test_graph_extractor.py +++ b/tests/test_graph_extractor/test_graph_extractor.py @@ -8,7 +8,10 @@ @pytest.mark.parametrize("snn, input_dummy, expected_output", args_GraphExtractor) def test_GraphExtractor(snn, input_dummy, expected_output): - """ Tests the graph extraction from the original SNN being turned into a DynapcnnNetwork.""" + """ Tests the graph extraction from the original SNN being turned into a `DynapcnnNetwork`. These tests + verify the correct functionality of the `NIRtoDynapcnnNetworkGraph` class, which implements the first pre-processing + step on the conversion of the SNN into a DynapcnnNetwork. + """ graph_tracer = NIRtoDynapcnnNetworkGraph(snn, input_dummy) From 59daf6623b8fee7c0c74e9e88ace1d22d8d38dcf Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Tue, 28 May 2024 15:45:55 +0200 Subject: [PATCH 110/379] Refactor Added @property methods to access private properties for unit testing purposes --- sinabs/backend/dynapcnn/NIRGraphExtractor.py | 25 +++++++------- sinabs/backend/dynapcnn/dynapcnn_layer.py | 4 ++- sinabs/backend/dynapcnn/dynapcnn_network.py | 34 ++++++++------------ sinabs/backend/dynapcnn/utils.py | 13 -------- 4 files changed, 29 insertions(+), 47 deletions(-) diff --git a/sinabs/backend/dynapcnn/NIRGraphExtractor.py b/sinabs/backend/dynapcnn/NIRGraphExtractor.py index 7ec5a0d7..7dd970b2 100644 --- a/sinabs/backend/dynapcnn/NIRGraphExtractor.py +++ b/sinabs/backend/dynapcnn/NIRGraphExtractor.py @@ -14,10 +14,10 @@ def __init__(self, spiking_model: nn.Module, dummy_input: torch.tensor): Parameters ---------- - spiking_model (nn.Module): a sinabs-compatible spiking network. - dummy_input (torch.tensor): a random input sample to be fed through the model to acquire both - the computational graph (via `nirtorch`) and the I/O shapes of each node. Its a 4-D shape - with `(batch, channels, heigh, width)`. + - spiking_model (nn.Module): a sinabs-compatible spiking network. + - dummy_input (torch.tensor): a random input sample to be fed through the model to acquire both + the computational graph (via `nirtorch`) and the I/O shapes of each node. Its a 4-D shape + with `(batch, channels, heigh, width)`. """ # extract computational graph. @@ -25,15 +25,6 @@ def __init__(self, spiking_model: nn.Module, dummy_input: torch.tensor): # converts the NIR representation into a list of edges with nodes represented as integers. self._edges_list, self._name_2_indx_map, self._entry_nodes = self._get_edges_from_nir(nir_graph) - - # print('self._entry_nodes: ', self._entry_nodes) - - # for key, val in self._name_2_indx_map.items(): - # print(key, val) - # print('---------------------------------------------------') - # for edge in self._edges_list: - # print(edge) - # print('---------------------------------------------------') # recovers the associated `nn.Module` (layer) of each node. self.modules_map = self._get_named_modules(spiking_model) @@ -50,6 +41,14 @@ def entry_nodes(self) -> List[int]: @property def get_edges_list(self): return self._edges_list + + @property + def name_2_indx_map(self): + return self._name_2_indx_map + + @property + def nodes_io_shapes(self): + return self._nodes_io_shapes def remove_ignored_nodes(self, default_ignored_nodes): """ Recreates the edges list based on layers that 'DynapcnnNetwork' will ignore. This diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer.py b/sinabs/backend/dynapcnn/dynapcnn_layer.py index 891dbc94..48d33743 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer.py @@ -1,6 +1,8 @@ +# author : Willian Soares Girao +# contact : wsoaresgirao@gmail.com + from copy import deepcopy from typing import Dict, Callable, Tuple, Union, List -from warnings import warn import numpy as np import torch diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index 18ce6b16..976b0417 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -1,9 +1,5 @@ -""" -functionality : extracts the computational graph of a network defined as a `nn.Module` and converts it into a set of `DynapcnnLayer`s - that implement a network ()`DynapcnnNetwork`) instance that can be deployed to a Speck chip. -author : Willian Soares Girao -contact : williansoaresgirao@gmail.com -""" +# author : Willian Soares Girao +# contact : wsoaresgirao@gmail.com import time, copy from typing import List, Optional, Sequence, Tuple, Union, Dict, Callable @@ -87,7 +83,6 @@ def __init__( assert len(self.input_shape) == 3, "infer_input_shape did not return 3-tuple" # computational graph from original PyTorch module. - # TODO - bacth size must be passed as argument. self._graph_tracer = NIRtoDynapcnnNetworkGraph( snn, torch.randn((batch_size, *self.input_shape))) # needs the batch dimension. @@ -104,20 +99,10 @@ def __init__( self._nodes_to_dcnnl_map = build_nodes_to_dcnnl_map( layers=self._sinabs_modules_map, edges=self._sinabs_edges) - - # print('-----------------------------------------------------------------') - # for edge in self._sinabs_edges: - # print(edge) - # print('-----------------------------------------------------------------') # updates 'self._nodes_to_dcnnl_map' to include the I/O shape for each node. self._populate_nodes_io() - # print('-----------------------------------------------------------------') - # for key, val in self._nodes_to_dcnnl_map.items(): - # print(key, val) - # print('-----------------------------------------------------------------') - # build `DynapcnnLayer` instances from graph edges and mapper. self._dynapcnn_layers = build_from_graph( discretize = discretize, @@ -129,9 +114,6 @@ def __init__( # these gather all data necessay to implement the forward method for this class. self._dcnnl_edges, self._forward_map, self._merge_points, self._topological_order = self._get_network_module() - # for edge in self._dcnnl_edges: - # print(edge) - # all necessary `DynapcnnLayer` data held in `self._forward_map`: removing intermediary data structures no longer necessary. del self._graph_tracer del self._sinabs_edges @@ -142,6 +124,18 @@ def __init__( del self._entry_nodes ####################################################### Public Methods ####################################################### + + @property + def dcnnl_edges(self): + return self._dcnnl_edges + + @property + def merge_points(self): + return self._merge_points + + @property + def topological_order(self): + return self._topological_order @property def forward_map(self) -> Dict[int, DynapcnnLayer]: diff --git a/sinabs/backend/dynapcnn/utils.py b/sinabs/backend/dynapcnn/utils.py index 75a2ee58..21cc5c08 100644 --- a/sinabs/backend/dynapcnn/utils.py +++ b/sinabs/backend/dynapcnn/utils.py @@ -180,19 +180,6 @@ def construct_dynapcnnlayers_from_mapper( dynapcnnlayer = construct_dynapcnnlayer( dpcnnl_idx, discretize, edges, nodes_to_dcnnl_map, weight_rescaling_fn, entry_nodes) - # print('-----------------------------------------------------------------') - # print('dpcnnl_index: ', dynapcnnlayer.dpcnnl_index) - # print('conv_node_id: ', dynapcnnlayer.conv_node_id) - # print('conv_in_shape: ', dynapcnnlayer.conv_in_shape) - # print('conv_out_shape: ', dynapcnnlayer.conv_out_shape) - # print('spk_node_id: ', dynapcnnlayer.spk_node_id) - # print('pool_node_id: ', dynapcnnlayer.pool_node_id) - # print('conv_rescaling_factor: ', dynapcnnlayer.conv_rescaling_factor) - # print('dynapcnnlayer_destination: ', dynapcnnlayer.dynapcnnlayer_destination) - # print('nodes_destinations: ', dynapcnnlayer.nodes_destinations) - # print('entry_point: ', dynapcnnlayer.entry_point) - # print('-----------------------------------------------------------------') - dynapcnn_layers[dpcnnl_idx] = { 'layer': dynapcnnlayer, 'destinations': nodes_to_dcnnl_map[dpcnnl_idx]['destinations'] From 90d6d4952ec6975a7223a2b2c3b85da32073c896 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Tue, 28 May 2024 16:06:37 +0200 Subject: [PATCH 111/379] Documentation Updated function headers and type-hints. --- sinabs/backend/dynapcnn/NIRGraphExtractor.py | 63 ++++++++------------ sinabs/backend/dynapcnn/dynapcnn_network.py | 34 +++++++---- 2 files changed, 48 insertions(+), 49 deletions(-) diff --git a/sinabs/backend/dynapcnn/NIRGraphExtractor.py b/sinabs/backend/dynapcnn/NIRGraphExtractor.py index 7dd970b2..5f89001c 100644 --- a/sinabs/backend/dynapcnn/NIRGraphExtractor.py +++ b/sinabs/backend/dynapcnn/NIRGraphExtractor.py @@ -50,10 +50,19 @@ def name_2_indx_map(self): def nodes_io_shapes(self): return self._nodes_io_shapes - def remove_ignored_nodes(self, default_ignored_nodes): - """ Recreates the edges list based on layers that 'DynapcnnNetwork' will ignore. This + def remove_ignored_nodes(self, default_ignored_nodes: tuple) -> Tuple[list, dict]: + """ Recreates the edges list based on layers that `DynapcnnNetwork` will ignore. This is done by setting the source (target) node of an edge where the source (target) node will be dropped as the node that originally targeted this node to be dropped. + + Parameters + ---------- + - default_ignored_nodes (tuple): a set of layers (`nn.Module`) that should be ignored from the graph. + + Returns + ---------- + - remapped_edges (list): the new list of edges after nodes flagged by `default_ignored_nodes` have been removed. + - remapped_nodes (dict): updated nodes' IDs after nodes flagged by `default_ignored_nodes` have been removed. """ edges = copy.deepcopy(self._edges_list) parsed_edges = [] @@ -102,7 +111,13 @@ def remove_ignored_nodes(self, default_ignored_nodes): # TODO - it would be good if I/O shapes were returned by the NIR graph. def get_node_io_shapes(self, node: int) -> Tuple[torch.Size, torch.Size]: - """ Returns the I/O tensors' shapes of `node`. """ + """ Returns the I/O tensors' shapes of `node`. + + Returns + ---------- + - input shape (torch.Size): shape of the input tensor to `node`. + - output shape (torch.Size): shape of the output tensor from `node`. + """ return self._nodes_io_shapes[node]['input'], self._nodes_io_shapes[node]['output'] ####################################################### Pivate Methods ####################################################### @@ -159,11 +174,11 @@ def _get_named_modules(self, model: nn.Module) -> Dict[int, nn.Module]: Parameters ---------- - model (nn.Module): the `spiking_model` used as argument to the class instance. + - model (nn.Module): the `spiking_model` used as argument to the class instance. Returns ---------- - modules_map (dict): the mapping between a node (`key` as an `int`) and its module (`value` as a `nn.Module`). + - modules_map (dict): the mapping between a node (`key` as an `int`) and its module (`value` as a `nn.Module`). """ modules_map = {} @@ -253,37 +268,6 @@ def _get_nodes_io_shapes(self, input_dummy: torch.tensor) -> Dict[int, Dict[str, nodes_io_map[node]['output'] = io['output'].shape return nodes_io_map - - def _handle_merge_source(self, merge_node_id: int, nodes_io_map: dict) -> None: - """ This method finds the I/O shapes for node `merge_node_id` if they haven't been computed yet. When `self._find_source_of_input_to()` is - called the returned node might be a `Merge` layer for which the I/O shapes have yet to be computed. - - NOTE: In the current implemente both arguments to a `Merge` layer need to have the same output shapes. - - Parameters - ---------- - - merge_node_id (int): the ID of the node representing a `Merge` layer. - - nodes_io_map (dict): a dictionary mapping nodes to their I/O shapes. - """ - - if merge_node_id in nodes_io_map: - # I/O shapes have been computed already. - return None - - # finding nodes serving as argument to the `Merge` node... - for edge in self._edges_list: - - if edge[1] == merge_node_id: - # node `edge[0]` is one of the arguments for the `Merge` layer. - if edge[0] in nodes_io_map: - # I/O shapes of one of the arguments for the `Merge` node has been computed. - - # both arguments to `Merge` have the same I/O shape and merge outputs the same shape: updating I/O shape of `merge_node_id`. - nodes_io_map[merge_node_id] = {'input': nodes_io_map[edge[0]]['output'], 'output': nodes_io_map[edge[0]]['output']} - - return None - - raise ValueError(f'Node {merge_node_id} is a \'Merge\' layer and I/O shape for none of its arguments have been computed yet.') def _find_source_of_input_to(self, node: int) -> int: """ Finds the first edge `(X, node)` returns `X`. @@ -306,7 +290,12 @@ def _find_source_of_input_to(self, node: int) -> int: return -1 def _find_merge_arguments(self, merge_node: int) -> Tuple[int, int]: - """ A `Merge` layer receives two inputs. Return the two inputs to `merge_node` representing a `Merge` layer. """ + """ A `Merge` layer receives two inputs. Return the two inputs to `merge_node` representing a `Merge` layer. + + Returns + ---------- + - args (tuple): the IDs of the nodes that provice the input arguments to a `Merge` layer. + """ args = [] for edge in self._edges_list: diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index 976b0417..6423fe94 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -157,8 +157,7 @@ def forward(self, x): the edges are the indices of these layers). An `edge` is used to index a mapper (using `edge[0]`) in order to retrieve the output to be fed as input to a `DynapcnnLayer` instance (indexed by `edge[1]`). - `self._forward_map` (dict): a mapper used to forward data through the `DynapcnnNetwork` instances. Each `key` is the indice associated - with a `DynapcnnLayer` instance. The edges in `self._dcnnl_edges` are accessed sequentially and each node in an edge is used to index - a forward call via `self._forward_map`. + with a `DynapcnnLayer` instance. - `self._merge_points` (dict): this mapper has a "support" role. It indexes wich convolutional layers in the set of `DynapcnnLayer`s composing the network require two sources of input (because their input tensor is the output of a `Merge` layer). """ @@ -366,7 +365,7 @@ def to( ####################################################### Private Methods ####################################################### def _get_input_to_dcnnl(self, dcnnl_ID) -> int: - """ . """ + """ Returns the ID of the first `DynapcnnLayer` forwarding its input to `dcnnl_ID`. """ for edge in self._dcnnl_edges: if edge[1] == dcnnl_ID: return edge[0] @@ -499,7 +498,13 @@ def _get_network_module(self) -> Union[list, dict, dict]: return dcnnnet_module.dcnnl_edges, dcnnnet_module.forward_map, dcnnnet_module.merge_points, topological_sorting(dcnnl_edges) def _get_dynapcnnlayers_edges(self) -> List[Tuple[int, int]]: - """ Create edges representing connections between `DynapcnnLayer` instances. """ + """ Create edges representing connections between `DynapcnnLayer` instances. + + Returns + ---------- + - dcnnl_edges (list): a list of edges using the IDs of `DynapcnnLayer` instances. These edges describe the computational + graph implemented by the layers of the model (i.e., how the `DynapcnnLayer` instances address each other). + """ dcnnl_edges = [] for dcnnl_idx, layer_data in self._dynapcnn_layers.items(): @@ -547,8 +552,13 @@ def _populate_nodes_io(self): """ Loops through the nodes in the original graph to retrieve their I/O tensor shapes and add them to their respective representations in `self._nodes_to_dcnnl_map`.""" - def find_original_node_name(name_mapper: dict, node: int): - """ Find what a node is originally named when built in `self._graph_tracer`. """ + def find_original_node_name(name_mapper: dict, node: int) -> str: + """ Find what a node is originally named when built in `self._graph_tracer`. + + Returns + ---------- + - orig_name (str): a string with the original variable name given to `node`. + """ for orig_name, new_name in name_mapper.items(): if new_name == node: return orig_name @@ -559,14 +569,14 @@ def find_my_input(edges_list: list, node: int) -> int: Parameters ---------- - node (int): the node in the computational graph for which we whish to find the input source (either another node in the - graph or the original input itself to the network). + - node (int): the node in the computational graph for which we whish to find the input source (either another node in the + graph or the original input itself to the network). Returns ---------- - input source (int): this indicates the node in the computational graph providing the input to `node`. If `node` is - receiving outside input (i.e., it is a starting node) the return will be -1. For example, this will be the case - when a network with two independent branches (each starts from a different "input node") merge along the computational graph. + - input source (int): this indicates the node in the computational graph providing the input to `node`. If `node` is + receiving outside input (i.e., it is a starting node) the return will be -1. For example, this will be the case + when a network with two independent branches (each starts from a different "input node") merge along the computational graph. """ for edge in edges_list: if edge[1] == node: @@ -607,7 +617,7 @@ def find_my_input(edges_list: list, node: int) -> int: node_data['output_shape'] = tuple(list(_out)[1:]) def _to_device(self, device: torch.device) -> None: - """ .""" + """ Access each sub-layer within all `DynapcnnLayer` instances and call `.to(device)` on them.""" for layer in self._forward_map.values(): if isinstance(layer, sinabs.backend.dynapcnn.dynapcnn_layer_new.DynapcnnLayer): layer.conv_layer.to(device) From bb770e8c6e17ff7512be592be80c8ae0fcbc6dbb Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Tue, 28 May 2024 16:10:55 +0200 Subject: [PATCH 112/379] Documentation Updated function headers and type-hints. --- sinabs/backend/dynapcnn/utils.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/sinabs/backend/dynapcnn/utils.py b/sinabs/backend/dynapcnn/utils.py index 21cc5c08..bf140d07 100644 --- a/sinabs/backend/dynapcnn/utils.py +++ b/sinabs/backend/dynapcnn/utils.py @@ -305,8 +305,10 @@ def convert_Avg_to_Sum_pooling( # update the rescale factor for the target of node `key`. nodes_to_dcnnl_map[trg_dcnnl_idx]['conv_rescale_factor'].append(rescale_factor) -def find_nodes_dcnnl_idx(node, nodes_to_dcnnl_map): - """ .""" +def find_nodes_dcnnl_idx(node: int, nodes_to_dcnnl_map: dict) -> int: + """ Find the ID of the (future) `DynapcnnLayer` instance to which `node` belongs to.""" + + # looping over sets of layers (nodes) that will be used to instantiate `DynapcnnLayer`s. for dcnnl_idx, dcnnl_data in nodes_to_dcnnl_map.items(): for key, value in dcnnl_data.items(): if isinstance(key, int): @@ -323,11 +325,11 @@ def build_SumPool2d(module: nn.AvgPool2d) -> Tuple[sl.SumPool2d, int]: Parameters ---------- - module (torch.nn.AvgPool2d): the average pooling layer being converted into a sum pooling layer. + - module (torch.nn.AvgPool2d): the average pooling layer being converted into a sum pooling layer. Returns ---------- - lyr_pool (sinabs.layers.SumPool2d): the equivalent sum pooling layer. + - lyr_pool (sinabs.layers.SumPool2d): the equivalent sum pooling layer. rescale_factor (int): the weight re-scaling computed for the weights of the convolution layer targeted by the pooling. """ @@ -370,7 +372,7 @@ def topological_sorting(edges: List[Tuple[int, int]]) -> List[int]: Parameters ---------- - edges (list): the edges describing the *acyclic* graph. + - edges (list): the edges describing the *acyclic* graph. Returns ---------- From 221a95883e79fbc47f219fe61eba870461365856 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Tue, 28 May 2024 18:21:25 +0200 Subject: [PATCH 113/379] Refactor - DynapcnnNetwork._to_device() calling DynapcnnLayer.to() (no need to call .to() from each individual layer within). - _to_device() is calling .to() on the new Merge() layers created for the forward call to. - forward method works like a charm: if DynapcnnNetwork(discretize=True) the loss does not change during training. --- sinabs/backend/dynapcnn/dynapcnn_network.py | 25 +++++++-------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index 6423fe94..1aedcc57 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -194,7 +194,7 @@ def forward(self, x): # call the forward. layers_outputs[i] = self._forward_map[i](merge_output) - elif i not in layers_outputs: + else: # there's a single source of input for `DynapcnnLayer i`. # input source for `i`. @@ -206,10 +206,6 @@ def forward(self, x): # call the forward. layers_outputs[i] = self._forward_map[i](layers_outputs[src_dcnnl][return_index]) - - else: - - pass # TODO - this assumes the network has a single output node. return layers_outputs[self._topological_order[-1]][0] @@ -227,7 +223,7 @@ def parameters(self) -> list: parameters = [] for layer in self._forward_map.values(): - if isinstance(layer, sinabs.backend.dynapcnn.dynapcnn_layer_new.DynapcnnLayer): + if isinstance(layer, sinabs.backend.dynapcnn.dynapcnn_layer.DynapcnnLayer): parameters.extend(layer.conv_layer.parameters()) return parameters @@ -240,14 +236,14 @@ def init_weights(self, init_fn: nn.init = nn.init.xavier_normal_) -> None: - init_fn (torch.nn.init): the weight initialization method to be used. """ for layer in self._forward_map.values(): - if isinstance(layer, sinabs.backend.dynapcnn.dynapcnn_layer_new.DynapcnnLayer): + if isinstance(layer, sinabs.backend.dynapcnn.dynapcnn_layer.DynapcnnLayer): init_fn(layer.conv_layer.weight.data) def detach_neuron_states(self) -> None: """ Detach the neuron states and activations from current computation graph (necessary). """ for module in self._forward_map.values(): - if isinstance(module, sinabs.backend.dynapcnn.dynapcnn_layer_new.DynapcnnLayer): + if isinstance(module, sinabs.backend.dynapcnn.dynapcnn_layer.DynapcnnLayer): if isinstance(module.spk_layer, sl.StatefulLayer): for name, buffer in module.spk_layer.named_buffers(): buffer.detach_() @@ -619,16 +615,11 @@ def find_my_input(edges_list: list, node: int) -> int: def _to_device(self, device: torch.device) -> None: """ Access each sub-layer within all `DynapcnnLayer` instances and call `.to(device)` on them.""" for layer in self._forward_map.values(): - if isinstance(layer, sinabs.backend.dynapcnn.dynapcnn_layer_new.DynapcnnLayer): - layer.conv_layer.to(device) - layer.spk_layer.to(device) - - # if there's more than one pooling each of them becomes a node that is catched by the `else` statement. - if len(layer.pool_layer) == 1: - layer.pool_layer[0].to(device) - else: - # this nodes are created from `DynapcnnLayer`s that have multiple poolings (each pooling becomes a new node). + if isinstance(layer, sinabs.backend.dynapcnn.dynapcnn_layer.DynapcnnLayer): layer.to(device) + + for _, data in self._merge_points.items(): + data['merge'].to(device) def __str__(self): pretty_print = '' From 1eaafe4f524e64384c4242905e68cacc05080a00 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Tue, 28 May 2024 18:33:01 +0200 Subject: [PATCH 114/379] Converting SNN to DynapcnnNetwork and training the hw model directly --- .../DynapcnnNetwork-example_1.ipynb | 275 ++++++++++++++---- 1 file changed, 224 insertions(+), 51 deletions(-) diff --git a/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_1.ipynb b/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_1.ipynb index 52e549d1..c4a3c8b7 100644 --- a/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_1.ipynb +++ b/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_1.ipynb @@ -11,7 +11,7 @@ "import torch\n", "import torch.nn as nn\n", "from sinabs.backend.dynapcnn import DynapcnnNetwork\n", - "from sinabs.layers import Merge, IAFSqueeze\n", + "from sinabs.layers import Merge, IAFSqueeze, SumPool2d\n", "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential" ] }, @@ -25,7 +25,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 2, @@ -48,7 +48,7 @@ "channels = 2\n", "height = 34\n", "width = 34\n", - "batch_size = 3\n", + "batch_size = 8\n", "\n", "input_shape = (channels, height, width)" ] @@ -153,7 +153,8 @@ "hw_model = DynapcnnNetwork(\n", " snn=snn,\n", " input_shape=input_shape,\n", - " batch_size=batch_size\n", + " batch_size=batch_size,\n", + " discretize=False\n", ")" ] }, @@ -180,13 +181,15 @@ "\n", "(node 0): Conv2d(2, 10, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", "(node 1): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(361.), min_v_mem=Parameter containing:\n", - "tensor(-361.), batch_size=3, num_timesteps=-1)\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-1.), batch_size=8, num_timesteps=-1)\n", "(node 2): SumPool2d(norm_type=1, kernel_size=(3, 3), stride=None, ceil_mode=False)\n", "(node 3): SumPool2d(norm_type=1, kernel_size=(4, 4), stride=None, ceil_mode=False)\n", "\n", "METADATA:\n", "\n", + "> network's entry point: True\n", + "> convolution's weight re-scaling factor: None\n", "> assigned core index: None\n", "> destination DynapcnnLayers: [1, 2]\n", "> node 2 feeds input to nodes [4]\n", @@ -198,11 +201,13 @@ "\n", "(node 4): Conv2d(10, 10, kernel_size=(4, 4), stride=(1, 1), bias=False)\n", "(node 6): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(14463.), min_v_mem=Parameter containing:\n", - "tensor(-14463.), batch_size=3, num_timesteps=-1)\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-1.), batch_size=8, num_timesteps=-1)\n", "\n", "METADATA:\n", "\n", + "> network's entry point: False\n", + "> convolution's weight re-scaling factor: 4.5\n", "> assigned core index: None\n", "> destination DynapcnnLayers: [2]\n", "> node 6 feeds input to nodes [7]\n", @@ -213,11 +218,13 @@ "\n", "(node 7): Conv2d(10, 1, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", "(node 8): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(13042.), min_v_mem=Parameter containing:\n", - "tensor(-13042.), batch_size=3, num_timesteps=-1)\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-1.), batch_size=8, num_timesteps=-1)\n", "\n", "METADATA:\n", "\n", + "> network's entry point: False\n", + "> convolution's weight re-scaling factor: 8.0\n", "> assigned core index: None\n", "> destination DynapcnnLayers: [3]\n", "> node 8 feeds input to nodes [9]\n", @@ -228,11 +235,13 @@ "\n", "(node 9): Conv2d(1, 500, kernel_size=(7, 7), stride=(1, 1), bias=False)\n", "(node 10): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(889.), min_v_mem=Parameter containing:\n", - "tensor(-889.), batch_size=3, num_timesteps=-1)\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-1.), batch_size=8, num_timesteps=-1)\n", "\n", "METADATA:\n", "\n", + "> network's entry point: False\n", + "> convolution's weight re-scaling factor: None\n", "> assigned core index: None\n", "> destination DynapcnnLayers: [4]\n", "> node 10 feeds input to nodes [11]\n", @@ -243,11 +252,13 @@ "\n", "(node 11): Conv2d(500, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", "(node 12): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(2842.), min_v_mem=Parameter containing:\n", - "tensor(-2842.), batch_size=3, num_timesteps=-1)\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-1.), batch_size=8, num_timesteps=-1)\n", "\n", "METADATA:\n", "\n", + "> network's entry point: False\n", + "> convolution's weight re-scaling factor: None\n", "> assigned core index: None\n", "> destination DynapcnnLayers: []\n", "\n", @@ -259,32 +270,6 @@ "print(hw_model)" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Lets forward a sample data through our DynapcnnNetwork instance to see if the produces the correct output:" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "torch.Size([3, 10, 1, 1])\n" - ] - } - ], - "source": [ - "x = torch.randn((batch_size, *input_shape))\n", - "out = hw_model(x)\n", - "print(out.shape)" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -296,7 +281,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 8, "metadata": { "metadata": {} }, @@ -313,13 +298,15 @@ "\n", "(node 0): Conv2d(2, 10, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", "(node 1): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(361.), min_v_mem=Parameter containing:\n", - "tensor(-361.), batch_size=3, num_timesteps=-1)\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-1.), batch_size=8, num_timesteps=-1)\n", "(node 2): SumPool2d(norm_type=1, kernel_size=(3, 3), stride=None, ceil_mode=False)\n", "(node 3): SumPool2d(norm_type=1, kernel_size=(4, 4), stride=None, ceil_mode=False)\n", "\n", "METADATA:\n", "\n", + "> network's entry point: True\n", + "> convolution's weight re-scaling factor: None\n", "> assigned core index: 0\n", "> destination DynapcnnLayers: [1, 2]\n", "> node 2 feeds input to nodes [4]\n", @@ -331,11 +318,13 @@ "\n", "(node 4): Conv2d(10, 10, kernel_size=(4, 4), stride=(1, 1), bias=False)\n", "(node 6): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(14463.), min_v_mem=Parameter containing:\n", - "tensor(-14463.), batch_size=3, num_timesteps=-1)\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-1.), batch_size=8, num_timesteps=-1)\n", "\n", "METADATA:\n", "\n", + "> network's entry point: False\n", + "> convolution's weight re-scaling factor: 4.5\n", "> assigned core index: 1\n", "> destination DynapcnnLayers: [2]\n", "> node 6 feeds input to nodes [7]\n", @@ -346,11 +335,13 @@ "\n", "(node 7): Conv2d(10, 1, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", "(node 8): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(13042.), min_v_mem=Parameter containing:\n", - "tensor(-13042.), batch_size=3, num_timesteps=-1)\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-1.), batch_size=8, num_timesteps=-1)\n", "\n", "METADATA:\n", "\n", + "> network's entry point: False\n", + "> convolution's weight re-scaling factor: 8.0\n", "> assigned core index: 2\n", "> destination DynapcnnLayers: [3]\n", "> node 8 feeds input to nodes [9]\n", @@ -361,11 +352,13 @@ "\n", "(node 9): Conv2d(1, 500, kernel_size=(7, 7), stride=(1, 1), bias=False)\n", "(node 10): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(889.), min_v_mem=Parameter containing:\n", - "tensor(-889.), batch_size=3, num_timesteps=-1)\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-1.), batch_size=8, num_timesteps=-1)\n", "\n", "METADATA:\n", "\n", + "> network's entry point: False\n", + "> convolution's weight re-scaling factor: None\n", "> assigned core index: 3\n", "> destination DynapcnnLayers: [4]\n", "> node 10 feeds input to nodes [11]\n", @@ -376,11 +369,13 @@ "\n", "(node 11): Conv2d(500, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", "(node 12): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(2842.), min_v_mem=Parameter containing:\n", - "tensor(-2842.), batch_size=3, num_timesteps=-1)\n", + "tensor(1.), min_v_mem=Parameter containing:\n", + "tensor(-1.), batch_size=8, num_timesteps=-1)\n", "\n", "METADATA:\n", "\n", + "> network's entry point: False\n", + "> convolution's weight re-scaling factor: None\n", "> assigned core index: 4\n", "> destination DynapcnnLayers: []\n", "\n", @@ -398,6 +393,184 @@ "source": [ "Notice above now how the layers of the model have been assigned to a chip core." ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Training the HW model" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "hw_model.init_weights()" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "device: NVIDIA GeForce RTX 3070 Ti\n" + ] + } + ], + "source": [ + "if torch.cuda.is_available():\n", + " device = torch.device('cuda:0')\n", + " print('device: ', torch.cuda.get_device_name(0))\n", + "else:\n", + " device = torch.device('cpu')" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "hw_model.to(device)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "\n", + "sys.path.append('../utils')\n", + "\n", + "from train_test_fn import training_loop, load_dataset, split_train_validation, load_architecture" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "batch_size = 8\n", + "num_workers = 4\n", + "epochs = 5\n", + "lr = 5e-4\n", + "\n", + "n_time_steps = 50" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "from torch.utils.data import DataLoader\n", + "from torch.nn import CrossEntropyLoss\n", + "from torch.optim import Adam" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(34, 34, 2)\n" + ] + } + ], + "source": [ + "snn_train_dataset, snn_test_dataset, sensor_size, nb_classes = load_dataset('NMNIST', n_time_steps, \"../NMNIST\")\n", + "\n", + "print(sensor_size)\n", + "\n", + "snn_train_dataloader = DataLoader(snn_train_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True)\n", + "snn_test_dataloader = DataLoader(snn_test_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "optimizer = Adam(hw_model.parameters(), lr=lr, betas=(0.9, 0.999), eps=1e-8)\n", + "loss_fn = CrossEntropyLoss()" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "70c30996c7164a019d9a9b26397a4a6c", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/7500 [00:00 1\u001b[0m epochs_x, epochs_y, epochs_acc \u001b[38;5;241m=\u001b[39m training_loop(\n\u001b[1;32m 2\u001b[0m device, \n\u001b[1;32m 3\u001b[0m n_time_steps,\n\u001b[1;32m 4\u001b[0m batch_size,\n\u001b[1;32m 5\u001b[0m sensor_size,\n\u001b[1;32m 6\u001b[0m snn_train_dataloader, \n\u001b[1;32m 7\u001b[0m hw_model, \n\u001b[1;32m 8\u001b[0m loss_fn, \n\u001b[1;32m 9\u001b[0m optimizer, \n\u001b[1;32m 10\u001b[0m epochs, \n\u001b[1;32m 11\u001b[0m snn_test_dataloader)\n", + "File \u001b[0;32m~/Documents/github/sinabs/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/../utils/train_test_fn.py:132\u001b[0m, in \u001b[0;36mtraining_loop\u001b[0;34m(device, nb_time_steps, batch_size, feature_map_size, dataloader_train, model, loss_fn, optimizer, epochs, dataloader_test)\u001b[0m\n\u001b[1;32m 130\u001b[0m \u001b[38;5;66;03m# gradient update\u001b[39;00m\n\u001b[1;32m 131\u001b[0m optimizer\u001b[38;5;241m.\u001b[39mzero_grad()\n\u001b[0;32m--> 132\u001b[0m loss\u001b[38;5;241m.\u001b[39mbackward()\n\u001b[1;32m 133\u001b[0m optimizer\u001b[38;5;241m.\u001b[39mstep()\n\u001b[1;32m 135\u001b[0m \u001b[38;5;66;03m# detach the neuron states and activations from current computation graph(necessary)\u001b[39;00m\n", + "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/_tensor.py:522\u001b[0m, in \u001b[0;36mTensor.backward\u001b[0;34m(self, gradient, retain_graph, create_graph, inputs)\u001b[0m\n\u001b[1;32m 512\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m has_torch_function_unary(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m 513\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m handle_torch_function(\n\u001b[1;32m 514\u001b[0m Tensor\u001b[38;5;241m.\u001b[39mbackward,\n\u001b[1;32m 515\u001b[0m (\u001b[38;5;28mself\u001b[39m,),\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 520\u001b[0m inputs\u001b[38;5;241m=\u001b[39minputs,\n\u001b[1;32m 521\u001b[0m )\n\u001b[0;32m--> 522\u001b[0m torch\u001b[38;5;241m.\u001b[39mautograd\u001b[38;5;241m.\u001b[39mbackward(\n\u001b[1;32m 523\u001b[0m \u001b[38;5;28mself\u001b[39m, gradient, retain_graph, create_graph, inputs\u001b[38;5;241m=\u001b[39minputs\n\u001b[1;32m 524\u001b[0m )\n", + "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/autograd/__init__.py:266\u001b[0m, in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[0m\n\u001b[1;32m 261\u001b[0m retain_graph \u001b[38;5;241m=\u001b[39m create_graph\n\u001b[1;32m 263\u001b[0m \u001b[38;5;66;03m# The reason we repeat the same comment below is that\u001b[39;00m\n\u001b[1;32m 264\u001b[0m \u001b[38;5;66;03m# some Python versions print out the first line of a multi-line function\u001b[39;00m\n\u001b[1;32m 265\u001b[0m \u001b[38;5;66;03m# calls in the traceback and some print out the last line\u001b[39;00m\n\u001b[0;32m--> 266\u001b[0m Variable\u001b[38;5;241m.\u001b[39m_execution_engine\u001b[38;5;241m.\u001b[39mrun_backward( \u001b[38;5;66;03m# Calls into the C++ engine to run the backward pass\u001b[39;00m\n\u001b[1;32m 267\u001b[0m tensors,\n\u001b[1;32m 268\u001b[0m grad_tensors_,\n\u001b[1;32m 269\u001b[0m retain_graph,\n\u001b[1;32m 270\u001b[0m create_graph,\n\u001b[1;32m 271\u001b[0m inputs,\n\u001b[1;32m 272\u001b[0m allow_unreachable\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m 273\u001b[0m accumulate_grad\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m 274\u001b[0m )\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m: " + ] + } + ], + "source": [ + "epochs_x, epochs_y, epochs_acc = training_loop(\n", + " device, \n", + " n_time_steps,\n", + " batch_size,\n", + " sensor_size,\n", + " snn_train_dataloader, \n", + " hw_model, \n", + " loss_fn, \n", + " optimizer, \n", + " epochs, \n", + " snn_test_dataloader)" + ] } ], "metadata": { From d744dd6bb2857e05f67bd70ceec1963b644c98c0 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Thu, 30 May 2024 14:49:47 +0200 Subject: [PATCH 115/379] Refactor + Bug Fixing - Extraction of config. dict of a DynapcnnLayer removed from class and moved back to ConfigBuilder class. - Updated function headers/type hints/in-line documentation. - Refactored 'ConfigBuilder.get_dynapcnn_layer_config_dict' to create 'destinations' dict entry to handle setting of the destinations config. object. - Ranamed '_forward_map' into 'layers_mapper' in DynapcnnNetwork to better fit its role in the network construction/chip deployment. --- sinabs/backend/dynapcnn/chips/dynapcnn.py | 121 ++++++---------- sinabs/backend/dynapcnn/chips/speck2cmini.py | 6 +- sinabs/backend/dynapcnn/chips/speck2e.py | 6 +- sinabs/backend/dynapcnn/chips/speck2f.py | 6 +- sinabs/backend/dynapcnn/config_builder.py | 16 ++- sinabs/backend/dynapcnn/dynapcnn_layer.py | 143 +++++-------------- sinabs/backend/dynapcnn/dynapcnn_network.py | 120 +++++++++++----- sinabs/backend/dynapcnn/mapping.py | 2 +- 8 files changed, 185 insertions(+), 235 deletions(-) diff --git a/sinabs/backend/dynapcnn/chips/dynapcnn.py b/sinabs/backend/dynapcnn/chips/dynapcnn.py index 1039e8bf..e766f03a 100644 --- a/sinabs/backend/dynapcnn/chips/dynapcnn.py +++ b/sinabs/backend/dynapcnn/chips/dynapcnn.py @@ -74,7 +74,7 @@ def set_kill_bits(cls, layer: DynapcnnLayer, config_dict: dict) -> dict: return config_dict @classmethod - def get_dynapcnn_layer_config_dict(cls, layer: DynapcnnLayer): + def get_dynapcnn_layer_config_dict(cls, layer: DynapcnnLayer, layers_mapper: Dict[int, DynapcnnLayer]) -> dict: config_dict = {} config_dict["destinations"] = [{}, {}] @@ -113,8 +113,6 @@ def get_dynapcnn_layer_config_dict(cls, layer: DynapcnnLayer): config_dict["weights"] = weights.int().tolist() config_dict["biases"] = biases.int().tolist() config_dict["leak_enable"] = biases.bool().any() - # config_dict["weights_kill_bit"] = torch.zeros_like(weights).bool().tolist() - # config_dict["biases_kill_bit"] = torch.zeros_like(biases).bool().tolist() # Update parameters from the spiking layer @@ -141,10 +139,6 @@ def get_dynapcnn_layer_config_dict(cls, layer: DynapcnnLayer): "Unknown reset mechanism. Only MembraneReset and MembraneSubtract are currently understood." ) - # if (not return_to_zero) and self.spk_layer.membrane_subtract != self.spk_layer.threshold: - # warn( - # "SpikingConv2dLayer: Subtraction of membrane potential is always by high threshold." - # ) if layer.spk_layer.min_v_mem is None: min_v_mem = -(2**15) else: @@ -156,126 +150,96 @@ def get_dynapcnn_layer_config_dict(cls, layer: DynapcnnLayer): "threshold_low": min_v_mem, "monitor_enable": False, "neurons_initial_value": neurons_state.int().tolist(), - # "neurons_value_kill_bit" : torch.zeros_like(neurons_state).bool().tolist() } ) - # Update parameters from pooling - if layer.pool_layer is not None: - config_dict["destinations"][0]["pooling"] = expand_to_pair( - layer.pool_layer.kernel_size - )[0] - config_dict["destinations"][0]["enable"] = True - else: - pass - # Set kill bits - config_dict = cls.set_kill_bits(layer=layer, config_dict=config_dict) + # setting destinations config. based on destinations destination nodes of the nodes withing this `dcnnl`. + destinations = [] + for node_id, destination_nodes in layer.nodes_destinations.items(): + for dest_node in destination_nodes: + core_id = DynapcnnLayer.find_nodes_core_id(dest_node, layers_mapper) + kernel_size = layer.get_pool_kernel_size(node_id) - return config_dict + dest_data = { + 'layer': core_id, + 'enable': True, + 'pooling': kernel_size if kernel_size else 1 + } - @classmethod - def write_dynapcnn_layer_config( - cls, layer: DynapcnnLayer, chip_layer: "CNNLayerConfig" - ): - """Write a single layer configuration to the dynapcnn conf object. + destinations.append(dest_data) + config_dict["destinations"] = destinations - Parameters - ---------- - layer: - The dynapcnn layer to write the configuration for - chip_layer: CNNLayerConfig - DYNAPCNN configuration object representing the layer to which - configuration is written. - """ - config_dict = cls.get_dynapcnn_layer_config_dict(layer=layer) - - # Update configuration of the DYNAPCNN layer - chip_layer.dimensions = config_dict["dimensions"] - config_dict.pop("dimensions") + # Set kill bits + config_dict = cls.set_kill_bits(layer=layer, config_dict=config_dict) - for i in range(len(config_dict["destinations"])): - if "pooling" in config_dict["destinations"][i]: - chip_layer.destinations[i].pooling = config_dict["destinations"][i][ - "pooling" - ] - config_dict.pop("destinations") - for param, value in config_dict.items(): - try: - setattr(chip_layer, param, value) - except TypeError as e: - raise TypeError(f"Unexpected parameter {param} or value. {e}") + return config_dict @classmethod - def write_dynapcnn_layer_config_graph(cls, dcnnl: DynapcnnLayer, chip_layer: "CNNLayerConfig", forward_map: dict) -> None: - """ Uses the data in `dcnnl` to configure a `CNNLayerConfig` to be deployed on chip. + def write_dynapcnn_layer_config(cls, layer: DynapcnnLayer, chip_layer: "CNNLayerConfig", layers_mapper: Dict[int, DynapcnnLayer]) -> None: + """ Write a single layer configuration to the dynapcnn conf object. Uses the data in `layer` to configure a `CNNLayerConfig` to be + deployed on chip. Parameters ---------- - dcnnl (DynapcnnLayer): the layer for which the condiguration will be written. - chip_layer (CNNLayerConfig): used to represent/configure `dcnnl` onto the chip. - forward_map (dict): a dictionary with keys being the ID of each DynapcnnLayer and values being the layer - itself. This is used to retrieve the `.assigned_core` for each of the layers in `.dynapcnnlayer_destination` - such that `chip_layer.destinations` can be configured. + - layer (DynapcnnLayer): the layer for which the condiguration will be written. + - chip_layer (CNNLayerConfig): configuration object representing the layer to which configuration is written. + - layers_mapper (dict): a dictionary with keys being the ID of each `DynapcnnLayer` and values being the layer + instance. This is used to retrieve the `.assigned_core` for each of the layers in `.dynapcnnlayer_destination` + such that `chip_layer.destinations` can be configured. """ # extracting from a DynapcnnLayer the config. variables for its CNNLayerConfig. - config_dict = dcnnl.get_layer_config_dict() - - # use core indexing instead of DynapcnnLayer indexing for destinations. - for dest_config in config_dict['destinations']: - dcnnl_idx = dest_config['layer'] + config_dict = cls.get_dynapcnn_layer_config_dict(layer=layer, layers_mapper=layers_mapper) - # get the core the destination DynapcnnLayer is using. - dcnnl_core_idx = forward_map[dcnnl_idx].assigned_core - - dest_config['layer'] = dcnnl_core_idx + # update configuration of the DYNAPCNN layer. + chip_layer.dimensions = config_dict["dimensions"] + config_dict.pop("dimensions") # set the destinations configuration. for i in range(len(config_dict['destinations'])): - chip_layer.destinations[i] = config_dict['destinations'][i] - config_dict.pop("destinations") + chip_layer.destinations[i].layer = config_dict['destinations'][i]['layer'] + chip_layer.destinations[i].enable = config_dict['destinations'][i]['enable'] + chip_layer.destinations[i].pooling = config_dict['destinations'][i]['pooling'] + + config_dict.pop('destinations') # set remaining configuration. - for param, value in config_dict.items(): # set remaining attributes. + for param, value in config_dict.items(): try: setattr(chip_layer, param, value) except TypeError as e: raise TypeError(f"Unexpected parameter {param} or value. {e}") @classmethod - def build_config(cls, model: Union["DynapcnnNetwork"], chip_layers: Union[List[int], None]) -> DynapcnnConfiguration: + def build_config(cls, model: Union["DynapcnnNetwork"]) -> DynapcnnConfiguration: """ Uses `DynapcnnLayer` objects to configure their equivalent chip core via a `CNNLayerConfig` object that is built using using the `DynapcnnLayer` properties. Parameters ---------- - model: - either a `DynapcnnNetwork` or a `DynapcnnNetworkGraph` instance where the model (DynapcnnLayer) layers can be found. - chip_layers: - a list containing the core indexes where each `DynapcnnLayer` will be mapped to (if `model` is an instance of `DynapcnnNetwork`, otherwise `None`). + - model (DynapcnnNetwork): network instance used to read out `DynapcnnLayer` instances. Returns ---------- - config: - an instance of a `DynapcnnConfiguration`. + - config (DynapcnnConfiguration): an instance of a `DynapcnnConfiguration`. """ config = cls.get_default_config() if type(model) == sinabs.backend.dynapcnn.dynapcnn_network.DynapcnnNetwork: - """ Loops through `DynapcnnNetworkGraph._forward_map`, containing all `DynapcnnLayer`s in the model, their - core ID to be loaded onto and their target destinations. Each `ith_dcnnl` has all the info. necessary to config. + """ Loops through `DynapcnnNetworkGraph._layers_mapper`, containing all `DynapcnnLayer`s in the model, their + core ID (where they are configured onto) and their target destinations. Each `ith_dcnnl` has all the info. necessary to config. their respective `CNNLayerConfig` object. """ has_dvs_layer = False # TODO DVSLayer not supported yet. - for layer_index, ith_dcnnl in model.forward_map.items(): + for layer_index, ith_dcnnl in model.layers_mapper.items(): if isinstance(ith_dcnnl, DVSLayer): # TODO DVSLayer not supported yet. pass elif isinstance(ith_dcnnl, DynapcnnLayer): chip_layer = config.cnn_layers[ith_dcnnl.assigned_core] - cls.write_dynapcnn_layer_config_graph(ith_dcnnl, chip_layer, model.forward_map) + cls.write_dynapcnn_layer_config(ith_dcnnl, chip_layer, model.layers_mapper) else: # shouldn't happen since type checks are made previously. @@ -355,6 +319,7 @@ def monitor_layers(cls, config: "DynapcnnConfiguration", layers: List): monitor_layers.remove("dvs") for lyr_indx in monitor_layers: config.cnn_layers[lyr_indx].monitor_enable = True + if any( dest.pooling != 1 for dest in config.cnn_layers[lyr_indx].destinations ): diff --git a/sinabs/backend/dynapcnn/chips/speck2cmini.py b/sinabs/backend/dynapcnn/chips/speck2cmini.py index b044e8da..327f938e 100644 --- a/sinabs/backend/dynapcnn/chips/speck2cmini.py +++ b/sinabs/backend/dynapcnn/chips/speck2cmini.py @@ -1,4 +1,4 @@ -from typing import List +from typing import List, Dict import samna from samna.speck2cMini.configuration import SpeckConfiguration @@ -29,8 +29,8 @@ def get_output_buffer(cls): return samna.BasicSinkNode_speck2c_mini_event_output_event() @classmethod - def get_dynapcnn_layer_config_dict(cls, layer: DynapcnnLayer): - config_dict = super().get_dynapcnn_layer_config_dict(layer=layer) + def get_dynapcnn_layer_config_dict(cls, layer: DynapcnnLayer, layers_mapper: Dict[int, DynapcnnLayer]) -> dict: + config_dict = super().get_dynapcnn_layer_config_dict(layer=layer, layers_mapper=layers_mapper) config_dict.pop("weights_kill_bit") config_dict.pop("biases_kill_bit") config_dict.pop("neurons_value_kill_bit") diff --git a/sinabs/backend/dynapcnn/chips/speck2e.py b/sinabs/backend/dynapcnn/chips/speck2e.py index 1e170a9f..3904e098 100644 --- a/sinabs/backend/dynapcnn/chips/speck2e.py +++ b/sinabs/backend/dynapcnn/chips/speck2e.py @@ -5,6 +5,8 @@ from .dynapcnn import DynapcnnConfigBuilder +from typing import Dict + # Since most of the configuration is identical to DYNAP-CNN, we can simply inherit this class @@ -30,6 +32,6 @@ def set_kill_bits(cls, layer: DynapcnnLayer, config_dict: dict) -> dict: return config_dict @classmethod - def get_dynapcnn_layer_config_dict(cls, layer: DynapcnnLayer): - config_dict = super().get_dynapcnn_layer_config_dict(layer=layer) + def get_dynapcnn_layer_config_dict(cls, layer: DynapcnnLayer, layers_mapper: Dict[int, DynapcnnLayer]) -> dict: + config_dict = super().get_dynapcnn_layer_config_dict(layer=layer, layers_mapper=layers_mapper) return config_dict diff --git a/sinabs/backend/dynapcnn/chips/speck2f.py b/sinabs/backend/dynapcnn/chips/speck2f.py index b43a5417..2f1a0b2d 100644 --- a/sinabs/backend/dynapcnn/chips/speck2f.py +++ b/sinabs/backend/dynapcnn/chips/speck2f.py @@ -5,6 +5,8 @@ from .dynapcnn import DynapcnnConfigBuilder +from typing import Dict + # Since most of the configuration is identical to DYNAP-CNN, we can simply inherit this class @@ -26,8 +28,8 @@ def get_output_buffer(cls): return samna.BasicSinkNode_speck2f_event_output_event() @classmethod - def get_dynapcnn_layer_config_dict(cls, layer: DynapcnnLayer): - config_dict = super().get_dynapcnn_layer_config_dict(layer=layer) + def get_dynapcnn_layer_config_dict(cls, layer: DynapcnnLayer, layers_mapper: Dict[int, DynapcnnLayer]) -> dict: + config_dict = super().get_dynapcnn_layer_config_dict(layer=layer, layers_mapper=layers_mapper) config_dict.pop("weights_kill_bit") config_dict.pop("biases_kill_bit") config_dict.pop("neurons_value_kill_bit") diff --git a/sinabs/backend/dynapcnn/config_builder.py b/sinabs/backend/dynapcnn/config_builder.py index 3843a6a8..01f32198 100644 --- a/sinabs/backend/dynapcnn/config_builder.py +++ b/sinabs/backend/dynapcnn/config_builder.py @@ -74,9 +74,8 @@ def get_valid_mapping(cls, model: "DynapcnnNetwork") -> List[int]: Returns ------- - List of core indices corresponding to each layer of the model: - The index of the core on chip to which the i-th layer in the - model is mapped is the value of the i-th entry in the list. + - chip_layers_ordering (list): the core indices corresponding to each layer of the model. Though this list is being returned, each core index + `core_idx` is assigned directyl to each `DynapcnnLayer` instance via accesses to `model.layers_mapper`. """ chip_layers_ordering = [] @@ -84,18 +83,21 @@ def get_valid_mapping(cls, model: "DynapcnnNetwork") -> List[int]: if type(model) == sinabs.backend.dynapcnn.dynapcnn_network.DynapcnnNetwork: mapping = get_valid_mapping(model, cls.get_constraints()) - if isinstance(model.forward_map[0], DVSLayer): + if isinstance(model.layers_mapper[0], DVSLayer): # TODO not handling DVSLayer yet. # TODO if the architecture has more than one `DynapcnnLayer`s acting as input node of the model - # thi check will be wrong since it assumes the network has a single input node `model.forward_map[0]`. + # thi check will be wrong since it assumes the network has a single input node `model.layers_mapper[0]`. pass - for (dcnnl, core_idx) in mapping: - model.forward_map[dcnnl].assigned_core = core_idx + for (dcnnl_idx, core_idx) in mapping: + # save the core index information directly in the `DynapcnnLayer` object assigned to it. + model.layers_mapper[dcnnl_idx].assigned_core = core_idx + chip_layers_ordering.append(core_idx) else: raise InvalidModel(model) + # return kept but its information is not used beyond this point (core indices already part of each `DynapcnnLayer` instance). return chip_layers_ordering @classmethod diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer.py b/sinabs/backend/dynapcnn/dynapcnn_layer.py index 48d33743..f4433680 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer.py @@ -120,6 +120,8 @@ def __init__( # input shape of conv layer. self.conv_in_shape = dcnnl_data[self.conv_node_id]['input_shape'] + # input shape of the `DynapcnnLayer` instance. + self.input_shape = self.conv_in_shape # this weight rescale comes from the node projecting into this 'conv' node. if len(dcnnl_data['conv_rescale_factor']): @@ -157,6 +159,16 @@ def __init__( ####################################################### Public Methods ####################################################### + def get_neuron_shape(self) -> Tuple[int, int, int]: + """Return the output shape of the neuron layer. + + Returns + ------- + features, height, width + """ + # same as the convolution's output. + return self.conv_out_shape + def get_destination_dcnnl_index(self, dcnnl_id: int) -> int: """ The `forward` method will return as many tensors as there are elements in `self.dynapcnnlayer_destination`. Since the i-th returned tensor is to be fed to the i-th destionation in `self.dynapcnnlayer_destination`, the return of this method can be used to index a tensor returned in the `forward` method. @@ -181,16 +193,14 @@ def forward(self, x): Example ---------- + TODO this example needs to be revised because the spiking layer only appears if it is projecting to outside this layer. + - With `self.nodes_destinations = {1: [5, 8], 2: [4], 3: [7]}` this method will return 4 tensors (each to be sent to a different DynapcnnLayer): the 1st and 2nd are the outputs of the spiking layer (`node 1`); the 3rd is the output of the first pooling layer (`node 2`, receiving the output of node 1) appearing right after the spiking layer as defined in the original `nn.Module` being converted to a `DynapcnnNetwork`; the 4th is the output of the second pooling layer (`node 3`) to appear after the spiking layer. Thus, the return will be `return node_1_out, node_1_out, node_2_out, node_3_out`. This means the edges in the computational graph involved in this mapping were: - - - 1 --> 2 # `2` is one of the pooling layers of this DynapcnnLayer. - - 1 --> 3 # `3` is one of the pooling layers of this DynapcnnLayer. - - 1 --> 5 # `5` is a conv layer belonging to another DynapcnnLayer U. - - 1 --> 8 # `8` is a conv layer belonging to another DynapcnnLayer V. + - 2 --> 4 # `4` is a conv layer belonging to another DynapcnnLayer X. - 3 --> 7 # `7` is a conv layer belonging to another DynapcnnLayer Y. """ @@ -301,108 +311,6 @@ def summary(self) -> dict: "kernel": list(self.conv_layer.weight.data.shape), "neuron": self.conv_out_shape, # neuron layer output has the same shape as the convolution layer ouput. } - - def get_layer_config_dict(self) -> dict: - """ Returns a dict containing the properties required to configure a `CNNLayerConfig` instance that - will map this DynapcnnLayer onto the chip. - - Returns - ---------- - - config_dict (dict): a nested dictionary containing of the variables necessary to configure a `CNNLayerConfig` instance. - """ - config_dict = {} - - # configures `CNNLayerConfig.dimensions` (instance of `CNNLayerDimensions`). - dimensions = {} - - # input shape of convolution. - dimensions['input_shape'] = { - 'size': {'x': self.conv_in_shape[2], 'y': self.conv_in_shape[1]}, - 'feature_count': self.conv_in_shape[0] - } - - # ouput shape of convolution. - dimensions['output_shape'] = { - 'size': {'x': self.conv_out_shape[2], 'y': self.conv_out_shape[1]}, - 'feature_count': self.conv_out_shape[0] - } - - # convolution padding, stride and kernel sizes. - dimensions['padding'] = {'x': self.conv_layer.padding[1], 'y': self.conv_layer.padding[0]} - dimensions['stride'] = {'x': self.conv_layer.stride[1], 'y': self.conv_layer.stride[0]} - dimensions['kernel_size'] = self.conv_layer.kernel_size[0] - - config_dict['dimensions'] = dimensions # update config dict. - - # update parameters from convolution. - if self.conv_layer.bias is not None: - (weights, biases) = self.conv_layer.parameters() - else: - (weights,) = self.conv_layer.parameters() - biases = torch.zeros(self.conv_layer.out_channels) - - # parameters of the convolution in the DynapcnnLayer. - - weights = weights.transpose(2, 3) # need this to match samna convention. - config_dict['weights'] = weights.int().tolist() # 4-D list of lists representing kernel parameters. - config_dict['biases'] = biases.int().tolist() - config_dict['leak_enable'] = biases.bool().any() - - # parameters of the neurons in the DynapcnnLayer. - - # set neuron states. # TODO coppied from the old implementation. - if not self.spk_layer.is_state_initialised(): - # then we assign no initial neuron state to DYNAP-CNN. - f, h, w = self.conv_out_shape # same as the convolution layer. - neurons_state = torch.zeros(f, w, h) - - elif self.spk_layer.v_mem.dim() == 4: - # 4-D states should be the norm when there is a batch dim. - neurons_state = self.spk_layer.v_mem.transpose(2, 3)[0] - - else: - raise ValueError(f"Current v_mem (shape: {self.spk_layer.v_mem.shape}) of spiking layer not understood.") - # TODO error here: find where `self.spk_layer.v_mem` is being initialized. - - # resetting vs returning to 0. # TODO coppied from the old implementation. - if isinstance(self.spk_layer.reset_fn, sinabs.activation.MembraneReset): - return_to_zero = True # neurons in this layer will return to 0 when firing. - elif isinstance(self.spk_layer.reset_fn, sinabs.activation.MembraneSubtract): - return_to_zero = False # threshold will be subtracted from the value their membrane potential reached before firing. - else: - raise Exception("Unknown reset mechanism. Only MembraneReset and MembraneSubtract are currently understood.") - - if self.spk_layer.min_v_mem is None: - min_v_mem = -(2**15) - else: - min_v_mem = int(self.spk_layer.min_v_mem) - - # set neuron configuration for this DynapcnnLayer. - config_dict.update( - { - "return_to_zero": return_to_zero, - "threshold_high": int(self.spk_layer.spike_threshold), - "threshold_low": min_v_mem, - "monitor_enable": False, - "neurons_initial_value": neurons_state.int().tolist() - } - ) - - # set pooling configuration for each destinaition. This configures a `CNNLayerConfig.destinations` (instance of `CNNLayerDimensions`). - config_dict['destinations'] = [] - if len(self.pool_layer) != 0: - for i in range(len(self.pool_layer)): - dest_config = { - 'layer': self.dynapcnnlayer_destination[i],# TODO this destination index is not the core index yet, just the index of the DynapcnnLayers themselves. - 'enable': True, - 'pooling': self.pool_layer[i].kernel_size[0] if isinstance(self.pool_layer[i].kernel_size, tuple) else self.pool_layer[i].kernel_size # TODO make sure the kernel is a square. - } - - config_dict['destinations'].append(dest_config) - - # setting of the kill bits need to be done outside this method. - - return config_dict def memory_summary(self): """Computes the amount of memory required for each of the components. Note that this is not @@ -568,4 +476,23 @@ def _get_destinations_input_source(self, sinabs_edges: list) -> dict: if edge[0] == id: destinations_input_source[id].append(edge[1]) - return destinations_input_source \ No newline at end of file + return destinations_input_source + + def get_pool_kernel_size(self, node: int): + """ Returns the pooling kernel size if `node` is a pooling layer.""" + + if node in self.pool_node_id: + i = self.pool_node_id.index(node) + return self.pool_layer[i].kernel_size[0] if isinstance(self.pool_layer[i].kernel_size, tuple) else self.pool_layer[i].kernel_size + else: + return None + + @staticmethod + def find_nodes_core_id(node: int, forward_map: dict) -> int: + + for _, dcnnl in forward_map.items(): + + if node == dcnnl.conv_node_id or node == dcnnl.spk_node_id or node in dcnnl.pool_node_id: + return dcnnl.assigned_core + + raise ValueError(f'Node {node} not found in any of the cores.') diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index 1aedcc57..b065bfcb 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -13,7 +13,7 @@ from .chip_factory import ChipFactory from .dvs_layer import DVSLayer -from .io import open_device +from .io import open_device, disable_timestamps, enable_timestamps, reset_timestamps from .utils import ( DEFAULT_IGNORED_LAYER_TYPES, build_from_graph, @@ -25,7 +25,7 @@ from .sinabs_edges_handler import merge_handler from .dynapcnnnetwork_module import DynapcnnNetworkModule -from .weight_rescaling_methods import rescale_method_1, rescale_method_2 +from .weight_rescaling_methods import rescale_method_1 from .dynapcnn_layer import DynapcnnLayer @@ -61,7 +61,7 @@ def __init__( Some of the properties defined within the class constructor are meant to be temporary data structures handling the conversion of the `snn` (the original `nn.Module`) into a set of `DynapcnnLayer`s composing a `DynapcnnNetwork` instance. Once their role in preprocessing `snn` is finished, all required data to train/deploy the `DynapcnnNetwork` instance is within `self._dcnnl_edges` - (the connectivity between each `DynapcnnLayer`/core), `self._forward_map` (every `DynapcnnLayer` in the network) and `self._merge_points` + (the connectivity between each `DynapcnnLayer`/core), `self._layers_mapper` (every `DynapcnnLayer` in the network) and `self._merge_points` (the `DynapcnnLayer`s that need a `Merge` input). Thus, the following private properties are delted as last step of the constructor: - self._graph_tracer @@ -112,9 +112,9 @@ def __init__( entry_nodes = self._entry_nodes) # these gather all data necessay to implement the forward method for this class. - self._dcnnl_edges, self._forward_map, self._merge_points, self._topological_order = self._get_network_module() + self._dcnnl_edges, self._layers_mapper, self._merge_points, self._topological_order = self._get_network_module() - # all necessary `DynapcnnLayer` data held in `self._forward_map`: removing intermediary data structures no longer necessary. + # all necessary `DynapcnnLayer` data held in `self._layers_mapper`: removing intermediary data structures no longer necessary. del self._graph_tracer del self._sinabs_edges del self._sinabs_modules_map @@ -138,25 +138,74 @@ def topological_order(self): return self._topological_order @property - def forward_map(self) -> Dict[int, DynapcnnLayer]: - """ This dictionary contains each `DynapcnnLayer` in the model indexed by their ID (layer index). - - Returns - ---------- - - self._forward_map (dict): a mapper used to forward data through the `DynapcnnNetwork` instance when `self.forward` is called. + def layers_mapper(self) -> Dict[int, DynapcnnLayer]: + return self._layers_mapper + + @property + def chip_layers_ordering(self): + return self._chip_layers_ordering + + def get_output_core_id(self) -> int: + """ .""" + + # TODO if a network with two output layers is deployed, which is not supported yet btw, this monitoring part needs to be revised. + for _, ith_dcnnl in self._layers_mapper.items(): + if len(ith_dcnnl.dynapcnnlayer_destination) == 0: + # a DynapcnnLayer without destinations is taken to be the output layer of the network. + return ith_dcnnl.assigned_core + + def get_input_core_id(self) -> list: + """ Since the chip allows for multiple input layers (that merge into a single output at some point), this method returns + a list of all core IDs to which an input layer of the network has been assigned to. """ - return self._forward_map + entry_points = [] + for _, ith_dcnnl in self._layers_mapper.items(): + if ith_dcnnl.entry_point: + entry_points.append(ith_dcnnl.assigned_core) + + return entry_points + + def hw_forward(self, x): + """ Forwards data through the chip. """ + + # flush buffer. + _ = self.samna_output_buffer.get_events() + + # NOTE: The code to start and stop time stamping is device specific + reset_timestamps(self.device) + enable_timestamps(self.device) + + # send input. + self.samna_input_buffer.write(x) + received_evts = [] + + # record at least until the last event has been replayed. + min_duration = max(event.timestamp for event in x) * 1e-6 + time.sleep(min_duration) + + # keep recording if more events are being registered. + while True: + prev_length = len(received_evts) + time.sleep(0.1) + received_evts.extend(self.samna_output_buffer.get_events()) + if prev_length == len(received_evts): + break + + # disable timestamp + disable_timestamps(self.device) + + return received_evts def forward(self, x): """ Forwards data through the `DynapcnnNetwork` instance. This method relies on three main data structures created to represent the `DynapcnnLayer`s in the network and the data propagation through them during the forward pass: - - `self._topological_order` (list): this is used to guide the sequence in which the `DynapcnnLayer`s in `self._forward_map` are to be called + - `self._topological_order` (list): this is used to guide the sequence in which the `DynapcnnLayer`s in `self._layers_mapper` are to be called to generate the input tensors to be propagated through the network during the forward pass. - `self._dcnnl_edges` (list): this list of edges represent the graph describing the interactions between each `DynapcnnLayer` (the nodes in the edges are the indices of these layers). An `edge` is used to index a mapper (using `edge[0]`) in order to retrieve the output to be fed as input to a `DynapcnnLayer` instance (indexed by `edge[1]`). - - `self._forward_map` (dict): a mapper used to forward data through the `DynapcnnNetwork` instances. Each `key` is the indice associated + - `self._layers_mapper` (dict): a mapper used to forward data through the `DynapcnnNetwork` instances. Each `key` is the indice associated with a `DynapcnnLayer` instance. - `self._merge_points` (dict): this mapper has a "support" role. It indexes wich convolutional layers in the set of `DynapcnnLayer`s composing the network require two sources of input (because their input tensor is the output of a `Merge` layer). @@ -166,9 +215,9 @@ def forward(self, x): for i in self._topological_order: - if self._forward_map[i].entry_point: + if self._layers_mapper[i].entry_point: # `DynapcnnLayer i` is an entry point of the network. - layers_outputs[i] = self._forward_map[i](x) + layers_outputs[i] = self._layers_mapper[i](x) else: # input to `DynapcnnLayer i` is the output of another instance. @@ -181,8 +230,8 @@ def forward(self, x): # find which returned tensor from the `forward` call of DynapcnnLayers `arg1` and `arg2` are to be fed # to the target DynapcnnLayer `i`. - return_index_arg1 = self._forward_map[arg1].get_destination_dcnnl_index(i) - return_index_arg2 = self._forward_map[arg2].get_destination_dcnnl_index(i) + return_index_arg1 = self._layers_mapper[arg1].get_destination_dcnnl_index(i) + return_index_arg2 = self._layers_mapper[arg2].get_destination_dcnnl_index(i) # retrieve input tensors to `Merge`. _arg1 = layers_outputs[arg1][return_index_arg1] @@ -192,7 +241,7 @@ def forward(self, x): merge_output = self._merge_points[i]['merge'](_arg1, _arg2) # call the forward. - layers_outputs[i] = self._forward_map[i](merge_output) + layers_outputs[i] = self._layers_mapper[i](merge_output) else: # there's a single source of input for `DynapcnnLayer i`. @@ -202,10 +251,10 @@ def forward(self, x): # find which returned tensor from the `forward` call of the source DynapcnnLayer `src_dcnnl` is to be fed # to the target DynapcnnLayer `i`. - return_index = self._forward_map[src_dcnnl].get_destination_dcnnl_index(i) + return_index = self._layers_mapper[src_dcnnl].get_destination_dcnnl_index(i) # call the forward. - layers_outputs[i] = self._forward_map[i](layers_outputs[src_dcnnl][return_index]) + layers_outputs[i] = self._layers_mapper[i](layers_outputs[src_dcnnl][return_index]) # TODO - this assumes the network has a single output node. return layers_outputs[self._topological_order[-1]][0] @@ -222,7 +271,7 @@ def parameters(self) -> list: """ parameters = [] - for layer in self._forward_map.values(): + for layer in self._layers_mapper.values(): if isinstance(layer, sinabs.backend.dynapcnn.dynapcnn_layer.DynapcnnLayer): parameters.extend(layer.conv_layer.parameters()) @@ -235,14 +284,14 @@ def init_weights(self, init_fn: nn.init = nn.init.xavier_normal_) -> None: ---------- - init_fn (torch.nn.init): the weight initialization method to be used. """ - for layer in self._forward_map.values(): + for layer in self._layers_mapper.values(): if isinstance(layer, sinabs.backend.dynapcnn.dynapcnn_layer.DynapcnnLayer): init_fn(layer.conv_layer.weight.data) def detach_neuron_states(self) -> None: """ Detach the neuron states and activations from current computation graph (necessary). """ - for module in self._forward_map.values(): + for module in self._layers_mapper.values(): if isinstance(module, sinabs.backend.dynapcnn.dynapcnn_layer.DynapcnnLayer): if isinstance(module.spk_layer, sl.StatefulLayer): for name, buffer in module.spk_layer.named_buffers(): @@ -350,7 +399,7 @@ def to( self.device_output_graph.start() self.samna_config = config - return print(self) + return self else: self._to_device(device) @@ -418,11 +467,11 @@ def _make_config( config_builder = ChipFactory(device).get_config_builder() # TODO not handling DVSLayer yet. - has_dvs_layer = isinstance(self._forward_map[0], DVSLayer) + has_dvs_layer = isinstance(self._layers_mapper[0], DVSLayer) if chip_layers_ordering == "auto": - # figure out mapping of each DynapcnnLayer into one core. - chip_layers_ordering = config_builder.get_valid_mapping(self) + # figure out mapping of each DynapcnnLayer into one core (core ID will be set in the layer instance via `layer.assigned_core`). + _ = config_builder.get_valid_mapping(self) else: # TODO - mapping from each DynapcnnLayer into cores has been provided by the user: NOT IMPLEMENTED YET. @@ -431,7 +480,7 @@ def _make_config( pass # update config. - config = config_builder.build_config(self, None) + config = config_builder.build_config(self) # TODO not handling DVSLayer yet (this is from the old implementation, should be revised). if self.input_shape and self.input_shape[0] == 1: @@ -441,12 +490,15 @@ def _make_config( monitor_chip_layers = [] if monitor_layers is None: # check if any monitoring is enabled (if not, enable monitoring for the last layer). - for dcnnl_index, ith_dcnnl in self._forward_map.items(): + for dcnnl_index, ith_dcnnl in self._layers_mapper.items(): + + # TODO if a network with two output layers is deployed, which is not supported yet btw, this monitoring part needs to be revised. if len(ith_dcnnl.dynapcnnlayer_destination) == 0: + # a DynapcnnLayer without destinations is taken to be the output layer of the network. monitor_chip_layers.append(ith_dcnnl.assigned_core) - break + elif monitor_layers == "all": - for dcnnl_index, ith_dcnnl in self._forward_map.items(): + for dcnnl_index, ith_dcnnl in self._layers_mapper.items(): # TODO not handling DVSLayer yet # monitor each chip core (if not a DVSLayer). if not isinstance(ith_dcnnl, DVSLayer): @@ -614,7 +666,7 @@ def find_my_input(edges_list: list, node: int) -> int: def _to_device(self, device: torch.device) -> None: """ Access each sub-layer within all `DynapcnnLayer` instances and call `.to(device)` on them.""" - for layer in self._forward_map.values(): + for layer in self._layers_mapper.values(): if isinstance(layer, sinabs.backend.dynapcnn.dynapcnn_layer.DynapcnnLayer): layer.to(device) @@ -623,7 +675,7 @@ def _to_device(self, device: torch.device) -> None: def __str__(self): pretty_print = '' - for idx, layer_data in self._forward_map.items(): + for idx, layer_data in self._layers_mapper.items(): pretty_print += f'----------------------- [ DynapcnnLayer {idx} ] -----------------------\n' pretty_print += f'{layer_data}\n\n' diff --git a/sinabs/backend/dynapcnn/mapping.py b/sinabs/backend/dynapcnn/mapping.py index c93c6b3e..964cf987 100644 --- a/sinabs/backend/dynapcnn/mapping.py +++ b/sinabs/backend/dynapcnn/mapping.py @@ -64,7 +64,7 @@ def get_valid_mapping( layer_mapping = [] if type(model) == sinabs.backend.dynapcnn.dynapcnn_network.DynapcnnNetwork: - for dcnnl_index, ith_dcnnl in model.forward_map.items(): + for dcnnl_index, ith_dcnnl in model.layers_mapper.items(): if isinstance(ith_dcnnl, DynapcnnLayer): layer_mapping.append(find_chip_layers(ith_dcnnl, constraints)) else: From 5edcf9a6fe9b430c887b250a5cead0cb9acfe6aa Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Thu, 30 May 2024 15:06:57 +0200 Subject: [PATCH 116/379] Refactor Added verification to '.get_pool_kernel_size()' to make sure node passed as argument is one of the layers in the instance. --- sinabs/backend/dynapcnn/dynapcnn_layer.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer.py b/sinabs/backend/dynapcnn/dynapcnn_layer.py index f4433680..caca930a 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer.py @@ -478,14 +478,16 @@ def _get_destinations_input_source(self, sinabs_edges: list) -> dict: return destinations_input_source - def get_pool_kernel_size(self, node: int): + def get_pool_kernel_size(self, node: int) -> int: """ Returns the pooling kernel size if `node` is a pooling layer.""" if node in self.pool_node_id: i = self.pool_node_id.index(node) return self.pool_layer[i].kernel_size[0] if isinstance(self.pool_layer[i].kernel_size, tuple) else self.pool_layer[i].kernel_size + elif node == self.spk_node_id: + return 1 else: - return None + raise ValueError(f'Node {node} does not belong to DynapcnnLayer {self.dpcnnl_index}.') @staticmethod def find_nodes_core_id(node: int, forward_map: dict) -> int: From c55d1859aa8566c90b4b333be98d8d2e17bc6993 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Thu, 30 May 2024 15:35:54 +0200 Subject: [PATCH 117/379] Refactor Updated call from '.forward_map' to '.layers_mapper'. --- tests/test_dynapcnnnetwork/test_dynapcnnnetwork.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_dynapcnnnetwork/test_dynapcnnnetwork.py b/tests/test_dynapcnnnetwork/test_dynapcnnnetwork.py index 4f8f3bd7..8f7471b7 100644 --- a/tests/test_dynapcnnnetwork/test_dynapcnnnetwork.py +++ b/tests/test_dynapcnnnetwork/test_dynapcnnnetwork.py @@ -29,7 +29,7 @@ def test_DynapcnnNetwork(snn, input_shape, batch_size, expected_output): f'DynapcnnLayer {node} has wrong input sources ({args}).' for entry_point in expected_output['entry_point']: - assert dcnnnet.forward_map[entry_point].entry_point, \ + assert dcnnnet.layers_mapper[entry_point].entry_point, \ f'DynapcnnLayer {entry_point} should be an entry point.' assert expected_output['topological_order'] == dcnnnet.topological_order, \ From db9c670e25548fa5e4aa5238b87a4f0b97ed13cb Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Fri, 31 May 2024 16:40:42 +0200 Subject: [PATCH 118/379] Missing from previous commit - DynapcnnLayer accessible from dynapcnn/__init__. - Removed wild prints. --- sinabs/backend/dynapcnn/__init__.py | 9 +++++---- sinabs/backend/dynapcnn/chips/dynapcnn.py | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/sinabs/backend/dynapcnn/__init__.py b/sinabs/backend/dynapcnn/__init__.py index b783baca..4c32f1ee 100644 --- a/sinabs/backend/dynapcnn/__init__.py +++ b/sinabs/backend/dynapcnn/__init__.py @@ -1,8 +1,9 @@ -from .dynapcnn_network import ( # second one for compatibility purposes - DynapcnnCompatibleNetwork, +from .dynapcnn_network import ( DynapcnnNetwork, ) -from .dynapcnn_network_graph import ( - DynapcnnNetworkGraph, + +from .dynapcnn_layer import ( + DynapcnnLayer, ) + from .dynapcnn_visualizer import DynapcnnVisualizer diff --git a/sinabs/backend/dynapcnn/chips/dynapcnn.py b/sinabs/backend/dynapcnn/chips/dynapcnn.py index e766f03a..3a55063e 100644 --- a/sinabs/backend/dynapcnn/chips/dynapcnn.py +++ b/sinabs/backend/dynapcnn/chips/dynapcnn.py @@ -194,7 +194,7 @@ def write_dynapcnn_layer_config(cls, layer: DynapcnnLayer, chip_layer: "CNNLayer # update configuration of the DYNAPCNN layer. chip_layer.dimensions = config_dict["dimensions"] config_dict.pop("dimensions") - + # set the destinations configuration. for i in range(len(config_dict['destinations'])): chip_layer.destinations[i].layer = config_dict['destinations'][i]['layer'] From b938d3f7ff5b62a8895e07846cc21f2e7f4777ff Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Fri, 31 May 2024 16:42:56 +0200 Subject: [PATCH 119/379] full new deployment example #1 --- .../dynapcnn_network/snn_deployment.ipynb | 830 ++++++++++++++++++ 1 file changed, 830 insertions(+) create mode 100644 examples/dynapcnn_network/snn_deployment.ipynb diff --git a/examples/dynapcnn_network/snn_deployment.ipynb b/examples/dynapcnn_network/snn_deployment.ipynb new file mode 100644 index 00000000..e89e9f1b --- /dev/null +++ b/examples/dynapcnn_network/snn_deployment.ipynb @@ -0,0 +1,830 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "metadata": {} + }, + "outputs": [], + "source": [ + "import torch\n", + "import torch.nn as nn\n", + "from sinabs.backend.dynapcnn import DynapcnnNetwork\n", + "from sinabs.layers import Merge, IAFSqueeze, SumPool2d\n", + "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential\n", + "import sinabs.layers as sl\n", + "\n", + "from torch.nn import CrossEntropyLoss\n", + "from torch.optim import Adam\n", + "\n", + "from tonic.datasets.nmnist import NMNIST\n", + "from tonic.transforms import ToFrame\n", + "from torch.utils.data import DataLoader\n", + "import numpy as np\n", + "from tqdm.notebook import tqdm\n", + "from statistics import mode" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "metadata": {} + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "torch.manual_seed(0)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Network Module\n", + "\n", + "We need to define a `nn.Module` implementing the Spiking Neural Network (SNN) we want to deploy on chip. The configuration of the network on the chip needs to know in advance the shape of the input data and the batch size that will be used." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "channels = 2\n", + "height = 34\n", + "width = 34\n", + "batch_size = 8\n", + "\n", + "input_shape = (channels, height, width)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "metadata": {} + }, + "outputs": [], + "source": [ + "class SNN(nn.Module):\n", + " def __init__(self) -> None:\n", + " super().__init__()\n", + " # -- chip core A --\n", + " self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False)\n", + " self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential())\n", + " self.pool1 = nn.AvgPool2d(2,2)\n", + " # -- chip core B --\n", + " self.conv2 = nn.Conv2d(10, 10, 4, 1, bias=False)\n", + " self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential())\n", + " # -- chip core C --\n", + " self.conv3 = nn.Conv2d(10, 1, 2, 1, bias=False)\n", + " self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential())\n", + " # -- chip core D --\n", + " self.fc1 = nn.Linear(144, 200, bias=False)\n", + " self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential())\n", + " # -- chip core E --\n", + " self.fc2 = nn.Linear(200, 10, bias=False)\n", + " self.iaf5 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential())\n", + "\n", + " # -- layers ignored during deployment --\n", + " self.flat = nn.Flatten()\n", + "\n", + " def init_weights(self):\n", + " for name, layer in self.named_modules():\n", + " if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear):\n", + " nn.init.xavier_normal_(layer.weight.data)\n", + "\n", + " def detach_neuron_states(self):\n", + " for name, layer in self.named_modules():\n", + " if name != '':\n", + " if isinstance(layer, sl.StatefulLayer):\n", + " for name, buffer in layer.named_buffers():\n", + " buffer.detach_()\n", + "\n", + " def forward(self, x):\n", + " \n", + " con1_out = self.conv1(x)\n", + " iaf1_out = self.iaf1(con1_out)\n", + " pool1_out = self.pool1(iaf1_out)\n", + "\n", + " conv2_out = self.conv2(pool1_out)\n", + " iaf2_out = self.iaf2(conv2_out)\n", + "\n", + " conv3_out = self.conv3(iaf2_out)\n", + " iaf3_out = self.iaf3(conv3_out)\n", + "\n", + " flat_out = self.flat(iaf3_out)\n", + " \n", + " fc1_out = self.fc1(flat_out)\n", + " iaf4_out = self.iaf4(fc1_out)\n", + " fc2_out = self.fc2(iaf4_out)\n", + " iaf5_out = self.iaf5(fc2_out)\n", + "\n", + " return iaf5_out" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "metadata": {} + }, + "outputs": [], + "source": [ + "snn = SNN()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's train the model to see what kind of accuracy the software model gets:" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The transformed array is in shape [Time-Step, Channel, Height, Width] --> (50, 2, 34, 34)\n" + ] + } + ], + "source": [ + "_ = NMNIST(save_to='./NMNIST', train=True)\n", + "_ = NMNIST(save_to='./NMNIST', train=False)\n", + "\n", + "nb_time_steps = 50\n", + "to_raster = ToFrame(sensor_size=NMNIST.sensor_size, n_time_bins=nb_time_steps)\n", + "\n", + "snn_train_dataset = NMNIST(save_to='./NMNIST', train=True, transform=to_raster)\n", + "snn_test_dataset = NMNIST(save_to='./NMNIST', train=False, transform=to_raster)\n", + "\n", + "sample_data, label = snn_train_dataset[0]\n", + "print(f\"The transformed array is in shape [Time-Step, Channel, Height, Width] --> {sample_data.shape}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "train_indices = [i for i in range(1000)]\n", + "test_indices = [i for i in range(100)]\n", + "\n", + "snn_train_dataset_subset = torch.utils.data.Subset(snn_train_dataset, train_indices)\n", + "snn_test_subset = torch.utils.data.Subset(snn_train_dataset, test_indices)\n", + "\n", + "snn_train_dataloader = DataLoader(snn_train_dataset_subset, batch_size=batch_size, num_workers=4, drop_last=True, shuffle=True)\n", + "snn_test_dataloader = DataLoader(snn_test_subset, batch_size=batch_size, num_workers=4, drop_last=True, shuffle=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "SNN(\n", + " (conv1): Conv2d(2, 10, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + " (iaf1): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(1.), min_v_mem=Parameter containing:\n", + " tensor(-1.), batch_size=8, num_timesteps=-1)\n", + " (pool1): AvgPool2d(kernel_size=2, stride=2, padding=0)\n", + " (conv2): Conv2d(10, 10, kernel_size=(4, 4), stride=(1, 1), bias=False)\n", + " (iaf2): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(1.), min_v_mem=Parameter containing:\n", + " tensor(-1.), batch_size=8, num_timesteps=-1)\n", + " (conv3): Conv2d(10, 1, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + " (iaf3): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(1.), min_v_mem=Parameter containing:\n", + " tensor(-1.), batch_size=8, num_timesteps=-1)\n", + " (fc1): Linear(in_features=144, out_features=200, bias=False)\n", + " (iaf4): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(1.), min_v_mem=Parameter containing:\n", + " tensor(-1.), batch_size=8, num_timesteps=-1)\n", + " (fc2): Linear(in_features=200, out_features=10, bias=False)\n", + " (iaf5): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(1.), min_v_mem=Parameter containing:\n", + " tensor(-1.), batch_size=8, num_timesteps=-1)\n", + " (flat): Flatten(start_dim=1, end_dim=-1)\n", + ")" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "device = torch.device('cpu')\n", + "\n", + "snn.init_weights()\n", + "\n", + "snn.to(device)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "optimizer = Adam(snn.parameters(), lr=1e-4, betas=(0.9, 0.999), eps=1e-8)\n", + "loss_fn = CrossEntropyLoss()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "training the model..." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "c5579aac6828434dbac67d04236a87c0", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/125 [00:00DynapcnnLayer in the model has yet to be assigned to a core. This is only done once\n", + "DynapcnnNetworkGraph.to() is called." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "----------------------- [ DynapcnnLayer 0 ] -----------------------\n", + "\n", + "COMPUTATIONAL NODES:\n", + "\n", + "(node 0): Conv2d(2, 10, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + "(node 1): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(241.), min_v_mem=Parameter containing:\n", + "tensor(-241.), batch_size=8, num_timesteps=-1)\n", + "(node 2): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", + "\n", + "METADATA:\n", + "\n", + "> network's entry point: True\n", + "> convolution's weight re-scaling factor: None\n", + "> assigned core index: None\n", + "> destination DynapcnnLayers: [1]\n", + "> node 2 feeds input to nodes [3]\n", + "\n", + "----------------------- [ DynapcnnLayer 1 ] -----------------------\n", + "\n", + "COMPUTATIONAL NODES:\n", + "\n", + "(node 3): Conv2d(10, 10, kernel_size=(4, 4), stride=(1, 1), bias=False)\n", + "(node 4): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1041.), min_v_mem=Parameter containing:\n", + "tensor(-1041.), batch_size=8, num_timesteps=-1)\n", + "\n", + "METADATA:\n", + "\n", + "> network's entry point: False\n", + "> convolution's weight re-scaling factor: 2.0\n", + "> assigned core index: None\n", + "> destination DynapcnnLayers: [2]\n", + "> node 4 feeds input to nodes [5]\n", + "\n", + "----------------------- [ DynapcnnLayer 2 ] -----------------------\n", + "\n", + "COMPUTATIONAL NODES:\n", + "\n", + "(node 5): Conv2d(10, 1, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + "(node 6): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(247.), min_v_mem=Parameter containing:\n", + "tensor(-247.), batch_size=8, num_timesteps=-1)\n", + "\n", + "METADATA:\n", + "\n", + "> network's entry point: False\n", + "> convolution's weight re-scaling factor: None\n", + "> assigned core index: None\n", + "> destination DynapcnnLayers: [3]\n", + "> node 6 feeds input to nodes [7]\n", + "\n", + "----------------------- [ DynapcnnLayer 3 ] -----------------------\n", + "\n", + "COMPUTATIONAL NODES:\n", + "\n", + "(node 7): Conv2d(1, 200, kernel_size=(12, 12), stride=(1, 1), bias=False)\n", + "(node 8): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(363.), min_v_mem=Parameter containing:\n", + "tensor(-363.), batch_size=8, num_timesteps=-1)\n", + "\n", + "METADATA:\n", + "\n", + "> network's entry point: False\n", + "> convolution's weight re-scaling factor: None\n", + "> assigned core index: None\n", + "> destination DynapcnnLayers: [4]\n", + "> node 8 feeds input to nodes [9]\n", + "\n", + "----------------------- [ DynapcnnLayer 4 ] -----------------------\n", + "\n", + "COMPUTATIONAL NODES:\n", + "\n", + "(node 9): Conv2d(200, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + "(node 10): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(422.), min_v_mem=Parameter containing:\n", + "tensor(-422.), batch_size=8, num_timesteps=-1)\n", + "\n", + "METADATA:\n", + "\n", + "> network's entry point: False\n", + "> convolution's weight re-scaling factor: None\n", + "> assigned core index: None\n", + "> destination DynapcnnLayers: []\n", + "\n", + "\n" + ] + } + ], + "source": [ + "print(hw_model)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The `hw_model.to()` call will figure out into which core each `DynapcnnLayer` instance will be assigned to. Once this assingment is made the instance itself is used to configure the `CNNLayerConfig` instance representing the core's configuration assigned to it.\n", + "\n", + "If the call is sucessfull, the layers comprising the network and their associated metadata will be printed. To deploy the model, we need to provide the device string defining what Speck devkit is being used." + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "speck_device = \"speck2fmodule:0\"" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": { + "metadata": {} + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Network is valid: \n", + "\n" + ] + }, + { + "data": { + "text/plain": [ + "DynapcnnNetwork()" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "hw_model.to(device=speck_device)" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "----------------------- [ DynapcnnLayer 0 ] -----------------------\n", + "\n", + "COMPUTATIONAL NODES:\n", + "\n", + "(node 0): Conv2d(2, 10, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + "(node 1): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(241.), min_v_mem=Parameter containing:\n", + "tensor(-241.), batch_size=8, num_timesteps=-1)\n", + "(node 2): SumPool2d(norm_type=1, kernel_size=(2, 2), stride=None, ceil_mode=False)\n", + "\n", + "METADATA:\n", + "\n", + "> network's entry point: True\n", + "> convolution's weight re-scaling factor: None\n", + "> assigned core index: 0\n", + "> destination DynapcnnLayers: [1]\n", + "> node 2 feeds input to nodes [3]\n", + "\n", + "----------------------- [ DynapcnnLayer 1 ] -----------------------\n", + "\n", + "COMPUTATIONAL NODES:\n", + "\n", + "(node 3): Conv2d(10, 10, kernel_size=(4, 4), stride=(1, 1), bias=False)\n", + "(node 4): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(1041.), min_v_mem=Parameter containing:\n", + "tensor(-1041.), batch_size=8, num_timesteps=-1)\n", + "\n", + "METADATA:\n", + "\n", + "> network's entry point: False\n", + "> convolution's weight re-scaling factor: 2.0\n", + "> assigned core index: 1\n", + "> destination DynapcnnLayers: [2]\n", + "> node 4 feeds input to nodes [5]\n", + "\n", + "----------------------- [ DynapcnnLayer 2 ] -----------------------\n", + "\n", + "COMPUTATIONAL NODES:\n", + "\n", + "(node 5): Conv2d(10, 1, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + "(node 6): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(247.), min_v_mem=Parameter containing:\n", + "tensor(-247.), batch_size=8, num_timesteps=-1)\n", + "\n", + "METADATA:\n", + "\n", + "> network's entry point: False\n", + "> convolution's weight re-scaling factor: None\n", + "> assigned core index: 2\n", + "> destination DynapcnnLayers: [3]\n", + "> node 6 feeds input to nodes [7]\n", + "\n", + "----------------------- [ DynapcnnLayer 3 ] -----------------------\n", + "\n", + "COMPUTATIONAL NODES:\n", + "\n", + "(node 7): Conv2d(1, 200, kernel_size=(12, 12), stride=(1, 1), bias=False)\n", + "(node 8): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(363.), min_v_mem=Parameter containing:\n", + "tensor(-363.), batch_size=8, num_timesteps=-1)\n", + "\n", + "METADATA:\n", + "\n", + "> network's entry point: False\n", + "> convolution's weight re-scaling factor: None\n", + "> assigned core index: 5\n", + "> destination DynapcnnLayers: [4]\n", + "> node 8 feeds input to nodes [9]\n", + "\n", + "----------------------- [ DynapcnnLayer 4 ] -----------------------\n", + "\n", + "COMPUTATIONAL NODES:\n", + "\n", + "(node 9): Conv2d(200, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + "(node 10): IAFSqueeze(spike_threshold=Parameter containing:\n", + "tensor(422.), min_v_mem=Parameter containing:\n", + "tensor(-422.), batch_size=8, num_timesteps=-1)\n", + "\n", + "METADATA:\n", + "\n", + "> network's entry point: False\n", + "> convolution's weight re-scaling factor: None\n", + "> assigned core index: 3\n", + "> destination DynapcnnLayers: []\n", + "\n", + "\n" + ] + } + ], + "source": [ + "print(hw_model)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Spikes IN/Out of the Chip\n", + "\n", + "Let's try to use our network configured on the chip to forward some data. We'll get a sample from the NMNIST dataset to do that:" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "event_dataset = NMNIST(save_to='./NMNIST', train=False)\n", + "event_subset = torch.utils.data.Subset(event_dataset, test_indices)" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "targets = np.array(event_dataset.targets)\n", + "target_indices = {idx: np.where(targets == idx)[0] for idx in range(10)}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If you have a tensor with data and want to convert it to input_events, you would instantiate a ChipFactory object providing the device string (\"speck2fsomethingsomething\") as instantiation argument. For further details consult the [documentation](https://sinabs.readthedocs.io/en/v2.0.0/tutorials/nir_to_speck.html#prepare-dataset)." + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [], + "source": [ + "from sinabs.backend.dynapcnn.chip_factory import ChipFactory\n", + "\n", + "chip_factory = ChipFactory(speck_device)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This object has a method raster_to_events (see more [here](https://sinabs.readthedocs.io/en/v2.0.0/speck/api/dynapcnn/chip_factory.html#sinabs.backend.dynapcnn.chip_factory.ChipFactory.raster_to_events)) that can convert your data to an event list, which is what the chip expects. This method requires a 4 dimensional tensor of spike events with the dimensions [Time, Channel, Height, Width]." + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "output core id: 3\n", + "input core id: 0\n" + ] + } + ], + "source": [ + "layer_out = hw_model.get_output_core_id() # core assigned to the output layer of the model\n", + "layer_in = hw_model.get_input_core_id()[-1] # core assigned to the input layyer of the model\n", + "\n", + "print(f'output core id: {layer_out}')\n", + "print(f'input core id: {layer_in}')" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Output layer monitoring: True\n" + ] + } + ], + "source": [ + "print(f'Output layer monitoring: {hw_model.samna_config.cnn_layers[layer_out].monitor_enable}')" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "646e50af00044f1ab9625886fe3f36ed", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/100 [00:00 Date: Fri, 31 May 2024 16:51:00 +0200 Subject: [PATCH 120/379] removed folder with algo. exploration simulations --- .../DynapcnnNetwork-example_1.ipynb | 597 --- .../DynapcnnNetwork-example_2.ipynb | 396 -- .../DynapcnnNetwork-example_3.ipynb | 402 -- .../DynapcnnNetwork-example_4.ipynb | 380 -- .../DynapcnnNetwork-example_5.ipynb | 400 -- .../DynapcnnNetwork-example_5a.ipynb | 390 -- .../DynapcnnNetwork-example_6.ipynb | 422 -- .../complex_network_structure.ipynb | 337 -- .../split_and_merge.ipynb | 359 -- .../two_networks_merging_outputs.ipynb | 378 -- .../DynapcnnNetwork-example_1.ipynb | 679 --- tests/test_nonsequential/NNI-test/main.ipynb | 136 - tests/test_nonsequential/NNI-test/model.py | 175 - .../baseline-SCNN-example_1-NNI.ipynb | 613 --- .../baseline-SCNN-example_2.ipynb | 630 --- .../baseline-SCNN-example_3.ipynb | 1500 ------ .../exp_set_A/baseline-SCNN-example_3.ipynb | 1500 ------ .../non-sequential-SCNN-example_3.ipynb | 1509 ------ .../exp_set_B/baseline-SCNN-example_3.ipynb | 1512 ------ .../baseline_exp_set_B_training_metrics.npy | Bin 172944 -> 0 bytes .../non-sequential-SCNN-example_3.ipynb | 1521 ------ .../exp_set_B1/baseline-SCNN-example_3.ipynb | 1071 ---- .../non-sequential-SCNN-example_3.ipynb | 1509 ------ .../exp_set_TA1/main_loop.py | 5 - .../exp_set_TA1/nonseq_conv1_weights.pth | Bin 1693 -> 0 bytes .../exp_set_TA1/nonseq_conv2_weights.pth | Bin 2973 -> 0 bytes .../exp_set_TA1/nonseq_conv3_weights.pth | Bin 4957 -> 0 bytes .../exp_set_TA1/nonseq_fc2_weights.pth | Bin 41299 -> 0 bytes .../exp_set_TA1/nonseq_fc3_weights.pth | Bin 41299 -> 0 bytes .../exp_set_TA1/nonseq_model.py | 117 - .../exp_set_TA1/train_script.py | 152 - .../non-sequential-SCNN-example_1.ipynb | 649 --- .../non-sequential-SCNN-example_2.ipynb | 639 --- .../non-sequential-SCNN-example_3.ipynb | 1509 ------ .../transfer-learning/baseline-SCNN-3.ipynb | 525 -- .../transfer-learning/seq_model.py | 86 - .../baseline-SCNN-example_3-SumPool.ipynb | 1539 ------ .../exp_set_A/baseline-SCNN-example_3.ipynb | 1500 ------ .../ARCHITECTURES_SEARCH/Res-SCNN3.ipynb | 1380 ----- .../architectures_results.ipynb | 306 -- .../ARCHITECTURES_SEARCH/main.py | 136 - .../ARCHITECTURES_SEARCH/model_training.py | 92 - .../single_training.ipynb | 4716 ----------------- .../ARCHITECTURES_SEARCH/train_all.py | 4 - .../HPO_GAUSSIAN_SEARCH/GS_utils.py | 54 - .../gaussian_search_history.csv | 57 - .../HPO_GAUSSIAN_SEARCH/main.py | 202 - .../HPO_GAUSSIAN_SEARCH/network.py | 83 - .../using_SumPool2d/Res-SCNN3.ipynb | 1509 ------ .../TOP_2_ARCHITECTURES/single_training.ipynb | 3837 -------------- .../using_SumPool2d/models/ResSCNN_1.py | 152 - .../using_SumPool2d/models/ResSCNN_10.py | 121 - .../using_SumPool2d/models/ResSCNN_11.py | 112 - .../using_SumPool2d/models/ResSCNN_12.py | 91 - .../using_SumPool2d/models/ResSCNN_13.py | 96 - .../using_SumPool2d/models/ResSCNN_2.py | 148 - .../using_SumPool2d/models/ResSCNN_3.py | 151 - .../using_SumPool2d/models/ResSCNN_4.py | 154 - .../using_SumPool2d/models/ResSCNN_5.py | 143 - .../using_SumPool2d/models/ResSCNN_6.py | 146 - .../using_SumPool2d/models/ResSCNN_7.py | 146 - .../using_SumPool2d/models/ResSCNN_8.py | 151 - .../using_SumPool2d/models/ResSCNN_9.py | 124 - .../using_SumPool2d/models/SCNN.py | 130 - .../test_nonsequential/utils/train_test_fn.py | 274 - .../utils/weight_initialization.py | 49 - 66 files changed, 37701 deletions(-) delete mode 100644 tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_1.ipynb delete mode 100644 tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_2.ipynb delete mode 100644 tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_3.ipynb delete mode 100644 tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_4.ipynb delete mode 100644 tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_5.ipynb delete mode 100644 tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_5a.ipynb delete mode 100644 tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_6.ipynb delete mode 100644 tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/complex_network_structure.ipynb delete mode 100644 tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/split_and_merge.ipynb delete mode 100644 tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/two_networks_merging_outputs.ipynb delete mode 100644 tests/test_nonsequential/DynapcnnNetwork-example_1.ipynb delete mode 100644 tests/test_nonsequential/NNI-test/main.ipynb delete mode 100644 tests/test_nonsequential/NNI-test/model.py delete mode 100644 tests/test_nonsequential/baseline-SCNN-example_1-NNI.ipynb delete mode 100644 tests/test_nonsequential/baseline-SCNN-example_2.ipynb delete mode 100644 tests/test_nonsequential/baseline-SCNN-example_3.ipynb delete mode 100644 tests/test_nonsequential/exp_set_A/baseline-SCNN-example_3.ipynb delete mode 100644 tests/test_nonsequential/exp_set_A/non-sequential-SCNN-example_3.ipynb delete mode 100644 tests/test_nonsequential/exp_set_B/baseline-SCNN-example_3.ipynb delete mode 100644 tests/test_nonsequential/exp_set_B/baseline_exp_set_B_training_metrics.npy delete mode 100644 tests/test_nonsequential/exp_set_B/non-sequential-SCNN-example_3.ipynb delete mode 100644 tests/test_nonsequential/exp_set_B1/baseline-SCNN-example_3.ipynb delete mode 100644 tests/test_nonsequential/exp_set_B1/non-sequential-SCNN-example_3.ipynb delete mode 100644 tests/test_nonsequential/exp_set_TA1/main_loop.py delete mode 100644 tests/test_nonsequential/exp_set_TA1/nonseq_conv1_weights.pth delete mode 100644 tests/test_nonsequential/exp_set_TA1/nonseq_conv2_weights.pth delete mode 100644 tests/test_nonsequential/exp_set_TA1/nonseq_conv3_weights.pth delete mode 100644 tests/test_nonsequential/exp_set_TA1/nonseq_fc2_weights.pth delete mode 100644 tests/test_nonsequential/exp_set_TA1/nonseq_fc3_weights.pth delete mode 100644 tests/test_nonsequential/exp_set_TA1/nonseq_model.py delete mode 100644 tests/test_nonsequential/exp_set_TA1/train_script.py delete mode 100644 tests/test_nonsequential/non-sequential-SCNN-example_1.ipynb delete mode 100644 tests/test_nonsequential/non-sequential-SCNN-example_2.ipynb delete mode 100644 tests/test_nonsequential/non-sequential-SCNN-example_3.ipynb delete mode 100644 tests/test_nonsequential/transfer-learning/baseline-SCNN-3.ipynb delete mode 100644 tests/test_nonsequential/transfer-learning/seq_model.py delete mode 100644 tests/test_nonsequential/using_AvgPool2d/exp_set_A/baseline-SCNN-example_3-SumPool.ipynb delete mode 100644 tests/test_nonsequential/using_AvgPool2d/exp_set_A/baseline-SCNN-example_3.ipynb delete mode 100644 tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/Res-SCNN3.ipynb delete mode 100644 tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/architectures_results.ipynb delete mode 100644 tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/main.py delete mode 100644 tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/model_training.py delete mode 100644 tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/single_training.ipynb delete mode 100644 tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/train_all.py delete mode 100644 tests/test_nonsequential/using_SumPool2d/HPO_GAUSSIAN_SEARCH/GS_utils.py delete mode 100644 tests/test_nonsequential/using_SumPool2d/HPO_GAUSSIAN_SEARCH/gaussian_search_history.csv delete mode 100644 tests/test_nonsequential/using_SumPool2d/HPO_GAUSSIAN_SEARCH/main.py delete mode 100644 tests/test_nonsequential/using_SumPool2d/HPO_GAUSSIAN_SEARCH/network.py delete mode 100644 tests/test_nonsequential/using_SumPool2d/Res-SCNN3.ipynb delete mode 100644 tests/test_nonsequential/using_SumPool2d/TOP_2_ARCHITECTURES/single_training.ipynb delete mode 100644 tests/test_nonsequential/using_SumPool2d/models/ResSCNN_1.py delete mode 100644 tests/test_nonsequential/using_SumPool2d/models/ResSCNN_10.py delete mode 100644 tests/test_nonsequential/using_SumPool2d/models/ResSCNN_11.py delete mode 100644 tests/test_nonsequential/using_SumPool2d/models/ResSCNN_12.py delete mode 100644 tests/test_nonsequential/using_SumPool2d/models/ResSCNN_13.py delete mode 100644 tests/test_nonsequential/using_SumPool2d/models/ResSCNN_2.py delete mode 100644 tests/test_nonsequential/using_SumPool2d/models/ResSCNN_3.py delete mode 100644 tests/test_nonsequential/using_SumPool2d/models/ResSCNN_4.py delete mode 100644 tests/test_nonsequential/using_SumPool2d/models/ResSCNN_5.py delete mode 100644 tests/test_nonsequential/using_SumPool2d/models/ResSCNN_6.py delete mode 100644 tests/test_nonsequential/using_SumPool2d/models/ResSCNN_7.py delete mode 100644 tests/test_nonsequential/using_SumPool2d/models/ResSCNN_8.py delete mode 100644 tests/test_nonsequential/using_SumPool2d/models/ResSCNN_9.py delete mode 100644 tests/test_nonsequential/using_SumPool2d/models/SCNN.py delete mode 100644 tests/test_nonsequential/utils/train_test_fn.py delete mode 100644 tests/test_nonsequential/utils/weight_initialization.py diff --git a/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_1.ipynb b/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_1.ipynb deleted file mode 100644 index c4a3c8b7..00000000 --- a/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_1.ipynb +++ /dev/null @@ -1,597 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "import torch\n", - "import torch.nn as nn\n", - "from sinabs.backend.dynapcnn import DynapcnnNetwork\n", - "from sinabs.layers import Merge, IAFSqueeze, SumPool2d\n", - "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "metadata": {} - }, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "torch.manual_seed(0)" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "channels = 2\n", - "height = 34\n", - "width = 34\n", - "batch_size = 8\n", - "\n", - "input_shape = (channels, height, width)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Network Module\n", - "\n", - "We need to define a `nn.Module` implementing the network we want the chip to reproduce." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "class SNN(nn.Module):\n", - " def __init__(self) -> None:\n", - " super().__init__()\n", - "\n", - " self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False) # node 0\n", - " self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) # node 1\n", - " self.pool1 = nn.AvgPool2d(3,3) # node 2\n", - " self.pool1a = nn.AvgPool2d(4,4) # node 3\n", - "\n", - " self.conv2 = nn.Conv2d(10, 10, 4, 1, bias=False)# node 4\n", - " self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) # node 6\n", - "\n", - " self.conv3 = nn.Conv2d(10, 1, 2, 1, bias=False) # node 8\n", - " self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) # node 9\n", - "\n", - " self.flat = nn.Flatten()\n", - "\n", - " self.fc1 = nn.Linear(49, 500, bias=False) # node 10\n", - " self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) # node 11\n", - " \n", - " self.fc2 = nn.Linear(500, 10, bias=False) # node 12\n", - " self.iaf5 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) # node 13\n", - "\n", - " self.adder = Merge()\n", - "\n", - " def forward(self, x):\n", - " \n", - " con1_out = self.conv1(x)\n", - " iaf1_out = self.iaf1(con1_out)\n", - " pool1_out = self.pool1(iaf1_out)\n", - " pool1a_out = self.pool1a(iaf1_out)\n", - "\n", - " conv2_out = self.conv2(pool1_out)\n", - " iaf2_out = self.iaf2(conv2_out)\n", - "\n", - " conv3_out = self.conv3(self.adder(pool1a_out, iaf2_out))\n", - " iaf3_out = self.iaf3(conv3_out)\n", - "\n", - " flat_out = self.flat(iaf3_out)\n", - " \n", - " fc1_out = self.fc1(flat_out)\n", - " iaf4_out = self.iaf4(fc1_out)\n", - " fc2_out = self.fc2(iaf4_out)\n", - " iaf5_out = self.iaf5(fc2_out)\n", - "\n", - " return iaf5_out" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "snn = SNN()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## DynapcnnNetwork Class\n", - "\n", - "In the constructor of `DynapcnnNetworkGraph` the SNN passed as argument (defined as a `nn.Module`) will be parsed such that each layer is represented in a computational graph (using `nirtorch.extract_torch_graph`). \n", - "\n", - "The layers are the `nodes` of the graph, while their connectivity (how the outputs from a layer are sent to other layers) is represented as `edges`, represented in a `list` of `tuples`.\n", - "\n", - "Once the constructor finishes its initialization, the `hw_model.dynapcnn_layers` property is a dictionary where each entry represents the ID of a `DynapcnnLayer` instance (an `int` from `0` to `L`), with this entry containing a `DynapcnnLayer` instance where a subset of the layers in the original SNN has been incorporated into, the core such instance has been assigned to, and the list of `DynapcnnLayer` instances (their IDs) the layer targets." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "hw_model = DynapcnnNetwork(\n", - " snn=snn,\n", - " input_shape=input_shape,\n", - " batch_size=batch_size,\n", - " discretize=False\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Notice in the model bellow how the property DynapcnnLayer in the model has yet to be assigned to a core. This is only done once\n", - "DynapcnnNetworkGraph.to() is called." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "----------------------- [ DynapcnnLayer 0 ] -----------------------\n", - "\n", - "COMPUTATIONAL NODES:\n", - "\n", - "(node 0): Conv2d(2, 10, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", - "(node 1): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-1.), batch_size=8, num_timesteps=-1)\n", - "(node 2): SumPool2d(norm_type=1, kernel_size=(3, 3), stride=None, ceil_mode=False)\n", - "(node 3): SumPool2d(norm_type=1, kernel_size=(4, 4), stride=None, ceil_mode=False)\n", - "\n", - "METADATA:\n", - "\n", - "> network's entry point: True\n", - "> convolution's weight re-scaling factor: None\n", - "> assigned core index: None\n", - "> destination DynapcnnLayers: [1, 2]\n", - "> node 2 feeds input to nodes [4]\n", - "> node 3 feeds input to nodes [7]\n", - "\n", - "----------------------- [ DynapcnnLayer 1 ] -----------------------\n", - "\n", - "COMPUTATIONAL NODES:\n", - "\n", - "(node 4): Conv2d(10, 10, kernel_size=(4, 4), stride=(1, 1), bias=False)\n", - "(node 6): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-1.), batch_size=8, num_timesteps=-1)\n", - "\n", - "METADATA:\n", - "\n", - "> network's entry point: False\n", - "> convolution's weight re-scaling factor: 4.5\n", - "> assigned core index: None\n", - "> destination DynapcnnLayers: [2]\n", - "> node 6 feeds input to nodes [7]\n", - "\n", - "----------------------- [ DynapcnnLayer 2 ] -----------------------\n", - "\n", - "COMPUTATIONAL NODES:\n", - "\n", - "(node 7): Conv2d(10, 1, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", - "(node 8): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-1.), batch_size=8, num_timesteps=-1)\n", - "\n", - "METADATA:\n", - "\n", - "> network's entry point: False\n", - "> convolution's weight re-scaling factor: 8.0\n", - "> assigned core index: None\n", - "> destination DynapcnnLayers: [3]\n", - "> node 8 feeds input to nodes [9]\n", - "\n", - "----------------------- [ DynapcnnLayer 3 ] -----------------------\n", - "\n", - "COMPUTATIONAL NODES:\n", - "\n", - "(node 9): Conv2d(1, 500, kernel_size=(7, 7), stride=(1, 1), bias=False)\n", - "(node 10): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-1.), batch_size=8, num_timesteps=-1)\n", - "\n", - "METADATA:\n", - "\n", - "> network's entry point: False\n", - "> convolution's weight re-scaling factor: None\n", - "> assigned core index: None\n", - "> destination DynapcnnLayers: [4]\n", - "> node 10 feeds input to nodes [11]\n", - "\n", - "----------------------- [ DynapcnnLayer 4 ] -----------------------\n", - "\n", - "COMPUTATIONAL NODES:\n", - "\n", - "(node 11): Conv2d(500, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - "(node 12): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-1.), batch_size=8, num_timesteps=-1)\n", - "\n", - "METADATA:\n", - "\n", - "> network's entry point: False\n", - "> convolution's weight re-scaling factor: None\n", - "> assigned core index: None\n", - "> destination DynapcnnLayers: []\n", - "\n", - "\n" - ] - } - ], - "source": [ - "print(hw_model)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The `hw_model.to()` call will figure out into which core each `DynapcnnLayer` instance will be assigned to. Once this assingment is made the instance itself is used to configure the `CNNLayerConfig` instance representing the core's configuration assigned to it.\n", - "\n", - "If the call is sucessfull, the layers comprising the network and their associated metadata will be printed." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": { - "metadata": {} - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Network is valid: \n", - "\n", - "----------------------- [ DynapcnnLayer 0 ] -----------------------\n", - "\n", - "COMPUTATIONAL NODES:\n", - "\n", - "(node 0): Conv2d(2, 10, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", - "(node 1): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-1.), batch_size=8, num_timesteps=-1)\n", - "(node 2): SumPool2d(norm_type=1, kernel_size=(3, 3), stride=None, ceil_mode=False)\n", - "(node 3): SumPool2d(norm_type=1, kernel_size=(4, 4), stride=None, ceil_mode=False)\n", - "\n", - "METADATA:\n", - "\n", - "> network's entry point: True\n", - "> convolution's weight re-scaling factor: None\n", - "> assigned core index: 0\n", - "> destination DynapcnnLayers: [1, 2]\n", - "> node 2 feeds input to nodes [4]\n", - "> node 3 feeds input to nodes [7]\n", - "\n", - "----------------------- [ DynapcnnLayer 1 ] -----------------------\n", - "\n", - "COMPUTATIONAL NODES:\n", - "\n", - "(node 4): Conv2d(10, 10, kernel_size=(4, 4), stride=(1, 1), bias=False)\n", - "(node 6): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-1.), batch_size=8, num_timesteps=-1)\n", - "\n", - "METADATA:\n", - "\n", - "> network's entry point: False\n", - "> convolution's weight re-scaling factor: 4.5\n", - "> assigned core index: 1\n", - "> destination DynapcnnLayers: [2]\n", - "> node 6 feeds input to nodes [7]\n", - "\n", - "----------------------- [ DynapcnnLayer 2 ] -----------------------\n", - "\n", - "COMPUTATIONAL NODES:\n", - "\n", - "(node 7): Conv2d(10, 1, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", - "(node 8): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-1.), batch_size=8, num_timesteps=-1)\n", - "\n", - "METADATA:\n", - "\n", - "> network's entry point: False\n", - "> convolution's weight re-scaling factor: 8.0\n", - "> assigned core index: 2\n", - "> destination DynapcnnLayers: [3]\n", - "> node 8 feeds input to nodes [9]\n", - "\n", - "----------------------- [ DynapcnnLayer 3 ] -----------------------\n", - "\n", - "COMPUTATIONAL NODES:\n", - "\n", - "(node 9): Conv2d(1, 500, kernel_size=(7, 7), stride=(1, 1), bias=False)\n", - "(node 10): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-1.), batch_size=8, num_timesteps=-1)\n", - "\n", - "METADATA:\n", - "\n", - "> network's entry point: False\n", - "> convolution's weight re-scaling factor: None\n", - "> assigned core index: 3\n", - "> destination DynapcnnLayers: [4]\n", - "> node 10 feeds input to nodes [11]\n", - "\n", - "----------------------- [ DynapcnnLayer 4 ] -----------------------\n", - "\n", - "COMPUTATIONAL NODES:\n", - "\n", - "(node 11): Conv2d(500, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - "(node 12): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-1.), batch_size=8, num_timesteps=-1)\n", - "\n", - "METADATA:\n", - "\n", - "> network's entry point: False\n", - "> convolution's weight re-scaling factor: None\n", - "> assigned core index: 4\n", - "> destination DynapcnnLayers: []\n", - "\n", - "\n" - ] - } - ], - "source": [ - "hw_model.to(device=\"speck2fmodule:0\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Notice above now how the layers of the model have been assigned to a chip core." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Training the HW model" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "hw_model.init_weights()" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "device: NVIDIA GeForce RTX 3070 Ti\n" - ] - } - ], - "source": [ - "if torch.cuda.is_available():\n", - " device = torch.device('cuda:0')\n", - " print('device: ', torch.cuda.get_device_name(0))\n", - "else:\n", - " device = torch.device('cpu')" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "hw_model.to(device)" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "import sys\n", - "\n", - "sys.path.append('../utils')\n", - "\n", - "from train_test_fn import training_loop, load_dataset, split_train_validation, load_architecture" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "batch_size = 8\n", - "num_workers = 4\n", - "epochs = 5\n", - "lr = 5e-4\n", - "\n", - "n_time_steps = 50" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [], - "source": [ - "from torch.utils.data import DataLoader\n", - "from torch.nn import CrossEntropyLoss\n", - "from torch.optim import Adam" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(34, 34, 2)\n" - ] - } - ], - "source": [ - "snn_train_dataset, snn_test_dataset, sensor_size, nb_classes = load_dataset('NMNIST', n_time_steps, \"../NMNIST\")\n", - "\n", - "print(sensor_size)\n", - "\n", - "snn_train_dataloader = DataLoader(snn_train_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True)\n", - "snn_test_dataloader = DataLoader(snn_test_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [], - "source": [ - "optimizer = Adam(hw_model.parameters(), lr=lr, betas=(0.9, 0.999), eps=1e-8)\n", - "loss_fn = CrossEntropyLoss()" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "70c30996c7164a019d9a9b26397a4a6c", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - " 0%| | 0/7500 [00:00 1\u001b[0m epochs_x, epochs_y, epochs_acc \u001b[38;5;241m=\u001b[39m training_loop(\n\u001b[1;32m 2\u001b[0m device, \n\u001b[1;32m 3\u001b[0m n_time_steps,\n\u001b[1;32m 4\u001b[0m batch_size,\n\u001b[1;32m 5\u001b[0m sensor_size,\n\u001b[1;32m 6\u001b[0m snn_train_dataloader, \n\u001b[1;32m 7\u001b[0m hw_model, \n\u001b[1;32m 8\u001b[0m loss_fn, \n\u001b[1;32m 9\u001b[0m optimizer, \n\u001b[1;32m 10\u001b[0m epochs, \n\u001b[1;32m 11\u001b[0m snn_test_dataloader)\n", - "File \u001b[0;32m~/Documents/github/sinabs/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/../utils/train_test_fn.py:132\u001b[0m, in \u001b[0;36mtraining_loop\u001b[0;34m(device, nb_time_steps, batch_size, feature_map_size, dataloader_train, model, loss_fn, optimizer, epochs, dataloader_test)\u001b[0m\n\u001b[1;32m 130\u001b[0m \u001b[38;5;66;03m# gradient update\u001b[39;00m\n\u001b[1;32m 131\u001b[0m optimizer\u001b[38;5;241m.\u001b[39mzero_grad()\n\u001b[0;32m--> 132\u001b[0m loss\u001b[38;5;241m.\u001b[39mbackward()\n\u001b[1;32m 133\u001b[0m optimizer\u001b[38;5;241m.\u001b[39mstep()\n\u001b[1;32m 135\u001b[0m \u001b[38;5;66;03m# detach the neuron states and activations from current computation graph(necessary)\u001b[39;00m\n", - "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/_tensor.py:522\u001b[0m, in \u001b[0;36mTensor.backward\u001b[0;34m(self, gradient, retain_graph, create_graph, inputs)\u001b[0m\n\u001b[1;32m 512\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m has_torch_function_unary(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m 513\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m handle_torch_function(\n\u001b[1;32m 514\u001b[0m Tensor\u001b[38;5;241m.\u001b[39mbackward,\n\u001b[1;32m 515\u001b[0m (\u001b[38;5;28mself\u001b[39m,),\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 520\u001b[0m inputs\u001b[38;5;241m=\u001b[39minputs,\n\u001b[1;32m 521\u001b[0m )\n\u001b[0;32m--> 522\u001b[0m torch\u001b[38;5;241m.\u001b[39mautograd\u001b[38;5;241m.\u001b[39mbackward(\n\u001b[1;32m 523\u001b[0m \u001b[38;5;28mself\u001b[39m, gradient, retain_graph, create_graph, inputs\u001b[38;5;241m=\u001b[39minputs\n\u001b[1;32m 524\u001b[0m )\n", - "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/autograd/__init__.py:266\u001b[0m, in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[0m\n\u001b[1;32m 261\u001b[0m retain_graph \u001b[38;5;241m=\u001b[39m create_graph\n\u001b[1;32m 263\u001b[0m \u001b[38;5;66;03m# The reason we repeat the same comment below is that\u001b[39;00m\n\u001b[1;32m 264\u001b[0m \u001b[38;5;66;03m# some Python versions print out the first line of a multi-line function\u001b[39;00m\n\u001b[1;32m 265\u001b[0m \u001b[38;5;66;03m# calls in the traceback and some print out the last line\u001b[39;00m\n\u001b[0;32m--> 266\u001b[0m Variable\u001b[38;5;241m.\u001b[39m_execution_engine\u001b[38;5;241m.\u001b[39mrun_backward( \u001b[38;5;66;03m# Calls into the C++ engine to run the backward pass\u001b[39;00m\n\u001b[1;32m 267\u001b[0m tensors,\n\u001b[1;32m 268\u001b[0m grad_tensors_,\n\u001b[1;32m 269\u001b[0m retain_graph,\n\u001b[1;32m 270\u001b[0m create_graph,\n\u001b[1;32m 271\u001b[0m inputs,\n\u001b[1;32m 272\u001b[0m allow_unreachable\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m 273\u001b[0m accumulate_grad\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m 274\u001b[0m )\n", - "\u001b[0;31mKeyboardInterrupt\u001b[0m: " - ] - } - ], - "source": [ - "epochs_x, epochs_y, epochs_acc = training_loop(\n", - " device, \n", - " n_time_steps,\n", - " batch_size,\n", - " sensor_size,\n", - " snn_train_dataloader, \n", - " hw_model, \n", - " loss_fn, \n", - " optimizer, \n", - " epochs, \n", - " snn_test_dataloader)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "speck-rescnn", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_2.ipynb b/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_2.ipynb deleted file mode 100644 index abdda89e..00000000 --- a/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_2.ipynb +++ /dev/null @@ -1,396 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "import torch\n", - "import torch.nn as nn\n", - "from sinabs.backend.dynapcnn import DynapcnnNetworkGraph\n", - "from sinabs.layers import Merge, IAFSqueeze" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "metadata": {} - }, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "torch.manual_seed(0)" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "channels = 2\n", - "height = 34\n", - "width = 34\n", - "\n", - "input_shape = (channels, height, width)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Network Module\n", - "\n", - "We need to define a `nn.Module` implementing the network we want the chip to reproduce." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "class SNN(nn.Module):\n", - " def __init__(self) -> None:\n", - " super().__init__()\n", - "\n", - " self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False)\n", - " self.conv1_iaf = IAFSqueeze(batch_size=1)\n", - " self.pool1 = nn.AvgPool2d(3,3)\n", - " self.pool1a = nn.AvgPool2d(4,4)\n", - "\n", - " self.conv2 = nn.Conv2d(10, 10, 4, 1, bias=False)\n", - " self.conv2_iaf = IAFSqueeze(batch_size=1)\n", - "\n", - " self.conv3 = nn.Conv2d(10, 1, 2, 1, bias=False)\n", - " self.conv3_iaf = IAFSqueeze(batch_size=1)\n", - "\n", - " self.flat = nn.Flatten()\n", - "\n", - " self.fc1 = nn.Linear(49, 100, bias=False)\n", - " self.fc1_iaf = IAFSqueeze(batch_size=1)\n", - " \n", - " self.fc2 = nn.Linear(100, 100, bias=False)\n", - " self.fc2_iaf = IAFSqueeze(batch_size=1)\n", - "\n", - " self.fc3 = nn.Linear(100, 10, bias=False)\n", - " self.fc3_iaf = IAFSqueeze(batch_size=1)\n", - "\n", - " self.merge1 = Merge()\n", - "\n", - " def forward(self, x):\n", - " # -- conv. block 1 --\n", - " con1_out = self.conv1(x)\n", - " conv1_iaf_out = self.conv1_iaf(con1_out)\n", - " pool1_out = self.pool1(conv1_iaf_out)\n", - " pool1a_out = self.pool1a(conv1_iaf_out)\n", - " # -- conv. block 2 --\n", - " conv2_out = self.conv2(pool1_out)\n", - " conv2_iaf_out = self.conv2_iaf(conv2_out)\n", - " # -- conv. block 3 --\n", - " merge1_out = self.merge1(pool1a_out, conv2_iaf_out)\n", - " conv3_out = self.conv3(merge1_out)\n", - " conv3_iaf_out = self.conv3_iaf(conv3_out)\n", - " flat_out = self.flat(conv3_iaf_out)\n", - " # -- fc clock 1 --\n", - " fc1_out = self.fc1(flat_out)\n", - " fc1_iaf_out = self.fc1_iaf(fc1_out)\n", - " # -- fc clock 2 --\n", - " fc2_out = self.fc2(fc1_iaf_out)\n", - " fc2_iaf_out = self.fc2_iaf(fc2_out)\n", - " # -- fc clock 3 --\n", - " fc3_out = self.fc3(fc2_iaf_out)\n", - " fc3_iaf_out = self.fc3_iaf(fc3_out)\n", - "\n", - " return fc3_iaf_out" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "snn = SNN()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The following exemplifies how each of the layers in the SNN should be grouped together to form a `DynapcnnLayer` instance. Let's check the shapes of the tensors each of the (original) layers in the model inputs and outputs by feeding a fake input through the model:" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "metadata": {} - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "torch.Size([1, 10, 33, 33])\n", - "torch.Size([1, 10, 33, 33])\n", - "torch.Size([1, 10, 11, 11])\n", - "torch.Size([1, 10, 8, 8])\n", - "torch.Size([1, 10, 8, 8])\n", - "torch.Size([1, 10, 8, 8])\n", - "torch.Size([1, 10, 8, 8])\n", - "torch.Size([1, 1, 7, 7])\n", - "torch.Size([1, 1, 7, 7])\n", - "torch.Size([1, 49])\n", - "torch.Size([1, 100])\n", - "torch.Size([1, 100])\n", - "torch.Size([1, 100])\n", - "torch.Size([1, 100])\n", - "torch.Size([1, 10])\n", - "torch.Size([1, 10])\n" - ] - } - ], - "source": [ - "x = torch.randn((1, *input_shape))\n", - "\n", - "# -- conv. block 1 --\n", - "con1_out = snn.conv1(x)\n", - "print(con1_out.shape)\n", - "conv1_iaf_out = snn.conv1_iaf(con1_out)\n", - "print(conv1_iaf_out.shape)\n", - "pool1_out = snn.pool1(conv1_iaf_out)\n", - "print(pool1_out.shape)\n", - "pool1a_out = snn.pool1a(conv1_iaf_out)\n", - "print(pool1a_out.shape)\n", - "# -- conv. block 2 --\n", - "conv2_out = snn.conv2(pool1_out)\n", - "print(conv2_out.shape)\n", - "conv2_iaf_out = snn.conv2_iaf(conv2_out)\n", - "print(conv2_iaf_out.shape)\n", - "# -- conv. block 3 --\n", - "merge1_out = snn.merge1(pool1a_out, conv2_iaf_out)\n", - "print(merge1_out.shape)\n", - "conv3_out = snn.conv3(merge1_out)\n", - "print(conv3_out.shape)\n", - "conv3_iaf_out = snn.conv3_iaf(conv3_out)\n", - "print(conv3_iaf_out.shape)\n", - "flat_out = snn.flat(conv3_iaf_out)\n", - "print(flat_out.shape)\n", - "# -- fc clock 1 --\n", - "fc1_out = snn.fc1(flat_out)\n", - "print(fc1_out.shape)\n", - "fc1_iaf_out = snn.fc1_iaf(fc1_out)\n", - "print(fc1_iaf_out.shape)\n", - "# -- fc clock 2 --\n", - "fc2_out = snn.fc2(fc1_iaf_out)\n", - "print(fc2_out.shape)\n", - "fc2_iaf_out = snn.fc2_iaf(fc2_out)\n", - "print(fc2_iaf_out.shape)\n", - "# -- fc clock 3 --\n", - "fc3_out = snn.fc3(fc2_iaf_out)\n", - "print(fc3_out.shape)\n", - "fc3_iaf_out = snn.fc3_iaf(fc3_out)\n", - "print(fc3_iaf_out.shape)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## DynapcnnNetwork Class\n", - "\n", - "In the constructor of `DynapcnnNetworkGraph` the SNN passed as argument (defined as a `nn.Module`) will be parsed such that each layer is represented in a computational graph (using `nirtorch.extract_torch_graph`). \n", - "\n", - "The layers are the `nodes` of the graph, while their connectivity (how the outputs from a layer are sent to other layers) is represented as `edges`, represented in a `list` of `tuples`.\n", - "\n", - "Once the constructor finishes its initialization, the `hw_model.dynapcnn_layers` property is a dictionary where each entry represents the ID of a `DynapcnnLayer` instance (an `int` from `0` to `L`), with this entry containing a `DynapcnnLayer` instance where a subset of the layers in the original SNN has been incorporated into, the core such instance has been assigned to, and the list of `DynapcnnLayer` instances (their IDs) the layer targets." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "hw_model = DynapcnnNetworkGraph(\n", - " snn,\n", - " discretize=True,\n", - " input_shape=input_shape\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The `hw_model.to()` call will figure out into which core eac `DynapcnnLayer` instance will be assigned to. Once this assingment is made the instance itself is used to configure the `CNNLayerConfig` instance representing the core's configuration.\n", - "\n", - "If the cores' configuration is valid, each `DynapcnnLayer` instance and their respective destinations will be used to create a computational graph that encodes how the `forward` method of `hw_model.network` (a `nn.Module` using the `DynapcnnLayer` instances) propagates that through the network." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": { - "metadata": {} - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Network is valid\n", - "(0, 1)\n", - "(0, 2)\n", - "(1, 2)\n", - "(2, 3)\n", - "(3, 4)\n", - "(4, 5)\n" - ] - }, - { - "ename": "RuntimeError", - "evalue": "Device is already opened!", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[8], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m hw_model\u001b[38;5;241m.\u001b[39mto(device\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mspeck2fmodule:0\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", - "File \u001b[0;32m~/Documents/github/sinabs/sinabs/backend/dynapcnn/dynapcnn_network_graph.py:172\u001b[0m, in \u001b[0;36mDynapcnnNetworkGraph.to\u001b[0;34m(self, device, chip_layers_ordering, monitor_layers, config_modifier, slow_clk_frequency)\u001b[0m\n\u001b[1;32m 163\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m device_name \u001b[38;5;129;01min\u001b[39;00m ChipFactory\u001b[38;5;241m.\u001b[39msupported_devices: \u001b[38;5;66;03m# pragma: no cover\u001b[39;00m\n\u001b[1;32m 165\u001b[0m config \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmake_config( \u001b[38;5;66;03m# generate config.\u001b[39;00m\n\u001b[1;32m 166\u001b[0m chip_layers_ordering\u001b[38;5;241m=\u001b[39mchip_layers_ordering,\n\u001b[1;32m 167\u001b[0m device\u001b[38;5;241m=\u001b[39mdevice,\n\u001b[1;32m 168\u001b[0m monitor_layers\u001b[38;5;241m=\u001b[39mmonitor_layers,\n\u001b[1;32m 169\u001b[0m config_modifier\u001b[38;5;241m=\u001b[39mconfig_modifier,\n\u001b[1;32m 170\u001b[0m )\n\u001b[0;32m--> 172\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msamna_device \u001b[38;5;241m=\u001b[39m open_device(device) \u001b[38;5;66;03m# apply configuration to device.\u001b[39;00m\n\u001b[1;32m 173\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msamna_device\u001b[38;5;241m.\u001b[39mget_model()\u001b[38;5;241m.\u001b[39mapply_configuration(config)\n\u001b[1;32m 174\u001b[0m time\u001b[38;5;241m.\u001b[39msleep(\u001b[38;5;241m1\u001b[39m)\n", - "File \u001b[0;32m~/Documents/github/sinabs/sinabs/backend/dynapcnn/io.py:256\u001b[0m, in \u001b[0;36mopen_device\u001b[0;34m(device_id)\u001b[0m\n\u001b[1;32m 254\u001b[0m device_map \u001b[38;5;241m=\u001b[39m get_device_map()\n\u001b[1;32m 255\u001b[0m device_info \u001b[38;5;241m=\u001b[39m device_map[device_id]\n\u001b[0;32m--> 256\u001b[0m device_handle \u001b[38;5;241m=\u001b[39m samna\u001b[38;5;241m.\u001b[39mdevice\u001b[38;5;241m.\u001b[39mopen_device(device_info)\n\u001b[1;32m 258\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m device_handle \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 259\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m device_handle\n", - "\u001b[0;31mRuntimeError\u001b[0m: Device is already opened!" - ] - } - ], - "source": [ - "hw_model.to(device=\"speck2fmodule:0\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The layers comprising our `hw_model` and their respective metadata can be inspected by calling `print` on a `DynapcnnNetworkGraph` instance." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "---- DynapcnnLayer 0 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 0): Conv2d(2, 10, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", - "(node 1): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "(node 2): SumPool2d(norm_type=1, kernel_size=(3, 3), stride=None, ceil_mode=False)\n", - "(node 3): SumPool2d(norm_type=1, kernel_size=(4, 4), stride=None, ceil_mode=False)\n", - "> layer destinations: [1, 2]\n", - "> assigned core: 0\n", - "\n", - "---- DynapcnnLayer 1 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 4): Conv2d(10, 10, kernel_size=(4, 4), stride=(1, 1), bias=False)\n", - "(node 6): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [2]\n", - "> assigned core: 1\n", - "\n", - "---- DynapcnnLayer 2 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 7): Conv2d(10, 1, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", - "(node 8): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [3]\n", - "> assigned core: 2\n", - "\n", - "---- DynapcnnLayer 3 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 9): Conv2d(1, 100, kernel_size=(7, 7), stride=(1, 1), bias=False)\n", - "(node 10): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [4]\n", - "> assigned core: 3\n", - "\n", - "---- DynapcnnLayer 4 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 11): Conv2d(100, 100, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - "(node 12): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [5]\n", - "> assigned core: 4\n", - "\n", - "---- DynapcnnLayer 5 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 13): Conv2d(100, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - "(node 14): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: []\n", - "> assigned core: 5\n", - "\n", - "\n" - ] - } - ], - "source": [ - "print(hw_model)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "speck-rescnn", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_3.ipynb b/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_3.ipynb deleted file mode 100644 index 5b8e7216..00000000 --- a/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_3.ipynb +++ /dev/null @@ -1,402 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "import torch\n", - "import torch.nn as nn\n", - "from sinabs.backend.dynapcnn import DynapcnnNetworkGraph\n", - "from sinabs.layers import Merge, IAFSqueeze" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "metadata": {} - }, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "torch.manual_seed(0)" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "channels = 2\n", - "height = 34\n", - "width = 34\n", - "\n", - "input_shape = (channels, height, width)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Network Module\n", - "\n", - "We need to define a `nn.Module` implementing the network we want the chip to reproduce." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "class SNN(nn.Module):\n", - " def __init__(self) -> None:\n", - " super().__init__()\n", - "\n", - " self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False)\n", - " self.conv1_iaf = IAFSqueeze(batch_size=1)\n", - " self.pool1 = nn.AvgPool2d(3,3)\n", - " self.pool1a = nn.AvgPool2d(4,4)\n", - "\n", - " self.conv2 = nn.Conv2d(10, 10, 4, 1, bias=False)\n", - " self.conv2_iaf = IAFSqueeze(batch_size=1)\n", - "\n", - " self.conv3 = nn.Conv2d(10, 1, 2, 1, bias=False)\n", - " self.conv3_iaf = IAFSqueeze(batch_size=1)\n", - "\n", - " self.flat = nn.Flatten()\n", - "\n", - " self.fc1 = nn.Linear(49, 100, bias=False)\n", - " self.fc1_iaf = IAFSqueeze(batch_size=1)\n", - " \n", - " self.fc2 = nn.Linear(100, 100, bias=False)\n", - " self.fc2_iaf = IAFSqueeze(batch_size=1)\n", - "\n", - " self.fc3 = nn.Linear(100, 10, bias=False)\n", - " self.fc3_iaf = IAFSqueeze(batch_size=1)\n", - "\n", - " self.merge1 = Merge()\n", - " self.merge2 = Merge()\n", - "\n", - " def forward(self, x):\n", - " # -- conv. block 0 --\n", - " con1_out = self.conv1(x)\n", - " conv1_iaf_out = self.conv1_iaf(con1_out)\n", - " pool1_out = self.pool1(conv1_iaf_out)\n", - " pool1a_out = self.pool1a(conv1_iaf_out)\n", - " # -- conv. block 1 --\n", - " conv2_out = self.conv2(pool1_out)\n", - " conv2_iaf_out = self.conv2_iaf(conv2_out)\n", - " # -- conv. block 2 --\n", - " merge1_out = self.merge1(pool1a_out, conv2_iaf_out)\n", - " conv3_out = self.conv3(merge1_out)\n", - " conv3_iaf_out = self.conv3_iaf(conv3_out)\n", - " flat_out = self.flat(conv3_iaf_out)\n", - " # -- fc clock 3 --\n", - " fc1_out = self.fc1(flat_out)\n", - " fc1_iaf_out = self.fc1_iaf(fc1_out)\n", - " # -- fc clock 4 --\n", - " fc2_out = self.fc2(fc1_iaf_out)\n", - " fc2_iaf_out = self.fc2_iaf(fc2_out)\n", - " # -- fc clock 5 --\n", - " merge2_out = self.merge2(fc1_iaf_out, fc2_iaf_out)\n", - " fc3_out = self.fc3(merge2_out)\n", - " fc3_iaf_out = self.fc3_iaf(fc3_out)\n", - "\n", - " return fc3_iaf_out" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "snn = SNN()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The following exemplifies how each of the layers in the SNN should be grouped together to form a `DynapcnnLayer` instance. Let's check the shapes of the tensors each of the (original) layers in the model inputs and outputs by feeding a fake input through the model:" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "metadata": {} - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "torch.Size([1, 10, 33, 33])\n", - "torch.Size([1, 10, 33, 33])\n", - "torch.Size([1, 10, 11, 11])\n", - "torch.Size([1, 10, 8, 8])\n", - "torch.Size([1, 10, 8, 8])\n", - "torch.Size([1, 10, 8, 8])\n", - "merge1: torch.Size([1, 10, 8, 8])\n", - "torch.Size([1, 1, 7, 7])\n", - "torch.Size([1, 1, 7, 7])\n", - "torch.Size([1, 49])\n", - "torch.Size([1, 100])\n", - "torch.Size([1, 100])\n", - "torch.Size([1, 100])\n", - "torch.Size([1, 100])\n", - "merge2: torch.Size([1, 100])\n", - "torch.Size([1, 10])\n", - "torch.Size([1, 10])\n" - ] - } - ], - "source": [ - "x = torch.randn((1, *input_shape))\n", - "\n", - "# -- conv. block 0 --\n", - "con1_out = snn.conv1(x)\n", - "print(con1_out.shape)\n", - "conv1_iaf_out = snn.conv1_iaf(con1_out)\n", - "print(conv1_iaf_out.shape)\n", - "pool1_out = snn.pool1(conv1_iaf_out)\n", - "print(pool1_out.shape)\n", - "pool1a_out = snn.pool1a(conv1_iaf_out)\n", - "print(pool1a_out.shape)\n", - "# -- conv. block 1 --\n", - "conv2_out = snn.conv2(pool1_out)\n", - "print(conv2_out.shape)\n", - "conv2_iaf_out = snn.conv2_iaf(conv2_out)\n", - "print(conv2_iaf_out.shape)\n", - "# -- conv. block 2 --\n", - "merge1_out = snn.merge1(pool1a_out, conv2_iaf_out)\n", - "print(f'merge1: {merge1_out.shape}')\n", - "conv3_out = snn.conv3(merge1_out)\n", - "print(conv3_out.shape)\n", - "conv3_iaf_out = snn.conv3_iaf(conv3_out)\n", - "print(conv3_iaf_out.shape)\n", - "flat_out = snn.flat(conv3_iaf_out)\n", - "print(flat_out.shape)\n", - "# -- fc clock 3 --\n", - "fc1_out = snn.fc1(flat_out)\n", - "print(fc1_out.shape)\n", - "fc1_iaf_out = snn.fc1_iaf(fc1_out)\n", - "print(fc1_iaf_out.shape)\n", - "# -- fc clock 4 --\n", - "fc2_out = snn.fc2(fc1_iaf_out)\n", - "print(fc2_out.shape)\n", - "fc2_iaf_out = snn.fc2_iaf(fc2_out)\n", - "print(fc2_iaf_out.shape)\n", - "# -- fc clock 5 --\n", - "merge2_out = snn.merge2(fc1_iaf_out, fc2_iaf_out)\n", - "print(f'merge2: {merge2_out.shape}')\n", - "fc3_out = snn.fc3(merge2_out)\n", - "print(fc3_out.shape)\n", - "fc3_iaf_out = snn.fc3_iaf(fc3_out)\n", - "print(fc3_iaf_out.shape)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## DynapcnnNetwork Class\n", - "\n", - "In the constructor of `DynapcnnNetworkGraph` the SNN passed as argument (defined as a `nn.Module`) will be parsed such that each layer is represented in a computational graph (using `nirtorch.extract_torch_graph`). \n", - "\n", - "The layers are the `nodes` of the graph, while their connectivity (how the outputs from a layer are sent to other layers) is represented as `edges`, represented in a `list` of `tuples`.\n", - "\n", - "Once the constructor finishes its initialization, the `hw_model.dynapcnn_layers` property is a dictionary where each entry represents the ID of a `DynapcnnLayer` instance (an `int` from `0` to `L`), with this entry containing a `DynapcnnLayer` instance where a subset of the layers in the original SNN has been incorporated into, the core such instance has been assigned to, and the list of `DynapcnnLayer` instances (their IDs) the layer targets." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "hw_model = DynapcnnNetworkGraph(\n", - " snn,\n", - " discretize=True,\n", - " input_shape=input_shape\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The `hw_model.to()` call will figure out into which core eac `DynapcnnLayer` instance will be assigned to. Once this assingment is made the instance itself is used to configure the `CNNLayerConfig` instance representing the core's configuration.\n", - "\n", - "If the cores' configuration is valid, each `DynapcnnLayer` instance and their respective destinations will be used to create a computational graph that encodes how the `forward` method of `hw_model.network` (a `nn.Module` using the `DynapcnnLayer` instances) propagates that through the network." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": { - "metadata": {} - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Network is valid\n", - "(0, 1)\n", - "(0, 2)\n", - "(1, 2)\n", - "(2, 3)\n", - "(3, 4)\n", - "(3, 5)\n", - "(4, 5)\n" - ] - }, - { - "ename": "RuntimeError", - "evalue": "Device is already opened!", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[8], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m hw_model\u001b[38;5;241m.\u001b[39mto(device\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mspeck2fmodule:0\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", - "File \u001b[0;32m~/Documents/github/sinabs/sinabs/backend/dynapcnn/dynapcnn_network_graph.py:172\u001b[0m, in \u001b[0;36mDynapcnnNetworkGraph.to\u001b[0;34m(self, device, chip_layers_ordering, monitor_layers, config_modifier, slow_clk_frequency)\u001b[0m\n\u001b[1;32m 163\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m device_name \u001b[38;5;129;01min\u001b[39;00m ChipFactory\u001b[38;5;241m.\u001b[39msupported_devices: \u001b[38;5;66;03m# pragma: no cover\u001b[39;00m\n\u001b[1;32m 165\u001b[0m config \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmake_config( \u001b[38;5;66;03m# generate config.\u001b[39;00m\n\u001b[1;32m 166\u001b[0m chip_layers_ordering\u001b[38;5;241m=\u001b[39mchip_layers_ordering,\n\u001b[1;32m 167\u001b[0m device\u001b[38;5;241m=\u001b[39mdevice,\n\u001b[1;32m 168\u001b[0m monitor_layers\u001b[38;5;241m=\u001b[39mmonitor_layers,\n\u001b[1;32m 169\u001b[0m config_modifier\u001b[38;5;241m=\u001b[39mconfig_modifier,\n\u001b[1;32m 170\u001b[0m )\n\u001b[0;32m--> 172\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msamna_device \u001b[38;5;241m=\u001b[39m open_device(device) \u001b[38;5;66;03m# apply configuration to device.\u001b[39;00m\n\u001b[1;32m 173\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msamna_device\u001b[38;5;241m.\u001b[39mget_model()\u001b[38;5;241m.\u001b[39mapply_configuration(config)\n\u001b[1;32m 174\u001b[0m time\u001b[38;5;241m.\u001b[39msleep(\u001b[38;5;241m1\u001b[39m)\n", - "File \u001b[0;32m~/Documents/github/sinabs/sinabs/backend/dynapcnn/io.py:256\u001b[0m, in \u001b[0;36mopen_device\u001b[0;34m(device_id)\u001b[0m\n\u001b[1;32m 254\u001b[0m device_map \u001b[38;5;241m=\u001b[39m get_device_map()\n\u001b[1;32m 255\u001b[0m device_info \u001b[38;5;241m=\u001b[39m device_map[device_id]\n\u001b[0;32m--> 256\u001b[0m device_handle \u001b[38;5;241m=\u001b[39m samna\u001b[38;5;241m.\u001b[39mdevice\u001b[38;5;241m.\u001b[39mopen_device(device_info)\n\u001b[1;32m 258\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m device_handle \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 259\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m device_handle\n", - "\u001b[0;31mRuntimeError\u001b[0m: Device is already opened!" - ] - } - ], - "source": [ - "hw_model.to(device=\"speck2fmodule:0\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The layers comprising our `hw_model` and their respective metadata can be inspected by calling `print` on a `DynapcnnNetworkGraph` instance." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "---- DynapcnnLayer 0 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 0): Conv2d(2, 10, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", - "(node 1): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "(node 2): SumPool2d(norm_type=1, kernel_size=(3, 3), stride=None, ceil_mode=False)\n", - "(node 3): SumPool2d(norm_type=1, kernel_size=(4, 4), stride=None, ceil_mode=False)\n", - "> layer destinations: [1, 2]\n", - "> assigned core: 0\n", - "\n", - "---- DynapcnnLayer 1 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 4): Conv2d(10, 10, kernel_size=(4, 4), stride=(1, 1), bias=False)\n", - "(node 6): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [2]\n", - "> assigned core: 1\n", - "\n", - "---- DynapcnnLayer 2 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 7): Conv2d(10, 1, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", - "(node 8): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [3]\n", - "> assigned core: 2\n", - "\n", - "---- DynapcnnLayer 3 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 9): Conv2d(1, 100, kernel_size=(7, 7), stride=(1, 1), bias=False)\n", - "(node 10): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [4, 5]\n", - "> assigned core: 3\n", - "\n", - "---- DynapcnnLayer 4 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 11): Conv2d(100, 100, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - "(node 13): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [5]\n", - "> assigned core: 4\n", - "\n", - "---- DynapcnnLayer 5 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 14): Conv2d(100, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - "(node 15): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: []\n", - "> assigned core: 5\n", - "\n", - "\n" - ] - } - ], - "source": [ - "print(hw_model)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "speck-rescnn", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_4.ipynb b/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_4.ipynb deleted file mode 100644 index 1f190961..00000000 --- a/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_4.ipynb +++ /dev/null @@ -1,380 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "import torch\n", - "import torch.nn as nn\n", - "from sinabs.backend.dynapcnn import DynapcnnNetworkGraph\n", - "from sinabs.layers import Merge, IAFSqueeze" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "metadata": {} - }, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "torch.manual_seed(0)" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "channels = 2\n", - "height = 34\n", - "width = 34\n", - "\n", - "input_shape = (channels, height, width)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Network Module\n", - "\n", - "We need to define a `nn.Module` implementing the network we want the chip to reproduce." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "class SNN(nn.Module):\n", - " def __init__(self) -> None:\n", - " super().__init__()\n", - "\n", - " self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False)\n", - " self.conv1_iaf = IAFSqueeze(batch_size=1)\n", - " self.pool1 = nn.AvgPool2d(3,3)\n", - " self.pool1a = nn.AvgPool2d(4,4)\n", - "\n", - " self.conv2 = nn.Conv2d(10, 10, 4, 1, bias=False)\n", - " self.conv2_iaf = IAFSqueeze(batch_size=1)\n", - "\n", - " self.conv3 = nn.Conv2d(10, 10, 2, 1, bias=False)\n", - " self.conv3_iaf = IAFSqueeze(batch_size=1)\n", - "\n", - " self.conv4 = nn.Conv2d(10, 1, 2, 1, bias=False)\n", - " self.conv4_iaf = IAFSqueeze(batch_size=1)\n", - "\n", - " self.flat = nn.Flatten()\n", - "\n", - " self.fc1 = nn.Linear(36, 100, bias=False)\n", - " self.fc1_iaf = IAFSqueeze(batch_size=1)\n", - " \n", - " self.fc2 = nn.Linear(100, 100, bias=False)\n", - " self.fc2_iaf = IAFSqueeze(batch_size=1)\n", - "\n", - " self.fc3 = nn.Linear(100, 10, bias=False)\n", - " self.fc3_iaf = IAFSqueeze(batch_size=1)\n", - "\n", - " self.merge1 = Merge()\n", - " self.merge2 = Merge()\n", - "\n", - " def forward(self, x):\n", - " # -- conv. block 0 --\n", - " con1_out = self.conv1(x)\n", - " conv1_iaf_out = self.conv1_iaf(con1_out)\n", - " pool1_out = self.pool1(conv1_iaf_out)\n", - " pool1a_out = self.pool1a(conv1_iaf_out)\n", - " # -- conv. block 1 --\n", - " conv2_out = self.conv2(pool1_out)\n", - " conv2_iaf_out = self.conv2_iaf(conv2_out)\n", - " # -- conv. block 2 --\n", - " merge1_out = self.merge1(pool1a_out, conv2_iaf_out)\n", - " conv3_out = self.conv3(merge1_out)\n", - " conv3_iaf_out = self.conv3_iaf(conv3_out)\n", - " # -- conv. block 3 --\n", - " conv4_out = self.conv4(conv3_iaf_out)\n", - " conv4_iaf_out = self.conv4_iaf(conv4_out)\n", - " flat_out = self.flat(conv4_iaf_out)\n", - " # -- fc clock 4 --\n", - " fc1_out = self.fc1(flat_out)\n", - " fc1_iaf_out = self.fc1_iaf(fc1_out)\n", - " # -- fc clock 5 --\n", - " fc2_out = self.fc2(fc1_iaf_out)\n", - " fc2_iaf_out = self.fc2_iaf(fc2_out)\n", - " # -- fc clock 6 --\n", - " merge2_out = self.merge2(fc1_iaf_out, fc2_iaf_out)\n", - " fc3_out = self.fc3(merge2_out)\n", - " fc3_iaf_out = self.fc3_iaf(fc3_out)\n", - "\n", - " return fc3_iaf_out" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "snn = SNN()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The following exemplifies how each of the layers in the SNN should be grouped together to form a `DynapcnnLayer` instance. Let's check the shapes of the tensors each of the (original) layers in the model inputs and outputs by feeding a fake input through the model:" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "x = torch.randn((1, *input_shape))\n", - "\n", - "# -- conv. block 0 --\n", - "con1_out = snn.conv1(x)\n", - "conv1_iaf_out = snn.conv1_iaf(con1_out)\n", - "pool1_out = snn.pool1(conv1_iaf_out)\n", - "pool1a_out = snn.pool1a(conv1_iaf_out)\n", - "# -- conv. block 1 --\n", - "conv2_out = snn.conv2(pool1_out)\n", - "conv2_iaf_out = snn.conv2_iaf(conv2_out)\n", - "# -- conv. block 2 --\n", - "merge1_out = snn.merge1(pool1a_out, conv2_iaf_out)\n", - "conv3_out = snn.conv3(merge1_out)\n", - "conv3_iaf_out = snn.conv3_iaf(conv3_out)\n", - "# -- conv. block 3 --\n", - "conv4_out = snn.conv4(conv3_iaf_out)\n", - "conv4_iaf_out = snn.conv4_iaf(conv4_out)\n", - "flat_out = snn.flat(conv4_iaf_out)\n", - "# -- fc clock 4 --\n", - "fc1_out = snn.fc1(flat_out)\n", - "fc1_iaf_out = snn.fc1_iaf(fc1_out)\n", - "# -- fc clock 5 --\n", - "fc2_out = snn.fc2(fc1_iaf_out)\n", - "fc2_iaf_out = snn.fc2_iaf(fc2_out)\n", - "# -- fc clock 6 --\n", - "merge2_out = snn.merge2(fc1_iaf_out, fc2_iaf_out)\n", - "fc3_out = snn.fc3(merge2_out)\n", - "fc3_iaf_out = snn.fc3_iaf(fc3_out)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## DynapcnnNetwork Class\n", - "\n", - "In the constructor of `DynapcnnNetworkGraph` the SNN passed as argument (defined as a `nn.Module`) will be parsed such that each layer is represented in a computational graph (using `nirtorch.extract_torch_graph`). \n", - "\n", - "The layers are the `nodes` of the graph, while their connectivity (how the outputs from a layer are sent to other layers) is represented as `edges`, represented in a `list` of `tuples`.\n", - "\n", - "Once the constructor finishes its initialization, the `hw_model.dynapcnn_layers` property is a dictionary where each entry represents the ID of a `DynapcnnLayer` instance (an `int` from `0` to `L`), with this entry containing a `DynapcnnLayer` instance where a subset of the layers in the original SNN has been incorporated into, the core such instance has been assigned to, and the list of `DynapcnnLayer` instances (their IDs) the layer targets." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "hw_model = DynapcnnNetworkGraph(\n", - " snn,\n", - " discretize=True,\n", - " input_shape=input_shape\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The `hw_model.to()` call will figure out into which core eac `DynapcnnLayer` instance will be assigned to. Once this assingment is made the instance itself is used to configure the `CNNLayerConfig` instance representing the core's configuration.\n", - "\n", - "If the cores' configuration is valid, each `DynapcnnLayer` instance and their respective destinations will be used to create a computational graph that encodes how the `forward` method of `hw_model.network` (a `nn.Module` using the `DynapcnnLayer` instances) propagates that through the network." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": { - "metadata": {} - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Network is valid\n", - "(0, 1)\n", - "(0, 2)\n", - "(1, 2)\n", - "(2, 3)\n", - "(3, 4)\n", - "(4, 5)\n", - "(4, 6)\n", - "(5, 6)\n" - ] - }, - { - "ename": "RuntimeError", - "evalue": "Device is already opened!", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[8], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m hw_model\u001b[38;5;241m.\u001b[39mto(device\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mspeck2fmodule:0\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", - "File \u001b[0;32m~/Documents/github/sinabs/sinabs/backend/dynapcnn/dynapcnn_network_graph.py:172\u001b[0m, in \u001b[0;36mDynapcnnNetworkGraph.to\u001b[0;34m(self, device, chip_layers_ordering, monitor_layers, config_modifier, slow_clk_frequency)\u001b[0m\n\u001b[1;32m 163\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m device_name \u001b[38;5;129;01min\u001b[39;00m ChipFactory\u001b[38;5;241m.\u001b[39msupported_devices: \u001b[38;5;66;03m# pragma: no cover\u001b[39;00m\n\u001b[1;32m 165\u001b[0m config \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmake_config( \u001b[38;5;66;03m# generate config.\u001b[39;00m\n\u001b[1;32m 166\u001b[0m chip_layers_ordering\u001b[38;5;241m=\u001b[39mchip_layers_ordering,\n\u001b[1;32m 167\u001b[0m device\u001b[38;5;241m=\u001b[39mdevice,\n\u001b[1;32m 168\u001b[0m monitor_layers\u001b[38;5;241m=\u001b[39mmonitor_layers,\n\u001b[1;32m 169\u001b[0m config_modifier\u001b[38;5;241m=\u001b[39mconfig_modifier,\n\u001b[1;32m 170\u001b[0m )\n\u001b[0;32m--> 172\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msamna_device \u001b[38;5;241m=\u001b[39m open_device(device) \u001b[38;5;66;03m# apply configuration to device.\u001b[39;00m\n\u001b[1;32m 173\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msamna_device\u001b[38;5;241m.\u001b[39mget_model()\u001b[38;5;241m.\u001b[39mapply_configuration(config)\n\u001b[1;32m 174\u001b[0m time\u001b[38;5;241m.\u001b[39msleep(\u001b[38;5;241m1\u001b[39m)\n", - "File \u001b[0;32m~/Documents/github/sinabs/sinabs/backend/dynapcnn/io.py:256\u001b[0m, in \u001b[0;36mopen_device\u001b[0;34m(device_id)\u001b[0m\n\u001b[1;32m 254\u001b[0m device_map \u001b[38;5;241m=\u001b[39m get_device_map()\n\u001b[1;32m 255\u001b[0m device_info \u001b[38;5;241m=\u001b[39m device_map[device_id]\n\u001b[0;32m--> 256\u001b[0m device_handle \u001b[38;5;241m=\u001b[39m samna\u001b[38;5;241m.\u001b[39mdevice\u001b[38;5;241m.\u001b[39mopen_device(device_info)\n\u001b[1;32m 258\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m device_handle \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 259\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m device_handle\n", - "\u001b[0;31mRuntimeError\u001b[0m: Device is already opened!" - ] - } - ], - "source": [ - "hw_model.to(device=\"speck2fmodule:0\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The layers comprising our `hw_model` and their respective metadata can be inspected by calling `print` on a `DynapcnnNetworkGraph` instance." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "---- DynapcnnLayer 0 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 0): Conv2d(2, 10, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", - "(node 1): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "(node 2): SumPool2d(norm_type=1, kernel_size=(3, 3), stride=None, ceil_mode=False)\n", - "(node 3): SumPool2d(norm_type=1, kernel_size=(4, 4), stride=None, ceil_mode=False)\n", - "> layer destinations: [1, 2]\n", - "> assigned core: 0\n", - "\n", - "---- DynapcnnLayer 1 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 4): Conv2d(10, 10, kernel_size=(4, 4), stride=(1, 1), bias=False)\n", - "(node 6): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [2]\n", - "> assigned core: 1\n", - "\n", - "---- DynapcnnLayer 2 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 7): Conv2d(10, 10, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", - "(node 8): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [3]\n", - "> assigned core: 2\n", - "\n", - "---- DynapcnnLayer 3 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 9): Conv2d(10, 1, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", - "(node 10): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [4]\n", - "> assigned core: 3\n", - "\n", - "---- DynapcnnLayer 4 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 11): Conv2d(1, 100, kernel_size=(6, 6), stride=(1, 1), bias=False)\n", - "(node 12): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [5, 6]\n", - "> assigned core: 4\n", - "\n", - "---- DynapcnnLayer 5 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 13): Conv2d(100, 100, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - "(node 15): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [6]\n", - "> assigned core: 5\n", - "\n", - "---- DynapcnnLayer 6 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 16): Conv2d(100, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - "(node 17): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: []\n", - "> assigned core: 6\n", - "\n", - "\n" - ] - } - ], - "source": [ - "print(hw_model)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "speck-rescnn", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_5.ipynb b/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_5.ipynb deleted file mode 100644 index 4b3fcddb..00000000 --- a/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_5.ipynb +++ /dev/null @@ -1,400 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "import torch\n", - "import torch.nn as nn\n", - "from sinabs.backend.dynapcnn import DynapcnnNetworkGraph\n", - "from sinabs.layers import Merge, IAFSqueeze" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "metadata": {} - }, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "torch.manual_seed(0)" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "channels = 2\n", - "height = 34\n", - "width = 34\n", - "\n", - "input_shape = (channels, height, width)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Network Module\n", - "\n", - "We need to define a `nn.Module` implementing the network we want the chip to reproduce." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "class SNN(nn.Module):\n", - " def __init__(self) -> None:\n", - " super().__init__()\n", - "\n", - " self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False)\n", - " self.conv1_iaf = IAFSqueeze(batch_size=1)\n", - " self.pool1 = nn.AvgPool2d(3,3)\n", - " self.pool1a = nn.AvgPool2d(4,4)\n", - "\n", - " self.conv2 = nn.Conv2d(10, 10, 4, 1, bias=False)\n", - " self.conv2_iaf = IAFSqueeze(batch_size=1)\n", - "\n", - " self.conv3 = nn.Conv2d(10, 10, 2, 1, bias=False)\n", - " self.conv3_iaf = IAFSqueeze(batch_size=1)\n", - "\n", - " self.conv4 = nn.Conv2d(10, 1, 2, 1, bias=False)\n", - " self.conv4_iaf = IAFSqueeze(batch_size=1)\n", - "\n", - " self.flat = nn.Flatten()\n", - "\n", - " self.fc1 = nn.Linear(36, 100, bias=False)\n", - " self.fc1_iaf = IAFSqueeze(batch_size=1)\n", - " \n", - " self.fc2 = nn.Linear(100, 100, bias=False)\n", - " self.fc2_iaf = IAFSqueeze(batch_size=1)\n", - "\n", - " self.fc3 = nn.Linear(100, 100, bias=False)\n", - " self.fc3_iaf = IAFSqueeze(batch_size=1)\n", - "\n", - " self.fc4 = nn.Linear(100, 10, bias=False)\n", - " self.fc4_iaf = IAFSqueeze(batch_size=1)\n", - "\n", - " self.merge1 = Merge()\n", - " self.merge2 = Merge()\n", - " self.merge3 = Merge()\n", - "\n", - " def forward(self, x):\n", - " # -- conv. block 0 --\n", - " con1_out = self.conv1(x)\n", - " conv1_iaf_out = self.conv1_iaf(con1_out)\n", - " pool1_out = self.pool1(conv1_iaf_out)\n", - " pool1a_out = self.pool1a(conv1_iaf_out)\n", - " # -- conv. block 1 --\n", - " conv2_out = self.conv2(pool1_out)\n", - " conv2_iaf_out = self.conv2_iaf(conv2_out)\n", - " # -- conv. block 2 --\n", - " merge1_out = self.merge1(pool1a_out, conv2_iaf_out)\n", - " conv3_out = self.conv3(merge1_out)\n", - " conv3_iaf_out = self.conv3_iaf(conv3_out)\n", - " # -- conv. block 3 --\n", - " conv4_out = self.conv4(conv3_iaf_out)\n", - " conv4_iaf_out = self.conv4_iaf(conv4_out)\n", - " flat_out = self.flat(conv4_iaf_out)\n", - " # -- fc clock 4 --\n", - " fc1_out = self.fc1(flat_out)\n", - " fc1_iaf_out = self.fc1_iaf(fc1_out)\n", - " # -- fc clock 5 --\n", - " fc2_out = self.fc2(fc1_iaf_out)\n", - " fc2_iaf_out = self.fc2_iaf(fc2_out)\n", - " # -- fc clock 6 --\n", - " merge2_out = self.merge2(fc1_iaf_out, fc2_iaf_out)\n", - " fc3_out = self.fc3(merge2_out)\n", - " fc3_iaf_out = self.fc3_iaf(fc3_out)\n", - " # -- fc clock 7 --\n", - " merge3_out = self.merge3(fc2_iaf_out, fc3_iaf_out)\n", - " fc4_out = self.fc4(merge3_out)\n", - " fc4_iaf_out = self.fc4_iaf(fc4_out)\n", - "\n", - " return fc4_iaf_out" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "snn = SNN()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The following exemplifies how each of the layers in the SNN should be grouped together to form a `DynapcnnLayer` instance. Let's check the shapes of the tensors each of the (original) layers in the model inputs and outputs by feeding a fake input through the model:" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "x = torch.randn((1, *input_shape))\n", - "\n", - "# -- conv. block 0 --\n", - "con1_out = snn.conv1(x)\n", - "conv1_iaf_out = snn.conv1_iaf(con1_out)\n", - "pool1_out = snn.pool1(conv1_iaf_out)\n", - "pool1a_out = snn.pool1a(conv1_iaf_out)\n", - "# -- conv. block 1 --\n", - "conv2_out = snn.conv2(pool1_out)\n", - "conv2_iaf_out = snn.conv2_iaf(conv2_out)\n", - "# -- conv. block 2 --\n", - "merge1_out = snn.merge1(pool1a_out, conv2_iaf_out)\n", - "conv3_out = snn.conv3(merge1_out)\n", - "conv3_iaf_out = snn.conv3_iaf(conv3_out)\n", - "# -- conv. block 3 --\n", - "conv4_out = snn.conv4(conv3_iaf_out)\n", - "conv4_iaf_out = snn.conv4_iaf(conv4_out)\n", - "flat_out = snn.flat(conv4_iaf_out)\n", - "# -- fc clock 4 --\n", - "fc1_out = snn.fc1(flat_out)\n", - "fc1_iaf_out = snn.fc1_iaf(fc1_out)\n", - "# -- fc clock 5 --\n", - "fc2_out = snn.fc2(fc1_iaf_out)\n", - "fc2_iaf_out = snn.fc2_iaf(fc2_out)\n", - "# -- fc clock 6 --\n", - "merge2_out = snn.merge2(fc1_iaf_out, fc2_iaf_out)\n", - "fc3_out = snn.fc3(merge2_out)\n", - "fc3_iaf_out = snn.fc3_iaf(fc3_out)\n", - "# -- fc clock 7 --\n", - "merge3_out = snn.merge3(fc2_iaf_out, fc3_iaf_out)\n", - "fc4_out = snn.fc4(merge3_out)\n", - "fc4_iaf_out = snn.fc4_iaf(fc4_out)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## DynapcnnNetwork Class\n", - "\n", - "In the constructor of `DynapcnnNetworkGraph` the SNN passed as argument (defined as a `nn.Module`) will be parsed such that each layer is represented in a computational graph (using `nirtorch.extract_torch_graph`). \n", - "\n", - "The layers are the `nodes` of the graph, while their connectivity (how the outputs from a layer are sent to other layers) is represented as `edges`, represented in a `list` of `tuples`.\n", - "\n", - "Once the constructor finishes its initialization, the `hw_model.dynapcnn_layers` property is a dictionary where each entry represents the ID of a `DynapcnnLayer` instance (an `int` from `0` to `L`), with this entry containing a `DynapcnnLayer` instance where a subset of the layers in the original SNN has been incorporated into, the core such instance has been assigned to, and the list of `DynapcnnLayer` instances (their IDs) the layer targets." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "hw_model = DynapcnnNetworkGraph(\n", - " snn,\n", - " discretize=True,\n", - " input_shape=input_shape\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The `hw_model.to()` call will figure out into which core eac `DynapcnnLayer` instance will be assigned to. Once this assingment is made the instance itself is used to configure the `CNNLayerConfig` instance representing the core's configuration.\n", - "\n", - "If the cores' configuration is valid, each `DynapcnnLayer` instance and their respective destinations will be used to create a computational graph that encodes how the `forward` method of `hw_model.network` (a `nn.Module` using the `DynapcnnLayer` instances) propagates that through the network." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": { - "metadata": {} - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Network is valid\n", - "(0, 1)\n", - "(0, 2)\n", - "(1, 2)\n", - "(2, 3)\n", - "(3, 4)\n", - "(4, 5)\n", - "(4, 6)\n", - "(5, 6)\n", - "(5, 7)\n", - "(6, 7)\n" - ] - }, - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "hw_model.to(device=\"speck2fmodule:0\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The layers comprising our `hw_model` and their respective metadata can be inspected by calling `print` on a `DynapcnnNetworkGraph` instance." - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": { - "metadata": {} - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "---- DynapcnnLayer 0 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 0): Conv2d(2, 10, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", - "(node 1): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "(node 2): SumPool2d(norm_type=1, kernel_size=(3, 3), stride=None, ceil_mode=False)\n", - "(node 3): SumPool2d(norm_type=1, kernel_size=(4, 4), stride=None, ceil_mode=False)\n", - "> layer destinations: [1, 2]\n", - "> assigned core: 0\n", - "\n", - "---- DynapcnnLayer 1 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 4): Conv2d(10, 10, kernel_size=(4, 4), stride=(1, 1), bias=False)\n", - "(node 6): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [2]\n", - "> assigned core: 1\n", - "\n", - "---- DynapcnnLayer 2 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 7): Conv2d(10, 10, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", - "(node 8): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [3]\n", - "> assigned core: 2\n", - "\n", - "---- DynapcnnLayer 3 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 9): Conv2d(10, 1, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", - "(node 10): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [4]\n", - "> assigned core: 3\n", - "\n", - "---- DynapcnnLayer 4 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 11): Conv2d(1, 100, kernel_size=(6, 6), stride=(1, 1), bias=False)\n", - "(node 12): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [5, 6]\n", - "> assigned core: 4\n", - "\n", - "---- DynapcnnLayer 5 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 13): Conv2d(100, 100, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - "(node 15): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [6, 7]\n", - "> assigned core: 5\n", - "\n", - "---- DynapcnnLayer 6 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 17): Conv2d(100, 100, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - "(node 18): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [7]\n", - "> assigned core: 6\n", - "\n", - "---- DynapcnnLayer 7 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 19): Conv2d(100, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - "(node 20): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: []\n", - "> assigned core: 7\n", - "\n", - "\n" - ] - } - ], - "source": [ - "print(hw_model)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "speck-rescnn", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_5a.ipynb b/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_5a.ipynb deleted file mode 100644 index 991e48fa..00000000 --- a/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_5a.ipynb +++ /dev/null @@ -1,390 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "import torch\n", - "import torch.nn as nn\n", - "from sinabs.backend.dynapcnn import DynapcnnNetworkGraph\n", - "from sinabs.layers import Merge, IAFSqueeze, SumPool2d" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "metadata": {} - }, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "torch.manual_seed(0)" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "channels = 2\n", - "height = 34\n", - "width = 34\n", - "\n", - "input_shape = (channels, height, width)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Network Module\n", - "\n", - "We need to define a `nn.Module` implementing the network we want the chip to reproduce." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "class SNN(nn.Module):\n", - " def __init__(self) -> None:\n", - " super().__init__()\n", - "\n", - " self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False)\n", - " self.conv1_iaf = IAFSqueeze(batch_size=1)\n", - " self.pool1 = SumPool2d(3,3)\n", - " self.pool1a = SumPool2d(4,4)\n", - "\n", - " self.conv2 = nn.Conv2d(10, 10, 4, 1, bias=False)\n", - " self.conv2_iaf = IAFSqueeze(batch_size=1)\n", - "\n", - " self.conv3 = nn.Conv2d(10, 10, 2, 1, bias=False)\n", - " self.conv3_iaf = IAFSqueeze(batch_size=1)\n", - "\n", - " self.conv4 = nn.Conv2d(10, 1, 2, 1, bias=False)\n", - " self.conv4_iaf = IAFSqueeze(batch_size=1)\n", - "\n", - " self.flat = nn.Flatten()\n", - "\n", - " self.fc1 = nn.Linear(36, 100, bias=False)\n", - " self.fc1_iaf = IAFSqueeze(batch_size=1)\n", - " \n", - " self.fc2 = nn.Linear(100, 100, bias=False)\n", - " self.fc2_iaf = IAFSqueeze(batch_size=1)\n", - "\n", - " self.fc3 = nn.Linear(100, 100, bias=False)\n", - " self.fc3_iaf = IAFSqueeze(batch_size=1)\n", - "\n", - " self.fc4 = nn.Linear(100, 10, bias=False)\n", - " self.fc4_iaf = IAFSqueeze(batch_size=1)\n", - "\n", - " self.merge1 = Merge()\n", - " self.merge2 = Merge()\n", - " self.merge3 = Merge()\n", - "\n", - " def forward(self, x):\n", - " # -- conv. block 0 --\n", - " con1_out = self.conv1(x)\n", - " conv1_iaf_out = self.conv1_iaf(con1_out)\n", - " pool1_out = self.pool1(conv1_iaf_out)\n", - " pool1a_out = self.pool1a(conv1_iaf_out)\n", - " # -- conv. block 1 --\n", - " conv2_out = self.conv2(pool1_out)\n", - " conv2_iaf_out = self.conv2_iaf(conv2_out)\n", - " # -- conv. block 2 --\n", - " merge1_out = self.merge1(pool1a_out, conv2_iaf_out)\n", - " conv3_out = self.conv3(merge1_out)\n", - " conv3_iaf_out = self.conv3_iaf(conv3_out)\n", - " # -- conv. block 3 --\n", - " conv4_out = self.conv4(conv3_iaf_out)\n", - " conv4_iaf_out = self.conv4_iaf(conv4_out)\n", - " flat_out = self.flat(conv4_iaf_out)\n", - " # -- fc clock 4 --\n", - " fc1_out = self.fc1(flat_out)\n", - " fc1_iaf_out = self.fc1_iaf(fc1_out)\n", - " # -- fc clock 5 --\n", - " fc2_out = self.fc2(fc1_iaf_out)\n", - " fc2_iaf_out = self.fc2_iaf(fc2_out)\n", - " # -- fc clock 6 --\n", - " merge2_out = self.merge2(fc1_iaf_out, fc2_iaf_out)\n", - " fc3_out = self.fc3(merge2_out)\n", - " fc3_iaf_out = self.fc3_iaf(fc3_out)\n", - " # -- fc clock 7 --\n", - " merge3_out = self.merge3(fc2_iaf_out, fc3_iaf_out)\n", - " fc4_out = self.fc4(merge3_out)\n", - " fc4_iaf_out = self.fc4_iaf(fc4_out)\n", - "\n", - " return fc4_iaf_out" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "snn = SNN()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The following exemplifies how each of the layers in the SNN should be grouped together to form a `DynapcnnLayer` instance. Let's check the shapes of the tensors each of the (original) layers in the model inputs and outputs by feeding a fake input through the model:" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "x = torch.randn((1, *input_shape))\n", - "\n", - "# -- conv. block 0 --\n", - "con1_out = snn.conv1(x)\n", - "conv1_iaf_out = snn.conv1_iaf(con1_out)\n", - "pool1_out = snn.pool1(conv1_iaf_out)\n", - "pool1a_out = snn.pool1a(conv1_iaf_out)\n", - "# -- conv. block 1 --\n", - "conv2_out = snn.conv2(pool1_out)\n", - "conv2_iaf_out = snn.conv2_iaf(conv2_out)\n", - "# -- conv. block 2 --\n", - "merge1_out = snn.merge1(pool1a_out, conv2_iaf_out)\n", - "conv3_out = snn.conv3(merge1_out)\n", - "conv3_iaf_out = snn.conv3_iaf(conv3_out)\n", - "# -- conv. block 3 --\n", - "conv4_out = snn.conv4(conv3_iaf_out)\n", - "conv4_iaf_out = snn.conv4_iaf(conv4_out)\n", - "flat_out = snn.flat(conv4_iaf_out)\n", - "# -- fc clock 4 --\n", - "fc1_out = snn.fc1(flat_out)\n", - "fc1_iaf_out = snn.fc1_iaf(fc1_out)\n", - "# -- fc clock 5 --\n", - "fc2_out = snn.fc2(fc1_iaf_out)\n", - "fc2_iaf_out = snn.fc2_iaf(fc2_out)\n", - "# -- fc clock 6 --\n", - "merge2_out = snn.merge2(fc1_iaf_out, fc2_iaf_out)\n", - "fc3_out = snn.fc3(merge2_out)\n", - "fc3_iaf_out = snn.fc3_iaf(fc3_out)\n", - "# -- fc clock 7 --\n", - "merge3_out = snn.merge3(fc2_iaf_out, fc3_iaf_out)\n", - "fc4_out = snn.fc4(merge3_out)\n", - "fc4_iaf_out = snn.fc4_iaf(fc4_out)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## DynapcnnNetwork Class\n", - "\n", - "In the constructor of `DynapcnnNetworkGraph` the SNN passed as argument (defined as a `nn.Module`) will be parsed such that each layer is represented in a computational graph (using `nirtorch.extract_torch_graph`). \n", - "\n", - "The layers are the `nodes` of the graph, while their connectivity (how the outputs from a layer are sent to other layers) is represented as `edges`, represented in a `list` of `tuples`.\n", - "\n", - "Once the constructor finishes its initialization, the `hw_model.dynapcnn_layers` property is a dictionary where each entry represents the ID of a `DynapcnnLayer` instance (an `int` from `0` to `L`), with this entry containing a `DynapcnnLayer` instance where a subset of the layers in the original SNN has been incorporated into, the core such instance has been assigned to, and the list of `DynapcnnLayer` instances (their IDs) the layer targets." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "hw_model = DynapcnnNetworkGraph(\n", - " snn,\n", - " discretize=True,\n", - " input_shape=input_shape\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The `hw_model.to()` call will figure out into which core eac `DynapcnnLayer` instance will be assigned to. Once this assingment is made the instance itself is used to configure the `CNNLayerConfig` instance representing the core's configuration.\n", - "\n", - "If the cores' configuration is valid, each `DynapcnnLayer` instance and their respective destinations will be used to create a computational graph that encodes how the `forward` method of `hw_model.network` (a `nn.Module` using the `DynapcnnLayer` instances) propagates that through the network." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": { - "metadata": {} - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Network is valid\n" - ] - }, - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "hw_model.to(device=\"speck2fmodule:0\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The layers comprising our `hw_model` and their respective metadata can be inspected by calling `print` on a `DynapcnnNetworkGraph` instance." - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": { - "metadata": {} - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "---- DynapcnnLayer 0 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 0): Conv2d(2, 10, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", - "(node 1): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "(node 2): SumPool2d(norm_type=1, kernel_size=3, stride=3, ceil_mode=False)\n", - "(node 3): SumPool2d(norm_type=1, kernel_size=4, stride=4, ceil_mode=False)\n", - "> layer destinations: [1, 2]\n", - "> assigned core: 0\n", - "\n", - "---- DynapcnnLayer 1 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 4): Conv2d(10, 10, kernel_size=(4, 4), stride=(1, 1), bias=False)\n", - "(node 6): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [2]\n", - "> assigned core: 1\n", - "\n", - "---- DynapcnnLayer 2 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 7): Conv2d(10, 10, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", - "(node 8): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [3]\n", - "> assigned core: 2\n", - "\n", - "---- DynapcnnLayer 3 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 9): Conv2d(10, 1, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", - "(node 10): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [4]\n", - "> assigned core: 3\n", - "\n", - "---- DynapcnnLayer 4 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 11): Conv2d(1, 100, kernel_size=(6, 6), stride=(1, 1), bias=False)\n", - "(node 12): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [5, 6]\n", - "> assigned core: 4\n", - "\n", - "---- DynapcnnLayer 5 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 13): Conv2d(100, 100, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - "(node 15): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [6, 7]\n", - "> assigned core: 5\n", - "\n", - "---- DynapcnnLayer 6 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 17): Conv2d(100, 100, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - "(node 18): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [7]\n", - "> assigned core: 6\n", - "\n", - "---- DynapcnnLayer 7 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 19): Conv2d(100, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - "(node 20): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: []\n", - "> assigned core: 7\n", - "\n", - "\n" - ] - } - ], - "source": [ - "print(hw_model)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "speck-rescnn", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_6.ipynb b/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_6.ipynb deleted file mode 100644 index 9a8dc655..00000000 --- a/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/DynapcnnNetwork-example_6.ipynb +++ /dev/null @@ -1,422 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "import torch\n", - "import torch.nn as nn\n", - "from sinabs.backend.dynapcnn import DynapcnnNetworkGraph\n", - "from sinabs.layers import Merge, IAFSqueeze, SumPool2d\n", - "import sinabs.layers as sl\n", - "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "metadata": {} - }, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "torch.manual_seed(0)" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "channels = 2\n", - "height = 34\n", - "width = 34\n", - "\n", - "input_shape = (channels, height, width)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Network Module\n", - "\n", - "We need to define a `nn.Module` implementing the network we want the chip to reproduce." - ] - }, - { - "cell_type": "code", - "execution_count": 67, - "metadata": { - "metadata": {} - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "torch.Size([8, 128])\n" - ] - }, - { - "data": { - "text/plain": [ - "tensor([[0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", - " [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", - " [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", - " [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", - " [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", - " [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", - " [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", - " [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]], grad_fn=)" - ] - }, - "execution_count": 67, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "class SNN(nn.Module):\n", - " def __init__(self, nb_classes, batch_size, surrogate_fn, min_v_mem=-0.313, spk_thr=2.0) -> None:\n", - " super().__init__()\n", - "\n", - " self.conv1 = nn.Conv2d(2, 1, 2, 1, bias=False)\n", - " self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", - " self.pool1 = sl.SumPool2d(2,2)\n", - "\n", - " self.conv2 = nn.Conv2d(1, 8, 2, 1, bias=False)\n", - " self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", - " self.pool2 = sl.SumPool2d(2,2)\n", - "\n", - " self.conv3 = nn.Conv2d(8, 16, 2, 1, bias=False)\n", - " self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", - " self.pool3 = sl.SumPool2d(2,2)\n", - "\n", - " self.conv4 = nn.Conv2d(16, 32, 2, 1, bias=False)\n", - " self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", - "\n", - " self.flat = nn.Flatten()\n", - "\n", - " self.fc1 = nn.Linear(128, 1024, bias=False)\n", - " self.iaf1_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", - "\n", - " self.fc2 = nn.Linear(1024, 512, bias=False)\n", - " self.iaf2_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", - "\n", - " self.fc3 = nn.Linear(512, 256, bias=False)\n", - " self.iaf3_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", - "\n", - " self.fc4 = nn.Linear(256, 128, bias=False)\n", - " self.iaf4_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", - "\n", - " self.fc5 = nn.Linear(128, nb_classes, bias=False)\n", - " self.iaf5_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", - "\n", - " def forward(self, x):\n", - " # conv 1\n", - " con1_out = self.conv1(x)\n", - " iaf1_out = self.iaf1(con1_out)\n", - " pool1_out = self.pool1(iaf1_out)\n", - "\n", - " # conv 2\n", - " conv2_out = self.conv2(pool1_out)\n", - " iaf2_out = self.iaf2(conv2_out)\n", - " pool2_out = self.pool2(iaf2_out)\n", - "\n", - " # conv 3\n", - " conv3_out = self.conv3(pool2_out)\n", - " iaf3_out = self.iaf3(conv3_out)\n", - " pool3_out = self.pool3(iaf3_out)\n", - "\n", - " # conv 4\n", - " conv4_out = self.conv4(pool3_out)\n", - " iaf4_out = self.iaf4(conv4_out)\n", - "\n", - " flat_out = self.flat(iaf4_out)\n", - " \n", - " # fc 1\n", - " print(flat_out.shape)\n", - " fc1_out = self.fc1(flat_out)\n", - " iaf1_fc_out = self.iaf1_fc(fc1_out)\n", - "\n", - " # fc 2\n", - " fc2_out = self.fc2(iaf1_fc_out)\n", - " iaf2_fc_out = self.iaf2_fc(fc2_out)\n", - "\n", - " # fc 3\n", - " fc3_out = self.fc3(iaf2_fc_out)\n", - " iaf3_fc_out = self.iaf3_fc(fc3_out)\n", - "\n", - " # fc 4\n", - " fc4_out = self.fc4(iaf3_fc_out)\n", - " iaf4_fc_out = self.iaf4_fc(fc4_out)\n", - "\n", - " # fc 5\n", - " fc5_out = self.fc5(iaf4_fc_out)\n", - " iaf5_fc_out = self.iaf5_fc(fc5_out)\n", - "\n", - " return iaf5_fc_out\n", - " \n", - "snn = SNN(11, 8, PeriodicExponential())\n", - "\n", - "x = torch.randn((8, *input_shape))\n", - "\n", - "snn(x)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The following exemplifies how each of the layers in the SNN should be grouped together to form a `DynapcnnLayer` instance. Let's check the shapes of the tensors each of the (original) layers in the model inputs and outputs by feeding a fake input through the model:" - ] - }, - { - "cell_type": "code", - "execution_count": 30, - "metadata": { - "metadata": {} - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "torch.Size([8, 8, 17, 17])\n", - "torch.Size([8, 8, 16, 16])\n", - "torch.Size([8, 8, 1, 1])\n", - "torch.Size([8, 8])\n" - ] - }, - { - "ename": "RuntimeError", - "evalue": "mat1 and mat2 shapes cannot be multiplied (8x8 and 800x11)", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[30], line 3\u001b[0m\n\u001b[1;32m 1\u001b[0m x \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mrandn((\u001b[38;5;241m8\u001b[39m, \u001b[38;5;241m*\u001b[39minput_shape))\n\u001b[0;32m----> 3\u001b[0m snn(x)\n", - "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/nn/modules/module.py:1511\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1509\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs) \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m 1510\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1511\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n", - "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/nn/modules/module.py:1520\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1515\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1516\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1517\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m 1518\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1519\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1520\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m forward_call(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m 1522\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 1523\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n", - "Cell \u001b[0;32mIn[28], line 74\u001b[0m, in \u001b[0;36mSNN.forward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m 72\u001b[0m flat \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mflat(iaf8_out)\n\u001b[1;32m 73\u001b[0m \u001b[38;5;28mprint\u001b[39m(flat\u001b[38;5;241m.\u001b[39mshape)\n\u001b[0;32m---> 74\u001b[0m fc_out \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfc_out(flat)\n\u001b[1;32m 75\u001b[0m iaf_fc_out \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39miaf_fc_out(fc_out)\n\u001b[1;32m 78\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m iaf_fc_out\n", - "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/nn/modules/module.py:1511\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1509\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs) \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m 1510\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1511\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n", - "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/nn/modules/module.py:1520\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1515\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1516\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1517\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m 1518\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1519\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1520\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m forward_call(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m 1522\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 1523\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n", - "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/nn/modules/linear.py:116\u001b[0m, in \u001b[0;36mLinear.forward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m 115\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;28minput\u001b[39m: Tensor) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tensor:\n\u001b[0;32m--> 116\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m F\u001b[38;5;241m.\u001b[39mlinear(\u001b[38;5;28minput\u001b[39m, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mweight, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbias)\n", - "\u001b[0;31mRuntimeError\u001b[0m: mat1 and mat2 shapes cannot be multiplied (8x8 and 800x11)" - ] - } - ], - "source": [ - "x = torch.randn((8, *input_shape))\n", - "\n", - "snn(x)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## DynapcnnNetwork Class\n", - "\n", - "In the constructor of `DynapcnnNetworkGraph` the SNN passed as argument (defined as a `nn.Module`) will be parsed such that each layer is represented in a computational graph (using `nirtorch.extract_torch_graph`). \n", - "\n", - "The layers are the `nodes` of the graph, while their connectivity (how the outputs from a layer are sent to other layers) is represented as `edges`, represented in a `list` of `tuples`.\n", - "\n", - "Once the constructor finishes its initialization, the `hw_model.dynapcnn_layers` property is a dictionary where each entry represents the ID of a `DynapcnnLayer` instance (an `int` from `0` to `L`), with this entry containing a `DynapcnnLayer` instance where a subset of the layers in the original SNN has been incorporated into, the core such instance has been assigned to, and the list of `DynapcnnLayer` instances (their IDs) the layer targets." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "hw_model = DynapcnnNetworkGraph(\n", - " snn,\n", - " discretize=True,\n", - " input_shape=input_shape\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The `hw_model.to()` call will figure out into which core eac `DynapcnnLayer` instance will be assigned to. Once this assingment is made the instance itself is used to configure the `CNNLayerConfig` instance representing the core's configuration.\n", - "\n", - "If the cores' configuration is valid, each `DynapcnnLayer` instance and their respective destinations will be used to create a computational graph that encodes how the `forward` method of `hw_model.network` (a `nn.Module` using the `DynapcnnLayer` instances) propagates that through the network." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": { - "metadata": {} - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Network is valid\n" - ] - }, - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "hw_model.to(device=\"speck2fmodule:0\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The layers comprising our `hw_model` and their respective metadata can be inspected by calling `print` on a `DynapcnnNetworkGraph` instance." - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": { - "metadata": {} - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "---- DynapcnnLayer 0 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 0): Conv2d(2, 10, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", - "(node 1): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "(node 2): SumPool2d(norm_type=1, kernel_size=3, stride=3, ceil_mode=False)\n", - "(node 3): SumPool2d(norm_type=1, kernel_size=4, stride=4, ceil_mode=False)\n", - "> layer destinations: [1, 2]\n", - "> assigned core: 0\n", - "\n", - "---- DynapcnnLayer 1 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 4): Conv2d(10, 10, kernel_size=(4, 4), stride=(1, 1), bias=False)\n", - "(node 6): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [2]\n", - "> assigned core: 1\n", - "\n", - "---- DynapcnnLayer 2 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 7): Conv2d(10, 10, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", - "(node 8): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [3]\n", - "> assigned core: 2\n", - "\n", - "---- DynapcnnLayer 3 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 9): Conv2d(10, 1, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", - "(node 10): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [4]\n", - "> assigned core: 3\n", - "\n", - "---- DynapcnnLayer 4 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 11): Conv2d(1, 100, kernel_size=(6, 6), stride=(1, 1), bias=False)\n", - "(node 12): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [5, 6]\n", - "> assigned core: 4\n", - "\n", - "---- DynapcnnLayer 5 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 13): Conv2d(100, 100, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - "(node 15): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [6, 7]\n", - "> assigned core: 5\n", - "\n", - "---- DynapcnnLayer 6 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 17): Conv2d(100, 100, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - "(node 18): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [7]\n", - "> assigned core: 6\n", - "\n", - "---- DynapcnnLayer 7 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 19): Conv2d(100, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - "(node 20): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: []\n", - "> assigned core: 7\n", - "\n", - "\n" - ] - } - ], - "source": [ - "print(hw_model)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "speck-rescnn", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/complex_network_structure.ipynb b/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/complex_network_structure.ipynb deleted file mode 100644 index 4494924e..00000000 --- a/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/complex_network_structure.ipynb +++ /dev/null @@ -1,337 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "import torch\n", - "import torch.nn as nn\n", - "from sinabs.backend.dynapcnn import DynapcnnNetworkGraph\n", - "from sinabs.layers import Merge, IAFSqueeze, SumPool2d\n", - "import sinabs.layers as sl\n", - "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "metadata": {} - }, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "torch.manual_seed(0)" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "channels = 2\n", - "height = 34\n", - "width = 34\n", - "\n", - "input_shape = (channels, height, width)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Network Module\n", - "\n", - "We need to define a `nn.Module` implementing the network we want the chip to reproduce." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "```mermaid\n", - "stateDiagram\n", - " [*] --> A\n", - " A --> B\n", - " A --> C\n", - " C --> D\n", - " C --> E\n", - " B --> D\n", - " D --> F\n", - " E --> F\n", - " F --> [*]\n", - "```" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "class SNN(nn.Module):\n", - " def __init__(self, nb_classes, batch_size, surrogate_fn, min_v_mem=-0.313, spk_thr=2.0) -> None:\n", - " super().__init__()\n", - "\n", - " self.conv1 = nn.Conv2d(2, 8, 2, 1, bias=False)\n", - " self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", - "\n", - " self.conv2 = nn.Conv2d(8, 8, 2, 1, bias=False)\n", - " self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", - " self.pool2 = sl.SumPool2d(2,2)\n", - "\n", - " self.conv3 = nn.Conv2d(8, 8, 2, 1, bias=False)\n", - " self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", - " self.pool3 = sl.SumPool2d(2,2)\n", - " self.pool3a = sl.SumPool2d(6,6)\n", - "\n", - " self.conv4 = nn.Conv2d(8, 8, 2, 1, bias=False)\n", - " self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", - " self.pool4 = sl.SumPool2d(3,3)\n", - "\n", - " self.flat = nn.Flatten()\n", - " self.flat_a = nn.Flatten()\n", - "\n", - " self.fc1 = nn.Linear(200, 200, bias=False)\n", - " self.iaf1_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", - "\n", - " self.fc2 = nn.Linear(200, nb_classes, bias=False)\n", - " self.iaf2_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", - "\n", - " # -- merges --\n", - " self.merge1 = Merge()\n", - " self.merge2 = Merge()\n", - "\n", - " def forward(self, x):\n", - " # conv 1 - A\n", - " con1_out = self.conv1(x)\n", - " iaf1_out = self.iaf1(con1_out)\n", - "\n", - " # conv 2 - B\n", - " conv2_out = self.conv2(iaf1_out)\n", - " iaf2_out = self.iaf2(conv2_out)\n", - " pool2_out = self.pool2(iaf2_out)\n", - "\n", - " # conv 3 - C\n", - " conv3_out = self.conv3(iaf1_out)\n", - " iaf3_out = self.iaf3(conv3_out)\n", - " pool3_out = self.pool3(iaf3_out)\n", - " pool3a_out = self.pool3a(iaf3_out)\n", - "\n", - " # conv 4 - D\n", - " merge1_out = self.merge1(pool2_out, pool3_out)\n", - " conv4_out = self.conv4(merge1_out)\n", - " iaf4_out = self.iaf4(conv4_out)\n", - " pool4_out = self.pool4(iaf4_out)\n", - " flat_out = self.flat(pool4_out)\n", - " \n", - " # fc 1 - E\n", - " flat_a_out = self.flat_a(pool3a_out)\n", - " fc1_out = self.fc1(flat_a_out)\n", - " iaf1_fc_out = self.iaf1_fc(fc1_out)\n", - "\n", - " # fc 2 - F\n", - " merge2_out = self.merge2(iaf1_fc_out, flat_out)\n", - " fc2_out = self.fc2(merge2_out)\n", - " iaf2_fc_out = self.iaf2_fc(fc2_out)\n", - "\n", - " return iaf2_fc_out\n", - " \n", - "snn = SNN(11, 1, PeriodicExponential())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## DynapcnnNetwork Class\n", - "\n", - "In the constructor of `DynapcnnNetworkGraph` the SNN passed as argument (defined as a `nn.Module`) will be parsed such that each layer is represented in a computational graph (using `nirtorch.extract_torch_graph`). \n", - "\n", - "The layers are the `nodes` of the graph, while their connectivity (how the outputs from a layer are sent to other layers) is represented as `edges`, represented in a `list` of `tuples`.\n", - "\n", - "Once the constructor finishes its initialization, the `hw_model.dynapcnn_layers` property is a dictionary where each entry represents the ID of a `DynapcnnLayer` instance (an `int` from `0` to `L`), with this entry containing a `DynapcnnLayer` instance where a subset of the layers in the original SNN has been incorporated into, the core such instance has been assigned to, and the list of `DynapcnnLayer` instances (their IDs) the layer targets." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "hw_model = DynapcnnNetworkGraph(\n", - " snn,\n", - " discretize=True,\n", - " input_shape=input_shape\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The `hw_model.to()` call will figure out into which core eac `DynapcnnLayer` instance will be assigned to. Once this assingment is made the instance itself is used to configure the `CNNLayerConfig` instance representing the core's configuration.\n", - "\n", - "If the cores' configuration is valid, each `DynapcnnLayer` instance and their respective destinations will be used to create a computational graph that encodes how the `forward` method of `hw_model.network` (a `nn.Module` using the `DynapcnnLayer` instances) propagates that through the network." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "metadata": {} - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Network is valid\n" - ] - }, - { - "ename": "RuntimeError", - "evalue": "Device is already opened!", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[6], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m hw_model\u001b[38;5;241m.\u001b[39mto(device\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mspeck2fmodule:0\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", - "File \u001b[0;32m~/Documents/github/sinabs/sinabs/backend/dynapcnn/dynapcnn_network_graph.py:172\u001b[0m, in \u001b[0;36mDynapcnnNetworkGraph.to\u001b[0;34m(self, device, chip_layers_ordering, monitor_layers, config_modifier, slow_clk_frequency)\u001b[0m\n\u001b[1;32m 163\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m device_name \u001b[38;5;129;01min\u001b[39;00m ChipFactory\u001b[38;5;241m.\u001b[39msupported_devices: \u001b[38;5;66;03m# pragma: no cover\u001b[39;00m\n\u001b[1;32m 165\u001b[0m config \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmake_config( \u001b[38;5;66;03m# generate config.\u001b[39;00m\n\u001b[1;32m 166\u001b[0m chip_layers_ordering\u001b[38;5;241m=\u001b[39mchip_layers_ordering,\n\u001b[1;32m 167\u001b[0m device\u001b[38;5;241m=\u001b[39mdevice,\n\u001b[1;32m 168\u001b[0m monitor_layers\u001b[38;5;241m=\u001b[39mmonitor_layers,\n\u001b[1;32m 169\u001b[0m config_modifier\u001b[38;5;241m=\u001b[39mconfig_modifier,\n\u001b[1;32m 170\u001b[0m )\n\u001b[0;32m--> 172\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msamna_device \u001b[38;5;241m=\u001b[39m open_device(device) \u001b[38;5;66;03m# apply configuration to device.\u001b[39;00m\n\u001b[1;32m 173\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msamna_device\u001b[38;5;241m.\u001b[39mget_model()\u001b[38;5;241m.\u001b[39mapply_configuration(config)\n\u001b[1;32m 174\u001b[0m time\u001b[38;5;241m.\u001b[39msleep(\u001b[38;5;241m1\u001b[39m)\n", - "File \u001b[0;32m~/Documents/github/sinabs/sinabs/backend/dynapcnn/io.py:256\u001b[0m, in \u001b[0;36mopen_device\u001b[0;34m(device_id)\u001b[0m\n\u001b[1;32m 254\u001b[0m device_map \u001b[38;5;241m=\u001b[39m get_device_map()\n\u001b[1;32m 255\u001b[0m device_info \u001b[38;5;241m=\u001b[39m device_map[device_id]\n\u001b[0;32m--> 256\u001b[0m device_handle \u001b[38;5;241m=\u001b[39m samna\u001b[38;5;241m.\u001b[39mdevice\u001b[38;5;241m.\u001b[39mopen_device(device_info)\n\u001b[1;32m 258\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m device_handle \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 259\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m device_handle\n", - "\u001b[0;31mRuntimeError\u001b[0m: Device is already opened!" - ] - } - ], - "source": [ - "hw_model.to(device=\"speck2fmodule:0\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The layers comprising our `hw_model` and their respective metadata can be inspected by calling `print` on a `DynapcnnNetworkGraph` instance." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "---- DynapcnnLayer 0 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 0): Conv2d(2, 8, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", - "(node 1): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(723.), min_v_mem=Parameter containing:\n", - "tensor(-113.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [1, 2]\n", - "> assigned core: 0\n", - "\n", - "---- DynapcnnLayer 1 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 2): Conv2d(8, 8, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", - "(node 4): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1452.), min_v_mem=Parameter containing:\n", - "tensor(-227.), batch_size=1, num_timesteps=-1)\n", - "(node 5): SumPool2d(norm_type=1, kernel_size=2, stride=2, ceil_mode=False)\n", - "> layer destinations: [3]\n", - "> assigned core: 1\n", - "\n", - "---- DynapcnnLayer 2 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 3): Conv2d(8, 8, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", - "(node 7): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1437.), min_v_mem=Parameter containing:\n", - "tensor(-225.), batch_size=1, num_timesteps=-1)\n", - "(node 8): SumPool2d(norm_type=1, kernel_size=2, stride=2, ceil_mode=False)\n", - "(node 9): SumPool2d(norm_type=1, kernel_size=6, stride=6, ceil_mode=False)\n", - "> layer destinations: [3, 4]\n", - "> assigned core: 2\n", - "\n", - "---- DynapcnnLayer 3 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 10): Conv2d(8, 8, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", - "(node 11): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1439.), min_v_mem=Parameter containing:\n", - "tensor(-225.), batch_size=1, num_timesteps=-1)\n", - "(node 12): SumPool2d(norm_type=1, kernel_size=3, stride=3, ceil_mode=False)\n", - "> layer destinations: [5]\n", - "> assigned core: 3\n", - "\n", - "---- DynapcnnLayer 4 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 14): Conv2d(8, 200, kernel_size=(5, 5), stride=(1, 1), bias=False)\n", - "(node 15): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(3592.), min_v_mem=Parameter containing:\n", - "tensor(-562.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [5]\n", - "> assigned core: 5\n", - "\n", - "---- DynapcnnLayer 5 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 16): Conv2d(200, 11, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - "(node 17): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(3592.), min_v_mem=Parameter containing:\n", - "tensor(-562.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: []\n", - "> assigned core: 4\n", - "\n", - "\n" - ] - } - ], - "source": [ - "print(hw_model)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "speck-rescnn", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/split_and_merge.ipynb b/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/split_and_merge.ipynb deleted file mode 100644 index 4ce4ffc8..00000000 --- a/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/split_and_merge.ipynb +++ /dev/null @@ -1,359 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "import torch\n", - "import torch.nn as nn\n", - "from sinabs.backend.dynapcnn import DynapcnnNetworkGraph\n", - "from sinabs.layers import Merge, IAFSqueeze, SumPool2d\n", - "import sinabs.layers as sl\n", - "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "metadata": {} - }, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "torch.manual_seed(0)" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "channels = 2\n", - "height = 34\n", - "width = 34\n", - "\n", - "input_shape = (channels, height, width)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Network Module\n", - "\n", - "We need to define a `nn.Module` implementing the network we want the chip to reproduce." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "```mermaid\n", - "stateDiagram\n", - " [*] --> A\n", - " A --> B\n", - " B --> C\n", - " C --> D\n", - " B --> E\n", - " E --> F\n", - " D --> G\n", - " F --> G\n", - " G --> [*]\n", - "```" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "class SNN(nn.Module):\n", - " def __init__(self, nb_classes, batch_size, surrogate_fn, min_v_mem=-0.313, spk_thr=2.0) -> None:\n", - " super().__init__()\n", - "\n", - " self.conv1 = nn.Conv2d(2, 4, 2, 1, bias=False)\n", - " self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", - "\n", - " self.conv2 = nn.Conv2d(4, 4, 2, 1, bias=False)\n", - " self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", - " self.pool2 = sl.SumPool2d(2,2)\n", - " self.pool2a = sl.SumPool2d(5,5)\n", - "\n", - " self.conv3 = nn.Conv2d(4, 4, 2, 1, bias=False)\n", - " self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", - " self.pool3 = sl.SumPool2d(2,2)\n", - "\n", - " self.conv4 = nn.Conv2d(4, 4, 2, 1, bias=False)\n", - " self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", - "\n", - " self.flat = nn.Flatten()\n", - " self.flat_a = nn.Flatten()\n", - "\n", - " self.fc1 = nn.Linear(144, 144, bias=False)\n", - " self.iaf1_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", - "\n", - " self.fc2 = nn.Linear(144, 144, bias=False)\n", - " self.iaf2_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", - "\n", - " self.fc3 = nn.Linear(144, nb_classes, bias=False)\n", - " self.iaf3_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", - "\n", - " # -- merges --\n", - " self.merge1 = Merge()\n", - "\n", - " def forward(self, x):\n", - " # conv 1 - A\n", - " con1_out = self.conv1(x)\n", - " iaf1_out = self.iaf1(con1_out)\n", - "\n", - " # conv 2 - B\n", - " conv2_out = self.conv2(iaf1_out)\n", - " iaf2_out = self.iaf2(conv2_out)\n", - " pool2_out = self.pool2(iaf2_out)\n", - " pool2a_out = self.pool2a(iaf2_out)\n", - "\n", - " # conv 3 - C\n", - " conv3_out = self.conv3(pool2_out)\n", - " iaf3_out = self.iaf3(conv3_out)\n", - " pool3_out = self.pool3(iaf3_out)\n", - "\n", - " # conv 4 - D\n", - " conv4_out = self.conv4(pool3_out)\n", - " iaf4_out = self.iaf4(conv4_out)\n", - " flat_out = self.flat(iaf4_out)\n", - " \n", - " # fc 1 - E\n", - " flat_a_out = self.flat_a(pool2a_out)\n", - " print(flat_a_out.shape)\n", - " fc1_out = self.fc1(flat_a_out)\n", - " iaf1_fc_out = self.iaf1_fc(fc1_out)\n", - "\n", - " # fc 2 - F\n", - " fc2_out = self.fc2(iaf1_fc_out)\n", - " iaf2_fc_out = self.iaf2_fc(fc2_out)\n", - "\n", - " # fc 2 - G\n", - " print(flat_out.shape, iaf2_fc_out.shape)\n", - " merge1_out = self.merge1(flat_out, iaf2_fc_out)\n", - " fc3_out = self.fc3(merge1_out)\n", - " iaf3_fc_out = self.iaf3_fc(fc3_out)\n", - "\n", - " return iaf3_fc_out\n", - " \n", - "snn = SNN(11, 1, PeriodicExponential())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## DynapcnnNetwork Class\n", - "\n", - "In the constructor of `DynapcnnNetworkGraph` the SNN passed as argument (defined as a `nn.Module`) will be parsed such that each layer is represented in a computational graph (using `nirtorch.extract_torch_graph`). \n", - "\n", - "The layers are the `nodes` of the graph, while their connectivity (how the outputs from a layer are sent to other layers) is represented as `edges`, represented in a `list` of `tuples`.\n", - "\n", - "Once the constructor finishes its initialization, the `hw_model.dynapcnn_layers` property is a dictionary where each entry represents the ID of a `DynapcnnLayer` instance (an `int` from `0` to `L`), with this entry containing a `DynapcnnLayer` instance where a subset of the layers in the original SNN has been incorporated into, the core such instance has been assigned to, and the list of `DynapcnnLayer` instances (their IDs) the layer targets." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "metadata": {} - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "torch.Size([1, 144])\n", - "torch.Size([1, 144]) torch.Size([1, 144])\n" - ] - } - ], - "source": [ - "hw_model = DynapcnnNetworkGraph(\n", - " snn,\n", - " discretize=True,\n", - " input_shape=input_shape\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The `hw_model.to()` call will figure out into which core eac `DynapcnnLayer` instance will be assigned to. Once this assingment is made the instance itself is used to configure the `CNNLayerConfig` instance representing the core's configuration.\n", - "\n", - "If the cores' configuration is valid, each `DynapcnnLayer` instance and their respective destinations will be used to create a computational graph that encodes how the `forward` method of `hw_model.network` (a `nn.Module` using the `DynapcnnLayer` instances) propagates that through the network." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "metadata": {} - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Network is valid\n" - ] - }, - { - "ename": "RuntimeError", - "evalue": "Device is already opened!", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[6], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m hw_model\u001b[38;5;241m.\u001b[39mto(device\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mspeck2fmodule:0\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", - "File \u001b[0;32m~/Documents/github/sinabs/sinabs/backend/dynapcnn/dynapcnn_network_graph.py:172\u001b[0m, in \u001b[0;36mDynapcnnNetworkGraph.to\u001b[0;34m(self, device, chip_layers_ordering, monitor_layers, config_modifier, slow_clk_frequency)\u001b[0m\n\u001b[1;32m 163\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m device_name \u001b[38;5;129;01min\u001b[39;00m ChipFactory\u001b[38;5;241m.\u001b[39msupported_devices: \u001b[38;5;66;03m# pragma: no cover\u001b[39;00m\n\u001b[1;32m 165\u001b[0m config \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmake_config( \u001b[38;5;66;03m# generate config.\u001b[39;00m\n\u001b[1;32m 166\u001b[0m chip_layers_ordering\u001b[38;5;241m=\u001b[39mchip_layers_ordering,\n\u001b[1;32m 167\u001b[0m device\u001b[38;5;241m=\u001b[39mdevice,\n\u001b[1;32m 168\u001b[0m monitor_layers\u001b[38;5;241m=\u001b[39mmonitor_layers,\n\u001b[1;32m 169\u001b[0m config_modifier\u001b[38;5;241m=\u001b[39mconfig_modifier,\n\u001b[1;32m 170\u001b[0m )\n\u001b[0;32m--> 172\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msamna_device \u001b[38;5;241m=\u001b[39m open_device(device) \u001b[38;5;66;03m# apply configuration to device.\u001b[39;00m\n\u001b[1;32m 173\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msamna_device\u001b[38;5;241m.\u001b[39mget_model()\u001b[38;5;241m.\u001b[39mapply_configuration(config)\n\u001b[1;32m 174\u001b[0m time\u001b[38;5;241m.\u001b[39msleep(\u001b[38;5;241m1\u001b[39m)\n", - "File \u001b[0;32m~/Documents/github/sinabs/sinabs/backend/dynapcnn/io.py:256\u001b[0m, in \u001b[0;36mopen_device\u001b[0;34m(device_id)\u001b[0m\n\u001b[1;32m 254\u001b[0m device_map \u001b[38;5;241m=\u001b[39m get_device_map()\n\u001b[1;32m 255\u001b[0m device_info \u001b[38;5;241m=\u001b[39m device_map[device_id]\n\u001b[0;32m--> 256\u001b[0m device_handle \u001b[38;5;241m=\u001b[39m samna\u001b[38;5;241m.\u001b[39mdevice\u001b[38;5;241m.\u001b[39mopen_device(device_info)\n\u001b[1;32m 258\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m device_handle \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 259\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m device_handle\n", - "\u001b[0;31mRuntimeError\u001b[0m: Device is already opened!" - ] - } - ], - "source": [ - "hw_model.to(device=\"speck2fmodule:0\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The layers comprising our `hw_model` and their respective metadata can be inspected by calling `print` on a `DynapcnnNetworkGraph` instance." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "---- DynapcnnLayer 0 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 0): Conv2d(2, 4, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", - "(node 1): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(758.), min_v_mem=Parameter containing:\n", - "tensor(-119.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [1]\n", - "> assigned core: 0\n", - "\n", - "---- DynapcnnLayer 1 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 2): Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", - "(node 3): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1022.), min_v_mem=Parameter containing:\n", - "tensor(-160.), batch_size=1, num_timesteps=-1)\n", - "(node 4): SumPool2d(norm_type=1, kernel_size=2, stride=2, ceil_mode=False)\n", - "(node 5): SumPool2d(norm_type=1, kernel_size=5, stride=5, ceil_mode=False)\n", - "> layer destinations: [2, 3]\n", - "> assigned core: 1\n", - "\n", - "---- DynapcnnLayer 2 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 6): Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", - "(node 7): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1062.), min_v_mem=Parameter containing:\n", - "tensor(-166.), batch_size=1, num_timesteps=-1)\n", - "(node 8): SumPool2d(norm_type=1, kernel_size=2, stride=2, ceil_mode=False)\n", - "> layer destinations: [4]\n", - "> assigned core: 2\n", - "\n", - "---- DynapcnnLayer 3 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 12): Conv2d(4, 144, kernel_size=(6, 6), stride=(1, 1), bias=False)\n", - "(node 13): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(3048.), min_v_mem=Parameter containing:\n", - "tensor(-477.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [6]\n", - "> assigned core: 5\n", - "\n", - "---- DynapcnnLayer 4 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 9): Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", - "(node 10): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1027.), min_v_mem=Parameter containing:\n", - "tensor(-161.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [5]\n", - "> assigned core: 3\n", - "\n", - "---- DynapcnnLayer 5 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 16): Conv2d(4, 11, kernel_size=(6, 6), stride=(1, 1), bias=False)\n", - "(node 17): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(3054.), min_v_mem=Parameter containing:\n", - "tensor(-478.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: []\n", - "> assigned core: 4\n", - "\n", - "---- DynapcnnLayer 6 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 14): Conv2d(144, 144, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - "(node 15): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(3048.), min_v_mem=Parameter containing:\n", - "tensor(-477.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [5]\n", - "> assigned core: 6\n", - "\n", - "\n" - ] - } - ], - "source": [ - "print(hw_model)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "speck-rescnn", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/two_networks_merging_outputs.ipynb b/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/two_networks_merging_outputs.ipynb deleted file mode 100644 index 8f1e2be3..00000000 --- a/tests/test_nonsequential/DYNAPCNNNETWORK_EXAMPLES/two_networks_merging_outputs.ipynb +++ /dev/null @@ -1,378 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "import torch\n", - "import torch.nn as nn\n", - "from sinabs.backend.dynapcnn import DynapcnnNetworkGraph\n", - "from sinabs.layers import Merge, IAFSqueeze, SumPool2d\n", - "import sinabs.layers as sl\n", - "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "metadata": {} - }, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "torch.manual_seed(0)" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "channels = 2\n", - "height = 34\n", - "width = 34\n", - "\n", - "input_shape = (channels, height, width)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Network Module\n", - "\n", - "We need to define a `nn.Module` implementing the network we want the chip to reproduce." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "```mermaid\n", - "stateDiagram\n", - " [*] --> A\n", - " A --> B\n", - " B --> C\n", - " D --> E\n", - " E --> F\n", - " C --> G\n", - " F --> G\n", - " G --> H\n", - " H --> I\n", - " I --> [*]\n", - "```" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "class SNN(nn.Module):\n", - " def __init__(self, nb_classes, batch_size, surrogate_fn, min_v_mem=-0.313, spk_thr=2.0) -> None:\n", - " super().__init__()\n", - "\n", - " self.conv_A = nn.Conv2d(2, 4, 2, 1, bias=False)\n", - " self.iaf_A = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", - "\n", - " self.conv_B = nn.Conv2d(4, 4, 2, 1, bias=False)\n", - " self.iaf_B = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", - " self.pool_B = sl.SumPool2d(2,2)\n", - "\n", - " self.conv_C = nn.Conv2d(4, 4, 2, 1, bias=False)\n", - " self.iaf_C = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", - " self.pool_C = sl.SumPool2d(2,2)\n", - "\n", - " self.conv_D = nn.Conv2d(2, 4, 2, 1, bias=False)\n", - " self.iaf_D = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", - "\n", - " self.conv_E = nn.Conv2d(4, 4, 2, 1, bias=False)\n", - " self.iaf_E = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", - " self.pool_E = sl.SumPool2d(2,2)\n", - "\n", - " self.conv_F = nn.Conv2d(4, 4, 2, 1, bias=False)\n", - " self.iaf_F = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", - " self.pool_F = sl.SumPool2d(2,2)\n", - "\n", - " self.flat_brach1 = nn.Flatten()\n", - " self.flat_brach2 = nn.Flatten()\n", - " self.merge = Merge()\n", - "\n", - " self.fc1 = nn.Linear(196, 200, bias=False)\n", - " self.iaf1_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", - "\n", - " self.fc2 = nn.Linear(200, 200, bias=False)\n", - " self.iaf2_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", - "\n", - " self.fc3 = nn.Linear(200, nb_classes, bias=False)\n", - " self.iaf3_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr)\n", - "\n", - " def forward(self, x):\n", - " # conv 1 - A\n", - " conv_A_out = self.conv_A(x)\n", - " iaf_A_out = self.iaf_A(conv_A_out)\n", - " # conv 2 - B\n", - " conv_B_out = self.conv_B(iaf_A_out)\n", - " iaf_B_out = self.iaf_B(conv_B_out)\n", - " pool_B_out = self.pool_B(iaf_B_out)\n", - " # conv 3 - C\n", - " conv_C_out = self.conv_C(pool_B_out)\n", - " iaf_C_out = self.iaf_C(conv_C_out)\n", - " pool_C_out = self.pool_C(iaf_C_out)\n", - "\n", - " # ---\n", - "\n", - " # conv 4 - D\n", - " conv_D_out = self.conv_D(x)\n", - " iaf_D_out = self.iaf_D(conv_D_out)\n", - " # conv 5 - E\n", - " conv_E_out = self.conv_E(iaf_D_out)\n", - " iaf_E_out = self.iaf_E(conv_E_out)\n", - " pool_E_out = self.pool_E(iaf_E_out)\n", - " # conv 6 - F\n", - " conv_F_out = self.conv_F(pool_E_out)\n", - " iaf_F_out = self.iaf_F(conv_F_out)\n", - " pool_F_out = self.pool_F(iaf_F_out)\n", - "\n", - " # ---\n", - "\n", - " flat_brach1_out = self.flat_brach1(pool_C_out)\n", - " flat_brach2_out = self.flat_brach2(pool_F_out)\n", - " merge_out = self.merge(flat_brach1_out, flat_brach2_out)\n", - "\n", - " # FC 7 - G\n", - " fc1_out = self.fc1(merge_out)\n", - " iaf1_fc_out = self.iaf1_fc(fc1_out)\n", - " # FC 8 - H\n", - " fc2_out = self.fc2(iaf1_fc_out)\n", - " iaf2_fc_out = self.iaf2_fc(fc2_out)\n", - " # FC 9 - I\n", - " fc3_out = self.fc3(iaf2_fc_out)\n", - " iaf3_fc_out = self.iaf3_fc(fc3_out)\n", - "\n", - " return iaf3_fc_out\n", - " \n", - "snn = SNN(11, 1, PeriodicExponential())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## DynapcnnNetwork Class\n", - "\n", - "In the constructor of `DynapcnnNetworkGraph` the SNN passed as argument (defined as a `nn.Module`) will be parsed such that each layer is represented in a computational graph (using `nirtorch.extract_torch_graph`). \n", - "\n", - "The layers are the `nodes` of the graph, while their connectivity (how the outputs from a layer are sent to other layers) is represented as `edges`, represented in a `list` of `tuples`.\n", - "\n", - "Once the constructor finishes its initialization, the `hw_model.dynapcnn_layers` property is a dictionary where each entry represents the ID of a `DynapcnnLayer` instance (an `int` from `0` to `L`), with this entry containing a `DynapcnnLayer` instance where a subset of the layers in the original SNN has been incorporated into, the core such instance has been assigned to, and the list of `DynapcnnLayer` instances (their IDs) the layer targets." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "hw_model = DynapcnnNetworkGraph(\n", - " snn,\n", - " discretize=True,\n", - " input_shape=input_shape\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The `hw_model.to()` call will figure out into which core eac `DynapcnnLayer` instance will be assigned to. Once this assingment is made the instance itself is used to configure the `CNNLayerConfig` instance representing the core's configuration.\n", - "\n", - "If the cores' configuration is valid, each `DynapcnnLayer` instance and their respective destinations will be used to create a computational graph that encodes how the `forward` method of `hw_model.network` (a `nn.Module` using the `DynapcnnLayer` instances) propagates that through the network." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "metadata": {} - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Network is valid\n" - ] - }, - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "hw_model.to(device=\"speck2fmodule:0\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The layers comprising our `hw_model` and their respective metadata can be inspected by calling `print` on a `DynapcnnNetworkGraph` instance." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "metadata": {} - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "---- DynapcnnLayer 0 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 0): Conv2d(2, 4, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", - "(node 1): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(758.), min_v_mem=Parameter containing:\n", - "tensor(-119.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [1]\n", - "> assigned core: 0\n", - "\n", - "---- DynapcnnLayer 1 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 2): Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", - "(node 3): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1022.), min_v_mem=Parameter containing:\n", - "tensor(-160.), batch_size=1, num_timesteps=-1)\n", - "(node 4): SumPool2d(norm_type=1, kernel_size=2, stride=2, ceil_mode=False)\n", - "> layer destinations: [2]\n", - "> assigned core: 1\n", - "\n", - "---- DynapcnnLayer 2 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 5): Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", - "(node 6): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1062.), min_v_mem=Parameter containing:\n", - "tensor(-166.), batch_size=1, num_timesteps=-1)\n", - "(node 7): SumPool2d(norm_type=1, kernel_size=2, stride=2, ceil_mode=False)\n", - "> layer destinations: [3]\n", - "> assigned core: 2\n", - "\n", - "---- DynapcnnLayer 3 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 17): Conv2d(4, 200, kernel_size=(7, 7), stride=(1, 1), bias=False)\n", - "(node 18): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(3556.), min_v_mem=Parameter containing:\n", - "tensor(-557.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [7]\n", - "> assigned core: 5\n", - "\n", - "---- DynapcnnLayer 4 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 8): Conv2d(2, 4, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", - "(node 9): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(796.), min_v_mem=Parameter containing:\n", - "tensor(-125.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [5]\n", - "> assigned core: 3\n", - "\n", - "---- DynapcnnLayer 5 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 10): Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", - "(node 11): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1027.), min_v_mem=Parameter containing:\n", - "tensor(-161.), batch_size=1, num_timesteps=-1)\n", - "(node 12): SumPool2d(norm_type=1, kernel_size=2, stride=2, ceil_mode=False)\n", - "> layer destinations: [6]\n", - "> assigned core: 4\n", - "\n", - "---- DynapcnnLayer 6 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 13): Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", - "(node 14): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1033.), min_v_mem=Parameter containing:\n", - "tensor(-162.), batch_size=1, num_timesteps=-1)\n", - "(node 15): SumPool2d(norm_type=1, kernel_size=2, stride=2, ceil_mode=False)\n", - "> layer destinations: [3]\n", - "> assigned core: 8\n", - "\n", - "---- DynapcnnLayer 7 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 19): Conv2d(200, 200, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - "(node 20): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(3592.), min_v_mem=Parameter containing:\n", - "tensor(-562.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [8]\n", - "> assigned core: 6\n", - "\n", - "---- DynapcnnLayer 8 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 21): Conv2d(200, 11, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - "(node 22): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(3592.), min_v_mem=Parameter containing:\n", - "tensor(-562.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: []\n", - "> assigned core: 7\n", - "\n", - "\n" - ] - } - ], - "source": [ - "print(hw_model)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "speck-rescnn", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/tests/test_nonsequential/DynapcnnNetwork-example_1.ipynb b/tests/test_nonsequential/DynapcnnNetwork-example_1.ipynb deleted file mode 100644 index a294325f..00000000 --- a/tests/test_nonsequential/DynapcnnNetwork-example_1.ipynb +++ /dev/null @@ -1,679 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [], - "source": [ - "import torch\n", - "import torch.nn as nn\n", - "from sinabs.backend.dynapcnn import DynapcnnNetworkGraph\n", - "from sinabs.layers import Merge, IAFSqueeze\n", - "import matplotlib.pyplot as plt" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "torch.manual_seed(0)" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "channels = 2\n", - "height = 34\n", - "width = 34\n", - "\n", - "input_shape = (channels, height, width)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Network Module\n", - "\n", - "We need to define a `nn.Module` implementing the network we want the chip to reproduce." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "class SNN(nn.Module):\n", - " def __init__(self) -> None:\n", - " super().__init__()\n", - "\n", - " self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False) # node 0\n", - " self.iaf1 = IAFSqueeze(batch_size=1) # node 1\n", - " self.pool1 = nn.AvgPool2d(3,3) # node 2\n", - " self.pool1a = nn.AvgPool2d(4,4) # node 3\n", - "\n", - " self.conv2 = nn.Conv2d(10, 10, 4, 1, bias=False)# node 4\n", - " self.iaf2 = IAFSqueeze(batch_size=1) # node 6\n", - " # self.pool2 = nn.AvgPool2d(3,3) # node 7\n", - "\n", - " self.conv3 = nn.Conv2d(10, 1, 2, 1, bias=False) # node 8\n", - " self.iaf3 = IAFSqueeze(batch_size=1) # node 9\n", - "\n", - " self.flat = nn.Flatten()\n", - "\n", - " self.fc1 = nn.Linear(49, 500, bias=False) # node 10\n", - " self.iaf4 = IAFSqueeze(batch_size=1) # node 11\n", - " \n", - " self.fc2 = nn.Linear(500, 10, bias=False) # node 12\n", - " self.iaf5 = IAFSqueeze(batch_size=1) # node 13\n", - "\n", - " self.adder = Merge()\n", - "\n", - " def forward(self, x):\n", - " \n", - " con1_out = self.conv1(x)\n", - " iaf1_out = self.iaf1(con1_out)\n", - " pool1_out = self.pool1(iaf1_out)\n", - " pool1a_out = self.pool1a(iaf1_out)\n", - "\n", - " conv2_out = self.conv2(pool1_out)\n", - " iaf2_out = self.iaf2(conv2_out)\n", - " # pool2_out = self.pool2(iaf2_out)\n", - "\n", - " conv3_out = self.conv3(self.adder(pool1a_out, iaf2_out))\n", - " iaf3_out = self.iaf3(conv3_out)\n", - "\n", - " flat_out = self.flat(iaf3_out)\n", - " \n", - " fc1_out = self.fc1(flat_out)\n", - " iaf4_out = self.iaf4(fc1_out)\n", - " fc2_out = self.fc2(iaf4_out)\n", - " iaf5_out = self.iaf5(fc2_out)\n", - "\n", - " return iaf5_out" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "snn = SNN()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The following exemplifies how each of the layers in the SNN should be grouped together to form a `DynapcnnLayer` instance. Let's check the shapes of the tensors each of the (original) layers in the model inputs and outputs by feeding a fake input through the model:" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "DynapcnnLayer 0: ... [1, 2, 34, 34]\n", - " conv1: [1, 10, 33, 33]\n", - " iaf1: [1, 10, 33, 33]\n", - " pool1: [1, 10, 11, 11]\n", - " pool1a: [1, 10, 8, 8]\n", - "\n", - "DynapcnnLayer 1: ... [1, 10, 11, 11]\n", - " conv2: [1, 10, 8, 8]\n", - " iaf2: [1, 10, 8, 8]\n", - "\n", - "DynapcnnLayer 2: ... [1, 10, 8, 8] [ Merge(pool1a, iaf2_out) ]\n", - " conv3: [1, 1, 7, 7]\n", - " iaf3: [1, 1, 7, 7]\n", - "\n", - "DynapcnnLayer 3: ... [1, 49]\n", - " fc1: [1, 500]\n", - " iaf4: [1, 500]\n", - "\n", - "DynapcnnLayer 4: ... [1, 500]\n", - " fc2: [1, 10]\n", - " iaf5: [1, 10]\n", - "\n" - ] - } - ], - "source": [ - "x = torch.randn((1, *input_shape))\n", - "\n", - "print(f'DynapcnnLayer 0: ... {list(x.shape)}')\n", - "con1_out = snn.conv1(x)\n", - "print(f' conv1: {list(con1_out.shape)}')\n", - "iaf1_out = snn.iaf1(con1_out)\n", - "print(f' iaf1: {list(iaf1_out.shape)}')\n", - "pool1_out = snn.pool1(iaf1_out)\n", - "print(f' pool1: {list(pool1_out.shape)}')\n", - "pool1a_out = snn.pool1a(iaf1_out)\n", - "print(f' pool1a: {list(pool1a_out.shape)}\\n')\n", - "\n", - "print(f'DynapcnnLayer 1: ... {list(pool1_out.shape)}')\n", - "conv2_out = snn.conv2(pool1_out)\n", - "print(f' conv2: {list(conv2_out.shape)}')\n", - "iaf2_out = snn.iaf2(conv2_out)\n", - "print(f' iaf2: {list(iaf2_out.shape)}\\n')\n", - "# pool2_out = snn.pool2(iaf2_out)\n", - "# print(f' pool2: {list(pool2_out.shape)}\\n')\n", - "\n", - "added = snn.adder(pool1a_out, iaf2_out)\n", - "\n", - "print(f'DynapcnnLayer 2: ... {list(added.shape)} [ Merge(pool1a, iaf2_out) ]')\n", - "conv3_out = snn.conv3(added)\n", - "print(f' conv3: {list(conv3_out.shape)}')\n", - "iaf3_out = snn.iaf3(conv3_out)\n", - "print(f' iaf3: {list(iaf3_out.shape)}\\n')\n", - "\n", - "flat_out = snn.flat(iaf3_out)\n", - "\n", - "print(f'DynapcnnLayer 3: ... {list(flat_out.shape)}')\n", - "fc1_out = snn.fc1(flat_out)\n", - "print(f' fc1: {list(fc1_out.shape)}')\n", - "iaf4_out = snn.iaf4(fc1_out)\n", - "print(f' iaf4: {list(iaf4_out.shape)}\\n')\n", - "\n", - "\n", - "print(f'DynapcnnLayer 4: ... {list(iaf4_out.shape)}')\n", - "fc2_out = snn.fc2(iaf4_out)\n", - "print(f' fc2: {list(fc2_out.shape)}')\n", - "iaf5_out = snn.iaf5(fc2_out)\n", - "print(f' iaf5: {list(iaf5_out.shape)}\\n')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## DynapcnnNetwork Class\n", - "\n", - "In the constructor of `DynapcnnNetworkGraph` the SNN passed as argument (defined as a `nn.Module`) will be parsed such that each layer is represented in a computational graph (using `nirtorch.extract_torch_graph`). \n", - "\n", - "The layers are the `nodes` of the graph, while their connectivity (how the outputs from a layer are sent to other layers) is represented as `edges`, represented in a `list` of `tuples`.\n", - "\n", - "Once the constructor finishes its initialization, the `hw_model.dynapcnn_layers` property is a dictionary where each entry represents the ID of a `DynapcnnLayer` instance (an `int` from `0` to `L`), with this entry containing a `DynapcnnLayer` instance where a subset of the layers in the original SNN has been incorporated into, the core such instance has been assigned to, and the list of `DynapcnnLayer` instances (their IDs) the layer targets." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "hw_model = DynapcnnNetworkGraph(\n", - " snn,\n", - " discretize=True,\n", - " input_shape=input_shape\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The `hw_model.to()` call will figure out into which core eac `DynapcnnLayer` instance will be assigned to. Once this assingment is made the instance itself is used to configure the `CNNLayerConfig` instance representing the core's configuration.\n", - "\n", - "If the cores' configuration is valid, each `DynapcnnLayer` instance and their respective destinations will be used to create a computational graph that encodes how the `forward` method of `hw_model.network` (a `nn.Module` using the `DynapcnnLayer` instances) propagates that through the network." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Network is valid\n" - ] - }, - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "hw_model.to(device=\"speck2fmodule:0\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The layers comprising our `hw_model` and their respective metadata can be inspected by calling `print` on a `DynapcnnNetworkGraph` instance." - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "---- DynapcnnLayer 0 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 0): Conv2d(2, 10, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", - "(node 1): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "(node 2): SumPool2d(norm_type=1, kernel_size=(3, 3), stride=None, ceil_mode=False)\n", - "(node 3): SumPool2d(norm_type=1, kernel_size=(4, 4), stride=None, ceil_mode=False)\n", - "> layer destinations: [1, 2]\n", - "> assigned core: 0\n", - "\n", - "---- DynapcnnLayer 1 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 4): Conv2d(10, 10, kernel_size=(4, 4), stride=(1, 1), bias=False)\n", - "(node 6): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [2]\n", - "> assigned core: 1\n", - "\n", - "---- DynapcnnLayer 2 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 7): Conv2d(10, 1, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", - "(node 8): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [3]\n", - "> assigned core: 2\n", - "\n", - "---- DynapcnnLayer 3 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 9): Conv2d(1, 500, kernel_size=(7, 7), stride=(1, 1), bias=False)\n", - "(node 10): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: [4]\n", - "> assigned core: 3\n", - "\n", - "---- DynapcnnLayer 4 ----------------------------------------------------------\n", - "> layer modules: \n", - "(node 11): Conv2d(500, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - "(node 12): IAFSqueeze(spike_threshold=Parameter containing:\n", - "tensor(1.), min_v_mem=Parameter containing:\n", - "tensor(-32768.), batch_size=1, num_timesteps=-1)\n", - "> layer destinations: []\n", - "> assigned core: 4\n", - "\n", - "\n" - ] - } - ], - "source": [ - "print(hw_model)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Training our DynapcnnNetwork\n", - "\n", - "Preparing the data..." - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "# https://synsense.gitlab.io/sinabs-dynapcnn/getting_started/notebooks/nmnist_quick_start.html\n", - "\n", - "from tonic.datasets.nmnist import NMNIST\n", - "from tonic.transforms import ToFrame\n", - "from torch.utils.data import DataLoader\n", - "from torch.nn import CrossEntropyLoss\n", - "from torch.optim import SGD\n", - "from tqdm.notebook import tqdm\n", - " \n", - "# download dataset\n", - "root_dir = \"./NMNIST\"\n", - "_ = NMNIST(save_to=root_dir, train=True)\n", - "_ = NMNIST(save_to=root_dir, train=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "type of data is: \n", - "(4686,)\n", - "time length of sample data is: 300760 micro seconds\n", - "there are 4686 events in the sample data\n", - "the label of the sample data is: 5\n" - ] - } - ], - "source": [ - "sample_data, label = NMNIST(save_to=root_dir, train=False)[0]\n", - "\n", - "print(f\"type of data is: {type(sample_data)}\")\n", - "print(sample_data.shape)\n", - "print(f\"time length of sample data is: {sample_data['t'][-1] - sample_data['t'][0]} micro seconds\")\n", - "print(f\"there are {len(sample_data)} events in the sample data\")\n", - "print(f\"the label of the sample data is: {label}\")" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The transformed array is in shape [Time-Step, Channel, Height, Width] --> (50, 2, 34, 34)\n" - ] - } - ], - "source": [ - "n_time_steps = 50\n", - "to_raster = ToFrame(sensor_size=NMNIST.sensor_size, n_time_bins=n_time_steps)\n", - "\n", - "snn_train_dataset = NMNIST(save_to=root_dir, train=True, transform=to_raster)\n", - "snn_test_dataset = NMNIST(save_to=root_dir, train=False, transform=to_raster)\n", - "\n", - "# check the transformed data\n", - "sample_data, label = snn_train_dataset[0]\n", - "print(f\"The transformed array is in shape [Time-Step, Channel, Height, Width] --> {sample_data.shape}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Setting the training hyperparameters..." - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "device: NVIDIA GeForce RTX 3070 Ti\n" - ] - } - ], - "source": [ - "if torch.cuda.is_available():\n", - " device = torch.device('cuda:0')\n", - " print('device: ', torch.cuda.get_device_name(0))\n", - "else:\n", - " device = torch.device('cpu')" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [], - "source": [ - "epochs = 1\n", - "lr = 1e-4\n", - "batch_size = 64\n", - "num_workers = 4\n", - "shuffle = True" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [], - "source": [ - "snn_train_dataloader = DataLoader(snn_train_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True)\n", - "snn_test_dataloader = DataLoader(snn_test_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=False)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Initializing our weights..." - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [], - "source": [ - "hw_cnn = hw_model.network" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [], - "source": [ - "hw_cnn.to(device)" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [], - "source": [ - "hw_cnn.init_weights()" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [], - "source": [ - "optimizer = SGD(params=hw_cnn.parameters(), lr=lr)\n", - "criterion = CrossEntropyLoss()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The training loop..." - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [], - "source": [ - "losses = []\n", - "batches = []\n", - "batch_count = 0" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "0a2474dc5d01479ead00063a63022136", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - " 0%| | 0/937 [00:00" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "plt.plot(batches, losses)\n", - "plt.ylabel('loss')\n", - "plt.xlabel('batches')\n", - "plt.show()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "speck-rescnn", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/tests/test_nonsequential/NNI-test/main.ipynb b/tests/test_nonsequential/NNI-test/main.ipynb deleted file mode 100644 index 553c1170..00000000 --- a/tests/test_nonsequential/NNI-test/main.ipynb +++ /dev/null @@ -1,136 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "from nni.experiment import Experiment\n", - "experiment = Experiment('local')" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "search_space = {\n", - " 'lr': {'_type': 'loguniform', '_value': [0.001, 0.0001]},\n", - "}" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "experiment.config.trial_command = 'python model.py'\n", - "experiment.config.trial_code_directory = '.'" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "experiment.config.search_space = search_space" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "experiment.config.tuner.name = 'TPE'\n", - "experiment.config.tuner.class_args['optimize_mode'] = 'maximize'" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "experiment.config.max_trial_number = 5\n", - "experiment.config.trial_concurrency = 2" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[2024-04-26 17:05:41] \u001b[32mCreating experiment, Experiment ID: \u001b[36mda0h79xv\u001b[0m\n", - "[2024-04-26 17:05:41] \u001b[32mStarting web server...\u001b[0m\n", - "[2024-04-26 17:05:42] \u001b[32mSetting up...\u001b[0m\n", - "[2024-04-26 17:05:42] \u001b[32mWeb portal URLs: \u001b[36mhttp://127.0.0.1:8080 http://192.168.11.68:8080 http://172.17.0.1:8080\u001b[0m\n" - ] - }, - { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "experiment.run(8080)" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[2024-04-26 17:05:52] \u001b[32mStopping experiment, please wait...\u001b[0m\n", - "[2024-04-26 17:05:52] \u001b[32mSaving experiment checkpoint...\u001b[0m\n", - "[2024-04-26 17:05:52] \u001b[32mStopping NNI manager, if any...\u001b[0m\n", - "[2024-04-26 17:05:54] \u001b[32mExperiment stopped.\u001b[0m\n" - ] - } - ], - "source": [ - "#input('Press enter to quit')\n", - "experiment.stop()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "speck-rescnn", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/tests/test_nonsequential/NNI-test/model.py b/tests/test_nonsequential/NNI-test/model.py deleted file mode 100644 index 2ba97930..00000000 --- a/tests/test_nonsequential/NNI-test/model.py +++ /dev/null @@ -1,175 +0,0 @@ -import torch -import torch.nn as nn -import sinabs.layers as sl -import nni - -from tonic.datasets.nmnist import NMNIST -from tonic.transforms import ToFrame -from torch.utils.data import DataLoader -from torch.nn import CrossEntropyLoss -from torch.optim import SGD - -params = { - 'lr': 0.001, -} - -optimized_params = nni.get_next_parameter() -params.update(optimized_params) -print(params) - -###### Loading Data ###### - -batch_size = 32 -num_workers = 4 -epochs = 1 - -root_dir = "./NMNIST" -_ = NMNIST(save_to=root_dir, train=True) -_ = NMNIST(save_to=root_dir, train=False) - -n_time_steps = 50 -to_raster = ToFrame(sensor_size=NMNIST.sensor_size, n_time_bins=n_time_steps) - -snn_train_dataset = NMNIST(save_to=root_dir, train=True, transform=to_raster) -snn_test_dataset = NMNIST(save_to=root_dir, train=False, transform=to_raster) - -snn_train_dataloader = DataLoader(snn_train_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True) -snn_test_dataloader = DataLoader(snn_test_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=False) - -###### Defining the Model ###### - -if torch.cuda.is_available(): - device = torch.device('cuda:0') - print('device: ', torch.cuda.get_device_name(0)) -else: - device = torch.device('cpu') - -class SNN(nn.Module): - def __init__(self) -> None: - super().__init__() - - self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False) - self.iaf1 = sl.IAFSqueeze(batch_size=1) - self.pool1 = sl.SumPool2d(3,3) - self.pool1a = sl.SumPool2d(4,4) - - self.conv2 = nn.Conv2d(10, 10, 4, 1, bias=False) - self.iaf2 = sl.IAFSqueeze(batch_size=1) - - self.conv3 = nn.Conv2d(10, 1, 2, 1, bias=False) - self.iaf3 = sl.IAFSqueeze(batch_size=1) - - self.flat = nn.Flatten() - - self.fc1 = nn.Linear(49, 100, bias=False) - self.iaf4 = sl.IAFSqueeze(batch_size=1) - - self.fc2 = nn.Linear(100, 10, bias=False) - self.iaf5 = sl.IAFSqueeze(batch_size=1) - - - def detach_neuron_states(self): - for name, layer in self.named_modules(): - if name != '': - if isinstance(layer, sl.StatefulLayer): - for name, buffer in layer.named_buffers(): - buffer.detach_() - - def init_weights(self): - for name, layer in self.named_modules(): - if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear): - nn.init.xavier_normal_(layer.weight.data) - - def forward(self, x): - - con1_out = self.conv1(x) - iaf1_out = self.iaf1(con1_out) - pool1_out = self.pool1(iaf1_out) - - conv2_out = self.conv2(pool1_out) - iaf2_out = self.iaf2(conv2_out) - - conv3_out = self.conv3(iaf2_out) - iaf3_out = self.iaf3(conv3_out) - - flat_out = self.flat(iaf3_out) - - fc1_out = self.fc1(flat_out) - iaf4_out = self.iaf4(fc1_out) - fc2_out = self.fc2(iaf4_out) - iaf5_out = self.iaf5(fc2_out) - - return iaf5_out - -snn = SNN().to(device) - -snn.init_weights() - -optimizer = SGD(snn.parameters(), lr=params['lr']) -loss_fn = CrossEntropyLoss() - -###### Defining Train/Test ###### - -def train(dataloader, model, loss_fn, optimizer): - size = len(dataloader.dataset) - model.train() - for batch, (X, y) in enumerate(dataloader): - # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width] - X = X.reshape(-1, 2, 34, 34).to(dtype=torch.float, device=device) - y = y.to(dtype=torch.long, device=device) - - # forward - pred = model(X) - - # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes] - pred = pred.reshape(batch_size, n_time_steps, -1) - - # accumulate all time-steps output for final prediction - pred = pred.sum(dim = 1) - loss = loss_fn(pred, y) - - # gradient update - optimizer.zero_grad() - loss.backward() - optimizer.step() - - # detach the neuron states and activations from current computation graph(necessary) - model.detach_neuron_states() - - break - -def test(dataloader, model): - correct_predictions = [] - with torch.no_grad(): - for X, y in dataloader: - # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width] - X = X.reshape(-1, 2, 34, 34).to(dtype=torch.float, device=device) - y = y.to(dtype=torch.long, device=device) - - # forward - output = model(X) - - # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes] - output = output.reshape(batch_size, n_time_steps, -1) - - # accumulate all time-steps output for final prediction - output = output.sum(dim=1) - - # calculate accuracy - pred = output.argmax(dim=1, keepdim=True) - - # compute the total correct predictions - correct_predictions.append(pred.eq(y.view_as(pred))) - - break - - return correct_predictions.sum().item()/(len(correct_predictions))*100 - -###### Training loop (HPO) ###### - -for t in range(epochs): - print(f"Epoch {t+1}\n-------------------------------") - train(snn_train_dataloader, snn, loss_fn, optimizer) - accuracy = test(snn_test_dataloader, snn) - nni.report_intermediate_result(accuracy) -nni.report_final_result(accuracy) \ No newline at end of file diff --git a/tests/test_nonsequential/baseline-SCNN-example_1-NNI.ipynb b/tests/test_nonsequential/baseline-SCNN-example_1-NNI.ipynb deleted file mode 100644 index 90889ea7..00000000 --- a/tests/test_nonsequential/baseline-SCNN-example_1-NNI.ipynb +++ /dev/null @@ -1,613 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [], - "source": [ - "import torch\n", - "import torch.nn as nn\n", - "import sinabs.layers as sl\n", - "from tqdm.notebook import tqdm\n", - "\n", - "from tonic.datasets.nmnist import NMNIST\n", - "from tonic.transforms import ToFrame\n", - "from torch.utils.data import DataLoader\n", - "from torch.nn import CrossEntropyLoss\n", - "from torch.optim import Adam\n", - "\n", - "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential\n", - "\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "torch.manual_seed(0)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Loading Data" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "batch_size = 64\n", - "num_workers = 4\n", - "epochs = 5\n", - "lr = 1e-3" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "root_dir = \"./NMNIST\"\n", - "_ = NMNIST(save_to=root_dir, train=True)\n", - "_ = NMNIST(save_to=root_dir, train=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "n_time_steps = 50\n", - "to_raster = ToFrame(sensor_size=NMNIST.sensor_size, n_time_bins=n_time_steps)\n", - "\n", - "snn_train_dataset = NMNIST(save_to=root_dir, train=True, transform=to_raster)\n", - "snn_test_dataset = NMNIST(save_to=root_dir, train=False, transform=to_raster)" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "snn_train_dataloader = DataLoader(snn_train_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True)\n", - "snn_test_dataloader = DataLoader(snn_test_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=False)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Network Module\n", - "\n", - "We need to define a `nn.Module` implementing the network we want the chip to reproduce." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "device: NVIDIA GeForce RTX 3070 Ti\n" - ] - } - ], - "source": [ - "if torch.cuda.is_available():\n", - " device = torch.device('cuda:0')\n", - " print('device: ', torch.cuda.get_device_name(0))\n", - "else:\n", - " device = torch.device('cpu')" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "class SNN(nn.Module):\n", - " def __init__(self) -> None:\n", - " super().__init__()\n", - "\n", - " self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False)\n", - " self.iaf1 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - " self.pool1 = nn.AvgPool2d(2,2)\n", - " self.pool1a = nn.AvgPool2d(6,6)\n", - "\n", - " self.conv2 = nn.Conv2d(10, 10, 2, 1, bias=False)\n", - " self.iaf2 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - " self.pool2 = nn.AvgPool2d(3,3)\n", - "\n", - " self.conv3 = nn.Conv2d(10, 10, 3, 1, bias=False)\n", - " self.iaf3 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - " self.pool3 = nn.AvgPool2d(2,2)\n", - "\n", - " self.flat = nn.Flatten()\n", - "\n", - " self.fc1 = nn.Linear(10, 10, bias=False)\n", - " self.iaf4 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - "\n", - " def detach_neuron_states(self):\n", - " for name, layer in self.named_modules():\n", - " if name != '':\n", - " if isinstance(layer, sl.StatefulLayer):\n", - " for name, buffer in layer.named_buffers():\n", - " buffer.detach_()\n", - "\n", - " def init_weights(self):\n", - " for name, layer in self.named_modules():\n", - " if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear):\n", - " nn.init.xavier_normal_(layer.weight.data)\n", - "\n", - " def forward(self, x):\n", - " \n", - " con1_out = self.conv1(x)\n", - " iaf1_out = self.iaf1(con1_out)\n", - " pool1_out = self.pool1(iaf1_out)\n", - "\n", - " conv2_out = self.conv2(pool1_out)\n", - " iaf2_out = self.iaf2(conv2_out)\n", - " pool2_out = self.pool2(iaf2_out)\n", - "\n", - " conv3_out = self.conv3(pool2_out)\n", - " iaf3_out = self.iaf3(conv3_out)\n", - " pool3_out = self.pool3(iaf3_out)\n", - "\n", - " flat_out = self.flat(pool3_out)\n", - " \n", - " fc1_out = self.fc1(flat_out)\n", - " iaf4_out = self.iaf4(fc1_out)\n", - "\n", - " return iaf4_out" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "snn = SNN().to(device)" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "snn.init_weights()" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "optimizer = Adam(snn.parameters(), lr=lr, betas=(0.9, 0.999), eps=1e-8)\n", - "loss_fn = CrossEntropyLoss()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Define train and test" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "def train(dataloader, model, loss_fn, optimizer, epochs, test_func, dataloader_test):\n", - " epochs_y = []\n", - " epochs_x = []\n", - " epochs_acc = []\n", - " model.train()\n", - "\n", - " for e in range(epochs):\n", - " losses = []\n", - " batches = []\n", - " batch_count = 0\n", - " train_p_bar = tqdm(snn_train_dataloader)\n", - "\n", - " for X, y in train_p_bar:\n", - " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", - " X = X.reshape(-1, 2, 34, 34).to(dtype=torch.float, device=device)\n", - " y = y.to(dtype=torch.long, device=device)\n", - "\n", - " # forward\n", - " pred = model(X)\n", - "\n", - " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", - " pred = pred.reshape(batch_size, n_time_steps, -1)\n", - "\n", - " # accumulate all time-steps output for final prediction\n", - " pred = pred.sum(dim = 1)\n", - " loss = loss_fn(pred, y)\n", - "\n", - " # gradient update\n", - " optimizer.zero_grad()\n", - " loss.backward()\n", - " optimizer.step()\n", - "\n", - " # detach the neuron states and activations from current computation graph(necessary)\n", - " model.detach_neuron_states()\n", - "\n", - " train_p_bar.set_description(f\"Epoch {e} - BPTT Training Loss: {round(loss.item(), 4)}\")\n", - "\n", - " batch_count += 1\n", - " losses.append(loss.item())\n", - " batches.append(batch_count)\n", - "\n", - " epochs_y.append(losses)\n", - " epochs_x.append(batches)\n", - "\n", - " acc = test_func(dataloader_test, model)\n", - " print(f'Epoch {e} accuracy: {acc}')\n", - " epochs_acc.append(acc)\n", - "\n", - " return epochs_x, epochs_y, epochs_acc\n" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "def test(dataloader, model):\n", - " correct_predictions = []\n", - " with torch.no_grad():\n", - " test_p_bar = tqdm(dataloader)\n", - " for X, y in test_p_bar:\n", - " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", - " X = X.reshape(-1, 2, 34, 34).to(dtype=torch.float, device=device)\n", - " y = y.to(dtype=torch.long, device=device)\n", - "\n", - " # forward\n", - " output = model(X)\n", - "\n", - " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", - " output = output.reshape(batch_size, n_time_steps, -1)\n", - "\n", - " # accumulate all time-steps output for final prediction\n", - " output = output.sum(dim=1)\n", - "\n", - " # calculate accuracy\n", - " pred = output.argmax(dim=1, keepdim=True)\n", - "\n", - " # compute the total correct predictions\n", - " correct_predictions.append(pred.eq(y.view_as(pred)))\n", - "\n", - " test_p_bar.set_description(f\"Testing Model...\")\n", - " \n", - " correct_predictions = torch.cat(correct_predictions)\n", - " return correct_predictions.sum().item()/(len(correct_predictions))*100" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Training loop (HPO)" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "0cd81220753e46039b5cac482961c5c5", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - " 0%| | 0/937 [00:00" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "plt.plot(epochs_x[-1], epochs_y[-1])\n", - "plt.xlabel('batches')\n", - "plt.ylabel('loss')\n", - "plt.ylim(0,)\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAGwCAYAAABVdURTAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAABTz0lEQVR4nO3de1xUZf4H8M9cgOGOgCAoV/GOIIIXUPJOqy6b1W+lbNVKS9Ty1mVRy9Ta2NwurluY5qV1K6PS3Eq2pJSLYhkG3vMCCAiDCMhwkwFmzu8PZBS5yChwmOHzfr3mJZx5zpnv49mczz7nnOeRCIIggIiIiMhISMUugIiIiKg9MdwQERGRUWG4ISIiIqPCcENERERGheGGiIiIjArDDRERERkVhhsiIiIyKnKxC+hsWq0W+fn5sLa2hkQiEbscIiIiagNBEFBeXg5XV1dIpa2PzXS7cJOfnw83NzexyyAiIqJ7kJubiz59+rTaptuFG2trawD1fzk2NjYiV0NERERtUVZWBjc3N933eGu6XbhpuBRlY2PDcENERGRg2nJLCW8obkFSUhLCw8Ph6uoKiUSCffv2tXnfI0eOQC6XY9iwYU3eKy0txeLFi+Hi4gKFQoFBgwYhLi6u/QonIiLq5rrdyE1bVVZWwt/fH0899RQeffTRNu+nUqkwZ84cTJo0CVevXm30Xk1NDaZMmQInJyd89dVX6NOnD3Jzc9s0xEZERERtw3DTgqlTp2Lq1Kl677dgwQLMmjULMpmsyWjPjh07UFJSgpSUFJiYmAAAPDw82qNcIiIiuomXpdrRzp07kZGRgddee63Z97/55hsEBwdj8eLFcHZ2hq+vL958801oNJpOrpSIiMh4ceSmnVy8eBFRUVFITk6GXN78X2tmZiYOHjyIJ554AnFxcbh48SIWL16Muro6rFmzppMrJiIiMk4MN+1Ao9Fg1qxZWLduHfr3799iO61WCycnJ2zduhUymQyBgYHIz8/HP/7xD4YbIiKidsJw0w7Ky8uRmpqKtLQ0PPfccwDqg4wgCJDL5Thw4AAmTpwIFxcXmJiYQCaT6fYdNGgQCgoKUFNTA1NTU7G6QEREZDQYbtqBjY0NTp061WhbTEwMDh48iK+++gpeXl4AgDFjxuCzzz6DVqvVTR194cIFuLi4MNgQERG1E4abFlRUVODSpUu637OyspCeng57e3u4u7tj5cqVyMvLw65duyCVSuHr69tofycnJygUikbbFy5ciH/9619YunQpnn/+eVy8eBFvvvkmlixZ0mn9IiIiMnYMNy1ITU3FhAkTdL+vWLECADB37lx8/PHHUCqVyMnJabSPUnUDWUWV8HK0bPaYbm5uOHDgAJYvXw4/Pz/07t0bS5cuxV//+teO6wgREVE3IxEEQRC7iM5UVlYGW1tbqFSqdl1+IfbXHKzcewpaAZBKgOhHhiJihHu7HZ+IiKg70+f7m/PctAOl6gaibgYbANAKwKq9p6FU3RC3MCIiom6I4aYdZBVV4s7xL40g4HJRlTgFERERdWMMN+3Ay9ES0jsWKZVJAE9HC3EKIiIi6sYYbtqBi605oh8Z2ijgzBvrDRdbc/GKIiIi6qYYbtpJxAh3HImaiMmDnAAA5wrKRK6IiIioe2K4aUcutuZ4LXwIZFIJki8W4dQVldglERERdTsMN+3Mzd4Cf/J3BQBsTrx0l9ZERETU3hhuOsDC8X0BAP87XYCMaxUiV0NERNS9MNx0gP7O1pg8yBmCAHyYkCF2OURERN0Kw00HWTShfvTm67Q85JdyMj8iIqLOwnDTQYa798Bob3vUaQV8lJwpdjlERETdBsNNB1o03gcA8PmxXJRU1ohcDRERUffAcNOBQvs5YmhvW9yo1eDjI1lil0NERNQtiBpukpKSEB4eDldXV0gkEuzbt6/N+x45cgRyuRzDhg3rsPrul0QiwaKbT059nHIZFeo6kSsiIiIyfqKGm8rKSvj7++P999/Xaz+VSoU5c+Zg0qRJHVRZ+3lwSC9497REWXUdPv05W+xyiIiIjJ6o4Wbq1Kl444038Mgjj+i134IFCzBr1iwEBwffta1arUZZWVmjV2eSSiWIHFc/erPtcBaqazWd+vlERETdjcHdc7Nz505kZGTgtddea1P76Oho2Nra6l5ubm4dXGFTM4b1houtAtfK1djz25VO/3wiIqLuxKDCzcWLFxEVFYVPP/0Ucrm8TfusXLkSKpVK98rNze3gKpsylUvxTKg3AGBLYibqNNpOr4GIiKi7MJhwo9FoMGvWLKxbtw79+/dv835mZmawsbFp9BLDYyPd0MPCBDklVdh/SilKDURERN2BwYSb8vJypKam4rnnnoNcLodcLsf69etx4sQJyOVyHDx4UOwSW2VhKsfTY7wAAJsTMiAIgsgVERERGSeDCTc2NjY4deoU0tPTda/IyEgMGDAA6enpGDVqlNgl3tWcYE9Ymsrwe0E5Dv5eKHY5RERERqltN650kIqKCly6dEn3e1ZWFtLT02Fvbw93d3esXLkSeXl52LVrF6RSKXx9fRvt7+TkBIVC0WR7V2VrYYK/jPbAlqRMxCRkYOJAJ0gkErHLIiIiMiqijtykpqYiICAAAQEBAIAVK1YgICAAa9asAQAolUrk5OSIWWK7mzfWC6ZyKY5nX8exrBKxyyEiIjI6EqGb3fxRVlYGW1tbqFQq0W4uXvX1KXz2Sw7G9e+Jfz89UpQaiIiIDIk+398Gc8+NMVnwgDekEiDxwjWczlOJXQ4REZFRYbgRgYeDJcL9XQEAmxMzRK6GiIjIuDDciGThzQU1404pkXmtQuRqiIiIjAfDjUgG9rLBpIFOEIT6WYuJiIiofTDciGjRhPrRm71pV6BU3RC5GiIiIuPAcCOiQA97jPSyR61GwLbkLLHLISIiMgoMNyJbdPPem93HcnC9skbkaoiIiAwfw43IxvXviSGuNqiq0eDjlMtil0NERGTwGG5EJpFIsGi8DwDg45TLqFDXiVwRERGRYWO46QL+4NsL3o6WUN2oxe5fjGu5CSIios7GcNMFyKQSLBjnDQDYdjgT6jqNyBUREREZLoabLuLhgD7oZaPA1TI19v6WJ3Y5REREBovhposwlUsxP9QLALAlMQMabbdaz5SIiKjdMNx0IY+PdIedhQkuF1ch7pRS7HKIiIgMEsNNF2JpJsdTIfWjNzEJGRAEjt4QERHpi+Gmi5kb4gFLUxnOKcuQcP6a2OUQEREZHIabLsbOwhSzRrkDAGISLolcDRERkeFhuOmC5od6w1Qmxa+Xr+PXyyVil0NERGRQGG66IGcbBR4N7A0AiDnE0RsiIiJ9MNx0UQse6AupBDh0/hrO5peJXQ4REZHBYLjpojwdLTHdzxUAsDkxQ+RqiIiIDAfDTRe2cFxfAMD+k/m4XFQpcjVERESGgeGmCxvsaoMJA3pCKwBbkjh6Q0RE1BYMN13cogk+AIA9x/Nwtaxa5GqIiIi6PoabLm6Epz1GePZAjUaLbcmZYpdDRETU5THcGIBF4+tHbz79JQelVTUiV0NERNS1MdwYgPEDemKQiw2qajT4d0q22OUQERF1aQw3BkAikWDR+Ponp3amZKFSXSdyRURERF0Xw42BmDbUBZ4OFiitqsXuYzlil0NERNRlMdwYCJlUggU3573ZlpwFdZ1G5IqIiIi6JoYbA/LI8N5wtjFDQVk19qXliV0OERFRl8RwY0DM5DLMH+sNAPgwMRMarSByRURERF0Pw42BeXyUO2zNTZBVVInvTxeIXQ4REVGXw3BjYKzM5HgyxBMAEJNwCYLA0RsiIqLbMdwYoCdDPGFhKsOZ/DIkXSwSuxwiIqIuheHGAPWwNMXjI90BAB8cuiRyNURERF0Lw42Bmh/qBROZBMeySnA8u0TscoiIiLoMUcNNUlISwsPD4erqColEgn379rXafu/evZgyZQp69uwJGxsbBAcH44cffuicYrsYF1tzPBLQBwAQcyhD5GqIiIi6DlHDTWVlJfz9/fH++++3qX1SUhKmTJmCuLg4HD9+HBMmTEB4eDjS0tI6uNKuacE4b0gkwE+/F+L3gjKxyyEiIuoSJEIXedxGIpHg66+/xowZM/Tab8iQIYiIiMCaNWva1L6srAy2trZQqVSwsbG5h0q7lsWf/Yb9J5V4aJgr/vlYgNjlEBERdQh9vr8N+p4brVaL8vJy2Nvbt9hGrVajrKys0cuYLLy5JMO3J/KRU1wlcjVERETiM+hw884776CyshIzZ85ssU10dDRsbW11Lzc3t06ssOP59rbFuP49oRWAD5N47w0REZHBhpvdu3dj7dq1iI2NhZOTU4vtVq5cCZVKpXvl5uZ2YpWdY9H4+tGbr1KvoLCsWuRqiIiIxGWQ4SY2Nhbz5s3DF198gcmTJ7fa1szMDDY2No1exmaklz0CPXqgRqPF9sNZYpdDREQkKoMLN7t378aTTz6Jzz77DNOnTxe7nC5BIpHoRm8++TkbqqpakSsiIiISj6jhpqKiAunp6UhPTwcAZGVlIT09HTk5OQDqLynNmTNH13737t2YM2cO3nnnHYwePRoFBQUoKCiASqUSo/wuZeJAJwzsZY3KGg12Hb0sdjlERESiETXcpKamIiAgAAEB9Y8wr1ixAgEBAbrHupVKpS7oAMCWLVtQV1eHxYsXw8XFRfdaunSpKPV3JRKJBAtvjt7sTLmMqpo6kSsiIiISR5eZ56azGNs8N7er02gx8Z1E5JRUYc0fB+PpsV5il0RERNQuus08N9SYXCbFgnHeAICPkjNRU6cVuSIiIqLOx3BjZB4d3gc9rc2gVFVjX3qe2OUQERF1OoYbI6MwkWH+zctRHyZmQKPtVlcdiYiIGG6M0ROjPWCjkCPzWiUOnCkQuxwiIqJOxXBjhKzM5HgyxBMAEJOQgW52zzgREXVzDDdG6skxXjA3keFUngqHLxWJXQ4REVGnYbgxUvaWpnhsZP0ioR8cuiRyNURERJ2H4caIPRPqDROZBD9nluC3nOtil0NERNQpGG6MmKudOWYM6w0AiDmUIXI1REREnYPhxshFju8LiQT48dxVnC8oF7scIiKiDsdwY+T69rTCVN9eAOrnvSEiIjJ2DDfdwKLxPgCAb07kI7ekSuRqiIiIOhbDTTfg29sWof0codEK2JLE0RsiIjJuDDfdRMPozRepV1BYXi1yNURERB2H4aabGO1tjwB3O9TUabHj8GWxyyEiIuowDDfdhEQi0Y3efPJzNlQ3akWuiIiIqGMw3HQjkwY6YYCzNSrUdfjk52yxyyEiIuoQDDfdiFQqwcLxfQEAOw5n4UaNRuSKiIiI2h/DTTfzRz8XuNmbo7iyBrG/5ohdDhERUbtjuOlm5DIpnn2gfvTmo+Qs1Gq0IldERETUvhhuuqE/B/aBo5UZ8kpv4L/p+WKXQ0RE1K4YbrohhYkM88Z6AahfkkGrFUSuiIiIqP0w3HRTfxntDmuFHJcKK3Dg7FWxyyEiImo3DDfdlLXCBHODPQEAmxMuQRA4ekNERMaB4aYbe2qMJxQmUpy4okJKRrHY5RAREbULhptuzMHKDI+NcAcAfHDoksjVEBERtQ+Gm27umQe8IZdKkJJRjPTcUrHLISIium8MN91cbztzPDSsNwAghqM3RERkBBhuCAvHe0MiAQ6cvYqLV8vFLoeIiOi+MNwQfJys8eDgXgCAzYkZIldDRER0fxhuCACwaEL9kgz/Tc9HbkmVyNUQERHdO4YbAgD49bHDWB9HaLQCPkrOFLscIiKie8ZwQzqLxteP3sT+motr5WqRqyEiIro3DDekE9zXAf5udlDXabHzSJbY5RAREd0ThhvSkUgkutGb/xzNRll1rcgVERER6Y/hhhqZMsgZ/ZysUK6uwyc/Z4tdDhERkd4YbqgRqVSChTdHb3YczkJ1rUbkioiIiPTDcENNhPu7oredOYoqavBFaq7Y5RAREelF1HCTlJSE8PBwuLq6QiKRYN++fXfdJzExEYGBgVAoFPD29saHH37Y8YV2MyYyKRaM8wYAbEnMRK1GK3JFREREbSdquKmsrIS/vz/ef//9NrXPysrCtGnTEBoairS0NKxatQpLlizBnj17OrjS7mdmkBscrUyRV3oD357IF7scIiKiNpOL+eFTp07F1KlT29z+ww8/hLu7OzZu3AgAGDRoEFJTU/H222/j0UcfbXYftVoNtfrWnC1lZWX3VXN3oTCR4akxXvjHD+exOSEDM4b1hlQqEbssIiKiuzKoe26OHj2KsLCwRtsefPBBpKamora2+ceWo6OjYWtrq3u5ubl1RqlGYXawB6zN5LhYWIEfz10VuxwiIqI2MahwU1BQAGdn50bbnJ2dUVdXh6Kiomb3WblyJVQqle6Vm8sbZNvKRmGC2cEeAICYhAwIgiByRURERHdnUOEGqJ9o7nYNX7h3bm9gZmYGGxubRi9qu6fHesFMLkV6bimOZhaLXQ4REdFdGVS46dWrFwoKChptKywshFwuh4ODg0hVGTdHKzNEjKi/lBdzKEPkaoiIiO7OoMJNcHAw4uPjG207cOAAgoKCYGJiIlJVxu+ZUG/IpBIcvlSEk1dKxS6HiIioVaKGm4qKCqSnpyM9PR1A/aPe6enpyMnJAVB/v8ycOXN07SMjI5GdnY0VK1bg3Llz2LFjB7Zv344XX3xRjPK7DTd7Czzk7wqAozdERNT1iRpuUlNTERAQgICAAADAihUrEBAQgDVr1gAAlEqlLugAgJeXF+Li4pCQkIBhw4bh9ddfx6ZNm1p8DJzaT+TNJRl+OFuAS4UVIldDRETUMonQzR6BKSsrg62tLVQqFW8u1tOzu1Jx4OxV/F9gH7z9Z3+xyyEiom5En+9vg7rnhsS1aIIPAGBfWh7ySm+IXA0REVHzGG6ozYa52SGkrwPqtAI+SsoUuxwiIqJmMdyQXhaNrx+9+fzXHBRXqO/SmoiIqPMx3JBexvg4wK+PLaprtdh55LLY5RARETXBcEN6kUgkWHTzyal/H72M8urm1/QiIiISC8MN6S1scC/07WmJ8uo6fPpLzt13ICIi6kQMN6Q3qVSChTfvvdmWnIXqWo3IFREREd3CcEP35KFhruhtZ46iCjW+PH5F7HKIiIh09A43N27cQFVVle737OxsbNy4EQcOHGjXwqhrM5FJ8UyoFwBga1IG6jRakSsiIiKqp3e4eeihh7Br1y4AQGlpKUaNGoV33nkHDz30EDZv3tzuBVLXFTHCHQ6WpsgtuYHvTirFLoeIiAjAPYSb3377DaGhoQCAr776Cs7OzsjOzsauXbuwadOmdi+Qui5zUxmeGuMJANickAGttlut5EFERF2U3uGmqqoK1tbWAIADBw7gkUcegVQqxejRo5Gdnd3uBVLXNjvYE1Zmcpy/Wo6DvxeKXQ4REZH+4cbHxwf79u1Dbm4ufvjhB4SFhQEACgsLuRBlN2RrboK/jPYAAMQkXEI3W4eViIi6IL3DzZo1a/Diiy/C09MTo0aNQnBwMID6UZyAgIB2L5C6vqfHesJULsVvOaX4JatE7HKIiKib0zvc/N///R9ycnKQmpqK77//Xrd90qRJeO+999q1ODIMTtYKzAzqAwD44NAlkashIqLu7p7muenVqxcCAgIglUpRVlaGffv2wdraGgMHDmzv+shALHigL2RSCZIvFuHUFZXY5RARUTemd7iZOXMm3n//fQD1c94EBQVh5syZ8PPzw549e9q9QDIMbvYWCPdzAQBsTuToDRERiUfvcJOUlKR7FPzrr7+GIAgoLS3Fpk2b8MYbb7R7gWQ4GpZk+N/pAmRcqxC5GiIi6q70DjcqlQr29vYAgO+//x6PPvooLCwsMH36dFy8eLHdCyTDMaCXNSYPcoYgAFsSM8Quh4iIuim9w42bmxuOHj2KyspKfP/997pHwa9fvw6FQtHuBZJhWTShLwDg67Q85JfeELkaIiLqjvQON8uWLcMTTzyBPn36wNXVFePHjwdQf7lq6NCh7V0fGZjh7j0w2tsetRoBHyVnil0OERF1Q3qHm0WLFuHo0aPYsWMHDh8+DKm0/hDe3t6854YAAItu3nvz+bFclFTWiFwNERF1NxLhPqaUbdhVIpG0W0EdraysDLa2tlCpVJxRuYMIgoDw9w/jdF4Zlkz0wYqwAWKXREREBk6f7+97mudm165dGDp0KMzNzWFubg4/Pz/85z//uadiyfhIJBLd6M3HKZdRoa4TuSIiIupO9A437777LhYuXIhp06bhiy++QGxsLP7whz8gMjKSMxSTzoNDesG7pyXKquvw2S9cUJWIiDqP3pelvLy8sG7dOsyZM6fR9n//+99Yu3YtsrKy2rXA9sbLUp3ni9RcvPzVSThZmyHp5QlQmMjELomIiAxUh16WUiqVCAkJabI9JCQESqVS38OREZsxrDdcbBUoLFdjz29XxC6HiIi6Cb3DjY+PD7744osm22NjY9GvX792KYqMg6lcimdCvQEAWxIzUafRilwRERF1B3J9d1i3bh0iIiKQlJSEMWPGQCKR4PDhw/jpp5+aDT3UvT020g3/OngROSVV2H9KiYeG9Ra7JCIiMnJ6j9w8+uij+OWXX+Do6Ih9+/Zh7969cHR0xLFjx/Dwww93RI1kwCxM5XhqjBcAYHNCBu5j5gEiIqI2ua95bgwRbyjufKqqWoT8/SdU1miw48kgTBzoLHZJRERkYPT5/m7TZamysrI2fzgDA93J1sIEfxntgS1JmYg5lMFwQ0REHapN4cbOzu6usxALggCJRAKNRtMuhZFxmTfWCztTLiM1+zqOZZVgpJe92CUREZGRalO4OXToUEfXQUbOyUaB/wvsg89+ycEHhy5hpNdIsUsiIiIj1aZwM27cuI6ug7qBBQ944/NjOUi8cA2n81Tw7W0rdklERGSE7mltKaJ74eFgiT/6uQIANidmiFwNEREZK4Yb6lQLx/cFAPzvlBJZRZUiV0NERMZI9HATExMDLy8vKBQKBAYGIjk5udX2n376Kfz9/WFhYQEXFxc89dRTKC4u7qRq6X4NcrHBpIFO0ArAFo7eEBFRBxA13MTGxmLZsmVYvXo10tLSEBoaiqlTpyInJ6fZ9ocPH8acOXMwb948nDlzBl9++SV+/fVXzJ8/v5Mrp/uxaEL96M2e366gQFUtcjVERGRs7inc1NXV4ccff8SWLVtQXl4OAMjPz0dFRYVex3n33Xcxb948zJ8/H4MGDcLGjRvh5uaGzZs3N9v+559/hqenJ5YsWQIvLy+MHTsWCxYsQGpq6r10g0QS6GGPkV72qNUI+Cg5U+xyiIjIyOgdbrKzszF06FA89NBDWLx4Ma5duwYA2LBhA1588cU2H6empgbHjx9HWFhYo+1hYWFISUlpdp+QkBBcuXIFcXFxEAQBV69exVdffYXp06e3+DlqtRplZWWNXiS+RTfvvdl9LAfXK2tEroaIiIyJ3uFm6dKlCAoKwvXr12Fubq7b/vDDD+Onn35q83GKioqg0Wjg7Nx4tlpnZ2cUFBQ0u09ISAg+/fRTREREwNTUFL169YKdnR3+9a9/tfg50dHRsLW11b3c3NzaXCN1nHH9e2KIqw2qajT4OOWy2OUQEZER0TvcHD58GK+88gpMTU0bbffw8EBeXp7eBdw583HDTMfNOXv2LJYsWYI1a9bg+PHj+P7775GVlYXIyMgWj79y5UqoVCrdKzc3V+8aqf1JJBLdk1Mfp1xGpbpO5IqIiMhYtGkSv9tptdpml1i4cuUKrK2t23wcR0dHyGSyJqM0hYWFTUZzGkRHR2PMmDF46aWXAAB+fn6wtLREaGgo3njjDbi4uDTZx8zMDGZmZm2uizrPVF8XeDleQFZRJXYfy8H8UG+xSyIiIiOg98jNlClTsHHjRt3vEokEFRUVeO211zBt2rQ2H8fU1BSBgYGIj49vtD0+Ph4hISHN7lNVVQWptHHJMpkMQP2IDxkWmVSCyHH1geaj5Eyo67guGRER3T+9w817772HxMREDB48GNXV1Zg1axY8PT2Rl5eHt956S69jrVixAtu2bcOOHTtw7tw5LF++HDk5ObrLTCtXrsScOXN07cPDw7F3715s3rwZmZmZOHLkCJYsWYKRI0fC1dVV365QF/BwQB/0slHgapkae3/T/7ImERHRnfS+LOXq6or09HTs3r0bv/32G7RaLebNm4cnnnii0Q3GbREREYHi4mKsX78eSqUSvr6+iIuLg4eHBwBAqVQ2mvPmySefRHl5Od5//3288MILsLOzw8SJE/UOVdR1mMqlmB/qhTf2n8OWxAzMDHKDTNr6CvREREStkQjd7HpOWVkZbG1toVKpYGNjI3Y5BKBSXYcxbx1EaVUt/vV4AML9OQpHRESN6fP9rffIzTfffNPsdolEAoVCAR8fH3h5eel7WOrGLM3keDLEExt/vIiYhAz80c+lxSfmiIiI7kbvcDNjxgxIJJImN/A2bJNIJBg7diz27duHHj16tFuhZNyeDPHE1qRMnFOWIeHCNUwY4CR2SUREZKD0vqE4Pj4eI0aMQHx8vG7umPj4eIwcORLfffcdkpKSUFxcrNdsxUR2FqZ4YpQ7AGDzIS6oSURE907vkZulS5di69atjR7XnjRpEhQKBZ599lmcOXMGGzduxNNPP92uhZLxmx/qjX+nZOPY5RL8erkEIzztxS6JiIgMkN4jNxkZGc3eyGNjY4PMzPpFEPv164eioqL7r466FWcbBR4N7A0AiDl0SeRqiIjIUOkdbgIDA/HSSy/pFswEgGvXruHll1/GiBEjAAAXL15Enz592q9K6jYWPNAXUglw6Pw1nM3nIqdERKQ/vcPN9u3bkZWVhT59+sDHxwf9+vVDnz59cPnyZWzbtg0AUFFRgVdffbXdiyXj5+loiWlD65fR2JzIe2+IiEh/9zTPjSAI+OGHH3DhwgUIgoCBAwdiypQpTZZG6Io4z03XdyZfhembDkMqAQ6+MB6ejpZil0RERCLT5/ubk/hRl/TUzmM4dP4aHh/pjuhHhopdDhERiaxDJ/EDgMrKSiQmJiInJwc1NTWN3luyZMm9HJKokUUTfHDo/DXsOX4Fyyb3g7ONQuySiIjIQOgdbtLS0jBt2jRUVVWhsrIS9vb2KCoqgoWFBZycnBhuqF2M8LTHCM8e+PXydWxLzsTq6YPFLomIiAyE3jfJLF++HOHh4SgpKYG5uTl+/vlnZGdnIzAwEG+//XZH1Ejd1KLxPgCAT3/JQWlVzV1aExER1dM73KSnp+OFF16ATCaDTCaDWq2Gm5sbNmzYgFWrVnVEjdRNjR/QE4NcbFBVo8G/U7LFLoeIiAyE3uHGxMREt6ihs7MzcnJyAAC2tra6n4nag0QiwcLxfQEAH6dkoaqmTuSKiIjIEOgdbgICApCamgoAmDBhAtasWYNPP/0Uy5Ytw9ChfKqF2tf0oS7wdLDA9apa7D6WK3Y5RERkAPQON2+++SZcXOonWXv99dfh4OCAhQsXorCwEFu3bm33Aql7k0klWDCufvTmo6RM1NRpRa6IiIi6Or2elhIEAT179sSQIUMAAD179kRcXFyHFEbU4JHhvbHxxwsoKKvG12lXEDHCXeySiIioC9Nr5EYQBPTr1w9XrlzpqHqImjCTyzB/rDcA4MPETGi03WreSSIi0pNe4UYqlaJfv34oLi7uqHqImvX4KHfYmpsgq6gS358uELscIiLqwvS+52bDhg146aWXcPr06Y6oh6hZVmZyzA3xBADEJFxCN1s1hIiI9KD32lI9evRAVVUV6urqYGpqCnNz80bvl5SUtGuB7Y1rSxmu65U1GPPWwfp5b54eiXH9e4pdEhERdZIOXVtq48aN91oX0X3pYWmKx0e6Y/vhLMQcusRwQ0REzdI73MydO7cj6iBqk/mhXth19DJ+ySrB8ewSBHrYi10SERF1MXrfcwMAGRkZeOWVV/D444+jsLAQAPD999/jzJkz7Voc0Z1cbM3xSEAfAEDMoQyRqyEioq5I73CTmJiIoUOH4pdffsHevXtRUVEBADh58iRee+21di+Q6E4LxnlDIgF++r0QvxeUiV0OERF1MXqHm6ioKLzxxhuIj4+HqampbvuECRNw9OjRdi2OqDnePa0wzbd+luzNCRy9ISKixvQON6dOncLDDz/cZHvPnj05/w11moYFNb89kY+c4iqRqyEioq5E73BjZ2cHpVLZZHtaWhp69+7dLkUR3Y1vb1uM698TWgHYksTRGyIiukXvcDNr1iz89a9/RUFBASQSCbRaLY4cOYIXX3wRc+bM6YgaiZq16ObozZfHr6CwrFrkaoiIqKvQO9z87W9/g7u7O3r37o2KigoMHjwYDzzwAEJCQvDKK690RI1EzRrpZY9Ajx6oqdNi++EsscshIqIuQu8ZihtkZGQgLS0NWq0WAQEB6NevX3vX1iE4Q7Fx+encVcz7dyosTWVIiZoEWwsTsUsiIqIO0KEzFCcmJmLcuHHo27cv+vbte89FErWHiQOdMLCXNX4vKMeuo5fx/CTDCNlERNRx9L4sNWXKFLi7uyMqKoqLZ5LoJBKJ7smpnSmXcaNGI3JFREQkNr3DTX5+Pl5++WUkJyfDz88Pfn5+2LBhA65cudIR9RHd1fShLnC3t0BJZQ0+/zVH7HKIiEhkeocbR0dHPPfcczhy5AgyMjIQERGBXbt2wdPTExMnTuyIGolaJZdJsWCcNwDgo6RM1NRpRa6IiIjEdE9rSzXw8vJCVFQU/v73v2Po0KFITExsr7qI9PLo8D7oaW2GfFU19qXniV0OERGJ6J7DzZEjR7Bo0SK4uLhg1qxZGDJkCL777rv2rI2ozRQmMswf6wUA+DAxAxrtPT0ESERERkDvcLNq1Sp4eXlh4sSJyM7OxsaNG1FQUIBPPvkEU6dO7YgaidrkidEesFHIkXmtEgfOFIhdDhERiUTvcJOQkIAXX3wReXl52L9/P2bNmgULC4t7LiAmJgZeXl5QKBQIDAxEcnJyq+3VajVWr14NDw8PmJmZoW/fvtixY8c9fz4ZDyszOeaGeAIAYhIycI9TOBERkYHTe56blJSUdvvw2NhYLFu2DDExMRgzZgy2bNmCqVOn4uzZs3B3d292n5kzZ+Lq1avYvn07fHx8UFhYiLq6unariQzbU2O8sC05C6fyVDh8qQih/XqKXRIREXWye56h+OzZs8jJyUFNTU2j7X/605/afIxRo0Zh+PDh2Lx5s27boEGDMGPGDERHRzdp//333+Oxxx5DZmYm7O3t2/QZarUaarVa93tZWRnc3Nw4Q7ERW/ftGew8chnB3g7Y/exoscshIqJ20KEzFGdmZuLhhx/GqVOnIJFIdEP/EokEAKDRtG0StZqaGhw/fhxRUVGNtoeFhbU4OvTNN98gKCgIGzZswH/+8x9YWlriT3/6E15//XWYm5s3u090dDTWrVvX1u6REXgm1Buf/JyNo5nF+C3nOoa79xC7JCIi6kR633OzdOlSeHl54erVq7CwsMCZM2eQlJSEoKAgJCQktPk4RUVF0Gg0cHZ2brTd2dkZBQXN3wyamZmJw4cP4/Tp0/j666+xceNGfPXVV1i8eHGLn7Ny5UqoVCrdKzc3t801kmFytTPHjGG9AQAxhzJEroaIiDqb3uHm6NGjWL9+PXr27AmpVAqpVIqxY8ciOjoaS5Ys0buAhhGfBoIgNNnWQKvVQiKR4NNPP8XIkSMxbdo0vPvuu/j4449x48aNZvcxMzODjY1NoxcZv8jxfSGRAD+eu4rzBeVil0NERJ1I73Cj0WhgZWUFoH624vz8fACAh4cHzp8/3+bjODo6QiaTNRmlKSwsbDKa08DFxQW9e/eGra2tbtugQYMgCAKXf6BG+va0wh+G9AJQP+8NERF1H3qHG19fX5w8eRJA/Q3BGzZswJEjR7B+/Xp4e3u3+TimpqYIDAxEfHx8o+3x8fEICQlpdp8xY8YgPz8fFRUVum0XLlyAVCpFnz599O0KGblF430AAN+cyEduSZXI1RARUWfRO9y88sor0Grr1+554403kJ2djdDQUMTFxWHTpk16HWvFihXYtm0bduzYgXPnzmH58uXIyclBZGQkgPr7ZebMmaNrP2vWLDg4OOCpp57C2bNnkZSUhJdeeglPP/10izcUU/c1tI8tQvs5QqMVsDUpU+xyiIiok+j9tNSDDz6o+9nb2xtnz55FSUkJevTo0eK9Mi2JiIhAcXEx1q9fD6VSCV9fX8TFxcHDwwMAoFQqkZNza5VnKysrxMfH4/nnn0dQUBAcHBwwc+ZMvPHGG/p2g7qJReN9kHyxCLGpuXh+kg+crBVil0RERB3snue5MVT6PCdPhk8QBDyyOQVpOaWIHNcXUVMHil0SERHdA32+v+9rVXCirk4ikejuvfnk52yobtSKXBEREXU0hhsyepMGOqG/sxUq1HX45OdsscshIqIOxnBDRk8qlWDh+L4AgB2Hs3Cjpm2zaBMRkWFiuKFuIdzPFX16mKO4sgZfpHKWaiIiY8ZwQ92CXCbFgnH1ozdbkzJRq9GKXBEREXUUhhvqNv4c2AeOVmbIK72Bb9LzxS6HiIg6CMMNdRsKExnmjfUCAGxOzIBW261mQSAi6jYYbqhb+ctod1gr5LhUWIEDZ6+KXQ4REXUAhhvqVqwVJpgTXD8D9uaES+hmc1gSEXULDDdk1GJiYuDl5QWFQoHAwEAkJyfjqTFeUJhIceKKCikZxc3ud+TIEcjlcgwbNqzJexs3bsSAAQNgbm4ONzc3LF++HNXV1R3cEyIiaiuGGzJasbGxWLZsGVavXo20tDSEhoZi6tSpqCq5isdGuAMAYhIuNdlPpVJhzpw5mDRpUpP3Pv30U0RFReG1117DuXPnsH37dsTGxmLlypUd3h8iImobhhsyWu+++y7mzZuH+fPnY9CgQdi4cSPc3NywefNmPPOAN+RSCY5cKkZ6bmmj/RYsWIBZs2YhODi4yTGPHj2KMWPGYNasWfD09ERYWBgef/xxpKamdlKviIjobhhuyCjV1NTg+PHjCAsLa7Q9LCwMKSkp6G1njoeG9QYAxBy6NXqzc+dOZGRk4LXXXmv2uGPHjsXx48dx7NgxAEBmZibi4uIwffr0DuoJERHpSy52AUQdoaioCBqNBs7Ozo22Ozs7o6CgAACwcLw39qZdwYGzV3HxajlQVoCoqCgkJydDLm/+P43HHnsM165dw9ixYyEIAurq6rBw4UJERUV1eJ+IiKhtOHJDRk0ikTT6XRAE3TYfJ2uEDa4PPx8cuoBZs2Zh3bp16N+/f4vHS0hIwN/+9jfExMTgt99+w969e/Hdd9/h9ddf77hOEBGRXjhyQ0bJ0dERMplMN0rToLCwsNFozqLxPvjhzFX891gGLqemIi0tDc899xwAQKvVQhAEyOVyHDhwABMnTsSrr76K2bNnY/78+QCAoUOHorKyEs8++yxWr14NqZT/f4GISGz8l5iMkqmpKQIDAxEfH99oe3x8PEJCQnS/+7vZYayPI7Qm5oj8516kp6frXpGRkRgwYADS09MxatQoAEBVVVWTACOTySAIAufMISLqIjhyQ0ZrxYoVmD17NoKCghAcHIytW7ciJycHkZGRAICVK1ciLy8Pi9a8i8OXinCwUIHXPfvB0coMAODk5ASFQgFfX1/dMcPDw/Huu+8iICAAo0aNwqVLl/Dqq6/iT3/6E2QymSj9JCKixhhuyGhFRESguLgY69evh1KphK+vL+Li4uDhUT9DsVKpRE5ODoL7OsDfzQ4nckux6ceL+MPQXvBytGz2mK+88gokEgleeeUV5OXloWfPnggPD8ff/va3zuwaERG1QiJ0s7H0srIy2NraQqVSwcbGRuxyqIv44UwBFvznuO53qQSIfmQoIm5O9kdEROLS5/ub99wQAfB1bfwfilYAVu09BaXqhkgVERHRvWK4IQKQXVLVZJtGAN6M+x2n81S8WZiIyIDwnhsiAF6OlpBK6kdsbvftiXx8eyIf3o6W+KO/K8L9XNDP2VqcIomIqE14zw3RTbG/5mDV3tPQCAKkEuDxke4oqazBwd8Loa7T6toN7GWNcH9X/NHPBR4Ozd94TERE7Uuf72+GG6LbKFU3cLmoCp6OFnCxNQcAVKjr8OPZq/j2RD6SLl5DrebWfzJ+fWwR7ueK6X4ucLUzF6tsIiKjx3DTCoYbuh+qqlr8cKYA357MR0pGMTS3XccK8uiBcH9XTB3aC07WChGrJCIyPgw3rWC4ofZSVKHG/04X4NsT+fj1cgka/kuSSoDR3g4I93fFH4b0Qg9LU3ELJSIyAgw3rWC4oY5QoKrG/lNKfHsiH+m5pbrtcqkEY/s5ItzPFVOGOMNGYSJekUREBozhphUMN9TRckuq8N1JJb47mY8z+WW67aYyKcYP6Ilwf1dMGuQEC1M+rEhE1FYMN61guKHOlHGtAt+dUOLbk/m4VFih225uIsOkQU4I93fFuP49oTDhulRERK1huGkFww2JQRAEnL9ajm9P5OO7k0pkF9+aNNDaTI4pQ5wR7ueKsf0cYSLj3JpERHdiuGkFww2JTRAEnMpT6YKOUlWte8/OwgRTfXsh3M8Vo7wdIJNKRKyUiKjrYLhpBcMNdSVarYDfcq7j2xP52H+qAEUVat17jlZmmD60F/7o74pA9x6QMugQUTfGcNMKhhvqqjRaAb9kFuPbk/n43+kClFbV6t5zsVXgj34uCPd3xdDetpBIGHSIqHthuGkFww0ZglqNFocvFeHbE/mIP3MV5eo63Xvu9hYI93fBH/1cMbCXNYMOEXULDDetYLghQ1Ndq0HihWv49kQ+fjpXiBu1Gt17Pk5WCPdzxR/9XdC3p5WIVRIRdSyGm1Yw3JAhq6qpw0/nCvHtiXwkXLiGmtsW9BzsYqNb0NPN3kLEKomI2p8+39+iP3MaExMDLy8vKBQKBAYGIjk5uU37HTlyBHK5HMOGDevYAom6EAtTOcL9XbF1ThBSX5mMd/7sj/EDekIuleCssgxvff87QjccwowPjmD74SwU3PYkFhFRdyHqyE1sbCxmz56NmJgYjBkzBlu2bMG2bdtw9uxZuLu7t7ifSqXC8OHD4ePjg6tXryI9Pb3Nn8mRGzJG1ytr8P2Z+nWufs4sRsN6nhIJMMLTvn5BT99ecLQyE7dQIqJ7ZDCXpUaNGoXhw4dj8+bNum2DBg3CjBkzEB0d3eJ+jz32GPr16weZTIZ9+/Yx3BDdprC8Gv87VR90UrOv67bLpBKE9HVAuJ8rHhzSC7YWXOeKiAyHPt/foi1uU1NTg+PHjyMqKqrR9rCwMKSkpLS4386dO5GRkYFPPvkEb7zxxl0/R61WQ62+NXdIWVlZK62JDJ+TtQJzQzwxN8QT+aU3sP9k/fIPJ6+okHyxCMkXi7B63yk80K9+navJg51hZcZ1rojIeIj2L1pRURE0Gg2cnZ0bbXd2dkZBQUGz+1y8eBFRUVFITk6GXN620qOjo7Fu3br7rpfIELnameOZB7zxzAPeyC6uxHcn61cu/72gHD/9Xoiffi+EmVyKiQPr17maMMAJ5qZc54qIDJvo/3ftzjk6BEFodt4OjUaDWbNmYd26dejfv3+bj79y5UqsWLFC93tZWRnc3NzuvWAiA+XhYInFE3yweIIPLl4tx7cnlfjuRD4yiyrxv9MF+N/pAliYyjBlcP06V6H9HWEmZ9AhIsMj2j03NTU1sLCwwJdffomHH35Yt33p0qVIT09HYmJio/alpaXo0aMHZLJb/9hqtVoIggCZTIYDBw5g4sSJd/1c3nNDdIsgCDirLMO3J+pHdPJKb+jes1bI8YchvRDu74qQvg6Qc0FPIhKRQd1QHBgYiJiYGN22wYMH46GHHmpyQ7FWq8XZs2cbbYuJicHBgwfx1VdfwcvLC5aWlnf9TIYbouYJgoD03FJ8e0KJ/afycbXs1r1q9pam9Qt6+rtihKc9F/Qkok5nEDcUA8CKFSswe/ZsBAUFITg4GFu3bkVOTg4iIyMB1F9SysvLw65duyCVSuHr69tofycnJygUiibbiUh/EokEAe49EODeA69MH4RfL5fg25P5iDtVgJLKGnz6Sw4+/SUHTtZmmH5znasANzsu/0BEXY6o4SYiIgLFxcVYv349lEolfH19ERcXBw8PDwCAUqlETk6OmCUSdUtSqQSjvB0wytsBa8OH4GhmMb49kY/vTxegsFyNnUcuY+eRy+htZ44/+rsg3M8VQ1xtGHSIqEvg8gtE1GY1dVokX6xf5yr+7FVU1txa58rL0RLhN0d0+jlbi1glERkjg1p+gYgMh6lcikmDnLHxsQAcf3UKNj8xHNOG9oKZXIqsokpsOngJU95LwoPvJeH9gxdxuajyvj9TnyVaDh8+jDFjxsDBwQHm5uYYOHAg3nvvvUZtPvroI4SGhqJHjx7o0aMHJk+ejGPHjt13nUTUdXDkhojuW4W6Dj+du4pvT+Qj8cI11Gpu/bMytLctwv1dMN3PFb3tzPU6rr5LtKSlpeH333+Hn58fLC0tcfjwYSxYsADvvfcenn32WQDAE088gTFjxiAkJAQKhQIbNmzA3r17cebMGfTu3fv+/iKIqMMYzNNSYmC4IepYqqpa/HC2fvmHlIxiaLS3/okJ9OiBcD8XTPNzgZO14q7HutclWm73yCOPwNLSEv/5z3+afV+j0aBHjx54//33MWfOnDYdk4g6n8E8LUVExsfWwgQzg9wwM8gNxRVq/O90fdA5drkEx7Ov43j2daz/7ixGeTkg3N8Vf/DtBXtL0ybHudclWm6XlpaGlJSUVpdqqaqqQm1tLezt7fXrKBF1WQw3RNRhHKzM8JfRHvjLaA9cLavWrXOVllOKo5nFOJpZjDX/PY0xPo4I93dF2BBn2CjqF/S8lyVaGvTp0wfXrl1DXV0d1q5di/nz57fYNioqCr1798bkyZPvv8NE1CUw3BBRp3C2UeDpsV54eqwXckuqsP9U/azIZ/LLkHjhGhIvXIPpXinGDahf0HOIbR2Ati/Rcrvk5GRUVFTg559/RlRUFHx8fPD44483abdhwwbs3r0bCQkJUCjufpmMiAwDww0RdTo3ewtEjuuLyHF9kXmtAt+dVOKbE/m4VFiB+LNXEX/2KhRSLSRSGeKOnUNA0EgoTOqXXiksLGwymnMnLy8vAMDQoUNx9epVrF27tkm4efvtt/Hmm2/ixx9/hJ+fX8d0lIhEwXBDRKLy7mmFJZP64fmJPjh/tRzfnai/dJVdXAUT577Y+O89+LqkN8IGOyPc3xUH4uMx46GH2nx8QRCgVqsbbfvHP/6BN954Az/88AOCgoLau0tEJDKGGyLqEiQSCQb2ssHAXjZ4Iaw/TuWp8CYW4Iu3/4qCXj6IzRuEbW9/j8qMLJR6jMORS0X4ZtvbUObnY9euXQCADz74AO7u7hg4cCCA+nlv3n77bTz//PO6z9mwYQNeffVVfPbZZ/D09NTdv2NlZQUrK6vO7zgRtTs+Ck5EXdoHH3yAv0W/hcKrBTBz8oT1+HlQuNWvJ1dx4J+wqrmOL7/7AYHuPfDm2+9hx7aPUJCXAxO5HH379sUzzzyDBQsWQCqtn7PU09MT2dnZTT7ntddew9q1azuza0SkB85z0wqGGyLDpdEK+CWrGN+eUOJ/p5UorarVvWdrLkfZjToIAKQS4M2Hh+KxkU0n+iMiw8Rw0wqGGyLjUKvR4vClInx3QonvTysbrXPVwMPeHO4OlnC1NYeLnUL3p4utOVztFLAw5ZV5IkPBcNMKhhsi45NwvhBP7vxV7/1szU3gYquAq515oz8bwk8vWwXM5LIOqJiI9MUZiomoWxnQyxpSCXDbSg+QSoD3IoZBXaeFsrQaStUN5KuqoSy9AaWqGhXqOqhu1EJ1oxa/F5S3eGxHKzO42ikahZ7b/3SyNoNcxjWIWxMTE4N//OMfUCqVGDJkCDZu3IjQ0NBm2+7duxebN29Geno61Go1hgwZgrVr1+LBBx/UtRk/fjwSExOb7Dtt2jTs37+/w/pBhoPhhogMnoutOaIfGYpVe09DIwiQSSR48xFfPDSs5YUwy6proSytRr7qxq3wc/NPpaoa+aU3oK7ToqhCjaIKNU5eUTV7HJlUAidrs/rwY2cOV10IuhWAHCxNIZW2PvGgsYqNjcWyZcsaLX46derUFhc/TUpKwpQpU/Dmm2/Czs4OO3fuRHh4OH755RcEBAQAqA9ANTU1un2Ki4vh7++PP//5z53WL+raeFmKiIyGUnUDl4uq4OloARdb/VYgv5MgCCiprNEFHaWqaRC6WlaNOu3d/wk1lUnRy1bR+NJXoyCkgK25yV1nXjZE7bH46ZAhQxAREYE1a9Y0+/7GjRuxZs0aKJVKWFpatkvd1PXwshQRdUsutub3HWoaSCQSOFiZwcHKDL69bZtto9EKKKpQ3wo/N/9Uqm4gr7T+Eti1CjVqNFrklFQhp6Sqxc8zN5Hpbnq+89JXw5+WZob1T3Z7LH6q1WpRXl7e6sKm27dvx2OPPcZgQzqG9V8KEVEXIpNK4GyjgLONAgEttKmp0+JqWbUu9OQ3cwmspLIGN2o1yLxWicxrlS1+no1C3uzIT0Mo6mWr0C1T0RXcz+KnDd555x1UVlZi5syZzb5/7NgxnD59Gtu3b7/vesl4MNwQEXUgU7kUbvYWcLO3aLFNda2mPvyU1t/0XD8CdFsAKq1GuboOZdV1KCsov8sN0KY3R7CauQRmZw5nEW6AvpfFTwFg9+7dWLt2Lf773//Cycmp2Tbbt2+Hr68vRo4c2S61knFguCEiEpnCRAYvR0t4ObZ8WaW8urbxpa+bQagh/OSrbqC6VouiihoUVdTgVF7zN0BLJYCTteLWvD+3hR9Xu/pRIEdLs3a5AdrR0REymazJKE1bFj+NjY3FvHnz8OWXX2Ly5MnNtqmqqsLnn3+O9evX33etZFwYboiIDIC1wgTWChP0d7Zu9n1BEFBaVYu82+77uX3kJ191A1fLqlGrEVBQVo2CsmqkobTZY5nIJDdvgL414nPnJTA7i7vfAG1qaorAwEDEx8fj4Ycf1m2Pj4/HQ60sfrp79248/fTT2L17N6ZPn95iuy+++AJqtRp/+ctfWq2Duh+GGyIiIyCRSNDD0hQ9LE1bvAFa23AD9O0jP3c8CXa1vD4A5ZbcQG7JjRY/z9xEdnPU57ZH3++4BGZlJseKFSswe/ZsBAUFITg4GFu3bkVOTg4iIyMBACtXrkReXp5u8dPdu3djzpw5+Oc//4nRo0frRn3Mzc1ha9u4X9u3b8eMGTPg4ODQHn+FZET4KDgREenUam7dAN3SJbDiypq7HwiAtUIOV1tzqH77Dud++BSVpdfg3ncgXnz1DUwNmwQXWwUin5mHy5cvIyEhAQAQMvYBHD2S3ORYc+fOxccff6z7/cKFCxgwYAAOHDiAKVOmtEfXqQXtPQnj3r178eabb+LSpUuora1Fv3798MILL2D27Nmt1sHlF1rBcENEdH+qazUouHPen4YwdPMSWHl1XZuOZW9pqnvUvbK6FkczSyAAkEiA5yf64IlRHuhhYQpTOWeBFkNsbCxmz57daBLGbdu2tTgJ47Jly+Dq6ooJEyboJmF8++23G03CmJCQgOvXr2PgwIEwNTXFd999hxdeeAH79+9vFILuxHDTCoYbIqKOV6Gua3Tpq7lLYDdqmy522hIbhbx+3iFLU9hbmup+drCq/93Rykz3s72FKZfEaCedMQkjAAwfPhzTp0/H66+/3mIbTuJHRESisjKTo5+zNfq1cgO06kat7qbnwxeLsDPlcpN2EgACUP8YfHUdsopangfodnYWJvXhx/JW6Gk2EFmaws7CFLJuujxGazpjEkZBEHDw4EGcP38eb7311n3X3IDhhoiIOp1EIoGdRX2wGOxqg8GuNvj30cuNFj+VSSRIenk8LEzlKK5Uo7iiBsWVN18VapRU1tzcduu961U1EASgtKoWpVW1yGhlUsRbtQD2Fg0BqIVAdNt7tuYm3WKtsI6chFGlUqF3795Qq9WQyWSIiYlp13unGG6IiEh0LS1+2rtH/eSHPSxN4dP8PH6NaLQCrlfVoKSyBkWNAlDjQFRUWf9zaVUtBAG60HSx8O6fIZNK0MPCFI7NBCD728JRw8iRjbncoNcN64hJGK2trZGeno6Kigr89NNPWLFiBby9vTF+/Ph2qZnhhoiIuoSIEe54oH/P+1r8VCaVwNHKDI5WZi3OCXS7Wo0W16vqA0+rgejmz2XVdbo1xYoq1G2qyUQmqb8XyNLsViC6LQDdecnMyqxrhKGOnIRRKpXCx8cHADBs2DCcO3cO0dHRDDdERGR82nPx07YwkUnhZK2Ak7WiTe1r6rQ3w4665UBUeev3CnUdajUCrpapcbWsbWHIVC697d4gMzi2chO1vaUpLExlHRKGOnoSxtsJggC1um1/P23BcENERNRGpnIpetkq0Mu2bWGoulbT5N6gksqbl8XuuIeouKJ+AdWaOu3NWaar2/QZChPpHSNBZk0vmd0WiPRZXLUjJmGMjo5GUFAQ+vbti5qaGsTFxWHXrl2Nnsi6Xww3REREHURhIqufvdmubaNRVTV1ugB0+43Sd44QNfyurtOiulaLvNIbyCtteUbp21mYyhqNCul+biYQzXj0/7CxuBjr16+HUqmEr68v4uLi4OHhAQBQKpXIycnRHXvLli2oq6vD4sWLsXjxYt322ydhrKysxKJFi3DlyhWYm5tj4MCB+OSTTxAREdHGv9W74zw3REREBkgQBFTWaFBScftIkPrmSFBzl8zUqNXo/5VvbSa/eaP0nSNBTe8h+uncVbyy7zS0Qv0irdGPDEXEiKaT/d0LznNDRERk5CQSCazM5LAyk8PdweKu7QVBQLm6YWRIjaKGEaKK5gNRSWUN6rT1+5Sr65BdXKVXfVoBWLX3NB7o37NT76MCGG6IiIi6BYlEAhuFCWwUJvBytLxre61WQFl17W3Bp+VA1HAJ7c5xIY0g4HJRFcMNERERiU8qvTXRYt+ed2+fd70KoRsONZmI0dPx7qNK7Y2LbxAREdF9693DAtGPDIXs5mPpDRMxdvaoDcCRGyIiImon7TERY3sQfeQmJiYGXl5eUCgUCAwMRHJycott9+7diylTpqBnz56wsbFBcHAwfvjhh06sloiIiFrjYmuO4L4OogUbQORwExsbi2XLlmH16tVIS0tDaGgopk6d2uiZ+dslJSVhypQpiIuLw/HjxzFhwgSEh4cjLS2tkysnIiKirkrUeW5GjRqF4cOHN5qVcNCgQZgxYwaio6PbdIwhQ4YgIiICa9asafZ9tVrdaErnsrIyuLm5cZ4bIiIiA6LPPDeijdzU1NTg+PHjCAsLa7Q9LCwMKSkpbTqGVqtFeXk57O3tW2wTHR0NW1tb3cvNze2+6iYiIqKuTbRwU1RUBI1G02RlUWdn5yYrkLbknXfeQWVlJWbOnNlim5UrV0KlUuleubm591U3ERERdW2iPy1150qmgiC0aXXT3bt3Y+3atfjvf/8LJyenFtuZmZnBzMzsvuskIiIiwyBauHF0dIRMJmsySlNYWNhkNOdOsbGxmDdvHr788ktMnjy5I8skIiIiAyPaZSlTU1MEBgYiPj6+0fb4+HiEhIS0uN/u3bvx5JNP4rPPPsP06dM7ukwiIiIyMKJellqxYgVmz56NoKAgBAcHY+vWrcjJyUFkZCSA+vtl8vLysGvXLgD1wWbOnDn45z//idGjR+tGfczNzWFraytaP4iIiKjrEDXcREREoLi4GOvXr4dSqYSvry/i4uLg4eEBAFAqlY3mvNmyZQvq6uqwePFiLF68WLd97ty5+Pjjjzu7fCIiIuqCRJ3nRgz6PCdPREREXYNBzHNDRERE1BEYboiIiMioMNwQERGRUWG4ISIiIqPCcENERERGheGGiIiIjArDDRERERkVhhsiIiIyKgw3REREZFQYboiIiMioMNwQERGRUWG4ISIiIqPCcENERERGheGGiIiIjArDDRERERkVhhsiIiIyKgw3REREZFQYboiIiMioMNwQERGRUWG4ISIiIqPCcENERERGheGGiIiIjArDDRERERkVhhsiIiIyKgw3REREZFQYboiIiMioMNwQERGRUWG4ISIiIqPCcENERERGheGGiIiIjArDDRERERkVhhsiIiIyKgw3REREZFQYboiIiMioMNwQERGRUWG4ISIiIqPCcENERERGRfRwExMTAy8vLygUCgQGBiI5ObnV9omJiQgMDIRCoYC3tzc+/PDDTqqUiIiIDIGo4SY2NhbLli3D6tWrkZaWhtDQUEydOhU5OTnNts/KysK0adMQGhqKtLQ0rFq1CkuWLMGePXs6uXIiIiLqqiSCIAhiffioUaMwfPhwbN68Wbdt0KBBmDFjBqKjo5u0/+tf/4pvvvkG586d022LjIzEiRMncPTo0TZ9ZllZGWxtbaFSqWBjY3P/nSAiIqIOp8/3t7yTamqipqYGx48fR1RUVKPtYWFhSElJaXafo0ePIiwsrNG2Bx98ENu3b0dtbS1MTEya7KNWq6FWq3W/q1QqAPV/SURERGQYGr632zImI1q4KSoqgkajgbOzc6Ptzs7OKCgoaHafgoKCZtvX1dWhqKgILi4uTfaJjo7GunXrmmx3c3O7j+qJiIhIDOXl5bC1tW21jWjhpoFEImn0uyAITbbdrX1z2xusXLkSK1as0P2u1WpRUlICBweHVj/nXpSVlcHNzQ25ublGecnL2PsHGH8f2T/DZ+x9ZP8MX0f1URAElJeXw9XV9a5tRQs3jo6OkMlkTUZpCgsLm4zONOjVq1ez7eVyORwcHJrdx8zMDGZmZo222dnZ3XvhbWBjY2O0/6MFjL9/gPH3kf0zfMbeR/bP8HVEH+82YtNAtKelTE1NERgYiPj4+Ebb4+PjERIS0uw+wcHBTdofOHAAQUFBzd5vQ0RERN2PqI+Cr1ixAtu2bcOOHTtw7tw5LF++HDk5OYiMjARQf0lpzpw5uvaRkZHIzs7GihUrcO7cOezYsQPbt2/Hiy++KFYXiIiIqIsR9Z6biIgIFBcXY/369VAqlfD19UVcXBw8PDwAAEqlstGcN15eXoiLi8Py5cvxwQcfwNXVFZs2bcKjjz4qVhcaMTMzw2uvvdbkMpixMPb+AcbfR/bP8Bl7H9k/w9cV+ijqPDdERERE7U305ReIiIiI2hPDDRERERkVhhsiIiIyKgw3REREZFQYbvQUExMDLy8vKBQKBAYGIjk5udX2iYmJCAwMhEKhgLe3Nz788MNOqvTe6NO/hIQESCSSJq/ff/+9Eytuu6SkJISHh8PV1RUSiQT79u276z6Gdv707aMhncPo6GiMGDEC1tbWcHJywowZM3D+/Pm77mdI5/Be+mhI53Dz5s3w8/PTTe4WHByM//3vf63uY0jnT9/+GdK5a050dDQkEgmWLVvWajsxziHDjR5iY2OxbNkyrF69GmlpaQgNDcXUqVMbPa5+u6ysLEybNg2hoaFIS0vDqlWrsGTJEuzZs6eTK28bffvX4Pz581AqlbpXv379Oqli/VRWVsLf3x/vv/9+m9ob2vkD9O9jA0M4h4mJiVi8eDF+/vlnxMfHo66uDmFhYaisrGxxH0M7h/fSxwaGcA779OmDv//970hNTUVqaiomTpyIhx56CGfOnGm2vaGdP33718AQzt2dfv31V2zduhV+fn6tthPtHArUZiNHjhQiIyMbbRs4cKAQFRXVbPuXX35ZGDhwYKNtCxYsEEaPHt1hNd4Pfft36NAhAYBw/fr1TqiufQEQvv7661bbGNr5u1Nb+mjI57CwsFAAICQmJrbYxtDPYVv6aMjnUBAEoUePHsK2bduafc/Qz58gtN4/Qz135eXlQr9+/YT4+Hhh3LhxwtKlS1tsK9Y55MhNG9XU1OD48eMICwtrtD0sLAwpKSnN7nP06NEm7R988EGkpqaitra2w2q9F/fSvwYBAQFwcXHBpEmTcOjQoY4ss1MZ0vm7X4Z4DlUqFQDA3t6+xTaGfg7b0scGhnYONRoNPv/8c1RWViI4OLjZNoZ8/trSvwaGdu4WL16M6dOnY/LkyXdtK9Y5ZLhpo6KiImg0miaLejo7OzdZzLNBQUFBs+3r6upQVFTUYbXei3vpn4uLC7Zu3Yo9e/Zg7969GDBgACZNmoSkpKTOKLnDGdL5u1eGeg4FQcCKFSswduxY+Pr6ttjOkM9hW/toaOfw1KlTsLKygpmZGSIjI/H1119j8ODBzbY1xPOnT/8M7dwBwOeff47ffvsN0dHRbWov1jkUdfkFQySRSBr9LghCk213a9/c9q5Cn/4NGDAAAwYM0P0eHByM3NxcvP3223jggQc6tM7OYmjnT1+Geg6fe+45nDx5EocPH75rW0M9h23to6GdwwEDBiA9PR2lpaXYs2cP5s6di8TExBYDgKGdP336Z2jnLjc3F0uXLsWBAwegUCjavJ8Y55AjN23k6OgImUzWZBSjsLCwSSpt0KtXr2bby+VyODg4dFit9+Je+tec0aNH4+LFi+1dnigM6fy1p65+Dp9//nl88803OHToEPr06dNqW0M9h/r0sTld+RyamprCx8cHQUFBiI6Ohr+/P/75z38229YQz58+/WtOVz53x48fR2FhIQIDAyGXyyGXy5GYmIhNmzZBLpdDo9E02Uesc8hw00ampqYIDAxEfHx8o+3x8fEICQlpdp/g4OAm7Q8cOICgoCCYmJh0WK334l7615y0tDS4uLi0d3miMKTz15666jkUBAHPPfcc9u7di4MHD8LLy+uu+xjaObyXPjanq57D5giCALVa3ex7hnb+mtNa/5rTlc/dpEmTcOrUKaSnp+teQUFBeOKJJ5Ceng6ZTNZkH9HOYYfermxkPv/8c8HExETYvn27cPbsWWHZsmWCpaWlcPnyZUEQBCEqKkqYPXu2rn1mZqZgYWEhLF++XDh79qywfft2wcTERPjqq6/E6kKr9O3fe++9J3z99dfChQsXhNOnTwtRUVECAGHPnj1idaFV5eXlQlpampCWliYAEN59910hLS1NyM7OFgTB8M+fIOjfR0M6hwsXLhRsbW2FhIQEQalU6l5VVVW6NoZ+Du+lj4Z0DleuXCkkJSUJWVlZwsmTJ4VVq1YJUqlUOHDggCAIhn/+9O2fIZ27ltz5tFRXOYcMN3r64IMPBA8PD8HU1FQYPnx4o0c0586dK4wbN65R+4SEBCEgIEAwNTUVPD09hc2bN3dyxfrRp39vvfWW0LdvX0GhUAg9evQQxo4dK+zfv1+Eqtum4bHLO19z584VBME4zp++fTSkc9hcvwAIO3fu1LUx9HN4L300pHP49NNP6/596dmzpzBp0iTdF78gGP7507d/hnTuWnJnuOkq51AiCDfv7CEiIiIyArznhoiIiIwKww0REREZFYYbIiIiMioMN0RERGRUGG6IiIjIqDDcEBERkVFhuCEiIiKjwnBDRERERoXhhoi6vYSEBEgkEpSWlopdChG1A4YbIiIiMioMN0RERGRUGG6ISHSCIGDDhg3w9vaGubk5/P398dVXXwG4dclo//798Pf3h0KhwKhRo3Dq1KlGx9izZw+GDBkCMzMzeHp64p133mn0vlqtxssvvww3NzeYmZmhX79+2L59e6M2x48fR1BQECwsLBASEoLz5893bMeJqEMw3BCR6F555RXs3LkTmzdvxpkzZ7B8+XL85S9/QWJioq7NSy+9hLfffhu//vornJyc8Kc//Qm1tbUA6kPJzJkz8dhjj+HUqVNYu3YtXn31VXz88ce6/efMmYPPP/8cmzZtwrlz5/Dhhx/CysqqUR2rV6/GO++8g9TUVMjlcjz99NOd0n8ial9cFZyIRFVZWQlHR0ccPHgQwcHBuu3z589HVVUVnn32WUyYMAGff/45IiIiAAAlJSXo06cPPv74Y8ycORNPPPEErl27hgMHDuj2f/nll7F//36cOXMGFy5cwIABAxAfH4/Jkyc3qSEhIQETJkzAjz/+iEmTJgEA4uLiMH36dNy4cQMKhaKD/xaIqD1x5IaIRHX27FlUV1djypQpsLKy0r127dqFjIwMXbvbg4+9vT0GDBiAc+fOAQDOnTuHMWPGNDrumDFjcPHiRWg0GqSnp0Mmk2HcuHGt1uLn56f72cXFBQBQWFh4330kos4lF7sAIuretFotAGD//v3o3bt3o/fMzMwaBZw7SSQSAPX37DT83OD2QWlzc/M21WJiYtLk2A31EZHh4MgNEYlq8ODBMDMzQ05ODnx8fBq93NzcdO1+/vln3c/Xr1/HhQsXMHDgQN0xDh8+3Oi4KSkp6N+/P2QyGYYOHQqtVtvoHh4iMl4cuSEiUVlbW+PFF1/E8uXLodVqMXbsWJSVlSElJQVWVlbw8PAAAKxfvx4ODg5wdnbG6tWr4ejoiBkzZgAAXnjhBYwYMQKvv/46IiIicPToUbz//vuIiYkBAHh6emLu3Ll4+umnsWnTJvj7+yM7OxuFhYWYOXOmWF0nog7CcENEonv99dfh5OSE6OhoZGZmws7ODsOHD8eqVat0l4X+/ve/Y+nSpbh48SL8/f3xzTffwNTUFAAwfPhwfPHFF1izZg1ef/11uLi4YP369XjyySd1n7F582asWrUKixYtQnFxMdzd3bFq1SoxuktEHYxPSxFRl9bwJNP169dhZ2cndjlEZAB4zw0REREZFYYbIiIiMiq8LEVERERGhSM3REREZFQYboiIiMioMNwQERGRUWG4ISIiIqPCcENERERGheGGiIiIjArDDRERERkVhhsiIiIyKv8PfJNtMHVpF1wAAAAASUVORK5CYII=", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "y_avg = []\n", - "for y in epochs_y:\n", - " y_avg.append(np.mean(y))\n", - "\n", - "plt.plot(np.arange(len(epochs_x)), y_avg, marker = '.')\n", - "plt.xlabel('epoch')\n", - "plt.ylabel('average loss')\n", - "plt.ylim(0,)\n", - "for i, txt in enumerate(y_avg):\n", - " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom')\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkkAAAG2CAYAAABrrBJlAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAABe1ElEQVR4nO3deVhTV/4/8HcIEBYhiigJiCyKIIuIO6617tW64T6jVmtrRzuV2rqg0nFH7NTWpdVpf221OlW/U9dxKWKtWqpj3VAQRa0KLiAukLAGSO7vD2s0JSDBQBJ4v54nz8jNuSef0+uYN+eee69IEAQBRERERKTDytQFEBEREZkjhiQiIiIiPRiSiIiIiPRgSCIiIiLSgyGJiIiISA+GJCIiIiI9GJKIiIiI9GBIIiIiItKDIYmIiIhID4YkIiIiIj1MGpKOHz+O119/He7u7hCJRNi9e7fO+4IgYOHChXB3d4e9vT1eeeUVXLp0SaeNSqXC3//+d7i6usLR0RGDBw/GnTt3XvjZX3zxBXx8fGBnZ4e2bdvil19+MebQiIiIyMKZNCTl5+cjNDQU69at0/v+ypUrsWrVKqxbtw6nT5+GTCZDnz59kJubq20TGRmJXbt2Ydu2bUhISEBeXh4GDRoEtVpd7udu374dkZGRmD9/Ps6fP49u3bphwIABSE9PN/oYiYiIyDKJzOUBtyKRCLt27cLQoUMBPJlFcnd3R2RkJObMmQPgyayRm5sbYmNjMXXqVCgUCjRq1AibN2/G6NGjAQD37t2Dp6cnDhw4gH79+un9rI4dO6JNmzZYv369dlvLli0xdOhQxMTEVO9AiYiIyCJYm7qA8ty8eROZmZno27evdptEIkGPHj1w4sQJTJ06FWfPnkVJSYlOG3d3dwQHB+PEiRN6Q1JxcTHOnj2LuXPn6mzv27cvTpw4UW49KpUKKpVK+7NGo8Hjx4/RsGFDiESilxkqERER1RBBEJCbmwt3d3dYWVV8Qs1sQ1JmZiYAwM3NTWe7m5sb0tLStG1sbW3RoEGDMm2e7v9nDx8+hFqt1ttvefsAQExMDBYtWmTwOIiIiMj83L59G02aNKmwjdmGpKf+PEsjCMILZ24q08bQfqOiojBz5kztzwqFAk2bNsXt27fh7Oxc4WcRERGReVAqlfD09ISTk9ML25ptSJLJZACezBbJ5XLt9qysLO0skEwmQ3FxMbKzs3Vmk7KystC5c2e9/bq6ukIsFpeZNXq+X30kEgkkEkmZ7c7OzgxJREREFqYyS2XM9j5JPj4+kMlkiI+P124rLi7GsWPHtAGobdu2sLGx0WmTkZGB5OTkckOSra0t2rZtq7MPAMTHx5e7DxEREdU9Jp1JysvLw/Xr17U/37x5E4mJiXBxcUHTpk0RGRmJ5cuXw8/PD35+fli+fDkcHBwwbtw4AIBUKsWbb76JDz74AA0bNoSLiws+/PBDhISEoHfv3tp+e/XqhWHDhuHdd98FAMycORPjx49Hu3btEB4eji+//BLp6el45513avY/ABEREZktk4akM2fOoGfPntqfn675mThxIjZu3IjZs2ejsLAQ06ZNQ3Z2Njp27IhDhw7pnEf89NNPYW1tjVGjRqGwsBC9evXCxo0bIRaLtW1+//13PHz4UPvz6NGj8ejRIyxevBgZGRkIDg7GgQMH4OXlVQOjJiIiIktgNvdJsjRKpRJSqRQKhYJrkoiIiCyEId/fZrsmiYiIiMiUGJKIiIiI9GBIIiIiItKDIYmIiIhID4YkIiIiIj0YkoiIiIj0YEgiIiIi0oMhiYiIiEgPhiQiIiIiPRiSiIiIiPRgSCIiIiLSgyGJiIiISA+GJCIiIiI9GJKIiIiI9GBIIiIiItKDIYmIiIhID4YkIiIiIj0YkoiIiIj0YEgiIiIi0oMhiYiIiEgPhiQiIiIiPRiSiIiIiPRgSCIiIiLSgyGJiIiISA+GJCIiIiI9GJKIiIiI9GBIIiIiItKDIYmIiIhID4YkIiIiIj0YkoiIiIj0YEgiIiIi0oMhiYiIiEgPhiQiIiIiPRiSiIiIiPRgSCIiIiLSw+xDUm5uLiIjI+Hl5QV7e3t07twZp0+f1r4vEon0vj7++ONy+9y4caPefYqKimpiSERERGQBrE1dwItMmTIFycnJ2Lx5M9zd3bFlyxb07t0bKSkp8PDwQEZGhk77gwcP4s0330RERESF/To7OyM1NVVnm52dndHrJyIiIstk1iGpsLAQO3bswJ49e9C9e3cAwMKFC7F7926sX78eS5cuhUwm09lnz5496NmzJ3x9fSvsWyQSldmXiIiI6CmzPt1WWloKtVpdZobH3t4eCQkJZdrfv38f+/fvx5tvvvnCvvPy8uDl5YUmTZpg0KBBOH/+fIXtVSoVlEqlzouIiIhqL7MOSU5OTggPD8eSJUtw7949qNVqbNmyBadOnSpzmg0ANm3aBCcnJwwfPrzCfgMCArBx40bs3bsXW7duhZ2dHbp06YJr166Vu09MTAykUqn25enp+dLjIyIiIvMlEgRBMHURFfn9998xefJkHD9+HGKxGG3atEGLFi1w7tw5pKSk6LQNCAhAnz59sHbtWoM+Q6PRoE2bNujevTvWrFmjt41KpYJKpdL+rFQq4enpCYVCAWdnZ8MHRkRERDVOqVRCKpVW6vvbrNckAUCzZs1w7Ngx5OfnQ6lUQi6XY/To0fDx8dFp98svvyA1NRXbt283+DOsrKzQvn37CmeSJBIJJBKJwX0TERGRZTLr023Pc3R0hFwuR3Z2NuLi4jBkyBCd97/++mu0bdsWoaGhBvctCAISExMhl8uNVS4RERFZOLOfSYqLi4MgCPD398f169cxa9Ys+Pv7Y9KkSdo2SqUS//nPf/DJJ5/o7WPChAnw8PBATEwMAGDRokXo1KkT/Pz8oFQqsWbNGiQmJuLzzz+vkTERERGR+TP7kKRQKBAVFYU7d+7AxcUFERERWLZsGWxsbLRttm3bBkEQMHbsWL19pKenw8rq2aRZTk4O3n77bWRmZkIqlSIsLAzHjx9Hhw4dqn08REREZBnMfuG2uTJk4RcRERGZB0O+vy1mTRIRERFRTWJIIiIiItKDIYmIiIhID4YkIiIiIj0YkoiIiIj0YEgiIiIi0oMhiYiIiEgPhiQiIiIiPRiSiIiIiPRgSCIiIiLSgyGJiIiISA+GJCIiIiI9GJKIiIiI9GBIIiIiItKDIYmIiIhID4YkIiIiIj0YkoiIiIj0YEgiIiIi0oMhiYiIiEgPhiQiIiIiPRiSiIiIiPRgSCIiIiLSgyGJiIiISA+GJCIiIiI9GJKIiIiI9GBIIiIiItKDIYmIiIhID4YkIiIiIj0YkoiIiMiocnNzERkZCS8vL9jb26Nz5844ffq03rZTp06FSCTCZ599VmGfJSUlWLx4MZo1awY7OzuEhobixx9/LNPu7t27+Otf/4qGDRvCwcEBrVu3xtmzZ6s0Dusq7UVERERUjilTpiA5ORmbN2+Gu7s7tmzZgt69eyMlJQUeHh7adrt378apU6fg7u7+wj4XLFiALVu24KuvvkJAQADi4uIwbNgwnDhxAmFhYQCA7OxsdOnSBT179sTBgwfRuHFj/P7776hfv36VxiESBEGo0p51nFKphFQqhUKhgLOzs6nLISIiMguFhYVwcnLCnj17MHDgQO321q1bY9CgQVi6dCmAJzM+HTt2RFxcHAYOHIjIyEhERkaW26+7uzvmz5+P6dOna7cNHToU9erVw5YtWwAAc+fOxa+//opffvml3H4M+f7m6TYiIiIymtLSUqjVatjZ2elst7e3R0JCAgBAo9Fg/PjxmDVrFoKCgirVr0qlqrBPANi7dy/atWuHkSNHonHjxggLC8NXX31V5bEwJBEREZHRODk5ITw8HEuWLMG9e/egVquxZcsWnDp1ChkZGQCA2NhYWFtb47333qt0v/369cOqVatw7do1aDQaxMfHY8+ePdo+AeDGjRtYv349/Pz8EBcXh3feeQfvvfcevvvuuyqNhSGJiIiIjGrz5s0QBAEeHh6QSCRYs2YNxo0bB7FYjLNnz2L16tXYuHEjRCJRpftcvXo1/Pz8EBAQAFtbW7z77ruYNGkSxGKxto1Go0GbNm2wfPlyhIWFYerUqXjrrbewfv36Ko2DIYmIiIiMqlmzZjh27Bjy8vJw+/Zt/PbbbygpKYGPjw9++eUXZGVloWnTprC2toa1tTXS0tLwwQcfwNvbu9w+GzVqhN27dyM/Px9paWm4cuUK6tWrBx8fH20buVyOwMBAnf1atmyJ9PT0Ko3D7EPSiy4jfOONNyASiXRenTp1emG/O3bsQGBgICQSCQIDA7Fr167qHAYREVGd4+joCLlcjuzsbMTFxWHIkCEYP348Ll68iMTERO3L3d0ds2bNQlxc3Av7tLOzg4eHB0pLS7Fjxw4MGTJE+16XLl2Qmpqq0/7q1avw8vKqUv1mH5KmTJmC+Ph4bN68GUlJSejbty969+6Nu3fvatv0798fGRkZ2teBAwcq7PPkyZMYPXo0xo8fjwsXLmD8+PEYNWoUTp06Vd3DISKiF6iOe+w8PbXz51dRUZFOuy+++AI+Pj6ws7ND27ZtK7xKisoXFxeHH3/8ETdv3kR8fDx69uwJf39/TJo0CQ0bNkRwcLDOy8bGBjKZDP7+/to+JkyYgKioKO3Pp06dws6dO3Hjxg388ssv6N+/PzQaDWbPnq1t8/777+N///sfli9fjuvXr+P777/Hl19+qXNFnEEEM1ZQUCCIxWJh3759OttDQ0OF+fPnC4IgCBMnThSGDBliUL+jRo0S+vfvr7OtX79+wpgxYyrdh0KhEAAICoXCoM8mIqKKjRo1SggMDBSOHTsmXLt2TfjHP/4hODs7C3fu3NFpt2vXLiE0NFRwd3cXPv300wr7/PbbbwVnZ2chIyND5/W8bdu2CTY2NsJXX30lpKSkCDNmzBAcHR2FtLQ0Yw+x1tu+fbvg6+sr2NraCjKZTJg+fbqQk5NTbnsvL68yxzC8SzdhwPDRwr2cAkEQBOHo0aNCy5YtBYlEIjRs2FAYP368cPfu3TJ9/fe//xWCg4MFiUQiBAQECF9++aXO+4Z8f5t1SFIqlQIA4fDhwzrbO3XqJPTo0UMQhCchSSqVCo0aNRL8/PyEKVOmCPfv36+wX09PT2HVqlU621atWiU0bdq00rUxJBERGV9lfjkWBEG4c+eO4OHhISQnJ+v9gv2zb7/9VpBKpRW26dChg/DOO+/obAsICBDmzp1r0Bjo5W37LU3wmbtP8JqzT/CZu0/Y9pvxgqoh399mfcft5y8jbNmyJdzc3LB161acOnUKfn5+AIABAwZg5MiR8PLyws2bNxEdHY1XX30VZ8+ehUQi0dtvZmYm3NzcdLa5ubkhMzOz3FpUKhVUKpX2Z6VSaYQREhHR86rrHjsAkJeXBy8vL6jVarRu3RpLlizR3qm5uLgYZ8+exdy5c3X26du3L06cOPGSo6KnBEFArqoUOfkleFxQjOyCYuQUFONxfgly/vj5Xk4RjlzJ0u6jEYB5O5PRvUUjyKX2NVqvWYck4MllhJMnT4aHhwfEYjHatGmDcePG4dy5cwCA0aNHa9sGBwejXbt28PLywv79+zF8+PBy+/3zZYeCIFR4KWJMTAwWLVr0kqMhIqKKVOaX46rcYycgIAAbN25ESEgIlEolVq9ejS5duuDChQvw8/PDw4cPoVarDf4Fui5TawQoC5+EnadBR1/oyf5je3bBk22lGsMf9KEWBNx6WMCQ9GdPLyPMz8+HUqmEXC7H6NGjdS75e55cLoeXlxeuXbtWbp8ymazMX/qsrKwy/+d4XlRUFGbOnKn9WalUwtPT08DREBHRi1T0y/HTe+ycO3fOoHvsdOrUSefK5y5duqBNmzZYu3Yt1qxZo91u6C/QtUWJWvNHwCnB4/ziPwKO7p+z84uftSkohqKwBFV9sJm9jRgNHGxQ38EWLo62qO9ggwYOtmjgaAuxCPjs8DU837VYJIK3q4NRxmoIsw9JTzk6OsLR0VF7GeHKlSv1tnv06BFu374NuVxebl/h4eGIj4/H+++/r9126NAhdO7cudx9JBJJuafviIjIeCr65fj5e+w8pVar8cEHH+Czzz7DrVu3KvUZVlZWaN++vfYXaldXV4jFYoN/gTZHRSVqPH4+0JQXep6e7sovQa6qtMqf5ySxRgNH23JDT4Onf3awRQPHJ3+2sxFX2KdMaod5O5OhFgSIRSIsHx5c47NIgAWEpLi4OAiCAH9/f1y/fh2zZs3SXkaYl5eHhQsXIiIiAnK5HLdu3cK8efPg6uqKYcOGafuYMGECPDw8EBMTAwCYMWMGunfvjtjYWAwZMgR79uzB4cOHdZ7/QkREpqXvl+OIiAj07t1bp12/fv0wfvx4TJo0qdJ9C4KAxMREhISEAABsbW3Rtm1bxMfH63x/xMfH69yHpyYJgoA8Vak26OgNPX+c3np2SqsYRSWaKn2eSATUt9cNN2VCj8MfocfxyZ/rO9jARmz8uwmNbt8U3Vs0wq2HBfB2dTBJQAIsICQpFApERUXhzp07cHFxQUREBJYtWwYbGxuUlpYiKSkJ3333HXJyciCXy9GzZ09s374dTk5O2j7S09NhZfXsIHbu3Bnbtm3DggULEB0djWbNmmH79u3o2LGjKYZIRETPqeiXYxsbGzRs2FCnfXn32Hn+l+NFixahU6dO8PPzg1KpxJo1a5CYmIjPP/9cu8/MmTMxfvx4tGvXDuHh4fjyyy+Rnp6Od95556XHpNEIUBQ+W5vz51NXT4OO9s9/rN8pUVftfJa1lUg36Pwxi/P0z/pmepztbSC2Mp9Ti3KpvcnC0VNmH5JGjRqFUaNG6X3P3t6+UnfnPHr0aJltI0aMwIgRI162PCIiMrKKfjmurD//cpyTk4O3334bmZmZkEqlCAsLw/Hjx9GhQwdtm9GjR+PRo0dYvHgxMjIyEBwcjAMHDpS5W3OJWoOcp6er8v906qqcdTyKwhJUYb0yAMDOxkrndJU29OiZ6Xn653oS6zqxlqq6iQShqsuu6jalUgmpVAqFQgFnZ2dTl0NERM/JUBTi5sN8+Lg6VjgbUVSi1l6BlVNQ/Mdl6SXIyS/+Y1anbOh52fU79R11T109DTrlnd6yt614/Q4ZxpDvb7OfSSIiIjLEt7/exOJ9KRAEQATg1ZaN4VHf/o+wozsDVFiirtJniESA1N5G76mr8kJPfXtb2Fqb/dPA6DkMSUREZJHUGgG3HuXjcobyj1cuku8qkJX77Ma/AoCfLmeV3wmerN95EmaehJoGDjZ/BBv9V2aZ4/odqh4MSUREZPbyVKW48kcYSsnIxeUMJVIzcys9EzQ8zB3BHvV1gs7T4MP1O1QehiQiIjIbgiDgTnYhUrSzQ09miNIfF+htb2djBX83JwS6O6Ol3BmNnSSY9u9zOoukxSIRZvUPMPmVUmR5GJKIiMgkikrUSM3M/WN26EkgupKRW+7CaJmzHVrKndBS7qx9+bg6ljntFTM8xCxuREiWjyGJiIiqlSAIuK9U6YShyxlK3HyYr/eyeBuxCM0bOyFQ7oyW8if/GyB3houjbaU+z1xuREiWjyGJiIiMprhUg+tZeX86XaZEdkGJ3vYNHW3/mBV6NkPUrFG9l74KzBxuREiWjyGJiIiq5FGeCpf/WET9dJbo9wd5eu8SbSUCmjWq99ypsiczRI2cJFw0TWaLIYmIiCpUqtbg5sP8P2aHnoWi5y+1f56TnTVayp21p8tayp3Rws3phQ81JTI3DElERKSlKCzRXmp/OSMXlzOfXGqvKtX/0FTvhg46C6lbyp3gUd+es0NUKzAkEVGtkpubi+joaOzatQtZWVkICwvD6tWr0b59ewDAwoULsW3bNty+fVv75Pdly5ZV+IDrnTt3Yvny5bh+/TpKSkrg5+eHDz74AOPHj9e2KS0txcKFC/Hvf/8bmZmZkMvleOONN7BgwQKdZ4iZC41GQPrjgudOlT2ZIbqbU6i3vYOtGP6yZ+uGAuVO8Jc5o56EXyNUe/FvNxHVKlOmTEFycjI2b94Md3d3bNmyBb1790ZKSgo8PDzQokULrFu3Dr6+vigsLMSnn36Kvn374vr162jUqJHePl1cXDB//nwEBATA1tYW+/btw6RJk9C4cWP069cPABAbG4sNGzZg06ZNCAoKwpkzZzBp0iRIpVLMmDGjJv8TlFFQXIormc+tHbr3ZHYov1j/jRg96tuXudTey8UBVrzDNNUxfMBtFfEBt0Tmp7CwEE5OTtizZw8GDhyo3d66dWsMGjQIS5cuLbPP0/8vHz58GL169ar0Z7Vp0wYDBw7EkiVLAACDBg2Cm5sbvv76a22biIgIODg4YPPmzS8xqsoTBAH3FEW4fO+P02WZT06Z3XqUD33/0ttaW6GFWz20lDlrb8bYUuYMqYNNjdRLZAp8wC0R1UmlpaVQq9Wws7PT2W5vb4+EhIQy7YuLi/Hll19CKpUiNDS0Up8hCAKOHDmC1NRUxMbGard37doVGzZswNWrV9GiRQtcuHABCQkJ+Oyzz15qTOUpKlE/udT+3nM3YszMhaJQ/6X2jZwkOleVtZQ7w9fVEdZi8zsVSGQuGJKIqNZwcnJCeHg4lixZgpYtW8LNzQ1bt27FqVOn4Ofnp223b98+jBkzBgUFBZDL5YiPj4erq2uFfSsUCnh4eEClUkEsFuOLL75Anz59tO/PmTMHCoUCAQEBEIvFUKvVWLZsGcaOHfvS48rKLdK5quxyhhK/P8iHWs+dGK2tRH9cav/kdNnTGSLXepKXroOormFIIqJaZfPmzZg8eTI8PDwgFovRpk0bjBs3DufOndO26dmzJxITE/Hw4UN89dVXGDVqFE6dOoXGjRuX26+TkxMSExORl5eHn376CTNnzoSvry9eeeUVAMD27duxZcsWfP/99wgKCkJiYiIiIyPh7u6OiRMnVqr2ErUGvz/Ie3Zl2R+B6GFesd729R1s0FLmrHMzRj+3epBY81J7ImPgmqQq4pokIvOWn58PpVIJuVyO0aNHIy8vD/v379fb1s/PD5MnT0ZUVFSl+58yZQpu376NuLg4AICnpyfmzp2L6dOna9ssXboUW7ZswZUrV8rsn1NQXOa+Q9fu56FYXfZSe5EI8Gno+NzM0JNAJHO246X2RAbimiQiqvMcHR3h6OiI7OxsxMXFYeXKleW2FQQBKpX+GyNWdp+CgoIyl/qLxWJoNM/PDj0LRRmKIr391pNYI0DmpHPfIX+ZExxs+c81UU3j/+uIqFaJi4uDIAjw9/fH9evXMWvWLPj7+2PSpEnIz8/HsmXLMHjwYMjlcjx69AhffPEF7ty5g5EjR2r7mDBhAjw8PBATEwMAiImJQbt27dCsWTMUFxfjwIED+O6777B+/XrtPq+//jqWLlsGlaQBNPWbIOHUGez/YiXqhfRGr0+O6a3V08X+udNlT+5Q3aSBPS+1JzITDElEVKsoFApERUXhzp07cHFxQUREBJYtWwYbGxuo1WpcuXIFmzZtwsOHD9GwYUO0b98ev/zyC4KCgrR9XL9xCw/zi5GhKIRcao/8/HxMmzYNd+7cgb29PQICAvDZhq/h2akvPjt8FZczlLjabCRyk7Px4fvvQVOggLieCxxC+qFelzGws7GCv5vufYcC5E5wtuOl9kTmjGuSqohrkohqp+2n0xG1Mwka4clDWRcNDkZIE6nOlWVXMnKRqyrVu7/M2a7MjRh9XB0h5uwQkVngmiQioirIUBRqAxIAaAQgek+y3rY2YhGaN3bSue9QS7kzXBxta7BiIqpODElEVOfdyynEj8mZ2H46HXpuPQRnO2u0alJfZ4aoWaN6sLXmjRiJajOGJCKqk24/LsDB5AwcSMpE4u2ccttZiYAfI7vBvb5DzRVHRGaBIYmI6owbD/JwMDkTB5MzkHxXqd0uEgHtvBpgQLAcJWoNVv6YCrUgQCwSYfnwYAYkojqKIYmIai1BEHAtKw8HkjLwY3ImrmTmat+zEgGdfBtiQLAM/YJkaOz87Hlvg1u749bDAni7OkAutTdF6URkBhiSiKhWEQQBKRlKHEx6MmP0+4N87XvWViJ0bu6KAcEy9A10Q8Nynmcml9ozHBERQxIRWT5BEHDxjgIHkp/MGKU9KtC+Zyu2Qjc/V/QPlqFPoBvqO/DqMyKqHIYkIrJIGo2A87ezcSApEz8mZ+JuTqH2PYm1FV7xb4QBwXK82rIxb9pIRFXCkEREFkOtEXD61mMcTMrAj5cycV/57NlpDrZi9AxojAHBMvT0bwxHCf95I6KXw39FiMislao1+N+NxziQnIFDlzLxMK9Y+149iTV6t2yMASFy9GjRCHY2YhNWSkS1DUMSEZmd4lINfv39IQ4mZSA+5T6yC0q070ntbdAn0A2vhcjQpbkrJNYMRkRUPRiSiMgsFJWo8cu1P4LR5fvILXr2bDQXR1v0C3LDgGA5wps1hI2Yd7omourHkEREJlNQXIpjqQ9wIDkTRy7fR36xWvteIycJ+gfJMCBEhg7eLrBmMCKiGsaQREQ1Kk9ViiNXsnAwKQM/p2ahqESjfU8utUP/YBleC5GjTdMGEFuJTFgpEdV1Zh+ScnNzER0djV27diErKwthYWFYvXo12rdvj5KSEixYsAAHDhzAjRs3IJVK0bt3b6xYsQLu7u7l9rlx40ZMmjSpzPbCwkLY2dnp2YOIXoaisAQ/Xb6PA0mZOH7tAYpLnwUjTxd7DAiWY0CwDKFN6sOKwYiIzITZh6QpU6YgOTkZmzdvhru7O7Zs2YLevXsjJSUF9erVw7lz5xAdHY3Q0FBkZ2cjMjISgwcPxpkzZyrs19nZGampqTrbGJCIjCc7vxjxKfdxIDkDv15/iBK1oH3Px9URA/6YMQpyd4ZIxGBEROZHJAiC8OJmplFYWAgnJyfs2bMHAwcO1G5v3bo1Bg0ahKVLl5bZ5/Tp0+jQoQPS0tLQtGlTvf1u3LgRkZGRyMnJqXJtSqUSUqkUCoUCzs7OVe6HqDZ5kKvCoZRMHEzKxMkbj6DWPPvnpYVbPfQPluO1EBn83ZwYjIjIJAz5/jbrmaTS0lKo1eoyMzz29vZISEjQu49CoYBIJEL9+vUr7DsvLw9eXl5Qq9Vo3bo1lixZgrCwsHLbq1QqqFTPblynVCrLbUtUl2QqihB3KRMHkjJw+tZjPJeLECh3xmshMvQPlqN543qmK5KIqAoMDklHjx7FK6+8Ug2llOXk5ITw8HAsWbIELVu2hJubG7Zu3YpTp07Bz8+vTPuioiLMnTsX48aNqzAdBgQEYOPGjQgJCYFSqcTq1avRpUsXXLhwQW+/ABATE4NFixYZbWxEluxOdgF+TM7EweRMnE3L1nkvtIkUA0KerDHyauhoogqJiF6ewafb7Ozs4OHhgUmTJmHixInw9PSsrtoAAL///jsmT56M48ePQywWo02bNmjRogXOnTuHlJQUbbuSkhKMHDkS6enpOHr0qEGnwDQaDdq0aYPu3btjzZo1etvom0ny9PTk6TaqM9Ie5eNgciYOJmXgwh2FznttvRpgQLAM/YNlaNLAwUQVEhG9WLWebrt37x62bNmCjRs3YuHChejVqxfefPNNDB06FLa2xn+6drNmzXDs2DHk5+dDqVRCLpdj9OjR8PHx0bYpKSnBqFGjcPPmTRw5csTg0GJlZYX27dvj2rVr5baRSCSQSCRVHgeRJbqelYcfkzNwICkTKRnPTjGLREAHbxe8FiJHvyAZZFJe9EBEtc9LLdxOTEzEN998g61bt0Kj0eAvf/kL3nzzTYSGhhqzRh3Z2dnw8fHBypUr8fbbb2sD0rVr1/Dzzz+jUaNGBvcpCAI6dOiAkJAQfPPNN5Xahwu3qTYSBAGp93NxMCkTB5MzcPV+nvY9sZUI4b4NMSBEhr6BMjRy4i8NRGR5DPn+fumr2+7du4cvv/wSK1asgLW1NYqKihAeHo4NGzYgKCjoZboGAMTFxUEQBPj7++P69euYNWsWJBIJEhISIBKJEBERgXPnzmHfvn1wc3PT7ufi4qKd2ZowYQI8PDwQExMDAFi0aBE6deoEPz8/KJVKrFmzBps3b8avv/6KDh06VKouhiSqLQRBwKV7ShxIysCPyZm48TBf+56NWIQuzV3xWrAcfQLd0MDR+LPFREQ1qdqvbispKcGePXvwzTffID4+Hu3atcO6deswduxYPH78GHPmzMHIkSN11gxVlUKhQFRUFO7cuQMXFxdERERg2bJlsLGxwa1bt7B3714AT24L8Lyff/5Zu8A8PT0dVlbPHmmQk5ODt99+G5mZmZBKpQgLC8Px48crHZCILJ0gCEi8nfNkjVFyBm4/LtS+Z2tthe5+jfBaiAy9WrpBam9jwkqJiEzH4Jmkv//979i6dSsA4K9//SumTJmC4OBgnTbp6enw9vaGRqPR10WtwJkksjQajYCz6dk4kJSBuORM3FMUad+zs7FCT//GGBAix6sBjVFPYtZ3ByEiqrJqnUlKSUnB2rVrERERUe5CbXd3d/z888+Gdk1ERlaq1uC3W49xMCkTcZcykZX77ApNR1sxXm3phteCZejh3wgOtgxGRETPM/ix2j/99BPGjh1b4ZVs1tbW6NGjx0sVRmQKubm5iIyMhJeXF+zt7dG5c2ecPn1a+/7OnTvRr18/uLq6QiQSITEx8YV97ty5E+3atUP9+vXh6OiI1q1bY/PmzQZ9riFK1Bocv/oAUTsvouPynzDuq1PY/L80ZOWq4GRnjeFtPPDVhHY4G90Ha8eGYUCInAGJiEgPg/9ljImJgZubGyZPnqyz/ZtvvsGDBw8wZ84coxVHVNMqelagh4cH8vPz0aVLF4wcORJvvfVWpfp0cXHB/PnzERAQAFtbW+zbtw+TJk1C48aN0a9fv0p97ouoStX49fpDHEjKRHzKfSgKS7Tv1XewQb9AGfqHyNClmStsrQ3+3YiIqE4yeE2St7c3vv/+e3Tu3Fln+6lTpzBmzBjcvHnTqAWaK65Jqn0MeVbgrVu34OPjg/Pnz5e5aKAy2rRpg4EDB2LJkiVVekYhABSVqHHs6gMcTMrAT5ezkKsq1b7nWs8W/YJkGBAsR0dfF9iIGYyIiIBqXpOUmZkJuVxeZnujRo2QkZFhaHdEZqMqzwo0lCAIOHLkCFJTUxEbG2vw5+arSnE09QEOJGfg5ytZKChWa99zc5ZgQLAc/YNlaO/tArEVHyBLRPQyDA5Jnp6e+PXXX3XueA0Av/76K9zd3Y1WGFFNM/RZgYZQKBTw8PCASqWCWCzGF198gT59+lTqc3OLSnDkShYOJGXgaOoDqEqfXTXqUd8eA4JlGBAiQ5hnA1gxGBERGY3BIWnKlCmIjIxESUkJXn31VQBPFnPPnj0bH3zwgdELJKpJmzdvxuTJk+Hh4aF9VuC4ceNw7ty5l+rXyckJiYmJyMvLw08//YSZM2fC19dXey+vP39uaOswdO0/FEkXEtF2yWEUq58FI6+GDhgQ/OQBsq2aSCESMRgREVUHg0PS7Nmz8fjxY0ybNg3FxcUAnjz0ds6cOYiKijJ6gUQ1qTLPCqwKKysrNG/eHMCTtUaXL19GTEyMNiQ1a9YMO/cfwn/P3sS+szeQ+BC4smsFBLuGKFZr0KyRI14LkWNAsBwt5U4MRkRENcDgkCQSiRAbG4vo6GhcvnwZ9vb28PPz48NfqVZxdHSEo6MjsrOzERcXh5UrVxq1f0EQoFKpkJVbhLhL93EwKQP/u/EImj8uo1AX5aH41nkMf2c2lr7fHX5uTkb9fCIierEq3xylXr16aN++vTFrITI5fc8K9Pf3x6RJkwAAjx8/Rnp6Ou7duwcASE1NBQDIZDLIZDIAZZ8VGBMTg3bt2qFZs2YoLi7Gth17sGnTd2g95kN0XP4TBAEovHEWABAcGICWjgU4smkV3FsFYsvKubCx4WNBiIhMoUoh6fTp0/jPf/6D9PR07Sm3p3bu3GmUwohMoaJnBQLA3r17tYEJAMaMGQMA+Mc//oGFCxcCAK7fuIWH+cXIUBRCLrVHfn4+3n7nb7h75w5gbQur+h5oMHAmHjbpCghAWNP6aGRXH3GbPkPCnrtI0fO5RERU8wy+T9K2bdswYcIE9O3bF/Hx8ejbty+uXbuGzMxMDBs2DN9++2111WpWeJ8k0mf76XRE7UyCRgCsRECfQDfcyylC0l2Fto1IBLT3ckH/YBn6B8vgXt/ehBUTEdUt1XqfpOXLl+PTTz/F9OnT4eTkhNWrV8PHxwdTp07Ve/8koroiQ1GoDUgAoBGAuEv3ATwJTJ18G2JAsAz9gmRo7GxXQU9ERGQODA5Jv//+u/auwBKJBPn5+RCJRHj//ffx6quvYtGiRUYvksgS3HyYrw1Iz3urmw/e6dEMDevx4gYiIkti8LMKXFxckJubCwDw8PBAcnIyACAnJwcFBQXGrY7Igvi4OpbZJhaJMLmrDwMSEZEFMngmqVu3boiPj0dISAhGjRqFGTNm4MiRI4iPj0evXr2qo0Yii5D2SPeXBLFIhOXDgyGXcs0REZElMjgkrVu3DkVFRQCAqKgo2NjYICEhAcOHD0d0dLTRCySyBEUlaszdcREAMLS1O0a3bwpvVwcGJCIiC2bQ1W2lpaX497//jX79+mnvCVNX8eo2el7Mwcv417EbkDnb4dDM7nC246X7RETmyJDvb4PWJFlbW+Nvf/sbVCrVSxVIVJtcvJODr47fAAAsHRrMgEREVEsYvHC7Y8eOOH/+fHXUQmRxStQazP7hIjQCMDjUHb0D3UxdEhERGYnBa5KmTZuGDz74AHfu3EHbtm3h6Kh7RU+rVq2MVhyRufvXsd9xJTMXDRxs8I/XA01dDhERGZHBd9y2sio7+SQSiSAIAkQiEdRqtdGKM2dck0TXs3Lx2uoEFKs1WD2mNYa09jB1SURE9ALVesftmzdvVrkwotpCrREw+4eLKFZr0NO/EQaHupu6JCIiMjKDQ5KXl1d11EFkUTafvIVz6TlwtBVj2bAQiEQiU5dERERGZnBI+u677yp8f8KECVUuhsgS3MkuwMq4VADA3Nda8gG1RES1lMEhacaMGTo/l5SUoKCgALa2tnBwcGBIolpNEATM25WMgmI1Oni74C8dmpq6JCIiqiYG3wIgOztb55WXl4fU1FR07doVW7durY4aiczGznN3cfzqA9haW2FFRAisrHiajYiotjI4JOnj5+eHFStWlJllIqpNHuSqsHhfCgDg/d4t4NuonokrIiKi6mSUkAQAYrEY9+7dM1Z3RGZn4d5LUBSWINjDGW918zF1OUREVM0MXpO0d+9enZ8FQUBGRgbWrVuHLl26GK0wInPyY3Im9idlQGwlQmxEK1iLjfb7BRERmSmDQ9LQoUN1fhaJRGjUqBFeffVVfPLJJ8aqi8hsKApL8NGeZADA1O6+CHKXmrgiIiKqCQaHJI1GUx11EJmtmAOXkZWrgq+rI97r5WfqcoiIqIbwnAFRBX69/hDbTt8GAMSOaAU7G7GJKyIioppicEgaMWIEVqxYUWb7xx9/jJEjRxqlKCJzUFBciqidSQCACeFeaO/tYuKKiIioJhkcko4dO4aBAweW2d6/f38cP37cKEURmYNVh64i/XEB3KV2mN0/wNTlEBFRDTM4JOXl5cHW1rbMdhsbGyiVSqMU9bzc3FxERkbCy8sL9vb26Ny5M06fPq19XxAELFy4EO7u7rC3t8crr7yCS5cuvbDfHTt2IDAwEBKJBIGBgdi1a5fRayfLdT49G9/8+uRhzsuGh6CexODle0REZOEMDknBwcHYvn17me3btm1DYGCgUYp63pQpUxAfH4/NmzcjKSkJffv2Re/evXH37l0AwMqVK7Fq1SqsW7cOp0+fhkwmQ58+fZCbm1tunydPnsTo0aMxfvx4XLhwAePHj8eoUaNw6tQpo9dPlqe4VIM5Oy5CIwDDwjzQ07+xqUsiIiITEAmCIBiyw969exEREYFx48bh1VdfBQD89NNP2Lp1K/7zn/+UuUXAyygsLISTkxP27Nmjc4qvdevWGDRoEJYsWQJ3d3dERkZizpw5AACVSgU3NzfExsZi6tSpevsdPXo0lEolDh48qN3Wv39/NGjQoNKPVlEqlZBKpVAoFHB2dn6JUZK5+ezwVXx2+BoaOtoifmYPuDiWnTklIiLLZMj3t8EzSYMHD8bu3btx/fp1TJs2DR988AHu3LmDw4cPGzUgAUBpaSnUajXs7Ox0ttvb2yMhIQE3b95EZmYm+vbtq31PIpGgR48eOHHiRLn9njx5UmcfAOjXr1+F+6hUKiiVSp0X1T5X7+fi85+vAwAWDg5iQCIiqsOqtNBi4MCBehdvG5uTkxPCw8OxZMkStGzZEm5ubti6dStOnToFPz8/ZGZmAgDc3Nx09nNzc0NaWlq5/WZmZurd52l/+sTExGDRokUvMRoyd2qNgNk/XESJWkDvlm4Y1Epu6pKIiMiEDJ5JOn36tN61O6dOncKZM2eMUtTzNm/eDEEQ4OHhAYlEgjVr1mDcuHEQi5/dr0Yk0n0SuyAIZbb9maH7REVFQaFQaF+3b9+uwmjInG08cQuJt3PgJLHG0qHBL/w7REREtZvBIWn69Ol6A8Ldu3cxffp0oxT1vGbNmuHYsWPIy8vD7du38dtvv6GkpAQ+Pj6QyWQAUGYGKCsrq8xM0fNkMpnB+0gkEjg7O+u8qPZIf1SAf8alAgDmDWwJmdTuBXsQEVFtZ3BISklJQZs2bcpsDwsLQ0pKilGK0sfR0RFyuRzZ2dmIi4vDkCFDtEEpPj5e2664uBjHjh1D586dy+0rPDxcZx8AOHToUIX7UO0lCAKidl1EYYka4b4NMaa9p6lLIiIiM2DwmiSJRIL79+/D19dXZ3tGRgasrY1/L5m4uDgIggB/f39cv34ds2bNgr+/PyZNmgSRSITIyEgsX74cfn5+8PPzw/Lly+Hg4IBx48Zp+5gwYQI8PDwQExMDAJgxYwa6d++O2NhYDBkyBHv27MHhw4eRkJBg9PrJ/P3nzB38ev0RJNZWiBkewtNsREQEoAohqU+fPoiKisKePXsglT55GnpOTg7mzZuHPn36GL1AhUKBqKgo3LlzBy4uLoiIiMCyZctgY2MDAJg9ezYKCwsxbdo0ZGdno2PHjjh06BCcnJy0faSnp8PK6tmkWefOnbFt2zYsWLAA0dHRaNasGbZv346OHTsavX4yb1nKIizZ/2QG9IO+LeDt6mjiioiIyFwYfJ+ku3fvonv37nj06BHCwsIAAImJiXBzc0N8fDw8PevGqQreJ6l2eGfzWfx4KROtmkix82+dYS3mM5+JiGozQ76/DZ5J8vDwwMWLF/Hvf/8bFy5cgL29PSZNmoSxY8dqZ3eILMHBpAz8eCkT1lYixEa0YkAiIiIdVVpE5OjoiLffftvYtRDVmJyCYkTvefKMv2mvNENLOWcDiYhIV5VXWqekpCA9PR3FxcU62wcPHvzSRRFVt6X7L+NhngrNG9fD9Febm7ocIiIyQwaHpBs3bmDYsGFISkqCSCTC0yVNT68IUqvVxq2QyMiOX32AH87egUgExEa0gsRa/OKdiIiozjF4EcaMGTPg4+OD+/fvw8HBAZcuXcLx48fRrl07HD16tBpKJDKefFUponYmAQAmhnujrVcDE1dERETmyuCZpJMnT+LIkSNo1KgRrKysYGVlha5duyImJgbvvfcezp8/Xx11EhnFx3GpuJtTCI/69pjVz9/U5RARkRkzeCZJrVajXr16AABXV1fcu3cPAODl5YXU1FTjVkdkRGfTsrHp5C0AQMzwEDhKjH/zUyIiqj0M/pYIDg7GxYsX4evri44dO2LlypWwtbXFl19+WeYu3ETmQlWqxpwdFyEIwIi2TdC9RSNTl0RERGbO4JC0YMEC5OfnAwCWLl2KQYMGoVu3bmjYsCG2b99u9AKJjOHzI9dxPSsPrvUkWDCwpanLISIiC2BwSOrXr5/2z76+vkhJScHjx4/RoEEDPvOKzNLlDCW+OPo7AGDJkCDUd7A1cUVERGQJjLIow8XFxRjdEBldqVqDOTsuolQjoH+QDANC5KYuiYiILASfw0C12je/3sTFOwo421lj8ZAgU5dDREQWhCGJaq1bD/PxyaGrAIAFAwPR2NnOxBUREZElYUiiWkkQBMzdeRGqUg26NG+Ike2amLokIiKyMAaHpOPHj6O0tLTM9tLSUhw/ftwoRRG9rG2nb+N/Nx7D3kaMmGGteFEBEREZzOCQ1LNnTzx+/LjMdoVCgZ49exqlKKKXkakowvL9lwEAH/bzR9OGDiauiIiILJHBIUkQBL2/lT969AiOjo5GKYqoqgRBwILdSchVlaK1Z3280dnb1CUREZGFqvQtAIYPHw4AEIlEeOONNyCRSLTvqdVqXLx4EZ07dzZ+hUQG2HcxA4cvZ8FGLMLKEa0gtuJpNiIiqppKhySpVArgyW/qTk5OsLe3175na2uLTp064a233jJ+hUSV9Di/GAv3XgIATO/ZHC3cnExcERERWbJKh6Rvv/0WAODt7Y0PP/yQp9bI7CzZl4JH+cXwd3PCtFeam7ocIiKycAavSZo9e7bOmqS0tDR89tlnOHTokFELIzLEz6lZ2HX+LqxEQOyIVrC15t0tiIjo5Rj8TTJkyBB89913AICcnBx06NABn3zyCYYMGYL169cbvUCiF8lTlWL+ziQAwOQuPmjtWd+0BRERUa1gcEg6d+4cunXrBgD44YcfIJPJkJaWhu+++w5r1qwxeoFEL7Lyxyu4pyhCUxcHzOzbwtTlEBFRLWFwSCooKICT05MFsYcOHcLw4cNhZWWFTp06IS0tzegFElXkt5uP8d3JJ3/vVgwPgYOtUZ7ZTEREZHhIat68OXbv3o3bt28jLi4Offv2BQBkZWXB2dnZ6AUSlaeoRI25Oy4CAMa090Tn5q4mroiIiGoTg0PSRx99hA8//BDe3t7o0KEDwsPDATyZVQoLCzN6gUTlWfPTNdx4mI/GThJEvdbS1OUQEVEtY/C5iREjRqBr167IyMhAaGiodnuvXr0wbNgwoxZHVJ7kuwr86/gNAMCSocGQ2tuYuCIiIqptqnSdtEwmg5OTE+Lj41FYWAgAaN++PQICAoxaHJE+pWoN5uy4CLVGwMAQOfoFyUxdEhER1UIGh6RHjx6hV69eaNGiBV577TVkZGQAAKZMmYIPPvjA6AUS/dlXv9zEpXtKSO1tsHBwkKnLISKiWsrgkPT+++/DxsYG6enpcHB49nT10aNH48cffzRqcUR/duNBHj49fBUA8NGgQDRykrxgDyIioqoxeE3SoUOHEBcXhyZNmuhs9/Pz4y0AqFppNALm7khCcakG3Vs0wvA2HqYuiYiIajGDZ5Ly8/N1ZpCeevjwISQS/lZP1effv6Xjt1uP4WArxvJhwTqPxyEiIjI2g0NS9+7dtY8lAQCRSASNRoOPP/4YPXv2NGpxRE/dzSnEigOXAQCz+/mjSYOyQZ2IiMiYDD7d9vHHH+OVV17BmTNnUFxcjNmzZ+PSpUt4/Pgxfv311+qokeo4QRAwf1cS8ovVaOvVAOPDvU1dEhER1QEGzyQFBgbi4sWL6NChA/r06YP8/HwMHz4c58+fR7NmzaqjRqrj9iTew9HUB7AVWyE2IgRiK55mIyKi6mdwSEpPT4ebmxsWLVqEffv24cCBA1i6dCnkcjnS09ONWlxpaSkWLFgAHx8f2Nvbw9fXF4sXL4ZGo9G2EYlEel8ff/xxuf1u3LhR7z5FRUVGrZ9e3qM8FRb99xIA4L1ezdG8sZOJKyIiorrC4NNtPj4+yMjIQOPGjXW2P3r0CD4+PlCr1UYrLjY2Fhs2bMCmTZsQFBSEM2fOYNKkSZBKpZgxYwYAaO/T9NTBgwfx5ptvIiIiosK+nZ2dkZqaqrPNzs7OaLWTcSz6bwqyC0oQIHPC1B6cqSQioppjcEgSBEHvVUV5eXlGDxknT57EkCFDMHDgQACAt7c3tm7dijNnzmjbyGS6d1ves2cPevbsCV9f3wr7FolEZfYl83I45T72XrgHKxHw8YhQ2IirdIN4IiKiKql0SJo5cyaAJ+EiOjpa5zYAarUap06dQuvWrY1aXNeuXbFhwwZcvXoVLVq0wIULF5CQkIDPPvtMb/v79+9j//792LRp0wv7zsvLg5eXF9RqNVq3bo0lS5ZU+IBelUoFlUql/VmpVBo8Hqo8ZVEJFuxOBgC81c0XIU2kJq6IiIjqmkqHpPPnzwN4MpOUlJQEW1tb7Xu2trYIDQ3Fhx9+aNTi5syZA4VCgYCAAIjFYqjVaixbtgxjx47V237Tpk1wcnLC8OHDK+w3ICAAGzduREhICJRKJVavXo0uXbrgwoUL8PPz07tPTEwMFi1a9NJjospZcfAKMpVF8G7ogMjeLUxdDhER1UEiQRAEQ3aYNGkSVq9eDWdn5+qqSWvbtm2YNWsWPv74YwQFBSExMRGRkZFYtWoVJk6cWKZ9QEAA+vTpg7Vr1xr0ORqNBm3atEH37t2xZs0avW30zSR5enpCoVDUyH+LuuTk748w9qv/AQC2vtUJ4c0amrgiIiKqLZRKJaRSaaW+vw1ek/Ttt99WuTBDzZo1C3PnzsWYMWMAACEhIUhLS0NMTEyZkPTLL78gNTUV27dvN/hzrKys0L59e1y7dq3cNhKJhHcUrwFFJWpE7bwIABjXsSkDEhERmYxZr4QtKCiAlZVuiWKxWOcWAE99/fXXaNu2LUJDQw3+HEEQkJiYCLlcXuVayTg+PXwVtx4VQOZsh7kDAkxdDhER1WEGzyTVpNdffx3Lli1D06ZNERQUhPPnz2PVqlWYPHmyTjulUon//Oc/+OSTT/T2M2HCBHh4eCAmJgYAsGjRInTq1Al+fn5QKpVYs2YNEhMT8fnnn1f7mKh8F+/k4KvjNwAAS4cGw9nOxsQVERFRXWbWIWnt2rWIjo7GtGnTkJWVBXd3d0ydOhUfffSRTrtt27ZBEIRyF3Snp6frzEjl5OTg7bffRmZmJqRSKcLCwnD8+HF06NChWsdD5StRazD7h4vQCMDgUHf0DnQzdUlERFTHGbxwm54wZOEXvdi6I9fwz0NX0cDBBodn9kDDelz/RURExmfI97dZr0miuuF6Vi7W/HQdAPCP14MYkIiIyCwwJJFJqTUCZv9wEcVqDXr6N8KQ1u6mLomIiAgAQxKZ2OaTt3AuPQeOtmIsGxai95E3REREpsCQRCZzJ7sAK+OePGR47mst4V7f3sQVERERPcOQRCYhCALm7UpGQbEaHbxd8JcOTU1dEhERkQ6GJDKJnefu4vjVB7C1tsKKiBBYWfE0GxERmReGJKpxD3JVWLwvBQAQ2dsPvo3qmbgiIiKishiSqMYt3HsJisISBLk7461uvqYuh4iISC+GJKpRPyZnYn9SBsRWIsRGtIKNmH8FiYjIPPEbimqMoqAE0XuSAQBTu/si2ENq4oqIiIjKx5BENWb5gct4kKuCr6sj3uvlZ+pyiIiIKsSQRDXi1+sPsf3MbQBA7IhWsLMRm7giIiKiijEkUbUrKC5F1M4kAMCEcC+093YxcUVEREQvxpBE1W7VoatIf1wAd6kdZvcPMHU5RERElcKQRNXqfHo2vvn1JgBg2bAQ1JNYm7giIiKiymFIompTXKrBnB0XoRGAYWEe6BnQ2NQlERERVRpDElWbL45ex9X7eWjoaIvoQYGmLoeIiMggDElULa7ez8XnP18HACwcHAQXR1sTV0RERGQYhiQyOrVGwOwfLqJELaB3SzcMaiU3dUlEREQGY0gio9t44hYSb+fASWKNpUODIRKJTF0SERGRwRiSyKjSHxXgn3GpAICo11pCJrUzcUVERERVw5BERiMIAqJ2XURhiRqdfF0wpr2nqUsiIiKqMoYkMpr/nLmDX68/gsTaCiuGt4KVFU+zERGR5WJIIqO4ryzCkv0pAIAP+raAt6ujiSsiIiJ6OQxJZBQf7UlGblEpWjWRYnIXH1OXQ0RE9NIYkuilHUzKQNyl+7C2EiE2ohWsxfxrRURElo/fZvRScgqKEb3nEgBg2ivN0FLubOKKiIiIjIMhiV7K0v2X8TBPheaN62H6q81NXQ4REZHRMCRRlR2/+gA/nL0DkQiIjQiBxFps6pKIiIiMhiGJqiRfVYqonUkAgInh3mjr5WLiioiIiIyLIYmq5OO4VNzNKYRHfXvM6udv6nKIiIiMjiGJDHY2LRubTt4CAMQMD4GjxNq0BREREVUDhiQyiKpUjTk7LkIQgBFtm6B7i0amLomIiKhaMCSRQT4/ch3Xs/LgWk+CBQNbmrocIiKiasOQRJV2OUOJL47+DgBYPCQI9R1sTVwRERFR9THrkFRaWooFCxbAx8cH9vb28PX1xeLFi6HRaLRt3njjDYhEIp1Xp06dXtj3jh07EBgYCIlEgsDAQOzatas6h2LxStUazNlxEaUaAf2C3DAgWGbqkoiIiKqVWa+4jY2NxYYNG7Bp0yYEBQXhzJkzmDRpEqRSKWbMmKFt179/f3z77bfan21tK57hOHnyJEaPHo0lS5Zg2LBh2LVrF0aNGoWEhAR07Nix2sZjyb759SYu3lHAyc4aS4YEQyQSmbokIiKiaiUSBEEwdRHlGTRoENzc3PD1119rt0VERMDBwQGbN28G8GQmKScnB7t37650v6NHj4ZSqcTBgwe12/r3748GDRpg69atlepDqVRCKpVCoVDA2bl2P4rj1sN89PvsOFSlGqyMaIVR7T1NXRIREVGVGPL9bdan27p27YqffvoJV69eBQBcuHABCQkJeO2113TaHT16FI0bN0aLFi3w1ltvISsrq8J+T548ib59++ps69evH06cOFHuPiqVCkqlUudVF2g0AubsuAhVqQZdmjfEyHZNTF0SERFRjTDr021z5syBQqFAQEAAxGIx1Go1li1bhrFjx2rbDBgwACNHjoSXlxdu3ryJ6OhovPrqqzh79iwkEonefjMzM+Hm5qazzc3NDZmZmeXWEhMTg0WLFhlnYBZk2+nbOHXzMextxIgZ1oqn2YiIqM4w65C0fft2bNmyBd9//z2CgoKQmJiIyMhIuLu7Y+LEiQCenDp7Kjg4GO3atYOXlxf279+P4cOHl9v3n7/sBUGoMABERUVh5syZ2p+VSiU8PWv3aadMRRFiDlwGAHzYzx9NGzqYuCIiIqKaY9YhadasWZg7dy7GjBkDAAgJCUFaWhpiYmK0IenP5HI5vLy8cO3atXL7lclkZWaNsrKyyswuPU8ikZQ7M1UbCYKABbuTkKsqRWvP+nijs7epSyIiIqpRZr0mqaCgAFZWuiWKxWKdWwD82aNHj3D79m3I5fJy24SHhyM+Pl5n26FDh9C5c+eXK7gW2XcxA4cvZ8FGLEJsRCuIrXiajYiI6haznkl6/fXXsWzZMjRt2hRBQUE4f/48Vq1ahcmTJwMA8vLysHDhQkREREAul+PWrVuYN28eXF1dMWzYMG0/EyZMgIeHB2JiYgAAM2bMQPfu3REbG4shQ4Zgz549OHz4MBISEkwyTnPzOL8YC/deAgBM79kc/jInE1dERERU88w6JK1duxbR0dGYNm0asrKy4O7ujqlTp+Kjjz4C8GRWKSkpCd999x1ycnIgl8vRs2dPbN++HU5Oz77Y09PTdWakOnfujG3btmHBggWIjo5Gs2bNsH37dt4j6Q9L9qXgUX4x/N2cMO2V5qYuh4iIyCTM+j5J5qy23ifp5ytZmLTxNKxEwM5pXdDas76pSyIiIjKaWnOfJKpZeapSzN+VBACY3MWHAYmIiOo0hiTSWvnjFdxTFKGpiwNm9m1h6nKIiIhMiiGJAAC/3XyM706mAQBihofAwdasl6sRERFVO4YkQlGJGnN3XAQAjG7niS7NXU1cERERkekxJBHW/HQNNx7mo7GTBPMGtjR1OURERGaBIamOS76rwL+O3wAALBkaDKm9jYkrIiIiMg8MSXVYiVqD2T9chFojYGCIHP2CZKYuiYiIyGwwJNVhX/1yAykZSkjtbbBwcJCpyyEiIjIrDEl11I0Hefjs8JOHAH80KBCNnOrOw3uJiIgqgyGpDtJoBMzdkYTiUg26+blieBsPU5dERERkdhiS6qB//5aO3249hoOtGMuHhUAkEpm6JCIiIrPDkFTH3M0pxIoDlwEAs/v5w9PFwcQVERERmSeGpDpEEATM35WE/GI12no1wPhwb1OXREREZLYYkuqQPYn3cDT1AWzFVoiNCIHYiqfZiIiIysOQVEc8ylNh0X8vAQDe69UczRs7mbgiIiIi88aQVEcs+m8KsgtKECBzwtQezUxdDhERkdljSKoDDqfcx94L92AlAlaOaAUbMQ87ERHRi/DbspZTFpVgwe5kAMBb3XzRqkl90xZERERkIRiSarkVB68gU1kE74YOiOzdwtTlEBERWQyGpFrs5O+P8P2pdABAzPBWsLcVm7giIiIiy8GQVEsVFqsRtfMiAGBcx6YIb9bQxBURERFZFoakWuqzw1dx61EBZM52mDsgwNTlEBERWRyGpFro4p0cfPXLDQDA0qHBcLazMXFFRERElochqZYpUWsw+4eL0AjA66Hu6B3oZuqSiIiILBJDUi3zr2O/40pmLho42OAfrweauhwiIiKLxZBUi1zPysWan64DAP7xehBc60lMXBEREZHlYkiqJdQaAbN/uIhitQY9/RthSGt3U5dERERk0RiSaonNJ2/hXHoOHG3FWDYsBCKRyNQlERERWTSGpFrg9uMCrIxLBQDMfa0l3Ovbm7giIiIiy8eQZOEEQcC8XUkoKFajg7cL/tKhqalLIiIiqhUYkizcznN38cu1h7C1tsKKiBBYWfE0GxERkTEwJFmwB7kqLN6XAgCI7O0H30b1TFwRERFR7cGQZMEW7r0ERWEJgtyd8VY3X1OXQ0REVKswJFmoH5MzsT8pA2IrEWIjWsFGzENJRERkTPxmtUCKghJE70kGAEzt7otgD6mJKyIiIqp9zDoklZaWYsGCBfDx8YG9vT18fX2xePFiaDQaAEBJSQnmzJmDkJAQODo6wt3dHRMmTMC9e/cq7Hfjxo0QiURlXkVFRTUxrJe2/MBlPMhVwdfVEe/18jN1OURERLWStakLqEhsbCw2bNiATZs2ISgoCGfOnMGkSZMglUoxY8YMFBQU4Ny5c4iOjkZoaCiys7MRGRmJwYMH48yZMxX27ezsjNTUVJ1tdnZ21Tkco/j1+kNsP3MbABA7ohXsbMQmroiIiKh2MuuZpJMnT2LIkCEYOHAgvL29MWLECPTt21cbgKRSKeLj4zFq1Cj4+/ujU6dOWLt2Lc6ePYv09PQK+xaJRJDJZDqvl/WimS8A2LlzJ/r16wdXV1eIRCIkJia+sN+SkhIsXrwYvr7N0K2lO+598y46295Ge28XbRtvb2+9s2PTp09/6XERERHVRWYdkrp27YqffvoJV69eBQBcuHABCQkJeO2118rdR6FQQCQSoX79+hX2nZeXBy8vLzRp0gSDBg3C+fPnX7repzNf69atw+XLl7Fy5Up8/PHHWLt2rbZNfn4+unTpghUrVlS63wULFuBf//oXuk6YBfmb69EkfDB2ffy+Ts2nT59GRkaG9hUfHw8AGDly5EuPi4iIqC4y69Ntc+bMgUKhQEBAAMRiMdRqNZYtW4axY8fqbV9UVIS5c+di3LhxcHZ2LrffgIAAbNy4ESEhIVAqlVi9ejW6dOmCCxcuwM9P/xoflUoFlUql/VmpVJZp8/zMF/Bkdmfr1q06p/7Gjx8PALh169YLx//U5s2bMeFv7+P/CjxhUx/4OvJ1rBal4ZNPPsGWLVsAAI0aNdLZZ8WKFWjWrBl69OhR6c8hIiKiZ8x6Jmn79u3YsmULvv/+e5w7dw6bNm3CP//5T2zatKlM25KSEowZMwYajQZffPFFhf126tQJf/3rXxEaGopu3brh//7v/9CiRQudGZ8/i4mJgVQq1b48PT3LtKnKzFdlqFQq/Df5ATQCMCzMAz0DGsPe3h4JCQl62xcXF2PLli2YPHkyH3RLRERURWY9kzRr1izMnTsXY8aMAQCEhIQgLS0NMTExmDhxorZdSUkJRo0ahZs3b+LIkSMVziLpY2Vlhfbt2+PatWvltomKisLMmTO1PyuVyjJBydCZr8ryDu2MpJ+2w++vgZj/Wi/Ex8djz549UKvVetvv3r0bOTk5eOONN17qc4mIiOoys55JKigogJWVbolisVhnIfTTgHTt2jUcPnwYDRs2NPhzBEFAYmIi5HJ5uW0kEgmcnZ11Xn9myMxXZaVm5iK71ThYu7jjyto3IWtQD++++y4mTZoEsVj/lW1ff/01BgwYAHd39yp/LhERUV1n1jNJr7/+OpYtW4amTZsiKCgI58+fx6pVqzB58mQAT64mGzFiBM6dO4d9+/ZBrVYjMzMTAODi4gJbW1sAwIQJE+Dh4YGYmBgAwKJFi9CpUyf4+flBqVRizZo1SExMxOeff/5S9VZ25quy1BoBc3ZchMbOGePmr8XaUUF4/Pgx3N3dMXfuXPj4+JTZJy0tDYcPH8bOnTtfaixERER1nVmHpLVr1yI6OhrTpk1DVlYW3N3dMXXqVHz00UcAgDt37mDv3r0AgNatW+vs+/PPP+OVV14BAKSnp+vMSOXk5ODtt99GZmYmpFIpwsLCcPz4cXTo0OGl6q3MzJchNp64hcTbOXCSWGPp0GDY29vBw8MDJSUl2LFjB0aNGlVmn2+//RaNGzfWLh4nIiKiqhEJgiCYughLpFQqIZVKoVAotKfe3njjDRw+fBj/+te/tDNfb7/9NiZPnozY2FgAwOPHj5Geno579+5h4MCB2LZtG/z9/XXu1TRhwgQ4uTTGEafeKCxRY7JfKYIbqNG6dWvcvXsXCxcuxM2bN3Hu3DmdWx1oNBr4+Phg7NixBt1igIiIqK7Q9/1dHrNek2Rp1q5dixEjRmDatGlo2bIlPvzwQ0ydOhVLlizRttm7dy/CwsK0Mz1jxoxBWFgYNmzYoG1z/cYt7Dl5CYUlanTydUE3XykWLFiAwMBADBs2DB4eHkhISChzL6jDhw8jPT1dezqSiIiIqo4zSVVkSBI1xPbT6Zi7IwlPD8qsfv6Y3rO50fonIiKqyziTZKEyFIWI2vksIAHAqkNXkaEoNFlNREREdRVDkhm5+TAfmj/N66kFAbceFpimICIiojqMIcmM+Lg6wupPN8gWi0TwdnUwTUFERER1GEOSGZFL7REzPATiPx4lIhaJsHx4MORSexNXRkREVPeY9X2S6qLR7Zuie4tGuPWwAN6uDgxIREREJsKQZIbkUnuGIyIiIhPj6TYiIiIiPRiSiIiIiPRgSCIiIiLSgyGJiIiISA+GJCIiIiI9GJKIiIiI9GBIIiIiItKDIYmIiIhID4YkIiIiIj0YkoiIiIj0YEgiIiIi0oMhiYiIiEgPhiQiIiIiPRiSiIiIiPRgSCIiIiLSgyGJiIiISA+GJCIiIiI9GJKIiIiI9GBIIiIiItKDIYmIiIhID4YkIiIiIj0YkoiIiIj0YEgiIiIi0oMhiYiIiEgPhiQiIiIiPRiSiIiIiPRgSCIiIiLSgyGJiIiISA+zDkmlpaVYsGABfHx8YG9vD19fXyxevBgajUbbRhAELFy4EO7u7rC3t8crr7yCS5cuvbDvHTt2IDAwEBKJBIGBgdi1a1d1DoWIiIgsjFmHpNjYWGzYsAHr1q3D5cuXsXLlSnz88cdYu3atts3KlSuxatUqrFu3DqdPn4ZMJkOfPn2Qm5tbbr8nT57E6NGjMX78eFy4cAHjx4/HqFGjcOrUqZoYFhEREVkAkSAIgqmLKM+gQYPg5uaGr7/+WrstIiICDg4O2Lx5MwRBgLu7OyIjIzFnzhwAgEqlgpubG2JjYzF16lS9/Y4ePRpKpRIHDx7Ubuvfvz8aNGiArVu3Vqo2pVIJqVQKhUIBZ2fnlxglERER1RRDvr+ta6imKunatSs2bNiAq1evokWLFrhw4QISEhLw2WefAQBu3ryJzMxM9O3bV7uPRCJBjx49cOLEiXJD0smTJ/H+++/rbOvXr5+2X31UKhVUKpX2Z4VCAeDJf2wiIiKyDE+/tyszR2TWIWnOnDlQKBQICAiAWCyGWq3GsmXLMHbsWABAZmYmAMDNzU1nPzc3N6SlpZXbb2Zmpt59nvanT0xMDBYtWlRmu6enZ6XHQ0REROYhNzcXUqm0wjZmHZK2b9+OLVu24Pvvv0dQUBASExMRGRkJd3d3TJw4UdtOJBLp7CcIQpltf2boPlFRUZg5c6b2Z41Gg8ePH6Nhw4Yv/CxDKZVKeHp64vbt27XyVB7HZ/lq+xhr+/iA2j9Gjs/yVdcYBUFAbm4u3N3dX9jWrEPSrFmzMHfuXIwZMwYAEBISgrS0NMTExGDixImQyWQAnswMyeVy7X5ZWVllZoqeJ5PJyswavWgfiUQCiUSis61+/fqGDskgzs7OtfYvP8Dx1Qa1fYy1fXxA7R8jx2f5qmOML5pBesqsr24rKCiAlZVuiWKxWHsLAB8fH8hkMsTHx2vfLy4uxrFjx9C5c+dy+w0PD9fZBwAOHTpU4T5ERERUt5j1TNLrr7+OZcuWoWnTpggKCsL58+exatUqTJ48GcCTU2aRkZFYvnw5/Pz84Ofnh+XLl8PBwQHjxo3T9jNhwgR4eHggJiYGADBjxgx0794dsbGxGDJkCPbs2YPDhw8jISHBJOMkIiIi82PWIWnt2rWIjo7GtGnTkJWVBXd3d0ydOhUfffSRts3s2bNRWFiIadOmITs7Gx07dsShQ4fg5OSkbZOenq4zI9W5c2ds27YNCxYsQHR0NJo1a4bt27ejY8eONTq+8kgkEvzjH/8oc3qvtuD4LF9tH2NtHx9Q+8fI8Vk+cxijWd8niYiIiMhUzHpNEhEREZGpMCQRERER6cGQRERERKQHQxIRERGRHgxJJvLFF1/Ax8cHdnZ2aNu2LX755ZcK2x87dgxt27aFnZ0dfH19sWHDhhqqtGoMGd/Ro0chEonKvK5cuVKDFVfe8ePH8frrr8Pd3R0ikQi7d+9+4T6WdPwMHZ+lHb+YmBi0b98eTk5OaNy4MYYOHYrU1NQX7mcpx7Aq47O0Y7h+/Xq0atVKe5PB8PBwnQeW62Mpxw8wfHyWdvz+LCYmRntLn4qY4hgyJJnA9u3bERkZifnz5+P8+fPo1q0bBgwYgPT0dL3tb968iddeew3dunXD+fPnMW/ePLz33nvYsWNHDVdeOYaO76nU1FRkZGRoX35+fjVUsWHy8/MRGhqKdevWVaq9pR0/Q8f3lKUcv2PHjmH69On43//+h/j4eJSWlqJv377Iz88vdx9LOoZVGd9TlnIMmzRpghUrVuDMmTM4c+YMXn31VQwZMgSXLl3S296Sjh9g+PiespTj97zTp0/jyy+/RKtWrSpsZ7JjKFCN69Chg/DOO+/obAsICBDmzp2rt/3s2bOFgIAAnW1Tp04VOnXqVG01vgxDx/fzzz8LAITs7OwaqM64AAi7du2qsI2lHb/nVWZ8lnz8BEEQsrKyBADCsWPHym1jycewMuOz9GMoCILQoEED4f/9v/+n9z1LPn5PVTQ+Sz1+ubm5gp+fnxAfHy/06NFDmDFjRrltTXUMOZNUw4qLi3H27Fn07dtXZ3vfvn1x4sQJvfucPHmyTPt+/frhzJkzKCkpqbZaq6Iq43sqLCwMcrkcvXr1ws8//1ydZdYoSzp+L8NSj59CoQAAuLi4lNvGko9hZcb3lCUeQ7VajW3btiE/Px/h4eF621jy8avM+J6ytOM3ffp0DBw4EL17935hW1MdQ4akGvbw4UOo1eoyD9N1c3Mr89DdpzIzM/W2Ly0txcOHD6ut1qqoyvjkcjm+/PJL7NixAzt37oS/vz969eqF48eP10TJ1c6Sjl9VWPLxEwQBM2fORNeuXREcHFxuO0s9hpUdnyUew6SkJNSrVw8SiQTvvPMOdu3ahcDAQL1tLfH4GTI+Szx+27Ztw7lz57SPC3sRUx1Ds34sSW0mEol0fhYEocy2F7XXt91cGDI+f39/+Pv7a38ODw/H7du38c9//hPdu3ev1jpriqUdP0NY8vF79913cfHixUo9t9ESj2Flx2eJx9Df3x+JiYnIycnBjh07MHHiRBw7dqzcIGFpx8+Q8Vna8bt9+zZmzJiBQ4cOwc7OrtL7meIYciaphrm6ukIsFpeZVcnKyiqTkp+SyWR621tbW6Nhw4bVVmtVVGV8+nTq1AnXrl0zdnkmYUnHz1gs4fj9/e9/x969e/Hzzz+jSZMmFba1xGNoyPj0MfdjaGtri+bNm6Ndu3aIiYlBaGgoVq9erbetJR4/Q8anjzkfv7NnzyIrKwtt27aFtbU1rK2tcezYMaxZswbW1tZQq9Vl9jHVMWRIqmG2trZo27Yt4uPjdbbHx8ejc+fOevcJDw8v0/7QoUNo164dbGxsqq3WqqjK+PQ5f/485HK5scszCUs6fsZizsdPEAS8++672LlzJ44cOQIfH58X7mNJx7Aq49PHnI+hPoIgQKVS6X3Pko5feSoanz7mfPx69eqFpKQkJCYmal/t2rXDX/7yFyQmJkIsFpfZx2THsFqXhZNe27ZtE2xsbISvv/5aSElJESIjIwVHR0fh1q1bgiAIwty5c4Xx48dr29+4cUNwcHAQ3n//fSElJUX4+uuvBRsbG+GHH34w1RAqZOj4Pv30U2HXrl3C1atXheTkZGHu3LkCAGHHjh2mGkKFcnNzhfPnzwvnz58XAAirVq0Szp8/L6SlpQmCYPnHz9DxWdrx+9vf/iZIpVLh6NGjQkZGhvZVUFCgbWPJx7Aq47O0YxgVFSUcP35cuHnzpnDx4kVh3rx5gpWVlXDo0CFBECz7+AmC4eOztOOnz5+vbjOXY8iQZCKff/654OXlJdja2gpt2rTRuTx34sSJQo8ePXTaHz16VAgLCxNsbW0Fb29vYf369TVcsWEMGV9sbKzQrFkzwc7OTmjQoIHQtWtXYf/+/SaounKeXm7759fEiRMFQbD842fo+Czt+OkbGwDh22+/1bax5GNYlfFZ2jGcPHmy9t+XRo0aCb169dIGCEGw7OMnCIaPz9KOnz5/DknmcgxFgvDHyiciIiIi0uKaJCIiIiI9GJKIiIiI9GBIIiIiItKDIYmIiIhID4YkIiIiIj0YkoiIiIj0YEgiIiIi0oMhiYjISI4ePQqRSIScnBxTl0JERsCQRERERKQHQxIRERGRHgxJRFRrCIKAlStXwtfXF/b29ggNDcUPP/wA4NmpsP379yM0NBR2dnbo2LEjkpKSdPrYsWMHgoKCIJFI4O3tjU8++UTnfZVKhdmzZ8PT0xMSiQR+fn74+uuvddqcPXsW7dq1g4ODAzp37ozU1NTqHTgRVQuGJCKqNRYsWIBvv/0W69evx6VLl/D+++/jr3/9K44dO6ZtM2vWLPzzn//E6dOn0bhxYwwePBglJSUAnoSbUaNGYcyYMUhKSsLChQsRHR2NjRs3avefMGECtm3bhjVr1uDy5cvYsGED6tWrp1PH/Pnz8cknn+DMmTOwtrbG5MmTa2T8RGRcfMAtEdUK+fn5cHV1xZEjRxAeHq7dPmXKFBQUFODtt99Gz549sW3bNowePRoA8PjxYzRp0gQbN27EqFGj8Je//AUPHjzAoUOHtPvPnj0b+/fvx6VLl3D16lX4+/sjPj4evXv3LlPD0aNH0bNnTxw+fBi9evUCABw4cAADBw5EYWEh7Ozsqvm/AhEZE2eSiKhWSElJQVFREfr06YN69eppX9999x1+//13bbvnA5SLiwv8/f1x+fJlAMDly5fRpUsXnX67dOmCa9euQa1WIzExEWKxGD169KiwllatWmn/LJfLAQBZWVkvPUYiqlnWpi6AiMgYNBoNAGD//v3w8PDQeU8ikegEpT8TiUQAnqxpevrnp56fbLe3t69ULTY2NmX6flofEVkOziQRUa0QGBgIiUSC9PR0NG/eXOfl6empbfe///1P++fs7GxcvXoVAQEB2j4SEhJ0+j1x4gRatGgBsViMkJAQaDQanTVORFR7cSaJiGoFJycnfPjhh3j//feh0WjQtWtXKJVKnDhxAvXq1YOXlxcAYPHixWjYsCHc3Nwwf/58uLq6YujQoQCADz74AO3bt8eSJUswevRonDx5EuvWrcMXX3wBAPD29sbEiRMxefJkrFmzBqGhoUhLS0NWVhZGjRplqqETUTVhSCKiWmPJkiVo3LgxYmJicOPGDdSvXx9t2rTBvHnztKe7VqxYgRkzZuDatWsIDQ3F3r17YWtrCwBo06YN/u///g8fffQRlixZArlcjsWLF+ONN97Qfsb69esxb948TJs2DY8ePULTpk0xb948UwyXiKoZr24jojrh6ZVn2dnZqF+/vqnLISILwDVJRERERHowJBERERHpwdNtRERERHpwJomIiIhID4YkIiIiIj0YkoiIiIj0YEgiIiIi0oMhiYiIiEgPhiQiIiIiPRiSiIiIiPRgSCIiIiLSgyGJiIiISI//D674pTXVsk6xAAAAAElFTkSuQmCC", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "plt.plot(np.arange(len(epochs_x)), epochs_acc, marker = '.')\n", - "plt.xlabel('epoch')\n", - "plt.ylabel('test accuracy')\n", - "plt.ylim(80, 100)\n", - "for i, txt in enumerate(epochs_acc):\n", - " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom')\n", - "plt.show()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "speck-rescnn", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/tests/test_nonsequential/baseline-SCNN-example_2.ipynb b/tests/test_nonsequential/baseline-SCNN-example_2.ipynb deleted file mode 100644 index 893b5e79..00000000 --- a/tests/test_nonsequential/baseline-SCNN-example_2.ipynb +++ /dev/null @@ -1,630 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import torch\n", - "import torch.nn as nn\n", - "import sinabs.layers as sl\n", - "from tqdm.notebook import tqdm\n", - "\n", - "from tonic.datasets.nmnist import NMNIST\n", - "from tonic.transforms import ToFrame\n", - "from torch.utils.data import DataLoader\n", - "from torch.nn import CrossEntropyLoss\n", - "from torch.optim import Adam\n", - "\n", - "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential\n", - "\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "torch.manual_seed(0)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Loading Data" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "batch_size = 64\n", - "num_workers = 4\n", - "epochs = 5\n", - "lr = 1e-3" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "root_dir = \"./NMNIST\"\n", - "_ = NMNIST(save_to=root_dir, train=True)\n", - "_ = NMNIST(save_to=root_dir, train=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "n_time_steps = 50\n", - "to_raster = ToFrame(sensor_size=NMNIST.sensor_size, n_time_bins=n_time_steps)\n", - "\n", - "snn_train_dataset = NMNIST(save_to=root_dir, train=True, transform=to_raster)\n", - "snn_test_dataset = NMNIST(save_to=root_dir, train=False, transform=to_raster)" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "snn_train_dataloader = DataLoader(snn_train_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True)\n", - "snn_test_dataloader = DataLoader(snn_test_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=False)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Network Module\n", - "\n", - "We need to define a `nn.Module` implementing the network we want the chip to reproduce." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "device: NVIDIA GeForce RTX 3070 Ti\n" - ] - } - ], - "source": [ - "if torch.cuda.is_available():\n", - " device = torch.device('cuda:0')\n", - " print('device: ', torch.cuda.get_device_name(0))\n", - "else:\n", - " device = torch.device('cpu')" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "class SNN(nn.Module):\n", - " def __init__(self) -> None:\n", - " super().__init__()\n", - "\n", - " self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False)\n", - " self.iaf1 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - " self.pool1 = nn.AvgPool2d(2,2)\n", - "\n", - " self.conv2 = nn.Conv2d(10, 10, 2, 1, bias=False)\n", - " self.iaf2 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - " self.pool2 = nn.AvgPool2d(3,3)\n", - "\n", - " self.conv3 = nn.Conv2d(10, 10, 3, 1, bias=False)\n", - " self.iaf3 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - " self.pool3 = nn.AvgPool2d(2,2)\n", - "\n", - " self.flat = nn.Flatten()\n", - "\n", - " self.fc1 = nn.Linear(10, 100, bias=False)\n", - " self.iaf4 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - "\n", - " self.fc2 = nn.Linear(100, 100, bias=False)\n", - " self.iaf5 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - "\n", - " self.fc3 = nn.Linear(100, 100, bias=False)\n", - " self.iaf6 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - "\n", - " self.fc4 = nn.Linear(100, 10, bias=False)\n", - " self.iaf7 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - "\n", - " def detach_neuron_states(self):\n", - " for name, layer in self.named_modules():\n", - " if name != '':\n", - " if isinstance(layer, sl.StatefulLayer):\n", - " for name, buffer in layer.named_buffers():\n", - " buffer.detach_()\n", - "\n", - " def init_weights(self):\n", - " for name, layer in self.named_modules():\n", - " if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear):\n", - " nn.init.xavier_normal_(layer.weight.data)\n", - "\n", - " def forward(self, x):\n", - " \n", - " con1_out = self.conv1(x)\n", - " iaf1_out = self.iaf1(con1_out)\n", - " pool1_out = self.pool1(iaf1_out)\n", - "\n", - " conv2_out = self.conv2(pool1_out)\n", - " iaf2_out = self.iaf2(conv2_out)\n", - " pool2_out = self.pool2(iaf2_out)\n", - "\n", - " conv3_out = self.conv3(pool2_out)\n", - " iaf3_out = self.iaf3(conv3_out)\n", - " pool3_out = self.pool3(iaf3_out)\n", - "\n", - " flat_out = self.flat(pool3_out)\n", - " \n", - " fc1_out = self.fc1(flat_out)\n", - " iaf4_out = self.iaf4(fc1_out)\n", - "\n", - " fc2_out = self.fc2(iaf4_out)\n", - " iaf5_out = self.iaf5(fc2_out)\n", - "\n", - " fc3_out = self.fc3(iaf5_out)\n", - " iaf6_out = self.iaf6(fc3_out)\n", - "\n", - " fc4_out = self.fc4(iaf6_out)\n", - " iaf7_out = self.iaf7(fc4_out)\n", - "\n", - " return iaf7_out" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "snn = SNN().to(device)" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "snn.init_weights()" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "optimizer = Adam(snn.parameters(), lr=lr, betas=(0.9, 0.999), eps=1e-8)\n", - "loss_fn = CrossEntropyLoss()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Define train and test" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "def train(dataloader, model, loss_fn, optimizer, epochs, test_func, dataloader_test):\n", - " epochs_y = []\n", - " epochs_x = []\n", - " epochs_acc = []\n", - " model.train()\n", - "\n", - " for e in range(epochs):\n", - " losses = []\n", - " batches = []\n", - " batch_count = 0\n", - " train_p_bar = tqdm(snn_train_dataloader)\n", - "\n", - " for X, y in train_p_bar:\n", - " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", - " X = X.reshape(-1, 2, 34, 34).to(dtype=torch.float, device=device)\n", - " y = y.to(dtype=torch.long, device=device)\n", - "\n", - " # forward\n", - " pred = model(X)\n", - "\n", - " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", - " pred = pred.reshape(batch_size, n_time_steps, -1)\n", - "\n", - " # accumulate all time-steps output for final prediction\n", - " pred = pred.sum(dim = 1)\n", - " loss = loss_fn(pred, y)\n", - "\n", - " # gradient update\n", - " optimizer.zero_grad()\n", - " loss.backward()\n", - " optimizer.step()\n", - "\n", - " # detach the neuron states and activations from current computation graph(necessary)\n", - " model.detach_neuron_states()\n", - "\n", - " train_p_bar.set_description(f\"Epoch {e} - BPTT Training Loss: {round(loss.item(), 4)}\")\n", - "\n", - " batch_count += 1\n", - " losses.append(loss.item())\n", - " batches.append(batch_count)\n", - "\n", - " epochs_y.append(losses)\n", - " epochs_x.append(batches)\n", - "\n", - " acc = test_func(dataloader_test, model)\n", - " print(f'Epoch {e} accuracy: {acc}')\n", - " epochs_acc.append(acc)\n", - "\n", - " return epochs_x, epochs_y, epochs_acc\n" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "def test(dataloader, model):\n", - " correct_predictions = []\n", - " with torch.no_grad():\n", - " test_p_bar = tqdm(dataloader)\n", - " for X, y in test_p_bar:\n", - " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", - " X = X.reshape(-1, 2, 34, 34).to(dtype=torch.float, device=device)\n", - " y = y.to(dtype=torch.long, device=device)\n", - "\n", - " # forward\n", - " output = model(X)\n", - "\n", - " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", - " output = output.reshape(batch_size, n_time_steps, -1)\n", - "\n", - " # accumulate all time-steps output for final prediction\n", - " output = output.sum(dim=1)\n", - "\n", - " # calculate accuracy\n", - " pred = output.argmax(dim=1, keepdim=True)\n", - "\n", - " # compute the total correct predictions\n", - " correct_predictions.append(pred.eq(y.view_as(pred)))\n", - "\n", - " test_p_bar.set_description(f\"Testing Model...\")\n", - " \n", - " correct_predictions = torch.cat(correct_predictions)\n", - " return correct_predictions.sum().item()/(len(correct_predictions))*100" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Training loop (HPO)" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "392406a27a8146319e1aab0a40fe2f59", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - " 0%| | 0/937 [00:00" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "plt.plot(epochs_x[-1], epochs_y[-1])\n", - "plt.xlabel('batches')\n", - "plt.ylabel('loss')\n", - "plt.ylim(0,)\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAGwCAYAAABVdURTAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAABTAElEQVR4nO3de1xUZf4H8M/MwDCgMIhc5C4o4g1NQREVlRRMXVe7rJSu1qa/1rL1QmqirWu2vygvZW6pWZo/dxMpRXNXNsVU0CAvBOUVSRAQQQRluAk4zPn9gU6NXGSQ4TDD5/16zevVPPOcM9/HU83H55zzHIkgCAKIiIiITIRU7AKIiIiIWhPDDREREZkUhhsiIiIyKQw3REREZFIYboiIiMikMNwQERGRSWG4ISIiIpNiJnYBbU2j0eDGjRuwtraGRCIRuxwiIiJqBkEQUFZWBhcXF0ilTc/NdLhwc+PGDbi7u4tdBhEREbVAbm4u3NzcmuzT4cKNtbU1gLo/HBsbG5GrISIiouYoLS2Fu7u79ne8KR0u3Dw4FWVjY8NwQ0REZGSac0kJLyhuRGJiIiZPngwXFxdIJBLs37+/yf6xsbEIDQ2Fg4MDbGxsEBQUhEOHDtXrV1JSgnnz5sHZ2RkKhQJ9+vRBXFycgUZBRETU8TDcNKKiogIDBw7Exx9/3Kz+iYmJCA0NRVxcHFJSUhASEoLJkycjNTVV26empgahoaG4du0a9uzZg/T0dHz22WdwdXU11DCIiIg6nA53Wqq5JkyYgAkTJjS7/4YNG3Tev/vuu/jmm2/w73//G4MGDQIAbN++Hbdv30ZSUhLMzc0BAJ6enq1WMxEREXHmxmA0Gg3KyspgZ2enbTtw4ACCgoIwb948ODk5oX///nj33XdRW1srYqVERESmhTM3BrJ+/XpUVFRg2rRp2rbMzEwcPXoUM2bMQFxcHDIyMjBv3jyo1WqsXLlSxGqJiIhMB8ONAURHR2PVqlX45ptv4OjoqG3XaDRwdHTE1q1bIZPJ4O/vjxs3bmDt2rUMN0RERK2E4aaVxcTEYPbs2fj6668xbtw4nc+cnZ1hbm4OmUymbevTpw8KCgpQU1MDuVze1uUSERGZHF5z04qio6Px0ksvYdeuXZg0aVK9z0eMGIFffvkFGo1G23blyhU4Ozsz2BAREbUShptGlJeXIy0tDWlpaQCArKwspKWlIScnBwAQGRmJWbNmaftHR0dj1qxZWL9+PYYNG4aCggIUFBRApVJp+7z66qsoLi7GggULcOXKFRw8eBDvvvsu5s2b16ZjIyIiMmUSQRAEsYtoS6WlpVAqlVCpVE2uUHz8+HGEhITUa3/xxRexY8cOvPTSS7h27RqOHz8OABgzZgwSEhIa7f9AcnIyFi1ahLS0NLi6umL27Nl48803dU5VERERka7m/n4DDDettt/dp3MQGXsOAgCpBIh6xg/hQzxabf9EREQdmT6/3zwt1QryVXcRua8u2ACARgCWx55HvuquqHURERF1RAw3rSCrqAIPz3/VCgKuFVWKUxAREVEHxnDTCrzsO0H60ENKpRKgu72VOAURERF1YAw3rcBZaYmoZ/wg+81j2Ae628JZaSliVURERB0Tw00rCR/igZPLQvDu1P4AgLTcEqQXlIlcFRERUcfDcNOKnJWWmD7MExP6d4MgAOsOp4tdEhERUYfDcGMAb4T1glQCxF+8iR9z7ohdDhERUYfCcGMAPR2t8exgNwDA2m/T0cGWEiIiIhIVw42BLBjnA7lMiuTMYnz/S7HY5RAREXUYDDcG4tbFCtMD61YoXnvoMmdviIiI2gjDjQG9/mRPWMll+Om6CocuFIhdDhERUYfAcGNA9p0tMHukFwBg3eErqNVw9oaIiMjQGG4M7H9GecPWyhy/FJYj9sfrYpdDRERk8hhuDMxGYY5XR/cAAGw4koFqda3IFREREZk2UcNNYmIiJk+eDBcXF0gkEuzfv7/J/rGxsQgNDYWDgwNsbGwQFBSEQ4cOtU2xj2FWUHc4Wlsgr+Quok/liF0OERGRSRM13FRUVGDgwIH4+OOPm9U/MTERoaGhiIuLQ0pKCkJCQjB58mSkpqYauNLHYymXYf5YHwDAx8d+QUW1WuSKiIiITJdEaCf3KEskEuzbtw9Tp07Va7t+/fohPDwcK1eubPDz6upqVFdXa9+XlpbC3d0dKpUKNjY2j1OyXu7VajDugwRkF1dicVgvvP6kT5t9NxERkbErLS2FUqls1u+3UV9zo9FoUFZWBjs7u0b7REVFQalUal/u7u5tWOGvzGVSRIT2AgB8mpiJksoaUeogIiIydUYdbtavX4+KigpMmzat0T6RkZFQqVTaV25ubhtWqGvyABf07maNsio1tiRkilYHERGRKTPacBMdHY1Vq1YhJiYGjo6OjfazsLCAjY2NzkssUqkEi8N8AQA7krJQWFolWi1ERESmyijDTUxMDGbPno2vvvoK48aNE7scvYzt44jBHraouqfBxqMZYpdDRERkcowu3ERHR+Oll17Crl27MGnSJLHL0ZtEIsHSp3oDAHafzkVOcaXIFREREZkWUcNNeXk50tLSkJaWBgDIyspCWloacnLq1oKJjIzErFmztP2jo6Mxa9YsrF+/HsOGDUNBQQEKCgqgUqnEKL/Fhnl3RbCPPdQaAR8euSJ2OURERCZF1HBz9uxZDBo0CIMGDQIAREREYNCgQdrbuvPz87VBBwA+/fRTqNVqzJs3D87OztrXggULRKn/cSwdXzd7sz8tD5cLSkWuhoiIyHS0m3Vu2oo+98kb2mtfpiDuXAHG9XHC5y8GiFoLERFRe9Zh1rkxdhGhvpBKgCOXbuLHnDtil0NERGQSGG5E1NOxM54d7AYAWPttOjrYJBoREZFBMNyIbGFoL8hlUiRnFuPkL0Vil0NERGT0GG5E5mpriRnDPAAAaw9x9oaIiOhxMdy0A/NCesJKLsPP11U4dKFA7HKIiIiMGsNNO2Df2QKzR3oBANYdvoJaDWdviIiIWorhpp34n1HesLUyxy+F5Yj98brY5RARERkthpt2wkZhjldH9wAAbDiSgWp1rcgVERERGSeGm3bkxeHd4WRjgbySu4g+lfPoDYiIiKgehpt2RGEuw1+e9AEAfHzsF1RUq0WuiIiIyPgw3LQz4UPc4dnVCkXlNfji+yyxyyEiIjI6DDftjLlMiojQXgCATxMzUVJZI3JFRERExoXhph2aPMAFvbtZo6xKjS0JmWKXQ0REZFQYbtohqVSCxWG+AIAdSVkoLK0SuSIiIiLjwXDTTo3t44jBHraouqfBxqMZYpdDRERkNBhu2imJRIKlT/UGAOw+nYvs4gqRKyIiIjIODDft2DDvrhjVywFqjYANRzh7Q0RE1BwMN+3ckvvX3uxPy8PlglKRqyEiImr/GG7aOT83JSb6dYMgAOsOXRG7HCIionaP4cYIRIT6QioBjly6iZTsO2KXQ0RE1K4x3BiBno6d8Zy/GwBg7aHLEARB5IqIiIjaL4YbI7FgXC/IZVL8kHkbJ38pErscIiKidovhxki42lpixjAPAMDaQ+mcvSEiImoEw40RmRfSE1ZyGX6+rsK35wvELoeIiKhdYrgxIvadLTBnpBcAYN3hdNRqOHtDRET0MIYbIzNnlDdsrcxx9VYFYn+8LnY5RERE7Q7DjZGxUZjj1dE9AAAbjmSgWl0rckVERETtC8ONEXpxeHc42Vggr+Qudp3KEbscIiKidoXhxggpzGWYP9YHAPDJsV9QUa0WuSIiIqL2g+HGSE0LcIdnVysUldfgi++zxC6HiIio3WC4MVLmMikiQnsBAD5NzERJZY3IFREREbUPDDdGbPIAF/TuZo2yKjU2J1wVuxwiIqJ2geHGiEmlEiwZ7wsA+L+ka7hZWiVyRUREROJjuDFyT/Z2hL9nF1Td0+AfRzPELoeIiEh0DDdGTiL5dfZm9+lcZBdXiFwRERGRuBhuTMAw764Y1csBao2AD+OviF0OERGRqBhuTMTS+7M33/x0A5cLSkWuhoiISDwMNyaiv6sSk/ycIQjAukOcvSEioo6L4caELArtBakEOHLpJlKy74hdDhERkSgYbkxIT8fOeM7fDQCw9tBlCIIgckVERERtj+HGxCwY1wtymRQ/ZN7GyV+KxC6HiIiozTHcmBhXW0v8cZgnAGDtoXTO3hARUYfDcGOCXgvpASu5DD9fV+Hb8wVil0NERNSmGG5MkH1nC8wZ6QUAWHc4HepajcgVERERtR2GGxM1Z5Q3bK3McfVWBfal5oldDhERUZsRNdwkJiZi8uTJcHFxgUQiwf79+x+5TUJCAvz9/aFQKODt7Y0tW7YYvlAjZKMwx2tjegAANhzJQLW6VuSKiIiI2oao4aaiogIDBw7Exx9/3Kz+WVlZmDhxIoKDg5Gamorly5dj/vz52Lt3r4ErNU6zgrrDycYCeSV3setUjtjlEBERtQmJ0E5up5FIJNi3bx+mTp3aaJ8333wTBw4cwKVLl7Rtc+fOxU8//YTk5ORmfU9paSmUSiVUKhVsbGwet+x278tT2Vix7zy6dpIjcWkIOlmYiV0SERGR3vT5/Taqa26Sk5MRFham0zZ+/HicPXsW9+7da3Cb6upqlJaW6rw6kmkB7uje1QrFFTX44vssscshIiIyOKMKNwUFBXByctJpc3JyglqtRlFRwwvWRUVFQalUal/u7u5tUWq7YS6TYlFoLwDAp4mZKKmsEbkiIiIiwzKqcAPUnb76rQdn1R5ufyAyMhIqlUr7ys3NNXiN7c3kAS7o3c0aZVVqbE64KnY5REREBmVU4aZbt24oKNBdlK6wsBBmZmbo2rVrg9tYWFjAxsZG59XRSKUSLBnvCwDY8f013CytErkiIiIiwzGqcBMUFIT4+HidtsOHDyMgIADm5uYiVWUcnuztCH/PLqhWa/CPoxlil0NERGQwooab8vJypKWlIS0tDUDdrd5paWnIyam7bTkyMhKzZs3S9p87dy6ys7MRERGBS5cuYfv27di2bRsWL14sRvlGRSKRYOn92Zvdp3ORXVwhckVERESGIWq4OXv2LAYNGoRBgwYBACIiIjBo0CCsXLkSAJCfn68NOgDg5eWFuLg4HD9+HE888QTeeecdbNy4Ec8++6wo9RubQO+uGN3LAWqNgA/jr4hdDhERkUG0m3Vu2kpHW+fmYefzVPjdP05CIgH+uyAYvbt1vD8DIiIyPia7zg09vv6uSkzyc4YgAOsOcfaGiIhMD8NNBxQR1gsyqQRHLt1ESvYdscshIiJqVQw3HVAPh854brAbAGDtocvoYGcmiYjIxDHcdFDzx/lALpPih8zbOJHR8OrORERExojhpoNytbXEH4d5AgDWHkrn7A0REZkMhpsObF5ID3SSy3AuT4Vvzxc8egMiIiIjwHDTgXXtbIHZwd4AgHWH06Gu1YhcERER0eNjuOng5gR7wdbKHFdvVSA2NU/scoiIiB4bw00HZ6Mwx2tjegAAPjqSgWp1rcgVERERPR6GG8KsoO7oZqNAXsld7DqV8+gNiIiI2jGGG4LCXIb5Y30AAB8f/QUV1WqRKyIiImo5hhsCAPwhwA3du1qhuKIG209miV0OERFRizHcEADAXCbFotBeAICtiZkoqawRuSIiIqKWYbghrckDXNDH2QZl1WpsTrgqdjlEREQtwnBDWlKpBEvG183e7Pj+Gm6WVolcERERkf4YbkhHiK8j/D27oFqtwcbvMsQuh4iISG8MN6RDIpFg6XhfAEDMmVxkF1eIXBEREZF+GG6onkDvrhjdywFqjYAP46+IXQ4REZFeGG6oQUvuz95889MNXC4oFbkaIiKi5mO4oQb1d1Vikp8zBAFYdyhd7HKIiIiajeGGGhUR1gsyqQRHLhUiJfuO2OUQERE1C8MNNaqHQ2c8N9gNALD20GUIgiByRURERI/GcENNWjDOB3IzKX7IvI0TGUVil0NERPRIDDfUJBdbS8wc5gkAWHsonbM3RETU7jHc0CO9NqYHOsllOJenwrfnC8Quh4iIqEkMN/RIXTtbYHawNwBg3eF0qGs1IldERETUOIYbapb/CfZCFytzXL1VgdjUPLHLISIiahTDDTWLtcIcr43pCQD46EgGqtW1IldERETUMIYbaraZQZ7oZqNAXsldfPlDjtjlEBERNYjhhppNYS7D/LE+AIBPjv2Cimq1yBURERHVx3BDevlDgBu6d7VCcUUNtp/MErscIiKiehhuSC/mMikiwuoeqrk1MRN3KmpEroiIiEgXww3p7Xd+zujjbIOyajW2JFwVuxwiIiIdDDekN6lUgiXjewEAdiRdw83SKpErIiIi+hXDDbVIiK8jAjy7oFqtwcbvMsQuh4iISIvhhlpEIpFg6VO9AQAxZ3KRXVwhckVERER1GG6oxYZ62WF0LweoNQI+iL8idjlEREQAGG7oMS0ZX3fn1IGfbuBSfqnI1RARETHc0GPq76rEpAHOEARg/eF0scshIiJiuKHH90ZoL8ikEhy5VIiU7Ntil0NERB0cww09Nm+HznhusBsAYM236RAEQeSKiIioI2O4oVaxYJwP5GZSnMq6jRMZRWKXQ0REHRjDDbUKF1tLzBzmCQBYe4izN0REJB6GG2o1r43pgU5yGc7lqfDf8wVil0NERB0Uww21mq6dLTA72BsAsO5wOtS1GpErIiKijojhhlrV/wR7oYuVOTJvVSA2NU/scoiIqAMSPdxs2rQJXl5eUCgU8Pf3x4kTJ5rs/+WXX2LgwIGwsrKCs7Mz/vSnP6G4uLiNqqVHsVaY47UxPQEAHx3JQLW6VuSKiIiooxE13MTExGDhwoVYsWIFUlNTERwcjAkTJiAnJ6fB/idPnsSsWbMwe/ZsXLhwAV9//TXOnDmDOXPmtHHl1JSZQZ7oZqNAXsldfPlDw8eSiIjIUPQON3fv3kVlZaX2fXZ2NjZs2IDDhw/r/eUffPABZs+ejTlz5qBPnz7YsGED3N3dsXnz5gb7//DDD+jevTvmz58PLy8vjBw5En/+859x9uzZRr+juroapaWlOi8yLIW5DPPH+gAAPjn2C8qr1SJXREREHYne4WbKlCnYuXMnAKCkpASBgYFYv349pkyZ0mgoaUhNTQ1SUlIQFham0x4WFoakpKQGtxk+fDiuX7+OuLg4CIKAmzdvYs+ePZg0aVKj3xMVFQWlUql9ubu7N7tGark/BLihe1crFFfU4IuTWWKXQ0REHYje4ebHH39EcHAwAGDPnj1wcnJCdnY2du7ciY0bNzZ7P0VFRaitrYWTk5NOu5OTEwoKGr6NePjw4fjyyy8RHh4OuVyObt26wdbWFv/4xz8a/Z7IyEioVCrtKzc3t9k1UsuZy6SICKt7qObWxEzcqagRuSIiIuoo9A43lZWVsLa2BgAcPnwYzzzzDKRSKYYNG4bs7Gy9C5BIJDrvBUGo1/bAxYsXMX/+fKxcuRIpKSn49ttvkZWVhblz5za6fwsLC9jY2Oi8qG38zs8ZfZ1tUFatxpaEq2KXQ0REHYTe4aZnz57Yv38/cnNzcejQIe1ppcLCQr2Cg729PWQyWb1ZmsLCwnqzOQ9ERUVhxIgRWLJkCQYMGIDx48dj06ZN2L59O/Lz8/UdChmYVCrBkvF1szc7kq6hQFUlckVERNQR6B1uVq5cicWLF6N79+4IDAxEUFAQgLpZnEGDBjV7P3K5HP7+/oiPj9dpj4+Px/DhwxvcprKyElKpbskymQwAuNx/OzXG1wEBnl1QrdbgH0czxC6HiIg6AL3DzXPPPYecnBycPXsW3377rbZ97Nix+PDDD/XaV0REBD7//HNs374dly5dwqJFi5CTk6M9zRQZGYlZs2Zp+0+ePBmxsbHYvHkzMjMz8f3332P+/PkYOnQoXFxc9B0KtQGJRIKlT/UGAMScyUV2cYXIFRERkakza8lG3bp1Q7du3QAApaWlOHr0KHx9fdG7d2+99hMeHo7i4mKsXr0a+fn56N+/P+Li4uDpWfcAxvz8fJ01b1566SWUlZXh448/xhtvvAFbW1s8+eSTeP/991syDGojQ73sMMbXAcfTb+GD+Cv46Pnmz/ARERHpSyLoeT5n2rRpGDVqFF5//XXcvXsXAwcOxLVr1yAIAnbv3o1nn33WULW2itLSUiiVSqhUKl5c3IbO56nwu3+chEQCxM0PRh9n/tkTEVHz6fP7rfdpqcTERO2t4Pv27YMgCCgpKcHGjRvx97//vWUVk8nr76rEpAHOEARg/eF0scshIiITpne4UalUsLOzAwB8++23ePbZZ2FlZYVJkyYhI4MXjFLj3gjtBZlUgiOXCpGSfVvscoiIyETpHW7c3d2RnJyMiooKfPvtt9pbwe/cuQOFQtHqBZLp8HbojD/4uwEA1nybzjvciIjIIPQONwsXLsSMGTPg5uYGFxcXjBkzBkDd6So/P7/Wro9MzPyxPpCbSXEq6zYSM4rELoeIiEyQ3uHmtddeQ3JyMrZv346TJ09q153x9vbmNTf0SC62lpg5rO5uuLWHLnP2hoiIWp3ed0v91oNNG3tcQnvEu6XEV1xejVFrjqGiphabZgzGRD9nsUsiIqJ2zqB3SwHAzp074efnB0tLS1haWmLAgAH45z//2aJiqePp2tkCc4K9AQDrDqdDXasRuSIiIjIleoebDz74AK+++iomTpyIr776CjExMXjqqacwd+5cvVcopo5rTrAXuliZI/NWBWJ/zBO7HCIiMiF6n5by8vLC22+/rfNYBAD4v//7P6xatQpZWVmtWmBr42mp9uOzxEz8b9wluCgVOLZkDCzMZGKXRERE7ZRBT0vl5+c3+GDL4cOH88ncpJeZQZ7oZqPADVUVvvwh59EbEBERNYPe4aZnz5746quv6rXHxMTAx8enVYqijkFhLsOCcXX/znxy7BeUV6tFroiIiEyB3g/OfPvttxEeHo7ExESMGDECEokEJ0+exHfffddg6CFqynP+btiamImsogpsP5mF+WMZkImI6PHoPXPz7LPP4tSpU7C3t8f+/fsRGxsLe3t7nD59Gk8//bQhaiQTZi6TYlFoLwB11+DcqagRuSIiIjJ2j7XOjTHiBcXtj0Yj4Hf/OImL+aX48yhvRE7sI3ZJRETUzrT6BcWlpaXNfhHpSyqVYMl4XwDAjqRrKFBViVwREREZs2Zdc2Nra/vIVYgFQYBEIkFtbW2rFEYdyxhfBwzp3gVnrt3BxqMZePdpPqeMiIhaplnh5tixY4augzo4iUSCJeN7Y9qnyfjqTC5eCfZGd/tOYpdFRERGqFnhZvTo0YaugwhDvewwxtcBx9Nv4cMjV/DR84PELomIiIxQi54tRWQoi8Pqrr058NMNXMrnNVxERKQ/hhtqV/q7KvG7Ac4QBGDdoXSxyyEiIiPEcEPtTkRoL8ikEnx3uRAp2bfFLoeIiIwMww21O94OnfEHfzcAwJpv09HBlmIiIqLH1KJwo1arceTIEXz66acoKysDANy4cQPl5eWtWhx1XAvG+UBuJsWprNtIzCgSuxwiIjIieoeb7Oxs+Pn5YcqUKZg3bx5u3boFAFizZg0WL17c6gVSx+SstMSsYZ4AgLWHLkOj4ewNERE1j97hZsGCBQgICMCdO3dgaWmpbX/66afx3XfftWpx1LG9FtITneQynM8rxbcXCsQuh4iIjITe4ebkyZN46623IJfLddo9PT2Rl5fXaoUR2XWSY06wNwBg3eF0qGs1IldERETGQO9wo9FoGnzEwvXr12Ftbd0qRRE9MCfYC12szJF5qwKxPzI8ExHRo+kdbkJDQ7Fhwwbte4lEgvLycvztb3/DxIkTW7M2IlgrzDEvpCcAYMORK6i6x2eXERFR0/QONx9++CESEhLQt29fVFVVYfr06ejevTvy8vLw/vvvG6JG6uD+OMwTzkoFbqiqsOtUjtjlEBFROycRWrCIyN27dxEdHY0ff/wRGo0GgwcPxowZM3QuMG6vSktLoVQqoVKpYGNjI3Y51EzRp3MQGXsOXTvJkbA0BJ0tmvVYNCIiMhH6/H63KNwYM4Yb46Su1SD0w0RkFVUgIrQX5o/1EbskIiJqQ/r8fuv9198DBw402C6RSKBQKNCzZ094eXnpu1uiJpnJpIgI7YW/RKfis8RMzBzmiS6d5I/ekIiIOhy9w83UqVMhkUjqLYn/oE0ikWDkyJHYv38/unTp0mqFEk3yc8bm41dxMb8UWxKuInJiH7FLIiKidkjvC4rj4+MxZMgQxMfHQ6VSQaVSIT4+HkOHDsV//vMfJCYmori4mKsVU6uTSiVYMt4XALAj6RoKVFUiV0RERO2R3jM3CxYswNatWzF8+HBt29ixY6FQKPDKK6/gwoUL2LBhA15++eVWLZQIAMb4OmBI9y44c+0ONh7NwLtP+4ldEhERtTN6z9xcvXq1wQt5bGxskJmZCQDw8fFBUREfdkitTyKRYOlTvQEAX53JxbWiCpErIiKi9kbvcOPv748lS5ZoH5gJALdu3cLSpUsxZMgQAEBGRgbc3Nxar0qi3xjS3Q4hvg5QawR8eOSK2OUQEVE7o3e42bZtG7KysuDm5oaePXvCx8cHbm5uuHbtGj7//HMAQHl5Of7617+2erFED7wRVnftzYGfbuBSfqnI1RARUXvSonVuBEHAoUOHcOXKFQiCgN69eyM0NBRSqd5Zqc1xnRvT8fquH/Gfn/Mxtrcjtr00ROxyiIjIgLiIXxMYbkxHVlEFxn2QgFqNgD1zgxDQ3U7skoiIyEAMuogfAFRUVCAhIQE5OTmoqanR+Wz+/Pkt2SWR3rzsO2FagBuiT+dizaF0xLwyDBKJROyyiIhIZHqHm9TUVEycOBGVlZWoqKiAnZ0dioqKYGVlBUdHR4YbalPzx/pg7495OJ11G4kZRRjdy0HskoiISGR6XySzaNEiTJ48Gbdv34alpSV++OEHZGdnw9/fH+vWrTNEjUSNclZaYtYwTwDA2kOXodF0qLOsRETUAL3DTVpaGt544w3IZDLIZDJUV1fD3d0da9aswfLlyw1RI1GTXgvpic4WZjifV4r/ni8QuxwiIhKZ3uHG3Nxce12Dk5MTcnJyAABKpVL7z0Rtya6THHOC6x7Wuj4+HepajcgVERGRmPQON4MGDcLZs2cBACEhIVi5ciW+/PJLLFy4EH5++i+Fv2nTJnh5eUGhUMDf3x8nTpxosn91dTVWrFgBT09PWFhYoEePHti+fbve30umZfZIL3SxMkfmrQrE/pgndjlERCQivcPNu+++C2dnZwDAO++8g65du+LVV19FYWEhtm7dqte+YmJisHDhQqxYsQKpqakIDg7GhAkTmpwBmjZtGr777jts27YN6enpiI6ORu/evfUdBpkYa4U55oX0BABsOHIFVfdqRa6IiIjEotc6N4IgICcnB46OjrC0tHzsLw8MDMTgwYOxefNmbVufPn0wdepUREVF1ev/7bff4vnnn0dmZibs7Fq2pgnXuTFdVfdqEbLuOPJVVfjr7/pi9kgvsUsiIqJWos/vt14zN4IgwMfHB9evX3+sAgGgpqYGKSkpCAsL02kPCwtDUlJSg9scOHAAAQEBWLNmDVxdXdGrVy8sXrwYd+/ebfR7qqurUVpaqvMi06Qwl2HBWB8AwKZjv6C8Wi1yRUREJAa9wo1UKoWPjw+Ki4sf+4uLiopQW1sLJycnnXYnJycUFDR8x0tmZiZOnjyJ8+fPY9++fdiwYQP27NmDefPmNfo9UVFRUCqV2pe7u/tj107t13P+bvCy74TiihpsP5kldjlERCQCva+5WbNmDZYsWYLz58+3SgEPrygrCEKjq8xqNBpIJBJ8+eWXGDp0KCZOnIgPPvgAO3bsaHT2JjIyEiqVSvvKzc1tlbqpfTKTSRER2gsA8FliJu5U1DxiCyIiMjV6h5s//vGPOH36NAYOHAhLS0vY2dnpvJrL3t4eMpms3ixNYWFhvdmcB5ydneHq6gqlUqlt69OnDwRBaPRUmYWFBWxsbHReZNom+Tmjr7MNyqrV2JxwVexyiIiojen9+IUNGza0yhfL5XL4+/sjPj4eTz/9tLY9Pj4eU6ZMaXCbESNG4Ouvv0Z5eTk6d+4MALhy5QqkUinc3NxapS4yflKpBEue8sWfvjiD/0u6hpdHeKGbUiF2WURE1EZEfSp4TEwMZs6ciS1btiAoKAhbt27FZ599hgsXLsDT0xORkZHIy8vDzp07AQDl5eXo06cPhg0bhrfffhtFRUWYM2cORo8ejc8++6xZ38m7pToGQRAQ/ukPOH3tNqYHeuDdp/Vfg4mIiNoPg90t9cDVq1fx1ltv4YUXXkBhYSGAutu0L1y4oNd+wsPDsWHDBqxevRpPPPEEEhMTERcXB0/PumcF5efn66x507lzZ8THx6OkpAQBAQGYMWMGJk+ejI0bN7ZkGGTCJJK62RsA+OpMLq4VVYhcERERtRW9Z24SEhIwYcIEjBgxAomJibh06RK8vb2xZs0anD59Gnv27DFUra2CMzcdy5++OI1j6bfw+4Eu2PjCILHLISKiFjLozM2yZcvw97//HfHx8ZDL5dr2kJAQJCcn618tkQEtHl83e3Pgpxu4eINrHBERdQR6h5tz587pXAD8gIODQ6usf0PUmvq5KDF5oAsAYP3hdJGrISKitqB3uLG1tUV+fn699tTUVLi6urZKUUStKSK0F2RSCb67XIiz126LXQ4RERmY3uFm+vTpePPNN1FQUACJRAKNRoPvv/8eixcvxqxZswxRI9Fj8bLvhGkBdUsFrDmUDhFvECQiojagd7j53//9X3h4eMDV1RXl5eXo27cvRo0aheHDh+Ott94yRI1Ej23+WB/IzaQ4nXUbiRlFYpdDREQG1OJ1bq5evYrU1FRoNBoMGjQIPj4+rV2bQfBuqY7rfw9exGcnstDf1QYH5o2EVNrwYz6IiKj90ef3W+8VihMSEjB69Gj06NEDPXr0aHGRRG3t1TE9EX06F+fzSvHf8wWYNMBZ7JKIiMgA9D4tFRoaCg8PDyxbtqzVHp5J1BbsOskxJ9gLALA+Ph3qWo3IFRERkSHoHW5u3LiBpUuX4sSJExgwYAAGDBiANWvWNPrgSqL2ZE6wN+w6yZF5qwKxP+aJXQ4RERmA3uHG3t4er7/+Or7//ntcvXoV4eHh2LlzJ7p3744nn3zSEDUStdimTZvg5eUFhUIBf39/pJ5Oxmtj6k6nbjhyBVX3arV9jx8/DolEUu91+fJlnX3u3bsXffv2hYWFBfr27Yt9+/a16ZiIiKhpLXq21ANeXl5YtmwZ3nvvPfj5+SEhIaG16iJ6bDExMVi4cCFWrFiB1NRUBAcHY8KECRjlIoGzUoEbqip8eSqn3nbp6enIz8/Xvn57sXxycjLCw8Mxc+ZM/PTTT5g5cyamTZuGU6dOteXQiIioCS2+W+r777/Hl19+iT179qCqqgq///3vMWPGDEyYMKG1a2xVvFuq4wgMDMTgwYOxefNmbVufPn0wdepUDHz6VSyLPQe7TnIkLg1BZwszHD9+HCEhIbhz5w5sbW0b3Gd4eDhKS0vx3//+V9v21FNPoUuXLoiOjjb0kIiIOiyDPltq+fLl8PLywpNPPons7Gxs2LABBQUF+Ne//tXugw11HDU1NUhJSUFYWJhOe1hYGJKSkvCcvxu87TvhdkUNtp/M0ukzaNAgODs7Y+zYsTh27JjOZ8nJyfX2OX78eCQlJRlmIEREpDe9w83x48exePFi5OXl4eDBg5g+fTqsrKwMURtRixUVFaG2thZOTk467U5OTigoKICZTIqIsF4AgM8SM3GnogbOzs7YunUr9u7di9jYWPj6+mLs2LFITEzUbl9QUNDoPomIqH3Qe50b/g2VjIlEortQnyAI2raJ/Z3R1/kqLuaXYnPCVSyf2Ae+vr7avkFBQcjNzcW6deswatSoZu2TiIjEp3e4eeDixYvIyclBTU2NTvvvf//7xy6K6HHZ29tDJpPVm1EpLCzUzrxIpRIsecoXf/riDP4v6RpeHuGFbkqFTv9hw4bhX//6l/Z9t27dmtwnERGJT+9wk5mZiaeffhrnzp2DRCLRPoTwwd9ca2trm9qcqE3I5XL4+/sjPj4eTz/9tLY9Pj4eU6ZM0b4f08sBQ7vb4fS12/jouwxEPeOns5/U1FQ4O/+6knFQUBDi4+OxaNEibdvhw4cxfPhwA46GiIj0oXe4WbBgAby8vHDkyBF4e3vj9OnTKC4uxhtvvIF169YZokaiFomIiMDMmTMREBCAoKAgbN26FTk5OZg7dy4AIDIyEnl5eViyegP+sCUZn236B7zKn0RIkD9qamrwr3/9C3v37sXevXu1+1ywYAFGjRqF999/H1OmTME333yDI0eO4OTJk2INk4iIHqJ3uElOTsbRo0fh4OAAqVQKqVSKkSNHIioqCvPnz0dqaqoh6iTSW3h4OIqLi7F69Wrk5+ejf//+iIuLg6enJwAgPz8fOTk5GNLdDiG+Dth/6h6WLFmCKlURrCwt0b9/Pxw8eBATJ07U7nP48OHYvXs33nrrLfz1r39Fjx49EBMTg8DAQLGGSURED9F7nZsuXbogJSUF3t7e6NGjBz7//HOEhITg6tWr8PPzQ2VlpaFqbRVc54YacuGGCpM2/jr7IpUAUc/4IXyIh4hVERHRAwZ9Knj//v3x888/w9vbG4GBgVizZg3kcjm2bt0Kb2/vFhdNJCa7TnKd9xoBiIw9h1G9HOCstBSpKiIiagm917l56623oNHUPU3573//O7KzsxEcHIy4uDhs3Lix1QskagtZRRX12jQCMP2zU4g+nYOKarUIVRERUUu0+PELv3X79m106dLFKNb64Gkpaki+6i5GvHcUmkb+a+hsYYapg1wwfagn+rrw3xsioramz+93q4QbY8JwQ42JOZOD5bHnUSsIkEkkWDGpD9QaDaJP5+rM7DzhbosZgR743QAXWMplIlZMRNRxMNw0geGGmpKvuotrRZXobm+lvdZGEAQkXy3Gl6dzcOh8AdT3p3esFWZ4drAbpgd6oJeTtZhlExGZPIabJjDc0OO4VVaNr1NyEX06B7m372rbh3TvgumBHpjQ3xkKc87mEBG1NoabJjDcUGvQaASc+KUIu05l48ilQtTen82xtTLHc4Pd8EKgB3o4dBa5SiIi08Fw0wSGG2ptN0urEHMmF7tP5+CGqkrbPszbDjMCPTG+XzfIzfS+MZGIiH6D4aYJDDdkKLUaAQlXCvHlDzk4ll6ovfOqayc5/hDgjheGusOzaydxiyQiMlIMN01guKG2kFdyFzFnchFzJgc3S6u17cE+9pgR6IGxfZxgLuNsDhFRczHcNIHhhtqSulaD7y4XYtepHCRm3MKD/9ocrC0QHuCO54e6w62LlbhFEhEZAYabJjDckFhyb1ci+nQOvjp7HUXldbM5EgkwppcDpgd6IsTXAWaczSEiahDDTRMYbkhsNWoN4i/exK7T2fj+l2Jtu7NSgfAh7ggf4s7nWRERPYThpgkMN9SeZBVVIPp0DvakXMftihoAdU8kf7K3E2YM88AoHwfIpO3/sSZERIbGcNMEhhtqj6rVtfj2fAF2ncrBqazb2nZXW0u8MNQd0wLc4WijELFCIiJxMdw0geGG2rtfCsuw61Qu9qTkorSq7mnkZlIJQvs6YXqgB0b0sIeUszlE1MEw3DSB4YaMRdW9Whz8OR+7TucgJfuOtt2zqxVeGOqB5/zdYN/ZQsQKiYjaDsNNExhuyBhdLijFrlM52PdjHsqq62ZzzGUSjO/XDTMCPTHM2w4SCWdziMh0Mdw0geGGjFlljRr/+SkfX57Kxk/XVdp2b4dOmH5/NsfWSi5ihUREhsFw0wSGGzIV5/NU2HU6B9+k5qGiphYAIDeTYpKfM6YHeiDAswtnc4jIZDDcNIHhhkxNebUa36TlYdepHFy4Uapt7+XUGdOHeuDpwW5QWpqLWCER0eNjuGkCww2ZKkEQ8NN1FXadysaBn26g6p4GAKAwl2LyABdMD/TAE+62nM0hIqPEcNMEhhvqCFR372F/at1sTvrNMm17H2cbTA/0wNQnXGCt4GwOERkPhpsmMNxQRyIIAn7MuYMvf8jBf87lo0ZdN5tjJZdhyhMumD7UE35uSpGrJCJ6NIabJjDcUEdVUlmDvT/mYdepbFy9VaFtH+CmxPShHvj9Ey6wkpuJWCERUeP0+f0W/RHEmzZtgpeXFxQKBfz9/XHixIlmbff999/DzMwMTzzxhGELJDIRtlZyzB7phSMRo7H7lWH4/UAXmMsk+Pm6CstizyHwf7/DX/efx6X80kfvjIioHRN15iYmJgYzZ87Epk2bMGLECHz66af4/PPPcfHiRXh4eDS6nUqlwuDBg9GzZ0/cvHkTaWlpzf5OztwQ/aq4vBp7Uq4j+nQOrhVXatsHedhiRqAnfjfAGQpzmYgVEhHVMZrTUoGBgRg8eDA2b96sbevTpw+mTp2KqKioRrd7/vnn4ePjA5lMhv379zPcED0mjUZA0tVi7DqdjcMXbkKtqfvfgo3CDM8MdsOMQA/4OFmLXCURdWRGcVqqpqYGKSkpCAsL02kPCwtDUlJSo9t98cUXuHr1Kv72t78163uqq6tRWlqq8yIiXVKpBCN97LFphj+SIp/EkvG+cOtiidIqNXYkXUPoh4mYtiUZ36TloVpdK3a5RERNEu3qwaKiItTW1sLJyUmn3cnJCQUFBQ1uk5GRgWXLluHEiRMwM2te6VFRUXj77bcfu16ijsLRWoF5IT3x6ugeSMy4hV2ncvDd5UKcvnYbp6/dRhcrczzn74YXhnrA26Gz2OUSEdUj+q0RDy8oJghCg4uM1dbWYvr06Xj77bfRq1evZu8/MjISERER2velpaVwd3dvecFEHYRUKsEYX0eM8XVEgaoKMWdysftMDvJVVfjsRBY+O5GF4T26YnqgB8L6doPcTPT7E4iIAIgYbuzt7SGTyerN0hQWFtabzQGAsrIynD17FqmpqXj99dcBABqNBoIgwMzMDIcPH8aTTz5ZbzsLCwtYWFgYZhBEHUQ3pQILxvlgXkgPHE+/hV2nc3AsvRBJV4uRdLUY9p3l+EOAO14Y4gGPrlZil0tEHZzoFxT7+/tj06ZN2ra+fftiypQp9S4o1mg0uHjxok7bpk2bcPToUezZswdeXl7o1KnTI7+TFxQTtY7rdyoRcyYXMWdyUVhWrW0P9rHHjEAPjO3jBHMZZ3OIqHXo8/st6mmpiIgIzJw5EwEBAQgKCsLWrVuRk5ODuXPnAqg7pZSXl4edO3dCKpWif//+Ots7OjpCoVDUayciw3PrYoU3wnwxf6wPvrtUiC9PZeNERpH25WhtgfAh7nh+qAdcbS3FLpeIOhBR/1oVHh6ODRs2YPXq1XjiiSeQmJiIuLg4eHp6AgDy8/ORk5MjZolE9AjmMime6t8N/5wdiMQlIXh1TA/Yd5ajsKwa/zj6C4LfP4qXd5zBkYs3UavRf6JYn4U+T548iREjRqBr166wtLRE79698eGHHzbaf/fu3ZBIJJg6daredRFR+8XHLxBRq6tRa3D4YgF2ncpB0tVibbuLUoHwIR4IH+KObkrFI/ej70KfqampuHz5MgYMGIBOnTrh5MmT+POf/4wPP/wQr7zyik7f7OxsjBgxAt7e3rCzs8P+/fsfe9xEZDhGs4ifGBhuiNpW5q1yRJ/OwZ6U67hTeQ8AIJNK8GRvR8wI9MAoHwdIpfXvkARavtDnbz3zzDPo1KkT/vnPf2rbamtrMXr0aPzpT3/CiRMnUFJSwnBD1M4ZxSJ+RNQxeDt0xopJfZEcORYbwp/A0O52qNUIiL94Ey99cQaj1h7DJ8d+QWFZlc52LV3o87dSU1ORlJSE0aNH67SvXr0aDg4OmD179uMNjojaJdHXuSGijkFhLsPUQa6YOsgVGTfLsOt0DvamXMf1O3ex9lA6Poy/grB+Tpg+1BPDe3Rt0UKfD7i5ueHWrVtQq9VYtWoV5syZo/3s+++/x7Zt2/R6bAsRGReGGyJqcz5O1vjb5H5YOr43Dp7Lx65T2fgxpwRx5woQd64A3btaYYJ33fpUzV3o87dOnDiB8vJy/PDDD1i2bBl69uyJF154AWVlZfjjH/+Izz77DPb29gYbHxGJi+GGiERjKZfhOX83POfvhkv5pdh1Kgf7UvNwrbgSmwpVgESKv3+dhL869UKglx0kEkmjC33+lpeXFwDAz88PN2/exKpVq/DCCy/g6tWruHbtGiZPnqztq9FoAABmZmZIT09Hjx49DDdgImoTDDdE1C70cbbBO1P7Y9mE3vj3Tzew63QOCrr1ROLxo3he3hs9HDrhhaEeOHT4MJ7W49ZtQRBQXV23yGDv3r1x7tw5nc/feustlJWV4aOPPuKjWYhMBMMNEbUrnSzM8PxQDzw/1ANrpcuwbMErsHbrhcuOvojYvQHlV6/hlmswzl67jT1b1uDGjRvYuXMnAOCTTz6Bh4cHevfuDaBu3Zt169bhL3/5CwA0uOinra0tAHAxUCITwnBDRO3Wktf+hE64i/ffX4Mb+fmwcuoOxz+swpHrAo5sSca9oz/BWn0HpVX3YKMwx52Kaqx/YykK8nJgbmaGHj164L333sOf//xnsYdCRG2I69wQkdEQBAFpuSXYdSoH//75Bqru1V0vozCXor+LEik5dyAIgFQCRD3jh/Ah9Rf6IyLjxEX8msBwQ2QaVHfvYd+P17HrdA6u3Cyv97lEAkT/zzAM7W7X6CKBRGQ8GG6awHBDZFoEQcCOpGt4+98XG/zc2sIM/V2VGOCmxAA3WwxwU8Kti+UjbycnovbFaJ4KTkT0uCQSCZ7q3w3v/OciHn4up1wmQVm1GsmZxUjO/PUZV12szOHnZosBvwk9TjYWDDxEJoIzN0RkEmLO5GB57HnUCgJkEgnefaY/nh3shozCcvx8vQQ/X1fh5+sqXC4oxb3a+v/bc7C2wEA3Jfxcbe8HHiW6drYQYSRE1BCelmoCww2R6cpX3cW1okp0t7eCs9KywT7V6lqkF5Thp+sqnLsfejIKy1H78LQPAFdbSwxwU8LPTYkBrrbwc1NCaWlu6GEQUQMYbprAcENED7tbU4uL+Sr8lKvCuTwVfr5egsyiCjT0f8fuXa201+74uSrR31WJThY8w09kaAw3TWC4IaLmKKu6h/N5pXWntPJUOHddhZzblfX6SSRAT4fOvwYeNyX6OttAYS4ToWoi08Vw0wSGGyJqqTsVNdqZnZ+v183y5Kuq6vUzk0rQy8kaA91/vYbHt5s1zGVSEaomMg0MN01guCGi1lRYWoVzeSqda3iKK2rq9ZObSdHH2eb+Rct1d2j1dOwMGdfgIWoWhpsmMNwQkSEJgoAbqiqcu15yP/DUzfSUVqnr9bU0l6G/q43ONTzdu3biooNEDWC4aQLDDRG1NUEQkF1cef/anbrQcz5Phcqa2np9rRVm2pmdB4GHiw4SMdw0ieGGiNqDWo2AzFvl2mt3frpegos3SlGt1tTra9dJDj9XZd0prfuhx8lGIULVROJhuGkCww0RtVf3ajW4crOs7lTW/QuXL+eXQd3AGjxONhbwc7W9H3jqZnrsOslFqPrRNm3ahLVr1yI/Px/9+vXDhg0bEBwc3GDf2NhYbN68GWlpaaiurka/fv2watUqjB8/XqdfSUkJVqxYgdjYWNy5cwdeXl5Yv349Jk6c2BZDIhHw8QtEREbIXCZFPxcl+rko8fz9tqp7tbhcUKa9WLlu0cEy3Cytxs3Smzhy6aZ2e7cu9xcdvB96+rspYaMQd9HBmJgYLFy4EJs2bcKIESPw6aefYsKECbh48SI8POo/tT0xMRGhoaF49913YWtriy+++AKTJ0/GqVOnMGjQIABATU0NQkND4ejoiD179sDNzQ25ubmwtrZu6+FRO8WZGyIiI1NZo8bFG6U6d2hlFlU02NfbvhP87l+7M9DdFv1cbGAlb7u/1wYGBmLw4MHYvHmztq1Pnz6YOnUqoqKimrWPfv36ITw8HCtXrgQAbNmyBWvXrsXly5dhbs4VozsKztwQEZkwK7kZArrbIaC7nbattOoezufVzeycu153Dc/1O3eRWVSBzKIKfJN2AwAglQA+jtbwc/v1Gp7e3awNsuhgTU0NUlJSsGzZMp32sLAwJCUlNWsfGo0GZWVlsLP7dawHDhxAUFAQ5s2bh2+++QYODg6YPn063nzzTchkXDyRGG6IiEyCjcIcw3vYY3gPe23b7Yoa/Hy9ROcanpul1Ui/WYb0m2XYk3IdAGAuk8C3m7XOQ0N7OT3+ooNFRUWora2Fk5OTTruTkxMKCgqatY/169ejoqIC06ZN07ZlZmbi6NGjmDFjBuLi4pCRkYF58+ZBrVZrZ3eoY2O4ISIyUXad5Bjj64gxvo7atpulVdq1d36+P9Nzu6IG5/NKcT6vFNGn6/pZmEnR18UGA1zrZncGuinh7dCyRQcfvo1dEIRm3doeHR2NVatW4ZtvvoGj469j0Gg0cHR0xNatWyGTyeDv748bN25g7dq1DDcEgOGGiKhDcbJRwKmvAuP61s2mCIKAvJK7909lqXAur+4anrIqNVJzSpCaUwIgGwDQSS5DP1fl/cCjxEA3W3h2tWo0qNjb20Mmk9WbpSksLKw3m/OwmJgYzJ49G19//TXGjRun85mzszPMzc11TkH16dMHBQUFqKmpgVzePu8ao7bDcENE1IFJJBK4dbGCWxcrTPBzBgBoNAKyb1f++gyt6yqcv6FCRU0tTmfdxums29rtbRRmGOBmW3c7uqsSA9xt4aJUQCKRQC6Xw9/fH/Hx8Xj66ae128THx2PKlCmN1hQdHY2XX34Z0dHRmDRpUr3PR4wYgV27dkGj0UAqrTt1duXKFTg7OzPYEADeLSV2OURERqFWI+Dq/UUHH4Sei/mlqGlg0cGuneT3n5Bui9s/H8OayL9gy5YtCAoKwtatW/HZZ5/hwoUL8PT0RGRkJPLy8rBz504AdcFm1qxZ+Oijj/DMM89o92lpaQmlUgkAyM3NRd++ffHSSy/hL3/5CzIyMvDyyy9j/vz5WLFiRdv8gXQgrb1OUWxsLN5991388ssvuHfvHnx8fPDGG29g5syZTdbBRfyawHBDRNQ67tVqkF5QpvOk9PSC+osOlv14EOVnYqEuvw1Xr15Y/Ld3MXPqU+jSSY6XXnoJ165dw/HjxwEAw0eOQvL3J+p914svvogdO3Zo3ycnJ2PRokVIS0uDq6srZs+ezbulDCAmJgYzZ87UWafo888/b3SdooULF8LFxQUhISHadYrWrVuns07R8ePHcefOHfTu3RtyuRz/+c9/8MYbb+DgwYP1Fmv8LYabJjDcEBEZTtW9WlzKL617pERu3TU8vxSWo4FFluFuZ4kB9+/Q8nNTIuNmGd7+90VohLpb1qOe8UP4kPo/oNR2DLFOUUMGDx6MSZMm4Z133mm0D9e5ISIiUSjMZRjk0QWDPLoAQXVtFdVqXLhR+us1PHkqZBVVIPf2XeTevouD5/Lr7UcjAMv2nkNWUQXculjB1socSkvdl7XCvEV3b1HzGGqdot8SBAFHjx5Feno63n///ceu+QGGGyIiMqhOFmYY6mWHoV6//sCp7v666ODP10twJus2iipqdLYTAGxJyGx0vxIJ0NnCTBt2fhuAbCzrhyGlpTlsLeX3g5EZpAxGTTLUOkUAoFKp4OrqiurqashkMmzatAmhoaGtVjvDDRERtTmlpTlG9LTHiJ51iw7mq+5ixHtHdU5fSQBM8OuGe7UCVHfvofTuPajuvypraiEIQFmVGmVValy/c1ev75dIAGsLMygbmBGy+U0IaujV0YJRa69TBADW1tZIS0tDeXk5vvvuO0RERMDb2xtjxoxplZoZboiISHTOSktEPeOH5bHnUSsIkEkkePeZ/o1ec1Oj1miDzsPBp96rUvf93Xt1wai0So3SKjVy0bJgZGslrxeKGpwtsvr1c2sL4wlGhlqnCACkUil69uwJAHjiiSdw6dIlREVFMdwQEZFpCR/igVG9HHCtqBLd7a3grLRstK/cTAoHaws4WFvo/T2NBaOSyhqo7qqbDE0PByN9SSWAtaKBENTADNLDwamtg5Gh1ilqiCAIqK6ufuyaH2C4ISKidsNZadlkqGkNjxOMqtW1Dc8UVd7TBqOSuzUNziRV3dNAI0D7Xl9SCerNDjU0W2T78GdWdcGoOaeSHhYREYGZM2ciICBAu05RTk4O5s6dCwBNrlM0bNgw7azPb9cpioqKQkBAAHr06IGamhrExcVh586dOndkPS6GGyIiomayMJPB0VoGR2uF3ts+HIxKKhs+jfbbYPSgT7W6LhiVVNa16auhYPSomSJbK3NMnPIMPvywCKtXr0Z+fj769++PuLg4eHp6AgDy8/ORk5Oj/Z5PP/0UarUa8+bNw7x587Ttv12nqKKiAq+99hquX78OS0tL9O7dG//6178QHh6u97gaw3VuiIiI2rmqe7VNXldUUtn4dUfVDawirQ+ZVAIbhVmjs0W/vUstJfsOPj+ZBcEAaxVxnRsiIiITojCXQWEug6ON/jNGD4JRSSMXWNebLfrNP9eoNajVCLhTeQ939Jwx0gjA8tjzGNXLweCnGh/GcENERGTCHjcYNXTnWUkDoej6nUpcuVmus32tIOBaUSXDDREREbUPD4KRUzOCUUNrFckkEnS3tzJghQ2Ttvk3EhERkcl5sFaR7P5dWQ/WKmrrWRuAMzdERETUSvRZq8iQGG6IiIio1bTFWkWPIvppqU2bNsHLywsKhQL+/v44ceJEo31jY2MRGhoKBwcH2NjYICgoCIcOHWrDaomIiKi9EzXcxMTEYOHChVixYgVSU1MRHByMCRMm6CwI9FuJiYkIDQ1FXFwcUlJSEBISgsmTJyM1NbWNKyciIqL2StRF/AIDAzF48GCdJZf79OmDqVOnIioqqln76NevH8LDw7Fy5coGP6+urtZ5XkVpaSnc3d25iB8REZER0WcRP9FmbmpqapCSkoKwsDCd9rCwMCQlJTVrHxqNBmVlZbCzs2u0T1RUFJRKpfbl7u7+WHUTERFR+yZauCkqKkJtbW29x6Y7OTnVe7x6Y9avX4+KigpMmzat0T6RkZFQqVTaV25u7mPVTURERO2b6HdLPfyUUkEQmvXk0ujoaKxatQrffPMNHB0dG+1nYWEBCwv9n/xKRERExkm0cGNvbw+ZTFZvlqawsLDebM7DYmJiMHv2bHz99dcYN26cIcskIiIiIyPaaSm5XA5/f3/Ex8frtMfHx2P48OGNbhcdHY2XXnoJu3btwqRJkwxdJhERERkZUU9LRUREYObMmQgICEBQUBC2bt2KnJwczJ07F0Dd9TJ5eXnYuXMngLpgM2vWLHz00UcYNmyYdtbH0tISSqVStHEQERFR+yFquAkPD0dxcTFWr16N/Px89O/fH3FxcfD09AQA5Ofn66x58+mnn0KtVmPevHmYN2+etv3FF1/Ejh072rp8IiIiaodEXedGDPrcJ09ERETtg1Gsc0NERERkCAw3REREZFIYboiIiMikMNwQERGRSWG4ISIiIpPCcENEREQmheGGiIiITArDDREREZkUhhsiIiIyKQw3REREZFIYboiIiMikMNwQERGRSWG4ISIiIpPCcENEREQmheGGiIiITArDDREREZkUhhsiIiIyKQw3REREZFIYboiIiMikMNwQERGRSWG4ISIiIpPCcENEREQmheGGiIiITArDDREREZkUhhsiIiIyKQw3REREZFIYboiIiMikMNwQERGRSWG4ISIiIpPCcENEREQmheGGiIiITArDDREREZkUhhsiIiIyKQw3REREZFIYboiIiMikMNwQERGRSWG4ISIiIpPCcENEREQmheGGiIiITArDDREREZkUhhsiIiIyKQw3REREZFIYboiIiMikMNwQERGRSWG4ISIiIpMierjZtGkTvLy8oFAo4O/vjxMnTjTZPyEhAf7+/lAoFPD29saWLVvaqFIiIiIyBqKGm5iYGCxcuBArVqxAamoqgoODMWHCBOTk5DTYPysrCxMnTkRwcDBSU1OxfPlyzJ8/H3v37m3jyomIiKi9kgiCIIj15YGBgRg8eDA2b96sbevTpw+mTp2KqKioev3ffPNNHDhwAJcuXdK2zZ07Fz/99BOSk5Ob9Z2lpaVQKpVQqVSwsbF5/EEQERGRwenz+23WRjXVU1NTg5SUFCxbtkynPSwsDElJSQ1uk5ycjLCwMJ228ePHY9u2bbh37x7Mzc3rbVNdXY3q6mrte5VKBaDuD4mIiIiMw4Pf7ebMyYgWboqKilBbWwsnJyeddicnJxQUFDS4TUFBQYP91Wo1ioqK4OzsXG+bqKgovP322/Xa3d3dH6N6IiIiEkNZWRmUSmWTfUQLNw9IJBKd94Ig1Gt7VP+G2h+IjIxERESE9r1Go8Ht27fRtWvXJr+nJUpLS+Hu7o7c3FyTPOVl6uMDTH+MHJ/xM/UxcnzGz1BjFAQBZWVlcHFxeWRf0cKNvb09ZDJZvVmawsLCerMzD3Tr1q3B/mZmZujatWuD21hYWMDCwkKnzdbWtuWFN4ONjY3J/ksLmP74ANMfI8dn/Ex9jByf8TPEGB81Y/OAaHdLyeVy+Pv7Iz4+Xqc9Pj4ew4cPb3CboKCgev0PHz6MgICABq+3ISIioo5H1FvBIyIi8Pnnn2P79u24dOkSFi1ahJycHMydOxdA3SmlWbNmafvPnTsX2dnZiIiIwKVLl7B9+3Zs27YNixcvFmsIRERE1M6Ies1NeHg4iouLsXr1auTn56N///6Ii4uDp6cnACA/P19nzRsvLy/ExcVh0aJF+OSTT+Di4oKNGzfi2WefFWsIOiwsLPC3v/2t3mkwU2Hq4wNMf4wcn/Ez9TFyfMavPYxR1HVuiIiIiFqb6I9fICIiImpNDDdERERkUhhuiIiIyKQw3BAREZFJYbjR06ZNm+Dl5QWFQgF/f3+cOHGiyf4JCQnw9/eHQqGAt7c3tmzZ0kaVtow+4zt+/DgkEkm91+XLl9uw4uZLTEzE5MmT4eLiAolEgv379z9yG2M7fvqO0ZiOYVRUFIYMGQJra2s4Ojpi6tSpSE9Pf+R2xnQMWzJGYzqGmzdvxoABA7SLuwUFBeG///1vk9sY0/HTd3zGdOwaEhUVBYlEgoULFzbZT4xjyHCjh5iYGCxcuBArVqxAamoqgoODMWHCBJ3b1X8rKysLEydORHBwMFJTU7F8+XLMnz8fe/fubePKm0ff8T2Qnp6O/Px87cvHx6eNKtZPRUUFBg4ciI8//rhZ/Y3t+AH6j/EBYziGCQkJmDdvHn744QfEx8dDrVYjLCwMFRUVjW5jbMewJWN8wBiOoZubG9577z2cPXsWZ8+exZNPPokpU6bgwoULDfY3tuOn7/geMIZj97AzZ85g69atGDBgQJP9RDuGAjXb0KFDhblz5+q09e7dW1i2bFmD/ZcuXSr07t1bp+3Pf/6zMGzYMIPV+Dj0Hd+xY8cEAMKdO3faoLrWBUDYt29fk32M7fg9rDljNOZjWFhYKAAQEhISGu1j7MewOWM05mMoCILQpUsX4fPPP2/wM2M/foLQ9PiM9diVlZUJPj4+Qnx8vDB69GhhwYIFjfYV6xhy5qaZampqkJKSgrCwMJ32sLAwJCUlNbhNcnJyvf7jx4/H2bNnce/ePYPV2hItGd8DgwYNgrOzM8aOHYtjx44Zssw2ZUzH73EZ4zFUqVQAADs7u0b7GPsxbM4YHzC2Y1hbW4vdu3ejoqICQUFBDfYx5uPXnPE9YGzHbt68eZg0aRLGjRv3yL5iHUOGm2YqKipCbW1tvYd6Ojk51XuY5wMFBQUN9ler1SgqKjJYrS3RkvE5Oztj69at2Lt3L2JjY+Hr64uxY8ciMTGxLUo2OGM6fi1lrMdQEARERERg5MiR6N+/f6P9jPkYNneMxnYMz507h86dO8PCwgJz587Fvn370Ldv3wb7GuPx02d8xnbsAGD37t348ccfERUV1az+Yh1DUR+/YIwkEonOe0EQ6rU9qn9D7e2FPuPz9fWFr6+v9n1QUBByc3Oxbt06jBo1yqB1thVjO376MtZj+Prrr+Pnn3/GyZMnH9nXWI9hc8dobMfQ19cXaWlpKCkpwd69e/Hiiy8iISGh0QBgbMdPn/EZ27HLzc3FggULcPjwYSgUimZvJ8Yx5MxNM9nb20Mmk9WbxSgsLKyXSh/o1q1bg/3NzMzQtWtXg9XaEi0ZX0OGDRuGjIyM1i5PFMZ0/FpTez+Gf/nLX3DgwAEcO3YMbm5uTfY11mOozxgb0p6PoVwuR8+ePREQEICoqCgMHDgQH330UYN9jfH46TO+hrTnY5eSkoLCwkL4+/vDzMwMZmZmSEhIwMaNG2FmZoba2tp624h1DBlumkkul8Pf3x/x8fE67fHx8Rg+fHiD2wQFBdXrf/jwYQQEBMDc3NxgtbZES8bXkNTUVDg7O7d2eaIwpuPXmtrrMRQEAa+//jpiY2Nx9OhReHl5PXIbYzuGLRljQ9rrMWyIIAiorq5u8DNjO34NaWp8DWnPx27s2LE4d+4c0tLStK+AgADMmDEDaWlpkMlk9bYR7Rga9HJlE7N7927B3Nxc2LZtm3Dx4kVh4cKFQqdOnYRr164JgiAIy5YtE2bOnKntn5mZKVhZWQmLFi0SLl68KGzbtk0wNzcX9uzZI9YQmqTv+D788ENh3759wpUrV4Tz588Ly5YtEwAIe/fuFWsITSorKxNSU1OF1NRUAYDwwQcfCKmpqUJ2drYgCMZ//ARB/zEa0zF89dVXBaVSKRw/flzIz8/XviorK7V9jP0YtmSMxnQMIyMjhcTERCErK0v4+eefheXLlwtSqVQ4fPiwIAjGf/z0HZ8xHbvGPHy3VHs5hgw3evrkk08ET09PQS6XC4MHD9a5RfPFF18URo8erdP/+PHjwqBBgwS5XC50795d2Lx5cxtXrB99xvf+++8LPXr0EBQKhdClSxdh5MiRwsGDB0Wounke3Hb58OvFF18UBME0jp++YzSmY9jQuAAIX3zxhbaPsR/DlozRmI7hyy+/rP3/i4ODgzB27FjtD78gGP/x03d8xnTsGvNwuGkvx1AiCPev7CEiIiIyAbzmhoiIiEwKww0RERGZFIYbIiIiMikMN0RERGRSGG6IiIjIpDDcEBERkUlhuCEiIiKTwnBDREREJoXhhog6vOPHj0MikaCkpETsUoioFTDcEBERkUlhuCEiIiKTwnBDRKITBAFr1qyBt7c3LC0tMXDgQOzZswfAr6eMDh48iIEDB0KhUCAwMBDnzp3T2cfevXvRr18/WFhYoHv37li/fr3O59XV1Vi6dCnc3d1hYWEBHx8fbNu2TadPSkoKAgICYGVlheHDhyM9Pd2wAycig2C4ISLRvfXWW/jiiy+wefNmXLhwAYsWLcIf//hHJCQkaPssWbIE69atw5kzZ+Do6Ijf//73uHfvHoC6UDJt2jQ8//zzOHfuHFatWoW//vWv2LFjh3b7WbNmYffu3di4cSMuXbqELVu2oHPnzjp1rFixAuvXr8fZs2dhZmaGl19+uU3GT0Sti08FJyJRVVRUwN7eHkePHkVQUJC2fc6cOaisrMQrr7yCkJAQ7N69G+Hh4QCA27dvw83NDTt27MC0adMwY8YM3Lp1C4cPH9Zuv3TpUhw8eBAXLlzAlStX4Ovri/j4eIwbN65eDcePH0dISAiOHDmCsWPHAgDi4uIwadIk3L17FwqFwsB/CkTUmjhzQ0SiunjxIqqqqhAaGorOnTtrXzt37sTVq1e1/X4bfOzs7ODr64tLly4BAC5duoQRI0bo7HfEiBHIyMhAbW0t0tLSIJPJMHr06CZrGTBggPafnZ2dAQCFhYWPPUYialtmYhdARB2bRqMBABw8eBCurq46n1lYWOgEnIdJJBIAddfsPPjnB347KW1padmsWszNzevt+0F9RGQ8OHNDRKLq27cvLCwskJOTg549e+q83N3dtf1++OEH7T/fuXMHV65cQe/evbX7OHnypM5+k5KS0KtXL8hkMvj5+UGj0ehcw0NEposzN0QkKmtrayxevBiLFi2CRqPByJEjUVpaiqSkJHTu3Bmenp4AgNWrV6Nr165wcnLCihUrYG9vj6lTpwIA3njjDQwZMgTvvPMOwsPDkZycjI8//hibNm0CAHTv3h0vvvgiXn75ZWzcuBEDBw5EdnY2CgsLMW3aNLGGTkQGwnBDRKJ755134OjoiKioKGRmZsLW1haDBw/G8uXLtaeF3nvvPSxYsAAZGRkYOHAgDhw4ALlcDgAYPHgwvvrqK6xcuRLvvPMOnJ2dsXr1arz00kva79i8eTOWL1+O1157DcXFxfDw8MDy5cvFGC4RGRjvliKidu3BnUx37tyBra2t2OUQkRHgNTdERERkUhhuiIiIyKTwtBQRERGZFM7cEBERkUlhuCEiIiKTwnBDREREJoXhhoiIiEwKww0RERGZFIYbIiIiMikMN0RERGRSGG6IiIjIpPw/eoVjEwZCx54AAAAASUVORK5CYII=", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "y_avg = []\n", - "for y in epochs_y:\n", - " y_avg.append(np.mean(y))\n", - "\n", - "plt.plot(np.arange(len(epochs_x)), y_avg, marker = '.')\n", - "plt.xlabel('epoch')\n", - "plt.ylabel('average loss')\n", - "plt.ylim(0,)\n", - "for i, txt in enumerate(y_avg):\n", - " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom')\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkkAAAG2CAYAAABrrBJlAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAABcAklEQVR4nO3deVxU5f4H8M8wwLCjgAKDyKIIiog7omaaa+4rpKWmWd703iTLNbXUlNS0Ukuvv1tRWmqmqDf1CuYWSeYCuOCCioAKorIM6wAz5/cHOTkyIIMDMwOf9+vF68bxOc98n063+fScc55HJAiCACIiIiJSY6LvAoiIiIgMEUMSERERkQYMSUREREQaMCQRERERacCQRERERKQBQxIRERGRBgxJRERERBowJBERERFpwJBEREREpAFDEhEREZEGeg1JJ0+exLBhwyCVSiESibB37161PxcEAR999BGkUiksLS3Ru3dvXL58Wa2NXC7Hv/71Lzg5OcHa2hrDhw/HnTt3nvnZX331Fby8vGBhYYFOnTrht99+0+XQiIiIyMjpNSQVFBQgMDAQGzdu1Pjnq1evxrp167Bx40acOXMGLi4u6N+/P/Ly8lRtwsLCEBkZiR07diAmJgb5+fkYOnQoFApFpZ+7c+dOhIWF4YMPPkBcXBxeeOEFvPzyy0hNTdX5GImIiMg4iQxlg1uRSITIyEiMHDkSQPksklQqRVhYGObNmwegfNbI2dkZq1atwvTp05Gbm4smTZpg69atCA0NBQDcu3cP7u7uOHjwIAYOHKjxs4KCgtCxY0ds2rRJdax169YYOXIkwsPDa3egREREZBRM9V1AZZKTk5GRkYEBAwaojkkkErz44os4deoUpk+fjnPnzqG0tFStjVQqRdu2bXHq1CmNIamkpATnzp3D/Pnz1Y4PGDAAp06dqrQeuVwOuVyu+l2pVCIrKwuOjo4QiUTPM1QiIiKqI4IgIC8vD1KpFCYmVd9QM9iQlJGRAQBwdnZWO+7s7IyUlBRVG3NzczRu3LhCm8fnP+3hw4dQKBQa+63sHAAIDw/H0qVLtR4HERERGZ60tDQ0a9asyjYGG5Iee3qWRhCEZ87cVKeNtv0uWLAAs2fPVv2em5uL5s2bIy0tDXZ2dlV+FhERERkGmUwGd3d32NraPrOtwYYkFxcXAOWzRa6urqrjmZmZqlkgFxcXlJSUIDs7W202KTMzE927d9fYr5OTE8RicYVZoyf71UQikUAikVQ4bmdnx5BERERkZKrzqIzBrpPk5eUFFxcXREdHq46VlJTgxIkTqgDUqVMnmJmZqbVJT0/HpUuXKg1J5ubm6NSpk9o5ABAdHV3pOURERNTw6HUmKT8/Hzdu3FD9npycjPj4eDg4OKB58+YICwvDypUr4ePjAx8fH6xcuRJWVlaYMGECAMDe3h5vvPEG3nvvPTg6OsLBwQHvv/8+AgIC0K9fP1W/ffv2xahRo/DPf/4TADB79mxMnDgRnTt3RnBwMLZs2YLU1FT84x//qNu/AURERGSw9BqSzp49iz59+qh+f/zMz+TJkxEREYG5c+eiqKgIM2bMQHZ2NoKCghAVFaV2H/Gzzz6DqakpQkJCUFRUhL59+yIiIgJisVjV5ubNm3j48KHq99DQUDx69AjLli1Deno62rZti4MHD8LDw6MORk1ERETGwGDWSTI2MpkM9vb2yM3N5TNJRERERkKb72+DfSaJiIiISJ8YkoiIiIg0YEgiIiIi0oAhiYiIiEgDhiQiIiIiDRiSiIiIiDRgSCIiIiLSgCGJiIiISAOGJCIiIiINGJKIiIiINGBIIiIiItKAIYmIiIhIA4YkIiIiIg0YkoiIiIg0YEgiIiIi0oAhiYiIiEgDhiQiIiIiDRiSiIiIiDRgSCIiIiLSgCGJiIiISAOGJCIiIiINGJKIiIiINGBIIiIiItKAIYmIiIhIA4YkIiIiIg0YkoiIiIg0YEgiIiIi0oAhiYiIiEgDhiQiIiIiDRiSiIiIiDRgSCIiIiLSgCGJiIiISAOGJCIiIiINGJKIiIiINGBIIiIiItLA4ENSXl4ewsLC4OHhAUtLS3Tv3h1nzpxR/blIJNL4s2bNmkr7jIiI0HhOcXFxXQyJiIiIjICpvgt4lmnTpuHSpUvYunUrpFIptm3bhn79+iExMRFubm5IT09Xa3/o0CG88cYbGDNmTJX92tnZ4dq1a2rHLCwsdF4/ERERGSeDDklFRUXYvXs39u3bh169egEAPvroI+zduxebNm3Cxx9/DBcXF7Vz9u3bhz59+sDb27vKvkUiUYVziYiIiB4z6NttZWVlUCgUFWZ4LC0tERMTU6H9/fv3ceDAAbzxxhvP7Ds/Px8eHh5o1qwZhg4diri4uCrby+VyyGQytR8iIiKqvww6JNna2iI4OBjLly/HvXv3oFAosG3bNpw+fbrCbTYA+O6772Bra4vRo0dX2a+fnx8iIiKwf/9+bN++HRYWFujRoweSkpIqPSc8PBz29vaqH3d39+ceHxERERkukSAIgr6LqMrNmzcxdepUnDx5EmKxGB07dkSrVq1w/vx5JCYmqrX18/ND//79sWHDBq0+Q6lUomPHjujVqxfWr1+vsY1cLodcLlf9LpPJ4O7ujtzcXNjZ2Wk/MCIiIqpzMpkM9vb21fr+NuhnkgCgRYsWOHHiBAoKCiCTyeDq6orQ0FB4eXmptfvtt99w7do17Ny5U+vPMDExQZcuXaqcSZJIJJBIJFr3TURERMbJoG+3Pcna2hqurq7Izs7G4cOHMWLECLU///rrr9GpUycEBgZq3bcgCIiPj4erq6uuyiUiIiIjZ/AzSYcPH4YgCPD19cWNGzcwZ84c+Pr6YsqUKao2MpkMu3btwtq1azX2MWnSJLi5uSE8PBwAsHTpUnTr1g0+Pj6QyWRYv3494uPj8eWXX9bJmIiIiMjwGXxIys3NxYIFC3Dnzh04ODhgzJgxWLFiBczMzFRtduzYAUEQMH78eI19pKamwsTk70mznJwcvPXWW8jIyIC9vT06dOiAkydPomvXrrU+HiIiIjIOBv/gtqHS5sEvIiIiMgzafH8bzTNJRERERHWJIYmIiIhIA4YkIiIiIg0YkoiIiIg0YEgiIiIi0oAhiYiIiEgDhiQiIiIiDRiSiIiIiDRgSCIiIiLSgCGJiIiISAOGJCIiIiINGJKIiIiINGBIIiIiItKAIYmIiIhIA4YkIiIiIg0YkoiIiIg0YEgiIiIi0oAhiYiIiEgDhiQiIiIiDRiSiIiIiDRgSCIiIiLSgCGJiIiISAOGJCIiIiINGJKIiIiINGBIIiIiItKAIYmIiIhIA4YkIiIiIg0YkoiIiIg0YEgiIiIi0oAhiYiIiEgDhiQiIiIiDRiSiIiIiDRgSCIiIiLSgCGJiIiISAOGJCIiIiINDD4k5eXlISwsDB4eHrC0tET37t1x5swZ1Z+//vrrEIlEaj/dunV7Zr+7d+9GmzZtIJFI0KZNG0RGRtbmMIiIiMjIGHxImjZtGqKjo7F161ZcvHgRAwYMQL9+/XD37l1Vm0GDBiE9PV31c/DgwSr7jI2NRWhoKCZOnIiEhARMnDgRISEhOH36dG0Ph4iIiIyESBAEQd9FVKaoqAi2trbYt28fhgwZojrevn17DB06FB9//DFef/115OTkYO/evdXuNzQ0FDKZDIcOHVIdGzRoEBo3bozt27dXqw+ZTAZ7e3vk5ubCzs6u2p9NRERE+qPN97dBzySVlZVBoVDAwsJC7bilpSViYmJUvx8/fhxNmzZFq1at8OabbyIzM7PKfmNjYzFgwAC1YwMHDsSpU6d0VzwREREZNYMOSba2tggODsby5ctx7949KBQKbNu2DadPn0Z6ejoA4OWXX8YPP/yAo0ePYu3atThz5gxeeuklyOXySvvNyMiAs7Oz2jFnZ2dkZGRUeo5cLodMJlP7ISIiovrLoEMSAGzduhWCIMDNzQ0SiQTr16/HhAkTIBaLAZTfOhsyZAjatm2LYcOG4dChQ7h+/ToOHDhQZb8ikUjtd0EQKhx7Unh4OOzt7VU/7u7uzz84IiIiMlgGH5JatGiBEydOID8/H2lpafjzzz9RWloKLy8vje1dXV3h4eGBpKSkSvt0cXGpMGuUmZlZYXbpSQsWLEBubq7qJy0trWYDIiIiIqNg8CHpMWtra7i6uiI7OxuHDx/GiBEjNLZ79OgR0tLS4OrqWmlfwcHBiI6OVjsWFRWF7t27V3qORCKBnZ2d2g8RERHVX6b6LuBZDh8+DEEQ4Ovrixs3bmDOnDnw9fXFlClTkJ+fj48++ghjxoyBq6srbt++jYULF8LJyQmjRo1S9TFp0iS4ubkhPDwcADBr1iz06tULq1atwogRI7Bv3z4cOXJE7WFwIiIiatgMfiYpNzcXM2fOhJ+fHyZNmoSePXsiKioKZmZmEIvFuHjxIkaMGIFWrVph8uTJaNWqFWJjY2Fra6vqIzU1VfWgNwB0794dO3bswLfffot27dohIiICO3fuRFBQkD6GSERET3jWIsJPmj59OkQiET7//PNn9vusRYS1+VxqGAx+JikkJAQhISEa/8zS0hKHDx9+Zh/Hjx+vcGzs2LEYO3bs85ZHREQ6Nm3aNFy6dAlbt26FVCrFtm3b0K9fPyQmJsLNzU3Vbu/evTh9+jSkUukz+3y8iPDy5csxatQoREZGIiQkBDExMar/QK7u51LDYdCLSRoyLiZJRKR71VlEGADu3r2LoKAgHD58GEOGDEFYWBjCwsIq7fdZiwhX93PJ+NWbxSSJiKhhqc4iwkqlEhMnTsScOXPg7+9frX6ftYhwdRcvpoaFIYmIiAxGdRYRXrVqFUxNTfHOO+9Uu99nLSJcnc+lhochiYiIDEpViwifO3cOX3zxBSIiIqpcAFiTZy0i/KzFi6nhYUgiIiKDUtUiwr/99hsyMzPRvHlzmJqawtTUFCkpKXjvvffg6elZaZ/VWURY28WLqf5jSCIiIoOkaRHhiRMn4sKFC4iPj1f9SKVSzJkzp8q3nbVZRLi6ixdT/WfwSwAQEVHDUtUiwmZmZnB0dFRrb2ZmBhcXF/j6+qqO1WQR4ao+lxomziQREZFBqWoR4eq6ces2Eq4nIz23CED1FhHWxedS/cJ1kmqI6yQRERmmnWdSsWDPRSgFwEQEhI8OQGiX5vouiwwE10kiIqIGRxAEnLrxEPN3lwckAFAKwII9F3ElXabf4sgo8ZkkIiIySkqlgCsZMvyZnIXTt7Lw5+0sZBWUVGwnAC9/8Rua2krQ2tUOfq62aO1S/r8tmtjATMz5AtKMIYmIiIxCmUKJy/dkOJ38CKdvZeHM7SzIisvU2khMRZCXaX6KJDNPjsy8Bzhx/YHqmJlYhJZNbdHaxbY8PLnawc/FDk1sJbU6FjIODElERGSQSsqUuHAnB6eTs3A6OQvnbmehoESh1sbaXIxOng4I8ir/adesESLj7mDhnktQCALEIhFWjm6LIe2kuJaRh6sZMlxN//t/8+RluJIuK78dF/d3v0425vBzsYOfi61q9qllUxtITLmwZEPCB7driA9uExHpVnGpAnGpOaqZori0bBSXKtXa2FmYoquXA7p6OSDIyxH+UjuYarhdlp5bhNsPC+HpZAVXe0uNnycIAu5kF+FqRh6upstw5a/glPyoAJq+GcUmIrRoYq2abXp8287ZTqL16t+kP9p8fzMk1RBDEhHR8ymQl+FcSnb5M0XJj5CQlosShXoocrA2R1dPBwR5lwcjPxc7iE1qN5AUlShw/X4erqTLcDUjTzXT9PStvccaW5mphSY/V1u0craFhRlnnQwRQ1IdYEgiItJOblEpzqWUP2T9R3IWLt3NhUKp/hXU1FaCIG9HdPVyQDcvB7RsamMQszSCICBDVvxXYMpTzT7delhQYQxA+dIDXk7W8HO1Q2vVLTs7SO0tDGI8DRlDUh1gSCIiqlpWQQn+TM5SzRQlpssq3MZya2RZ/jyRtwO6ejnC09HKqEJEcakCNzLzK8w6ZReWamxva2GK1i52aO1qCz/X8meefF1sYWXOR4TrCkNSHWBIIiJSl5lXrHod/3TyI1y/n1+hjaejFYK8ymeKgrwd0KyxlR4qrV2CIOBBnhxXHj/r9FeAupGZjzINs04iEeDhYFXhWadmjS1hUsu3FhsihqQ6wJBERA3dvZwinE5+pApGtx4WVGjj09Tmr0DkiK6eDnCxt9BDpYahpEyJmw/UZ52uZuThQZ5cY3sbiSl8XWzh51I+69Tmr2edbC0Mf5uUvLw8LF68GJGRkcjMzESHDh3wxRdfoEuXLigtLcWiRYtw8OBB3Lp1C/b29ujXrx8++eQTSKXSSvssLS1FeHg4vvvuO9y9exe+vr5YtWoVBg0aVK3PfUyb72/O7xER0TMJgoDUrMLy1/H/mim6k12k1kYkAvxc7FSv43fxcoCTDdcbeszc1AStXe3Q2lX9i/lhvly1LMGV9PLwdCMzH/l/Pdh+LiVbrb27gyX8XMr7af1XgPJwsDKoWadp06bh0qVL2Lp1K6RSKbZt24Z+/fohMTERNjY2OH/+PBYvXozAwEBkZ2cjLCwMw4cPx9mzZyvtc9GiRdi2bRv+7//+D35+fjh8+DBGjRqFU6dOoUOHDs/8XDc3N63HwZmkGuJMEhHVZ4Ig4OaDArWZogxZsVobsYkIbaV2qtfxu3g6wN7K8Gc5jEGpQonkhwXqs07peRWuwWOWZmL4utiWP+v01/pOfi52erkeRUVFsLW1xb59+zBkyBDV8fbt22Po0KH4+OOPK5xz5swZdO3aFSkpKWjeXPM+e1KpFB988AFmzpypOjZy5EjY2Nhg27Zt1f5cziQREZFWlEoB1+7nqR6y/jM5Cw/z1bf4MBOL0K5ZIwT9tU5RZ08H2Ej4NVIbzMQmaOVcfnttxBPHswtKnrhVVx6grmXkoahUgfi0HMSn5aj149bI8q/bdbaq2SdPRyuNa0vpSllZGRQKBSws1G+tWlpaIiYmRuM5ubm5EIlEaNSoUaX9yuXyKvusyec+C2eSaogzSURkzBRKAYmPt/hILt/iI+epN7LMTU3Qwb0Rgrwd0c3LAR2aN4alOdf+MTQKpYDkhwVqq4lfSc/D3Zwije0lpuUB7MnVxFu72KGxtbnOaurevTvMzc3x448/wtnZGdu3b8ekSZPg4+ODa9euqbUtLi5Gz5494efnh23btlXa54QJE5CQkIC9e/eiRYsW+PXXXzFixAgoFArI5fJqfy4f3K4DDElEZExKFUpcuJOrmik6dzsbeXL1xREtzcTo7Nn4r8UbHRHobs9tOIxYblGpaiuWx886PZ510sTZTvL3s05/zTx5N7Gu0QbAN2/exNSpU3Hy5EmIxWJ07NgRrVq1wvnz55GYmKhqV1painHjxiE1NRXHjx+v8vv0wYMHePPNN/Hf//4XIpEILVq0QL9+/fDtt9+isLCw2p/LkFQHGJKIyJAVlyqQkFa+79mfyVk4l5Jd4cvRVmKKzp6NEeTtiCAvB7R1s6/RFyIZD6Wy/AH8qxkyJKaXL1FwNSMPqVmFGtubi03QsqmN2mrirV3tqv1AfkFBAWQyGVxdXREaGor8/HwcOHAAQHlACgkJwa1bt3D06FE4OjpWq8/i4mI8evQIUqkU8+fPxy+//ILLly9X+3P5TBIRNVi18eoxAOTk5OCDDz7Anj17kJ2dDS8vL6xduxaDBw9Wtfnqq6+wZs0apKenw9/fH59//jleeOGF2h4yAKCwpKx837Nbj/BHchbi03JQUqa+xUcjKzN09Sx/nqibtyNau9b+Fh9kWExMRPB0soankzUGtXVVHc+Xl+FaxuPVxB/ftstDvrwMiekyJKbLANxVtXeykfw122SrWt+pRVPrCjOP1tbWsLa2RnZ2Ng4fPozVq1cD+DsgJSUl4dixY9UOSABgYWEBNzc3lJaWYvfu3QgJCanQprLP1RZnkmqIM0lEhik0NBSXLl3Cpk2bVK8Af/bZZ6pXj8eOHYs333xT7dXjsrKyKl89LikpQY8ePdC0aVMsXLgQzZo1Q1paGmxtbREYGAgA2LlzJyZOnIivvvoKPXr0wL///W/85z//QWJiYqVv6zyPvOJSnE3JxulbWfgz+REu3MmtsFChk41EtZp1kJcjfJraGNRr4mTYHm8A/PgNu8e37W5XsgGwqYkILZqUzzoJafFo1tgSvbu2R05GKubOnQuJRIKYmBiIRCKMGTMG58+fxy+//AJnZ2dVHw4ODjA3L382atKkSXBzc0N4eDgA4PTp07h79y7at2+Pu3fv4qOPPkJycjLOnz+veuD78OHDEAQBvr6+uHHjBubMmaP6XDOz8jf9eLutDjAkERme2nr1ePPmzVizZg2uXr2q+hft04KCgtCxY0ds2rRJdax169YYOXKk6l/yzyOnsARnbmfj9K3yB60v38vF04s3u9pb/PXmmSOCvB3g7WRtVFt8kHEoLCnD9fv5qtXEH68s/uQGwAVXfkPOye9QlvcQppZ28OrcB2Onz0H7llLYlWbj5e6BGvs+duwYevfuDQDo3bs3PD09ERERAQA4ceIE3n77bdy6dQs2NjYYPHhwhVngn376CQsWLMCdO3fg4OCAMWPGYMWKFbC3t1e1YUiqAwxJRIYnLy8PdnZ2OHLkCPr27as6HhwcDIlEguPHj1c458iRIxgwYABycnIq/f/y4MGD4eDgACsrK+zbtw9NmjTBhAkTMG/ePIjFYpSUlMDKygq7du3CqFGjVOfNmjUL8fHxOHHihNZjeZgv/2t9ovJQdO1+XoX/em/uYPXXGkXlM0XuDpYMRaQXgiAgPbe4wmritx7kVwjzQPkGwN5NbJ64XVf+v65PbACcnluE5IcF8HKyhqu9pc5q5TNJRNQg2draIjg4GMuXL0fr1q1VrwCfPn0aPj4+FdoXFxdj/vz5mDBhQpX/snz8YOmrr76KgwcPIikpCTNnzkRZWRmWLFmChw8fQqFQqN02AABnZ2dkZGRUq/aM3GLV6/inbz3CzQcVt/jwbmKNIC9H1TpF0ka6++Igeh4ikQjSRpaQNrJE39Z///+guFSBpPv5uPLXc07lM08y5BSW4kZmPm5k5uOXC+mq9nYWpvBztYOpCIi9lQUB5YEqfHQAQrvo/rb1szAkEVG9snXrVkydOhVubm6qV4AnTJiA8+fPq7UrLS3FK6+8AqVSia+++qrKPpVKJZo2bYotW7ZALBajU6dOuHfvHtasWYMlS5ao2j09iyMIQqUzO2l/bfHx51/BKOVRxbeL/Fxs/17N2qsxmto23H3PyDhZmIkR0MweAc3+vt0lCAIy8+QVVhO/+SAfsuIy/JmcpdaHUgAW7rmEXq2a6HRGqToYkoioXmnRogVOnDhR4RVgLy8vVZvHb9YkJyfj6NGjz5xyd3V1hZmZGcTiv9/cad26NTIyMlBSUgInJyeIxeIKs0aZmZlwdnaGIJQv9le+RlH5K/lPL/RnIgLaSO0Q5OWIrl4O6OrpoNPF/YgMhUgkgrOdBZztLNDbt6nquLxMgZuZBfjvhXvYdPym2jkKQcDth4UMSUREuqDLV4979OiBH3/8EUqlEiYm5esIXb9+Ha6urqo3cTp16oTo6GiMGjUKSqWAGw/y8fP+g3AJ6ImuK3+tsNO72ESEADd7BHk7oJuXIzp5NoadEezuTlRbJKZitJHaobG1Gf594qbas0xikQieTlZ1XhNDEhHVK5peAfb19cWUKVNQVlaGsWPHql49VigUqtmfql49fvvtt7FhwwbMmjUL//rXv5CUlISVK1finXfeAVC+LcS41/+B+e9Mx+USJzyw8sCd2P8i/+4diAb1gmmeHOZiE7R3b4Qg7/LniTo2bwxr7ntGVIGrvSXCRwdg4Z5LUAgCxCIRVo5uW+ezSABDEhHVM7m5uRpfATYzM8Pt27exf/9+AOXLAjzpyVePU1NTVTNGAODu7o6oqCi8++67aNeuHdzc3BDy+ltwCB6HNyLO4M/bWcgrbgr7PtPw+65/Q1GQBUkTT7w0ax2GDeyJIG8HtHdvBAszbvFBVB2hXZqjV6smuP2wEJ5OVnoJSACXAKgxLgFAVH89/eqxvEyh2vfsj1uPcC4lG4Ul6lt8WJuL0dnz8cKNDghwawRzU27xQWRo6tUSALWxxUBERASmTJlS4XhRUREsLPj2CFFDtvNMKhbsuQilAIgAeDlZ425OEeRPbfFhZ2GqevMsyNsBbVztYMp9z4jqFYMPSdOmTcOlS5ewdetW1RYD/fr1U20xcP78eSxevFhti4Hhw4dXucUAANjZ2eHatWtqxxiQiBq2s7ezMH/3RTyeXhcA3HpYvl6Ro7W5auHGrl6O8HOx5RYfRPWcQd9uq60tBiIiIhAWFoacnJwa18bbbUT1g0Ip4Pi1TGz9IwXHrz3Q2ObTce0wpmMzrmZNVA/Um9ttZWVlUCgUFWZ4LC0tERMTo/Gc3NxciEQi1WZ3lcnPz4eHhwcUCgXat2+P5cuXo0OHDpW2l8vlkMv/foVXJpNVfyBEZHAe5svx09k0/PBHaoU1i54kFonQo6UTAxJRA6T1DXRNex/Vlie3GLh37x4UCgW2bduG06dPIz09vUL76m4x4Ofnh4iICOzfvx/bt2+HhYUFevTogaSkpErPCQ8Ph729verH3d1dJ2MkorojCALOpWQhbEccuocfxer/XcPdnCLYW5rhzRe8cPz93lg1JgDivwKRPl89JiL90/p2m4WFBdzc3DBlyhRMnjy51sPCzZs3MXXqVJw8eVK1xUCrVq1w/vx5JCYmqtqVlpZi3LhxSE1NxfHjx7W6BaZUKtGxY0f06tUL69ev19hG00ySu7s7b7cRGYECeRn2xt/F1tgUXM3IUx0PdG+E14KaY1igVO31/PTcIr2/ekxEtaNWb7fdu3cP27ZtQ0REBD766CP07dsXb7zxBkaOHKlaiE2XamOLgaeZmJigS5cuVc4kSSQSSCSSGo+DiOpe0v08bPsjBbvP30W+vAwAIDE1wYj2UrzWzQPtmjXSeJ6rvSXDERE934Pb8fHx+Oabb7B9+3YolUq8+uqreOONNxAYGKjLGtVkZ2fDy8sLq1evxltvvVVhi4EmTZpo3acgCOjatSsCAgLwzTffVOscPrhNZJhKFUpEXb6PrX/cxh+3/t4o08vJGq8GNcfYTs3QyIp7ohE1VNp8fz/322337t3Dli1b8Mknn8DU1BTFxcUIDg7G5s2b4e/v/zxdA9C8xYBEIkFMTAxEIhHGjBmj2mLA2dlZdV5VWwwsXboU3bp1g4+PD2QyGdavX4+tW7fi999/R9euXatVF0MSkWFJzy3C9j/TsOPPVGT+tU+aiQjo19oZE4M90KOFE1/ZJ6Laf7uttLQU+/btwzfffIPo6Gh07twZGzduxPjx45GVlYV58+Zh3Lhxas8M1VRtbDGQk5ODt956CxkZGbC3t0eHDh1w8uTJagckIjIMgiDg1M1H2Bqbgugr96H4a0dMJxsJxnd1x/iuzSFtxNtmRFQzWs8k/etf/8L27dsBAK+99hqmTZuGtm3bqrVJTU2Fp6cnlEqlpi7qBc4kEelPblEpdp+7g22nU3DrQYHqeJCXAyYGe2BAGxduCUJEGtXqTFJiYiI2bNiAMWPGVPqgtlQqxbFjx7TtmoioSpfu5mJrbAr2JdxFcWn5f4TZSEwxuqMbXuvmgVbOtnqukIjqE4NecduQcSaJqG4Ulypw4EI6tv6Rgvi0HNVxPxdbvNbNAyM7uMFGYtDr4hKRAanVmaTw8HA4Oztj6tSpase/+eYbPHjwAPPmzdO2SyKiClIfFeKH0yn46WwasgtLAQBmYhFebuuKicEe6OzRmKtgE1Gt0jok/fvf/8aPP/5Y4bi/vz9eeeUVhiQiqrEn91E7cf0BHs9zuzWyxISg5gjp7I4mtlyvjIjqhtYhKSMjA66urhWON2nSRONWIUREz1LZPmovtmqCid080MevKcR8fZ+I6pjWIcnd3R2///672orXAPD7779DKpXqrDAiqt/K91HLxtY/UnDwYjpKFeXTRo2szBDS2R0TujaHp5O1nqskooZM65A0bdo0hIWFobS0FC+99BIA4Ndff8XcuXPx3nvv6bxAIqpfqtpHbWI3Dwxt56q2jxoRkb5oHZLmzp2LrKwszJgxAyUlJQDKN72dN28eFixYoPMCiah+qOk+akRE+lLjJQDy8/Nx5coVWFpawsfHp8Ft/solAIie7Vn7qI3r5A57KzM9VkhEDU2tb0sCADY2NujSpUtNTyeieoz7qBFRfVCjkHTmzBns2rULqampqltuj+3Zs0cnhRGRcREEAb/feIRtf1TcR21CV3e8wn3UiMjIaB2SduzYgUmTJmHAgAGIjo7GgAEDkJSUhIyMDIwaNao2aiQiA5ZbVIqfz93BD3+k4NZD7qNGRPWH1iFp5cqV+OyzzzBz5kzY2triiy++gJeXF6ZPn65x/SQiqp+4jxoR1Xdah6SbN29iyJAhAACJRIKCggKIRCK8++67eOmll7B06VKdF0lEhoH7qBFRQ6L1v80cHByQl1e+tombmxsuXbqEgIAA5OTkoLCwUOcFEpH+VbaP2uAAV7zWjfuoEVH9pHVIeuGFFxAdHY2AgACEhIRg1qxZOHr0KKKjo9G3b9/aqJGI9ID7qBFRQ6d1SNq4cSOKi4sBAAsWLICZmRliYmIwevRoLF68WOcFElHd4j5qRETltFpMsqysDD/88AMGDhwIFxeX2qzL4HExSapPuI8aETUUtbaYpKmpKd5++21cuXLluQokIsPAfdSIiCqn9e22oKAgxMXFwcPDozbqIaI6UNU+ahO7eSKgmb2eKyQi0j+tQ9KMGTPw3nvv4c6dO+jUqROsrdWn4Nu1a6ez4ohId6raR+21bh4Y27EZ91EjInqC1hvcmphUXDlXJBJBEASIRCIoFAqdFWfI+EwSGYvK9lHr38YZE7t5onsLR+6jRkQNRq1ucJucnFzjwoiobnAfNSKi56d1SOKzSESGi/uoERHpjtYh6fvvv6/yzydNmlTjYoioZriPGhGR7mn9TFLjxo3Vfi8tLUVhYSHMzc1hZWWFrKysSs6sX/hMUv2Ul5eHxYsXIzIyEpmZmejQoQO++OILdOnSBUD5baylS5diy5YtyM7ORlBQEL788kv4+/tX2e/u3buxePFi3Lx5Ey1atMCKFSswatQo1Z97enoiJSWlwnkzZszAl19+qbHPqvZRmxjsgRHtuY8aEdHTavWZpOzs7ArHkpKS8Pbbb2POnDnadkdkUKZNm4ZLly5h69atkEql2LZtG/r164fExES4ublh9erVWLduHSIiItCqVSt8/PHH6N+/P65duwZbW82zNbGxsQgNDcXy5csxatQoREZGIiQkBDExMQgKCgIAnDlzRu2lh0uXLqF///4YN25chf6q2kdtYjcPdOI+akREOqH1TFJlzp49i9deew1Xr17VRXcGjzNJ9U9RURFsbW2xb98+DBkyRHW8ffv2GDp0KJYvXw6pVIqwsDDMmzcPACCXy+Hs7IxVq1Zh+vTpGvsNDQ2FTCbDoUOHVMcGDRqExo0bY/v27RrPCQsLwy+//IKkpKTyt0a5jxoRkU7U6kxSZcRiMe7du6er7ojqXFlZGRQKBSwsLNSOW1paIiYmBsnJycjIyMCAAQNUfyaRSPDiiy/i1KlTlYak2NhYvPvuu2rHBg4ciM8//1xj+5KSEmzbtg2zZ8/Go4IS7qNGRKQnWoek/fv3q/0uCALS09OxceNG9OjRQ2eFEdU1W1tbBAcHY/ny5WjdujWcnZ2xfft2nD59Gj4+PsjIyAAAODs7q53n7Oys8XmixzIyMjSe87i/p0VGRiInJwc3G3VGcPivFfZRezWoOTwcuY8aEVFt0zokjRw5Uu13kUiEJk2a4KWXXsLatWt1VReRXmzduhVTp06Fm5sbxGIxOnbsiAkTJuD8+fOqNk8/7/N4IdWqVOecx/uozfpwLcw9O+LX1PLnjdr/tY/aEO6jRkRUp7QOSUqlsjbqIDIILVq0wIkTJ1BQUACZTAZXV1eEhobCy8sLLi4uAMpnhlxdXVXnZGZmVpgpepKLi0uFWaMnz3lyH7WczHt4dP0cpGM/QGhnd7zWzYP7qBER6QlXlSPSwNraGq6ursjOzsbhw4cxYsQIVVCKjo5WtSspKcGJEyfQvXv3SvsKDg5WOwcADh8+DM82HfDKllj0/+wkvotNQb68DKY3T8C+sSMu/GcBVo1tx4BERKRHWoeksWPH4pNPPqlwfM2aNRpfVyYyJocPH8b//vc/JCcnIzo6Gn369IGvry+mTJkCkUiEsLAwrFy5EpGRkbh06RJef/11WFlZYcKECao+Jk2ahAULFqh+nzVrFqKiorBq1SqcPBOPodPeR1T0EVxv0gt/3MqCiQgY6O+M76d0gXDtGP7x5lQ42nHLECIifdP6dtuJEyfw4YcfVjg+aNAgfPrppzopikhfcnNzsWDBAty5cwcODg4YM2YMVqxYATMzMwDA3LlzUVRUhBkzZqgWk4yKilJbIyk1NVVtI+jg4GB8+NkWfBK+DDkLF8G0kQuchs9DM992GN/l733UoqKikJqaiqlTp9b5uImIqCKt10mytLREfHw8fH191Y5fvXoVHTp0QFFRUSVn1oy+VkB+Fq6TRJVJzy1C8sMCOFlL8NuNhxX2Uevm7YDXunEfNSIifajVdZLatm2LnTt3YsmSJWrHd+zYgTZt2mjb3TPpawVkoprYeSYVC/ZchPKp//SwkZhiTEc3vMp91IiIjIbWM0n79+/HmDFjMGHCBLz00ksAgF9//RXbt2/Hrl27KiwR8DwMaQXkp3EmiZ6WnluEHp8crRCQ5g7yxeRgT1hzHzUiIr3T5vtb67n+4cOHY+/evbhx4wZmzJiB9957D3fu3MGRI0d0GpCA518BuTKxsbFq5wDlKyBXdY5cLodMJlP7IXpS8sOCCgEJADq4N2ZAIiIyQjX6N/eQIUPUZnZqi6GsgAwA4eHhWLp06XOMhuq7krKKa4iJRSJ4OlnpoRoiInpeWs8knTlzBqdPn65w/PTp0zh79qxOinrS1q1bIQgC3NzcIJFIsH79ekyYMAFi8d8rD9fWCshPWrBgAXJzc1U/aWlpNRgN1VfyMgXCD6pv7iwWibBydFu42vN1fiIiY6R1SJo5c6bGgHD37l3MnDlTJ0U96fEKyPn5+UhLS8Off/6J0tLSCisgP+l5V0DWRCKRwM7OTu2H6LF10ddx7X4enGzMcfCdntj+ZjfEzO+D0C7N9V0aERHVkNYhKTExER07dqxwvEOHDkhMTNRJUZrU9grIUVFRVZ5DVJlzKVnYcvIWACB8dDu0kdojuIUjZ5CIiIyc1s8kSSQS3L9/H97e3mrH09PTYWqq+4dTDx8+DEEQ4Ovrixs3bmDOnDkaV0D28fGBj48PVq5cqXEFZDc3N4SHhwMoXwG5V69eWLVqFUaMGIF9+/bhyJEjiImJ0Xn9VL8VlpRh9k8JEARgTMdm6N+m8tlIIiIyLlqnmv79+2PBggXYt28f7O3L95XKycnBwoUL0b9/f50XWBsrIHfv3h07duzAokWLsHjxYrRo0QI7d+7kGkmktfCDV5HyqBBSewt8OFz364QREZH+aL1O0t27d9GrVy88evQIHTp0AADEx8fD2dkZ0dHRcHd3r5VCDQ3XSaLfkh5g4td/AgB+mBaEHi2d9FwRERE9S62uuO3m5oYLFy7ghx9+QEJCAiwtLTFlyhSMHz9eNbtDVN/lFpVizq4LAIDJwR4MSERE9VCNHiKytrbGW2+9petaiIzG0v9eRoasGF5O1pj/cmt9l0NERLWgxk9aJyYmIjU1FSUlJWrHhw8f/txFERmyw5czsOf8XZiIgE/HBcLSXPzsk4iIyOhoHZJu3bqFUaNG4eLFixCJRHj8SNPjhRgVCoVuKyQyIA/z5Vi45yIAYPqLLdDJo7GeKyIiotqi9TpJs2bNgpeXF+7fvw8rKytcvnwZJ0+eROfOnXH8+PFaKJHIMAiCgA8iL+JRQQn8XGwR1s9H3yUREVEt0nomKTY2FkePHkWTJk1gYmICExMT9OzZE+Hh4XjnnXcQFxdXG3US6d3e+Ls4fPk+zMQirAtpD4kpb7MREdVnWs8kKRQK2NjYAACcnJxw7949AICHhweuXbum2+qIDER6bhGW7LsMAJjV1wdtpFz2gYiovtN6Jqlt27a4cOECvL29ERQUhNWrV8Pc3BxbtmypsAo3UX0gCALm/nwBecVlaO/eCP94sYW+SyIiojqgdUhatGgRCgoKAAAff/wxhg4dihdeeAGOjo7YuXOnzgsk0rdtp1PxW9JDWJiZYG1IIEzFWk/AEhGREdI6JA0cOFD1197e3khMTERWVhYaN26sesONqL64/bAAKw9cAQDMG+SHFk1s9FwRERHVFZ3sSOvg4KCLbogMikIp4P1dCSgqVSDY2xGTgz31XRIREdUh3jcgqsR/fruFsynZsJGYYs24djAx4UwpEVFDwpBEpMG1jDysjboOAFgytA2aNbbSc0VERFTXGJKInlJSpsTsn+JRolCir19TjOvcTN8lERGRHmgdkk6ePImysrIKx8vKynDy5EmdFEWkTxuPJuHyPRkaWZkhfEwAX0ggImqgtA5Jffr0QVZWVoXjubm56NOnj06KItKXhLQcfHn8JgDg45Ft0dTWQs8VERGRvmgdkgRB0Phf1o8ePYK1tbVOiiLSh+JSBWb/FA+FUsCwQCmGtpPquyQiItKjai8BMHr0aACASCTC66+/DolEovozhUKBCxcuoHv37rqvkKiOrDl8DTcfFKCprQTLR/jruxwiItKzaocke3t7AOUzSba2trC0tFT9mbm5Obp164Y333xT9xUS1YE/bj3CN78nAwBWjWmHRlbmeq6IiIj0rdoh6dtvvwUAeHp64v333+etNao38uVleH9XAgQBGN/VHX38muq7JCIiMgBaP5M0d+5ctWeSUlJS8PnnnyMqKkqnhRHVlRUHEnEnuwjNGlvigyFt9F0OEREZCK1D0ogRI/D9998DAHJyctC1a1esXbsWI0aMwKZNm3ReIFFtOnY1E9v/TINIBHw6LhA2Ep3s1ENERPWA1iHp/PnzeOGFFwAAP//8M1xcXJCSkoLvv/8e69ev13mBRLUlp7AE83ZfAABM7eGFbt6Oeq6IiIgMidYhqbCwELa2tgCAqKgojB49GiYmJujWrRtSUlJ0XiBRbVm87zIy8+Ro2dQGcwb66rscIiIyMFqHpJYtW2Lv3r1IS0vD4cOHMWDAAABAZmYm7OzsdF4gUW345cI9/DfhHsQmIqwdFwgLM7G+SyIiIgOjdUhasmQJ3n//fXh6eqJr164IDg4GUD6r1KFDB50XSKRrmXnFWLT3EgBgZu8WCHRvpN+CiIjIIGn9lOrYsWPRs2dPpKenIzAwUHW8b9++GDVqlE6LI9I1QRCwYPdF5BSWwl9qh3++5KPvkoiIyEBpPZMEAC4uLrC1tUV0dDSKiooAAF26dIGfn59OiyPStV1n7+DXq5kwF5tgXUh7mJvW6P8CRETUAGj9DfHo0SP07dsXrVq1wuDBg5Geng4AmDZtGt577z2dF0ikK2lZhVj2SyIA4L0BreDrYqvnioiIyJBpHZLeffddmJmZITU1FVZWVqrjoaGh+N///qfT4oh0RakUMPfnC8iXl6GzR2NMe8Fb3yUREZGB0/qZpKioKBw+fBjNmjVTO+7j48MlAMhgfRd7G7G3HsHSTIxPxwVCbCJ69klERNSgaT2TVFBQoDaD9NjDhw8hkUh0UhSRLt18kI9PDl0FACwc0hqeTtx3kIiInk3rkNSrVy/VtiQAIBKJoFQqsWbNGvTp00enxRE9rzKFErN/SoC8TIkXfJzwWlBzfZdERERGQuvbbWvWrEHv3r1x9uxZlJSUYO7cubh8+TKysrLw+++/10aNRDW2+cRNJKTlwNbCFKvHtlPbnJmIiKgqWs8ktWnTBhcuXEDXrl3Rv39/FBQUYPTo0YiLi0OLFi1qo0aiGrl8Lxdf/JoEAFg63B+u9pZ6roiIiIyJ1iEpNTUVzs7OWLp0KX755RccPHgQH3/8MVxdXZGamqrT4srKyrBo0SJ4eXnB0tIS3t7eWLZsGZRKpaqNSCTS+LNmzZpK+42IiNB4TnFxsU7rJ/2Rlynw3k8JKFUIGOjvjFEd3PRdEhERGRmtb7d5eXkhPT0dTZs2VTv+6NEjeHl5QaFQ6Ky4VatWYfPmzfjuu+/g7++Ps2fPYsqUKbC3t8esWbMAQLVO02OHDh3CG2+8gTFjxlTZt52dHa5du6Z2zMLCQme1k359fiQJVzPy4GhtjpWjAnibjYiItKZ1SBIEQeMXTn5+vs5DRmxsLEaMGIEhQ4YAADw9PbF9+3acPXtW1cbFxUXtnH379qFPnz7w9q56HRyRSFThXKofzqVk4d8nbgIAVo4OgKMN37okIiLtVTskzZ49G0B5uFi8eLHaMgAKhQKnT59G+/btdVpcz549sXnzZly/fh2tWrVCQkICYmJi8Pnnn2tsf//+fRw4cADffffdM/vOz8+Hh4cHFAoF2rdvj+XLl1e5Qa9cLodcLlf9LpPJtB4P1b7CkjK891MClAIwuoMbBvozCBMRUc1UOyTFxcUBKJ9JunjxIszNzVV/Zm5ujsDAQLz//vs6LW7evHnIzc2Fn58fxGIxFAoFVqxYgfHjx2ts/91338HW1hajR4+usl8/Pz9EREQgICAAMpkMX3zxBXr06IGEhAT4+Gje8DQ8PBxLly597jFR7Vp16CpuPyqEq70FPhzur+9yiIjIiIkEQRC0OWHKlCn44osvYGdnV1s1qezYsQNz5szBmjVr4O/vj/j4eISFhWHdunWYPHlyhfZ+fn7o378/NmzYoNXnKJVKdOzYEb169cL69es1ttE0k+Tu7o7c3Nw6+XtBz/b7jYd49T+nAQBb3+iKF3ya6LkiIiIyNDKZDPb29tX6/tb6maRvv/22xoVpa86cOZg/fz5eeeUVAEBAQABSUlIQHh5eIST99ttvuHbtGnbu3Kn155iYmKBLly5ISkqqtI1EIuGK4gZMVlyKObsSAAATu3kwIBER0XPTegmAulRYWAgTE/USxWKx2hIAj3399dfo1KkTAgMDtf4cQRAQHx8PV1fXGtdK+rV0fyLu5RbDw9EKCwb76bscIiKqB7SeSapLw4YNw4oVK9C8eXP4+/sjLi4O69atw9SpU9XayWQy7Nq1C2vXrtXYz6RJk+Dm5obw8HAAwNKlS9GtWzf4+PhAJpNh/fr1iI+Px5dfflnrYyLdi7qcgd3n70AkAtaOC4SVuUH/Y01EREbCoL9NNmzYgMWLF2PGjBnIzMyEVCrF9OnTsWTJErV2O3bsgCAIlT7QnZqaqjYjlZOTg7feegsZGRmwt7dHhw4dcPLkSXTt2rVWx0O69yhfjoWRFwEAb/XyRmdPBz1XRERE9YXWD25TOW0e/KLaIQgCZvxwHocuZcDX2Rb7/9UDElOxvssiIiIDps33t0E/k0RUlX3x93DoUgZMTURYGxLIgERERDrFkERGKSO3GEv2XQIAvNPXB23d7PVcERER1TcMSWR0BEHA3N0XICsuQ2Aze8zo3ULfJRERUT3EkERG58c/U3Hy+gNITE2wNqQ9TMX8x5iIiHSP3y5kVFIeFWDFgSsAgLmD/NCyqY2eKyIiovqKIYmMhkIp4P1dCSgsUSDIywFTunvquyQiIqrHGJLIaHwdcwtnbmfD2lyMT8cFwsREpO+SiIioHmNIIqNw/X4ePj18HQCweGgbuDtY6bkiIiKq7xiSyOCVKpSY/VM8ShRKvOTXFKFd3PVdEhERNQAMSWTwNh69gUt3ZbC3NMMnowMgEvE2GxER1T6GJDJoF+7kYOOxGwCA5SPboqmdhZ4rIiKihoIhiQxWcakCs39KgEIpYEg7VwwPlOq7JCIiakAYkshgrY26hhuZ+WhiK8HHI9rquxwiImpgGJLIIJ2+9Qj/iUkGAKwaE4DG1uZ6roiIiBoahiQyOPnyMrz/cwIEAQjt7I6X/Jz1XRIRETVADElkcFYcuIK0rCK4NbLEoqGt9V0OERE1UAxJZFCOXcvE9j9TAQBrxrWDrYWZnisiIqKGiiGJDEZOYQnm/XwBADClhye6t3DSc0VERNSQMSSRwfhw/2Vk5snh3cQa8wb56bscIiJq4BiSyCAcvJiOffH3YCIC1oW0h4WZWN8lERFRA8eQRHqXmVeMDyIvAgBm9G6J9u6N9FsQERERGJJIzwRBwMI9l5BdWIo2rnZ4p6+PvksiIiICwJBEevbzuTs4cuU+zMUmWBcaCHNT/iNJRESGgd9IpDd3c4qw7L+JAIB3+7eCn4udnisiIiL6G0MS6YVSKWDOrgTkycvQsXkjvNXLW98lERERqWFIIr34PvY2Tt18BEszMdaGtIfYRKTvkoiIiNQwJFGdu/UgH5/87yoAYMFgP3g5Weu5IiIioooYkqhOlSmUeG9XAopLlejZ0gmvBXnouyQiIiKNGJKoTv375C3EpebAVmKK1WPbwYS32YiIyEAxJFGdSbwnw+dHrgMAPhzuD2kjSz1XREREVDmGJKoT8jIFZv8Uj1KFgP5tnDGmo5u+SyIiIqoSQxLVifW/JuFqRh4crM0RPjoAIhFvsxERkWFjSKJadz41G5uO3wQArBzVFk42Ej1XRERE9GwMSVSrikoUeP+nBCgFYGR7KQa1ddV3SURERNXCkES1atX/ruLWwwK42Flg6fC2+i6HiIio2gw6JJWVlWHRokXw8vKCpaUlvL29sWzZMiiVSlWb119/HSKRSO2nW7duz+x79+7daNOmDSQSCdq0aYPIyMjaHEqD9PuNh4g4dRsAsGpsO9hbmem3ICIiIi2Y6ruAqqxatQqbN2/Gd999B39/f5w9exZTpkyBvb09Zs2apWo3aNAgfPvtt6rfzc3Nq+w3NjYWoaGhWL58OUaNGoXIyEiEhIQgJiYGQUFBtTaehkRWXIq5P18AALwa1Bwvtmqi54qIiIi0IxIEQdB3EZUZOnQonJ2d8fXXX6uOjRkzBlZWVti6dSuA8pmknJwc7N27t9r9hoaGQiaT4dChQ6pjgwYNQuPGjbF9+/Zq9SGTyWBvb4/c3FzY2XH3+qfN2ZWAXefuoLmDFQ7NegHWEoPO40RE1EBo8/1t0LfbevbsiV9//RXXr5cvQJiQkICYmBgMHjxYrd3x48fRtGlTtGrVCm+++SYyMzOr7Dc2NhYDBgxQOzZw4ECcOnWq0nPkcjlkMpnaD2l2JPE+dp27A5EI+HRcIAMSEREZJYP+9po3bx5yc3Ph5+cHsVgMhUKBFStWYPz48ao2L7/8MsaNGwcPDw8kJydj8eLFeOmll3Du3DlIJJpfNc/IyICzs7PaMWdnZ2RkZFRaS3h4OJYuXaqbgdVjWQUlmL/nIgDgzRe80dXLQc8VERER1YxBh6SdO3di27Zt+PHHH+Hv74/4+HiEhYVBKpVi8uTJAMpvnT3Wtm1bdO7cGR4eHjhw4ABGjx5dad9PL2YoCEKVCxwuWLAAs2fPVv0uk8ng7u5e06HVS4IgYNHei3iYL0crZxvM7t9K3yURERHVmEGHpDlz5mD+/Pl45ZVXAAABAQFISUlBeHi4KiQ9zdXVFR4eHkhKSqq0XxcXlwqzRpmZmRVml54kkUgqnZmicvsT7uHgxQyYmoiwdlx7WJiJ9V0SERFRjRn0M0mFhYUwMVEvUSwWqy0B8LRHjx4hLS0Nrq6VL1oYHByM6OhotWNRUVHo3r378xXcgN2XFWPJvssAgH++1BIBzez1XBEREdHzMeiZpGHDhmHFihVo3rw5/P39ERcXh3Xr1mHq1KkAgPz8fHz00UcYM2YMXF1dcfv2bSxcuBBOTk4YNWqUqp9JkybBzc0N4eHhAIBZs2ahV69eWLVqFUaMGIF9+/bhyJEjiImJ0cs4jZ0gCJj78wXkFpUiwM0eM/u01HdJREREz82gQ9KGDRuwePFizJgxA5mZmZBKpZg+fTqWLFkCoHxW6eLFi/j++++Rk5MDV1dX9OnTBzt37oStra2qn9TUVLUZqe7du2PHjh1YtGgRFi9ejBYtWmDnzp1cI6mGtv+ZhhPXH8Dc1ATrQgJhJjboCUoiIqJqMeh1kgwZ10kql/qoEIO+OInCEgUWDWmNaS9467skIiKiStWbdZLIsCmVAt7/OQGFJQp09XLA1B5e+i6JiIhIZxiSqMa++T0ZfyZnwcpcjE/HBsLEpPIlFIiIiIwNQxLVSNL9PKw+fA0AsGhIGzR3tNJzRURERLrFkERaK1UoMfunBJSUKdHbtwnGd+WimkREVP8wJJHWvjp2Exfv5sLe0gyrxrSrcqVyIiIiY8WQRFq5eCcXG46Wr2a+bIQ/nO0s9FwRERFR7WBIomorLlVg9k/xKFMKGBzgguGBUn2XREREVGsYkqja1kVfR1JmPpxsJPh4ZABvsxERUb3GkETV8mdyFv7vt1sAgE9GB8DB2lzPFREREdUuhiR6pgJ5Gd7flQBBAMZ1aoZ+bZz1XRIREVGtY0iiZ1p58ApSswrh1sgSi4e10Xc5REREdYIhiap04voD/HA6FQCwZmw72FmY6bkiIiKiusGQRJXKLSzF3J8TAACvd/dE95ZOeq6IiIio7jAkUaU++u9l3JfJ4eVkjXmD/PRdDhERUZ1iSCKN/ncpHZFxd2EiAtaGBMLSXKzvkoiIiOoUQxJV8CBPjoWRlwAA/3ixBTo2b6znioiIiOoeQxKpEQQBCyMvIqugBH4utpjVz0ffJREREekFQxKp2X3+LqIT78NMLMJnoe0hMeVtNiIiapgYkkjlXk4Rlu6/DAAI69cKrV3t9FwRERGR/jAkEQBAqRQw9+cLyJOXoUPzRpjey1vfJREREekVQxIBALadTkHMjYewMDPB2nGBMBXzHw0iImrY+E1ISH5YgJUHrwAA5g/yg3cTGz1XREREpH8MSQ2cQingvZ/iUVyqRPcWjpgU7KnvkoiIiAwCQ1IDt+XkLZxPzYGNxBRrxgXCxESk75KIiIgMAkNSA3Y1Q4bPoq8DAJYMawO3RpZ6roiIiMhwMCQ1UCVlSry7MwElCiX6tW6KcZ2a6bskIiIig8KQ1ECt/zUJV9JlaGxlhpWjAyAS8TYbERHRkxiSGqC41Gx8dfwGAGDFqAA0tbXQc0VERESGhyGpgSkqUeC9nxKgFIDhgVIMDnDVd0lEREQGiSGpgVl9+CpuPSxAU1sJlo3w13c5REREBoshqQE5dfMhvv39NgBg1dh2aGRlrt+CiIiIDBhDUgORV1yKObsuAADGd22OPr5N9VwRERGRYWNIaiCW/5KIuzlFcHewxAdDWuu7HCIiIoPHkNQA/HrlPn46ewciEfDp2EDYSEz1XRIREZHBY0iq57ILSjB/z0UAwBs9vBDk7ajnioiIiIyDQYeksrIyLFq0CF5eXrC0tIS3tzeWLVsGpVIJACgtLcW8efMQEBAAa2trSKVSTJo0Cffu3auy34iICIhEogo/xcXFdTGsOrVo3yU8yJOjZVMbvD/QV9/lEBERGQ2Dvu+yatUqbN68Gd999x38/f1x9uxZTJkyBfb29pg1axYKCwtx/vx5LF68GIGBgcjOzkZYWBiGDx+Os2fPVtm3nZ0drl27pnbMwqJ+Laq4P+EeDlxIh9hEhHUhgbAwE+u7JCIiIqNh0CEpNjYWI0aMwJAhQwAAnp6e2L59uyoA2dvbIzo6Wu2cDRs2oGvXrkhNTUXz5s0r7VskEsHFxaX2itezTFkxFu+9BACY2acl2jVrpN+CiIiIjIxB327r2bMnfv31V1y/Xr5TfUJCAmJiYjB48OBKz8nNzYVIJEKjRo2q7Ds/Px8eHh5o1qwZhg4diri4OF2WrleCIGDe7gvILSpFWzc7/OullvouiYiIyOgY9EzSvHnzkJubCz8/P4jFYigUCqxYsQLjx4/X2L64uBjz58/HhAkTYGdnV2m/fn5+iIiIQEBAAGQyGb744gv06NEDCQkJ8PHx0XiOXC6HXC5X/S6TyZ5vcLVo55k0HLv2AOamJlgX0h5mYoPOwkRERAbJoEPSzp07sW3bNvz444/w9/dHfHw8wsLCIJVKMXnyZLW2paWleOWVV6BUKvHVV19V2W+3bt3QrVs31e89evRAx44dsWHDBqxfv17jOeHh4Vi6dOnzD6qWpWUVYvkviQCA9we0QitnWz1XREREZJxEgiAI+i6iMu7u7pg/fz5mzpypOvbxxx9j27ZtuHr1qupYaWkpQkJCcOvWLRw9ehSOjtq/5v7mm2/izp07OHTokMY/1zST5O7ujtzc3CpnreqSUilg/P/9gdPJWeji2Rg73gqG2ESk77KIiIgMhkwmg729fbW+vw16JqmwsBAmJuq3isRisWoJAODvgJSUlIRjx47VKCAJgoD4+HgEBARU2kYikUAikWjdd1369tRtnE7OgpW5GJ+OC2RAIiIieg4GHZKGDRuGFStWoHnz5vD390dcXBzWrVuHqVOnAihfR2ns2LE4f/48fvnlFygUCmRkZAAAHBwcYG5evoHrpEmT4ObmhvDwcADA0qVL0a1bN/j4+EAmk2H9+vWIj4/Hl19+qZ+B6sCNzHys/l/57NrCwa3h4Wit54qIiIiMm0GHpA0bNmDx4sWYMWMGMjMzIZVKMX36dCxZsgQAcOfOHezfvx8A0L59e7Vzjx07ht69ewMAUlNT1WakcnJy8NZbbyEjIwP29vbo0KEDTp48ia5du9bJuHStTKHEez/FQ16mRK9WTfBqUOVLHxAREVH1GPQzSYZMm3uatW39r0lYF30ddhamiHr3RbjY169FMYmIiHRFm+9vvhtu5C7dzcX6X5MAAEtH+DMgERER6QhDkhGTlykw+6d4lCkFDPJ3wcj2bvouiYiIqN5gSDJi66Kv4/r9fDjZmGPFqLYQifg2GxERka4wJBmps7ezsOXkLQDAylEBcLQx7OUJiIiIjA1DkhEqkJfhvV0JEARgTMdmGOBffzfqJSIi0heGJCMUfugKUh4VwtXeAkuGtdF3OURERPUSQ5IOlZWVYdGiRfDy8oKlpSW8vb2xbNkytRXC9+zZg4EDB8LJyQkikQjx8fHP7Pfy5csYM2YMPD09IRKJ8NWGDQCANWMDYW9pVqF9eHg4RCIRwsLCdDU0IiKiBochSYdWrVqFzZs3Y+PGjbhy5QpWr16NNWvWYMNfoQYACgoK0KNHD3zyySfV7rewsBDe3t5YsuxjmNk4AAAmBXugp49ThbZnzpzBli1b0K5du+cfEBERUQNm0CtuG5vY2FiMGDECQ4YMAQB4enpi+/btOHv2rKrNxIkTAQC3b9+udr9dunRBly5dMHtnPJQmpnCwNsP8l/0qtMvPz8err76K//u//8PHH3/8fIMhIiJq4DiTpEM9e/bEr7/+iuvXrwMAEhISEBMTg8GDBz933/+7lIE9cXchAjCivRuszCvm25kzZ2LIkCHo16/fc38eERFRQ8eZJB2aN28ecnNz4efnB7FYDIVCgRUrVmD8+PHP1e/DfDk+iLwIALCxMIW7g1WFNjt27MD58+dx5syZ5/osIiIiKseQpEM7d+7Etm3b8OOPP8Lf3x/x8fEICwuDVCrF5MmTa9SnIAj4IPIiHhWUwM/FFlc0PKidlpaGWbNmISoqChYW3JaEiIhIFxiSdGjOnDmYP38+XnnlFQBAQEAAUlJSEB4eXuOQFBl3F4cv34eZWIS1IYEY8nnFNufOnUNmZiY6deqkOqZQKHDy5Els3LgRcrkcYrG4Rp9PRETUUDEk6VBhYSFMTNQf8xKLxWpLAGjjXk4RPtx/GQAwq68P/KX2Gtv17dsXFy9eVDs2ZcoU+Pn5Yd68eQxIRERENcCQpEPDhg3DihUr0Lx5c/j7+yMuLg7r1q3D1KlTVW2ysrKQmpqKe/fuAQCuXbsGAHBxcYGLS/nK2ZMmTYJUKsVdn1HIKy5DgKs1utnnIT4+HiUlJbh79y7i4+NhY2ODli1bwtbWFm3btlWrxdraGo6OjhWOExERUfWIBEEQ9F2EMZLJZLC3t0dubi7s7OwAAHl5eVi8eDEiIyORmZkJqVSK8ePHY8mSJTA3NwcAREREYMqUKRX6+/DDD/HRRx8BAHr37g2FtRPSAqZAYmqCf49sjj5dKoadF198EcePH9dYX+/evdG+fXt8/vnnOhkvERFRfaDp+7syDEk1pM3fZG39mfwIr/3nT5QolFgytA2m9vTSaf9EREQNlTbf37zdZmC2/5mKBXv+fr7I0pzPExEREekDF5M0IOm5RVi4R/0B7EWRl5CeW6SnioiIiBouhiQDkvywAE/f+1QIAm4/LNRLPURERA0ZQ5IB8XKyholI/ZhYJIKnU8UVtomIiKh2MSQZEFd7S4SPDoBYVJ6UxCIRVo5uC1d7Sz1XRkRE1PDwwW0DE9qlOXq1aoLbDwvh6WTFgERERKQnDEkGyNXekuGIiIhIz3i7jYiIiEgDhiQiIiIiDRiSiIiIiDRgSCIiIiLSgCGJiIiISAOGJCIiIiINGJKIiIiINGBIIiIiItKAIYmIiIhIA4YkIiIiIg0YkoiIiIg0YEgiIiIi0sCgQ1JZWRkWLVoELy8vWFpawtvbG8uWLYNSqVS1EQQBH330EaRSKSwtLdG7d29cvnz5mX3v3r0bbdq0gUQiQZs2bRAZGVmbQyEiIiIjY9AhadWqVdi8eTM2btyIK1euYPXq1VizZg02bNigarN69WqsW7cOGzduxJkzZ+Di4oL+/fsjLy+v0n5jY2MRGhqKiRMnIiEhARMnTkRISAhOnz5dF8MiIiIiIyASBEHQdxGVGTp0KJydnfH111+rjo0ZMwZWVlbYunUrBEGAVCpFWFgY5s2bBwCQy+VwdnbGqlWrMH36dI39hoaGQiaT4dChQ6pjgwYNQuPGjbF9+/Zq1SaTyWBvb4/c3FzY2dk9xyiJiIiormjz/W1aRzXVSM+ePbF582Zcv34drVq1QkJCAmJiYvD5558DAJKTk5GRkYEBAwaozpFIJHjxxRdx6tSpSkNSbGws3n33XbVjAwcOVPWriVwuh1wuV/2em5sLoPxvNhERERmHx9/b1ZkjMuiQNG/ePOTm5sLPzw9isRgKhQIrVqzA+PHjAQAZGRkAAGdnZ7XznJ2dkZKSUmm/GRkZGs953J8m4eHhWLp0aYXj7u7u1R4PERERGYa8vDzY29tX2cagQ9LOnTuxbds2/Pjjj/D390d8fDzCwsIglUoxefJkVTuRSKR2niAIFY49TdtzFixYgNmzZ6t+VyqVyMrKgqOj4zM/S1symQzu7u5IS0url7fyOD7jV9/HWN/HB9T/MXJ8xq+2xigIAvLy8iCVSp/Z1qBD0pw5czB//ny88sorAICAgACkpKQgPDwckydPhouLC4DymSFXV1fVeZmZmRVmip7k4uJSYdboWedIJBJIJBK1Y40aNdJ2SFqxs7Ort//wAxxffVDfx1jfxwfU/zFyfMavNsb4rBmkxwz67bbCwkKYmKiXKBaLVUsAeHl5wcXFBdHR0ao/LykpwYkTJ9C9e/dK+w0ODlY7BwCioqKqPIeIiIgaFoOeSRo2bBhWrFiB5s2bw9/fH3FxcVi3bh2mTp0KoPyWWVhYGFauXAkfHx/4+Phg5cqVsLKywoQJE1T9TJo0CW5ubggPDwcAzJo1C7169cKqVaswYsQI7Nu3D0eOHEFMTIxexklERESGx6BD0oYNG7B48WLMmDEDmZmZkEqlmD59OpYsWaJqM3fuXBQVFWHGjBnIzs5GUFAQoqKiYGtrq2qTmpqqNiPVvXt37NixA4sWLcLixYvRokUL7Ny5E0FBQXU6vspIJBJ8+OGHFW7v1Rccn/Gr72Os7+MD6v8YOT7jZwhjNOh1koiIiIj0xaCfSSIiIiLSF4YkIiIiIg0YkoiIiIg0YEgiIiIi0oAhSU+++uoreHl5wcLCAp06dcJvv/1WZfsTJ06gU6dOsLCwgLe3NzZv3lxHldaMNuM7fvw4RCJRhZ+rV6/WYcXVd/LkSQwbNgxSqRQikQh79+595jnGdP20HZ+xXb/w8HB06dIFtra2aNq0KUaOHIlr16498zxjuYY1GZ+xXcNNmzahXbt2qkUGg4OD1TYs18RYrh+g/fiM7fo9LTw8XLWkT1X0cQ0ZkvRg586dCAsLwwcffIC4uDi88MILePnll5GamqqxfXJyMgYPHowXXngBcXFxWLhwId555x3s3r27jiuvHm3H99i1a9eQnp6u+vHx8amjirVTUFCAwMBAbNy4sVrtje36aTu+x4zl+p04cQIzZ87EH3/8gejoaJSVlWHAgAEoKCio9BxjuoY1Gd9jxnINmzVrhk8++QRnz57F2bNn8dJLL2HEiBG4fPmyxvbGdP0A7cf3mLFcvyedOXMGW7ZsQbt27apsp7drKFCd69q1q/CPf/xD7Zifn58wf/58je3nzp0r+Pn5qR2bPn260K1bt1qr8XloO75jx44JAITs7Ow6qE63AAiRkZFVtjG26/ek6ozPmK+fIAhCZmamAEA4ceJEpW2M+RpWZ3zGfg0FQRAaN24s/Oc//9H4Z8Z8/R6ranzGev3y8vIEHx8fITo6WnjxxReFWbNmVdpWX9eQM0l1rKSkBOfOncOAAQPUjg8YMACnTp3SeE5sbGyF9gMHDsTZs2dRWlpaa7XWRE3G91iHDh3g6uqKvn374tixY7VZZp0ypuv3PIz1+uXm5gIAHBwcKm1jzNewOuN7zBivoUKhwI4dO1BQUIDg4GCNbYz5+lVnfI8Z2/WbOXMmhgwZgn79+j2zrb6uIUNSHXv48CEUCkWFzXSdnZ0rbLr7WEZGhsb2ZWVlePjwYa3VWhM1GZ+rqyu2bNmC3bt3Y8+ePfD19UXfvn1x8uTJuii51hnT9asJY75+giBg9uzZ6NmzJ9q2bVtpO2O9htUdnzFew4sXL8LGxgYSiQT/+Mc/EBkZiTZt2mhsa4zXT5vxGeP127FjB86fP6/aLuxZ9HUNDXpbkvpMJBKp/S4IQoVjz2qv6bih0GZ8vr6+8PX1Vf0eHByMtLQ0fPrpp+jVq1et1llXjO36acOYr98///lPXLhwoVr7NhrjNazu+IzxGvr6+iI+Ph45OTnYvXs3Jk+ejBMnTlQaJIzt+mkzPmO7fmlpaZg1axaioqJgYWFR7fP0cQ05k1THnJycIBaLK8yqZGZmVkjJj7m4uGhsb2pqCkdHx1qrtSZqMj5NunXrhqSkJF2XpxfGdP10xRiu37/+9S/s378fx44dQ7Nmzapsa4zXUJvxaWLo19Dc3BwtW7ZE586dER4ejsDAQHzxxRca2xrj9dNmfJoY8vU7d+4cMjMz0alTJ5iamsLU1BQnTpzA+vXrYWpqCoVCUeEcfV1DhqQ6Zm5ujk6dOiE6OlrteHR0NLp3767xnODg4Arto6Ki0LlzZ5iZmdVarTVRk/FpEhcXB1dXV12XpxfGdP10xZCvnyAI+Oc//4k9e/bg6NGj8PLyeuY5xnQNazI+TQz5GmoiCALkcrnGPzOm61eZqsaniSFfv759++LixYuIj49X/XTu3Bmvvvoq4uPjIRaLK5yjt2tYq4+Fk0Y7duwQzMzMhK+//lpITEwUwsLCBGtra+H27duCIAjC/PnzhYkTJ6ra37p1S7CyshLeffddITExUfj6668FMzMz4eeff9bXEKqk7fg+++wzITIyUrh+/bpw6dIlYf78+QIAYffu3foaQpXy8vKEuLg4IS4uTgAgrFu3ToiLixNSUlIEQTD+66ft+Izt+r399tuCvb29cPz4cSE9PV31U1hYqGpjzNewJuMztmu4YMEC4eTJk0JycrJw4cIFYeHChYKJiYkQFRUlCIJxXz9B0H58xnb9NHn67TZDuYYMSXry5ZdfCh4eHoK5ubnQsWNHtddzJ0+eLLz44otq7Y8fPy506NBBMDc3Fzw9PYVNmzbVccXa0WZ8q1atElq0aCFYWFgIjRs3Fnr27CkcOHBAD1VXz+PXbZ/+mTx5siAIxn/9tB2fsV0/TWMDIHz77beqNsZ8DWsyPmO7hlOnTlX9+6VJkyZC3759VQFCEIz7+gmC9uMztuunydMhyVCuoUgQ/nryiYiIiIhU+EwSERERkQYMSUREREQaMCQRERERacCQRERERKQBQxIRERGRBgxJRERERBowJBERERFpwJBERKQjx48fh0gkQk5Ojr5LISIdYEgiIiIi0oAhiYiIiEgDhiQiqjcEQcDq1avh7e0NS0tLBAYG4ueffwbw962wAwcOIDAwEBYWFggKCsLFixfV+ti9ezf8/f0hkUjg6emJtWvXqv25XC7H3Llz4e7uDolEAh8fH3z99ddqbc6dO4fOnTvDysoK3bt3x7Vr12p34ERUKxiSiKjeWLRoEb799lts2rQJly9fxrvvvovXXnsNJ06cULWZM2cOPv30U5w5cwZNmzbF8OHDUVpaCqA83ISEhOCVV17BxYsX8dFHH2Hx4sWIiIhQnT9p0iTs2LED69evx5UrV7B582bY2Nio1fHBBx9g7dq1OHv2LExNTTF16tQ6GT8R6RY3uCWieqGgoABOTk44evQogoODVcenTZuGwsJCvPXWW+jTpw927NiB0NBQAEBWVhaaNWuGiIgIhISE4NVXX8WDBw8QFRWlOn/u3Lk4cOAALl++jOvXr8PX1xfR0dHo169fhRqOHz+OPn364MiRI+jbty8A4ODBgxgyZAiKiopgYWFRy38XiEiXOJNERPVCYmIiiouL0b9/f9jY2Kh+vv/+e9y8eVPV7skA5eDgAF9fX1y5cgUAcOXKFfTo0UOt3x49eiApKQkKhQLx8fEQi8V48cUXq6ylXbt2qr92dXUFAGRmZj73GImobpnquwAiIl1QKpUAgAMHDsDNzU3tzyQSiVpQeppIJAJQ/kzT479+7MnJdktLy2rVYmZmVqHvx/URkfHgTBIR1Qtt2rSBRCJBamoqWrZsqfbj7u6uavfHH3+o/jo7OxvXr1+Hn5+fqo+YmBi1fk+dOoVWrVpBLBYjICAASqVS7RknIqq/OJNERPWCra0t3n//fbz77rtQKpXo2bMnZDIZTp06BRsbG3h4eAAAli1bBkdHRzg7O+ODDz6Ak5MTRo4cCQB477330KVLFyxfvhyhoaGIjY3Fxo0b8dVXXwEAPD09MXnyZEydOhXr169HYGAgUlJSkJmZiZCQEH0NnYhqCUMSEdUby5cvR9OmTREeHo5bt26hUaNG6NixIxYuXKi63fXJJ59g1qxZSEpKQmBgIPbv3w9zc3MAQMeOHfHTTz9hyZIlWL58OVxdXbFs2TK8/vrrqs/YtGkTFi5ciBkzZuDRo0do3rw5Fi5cqI/hElEt49ttRNQgPH7zLDs7G40aNdJ3OURkBPhMEhEREZEGDElEREREGvB2GxEREZEGnEkiIiIi0oAhiYiIiEgDhiQiIiIiDRiSiIiIiDRgSCIiIiLSgCGJiIiISAOGJCIiIiINGJKIiIiINGBIIiIiItLg/wE7w+URUhR3kQAAAABJRU5ErkJggg==", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "plt.plot(np.arange(len(epochs_x)), epochs_acc, marker = '.')\n", - "plt.xlabel('epoch')\n", - "plt.ylabel('test accuracy')\n", - "plt.ylim(80, 100)\n", - "for i, txt in enumerate(epochs_acc):\n", - " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom')\n", - "plt.show()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "speck-rescnn", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/tests/test_nonsequential/baseline-SCNN-example_3.ipynb b/tests/test_nonsequential/baseline-SCNN-example_3.ipynb deleted file mode 100644 index dc53313a..00000000 --- a/tests/test_nonsequential/baseline-SCNN-example_3.ipynb +++ /dev/null @@ -1,1500 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import torch, random\n", - "import torch.nn as nn\n", - "import sinabs.layers as sl\n", - "from tqdm.notebook import tqdm\n", - "\n", - "from tonic.datasets.dvsgesture import DVSGesture\n", - "from tonic.transforms import ToFrame\n", - "from torch.utils.data import DataLoader\n", - "from torch.nn import CrossEntropyLoss\n", - "from torch.optim import Adam\n", - "\n", - "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential\n", - "\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "torch.backends.cudnn.deterministic = True\n", - "random.seed(1)\n", - "torch.manual_seed(1)\n", - "torch.cuda.manual_seed(1)\n", - "np.random.seed(1)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Loading Data" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "batch_size = 3\n", - "num_workers = 1\n", - "epochs = 30\n", - "lr = 1e-3" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "root_dir = \"./DVSGESTURE\"\n", - "_ = DVSGesture(save_to=root_dir, train=True)\n", - "_ = DVSGesture(save_to=root_dir, train=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "n_time_steps = 50\n", - "to_raster = ToFrame(sensor_size=DVSGesture.sensor_size, n_time_bins=n_time_steps)\n", - "\n", - "snn_train_dataset = DVSGesture(save_to=root_dir, train=True, transform=to_raster)\n", - "snn_test_dataset = DVSGesture(save_to=root_dir, train=False, transform=to_raster)" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The transformed array is in shape [Time-Step, Channel, Height, Width] --> (50, 2, 128, 128)\n" - ] - } - ], - "source": [ - "sample_data, label = snn_train_dataset[0]\n", - "print(f\"The transformed array is in shape [Time-Step, Channel, Height, Width] --> {sample_data.shape}\")" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "snn_train_dataloader = DataLoader(snn_train_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True)\n", - "snn_test_dataloader = DataLoader(snn_test_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=False)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Network Module\n", - "\n", - "We need to define a `nn.Module` implementing the network we want the chip to reproduce." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "device: NVIDIA RTX A4000\n" - ] - } - ], - "source": [ - "if torch.cuda.is_available():\n", - " device = torch.device('cuda:0')\n", - " print('device: ', torch.cuda.get_device_name(0))\n", - "else:\n", - " device = torch.device('cpu')" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "class SNN(nn.Module):\n", - " def __init__(self) -> None:\n", - " super().__init__()\n", - "\n", - " self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False)\n", - " self.iaf1 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - " self.pool1 = nn.AvgPool2d(2,2)\n", - "\n", - " self.conv2 = nn.Conv2d(10, 10, 2, 1, bias=False)\n", - " self.iaf2 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - " self.pool2 = nn.AvgPool2d(3,3)\n", - "\n", - " self.conv3 = nn.Conv2d(10, 10, 3, 1, bias=False)\n", - " self.iaf3 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - " self.pool3 = nn.AvgPool2d(2,2)\n", - "\n", - " self.flat = nn.Flatten()\n", - "\n", - " self.fc1 = nn.Linear(810, 100, bias=False)\n", - " self.iaf4 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - "\n", - " self.fc2 = nn.Linear(100, 100, bias=False)\n", - " self.iaf5 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - "\n", - " self.fc3 = nn.Linear(100, 100, bias=False)\n", - " self.iaf6 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - "\n", - " self.fc4 = nn.Linear(100, 11, bias=False)\n", - " self.iaf7 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - "\n", - " def detach_neuron_states(self):\n", - " for name, layer in self.named_modules():\n", - " if name != '':\n", - " if isinstance(layer, sl.StatefulLayer):\n", - " for name, buffer in layer.named_buffers():\n", - " buffer.detach_()\n", - "\n", - " def init_weights(self):\n", - " for name, layer in self.named_modules():\n", - " if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear):\n", - " nn.init.xavier_normal_(layer.weight.data)\n", - "\n", - " def forward(self, x):\n", - " \n", - " con1_out = self.conv1(x)\n", - " iaf1_out = self.iaf1(con1_out)\n", - " pool1_out = self.pool1(iaf1_out)\n", - "\n", - " conv2_out = self.conv2(pool1_out)\n", - " iaf2_out = self.iaf2(conv2_out)\n", - " pool2_out = self.pool2(iaf2_out)\n", - "\n", - " conv3_out = self.conv3(pool2_out)\n", - " iaf3_out = self.iaf3(conv3_out)\n", - " pool3_out = self.pool3(iaf3_out)\n", - "\n", - " flat_out = self.flat(pool3_out)\n", - " \n", - " fc1_out = self.fc1(flat_out)\n", - " iaf4_out = self.iaf4(fc1_out)\n", - "\n", - " fc2_out = self.fc2(iaf4_out)\n", - " iaf5_out = self.iaf5(fc2_out)\n", - "\n", - " fc3_out = self.fc3(iaf5_out)\n", - " iaf6_out = self.iaf6(fc3_out)\n", - "\n", - " fc4_out = self.fc4(iaf6_out)\n", - " iaf7_out = self.iaf7(fc4_out)\n", - "\n", - " return iaf7_out" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "snn = SNN().to(device)" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "snn.init_weights()" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "optimizer = Adam(snn.parameters(), lr=lr, betas=(0.9, 0.999), eps=1e-8)\n", - "loss_fn = CrossEntropyLoss()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Define train and test" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "def train(dataloader, model, loss_fn, optimizer, epochs, test_func, dataloader_test):\n", - " epochs_y = []\n", - " epochs_x = []\n", - " epochs_acc = []\n", - " model.train()\n", - "\n", - " for e in range(epochs):\n", - " losses = []\n", - " batches = []\n", - " batch_count = 0\n", - " train_p_bar = tqdm(snn_train_dataloader)\n", - "\n", - " for X, y in train_p_bar:\n", - " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", - " X = X.reshape(-1, 2, 128, 128).to(dtype=torch.float, device=device)\n", - " y = y.to(dtype=torch.long, device=device)\n", - "\n", - " # forward\n", - " pred = model(X)\n", - "\n", - " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", - " pred = pred.reshape(batch_size, n_time_steps, -1)\n", - "\n", - " # accumulate all time-steps output for final prediction\n", - " pred = pred.sum(dim = 1)\n", - " loss = loss_fn(pred, y)\n", - "\n", - " # gradient update\n", - " optimizer.zero_grad()\n", - " loss.backward()\n", - " optimizer.step()\n", - "\n", - " # detach the neuron states and activations from current computation graph(necessary)\n", - " model.detach_neuron_states()\n", - "\n", - " train_p_bar.set_description(f\"Epoch {e} - BPTT Training Loss: {round(loss.item(), 4)}\")\n", - "\n", - " batch_count += 1\n", - " losses.append(loss.item())\n", - " batches.append(batch_count)\n", - "\n", - " epochs_y.append(losses)\n", - " epochs_x.append(batches)\n", - "\n", - " acc = test_func(dataloader_test, model)\n", - " print(f'Epoch {e} accuracy: {acc}')\n", - " epochs_acc.append(acc)\n", - "\n", - " return epochs_x, epochs_y, epochs_acc\n" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [], - "source": [ - "def test(dataloader, model):\n", - " correct_predictions = []\n", - " with torch.no_grad():\n", - " test_p_bar = tqdm(dataloader)\n", - " for X, y in test_p_bar:\n", - " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", - " X = X.reshape(-1, 2, 128, 128).to(dtype=torch.float, device=device)\n", - " y = y.to(dtype=torch.long, device=device)\n", - "\n", - " # forward\n", - " output = model(X)\n", - "\n", - " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", - " output = output.reshape(batch_size, n_time_steps, -1)\n", - "\n", - " # accumulate all time-steps output for final prediction\n", - " output = output.sum(dim=1)\n", - "\n", - " # calculate accuracy\n", - " pred = output.argmax(dim=1, keepdim=True)\n", - "\n", - " # compute the total correct predictions\n", - " correct_predictions.append(pred.eq(y.view_as(pred)))\n", - "\n", - " test_p_bar.set_description(f\"Testing Model...\")\n", - " \n", - " correct_predictions = torch.cat(correct_predictions)\n", - " return correct_predictions.sum().item()/(len(correct_predictions))*100" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Training loop" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "ebf2bea3d0124365abf1f2aa248ceab6", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - " 0%| | 0/359 [00:00" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "y_avg = []\n", - "for y in epochs_y:\n", - " y_avg.append(np.mean(y))\n", - "\n", - "plt.plot(np.arange(len(epochs_x)), y_avg, marker = '.')\n", - "plt.xlabel('epoch')\n", - "plt.ylabel('average loss')\n", - "plt.ylim(0,)\n", - "plt.xticks(np.arange(len(epochs_x)))\n", - "for i, txt in enumerate(y_avg):\n", - " if i%2 == 0:\n", - " pass\n", - " else:\n", - " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAG2CAYAAACZEEfAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAAB9HElEQVR4nO3dd1RUx98G8GfpRUB6kaqg2HvB3oIt9qiJJjHGWH5irLFgYkw0ETWJJsbE9ipqrDGxG3vBhgiKXSmKglQLLL3uvH8QNq60XYrg+nzO2XNk7p07c3evu987d4pECCFAREREpKY0qroCRERERJWJwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqbUqDXbOnTuH/v37w87ODhKJBPv27VPYLoTA119/DVtbW+jr66Nnz54ICwtT2OfFixcYNWoUjI2NUbNmTYwdOxapqamv8SyIiIioOqvSYCctLQ1NmzbFb7/9VuT2ZcuWYeXKlVizZg0CAgJgaGiIXr16ITMzU77PqFGjcOfOHZw4cQKHDh3CuXPnMH78+Nd1CkRERFTNSarLQqASiQR79+7FoEGDAOS36tjZ2WHmzJn44osvAABSqRTW1tbYtGkT3n//fdy7dw8NGjRAYGAgWrVqBQA4evQo+vbtiydPnsDOzq6qToeIiIiqCa2qrkBxIiIiEBcXh549e8rTTExM0LZtW/j7++P999+Hv78/atasKQ90AKBnz57Q0NBAQEAABg8eXOSxs7KykJWVJf9bJpPhxYsXMDc3h0QiqbyTIiIiogojhEBKSgrs7OygoVH8w6pqG+zExcUBAKytrRXSra2t5dvi4uJgZWWlsF1LSwtmZmbyfYri4+ODb7/9toJrTERERFUhKioK9vb2xW6vtsFOZfL29saMGTPkf0ulUjg6OiIqKgrGxsZVWDMiIiJSVnJyMhwcHGBkZFTiftU22LGxsQEAxMfHw9bWVp4eHx+PZs2ayfdJSEhQyJebm4sXL17I8xdFV1cXurq6hdKNjY0Z7BAREb1hSuuCUm3n2XFxcYGNjQ1OnTolT0tOTkZAQAA8PDwAAB4eHkhKSsLVq1fl+5w+fRoymQxt27Z97XUmIiKi6qdKW3ZSU1MRHh4u/zsiIgLXr1+HmZkZHB0dMW3aNHz33Xdwc3ODi4sL5s+fDzs7O/mIrfr166N3794YN24c1qxZg5ycHEyePBnvv/8+R2IRERERgCoOdoKCgtCtWzf53wX9aEaPHo1NmzZh9uzZSEtLw/jx45GUlISOHTvi6NGj0NPTk+fZtm0bJk+ejB49ekBDQwNDhw7FypUrX/u5EBERUfVUbebZqUrJyckwMTGBVCplnx0iIqI3hLK/39W2zw4RERFRRWCwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGptWod7OTl5WH+/PlwcXGBvr4+6tSpg0WLFkEIId9HCIGvv/4atra20NfXR8+ePREWFlaFtSYiIqLqpFoHO0uXLsXq1auxatUq3Lt3D0uXLsWyZcvw66+/yvdZtmwZVq5ciTVr1iAgIACGhobo1asXMjMzq7DmREREVF1IxMvNJNXMu+++C2tra2zYsEGeNnToUOjr62Pr1q0QQsDOzg4zZ87EF198AQCQSqWwtrbGpk2b8P777ytVTnJyMkxMTCCVSmFsbFwp50JEREQVS9nf72rdstO+fXucOnUKoaGhAIAbN27gwoUL6NOnDwAgIiICcXFx6NmzpzyPiYkJ2rZtC39//2KPm5WVheTkZIUXERERqSetqq5ASebOnYvk5GS4u7tDU1MTeXl5+P777zFq1CgAQFxcHADA2tpaIZ+1tbV8W1F8fHzw7bffVl7FiYiIqNqo1i07f/75J7Zt24bt27fj2rVr2Lx5M3788Uds3ry5XMf19vaGVCqVv6KioiqoxkRERFTdVOuWnVmzZmHu3LnyvjeNGzfG48eP4ePjg9GjR8PGxgYAEB8fD1tbW3m++Ph4NGvWrNjj6urqQldXt1LrTkRERNVDtW7ZSU9Ph4aGYhU1NTUhk8kAAC4uLrCxscGpU6fk25OTkxEQEAAPD4/XWlciqhqx0gxcevAMsdKMqq4KVSJnZ2dIJJJCLy8vLwBA165dC22bOHFiiccs6ngSiQQ//PCDwn6HDx9G27Ztoa+vD1NTUwwaNKiyTpMqSbVu2enfvz++//57ODo6omHDhggODsby5cvx6aefAsi/UKdNm4bvvvsObm5ucHFxwfz582FnZ8eLkegtsCswEt57bkEmAA0J4DOkMUa0dqzqalElCAwMRF5envzv27dv45133sGwYcPkaePGjcPChQvlfxsYGJR4zNjYWIW/jxw5grFjx2Lo0KHytL///hvjxo3D4sWL0b17d+Tm5uL27dvlPR16zap1sPPrr79i/vz5mDRpEhISEmBnZ4cJEybg66+/lu8ze/ZspKWlYfz48UhKSkLHjh1x9OhR6OnpVWHNiaiyxUozMHfPLRRMniETwLw9t9G5riVsTfSrtnJU4SwtLRX+XrJkCerUqYMuXbrI0wwMDOTdG5Tx6r779+9Ht27dULt2bQBAbm4upk6dih9++AFjx46V79egQYOynAJVoWr9GMvIyAg///wzHj9+jIyMDDx48ADfffcddHR05PtIJBIsXLgQcXFxyMzMxMmTJ1G3bt0qrDURvQ7XHifi1VnC8oTAo2fpVVMhem2ys7OxdetWfPrpp5BIJPL0bdu2wcLCAo0aNYK3tzfS05W/FuLj43H48GGFoObatWuIjo6GhoYGmjdvDltbW/Tp04ctO2+gat2yQ0RUlPTsXKw8XXhZGE0J4GxR8qMLevPt27cPSUlJ+OSTT+RpI0eOhJOTE+zs7HDz5k3MmTMHISEh2LNnj1LH3Lx5M4yMjDBkyBB52sOHDwEA33zzDZYvXw5nZ2f89NNP6Nq1K0JDQ2FmZlah50WVh8EOEb1R8mQCU3ZcR0hcKgx1NJGRkwfZvy08ozu48BHWW2DDhg3o06cP7Ozs5Gnjx4+X/7tx48awtbVFjx498ODBA9SpU6fUY27cuBGjRo1S6AJRMBjmyy+/lPfj8fX1hb29PXbv3o0JEyZU1ClRJavWj7GIiF7l8889nLwXDx0tDWwZ2xYX53ZH30b5fS9uRiWhGq+AQxXg8ePHOHnyJD777LMS92vbti0AIDw8vNRjnj9/HiEhIYWOWTClyct9dHR1dVG7dm1ERkaqWnWqQgx2iOiNsfXyY/zfhQgAwE/DmqKlkylsTfSxYEBD6GhqIOhxIgIiXlRxLaky+fr6wsrKCv369Stxv+vXrwOAwhxsxdmwYQNatmyJpk2bKqS3bNkSurq6CAkJkafl5OTg0aNHcHJyUr3yVGUY7BBRmZU298mECRNQp04d6Ovrw9LSEgMHDsT9+/eVPv7EiRMhkUjw888/wy/0KRYcuAMA+Li+Nv7v6//BwsICxsbGGNq3J1rrxgAAfjtT+p08vZlkMhl8fX0xevRoaGn91wvjwYMHWLRoEa5evYpHjx7hwIED+Pjjj9G5c2c0adJEvp+7uzv27t2rcMzk5GTs3r27yJYiY2NjTJw4ERMmTJBf2zo6OkhISMCECRMq5Tp/WWhoKAYOHCi/zjt27IgzZ84ofVz6D4MdIiqzwMBAxMbGyl8nTpwAAPncJy1btoSvry/u3buHY8eOQQgBT09PhflSirN3715cvnwZdnZ2SEjOhNe2a8iTCQxtYY+d309Gbm4uTp8+jatXr6Jp06bYu3QKRHoizoc9Q3BkYqWeN1WNkydPIjIyUj7XWgEdHR2cPHkSnp6ecHd3x8yZMzF06FAcPHhQYb+QkBBIpVKFtJ07d0IIgQ8++KDIMn/44QeMHz8e5ubmMDQ0RKdOnbB+/XoAFX+dv+rdd98tdJ2/++67Ja79SMUQJKRSqQAgpFJpVVeF6I02depUUadOHSGTyYrcfuPGDQFAhIeHl3icJ0+eiFq1aonbt28LB0dH4dxvknCac0gMX3NJRMfGCwDi3Llz8v2Tk5MFADH0yzXCac4hMXbTlQo9L8rn5OQkABR6TZo0SQghxPjx40Xt2rWFnp6esLCwEAMGDBD37t0r8Zh///23eOedd4SZmZkAIIKDgwvts3btWtGlSxdhZGQkAIjExMQyn0NMUrq4GP5UxCSll/kYlXGdOzk5iRUrVsi3PX36tNjr/MSJE2WuuyrehM9b2d9vtuwQUYUobu6TAmlpafD19YWLiwscHByKPY5MJsNHH32EWbNmobabO56lZkOakQMXC0Os/aglbK0tUa9ePWzZsgVpaWnIzc3F2rVrYWVlhXmj+0IiAU7eS8DdmOTKPN23UmW05KWlpaFjx45YunRpsfukp6ejd+/emDdvXrnqvyswEh2WnMbI9QHosOQ0dgWq3sm4Mq7zhg0bFtpubm6OevXqoW/fvvJHaMbGxgCAd955B15eXnjx4gU+//xz1KtXD/r6+nB0dMSUKVMKtV69SpllMgYMGICcnBzo6OjA0tISQ4cOxY4dOwC8OZ+3glICu7cCW3aIym/Xrl1CU1NTREdHK6T/9ttvwtDQUAAQ9erVK/Vud/HixeKdd94Rubl5YuIfQULT2ErY9Z4gHj5Nle8TFRUlWrZsKSQSidDU1BS2trbi2rVrQgghvLZdFU5zDolJ265W/EmSgopq4RBCiIiIiGLv9AucOXOmzC07MUnpwnnuIeE0579X7bmHVW7hqejrvOC9e7VlR4j867xJkyYCgNDQ0BBWVlZi9erVAoA4c+aMuHXrlhgyZIg4cOCACA8PF6dOnRJubm5i6NChJZYdGxur8Nq4caOQSCTiwYMH8n2WL18u/P39xaNHj8TFixeFh4eHsLGxqXafN1t2iN4SJXUSLuud3zfffAN3d3cYGhrC1NQUPXv2REBAQInljhgxAm5uboX6HowaNQrBwcHw8/ND3bp1MXz4cGRmZhZZ7tWrV/HLL79g06ZN+OF4KI7cjoNEAgxv5QgXC0MAgBACXl5esLKywvnz53HlyhUMGjQI/fv3R2xsLLy6uQIA/rkViwdPU8v6tlIpKqqF43WJeJZWITNuFzXHD1D267yo9w747zqvVasWLly4gMDAQAwdOhRffPEFnJ2d0aVLFzRq1Ah///03+vfvjzp16qB79+74/vvvcfDgQeTm5hZ7DjY2NgqvV5fJAIDp06ejXbt2cHJyQvv27TFz5kzExcVh9OjRb8TnXUip4ddbgC079CZLSEhQuEs7ceJEue/8tm3bJk6cOCEePHggbt++LcaOHSuMjY1FQkKCfB8nJyexcOFCERsbK65cuSI0NDTEzp07SzxuVlaWMDAwENu3by9y+4oVK4REIhEampoCEo381793tU5OTkIIIU6ePCk0NDQK/X91dXUVPj4+Qgghxm4KFE5zDokZu66X9vZRGVVUC0eBym7ZiU5MU2jVKUvLzqNHj4SGhobYt29fifspe51ramrKX8pc51lZWUJDQ0N4enoWW/b69euFhYWF0ucUFxcntLS0xLZt24rd5/nz58LDw0MAqHafN1t2iN4SlpaWCndphw4dki+QWNY7v5EjR6Jnz56oXbs2GjZsiOXLlyM5ORk3b95U2M/IyAg2NjY4fPgwrKysFFaLLooQAkIIZGVlFbn9o48+wuZDfrAbsxK2Y1biizX7YWdnh1mzZuHYsWMAIF/vSEND8etLQ0NDPuPt5O75rTv7rkcj6gXXyqoMFdHC8TrFJRe+5ub2cVdpxm1l5/hR5jq/efMmrl+/Ln8pc53v27cPMpkMzZs3L/K4z549w6JFixRmky5NUctkFJgzZw4MDQ1hbm6Ou3fvwtPT8435vF/FYIdIjZT2aAEApFIpjI2NFeYpKe2Y69atg4mJSaFJ15YsWQIzMzMsXrwY7u7uCtsePnwIHx8fXL16FZGRkbh06RKGDRsGfX199O3bV77fy3OfJObpYNmVdGiaO2HYO+2xbFw/aGtrw8bGBvXq1QMAeHh4wNTUFKNHj8aNGzcQGhqKWbNmISIiQv4j1MyhJjq5WSBPJrDG74Fybx4praRZjE1MTODm5obOnTvjr7/+wv379wvNbVMVDt2IBQB4NrBGPesaAIC07OID/lcVN8dPWa5zc3NzNGrUSOGlzHU+Z84cSCSSIofJJycno1+/fmjQoAG++eYbpc+rqGUyCsyaNQvBwcH4448/IJVK8fz580IzlFfXz/tVDHaI1EhRCyS+TJU7v0OHDqFGjRrQ09PDihUrcOLECVhYWMi3T5kyBTt37sTixYuRk5ODq1evYvbs2fLtenp6OH/+PPr27QtXV1eMGDECRkZGuHTpEqysrOT7Fcx98iw1C2M2BSIlMxetnEyxdGiTIgM2CwsLHD16FKmpqejevTtatWqFCxcuYP/+/QrB2OR/++7sDnqCOGn1u9N8k1VUC8frIpMJ/HMrP9gZ1soBk/69NnZeiUJunkypYxQ3x4+q17myXr3OW7RogUePHuGrr74qdNORkpKC3r17w8jICHv37oW2trZSZRS3TMbLdahbty7Cw8NhZWWFq1ev4vLly8Uer7p83kVS6uGammOfHVIXnp6e4t133y1ym1QqFW3atBG9e/cW2dnZpR4rNTVVhIWFCX9/f/Hpp58KZ2dnER8fX+z+GzZsEFpaWiIzM1OlOsckpYsz9+NFv1/OCac5h0SnpafFsxTVjlGc91ZfFE5zDomFB+9UyPFIiLy8POHo6CjmzJmjkP7gwQOxePFiERQUJB4/fiwuXrwo+vfvL8zMzBSum3r16ok9e/bI/37+/LkIDg4Whw8fFgDEzp07RXBwsIiNjZXvExsbK4KDg8X69evlc88EBweL58+fK1XnwIjnwmnOIdHo66MiIztXZObkihYLjwunOYfEkVuxpR+ggpRnjp8FCxYIGxsbkZOTo5AulUpFu3btRJcuXURaWppKxxw9erRo2bJlifsUfN7/+9//5H0Bhag+n7eyv98MdgSDHVIPJXWeTE5OFh4eHqJHjx4iIyOjTMd3dXUVixcvLnb77du3BQBx//59pY+588pj4fLScOB6X/4jwuJTylS/opwNScg/7lf/VFgA9bY7duyYACBCQkIU0qOjo0WfPn2ElZWV0NbWFvb29mLkyJGFrgcAwtfXV/63r69vkRPXLViwQL7PggULitzn5eOUZMH+28JpziExfWewPG3JkXvCac4h8eH/XVb1LSiTl691l7mHxM4rj5XOW1yAKZVKRdu2bUXjxo1FeHi4wkCF3Nxc+X6vBhwFeQ0MDMTq1asLlXf58mXx66+/iuDgYLFlyxYBQDRv3lzUqVNHfjNTXT5vBjsqYLBD6qAy7vxeVrt2bYUvpFdt3bpVaGhoiBcvXih1vIinqcL5ldExLnMPlWtm21fJZDLR/9fzwmnOIbH0SMkzu74pSpvVNiMjQ0yaNEmYmZkJQ0NDMWTIEBEXF6f08SdMmCAAFJrzpahyC0a/qaoiZjFWVl6eTLT+7oRwmnNInLz73/sQ+TxNPu/Oy3M4VYaYpHSFoF7VkWDFBZgFo5WKekVERMj3KypQWLt2rdDX1xdJSUmFyrt586bo1q2bMDMzE7q6usLZ2Vl8/OlnYt+Fm2X6zCrz81b291u5HopEVK3JZDL834aN6DlgGJ6m5cDWJP+/dnJyMjw9PZGeno6tW7ciOTkZycn5MwtbWlpCU1MTQH7nSR8fHwwePBhpaWn4/vvvMWDAANja2uLZs2f47bffEB0dLZ851d/fHwEBAejWrRuMjIzg7++P6dOn48MPP4SpqWkJ9RQIiHiBfcHROHAjGuLV7QJ49CxdpREyJZFIJPDq5ooJf1zFFv/HmNC5DkwMlOvPUF0FBgYqzFB7+/ZtvPPOO/LPZvr06Th8+DB2794NExMTTJ48GUOGDMHFixdLPXZJ6zQBwMKFCzFu3Dj530ZGRirXf3vAY3y59zYEAA0J4DOkMUa0dlT5OMoKepyIhJQsGOlpoaPbf33OHMwM0K2eFU7fT8C2y4/x1bsNKq0OEc/SICtmjh9lrnVPT89CHYMBoGvXrkWmv0oIgVhpBi49eAYXC0PYmuhj/PjxRfbdE0LAvnY9rN2xH09Ts/AsNRsn7sTh0M1Y+B2MhORgJLrWs0QDO+NSywWAuzHJOBvy9LV93sVhsEOkBr76bTuin0ThtGiEDktOy79Qrl27Jp8M0NXVVSFPREQEnJ2dAeR3ngwKe4J20gyY6mri/v372Lx5M549ewZzc3O0bt0a58+fl09rr6uri507d+Kbb75BVlYWXFxcMH36dMyYMaPI+oXGp2BvcDT2B0cjpoTOwpoSCZwtDJQ+7+joaMyZMwdHjhxBeno6XF1d4evri1atWgEA4uPjsX3ZXMTtO4Ts9FS0+6cNDmzfADc3t2KP2bVrV/j5+RVK79u3Lw4fPgwg/wdh+vTpWLN2LbIyM6GhoQFXV1ds27ZNoew5c+bg+PHjSEpKQufOnfHrr7+Wu2wLCwssWLAA69evR1JSEiwtLeHo6IguXbpAKpViw4YN2L59O7p37w4gvzNx/fr1cfnyZbRr167E9/Lzzz/HsWPHiu14XDDVQFnFSjPkgQ6QH9zO23MbnetaVliA+6pDN2MAAJ4NbKCrpamw7cN2jjh9PwG7rz7BF73qQU9bs6hDlJuFoW6hNA0JVLrWy2NXYCS899yCTAASCTCilQNcrWrkBzMp2XiWmiV/PU/NRu6rkdlLBIAzIU9xJuSpyvV4HZ93cRjsEL3hYpLSsSPaFE5zDgHI/0KZ8/ctbLr0GLVqGmLOXzdgUUMXFjV0YGmkB4saOrAw0oVpDV0IIfBnUBRc5h7CthfAjn8DpT179pRYZosWLUoclQEA8cmZOHA9BnuDo3E39r91qoz0tNCvsS0GNa+FiGdp+GrvbeQJAU2JBIuHNFL6SzAxMREdOnRAt27dcOTIEVhaWiIsLEzesiSEwKBBg6CtrY3vf9+C5WejkBB8AD169MS9e3dhaGhY5HH37NmD7Oxs+d/Pnz9H06ZN5S0nQP4M0ytXroSuQ2OYN3kHabdOIv5ZDAwMDAqVvX//fhgbG2P58uXo2bMn7t4tX9nLli3DypUrsXnzZtjb26N9+/YwNDREVlYWrl69ipycHPTs2VO+v7u7OxwdHeHv719ssFPaOk0FlixZgkWLFsHR0REjR47E9OnTlZ7CAADO3E8o1JqnSguHqvJkAv/cyl8h/N2mtoW2d6lrBXtTfTxJzMDBGzEY1qpyZv7dez26UJqbVQ3YGBce7l3RYqUZ8kAHAIQAdgZGlZrPWE8LFka60NHUwP24lELbezeyLvUzi03KwNE78Qpplfl5l4TBDlEFUKaFQdW7/D179mDx4sUIDw9HTk4O3NzcMHPmTHz00UcAgJycHMz1nocNO/Yg+Wk0NHQNoefUFDW7fAItI3Pci03GvdiSF8PU1pAg56W7OJkA5v59C7FJmahtVSM/QKqhC4sauqhpoF3s3D2x0gxEPEuDlZEebkQlYd/1aFwMfyb/gtXWlKBrPSsMbl4L3d2t5HfQ7Wqbo2s9Szx6lg5nCwOVvgCXLl0KBwcH+Pr6ytNcXFzk/w4LC8Ply5dx+/ZtuNdvgEOxfnhoNhGJ6z7Bjh07ih1ua2ZmpvD3zp07YWBgIA84hBD48ccfoWlsBesPFgMADFzb4MmvH8LvchAaNGigUHZB8LB69WrY2NiUu+yff/4ZX331FQYOHIg///wTeXl5SEtLk084p6Ojg5o1ayocx9raGnFxcSW+l1paWpgyZUqx+0yZMgUtWrSAmZkZLl26BG9vb8TGxmL58uXF5nlZTp4Mmy49KpSuamueKq5EvMCz1CyY6GujQx2LQts1NSQY2dYRy46GYGtAZKUEOwnJmfC9GAEAWDq0CTQ1AO+/byEkPhV7g6MxpIV9hZf5socJhR+hAYBHHXO42xjBooZu/v9xI51/b4p0YV5DR94KFivNQIclpxWOoSmRYEH/hqUHO9IMHL8bXyjv62rRehmDHaJyUqWFQZW7fDMzM3z55Zdwd3eHjo4ODh06hDFjxsDKygq9evVCeMxzbD54FtqthsHWygWyzFS8OLUOT/csQq1PfobP4MbIFaJQM/Wz1Gw8S8lCSlauQqBTQAD4+VRYoXQtDQnMa+R/IVoa6cq/GKNepOOfW7GF7tgBoKWTKQY3r4V+jW1haqhT5HnamuiX6S7vwIED6NWrF4YNGwY/Pz/UqlULkyZNkvcpKZjrQ09PD5oaEvyvax3M/usmMmWa8Dt3vtiA41UbNmzA+++/L/+cHj58iPT0dBg2aIOn+3yQGXUbmjXMoWlsiTPnLuB/n36sUHYBDQ0N6Orq4sKFC2UuOyIiAnFxcfKWm4JZjJOTk+Hv74+2bdsqddyXFazTdO3atWKDWQAKjyibNGkCHR0dTJgwAT4+PtDVLfyY5lXrzj1EaHwq9LU1kZWbJ/8BnOlZt9Lu8g/fyn+E1auhNXS0ip5WbngrB6w4EYobUUm49USKxvYmFVqHX0+HIzNHhhaONTG8lT0kEgnik7Pww7EQfHvwLjq6WcDKqPJaeC4+eFYoTVMiwfLhTZV6321N9OEzpDHm7VG9BbY8eSsagx2iclKlhUGVu/yuXbsq/D116lRs3rwZFy5cQE23Vpi07RZqDP4G5oY6eK+lPf7vfATwzkTEbZmBme3NMKJNyZ0AM3PycDdGiqFr/BUWSJQA6OZuhbSsXHlwJM3IQa5MID45C/FFTLv/qnGdXPBRO2c4mlfeHdzDhw+xevVqzJgxA/PmzUNgYCCmTJkCHR0djB49Wv74xtvbG2vXrkW/hpaYNX8fcpKf4npIhFJlXLlyBbdv38aGDRsAAJHP0+G14QwAIC3kIoxbD4K1x3BkxYbhxfHfccrvImKSMgqVbWhoiBUrVuDJkyeIjY0tU9kA5K0z1tbW8lmM9+zZg23btiEuLg42NjbIzs5GUlKSQutOfHx8sX1tzp8/j4SEBDg6/ne95OXlYebMmfj555/x6NGjIvO1bdsWubm5ePTokXzW3+KEJ6Tgl38D6O8GNUJ7V3NM+OMqbj6R4kVadol5yyo3T4ajt/Pfr35Niu5wDQAWNXTRt7Et9l+PwdbLj7H0vSYVVofHz9Ow40okAGB2b3d5MDm+c238cysWd2KSsWD/Haz+sGWFlfmyq48Tse7cQwD5fXWEQJkCjhGtHdG5btlaYMuTtyJxBmWicjpw4ABatWqFYcOGwcrKCs2bN8f69evl20u7y1eGEAKnTp1CSEgIsi3rYdT/BeB5WjYa2hnjwOcd4d23Pi7M7QbvHk6QSCT4qEvx/S4K6GlrooWTGZYMaQzNf7+ENSUSLBnaGBs/aY1dEzxwamZX3FjgidDv+sDfuzsOTO6AjZ+0wrKhTTCrVz30bmhd5LG7u1tXaqAD5PczadGiBRYvXozmzZtj/PjxGDduHNasWQMA0NbWxp49exAaGgozMzOYGNWASWIo9Gq3xJOkTGTnlj5z7oYNG9C4cWM0b9kKa/wewPNnP1yPTAIAWDi4wrzraOhY14Fxs97QqmmNpLgovLPcD1uvPMHuv/6Wl21gYIAzZ86gT58+hdb0Kq3sNm3aFLm9qFmMW7ZsCW1tbZw6dUqeFhISgsjISHh4eBR5HGXWaSrK9evXoaGhoTBLcFHyZAKz/7qJ7FwZutazxJAWtWBroo/pPesCAP4MikK6Css2KCv/EVY2ahpoo30d8xL3/bCdEwBg/41oSDNyKqwOK06EIlcm0LmuJdrV/q8O2poaWPZeE2hpSHDkdhyO3FIuAFaFND0HU3YEI1cm8G4TW1yc0w07xrXDhbndyjQaytZEHx51zMsUrJQnb0Vhyw5ROanawqDKXb5UKkWtWrWQlZUFTU1NdB/rjV0xpgAE+je1w7KhTaCvk/9s3VRXgs2/fI8PPvgAxsbKDQsFlLvz0tHSKPJxU1U+k7e1tUWDBorDhevXr4+///5b/nfLli1x/fp1SKVSZGdnw6imGSxqN0KuSR3sDX5S4pd+Wloadu7ciQnTvdH/1wvyTpqt67vgEICOzerh97nd5e+bxy59PElLRFp2Hr49eBdN7U2w7R8/2Bvmry9maWmJtm3byvtxlaSg7IULFyqkF7TOxMbGKkw1EB8fj2bNmsHExARjx47FjBkzYGZmBmNjY3z++efw8PBQ6Jz88lQD5ubmMDdXDAZeXaeprFMNAMDmS49wLTIJNXS1sHhwY3nrRpe6lnA2N8Cj5+nYGxyNUW2dSn1fVHHo3wCid0MbaGuWHGC2cjJFPWsjhMSnYM+1JxjTwaXE/ZVxLzYZ+2/kP0ab3atwy1dDOxNM7FIHq86EY/7+O2hX27zYR72qEkJg9t83EJ2UAUczA/gMaQwjPW3Y1Xz9fWWqC7bsEJWTqi0MqtzlGxkZ4fr16zh29gLq9huHoxt+QGbkTczt446V7zeTBzo5OTkYPnw4hBBYvXq1yudQ1juvgmfyL7cMva5n8h06dEBISIhCWmhoKJycCv9ompiYwNLSElGPHiI9JhQGbm2x+uyDEtdF+mP7TqRlZGJnkhPux6WgpoE2fnivCfZ7D4aenh5u3Lghf98MJTl4EvUYtZ0c8N2gRjDS1cKNJ1IMWHURv1+MgaGJKcLCwhAUFISBAweWem67d+9GVlYWPvzwQ4V0FxcX2NjYYNaiH+VTDXh8ewj+lwPkLTcrVqzAu+++i6FDh6Jz586wsbEpNLouJCQEj2Of4tKDZ4iVZpRan4KpBrp06YKGDRvi+++/x/Tp07Fu3boS80U+T8cPx/I/o7l93GFX87/rQkNDgo88nAHkB0TKzBejrJcfYb1bwiOsAhKJBB965F83Wy8/rpC6/HgsBEIA/ZrYolGtovsBfd7DFa5WNfAsNQuLDt0td5kFtvg/xrE78dDWlGDVyOYw0nuz55aqEBU+neEbiDMoq4cnT56IUaNGCTMzM6GnpycaNWokAgMD5dtRzEyjy5YtK/aYubm54quvvhLOzs5CT09P1K5dWyxcuFDIZDL5Po6OjqJZs2bCxsZG6OnpiR49eohvvvlG2NnZFTpeUlKSSEhIEEII0aZNG/mstyW5EZUo2n5/UjjNOSRMm/cSrTp0VdienZ0tBg0aJJo0aSKePXtW6vEqQ0xSurgU/uy1zIhb4MqVK0JLS0t8//33IiwsTGzbtk0YGBiIrVu3yvf5888/xZkzZ8SDBw/Evn37hJOTkxgwaLBo9u0x4TTnkNgX/ER89NFHYu7cuQrHPnY7Vhg5NRIG7p3kywy8vNyEl5eXACA+/PBDcejQIdGyZUshkUjExo0bhRBCxEkzhOfnS4T1B4uF3YT/E+4fLRTWdg5iyJAhCuUUVbYQQnTs2FGMGDFCIS0vTyZuPUkSvcfMEBq6hsJyyHxh++kqoe/WTmiZWIsuPsfEqPWXxdQd18Sig3fE6rPh4q+gKHE2JEHcjk4S8dIMkZObJ4Qo39IFypLJZOKDdf7Cac4hMWLtJZGXJyu0jzQjW9Sff0Q4zTkkLoY9rbCyz4XmLxPSfOFx+TmXJiUzRzQoqEt4+epSsBZXbe/D4kFCycufXH38Qj6T8+l7xa89p6xbT5KE27x/hNOcQ2LD+YflPl51xxmU6a1S2ogoAIUeGR05cgRjx47F0KFDiz3u0qVLsXr1amzevBkNGzZEUFAQxowZAxMTE/kwXVNTU9y6dQt///03XFxcMH/+fCxfvhz169cvdDwTk/w7vIK7/EWLFpV4XnuuPcHcPbeQnStDHUtDmLlb4mnMf3NkFLTohIWF4cyZM4UeR7wuZR1RVR6tW7fG3r174e3tjYULF8LFxQU///wzRo0aJd8nNjYWM2bMQHx8PGxtbfHxxx9j/vz5WHfhMX48HorfzoQjOzJS3sIWJ83EggO3cejcVaQ8vo1GY5fh97Ft0MnNUqHsX3/9Fc+ePcPOnTuxdetWGBgYYOHChRgzZgwAwNpYD/1c9RG8eykSniYg3tAUNRp2h82gGXiakgVLo/zRS5EvlV0gJCQEFy5cwPHjxxGdlIELYU9xPuwZLj14jhdp2RCW3WDUMgbPj/0KWWYa9OwbwGr4QjxKysGjpMIjb14mkQAmetpIeqlfSmVN9LYzMAqXHjyHnrYGlg5tAg2NwiO9jPW0MbSFPf64/BibLj1Ce9fCw8PL4vDNfx9hNbKBVimPsArU0NXCoOa1sC0gEtsuR6J9EUPVlSGEwLKj+a1Zw1vZo7ZljRL3b+Foik87uGDDhQjM23sLx6d3LnNLTGpWLiZvv4bsPBl61rfGmA7OZTqOOpIIUYFth2+o5ORkmJiYQCqVqtTXgaqPuXPn4uLFizh//rzSeQYNGoSUlBSFzpyvevfdd2Ftba0wImbo0KHQ19fH1q1bIYSApaUlEhMTsWjRIgwfPhxnz57FuHHj4OXlhVWrVgHIfyxRMNPtrVu3MHXqVLRs2VKhf8nHH3+MWrVqwcfHB7l5MvT5dCZuZppDy9QW7RyN0FLjERbM/xKrV6/GZ599hpycHLz33nu4du0aDh06BGvr/zoLm5mZQUenYp7/qyNpRg46LjmNlKxcTO9ZF0Nb1sLp+wlYdjQEqVm50NKQYFzn2pjS3U3+qLCs0rJy8dPxUGy6FAGZyJ+sbV7f+hjeygHxKZmIeJYmn8I/JTMHlx++yA9wwp/h4dM0hWMZ6miiuWNNXAx/rjDUX0MC/DS8GWQyoTjFQGoWnqbk//tFWlaR860U2DGuHTxK6cirrFhpBjyXn0NKVi6+6lcfn3WqXey+4Qkp6Ln8HDQkgN+sbnAwK1+/kpw8GVp/fxJJ6TnY/llblQKoe7HJ6PPLeWhpSHBpbndYlWHSvzMhCRjjGwgdLQ34zeqqVACZkZ2HXj+fQ+SLdIxq64jvBzdWuVwhBKbtuo7912NgZ6KHf6Z2Qk0D9f8OUPb3my07pBZKm3PlVfHx8Th8+DA2b95c4nHbt2+PdevWITQ0FHXr1sWNGzdw4cIF+URqEREReP78OX755ResX79e3sLg5uYmX3cKKL6F4WUFd/lJ6dn4fEcwroTFIv3+LiDtBc4YGiDW3R1bt27FiBEjAORPZHjgwAEAQLNmzRSOdebMmUJD1+k/JvraaO1shtMhCVhxMhQrTobKtzV3rAmfIY3hblMxNz6Gulr4un8DDGpuB+89t3AnJhlz99zCWr+HePwif8I3CQBHMwM8ScpA3ksRiYYEaOpQE51cLdCpriWaOdSEtqYGdgVGFpq7ZHDzWiXWI08mkJiejXuxyfh445VC0w04mVdMq44QAl/tvY2UrFw0c6hZamdfVysjdHKzwPmwZ9h6+TG8+xZuEVXFpQfPkZSeA4saOmjjYlZ6hpfUtzVGKydTBD1OxM7AKEzpUfykn0WRyf5r1fmkvbPSLWX6OppYMrQxRq4PwLaASLzbxE7lwHN30BPsvx4DTQ0JVn7Q/K0IdFTBlh2wZUcdFAzrnjFjBoYNG4bAwEBMnToVa9aswejRowvtv2zZMixZsgQxMTEKQ8JfJZPJMG/ePCxbtgyamprIy8vD999/D29vbwDApUuX0KFDB8TExMDW9r/p6IcPHw6JRIJdu3YpfQ6x0gycD3uGn0+GIiYpE/ramvhxWFP0a1J4mnsqn6JmhQWALzzr4n9dXaFZxCOXipCbJ4PvxUf46XgIMosZ+u5sboCObhbo6GoJjzrmMNEv+pFGrDSjzHOX5AdLt5D30vmPauuIhQMblfvc91+PxtSd16GjqYFDUzqirnXpi4WevBuPz7YEwURfG5e9e5SrNW32XzfwZ9ATfNjOEd8NUr2FZF9wNKbtug5bEz2cn91N6cdgAHDgRgym7AiGka4Wzs3upvLoqnl7b2F7QCSczA1wdGpnpd+HsPgU9F91AZk5MszqVQ9e3VxLz6Qm2LJDbxWZTIZWrVph8eL86fubN2+O27dvFxvsbNy4EaNGjSox0AGAP//8E9u2bcP27dvRsGFDXL9+HdOmTYOdnV2Rxy2rLZceYcGBO/JHE6YG2tj2WTulVxYm1RS1CjUAtHQyq7RABwC0NDUwrnNtWBnpYuqu64W2r3y/GQY0K7mFpkB5+km9PN1A0OMXWH4iFNsCIvEsNQu/vN+8zAtiPkvNwjcH7gAAPu/uqlSgA+RPYuloZoDIF+nYdz0aH5QyIWZxsnNlOPbvWkzKjMIqSp/GNlh4SAex0kycvp8Az4bKLXyakyfDT8fzW3XGda5dpmHk3n3cceZ+Ah4/T8dPx0OUWok9IzsPXtuvITNHhk5uFvhflzoql/s24NBzUgvFzbkSGRlZaN/z588jJCREqSn7Z82ahblz5+L9999H48aN8dFHH2H69Onw8fEB8N+8J/HxiovdlTRjbYHcPBn8Qp9iwpYgfP1SoAPk9ykxNeRw0criYmGIV2Oa17lmT5vaZkWW31rFxy7lUTBs/vPubvhtZAvoaGrg2J14fLzhCqTpZZtYb8GBO0hMz0F9W2NM7Kr8j66mhgQf/zv0uzzD0C+GP4M0IweWRrpo7Vy291JXSxPD/10j64/Lj5XO92dQFB4/T4e5oQ7GdizbPD1GetpY/G9/nY0XI3AtMrHUPAsP3UFofCosauhi+fBmRXYEJwY7pCZUmXNlw4YNaNmyJZo2bVrqcdPT0wuNltHU1IRMlv8IomDek5c7OScnJyMgIKDIGWuFELj1RIqFB++inc9pjN54BcfuxhfaTyaAR8/SS60flU1Vzg9UHcp/Vd/Gttj8aRsY6WrhyqMXGL7WX6n5d1527E4cDt+MhaaGBD+816TUifxeNayVA/S1NXE/LgWXH75QKW+BQ/+OwurbyKZcLXSj2jpCIgHOhz3Do2dppe6fkZ2Hlf8uhzG5uysMdcv+0KSbe/6CuTIBzP7rJrJy84rd98CNGOy4EgWJBPjl/WbyUX5UGB9jkVqYPn062rdvj8WLF2P48OG4cuUK1q1bV2jSs+TkZOzevRs//fRTkcfp0aMHBg8ejMmTJwMA+vfvj++//x6Ojo5o2LAhgoODsXz5cnz66acA8icjmzZtGr777ju4ubnJh57b2dlh0KBB8uNGvUjHgRsx2BscjfCEVHm6qYE2urtbYU9wtEKH0apaGfhtUtVr9lR1+a/yqGOOPyd6YPTGKwiJT8HQ3y9hy9g2cLUq/VGUND0HX+27DQCY0Ll2sZPolcREXxtDWuQP/d586ZHKHXSzcvNw/G7pa2Epw8HMAF3qWuJsyFNsvxKJeaV0mt7s/wjxyVmoVVMfI9uW7RHcy75+twHOhz1FeEIqVp0Ox0zPwjMwP3qWhnl7bgEAJndzRYcKGravrhjskFpQZs4VANi5cyeEEPjggw+KPE5oWDiCQyMRK82ArYk+fv31V8yfPx+TJk1CQkIC7OzsMGHCBHz99dfyPLNnz0ZaWhrGjx+PpKQkNG7ZFlt270OWTBN7r0Ri77VoXHn0352qrpYG3mlgjcHNa6FzXUtoa2qgjYtZtVgZ+G1TFfMDVafyX1Xf1hh7JrXHxxuv4OHTNAxd7Y+Nn7RCS6eSHwl9d/gunqZkoY6locojmF42ur0ztgVE4vjdOEQnZaBWTeXfmwthz5CSmQsrI120cip5CQtlfNTOCWdDnuLPoCjMeKdusf2YpBk5WH32AQBg+jt1oatVvqkKAMDUUAcLBzbCpG3XsPrsA/RpZKvQfy8rNw+Td1xDalYu2jibYWo53vO3BUdjgaOxKN+uwEh477kFmcgf8uszpLFKC+a9nB/Ib53J+/e/l0QCeNQ2x6DmtdC7kQ2Mi5g0rDyja4gq0ou0bHy6KRDXo5Kgq6WBVSNb4J0GRS/66hf6FKM3XoFEAvw10aPUwKg0I9dfxqUHzzGxSx3M7eOudL4Zu65jT3A0PmnvjG8GlL4QbmnyZAKdl51BdFIGfhrWFENb2he53w/H7uO3Mw/gZlUDR6d1rtAO7hP/uIqjd+LQ0M4Y+7w6yB8NfnvwDnwvPoKpgTb+mdrprf6+4GgsIiVl5uTh6O04zP37lryTsEwAc/6+BZ9/7ivV4U8mEwqz0gJAnhCoY2mI4a0cMKCZXalfSNXtLp/eXmaGOtg+ri28tl3DmZCnmPBHEBYPboz3XxkllZqVK3+UMtrDudyBDpA/P82lB8+xMzAS03q6KTUyLDMnDyf+7fvWv2nFTNWgqSHByLaO+OFYCLYGPC4y2ElIycTGC48AAF/0qlfhI/kWDmoI/4fPcScmGevOPYRXN1ccvxMH34v5Zf40vCm/M5TEYIfeOjKZwN3YZJwPe4YL4U8R+CgR2cXMefJqAKOq7wY1rrBZaYleJwMdLaz7uBW899zCX1fzly15mpKFyd1d5SuXLzt6H9FJGXAw08fs3oX7lZRFj/rWsDfVx5PEDBy4HoPhrR1KzXMu9ClSsnJha6KH5g7lf4RVYHgrB/x8MhTBkUm4HS0t1Bfpt9PhyMjJQzOHmvAspuWrPKyM9PD1uw0wc/cN/HwyFJoaEvx2JhwA8FlHF3R3r/gy1RWDHVIrsdIMhen3C8QkZeBC2DOcC3sqX2PoZVZGukhIyVJI05AAW8e2VWqEw9OULHy4IUBh7hZ2MqY3nbamBn54rwmsjXXx25kH+OlEKOJTMjGxSx0cvxOPLf75Q7OXDGkCA52K+TnR1JDgo3ZO8DlyH76XHmFYK3t5cFWcw7f+HYXV2LZCh15bGumidyNbHLwRg20Bj+EzpIl8W9SLdGy/kj+1xeze9UqtY1kNaVELa889QGh8KpYcuQ8AsDfVx+zeyj/iIwY7pEZe7XPzYTsnSIBi1xjyqGOOjq4W6OhmiTqWhvgzKKpQJ2Fl19VxszaCz5DG7GRMakcikWBWL3dY1tDFt4fuYuvlSGy9/N/8Va2dTSt8JNCI1g5YcTIU92KTEfgoscRlHzJz8nDy30dYlTHb+IdtHXHwRgz2BcfAu299eX+7FSdCkZMn0MnNosyLhiojLjlTYQQnkH/z9jwti98vKmCwQ2ohVpqh0DlYJiC/6wSKX2PoZeUdClzdhhITVaRPOrhAU1OC+fvuKKRffZwoH71YUWoa6GBw81rYcSUKmy89KjHYORvyFGnZeahVUx/NHWpWWB0KtHExQ13rGgiNT8Xea9EY3d4Z9+OSsfd6NABgVq+KeXxXnKJm+y6Yh4vfMcpjsENqobjp/3vWt8J7LR1KXGPoZeXtJMxOxqTO6ljWKJRWWT+8o9s7Y8eVKBy9E4eYpAzYFTMM/b9HWDaV8ihJIpHgw3ZO+Hr/HWy9/Bgfezjhx2OhECK/zCb2NSu8zJcVzPbNR+TlwxmUSS24WBji1a85TYkEiwY1Qu9GNkoFOkRUste5zIa7jTHa1TZDnkxgW0DRyzZkZOfh1L3yrYWljMHNa8FARxNhCalY7fcAJ+/FQ0MCzHinclt1gOo32/abisEOqYVnKdl4OdrhFwJRxXvdP7yftHcGAOy4EoXMnMLLJpwNSUB6dh7sTfXRxF71WZuVZaSnjUHN8xdoXXY0f1maYS0d4GpVuKWrMoxo7YgLc7thx7h2uDC3m0rzf1E+PsaiClfciKjKkp0rw6y/bkAIoIe7FT7rVJt9Zogqyevsm9azvjXsTPQQI83EwRsxGNZKcRh6wVpY/ZrYVtpoqAKWNRRHZda2NKzU8l7FR+Tlw5YdNRQdHY0PP/wQ5ubm0NfXR+PGjREUFAQAyMnJwZw5c9C4cWMYGhrCzs4OH3/8MWJiYko85rlz59C/f3/Y2dlBIpFg3759CtsLjuvo6o5alqbo3Kwe6nboh9X/BFbWacqt8XuA+3EpMDXQxrL3msCjjjm/FIgqUcGK6ZX9/0xLUwMfeTgDADa9shp6enYuTt3/9xFW48p7hAXk38D9ejpMIW3Z0RCVF0ulqsNgR80kJiaiQ4cO0NbWxpEjR3D37l389NNPMDXNn2grPT0d165dw/z583Ht2jXs2bMHISEhGDBgQInHTUtLQ9OmTfHbb78VuT09PR3+V4KQ2WgQbEf/AstB85D9IhrTxo6s1C+E0PgU+ZfQNwMawrwGV/0lUifvt3aArpYG7sQk4+rjRHn66fsJyMyRwdHMAI1qVe4yP0UNgMgTAo+epVdquVRx+BhLzSxduhQODg7w9fWVp7m4uMj/bWJighMnTijkWbVqFdq0aYPIyEg4Ohb9LLhPnz7o06dPseWamJjggwVrsfTf59kAYPbORMRtmYHLN0MxuFPTsp5SsfJkArP+uomcPIGe9a0woGnl3t0R0etnaqiDgc3s8GfQE2y69AitnPOHoR9+jY+wOCLqzceWHTVz4MABtGrVCsOGDYOVlRWaN2+O9evXl5hHKpVCIpGgZs2aZSozKzcPCw/eVQh0AECWlQ5AAoMalXPXtfFCBG5EJcFIVwvfDWpc6V94RFQ1Rv/bUfno7TjESTORlpWL0/cTAADvVsJEgq/iiKg3H1t21MzDhw+xevVqzJgxA/PmzUNgYCCmTJkCHR0djB49utD+mZmZmDNnDj744IMyrfgenpCCz3dcx73YZABA+zrmuPzwOfJyspF01hcGDTpj4bEItHC1U2rZBWU9epaGH4/nB1df9qsPGxO9Cjs2EVUvDe1M0MbZDFcevcC2gMdwszZCVq4MLhaGaGBbuY+wCnDS0Dcbgx01I5PJ0KpVKyxevBgA0Lx5c9y+fRtr1qwpFOzk5ORg+PDhEEJg9erVKpUjhMD2gEgsPHQHmTkymBvq4IdhTdDd3RqRz5IxasRwGFoawnzELEQlZeCzLUHYNb6dUisYl36OAnP+vomsXBk6uJpjhBILBRLRm210e2dcefQC2wMi5cPM+zWu/EdYL+OIqDcXH2OpGVtbWzRo0EAhrX79+oiMjFRIKwh0Hj9+jBMnTqjcqrPG7yHm7b2FzBwZOrlZ4MjUTujubo2cnBxMHTcayc9icf7saWyZ2BU1DbRxIyoJM/68DllR0xyraPuVSAREvIC+tiaWDGnCx1dEbwHPhtawNdHD87RsnAl5CqBy1sIi9cRgR8106NABISGKfWdCQ0Ph5OQk/7sg0AkLC8PJkydhbm6u9PH9HzwHAARHJkJbU4Iv+9bH5jFtYGWsh5ycHPTv3x9nzpxBVFQU7O3tMaBbO0xrogFtTQn+uRWHZcfu4+uvv4atrS309fXRs2dPhIWFlVimj48PWrduDSMjI1hYWmHS6A+Q8/wJZveuBwez/A6CEyZMQJ06daCvrw9LS0sMHDgQ9+/fV/q8iKh609bUKDRx4I2opKqpDL1xGOyomenTp+Py5ctYvHgxwsPDsX37dqxbtw5eXl4A8gOd9957D0FBQdi2bRvy8vIQFxeHuLg4ZGdny4/To0cPrFq1Sv53ojQZ03/fh6HfbwcAmOQmwqeLEXo5a0FDQ4KcnBwMGDAAJ0+eRJcuXbBt2zb4+fnhyy+/RNt6tbBkSBMAwNKly/DTil+wZs0aBAQEwNDQEL169UJmZmax5+Tn5wcvLy/4+/vDY/Jy5ObmIHHPArzXxFK+T8uWLeHr64t79+7h2LFjEELA09MTeXmFZ10lojdPrDQDJ/5d3bzAl3tvc64bUo4gIZVKBQAhlUqruioV4uDBg6JRo0ZCV1dXuLu7i3Xr1sm3RURECABFvs6cOSPfz8nJSSxYsEAIIcTjZ2miw5SVReYZPXq00sf98eg9oWloKsy7fyouhj0VQgiRlJQkdHV1xY4dO0o9r7+CooTTnEPCZdoOAUD4+fkVu++NGzcEABEeHq7am0dE1dLF8KfCac6hQq9L4c+qumpUhZT9/WYHZTX07rvv4t133y1ym7Ozs8IspMXxv3EPEc/SsOliBH48HopU/dpovOAofIY0KfI5ubOzM+rXr49evXrhyZMn8PPzQ61atTBp0iR07doVADDIVRtfpCVCx7EZJm69ij2TOsDVygRt27aFv78/3n///WLrk5CSiYWH7gIAPm5liW8BmJmZFblvWloafH194eLiAgcHdl4mUgec64bKg4+xqJBdgZHosOQ0Rq4PwDcH7yI1KxetnU1xZFrnEjsEFgx7d3Nzw7Fjx/C///0PU6ZMwebNmwEA8fH5TdDN6jkhOTMXYzZdwfPULFhbWyMuLq7EOn297w6kGTloaFsDgduXo0OHDmjUqJHCPr///jtq1KiBGjVq4MiRIzhx4gR0dHTK+W4QUXXAuW6oPCRCmdv8l5w5cwbdunWrrPpUieTkZJiYmEAqlZZprhl1IU3PweFbMZi397ZCugTAudld4WBW8sJ3Ojo6aNWqFS5duiRPmzJlCgIDA+Hv749Lly6hQ4cOuB0Wgf/tiUDki3S0dDIFTq2AlqYGdu3aVeRx/7kVi0nbrkFLQ4JWT/7G5XOncOHCBdjb2yvWXypFQkICYmNj8eOPPyI6OhoXL16Enh7n4CFSF7HSDM51Q3LK/n6r/Bird+/esLe3x5gxYzB69Gg+JniDZefKcC0yERfCnuF8+DPcepJUaP0XIL/jzZPEzFKDneKGvf/9998AABsbGwBATmoSNn7SGoN/v4irjxORc+chhr7TochjJqZl4+v9+cGX9e1tuBh8DufOnSsU6AD5S1aYmJjAzc0N7dq1g6mpKfbu3YsPPvigtLeCiN4QnOuGykLlx1jR0dGYPHky/vrrL9SuXRu9evXCn3/+qTCSh6perDQDlx48UxipIIRAaHwKNlyIwBjfK2i28DjeX3cZq86E40ZUfqDjbG6AV2etUfa5eGnD3l1cXGBjY4NTp07B1aoG1n7YEho5GYgJu4XEGs5FHnPhobt4mpKFvAv/h0dXz+L06dMKa30VRwgBIQSysrJK3ZeIiNSbyi07FhYWmD59OqZPn45r167B19cXkyZNwqRJkzBy5EiMHTsWTZtW/KKPpLxdgZHw3nMLMgFoSIDhrRyQkydwIfwp4pMVf/wtauigg6sFOrpaoKObBWxN9LErMBLz9txGnhAqPRefPn062rdvj8WLF2P48OG4cuUK1q1bh3Xr1gEAJBIJpk2bhu+++w5ubm5wcXGBedB6RNUww7lsF/x99QmGtrRHjx49MHjwYDTsORx7g6OReGI1xIMLOHTgAIyMjOT9e0xMTKCvr4+HDx9i165d8PT0hKWlJZ48eYIlS5ZAX18fffv2rfg3mIiI3igq99l5VUxMDNatW4clS5ZAS0sLmZmZ8PDwwJo1a9CwYcOKqmelUqc+O7HSDHRYcrrIx1EAoKulgTYuZujoaoFObpZwtzGChkbhGYjL+lz80KFD8Pb2RlhYGFxcXDBjxgyMGzdOvl0IgQULFmDdunVISkpCx44d0ez9mfgrXAZtTQm2jm2LEd1b4oMPP8JZw66IlWbi8dKiR5b5+vrik08+QUxMDD777DNcvXoViYmJsLa2RufOnfH111+jXr16StediIjeLMr+fpcp2MnJycH+/fuxceNGnDhxAq1atcLYsWPxwQcf4OnTp/jqq69w7do13L17t1wnAeQ/NpszZw6OHDmC9PR0uLq6wtfXF61atQLw34/n+vXrkZSUhA4dOshHBClLnYKdSw+eYeT6gELp/ZvaYkQrR7RyNq2Q9akqkkwmMHnHNfxzKw41DbSx7qOWWOP3EKfvJ8DJ3ABHp3aGvk71qjMREVW9Suug/Pnnn2PHjh0QQuCjjz7CsmXLFIYAGxoa4scff4SdnV3Zav6SxMREdOjQAd26dcORI0dgaWmJsLAwmJqayvdZtmwZVq5cic2bN8PFxQXz589Hr169cPfu3bdyFE5xc1HM61u/2nbq09CQYPnwZohOuowbUUkYvvayfFuvBjYMdIiIqFxUbtnp0aMHPvvsMwwZMgS6urpF7pObm4uLFy+iS5cu5arc3LlzcfHiRZw/f77I7UII2NnZYebMmfjiiy8A5A8/tra2xqZNm0qcpO5l6tSyAwDTdgZj3/UYAP/NRTGitWMV16p0t6OlePfXCwppmhIJLsztVm0DNSIiqjrK/n6rPBrr1KlT+OCDD4oNdABAS0ur3IEOABw4cACtWrXCsGHDYGVlhebNm2P9+vXy7REREYiLi0PPnj3laSYm/83IW5ysrCwkJycrvNRJSmYuAGBEawdcmNvtjQh0ACA5M6dQWp4QePQsvQpqQ0RE6kLlYMfHxwcbN24slL5x40YsXbq0QipVoLQZeQtG5VhbWyvkK21GXh8fH/mcLCYmJmo1V1BWbh4u/bsy+UftnN6oFpGCR3Av43TwRERUXioHO2vXroW7u3uh9IYNG2LNmjUVUqkCMpkMLVq0wOLFi9G8eXOMHz8e48aNK3c53t7ekEql8ldUVFQF1bjqBUYkIiMnD5ZGumho92Y9kuN08EREVBlU7qAcFxcHW9vC6yNZWloiNja2QipVQNkZeePj4xXqFB8fj2bNmhV7XF1d3RIfw73JzoYkAAC61LWERFJ4SHl1N6K1IzrXteR08EREVGFUbtlxcHDAxYsXC6VfvHixQkZgvUyVGXkLJCcnIyAgAB4eHhValzfF2dCnAICu9SyruCZlZ2uiD4865gx0iIioQqjcsjNu3DhMmzYNOTk56N69O4D8TsuzZ8/GzJkzK7RyZZmRd/78+bCzs8OgQYMqtC5vgieJ6QhPSIWGBOjk+uYGO0RERBVJ5WBn1qxZeP78OSZNmiRfD0tPTw9z5syBt7d3hVaudevW2Lt3L7y9vbFw4UK4uLjg559/xqhRo+T7zJ49G2lpaRg/frx8Rt6jR4++lXPsnA3Jb9Vp4WgKEwPtKq4NERFR9VDm5SJSU1Nx79496Ovrw83N7Y3uA6Mu8+x8tjkIJ+/F4wvPupjcXfkZpImIiN5ElTaDcoEaNWqgdevWZc1OFSx/yPkzAEDXelZVXBsiIqLqo0zBTlBQEP78809ERkbKH2UV2LNnT4VUjFQT9CgR6dl5sKihiwa2b27rFBERUUVTeTTWzp070b59e9y7dw979+5FTk4O7ty5g9OnT8PExKQy6khKeHnIeVGrmBMREb2tVA52Fi9ejBUrVuDgwYPQ0dHBL7/8gvv372P48OFwdHwzliVQRwWdk7u8wUPOiYiIKoPKwc6DBw/Qr18/AICOjg7S0tIgkUgwffp0+ZBwer2ikzIQ9u+Q885uFlVdHSIiompF5WDH1NQUKSkpAIBatWrh9u3bAICkpCSkp3PBxqrg92+rTjOHmqhpoFPFtSEiIqpeVO6g3LlzZ5w4cQKNGzfGsGHDMHXqVJw+fRonTpxAjx49KqOOVIqC/jochUVERFSYysHOqlWrkJmZCQD48ssvoa2tjUuXLmHo0KH46quvKryCVLLsXBkuhhcMOWd/HSIiolepFOzk5ubi0KFD6NWrFwBAQ0MDc+fOrZSKkXKCHr9AWnYeLGrooJEdR8MRERG9SqU+O1paWpg4caK8ZYeqXkF/nc5uHHJORERUFJU7KLdp0wbXr1+vhKpQWXDIORERUclU7rMzadIkzJgxA1FRUWjZsiUMDQ0Vtjdp0qTCKkcli0nKQEh8yr9DzhnsEBERFUXlYOf9998HAEyZMkWeJpFIIISARCJBXl5exdWOSuQXmt+q09ShJkwNOeSciIioKCoHOxEREZVRDyqDl5eIICIioqKpHOw4OTlVRj1IRTl5MlwMfw6A8+sQERGVROVgZ8uWLSVu//jjj8tcGVLe1ceJSM3KhZmhDprU4pBzIiKi4qgc7EydOlXh75ycHKSnp0NHRwcGBgYMdl6Ts/Ih5xYcck5ERFQClYeeJyYmKrxSU1MREhKCjh07YseOHZVRRyoCl4ggIiJSjsrBTlHc3NywZMmSQq0+VDnipJm4H5cCiQTozM7JREREJaqQYAfIn105Jiamog5HJfALzW/VaWJfE2Ycck5ERFQilfvsHDhwQOFvIQRiY2OxatUqdOjQocIqRsUr6K/Tla06REREpVI52Bk0aJDC3xKJBJaWlujevTt++umniqoXFSMnT4YLYVzlnIiISFkqBzsymawy6kFKuvY4ESlZuTA10EYT+5pVXR0iIqJqr8L67NDrcfbfJSI6uVlCk0POiYiISqVysDN06FAsXbq0UPqyZcswbNiwCqkUFc+voL8OH2EREREpReVg59y5c+jbt2+h9D59+uDcuXMVUikqWkJyJu7GJgPgkHMiIiJlqRzspKamQken8HBnbW1tJCcnV0ilqGgFj7Ca2JvAooZuFdeGiIjozaBysNO4cWPs2rWrUPrOnTvRoEGDCqkUFc2PQ86JiIhUpvJorPnz52PIkCF48OABunfvDgA4deoUduzYgd27d1d4BSlfbp4M58Pyg50uXCKCiIhIaSoHO/3798e+ffuwePFi/PXXX9DX10eTJk1w8uRJdOnSpTLqSACCo5KQnJmLmgbaaOZQs6qrQ0RE9MZQOdgBgH79+qFfv34VXRcqQcHCnxxyTkREpBqV++wEBgYiICCgUHpAQACCgoIqpFLqZsmSJZBIJJg2bZpCur+/P7p37w5DQ0MYGxujc+fOyMjIKPIYZ0OeQur/J4589wmMjIxgZWWFQYMGISQkRGG/devWoWvXrjA2NoZEIkFSUlIlnRUREdGbQeVgx8vLC1FRUYXSo6Oj4eXlVSGVUieBgYFYu3YtmjRpopDu7++P3r17w9PTE1euXEFgYCAmT54MDY3CH0lCSibuxCQjM+o2pn4+GZcvX8aJEyeQk5MDT09PpKWlyfdNT09H7969MW/evEo/NyIiojeByo+x7t69ixYtWhRKb968Oe7evVshlVIXqampGDVqFNavX4/vvvtOYdv06dMxZcoUzJ07V55Wr169Io9TMAqrx/Sf8fnETvL0TZs2wcrKClevXkXnzp0BQN56dPbs2Qo8EyIiojeXyi07urq6iI+PL5QeGxsLLa0ydQFSW15eXujXrx969uypkJ6QkICAgABYWVmhffv2sLa2RpcuXXDhwoUij1Mwv07XuoqjsKRSKQDAzMysEmpPRESkHlQOdjw9PeHt7S3/oQWApKQkzJs3D++8806FVu5NtnPnTly7dg0+Pj6Ftj18+BAA8M0332DcuHE4evQoWrRogR49eiAsLExh39xiVjmXyWSYNm0aOnTogEaNGlXimRAREb3ZVG6K+fHHH9G5c2c4OTmhefPmAIDr16/D2toaf/zxR4VX8E0UFRWFqVOn4sSJE9DT0yu0vWDl+AkTJmDMmDEA8h8Dnjp1Chs3blQIkG48SYI0IwfGeloKQ869vLxw+/btYluDiIiIKJ/KwU6tWrVw8+ZNbNu2DTdu3IC+vj7GjBmDDz74ANra2pVRxzfO1atXkZCQoNC3KS8vD+fOncOqVavkI6henXG6fv36iIyMVEg7+29/nU51LaGlmd8QN3nyZBw6dAjnzp2Dvb19ZZ4KERHRG69MnWwMDQ0xfvz4iq6L2ujRowdu3bqlkDZmzBi4u7tjzpw5qF27Nuzs7AoNGw8NDUWfPn0U0s6+tESEEAKff/459u7di7Nnz8LFxaVyT4SIiEgNlLlH8d27dxEZGYns7GyF9AEDBpS7Um86IyOjQv1oDA0NYW5uLk+fNWsWFixYgKZNm6JZs2bYvHkz7t+/j7/++kuep3PXbritXQ/GLfujSz1LeHl5Yfv27di/fz+MjIwQFxcHADAxMYG+vj4AIC4uDnFxcQgPDwcA3Lp1C0ZGRnB0dGRHZiIieiupHOw8fPgQgwcPxq1btyCRSCCEAABIJPmz+ubl5VVsDdXUtGnTkJmZienTp+PFixdo2rQpTpw4gTp16sj3uR8aDlltWzS0M4aVkR5Wr14NAOjatavCsXx9ffHJJ58AANasWYNvv/1Wvq1gSPrL+xAREb1NJKIgWlFS//79oampif/7v/+Di4sLrly5gufPn2PmzJn48ccf0alTp9IPUs0kJyfDxMQEUqkUxsbGVV0duc82B+HkvXiM9nDCtwM54oqIiOhlyv5+qzz03N/fHwsXLoSFhQU0NDSgoaGBjh07wsfHB1OmTClXpek/O65E4uS9/PmMtlx+jF2BkaXkICIioqKoHOzk5eXByMgIAGBhYYGYmBgAgJOTU6EOt1Q2sdIMzNv7XwdnIYB5e24jVlr0ullERERUPJX77DRq1Ag3btyAi4sL2rZti2XLlkFHRwfr1q1D7dq1K6OOb52IZ2l49eFinhB49Cwdtib6VVMpIiKiN5TKwc5XX30lX3hy4cKFePfdd9GpUyeYm5tj165dFV7Bt5Hmv529X01ztjCogtoQERG92VQOdnr16iX/t6urK+7fv48XL17A1NRUPiKLymff9RiFvzUlEiwe0oitOkRERGVQISt3cv6WipOQkom/rz0BAKwe1QI1DXTgbGHAQIeIiKiMuEx5NbPp4iNk58rQwrEmejeyYWsZERFROak8GosqT2pWLv64/BgAMKFLHQY6REREFYDBTjWyIyASKZm5qG1piHfqW1d1dYiIiNSCysHOuXPnkJubWyg9NzcX586dq5BKvY2yc2XYcCECADChc21oaLBVh4iIqCKoHOx069YNL168KJQulUrRrVu3CqnU22j/9WjEJWfCykgXg5rXqurqEBERqQ2Vgx0hRJF9SZ4/fw5DQ8MKqdTbRiYTWHfuIQDg044u0NXSrOIaERERqQ+lR2MNGTIEQP7q5p988gl0dXXl2/Ly8nDz5k20b9++4mv4Fjh9PwFhCakw0tXCyLaOVV0dIiIitaJ0sGNiYgIgv2XHyMgI+vr/zfuio6ODdu3aYdy4cRVfw7fA2nMPAAAj2znCWE+7imtDRESkXpQOdnx9fQEAzs7O+OKLL/jIqoJcffwCgY8SoaOpgU87uFR1dYiIiNSOyn12Zs+erdBn5/Hjx/j5559x/PjxCq3Y22KNX35fncHNa8HaWK+Ka0NERKR+VA52Bg4ciC1btgAAkpKS0KZNG/z0008YOHAgVq9eXeEVVGfhCak4cTceEgkwrjNXjCciIqoMKgc7165dQ6dOnQAAf/31F2xsbPD48WNs2bIFK1eurPAKqrN1//bVeae+NVytalRxbYiIiNSTysFOeno6jIyMAADHjx/HkCFDoKGhgXbt2uHx48cVXkF1FZ+cib3B0QDyl4YgIiKiyqFysOPq6op9+/YhKioKx44dg6enJwAgISEBxsbGFV5BdbXxQgRy8gTaOJuhpZNpVVeHiIhIbakc7Hz99df44osv4OzsjDZt2sDDwwNAfitP8+bNK7yC6ig5MwfbAiIBABO6sK8OERFRZVI52HnvvfcQGRmJoKAgHDt2TJ7eo0cPrFixokIr96olS5ZAIpFg2rRp8rTMzEx4eXnB3NwcNWrUwNChQxEfH1+p9SivbZcjkZqVCzerGuhWz6qqq0NERKTWyrTquY2NDYyMjHDixAlkZGQAAFq3bg13d/cKrdzLAgMDsXbtWjRp0kQhffr06Th48CB2794NPz8/xMTEyGd7ro6ycvOw8eK/C352qcMFP4mIiCqZysHO8+fP0aNHD9StWxd9+/ZFbGwsAGDs2LGYOXNmhVcQAFJTUzFq1CisX78epqb/9W+RSqXYsGEDli9fju7du6Nly5bw9fXFpUuXcPny5UqpS3ntC47G05Qs2JroYUBTu6quDhERkdpTOdiZPn06tLW1ERkZCQMDA3n6iBEjcPTo0QqtXAEvLy/069cPPXv2VEi/evUqcnJyFNLd3d3h6OgIf3//Yo+XlZWF5ORkhdfrIJMJrP13wc+xHV2go1WmhjUiIiJSgdLLRRQ4fvw4jh07Bnt7e4V0Nze3Shl6vnPnTly7dg2BgYGFtsXFxUFHRwc1a9ZUSLe2tkZcXFyxx/Tx8cG3335b0VUt1Yl78Xj4NA3Gelp4vw0X/CQiInodVG5aSEtLU2jRKfDixQuFldArQlRUFKZOnYpt27ZBT6/illLw9vaGVCqVv6Kioirs2MURQmCNX/4kgh95OKGGrspxJhEREZWBysFOp06d5MtFAIBEIoFMJsOyZcvQrVu3Cq3c1atXkZCQgBYtWkBLSwtaWlrw8/PDypUroaWlBWtra2RnZyMpKUkhX3x8PGxsbIo9rq6uLoyNjRVelS3wUSKCI5Ogo6WBT9pzwU8iIqLXReXmhWXLlqFHjx4ICgpCdnY2Zs+ejTt37uDFixe4ePFihVauR48euHXrlkLamDFj4O7ujjlz5sDBwQHa2to4deoUhg4dCgAICQlBZGSkfP6f6qKgVee9lvawNKrYFjAiIiIqnsrBTqNGjRAaGopVq1bByMgIqampGDJkCLy8vGBra1uhlTMyMkKjRo0U0gwNDWFubi5PHzt2LGbMmAEzMzMYGxvj888/h4eHB9q1a1ehdSmPkLgUnL6fkL/gZydOIkhERPQ6qRzsREZGwsHBAV9++WWR2xwdX2/H2xUrVkBDQwNDhw5FVlYWevXqhd9///211qE0a/9d8LN3Qxu4WBhWcW2IiIjeLhIhhFAlg6amJmJjY2FlpTjz7/Pnz2FlZYW8vLwKreDrkJycDBMTE0il0grvvxOTlIHOy84gVyaw36sDmjrUrNDjExERva2U/f1WuYOyEAISSeFZf1NTUyt0xJS62HghArkygXa1zRjoEBERVQGlH2PNmDEDQP7oq/nz5ysMP8/Ly0NAQACaNWtW4RV8k4XGpWDr5fy5hyZ2qVPFtSEiIno7KR3sBAcHA8hv2bl16xZ0dHTk23R0dNC0aVN88cUXFV/DN9SuwEjM/fsWCp4Rxkkzq7Q+REREbyulg50zZ84AyB/6/csvv7yWuWneVLHSDHjv+S/QAYAv995Gl3qWsDXRr7J6ERERvY1U7rPj6+vLQKcUEc/SIHul23eeEHj0LL1qKkRERPQW40qUlcDFwhAar/Th1pRI4GxReJkNIiIiqlwMdiqBrYk+fIY0hua/o9Y0JRIsHtKIj7CIiIiqAFejrCQjWjuic11LPHqWDmcLAwY6REREVYTBTiWyNdFnkENERFTF+BiLiIiI1BqDHSIiIlJrDHaIiIhIrTHYISIiIrXGYIeIiIjUGoMdIiIiUmsMdoiIiEitMdghIiIitcZgh4iIiNQagx0iIiJSawx2iIiISK0x2CEiIiK1xmCHiIiI1BqDHSIiIlJrDHaIiIhIrTHYISIiIrXGYIeIiIjUGoMdIiIiUmsMdoiIiEitMdghIiIitcZgh4iIiNQagx0iIiJSawx2iIiISK0x2CEiIiK1xmCHiIiI1BqDHSIiIlJrDHaIiIhIrTHYISIiIrXGYIeIiIjUGoMdIiIiUmsMdoiIiEitMdghIiIitcZgh4iIiNQagx0iIiJSawx2iIiISK0x2CEiIiK1xmCHiIiI1BqDHSIiIlJrDHaIiIhIrTHYISIiIrXGYIeIiIjUGoMdIiIiUmsMdoiIiEitMdghIiIitcZgh4iIiNQagx0iIiJSawx2iIiISK0x2CEiIiK1xmCHiIiI1BqDHSIiIlJrDHaIiIhIrTHYISIiIrXGYIeIiIjUGoMdIiIiUmvVOtjx8fFB69atYWRkBCsrKwwaNAghISEK+2RmZsLLywvm5uaoUaMGhg4divj4+CqqMREREVU31TrY8fPzg5eXFy5fvowTJ04gJycHnp6eSEtLk+8zffp0HDx4ELt374afnx9iYmIwZMiQKqw1ERERVScSIYSo6koo6+nTp7CysoKfnx86d+4MqVQKS0tLbN++He+99x4A4P79+6hfvz78/f3Rrl07pY6bnJwMExMTSKVSGBsbV+YpEBERUQVR9ve7WrfsvEoqlQIAzMzMAABXr15FTk4OevbsKd/H3d0djo6O8Pf3L/Y4WVlZSE5OVngRERGRenpjgh2ZTIZp06ahQ4cOaNSoEQAgLi4OOjo6qFmzpsK+1tbWiIuLK/ZYPj4+MDExkb8cHBwqs+pERERUhd6YYMfLywu3b9/Gzp07y30sb29vSKVS+SsqKqoCakhERETVkVZVV0AZkydPxqFDh3Du3DnY29vL021sbJCdnY2kpCSF1p34+HjY2NgUezxdXV3o6upWZpWJiIiomqjWLTtCCEyePBl79+7F6dOn4eLiorC9ZcuW0NbWxqlTp+RpISEhiIyMhIeHx+uuLhEREVVD1bplx8vLC9u3b8f+/fthZGQk74djYmICfX19mJiYYOzYsZgxYwbMzMxgbGyMzz//HB4eHkqPxCIiIiL1Vq2HnkskkiLTfX198cknnwDIn1Rw5syZ2LFjB7KystCrVy/8/vvvJT7GehWHnhMREb15lP39rtbBzuvCYIeIiOjNo5bz7BARERGpisEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTW1CbY+e233+Ds7Aw9PT20bdsWV65cqeoqERERUTWgFsHOrl27MGPGDCxYsADXrl1D06ZN0atXLyQkJFR11YiIiKiKqUWws3z5cowbNw5jxoxBgwYNsGbNGhgYGGDjxo1VXTUiIiKqYlpVXYHyys7OxtWrV+Ht7S1P09DQQM+ePeHv719knqysLGRlZcn/lkqlAIDk5OTKrSwRERFVmILfbSFEifu98cHOs2fPkJeXB2tra4V0a2tr3L9/v8g8Pj4++PbbbwulOzg4VEodiYiIqPKkpKTAxMSk2O1vfLBTFt7e3pgxY4b8b5lMhhcvXsDc3BwSiaTCyklOToaDgwOioqJgbGz8WvOz7Ndfdnnzs+y3q+zy5mfZLPtNyV/esksihEBKSgrs7OxK3O+ND3YsLCygqamJ+Ph4hfT4+HjY2NgUmUdXVxe6uroKaTVr1qysKsLY2LhcH3B58rPs1192efOz7Ler7PLmZ9ks+03JX96yi1NSi06BN76Dso6ODlq2bIlTp07J02QyGU6dOgUPD48qrBkRERFVB298yw4AzJgxA6NHj0arVq3Qpk0b/Pzzz0hLS8OYMWOqumpERERUxdQi2BkxYgSePn2Kr7/+GnFxcWjWrBmOHj1aqNPy66arq4sFCxYUemT2OvKz7Ndfdnnzs+y3q+zy5mfZLPtNyV/esiuCRJQ2XouIiIjoDfbG99khIiIiKgmDHSIiIlJrDHaIiIhIrTHYISIiIrXGYKcS/fbbb3B2doaenh7atm2LK1euKJXv3Llz6N+/P+zs7CCRSLBv3z6ly/Tx8UHr1q1hZGQEKysrDBo0CCEhIUrnX716NZo0aSKf/MnDwwNHjhxROv/LlixZAolEgmnTpim1/zfffAOJRKLwcnd3V7q86OhofPjhhzA3N4e+vj4aN26MoKAgpfI6OzsXKlsikcDLy6vUvHl5eZg/fz5cXFygr6+POnXqYNGiRaWu1fKylJQUTJs2DU5OTtDX10f79u0RGBhYaL/Srg0hBL7++mvY2tpCX18fPXv2RFhYmNL59+zZA09PT/ls4tevX1e6/JycHMyZMweNGzeGoaEh7Ozs8PHHHyMmJkapsr/55hu4u7vD0NAQpqam6NmzJwICApSu+8smTpwIiUSCn3/+Wam8n3zySaHPvnfv3iqVfe/ePQwYMAAmJiYwNDRE69atERkZWWreoq47iUSCH374QamyU1NTMXnyZNjb20NfX1++GLIyeePj4/HJJ5/Azs4OBgYG6N27t/x6Uea7JDMzE15eXjA3N0eNGjUwdOhQ+QSvyuRft24dunbtCmNjY0gkEiQlJcm3lZb/xYsX+Pzzz1GvXj3o6+vD0dERU6ZMgVQqVarsCRMmoE6dOtDX14elpSUGDhwoX2JIle9RIQT69Okjf3+Vydu1a9dCn/fEiRNVKtvf3x/du3eHoaEhjI2N0blzZyxcuLDEvI8ePSr2etu9e7dSZcfFxeGjjz6CjY0NDA0N0aJFC/z9999K5X3w4AEGDx4MS0tLGBsbY/jw4YUmBK4sDHYqya5duzBjxgwsWLAA165dQ9OmTdGrVy8kJCSUmjctLQ1NmzbFb7/9pnK5fn5+8PLywuXLl3HixAnk5OTA09MTaWlpSuW3t7fHkiVLcPXqVQQFBaF79+4YOHAg7ty5o1I9AgMDsXbtWjRp0kSlfA0bNkRsbKz8deHCBaXyJSYmokOHDtDW1saRI0dw9+5d/PTTTzA1NVW6vi+Xe+LECQDAsGHDSs27dOlSrF69GqtWrcK9e/ewdOlSLFu2DL/++qtSZQPAZ599hhMnTuCPP/7ArVu34OnpiZ49eyI6Olphv9KujWXLlmHlypVYs2YNAgICYGhoiF69eiEzM1Op/GlpaejYsSOWLl1a7Pbi8qenp+PatWuYP38+rl27hj179iAkJAQDBgxQquy6deti1apVuHXrFi5cuABnZ2d4enri6dOnSuUvsHfvXly+fFlh+nhl8vbu3VvhGtixY4fS+R88eICOHTvC3d0dZ8+exc2bNzF//nzo6emVmvflMmNjY7Fx40ZIJBIMHTpUqbJnzJiBo0ePYuvWrbh37x6mTZuGyZMn48CBAyXmFUJg0KBBePjwIfbv34/g4GA4OTmhZ8+eSEtLU+q7ZPr06Th48CB2794NPz8/xMTEYMiQIQCU+y5KT09H7969MW/evEL1Ky1/TEwMYmJi8OOPP+L27dvYtGkTjh49irFjxypVdsuWLeHr64t79+7h2LFjEELA09MTeXl5Kn2P/vzzzwrLDCmbd9y4cQqf+7Jly5TO7+/vj969e8PT0xNXrlxBYGAgJk+ejAsXLpSY18HBodD19u2336JGjRro06ePUmV//PHHCAkJwYEDB3Dr1i0MGTIEw4cPx8GDB0vMm5aWBk9PT0gkEpw+fRoXL15EdnY2+vfvD5lMVuh9rXCCKkWbNm2El5eX/O+8vDxhZ2cnfHx8VDoOALF3794y1yMhIUEAEH5+fmU+hqmpqfi///s/pfdPSUkRbm5u4sSJE6JLly5i6tSpSuVbsGCBaNq0aZnqOGfOHNGxY8cy5S3K1KlTRZ06dYRMJit13379+olPP/1UIW3IkCFi1KhRSpWVnp4uNDU1xaFDhxTSW7RoIb788sti8716bchkMmFjYyN++OEHeVpSUpLQ1dUVO3bsKDX/yyIiIgQAERwcrHT5Rbly5YoAIB4/fqxyXqlUKgCIkydPKl32kydPRK1atcTt27eFk5OTWLFihVJ5R48eLQYOHFhifUrKP2LECPHhhx+WKe+rBg4cKLp37650/oYNG4qFCxcqpBV17byaNyQkRAAQt2/flqfl5eUJS0tLsX79+kJlv/pdkpSUJLS1tcXu3bvl+9y7d08AEP7+/qXmf9mZM2cEAJGYmFjkeZeWv8Cff/4pdHR0RE5Ojsp5b9y4IQCI8PBwpcsODg4WtWrVErGxscV+tkXlVeV7saj8bdu2FV999VWZ8r6qWbNmhb6/SspvaGgotmzZorCfmZlZoWvm1bzHjh0TGhoaQiqVyvdJSkoSEolEnDhxotRzKS+27FSC7OxsXL16FT179pSnaWhooGfPnvD393+tdZFKpQAAMzMzlfPm5eVh586dSEtLU2npDS8vL/Tr10/h/JUVFhYGOzs71K5dG6NGjUJkZKRS+Q4cOIBWrVph2LBhsLKyQvPmzbF+/XqVywfyP7+tW7fi008/VWph2Pbt2+PUqVMIDQ0FANy4cQMXLlxAnz59lCovNzcXeXl50NPTU0jX19dXumULACIiIhAXF6fwvpuYmKBt27av/borIJVKIZFIVF57Ljs7G+vWrYOJiQmaNm2qVB6ZTIaPPvoIs2bNQsOGDVWu69mzZ2FlZYV69erhf//7H54/f650uYcPH0bdunXRq1cvWFlZoW3btio9fi4QHx+Pw4cPY+zYsUrnad++PQ4cOIDo6GgIIXDmzBmEhobC09OzxHxZWVkAoHDdaWhoQFdXt8jr7tXvkqtXryInJ0fhenN3d4ejo2OR11t5vouUzS+VSmFsbAwtLa1C6SXlTUtLg6+vL1xcXODg4KBU2enp6Rg5ciR+++23YtdhLKnsbdu2wcLCAo0aNYK3tzfS09OVyp+QkICAgABYWVmhffv2sLa2RpcuXZT6zF519epVXL9+vdjrraj87du3x65du/DixQvIZDLs3LkTmZmZ6Nq1a4l5s7KyIJFIFCYW1NPTg4aGhkrfc2VW6eHUWyg6OloAEJcuXVJInzVrlmjTpo1Kx0I5Wnby8vJEv379RIcOHVTKd/PmTWFoaCg0NTWFiYmJOHz4sNJ5d+zYIRo1aiQyMjKEEKrdwfzzzz/izz//FDdu3BBHjx4VHh4ewtHRUSQnJ5eaV1dXV+jq6gpvb29x7do1sXbtWqGnpyc2bdqkdN0L7Nq1S2hqaoro6Gil9s/LyxNz5swREolEaGlpCYlEIhYvXqxSmR4eHqJLly4iOjpa5Obmij/++ENoaGiIunXrFpvn1Wvj4sWLAoCIiYlR2G/YsGFi+PDhpeZ/WUW07GRkZIgWLVqIkSNHKp334MGDwtDQUEgkEmFnZyeuXLmidNmLFy8W77zzjrw1TpWWnR07doj9+/eLmzdvir1794r69euL1q1bi9zc3FLzF9zVGxgYiOXLl4vg4GDh4+MjJBKJOHv2rFLnXWDp0qXC1NRU/v9HmbpnZmaKjz/+WAAQWlpaQkdHR2zevLnUvNnZ2cLR0VEMGzZMvHjxQmRlZYklS5YIAMLT01Mhb1HfJdu2bRM6OjqFymndurWYPXt2qflfVlrLjjLfZU+fPhWOjo5i3rx5Suf97bffhKGhoQAg6tWrV2SrTnH5x48fL8aOHSv/u6jPpri8a9euFUePHhU3b94UW7duFbVq1RKDBw9Wqmx/f38BQJiZmYmNGzeKa9euiWnTpgkdHR0RGhqq1HkX+N///ifq169f5Lbi8icmJgpPT0/59WZsbCyOHTtWat6EhARhbGwspk6dKtLS0kRqaqqYPHmyACDGjx9fbB0rCoOdSlBdgp2JEycKJycnERUVpVK+rKwsERYWJoKCgsTcuXOFhYWFuHPnTqn5IiMjhZWVlbhx44Y8TZVg51WJiYnC2NhYqUdo2trawsPDQyHt888/F+3atVO5XE9PT/Huu+8qvf+OHTuEvb292LFjh7h586bYsmWLMDMzUynQCg8PF507dxYAhKampmjdurUYNWqUcHd3LzZPdQ52srOzRf/+/UXz5s0Vmq1Ly5uamirCwsKEv7+/+PTTT4Wzs7OIj48vNX9QUJCwtrZWCFBVCXZe9eDBA6UfoRX8f//ggw8U9uvfv794//33VSq7Xr16YvLkycVuLyr/Dz/8IOrWrSsOHDggbty4IX799VdRo0aNQo8GisobFBQkmjZtKr/uevXqJfr06SN69+6tsF9R3yWqBDulfReVFuyUll8qlYo2bdqI3r17i+zsbKXzJiUlidDQUOHn5yf69+8vWrRoUSjQLCr//v37haurq0hJSZGnFfX+KvsdfOrUqSIfoRWVv+D/ube3t8K+jRs3FnPnzlW67PT0dGFiYiJ+/PHHIrcXl3/y5MmiTZs24uTJk+L69evim2++ESYmJuLmzZul5j127JioXbu2kEgkQlNTU3z44YeiRYsWYuLEiSW8OxWDwU4lyMrKEpqamoUu/I8//lgMGDBApWOVNdjx8vIS9vb24uHDhyrnfVWPHj2Uirz37t0r/9IseAGQX9hF3SWXplWrVgr/gYvj6OiocJclhBC///67sLOzU6m8R48eCQ0NDbFv3z6l89jb24tVq1YppC1atEjUq1dPpbKFyP+xLwhWhg8fLvr27Vvsvq9eGwU/0K8GKJ07dxZTpkwpNf/LyhPsZGdni0GDBokmTZqIZ8+eqZT3Va6urkW2kr2af8WKFfLr7OVrT0NDQzg5OZWpbAsLC7FmzZpSy87KyhJaWlpi0aJFCvvNnj1btG/fXumyz507JwCI69evF1unV/Onp6cLbW3tQv29xo4dK3r16qV02UlJSSIhIUEIkd/fcNKkSfJtxX2XFPxAvxqgODo6iuXLl5ea/2UlBTul5U9OThYeHh6iR48ehQIVVb4Hs7KyhIGBgdi+fXup+adOnVrs9dalSxeVy05NTRUAxNGjR0st++HDhwKA+OOPPxTShw8fLm9FVabsLVu2CG1tbfnn/rLi8oeHhxfq5yVE/m/EhAkTlC776dOn8s/a2tpaLFu2rNh9Kwr77FQCHR0dtGzZEqdOnZKnyWQynDp1SqW+L2UhhMDkyZOxd+9enD59Gi4uLuU+pkwmkz/fL0mPHj1w69YtXL9+Xf5q1aoVRo0ahevXr0NTU1OlclNTU/HgwQPY2tqWum+HDh0KDXMMDQ2Fk5OTSmX6+vrCysoK/fr1UzpPeno6NDQU/ytpamqWaYSBoaEhbG1tkZiYiGPHjmHgwIFK53VxcYGNjY3CdZecnIyAgIBKv+4K5OTkYPjw4QgLC8PJkydhbm5eruMpe+199NFHuHnzpsK1Z2dnh1mzZuHYsWMql/vkyRM8f/5cqWtPR0cHrVu3Lvf1t2HDBrRs2VLpPkpA/vudk5NT7uvPxMQElpaWCAsLQ1BQEAYOHFjqd0nLli2hra2tcL2FhIQgMjISHh4e5f4uUiZ/cnIyPD09oaOjgwMHDsj7H5WlbJF/84+srKxS88+dO7fQ9QYAK1aswMaNG1UuuyC/ra1tqWU7OzvDzs6uyOvN0dFR6bI3bNiAAQMGwNLSUuE9KCl/Qb+ioq63vLw8pcu2sLBAzZo1cfr0aSQkJMhHbFaqSg+n3lI7d+4Uurq6YtOmTeLu3bti/PjxombNmiIuLq7UvCkpKSI4OFgEBwcLAPJ+AK+OaCnK//73P2FiYiLOnj0rYmNj5a/09HSl6j137lzh5+cnIiIixM2bN8XcuXOFRCIRx48fVyr/q1R5jDVz5kxx9uxZERERIS5evCh69uwpLCwsirzzeNWVK1eElpaW+P7770VYWJjYtm2bMDAwEFu3blW6rnl5ecLR0VHMmTNH6TxC5I/kqVWrljh06JCIiIgQe/bsERYWFoWa8kty9OhRceTIEfHw4UNx/Phx0bRpU9G2bdtCTfKlXRtLliwRNWvWlPc/GThwoHBxcZHf8ZaW//nz5yI4OFgcPnxYABA7d+4UwcHBIjY2ttT82dnZYsCAAcLe3l5cv35d4frLysoqMW9qaqrw9vYW/v7+4tGjRyIoKEiMGTNG6Orqyu8iVf1/8fJjrJLypqSkiC+++EL4+/uLiIgIcfLkSdGiRQvh5uYmMjMzlSp7z549QltbW6xbt06EhYWJX3/9VWhqaorz588rVW+pVCoMDAzE6tWrC51Hafm7dOkiGjZsKM6cOSMePnwofH19hZ6envj9999Lzfvnn3+KM2fOiAcPHoh9+/YJJycnMWTIECGEct8lEydOFI6OjuL06dMiKChIeHh4yB8nK5M/NjZWBAcHi/Xr1wsA4ty5cyI4OFg8f/681PxSqVS0bdtWNG7cWISHhyvsM3HixBLzPnjwQCxevFgEBQWJx48fi4sXL4r+/fsLMzMzER8fX6bvUfzbclZa3vDwcLFw4UIRFBQkIiIixP79+0Xt2rVF586dlX7fVqxYIYyNjcXu3btFWFiY+Oqrr4Senp4YOXKkUvUOCwsTEolEHDlyRCG9tLKzs7OFq6ur6NSpkwgICBDh4eHixx9/FBKJRPTt27fUsjdu3Cj8/f1FeHi4+OOPP4SZmZmYMWNGse9pRWKwU4l+/fVX4ejoKHR0dESbNm3E5cuXlcpX0KT76mv06NGl5i0qHwDh6+urVNmffvqpcHJyEjo6OsLS0lL06NGjzIGOEKoFOyNGjBC2trZCR0dH1KpVS4wYMaLIDoPFOXjwoGjUqJHQ1dUV7u7uYt26dSrV9dixYwKACAkJUSlfcnKymDp1qnB0dBR6enqidu3a4ssvvxRZWVlKH2PXrl2idu3aQkdHR9jY2AgvLy+RlJRUaL/Srg2ZTCbmz58vrK2tha6urujRo4fC+ZSW39fXt8jtCxYsKDV/waOvol5nzpwpMW9GRoYYPHiwsLOzEzo6OsLW1lYMGDBAoYOyqv8vXg52Ssqbnp4uPD09haWlpdDW1hZOTk5i3LhxCjcmypS9YcMG4erqKvT09ETTpk3lj0KVybt27Vqhr69fps88NjZWfPLJJ8LOzk7o6emJevXqiZ9++knIZLJS8/7yyy/C3t5eaGtrC0dHR/HVV1/Jr1tlvksyMjLEpEmThKmpqTAwMBCDBw+WB8bK5F+wYEGx+5SWv7hzK+lVkDc6Olr06dNHWFlZCW1tbWFvby9Gjhwp7t+/r3TdX1UQ7JSWNzIyUnTu3FmYmZkJXV1d4erqKmbNmiXv26Zs2T4+PsLe3l4YGBgIDw8Pcf78eaXzent7CwcHB5GXl1foHErLHxoaKoYMGSKsrKyEgYGBaNKkidiyZYtSeefMmSOsra2Ftra2cHNzk1+nr4Pk3xMkIiIiUkvss0NERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BARveLs2bOQSCRISkqq6qoQUQVgsENERERqjcEOERERqTUGO0RU7chkMvj4+MDFxQX6+vpo2rQp/vrrLwD/PWI6fPgwmjRpAj09PbRr1w63b99WOMbff/+Nhg0bQldXF87Ozvjpp58UtmdlZWHOnDlwcHCArq4uXF1dsWHDBoV9rl69ilatWsHAwADt27cvtNI0Eb0ZGOwQUbXj4+ODLVu2YM2aNbhz5w6mT5+ODz/8EH5+fvJ9Zs2ahZ9++gmBgYGwtLRE//79kZOTAyA/SBk+fDjef/993Lp1C9988w3mz5+PTZs2yfN//PHH2LFjB1auXIl79+5h7dq1qFGjhkI9vvzyS/z0008ICgqClpYWPv3009dy/kRUsbgQKBFVK1lZWTAzM8PJkyfh4eEhT//ss8+Qnp6O8ePHo1u3bti5cydGjBgBAHjx4gXs7e2xadMmDB8+HKNGjcLTp09x/Phxef7Zs2fj8OHDuHPnDkJDQ1GvXj2cOHECPXv2LFSHs2fPolu3bjh58iR69OgBAPjnn3/Qr18/ZGRkQE9Pr5LfBSKqSGzZIaJqJTw8HOnp6XjnnXdQo0YN+WvLli148OCBfL+XAyEzMzPUq1cP9+7dAwDcu3cPHTp0UDhuhw4dEBYWhry8PFy/fh2ampro0qVLiXVp0qSJ/N+2trYAgISEhHKfIxG9XlpVXQEiopelpqYCAA4fPoxatWopbNPV1VUIeMpKX19fqf20tbXl/5ZIJADy+xMR0ZuFLTtEVK00aNAAurq6iIyMhKurq8LLwcFBvt/ly5fl/05MTERoaCjq168PAKhfvz4uXryocNyLFy+ibt260NTUROPGjSGTyRT6ABGR+mLLDhFVK0ZGRvjiiy8wffp0yGQydOzYEVKpFBcvXoSxsTGcnJwAAAsXLoS5uTmsra3x5ZdfwsLCAoMGDQIAzJw5E61bt8aiRYswYsQI+Pv7Y9WqVfj9998BAM7Ozhg9ejQ+/fRTrFy5Ek2bNsXjx4+RkJCA4cOHV9WpE1ElYbBDRNXOokWLYGlpCR8fHzx8+BA1a9ZEixYtMG/ePPljpCVLlmDq1KkICwtDs2bNcPDgQejo6AAAWrRogT///BNff/01Fi1aBFtbWyxcuBCffPKJvIzVq1dj3rx5mDRpEp4/fw5HR0fMmzevKk6XiCoZR2MR0RulYKRUYmIiatasWdXVIaI3APvsEBERkVpjsENERERqjY+xiIiISK2xZYeIiIjUGoMdIiIiUmsMdoiIiEitMdghIiIitcZgh4iIiNQagx0iIiJSawx2iIiISK0x2CEiIiK1xmCHiIiI1Nr/A45Ofu2hDDlOAAAAAElFTkSuQmCC", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "plt.plot(np.arange(len(epochs_x)), epochs_acc, marker = '.')\n", - "plt.xlabel('epoch')\n", - "plt.ylabel('test accuracy')\n", - "plt.ylim(0, 100)\n", - "plt.xticks(np.arange(len(epochs_x)))\n", - "for i, txt in enumerate(epochs_acc):\n", - " if i%2 == 0:\n", - " pass\n", - " else:\n", - " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", - "plt.show()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "speck-rescnn", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/tests/test_nonsequential/exp_set_A/baseline-SCNN-example_3.ipynb b/tests/test_nonsequential/exp_set_A/baseline-SCNN-example_3.ipynb deleted file mode 100644 index dc53313a..00000000 --- a/tests/test_nonsequential/exp_set_A/baseline-SCNN-example_3.ipynb +++ /dev/null @@ -1,1500 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import torch, random\n", - "import torch.nn as nn\n", - "import sinabs.layers as sl\n", - "from tqdm.notebook import tqdm\n", - "\n", - "from tonic.datasets.dvsgesture import DVSGesture\n", - "from tonic.transforms import ToFrame\n", - "from torch.utils.data import DataLoader\n", - "from torch.nn import CrossEntropyLoss\n", - "from torch.optim import Adam\n", - "\n", - "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential\n", - "\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "torch.backends.cudnn.deterministic = True\n", - "random.seed(1)\n", - "torch.manual_seed(1)\n", - "torch.cuda.manual_seed(1)\n", - "np.random.seed(1)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Loading Data" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "batch_size = 3\n", - "num_workers = 1\n", - "epochs = 30\n", - "lr = 1e-3" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "root_dir = \"./DVSGESTURE\"\n", - "_ = DVSGesture(save_to=root_dir, train=True)\n", - "_ = DVSGesture(save_to=root_dir, train=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "n_time_steps = 50\n", - "to_raster = ToFrame(sensor_size=DVSGesture.sensor_size, n_time_bins=n_time_steps)\n", - "\n", - "snn_train_dataset = DVSGesture(save_to=root_dir, train=True, transform=to_raster)\n", - "snn_test_dataset = DVSGesture(save_to=root_dir, train=False, transform=to_raster)" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The transformed array is in shape [Time-Step, Channel, Height, Width] --> (50, 2, 128, 128)\n" - ] - } - ], - "source": [ - "sample_data, label = snn_train_dataset[0]\n", - "print(f\"The transformed array is in shape [Time-Step, Channel, Height, Width] --> {sample_data.shape}\")" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "snn_train_dataloader = DataLoader(snn_train_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True)\n", - "snn_test_dataloader = DataLoader(snn_test_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=False)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Network Module\n", - "\n", - "We need to define a `nn.Module` implementing the network we want the chip to reproduce." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "device: NVIDIA RTX A4000\n" - ] - } - ], - "source": [ - "if torch.cuda.is_available():\n", - " device = torch.device('cuda:0')\n", - " print('device: ', torch.cuda.get_device_name(0))\n", - "else:\n", - " device = torch.device('cpu')" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "class SNN(nn.Module):\n", - " def __init__(self) -> None:\n", - " super().__init__()\n", - "\n", - " self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False)\n", - " self.iaf1 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - " self.pool1 = nn.AvgPool2d(2,2)\n", - "\n", - " self.conv2 = nn.Conv2d(10, 10, 2, 1, bias=False)\n", - " self.iaf2 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - " self.pool2 = nn.AvgPool2d(3,3)\n", - "\n", - " self.conv3 = nn.Conv2d(10, 10, 3, 1, bias=False)\n", - " self.iaf3 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - " self.pool3 = nn.AvgPool2d(2,2)\n", - "\n", - " self.flat = nn.Flatten()\n", - "\n", - " self.fc1 = nn.Linear(810, 100, bias=False)\n", - " self.iaf4 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - "\n", - " self.fc2 = nn.Linear(100, 100, bias=False)\n", - " self.iaf5 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - "\n", - " self.fc3 = nn.Linear(100, 100, bias=False)\n", - " self.iaf6 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - "\n", - " self.fc4 = nn.Linear(100, 11, bias=False)\n", - " self.iaf7 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - "\n", - " def detach_neuron_states(self):\n", - " for name, layer in self.named_modules():\n", - " if name != '':\n", - " if isinstance(layer, sl.StatefulLayer):\n", - " for name, buffer in layer.named_buffers():\n", - " buffer.detach_()\n", - "\n", - " def init_weights(self):\n", - " for name, layer in self.named_modules():\n", - " if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear):\n", - " nn.init.xavier_normal_(layer.weight.data)\n", - "\n", - " def forward(self, x):\n", - " \n", - " con1_out = self.conv1(x)\n", - " iaf1_out = self.iaf1(con1_out)\n", - " pool1_out = self.pool1(iaf1_out)\n", - "\n", - " conv2_out = self.conv2(pool1_out)\n", - " iaf2_out = self.iaf2(conv2_out)\n", - " pool2_out = self.pool2(iaf2_out)\n", - "\n", - " conv3_out = self.conv3(pool2_out)\n", - " iaf3_out = self.iaf3(conv3_out)\n", - " pool3_out = self.pool3(iaf3_out)\n", - "\n", - " flat_out = self.flat(pool3_out)\n", - " \n", - " fc1_out = self.fc1(flat_out)\n", - " iaf4_out = self.iaf4(fc1_out)\n", - "\n", - " fc2_out = self.fc2(iaf4_out)\n", - " iaf5_out = self.iaf5(fc2_out)\n", - "\n", - " fc3_out = self.fc3(iaf5_out)\n", - " iaf6_out = self.iaf6(fc3_out)\n", - "\n", - " fc4_out = self.fc4(iaf6_out)\n", - " iaf7_out = self.iaf7(fc4_out)\n", - "\n", - " return iaf7_out" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "snn = SNN().to(device)" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "snn.init_weights()" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "optimizer = Adam(snn.parameters(), lr=lr, betas=(0.9, 0.999), eps=1e-8)\n", - "loss_fn = CrossEntropyLoss()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Define train and test" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "def train(dataloader, model, loss_fn, optimizer, epochs, test_func, dataloader_test):\n", - " epochs_y = []\n", - " epochs_x = []\n", - " epochs_acc = []\n", - " model.train()\n", - "\n", - " for e in range(epochs):\n", - " losses = []\n", - " batches = []\n", - " batch_count = 0\n", - " train_p_bar = tqdm(snn_train_dataloader)\n", - "\n", - " for X, y in train_p_bar:\n", - " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", - " X = X.reshape(-1, 2, 128, 128).to(dtype=torch.float, device=device)\n", - " y = y.to(dtype=torch.long, device=device)\n", - "\n", - " # forward\n", - " pred = model(X)\n", - "\n", - " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", - " pred = pred.reshape(batch_size, n_time_steps, -1)\n", - "\n", - " # accumulate all time-steps output for final prediction\n", - " pred = pred.sum(dim = 1)\n", - " loss = loss_fn(pred, y)\n", - "\n", - " # gradient update\n", - " optimizer.zero_grad()\n", - " loss.backward()\n", - " optimizer.step()\n", - "\n", - " # detach the neuron states and activations from current computation graph(necessary)\n", - " model.detach_neuron_states()\n", - "\n", - " train_p_bar.set_description(f\"Epoch {e} - BPTT Training Loss: {round(loss.item(), 4)}\")\n", - "\n", - " batch_count += 1\n", - " losses.append(loss.item())\n", - " batches.append(batch_count)\n", - "\n", - " epochs_y.append(losses)\n", - " epochs_x.append(batches)\n", - "\n", - " acc = test_func(dataloader_test, model)\n", - " print(f'Epoch {e} accuracy: {acc}')\n", - " epochs_acc.append(acc)\n", - "\n", - " return epochs_x, epochs_y, epochs_acc\n" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [], - "source": [ - "def test(dataloader, model):\n", - " correct_predictions = []\n", - " with torch.no_grad():\n", - " test_p_bar = tqdm(dataloader)\n", - " for X, y in test_p_bar:\n", - " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", - " X = X.reshape(-1, 2, 128, 128).to(dtype=torch.float, device=device)\n", - " y = y.to(dtype=torch.long, device=device)\n", - "\n", - " # forward\n", - " output = model(X)\n", - "\n", - " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", - " output = output.reshape(batch_size, n_time_steps, -1)\n", - "\n", - " # accumulate all time-steps output for final prediction\n", - " output = output.sum(dim=1)\n", - "\n", - " # calculate accuracy\n", - " pred = output.argmax(dim=1, keepdim=True)\n", - "\n", - " # compute the total correct predictions\n", - " correct_predictions.append(pred.eq(y.view_as(pred)))\n", - "\n", - " test_p_bar.set_description(f\"Testing Model...\")\n", - " \n", - " correct_predictions = torch.cat(correct_predictions)\n", - " return correct_predictions.sum().item()/(len(correct_predictions))*100" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Training loop" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "ebf2bea3d0124365abf1f2aa248ceab6", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - " 0%| | 0/359 [00:00" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "y_avg = []\n", - "for y in epochs_y:\n", - " y_avg.append(np.mean(y))\n", - "\n", - "plt.plot(np.arange(len(epochs_x)), y_avg, marker = '.')\n", - "plt.xlabel('epoch')\n", - "plt.ylabel('average loss')\n", - "plt.ylim(0,)\n", - "plt.xticks(np.arange(len(epochs_x)))\n", - "for i, txt in enumerate(y_avg):\n", - " if i%2 == 0:\n", - " pass\n", - " else:\n", - " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAG2CAYAAACZEEfAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAAB9HElEQVR4nO3dd1RUx98G8GfpRUB6kaqg2HvB3oIt9qiJJjHGWH5irLFgYkw0ETWJJsbE9ipqrDGxG3vBhgiKXSmKglQLLL3uvH8QNq60XYrg+nzO2XNk7p07c3evu987d4pECCFAREREpKY0qroCRERERJWJwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqbUqDXbOnTuH/v37w87ODhKJBPv27VPYLoTA119/DVtbW+jr66Nnz54ICwtT2OfFixcYNWoUjI2NUbNmTYwdOxapqamv8SyIiIioOqvSYCctLQ1NmzbFb7/9VuT2ZcuWYeXKlVizZg0CAgJgaGiIXr16ITMzU77PqFGjcOfOHZw4cQKHDh3CuXPnMH78+Nd1CkRERFTNSarLQqASiQR79+7FoEGDAOS36tjZ2WHmzJn44osvAABSqRTW1tbYtGkT3n//fdy7dw8NGjRAYGAgWrVqBQA4evQo+vbtiydPnsDOzq6qToeIiIiqCa2qrkBxIiIiEBcXh549e8rTTExM0LZtW/j7++P999+Hv78/atasKQ90AKBnz57Q0NBAQEAABg8eXOSxs7KykJWVJf9bJpPhxYsXMDc3h0QiqbyTIiIiogojhEBKSgrs7OygoVH8w6pqG+zExcUBAKytrRXSra2t5dvi4uJgZWWlsF1LSwtmZmbyfYri4+ODb7/9toJrTERERFUhKioK9vb2xW6vtsFOZfL29saMGTPkf0ulUjg6OiIqKgrGxsZVWDMiIiJSVnJyMhwcHGBkZFTiftU22LGxsQEAxMfHw9bWVp4eHx+PZs2ayfdJSEhQyJebm4sXL17I8xdFV1cXurq6hdKNjY0Z7BAREb1hSuuCUm3n2XFxcYGNjQ1OnTolT0tOTkZAQAA8PDwAAB4eHkhKSsLVq1fl+5w+fRoymQxt27Z97XUmIiKi6qdKW3ZSU1MRHh4u/zsiIgLXr1+HmZkZHB0dMW3aNHz33Xdwc3ODi4sL5s+fDzs7O/mIrfr166N3794YN24c1qxZg5ycHEyePBnvv/8+R2IRERERgCoOdoKCgtCtWzf53wX9aEaPHo1NmzZh9uzZSEtLw/jx45GUlISOHTvi6NGj0NPTk+fZtm0bJk+ejB49ekBDQwNDhw7FypUrX/u5EBERUfVUbebZqUrJyckwMTGBVCplnx0iIqI3hLK/39W2zw4RERFRRWCwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGptWod7OTl5WH+/PlwcXGBvr4+6tSpg0WLFkEIId9HCIGvv/4atra20NfXR8+ePREWFlaFtSYiIqLqpFoHO0uXLsXq1auxatUq3Lt3D0uXLsWyZcvw66+/yvdZtmwZVq5ciTVr1iAgIACGhobo1asXMjMzq7DmREREVF1IxMvNJNXMu+++C2tra2zYsEGeNnToUOjr62Pr1q0QQsDOzg4zZ87EF198AQCQSqWwtrbGpk2b8P777ytVTnJyMkxMTCCVSmFsbFwp50JEREQVS9nf72rdstO+fXucOnUKoaGhAIAbN27gwoUL6NOnDwAgIiICcXFx6NmzpzyPiYkJ2rZtC39//2KPm5WVheTkZIUXERERqSetqq5ASebOnYvk5GS4u7tDU1MTeXl5+P777zFq1CgAQFxcHADA2tpaIZ+1tbV8W1F8fHzw7bffVl7FiYiIqNqo1i07f/75J7Zt24bt27fj2rVr2Lx5M3788Uds3ry5XMf19vaGVCqVv6KioiqoxkRERFTdVOuWnVmzZmHu3LnyvjeNGzfG48eP4ePjg9GjR8PGxgYAEB8fD1tbW3m++Ph4NGvWrNjj6urqQldXt1LrTkRERNVDtW7ZSU9Ph4aGYhU1NTUhk8kAAC4uLrCxscGpU6fk25OTkxEQEAAPD4/XWlciqhqx0gxcevAMsdKMqq4KVSJnZ2dIJJJCLy8vLwBA165dC22bOHFiiccs6ngSiQQ//PCDwn6HDx9G27Ztoa+vD1NTUwwaNKiyTpMqSbVu2enfvz++//57ODo6omHDhggODsby5cvx6aefAsi/UKdNm4bvvvsObm5ucHFxwfz582FnZ8eLkegtsCswEt57bkEmAA0J4DOkMUa0dqzqalElCAwMRF5envzv27dv45133sGwYcPkaePGjcPChQvlfxsYGJR4zNjYWIW/jxw5grFjx2Lo0KHytL///hvjxo3D4sWL0b17d+Tm5uL27dvlPR16zap1sPPrr79i/vz5mDRpEhISEmBnZ4cJEybg66+/lu8ze/ZspKWlYfz48UhKSkLHjh1x9OhR6OnpVWHNiaiyxUozMHfPLRRMniETwLw9t9G5riVsTfSrtnJU4SwtLRX+XrJkCerUqYMuXbrI0wwMDOTdG5Tx6r779+9Ht27dULt2bQBAbm4upk6dih9++AFjx46V79egQYOynAJVoWr9GMvIyAg///wzHj9+jIyMDDx48ADfffcddHR05PtIJBIsXLgQcXFxyMzMxMmTJ1G3bt0qrDURvQ7XHifi1VnC8oTAo2fpVVMhem2ys7OxdetWfPrpp5BIJPL0bdu2wcLCAo0aNYK3tzfS05W/FuLj43H48GGFoObatWuIjo6GhoYGmjdvDltbW/Tp04ctO2+gat2yQ0RUlPTsXKw8XXhZGE0J4GxR8qMLevPt27cPSUlJ+OSTT+RpI0eOhJOTE+zs7HDz5k3MmTMHISEh2LNnj1LH3Lx5M4yMjDBkyBB52sOHDwEA33zzDZYvXw5nZ2f89NNP6Nq1K0JDQ2FmZlah50WVh8EOEb1R8mQCU3ZcR0hcKgx1NJGRkwfZvy08ozu48BHWW2DDhg3o06cP7Ozs5Gnjx4+X/7tx48awtbVFjx498ODBA9SpU6fUY27cuBGjRo1S6AJRMBjmyy+/lPfj8fX1hb29PXbv3o0JEyZU1ClRJavWj7GIiF7l8889nLwXDx0tDWwZ2xYX53ZH30b5fS9uRiWhGq+AQxXg8ePHOHnyJD777LMS92vbti0AIDw8vNRjnj9/HiEhIYWOWTClyct9dHR1dVG7dm1ERkaqWnWqQgx2iOiNsfXyY/zfhQgAwE/DmqKlkylsTfSxYEBD6GhqIOhxIgIiXlRxLaky+fr6wsrKCv369Stxv+vXrwOAwhxsxdmwYQNatmyJpk2bKqS3bNkSurq6CAkJkafl5OTg0aNHcHJyUr3yVGUY7BBRmZU298mECRNQp04d6Ovrw9LSEgMHDsT9+/eVPv7EiRMhkUjw888/wy/0KRYcuAMA+Li+Nv7v6//BwsICxsbGGNq3J1rrxgAAfjtT+p08vZlkMhl8fX0xevRoaGn91wvjwYMHWLRoEa5evYpHjx7hwIED+Pjjj9G5c2c0adJEvp+7uzv27t2rcMzk5GTs3r27yJYiY2NjTJw4ERMmTJBf2zo6OkhISMCECRMq5Tp/WWhoKAYOHCi/zjt27IgzZ84ofVz6D4MdIiqzwMBAxMbGyl8nTpwAAPncJy1btoSvry/u3buHY8eOQQgBT09PhflSirN3715cvnwZdnZ2SEjOhNe2a8iTCQxtYY+d309Gbm4uTp8+jatXr6Jp06bYu3QKRHoizoc9Q3BkYqWeN1WNkydPIjIyUj7XWgEdHR2cPHkSnp6ecHd3x8yZMzF06FAcPHhQYb+QkBBIpVKFtJ07d0IIgQ8++KDIMn/44QeMHz8e5ubmMDQ0RKdOnbB+/XoAFX+dv+rdd98tdJ2/++67Ja79SMUQJKRSqQAgpFJpVVeF6I02depUUadOHSGTyYrcfuPGDQFAhIeHl3icJ0+eiFq1aonbt28LB0dH4dxvknCac0gMX3NJRMfGCwDi3Llz8v2Tk5MFADH0yzXCac4hMXbTlQo9L8rn5OQkABR6TZo0SQghxPjx40Xt2rWFnp6esLCwEAMGDBD37t0r8Zh///23eOedd4SZmZkAIIKDgwvts3btWtGlSxdhZGQkAIjExMQyn0NMUrq4GP5UxCSll/kYlXGdOzk5iRUrVsi3PX36tNjr/MSJE2WuuyrehM9b2d9vtuwQUYUobu6TAmlpafD19YWLiwscHByKPY5MJsNHH32EWbNmobabO56lZkOakQMXC0Os/aglbK0tUa9ePWzZsgVpaWnIzc3F2rVrYWVlhXmj+0IiAU7eS8DdmOTKPN23UmW05KWlpaFjx45YunRpsfukp6ejd+/emDdvXrnqvyswEh2WnMbI9QHosOQ0dgWq3sm4Mq7zhg0bFtpubm6OevXqoW/fvvJHaMbGxgCAd955B15eXnjx4gU+//xz1KtXD/r6+nB0dMSUKVMKtV69SpllMgYMGICcnBzo6OjA0tISQ4cOxY4dOwC8OZ+3glICu7cCW3aIym/Xrl1CU1NTREdHK6T/9ttvwtDQUAAQ9erVK/Vud/HixeKdd94Rubl5YuIfQULT2ErY9Z4gHj5Nle8TFRUlWrZsKSQSidDU1BS2trbi2rVrQgghvLZdFU5zDolJ265W/EmSgopq4RBCiIiIiGLv9AucOXOmzC07MUnpwnnuIeE0579X7bmHVW7hqejrvOC9e7VlR4j867xJkyYCgNDQ0BBWVlZi9erVAoA4c+aMuHXrlhgyZIg4cOCACA8PF6dOnRJubm5i6NChJZYdGxur8Nq4caOQSCTiwYMH8n2WL18u/P39xaNHj8TFixeFh4eHsLGxqXafN1t2iN4SJXUSLuud3zfffAN3d3cYGhrC1NQUPXv2REBAQInljhgxAm5uboX6HowaNQrBwcHw8/ND3bp1MXz4cGRmZhZZ7tWrV/HLL79g06ZN+OF4KI7cjoNEAgxv5QgXC0MAgBACXl5esLKywvnz53HlyhUMGjQI/fv3R2xsLLy6uQIA/rkViwdPU8v6tlIpKqqF43WJeJZWITNuFzXHD1D267yo9w747zqvVasWLly4gMDAQAwdOhRffPEFnJ2d0aVLFzRq1Ah///03+vfvjzp16qB79+74/vvvcfDgQeTm5hZ7DjY2NgqvV5fJAIDp06ejXbt2cHJyQvv27TFz5kzExcVh9OjRb8TnXUip4ddbgC079CZLSEhQuEs7ceJEue/8tm3bJk6cOCEePHggbt++LcaOHSuMjY1FQkKCfB8nJyexcOFCERsbK65cuSI0NDTEzp07SzxuVlaWMDAwENu3by9y+4oVK4REIhEampoCEo381793tU5OTkIIIU6ePCk0NDQK/X91dXUVPj4+Qgghxm4KFE5zDokZu66X9vZRGVVUC0eBym7ZiU5MU2jVKUvLzqNHj4SGhobYt29fifspe51ramrKX8pc51lZWUJDQ0N4enoWW/b69euFhYWF0ucUFxcntLS0xLZt24rd5/nz58LDw0MAqHafN1t2iN4SlpaWCndphw4dki+QWNY7v5EjR6Jnz56oXbs2GjZsiOXLlyM5ORk3b95U2M/IyAg2NjY4fPgwrKysFFaLLooQAkIIZGVlFbn9o48+wuZDfrAbsxK2Y1biizX7YWdnh1mzZuHYsWMAIF/vSEND8etLQ0NDPuPt5O75rTv7rkcj6gXXyqoMFdHC8TrFJRe+5ub2cVdpxm1l5/hR5jq/efMmrl+/Ln8pc53v27cPMpkMzZs3L/K4z549w6JFixRmky5NUctkFJgzZw4MDQ1hbm6Ou3fvwtPT8435vF/FYIdIjZT2aAEApFIpjI2NFeYpKe2Y69atg4mJSaFJ15YsWQIzMzMsXrwY7u7uCtsePnwIHx8fXL16FZGRkbh06RKGDRsGfX199O3bV77fy3OfJObpYNmVdGiaO2HYO+2xbFw/aGtrw8bGBvXq1QMAeHh4wNTUFKNHj8aNGzcQGhqKWbNmISIiQv4j1MyhJjq5WSBPJrDG74Fybx4praRZjE1MTODm5obOnTvjr7/+wv379wvNbVMVDt2IBQB4NrBGPesaAIC07OID/lcVN8dPWa5zc3NzNGrUSOGlzHU+Z84cSCSSIofJJycno1+/fmjQoAG++eYbpc+rqGUyCsyaNQvBwcH4448/IJVK8fz580IzlFfXz/tVDHaI1EhRCyS+TJU7v0OHDqFGjRrQ09PDihUrcOLECVhYWMi3T5kyBTt37sTixYuRk5ODq1evYvbs2fLtenp6OH/+PPr27QtXV1eMGDECRkZGuHTpEqysrOT7Fcx98iw1C2M2BSIlMxetnEyxdGiTIgM2CwsLHD16FKmpqejevTtatWqFCxcuYP/+/QrB2OR/++7sDnqCOGn1u9N8k1VUC8frIpMJ/HMrP9gZ1soBk/69NnZeiUJunkypYxQ3x4+q17myXr3OW7RogUePHuGrr74qdNORkpKC3r17w8jICHv37oW2trZSZRS3TMbLdahbty7Cw8NhZWWFq1ev4vLly8Uer7p83kVS6uGammOfHVIXnp6e4t133y1ym1QqFW3atBG9e/cW2dnZpR4rNTVVhIWFCX9/f/Hpp58KZ2dnER8fX+z+GzZsEFpaWiIzM1OlOsckpYsz9+NFv1/OCac5h0SnpafFsxTVjlGc91ZfFE5zDomFB+9UyPFIiLy8POHo6CjmzJmjkP7gwQOxePFiERQUJB4/fiwuXrwo+vfvL8zMzBSum3r16ok9e/bI/37+/LkIDg4Whw8fFgDEzp07RXBwsIiNjZXvExsbK4KDg8X69evlc88EBweL58+fK1XnwIjnwmnOIdHo66MiIztXZObkihYLjwunOYfEkVuxpR+ggpRnjp8FCxYIGxsbkZOTo5AulUpFu3btRJcuXURaWppKxxw9erRo2bJlifsUfN7/+9//5H0Bhag+n7eyv98MdgSDHVIPJXWeTE5OFh4eHqJHjx4iIyOjTMd3dXUVixcvLnb77du3BQBx//59pY+588pj4fLScOB6X/4jwuJTylS/opwNScg/7lf/VFgA9bY7duyYACBCQkIU0qOjo0WfPn2ElZWV0NbWFvb29mLkyJGFrgcAwtfXV/63r69vkRPXLViwQL7PggULitzn5eOUZMH+28JpziExfWewPG3JkXvCac4h8eH/XVb1LSiTl691l7mHxM4rj5XOW1yAKZVKRdu2bUXjxo1FeHi4wkCF3Nxc+X6vBhwFeQ0MDMTq1asLlXf58mXx66+/iuDgYLFlyxYBQDRv3lzUqVNHfjNTXT5vBjsqYLBD6qAy7vxeVrt2bYUvpFdt3bpVaGhoiBcvXih1vIinqcL5ldExLnMPlWtm21fJZDLR/9fzwmnOIbH0SMkzu74pSpvVNiMjQ0yaNEmYmZkJQ0NDMWTIEBEXF6f08SdMmCAAFJrzpahyC0a/qaoiZjFWVl6eTLT+7oRwmnNInLz73/sQ+TxNPu/Oy3M4VYaYpHSFoF7VkWDFBZgFo5WKekVERMj3KypQWLt2rdDX1xdJSUmFyrt586bo1q2bMDMzE7q6usLZ2Vl8/OlnYt+Fm2X6zCrz81b291u5HopEVK3JZDL834aN6DlgGJ6m5cDWJP+/dnJyMjw9PZGeno6tW7ciOTkZycn5MwtbWlpCU1MTQH7nSR8fHwwePBhpaWn4/vvvMWDAANja2uLZs2f47bffEB0dLZ851d/fHwEBAejWrRuMjIzg7++P6dOn48MPP4SpqWkJ9RQIiHiBfcHROHAjGuLV7QJ49CxdpREyJZFIJPDq5ooJf1zFFv/HmNC5DkwMlOvPUF0FBgYqzFB7+/ZtvPPOO/LPZvr06Th8+DB2794NExMTTJ48GUOGDMHFixdLPXZJ6zQBwMKFCzFu3Dj530ZGRirXf3vAY3y59zYEAA0J4DOkMUa0dlT5OMoKepyIhJQsGOlpoaPbf33OHMwM0K2eFU7fT8C2y4/x1bsNKq0OEc/SICtmjh9lrnVPT89CHYMBoGvXrkWmv0oIgVhpBi49eAYXC0PYmuhj/PjxRfbdE0LAvnY9rN2xH09Ts/AsNRsn7sTh0M1Y+B2MhORgJLrWs0QDO+NSywWAuzHJOBvy9LV93sVhsEOkBr76bTuin0ThtGiEDktOy79Qrl27Jp8M0NXVVSFPREQEnJ2dAeR3ngwKe4J20gyY6mri/v372Lx5M549ewZzc3O0bt0a58+fl09rr6uri507d+Kbb75BVlYWXFxcMH36dMyYMaPI+oXGp2BvcDT2B0cjpoTOwpoSCZwtDJQ+7+joaMyZMwdHjhxBeno6XF1d4evri1atWgEA4uPjsX3ZXMTtO4Ts9FS0+6cNDmzfADc3t2KP2bVrV/j5+RVK79u3Lw4fPgwg/wdh+vTpWLN2LbIyM6GhoQFXV1ds27ZNoew5c+bg+PHjSEpKQufOnfHrr7+Wu2wLCwssWLAA69evR1JSEiwtLeHo6IguXbpAKpViw4YN2L59O7p37w4gvzNx/fr1cfnyZbRr167E9/Lzzz/HsWPHiu14XDDVQFnFSjPkgQ6QH9zO23MbnetaVliA+6pDN2MAAJ4NbKCrpamw7cN2jjh9PwG7rz7BF73qQU9bs6hDlJuFoW6hNA0JVLrWy2NXYCS899yCTAASCTCilQNcrWrkBzMp2XiWmiV/PU/NRu6rkdlLBIAzIU9xJuSpyvV4HZ93cRjsEL3hYpLSsSPaFE5zDgHI/0KZ8/ctbLr0GLVqGmLOXzdgUUMXFjV0YGmkB4saOrAw0oVpDV0IIfBnUBRc5h7CthfAjn8DpT179pRYZosWLUoclQEA8cmZOHA9BnuDo3E39r91qoz0tNCvsS0GNa+FiGdp+GrvbeQJAU2JBIuHNFL6SzAxMREdOnRAt27dcOTIEVhaWiIsLEzesiSEwKBBg6CtrY3vf9+C5WejkBB8AD169MS9e3dhaGhY5HH37NmD7Oxs+d/Pnz9H06ZN5S0nQP4M0ytXroSuQ2OYN3kHabdOIv5ZDAwMDAqVvX//fhgbG2P58uXo2bMn7t4tX9nLli3DypUrsXnzZtjb26N9+/YwNDREVlYWrl69ipycHPTs2VO+v7u7OxwdHeHv719ssFPaOk0FlixZgkWLFsHR0REjR47E9OnTlZ7CAADO3E8o1JqnSguHqvJkAv/cyl8h/N2mtoW2d6lrBXtTfTxJzMDBGzEY1qpyZv7dez26UJqbVQ3YGBce7l3RYqUZ8kAHAIQAdgZGlZrPWE8LFka60NHUwP24lELbezeyLvUzi03KwNE78Qpplfl5l4TBDlEFUKaFQdW7/D179mDx4sUIDw9HTk4O3NzcMHPmTHz00UcAgJycHMz1nocNO/Yg+Wk0NHQNoefUFDW7fAItI3Pci03GvdiSF8PU1pAg56W7OJkA5v59C7FJmahtVSM/QKqhC4sauqhpoF3s3D2x0gxEPEuDlZEebkQlYd/1aFwMfyb/gtXWlKBrPSsMbl4L3d2t5HfQ7Wqbo2s9Szx6lg5nCwOVvgCXLl0KBwcH+Pr6ytNcXFzk/w4LC8Ply5dx+/ZtuNdvgEOxfnhoNhGJ6z7Bjh07ih1ua2ZmpvD3zp07YWBgIA84hBD48ccfoWlsBesPFgMADFzb4MmvH8LvchAaNGigUHZB8LB69WrY2NiUu+yff/4ZX331FQYOHIg///wTeXl5SEtLk084p6Ojg5o1ayocx9raGnFxcSW+l1paWpgyZUqx+0yZMgUtWrSAmZkZLl26BG9vb8TGxmL58uXF5nlZTp4Mmy49KpSuamueKq5EvMCz1CyY6GujQx2LQts1NSQY2dYRy46GYGtAZKUEOwnJmfC9GAEAWDq0CTQ1AO+/byEkPhV7g6MxpIV9hZf5socJhR+hAYBHHXO42xjBooZu/v9xI51/b4p0YV5DR94KFivNQIclpxWOoSmRYEH/hqUHO9IMHL8bXyjv62rRehmDHaJyUqWFQZW7fDMzM3z55Zdwd3eHjo4ODh06hDFjxsDKygq9evVCeMxzbD54FtqthsHWygWyzFS8OLUOT/csQq1PfobP4MbIFaJQM/Wz1Gw8S8lCSlauQqBTQAD4+VRYoXQtDQnMa+R/IVoa6cq/GKNepOOfW7GF7tgBoKWTKQY3r4V+jW1haqhT5HnamuiX6S7vwIED6NWrF4YNGwY/Pz/UqlULkyZNkvcpKZjrQ09PD5oaEvyvax3M/usmMmWa8Dt3vtiA41UbNmzA+++/L/+cHj58iPT0dBg2aIOn+3yQGXUbmjXMoWlsiTPnLuB/n36sUHYBDQ0N6Orq4sKFC2UuOyIiAnFxcfKWm4JZjJOTk+Hv74+2bdsqddyXFazTdO3atWKDWQAKjyibNGkCHR0dTJgwAT4+PtDVLfyY5lXrzj1EaHwq9LU1kZWbJ/8BnOlZt9Lu8g/fyn+E1auhNXS0ip5WbngrB6w4EYobUUm49USKxvYmFVqHX0+HIzNHhhaONTG8lT0kEgnik7Pww7EQfHvwLjq6WcDKqPJaeC4+eFYoTVMiwfLhTZV6321N9OEzpDHm7VG9BbY8eSsagx2iclKlhUGVu/yuXbsq/D116lRs3rwZFy5cQE23Vpi07RZqDP4G5oY6eK+lPf7vfATwzkTEbZmBme3NMKJNyZ0AM3PycDdGiqFr/BUWSJQA6OZuhbSsXHlwJM3IQa5MID45C/FFTLv/qnGdXPBRO2c4mlfeHdzDhw+xevVqzJgxA/PmzUNgYCCmTJkCHR0djB49Wv74xtvbG2vXrkW/hpaYNX8fcpKf4npIhFJlXLlyBbdv38aGDRsAAJHP0+G14QwAIC3kIoxbD4K1x3BkxYbhxfHfccrvImKSMgqVbWhoiBUrVuDJkyeIjY0tU9kA5K0z1tbW8lmM9+zZg23btiEuLg42NjbIzs5GUlKSQutOfHx8sX1tzp8/j4SEBDg6/ne95OXlYebMmfj555/x6NGjIvO1bdsWubm5ePTokXzW3+KEJ6Tgl38D6O8GNUJ7V3NM+OMqbj6R4kVadol5yyo3T4ajt/Pfr35Niu5wDQAWNXTRt7Et9l+PwdbLj7H0vSYVVofHz9Ow40okAGB2b3d5MDm+c238cysWd2KSsWD/Haz+sGWFlfmyq48Tse7cQwD5fXWEQJkCjhGtHdG5btlaYMuTtyJxBmWicjpw4ABatWqFYcOGwcrKCs2bN8f69evl20u7y1eGEAKnTp1CSEgIsi3rYdT/BeB5WjYa2hnjwOcd4d23Pi7M7QbvHk6QSCT4qEvx/S4K6GlrooWTGZYMaQzNf7+ENSUSLBnaGBs/aY1dEzxwamZX3FjgidDv+sDfuzsOTO6AjZ+0wrKhTTCrVz30bmhd5LG7u1tXaqAD5PczadGiBRYvXozmzZtj/PjxGDduHNasWQMA0NbWxp49exAaGgozMzOYGNWASWIo9Gq3xJOkTGTnlj5z7oYNG9C4cWM0b9kKa/wewPNnP1yPTAIAWDi4wrzraOhY14Fxs97QqmmNpLgovLPcD1uvPMHuv/6Wl21gYIAzZ86gT58+hdb0Kq3sNm3aFLm9qFmMW7ZsCW1tbZw6dUqeFhISgsjISHh4eBR5HGXWaSrK9evXoaGhoTBLcFHyZAKz/7qJ7FwZutazxJAWtWBroo/pPesCAP4MikK6Css2KCv/EVY2ahpoo30d8xL3/bCdEwBg/41oSDNyKqwOK06EIlcm0LmuJdrV/q8O2poaWPZeE2hpSHDkdhyO3FIuAFaFND0HU3YEI1cm8G4TW1yc0w07xrXDhbndyjQaytZEHx51zMsUrJQnb0Vhyw5ROanawqDKXb5UKkWtWrWQlZUFTU1NdB/rjV0xpgAE+je1w7KhTaCvk/9s3VRXgs2/fI8PPvgAxsbKDQsFlLvz0tHSKPJxU1U+k7e1tUWDBorDhevXr4+///5b/nfLli1x/fp1SKVSZGdnw6imGSxqN0KuSR3sDX5S4pd+Wloadu7ciQnTvdH/1wvyTpqt67vgEICOzerh97nd5e+bxy59PElLRFp2Hr49eBdN7U2w7R8/2Bvmry9maWmJtm3byvtxlaSg7IULFyqkF7TOxMbGKkw1EB8fj2bNmsHExARjx47FjBkzYGZmBmNjY3z++efw8PBQ6Jz88lQD5ubmMDdXDAZeXaeprFMNAMDmS49wLTIJNXS1sHhwY3nrRpe6lnA2N8Cj5+nYGxyNUW2dSn1fVHHo3wCid0MbaGuWHGC2cjJFPWsjhMSnYM+1JxjTwaXE/ZVxLzYZ+2/kP0ab3atwy1dDOxNM7FIHq86EY/7+O2hX27zYR72qEkJg9t83EJ2UAUczA/gMaQwjPW3Y1Xz9fWWqC7bsEJWTqi0MqtzlGxkZ4fr16zh29gLq9huHoxt+QGbkTczt446V7zeTBzo5OTkYPnw4hBBYvXq1yudQ1juvgmfyL7cMva5n8h06dEBISIhCWmhoKJycCv9ompiYwNLSElGPHiI9JhQGbm2x+uyDEtdF+mP7TqRlZGJnkhPux6WgpoE2fnivCfZ7D4aenh5u3Lghf98MJTl4EvUYtZ0c8N2gRjDS1cKNJ1IMWHURv1+MgaGJKcLCwhAUFISBAweWem67d+9GVlYWPvzwQ4V0FxcX2NjYYNaiH+VTDXh8ewj+lwPkLTcrVqzAu+++i6FDh6Jz586wsbEpNLouJCQEj2Of4tKDZ4iVZpRan4KpBrp06YKGDRvi+++/x/Tp07Fu3boS80U+T8cPx/I/o7l93GFX87/rQkNDgo88nAHkB0TKzBejrJcfYb1bwiOsAhKJBB965F83Wy8/rpC6/HgsBEIA/ZrYolGtovsBfd7DFa5WNfAsNQuLDt0td5kFtvg/xrE78dDWlGDVyOYw0nuz55aqEBU+neEbiDMoq4cnT56IUaNGCTMzM6GnpycaNWokAgMD5dtRzEyjy5YtK/aYubm54quvvhLOzs5CT09P1K5dWyxcuFDIZDL5Po6OjqJZs2bCxsZG6OnpiR49eohvvvlG2NnZFTpeUlKSSEhIEEII0aZNG/mstyW5EZUo2n5/UjjNOSRMm/cSrTp0VdienZ0tBg0aJJo0aSKePXtW6vEqQ0xSurgU/uy1zIhb4MqVK0JLS0t8//33IiwsTGzbtk0YGBiIrVu3yvf5888/xZkzZ8SDBw/Evn37hJOTkxgwaLBo9u0x4TTnkNgX/ER89NFHYu7cuQrHPnY7Vhg5NRIG7p3kywy8vNyEl5eXACA+/PBDcejQIdGyZUshkUjExo0bhRBCxEkzhOfnS4T1B4uF3YT/E+4fLRTWdg5iyJAhCuUUVbYQQnTs2FGMGDFCIS0vTyZuPUkSvcfMEBq6hsJyyHxh++kqoe/WTmiZWIsuPsfEqPWXxdQd18Sig3fE6rPh4q+gKHE2JEHcjk4S8dIMkZObJ4Qo39IFypLJZOKDdf7Cac4hMWLtJZGXJyu0jzQjW9Sff0Q4zTkkLoY9rbCyz4XmLxPSfOFx+TmXJiUzRzQoqEt4+epSsBZXbe/D4kFCycufXH38Qj6T8+l7xa89p6xbT5KE27x/hNOcQ2LD+YflPl51xxmU6a1S2ogoAIUeGR05cgRjx47F0KFDiz3u0qVLsXr1amzevBkNGzZEUFAQxowZAxMTE/kwXVNTU9y6dQt///03XFxcMH/+fCxfvhz169cvdDwTk/w7vIK7/EWLFpV4XnuuPcHcPbeQnStDHUtDmLlb4mnMf3NkFLTohIWF4cyZM4UeR7wuZR1RVR6tW7fG3r174e3tjYULF8LFxQU///wzRo0aJd8nNjYWM2bMQHx8PGxtbfHxxx9j/vz5WHfhMX48HorfzoQjOzJS3sIWJ83EggO3cejcVaQ8vo1GY5fh97Ft0MnNUqHsX3/9Fc+ePcPOnTuxdetWGBgYYOHChRgzZgwAwNpYD/1c9RG8eykSniYg3tAUNRp2h82gGXiakgVLo/zRS5EvlV0gJCQEFy5cwPHjxxGdlIELYU9xPuwZLj14jhdp2RCW3WDUMgbPj/0KWWYa9OwbwGr4QjxKysGjpMIjb14mkQAmetpIeqlfSmVN9LYzMAqXHjyHnrYGlg5tAg2NwiO9jPW0MbSFPf64/BibLj1Ce9fCw8PL4vDNfx9hNbKBVimPsArU0NXCoOa1sC0gEtsuR6J9EUPVlSGEwLKj+a1Zw1vZo7ZljRL3b+Foik87uGDDhQjM23sLx6d3LnNLTGpWLiZvv4bsPBl61rfGmA7OZTqOOpIIUYFth2+o5ORkmJiYQCqVqtTXgaqPuXPn4uLFizh//rzSeQYNGoSUlBSFzpyvevfdd2Ftba0wImbo0KHQ19fH1q1bIYSApaUlEhMTsWjRIgwfPhxnz57FuHHj4OXlhVWrVgHIfyxRMNPtrVu3MHXqVLRs2VKhf8nHH3+MWrVqwcfHB7l5MvT5dCZuZppDy9QW7RyN0FLjERbM/xKrV6/GZ599hpycHLz33nu4du0aDh06BGvr/zoLm5mZQUenYp7/qyNpRg46LjmNlKxcTO9ZF0Nb1sLp+wlYdjQEqVm50NKQYFzn2pjS3U3+qLCs0rJy8dPxUGy6FAGZyJ+sbV7f+hjeygHxKZmIeJYmn8I/JTMHlx++yA9wwp/h4dM0hWMZ6miiuWNNXAx/rjDUX0MC/DS8GWQyoTjFQGoWnqbk//tFWlaR860U2DGuHTxK6cirrFhpBjyXn0NKVi6+6lcfn3WqXey+4Qkp6Ln8HDQkgN+sbnAwK1+/kpw8GVp/fxJJ6TnY/llblQKoe7HJ6PPLeWhpSHBpbndYlWHSvzMhCRjjGwgdLQ34zeqqVACZkZ2HXj+fQ+SLdIxq64jvBzdWuVwhBKbtuo7912NgZ6KHf6Z2Qk0D9f8OUPb3my07pBZKm3PlVfHx8Th8+DA2b95c4nHbt2+PdevWITQ0FHXr1sWNGzdw4cIF+URqEREReP78OX755ResX79e3sLg5uYmX3cKKL6F4WUFd/lJ6dn4fEcwroTFIv3+LiDtBc4YGiDW3R1bt27FiBEjAORPZHjgwAEAQLNmzRSOdebMmUJD1+k/JvraaO1shtMhCVhxMhQrTobKtzV3rAmfIY3hblMxNz6Gulr4un8DDGpuB+89t3AnJhlz99zCWr+HePwif8I3CQBHMwM8ScpA3ksRiYYEaOpQE51cLdCpriWaOdSEtqYGdgVGFpq7ZHDzWiXWI08mkJiejXuxyfh445VC0w04mVdMq44QAl/tvY2UrFw0c6hZamdfVysjdHKzwPmwZ9h6+TG8+xZuEVXFpQfPkZSeA4saOmjjYlZ6hpfUtzVGKydTBD1OxM7AKEzpUfykn0WRyf5r1fmkvbPSLWX6OppYMrQxRq4PwLaASLzbxE7lwHN30BPsvx4DTQ0JVn7Q/K0IdFTBlh2wZUcdFAzrnjFjBoYNG4bAwEBMnToVa9aswejRowvtv2zZMixZsgQxMTEKQ8JfJZPJMG/ePCxbtgyamprIy8vD999/D29vbwDApUuX0KFDB8TExMDW9r/p6IcPHw6JRIJdu3YpfQ6x0gycD3uGn0+GIiYpE/ramvhxWFP0a1J4mnsqn6JmhQWALzzr4n9dXaFZxCOXipCbJ4PvxUf46XgIMosZ+u5sboCObhbo6GoJjzrmMNEv+pFGrDSjzHOX5AdLt5D30vmPauuIhQMblfvc91+PxtSd16GjqYFDUzqirnXpi4WevBuPz7YEwURfG5e9e5SrNW32XzfwZ9ATfNjOEd8NUr2FZF9wNKbtug5bEz2cn91N6cdgAHDgRgym7AiGka4Wzs3upvLoqnl7b2F7QCSczA1wdGpnpd+HsPgU9F91AZk5MszqVQ9e3VxLz6Qm2LJDbxWZTIZWrVph8eL86fubN2+O27dvFxvsbNy4EaNGjSox0AGAP//8E9u2bcP27dvRsGFDXL9+HdOmTYOdnV2Rxy2rLZceYcGBO/JHE6YG2tj2WTulVxYm1RS1CjUAtHQyq7RABwC0NDUwrnNtWBnpYuqu64W2r3y/GQY0K7mFpkB5+km9PN1A0OMXWH4iFNsCIvEsNQu/vN+8zAtiPkvNwjcH7gAAPu/uqlSgA+RPYuloZoDIF+nYdz0aH5QyIWZxsnNlOPbvWkzKjMIqSp/GNlh4SAex0kycvp8Az4bKLXyakyfDT8fzW3XGda5dpmHk3n3cceZ+Ah4/T8dPx0OUWok9IzsPXtuvITNHhk5uFvhflzoql/s24NBzUgvFzbkSGRlZaN/z588jJCREqSn7Z82ahblz5+L9999H48aN8dFHH2H69Onw8fEB8N+8J/HxiovdlTRjbYHcPBn8Qp9iwpYgfP1SoAPk9ykxNeRw0criYmGIV2Oa17lmT5vaZkWW31rFxy7lUTBs/vPubvhtZAvoaGrg2J14fLzhCqTpZZtYb8GBO0hMz0F9W2NM7Kr8j66mhgQf/zv0uzzD0C+GP4M0IweWRrpo7Vy291JXSxPD/10j64/Lj5XO92dQFB4/T4e5oQ7GdizbPD1GetpY/G9/nY0XI3AtMrHUPAsP3UFofCosauhi+fBmRXYEJwY7pCZUmXNlw4YNaNmyJZo2bVrqcdPT0wuNltHU1IRMlv8IomDek5c7OScnJyMgIKDIGWuFELj1RIqFB++inc9pjN54BcfuxhfaTyaAR8/SS60flU1Vzg9UHcp/Vd/Gttj8aRsY6WrhyqMXGL7WX6n5d1527E4cDt+MhaaGBD+816TUifxeNayVA/S1NXE/LgWXH75QKW+BQ/+OwurbyKZcLXSj2jpCIgHOhz3Do2dppe6fkZ2Hlf8uhzG5uysMdcv+0KSbe/6CuTIBzP7rJrJy84rd98CNGOy4EgWJBPjl/WbyUX5UGB9jkVqYPn062rdvj8WLF2P48OG4cuUK1q1bV2jSs+TkZOzevRs//fRTkcfp0aMHBg8ejMmTJwMA+vfvj++//x6Ojo5o2LAhgoODsXz5cnz66acA8icjmzZtGr777ju4ubnJh57b2dlh0KBB8uNGvUjHgRsx2BscjfCEVHm6qYE2urtbYU9wtEKH0apaGfhtUtVr9lR1+a/yqGOOPyd6YPTGKwiJT8HQ3y9hy9g2cLUq/VGUND0HX+27DQCY0Ll2sZPolcREXxtDWuQP/d586ZHKHXSzcvNw/G7pa2Epw8HMAF3qWuJsyFNsvxKJeaV0mt7s/wjxyVmoVVMfI9uW7RHcy75+twHOhz1FeEIqVp0Ox0zPwjMwP3qWhnl7bgEAJndzRYcKGravrhjskFpQZs4VANi5cyeEEPjggw+KPE5oWDiCQyMRK82ArYk+fv31V8yfPx+TJk1CQkIC7OzsMGHCBHz99dfyPLNnz0ZaWhrGjx+PpKQkNG7ZFlt270OWTBN7r0Ri77VoXHn0352qrpYG3mlgjcHNa6FzXUtoa2qgjYtZtVgZ+G1TFfMDVafyX1Xf1hh7JrXHxxuv4OHTNAxd7Y+Nn7RCS6eSHwl9d/gunqZkoY6locojmF42ur0ztgVE4vjdOEQnZaBWTeXfmwthz5CSmQsrI120cip5CQtlfNTOCWdDnuLPoCjMeKdusf2YpBk5WH32AQBg+jt1oatVvqkKAMDUUAcLBzbCpG3XsPrsA/RpZKvQfy8rNw+Td1xDalYu2jibYWo53vO3BUdjgaOxKN+uwEh477kFmcgf8uszpLFKC+a9nB/Ib53J+/e/l0QCeNQ2x6DmtdC7kQ2Mi5g0rDyja4gq0ou0bHy6KRDXo5Kgq6WBVSNb4J0GRS/66hf6FKM3XoFEAvw10aPUwKg0I9dfxqUHzzGxSx3M7eOudL4Zu65jT3A0PmnvjG8GlL4QbmnyZAKdl51BdFIGfhrWFENb2he53w/H7uO3Mw/gZlUDR6d1rtAO7hP/uIqjd+LQ0M4Y+7w6yB8NfnvwDnwvPoKpgTb+mdrprf6+4GgsIiVl5uTh6O04zP37lryTsEwAc/6+BZ9/7ivV4U8mEwqz0gJAnhCoY2mI4a0cMKCZXalfSNXtLp/eXmaGOtg+ri28tl3DmZCnmPBHEBYPboz3XxkllZqVK3+UMtrDudyBDpA/P82lB8+xMzAS03q6KTUyLDMnDyf+7fvWv2nFTNWgqSHByLaO+OFYCLYGPC4y2ElIycTGC48AAF/0qlfhI/kWDmoI/4fPcScmGevOPYRXN1ccvxMH34v5Zf40vCm/M5TEYIfeOjKZwN3YZJwPe4YL4U8R+CgR2cXMefJqAKOq7wY1rrBZaYleJwMdLaz7uBW899zCX1fzly15mpKFyd1d5SuXLzt6H9FJGXAw08fs3oX7lZRFj/rWsDfVx5PEDBy4HoPhrR1KzXMu9ClSsnJha6KH5g7lf4RVYHgrB/x8MhTBkUm4HS0t1Bfpt9PhyMjJQzOHmvAspuWrPKyM9PD1uw0wc/cN/HwyFJoaEvx2JhwA8FlHF3R3r/gy1RWDHVIrsdIMhen3C8QkZeBC2DOcC3sqX2PoZVZGukhIyVJI05AAW8e2VWqEw9OULHy4IUBh7hZ2MqY3nbamBn54rwmsjXXx25kH+OlEKOJTMjGxSx0cvxOPLf75Q7OXDGkCA52K+TnR1JDgo3ZO8DlyH76XHmFYK3t5cFWcw7f+HYXV2LZCh15bGumidyNbHLwRg20Bj+EzpIl8W9SLdGy/kj+1xeze9UqtY1kNaVELa889QGh8KpYcuQ8AsDfVx+zeyj/iIwY7pEZe7XPzYTsnSIBi1xjyqGOOjq4W6OhmiTqWhvgzKKpQJ2Fl19VxszaCz5DG7GRMakcikWBWL3dY1tDFt4fuYuvlSGy9/N/8Va2dTSt8JNCI1g5YcTIU92KTEfgoscRlHzJz8nDy30dYlTHb+IdtHXHwRgz2BcfAu299eX+7FSdCkZMn0MnNosyLhiojLjlTYQQnkH/z9jwti98vKmCwQ2ohVpqh0DlYJiC/6wSKX2PoZeUdClzdhhITVaRPOrhAU1OC+fvuKKRffZwoH71YUWoa6GBw81rYcSUKmy89KjHYORvyFGnZeahVUx/NHWpWWB0KtHExQ13rGgiNT8Xea9EY3d4Z9+OSsfd6NABgVq+KeXxXnKJm+y6Yh4vfMcpjsENqobjp/3vWt8J7LR1KXGPoZeXtJMxOxqTO6ljWKJRWWT+8o9s7Y8eVKBy9E4eYpAzYFTMM/b9HWDaV8ihJIpHgw3ZO+Hr/HWy9/Bgfezjhx2OhECK/zCb2NSu8zJcVzPbNR+TlwxmUSS24WBji1a85TYkEiwY1Qu9GNkoFOkRUste5zIa7jTHa1TZDnkxgW0DRyzZkZOfh1L3yrYWljMHNa8FARxNhCalY7fcAJ+/FQ0MCzHinclt1gOo32/abisEOqYVnKdl4OdrhFwJRxXvdP7yftHcGAOy4EoXMnMLLJpwNSUB6dh7sTfXRxF71WZuVZaSnjUHN8xdoXXY0f1maYS0d4GpVuKWrMoxo7YgLc7thx7h2uDC3m0rzf1E+PsaiClfciKjKkp0rw6y/bkAIoIe7FT7rVJt9Zogqyevsm9azvjXsTPQQI83EwRsxGNZKcRh6wVpY/ZrYVtpoqAKWNRRHZda2NKzU8l7FR+Tlw5YdNRQdHY0PP/wQ5ubm0NfXR+PGjREUFAQAyMnJwZw5c9C4cWMYGhrCzs4OH3/8MWJiYko85rlz59C/f3/Y2dlBIpFg3759CtsLjuvo6o5alqbo3Kwe6nboh9X/BFbWacqt8XuA+3EpMDXQxrL3msCjjjm/FIgqUcGK6ZX9/0xLUwMfeTgDADa9shp6enYuTt3/9xFW48p7hAXk38D9ejpMIW3Z0RCVF0ulqsNgR80kJiaiQ4cO0NbWxpEjR3D37l389NNPMDXNn2grPT0d165dw/z583Ht2jXs2bMHISEhGDBgQInHTUtLQ9OmTfHbb78VuT09PR3+V4KQ2WgQbEf/AstB85D9IhrTxo6s1C+E0PgU+ZfQNwMawrwGV/0lUifvt3aArpYG7sQk4+rjRHn66fsJyMyRwdHMAI1qVe4yP0UNgMgTAo+epVdquVRx+BhLzSxduhQODg7w9fWVp7m4uMj/bWJighMnTijkWbVqFdq0aYPIyEg4Ohb9LLhPnz7o06dPseWamJjggwVrsfTf59kAYPbORMRtmYHLN0MxuFPTsp5SsfJkArP+uomcPIGe9a0woGnl3t0R0etnaqiDgc3s8GfQE2y69AitnPOHoR9+jY+wOCLqzceWHTVz4MABtGrVCsOGDYOVlRWaN2+O9evXl5hHKpVCIpGgZs2aZSozKzcPCw/eVQh0AECWlQ5AAoMalXPXtfFCBG5EJcFIVwvfDWpc6V94RFQ1Rv/bUfno7TjESTORlpWL0/cTAADvVsJEgq/iiKg3H1t21MzDhw+xevVqzJgxA/PmzUNgYCCmTJkCHR0djB49utD+mZmZmDNnDj744IMyrfgenpCCz3dcx73YZABA+zrmuPzwOfJyspF01hcGDTpj4bEItHC1U2rZBWU9epaGH4/nB1df9qsPGxO9Cjs2EVUvDe1M0MbZDFcevcC2gMdwszZCVq4MLhaGaGBbuY+wCnDS0Dcbgx01I5PJ0KpVKyxevBgA0Lx5c9y+fRtr1qwpFOzk5ORg+PDhEEJg9erVKpUjhMD2gEgsPHQHmTkymBvq4IdhTdDd3RqRz5IxasRwGFoawnzELEQlZeCzLUHYNb6dUisYl36OAnP+vomsXBk6uJpjhBILBRLRm210e2dcefQC2wMi5cPM+zWu/EdYL+OIqDcXH2OpGVtbWzRo0EAhrX79+oiMjFRIKwh0Hj9+jBMnTqjcqrPG7yHm7b2FzBwZOrlZ4MjUTujubo2cnBxMHTcayc9icf7saWyZ2BU1DbRxIyoJM/68DllR0xyraPuVSAREvIC+tiaWDGnCx1dEbwHPhtawNdHD87RsnAl5CqBy1sIi9cRgR8106NABISGKfWdCQ0Ph5OQk/7sg0AkLC8PJkydhbm6u9PH9HzwHAARHJkJbU4Iv+9bH5jFtYGWsh5ycHPTv3x9nzpxBVFQU7O3tMaBbO0xrogFtTQn+uRWHZcfu4+uvv4atrS309fXRs2dPhIWFlVimj48PWrduDSMjI1hYWmHS6A+Q8/wJZveuBwez/A6CEyZMQJ06daCvrw9LS0sMHDgQ9+/fV/q8iKh609bUKDRx4I2opKqpDL1xGOyomenTp+Py5ctYvHgxwsPDsX37dqxbtw5eXl4A8gOd9957D0FBQdi2bRvy8vIQFxeHuLg4ZGdny4/To0cPrFq1Sv53ojQZ03/fh6HfbwcAmOQmwqeLEXo5a0FDQ4KcnBwMGDAAJ0+eRJcuXbBt2zb4+fnhyy+/RNt6tbBkSBMAwNKly/DTil+wZs0aBAQEwNDQEL169UJmZmax5+Tn5wcvLy/4+/vDY/Jy5ObmIHHPArzXxFK+T8uWLeHr64t79+7h2LFjEELA09MTeXmFZ10lojdPrDQDJ/5d3bzAl3tvc64bUo4gIZVKBQAhlUqruioV4uDBg6JRo0ZCV1dXuLu7i3Xr1sm3RURECABFvs6cOSPfz8nJSSxYsEAIIcTjZ2miw5SVReYZPXq00sf98eg9oWloKsy7fyouhj0VQgiRlJQkdHV1xY4dO0o9r7+CooTTnEPCZdoOAUD4+fkVu++NGzcEABEeHq7am0dE1dLF8KfCac6hQq9L4c+qumpUhZT9/WYHZTX07rvv4t133y1ym7Ozs8IspMXxv3EPEc/SsOliBH48HopU/dpovOAofIY0KfI5ubOzM+rXr49evXrhyZMn8PPzQ61atTBp0iR07doVADDIVRtfpCVCx7EZJm69ij2TOsDVygRt27aFv78/3n///WLrk5CSiYWH7gIAPm5liW8BmJmZFblvWloafH194eLiAgcHdl4mUgec64bKg4+xqJBdgZHosOQ0Rq4PwDcH7yI1KxetnU1xZFrnEjsEFgx7d3Nzw7Fjx/C///0PU6ZMwebNmwEA8fH5TdDN6jkhOTMXYzZdwfPULFhbWyMuLq7EOn297w6kGTloaFsDgduXo0OHDmjUqJHCPr///jtq1KiBGjVq4MiRIzhx4gR0dHTK+W4QUXXAuW6oPCRCmdv8l5w5cwbdunWrrPpUieTkZJiYmEAqlZZprhl1IU3PweFbMZi397ZCugTAudld4WBW8sJ3Ojo6aNWqFS5duiRPmzJlCgIDA+Hv749Lly6hQ4cOuB0Wgf/tiUDki3S0dDIFTq2AlqYGdu3aVeRx/7kVi0nbrkFLQ4JWT/7G5XOncOHCBdjb2yvWXypFQkICYmNj8eOPPyI6OhoXL16Enh7n4CFSF7HSDM51Q3LK/n6r/Bird+/esLe3x5gxYzB69Gg+JniDZefKcC0yERfCnuF8+DPcepJUaP0XIL/jzZPEzFKDneKGvf/9998AABsbGwBATmoSNn7SGoN/v4irjxORc+chhr7TochjJqZl4+v9+cGX9e1tuBh8DufOnSsU6AD5S1aYmJjAzc0N7dq1g6mpKfbu3YsPPvigtLeCiN4QnOuGykLlx1jR0dGYPHky/vrrL9SuXRu9evXCn3/+qTCSh6perDQDlx48UxipIIRAaHwKNlyIwBjfK2i28DjeX3cZq86E40ZUfqDjbG6AV2etUfa5eGnD3l1cXGBjY4NTp07B1aoG1n7YEho5GYgJu4XEGs5FHnPhobt4mpKFvAv/h0dXz+L06dMKa30VRwgBIQSysrJK3ZeIiNSbyi07FhYWmD59OqZPn45r167B19cXkyZNwqRJkzBy5EiMHTsWTZtW/KKPpLxdgZHw3nMLMgFoSIDhrRyQkydwIfwp4pMVf/wtauigg6sFOrpaoKObBWxN9LErMBLz9txGnhAqPRefPn062rdvj8WLF2P48OG4cuUK1q1bh3Xr1gEAJBIJpk2bhu+++w5ubm5wcXGBedB6RNUww7lsF/x99QmGtrRHjx49MHjwYDTsORx7g6OReGI1xIMLOHTgAIyMjOT9e0xMTKCvr4+HDx9i165d8PT0hKWlJZ48eYIlS5ZAX18fffv2rfg3mIiI3igq99l5VUxMDNatW4clS5ZAS0sLmZmZ8PDwwJo1a9CwYcOKqmelUqc+O7HSDHRYcrrIx1EAoKulgTYuZujoaoFObpZwtzGChkbhGYjL+lz80KFD8Pb2RlhYGFxcXDBjxgyMGzdOvl0IgQULFmDdunVISkpCx44d0ez9mfgrXAZtTQm2jm2LEd1b4oMPP8JZw66IlWbi8dKiR5b5+vrik08+QUxMDD777DNcvXoViYmJsLa2RufOnfH111+jXr16StediIjeLMr+fpcp2MnJycH+/fuxceNGnDhxAq1atcLYsWPxwQcf4OnTp/jqq69w7do13L17t1wnAeQ/NpszZw6OHDmC9PR0uLq6wtfXF61atQLw34/n+vXrkZSUhA4dOshHBClLnYKdSw+eYeT6gELp/ZvaYkQrR7RyNq2Q9akqkkwmMHnHNfxzKw41DbSx7qOWWOP3EKfvJ8DJ3ABHp3aGvk71qjMREVW9Suug/Pnnn2PHjh0QQuCjjz7CsmXLFIYAGxoa4scff4SdnV3Zav6SxMREdOjQAd26dcORI0dgaWmJsLAwmJqayvdZtmwZVq5cic2bN8PFxQXz589Hr169cPfu3bdyFE5xc1HM61u/2nbq09CQYPnwZohOuowbUUkYvvayfFuvBjYMdIiIqFxUbtnp0aMHPvvsMwwZMgS6urpF7pObm4uLFy+iS5cu5arc3LlzcfHiRZw/f77I7UII2NnZYebMmfjiiy8A5A8/tra2xqZNm0qcpO5l6tSyAwDTdgZj3/UYAP/NRTGitWMV16p0t6OlePfXCwppmhIJLsztVm0DNSIiqjrK/n6rPBrr1KlT+OCDD4oNdABAS0ur3IEOABw4cACtWrXCsGHDYGVlhebNm2P9+vXy7REREYiLi0PPnj3laSYm/83IW5ysrCwkJycrvNRJSmYuAGBEawdcmNvtjQh0ACA5M6dQWp4QePQsvQpqQ0RE6kLlYMfHxwcbN24slL5x40YsXbq0QipVoLQZeQtG5VhbWyvkK21GXh8fH/mcLCYmJmo1V1BWbh4u/bsy+UftnN6oFpGCR3Av43TwRERUXioHO2vXroW7u3uh9IYNG2LNmjUVUqkCMpkMLVq0wOLFi9G8eXOMHz8e48aNK3c53t7ekEql8ldUVFQF1bjqBUYkIiMnD5ZGumho92Y9kuN08EREVBlU7qAcFxcHW9vC6yNZWloiNja2QipVQNkZeePj4xXqFB8fj2bNmhV7XF1d3RIfw73JzoYkAAC61LWERFJ4SHl1N6K1IzrXteR08EREVGFUbtlxcHDAxYsXC6VfvHixQkZgvUyVGXkLJCcnIyAgAB4eHhValzfF2dCnAICu9SyruCZlZ2uiD4865gx0iIioQqjcsjNu3DhMmzYNOTk56N69O4D8TsuzZ8/GzJkzK7RyZZmRd/78+bCzs8OgQYMqtC5vgieJ6QhPSIWGBOjk+uYGO0RERBVJ5WBn1qxZeP78OSZNmiRfD0tPTw9z5syBt7d3hVaudevW2Lt3L7y9vbFw4UK4uLjg559/xqhRo+T7zJ49G2lpaRg/frx8Rt6jR4++lXPsnA3Jb9Vp4WgKEwPtKq4NERFR9VDm5SJSU1Nx79496Ovrw83N7Y3uA6Mu8+x8tjkIJ+/F4wvPupjcXfkZpImIiN5ElTaDcoEaNWqgdevWZc1OFSx/yPkzAEDXelZVXBsiIqLqo0zBTlBQEP78809ERkbKH2UV2LNnT4VUjFQT9CgR6dl5sKihiwa2b27rFBERUUVTeTTWzp070b59e9y7dw979+5FTk4O7ty5g9OnT8PExKQy6khKeHnIeVGrmBMREb2tVA52Fi9ejBUrVuDgwYPQ0dHBL7/8gvv372P48OFwdHwzliVQRwWdk7u8wUPOiYiIKoPKwc6DBw/Qr18/AICOjg7S0tIgkUgwffp0+ZBwer2ikzIQ9u+Q885uFlVdHSIiompF5WDH1NQUKSkpAIBatWrh9u3bAICkpCSkp3PBxqrg92+rTjOHmqhpoFPFtSEiIqpeVO6g3LlzZ5w4cQKNGzfGsGHDMHXqVJw+fRonTpxAjx49KqOOVIqC/jochUVERFSYysHOqlWrkJmZCQD48ssvoa2tjUuXLmHo0KH46quvKryCVLLsXBkuhhcMOWd/HSIiolepFOzk5ubi0KFD6NWrFwBAQ0MDc+fOrZSKkXKCHr9AWnYeLGrooJEdR8MRERG9SqU+O1paWpg4caK8ZYeqXkF/nc5uHHJORERUFJU7KLdp0wbXr1+vhKpQWXDIORERUclU7rMzadIkzJgxA1FRUWjZsiUMDQ0Vtjdp0qTCKkcli0nKQEh8yr9DzhnsEBERFUXlYOf9998HAEyZMkWeJpFIIISARCJBXl5exdWOSuQXmt+q09ShJkwNOeSciIioKCoHOxEREZVRDyqDl5eIICIioqKpHOw4OTlVRj1IRTl5MlwMfw6A8+sQERGVROVgZ8uWLSVu//jjj8tcGVLe1ceJSM3KhZmhDprU4pBzIiKi4qgc7EydOlXh75ycHKSnp0NHRwcGBgYMdl6Ts/Ih5xYcck5ERFQClYeeJyYmKrxSU1MREhKCjh07YseOHZVRRyoCl4ggIiJSjsrBTlHc3NywZMmSQq0+VDnipJm4H5cCiQTozM7JREREJaqQYAfIn105Jiamog5HJfALzW/VaWJfE2Ycck5ERFQilfvsHDhwQOFvIQRiY2OxatUqdOjQocIqRsUr6K/Tla06REREpVI52Bk0aJDC3xKJBJaWlujevTt++umniqoXFSMnT4YLYVzlnIiISFkqBzsymawy6kFKuvY4ESlZuTA10EYT+5pVXR0iIqJqr8L67NDrcfbfJSI6uVlCk0POiYiISqVysDN06FAsXbq0UPqyZcswbNiwCqkUFc+voL8OH2EREREpReVg59y5c+jbt2+h9D59+uDcuXMVUikqWkJyJu7GJgPgkHMiIiJlqRzspKamQken8HBnbW1tJCcnV0ilqGgFj7Ca2JvAooZuFdeGiIjozaBysNO4cWPs2rWrUPrOnTvRoEGDCqkUFc2PQ86JiIhUpvJorPnz52PIkCF48OABunfvDgA4deoUduzYgd27d1d4BSlfbp4M58Pyg50uXCKCiIhIaSoHO/3798e+ffuwePFi/PXXX9DX10eTJk1w8uRJdOnSpTLqSACCo5KQnJmLmgbaaOZQs6qrQ0RE9MZQOdgBgH79+qFfv34VXRcqQcHCnxxyTkREpBqV++wEBgYiICCgUHpAQACCgoIqpFLqZsmSJZBIJJg2bZpCur+/P7p37w5DQ0MYGxujc+fOyMjIKPIYZ0OeQur/J4589wmMjIxgZWWFQYMGISQkRGG/devWoWvXrjA2NoZEIkFSUlIlnRUREdGbQeVgx8vLC1FRUYXSo6Oj4eXlVSGVUieBgYFYu3YtmjRpopDu7++P3r17w9PTE1euXEFgYCAmT54MDY3CH0lCSibuxCQjM+o2pn4+GZcvX8aJEyeQk5MDT09PpKWlyfdNT09H7969MW/evEo/NyIiojeByo+x7t69ixYtWhRKb968Oe7evVshlVIXqampGDVqFNavX4/vvvtOYdv06dMxZcoUzJ07V55Wr169Io9TMAqrx/Sf8fnETvL0TZs2wcrKClevXkXnzp0BQN56dPbs2Qo8EyIiojeXyi07urq6iI+PL5QeGxsLLa0ydQFSW15eXujXrx969uypkJ6QkICAgABYWVmhffv2sLa2RpcuXXDhwoUij1Mwv07XuoqjsKRSKQDAzMysEmpPRESkHlQOdjw9PeHt7S3/oQWApKQkzJs3D++8806FVu5NtnPnTly7dg0+Pj6Ftj18+BAA8M0332DcuHE4evQoWrRogR49eiAsLExh39xiVjmXyWSYNm0aOnTogEaNGlXimRAREb3ZVG6K+fHHH9G5c2c4OTmhefPmAIDr16/D2toaf/zxR4VX8E0UFRWFqVOn4sSJE9DT0yu0vWDl+AkTJmDMmDEA8h8Dnjp1Chs3blQIkG48SYI0IwfGeloKQ869vLxw+/btYluDiIiIKJ/KwU6tWrVw8+ZNbNu2DTdu3IC+vj7GjBmDDz74ANra2pVRxzfO1atXkZCQoNC3KS8vD+fOncOqVavkI6henXG6fv36iIyMVEg7+29/nU51LaGlmd8QN3nyZBw6dAjnzp2Dvb19ZZ4KERHRG69MnWwMDQ0xfvz4iq6L2ujRowdu3bqlkDZmzBi4u7tjzpw5qF27Nuzs7AoNGw8NDUWfPn0U0s6+tESEEAKff/459u7di7Nnz8LFxaVyT4SIiEgNlLlH8d27dxEZGYns7GyF9AEDBpS7Um86IyOjQv1oDA0NYW5uLk+fNWsWFixYgKZNm6JZs2bYvHkz7t+/j7/++kuep3PXbritXQ/GLfujSz1LeHl5Yfv27di/fz+MjIwQFxcHADAxMYG+vj4AIC4uDnFxcQgPDwcA3Lp1C0ZGRnB0dGRHZiIieiupHOw8fPgQgwcPxq1btyCRSCCEAABIJPmz+ubl5VVsDdXUtGnTkJmZienTp+PFixdo2rQpTpw4gTp16sj3uR8aDlltWzS0M4aVkR5Wr14NAOjatavCsXx9ffHJJ58AANasWYNvv/1Wvq1gSPrL+xAREb1NJKIgWlFS//79oampif/7v/+Di4sLrly5gufPn2PmzJn48ccf0alTp9IPUs0kJyfDxMQEUqkUxsbGVV0duc82B+HkvXiM9nDCtwM54oqIiOhlyv5+qzz03N/fHwsXLoSFhQU0NDSgoaGBjh07wsfHB1OmTClXpek/O65E4uS9/PmMtlx+jF2BkaXkICIioqKoHOzk5eXByMgIAGBhYYGYmBgAgJOTU6EOt1Q2sdIMzNv7XwdnIYB5e24jVlr0ullERERUPJX77DRq1Ag3btyAi4sL2rZti2XLlkFHRwfr1q1D7dq1K6OOb52IZ2l49eFinhB49Cwdtib6VVMpIiKiN5TKwc5XX30lX3hy4cKFePfdd9GpUyeYm5tj165dFV7Bt5Hmv529X01ztjCogtoQERG92VQOdnr16iX/t6urK+7fv48XL17A1NRUPiKLymff9RiFvzUlEiwe0oitOkRERGVQISt3cv6WipOQkom/rz0BAKwe1QI1DXTgbGHAQIeIiKiMuEx5NbPp4iNk58rQwrEmejeyYWsZERFROak8GosqT2pWLv64/BgAMKFLHQY6REREFYDBTjWyIyASKZm5qG1piHfqW1d1dYiIiNSCysHOuXPnkJubWyg9NzcX586dq5BKvY2yc2XYcCECADChc21oaLBVh4iIqCKoHOx069YNL168KJQulUrRrVu3CqnU22j/9WjEJWfCykgXg5rXqurqEBERqQ2Vgx0hRJF9SZ4/fw5DQ8MKqdTbRiYTWHfuIQDg044u0NXSrOIaERERqQ+lR2MNGTIEQP7q5p988gl0dXXl2/Ly8nDz5k20b9++4mv4Fjh9PwFhCakw0tXCyLaOVV0dIiIitaJ0sGNiYgIgv2XHyMgI+vr/zfuio6ODdu3aYdy4cRVfw7fA2nMPAAAj2znCWE+7imtDRESkXpQOdnx9fQEAzs7O+OKLL/jIqoJcffwCgY8SoaOpgU87uFR1dYiIiNSOyn12Zs+erdBn5/Hjx/j5559x/PjxCq3Y22KNX35fncHNa8HaWK+Ka0NERKR+VA52Bg4ciC1btgAAkpKS0KZNG/z0008YOHAgVq9eXeEVVGfhCak4cTceEgkwrjNXjCciIqoMKgc7165dQ6dOnQAAf/31F2xsbPD48WNs2bIFK1eurPAKqrN1//bVeae+NVytalRxbYiIiNSTysFOeno6jIyMAADHjx/HkCFDoKGhgXbt2uHx48cVXkF1FZ+cib3B0QDyl4YgIiKiyqFysOPq6op9+/YhKioKx44dg6enJwAgISEBxsbGFV5BdbXxQgRy8gTaOJuhpZNpVVeHiIhIbakc7Hz99df44osv4OzsjDZt2sDDwwNAfitP8+bNK7yC6ig5MwfbAiIBABO6sK8OERFRZVI52HnvvfcQGRmJoKAgHDt2TJ7eo0cPrFixokIr96olS5ZAIpFg2rRp8rTMzEx4eXnB3NwcNWrUwNChQxEfH1+p9SivbZcjkZqVCzerGuhWz6qqq0NERKTWyrTquY2NDYyMjHDixAlkZGQAAFq3bg13d/cKrdzLAgMDsXbtWjRp0kQhffr06Th48CB2794NPz8/xMTEyGd7ro6ycvOw8eK/C352qcMFP4mIiCqZysHO8+fP0aNHD9StWxd9+/ZFbGwsAGDs2LGYOXNmhVcQAFJTUzFq1CisX78epqb/9W+RSqXYsGEDli9fju7du6Nly5bw9fXFpUuXcPny5UqpS3ntC47G05Qs2JroYUBTu6quDhERkdpTOdiZPn06tLW1ERkZCQMDA3n6iBEjcPTo0QqtXAEvLy/069cPPXv2VEi/evUqcnJyFNLd3d3h6OgIf3//Yo+XlZWF5ORkhdfrIJMJrP13wc+xHV2go1WmhjUiIiJSgdLLRRQ4fvw4jh07Bnt7e4V0Nze3Shl6vnPnTly7dg2BgYGFtsXFxUFHRwc1a9ZUSLe2tkZcXFyxx/Tx8cG3335b0VUt1Yl78Xj4NA3Gelp4vw0X/CQiInodVG5aSEtLU2jRKfDixQuFldArQlRUFKZOnYpt27ZBT6/illLw9vaGVCqVv6Kioirs2MURQmCNX/4kgh95OKGGrspxJhEREZWBysFOp06d5MtFAIBEIoFMJsOyZcvQrVu3Cq3c1atXkZCQgBYtWkBLSwtaWlrw8/PDypUroaWlBWtra2RnZyMpKUkhX3x8PGxsbIo9rq6uLoyNjRVelS3wUSKCI5Ogo6WBT9pzwU8iIqLXReXmhWXLlqFHjx4ICgpCdnY2Zs+ejTt37uDFixe4ePFihVauR48euHXrlkLamDFj4O7ujjlz5sDBwQHa2to4deoUhg4dCgAICQlBZGSkfP6f6qKgVee9lvawNKrYFjAiIiIqnsrBTqNGjRAaGopVq1bByMgIqampGDJkCLy8vGBra1uhlTMyMkKjRo0U0gwNDWFubi5PHzt2LGbMmAEzMzMYGxvj888/h4eHB9q1a1ehdSmPkLgUnL6fkL/gZydOIkhERPQ6qRzsREZGwsHBAV9++WWR2xwdX2/H2xUrVkBDQwNDhw5FVlYWevXqhd9///211qE0a/9d8LN3Qxu4WBhWcW2IiIjeLhIhhFAlg6amJmJjY2FlpTjz7/Pnz2FlZYW8vLwKreDrkJycDBMTE0il0grvvxOTlIHOy84gVyaw36sDmjrUrNDjExERva2U/f1WuYOyEAISSeFZf1NTUyt0xJS62HghArkygXa1zRjoEBERVQGlH2PNmDEDQP7oq/nz5ysMP8/Ly0NAQACaNWtW4RV8k4XGpWDr5fy5hyZ2qVPFtSEiIno7KR3sBAcHA8hv2bl16xZ0dHTk23R0dNC0aVN88cUXFV/DN9SuwEjM/fsWCp4Rxkkzq7Q+REREbyulg50zZ84AyB/6/csvv7yWuWneVLHSDHjv+S/QAYAv995Gl3qWsDXRr7J6ERERvY1U7rPj6+vLQKcUEc/SIHul23eeEHj0LL1qKkRERPQW40qUlcDFwhAar/Th1pRI4GxReJkNIiIiqlwMdiqBrYk+fIY0hua/o9Y0JRIsHtKIj7CIiIiqAFejrCQjWjuic11LPHqWDmcLAwY6REREVYTBTiWyNdFnkENERFTF+BiLiIiI1BqDHSIiIlJrDHaIiIhIrTHYISIiIrXGYIeIiIjUGoMdIiIiUmsMdoiIiEitMdghIiIitcZgh4iIiNQagx0iIiJSawx2iIiISK0x2CEiIiK1xmCHiIiI1BqDHSIiIlJrDHaIiIhIrTHYISIiIrXGYIeIiIjUGoMdIiIiUmsMdoiIiEitMdghIiIitcZgh4iIiNQagx0iIiJSawx2iIiISK0x2CEiIiK1xmCHiIiI1BqDHSIiIlJrDHaIiIhIrTHYISIiIrXGYIeIiIjUGoMdIiIiUmsMdoiIiEitMdghIiIitcZgh4iIiNQagx0iIiJSawx2iIiISK0x2CEiIiK1xmCHiIiI1BqDHSIiIlJrDHaIiIhIrTHYISIiIrXGYIeIiIjUGoMdIiIiUmsMdoiIiEitMdghIiIitcZgh4iIiNQagx0iIiJSawx2iIiISK0x2CEiIiK1xmCHiIiI1BqDHSIiIlJrDHaIiIhIrTHYISIiIrXGYIeIiIjUGoMdIiIiUmvVOtjx8fFB69atYWRkBCsrKwwaNAghISEK+2RmZsLLywvm5uaoUaMGhg4divj4+CqqMREREVU31TrY8fPzg5eXFy5fvowTJ04gJycHnp6eSEtLk+8zffp0HDx4ELt374afnx9iYmIwZMiQKqw1ERERVScSIYSo6koo6+nTp7CysoKfnx86d+4MqVQKS0tLbN++He+99x4A4P79+6hfvz78/f3Rrl07pY6bnJwMExMTSKVSGBsbV+YpEBERUQVR9ve7WrfsvEoqlQIAzMzMAABXr15FTk4OevbsKd/H3d0djo6O8Pf3L/Y4WVlZSE5OVngRERGRenpjgh2ZTIZp06ahQ4cOaNSoEQAgLi4OOjo6qFmzpsK+1tbWiIuLK/ZYPj4+MDExkb8cHBwqs+pERERUhd6YYMfLywu3b9/Gzp07y30sb29vSKVS+SsqKqoCakhERETVkVZVV0AZkydPxqFDh3Du3DnY29vL021sbJCdnY2kpCSF1p34+HjY2NgUezxdXV3o6upWZpWJiIiomqjWLTtCCEyePBl79+7F6dOn4eLiorC9ZcuW0NbWxqlTp+RpISEhiIyMhIeHx+uuLhEREVVD1bplx8vLC9u3b8f+/fthZGQk74djYmICfX19mJiYYOzYsZgxYwbMzMxgbGyMzz//HB4eHkqPxCIiIiL1Vq2HnkskkiLTfX198cknnwDIn1Rw5syZ2LFjB7KystCrVy/8/vvvJT7GehWHnhMREb15lP39rtbBzuvCYIeIiOjNo5bz7BARERGpisEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTW1CbY+e233+Ds7Aw9PT20bdsWV65cqeoqERERUTWgFsHOrl27MGPGDCxYsADXrl1D06ZN0atXLyQkJFR11YiIiKiKqUWws3z5cowbNw5jxoxBgwYNsGbNGhgYGGDjxo1VXTUiIiKqYlpVXYHyys7OxtWrV+Ht7S1P09DQQM+ePeHv719knqysLGRlZcn/lkqlAIDk5OTKrSwRERFVmILfbSFEifu98cHOs2fPkJeXB2tra4V0a2tr3L9/v8g8Pj4++PbbbwulOzg4VEodiYiIqPKkpKTAxMSk2O1vfLBTFt7e3pgxY4b8b5lMhhcvXsDc3BwSiaTCyklOToaDgwOioqJgbGz8WvOz7Ndfdnnzs+y3q+zy5mfZLPtNyV/esksihEBKSgrs7OxK3O+ND3YsLCygqamJ+Ph4hfT4+HjY2NgUmUdXVxe6uroKaTVr1qysKsLY2LhcH3B58rPs1192efOz7Ler7PLmZ9ks+03JX96yi1NSi06BN76Dso6ODlq2bIlTp07J02QyGU6dOgUPD48qrBkRERFVB298yw4AzJgxA6NHj0arVq3Qpk0b/Pzzz0hLS8OYMWOqumpERERUxdQi2BkxYgSePn2Kr7/+GnFxcWjWrBmOHj1aqNPy66arq4sFCxYUemT2OvKz7Ndfdnnzs+y3q+zy5mfZLPtNyV/esiuCRJQ2XouIiIjoDfbG99khIiIiKgmDHSIiIlJrDHaIiIhIrTHYISIiIrXGYKcS/fbbb3B2doaenh7atm2LK1euKJXv3Llz6N+/P+zs7CCRSLBv3z6ly/Tx8UHr1q1hZGQEKysrDBo0CCEhIUrnX716NZo0aSKf/MnDwwNHjhxROv/LlixZAolEgmnTpim1/zfffAOJRKLwcnd3V7q86OhofPjhhzA3N4e+vj4aN26MoKAgpfI6OzsXKlsikcDLy6vUvHl5eZg/fz5cXFygr6+POnXqYNGiRaWu1fKylJQUTJs2DU5OTtDX10f79u0RGBhYaL/Srg0hBL7++mvY2tpCX18fPXv2RFhYmNL59+zZA09PT/ls4tevX1e6/JycHMyZMweNGzeGoaEh7Ozs8PHHHyMmJkapsr/55hu4u7vD0NAQpqam6NmzJwICApSu+8smTpwIiUSCn3/+Wam8n3zySaHPvnfv3iqVfe/ePQwYMAAmJiYwNDRE69atERkZWWreoq47iUSCH374QamyU1NTMXnyZNjb20NfX1++GLIyeePj4/HJJ5/Azs4OBgYG6N27t/x6Uea7JDMzE15eXjA3N0eNGjUwdOhQ+QSvyuRft24dunbtCmNjY0gkEiQlJcm3lZb/xYsX+Pzzz1GvXj3o6+vD0dERU6ZMgVQqVarsCRMmoE6dOtDX14elpSUGDhwoX2JIle9RIQT69Okjf3+Vydu1a9dCn/fEiRNVKtvf3x/du3eHoaEhjI2N0blzZyxcuLDEvI8ePSr2etu9e7dSZcfFxeGjjz6CjY0NDA0N0aJFC/z9999K5X3w4AEGDx4MS0tLGBsbY/jw4YUmBK4sDHYqya5duzBjxgwsWLAA165dQ9OmTdGrVy8kJCSUmjctLQ1NmzbFb7/9pnK5fn5+8PLywuXLl3HixAnk5OTA09MTaWlpSuW3t7fHkiVLcPXqVQQFBaF79+4YOHAg7ty5o1I9AgMDsXbtWjRp0kSlfA0bNkRsbKz8deHCBaXyJSYmokOHDtDW1saRI0dw9+5d/PTTTzA1NVW6vi+Xe+LECQDAsGHDSs27dOlSrF69GqtWrcK9e/ewdOlSLFu2DL/++qtSZQPAZ599hhMnTuCPP/7ArVu34OnpiZ49eyI6Olphv9KujWXLlmHlypVYs2YNAgICYGhoiF69eiEzM1Op/GlpaejYsSOWLl1a7Pbi8qenp+PatWuYP38+rl27hj179iAkJAQDBgxQquy6deti1apVuHXrFi5cuABnZ2d4enri6dOnSuUvsHfvXly+fFlh+nhl8vbu3VvhGtixY4fS+R88eICOHTvC3d0dZ8+exc2bNzF//nzo6emVmvflMmNjY7Fx40ZIJBIMHTpUqbJnzJiBo0ePYuvWrbh37x6mTZuGyZMn48CBAyXmFUJg0KBBePjwIfbv34/g4GA4OTmhZ8+eSEtLU+q7ZPr06Th48CB2794NPz8/xMTEYMiQIQCU+y5KT09H7969MW/evEL1Ky1/TEwMYmJi8OOPP+L27dvYtGkTjh49irFjxypVdsuWLeHr64t79+7h2LFjEELA09MTeXl5Kn2P/vzzzwrLDCmbd9y4cQqf+7Jly5TO7+/vj969e8PT0xNXrlxBYGAgJk+ejAsXLpSY18HBodD19u2336JGjRro06ePUmV//PHHCAkJwYEDB3Dr1i0MGTIEw4cPx8GDB0vMm5aWBk9PT0gkEpw+fRoXL15EdnY2+vfvD5lMVuh9rXCCKkWbNm2El5eX/O+8vDxhZ2cnfHx8VDoOALF3794y1yMhIUEAEH5+fmU+hqmpqfi///s/pfdPSUkRbm5u4sSJE6JLly5i6tSpSuVbsGCBaNq0aZnqOGfOHNGxY8cy5S3K1KlTRZ06dYRMJit13379+olPP/1UIW3IkCFi1KhRSpWVnp4uNDU1xaFDhxTSW7RoIb788sti8716bchkMmFjYyN++OEHeVpSUpLQ1dUVO3bsKDX/yyIiIgQAERwcrHT5Rbly5YoAIB4/fqxyXqlUKgCIkydPKl32kydPRK1atcTt27eFk5OTWLFihVJ5R48eLQYOHFhifUrKP2LECPHhhx+WKe+rBg4cKLp37650/oYNG4qFCxcqpBV17byaNyQkRAAQt2/flqfl5eUJS0tLsX79+kJlv/pdkpSUJLS1tcXu3bvl+9y7d08AEP7+/qXmf9mZM2cEAJGYmFjkeZeWv8Cff/4pdHR0RE5Ojsp5b9y4IQCI8PBwpcsODg4WtWrVErGxscV+tkXlVeV7saj8bdu2FV999VWZ8r6qWbNmhb6/SspvaGgotmzZorCfmZlZoWvm1bzHjh0TGhoaQiqVyvdJSkoSEolEnDhxotRzKS+27FSC7OxsXL16FT179pSnaWhooGfPnvD393+tdZFKpQAAMzMzlfPm5eVh586dSEtLU2npDS8vL/Tr10/h/JUVFhYGOzs71K5dG6NGjUJkZKRS+Q4cOIBWrVph2LBhsLKyQvPmzbF+/XqVywfyP7+tW7fi008/VWph2Pbt2+PUqVMIDQ0FANy4cQMXLlxAnz59lCovNzcXeXl50NPTU0jX19dXumULACIiIhAXF6fwvpuYmKBt27av/borIJVKIZFIVF57Ljs7G+vWrYOJiQmaNm2qVB6ZTIaPPvoIs2bNQsOGDVWu69mzZ2FlZYV69erhf//7H54/f650uYcPH0bdunXRq1cvWFlZoW3btio9fi4QHx+Pw4cPY+zYsUrnad++PQ4cOIDo6GgIIXDmzBmEhobC09OzxHxZWVkAoHDdaWhoQFdXt8jr7tXvkqtXryInJ0fhenN3d4ejo2OR11t5vouUzS+VSmFsbAwtLa1C6SXlTUtLg6+vL1xcXODg4KBU2enp6Rg5ciR+++23YtdhLKnsbdu2wcLCAo0aNYK3tzfS09OVyp+QkICAgABYWVmhffv2sLa2RpcuXZT6zF519epVXL9+vdjrraj87du3x65du/DixQvIZDLs3LkTmZmZ6Nq1a4l5s7KyIJFIFCYW1NPTg4aGhkrfc2VW6eHUWyg6OloAEJcuXVJInzVrlmjTpo1Kx0I5Wnby8vJEv379RIcOHVTKd/PmTWFoaCg0NTWFiYmJOHz4sNJ5d+zYIRo1aiQyMjKEEKrdwfzzzz/izz//FDdu3BBHjx4VHh4ewtHRUSQnJ5eaV1dXV+jq6gpvb29x7do1sXbtWqGnpyc2bdqkdN0L7Nq1S2hqaoro6Gil9s/LyxNz5swREolEaGlpCYlEIhYvXqxSmR4eHqJLly4iOjpa5Obmij/++ENoaGiIunXrFpvn1Wvj4sWLAoCIiYlR2G/YsGFi+PDhpeZ/WUW07GRkZIgWLVqIkSNHKp334MGDwtDQUEgkEmFnZyeuXLmidNmLFy8W77zzjrw1TpWWnR07doj9+/eLmzdvir1794r69euL1q1bi9zc3FLzF9zVGxgYiOXLl4vg4GDh4+MjJBKJOHv2rFLnXWDp0qXC1NRU/v9HmbpnZmaKjz/+WAAQWlpaQkdHR2zevLnUvNnZ2cLR0VEMGzZMvHjxQmRlZYklS5YIAMLT01Mhb1HfJdu2bRM6OjqFymndurWYPXt2qflfVlrLjjLfZU+fPhWOjo5i3rx5Suf97bffhKGhoQAg6tWrV2SrTnH5x48fL8aOHSv/u6jPpri8a9euFUePHhU3b94UW7duFbVq1RKDBw9Wqmx/f38BQJiZmYmNGzeKa9euiWnTpgkdHR0RGhqq1HkX+N///ifq169f5Lbi8icmJgpPT0/59WZsbCyOHTtWat6EhARhbGwspk6dKtLS0kRqaqqYPHmyACDGjx9fbB0rCoOdSlBdgp2JEycKJycnERUVpVK+rKwsERYWJoKCgsTcuXOFhYWFuHPnTqn5IiMjhZWVlbhx44Y8TZVg51WJiYnC2NhYqUdo2trawsPDQyHt888/F+3atVO5XE9PT/Huu+8qvf+OHTuEvb292LFjh7h586bYsmWLMDMzUynQCg8PF507dxYAhKampmjdurUYNWqUcHd3LzZPdQ52srOzRf/+/UXz5s0Vmq1Ly5uamirCwsKEv7+/+PTTT4Wzs7OIj48vNX9QUJCwtrZWCFBVCXZe9eDBA6UfoRX8f//ggw8U9uvfv794//33VSq7Xr16YvLkycVuLyr/Dz/8IOrWrSsOHDggbty4IX799VdRo0aNQo8GisobFBQkmjZtKr/uevXqJfr06SN69+6tsF9R3yWqBDulfReVFuyUll8qlYo2bdqI3r17i+zsbKXzJiUlidDQUOHn5yf69+8vWrRoUSjQLCr//v37haurq0hJSZGnFfX+KvsdfOrUqSIfoRWVv+D/ube3t8K+jRs3FnPnzlW67PT0dGFiYiJ+/PHHIrcXl3/y5MmiTZs24uTJk+L69evim2++ESYmJuLmzZul5j127JioXbu2kEgkQlNTU3z44YeiRYsWYuLEiSW8OxWDwU4lyMrKEpqamoUu/I8//lgMGDBApWOVNdjx8vIS9vb24uHDhyrnfVWPHj2Uirz37t0r/9IseAGQX9hF3SWXplWrVgr/gYvj6OiocJclhBC///67sLOzU6m8R48eCQ0NDbFv3z6l89jb24tVq1YppC1atEjUq1dPpbKFyP+xLwhWhg8fLvr27Vvsvq9eGwU/0K8GKJ07dxZTpkwpNf/LyhPsZGdni0GDBokmTZqIZ8+eqZT3Va6urkW2kr2af8WKFfLr7OVrT0NDQzg5OZWpbAsLC7FmzZpSy87KyhJaWlpi0aJFCvvNnj1btG/fXumyz507JwCI69evF1unV/Onp6cLbW3tQv29xo4dK3r16qV02UlJSSIhIUEIkd/fcNKkSfJtxX2XFPxAvxqgODo6iuXLl5ea/2UlBTul5U9OThYeHh6iR48ehQIVVb4Hs7KyhIGBgdi+fXup+adOnVrs9dalSxeVy05NTRUAxNGjR0st++HDhwKA+OOPPxTShw8fLm9FVabsLVu2CG1tbfnn/rLi8oeHhxfq5yVE/m/EhAkTlC776dOn8s/a2tpaLFu2rNh9Kwr77FQCHR0dtGzZEqdOnZKnyWQynDp1SqW+L2UhhMDkyZOxd+9enD59Gi4uLuU+pkwmkz/fL0mPHj1w69YtXL9+Xf5q1aoVRo0ahevXr0NTU1OlclNTU/HgwQPY2tqWum+HDh0KDXMMDQ2Fk5OTSmX6+vrCysoK/fr1UzpPeno6NDQU/ytpamqWaYSBoaEhbG1tkZiYiGPHjmHgwIFK53VxcYGNjY3CdZecnIyAgIBKv+4K5OTkYPjw4QgLC8PJkydhbm5eruMpe+199NFHuHnzpsK1Z2dnh1mzZuHYsWMql/vkyRM8f/5cqWtPR0cHrVu3Lvf1t2HDBrRs2VLpPkpA/vudk5NT7uvPxMQElpaWCAsLQ1BQEAYOHFjqd0nLli2hra2tcL2FhIQgMjISHh4e5f4uUiZ/cnIyPD09oaOjgwMHDsj7H5WlbJF/84+srKxS88+dO7fQ9QYAK1aswMaNG1UuuyC/ra1tqWU7OzvDzs6uyOvN0dFR6bI3bNiAAQMGwNLSUuE9KCl/Qb+ioq63vLw8pcu2sLBAzZo1cfr0aSQkJMhHbFaqSg+n3lI7d+4Uurq6YtOmTeLu3bti/PjxombNmiIuLq7UvCkpKSI4OFgEBwcLAPJ+AK+OaCnK//73P2FiYiLOnj0rYmNj5a/09HSl6j137lzh5+cnIiIixM2bN8XcuXOFRCIRx48fVyr/q1R5jDVz5kxx9uxZERERIS5evCh69uwpLCwsirzzeNWVK1eElpaW+P7770VYWJjYtm2bMDAwEFu3blW6rnl5ecLR0VHMmTNH6TxC5I/kqVWrljh06JCIiIgQe/bsERYWFoWa8kty9OhRceTIEfHw4UNx/Phx0bRpU9G2bdtCTfKlXRtLliwRNWvWlPc/GThwoHBxcZHf8ZaW//nz5yI4OFgcPnxYABA7d+4UwcHBIjY2ttT82dnZYsCAAcLe3l5cv35d4frLysoqMW9qaqrw9vYW/v7+4tGjRyIoKEiMGTNG6Orqyu8iVf1/8fJjrJLypqSkiC+++EL4+/uLiIgIcfLkSdGiRQvh5uYmMjMzlSp7z549QltbW6xbt06EhYWJX3/9VWhqaorz588rVW+pVCoMDAzE6tWrC51Hafm7dOkiGjZsKM6cOSMePnwofH19hZ6envj9999Lzfvnn3+KM2fOiAcPHoh9+/YJJycnMWTIECGEct8lEydOFI6OjuL06dMiKChIeHh4yB8nK5M/NjZWBAcHi/Xr1wsA4ty5cyI4OFg8f/681PxSqVS0bdtWNG7cWISHhyvsM3HixBLzPnjwQCxevFgEBQWJx48fi4sXL4r+/fsLMzMzER8fX6bvUfzbclZa3vDwcLFw4UIRFBQkIiIixP79+0Xt2rVF586dlX7fVqxYIYyNjcXu3btFWFiY+Oqrr4Senp4YOXKkUvUOCwsTEolEHDlyRCG9tLKzs7OFq6ur6NSpkwgICBDh4eHixx9/FBKJRPTt27fUsjdu3Cj8/f1FeHi4+OOPP4SZmZmYMWNGse9pRWKwU4l+/fVX4ejoKHR0dESbNm3E5cuXlcpX0KT76mv06NGl5i0qHwDh6+urVNmffvqpcHJyEjo6OsLS0lL06NGjzIGOEKoFOyNGjBC2trZCR0dH1KpVS4wYMaLIDoPFOXjwoGjUqJHQ1dUV7u7uYt26dSrV9dixYwKACAkJUSlfcnKymDp1qnB0dBR6enqidu3a4ssvvxRZWVlKH2PXrl2idu3aQkdHR9jY2AgvLy+RlJRUaL/Srg2ZTCbmz58vrK2tha6urujRo4fC+ZSW39fXt8jtCxYsKDV/waOvol5nzpwpMW9GRoYYPHiwsLOzEzo6OsLW1lYMGDBAoYOyqv8vXg52Ssqbnp4uPD09haWlpdDW1hZOTk5i3LhxCjcmypS9YcMG4erqKvT09ETTpk3lj0KVybt27Vqhr69fps88NjZWfPLJJ8LOzk7o6emJevXqiZ9++knIZLJS8/7yyy/C3t5eaGtrC0dHR/HVV1/Jr1tlvksyMjLEpEmThKmpqTAwMBCDBw+WB8bK5F+wYEGx+5SWv7hzK+lVkDc6Olr06dNHWFlZCW1tbWFvby9Gjhwp7t+/r3TdX1UQ7JSWNzIyUnTu3FmYmZkJXV1d4erqKmbNmiXv26Zs2T4+PsLe3l4YGBgIDw8Pcf78eaXzent7CwcHB5GXl1foHErLHxoaKoYMGSKsrKyEgYGBaNKkidiyZYtSeefMmSOsra2Ftra2cHNzk1+nr4Pk3xMkIiIiUkvss0NERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BARveLs2bOQSCRISkqq6qoQUQVgsENERERqjcEOERERqTUGO0RU7chkMvj4+MDFxQX6+vpo2rQp/vrrLwD/PWI6fPgwmjRpAj09PbRr1w63b99WOMbff/+Nhg0bQldXF87Ozvjpp58UtmdlZWHOnDlwcHCArq4uXF1dsWHDBoV9rl69ilatWsHAwADt27cvtNI0Eb0ZGOwQUbXj4+ODLVu2YM2aNbhz5w6mT5+ODz/8EH5+fvJ9Zs2ahZ9++gmBgYGwtLRE//79kZOTAyA/SBk+fDjef/993Lp1C9988w3mz5+PTZs2yfN//PHH2LFjB1auXIl79+5h7dq1qFGjhkI9vvzyS/z0008ICgqClpYWPv3009dy/kRUsbgQKBFVK1lZWTAzM8PJkyfh4eEhT//ss8+Qnp6O8ePHo1u3bti5cydGjBgBAHjx4gXs7e2xadMmDB8+HKNGjcLTp09x/Phxef7Zs2fj8OHDuHPnDkJDQ1GvXj2cOHECPXv2LFSHs2fPolu3bjh58iR69OgBAPjnn3/Qr18/ZGRkQE9Pr5LfBSKqSGzZIaJqJTw8HOnp6XjnnXdQo0YN+WvLli148OCBfL+XAyEzMzPUq1cP9+7dAwDcu3cPHTp0UDhuhw4dEBYWhry8PFy/fh2ampro0qVLiXVp0qSJ/N+2trYAgISEhHKfIxG9XlpVXQEiopelpqYCAA4fPoxatWopbNPV1VUIeMpKX19fqf20tbXl/5ZIJADy+xMR0ZuFLTtEVK00aNAAurq6iIyMhKurq8LLwcFBvt/ly5fl/05MTERoaCjq168PAKhfvz4uXryocNyLFy+ibt260NTUROPGjSGTyRT6ABGR+mLLDhFVK0ZGRvjiiy8wffp0yGQydOzYEVKpFBcvXoSxsTGcnJwAAAsXLoS5uTmsra3x5ZdfwsLCAoMGDQIAzJw5E61bt8aiRYswYsQI+Pv7Y9WqVfj9998BAM7Ozhg9ejQ+/fRTrFy5Ek2bNsXjx4+RkJCA4cOHV9WpE1ElYbBDRNXOokWLYGlpCR8fHzx8+BA1a9ZEixYtMG/ePPljpCVLlmDq1KkICwtDs2bNcPDgQejo6AAAWrRogT///BNff/01Fi1aBFtbWyxcuBCffPKJvIzVq1dj3rx5mDRpEp4/fw5HR0fMmzevKk6XiCoZR2MR0RulYKRUYmIiatasWdXVIaI3APvsEBERkVpjsENERERqjY+xiIiISK2xZYeIiIjUGoMdIiIiUmsMdoiIiEitMdghIiIitcZgh4iIiNQagx0iIiJSawx2iIiISK0x2CEiIiK1xmCHiIiI1Nr/A45Ofu2hDDlOAAAAAElFTkSuQmCC", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "plt.plot(np.arange(len(epochs_x)), epochs_acc, marker = '.')\n", - "plt.xlabel('epoch')\n", - "plt.ylabel('test accuracy')\n", - "plt.ylim(0, 100)\n", - "plt.xticks(np.arange(len(epochs_x)))\n", - "for i, txt in enumerate(epochs_acc):\n", - " if i%2 == 0:\n", - " pass\n", - " else:\n", - " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", - "plt.show()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "speck-rescnn", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/tests/test_nonsequential/exp_set_A/non-sequential-SCNN-example_3.ipynb b/tests/test_nonsequential/exp_set_A/non-sequential-SCNN-example_3.ipynb deleted file mode 100644 index 956dfb88..00000000 --- a/tests/test_nonsequential/exp_set_A/non-sequential-SCNN-example_3.ipynb +++ /dev/null @@ -1,1509 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import torch, random\n", - "import torch.nn as nn\n", - "import sinabs.layers as sl\n", - "from tqdm.notebook import tqdm\n", - "\n", - "from tonic.datasets.dvsgesture import DVSGesture\n", - "from tonic.transforms import ToFrame\n", - "from torch.utils.data import DataLoader\n", - "from torch.nn import CrossEntropyLoss\n", - "from torch.optim import Adam\n", - "\n", - "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential\n", - "\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "torch.backends.cudnn.deterministic = True\n", - "random.seed(1)\n", - "torch.manual_seed(1)\n", - "torch.cuda.manual_seed(1)\n", - "np.random.seed(1)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Loading Data" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "batch_size = 3\n", - "num_workers = 1\n", - "epochs = 30\n", - "lr = 1e-3" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "root_dir = \"./DVSGESTURE\"\n", - "_ = DVSGesture(save_to=root_dir, train=True)\n", - "_ = DVSGesture(save_to=root_dir, train=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "n_time_steps = 50\n", - "to_raster = ToFrame(sensor_size=DVSGesture.sensor_size, n_time_bins=n_time_steps)\n", - "\n", - "snn_train_dataset = DVSGesture(save_to=root_dir, train=True, transform=to_raster)\n", - "snn_test_dataset = DVSGesture(save_to=root_dir, train=False, transform=to_raster)" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The transformed array is in shape [Time-Step, Channel, Height, Width] --> (50, 2, 128, 128)\n" - ] - } - ], - "source": [ - "sample_data, label = snn_train_dataset[0]\n", - "print(f\"The transformed array is in shape [Time-Step, Channel, Height, Width] --> {sample_data.shape}\")" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "snn_train_dataloader = DataLoader(snn_train_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True)\n", - "snn_test_dataloader = DataLoader(snn_test_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=False)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Network Module\n", - "\n", - "We need to define a `nn.Module` implementing the network we want the chip to reproduce." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "device: NVIDIA RTX A4000\n" - ] - } - ], - "source": [ - "if torch.cuda.is_available():\n", - " device = torch.device('cuda:0')\n", - " print('device: ', torch.cuda.get_device_name(0))\n", - "else:\n", - " device = torch.device('cpu')" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "class SNN(nn.Module):\n", - " def __init__(self) -> None:\n", - " super().__init__()\n", - "\n", - " self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False)\n", - " self.iaf1 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - " self.pool1 = nn.AvgPool2d(2,2)\n", - " self.pool1a = nn.AvgPool2d(6,6)\n", - "\n", - " self.conv2 = nn.Conv2d(10, 10, 2, 1, bias=False)\n", - " self.iaf2 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - " self.pool2 = nn.AvgPool2d(3,3)\n", - "\n", - " self.conv3 = nn.Conv2d(10, 10, 3, 1, bias=False)\n", - " self.iaf3 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - " self.pool3 = nn.AvgPool2d(2,2)\n", - "\n", - " self.flat = nn.Flatten()\n", - "\n", - " self.fc1 = nn.Linear(810, 100, bias=False)\n", - " self.iaf4 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - "\n", - " self.fc2 = nn.Linear(100, 100, bias=False)\n", - " self.iaf5 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - "\n", - " self.fc3 = nn.Linear(100, 100, bias=False)\n", - " self.iaf6 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - "\n", - " self.fc4 = nn.Linear(100, 11, bias=False)\n", - " self.iaf7 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - "\n", - " self.merge_fc = sl.Merge()\n", - " self.merge_conv = sl.Merge()\n", - "\n", - " def detach_neuron_states(self):\n", - " for name, layer in self.named_modules():\n", - " if name != '':\n", - " if isinstance(layer, sl.StatefulLayer):\n", - " for name, buffer in layer.named_buffers():\n", - " buffer.detach_()\n", - "\n", - " def init_weights(self):\n", - " for name, layer in self.named_modules():\n", - " if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear):\n", - " nn.init.xavier_normal_(layer.weight.data)\n", - "\n", - " def forward(self, x):\n", - " \n", - " con1_out = self.conv1(x)\n", - " iaf1_out = self.iaf1(con1_out)\n", - " pool1_out = self.pool1(iaf1_out)\n", - " pool1a_out = self.pool1a(iaf1_out)\n", - "\n", - " conv2_out = self.conv2(pool1_out)\n", - " iaf2_out = self.iaf2(conv2_out)\n", - " pool2_out = self.pool2(iaf2_out)\n", - "\n", - " merged_conv_out = self.merge_conv(pool1a_out, pool2_out)\n", - "\n", - " conv3_out = self.conv3(merged_conv_out)\n", - " iaf3_out = self.iaf3(conv3_out)\n", - " pool3_out = self.pool3(iaf3_out)\n", - "\n", - " flat_out = self.flat(pool3_out)\n", - " \n", - " fc1_out = self.fc1(flat_out)\n", - " iaf4_out = self.iaf4(fc1_out)\n", - "\n", - " fc2_out = self.fc2(iaf4_out)\n", - " iaf5_out = self.iaf5(fc2_out)\n", - "\n", - " fc3_out = self.fc3(iaf5_out)\n", - " iaf6_out = self.iaf6(fc3_out)\n", - "\n", - " merge_fc_out = self.merge_fc(iaf4_out, iaf6_out)\n", - "\n", - " fc4_out = self.fc4(merge_fc_out)\n", - " iaf7_out = self.iaf7(fc4_out)\n", - "\n", - " return iaf7_out" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "snn = SNN().to(device)" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "snn.init_weights()" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "optimizer = Adam(snn.parameters(), lr=lr, betas=(0.9, 0.999), eps=1e-8)\n", - "loss_fn = CrossEntropyLoss()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Define train and test" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "def train(dataloader, model, loss_fn, optimizer, epochs, test_func, dataloader_test):\n", - " epochs_y = []\n", - " epochs_x = []\n", - " epochs_acc = []\n", - " model.train()\n", - "\n", - " for e in range(epochs):\n", - " losses = []\n", - " batches = []\n", - " batch_count = 0\n", - " train_p_bar = tqdm(snn_train_dataloader)\n", - "\n", - " for X, y in train_p_bar:\n", - " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", - " X = X.reshape(-1, 2, 128, 128).to(dtype=torch.float, device=device)\n", - " y = y.to(dtype=torch.long, device=device)\n", - "\n", - " # forward\n", - " pred = model(X)\n", - "\n", - " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", - " pred = pred.reshape(batch_size, n_time_steps, -1)\n", - "\n", - " # accumulate all time-steps output for final prediction\n", - " pred = pred.sum(dim = 1)\n", - " loss = loss_fn(pred, y)\n", - "\n", - " # gradient update\n", - " optimizer.zero_grad()\n", - " loss.backward()\n", - " optimizer.step()\n", - "\n", - " # detach the neuron states and activations from current computation graph(necessary)\n", - " model.detach_neuron_states()\n", - "\n", - " train_p_bar.set_description(f\"Epoch {e} - BPTT Training Loss: {round(loss.item(), 4)}\")\n", - "\n", - " batch_count += 1\n", - " losses.append(loss.item())\n", - " batches.append(batch_count)\n", - "\n", - " epochs_y.append(losses)\n", - " epochs_x.append(batches)\n", - "\n", - " acc = test_func(dataloader_test, model)\n", - " print(f'Epoch {e} accuracy: {acc}')\n", - " epochs_acc.append(acc)\n", - "\n", - " return epochs_x, epochs_y, epochs_acc\n" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [], - "source": [ - "def test(dataloader, model):\n", - " correct_predictions = []\n", - " with torch.no_grad():\n", - " test_p_bar = tqdm(dataloader)\n", - " for X, y in test_p_bar:\n", - " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", - " X = X.reshape(-1, 2, 128, 128).to(dtype=torch.float, device=device)\n", - " y = y.to(dtype=torch.long, device=device)\n", - "\n", - " # forward\n", - " output = model(X)\n", - "\n", - " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", - " output = output.reshape(batch_size, n_time_steps, -1)\n", - "\n", - " # accumulate all time-steps output for final prediction\n", - " output = output.sum(dim=1)\n", - "\n", - " # calculate accuracy\n", - " pred = output.argmax(dim=1, keepdim=True)\n", - "\n", - " # compute the total correct predictions\n", - " correct_predictions.append(pred.eq(y.view_as(pred)))\n", - "\n", - " test_p_bar.set_description(f\"Testing Model...\")\n", - " \n", - " correct_predictions = torch.cat(correct_predictions)\n", - " return correct_predictions.sum().item()/(len(correct_predictions))*100" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Training loop (HPO)" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "12d134e3b89e41888c9c47892b8e6491", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - " 0%| | 0/359 [00:00" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "y_avg = []\n", - "for y in epochs_y:\n", - " y_avg.append(np.mean(y))\n", - "\n", - "plt.plot(np.arange(len(epochs_x)), y_avg, marker = '.')\n", - "plt.xlabel('epoch')\n", - "plt.ylabel('average loss')\n", - "plt.ylim(0,)\n", - "plt.xticks(np.arange(len(epochs_x)))\n", - "for i, txt in enumerate(y_avg):\n", - " if i%5 == 0:\n", - " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", - " else:\n", - " pass\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAG2CAYAAACZEEfAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABq60lEQVR4nO3dd1hT1+MG8DfsHUW2TAfixK3gqqO46q6jtVWr1bZi62idraPVFrWt7dfROn5urauOOloXrqqooOBEBERBZYhI2DPn9weaGlmJBIH4fp4nT8vNPfecQMx9c+6550iEEAJEREREWkqnohtAREREVJ4YdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirVWjYOXPmDPr06QMHBwdIJBLs27dP6XkhBObMmQN7e3sYGxujW7duCA8PV9onKSkJw4cPh4WFBapVq4YxY8YgLS3tNb4KIiIiqswqNOykp6fD09MTK1asKPL5xYsXY+nSpVi5ciUuXrwIU1NTdO/eHVlZWYp9hg8fjps3b+LYsWM4ePAgzpw5g3Hjxr2ul0BERESVnKSyLAQqkUiwd+9e9O/fH0BBr46DgwO+/PJLfPXVVwAAmUwGW1tbbNiwAcOGDUNoaCgaNGiAwMBAtGzZEgBw+PBh9OrVCw8ePICDg0NFvRwiIiKqJPQqugHFiYqKQlxcHLp166bYJpVK0aZNGwQEBGDYsGEICAhAtWrVFEEHALp16wYdHR1cvHgRAwYMKPLY2dnZyM7OVvwsl8uRlJSEGjVqQCKRlN+LIiIiIo0RQiA1NRUODg7Q0Sn+YlWlDTtxcXEAAFtbW6Xttra2iufi4uJgY2Oj9Lyenh4sLS0V+xTFz88P3377rYZbTERERBUhJiYGjo6OxT5facNOeZo5cyamTJmi+Fkmk8HZ2RkxMTGwsLCowJYRERGRqlJSUuDk5ARzc/MS96u0YcfOzg4AEB8fD3t7e8X2+Ph4NG3aVLFPQkKCUrm8vDwkJSUpyhfF0NAQhoaGhbZbWFgw7BAREVUxpQ1BqbTz7Li5ucHOzg7+/v6KbSkpKbh48SK8vLwAAF5eXkhOTsbly5cV+5w4cQJyuRxt2rR57W0mIiKiyqdCe3bS0tIQERGh+DkqKgohISGwtLSEs7MzJk2ahAULFqBu3bpwc3PD7Nmz4eDgoLhjq379+ujRowfGjh2LlStXIjc3FxMmTMCwYcN4JxYREREBqOCwExQUhM6dOyt+fj6OZuTIkdiwYQOmTZuG9PR0jBs3DsnJyWjfvj0OHz4MIyMjRZmtW7diwoQJ6Nq1K3R0dDBo0CAsXbr0tb8WIiIiqpwqzTw7FSklJQVSqRQymYxjdoiIiKoIVc/flXbMDhEREZEmMOwQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEqQn5+P2bNnw83NDcbGxqhduzbmz58PIYRin1GjRkEikSg9evToUeJxU1NTMWnSJLi4uMDY2Bje3t4IDAxU2ictLQ0TJkyAo6MjjI2N0aBBA6xcubJcXqc206voBhAREVVmixYtwu+//46NGzeiYcOGCAoKwkcffQSpVIovvvhCsV+PHj2wfv16xc+GhoYlHvfjjz/GjRs3sHnzZjg4OGDLli3o1q0bbt26hZo1awIApkyZghMnTmDLli1wdXXF0aNHMX78eDg4OKBv377l84K1EHt2iIiISnD+/Hn069cPvXv3hqurK9599134+Pjg0qVLSvsZGhrCzs5O8ahevXqxx8zMzMTu3buxePFidOzYEXXq1MG8efNQp04d/P7770p1jxw5Em+99RZcXV0xbtw4eHp6FqqbSsawQ0T0hiivyzF+fn5o1aoVzM3NYWNjg/79+yMsLExpn08++QS1a9eGsbExalhZo0O3njgTGFIeL1PjvL294e/vjzt37gAArl69irNnz6Jnz55K+506dQo2NjaoV68ePvvsMzx58qTYY+bl5SE/Px9GRkZK242NjXH27Fmluvfv34+HDx9CCIGTJ0/izp078PHx0eArfAMIEjKZTAAQMpmsoptCRFRuvv/+e1GjRg1x8OBBERUVJXbt2iXMzMzE//73P8U+I0eOFD169BCxsbGKR1JSUonH7d69u1i/fr24ceOGCAkJEb169RLOzs4iLS1Nsc+qVavE6dOnxdJ9Z4XDqF+FcZ3WQtfCWvwRcLfcXq+m5Ofni+nTpwuJRCJ09fSERCIRP/zwg9I+27ZtE3/99Ze4du2a2Lt3r6hfv75o1aqVyMvLK/a4Xl5eolOnTuLhw4ciLy9PbN68Wejo6Ah3d3fFPllZWWLEiBECgNDT0xMGBgZi48aN5fZaqxpVz98MO4Jhh4jeDL179xajR49W2jZw4EAxfPhwxc8jR44U/fr1K1M9CQkJAoA4ffq00vbIhFThOv2gcJl+UNh/tEwAEE6f/J94lJxRpvrK27Zt24Sljb2w7jtV2I9eLqzemSLMLKqJDRs2FFsmMjJSABDHjx8vdp+IiAjRsWNHAUDo6uqKVq1aieHDhwsPDw/FPj/++KNwd3cX+/fvF1evXhXLli0TZmZm4tixYxp9jVWVqudvXsYiInpDlMflmKLIZDIAgKWlpWLbydsJGLIyAAKAPCcLadePQ09qC4l5DdxLzCjbCytnU776ChLPfjCp3wkG1q4wbdgFep7vYP73PxRbplatWrCyskJERESx+9SuXRunT59GWloaYmJicOnSJeTm5qJWrVoACsb1zJo1C0uWLEGfPn3QpEkTTJgwAUOHDsVPP/2k8ddZklhZJs5HJiJWlvla69UU3o1FRPSGmDFjBlJSUuDh4QFdXV3k5+fj+++/x/DhwxX79OjRAwMHDoSbmxsiIyMxa9Ys9OzZEwEBAdDV1S21DrlcjkmTJqFdu3Zo1KgRElKz8N2BWzh4LRapVw7h6an1ELlZ0LN0hM3QBdDV04erlUl5vuwyEUJAlpIOI8lLfQMSHcTLMuEfGo+O7tbQ11V+/sGDB3jy5Ans7e1LrcPU1BSmpqZ4+vQpjhw5gsWLFwMAcnNzkZubCx0d5WPr6upCLper9TpiZZmISkyHm5Up7KXGpe4vhEBadh4S03KwIzAaq87chRCAjgTwG9gYQ1s5q1V/RWPYISJ6Q+zcuRNbt27FH3/8gYYNGyIkJASTJk2Cg4MDRo4cCQAYNmyYYv/GjRujSZMmqF27Nk6dOoWuXbuWWoevry9u3LiBM2f+xbZL0fD7OxQpWXnQkQDjPx4Ji48H45e/ApF8aTcS/1oI51E/IyMnv9xec1lcikqC3z+hkLi0gOz8DuhaWMPAyhk58ZFICdwHsyZvY8zGIEj18mF6cy8+GfkeOnrWxd27dzFt2jTUqVMH3bt3Vxyva9euGDBgACZMmAAAOHLkCIQQqFevHiIiIjB16lR4eHjgo48+AgBYWFigU6dOmDp1KoyNjeHi4oLTp09j06ZNWLJkicqvY0dgNGbuuQ75s7Ay+W13tHa1RGJaDhLTshWPx6nKP2flFg5UcgFM330dJ8Meo4VzdXjYm8PDzgLW5iXfZl/RJEK8MAz/DZWSkgKpVAqZTAYLC4uKbg4RqSA/Px/z5s3Dli1bEBcXBwcHB4waNQrffPMNJBIJAGDevHnYvn07YmJiYGBggBYtWuD7779HmzZtij2uq6sr7t+/X2j7+PHjsWLFCgDA6tWr8ccff+DKlStITU3F06dPUa1atXJ5nZrk5OSEGTNmwNfXV7FtwYIF2LJlC27fvl1sOWtrayxYsACffPJJicefMGEC/vrrL2za8zd+u5yKwHtPAQCNa0rhN7AxGtWUAijoZbj9IAlvN6+D6t0/h+dbvbHPtx3MDCvH9+878alYfPg2jocmAAAM5NnQD9mFsIv+kGfIoGtmiU49+8P73U/w961EJDxNweM9C5CTcBciOx3VrGzR3edt/PrjQtja2iqO6+rqilGjRmHevHkACsLnzJkz8eDBA1haWmLQoEH4/vvvIZVKFWXi4uIwc+ZMHD16FElJSXBxccG4ceMwefJkxfv8ZUIIPHiaidtxqQiMeoLV/0a98u/CUE8H2Xml9yJZmRmivr05POwKwo+HvTnq2JjBUE9X7V4ldah6/q4c7ywiIjWpMtGbu7s7li9fjlq1aiEzMxO//PILfHx8EBERAWtr6yKPGxgYiPz8/3oabty4gbfffhuDBw9WbMvIyECPHj3Qo0cPzJw5s3xfqAZlZGSofUlElcsxQgh8/vnn2LN3L0Z+vx7j9sUgN1/AxEAXU952xyhvV+i9cJnHXmoMSyMrGOhKYGEARCSk4audV/H7B82LPYG/DrGyTPxy7A7+vPwAcgHo6kgwrJUTJnatCxuLgYiVZeJeYgZcrUwUJ+05/eQ4F/kEe1utxZGb8cjMLXjvBAD4dHcEBjTLxDtNHGBpaoCAq6GISkxHrCwT9lJjDBkyBEOGDCmxTXZ2dli/fn2xgSE1Kxd34lMRGpuK23EpuB2bittxqUjLzivxuLYWhnCqbgIrM0NYmRvAyswQ1uaGBT+bGcL62XZZZi7aLTwB+QvdIjoSYEx7NzxMzsTt2FREPUlHYlo2/g3Pxr/hiYr99HQkqGFmgPiUbEW5iroExp4dsGeHqCp65513YGtri7Vr1yq2DRo0CMbGxtiyZUuRZZ7/Wz9+/LhKl2QAYNKkSTh48CDCw8MLnYhPnTqFzp07V5menVGjRuH48eNYtWoVrJ1qw//cRfw0+yt8PGY0Fi1ahLS0NHz77bcYNGgQ7OzsEBkZiWnTpiE1NRXXr19XzAj88uWY8ePHY/OWrajzwbdI1LECALSrWwPzB7dGXYcauHv3Lnbs2AEfHx9YW1vjwYMHWLhwIc6dO4edxwLw2e5I5OTLMbV7Pfh2rvPafy+yjFz8djoCG87dU/Ri9Gxkh6+610NtazOVj5OenYejt+KwN/gRzoY/VgQEPR0J6tqa4XZsKgQKTvrTutfDO54OKh334NVHWHwkDHIBSAB08bCBjo4Et+NSEJNU9IBhfV0J6tiYw8XSBEduxuHFE72uRIKzMzqr3MuyIzAas/bcQL4Q0JVI8MPARkqBJSMnD+HxaQiNTcHtuFTFf2WZuYWOpW7dpWHPDhFpNW9vb6xevRp37tyBu7u74s6i4sYy5OTkYPXq1ZBKpfD09FSpjpycHGzZsgVTpkyp0B4HTVm2bBlmz56NUR9/gieJj6FrZgnTBl3RpN9YAAW9PNeuXcPGjRuRnJwMBwcH+Pj4YP78+UpLH0RGRiIxseAbfHJGjmLG35DfJyv22QGgh3Q96o4aBSMjI/z777/49ddf8fTpU9ja2qJjx444f/486tVzx7d5Rpi55zp+OhqGRjWl6ORedK+bpmXl5mNTwD2sOBmpODG3drPEjJ4eaO5c/OzHxTE11MOAZo4Y0MwRCalZ2B/yCPtCHuLGwxSExqYq9pMLYOHhMCw8HFbC0YomAPjfTlDaZmdhpBg7U//Zf2tZmyoGTRcVVtQJG0NbOaOju3WhXq3nTAz04OlUDZ5O1f5rpxA4dD0WE/4IVto3XwjcS8zQ+OWs0rBnB+zZIaqK5HI5Zs2ahcWLFyvdWfTyZaWDBw9i2LBhyMjIgL29Pfbt24dWrVqpVMfOnTvx/vvvIzo6Gg4Ohb+FV7WeHaDgUs3LlyUkAGb28kBdG3PFZY0apoYw0Ct6dpJYWSaiHqcjPCEVy05EIDEtBwAwvI0zpvXwgNRYX+12zdxzDdsuxUBqrI8DE9rDuUb53KEVK8tEZEI6QmNlWH/uHh7JsgAA9WzNMb1nPXSuZ6PxYLsrKAZT/7xWaLu+jgQ6OiXXJZcL5MoLn6ZHtHVBj8Z28LCzgKWpQaltKOoSXHkr6r3Gnh0iqjDlOYCwvOpW5c4iAOjcuTNCQkKQmJiINWvWYMiQIbh48SJsbGxKrWPt2rXo2bNnkUGnqop6nI6Xz50CwA9/Fx6gLDXWh5WZwbMAVDCOIzY5E0dvxStdFqlrYwa/gY3R0tWy0DFUNa9vQ4TGpiIkJhnjNgdhz3hvmBho9hS1IzAaM/Zcx4tf8R2kRpjiUw8DmtWEbinB41W1r2sFHQkKnfTPTC/9pF9cYPisc221/r3YS41f+79te6kx/AY2LlOvkqawZwfs2aE328u3pb7OAYRlqftV7yyqW7cuRo8eXerA4vv376NWrVrYs2cP+vXrV+Q+VbFnZ8uF+/hm3w2lbRIUXL5JzcpDYlo2nqTnIL+I3oSiSACcnvoWnGuYlrltsbJM9Fl2FolpOejr6YD/DWuqsV6WqzFP0W/FeaVtEgAnv3oLrlZlb3tpShv3Ul5lK4Py7FVizw4RlSpWlqkIG0DBN89Ze26go7t1uX/7KqrumXuuq1z3q9xZBBRc/srOzi71+OvXr4eNjQ169+5d6r5VRWpWLpb6hwMoONELoMiTp1wukJyZWzDfSmo2HqdlIzEtB1djkrH/6iOlYwoAD5OzNBJ27KXGWPF+cwz/v4vYf/URmjhK8XGHWmU6Zr5cYMuF+/D7J7TQcwJArCzrtYSd0sa9lFfZyqAiepVexrBD9Aa7HZtS6JLG6xpAGJVY+HKKXABbL9zHlLfrlTqWoU+fPvj+++/h7OyMhg0bIjg4GEuWLMHo0aMBAOnp6fj+++/Rt29f2NvbIzExEStWrMDDhw+VbiN/+c4ioCAQrV+/HiNHjoSeXuGPybi4OMTFxSmWArh+/TrMzc3h7OystERCaV735cOfj95BQmo2XGuYYMNHrZ+d6AufPHV0JLA0NYClqQHcbc2V2nvw2qNCl1Q0OQNym1o18HXv+vj2wC34/XMbDRws4F3b6pWOFRqbgpl7riMkJrnI5zXd9tKU5aRfGQJDVca1sYgqWH5+PmbPng03NzcYGxujdu3amD9/Pl68wrxnzx74+PigRo0akEgkCAkJUenYu3btgoeHB4yMjNC4cWP8/fffiueS0nOw+Mgd5CbGIGH3d4j+ZQiilwxC7MbJ0MtUby2kV+FWzLfp5Scj0Wf5Wfwb/rjE8suWLcO7776L8ePHo379+vjqq6/wySefYP78+QAKenlu376NQYMGwd3dHX369MGTJ0/w77//omHDhgAKTt63wsIR9SBW6djHjx9HdHS0Iji9bOXKlWjWrBnGji24i6ljx45o1qwZ9u/fr/Lr3xEYjXYLT+D9NRfRbuEJ7AiMVrnsq7j+QIZNAfcAAPP7N4KrlSm8atdQe9yH38DG0H12aam8xmCM8nbFgGY1kS8X+PyPYDxKVm89psycfCz85zb6LDuLkJhkmBvqYX7/Rq+l7VQ5ccwOOGaHKtYPP/yAJUuWFJoc7/vvv1dMjrd582ZERUXBwcEBY8eORXBwMJo2bVricc+fP4+OHTvCz88P77zzDv744w8sWrQIV65cgdShFkauu4Sw8AjEbZ4C8yZvw6R+J0gMTGCW8QgnF4+Dg71dub7uM3ceY8S6S4qfdSTA2w1scT7iCVKfTYjWoa4VpvfwUMy8q0kVOVbp4dMMtF90skxzn6gjXy4w4LdzuPZAhr6eDlj6XrMyHe913NmTmZOPQb+fx63YFDRxlGLnJ14w0i99ba5/wx/j6703EJ1UsLhor8Z2mNunIWwtjF5b2+n1UfX8zbADhh2qWOpMjnfv3j24ubmpFHaGDh2K9PR0HDx4ULGtbdu2cKnbAHc93sfj1Gyk/f0T2tezxe//tx43HsgwZddVpGblYX6/hvjQy1WTL1NJbr4cPf/3LyIS0jCkZcG8JM9PPknpOVh+IgKbL9xDbn7Bx1NfTwd85VNPY7cjRyelo9OPp5TuyinPsPGi7Lx8jFx3CRfuJhV6btvYtvCqXUPjdW4KuIc5f92EuZEe/L/sBBtzI43XUR5ikjLQZ/lZJGfkYkhLRywa1KTYActP0rKx4FAo9gY/BFBwl9V3/RqhWwPbIvcn7aDq+ZuXsYgqmLe3N/z9/XHnzh0AUEyO17NnzzIdNyAgAN26dVPa1rBVB+w/dgqPU7Phbm2KrKggtGjSEKOG9Mfwzk2QsWs6Mu4E4Odjd/A0PadM9ZdkU8B9RCSkwdLUAF/3bqB0OcXS1ABz+jTAiS/fwoBmNSGRAPuvPkLXJacwb/9NPEkrfXDxy4QQiEhIxfpzURizIRA+S87g5a95z8cqlafUrFyM3hBYZNDRkaBcxo8kpGThx2eT103rXq/KBB0AcLI0wbL3mkFHAuwMeoCtFwtf6hNCYGdQDLouOY29wQ+hIwE+aueKo1M6MeiQAgcoE1WwGTNmICUlBR4eHkqT4w0fPrxMx42Li1NahPDQtVgcCM9ATupTtHazxAIfR9T7Kg0LFy7EggULsGjRIvz99z/4+puvoWNkhiXHHDC/f6OyvrxCEtOy8euxgmA3rXu9Yiegc7I0wS9Dm+LjDm5YdDgMZ+48xobz9/Dn5Qf4pGMtjOngVuI8LE/SsnE2IhFnwxNxNiIRsc8mjyvJoeuxaO1mWS7zrSSkZmHUukDcik2BqYEuhrV2xvpzUYrBvpamBuWyEOb8Q6FIzc6Dp6MU77dx0fjxy1uHutaY2t0Diw7fxrcHbqK+vQVauBTMbnz3cRpm7b2uCI8N7C2wcFBjNHGsVoEtpsqIYYeogqk6OV5ZbDx/D/MO3ES+XMBQTwebRrdG0uN4AEC/fv0weXLBNP9NmzbFP/6ncSXkH2x1aYz3WjujgYNmL+3+eDgMqdl5aFTTAoNbOpW6f0MHKTaNbo1zEYlY+M9tXH8ow8/H7mDThfuY2LUuOrpb4cHTTDhIjfHgaSb+DX+Mf8MTcSs2Rek4Bno6aO1qiQ51rdC+rhWuPZDhm70Fc5c8vw17y4X7ePA0A/8b1uyVZgEuTlRiOkasu4iYpExYmRlg/ajWaOwoxccd3HA1Jhlz/rqJhNRsTNl5Fas+aFHqnWiqOnPnMQ5cfQQdCfD9gMblNmleefu0Uy1cf5iMv6/HYdymIMzt0wDXH8qwMeA+cvLkMNbXxeS362J0OzelBUeJnmPYIapgU6dOxYwZMzBs2DAAQOPGjXH//n34+fmVKezY2dkhLi4OPx65jRUnIwEADarLkeLqCCN9XVhZWUFPTw8NGjRQKufd0hMRe49ALoBvD9zE9nFtNTax27UHydh5OQYAMK9PQ7VOvu3qWOEv33Y4dD0WPx4JQ3RSRqHJ8V5W394CHZ+Fm1aulkoDXBs6SPFWvf/mLrkUlYRpf17DqbDH6L/iHNaMaIE6NuYlHF01V2OS8dGGQCSl58Clhgk2jW4Nl2dz0jy/ndheaozBqwJw7FY8VpyMwOdd65a53qzcfMz+q+D3M9LbtVwGeb8uEokEi9/1RNC9p0hIzcYX20MUz3Vyt8aC/o3gZPn6biGnqocRmEiDYmWZOB+ZiFiZ6rfKvurkeKVp07Ytlm/Zpwg6X77tjvyYa/Dy8gIAGBgYoFWrVggLU16M8M6dO2jTxB1G+jq4GJWEQ9djCx37VcjlAvP234QQwIBmNV9paQEdHQn6eDrg+JROmPK2e5H79Gpsh/8Na4rAr7vhn4kdMLNXfXSoa13knTz2UmPFeKF+TWti92fecJAaISoxHf1XnMfxW/Fqt/FFp8IS8N6aC0hKz0HjmlLs/sxbEXRe5OlUDQv6FVwyXHL8Dk6GJRTaR12/nYrE/ScZsLUwLPZ3VZWkZhVMcvgiiQTwG8igQ6Vj2CHSkFedN+X55HiHDh3CvXv3sHfvXixZsgQDBgxQ7JOUlISQkBDcunULABAWFoaQkBDExcUp9hkxYoRiCYSMnDxk1PXB3ZBzSL20B583N8aTf7ciKChIafK8qVOnYseOHVizZg0iIiKwfPlyHDhwAF9N+gKfdaoDAPjhUCgyc/LL/PvZF/IQV6KTYWKgixk9Pcp0LAM9HbR0LXpV6g/buqJf05qwNjcs8vmSNKopxf7P26O1qyXSsvMwdnMQlp8Ix6vctLrnygN8vDEIGTn56FDXCtvGtYWVWfFtGtLKCcPbOEMIYOK2YNxLTFe7zuciH6dh5amCkDvnnYYwN9LcJbmKUtQklEIA95+oNwcPvZkYdog0oLhlF1Tp4SltcjwA2L9/P5o1a6ZYumDYsGFo1qwZVq5cqdgnOjoasbGxSErPwftrLuJmvj3s+0+DSdRpzPqwJ/7880/s27cPjRr9N+h4wIABWLlyJRYvXozGjRvj//7v/7B79260b98en3SqhZrVjPFIloXfT0eW6feTlp2Hhf8UrFc1oUsdxZwnZeFmZYqXr4JpYkZcKzNDbPm4DT5oWxA8fjp6B75/XEH6s7l/SiOEwKrTkZiy8yry5AL9mzpg7chWKg0+ntunIZo7V0NKVh4+2XxZ5Tpfrn/2vhvIyZejk7s1ejUu3/mSXpfy+nvTm4Hz7IDz7FDZ7b/6CF9sCy60vbzmTSlKrCwTgVFJ+OloGKKTMiE11se6US3RwuXVV6L+53osPtt6BQZ6OvCf0umVLxcs/Oc2Vp6OhEsNExyd3BGGeqVPDqeK8l4gcdulaMz56wZy8wU87MyxZkTLEn8HcrnA93+HYu3ZKADA2A5umNmzvloDjuNTsvDOsrN4nJqN3k3ssfy9ZmqNmdoX/BCTdoTAUE8HRyd3LPKyWVVV1RfEJM3jpIJqYNihsniSlo0Bv51DdJJyL46OBDg3o8trmaV1R2A0Zuy5rpg7Rmqsh92feZd5gK0QAsP/7yLORz5Bj4Z2WPlhC7WPEZWYDp9fTiM3X+D/RrTU+Nwn5T0jbtC9JHy65QoS07JR3UQfK95vDu86hddqys7Lx1e7ruHAs4Uyv+5VH2M7vtoiloH3kvDe6gvIkwvM6uWBcR1rq1ROlpGLrktOITEtB1/5uGNCl7IPdK5sOAMyvYiTCtJr5+rqColEUujh6+sLoGDelw8//BB2dnYwNTVF8+bNsXv37lKPu2LFCri6usLIyAht2rTBpUuXlJ7/5JNPULt2bRgbG8Pa2hr9+vXD7du3y+U1viw1Kxcj11961pOip9TN3sRRCjsNXK4pTawsUynoFLQrD6YamLNFIpFg7rO7pg7fjMO5iES1j7Hg4C3k5gt0crdG1/o2ZW7Ty14cZFweWrpa4sDn7dDEUYqnGbn4cN0lbDgXpTSO5/lkgQeuPoK+rgS/Dm36ykEHAFq5WmJun4K75Bb+c1vl3/viI7eRmJaD2tamZaq/Mivvvzdpp0oddlRZIFEIgTlz5sDe3h7Gxsbo1q0bwsPDK7DVb67AwECs/DsQjhM2w9F3M+yGLQAAxQrTI0aMQFhYGPbv34/r169j4MCBGDJkCIKDC1/+eW7Hjh2YMmUK5s6diytXrsDT0xPdu3dHQsJ/d6u0aNEC69evR2hoKI4cOQIhBHx8fJCfX/ZBtSXJys3H2E1BuPEwBTVMDbBnfDucm9EFc96pD10JEBIjw7pz98q1DQDwb3hiodmA5QIamw24np05PmxbMBndtwduIi9f9bvEToYlwP92AvR0JJjTp4HGbmF/3eylxtj5iZdiccp5B25h2p/XcP9JOv6+/giDfj+PcxFPYGqgi3WjWqF/s5plrvODti54t4Uj5AKY8McVPHha8t8zOPop/rhUMCh+Qf/GGrtUSKQNKnXYWbRoEX7//XcsX74coaGhWLRoERYvXoxly5Yp9lm8eDGWLl2KlStX4uLFizA1NUX37t2RlVX6bKmkOflygeN3M+B3Oh66ptWha1Yd6eGXoFfNHnU9WwEoWJjy888/R+vWrVGrVi188803qFatGi5fvlzscZcsWYKxY8fio48+QoMGDbBy5UqYmJhg3bp1in3GjRuHjh07wtXVFc2bN8eCBQsQExODe/fuldvrzcuXY8IfwbhwNwlmhnrYOLo1alubwV5qjNHta+Gbdwq+lf/wdygCIstvBfGYpAwsPly4F0vTAzcnd3NHdRN93IlPw5YL91Uqk5Mnx/wDBXePfdTOFbWtzTTWnopgpK+LJUM88U3v+tCRALsuP0CnH09h/NZg3IlPg6mBLraP80KHutYaqU8ikWBB/0ZoXLOgR+nTLZeRlVt0gM/Ll+PrvTcgBDCwec3XNk6MqKqo1GHn/Pnz6NevH3r37g1XV1e8++678PHxUVzGEELg119/xTfffIN+/fqhSZMm2LRpEx49eoR9+/ZVbOPfEBk5edh4/h66/HwKM/f+N8GbyM9F+q1TMGvyNj7eeBmRj9Pg7e2NHTt2ICkpCXK5HNu3b0dWVhbeeuutIo+dk5ODy5cvK63vpKOjg27duiEgIKDIMunp6Vi/fj3c3Nzg5FT67LyvQi4XmL77Oo6HxsNATwf/N7JloQnbRnm7KnoBJvxxBY+SNX977OPUbHy49iIS03JgY26ouIT2fOCmJrv5pSb6+Kp7PQDAkmN3VFqfasP5KNxNTIeVmSG+0MAkeZWBRCLBxx1q4ZehTQs9l5mbDytzA43WZ6Svi5UftoClqQFuPEx5FmgKD7PcGHAft2JTIDXWx6xe9TXaBiJtUKnDTmkLJEZFRSEuLk7pZCiVStGmTZtiT4YAkJ2djZSUFKUHqSchJQs/HrkNL78TmLv/Ju4/yYCFkR6eX6TIuHMB8qw0mDbqitC4VPT89V90Hu+H7Jwc1KhRA4aGhvjkk0+wd+9e1KlTp8g6EhMTkZ+fr7S+EwDY2toqzS8DAL/99hvMzMxgZmaGf/75B8eOHYOBgWZPPEBBwP7+71DsvvIAujoSrHi/OdrWKvwtWiKR4IcBjdHA3gJP0nPwWQnfyl9FSlYuRq67hHtPMuBY3Rj7J7THuRldsG1sW5yd0blc7lAZ1soZDewtkJKVh5+O3ilx34TULCz1jwAATO9RTyvmeXlRUXP4aPLS4YtqVjPG8meLYe6+8gCbX+pZi5VlYsnRgokhZ/T0KHEuH6I3VaUOO8+n0Pfw8IC+vj6aNWuGSZMmKRZIfH7CU+Vk+CI/Pz9IpVLFo7x6ALRRWFwqpu66ivaLTmLFyUjIMnPhUsME3/VriAuzumLhoMbQlUiQdu0oTGq1xNeDvdHJ3Ro5+XL4zZ+HS7djsHzzHgQFBWHKlCkYMmQIrl+/XuZ2DR8+HMHBwTh9+jTc3d0xZMiQcrmUueJkhOK24sWDmuDtEu4sMjbQxaoPW6CaiT6uPpBh7l83X2lyupdl5uTj4w1BuBWbAiszA2we0wZ2UqNyH7ipqyPBvL4NAQDbA6Nx46Gs2H0XHw5DWnYePJ2qYVBzx3JpT0V63XO+eNexwsyeBT023x24hcB7/62a/t2BW0jPyUdz52oYqsJaY0Rvokoddl5cIPHKlSvYuHEjfvrpJ2zcuLFMx505cyZkMpniERMTo6EWaychBM6GJ2LEukvo/usZ7Lr8ADn5crR0qY6VH7TAiS/fwggvV5gY6GFoK2dsf782cqKvYvl3X+Kzt+pgw0etMLN9daReOQjTtz/HjzcMsD1SgklTZ6Fly5ZYsWJFkfVaWVlBV1cX8fHKU/bHx8fDzk55ojSpVIq6deuiY8eO+PPPP3H79m3s3btXo7+HzRfuK3o0Zr/TAINalH4Sd7I0wbJn38p3BMUoBpC+qtx8OXz/uIJL95Jg/myskJvV65tHpbWbJfp6OkAIPFv6oXB4C45+ij8vPwAAzOvTQGOLWlYm9lJj+A0sCPZA+Vw6fNnHHdzQx9MBeXKB8VuvID4lCydux+OfG3HQ1ZHg+wGNtfJ3TaQJlXoh0NIWSHx+wouPj4e9vb2iXHx8PJo2bVrscQ0NDWFoyK7eksTKMhEen4bw+FTsuvwAt+NSARTMHdOzkT0+7uCGZs5FT9d/aPc22NjY4MMhAwEUXNLxdimY76VXEwccjwW2XYrBsVsJkGfkFXvXlIGBAVq0aAF/f3/0798fACCXy+Hv76+05MHLhBAQQiA7u/RxJao6cPUR5jxbVPHzLnUwpr2bymU71LXG1O4eWHT4NubtvwkPOwu0cCn6d1cSuVxg6q6rOHE7AYZ6Olg7qhUaOrz+xR1n9vLAsVvxCLr/FPuvPkK/pv/defR8/SsAeLeFY7HvEW0wtJUzOrpbv7Y5XyQSCRYNaozw+FTcjkvF6A2BiE8p6L0c094N9e05RxhRcSp1z05pCyS6ubnBzs4O/v7+iudTUlJw8eJFxWKHpL7tl6Lh7XcCI9ZdwvxDobgdlwoTA12M8nbF6amdsWJ482JPYnK5HOvXr8fIkSOhp/dflvbw8ECdOnVwZ/cSzGljAHtJMu6e2IbggDOIMmuAmKSCsQ5du3bF8uXLFeWmTJmCNWvWYOPGjQgNDcVnn32G9PR0fPTRRwCAu3fvws/PD5cvX0Z0dDTOnz+PwYMHw9jYGL169dLI7+P0nceYsjMEQgAftnV5pUUVP+1UC70a2yE3X2D81stISFXvEpsQAt8dvIV9IY+gpyPB7x80R2u3V58ZuSzspcbw7VwwyZ3f37eVljTYfeUBrj6QwcxQD9N61KuQ9r1Or3vOFxMDPaz6sAWM9HRw81EKEtNyAACO1TjnDFFJKnXYKW2BRIlEgkmTJmHBggWKuVtGjBgBBwcHRU8Aqef5Gk8vXpyQANg3vh3m9W1Y6nIBx48fR3R0NEaPHq20XV9fH3///Tesra0x89MPcG3pOBjfPwfbPlMQYegOn1/OYM2Zu4iMjETUg1jFyuFDhw7FTz/9hDlz5qBp06YICQnB4cOHFeO0jIyM8O+//6JXr16oU6cOhg4dCnNzc5w/fx42NmWfwO7y/SR8uvkycvMF+ng64Nu+DV9prhiJRILF73qiro0Z4lOy4bv1CnLyVJ+vZql/BDacvwcA+GmwJ7p4aHYWYnV93KEWnCyNEZeShd9OFQxETsnKxaLDBQNlv+haBzbm5T+h4pvIQE8H2S/NdfTtgVsqrcNG9Kaq1MtFpKamYvbs2di7dy8SEhLg4OCA9957D3PmzFHcaSOEwNy5c7F69WokJyejffv2+O233+Durvq3by4X8Z/dl2Pw5a5rhbaX1xpPEQlpmLX3Oi5FFQy4dJAaITYlC0IUXDLzG9i4wta+uR2XgiErA5CSlYdO7tZYM6IlDPTK9v3g7uM09Ft+DqnZeRjp5YJv+zUqtczG8/cw99mloW/7NsRIb9cytUFTjt6Mw7jNl2Ggq4NjUzpiy4X7WPNvFGpZmeLwpI5l/l1R0c5HJuL9NRcLbX+d67ARVRZcG0sNDDsF8vLl6LP8HEJjlW/F15VIcHZG53LrqpfLBXZdjsGCQ6FIzVJe5VlHAvzl2w6Nakpf6+y70U8yMGjleTxOzUYLl+rYPKY1TAw0M8TNPzQeYzYGAQB+HuxZ4kDnv0IeYuL2EADApG51Mamb+pfQyosQAiPWXcK/4Ylo5GCBW7EpkAtg/Uet0Lme5peFoAKxsky0W3gC8hc+ucv73yhRZcW1sUhtK05GIjQ2BUZ6OuU6Qd3LdHQkGNrKGT++26TQc3IB9Fl+Di0WHMf7ay7guwO3sCsoBjceyoqctyZWlqm4BPYqYmWZ+PvaIwxbE4DHqdnwsDPHupGtNBZ0AKBrfVtMfDbJ3qy914u9hfvk7QR8ufMqgIJJCidWson5CtbNagAdCXDjUYri5JuQwtnLy1NF3AlGVNVV6rux6PW5GpOMpScK1hRb9G4TtHazfO0rC3s6VYOOBErfWIGCMUNJ6Tk4H/kE519YekFXRwI3K1N42Jmjvr0FHqdmYVPAfchf8RLYjsBozNxzXVG/pakBNo1uDamJ5ifEm9i1Lm48lMH/dgI+2XwZBz5vD0vT/yZBDLyXhE+3XEaeXGBAs5qY807lXFfK1FCv0Lpcs/bcQEd3a558y9HrvhOMqKrjZSzwMlZmTj56L/sXdx+n450m9lj+fvMKa8uOwGjM2nMD+UIovrH2a1oT4fFpCI1LQWhsCm7HpiI0LgXJGbmlHq+asb5Kc4/I5QLJmcrH05EA52Z0KbcTiSwzF/1XnENUYjq8a9fAptGtoaerg1uPUjB0dQBSs/LQ1cMGKz9sAX3dytkJy/EjRFSRVD1/s2eHsPCfUNx9nA5bC0Ms6F/6gNnyVNw31saOUjR2/G9OGSEEElKzC8JPXCr+vfMY54pYcPPlAKOO59P/l1fYkRrrY9WHLdB/xTmcj3yCOftvopVLdXx3sGDsUmtXS6wY3rzSBh3gv5mEXx4/Ul4zCRMRvQr27ODN7tk5fecxRq4rWFh185jWGlux+XUratCmjgTYMqZNkesYvexxajY+WHuxQgZ9/n09FuO3XlHaZi81wpHJHWFRBdaUKqo3rqLuoCOiNwt7dqhUyRk5mLrrvwGwVTXoAP8N2nz5pOtdx0ql8nVtzYss/zrGQjRzrgYJoDS3UXxKFtKz86pE2OH4ESKq7Bh23lBCCHy97wYSUrNRy9oU03t4VHSTyqysJ92KOmlHJabj5e7V8r6Epmn2UuMq01YievMw7Lyh9l99hEPXYqGnI8GvQ5vC2EC3opukEWU96VbESZvjXoiIylflHflI5eZRcia+2VewqOUXXeuiiWO1im3QG47zphARlS/27Lxh5HKBqX9eRWpWHpo6VcP4t2pXdJMIHPdCRFSeGHbeMBvO38O5iCcw1tfFkiGe0KvEtzW/aTjuhYiofPBM9wYJj0/FwsO3AQBf966PWtZmFdwiIiKi8sew84bIyZNj8s4Q5OTJ8VY9awxvw3lQiIjozcCw84ZY6h+OGw9TUM1EH4sHNamU6ywRERGVB4adN8Dl+0n47VQEAMBvQGPYWBhVcIuIiIheH4YdLZeenYcpO69CLoCBzWqiZ2P7im4SERHRa8Wwo+UWHArF/ScZqFnNGPP6Nazo5hAREb12vPVcS8XKMrHn8gNsuxQNiQT4abBnlVhniYiISNMYdrTQjsBozNxzXbH8QPvaVvCqXaNiG0VERFRBeBlLy8TKMpWCDgCci0xErCyz4hpFRERUgRh2tExUYrpS0AH+W0GbiIjoTcSwo2XcrEzx8hQ6XEGbiIjeZAw7WsZeaoz2dawUP3MFbSIietNxgLIWepRcMD5nYte6GNbaiUGHiIjeaAw7Wib6SQYiH6dDV0eC0e3dIDXm7eZERPRm42UsLXMyLAEA0NKlOoMOERERGHa0zonbBWGni4dNBbeEiIiocmDY0SIZOXkIuPsEAMMOERHRcww7WuR8xBPk5MnhWN0YdWzMKro5RERElQLDjhZ5Pl6ni4cNJC9PtkNERPSGYtjREkIInHw2XqdzPV7CIiIieo5hR0uExafikSwLRvo6XPSTiIjoBQw7WuL5XVjeta1gpK9bwa0hIiKqPBh2tITiEhbvwiIiIlLCsKMFkjNycPn+UwBA53rWFdwaIiKiyoVhRwucCU+EXADutmZwrM7VzYmIiF7EsKMFeAmLiIioeAw7VVy+XODU8/l1eMs5ERFRIQw7VVxITDKeZuTCwkgPLVyqV3RziIiIKh2GnSru+SWsju7W0NPln5OIiOhlPDtWcS8uEUFERESFMexUYfEpWbj5KAUSCdDJnbecExERFYVhpwp7fgnL07EaapgZVnBriIiIKieGnSrs+RIRvIRFRERUPIadKio7Lx9nIxIBMOwQERGVhGGniroUlYSMnHzYmBuioYNFRTeHiIio0mLYqaJO3n4MAHirnjUkEkkFt4aIiKjyYtiponjLORERkWoYdqqgqMR0RCWmQ19XgvZ1ecs5ERFRSRh2qqDnd2G1drOEmaFeBbeGiIiocmPYqYIUq5xz4U8iIqJSMexUMenZebgY9QQAx+sQERGpgmGnijkbkYjcfAGXGiZwszKt6OYQERFVegw7VcyLl7B4yzkREVHpGHaqECEEbzknIiJSE8NOFXLzUQriU7JhYqCLNrUsK7o5REREVQLDThVy6lmvTrs6VjDU063g1hAREVUNDDtVyAneck5ERKQ2tcPOyZMny6MdVIqk9BwExyQDADp7cNZkIiIiVakddnr06IHatWtjwYIFiImJKY82URFO30mAEEB9ewvYS40rujlERERVhtph5+HDh5gwYQL+/PNP1KpVC927d8fOnTuRk5NTHu2jZ048W+W8C3t1iIiI1KJ22LGyssLkyZMREhKCixcvwt3dHePHj4eDgwO++OILXL16tTza+UbLy5fjNG85JyIieiVlGqDcvHlzzJw5ExMmTEBaWhrWrVuHFi1aoEOHDrh586am2vjGC45JRkpWHqqZ6KOpU/WKbg4REVGV8kphJzc3F3/++Sd69eoFFxcXHDlyBMuXL0d8fDwiIiLg4uKCwYMHa6SBDx8+xAcffIAaNWrA2NgYjRs3RlBQkOJ5IQTmzJkDe3t7GBsbo1u3bggPD9dI3ZXF87uwOrlbQ1eHsyYTERGpQ+2w8/nnn8Pe3h6ffPIJ3N3dERwcjICAAHz88ccwNTWFq6srfvrpJ9y+fbvMjXv69CnatWsHfX19/PPPP7h16xZ+/vlnVK/+X+/G4sWLsXTpUqxcuRIXL16EqakpunfvjqysrDLXX1k8XyKCl7CIiIjUp6dugVu3bmHZsmUYOHAgDA0Ni9zHyspKI7eoL1q0CE5OTli/fr1im5ubm+L/hRD49ddf8c0336Bfv34AgE2bNsHW1hb79u3DsGHDytyGivYwORO341KhIyno2SEiIiL1qN2z4+/vj/fee6/YoAMAenp66NSpU5kaBgD79+9Hy5YtMXjwYNjY2KBZs2ZYs2aN4vmoqCjExcWhW7duim1SqRRt2rRBQEBAscfNzs5GSkqK0qOyet6r09y5OqqZGFRwa4iIiKoetcOOn58f1q1bV2j7unXrsGjRIo006rm7d+/i999/R926dXHkyBF89tln+OKLL7Bx40YAQFxcHADA1tZWqZytra3iuaL4+flBKpUqHk5OThpttyY9XyKiMy9hERERvRK1w86qVavg4eFRaHvDhg2xcuVKjTTqOblcjubNm+OHH35As2bNMG7cOIwdO7bM9cycORMymUzxqKyTI2bl5uNcxBMAXCKCiIjoVakdduLi4mBvb19ou7W1NWJjYzXSqOfs7e3RoEEDpW3169dHdHQ0AMDOzg4AEB8fr7RPfHy84rmiGBoawsLCQulRGV24+wSZufmwlxqhvr15RTeHiIioSlI77Dg5OeHcuXOFtp87dw4ODg4aadRz7dq1Q1hYmNK2O3fuwMXFBUDBYGU7Ozv4+/srnk9JScHFixfh5eWl0bZUhINXHwEAWrtZQiLhLedERESvQu27scaOHYtJkyYhNzcXXbp0AVAwaHnatGn48ssvNdq4yZMnw9vbGz/88AOGDBmCS5cuYfXq1Vi9ejUAQCKRYNKkSViwYAHq1q0LNzc3zJ49Gw4ODujfv79G2/K6bb8UjT+vPAQA7L/6CN61a2BoK+cKbhUREVHVIxFCCHUKCCEwY8YMLF26VLEelpGREaZPn445c+ZovIEHDx7EzJkzER4eDjc3N0yZMgVjx45Vas/cuXOxevVqJCcno3379vjtt9/g7u6uch0pKSmQSqWQyWSV4pJWrCwT7RaegPyFv4yuRIKzMzpzEVAiIqJnVD1/qx12nktLS0NoaCiMjY1Rt27dEm9Fr+wqW9g5H5mI99dcLLR929i28KpdowJaREREVPmoev5W+zLWc2ZmZmjVqtWrFqcSOFYv3HujK5HA1cqkAlpDRERUtb1S2AkKCsLOnTsRHR2tuJT13J49ezTSsDfZrUepSj/rSiT4YWAjXsIiIiJ6BWrfjbV9+3Z4e3sjNDQUe/fuRW5uLm7evIkTJ05AKpWWRxvfONsuFdxa/2FbF2wb2xZnZ3Tm4GQiIqJXpHbY+eGHH/DLL7/gwIEDMDAwwP/+9z/cvn0bQ4YMgbMzT8hl9eBpBs6EPwYAfNzBDV61a7BHh4iIqAzUDjuRkZHo3bs3AMDAwADp6emQSCSYPHmy4pZwenU7A2MgBNCuTg241DCt6OYQERFVeWqHnerVqyM1tWBMSc2aNXHjxg0AQHJyMjIyMjTbujdMXr4cO4IKlq54rzV7yYiIiDRB7QHKHTt2xLFjx9C4cWMMHjwYEydOxIkTJ3Ds2DF07dq1PNr4xjgV9hjxKdmwNDXA2w1sSy9AREREpVI77CxfvhxZWVkAgK+//hr6+vo4f/48Bg0ahG+++UbjDXyTPB+Y/G4LRxjq6VZwa4iIiLSDWmEnLy8PBw8eRPfu3QEAOjo6mDFjRrk07E0TK8vEybAEAMDQVk4V3BoiIiLtodaYHT09PXz66aeKnh3SnJ2BDyAXQBs3S9S2Nqvo5hAREWkNtQcot27dGiEhIeXQlDdXvlxg57OBye+34cBkIiIiTVJ7zM748eMxZcoUxMTEoEWLFjA1Vb49ukmTJhpr3JviTPhjPEzORDUTfXRvaFfRzSEiItIqaoedYcOGAQC++OILxTaJRAIhBCQSCfLz8zXXujfE9mcDkwc2c4SRPgcmExERaZLaYScqKqo82vHGSkjJwvHQgoHJ77XmwGQiIiJNUzvsuLi4lEc73li7Lj9AvlygpUt11LU1r+jmEBERaR21w86mTZtKfH7EiBGv3Jg3jVwusD2w4BLWMM6YTEREVC7UDjsTJ05U+jk3NxcZGRkwMDCAiYkJw44azkUmIiYpE+ZGeujd2L6im0NERKSV1L71/OnTp0qPtLQ0hIWFoX379ti2bVt5tFFrbb9UcLv5gGY1YWzAgclERETlQe2wU5S6deti4cKFhXp9qHiJadk4eisOADCsFS9hERERlReNhB2gYHblR48eaepwWm/35QfIzRfwdKqGBg4WFd0cIiIiraX2mJ39+/cr/SyEQGxsLJYvX4527dpprGHaTAiB7YHPZkzm7eZERETlSu2w079/f6WfJRIJrK2t0aVLF/z888+aapdWu3A3CVGJ6TAz1MM7TRwqujlERERaTe2wI5fLy6Mdb5Rtz2ZM7tvUAaaGav8JiIiISA0aG7NDqnmanoPDNwoGJr/PuXWIiIjKndphZ9CgQVi0aFGh7YsXL8bgwYM10ihttvvKA+Tky9GopgUa1ZRWdHOIiIi0ntph58yZM+jVq1eh7T179sSZM2c00iht9eLA5PfYq0NERPRaqB120tLSYGBgUGi7vr4+UlJSNNKoymbhwoWQSCSYNGkSAODevXuQSCRFPnbt2lXscXR0dOD/5Vu4v+gdfNDWVVHmxx9/VOzTt29fODs7w8jICPb29vjwww95Sz8REVEZqB12GjdujB07dhTavn37djRo0EAjjapMAgMDsWrVKjRp0kSxzcnJCbGxsUqPb7/9FmZmZujZs2exxxq38hgcfTdj/Gp/xMbGYt26dZBIJBg0aJBin86dO2Pnzp0ICwvD7t27ERkZiXfffbdcXyMREZE2U/tWoNmzZ2PgwIGIjIxEly5dAAD+/v7Ytm1bib0aVVFaWhqGDx+ONWvWYMGCBYrturq6sLOzU9p37969GDJkCMzMzIo8liwjF6dicqFrVh0fd28GO7vq+Ouvv9C5c2fUqlVLsd/kyZMV/+/i4oIZM2agf//+yM3Nhb6+voZfIRERkfZTu2enT58+2LdvHyIiIjB+/Hh8+eWXePDgAY4fP15oDp6qztfXF71790a3bt1K3O/y5csICQnBmDFjit1nb/ADZOfJ4WFnjqZO1RAfH49Dhw6VWCYpKQlbt26Ft7c3gw4REdEreqVJXnr37o3evXtrui2Vyvbt23HlyhUEBgaWuu/atWtRv359eHt7F/n8ywOTJRIJNm7cCHNzcwwcOLDQ/tOnT8fy5cuRkZGBtm3b4uDBg2V7MURERG8wtXt2AgMDcfHixULbL168iKCgII00qqLFxMRg4sSJ2Lp1K4yMjErcNzMzE3/88UeJPTTBMcm4HZcKQz0d9G9WEwCwbt06DB8+vMjjT506FcHBwTh69Ch0dXUxYsQICCHK9qKIiIjeUGqHHV9fX8TExBTa/vDhQ/j6+mqkURXt8uXLSEhIQPPmzaGnpwc9PT2cPn0aS5cuhZ6eHvLz8xX7/vnnn8jIyMCIESOKPd72ZzMmv9PEAVJjffz7778ICwvDxx9/XOT+VlZWcHd3x9tvv43t27fj77//xoULFzT7IomIiN4Qal/GunXrFpo3b15oe7NmzXDr1i2NNKqide3aFdevX1fa9tFHH8HDwwPTp0+Hrq6uYvvatWvRt29fWFtbF3ms1KxcHLgaCwB479min2vXrkWLFi3g6elZalueL8+RnZ39Sq+FiIjoTad22DE0NER8fLzSHUQAEBsbCz097VjnydzcHI0aNVLaZmpqiho1aihtj4iIwJkzZ/D3338XeRwPDw/0GDUZmbmOqGtjhhYu1ZGSkoJdu3YVuWjqxYsXERgYiPbt26N69eqIjIzE7NmzUbt2bXh5eWn2RRIREb0h1L6M5ePjg5kzZ0Imkym2JScnY9asWXj77bc12rjKbt26dXB0dISPj0+Rz4eFheFw8F0AwLBnA5O3b98OIQTee++9QvubmJhgz5496Nq1K+rVq4cxY8agSZMmOH36NAwNDcv1tRAREWkriVBz5OvDhw/RsWNHPHnyBM2aNQMAhISEwNbWFseOHYOTk1O5NLQ8paSkQCqVQiaTwcLCQiPH3BEYjRl7ruP5b3fOOw0wur2bRo5NREREqp+/1Q47AJCeno6tW7fi6tWrMDY2RpMmTfDee+9V2blgNB12YmWZaLfwBOQv/GZ1JRKcndEZ9lLjMh+fiIiIVD9/v9IgG1NTU4wbN+6VG6ftohLTlYIOAOQLgXuJGQw7REREr9krjyi+desWoqOjkZOTo7S9b9++ZW5UVedmZQodCQr17LhamVRco4iIiN5Qaoedu3fvYsCAAbh+/TokEolisjuJRAIASnPQvKnspcbwG9gYs/bcQL4Q0JVI8MPARuzVISIiqgBqh52JEyfCzc0N/v7+cHNzw6VLl/DkyRN8+eWX+Omnn8qjjVXS0FbO6OhujXuJGXC1MmHQISIiqiBqh52AgACcOHECVlZW0NHRgY6ODtq3bw8/Pz988cUXCA4OLo92Vkn2UmOGHCIiogqm9jw7+fn5MDc3B1CwrMGjR48AAC4uLggLC9Ns64iIiIjKSO2enUaNGuHq1atwc3NDmzZtsHjxYhgYGGD16tWFZlUmIiIiqmhqh51vvvkG6enpAIDvvvsO77zzDjp06IAaNWpgx44dGm8gERERUVm80qSCL0tKSkL16tUVd2RVNeUxgzIRERGVr3KdVPBllpaWmjgMERERkcapPUCZiIiIqCph2CEiIiKtxrBDREREWk3tsHPmzBnk5eUV2p6Xl4czZ85opFFEREREmqJ22OncuTOSkpIKbZfJZOjcubNGGkVERESkKWqHHSFEkbeYP3nyBKamphppFBEREZGmqHzr+cCBAwEUrG4+atQoGBoaKp7Lz8/HtWvX4O3trfkWEhEREZWBymFHKpUCKOjZMTc3h7HxfwtcGhgYoG3bthg7dqzmW0hERERUBiqHnfXr1wMAXF1d8dVXX/GSFREREVUJao/ZmTZtmtKYnfv37+PXX3/F0aNHNdowIiIiIk1QO+z069cPmzZtAgAkJyejdevW+Pnnn9GvXz/8/vvvGm8gERERUVmoHXauXLmCDh06AAD+/PNP2NnZ4f79+9i0aROWLl2q8QYSERERlYXaYScjIwPm5uYAgKNHj2LgwIHQ0dFB27Ztcf/+fY03kIiIiKgs1A47derUwb59+xATE4MjR47Ax8cHAJCQkFDi8upEREREFUHtsDNnzhx89dVXcHV1RevWreHl5QWgoJenWbNmGm8gERERUVmoHXbeffddREdHIygoCEeOHFFs79q1K3755ReNNu5lCxcuhEQiwaRJkxTbsrKy4Ovrixo1asDMzAyDBg1CfHx8ubaDiIiIqo5XWvXczs4O5ubmOHbsGDIzMwEArVq1goeHh0Yb96LAwECsWrUKTZo0Udo+efJkHDhwALt27cLp06fx6NEjxWzPRERERGqHnSdPnqBr165wd3dHr169EBsbCwAYM2YMvvzyS403EADS0tIwfPhwrFmzBtWrV1dsl8lkWLt2LZYsWYIuXbqgRYsWWL9+Pc6fP48LFy6US1uIiIioalE77EyePBn6+vqIjo6GiYmJYvvQoUNx+PBhjTbuOV9fX/Tu3RvdunVT2n758mXk5uYqbffw8ICzszMCAgKKPV52djZSUlKUHkRERKSdVF4u4rmjR4/iyJEjcHR0VNpet27dcrn1fPv27bhy5QoCAwMLPRcXFwcDAwNUq1ZNabutrS3i4uKKPaafnx++/fZbTTeViIiIKiG1e3bS09OVenSeS0pKUloJXRNiYmIwceJEbN26FUZGRho77syZMyGTyRSPmJgYjR2biIiIKhe1w06HDh0Uy0UAgEQigVwux+LFi9G5c2eNNu7y5ctISEhA8+bNoaenBz09PZw+fRpLly6Fnp4ebG1tkZOTg+TkZKVy8fHxsLOzK/a4hoaGsLCwUHoQERGRdlL7MtbixYvRtWtXBAUFIScnB9OmTcPNmzeRlJSEc+fOabRxXbt2xfXr15W2ffTRR/Dw8MD06dPh5OQEfX19+Pv7Y9CgQQCAsLAwREdHK+b/ISIiojeb2mGnUaNGuHPnDpYvXw5zc3OkpaVh4MCB8PX1hb29vUYbZ25ujkaNGiltMzU1RY0aNRTbx4wZgylTpsDS0hIWFhb4/PPP4eXlhbZt22q0LURERFQ1qR12oqOj4eTkhK+//rrI55ydnTXSMFX98ssv0NHRwaBBg5CdnY3u3bvjt99+e61tICIiospLIoQQ6hTQ1dVFbGwsbGxslLY/efIENjY2yM/P12gDX4eUlBRIpVLIZDKO3yEiIqoiVD1/qz1AWQgBiURSaHtaWppG75giIiIi0gSVL2NNmTIFQMHdV7Nnz1a6/Tw/Px8XL15E06ZNNd5AIiIiorJQOewEBwcDKOjZuX79OgwMDBTPGRgYwNPTE1999ZXmW0hERERUBiqHnZMnTwIouPX7f//7H8e2EBERUZWg9t1Y69evL492EBEREZULtQcoExEREVUlDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Sp12PHz80OrVq1gbm4OGxsb9O/fH2FhYUr7ZGVlwdfXFzVq1ICZmRkGDRqE+Pj4CmoxERERVTaVOuycPn0avr6+uHDhAo4dO4bc3Fz4+PggPT1dsc/kyZNx4MAB7Nq1C6dPn8ajR48wcODACmw1ERERVSYSIYSo6Eao6vHjx7CxscHp06fRsWNHyGQyWFtb448//sC7774LALh9+zbq16+PgIAAtG3bVqXjpqSkQCqVQiaTwcLCojxfAhEREWmIqufvSt2z8zKZTAYAsLS0BABcvnwZubm56Natm2IfDw8PODs7IyAgoNjjZGdnIyUlRelBRERE2qnKhB25XI5JkyahXbt2aNSoEQAgLi4OBgYGqFatmtK+tra2iIuLK/ZYfn5+kEqlioeTk1N5Np2IiIgqUJUJO76+vrhx4wa2b99e5mPNnDkTMplM8YiJidFAC4mIiKgy0qvoBqhiwoQJOHjwIM6cOQNHR0fFdjs7O+Tk5CA5OVmpdyc+Ph52dnbFHs/Q0BCGhobl2WQiIiKqJCp1z44QAhMmTMDevXtx4sQJuLm5KT3fokUL6Ovrw9/fX7EtLCwM0dHR8PLyet3NJSIiokqoUvfs+Pr64o8//sBff/0Fc3NzxTgcqVQKY2NjSKVSjBkzBlOmTIGlpSUsLCzw+eefw8vLS+U7sYiIiEi7VepbzyUSSZHb169fj1GjRgEomFTwyy+/xLZt25CdnY3u3bvjt99+K/Ey1st46zkREVHVo+r5u1KHndeFYYeIiKjq0cp5doiIiIjUxbBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIq2lN2FmxYgVcXV1hZGSENm3a4NKlSxXdJCIiIqoEtCLs7NixA1OmTMHcuXNx5coVeHp6onv37khISKjophEREVEF04qws2TJEowdOxYfffQRGjRogJUrV8LExATr1q2r6KYRERFRBdOr6AaUVU5ODi5fvoyZM2cqtuno6KBbt24ICAgoskx2djays7MVP8tkMgBASkpK+TaWiIiINOb5eVsIUeJ+VT7sJCYmIj8/H7a2tkrbbW1tcfv27SLL+Pn54dtvvy203cnJqVzaSEREROUnNTUVUqm02OerfNh5FTNnzsSUKVMUP8vlciQlJaFGjRqQSCQaqyclJQVOTk6IiYmBhYXFay3Pul9/3WUtz7rfrLrLWp51s+6qUr6sdZdECIHU1FQ4ODiUuF+VDztWVlbQ1dVFfHy80vb4+HjY2dkVWcbQ0BCGhoZK26pVq1ZeTYSFhUWZ/sBlKc+6X3/dZS3Put+sustannWz7qpSvqx1F6ekHp3nqvwAZQMDA7Ro0QL+/v6KbXK5HP7+/vDy8qrAlhEREVFlUOV7dgBgypQpGDlyJFq2bInWrVvj119/RXp6Oj766KOKbhoRERFVMK0IO0OHDsXjx48xZ84cxMXFoWnTpjh8+HChQcuvm6GhIebOnVvoktnrKM+6X3/dZS3Put+sustannWz7qpSvqx1a4JElHa/FhEREVEVVuXH7BARERGVhGGHiIiItBrDDhEREWk1hh0iIiLSagw75WjFihVwdXWFkZER2rRpg0uXLqlU7syZM+jTpw8cHBwgkUiwb98+lev08/NDq1atYG5uDhsbG/Tv3x9hYWEql//999/RpEkTxeRPXl5e+Oeff1Qu/6KFCxdCIpFg0qRJKu0/b948SCQSpYeHh4fK9T18+BAffPABatSoAWNjYzRu3BhBQUEqlXV1dS1Ut0Qiga+vb6ll8/PzMXv2bLi5ucHY2Bi1a9fG/PnzS12r5UWpqamYNGkSXFxcYGxsDG9vbwQGBhbar7T3hhACc+bMgb29PYyNjdGtWzeEh4erXH7Pnj3w8fFRzCYeEhKicv25ubmYPn06GjduDFNTUzg4OGDEiBF49OiRSnXPmzcPHh4eMDU1RfXq1dGtWzdcvHhR5ba/6NNPP4VEIsGvv/6qUtlRo0YV+tv36NFDrbpDQ0PRt29fSKVSmJqaolWrVoiOji61bFHvO4lEgh9//FGlutPS0jBhwgQ4OjrC2NhYsRiyKmXj4+MxatQoODg4wMTEBD169FC8X1T5LMnKyoKvry9q1KgBMzMzDBo0SDHBqyrlV69ejbfeegsWFhaQSCRITk5WPFda+aSkJHz++eeoV68ejI2N4ezsjC+++AIymUyluj/55BPUrl0bxsbGsLa2Rr9+/RRLDKnzOSqEQM+ePRW/X1XKvvXWW4X+3p9++qladQcEBKBLly4wNTWFhYUFOnbsiO+++67Esvfu3Sv2/bZr1y6V6o6Li8OHH34IOzs7mJqaonnz5ti9e7dKZSMjIzFgwABYW1vDwsICQ4YMKTQhcHlh2CknO3bswJQpUzB37lxcuXIFnp6e6N69OxISEkotm56eDk9PT6xYsULtek+fPg1fX19cuHABx44dQ25uLnx8fJCenq5SeUdHRyxcuBCXL19GUFAQunTpgn79+uHmzZtqtSMwMBCrVq1CkyZN1CrXsGFDxMbGKh5nz55VqdzTp0/Rrl076Ovr459//sGtW7fw888/o3r16iq398V6jx07BgAYPHhwqWUXLVqE33//HcuXL0doaCgWLVqExYsXY9myZSrVDQAff/wxjh07hs2bN+P69evw8fFBt27d8PDhQ6X9SntvLF68GEuXLsXKlStx8eJFmJqaonv37sjKylKpfHp6Otq3b49FixYV+3xx5TMyMnDlyhXMnj0bV65cwZ49exAWFoa+ffuqVLe7uzuWL1+O69ev4+zZs3B1dYWPjw8eP36sUvnn9u7diwsXLihNH69K2R49eii9B7Zt26Zy+cjISLRv3x4eHh44deoUrl27htmzZ8PIyKjUsi/WGRsbi3Xr1kEikWDQoEEq1T1lyhQcPnwYW7ZsQWhoKCZNmoQJEyZg//79JZYVQqB///64e/cu/vrrLwQHB8PFxQXdunVDenq6Sp8lkydPxoEDB7Br1y6cPn0ajx49wsCBAwGo9lmUkZGBHj16YNasWYXaV1r5R48e4dGjR/jpp59w48YNbNiwAYcPH8aYMWNUqrtFixZYv349QkNDceTIEQgh4OPjg/z8fLU+R3/99VelZYZULTt27Filv/vixYtVLh8QEIAePXrAx8cHly5dQmBgICZMmICzZ8+WWNbJyanQ++3bb7+FmZkZevbsqVLdI0aMQFhYGPbv34/r169j4MCBGDJkCA4cOFBi2fT0dPj4+EAikeDEiRM4d+4ccnJy0KdPH8jl8kK/V40TVC5at24tfH19FT/n5+cLBwcH4efnp9ZxAIi9e/e+cjsSEhIEAHH69OlXPkb16tXF//3f/6m8f2pqqqhbt644duyY6NSpk5g4caJK5ebOnSs8PT1fqY3Tp08X7du3f6WyRZk4caKoXbu2kMvlpe7bu3dvMXr0aKVtAwcOFMOHD1eproyMDKGrqysOHjyotL158+bi66+/Lrbcy+8NuVwu7OzsxI8//qjYlpycLAwNDcW2bdtKLf+iqKgoAUAEBwerXH9RLl26JACI+/fvq11WJpMJAOL48eMq1/3gwQNRs2ZNcePGDeHi4iJ++eUXlcqOHDlS9OvXr8T2lFR+6NCh4oMPPnilsi/r16+f6NKli8rlGzZsKL777julbUW9d14uGxYWJgCIGzduKLbl5+cLa2trsWbNmkJ1v/xZkpycLPT19cWuXbsU+4SGhgoAIiAgoNTyLzp58qQAIJ4+fVrk6y6t/HM7d+4UBgYGIjc3V+2yV69eFQBERESEynUHBweLmjVritjY2GL/tkWVVedzsajybdq0Ed98880rlX1Z06ZNC31+lVTe1NRUbNq0SWk/S0vLQu+Zl8seOXJE6OjoCJlMptgnOTlZSCQScezYsVJfS1mxZ6cc5OTk4PLly+jWrZtim46ODrp164aAgIDX2haZTAYAsLS0VLtsfn4+tm/fjvT0dLWW3vD19UXv3r2VXr+qwsPD4eDggFq1amH48OGIjo5Wqdz+/fvRsmVLDB48GDY2NmjWrBnWrFmjdv1Awd9vy5YtGD16tEoLw3p7e8Pf3x937twBAFy9ehVnz55Fz549VaovLy8P+fn5MDIyUtpubGyscs8WAERFRSEuLk7p9y6VStGmTZvX/r57TiaTQSKRqL32XE5ODlavXg2pVApPT0+Vysjlcnz44YeYOnUqGjZsqHZbT506BRsbG9SrVw+fffYZnjx5onK9hw4dgru7O7p37w4bGxu0adNGrcvPz8XHx+PQoUMYM2aMymW8vb2xf/9+PHz4EEIInDx5Enfu3IGPj0+J5bKzswFA6X2no6MDQ0PDIt93L3+WXL58Gbm5uUrvNw8PDzg7Oxf5fivLZ5Gq5WUyGSwsLKCnp1doe0ll09PTsX79eri5ucHJyUmlujMyMvD+++9jxYoVxa7DWFLdW7duhZWVFRo1aoSZM2ciIyNDpfIJCQm4ePEibGxs4O3tDVtbW3Tq1Emlv9nLLl++jJCQkGLfb0WV9/b2xo4dO5CUlAS5XI7t27cjKysLb731Volls7OzIZFIlCYWNDIygo6Ojlqfc6+s3OPUG+jhw4cCgDh//rzS9qlTp4rWrVurdSyUoWcnPz9f9O7dW7Rr106tcteuXROmpqZCV1dXSKVScejQIZXLbtu2TTRq1EhkZmYKIdT7BvP333+LnTt3iqtXr4rDhw8LLy8v4ezsLFJSUkota2hoKAwNDcXMmTPFlStXxKpVq4SRkZHYsGGDym1/bseOHUJXV1c8fPhQpf3z8/PF9OnThUQiEXp6ekIikYgffvhBrTq9vLxEp06dxMOHD0VeXp7YvHmz0NHREe7u7sWWefm9ce7cOQFAPHr0SGm/wYMHiyFDhpRa/kWa6NnJzMwUzZs3F++//77KZQ8cOCBMTU2FRCIRDg4O4tKlSyrX/cMPP4i3335b0RunTs/Otm3bxF9//SWuXbsm9u7dK+rXry9atWol8vLySi3//Fu9iYmJWLJkiQgODhZ+fn5CIpGIU6dOqfS6n1u0aJGoXr264t+PKm3PysoSI0aMEACEnp6eMDAwEBs3biy1bE5OjnB2dhaDBw8WSUlJIjs7WyxcuFAAED4+Pkpli/os2bp1qzAwMChUT6tWrcS0adNKLf+i0np2VPkse/z4sXB2dhazZs1SueyKFSuEqampACDq1atXZK9OceXHjRsnxowZo/i5qL9NcWVXrVolDh8+LK5duya2bNkiatasKQYMGKBS3QEBAQKAsLS0FOvWrRNXrlwRkyZNEgYGBuLOnTsqve7nPvvsM1G/fv0inyuu/NOnT4WPj4/i/WZhYSGOHDlSatmEhARhYWEhJk6cKNLT00VaWpqYMGGCACDGjRtXbBs1hWGnHFSWsPPpp58KFxcXERMTo1a57OxsER4eLoKCgsSMGTOElZWVuHnzZqnloqOjhY2Njbh69apimzph52VPnz4VFhYWKl1C09fXF15eXkrbPv/8c9G2bVu16/Xx8RHvvPOOyvtv27ZNODo6im3btolr166JTZs2CUtLS7WCVkREhOjYsaMAIHR1dUWrVq3E8OHDhYeHR7FlKnPYycnJEX369BHNmjVT6rYurWxaWpoIDw8XAQEBYvTo0cLV1VXEx8eXWj4oKEjY2toqBVR1ws7LIiMjVb6E9vzf+3vvvae0X58+fcSwYcPUqrtevXpiwoQJxT5fVPkff/xRuLu7i/3794urV6+KZcuWCTMzs0KXBooqGxQUJDw9PRXvu+7du4uePXuKHj16KO1X1GeJOmGntM+i0sJOaeVlMplo3bq16NGjh8jJyVG5bHJysrhz5444ffq06NOnj2jevHmhoFlU+b/++kvUqVNHpKamKrYV9ftV9TPY39+/yEtoRZV//u985syZSvs2btxYzJgxQ+W6MzIyhFQqFT/99FORzxdXfsKECaJ169bi+PHjIiQkRMybN09IpVJx7dq1UsseOXJE1KpVS0gkEqGrqys++OAD0bx5c/Hpp5+W8NvRDIadcpCdnS10dXULvfFHjBgh+vbtq9axXjXs+Pr6CkdHR3H37l21y76sa9euKiXvvXv3Kj40nz8AKN7YRX1LLk3Lli2V/gEXx9nZWelblhBC/Pbbb8LBwUGt+u7duyd0dHTEvn37VC7j6Ogoli9frrRt/vz5ol69emrVLUTByf55WBkyZIjo1atXsfu+/N54foJ+OaB07NhRfPHFF6WWf1FZwk5OTo7o37+/aNKkiUhMTFSr7Mvq1KlTZC/Zy+V/+eUXxfvsxfeejo6OcHFxeaW6raysxMqVK0utOzs7W+jp6Yn58+cr7Tdt2jTh7e2tct1nzpwRAERISEixbXq5fEZGhtDX1y803mvMmDGie/fuKtednJwsEhIShBAF4w3Hjx+veK64z5LnJ+iXA4qzs7NYsmRJqeVfVFLYKa18SkqK8PLyEl27di0UVNT5HMzOzhYmJibijz/+KLX8xIkTi32/derUSe2609LSBABx+PDhUuu+e/euACA2b96stH3IkCGKXlRV6t60aZPQ19dX/N1fVFz5iIiIQuO8hCg4R3zyyScq1/348WPF39rW1lYsXry42H01hWN2yoGBgQFatGgBf39/xTa5XA5/f3+1xr68CiEEJkyYgL179+LEiRNwc3Mr8zHlcrni+n5JunbtiuvXryMkJETxaNmyJYYPH46QkBDo6uqqVW9aWhoiIyNhb29f6r7t2rUrdJvjnTt34OLiolad69evh42NDXr37q1ymYyMDOjoKP9T0tXVfaU7DExNTWFvb4+nT5/iyJEj6Nevn8pl3dzcYGdnp/S+S0lJwcWLF8v9ffdcbm4uhgwZgvDwcBw/fhw1atQo0/FUfe99+OGHuHbtmtJ7z8HBAVOnTsWRI0fUrvfBgwd48uSJSu89AwMDtGrVqszvv7Vr16JFixYqj1ECCn7fubm5ZX7/SaVSWFtbIzw8HEFBQejXr1+pnyUtWrSAvr6+0vstLCwM0dHR8PLyKvNnkSrlU1JS4OPjAwMDA+zfv18x/uhV6hYFX/6RnZ1davkZM2YUer8BwC+//IJ169apXffz8vb29qXW7erqCgcHhyLfb87OzirXvXbtWvTt2xfW1tZKv4OSyj8fV1TU+y0/P1/luq2srFCtWjWcOHECCQkJijs2y1W5x6k31Pbt24WhoaHYsGGDuHXrlhg3bpyoVq2aiIuLK7VsamqqCA4OFsHBwQKAYhzAy3e0FOWzzz4TUqlUnDp1SsTGxioeGRkZKrV7xowZ4vTp0yIqKkpcu3ZNzJgxQ0gkEnH06FGVyr9MnctYX375pTh16pSIiooS586dE926dRNWVlZFfvN42aVLl4Senp74/vvvRXh4uNi6daswMTERW7ZsUbmt+fn5wtnZWUyfPl3lMkIU3MlTs2ZNcfDgQREVFSX27NkjrKysCnXll+Tw4cPin3/+EXfv3hVHjx4Vnp6eok2bNoW65Et7byxcuFBUq1ZNMf6kX79+ws3NTfGNt7TyT548EcHBweLQoUMCgNi+fbsIDg4WsbGxpZbPyckRffv2FY6OjiIkJETp/ZednV1i2bS0NDFz5kwREBAg7t27J4KCgsRHH30kDA0NFd8i1f138eJlrJLKpqamiq+++koEBASIqKgocfz4cdG8eXNRt25dkZWVpVLde/bsEfr6+mL16tUiPDxcLFu2TOjq6op///1XpXbLZDJhYmIifv/990Kvo7TynTp1Eg0bNhQnT54Ud+/eFevXrxdGRkbit99+K7Xszp07xcmTJ0VkZKTYt2+fcHFxEQMHDhRCqPZZ8umnnwpnZ2dx4sQJERQUJLy8vBSXk1UpHxsbK4KDg8WaNWsEAHHmzBkRHBwsnjx5Ump5mUwm2rRpIxo3biwiIiKU9vn0009LLBsZGSl++OEHERQUJO7fvy/OnTsn+vTpIywtLUV8fPwrfY7iWc9ZaWUjIiLEd999J4KCgkRUVJT466+/RK1atUTHjh1V/r398ssvwsLCQuzatUuEh4eLb775RhgZGYn3339fpXaHh4cLiUQi/vnnH6XtpdWdk5Mj6tSpIzp06CAuXrwoIiIixE8//SQkEono1atXqXWvW7dOBAQEiIiICLF582ZhaWkppkyZUuzvVJMYdsrRsmXLhLOzszAwMBCtW7cWFy5cUKnc8y7dlx8jR44stWxR5QCI9evXq1T36NGjhYuLizAwMBDW1taia9eurxx0hFAv7AwdOlTY29sLAwMDUbNmTTF06NAiBwwW58CBA6JRo0bC0NBQeHh4iNWrV6vV1iNHjggAIiwsTK1yKSkpYuLEicLZ2VkYGRmJWrVqia+//lpkZ2erfIwdO3aIWrVqCQMDA2FnZyd8fX1FcnJyof1Ke2/I5XIxe/ZsYWtrKwwNDUXXrl2VXk9p5devX1/k83Pnzi21/PNLX0U9Tp48WWLZzMxMMWDAAOHg4CAMDAyEvb296Nu3r9IAZXX/XbwYdkoqm5GRIXx8fIS1tbXQ19cXLi4uYuzYsUpfTFSpe+3ataJOnTrCyMhIeHp6Ki6FqlJ21apVwtjY+JX+5rGxsWLUqFHCwcFBGBkZiXr16omff/5ZyOXyUsv+73//E46OjkJfX184OzuLb775RvG+VeWzJDMzU4wfP15Ur15dmJiYiAEDBiiCsSrl586dW+w+pZUv7rWV9Hhe9uHDh6Jnz57CxsZG6OvrC0dHR/H++++L27dvq9z2lz0PO6WVjY6OFh07dhSWlpbC0NBQ1KlTR0ydOlUxtk3Vuv38/ISjo6MwMTERXl5e4t9//1W57MyZM4WTk5PIz88v9BpKK3/nzh0xcOBAYWNjI0xMTESTJk3Epk2bVCo7ffp0YWtrK/T19UXdunUV79PXQfLsBRIRERFpJY7ZISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQEb3k1KlTkEgkSE5OruimEJEGMOwQERGRVmPYISIiIq3GsENElY5cLoefnx/c3NxgbGwMT09P/PnnnwD+u8R06NAhNGnSBEZGRmjbti1u3LihdIzdu3ejYcOGMDQ0hKurK37++Wel57OzszF9+nQ4OTnB0NAQderUwdq1a5X2uXz5Mlq2bAkTExN4e3sXWmmaiKoGhh0iqnT8/PywadMmrFy5Ejdv3sTkyZPxwQcf4PTp04p9pk6dip9//hmBgYGwtrZGnz59kJubC6AgpAwZMgTDhg3D9evXMW/ePMyePRsbNmxQlB8xYgS2bduGpUuXIjQ0FKtWrYKZmZlSO77++mv8/PPPCAoKgp6eHkaPHv1aXj8RaRYXAiWiSiU7OxuWlpY4fvw4vLy8FNs//vhjZGRkYNy4cejcuTO2b9+OoUOHAgCSkpLg6OiIDRs2YMiQIRg+fDgeP36Mo0ePKspPmzYNhw4dws2bN3Hnzh3Uq1cPx44dQ7du3Qq14dSpU+jcuTOOHz+Orl27AgD+/vtv9O7dG5mZmTAyMirn3wIRaRJ7doioUomIiEBGRgbefvttmJmZKR6bNm1CZGSkYr8Xg5ClpSXq1auH0NBQAEBoaCjatWundNx27dohPDwc+fn5CAkJga6uLjp16lRiW5o0aaL4f3t7ewBAQkJCmV8jEb1eehXdACKiF6WlpQEADh06hJo1ayo9Z2hoqBR4XpWxsbFK++nr6yv+XyKRACgYT0REVQt7doioUmnQoAEMDQ0RHR2NOnXqKD2cnJwU+124cEHx/0+fPsWdO3dQv359AED9+vVx7tw5peOeO3cO7u7u0NXVRePGjSGXy5XGABGR9mLPDhFVKubm5vjqq68wefJkyOVytG/fHjKZDOfOnYOFhQVcXFwAAN999x1q1KgBW1tbfP3117CyskL//v0BAF9++SVatWqF+fPnY+jQoQgICMDy5cvx22+/AQBcXV0xcuRIjB49GkuXLoWnpyfu37+PhIQEDBkypKJeOhGVE4YdIqp05s+fD2tra/j5+eHu3buoVq0amjdvjlmzZikuIy1cuBATJ05EeHg4mjZtigMHDsDAwAAA0Lx5c+zcuRNz5szB/PnzYW9vj++++w6jRo1S1PH7779j1qxZGD9+PJ48eQJnZ2fMmjWrIl4uEZUz3o1FRFXK8zulnj59imrVqlV0c4ioCuCYHSIiItJqDDtERESk1XgZi4iIiLQae3aIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIq/0/aNuF5S+4oysAAAAASUVORK5CYII=", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "plt.plot(np.arange(len(epochs_x)), epochs_acc, marker = '.')\n", - "plt.xlabel('epoch')\n", - "plt.ylabel('test accuracy')\n", - "plt.ylim(0, 100)\n", - "plt.xticks(np.arange(len(epochs_x)))\n", - "for i, txt in enumerate(epochs_acc):\n", - " if i%5 == 0:\n", - " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", - " else:\n", - " pass\n", - "plt.show()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "speck-rescnn", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/tests/test_nonsequential/exp_set_B/baseline-SCNN-example_3.ipynb b/tests/test_nonsequential/exp_set_B/baseline-SCNN-example_3.ipynb deleted file mode 100644 index a29b616b..00000000 --- a/tests/test_nonsequential/exp_set_B/baseline-SCNN-example_3.ipynb +++ /dev/null @@ -1,1512 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import torch, random\n", - "import torch.nn as nn\n", - "import sinabs.layers as sl\n", - "from tqdm.notebook import tqdm\n", - "\n", - "from tonic.datasets.dvsgesture import DVSGesture\n", - "from tonic.transforms import ToFrame\n", - "from torch.utils.data import DataLoader\n", - "from torch.nn import CrossEntropyLoss\n", - "from torch.optim import Adam\n", - "\n", - "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential\n", - "\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "torch.backends.cudnn.deterministic = True\n", - "random.seed(1)\n", - "torch.manual_seed(1)\n", - "torch.cuda.manual_seed(1)\n", - "np.random.seed(1)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Loading Data" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "batch_size = 3\n", - "num_workers = 1\n", - "epochs = 30\n", - "lr = 5e-4" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "root_dir = \"../DVSGESTURE\"\n", - "_ = DVSGesture(save_to=root_dir, train=True)\n", - "_ = DVSGesture(save_to=root_dir, train=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "n_time_steps = 50\n", - "to_raster = ToFrame(sensor_size=DVSGesture.sensor_size, n_time_bins=n_time_steps)\n", - "\n", - "snn_train_dataset = DVSGesture(save_to=root_dir, train=True, transform=to_raster)\n", - "snn_test_dataset = DVSGesture(save_to=root_dir, train=False, transform=to_raster)" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The transformed array is in shape [Time-Step, Channel, Height, Width] --> (50, 2, 128, 128)\n" - ] - } - ], - "source": [ - "sample_data, label = snn_train_dataset[0]\n", - "print(f\"The transformed array is in shape [Time-Step, Channel, Height, Width] --> {sample_data.shape}\")" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "snn_train_dataloader = DataLoader(snn_train_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True)\n", - "snn_test_dataloader = DataLoader(snn_test_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=False)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Network Module\n", - "\n", - "We need to define a `nn.Module` implementing the network we want the chip to reproduce." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "device: NVIDIA GeForce RTX 3070 Ti\n" - ] - } - ], - "source": [ - "if torch.cuda.is_available():\n", - " device = torch.device('cuda:0')\n", - " print('device: ', torch.cuda.get_device_name(0))\n", - "else:\n", - " device = torch.device('cpu')" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "class SNN(nn.Module):\n", - " def __init__(self) -> None:\n", - " super().__init__()\n", - "\n", - " self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False)\n", - " self.iaf1 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - " self.pool1 = nn.AvgPool2d(2,2)\n", - "\n", - " self.conv2 = nn.Conv2d(10, 10, 2, 1, bias=False)\n", - " self.iaf2 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - " self.pool2 = nn.AvgPool2d(3,3)\n", - "\n", - " self.conv3 = nn.Conv2d(10, 10, 3, 1, bias=False)\n", - " self.iaf3 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - " self.pool3 = nn.AvgPool2d(2,2)\n", - "\n", - " self.flat = nn.Flatten()\n", - "\n", - " self.fc1 = nn.Linear(810, 100, bias=False)\n", - " self.iaf4 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - "\n", - " self.fc2 = nn.Linear(100, 100, bias=False)\n", - " self.iaf5 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - "\n", - " self.fc3 = nn.Linear(100, 100, bias=False)\n", - " self.iaf6 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - "\n", - " self.fc4 = nn.Linear(100, 11, bias=False)\n", - " self.iaf7 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - "\n", - " def detach_neuron_states(self):\n", - " for name, layer in self.named_modules():\n", - " if name != '':\n", - " if isinstance(layer, sl.StatefulLayer):\n", - " for name, buffer in layer.named_buffers():\n", - " buffer.detach_()\n", - "\n", - " def init_weights(self):\n", - " for name, layer in self.named_modules():\n", - " if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear):\n", - " nn.init.xavier_normal_(layer.weight.data)\n", - "\n", - " def forward(self, x):\n", - " \n", - " con1_out = self.conv1(x)\n", - " iaf1_out = self.iaf1(con1_out)\n", - " pool1_out = self.pool1(iaf1_out)\n", - "\n", - " conv2_out = self.conv2(pool1_out)\n", - " iaf2_out = self.iaf2(conv2_out)\n", - " pool2_out = self.pool2(iaf2_out)\n", - "\n", - " conv3_out = self.conv3(pool2_out)\n", - " iaf3_out = self.iaf3(conv3_out)\n", - " pool3_out = self.pool3(iaf3_out)\n", - "\n", - " flat_out = self.flat(pool3_out)\n", - " \n", - " fc1_out = self.fc1(flat_out)\n", - " iaf4_out = self.iaf4(fc1_out)\n", - "\n", - " fc2_out = self.fc2(iaf4_out)\n", - " iaf5_out = self.iaf5(fc2_out)\n", - "\n", - " fc3_out = self.fc3(iaf5_out)\n", - " iaf6_out = self.iaf6(fc3_out)\n", - "\n", - " fc4_out = self.fc4(iaf6_out)\n", - " iaf7_out = self.iaf7(fc4_out)\n", - "\n", - " return iaf7_out" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "snn = SNN().to(device)" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "snn.init_weights()" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "optimizer = Adam(snn.parameters(), lr=lr, betas=(0.9, 0.999), eps=1e-8)\n", - "loss_fn = CrossEntropyLoss()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Define train and test" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "def train(dataloader, model, loss_fn, optimizer, epochs, test_func, dataloader_test):\n", - " epochs_y = []\n", - " epochs_x = []\n", - " epochs_acc = []\n", - " model.train()\n", - "\n", - " for e in range(epochs):\n", - " losses = []\n", - " batches = []\n", - " batch_count = 0\n", - " train_p_bar = tqdm(snn_train_dataloader)\n", - "\n", - " for X, y in train_p_bar:\n", - " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", - " X = X.reshape(-1, 2, 128, 128).to(dtype=torch.float, device=device)\n", - " y = y.to(dtype=torch.long, device=device)\n", - "\n", - " # forward\n", - " pred = model(X)\n", - "\n", - " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", - " pred = pred.reshape(batch_size, n_time_steps, -1)\n", - "\n", - " # accumulate all time-steps output for final prediction\n", - " pred = pred.sum(dim = 1)\n", - " loss = loss_fn(pred, y)\n", - "\n", - " # gradient update\n", - " optimizer.zero_grad()\n", - " loss.backward()\n", - " optimizer.step()\n", - "\n", - " # detach the neuron states and activations from current computation graph(necessary)\n", - " model.detach_neuron_states()\n", - "\n", - " train_p_bar.set_description(f\"Epoch {e} - BPTT Training Loss: {round(loss.item(), 4)}\")\n", - "\n", - " batch_count += 1\n", - " losses.append(loss.item())\n", - " batches.append(batch_count)\n", - "\n", - " epochs_y.append(losses)\n", - " epochs_x.append(batches)\n", - "\n", - " acc = test_func(dataloader_test, model)\n", - " print(f'Epoch {e} accuracy: {acc}')\n", - " epochs_acc.append(acc)\n", - "\n", - " return epochs_x, epochs_y, epochs_acc\n" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [], - "source": [ - "def test(dataloader, model):\n", - " correct_predictions = []\n", - " with torch.no_grad():\n", - " test_p_bar = tqdm(dataloader)\n", - " for X, y in test_p_bar:\n", - " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", - " X = X.reshape(-1, 2, 128, 128).to(dtype=torch.float, device=device)\n", - " y = y.to(dtype=torch.long, device=device)\n", - "\n", - " # forward\n", - " output = model(X)\n", - "\n", - " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", - " output = output.reshape(batch_size, n_time_steps, -1)\n", - "\n", - " # accumulate all time-steps output for final prediction\n", - " output = output.sum(dim=1)\n", - "\n", - " # calculate accuracy\n", - " pred = output.argmax(dim=1, keepdim=True)\n", - "\n", - " # compute the total correct predictions\n", - " correct_predictions.append(pred.eq(y.view_as(pred)))\n", - "\n", - " test_p_bar.set_description(f\"Testing Model...\")\n", - " \n", - " correct_predictions = torch.cat(correct_predictions)\n", - " return correct_predictions.sum().item()/(len(correct_predictions))*100" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Training loop" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "a1826a7c572749a4ae111dcc8af0ec3b", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - " 0%| | 0/359 [00:00" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "y_avg = []\n", - "for y in epochs_y:\n", - " y_avg.append(np.mean(y))\n", - "\n", - "plt.plot(np.arange(len(epochs_x)), y_avg, marker = '.')\n", - "plt.xlabel('epoch')\n", - "plt.ylabel('average loss')\n", - "plt.ylim(0,)\n", - "plt.xticks(np.arange(len(epochs_x)))\n", - "for i, txt in enumerate(y_avg):\n", - " if i%2 == 0:\n", - " pass\n", - " else:\n", - " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAG2CAYAAACZEEfAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAAB7hklEQVR4nO3deXhM1/8H8PdkskeE7DNJJEESBLEWsRdRVUupXe2qpSXUvrSqJGiptoryU2uLllhqj5bYiSVEBCEhEYkIyWTfz++PfDM1ss2QiIz363nmaXPvPcuNY+7HuWeRCCEEiIiIiLSUTkVXgIiIiKg8MdghIiIircZgh4iIiLQagx0iIiLSagx2iIiISKsx2CEiIiKtxmCHiIiItBqDHSIiItJqDHaIiIhIqzHYISIiIq1WocHOyZMn0aNHD8jlckgkEuzZs0flvBAC8+fPh1wuh5GRETp06ICQkBCVazIzM/HFF1/A0tISJiYm6NmzJx4+fPga74KIiIjeZBUa7KSmpsLDwwMrV64s8vzSpUuxfPlyrFy5EoGBgbC1tUWXLl2QnJysvMbb2xu7d+/G9u3bcfr0aaSkpOCDDz5Abm7u67oNIiIieoNJ3pSNQCUSCXbv3o3evXsDyO/Vkcvl8Pb2xowZMwDk9+LY2NhgyZIlGDduHBQKBaysrLBlyxYMGDAAAPDo0SM4ODjg4MGD6Nq1a0XdDhEREb0hdCu6AsWJiIhAbGwsvLy8lMcMDAzQvn17nD17FuPGjcPly5eRnZ2tco1cLkf9+vVx9uzZYoOdzMxMZGZmKn/Oy8vDs2fPYGFhAYlEUn43RURERGVGCIHk5GTI5XLo6BT/suqNDXZiY2MBADY2NirHbWxs8ODBA+U1+vr6qF69eqFrCtIXxdfXF998800Z15iIiIgqQlRUFOzt7Ys9/8YGOwVe7GkRQpTa+1LaNbNmzcKUKVOUPysUCtSoUQNRUVGoWrXqq1WYiIiIXoukpCQ4ODjA1NS0xOve2GDH1tYWQH7vjUwmUx6Pi4tT9vbY2toiKysLCQkJKr07cXFx8PT0LDZvAwMDGBgYFDpetWpVBjtERESVTGmdIG/sOjvOzs6wtbWFv7+/8lhWVhYCAgKUgUzTpk2hp6enck1MTAxu3LhRYrBDREREb48K7dlJSUnB3bt3lT9HREQgKCgI5ubmqFGjBry9veHj4wMXFxe4uLjAx8cHxsbGGDx4MADAzMwMo0ePxpdffgkLCwuYm5tj6tSpaNCgATp37lxRt0VERERvkAoNdi5duoSOHTsqfy4YRzN8+HBs3LgR06dPR3p6OsaPH4+EhAS0aNECR48eVXk398MPP0BXVxf9+/dHeno6OnXqhI0bN0Iqlb72+yEiIqI3zxuzzk5FSkpKgpmZGRQKBcfsEBERVRLqPr/f2DE7RERERGWBwQ4RERFpNQY7REREpNUY7BAREZFWY7BDREREWo3BDhEREWk1BjtERESk1RjsEBERkVZjsENERERajcEOERERaTUGO0RERKTVGOwQERGRVmOwQ0RERFqNwQ4RERFpNQY7REREpNUY7BAREZFWY7BDREREWo3BDhEREWk1BjtERESk1RjsEBERkVZjsENERERajcEOERERaTUGO0RERKTVGOwQERGRVmOwQ0RERFqNwQ4RERFpNQY7REREpNUY7BAREZFWY7BDREREWo3BDhEREWk1BjtERESk1RjsEBERkVZjsENERERajcEOERERaTUGO0RERKTVGOwQERGRVmOwQ0RERFqNwQ4RERFpNQY7REREpNUY7BAREZFWY7BDREREWo3BDhEREWk1BjtERESk1RjsEBERkVZjsENERERajcEOERERaTUGO0RERKTVGOwQERGRVmOwQ0RERFqNwQ4RERFpNQY7REREpNUY7BAREZFWY7BDREREWo3BDhERERWSk5ODuXPnwtnZGUZGRqhZsyYWLFiAvLw85TVCCMyfPx9yuRxGRkbo0KEDQkJCSszXz88PzZo1Q7Vq1WBiYoJGjRphy5Ytha5btWoVnJ2dYWhoiKZNm+LUqVMvfS8MdoiIiKiQJUuWYM2aNVi5ciVCQ0OxdOlSfPfdd/j555+V1yxduhTLly/HypUrERgYCFtbW3Tp0gXJycnF5mtubo45c+bg3LlzuH79OkaOHImRI0fiyJEjymt27NgBb29vzJkzB1evXkXbtm3RrVs3REZGvtS9SIQQ4qVSapGkpCSYmZlBoVCgatWqFV0dIiKiCvfBBx/AxsYG69evVx7r27cvjI2NsWXLFgghIJfL4e3tjRkzZgAAMjMzYWNjgyVLlmDcuHFql9WkSRN0794d3377LQCgRYsWaNKkCVavXq28pm7duujduzd8fX2Vx9R9frNnh4iIiApp06YN/vnnH9y5cwcAcO3aNZw+fRrvv/8+ACAiIgKxsbHw8vJSpjEwMED79u1x9uxZtcoQQuCff/7B7du30a5dOwBAVlYWLl++rJIvAHh5eamd74t0XyoVERERabUZM2ZAoVCgTp06kEqlyM3NxaJFizBo0CAAQGxsLADAxsZGJZ2NjQ0ePHhQYt4KhQJ2dnbIzMyEVCrFqlWr0KVLFwBAfHw8cnNzi8y3oExNsWeHiIioBE5OTpBIJIU+EyZMAAA8fvwYI0aMgFwuh7GxMd577z2EhYWVmGdISAj69u2rzHvFihUal1veduzYga1bt+KPP/7AlStXsGnTJnz//ffYtGmTynUSiUTlZyFEoWMvMjU1RVBQEAIDA7Fo0SJMmTIFJ06ceOV8i8Ngh4iIqASBgYGIiYlRfvz9/QEA/fr1gxACvXv3Rnh4OPbu3YurV6/C0dERnTt3RmpqarF5pqWloWbNmli8eDFsbW2LLdfe3r7Q8VWrVikDnpSUFHz++eewt7eHkZER6tatqzLOpSgdOnQoMojq3r278prk5GSMHTsWqampGDlyJMaNG4c6depg8uTJyjEzBfV+sbclLi6uUK/Mi3R0dFC7dm00atQIX375JT766CNlvpaWlpBKpS+Vb7HlvVQqIiJ6a8Uo0nH2XjxiFOkVXZXXwsrKCra2tsrP/v37UatWLbRv3x5hYWE4f/48Vq9ejebNm8PNzQ2rVq1CSkoKtm3bVmyezZs3x3fffYeBAwfCwMCg2HKvXLmiDLLGjBkDmUwGID/QAoDJkyfj8OHD2Lp1K0JDQzF58mR88cUX2Lt3b7Fl+/n5qQRvN27cgFQqVeYJAGPGjEF6ejpGjBiB4OBgeHl5KQO4gqnnzs7OsLW1VQZ/QP54m4CAAHh6eqr/C0Z+r01mZiYAQF9fH02bNlXJFwD8/f01zvf5At56CoVCABAKhaKiq0JE9EbbfvGBcJ65XzjO2C+cZ+4X2y8+qOgqvVaZmZnCwsJCLFq0SAghxPXr1wUAcffuXZXrbG1txfDhw9XK09HRUfzwww9qlevp6Slq1aol8vLyhBBCuLu7iwULFqhc26RJEzF37lz1bkgI8cMPPwhTU1ORkpIihBAiLS1NSKVS0alTJ2FnZyf2798vIiIihJOTkzA2NhbTp09Xpl28eLEwMzMTfn5+Ijg4WAwaNEjIZDKRlJSkvObjjz8WM2fOVP7s4+Mjjh49Ku7duydCQ0PFsmXLhK6urli3bp3ymu3btws9PT2xfv16cfPmTeHt7S1MTEzE/fv3Vequ7vObA5SJiEgtMYp0zPILRt7/FizJE8Bsvxto52oFmZlRxVbuNdmzZw8SExMxYsQIAECdOnXg6OiIWbNm4ddff4WJiQmWL1+O2NhYxMTElGm5CQkJCA0NxdSpU5VjV9q0aYN9+/Zh1KhRkMvlOHHiBO7cuYMff/xR7bzXr1+PgQMHwsTEBED+YoK5ubmYNGkS/vnnH4wfPx5xcXEQQsDGxkY5PRwApk+fjvT0dIwfPx4JCQlo0aIFjh49ClNTU+U1kZGR0NH570VSamoqxo8fj4cPH8LIyAh16tTB1q1bMWDAAOU1AwYMwNOnT7FgwQLExMSgfv36OHjwIBwdHV/uF6h26FcBsrOzxZw5c4STk5MwNDQUzs7O4ptvvhG5ubnKa/Ly8sTXX38tZDKZMDQ0FO3btxc3btzQqBz27BARle7M3SfCccb+Qp+zd+PLvWxHR0cBoNBn/PjxQghR5DkAYunSpSXmm5CQIMaPHy9sbW2FgYGBqFOnjjhw4IDy/Ndff10oT319fZU8Ll26JDw8PAQAIZVKRdeuXUW3bt1Et27d1L630np2vLy8RJMmTYRUKhXR0dHK45mZmWLYsGECgNDV1RX6+vpi8+bNapUrhBAXLlwQAMSFCxdUjrdq1Uq0b99eREdHi5ycHLFlyxYhkUiEq6ur2nkXeJSYJs7cfSIeJaZpnLY0WtGzU7B646ZNm+Du7o5Lly5h5MiRMDMzw6RJkwD8t3rjxo0b4erqioULF6JLly64ffu2SmRJRESvxtnSBBLkP/Gfdz8+Fa1qWZRr2YGBgcjNzVX+fOPGDXTp0kU5zuTFXpRDhw5h9OjR6Nu3b7F5ZmVloUuXLrC2tsbOnTthb2+PqKioQs8Od3d3HDt2DFFRUWjZsiXWrVuncr5p06YICgqCQqFAVlYWrKys0KJFCzRr1uxVbxsA8ODBAxw7dgwNGzZEt27dIJfLled++uknnD9/Hvv27YOjoyNOnjyJ8ePHQyaToXPnzqXmvX79etSvXx/vvPOOyvEtW7Zg1KhRsLOzg1QqRZMmTTB48GBcuXJFo7rvCIxU9gbqSADfPg0woHkNjfIoC290sHPu3Dn06tVLOULcyckJ27Ztw6VLlwDkD2hasWIF5syZgz59+gAANm3aBBsbG/zxxx8ard5IREQls6xiADMjPSSmZwOAMvCZt/cGqhnroVsDWbmVbWVlpfLz4sWLlYOEARSa0bR371507NgRNWvWLDbP3377Dc+ePcPZs2ehp6cHAEW+JtHV1YWtrS3WrFkDa2trDB48uMj8zMzMAABhYWG4dOmSyuueV7FhwwZYWFjg+vXrmD9/vvJ4eno6Zs+ejd27dyufkw0bNkRQUBC+//77UoOdtLQ0bN++HQsWLCh0rlatWggICEBqaiqSkpIgk8kwYMAAODs7q13vN+m15xs9G6u8Vm/MzMxEUlKSyoeIiEpm5+CIa/O74sGSD/BgyQe4/7//Pj68Cp9vu4oD12MQGhqKnj17wszMDKampmjZsmWp+xmtWLECbm5uMDIygoODAyZPnoyMjAzl+eTkZHh7e8PR0RFGRkZo2bIlNm7ciFGjRhW57srjx49x4MABjB49usRy9+3bh1atWmHChAmwsbFB/fr14ePjo9KDBOQHLzKZDAsXLkS1atUK3c9ff/2FEydOKKefd+nSBb1791Z5Ng0bNgyzZs1S/pyVlYWgoCAEBQUhKysL0dHRCAoKwt27d1XyzsvLw4YNG+Dq6gpra2uV6eHZ2dnIzs5WGQ8DAFKpVGWzzuL8+eefyMzMxNChQ4u9xsTEBDKZDAkJCThy5Ah69epVar4FIp6kKgOdArlC4H58mtp5lJkyf4FWhvLy8sTMmTOFRCIRurq6QiKRCB8fH+X5M2fOCAAq7y+FEGLs2LHCy8ur2HyLegcLjtkhIipWTm6eaP31bmE/YYtYvOuciImJEf7+/gKA+GjeOuE4Y79w+PT/hKlZNTFt2jRx5coVce/ePbF//37x+PHjYvPdunWrMDAwEL///ruIiIgQR44cETKZTHh7eyuv6d+/v6hXr54ICAgQYWFh4qOPPhIAxKVLl4rMc8mSJaJ69eoiPT29xHtyc3MTBgYGYtSoUeLSpUti27ZtwtzcXHzzzTfKaw4ePCh27twp1qxZIwCI5s2bCxsbGxEf/984pR9//FHY29sLPT09UaNGDTF37lyRmZmpUlb79u1F/0FDlWNXIiIiinwOtW/fXiXdkSNHBAAhl8vFjBkzCt1D+/bthbu7uzh+/LgIDw8XGzZsEIaGhmLVqlXKa16cDVWgTZs2YsCAAUX+bg4fPiwOHTokwsPDxdGjR4WHh4d45513RFZWVom/0+fN3R1caHxXzZkHynTsjrpjdt7oYGfbtm3C3t5ebNu2TVy/fl1s3rxZmJubi40bNwoh/gt2Hj16pJJuzJgxomvXrsXmm5GRIRQKhfITFRXFYIeIqAR/X4sWjjP2i4bzj4jkjGwhhBCTJk0StWrVEtk5uWLKjiBhXKetMHHvKPZcfah2vhMmTBDvvvuuyrEpU6aINm3aCCH+mwa9f/9+5XkvLy9RtWpVMWfOnCLzdHNzE59//nmpZbu4uAgHBweRk5OjPLZs2TJha2tbbJqUlBRhY2Mjli1bVmr+z3uVKfsFAc/t27cLnYuJiREjRowQcrlcGBoaCjc3N7Fs2TLl1HQh8gOiF6fB3759WwAQR48eLbLMHTt2iJo1awp9fX1ha2srJkyYIBITE9Wu868Bd5UBjtPM/wKdsl6qQCsGKE+bNg0zZ87EwIEDAQANGjTAgwcP4Ovri+HDh6us3liw0BJQ+iqLBgYGxS7iREREqvLyBFb+m/96ZWRrJ1Qx0EVWVha2bt2KKVOmQFeqg8V96uPn4Zdh3OxDDOrTA/qJkXBzqYVZs2ahd+/exebdpk0bbN26FRcvXsQ777yD8PBwHDx4EMOHDwfw3zRoQ0NDAP8N1q1duzZOnz5dKL9Tp07h9u3b2LFjR6n3JZPJoKenB6lUqjxWt25dxMbGIisrC/r6+oXSmJiYoEGDBqVuB1EgKycPx27GYuauYOXAbk3Hrnh5eUGIF4eF57O1tcWGDRtKTH/ixAnlQpDOliaQmRnB1dW12DwBoH///ujfv3+pdSvKn5ei4HPwFgBgxnt10LuxHPfj0+BkaVxhSxS80cFOWlpaie8in1+9sXHjxgD+W71xyZIlr72+RETa6J9bcbgVm4wqBroY4ekEoPB6M0/jnyA7Iw1pgbtQpfVQGDmOhGuVR+jTpw+OHz+uHEj8ooEDB+LJkydo06YNhBDIycnBZ599hpkzZwLI30OpVatW+Pbbb1G3bl2sX78epqamyrGcL1q/fj2aNm0KDw+PUu+rdevW+OOPP5CXl6d81ty5cwcymazIQAfIH/MZGhqKtm3bFnleCIF7T1JwKiwep8PicT78KVKzcgtdVzB25XU8/F/njKjDN2Ixc9d1AMC4djXxWYdaAFDh6zC90cFOjx49sGjRItSoUQPu7u64evUqli9fjlGjRgHI3yTM29sbPj4+cHFxgYuLC3x8fGBsbFzsaHkiIlKfEAIrj+f36gxt6YhqxvlBwPr161WmQRf8I7Tvhx/Cqd8X2HYxCqclNdGkzSWsWbOm2GDnxIkTWLRoEVatWoUWLVrg7t27mDRpEmQyGebNmwdAdRo0kN+b8cEHHxSaBp2UlIS//voLy5YtK7KsYcOGwc7OTrkH02effYaff/4ZkyZNwhdffIGwsDD4+Phg4sSJyjRTp05Fjx49UKNGDcTFxWHu198gIVGBbh/+twBefEomztyNVwY4sUkZKuVWe24GWwEJACdL42J+62WnqBlRs/yCy2VG1Nm78Zi47SryBDCgmQNmdqtTpvm/ijc62Pn5558xb9485eqNcrkc48aNw1dffaW8Rp3VG4mI6OWcvhuPa1GJMNTTwZi2+dOOC14l+fn5Ka+ztLSErq4u3N3rYXbvBtCRSPD7hUiEZZkh+Wbxr3zmzZuHjz/+GGPGjAGQP1whNTUVn3zyCebMmQMdHR3lNOh9+/ahV69eCAgIwLx58wpNg96+fTuEEBg0aFCRZb24kq+DgwOOHj2KyZMno2HDhrCzs8OkSZMwY8YM5TUPHz7EoEGDEB8fjyrVzJFZvRbMBi5F/z/uol3tRMSlZCE0RnVGr76uDt5xMkcbF0u0qW2JerKq+OtyFGb73UDuc6+OHiakl3uPx+2Y5EIzovIEsObEPczv6f7Su4i/6FpUIsZuvoSs3Dy8526LRR/WL7O8y4JElPTS7i2RlJQEMzMzKBQKVK1ataKrQ0T0xhjw6zlciHiGka2d8HUPdwDA/Pnz8euvvyIqKgq6uv/9m9nT0xO1atXCli1bIITAV3tDsGz6WOjoGWD9xs1Fvjpp2rQpOnfurDL0YNu2bRg1ahRSUlJUxtMUSEhIgLOzM5YuXYpPPvlEo/uJUaQjIj5VOXblRbl5AglpWYhPyUR8cv5/nyRn4v7TVPx+ofgp9HVlVdHOxRJtXCzR3MkchnqF6x2jSMf9+FRsOvsAh0NiITMzxMGJbVHdpOhXZq9KCIFPt1zGkZuPizzf0c0K3/XzgGWVVxvDejcuGf3WnENCWjY8a1ngtxHNi7z/8qDu8/uN7tkhIqKKczHiGS5EPIOeVIJP2uUvzlew7svw4cNVAh0gf1LJgAED0K5dO3Ts2BHWUceRcS8Q1oN8MGNX/quUQyvnqrxK6tGjB5YvX47GjRsrX2PNmzcPPXv2VAY6R44cgRACbm5uuHv3LqZNmwY3NzeMHDlSo/t5fuyKBEAbF0tYVjFQBjTxKVl4lppZqCekJJ93rIXhns6wMi09YJCZGUFmZoSG9tVw5+dkhMenYupf1/B/w5uVSy/IhjP3ceTmY0gk+febJwCpBOjeUI7DIbE4fvsJ3ltxCsv7e6Cdq1Wp+RUlOjEdH6+/iIS0bHjYm2HtsGavLdDRBIMdIiIqUsFYnY+aOih7QY4dO4bIyEjl2Mnnffjhh1izZg18fX0xceJEuLm5wW/XTlyTumDDmfuY5RcMoxt3VF4lzZ07FxKJBHPnzkV0dDSsrKyU4zULKBQKzJo1Cw8fPoS5uTn69u2LRYsWKVc9VsfVyATM2BWs/FkAOBUWX+z15ib6sKyiD8sqBrCsYgAjPSn+vBSlslWGVCLBkJaOagU6zzMx0MXPgxvjw1Vn8c+tOPx25j5Gt1F/ZWJ1nA9/ikUHQwEA87rXQ7cGtiozom7FJmHitqu48zgFw367iLFtnTG1qxsMdNUPVOJTMvHx/11AjCIDta2rYMPId1DF4M0MK/gaC3yNRUT0ousPE9Fz5RlIdSQ4/mUH1LB4+cG0QggsPBCK9acjAABTvVzRxLF6sa+SypIiLRurTtzF+tMRyCmiy2bQOw5o6mgOyyr6sDI1gFUVA5ib6ENXWniDgR2BkcpxN1KJBD596r/SrKYt5+5j3t4Q6Ekl2PWZJxraV3vpvJ73KDEdPX4+jaepWejdSI4fBjQqsucoIzsXiw6EYsv5BwAAd3lV/DSoMWpZVSm1jOSMbAxadx43opNgV80IOz9rVSEzrtR9fjPYAYMdIqIXfbL5Eo7efIw+je2wfECjV85PCAGfg6FYdypCeaw8p0FnZOdi09n7+OX4XSRl5BR5jVQiwemZHTV6SOePuymbNWOEEBj/+xUcuhGLGubG2D+xDaoaqt9bVZSM7FwM+PUcrj1UoJ6sKnZ95gkj/ZJ7a/xvPsb0ndeQkJYNIz0p5vesh/7NHIp9tZaRnYsRGy7ifPgzWJjo469PW6GmGgFSeVD3+f1G741FRESv3+3YZBz931iP8R1rlUmeEokEI1s74fnHZ54AZuwKxrYLkcjMKbwWzcvIzRP481IUOn5/Ar6HbiEpIwduNqbYMKI5FvdpAOn/HuAFPTOaBiwyMyO0qmVRJr0YEokEi/s2hH11I0Q+S8Msv+ASF/orjRACX+8NwbWHClQz1sOvHzctNdABgC71bHBoUjt41rJAenYuZuwKxoQ/rkCRll3o2pzcPHz+x1WcD3+GKga62DTqnQoLdDTxZr5cIyKiCvPL/8bqdKtvi9rWZbeMx/2naSjqUT5rdzAWH76F7g1l+LCxHZrWqA4dHc0G7Aoh8O+tOCw5fAt3HqcAAORmhpji5YYPG9tB+r/82rtZVfhqvs8zM9LDz4Mao9+aczhwPQata1licIuX6+n642IkdlyKgo4E+GlgYziYq//q0dbMEFtHt8DaU+H4/shtHAyORVBkIlYMbIx3nM0B5K+kPWNXMI6FPoaBrg7+b3gz1Lcze6m6vm58jQW+xiIiKhARn4pOy04gTwAHJraBu7zsHmYxinS0XvyvymwnCQCLKvqIT8lSHrOvboTejezQu7EdaluX3mtwJTIBiw/ewsX7zwDkBxATOtbCsFZOb+TMoKKsOxmORQdDYaCrg72ft0YdW82eRZcfJGDg2nPIzhWY/p4bxneo/dJ1uRaViEnbr+L+0zToSIDP33VBv6Z28Dl4C4duxEKqI8GvQ5uic73it2V6XThmRwMMdoiI8k3feQ1/XnqITnWssX5E8zLPv6hBvh81dcD58KfYfTUah4JjVLZXaGhvht6N7NDDQ66c9VSwVo6ORIKNZ+7jcEgsAMBAVwcjWzvjs/a1YGb8amNfXre8PIHRmwJx/PYT1LIywd9ftIGxvnovX+KSM9Dj59N4nJSJ9xvY4pfBTV55KntKZg7m7wvBzssPC53r38weSz8qfTuO14HBjgYY7BARAQ8T0tDhuxPIyRPwG++JJjWql0s5JQ3yTc/KxbHQx9h9NRoBd54g93/dQFIdCdrUtoTMzBB/XopS6R3SkQAfNbWHd2dXyKtV/Kupl/UsNQvdfjyJx0mZ+KipPb7vV3pAkZWThyH/dx6B9xPgYl0Fuye0LtPp35vO3cfXe0NUjr3MwO7ywgHK9NYq2N03RpFe0VUhKnNOTk6QSCSFPhMmTEB2djZmzJiBBg0awMTEBHK5HMOGDcOjR49KzNPPzw/NmjVDbXtbhH/fB4rfJyMkYL/KNfPnzy9Upq2t7UvdQ0mDfI30pejhIcdvI5rj4uxO+KanOxo5VENunkDAnSfYHhhVaNG/LaNbYOlHHpU60AHy1/b5aWBj6EiAnZcfwu9K4V6VFy06cBOB9xNgaqCLXz9uWubr3LgU8RqxYBPTyoQDlEmr/HHhAebsvgGB8t/dl6giBAYGIjf3v9c8N27cQJcuXdCvXz+kpaXhypUrmDdvHjw8PJCQkABvb2/07NkTly5dKjZPc3NzfD55GuafSkS2kKKfdRxGjhwJa2trdO3aVXmdu7s7jh07pvy5qK0cypJFFQMM93TCcE8nRMSn4ud/wuB3NbrQdTpv0B5Mr6pFTQt4d3bFcv87mLvnBjwcqhW77s3Oyw+x6Vz+GjkrBjYql1lRzpYm0JFAJcCUSiSvZRPTssSeHdIaMYp0ZaAD5P/lnO13gz08pFWsrKxga2sLYWSG8FRd7Ni1B7Vq1UL79u1hZmYGf39/9O/fH25ubmjZsiV+/vlnXL58GZGRxe/r1KFDB0RXawBUs0fLRvXww4JZaNiwIU6fPq1yna6uLmxtbZUfK6uX22LgZThbmmDae254cZJWZXzwlmZCx9rwrGWBtKxcTPj9CjKyC0/LD36owOzd+StCT+rkgk51y2ewsMzMCL5lMGW/ojHYIa1xIfxZoWmtlbG7lag0OwIj0Xrxvxi05jR+27QFzbz6FDsgVaFQQCKRoFq1asXm9yw1S7nJ5YSOtfDvv//i9u3baNeuncp1YWFhkMvlcHZ2xsCBAxEeHl5m96QObXnwlkaqI8GKAY1gYaKPW7HJWHQgVOX805RMfLr1MrJy8tCpjjUmdXIp1/oMaF4Dp2d2xLaxLXF6ZsdK2VvO11ikFYQQ2B5Y9L9cZWaGr7k2ROUnRpGu3Mwy7c555GWk4JxOfcQo0gs99DMyMjBz5kwMHjy4xMGbG85EICU5CY9WjUDXZdmQSqVYtWoVunTporymRYsW2Lx5M1xdXfH48WMsXLgQnp6eCAkJgYWFRbnd74sGNK+Bdq5v1lo55cG6qiGWD2iE4b9dxJbzD+BZywLdGsiQk5uHL7ZdRXRiOpwtTbB8QCON1yR6GQWbmFZW7NkhrXD4RizOhz+DVCIp1M29zP/OK61KSm+mkgbqAvkB8Pz58yGXy2FkZIQOHTogJCSklFyBFStWwM3NDUZGRnBwcMDkyZORkZGhPL969Wo0bNgQVatWRdWqVdGqVSscOnSo3O7zRRHxqcrxEynXj8KoZlNIqljgt9MRKq87srOzMXDgQOTl5WHVqlXF5peUkY2NZ+9Dom+EtbuPITAwEIsWLcKUKVNw4sQJ5XXdunVD37590aBBA3Tu3BkHDhwAAGzatKlc7rMkZbmK8ZusvasVPuuQv4L19F3XcfnBM3jvCMLZe09hrC/Frx83hZlR5ZpiX1HYs0OVXkpmDub/nf8Qm9CxFga1qIH78WmIT8nA5B3X8Pe1R3C2MMYUL7cKrimVpZIG6gLA0qVLsXz5cmzcuBGurq5YuHAhunTpgtu3b8PUtOhVgX///XfMnDkTv/32Gzw9PXHnzh2MGDECAPDDDz8AAOzt7bF48WLUrp2/aNumTZvQq1cvXL16Fe7u7uV4x/luxSQBAHIUcch4cA1WH84GAKw7FYHdV6PxcUsnDGwmx6cjhyIiIgL//vtvib06W849QHJGDlxtqmLEe62goyNBo0aNEBoaCl9fX3To0KHIdCYmJmjQoAHCwsLK/B7pP1O6uOJC+FNciUxE39XnlMc/bGwHV5uyW91a27Fnhyq95Ufv4HFSJhwtjDG+Y23lv/p6eNjB58MGAICf/r2LXUUsjkWVV8FA3YLP/v37lQN1hRBYsWIF5syZgz59+qB+/frYtGkT0tLS8McffxSb57lz59C6dWsMHjwYTk5O8PLywqBBg1RmMvXo0QPvv/8+XF1d4erqikWLFqFKlSo4f/58ud/zzssP8e3/xm+kBPtDamwGk9rN0b2BDHbVjBCfkoXlR26iVosuOH05GP+3fU+Jr5jSsnKUO5FP6Fhb5XWIEAKZmZnFps3MzERoaChkMlkZ3R0VRU+qg7nd6xY6vv1iFCdfaIDBDlVqN6IV2Hg2Ag9Xj8LJ6e/CSF9X5ZVGwMbF+KxDLQghMNZ7BqxsbDV6pbFr1y7Uq1cPBgYGqFevHnbv3q1yPicnB3PnzoWzszOMjIxQs2ZNLFiwAHl5eeV1y8V6m9cXysrKwtatWzFq1ChIJBJEREQgNjYWXl5eymsMDAzQvn17nD17tth82rRpg8uXL+PixYsAgPDwcBw8eBDdu3dXue7FV2gJCQkYM2bMK79CK6m9/XkpClP/vIJnAVuQ8NtYKM5sg5FOLj6SBuLnQY1wYloHLP+oPjIPf4+MmDDodZ6EPqvOYPCPh3Hwwk2VwGXYsGGYNWsW/rgQiWepWZBc2wPDxzcQHh6OW7duYfny5di8eTOGDh2qTDN16lQEBAQgIiICFy5cwEcffYSkpCQMHz681PuiV5ORU/j7hJMvNCRIKBQKAUAoFIqKrgppICc3T/T8+ZRwnLFfjFp9TMTExCg//v7+AoA4fvy4yM3NEy0HfC4k+kbCsf88cTDgghgwYICQyWQiKSmp2PzPnj0rpFKp8PHxEaGhocLHx0fo6uqK8+fPK69ZuHChsLCwEPv37xcRERHir7/+ElWqVBErVqx4Hb8Cpe0XHwjnmfuF44z9wnnmfrH94oPXWn5F27Fjh5BKpSI6OloIIcSZM2cEAOXPBcaOHSu8vLxKzOunn34Senp6QldXVwAQn332WaFrTpw4IYyMjISOjo4wNTUVCxcuVLY3IYRYvHixMDU1Fbt27RLBwcGv3N62X3wgnGbuF9XafiyMTKuJb775RgAQP/74o0p7i4iIEACK/HhO/EnsC4oW2Tm5on379qLvgCHCY/4R4Thjv/hw5Oeidu3awtDQUFSvXl20atVKbN++XaV+Bfegp6cn5HK56NOnjwgJCSn1z4Ze3aPENOXf74JPzZkHxKPEtIquWoVT9/nN7SLA7SIqqy3nH2DenhswNdDFsS/bw6bqf7OuvL29sX//fuV4ArlcjuoteiOtzgdwsjDG9tHNUKemA5YsWYJx48YVmf+AAQOQlJSkMvj0vffeQ/Xq1bFt2zYAwAcffAAbGxusX79eeU3fvn1hbGyMLVu2lMdtF1LU5opv0nLur0PXrl2hr6+Pv//+GwBw9uxZtG7dGo8ePVJ5zTJ27FhERUXh8OHDReZz4sQJDBw4EAsXLkSLFi1w9+5dTJo0CWPHjsW8efOU12VlZSEyMhKJiYnYtWsXVqxYASsrKzx4kL/Am1wuh7e3N2bMmAEg/5WPjY3NS7W3pDwDPGryCQBA/5+laNOgFn777TflNSW1t7txKVh/OgJ+Vx4i83+9A3bVjNDIoRoOBscol2pY1Ls+hrR0LPF3TBWrqD3FKuMU8LLG7SJIq8UlZ2Dp4VsAgKld3VQCneJeaaycOgJ21Yxw/2kaJv55A23btSvxlca5c+dUXoMA+Q/V59O0adMG//zzD+7cuQMAuHbtGk6fPo3333+/LG+3RLdikgotn/82dXE/ePAAx44dw5gxY5THCrYxiI2NVbk2Li4ONjbFL742b948fPzxxxgzZgwaNGiADz/8ED4+PvD19VV5Namvr4/atWujWbNm+Oabb5CbmwtbW9tXeoVWVHuzcGuOixfyB6WObO2EkX3ew7///qt2e6ttXQW+fRrg7Mx34d3ZBeYm+ohOTMeB5wIdAPhqb8hb+fqzMtGGtW4qEmdjUaXkcyAUyRk5aGBnhqEv/It0z549SExMVM6iKXjg1anpgA0upui76iwu3n8G01Q9ZGXGvpi1UmxsbKEHo42NjcoDdMaMGVAoFKhTpw6kUilyc3OxaNEiDBo0qIzutGRCCPx+IarQcR0JtG5V2eJs2LAB1tbWKuNqnJ2dYWtrC39/fzRu3BhAfhAcEBCAJUuWFJtXWloadHRU/w0olUohhCh2+YI9e/YgJycHTk5OAP5rb0W1nYKen6K82N62nLuPoxEZyE1NwOg2zvmDVD+oh6SkJI3bm0UVA3h3dsWn7WvhuyO3lYOSCxQEx29LT2BlVdnXuqlI7NmhSufM3XjsCXoEHQng82EDSF9YWGf9+vXo1q0b5HK5ynGJRAJXG1P8MqQJpDoS3I9PQeSzkv81++KqtEIIlWM7duzA1q1b8ccff+DKlSvYtGkTvv/++9e29sj60xE4FvoYOhKorC9krC+FvlT7/3rn5eVhw4YNGD58OHR1//u3m0Qigbe3N3x8fLB7927cuHEDI0aMgLGxMQYPHqy8rmCgboEePXpg9erV2L59OyIiIuDv74958+ahZ8+eyn2gZs+ejVOnTuH+/fsIDg7GzJkz8wfAjx2rUrfS2k5RCs5vOnsf8/aGQAgBXZ382TgSieSV25uhnhRj2jq/FVsuED2PPTtUqWRk52LunhsAgGGtnNDA3kzlfMErDT8/P+Wx519pyGQytHO1woJe7hj7lwJReSbYGxSNXo3sCpVla2tb6muQadOmYebMmRg4cCAAoEGDBnjw4AF8fX3LfZbK2Xvx8D2U/yrvqw/qoWt9W9x5nIxv9t1EeHwqZvkF49ePm5b6gK3Mjh07hsjISIwaNarQuenTpyM9PR3jx49HQkICWrRogaNHj6qssRMZGanSkzN37lxIJBLMnTsX0dHRsLKyQo8ePbBo0SLlNY8fP8bHH3+MmJgYGJtUQWLCM3w5c45yteEX21uB0l6hFbS3305HYMH+mwCAFjJd3JDZKv8My6K9FWy58OL4D/YYkDbT/n/60WsVHR2NoUOHwsLCAsbGxmjUqBEuX76sPJ+SkoLPP/8c9vb2MDIyQt26dbF69eoS8wwJCUHfvn3h5OQEI31dXDv0B6xNDTDFy1V5TcEUcA8PD+Tl5cHb21s5Bfz5VxoF+jWWQcTchIFdXUz76zou3X9WqNxWrVqppAGAo0ePwtPTU/lzca89ynvqeXRiOj7/4ypy8wT6NLbDcE8nyMyM0N7VGisHN4GeVIKjNx/jr9e0tlBFTXv38vKCEAKurq6FzkkkEsyfPx8xMTHIyMhAQEAA6tevr3LNiRMn4PvjamXddXV18fXXX+Pu3btIT09HZGQkfvnlF1SrVg3ZuXl4nJSByQuWY+ORi5j4+0WIel0hNamOnXnvYNvF/O1KimpvBa/Qnm87L2rVqhV+27FXGeiM71ALeVHXyqW9cfwHvW3Ys0NlJiEhAa1bt0bHjh1x6NAhWFtb4969eyobEE6ePBnHjx/H1q1b4eTkhKNHj2L8+PGQy+Xo1atXkfmmpaWhZs2aaP9eD0yZPAUA8FWPeqhq+N8y6UuWLMHq1auhr6+PTz/9FJ06dcLIkSNhZmaGSZMmKV9puLi4wMXFBT4+PqhetQq8PuyHExEp+GTLZTjf2ADXmo7w9fUFAEyaNAnt2rXDkiVL0KtXL+zduxfHjh1T2Qm64F/9NWrUgLu7O65evYrly5cX2dNQVjKyc/HZ1st4lpoFd3lV+PRpoNJ7U09eFVO6uGHJ4VtY8PdNtKppAQfz8ntFsSMwUrlXk44E8O3ToNI8PJ+vu0QC9G1sD2crE8SnZCI+JQvxyZn/+/9MJKRlq6QVIg8pwcdgUr8TIJFill8wjtyIRed6Nhj+yfhC7a2oV2h2dnbK9ubc/iP8+flAVDN0xtih/ZEXtKdc2xvHf9DbhFPPwannZWXmzJk4c+YMTp06Vew19evXx4ABA1Sm8TZt2hTvv/8+vv3222LTCSEw7LeL2DalJxq/PxgX/1iu8oD/4IMPkJOTgyNHjuD27dtwdXVVmZIrhMA333yDX3/9VflK45dffkFN1zoY8Ot5BEcroPhrDrq1aog/tm5W5rtz507MnTsX4eHhqFWrFhYtWoQ+ffoozycnJ2PevHnYvXs34uLiIJfLMWjQIHz11VfQ19d/2V9lib+HaTuvY+flh6hurId9n7cpMpDJzRMYuPYcAu8n4B0nc2z7pGWhsU1loTJPe3+UmIbWi49Dky9AHUn+YF9DXR3cuXIGcX9+BfnYX6FnrvoaVAgBceUvPLt0EFlpyWj+zjv4dfUqlZ6lDh06wMnJCRs3bsSagHtYfOgWUm+dhuTyDiQ+fvhGtDeiN526z28GO2CwU1bq1auHrl274uHDhwgICICdnR3Gjx+vMnDz008/xeXLl7Fnzx7I5XKcOHECPXv2xKFDh9CmTZti89537REmbruK6DWjMGval1gwZ7rK+cWLF2PNmjU4evQoXF1dce3aNXh5eWHFihWlzlR5nJSB3r+cQYwiA01qVIN3Z1e42FTR6GEdHR2NGTNm4NChQ0hPT4erqyvWr1+Ppk2bAig8WLXA0qVLMW3atFLz3759OwYNGoQmbbvgqeck6EiAzaNa4NTOdfDz88OtW7dgZGQET09PLFmyBG5uboh6lob3VpxEalYuZnarg0/b11L7ftTld+Uhpvx5rdDxbWNbolWt17cTtqbSsnIwamMgzocXfn3ZztUS9WRmsKyiDytTA1hWKfjoo7qxPnR0JEUGeToSYHQbZ1yLUuBKZAJynjupIwEa2FdD29qWaONiiSY1qkNfVwcxinSsOBaGHYH5M+q8O7vAu3PhV3JEVDR1n998jUVlJjw8HKtXr8aUKVMwe/ZsXLx4ERMnToSBgQGGDRsGAPjpp58wduxY2NvbQ1dXFzo6Ovi///u/EgMdRXo2vv3fOIaqhnowNyn8L9hXmQJuU9UQv41ojl4rT+NKZCKG/XZRo9cx6ry+i4mJUUlz6NAhjB49Gn379i01/wcPHmDq1Klo3LwVbsUmwwrA9PfqoI2LJRYGBGDChAlo3rw5cnJyMGfOHHh5eeHmzZtwMDfB1z3cMX3XdSw7ehvtXKxQT152wfy9JylY9L99mp73pk97f5iQhk82X8bN/22o+TypRIIlfRuWGugWN8i3oL2kZObgQvhTnAqLx+m78bgbl4JrUYm4FpWIlcfvwlhfCofqRrj9OEWZZ5d6Ngx0iMoJgx0qM3l5eWjWrBl8fHwAAI0bN0ZISAhWr16tEuycP38e+/btg6OjI06ePInx48dDJpOhc+fORea77OhtPEnORE0rE9wzLLrJPj8l193dHUFBQfD29oZcLldrlko1Yz1kP/cv8TwBzNwVjGpGevByty1xRtOSJUvg4OCADRs2KI8VrLlSoGCGToG9e/eiY8eOqFmzZon1ys3NxZAhQ/DlzLlY9JsfBJLRvYEM49rlp3txJeCCNWcuX76Mdu3aoV8ze/iHPob/zceY8mcQ9kxoDUM9aYllquNuXAoGrTuPp6lZsKlqgCfJmcpeDicLE9g+t8jjm+R8+FOM//0KnqVmwcJEH/2a2WPdyYiXmpU0oHkNtHO1wv34NDhZGqukq2Kgi051bdCpbv7sqxhFOk7/L/A5HRaPp6lZKoEOAPwbGocYRfob//qPqDJisENlRiaToV69eirH6tati127dgEA0tPTMXv2bOzevVu5AFzDhg0RFBSE77//vshg51pUIracz1+IbWHv+hj8c9Flv+qU3Ij4VLz4QlcAGLf1ChzMjfBhIzv0amyHWlZVCqXdt28funbtin79+hX7+u55jx8/xoEDB9RaG2XBggWwsLTEaWlDZGTvRBUDXSz9qGGh4KvgNdqBAwcAAJ988gl+//13NG3aFL59GuBqZAKCb9xE4zbz8ejWZeTl5cHd3R1//vknatQovvcqMTERc+bMgZ+fHxISEuDs7Iwv532LNeHVEJ+Sicdrx+BBgur0/AcAelwajv3bN5Z6f6+LEAJbzj/Agr9vIidPoL5dVfz6cTPYVTPCcE+nIgMWdag7yFdmZoR+zRzQr5kD8vIEtgdGYvbuGyrXcGE/ovLDYIfKTOvWrXH79m2VY3fu3IGjY/4Kx9nZ2cjOzlZ76mxObh5m7w6GEMCHje3gWcuy2LJfdUqus6UJdCRQGYMhAWCop4OoZ+n46d+7+Onfu/CwN0Pvxnbo4SGHZRUDAOq9vnvepk2bYGpqqjLwtChnzpzB+vXr0d/nD/jdTICeVIJG8mowMVD9a/v8a7T69esjIyMDixcvVr5Gs6xigInNqmLkoulI8eiCn7buQVt3R4SGhsLQsPgemKysLHTp0gXW1tbYuXMn7O3tcebabSw8Eo40UyPUlVXF0SuXUNXwv56i+ZsO49eZI3G/akOkZ+XCSP/Ve5FeVWZOLr7aE4Idl/LHxfRqJMfiPg2VdXvds5J0dCToWMe6UHvjwn5E5YfBDpWZyZMnw9PTEz4+Pujfvz8uXryItWvXYu3atQCAqlWron379pg2bRqMjIzg6OiIgIAAbN68GcuXL1fmUzAl1/WDTxDyKAlV9AR62GchKCgIWVlZiI6ORlBQEKpUqYLatWsDePUpucWNwejpYYejN2Ox52o0TobF49pDBa49VGDhgVC0c7FE78Z2Kq/vYhTpaFDVAYOHXVN5ffe83377DUOGDCkx0EhOTsbQoUPx8TQfbLuZDIkEeMfZHEYis9C1Ba/RjI2NERUVhdOnT8Pe3l7lmoMbf0Ddd9ohpdUorA0R+LCLA7qX8grtt99+w7Nnz3D27Fno6enhdmwyll2TIM3UAfVkVfH7mBao/sL4Kd2HV2FgLkdydVf8evJehY9BiUvKwKdbL+NKZCJ0JMDMbnUwtm3NCl9okQv7Eb1enI0FzsYqS/v378esWbMQFhYGZ2dnTJkyReV1TmxsLGbNmoWjR4/i2bNncHR0xCeffILJkycrH0AdOnSAjdwBIS5DkZKZA+8WZpjcp22hstq3b48TJ04AKLspuTGK9GJfacSnZGL/tUfYHfQI16ISlccfrRkFtyaeGDBlEX47E4E8AaRcPYi8q7vwLE71Fc+pU6fQrl07BAUFwcPDo9h6BAUF5e/pJMnvrdKRSCBEfi+Vjo4Obt++jVq18mdX1atXD/r6+rhz5w6MjY3h4OCg8hotLy8PZmZm8J4yFSu37UdSdBis5Q5Y/d236N27d7F1eP/992Fubg5jY2P47d6DVB1jGNZpD88+I/H7WE9UM1b9vWZlZUEul+P9wWNw0rgtDHR18M+X7WFfvWJ6K65FJWLclsuITcpAVUNd/Dy4Cdq7WlVIXYpTUnsjotJx6rkGGOy8WWIU6fDeHoQLEc/QuEY17PrUEzrlsEbMqwh/koI9V6OxOygaVzYuQG7yE9gOWao8/+yfdciKuY2IkCsqD7ERI0bgxo0buHTpUon5P4xX4INv/0JcSiZa1rTA/B7u+OqreUhOTsaPP/4IV1dX6OvrQwgBPT095ObmYty4cRg3bhwuXrwIb29v/Prrrxg2bJhy2wJjY2OMmzIbO6KrIi38MhQnN+P48eNo3759kXWoU6cO7t+/j+4f9sMNs5Z49ugBFP/8iqlTvOHz7TeFrv/zzz8xePBgPHjwAFP2R+JCxDN0byDDL0OavORv+eXtuvwQs3YHIysnD7Wtq2DdsGZwtjR57fUgovKl7vOb20XQG2VHYCQ8F/+LCxH565+0c7F64wIdAKhpVQVTvNxwclpHrF48F1kxt6E49yeyEx4h9eYJpFw7jCqNu2P4bxfxg/8dXLr/DE8TEvHXX39hzJgxReZZsCllTm4epvrdRKKRDHXqumPjl33RsGEDVKtWDaampqhfv76yt2rChAnIzc2Fu7s75s+fD5lMhl69emHkyJHKbTgKxi316tULy7+dg0n9u8CsZT+YurbAip9/KfYe8/LyUN3CCnfdBiOzmhM8vXri66/mYsP/rS3y+oINWO3s7DC/pzt0JMCB4BicvRf/Kr9qjeTk5uHb/Tfx5V/XkJWTh851rbF7vCcDHaK3HIMdemPEKNIxyy9YZVbUyn/vvvb9ljQhkUgwrGdnbPx9B1JDA/Bo/QQkntmO6u+ORRX3jrjzOAU//hOGj9acQ4OPv0ZmTi5ETU/ce5KCFztVIyMjERMTg8WHbuF8+DOY6Evx68dNVbbFeFFBQBMSEgKZTKb8pKSkIDIyf68mS0tL6OrqKmfKeXd2RT1ZVaCaHc5eu12oHgWqmlshxcASiow8NHKohi1jWqBJw/qIjY1FVlaWyrUFG7AWBHJ1ZVUxpEX+wPQFf99ETm757hUGAIlpWRixIRDrT0cAACa+WxtrP24G0xJ+f0T0duAAZXpjRMSnqsxOASrPdNxhA/rCoGZzlQGn07q6opqxPk7djceZu/FIrO8F+/peWPJvFJb8GwW5mSHaulihjYslWte2xLa9h/D7+QdYefweAOD7fh5wsflvh+6NGzcWKlcIgcGDByMqKkplm47JkycrZ8Hp6+ujefPmyply+ro6WDGwEZqseoQUver442KkMjApEPxQgUg9B6Q/PY5G9lWxefQ7qGqohzt37kAmkxUaB1Wwvk/BkgIAMKWLK/6+/gi3YpPxx8VIDGvl9Eq/4+LEKNJx8k48fvznDh4lZsBYX4pl/TzQrYGs9MRE9FZgsENvjAfxaYWOVabpuMUtMjfwnRrIzRMIeaTIX1E3LB6XHyTgkSIDOy5FKadEP6+Dm5XaD+vSZsEB+esQDRgwAO3atUPHjh1x9PBhpN+7CKuBPli4PxSetSzx9ZTPYGdnh4HjZ+Dj9Reg3+A9SAL3wSZkO2IjbXAqLAw+Pj6YOHGiSvl5eXnYsGEDhg8fDl3d/75Sqpvo40svN8zbcwPLjt5Bj4byQrO3XtWOwEjM3BWs3N+quoketo1tiTq2HHtHRP/hAGVwgPKbIC45A++tOIVnqVmQIH9BvxeX4NcmaVk5uBjxDKfD4nH8dhzuPUlVOa8jAc7MfFftHq3SZsEB+VPJfX198fDhQ7i5ueHrr+fjzye2OBf+FO7yqnj0+0xYyewR02gUkjNy0NypOj6tm4s5M6YhKCgIdnZ2GD16NGbMmAGp9L/1c44ePYquXbsqN2B9Xm6eQPefTuFWbDKGtqyBhb0bvORvrLAYRTo8ff9V2chT098bEVVunI2lAQY7FUsIgTGbLuGfW3GoK6uKNUOb4FFixlszHffsvXgMXneh0PHXsZlmdGI63v3+BDJzVMfUvONkjg0jmxdawPBlnA9/ioFrz0NHAvz9RRu4y81eOU8hBCbvCMKeoEeFzr3pm5ASUdnhbCyqNLYHRuGfW3HQl+pgxYBGcLQwQataFm9FoAP8t3rz817X6zsdCZCVU3jwsG+fBmoHOtHR0Rg6dCgsLCxgbGyMRo0a4fLly8rzhzf/DMXmCbi/rC8a13ZA586dceFC4eDuRYmJiZgwYQJkMhkMDQ1Rt25dHDx4EDm5eZix67pKoKM49yceLPkACf+sqzSvPYno9eGYHapQD56mKnc0n/6eG9xsTUtJoX0qcjXdiPhUFNW1G5eciVrWhfcBe5E6O767urpi9apfMPvYY2SkZ0An4TS8vLxw9+5dWFkVvchfUVtVREVFQc/QGOO2XMY/t+KgI8nfRmT7wRNIvnYE+lbOaF377QmSiUh9DHaowuTk5mHyjiCkZeWiZU1zjGrtXNFVqjAl7aBdnoraE0yTXiV1dnwfPHgwACDONAzL/e8gwd4eSbu34fr16+jUqVOR+b64VQUAVLWUYdSmQFyNTISBrg5WDm6CVjVMsGd2Pyz/aRW2r10BV5u3L1gmotLxNRZVmF9PhuNKZCJMDXTxfT+PN3LxwNdJZmb02l/fFfQqSf+3VYemvUr79u1Ds2bN0K9fP1hbW6Nx48ZYt25dkdd+0q4m5Ka6uHtyLwxNTEvcLmPfvn1o1aoVJkyYABsbG7jVrYem/cbjyv2nMDPSw+9jWqBLPRtMmDABPXt8gInD+kBfl19nRFQ09uxQhbgRrcAP/ncAAN/0cq+w/ZPo1XqV1N3xff/+/Rg4cGD+7vQm1SHv/y3SdYr/Mw8PD8e///6LIUOG4JfNf2LGb0fxYN/PkGflYOfvP8PFxhTbt2/HlStXEBgY+Er3T0Taj8EOvXYZ2bnw3hGEnDyBbvVt8WFju4qu0ltPZmb0Uj1Kz+/4DgCNGzdGSEhIoR3fO3bsiKCgIDx58gRDp/nggZ8v5jSti80TvIrN19raGqNm+uLT368i19kTtb0SkXhhF1xsTBEVFYVJkybh6NGjJe4eT0QE8DUWVYClh2/jblwKrEwNsOjDBsrdzqnykclkym0oCtStW1e5VUUBExMT1K5dG61atcKhXX9AoiPF33/+jtNhRe+bJZPJYC53xMiNl5Vr/iwa8R7iHj9GVlYWLl++jLi4ODRt2hS6urrQ1dVFQEAAfvrpJ+jq6iI3N7fc7pmIKh8GO/Ranbkbj9/O5O9dtPSjhjAv4xV16fVq3bq1chuKAnfu3FFuVVEUVxtTmBpIIXKz8c3fIcguYt8sM6f6uBF6B5k5OfCqZ4Mto1sgOjJcuVVFp06dEBwcjKCgIOWnWbNmGDJkCIKCglQWPSQi4mssem0UadmY+tc1AMCQFjXQ0c26gmtEr6q0rSpSU1OxaNEi9OzZEzKZDE+fPsWqVauQlhAH58YdERaXgi3nHuDE2q9hZ2cHHx8fLDt6B1dMmiMvYwOsgrdh8siv8c/RwypbVRTs/v48ExMTWFhYFDpORMRgh16br/bdQIwiA04WxpjTvW5FV4fKQPPmzbF7927MmjULCxYsgLOzM1asWIEhQ4YAAKRSKW7duoVNmzYhPj4eFhYWaN68OU6dOoW7wgaz/ILxw7E7qBpxH5BIMGPXdfx56SF0q1ph4ncbcf6P5WjcyAN2dnaYNGkSZsyYUbE3TESVEreLALeLeB3+vvYIX2y7CqmOBDs/bYXGNapXdJWoguXmCfRceRohj5LwnrstHidl4GpUInQkwMLeDTC4hfbtiUZEZYvbRdAbI1aRgbl7bgAAJnSoxUCHAABSHQm+6ekOADgcEourUYkAgKEtHBnoEFGZYrBD5UoIgWk7r0GRno0Gdmb4opNLRVeJ3iB21QtPd//9QiRiFOkVUBsi0lYMdqhcbTn/AKfC4mGgq4MfBjSCnpRNjv4TEZ9a6FiuELgfn1YBtSEibcUnD5Wbu3Ep8DkYCgCY1a0OaquxsSS9XSpyx3ciensw2NEy8+fPh0QiUfnY2toqzz9+/BgjRoyAXC6HsbEx3nvvPYSFhamd//bt2yGRSNC7d2+V405OToXKdbExRfSBX9DWxRLDWjmV0R2SNnnVvbmIiNTBqedayN3dHceOHVP+XLDAmhACvXv3hp6eHvbu3YuqVati+fLl6Ny5M27evAkTE5MS833w4AGmTp2Ktm3bFjoXGBioXLV279WH+HrTETzeMRcmdVqjrYvVW7/JJxWvonZ8J6K3h8Y9OydOnCiHalBZ0tXVha2trfJjZWUFAAgLC8P58+exevVqNG/eHG5ubli1ahVSUlKwbdu2EvPMzc3FkCFD8M0336BmzZqFzltZWcHW1ha5BlWxOOAx0u5ehG41GQwcGmDJoVsccEolqogd34no7aFxsPPee++hVq1aWLhwIaKiosqjTvSKwsLCIJfL4ezsjIEDByI8PBwAkJmZCQAqGydKpVLo6+vj9OnTJea5YMECWFlZYfTo0cVeczcuGaM3XkJebjZSb55AlYZdIJFIOOCUiIgqlMbBzqNHjzBp0iT4+fnB2dkZXbt2xZ9//omsrKzyqB9pqEWLFti8eTOOHDmCdevWITY2Fp6ennj69Cnq1KkDR0dHzJo1CwkJCcjKysLixYsRGxuLmJiYYvM8c+YM1q9fj3Xr1hV5PiM7F8v976Dbj6dw63Ey0u6cR15GCkzqdwLAAadERFSxNA52zM3NMXHiRFy5cgWXLl2Cm5sbJkyYAJlMhokTJ+LatWvlUU9SU7du3dC3b180aNAAnTt3xoEDBwAAmzZtgp6eHnbt2oU7d+7A3NwcxsbGOHHiBLp161bsxonJyckYOnQo1q1bB0tLy0Lnz4c/xfs/ncJP/4QhO1fg3TrWsHp0BsY1m0HX1IIDTomIqMK98nYRjx49wtq1a7F48WLo6uoiIyMDrVq1wpo1a+Du7l5W9SxX2r5dRJcuXVC7dm2sXr1aeUyhUCArKwtWVlZo0aIFmjVrhl9++aVQ2qCgIDRu3FglGMrL+98u1RIdyMasgV51GSyrGGB+z3qoXzULtWrVwv9t2YY6LTpxwCkREZWbct0uIjs7Gzt37sT7778PR0dHHDlyBCtXrsTjx48REREBBwcH9OvX76Ur/7zo6GgMHToUFhYWMDY2RqNGjXD58mXleSEE5s+fD7lcDiMjI3To0AEhISFlUrY2yMzMRGhoKGQymcpxMzMzWFlZISwsDJcuXUKvXr2KTF+nTh0EBwcjKCgIQUFBuHr1Kpq390IVZw/YjvgRulUtMbhFDfzzZXt80FCOjRs3wtraGh/378MBp0RE9EbQeOr5F198oZy5M3ToUCxduhT169dXnjcxMcHixYvh5OT0ypVLSEhA69at0bFjRxw6dAjW1ta4d+8eqlWrprxm6dKlWL58OTZu3AhXV1csXLgQXbp0we3bt2FqavrKdahspk6dih49eqBGjRqIi4vDwoULkZSUhOHDhwMA/vrrL1hZWaFGjRoIDg7GpEmT0Lt3b3h5eSnzGDZsGOzs7ODr6wtDQ0Pln2/UszTM2XMDIfE5yJMawt29Pnz6NEBzJ3MA+T0+GzZswPDhw6Gry1UNiIjozaDxE+nmzZv4+eef0bdvX+jr6xd5jVwux/Hjx1+5ckuWLIGDgwM2bNigPPZ8ECWEwIoVKzBnzhz06dMHQP7YFBsbG/zxxx8YN27cK9ehsnn48CEGDRqE+Ph4WFlZoWXLljh//jwcHR0BADExMZgyZQoeP34MmUyGYcOGYd68eSp5REZGQkfnv06/7Nw8rD8dgRXH7iAjOw86OhLUtDLBgYltoa/733XHjh1DZGQkRo0a9XpuloiISA2vPGanPNWrVw9du3bFw4cPERAQADs7O4wfPx5jx44FAISHh6NWrVq4cuUKGjdurEzXq1cvVKtWDZs2bSoy38zMTOU0bCD/nZ+Dg4PWjtl5GTGKdETEpyItKxffH7mNW7HJAICWNc3h82ED1LTi1g9ERFSx1B2zo3HPjq+vL2xsbAr96/23337DkydPMGPGDM1rW4zw8HCsXr0aU6ZMwezZs3Hx4kVMnDgRBgYGGDZsGGJjYwEANjY2KulsbGzw4MGDEu/hm2++KbN6apsdgZGY5ReMvOfC4GrGepjzfl181NQeEglXQyYiospD4wHKv/76K+rUqVPouLu7O9asWVMmlSqQl5eHJk2awMfHB40bN8a4ceMwduxYlVlFAAo9fIUQJT6QZ82aBYVCofxwcUQgL0/gRrQCSw/fwoxdqoGOBMAfY1qgXzMHBjpERFTpaNyzExsbW2hmD5C/XUBJC9O9DJlMhnr16qkcq1u3Lnbt2gUAyg0uX6xTXFxcod6e5xkYGMDAwKBM61oZxSjScSosHqfD4nHmbjyepha9MKQAoEjPeb2VIyIiKiMaBzsODg44c+YMnJ2dVY6fOXMGcrm8zCoGAK1bt8bt27dVjt25c0c52NbZ2Rm2trbw9/dXjtnJyspCQEAAlixZUqZ1qYwKxt04W5pAZmaElMwcnL/3FKfvxuNU2BPce5Kqcr2xvhSNHKrh3L2neH4gF1dAJiKiykzjYGfMmDHw9vZGdnY23n33XQDAP//8g+nTp+PLL78s08pNnjwZnp6e8PHxQf/+/XHx4kWsXbsWa9euBZD/+srb2xs+Pj5wcXGBi4sLfHx8YGxsjMGDB5dpXSqb58fdSAA4WZggKiENOc+9n9KRAA3sq6GdiyXa1LZE4xrVoa+rgx2BkZjtdwO5QnAFZCIiqvQ0no0lhMDMmTPx008/KffDMjQ0xIwZM/DVV1+VeQX379+PWbNmISwsDM7OzpgyZYpyNlZBfb755hv8+uuvSEhIQIsWLfDLL7+orP1TGm1bQTlGkQ7Pxf+iqD/ZGubGaONiiba1LeFZyxJmxnrF5nE/Po0rIBMR0RtL3ef3S089T0lJQWhoKIyMjODi4lKpx8BoW7Cz7uQ9LDp4q9DxHwc0Qq/GdhVQIyIiorJXblPPC1SpUgXNmzd/2eRUTq5FJWK5/51Cx6USCd6paV4BNSIiIqpYLxXsBAYG4q+//kJkZKTyVVYBPz+/MqkYae5uXDJGbLiI9Ow81LIyQUR8KvIEOO6GiIjeahoHO9u3b8ewYcPg5eUFf39/eHl5ISwsDLGxsfjwww/Lo46khujEdHy8/iIS0rLhYW+G38e2RHJGNsfdEBHRW0/jRQV9fHzwww8/YP/+/dDX18ePP/6I0NBQ9O/fHzVq1CiPOlY68+fPh0QiUfkUrAlUcL5OnTowMTFB9erV0blzZ1y4cKHEPDt06FAoT4lEgu7duyM+JRMf/98FRMc9Q97ZDbiydAisqpmib7dO0H0WzkCHiIjeahoHO/fu3UP37t0B5C/Ol5qaColEgsmTJyunhFP+itIxMTHKT3BwsPKcq6srVq5cieDgYJw+fRpOTk7w8vLCkydPis3Pz89PJb8bN25AKpWiR+8+GLHhIsLjU5H2zyoYPbmJ37duQXBwMLy8vNC5c2dER0e/jlsmIiJ6I2kc7JibmyM5OX9TSDs7O9y4cQMAkJiYiLS0tLKtXSWmq6sLW1tb5cfKykp5bvDgwejcuTNq1qwJd3d3LF++HElJSbh+/Xqx+Zmbm6vk5+/vD2NjY/inO+NGdBKq6wsk3jyF5d9/h3bt2qF27dqYP38+nJ2dC22vQURE9DbRONhp27Yt/P39AQD9+/fHpEmTMHbsWAwaNAidOnUq8wpWVmFhYZDL5XB2dsbAgQMRHh5e5HVZWVlYu3YtzMzM4OHhoXb+/7d+PeRNOuHyo3RUMdDFmiGNkJubC0NDQ5XrjIyMcPr06Ve6FyIiospM4wHKK1euREZGBoD8DTX19PRw+vRp9OnTB/PmzSvzClZGLVq0wObNm+Hq6orHjx9j4cKF8PT0REhICCwsLADkL5Y4cOBApKWlQSaTwd/fH5aWlmrlf/78BYTcuAHbj0eiqq4O/m94M7SoaYFWrVrh22+/Rd26dWFjY4Nt27bhwoULcHFxKc/bJSIieqNptKhgTk4Ofv/9d3Tt2lVlwG1lV96LCqampqJWrVqYPn06pkyZojwWExOD+Ph4rFu3Dv/++y8uXLgAa2vrEvMSQqB5twG4fvkiHMauwq9Dm6JzvfxNT+/du4dRo0bh5MmTkEqlaNKkCVxdXXHlyhXcvHmzzO+LiIioIqn7/NboNZauri4+++wzZGZmvnIF3yYmJiZo0KABwsLCVI7Vrl0bLVu2xPr166Grq4v169eXmtcPh4Jx5fgBVPHwwncfNVQGOgBQq1YtBAQEICUlBVFRUbh48SKys7MLbdpKRET0NtF4zE6LFi1w9erV8qiL1srMzERoaChkMlmx1wghSg0it5x/AJ9fNkDkZmP+5E/Rp4l9kdeZmJhAJpMhISEBR44cQa9evV6p/kRERJWZxmN2xo8fjy+//BIPHz5E06ZNYWJionK+YcOGZVa5ymrq1Kno0aMHatSogbi4OCxcuBBJSUkYPnw4UlNTsWjRIvTs2RMymQxPnz7FqlWr8PDhQ/Tr10+Zx7Bhw2BnZwdfX18AwL5rj/DV3htIuX4Ujdp0wcTuTQqVe+TIEQgh4Obmhrt372LatGlwc3PDyJEjX9u9ExERvWk0DnYGDBgAAJg4caLymEQigRACEokEubm5ZVe7Surhw4cYNGgQ4uPjYWVlhZYtW+L8+fNwdHRERkYGbt26hU2bNiE+Ph4WFhZo3rw5Tp06BXd3d2UekZGR0NHJ73g7cTsOU3YEIetpNDIf3sSS9T8UWa5CocCsWbPw8OFDmJubo2/fvli0aBH09Ire2ZyIiOhtoPGu5w8ePCjxvKOj4ytVqCK8qbuexyjScSQkFr4HQ5GZI9DDQ44fBzSCjo6koqtGRERU4cpt1/PKGMxURjsCIzHTLxgFoairTRUs6+fBQIeIiEhDGgc7mzdvLvH8sGHDXroylC9GkY5ZzwU6AHA3LgVPUzO5zxUREZGGNA52Jk2apPJzdnY20tLSoK+vD2NjYwY7ZSDscQryXni5mCeA+/FpDHaIiIg0pPHU84SEBJVPSkoKbt++jTZt2mDbtm3lUce3zu6rDwsdk0okcLI0roDaEBERVW4aBztFcXFxweLFiwv1+pDm/gyMwu6rjwAABcNzpBIJfPrUZ68OERHRS9D4NVZxpFIpHj16VFbZvZWuRSVi7p78XeSndHFFv2b2uB+fBidLYwY6REREL0njYGffvn0qPwshEBMTg5UrV6J169ZlVrG3TXxKJj7dehlZuXnoXNcGn3esDR0dCYMcIiKiV6RxsNO7d2+VnyUSCaysrPDuu+9i2bJlZVWvt0pObh4+/+MKYhQZqGlpguUDOMWciIiorGgc7OTl5ZVHPd5qvodu4Xz4M5joS7F2WFNUNeSKx0RERGWlTAYo08vbGxSN9acjAADL+nugtrVpBdeIiIhIu2gc7Hz00UdYvHhxoePfffedykaWVLqbj5IwY9d1AMD4DrXwXv3id0UnIiKil6NxsBMQEIDu3bsXOv7ee+/h5MmTZVKpt0FiWhbGbb2EjOw8tHO1wpdebhVdJSIiIq2kcbCTkpICfX39Qsf19PSQlJRUJpXSdrl5AhO3ByHqWToczI3w08BGkHJAMhERUbnQONipX78+duzYUej49u3bUa9evTKplLZb7n8bJ+88gaGeDn4d2gzVjAsHj0RERFQ2NJ6NNW/ePPTt2xf37t3Du+++CwD4559/sG3bNvz1119lXkFtc/hGDH45fg8AsKRvQ9STF78lPREREb06jYOdnj17Ys+ePfDx8cHOnTthZGSEhg0b4tixY2jfvn151FFrhD1Oxpd/XgMAjG7jjF6N7Cq4RkRERNpPIoQQpV+m3ZKSkmBmZgaFQoGqVcunpyUpIxu9V55BeHwqWtY0x9bRLaAr5cx/IiKil6Xu81vjp21gYCAuXLhQ6PiFCxdw6dIlTbN7K+TlCUzZcQ3h8amQmRli5eAmDHSIiIheE42fuBMmTEBUVFSh49HR0ZgwYUKZVErbrDx+F8dCH0NfVwdrhjaFZRWDiq4SERHRW0PjMTs3b95EkyZNCh1v3Lgxbt68WSaV0hYxinTsvhKN5f53AAALe9WHh0O1iq0UERHRW0bjYMfAwACPHz9GzZo1VY7HxMRAV1fj7LTWjsBIzPILRt7/RkS1cDZH/+YOFVspIiKit5DGr7G6dOmCWbNmQaFQKI8lJiZi9uzZ6NKlS5lWrrKKUaSrBDoAEHj/GWIU6RVXKSIioreUxl0xy5YtQ7t27eDo6IjGjRsDAIKCgmBjY4MtW7aUeQUro4j4VJVABwDyBHA/Pg0yM6OKqRQREdFbSuNgx87ODtevX8fvv/+Oa9euwcjICCNHjsSgQYOgp6dXHnWsdJwtTaAjgUrAI5VI4GRpXHGVIiIieku91CAbExMTfPLJJ2VdF60hMzOCb58GmO13A7lCQCqRwKdPffbqEBERVYCXHlF88+ZNREZGIisrS+V4z549X7lS2mBA8xpo52qF+/FpcLI0ZqBDRERUQTQOdsLDw/Hhhx8iODgYEokEBQswSyT5u3bn5uaWbQ0rMZmZEYMcIiKiCqbxbKxJkybB2dkZjx8/hrGxMUJCQnDy5Ek0a9YMJ06cKIcqEhEREb08jXt2zp07h3///RdWVlbQ0dGBjo4O2rRpA19fX0ycOBFXr14tj3oSERERvRSNe3Zyc3NRpUoVAIClpSUePXoEAHB0dMTt27fLtnZEREREr0jjnp369evj+vXrqFmzJlq0aIGlS5dCX18fa9euLbSqMhEREVFF0zjYmTt3LlJTUwEACxcuxAcffIC2bdvCwsICO3bsKPMKEhEREb0KiSiYTvUKnj17hurVqytnZFU2SUlJMDMzg0KhQNWqVSu6OkRERKQGdZ/fZbJzp7m5eVlkQ0RERFTmNB6gTERERFSZMNghIiIircZgh4iIiLSaxsHOyZMnkZOTU+h4Tk4OTp48WSaVIiIiIiorGgc7HTt2xLNnzwodVygU6NixY5lUioiIiKisaBzsCCGKnGL+9OlTmJiYlEmliIiIiMqK2lPP+/TpAyB/d/MRI0bAwMBAeS43NxfXr1+Hp6dn2deQiIiI6BWoHeyYmZkByO/ZMTU1hZGRkfKcvr4+WrZsibFjx5Z9DYmIiIhegdrBzoYNGwAATk5OmDp1Kl9ZERERUaWg8Zid6dOnq4zZefDgAVasWIGjR4+WacWIiIiIyoLGwU6vXr2wefNmAEBiYiLeeecdLFu2DL169cLq1avLvIJEREREr0LjYOfKlSto27YtAGDnzp2wtbXFgwcPsHnzZvz0009lXkEiIiKiV6FxsJOWlgZTU1MAwNGjR9GnTx/o6OigZcuWePDgQZlXkIiIiOhVaBzs1K5dG3v27EFUVBSOHDkCLy8vAEBcXFyJ26sTERERVQSNg52vvvoKU6dOhZOTE9555x20atUKQH4vT+PGjcu8gkRERESvQuNg56OPPkJkZCQuXbqEI0eOKI936tQJP/zwQ5lW7kW+vr6QSCTw9vZWHhNCYP78+ZDL5TAyMkKHDh0QEhJSrvUgIiKiyuOldj23tbWFqakp/P39kZ6eDgBo3rw56tSpU6aVe15gYCDWrl2Lhg0bqhxfunQpli9fjpUrVyIwMBC2trbo0qULkpOTy60uREREVHloHOw8ffoUnTp1gqurK95//33ExMQAAMaMGYMvv/yyzCsIACkpKRgyZAjWrVuH6tWrK48LIbBixQrMmTMHffr0Qf369bFp0yakpaXhjz/+KJe6EBERUeWicbAzefJk6OnpITIyEsbGxsrjAwYMwOHDh8u0cgUmTJiA7t27o3PnzirHIyIiEBsbqxwkDQAGBgZo3749zp49W2x+mZmZSEpKUvkQERGRdlJ7u4gCR48exZEjR2Bvb69y3MXFpVymnm/fvh1XrlxBYGBgoXOxsbEAABsbG5XjNjY2JdbF19cX33zzTdlWlIiIiN5IGvfspKamqvToFIiPj1fZCb0sREVFYdKkSdi6dSsMDQ2Lve757SuA/NdbLx573qxZs6BQKJSfqKioMqszERERvVk0DnbatWun3C4CyA808vLy8N1336Fjx45lWrnLly8jLi4OTZs2ha6uLnR1dREQEICffvoJurq6yh6dgh6eAnFxcYV6e55nYGCAqlWrqnyIiIhIO2n8Guu7775Dhw4dcOnSJWRlZWH69OkICQnBs2fPcObMmTKtXKdOnRAcHKxybOTIkahTpw5mzJiBmjVrwtbWFv7+/so1frKyshAQEIAlS5aUaV2IiIioctI42KlXrx6uX7+O1atXQyqVIjU1FX369MGECRMgk8nKtHKmpqaoX7++yjETExNYWFgoj3t7e8PHxwcuLi5wcXGBj48PjI2NMXjw4DKtCxEREVVOGgc7kZGRcHBwKHKAb2RkJGrUqFEmFVPX9OnTkZ6ejvHjxyMhIQEtWrTA0aNHlft3ERER0dtNIoQQmiSQSqWIiYmBtbW1yvGnT5/C2toaubm5ZVrB1yEpKQlmZmZQKBQcv0NERFRJqPv81niAcnEznVJSUkqcMUVERERUEdR+jTVlyhQA+bOv5s2bpzL9PDc3FxcuXECjRo3KvIJEREREr0LtYOfq1asA8nt2goODoa+vrzynr68PDw8PTJ06texrSERERPQK1A52jh8/DiB/6vePP/7IsS1ERERUKWg8G2vDhg3lUQ8iIiKicqHxAGUiIiKiyoTBDhEREWk1BjtERESk1RjsEBERkVZjsENERERajcEOERERaTUGO0RERKTVGOwQERGRVmOwQ0RERFqNwQ4RERFpNQY7REREpNUY7BAREZFWY7BDREREWo3BDhEREWk1BjtERESk1RjsEBERkVZjsENERERajcEOERERaTUGO0RERKTVGOwQERGRVmOwQ0RERFqNwQ4RERFpNQY7REREpNUY7BAREZFWY7BDREREWo3BDhEREWk1BjtERESk1RjsEBERkVZjsENERERajcEOERERaTUGO0RERKTVGOwQERGRVmOwQ0RERFqNwQ4RERFpNQY7REREpNUY7BAREZFWY7BDREREWo3BDhEREWk1BjtERESk1RjsEBERkVZjsENERERajcEOERERaTUGO0RERKTVGOwQERGRVmOwQ0RERFqNwQ4RERFpNQY7REREpNUY7BAREZFWY7BDREREWo3BDhEREWk1BjtERESk1RjsEBERkVZjsENERERajcEOERERaTUGO0RERKTVGOwQERGRVmOwQ0RERFqNwQ4RERFptTc62PH19UXz5s1hamoKa2tr9O7dG7dv31a5RgiB+fPnQy6Xw8jICB06dEBISEgF1ZiIiIjeNG90sBMQEIAJEybg/Pnz8Pf3R05ODry8vJCamqq8ZunSpVi+fDlWrlyJwMBA2NraokuXLkhOTq7AmhMREdGbQiKEEBVdCXU9efIE1tbWCAgIQLt27SCEgFwuh7e3N2bMmAEAyMzMhI2NDZYsWYJx48aplW9SUhLMzMygUChQtWrV8rwFIiIiKiPqPr/f6J6dFykUCgCAubk5ACAiIgKxsbHw8vJSXmNgYID27dvj7NmzxeaTmZmJpKQklQ8RERFpp0oT7AghMGXKFLRp0wb169cHAMTGxgIAbGxsVK61sbFRniuKr68vzMzMlB8HB4fyqzgRERFVqEoT7Hz++ee4fv06tm3bVuicRCJR+VkIUejY82bNmgWFQqH8REVFlXl9iYiI6M2gW9EVUMcXX3yBffv24eTJk7C3t1cet7W1BZDfwyOTyZTH4+LiCvX2PM/AwAAGBgblV2EiIiJ6Y7zRPTtCCHz++efw8/PDv//+C2dnZ5Xzzs7OsLW1hb+/v/JYVlYWAgIC4Onp+bqrS0RERG+gN7pnZ8KECfjjjz+wd+9emJqaKsfhmJmZwcjICBKJBN7e3vDx8YGLiwtcXFzg4+MDY2NjDB48uIJrT0RERG+CNzrYWb16NQCgQ4cOKsc3bNiAESNGAACmT5+O9PR0jB8/HgkJCWjRogWOHj0KU1PT11xbIiIiehNVqnV2ygvX2SEiIqp8tHKdHSIiIiJNMdghIiIircZgh4iIiLQagx0iIiLSagx2iIiISKsx2CEiIiKtxmCHiIiItBqDHSIiItJqDHaIiIhIqzHYISIiIq3GYIeIiIi0GoMdIiIi0moMdoiIiEirMdghIiIircZgh4iIiLQagx0iIiLSagx2iIiISKsx2CEiIiKtxmCHiIiItBqDHSIiItJqDHaIiIhIqzHYISIiIq3GYIeIiIi0GoMdIiIi0moMdoiIiEirMdghIiIircZgh4iIiLQagx0iIiLSagx2iIiISKsx2CEiIiKtxmCHiIiItBqDHSIiItJqDHaIiIhIqzHYISIiIq3GYIeIiIi0GoMdIiIi0moMdoiIiEirMdghIiIircZgh4iIiLQagx0iIiLSagx2iIiISKsx2CEiIiKtxmCHiIiItBqDHSIiItJqDHaIiIhIqzHYISIiIq3GYIeIiIi0GoMdIiIi0moMdoiIiEirMdghIiIircZgh4iIiLQagx0iIiLSagx2iIiISKsx2CEiIiKtxmCHiIiItBqDHSIiItJqDHaIiIhIqzHYISIiIq3GYIeIiIi0GoMdIiIi0moMdoiIiEirMdghIiIircZgh4iIiLSa1gQ7q1atgrOzMwwNDdG0aVOcOnWqoqtEREREbwCtCHZ27NgBb29vzJkzB1evXkXbtm3RrVs3REZGVnTViIiIqIJJhBCioivxqlq0aIEmTZpg9erVymN169ZF79694evrW2r6pKQkmJmZQaFQoGrVquVZVSIiIioj6j6/dV9jncpFVlYWLl++jJkzZ6oc9/LywtmzZ4tMk5mZiczMTOXPCoUCQP4vjYiIiCqHgud2af02lT7YiY+PR25uLmxsbFSO29jYIDY2tsg0vr6++Oabbwodd3BwKJc6EhERUflJTk6GmZlZsecrfbBTQCKRqPwshCh0rMCsWbMwZcoU5c95eXl49uwZLCwsik3zMpKSkuDg4ICoqKiXej1WkelZNsuuLOlZ9ttV9qumZ9mVr+ySCCGQnJwMuVxe4nWVPtixtLSEVCot1IsTFxdXqLengIGBAQwMDFSOVatWrbyqiKpVq77SH3BFpmfZLLuypGfZb1fZr5qeZVe+sotTUo9OgUo/G0tfXx9NmzaFv7+/ynF/f394enpWUK2IiIjoTVHpe3YAYMqUKfj444/RrFkztGrVCmvXrkVkZCQ+/fTTiq4aERERVTCtCHYGDBiAp0+fYsGCBYiJiUH9+vVx8OBBODo6Vmi9DAwM8PXXXxd6ZVYZ0rNsll1Z0rPst6vsV03Psitf2WVBK9bZISIiIipOpR+zQ0RERFQSBjtERESk1RjsEBERkVZjsENERERajcFOOVq1ahWcnZ1haGiIpk2b4tSpU2qlO3nyJHr06AG5XA6JRII9e/aoXaavry+aN28OU1NTWFtbo3fv3rh9+7ba6VevXo2GDRsqF39q1aoVDh06pHb6F+sikUjg7e2t1vXz58+HRCJR+dja2qpdXnR0NIYOHQoLCwsYGxujUaNGuHz5slppnZycCpUtkUgwYcKEUtPm5ORg7ty5cHZ2hpGREWrWrIkFCxYgLy9P7bonJyfD29sbjo6OMDIygqenJwIDA4u8trT2IYTA/PnzIZfLYWRkhA4dOiAkJESttH5+fujatSssLS0hkUgQFBSkdtnZ2dmYMWMGGjRoABMTE8jlcgwbNgyPHj1Sq+z58+ejTp06MDExQfXq1dG5c2dcuHBB7ft+3rhx4yCRSLBixQq10o4YMaLQn33Lli01Kjs0NBQ9e/aEmZkZTE1N0bJlS0RGRpaatqh2J5FI8N1336lVdkpKCj7//HPY29vDyMgIdevWVdkUubT0jx8/xogRIyCXy2FsbIz33nsPYWFhANT7PimuvamTtqT2Vlr6ktqbOmWX1N40/R59vr2pk7ak9qZu2UW1txkzZpSatqT2pk7ZxbU3ddKW1NbKG4OdcrJjxw54e3tjzpw5uHr1Ktq2bYtu3bohMjKy1LSpqanw8PDAypUrNS43ICAAEyZMwPnz5+Hv74+cnBx4eXkhNTVVrfT29vZYvHgxLl26hEuXLuHdd99Fr169lA9LdQUGBmLt2rVo2LChRunc3d0RExOj/AQHB6uVLiEhAa1bt4aenh4OHTqEmzdvYtmyZWqvjB0YGKhSbsEilf369Ss17ZIlS7BmzRqsXLkSoaGhWLp0Kb777jv8/PPPapUNAGPGjIG/vz+2bNmC4OBgeHl5oXPnzoiOji50bWntY+nSpVi+fDlWrlyJwMBA2NraokuXLkhOTi41bWpqKlq3bo3FixcXe7649Glpabhy5QrmzZuHK1euwM/PD3fu3EHPnj3VqrerqytWrlyJ4OBgnD59Gk5OTvDy8sKTJ0/USl9gz549uHDhgsry8eqkfe+991TawMGDB9VOf+/ePbRp0wZ16tTBiRMncO3aNcybNw+Ghoalpn2+zJiYGPz222+QSCTo27evWmVPnjwZhw8fxtatWxEaGorJkyfjiy++wN69e0tNL4RA7969ER4ejr179+Lq1atwdHRE586dkZqaqtb3SXHt7Z9//ik1bUntrbSyS2pv6tS7pPamyffoi+1N3bTFtTd10hfX3gIDA0tNW1J7U6fs4trbX3/9VWLa0tpauRNULt555x3x6aefqhyrU6eOmDlzpkb5ABC7d+9+6XrExcUJACIgIOCl86hevbr4v//7P7WvT05OFi4uLsLf31+0b99eTJo0Sa10X3/9tfDw8HipOs6YMUO0adPmpdIWZdKkSaJWrVoiLy+v1Gu7d+8uRo0apXKsT58+YujQoWqVlZaWJqRSqdi/f7/KcQ8PDzFnzpwS077YPvLy8oStra1YvHix8lhGRoYwMzMTa9asKTHt8yIiIgQAcfXqVbXLLsrFixcFAPHgwQON0yoUCgFAHDt2TO2yHz58KOzs7MSNGzeEo6Oj+OGHH9RKO3z4cNGrV68S61NS+gEDBqj1563Offfq1Uu8++67aqd3d3cXCxYsUDnWpEkTMXfu3FLT3759WwAQN27cUB7LyckR5ubmYt26dYXSv/h9okl7K+m7SJ32ps53WXHtTZ20JbW34tKr096KSqtJeysqvbrtTZ37Lqm9FZVe3fb2YlpN21pZY89OOcjKysLly5fh5eWlctzLywtnz559rXVRKBQAAHNzc43T5ubmYvv27UhNTUWrVq3UTjdhwgR0794dnTt31rjMsLAwyOVyODs7Y+DAgQgPD1cr3b59+9CsWTP069cP1tbWaNy4MdatW6dx+UD+n9/WrVsxatQotTaGbdOmDf755x/cuXMHAHDt2jWcPn0a77//vlrl5eTkIDc3F4aGhirHjYyMcPr0aY3qHhERgdjYWJW2Z2BggPbt27/2tgfktz+JRKLx3nNZWVlYu3YtzMzM4OHhoVaavLw8fPzxx5g2bRrc3d01ruuJEydgbW0NV1dXjB07FnFxcWqXe+DAAbi6uqJr166wtrZGixYtNHr9XODx48c4cOAARo8erXaaNm3aYN++fYiOjoYQAsePH8edO3fQtWvXUtNmZmYCgErbk0ql0NfXL7Ltvfh9okl7e5XvInXTF9feSktbWnsrKr267a24stVtby+m16S9lXbfpbW3otKr295eTKtpWytz5R5OvYWio6MFAHHmzBmV44sWLRKurq4a5YVX6NnJy8sTPXr00LjH4/r168LExERIpVJhZmYmDhw4oHbabdu2ifr164v09HQhhNCoZ+fgwYNi586d4vr168peIRsbGxEfH19qWgMDA2FgYCBmzZolrly5ItasWSMMDQ3Fpk2b1K57gR07dgipVCqio6PVuj4vL0/MnDlTSCQSoaurKyQSifDx8dGozFatWon27duL6OhokZOTI7Zs2SIkEkmp7eXF9nHmzBkBoFDdx44dK7y8vEpM+7yy6NlJT08XTZs2FUOGDFE77d9//y1MTEyERCIRcrlcXLx4Ue2yfXx8RJcuXZS9cZr07Gzfvl3s379fBAcHi3379gkPDw/h7u4uMjIySk0fExMjAAhjY2OxfPlycfXqVeHr6yskEok4ceKEWvddYMmSJaJ69erKvz/q1D0zM1MMGzZMABC6urpCX19fbN68Wa30WVlZwtHRUfTr1088e/ZMZGZmCl9fXwGgUHsp6vtE3fZW2ndRae1Nne+y4tpbSWnVaW/FpVenvRWXVt32VlR6ddubOr+zktpbcenVaW9FpdWkrZUHBjvloCDYOXv2rMrxhQsXCjc3N43yepVgZ/z48cLR0VFERUVplC4zM1OEhYWJwMBAMXPmTGFpaSlCQkJKTRcZGSmsra1FUFCQ8pgmwc6LUlJShI2NjVi2bFmp1+rp6YlWrVqpHPviiy9Ey5YtNS7Xy8tLfPDBB2pfv23bNmFvby+2bdsmrl+/LjZv3izMzc3Fxo0b1c7j7t27ol27dgKAkEqlonnz5mLIkCGibt26JaYrLth59OiRynVjxowRXbt2LTHt81412MnKyhK9evUSjRs3FgqFQu20KSkpIiwsTJw7d06MGjVKODk5icePH5ea/tKlS8LGxkbloatJsPOiR48eCT09PbFr165S0xf8fR80aJDKdT169BADBw7UqGw3Nzfx+eefF3u+qPTfffedcHV1Ffv27RPXrl0TP//8s6hSpYrw9/dXK/2lS5eEh4eHsu117dpVdOvWTXTr1k3luqK+T9Rtb6V9F5XW3kpLX1J7KymtOu2tqPTqtjd1v4OLa29FpVe3valTdkntrbj06rS34tKq29bKA4OdcpCZmSmkUqnw8/NTOT5x4kTRrl07jfJ62WDn888/F/b29iI8PFzjtC/q1KmT+OSTT0q9bvfu3cpGXPABICQSiZBKpSInJ0fjsjt37lxo7FNRatSoIUaPHq1ybNWqVUIul2tU3v3794WOjo7Ys2eP2mns7e3FypUrVY59++23Gge2QuR/+RY8OPr37y/ef//9Eq9/sX3cu3dPABBXrlxRua5nz55i2LBhJaZ93qsEO1lZWaJ3796iYcOGxfbKqduua9euXWQv2Yvpf/jhB2U7e77t6ejoCEdHx5cu+/mxKMWlz8zMFLq6uuLbb79VuW769OnC09NT7bJPnjwpAKj8Y6G0stPS0oSenl6h8V6jR48uFNyWVn5iYqKIi4sTQuSPORw/frzyXHHfJ+q0N3W+i0pqb6WlL6m9afo9+GJ7Ky69Ou3tZcp+vr0Vl16d9qZO2SW1t+LSq9Pe1Cm7pLZWXjhmpxzo6+ujadOmyhk9Bfz9/eHp6VmuZQsh8Pnnn8PPzw///vsvnJ2dyyTPgvetJenUqROCg4MRFBSk/DRr1gxDhgxBUFAQpFKpRuVmZmYiNDQUMpms1Gtbt25daJrjnTt3NN4MdsOGDbC2tkb37t3VTpOWlgYdHdW/SlKpVKOp5wVMTEwgk8mQkJCAI0eOoFevXhqld3Z2hq2trUrby8rKQkBAQLm3PSB/OnD//v0RFhaGY8eOwcLC4pXyU7ftffzxx7h+/bpK25PL5Zg2bRqOHDmicblPnz5FVFSUWm1PX18fzZs3f+X2t379ejRt2lTtMUpA/u87Ozu7TNqfmZkZrKysEBYWhkuXLqFXr16lfp+U1N5atWr1St9F6nyXFdfeXvZ7sKC9lZa+pPZ2+PBhjct+vr2VVnZJ7a1GjRpql11Ueyut7JLaW25urtplF9XWyl25h1Nvqe3btws9PT2xfv16cfPmTeHt7S1MTEzE/fv3S02bnJwsrl69Kq5evSoAKN/LvjjDoCifffaZMDMzEydOnBAxMTHKT1pamlr1njVrljh58qSIiIgQ169fF7NnzxY6Ojri6NGjaqV/kSavsb788ktx4sQJER4eLs6fPy8++OADYWpqqtbv7OLFi0JXV1csWrRIhIWFid9//10YGxuLrVu3ql3X3NxcUaNGDTFjxgy10wiRP7PCzs5O7N+/X0RERAg/Pz9haWkppk+frnYehw8fFocOHRLh4eHi6NGjwsPDQ7zzzjsiKyur0LWltY/FixcLMzMz4efnJ4KDg8WgQYOETCYTSUlJpaZ9+vSpuHr1qjhw4IAAILZv3y6uXr0qYmJiSi07Oztb9OzZU9jb24ugoCCV9peZmVli2pSUFDFr1ixx7tw5cf/+fXH58mUxevRoYWBgoJy9oenfi+dfK5SUNjk5WXz55Zfi7NmzIiIiQhw/fly0atVK2NnZiaSkJLXK9vPzE3p6emLt2rUiLCxM/Pzzz0IqlYpTp06pVW+FQiGMjY3F6tWrNf7zbt++vXB3dxfHjx8X4eHhYsOGDcLQ0FCsWrVKrfR//vmnOH78uLh3757Ys2ePcHR0FH369BFCqPd9Ulx7Gz16dKlpS2pvpZVdUnv75JNPSkxbWnt7me/RgvZWWtrS2ps6ZRfX3nr37q1WvYtrb+qUXVx7a9u2balpS2pr5Y3BTjn65ZdfhKOjo9DX1xdNmjRRe/r38ePHBYBCn+HDh5eatqh0AMSGDRvUKnvUqFHKOltZWYlOnTq9dKAjhGbBzoABA4RMJhN6enpCLpeLPn36qDVWqMDff/8t6tevLwwMDESdOnXE2rVrNarrkSNHBABx+/ZtjdIlJSWJSZMmiRo1aghDQ0NRs2ZNMWfOHJGZmal2Hjt27BA1a9YU+vr6wtbWVkyYMEEkJiYWeW1p7SMvL098/fXXwtbWVhgYGIh27dqJ4OBgtdJu2LChyPNff/11qekLXkUU9Tl+/HiJadPT08WHH34o5HK50NfXFzKZTPTs2VNlwKimfy+eD3ZKSpuWlia8vLyElZWV0NPTEzVq1BDDhw8XkZGRGpW9fv16Ubt2bWFoaCg8PDyUr0LVSfvrr78KIyOjIv/MS0sfExMjRowYIeRyuTA0NBRubm5i2bJlyoGzpaX/8ccfhb29vfLe586dq2y76nyfFNfe1ElbUnsrLX1J7a20tKW1t5f5Hi1ob6WlLa29qVt2Ue1N3bTFtTd10hfX3tRJW1JbK2+S/90gERERkVbimB0iIiLSagx2iIiISKsx2CEiIiKtxmCHiIiItBqDHSIiItJqDHaIiIhIqzHYISIiIq3GYIeIqAgnTpyARCJBYmJiRVeFiF4Rgx0iIiLSagx2iIiISKsx2CGiN5IQAkuXLkXNmjVhZGQEDw8P7Ny5E8B/r5gOHDgADw8PGBoaokWLFggODlbJY9euXXB3d4eBgQGcnJywbNkylfOZmZmYPn06HBwcYGBgABcXF6xfv17lmsuXL6NZs2YwNjaGp6dnod2miejNx2CHiN5Ic+fOxYYNG7B69WqEhIRg8uTJGDp0KAICApTXTJs2Dd9//z0CAwNhbW2Nnj17Ijs7G0B+kNK/f38MHDgQwcHBmD9/PubNm4eNGzcq0w8bNgzbt2/HTz/9hNDQUKxZswZVqlRRqcecOXOwbNkyXLp0Cbq6uhg1atRruX8iKjvcCJSI3jipqamwtLTEv//+i1atWimPjxkzBmlpafjkk0/QsWNHbN++HQMGDAAAPHv2DPb29ti4cSP69++PIUOG4MmTJzh69Kgy/fTp03HgwAGEhITgzp07cHNzg7+/Pzp37lyoDidOnEDHjh1x7NgxdOrUCQBw8OBBdO/eHenp6TA0NCzn3wIRlRX27BDRG+fmzZvIyMhAly5dUKVKFeVn8+bNuHfvnvK65wMhc3NzuLm5ITQ0FAAQGhqK1q1bq+TbunVrhIWFITc3F0FBQZBKpWjfvn2JdWnYsKHy/2UyGQAgLi7ule+RiF4f3YquABHRi/Ly8gAABw4cgJ2dnco5AwMDlYDnRRKJBED+mJ+C/y/wfEe2kZGRWnXR09MrlHdB/YiocmDPDhG9cerVqwcDAwNERkaidu3aKh8HBwfldefPn1f+f0JCAu7cuYM6deoo8zh9+rRKvmfPnoWrqyukUikaNGiAvLw8lTFARKSd2LNDRG8cU1NTTJ06FZMnT0ZeXh7atGmDpKQknD17FlWqVIGjoyMAYMGCBbCwsICNjQ3mzJkDS0tL9O7dGwDw5Zdfonnz5vj2228xYMAAnDt3DitXrsSqVasAAE5OThg+fDhGjRqFn376CR4eHnjw4AHi4uLQv3//irp1IioHDHaI6I307bffwtraGr6+vggPD0e1atXQpEkTzJ49W/kaafHixZg0aRLCwsLg4eGBffv2QV9fHwDQpEkT/Pnnn/jqq6/w7bffQiaTYcGCBRgxYoSyjNWrV2P27NkYP348nj59iho1amD27NkVcbtEVI44G4uIKp2CmVIJCQmoVq1aRVeHiN5wHLNDREREWo3BDhEREWk1vsYiIiIircaeHSIiItJqDHaIiIhIqzHYISIiIq3GYIeIiIi0GoMdIiIi0moMdoiIiEirMdghIiIircZgh4iIiLQagx0iIiLSav8Pap8fE7Joju4AAAAASUVORK5CYII=", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "plt.plot(np.arange(len(epochs_x)), epochs_acc, marker = '.')\n", - "plt.xlabel('epoch')\n", - "plt.ylabel('test accuracy')\n", - "plt.ylim(0, 100)\n", - "plt.xticks(np.arange(len(epochs_x)))\n", - "for i, txt in enumerate(epochs_acc):\n", - " if i%2 == 0:\n", - " pass\n", - " else:\n", - " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [], - "source": [ - "with open('baseline_exp_set_B_training_metrics.npy', 'wb') as f:\n", - " np.save(f, np.array(epochs_x))\n", - " np.save(f, np.array(epochs_y))\n", - " np.save(f, np.array(epochs_acc))" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "speck-rescnn", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/tests/test_nonsequential/exp_set_B/baseline_exp_set_B_training_metrics.npy b/tests/test_nonsequential/exp_set_B/baseline_exp_set_B_training_metrics.npy deleted file mode 100644 index eafbf40a061d3e9f1e60476b58a21fd33ee6d88e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 172944 zcmeF$RhV4Ixd!U71I`X7JDez)pu^yhgBme2bBmdoTg=SdVrFJ$W|l_G%zQkx-hHmm zbFN!AuX|zDSFnD`LkgBCP`J!{DJ@dkepsz~)2dBA{AcQi|E~S)YSlB2=d4_>Y4uNzH?38i@q#pMNhU`tv?N z#ZU7y{477<=lDPTJiov%@=N?OzrwHbYy3L@m*3zw`7M5%-{E)pJ$|1*U?%>MKVm9> z%%AY5{271FU+_czlE31w`5XS0zvJ)u2mXb1)}!F*oxtFY_@!3$P#yu`r9UD2uT;ORywMu{6uDEX%PxE3hIfF~-WQ z!m6ys>a4+1Y{k}W!?tY4_UyopjI$FvvkSYj z8@sayd$JdMvk&{SANz9v2XYVxa|nlW7>9ENM{*QLa}39F9LIA46P(CNoXjbl%4wX= z8Jx*koXt6$%Xys71zgBQT+Ah0%4J;6613bt>Jj^3J%40mv6FkXNJk2va%X2)>bY9>^UgBk5;Z84j-r{ZE z;a%S2eLmnrKH_6O;Zr{2bH3n9zT#`X;ak3A%6rj&yvO(XDSn!t;b-{)Kga*!{}mEn z@cWDW62Hu^@T>e9zs~>VH~39{i{IvV_+5UF-{%jQi9h6zn93jXC;TaY#-H;S{E)xo zulQ^JhQHg@*Dgnzr}C!JNz!c$M5q8%)}q^ zM@;39`4j$>KjY8&3x3F7@>l#df5YGMcliSA5Mke9L!C*3D56$+|hpAz3#^IV9`mD2GfD zDL>?on93jXC;TaY#-H;S{E)xoulQ^JhQH3! z4w)j7b+a5YGmTl8mD!k`Ihd2Vn45W+m-(2V1z3=USeQjvl*L$_C0LTBSej*6mgQKU z625(tir0S#_FuWnykgzti!sj$NFr*hHS*fY{I5&#^!9nmTbk=Y{Rx}$M)>N zj*PPtJF^SBvKzaz2Ya#?d$SMwvLE|%00(jq2XhFAau|nm1V?fdM{^9vavaBV0u!9b zNu10noXTmO&KaD^S)9!|oXdHf&jnn_MO@4!T*_r!&J|qARb0(AT+4M_&kfwjP29{a z+{$g-&K=yzUEIw*+{=C3&jUQjLp;nQJj!D{&J#SzQ#{QxJj-)D&vahkMPA}%Ug1?< z<8|KPP2S>d-r-%|<9$BhLq6hTKH*b7<8!{?OTOZ3zTsQGW3q0Jf=JfQQ4YzvIm#hf zH%B>Sib&SYa>&dyW?@!lV|M0XPUd26=3!puV}2H3K^9_R7GY5qV{w*XNtR-1mSI_z zV|i9!MOI>rm05*VS&h|MgEd);wONOCS&#MEfDPG*joE}v*^JHEf-TvKt=Wcc*^cem zfgKrVCw68Rc4aqqXAkydFZO01_GLfz=Kv1mAP(jb4&^Wo=LnAED30bBj^#Lx=L9A= zk&`%?Q#h5=IGr;%le0LRb2yjtIG+o+kc+sOOSqKFxST7vlB>9yYq*x{xSkuhk(;=g zTey|mxScz=le@T^d$^bTxSt1jkcW7fM|hOSc$_DAlBal@XLy$9c%JFJz>B=Z%e=y? zyvFOi!JE9r+q}cOyvO@|z=wRq$9%%4e8%T|!Iyl+*L=gbe8*(n90ifAo1+|(b#s(M zvTlxY$P|&Ro8^$1Y0Sc`%*O1@!JN#++|0wg%*XsJz=ABq!Ysm~EXLw2!ICV+(k#QW zEXVS!z>2KI7%Q_1tFjuavj%Ij7HhK(>#`o}vjH2j5gW4!o3a_3vjtnS6)0*Ks{Ja3eQy zGq-Rnw{bgna3^!V%Px*|``GPO`im&;GZ~2bNx;Y9WSvN;HB?WG&Wa9oA(%)@K7YWFt0a6Eau{Zm$FZ;1S2XG(72otoWfJjBC1!lOLK<2=EWJjK&I!?Qfc^GxRjUgRZS z<`rJ$HD2cp-sCOb<{jSUJ>KU7KI9`l<`X{UGd|}FzT_*u<{Q4{J0|PqD2Qa;9OaO# zo1+|(b#s(Mrif(SEQicYV-{v*HfCoI=43ABW*+8cKIUfu7Gxn7W)T);F&1YDmSicG zW*L@cIhJPyR%9i{SeaE=mDO0CHCU6iSetcNm-Sem4cL&4*qBY&l+Djng@UGdYX1IfrvOkMp^J3%Q7kxr9r(jLW%#E4hlRxrS@Gj_bLB z8@Y*_xrJM~joZ0{JGqOyxrckXkNbIm2YHBxd4xxKjK_I`CwYped4^|sj^~-q3%tln zyv!@S%4@vN8@$O|yv;kj%X_@f2Ykp!e9R|&%4dAe7ktTAe9bp}%XduH%~24^x;e@r zSvN;HB*jHnZ_*4%52Qe9L&jF%*{N^%Y4kw0xZZvEX*P-%3>_e5-iD5 zEX^`3%W^Ew3arRVjIlDSuqvyuI%}{dYq2)#urBMdJ{zzh8?iB)uqm6dIa{zLTd_6U zur1rMJv*=?#_sIFp6tcm?8Cn7$Nn6^fgHra9KxX-#^D^nksQU*9K*33 z$MKxN1SfJ5Cvys?avG;|24`{>XLAncavtY%0T*%+7jp@hav7I%1y^zvS91;5avj%m z12=LLH**WOavQgE2X}H8cXJQ-av%5e01xsI5Az6*@)(cv1W)o5PxB1V@*K}IofmkK zmw1_1c$L?9oi})sw|JX(c$fEhpAYzukNB8R_>|B1oGojI73xtN=In3wsOp9NTug;fCD**gE@plIgG!UvoI^OF*|cGCv!13^Dr;-F+U5iAPccDi?Aq*u{cYx zBulY0%djlVu{bWw6FajDyRsX*vj=;!7kjf0`?4SVa{vc&5C?MzhjJK)a|B0n6i0Im z$8sFUa{?2b$Vr^cDV)k_oX#1X$yuDuIh@ORoX-VZ$VFVtC0xp7T+S6-$yHpM$W7eLE!@g&+|C``$z9ydJ>1KE+|L6%$U{8LBRtAuJkAq5$x}SdGd#<4JkNAq z;6+~IWnSS`UgLG%;7#7*ZQkKs-s62f;6py*V?Nh8VP1%gi*@7+E zimlm(ZP||P*?}DyXD4=M7j|Vgc4rUvWH0t+ANFNG_U8Z&?yQj^_j>IFXY$nNv8G(>R?oIFqwDn{zmq^EjUixR8sum`k{n%eb5?xRR^5nrpb0 z>$sj9xRINH=Xjp! zyugdR#LK+GtGveRyuq8i#oN5YyS&Hye87i%#K(NXr+miee8HD|#n*hpw|vKB-5dpx ztec}8l67;GL$Yp;a>x{stefSKnQ6?ztjxyj%)y+@#oWxpyv)b^EWm;+#KJ7XqAbSZ zEWwg2#nLRpvMk5)tiXz_#272H3ahdjtFs1cvKDKz4(qZW>$3qHvJo4z37fJRo3jO5 zvK3pi4coFE+p_~ZGR{uy%r5N8ZtTt;?8#p2%|7hQe(cWy9LPZ&%pn}gVI0m89LZ4} z%`qIyaU9PHOmHG6aWbcHDyMNeXK*HGaW?00F6VJR7jPjLaWR*0DVK3MS8yd)aW&U) zE!S~9H*h02aWl7YE4OhwcW@_naX0sHFZXdj5AYxl@i33@D39?tPw*s9@ifoyEYI;g z(|Lgxd5M>Kg;#lv*Lj0Cd5gDshj)38_xXSi`G}ACgira5&-sEc`HHXkhHv?f$+|fT zB3U;_IV9`mD2HU-9OaNHB3U=fAv4pMg;|-6*_nemnTxrZhk2Qg`B{JkS%`&Mghg45 z#aV(SS&F4uhGkif8n5#PZ}Jvz^A7Lw9`Ex3AMz0&^9i5w8K3h7U-A`S^9|qf9g}r) z6hyLaj&exW%~1}?x;e@rQ$(_EmP2NyF$=RY8?!S9b21lmGY|7JAM>*S3$hRkvj~f_ z7>lz6OR^M8vkc3!9Luu;E3y(}tjsE`%4)368m!4$tj#*C%X+NO25iViY|JKX%4TfN z7Hr8@Y|S=o%XVzf4(!M{JFzpnuq(TCi2XQcma43gyI7e_K zM{zXAa4g4hJSQ;0iJZjAoWiM`#_62FnViMhoWr@C$N5~qgiSA5Mke9L!C z*3D56$+|hpAz3#^IV9`mD2GfD$+}q%nVH5c%*t%c&K%6iT+Gcp%*%Yt&jKvSLM+T8 zEXram&JrxiQY_6fEX#5%&kC%_N{q2GtFS7ou{vw8CTp=a>##2Cu|6BHAsewVo3JUH zu{m3?C0nsI+psO$u{}GmBjfDE&g{aj?8ffw!Jh2J-t5D^?8p8bz=0gZ!5qS&9LC`s z!I2!r(Hz6E9LMpTzyv395+`#Cr*ayna|UN}7H4w~=W-tBa{(7}5f^g_mvR}Ga|Ks& z6<2c&*K!@#a|1VW6E|}Uw{jb|a|d^F7k6_H_i`Wi^8gR>5D)VRkMbCg^8`=w6i@RE z&+;74Go2TBk(YRxS9q1zc%3(Rlec)AcX*fgc%KjWkdOG7PxzG2_?$2JlCSuhZ}^t) zn5>(lAd+=+ltZ#^j&exW%~1}SB9e8p95OSFS(ugCn4LM8lew6id6<{^n4bk$kcC*7 zMOc)@SezwTlBHOhWmuNwSe_MFk(C%@WmaKTR%3P6U`^IyZPsC3)?V$^ zHe++PU`w`QYqnuqwqtvCU`NK;iJjSnUD=J@*@HdVi@n*0ec6xwIe-H>h=VzVLphAY zIf5fOilaG(V>yoFIe`gIZs!i}!9`5Bn?&kp>@Fs8ZHt+B*@9{n#@F5@ZF`w`$pYb_g@FidIHQ(?p z-!WM?M?oa(<|v0`-5lkRtec}8GDRfoW;tYL8nZAfvoSk!Feh^{H}fzr^D#dQupkSu zFpID#i?KLMup~>dG|R9o%dtEwup%ol#>%Y1s;tK9tihVB#oDaHx~#|gY`}(W#KvsG zrfkOMY{8an#nx=Ywrt1t?7)tUvlBbB3%jx#yR!#-vKM=^5Bsto`*Q#Xau5e|2#0bQ zhjRo+aui2%499XD$8!P`oXAO>%qg78X`Id(oXJ_7%{iRQd7RG$T*yUS%q3jPWn9h` zT**~j%{5%hbzIL4+{jJb%q`r?ZQRZs+{sl%p*L?V?53iJjqi$ z%`-g9b3D&dpRbJzD-r!B%;%(mHUEbq;KHx(>;$uGHQ$FK!zTiu~;%mO) zTfSqmZjORT*3D54$+|hpAz3#^Ib@1R*3ELr%rs_UR%T;%=3q|dVs7SPUgl$d7GOaZ zVqq3xQ5IuymS9PiVriCPS(amYR$xU|VvLnpg;iON)meizS&OwAb*;yu{1A!mGT->%766yv5tR!@Io4`+UHMe8k6m!l!)3=X}AJe8ty% z!?%3LWZfJEk*u4e9Flc<Z#^j&jHpk*u5LkeO-B!mP~3?99QO%*EWy!@SJL{4BtN zEX2Yr!lEq3;w-_EEXC3+!?G;L@~ps$ti%{AvkI%S8mqGgYqAz=vkvRB9_zCK8?q4_ zvk9BB8Jn{OTe1~fvklv_9ow@5J2K8r?949g%5Ln=9_-0p?9D#x%YN+70UXFd9Lymc z%3&PN5gf@;9L+Ht%W)jf2~2PzCvh^Ta4M&9I%jYuXK^;?a4zR@J{NEy7jZF{a4DB@ zIahEcS8+Aha4pwyJvVS8H*qt!a4WZQJ9ls=cX2oOa4+|9KM(LA5AiUM@F)!-$ju|EfJAO~?U zhj1u|aX3eCBu8;H$8apiaXcq5!HJy2$(+KeoW|*#!I_-J*_^|G!IfOa)m+21T*vj?z>VC*&D_GR+{W$P!JXX2-Q2^y+{gVqz=J%*!#u*HJjUZZ z!IM12(>%koJje4)=LKHmC0^zgUgb4j=MCQEE#BrG-sL^s=L0_EBR=L6KIJn$=L^2% zE57C%zU4b6>*gqkWZfL)kgS`d9Flc<ZS7WZf)>%uHhzW@R>JXAb6MF6L$)=4C$S zX8{&uAr@v47G*IOX9<>MDVAm#mSs7XX9ZSdCB|5pRalkPSe-RkleJizby%16Sf35p zkd4@wP1uyp*qklclC9X9ZP=FW*q$BOk#TlnXLey%c4K$;U{Cg9Z}wqd_G5nz;6M)I zU=HC>4&!i+;7E?*XpZ4nj^lVvV1g4liIX{nQ#p;(IfFAfi?cb0b2*Rmxqu6~h>N*| zOSz28xq>UXimSPXYq^f=xq%zGiJQ5FTe*$fxq~~oi@Ujpd%2JMd4LCbh=+NEM|q6L zd4eZ-il=#oXL*k2na&Hm$Vb5JG{$#yw3-G$VYt4Cw$6he9jkq z$ya>MH+;)?OxDd&5XrhZ${|@dM>!Mm%+4Il$z06MJj}~{ z%+CTW$U-d4A}q>cEY1=v$x#;r? zupt|U62#@j@ zkMjgi@)S?=4A1f$&oiADc#)TQnOAs~*La;bc$2qyn|FAZ_jsQV_>hnIm{0hW&-k1# z_>!;qns4})@0hHcqac!XbCg4}ZjN$D*3D54nIe*Pvm7!rjaitL*_fROmghGRL7<2iu|PUIv`<`holG*0IX&g3l4<{ZxDJkI9=F61IE z<`ORDGA`!|uH-7N<{GZ$I<{6&lIi6=aFYqES@iMRQDzEW6Z}28>@iy=9F7NR^AMha`@iCw9DWCB< zU+^Vg@ipJ@E#EO&H%CDv>*gqjWZfL)kgS`d95O{D>t;D*W*W0FE3+{>b1)}!F*oxt zFY_@!3$P#yu`r9UD2uT;ORywMu{6uDEX%PxE3hIfF~-WQ!m6ys>a4+1Y{k}W!?tY4_UyopjI$FvvkSYj8@sayd$JdMvk&{SANz9v z2XYVxa|nlW7>9ENM{*QLa}39F9LIA46P(CNoXjbl%4wX=8Jx*koXt6$%Xys71zgBQ zT+Ah0%4J;6613bt>Jj^3J z%40mv6FkXNJk2va%X2)>bY9>^UgBk5;Z84j-r{ZE;a%S2eLmnrKH_6O;Zr{2 zbH3n9zT#`X;ak3AvTlxoNY>3!4#~PX${|@dM>%AQNY>4A$jmfmVOC~icIIGC=3;K< zVP58AeimRs7Ghx*VNn)iah707mSSm^VOf@Ac~)RWR$`2mS%pkyCwJJBN{>krO{`rr;P5tv< z|M9EernXQ0|M%zfpR_hM`G?<3`^5O4fBNkVQ}SkeFT=FrML(Tk;yWLIBg51U_fs=W z@6s`xPoH)4a~Z#5_g;P_9RKV`?`N3!#^BFn82?Jvl<@m2Pd*h~+%y=!GxtU2PtM2w zF*aP6*fS#-Yj-`IkJXqM?n@c+hYvFDOB5#_67XIvNi)z;wu``oFE-ez1MFM1+scdpXmy{S)* zhU*d~JA~)P=l?Eh=Yh%hGpSy%*7VHhGujvLRV7>>Te2X^eagP@o|HokzmRc0rBb=5z5Vw^ z?fZG@sNHwlKg)Q}hRau?_7*B0j>o=w>&qGU#ov7D{R~r1GGp+f6AT7(YT)6 z^3{y<@hd~ab5g3+iQc=dTDUG&w$PU{&c}NGBIF+b*7|T=;%h0tXjeje(m(&p z3mNT6k9{kQb9%>OQEoq28qL37{XUFSdZulmzo~!vY&34w8=eZ|_4mC|zUi}~`L(e| zXlLxFXP#!9Px)YeXh%w>W#N4K59VA7$5Y>ra{YFvX#PH`8qTLJsQWPEeEc^@Lwh#t zDj(Kc{86pD8OKw{d@&lgrBB28*f+X{exRb)&h!^P;-jn*5*Wr2bb48==X%uVH{#@Uq$Qf#)a_y#Kj(A-KKngdvw0ff#`k3t=qpx?MR7li`MNVY zTl%MtugSPRvF5vBKE%&o2-l@9{7>jtqQ>c68RyeC{4C@W%kt`S#_tXH%Y^xn@~B3b z*Qss4zAd~r-sOMOGR{A09CA%*IV3zM<@%Sx`{HfahV_y1@$T>)Nolb*T5tK6gz?<4 ztz&w4-ALs-X&DNjRA>22d-y{6wB`+mlAV$=T`a!$|xFk0WQ zhKG5Z_{#2(Q+nGq;d$wQIuKnyctQ9bJ5n%Ouif4Y;~8)J@qvu?#|v+{68bUjyTSDQ zyQ1}4V@ouC#ny#+ke+vInD>cwHKO@3xl8yiq@L^+a!aJm2>nkP_RVO2$n@th-{Zgk zMW~C?zW$eJ{ST@b#yz!kwBN`2d_S}&7Aq62-(URaWXSnD|BmKk!_J`}iCRAi^C<1k zXQKCX`9kzv{a1rff2B@(Kg^%BsRzS2r~h(6w630i5bZOutYKaxKI$IL>+_?-x{PmK z7M`E-L6l$Y+#jc9yg&8Pi_pLSt6SoWeh}^FU-%#x|MmOP{C)3wFrIB<^d0)y&%(a) zf8Y7om48GzCH@iSXG-oRA;+|hwL?E*2l_|`zXYYjT;zvJ@^1j#nVrcgdj)do@e)N+tkJ7(8H~J1w?ia0#;v+&kQ@vd--LCM znE$iL{KdlfrWDx{o|o2PeCS`|Y}K${QhxX4u+Cz+x`cHRoBo?0rDcrg7q*1`A~E7? z=feE@>H6rt-&_y(#W(!@TE_8s_QoOC^q>DQv_EY^)o7d!|12C&{nYrV|Ia>-+F#_? z(faFt$okt7&D%qdqVf1^(P-Y6x%Rb;cBHMT8}^ad%9Lmwj>;X@XS~_Okaw*2&EpyE zNXa`P^e-jlZnVF2svGi-|F&q@M^k^aH?%Lk-n=uR-9H!`u8V#DS+wtzTo7`KckL4G zr!A+3@r&&*AKH^Lqg2Qtk>hT(|9$I&=sBf&MfKL-I*0j_I-_aW7vhI^NBzF``OwaI zo>-XoDSZ}3>+DmXIg-(i)I!5Te-cafhdfjNa6Igj8{Uo&Ii`;J?VixyU!+_O{rpp< zFyB&6PK(yhtd?Qi<3HLIa!4QXKOwJpk!)MSec$LB^=DqyFz-^1?+Wcs`Nqig;r%}w z7III?H!IpdJM}-BaXeP@Kg)t^s)csMzf?Z-C%yXC!x_gDvpcTKaKpnN917Rf&mPuG ztbYAyKYlYXdT*_0KTZ8-;cz^5t>f5?>r*Qq4*N#z!9Sww8~rF6-`Joq?y)LkLS35j zJXQi{|AY_eA^YneT>vZy28z?Mvs6gyV^`!=n9q%Yb0|%repb^4Y(K{VHYH z%4l6)j78tO4}TKo?S?{s4bM-_ZlCUxBjlVq`*YF0dgMR*L;Kbx!uY4Cr_=vuahQj( zR;$CjjJ60NJ(wZnHHcBxBL zub#*j^4{=4g=n4iiii20-r#Rxd}61Dggj$4uZHnYEZrL2*Kp?6jQ6IVY%(Lm#BZ{P zd6ZaNH|&GyKYtw7S!&z%Vco?h#KL@!ogNqZpL+K3`)QxtmvZ!PQM>A_2=gTM8|_0* ziFtFvb?H4W9}Lg`)W|j&rdZD@g$sT??USF0Z~rA)m-D0XNvXN;hZ)yx_}Yxg;ri6~ zLc3G`cqH0y9`xxFp7+TL3 zCKl$6&d>N|v@d-sSJ-FLW`1u&#`9y@o`w7p4PS*kQXUSE-dCVbv@X9CkNUkSd)VjF zKXo&_H?_>zZ)Nl=W$nFap1$};^!==RC%ivx_w2C0q-TzY{-pi$qj24ZtT)2%_y@z+ zWV}B;$C5CQV!xdm+Ld_qFpO_%mC4b5y)mlC;@|(}xp4h=c87MPRC^cR7ym+;Xul{o zJz76`ejI(zo)-xFV|qt_AN@(wF#lpz_K(SUPU_>_p&hA@8%FPa_palAKRF(&_#mve z)Eni(`IO?XqJL+Z+GlOZ>q5scE-6b|emy*=X3=PUH2y`{?-Qw?`F`3b&q?`cMrcoL z$NF$SWpcaF{@73IhkY=m|AVNXt!sr`;ys#&+)}=rDXiO+{q>`HdNUeYpV!?>oF>N+pu{?vmbqWvuIS3({sOSgu3m)dh~^qqP0 zk1&4|pPjHNyl-5YX#RcgQph2t@pr<0l2RmdH2!}W7WSRgE48Eiv&;(Pk~+9|*bmaD zjthCG6dM-Jlh1z`?GvS=zt1PW`zo{}ajbGQpQoM*`$wYn7sLLVxLz;pXX#7oMcAxNm z)_;7#cQ$1Fj!(QE>W!3*)uR38J2Rto*|*!l@cd5?3E#_<+}{iROTU>om2o_EWAy!q zFTEDc-{s%g9iDfr<@s>F*YDm>`{cTm&we~9{4Rea>SzCT(Y(v~ING=KT?_kd%E$kX z_KWx*L+-ILABTKmFZPG_rqsV0@=X2u`e;7hC>g$|@ofj9_Wh!Lcwg%0{u9lssU@Rv z?Dvh3OZ=C$qy22fxhT&eRiire?0=$l(I@(MvDn7bp&zkKN1}X6M9)phvOBEv#Jx<> zJo;tsuz#nn{A}n){OOmXee=gJqWM3lLo{!)ek1fZHfBroUCIC9(TqG2c|RNWt&{=x zqw}r48^$s5yJw-DY5)CCw9eMl4)2YR&JxzyhR)r>c*YM`iSnqiB&>(nxiO*NiK{z9 ze`9lVgz-=LYt)|fsbxaD(+52c_oq+m5$0p;RKYN=Dfe@S`_j*UI%;>kdNfalpE#Y- z-juvW!+MGRxkfl1>l%F*5}$7z=0jrVKcji@+4n>4@o~?;llIAN@x0fTgz;~`D0=_= zVIjBp!+lY`QQ+&Lf2reoMB~2nN!UN*jjycAxG#N9>c#N>*~4;%-|sz(*84{}LmiNK z^uz7p`TtoNo|ib)@)sGuW3@jW+OwhK>}X%P{xakdo027(cM}rfd|J&vg&bq^8%O)z zwm(F5aC(h!K6UEnLLO;nnnk()`b+PpWjz1qBSRkP)8~ICy#E(FLcbF~8yChi{nI7F zxTO4LO!(gYUp<%JX;*aq%+b(}l*b!FKjYi}7VU3Sw@nG{AGS1{kA3@V`7(Z||8`?E z?@zCa*5zj^MeYA(!K&fFhJ9RDW zlPMdYMtOYtboRgvtNYgCjNLS znkStWhIyJO*(l7D#Np3GIi_C@?Mk0MYhK3pFaDV>Vcw;`+Y+tETD6X4yg%``&qd?0 zE&49T3oHozip}|2bbq&9(LO(7U#pDkQ+n)++Lx(ww2$q0IymEadTP$F-^AWN4A-TO z=pFXOlxgpUeKoa2iRR%u+~%8gLcf0y{r{fS$-ScYe>g6DU*eOxgx~R<@1G0LjkiCL zVPa>i@V!fUTsQ0^v92l6`2#z`^(haVhJ7U7Z*KIR`ts?JL!v<5*BST6KWrM!r-%2W zzwb@C7Uo&(&aXnQ@quf?zP90DqiEa?T#e5E@SEYj*y8)qzW1AYVLqk)_AgVI2=_9vR%3V&~jo!JxK zn>znW*tb%@+9<5k)WI{SW;{Q&X7lL%tvZJ1#2Y+~=HI+4YctNr1~ojJ;fCA43wgyR z4h!=q@m|enzq>Ufyf6J9WuyKz9~T|};B@%^CaK@q6y{0lsOayNv7gTf=VP4)oea;P zo;6xu3#P6P?Kzb{^f!I>h0w0_q2CJaiMRMLj87~}v*j7rC615WkzvZS@?qY^_WU>+ zrz_Q?eeT=QzgHw4tqsqM-}qM8-xCMBh5pBfmYg2${~%NNPNdcT#rtWWJTARQ@la2s ze6Q`{@Sb6_qVH6pN?|>vZr>g9NG)ICe8&0ozby>om0H91J1wd!6ZN)+_NATqXUHw} zTNR@9^x5cpocQ0jq5g|i%@>{*`|~#!XFM-;?ZBuWzy7%}o+*p6M*HH(w2*H~q3?uT zV!eulb-JNj-S^Wzxi8WEyJ23(ru}7p#&hD^@`d>t`~3%N!gE(gxg;k2Z_RN2qpM*) z#j3stoaBbh3NO^>O|+KJ`U|n zt@pL4oj0FF`(6DXgnq}5&W+|-x!Pe~B|i0)@cy(H-;e6@1vA5XNaVZrlZ^hPEM6P! zYrpAnB%{5FJL94pH!qIX!QjvTEaSQj&o(Z~IG>)Qdzgo@YUf&o{?4uz+MoDyl`!5Z zEeb~a&x18p!t*}YI(#Qma^(v1Ctm1g7|+CsqT%0TQ>&$hc^*GmEgIhr(f=ok-yIkF zoBF+J`!k*spI$0_$5Q^eH0+7M()0Z{%*XhO zkE3<`o2#M!v7U9J^;F?YTf_5rTnO_z^wkSLN^hCw7b7&hEyp zYxl4!g-)(R!_HE!F|Q+n2mtp7vGW*`z0PwT1kuu#wP1 zl062UblffZVb2Z(x5_tS8V5vYGEzqwiH{VvMJH-v{}846T1DpML0% zym$!h@KL9MKW}{!beAt{Kt6a#2GEW1m zum?(xAifGOPvcg!1f5@)Tm}5CXJ)`S=%Oz%j?6s+_K$U~gi!pddLQ)fdLMCJJr2fI zylM@3mMAD@4NKiIkB=!Z<$26W|v z<2?dCX?$P@g{unsuda;(esax!XotCJT|n-e2R@W60`OHp1<=`DF8^S(|EU-7_v~K~ z+~L3PAYGl`VM1BQ_om@5*P<5C!^+hH{3Hh4#JDK0RG=$ARTTDsr^`Y1qw}WfroMW0 z6#S(7(!R2o;ZOCmwM2hds{`Om`CX74ojU|MlfhXapFG`?#lWNCQQ)aI{625uVoFQm zw*qJ3XYlfQA*ZZUEbj8;^Ed^GPdxvdPvrRse~P!eigxwpLel{^`v&q84&**!;<|WI z&{h780KfC<$4i@Zc62H9pKf1+^kMO6^wa%iR=COM`HCQvo9;q?@tkMTo*Z2faHM}J z@T<5r1NxOOOai*8#tR}*zd#kxlVyHI_Pou3<0f6kZScW_D!uPr6N(~x$$sRI4|>Vf zdqEdIY7*uFt_nVr0WZJ@?de+UVaK#QN&GmyDfF`%+XUs+APxGtrWL_B@u*~^AAFji zpL*9PDz|17#zmbg0z6cXNr>~psX=G?{S)wH-3ox;`TIU01`qc45y{oy*5GHcU^T{% zjTuIKk?gIP%Tr8lD@OUNUhYP{JDEP2P@anhd~r7k_*w>T`VZ-T=fSV~Dy@slrjyVf zFaK_U$>%xafu5qSuQvSazP6O;xcK{ilds!fN4`$~81lhWEpnN3RX9GzMVF|E_IaXO zsHdB*`2x7z{)1hR%Y87uVt^On{e2_kiXWo&c~NN%^dcYkaF?ma*e%j0b@!9Kd2*TL zZS_vFcdJWbT=czsXh-i(PW*E21L&!OQi2}@OGxYWz4qKRaAl?67!Mxr3iJl=Qxo;w z#RgD+oBJ-teBj|S*im(%B=j3Uyc>E_lFI-`tm#GL+%OpU zsxuOFVLgM<58a;T&8i{kJ9c3<_(@DI3;d;>ugO)xXjhe34LazJWbZ`3HDgSBqA}TB z@jd|Kt+ocPFnGy|@u9c$r9|+D^q%XWhsa6iuz0G}<4t{?CqDSY-D@f2O^qOY)TZVe zO+HHzfc`VP?xQZOxsHAYccAeL@c9Ec+1$jxgV&(E=s6<7q{}tkF}`}q=MgCPnuCAb z?~bEg_N5-_p^lNncQ-iT@|p=SKCHx^c7W6P6!ZzfC9_#>F`5 znm)?Jm7x7;S+;H<;@{JOpY($7;9Ie#3iOA%7z4dvK;gTbfbdC}=m_AyiVJy`0m*0_ zgPu_TuG78++ZG%BR+s%?x5fOUH2zEL055j37UF!yk*&bwJ676ML37?KG&_&m~j(Q^981S>+kRSF#E?Groe2$AFH!Mp~+WpdI`dz2G>? zs|t5v55z(@^q1IGg~n_40rX2`-3NR{o8$1q_~65#1}+P;>jo#A9z{D9S1&Z7Xps-? zs6%g$nYb&^X~eL9l(BG?~v7rD2AFLIW7D@ zgqs7;7`?%dZG_!oBL^bXH`Y=+qiFqz-=Td0k?p3J%Tr!7@uGFT)Z=LVsAhWVXXt_M zz^4+e_vxu^fxozQ40P9XmZQGv@r3w(G0C^;{0MxhydRQW`<6vN^s<%U3)ZU{_=&T% zJq>*EHWt~3TW!vwABir2Z*;-`NdEo{#=3!A-?6^QXTjUhF6(`O^o?(Ssy{p9Ti~1Q zB(2}%S=*>MdkS5H(fgrc<_~%As0Hw zKER*r(6*6&FVsifI4emZ9` z_+9;uaT~s1jRG)_U|V-0UEXX@dMRhm0l;H_=PST>&sy-6`1BP0Vx==(FzGDl*k%*D z)7yRihg~93{`$B(Ce%N+KpynjiqJ!%>pjvZWmeur{g+m6wW9g3ZdJb>((8QLX+qgN zEzxn zjE~qo_c-9L)1-eMHO08N^DJosI1_%OJvHTn?Crr9H`WO9DTLJ?yP9gn2AU*N* zrzTY6hGINq)Bm7P#NYjp3!ZH0Jmjwl1^?3Z*c7y*3$FkktPQO*>D*IE zpZT|f9#tbr@3JkeF>ZQm3F5B`578f;yUK9W4m&y@`k47fq5rJZ1IRg>x}W%n{|9=o zp^@M}7MU3Qp_>*aI^E7k^*&VvAG+5R$_@P2e?izlWgpJ%g{v^ zZ%yO<={4j@)d{6~{&qePyFKZr=GW#MxZ(!wOR2JtsQ+6RfIhP42;e6x{{TGx`3K>3 zeh2uBC&&ms7w1j_Pnjt#`ll1p`D+&2jd~(i1JrZHN{n)bMCKFx0m0+(vYN<+abEU*b$4;D|;cz+ZS>%!_&%j*?tVT8Q!GjlD?! zr5#A+^HoN=URn-(!F+uugRfp`nlHtFlLhn{+=0e_k{V#*uH%(}FOSm>euz9e3wB#P zEWHWscC`M@=D<@1o*cQ7`agXR=)=B^gq(==e~+5-{N!`ugMb|9m-1-_xmF{_0zVe+ z<>m5}6HDnl8t*U+IIl$XrI!cqsX6^_&weh&`-MUeoN|oGfa6g zcjYe=vf$p}C-z|g?2cNucpmbnucvW|b06*LH~z4zqQYdrmCI=TMP_;le}E<47!U9& z4Ip}DegJx~4qG5EGWtB^MJ%NCbKP$NmZ=)a$y$ z=gob08@j5sCoxX)Y8vWiokYl2W$e08{Su%f4ZStTD&`FMPqoS33HfU;V;-w}#rJY~@MB5KjY0b4 z2GIyjn&;!6^8sJ+IsSW-uJgCXdIxj8|AcY@?}_dQGhH-s^{NT%Ay4)YdQN`Y34Y<5 z_R~5+jDHx)wfY8os*V(f+{-t4h#p^`lK#;*U|-a9D@T*%FtqOqMY^6l0`0O)qrev` z^u&A6nHM?_y!OADfbx4vfDW!H4<4DgIAQnOI^Ll1O}}=RNmt36px+{MFvefxItctd zS@=Ym5R%VVPY^O!0G$`}jz|0d8_3^aISZ2AcsRSdfy-lJmKwZtBJZ1M=T6C=Ce(Ec z(K)l4NrI8?_YHg~3ooB%;xcJ$z+r3==*hB{03D_Oani>PS`nX~sZ4sfQ)~DwtQhg3 z7)k4S;_?d2i(S3wyp3!-2mH^U#|GYP(o$vG<342r(C)!XXiuE44tnq>!$tzXp}xNn zURgqV=++eA%Lngg*A*sNv5|C=tN9Z_l% z_}6{S-^=C6)E;Y3mt0wT$E0Q3t^3ZyzKC{f~ zi~bkT?gYC&8#E4a)%FtXqQ2V=^mhd;M?3D`xi*;cqVzb(r)rg87ust)5`4nHo`wF@ z8~bA%`N@+MMw0&{3;Us*O4A2=T90WBy~2x6huz>`1~mixwVR2LQ@N>~%67hcsU7Go z8~p}ceK}1i@PGE~mmoR?fS#jt+_mSseDEF-t#zjv`d)mb1(asnz_MrAy z6X*WrK!08#73jr|F2OiC`{**qTF}S+>mlIDP`mEh?GN>P$z%9Q>{BW56Pq%24BEXy zewPS|`D)~q&uxHlmkr{fKfLKaFPBMAMEcuxCk4irg%sUv>Z`CxvrMS74kJ1bK7(;# zv5tXG|L|L%5BRIX&B3?)LNws3l4N(q?&P4isGpALI3pP!b>%^JJ&V`%=9ppRG17h7r$cJdq9drMvbr-tWmXpIk})h-&$n{FMut5z5cYz_0AcVbYsPdlG%>*GBtN z)_|O;4Xq&u>Q5t*^LK-0fF23cfZsgu%K(qs8?T#?_gId8um^3x@3KyjawtFc0r72) znKZt|Znrn-V!)H*2>m`m-bJU++fm++=wxHhfD&S{PGZ*Z#YPNvvaQu_Akw3gV`pbId zLg=2i7Ib1+8eyE-5%Nn!al&663cG3QdG5KrLAzTs(m3s0HvsSw+#-73KSuOTLh~Hq zDvok`rQQE5-j?jZtOcNh7!eG5>6+s(ZjOKEo-cP9xNPIjjcBi6Ka8LHkNBR&J%@I9 zz)9rmkWy|_PK|s+{ZDT9qpM`E0XUBp+3l{wUz@mSPV2{VfSu>nT1|LH>?L^<<&m&8BrtsTV>2xo7X5 zhy34FpcmDgf^;5rTNaG38u^0cwDLlPqC#Tesl2b!J>S{cQ zPsHNge-MA&8GItESH^gWHqFrcdyrX0^l7 zjW3@v^_@?>rgdf=6%2mWImvGprC$+!dVj*Ws&GYmB5O4GmbWPf`irQUs3*EKfgaN5 z%bx^(?c-s-!R)z2S$7}!UdGrDYqDuYZ1{b=KHj!U5;igdJ!na}VIHu7(hw zcApMAEMFG*i*~=0{g-pdKIw<|ucChJLy#Z2tl~nzsSybNlH-frq zt_olFnNXBlPJFa-2kbEWZz$SR)n*%5Rt`k} zMDS?97ctGdoANqoCD317Y< zS2gwBZHk~BAZ6gS**Ok)HI=Yex(DrdsP4-ap}l;xk0tBOMLT+Cfz2jeJqz1oLRmRK z_`*3CpzFtn+~{k4Adjxw?>?aX-Jhs0-_t!o^?~Gqy^jH%RP{R8_he&RO-8$gLq`Dq zvq{fQ$RFk=J(H~|_?Fd93HcJ|reK_8ty_?9xitpz$BWT-^KB%$7WHx(ifnY*~Bp5E#g%|KlRv!pqFbrt!L^+@i6X6MS-4ds~AgydLRILHnUR_ypPUPN|m|ypmIo*)(m8Xs(U&$`eqpZ|L&{20Kz3%Y& zKRL@x>*c!m>W`*>EEml~RF43R7e5;2HtDiI@w=)%>>J8uqxRUkEyTZFGLs$g^UdHg zaP;eGrW`L;8uG7?zlI#EIbX=W+*<{@>HdE3Lq z05|z=vU6Tt!GCg22jC$iXFyKmR(n3aS3;zV{T<2AI85t0I@M;3pJ>>AnStl}Is^T7 z*P(NvY-4aewEJiR;PLmG+N(wDY%(%)XOqvSS0TBXe+c$oN9T%M) zo~kk3XA+rvL!ZdXX-gR?q1d$im#Hs4W{fnU9GnmIQlk=jxlH=h%ot~}eK+XE zuMdGdX#Zz`$IlllfqHevX9Ju%@}LQIY1+TwX%B2iyxsfjCgfei_nA=Mn7jk=lq)d4 zGG7GIXCkdGkMf=9{TN>95|!Wojr7jME0A~gg7(40l%!-YXOMiV zrWGKU%%wvO99NpZq(^cEpndTr5uI~O+5a!_Yh(4{#-u(dA6%IHpi5PLo4BYM8~W9C ziQbP;RZ?Tz)%N7TM--#|dY<|v=*p-5kfxqT-&_K`fbttmC{vyj>(}qJ*ZrsO@_jAGU6Xwxpg*ah`ujNLE3{uz9t z?L5lm>)RFO5?2IWoOK`We`KS93Xyc~}D{QVE;BRxL?vc)wrzodPDCRt!W2IaGF z4>qBg&;fMgewHunJ$O-P1lnU+XS9B;)UzNpL3 zn{qruU-B#7j|1IQ!WH0Ok%#W@vz8AqK5DP^!@5~{Irz)6X&JtN;C{Ti-K4ws?|?mH=jh%M zyR<&sq{}w6Ue9XXgnz3V(tJk*?|>av4YPsXq6z6my+3s`Q%|%Wm&}BGb7|-w<`-)- z;PuY{x`-g!U*XptLav!N`7!KxDeylNzkwHjd|ozE06 zZCJ?qkMcapdAWEudhqrWKzF`B^HBqr7wq;N@S75!IqM@T0j=|xE}Bnd z-!spYbI-2_J*_^{dL!R>aG^<8b$1L${j^a9<69DdkJy1Y#ODRV764wiRP_;#Zdlla($D(c`_F-Yo%>!on8r(#PfO!u z_n+A8I?($p_Eg}ZYFIyEm)8yhPqnT~dbEY5Ur(}!ZoeGR8~hL5yCLj7bgzWK%%E|lJnowPy*%0x$lfjp)bbr!snL2Dm`4x0NLM5_(>4mevi&wo4 zcpiTM_ZQs|keKY!t_CKR=Z4zeGu8?vi3e(LYFQ>gcHz)2Ibc@pc} zdLr$ci=0Ium#SR};HM^$oQOY79+~oTMgz!^wEHJ&{0WSYyg#)V+Rc&)`D|J{=o>MB z?lZAOONl=QTe%(nb(|^h@@Y$UCt0FM6W4nq`kIhm`bGWjNB2fV9P$f9Er0NVR7Z#p z{_KkjI1^if&Z?)iOB?Nb8L3VKKOMK(Yg10tnhpJ_d+dii=!|P2XPzv4yjpllQ%)u+ zFHOjL#HD%SlX>Ld#3uhx?KnHi%c4{E%n$qF67q7ftsy`8T^?`?wG2 z;@QsyUg~2C!1bK#+5>)S-Jkkl&!x*4I`7Q4^(Q^I_$Ty?Bj4=qBFMY`MDX>?*9Chabd;*?B;h?C=BFxY##Qu2|}0Ce&-Q^)sPJN_JNl z@gqL0MeC8WS2y@!Jo#T8Q{#XR| zOO#oU@ZY+{NUwK(P5t$?blce){p2~14Kn3qk?R`~hL@-Q z_3cXfFyR7{$LaQ*o?Sl`i)j3G(RMD>>--(?Me;qg&U&Rb>Zz>X0bg}Hk9mW7M0%9T ziL*^T=35$k=hy-F?;q6xFB|c%DPlg}ItceYSc%o>uS!1~e8WnSK6EAj{uuQV{s!Ok zf!7eqt8+mQ(QP;6kk_a=-jw6%x`98~Ad-9L*Msy=&@RxyZQr#JlLPky&%D!Nmsp=K z7+2ZM4LOqsA6-WNCm+%;4SIl&ScB#Hklwu_%ISRy?WJ76L)WW zPIj=9J%>3s0qCpiCP6!@c3;>Rk&@0Oh<)Qp?{u%U(A4MWX#I~Jc;3*&^^Mb@r}};f z<1W*s?SXbHYz7_q@Kq%56CR^~I@y*zs5gk_39`LkClhCN%Y#qZHoK1B%=Zw+bFbZ} z_qv5~Hbu>+ubhTFNbecxO?_UgGTFD$)j$v4nfA|E(aX~SKbC!G?wCF2c!%yI1DJtl z*PVIAj?kxaYsU~%Pp%HcJxAgA-S4N&H|hHN*X<@0H%Ly^8`^(U{X#)!HNP+L<-hn& zQ;wHw{ucF~#v{G5p5AxjUo##z>9SfQ;6|s68M`BoU_Z+V@sca z?&9ZV$g`Nye<0GA*!$GY=-i~fvP+VqbP77RquyNwU0HLo z!@O+IaMYXg>V+9+{^1z-#q%#dHI?j$3?+Rnf=^+5yQ#0j+C%>M zT`!aq*~pHGYSy0ru0r&wkpX&2jolAAu#PmZQobP{(4WfWcc{^=u>NY%VXo*{G~X;1 z3H>8yE+Dyd3d|+HLtdKH*uascqaZh|U@6dzU+)J$SG}SA3RNHz^d>7w>u5aLjZm~7 z=Op-71h$xN;;vTA)E{~B5$LRY(LNyGNB2bBgYA14c^?4281w++#Ea2A z76)DT?eW%V!k11E7b^p1S=fb?DqI=`ce=EOM2fX>8+x4ytnWS6(oe%pvy z&~LI7*+bUW`;lo!g}pw7_8t*Ft_2@KH{Q4?_BHr|0<`Yf<|X0trSeZxUXG&kQ+z|w zvw%}vVqU|ZJq5k>{1j+U{m8M@0Yaf8xQo#6yc zzXtCm{DMfrA>$&Yi^=N`=^KjL;b*eQ|Y z(hHR98sz2jz;kB^2HkbzjNk{|W*_op2U;g&pTglUse=cgU)Y?ruoq%{71%j;=o#v> z$uXdZnz?h3|*_esQ=XQR;TeihG=)$xvC?hOCPNRD(Y}hTxr3j?+jk?uK^6NWT zeYnZa1MXh{U3AIE$3d6TIWUfD*H+L?Bo(j&&ib-A))Dt*jhmuLZba!1yt zb=+jME~PY`Z(}W@(2fYUeASxXNtfqy(tP_fy$_+P(tLs6=tq2ag8Hx9ksV~YN?tH{ ziphBZPZnx-8}SkG;omv_k={*y3u|um&aO7_$K>CbHHKew(=>EH_s8lS=yzS;Yp|zb6Y|v^yj-48q_OlXI|lN@74?%JTD0Ai zbKgmccG!~JTTI+_BVe5g#hT5fO(?%T%5Opz&(3?7H^98--@dg^=SC=>v}R|NGi5!W zSWW(v?(rCULl?K}4IOu9ae49`JajDk&f)JQ-lo1>XZ#rzGqHf3g(* z3okYoeiRxr?L~AUzM1p~@}kQ`!v3jZKCn9~OC`t!D?;c0*n=0)A8hm<;H~2=yJ7Hg zK1a{;nKy|}K^$@_W6}P#DtL9CDX)T-2bfUACpxG`54)JSd_eb{bsaiCqV9}hm`PG5+cxt)HlF^zMZ$VAtQP_=EpsmBmlN@5lUL z-}#U$hk;N2QlN)(&r1cJ1l?5os!dJ4^Zh%|;k*x9iOz?Lr?(yfPHTEMP=pnPzbq2i z`MI4>s6L76n{ul1wM__%m!WaXZP$G_6JM(v@9zTM4Ihk;3_lLPn`NFw?Ht@k@|5Ap zNtDY~8vW;I(n6oB12q3qfwb<=a~uNQgmaEJjLPvgO~9Y>KeA_h1=(r0_hZ_BJ3R~d z>iBejT2`_5fhry8Yue=le#}O=b~yQ6rD%}UvL9RM9G-)@kv+1#d^c zE5W{ppnaBv&Iz*hdEuw&p0r=(tRJ~g(E7VU(R`{t(ZlWg5PlWEOXm++IeX6d%0AGU z`Nu#W-F6>RCL{eKi!4Mxl>6~B(8sGmDHEzEWPjYLvku2%=$wrc*PFhPf9Gm@)W8=vF8(s1UP1e$`gn8V<3rRg&qDa?-gIt)#bK}q z^3UaCrXCwl>wnI^x+}>#;H6)TgI}ym(*1gN(C4iwFRR=PM0>|Jg1=OR)sORQfnVih zdtRp({f>;d$jLq*_z3-=VwGHuc4a2;n~ESm!)@>7^Wkf+n|$tT?clh*cTj#-3;09& zg*_Man)dbh*FMneN}Po~7Hi1v^Bdi1o|fSf`5*bJ1Q|Hug>SS8_3g_TN8TnD>=WO! z2L175;S-ye6aRIgdptVx@~5Vr>UodKJ+R}FkbJNCmrzFCV4^|FA~8 z&H=7{KhYia3HqH!*n3*Pqlmw+oGp)X89#yku3Q>)R5KQ%UD4;oag#4YuTC`~%W3s* z(MRAjS)dc_r4*;g&YkWKyRUu*08f#-G5XKzksecLX?|cpnUB|B27XahvJrnf?@5K0 zK!4bX8ARU!v<}K@T?UGjpVOKr>;`6-kj`_8&OMbulMs($$V+fCk=uv?8%dNrVqT)+ zeuX?a>u+k=8@lINIs@pXFJFNEWfyvoKT;$;_)T7)c-OR}QVt_M^pfuH=)#$ye^m_q zz7X$t5%lHH=8r-DHnqKldM`SFAJq?97nhIDz<**7E@K>3wTSzepS2~pDoy&ErXG(s zucir|b8zBfnG`0@hSGXGdsbE<-)9r?bM}{zb3VhKs|+bY@}8~RM${`v`)B&H*LK9W z$N7NzAMLv1lXbx+&c~HNfBD@b;B!8@H}s!udclo)Q_mkZpzMZg8pm%m z-%?XsQNIVb%LTZ7W^xlc{@lNL$e5DE4|ZK%Zu#)RBZoC}*j)qOw(r!*#MKQAxpK}gh`T$%$GpbV zc_v@f-_Q>I2n<|+@jqtQy?hH`JlSm;KOpP*!|TT-Ju-X?_*M0%dtZ(}t#l=fk9gIJ z+IdEPBCB~0__`)Mf}R!L$JUy5<=FV)z_arT8t2^y!6&X1?-rPJwn`(edhdZ;=;IxJ z8a)54yB)UYc8`!>&fNdIZsV$Pe-HZCGy?phHqkopSm-rm&q63FA_hsuk7XW z#MvqGdvyakAL!}de!y8)t)~f9?taRI&d2M#Cp}t-=F=i6L%-O4(pNIF8|1)QCuRY( z&L^_JMnCoHO&2>hg0w#0ag_H=%k z-=T5PTQdS*c04ob`y3_5n09oX*rQD-yU~4LX8i%VY%=^nHhf%DldrO6f_@PT^P@lf zChc$V?a|*%KI8o{&gNh93FZa=OkGcm@seVdKgtE1djfdF!ofG3(LMw#wQoJrHw7Ly z*#<_YR;PbizFmAeM5T%dn*ah(H{*fL^@*`^z zKFasYc9YL*k^XhfsJa31S<9Q7Q2b5z4q-}7cKkM-Z*u>j-zm}=vg9@Ss>2_^VcX9` z|G1*6PcZ3h{$s@5U#ekV>^gE7`0+Dq|Ds<0D6*e+J|HFC^W*mSOPu=#jz6x#FTOJM zoOK>o!#Om5Ju`hpd(L@?o7eUtZu!u2?hW$D&yjv*#an}3B2W64rkt!s?;NO;R8D)Z zhCH*nA-zq$7*F!dqGv6Lz2=vBVmo`>q2>n!RIs0%sr%4#^gDY%@1ck|H%ZSF z+6Ft!#@+y5I(Ck4%ytj>eE4zRgszi00Z(PJ``-4v0oz6Qc$hc&mD0XjAY=8w`0H~g zfuBgcZkK_>Qzk)wS+Mnw$9_it#8|)YXm1zo?y5J@dIDa0iKm37I=>I)X7uC@4o{y=<$^7q-uQzdQHa(0)NVCv7q1N>P@8g zU-W|AkxA_Pl~3utDE5)$L7c6DaS#jX9VOA9_D6Y^c)_L}5nd5+#k&h+C+zthm26!L zlh4B*5}z);2f2|GQqs7N&I~;vLhU^89j(u+vxy&>dak)kiSO3T1wZJp3*>h_`3Af_ zMey-X%b^ch7@ePxcK)NYEg^oHa|+|?lxGVw!;Y)!Mc_Bfug9L5_T*#2OEp-4_SN*F ziA?%G`5%xFcbW}E(GGh;eveK|@3XM9J0a)n<|>T0?0TKXdu)5?F~<(_o_>Q&yJ9rm zYhYPPzS*C~t4zA8?RyvP)gXS-DPBO|^0<-Q}{V#ienrUkmoa@grDD zk_R32pg-#6Yu*{~!y1$^p+2*&hY7{gN*G67^fl~`4p|So#U9Z<9M4AkXkycwAk@p- z^n(fch|lPsm}~dztbZUrT=Q~y^5u#U!1L%m^q2pc3VS0u(mYl-7>@NDzRvo=8xC&; z{5QW|n2_~&Gt`7);l!m#|4|(6%B>f`FQPP^Uvt)9+0daF7ZFJILfUmn(ezSWQ(r~d zb+$xaq>t;k(^n7OU63}_-w0xcLUeGyrcsB2~(gablUR7SAg!JYf3tg z!B&nxYRa)RA^T0p^Im~Il;>!DkX8JMe0ixC^e9W;X9D0p?;C1D`L;!*301DFgx~v) z|Dz`ze~hh50zLT8er0Pr4E3D#;u!m0hW|$B0TsLkp{Qp2pK}-dDfaU65>tUWzPIz5^wCDMK%H6o|>q;(Vdh3vb|{}uGnOFx1?*trdf zfxmO^e()XefgC~W>0*n0hkxrj8n-5KqD^`6aUSGN7NBz)?)`RMVhWwxQLo|?o;|Gn zEJygrzt>9r<)cF)9+02xfx>_IF8=oiPsUn zGw2)?|L_j_TcxG*=OQkhi_wMc{z1TLjGO3^33TLP5B|U}cueO{^kVvbE(gA|j?D*? zJ{IRU_crbG>g<#W-4kdYFFh|XyF77M=PAvA-{S71hc^YDhJKCJb*M?#VIy}SKc~gV z-iH%MwpB9e{HXOm3wDKmRaYyVLwbK&kNvlfKVhCU>8_9~3rr|Bl3&k?Sic}$IgFop zy|IAF|7W+Hdkl~1oqM_WpqIY{v^?y&3JIahDjPesI>cEHPd|7IJVi*aqpO7_<3Ls{%zBfzOZ_j35( zr6j)%=)SYFe#c+V1mDXghkly!tlTchxB1t6s_`r0x1;v_QWfG)+3_IsyZh^QFPEpB zp1f=x;XMU@znWzIo5`n1p6$M_3fX~tF85P=S8qq5p8Y)mm79J~*LjD|eYngAlh5y3 zJKlioFi%bErL3M;sL9ux2D~?+sz>Xw@`qjDaQwR~f6y=KT>^9vk)L1}RmKfS7tIRN z{KmO=xW>B{+HGLhMZd>jJ>1g-K2a&wShO2N_r7I6JJ0As_fK`*O3zI`bN1z&`)n(w z1CH8#0P-e+Ucg_FkM?8UC<2av9_o|*J*=Cw|Do(Tcr}UcZ?iJr2>)tpKzBaGt{?SF z0X-q`^avpXU`eBc5Nko+cg*ZU#6yacb#)P>ew#WIli0p zD<4kpin*q*qj`Q(_aFmbJdC=8uwpMC^nWeQTil0^gAZ8wQ_#;cTlRg(Ur6V&*f!D= zyl5oGL#J;9I%zu}5sM2%LH{}P{k|dKAMNbVHOpNd@CuLkWkQ~Z<~6*o_21su`%|ar zoVxB!zh5l|doMTTm0y0+Z}(3h1Ds#|(7v$#35z}eJ;dHx|MKl!j34`L&v#inqdMAt zRBr=#YG+6b1MlB_#Ln;3=NRl)vqyH^`Xq+l@f5|Uvyk1C9q+v}^>vcPw9mCS5a+7? z+5H{tCh-5;cMb54UQXjbygU3+b)_fxP_SW}OupD2KN{in8dFWE{iDVpyi50jRp%hQ z|07!+ApYGFc-GKePwY+j?P-B`3DgGqT@W^TJ;uxRtl=aB$N6MG+J9l+V=ke7-|xr& ze?Bij{T6n8PTd(c(d4VD)*snigZP7I7-iCBMf>|_wgH*R-2~~==yxW?>DaK#;x4^M z#)h?r9T2l<{YsDDMt;G8G|)fpzcG6aUTot1KY%mo`D?U)k=EB)v8Nb6-TVstY7J)YyNgfIcoh8v z|HfGl;_v8vV)2&rn27bhB+4J8eP`2z`FMkqr1w*gJYn*+|E9-if8b5%XZ@_(E~MAZ z??U?Y=AkB3(WT)($iK9|sb4gL{EIvn!Phd)f*L4q8u5Jm;Ct9@H8(Hlr{~aogWVy$ zERK4E{`%W@^oPA}0Qpzh=)SFZ*BO4R^Dc!PmK^+`tpB3zeihsDwhG#PLg#puT~BuI zg^O17t|wvT>F<)I=}_LY(O^QE#=bweWex0>GcHbf`KL7aT3h?6r;SEEmhA$@S$wee zrS}Wa?caTDTV8$lhJ3kd`=Ec~_IvmxGT;^UCoTI3dCOjNxP6chxxqN2Mb9apa0` z3EYSLmZX=&o@et=uIfSfak_lZYbGvI`hP+Cv-zz|=(NxO#^Bvu_R*gA9rp+Rlcxwi zJ(hkiNj+MHc?$2mxs0hVM|E!vcz)x+7rauxZYJ*TSO{{(7F&NS`0PoO&K8%Rg?dp_ zE1OVsjSWA@*C*ans`C@jlgFX=G5CqE&_m9-7(J^8#=*V50M1jo^V7W| z_ANE^it@JattQC}y{7jaB|4<{pKsddV^Y9gvNBshFSao^>DBPou%G-yGSm}o)1Vw5 zypH&3wS7@8%QfQn zTP?9(z;@c_y~7DIBY_mX6belDyyIVovV8r4n5}i9S)NF?j5t4a-!Dqn2;lNfqhem@SsON|-_dE#&L zfe+Z0VCWNZmd*>Xaz`MK>{a102EIs4^k!3OexiDHC%Qbd^V0tDMgdPl8_(z5lMyeX z$&a(|lX81LN;iz4^UK!n)k8Vth8?8$3J8j4To~DZJtgop`epreJ+Ue7S@5>@ch?Kj zd`VXP`qSW{>^(tN;3N1+<=a*h_*wqdRcKxz%1yz2KY4`ae|itug@5Nl570U)8~7@P zsV}!zp|Dh}_5Z)W^2_+9CN5go^{|6>Kc*qQFUiVHBKh|#4|^lZ(0p3Ywfe=^mwq?o zedYKZLac)*`g) zdVF9}lDG8D`4N0I!fAL)CVxSB`rT=0eQrkFTO*N600uV$Z6zEI_N z2zQg+(oHu*?!`u0C(seJE+g6olV5!xfXcti6>jRuKRAWqgK<{?x#=h{=)n(fMkP&o#$@%?aR%1Nrky57?892@-dnxos7yb_W z#a|R9ev3T}`j;o#3;K#Fp_ffP?binSMJ6a8Wa4Tv?Gy5p1L1!#pCLO34KR`H7@QoeIu- zGSk1%`v2-3eNB0u;1c>Himm4+?wU#VmRbEL-_EIk^gz45AAb+&ZwKzXek32UW`l03 z{yoHHw>9TXJyCuG(#4BxIOi!o(Eb!39EJ5>b*)Ehl)JST^2d8N#JGrO#nea(^1?C-tl=M40cb*UWxJQ94XQf;R6ZdDPAguMey`^@>> z?*%{59?#i3x66~P_FSuE^3_56J4hYx-9>y+6y(&CMPHimhe)@-hpyLT2s3e=o7R1G z?`q(C_VgS1>2EUtZ^yr6dtbFf`9{fWn~>$*w+-;@{Ylk;^oaaR_bu45wabw|fX=t+ z>_cNA&hw*Ry!MInCN7$v@N#)V-FhbQb}Q z0$lr^1Ka->dQ5(9ihNcr65}N!FTtO1U5X7pbZ^cJJXBL+?xKbE_f8>>$nW)FN6xQsg)^ak&jbBRE!{mG3ktMzMH9z=ShSBf3i8)sh7pK#FN4Z6XR---ZS9z-9Q=KkD zU-0&H-riYX(LcsEHRZK^H%PR!<5PVF^e1!nG3|N{|NEKjw(kS@3!+T3KvSPz8jA69 z_WeXhYbVzS{Rg~!J^)WW+TOS9k*hw^v)zZ^$d{1+q>@a(W70+YJFsU!-u&q~p7>)G ztwX8W2Wz3+pfjsX$kNj9rOMAW;sdVtAK3MO?^raix*z3V(Yl73_zvUoZ(Vm4?UTxV zqeu^Y>IJ;T89EncnlPU%uz#$9qdQ+aW>)C-Jf;ykM8SELUdJKBUS&X%DFo%ac9(eKdmfL)Mp ze!B;7(x1KV&K`b5q?kPUFttRgzB z>LL`ieXu?&wpL$k(%FF!z~hPfKz?|u`ON^|-n(`+px>>KA8wpC z>H2c&#DLd&+Y!WjT&{+AWiM|N%9!%tLtUCH6BjZ0sN8~t@RxM0#SaZVXhO#MuBkN6X6kp#LVxe9GEk?nn1d#DO-Tzqb8lc0EP! zu6o755r4?PR~HUWGI8zR`vUN%ZM|e+duI(-NHE^BrpzRFz{ehc4X@3p7he$tdzBWRweiqSbiF@ETDlkR+K%qyIy;!z}b zJOiEQ6`Q_8&h+b;Cy;|f){lO1;5P86lm>pKE2}+M``?zOCS5muvzygAStCQ)LHUDO11?69vk{{)jAY&`15X_rM-E2j4sIKB%!>dI0a24@r-vOAR^GUt=yC z{M5Wa;Nz^<=;E0GUoE2fzivtMNb#KZg_QjrEb*)e?36C^nDoUtdf(1juUA<~uX}#r zk^H{iE77jwuX*0t0zc|?Nka{OVqGBSQ+#gSM!0C1#*uM17+te5L&f1qcWdnEifJ~kD}d6`0)U7m8{_YjP~OYZhEac7<*n$kNe zVpSl1C(x7iKc71M5cXKNw(nQi^>{}wtMxfs8a!O9FG9}wRUi1z@{--}O+xc*XFp10 zNc_f>SHV-DM_93b(3ei5a`$kux3dO+L^*p-CN-qE@yOZOCfsE+J|JUH)v$+I-SVND-{A9dX1XVCuGuJBL9hQ2#ZT-AvL ze<^3*-u@n*+GfuKW*tU)bjr;`z+-dZLCD*PHlUwpeWHV#1low&|P_u4!l@rh;Sei-=d)`Wa$ z$L_}}yY9jA(RqA#zAvDIwC^Q~r;8E*|MPYoZvS+QzoXaGrc)R{89?@!&9Z*P>)Y_l znDqnsK>EEPz2#R)1JCuC-k)-{NC1CF*!??EgMJ@LSL?b3_>Qjw`E%R%0mLWyAL`q4 zOz52FkI0beg-Pd8wC<^0)*d_W{vF&)cz-GZx#qrS%A4}?Zl4Aw6tCJrK3KXk@YmJ% zUflq1VQ1)bHjnTSb7`MRM0QDO^4X~j&?k<(nwXrE^4lBkcn# zPWPr9{@@{xs-m1dzu~EZPt?ko&ZP5ZD=@#Z$)5X&cAXnY#+&PV?9jTmP+`Oy-o+uxD$xmy2XFEMeizUA8gS+McP_vi*$aLfYp@Y==>bjO? z*Dj8B`Jm2UOk8B5eFYVf;T!UYq}pym=TqK8vY5D|FP;4iwv5h!@hh^a$=3rqjzs+^ z+8=i0QlznZ)j5CGo6aTbur??!*4q0|vJdc9i|KcQ+-L9N9Id=<&jb4Xg}(Zy|Lpq# z;x4@pqe~K<#Ot=d3?AZsW{f*aO!IyHc^UjdU54)A%HCekBhI?MIBn1MrSA>Aoc9ex zszAt#nv>w1fy)ljyi~8aN_^kT^6N@E@9iF#WjERvv>(IOQ`ix4zQbV9r`bHf)2R;b zLj2;s5EJS%{;+?n)z`8}x9hUb`{X?NTJV#2+aB;(qVB{$mJa%4^deJF9Io}jglz2$ ztS^WOnPU)-r1Q*<9@4uX{D!^sDLT{StHj=6CX}(ruITFxpy$+B+J}`tVlF}t&iqaL zAGOP;K)oNN-}R4H(I&1ZbogUJw!X<26Y{!|rA;WepMd^VtID@Cagm{3Z^W{8ltK0t{E}uRI{c_eF z_!fFU%-!wm1mNMEt9JH-GT8eqJ!qZYwYWO^CAQKze-%mVzAE#6_-lF!&7)OaI$tNw z(mDqFMfVHXs^QRIYV}l%GwVb4m~Rfm@3yc?#J9@(An4-wb-V-lr|c%Z2clMv;09k; ztoo1x<(&?E!0rC9b01l>wDaUYhnAc2DqFu`6FPFEcL$Q)ebE7azgm!k_}F>R?@S}e z1^-(W^>kNyPh4lO77BQ6?Yy))twV}KEpXpgcBS*1JWU(S|9BX^7cP!xfW8y=#(gsU zBOdIpW8mm4d5DkG(76CMCo}AYGp|>(hkKiH&d24f3uu4dGL+L@q8FIB<45ypg`u~c zesX`>k72{Ef^S5AbrJ2J%>}-6`~s2J7xOq>^daH>Hw^UPd-KB%ssiH(|6YBd7ufQ8 z@Hw$qJ6u5;`EE9I<*+(t)I)k&3k6z ztOEHBGR+NY@AGZYOFiumy~eNHhP{;+Z;~CDLwutBat<{0MIUQ7?6~S!be~D=yNdF% zKK-7PXWyX<+WUtP{lni3r+eIoMjryahBUsiQp5w(Z)d(EZ;f`Fe0}o~;5zd=^>b?) zlkWIq|K4MXpz~77;Uj8}X-W6@`77Jcl;p3;(p}MCXZ?e{Oy1`I!<7f=UKLO01Nz8d zIw$J*e-=EGzP?zGbJ=(D^p>+kx7`~^uIxEhr#wrZf%vg|h3V-3g0|1m?$pIkOvn}{ zg8b;j^lq(KNb6Xt#XI=fY@%IfzDw)ruE*m(fd4;t0N=8v--utGbIN&qao%6V+^J~b ztMSFRq8)DUDckqn)R27Wr+$$6DDoS&j%Pwg4%DPn;CC6Be2qzW_V48g`yQ9`Ui6v0 z=TWZ^{eF&PubJ2RElBt0KPV)-q1qFl^Swl0X5Z6h_V?7}xxN@D^)svWC+)hBb55ZI z&7T~*%BS|*W8kwaN8mrn35UlbUMVBwUO(D9-^3j`{a5dbeNW83A0-AZZ3;XuKAYZ~^4zyCV(RHH>tLTme)7{qH_BJ@(}n?VLOPemw%gx_S{((y zMA-chmbE3;8Fbb%kRvsV=1HuYednSYt?TG5fq@u*@1x()uJi5;u=3z#`M}v<6LU$P zMYpZlOnFhN75I<8iMI>!bq9U|zYi8aXTA4EV(1~c#`<;PG=EYj>E1aHpmieQy$W_s zHTFFNyrz$bep7Ggoe@WGIdUXE#9%#y+jA-+7Ws7+eb0NR(aQ}!&UqYd-&0fV)?>ZS z8Ar9Q5%^UtwD+gBCx$%}U+F%KqyPWS*WK6e1Fn@ft{Y)~q!``Pbo7nrQIqiREytm} zMa-e6w!d4zGJU~0EnS*^_d&foKh2bP_}|^)W%d85AKq!biK~DQkXtb!(HBE6Hpg$Y zNf+mH-b0wizUQBi*7cano-_BOd8VoI|36jd9psdk?A+MYcfYguSe^5S&N_rs$3m&~{TyoTsY@`{CRl<3Y*Cqg-A4POZIPC>qi}lDqyN@bUk3 z_T}+D7vKLwmXuwHEK}LDgs7;zWs9=AB+AIXl(qUKF|rgTOGWk}>qVB5vc4@9A`!|y z*+ped*86+U^PKbgbbTJ*e}3;j?%a9J%$e=XIcLtCIio+g3+Gb01HH?p`;!0T2rur- zzdOKDFC>9)+V|#Q9lBz4?jCz-0OU7WkA}|^{E-D%-)`q|jHisdbkpdDplf>dY{(n& z$)AGH>6Yg}mu!>TokG2_bA#KVe^qEduM50~`c;l2fU63SK3C0227asTWRKt`{ss7M z{u$t%U(gA9K_~C94y(~Wz_n9J?o;hEVSfEe@-w8f_Uwm#?myw`UFYnJ#5euYI}GS0U~;y$v|B zFH_Sto(SLl%u`j-zBZ?U4??%pw5#Kh&hOsIdbFZfn_4YnNVi$_7a`P*u3?=eynGx9 z_-?y>dpdk~^Hz;U_%zuSi=NqkynAB!e)}(~CuJka^WD|}AI0v(k0JkD<|g?kbF&X( z-u2Ad=&$aar3>oIknOb)nkSi$RG{&+>^Iq!UW5IQ=)>Ft*=e4R52Sb1PY}MT)ptPd z*E0))4*80so(gbv?Qd6v(4M4yoNKc071X<$?oX+o$<9a3qjbCLlPMux<)wQF<_q?Z zSiD_te9v}or4X9Ww_Za$>!+;m6<3{j3&-c^JXXGbN<)`~9<|%s5#N}uGfu;3n zTEBsDw5tyl3~*if$(RqjaxC=Hk>2kqwg)}^>Kf!Ln{)~D=*jH zUJvu+5zzcJ*C?9>kN{S$_xntcg+uJ7(KJLD63u3lt$s{+Y2V$Wc{EDZdz zBa1=4vSo&%oJoHJ^Q$M*y=J|I{5qMMn>U~zy|-Y0s#cKPE$h+Lruztb(2|c(f9aPm zpuK(g9P}4DhZQ>s!FSL8wB6JBh<^$F?$ovn%Cr9tX?OgOwMWDJ=!X^L?)b%Sm}kZN zP&HvTt!o|^-De}(EBhVU-}+X(Ka=s}&^$}$4-vokl-94WbE8mbk7`W!m2K~edqdpR zx#xTcMUDztQvKM@+Fv&$N6WpRjBSXX-b|T~{v2F@ez@|TfcK^~?fZ0v7cvF-y6)XY zLMZz^sZYy2rZi>xgmgbS!^0s|w`OA9>RVreKB*nFF7&dZn0KFl7mW+wgD{6^f8^7= z(G2yle}DJfRLFgP4e6s~Tj`(s_g`eBFk0o1gU@OG;oJ>pavCq3=3(P4G9H z#OM25k=gY##$T0th3IO#!>0l~ccBc%RnH^2Q{YJZbfc~s&q`keV8q|5%-jy)0!aaDrtJXVrj zdH585LU$)1;e9w9Y$sw|y^fi(@Tw}UN;Ljf?2(I?|#JiTqL^{`&lyV!iBF29{bfShb*9TJHI*ePoZ4wpL>A+t_aCdetc7` z6T5ji_=5S9*Jo#b7pnVk(9KBhPID6FO;S?9SOT8x^L4iyp921EdLmnj-V=}d*Capi zOE)b<os#=!W^@DS4OGsF4+J=x_1cMZQ0O7_jA^9z*rQR8sY?5is2>lKp8@yRF5+Ke z|H=1w)X^WI7xiCehy1MM-qqbRA#bX#-_U#>sIWQI@1J7%mG4oQ8Z`fQF8R$f&FI|~ z{S@E#%5w{JC2qRV`TcQvugrhM{U0~JZ-6U5{W#tK_dn=H=&$%a=#~TXYhU_=+SBeo zprf|;cI?aC$U#IG_&$#6c>O5ayS>8D5XP(XyPduGo-0@A-|7Z<{_f=`(2k$(>WAX80Of|DW~5df?_J>=H>Y;pQyQ9r8zX?uA^CH}(6)yP%z& z<6A?Wl2GP zKXDb{sdyHek5+FI{DVYy+`D8yW}22ifbvaOqz|F5O>{;0y!$vA_rRU_{d)F`;US-E zPUp++z3BvZKr!ln<<8Linv4w&p*?rAeZo&{m&N!jbVzru4Sr+m^Z*#LEj_L5#As{a=8yC^lT_*&@CWZz=1 zG{k(nwfnJ-^tL*npTZ~nHzPo&^fOb$Z+=g_SL%P%> z=VN|*nYY6CNKWB_@imJyvTk}{Gi)hZ$6B9U^*@P zN@=IcIvH|eZ0nG7A)mXS^mL*}lzb6he9!0HtCoVU`Df_;mV_R;73$#vE)&sf`|G>K z09X9_^Si}vC+pFcWN!HHa<(nmm#O9%=>FoqX=uM~x)ylgEZm+E^4mC_51W^% zK9lWNvLBpG@1cquulgT?{!#pJsKsxOL%nyBd=bgFyq}1D-Q9(B!2?r}^cQMLR_F)( zfR8(dbbYlg-LK;Y>&1-Ua*pySt$SC!WPjwpbYXM|-JoYNuO??sqOU`85WgE+m-tHd zCBQ5DBHQuFc?SE-)HBk~4sgtf7YNSH^`HZ0`i4D#U#AD?mc8%;$ssvMU>&JJWLFwH zME$ls7cL9={pk1-gwq#7p3$Qwp`Fry?q1?OpWf)d>QD%DFuJ$Ec5<>mbq#p^jpzHF zImbY6qgK`{7wXZkmBsia;KT2efBpL!sDB*2t7q2nyyV_a{B1MoCG_C!nLD9y^ z?E4mUQqIpz)w-B(cTZ#Bzx2cJ-jNsO$I*NWy-^tt6vX#?zW|rCUNw5pn&-{+_ z96z->;rD%K3xxc3-D@R67$4pJrx5x%fBYRnUH0_DAylu>`FcWcH!CLGWa-I-74$mTcN2g*57s4DzoMR?p^q;b&&9$t+0 z@cBgIT!_wtdEMKQeD1{Zy_rn=Ayq5sceMAzE5Iwu@5%cXe`CLC+dM$y@R@oF@GlM| z``bdKe|2kkpH=P9g^-@G7a{p#G`$yTe|ew^%AF$q5vzC(_+(i>rQVc!Gj?*vHP)3&GzHyhab*%)XkS-y!H8)=Y@R2M-1afVjls2FDCm}z$?x7 zfV`^=dB~&?9TGi_$p7}z{6Ybq8+-)#YP*wP5I<#V@$lWR_;e@QRfY9yEiPdG&D3Pv zlQ8v&|0+xP8GCFQ^gR0Bv~<49{?q;L-(C-J;(b`o_$l9_5KqX7C$ay}x9oZd?WV%G zkCONoKg;JfZsJaocRqa;=Sk82>DnJX0~~jf?{jueDih-JGndH!i`Z$}i)2@=r!@i` z$tU}U#D0&)Pxehyza=?k;9RUfpKT@S1-YPq#H>Jl*XZ0x-epvCFU`Ssw!aj7v`>sUmiTL~?!d?R9jtF-`YPoTd=L0wre-b@>M`kG#rknwX+7&_lCU0K zO|qxdU$S4NDW&k;v>Af_2>rD$@_NoZ=BEHp&rGIw(7D3f&*hzEzP7Ul^dA13p)^0+ z*#0Re(<}Cqt9aio`$o|V=(N|eg!<%OgPW3p^qlN>!Jo?qxOzy=?if#Qk9%bx=#si} z2J(cwr_ISqvA<@qxAC{;lN}t(pKjgBk^!FY!g5O8NO|(#V9PJW`1wx1ynyy{eWvO% z*u#iEBlZ#Lvt(Xuy<-oc-N&b_454b91NsU7G1*hQ^mE<;yyUOYZhJI^&h748j(IYR zienv_`F!pa67>G-`Wy@BtWvj0EzWp&`aZodQT z#8=-l8tuJ0m*mI-G%oSE7eU``m%o7T`X$y!wCD`FtC;V(eW~@x!iOdFgWoBJ{?^-G zALBvIPuwTXGBLzuzssMM&)O8@F5@ZV9Dn{W){7hJ#{nLnOSuj#4>JH?oBeKz-@r5F zA@9YX?~QdHe{LT7Ytp3yo$+(aWk}U7xrf2}bNl`yT~J^5yu`N)(E7B`v0U1@WaW^q zbL|1&vGYa@M0_9Lo3BdphJTIDXN(;|deJEZv3}K_KOv{7Iz=$9t~$Llqgti`fAKrX zua|9Ehx&hz;0hg=adQ{?9{G1a#Qx1xBfBU6KJj^>M>4MZ@Kc>bIX`{FaD+!m4tHWl zU-cQRS2;&^!)W}XbIg|}%|yNH=zTJ`Z1{~3m-4Yn>9Mb{yx&*7w!kh$^pN_VM?tsb z{64mY?4Im^rNFyr-KM0B2YujuU&N1#?S`K##rntCjp~;|{pLNgt1^pL1E2g@!e2Ep z8Sk{15f#p$-Ll`{^&$I0_f&O~g9{gZ8}Rtt*9=?nSBUF0Cy^e_!*ufB;?`!qC#1Up zyk4FpyFFKj-XqoGH%IKM5_!WfI#*s#=j@u#6GcC8`+Cp==JyZv$o|)}KEO|*`&?${ zbE(#wtG8)&$nU#V|2>4#&&0p@Nn9^p3w?)t_xndcK6As4o)6_z_GP_7=%ys&zNMVw zxN7V-svYZVcw8eo@Yg`tSLl`-G~jF`I}_h!J>(hL=SOzX+yE#3gT+363ZJts-G%jL zCX-#5wBJ|!^IU*0bj7S?dls=jlzGfw`bYGmOT)})kLXiAO@@7Sqz{sIF#YFfbUyD} z`~&`>`!2)2SJEYPZUXdJ;s?RKNqRtOpQ}!E+f{x6{70vdou0_QqDOb-iH<~Y7boN0 z*=YP(4M>94me)kc=b+L~cj^PIBmYTG(u;2= zJ&2^+$^+_za{f`0o8vu*Z~3*&i^sxuzBjJx5`MVFmi!fPJ^IR)$(VNr=<+^40`VuO zV!g*>U}I*3&71=0q|W1 z@+at~Z%&K!xi^lXo_0JBCkqx0arv3m{~3(G1@tnMw9^5?B4$Zv~$hyQXA+ zV`h;(n7fzcKAn^_6#J>uWY21IlHIKt)C1$MQ)oZnucqML5mk%stJpsOVE?N2vR|U7 z-FS@CBziBz)!RHb#Qm5epnt*-qjko5UQ=S%hLA7*;lVh{*G_sXgqHm@#)|U&+A;i2 z#=YPEhJ1YQNY2|-k5O5}cNtHeyc+V45v!49DPr_kBh z<{!XUP1khYP|pG8ulKUw=S4RbgztU`y&Gs&kpDTeg!TWSZHb@R)PGILXYNRfh0wBn zjww$5Q)7S9J36XEmRhLCHiG=_vJS&|CeHuwk0r|aJD+WYrFTr?-%dk2^x67o zuWL>Eso1qNXlJB1tU&rnm*HXTLrqc={PFrcM91_+x>xCsf3*btN&c=+2+f$rgFts--JGX;hEl)KU*G(1!lzeESCrq>7VXe= z>D@E)4C(74J;NWo4~nH<9P(@R(BFVRoZb!8`E3!vk&5^prK~5%dLl_S7d8|M`9=S# z&e6P^Q-7=ne70vX`IcMMQ;+;n=uO`~72>{R&K=0t=+#{zv~xZ`h40k6^vb(dLyA%YxZemxSJR;=t7s(#dq$9arU4L?A z$Y<}_S}ugX)=TKWyFurNeq~wUtE+alJK!bqgrCTDz83lYv~pjE{py-7L{IG~e>{|n zf4$xSKHJ~8qNLAF_~)Z@GC=*m|M)qFGludy&G(H`wWB4=O_z@WuWUa(E~L9VpWcal zS&sp4Wgnte&qF(6HFLv`)3&?_er0}6B05o*-%Y&9ezdtsrqXLVoc@kM zjtIE}ZhnUCAs(IEEH94zjQxuI4X7NSf?m1tCq6^IKWku~M6Pt*$ls>RdK%;Ho8$sL zHv8vIPE`;4edKDDeKHg5jC?MVk94aS2(RYerU&8z{+ZkKwhL1#2KchSvTRSI7I!!s zzMFaEm)e(SyYqp(jz+aUhjy`ESlW}IpO`;=d^-4;Iz#@=V|O2Df_hkwu8wwt9!}My z`?+F2Bye2MDKrn_C$i5z8jm4l_vyOR{mtmyX-t<60lsGak(vA>=(U=$Js;Y6{*F^2 zG|k@wzUuV9V1C@yY2bH$(#w#06LfcJz@J!$!JyabK!4CTeZRM?-m?Z#3awTC%Hj^XME){jnJ7@joZEL;VX<FVTOn-ZzrtgkRoBvlI07KQ1cb@ox*l~}2bvWSOCa1`Jxwp5_JZAMI&o11NY8|O> zZXsRlk2X<|0cv$NwU+A73BMp;alPt zokO1@J2RPoRg~@_YknWk{;_v8>Z|ix1BBc!tInf7wX74?IYCX}UvBb)WNYq*KG5W1 z|JN0>fq$ycb8SMq*7JUQ6x}P*1*a096hIu>o(E8moBuiJsUJ&zyJVju>&*R<pz z-zD~Y^t%f2Ysc$KkEC-08Fzhs z;BN4%8Oujw{C8}j^~2?(agvVQ_T5clJ%Q9WuQA3)^b-C6omZK?*H5DV{xJM02spEq z{E|vsm8?zW8!Q68iayj29Wa5$?|0BEwd%#nA#MvLV_z%%F_~szzJy-5Q)LhRK^8r8YMCjAa$Vx{;JoXFmH@_j7#zzWr=+|7X67s2YBS4RYpQ-b7UK?dU zMt-Ie=M`92YUvYIQEoxmRv|QvN8E|}YOx%CN0uK#T;wY0pR99tf$U=CzJ&W~P`;2~ z&J|oQ=GUwTw|uYG+(^MYKdyBf)UP{rdM=cU?`lvzgzBM_c>zcCa}9FRy4p0I^Z+4G z;%A!vo*wy%d_w%wkO!gsxt>`C1o2fpCD)4P`DIp%K_ z3!Xwg*;j@(gkOL1U+C+RpCqCC>Ro!b!9T|KV`-ZnM)^MXfv!gMfuJa#s~z^k;_ptM zOQCZ$KHpNTzgAaBZ{fz0{|-H_O*X)NG7I&mC9NkP`<&#wu5{nc?3#EnKyhfTzx`+@C)Qo|K)~-Azhs!yEsX=ZPP<9X-?=3NN;i%ozHRxQ~V*&NmGI36a579 zn^r$<4f!3vcjk65ANZgY=!F~fD(!bgzN<`r&!k?LJIjGkUO$q)aR|+TE2RK$JK38> z_YTjsh==d$JfGKXU-DLn%X%~E)IG68TdcRY`Wd6xEZx>GOECo_%4A-X@tem+d| zYE4lu?>oisP;B0pqtUMOwY!ASozFErgpq&8CP~W!|Mq3cZ-LxTcgOY&Lb>nxT?0A4 zokQcAD1x8x72zAcXE&N>PPZf2f850_G|pFhz7pUj#*_BxH&S4iDR89!uHdiGuc-2w z&xHK)GyUnkKRsvNHT0wBP06pLdwOm=#9yO(6>e}h$TyblwcKO0pHg4%CVG33 z_9v!$UEqt<6Kj43@|1pVKj}}`ys#K>tI+wSpZ+1_QFn&*L?7hnhxXim@qXl6KYwEg z<>w|m0D5B|m_YLW(!RMvx~+74TnJ5k3ixO&IX%(;ja_>o{odxG}H&r-{gzW9J6fGhCW4$v0T zn0Mi0vd-Psv=03oQgs*FHN6h-RL>$mEiUPaYN_CreVh2wt^6dPa6N8yP8w%^FF@zW zebN zrS^iZvfne$_AV|D=`VZ-y3cO9E*Tr@w;Ky}3!(hf~Xde_fBN_uE_tuf>lH-`K{n}wv; zP87jUe_Mmr-Dk=0n=Ruh`zn)iFYsUNu+>NP$UoS=nlsU!AIY9kC08gL;_g?HGi>{* zkncpERtt6l-(~#NnVjcCei>K)T`!`ma=(WC8Tg|EPla)|>F8WtiM)f>C+a)(5!Q{p zM)XenQRxz2b`0?4C+*M`bCbWm<+Se1u;QQ>?$cG@qd!e2f^Pe#$?i0+))U@-d)I8F zzw$EZqHpv9jq3u^f2vYx!B3-e(lt#mu2KEGkBs8~Y+es|eZIzemvJz)VyWneVn4ky zZhk;p;D?=(485Bw`Vi6S&$~jeDe!gi@@S7=JrwD>aqp_3J~x#82+4im0d$W|^notl zhoF1<*0J;TX`7mcPFF4q=)dnsa+mZ&=!(SMJAEDx^|?$$Z|(Ve zKv&fx!y#|PI-kM(*v3SMWc}Jgbbc>zd^XY-o0_EOHti;+3vlD}$X+pempMD1TkI!! z#q@52@sj&or_w0}u z^dV{;*)c}>+tK^Z3A#K3@&G58hj-)tLX>ZFxnZF{emnX7R0n5mz&v%~cNE9bJLGog zws*sK(X*;wNv{~eVY(su1@+=fwE#ylaHwDBdlejaXMU_4@>zaISo~yo_DA4F-&p$E zLco*p6FSAARAb~G_&=oY7J1ja{1Dc)Ua|mmT+aQx`ik&)75N?Z$DZ$v_M~K|btZDz zkJ&NKrq|gvTQC7)K_eew!u$|;ru`^2OxuH*r^by=p)3Hc5^oQr!iCs_&)}Jxm zF&FM6{u0Ys4)q>rF$3ioj^0(A@M2n9$RlpkaL_Xu4<+x`h+PaPxXv3v@72oAB)^C~ z=H_8t0(^O2!@N%S3k06~V9$z>F8#4<*?yV%g{qeU_+$G#(IMnB_37M4%Eww(ZIAE# z4viUK5bN0;$oWC|ZZpr^5kfuhBJznJG5gO$eZ%)y=@y{demmWpcNzKoiQgOcvq&%G zifqBYMb%BM$CCZMgwig5av;vjU9Pm>p?(>U$0xQ6abL1E;Y9~pr}jCLFWfkKhh5-^ zeXzS&XBFV^`ZK!J-GH-`_LpiB`*Glc`f@M87dQ#J_+H4b7M8|7A^N^?Eb#~4-+rp}Q{s9Sg#D_snpTm-nWnaerhR`sbHyzN$}D`Neg}gYWi-ijX%{ocL4ZkMQv%^7|v_O056Y>)Bpo z5#hNXpSR?{$4%C`_?1%o>AiZ9lLXJ=6>dO|@v+jNH*U$R&|CQdd=HSj)EZLKPpFPV4W{44a+ zGQLOh+X3#eT-~W{ca;U!~J%j8Ith+#ZhlQolZlNPm%zpVk!Ryx% zbiph;Rsis4C9es5OymKP$Hc#4RG$5vsBLXBV4j3uJiDIo;@PBK;D@}vrJQ5?SDDY- z@l58?{Xy@m>qoQWoZa<42|8mtwE!I!e3tPN{j*EMb``hDp{DNTHE1U{MCgaKOYq2E zA^*lQpN7||DNpA`wivJPkIO=zsHc*DK*0l*J;8xAz!pl+DrDi3B4=PZ?S9oncn?$Kc&DAcf1JkvHyX~71%lRL1)|ovVYg? zFIDQ7dA)sp%v-;LKL3xL#dZWy`5Yedo!d-ul^vFX{foIp>%nejyka>*%0=Uu%=T~{ z58sDzx4f><%bvh8>=ze4`EO7oO9DboL z^4(MKX^eY^G_+sia{xOsOR9aA+d+OwM9&%7C-6BztPh==NImg|w9hbOy3##Awj*&j zX@4&KO7gqkQb4Co>1HmpNA^XME}_h`;JwVBq)X__ox!}=4)+27ZI%Ho(BEG=mWd@Y z-A)E)lvS@~GtVQ#wG7lY2_C|99C* z-foNe3&8z%w zv?4nSoA(FEx312YZTgRt~5jW*vPS%C%OM8=lFtuZd`+NXve(D zg(#$RQa5lT>aqLj99sILR?~hf{zHeIA-_JgmhgSXiLoIrKRtxU=_dIdHS9-4(%lS_ zGwq?aj7Ou=1D^QBWO>0H>3AINVtt_L&F4hy$J5pMa3IRPxvn$vF&>EBu6v{)(e+|v zKOfaE@5A#4ll|8EP+rad>T|%^KLzy2R#|pA;GgZ93G$se^Jgmj(bo%*{8y3JP5Ujx zcjTQ5hA;EWQDs**fqZM@nJItYG>Dr*J0O49V{O1M4Etv_tT(p<=>2QGY%chhnsNu- zhq{CAf2)FZFrVh{T{veG`V`R@zNcymvV6o1(jSm~l*pgze9pa0@}WkH+y}zNwfu>Bs6M%(rXBc5J7M{vFcICVIb7)TNm#a$fw}Ygiv{P5p() z$NZ9;{oi{z%jkUry=4mF8S67;9@Xv>IA_t_$nGrOzbmbuCCy2`+xZpFnRN3Pu}_in zQ+cP`elQ$#K}{(~^6j)?z<2u@%VQ^BfIOw^lHX|aY@wQ=y>4!A;;Z@j{%XZa*U*2l ze|eDZ^~Nu6=#Bhm2NV5dyNlSU@ryAYYxrJP&XK9~a?v<6OPUhk$}jd_yiq6@KR0`H z2=yU4PxPnKz9b$>C&|Cahbj%I@~E!LzK%+esX-lrf-F~KTG%6 zT+tp_2e!{4qSJiuS>XG3i4W@Bw69QYOM#AQwo|v2c)!l?z=~cyqK~h}3&s+_T$80Z zXRw{wZfSd;o@j^Iv#fow1LDofV1F$9#T6mGE$?MV`@G!Oa)tc1!Rf*X-yefZN3t8rd^<>H+>L^04B4pp^5IH-rE9T6-aHnCsw8+<-3beqRr@FVhD z_u_l7OHo_MFOWHY7;>Xcq4QDu8~H~yzY`xYciu(vz|(QuqchE(Kl$(aMNbmhSMt3k zsaN*JVuvg5$-Co4H(a>_v(Q7= zhWw=4PtLva?{>;Mj8(3L^=&`<>&}p_JD0?`3LN#))h^+?yPQJ$2p;Iz0`iNmm(%;$ zGX82S*;Tk4TLDM2{hiQ1$tQFp@)yASUvqaq%%`+Z=!Dq6NWa7{gut`sf1HK>@%!-c z%cNiR!-kT4&hNkYMQra_ko{^&y@EG6$^Jm(Vi^aaCsLpKbTZ~$4<~s~ZM$?A#&6;G z#Y5rRfe&gg z>2al=$e#p%5V_sPo}>KyUDszjNs*`CrG0U1Ne;*ls@x}MLqB9bH0yH(k7b>@CiISu z#C6^eKSle6zF*%9x~baFNE#x=y&cYPZlSzK@ z$80*TMZ3*%(yNKyP57JSm-&@?RFn0fvm*EEr}hzD=l8>JobTA1MCar^Ab)c*_a=)Z=_$s@;B2_Dep3jf_!IFib39$ z@r|F3fnUk|s3g9}!3CurX|KRjzjk>j)GI&1JIOEe>wa!EC*+fRreZ%Q<>b9u|Mr+_ z$jAFv{qh9JV{$Lgmd*jXWLS@+Z?g+^tWR()=-gM}3m?&w49V;3*#FSx4o>1v#QrU|^T#6LyU;y5@nhhvYeD{hP48L28`qTl zR{P`RFI)U^`Ol}3JhGPji756XDD~>zH*A0(`7x7s`PlEVU48jlNVk9d1-dTt8;XRV z-FAQ8SRyo?I=qec^LtGp4ZrmCE~7-<$i=DXi}-Cca-StC$RvaA{zDNA`iG|qTo7TURe0!T-Sg+votdD6O$1B_m zKHzK9J>LK-{LHF%7Nec>=-nOqn)q9f{6e@g&Ecm-zUzgwzmvFI%KORF?_xak_h7zB(}vutOC)8sV2RukLa3lP+|{y`HQ>d-Vbf|Jt%HrJhJm;Sc6D)<1Gw z()AZ9SSOzScnCj=Cnx8OCGy(bWPicMLhwK=@yAX74*OB{#1iuNlbo!>cab}_yo(pU zC4PdJ_Gz-WH0;k==$fy1{1C=p^f**3de6>?-su+Gi+1@3@~SGdFKsN5SH>&$;wZ`DOJBhhm>&-XMD@ zZApKt9$8l_ly^6d{ss8i)`Gs652;;#E#31~H~8IPq5pS2dLiVOpT6<>TC^i?LGVf2 zv=sD2w&j+a=ud^o^N_zl_kAJs`DmY}cD_jCw}<2gy{HZH`#$7v)SPVq{v+>&xv7O= zH*DCCfc^W|e*bPqEaM;dgt*|P&2t`l1an_j*pujv$LLq2^uWq0U@yH$wd#bclSyI#3>q-);a5iN=rZGou)dhx4EA03)J5v59U?qV)Ty3oJF9Yy4d|c zhkWL@te_+22HBO{@85&`mq^0TR4)8u$QQ{yV!twx?9NrDD%dy6I@68V-m*p3M*yGw zPQ@>zz+TEfuQ9&5!tTx?UFZID4f1up2lyf9gtqgWBu}({lEx!%0i^3J7v>^=x^tyM zDE)FZ=pKst@=KEUSUwecqi(RCl)214rpZ_z zkseI^(aL+_bMGD$;H#cno1lHwli=6Sm(PZI)T6$`_{uwTw$ci;)3>B^6|DzWYuspl5Q!Ug?VOS5xqQbY#E4b`JKX z_E^utcWo(;a-Q{Ms?}Xsw|42Dz<<+??7d?{$IyD}`QSl}*TPcJTgbYL;IZCNvE9A+ zLsje9&So3oqrSf(=(70dlkraQf9Pl9X_sJL+%39y;mgoHVm+kPpMl<4elOaWCOf-G zZV>;g6Y4+?5O*)O$JB=q!93iLBl{`UQ!D;mT68unKs<82>df2VwIC14ZzmW1~C zR>#0MBYTA=E3hwDd|su$yp{@I&3y6)?`rPG`y=N3Mx@KQ-tM0<;-}yINqQhYFOhb* zHUBgU^Cb7Ze9j+!K%C!2G^@LUU-+>bONZ~8@Ba$E8nH82H5PQ=b{j+MeF5!H;$>H2 zeshgeJe>!JJ zwD0w^D?(_~R094b>@wiD*!**o_EEoA!?@X>)*+u``$>_T06X!=W&G+yfFt87-zBvC zF0y*^7x+t*{`ubHYo=-^7Z5unx9u?KoJ!*RF)8G)MUS9!yo7%N=sRRRy4-`{KTqm) zg9eb?&+nP|akMWLekA)C$**#@#`=`+{!P9wCFB1>26|t8_#Les|^9^WJ7 zcG;<9mk_;o%KKHvenDm21g^c!Z{`2Y^K-;+MPF|Ql7BU6zx$>Q*+1Vi{UpZcO3D_j zTVD{5;=>7I|EcD{wfJS2+kTyxz77@mEL=()D>C_k8MokQ*7l zhI+-{*1iISXTQ)p7dGFem4G7^Pp|)Yi2EsrAqU83BYxIKyh`#Quj44)pRWNpybo~& z>Hb}UKj6I#$>%2h4nO=7m(c%pvUe!2_n!rx$#?tC#>r^Uk%^EyWd0QUyNac0bsqJO zBl{_r#P@kekv{?7uKs+aGr;)fF6Tl#UN1f9tJEX-8~b)8$uFEB`l*PokIY1L?&F_H zKIeS4kOLhuO;3Fp;M?gm|8@!apA`5Bx(NPe7JiHR<-5u|8uFzbTMu%XzB~(bH>$tJ zG0-)0c`N9Q;d@ki%W{%8M^-L|e$Fc2KZIt&8*d|Ie}uZ)F~C#2A2-c9L5`JvnG!4) zGX3CgCC-6~KSq51lTx6wYE7fp1ALd9jCaB8rwaRHz2;fa4>j;!k_WT@I05Zo{L?c^ z{T%RIx6KUxsLS6R64K-CceDpQJ_iy0p?;-v1haM)>;_a>qEqG(zVG_zfBp#N)Oz+; z^XQg4QD4vR36C$;?|}5*u3;TWe-r+nCWdyHv~?lZ$T-_)`(yuNelAS@;ot6aA(Z#M z(xE?YLmsRb!}|fx{yN0Jk#9kM7(`z#^v5;KjC#~He&=5H>#VmmYcDNCJ6H58gZe5G zUA67$ok4Z=gD1jw`%ez=N5^(Vk-YK*`FTrZ!Ox6r)(mj2lK$D19fI{G^u+P|+xC|t zkA{3EgTgqO=H1awGw`t&P~YN>gomtm6S+g~%lUsu@1XUyk;wn-65yvl`5NeiDR^i< zz6+h7P!Q`!@IE%;q znqKyu?E?Jh_Zii%e&0AOd^aC&1YK}*OAbhtPYN6-J3_aE_D8zW$@@b-dx7^Y`Hnzt z&~GqZyOy%{-+aP-Mfp@2UMI~+&d^Phz6kl`T>>rg;ha&K0(=<y=sT=ZpIsg zr-}c=&(4^=4eb$oYCc!8|(JAPd602O|^c2m$@m)zh3z#Co`RI6>ha`H!16kjbj|o4 zn??IXQ=0W}7msBKaOLN6k{=m~%Y9h$7uh@5<6QpS;L_+1kAu(w8Bd`@CWX#FeW5wP zBendwd7+$dcOLSwxp@uz>2^6&>h~FFU#+TTB>u<#Uu@Z(pof~r$&dOS?*~ac^x=%Z zg?hyAtDg5R@zq_MLEp?LSuk(Hf93w8s#6>CgRY!Rc+L0tWqzalaz4!KSL~|XEIJQQ z49=2iM9=RS)40ogVS4hA^#ey6XK z8~nvIZbkmmnO_TBp^p+5d~kdFU>!?6lJ1tS0bOu!kX@k6qon(5DX=rJjgxRMSJH*v zMfq6H68^2bZ8(APUUUZXYGNLDV4TN2h<$?jg5*Kfp8Rgxe>SHYA60TK@I>J2)=f3W zb4qs5&1n4C?o9HzmaqRA^63k7kI^;y9qZTcrF}*0z=wm;F6qyp5$Qr)@GRKq#`k`*kB>K(rt{uCD?4r61g&xFwmQ480`+C2Q-Xr(t z>D`g2KGCzMC%z!~m%sz;tMol|&n#Yu{B!HVxrzVYb}!D%Y*9Ikn{Aqm`SiUmmJe~o zcCnt{Ig)w>ucbcu9`CUo@`v=(KmJNmfD>=Q`*6|s&ExxZt3IN77=X}QfhY7*;^OyG z_`GeF5%Phg$LfB9{j|BBLjI`b-j_@VKQt%TfZje>9h zFlTi2uEfH3`N@4doAq1Zhn=0aWk{EEHknAJW75UA)2OiC35Bo8eNnlGp|`L7$E@>Z>L=4j_u`MA zqTY$LPZT=kO0Ydq@ky&gdUU?b=lJH-K->qD{`)%QCsn`pJN$S=>um?gV`9fB{Sx`d zG+{j~x6^0M40%MgOCtYCx8L8{_eL?ahXF-+{0+ZrBK8x!ZlrxuuY?J?3hxWVpJ025 z37_l;aBrVS|LWRnsITt<8nA|IZyhA$B74z_CkYlACCNtSZn#pUSACUZNFue<-`Momr1?@+o z^nvXEh2@I~uXZjy1iTP?Z?y5uO9nBRp>tMe`}`y#2>TR_xm+z z!RPF_OA}JTnO*KK)b}dwTkNSx8vwTf$(MRzUf{FLqrGrfose$oOw5XM%U^@Mp?{+H zab&%V{i(D|H$DaW5g-0(qfk!Pg_$=Dc&t0u#riXaX&hv{M33QH(RotzUF0{Z;Qt6c z)7AKQB%ej+o{Y~f#~9*^a{k}C6zrJXo(rky%k6Yg6=V6D?RvzYeS%NGU$*-rCAHt| z)BJ9+=$k}7XFJ{7n=gFRea-%mZp$Ei)XbR*`N2Ou{=QfukN%PD5QQG-bd_!edLlpf z<~q!$=nu{BrJIELus@JJlgJkd`ZO<;llwEi>T9s8b8H`OR*Hke-I@90gD1oZ0<7 z3HmekQGbj}Yyq7I$-M@rPeVWF>XV-VJ^P`Vp`6GauEgx7sQIDLf9stjU$`gf-DBH!?FzJ){d<{1C5TQRSpfTB&-cIo zT^0VXLH@3UzWJm%z+;n^-jlUkCrk-&&9q}PyE9PT~KT%KCy?>Cw!2y5A}J%tF?;cS}#}efc!-9X+1TZTunf+vjI@ z#eO4xf#1=aRSJ3^Rq03IxA+~@Exx*d_GTl$_mQ0OU;QZj8S5d*xIZQBkkF=7Avs`f zKgc_F0p0Uc$#kFG#ro1YQkCK0+bZ2=;H#N5nDk(`?ZcyYj=N=qoME0H+BdY%FF6dm zeD(Gk@Kb$(&zA~!fZXBMk-ej=Z@rEE9jTF^W76IPJt-65$-QeeZEXg?XFYMOSob0! zEXrJkq6I@AmN$j`hbKcKhdI0sAC3HVrFJ*fl*-x?G24DTT0imAg9X0{T zq1fF>fBf@nhll(Tf4g#!#*_2w++#+rH|4wI6pR7y1w-niF`mCVil3ncV z)$N{QH}G`CPfg zTadSX-h=Z(IluEBjFWzd&UZ{j(zELa8?M55FrO1Wi2W!V*(L0xcVwgS>!GloV=E7m z9L(pf(qAY3bw(%byFO}=RKHQnH0pKrR`K>7`n2m^yoxUVM90U9m)6BQ_UPTWciUIn z^yuBbL-=01ZTCJM65soDZrig%B3`26!=+1>)^F?o|KD9t{o@B`d+M23`;Y2Z-g~TG zY<_a$SE_!j|G9M;3$Lml%Z>~UWAB$;acaihjbrFUyh*G>$UmfUY)H05h6fwRCUsxB zx!C?jv2uxQ@0V>73t*Cy8^xL@C4S2q$9B}p`(oc64WoSWy+P%@?LXSqID&`zMm33r z&-L>(A^0_##X_|5<4t2|?ypT^TZ$!W?9w#0K4anj=LR*U`nEKUC1UF`wrLVev?h_E TX)JW3|G6fy(A@U9n#KMf7dXz^ diff --git a/tests/test_nonsequential/exp_set_B/non-sequential-SCNN-example_3.ipynb b/tests/test_nonsequential/exp_set_B/non-sequential-SCNN-example_3.ipynb deleted file mode 100644 index f8b7f3ff..00000000 --- a/tests/test_nonsequential/exp_set_B/non-sequential-SCNN-example_3.ipynb +++ /dev/null @@ -1,1521 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import torch, random\n", - "import torch.nn as nn\n", - "import sinabs.layers as sl\n", - "from tqdm.notebook import tqdm\n", - "\n", - "from tonic.datasets.dvsgesture import DVSGesture\n", - "from tonic.transforms import ToFrame\n", - "from torch.utils.data import DataLoader\n", - "from torch.nn import CrossEntropyLoss\n", - "from torch.optim import Adam\n", - "\n", - "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential\n", - "\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "torch.backends.cudnn.deterministic = True\n", - "random.seed(1)\n", - "torch.manual_seed(1)\n", - "torch.cuda.manual_seed(1)\n", - "np.random.seed(1)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Loading Data" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "batch_size = 3\n", - "num_workers = 1\n", - "epochs = 30\n", - "lr = 5e-4" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "root_dir = \"../DVSGESTURE\"\n", - "_ = DVSGesture(save_to=root_dir, train=True)\n", - "_ = DVSGesture(save_to=root_dir, train=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "n_time_steps = 50\n", - "to_raster = ToFrame(sensor_size=DVSGesture.sensor_size, n_time_bins=n_time_steps)\n", - "\n", - "snn_train_dataset = DVSGesture(save_to=root_dir, train=True, transform=to_raster)\n", - "snn_test_dataset = DVSGesture(save_to=root_dir, train=False, transform=to_raster)" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The transformed array is in shape [Time-Step, Channel, Height, Width] --> (50, 2, 128, 128)\n" - ] - } - ], - "source": [ - "sample_data, label = snn_train_dataset[0]\n", - "print(f\"The transformed array is in shape [Time-Step, Channel, Height, Width] --> {sample_data.shape}\")" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "snn_train_dataloader = DataLoader(snn_train_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True)\n", - "snn_test_dataloader = DataLoader(snn_test_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=False)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Network Module\n", - "\n", - "We need to define a `nn.Module` implementing the network we want the chip to reproduce." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "device: NVIDIA RTX A4000\n" - ] - } - ], - "source": [ - "if torch.cuda.is_available():\n", - " device = torch.device('cuda:0')\n", - " print('device: ', torch.cuda.get_device_name(0))\n", - "else:\n", - " device = torch.device('cpu')" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "class SNN(nn.Module):\n", - " def __init__(self) -> None:\n", - " super().__init__()\n", - "\n", - " self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False)\n", - " self.iaf1 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - " self.pool1 = nn.AvgPool2d(2,2)\n", - " self.pool1a = nn.AvgPool2d(6,6)\n", - "\n", - " self.conv2 = nn.Conv2d(10, 10, 2, 1, bias=False)\n", - " self.iaf2 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - " self.pool2 = nn.AvgPool2d(3,3)\n", - "\n", - " self.conv3 = nn.Conv2d(10, 10, 3, 1, bias=False)\n", - " self.iaf3 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - " self.pool3 = nn.AvgPool2d(2,2)\n", - "\n", - " self.flat = nn.Flatten()\n", - "\n", - " self.fc1 = nn.Linear(810, 100, bias=False)\n", - " self.iaf4 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - "\n", - " self.fc2 = nn.Linear(100, 100, bias=False)\n", - " self.iaf5 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - "\n", - " self.fc3 = nn.Linear(100, 100, bias=False)\n", - " self.iaf6 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - "\n", - " self.fc4 = nn.Linear(100, 11, bias=False)\n", - " self.iaf7 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - "\n", - " self.merge_fc = sl.Merge()\n", - " self.merge_conv = sl.Merge()\n", - "\n", - " def detach_neuron_states(self):\n", - " for name, layer in self.named_modules():\n", - " if name != '':\n", - " if isinstance(layer, sl.StatefulLayer):\n", - " for name, buffer in layer.named_buffers():\n", - " buffer.detach_()\n", - "\n", - " def init_weights(self):\n", - " for name, layer in self.named_modules():\n", - " if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear):\n", - " nn.init.xavier_normal_(layer.weight.data)\n", - "\n", - " def forward(self, x):\n", - " \n", - " con1_out = self.conv1(x)\n", - " iaf1_out = self.iaf1(con1_out)\n", - " pool1_out = self.pool1(iaf1_out)\n", - " pool1a_out = self.pool1a(iaf1_out)\n", - "\n", - " conv2_out = self.conv2(pool1_out)\n", - " iaf2_out = self.iaf2(conv2_out)\n", - " pool2_out = self.pool2(iaf2_out)\n", - "\n", - " merged_conv_out = self.merge_conv(pool1a_out, pool2_out)\n", - "\n", - " conv3_out = self.conv3(merged_conv_out)\n", - " iaf3_out = self.iaf3(conv3_out)\n", - " pool3_out = self.pool3(iaf3_out)\n", - "\n", - " flat_out = self.flat(pool3_out)\n", - " \n", - " fc1_out = self.fc1(flat_out)\n", - " iaf4_out = self.iaf4(fc1_out)\n", - "\n", - " fc2_out = self.fc2(iaf4_out)\n", - " iaf5_out = self.iaf5(fc2_out)\n", - "\n", - " fc3_out = self.fc3(iaf5_out)\n", - " iaf6_out = self.iaf6(fc3_out)\n", - "\n", - " merge_fc_out = self.merge_fc(iaf4_out, iaf6_out)\n", - "\n", - " fc4_out = self.fc4(merge_fc_out)\n", - " iaf7_out = self.iaf7(fc4_out)\n", - "\n", - " return iaf7_out" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "snn = SNN().to(device)" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "snn.init_weights()" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "optimizer = Adam(snn.parameters(), lr=lr, betas=(0.9, 0.999), eps=1e-8)\n", - "loss_fn = CrossEntropyLoss()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Define train and test" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "def train(dataloader, model, loss_fn, optimizer, epochs, test_func, dataloader_test):\n", - " epochs_y = []\n", - " epochs_x = []\n", - " epochs_acc = []\n", - " model.train()\n", - "\n", - " for e in range(epochs):\n", - " losses = []\n", - " batches = []\n", - " batch_count = 0\n", - " train_p_bar = tqdm(snn_train_dataloader)\n", - "\n", - " for X, y in train_p_bar:\n", - " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", - " X = X.reshape(-1, 2, 128, 128).to(dtype=torch.float, device=device)\n", - " y = y.to(dtype=torch.long, device=device)\n", - "\n", - " # forward\n", - " pred = model(X)\n", - "\n", - " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", - " pred = pred.reshape(batch_size, n_time_steps, -1)\n", - "\n", - " # accumulate all time-steps output for final prediction\n", - " pred = pred.sum(dim = 1)\n", - " loss = loss_fn(pred, y)\n", - "\n", - " # gradient update\n", - " optimizer.zero_grad()\n", - " loss.backward()\n", - " optimizer.step()\n", - "\n", - " # detach the neuron states and activations from current computation graph(necessary)\n", - " model.detach_neuron_states()\n", - "\n", - " train_p_bar.set_description(f\"Epoch {e} - BPTT Training Loss: {round(loss.item(), 4)}\")\n", - "\n", - " batch_count += 1\n", - " losses.append(loss.item())\n", - " batches.append(batch_count)\n", - "\n", - " epochs_y.append(losses)\n", - " epochs_x.append(batches)\n", - "\n", - " acc = test_func(dataloader_test, model)\n", - " print(f'Epoch {e} accuracy: {acc}')\n", - " epochs_acc.append(acc)\n", - "\n", - " return epochs_x, epochs_y, epochs_acc\n" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [], - "source": [ - "def test(dataloader, model):\n", - " correct_predictions = []\n", - " with torch.no_grad():\n", - " test_p_bar = tqdm(dataloader)\n", - " for X, y in test_p_bar:\n", - " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", - " X = X.reshape(-1, 2, 128, 128).to(dtype=torch.float, device=device)\n", - " y = y.to(dtype=torch.long, device=device)\n", - "\n", - " # forward\n", - " output = model(X)\n", - "\n", - " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", - " output = output.reshape(batch_size, n_time_steps, -1)\n", - "\n", - " # accumulate all time-steps output for final prediction\n", - " output = output.sum(dim=1)\n", - "\n", - " # calculate accuracy\n", - " pred = output.argmax(dim=1, keepdim=True)\n", - "\n", - " # compute the total correct predictions\n", - " correct_predictions.append(pred.eq(y.view_as(pred)))\n", - "\n", - " test_p_bar.set_description(f\"Testing Model...\")\n", - " \n", - " correct_predictions = torch.cat(correct_predictions)\n", - " return correct_predictions.sum().item()/(len(correct_predictions))*100" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Training loop (HPO)" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "81911e3a7aa94b6299e5943b8f751b65", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - " 0%| | 0/359 [00:00" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "y_avg = []\n", - "for y in epochs_y:\n", - " y_avg.append(np.mean(y))\n", - "\n", - "plt.plot(np.arange(len(epochs_x)), y_avg, marker = '.')\n", - "plt.xlabel('epoch')\n", - "plt.ylabel('average loss')\n", - "plt.ylim(0,)\n", - "plt.xticks(np.arange(len(epochs_x)))\n", - "for i, txt in enumerate(y_avg):\n", - " if i%5 == 0:\n", - " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", - " else:\n", - " pass\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAG2CAYAAACZEEfAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABqLElEQVR4nO3dd1RUR8MG8GfpRUDpINWKir2iRmMJthhbLLGXaEww1lgwMTExippoiia2L2KNLVFjib1GgwgodhGwoEgRkaW33fn+4GXjStuVpbg8v3P2nHD3zp25uNn7MHfujEQIIUBERESkpXQqugFEREREZYlhh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLRahYad8+fPo2/fvnB0dIREIsH+/fuV3hdC4Msvv4SDgwOMjY3RvXt3hIeHK+2TmJiIESNGwNzcHNWrV8eECROQmppajmdBRERElVmFhp20tDQ0bdoUv/zyS6HvL1++HD///DPWrl2LwMBAmJqaokePHsjMzFTsM2LECNy6dQsnTpzAoUOHcP78eUyaNKm8ToGIiIgqOUllWQhUIpFg37596N+/P4C8Xh1HR0fMmjULn332GQBAKpXCzs4OmzZtwrBhw3Dnzh00bNgQQUFBaNWqFQDg6NGj6N27N548eQJHR8eKOh0iIiKqJPQqugFFefDgAWJjY9G9e3fFNgsLC7Rt2xYBAQEYNmwYAgICUL16dUXQAYDu3btDR0cHgYGBGDBgQKHHzsrKQlZWluJnuVyOxMREWFlZQSKRlN1JERERkcYIIZCSkgJHR0fo6BR9s6rShp3Y2FgAgJ2dndJ2Ozs7xXuxsbGwtbVVel9PTw+WlpaKfQrj5+eHr7/+WsMtJiIioorw+PFjODk5Ffl+pQ07ZcnX1xczZ85U/CyVSuHi4oLHjx/D3Ny8AltGREREqkpOToazszPMzMyK3a/Shh17e3sAQFxcHBwcHBTb4+Li0KxZM8U+8fHxSuVyc3ORmJioKF8YQ0NDGBoaFthubm7OsENERPSGKWkISqWdZ8fd3R329vY4deqUYltycjICAwPh5eUFAPDy8kJSUhJCQkIU+5w+fRpyuRxt27Yt9zYTERFR5VOhPTupqamIiIhQ/PzgwQOEhobC0tISLi4umD59Or799lvUrVsX7u7uWLBgARwdHRVPbDVo0AA9e/bExIkTsXbtWuTk5GDKlCkYNmwYn8QiIiIiABUcdoKDg9GlSxfFz/njaMaMGYNNmzZhzpw5SEtLw6RJk5CUlISOHTvi6NGjMDIyUpTZvn07pkyZgm7dukFHRweDBg3Czz//XO7nQkRERJVTpZlnpyIlJyfDwsICUqmUY3aIiIjeEKpevyvtmB0iIiIiTWDYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQERGRVmPYISIiIq3GsENERFQMmUyGBQsWwN3dHcbGxqhduzYWLVoEIYRin7Fjx0IikSi9evbsWeKxo6OjMXLkSFhZWcHY2BiNGzdGcHCw4v29e/fC29sbVlZWkEgkCA0NLYtT1Hp6Fd0AIiKiymzZsmVYs2YNNm/ejEaNGiE4OBjjxo2DhYUFpk6dqtivZ8+e8Pf3V/xsaGhY7HFfvHiBDh06oEuXLjhy5AhsbGwQHh6OGjVqKPZJS0tDx44dMWTIEEycOFHzJ1dFMOwQEREV499//0W/fv3Qp08fAICbmxt27NiBy5cvK+1naGgIe3t7lY+7bNkyODs7KwUkd3d3pX1GjRoFAHj48OFrtp4A3sYiIiIqVvv27XHq1Cncu3cPAHDt2jVcuHABvXr1Utrv7NmzsLW1Rf369fHxxx/j+fPnxR73wIEDaNWqFQYPHgxbW1s0b94cGzZsKLPzqMrYs0NERFSMefPmITk5GR4eHtDV1YVMJsPixYsxYsQIxT49e/bEwIED4e7ujsjISMyfPx+9evVCQEAAdHV1Cz3u/fv3sWbNGsycORPz589HUFAQpk6dCgMDA4wZM6a8Tq9KYNghIiIqxu7du7F9+3b8/vvvaNSoEUJDQzF9+nQ4OjoqQsmwYcMU+zdu3BhNmjRB7dq1cfbsWXTr1q3Q48rlcrRq1QpLliwBADRv3hw3b97E2rVrGXY0jLexiIiIijF79mzMmzcPw4YNQ+PGjTFq1CjMmDEDfn5+RZapVasWrK2tERERUeQ+Dg4OaNiwodK2Bg0aICoqSmNtpzwMO0RERMVIT0+Hjo7y5VJXVxdyubzIMk+ePMHz58/h4OBQ5D4dOnRAWFiY0rZ79+7B1dW1dA2mAhh2iIiIitG3b18sXrwYhw8fxsOHD7Fv3z6sXLkSAwYMAACkpqZi9uzZuHTpEh4+fIhTp06hX79+qFOnDnr06KE4Trdu3bB69WrFzzNmzMClS5ewZMkSRERE4Pfff8f69evh4+Oj2CcxMRGhoaG4EHQVAHDp6g2EhoYiNja2nM5eSwgSUqlUABBSqbSim0JERJVMcnKymDZtmnBxcRFGRkaiVq1a4vPPPxdZWVlCCCHS09OFt7e3sLGxEfr6+sLV1VVMnDhRxMbGKh3HydlFjP90tnialK7YdvDgQeHp6SkMDQ2Fh4eHWL9+vVIZf39/AaDA66uvvirz834TqHr9lgjx0hSQVVRycjIsLCwglUphbm5e0c0hIiItsysoCr57b0AuAB0JsGRAY7zb1BEJKVlISM17PUvNRkJKFp6lZim2xyZn4mlSptKxdCTAxXld4WBhXEFnU3moev1m2AHDDhERlZ0YaQY6LD0NuQavtkv6e2J4O47tUfX6zUfPiYiIytCDhLQig46xvi6szQxgXc1Q8bKpZgAbs7z/lkiAT7ZfKVD+ywM3kZqdiw871oKOjqTsT+INx7BDRERUhu7FphTYpiMBTszojNq21Uos7zewMebvvQmZENCRAA0dzHHzaTKW/H0X/4QnYMXgprA1NyqLpmsN3sYCb2MREVHZuPVUikFr/kVmjhwS5I0u1pVIsGSgJ4a2dlH5ODHSDDxMSIebtQnszY2wM+gxvj54C5k5cliaGuC795ugWwO7MjuPyopjdtTAsENEVDXESDPwICEN7tamZT7A90VaNvquvoAnLzLQuZ4NFg/wxOPEDLhZm2ik7oj4FHy6IxR3YpIBAGPbu2FeLw8Y6Re+PIU2YthRA8MO0ZurPC9e9GZ79Ykov4GN1epdUYdMLjDW/zL+CU+Ai6UJDk7pCAsTfY3Xk5Urw7IjYdh48QEAwMPeDD9/0Bz17Mw0XldlpOr1m5MKEtEba1dQFDosPY3hGwLRYelp7AriNPtUuBhpBub9L+gAgFwA8/feRIw0o0zq+/54GP4JT4Cxvi7WjWpZJkEHAAz1dPFl34bwH9ca1tUMcDc2BX1XXcDWS4/Avoz/MOwQaVCMNAP/RiaU2Rco/SdGmqH4Kx0o+4uXNpDJZFiwYAHc3d1hbGyM2rVrY9GiRUoXRSEEvvzySzg4OMDY2Bjdu3dHeHh4iceOjo7GyJEjYWVlBWNjYzRu3BjBwcGK91NTUzFlyhQ4OTnB2NgYDRs2xNq1a8vkPAtz+k48Xr32y4TAw4R0jdd15EYM1pyNBAAse78JGjiU/R2DLvVtcWRaJ3SuZ4OsXDkW7L+JiVtCkJiWXeZ1vwkYdog0ZMflKLRnL0O5Kexx3rK6eBXlTQu3y5Ytw5o1a7B69WrcuXMHy5Ytw/Lly7Fq1SrFPsuXL8fPP/+MtWvXIjAwEKampujRowcyMzOLPO6LFy/QoUMH6Ovr48iRI7h9+zZWrFiBGjVqKPaZOXMmjh49im3btuHOnTuYPn06pkyZggMHDpTpOQNA5LNUfHcsrND30rNzNVpXeFwKPttzDQDwYUd3vNfUUaPHL46NmSH8x7bGgncbwkBXByfvxKHXT+fxb0TCG/dZ1TSO2QHH7FDpRb9IR4dlZ5S26UokuDCvC8eRlJGHCWl4+/uzBbafntUZtWxKfpy3tMpz/IemvPvuu7Czs8Nvv/2m2DZo0CAYGxtj27ZtEELA0dERs2bNwmeffQYAkEqlsLOzw6ZNmzBs2LBCjztv3jxcvHgR//zzT5F1e3p6YujQoViwYIFiW8uWLdGrVy98++23GjrDgp4mZeD9Nf/iqTQTNasbI0aaoRSSjfV1sWZkC7xd37bUdSVn5qDf6ot4kJAGr1pW2DqhDfR0K6ZP4Wa0FNN2XkXkszQAUDwJ9qZ8VlXFMTtE5UQIgW8P3ymwvbx7GaqaVo3r49Gydwu8ug0ajcS0bERGRmLAgAGwsbGBubk5hgwZgri4uGKPmZKSgunTp8PV1RXGxsZo3749goKCFO/n5ORg7ty5aNDIEx90qIeo1aORcGgFspOfvxG30Nq3b49Tp07h3r17AIBr167hwoUL6NWrFwDgwYMHiI2NRffu3RVlLCws0LZtWwQEBBR53AMHDqBVq1YYPHgwbG1t0bx5c2zYsKFA3QcOHEB0dDSEEDhz5gzu3bsHb2/vMjjTPIlp2Rj1WyCeSjNRy8YUB6Z0wMV5XbFjYjucmtkZb9W1RkaODB9uDsafIU9KVZdcLjBzVygeJKTB0cIIq4c3r7CgAwCeNS1w8NOO6Ncsr2cpP9/JBeC790al/6xqGsMOaYybmxskEkmBV/4Kvq9z8Vm4cGGB43l4eCjt8/bbbxfYZ/LkyWV2nq/65UwEjtwsuAKxjgRwszYpt3ZUJcmZObAfvRJOPluxfG8ADgbcgt+GnQCADKfWGPDTKXTt/g4kEglOnz6NixcvIjs7G3379oVcLi/yuB9++CFOnDiBrVu34saNG/D29kb37t0RHR0NAEhPT8eVK1fQrO94OIz5CTb95yMnMRrP9i6CTAj8cy+hXM7/dc2bNw/Dhg2Dh4cH9PX10bx5c0yfPh0jRowAAMVK2nZ2yvO12NnZFbvK9v3797FmzRrUrVsXx44dw8cff4ypU6di8+bNin1WrVqFhg0bwsnJCQYGBujZsyd++eUXdOrUqQzOFEjNysVY/8uIfJYXPrZNaAuraoZwsDCGV20r1Latht/GtMaA5jWRKxeYteca1p6LfO1BvatOR+DknXgY6Olg7aiWsKpmqOEzUp+JgR6GtnYusF0ugImbg/FHyBOkZmn2Nl5lxbBDGhMUFISYmBjF68SJEwCAwYMHIy0tDd7e3mpffACgUaNGSse9cOFCgX0mTpyotM/y5cvL5BxftfXSI3x/PO+v5L5NHKAr+W/adj0dCZIzqsYXSXnbcP4+UiWmqF/LGTPea4N32zVE7M0AuLi5w92zNe5eC0bUo0fwXfozGjdujMaNG2Pz5s0IDg7G6dOnCz1mRkYG/vzzTyxfvhydOnVCnTp1sHDhQtSpUwdr1qzJ28nABPXGLUMA6kPfygmGNT1g+c5kZMdGIDc5HnP/vI7Ze67haVLl/Kt59+7d2L59O37//XdcuXIFmzdvxvfff68USl6HXC5HixYtsGTJEjRv3hyTJk3CxIkTlQYgr1q1CpcuXcKBAwcQEhKCFStWwMfHBydPniztaRWQmSPDxM3BuP5ECktTA2yZ0BaO1QveTjbQ08GKwU0xqVMtAMDSI3ex6NAdyNVcxOr03Tj8eCrve2Bxf080cape6nPQFHdrUxS2msTNp8n4bM81tPr2BKbuuIozd+ORIyv+u/hNxrBDGmNjYwN7e3vF69ChQ6hduzY6d+6Mixcv4uHDh9i0aZPKF598enp6Sse1trYusI+JiYnSPuUx9urAtaf48q+bAIBPu9bBquEtcGFeF2yb0Aat3KojWybw0dZgSDNyyrwtVcmzlCz8diFvTpHPvOtBT1cH2dnZ2LZtGz6a+CH2+XSEo1neSjhjN19FQORzAICRkRF0dHQKDcsAkJubC5lMBiMj5Wn3jY2NceHCBQQ9TETvn/7B4esx0NORoKenPXQkgDwrHYAEnm4OEAD2hDzB29+fhd/fdyBNr1z/9rNnz1b07jRu3BijRo3CjBkz4OfnBwCwt7cHgAI9rnFxcYr3CuPg4ICGDRsqbWvQoAGiovIG6WdkZGD+/PlYuXIl+vbtiyZNmmDKlCkYOnQovv/+e02eInJlckzdcRUB95/D1EAXm8a1Rp1ilmTQ0ZFgfu8G+KJPAwDAxosPMHXnVWTlylSq70FCGqbtDIUQwKh2rhjcqmBPSkVysDCG38DGij/EdCUSzOlRH7PeqYda1qbIzJHjwLWnGLcpCO2WnMLCA7cQ+jhJ6x5bZ9ihMpF/8Rk/fjwkEgmysrIgkUhgaPhf125JF5984eHhcHR0RK1atTBixAjFF+jLtm/fDmtra3h6esLX1xfp6WU7VuZsWDxm7vrvC27mO/UA5H2xdKxrg/WjWqNmdWM8fJ6OmbtC1f5LkYr2y5kIpGfL0NTJAj0a5V2A9+/fj6SkJIwdOxb2Fkb4Y8EY6Bka4/Gx/8OodeexNzASn332GWQyGWJiYgo9rpmZGby8vLBo0SI8ffoUMpkM27ZtQ0BAAG5HPsLQdQGITsqAq5UJ/vy4PdaObIlT09ujxs3dGDB4CI7O6Yl9n7RHG3dLZOfKse78fby1/DTWnYtEZo5qF86ylp6eDh0d5a99XV1dRe+qu7s77O3tcerUKcX7ycnJCAwMhJeXV5HH7dChA8LClJ92unfvHlxd81blzsnJQU5OTrF1a4IQAr57b+D47TgY6Olgw5hWKveyfPhWLfw0rBn0dSU4dD0G4/yDkJJZfFhNy8rFpC3BSMnMRSvXGljwbsNi968oQ1u74MK8LtgxsR0uzOuCT7rUwafd6uLUrM74y6cDxrZ3g5WpAZ6nZWPTvw/R/5eL6LbiHH4+FY6o53nfpaV5mqsyPAnGp7HAp7HKwu7duzF8+HBERUXB0dERz549Q506dTBu3DgsWbIEQgjMmzcPq1evxqRJk7Bu3bpCj3PkyBGkpqaifv36iImJwddff43o6GjcvHkTZmZ5M4SuX78erq6ucHR0xPXr1zF37ly0adMGe/fuLZNzC3mUiBH/F4jMHDn6NnXET0ObFbrq8M3ovDVxsnLlmNatLmb8LxDR63ucmI6uK84iRyaw/cO26FAnr5evR48eMDAwwMGDBxX7Hvr7CIaPm4iU+KeARIL23u8h81kU2rRp899tqVdERkZi/PjxOH/+PHR1deHZpBkS9awQE3kbNSeuxcAWNfFNP09UM9RDTk4OBg0ahCdPnuDs2bOK7w4hBM6GPcPSI3cRFpe3AKSjhRFmvFMPA1s4QVdDK1S/zszRY8eOxcmTJ7Fu3TrYONfGqYuB+H7BZ/hwwngsW7YMQN7j6UuXLsXmzZvh7u6OBQsW4Pr167h9+7ai16tbt24YMGAApkyZAiDvFnb79u3x9ddfY8iQIbh8+TImTpyI9evXK8YDvf3220hISMDq1avh6uqKc+fO4eOPP8bKlSvx8ccfl/r3IYTAkr/vYMM/D6AjAdaMbKkIw+r4J/wZJm8NQVq2DA0dzLFpfGvYmhVcZFMIAZ/fr+DvG7GwNTPEoU87vtGLcebI5LgQkYB9V6Jx/HYsMnP+C6GuliaISkyHQN5TXSPbuSj+3yvJxYgEbLsUVWZPgnG5CDUw7GheYRef48eP4+OPP8aDBw+go6ODDz74ALdv3y724vOqpKQkuLq6YuXKlZgwYUKh+5w+fRrdunVDREQEateurZHzyXcnJhlD1wUgOTMXnevZYMPoVjDQK7qD9M+QJ5j1vzk3/m90K3RvWPUW6tOkmbtDsfdKNDrUscL2D9sBAB49eoRatWph79696Nevn9L+MrnAZ1v/wZ9XY6BjVA3P14/FgnmzMWfOnGLrSUtLwx8B9/D9hXjc37UYOrmZ2LZnH/o1qwkgr6diyJAhuH//Pk6fPg0rK6sCx5DJBfZdjcbK42F4Ks2bo6aeXTXM7emBrh62kEheP/S87mPvKSkpWLBgAbbv+gPPE55Bt5olTBt2xi/ffYsR7esAyLuIf/XVV1i/fj2SkpLQsWNH/Prrr6hX77+w7ubmhrFjx2LhwoWKbYcOHYKvry/Cw8Ph7u6OmTNnYuLEiYr3Y2Nj4evri+PHjyMxMRGurq6YNGkSZsyYUarfRb5fzkQo5tL57v0mpbqddDNairH+l5GQmg1nS2NsGd8W7tamSvusPReJpUfuQl9Xgp2T2qGlq2Wp2l+ZpGbl4tjNWOwPjcaF8ARoMiRoekoOhh01MOxoVnEXHwBISEiAnp4eqlevDnt7e8yaNQuzZ89W+fitW7dG9+7dFeMMXpWWloZq1arh6NGj6NGjx2ufx6uinqdj0Np/8SwlCy1da2DbhLYwNih5wb2v/rqJzQGPYGaoh7+mdCiXOWDU9SasLxUWm4KeP52HEMBfPh3Q1Lk6gLwn9tatW4fHjx9DT0+vQDkhBFadjsDi/9uD+J1fYPyP+7BuSt8iHwtOzcrFwgO38EfIE8gyUxG3/kN8s9gP82bk9WLkB53w8HCcOXMGNjY2xbY7M0eGrQGPsPpMhGL8Vht3S8zr5QEHC6MCv3chBJIzcvEsNQsJ+a+ULCSkZiMhNQtPXqTjQsTzAvVUN9YvtIfxVXK5QFIh48gGtagJVytTWFczhHU1A1ibGcKmmiGsqxkW+JyX9vOi6c/b9sBH+Hxf3vi5L/o0wIdv1Sr1MR89T8PojZfx6Hk6LE0N4D+2teIz90/4M4zZeBlyASzq74lR7VxLXV9ldfj6U/j8frXA9np21WBuVPwSGMmZObgXl1pg+46J7eBVu+AfCK9D1et3wW+GSkQmk2HhwoXYtm0bYmNj4ejoiLFjx+KLL75Q/CWQ/1fIhg0bkJSUhA4dOigegaSK4e/vD1tbW/Tp06fQ9/MHGJ8+fRrx8fF47733VD52amoqIiMjMWrUqCL3CQ0NBZA3aFJT4pMzMfK3QDxLyYKHvRk2jmmtUtABgC/ebYjbMckIevgCH20NwT6fDqhmWHn+13tTJsf7/ngYhAB6NrJXXHTkcjn8/f0xZsyYAkHH398fDRo0gI2NDSxjApH+93cwb90Pp2L0MGlrCH4Z3gLv9vJWuh2zZtufWHM2As/1rCFPioFO0HY0b9wIs6Z8BCAv6Lz//vu4cuUKDh06BJlMpngk29LSEgYGBgXabaSvi4mdamFIK2esORcJ/4sPcPlBIgb++q/SfjWrG0MuBJ6nZiP7NZ6KKSzAqOPPK9FFvmdqoAtrs7zgk5kjw62neatsSwC8Xd8GDR1V/yPx9tNknA17prHbGoeuP8UX+/OCjk+X2hoJOgDgamWKPz9uj3H+QbgRLcWw9ZeweIAndCUSLPjrJuQCGNzSCSPbVr7/VzSphWuNvIH4L3WL6Eok2Dy+TYlBNUaagQ5LTxcoWxFTclTqnp0lS5Zg5cqV2Lx5Mxo1aoTg4GCMGzcOixcvxtSpUwHk3V/28/NTur9848YNpfvLJWHPjubI5XK4u7vjgw8+wNKlS5Xee/niExAQgGnTpmHs2LFYsWKFYp9XxwJ89tln6Nu3L1xdXfH06VN89dVXCA0Nxe3bt2FjY4PIyEj8/vvv6N27N6ysrHD9+nXMmDEDTk5OOHfunEbOSZqeg6HrA3A3NgUulib4Y7KX2vfm41My0XfVBcQlZ6GXpz1+HdFCI133pVXYl5GOBLg4r2ul6uG5EvUCA3/9FzoS4PiMTqhjmzde6/jx4+jRowfCwsKUbrMAeXPKbNq0CYmJiXBzc8PkyZPh2WM4Pt1xFVm5cjR3qY7LfsPRc8BQfPP1Qhy8HoOFP/4fnp/dBFnKc1ha1sCwIYOxePFiWFhYAAAePnwId3f3Qtt45swZvP322yWey9OkDCz5+zYOXS963hoAMDPSU/SsWJsZ/K/HxRD6ujpYfvSu0q0FHQmwbUJb2JiVPLfLs5QsjPwtUOnfXCIBxrRzQ2auDAmpWXiWmo2ElCw8S81Cdm7ZP448oq0Lenrao7WbJYz0VfsjAgDO3XuGDzcHIUcmMLytCxb399T4/1epWbn4eFsI/glXnkPJqYYxTs7srFZ731S7gqIwf+9NyISArkSCJQM9VQ6opSmrCq24jVVWU5u/imFHc9S9+Lx6v/7VsQDDhg3D+fPn8fz5c9jY2KBjx45YvHixYizO48ePMXLkSNy8eRNpaWlwdnbGgAED8MUXX2jk3zI9OxejfruMkEcvYGtmiD8mt4eL1ev9VRLy6AWGrQ9AjkxgTs/6+OTtOqVuX2n9G5mA4RsCC2zfOr4N3qpX/O2Z8iKEwAcbLuHS/UQMbumE7wY3LdXxgh8mYsLmoqcE6OVpj6UDm5TZKtVA0b/3Rf090dXDFlamBsVeREt7AVG1vBACKVm5ittoFyMS8NOpgouC9vS0UykcxyRl4OitoicSNdTTQWs3S3Ssa4236lqjgb15kbfmQh69wMj/C0RGjgzvNnHAT8Oaa2zw96uinqeh03dnlbZVxj8KylKMNAMPE9LhZm2i9jmXpmxJtCLsLFmyBOvXr8fx48dRr149XLt2Dd7e3li5ciVGjBiB+/fvo3bt2rh69SqaNWumKNe5c2c0a9YMP/30U6HHzcrKQlZWluLn5ORkODs7M+xUEpVl/Eh2rhwTtwTj3L1nMDfSw+7JXvCwL93nI39sgY4E2DSuDTpVcKC4/jgJ7/1yscD2t+pa47cxrYsdfF1ezt97htEbL8NAVwdnZr+NmoVMDqeufyMSMPz/CoYN354emNS5Vpn3uhXVva/OwM3SXkBep3xp211YeYkkL2BeeZSE2GTlxUatTA3QoY61Ivw4WOStbXX+XgK+PXwbKZm56FTPBv9XwoMCpVVUONXk2BN6PVoxZmfevHlITk6Gh4cHdHV1IZPJsHjx4lJPbe7n54evv/667BpOr62yjB+R/W/6+HP3nsFYXxf+49qUOugAwPA2Lrj+WIpdwY/x6Y6rOPRpRzhbVtySElsvPVL6WUcCSCQS/BOegE+2h+CXES1gqFdx3fRCCMUTNiPbuWok6ADIG2xSiCbO1cvl9mL+RG+v9q6oE1ocLIxL9cfA65QvbbuLKj+0tQuEEIh8lop/whPwT3gCLt1/judp2Thw7SkOXHsKALCpZoCE1GzFLTwXS2OsHdmizEN5/izElWHsCb2eSh12Xp7avFGjRggNDcX06dPh6OiIMWPGvPZxfX19MXPmTMXP+T07VLFipBmY9+cNpQXr5u+9iU71bMq1h0cIga8O3MTBa0+hryvB2lEt0dK1hkaOLZFI8HW/Rrgbm4xrT6SYtDUEez9ur/JgZ026/TQZf1zJW/xw/eiWMDPUh5u1CcLjUjFxSzBO3onHx9uu4NcRLSpsXMKRm7G4ES2FqYEufLpobhqBynDxGtraBZ3q2ZRZ935ZKW27iyovkUhQx9YMdWzNMK6DO7Jz5bga9QIXIvLCz7XHSXiWmq10rCcvMiDNyIGJQdleyjQRTqliVeqw8/LU5gDQuHFjPHr0CH5+fhgzZozS1OYvP3kTFxendFvrVYaGhkoz+VLlsP3SowLzOeSvHF5eXyox0gwsP3oX+64+hUQCrBzSDJ01fKvJSF8Xa0a2RN9VF3AnJhm+e6/jh6HNynXAcv4EbEIAfZo4wLvhf5OvOVgY47cxrfHhliCcvhuPydtCsHZky3IPPLkyOb4/nter8+FbtTS6sGJluXiVtnemopRHr5KBng7a1rJC21pWmOVdHyduxWHi1mClfeQC5fb98KaGU8pT8Tfki1FWU5tT5bP/ajR+ORNZYLsE5bdy+K6gKLT3O419V/O6zPs3c0Tfpo5lUpdjdWP8MqIFdHUk2B/6FP4XH5ZJPUU5d+8ZLkQkwEBXB3N7eBR4v2Nda2wc0xpG+jo4G/YMk7aGlPuSB39eeYL7z9JQw0QfH75V+BNQpfHqFPqV8XF7+o+nk3mBBS3Luzcuf8V0Bp03T6UOO3379sXixYtx+PBhPHz4EPv27cPKlSsxYMAAAHndntOnT8e3336LAwcO4MaNGxg9ejQcHR3Rv3//im38G6481zLZe+UJZu4OhQDQ2rUGdF/6QhPIu91S1l69hQYAB0JjyvT829WywvzeeYsPLv77Dg5ff1ouv/NcmRxL/r4DABjT3rXIp8va17GG/9g2MNbXxfl7zzBxS3C5BZ7MHBl+PJn31I9PlzowK2HystfFi9ebo7AFLXkriVRVqW9jrVq1CgsWLMAnn3yC+Ph4ODo64qOPPsKXX36p2GfOnDlIS0vDpEmTFFObHz16VOU5dqig8hwk/EfIE8z+4xqEAD5o44zF/RsjLiUTDxPSsf9qNHYFP8asPddwZNpbZfqldvJ2XIXcQhvfwQ03niRhf+h/s5SW9e98T8gT3ItLhYWxPqZ0KX7yTa/aVvAf1xrjNwXhn/AETNgchP8brfqEiq9r26VHiJFmwsHCCCO1eHZaUg9vJdHrqtSPnpcXzrPzH008Equq3cGPMffP6xAib1KxRf08lebUyMqV4f01AbgRLUVrtxrYMbFdkVP8l8aTF+nou+oiXqQrD34sq/N+1YNnaeiy4my51J2WlYvO351FQmoWFrzbEBM6qnZ76PKDRIz1v4z0bBm8alnht7GtymxQaEpmDjotP4MX6TlYNqhyzuZMRJWDqtfvSn0bi8rfg4Q0paAD/NfDoUm7gqIUQWdUO1d829+zwORhhnq6WD28OaoZ6iHo4YtCJzMrreTMHEzYFIwX6dlwsDBSjAkozy7ymOSCt63K4ncOAOvO30dCahZcrUzUWs+njbsltoxvA1MDXQTcf47xm4KQnp2r8fYBwIZ/HuBFeg5q2ZhiUAunMqmDiKoWhh1S8urKvvl2XH4EaXrp1t7J93tgFOb+eQNCAGO8XPFNv0ZFPonkamUKv4GNAQCrz0TgwitTtpdGrkyOKb9fRVhcCmzNDPHnx+1xcV7Xch+wmv8Y9MvKYmB2rDQT68/nDQKf29ND7blJWrlZYsuENqhmqIdL9xMx1j8IaVmaDTzPU7Pw2z/3AQCfedcvk548Iqp6+E1CSmSvdOvkX4MPXItBt5VnceDaU5Tmzuf2wEeYv+8GAGBcBzcsfK/ooJOvb1NHfNDGBUIA03eF4llKVrH7qyJvLp1bOP+/SQN/G9MajtWNK2TA6qsDL4G8gdkXC1nZujRWnghDZo4cLV1roJenfckFCtHSNS/wmBnqKW5tpWow8PxyJhJp2TI0rmnx2m0kInoVww4p+TMkb+XjFi7VsWNiO/zr2xV7Jnuhjm01JKRmY+qOqxjrH4THierfYtka8BCf78tbnXhCR3d8+W5DleeW+apvQ9S3M0NCahZm7AqF/NV7bWr67cIDbA+MgkQC/DSsGRo7WZTqeKX18mPQo/93e8l373UERGom8Nx+mow9IXkTCH7ep0Gp5vRp4VIDWz9sCzOjvNuLYzZeRnhcSqmfJHvyIh3b/jej85ye9SvFQqlEpB0YdkhBLhfYE/IYADDKy1XRw9HazRKHp3bEzHfqwUBXB+fuPcM7P5zD+vORyJWptiLy5n8fYsFftwAAE99yxxdqXnCN9PPG7xjr6+JCRALWnCs4J4+qjt+KxeL/PXr9ee8G8G5UOXoQ8nuVFr7XCH0aOyBHJjB5WwjuP0st1XFfnUCwhUvpZ4Nu5lwd2ybkBZ6QRy/wzg/nMXxDIDosPY1dQVGvdcyfToYjWyaHVy0rdKxjXeo2EhHlY9ghhUv3n+PJiwyYGeqhZyMHpfcM9XQxtVtdHJ3+FtrVskRmjhxL/r6L91ZfxLXHScUe1//iA3x1IC/ofNSpFub3fr2ehbp2Zvi6XyMAwMoT9xD0MFHtY9x4IsW0naGKJ8BUfRqpPOnoSLBiSFM0c64OaUYOxm8KQmJadskFi5A/gaC+rqTQCQRfV1Pn6vhpWDOlbXIBzPvzBpYdvYMzd+MRI81Q6bZneFwK/vzf0hXs1SEiTWPYIYXdwXm9Ou81cyxyHpVaNtWwY2I7LH+/Caqb6ON2TDIG/HoRXx+8VejYjf/75z6+PngbAPDx27Uxr5dHqS5kg1s6YUDzmpDJBabuuIoXaoSAGGkGJmwOQkaODG/VtVZpvFB5cHNzg0QiUXoZG+jBMnQrnGoYI2T7cri4ucPY2Bg2Njbo168f7t69W+wx9+7dC29vb1hZWaGLhx2y4+5jjJeb0gSC69evx9tvvw1zc3NIJBIkJSWp3fbClpAQANacvY9xm4Lg5Xcazb45gaHrArDwwC3sCorCtcdJyMj+b3LCGGkG5u/Lm9fJu6Edmmug54mI6GWVelJBKj/SjBwcuZm3UvyQVsUviiqRSDCklTO6etji20O3FcsdHLsZi2/6eaJRTXM8SEhDQMRzrDoTAQDw6VIbn3mX/i92iUSCRf09Efo4CQ8S0jD7j2vYMLpVicdNzcrF+E3BiE/JQj27avhlRAvoV5InfYKCgiCT/Xfxv3nzJt555x2MHjEMNRu0xNv/1IOs0dvo3a4xfNrb4euvv4a3tzcePHgAXd3CQ2laWho6duwIt9bdsGHJPFQz1MOUrnWU9klPT0fPnj3Rs2dP+Pr6vlbbC1tQUwKgewM7PEpMQ+SzNEgzchD4IBGBD/7riZNIAHcrU5ga6uJmdLJiQsdGNav2PFdEVDY4qSA4qSCQN2PtF/tvor6dGY5Of0utUHL+3jN8vv8GHicWPjh1atc6mPFOPY32otx6KsWAX/9Fdq68xMnxcmVyTNoagtN342FdzQD7fTrAqUb5raejrunTp+PQoUMIDw+HRCLBP+HPMNY/CDK5wIzu9dDFNgNNmzZFREQEatcueiXwtKxceH2+EzdWjMKXGw/h63F9Ct3v7Nmz6NKlC168eIHq1aur3d5dQVEFFtTMf2w/M0eGyGepuBOTgrsxybgbm4I7Mcl4XkSPXHlN5EhE2kHV6zd7dggAsOd/t7AGt3JSO5R0qmeD49M7Y/HfdxRP0+STAPigrYvGbxc1crTAF30a4Mu/bmHpkTto7VYDTZyqF7rvt4fv4PTdeBjq6WDD6FaVOuhkZ2dj27ZtmDlzpuJ39lZdGyzq54n5+25gxd/XcSr+KNzd3eHsXHwP3Lrz9xVjfd5t7FDsvqVR3BT+Rvq6aORogUaOyk+7PUvJwt4rT+B3RPl2XHmvck9EVUPl6MenCnU3NhnXnkihpyPBgOY1X+sYxga66N244FNNAiiTmYCBvJmXezayR45MYMrvV5GcWXDSw83/PsSmfx8CAH4Y2qzSjwfZv38/kpKSMHbsWKXtSSGHEPPTYDz+4X0cPXIUK/33wMDAoMjjvDyBIADoqzmBoLrUnZ/IxswQ7zVzrPBVrImoamDYIewJznsKpnsDO1hVM3zt4xQ2E3BZXrwkEgmWvd8ENasbIyoxHb57byg9+XP6bhy+Ppj3FNjcnh7oXYa9G5ry22+/oVevXnB0dFTaPmLECFwPDcW7vuugZ+mI4R8MQ1h00XPw5E8g2Mix8t6W5SrWRFReGHaquOxcOfZdzZtIcEjr0q1DVBEXLwtjfawa3hx6OhIcvh6DHZfzbsfdfpqMT3+/CrkAhrZyxuTOtcqsDZry6NEjnDx5Eh9++GGB9ywsLFC/fj3sWjgeXT5ZisyExxg498dCl/C4E/PfBIIlrWpe0V6eTLE8l+ggoqqFY3aquNN345CYlg1bM0N0qmtT6uMVN36jrLRwqYHZPerD78hdLDxwE8mZOdh44QHSsmVoX9sK3w7wrBSPmJfE398ftra26NOn8IHEAGBioIe1I1vAbS4Q9yIVk7eFYPP4NkrrXL08gWBFzwytCgcLY/bmEFGZYs9OFbf7f7ewBrV00tiiixWxvtTEt2qhvl01ZMsElh65i/iULNhUM8CaES0rzSPmxZHL5fD398eYMWOgp/ff3yD379+Hn58fQkJCEBUVhX///Rc+E0bBrJoJrDzaIuD+c3y+7wY8PDywb98+nLv3DP+EJ0AnOxXvOmbh9u28OY7CwsIQGhqK2NhYxbFjY2MRGhqKiIi86QFu3LiB0NBQJCaqP1kjEVFlVvmvAlRm4pIzcTYsHkDeZH1vsriUTITHKy+r8DwtG+k5ml2Vu6ycPHkSUVFRGD9+vNJ2IyMj/PPPP+jduzfq1KmDoUOHwszMDIEBAVg7sSt0JMCekCcICwvDixdJWHI4bxmMZvII9O7SXtFLNGzYMDRv3hxr165VHHvt2rVo3rw5Jk6cCADo1KkTmjdvjgMHDpTTWRMRlQ/Os4OqO8/Or2cjsPxoGFq71cCeye0rujml8m9kAoZvCCywfcfEdvCqbVUBLSofWwIe4sv/rTnW1cMWp+/Gw8xID//M6YLqJkU/rUVEpA1UvX6zZ6eKEkIonsIaXMKMyW+C8n4SrLIY7eWGcR3cAACn7+b10qVm5uLYrdhiShERVS0MO1VU8KMXeJCQBhMDXfR5Ax7JLklVfoz51dmjBYD5e28iRlr4jNZERFUNn8aqonYH5T2i/W4TB5gaasfHoCKeBKsMohILTtrImYiJiP6jHVc5UktqVi4O34gBUPKin2+aqvgYc2GLcVaFW3hERKribawq6O/rMUjPlqGWtSlaulbu5ROoZFX5Fh4RkSrYs1MF7VYs+un8Rky2RyWrqrfwiIhUwbBTxUTEpyL40Qvo6kgwqMXrLfpJlVNVvIVHRKQK3saqYvaE5PXqvF3PBrbmRhXcGiIiorLHsFOF5Mjk+DMkb9FPbZhbh4iISBUMO1XIubBnSEjNgpWpAbp62FZ0c4iIiMoFw04Vkj8weUDzmkqrZBMREWkzXvGqiGcpWYrlBHgLi4iIqhKGnSpi/9Vo5MoFmjpXR317s4puDhERUblh2KkChBCKW1hDWjlVcGuIiIjKF8NOFRD6OAnh8akw1NNB36aOFd0cIiKicsWwUwXsDn4CAOjd2AHmRvoV3BoiIqLyxbCj5TKyZTh47SkAYDBvYRERURXEsKPljtyMQWpWLlwsTdDO3aqim0NERFTuGHa0nGLRz5ZO0NHhop9ERFT1MOxosUfP03DpfiIkEmBQS97CIiKiqolhR4v9EZI3MPmtujZwrM7VsImIqGpi2NFST16kY3vgIwCcW4eIiKo2vYpuAGnerqAozNt7A0Lk/ZyUnlOxDSIiIqpA7NnRMjHSDPi+FHQA4Ku/biFGmlFxjSIiIqpADDtaZvulR5AL5W0yIfAwIb1iGkRERFTBeBtLS6Rk5uCrv25h79XoAu/pSiRwszapgFYRERFVPPbsaIHQx0no8/MF7L0aDR0J0L2BLfKn1NGVSLBkoCccLPg0FhERVU3s2XmDyeQCa89F4ocT95ArF6hZ3Rg/DWuGVm6WiJFm4GFCOtysTRh0iIioSmPYeUPFSjMxY1coAu4/BwC828QBiwc0hoVx3kKfDhbGDDlERERg2HkjHbsVi7l/XkdSeg5MDHTx9XuN8H5LJ0gkXA6CiIjoVQw7b5CMbBm+PXwb2wOjAACNa1rgp2HNUMumWgW3jIiIqPJi2HlD3IlJxtQdVxEenwoA+KhTLczyrg8DPY4xJyIiKg7DTiUWI83Ag2dpCHqYiF/ORiI7Vw4bM0OsHNIUb9W1qejmERERvREYdiqpXUFR8N17Q2mCwG4etlj+fhNYVTOsuIYRERG9YXgPpBLKX/Lh5aAjkQCL+jdi0CEiIlITw04l9CAhrcCSD0IAj55zfSsiIiJ1MexUQu7WpgW2cckHIiKi18OwUwk5WBjD0tRA8TOXfCAiInp9HKBcCcWnZCIxLRsA8NuYVmjoaM6gQ0RE9JoYdiqhK4+SAAAe9mbo1sCuYhtDRET0huNtrEroStQLAEAL1xoV3BIiIqI3n9ph58yZM2XRDnpJyKO8sNPShWGHiIiotNQOOz179kTt2rXx7bff4vHjx2XRpiotK1eGG0+kAICW7NkhIiIqNbXDTnR0NKZMmYI//vgDtWrVQo8ePbB7925kZ2eXRfuqnJvRyciWyWFlagBXKz5qTkREVFpqhx1ra2vMmDEDoaGhCAwMRL169fDJJ5/A0dERU6dOxbVr18qinVXGlUf/jdeRSCQV3BoiIqI3X6kGKLdo0QK+vr6YMmUKUlNTsXHjRrRs2RJvvfUWbt26pak2VimK8Tq8hUVERKQRrxV2cnJy8Mcff6B3795wdXXFsWPHsHr1asTFxSEiIgKurq4YPHiwRhoYHR2NkSNHwsrKCsbGxmjcuDGCg4MV7wsh8OWXX8LBwQHGxsbo3r07wsPDNVJ3eRNCICSKYYeIiEiT1A47n376KRwcHPDRRx+hXr16uHr1KgICAvDhhx/C1NQUbm5u+P7773H37t1SN+7Fixfo0KED9PX1ceTIEdy+fRsrVqxAjRr/BYHly5fj559/xtq1axEYGAhTU1P06NEDmZmZpa6/vD15kYFnKVnQ15WgcU2Lim4OERGRVlB7UsHbt29j1apVGDhwIAwNC1+B29raWiOPqC9btgzOzs7w9/dXbHN3d1f8txACP/74I7744gv069cPALBlyxbY2dlh//79GDZsWKnbUJ7yb2E1crSAkb5uBbeGiIhIO6jds3Pq1Cl88MEHRQYdANDT00Pnzp1L1TAAOHDgAFq1aoXBgwfD1tYWzZs3x4YNGxTvP3jwALGxsejevbtim4WFBdq2bYuAgIAij5uVlYXk5GSlV2XA8TpERESap3bY8fPzw8aNGwts37hxI5YtW6aRRuW7f/8+1qxZg7p16+LYsWP4+OOPMXXqVGzevBkAEBsbCwCws1NeUsHOzk7xXmH8/PxgYWGheDk7O2u03a+LYYeIiEjz1A4769atg4eHR4HtjRo1wtq1azXSqHxyuRwtWrTAkiVL0Lx5c0yaNAkTJ04sdT2+vr6QSqWKV2WYHDE1Kxd3Y/N6mBh2iIiINEftsBMbGwsHB4cC221sbBATE6ORRuVzcHBAw4YNlbY1aNAAUVFRAAB7e3sAQFxcnNI+cXFxivcKY2hoCHNzc6VXRbv2OAlyAdSsbgw7c6OKbg4REZHWUDvsODs74+LFiwW2X7x4EY6OjhppVL4OHTogLCxMadu9e/fg6uoKIG+wsr29PU6dOqV4Pzk5GYGBgfDy8tJoW8oab2ERERGVDbWfxpo4cSKmT5+OnJwcdO3aFUDeoOU5c+Zg1qxZGm3cjBkz0L59eyxZsgRDhgzB5cuXsX79eqxfvx4AIJFIMH36dHz77beoW7cu3N3dsWDBAjg6OqJ///4abUtZY9ghIiIqG2qHndmzZ+P58+f45JNPFOthGRkZYe7cufD19dVo41q3bo19+/bB19cX33zzDdzd3fHjjz9ixIgRin3mzJmDtLQ0TJo0CUlJSejYsSOOHj0KI6M351aQXC5whZMJEhERlQmJEEK8TsHU1FTcuXMHxsbGqFu3brGPold2ycnJsLCwgFQqrZDxO/fiUuD9w3kY6+vixkJv6OmWahUPIiKiKkHV67faPTv5qlWrhtatW79ucXpJ/i2sZs7VGXSIiIg07LXCTnBwMHbv3o2oqCjFrax8e/fu1UjDqhKO1yEiIio7ancj7Ny5E+3bt8edO3ewb98+5OTk4NatWzh9+jQsLLie0+u4wrBDRERUZtQOO0uWLMEPP/yAgwcPwsDAAD/99BPu3r2LIUOGwMXFpSzaqNUS07JxPyENANDcpXrFNoaIiEgLqR12IiMj0adPHwCAgYEB0tLSIJFIMGPGDMUj4aS6/F6dOrbVUN3EoIJbQ0REpH3UDjs1atRASkoKAKBmzZq4efMmACApKQnp6emabV0VEJL/yLkLb2ERERGVBbUHKHfq1AknTpxA48aNMXjwYEybNg2nT5/GiRMn0K1bt7Joo1bj4GQiIqKypXbYWb16NTIzMwEAn3/+OfT19fHvv/9i0KBB+OKLLzTeQG2WI5Pj2uMkAEALhh0iIqIyoVbYyc3NxaFDh9CjRw8AgI6ODubNm1cmDasKbj9NRlauHNVN9FHL2rSim0NERKSV1Bqzo6enh8mTJyt6dqh08m9htXCpAR0dSQW3hoiISDupPUC5TZs2CA0NLYOmVD0hXA+LiIiozKk9ZueTTz7BzJkz8fjxY7Rs2RKmpsq3X5o0aaKxxmm7Ky/17BAREVHZUDvsDBs2DAAwdepUxTaJRAIhBCQSCWQymeZap8WeJmUgRpoJXR0Jmjpz5mkiIqKyonbYefDgQVm0o8rJH6/T0MEcJgavvR4rERERlUDtq6yrq2tZtKPK4fw6RERE5UPtsLNly5Zi3x89evRrN6YqYdghIiIqH2qHnWnTpin9nJOTg/T0dBgYGMDExIRhRwXp2bm4HZMMgGGHiIiorKn96PmLFy+UXqmpqQgLC0PHjh2xY8eOsmij1rn2WAqZXMDBwgiO1Y0rujlERERaTe2wU5i6deti6dKlBXp9qHBX/je/DpeIICIiKnsaCTtA3uzKT58+1dThtJpivA7n1yEiIipzao/ZOXDggNLPQgjExMRg9erV6NChg8Yapq3kcqHo2eF4HSIiorKndtjp37+/0s8SiQQ2Njbo2rUrVqxYoal2aa37CWlISs+Bkb4OGjqaV3RziIiItJ7aYUcul5dFO6qM/CUimjhVh76uxu4iEhERURF4tS1nnF+HiIiofKkddgYNGoRly5YV2L58+XIMHjxYI43SZoqVzjk4mYiIqFyoHXbOnz+P3r17F9jeq1cvnD9/XiON0lZJ6dmIiE8FwMfOiYiIyovaYSc1NRUGBgYFtuvr6yM5OVkjjdJWV6OSAAC1rE1haVrwd0hERESap3bYady4MXbt2lVg+86dO9GwYUONNEpb5Y/XYa8OERFR+VH7aawFCxZg4MCBiIyMRNeuXQEAp06dwo4dO7Bnzx6NN1CbcHAyERFR+VM77PTt2xf79+/HkiVL8Mcff8DY2BhNmjTByZMn0blz57Joo1bIlckR+jgJAMMOERFReVI77ABAnz590KdPH023RavdjU1BRo4MZkZ6qGNTraKbQ0REVGWoPWYnKCgIgYGBBbYHBgYiODhYI43SRorxOi41oKMjqeDWEBERVR1qhx0fHx88fvy4wPbo6Gj4+PhopFHaiON1iIiIKobaYef27dto0aJFge3NmzfH7du3NdIobcSwQ0REVDHUDjuGhoaIi4srsD0mJgZ6eq81BKjSW7p0KSQSCaZPn67Y9vbbb0MikSi9Jk+eXGj5WGkmopMyoCMBmjpXBwBMnjwZEokEP/74o9K+bm5uBY67dOnSMjozIiIi7ad2OvH29oavry/++usvWFhYAACSkpIwf/58vPPOOxpvYEULCgrCunXr0KRJkwLvTZw4Ed98843iZxMTk0KPceV/S0R42JujmqEe9u3bh0uXLsHR0bHQ/b/55htMnDhR8bOZmVlpToGIiKhKUzvsfP/99+jUqRNcXV3RvHlzAEBoaCjs7OywdetWjTewIqWmpmLEiBHYsGEDvv322wLvm5iYwN7evsTjvHwLKzo6Gp9++imOHTtW5BNtZmZmKh2XiIiISqb2bayaNWvi+vXrWL58ORo2bIiWLVvip59+wo0bN+Ds7FwWbawwPj4+6NOnD7p3717o+9u3b4e1tTU8PT3h6+uL9PT0QvfLDzvNnS0watQozJ49G40aNSqy3qVLl8LKygrNmzfHd999h9zc3NKfDBERURX1WoNsTE1NMWnSJE23pVLZuXMnrly5gqCgoELfHz58OFxdXeHo6Ijr169j7ty5CAsLw969e5X2y8yR4dZTKQAgcP9G6OnpYerUqUXWO3XqVLRo0QKWlpb4999/4evri5iYGKxcuVJzJ0dERFSFvPaI4tu3byMqKgrZ2dlK2997771SN6qiPX78GNOmTcOJEydgZGRU6D4vh73GjRvDwcEB3bp1Q2RkJGrXrq1470a0FDkyAZPkR9i8dw2uXLkCiaToeXZmzpyp+O8mTZrAwMAAH330Efz8/GBoaKiBsyMiIqpa1A479+/fx4ABA3Djxg1IJBIIIQBAcQGXyWSabWEFCAkJQXx8vNIj9jKZDOfPn8fq1auRlZUFXV1dpTJt27YFAERERCiFnfxbWNWlkbgbHw8XFxelY86aNQs//vgjHj58WGhb2rZti9zcXDx8+BD169fX1CkSERFVGWqP2Zk2bRrc3d0RHx8PExMT3Lp1C+fPn0erVq1w9uzZMmhi+evWrRtu3LiB0NBQxatVq1YYMWIEQkNDCwQdIG+QNgA4ODgobc8PO4OHDcf169eVjuno6IjZs2fj2LFjRbYlNDQUOjo6sLW11dwJEhERVSFq9+wEBATg9OnTsLa2ho6ODnR0dNCxY0f4+flh6tSpuHr1alm0s1yZmZnB09NTaZupqSmsrKzg6emJyMhI/P777+jduzesrKxw/fp1zJgxA506dVJ6RN3DwwM5zYcCrm3QqUkteL4yoaC+vj7s7e0VPTYBAQEIDAxEly5dYGZmhoCAAMyYMQMjR45EjRqcjJCIiOh1qB12ZDKZYt4Xa2trPH36FPXr14erqyvCwsI03sDKyMDAACdPnsSPP/6ItLQ0ODs7Y9CgQfjiiy+U9gsLC4NV7WRY6urAs6Z5icc1NDTEzp07sXDhQmRlZcHd3R0zZsxQGsdDRERE6lE77Hh6euLatWtwd3dH27ZtsXz5chgYGGD9+vWoVatWWbSxUnj5Fp2zszPOnTtXYpn/Ox+JRYfvoL69GQz1Ct76enWcTosWLXDp0qXSNpWIiIheovaYnS+++AJyuRxA3ky/Dx48wFtvvYW///4bP//8s8Yb+KbaFRSFRYfvAABuRkuxKyiqgltERERUNUlE/uNUpZCYmIgaNWoU+0h1ZZacnAwLCwtIpVKYm5d8u6kkMdIMdFh6GvKXfrO6EgkuzOsCBwvjUh+fiIiIVL9+q92zUxhLS8s3NuiUhQcJaUpBBwBkQuBhQuEzLBMREVHZ0UjYIWXu1qbQeSX76UokcLMufKFQIiIiKjsMO2XAwcIYfgMbQ/d/vV26EgmWDPTkLSwiIqIK8NrLRVDxhrZ2Qad6NniYkA43axMGHSIiogqids/O+fPnC12FOzc3F+fPn9dIo7SFg4UxvGpbMegQERFVILXDTpcuXZCYmFhgu1QqRZcuXTTSKCIiIiJNUTvsCCEKffLq+fPnMDU11UijiIiIiDRF5TE7AwcOBJC3uvnYsWNhaGioeE8mk+H69eto37695ltIREREVAoqhx0LCwsAeT07ZmZmMDb+bxyKgYEB2rVrh4kTJ2q+hURERESloHLY8ff3BwC4ubnhs88+4y0rIiIieiOoPWZnzpw5SmN2Hj16hB9//BHHjx/XaMOIiIiINEHtsNOvXz9s2bIFAJCUlIQ2bdpgxYoV6NevH9asWaPxBhIRERGVhtph58qVK3jrrbcAAH/88Qfs7e3x6NEjbNmyhaueExERUaWjdthJT0+HmZkZAOD48eMYOHAgdHR00K5dOzx69EjjDSQiIiIqDbXDTp06dbB//348fvwYx44dg7e3NwAgPj6+2OXViYiIiCqC2mHnyy+/xGeffQY3Nze0adMGXl5eAPJ6eZo3b67xBhIRERGVhtph5/3330dUVBSCg4Nx7NgxxfZu3brhhx9+0GjjXrV06VJIJBJMnz5dsS0zMxM+Pj6wsrJCtWrVMGjQIMTFxZVpO4iIiOjNoXbYAQB7e3uYmZnhxIkTyMjIAAC0bt0aHh4eGm3cy4KCgrBu3To0adJEafuMGTNw8OBB7NmzB+fOncPTp08Vsz0TERERqR12nj9/jm7duqFevXro3bs3YmJiAAATJkzArFmzNN5AAEhNTcWIESOwYcMG1KhRQ7FdKpXit99+w8qVK9G1a1e0bNkS/v7++Pfff3Hp0qUyaQsRERG9WdQOOzNmzIC+vj6ioqJgYmKi2D506FAcPXpUo43L5+Pjgz59+qB79+5K20NCQpCTk6O03cPDAy4uLggICCjyeFlZWUhOTlZ6ERERkXZSebmIfMePH8exY8fg5OSktL1u3bpl8uj5zp07ceXKFQQFBRV4LzY2FgYGBqhevbrSdjs7O8TGxhZ5TD8/P3z99deabioRERFVQmr37KSlpSn16ORLTExUWgldEx4/foxp06Zh+/btMDIy0thxfX19IZVKFa/Hjx9r7NhERERUuagddt566y3FchEAIJFIIJfLsXz5cnTp0kWjjQsJCUF8fDxatGgBPT096Onp4dy5c/j555+hp6cHOzs7ZGdnIykpSalcXFwc7O3tizyuoaEhzM3NlV5ERESkndS+jbV8+XJ069YNwcHByM7Oxpw5c3Dr1i0kJibi4sWLGm1ct27dcOPGDaVt48aNg4eHB+bOnQtnZ2fo6+vj1KlTGDRoEAAgLCwMUVFRivl/iIiIqGpTO+x4enri3r17WL16NczMzJCamoqBAwfCx8cHDg4OGm2cmZkZPD09lbaZmprCyspKsX3ChAmYOXMmLC0tYW5ujk8//RReXl5o166dRttCREREbya1w05UVBScnZ3x+eefF/qei4uLRhqmqh9++AE6OjoYNGgQsrKy0KNHD/z666/l2gYiIiKqvCRCCKFOAV1dXcTExMDW1lZp+/Pnz2FrawuZTKbRBpaH5ORkWFhYQCqVcvwOERHRG0LV67faA5SFEJBIJAW2p6amavSJKSIiIiJNUPk21syZMwHkPX21YMECpcfPZTIZAgMD0axZM403kIiIiKg0VA47V69eBZDXs3Pjxg0YGBgo3jMwMEDTpk3x2Wefab6FRERERKWgctg5c+YMgLxHv3/66SeObSEiIqI3gtpPY/n7+5dFO4iIiIjKhNoDlImIiIjeJAw7REREpNUYdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUqddjx8/ND69atYWZmBltbW/Tv3x9hYWFK+2RmZsLHxwdWVlaoVq0aBg0ahLi4uApqMREREVU2lTrsnDt3Dj4+Prh06RJOnDiBnJwceHt7Iy0tTbHPjBkzcPDgQezZswfnzp3D06dPMXDgwApsNREREVUmEiGEqOhGqOrZs2ewtbXFuXPn0KlTJ0ilUtjY2OD333/H+++/DwC4e/cuGjRogICAALRr106l4yYnJ8PCwgJSqRTm5uZleQpERESkIapevyt1z86rpFIpAMDS0hIAEBISgpycHHTv3l2xj4eHB1xcXBAQEFDkcbKyspCcnKz0IiIiIu30xoQduVyO6dOno0OHDvD09AQAxMbGwsDAANWrV1fa187ODrGxsUUey8/PDxYWFoqXs7NzWTadiIiIKtAbE3Z8fHxw8+ZN7Ny5s9TH8vX1hVQqVbweP36sgRYSERFRZaRX0Q1QxZQpU3Do0CGcP38eTk5Oiu329vbIzs5GUlKSUu9OXFwc7O3tizyeoaEhDA0Ny7LJREREVElU6p4dIQSmTJmCffv24fTp03B3d1d6v2XLltDX18epU6cU28LCwhAVFQUvL6/ybi4RERFVQpW6Z8fHxwe///47/vrrL5iZmSnG4VhYWMDY2BgWFhaYMGECZs6cCUtLS5ibm+PTTz+Fl5eXyk9iERERkXar1I+eSySSQrf7+/tj7NixAPImFZw1axZ27NiBrKws9OjRA7/++muxt7FexUfPiYiI3jyqXr8rddgpLww7REREbx6tnGeHiIiISF0MO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLSa1oSdX375BW5ubjAyMkLbtm1x+fLlim4SERERVQJaEXZ27dqFmTNn4quvvsKVK1fQtGlT9OjRA/Hx8RXdNCIiIqpgWhF2Vq5ciYkTJ2LcuHFo2LAh1q5dCxMTE2zcuLGim0ZEREQVTK+iG1Ba2dnZCAkJga+vr2Kbjo4OunfvjoCAgELLZGVlISsrS/GzVCoFACQnJ5dtY4mIiEhj8q/bQohi93vjw05CQgJkMhns7OyUttvZ2eHu3buFlvHz88PXX39dYLuzs3OZtJGIiIjKTkpKCiwsLIp8/40PO6/D19cXM2fOVPwsl8uRmJgIKysrSCQSjdWTnJwMZ2dnPH78GObm5uVannWXf92lLc+6q1bdpS3Puln3m1K+tHUXRwiBlJQUODo6FrvfGx92rK2toauri7i4OKXtcXFxsLe3L7SMoaEhDA0NlbZVr169rJoIc3PzUv0Dl6Y86y7/uktbnnVXrbpLW551s+43pXxp6y5KcT06+d74AcoGBgZo2bIlTp06pdgml8tx6tQpeHl5VWDLiIiIqDJ443t2AGDmzJkYM2YMWrVqhTZt2uDHH39EWloaxo0bV9FNIyIiogqmFWFn6NChePbsGb788kvExsaiWbNmOHr0aIFBy+XN0NAQX331VYFbZuVRnnWXf92lLc+6q1bdpS3Puln3m1K+tHVrgkSU9LwWERER0RvsjR+zQ0RERFQchh0iIiLSagw7REREpNUYdoiIiEirMeyUoV9++QVubm4wMjJC27ZtcfnyZZXKnT9/Hn379oWjoyMkEgn279+vcp1+fn5o3bo1zMzMYGtri/79+yMsLEzl8mvWrEGTJk0Ukz95eXnhyJEjKpd/2dKlSyGRSDB9+nSV9l+4cCEkEonSy8PDQ+X6oqOjMXLkSFhZWcHY2BiNGzdGcHCwSmXd3NwK1C2RSODj41NiWZlMhgULFsDd3R3GxsaoXbs2Fi1aVOJaLS9LSUnB9OnT4erqCmNjY7Rv3x5BQUEF9ivpsyGEwJdffgkHBwcYGxuje/fuCA8PV7n83r174e3trZhNPDQ0VOX6c3JyMHfuXDRu3BimpqZwdHTE6NGj8fTpU5XqXrhwITw8PGBqaooaNWqge/fuCAwMVLntL5s8eTIkEgl+/PFHlcqOHTu2wL99z5491ar7zp07eO+992BhYQFTU1O0bt0aUVFRJZYt7HMnkUjw3XffqVR3amoqpkyZAicnJxgbGysWQ1albFxcHMaOHQtHR0eYmJigZ8+eis+LKt8lmZmZ8PHxgZWVFapVq4ZBgwYpJnhVpfz69evx9ttvw9zcHBKJBElJSYr3SiqfmJiITz/9FPXr14exsTFcXFwwdepUSKVSler+6KOPULt2bRgbG8PGxgb9+vVTLDGkzveoEAK9evVS/H5VKfv2228X+PeePHmyWnUHBASga9euMDU1hbm5OTp16oRvvvmm2LIPHz4s8vO2Z88eleqOjY3FqFGjYG9vD1NTU7Ro0QJ//vmnSmUjIyMxYMAA2NjYwNzcHEOGDCkwIXBZYdgpI7t27cLMmTPx1Vdf4cqVK2jatCl69OiB+Pj4EsumpaWhadOm+OWXX9Su99y5c/Dx8cGlS5dw4sQJ5OTkwNvbG2lpaSqVd3JywtKlSxESEoLg4GB07doV/fr1w61bt9RqR1BQENatW4cmTZqoVa5Ro0aIiYlRvC5cuKBSuRcvXqBDhw7Q19fHkSNHcPv2baxYsQI1atRQub0v13vixAkAwODBg0ssu2zZMqxZswarV6/GnTt3sGzZMixfvhyrVq1SqW4A+PDDD3HixAls3boVN27cgLe3N7p3747o6Gil/Ur6bCxfvhw///wz1q5di8DAQJiamqJHjx7IzMxUqXxaWho6duyIZcuWFfl+UeXT09Nx5coVLFiwAFeuXMHevXsRFhaG9957T6W669Wrh9WrV+PGjRu4cOEC3Nzc4O3tjWfPnqlUPt++fftw6dIlpenjVSnbs2dPpc/Ajh07VC4fGRmJjh07wsPDA2fPnsX169exYMECGBkZlVj25TpjYmKwceNGSCQSDBo0SKW6Z86ciaNHj2Lbtm24c+cOpk+fjilTpuDAgQPFlhVCoH///rh//z7++usvXL16Fa6urujevTvS0tJU+i6ZMWMGDh48iD179uDcuXN4+vQpBg4cCEC176L09HT07NkT8+fPL9C+kso/ffoUT58+xffff4+bN29i06ZNOHr0KCZMmKBS3S1btoS/vz/u3LmDY8eOQQgBb29vyGQytb5Hf/zxR6VlhlQtO3HiRKV/9+XLl6tcPiAgAD179oS3tzcuX76MoKAgTJkyBRcuXCi2rLOzc4HP29dff41q1aqhV69eKtU9evRohIWF4cCBA7hx4wYGDhyIIUOG4ODBg8WWTUtLg7e3NyQSCU6fPo2LFy8iOzsbffv2hVwuL/B71ThBZaJNmzbCx8dH8bNMJhOOjo7Cz89PreMAEPv27XvtdsTHxwsA4ty5c699jBo1aoj/+7//U3n/lJQUUbduXXHixAnRuXNnMW3aNJXKffXVV6Jp06av1ca5c+eKjh07vlbZwkybNk3Url1byOXyEvft06ePGD9+vNK2gQMHihEjRqhUV3p6utDV1RWHDh1S2t6iRQvx+eefF1nu1c+GXC4X9vb24rvvvlNsS0pKEoaGhmLHjh0lln/ZgwcPBABx9epVlesvzOXLlwUA8ejRI7XLSqVSAUCcPHlS5bqfPHkiatasKW7evClcXV3FDz/8oFLZMWPGiH79+hXbnuLKDx06VIwcOfK1yr6qX79+omvXriqXb9Sokfjmm2+UthX22Xm1bFhYmAAgbt68qdgmk8mEjY2N2LBhQ4G6X/0uSUpKEvr6+mLPnj2Kfe7cuSMAiICAgBLLv+zMmTMCgHjx4kWh511S+Xy7d+8WBgYGIicnR+2y165dEwBERESEynVfvXpV1KxZU8TExBT5b1tYWXW+Fwsr37ZtW/HFF1+8VtlXNWvWrMD3V3HlTU1NxZYtW5T2s7S0LPCZebXssWPHhI6OjpBKpYp9kpKShEQiESdOnCjxXEqLPTtlIDs7GyEhIejevbtim46ODrp3746AgIBybYtUKgUAWFpaql1WJpNh586dSEtLU2vpDR8fH/Tp00fp/FUVHh4OR0dH1KpVCyNGjEBUVJRK5Q4cOIBWrVph8ODBsLW1RfPmzbFhwwa16wfy/v22bduG8ePHq7QwbPv27XHq1Cncu3cPAHDt2jVcuHABvXr1Uqm+3NxcyGQyGBkZKW03NjZWuWcLAB48eIDY2Fil37uFhQXatm1b7p+7fFKpFBKJRO2157Kzs7F+/XpYWFigadOmKpWRy+UYNWoUZs+ejUaNGqnd1rNnz8LW1hb169fHxx9/jOfPn6tc7+HDh1GvXj306NEDtra2aNu2rVq3n/PFxcXh8OHDmDBhgspl2rdvjwMHDiA6OhpCCJw5cwb37t2Dt7d3seWysrIAQOlzp6OjA0NDw0I/d69+l4SEhCAnJ0fp8+bh4QEXF5dCP2+l+S5StbxUKoW5uTn09PQKbC+ubFpaGvz9/eHu7g5nZ2eV6k5PT8fw4cPxyy+/FLkOY3F1b9++HdbW1vD09ISvry/S09NVKh8fH4/AwEDY2tqiffv2sLOzQ+fOnVX6N3tVSEgIQkNDi/y8FVa+ffv22LVrFxITEyGXy7Fz505kZmbi7bffLrZsVlYWJBKJ0sSCRkZG0NHRUet77rWVeZyqgqKjowUA8e+//yptnz17tmjTpo1ax0IpenZkMpno06eP6NChg1rlrl+/LkxNTYWurq6wsLAQhw8fVrnsjh07hKenp8jIyBBCqPcXzN9//y12794trl27Jo4ePSq8vLyEi4uLSE5OLrGsoaGhMDQ0FL6+vuLKlSti3bp1wsjISGzatEnltufbtWuX0NXVFdHR0SrtL5PJxNy5c4VEIhF6enpCIpGIJUuWqFWnl5eX6Ny5s4iOjha5ubli69atQkdHR9SrV6/IMq9+Ni5evCgAiKdPnyrtN3jwYDFkyJASy79MEz07GRkZokWLFmL48OEqlz148KAwNTUVEolEODo6isuXL6tc95IlS8Q777yj6I1Tp2dnx44d4q+//hLXr18X+/btEw0aNBCtW7cWubm5JZbP/6vexMRErFy5Uly9elX4+fkJiUQizp49q9J551u2bJmoUaOG4v8fVdqemZkpRo8eLQAIPT09YWBgIDZv3lxi2ezsbOHi4iIGDx4sEhMTRVZWlli6dKkAILy9vZXKFvZdsn37dmFgYFCgntatW4s5c+aUWP5lJfXsqPJd9uzZM+Hi4iLmz5+vctlffvlFmJqaCgCifv36hfbqFFV+0qRJYsKECYqfC/u3KarsunXrxNGjR8X169fFtm3bRM2aNcWAAQNUqjsgIEAAEJaWlmLjxo3iypUrYvr06cLAwEDcu3dPpfPO9/HHH4sGDRoU+l5R5V+8eCG8vb0Vnzdzc3Nx7NixEsvGx8cLc3NzMW3aNJGWliZSU1PFlClTBAAxadKkItuoKQw7ZaCyhJ3JkycLV1dX8fjxY7XKZWVlifDwcBEcHCzmzZsnrK2txa1bt0osFxUVJWxtbcW1a9cU29QJO6968eKFMDc3V+kWmr6+vvDy8lLa9umnn4p27dqpXa+3t7d49913Vd5/x44dwsnJSezYsUNcv35dbNmyRVhaWqoVtCIiIkSnTp0EAKGrqytat24tRowYITw8PIosU5nDTnZ2tujbt69o3ry5Urd1SWVTU1NFeHi4CAgIEOPHjxdubm4iLi6uxPLBwcHCzs5OKaCqE3ZeFRkZqfIttPz/3z/44AOl/fr27SuGDRumVt3169cXU6ZMKfL9wsp/9913ol69euLAgQPi2rVrYtWqVaJatWoFbg0UVjY4OFg0bdpU8bnr0aOH6NWrl+jZs6fSfoV9l6gTdkr6Liop7JRUXiqVijZt2oiePXuK7OxslcsmJSWJe/fuiXPnzom+ffuKFi1aFAiahZX/66+/RJ06dURKSopiW2G/X1W/g0+dOlXoLbTCyuf/f+7r66u0b+PGjcW8efNUrjs9PV1YWFiI77//vtD3iyo/ZcoU0aZNG3Hy5EkRGhoqFi5cKCwsLMT169dLLHvs2DFRq1YtIZFIhK6urhg5cqRo0aKFmDx5cjG/Hc1g2CkDWVlZQldXt8AHf/To0eK9995T61ivG3Z8fHyEk5OTuH//vtplX9WtWzeVkve+ffsUX5r5LwCKD3ZhfyWXpFWrVkr/AxfFxcVF6a8sIYT49ddfhaOjo1r1PXz4UOjo6Ij9+/erXMbJyUmsXr1aaduiRYtE/fr11apbiLyLfX5YGTJkiOjdu3eR+7762ci/QL8aUDp16iSmTp1aYvmXlSbsZGdni/79+4smTZqIhIQEtcq+qk6dOoX2kr1a/ocfflB8zl7+7Ono6AhXV9fXqtva2lqsXbu2xLqzsrKEnp6eWLRokdJ+c+bMEe3bt1e57vPnzwsAIjQ0tMg2vVo+PT1d6OvrFxjvNWHCBNGjRw+V605KShLx8fFCiLzxhp988onivaK+S/Iv0K8GFBcXF7Fy5coSy7+suLBTUvnk5GTh5eUlunXrViCoqPM9mJWVJUxMTMTvv/9eYvlp06YV+Xnr3Lmz2nWnpqYKAOLo0aMl1n3//n0BQGzdulVp+5AhQxS9qKrUvWXLFqGvr6/4d39ZUeUjIiIKjPMSIu8a8dFHH6lc97NnzxT/1nZ2dmL58uVF7qspHLNTBgwMDNCyZUucOnVKsU0ul+PUqVNqjX15HUIITJkyBfv27cPp06fh7u5e6mPK5XLF/f3idOvWDTdu3EBoaKji1apVK4wYMQKhoaHQ1dVVq97U1FRERkbCwcGhxH07dOhQ4DHHe/fuwdXVVa06/f39YWtriz59+qhcJj09HTo6yv8r6erqvtYTBqampnBwcMCLFy9w7Ngx9OvXT+Wy7u7usLe3V/rcJScnIzAwsMw/d/lycnIwZMgQhIeH4+TJk7CysirV8VT97I0aNQrXr19X+uw5Ojpi9uzZOHbsmNr1PnnyBM+fP1fps2dgYIDWrVuX+vP322+/oWXLliqPUQLyft85OTml/vxZWFjAxsYG4eHhCA4ORr9+/Ur8LmnZsiX09fWVPm9hYWGIioqCl5dXqb+LVCmfnJwMb29vGBgY4MCBA4rxR69Tt8j74x9ZWVkllp83b16BzxsA/PDDD9i4caPadeeXd3BwKLFuNzc3ODo6Fvp5c3FxUbnu3377De+99x5sbGyUfgfFlc8fV1TY500mk6lct7W1NapXr47Tp08jPj5e8cRmmSrzOFVF7dy5UxgaGopNmzaJ27dvi0mTJonq1auL2NjYEsumpKSIq1eviqtXrwoAinEArz7RUpiPP/5YWFhYiLNnz4qYmBjFKz09XaV2z5s3T5w7d048ePBAXL9+XcybN09IJBJx/Phxlcq/Sp3bWLNmzRJnz54VDx48EBcvXhTdu3cX1tbWhf7l8arLly8LPT09sXjxYhEeHi62b98uTExMxLZt21Ruq0wmEy4uLmLu3LkqlxEi70memjVrikOHDokHDx6IvXv3Cmtr6wJd+cU5evSoOHLkiLh//744fvy4aNq0qWjbtm2BLvmSPhtLly4V1atXV4w/6devn3B3d1f8xVtS+efPn4urV6+Kw4cPCwBi586d4urVqyImJqbE8tnZ2eK9994TTk5OIjQ0VOnzl5WVVWzZ1NRU4evrKwICAsTDhw9FcHCwGDdunDA0NFT8Fanu/xcv38YqrmxKSor47LPPREBAgHjw4IE4efKkaNGihahbt67IzMxUqe69e/cKfX19sX79ehEeHi5WrVoldHV1xT///KNSu6VSqTAxMRFr1qwpcB4lle/cubNo1KiROHPmjLh//77w9/cXRkZG4tdffy2x7O7du8WZM2dEZGSk2L9/v3B1dRUDBw4UQqj2XTJ58mTh4uIiTp8+LYKDg4WXl5fidrIq5WNiYsTVq1fFhg0bBABx/vx5cfXqVfH8+fMSy0ulUtG2bVvRuHFjERERobTP5MmTiy0bGRkplixZIoKDg8WjR4/ExYsXRd++fYWlpaWIi4t7re9R/K/nrKSyERER4ptvvhHBwcHiwYMH4q+//hK1atUSnTp1Uvn39sMPPwhzc3OxZ88eER4eLr744gthZGQkhg8frlK7w8PDhUQiEUeOHFHaXlLd2dnZok6dOuKtt94SgYGBIiIiQnz//fdCIpGI3r17l1j3xo0bRUBAgIiIiBBbt24VlpaWYubMmUX+TjWJYacMrVq1Sri4uAgDAwPRpk0bcenSJZXK5XfpvvoaM2ZMiWULKwdA+Pv7q1T3+PHjhaurqzAwMBA2NjaiW7durx10hFAv7AwdOlQ4ODgIAwMDUbNmTTF06NBCBwwW5eDBg8LT01MYGhoKDw8PsX79erXaeuzYMQFAhIWFqVUuOTlZTJs2Tbi4uAgjIyNRq1Yt8fnnn4usrCyVj7Fr1y5Rq1YtYWBgIOzt7YWPj49ISkoqsF9Jnw25XC4WLFgg7OzshKGhoejWrZvS+ZRU3t/fv9D3v/rqqxLL59/6Kux15syZYstmZGSIAQMGCEdHR2FgYCAcHBzEe++9pzRAWd3/L14OO8WVTU9PF97e3sLGxkbo6+sLV1dXMXHiRKU/TFSp+7fffhN16tQRRkZGomnTpopboaqUXbdunTA2Nn6tf/OYmBgxduxY4ejoKIyMjET9+vXFihUrhFwuL7HsTz/9JJycnIS+vr5wcXERX3zxheJzq8p3SUZGhvjkk09EjRo1hImJiRgwYIAiGKtS/quvvipyn5LKF3Vuxb3yy0ZHR4tevXoJW1tboa+vL5ycnMTw4cPF3bt3VW77q/LDTkllo6KiRKdOnYSlpaUwNDQUderUEbNnz1aMbVO1bj8/P+Hk5CRMTEyEl5eX+Oeff1Qu6+vrK5ydnYVMJitwDiWVv3fvnhg4cKCwtbUVJiYmokmTJmLLli0qlZ07d66ws7MT+vr6om7duorPaXmQ/O8EiYiIiLQSx+wQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIXnH27FlIJBIkJSVVdFOISAMYdoiIiEirMewQERGRVmPYIaJKRy6Xw8/PD+7u7jA2NkbTpk3xxx9/APjvFtPhw4fRpEkTGBkZoV27drh586bSMf788080atQIhoaGcHNzw4oVK5Tez8rKwty5c+Hs7AxDQ0PUqVMHv/32m9I+ISEhaNWqFUxMTNC+ffsCK00T0ZuBYYeIKh0/Pz9s2bIFa9euxa1btzBjxgyMHDkS586dU+wze/ZsrFixAkFBQbCxsUHfvn2Rk5MDIC+kDBkyBMOGDcONGzewcOFCLFiwAJs2bVKUHz16NHbs2IGff/4Zd+7cwbp161CtWjWldnz++edYsWIFgoODoaenh/Hjx5fL+RORZnEhUCKqVLKysmBpaYmTJ0/Cy8tLsf3DDz9Eeno6Jk2ahC5dumDnzp0YOnQoACAxMRFOTk7YtGkThgwZghEjRuDZs2c4fvy4ovycOXNw+PBh3Lp1C/fu3UP9+vVx4sQJdO/evUAbzp49iy5duuDkyZPo1q0bAODvv/9Gnz59kJGRASMjozL+LRCRJrFnh4gqlYiICKSnp+Odd95BtWrVFK8tW7YgMjJSsd/LQcjS0hL169fHnTt3AAB37txBhw4dlI7boUMHhIeHQyaTITQ0FLq6uujcuXOxbWnSpInivx0cHAAA8fHxpT5HIipfehXdACKil6WmpgIADh8+jJo1ayq9Z2hoqBR4XpexsbFK++nr6yv+WyKRAMgbT0REbxb27BBRpdKwYUMYGhoiKioKderUUXo5Ozsr9rt06ZLiv1+8eIF79+6hQYMGAIAGDRrg4sWLSse9ePEi6tWrB11dXTRu3BhyuVxpDBARaS/27BBRpWJmZobPPvsMM2bMgFwuR8eOHSGVSnHx4kWYm5vD1dUVAPDNN9/AysoKdnZ2+Pzzz2FtbY3+/fsDAGbNmoXWrVtj0aJFGDp0KAICArB69Wr8+uuvAAA3NzeMGTMG48ePx88//4ymTZvi0aNHiI+Px5AhQyrq1ImojDDsEFGls2jRItjY2MDPzw/3799H9erV0aJFC8yfP19xG2np0qWYNm0awsPD0axZMxw8eBAGBgYAgBYtWmD37t348ssvsWjRIjg4OOCbb77B2LFjFXWsWbMG8+fPxyeffILnz5/DxcUF8+fPr4jTJaIyxqexiOiNkv+k1IsXL1C9evWKbg4RvQE4ZoeIiIi0GsMOERERaTXexiIiIiKtxp4dIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0mr/DyhqLZkmhFwSAAAAAElFTkSuQmCC", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "plt.plot(np.arange(len(epochs_x)), epochs_acc, marker = '.')\n", - "plt.xlabel('epoch')\n", - "plt.ylabel('test accuracy')\n", - "plt.ylim(0, 100)\n", - "plt.xticks(np.arange(len(epochs_x)))\n", - "for i, txt in enumerate(epochs_acc):\n", - " if i%5 == 0:\n", - " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", - " else:\n", - " pass\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [], - "source": [ - "with open('nonseq_exp_set_B_training_metrics.npy', 'wb') as f:\n", - " np.save(f, np.array(epochs_x))\n", - " np.save(f, np.array(epochs_y))\n", - " np.save(f, np.array(epochs_acc))" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "speck-rescnn", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/tests/test_nonsequential/exp_set_B1/baseline-SCNN-example_3.ipynb b/tests/test_nonsequential/exp_set_B1/baseline-SCNN-example_3.ipynb deleted file mode 100644 index 2eb99dfc..00000000 --- a/tests/test_nonsequential/exp_set_B1/baseline-SCNN-example_3.ipynb +++ /dev/null @@ -1,1071 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import torch, random\n", - "import torch.nn as nn\n", - "import sinabs.layers as sl\n", - "from tqdm.notebook import tqdm\n", - "\n", - "from tonic.datasets.dvsgesture import DVSGesture\n", - "from tonic.transforms import ToFrame\n", - "from torch.utils.data import DataLoader\n", - "from torch.nn import CrossEntropyLoss\n", - "from torch.optim import Adam\n", - "\n", - "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential\n", - "\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "torch.backends.cudnn.deterministic = True\n", - "random.seed(1)\n", - "torch.manual_seed(1)\n", - "torch.cuda.manual_seed(1)\n", - "np.random.seed(1)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Loading Data" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "batch_size = 3\n", - "num_workers = 1\n", - "epochs = 30\n", - "lr = 5e-4" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "root_dir = \"../DVSGESTURE\"\n", - "_ = DVSGesture(save_to=root_dir, train=True)\n", - "_ = DVSGesture(save_to=root_dir, train=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "n_time_steps = 50\n", - "to_raster = ToFrame(sensor_size=DVSGesture.sensor_size, n_time_bins=n_time_steps)\n", - "\n", - "snn_train_dataset = DVSGesture(save_to=root_dir, train=True, transform=to_raster)\n", - "snn_test_dataset = DVSGesture(save_to=root_dir, train=False, transform=to_raster)" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The transformed array is in shape [Time-Step, Channel, Height, Width] --> (50, 2, 128, 128)\n" - ] - } - ], - "source": [ - "sample_data, label = snn_train_dataset[0]\n", - "print(f\"The transformed array is in shape [Time-Step, Channel, Height, Width] --> {sample_data.shape}\")" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "snn_train_dataloader = DataLoader(snn_train_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True)\n", - "snn_test_dataloader = DataLoader(snn_test_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=False)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Network Module\n", - "\n", - "We need to define a `nn.Module` implementing the network we want the chip to reproduce." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "device: NVIDIA GeForce RTX 3070 Ti\n" - ] - } - ], - "source": [ - "if torch.cuda.is_available():\n", - " device = torch.device('cuda:0')\n", - " print('device: ', torch.cuda.get_device_name(0))\n", - "else:\n", - " device = torch.device('cpu')" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "class SNN(nn.Module):\n", - " def __init__(self) -> None:\n", - " super().__init__()\n", - "\n", - " self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False)\n", - " self.iaf1 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential(), spike_threshold=0.5)\n", - " self.pool1 = nn.AvgPool2d(2,2)\n", - "\n", - " self.conv2 = nn.Conv2d(10, 10, 2, 1, bias=False)\n", - " self.iaf2 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential(), spike_threshold=0.5)\n", - " self.pool2 = nn.AvgPool2d(3,3)\n", - "\n", - " self.conv3 = nn.Conv2d(10, 10, 3, 1, bias=False)\n", - " self.iaf3 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential(), spike_threshold=0.5)\n", - " self.pool3 = nn.AvgPool2d(2,2)\n", - "\n", - " self.flat = nn.Flatten()\n", - "\n", - " self.fc1 = nn.Linear(810, 100, bias=False)\n", - " self.iaf4 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential(), spike_threshold=0.5)\n", - "\n", - " self.fc2 = nn.Linear(100, 100, bias=False)\n", - " self.iaf5 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential(), spike_threshold=0.5)\n", - "\n", - " self.fc3 = nn.Linear(100, 100, bias=False)\n", - " self.iaf6 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential(), spike_threshold=0.5)\n", - "\n", - " self.fc4 = nn.Linear(100, 11, bias=False)\n", - " self.iaf7 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential(), spike_threshold=0.5)\n", - "\n", - " def detach_neuron_states(self):\n", - " for name, layer in self.named_modules():\n", - " if name != '':\n", - " if isinstance(layer, sl.StatefulLayer):\n", - " for name, buffer in layer.named_buffers():\n", - " buffer.detach_()\n", - "\n", - " def init_weights(self):\n", - " for name, layer in self.named_modules():\n", - " if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear):\n", - " nn.init.xavier_normal_(layer.weight.data)\n", - "\n", - " def forward(self, x):\n", - " \n", - " con1_out = self.conv1(x)\n", - " iaf1_out = self.iaf1(con1_out)\n", - " pool1_out = self.pool1(iaf1_out)\n", - "\n", - " conv2_out = self.conv2(pool1_out)\n", - " iaf2_out = self.iaf2(conv2_out)\n", - " pool2_out = self.pool2(iaf2_out)\n", - "\n", - " conv3_out = self.conv3(pool2_out)\n", - " iaf3_out = self.iaf3(conv3_out)\n", - " pool3_out = self.pool3(iaf3_out)\n", - "\n", - " flat_out = self.flat(pool3_out)\n", - " \n", - " fc1_out = self.fc1(flat_out)\n", - " iaf4_out = self.iaf4(fc1_out)\n", - "\n", - " fc2_out = self.fc2(iaf4_out)\n", - " iaf5_out = self.iaf5(fc2_out)\n", - "\n", - " fc3_out = self.fc3(iaf5_out)\n", - " iaf6_out = self.iaf6(fc3_out)\n", - "\n", - " fc4_out = self.fc4(iaf6_out)\n", - " iaf7_out = self.iaf7(fc4_out)\n", - "\n", - " return iaf7_out" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "snn = SNN().to(device)" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "snn.init_weights()" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "optimizer = Adam(snn.parameters(), lr=lr, betas=(0.9, 0.999), eps=1e-8)\n", - "loss_fn = CrossEntropyLoss()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Define train and test" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "def train(dataloader, model, loss_fn, optimizer, epochs, test_func, dataloader_test):\n", - " epochs_y = []\n", - " epochs_x = []\n", - " epochs_acc = []\n", - " model.train()\n", - "\n", - " for e in range(epochs):\n", - " losses = []\n", - " batches = []\n", - " batch_count = 0\n", - " train_p_bar = tqdm(snn_train_dataloader)\n", - "\n", - " for X, y in train_p_bar:\n", - " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", - " X = X.reshape(-1, 2, 128, 128).to(dtype=torch.float, device=device)\n", - " y = y.to(dtype=torch.long, device=device)\n", - "\n", - " # forward\n", - " pred = model(X)\n", - "\n", - " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", - " pred = pred.reshape(batch_size, n_time_steps, -1)\n", - "\n", - " # accumulate all time-steps output for final prediction\n", - " pred = pred.sum(dim = 1)\n", - " loss = loss_fn(pred, y)\n", - "\n", - " # gradient update\n", - " optimizer.zero_grad()\n", - " loss.backward()\n", - " optimizer.step()\n", - "\n", - " # detach the neuron states and activations from current computation graph(necessary)\n", - " model.detach_neuron_states()\n", - "\n", - " train_p_bar.set_description(f\"Epoch {e} - BPTT Training Loss: {round(loss.item(), 4)}\")\n", - "\n", - " batch_count += 1\n", - " losses.append(loss.item())\n", - " batches.append(batch_count)\n", - "\n", - " epochs_y.append(losses)\n", - " epochs_x.append(batches)\n", - "\n", - " acc = test_func(dataloader_test, model)\n", - " print(f'Epoch {e} accuracy: {acc}')\n", - " epochs_acc.append(acc)\n", - "\n", - " return epochs_x, epochs_y, epochs_acc\n" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [], - "source": [ - "def test(dataloader, model):\n", - " correct_predictions = []\n", - " with torch.no_grad():\n", - " test_p_bar = tqdm(dataloader)\n", - " for X, y in test_p_bar:\n", - " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", - " X = X.reshape(-1, 2, 128, 128).to(dtype=torch.float, device=device)\n", - " y = y.to(dtype=torch.long, device=device)\n", - "\n", - " # forward\n", - " output = model(X)\n", - "\n", - " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", - " output = output.reshape(batch_size, n_time_steps, -1)\n", - "\n", - " # accumulate all time-steps output for final prediction\n", - " output = output.sum(dim=1)\n", - "\n", - " # calculate accuracy\n", - " pred = output.argmax(dim=1, keepdim=True)\n", - "\n", - " # compute the total correct predictions\n", - " correct_predictions.append(pred.eq(y.view_as(pred)))\n", - "\n", - " test_p_bar.set_description(f\"Testing Model...\")\n", - " \n", - " correct_predictions = torch.cat(correct_predictions)\n", - " return correct_predictions.sum().item()/(len(correct_predictions))*100" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Training loop" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "7c174908d3784728bf9706cd7683230e", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - " 0%| | 0/359 [00:00" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "y_avg = []\n", - "for y in epochs_y:\n", - " y_avg.append(np.mean(y))\n", - "\n", - "plt.plot(np.arange(len(epochs_x)), y_avg, marker = '.')\n", - "plt.xlabel('epoch')\n", - "plt.ylabel('average loss')\n", - "plt.ylim(0,)\n", - "plt.xticks(np.arange(len(epochs_x)))\n", - "for i, txt in enumerate(y_avg):\n", - " if i%2 == 0:\n", - " pass\n", - " else:\n", - " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAG2CAYAAACZEEfAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAAB7hklEQVR4nO3deXhM1/8H8PdkskeE7DNJJEESBLEWsRdRVUupXe2qpSXUvrSqJGiptoryU2uLllhqj5bYiSVEBCEhEYkIyWTfz++PfDM1ss2QiIz363nmaXPvPcuNY+7HuWeRCCEEiIiIiLSUTkVXgIiIiKg8MdghIiIircZgh4iIiLQagx0iIiLSagx2iIiISKsx2CEiIiKtxmCHiIiItBqDHSIiItJqDHaIiIhIqzHYISIiIq1WocHOyZMn0aNHD8jlckgkEuzZs0flvBAC8+fPh1wuh5GRETp06ICQkBCVazIzM/HFF1/A0tISJiYm6NmzJx4+fPga74KIiIjeZBUa7KSmpsLDwwMrV64s8vzSpUuxfPlyrFy5EoGBgbC1tUWXLl2QnJysvMbb2xu7d+/G9u3bcfr0aaSkpOCDDz5Abm7u67oNIiIieoNJ3pSNQCUSCXbv3o3evXsDyO/Vkcvl8Pb2xowZMwDk9+LY2NhgyZIlGDduHBQKBaysrLBlyxYMGDAAAPDo0SM4ODjg4MGD6Nq1a0XdDhEREb0hdCu6AsWJiIhAbGwsvLy8lMcMDAzQvn17nD17FuPGjcPly5eRnZ2tco1cLkf9+vVx9uzZYoOdzMxMZGZmKn/Oy8vDs2fPYGFhAYlEUn43RURERGVGCIHk5GTI5XLo6BT/suqNDXZiY2MBADY2NirHbWxs8ODBA+U1+vr6qF69eqFrCtIXxdfXF998800Z15iIiIgqQlRUFOzt7Ys9/8YGOwVe7GkRQpTa+1LaNbNmzcKUKVOUPysUCtSoUQNRUVGoWrXqq1WYiIiIXoukpCQ4ODjA1NS0xOve2GDH1tYWQH7vjUwmUx6Pi4tT9vbY2toiKysLCQkJKr07cXFx8PT0LDZvAwMDGBgYFDpetWpVBjtERESVTGmdIG/sOjvOzs6wtbWFv7+/8lhWVhYCAgKUgUzTpk2hp6enck1MTAxu3LhRYrBDREREb48K7dlJSUnB3bt3lT9HREQgKCgI5ubmqFGjBry9veHj4wMXFxe4uLjAx8cHxsbGGDx4MADAzMwMo0ePxpdffgkLCwuYm5tj6tSpaNCgATp37lxRt0VERERvkAoNdi5duoSOHTsqfy4YRzN8+HBs3LgR06dPR3p6OsaPH4+EhAS0aNECR48eVXk398MPP0BXVxf9+/dHeno6OnXqhI0bN0Iqlb72+yEiIqI3zxuzzk5FSkpKgpmZGRQKBcfsEBERVRLqPr/f2DE7RERERGWBwQ4RERFpNQY7REREpNUY7BAREZFWY7BDREREWo3BDhEREWk1BjtERESk1RjsEBERkVZjsENERERajcEOERERaTUGO0RERKTVGOwQERGRVmOwQ0RERFqNwQ4RERFpNQY7REREpNUY7BAREZFWY7BDREREWo3BDhEREWk1BjtERESk1RjsEBERkVZjsENERERajcEOERERaTUGO0RERKTVGOwQERGRVmOwQ0RERFqNwQ4RERFpNQY7REREpNUY7BAREZFWY7BDREREWo3BDhEREWk1BjtERESk1RjsEBERkVZjsENERERajcEOERERaTUGO0RERKTVGOwQERGRVmOwQ0RERFqNwQ4RERFpNQY7REREpNUY7BAREZFWY7BDREREWo3BDhEREWk1BjtERESk1RjsEBERkVZjsENERERajcEOERERaTUGO0RERKTVGOwQERGRVmOwQ0RERFqNwQ4RERFpNQY7REREpNUY7BAREZFWY7BDREREWo3BDhERERWSk5ODuXPnwtnZGUZGRqhZsyYWLFiAvLw85TVCCMyfPx9yuRxGRkbo0KEDQkJCSszXz88PzZo1Q7Vq1WBiYoJGjRphy5Ytha5btWoVnJ2dYWhoiKZNm+LUqVMvfS8MdoiIiKiQJUuWYM2aNVi5ciVCQ0OxdOlSfPfdd/j555+V1yxduhTLly/HypUrERgYCFtbW3Tp0gXJycnF5mtubo45c+bg3LlzuH79OkaOHImRI0fiyJEjymt27NgBb29vzJkzB1evXkXbtm3RrVs3REZGvtS9SIQQ4qVSapGkpCSYmZlBoVCgatWqFV0dIiKiCvfBBx/AxsYG69evVx7r27cvjI2NsWXLFgghIJfL4e3tjRkzZgAAMjMzYWNjgyVLlmDcuHFql9WkSRN0794d3377LQCgRYsWaNKkCVavXq28pm7duujduzd8fX2Vx9R9frNnh4iIiApp06YN/vnnH9y5cwcAcO3aNZw+fRrvv/8+ACAiIgKxsbHw8vJSpjEwMED79u1x9uxZtcoQQuCff/7B7du30a5dOwBAVlYWLl++rJIvAHh5eamd74t0XyoVERERabUZM2ZAoVCgTp06kEqlyM3NxaJFizBo0CAAQGxsLADAxsZGJZ2NjQ0ePHhQYt4KhQJ2dnbIzMyEVCrFqlWr0KVLFwBAfHw8cnNzi8y3oExNsWeHiIioBE5OTpBIJIU+EyZMAAA8fvwYI0aMgFwuh7GxMd577z2EhYWVmGdISAj69u2rzHvFihUal1veduzYga1bt+KPP/7AlStXsGnTJnz//ffYtGmTynUSiUTlZyFEoWMvMjU1RVBQEAIDA7Fo0SJMmTIFJ06ceOV8i8Ngh4iIqASBgYGIiYlRfvz9/QEA/fr1gxACvXv3Rnh4OPbu3YurV6/C0dERnTt3RmpqarF5pqWloWbNmli8eDFsbW2LLdfe3r7Q8VWrVikDnpSUFHz++eewt7eHkZER6tatqzLOpSgdOnQoMojq3r278prk5GSMHTsWqampGDlyJMaNG4c6depg8uTJyjEzBfV+sbclLi6uUK/Mi3R0dFC7dm00atQIX375JT766CNlvpaWlpBKpS+Vb7HlvVQqIiJ6a8Uo0nH2XjxiFOkVXZXXwsrKCra2tsrP/v37UatWLbRv3x5hYWE4f/48Vq9ejebNm8PNzQ2rVq1CSkoKtm3bVmyezZs3x3fffYeBAwfCwMCg2HKvXLmiDLLGjBkDmUwGID/QAoDJkyfj8OHD2Lp1K0JDQzF58mR88cUX2Lt3b7Fl+/n5qQRvN27cgFQqVeYJAGPGjEF6ejpGjBiB4OBgeHl5KQO4gqnnzs7OsLW1VQZ/QP54m4CAAHh6eqr/C0Z+r01mZiYAQF9fH02bNlXJFwD8/f01zvf5At56CoVCABAKhaKiq0JE9EbbfvGBcJ65XzjO2C+cZ+4X2y8+qOgqvVaZmZnCwsJCLFq0SAghxPXr1wUAcffuXZXrbG1txfDhw9XK09HRUfzwww9qlevp6Slq1aol8vLyhBBCuLu7iwULFqhc26RJEzF37lz1bkgI8cMPPwhTU1ORkpIihBAiLS1NSKVS0alTJ2FnZyf2798vIiIihJOTkzA2NhbTp09Xpl28eLEwMzMTfn5+Ijg4WAwaNEjIZDKRlJSkvObjjz8WM2fOVP7s4+Mjjh49Ku7duydCQ0PFsmXLhK6urli3bp3ymu3btws9PT2xfv16cfPmTeHt7S1MTEzE/fv3Vequ7vObA5SJiEgtMYp0zPILRt7/FizJE8Bsvxto52oFmZlRxVbuNdmzZw8SExMxYsQIAECdOnXg6OiIWbNm4ddff4WJiQmWL1+O2NhYxMTElGm5CQkJCA0NxdSpU5VjV9q0aYN9+/Zh1KhRkMvlOHHiBO7cuYMff/xR7bzXr1+PgQMHwsTEBED+YoK5ubmYNGkS/vnnH4wfPx5xcXEQQsDGxkY5PRwApk+fjvT0dIwfPx4JCQlo0aIFjh49ClNTU+U1kZGR0NH570VSamoqxo8fj4cPH8LIyAh16tTB1q1bMWDAAOU1AwYMwNOnT7FgwQLExMSgfv36OHjwIBwdHV/uF6h26FcBsrOzxZw5c4STk5MwNDQUzs7O4ptvvhG5ubnKa/Ly8sTXX38tZDKZMDQ0FO3btxc3btzQqBz27BARle7M3SfCccb+Qp+zd+PLvWxHR0cBoNBn/PjxQghR5DkAYunSpSXmm5CQIMaPHy9sbW2FgYGBqFOnjjhw4IDy/Ndff10oT319fZU8Ll26JDw8PAQAIZVKRdeuXUW3bt1Et27d1L630np2vLy8RJMmTYRUKhXR0dHK45mZmWLYsGECgNDV1RX6+vpi8+bNapUrhBAXLlwQAMSFCxdUjrdq1Uq0b99eREdHi5ycHLFlyxYhkUiEq6ur2nkXeJSYJs7cfSIeJaZpnLY0WtGzU7B646ZNm+Du7o5Lly5h5MiRMDMzw6RJkwD8t3rjxo0b4erqioULF6JLly64ffu2SmRJRESvxtnSBBLkP/Gfdz8+Fa1qWZRr2YGBgcjNzVX+fOPGDXTp0kU5zuTFXpRDhw5h9OjR6Nu3b7F5ZmVloUuXLrC2tsbOnTthb2+PqKioQs8Od3d3HDt2DFFRUWjZsiXWrVuncr5p06YICgqCQqFAVlYWrKys0KJFCzRr1uxVbxsA8ODBAxw7dgwNGzZEt27dIJfLled++uknnD9/Hvv27YOjoyNOnjyJ8ePHQyaToXPnzqXmvX79etSvXx/vvPOOyvEtW7Zg1KhRsLOzg1QqRZMmTTB48GBcuXJFo7rvCIxU9gbqSADfPg0woHkNjfIoC290sHPu3Dn06tVLOULcyckJ27Ztw6VLlwDkD2hasWIF5syZgz59+gAANm3aBBsbG/zxxx8ard5IREQls6xiADMjPSSmZwOAMvCZt/cGqhnroVsDWbmVbWVlpfLz4sWLlYOEARSa0bR371507NgRNWvWLDbP3377Dc+ePcPZs2ehp6cHAEW+JtHV1YWtrS3WrFkDa2trDB48uMj8zMzMAABhYWG4dOmSyuueV7FhwwZYWFjg+vXrmD9/vvJ4eno6Zs+ejd27dyufkw0bNkRQUBC+//77UoOdtLQ0bN++HQsWLCh0rlatWggICEBqaiqSkpIgk8kwYMAAODs7q13vN+m15xs9G6u8Vm/MzMxEUlKSyoeIiEpm5+CIa/O74sGSD/BgyQe4/7//Pj68Cp9vu4oD12MQGhqKnj17wszMDKampmjZsmWp+xmtWLECbm5uMDIygoODAyZPnoyMjAzl+eTkZHh7e8PR0RFGRkZo2bIlNm7ciFGjRhW57srjx49x4MABjB49usRy9+3bh1atWmHChAmwsbFB/fr14ePjo9KDBOQHLzKZDAsXLkS1atUK3c9ff/2FEydOKKefd+nSBb1791Z5Ng0bNgyzZs1S/pyVlYWgoCAEBQUhKysL0dHRCAoKwt27d1XyzsvLw4YNG+Dq6gpra2uV6eHZ2dnIzs5WGQ8DAFKpVGWzzuL8+eefyMzMxNChQ4u9xsTEBDKZDAkJCThy5Ah69epVar4FIp6kKgOdArlC4H58mtp5lJkyf4FWhvLy8sTMmTOFRCIRurq6QiKRCB8fH+X5M2fOCAAq7y+FEGLs2LHCy8ur2HyLegcLjtkhIipWTm6eaP31bmE/YYtYvOuciImJEf7+/gKA+GjeOuE4Y79w+PT/hKlZNTFt2jRx5coVce/ePbF//37x+PHjYvPdunWrMDAwEL///ruIiIgQR44cETKZTHh7eyuv6d+/v6hXr54ICAgQYWFh4qOPPhIAxKVLl4rMc8mSJaJ69eoiPT29xHtyc3MTBgYGYtSoUeLSpUti27ZtwtzcXHzzzTfKaw4ePCh27twp1qxZIwCI5s2bCxsbGxEf/984pR9//FHY29sLPT09UaNGDTF37lyRmZmpUlb79u1F/0FDlWNXIiIiinwOtW/fXiXdkSNHBAAhl8vFjBkzCt1D+/bthbu7uzh+/LgIDw8XGzZsEIaGhmLVqlXKa16cDVWgTZs2YsCAAUX+bg4fPiwOHTokwsPDxdGjR4WHh4d45513RFZWVom/0+fN3R1caHxXzZkHynTsjrpjdt7oYGfbtm3C3t5ebNu2TVy/fl1s3rxZmJubi40bNwoh/gt2Hj16pJJuzJgxomvXrsXmm5GRIRQKhfITFRXFYIeIqAR/X4sWjjP2i4bzj4jkjGwhhBCTJk0StWrVEtk5uWLKjiBhXKetMHHvKPZcfah2vhMmTBDvvvuuyrEpU6aINm3aCCH+mwa9f/9+5XkvLy9RtWpVMWfOnCLzdHNzE59//nmpZbu4uAgHBweRk5OjPLZs2TJha2tbbJqUlBRhY2Mjli1bVmr+z3uVKfsFAc/t27cLnYuJiREjRowQcrlcGBoaCjc3N7Fs2TLl1HQh8gOiF6fB3759WwAQR48eLbLMHTt2iJo1awp9fX1ha2srJkyYIBITE9Wu868Bd5UBjtPM/wKdsl6qQCsGKE+bNg0zZ87EwIEDAQANGjTAgwcP4Ovri+HDh6us3liw0BJQ+iqLBgYGxS7iREREqvLyBFb+m/96ZWRrJ1Qx0EVWVha2bt2KKVOmQFeqg8V96uPn4Zdh3OxDDOrTA/qJkXBzqYVZs2ahd+/exebdpk0bbN26FRcvXsQ777yD8PBwHDx4EMOHDwfw3zRoQ0NDAP8N1q1duzZOnz5dKL9Tp07h9u3b2LFjR6n3JZPJoKenB6lUqjxWt25dxMbGIisrC/r6+oXSmJiYoEGDBqVuB1EgKycPx27GYuauYOXAbk3Hrnh5eUGIF4eF57O1tcWGDRtKTH/ixAnlQpDOliaQmRnB1dW12DwBoH///ujfv3+pdSvKn5ei4HPwFgBgxnt10LuxHPfj0+BkaVxhSxS80cFOWlpaie8in1+9sXHjxgD+W71xyZIlr72+RETa6J9bcbgVm4wqBroY4ekEoPB6M0/jnyA7Iw1pgbtQpfVQGDmOhGuVR+jTpw+OHz+uHEj8ooEDB+LJkydo06YNhBDIycnBZ599hpkzZwLI30OpVatW+Pbbb1G3bl2sX78epqamyrGcL1q/fj2aNm0KDw+PUu+rdevW+OOPP5CXl6d81ty5cwcymazIQAfIH/MZGhqKtm3bFnleCIF7T1JwKiwep8PicT78KVKzcgtdVzB25XU8/F/njKjDN2Ixc9d1AMC4djXxWYdaAFDh6zC90cFOjx49sGjRItSoUQPu7u64evUqli9fjlGjRgHI3yTM29sbPj4+cHFxgYuLC3x8fGBsbFzsaHkiIlKfEAIrj+f36gxt6YhqxvlBwPr161WmQRf8I7Tvhx/Cqd8X2HYxCqclNdGkzSWsWbOm2GDnxIkTWLRoEVatWoUWLVrg7t27mDRpEmQyGebNmwdAdRo0kN+b8cEHHxSaBp2UlIS//voLy5YtK7KsYcOGwc7OTrkH02effYaff/4ZkyZNwhdffIGwsDD4+Phg4sSJyjRTp05Fjx49UKNGDcTFxWHu198gIVGBbh/+twBefEomztyNVwY4sUkZKuVWe24GWwEJACdL42J+62WnqBlRs/yCy2VG1Nm78Zi47SryBDCgmQNmdqtTpvm/ijc62Pn5558xb9485eqNcrkc48aNw1dffaW8Rp3VG4mI6OWcvhuPa1GJMNTTwZi2+dOOC14l+fn5Ka+ztLSErq4u3N3rYXbvBtCRSPD7hUiEZZkh+Wbxr3zmzZuHjz/+GGPGjAGQP1whNTUVn3zyCebMmQMdHR3lNOh9+/ahV69eCAgIwLx58wpNg96+fTuEEBg0aFCRZb24kq+DgwOOHj2KyZMno2HDhrCzs8OkSZMwY8YM5TUPHz7EoEGDEB8fjyrVzJFZvRbMBi5F/z/uol3tRMSlZCE0RnVGr76uDt5xMkcbF0u0qW2JerKq+OtyFGb73UDuc6+OHiakl3uPx+2Y5EIzovIEsObEPczv6f7Su4i/6FpUIsZuvoSs3Dy8526LRR/WL7O8y4JElPTS7i2RlJQEMzMzKBQKVK1ataKrQ0T0xhjw6zlciHiGka2d8HUPdwDA/Pnz8euvvyIqKgq6uv/9m9nT0xO1atXCli1bIITAV3tDsGz6WOjoGWD9xs1Fvjpp2rQpOnfurDL0YNu2bRg1ahRSUlJUxtMUSEhIgLOzM5YuXYpPPvlEo/uJUaQjIj5VOXblRbl5AglpWYhPyUR8cv5/nyRn4v7TVPx+ofgp9HVlVdHOxRJtXCzR3MkchnqF6x2jSMf9+FRsOvsAh0NiITMzxMGJbVHdpOhXZq9KCIFPt1zGkZuPizzf0c0K3/XzgGWVVxvDejcuGf3WnENCWjY8a1ngtxHNi7z/8qDu8/uN7tkhIqKKczHiGS5EPIOeVIJP2uUvzlew7svw4cNVAh0gf1LJgAED0K5dO3Ts2BHWUceRcS8Q1oN8MGNX/quUQyvnqrxK6tGjB5YvX47GjRsrX2PNmzcPPXv2VAY6R44cgRACbm5uuHv3LqZNmwY3NzeMHDlSo/t5fuyKBEAbF0tYVjFQBjTxKVl4lppZqCekJJ93rIXhns6wMi09YJCZGUFmZoSG9tVw5+dkhMenYupf1/B/w5uVSy/IhjP3ceTmY0gk+febJwCpBOjeUI7DIbE4fvsJ3ltxCsv7e6Cdq1Wp+RUlOjEdH6+/iIS0bHjYm2HtsGavLdDRBIMdIiIqUsFYnY+aOih7QY4dO4bIyEjl2Mnnffjhh1izZg18fX0xceJEuLm5wW/XTlyTumDDmfuY5RcMoxt3VF4lzZ07FxKJBHPnzkV0dDSsrKyU4zULKBQKzJo1Cw8fPoS5uTn69u2LRYsWKVc9VsfVyATM2BWs/FkAOBUWX+z15ib6sKyiD8sqBrCsYgAjPSn+vBSlslWGVCLBkJaOagU6zzMx0MXPgxvjw1Vn8c+tOPx25j5Gt1F/ZWJ1nA9/ikUHQwEA87rXQ7cGtiozom7FJmHitqu48zgFw367iLFtnTG1qxsMdNUPVOJTMvHx/11AjCIDta2rYMPId1DF4M0MK/gaC3yNRUT0ousPE9Fz5RlIdSQ4/mUH1LB4+cG0QggsPBCK9acjAABTvVzRxLF6sa+SypIiLRurTtzF+tMRyCmiy2bQOw5o6mgOyyr6sDI1gFUVA5ib6ENXWniDgR2BkcpxN1KJBD596r/SrKYt5+5j3t4Q6Ekl2PWZJxraV3vpvJ73KDEdPX4+jaepWejdSI4fBjQqsucoIzsXiw6EYsv5BwAAd3lV/DSoMWpZVSm1jOSMbAxadx43opNgV80IOz9rVSEzrtR9fjPYAYMdIqIXfbL5Eo7efIw+je2wfECjV85PCAGfg6FYdypCeaw8p0FnZOdi09n7+OX4XSRl5BR5jVQiwemZHTV6SOePuymbNWOEEBj/+xUcuhGLGubG2D+xDaoaqt9bVZSM7FwM+PUcrj1UoJ6sKnZ95gkj/ZJ7a/xvPsb0ndeQkJYNIz0p5vesh/7NHIp9tZaRnYsRGy7ifPgzWJjo469PW6GmGgFSeVD3+f1G741FRESv3+3YZBz931iP8R1rlUmeEokEI1s74fnHZ54AZuwKxrYLkcjMKbwWzcvIzRP481IUOn5/Ar6HbiEpIwduNqbYMKI5FvdpAOn/HuAFPTOaBiwyMyO0qmVRJr0YEokEi/s2hH11I0Q+S8Msv+ASF/orjRACX+8NwbWHClQz1sOvHzctNdABgC71bHBoUjt41rJAenYuZuwKxoQ/rkCRll3o2pzcPHz+x1WcD3+GKga62DTqnQoLdDTxZr5cIyKiCvPL/8bqdKtvi9rWZbeMx/2naSjqUT5rdzAWH76F7g1l+LCxHZrWqA4dHc0G7Aoh8O+tOCw5fAt3HqcAAORmhpji5YYPG9tB+r/82rtZVfhqvs8zM9LDz4Mao9+aczhwPQata1licIuX6+n642IkdlyKgo4E+GlgYziYq//q0dbMEFtHt8DaU+H4/shtHAyORVBkIlYMbIx3nM0B5K+kPWNXMI6FPoaBrg7+b3gz1Lcze6m6vm58jQW+xiIiKhARn4pOy04gTwAHJraBu7zsHmYxinS0XvyvymwnCQCLKvqIT8lSHrOvboTejezQu7EdaluX3mtwJTIBiw/ewsX7zwDkBxATOtbCsFZOb+TMoKKsOxmORQdDYaCrg72ft0YdW82eRZcfJGDg2nPIzhWY/p4bxneo/dJ1uRaViEnbr+L+0zToSIDP33VBv6Z28Dl4C4duxEKqI8GvQ5uic73it2V6XThmRwMMdoiI8k3feQ1/XnqITnWssX5E8zLPv6hBvh81dcD58KfYfTUah4JjVLZXaGhvht6N7NDDQ66c9VSwVo6ORIKNZ+7jcEgsAMBAVwcjWzvjs/a1YGb8amNfXre8PIHRmwJx/PYT1LIywd9ftIGxvnovX+KSM9Dj59N4nJSJ9xvY4pfBTV55KntKZg7m7wvBzssPC53r38weSz8qfTuO14HBjgYY7BARAQ8T0tDhuxPIyRPwG++JJjWql0s5JQ3yTc/KxbHQx9h9NRoBd54g93/dQFIdCdrUtoTMzBB/XopS6R3SkQAfNbWHd2dXyKtV/Kupl/UsNQvdfjyJx0mZ+KipPb7vV3pAkZWThyH/dx6B9xPgYl0Fuye0LtPp35vO3cfXe0NUjr3MwO7ywgHK9NYq2N03RpFe0VUhKnNOTk6QSCSFPhMmTEB2djZmzJiBBg0awMTEBHK5HMOGDcOjR49KzNPPzw/NmjVDbXtbhH/fB4rfJyMkYL/KNfPnzy9Upq2t7UvdQ0mDfI30pejhIcdvI5rj4uxO+KanOxo5VENunkDAnSfYHhhVaNG/LaNbYOlHHpU60AHy1/b5aWBj6EiAnZcfwu9K4V6VFy06cBOB9xNgaqCLXz9uWubr3LgU8RqxYBPTyoQDlEmr/HHhAebsvgGB8t/dl6giBAYGIjf3v9c8N27cQJcuXdCvXz+kpaXhypUrmDdvHjw8PJCQkABvb2/07NkTly5dKjZPc3NzfD55GuafSkS2kKKfdRxGjhwJa2trdO3aVXmdu7s7jh07pvy5qK0cypJFFQMM93TCcE8nRMSn4ud/wuB3NbrQdTpv0B5Mr6pFTQt4d3bFcv87mLvnBjwcqhW77s3Oyw+x6Vz+GjkrBjYql1lRzpYm0JFAJcCUSiSvZRPTssSeHdIaMYp0ZaAD5P/lnO13gz08pFWsrKxga2sLYWSG8FRd7Ni1B7Vq1UL79u1hZmYGf39/9O/fH25ubmjZsiV+/vlnXL58GZGRxe/r1KFDB0RXawBUs0fLRvXww4JZaNiwIU6fPq1yna6uLmxtbZUfK6uX22LgZThbmmDae254cZJWZXzwlmZCx9rwrGWBtKxcTPj9CjKyC0/LD36owOzd+StCT+rkgk51y2ewsMzMCL5lMGW/ojHYIa1xIfxZoWmtlbG7lag0OwIj0Xrxvxi05jR+27QFzbz6FDsgVaFQQCKRoFq1asXm9yw1S7nJ5YSOtfDvv//i9u3baNeuncp1YWFhkMvlcHZ2xsCBAxEeHl5m96QObXnwlkaqI8GKAY1gYaKPW7HJWHQgVOX805RMfLr1MrJy8tCpjjUmdXIp1/oMaF4Dp2d2xLaxLXF6ZsdK2VvO11ikFYQQ2B5Y9L9cZWaGr7k2ROUnRpGu3Mwy7c555GWk4JxOfcQo0gs99DMyMjBz5kwMHjy4xMGbG85EICU5CY9WjUDXZdmQSqVYtWoVunTporymRYsW2Lx5M1xdXfH48WMsXLgQnp6eCAkJgYWFRbnd74sGNK+Bdq5v1lo55cG6qiGWD2iE4b9dxJbzD+BZywLdGsiQk5uHL7ZdRXRiOpwtTbB8QCON1yR6GQWbmFZW7NkhrXD4RizOhz+DVCIp1M29zP/OK61KSm+mkgbqAvkB8Pz58yGXy2FkZIQOHTogJCSklFyBFStWwM3NDUZGRnBwcMDkyZORkZGhPL969Wo0bNgQVatWRdWqVdGqVSscOnSo3O7zRRHxqcrxEynXj8KoZlNIqljgt9MRKq87srOzMXDgQOTl5WHVqlXF5peUkY2NZ+9Dom+EtbuPITAwEIsWLcKUKVNw4sQJ5XXdunVD37590aBBA3Tu3BkHDhwAAGzatKlc7rMkZbmK8ZusvasVPuuQv4L19F3XcfnBM3jvCMLZe09hrC/Frx83hZlR5ZpiX1HYs0OVXkpmDub/nf8Qm9CxFga1qIH78WmIT8nA5B3X8Pe1R3C2MMYUL7cKrimVpZIG6gLA0qVLsXz5cmzcuBGurq5YuHAhunTpgtu3b8PUtOhVgX///XfMnDkTv/32Gzw9PXHnzh2MGDECAPDDDz8AAOzt7bF48WLUrp2/aNumTZvQq1cvXL16Fe7u7uV4x/luxSQBAHIUcch4cA1WH84GAKw7FYHdV6PxcUsnDGwmx6cjhyIiIgL//vtvib06W849QHJGDlxtqmLEe62goyNBo0aNEBoaCl9fX3To0KHIdCYmJmjQoAHCwsLK/B7pP1O6uOJC+FNciUxE39XnlMc/bGwHV5uyW91a27Fnhyq95Ufv4HFSJhwtjDG+Y23lv/p6eNjB58MGAICf/r2LXUUsjkWVV8FA3YLP/v37lQN1hRBYsWIF5syZgz59+qB+/frYtGkT0tLS8McffxSb57lz59C6dWsMHjwYTk5O8PLywqBBg1RmMvXo0QPvv/8+XF1d4erqikWLFqFKlSo4f/58ud/zzssP8e3/xm+kBPtDamwGk9rN0b2BDHbVjBCfkoXlR26iVosuOH05GP+3fU+Jr5jSsnKUO5FP6Fhb5XWIEAKZmZnFps3MzERoaChkMlkZ3R0VRU+qg7nd6xY6vv1iFCdfaIDBDlVqN6IV2Hg2Ag9Xj8LJ6e/CSF9X5ZVGwMbF+KxDLQghMNZ7BqxsbDV6pbFr1y7Uq1cPBgYGqFevHnbv3q1yPicnB3PnzoWzszOMjIxQs2ZNLFiwAHl5eeV1y8V6m9cXysrKwtatWzFq1ChIJBJEREQgNjYWXl5eymsMDAzQvn17nD17tth82rRpg8uXL+PixYsAgPDwcBw8eBDdu3dXue7FV2gJCQkYM2bMK79CK6m9/XkpClP/vIJnAVuQ8NtYKM5sg5FOLj6SBuLnQY1wYloHLP+oPjIPf4+MmDDodZ6EPqvOYPCPh3Hwwk2VwGXYsGGYNWsW/rgQiWepWZBc2wPDxzcQHh6OW7duYfny5di8eTOGDh2qTDN16lQEBAQgIiICFy5cwEcffYSkpCQMHz681PuiV5ORU/j7hJMvNCRIKBQKAUAoFIqKrgppICc3T/T8+ZRwnLFfjFp9TMTExCg//v7+AoA4fvy4yM3NEy0HfC4k+kbCsf88cTDgghgwYICQyWQiKSmp2PzPnj0rpFKp8PHxEaGhocLHx0fo6uqK8+fPK69ZuHChsLCwEPv37xcRERHir7/+ElWqVBErVqx4Hb8Cpe0XHwjnmfuF44z9wnnmfrH94oPXWn5F27Fjh5BKpSI6OloIIcSZM2cEAOXPBcaOHSu8vLxKzOunn34Senp6QldXVwAQn332WaFrTpw4IYyMjISOjo4wNTUVCxcuVLY3IYRYvHixMDU1Fbt27RLBwcGv3N62X3wgnGbuF9XafiyMTKuJb775RgAQP/74o0p7i4iIEACK/HhO/EnsC4oW2Tm5on379qLvgCHCY/4R4Thjv/hw5Oeidu3awtDQUFSvXl20atVKbN++XaV+Bfegp6cn5HK56NOnjwgJCSn1z4Ze3aPENOXf74JPzZkHxKPEtIquWoVT9/nN7SLA7SIqqy3nH2DenhswNdDFsS/bw6bqf7OuvL29sX//fuV4ArlcjuoteiOtzgdwsjDG9tHNUKemA5YsWYJx48YVmf+AAQOQlJSkMvj0vffeQ/Xq1bFt2zYAwAcffAAbGxusX79eeU3fvn1hbGyMLVu2lMdtF1LU5opv0nLur0PXrl2hr6+Pv//+GwBw9uxZtG7dGo8ePVJ5zTJ27FhERUXh8OHDReZz4sQJDBw4EAsXLkSLFi1w9+5dTJo0CWPHjsW8efOU12VlZSEyMhKJiYnYtWsXVqxYASsrKzx4kL/Am1wuh7e3N2bMmAEg/5WPjY3NS7W3pDwDPGryCQBA/5+laNOgFn777TflNSW1t7txKVh/OgJ+Vx4i83+9A3bVjNDIoRoOBscol2pY1Ls+hrR0LPF3TBWrqD3FKuMU8LLG7SJIq8UlZ2Dp4VsAgKld3VQCneJeaaycOgJ21Yxw/2kaJv55A23btSvxlca5c+dUXoMA+Q/V59O0adMG//zzD+7cuQMAuHbtGk6fPo3333+/LG+3RLdikgotn/82dXE/ePAAx44dw5gxY5THCrYxiI2NVbk2Li4ONjbFL742b948fPzxxxgzZgwaNGiADz/8ED4+PvD19VV5Namvr4/atWujWbNm+Oabb5CbmwtbW9tXeoVWVHuzcGuOixfyB6WObO2EkX3ew7///qt2e6ttXQW+fRrg7Mx34d3ZBeYm+ohOTMeB5wIdAPhqb8hb+fqzMtGGtW4qEmdjUaXkcyAUyRk5aGBnhqEv/It0z549SExMVM6iKXjg1anpgA0upui76iwu3n8G01Q9ZGXGvpi1UmxsbKEHo42NjcoDdMaMGVAoFKhTpw6kUilyc3OxaNEiDBo0qIzutGRCCPx+IarQcR0JtG5V2eJs2LAB1tbWKuNqnJ2dYWtrC39/fzRu3BhAfhAcEBCAJUuWFJtXWloadHRU/w0olUohhCh2+YI9e/YgJycHTk5OAP5rb0W1nYKen6K82N62nLuPoxEZyE1NwOg2zvmDVD+oh6SkJI3bm0UVA3h3dsWn7WvhuyO3lYOSCxQEx29LT2BlVdnXuqlI7NmhSufM3XjsCXoEHQng82EDSF9YWGf9+vXo1q0b5HK5ynGJRAJXG1P8MqQJpDoS3I9PQeSzkv81++KqtEIIlWM7duzA1q1b8ccff+DKlSvYtGkTvv/++9e29sj60xE4FvoYOhKorC9krC+FvlT7/3rn5eVhw4YNGD58OHR1//u3m0Qigbe3N3x8fLB7927cuHEDI0aMgLGxMQYPHqy8rmCgboEePXpg9erV2L59OyIiIuDv74958+ahZ8+eyn2gZs+ejVOnTuH+/fsIDg7GzJkz8wfAjx2rUrfS2k5RCs5vOnsf8/aGQAgBXZ382TgSieSV25uhnhRj2jq/FVsuED2PPTtUqWRk52LunhsAgGGtnNDA3kzlfMErDT8/P+Wx519pyGQytHO1woJe7hj7lwJReSbYGxSNXo3sCpVla2tb6muQadOmYebMmRg4cCAAoEGDBnjw4AF8fX3LfZbK2Xvx8D2U/yrvqw/qoWt9W9x5nIxv9t1EeHwqZvkF49ePm5b6gK3Mjh07hsjISIwaNarQuenTpyM9PR3jx49HQkICWrRogaNHj6qssRMZGanSkzN37lxIJBLMnTsX0dHRsLKyQo8ePbBo0SLlNY8fP8bHH3+MmJgYGJtUQWLCM3w5c45yteEX21uB0l6hFbS3305HYMH+mwCAFjJd3JDZKv8My6K9FWy58OL4D/YYkDbT/n/60WsVHR2NoUOHwsLCAsbGxmjUqBEuX76sPJ+SkoLPP/8c9vb2MDIyQt26dbF69eoS8wwJCUHfvn3h5OQEI31dXDv0B6xNDTDFy1V5TcEUcA8PD+Tl5cHb21s5Bfz5VxoF+jWWQcTchIFdXUz76zou3X9WqNxWrVqppAGAo0ePwtPTU/lzca89ynvqeXRiOj7/4ypy8wT6NLbDcE8nyMyM0N7VGisHN4GeVIKjNx/jr9e0tlBFTXv38vKCEAKurq6FzkkkEsyfPx8xMTHIyMhAQEAA6tevr3LNiRMn4PvjamXddXV18fXXX+Pu3btIT09HZGQkfvnlF1SrVg3ZuXl4nJSByQuWY+ORi5j4+0WIel0hNamOnXnvYNvF/O1KimpvBa/Qnm87L2rVqhV+27FXGeiM71ALeVHXyqW9cfwHvW3Ys0NlJiEhAa1bt0bHjh1x6NAhWFtb4969eyobEE6ePBnHjx/H1q1b4eTkhKNHj2L8+PGQy+Xo1atXkfmmpaWhZs2aaP9eD0yZPAUA8FWPeqhq+N8y6UuWLMHq1auhr6+PTz/9FJ06dcLIkSNhZmaGSZMmKV9puLi4wMXFBT4+PqhetQq8PuyHExEp+GTLZTjf2ADXmo7w9fUFAEyaNAnt2rXDkiVL0KtXL+zduxfHjh1T2Qm64F/9NWrUgLu7O65evYrly5cX2dNQVjKyc/HZ1st4lpoFd3lV+PRpoNJ7U09eFVO6uGHJ4VtY8PdNtKppAQfz8ntFsSMwUrlXk44E8O3ToNI8PJ+vu0QC9G1sD2crE8SnZCI+JQvxyZn/+/9MJKRlq6QVIg8pwcdgUr8TIJFill8wjtyIRed6Nhj+yfhC7a2oV2h2dnbK9ubc/iP8+flAVDN0xtih/ZEXtKdc2xvHf9DbhFPPwannZWXmzJk4c+YMTp06Vew19evXx4ABA1Sm8TZt2hTvv/8+vv3222LTCSEw7LeL2DalJxq/PxgX/1iu8oD/4IMPkJOTgyNHjuD27dtwdXVVmZIrhMA333yDX3/9VflK45dffkFN1zoY8Ot5BEcroPhrDrq1aog/tm5W5rtz507MnTsX4eHhqFWrFhYtWoQ+ffoozycnJ2PevHnYvXs34uLiIJfLMWjQIHz11VfQ19d/2V9lib+HaTuvY+flh6hurId9n7cpMpDJzRMYuPYcAu8n4B0nc2z7pGWhsU1loTJPe3+UmIbWi49Dky9AHUn+YF9DXR3cuXIGcX9+BfnYX6FnrvoaVAgBceUvPLt0EFlpyWj+zjv4dfUqlZ6lDh06wMnJCRs3bsSagHtYfOgWUm+dhuTyDiQ+fvhGtDeiN526z28GO2CwU1bq1auHrl274uHDhwgICICdnR3Gjx+vMnDz008/xeXLl7Fnzx7I5XKcOHECPXv2xKFDh9CmTZti89537REmbruK6DWjMGval1gwZ7rK+cWLF2PNmjU4evQoXF1dce3aNXh5eWHFihWlzlR5nJSB3r+cQYwiA01qVIN3Z1e42FTR6GEdHR2NGTNm4NChQ0hPT4erqyvWr1+Ppk2bAig8WLXA0qVLMW3atFLz3759OwYNGoQmbbvgqeck6EiAzaNa4NTOdfDz88OtW7dgZGQET09PLFmyBG5uboh6lob3VpxEalYuZnarg0/b11L7ftTld+Uhpvx5rdDxbWNbolWt17cTtqbSsnIwamMgzocXfn3ZztUS9WRmsKyiDytTA1hWKfjoo7qxPnR0JEUGeToSYHQbZ1yLUuBKZAJynjupIwEa2FdD29qWaONiiSY1qkNfVwcxinSsOBaGHYH5M+q8O7vAu3PhV3JEVDR1n998jUVlJjw8HKtXr8aUKVMwe/ZsXLx4ERMnToSBgQGGDRsGAPjpp58wduxY2NvbQ1dXFzo6Ovi///u/EgMdRXo2vv3fOIaqhnowNyn8L9hXmQJuU9UQv41ojl4rT+NKZCKG/XZRo9cx6ry+i4mJUUlz6NAhjB49Gn379i01/wcPHmDq1Klo3LwVbsUmwwrA9PfqoI2LJRYGBGDChAlo3rw5cnJyMGfOHHh5eeHmzZtwMDfB1z3cMX3XdSw7ehvtXKxQT152wfy9JylY9L99mp73pk97f5iQhk82X8bN/22o+TypRIIlfRuWGugWN8i3oL2kZObgQvhTnAqLx+m78bgbl4JrUYm4FpWIlcfvwlhfCofqRrj9OEWZZ5d6Ngx0iMoJgx0qM3l5eWjWrBl8fHwAAI0bN0ZISAhWr16tEuycP38e+/btg6OjI06ePInx48dDJpOhc+fORea77OhtPEnORE0rE9wzLLrJPj8l193dHUFBQfD29oZcLldrlko1Yz1kP/cv8TwBzNwVjGpGevByty1xRtOSJUvg4OCADRs2KI8VrLlSoGCGToG9e/eiY8eOqFmzZon1ys3NxZAhQ/DlzLlY9JsfBJLRvYEM49rlp3txJeCCNWcuX76Mdu3aoV8ze/iHPob/zceY8mcQ9kxoDUM9aYllquNuXAoGrTuPp6lZsKlqgCfJmcpeDicLE9g+t8jjm+R8+FOM//0KnqVmwcJEH/2a2WPdyYiXmpU0oHkNtHO1wv34NDhZGqukq2Kgi051bdCpbv7sqxhFOk7/L/A5HRaPp6lZKoEOAPwbGocYRfob//qPqDJisENlRiaToV69eirH6tati127dgEA0tPTMXv2bOzevVu5AFzDhg0RFBSE77//vshg51pUIracz1+IbWHv+hj8c9Flv+qU3Ij4VLz4QlcAGLf1ChzMjfBhIzv0amyHWlZVCqXdt28funbtin79+hX7+u55jx8/xoEDB9RaG2XBggWwsLTEaWlDZGTvRBUDXSz9qGGh4KvgNdqBAwcAAJ988gl+//13NG3aFL59GuBqZAKCb9xE4zbz8ejWZeTl5cHd3R1//vknatQovvcqMTERc+bMgZ+fHxISEuDs7Iwv532LNeHVEJ+Sicdrx+BBgur0/AcAelwajv3bN5Z6f6+LEAJbzj/Agr9vIidPoL5dVfz6cTPYVTPCcE+nIgMWdag7yFdmZoR+zRzQr5kD8vIEtgdGYvbuGyrXcGE/ovLDYIfKTOvWrXH79m2VY3fu3IGjY/4Kx9nZ2cjOzlZ76mxObh5m7w6GEMCHje3gWcuy2LJfdUqus6UJdCRQGYMhAWCop4OoZ+n46d+7+Onfu/CwN0Pvxnbo4SGHZRUDAOq9vnvepk2bYGpqqjLwtChnzpzB+vXr0d/nD/jdTICeVIJG8mowMVD9a/v8a7T69esjIyMDixcvVr5Gs6xigInNqmLkoulI8eiCn7buQVt3R4SGhsLQsPgemKysLHTp0gXW1tbYuXMn7O3tcebabSw8Eo40UyPUlVXF0SuXUNXwv56i+ZsO49eZI3G/akOkZ+XCSP/Ve5FeVWZOLr7aE4Idl/LHxfRqJMfiPg2VdXvds5J0dCToWMe6UHvjwn5E5YfBDpWZyZMnw9PTEz4+Pujfvz8uXryItWvXYu3atQCAqlWron379pg2bRqMjIzg6OiIgIAAbN68GcuXL1fmUzAl1/WDTxDyKAlV9AR62GchKCgIWVlZiI6ORlBQEKpUqYLatWsDePUpucWNwejpYYejN2Ox52o0TobF49pDBa49VGDhgVC0c7FE78Z2Kq/vYhTpaFDVAYOHXVN5ffe83377DUOGDCkx0EhOTsbQoUPx8TQfbLuZDIkEeMfZHEYis9C1Ba/RjI2NERUVhdOnT8Pe3l7lmoMbf0Ddd9ohpdUorA0R+LCLA7qX8grtt99+w7Nnz3D27Fno6enhdmwyll2TIM3UAfVkVfH7mBao/sL4Kd2HV2FgLkdydVf8evJehY9BiUvKwKdbL+NKZCJ0JMDMbnUwtm3NCl9okQv7Eb1enI0FzsYqS/v378esWbMQFhYGZ2dnTJkyReV1TmxsLGbNmoWjR4/i2bNncHR0xCeffILJkycrH0AdOnSAjdwBIS5DkZKZA+8WZpjcp22hstq3b48TJ04AKLspuTGK9GJfacSnZGL/tUfYHfQI16ISlccfrRkFtyaeGDBlEX47E4E8AaRcPYi8q7vwLE71Fc+pU6fQrl07BAUFwcPDo9h6BAUF5e/pJMnvrdKRSCBEfi+Vjo4Obt++jVq18mdX1atXD/r6+rhz5w6MjY3h4OCg8hotLy8PZmZm8J4yFSu37UdSdBis5Q5Y/d236N27d7F1eP/992Fubg5jY2P47d6DVB1jGNZpD88+I/H7WE9UM1b9vWZlZUEul+P9wWNw0rgtDHR18M+X7WFfvWJ6K65FJWLclsuITcpAVUNd/Dy4Cdq7WlVIXYpTUnsjotJx6rkGGOy8WWIU6fDeHoQLEc/QuEY17PrUEzrlsEbMqwh/koI9V6OxOygaVzYuQG7yE9gOWao8/+yfdciKuY2IkCsqD7ERI0bgxo0buHTpUon5P4xX4INv/0JcSiZa1rTA/B7u+OqreUhOTsaPP/4IV1dX6OvrQwgBPT095ObmYty4cRg3bhwuXrwIb29v/Prrrxg2bJhy2wJjY2OMmzIbO6KrIi38MhQnN+P48eNo3759kXWoU6cO7t+/j+4f9sMNs5Z49ugBFP/8iqlTvOHz7TeFrv/zzz8xePBgPHjwAFP2R+JCxDN0byDDL0OavORv+eXtuvwQs3YHIysnD7Wtq2DdsGZwtjR57fUgovKl7vOb20XQG2VHYCQ8F/+LCxH565+0c7F64wIdAKhpVQVTvNxwclpHrF48F1kxt6E49yeyEx4h9eYJpFw7jCqNu2P4bxfxg/8dXLr/DE8TEvHXX39hzJgxReZZsCllTm4epvrdRKKRDHXqumPjl33RsGEDVKtWDaampqhfv76yt2rChAnIzc2Fu7s75s+fD5lMhl69emHkyJHKbTgKxi316tULy7+dg0n9u8CsZT+YurbAip9/KfYe8/LyUN3CCnfdBiOzmhM8vXri66/mYsP/rS3y+oINWO3s7DC/pzt0JMCB4BicvRf/Kr9qjeTk5uHb/Tfx5V/XkJWTh851rbF7vCcDHaK3HIMdemPEKNIxyy9YZVbUyn/vvvb9ljQhkUgwrGdnbPx9B1JDA/Bo/QQkntmO6u+ORRX3jrjzOAU//hOGj9acQ4OPv0ZmTi5ETU/ce5KCFztVIyMjERMTg8WHbuF8+DOY6Evx68dNVbbFeFFBQBMSEgKZTKb8pKSkIDIyf68mS0tL6OrqKmfKeXd2RT1ZVaCaHc5eu12oHgWqmlshxcASiow8NHKohi1jWqBJw/qIjY1FVlaWyrUFG7AWBHJ1ZVUxpEX+wPQFf99ETm757hUGAIlpWRixIRDrT0cAACa+WxtrP24G0xJ+f0T0duAAZXpjRMSnqsxOASrPdNxhA/rCoGZzlQGn07q6opqxPk7djceZu/FIrO8F+/peWPJvFJb8GwW5mSHaulihjYslWte2xLa9h/D7+QdYefweAOD7fh5wsflvh+6NGzcWKlcIgcGDByMqKkplm47JkycrZ8Hp6+ujefPmyply+ro6WDGwEZqseoQUver442KkMjApEPxQgUg9B6Q/PY5G9lWxefQ7qGqohzt37kAmkxUaB1Wwvk/BkgIAMKWLK/6+/gi3YpPxx8VIDGvl9Eq/4+LEKNJx8k48fvznDh4lZsBYX4pl/TzQrYGs9MRE9FZgsENvjAfxaYWOVabpuMUtMjfwnRrIzRMIeaTIX1E3LB6XHyTgkSIDOy5FKadEP6+Dm5XaD+vSZsEB+esQDRgwAO3atUPHjh1x9PBhpN+7CKuBPli4PxSetSzx9ZTPYGdnh4HjZ+Dj9Reg3+A9SAL3wSZkO2IjbXAqLAw+Pj6YOHGiSvl5eXnYsGEDhg8fDl3d/75Sqpvo40svN8zbcwPLjt5Bj4byQrO3XtWOwEjM3BWs3N+quoketo1tiTq2HHtHRP/hAGVwgPKbIC45A++tOIVnqVmQIH9BvxeX4NcmaVk5uBjxDKfD4nH8dhzuPUlVOa8jAc7MfFftHq3SZsEB+VPJfX198fDhQ7i5ueHrr+fjzye2OBf+FO7yqnj0+0xYyewR02gUkjNy0NypOj6tm4s5M6YhKCgIdnZ2GD16NGbMmAGp9L/1c44ePYquXbsqN2B9Xm6eQPefTuFWbDKGtqyBhb0bvORvrLAYRTo8ff9V2chT098bEVVunI2lAQY7FUsIgTGbLuGfW3GoK6uKNUOb4FFixlszHffsvXgMXneh0PHXsZlmdGI63v3+BDJzVMfUvONkjg0jmxdawPBlnA9/ioFrz0NHAvz9RRu4y81eOU8hBCbvCMKeoEeFzr3pm5ASUdnhbCyqNLYHRuGfW3HQl+pgxYBGcLQwQataFm9FoAP8t3rz817X6zsdCZCVU3jwsG+fBmoHOtHR0Rg6dCgsLCxgbGyMRo0a4fLly8rzhzf/DMXmCbi/rC8a13ZA586dceFC4eDuRYmJiZgwYQJkMhkMDQ1Rt25dHDx4EDm5eZix67pKoKM49yceLPkACf+sqzSvPYno9eGYHapQD56mKnc0n/6eG9xsTUtJoX0qcjXdiPhUFNW1G5eciVrWhfcBe5E6O767urpi9apfMPvYY2SkZ0An4TS8vLxw9+5dWFkVvchfUVtVREVFQc/QGOO2XMY/t+KgI8nfRmT7wRNIvnYE+lbOaF377QmSiUh9DHaowuTk5mHyjiCkZeWiZU1zjGrtXNFVqjAl7aBdnoraE0yTXiV1dnwfPHgwACDONAzL/e8gwd4eSbu34fr16+jUqVOR+b64VQUAVLWUYdSmQFyNTISBrg5WDm6CVjVMsGd2Pyz/aRW2r10BV5u3L1gmotLxNRZVmF9PhuNKZCJMDXTxfT+PN3LxwNdJZmb02l/fFfQqSf+3VYemvUr79u1Ds2bN0K9fP1hbW6Nx48ZYt25dkdd+0q4m5Ka6uHtyLwxNTEvcLmPfvn1o1aoVJkyYABsbG7jVrYem/cbjyv2nMDPSw+9jWqBLPRtMmDABPXt8gInD+kBfl19nRFQ09uxQhbgRrcAP/ncAAN/0cq+w/ZPo1XqV1N3xff/+/Rg4cGD+7vQm1SHv/y3SdYr/Mw8PD8e///6LIUOG4JfNf2LGb0fxYN/PkGflYOfvP8PFxhTbt2/HlStXEBgY+Er3T0Taj8EOvXYZ2bnw3hGEnDyBbvVt8WFju4qu0ltPZmb0Uj1Kz+/4DgCNGzdGSEhIoR3fO3bsiKCgIDx58gRDp/nggZ8v5jSti80TvIrN19raGqNm+uLT368i19kTtb0SkXhhF1xsTBEVFYVJkybh6NGjJe4eT0QE8DUWVYClh2/jblwKrEwNsOjDBsrdzqnykclkym0oCtStW1e5VUUBExMT1K5dG61atcKhXX9AoiPF33/+jtNhRe+bJZPJYC53xMiNl5Vr/iwa8R7iHj9GVlYWLl++jLi4ODRt2hS6urrQ1dVFQEAAfvrpJ+jq6iI3N7fc7pmIKh8GO/Ranbkbj9/O5O9dtPSjhjAv4xV16fVq3bq1chuKAnfu3FFuVVEUVxtTmBpIIXKz8c3fIcguYt8sM6f6uBF6B5k5OfCqZ4Mto1sgOjJcuVVFp06dEBwcjKCgIOWnWbNmGDJkCIKCglQWPSQi4mssem0UadmY+tc1AMCQFjXQ0c26gmtEr6q0rSpSU1OxaNEi9OzZEzKZDE+fPsWqVauQlhAH58YdERaXgi3nHuDE2q9hZ2cHHx8fLDt6B1dMmiMvYwOsgrdh8siv8c/RwypbVRTs/v48ExMTWFhYFDpORMRgh16br/bdQIwiA04WxpjTvW5FV4fKQPPmzbF7927MmjULCxYsgLOzM1asWIEhQ4YAAKRSKW7duoVNmzYhPj4eFhYWaN68OU6dOoW7wgaz/ILxw7E7qBpxH5BIMGPXdfx56SF0q1ph4ncbcf6P5WjcyAN2dnaYNGkSZsyYUbE3TESVEreLALeLeB3+vvYIX2y7CqmOBDs/bYXGNapXdJWoguXmCfRceRohj5LwnrstHidl4GpUInQkwMLeDTC4hfbtiUZEZYvbRdAbI1aRgbl7bgAAJnSoxUCHAABSHQm+6ekOADgcEourUYkAgKEtHBnoEFGZYrBD5UoIgWk7r0GRno0Gdmb4opNLRVeJ3iB21QtPd//9QiRiFOkVUBsi0lYMdqhcbTn/AKfC4mGgq4MfBjSCnpRNjv4TEZ9a6FiuELgfn1YBtSEibcUnD5Wbu3Ep8DkYCgCY1a0OaquxsSS9XSpyx3ciensw2NEy8+fPh0QiUfnY2toqzz9+/BgjRoyAXC6HsbEx3nvvPYSFhamd//bt2yGRSNC7d2+V405OToXKdbExRfSBX9DWxRLDWjmV0R2SNnnVvbmIiNTBqedayN3dHceOHVP+XLDAmhACvXv3hp6eHvbu3YuqVati+fLl6Ny5M27evAkTE5MS833w4AGmTp2Ktm3bFjoXGBioXLV279WH+HrTETzeMRcmdVqjrYvVW7/JJxWvonZ8J6K3h8Y9OydOnCiHalBZ0tXVha2trfJjZWUFAAgLC8P58+exevVqNG/eHG5ubli1ahVSUlKwbdu2EvPMzc3FkCFD8M0336BmzZqFzltZWcHW1ha5BlWxOOAx0u5ehG41GQwcGmDJoVsccEolqogd34no7aFxsPPee++hVq1aWLhwIaKiosqjTvSKwsLCIJfL4ezsjIEDByI8PBwAkJmZCQAqGydKpVLo6+vj9OnTJea5YMECWFlZYfTo0cVeczcuGaM3XkJebjZSb55AlYZdIJFIOOCUiIgqlMbBzqNHjzBp0iT4+fnB2dkZXbt2xZ9//omsrKzyqB9pqEWLFti8eTOOHDmCdevWITY2Fp6ennj69Cnq1KkDR0dHzJo1CwkJCcjKysLixYsRGxuLmJiYYvM8c+YM1q9fj3Xr1hV5PiM7F8v976Dbj6dw63Ey0u6cR15GCkzqdwLAAadERFSxNA52zM3NMXHiRFy5cgWXLl2Cm5sbJkyYAJlMhokTJ+LatWvlUU9SU7du3dC3b180aNAAnTt3xoEDBwAAmzZtgp6eHnbt2oU7d+7A3NwcxsbGOHHiBLp161bsxonJyckYOnQo1q1bB0tLy0Lnz4c/xfs/ncJP/4QhO1fg3TrWsHp0BsY1m0HX1IIDTomIqMK98nYRjx49wtq1a7F48WLo6uoiIyMDrVq1wpo1a+Du7l5W9SxX2r5dRJcuXVC7dm2sXr1aeUyhUCArKwtWVlZo0aIFmjVrhl9++aVQ2qCgIDRu3FglGMrL+98u1RIdyMasgV51GSyrGGB+z3qoXzULtWrVwv9t2YY6LTpxwCkREZWbct0uIjs7Gzt37sT7778PR0dHHDlyBCtXrsTjx48REREBBwcH9OvX76Ur/7zo6GgMHToUFhYWMDY2RqNGjXD58mXleSEE5s+fD7lcDiMjI3To0AEhISFlUrY2yMzMRGhoKGQymcpxMzMzWFlZISwsDJcuXUKvXr2KTF+nTh0EBwcjKCgIQUFBuHr1Kpq390IVZw/YjvgRulUtMbhFDfzzZXt80FCOjRs3wtraGh/378MBp0RE9EbQeOr5F198oZy5M3ToUCxduhT169dXnjcxMcHixYvh5OT0ypVLSEhA69at0bFjRxw6dAjW1ta4d+8eqlWrprxm6dKlWL58OTZu3AhXV1csXLgQXbp0we3bt2FqavrKdahspk6dih49eqBGjRqIi4vDwoULkZSUhOHDhwMA/vrrL1hZWaFGjRoIDg7GpEmT0Lt3b3h5eSnzGDZsGOzs7ODr6wtDQ0Pln2/UszTM2XMDIfE5yJMawt29Pnz6NEBzJ3MA+T0+GzZswPDhw6Gry1UNiIjozaDxE+nmzZv4+eef0bdvX+jr6xd5jVwux/Hjx1+5ckuWLIGDgwM2bNigPPZ8ECWEwIoVKzBnzhz06dMHQP7YFBsbG/zxxx8YN27cK9ehsnn48CEGDRqE+Ph4WFlZoWXLljh//jwcHR0BADExMZgyZQoeP34MmUyGYcOGYd68eSp5REZGQkfnv06/7Nw8rD8dgRXH7iAjOw86OhLUtDLBgYltoa/733XHjh1DZGQkRo0a9XpuloiISA2vPGanPNWrVw9du3bFw4cPERAQADs7O4wfPx5jx44FAISHh6NWrVq4cuUKGjdurEzXq1cvVKtWDZs2bSoy38zMTOU0bCD/nZ+Dg4PWjtl5GTGKdETEpyItKxffH7mNW7HJAICWNc3h82ED1LTi1g9ERFSx1B2zo3HPjq+vL2xsbAr96/23337DkydPMGPGDM1rW4zw8HCsXr0aU6ZMwezZs3Hx4kVMnDgRBgYGGDZsGGJjYwEANjY2KulsbGzw4MGDEu/hm2++KbN6apsdgZGY5ReMvOfC4GrGepjzfl181NQeEglXQyYiospD4wHKv/76K+rUqVPouLu7O9asWVMmlSqQl5eHJk2awMfHB40bN8a4ceMwduxYlVlFAAo9fIUQJT6QZ82aBYVCofxwcUQgL0/gRrQCSw/fwoxdqoGOBMAfY1qgXzMHBjpERFTpaNyzExsbW2hmD5C/XUBJC9O9DJlMhnr16qkcq1u3Lnbt2gUAyg0uX6xTXFxcod6e5xkYGMDAwKBM61oZxSjScSosHqfD4nHmbjyepha9MKQAoEjPeb2VIyIiKiMaBzsODg44c+YMnJ2dVY6fOXMGcrm8zCoGAK1bt8bt27dVjt25c0c52NbZ2Rm2trbw9/dXjtnJyspCQEAAlixZUqZ1qYwKxt04W5pAZmaElMwcnL/3FKfvxuNU2BPce5Kqcr2xvhSNHKrh3L2neH4gF1dAJiKiykzjYGfMmDHw9vZGdnY23n33XQDAP//8g+nTp+PLL78s08pNnjwZnp6e8PHxQf/+/XHx4kWsXbsWa9euBZD/+srb2xs+Pj5wcXGBi4sLfHx8YGxsjMGDB5dpXSqb58fdSAA4WZggKiENOc+9n9KRAA3sq6GdiyXa1LZE4xrVoa+rgx2BkZjtdwO5QnAFZCIiqvQ0no0lhMDMmTPx008/KffDMjQ0xIwZM/DVV1+VeQX379+PWbNmISwsDM7OzpgyZYpyNlZBfb755hv8+uuvSEhIQIsWLfDLL7+orP1TGm1bQTlGkQ7Pxf+iqD/ZGubGaONiiba1LeFZyxJmxnrF5nE/Po0rIBMR0RtL3ef3S089T0lJQWhoKIyMjODi4lKpx8BoW7Cz7uQ9LDp4q9DxHwc0Qq/GdhVQIyIiorJXblPPC1SpUgXNmzd/2eRUTq5FJWK5/51Cx6USCd6paV4BNSIiIqpYLxXsBAYG4q+//kJkZKTyVVYBPz+/MqkYae5uXDJGbLiI9Ow81LIyQUR8KvIEOO6GiIjeahoHO9u3b8ewYcPg5eUFf39/eHl5ISwsDLGxsfjwww/Lo46khujEdHy8/iIS0rLhYW+G38e2RHJGNsfdEBHRW0/jRQV9fHzwww8/YP/+/dDX18ePP/6I0NBQ9O/fHzVq1CiPOlY68+fPh0QiUfkUrAlUcL5OnTowMTFB9erV0blzZ1y4cKHEPDt06FAoT4lEgu7duyM+JRMf/98FRMc9Q97ZDbiydAisqpmib7dO0H0WzkCHiIjeahoHO/fu3UP37t0B5C/Ol5qaColEgsmTJyunhFP+itIxMTHKT3BwsPKcq6srVq5cieDgYJw+fRpOTk7w8vLCkydPis3Pz89PJb8bN25AKpWiR+8+GLHhIsLjU5H2zyoYPbmJ37duQXBwMLy8vNC5c2dER0e/jlsmIiJ6I2kc7JibmyM5OX9TSDs7O9y4cQMAkJiYiLS0tLKtXSWmq6sLW1tb5cfKykp5bvDgwejcuTNq1qwJd3d3LF++HElJSbh+/Xqx+Zmbm6vk5+/vD2NjY/inO+NGdBKq6wsk3jyF5d9/h3bt2qF27dqYP38+nJ2dC22vQURE9DbRONhp27Yt/P39AQD9+/fHpEmTMHbsWAwaNAidOnUq8wpWVmFhYZDL5XB2dsbAgQMRHh5e5HVZWVlYu3YtzMzM4OHhoXb+/7d+PeRNOuHyo3RUMdDFmiGNkJubC0NDQ5XrjIyMcPr06Ve6FyIiospM4wHKK1euREZGBoD8DTX19PRw+vRp9OnTB/PmzSvzClZGLVq0wObNm+Hq6orHjx9j4cKF8PT0REhICCwsLADkL5Y4cOBApKWlQSaTwd/fH5aWlmrlf/78BYTcuAHbj0eiqq4O/m94M7SoaYFWrVrh22+/Rd26dWFjY4Nt27bhwoULcHFxKc/bJSIieqNptKhgTk4Ofv/9d3Tt2lVlwG1lV96LCqampqJWrVqYPn06pkyZojwWExOD+Ph4rFu3Dv/++y8uXLgAa2vrEvMSQqB5twG4fvkiHMauwq9Dm6JzvfxNT+/du4dRo0bh5MmTkEqlaNKkCVxdXXHlyhXcvHmzzO+LiIioIqn7/NboNZauri4+++wzZGZmvnIF3yYmJiZo0KABwsLCVI7Vrl0bLVu2xPr166Grq4v169eXmtcPh4Jx5fgBVPHwwncfNVQGOgBQq1YtBAQEICUlBVFRUbh48SKys7MLbdpKRET0NtF4zE6LFi1w9erV8qiL1srMzERoaChkMlmx1wghSg0it5x/AJ9fNkDkZmP+5E/Rp4l9kdeZmJhAJpMhISEBR44cQa9evV6p/kRERJWZxmN2xo8fjy+//BIPHz5E06ZNYWJionK+YcOGZVa5ymrq1Kno0aMHatSogbi4OCxcuBBJSUkYPnw4UlNTsWjRIvTs2RMymQxPnz7FqlWr8PDhQ/Tr10+Zx7Bhw2BnZwdfX18AwL5rj/DV3htIuX4Ujdp0wcTuTQqVe+TIEQgh4Obmhrt372LatGlwc3PDyJEjX9u9ExERvWk0DnYGDBgAAJg4caLymEQigRACEokEubm5ZVe7Surhw4cYNGgQ4uPjYWVlhZYtW+L8+fNwdHRERkYGbt26hU2bNiE+Ph4WFhZo3rw5Tp06BXd3d2UekZGR0NHJ73g7cTsOU3YEIetpNDIf3sSS9T8UWa5CocCsWbPw8OFDmJubo2/fvli0aBH09Ire2ZyIiOhtoPGu5w8ePCjxvKOj4ytVqCK8qbuexyjScSQkFr4HQ5GZI9DDQ44fBzSCjo6koqtGRERU4cpt1/PKGMxURjsCIzHTLxgFoairTRUs6+fBQIeIiEhDGgc7mzdvLvH8sGHDXroylC9GkY5ZzwU6AHA3LgVPUzO5zxUREZGGNA52Jk2apPJzdnY20tLSoK+vD2NjYwY7ZSDscQryXni5mCeA+/FpDHaIiIg0pPHU84SEBJVPSkoKbt++jTZt2mDbtm3lUce3zu6rDwsdk0okcLI0roDaEBERVW4aBztFcXFxweLFiwv1+pDm/gyMwu6rjwAABcNzpBIJfPrUZ68OERHRS9D4NVZxpFIpHj16VFbZvZWuRSVi7p78XeSndHFFv2b2uB+fBidLYwY6REREL0njYGffvn0qPwshEBMTg5UrV6J169ZlVrG3TXxKJj7dehlZuXnoXNcGn3esDR0dCYMcIiKiV6RxsNO7d2+VnyUSCaysrPDuu+9i2bJlZVWvt0pObh4+/+MKYhQZqGlpguUDOMWciIiorGgc7OTl5ZVHPd5qvodu4Xz4M5joS7F2WFNUNeSKx0RERGWlTAYo08vbGxSN9acjAADL+nugtrVpBdeIiIhIu2gc7Hz00UdYvHhxoePfffedykaWVLqbj5IwY9d1AMD4DrXwXv3id0UnIiKil6NxsBMQEIDu3bsXOv7ee+/h5MmTZVKpt0FiWhbGbb2EjOw8tHO1wpdebhVdJSIiIq2kcbCTkpICfX39Qsf19PSQlJRUJpXSdrl5AhO3ByHqWToczI3w08BGkHJAMhERUbnQONipX78+duzYUej49u3bUa9evTKplLZb7n8bJ+88gaGeDn4d2gzVjAsHj0RERFQ2NJ6NNW/ePPTt2xf37t3Du+++CwD4559/sG3bNvz1119lXkFtc/hGDH45fg8AsKRvQ9STF78lPREREb06jYOdnj17Ys+ePfDx8cHOnTthZGSEhg0b4tixY2jfvn151FFrhD1Oxpd/XgMAjG7jjF6N7Cq4RkRERNpPIoQQpV+m3ZKSkmBmZgaFQoGqVcunpyUpIxu9V55BeHwqWtY0x9bRLaAr5cx/IiKil6Xu81vjp21gYCAuXLhQ6PiFCxdw6dIlTbN7K+TlCUzZcQ3h8amQmRli5eAmDHSIiIheE42fuBMmTEBUVFSh49HR0ZgwYUKZVErbrDx+F8dCH0NfVwdrhjaFZRWDiq4SERHRW0PjMTs3b95EkyZNCh1v3Lgxbt68WSaV0hYxinTsvhKN5f53AAALe9WHh0O1iq0UERHRW0bjYMfAwACPHz9GzZo1VY7HxMRAV1fj7LTWjsBIzPILRt7/RkS1cDZH/+YOFVspIiKit5DGr7G6dOmCWbNmQaFQKI8lJiZi9uzZ6NKlS5lWrrKKUaSrBDoAEHj/GWIU6RVXKSIioreUxl0xy5YtQ7t27eDo6IjGjRsDAIKCgmBjY4MtW7aUeQUro4j4VJVABwDyBHA/Pg0yM6OKqRQREdFbSuNgx87ODtevX8fvv/+Oa9euwcjICCNHjsSgQYOgp6dXHnWsdJwtTaAjgUrAI5VI4GRpXHGVIiIieku91CAbExMTfPLJJ2VdF60hMzOCb58GmO13A7lCQCqRwKdPffbqEBERVYCXHlF88+ZNREZGIisrS+V4z549X7lS2mBA8xpo52qF+/FpcLI0ZqBDRERUQTQOdsLDw/Hhhx8iODgYEokEBQswSyT5u3bn5uaWbQ0rMZmZEYMcIiKiCqbxbKxJkybB2dkZjx8/hrGxMUJCQnDy5Ek0a9YMJ06cKIcqEhEREb08jXt2zp07h3///RdWVlbQ0dGBjo4O2rRpA19fX0ycOBFXr14tj3oSERERvRSNe3Zyc3NRpUoVAIClpSUePXoEAHB0dMTt27fLtnZEREREr0jjnp369evj+vXrqFmzJlq0aIGlS5dCX18fa9euLbSqMhEREVFF0zjYmTt3LlJTUwEACxcuxAcffIC2bdvCwsICO3bsKPMKEhEREb0KiSiYTvUKnj17hurVqytnZFU2SUlJMDMzg0KhQNWqVSu6OkRERKQGdZ/fZbJzp7m5eVlkQ0RERFTmNB6gTERERFSZMNghIiIircZgh4iIiLSaxsHOyZMnkZOTU+h4Tk4OTp48WSaVIiIiIiorGgc7HTt2xLNnzwodVygU6NixY5lUioiIiKisaBzsCCGKnGL+9OlTmJiYlEmliIiIiMqK2lPP+/TpAyB/d/MRI0bAwMBAeS43NxfXr1+Hp6dn2deQiIiI6BWoHeyYmZkByO/ZMTU1hZGRkfKcvr4+WrZsibFjx5Z9DYmIiIhegdrBzoYNGwAATk5OmDp1Kl9ZERERUaWg8Zid6dOnq4zZefDgAVasWIGjR4+WacWIiIiIyoLGwU6vXr2wefNmAEBiYiLeeecdLFu2DL169cLq1avLvIJEREREr0LjYOfKlSto27YtAGDnzp2wtbXFgwcPsHnzZvz0009lXkEiIiKiV6FxsJOWlgZTU1MAwNGjR9GnTx/o6OigZcuWePDgQZlXkIiIiOhVaBzs1K5dG3v27EFUVBSOHDkCLy8vAEBcXFyJ26sTERERVQSNg52vvvoKU6dOhZOTE9555x20atUKQH4vT+PGjcu8gkRERESvQuNg56OPPkJkZCQuXbqEI0eOKI936tQJP/zwQ5lW7kW+vr6QSCTw9vZWHhNCYP78+ZDL5TAyMkKHDh0QEhJSrvUgIiKiyuOldj23tbWFqakp/P39kZ6eDgBo3rw56tSpU6aVe15gYCDWrl2Lhg0bqhxfunQpli9fjpUrVyIwMBC2trbo0qULkpOTy60uREREVHloHOw8ffoUnTp1gqurK95//33ExMQAAMaMGYMvv/yyzCsIACkpKRgyZAjWrVuH6tWrK48LIbBixQrMmTMHffr0Qf369bFp0yakpaXhjz/+KJe6EBERUeWicbAzefJk6OnpITIyEsbGxsrjAwYMwOHDh8u0cgUmTJiA7t27o3PnzirHIyIiEBsbqxwkDQAGBgZo3749zp49W2x+mZmZSEpKUvkQERGRdlJ7u4gCR48exZEjR2Bvb69y3MXFpVymnm/fvh1XrlxBYGBgoXOxsbEAABsbG5XjNjY2JdbF19cX33zzTdlWlIiIiN5IGvfspKamqvToFIiPj1fZCb0sREVFYdKkSdi6dSsMDQ2Lve757SuA/NdbLx573qxZs6BQKJSfqKioMqszERERvVk0DnbatWun3C4CyA808vLy8N1336Fjx45lWrnLly8jLi4OTZs2ha6uLnR1dREQEICffvoJurq6yh6dgh6eAnFxcYV6e55nYGCAqlWrqnyIiIhIO2n8Guu7775Dhw4dcOnSJWRlZWH69OkICQnBs2fPcObMmTKtXKdOnRAcHKxybOTIkahTpw5mzJiBmjVrwtbWFv7+/so1frKyshAQEIAlS5aUaV2IiIioctI42KlXrx6uX7+O1atXQyqVIjU1FX369MGECRMgk8nKtHKmpqaoX7++yjETExNYWFgoj3t7e8PHxwcuLi5wcXGBj48PjI2NMXjw4DKtCxEREVVOGgc7kZGRcHBwKHKAb2RkJGrUqFEmFVPX9OnTkZ6ejvHjxyMhIQEtWrTA0aNHlft3ERER0dtNIoQQmiSQSqWIiYmBtbW1yvGnT5/C2toaubm5ZVrB1yEpKQlmZmZQKBQcv0NERFRJqPv81niAcnEznVJSUkqcMUVERERUEdR+jTVlyhQA+bOv5s2bpzL9PDc3FxcuXECjRo3KvIJEREREr0LtYOfq1asA8nt2goODoa+vrzynr68PDw8PTJ06texrSERERPQK1A52jh8/DiB/6vePP/7IsS1ERERUKWg8G2vDhg3lUQ8iIiKicqHxAGUiIiKiyoTBDhEREWk1BjtERESk1RjsEBERkVZjsENERERajcEOERERaTUGO0RERKTVGOwQERGRVmOwQ0RERFqNwQ4RERFpNQY7REREpNUY7BAREZFWY7BDREREWo3BDhEREWk1BjtERESk1RjsEBERkVZjsENERERajcEOERERaTUGO0RERKTVGOwQERGRVmOwQ0RERFqNwQ4RERFpNQY7REREpNUY7BAREZFWY7BDREREWo3BDhEREWk1BjtERESk1RjsEBERkVZjsENERERajcEOERERaTUGO0RERKTVGOwQERGRVmOwQ0RERFqNwQ4RERFpNQY7REREpNUY7BAREZFWY7BDREREWo3BDhEREWk1BjtERESk1RjsEBERkVZjsENERERajcEOERERaTUGO0RERKTVGOwQERGRVmOwQ0RERFqNwQ4RERFpNQY7REREpNUY7BAREZFWY7BDREREWo3BDhEREWk1BjtERESk1RjsEBERkVZjsENERERajcEOERERaTUGO0RERKTVGOwQERGRVmOwQ0RERFqNwQ4RERFptTc62PH19UXz5s1hamoKa2tr9O7dG7dv31a5RgiB+fPnQy6Xw8jICB06dEBISEgF1ZiIiIjeNG90sBMQEIAJEybg/Pnz8Pf3R05ODry8vJCamqq8ZunSpVi+fDlWrlyJwMBA2NraokuXLkhOTq7AmhMREdGbQiKEEBVdCXU9efIE1tbWCAgIQLt27SCEgFwuh7e3N2bMmAEAyMzMhI2NDZYsWYJx48aplW9SUhLMzMygUChQtWrV8rwFIiIiKiPqPr/f6J6dFykUCgCAubk5ACAiIgKxsbHw8vJSXmNgYID27dvj7NmzxeaTmZmJpKQklQ8RERFpp0oT7AghMGXKFLRp0wb169cHAMTGxgIAbGxsVK61sbFRniuKr68vzMzMlB8HB4fyqzgRERFVqEoT7Hz++ee4fv06tm3bVuicRCJR+VkIUejY82bNmgWFQqH8REVFlXl9iYiI6M2gW9EVUMcXX3yBffv24eTJk7C3t1cet7W1BZDfwyOTyZTH4+LiCvX2PM/AwAAGBgblV2EiIiJ6Y7zRPTtCCHz++efw8/PDv//+C2dnZ5Xzzs7OsLW1hb+/v/JYVlYWAgIC4Onp+bqrS0RERG+gN7pnZ8KECfjjjz+wd+9emJqaKsfhmJmZwcjICBKJBN7e3vDx8YGLiwtcXFzg4+MDY2NjDB48uIJrT0RERG+CNzrYWb16NQCgQ4cOKsc3bNiAESNGAACmT5+O9PR0jB8/HgkJCWjRogWOHj0KU1PT11xbIiIiehNVqnV2ygvX2SEiIqp8tHKdHSIiIiJNMdghIiIircZgh4iIiLQagx0iIiLSagx2iIiISKsx2CEiIiKtxmCHiIiItBqDHSIiItJqDHaIiIhIqzHYISIiIq3GYIeIiIi0GoMdIiIi0moMdoiIiEirMdghIiIircZgh4iIiLQagx0iIiLSagx2iIiISKsx2CEiIiKtxmCHiIiItBqDHSIiItJqDHaIiIhIqzHYISIiIq3GYIeIiIi0GoMdIiIi0moMdoiIiEirMdghIiIircZgh4iIiLQagx0iIiLSagx2iIiISKsx2CEiIiKtxmCHiIiItBqDHSIiItJqDHaIiIhIqzHYISIiIq3GYIeIiIi0GoMdIiIi0moMdoiIiEirMdghIiIircZgh4iIiLQagx0iIiLSagx2iIiISKsx2CEiIiKtxmCHiIiItBqDHSIiItJqDHaIiIhIqzHYISIiIq3GYIeIiIi0GoMdIiIi0moMdoiIiEirMdghIiIircZgh4iIiLQagx0iIiLSagx2iIiISKsx2CEiIiKtxmCHiIiItBqDHSIiItJqDHaIiIhIqzHYISIiIq3GYIeIiIi0GoMdIiIi0moMdoiIiEirMdghIiIircZgh4iIiLSa1gQ7q1atgrOzMwwNDdG0aVOcOnWqoqtEREREbwCtCHZ27NgBb29vzJkzB1evXkXbtm3RrVs3REZGVnTViIiIqIJJhBCioivxqlq0aIEmTZpg9erVymN169ZF79694evrW2r6pKQkmJmZQaFQoGrVquVZVSIiIioj6j6/dV9jncpFVlYWLl++jJkzZ6oc9/LywtmzZ4tMk5mZiczMTOXPCoUCQP4vjYiIiCqHgud2af02lT7YiY+PR25uLmxsbFSO29jYIDY2tsg0vr6++Oabbwodd3BwKJc6EhERUflJTk6GmZlZsecrfbBTQCKRqPwshCh0rMCsWbMwZcoU5c95eXl49uwZLCwsik3zMpKSkuDg4ICoqKiXej1WkelZNsuuLOlZ9ttV9qumZ9mVr+ySCCGQnJwMuVxe4nWVPtixtLSEVCot1IsTFxdXqLengIGBAQwMDFSOVatWrbyqiKpVq77SH3BFpmfZLLuypGfZb1fZr5qeZVe+sotTUo9OgUo/G0tfXx9NmzaFv7+/ynF/f394enpWUK2IiIjoTVHpe3YAYMqUKfj444/RrFkztGrVCmvXrkVkZCQ+/fTTiq4aERERVTCtCHYGDBiAp0+fYsGCBYiJiUH9+vVx8OBBODo6Vmi9DAwM8PXXXxd6ZVYZ0rNsll1Z0rPst6vsV03Psitf2WVBK9bZISIiIipOpR+zQ0RERFQSBjtERESk1RjsEBERkVZjsENERERajcFOOVq1ahWcnZ1haGiIpk2b4tSpU2qlO3nyJHr06AG5XA6JRII9e/aoXaavry+aN28OU1NTWFtbo3fv3rh9+7ba6VevXo2GDRsqF39q1aoVDh06pHb6F+sikUjg7e2t1vXz58+HRCJR+dja2qpdXnR0NIYOHQoLCwsYGxujUaNGuHz5slppnZycCpUtkUgwYcKEUtPm5ORg7ty5cHZ2hpGREWrWrIkFCxYgLy9P7bonJyfD29sbjo6OMDIygqenJwIDA4u8trT2IYTA/PnzIZfLYWRkhA4dOiAkJESttH5+fujatSssLS0hkUgQFBSkdtnZ2dmYMWMGGjRoABMTE8jlcgwbNgyPHj1Sq+z58+ejTp06MDExQfXq1dG5c2dcuHBB7ft+3rhx4yCRSLBixQq10o4YMaLQn33Lli01Kjs0NBQ9e/aEmZkZTE1N0bJlS0RGRpaatqh2J5FI8N1336lVdkpKCj7//HPY29vDyMgIdevWVdkUubT0jx8/xogRIyCXy2FsbIz33nsPYWFhANT7PimuvamTtqT2Vlr6ktqbOmWX1N40/R59vr2pk7ak9qZu2UW1txkzZpSatqT2pk7ZxbU3ddKW1NbKG4OdcrJjxw54e3tjzpw5uHr1Ktq2bYtu3bohMjKy1LSpqanw8PDAypUrNS43ICAAEyZMwPnz5+Hv74+cnBx4eXkhNTVVrfT29vZYvHgxLl26hEuXLuHdd99Fr169lA9LdQUGBmLt2rVo2LChRunc3d0RExOj/AQHB6uVLiEhAa1bt4aenh4OHTqEmzdvYtmyZWqvjB0YGKhSbsEilf369Ss17ZIlS7BmzRqsXLkSoaGhWLp0Kb777jv8/PPPapUNAGPGjIG/vz+2bNmC4OBgeHl5oXPnzoiOji50bWntY+nSpVi+fDlWrlyJwMBA2NraokuXLkhOTi41bWpqKlq3bo3FixcXe7649Glpabhy5QrmzZuHK1euwM/PD3fu3EHPnj3VqrerqytWrlyJ4OBgnD59Gk5OTvDy8sKTJ0/USl9gz549uHDhgsry8eqkfe+991TawMGDB9VOf+/ePbRp0wZ16tTBiRMncO3aNcybNw+Ghoalpn2+zJiYGPz222+QSCTo27evWmVPnjwZhw8fxtatWxEaGorJkyfjiy++wN69e0tNL4RA7969ER4ejr179+Lq1atwdHRE586dkZqaqtb3SXHt7Z9//ik1bUntrbSyS2pv6tS7pPamyffoi+1N3bTFtTd10hfX3gIDA0tNW1J7U6fs4trbX3/9VWLa0tpauRNULt555x3x6aefqhyrU6eOmDlzpkb5ABC7d+9+6XrExcUJACIgIOCl86hevbr4v//7P7WvT05OFi4uLsLf31+0b99eTJo0Sa10X3/9tfDw8HipOs6YMUO0adPmpdIWZdKkSaJWrVoiLy+v1Gu7d+8uRo0apXKsT58+YujQoWqVlZaWJqRSqdi/f7/KcQ8PDzFnzpwS077YPvLy8oStra1YvHix8lhGRoYwMzMTa9asKTHt8yIiIgQAcfXqVbXLLsrFixcFAPHgwQON0yoUCgFAHDt2TO2yHz58KOzs7MSNGzeEo6Oj+OGHH9RKO3z4cNGrV68S61NS+gEDBqj1563Offfq1Uu8++67aqd3d3cXCxYsUDnWpEkTMXfu3FLT3759WwAQN27cUB7LyckR5ubmYt26dYXSv/h9okl7K+m7SJ32ps53WXHtTZ20JbW34tKr096KSqtJeysqvbrtTZ37Lqm9FZVe3fb2YlpN21pZY89OOcjKysLly5fh5eWlctzLywtnz559rXVRKBQAAHNzc43T5ubmYvv27UhNTUWrVq3UTjdhwgR0794dnTt31rjMsLAwyOVyODs7Y+DAgQgPD1cr3b59+9CsWTP069cP1tbWaNy4MdatW6dx+UD+n9/WrVsxatQotTaGbdOmDf755x/cuXMHAHDt2jWcPn0a77//vlrl5eTkIDc3F4aGhirHjYyMcPr0aY3qHhERgdjYWJW2Z2BggPbt27/2tgfktz+JRKLx3nNZWVlYu3YtzMzM4OHhoVaavLw8fPzxx5g2bRrc3d01ruuJEydgbW0NV1dXjB07FnFxcWqXe+DAAbi6uqJr166wtrZGixYtNHr9XODx48c4cOAARo8erXaaNm3aYN++fYiOjoYQAsePH8edO3fQtWvXUtNmZmYCgErbk0ql0NfXL7Ltvfh9okl7e5XvInXTF9feSktbWnsrKr267a24stVtby+m16S9lXbfpbW3otKr295eTKtpWytz5R5OvYWio6MFAHHmzBmV44sWLRKurq4a5YVX6NnJy8sTPXr00LjH4/r168LExERIpVJhZmYmDhw4oHbabdu2ifr164v09HQhhNCoZ+fgwYNi586d4vr168peIRsbGxEfH19qWgMDA2FgYCBmzZolrly5ItasWSMMDQ3Fpk2b1K57gR07dgipVCqio6PVuj4vL0/MnDlTSCQSoaurKyQSifDx8dGozFatWon27duL6OhokZOTI7Zs2SIkEkmp7eXF9nHmzBkBoFDdx44dK7y8vEpM+7yy6NlJT08XTZs2FUOGDFE77d9//y1MTEyERCIRcrlcXLx4Ue2yfXx8RJcuXZS9cZr07Gzfvl3s379fBAcHi3379gkPDw/h7u4uMjIySk0fExMjAAhjY2OxfPlycfXqVeHr6yskEok4ceKEWvddYMmSJaJ69erKvz/q1D0zM1MMGzZMABC6urpCX19fbN68Wa30WVlZwtHRUfTr1088e/ZMZGZmCl9fXwGgUHsp6vtE3fZW2ndRae1Nne+y4tpbSWnVaW/FpVenvRWXVt32VlR6ddubOr+zktpbcenVaW9FpdWkrZUHBjvloCDYOXv2rMrxhQsXCjc3N43yepVgZ/z48cLR0VFERUVplC4zM1OEhYWJwMBAMXPmTGFpaSlCQkJKTRcZGSmsra1FUFCQ8pgmwc6LUlJShI2NjVi2bFmp1+rp6YlWrVqpHPviiy9Ey5YtNS7Xy8tLfPDBB2pfv23bNmFvby+2bdsmrl+/LjZv3izMzc3Fxo0b1c7j7t27ol27dgKAkEqlonnz5mLIkCGibt26JaYrLth59OiRynVjxowRXbt2LTHt81412MnKyhK9evUSjRs3FgqFQu20KSkpIiwsTJw7d06MGjVKODk5icePH5ea/tKlS8LGxkbloatJsPOiR48eCT09PbFr165S0xf8fR80aJDKdT169BADBw7UqGw3Nzfx+eefF3u+qPTfffedcHV1Ffv27RPXrl0TP//8s6hSpYrw9/dXK/2lS5eEh4eHsu117dpVdOvWTXTr1k3luqK+T9Rtb6V9F5XW3kpLX1J7KymtOu2tqPTqtjd1v4OLa29FpVe3valTdkntrbj06rS34tKq29bKA4OdcpCZmSmkUqnw8/NTOT5x4kTRrl07jfJ62WDn888/F/b29iI8PFzjtC/q1KmT+OSTT0q9bvfu3cpGXPABICQSiZBKpSInJ0fjsjt37lxo7FNRatSoIUaPHq1ybNWqVUIul2tU3v3794WOjo7Ys2eP2mns7e3FypUrVY59++23Gge2QuR/+RY8OPr37y/ef//9Eq9/sX3cu3dPABBXrlxRua5nz55i2LBhJaZ93qsEO1lZWaJ3796iYcOGxfbKqduua9euXWQv2Yvpf/jhB2U7e77t6ejoCEdHx5cu+/mxKMWlz8zMFLq6uuLbb79VuW769OnC09NT7bJPnjwpAKj8Y6G0stPS0oSenl6h8V6jR48uFNyWVn5iYqKIi4sTQuSPORw/frzyXHHfJ+q0N3W+i0pqb6WlL6m9afo9+GJ7Ky69Ou3tZcp+vr0Vl16d9qZO2SW1t+LSq9Pe1Cm7pLZWXjhmpxzo6+ujadOmyhk9Bfz9/eHp6VmuZQsh8Pnnn8PPzw///vsvnJ2dyyTPgvetJenUqROCg4MRFBSk/DRr1gxDhgxBUFAQpFKpRuVmZmYiNDQUMpms1Gtbt25daJrjnTt3NN4MdsOGDbC2tkb37t3VTpOWlgYdHdW/SlKpVKOp5wVMTEwgk8mQkJCAI0eOoFevXhqld3Z2hq2trUrby8rKQkBAQLm3PSB/OnD//v0RFhaGY8eOwcLC4pXyU7ftffzxx7h+/bpK25PL5Zg2bRqOHDmicblPnz5FVFSUWm1PX18fzZs3f+X2t379ejRt2lTtMUpA/u87Ozu7TNqfmZkZrKysEBYWhkuXLqFXr16lfp+U1N5atWr1St9F6nyXFdfeXvZ7sKC9lZa+pPZ2+PBhjct+vr2VVnZJ7a1GjRpql11Ueyut7JLaW25urtplF9XWyl25h1Nvqe3btws9PT2xfv16cfPmTeHt7S1MTEzE/fv3S02bnJwsrl69Kq5evSoAKN/LvjjDoCifffaZMDMzEydOnBAxMTHKT1pamlr1njVrljh58qSIiIgQ169fF7NnzxY6Ojri6NGjaqV/kSavsb788ktx4sQJER4eLs6fPy8++OADYWpqqtbv7OLFi0JXV1csWrRIhIWFid9//10YGxuLrVu3ql3X3NxcUaNGDTFjxgy10wiRP7PCzs5O7N+/X0RERAg/Pz9haWkppk+frnYehw8fFocOHRLh4eHi6NGjwsPDQ7zzzjsiKyur0LWltY/FixcLMzMz4efnJ4KDg8WgQYOETCYTSUlJpaZ9+vSpuHr1qjhw4IAAILZv3y6uXr0qYmJiSi07Oztb9OzZU9jb24ugoCCV9peZmVli2pSUFDFr1ixx7tw5cf/+fXH58mUxevRoYWBgoJy9oenfi+dfK5SUNjk5WXz55Zfi7NmzIiIiQhw/fly0atVK2NnZiaSkJLXK9vPzE3p6emLt2rUiLCxM/Pzzz0IqlYpTp06pVW+FQiGMjY3F6tWrNf7zbt++vXB3dxfHjx8X4eHhYsOGDcLQ0FCsWrVKrfR//vmnOH78uLh3757Ys2ePcHR0FH369BFCqPd9Ulx7Gz16dKlpS2pvpZVdUnv75JNPSkxbWnt7me/RgvZWWtrS2ps6ZRfX3nr37q1WvYtrb+qUXVx7a9u2balpS2pr5Y3BTjn65ZdfhKOjo9DX1xdNmjRRe/r38ePHBYBCn+HDh5eatqh0AMSGDRvUKnvUqFHKOltZWYlOnTq9dKAjhGbBzoABA4RMJhN6enpCLpeLPn36qDVWqMDff/8t6tevLwwMDESdOnXE2rVrNarrkSNHBABx+/ZtjdIlJSWJSZMmiRo1aghDQ0NRs2ZNMWfOHJGZmal2Hjt27BA1a9YU+vr6wtbWVkyYMEEkJiYWeW1p7SMvL098/fXXwtbWVhgYGIh27dqJ4OBgtdJu2LChyPNff/11qekLXkUU9Tl+/HiJadPT08WHH34o5HK50NfXFzKZTPTs2VNlwKimfy+eD3ZKSpuWlia8vLyElZWV0NPTEzVq1BDDhw8XkZGRGpW9fv16Ubt2bWFoaCg8PDyUr0LVSfvrr78KIyOjIv/MS0sfExMjRowYIeRyuTA0NBRubm5i2bJlyoGzpaX/8ccfhb29vfLe586dq2y76nyfFNfe1ElbUnsrLX1J7a20tKW1t5f5Hi1ob6WlLa29qVt2Ue1N3bTFtTd10hfX3tRJW1JbK2+S/90gERERkVbimB0iIiLSagx2iIiISKsx2CEiIiKtxmCHiIiItBqDHSIiItJqDHaIiIhIqzHYISIiIq3GYIeIqAgnTpyARCJBYmJiRVeFiF4Rgx0iIiLSagx2iIiISKsx2CGiN5IQAkuXLkXNmjVhZGQEDw8P7Ny5E8B/r5gOHDgADw8PGBoaokWLFggODlbJY9euXXB3d4eBgQGcnJywbNkylfOZmZmYPn06HBwcYGBgABcXF6xfv17lmsuXL6NZs2YwNjaGp6dnod2miejNx2CHiN5Ic+fOxYYNG7B69WqEhIRg8uTJGDp0KAICApTXTJs2Dd9//z0CAwNhbW2Nnj17Ijs7G0B+kNK/f38MHDgQwcHBmD9/PubNm4eNGzcq0w8bNgzbt2/HTz/9hNDQUKxZswZVqlRRqcecOXOwbNkyXLp0Cbq6uhg1atRruX8iKjvcCJSI3jipqamwtLTEv//+i1atWimPjxkzBmlpafjkk0/QsWNHbN++HQMGDAAAPHv2DPb29ti4cSP69++PIUOG4MmTJzh69Kgy/fTp03HgwAGEhITgzp07cHNzg7+/Pzp37lyoDidOnEDHjh1x7NgxdOrUCQBw8OBBdO/eHenp6TA0NCzn3wIRlRX27BDRG+fmzZvIyMhAly5dUKVKFeVn8+bNuHfvnvK65wMhc3NzuLm5ITQ0FAAQGhqK1q1bq+TbunVrhIWFITc3F0FBQZBKpWjfvn2JdWnYsKHy/2UyGQAgLi7ule+RiF4f3YquABHRi/Ly8gAABw4cgJ2dnco5AwMDlYDnRRKJBED+mJ+C/y/wfEe2kZGRWnXR09MrlHdB/YiocmDPDhG9cerVqwcDAwNERkaidu3aKh8HBwfldefPn1f+f0JCAu7cuYM6deoo8zh9+rRKvmfPnoWrqyukUikaNGiAvLw8lTFARKSd2LNDRG8cU1NTTJ06FZMnT0ZeXh7atGmDpKQknD17FlWqVIGjoyMAYMGCBbCwsICNjQ3mzJkDS0tL9O7dGwDw5Zdfonnz5vj2228xYMAAnDt3DitXrsSqVasAAE5OThg+fDhGjRqFn376CR4eHnjw4AHi4uLQv3//irp1IioHDHaI6I307bffwtraGr6+vggPD0e1atXQpEkTzJ49W/kaafHixZg0aRLCwsLg4eGBffv2QV9fHwDQpEkT/Pnnn/jqq6/w7bffQiaTYcGCBRgxYoSyjNWrV2P27NkYP348nj59iho1amD27NkVcbtEVI44G4uIKp2CmVIJCQmoVq1aRVeHiN5wHLNDREREWo3BDhEREWk1vsYiIiIircaeHSIiItJqDHaIiIhIqzHYISIiIq3GYIeIiIi0GoMdIiIi0moMdoiIiEirMdghIiIircZgh4iIiLQagx0iIiLSav8Pap8fE7Joju4AAAAASUVORK5CYII=", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "plt.plot(np.arange(len(epochs_x)), epochs_acc, marker = '.')\n", - "plt.xlabel('epoch')\n", - "plt.ylabel('test accuracy')\n", - "plt.ylim(0, 100)\n", - "plt.xticks(np.arange(len(epochs_x)))\n", - "for i, txt in enumerate(epochs_acc):\n", - " if i%2 == 0:\n", - " pass\n", - " else:\n", - " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "with open('baseline_exp_set_B1_training_metrics.npy', 'wb') as f:\n", - " np.save(f, np.array(epochs_x))\n", - " np.save(f, np.array(epochs_y))\n", - " np.save(f, np.array(epochs_acc))" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "speck-rescnn", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/tests/test_nonsequential/exp_set_B1/non-sequential-SCNN-example_3.ipynb b/tests/test_nonsequential/exp_set_B1/non-sequential-SCNN-example_3.ipynb deleted file mode 100644 index fca63b7d..00000000 --- a/tests/test_nonsequential/exp_set_B1/non-sequential-SCNN-example_3.ipynb +++ /dev/null @@ -1,1509 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import torch, random\n", - "import torch.nn as nn\n", - "import sinabs.layers as sl\n", - "from tqdm.notebook import tqdm\n", - "\n", - "from tonic.datasets.dvsgesture import DVSGesture\n", - "from tonic.transforms import ToFrame\n", - "from torch.utils.data import DataLoader\n", - "from torch.nn import CrossEntropyLoss\n", - "from torch.optim import Adam\n", - "\n", - "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential\n", - "\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "torch.backends.cudnn.deterministic = True\n", - "random.seed(1)\n", - "torch.manual_seed(1)\n", - "torch.cuda.manual_seed(1)\n", - "np.random.seed(1)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Loading Data" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "batch_size = 3\n", - "num_workers = 1\n", - "epochs = 30\n", - "lr = 5e-4" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "root_dir = \"./DVSGESTURE\"\n", - "_ = DVSGesture(save_to=root_dir, train=True)\n", - "_ = DVSGesture(save_to=root_dir, train=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "n_time_steps = 50\n", - "to_raster = ToFrame(sensor_size=DVSGesture.sensor_size, n_time_bins=n_time_steps)\n", - "\n", - "snn_train_dataset = DVSGesture(save_to=root_dir, train=True, transform=to_raster)\n", - "snn_test_dataset = DVSGesture(save_to=root_dir, train=False, transform=to_raster)" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The transformed array is in shape [Time-Step, Channel, Height, Width] --> (50, 2, 128, 128)\n" - ] - } - ], - "source": [ - "sample_data, label = snn_train_dataset[0]\n", - "print(f\"The transformed array is in shape [Time-Step, Channel, Height, Width] --> {sample_data.shape}\")" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "snn_train_dataloader = DataLoader(snn_train_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True)\n", - "snn_test_dataloader = DataLoader(snn_test_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=False)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Network Module\n", - "\n", - "We need to define a `nn.Module` implementing the network we want the chip to reproduce." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "device: NVIDIA RTX A4000\n" - ] - } - ], - "source": [ - "if torch.cuda.is_available():\n", - " device = torch.device('cuda:0')\n", - " print('device: ', torch.cuda.get_device_name(0))\n", - "else:\n", - " device = torch.device('cpu')" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "class SNN(nn.Module):\n", - " def __init__(self) -> None:\n", - " super().__init__()\n", - "\n", - " self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False)\n", - " self.iaf1 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - " self.pool1 = nn.AvgPool2d(2,2)\n", - " self.pool1a = nn.AvgPool2d(6,6)\n", - "\n", - " self.conv2 = nn.Conv2d(10, 10, 2, 1, bias=False)\n", - " self.iaf2 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - " self.pool2 = nn.AvgPool2d(3,3)\n", - "\n", - " self.conv3 = nn.Conv2d(10, 10, 3, 1, bias=False)\n", - " self.iaf3 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - " self.pool3 = nn.AvgPool2d(2,2)\n", - "\n", - " self.flat = nn.Flatten()\n", - "\n", - " self.fc1 = nn.Linear(810, 100, bias=False)\n", - " self.iaf4 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - "\n", - " self.fc2 = nn.Linear(100, 100, bias=False)\n", - " self.iaf5 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - "\n", - " self.fc3 = nn.Linear(100, 100, bias=False)\n", - " self.iaf6 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - "\n", - " self.fc4 = nn.Linear(100, 11, bias=False)\n", - " self.iaf7 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - "\n", - " self.merge_fc = sl.Merge()\n", - " self.merge_conv = sl.Merge()\n", - "\n", - " def detach_neuron_states(self):\n", - " for name, layer in self.named_modules():\n", - " if name != '':\n", - " if isinstance(layer, sl.StatefulLayer):\n", - " for name, buffer in layer.named_buffers():\n", - " buffer.detach_()\n", - "\n", - " def init_weights(self):\n", - " for name, layer in self.named_modules():\n", - " if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear):\n", - " nn.init.xavier_normal_(layer.weight.data)\n", - "\n", - " def forward(self, x):\n", - " \n", - " con1_out = self.conv1(x)\n", - " iaf1_out = self.iaf1(con1_out)\n", - " pool1_out = self.pool1(iaf1_out)\n", - " pool1a_out = self.pool1a(iaf1_out)\n", - "\n", - " conv2_out = self.conv2(pool1_out)\n", - " iaf2_out = self.iaf2(conv2_out)\n", - " pool2_out = self.pool2(iaf2_out)\n", - "\n", - " merged_conv_out = self.merge_conv(pool1a_out, pool2_out)\n", - "\n", - " conv3_out = self.conv3(merged_conv_out)\n", - " iaf3_out = self.iaf3(conv3_out)\n", - " pool3_out = self.pool3(iaf3_out)\n", - "\n", - " flat_out = self.flat(pool3_out)\n", - " \n", - " fc1_out = self.fc1(flat_out)\n", - " iaf4_out = self.iaf4(fc1_out)\n", - "\n", - " fc2_out = self.fc2(iaf4_out)\n", - " iaf5_out = self.iaf5(fc2_out)\n", - "\n", - " fc3_out = self.fc3(iaf5_out)\n", - " iaf6_out = self.iaf6(fc3_out)\n", - "\n", - " merge_fc_out = self.merge_fc(iaf4_out, iaf6_out)\n", - "\n", - " fc4_out = self.fc4(merge_fc_out)\n", - " iaf7_out = self.iaf7(fc4_out)\n", - "\n", - " return iaf7_out" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "snn = SNN().to(device)" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "snn.init_weights()" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "optimizer = Adam(snn.parameters(), lr=lr, betas=(0.9, 0.999), eps=1e-8)\n", - "loss_fn = CrossEntropyLoss()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Define train and test" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "def train(dataloader, model, loss_fn, optimizer, epochs, test_func, dataloader_test):\n", - " epochs_y = []\n", - " epochs_x = []\n", - " epochs_acc = []\n", - " model.train()\n", - "\n", - " for e in range(epochs):\n", - " losses = []\n", - " batches = []\n", - " batch_count = 0\n", - " train_p_bar = tqdm(snn_train_dataloader)\n", - "\n", - " for X, y in train_p_bar:\n", - " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", - " X = X.reshape(-1, 2, 128, 128).to(dtype=torch.float, device=device)\n", - " y = y.to(dtype=torch.long, device=device)\n", - "\n", - " # forward\n", - " pred = model(X)\n", - "\n", - " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", - " pred = pred.reshape(batch_size, n_time_steps, -1)\n", - "\n", - " # accumulate all time-steps output for final prediction\n", - " pred = pred.sum(dim = 1)\n", - " loss = loss_fn(pred, y)\n", - "\n", - " # gradient update\n", - " optimizer.zero_grad()\n", - " loss.backward()\n", - " optimizer.step()\n", - "\n", - " # detach the neuron states and activations from current computation graph(necessary)\n", - " model.detach_neuron_states()\n", - "\n", - " train_p_bar.set_description(f\"Epoch {e} - BPTT Training Loss: {round(loss.item(), 4)}\")\n", - "\n", - " batch_count += 1\n", - " losses.append(loss.item())\n", - " batches.append(batch_count)\n", - "\n", - " epochs_y.append(losses)\n", - " epochs_x.append(batches)\n", - "\n", - " acc = test_func(dataloader_test, model)\n", - " print(f'Epoch {e} accuracy: {acc}')\n", - " epochs_acc.append(acc)\n", - "\n", - " return epochs_x, epochs_y, epochs_acc\n" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [], - "source": [ - "def test(dataloader, model):\n", - " correct_predictions = []\n", - " with torch.no_grad():\n", - " test_p_bar = tqdm(dataloader)\n", - " for X, y in test_p_bar:\n", - " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", - " X = X.reshape(-1, 2, 128, 128).to(dtype=torch.float, device=device)\n", - " y = y.to(dtype=torch.long, device=device)\n", - "\n", - " # forward\n", - " output = model(X)\n", - "\n", - " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", - " output = output.reshape(batch_size, n_time_steps, -1)\n", - "\n", - " # accumulate all time-steps output for final prediction\n", - " output = output.sum(dim=1)\n", - "\n", - " # calculate accuracy\n", - " pred = output.argmax(dim=1, keepdim=True)\n", - "\n", - " # compute the total correct predictions\n", - " correct_predictions.append(pred.eq(y.view_as(pred)))\n", - "\n", - " test_p_bar.set_description(f\"Testing Model...\")\n", - " \n", - " correct_predictions = torch.cat(correct_predictions)\n", - " return correct_predictions.sum().item()/(len(correct_predictions))*100" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Training loop (HPO)" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "12d134e3b89e41888c9c47892b8e6491", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - " 0%| | 0/359 [00:00" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "y_avg = []\n", - "for y in epochs_y:\n", - " y_avg.append(np.mean(y))\n", - "\n", - "plt.plot(np.arange(len(epochs_x)), y_avg, marker = '.')\n", - "plt.xlabel('epoch')\n", - "plt.ylabel('average loss')\n", - "plt.ylim(0,)\n", - "plt.xticks(np.arange(len(epochs_x)))\n", - "for i, txt in enumerate(y_avg):\n", - " if i%5 == 0:\n", - " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", - " else:\n", - " pass\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAG2CAYAAACZEEfAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABq60lEQVR4nO3dd1hT1+MG8DfsHUW2TAfixK3gqqO46q6jtVWr1bZi62idraPVFrWt7dfROn5urauOOloXrqqooOBEBERBZYhI2DPn9weaGlmJBIH4fp4nT8vNPfecQMx9c+6550iEEAJEREREWkqnohtAREREVJ4YdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirVWjYOXPmDPr06QMHBwdIJBLs27dP6XkhBObMmQN7e3sYGxujW7duCA8PV9onKSkJw4cPh4WFBapVq4YxY8YgLS3tNb4KIiIiqswqNOykp6fD09MTK1asKPL5xYsXY+nSpVi5ciUuXrwIU1NTdO/eHVlZWYp9hg8fjps3b+LYsWM4ePAgzpw5g3Hjxr2ul0BERESVnKSyLAQqkUiwd+9e9O/fH0BBr46DgwO+/PJLfPXVVwAAmUwGW1tbbNiwAcOGDUNoaCgaNGiAwMBAtGzZEgBw+PBh9OrVCw8ePICDg0NFvRwiIiKqJPQqugHFiYqKQlxcHLp166bYJpVK0aZNGwQEBGDYsGEICAhAtWrVFEEHALp16wYdHR1cvHgRAwYMKPLY2dnZyM7OVvwsl8uRlJSEGjVqQCKRlN+LIiIiIo0RQiA1NRUODg7Q0Sn+YlWlDTtxcXEAAFtbW6Xttra2iufi4uJgY2Oj9Lyenh4sLS0V+xTFz88P3377rYZbTERERBUhJiYGjo6OxT5facNOeZo5cyamTJmi+Fkmk8HZ2RkxMTGwsLCowJYRERGRqlJSUuDk5ARzc/MS96u0YcfOzg4AEB8fD3t7e8X2+Ph4NG3aVLFPQkKCUrm8vDwkJSUpyhfF0NAQhoaGhbZbWFgw7BAREVUxpQ1BqbTz7Li5ucHOzg7+/v6KbSkpKbh48SK8vLwAAF5eXkhOTsbly5cV+5w4cQJyuRxt2rR57W0mIiKiyqdCe3bS0tIQERGh+DkqKgohISGwtLSEs7MzJk2ahAULFqBu3bpwc3PD7Nmz4eDgoLhjq379+ujRowfGjh2LlStXIjc3FxMmTMCwYcN4JxYREREBqOCwExQUhM6dOyt+fj6OZuTIkdiwYQOmTZuG9PR0jBs3DsnJyWjfvj0OHz4MIyMjRZmtW7diwoQJ6Nq1K3R0dDBo0CAsXbr0tb8WIiIiqpwqzTw7FSklJQVSqRQymYxjdoiIiKoIVc/flXbMDhEREZEmMOwQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEqQn5+P2bNnw83NDcbGxqhduzbmz58PIYRin1GjRkEikSg9evToUeJxU1NTMWnSJLi4uMDY2Bje3t4IDAxU2ictLQ0TJkyAo6MjjI2N0aBBA6xcubJcXqc206voBhAREVVmixYtwu+//46NGzeiYcOGCAoKwkcffQSpVIovvvhCsV+PHj2wfv16xc+GhoYlHvfjjz/GjRs3sHnzZjg4OGDLli3o1q0bbt26hZo1awIApkyZghMnTmDLli1wdXXF0aNHMX78eDg4OKBv377l84K1EHt2iIiISnD+/Hn069cPvXv3hqurK9599134+Pjg0qVLSvsZGhrCzs5O8ahevXqxx8zMzMTu3buxePFidOzYEXXq1MG8efNQp04d/P7770p1jxw5Em+99RZcXV0xbtw4eHp6FqqbSsawQ0T0hiivyzF+fn5o1aoVzM3NYWNjg/79+yMsLExpn08++QS1a9eGsbExalhZo0O3njgTGFIeL1PjvL294e/vjzt37gAArl69irNnz6Jnz55K+506dQo2NjaoV68ePvvsMzx58qTYY+bl5SE/Px9GRkZK242NjXH27Fmluvfv34+HDx9CCIGTJ0/izp078PHx0eArfAMIEjKZTAAQMpmsoptCRFRuvv/+e1GjRg1x8OBBERUVJXbt2iXMzMzE//73P8U+I0eOFD169BCxsbGKR1JSUonH7d69u1i/fr24ceOGCAkJEb169RLOzs4iLS1Nsc+qVavE6dOnxdJ9Z4XDqF+FcZ3WQtfCWvwRcLfcXq+m5Ofni+nTpwuJRCJ09fSERCIRP/zwg9I+27ZtE3/99Ze4du2a2Lt3r6hfv75o1aqVyMvLK/a4Xl5eolOnTuLhw4ciLy9PbN68Wejo6Ah3d3fFPllZWWLEiBECgNDT0xMGBgZi48aN5fZaqxpVz98MO4Jhh4jeDL179xajR49W2jZw4EAxfPhwxc8jR44U/fr1K1M9CQkJAoA4ffq00vbIhFThOv2gcJl+UNh/tEwAEE6f/J94lJxRpvrK27Zt24Sljb2w7jtV2I9eLqzemSLMLKqJDRs2FFsmMjJSABDHjx8vdp+IiAjRsWNHAUDo6uqKVq1aieHDhwsPDw/FPj/++KNwd3cX+/fvF1evXhXLli0TZmZm4tixYxp9jVWVqudvXsYiInpDlMflmKLIZDIAgKWlpWLbydsJGLIyAAKAPCcLadePQ09qC4l5DdxLzCjbCytnU776ChLPfjCp3wkG1q4wbdgFep7vYP73PxRbplatWrCyskJERESx+9SuXRunT59GWloaYmJicOnSJeTm5qJWrVoACsb1zJo1C0uWLEGfPn3QpEkTTJgwAUOHDsVPP/2k8ddZklhZJs5HJiJWlvla69UU3o1FRPSGmDFjBlJSUuDh4QFdXV3k5+fj+++/x/DhwxX79OjRAwMHDoSbmxsiIyMxa9Ys9OzZEwEBAdDV1S21DrlcjkmTJqFdu3Zo1KgRElKz8N2BWzh4LRapVw7h6an1ELlZ0LN0hM3QBdDV04erlUl5vuwyEUJAlpIOI8lLfQMSHcTLMuEfGo+O7tbQ11V+/sGDB3jy5Ans7e1LrcPU1BSmpqZ4+vQpjhw5gsWLFwMAcnNzkZubCx0d5WPr6upCLper9TpiZZmISkyHm5Up7KXGpe4vhEBadh4S03KwIzAaq87chRCAjgTwG9gYQ1s5q1V/RWPYISJ6Q+zcuRNbt27FH3/8gYYNGyIkJASTJk2Cg4MDRo4cCQAYNmyYYv/GjRujSZMmqF27Nk6dOoWuXbuWWoevry9u3LiBM2f+xbZL0fD7OxQpWXnQkQDjPx4Ji48H45e/ApF8aTcS/1oI51E/IyMnv9xec1lcikqC3z+hkLi0gOz8DuhaWMPAyhk58ZFICdwHsyZvY8zGIEj18mF6cy8+GfkeOnrWxd27dzFt2jTUqVMH3bt3Vxyva9euGDBgACZMmAAAOHLkCIQQqFevHiIiIjB16lR4eHjgo48+AgBYWFigU6dOmDp1KoyNjeHi4oLTp09j06ZNWLJkicqvY0dgNGbuuQ75s7Ay+W13tHa1RGJaDhLTshWPx6nKP2flFg5UcgFM330dJ8Meo4VzdXjYm8PDzgLW5iXfZl/RJEK8MAz/DZWSkgKpVAqZTAYLC4uKbg4RqSA/Px/z5s3Dli1bEBcXBwcHB4waNQrffPMNJBIJAGDevHnYvn07YmJiYGBggBYtWuD7779HmzZtij2uq6sr7t+/X2j7+PHjsWLFCgDA6tWr8ccff+DKlStITU3F06dPUa1atXJ5nZrk5OSEGTNmwNfXV7FtwYIF2LJlC27fvl1sOWtrayxYsACffPJJicefMGEC/vrrL2za8zd+u5yKwHtPAQCNa0rhN7AxGtWUAijoZbj9IAlvN6+D6t0/h+dbvbHPtx3MDCvH9+878alYfPg2jocmAAAM5NnQD9mFsIv+kGfIoGtmiU49+8P73U/w961EJDxNweM9C5CTcBciOx3VrGzR3edt/PrjQtja2iqO6+rqilGjRmHevHkACsLnzJkz8eDBA1haWmLQoEH4/vvvIZVKFWXi4uIwc+ZMHD16FElJSXBxccG4ceMwefJkxfv8ZUIIPHiaidtxqQiMeoLV/0a98u/CUE8H2Xml9yJZmRmivr05POwKwo+HvTnq2JjBUE9X7V4ldah6/q4c7ywiIjWpMtGbu7s7li9fjlq1aiEzMxO//PILfHx8EBERAWtr6yKPGxgYiPz8/3oabty4gbfffhuDBw9WbMvIyECPHj3Qo0cPzJw5s3xfqAZlZGSofUlElcsxQgh8/vnn2LN3L0Z+vx7j9sUgN1/AxEAXU952xyhvV+i9cJnHXmoMSyMrGOhKYGEARCSk4audV/H7B82LPYG/DrGyTPxy7A7+vPwAcgHo6kgwrJUTJnatCxuLgYiVZeJeYgZcrUwUJ+05/eQ4F/kEe1utxZGb8cjMLXjvBAD4dHcEBjTLxDtNHGBpaoCAq6GISkxHrCwT9lJjDBkyBEOGDCmxTXZ2dli/fn2xgSE1Kxd34lMRGpuK23EpuB2bittxqUjLzivxuLYWhnCqbgIrM0NYmRvAyswQ1uaGBT+bGcL62XZZZi7aLTwB+QvdIjoSYEx7NzxMzsTt2FREPUlHYlo2/g3Pxr/hiYr99HQkqGFmgPiUbEW5iroExp4dsGeHqCp65513YGtri7Vr1yq2DRo0CMbGxtiyZUuRZZ7/Wz9+/LhKl2QAYNKkSTh48CDCw8MLnYhPnTqFzp07V5menVGjRuH48eNYtWoVrJ1qw//cRfw0+yt8PGY0Fi1ahLS0NHz77bcYNGgQ7OzsEBkZiWnTpiE1NRXXr19XzAj88uWY8ePHY/OWrajzwbdI1LECALSrWwPzB7dGXYcauHv3Lnbs2AEfHx9YW1vjwYMHWLhwIc6dO4edxwLw2e5I5OTLMbV7Pfh2rvPafy+yjFz8djoCG87dU/Ri9Gxkh6+610NtazOVj5OenYejt+KwN/gRzoY/VgQEPR0J6tqa4XZsKgQKTvrTutfDO54OKh334NVHWHwkDHIBSAB08bCBjo4Et+NSEJNU9IBhfV0J6tiYw8XSBEduxuHFE72uRIKzMzqr3MuyIzAas/bcQL4Q0JVI8MPARkqBJSMnD+HxaQiNTcHtuFTFf2WZuYWOpW7dpWHPDhFpNW9vb6xevRp37tyBu7u74s6i4sYy5OTkYPXq1ZBKpfD09FSpjpycHGzZsgVTpkyp0B4HTVm2bBlmz56NUR9/gieJj6FrZgnTBl3RpN9YAAW9PNeuXcPGjRuRnJwMBwcH+Pj4YP78+UpLH0RGRiIxseAbfHJGjmLG35DfJyv22QGgh3Q96o4aBSMjI/z777/49ddf8fTpU9ja2qJjx444f/486tVzx7d5Rpi55zp+OhqGRjWl6ORedK+bpmXl5mNTwD2sOBmpODG3drPEjJ4eaO5c/OzHxTE11MOAZo4Y0MwRCalZ2B/yCPtCHuLGwxSExqYq9pMLYOHhMCw8HFbC0YomAPjfTlDaZmdhpBg7U//Zf2tZmyoGTRcVVtQJG0NbOaOju3WhXq3nTAz04OlUDZ5O1f5rpxA4dD0WE/4IVto3XwjcS8zQ+OWs0rBnB+zZIaqK5HI5Zs2ahcWLFyvdWfTyZaWDBw9i2LBhyMjIgL29Pfbt24dWrVqpVMfOnTvx/vvvIzo6Gg4Ohb+FV7WeHaDgUs3LlyUkAGb28kBdG3PFZY0apoYw0Ct6dpJYWSaiHqcjPCEVy05EIDEtBwAwvI0zpvXwgNRYX+12zdxzDdsuxUBqrI8DE9rDuUb53KEVK8tEZEI6QmNlWH/uHh7JsgAA9WzNMb1nPXSuZ6PxYLsrKAZT/7xWaLu+jgQ6OiXXJZcL5MoLn6ZHtHVBj8Z28LCzgKWpQaltKOoSXHkr6r3Gnh0iqjDlOYCwvOpW5c4iAOjcuTNCQkKQmJiINWvWYMiQIbh48SJsbGxKrWPt2rXo2bNnkUGnqop6nI6Xz50CwA9/Fx6gLDXWh5WZwbMAVDCOIzY5E0dvxStdFqlrYwa/gY3R0tWy0DFUNa9vQ4TGpiIkJhnjNgdhz3hvmBho9hS1IzAaM/Zcx4tf8R2kRpjiUw8DmtWEbinB41W1r2sFHQkKnfTPTC/9pF9cYPisc221/r3YS41f+79te6kx/AY2LlOvkqawZwfs2aE328u3pb7OAYRlqftV7yyqW7cuRo8eXerA4vv376NWrVrYs2cP+vXrV+Q+VbFnZ8uF+/hm3w2lbRIUXL5JzcpDYlo2nqTnIL+I3oSiSACcnvoWnGuYlrltsbJM9Fl2FolpOejr6YD/DWuqsV6WqzFP0W/FeaVtEgAnv3oLrlZlb3tpShv3Ul5lK4Py7FVizw4RlSpWlqkIG0DBN89Ze26go7t1uX/7KqrumXuuq1z3q9xZBBRc/srOzi71+OvXr4eNjQ169+5d6r5VRWpWLpb6hwMoONELoMiTp1wukJyZWzDfSmo2HqdlIzEtB1djkrH/6iOlYwoAD5OzNBJ27KXGWPF+cwz/v4vYf/URmjhK8XGHWmU6Zr5cYMuF+/D7J7TQcwJArCzrtYSd0sa9lFfZyqAiepVexrBD9Aa7HZtS6JLG6xpAGJVY+HKKXABbL9zHlLfrlTqWoU+fPvj+++/h7OyMhg0bIjg4GEuWLMHo0aMBAOnp6fj+++/Rt29f2NvbIzExEStWrMDDhw+VbiN/+c4ioCAQrV+/HiNHjoSeXuGPybi4OMTFxSmWArh+/TrMzc3h7OystERCaV735cOfj95BQmo2XGuYYMNHrZ+d6AufPHV0JLA0NYClqQHcbc2V2nvw2qNCl1Q0OQNym1o18HXv+vj2wC34/XMbDRws4F3b6pWOFRqbgpl7riMkJrnI5zXd9tKU5aRfGQJDVca1sYgqWH5+PmbPng03NzcYGxujdu3amD9/Pl68wrxnzx74+PigRo0akEgkCAkJUenYu3btgoeHB4yMjNC4cWP8/fffiueS0nOw+Mgd5CbGIGH3d4j+ZQiilwxC7MbJ0MtUby2kV+FWzLfp5Scj0Wf5Wfwb/rjE8suWLcO7776L8ePHo379+vjqq6/wySefYP78+QAKenlu376NQYMGwd3dHX369MGTJ0/w77//omHDhgAKTt63wsIR9SBW6djHjx9HdHS0Iji9bOXKlWjWrBnGji24i6ljx45o1qwZ9u/fr/Lr3xEYjXYLT+D9NRfRbuEJ7AiMVrnsq7j+QIZNAfcAAPP7N4KrlSm8atdQe9yH38DG0H12aam8xmCM8nbFgGY1kS8X+PyPYDxKVm89psycfCz85zb6LDuLkJhkmBvqYX7/Rq+l7VQ5ccwOOGaHKtYPP/yAJUuWFJoc7/vvv1dMjrd582ZERUXBwcEBY8eORXBwMJo2bVricc+fP4+OHTvCz88P77zzDv744w8sWrQIV65cgdShFkauu4Sw8AjEbZ4C8yZvw6R+J0gMTGCW8QgnF4+Dg71dub7uM3ceY8S6S4qfdSTA2w1scT7iCVKfTYjWoa4VpvfwUMy8q0kVOVbp4dMMtF90skxzn6gjXy4w4LdzuPZAhr6eDlj6XrMyHe913NmTmZOPQb+fx63YFDRxlGLnJ14w0i99ba5/wx/j6703EJ1UsLhor8Z2mNunIWwtjF5b2+n1UfX8zbADhh2qWOpMjnfv3j24ubmpFHaGDh2K9PR0HDx4ULGtbdu2cKnbAHc93sfj1Gyk/f0T2tezxe//tx43HsgwZddVpGblYX6/hvjQy1WTL1NJbr4cPf/3LyIS0jCkZcG8JM9PPknpOVh+IgKbL9xDbn7Bx1NfTwd85VNPY7cjRyelo9OPp5TuyinPsPGi7Lx8jFx3CRfuJhV6btvYtvCqXUPjdW4KuIc5f92EuZEe/L/sBBtzI43XUR5ikjLQZ/lZJGfkYkhLRywa1KTYActP0rKx4FAo9gY/BFBwl9V3/RqhWwPbIvcn7aDq+ZuXsYgqmLe3N/z9/XHnzh0AUEyO17NnzzIdNyAgAN26dVPa1rBVB+w/dgqPU7Phbm2KrKggtGjSEKOG9Mfwzk2QsWs6Mu4E4Odjd/A0PadM9ZdkU8B9RCSkwdLUAF/3bqB0OcXS1ABz+jTAiS/fwoBmNSGRAPuvPkLXJacwb/9NPEkrfXDxy4QQiEhIxfpzURizIRA+S87g5a95z8cqlafUrFyM3hBYZNDRkaBcxo8kpGThx2eT103rXq/KBB0AcLI0wbL3mkFHAuwMeoCtFwtf6hNCYGdQDLouOY29wQ+hIwE+aueKo1M6MeiQAgcoE1WwGTNmICUlBR4eHkqT4w0fPrxMx42Li1NahPDQtVgcCM9ATupTtHazxAIfR9T7Kg0LFy7EggULsGjRIvz99z/4+puvoWNkhiXHHDC/f6OyvrxCEtOy8euxgmA3rXu9Yiegc7I0wS9Dm+LjDm5YdDgMZ+48xobz9/Dn5Qf4pGMtjOngVuI8LE/SsnE2IhFnwxNxNiIRsc8mjyvJoeuxaO1mWS7zrSSkZmHUukDcik2BqYEuhrV2xvpzUYrBvpamBuWyEOb8Q6FIzc6Dp6MU77dx0fjxy1uHutaY2t0Diw7fxrcHbqK+vQVauBTMbnz3cRpm7b2uCI8N7C2wcFBjNHGsVoEtpsqIYYeogqk6OV5ZbDx/D/MO3ES+XMBQTwebRrdG0uN4AEC/fv0weXLBNP9NmzbFP/6ncSXkH2x1aYz3WjujgYNmL+3+eDgMqdl5aFTTAoNbOpW6f0MHKTaNbo1zEYlY+M9tXH8ow8/H7mDThfuY2LUuOrpb4cHTTDhIjfHgaSb+DX+Mf8MTcSs2Rek4Bno6aO1qiQ51rdC+rhWuPZDhm70Fc5c8vw17y4X7ePA0A/8b1uyVZgEuTlRiOkasu4iYpExYmRlg/ajWaOwoxccd3HA1Jhlz/rqJhNRsTNl5Fas+aFHqnWiqOnPnMQ5cfQQdCfD9gMblNmleefu0Uy1cf5iMv6/HYdymIMzt0wDXH8qwMeA+cvLkMNbXxeS362J0OzelBUeJnmPYIapgU6dOxYwZMzBs2DAAQOPGjXH//n34+fmVKezY2dkhLi4OPx65jRUnIwEADarLkeLqCCN9XVhZWUFPTw8NGjRQKufd0hMRe49ALoBvD9zE9nFtNTax27UHydh5OQYAMK9PQ7VOvu3qWOEv33Y4dD0WPx4JQ3RSRqHJ8V5W394CHZ+Fm1aulkoDXBs6SPFWvf/mLrkUlYRpf17DqbDH6L/iHNaMaIE6NuYlHF01V2OS8dGGQCSl58Clhgk2jW4Nl2dz0jy/ndheaozBqwJw7FY8VpyMwOdd65a53qzcfMz+q+D3M9LbtVwGeb8uEokEi9/1RNC9p0hIzcYX20MUz3Vyt8aC/o3gZPn6biGnqocRmEiDYmWZOB+ZiFiZ6rfKvurkeKVp07Ytlm/Zpwg6X77tjvyYa/Dy8gIAGBgYoFWrVggLU16M8M6dO2jTxB1G+jq4GJWEQ9djCx37VcjlAvP234QQwIBmNV9paQEdHQn6eDrg+JROmPK2e5H79Gpsh/8Na4rAr7vhn4kdMLNXfXSoa13knTz2UmPFeKF+TWti92fecJAaISoxHf1XnMfxW/Fqt/FFp8IS8N6aC0hKz0HjmlLs/sxbEXRe5OlUDQv6FVwyXHL8Dk6GJRTaR12/nYrE/ScZsLUwLPZ3VZWkZhVMcvgiiQTwG8igQ6Vj2CHSkFedN+X55HiHDh3CvXv3sHfvXixZsgQDBgxQ7JOUlISQkBDcunULABAWFoaQkBDExcUp9hkxYoRiCYSMnDxk1PXB3ZBzSL20B583N8aTf7ciKChIafK8qVOnYseOHVizZg0iIiKwfPlyHDhwAF9N+gKfdaoDAPjhUCgyc/LL/PvZF/IQV6KTYWKgixk9Pcp0LAM9HbR0LXpV6g/buqJf05qwNjcs8vmSNKopxf7P26O1qyXSsvMwdnMQlp8Ix6vctLrnygN8vDEIGTn56FDXCtvGtYWVWfFtGtLKCcPbOEMIYOK2YNxLTFe7zuciH6dh5amCkDvnnYYwN9LcJbmKUtQklEIA95+oNwcPvZkYdog0oLhlF1Tp4SltcjwA2L9/P5o1a6ZYumDYsGFo1qwZVq5cqdgnOjoasbGxSErPwftrLuJmvj3s+0+DSdRpzPqwJ/7880/s27cPjRr9N+h4wIABWLlyJRYvXozGjRvj//7v/7B79260b98en3SqhZrVjPFIloXfT0eW6feTlp2Hhf8UrFc1oUsdxZwnZeFmZYqXr4JpYkZcKzNDbPm4DT5oWxA8fjp6B75/XEH6s7l/SiOEwKrTkZiy8yry5AL9mzpg7chWKg0+ntunIZo7V0NKVh4+2XxZ5Tpfrn/2vhvIyZejk7s1ejUu3/mSXpfy+nvTm4Hz7IDz7FDZ7b/6CF9sCy60vbzmTSlKrCwTgVFJ+OloGKKTMiE11se6US3RwuXVV6L+53osPtt6BQZ6OvCf0umVLxcs/Oc2Vp6OhEsNExyd3BGGeqVPDqeK8l4gcdulaMz56wZy8wU87MyxZkTLEn8HcrnA93+HYu3ZKADA2A5umNmzvloDjuNTsvDOsrN4nJqN3k3ssfy9ZmqNmdoX/BCTdoTAUE8HRyd3LPKyWVVV1RfEJM3jpIJqYNihsniSlo0Bv51DdJJyL46OBDg3o8trmaV1R2A0Zuy5rpg7Rmqsh92feZd5gK0QAsP/7yLORz5Bj4Z2WPlhC7WPEZWYDp9fTiM3X+D/RrTU+Nwn5T0jbtC9JHy65QoS07JR3UQfK95vDu86hddqys7Lx1e7ruHAs4Uyv+5VH2M7vtoiloH3kvDe6gvIkwvM6uWBcR1rq1ROlpGLrktOITEtB1/5uGNCl7IPdK5sOAMyvYiTCtJr5+rqColEUujh6+sLoGDelw8//BB2dnYwNTVF8+bNsXv37lKPu2LFCri6usLIyAht2rTBpUuXlJ7/5JNPULt2bRgbG8Pa2hr9+vXD7du3y+U1viw1Kxcj11961pOip9TN3sRRCjsNXK4pTawsUynoFLQrD6YamLNFIpFg7rO7pg7fjMO5iES1j7Hg4C3k5gt0crdG1/o2ZW7Ty14cZFweWrpa4sDn7dDEUYqnGbn4cN0lbDgXpTSO5/lkgQeuPoK+rgS/Dm36ykEHAFq5WmJun4K75Bb+c1vl3/viI7eRmJaD2tamZaq/Mivvvzdpp0oddlRZIFEIgTlz5sDe3h7Gxsbo1q0bwsPDK7DVb67AwECs/DsQjhM2w9F3M+yGLQAAxQrTI0aMQFhYGPbv34/r169j4MCBGDJkCIKDC1/+eW7Hjh2YMmUK5s6diytXrsDT0xPdu3dHQsJ/d6u0aNEC69evR2hoKI4cOQIhBHx8fJCfX/ZBtSXJys3H2E1BuPEwBTVMDbBnfDucm9EFc96pD10JEBIjw7pz98q1DQDwb3hiodmA5QIamw24np05PmxbMBndtwduIi9f9bvEToYlwP92AvR0JJjTp4HGbmF/3eylxtj5iZdiccp5B25h2p/XcP9JOv6+/giDfj+PcxFPYGqgi3WjWqF/s5plrvODti54t4Uj5AKY8McVPHha8t8zOPop/rhUMCh+Qf/GGrtUSKQNKnXYWbRoEX7//XcsX74coaGhWLRoERYvXoxly5Yp9lm8eDGWLl2KlStX4uLFizA1NUX37t2RlVX6bKmkOflygeN3M+B3Oh66ptWha1Yd6eGXoFfNHnU9WwEoWJjy888/R+vWrVGrVi188803qFatGi5fvlzscZcsWYKxY8fio48+QoMGDbBy5UqYmJhg3bp1in3GjRuHjh07wtXVFc2bN8eCBQsQExODe/fuldvrzcuXY8IfwbhwNwlmhnrYOLo1alubwV5qjNHta+Gbdwq+lf/wdygCIstvBfGYpAwsPly4F0vTAzcnd3NHdRN93IlPw5YL91Uqk5Mnx/wDBXePfdTOFbWtzTTWnopgpK+LJUM88U3v+tCRALsuP0CnH09h/NZg3IlPg6mBLraP80KHutYaqU8ikWBB/0ZoXLOgR+nTLZeRlVt0gM/Ll+PrvTcgBDCwec3XNk6MqKqo1GHn/Pnz6NevH3r37g1XV1e8++678PHxUVzGEELg119/xTfffIN+/fqhSZMm2LRpEx49eoR9+/ZVbOPfEBk5edh4/h66/HwKM/f+N8GbyM9F+q1TMGvyNj7eeBmRj9Pg7e2NHTt2ICkpCXK5HNu3b0dWVhbeeuutIo+dk5ODy5cvK63vpKOjg27duiEgIKDIMunp6Vi/fj3c3Nzg5FT67LyvQi4XmL77Oo6HxsNATwf/N7JloQnbRnm7KnoBJvxxBY+SNX977OPUbHy49iIS03JgY26ouIT2fOCmJrv5pSb6+Kp7PQDAkmN3VFqfasP5KNxNTIeVmSG+0MAkeZWBRCLBxx1q4ZehTQs9l5mbDytzA43WZ6Svi5UftoClqQFuPEx5FmgKD7PcGHAft2JTIDXWx6xe9TXaBiJtUKnDTmkLJEZFRSEuLk7pZCiVStGmTZtiT4YAkJ2djZSUFKUHqSchJQs/HrkNL78TmLv/Ju4/yYCFkR6eX6TIuHMB8qw0mDbqitC4VPT89V90Hu+H7Jwc1KhRA4aGhvjkk0+wd+9e1KlTp8g6EhMTkZ+fr7S+EwDY2toqzS8DAL/99hvMzMxgZmaGf/75B8eOHYOBgWZPPEBBwP7+71DsvvIAujoSrHi/OdrWKvwtWiKR4IcBjdHA3gJP0nPwWQnfyl9FSlYuRq67hHtPMuBY3Rj7J7THuRldsG1sW5yd0blc7lAZ1soZDewtkJKVh5+O3ilx34TULCz1jwAATO9RTyvmeXlRUXP4aPLS4YtqVjPG8meLYe6+8gCbX+pZi5VlYsnRgokhZ/T0KHEuH6I3VaUOO8+n0Pfw8IC+vj6aNWuGSZMmKRZIfH7CU+Vk+CI/Pz9IpVLFo7x6ALRRWFwqpu66ivaLTmLFyUjIMnPhUsME3/VriAuzumLhoMbQlUiQdu0oTGq1xNeDvdHJ3Ro5+XL4zZ+HS7djsHzzHgQFBWHKlCkYMmQIrl+/XuZ2DR8+HMHBwTh9+jTc3d0xZMiQcrmUueJkhOK24sWDmuDtEu4sMjbQxaoPW6CaiT6uPpBh7l83X2lyupdl5uTj4w1BuBWbAiszA2we0wZ2UqNyH7ipqyPBvL4NAQDbA6Nx46Gs2H0XHw5DWnYePJ2qYVBzx3JpT0V63XO+eNexwsyeBT023x24hcB7/62a/t2BW0jPyUdz52oYqsJaY0Rvokoddl5cIPHKlSvYuHEjfvrpJ2zcuLFMx505cyZkMpniERMTo6EWaychBM6GJ2LEukvo/usZ7Lr8ADn5crR0qY6VH7TAiS/fwggvV5gY6GFoK2dsf782cqKvYvl3X+Kzt+pgw0etMLN9daReOQjTtz/HjzcMsD1SgklTZ6Fly5ZYsWJFkfVaWVlBV1cX8fHKU/bHx8fDzk55ojSpVIq6deuiY8eO+PPPP3H79m3s3btXo7+HzRfuK3o0Zr/TAINalH4Sd7I0wbJn38p3BMUoBpC+qtx8OXz/uIJL95Jg/myskJvV65tHpbWbJfp6OkAIPFv6oXB4C45+ij8vPwAAzOvTQGOLWlYm9lJj+A0sCPZA+Vw6fNnHHdzQx9MBeXKB8VuvID4lCydux+OfG3HQ1ZHg+wGNtfJ3TaQJlXoh0NIWSHx+wouPj4e9vb2iXHx8PJo2bVrscQ0NDWFoyK7eksTKMhEen4bw+FTsuvwAt+NSARTMHdOzkT0+7uCGZs5FT9d/aPc22NjY4MMhAwEUXNLxdimY76VXEwccjwW2XYrBsVsJkGfkFXvXlIGBAVq0aAF/f3/0798fACCXy+Hv76+05MHLhBAQQiA7u/RxJao6cPUR5jxbVPHzLnUwpr2bymU71LXG1O4eWHT4NubtvwkPOwu0cCn6d1cSuVxg6q6rOHE7AYZ6Olg7qhUaOrz+xR1n9vLAsVvxCLr/FPuvPkK/pv/defR8/SsAeLeFY7HvEW0wtJUzOrpbv7Y5XyQSCRYNaozw+FTcjkvF6A2BiE8p6L0c094N9e05RxhRcSp1z05pCyS6ubnBzs4O/v7+iudTUlJw8eJFxWKHpL7tl6Lh7XcCI9ZdwvxDobgdlwoTA12M8nbF6amdsWJ482JPYnK5HOvXr8fIkSOhp/dflvbw8ECdOnVwZ/cSzGljAHtJMu6e2IbggDOIMmuAmKSCsQ5du3bF8uXLFeWmTJmCNWvWYOPGjQgNDcVnn32G9PR0fPTRRwCAu3fvws/PD5cvX0Z0dDTOnz+PwYMHw9jYGL169dLI7+P0nceYsjMEQgAftnV5pUUVP+1UC70a2yE3X2D81stISFXvEpsQAt8dvIV9IY+gpyPB7x80R2u3V58ZuSzspcbw7VwwyZ3f37eVljTYfeUBrj6QwcxQD9N61KuQ9r1Or3vOFxMDPaz6sAWM9HRw81EKEtNyAACO1TjnDFFJKnXYKW2BRIlEgkmTJmHBggWKuVtGjBgBBwcHRU8Aqef5Gk8vXpyQANg3vh3m9W1Y6nIBx48fR3R0NEaPHq20XV9fH3///Tesra0x89MPcG3pOBjfPwfbPlMQYegOn1/OYM2Zu4iMjETUg1jFyuFDhw7FTz/9hDlz5qBp06YICQnB4cOHFeO0jIyM8O+//6JXr16oU6cOhg4dCnNzc5w/fx42NmWfwO7y/SR8uvkycvMF+ng64Nu+DV9prhiJRILF73qiro0Z4lOy4bv1CnLyVJ+vZql/BDacvwcA+GmwJ7p4aHYWYnV93KEWnCyNEZeShd9OFQxETsnKxaLDBQNlv+haBzbm5T+h4pvIQE8H2S/NdfTtgVsqrcNG9Kaq1MtFpKamYvbs2di7dy8SEhLg4OCA9957D3PmzFHcaSOEwNy5c7F69WokJyejffv2+O233+Durvq3by4X8Z/dl2Pw5a5rhbaX1xpPEQlpmLX3Oi5FFQy4dJAaITYlC0IUXDLzG9i4wta+uR2XgiErA5CSlYdO7tZYM6IlDPTK9v3g7uM09Ft+DqnZeRjp5YJv+zUqtczG8/cw99mloW/7NsRIb9cytUFTjt6Mw7jNl2Ggq4NjUzpiy4X7WPNvFGpZmeLwpI5l/l1R0c5HJuL9NRcLbX+d67ARVRZcG0sNDDsF8vLl6LP8HEJjlW/F15VIcHZG53LrqpfLBXZdjsGCQ6FIzVJe5VlHAvzl2w6Nakpf6+y70U8yMGjleTxOzUYLl+rYPKY1TAw0M8TNPzQeYzYGAQB+HuxZ4kDnv0IeYuL2EADApG51Mamb+pfQyosQAiPWXcK/4Ylo5GCBW7EpkAtg/Uet0Lme5peFoAKxsky0W3gC8hc+ucv73yhRZcW1sUhtK05GIjQ2BUZ6OuU6Qd3LdHQkGNrKGT++26TQc3IB9Fl+Di0WHMf7ay7guwO3sCsoBjceyoqctyZWlqm4BPYqYmWZ+PvaIwxbE4DHqdnwsDPHupGtNBZ0AKBrfVtMfDbJ3qy914u9hfvk7QR8ufMqgIJJCidWson5CtbNagAdCXDjUYri5JuQwtnLy1NF3AlGVNVV6rux6PW5GpOMpScK1hRb9G4TtHazfO0rC3s6VYOOBErfWIGCMUNJ6Tk4H/kE519YekFXRwI3K1N42Jmjvr0FHqdmYVPAfchf8RLYjsBozNxzXVG/pakBNo1uDamJ5ifEm9i1Lm48lMH/dgI+2XwZBz5vD0vT/yZBDLyXhE+3XEaeXGBAs5qY807lXFfK1FCv0Lpcs/bcQEd3a558y9HrvhOMqKrjZSzwMlZmTj56L/sXdx+n450m9lj+fvMKa8uOwGjM2nMD+UIovrH2a1oT4fFpCI1LQWhsCm7HpiI0LgXJGbmlHq+asb5Kc4/I5QLJmcrH05EA52Z0KbcTiSwzF/1XnENUYjq8a9fAptGtoaerg1uPUjB0dQBSs/LQ1cMGKz9sAX3dytkJy/EjRFSRVD1/s2eHsPCfUNx9nA5bC0Ms6F/6gNnyVNw31saOUjR2/G9OGSEEElKzC8JPXCr+vfMY54pYcPPlAKOO59P/l1fYkRrrY9WHLdB/xTmcj3yCOftvopVLdXx3sGDsUmtXS6wY3rzSBh3gv5mEXx4/Ul4zCRMRvQr27ODN7tk5fecxRq4rWFh185jWGlux+XUratCmjgTYMqZNkesYvexxajY+WHuxQgZ9/n09FuO3XlHaZi81wpHJHWFRBdaUKqo3rqLuoCOiNwt7dqhUyRk5mLrrvwGwVTXoAP8N2nz5pOtdx0ql8nVtzYss/zrGQjRzrgYJoDS3UXxKFtKz86pE2OH4ESKq7Bh23lBCCHy97wYSUrNRy9oU03t4VHSTyqysJ92KOmlHJabj5e7V8r6Epmn2UuMq01YievMw7Lyh9l99hEPXYqGnI8GvQ5vC2EC3opukEWU96VbESZvjXoiIylflHflI5eZRcia+2VewqOUXXeuiiWO1im3QG47zphARlS/27Lxh5HKBqX9eRWpWHpo6VcP4t2pXdJMIHPdCRFSeGHbeMBvO38O5iCcw1tfFkiGe0KvEtzW/aTjuhYiofPBM9wYJj0/FwsO3AQBf966PWtZmFdwiIiKi8sew84bIyZNj8s4Q5OTJ8VY9awxvw3lQiIjozcCw84ZY6h+OGw9TUM1EH4sHNamU6ywRERGVB4adN8Dl+0n47VQEAMBvQGPYWBhVcIuIiIheH4YdLZeenYcpO69CLoCBzWqiZ2P7im4SERHRa8Wwo+UWHArF/ScZqFnNGPP6Nazo5hAREb12vPVcS8XKMrHn8gNsuxQNiQT4abBnlVhniYiISNMYdrTQjsBozNxzXbH8QPvaVvCqXaNiG0VERFRBeBlLy8TKMpWCDgCci0xErCyz4hpFRERUgRh2tExUYrpS0AH+W0GbiIjoTcSwo2XcrEzx8hQ6XEGbiIjeZAw7WsZeaoz2dawUP3MFbSIietNxgLIWepRcMD5nYte6GNbaiUGHiIjeaAw7Wib6SQYiH6dDV0eC0e3dIDXm7eZERPRm42UsLXMyLAEA0NKlOoMOERERGHa0zonbBWGni4dNBbeEiIiocmDY0SIZOXkIuPsEAMMOERHRcww7WuR8xBPk5MnhWN0YdWzMKro5RERElQLDjhZ5Pl6ni4cNJC9PtkNERPSGYtjREkIInHw2XqdzPV7CIiIieo5hR0uExafikSwLRvo6XPSTiIjoBQw7WuL5XVjeta1gpK9bwa0hIiKqPBh2tITiEhbvwiIiIlLCsKMFkjNycPn+UwBA53rWFdwaIiKiyoVhRwucCU+EXADutmZwrM7VzYmIiF7EsKMFeAmLiIioeAw7VVy+XODU8/l1eMs5ERFRIQw7VVxITDKeZuTCwkgPLVyqV3RziIiIKh2GnSru+SWsju7W0NPln5OIiOhlPDtWcS8uEUFERESFMexUYfEpWbj5KAUSCdDJnbecExERFYVhpwp7fgnL07EaapgZVnBriIiIKieGnSrs+RIRvIRFRERUPIadKio7Lx9nIxIBMOwQERGVhGGniroUlYSMnHzYmBuioYNFRTeHiIio0mLYqaJO3n4MAHirnjUkEkkFt4aIiKjyYtiponjLORERkWoYdqqgqMR0RCWmQ19XgvZ1ecs5ERFRSRh2qqDnd2G1drOEmaFeBbeGiIiocmPYqYIUq5xz4U8iIqJSMexUMenZebgY9QQAx+sQERGpgmGnijkbkYjcfAGXGiZwszKt6OYQERFVegw7VcyLl7B4yzkREVHpGHaqECEEbzknIiJSE8NOFXLzUQriU7JhYqCLNrUsK7o5REREVQLDThVy6lmvTrs6VjDU063g1hAREVUNDDtVyAneck5ERKQ2tcPOyZMny6MdVIqk9BwExyQDADp7cNZkIiIiVakddnr06IHatWtjwYIFiImJKY82URFO30mAEEB9ewvYS40rujlERERVhtph5+HDh5gwYQL+/PNP1KpVC927d8fOnTuRk5NTHu2jZ048W+W8C3t1iIiI1KJ22LGyssLkyZMREhKCixcvwt3dHePHj4eDgwO++OILXL16tTza+UbLy5fjNG85JyIieiVlGqDcvHlzzJw5ExMmTEBaWhrWrVuHFi1aoEOHDrh586am2vjGC45JRkpWHqqZ6KOpU/WKbg4REVGV8kphJzc3F3/++Sd69eoFFxcXHDlyBMuXL0d8fDwiIiLg4uKCwYMHa6SBDx8+xAcffIAaNWrA2NgYjRs3RlBQkOJ5IQTmzJkDe3t7GBsbo1u3bggPD9dI3ZXF87uwOrlbQ1eHsyYTERGpQ+2w8/nnn8Pe3h6ffPIJ3N3dERwcjICAAHz88ccwNTWFq6srfvrpJ9y+fbvMjXv69CnatWsHfX19/PPPP7h16xZ+/vlnVK/+X+/G4sWLsXTpUqxcuRIXL16EqakpunfvjqysrDLXX1k8XyKCl7CIiIjUp6dugVu3bmHZsmUYOHAgDA0Ni9zHyspKI7eoL1q0CE5OTli/fr1im5ubm+L/hRD49ddf8c0336Bfv34AgE2bNsHW1hb79u3DsGHDytyGivYwORO341KhIyno2SEiIiL1qN2z4+/vj/fee6/YoAMAenp66NSpU5kaBgD79+9Hy5YtMXjwYNjY2KBZs2ZYs2aN4vmoqCjExcWhW7duim1SqRRt2rRBQEBAscfNzs5GSkqK0qOyet6r09y5OqqZGFRwa4iIiKoetcOOn58f1q1bV2j7unXrsGjRIo006rm7d+/i999/R926dXHkyBF89tln+OKLL7Bx40YAQFxcHADA1tZWqZytra3iuaL4+flBKpUqHk5OThpttyY9XyKiMy9hERERvRK1w86qVavg4eFRaHvDhg2xcuVKjTTqOblcjubNm+OHH35As2bNMG7cOIwdO7bM9cycORMymUzxqKyTI2bl5uNcxBMAXCKCiIjoVakdduLi4mBvb19ou7W1NWJjYzXSqOfs7e3RoEEDpW3169dHdHQ0AMDOzg4AEB8fr7RPfHy84rmiGBoawsLCQulRGV24+wSZufmwlxqhvr15RTeHiIioSlI77Dg5OeHcuXOFtp87dw4ODg4aadRz7dq1Q1hYmNK2O3fuwMXFBUDBYGU7Ozv4+/srnk9JScHFixfh5eWl0bZUhINXHwEAWrtZQiLhLedERESvQu27scaOHYtJkyYhNzcXXbp0AVAwaHnatGn48ssvNdq4yZMnw9vbGz/88AOGDBmCS5cuYfXq1Vi9ejUAQCKRYNKkSViwYAHq1q0LNzc3zJ49Gw4ODujfv79G2/K6bb8UjT+vPAQA7L/6CN61a2BoK+cKbhUREVHVIxFCCHUKCCEwY8YMLF26VLEelpGREaZPn445c+ZovIEHDx7EzJkzER4eDjc3N0yZMgVjx45Vas/cuXOxevVqJCcno3379vjtt9/g7u6uch0pKSmQSqWQyWSV4pJWrCwT7RaegPyFv4yuRIKzMzpzEVAiIqJnVD1/qx12nktLS0NoaCiMjY1Rt27dEm9Fr+wqW9g5H5mI99dcLLR929i28KpdowJaREREVPmoev5W+zLWc2ZmZmjVqtWrFqcSOFYv3HujK5HA1cqkAlpDRERUtb1S2AkKCsLOnTsRHR2tuJT13J49ezTSsDfZrUepSj/rSiT4YWAjXsIiIiJ6BWrfjbV9+3Z4e3sjNDQUe/fuRW5uLm7evIkTJ05AKpWWRxvfONsuFdxa/2FbF2wb2xZnZ3Tm4GQiIqJXpHbY+eGHH/DLL7/gwIEDMDAwwP/+9z/cvn0bQ4YMgbMzT8hl9eBpBs6EPwYAfNzBDV61a7BHh4iIqAzUDjuRkZHo3bs3AMDAwADp6emQSCSYPHmy4pZwenU7A2MgBNCuTg241DCt6OYQERFVeWqHnerVqyM1tWBMSc2aNXHjxg0AQHJyMjIyMjTbujdMXr4cO4IKlq54rzV7yYiIiDRB7QHKHTt2xLFjx9C4cWMMHjwYEydOxIkTJ3Ds2DF07dq1PNr4xjgV9hjxKdmwNDXA2w1sSy9AREREpVI77CxfvhxZWVkAgK+//hr6+vo4f/48Bg0ahG+++UbjDXyTPB+Y/G4LRxjq6VZwa4iIiLSDWmEnLy8PBw8eRPfu3QEAOjo6mDFjRrk07E0TK8vEybAEAMDQVk4V3BoiIiLtodaYHT09PXz66aeKnh3SnJ2BDyAXQBs3S9S2Nqvo5hAREWkNtQcot27dGiEhIeXQlDdXvlxg57OBye+34cBkIiIiTVJ7zM748eMxZcoUxMTEoEWLFjA1Vb49ukmTJhpr3JviTPhjPEzORDUTfXRvaFfRzSEiItIqaoedYcOGAQC++OILxTaJRAIhBCQSCfLz8zXXujfE9mcDkwc2c4SRPgcmExERaZLaYScqKqo82vHGSkjJwvHQgoHJ77XmwGQiIiJNUzvsuLi4lEc73li7Lj9AvlygpUt11LU1r+jmEBERaR21w86mTZtKfH7EiBGv3Jg3jVwusD2w4BLWMM6YTEREVC7UDjsTJ05U+jk3NxcZGRkwMDCAiYkJw44azkUmIiYpE+ZGeujd2L6im0NERKSV1L71/OnTp0qPtLQ0hIWFoX379ti2bVt5tFFrbb9UcLv5gGY1YWzAgclERETlQe2wU5S6deti4cKFhXp9qHiJadk4eisOADCsFS9hERERlReNhB2gYHblR48eaepwWm/35QfIzRfwdKqGBg4WFd0cIiIiraX2mJ39+/cr/SyEQGxsLJYvX4527dpprGHaTAiB7YHPZkzm7eZERETlSu2w079/f6WfJRIJrK2t0aVLF/z888+aapdWu3A3CVGJ6TAz1MM7TRwqujlERERaTe2wI5fLy6Mdb5Rtz2ZM7tvUAaaGav8JiIiISA0aG7NDqnmanoPDNwoGJr/PuXWIiIjKndphZ9CgQVi0aFGh7YsXL8bgwYM10ihttvvKA+Tky9GopgUa1ZRWdHOIiIi0ntph58yZM+jVq1eh7T179sSZM2c00iht9eLA5PfYq0NERPRaqB120tLSYGBgUGi7vr4+UlJSNNKoymbhwoWQSCSYNGkSAODevXuQSCRFPnbt2lXscXR0dOD/5Vu4v+gdfNDWVVHmxx9/VOzTt29fODs7w8jICPb29vjwww95Sz8REVEZqB12GjdujB07dhTavn37djRo0EAjjapMAgMDsWrVKjRp0kSxzcnJCbGxsUqPb7/9FmZmZujZs2exxxq38hgcfTdj/Gp/xMbGYt26dZBIJBg0aJBin86dO2Pnzp0ICwvD7t27ERkZiXfffbdcXyMREZE2U/tWoNmzZ2PgwIGIjIxEly5dAAD+/v7Ytm1bib0aVVFaWhqGDx+ONWvWYMGCBYrturq6sLOzU9p37969GDJkCMzMzIo8liwjF6dicqFrVh0fd28GO7vq+Ouvv9C5c2fUqlVLsd/kyZMV/+/i4oIZM2agf//+yM3Nhb6+voZfIRERkfZTu2enT58+2LdvHyIiIjB+/Hh8+eWXePDgAY4fP15oDp6qztfXF71790a3bt1K3O/y5csICQnBmDFjit1nb/ADZOfJ4WFnjqZO1RAfH49Dhw6VWCYpKQlbt26Ft7c3gw4REdEreqVJXnr37o3evXtrui2Vyvbt23HlyhUEBgaWuu/atWtRv359eHt7F/n8ywOTJRIJNm7cCHNzcwwcOLDQ/tOnT8fy5cuRkZGBtm3b4uDBg2V7MURERG8wtXt2AgMDcfHixULbL168iKCgII00qqLFxMRg4sSJ2Lp1K4yMjErcNzMzE3/88UeJPTTBMcm4HZcKQz0d9G9WEwCwbt06DB8+vMjjT506FcHBwTh69Ch0dXUxYsQICCHK9qKIiIjeUGqHHV9fX8TExBTa/vDhQ/j6+mqkURXt8uXLSEhIQPPmzaGnpwc9PT2cPn0aS5cuhZ6eHvLz8xX7/vnnn8jIyMCIESOKPd72ZzMmv9PEAVJjffz7778ICwvDxx9/XOT+VlZWcHd3x9tvv43t27fj77//xoULFzT7IomIiN4Qal/GunXrFpo3b15oe7NmzXDr1i2NNKqide3aFdevX1fa9tFHH8HDwwPTp0+Hrq6uYvvatWvRt29fWFtbF3ms1KxcHLgaCwB479min2vXrkWLFi3g6elZalueL8+RnZ39Sq+FiIjoTad22DE0NER8fLzSHUQAEBsbCz097VjnydzcHI0aNVLaZmpqiho1aihtj4iIwJkzZ/D3338XeRwPDw/0GDUZmbmOqGtjhhYu1ZGSkoJdu3YVuWjqxYsXERgYiPbt26N69eqIjIzE7NmzUbt2bXh5eWn2RRIREb0h1L6M5ePjg5kzZ0Imkym2JScnY9asWXj77bc12rjKbt26dXB0dISPj0+Rz4eFheFw8F0AwLBnA5O3b98OIQTee++9QvubmJhgz5496Nq1K+rVq4cxY8agSZMmOH36NAwNDcv1tRAREWkriVBz5OvDhw/RsWNHPHnyBM2aNQMAhISEwNbWFseOHYOTk1O5NLQ8paSkQCqVQiaTwcLCQiPH3BEYjRl7ruP5b3fOOw0wur2bRo5NREREqp+/1Q47AJCeno6tW7fi6tWrMDY2RpMmTfDee+9V2blgNB12YmWZaLfwBOQv/GZ1JRKcndEZ9lLjMh+fiIiIVD9/v9IgG1NTU4wbN+6VG6ftohLTlYIOAOQLgXuJGQw7REREr9krjyi+desWoqOjkZOTo7S9b9++ZW5UVedmZQodCQr17LhamVRco4iIiN5Qaoedu3fvYsCAAbh+/TokEolisjuJRAIASnPQvKnspcbwG9gYs/bcQL4Q0JVI8MPARuzVISIiqgBqh52JEyfCzc0N/v7+cHNzw6VLl/DkyRN8+eWX+Omnn8qjjVXS0FbO6OhujXuJGXC1MmHQISIiqiBqh52AgACcOHECVlZW0NHRgY6ODtq3bw8/Pz988cUXCA4OLo92Vkn2UmOGHCIiogqm9jw7+fn5MDc3B1CwrMGjR48AAC4uLggLC9Ns64iIiIjKSO2enUaNGuHq1atwc3NDmzZtsHjxYhgYGGD16tWFZlUmIiIiqmhqh51vvvkG6enpAIDvvvsO77zzDjp06IAaNWpgx44dGm8gERERUVm80qSCL0tKSkL16tUVd2RVNeUxgzIRERGVr3KdVPBllpaWmjgMERERkcapPUCZiIiIqCph2CEiIiKtxrBDREREWk3tsHPmzBnk5eUV2p6Xl4czZ85opFFEREREmqJ22OncuTOSkpIKbZfJZOjcubNGGkVERESkKWqHHSFEkbeYP3nyBKamphppFBEREZGmqHzr+cCBAwEUrG4+atQoGBoaKp7Lz8/HtWvX4O3trfkWEhEREZWBymFHKpUCKOjZMTc3h7HxfwtcGhgYoG3bthg7dqzmW0hERERUBiqHnfXr1wMAXF1d8dVXX/GSFREREVUJao/ZmTZtmtKYnfv37+PXX3/F0aNHNdowIiIiIk1QO+z069cPmzZtAgAkJyejdevW+Pnnn9GvXz/8/vvvGm8gERERUVmoHXauXLmCDh06AAD+/PNP2NnZ4f79+9i0aROWLl2q8QYSERERlYXaYScjIwPm5uYAgKNHj2LgwIHQ0dFB27Ztcf/+fY03kIiIiKgs1A47derUwb59+xATE4MjR47Ax8cHAJCQkFDi8upEREREFUHtsDNnzhx89dVXcHV1RevWreHl5QWgoJenWbNmGm8gERERUVmoHXbeffddREdHIygoCEeOHFFs79q1K3755ReNNu5lCxcuhEQiwaRJkxTbsrKy4Ovrixo1asDMzAyDBg1CfHx8ubaDiIiIqo5XWvXczs4O5ubmOHbsGDIzMwEArVq1goeHh0Yb96LAwECsWrUKTZo0Udo+efJkHDhwALt27cLp06fx6NEjxWzPRERERGqHnSdPnqBr165wd3dHr169EBsbCwAYM2YMvvzyS403EADS0tIwfPhwrFmzBtWrV1dsl8lkWLt2LZYsWYIuXbqgRYsWWL9+Pc6fP48LFy6US1uIiIioalE77EyePBn6+vqIjo6GiYmJYvvQoUNx+PBhjTbuOV9fX/Tu3RvdunVT2n758mXk5uYqbffw8ICzszMCAgKKPV52djZSUlKUHkRERKSdVF4u4rmjR4/iyJEjcHR0VNpet27dcrn1fPv27bhy5QoCAwMLPRcXFwcDAwNUq1ZNabutrS3i4uKKPaafnx++/fZbTTeViIiIKiG1e3bS09OVenSeS0pKUloJXRNiYmIwceJEbN26FUZGRho77syZMyGTyRSPmJgYjR2biIiIKhe1w06HDh0Uy0UAgEQigVwux+LFi9G5c2eNNu7y5ctISEhA8+bNoaenBz09PZw+fRpLly6Fnp4ebG1tkZOTg+TkZKVy8fHxsLOzK/a4hoaGsLCwUHoQERGRdlL7MtbixYvRtWtXBAUFIScnB9OmTcPNmzeRlJSEc+fOabRxXbt2xfXr15W2ffTRR/Dw8MD06dPh5OQEfX19+Pv7Y9CgQQCAsLAwREdHK+b/ISIiojeb2mGnUaNGuHPnDpYvXw5zc3OkpaVh4MCB8PX1hb29vUYbZ25ujkaNGiltMzU1RY0aNRTbx4wZgylTpsDS0hIWFhb4/PPP4eXlhbZt22q0LURERFQ1qR12oqOj4eTkhK+//rrI55ydnTXSMFX98ssv0NHRwaBBg5CdnY3u3bvjt99+e61tICIiospLIoQQ6hTQ1dVFbGwsbGxslLY/efIENjY2yM/P12gDX4eUlBRIpVLIZDKO3yEiIqoiVD1/qz1AWQgBiURSaHtaWppG75giIiIi0gSVL2NNmTIFQMHdV7Nnz1a6/Tw/Px8XL15E06ZNNd5AIiIiorJQOewEBwcDKOjZuX79OgwMDBTPGRgYwNPTE1999ZXmW0hERERUBiqHnZMnTwIouPX7f//7H8e2EBERUZWg9t1Y69evL492EBEREZULtQcoExEREVUlDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Sp12PHz80OrVq1gbm4OGxsb9O/fH2FhYUr7ZGVlwdfXFzVq1ICZmRkGDRqE+Pj4CmoxERERVTaVOuycPn0avr6+uHDhAo4dO4bc3Fz4+PggPT1dsc/kyZNx4MAB7Nq1C6dPn8ajR48wcODACmw1ERERVSYSIYSo6Eao6vHjx7CxscHp06fRsWNHyGQyWFtb448//sC7774LALh9+zbq16+PgIAAtG3bVqXjpqSkQCqVQiaTwcLCojxfAhEREWmIqufvSt2z8zKZTAYAsLS0BABcvnwZubm56Natm2IfDw8PODs7IyAgoNjjZGdnIyUlRelBRERE2qnKhB25XI5JkyahXbt2aNSoEQAgLi4OBgYGqFatmtK+tra2iIuLK/ZYfn5+kEqlioeTk1N5Np2IiIgqUJUJO76+vrhx4wa2b99e5mPNnDkTMplM8YiJidFAC4mIiKgy0qvoBqhiwoQJOHjwIM6cOQNHR0fFdjs7O+Tk5CA5OVmpdyc+Ph52dnbFHs/Q0BCGhobl2WQiIiKqJCp1z44QAhMmTMDevXtx4sQJuLm5KT3fokUL6Ovrw9/fX7EtLCwM0dHR8PLyet3NJSIiokqoUvfs+Pr64o8//sBff/0Fc3NzxTgcqVQKY2NjSKVSjBkzBlOmTIGlpSUsLCzw+eefw8vLS+U7sYiIiEi7VepbzyUSSZHb169fj1GjRgEomFTwyy+/xLZt25CdnY3u3bvjt99+K/Ey1st46zkREVHVo+r5u1KHndeFYYeIiKjq0cp5doiIiIjUxbBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIq2lN2FmxYgVcXV1hZGSENm3a4NKlSxXdJCIiIqoEtCLs7NixA1OmTMHcuXNx5coVeHp6onv37khISKjophEREVEF04qws2TJEowdOxYfffQRGjRogJUrV8LExATr1q2r6KYRERFRBdOr6AaUVU5ODi5fvoyZM2cqtuno6KBbt24ICAgoskx2djays7MVP8tkMgBASkpK+TaWiIiINOb5eVsIUeJ+VT7sJCYmIj8/H7a2tkrbbW1tcfv27SLL+Pn54dtvvy203cnJqVzaSEREROUnNTUVUqm02OerfNh5FTNnzsSUKVMUP8vlciQlJaFGjRqQSCQaqyclJQVOTk6IiYmBhYXFay3Pul9/3WUtz7rfrLrLWp51s+6qUr6sdZdECIHU1FQ4ODiUuF+VDztWVlbQ1dVFfHy80vb4+HjY2dkVWcbQ0BCGhoZK26pVq1ZeTYSFhUWZ/sBlKc+6X3/dZS3Put+sustannWz7qpSvqx1F6ekHp3nqvwAZQMDA7Ro0QL+/v6KbXK5HP7+/vDy8qrAlhEREVFlUOV7dgBgypQpGDlyJFq2bInWrVvj119/RXp6Oj766KOKbhoRERFVMK0IO0OHDsXjx48xZ84cxMXFoWnTpjh8+HChQcuvm6GhIebOnVvoktnrKM+6X3/dZS3Put+sustannWz7qpSvqx1a4JElHa/FhEREVEVVuXH7BARERGVhGGHiIiItBrDDhEREWk1hh0iIiLSagw75WjFihVwdXWFkZER2rRpg0uXLqlU7syZM+jTpw8cHBwgkUiwb98+lev08/NDq1atYG5uDhsbG/Tv3x9hYWEql//999/RpEkTxeRPXl5e+Oeff1Qu/6KFCxdCIpFg0qRJKu0/b948SCQSpYeHh4fK9T18+BAffPABatSoAWNjYzRu3BhBQUEqlXV1dS1Ut0Qiga+vb6ll8/PzMXv2bLi5ucHY2Bi1a9fG/PnzS12r5UWpqamYNGkSXFxcYGxsDG9vbwQGBhbar7T3hhACc+bMgb29PYyNjdGtWzeEh4erXH7Pnj3w8fFRzCYeEhKicv25ubmYPn06GjduDFNTUzg4OGDEiBF49OiRSnXPmzcPHh4eMDU1RfXq1dGtWzdcvHhR5ba/6NNPP4VEIsGvv/6qUtlRo0YV+tv36NFDrbpDQ0PRt29fSKVSmJqaolWrVoiOji61bFHvO4lEgh9//FGlutPS0jBhwgQ4OjrC2NhYsRiyKmXj4+MxatQoODg4wMTEBD169FC8X1T5LMnKyoKvry9q1KgBMzMzDBo0SDHBqyrlV69ejbfeegsWFhaQSCRITk5WPFda+aSkJHz++eeoV68ejI2N4ezsjC+++AIymUyluj/55BPUrl0bxsbGsLa2Rr9+/RRLDKnzOSqEQM+ePRW/X1XKvvXWW4X+3p9++qladQcEBKBLly4wNTWFhYUFOnbsiO+++67Esvfu3Sv2/bZr1y6V6o6Li8OHH34IOzs7mJqaonnz5ti9e7dKZSMjIzFgwABYW1vDwsICQ4YMKTQhcHlh2CknO3bswJQpUzB37lxcuXIFnp6e6N69OxISEkotm56eDk9PT6xYsULtek+fPg1fX19cuHABx44dQ25uLnx8fJCenq5SeUdHRyxcuBCXL19GUFAQunTpgn79+uHmzZtqtSMwMBCrVq1CkyZN1CrXsGFDxMbGKh5nz55VqdzTp0/Rrl076Ovr459//sGtW7fw888/o3r16iq398V6jx07BgAYPHhwqWUXLVqE33//HcuXL0doaCgWLVqExYsXY9myZSrVDQAff/wxjh07hs2bN+P69evw8fFBt27d8PDhQ6X9SntvLF68GEuXLsXKlStx8eJFmJqaonv37sjKylKpfHp6Otq3b49FixYV+3xx5TMyMnDlyhXMnj0bV65cwZ49exAWFoa+ffuqVLe7uzuWL1+O69ev4+zZs3B1dYWPjw8eP36sUvnn9u7diwsXLihNH69K2R49eii9B7Zt26Zy+cjISLRv3x4eHh44deoUrl27htmzZ8PIyKjUsi/WGRsbi3Xr1kEikWDQoEEq1T1lyhQcPnwYW7ZsQWhoKCZNmoQJEyZg//79JZYVQqB///64e/cu/vrrLwQHB8PFxQXdunVDenq6Sp8lkydPxoEDB7Br1y6cPn0ajx49wsCBAwGo9lmUkZGBHj16YNasWYXaV1r5R48e4dGjR/jpp59w48YNbNiwAYcPH8aYMWNUqrtFixZYv349QkNDceTIEQgh4OPjg/z8fLU+R3/99VelZYZULTt27Filv/vixYtVLh8QEIAePXrAx8cHly5dQmBgICZMmICzZ8+WWNbJyanQ++3bb7+FmZkZevbsqVLdI0aMQFhYGPbv34/r169j4MCBGDJkCA4cOFBi2fT0dPj4+EAikeDEiRM4d+4ccnJy0KdPH8jl8kK/V40TVC5at24tfH19FT/n5+cLBwcH4efnp9ZxAIi9e/e+cjsSEhIEAHH69OlXPkb16tXF//3f/6m8f2pqqqhbt644duyY6NSpk5g4caJK5ebOnSs8PT1fqY3Tp08X7du3f6WyRZk4caKoXbu2kMvlpe7bu3dvMXr0aKVtAwcOFMOHD1eproyMDKGrqysOHjyotL158+bi66+/Lrbcy+8NuVwu7OzsxI8//qjYlpycLAwNDcW2bdtKLf+iqKgoAUAEBwerXH9RLl26JACI+/fvq11WJpMJAOL48eMq1/3gwQNRs2ZNcePGDeHi4iJ++eUXlcqOHDlS9OvXr8T2lFR+6NCh4oMPPnilsi/r16+f6NKli8rlGzZsKL777julbUW9d14uGxYWJgCIGzduKLbl5+cLa2trsWbNmkJ1v/xZkpycLPT19cWuXbsU+4SGhgoAIiAgoNTyLzp58qQAIJ4+fVrk6y6t/HM7d+4UBgYGIjc3V+2yV69eFQBERESEynUHBweLmjVritjY2GL/tkWVVedzsajybdq0Ed98880rlX1Z06ZNC31+lVTe1NRUbNq0SWk/S0vLQu+Zl8seOXJE6OjoCJlMptgnOTlZSCQScezYsVJfS1mxZ6cc5OTk4PLly+jWrZtim46ODrp164aAgIDX2haZTAYAsLS0VLtsfn4+tm/fjvT0dLWW3vD19UXv3r2VXr+qwsPD4eDggFq1amH48OGIjo5Wqdz+/fvRsmVLDB48GDY2NmjWrBnWrFmjdv1Awd9vy5YtGD16tEoLw3p7e8Pf3x937twBAFy9ehVnz55Fz549VaovLy8P+fn5MDIyUtpubGyscs8WAERFRSEuLk7p9y6VStGmTZvX/r57TiaTQSKRqL32XE5ODlavXg2pVApPT0+Vysjlcnz44YeYOnUqGjZsqHZbT506BRsbG9SrVw+fffYZnjx5onK9hw4dgru7O7p37w4bGxu0adNGrcvPz8XHx+PQoUMYM2aMymW8vb2xf/9+PHz4EEIInDx5Enfu3IGPj0+J5bKzswFA6X2no6MDQ0PDIt93L3+WXL58Gbm5uUrvNw8PDzg7Oxf5fivLZ5Gq5WUyGSwsLKCnp1doe0ll09PTsX79eri5ucHJyUmlujMyMvD+++9jxYoVxa7DWFLdW7duhZWVFRo1aoSZM2ciIyNDpfIJCQm4ePEibGxs4O3tDVtbW3Tq1Emlv9nLLl++jJCQkGLfb0WV9/b2xo4dO5CUlAS5XI7t27cjKysLb731Volls7OzIZFIlCYWNDIygo6Ojlqfc6+s3OPUG+jhw4cCgDh//rzS9qlTp4rWrVurdSyUoWcnPz9f9O7dW7Rr106tcteuXROmpqZCV1dXSKVScejQIZXLbtu2TTRq1EhkZmYKIdT7BvP333+LnTt3iqtXr4rDhw8LLy8v4ezsLFJSUkota2hoKAwNDcXMmTPFlStXxKpVq4SRkZHYsGGDym1/bseOHUJXV1c8fPhQpf3z8/PF9OnThUQiEXp6ekIikYgffvhBrTq9vLxEp06dxMOHD0VeXp7YvHmz0NHREe7u7sWWefm9ce7cOQFAPHr0SGm/wYMHiyFDhpRa/kWa6NnJzMwUzZs3F++//77KZQ8cOCBMTU2FRCIRDg4O4tKlSyrX/cMPP4i3335b0RunTs/Otm3bxF9//SWuXbsm9u7dK+rXry9atWol8vLySi3//Fu9iYmJWLJkiQgODhZ+fn5CIpGIU6dOqfS6n1u0aJGoXr264t+PKm3PysoSI0aMEACEnp6eMDAwEBs3biy1bE5OjnB2dhaDBw8WSUlJIjs7WyxcuFAAED4+Pkpli/os2bp1qzAwMChUT6tWrcS0adNKLf+i0np2VPkse/z4sXB2dhazZs1SueyKFSuEqampACDq1atXZK9OceXHjRsnxowZo/i5qL9NcWVXrVolDh8+LK5duya2bNkiatasKQYMGKBS3QEBAQKAsLS0FOvWrRNXrlwRkyZNEgYGBuLOnTsqve7nPvvsM1G/fv0inyuu/NOnT4WPj4/i/WZhYSGOHDlSatmEhARhYWEhJk6cKNLT00VaWpqYMGGCACDGjRtXbBs1hWGnHFSWsPPpp58KFxcXERMTo1a57OxsER4eLoKCgsSMGTOElZWVuHnzZqnloqOjhY2Njbh69apimzph52VPnz4VFhYWKl1C09fXF15eXkrbPv/8c9G2bVu16/Xx8RHvvPOOyvtv27ZNODo6im3btolr166JTZs2CUtLS7WCVkREhOjYsaMAIHR1dUWrVq3E8OHDhYeHR7FlKnPYycnJEX369BHNmjVT6rYurWxaWpoIDw8XAQEBYvTo0cLV1VXEx8eXWj4oKEjY2toqBVR1ws7LIiMjVb6E9vzf+3vvvae0X58+fcSwYcPUqrtevXpiwoQJxT5fVPkff/xRuLu7i/3794urV6+KZcuWCTMzs0KXBooqGxQUJDw9PRXvu+7du4uePXuKHj16KO1X1GeJOmGntM+i0sJOaeVlMplo3bq16NGjh8jJyVG5bHJysrhz5444ffq06NOnj2jevHmhoFlU+b/++kvUqVNHpKamKrYV9ftV9TPY39+/yEtoRZV//u985syZSvs2btxYzJgxQ+W6MzIyhFQqFT/99FORzxdXfsKECaJ169bi+PHjIiQkRMybN09IpVJx7dq1UsseOXJE1KpVS0gkEqGrqys++OAD0bx5c/Hpp5+W8NvRDIadcpCdnS10dXULvfFHjBgh+vbtq9axXjXs+Pr6CkdHR3H37l21y76sa9euKiXvvXv3Kj40nz8AKN7YRX1LLk3Lli2V/gEXx9nZWelblhBC/Pbbb8LBwUGt+u7duyd0dHTEvn37VC7j6Ogoli9frrRt/vz5ol69emrVLUTByf55WBkyZIjo1atXsfu+/N54foJ+OaB07NhRfPHFF6WWf1FZwk5OTo7o37+/aNKkiUhMTFSr7Mvq1KlTZC/Zy+V/+eUXxfvsxfeejo6OcHFxeaW6raysxMqVK0utOzs7W+jp6Yn58+cr7Tdt2jTh7e2tct1nzpwRAERISEixbXq5fEZGhtDX1y803mvMmDGie/fuKtednJwsEhIShBAF4w3Hjx+veK64z5LnJ+iXA4qzs7NYsmRJqeVfVFLYKa18SkqK8PLyEl27di0UVNT5HMzOzhYmJibijz/+KLX8xIkTi32/derUSe2609LSBABx+PDhUuu+e/euACA2b96stH3IkCGKXlRV6t60aZPQ19dX/N1fVFz5iIiIQuO8hCg4R3zyyScq1/348WPF39rW1lYsXry42H01hWN2yoGBgQFatGgBf39/xTa5XA5/f3+1xr68CiEEJkyYgL179+LEiRNwc3Mr8zHlcrni+n5JunbtiuvXryMkJETxaNmyJYYPH46QkBDo6uqqVW9aWhoiIyNhb29f6r7t2rUrdJvjnTt34OLiolad69evh42NDXr37q1ymYyMDOjoKP9T0tXVfaU7DExNTWFvb4+nT5/iyJEj6Nevn8pl3dzcYGdnp/S+S0lJwcWLF8v9ffdcbm4uhgwZgvDwcBw/fhw1atQo0/FUfe99+OGHuHbtmtJ7z8HBAVOnTsWRI0fUrvfBgwd48uSJSu89AwMDtGrVqszvv7Vr16JFixYqj1ECCn7fubm5ZX7/SaVSWFtbIzw8HEFBQejXr1+pnyUtWrSAvr6+0vstLCwM0dHR8PLyKvNnkSrlU1JS4OPjAwMDA+zfv18x/uhV6hYFX/6RnZ1davkZM2YUer8BwC+//IJ169apXffz8vb29qXW7erqCgcHhyLfb87OzirXvXbtWvTt2xfW1tZKv4OSyj8fV1TU+y0/P1/luq2srFCtWjWcOHECCQkJijs2y1W5x6k31Pbt24WhoaHYsGGDuHXrlhg3bpyoVq2aiIuLK7VsamqqCA4OFsHBwQKAYhzAy3e0FOWzzz4TUqlUnDp1SsTGxioeGRkZKrV7xowZ4vTp0yIqKkpcu3ZNzJgxQ0gkEnH06FGVyr9MnctYX375pTh16pSIiooS586dE926dRNWVlZFfvN42aVLl4Senp74/vvvRXh4uNi6daswMTERW7ZsUbmt+fn5wtnZWUyfPl3lMkIU3MlTs2ZNcfDgQREVFSX27NkjrKysCnXll+Tw4cPin3/+EXfv3hVHjx4Vnp6eok2bNoW65Et7byxcuFBUq1ZNMf6kX79+ws3NTfGNt7TyT548EcHBweLQoUMCgNi+fbsIDg4WsbGxpZbPyckRffv2FY6OjiIkJETp/ZednV1i2bS0NDFz5kwREBAg7t27J4KCgsRHH30kDA0NFd8i1f138eJlrJLKpqamiq+++koEBASIqKgocfz4cdG8eXNRt25dkZWVpVLde/bsEfr6+mL16tUiPDxcLFu2TOjq6op///1XpXbLZDJhYmIifv/990Kvo7TynTp1Eg0bNhQnT54Ud+/eFevXrxdGRkbit99+K7Xszp07xcmTJ0VkZKTYt2+fcHFxEQMHDhRCqPZZ8umnnwpnZ2dx4sQJERQUJLy8vBSXk1UpHxsbK4KDg8WaNWsEAHHmzBkRHBwsnjx5Ump5mUwm2rRpIxo3biwiIiKU9vn0009LLBsZGSl++OEHERQUJO7fvy/OnTsn+vTpIywtLUV8fPwrfY7iWc9ZaWUjIiLEd999J4KCgkRUVJT466+/RK1atUTHjh1V/r398ssvwsLCQuzatUuEh4eLb775RhgZGYn3339fpXaHh4cLiUQi/vnnH6XtpdWdk5Mj6tSpIzp06CAuXrwoIiIixE8//SQkEono1atXqXWvW7dOBAQEiIiICLF582ZhaWkppkyZUuzvVJMYdsrRsmXLhLOzszAwMBCtW7cWFy5cUKnc8y7dlx8jR44stWxR5QCI9evXq1T36NGjhYuLizAwMBDW1taia9eurxx0hFAv7AwdOlTY29sLAwMDUbNmTTF06NAiBwwW58CBA6JRo0bC0NBQeHh4iNWrV6vV1iNHjggAIiwsTK1yKSkpYuLEicLZ2VkYGRmJWrVqia+//lpkZ2erfIwdO3aIWrVqCQMDA2FnZyd8fX1FcnJyof1Ke2/I5XIxe/ZsYWtrKwwNDUXXrl2VXk9p5devX1/k83Pnzi21/PNLX0U9Tp48WWLZzMxMMWDAAOHg4CAMDAyEvb296Nu3r9IAZXX/XbwYdkoqm5GRIXx8fIS1tbXQ19cXLi4uYuzYsUpfTFSpe+3ataJOnTrCyMhIeHp6Ki6FqlJ21apVwtjY+JX+5rGxsWLUqFHCwcFBGBkZiXr16omff/5ZyOXyUsv+73//E46OjkJfX184OzuLb775RvG+VeWzJDMzU4wfP15Ur15dmJiYiAEDBiiCsSrl586dW+w+pZUv7rWV9Hhe9uHDh6Jnz57CxsZG6OvrC0dHR/H++++L27dvq9z2lz0PO6WVjY6OFh07dhSWlpbC0NBQ1KlTR0ydOlUxtk3Vuv38/ISjo6MwMTERXl5e4t9//1W57MyZM4WTk5PIz88v9BpKK3/nzh0xcOBAYWNjI0xMTESTJk3Epk2bVCo7ffp0YWtrK/T19UXdunUV79PXQfLsBRIRERFpJY7ZISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQEb3k1KlTkEgkSE5OruimEJEGMOwQERGRVmPYISIiIq3GsENElY5cLoefnx/c3NxgbGwMT09P/PnnnwD+u8R06NAhNGnSBEZGRmjbti1u3LihdIzdu3ejYcOGMDQ0hKurK37++Wel57OzszF9+nQ4OTnB0NAQderUwdq1a5X2uXz5Mlq2bAkTExN4e3sXWmmaiKoGhh0iqnT8/PywadMmrFy5Ejdv3sTkyZPxwQcf4PTp04p9pk6dip9//hmBgYGwtrZGnz59kJubC6AgpAwZMgTDhg3D9evXMW/ePMyePRsbNmxQlB8xYgS2bduGpUuXIjQ0FKtWrYKZmZlSO77++mv8/PPPCAoKgp6eHkaPHv1aXj8RaRYXAiWiSiU7OxuWlpY4fvw4vLy8FNs//vhjZGRkYNy4cejcuTO2b9+OoUOHAgCSkpLg6OiIDRs2YMiQIRg+fDgeP36Mo0ePKspPmzYNhw4dws2bN3Hnzh3Uq1cPx44dQ7du3Qq14dSpU+jcuTOOHz+Orl27AgD+/vtv9O7dG5mZmTAyMirn3wIRaRJ7doioUomIiEBGRgbefvttmJmZKR6bNm1CZGSkYr8Xg5ClpSXq1auH0NBQAEBoaCjatWundNx27dohPDwc+fn5CAkJga6uLjp16lRiW5o0aaL4f3t7ewBAQkJCmV8jEb1eehXdACKiF6WlpQEADh06hJo1ayo9Z2hoqBR4XpWxsbFK++nr6yv+XyKRACgYT0REVQt7doioUmnQoAEMDQ0RHR2NOnXqKD2cnJwU+124cEHx/0+fPsWdO3dQv359AED9+vVx7tw5peOeO3cO7u7u0NXVRePGjSGXy5XGABGR9mLPDhFVKubm5vjqq68wefJkyOVytG/fHjKZDOfOnYOFhQVcXFwAAN999x1q1KgBW1tbfP3117CyskL//v0BAF9++SVatWqF+fPnY+jQoQgICMDy5cvx22+/AQBcXV0xcuRIjB49GkuXLoWnpyfu37+PhIQEDBkypKJeOhGVE4YdIqp05s+fD2tra/j5+eHu3buoVq0amjdvjlmzZikuIy1cuBATJ05EeHg4mjZtigMHDsDAwAAA0Lx5c+zcuRNz5szB/PnzYW9vj++++w6jRo1S1PH7779j1qxZGD9+PJ48eQJnZ2fMmjWrIl4uEZUz3o1FRFXK8zulnj59imrVqlV0c4ioCuCYHSIiItJqDDtERESk1XgZi4iIiLQae3aIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIq/0/aNuF5S+4oysAAAAASUVORK5CYII=", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "plt.plot(np.arange(len(epochs_x)), epochs_acc, marker = '.')\n", - "plt.xlabel('epoch')\n", - "plt.ylabel('test accuracy')\n", - "plt.ylim(0, 100)\n", - "plt.xticks(np.arange(len(epochs_x)))\n", - "for i, txt in enumerate(epochs_acc):\n", - " if i%5 == 0:\n", - " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", - " else:\n", - " pass\n", - "plt.show()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "speck-rescnn", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/tests/test_nonsequential/exp_set_TA1/main_loop.py b/tests/test_nonsequential/exp_set_TA1/main_loop.py deleted file mode 100644 index c328a576..00000000 --- a/tests/test_nonsequential/exp_set_TA1/main_loop.py +++ /dev/null @@ -1,5 +0,0 @@ -import os - -for w_load in range(3, 9): - print(w_load) - os.system(f'python train_script.py {w_load}') \ No newline at end of file diff --git a/tests/test_nonsequential/exp_set_TA1/nonseq_conv1_weights.pth b/tests/test_nonsequential/exp_set_TA1/nonseq_conv1_weights.pth deleted file mode 100644 index a4ff9cf6e910398edaef8f119a23cac63e8bb7a9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1693 zcmWIWW@cev;NW1u0J03M40-u^#i@ny$@zI@hVkX8nduoN#ri3UC5d_k**R`bj0{l? zOv%alIXS7xC7D3AT>eEVsYR(NE}6+CT!jppL4}MFY(SGCS__$yOY)17GxXw1OEPnc zx#EjblS(slQsPTe^NRC};>(P<3Yj%DBG`dCih(K<(^CssAX>QGa`F>Pf+2!jg{%>b zKs5%1Y+%!qOH&f93<}x314;@x0=*eDyt%xYK(M5cQ^T9Xn+3>bEGgvb%mBHFyQGjO zsF1g~wvaD^6KG0&ZfZ#)$WMj*V0nQEpddq`U~Qog*d=ADMa4kB6$*PZ78e&M=>>SR za|Cxi5BdaD1;PP1Lr{*v33~`8RhFdYgF?#9$&Mk4B9N0=Qj(Jja#O4AV@Xh0gD`G4 zNi!H?cM~WC^bOpcbV+lN%im`E^LqE~S=&tP5C1i=pZ@*p-d}22`+N>;*mr)yK|2jc zJKHsP|Lt40XRaOV6ApX1(~I~0YcRBX_Kk1fg6@WWw(ZC5a_Ya@P0zR4Z(4X{pMyEy z9wtpLJ1arey-WE%?-f%DvHz9WZC?>|%3d&Vy?x`f>wE9=zux<+b=AI!G7WaCvVQN2 zd%(VbnfJ_n<|@wnnPa5wKRG#^ zr}Dh*`eN(7uMCyMVo%ms46#d+zWOu%#l!nj?a&7g|i1$yAbQ=D3qnV6GV zl?W;d;xkj+oD>N=$iURx%)->d!qnW@!qn2z%*for$N&fo4UG)V%`8og%}fm}EsRY; zuH10dH&X-XLJ$t{W&}AFUUDJlNLi2s3cy>Ap&Nyqk>pT}0Ty`J0|1_((al1RT4@xs zuHiBZk`d7jLXIIB6oV!(<8Tx-U7(wU9E#c~CJ6y+0<0m0J$?hc+1PZT2FNk%!i{2u qvS4&G(5E1<186)50QG@Ncm``wNd*KT=>Tt5Fpm{j>44Nj)B*re#oZqO diff --git a/tests/test_nonsequential/exp_set_TA1/nonseq_conv2_weights.pth b/tests/test_nonsequential/exp_set_TA1/nonseq_conv2_weights.pth deleted file mode 100644 index 8b615944c77d94845ff8d0b501c4e634d51b9799..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2973 zcmbtW3sjBi8s6P@w=I?Fc2vqJxs<)P(#8JYKi%0P3@Yii+p?#ded&e{x=}-eQ*?+r zg+zxVUBv$1kHpM`aY8MpB}SN0t1xmIxt+Z+7EaDtv(Edk_y523fA9CK=X<|zt>+05 zYH71rdU~w?2pg6TD>)@uE|x}#Qj*j7QR!ky{BnhSN?eR0#$`o9Vt|hZD}1e{NR*P8 zC>ALss#=4&vN*9!9OowyDGa2n3G<~Ip==fM+oe)Xks?JVTJ92+s*ohg4WeY?*i=bk zT$DnbEKiX|rST1M!hNN$T#!Gp{k!OsY#a9^bM6Xu}rSooOF~>LoS!cx`YUI^_r`9iw0Fy>J0g8 z3vF35hHqhPrb3*eZdHH}#R~s_B2l7HB#PCP@~B%@>Rzk!^OQ!jc*9dtH(&}kz{mNI z@kzH#^{kq%!94Ui%al)k%dt96r74A4 z?4yK;h5Mksi$(nD0pz(yK4mNKg3n{jG^kMhm?+P6~>A;;811~NM8F=jk6}i zgM&TRbilM6@_L|WI@h!uzrNByWgY5ehR#J$7X8-riindCJeJL*|LsdA{#0i3mGYg? zxut~S-P!_i9=h~-ybYP>3W>su}=l>AkQSN@7^=y?4IlRCPZ8j7|@-${um3#!Gs8+OX7 zggn@i(*bQ=-O#*#Hhs!5pXms+LcyFIsy#9f+i7FEU-<>CU0e)pXLV@%L+#AaGIQ9J zaFIMRp4lXVUHg$qmu1O`u(-kYtDMqoe_xkrM*hmyZLBw zrT})R?zL%Q6g=-($84w#0pk{Hs{iIiaMl#i$=`P|yxn}LeafQ;>PORG{z8)ri;7^2 zMFHa-xr$cqD1g}8k#tL*IW6AN4c#3cgf&os-)UHo?HfDrl7j)&{B1nARz!gzuLQh0 zT*&FpKatk4{dgeb3?A!tB#p&8$WVG2+5D=B+zq*dezKRC`z@Q)tWl7^mcCN5esyJL zhK;8jgOT!0?t_sQY(bT`8Eg}B$r5WD;^sV_)SXyL)^K)_9;qJ_cB2kn=%)bP+JIW) zw!xkyfTirmkk@_$gZNF#j6Q9uV9y5TXznt&dgBCDd-j#G*}b3IrpF;iB!+Zx*58@T z;0OpvZpOwkL(q_4#-5eEjK2u|7cg0E39#bQRxE7vgryDns1s&{y}mJ|R#Qaj6`7L> z))J`PxfiL6YcarPG06xZcsw$nnf{1Fdc@PIAZCcE>}zGRj~{`uBNvcaMXq$Bd-C6>qx5P+XeEq-58%{3H#fvNy!`z zEDP?!g_a_GHoBg&TV2NFD4ik0CJkSmF(GS$7c&)R0@UlehP-+wQXZWGmFvt<>y8!u z%)^rI`>lZ1@!3oV3H$L`eJ(DM`_PZZ?*?f#SIIP8#md4Q_%&}M*!6@_jYUp)j}`-$ z>Y*aMbFe9}M6hd73+NY_)An^nO&=h#9WJ{f&S1xPTwBeQOojwQn@c)=} zy^-Ci25JWFyWm0(NA25GmRpCVu76wmZ<6Ow`Tzg` diff --git a/tests/test_nonsequential/exp_set_TA1/nonseq_conv3_weights.pth b/tests/test_nonsequential/exp_set_TA1/nonseq_conv3_weights.pth deleted file mode 100644 index 8e179172d529aec383c7cae44f5c5fd693b64cb3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4957 zcmbtYc|6rw*uU10T^ZR*Mb^l^-1A&i%3u;jr6J2D5iVCri#;upv`W$*Ew_|r;{MKa zD~ZyC5-Kgyh?a>drHz+q-ucYTTkrCDpWo;CoO6EXIp6a=&pH1*UvCcyNft{+hV@@e zhb6^|iH%9%@`6HRW0qM3E$2o?EaoSehXwP4C&xuaySa+77Nm-YhQ>xmb3^%&L$PwR z;={P{+^}hpp?o;kXM^2|7E}OgI zv}6=Kcn7>JFW5g_8hmdI!q#U8X;-Ty7;tpyz>`NLFIiIbvim%G z6}vD33uRm^-^K7&R5Mx%6EJkp1O}IEr9tk?P=|3whmeoVV}n>&_N)km5BE{Go8~lL zRS#3IT%{XE8=>T{Z^(*sb>u;Q4Ae{H!{j&_(j%6^)TD@G!$B!DYFjIkw@bhmSAC%P z!g?^aE(haGAxSJ*2=h(VV2#B_OjDJ|f|02Zd?<}pn=N5FfA=T*CBt#nQ+3QtE29&4 zWr0Y`9Zb@!VT5%XUD|mc9zS#y`k6>$`hE_r-})ndO1npl88&7cABONxaU{@pE*VoH z%e=GZ;&#D9dO_|Z6`Xu5BzGl5FD+F$mvRKufi26a`}hP_dj(=gp&ZRvz7Pa*nxxLw zoKxj2#aSY?MRdnXofG@qo~#Wp6q#2HphoWzn7%NV+)a4J-jMwojKr#FkBu3}bNvL) zhwK(?p4ZLr_wOgpnKwZ85RaUT_avF|@5%I)5_oH}FK)drLyf$n;PlEEuuzf33hS4| zAZ8I{1#Ci}owJFv+)6AOm;twDO(aI2%4l|n4f~8#FzoTtfc8DPI7jIcsT`lhmR+8Q zie9c@WO+fDdu!CS&jLGso<4$mEpfCsO)konim zaCQ9mBz1c_`{|3{*rInIh2CcbY{kn(SfFqeQd{yc#F!1ES~G~#soO9qcNeWKamTma z6F3|5M~H0R-Nl+(S-Rv}0sg46omP!*V$V$S6scC?wv&98-f05$!N=ell<(6R4|f8Ob#fC&wDYfn}IUdUe$y_jw+XG1^M>G)$xClE>zB?=SCGcj_E3dkkHZa z0_8d4ez#yxTM(3&Nir`t9i`!2o-kD+7dkFQz^asZMv&w}_RZ4+^NCf$@I4On!8Jwt zu#^z~kpe96-^O^DZNtk~t5M;d7EYNgg(Y%(Nh1F}Q?=tHS;^`r*Y2o;TGT$a@AyWt zNVSeMnrq^l#u4b|U(ZhIUCL2#$R@srLr8BYAG?xnVb6-q)GYC5bhJsw68A(J+j9^% zbx4Q?KY0@IHCu6GNG>XbBQ9^5DhfNX9!q;|Ft>3w$9t0!G{Z7u##>%yA1)-_8LX4|7R%RySEGrHD}L2c33G z$cU|i%8S)BWLgJFkUc_bYV!#%?=yLll|zEW)X7^<1-c}!7PPC!;f1tQ#F{sWDtG=! zuRKb}9P?eM-8mP^m%T*0U*FKF{p;{fSq1vr)G;56Pm*8m&81^v456o10#?c!G4sk5 zKxcFjjbD^b^4pJ4XX8T3+BXODZs~z_rz!k2y?~BL^CnM9)=@#D8oYR@RH(48?g6v*aPxry^XAObpkpj2pToT%ea-GV$oT;*6 z7+Tzs6a_DQk5BfBi>j;r@J(_q+*0kLtOuzmKKdw?-?@oCsy#!;cnQeLeuf@RoQ@L{ zPeH+BCw%B?LHFxPLt$p+gQAiK4R$#ReZp?Nl3?EVpZHGzG7jz7LpFLm$^Y9ga$-Kk`1HnC)v zk^Y+!WEwvjO?pOPj9fA4dwGm@hrVJ5JRz`k4GYJ&9fd%_d6;(C5FhIO!mhco1w%tO z(r54WATDS&ri^N3lI=r?d2|q5ODaH)X81Wt#o!b2;jSJ1s_ zU@iZi?CwusjJeCPt*{W2d=&wOGx74?wdnU|ANJ=Qrw`W+I?4Tf4p_P~h`&n)U7(;@IXs+ z!0Q?#c$|*@@|WpS-Ghdj_;q>60`R%qIgPsR7IrB23j zWPiqKGNFC~)O4+-rG-1d>yEML?d)o-d!dcZZRe@uB54s#I*ZfaxPqigBIf?!OL~e8 z@yGE3D&F~=33#MO2J0@8<9>DYeD?;&oE@@oT7L_fn|p#?;%!W)`yQh?4N0)kUkwee z_MuUO01OPwVe2u#kiwiENI(*tHh~pafup%^F8ri$9d^$@1V*`%oXd`p zpyQti`l;#AB0kN@FeHFRat_13!bfC=l{80TZw^T02f~3GYiL$D$0WB-Al!Ex>W-#( zJbw{5FO0^;Q3iBfz+0z`(2w-(?Nj9Q@eHV3GmB1{XbNSo{FtZG8aVO&YbV2`&9uRC zJ#5^p1n=tNNuSMZGAKAq&U&f+h7a0p~|P}atF zlF%{}9XIE~W{K5wW#E3|voHwq8j6^zs5lZHdy+0(af#lv*9M!GD}DiAX;K{2@QV^01&8rHjzjhqB>wG4c_#^Rg&0ca#piXa&7)6^L&EVuQGqU6U8=S;{ zg6}PyP&;=K7*q*KtmP9D6*85SEDnckUl4Q=ed;S_p8lN2|p z;M%CYG~-$~^Ijz%b3V$7&YCu1_}j(UMLfax-g@Y~DZ_bi_ID7@?}6yi8kl~2?NBYV z1eaJ@krv+&s#i4;Al{UAT7PDLmK+o=7`2tSZJ$7&R$Jr3<~Xp>OoQ@nXD~^27SA!MEgauX*W_=d`8Nt3~cZAXLkmuizeM4Cu&+!i!;=D z80PGYT2Wcpp{a|dk8J4KDWP%-NcUPn>b-QF{MHxHvkbmoiT+t{ySa}4UzDO&a>F|| z?v0o$K2-M(`?q&AVd;N;$H{*bzX{y<$l&P6XKQa`Z*OB~ zWpCr)U~6e-Z)q`PQ*108>@01qtSl|;?Q9*ahgsg-P+wp;#BkWXJ;a8|{-wP9T~N>& zp7@9Tw>r$fq5due=>7v{>X2&kRj9x8h<{`K{VKKofz|vitS>^szk&XKL)!lUTBQ69 zQh&aKe zCNOACP~hZ{fN1$hF$=FqabIbX<^S#+DG?AIu_j=Jwg1}akgzCu|209&)`o-y`bP(? zii%j{zs^=ZQqtVqS4Pw$O4M@c@}Nkm|L!6`B`jiT^ql{`DIYn=S6tM_CQ|yptOu+O zTb;W;PL?Jp-(JN9ZYD=WDubil}e|S*z z(*F!5Qsuv%tNMy=ibW3I5~=oIFR(6XO_XRzkwZMhqoSggS$lfO$Y$=gdEPB*^`F!8 z|LHhAF{S^>amzMD2SxnHJyTrd{^Pm-KEgtxqr-y!V>}VMbpGS3|D6AC<2qt?|C90m z3}m#;6qnKe$MXOFhyVXodqn^_^mnV=CAi=U{>}T=w zEAeXlK3M8^7j$1;hpp>-`7Y`4EP32byvkX@oks_uvBzCF)_W)mbw5TLYdj#JcOq-4 zF~Gr&H~8YaT{>{QC>n(_zRRu`r|)FpS+)`vM9j z<@n}EI?g@fOpElDxVs022rAZj3kK!v#T%bCQG4bX(jTz_*Qh4&2UdT?RsQBA-EdwQ zG0Y6i7MH{HeRgcow=s#7IWRi&3F&+-AocOHDD%x}HY`7f8IG04 z!Mn70kIrx=*EWt-p3B0?KiXg&Q)i=mKlAyW?YQYHpp8=zCjHq-M~4pZ4`&T#*-d76 zUP4PSeWC-XzqF!|KuzqhEPw&cJVEZIFCd$?5Dq*yV^iA{>0?G8R<71z%S)~b=#dr~ zndtH1Klf1M(|)edIgX|7*oB)<%p+;D(_k<(1M<)5(zv+}xb%T8dbO>^f>|^0hkF;S z*>qVra>a5^A^c&T%DXvWQTiVDy)c4r#fhxcV;ZjPQm0|w*8KZ>dN9t+l&!YP#8huB z_Iha%UcPdVzcQ$p2EPi0MWHVtTwIxLUoxEO`M$xc?dhPGV@_W8KH{sr%W(Lv0FZUn zWi>A=Iq`I>y4-P6B<(v70{7@*!qpyFa&8x2U1>_H4Fs|;Teu0F0xfLR;~Uf_vYEqg z!itRLSafn1|NHe=w%*Bwv(FkzN8%*#QoSzCIbDjD4XTu^TO|DZu#vyR#$oH`Eu`0+ z#dVxr&N=-KLRWh?w$HGivyIpxcyGHERy&;J!!?dTc%~h{!l)bW1b6YjKA(f3e+ILq zAG%R^rGQ(tUyfCGl#uVWwRH9SQ7ma%L~D9?!;qb9yOM{z=14c zD!8Tp%t56lf;C#3LfzNHd}YT2K73ahR%&&ij`n1@+PDBWyGl{5!xY-;CWUUxesZC2 z7*%aLf+fSG$c8VaeeH!9;ub_jpWSHW*IHUQs);MU>rQjL?ICiCKL+ocj_DtEamugE zXvdS`P!gof28L_XkmVR&DoR@5(|dWQmi!RA&B?%!rj_Wyz5^}O4ho?Pp*@qO(abPp^4n1C*w$Z z(;$ktwUPd;2tbd?%Rs*(hKA*PfTN)XHLg_y;pzR{kA(ZsDxZy?EQlWwu?YumjbTL=CvaQ&W;C4QKodNp zu=Sn=Q(0IP2(ym$)~Y=M zjwO~Qy!gydID9FvlRewG^bJOIJ2eJ>Z*8W;l5^aQHInppkUPD8Z$~bV@}W2H7}gx$ zj%pn~%+$YyyV^aN?7CFp%Rx1EVUr_d)_YRF%r;n;ufh6%tOK?C+Wg!>gTd;z0h9lD znwwTQk||7DLpk!|8jkOxOz`nWcY0j*3YImfVde*4J}D`TrYNQZS8pSVSIfA} z;}_9E%LA7t)KJ^aVtDacoLzQ4hXZ+qFj6U(^Gx0eyJn=3|C(Cts`Pmw zeS~(wNERv3VLb(gG`6;w50(z4z2oYjFkgtZaXz%>p(gt^{y5*%H4-y)x1sLaxiDQ) zmPv)a2BF$fJbY{}%Nn1~HV;&ATaTL2xS;zmKG6vh9xuT4_HSWm*m1nIe1Lncmkd%H zG9j^Y7kjH~%cNGe7T8tq{63Mi`T9$u@3^MG3xfcmclZ4H#}^ zMC?BsY9EO84zQFm~VnKj}tw#513zYculOL&I zOJCNO@L>NKDjqc9<)|FCul+oYU#`VmPW;1zh6?Pk$2>T3n(}c$EyVU-Srb4Furp*iLuOa1bGQhf{brw%)b`hH*V$Zg>@9jOH_$uiMeU zP7$;AUjpqj7ja8ZF=zEygW69^lWBc9GsbCxLlfGB*gYpn>t>c&~mMtp6FzY~v%jV-KX+u;uNT7Z<}VXdS?xE;@KFZZH;PJV39E20Zw| z5jz@HP;>GIcFTEW!hLit?(Jy29Qnzyr^f^VNOW%VrOAO!!UT0)oqId1f;Np#sSo}7CYkwvE+TbVo`e)ot`aP$?}Y7~Rf z3$DZI0%a+P(?La0h zKEQ{MT!yoj$dQtKDpxs52|UZBL1B~+3)V@&1k)l6DN13S`dnJqHWST*S5unr1hN>W zM5f{@P&F=x`fuxCgw7wlmv;%WjMm|mgx|Q|vKu56p3|GOQnY9s$;=l`VK?G`VPt7N z_%{aQ&-hUg@MjGSgl?o|4zoymUmEFv3)tWBWrw#WQd_eeyRv%`w4aD(ZoA#slac0h zdr~1>%+O;RA)Yul>p1@~^NpbA(PB{6b7eE8?SYVH6S{Ry74LamL$BCfD8^Rf58t75 z=vE#E4jaTu-e1kmOliroM9`T-)qS%8|X4`pdLVUUoivG=Zbldsgyo`;0Lz zeFN*}-tv5gIWt|qhLgMp2&vQ!P~({FdYq$S3l-j%1v zgB4ich`m@a#Fj37NT;>C{kY}h5pCW&(^f4f8tFfmA6;@3xjY0g4P(#p`}%f9lw>%$35-mig+ANvtk zTFim5FN$ji0xE?Y7MP&=`Hd{9dtq{@WN?ecVb<8(Ns-H)%$Rj{(88+KZ*rl0A46thm8xz@+iki^~CYx@<;?)t*5 zw(k(reUY2}PLu2x{)RHaGgKUJjI-st;h*hYEV=(20&RUz^Xx^o>dJYvZZE=>38&cI zQHD&iWIo-!a~cA>(qU_UF=gI<$S+=IO?Ubq@Sponq0b^c*4rz=+(ww9di6b$dHk8n z{IZ)>ev=mEpK&zllnrqtwKkeLvU!5G`qXhnECQGxG^-5$}PN|qV_i7-*5LD(8J8`xf4Wg&9yPd(Jd9stXN^f92Z`!dhqpG~CFB;er;T#F*GpHbpT*$nf-Icvyn=cSAL6yh z82<0Ar=;_0GD{wVbeh-3zZtHyzukxO!m40ib03@;wUh>h%%VU^D|G8V#MOlAP}$uj zX!|1sKmKiiwyRdm^s^$_K^`p1PhfAoWT@Isoz>|6#!avX)HlC_e=CCE?C`PZb#gFO zJMV*w^O}X0Qx?;DI~%IlEJqtY{=~(lJZg0g6>5Grp|qoOFlyD9oE>iOY`!LTPS9e-A9QqBTdn&NN&b6)^j>Ic+Sggs-hb936f=f(!L-Br$0;Ce}y^lXn;MH!Dl=+sggW z#`Ngc?qsSmI{*Xmp_owd1OF%=fF;G3Fi3nVU46L{dyQ1wZm%~MYF zdSEEs6rT^LIDZoQ7?9@O$=rg8`k0tUf(`qm=w0C^)bO-m4g2~*`-UclEbzhDGZ}(Y zUKwQB@h_(q-74xd(h#nlMF1xj-scxtn$djW zC^VI?LZy>CaOs2`z9*>)DlL_0-Nt)xXYDL#zcU1rj?RFSvCHZCxU<3q+DSMj{URw^ zGitwg1E$`wVx!+BYgo9yj>id(z;jdOSS41H_GT;kV2WL(}-- z^y}|9I`vSAO!j_8*{6|I;(Z-Qza2$r0_5?o1KUo#fcVICh&_Lb+qiunQ;HJsgMaO& zQ+X3fV3|!J_P$^gkVqqD1Yp$0L#SF&MPa1MyccEAt2=%8)WU~eiuPRQQ*L9|l|oc* z9L9Ew4`LmYYH+!<0lTJI$&PydK+AqvmbyU~zqxqRnK#MUxqK3KT`1s(|DH*Dd1Bad zK$qEE@SvIz5i~(^63?a!Scms8czt;}ewtj0M}ID&I|f;3y}%N*LbF)YKTRg(9!d+3 zenMkcb=Ip|2^BAlSZc&vmil!MEXl~ivC1phf~+O*X!~8L-TxkFPppd=`*jQ-ardyHPD#XJMh+xL%8UG3Zt)snC?I!Rh*5Xq2nSc z`h^0kaa#;;G@U5kG?0z^mPgZW_HZ_D6JgOH4N5UNg871Awtw0x`m%L92K*S$R`1fo z_Tk4-Qff7qwBZ!DOr{849$Zcz%_i{PVc&V%Vg*(`{S*It)FfD9y_H%rim-LO9KZ8| z6b&T&hDJ1IeJQ7~c|jh_c^;2*JP)zd(?dz2ZzLMNiHBSA6VM?iRg{-JgELp=;ixyd zwTr~Yu%oH_VE-u>7XLzut=$nv&m6M(=_z_tmX<^d{f@FUD<^uczY!nS8Pn!E13Gul zkL@t-Vo{1g`Yutrsz$LIWr(57>k;Ok*(h8-qtMN)t z00cgf0)xtO>RdaP{FE-z)p;YxCMy(O1M^w)*I1-8A5qV(ic8rzne%G3r}KYSGbN32 zOig_PWXmVwj_hAtf7%z^KJyE-Z%}9ISL|5(Y)3Y+>lSQLh~^D{ufYr9KBSg>9QBm6 z=*yeOC^)S}f#18ZHp83-T^fV8q?B-+_HnlKx;PvCs}EMnsZqt734Bw2D~#Lx1-*I> zve{~ruzPARh^-W18fEiQ@`e#aJ$GhpZrR|JA_or=_oMqvKUR5QAzmrB!&if}=(X|` zx*;`1VEYc2_Vj=Z@HlY^^R

Kcjzn_Ob0%{Lbn-B zC{?};D?d(Tsj0`H@FZH_m>)GdwsYwi4LAFQ1vmsRQuOX8}{?x1ywuJKZ%krg3FQ>!!LL!DrhvghzrNq3__0 z=soQe7HFKt#$0Vu&Jbr6AJr*JgmL`3I1V$+HHn0kj`^S5DaY-JAbRd;oRVkB6f>-8 zp=~|iY~BNFhNfb|^BH{Ev0>DB&zAP4jKQnUQ`nR7uV`$^FU~bBnVeSUl7nM0mlEsA zJYI_Stvv=pEl&xSlfRqo7#zX$4s>Dfi^25zsU=(Z^*A&!Z~D-`jmn0{(uboWeDIJH z{mgOU3z!Bg_-+OKypg0jtAN>mF=n1WZ$o-xGZ&WB33DCp2^Xg{!k?0V(D^)t-TU_n z1&L9xZVp%P|5l9dy%liNSIfdt>0InD*+_ZrOJHi#ME=_+AHH2biKYydroIOXu+GGT zDVQp<#~k+c@x2*|xN!R%RBM>U zs$3HJ`m$Lp`HL7;pS_NKYnwn*tcf-^o`)j)Y%;K4N*6PJK*(lg%63+vz&Saf(lQpt zSaiXH>Yu#l>^1Q9nln{ydB`6z$fk$&e*D3VxwK#WiZErU1mm_?;W=1Hr)oZfZLm4M z&nu!U)kko1rVwO%3bAgr6F)=u6HQXwMAh0JSheRk9?(6>`;`@A4}Xg55icXV*tMLO z?G!pYA_uL_2D4$ij^XSe13I|flT|JD!i-`SlrM@WK428*%3AQzsI}yF_!~$*Jcv{{ z0C5T`#K}z;R{WfWG8<%==F7!+qB)73oZ7~PzRY0VGa@kW^LtJ~WjfuJRzdTQ3tWv} z2bL_2r)Zs>IR2!C&@IA|rpJ$`j%QnF?g$e&WuZjr0g~+Pa|Ieb>Mma!?hA9*v{LvJ z4Z2+Y2L`R|bMop`Cyo5IIG|R8zJvCH-Zwp%8G4rc`%RAJ27Kf$pR>V3SA{VD;4c)& z&F1enh2TutuTYw6itW!YLO|y;XpY?VGV6dL@_Z*DIW^ zG@L2|M$plie6nzfV)Fa9q1vL!G<3um)YW-XSGq_7%wMlZ>!qo@&s8_lJ^qIl&XXg@ zzb|oX(<|=WKPk2WVz9x+gbf=s0J~yr&?DObR}HU)lon-JkZ>M%l^F@*-iNYDkF@D# zcmzg`?W6}g$Fb#sW7y5#ulUKT*)Z`)A>Iy|#_VP-fxwVW~l=ZY%Wqwh|o zb$f8(b8Ec(Yz-b)sKkT)*6i=e7N}i1mCe}>sEg}atW7QVG}(f_H=RfOl{@I<=~uWb zSe{S2W6mseHJFKA8vQ!`4LNyGlcc%1H?7K<@M#9xxNy6c;hOT4ObJ*lbig}zctt) z{tlMwck>e0=irT-X-sR{QjFYujej?09!*FvM!osL|DY>v za;)Tvvrp29zhMHS`PHyNN{XBh-^G7N4)N}jyjgMj0)C|XB$76Fg^9u~5aHMh|8)QI z{@1OU^uOx@`KUo`k6;^f$yZ|KtyP@o_Y6vL8^dR1r}JCXx6#~>JnTs}B8M3U@Ww`r z&2-PguGmk!+$NsxztW&(YZkGNlZN=BTp5R|>cPKL7x^wd6|$VQ2>kUlC|t~%38jwU zuLCpagvU(MvD?QVz28QCk35jC&IOZ_&zzIAIhL$zCC|1L3{LoovNtclD47J3uP7j| zbAEJF$(|C@jB&I0QwkZ#0ez46_^e|B4Ln%HE2kQ;3yxES1IzO<&GRn4@vMW4?fc>U zj7H&yw9#y%&1&vL7s1uQLI|(4qR~Qk*e9>R)?9l)k5`7!@(F|4jBW{XIlq!Vc9?L6 zfs^>|c}4j9aSu29h8&eVk!6c2tZ>V`ZZ2eYHGl6<6z1pbLd_Oc8rGEuaanWNDxVi% zwYrxErMSR|?mT`?+g+?ZwUSSMB!g2YR&r5AU3jHfmVDZDDe~a~c)neQHpj-1`@}(7I9sM1Yc`r;b3u>rtjHH;M!TSR?oHvP z@afbkw-?r&+lwK8Y~Ula#cjh!VVkZh#d*Dw~>b{#*^pG!(brL#Pk)}(6&RJW>2i4P-QES9Hhj|VG2o1IgPQ#{-n3ZkERI| zNPq4zKK!#1AKb#>++{MD{~>}u2)Xq2Rxdt$84Y8O`BU?f>)2m$6Lkx>L7(*%;rNI8 zO#6$GXns42V$C18u`&`8wl7EbEwc4b15NoWxhq-2L4~>lb_rBH-HocxnBb4V1MoIo znqr4$V!QW7H0*mPLJfjx*0qz+U@BRsqxPHEcp=IS%uC_s(k^azvouREzX5(4K$F7b zaGy|#bYfi4x?>PC3b%oYYYyQMFB5;xxl}E6PFw+9&zEt-(XLpK*iDD~0p*_kco^fF9qorz__Q zVb!s56x~`%13|6eSSe0k!v)+sMHyT*JA$p9kWX*FhJ)(VLJ$fRS!1v!`kgw9Bl+E= z8eEPtr*5L&=yot*!{BJmn zo?OQtQ`kk?^H;F38lQR9wOLd$r5GCzJ5V)_qoc!)LTtMQXKH(lmMx3q7fVVrR&K;% z!%MJU_6*3Mm#29*#!{GUwUsLK|}O$w1>0nh09)$gHOrazci-c%_ny{F@`;@NMkkbncSd}9`sIU2e#}_ z8DXLOR?xg+lVHbuBX8N5$~w2k9#6~zilk-cvws9_Iln}ZU;*?KZ|ic?Qs3B z4w#_p#>!`lla;ely{C3Bj438*?xg{(fC30(ht$$Byq~LUk>lbf8x}9t_dYGmO56x-zS)= z5J`P^ooSTXK7?O6q%e3rDJ>tvj?H>clit7Pd^MuE@r!rUcIipfTke3nQ+!#-lHXW3 z{tj1Z`VNCmOA3~(GiF6MDro53WL#Cbl=x0h<`R=2m|gf79o~hqrB;D-OT-&~nKzV) z_ZTqks1z6#Ax#Elt|I-5KHHLYo4b`b4ot2_alhZXVETbf{Czc_f|oC++uJPJqJ!Td zb;wRk4jaQe{|Tl3`Jceg@)Bph;W+o;pb@dD9<+S6CiiugonUahEvV}YSW;#LMd!rB zo^yTZvs@q5ZSKRwfLSo3JOYwS4xpt(3{4YZFfnNtF=uHiEd9}o%F04c%A^9<%o3wJ zQ)hEMM&i7+oh2Pt3q{YUYdGnpKK4yV&cNY1RHb%`a+IZ%QYFL8Lca3;HMY=p>lOb| z-x%+$(Ez*a79^;c%g$S>vp0{Dxj*i<^hrMfPamH^VfPB4;&KY7=d8qh+a^%auZK9o zCY3(A#G}Es7&iZX7hV-}q@(3!ILOzYKec)n6xKcfFMC~Dks^w9Q|HqQ?PFlDMGhkT zW$Bp5Tj+Xi%`T0oLYbDGxc=)@3aXk+k%LyDl!FK(+bGIEN*XcQC5W$kCBp{H&M@ta zgIH7EOhK=o;Y!!xG%7-j1 z!=mc5kUo7ZDTO~oiz8of?D87L8=`3N(Fqi(E^d!X_ZdlY`g=JBvA=Ww$ z3SutvAEv2N|CuKIIp0b&M_-}jx4HPq?K6Ms@j;Y`8$n~QyWzS%dH8+*1IWieMT74X zu~yZeDd;ES%?rP9@7rxO{C7UPAkzggx9o7)5jEOv8t-)Z(;()&#*M^PM3|zmf;nbw zX5uAjWa>AZVXhj;`}Sdw-8X)hi$8l4a)W>T(vpo<9>Z3@a)5hv2gqSd0tURO#PYA( zxa0Evto!u^xU+pJE^f?a9~&Yu|FR1^zU3Q~yp#e3NQ5Ugk+`Zqo^$D}!G|KA_3GR4 z82^<(uI>T%%XBMy`rs;8@MI+EuWv!WsWo)BN0XiCtHgCzmO;Y$STZy95M;PZvT3&o z%`3{G85+RN<_cwPo6D^oCdW3IDUq*wAq{qZ3av&*;P&qM=%t^`)SF@$ir{rT`sy z_dJZ=K2V~nU%OB`;0Sv7Oa(g|TXbJ-NIP83Ve@eZUb5&AZ9cq)?#4O5qL~Uh2t=GSZLt`s(^V+k* z{ULoq<(4F>Snv`TTdc>5MjsI`yBS^6x59`K4LDB9$Laip2-yAU(T^C zPlSclcc$?QE)QwncjVn$$3Rr|891|3AH5VCnSFm9x^B*7T1kEQ=|BQ?sjVQJoZo^y zg_3OJNok5%a1=fmB*T#Ha_r&W?ZTP2E70Hi57=sK<3|LXr)%bmaM;~_@MP97<~sj5 z4*BFq^_uH>8z0en_eYhJI@XChyZ-VDiqpBDix$vQ(f4xq{2aP6LY39ji`wf-vf|kv zxX$l|qIukciBXNj`IwWF*%g%2l3|0Y0{Gf$Bl@)>9vY zmMP)LN|vkh>4{7DkkuY^XKgOJ{`vte%N2xNUIIFcFg4k{Pf+SKg%z)HBzN&J2D3KM z=8k{DoV_F2t&iWa*;Nyw$8_-PEAm;g#XjNZ#$A9T-RQ&LMmW^Y(cRCFKtfpych;?d znUxlpwKWrWB#$M>)X^+YW;Ok@Izro9oY~iNogpK3fA*0Fz17*6fa-9VE`Q0IFLrt1J zZKzY%VI%Mgzl3H-oxyDIes1~S#mrIDg-*;>WuZz*+|-S_xazhp25oo@2_Y-^8yhn@ zPs2aBt6Pi9-F%iUQ(D1vHtVv^m17yZB%tU*6Z#kw1p=>nh`Oar%A-@*`?`GgYMvXn zPfU#5JDyYZoH?v~>K9Pk^cS!8g`kO63Gok`_-QkZ1+tntVCne_N@FHqvdT0T{w|zt z`)NWFG9y{fdw(8%2phMopgd1Wc4!IVut%w+IaZuiORTN?K6)Y^s(TApHywjfC%dsa z;TTlx4TdL?BF^!FD0g~kL{DZD3|jRGC#{Rb#QJc0VloyP9p3(b^rtQPvSGt zI;42P80Nq1fm^?nsCKe;-TDe2zU|sgGV)$WG1Eo~t0J!Ai>egbQqqOlsikmAQJUto zopdTU6k$@`{{#wiL#gHMNnGc72>!I>QP~& z4n4u0ynGH-R{Ij4SH%r^?@Tw&HNb^iU3~L^GMC$u0r53){GpK?)37kR9q!zO;*M# z6n9??oDK4j{ksCrf1R<~`zesjKmOQ&2%p*0BM7-+iYMO{^GNyJOzZRbMW!1&M0iJA zo1@^<_6;!NF3l6vej9^PLF-W|AcoZ{NAvGAQ{eSZPn4LplQ1@vT4TPLy;T!XmbtvMLiRvL7xE0k(HIpHCxc$X|K-W3>Y#3x1%9u@3zW z&ZfoR@1c?5OU{12mPnjoMwu6i;joSyY_F4}_v?k|uvVKopN?iRKc}&jGD*5-70c3p ze#C24MdaME2Y!1L(sM~kQYxCyuKms92SNqt81ezH2OI#eB2_Y-bDGZ(Z6Icb?T1~_ zi4e7)!1z@uH$hHr)8? zyzqpQG+pYf;Kaf-S^uDJerXxdiP10cihdD)Z>0u>RL#cib7rB%kBb<;p#f_$!kB{L zOg8)ZDprt`27i~WR2^@UAKl7M*R|fXPe1(WGPPgDMz`zZWIw=3-1c1k@U|JbTA3V#Mll>J9ZAu zHzlxOZGU$5z8O8Ai)pAePm(h& z_(wD5JfwTK`}w#-!8mS5F+QEVlpa5=!)}K)I8A?~aQ%%egq-Ve@4*-r5ipfCDIbKX z<1eCk+8PL~yNk1;-7$3iZn(JjA7WoOxVKGbU$tIBX31)x)3WI1(1ocBT!<}chGdcc zbJF&gc&F2?Y!!B?WKklT%g>&Jez>L<9-33igE8zfV%$IF^2d#x0*lbapZ5=&??hmnHwfQd~ zvhfbO{PpH{%V^{9A<1lIj4^&wuE8B@1(>;~2tA7=>y4uhklkcyI-AZ`lLI>X% z%dxA%8*s>>8fAS%brO-{RJvC{dp1izRDln_MYmnx7IqHq8=pb<=X2ojWg|`_X)2p_ zW(Ak0tH(SBct`zHog$9hk0e_!Lz9~`|8XJlgI0O4@5=jxuGzNrF^B7DjBp6aE)k$y zwHn-BHJF0t9e~@zdT^h!=fJ8 zTlSXQm{ow?e{-2Z$^m$$`~yE|RpKk-@oZSaRy-eU2*0--f}t9DY@BL6y-$>2r&RLs zj_*uNwN=9QHWMa3?-xp+5v`k*_wZ4DHNM~fh@&INK*oIpB>CvjtYv1X_}l}g2P%Mi z{!HGl;46fDO(chR4Ew)^z>5uQp=H35Dn4g|->{h&9-)P2URv|w-%>f(Jbg^bkzyV? zIrv_r(Kpg5!iCM_z;t&$x_RuuIgaX}snSYul4|rU><(Hcs=y_68OpqMhTP1}nZ4Y0 zkq##vBeJol$%oZ!8P4qO2Y8#A+GKyC4Mwh%RH`IV(Q)_R47-()@k&^)$+AuKYlD- z>`5Y_)iOxE=fpfG+TiiGV1^Up;Z zgnJj`P^&qZD$KGexabP3bjgL$Z(BH>%|mI3==ts7G`8aF57gXTOIKqHFubD>XHLDt z*X=b!eceG2g`A9WRJ(9B1!k8DVfD6dIO$Wu4vzf6`x?%rn*8DPspls6 zch|$)lu%*OvvTNPFqmv-Z^YTNw{aiuHP$Ar-cI6{X3X_omGGy;67E~T9lk=O&3^IN zm>W~{gWn&xl%*OZ(WkYgbS=q(ZED)d8V!`GD=&@a4LQU=vOC5+E>35@0bAH-gQeUO z(-iRjF2%jSHXki}6v-?4G!@r3;!uZt)>*ZZR7ASVzBi&evHCC687aatv?HiWgqt`S zSd!xQV(|EF!6JwC^Sd3ix$d`_ICfYrYx*ljk?Z8y@gF;#;y zq^mfFnxP!t#~RUxkEU#z0;ywlj8`3g9gX=q`Lb)RY zDI;$@4(WXaU)LK^(AiIDJI0OnXS-5gt`@g-&3t&;^B0#a;t{?@;e?Mxc<$3_2+}#o zlw_n?=oV`xc1nw;YKAk54=xydcrhnAXqRYQajYm&iIUY7$jHi$RLja~e3=v7owo}g zJFjpGdo_awoBES>YcM6r{^PyVW>HV>dZ${$R+3W_VO$zlVA0Ftyl>A-?nmV?XuGBj zwp#5tr1>wV_x3^6+6$b);8y-p)DN(V$z`2nj-RS#plOYKefInuw5k|EI>!%SxgZJq z)$%A)eJ+39h!@ptox$oU1#tOd6r6P4$-b=k$vv!4rp(R^{B}K>S>~L>&|BiH!}=1= zOAseME}1oydlP%mi2>7hf=KRr{ z{tk*yJjrg591brPL*MgrG0;DVpZPix`0t(Mz)u0FxywE~k7Oqr%xLdnpxBk_%y74M zy;J;j_R41jfK;p~zSl9^Z7NK)s!lMJVF=XjD*&gK^^JI%T68q9o} z#o3*!2U)4AEE^m2Td+mz7ur603)8>5L+7C5cvChG?#ds6=ctd%lX^KD=}N4zlcE0d zNo-!|ec`=V^Vsmsli9Vi%NYKU6ZtCxq=4z1g8Hqrl0eBdAH3e*i6b4yL%qb>HNTVcRl%)JO1FOEgoz|t~#@M{DT(U+ed#^ zhGOiB419fMGhG>aonwD9aqVIWO8WZ)HIzjBW!^=R=2V?3Zq;GbWo@|QvJ%eh;K*)N z99?Q#N84v#!Cm1@`iiy1WP$t~DpKz-+MHw-voFjE0*M6?{za z5RiV|iKaS^@I!17^~IOL(9)4?`}ivY>qnbe+MKiO@QjV1+kKi!e9mEWyBP>ntXTdZ zT_zit$^NB{$LF^XQO}zv)Fi9SU07reY3GD!7JL#MHdvD0{16(jv&3+T9Pa+h5V*2Z zlc|pkr`|7%f$eVQi}QV$lSqr{c%KvTmBW~`sTnwbRH3gwrO0#KUOYKip5*qf<}YXM z6zFUTq|q<*n9sOO9Fg<|o8QIL$TeL;t+|OfPSTsUwHhs+u*Qd>`D}yU5TuWu z7}QVcxyk-w`}jzhEd&@vJB)6%WTtTstv$p=N! z;G-b2`37VTs>7=5FHm8l63L#B=ZEg(*{jaAXlqvEibE8G*ZgL#!U;P#CNqc~Ej$R~g zn78FP?v;{c<#CSO9=&;}~{~5lTIttWRtI*Dz z1f2Z7mUG}Q3Px|gP7U@YOj1-&GV+@X8P(+D|6=I8Zbp^cy-Z|>PQ1}rx8e-RLkn+@eO!;L`p&7fG@0Msb_kJCB*2NXV z)lP}BVtTkIgMM*`UfOZ~%Yx}&E)RWW?p(~|Vf6TxGARXk@!P&w@a7=m2fn$EtF4T& z%E*-BJ`Lt-f4;z26J3_4>5TlR4&K1#GS+*jb|&Ux)mIUx>Tjhrn&gB3Rte3DbgiajiXzh(*=oQ0XuHiHSOF&zCp+ zx9S6I{kwbI!S@oZ)HfgE)imIcfh8{MSxqnhW!JjYiD9CC6aRFz5zV~f!R2;Fvt7US zslM$9tghkEKG7Ag?`lV>Ig@C8YCSg1v7q4_jzin(BY4@z0rot~Mgu!BztBa2=5GFg zFv$zoyDp*676rIM=mEc(`Uv^BNcb1j0y}!IqtlU8obaq2e-z2F6+Yfn9bwN77FWT* zn}A+Z4fx|0ryvIT0p zTmnmJZ}=GTi?@lp!PZ|{OpX_TXx%348ugaTJ2?~I_876TW`Q`?`vk9DD&+5JN5Q_~ z8Eh-f!_CiTIY5;lfGc!pD7>b}EfVPKd$Qfk!bcVHuu}Rb#ym zlK2Y^Rrp=~Ecf93e$MH0Bxv<0VfXwm=yo>*ZpztE?f60b=&n6@QsO^utX?P#Ht>Ky zPBW=>iuFj96Os66){J_w9Lw|S3+%tfV*3PB6*4Z>(ZxtEcJc)B%KB3iPBe)-b zk0jSEfq7Ev!7%eMPTBT^dT-t)g?VWhzsP`wY40Z3FqHP)8$`BJ_AEO`ig@V+uHda0 z3sz)7Qrj@vKF^D6!iA3H%WdG}ZwsLcn)IVMifSGw(V4H~D8}wFmmuz>_WjWq-*gKX zE`9?q?g)K`|4d=v=z1O_>-jvtShV*_zIfr#esPz*_2(y zs6ZvuHXVi6oe#+VwHNlh2l)KmJ(yOz0)ofRCkM|Sl*?0uaS_S1;qr0(mf^=F51F#n z?e=(}_ya!q`x(n-%%xxLquD0K#Z2dy9WCqJM|-=eII#`4;T4Nn|xNiWH@``AVG4I8Qb-w2Sn`?*#`ILoZ5{bUQ1>q zE#zyVbx#}GsD9*4d)?XU`u+G~c{n*JU&G)h$H@1=Qr`NE7wO(SjOQAZF)im2wtcMR zw^+W!OBZ)gci4LI+dqp)_uzqv%%bVESsXwVD7UDnsj?S?^QT~KN6S%wWD&$ZSHu^@%>{yBFu=&c9x3`GsDql zTMiskb7zOHC(}u}S+qjvng<+AKyg(Etas=ljjzL*fy6XkrB9i@b*oV9f?UqWHkwUd zaR}OP6>_^H*091Gz5+Xw%8Ks1Cd~C_shcg?73qiU!xBw!zSs=m-4gZsS8LJ#MrbL{xdeT87vrA2=ds;xG~+X037O(V_-7tV zI`Y@Jt1Fb5b+7|Hp7$}UK|?f%@jq*WWlU^(Qxdjkl$H_V!PE;GOKOl zw4_$^)sC+z!Ooq%kIaSv{fP0#(rj++PIfq~SX_HM7y=INfrF!EDalpnYlaPHP2Uco z)|j(+u42FVUS2&aS6h;%u^xVU*Cfhqm`N>O&+zwnI~M+3hfVpu2J1wp*%;6N*i_}6 z)K{__#r|Pz#vo7jJmeJ~US~*e)O9HMur3R$@ul~pTy-$P9M2;f#Jz{{Whd>3}mXvh?@1hgd0T=e-Df!9OVuLO{<2b(7 zrT}|y>eIx@{%lfZJcix)>!3aDGM@CegTl~r%wzZoI(bP<_A0d)@FkZP*+fv@uW&Ff zHKlVOUh@AW^VszT8^woP<;Af}df=DCJUl7ZDvyj}K0Up7=fi&xEo51DgM!(@4ht|; zUkM|Y-{X=dAEnmu4lrVyB8%%7OokDLV7x7!uN(G|kOH~=PAHIqU4(!5R zhGXflx`uX2d1(1s*iFd|hA~^iP_tqx{F$rCdS1l9=71>tZ-+T9WJh@e|0uLh zb)(uDiBxEoCzcIKhLm*`;;F47n&5N@&ra>)lwu^=+Y25rEz=#V^xfg4!wqs+whv2xM8Z3xp8_Lc|Voq&{)woG|T2_BfdgS+Xg&y<=i~%U$a{DR!@S}#wBB|cO;!=om_gl3OUIi#|JI* zNVBs`Y*kqRiyAkvD;ez&5<4B_A4E{#{bO{r_b1FByAXq+wip)QHF-E0i(iX=Acy2QTA01Vp^5Y3Od$u;yDBZ|P zF&Eu!J=v6)U>0~N65f3CV?z!f=Cj&wVv%AwINy521zTkEEC23-kQq<;tz#G|Tg|6# zjZL(2u9?Qj&}iV(JF zz+p7$TE~3lwaIjf3%%QXh`;$(ibXkH!oP-VAaq|M9N80%fA=I2na^eYeeZBraSRpc z)N`)OlDLS!!$I|fz|?teq9l{M(DUXfj?BG_&%Jf&v5^;M3pwTH@O@a3Zzfu^;VT&Y zDTk(0;mE5J@6RFX;X6*|lq;D1E*F(+>xgUI`Y|p(U-Waq1$^S)NDY;Zu+Z3uThY@F zesZHB*{_C;JX``3qH<}<)|sTedITNby_@+z4WhD3m(g*;OZfS+nbuu1q=DJ1NZ#zW zczmE1nt&56*{DIK;~e?c4H?vXO^HoTw}Fm`zv$JmpR=o zI?vk#C0)|>o9FIBsdF#*@cd*f+dhPiY~Bx#JZ{msZ<|1@qD>ROZ=$+OB8uI1h1VbT ziR!Bx&|}GBG!D81DoPK8EH+r+zlp#1c`Qwpv4@0jLrAhZmUgA>rploQ*q{|naN1!y zOEQmVQ-rJ_*mW9RsTff|w~S-cZ+*j8@{(YB&XnCaU(4GGc}J`DB6L|VPx3Ld+3ElV zYCq)6T5`3~Y_f<}3I7|fU$%nr&Pr5}a~RteE5j1M1~^`A$H|W|r7z36$zyr~RPN|T z^+oAS;e`gQ_ZOI-j}({bdxX*V`hKc>iOF6`RIakO@!FE+?|(V{ofP=p+XAnDVTmc;$^5I;9B71Fb%hn4#eZduHp6^pcJHy-5 zU}2I%y{#^J({wfOPKc9xPdf%(0Nz!()DpkVsDxdgwt;Ow=txe3vdAx1U{ua! zv&cJT5Li$Mv##4=V$37{V33f{zS1X~-Q%b+u@3a&#*x(-RR_naH83;X8Jr{nagO3M zoUWb$$&;2dxh-4Kqx(3!AJD@eVOOcg_b_?uzvOf7`?0LmcH#$~o;d1KG%jDRg_5sA z394LJa&;8Ubz3W%v#9|_zaC1;*C((OIs@x(Pd0#oE=TE)#&dY2RKitX^uWyPvD6}6 zgyji;uu1(8{QUP2^Rs*S_)AUTGW#PpYMUJ%+h&Tn$x?iU=PIgttq2pt_Q9I@^YPIF z8G5Yog^yE@f;;~iLifaYs(7#pHO?roe}keZ_Mt7YtrB$q$2U0tQt%fU>|rH^7OZKg z0phGwPQy@zT{e6H@0HJ@>B8Zh)@Ns0w9yr>P2NC8)q&JBa38e$^Jus03_PsRrzyTl z0;d`UM<37*4o{)pIukVp9))G4S`?<|12g-auxwQcZcI9lE7Jxr z|KKS!`fdbloiiQ^?%#8`|8oU@>}DD{E$e0S)#}u4s}7HR#^K2-O=`$F1_=sE^-E$+ zv3Q0W^jF8Rv@I%_s@Kf9{n2-j?p#XSofo6c7#~qzOCYY${3`zYY$itBEMm6iiu|vow|M72 z<4JbNE^w58g4-S*;3Ctlup=-YMjTh5R}J;3l0Auho($s6xf>{6a*ES&kzxjR``J}{ z6&AJ8kS@9}hWL(^{Nojp#037W=l0f$C*s+wKWD?c3{=-J)&y| z?QqrMGmv%k0BrA4rv1@xK(ob$+ABICsUuA2u^xgSH`S@_lL5)xJP*IV?t_YRR!sM0 zEiZAFhrBlfX@sFIxAo~E8kTwwEq2(^j}{NM`$#pDI6shftr8fN@h{Qfg~vylVMW42S|zK_?pO~JVdh8_tsO%%Ln^3f*$jk%$FP2P8@@cU81G&ecE~$I z;Br+h+zGn}pJwb~bJLQz)aCMc{Z=eC<2JUajjSky5ZmkU|h-bW8m z@=ZE5+6S|)<;`4K?E#uM?h$b^sjMMTiSgG5P`u7d2-EpT!K0e$f)y6hF0F3);}$fofzz*Ag8oLbWD+P% zqHPB3e&__w{Z0b*&sK(z7Z>@Jfu^{A|3gT;rHitvx)d>CAw2)|8+)d%#wqWYFtB1kw| z&>&k&sLT0@!_pYnop2Z*{)xj(^EMb`H=jcDtHG;$JZX1LVTbjSQBy|^%_o;p*z+i| zOb~jknMo{hw>^2~){vK70*$l`Wof~=bjDtnwFthJ4+oAxds+f@nSEjw1wx*_%-xm^})VrvX+{cXdHg6~4!lRNOFT@o#)Cu7hf zZH(+_xqv^o?^@G2Qf{1@H8eg=pq+P2!@adUT z4*&L+!u#8ayz&oIwD|Q7=QMxB{|p1bZ1O;EOXeCl=&R4Z^+=HPm_}~-lqNjZwUSS{ z8%i^V%p^-yJ#u$`hor!HdaU{-WZH&3wCjBL6<$ zjXkuk;}1PM28SKO;h*JqZn5qXdi=P6{^`x-=IY0zWd_G?H4bCXk0hh9W)P%mLRK@D{T|OUkEoE&a0z@gdT+0j@_7*2_Lu;bxXtoPm=4hI&Anfel$#R>W z?{lE8Brpx(xZGQgdN!AHd)M2r_x|_s++J&FYI#K(|6rWNhW z&yn2;Pl2^_roeOp-QreIxm`fV&p44nwGy{da2X}{N1^51 zlgN)7#9ltO#p)j@?)9)rT=J> zc8$QDeIUKiMeO^q0qj#pxA2DfY6S?|qUYH$l8KOvXjxt8O;ZM9`>`##~* znfb7_c_j;Z{}e+bU1&_|Sx3*QdSVyDC2g5oE^$97cch4?(6P~#R{bm=9|kthU9 zbr}d~d4&>NL(x*~DBg9qiS-ALC5sMsbPR~bU(F5h+vFt}nKocnnJzmZbpL`s$T7Rl zV5auG5)}-cF!9VR=6$i4A6A<~2lN9$>)~U*^T87cIQ@m&(X3Cm7RQ1A;I(A;MZSKa z+;xgdt3l~Yaxf;^pPRk068r1>5DVR4P?-9=ck0SFQiT77W1K2T8_RZ?t{l^8}kZ)Z|9L4esUt`TcQ(P;*oj-rW9R{6m;j*lqn2Xm@ z?v||Z+5TY|USY@Kq0V1?5_1!d@)EGp=^NU5Yk}L)clcRN@SW;KWAMaMi2q%M?`A8B zoetYTaL9d-ay)>Vlk}PUr}fY<-HADE;Y2sjJBZ#i?}g*Khik8O%VEdfe0p(v1#TOq zMssdNl2B=ZC!gflg}Vl9*N-09e2K%{!~4J?;HNmyLYXaSzmNMQpFs4!A@!GM?t>5u zfmhw9#KvZ6ldZ`DDrlca7?*)nR+sTu<}m8h%7w1iCn?(JCR|ZXAV-}UX!&O(t=KSJ zaF&0i0pZo`h;=uV|6?R3 zmUZ)q8vVFEc^WRh(=WL8M$?gP$qsBp1b9>pLhait>5Nt`7OZ8oXZi;8Q+WzOmlL_Q zYi@83*Dj)u^*C}hilRGp_T1WuBT3#Z6&^|latdwI%)~I1L|HocN>&<_gc&rn)ssGt z%IBA-tJB|&wlwb1IezIfVK&W4#Olj7G{;>Ndj}`*Go&=B^S}r3%3XT&@2nMVDvX85 zQ$0*I{xRH9QKh7|`|xLb8J8KIC1j9`!S~lTK1ogR_HAdBSE$Dx@A=2iQJKXbo%I@5 zDc|RxK3R*IP0=uT*&lIO`E>dmnGaHRrFbT(8#Pbu#;0GMnZmPbzW1~_OfFfF+0;<~)k_zj`eG zPi!T+*f|SQqxWOEe>Qwco=TMRAGpLcV~EahlAj0^YvE3#ZVX^8rTgfk=?Mrq=S61u z|B-p(dr_QUHXZKy$<1B$5mp}<#fqwD;C>GekHZGV&vbt{U%IkG>-7AB$chf~Z z^$J>N9z-=UOL+IpH28K(M7M6opjmM=oxAu8FK3vtqQ+;!_gW)L-{gUIZ+Fm_v_j1C z)rRXiimYk63x(Y0ae$eI$p6=T7+J&9>`Uvgiq3N9GgUb)&!u>~s|kaR7P8&43E&`? zMQ-Y8yp_oolV=D$6pbWWJXwMY_spgdH>N{pfE*on(Zi6jPdV=TRvdmxmwvW*qpzho zPS`(@-`=#78n4x3q+&as68t;iX8R!~cswmwGK7`+ZRHmRP2{p>?B{2z&7l3UOQEvR z9fsTPMbE;WsMxiOP8vtD#H=QW44*^i7rYdIZBJrat(8Jk&Rl}(?#0c1ugG0pfFaBRN7b98=!Py9}t`$Cd! zvk`i%qqUg4WG|H^X)({40?Uwm1|v)pnful<*3^3#b=oxOzqeDEgL*GG%`zY}D@EF8 zp+s0oE*H0Z*wzCNrmmAT^x*~LiFW{-9 zhCB0(+4f`vDZ^@%T&m0zDrM2YC7O1X2uwq%97rGB%W8(@k#2xNnC6t@)6?g%SjB*RE*MZoYAv^6 z#xk;hu>|J~HDtFl*0L-O4K{m353gYH6|Mxz;PP{Mc)tD)ehRDO#mnwc(2godF!_a~ z{1d1;5k}ep4jopG`c*y-4{u44vq~V%i;!R~o0a%K7i!QUpc$44+hWNXQz>v_An*RB zpZA+ojK5+6NcQIq2oh$7)Qa7uLR}*--zDi&{$B}3HId=2G1T1-wj)NT+qM}a+ z+*j&AD=B9tZJP_n&Ms$agD3HqPu)Yaf^A%9Ng%nOj3ey>`@r7f1INZT@j?HJ@QU#O z_&eJSuIin@p*c@sa7~=(l-6M8HEs!9)*VkrhrJTXKluuY`uoT_sS_53RZ@7jJ{H}b z3a)-Kcx^`7js7jeRi+K4&W}Q z$|~F?Xl%w8w6B|i?HiX-*d|Zf#1CP+Bs1~Fpc2S^ZNsm-ewB~er^?#5E@xL7g}$O_ zK6iXx5{}$s!NR7R(lD)R%u%sw z{P(hFkaGSZy!p_>pV(?kZ(DD}m0IRMkz@)0Z{G3CE z^z_v$bf{Kh;oVMXmu~^@PBD5b_+1)*y3o33H$aWE;=ai0b9=6Nf~D!2raj6CTZ7{x#Pn zem!9VeN*YcO`?0at*4XUF)e`xDB9EFeqHQvGsVBRx1qi7RKB|M3|G3)oHFfK68-lX ze3g8JTMa9yxMU8Ue0~jcM1z1U?8W|4b5d-XMPr0a%;kwCB~6}BXKI|-r4ldF$ZG*7 z!Z^vQ`(|d_KCTyc4Id9>{ zc}Z%RyOOV(qDLOT=aKh5eZg@8u#!`wJ;Up{wc5e(^|%$e3Uk+$JN z+$;9{kV27A>gW*0v?H==8_7nNg9qXZ;nH$TWi=!x0z5e zJc%2p-%JBCHEDzBBW!!3!n|L85a-^Vi{&Rjv6^REFrdd6PBvS@m=-UJe8J(a+C!Y* z+qq!aPBcUjpww7`%MUli%SH$3)r=K*xUY+Ea1(YHe%r9qzn>dlSqk&+%kitl^Dvti zUMo+x(34T=Fg;)J(nQO^=gSkxcSHfp*Y&~HNLw=L-^8Z8IEt=4+Hid7OP;;mgW1yW zFk<0)GKrO81*Q8jW?em&OXtz$DGsb}?_17t@C;TV-0v}I-O5c0mlt=hJdItg*5p|7 z0lkjD2Q#_p*i&;5m#7bBQnNT%r}ULym*kEfDkoq@#&Osq_2G`#f)XA(P@S(i0% zZ3ff9xFa0@ekx_ZwCv&KjY7Ep_A_@-;N(?qp2p>|_rdn1&>vfJitTRsi5{|s0`qnn zv@#-5D=44#H3@9aUIahATj?&Rj&D8xnupn2zPS$;-MpYf!x_1QjElc=3BjDxI@|)CZ{0!QVR^rbQiq6Jti9gSoI9 zP)#L!*JwIrFOA`$*mEu@|K{6hjRE}{C`wj zJC<$x;fQT}jo2T-QxZNQkE?o^3kO9NIHIPHk4haX^aOjsu)_t%^dfeqUS36;Co@iS;MMqx1_XSF#F*<_f-xxPXpg zne3!6s8SvYUwh8*OM;cD>6vg-qbY(Sc8|d1&mLfXKa7s*LOSCb;nKEWi5m^tA?|_X;C0)oPil8N|34j z6K|cGN)7JYnO8?GSH02(UPVc<$)U#uM@~LVt-8m}nN-9!iAC_EVkmvedIei29|5b1 zQ5e%(FY?=vh|f+sAu1oE4PX7~lj|#7KWz^b-@XNQp<1+j?=JqR@EROFXgi+Eai=Wn zXHeLzL>5_-sIy{leZVEbMWbF1J(6m8=6Nst&>ANCXty8s^H%Koh<&{83tgQ2MT$Dh z(m~WO#&;91;w`HI?B0cQq8dG!dW+f0EN^2lp4sb14YS7c>+468obDEgsu>7Pmg6Y9 zRGtM6PKK^mN9f|{jVy8MTZat`)#%9nkI?<12CMFcg4*`;?1HevxSYF-jGPG;9sMJ! zU19_e%8O`Ay%ja~NU#A0qnV+}9_)QSlC|Hg1b);2!3Q;mUKBlpo$4K^cVr57KmP&K zOzY8TEkWy;MKs6n03?1m3{y?JxDbmHZguc{%6+qrcf2v126-N&7yT2M{Z3;F`YXlC zEaUNo;NA0c-hzD_b!fL%gj1}=R-Ha zj9r6S$=$Uqa_U*kY>0yGx}8w*I;k#qcQj5LdY|iSJ&vdDG(&v(A~tDU3e0bd!j?#Zjr3KNDg%Ol+m&RT^RB(nmNNu2r+b`^Y;1p zT;q9d-qbhZ^VcUp$h%6up-PRhgY|H}&Jj{vE6C|OD8y1M+VkUQcx4}Nf5$8ajk8A96T9(0nEV|PaGWVY7! z!t-1K+pl9q$6m~3Mu}O>ik5QLP4j8~`b`v5Ur!%`tudndGk0lW9Qwq5!p6&vkRPxT z|Aefff39}?6sr`fHB6+7&%fZx8J*noSG{7b+>Pvq&{5WT8YEgUqm&+;{myM)kjP|* z?V`q=SybhC2CUNuQqg)VYOoai1aa2D-Lj*=1JP_t!ceBzoFwFpd*H)k|O?`HWk23ARoHl!I^V{dx%vO<{v{Zf7q#Q)!TK?s-81i0FBkGZ~ zq1#_IDEin_Fih)5 zw2VLabGHioipV6HO|i@-VkE!qN|ZWt87`zrvzbe?xpPv1tcdwBsX-lN z&^w5>`A&tQ*{`{yf;v(_Gsex)(r$U7p}2h-TfH-RYqc@t-9rp^z!Cl+p%_%rdpCT^X{UN&U*Y%+c= zUtjS0Al*oB37?C3vmAMez13J| zeIGAXOH$sbXUIuzr%#&)iB!3}TvE??x-id7+;LZjmUF}Ci^VyZ?3u|Teh$SWNi*nD zU<%VfcSds;Fu8=uba03si@0+<4}oyERj}ZBTN*?Z9opbsMM#}pcBFk7YY2=D}3$={mF-$ zFTp!u-kKZs3RbH;#LX+UK=*JL`kw5}glR#76PQ^KyF|b%)fjhh-0`HH_z{jEatlg%F8i&|W@6*Y+b(#V@uQdoR)Q+bb z*I|^LYK=F_m1xDTA=r_gM+J5XTzh^o)HH6O*0FZnE0fEVZ21TpORVrkpcjo(UPRiD zGco;iHZ2Re$$38#ZZ%z1W9qp-K+8HChwUCp7k3omJ+~l!b3+_@JEziw>3)KDF+r54 zEh;-@Izy7;!| z!q*Ul+g0QECxeJjAvt!m@U4Sd=%FI`&`KF>lp3)=!81|ZNTma-@x_n0yf=Rmz|b3XOqTUrUflm zD8+UKSZR*p@>`5(SZ571pSXf?W!s?2NqA1%6%MwZGuff7Kl!zr@1Q|xE{&^o_W$8`|U-gmwY)0?wDezWzK0R4xh;iS`A#M96c0DYJc@2M! z7a&yp+2}TGm3fCguXcN=Iou%(9_8dI84Qx zj!W#t8`@@U%;r{7~$t7$`hB_)n9TeYOy#}IEPE$k86@1o`fD*-GDy%wWZ2 z1n2eO;+#nZf$exO&YQ*_Or?EA%UP|~d3VZ3XHeWFCKl9)i^6wM<*~Q; zSk8ofAI_M`3@Q3pARf{!D@3#u$~5eSEI=t z4lwm&H))2KCTuicgZ}MfXk4mV{rsxwFujEluJNUjS7q7TopOQ@QjYztJBWL1@4|{3 zs*H2H>s_Q4@LAW!K#sE^Yn>p0MKwjpjP^7AelsCrl4j;YALVgwE9(#KhAqR) zs46vv9n5UQ5VuO!nz<4T?9(7x;0-UI$fv%F7VfxOA=S+a0d1*3YCZD+X0@pD509-O zPG=FjJkX!=4!`4ccUJQj&ha8`OL}*w152>?hG}n`plnzitrarmjI{}H?Aa;i z_+FZ=a38^3Lo)dl#>VuqYC2oATez_`;~@I{lcvt&s&xKB5PY8z%AUQ>q;%OO{PYpG zDL3OiPLh!)tH&x-;3>tvy{d!9TNG&W5+8Arp#mwZe!_V({mJ+Hb^bxNDqUW%7Oz=1 zaEe-O;Gd?-4!b#Xe=R3*Qy1}^$Y%Mx}C`2#6xQx&%RT@i=c|H4CaTo42NVVm<& zEW29{rAaM#Meu#PbQJP2GJm+V?GEr^*JR4Ss7+l1mauQ`lVQlN1)Rd)Xx88*LAy?e zp{%^{?6bX|Y6~LaYeO|g{Mkgk+LBNzR|X39|AA_U4!`$O4!EqpMNNDZ~QaVS}jKzH#nGmw~D*?Vm|Fk%HjOfBI!@QlaP0ehl_4n^g$y9+r6*zH+6@@ z*78Sq@PI3=+~q^AQq#!GJ{run90FSliTc=1hVcWJ^0iWTnFpN3VizYiDJh1GE+1w+ ze}0K$CoX{zReBVnWrnWd77*m=fNj49)W@8-&YRr6%8wd$3{1YB72NXwi4F~vBXfhR zU~FZA+ai>qT{lH!urm+?$2PF%6LP^XE{Fykts<%I>tL<5CwYurgn35}L3dROT5o8^ zHQmc#lu)+088k(-;z}@V*y+SJ^rxWq%Q3<{Vu3S~C~_t0}1d!WCIcl{a4 zw_CJu?|U}U^xHS^{Hu+ua(WRBjPnCSH3g;|UJrry>LKloJjtl%^P-=|Z1fEu)>9%$ z-Y+azQr&;#yTp?9$+Xa_gl+6%ryZVe`h#@`?$MD~XCZP_2S0XQGXCeEKt6Y4xr1lz zVQ#o2UzC>&)0?l8yKF1ByX^tb3K~eJGZIZ+`{InKPh!sR2823KB26yxF$z{XYpco z?`b|id|4RYO5IBnV`Jg@dAlc|5YCjUWx zDg@VNa6fVox+XQ@MZ13x=GcsZv%2upm1(r~nFQM{e;wXfCcvYM(%gW8L1dDs&W%`X zjENP~xZIX`EFejVycO1$CKE%OP*QaEr>^ z5ekRzr^oA}`G}|h@;Ge+(ji*ht&Rz>uYVfX7?q8iP0sVOW_w`lap87xS(<2|o&t?> z{Ed%}9K(|?rgUsi0{=#J5^0r{z@_RMC^wx?H)egopE;GZLe#}J_2t6=e`RJK-wfM@ z2&TS*vCQntXwjU4v0-~)qH7nez2?Zeo)zE-i?Phf4tbMz44h=6*^q~3Y^6#7$t~9B z^GxTXu0aD{u|5EEM^2*9uFs55A5PaiS~(Zxndsn`AvRSTE6hccs3m9^oqJ}8V>h3H zs19j*=&%x|{#i*5Ju);i;56_0u!lQ)+J~kDtz%oYJ8+4av`EUs0rG7uQ7ml^_wKY~ zhA`L6evw9RHEPg6Z3u5RKnJHkx&f2o-oa&cb&7I!V@daAsp5V=Iyo3pOF<8W9v_K! zmh5KVmp_N-8zb?{*;CwmZyhYLUk}qtv*Fz06mVKziw0G<;cw*#=nZ?13u1r4!CQs! z;khYW-}njV@B0dyE^ijg&Ju2S>|2gw_7~$KxkA`+vw+-H1zY(VveF@}5q2e#LR&e^%Zn_^J3oePL;kAtpE*BYb(|RP} zi{@e)+>~a)A^G7sksqTZjss8*-^%Gq3n>*ag9nm!9ehTL_{34FO*v((8HWqwncW_jr zJK5-}i|QUqA*dR_Xm2}`y4+U#{*ffEaxB6%3p=1_N-$pX8VEt7$5P>kO@ga+G>v^y z#AHSZ4AfX}Si8oB(@Ned*2`66W0jwQ%Ii_2=@W?-6BC(UUl9sq4;9HAqHAA>OZ>mH zGXbZnYyUqoOJ*`e5}K6EoV^xNQB=lClQJ|QAr%@-nbKs=SSTcws0?S{w?t7=sUA-n z^rS?ZG^uF#?f&ok|6T9<{-6JK{jT3S*SgL**E#z>_gQ7Y=ip z1I{Rz6pTHK0TEGf+})LyeNP1&?PT^?r5>D~>_e8^y2kJRdISthb@1n>pX`PUUig>n zHe4;6Nd(5&p`rX;qF{WGT4O$0Gj9AL`t*BuGO?XEO1>>m zAgh{>6LAwowC+hG{0D0IXUien5!}Zb4}1rWXkWVDO_AK4R0W$(Zb!4D$KiQU4x_(5 zorc{O1Z`_2yqq?TW6?FE-j&HTI?J3!rO5DY@Cb_J48W&9-V?V5DeOBQL3hl%f-)(c zz}_okf4q}mrN;Dg{ijUSIM1CggkF&R{6To}Q~~{V`lEI|=X+hW42q7NVG9FBL2jfH zJmz}w)3yL>d}TJgDcw!}_;na!49rpbdIh!#b+VUjZQ&TlIMC44q^_LT{MyD&=E<8t z8rGeOIVM)@#7OH~!<-h-D}o#0$H=Wq~4njgX9vo_fOrVpaJLh0Fq zy?piaIrXd%x87vj7vhuRi1%v(Fx~VV>vYkQuOg$(=^>tzVPieELrIUG2w9Ko8J6uA z6QYtmt&IOnZz>t~C%h}(MU>NzQ-`YeWc(k~=xbXEoa&-Xu4k2l{8cTCJid%pdaWTj zLw?NRI~OqL)I@Z4b%D%7hl!nkA^)+6KV$!LB36cvgU~rM@Rl&ABYrJ_mj$&*?)X<| zw4jB^r8S{cMiFIG+DR;U(qE^i@{&h=@y4@W7@ky1aybvUN89yaXl6DMo0CDW)xHL| zh;UFiI?6;v9fG1_EvQ=aom-2JjPXVZD$u6_lUyfIRkx!se;hX^EBp{#H?fe{WyiSb zD*(B=m|p1CCjrkU8VGmdjs z6Ur`EX(VZTX43G*3D~t(f$HmLvnO@aQ0|~PZn&UC?4_>5oKkzZyyP%BT4f5AwdqV( z+G7ykQVOf?YQpk!m9(vpB}#{Gf_hs${<0SaOS3}qxjB!<%&`aap2rX~Uli;j&k|hf ziD?xjz~}tQDaXR-ix7l~?H}1EN-eyMvMcaZCWG=}Kbbff3$vfgQ}?*%^z?!XXozSf zN9^Lr>aqQt4)hIYfT&M*@bB`Pm&Xw&=b6;0L4(>olA!gA^SJ)a1+XlPATIuwYZqt7 zlC(}G`f-g12;IoU==Xs{Vj9MOH+ZVcqNjmP*?J|wtN z8HMk=G+g%9L^eg41}}ZfPWN!YdzNR})RsP6H|r{$J>wQ+CfZ<0!X`-JwAZqnSNMnS zN1(LKm%JFy`2cy%gUl}lwB&OL8+TBG$_T$^Od}(y`kZo-#uq{HXM?M zd8RbEvVuKjt<6{*;QYSc3E-RHGMxH{I}!RNfEpt!dHUxyas0gzwlLKhW_VVTOg4=3 z$9n;1^K7Wq`fRczGm0ccpQbm%?Qz9~@nCT2B-v_tnPhk9gN<1)Z?A(E)wn@XAh{Mk z=uE?sZ70Zf#d2!-po9cfMBzYjF&UhG9bVtfCeu8OvAcCS4NB-|_^G*gptqf5>r6-a zAp>MCDO0zv4QzD1HM|+LgsW5YN&G$@_WzWowyt-HlZpyW<#?QXbxOcYv4a?w8)2h{ z82xejI>#$hppQrPVO!i*I7)ccJt#ika8%wOI^5_YSb8Ui-j8w?=&coeXeiDhA zWVVTCPaMqJV2*b&sx=+LfbA(5duu=1SE!?8U_Y5%sHE+ZwKPO86&!-9b~D(Vb4^6E*nH*2{Ba9Qpj^GbLc){X)87A2%)?u#!&LJBJpy zXLI9L)2W!72szDp5Q^nkf%eh)4fxy1nbr+$2?J30Pc$7`0TX~4evS%ouv`n{Obysv2q*z)?7!93MHWN*y%JN_6ZT5 zeSu_;oFJRqd@-zK0}W7UgJahw)59FwGsS8g4!nxs@8mo$c5hOq(L{qL>LnA$6b;y| z+C${}#?#yS!Vnd{0w={P)6mv#DC!v?T5m3cVW&N=&Qc{U?t6*d*ViP3F(#2F!mwGb zi+5Rf46&Nu%r2L!L;=NO7&Y(!kuwz3E3&XFDhRdsi$QvNJ6~>UGnqVig6Z&R!k&|% zbZD5tkH^&;)$`+sR_`|Wd{YM7>{n91f;n`RZ9e$T@L|S1O@U0=wNPF}(d79r7?M&W zk~d^9`Aa;h+tF$H>68=3`G$kWGZ9R$lx_@6Ifa%|fAX7sUV%i|7p8AnF?Dj}la@f+9E81(?GZWx66D>~H0%ZsME$3T?Z4l3I}2?;+Lb!zpfiI5pJyZ9P5 z{V7a`lU7iRfxEzk~rAq#c_oc@c!O7v{-r#eof-Gd*}pfE^_lF zg^Ov3$bRBs(+M7#S&&$h&9pYX!xa&Kv0|x*$-(h4mS=kEp=e|ZF-6YHA^bb6UT6*8 z-`11fD0ht4alp-;Srr*LV(! z+0#*__%>!tS;{C@IFbPG8g@?22zli$%J|C1@xwgg=nTDwFyx)g=v#4p+?@eL+0PdD zNQ=@l4;3(cw441AR7pNMRuCnf3sC(ui9KB-2M0WtkuzG+m|I>9^%uusfk`oWvds|8 zJ-c!C5`yo`EIEDpAq*6piQRrAkAT-3M^8P9FBDtt1>F3P9o*HmyM)sN0I2*wsiYi5_V zHI3iLeWtKEttq#mCpz7+eSSGpQY=dERnrNs611=0r;(#xe6}DO ze(rn&iuYQeH+>GaZg-}w0g33BxfV`D3gg^gBE-OM1VYbE!=zXn-15PU8qzr2c*GXo zmRoVWwQ45u$3?Q@UKQ{^o(KJ(99#9y8aTIj157X!LxcK*tony!aOSk$o;Ss*%0h9X zS=mlPtz2+>dJ&--8usXd0eu^x{8rM_EOY^{m`%~D^Gxxxw(G^d3sbEq^ApLr}nZ8xo z3j=zl^w+z2R7mO_G+FD?;I$ODs-GeQPsZ>BS05pD$@hrvwFy-4-9+5+trR_drPy~{ z#?Xig6I9$;je52P__=W*xxo3fYM5A1h1v*aWc>`vCd&~k%>`73M)Eb~JaOxgC!Mt@ zi5$_iB=2n_i020p5M|ZT_~JzxZJtP$yFMnfOVv>Kb_F(a{5he~TB7Z79}ZrUAj)53 zLC;$T$%}Qg{jDxlS*OTm9&07i>QjgezYc0SyIn_S7B`p<0nITt$FF9H20moe(3ofs|=qEf=?WN&i~s3xdD zmtZxbB?7MT05VLR|=5Z8cKLWsF@fC9)Q(c-zs%ZFtNKkm6lIDK%Ta* z!Q97tKq@1hL_}th{enMu0om*5*h*J8HX(=@rgDsfAz#i$`>SGxr6O4kva{sektvk2c#JwLOY@qvtjv6a zNKaQh3Wy4z#qQ4_X=+7X)}LktWO<;hYfjSrTZmix6XM*~%&}I|iMnqewVgJPwLkQL z*;PA0I_uTAvA)eXah4O^@+OQX@u$*;)5zYB$z#u`x51-bYIM=LTb#aoKdw+phQngg z*fu^JuS|W%ILu7r`auy^uaxFd*5nrKR9iz8dW*1jeH_5ZI(+>^oY>Fj{-cE((7U?o zFsDYUF{49<-Va(+YY_X0Jl!%0?ex!ppvpTE^7Sqp8r)9C%8EeUQW<=ip+l?J*)cU| zL&@O=cQoUe`O;fC@7s}4R>FT3UR+vCHfb9*j_vZpJjOfpCHV|B#K zyb98vuOR6ru^8g91;=?Wz#yk4dJAV0k7Ku3m%O(in$d}uTB?AiaPvtUtZC(}Utlx4 z1u{PuLGW5VDzrxnCOf@gK1GU<%0fvR7^+V6UW(zNVj~(>sn}Q^x`0;nDdBOaI&g_E zg!V?j}MjTu}7U68y5t2e(XIOZs1~#-9xz;ZA4? zv+dp^=;_d*<8wmL*>V>-wpFsR;(;(uN$`WL*&SryM~8 z^^jx=LblODXBLJC4O@Y8Av$KpQm3*N2AAzdME-ftR}Zs_9kYSxmT zm3N5$UI+3f|1IbFx|awoKTS#%U9kGG1}!QM*$Pzcw4g^iO4=ZI%h?kCf4l zJ04w@Sfc#p-MH-PTueH540<`8s_3y+xGBDoNV>Jb{eT3GAc}a4P$tEK+8LIxyPoKhWuFeytcC=#)U_0R+$SI}Od74qUXZt2GRTS*S#amE zD?Pr?i^$E7B~m9G=zjH6utvO{%w2H?lI4XlSKo}D=Qts%o29Txa|zk1P)*uA2O#F# zM0z^yJ4k*xfTMmt$#;7hR5BhLmpfT=D_5aoX%wFIo2hp1R^I? zHE5Q0g6Ch?2>;t^v|1L+3>7I;jVT{MXYY4-Vs{aie_Me%IU20rW@AuKoeTb0KutzJ z63ZYd;u~Q_D}{CO*BwtP5&x1m*1(OnL2}Y>gQ`lXmG+aAdD_8O`*5PzB@Ru4t*}it3H=+a=yCBp5?pSGpFJ`lNSEV- z7vEvdE^=nPge38Gd>u?PmTEAT90A7{qV&btDcna zyb~S;1HB1&=;8!AQSJ=UiTMa3-70jcQVJ<;tHFq46Zk21x^(NqUob~SlT=4v0(q&s zxI9n?eMB8`U$Q%2TO_~w)p zZo9u2_uk&a`e*OPwyT2h&_xpq)NI*jlKc4A^<*HiR~^k8YhdH{R;E~9l0KO0jz;>^ zQS0OoX_wAm{e0|jgGY(g!gKNb8`_HK)nJCf7q?=s?7;deHJf8L9`&a@X34P+q)mPZsx?KaNvtO0T9b)2Z;JRm zV+LXIvZpYG^QiecC5sIiHAJ0}QFz{8NaDLD5}nX7ct1*rE-Yg}cI#GhB6=#7_8aE7 zOO4Ea*8>gpW~pT3^-B=#AxAsSlWEVqC&cN%0c^@|1q)MElC+j!^{O_0O79&g+rFP3 z_4#vj7o_kR5m@V1(IheGZ62+lmEfOp#OIn%=BFswrwuJinh5l!xf8DLB{}w6h?<4)6b~OCYF#o!5)c!4ubK&16^v@&WKjZvs z>rMT)ICuU&&fo7>XGc-d8UNf|bBw`%2NL<`qyKv!!F|WyCH&|=?z!#6em{DD>>tm6 VzwGQN^1GP@M7Z1U_y6O){{=!6a^?U4 diff --git a/tests/test_nonsequential/exp_set_TA1/nonseq_fc3_weights.pth b/tests/test_nonsequential/exp_set_TA1/nonseq_fc3_weights.pth deleted file mode 100644 index 64ca4119f3b8831591efb8201563032e21aa1371..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 41299 zcmaHSc{Enx_pdQSiA zoYJ6F8fXyJeZRloy6gUNfA^lX&N}Zp@AK?)o_(IRhtFs41x}JuVq&tgV*l4MLQF<1 zI3zgCFVxfD$I^3?UtqxM@Gvu9uW+vk>(>U&aF7&Rx^!ujgpW^1P>`Qbc%W#k{QM2R zejEIJ9Rq#BRx%6^m0jp0=H%<-n-D4|x`-!)%I{kJpBEV%9y-J&R3U78=+MP-qRF1?{KCEd zvzSoD|N38PvFN5)=&5vzBKa}??t2jb1wLQ zDsH5h!vCZ=?=9heA^%a&3>`zItOlvJ;$x7hU`^o2!H(4EIMzJ2}drpQf@>kbgoK*%vbHu zb#yo0UG|u_Hg%y{iycThbT89J3%-9Jf^M2EyZx>Z8?||E-L(uf9iPb^yO7IwxX93n z(NE~w?iS8ZaVIOevm3v>kY>_8TF|)F7G_G1WsCP&()7B1{)ws_`j~Lkr8Zpf?A|w! z=ET^m{R4P%jup$xF=2&+T$#RxE``*urgu56R8zAKWOeV-E6pLaEBO&WGyNKtZCykT zt2J0k&``D^Ie-=%+>dt_+A?w0S-rCT6gHN~K>ZjUq)BrivtSMv5IKULZ8M>bmBG+0 zFTxE6;G6D>4&2*x9$Kr#bv?aoo4v_ zQYx)~r^hxMM!=B%6nGokL@%+7+w=A-xX7&|-=meVp>!+_$yX-JPlXs?_Z04RYSFqL zC1Fyl55a3~a8h)w{!(f|$Bd#NN`*xK3D|8ofYSG<5%kEtQY<)Jk0PYKCM&Bf+GZ5Sq-jGDiuvhvCQ zAZhjp=ALNAn#B%M`Z_b{-zA1_J*pgP-o;zW@}Tjo9vu~;Al!E)n!Pq)T7LyB;^GYq zvwzPu#oIAXG6ai@&FPt0J|`|z!F@fmDDa~(JvLc}r>%-1WEcof9}K`De}dV0m+fro zTSr{ddIajsp9vSe&4s8y8Jco?CYv)dnN016(y*QdZ24ZI(>>up&!TvH_7G3*Jwq>! zEM!&UTKw-=O9-1AN8Qaws6NCSrVL{g_Ct;LzqS-FNQptQK0F!0{IOM z5H7DvCw5Arim4htZ}^o& zFO}OI?z~QI8QW2(L4lq-%;6^`UVx>A;*?urz{nJdOJsB z<20bkXhu)8$1>T6^Uy5Km`)x#!X2Ay4aY6tqTN#suzE6-nKdn-mlAIv`Q|8!6GTyN zusTG&)n=vg;x&=UKCsHE8sZbw=!In^s(i?S2~*_RqS!O&;&BY#Y&rz?S2oaxzJ0iH znG1D~)a8Y3b}&c#4IIiz!DSXNpvm4%wHi1Q^ zkgum&rYGgmM@8)lV?$#3Z&pUlgu1E(QCQ_SLCnD7O7Q%lHN$R<9a$c234b{(jm;< z^${E{n9!?k4fe7-o#u3X0pX7XdX%{kbCxRL;%A!hv(brf(H;Zi2TAblISZg^*KfQt zM1b};kHPZUMYwY8b8e{iAXDyBTp&!)Q z&gyD3{`drq)E&Tn`E`8obq)7$>LY0Um8?1RS*_aV7Xj7>V^Le^n@&~!l+ww2%D zb6503xu+q|ebFH=l}EVn;Z1(HLnguan=o*FFKM;s;fo+&ws)m7f2t!y5N(}@$BV7t zhFJ$0UK4b8_P>mAk)7cM^^#!{<{8Le9)DC*xtl5XFrW2qglq7BJIeJ)OlZZM`t6h zZl1wbTl8SbECbfwH-t&Gt--z>IdBzX$;4zY>X#@}+tYje$Gu*7=jKYf^Ei|#mFL3F zK|=2Ay>8BO?*%MNj}~kQ(1ruv`$6lRH1qm-iq~_9g)9k2)=_p9oYx-#i0wor%b`qQ z^F|=rlI*79C#Vfex1GH+nN6#!#5M;Nn(ktU;})I3*z6>l{_hbKTP|ZNv8&OoN{#8Q zbt85x2)1S4LrufYR8)MCHE1lw%c)Ix=-UUrI#HaolJa5Xg5hu?bUwT0okpI24CzeU zCtPp6kNc97$)*&(g;y)1+19RD+H|lE70kNOFT4cqId(wky@Qx9Cy#l40ODFE zQ@W`efA6-Xjq&p&GIqQJJNnM>i5F%gjXi-I?wGI=IWOEYb30k8Me%R;269&pOR@Q1 zw(vIItI4`$7F(CsCA1j*2>T0E+4_ii^($SFc@}%CqNUD_e^)a~{H%d*5(k z_yb;d>O3qTu@8%~&DjMBHFk9EMf|xi7Sn?l!qEt2PIb$8w)pxdzNqPput?!04EYv< z*AHC;Z}Wp-)P4qr*eKDJq%J5AN(32m!m5}bP&E8Js(Re!6|S#=J?!N%@LP;eeigYPBN$kayk%;@B#^tV7(v;@nO%D^(`Y3So4M=f8&SzG&a zFzy|S2J%Z_=dC*YD?J={Sqx$EZIx(0mfLoD+ z2i8c?2Qh*}8tR-)Ssn$*&877()2Lt1fu0nXQm>&N^}r7pvSKHV?XRTi3sYcQ(*Ue} zE5*;f+#`xNj=|ppij-A-5HE?ygXBbl>RQ~OPkeUm}dgIRs6!M3*=~d)Clrh zBnJH3GC1%yo=wem#>Bn$zUi#i(f4!S5_I zpi~uIF7((Un!oZDOdYXB(yv>4LngH>qlaUL^AO42CD z$#i$K2AwGC<6TxXVeDcJ!QV6LG-{72O;n4bh_0JhG24$9Ub+WOY8Oc*ZzKhuJI0QB z{)UrV{VCJig0?)+re`+>v;9>^@Xez(d~I(5vz8C18%J~bAa#x!R6n zQ^UYarGSRNDgsHznS9o#1K4h1ObvN6(IR9Ny*On^dmod-f3E)^SoM%&!OWWU~yjbCA9yKxPmJMA!eERo?pb%&s-`DU2x7Y}b+<-9{@Kz|m31 zz+dh+V8=9Er2G?h2g<yZGk4Q1YFx8&-uOW2VvEP)(|~TL*eJ`&dT#RsL@L4D#J! zO&9D8anR^#%tdw(DW$r>U3UqR*HUKwN6u1OfeoczN~5KQ&8Vq<9j5o^Q=I(>D!4e8 z63$Fx8H;_nJ!Zpcf9EV(-#H#_w6!R^V-ZA1rBi9|YEp^(hEIn{(l&h~T5<0QDQcFY zui09j6~I|C|5mvJYK<7ShqpdhAT>WLYa4n&?3jYn52wCp8}4gs_>@Mw77#@*a_s;7Ow< zHe9bEZDAcGCIoSZ4PDug;UjQJmM>Y*HsUuTey&rYM@kojR#bs;Y6{$y&ui%T_jBx) z;5fQmsG;Jgt^5%hL-v7Uv?MHt`L!ovV}c)Q&d_HqFG#4_lg7PWR?bg2s6wIsi%>Z~ z7KdAiQ*z-@L9?tYa^H}9-L{-f3U(lr11 zUCuSFoOOEoFzY9;F)$*XFAR3zZD-b#$wO<@@^B~9k>_lZ;!=^@X+*E97qHUBHW)MO z4Ysw0(fUp6KtgLU=6}%JhtdwsCt-skyBsA? z!@O)svaAIc{@h8)(Ti#LzjQY2Oa?`6cV_0s(Kyano?e^xf%;@C2)|*<7QY{kCo%)5 ze04d_93Bb2mp*|{_iJn@I)Eda7#$wH3(q9%hG7j6g4lVbWE1cT-~LLb8RvnPorvN1 zjBpn9ZVr7--AGgJZN-d+2uywU9-nsy(*~0$`lp^j89#2|y1YU`pZ{mBmE7q^fEb^B zV+tK<>%^WSS&EN)0%mI7u=m7T&iZ#8jb5S5j)fSo6LycFx-XO7jZ&l!SG8!-wxysg zCCU5OOxt*i$dor=?YqLPgYmEQi!FJy=oL6)RHD-3R^PoBZ+P;pc}a<_+)V}v@ps%5L7@;N2R&EL9UhsKfZ;ibE*Veq zt{@w!c)IyRnwJ0lgT;m?VQSa~*y!;ag&*c{XB}7I)~Mt7$FUlRn0yyFj;P@CCXA!3 zXDitDUPsznSPz5j`(W_VY*4;43g=i`GpWNb@y&WQHvdo`9`-v%E4yv*SMwlT+UX3x zv`tvlo+PNT)1}BeJ7HOj1~>kW8;w}L2XfQ<1X~iLAhK&J?R)&i_HkbajBnY-d~CF7 zuzw}_9C;6>tj`7uE5j8cy-)zhYoSv@0d8@gcDjhd8mKV3eJyO-CfoVxFcd zTe~wDDnrvaxspG;y;K+WF&(mPG=e?yQ6y!27JI%m^LuM1P`J`kEZn}E@>eK;$*5uo zV-Eb%TPrZSIF7n9Gx9}0?v@{iU`#HLm^Bqr{ zcw>l?DQLW+`#l)b32~Cz43-s59-e%z$s&qtUy553Rfr zLm?WfB-S*#=5I(j>8}pLdm5P}H)SDf_YkK;Z+DQpbry!?JjRZD=@={h9()Jb z82Lk=mfX3D2fMPEr-nV6>>LMc9lr8I`u~Bmr8pc^+e)n}0WeQzF1+!3f~va?;e+-+ z*t_@?W_er^?kw)d38MpGmz+0ouiRnD!fTRFx;iM!wpG6MI*X%)evgOn&5eZ9Rpn%@brbX~EG=GhvQ&23G#A z0|#LPtW~N9Wsi&W)Ju|14joPD{)L>FmLt>sD*`1v7)xPx?8Sf-^R0XiHiu(u^9Dbq z;`SYo>wOkS-)|6X-TMQRE<4h^2pM)<*^n|< z`D6)}`ZL%uc-4z^w-EG!Qq6 z?YWyyce>6YEy^Shaa{^3?Z)YE+XOeYcj1hkjr_N^SblA%CfiXM4-w0@(bIWn;dSN{ zu3}*>&56)w@B7VhkGCs7UoRIWp0r}^sW7&pXd8`5Fh zbh~&q%~MLGsfrrhn6SZ=zSfB@%P$A-Kb>^p)I;>EI{~`8Y@vR-1N)tO1(bA8vNwq( zYvFMkPR-0pNX`;7`SoVdDLOh$yeM#Ny~27QQMkqY5YPQ&8WUC;*#{IfZe+2M)G zuq^H}Tv+KzW7MC(ew(A{l-B}_PtL@+_1gI0juP8aHH*I4$uoD`0i2dJjeTBd!KSM8 zW0nYet*?8>1%F(K>UR5R-arCg{%4gW%J@!C8dw3V)Jp6ZDpYg9hVXoOX z*sWuSX6|QTk$O5iWu;72%h$6-yMjsi)l9m->?q8s$$?6*G$_BZ5M3LFlc04Gjav2` z8*1#?Y~6F%9CwOT+EVe5^+R}iP6F*uT_*b@D`D5KAr$3`v}KJlMLo@+%iAV@uo{T~!4((^r^F=G>N(vb~izS~LTwLVq<8Agk8 zpOKU(uQ;=~0IT=j;wo2ew->r#z%)5btC6E6x$&IywQ4?V(Q#Oz_z_%=RN{T7 zOse&{go{^NGJ|s_3AZ4>l;>b?dJ~yIgZyW?us466qTQSGSo6&i1Gg?6&Ix+e>lVQQ0UC>TjoGmY)`z@HmvyYz~7v7mY|< zFp~T?6hhL+og@hMg`$Q>*c2niZF^fKtol@i?{vX{S;<1;Rsvg6tc>I zmfw+`8Xd#)uYM_z*S3D}wY!O@%pW4|lV%fZ$AHB+8CtS(4-~G&~0WpzL+B9 zvR!kbHM5epxOWXU)tqO(yAtr~_mS}BMgr>p(*z?tM5W3_*rX*vud1xr?7a)X!S)M0 z9$LtG$Q=EtJXN>@m9*#~Q?%ISbGoiFwuf(LtRam&HsoONz7_^BhTyYL=uE@e<>rYqes z+DzB(s8Y>DkzM-iM=He+;Bc%fvwX?OStk8l|f!HGN$-k7^$M=bNm(zKZ zDSCS{P4L?$2;L|L%V%ZM=!jH0!RAv!<9ZBpd;&+$Nl~7uGM$pjB5CjI)rV!G>Fon| zoO@K34)j(*zrkR3HZq~Aq`(+-G>YkHU^KlN_Y#{h7kzurz^=bWbZlHcwT_Ktx_)yh z#iazVq)M?n)yv7_=W85p=|FWoX&9cNO`TesXqHtd#9GU-O{PoP$*mn+(0DPnuXimo zx0fJSy>GDg?>5vqlnJ47u6QJLEWFyX63b4Gpf4iK)@G*-H7!i!w@Rmzf=n$}9eE9u z{RSYq$rdW#6jSy2X6~qi1Nf|t;V$m*phxEAGkbFbjI+IhhW$%x5d5uF#wS z3EXQpn3`Q$Q1|x~IzL;!dgKBp5l*CpeR<=l@{u0Rn|m3Y=Feue4`L`UQTcBR%{FL{vE~4j+TU?&(4hXxx6i@$1 z!)eZYX!n@=_~?@vD^OlVGqxn)kAZt!uuBERA5owY8Me40(}2}B1q+=jQ?RaW8D6aa z$j1yZr?*WX;kR^5p>5lkLWTGS0-+mCC%{t*^Am@)F64 zUw}7?=JduP7W?OqU_GVz>}i%Nt?rr4gl3wY;k9_->U-B|?>;H^GtL|P+Ac=Yx96Y&<{nhp_owhJfV~ z{Nta&+|UKID{C}p9usjl))MSPtr4y3-;Bw>_S;U@F~V(rRw(mT6JxhHQNx;K&{J%~ zIhKE5_ka^t&M`+xKVx=W;uuICIf0`$Ux3I$7kFo>OI3yu?D54<`1xv#@UgHP-wdAx ze#iIX$5tbHX6H}6;!n8u3+7Sp&$Bd6#A8JLQs%~14xt;veq+drM5eY}g|y5%Aopz$ zZj@UOqwcD+4~v!f`pH|!XG8)`NVXz_XJ0r_)u*`{W-wY$9ZP~FFfLk+UZszxiLuG- z>2(>*YDnR_^M|ktpUNm+oUnROA8(p`l7;wWV&H)c+}I1S)GAZ({n=o);c{*DxHXbE zY~LPgI4#0Q%vI>1dza8N8-g>@ zQGP5N5feoPO9!#*Ze?`LIElbOWqVnr2isIuZRik<(Ao765rQkX4& zM!pX!%V)D?#>t?tZ#d?y%V5FkF;oz74=vpGGDkIO);Ir#EkYN#s~q5aM_$3c@Ew9} zFXj+8sS{IT3=RquC z<57%M|IID@Xh_kc`#EC#^n5o(o{?u`e+jvrYFD^F z!CLhDa})QiI*=NM*E7ZV4qUITg9~05lG`^YUTfD?&VJwkG?m>Ieh6^Gg>ptr_GSp^ zLjX>nupN##E#ZG??!~PqhH#5|^SB4sfzMnr1fCq|gdGYVIB39^uGp(EGj@#cSzdwj z-JD^GJ$yybLH>_}7!{S<325~}nB8i?x^uNrN93O(O%q7f^#vEKC*tx` zM7W`dn~gvIK%f{A1&tO5VVppjc3PTZZJjmqn)C!+LgiRWuRVPI_MR*5JO}Si#n5BL zM7D;rqTX*~*uVQ(s2v*%?MDy}{~S*XF0X?0pY_;cZp==F48_O$_1Se3br7UWGXA*~ ziI+~Msp400S*Hml`YvP`ugdNz>A>bLi zI+aoW?q{@89l)b~VPNyK8?xm@T#M;n30myh6_KA>ya%N- z1frO#0vqrc=a_qhH~XB11rZ-Ww`4t4${z&tEG=f!If)H=`We>S>+%x(U^qU>nEl$* z0w>JXX~!thcgUYYVQ~pGrNxa69ixaqo z&(k1MK_1H$g6K=)21*sa!>2!un6vVB8h*nDvY#xVe3|Lg`)N8nlvATcGB#K&@eVt3 zUU3~w=IqwU0Tg$;2$lw-I+tA|*~M80A+WoaJ1d;a*f~=cnLG&>1YG1VCV$56`=WYR zpJXoN^tO2oPonyc96w_XVs%Kf8j}|GMaNV60aKmgu{*ppl!)-m}PPsK?UlzO=8p98c^m#AoEr$zm+X+;+ax;|wYXxiLrkIH59LX>4Ov=?MatILE4P?Gqa*U`NAsy>`82q8ZV|jZHIu5xs?f4!f)TQ>Ag!mJwjP;A z{rk1q@FRzCkCinYIy(auBCBBJ*c&h|QiUxYI||S0d87Aw58NHs0(akTqkkU`WByc2 zig{tg<{!JxA8OU0tsBGHq7(A0=l0`MWxi9#&zISna?SA1Yn=ANo(_s=*To3yauX4j9 z0=d`6j#5Rrg!?acMZ)vnvb>BG+K*z8Ns! zY0iatU*%t^@8hOzNo2>&2C)XML)_i8I$%2aFePR%d3hSs%~p9z6UDPTHcqD2kq4=< zZ5|GJJ&vk%PLh<`JQV6D(aRDemXs`U`@|35biaIDUZ01k4xXc*1-MS44kks};DD(PsTG{z8mo`c zv1)gA`^$0uY`i*cUiTdLK9I!j6k}TWR*SaxOoFP`x18R?0`~98SLm;~1KAFv>A`8_ z$IV;?4Qv}OS@;t5QU3a68xbbv6ISQiiQc zSxzcHJ@7$9GQGU~8vW;=hO*hanZcnPj@B!%%MxBVZ{Bq#`Fl4Aw~wN;H7c}x+Yx-W z{UyFv8;|9Uh)UxuFx5hi&FtS(t-5$BoY_@MQ*`bK&PRyT6VGxMwmI~6Ofb9c-Uuxx&0*S(2PpGr3B0&Cm^nW;rtys-0?jWQsAAAy zuzdOo9)7BYCb7=~*@9oZSCbwsjMAYAck)2FPLgha{0JBCSkWVy9KPN?MQBoN!v2MQ zuda;z!5d2qz~5~r;hOew`n!J>95UO@>KyI3`5xgE8a%uz$f6mBoXIEeN^knw6vo|= zx{1bjwDIKO3)t7}PM_T>`J#)}IMnkE7!8-A#)>s;!>hsM^R*KCLaVq_qFTJgTlMMt zsI5#Z40+dIp0xa)D;#Wn1SZbWG~ZB$j1$cTebHuYcg_l|)E`G*_UvZMS}w4g=Dk8E zmkqqKcr>aPeaDL6I^MTJnIsC7*(?3IXfwiGgom8NJu~9@Z6gn(a<3)JpV-K~mPmr{ zYvORs#QUH;c@&%Z%#eNO9)WCoI8ODLOvCbavk&HFEKFIPC6%t?j~@_%^tK=rKU2k} z#a%-6ZELW#dk?@9Yq~X~2HzNE@LqwS93k<)hOx6c~qKjoTS*})KZ5|=)x#w=$E+Nl+e@U@aqe=8t zREW6U<{X!rt;lx&o(FY5RB8D*Kh$eh$61bAY;HAAjhol8!h*51?ou+!x0$lG zBOmas`D=)D&_xv`2hRTe1I*m86gQVQpx(U0+{Is4x&4RI;O30MZ1Jg;q}s6r{2OXv z(8JT9+Zjb2- zJ>@W%&NrgEUHx2FjWOGv^%N@pCBv=0a(J6kMDMc1nCzPE>`(D62cll*oBI{sU|Tw^A2=^ct$Q#$O^*4#4yMQIDp0SlIhK8{gXBkAI5jGk zSIAySl{IHD@Y4whD=y~FJru=AW?TL}npDB6#Qa0)R7*c!K z&s|oULK};9`NT=GOzc}_wXBLP919%Amf0;7+5081r-UOH2_5t(N~EIoy0l`zfc9_x z#&62fq02QJSpO;+xGT|y`$hS@K}HX(5-PHsV?dLS#IPmlU$~+5wtSf3F(}NHqYV)g z@xAgm=r?o3?RiJwU9kWXqyaLAw}6*j3Om)746|1>^Sv;W&2ZfgR^?Tg)hwh&{akeZ zp~)V8Is)v=YHI(Q$NXQ^;xU7H^kt_%JnbK#N!0HZ~M66T5S?!k7YTPXwM4?97}yDa*ldW$;K#WDHwX|{ITTsFKo zfOTythT1vHXxrs2#9gytFXKH0$={1uwbE?5_TG>MINydpLj@pa{1G4O-i8X30IVP1 zE$rf^lB(@V_Sx}P)nX41`g`;!E?c3FC(8@Ld$b$v(OQe84I?P)gg$JPR0W-s3jDkJ z4EOj%2i-fr3}(#P!ngcYq@1}Yq2Ixd;zS&jYon-WRK#ic&s3!L{n1o)fJ47c95a6! z%_{fqB3SGTYWEU=P3aKvnpSMhR&g38aA$2xGjPm-32e3eB(|%246SgxL4Oskn494n zysQ&W87S*Tw=F_75&;M0l099t4f^%3NBghzDPOa_kZjUO=B^`IMA4oRZd^#m9G^0xDj_hE8wZZT=w~o#I8e_+)<4 z^tQmH}#5CZnFSiI3>@7LB;P?1cOqP;mcVhI447zsPj9F~l6@^$!f# zdUFl-V%Tl$m>vRdotx|KKkDk0l z2TRYA*`rmU`EMF3?^A^G{0_Vo{}T_LAa0NQcxnmIrWq&G8Hw^S1wjDk7NbLfcd97j z{9m-wn?dp^Jk+Zlh3YpNv}1x;O~v?aEOBEVcOzg82_t~ciTY$pGNyX{oe!}_!BiwbIF?%BmT3u$x?#$MO@=3~^ ztkirqRDBp#%bC+E?{ucywt^)Eh*}oiB-unkF!Uu5Mu!TIv9YX{4Kln^{Bzc*vU{NncJW#khDX&xDn|0UHP_;K$e|j_&>n1|n zjtN{-->*zcXV8`&a1+kh|YB4K>!8}3V~G}~2s9S7e}B;VyZaCw~;E?RRM zeTI4C=v-HHo~cPW2OhJ<5~bXn<62ByYCH8^%A)APOc;oiB*pp;oFy%agPoq@D)BLF zf=7iwv402+sfgtgTTfB4{T|qw@JMK@r@(ZwHqoJHoAIBFJGXM)2o`8l&B|w#V)9%a z=701I443NU170|jY{p@_^-u*$G==oVG=OD|v!axD2k`VcXPBV1ot=})7fc;AmMsyl zhoIJzBp5FaE00w}_s>bZ;qF9=Ian!>2yutN-e_*4MIhcxQD6$QR?^dv^2s0kG zU};AM)_r?|TILn7+trZegtg*QhhE%%yGY=d>`Y|^n)Iaixvq0=n8rQt=Htyi!>#*+*~`{HSm9!R zsrq;qsO#OpbE`iK|F(+!{+I`>!zP)#xOg93dduL}5@TF?!H}lj8OJZap$U(3BC9hl z7W1Qiya2856Dd!o0ai6RFvS^hG`BSi=Ks_qsdvQ~U8jNZwkCMdAc{QtYw+EpF*G|{ z78f|Z=j(djabZ1{6iB}?ZdE#w-&XCvGsvoSXY|OZwoL7wGdN!dF5|a zR_9M_JhTH;LiWhU)b|p7Uf1wWfu>&V$=Q+w4~IT+k1aKI`b=N z#v~1PQhyulyUFPDo;ZGXDh{aASHe=&3(emVV7950o$wUsnP z5=Du|bI&(PLLo&+$%w2%WJW42m8SMEqLi{C_1tqJ6rx{|Q8wA5kVxov|AD7o&-1$X zd%owq-=C8vdY5F$J{_Dyd-P_*h{zyxc>aR#JG~!2p7GqMTAqFbwX_#e~u*0J!kP79CcaBa3i6bQ>anXu-AKgoVI~U=fGxm_GB+KS0L}J0# zp~5+P7Me?klDAtEY!Ms)v3wl-*JsNVK8Uy@2N$xZD$eY%=_Bs&>^hVk5JP)gok*_W zE98ti2nP=J^0nr>@RZUWR9=zF5*s2|wO&4q+6T13=MB!hoI-6186^5IiXWZ56q^h+ zxxC{)`0;O+V^e}K2i}n_sDWABpdLc2Io7oA#D036HG>^fw#LQmIJ`2N%S-ilk=>1I zl6b$FM!y-%dSDhjtslpdj2kIdavx)7l3}7Z(Gt--)_>v@otACJu@_iyn=PCx%voFPP*gT*+ao|st)`88Vu!>G`y+aBXB81a9&s&*u5? zzV0)d>8lMH1u3|4%qwV9w}ZI%U~P8g~z}qZjs&vV8!lZcc)`7|iB}nxNm}HT;E#ci=&>1ij3B zjK>G(!nG~(+`X7QJY;(Xt5(*c)1wxA;WU5@@`h4a;~>`UQ3ZEjI%D57U@s(Epw>X( zsNDqSe1Ipo_w0fH5-&r7z^OUTbHY{f18KIj1M-q?(4x2>ZECLL*xqCK@=G5kYI-qg z(I-&cNf#{y{>|0-SsO0o_#ghiSX+3LfVpl)G-pY~v%D z#9n=9?8?Khg)g9Y>v*`apqT9x=98xutps~5OPV+1J*s+I^JW6SGi}sEREQiyy^jZA zNQPpxM6h-COp)ZzgV;J}GmA3#hLMMtVg8~Q)oPx7T;8cd{$%?&a1c1m z%w?x}{lYP*<0DIB<}IM)Tc)%r&XB&Wio|3Y7lH?A=$Y(>9dB>pu##+;uyg^Pb}QqW zAFI)YH~##sz?ry4oLuwpsyTRfNKm<(B0V3VNnh5U#LvNBu;KG^I#4~Hw#P(3@HjR0 zXIMJ=DCkjyuh2icdl7ysDn30T;ZxZm6S{^^R zGLgGnScn=Q#L(`tnDs@<K4%0T&CNu)iPDsqwVvNvwI5zT zNPwJh#%@Y(0oZM42S+-G&`R}sC|IBZN*}_>p*R7K*<3;o9ma)qzd{3pKlrTe06t;|v1j2T zmiO*8x(a8K`@k+u~>Nlc+AO%s1q9~@9s1<+nK=%ItxBAH@L9> z((hESzkZZ0S~3xM(_?kta)&Z~LRz ztC#y2cY-&GzJQ-Du7w{iqsZESF|U~MkER9%vZcAt@W;CkfHnqj*H#bolT+B2d^7rY zbSJPY-?+)y$GGmd(UjBhk{){QpktT9`Tl1~?2_VLTt0jv`Nt=tRMU7CIDI=jFOi|E zYI5+pcR#19_W+-kt)r~CJ>=@5L<*bN(}we3(bVrba?P21!%e}t)Y8Gmo%g}6W*DhR z_lu{sE09NHENs$vOZK~UX}Vt*epX9^+-auBTOAd=_EXF-`ZHKojK_!mL+OcdUf({Y zK$_0q__TRpoR+^0g&m*48ZPT%<$qfAQ~Q}PE8of{ot!N2&`6T(_4Q)Ld0@f89R;| zmCH~vYNLH@b_YMldKK8Oy2gLs6N`J4wxHE}XXagf3^&#V<9NBz^u6jB8GrMm=qY-1 zgNp)LS4-A+|0@5%Z7;+;tHf0T`5^_S&lS;vp(XG)F%`SMd+>9jtFh?tYgq1U&HR=8nPSm6zHHeou-~x@P7Df#Kj|{0 z(7y{dE&nL=HBxYubS&G}bO!fmD6l+bOL)4>h-dw2?Aqcn;Fu-2-AT$c;kPgP&NQQz zQVX_?y%4=xdlu$-X)x1GCM;p?P~_H~7569YL>wK0>zfQT$*|9T2`sKU4Tl)?@C|W^cz)Rw$nEll9IfYITy$TwtT&q` z2pLwd%4c&`Rnuu=sv~`!V$ZMhvu3#qjj(mV2aFy(ioSH0lSzj;{hDM=W;Nl^ds{@= z&8NuDU@<*EDut!nn}r^08ooLs=;k*<(f_O>yJ9dKdHGFv-nJX~MQiBl{>>t-zx!F^ z)Jo1^O*yxEZxokO8o~T4f55XVNdyB2;fcXmv!M4CHf)!p)Ay<|s8EsJvRO?%ogXkl z;NkSvDMQDy9emcg?XXE%gKVxi;K67;eD6IU^E(@Hfx>p2A7X$@8W9z8!ePtV@0^p1 z0W1GK3L}r~qs;{~XeH#~lTKj{t8WiqT^MaoFQ(}3$$WNp98W4|VQ1w+bo(3&@nuiA zv~SgTJ1iO9_ejGe`)5eK%kb=%wa_v<7~>0du_?2TG-aZ2;)*I*YE?vg#p~els%`YI zX(ao=-QjwJgYZbgbF|gaBiHi-NVYQpmd-y7-M5!v$(;Yt_3&YMyK)mH8tI~jRSgbO z_=>JF$=H2w0Mz{uddc&iaFOQ>XxEDz-eBTm^fd5>k(F_D{fG;CiehNhv`zS;z=)>k zI5V$KF`O&=3l&_yz>0396?YOz;ruN6k$oTEef4JH{h6H4nyGMbmKS~PxKp2Z&Gn%CbdSZQdCOSKeVGa^Tly|xco~X-E z(tcC2Y`cR^J<;$vY%^TE-GaXMTI@sU46)gs5X_mrhlxrQ;JxM&SWw*s@mDj&V`fez z_2jjzey=tf{}Nd2#{!2eeE)AejhU)r1sW_UfM0_Q`6)a0mOSd(m&@HQqiKMQbKUg1So%_u}S#?wL4=vt8uFep=Xq z^2$-PVwo}=oU6l6Kda3)uGD6%rVg)cmZ7;5GVt|s6S%V~nf}c*VneUF!G8Y}XkfR7 zo)1?Ll4f3^x_u$0s0psj!7$eQx(qv`=2Ps+r||D-GhUdQ2?0s+Aj=g{$M*FUGc$rD zFKnXd?-zKfc2hLs%&1|HE!s^{rnz35=&j`>xEmdWUso5i-w_jWdte1$sv(JI{NiY* z=@ss4$6@@o`Y|4{41j5}moZs;G@Fq&k{doP8nRpvc3KR=+cx82gvWE7^I|Qgzt4cv z$<@53SsHkRbO|n*B^0{mQ^cLepqH})E`~`kIoW1z*F+uGI@Fijy!98XtvCYG!gC|v z$Jl`2AGl%ecsRFiKZIW%OcO zBuMx=ldgC!PK-~4Nws<4KVmH#E|0Lgv=TjIzH+zjE72rWyKuap1+)kCYeU@Uc7De!X*XQ$t`p?73Pz&7n`T~mEW%%V?iFEfUaD|_{L1yPH z<~&uAru|Gom8-9DOiT*r+-D8{p1+3e&i-`LK$8y6)xeCEn(Wy*#89Mk3iZw!!j`V-!@qye(-z6kI7^hxi8c$3!V#nB z)AeB5{PZSw*x|K^+@#6(s2z3eFTqTW5!j=BrMl3~QJ4qS;hc%WS$x@peOPS6&Tvn7 zl?^elvelSU9pB*M8Jaleurov`OeB*oNv2ik&Ocgb&kn1_qwPW?*k&~ZOpG_--TJ36 zGjK3-T^~vx+g`!#mjCR(T3zANw(sMApLM5WN3P>%f%A4&%La=wMbL~=VDSoDP=3$= zYCjdhg4SGsZQ2IxSY3mVQ!oGy_|Je(ru*@dx)KFgpQ4M8qai3i2c3havE$1w!UOlc ze7e*e+IeFpN;dD{)e<+cnePVB#QH#bEA)ImeQU2a`D6=b2bD?B?jOJLIxo^)`VorX zdcsrlFj}%Ao-W5nfa9q$uB-VYXSQuRGfpVy)5~u`VB&sw`Kca;T$oG~zDEk#Rqjly z$dQ>FI$@T1E;gMBWUC%(ki~StX_M5ams#Cdt0~XST1z49&jr4|fJ6p24cD&6BOc-Ke#i=%H#L{HcrYE?=TF8{6P|#);y@S>Zp4Bo zCgQvzb#h%!@WMlZw%OhTX$^CxuOLeU?nyi7zxu}4%&?+Y>DHF{6L*y}_Q6A^5Jp^_0;`j}<>L~9KNU}E@_`DB&5b#29 zsH>)se8K=yZkkLgC)`;_%PM;EuLPP;l%bo6TF`Z5&$0u6sWL znXw7X)nq(=sk#m6<6JR3Y6eyb&x|NvpKkyD44s>2lfA1xsRlcNP0$dmoiK=f&M{_( zyG=N)Qz!AO!)EGf66Rk6%*a;zKhB^s3);^`p?l_aT%SLNyOS9yo*ml@4^|Ciigx}i zYE>PK9WPG@?!?y2Ua!qO9jn<34Lh2mI*76Cbau`vfU@0XX{z7^`s5YjOvf;xUJa*S z&H^hQSL3eAN~WeWn&hSn`HV)t#G@y))@-dBCQelC;L2yqFk3mmlmn6QQ)elc6p~D? zA3lN3x|#T=K8-&z%^TJavSv@NoZu32t1!ml0)3jdmL3?6!ZSkwf}-0+jrQ-j$q$;~ zxpOyH)|-YmOiHNz{7Fu!_%)n;QbExd$FrLDD9XlJK5`x_lArk zi>)y=iSfP|s&E_P&aKB(vs3)zQZJ@dzJf*?8nA*=DOTu{PH&V(FipQhe8{T;Y7g24 z--jxb{t{iHXO~crvLQTUJpJ|>giE$1vBgE}z-!fhyerJq<>X>W@9Z^jzbge5uk`8L zK?apd!|17SXUUs9oof|!;t8(_l=)SGZ}B@#6Z31i*ojx5v3CIbGO!)j-m|6Kce811 z(0cl_KL9jTrsH&NL(0CULJ>{M%%LI*Myaev&BiWq>S|Y-d;1eN!@3077H2p;R+_$s z2tCM!1!7~n8LY}DpY`?BL4ca@Id9_e_2?I%uF}DmMa&_;Gbwn(aR`2V_}TvUp*@_{ zrqwX~vQmf8zm zRV>>*B1<&$I}b|LvGk_32%l>?h@I4*;@#&rq0J>8N6qqQ|C%Z}6X#rbvoM_tk+K!} zpKr%3fjzF9_z0%!FJfv1ne6E&H~yOhVDB|QI91%rKd6!+y`0Zj+cA=DdJ%?ORMU9b z$VA$}r$P6?B<}5-Eu>w4jz-ri(lQ-43<~JNqy59!LzT^ZL4N>SC=(%GnU=uLZdn39 zeLSFMOed%6c@r0I(PMKOfgYGAu|E@zvqF=2^gZ^IQ^*kV2u9|kc8vv_8$K49r7^7R zyMZ=S!su)BAetaOopJ?V!AJ8ccW1~^+-^Ff=G=Hm@(8I1FFi}r7+r~v-{_N*b}8;V zQ_cz&2_F3L)f6~A-o7+@6aibrmsK|7LdC^2bCed@|1rkweUqpGH`53!8`No3#D%Zc z*w25<=*2pH*lf9vL@l;xeqI%80^=y{~`?&v4J3r+7;v2mX{h zGyB0oNctD?-KKHSvO$7gEV6-#Mmn_3qX+mATlr%-OHi#m5L%K=ImPf)bei*?mpNDf zaf|med-I2!(z*~XJIWos=RaZ_RtK;vK7kOn?J1f(i=>)+cI0%o53TDC(e{>VDx4?F z`ebdG()|w@Xypg;%64?8>JT|fzZd2Js$?tV0Bo8f&+ST0M`y_wqQr((ERQ#)f>vL= z8wa#=Q#yS05`6KaXW-q=(RlIVT(Io55$zr(bi6CZz|7_DY-n>7ceV5djvd&{&sR`_ z52dDT`LQOrE^rqX3VK{qlLTlN>C%l9BP!Un3c7WrQ6hE${`&;r@8A!YB_4rp?nQ74 zHNXwiZ=MB{jXZr@7$HyDVg)?(ywVqPKz$`yGKc#bWTDEzCFFwjrw5E7Kj* z;VdmRhi^&`VaHbmqfG9af3i_QWdOkE0&FJI&GVgAj4QJu+Gi_Fuc7n#e z`wY&Dy;(-aQ@C_Qkrp11sXcqS6K`MB!MqvmsFo9inTfKjTT4jIDn3aioxAygG9RGi z!)kC^W+r^k5~!Uf$-1HxnR|&Q|8K@l!E;b#+7E45;oE9ZRrUr`pJD9$gS*_Fo)qr& z*dnqVcAwuTaSRIvnzGoe5B!IyI6C{)fSt8C&JDdM%^I(TLb{AP>Ydggz2ow@a(gbZaxOl;IDuA}DzL;o%gJuvBXIC&#Gk>Z@$S^)7`3$kjO>@feW#_YYTph1)wWoa ze=>w0vn!W((!%_6P&>DX%gT0V@`IY7Xsau}_Rhy^u{k&? zem#JspPoI&?V8f`jyZ=0)!Dgi@ICUkZQUO^%ID*F@P@dHlTZ<@EPVA&hh> zBM|QJeG4Xw{OTk?|D`f5IHpTj>P`Ey*cHV-w#6X!KA( z2^4v30^OgiCS;#Rk?9a$^3m`o^C#bM*_8vF*RXy}u880{;dgXsY=HuS*;TzMjipXL zFzZtg{XDW?V8=X}vO^|Dt?TE`Lvd{YGo{XyAM}azX|+JD+BeRx2suwu%NkaMhGn*EHeiOeb3R zbrdYGUBhZRcR=cFM!P-6(ht9(;@KT4qVwjy74#-(4H>P901cgBenF5i z({S}8nWvF#N%M5JN@Xnbwut1Nl zZ84n-sw0Xx_tA4{#DL`j({mc92d>~2A8IF3pN9vkGVx^`p{&#$Xbq8NB_Xo#Zm0%o{cEZ$*tk`}g2$Ej8mS&%cMH))dSRXYW)(L$CAQ_$eC zo4_|8ye=3Xkd(|RJruK@N*cyGIJS%mvqZ$W`Y zBMz#YhtZdo<9;z?PL2JrV|yH(-w;Wy+8UgddoBL5)C0rxsX}&lDTy2n*^UPiY?{_e z=J{4En#IklHVSFv-L|acX8T6r7bAJ{-4jhq_aDHazusctks18(=TfX`=MpZ87J@@< zC+}Js0>%bYnZXR<|MNYDUlvPZYD@)J`uQ_1+-SnKN87UN&U?inBUG8`T5a4vs*VdW zvEzH6uBM=@(eUX?6IJ&MjOUIQ*wJJLlE3-|mevtYkMlwAXrTsPsmk0X^`Nq9B?WOw zTwqcsD(zm+$G*)YyT@11+P4&5x5z<$36JKI6X}D>5GFr&FXUdHOpXe!l#;v{COuTZ zVzXP&Xfq!i)~mBI)_J(*vKj>rPiKys6wqEJh}sGySvs1VaRaROCHO=E91SK&qP+k%5#gVLe1s3S+0?zl@ZX<Gz3jwUEmc@Lru9uSzqK&j)_{{#Vdia~bEJFlXas52NR>Fz%N96`UPb z#=bjo>~x15ELs`~(XUdW<&z8hG5j*Fq+FcSqb4vyLZal;bh61iM-Lv~0RHhHN*6h@ z9kpJR*i;0=U)G}c#{}4{D)2_n#?a{h>Tu$ffozlRMf{T0U(>kyqd3H12I~DC2Degd z!71lF#41};O6+NvDu0a4QIue!wv%L^rHPiyZa{I?W&V5iC}`@CU{gc$sBr2#&~A09 z**#*rz@>Pwswp?2Ap9VIWMvH%4>}?`H>L`%x4ncB76J!7>moNv+=%O}Eii7RI{WAM z563@uVFrTApXS|xPhGCUKHC+nqxL5sC4CO7mm1Ou2Z8D6nF`v2)aX;vDH?J{ zg)Bap()e2nz*eKW1FYwHWR{BRRg zHB4qBKKXLHP7Gr^jCSDBK2;$bQi>CQCY?#7o|kX9bb1 ze8==<(;$`8()k*g%R{tpsfujK+r!XCav%!DK)%iTB^eHKlJT z-&&e(l$-LR%SC7>aRCRFo1ua3Q|wK?Kr5FFqXh#u^6Be7iy~L9g3~{=$#TL)e0$mq zTg+c_E86oZXtc0XqN)!C`3FN*EucT&^x)c|`OtJ~mw4TuF*SCF=Zkz6)L@aBEvyk{ zStlRPplP2cVZPlKQh!T_H z@NStcldc#|6Q0k(l$1rReP1vgwsE5k5w@gw-iU4WSV-@el=B-uT2N*AL0HVbL;2fy z-s#ps%F>Af+m*M(R*s4AdG1nvOS2v`8vPO;$mWA)k00zYnt(;qG^oU)5HqAc^KZ`k0Pqr?drT%{rY@^{28~1E!PLxjYE4jHauPN3n;? zHbH_o6$Y2h0R3JjF7XT(@~bwIrE zqQ&oYj6b5zD#j$z&fZwqF-yo_coIW5)aKBu)pvN^e!!2~)wqQVhjS(4sKv98^9=I^ zPGArvcRYup!lV2;r&D~h{ske|UWMH~n!s;=uLV-S{@^MrE7mz+KimCChwW4B0oBfR zY;(>)ni_Tka)-^v-f^QTr)x1TsTe>rhR&fa+U9W2U?2J^{Nm({c0ua45SlwAg-)A% z#(@tdvFAn+njR7Sy(NjbCR&3HY3s)G*Hj_IUZ1V8-;Rxu!}-51iC80K%LvRm=xNWU zEWeSM-8Y!^k2+iPq&1H3y!jTMWn^HAuZT8H+>6g0w$p#^HG*GlNbzCqR3XftJZ#ly z{!t~e3g1gUz5{A>H0F@A%0&LnSaV7+h=S(IBKBxx3?90@l1inOU{GfjZ#nrJJnD8q zXPuoOZzfC1pT0qg>M*7{-irNEP^EL_WtTb7V2FAt0jYsm#+lL4DZKuR9{HLJlWi2{> zb2{r*9*ZM8Z{yf$O>}SKVL0+Ft)_m;82o%%$kSh`PE$n3VPWi8%-_^Vo;>ieuUByj zS!0=6^CV_fA5FUDGPGv66_$G662I4;$(1?u!8KQoE-FOvdMQytE{7ODdCQ85ywzBn zgbQ_Xw)87Up5h9xQqNR1*1o2K#uT4|gjb94hIBF_sXJ8I#(gTU?GB5)nF z0OYxM{P=N|{G@+dxPUx07EG3+j~6ruc|{6+zl(m}JB^>`oW#XRnOHp}1iO{zh|M1t z;>Tub%GP~>DG%dl@66v=^+=m)?WVG*;h%Z`L_vxG9-f1Zq;4jIJD=WgGIaV}( z!yc+jsX#41XX?_*!hmVp@T+r0x3J7KP& zkRzV;57+DLryuj?Q?1~zuPE)nCz)d?cjg$f6+pZiJ2BTNw~Br@hmozyAU?!+FFX_6 z%_pI8_~d3c7P<@GRz|h`JEJlX+q&UH>x+1B_N$tDl6A6S8@>^doL7gY+OKpo!a2ZffY2fM}@t(MSlmn!z7e1!)NP($_u#4va%`}!FYEsjf+tsW;l17#$a|9r zfg?*{u-ZYiK71HAEn3Wv;$^Aj^G(c8E+XfW;k0buVJ=sB32rpefvgaOL3+=)Gheqt z`cgd_w>=%^*nZ)jRE}oVlgGgPM_}(Ub0c3B6oA8YDo zag$jI9-X4Yu7_#CpV_wTkb*X@Njbx|l&pmj|0T1BUE1^`Vl=b3ZNTmgF2=#H@A0v9 zl_X=-Nj{@E{>R0UX9l|a* zAH+HvQ!uPCX7AUQ;HW#s^k$9{XkrR%JuhSo2=&XXANMhUJIH#4brv&K7BK%8u{3MH zn7&-f#SuaV#Rsj4Z2gKv(v(O?ePM5b%>-9^`ptrc3R!u!5l{JBV^>42_BB|@NwLs| zh3xE69W0epXH{cE*hqUfcrsMTI2V|1zgy#Zn_Z*mvyKL3t}vzth5yiKY6EuJ4Psv- zX3?;LCPHq^a;#V*-0L*w!j(S*Nn|d@%)tYBgL+w}{r899N$D`YVmdRwcphF%ZNc2# zLeDkQi1QxdN|X0~0|U`D2Xe7k}%pEevGQ*SF&GIEop_c5giS< zin=ko(Jb{H_vxpeke88)2RiTJ-JZuV==pFms}QnAABT`}^G{Chsu;D5&R})RKpNqE z0xSM)By$NBYLyoly8X6%u+d6*h!*t@l;F;cYkIyHEwtVy9K`SeV`sqf;>1?xDT5DhC}%9x%8`HGqF{d0H)pq zj2VhUmYpH1A?w)0{YHGsi5qxb{w1Fm;lw769E+#sgi?l-6C2`?gWm*4Y_XF(@Ah9M zY(A*0Z&vv-Gsr`^0n#HtmehTi797M{>&rzIe#zoFHxcPjxY#ZGKsR@xUk)d8|mG+0#RK} z3D(4PV(N;c5c#YamW4^ue;by-BW@bI!d*e7VK(&X-WQPiyMbn;?832oro()R1>8eT zYf{=6gUZUE@b9Sv@JsWv-@C$}il-{FZ8aiPY_)~QJL)*CZzHL_(3s;=wLmcK6~X{`+7V&S~j6{A110`q6O^uqcC8w|v1o{b;V(!kq*Y85Ec7 zp>2AStg~}9gn9hqRhCCWa@S){=iyMMX`ozdpC3=pPRxMD+1gCy(^%&2p-dSqJMo`L z$iS<5A?kj6fJOvnQtJsT8fX3=j1(Q>Gdv<;P53-!u}qSgJaeZM-8t0i6$K|g?nVF9 z(==7nm)~?@8lLJYNB!}dEV|KyO$jr@BX>8TlVu6M%vjIvED)T&_GHnQSZgNPwG{u_ zAIH;wKH=FbF{~nX4l53sKvQIM$v46kmt?-?$DfvGi)Cu~ePe1tLDfWT*S8lX{Dqof zya5(PrEmc)8q|5nlFFkbMWa`SL2s@C^ZPu7ZOlxjufm?7%kQRuhGP;YDw?BHvjUm9 z=b%Hk2WVBflSsZ3uhg{S$1Nhx^v7L_zJ8G+!jj;TVmVwo?a8b<&T^%abGV7V^>9A& z6(5@S4J0~-!@SRquqNmiEZ#H+eT4hmBq1{~Qbz-FRYtOsjptCyUj`G6I>l#v$AkQ2 zqA`{V_AeKS_?FHAbh_e0sL)?ZuLoCXut*rQ8D-)c$Ez=u_~rog9rd32^Z zmOeiY6;(w@(ttZUA+ z?hRK@aNCvZdFQm_G;_%aI&O9mEt`X>;ly5ikh_%|`7Ibv%3K!@H3u4UFoedPzKo65 zIW%!jH1!Gd10%gCDx7@<2bZs=_02DFgViC@!bHm1t4Z0`)hI7yW{i}7j&(P4s6=xQ z9UpN5XVi70xymlGc~{TBxN#mYN=s3?+*;}xqfSmSTew`in?lCP629WpcV0SR5;gB# zK;Kv=?$RrR73)kX;;{+bKQg#>n4uI)S))W({tF?|#j%`2pad4&Jcu8!?jSSM#rXVO zH7`HkpkNmeg_A+JtfV}+fX-ZIjwl8OtF`SGqb&0 zC@yU@_WO^duxZEf-=v{rwQD}g6+A$Gd#(6d=qSuAH=*mgr@4ZT40zdh7GCsU76rOy zg5wHbzI%iO+P|8Eez=ErA62Juim6m6^t7+aDA1)-L+Q849E$z(9zQmG0CrVs!F*qp3$!YQ-0iOAp`nCIBEYFf_VM|K3H_WGL$0zjLGpMHHi*2){^TD%aAS}7 z{+Z!4(NEyU2FBy8e}_bG5|VJ6x(+iR9WJhWyB3m%BvpNsdBO#)4JYSo8Q3$@9NTs{ z(mksbF6_f5>`yoZD>5RPd(~^i`Xu&O!H)`mjH6-avtZEq(P%GZT1~Qv=CZv7j!@$; zCeHi7t!cO8Umiug<7r0AN3KNOt@_MjwhIdztVc?=V~80OZOZh3pd;$=&@YL?uLj_C zUrBa)e?3}v3TK$fO1x%v1{CxTq0@_7yk4?8L>7#K*EJ{cy!&93U6P7v)mijthdQ0g z-$p)vdeGTQmM==W#$Pe(!_dt@`iCFzpNA~Pk%B9?s|4}Z_6SnB0rWn8Dt}wMl~{cw zss~@io!5Ti8A)9llX{h3tQv!hw&~OOE;owJsppF(`(d1QJ%(y0@K%q6vw3tm<@y>@ z;XYr=;3rb{eHUm^vLd(RO;9O{#inOvIAua9eY49&yIVWS#yk(teC@>wt)W<8d{Gn_ zk%_l`$HD-eTKq9dj?TTYpb6XnbQ^e(%67+-RJ9}(Pb%g11YX6lbAamGVyU=iE6Rl4 z!QI31aOlv-c)))n^ZoD~Pd;u&&$y0iABBGI!;#zgfv&@&h>bMf%b!kJm*c#TZTN1H zI!2CmC*OmH?8c4f*fHb+X8hpMGH)4)<1=8(`565B@)hjxUrXPAdy%U{3%2<CAT?0cM^Y}sI-eIuCczD$JR^*d%0Z)89CsMb)h@Ii;tgOp~wK|2d z;_U0<)P+Zojz8q@hdbk~KY6sxbqCH^;s)}Ribo$_rP}@Ou)sPKwU;~*7rgt$m*gHs z!zc07g!3_Hiz;;m%;N422u0OXBSF5+8%EisqEFm<{$<>G+%a$riFTRL%TQM~V)tIW z)1yk!4n=g!dOlb2$Q%r2Zr~<4ynvg!-$lzjH0gZh8j$tuf({`+a7%$be%5jlZ_x2( z@nL^?wf|m%+NCMvZSP2>bOOr`yhMpydD_2wDU3*$1;77lAogT8OjzSVx=uRG+fRu% zT<*oD8TpGEDtFM`Kz$sh@SLuix#IAk9Jcq^07|gGL~q*cpwg=hr*(Rv)`S71Ib$+} zw>ApZwT3|oI}BU!C$4vO^-H2zk`~^w`^jIucvbYxJc!$7F^pa|5sY@@UeQ5%b1%t-!s_A<cLJGda*5BGnKh+ zV-F!5IwBEvl?N{8D3$b1bs@>V{y5M zA+d9;u%uj4`|$He zc^2$)59jupG9P&}{P*1(Jlpp``;8Fj@eqjB5#cn%_B+Jrj)j?j2BKx(eVAe{&$Q=p zRNf_pde$HC#Zg1L*lokyJ8657RTU z5TOJ&Zn=Z?fkw>5$B3Tf8G`<*aS$eC4V4LQN(LKCkN>sfwbBIq;-Amg%v?+3e%|EE zyFQ5{<$AGsQXW;v+!yukm8EQ^qsyH_t}Pe! zoRFj?xh1$=_7kpi*hFvM>SMR$UfjIW374iyl3lqi(^xPI?jF*o^!r1}LL(Fh22H0Q zF=jmXcL1AnZlA!yy#$@m9e8zjG(T~!Gxgptrt8m7LVz2A=jDO)Jz^hlLYBt;U$dD^ z$|x3hEr4%Z<-o_M4#I0%K6q~75qKGWpW8ZV8MO-#yUlAaxc#UJ+olX++fM1Tgbzyi zd4LuDp0Ec_WsL`^W_SCd^lX@vKbD_*P?y1HLwbZRB>T>k`m6Rp%1jBW^Nr?z`(5Jo z&TR&#*ib4L`fY9A?(qJUCPmIjMWaquWs7j`@y!^?;6;p?@DAid@>IRBYKUuS=VusuWBzTwgAzK0pTyg87$ z>WrZC%NtS6{v7$;mrsdESyAeczGwGI5&KT4!kDo^f($$Z48 zuW)9@b~>7*&I+o1q1@ve=lw9l-nieLMOV02*G`|w@{%PXrP`2reAtRd8k}%^eiCy$ z;l+B_=+NTtuVHEFSe9I(PQ!KX!=B72ex#L!@S=`KeYp_Ed)>mcgmafn+c#_9#Kl=5Q{Y|t&h%7g0oblo2~EbMgNl@SjU+-#tzIfkY+ z9H&}=*_4WpW7q4{Yu72}^9|a0Fz~-18lYAW!H4zO@Ue?P4JQ$K3cbL;X2RU(GMI4b zLXPAfht7Tu^K=Kx?ZupE zi5h!*$Cd4y=Kx<4cCa7(I?-b`jQXdY;qqQRq`~=n>9PM5dNrz2Y-t+A0v^_LAa@xb zsZV8B`h|I;%_6#d@g_b~ETFk(f+*&98eJ{5WUFV+1Bxv7F%Kk_qd@v8+4egF~;^7LrcIBwGDERqV; zr#Tnp*^P-o0{fMK5z@)*?%@nDwOYs~T3w_5MfteomJt*36%cR+ z6%{HO3W-Xou&?7Np)@19bKhy6$A<`M_^#gP`QG*1&v$=oz3V;qI@jKN?R_2l+~>K6 z>$;BP|NG-PuPl1o>k4`kd~i>n1bzFooY)~L$o}~p&uPd(op+lk{`FTbZ~bNQoYI3h zd50vlx5(1%Q#-{s5^aTJvN-Y$|A-0AbEvNEv)E1j0u*0T&Ve_u zD5C%>ZoI_qtuE*=T#ckYgkV^m5^I~=En2u$7iD(sV7X}`7N2nzuO?}-=mHVBM9<-# zomQYt^$X}h;}?PBnklgB09LRPCil7=-!~bupTnsIv>Lhm$jm~u}yGC#+SV1 z3Lw`zn3^W)!2~pfK9}(npJEBMpD$wdh--LAbt=IYUsOE%4u1~LLOa=HJTyjwq+Z0* z=HQ+1r0QLlQ0jwW)g0?R zwFR%-@}Vo=8$q_Yl2>38ur%))zqI5Y|MG_ywY;4}a+g2j&&C*96XVTY%YBbIg`EPQ zPX&{$MI<@0N3^?o3!K1}_}%9Yr1{&ky<=q1!})^k1KTrj z%d;0h{}@POI8F2I^SDjbNnjV)$+br>rojf+z;?a`Cw@GFF6WKsY9scrJp+%6{4Sc} zKZMVD?>9gCaXgG);y8qq8VVrZaR-|A+sI0m45jC>T5Ja1gp7mkBx!4j`4vaNsImop zT^6wVeRn}2a3?kFauoSiZr~oB7(CCLWKbPiZ z3QT4?7+BuInVn{^d3gY7|E|b=Jp2Y}&7ZJL$hlv4m!cJa%83>_DKnWqPs}S4E-5G6hH6{w3%*))SJE7t3I9_w;8d8bMyG< zVsmOPishUXGT=y8Cj==-k$Rmvt(oP@R2NH<*ii!)jTChH;v(qV+RVRd902aKO8ePrbg!Fy%oAe1^j~ zBViIYWK1BxtD8h?J?GQh77Iu-yN!-Eb6M(!vj|(t`PhQ7xX%757o6y5~w*|%h< z@Mn$Ka=AaPQPGCzQ}P1y{TS?1_25rLt%ngkHZgxdveTDV&17>CwTZ@syM)_|pU~;V1Xgf%|D9v_5?gHfkDEP4SAE#@IWf^lMo*{BjnScl9E5H)F|Z zC2|$=t!GbeAxCR}TKwB_aO!VCud`lNYQZ-4Ln0Y7>MjV{=;>I}wt-d)Y;SK*DSEoz zgd9^>l1-BfS(|0UF-v=BZyQJc&-9p%{9A$JQP2Mwwie%P8Oc<*M)(wQ6qYYvh7})W zNyxxM?7}fz{77|NcqxM9dx!Cxm4uvfb02hk84E9uuE$1kIC(rVCrPNGp5V=##iV_Z z=kyTQG$*01x-J#9rQ(x^Tlo~jKhQ2_0ym6bNk1Z0(NpyTj#GXFW4g}bh)tu&e$r=v zsSiXBv4)`E%CP3hHPi_|ii1zcvCqw^Ons3gW*E)n{cME)s~s(97#Km?p_BO$@rg{U z{~e}q#DNt?dDDpxM%4AxOwf2Kz^2Z-BIQJZ)B3CdzMjnFmrc8Xx}ldi_wwtco1sgq zYZhQYekhIH5<$DfCZOT?5z{M$^XprvVrE`~?dOrpxWYzbn5-}C7=YGfJd9%?%qah2h4PN?i13ZMv@?M;)ttFzuxtYl2*7) z*@OCD$VB=BIgS*iC1zlvzF}*1k#>MP}=Kjgd==3qso6z;l+ifnt zV4c7oy>=85HYQU$+8~-LGEcepP$D;%f^WWr+KWTk+tOlw=qxGLa(Nt;49|rf<9nfE zl_V`sauGb-Zex{8gbgpgf?2coBPa@UAX7#~0|!FK(mL_&M^-fIu_dGw3cq`PmUyl3 zCg-;77NXtH;-OlG+fWRp4pXtJ^1%J6`KI2)MYZ0lh7)He= zZ9;w80(j85m3{Br&63NPGtHr)P@eS~9y+hZ$dmoq=g<<8dfWhk3WDE~YbCmmNn}tW z4Mw}2VEIKg{!FO!6^#2M5BN$l=~3zKY)l?<>ba>~SO5@-ZJT9rnPDqlU1w zQ5|BHjj8bRfC7{*=ub;uFX24~&7-J~a~ZF8n^K3}#LyBu*m0`>bwdWwq~WCytK$Rv zKlS4`*7U=VbptB>T~*QBV=JY~nX-d5ZlZ6ML%6)etxy~yc%j`-=kM;;W+R_EkfdZR zBuS+Z^K3%%N9MF?!+5BDY)M0EQmL}+3T~eB1OEt;f^AC=V9Tc@d}ebIrl=Kji@U~i z?KgDUY+Zi}PThq;)rV=k*$Yvi;K9?f!H(iKwcxs8n`qOzrI7K+j8xvX;?R}mY_G8= z7v(>Mem)ya^SyS`Pw2+N;4;3YxejNgPvy3)ID&Rp;^1D31cW>qM6aX|V$g04n%XiL zi$9!)yIFlW_h+3rEasg^(_PStTc%@(-C!!q{)j#MOW}-2k9V(i0M+;wyn zoG+5*3X089ThbL`Y@$T@$u+jm7CF+T+k(f6tsGsv=gxC`8~C29MSR!Ad30{UBd#U4 zlzJNH;K^aSaNwIaJn8311)h>@+M>H)cH$T*J&qxj`PEnZ37!>7w$r>oMDG$vT!A1$yZ?`=|aPUS8h6xc8!o6qt7f$7-vQlIT`%ECmS z!JKL4K&JY75Jg<*!f$Z_B!BlIcs;oYj%*qI;xJu}Z$RG)2eLH~v@NC+>1(+GUhUL_ zu>spCRoGwSOXM(AJrWgSq?vrC3RCG`WgAZZ(d7n^%_pF; zoGZk6YSY*>ZQN3?!U9{Y;9bQR;5HS*(ZmGNEFHkDNmY71TRB;ngeK`NXUXT(*`nJM*2u>&JDrH!m7zo*0P< zYk%X1Wly5A-40Z1w26+B7dLwHL0r|Z7PkLk4@rx!z;1(g+^}$WQt*mpo?qHfY2#kab*B%` z)1W=+7Cx90$Da<1!X;btX-`=WeAP7~13{ChbvA(;1>OA6=$Uw{br2TzsXMpCFcy1DRgi`13patimKsh}Dmn%xPcs|!r(8nZ!yYC?OvFkm2bCqFQD{Z;F zXjPUSp8|d-{(zRRZmizmE#yWRGHdA~7CL7t`E?$IF6TNv+e-tVPc`Qnof3FIt&!9s zc19~nS#;D%hwjiL;HUeJTSK2v+clLfD_F?qlnUot27Ltim5F?m-Y${isXJ&;R7c)h z6W9Rv9LO}8gO?39kf(5*7n~cyt;lbJVO9rm`?y)qtGgf6p32e3>LZ+?tQxENP4Leg zeHMiT2HyMg46V7p2*PNP1qV%p@YCjmQfMDTZUqFO)F z&-5felP30P`C)QM3jhQ0UTRI<4kPEpgWBF{?0}ew5AN>*t6obkrBej=tbcMha2|CO z3tU3e@o>X(H@oBaN?=qFgdb?ZD>06u-IqqPmqum$RkM7oD2bq@UE`?rvkc2RA;wY3 zGMv#LQA}}Z2Y8h^!<3o}^uAo3ZT>ol>ed@juKsBVEGy$;W#{wRl9IGPJ)O7mJxkRm zzv0hJfqj1b0m_D3vC9sfILi7xbfo057mxzS<-XFe!7-?L=LrTjCUe;XPh)@SR}`8P z##WyfY6+c>z=uDTDb%JIhMU&#Dl?NQEXx9Q%^jKF#CT4J1TEx^)BFtU9BgpUMC*fY zywToxuzW8CgT0@_%wh}b=W++vJ=utn%N2!nSwTbAcVSy?2b!!;GcmlHN71*(^4qdyN z$X!k!&BtEntLGTf?9=bK+&jghhGiD4M8$%>b$gJNMJ?X&(qKbuhqL~|cWuq#M}k-E zHeTzh3;ND`4POs+<9e@nu0j+AO}V1VRP(tI>UtYIzRaN7d&=B-p)P*0nHAo=^#exa zb_mSemALWYRaz5WDe_9O2k+mcnZZfH15wcdOkoPl$eqIGpSGko^^#!LUC&kKCSma2 z>EOI&CS^{DL@T2U0=r;3H(dDdqbr~BK*$x$Zy&*uCrLBwFrlV;`c6tr7){A{{iyum zF32>pr&yn1OyR2nvzt)FIX&6KDPJhZf4rK`t=H?OlGlpDG58R=`E4@0Te6MjZ61ZE z2AAWL$uVdsc-48on@SDunxVMt2W<7XCH0mPT)0k>{5`DcMbUFi=Zksm#xyQgehtk} z>V7_b=+8jKBc?Ty042!()=JZK?Rf)_DeKZ zO@&u+r&-csEgC9sNcwA6(793mH2wN_T(sdmmvB}MAJRVdSbGxn)1HJ1$6TS^iLo;> zgJ`0+EBo_mHtKlhif4ZQlr{K{0fk3|Iaq_f&xCs5!*YCY?@;QMSVa2^D=dZL^x}W;6;EOUYQ+B0^H4E6b4TkK!pg9{bzzL$j0-f~(=zE(D-ccJy6FUC^ znl3U-W!)_8N3H><11lm?@M0LYtebn9ForU#~e9nRukSQN^fr9;{xu{o8zb1 z)!ZViQ;S1Qa}BUrS%5b>_JI78<8WZAkhh-UOg=mNQ_Smgczc`~EzeP)q8aBv+3YYz z_gv!_2Tg#dSNgDbMKHSDnF@>itKix7FmX$$JxJ=j#;z0#${Y0>ll+Ce@@%23CgUJ} z?zt>_a;})Q_Kl&Jw^Lcw1!a1(+>Tz%yNCDkr<0V9kQ+R%1Nl0&I9`;AjUOaf{jPCU z7slq&WXbtVRvRn5b|P0O;g~+8V2s)dZ07F1AVPYJqK77AKr;7UA$u~Nqw9w(&p?D}Ysao8bN zteZz6HsAR*_suEWsZ!uaI?>(hC0y-@RC-dZj73jhpjPG+Tvj~|+XOFe(_wvRQTr4+ zlF~)9-(KdTO77GCICFA!_ad#RV2G>`e8zshvyCkK0P)-9$Vjw|H8usYvd#K5%SBFp zzUr7MlMhKq{OYq@q@*Pz$Ow!u?-e>(>K+X@4_szXoj-zHIq_2HFnKG-QR1fA!&i z31TBD@po^3VRU5h%7~!gi2scA@5`$3mq`2nKGJ{9qsf1U`S-Qa{7aZw*?&8re|;kU zGtR%yp5b5OH2r;?zhBo`F0!(}{cCd_sR{obNakO+{_l0P@E(7)bw~brEo>+E>sEWt YKkxr~c$SOIuW6Q$5iY-;|Ig?CA4?LY@Bjb+ diff --git a/tests/test_nonsequential/exp_set_TA1/nonseq_model.py b/tests/test_nonsequential/exp_set_TA1/nonseq_model.py deleted file mode 100644 index 7e65c172..00000000 --- a/tests/test_nonsequential/exp_set_TA1/nonseq_model.py +++ /dev/null @@ -1,117 +0,0 @@ -import torch -import torch.nn as nn -import sinabs.layers as sl -from sinabs.activation.surrogate_gradient_fn import PeriodicExponential - -class SNN(nn.Module): - def __init__(self, nb_classes, pool2lin_size, batch_size) -> None: - super().__init__() - - self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False) - self.iaf1 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential()) - self.pool1 = nn.AvgPool2d(2,2) - self.pool1a = nn.AvgPool2d(6,6) - - self.conv2 = nn.Conv2d(10, 10, 2, 1, bias=False) - self.iaf2 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential()) - self.pool2 = nn.AvgPool2d(3,3) - - self.conv3 = nn.Conv2d(10, 10, 3, 1, bias=False) - self.iaf3 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential()) - self.pool3 = nn.AvgPool2d(2,2) - - self.flat = nn.Flatten() - - self.fc1 = nn.Linear(pool2lin_size, 100, bias=False) - self.iaf4 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential()) - - self.fc2 = nn.Linear(100, 100, bias=False) - self.iaf5 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential()) - - self.fc3 = nn.Linear(100, 100, bias=False) - self.iaf6 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential()) - - self.fc4 = nn.Linear(100, nb_classes, bias=False) - self.iaf7 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential()) - - self.merge_fc = sl.Merge() - self.merge_conv = sl.Merge() - - def export_conv_params(self): - torch.save(self.conv1.state_dict(), 'nonseq_conv1_weights.pth') - torch.save(self.conv2.state_dict(), 'nonseq_conv2_weights.pth') - torch.save(self.conv3.state_dict(), 'nonseq_conv3_weights.pth') - torch.save(self.fc2.state_dict(), 'nonseq_fc2_weights.pth') - torch.save(self.fc3.state_dict(), 'nonseq_fc3_weights.pth') - - def load_conv_params(self, w_load): - if w_load == 0: - self.conv1.load_state_dict(torch.load('nonseq_conv1_weights.pth')) - elif w_load == 1: - self.conv2.load_state_dict(torch.load('nonseq_conv2_weights.pth')) - elif w_load == 2: - self.conv3.load_state_dict(torch.load('nonseq_conv3_weights.pth')) - elif w_load == 4: - self.conv1.load_state_dict(torch.load('nonseq_conv1_weights.pth')) - self.conv2.load_state_dict(torch.load('nonseq_conv2_weights.pth')) - self.conv3.load_state_dict(torch.load('nonseq_conv3_weights.pth')) - elif w_load == 5: - self.fc2.load_state_dict(torch.load('nonseq_fc2_weights.pth')) - elif w_load == 6: - self.fc3.load_state_dict(torch.load('nonseq_fc3_weights.pth')) - elif w_load == 7: - self.fc2.load_state_dict(torch.load('nonseq_fc2_weights.pth')) - self.fc3.load_state_dict(torch.load('nonseq_fc3_weights.pth')) - elif w_load == 8: - self.conv1.load_state_dict(torch.load('nonseq_conv1_weights.pth')) - self.conv2.load_state_dict(torch.load('nonseq_conv2_weights.pth')) - self.conv3.load_state_dict(torch.load('nonseq_conv3_weights.pth')) - self.fc2.load_state_dict(torch.load('nonseq_fc2_weights.pth')) - self.fc3.load_state_dict(torch.load('nonseq_fc3_weights.pth')) - - def detach_neuron_states(self): - for name, layer in self.named_modules(): - if name != '': - if isinstance(layer, sl.StatefulLayer): - for name, buffer in layer.named_buffers(): - buffer.detach_() - - def init_weights(self): - for name, layer in self.named_modules(): - if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear): - nn.init.xavier_normal_(layer.weight.data) - - def forward(self, x): - - con1_out = self.conv1(x) - iaf1_out = self.iaf1(con1_out) - pool1_out = self.pool1(iaf1_out) - pool1a_out = self.pool1a(iaf1_out) - - conv2_out = self.conv2(pool1_out) - iaf2_out = self.iaf2(conv2_out) - pool2_out = self.pool2(iaf2_out) - - merged_conv_out = self.merge_conv(pool1a_out, pool2_out) - - conv3_out = self.conv3(merged_conv_out) - iaf3_out = self.iaf3(conv3_out) - pool3_out = self.pool3(iaf3_out) - - flat_out = self.flat(pool3_out) - - fc1_out = self.fc1(flat_out) - iaf4_out = self.iaf4(fc1_out) - - fc2_out = self.fc2(iaf4_out) - iaf5_out = self.iaf5(fc2_out) - - fc3_out = self.fc3(iaf5_out) - iaf6_out = self.iaf6(fc3_out) - - merge_fc_out = self.merge_fc(iaf4_out, iaf6_out) - - fc4_out = self.fc4(merge_fc_out) - iaf7_out = self.iaf7(fc4_out) - - return iaf7_out \ No newline at end of file diff --git a/tests/test_nonsequential/exp_set_TA1/train_script.py b/tests/test_nonsequential/exp_set_TA1/train_script.py deleted file mode 100644 index a752422b..00000000 --- a/tests/test_nonsequential/exp_set_TA1/train_script.py +++ /dev/null @@ -1,152 +0,0 @@ -import torch, random, sys -import torch.nn as nn -from tqdm.notebook import tqdm - -from tonic.transforms import ToFrame -from torch.utils.data import DataLoader -from torch.nn import CrossEntropyLoss -from torch.optim import Adam - -import numpy as np - -from nonseq_model import SNN - -torch.backends.cudnn.deterministic = True -random.seed(1) -torch.manual_seed(1) -torch.cuda.manual_seed(1) -np.random.seed(1) - -batch_size = 3 -num_workers = 1 -epochs = 30 -lr = 1e-3 -n_time_steps = 50 - -if torch.cuda.is_available(): - device = torch.device('cuda:0') - print('device: ', torch.cuda.get_device_name(0)) -else: - device = torch.device('cpu') - -def train(batch_size, feature_map_size, dataloader_train, model, loss_fn, optimizer, epochs, test_func, dataloader_test, phase): - epochs_y = [] - epochs_x = [] - epochs_acc = [] - model.train() - - for e in range(epochs): - losses = [] - batches = [] - batch_count = 0 - train_p_bar = tqdm(dataloader_train) - - for X, y in train_p_bar: - # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width] - X = X.reshape(-1, feature_map_size[2], feature_map_size[0], feature_map_size[1]).to(dtype=torch.float, device=device) - y = y.to(dtype=torch.long, device=device) - - # forward - pred = model(X) - - # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes] - pred = pred.reshape(batch_size, n_time_steps, -1) - - # accumulate all time-steps output for final prediction - pred = pred.sum(dim = 1) - loss = loss_fn(pred, y) - - # gradient update - optimizer.zero_grad() - loss.backward() - optimizer.step() - - # detach the neuron states and activations from current computation graph(necessary) - model.detach_neuron_states() - - train_p_bar.set_description(f"{phase} - Epoch {e} - BPTT Training Loss: {round(loss.item(), 4)}") - - batch_count += 1 - losses.append(loss.item()) - batches.append(batch_count) - - epochs_y.append(losses) - epochs_x.append(batches) - - acc = test_func(batch_size, feature_map_size, dataloader_test, model) - print(f'{phase} - Epoch {e} accuracy: {acc}') - epochs_acc.append(acc) - - return epochs_x, epochs_y, epochs_acc - -def test(batch_size, feature_map_size, dataloader, model): - correct_predictions = [] - with torch.no_grad(): - test_p_bar = tqdm(dataloader) - for X, y in test_p_bar: - # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width] - X = X.reshape(-1, feature_map_size[2], feature_map_size[0], feature_map_size[1]).to(dtype=torch.float, device=device) - y = y.to(dtype=torch.long, device=device) - - # forward - output = model(X) - - # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes] - output = output.reshape(batch_size, n_time_steps, -1) - - # accumulate all time-steps output for final prediction - output = output.sum(dim=1) - - # calculate accuracy - pred = output.argmax(dim=1, keepdim=True) - - # compute the total correct predictions - correct_predictions.append(pred.eq(y.view_as(pred))) - - test_p_bar.set_description(f"Testing Model...") - - correct_predictions = torch.cat(correct_predictions) - return correct_predictions.sum().item()/(len(correct_predictions))*100 - -from tonic.datasets.dvsgesture import DVSGesture - -root_dir = "../DVSGESTURE" -_ = DVSGesture(save_to=root_dir, train=True) -_ = DVSGesture(save_to=root_dir, train=False) - -to_raster = ToFrame(sensor_size=DVSGesture.sensor_size, n_time_bins=n_time_steps) - -snn_train_dataset = DVSGesture(save_to=root_dir, train=True, transform=to_raster) -snn_test_dataset = DVSGesture(save_to=root_dir, train=False, transform=to_raster) - -sample_data, label = snn_train_dataset[0] -print(f"The transformed array is in shape [Time-Step, Channel, Height, Width] --> {sample_data.shape}") - -snn_train_dataloader = DataLoader(snn_train_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True) -snn_test_dataloader = DataLoader(snn_test_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=False) - -snn = SNN(11, 810, batch_size).to(device) -snn.init_weights() - -snn.load_conv_params(int(sys.argv[1])) - -optimizer = Adam(snn.parameters(), lr=lr, betas=(0.9, 0.999), eps=1e-8) -loss_fn = CrossEntropyLoss() - -epochs_x, epochs_y, epochs_acc = train( - batch_size, - DVSGesture.sensor_size, - snn_train_dataloader, - snn, - loss_fn, - optimizer, - epochs, - test, - snn_test_dataloader, - 'post-training' - ) - -with open(f'nonseq_TA1_w_load_{sys.argv[1]}_training_metrics.npy', 'wb') as f: - np.save(f, np.array(epochs_x)) - np.save(f, np.array(epochs_y)) - np.save(f, np.array(epochs_acc)) \ No newline at end of file diff --git a/tests/test_nonsequential/non-sequential-SCNN-example_1.ipynb b/tests/test_nonsequential/non-sequential-SCNN-example_1.ipynb deleted file mode 100644 index 16c36d57..00000000 --- a/tests/test_nonsequential/non-sequential-SCNN-example_1.ipynb +++ /dev/null @@ -1,649 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [], - "source": [ - "import torch\n", - "import torch.nn as nn\n", - "import sinabs.layers as sl\n", - "from tqdm.notebook import tqdm\n", - "\n", - "from tonic.datasets.nmnist import NMNIST\n", - "from tonic.transforms import ToFrame\n", - "from torch.utils.data import DataLoader\n", - "from torch.nn import CrossEntropyLoss\n", - "from torch.optim import Adam\n", - "\n", - "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential\n", - "\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "torch.manual_seed(0)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Loading Data" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "batch_size = 64\n", - "num_workers = 4\n", - "epochs = 5\n", - "lr = 1e-3" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "root_dir = \"./NMNIST\"\n", - "_ = NMNIST(save_to=root_dir, train=True)\n", - "_ = NMNIST(save_to=root_dir, train=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "n_time_steps = 50\n", - "to_raster = ToFrame(sensor_size=NMNIST.sensor_size, n_time_bins=n_time_steps)\n", - "\n", - "snn_train_dataset = NMNIST(save_to=root_dir, train=True, transform=to_raster)\n", - "snn_test_dataset = NMNIST(save_to=root_dir, train=False, transform=to_raster)" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "snn_train_dataloader = DataLoader(snn_train_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True)\n", - "snn_test_dataloader = DataLoader(snn_test_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=False)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Network Module\n", - "\n", - "We need to define a `nn.Module` implementing the network we want the chip to reproduce." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "device: NVIDIA RTX A4000\n" - ] - } - ], - "source": [ - "if torch.cuda.is_available():\n", - " device = torch.device('cuda:0')\n", - " print('device: ', torch.cuda.get_device_name(0))\n", - "else:\n", - " device = torch.device('cpu')" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "class SNN(nn.Module):\n", - " def __init__(self) -> None:\n", - " super().__init__()\n", - "\n", - " self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False)\n", - " self.iaf1 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - " self.pool1 = nn.AvgPool2d(2,2)\n", - " self.pool1a = nn.AvgPool2d(6,6)\n", - "\n", - " self.conv2 = nn.Conv2d(10, 10, 2, 1, bias=False)\n", - " self.iaf2 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - " self.pool2 = nn.AvgPool2d(3,3)\n", - "\n", - " self.conv3 = nn.Conv2d(10, 10, 3, 1, bias=False)\n", - " self.iaf3 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - " self.pool3 = nn.AvgPool2d(2,2)\n", - "\n", - " self.flat = nn.Flatten()\n", - "\n", - " self.fc1 = nn.Linear(10, 10, bias=False)\n", - " self.iaf4 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - "\n", - " self.merge = sl.Merge()\n", - "\n", - "\n", - " def detach_neuron_states(self):\n", - " for name, layer in self.named_modules():\n", - " if name != '':\n", - " if isinstance(layer, sl.StatefulLayer):\n", - " for name, buffer in layer.named_buffers():\n", - " buffer.detach_()\n", - "\n", - " def init_weights(self):\n", - " for name, layer in self.named_modules():\n", - " if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear):\n", - " nn.init.xavier_normal_(layer.weight.data)\n", - "\n", - " def forward(self, x):\n", - " \n", - " con1_out = self.conv1(x)\n", - " iaf1_out = self.iaf1(con1_out)\n", - " pool1_out = self.pool1(iaf1_out)\n", - " pool1a_out = self.pool1a(iaf1_out)\n", - "\n", - " conv2_out = self.conv2(pool1_out)\n", - " iaf2_out = self.iaf2(conv2_out)\n", - " pool2_out = self.pool2(iaf2_out)\n", - "\n", - " merged = self.merge(pool1a_out, pool2_out)\n", - "\n", - " conv3_out = self.conv3(merged)\n", - " iaf3_out = self.iaf3(conv3_out)\n", - " pool3_out = self.pool3(iaf3_out)\n", - "\n", - " flat_out = self.flat(pool3_out)\n", - " \n", - " fc1_out = self.fc1(flat_out)\n", - " iaf4_out = self.iaf4(fc1_out)\n", - "\n", - " return iaf4_out" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "snn = SNN().to(device)" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "# x = torch.randn((batch_size, 2, 34, 34)).to(device)\n", - "\n", - "# con1_out = snn.conv1(x)\n", - "# iaf1_out = snn.iaf1(con1_out)\n", - "# pool1_out = snn.pool1(iaf1_out)\n", - "# pool1a_out = snn.pool1a(iaf1_out)\n", - "# print(pool1a_out.shape)\n", - "\n", - "# conv2_out = snn.conv2(pool1_out)\n", - "# iaf2_out = snn.iaf2(conv2_out)\n", - "# pool2_out = snn.pool2(iaf2_out)\n", - "# print(pool2_out.shape)\n", - "\n", - "# conv3_out = snn.conv3(pool2_out)\n", - "# iaf3_out = snn.iaf3(conv3_out)\n", - "# pool3_out = snn.pool3(iaf3_out)\n", - "\n", - "# flat_out = snn.flat(pool3_out)\n", - "# print(flat_out.shape)\n", - "\n", - "# fc1_out = snn.fc1(flat_out)\n", - "# iaf4_out = snn.iaf4(fc1_out)" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "snn.init_weights()" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "optimizer = Adam(snn.parameters(), lr=lr, betas=(0.9, 0.999), eps=1e-8)\n", - "loss_fn = CrossEntropyLoss()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Define train and test" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "def train(dataloader, model, loss_fn, optimizer, epochs, test_func, dataloader_test):\n", - " epochs_y = []\n", - " epochs_x = []\n", - " epochs_acc = []\n", - " model.train()\n", - "\n", - " for e in range(epochs):\n", - " losses = []\n", - " batches = []\n", - " batch_count = 0\n", - " train_p_bar = tqdm(snn_train_dataloader)\n", - "\n", - " for X, y in train_p_bar:\n", - " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", - " X = X.reshape(-1, 2, 34, 34).to(dtype=torch.float, device=device)\n", - " y = y.to(dtype=torch.long, device=device)\n", - "\n", - " # forward\n", - " pred = model(X)\n", - "\n", - " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", - " pred = pred.reshape(batch_size, n_time_steps, -1)\n", - "\n", - " # accumulate all time-steps output for final prediction\n", - " pred = pred.sum(dim = 1)\n", - " loss = loss_fn(pred, y)\n", - "\n", - " # gradient update\n", - " optimizer.zero_grad()\n", - " loss.backward()\n", - " optimizer.step()\n", - "\n", - " # detach the neuron states and activations from current computation graph(necessary)\n", - " model.detach_neuron_states()\n", - "\n", - " train_p_bar.set_description(f\"Epoch {e} - BPTT Training Loss: {round(loss.item(), 4)}\")\n", - "\n", - " batch_count += 1\n", - " losses.append(loss.item())\n", - " batches.append(batch_count)\n", - "\n", - " epochs_y.append(losses)\n", - " epochs_x.append(batches)\n", - "\n", - " acc = test_func(dataloader_test, model)\n", - " print(f'Epoch {e} accuracy: {acc}')\n", - " epochs_acc.append(acc)\n", - "\n", - " return epochs_x, epochs_y, epochs_acc\n" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [], - "source": [ - "def test(dataloader, model):\n", - " correct_predictions = []\n", - " with torch.no_grad():\n", - " test_p_bar = tqdm(dataloader)\n", - " for X, y in test_p_bar:\n", - " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", - " X = X.reshape(-1, 2, 34, 34).to(dtype=torch.float, device=device)\n", - " y = y.to(dtype=torch.long, device=device)\n", - "\n", - " # forward\n", - " output = model(X)\n", - "\n", - " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", - " output = output.reshape(batch_size, n_time_steps, -1)\n", - "\n", - " # accumulate all time-steps output for final prediction\n", - " output = output.sum(dim=1)\n", - "\n", - " # calculate accuracy\n", - " pred = output.argmax(dim=1, keepdim=True)\n", - "\n", - " # compute the total correct predictions\n", - " correct_predictions.append(pred.eq(y.view_as(pred)))\n", - "\n", - " test_p_bar.set_description(f\"Testing Model...\")\n", - " \n", - " correct_predictions = torch.cat(correct_predictions)\n", - " return correct_predictions.sum().item()/(len(correct_predictions))*100" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Training loop (HPO)" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "fbe5ba8d578d481b97c8e81bebb4d2c7", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - " 0%| | 0/937 [00:00" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "plt.plot(epochs_x[-1], epochs_y[-1])\n", - "plt.xlabel('batches')\n", - "plt.ylabel('loss')\n", - "plt.ylim(0,)\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAGwCAYAAABVdURTAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABVm0lEQVR4nO3de1xUZf4H8M/MADOgMIjcEQVFvIsoSl7SVJTUtbTd1dQNV2vL1NJQS800q40ualZeK0vbtryUWr90RcQLqZiJUl5RLooXriozMMAAM+f3Bzo5cpHBGQ4zfN6v17w2zjznzPfp7MSH5zzPORJBEAQQERER2Qip2AUQERERmRPDDREREdkUhhsiIiKyKQw3REREZFMYboiIiMimMNwQERGRTWG4ISIiIptiJ3YBDU2v1+PGjRtwdnaGRCIRuxwiIiKqA0EQUFhYCF9fX0iltY/NNLlwc+PGDfj7+4tdBhEREdXD1atX0apVq1rbNLlw4+zsDKDyX46Li4vI1RAREVFdqNVq+Pv7G36P16bJhZu7l6JcXFxqDTcJCQn48MMPkZSUhKysLOzYsQNjxoyp9dharRZvvfUWvvnmG2RnZ8PHxweLFy/G1KlTAQCPPfYYDh06VGW/kSNHYteuXfXvFBERURNRlyklTS7c1JVGo0FISAimTp2Kp556qk77jBs3Djk5OdiwYQOCgoKQlZUFvV5veH/79u0oKysz/Hzz5k2EhITg73//u9nrJyIiaqoYbmowYsQIjBgxos7t9+zZg0OHDiE9PR1ubm4AgICAAKM2d7fftXnzZjg5OTHcEBERmRGXgpvJTz/9hLCwMHzwwQfw8/NDcHAw5s6di5KSkhr32bBhA55++mk0a9asASslIiKybRy5MZP09HQcPnwYCoUCO3bsQH5+PqZPn46bN2/iq6++qtL++PHjOHPmDDZs2CBCtURERLaL4cZM9Ho9JBIJ/vvf/0KpVAIAVqxYgb/97W9Ys2YNHB0djdpv2LAB3bp1Q58+fcQol4iIyGbxspSZ+Pj4wM/PzxBsAKBTp04QBAHXrl0zaqvRaLB582Y8++yzDV0mERGRzWO4MZP+/fvjxo0bKCoqMmy7ePEipFJplZsNbdu2DVqtFv/4xz8aukwiIiKbx3BTg6KiIiQnJyM5ORkAkJGRgeTkZGRmZgIAFixYgKioKEP7iRMnomXLlpgyZQrOnTuHhIQEzJs3D1OnTq32ktSYMWPQsmXLBusPERFRU8E5NzU4ceIEBg8ebPg5OjoaADB58mRs3LgRWVlZhqADAM2bN8d/t/8f5rwyG73CwuDesiXGjRuHd955x+i4KSkpOHz4MPbu3dswHSEiImpiJIIgCGIX0ZDUajWUSiVUKpVZH7+w5bdMLNh+GnoBkEqAmKe6YXzv1mY7PhERUVNmyu9vXpYygyxViSHYAIBeABZuP4MsVc33uCEiIiLLYLgxg4x8jSHY3KUTBFzOLxanICIioiaM4cYMAt2bQXrfc7xkEiDA3UmcgoiIiJowhhsz8FE6IuapbkYBZ0Kf1vBROta8ExEREVkEw42ZjO/dGkfmD8HYUF8AwK8Zt6C7/1oVERERWRzDjRn5KB3x5hNd4aKww6XcIvz0+3WxSyIiImpyGG7MTOlojxcGtQMAfBR3CeU6vcgVERERNS0MNxYwpX8A3Js7IPNWMbaduPbgHYiIiMhsGG4swMnBDtMfCwIAfLr/EkrLdSJXRERE1HQw3FjIxPDW8FEqkKUqxX9/zXzwDkRERGQWDDcWorCX4eWh7QEAaw6kQqOtELkiIiKipoHhxoL+1qsV2rR0wk1NGTYevSx2OURERE2CqOEmISEBo0ePhq+vLyQSCXbu3PnAfbRaLV5//XW0adMGcrkcAQEB+PLLLy1fbD3Yy6SYHVE5erP+UBpUJeUiV0RERGT7RA03Go0GISEhWL16dZ33GTduHOLj47FhwwakpKTgu+++Q4cOHSxY5cN5IsQP7T2bQ11agS9+SRe7HCIiIptnJ+aHjxgxAiNGjKhz+z179uDQoUNIT0+Hm5sbACAgIKDWfbRaLbRareFntVpdr1rrSyaVYM7wYEz75iS+PJyBf/YLQMvm8gatgYiIqCmxqjk3P/30E8LCwvDBBx/Az88PwcHBmDt3LkpKSmrcJyYmBkql0vDy9/dvwIorRXbxRjc/JTRlOqw9mNbgn09ERNSUWFW4SU9Px+HDh3HmzBns2LEDK1euxPfff4/p06fXuM+CBQugUqkMr6tXrzZgxZUkksrRGwD4+tgVZKtKG7wGIiKipsKqwo1er4dEIsF///tf9OnTByNHjsSKFSuwadOmGkdv5HI5XFxcjF5iGBTsgd4BLVBWocen+y+JUgMREVFTYFXhxsfHB35+flAqlYZtnTp1giAIuHatcT/mQCKRYO7wyonPW367isybxSJXREREZJusKtz0798fN27cQFFRkWHbxYsXIZVK0apVKxErq5vwti3xaHt3VOgFrIy/KHY5RERENknUcFNUVITk5GQkJycDADIyMpCcnIzMzMrHFSxYsABRUVGG9hMnTkTLli0xZcoUnDt3DgkJCZg3bx6mTp0KR0dHMbpgsrujNztPXUdqbqHI1RAREdkeUcPNiRMnEBoaitDQUABAdHQ0QkNDsXjxYgBAVlaWIegAQPPmzREXF4eCggKEhYVh0qRJGD16ND755BNR6q+PEH9XDOvsBb0ArIjj6A0REZG5SQRBEMQuoiGp1WoolUqoVCrRJhdfyFZjxMe/QBCAn18agK5+ygfvRERE1ISZ8vvbqubc2IqO3i4Y3d0XAEdviIiIzI3hRiSvDAuGTCrB/gu5SLpyW+xyiIiIbAbDjUgC3Zvhbz0rV3gti00RuRoiIiLbwXAjopcj2sNBJkVi+k0cSc0XuxwiIiKbwHAjIj9XR0wMbw0A+DA2BU1sbjcREZFFMNyIbPrgdlDYS5F8tQDx53PFLoeIiMjqMdyIzNNZgcn9AgAAy/amQK/n6A0REdHDYLhpBKYNbAdnuR0uZBdi95ksscshIiKyagw3jUCLZg549tFAAJX3vanQ6UWuiIiIyHox3DQSzw4IRAsne6TnabDj1HWxyyEiIrJaDDeNhLPCHtMGtQMAfBx/CWUVHL0hIiKqD4abRiSqbwA8nOW4drsEW37LfPAOREREVAXDTSPi6CDDS0OCAACf7k9FSZlO5IqIiIisD8NNI/N079bwc3VEbqEW/zl2WexyiIiIrA7DTSPjYCfFrIj2AIC1B9NQWFouckVERETWheGmEXoq1A9t3ZvhdnE5vjx8WexyiIiIrArDTSNkJ5Ni9rBgAMAXv6SjoLhM5IqIiIisB8NNI/WXbj7o6O2MQm0F1ieki10OERGR1WC4aaSkUgnmDO8AANh45DJyC0tFroiIiMg6MNw0YhGdPBHi74qSch3WHEgTuxwiIiKrwHDTiEkkEsy7M3rz7a+ZuF5QInJFREREjR/DTSPXP6glHmnrhjKdHp/GXxK7HCIiokaP4aaRk0gkmBdZOXqzLekaMvI1IldERETUuDHcWIFebdwwuIMHdHoBK/ddFLscIiKiRo3hxkrcXTn10+83kJJdKHI1REREjRfDjZXo6qfEiK7eEARgRVyK2OUQERE1Wgw3ViR6WDAkEiD2bA7+uFYgdjlERESNEsONFWnv5YyxPfwAAMv2cu4NERFRdRhurMzsiGDYSSVIuJiH4xm3xC6HiIio0WG4sTKtWzphXG9/AMCy2BQIgiByRURERI0Lw40VemlIEBzspDh++RYSLuWLXQ4REVGjImq4SUhIwOjRo+Hr6wuJRIKdO3fWed8jR47Azs4OPXr0sFh9jZWP0hHPPNIGALB8L0dviIiI7iVquNFoNAgJCcHq1atN2q+goABRUVEYOnSohSpr/F58rB2cHGT445oKsWdzxC6HiIio0RA13IwYMQLvvPMOxo4da9J+06ZNw8SJE9G3b18LVdb4uTeXY0r/AACV973R6Tl6Q0REBFjhnJuvvvoK6enpWLJkSZ3aa7VaqNVqo5eteP7RdnBW2OFiThF+/uOG2OUQERE1ClYVbi5duoT58+fjm2++gZ2dXZ32iYmJgVKpNLz8/f0tXGXDUTrZ44WBbQEAH8VdRLlOL3JFRERE4rOacKPT6TBx4kQsXboUwcHBdd5vwYIFUKlUhtfVq1ctWGXDm9I/EC2bOeDyzWL8kHRN7HKIiIhEZzXhprCwECdOnMDMmTNhZ2cHOzs7vPXWW/j9999hZ2eH/fv3V7ufXC6Hi4uL0cuWNJPb4cXH2gEAPom/BG2FTuSKiIiIxGU14cbFxQWnT59GcnKy4TVt2jR06NABycnJCA8PF7tE0fzjkTbwdlHghqoU3/6aKXY5REREoqrbxBULKSoqQmpqquHnjIwMJCcnw83NDa1bt8aCBQtw/fp1fP3115BKpejatavR/p6enlAoFFW2NzUKexleGhqE13ecweoDqRjf2x9ODqKeWiIiItGIOnJz4sQJhIaGIjQ0FAAQHR2N0NBQLF68GACQlZWFzEyORNTFuDB/tHZzQn5RGTYevSx2OURERKKRCE3s9rZqtRpKpRIqlcrm5t/8kHQNc7b9DqWjPRJeHQylo73YJREREZmFKb+/rWbODT3YmFA/BHk2h6qkHBsOZ4hdDhERkSgYbmyITCpB9LDKZfIbfknHLU2ZyBURERE1PIYbG/N4F2908XWBpkyHdYfSxC6HiIiowTHc2BipVIK5wzsAADYdvYwcdanIFRERETUshhsb9FgHD/Rq0wLaCj1W7U998A5EREQ2hOHGBkkkf47ebP4tE1dvFYtcERERUcNhuLFRfdu1xIAgd5TrBHwcf0nscoiIiBoMw40NmzO8cuXU9pPXkJpbJHI1REREDYPhxoaFtm6BiE6e0AvAR/suil0OERFRg2C4sXHRwyrn3uz6IwvnbqhFroaIiMjyGG5sXGdfF/yluw8AYEVcisjVEBERWR7DTRPwyrBgSCXAvvO5OJl5W+xyiIiILIrhpglo59Ecf+3ZCgCwfC9Hb4iIyLYx3DQRLw9tD3uZBEdSb+JoWr7Y5RAREVkMw00T4e/mhAl9WgMAlsWmQBAEkSsiIiKyDIabJmTm4CDI7aQ4mVmAAym5YpdDRERkEQw3TYiniwKT+wUAAJbFXoRez9EbIiKyPQw3Tcy0Qe3QXG6Hc1lq/O9MttjlEBERmR3DTRPj1swBUwcEAqi8742OozdERGRjGG6aoOceDYTS0R5peRrsPHVd7HKIiIjMiuGmCXJR2GPaoHYAgJXxF1FWoRe5IiIiIvNhuGmiJvdrA/fmcly9VYKtJ66KXQ4REZHZMNw0UU4Odpg5uHL05tP9l1BarhO5IiIiIvNguGnCJoS3hq9SgRy1Ft8cuyJ2OURERGbBcNOEye1keHloewDAmoNpKNJWiFwRERHRw2O4aeL+2qsVAlo64ZamDF8dzhC7HCIioofGcNPE2cukeGVYMADgs1/SoSouF7kiIiKih8NwQxjd3RcdvJxRWFqBz35JE7scIiKih8JwQ5BKJYgeXjl689WRy8gv0opcERERUf0x3BAAYHhnL4S0UqK4TIc1Bzh6Q0RE1ovhhgAAEokEc4Z3AAB88+sVZKlKRK6IiIiofhhuyODR9u7oE+iGsgo9PolPFbscIiKiehE13CQkJGD06NHw9fWFRCLBzp07a22/fft2DBs2DB4eHnBxcUHfvn0RGxvbMMU2ARKJBHPvjN5sO3EVV25qRK6IiIjIdKKGG41Gg5CQEKxevbpO7RMSEjBs2DDs3r0bSUlJGDx4MEaPHo1Tp05ZuNKmo0+gGwYGe6BCL2Dlvktil0NERGQyiSAIgthFAJWjBjt27MCYMWNM2q9Lly4YP348Fi9eXKf2arUaSqUSKpUKLi4u9ajU9v1xrQBPrDoCiQSInT0QwV7OYpdERERNnCm/v616zo1er0dhYSHc3NxqbKPVaqFWq41eVLvurVwR2cULggB8FHdR7HKIiIhMYtXhZtmyZSgqKsK4ceNqbBMTEwOlUml4+fv7N2CF1mvO8A6QSID/ncnGmesqscshIiKqM6sNN99++y2WLl2KrVu3wtPTs8Z2CxYsgEqlMryuXr3agFVar2AvZzwZ4gsAWLY3ReRqiIiI6s4qw83mzZvx3HPPYevWrYiIiKi1rVwuh4uLi9GL6mZ2RDBkUgkOpuThxOVbYpdDRERUJ1YXbr777jtMmTIF3333HUaNGiV2OTYtwL0ZxoW1AgB8GJuCRjL3nIiIqFaihpuioiIkJycjOTkZAJCRkYHk5GRkZmYCqLykFBUVZWj/7bffIioqCsuXL0d4eDiys7ORnZ0NlYpzQizlpSHt4SCT4teMWzicmi92OURERA8karg5ceIEQkNDERoaCgCIjo5GaGioYVl3VlaWIegAwGeffYaKigrMmDEDPj4+htesWbNEqb8p8HV1xMTw1gCAZRy9ISIiK9Bo7nPTUHifG9PlFWox8IMDKCnX4bNnemF4F2+xSyIioiamydznhhqGh7Mc/+wfAABYEXcRen2TysNERGRlGG6oTl4Y2BbOcjtcyC7Ez6ezxC6HiIioRgw3VCeuTg7418C2AICVcRdRodOLXBEREVH1GG6ozqYOCIRbMwek52uw/eR1scshIiKqFsMN1VlzuR1eHNQOAPBx/CVoK3QiV0RERFQVww2Z5Jm+beDlIsf1ghJsPs5HWRARUePDcEMmUdjLMHNIewDAqgOpKCnj6A0RETUuDDdksvFh/mjVwhF5hVpsSrwsdjlERERGGG7IZA52UswaWjl6s+5QGtSl5SJXRERE9CeGG6qXsaF+aOvRDAXF5fjycIbY5RARERkw3FC92MmkiB4WDAD44pcM3NaUiVwRERFRJYYbqreRXX3QyccFRdoKrEtIE7scIiIiAAw39BCkUgnmDq8cvdl09DJy1aUiV0RERMRwQw9pSEdPhLZ2RWm5HqsPpIpdDhEREcMNPRyJRIJ5wzsAAL49nolrt4tFroiIiJo6hht6aP2C3NGvXUuU6wR8En9J7HKIiKiJY7ghs5hzZ/Tmh5PXkZ5XJHI1RETUlDHckFn0atMCQzp6QqcX8NE+jt4QEZF4GG7IbObcWTn1f7/fwPkstcjVEBFRU8VwQ2bTxVeJUd18AAAr4i6KXA0RETVVDDdkVq8MC4ZUAsSdy0Hy1QKxyyEioiaI4YbMKsizOcaGtgIALN+bInI1RETUFJkcbkpKSlBc/Oe9TK5cuYKVK1di7969Zi2MrNfsiPawl0nwy6V8HEu/KXY5RETUxJgcbp588kl8/fXXAICCggKEh4dj+fLlePLJJ7F27VqzF0jWx9/NCeN7+wMAlsWmQBAEkSsiIqKmxORwc/LkSTz66KMAgO+//x5eXl64cuUKvv76a3zyySdmL5Cs08zB7SG3k+LElds4eDFP7HKIiKgJMTncFBcXw9nZGQCwd+9ePPXUU5BKpXjkkUdw5coVsxdI1slbqcAzj7QBUDn3hqM3RETUUEwON0FBQdi5cyeuXr2K2NhYDB8+HACQm5sLFxcXsxdI1uvFx9qhmYMMZ66rsedMttjlEBFRE2FyuFm8eDHmzp2LgIAAhIeHo2/fvgAqR3FCQ0PNXiBZr5bN5Zg6IBBA5X1vdHqO3hARkeVJhHpcL8jOzkZWVhZCQkIglVbmo+PHj8PFxQUdO3Y0e5HmpFaroVQqoVKpONLUAFQl5Xj0/f1Ql1bgo/EhhmXiREREpjDl93e97nPj7e2N0NBQSKVSqNVq7Ny5E87Ozo0+2FDDUzra44VB7QAAH8VdQrlOL3JFRERk60wON+PGjcOqVasAVN7zJiwsDOPGjUP37t3xww8/mL1Asn5T+gfAvbkDMm8VY9uJa2KXQ0RENs7kcJOQkGBYCr5jxw4IgoCCggJ88skneOedd8xeIFk/Jwc7TH8sCADw6f5LKC3XiVwRERHZMpPDjUqlgpubGwBgz549+Otf/wonJyeMGjUKly5dMulYCQkJGD16NHx9fSGRSLBz584H7nPw4EH07NkTcrkcQUFB2Lhxo6ldIBFMDG8NH6UCWapS/PfXTLHLISIiG2ZyuPH390diYiI0Gg327NljWAp++/ZtKBQKk46l0WgQEhKC1atX16l9RkYGRo0ahcGDByM5ORmzZ8/Gc889h9jYWFO7QQ1MYS/DS0PaAwDWHEiFRlshckVERGSr7EzdYfbs2Zg0aRKaN2+ONm3a4LHHHgNQOQrTrVs3k441YsQIjBgxos7t161bh8DAQCxfvhwA0KlTJxw+fBgfffQRIiMjq91Hq9VCq9Uaflar1SbVSObz97BWWJ+Qhis3i7Hx6GXMGBwkdklERGSDTB65mT59OhITE/Hll1/i8OHDhqXgbdu2tficm8TERERERBhti4yMRGJiYo37xMTEQKlUGl7+/v4WrZFqZi+TYnZE5ejN+kNpUJWUi1wRERHZonotBQ8LC8PYsWPRrFkzw231R40ahf79+5u1uPtlZ2fDy8vLaJuXlxfUajVKSkqq3WfBggVQqVSG19WrVy1aI9XuiRA/tPdsDnVpBb74JV3scoiIyAbVK9x8/fXX6NatGxwdHeHo6Iju3bvjP//5j7lrMwu5XA4XFxejF4lHJpVgzvBgAMCXhzNws0j7gD2IiIhMY3K4WbFiBV588UWMHDkSW7duxdatW/H4449j2rRp+OijjyxRo4G3tzdycnKMtuXk5MDFxQWOjo4W/Wwyn8gu3ujmp4SmTIe1B9PELoeIiGyMyeHm008/xdq1a/H+++/jiSeewBNPPIEPPvgAa9aswSeffGKJGg369u2L+Ph4o21xcXGG51uRdZBI/hy9+frYFWSrSkWuiIiIbInJ4SYrKwv9+vWrsr1fv37Iysoy6VhFRUVITk5GcnIygMql3snJycjMrLwPyoIFCxAVFWVoP23aNKSnp+PVV1/FhQsXsGbNGmzduhWvvPKKqd0gkQ0K9kDvgBYoq9Dj0/2m3R+JiIioNiaHm6CgIGzdurXK9i1btqB9+/YmHevEiRMIDQ01PE08OjoaoaGhWLx4MYDKIHU36ABAYGAgdu3ahbi4OISEhGD58uX44osvalwGTo1X5ehNBwDAlt+uIvNmscgVERGRrTD5qeA//PADxo8fj4iICMPqqCNHjiA+Ph5bt27F2LFjLVKoufCp4I3LMxt+xS+X8vFUTz+sGNdD7HKIiKiRsuhTwf/617/i119/hbu7O3bu3ImdO3fC3d0dx48fb/TBhhqfu6M3O09dR2puocjVEBGRLTB55MbaceSm8fnX1ycQdy4Ho7r5YPWknmKXQ0REjZApv7/r9PgFUx5ZwMBAppozPBj7zudg1+ksvHhdha5+SrFLIiIiK1ancOPq6gqJRFJrG0EQIJFIoNPpzFIYNR0dvV0wursvfvr9BlbEXcSX/+wtdklERGTF6hRuDhw4YOk6qIl7ZVgwdp3Owv4LuUi6chu92rQQuyQiIrJSdQo3gwYNsnQd1MQFujfD33q2wpYTV7EsNgXfPf+I2CUREZGVqtezpYgs4eWI9nCQSZGYfhNHUvPFLoeIiKwUww01Gn6ujpjQxx8A8GFsCprYQj4iIjIThhtqVGYMCYLCXorkqwWIP58rdjlERGSFGG6oUfF0VmByvwAAwLK9KdDrOXpDRESmqVe4qaiowL59+7B+/XoUFlbeVfbGjRsoKioya3HUNE0b2A7OcjtcyC7E7jOmPYyViIjI5HBz5coVdOvWDU8++SRmzJiBvLw8AMD777+PuXPnmr1AanpaNHPAs48GAgBWxF1EhU4vckVERGRNTA43s2bNQlhYGG7fvg1HR0fD9rFjxyI+Pt6sxVHT9eyAQLRwskd6ngY7Tl0XuxwiIrIiJoebX375BYsWLYKDg4PR9oCAAFy/zl9CZB7OCntMG9QOAPBx/CWUVXD0hoiI6sbkcKPX66t9xMK1a9fg7OxslqKIACCqbwA8nOW4drsEW37LFLscIiKyEiaHm+HDh2PlypWGnyUSCYqKirBkyRKMHDnSnLVRE+foIMPMwUEAgE/3p6KkjM8tIyKiBzM53CxfvhxHjhxB586dUVpaiokTJxouSb3//vuWqJGasKf7+MPP1RG5hVr859hlscshIiIrIBHqcRvYiooKbN68GX/88QeKiorQs2dPTJo0yWiCcWOlVquhVCqhUqng4uIidjlUB1t/u4pXf/gDLZzskfDqYDgr7MUuiYiIGpgpv7/rFW6sGcON9anQ6TH8owSk52vwSkQwZkW0F7skIiJqYKb8/q7TU8Hv9dNPP1W7XSKRQKFQICgoCIGBgaYelqhGdjIpZg8LxsvfncIXv6Rjcr82cHVyePCORETUJJkcbsaMGQOJRFLloYZ3t0kkEgwYMAA7d+5EixYtzFYoNW1/6eaDNQdScSG7EOsT0vHa4x3FLomIiBopkycUx8XFoXfv3oiLi4NKpYJKpUJcXBzCw8Px888/IyEhATdv3uTdismspFIJ5gzvAADYeOQycgtLRa6IiIgaK5NHbmbNmoXPPvsM/fr1M2wbOnQoFAoFnn/+eZw9exYrV67E1KlTzVooUUQnT4T4u+L3qwVYcyANbz7RReySiIioETJ55CYtLa3aiTwuLi5IT08HALRv3x75+fkPXx3RPSQSCebdGb359tdMXC8oEbkiIiJqjEwON7169cK8efMMD8wEgLy8PLz66qvo3bs3AODSpUvw9/c3X5VEd/QPaonwQDeU6fT4NP6S2OUQEVEjZHK42bBhAzIyMtCqVSsEBQUhKCgIrVq1wuXLl/HFF18AAIqKirBo0SKzF0skkUgwL7Jy9GZb0jVk5GtEroiIiBqbet3nRq/XY+/evbh48SIAoEOHDhg2bBikUpOzUoPjfW5swz+/Oo6DKXl4socvPn46VOxyiIjIwngTv1ow3NiGM9dV+MunhyGRAHtmDUQHbz60lYjIlln0Jn4AoNFocOjQIWRmZqKsrMzovZdffrk+hyQySVc/JUZ09cb/zmRjRVwK1j8TJnZJRETUSJgcbk6dOoWRI0eiuLgYGo0Gbm5uyM/Ph5OTEzw9PRluqMFEDwvGnrPZiD2bgz+uFaB7K1exSyIiokbA5Ekyr7zyCkaPHo3bt2/D0dERx44dw5UrV9CrVy8sW7bMEjUSVau9lzPG9vADACzbe1HkaoiIqLEwOdwkJydjzpw5kEqlkMlk0Gq18Pf3xwcffICFCxfWq4jVq1cjICAACoUC4eHhOH78eK3tV65ciQ4dOsDR0RH+/v545ZVXUFrKO9Y2RbMjgmEnlSDhYh6OZ9wSuxwiImoETA439vb2hlVRnp6eyMzMBAAolUpcvXrV5AK2bNmC6OhoLFmyBCdPnkRISAgiIyORm5tbbftvv/0W8+fPx5IlS3D+/Hls2LABW7ZsqXewIuvWuqUTxvWuvKfSstiUKs88IyKipsfkcBMaGorffvsNADBo0CAsXrwY//3vfzF79mx07drV5AJWrFiBf/3rX5gyZQo6d+6MdevWwcnJCV9++WW17Y8ePYr+/ftj4sSJCAgIwPDhwzFhwoQHjvaQ7XppSBAc7KQ4fvkWEi7xzthERE2dyeHm3XffhY+PDwDg3//+N1q0aIEXX3wReXl5+Oyzz0w6VllZGZKSkhAREfFnQVIpIiIikJiYWO0+/fr1Q1JSkiHMpKenY/fu3Rg5cmS17bVaLdRqtdGLbIuP0hH/CG8DAFi+l6M3RERNnUmrpQRBgKenp2GExtPTE3v27Kn3h+fn50On08HLy8tou5eXFy5cuFDtPhMnTkR+fj4GDBgAQRBQUVGBadOm1XhZKiYmBkuXLq13jWQdpg9uh82/ZeKPayrEns3B4129xS6JiIhEYtLIjSAICAoKqtfcGnM5ePAg3n33XaxZswYnT57E9u3bsWvXLrz99tvVtl+wYAFUKpXhJWbtZDnuzeWY0j8AALAiLgU6PUdviIiaKpPCjVQqRfv27XHz5k2zfLi7uztkMhlycnKMtufk5MDbu/q/vN944w0888wzeO6559CtWzeMHTsW7777LmJiYqDX66u0l8vlcHFxMXqRbXr+0XZwVtjhYk4Rfv7jhtjlEBGRSEyec/Pee+9h3rx5OHPmzEN/uIODA3r16oX4+HjDNr1ej/j4ePTt27fafYqLi6s8w0omkwEA51o0cUone7wwsC0A4KO4iyjXVQ27RERk+0y+Q3FUVBSKi4sREhICBwcHODo6Gr1/65Zp9xqJjo7G5MmTERYWhj59+mDlypXQaDSYMmWK4fP8/PwQExMDABg9ejRWrFiB0NBQhIeHIzU1FW+88QZGjx5tCDnUdE3pH4ivjlzG5ZvF+CHpGp7u01rskoiIqIGZHG5Wrlxp1gLGjx+PvLw8LF68GNnZ2ejRowf27NljmGScmZlpNFKzaNEiSCQSLFq0CNevX4eHhwdGjx6Nf//732ati6xTM7kdXnysHd7ZdR6fxF/C2J5+kNsx9BIRNSV8KjjZnNJyHQZ9eAA5ai2WjO6MKf0DxS6JiIgekim/v02ecwMAaWlpWLRoESZMmGC4k/D//vc/nD17tj6HIzIrhb0MLw1pDwBYfSAVxWUVIldEREQNyeRwc+jQIXTr1g2//vortm/fjqKiIgDA77//jiVLlpi9QKL6GBfmD383R+QXlWHj0ctil0NERA3I5HAzf/58vPPOO4iLi4ODg4Nh+5AhQ3Ds2DGzFkdUXw52UsweGgwAWH8oHaqScpErIiKihmJyuDl9+jTGjh1bZbunpyfy8/lcH2o8xoT6IcizOVQl5dhwOEPscoiIqIGYHG5cXV2RlZVVZfupU6fg5+dnlqKIzEEmlSB6WOXozYZf0nFLUyZyRURE1BBMDjdPP/00XnvtNWRnZ0MikUCv1+PIkSOYO3cuoqKiLFEjUb093sUbXXxdoCnTYd2hNLHLISKiBlCvp4J37NgR/v7+KCoqQufOnTFw4ED069cPixYtskSNRPUmlUowd3gHAMCmo5eRoy4VuSIiIrK0et/nJjMzE2fOnEFRURFCQ0PRvn17c9dmEbzPTdMjCAL+ti4RSVdu45lH2uDtMV3FLomIiExk0fvcHD58GADQunVrjBw5EuPGjbOaYENNk0QiwZzhlXNvNv+Wiau3ikWuiIiILMnkcDNkyBAEBgZi4cKFOHfunCVqIjK7fu3c0T+oJcp1Aj6OvyR2OUREZEEmh5sbN25gzpw5OHToELp27YoePXrgww8/xLVr1yxRH5HZ3J17s/3kNaTmFolcDRERWYrJ4cbd3R0zZ87EkSNHkJaWhr///e/YtGkTAgICMGTIEEvUSGQWoa1bIKKTJ/QCsHLfRbHLISIiC6nXs6XuCgwMxPz58/Hee++hW7duOHTokLnqIrKI6GGVozc//5GFczfUIldDRESWUO9wc+TIEUyfPh0+Pj6YOHEiunbtil27dpmzNqKHtnr1agQEBEChUCA8PBxF1y7gL919AAAr4lKqtC8oKMCMGTPg4+MDuVyO4OBg7N692/D+m2++CYlEYvTq2LFjg/WHiIgezM7UHRYsWIDNmzfjxo0bGDZsGD7++GM8+eSTcHJyskR9RPW2ZcsWREdHY926dQgPD8fKlSsRGRmJ2CMnsft0Fvadz8XJzNvo2boFAKCsrAzDhg2Dp6cnvv/+e/j5+eHKlStwdXU1Om6XLl2wb98+w892diZ/jYiIyIJM/q9yQkIC5s2bh3HjxsHd3d0SNRGZxYoVK/Cvf/0LU6ZMAQCsW7cOu3btwv6ftuCvPUdgW9I1LN+bgv8+9wgA4Msvv8StW7dw9OhR2NvbAwACAgKqHNfOzg7e3t4N1g8iIjKNyZel7l6OYrChxqysrAxJSUmIiIgwbJNKpYiIiEBiYiJeHtoe9jIJjqTexNG0yge+/vTTT+jbty9mzJgBLy8vdO3aFe+++y50Op3RsS9dugRfX1+0bdsWkyZNQmZmZoP2jYiIalfv8fRz584hMzMTZWXGDyN84oknHrooooeVn58PnU4HLy8vo+1eXl64cOEC/N2c8HTv1vjPsStYFpuCH15sifT0dOzfvx+TJk3C7t27kZqaiunTp6O8vBxLliwBAISHh2Pjxo3o0KEDsrKysHTpUjz66KM4c+YMnJ2dxegqERHdx+Rwk56ejrFjx+L06dOQSCS4+/QGiUQCAFX+yiVqrGYOCcLWE1dxMrMAB1Jyodfr4enpic8++wwymQy9evXC9evX8eGHHxrCzYgRIwz7d+/eHeHh4WjTpg22bt2KZ599VqyuEBHRPUy+LDVr1iwEBgYiNzcXTk5OOHv2LBISEhAWFoaDBw9aoEQi07m7u0MmkyEnJ8doe05OjmG+jJeLApP7BQAAlsVehI+PD4KDgyGTyQztO3XqhOzs7CojlHe5uroiODgYqamplukIERGZzORwk5iYiLfeegvu7u6QSqWQSqUYMGAAYmJi8PLLL1uiRiKTOTg4oFevXoiPjzds0+v1iI+PR9++fQ3bpg1qh+ZyO5zLUsO7fQhSU1Oh1+sN71+8WBl6HBwcqv2coqIipKWlwcfHx3KdISIik5gcbnQ6nWFugbu7O27cuAEAaNOmDVJSqt43hEgs0dHR+Pzzz7Fp0yacP38eL774IjQajWH1VFRUFD58ZwmmDggEAFz3eRS3bt3CrFmzcPHiRezatQvvvvsuZsyYYTjm3LlzcejQIVy+fBlHjx7F2LFjIZPJMGHCBFH6SEREVZk856Zr1674/fffERgYiPDwcHzwwQdwcHDAZ599hrZt21qiRqJ6GT9+PPLy8rB48WJkZ2ejR48e2LNnj2GScWZmJqRSKRY8GohNRy/jWokTZr7/JX5c/x4+//xz+Pn5YdasWXjttdcMx7x27RomTJiAmzdvwsPDAwMGDMCxY8fg4eEhVjeJiOg+EuHujOA6io2NhUajwVNPPYXU1FT85S9/wcWLF9GyZUts2bKl0T9fSq1WQ6lUQqVSwcXFRexyqJFYezAN7++5YPhZKgFinuqG8b1bi1gVERHdZcrvb5PDTXVu3bqFFi1aGFZMNWYMN1Sd9LwiDFlu/Gw0mQQ4PH8IfJSOIlVFRER3mfL7+6EenHmXm5ubVQQboppkq0urbNMJwLu7L+BU5m3o9Q/9NwARETUQPhSHCECgezNIJcD9Geb/fr+B//v9Brxc5BjW2QuRXbzxSNuWsJeZ5e8CIiKyALNclrImvCxFNdnyWyYWbj8DnSBAKgHG924NdWk5Dl7Ihabsz5tTuijsMLSTFyK7eGFgsAecHPg3AhGRpTX4nBtrwnBDtclSleByfjEC3J0Mc21Ky3VITLuJ2LPZiDuXg5uaP2/oJ7eTYmCwByK7eGNoR0+0aFb9/XCIiOjhMNzUguGGHoZOLyDpym3Ens1G7NlsXLtdYnhPJpWgT4AbIrt4YXgXb/i6ciIyEZG5MNzUguGGzEUQBJzLUmPv2RzEns3GhexCo/e7t1Iisos3Irt4IciTD9UkInoYDDe1YLghS7lyU2MIOkmZt3HvN6utRzMM71wZdEJauUIq5epCIiJTNPhS8Ie1evVqBAQEQKFQIDw8HMePH6+1fUFBAWbMmAEfHx/I5XIEBwdj9+7dDVQtUfXatGyGfw1si+9f7IdfFw7Fu2O7YVCwB+xlEqTnabDuUBrGrjmKfu/txxs7z+DwpXyU6/QPPjAREZlE9JGbLVu2ICoqCuvWrUN4eDhWrlyJbdu2ISUlBZ6enlXal5WVoX///vD09MTChQvh5+eHK1euwNXVFSEhIQ/8PI7cUEMrLC3HgZQ8xJ7NrrLySuloj6EdPTGcK6+IiGplVZelwsPD0bt3b6xatQpA5ZOb/f398dJLL2H+/PlV2q9btw4ffvghLly4AHt7e5M/j+GGxFRarsPRtHzEnsnBvvPGK68U9lI82r5y5VVEJ0+4OnHlFRHRXVYTbsrKyuDk5ITvv/8eY8aMMWyfPHkyCgoK8OOPP1bZZ+TIkXBzc4OTkxN+/PFHeHh4YOLEiXjttdcgk8mqtNdqtdBqtYaf1Wo1/P39GW5IdA9aeRUe6IbILt4Y3sWLj4AgoibPlHAj6hh4fn4+dDqd4SnNd3l5eeHChQvV7pOeno79+/dj0qRJ2L17N1JTUzF9+nSUl5djyZIlVdrHxMRg6dKlFqmf6GHIpBL0CXRDn0A3LBrVCeey1Ig9m4O9d1ZeHU27iaNpN7Hkp7NceUVEZAJRR25u3LgBPz8/HD16FH379jVsf/XVV3Ho0CH8+uuvVfYJDg5GaWkpMjIyDCM1K1aswIcffoisrKwq7TlyQ9boyk3NnRGdHJysZuVVZdDxRnc/JVdeEVGTYDUjN+7u7pDJZMjJyTHanpOTA29v72r38fHxgb29vdElqE6dOiE7OxtlZWVwcDCepyCXyyGXy81fPJEFtWnZDM8PbIfnB7ZDbmEp9p3LRezZbBxNy0d6ngZrD6Zh7cE0eLsoMLyLF4Z39kZ4Wzc+84qICCKHGwcHB/Tq1Qvx8fGGOTd6vR7x8fGYOXNmtfv0798f3377LfR6PaTSyv+QX7x4ET4+PlWCDZEt8HRWYGJ4a0wMr3zW1YELudh7NgcHU3KRrS7F14lX8HXilXtWXnljULAHHB2qzkEjImoKRF8ttWXLFkyePBnr169Hnz59sHLlSmzduhUXLlyAl5cXoqKi4Ofnh5iYGADA1atX0aVLF0yePBkvvfQSLl26hKlTp+Lll1/G66+//sDP42opshUPWnk18M7Kq6FceUVENsBqLksBwPjx45GXl4fFixcjOzsbPXr0wJ49ewyTjDMzMw0jNADg7++P2NhYvPLKK+jevTv8/Pwwa9YsvPbaa2J1gUgUCnsZhnT0wpCOXtDpBZy4fAuxd+6QfL2gBHvP5WDvuRzIpBI80tYNwztz5RURNQ2ij9w0NI7ckK0TBAFnb6ix92w29p7LqfLMq5BWSgy/MyE5yLO5SFUSEZnGau5zIwaGG2pqLudrsPdc9Suv2t278qqVEhIJV14RUePEcFMLhhtqynILSxF3LgexZ3OQmJaPct2fX/+7K68iu3ijTyBXXhFR48JwUwuGG6JK9668OpCSi+L7n3nVyRORXbwxsD1XXhGR+BhuasFwQ1RVabkOR1LzEXs2G/vO5+LWfSuvBgV7YHhnrrwiIvEw3NSC4YaodhU6PU5cuY2996y8uuvuyqvILt4Y3tkb3kqFiJUSUVPCcFMLhhuiurt35VXs2Ryk5Ny38srfFZF37pDMlVdEZEmm/P7mjEEiqpFEIkFXPyWih3dA7CsDcXDuY1gwoiN6tWkBiQT4/WoBPtiTgogVhzB0+UF8sOcCfr9aAHP+zbR69WoEBARAoVAgPDwcx48fr9N+mzdvhkQiMdz9/K6ioiLMnDkTrVq1gqOjIzp37ox169aZrV4iEh9HboioXnLVpYg7X/3KKx+lAsM7/7nyyq6eK6+2bNmCqKgorFu3DuHh4Vi5ciW2bduGlJQUeHp61rjf5cuXMWDAALRt2xZubm7YuXOn4b3nn38e+/fvxxdffIGAgADs3bsX06dPx/bt2/HEE0/Uq04isjxelqoFww2R+d1deRV7NhsHU/KMVl65OtljSMf6rbwKDw9H7969sWrVKgCVz57z9/fHSy+9hPnz51e7j06nw8CBAzF16lT88ssvKCgoMAo3Xbt2xfjx4/HGG28YtvXq1QsjRozAO++8Y2LPiaih8LIUETUoF4U9nuzhhzWTeuHkG8OwYXIYxoW1glszBxQUl2P7yet44T9J6Pl2HF74zwlsP3kNquLyWo9ZVlaGpKQkREREGLZJpVJEREQgMTGxxv3eeusteHp64tlnn632/X79+uGnn37C9evXIQgCDhw4gIsXL2L48OH16zwRNTqiP1uKiGyLwl6GoZ28MLSTl2HlVezZbOw9m4PrBSV3nn+VAzupBI+0bYnILl4YVs3Kq/z8fOh0OsNz5u7y8vLChQsXqv3sw4cPY8OGDUhOTq6xvk8//RTPP/88WrVqBTs7O0ilUnz++ecYOHDgQ/ediBoHhhsishg7mRSPtG2JR9q2xOK/dK6y8upwaj4Op+bjjR/PGlZeRXbxRjsP01deFRYW4plnnsHnn38Od3f3Gtt9+umnOHbsGH766Se0adMGCQkJmDFjBnx9fY1GiYjIenHODRGJIiNfcyfoZONkZoHRe0GezTE0uAXeGNMT27Ztw9ixYw3vTZ48GQUFBfjxxx+N9klOTkZoaChksj/n9Oj1egCVl7NSUlLg6+sLpVKJHTt2YNSoUYZ2zz33HK5du4Y9e/ZYoKdEZA6m/P7myA0RiSLQvRleGNQOLwxqh1x1Kfaeq7xpYGLaTaTmFiE1twgyz3aY/sEm/C4LxvDOXghr44r4+HjMnDmzyvE6duyI06dPG21btGgRCgsL8fHHH8Pf3x+lpaUoLy+HVGo83VAmkxmCEBFZP4YbIhKdp4sC/3ikDf7xSBuoSspxMKVy5dVPfZ/C9R+XY9X6tvjcJxilyf8HTYEagX1HobRch+efnQI/Pz/ExMRAoVCga9euRsd1dXUFAMN2BwcHDBo0CPPmzYOjoyPatGmDQ4cO4euvv8aKFSsauttEZCEMN0TUqCgdK1dePdnDDyvG9cDcN53x9WercOt2Puw9A+H21Jt47X+ZeHPfdah+O4uOBSVQFZdD6WSPLFUJMvI1CHRvBh+lY7XH37x5MxYsWIBJkybh1q1baNOmDf79739j2rRpDdtRIrIYzrkhIqtQ3cqru+ykEgS4N0NabhEEAFIJEPNUN4zv3Vq8gonIrHgTv1ow3BBZv7vPvIq9MyH5Yk5Rte1GdPVGVz8lAt2bIdC9GQJaNjPpJoJE1Hgw3NSC4YbI9mw/eQ3RW3+vU1tfpQKBHs3uBJ7maHsn+LRq4Vjvx0QQkeVxtRQRNSl927WEVALo7/lTTSoBnh/YFnmFZUjPL0J6ngaqknLcUJXihqoUR1JvGh3DXiaBv5sT2ro3Q1uP5obRnrbuzeDhLIdEImngXhFRfTHcEJHV81E6Iuapbli4/Qx0ggCZRIJ3n+paZc7NbU0Z0vM1yMjXICO/CBn5GqTnaXD5pgal5Xqk51X+jPO5Rvs1c5DdGe1pbgg8ge7NEOjRDC4K+4bsKhHVAS9LEZHNyFKV4HJ+MQLcnWpcLVUdvV5Atrq0Muzka5CRVxl+0vM1uHqr2GhE6H7uzR3uBJ7mhstdbd2boXVLJ8jtOL+HyFw456YWDDdEZIqyCj0ybxVXGe3JyNcgt1Bb435SCeDXwtFoXs/dl6+rI2RSXuYiMgXDTS0YbojIXIq0Fbh8z2hP+p3wk5GnQaG2osb9HOykCGjpVDnK42F8qcutmYPNze9ZvXo1PvzwQ2RnZyMkJASffvop+vTp88D9Nm/ejAkTJuDJJ5/Ezp07DdsFQcCSJUvw+eefo6CgAP3798fatWvRvn17C/aCxMZwUwuGGyKyNEEQkF9UZhjt+fNSlwZXbhajTFfzox5cFHYI9Kg62hPo3gzN5NY3TXLLli2IiorCunXrEB4ejpUrV2Lbtm1ISUmBp6dnjftdvnwZAwYMQNu2beHm5mYUbt5//33ExMRg06ZNCAwMxBtvvIHTp0/j3LlzUCgUNR6TrBvDTS0YbohITDq9gBsFJXcCT5Fhnk96ngY3VCWo7b/IXi5yw2jPveHH380J9o10GXt4eDh69+6NVatWAah8mKm/vz9eeuklzJ8/v9p9dDodBg4ciKlTp+KXX35BQUGBIdwIggBfX1/MmTMHc+fOBQCoVCp4eXlh48aNePrppxukX9TwuBSciKiRkkkrl5z7uzlhULCH0Xul5TpcuVlcZbQnI1+Dm5oy5Ki1yFFrcSz9VpVjtnZzMhrlaXtnNZe3i0K0y1xlZWVISkrCggULDNukUikiIiKQmJhY435vvfUWPD098eyzz+KXX34xei8jIwPZ2dmIiIgwbFMqlQgPD0diYiLDDQFguCEiajQU9jJ08HZGB2/nKu+pisuRcfPOpOY8DdLuCT8l5TpDCLqfo70MAXfCTluPe8NPcyidLLuMPT8/HzqdDl5eXkbbvby8cOHChWr3OXz4MDZs2IDk5ORq38/OzjYc4/5j3n2PiOGGiMgKKJ3s0cPJFT38XY22C4KAHLXWaDLz3aCTeasYJeU6nM9S43yWusox3Zo5VDvaE9CyGRT2Db+MvbCwEM888ww+//xzuLu7N/jnk+1guCEismISiQTeSgW8lQr0a2ccCMp1ely7XVJ5metO6Ln7v9nqUtzSlOGWpgxJV27fd0zAV+n4Z+jx+HO0x69F3Zexu7u7QyaTIScnx2h7Tk4OvL29q7RPS0vD5cuXMXr0aMM2vb5y8rWdnR1SUlIM++Xk5MDHx8fomD169KhTXWT7GG6IiGyUvUxqCChDOhq/p9FW4PJNjdFoT+XE5iKoSytwvaAE1wtKcDg132g/B5kUre8uY793NZdHM3g0N35MhYODA3r16oX4+HiMGTMGQGVYiY+Px8yZM6vU27FjR5w+fdpo26JFi1BYWIiPP/4Y/v7+sLe3h7e3N+Lj4w1hRq1W49dff8WLL7748P/SyCYw3BARNUHN5Hbo4qtEF1+l0XZBEHC7uLza0Z6MmxqUVeiRmluE1NyqT2JvLrerMtrzt8nT8Eb0iwgLC0OfPn2wcuVKaDQaTJkyBQAQFRUFPz8/xMTEQKFQoGvXrkbHdHV1BQCj7bNnz8Y777yD9u3bG5aC+/r6GgIUUaMIN+a+wRMREdWPRCKBWzMHuDVzQ682bkbv6fUCbqhKDHN60u+Z33PtdjGKtBU4fV2F09dV9+zlCadH/4nnZ72KCs1teAV0wAvvfoHf8wW0lRTi8pUrkEr/XMaedef4ge7NanyExquvvgqNRoPnn38eBQUFGDBgAPbs2cN73JCB6Pe5scQNnmrD+9wQEZmftkKHq7eKq4z2pOdrkF9U+2MqWrWovMxVrtMjMe0mBFTO+1n8l874Z78Am7tjM9WPVd3Ez9w3eLqfVquFVvvnF0utVsPf35/hhoiogahLy3G5mtGejHwNimp5TAUAOMgk8FY6wttFAS+lAt4ucni5KODlUjmJ2ttFAQ9nuSiru5oKU66ubN++He+++y5SU1NRXl6O9u3bY86cOXjmmWcMbf75z39i06ZNRvtFRkZiz549tdZhNTfxs8QNnu4XExODpUuXmq1mIiIyjYvCHt1buaJ7K1ej7YIgIK9Ii4w8Dfadz8Hnv2RU2bdMJyDzVjEybxXX+hktnOwNgcfL+W4QUsBbKYenc+V2NycHSPnAUpNs2bIF0dHRRldXIiMja7y64ubmhtdffx0dO3aEg4MDfv75Z0yZMgWenp6IjIw0tHv88cfx1VdfGX6Wy+VmrVvUcGOJGzzdb8GCBYiOjjb8fHfkhoiIxCWRSODprICnswKtWzphw+EM6O+5liCVAFtf6AsBQLaqFDnq0sr/LdQiR1WKbHXlq6xCj9vF5bhdXI4L2YU1fp69rPLzvFzklSHIpTIA3T8S5OjAUaC7VqxYgX/961+GCeDr1q3Drl278OWXX1Z7deWxxx4z+nnWrFnYtGkTDh8+bBRu5HJ5tbcDMJdGMaG4rupzgye5XG72REhEROblo3REzFPdsHD7GegEATKJBO8+1RVhAW617icIAlQl5ZVB504AylFrka0uNQSgHLUWNzValOsEwxL32jgr7O6M+twNPnJDCLq7zb25vM73+7FW9b26cpcgCNi/fz9SUlLw/vvvG7138OBBeHp6okWLFhgyZAjeeecdtGzZ0my1ixpuLHGDp3bt2lm2aCIisojxvVtjYLAHLucXI8DdqcbVUveSSCRwdXKAq5MDOnrXPA+jXKdHbqEW2apS5Kr/HPXJUZXeeWZX5c/FZToUllagsLQIl6pZ7n6XTCqBR3M5vJQKeDlXHQnyVlbODXJWWPYRF5ZUn6srQOWDTP38/KDVaiGTybBmzRoMGzbM8P7jjz+Op556CoGBgUhLS8PChQsxYsQIJCYmQiYzz6iZqOHGEjd4IiIi6+WjdKxTqDGVvUwKP1dH+LnWfGxBEFCoragMP6o7oz/qey6H3QlAeYVa6PSCISDVppmDzOiyl9FI0D0TohvrU93rw9nZGcnJySgqKkJ8fDyio6PRtm1bwyWrex9u2q1bN3Tv3h3t2rXDwYMHMXToULPUIPplqejoaEyePNmsN3giIiIylUQigYvCHi4KewR5Vn146V06vYD8ospRoGz1PSNBKu2fYUhdisLSCmjKdJV3fq7moaZ/fi7Qspm8crTn3snQLgp43pkf5O2igNLRvkGXxZt6deUuqVSKoKAgAECPHj1w/vx5xMTEVJmPc1fbtm3h7u6O1NRU2wk348ePR15eHhYvXozs7Gz06NEDe/bsMQyDZWZmGt3giYiISEwyqcQwGhNSS7visoo7Iz5/XvbKVpUit7DUaHvFnbCUX6TFGVR9wOldcjvpn5e+GmBZvKlXV2qi1+uNbslyv2vXruHmzZtGzwp7WKLf56ah8SZ+RETUWOj1Am4Vlxld9spRa++ZDF35ul1cXudjmnNZ/JYtWzB58mSsX7/ecHVl69atuHDhAry8vIyurgCVt18JCwtDu3btoNVqsXv3bsyfPx9r167Fc889h6KiIixduhR//etf4e3tjbS0NLz66qsoLCzE6dOna10AZDX3uSEiImrKpFIJ3JvL4d5cjq5+yhrblZbrkKuuOg+o8rKY1mLL4k29uqLRaDB9+nRcu3YNjo6O6NixI7755huMHz8eACCTyfDHH39g06ZNKCgogK+vL4YPH463337brCubOXJDRERkAwRBQEFxOXIK75kArdIip7DUaCQov6iszsesz7L4ujwfrD44ckNERNTESCQStGjmgBbNal8WX1ahR96dCdF/3hjR+L5A2apSlJSbviy+vEKP81lqCKi8CWPMU90wvndrC/S2dgw3RERETYiDXd2Xxd8beIwvhT14WbxeABZuP4OBwR4WWd5fG4YbIiIiMnLvsvj2XjUvi6/Q6ZFfVIYcdSkOXczDiriLRu/rBAGX84sbPNxwjTURERHVi51MCm+lAiH+rvh7WCvcvwBLJpEgwN2pwetiuCEiIqKHdvf5YLI7Nxq8+3ywhh61AXhZioiIiMykPs8HswSGGyIiIjIbSz0fzBS8LEVEREQ2heGGiIiIbArDDREREdkUhhsiIiKyKQw3REREZFMYboiIiMimMNwQERGRTWG4ISIiIpvCcENEREQ2heGGiIiIbArDDREREdkUhhsiIiKyKQw3REREZFMYboiIiMimMNwQERGRTWG4ISIiIpvCcENEREQ2heGGiIiIbArDDREREdkUhhsiIiKyKQw3REREZFMYboiIiMimNIpws3r1agQEBEChUCA8PBzHjx+vse3nn3+ORx99FC1atECLFi0QERFRa3siIiJqWkQPN1u2bEF0dDSWLFmCkydPIiQkBJGRkcjNza22/cGDBzFhwgQcOHAAiYmJ8Pf3x/Dhw3H9+vUGrpyIiIgaI4kgCIKYBYSHh6N3795YtWoVAECv18Pf3x8vvfQS5s+f/8D9dTodWrRogVWrViEqKuqB7dVqNZRKJVQqFVxcXB66fiIiIrI8U35/izpyU1ZWhqSkJERERBi2SaVSREREIDExsU7HKC4uRnl5Odzc3Kp9X6vVQq1WG72IiIjIdokabvLz86HT6eDl5WW03cvLC9nZ2XU6xmuvvQZfX1+jgHSvmJgYKJVKw8vf3/+h6yYiIqLGS/Q5Nw/jvffew+bNm7Fjxw4oFIpq2yxYsAAqlcrwunr1agNXSURERA3JTswPd3d3h0wmQ05OjtH2nJwceHt717rvsmXL8N5772Hfvn3o3r17je3kcjnkcrlZ6iUiIqLGT9SRGwcHB/Tq1Qvx8fGGbXq9HvHx8ejbt2+N+33wwQd4++23sWfPHoSFhTVEqURERGQlRB25AYDo6GhMnjwZYWFh6NOnD1auXAmNRoMpU6YAAKKiouDn54eYmBgAwPvvv4/Fixfj22+/RUBAgGFuTvPmzdG8eXPR+kFERESNg+jhZvz48cjLy8PixYuRnZ2NHj16YM+ePYZJxpmZmZBK/xxgWrt2LcrKyvC3v/3N6DhLlizBm2++2ZClExERUSMk+n1uGhrvc0NERGR9rOY+N0RERETmxnBDRERENoXhhoiIiGwKww0RERHZFIYbIiIisikMN0RERGRTGG6IiIjIpjDcEBERkU1huCEiIiKbwnBDRERENoXhhoiIiGwKww0RERHZFIYbIiIisikMN0RERGRTGG6IiIjIpjDcEBERkU1huCEiIiKbwnBDRERENoXhhoiIiGwKww0RERHZFIYbIiIisikMN0RERGRTGG6IiIjIpjDcEBERkU1huCEiIiKbwnBDRERENoXhhoiIiGwKww0RERHZFIYbIiIisikMN0RERGRTGG6IiIjIpjDcEBERkU1pFOFm9erVCAgIgEKhQHh4OI4fP15r+23btqFjx45QKBTo1q0bdu/e3UCVEhERUWMnerjZsmULoqOjsWTJEpw8eRIhISGIjIxEbm5ute2PHj2KCRMm4Nlnn8WpU6cwZswYjBkzBmfOnGngyomIiKgxkgiCIIhZQHh4OHr37o1Vq1YBAPR6Pfz9/fHSSy9h/vz5VdqPHz8eGo0GP//8s2HbI488gh49emDdunUP/Dy1Wg2lUgmVSgUXFxfzdYSIiIgsxpTf33YNVFO1ysrKkJSUhAULFhi2SaVSREREIDExsdp9EhMTER0dbbQtMjISO3furLa9VquFVqs1/KxSqQBU/ksiIiIi63D393ZdxmREDTf5+fnQ6XTw8vIy2u7l5YULFy5Uu092dna17bOzs6ttHxMTg6VLl1bZ7u/vX8+qiYiISCyFhYVQKpW1thE13DSEBQsWGI306PV63Lp1Cy1btoREIjHrZ6nVavj7++Pq1as2ecnL1vsH2H4f2T/rZ+t9ZP+sn6X6KAgCCgsL4evr+8C2ooYbd3d3yGQy5OTkGG3PycmBt7d3tft4e3ub1F4ul0Mulxttc3V1rX/RdeDi4mKz/6cFbL9/gO33kf2zfrbeR/bP+lmijw8asblL1NVSDg4O6NWrF+Lj4w3b9Ho94uPj0bdv32r36du3r1F7AIiLi6uxPRERETUtol+Wio6OxuTJkxEWFoY+ffpg5cqV0Gg0mDJlCgAgKioKfn5+iImJAQDMmjULgwYNwvLlyzFq1Chs3rwZJ06cwGeffSZmN4iIiKiRED3cjB8/Hnl5eVi8eDGys7PRo0cP7NmzxzBpODMzE1LpnwNM/fr1w7fffotFixZh4cKFaN++PXbu3ImuXbuK1QUDuVyOJUuWVLkMZitsvX+A7feR/bN+tt5H9s/6NYY+in6fGyIiIiJzEv0OxURERETmxHBDRERENoXhhoiIiGwKww0RERHZFIYbE61evRoBAQFQKBQIDw/H8ePHa22/bds2dOzYEQqFAt26dcPu3bsbqNL6MaV/GzduhEQiMXopFIoGrNY0CQkJGD16NHx9fSGRSGp8Htm9Dh48iJ49e0IulyMoKAgbN260eJ0Pw9Q+Hjx4sMo5lEgkNT7OREwxMTHo3bs3nJ2d4enpiTFjxiAlJeWB+1nTd7A+fbSm7+HatWvRvXt3w83d+vbti//973+17mNN5w8wvY/WdP6q895770EikWD27Nm1tmvo88hwY4ItW7YgOjoaS5YswcmTJxESEoLIyEjk5uZW2/7o0aOYMGECnn32WZw6dQpjxozBmDFjcObMmQauvG5M7R9QeQfKrKwsw+vKlSsNWLFpNBoNQkJCsHr16jq1z8jIwKhRozB48GAkJydj9uzZeO655xAbG2vhSuvP1D7elZKSYnQePT09LVRh/R06dAgzZszAsWPHEBcXh/LycgwfPhwajabGfaztO1ifPgLW8z1s1aoV3nvvPSQlJeHEiRMYMmQInnzySZw9e7ba9tZ2/gDT+whYz/m732+//Yb169eje/futbYT5TwKVGd9+vQRZsyYYfhZp9MJvr6+QkxMTLXtx40bJ4waNcpoW3h4uPDCCy9YtM76MrV/X331laBUKhuoOvMCIOzYsaPWNq+++qrQpUsXo23jx48XIiMjLViZ+dSljwcOHBAACLdv326QmswpNzdXACAcOnSoxjbW9h28X136aM3fQ0EQhBYtWghffPFFte9Z+/m7q7Y+Wuv5KywsFNq3by/ExcUJgwYNEmbNmlVjWzHOI0du6qisrAxJSUmIiIgwbJNKpYiIiEBiYmK1+yQmJhq1B4DIyMga24upPv0DgKKiIrRp0wb+/v4P/OvE2ljT+XtYPXr0gI+PD4YNG4YjR46IXU6dqFQqAICbm1uNbaz9HNalj4B1fg91Oh02b94MjUZT4+NzrP381aWPgHWevxkzZmDUqFFVzk91xDiPDDd1lJ+fD51OZ7hz8l1eXl41zk/Izs42qb2Y6tO/Dh064Msvv8SPP/6Ib775Bnq9Hv369cO1a9caomSLq+n8qdVqlJSUiFSVefn4+GDdunX44Ycf8MMPP8Df3x+PPfYYTp48KXZptdLr9Zg9ezb69+9f693Jrek7eL+69tHavoenT59G8+bNIZfLMW3aNOzYsQOdO3eutq21nj9T+mht5w8ANm/ejJMnTxoei/QgYpxH0R+/QNarb9++Rn+N9OvXD506dcL69evx9ttvi1gZ1VWHDh3QoUMHw8/9+vVDWloaPvroI/znP/8RsbLazZgxA2fOnMHhw4fFLsVi6tpHa/sedujQAcnJyVCpVPj+++8xefJkHDp0qMZf/tbIlD5a2/m7evUqZs2ahbi4uEY98Znhpo7c3d0hk8mQk5NjtD0nJwfe3t7V7uPt7W1SezHVp3/3s7e3R2hoKFJTUy1RYoOr6fy5uLjA0dFRpKosr0+fPo06NMycORM///wzEhIS0KpVq1rbWtN38F6m9PF+jf176ODggKCgIABAr1698Ntvv+Hjjz/G+vXrq7S11vNnSh/v19jPX1JSEnJzc9GzZ0/DNp1Oh4SEBKxatQparRYymcxoHzHOIy9L1ZGDgwN69eqF+Ph4wza9Xo/4+Pgar6X27dvXqD0AxMXF1XrtVSz16d/9dDodTp8+DR8fH0uV2aCs6fyZU3JycqM8h4IgYObMmdixYwf279+PwMDAB+5jbeewPn28n7V9D/V6PbRabbXvWdv5q0ltfbxfYz9/Q4cOxenTp5GcnGx4hYWFYdKkSUhOTq4SbACRzqPFpirboM2bNwtyuVzYuHGjcO7cOeH5558XXF1dhezsbEEQBOGZZ54R5s+fb2h/5MgRwc7OTli2bJlw/vx5YcmSJYK9vb1w+vRpsbpQK1P7t3TpUiE2NlZIS0sTkpKShKefflpQKBTC2bNnxepCrQoLC4VTp04Jp06dEgAIK1asEE6dOiVcuXJFEARBmD9/vvDMM88Y2qenpwtOTk7CvHnzhPPnzwurV68WZDKZsGfPHrG68ECm9vGjjz4Sdu7cKVy6dEk4ffq0MGvWLEEqlQr79u0Tqws1evHFFwWlUikcPHhQyMrKMryKi4sNbaz9O1ifPlrT93D+/PnCoUOHhIyMDOGPP/4Q5s+fL0gkEmHv3r2CIFj/+RME0/toTeevJvevlmoM55HhxkSffvqp0Lp1a8HBwUHo06ePcOzYMcN7gwYNEiZPnmzUfuvWrUJwcLDg4OAgdOnSRdi1a1cDV2waU/o3e/ZsQ1svLy9h5MiRwsmTJ0Woum7uLnu+/3W3T5MnTxYGDRpUZZ8ePXoIDg4OQtu2bYWvvvqqwes2hal9fP/994V27doJCoVCcHNzEx577DFh//794hT/ANX1C4DRObH272B9+mhN38OpU6cKbdq0ERwcHAQPDw9h6NChhl/6gmD9508QTO+jNZ2/mtwfbhrDeZQIgiBYblyIiIiIqGFxzg0RERHZFIYbIiIisikMN0RERGRTGG6IiIjIpjDcEBERkU1huCEiIiKbwnBDRERENoXhhoiIiGwKww0RNXkHDx6ERCJBQUGB2KUQkRkw3BAREZFNYbghIiIim8JwQ0Si0+v1iImJQWBgIBwdHRESEoLvv/8ewJ+XjHbt2oXu3btDoVDgkUcewZkzZ4yO8cMPP6BLly6Qy+UICAjA8uXLjd7XarV47bXX4O/vD7lcjqCgIGzYsMGoTVJSEsLCwuDk5IR+/fohJSXFsh0nIotguCEi0cXExODrr7/GunXrcPbsWbzyyiv4xz/+gUOHDhnazJs3D8uXL8dvv/0GDw8PjB49GuXl5QAqQ8m4cePw9NNP4/Tp03jzzTfxxhtvYOPGjYb9o6Ki8N133+GTTz7B+fPnsX79ejRv3tyojtdffx3Lly/HiRMnYGdnh6lTpzZI/4nIvPhUcCISlVarhZubG/bt24e+ffsatj/33HMoLi7G888/j8GDB2Pz5s0YP348AODWrVto1aoVNm7ciHHjxmHSpEnIy8vD3r17Dfu/+uqr2LVrF86ePYuLFy+iQ4cOiIuLQ0RERJUaDh48iMGDB2Pfvn0YOnQoAGD37t0YNWoUSkpKoFAoLPxvgYjMiSM3RCSq1NRUFBcXY9iwYWjevLnh9fXXXyMtLc3Q7t7g4+bmhg4dOuD8+fMAgPPnz6N///5Gx+3fvz8uXboEnU6H5ORkyGQyDBo0qNZaunfvbvhnHx8fAEBubu5D95GIGpad2AUQUdNWVFQEANi1axf8/PyM3pPL5UYBp74cHR3r1M7e3t7wzxKJBEDlfCAisi4cuSEiUXXu3BlyuRyZmZkICgoyevn7+xvaHTt2zPDPt2/fxsWLF9GpUycAQKdOnXDkyBGj4x45cgTBwcGQyWTo1q0b9Hq90RweIrJdHLkhIlE5Oztj7ty5eOWVV6DX6zFgwACoVCocOXIELi4uaNOmDQDgrbfeQsuWLeHl5YXXX38d7u7uGDNmDABgzpw56N27N95++22MHz8eiYmJWLVqFdasWQMACAgIwOTJkzF16lR88sknCAkJwZUrV5Cbm4tx48aJ1XUishCGGyIS3dtvvw0PDw/ExMQgPT0drq6u6NmzJxYuXGi4LPTee+9h1qxZuHTpEnr06IH/+7//g4ODAwCgZ8+e2Lp1KxYvXoy3334bPj4+eOutt/DPf/7T8Blr167FwoULMX36dNy8eROtW7fGwoULxeguEVkYV0sRUaN2dyXT7du34erqKnY5RGQFOOeGiIiIbArDDREREdkUXpYiIiIim8KRGyIiIrIpDDdERERkUxhuiIiIyKYw3BAREZFNYbghIiIim8JwQ0RERDaF4YaIiIhsCsMNERER2ZT/B8tt/1iJ6d/9AAAAAElFTkSuQmCC", - "text/plain": [ - "

" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "y_avg = []\n", - "for y in epochs_y:\n", - " y_avg.append(np.mean(y))\n", - "\n", - "plt.plot(np.arange(len(epochs_x)), y_avg, marker = '.')\n", - "plt.xlabel('epoch')\n", - "plt.ylabel('average loss')\n", - "plt.ylim(0,)\n", - "for i, txt in enumerate(y_avg):\n", - " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom')\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAG2CAYAAACZEEfAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABSpklEQVR4nO3deVxU5f4H8M8wwLAIo+yMgiAuCO4LBJmmoGhmbr9cMsUtraxcSsUKy8xI780y66rdXFBcslLb7lUBU69FigpuKCoiuLCoyAz7MnN+f5CTE4uMAjNz/Lxfr3m9mnOeOX4f53L5+JznPI9EEAQBRERERCJlZugCiIiIiBoTww4RERGJGsMOERERiRrDDhEREYkaww4RERGJGsMOERERiRrDDhEREYkaww4RERGJGsMOERERiRrDDhEREYmaQcPO4cOHMWzYMCgUCkgkEuzZs0fnvCAIWLx4Mdzd3WFtbY3Q0FBcunRJp01eXh4mTJgAe3t7NG/eHNOmTUNhYWET9oKIiIiMmUHDTlFREbp27Yovv/yyxvMrVqzA559/jrVr1+Lo0aOwtbVFWFgYSktLtW0mTJiAc+fOITY2Fj///DMOHz6MGTNmNFUXiIiIyMhJjGUjUIlEgt27d2PEiBEAqkZ1FAoF3nzzTbz11lsAAKVSCVdXV2zatAnjxo3D+fPn4efnh8TERPTq1QsAsHfvXjzzzDO4fv06FAqFobpDRERERsLc0AXUJj09HdnZ2QgNDdUek8vlCAwMREJCAsaNG4eEhAQ0b95cG3QAIDQ0FGZmZjh69ChGjhxZ47XLyspQVlamfa/RaJCXlwdHR0dIJJLG6xQRERE1GEEQUFBQAIVCATOz2m9WGW3Yyc7OBgC4urrqHHd1ddWey87OhouLi855c3NzODg4aNvUJCoqCkuWLGngiomIiMgQrl27hlatWtV63mjDTmNatGgR5s2bp32vVCrh6emJa9euwd7e3oCVERERUX2pVCp4eHjAzs6uznZGG3bc3NwAADk5OXB3d9cez8nJQbdu3bRtcnNzdT5XWVmJvLw87edrIpPJIJPJqh23t7dn2CEiIjIxD5qCYrTr7Hh7e8PNzQ3x8fHaYyqVCkePHkVQUBAAICgoCPn5+Thx4oS2zYEDB6DRaBAYGNjkNRMREZHxMejITmFhIS5fvqx9n56ejuTkZDg4OMDT0xNz5szBhx9+iHbt2sHb2xuRkZFQKBTaJ7Y6duyIwYMH46WXXsLatWtRUVGB1157DePGjeOTWERERATAwGHn+PHj6N+/v/b9vXk04eHh2LRpExYsWICioiLMmDED+fn56NOnD/bu3QsrKyvtZ7Zu3YrXXnsNISEhMDMzw+jRo/H55583eV+IiIjIOBnNOjuGpFKpIJfLoVQqOWeHiIjIRNT397fRztkhIiIiaggMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkakYfdgoKCjBnzhy0bt0a1tbWCA4ORmJiovb85MmTIZFIdF6DBw82YMVERERkTMwNXcCDTJ8+HWfPnsWWLVugUCgQExOD0NBQpKSkoGXLlgCAwYMHY+PGjdrPyGQyQ5VLRERERsaoR3ZKSkrw/fffY8WKFejbty/atm2L999/H23btsWaNWu07WQyGdzc3LSvFi1aGLBqIiIiMiZGHXYqKyuhVqthZWWlc9za2hpHjhzRvj948CBcXFzQoUMHvPLKK7hz506d1y0rK4NKpdJ5ERERkTgZddixs7NDUFAQli5dips3b0KtViMmJgYJCQnIysoCUHULa/PmzYiPj8fy5ctx6NAhDBkyBGq1utbrRkVFQS6Xa18eHh5N1SUiIiJqYhJBEARDF1GXtLQ0TJ06FYcPH4ZUKkWPHj3Qvn17nDhxAufPn6/W/sqVK/Dx8UFcXBxCQkJqvGZZWRnKysq071UqFTw8PKBUKmFvb99ofSEiIqKGo1KpIJfLH/j726hHdgDAx8cHhw4dQmFhIa5du4Zjx46hoqICbdq0qbF9mzZt4OTkhMuXL9d6TZlMBnt7e50XERERiZPRh517bG1t4e7ujrt372Lfvn0YPnx4je2uX7+OO3fuwN3dvYkrJCIiImNk9I+e79u3D4IgoEOHDrh8+TLmz58PX19fTJkyBYWFhViyZAlGjx4NNzc3pKWlYcGCBWjbti3CwsIMXToREREZAaMf2VEqlZg1axZ8fX0xadIk9OnTB/v27YOFhQWkUilOnz6N5557Du3bt8e0adPQs2dP/O9//+NaO0RERATABCYoN4X6TnAiIiIi4yGaCcpEREREj4Jhh4iIiESNYYeIiIhEjWGHiIiIRI1hh4iIiESNYYeIiIhEjWGHiIiIRI1hh4iIiESNYYeIiIhEjWGHiIiIRI1hh4iIiESNYYeIiIhEjWGHiIiIRI1hh4iIiESNYYeIiIhEjWGHiIiIRI1hh4iIiGpUUFCAOXPmoHXr1rC2tkZwcDASExMBABUVFVi4cCE6d+4MW1tbKBQKTJo0CTdv3qzzmocPH8awYcOgUCggkUiwZ8+eam127dqFQYMGwdHRERKJBMnJyY/UD4YdIiIiqtH06dMRGxuLLVu24MyZMxg0aBBCQ0Nx48YNFBcX4+TJk4iMjMTJkyexa9cupKam4rnnnqvzmkVFRejatSu+/PLLOtv06dMHy5cvb5B+SARBEBrkSiZMpVJBLpdDqVTC3t7e0OUQEREZXElJCezs7PDDDz9g6NCh2uM9e/bEkCFD8OGHH1b7TGJiIgICApCRkQFPT88H/hkSiQS7d+/GiBEjajx/9epVeHt7IykpCd26dat2vr6/vzmyQ0RERNVUVlZCrVbDyspK57i1tTWOHDlS42eUSiUkEgmaN2/eBBXWH8MOERERVWNnZ4egoCAsXboUN2/ehFqtRkxMDBISEpCVlVWtfWlpKRYuXIjx48cb3V0Shh0iIiKq0ZYtWyAIAlq2bAmZTIbPP/8c48ePh5mZbnyoqKjAmDFjIAgC1qxZY6Bqa8ewQ0RERDXy8fHBoUOHUFhYiGvXruHYsWOoqKhAmzZttG3uBZ2MjAzExsYa3agOwLBDRERED2Brawt3d3fcvXsX+/btw/DhwwH8FXQuXbqEuLg4ODo6GrjSmpkbugAiIiIyTvv27YMgCOjQoQMuX76M+fPnw9fXF1OmTEFFRQX+7//+DydPnsTPP/8MtVqN7OxsAICDgwMsLS0BACEhIRg5ciRee+01AEBhYSEuX76s/TPS09ORnJwMBwcH7RNceXl5yMzM1K7Zk5qaCgBwc3ODm5ub3v3gyA4RERHVSKlUYtasWfD19cWkSZPQp08f7Nu3DxYWFrhx4wZ+/PFHXL9+Hd26dYO7u7v29fvvv2uvcfHSZSRdzESWsgQAcPz4cXTv3h3du3cHAMybNw/du3fH4sWLtZ/58ccf0b17d+0j7+PGjUP37t2xdu3ah+oH19kB19khIiJqDN8kZmLRrjPQCICZBIga1Rljez94/Z364jo7REREZDBnbygR8X1V0AEAjQC8veusdoSnKXHODhERET0SQRBw/W4JjqXnIfFqHo6l5+HK7aJq7dSCgKu3i+Eut27S+hh2iIiISC+CICDtViGOplcFm8T0PNxUlj7wc1KJBF5ONk1QoS6GHSIiIqpTpVqD81kFOHY1D8fS7yDx6l3kFZXrtDE3k6BTSzkCvR0Q4O2AXq0dsPdcFt7edRZqQYBUIsFHozo1+agOwLBDREREf1NWqcbp60oc+3Pk5kTGXRSWVeq0kZmboYdnC/T2dkCgtwO6ezaHjaVurBjb2xN92zvj6u1ieDnZGCToAAw7REREj73CskqczLiLxKt5OJqeh+Rr+Siv1Oi0sbMyR6/WLRDg7YgAbwd0bimHpfmDn3Nyl1sbLOTcw7BDRET0mLlbVK6dSJx4NQ9nb6qg1uiuROPUzBIB3g7o7VV1W8rXzR5SM4mBKn40DDtEREQil60s1c63OZaeh4s5hdXatGxurZ1v09vbAW2cbCGRmGa4+TuGHSIiIhERBAEZd4qr5tv8OXqTmVdcrV1bl2bo7VU136a3twNaNjfsrabGxLBDRERkwjQaARdzC3AsvWq+TWJ6HnILynTamEkAP4U9ArwcEeDdAr28HODUTGagipseww4REZEJqVBrcPaG8r45N3ehLKnQaWMpNUNXD7l2vk3P1i1gZ2VhoIoNj2GHiIjIiJVWqJGUma+dTHwi4y5KKtQ6bWwspejZugUCvKpuSXXzaA4rC6mBKjY+DDtERERGRFVagRMZd7Vr3Jy+no8Kte6TUs1tLNCrtYN2QrGfwh4WUm53WRuGHSIiIgO6XViGxPsmE5/PUuFvT4HD1V5Wtb6NV9U6N+1cmsHMRB8DNwSGHSIioiZ0/W6xdr7NsfQ8pN2qvmFma0cbBPw53ybA2wGeDjaieQzcEBh2iIiIGknVhplFOruB38gvqdbO181OO5k4wNsBrvZWBqhWvHiDj4iIGoVarUZkZCS8vb1hbW0NHx8fLF26FILw1z2anJwcTJ48GQqFAjY2Nhg8eDAuXbpU53UrKirwwQcfwMfHB1ZWVujatSv27t1brd2XX34JLy8vWFlZITAwEMeOHWvwPv6dWiPg7A0lNhxJx8tbTqDXh3EIXXkIb+8+g91JN3AjvwRSMwm6ejTHjL5t8PWkXkhePBB75/TF0hGdMKyrgkGnEXBkh4iIGsXy5cuxZs0aREdHw9/fH8ePH8eUKVMgl8vxxhtvQBAEjBgxAhYWFvjhhx9gb2+PlStXIjQ0FCkpKbC1ta3xuu+++y5iYmLw73//G76+vti3bx9GjhyJ33//Hd27dwcAfPPNN5g3bx7Wrl2LwMBAfPbZZwgLC0NqaipcXFwarI9llWqcua7Uzrc5cfUuCmrYMLO7Z/M/b0s5ortnc9jK+Ou3KUmE+yP2Y0qlUkEul0OpVMLe3t7Q5RARicKzzz4LV1dXrF+/Xnts9OjRsLa2RkxMDC5evIgOHTrg7Nmz8Pf3BwBoNBq4ubnho48+wvTp02u8rkKhwDvvvINZs2bVeF0ACAwMRO/evfHFF19or+vh4YHXX38dERERD92n4vJKnMzIr9p24WoekjLzUfa3DTObyczRy6tF1S0pLwd0biWHzJyPgTeG+v7+ZrQkIqJGERwcjK+++goXL15E+/btcerUKRw5cgQrV64EAJSVVa3ya2X1120bMzMzyGQyHDlypNawU1ZWpvMZALC2tsaRI0cAAOXl5Thx4gQWLVqkc93Q0FAkJCTo1Yf84nIcv3oXx/7cDfzcDSUq//aolKOtpc58m47uprthplgx7BARUaOIiIiASqWCr68vpFIp1Go1li1bhgkTJgAAfH194enpiUWLFmHdunWwtbXFp59+iuvXryMrK6vW64aFhWHlypXo27cvfHx8EB8fj127dkGtrlpo7/bt21Cr1XB1ddX5nKurKy5cuFBnzTmqUp3JxBeyC6q1adncWmc3cB9n8WyYKVYMO0RE1Ch27tyJrVu3Ytu2bfD390dycjLmzJkDhUKB8PBwWFhYYNeuXZg2bRocHBwglUoRGhqKIUOGoK4ZFqtWrcJLL70EX19fSCQS+Pj4YMqUKdiwYYNe9QmCgGt5JTj6507giVfzcPVO9Q0zfZxttaM2vb0c0KqFjd5/F2RYDDtERNQo5s+fj4iICIwbNw4A0LlzZ2RkZCAqKgrh4eEAgJ49eyI5ORlKpRLl5eVwdnZGYGAgevXqVet1nZ2dsWfPHpSWluLOnTtQKBSIiIhAmzZtAABOTk6QSqXIycnR+Vx2dg6atXDClj8y/lzj5g5yVLobZkokgJ+7vXa+TS8vBzjbPT4bZoqV0YedgoICREZGYvfu3cjNzUX37t2xatUq9O7dG0BVMn/vvffw73//G/n5+XjyySexZs0atGvXzsCVExE93oqLi2FmprvCiVQqhUajqdZWLpcDAC5duoTjx49j6dKlD7y+lZUVWrZsiYqKCnz//fcYM2YMAMDS0hI9e/ZEbFwcvHs+XbUb+JXb2PHDf2DbfSjS9pzVXsNCKkGXVs21Izc9W7eA/WO8YaZYGX3YmT59Os6ePYstW7ZAoVAgJiZG+1hiy5YtsWLFCnz++eeIjo6Gt7c3IiMjERYWhpSUlGoT2IiIqOkMGzYMy5Ytg6enJ/z9/ZGUlISVK1di6tSp2jbffvstnJ2d4enpiTNnzmD27NkYMWIEBg0apG0zadIktGzZElFRUQCAo0eP4saNG+jWrRtu3LiB999/HxqNBm/MfRN/XLmDxPQ8WHYbhjVrP8A36ZaQubeH6vgPUJeVwKl7GALbOmnn23T35IaZjwXBiBUXFwtSqVT4+eefdY736NFDeOeddwSNRiO4ubkJ//jHP7Tn8vPzBZlMJmzfvr3ef45SqRQACEqlssFqJyJ63KlUKmH27NmCp6enYGVlJbRp00Z45513hLKyMm2bVatWCa1atRIsLCwET09P4d1339U5LwiC0K9fPyE8PFz7/uDBg0LHjh0FmUwm2Dd3EHqEPCcMjdojtHv7P0LrhT9rXy1CZwoWchfBzNxC8OrYVYjes18or1Q3VfepCdT397dRr7NTUFAAe3t7xMXFISQkRHu8T58+MDc3x4YNG+Dj44OkpCR069ZNe75fv37o1q0bVq1aVeN1y8rKtI88AlXP6Xt4eHCdHSIiI5OlLEH67SJ4O9nCUmqGxKt/7gZ+9Q5SblbfMNPFTqa9JRXg7YD2LnbcMFPERLHOjp2dHYKCgrB06VJ07NgRrq6u2L59OxISEtC2bVtkZ2cDQI2PF947V5OoqCgsWbKkUWsnIqJHsyUhA4t/OIu6/kXu6WDzV7jxckBrR26YSdUZddgBgC1btmDq1Klo2bIlpFIpevTogfHjx+PEiRMPfc1FixZh3rx52vf3RnaIiMiw8orKEX8+Bz+euon/Xbpd7XwbJ1sEt3VEgLcjArwc4Cbn3Ex6MKMPOz4+Pjh06BCKioqgUqng7u6OsWPHok2bNnBzcwNQtZGcu7u79jM5OTk6t7X+TiaTQSbjo4RERMbgWl4x9p3LRmxKDhKv5lW7NXW/ZSM7I8jHsemKI1Ew+rBzj62tLWxtbXH37l3s27cPK1asgLe3N9zc3BAfH68NNyqVCkePHsUrr7xi2IKJiKhGgiDg3E0V9qfkYP+57GqrFPsr7BHUxhHrf0vH/bNKpRIJvJy4oB/pz+jDzr59+yAIAjp06IDLly9j/vz58PX1xZQpUyCRSDBnzhx8+OGHaNeunfbRc4VCgREjRhi6dCIi+lOlWoNjV/Ow/1wOYlNycCO/RHtOaiZBgJcDBvm7YqCfq3aF4nauzfD2rrNQCwKkEgk+GtUJ7nJrQ3WBTJjRhx2lUolFixbh+vXrcHBwwOjRo7Fs2TJYWFQt+rRgwQIUFRVhxowZyM/PR58+fbB3716usUNEZGDF5ZU4fPEW9p/LQfyFXChLKrTnrC2k6NveCYP83DDA1wUtbC2rfX5sb0/0be+Mq7eL4eVkw6BDD82oHz1vKvV9dI2IiOp2p7AM8edzsT8lG/+7dBtllX+tluxga4nQji4Y5OeGPu2cuJgfPTJRPHpORETGL+NOEWJTcrD/XA6OZ+hOMPZ0sMEgP1cM8ndDz9YtIOWaN2QADDtERKQXQRBw9oYK+1Oysf9cDlJzdCcYd24p1wac9q7NuO4NGRzDDhERPVCFWoOjV/IQm5KN/Sk5yFKWas9JzSR4oo0DBvm5IdTPFS2bc24NGReGHSIiqlFRWSUOXbyF/eeyceBCLlSlldpzNpZSPN3BGQP9XDGggyvkNtwpnIwXww4REWndKihD/Pkc7E/JwZHLt1F+3wRjp2aWCO3oikH+rgj24QRjMh0MO0REj7n020XYf67q9tTJzLs6C/l5OdogzN8NA/1c0d2TE4zJNDHsEBE9ZjQaAWduKLUTjC/lFuqc79pKjkH+bhjk54q2LpxgTKaPYYeI6DFQXqnBH1fuIDalagXjbNVfE4zNzSQI8nHEID9XhPq5cvE+Eh2GHSIikSoorfhzgnEOfr2Qi4KyvyYY21pK8bSvCwb5ueLpDi6QW3OCMYmX3mHn119/Rf/+/RujFiIiekS5qlLEnq8avfn98h2Uq++fYCzDQL97E4wdITPnBGN6POgddgYPHoxWrVphypQpCA8Ph4eHR2PURURE9ZR2qxD7z+Vgf0o2kjLzdc61cbLFoHsTjD2aw4wTjOkxpHfYuXHjBrZs2YLo6GgsWbIEAwYMwLRp0zBixAhYWlbfyI2IiBqWRiPg1PV87E/Jwf5z2Ui7VaRzvptHcwzyd8UgPze0dWlmoCqJjMcjbQR68uRJbNy4Edu3bwcAvPDCC5g2bRq6du3aYAU2BW4ESkTGrqxSjYS0O9ifkoO4lBzkFpRpz1lIJQj2ccJAP1cM9HOFq72VASslajr1/f39yLue37x5E1999RU+/vhjmJubo7S0FEFBQVi7di38/f0f5dJNhmGHiIyRqrQCB1OrVjA+mHoLhfdNMG4mM0f/PycY9+vgDHsrTjCmx0+j7npeUVGBH374ARs2bEBsbCx69eqFL774AuPHj8etW7fw7rvv4vnnn0dKSspDd4CI6HGUoyqt2kE8JQcJabdRof7r36MudvcmGLvhiTYOnGBMVE96j+y8/vrr2L59OwRBwMSJEzF9+nR06tRJp012djYUCgU0Gk0tVzEuHNkhIkMRBAFptwqx71xVwDl1LV/nfFuXZtodxLu0lHOCMdF9Gm1kJyUlBatXr8aoUaMgk8lqbOPk5IRff/1V30sTET0WNBoBSdfysT8lG7HncnDl9l8TjCUSoLtHc+0TVD7OnGBM9KjM9P1AfHw8xo8fX2vQAQBzc3P069fvkQojosebWq1GZGQkvL29YW1tDR8fHyxduhT3D0ZLJJIaX//4xz/qvPaXX34JLy8vWFlZITAwEMeOHdM5//TTT1e75ssvv/xI/SmtUOPXC7lYtOs0Aj6Kx+g1v2PdoSu4crsIllIz9O/gjKhRnXH07RDsevVJvNzPh0GHqIHoPbITFRUFV1dXTJ06Vef4hg0bcOvWLSxcuLDBiiOix9fy5cuxZs0aREdHw9/fH8ePH8eUKVMgl8vxxhtvAACysrJ0PvPf//4X06ZNw+jRo2u97jfffIN58+Zh7dq1CAwMxGeffYawsDCkpqbCxcVF2+6ll17CBx98oH1vY2Ojdx+UJRU4mJqL/edycDA1F0Xlau05OytzDPB1wSA/N/Tr4IxmMi5oT9RY9J6z4+XlhW3btiE4OFjn+NGjRzFu3Dikp6c3aIFNgXN2iIzPs88+C1dXV6xfv157bPTo0bC2tkZMTEyNnxkxYgQKCgoQHx9f63UDAwPRu3dvfPHFFwAAjUYDDw8PvP7664iIiABQNbLTrVs3fPbZZ3rXnaUsqZpgfC4Hf1y5g0rNX/8X62ZvhUH+VY+HB3o7wtJc78F1IrpPo83Zyc7Ohru7e7Xjzs7O1f6VRUT0sIKDg/HVV1/h4sWLaN++PU6dOoUjR45g5cqVNbbPycnBL7/8gujo6FqvWV5ejhMnTmDRokXaY2ZmZggNDUVCQoJO261btyImJgZubm4YNmwYIiMjaxzdEQQBl3ILsf9cNvan5OD0daXO+fauzTDIzw2D/F3RuaWcO4gTGYDeYcfDwwO//fYbvL29dY7/9ttvUCgUDVYYET3eIiIioFKp4OvrC6lUCrVajWXLlmHChAk1to+OjoadnR1GjRpV6zVv374NtVoNV1dXneOurq64cOGC9v0LL7yA1q1bQ6FQ4PTp01i4cCFSU1Oxa9cuAIBaI+Bk5t0/R3CycfVOsfazEgnQq3WLPxf4c4O3k+2j/DUQUQPQO+y89NJLmDNnDioqKjBgwAAAVZOWFyxYgDfffLPBCySix9POnTuxdetWbNu2Df7+/khOTsacOXOgUCgQHh5erf2GDRswYcIEWFk9+urBM2bM0P53586d4e7ujpCQEGzZdxRnC6wRdz4Hd4rKtW0szc3wVFsnDPJ3xQBfVzjb1f4ABxE1Pb3Dzvz583Hnzh28+uqrKC+v+mG3srLCwoULdYaGiYgexfz58xEREYFx48YBqAodGRkZiIqKqhZ2/ve//yE1NRXffPNNndd0cnKCVCpFTk6OzvGcnBy4ublVa68srsCB1Bz8cqVq37+31u+DdZueAAB7K3OEdHTFID9X9G3vDFtOMCYyWnr/dEokEixfvhyRkZE4f/48rK2t0a5duzofRSci0ldxcTHMzHQn8Eql0hoXK12/fj169uz5wH35LC0t0bNnT8THx2PEiBEAqiYox8fH47XXXgMA3MgvQeyf82+OpudBrRFQer1qNXg3N3eMDPbCID9X9PZ2gIWUE4yJTMFD/1OkWbNm6N27d0PWQkSkNWzYMCxbtgyenp7w9/dHUlISVq5cWW3ZC5VKhW+//RaffPJJjdcJCQnByJEjtWFm3rx5CA8PR69evRAQEIBPP/0UBYWFqGzbD8+u/h+SzqaiKOUgrH16Q2ptB5eKHGQeWIOegcFIXDWdE4yJTNBDhZ3jx49j586dyMzM1N7KuufeBD4iokexevVqREZG4tVXX0Vubi4UCgVmzpyJxYsX67TbsWMHBEHA+PHja7xOWloabt++rX0/duxY5OTmIuLtd3ErNwc27j6wfW4xvj6eBwCQmpvDIuccCk7/gsqyElh7eGDqi+Pw7rvvMugQmSi919nZsWMHJk2ahLCwMOzfvx+DBg3CxYsXkZOTg5EjR2Ljxo2NVWuj4To7ROKVpSxB+u0iKOTW2kfE4y/kIu++CcYyczM81c4Zg/xdEeLrAsdmvC1PZAoabZ2djz76CJ9++ilmzZoFOzs7rFq1Ct7e3pg5c2aN6+8QERlChVqDf/16GZ/FXUJN/6JrbmOBEN+qBf76tneCjSUnGBOJld4/3WlpaRg6dCiAqsl+RUVFkEgkmDt3LgYMGIAlS5Y0eJFERHVRlVbg/E0VzmepkPLnKzW7ABXq6jFnTK9WGNm9FXp7tYA5JxgTPRb0DjstWrRAQUEBAKBly5Y4e/YsOnfujPz8fBQXFz/g00RED08QBFy/W/JXqLmpwvlsFa7lldT7GiO7t0KQj2MjVklExkbvsNO3b1/Exsaic+fOeP755zF79mwcOHAAsbGxCAkJaYwaiegxVFapxqWcwr9CzZ8Bp6C0ssb2LZtbo6O7Pfzc7eCnsIdTMxnGrEvAfVtTQSqRwMtJ/w09ici06R12vvjiC5SWlgIA3nnnHVhYWOD333/H6NGj8e677zZ4gUQkfnlF5VVh5mZVoDmfpcLl3EKdTTTvsZBK0M6lKtBUhRt7dHS3Q3Mby2pto0Z1xtu7zkItCJBKJPhoVCe4y62boktEZET0ehqrsrIS27ZtQ1hYWLW9ZUwZn8YiahoajYCrd4pwPqsAKVnKP0dsCpCtKq2xfXMbiz/DTFWo8VPYw8e5mV67hWcpS3D1djG8nGwYdIhEplGexjI3N8fLL7+M8+fPP3KBRCRuxeWVSM0u0LkNdSG7AMXl6hrbeznaVI3WuNlrR23c5VaPvLaNu9yaIYfoMaf3bayAgAAkJyejdevWjVEPEZkYQRCQW1BWbW5N+u0i1DRuLDM3g++9uTV/jtZ0cLNHM+4tRUSNRO//d3n11Vcxb948XLt2DT179oStra3O+S5dujRYcURkXCrVGly5XaQztyblpkpnB/D7OdvJ/roNpai6FeXtZAupGVciJqKmo/cKyn/fmA+o2hxUEARIJBKo1TUPURszztkhqk5VWoELWQVIuan8M9gUIDWnAOWV1TfiNJMAPs7NdEJNR3d7ONtxJWIiajyNtoJyenr6IxVG1BTUajXef/99xMTEIDs7GwqFApMnT9bZ32jy5MmIjo7W+VxYWBj27t1b63XXrFmDNWvW4OrVqwAAf39/LF68GEOGDNG2efrpp3Ho0CGdz82cORNr165toN41LEEQcCO/RHe0Jqv2tWuayczR0d3uvieh7NHBzQ5WFtImrpyIqH70Djucq0OmYPny5VizZg2io6Ph7++P48ePY8qUKZDL5XjjjTe07QYPHqyzn5tMVvdIRKtWrfDxxx+jXbt2EAQB0dHRGD58OJKSkuDv769t99JLL+GDDz7QvrexMY61Xe5fu+b8fXNsVHWuXfPX3JqO7vbwaGEDM96GIiITonfY2bx5c53nJ02a9NDFEDWU33//HcOHD9dubeLl5YXt27fj2LFjOu1kMhnc3Nzqfd1hw4bpvF+2bBnWrFmDP/74Qyfs2NjY6HXdxnD/2jX3RmvqWrumrcv9oabqv2tau4aIyNToHXZmz56t876iogLFxcWwtLSEjY0Nww4ZheDgYHz11Ve4ePEi2rdvj1OnTuHIkSNYuXKlTruDBw/CxcUFLVq0wIABA/Dhhx/C0bF+Wwmo1Wp8++23KCoqQlBQkM65rVu3IiYmBm5ubhg2bBgiIyMbbXRHoxGQkVf8520oZdUaNjdVda5dc+/x7nu3odq66Ld2DRGRKdE77Ny9e7fasUuXLuGVV17B/PnzG6QookcVEREBlUoFX19fSKVSqNVqLFu2DBMmTNC2GTx4MEaNGgVvb2+kpaXh7bffxpAhQ5CQkACptPb5J2fOnEFQUBBKS0vRrFkz7N69G35+ftrzL7zwAlq3bg2FQoHTp09j4cKFSE1Nxa5dux65XyXlalzI1n0S6kFr19w/t8ZP0TBr1xARmRK9n8aqzfHjx/Hiiy/iwoULDXG5JsWnscRnx44dmD9/Pv7xj3/A398fycnJmDNnDlauXInw8PAaP3PlyhX4+PggLi6uzn3eysvLkZmZCaVSie+++w5ff/01Dh06pBN47nfgwAGEhITg8uXL8PHxqVf9giDgVkEZzumzdo2bnc5oja87164hInFrtKexar2QuTlu3rzZUJcjeiTz589HREQExo0bBwDo3LkzMjIyEBUVVWvYadOmDZycnHD58uU6w46lpSXatm0LAOjZsycSExOxatUqrFu3rsb2gYGBAFBr2Ll/7Zr7d/Oube0ap2ay+0KNHfwV9vBytIW5lLehiIhqonfY+fHHH3XeC4KArKwsfPHFF3jyyScbrDCiR1FcXFxtTSipVAqNpvoaMfdcv34dd+7cgbu7u15/lkajQVlZWa3nk5OTAQDu7u46a9dU7Q+lqnPtmjbOzXSehOrobgcXOyu96iMietzpHXZGjBih814ikcDZ2RkDBgzAJ5980lB1ET2SYcOGYdmyZfD09IS/vz+SkpKwcuVKTJ06FQBQWFiIJUuWYPTo0XBzc0NaWhoWLFiAtm3bIiwsTHudkJAQjBw5Eq+99hoAYNGiRRgyZAg8PT1RUFCAbdu24eDBg9i3bx8AIC0tDdu2bcOQIUNQYWGL/f87htUfRcK9Q3fM2nsb17btr7FeW0updk7NvTk2XLuGiKhh6B126vqXMZGxWL16NSIjI/Hqq68iNzcXCoUCM2fOxOLFiwFUjfKcPn0a0dHRyM/Ph0KhwKBBg7B06VKdtXbS0tJw+/Zt7fvc3FxMmjQJWVlZkMvl6NKlC376z3/g7heAncev4Y/T6YjZsgtLov4BdXkpzO2dYNMuCDbB47SL9CnkVjpza/wUXLuGiKgxNdgEZVPGCcpUmyxlCdJvF8HbyRbucmvt2jX3noSq79o1Hd3/mjzMtWuIiBpGo01QHj16NAICArBw4UKd4ytWrEBiYiK+/fZb/aslMkLfJGYiYtcZ7dNPcmtzKEtqXmlYbm2hM7fGj2vXEBEZDb1HdpydnXHgwAF07txZ5/iZM2cQGhqKnJycBi2wKXBkh/4uS1mC4KgDqOmHo7WjzV+3oLh2DRGRwdT397fe/+wsLCyEpWX1YXgLCwuoVCp9L1cntVqNyMhIeHt7w9raGj4+Pli6dCnuz2eTJ0+GRCLReQ0ePLhB66DHiyAIWP7fCzUGnQ3hvXBofn+sebEn3ghph1A/VyiaWzPoEBEZMb1vY3Xu3BnffPONdqLnPTt27Kh1UbWH1VibORLVRqMR8N6P57AnufqaUVKJBB0VHPkjIjI1eoedyMhIjBo1CmlpaRgwYAAAID4+Htu3b2/w+TqNtZkjUU0q1BrM//YU9iTfhEQCDO+qwE+nsqAWBEglEnw0qhPc5daGLpOIiPSkd9gZNmwY9uzZg48++gjfffcdrK2t0aVLF8TFxaFfv34NWlxjbeZYVlamswhcQ99+I9NTWqHGrK0nEX8hF+ZmEnwypiuGd2uJhUN8cfV2MbycbBh0iIhMlFE/eq7RaPD2229jxYoVOps5Llq0SNtmx44dsLGx0dnMsVmzZnVu5vj+++9jyZIl1Y5zgvLjqaC0AtOjj+Noeh5k5mZY82IPDPB1NXRZRET0APWdoKx32ElMTIRGo9Hu93PP0aNHIZVK0atXr4eruAaNtZljTSM7Hh4eDDuPoTuFZZi8MRFnbihhJzPH1+G9ENim9lFBIiIyHo32NNasWbNw7dq1asdv3LiBWbNm6Xu5Ot2/mWPnzp0xceJEzJ07F1FRUbV+5v7NHGsjk8lgb2+v86LHT5ayBGPWJeDMDSUcbC2xfcYTDDpERCKk95ydlJQU9OjRo9rx7t27IyUlpUGKuqcpN3Okx0v67SK8+PVR3MgvgUJuhc3TAtHWpZmhyyIiokag98iOTCarceHArKwsmJvrnZ3qdG8zx19++QVXr17F7t27sXLlSowcORJA1Zo/8+fPxx9//IGrV68iPj4ew4cPr7aZI9H9Um6q8Pza33EjvwRtnGzx7SvBDDpERCKm95yd8ePHIysrCz/88APkcjkAID8/HyNGjICLiwt27tzZYMUVFBQgMjISu3fv1m7mOH78eCxevBiWlpYoKSnBiBEjkJSUVG0zR1fX+k8w5QrKj4/jV/MwZVMiCkor4eduj83TAuDUjOsyERGZokaboHzjxg307dsXd+7cQffu3QEAycnJcHV1RWxsLDw8PB6tcgNg2Hk8HEzNxcsxJ1BaoUFvrxb4Orw35NYWhi6LiIgeUqNtBNqyZUucPn0aW7duxalTp2BtbY0pU6Zg/PjxsLDgLw4yTj+fvom53ySjQi3g6Q7OWDOhJ6wta16agIiIxOWhJtnY2tpixowZDV0LUaPYfiwTb++u2r382S7uWDmmG3cjJyJ6jDz0jOKUlBRkZmaivLxc5/hzzz33yEURNZR1h9IQ9d8LAIAXAj2xdHgnSM24aScR0eNE77Bz5coVjBw5EmfOnIFEItHuQH5v12e1Wt2wFRI9BEEQsGJfKtYcTAMAvPK0DxaEdeDu5EREjyG9x/Jnz54Nb29v5ObmwsbGBufOncPhw4fRq1cvHDx4sBFKJNKPWiPg3T1ntUFn4WBfLBzsy6BDRPSY0ntkJyEhAQcOHICTkxPMzMxgZmaGPn36ICoqCm+88QaSkpIao06ieqlQazBv5yn8dKpq5/JlIzrjhUBPQ5dFREQGpPfIjlqthp2dHQDAyckJN2/eBAC0bt0aqampDVsdkR5KytWYsfk4fjp1ExZSCT4f151Bh4iI9B/Z6dSpE06dOgVvb28EBgZixYoVsLS0xFdffYU2bdo0Ro1ED6QqrcD0Tcdx7GoerCzMsPbFnni6g4uhyyIiIiOgd9h59913UVRUBAD44IMP8Oyzz+Kpp56Co6MjvvnmmwYvkOhBbheWIXzDMZy7qYKdlTk2TO6N3l4Ohi6LiIiMhN4rKNckLy8PLVq0MNkJoFxB2XTdyC/BxK+P4srtIjjaWmLztAD4K+SGLouIiJpAo62gXBMHB/4rmppe2q1CTPz6KG4qS9GyuTW2TAtAG2du6ElERLoadptyoiZy9oYS4RuO4U5ROdo42yJmWiAUza0NXRYRERkhhh0yOcfS8zBtUyIKyirRqaU9oqcEwJE7lxMRUS0Ydsik/HqhaufyskoNArwd8HV4L9hbcQNaIiKqnd7r7Bw+fBiVlZXVjldWVuLw4cMNUhRRTX48dRMvbT6OskoNBvi6YPPUAAYdIiJ6IL3DTv/+/ZGXl1ftuFKpRP/+/RukKKK/23o0A7N3JKFSI2B4NwXWTewJKwupocsiIiIToPdtLEEQanzE/M6dO7C1tW2Qooju96+Dl7Fib9Xq3C8+4YkPnusEM+5cTkRE9VTvsDNq1CgAVbubT548GTLZXxNC1Wo1Tp8+jeDg4IavkB5bgiDg470XsO7QFQDArP4+eGsQdy4nIiL91DvsyOVVC7UJggA7OztYW//1mK+lpSWeeOIJvPTSSw1fIT2WqnYuP4Ptx64BAN5+xhcz+voYuCoiIjJF9Q47GzduBAB4eXnhrbfe4i0rajTllRrM3ZmMX05nwUwCRI3qjLG9uaEnERE9HL3n7CxYsAD37zCRkZGB3bt3w8/PD4MGDWrQ4ujxU1KuxssxJ3Do4i1YSCVYNa47nunsbuiyiIjIhOn9NNbw4cOxefNmAEB+fj4CAgLwySefYPjw4VizZk2DF0iPD2VJBSauP4pDF2/B2kKKr8N7M+gQEdEj0zvsnDx5Ek899RQA4LvvvoObmxsyMjKwefNmfP755w1eID0ebhWUYdxXf+B4xl3YW5kjZnoA+rV3NnRZREQkAnrfxiouLoadnR0AYP/+/Rg1ahTMzMzwxBNPICMjo8ELJPG7frcYE9cfQ/rtIjg1k2HLtAB0dOfu80RE1DD0Htlp27Yt9uzZg2vXrmHfvn3aeTq5ubl1bq9OVJPLuYV4fm0C0m8XoWVza3z7chCDDhERNSi9w87ixYvx1ltvwcvLCwEBAQgKCgJQNcrTvXv3Bi+QxOvMdSXGrEtAlrIUbV2a4ftXguHtxKf8iIioYUmE+x+tqqfs7GxkZWWha9euMDOrykvHjh2Dvb09fH19G7zIxqZSqSCXy6FUKjk61UT+uHIH06OPo7CsEl1aybFpSgAcbC0NXRYREZmQ+v7+1ntkBwDc3NxgZ2eH2NhYlJSUAAB69+5tkkGHml78+RyEbziGwrJKPNHGAVunBzLoEBFRo9E77Ny5cwchISFo3749nnnmGWRlZQEApk2bhjfffLPBCyRx+SH5BmZuOYGySg1CO7pg05QA2HHnciIiakR6h525c+fCwsICmZmZsLGx0R4fO3Ys9u7d26DFkbhsSbiKOd8ko1IjYGT3lljzIncuJyKixqf3o+f79+/Hvn370KpVK53j7dq146PnVCNBEPDlr5fxz/0XAQDhQa3x3jB/7lxORERNQu+wU1RUpDOic09eXp7OTuhEQFXQ+eg/5/Hv/6UDAN4Y0BZzB7bnzuVERNRk9L6N9dRTT2m3iwAAiUQCjUaDFStWoH///g1aHJk2tUZAxPdntEHn3aEdMW9QBwYdIiJqUnqP7KxYsQIhISE4fvw4ysvLsWDBApw7dw55eXn47bffGqNGMkFllWrM2ZGM/57NhpkE+HhUF4zp7WHosoiI6DGk98hOp06dcPHiRfTp0wfDhw9HUVERRo0ahaSkJPj4+DRGjWRiissrMT36OP57NhuWUjP8a0IPBh0iIjIYvRcVzMzMhIeHR423IjIzM+Hp6dlgxTUVLirYcJTFFZiy6RhOZubDxlKKdRN74ql23NCTiIgaXqMtKujt7Y1bt25VO37nzh14e3vrezkSkdyCUoz9KgEnM/P/3Lk8kEGHiIgMTu85O4Ig1DiqU1hYCCsrqwYpikzPtbxivLj+KDLuFMPZrmrncl83jpIREZHh1TvszJs3D0DV01eRkZE6j5+r1WocPXoU3bp1a/ACyfhdyinAi+uPIkdVBg8Ha8RMC0RrR27oSURExqHeYScpKQlA1cjOmTNnYGn5115GlpaW6Nq1K956662Gr5CM2qlr+Zi88RjuFlegnUszbJkWCDc5R/iIiMh41Dvs/PrrrwCAKVOmYNWqVZzIS0hIu4Pp0YkoKlej6587l7fghp5ERGRk9J6zs3Hjxsaog0xMbEoOZm07ifJKDYJ9HPHVpF5oJtP7f05ERESNjr+dSG+7k67jrW9PQ60RMNDPFavHd+eGnkREZLQYdkgv0b9fxXs/ngMAjOrREitGd4G5VO8VDIiIiJoMww7ViyAIWH3gMlbGVu1cPjnYC4uf9ePO5UREZPQYduiBNBoBy/5zHuuPVG3oOSe0HWaHtOOGnkREZBIYdqhOlWoNInadwXcnrgMAFj/rh6l9uFI2ERGZDoYdqlVZpRqztydj77lsSM0kWD66C/6vZytDl0VERKQXhh2qUVFZJWZuOYEjl2/DUmqG1S90R5i/m6HLIiIi0hvDDlWTX1yOyRsTkXytaufyf0/qhSfbOhm6LCIioodi1M8Mq9VqREZGwtvbG9bW1vDx8cHSpUshCIK2jSAIWLx4Mdzd3WFtbY3Q0FBcunTJgFWbtlxVKcau+wPJ1/LR3MYC2156gkGHiIhMmlGHneXLl2PNmjX44osvcP78eSxfvhwrVqzA6tWrtW1WrFiBzz//HGvXrsXRo0dha2uLsLAwlJaWGrBy05R5pxj/tzYBqTkFcLGT4ZsZQejm0dzQZRERET0SiXD/MImRefbZZ+Hq6or169drj40ePRrW1taIiYmBIAhQKBR48803tZuQKpVKuLq6YtOmTRg3bly9/hyVSgW5XA6lUvnY7vl1MacAL359FLkFZfB0sEHMtEB4Oto8+INEREQGUt/f30Y9shMcHIz4+HhcvFi1kN2pU6dw5MgRDBkyBACQnp6O7OxshIaGaj8jl8sRGBiIhIQEg9RsipIy72LMugTkFpShg6sdvns5iEGHiIhEw6gnKEdEREClUsHX1xdSqRRqtRrLli3DhAkTAADZ2dkAAFdXV53Pubq6as/VpKysDGVlZdr3KpWqEao3Db9dvo2XNh9Hcbka3TyaY9OU3mhuw53LiYhIPIx6ZGfnzp3YunUrtm3bhpMnTyI6Ohr//Oc/ER0d/UjXjYqKglwu1748PDwaqGLTsu9cNqZsTERxuRp92jph6/RABh0iIhIdow478+fPR0REBMaNG4fOnTtj4sSJmDt3LqKiogAAbm5V677k5OTofC4nJ0d7riaLFi2CUqnUvq5du9Z4nTBS3524jldiTqBcrcFgfzesn9wLtjKjHugjIiJ6KEYddoqLi2FmpluiVCqFRqMBAHh7e8PNzQ3x8fHa8yqVCkePHkVQUFCt15XJZLC3t9d5PU42HEnHW9+egkYA/q9nK3zxQnfIzKWGLouIiKhRGPU/5YcNG4Zly5bB09MT/v7+SEpKwsqVKzF16lQAgEQiwZw5c/Dhhx+iXbt28Pb2RmRkJBQKBUaMGGHY4o2QIAj4LO4SVsVXrUM0rY833nmmI3cuJyIiUTPqsLN69WpERkbi1VdfRW5uLhQKBWbOnInFixdr2yxYsABFRUWYMWMG8vPz0adPH+zduxdWVlYGrNz4aDQCPvg5BZt+vwoAeHNge7w2oC13LiciItEz6nV2morY19mpVGuw4PvT2HXyBgBgyXP+CA/2MmxRREREj6i+v7+NemSHHl1phRqvb09CbEoOpGYS/PP5LhjZnTuXExHR44NhR8QKyyoxY/Nx/J52B5bmZvjyhR4Y6Of64A8SERGJCMOOSN0tKsfkjcdw6roStpZS/Du8F4J9uKEnERE9fhh2RChbWYqJ64/iUm4hWthYIHpqALq0am7osoiIiAyCYUdkMu4UYcLXR3H9bgnc7K2wZVoA2rnaGbosIiIig2HYEZEL2SpMXH8MtwrK0NqxaudyDwdu6ElERI83hh2ROJl5F1M2JkJZUgFfNztsnhYAFzuuNURERMSwIwL/u3QLMzafQEmFGj08m2Pj5ADIbSwMXRYREZFRYNgxcXvPZuGN7ckoV2vwVDsnrJvYEzaW/FqJiIju4W9FE7bz+DVEfH8aGgF4prMbPh3bjRt6EhER/Q3Djon6+n9X8OEv5wEAY3t54KNRnSHlhp5ERETVMOyYGEEQsDL2IlYfuAwAmNG3DRYN8eWGnkRERLVg2DEhGo2AJT+dQ3RCBgBgflgHvPq0D4MOERFRHRh2TESFWoMF353G7qQbkEiAD57zx8QgL0OXRUREZPQYdkxAaYUar207ibjzuZCaSbByTFcM79bS0GURERGZBIYdI1dQWoHp0cdxND0PMnMz/GtCD4R05M7lRERE9cWwY8Ty/ty5/PR1JZrJzPF1eC880cbR0GURERGZFIYdI5WlLMHE9cdwObcQDraWiJ4SgM6t5IYui4iIyOQw7Bih9NtFePHro7iRXwJ3uRW2TAtEW5dmhi6LiIjIJDHsGJmUmypM2nAMtwvL4O1kiy3TAtCqBXcuJyIielhmhi7AGHl5eUEikVR7zZo1S9smISEBAwYMgK2tLezt7dG3b1+UlJTUek21Wo3IyEh4e3vD2toaPj4+WLp0KQRB0Lb5x7poBD7VH0nLRiFj+bOIDLJh0CEiInpEHNmpQWJiItRqtfb92bNnMXDgQDz//PMAqoLO4MGDsWjRIqxevRrm5uY4deoUzMxqz47Lly/HmjVrEB0dDX9/fxw/fhxTpkyBXC7HG2+8gUMXb2Hlf85AquiIbkEDcXLrcjjYWjZ6X4mIiMROItw/tPCYUqlUkMvlUCqVsLe3r3Z+zpw5+Pnnn3Hp0iVIJBI88cQTGDhwIJYuXVrvP+PZZ5+Fq6sr1q9frz02evRoWFtbY/yCf2DON0moUAvo194Zi/o4omOHtkhKSkK3bt0aootERESi86Df3/fwNtYDlJeXIyYmBlOnToVEIkFubi6OHj0KFxcXBAcHw9XVFf369cORI0fqvE5wcDDi4+Nx8eJFAMCpU6dw5MgRNG/fG69vP4kKtYChXdzx70m9YGXJncuJiIgaCm9jPcCePXuQn5+PyZMnAwCuXLkCAHj//ffxz3/+E926dcPmzZsREhKCs2fPol27djVeJyIiAiqVCr6+vpBKpVCr1Rg+fR5+Lm4LABgf4IEPR3DnciIioobGkZ0HWL9+PYYMGQKFQgEA0Gg0AICZM2diypQp6N69Oz799FN06NABGzZsqPU6O3fuxNatW7Ft2zacOHECY96Mwo9bvkLhmXjM7NcGH41k0CEiImoMDDt1yMjIQFxcHKZPn6495u7uDgDw8/PTaduxY0dkZmbWeq358+cjIiICY8aMxc4rEvwh7QS73sNhfuYHLBrSkTuXExERNRKGnTps3LgRLi4uGDp0qPaYl5cXFAoFUlNTddpevHgRrVu3rvVaxcXFEADM3ZmMmD8yIZEAYZ3cYW/F+TlERESNiXN2aqHRaLBx40aEh4fD3PyvvyaJRIL58+fjvffeQ9euXdGtWzdER0fjwoUL+O6777TtQkJCMHLkSLz22msAgGeGPouFkR/ANuQVWDu3xvh2Gnz99UZMnTpV+5m8vDxkZmbi5s2bAKANVG5ubnBzc2uKbhMREYkOw04t4uLikJmZqRNG7pkzZw5KS0sxd+5c5OXloWvXroiNjYWPj4+2TVpaGm7fvg0AUJVWoLDnREhT8nF3/7+gLCvAzpYKzJw5E4sXL9Z+5scff8SUKVO078eNGwcAeO+99/D+++83Uk+JiIjEjevsoP7P6esrS1mCU9fy8WncRaRmF8JOZo71k3sjwNuhwf4MIiKix1V9f39zZKeRfJOYiUW7zkDzZ5S0tZRi+4wn0Kkldy4nIiJqSpyg3AiylCU6QQcASirUcGzG7R+IiIiaGsNOI0i/XaQTdABAIwBXbxcbpiAiIqLHGMNOI/B2ssXf1weUSiTwcuIO5kRERE2NYacRuMutETWqM6R/LhQolUjw0ahOcJdbG7gyIiKixw8nKDeSsb090be9M67eLoaXkw2DDhERkYEw7DQid7k1Qw4REZGB8TYWERERiRrDDhEREYkaww4RERGJGsMOERERiRrDDhEREYkaww4RERGJGsMOERERiRrDDhEREYkaww4RERGJGsMOERERiRrDDhEREYkaww4RERGJGsMOERERiZrRhx0vLy9IJJJqr1mzZgEAnn766WrnXn75ZQNXTURERMbC3NAFPEhiYiLUarX2/dmzZzFw4EA8//zz2mMvvfQSPvjgA+17GxubJq2RiIiIjJfRhx1nZ2ed9x9//DF8fHzQr18/7TEbGxu4ubk1dWlERERkAoz+Ntb9ysvLERMTg6lTp0IikWiPb926FU5OTujUqRMWLVqE4uLiOq9TVlYGlUql8yIiIiJxMvqRnfvt2bMH+fn5mDx5svbYCy+8gNatW0OhUOD06dNYuHAhUlNTsWvXrlqvExUVhSVLljRBxURERGRoEkEQBEMXUV9hYWGwtLTETz/9VGubAwcOICQkBJcvX4aPj0+NbcrKylBWVqZ9r1Kp4OHhAaVSCXt7+wavm4iIiBqeSqWCXC5/4O9vkxnZycjIQFxcXJ0jNgAQGBgIAHWGHZlMBplM1uA1EhERkfExmTk7GzduhIuLC4YOHVpnu+TkZACAu7t7E1RFRERExs4kRnY0Gg02btyI8PBwmJv/VXJaWhq2bduGZ555Bo6Ojjh9+jTmzp2Lvn37okuXLgasmIiIiIyFSYSduLg4ZGZmYurUqTrHLS0tERcXh88++wxFRUXw8PDA6NGj8e677xqoUiIiIjI2JjVBubHUd4ITERERGY/6/v42mTk7RERERA+DYYeIiIhEjWGHiIiIRI1hh4iIiESNYYeIiIhEjWGHiIiIRI1hh4iIiESNYYeIiIhEjWGHiIiIRI1hh4iIiESNYYeIiIhEjWGHiIiIRI1hh4iIiESNYYeIiIhEjWGHiIiIRI1hh4iIiESNYYeIiIhEjWGHiIiIRI1hh4iIiESNYYeIiIhEjWGHiIiIRI1hh4iIiESNYYeIiIhEjWGHiIiIRI1hh4iIiESNYYeIiIhEjWGHiIiIRI1hh4iIiESNYYeIiIhEjWGHiIiIRI1hh4iIiESNYYeIiIhEjWGHiIiIRI1hh4iIiESNYYeIiIhEjWGHiIiIRI1hh4iIiESNYYeIiIhEjWGHiIiIRI1hh4iIiESNYYeIiIhEjWGHiIiIRI1hh4iIiESNYYeIiIhEjWGHiIiIRI1hh4iIiESNYYeIiIhEjWGHiIiIRI1hh4iIiESNYYeIiIhEjWGHiIiIRM3ow46XlxckEkm116xZswAApaWlmDVrFhwdHdGsWTOMHj0aOTk5Bq6aiIiIjIXRh53ExERkZWVpX7GxsQCA559/HgAwd+5c/PTTT/j2229x6NAh3Lx5E6NGjTJkyURERGREJIIgCIYuQh9z5szBzz//jEuXLkGlUsHZ2Rnbtm3D//3f/wEALly4gI4dOyIhIQFPPPFEva6pUqkgl8uhVCphb2/fmOUTERFRA6nv72/zJqzpkZWXlyMmJgbz5s2DRCLBiRMnUFFRgdDQUG0bX19feHp61hl2ysrKUFZWpn2vVCoBVP2lERERkWm493v7QeM2JhV29uzZg/z8fEyePBkAkJ2dDUtLSzRv3lynnaurK7Kzs2u9TlRUFJYsWVLtuIeHR0OWS0RERE2goKAAcrm81vMmFXbWr1+PIUOGQKFQPNJ1Fi1ahHnz5mnfazQa5OXlwdHRERKJ5FHL1FKpVPDw8MC1a9dEe3tM7H1k/0yf2PvI/pk+sfexMfsnCAIKCgoemAtMJuxkZGQgLi4Ou3bt0h5zc3NDeXk58vPzdUZ3cnJy4ObmVuu1ZDIZZDKZzrG/jw41JHt7e1H+D/h+Yu8j+2f6xN5H9s/0ib2PjdW/ukZ07jH6p7Hu2bhxI1xcXDB06FDtsZ49e8LCwgLx8fHaY6mpqcjMzERQUJAhyiQiIiIjYxIjOxqNBhs3bkR4eDjMzf8qWS6XY9q0aZg3bx4cHBxgb2+P119/HUFBQfV+EouIiIjEzSTCTlxcHDIzMzF16tRq5z799FOYmZlh9OjRKCsrQ1hYGP71r38ZoMrqZDIZ3nvvvWq3zMRE7H1k/0yf2PvI/pk+sffRGPpncuvsEBEREenDZObsEBERET0Mhh0iIiISNYYdIiIiEjWGHSIiIhI1hp1H9OWXX8LLywtWVlYIDAzEsWPH6mz/7bffwtfXF1ZWVujcuTP+85//NFGlD0ef/m3atAkSiUTnZWVl1YTV6ufw4cMYNmwYFAoFJBIJ9uzZ88DPHDx4ED169IBMJkPbtm2xadOmRq/zUejbx4MHD1b7DiUSSZ3brxhSVFQUevfuDTs7O7i4uGDEiBFITU194OdM5efwYfpnaj+Ha9asQZcuXbQLzgUFBeG///1vnZ8xle8P0L9/pvb9/d3HH38MiUSCOXPm1Nmuqb9Dhp1H8M0332DevHl47733cPLkSXTt2hVhYWHIzc2tsf3vv/+O8ePHY9q0aUhKSsKIESMwYsQInD17tokrrx99+wdUrZCZlZWlfWVkZDRhxfopKipC165d8eWXX9arfXp6OoYOHYr+/fsjOTkZc+bMwfTp07Fv375GrvTh6dvHe1JTU3W+RxcXl0aq8NEcOnQIs2bNwh9//IHY2FhUVFRg0KBBKCoqqvUzpvRz+DD9A0zr57BVq1b4+OOPceLECRw/fhwDBgzA8OHDce7cuRrbm9L3B+jfP8C0vr/7JSYmYt26dejSpUud7QzyHQr00AICAoRZs2Zp36vVakGhUAhRUVE1th8zZowwdOhQnWOBgYHCzJkzG7XOh6Vv/zZu3CjI5fImqq5hARB2795dZ5sFCxYI/v7+OsfGjh0rhIWFNWJlDac+ffz1118FAMLdu3ebpKaGlpubKwAQDh06VGsbU/s5vF99+mfKP4f3tGjRQvj6669rPGfK3989dfXPVL+/goICoV27dkJsbKzQr18/Yfbs2bW2NcR3yJGdh1ReXo4TJ04gNDRUe8zMzAyhoaFISEio8TMJCQk67QEgLCys1vaG9DD9A4DCwkK0bt0aHh4eD/zXi6kxpe/vUXXr1g3u7u4YOHAgfvvtN0OXU29KpRIA4ODgUGsbU/4e69M/wHR/DtVqNXbs2IGioqJat/wx5e+vPv0DTPP7mzVrFoYOHVrtu6mJIb5Dhp2HdPv2bajVari6uuocd3V1rXV+Q3Z2tl7tDelh+tehQwds2LABP/zwA2JiYqDRaBAcHIzr1683RcmNrrbvT6VSoaSkxEBVNSx3d3esXbsW33//Pb7//nt4eHjg6aefxsmTJw1d2gNpNBrMmTMHTz75JDp16lRrO1P6Obxffftnij+HZ86cQbNmzSCTyfDyyy9j9+7d8PPzq7GtKX5/+vTPFL+/HTt24OTJk4iKiqpXe0N8hyaxXQSZhqCgIJ1/rQQHB6Njx45Yt24dli5dasDKqL46dOiADh06aN8HBwcjLS0Nn376KbZs2WLAyh5s1qxZOHv2LI4cOWLoUhpFfftnij+HHTp0QHJyMpRKJb777juEh4fj0KFDtQYCU6NP/0zt+7t27Rpmz56N2NhYo55IzbDzkJycnCCVSpGTk6NzPCcnB25ubjV+xs3NTa/2hvQw/fs7CwsLdO/eHZcvX26MEptcbd+fvb09rK2tDVRV4wsICDD6APHaa6/h559/xuHDh9GqVas625rSz+E9+vTv70zh59DS0hJt27YFAPTs2ROJiYlYtWoV1q1bV62tKX5/+vTv74z9+ztx4gRyc3PRo0cP7TG1Wo3Dhw/jiy++QFlZGaRSqc5nDPEd8jbWQ7K0tETPnj0RHx+vPabRaBAfH1/rvdigoCCd9gAQGxtb571bQ3mY/v2dWq3GmTNn4O7u3lhlNilT+v4aUnJystF+h4Ig4LXXXsPu3btx4MABeHt7P/AzpvQ9Pkz//s4Ufw41Gg3KyspqPGdK319t6urf3xn79xcSEoIzZ84gOTlZ++rVqxcmTJiA5OTkakEHMNB32GhTnx8DO3bsEGQymbBp0yYhJSVFmDFjhtC8eXMhOztbEARBmDhxohAREaFt/9tvvwnm5ubCP//5T+H8+fPCe++9J1hYWAhnzpwxVBfqpG//lixZIuzbt09IS0sTTpw4IYwbN06wsrISzp07Z6gu1KmgoEBISkoSkpKSBADCypUrhaSkJCEjI0MQBEGIiIgQJk6cqG1/5coVwcbGRpg/f75w/vx54csvvxSkUqmwd+9eQ3XhgfTt46effirs2bNHuHTpknDmzBlh9uzZgpmZmRAXF2eoLtTplVdeEeRyuXDw4EEhKytL+youLta2MeWfw4fpn6n9HEZERAiHDh0S0tPThdOnTwsRERGCRCIR9u/fLwiCaX9/gqB//0zt+6vJ35/GMobvkGHnEa1evVrw9PQULC0thYCAAOGPP/7QnuvXr58QHh6u037nzp1C+/btBUtLS8Hf31/45Zdfmrhi/ejTvzlz5mjburq6Cs8884xw8uRJA1RdP/ces/77616fwsPDhX79+lX7TLdu3QRLS0uhTZs2wsaNG5u8bn3o28fly5cLPj4+gpWVleDg4CA8/fTTwoEDBwxTfD3U1DcAOt+LKf8cPkz/TO3ncOrUqULr1q0FS0tLwdnZWQgJCdEGAUEw7e9PEPTvn6l9fzX5e9gxhu9QIgiC0HjjRkRERESGxTk7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0REf3Pw4EFIJBLk5+cbuhQiagAMO0RERCRqDDtEREQkagw7RGR0NBoNoqKi4O3tDWtra3Tt2hXfffcdgL9uMf3yyy/o0qULrKys8MQTT+Ds2bM61/j+++/h7+8PmUwGLy8vfPLJJzrny8rKsHDhQnh4eEAmk6Ft27ZYv369TpsTJ06gV69esLGxQXBwMFJTUxu340TUKBh2iMjoREVFYfPmzVi7di3OnTuHuXPn4sUXX8ShQ4e0bebPn49PPvkEiYmJcHZ2xrBhw1BRUQGgKqSMGTMG48aNw5kzZ/D+++8jMjISmzZt0n5+0qRJ2L59Oz7//HOcP38e69atQ7NmzXTqeOedd/DJJ5/g+PHjMDc3x9SpU5uk/0TUsLgRKBEZlbKyMjg4OCAuLg5BQUHa49OnT0dxcTFmzJiB/v37Y8eOHRg7diwAIC8vD61atcKmTZswZswYTJgwAbdu3cL+/fu1n1+wYAF++eUXnDt3DhcvXkSHDh0QGxuL0NDQajUcPHgQ/fv3R1xcHEJCQgAA//nPfzB06FCUlJTAysqqkf8WiKghcWSHiIzK5cuXUVxcjIEDB6JZs2ba1+bNm5GWlqZtd38QcnBwQIcOHXD+/HkAwPnz5/Hkk0/qXPfJJ5/EpUuXoFarkZycDKlUin79+tVZS5cuXbT/7e7uDgDIzc195D4SUdMyN3QBRET3KywsBAD88ssvaNmypc45mUymE3gelrW1db3aWVhYaP9bIpEAqJpPRESmhSM7RGRU/Pz8IJPJkJmZibZt2+q8PDw8tO3++OMP7X/fvXsXFy9eRMeOHQEAHTt2xG+//aZz3d9++w3t27eHVCpF586dodFodOYAEZF4cWSHiIyKnZ0d3nrrLcydOxcajQZ9+vSBUqnEb7/9Bnt7e7Ru3RoA8MEHH8DR0RGurq5455134OTkhBEjRgAA3nzzTfTu3RtLly7F2LFjkZCQgC+++AL/+te/AABeXl4IDw/H1KlT8fnnn6Nr167IyMhAbm4uxowZY6iuE1EjYdghIqOzdOlSODs7IyoqCleuXEHz5s3Ro0cPvP3229rbSB9//DFmz56NS5cuoVu3bvjpp59gaWkJAOjRowd27tyJxYsXY+nSpXB3d8cHH3yAyZMna/+MNWvW4O2338arr76KO3fuwNPTE2+//bYhuktEjYxPYxGRSbn3pNTdu3fRvHlzQ5dDRCaAc3aIiIhI1Bh2iIiISNR4G4uIiIhEjSM7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkav8PYsojZWtCM4MAAAAASUVORK5CYII=", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "plt.plot(np.arange(len(epochs_x)), epochs_acc, marker = '.')\n", - "plt.xlabel('epoch')\n", - "plt.ylabel('test accuracy')\n", - "plt.ylim(70, 100)\n", - "for i, txt in enumerate(epochs_acc):\n", - " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom')\n", - "plt.show()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "speck-rescnn", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/tests/test_nonsequential/non-sequential-SCNN-example_2.ipynb b/tests/test_nonsequential/non-sequential-SCNN-example_2.ipynb deleted file mode 100644 index c6420691..00000000 --- a/tests/test_nonsequential/non-sequential-SCNN-example_2.ipynb +++ /dev/null @@ -1,639 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [], - "source": [ - "import torch\n", - "import torch.nn as nn\n", - "import sinabs.layers as sl\n", - "from tqdm.notebook import tqdm\n", - "\n", - "from tonic.datasets.nmnist import NMNIST\n", - "from tonic.transforms import ToFrame\n", - "from torch.utils.data import DataLoader\n", - "from torch.nn import CrossEntropyLoss\n", - "from torch.optim import Adam\n", - "\n", - "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential\n", - "\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "torch.manual_seed(0)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Loading Data" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "batch_size = 64\n", - "num_workers = 4\n", - "epochs = 5\n", - "lr = 1e-3" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "root_dir = \"./NMNIST\"\n", - "_ = NMNIST(save_to=root_dir, train=True)\n", - "_ = NMNIST(save_to=root_dir, train=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "n_time_steps = 50\n", - "to_raster = ToFrame(sensor_size=NMNIST.sensor_size, n_time_bins=n_time_steps)\n", - "\n", - "snn_train_dataset = NMNIST(save_to=root_dir, train=True, transform=to_raster)\n", - "snn_test_dataset = NMNIST(save_to=root_dir, train=False, transform=to_raster)" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "snn_train_dataloader = DataLoader(snn_train_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True)\n", - "snn_test_dataloader = DataLoader(snn_test_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=False)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Network Module\n", - "\n", - "We need to define a `nn.Module` implementing the network we want the chip to reproduce." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "device: NVIDIA RTX A4000\n" - ] - } - ], - "source": [ - "if torch.cuda.is_available():\n", - " device = torch.device('cuda:0')\n", - " print('device: ', torch.cuda.get_device_name(0))\n", - "else:\n", - " device = torch.device('cpu')" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "class SNN(nn.Module):\n", - " def __init__(self) -> None:\n", - " super().__init__()\n", - "\n", - " self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False)\n", - " self.iaf1 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - " self.pool1 = nn.AvgPool2d(2,2)\n", - " self.pool1a = nn.AvgPool2d(6,6)\n", - "\n", - " self.conv2 = nn.Conv2d(10, 10, 2, 1, bias=False)\n", - " self.iaf2 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - " self.pool2 = nn.AvgPool2d(3,3)\n", - "\n", - " self.conv3 = nn.Conv2d(10, 10, 3, 1, bias=False)\n", - " self.iaf3 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - " self.pool3 = nn.AvgPool2d(2,2)\n", - "\n", - " self.flat = nn.Flatten()\n", - "\n", - " self.fc1 = nn.Linear(10, 100, bias=False)\n", - " self.iaf4 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - "\n", - " self.fc2 = nn.Linear(100, 100, bias=False)\n", - " self.iaf5 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - "\n", - " self.fc3 = nn.Linear(100, 100, bias=False)\n", - " self.iaf6 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - "\n", - " self.fc4 = nn.Linear(100, 10, bias=False)\n", - " self.iaf7 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - "\n", - " self.merge_fc = sl.Merge()\n", - " self.merge_conv = sl.Merge()\n", - "\n", - " def detach_neuron_states(self):\n", - " for name, layer in self.named_modules():\n", - " if name != '':\n", - " if isinstance(layer, sl.StatefulLayer):\n", - " for name, buffer in layer.named_buffers():\n", - " buffer.detach_()\n", - "\n", - " def init_weights(self):\n", - " for name, layer in self.named_modules():\n", - " if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear):\n", - " nn.init.xavier_normal_(layer.weight.data)\n", - "\n", - " def forward(self, x):\n", - " \n", - " con1_out = self.conv1(x)\n", - " iaf1_out = self.iaf1(con1_out)\n", - " pool1_out = self.pool1(iaf1_out)\n", - " pool1a_out = self.pool1a(iaf1_out)\n", - "\n", - " conv2_out = self.conv2(pool1_out)\n", - " iaf2_out = self.iaf2(conv2_out)\n", - " pool2_out = self.pool2(iaf2_out)\n", - "\n", - " merged_conv_out = self.merge_conv(pool1a_out, pool2_out)\n", - "\n", - " conv3_out = self.conv3(merged_conv_out)\n", - " iaf3_out = self.iaf3(conv3_out)\n", - " pool3_out = self.pool3(iaf3_out)\n", - "\n", - " flat_out = self.flat(pool3_out)\n", - " \n", - " fc1_out = self.fc1(flat_out)\n", - " iaf4_out = self.iaf4(fc1_out)\n", - "\n", - " fc2_out = self.fc2(iaf4_out)\n", - " iaf5_out = self.iaf5(fc2_out)\n", - "\n", - " fc3_out = self.fc3(iaf5_out)\n", - " iaf6_out = self.iaf6(fc3_out)\n", - "\n", - " merge_fc_out = self.merge_fc(iaf4_out, iaf6_out)\n", - "\n", - " fc4_out = self.fc4(merge_fc_out)\n", - " iaf7_out = self.iaf7(fc4_out)\n", - "\n", - " return iaf7_out" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "snn = SNN().to(device)" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "snn.init_weights()" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "optimizer = Adam(snn.parameters(), lr=lr, betas=(0.9, 0.999), eps=1e-8)\n", - "loss_fn = CrossEntropyLoss()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Define train and test" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "def train(dataloader, model, loss_fn, optimizer, epochs, test_func, dataloader_test):\n", - " epochs_y = []\n", - " epochs_x = []\n", - " epochs_acc = []\n", - " model.train()\n", - "\n", - " for e in range(epochs):\n", - " losses = []\n", - " batches = []\n", - " batch_count = 0\n", - " train_p_bar = tqdm(snn_train_dataloader)\n", - "\n", - " for X, y in train_p_bar:\n", - " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", - " X = X.reshape(-1, 2, 34, 34).to(dtype=torch.float, device=device)\n", - " y = y.to(dtype=torch.long, device=device)\n", - "\n", - " # forward\n", - " pred = model(X)\n", - "\n", - " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", - " pred = pred.reshape(batch_size, n_time_steps, -1)\n", - "\n", - " # accumulate all time-steps output for final prediction\n", - " pred = pred.sum(dim = 1)\n", - " loss = loss_fn(pred, y)\n", - "\n", - " # gradient update\n", - " optimizer.zero_grad()\n", - " loss.backward()\n", - " optimizer.step()\n", - "\n", - " # detach the neuron states and activations from current computation graph(necessary)\n", - " model.detach_neuron_states()\n", - "\n", - " train_p_bar.set_description(f\"Epoch {e} - BPTT Training Loss: {round(loss.item(), 4)}\")\n", - "\n", - " batch_count += 1\n", - " losses.append(loss.item())\n", - " batches.append(batch_count)\n", - "\n", - " epochs_y.append(losses)\n", - " epochs_x.append(batches)\n", - "\n", - " acc = test_func(dataloader_test, model)\n", - " print(f'Epoch {e} accuracy: {acc}')\n", - " epochs_acc.append(acc)\n", - "\n", - " return epochs_x, epochs_y, epochs_acc\n" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "def test(dataloader, model):\n", - " correct_predictions = []\n", - " with torch.no_grad():\n", - " test_p_bar = tqdm(dataloader)\n", - " for X, y in test_p_bar:\n", - " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", - " X = X.reshape(-1, 2, 34, 34).to(dtype=torch.float, device=device)\n", - " y = y.to(dtype=torch.long, device=device)\n", - "\n", - " # forward\n", - " output = model(X)\n", - "\n", - " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", - " output = output.reshape(batch_size, n_time_steps, -1)\n", - "\n", - " # accumulate all time-steps output for final prediction\n", - " output = output.sum(dim=1)\n", - "\n", - " # calculate accuracy\n", - " pred = output.argmax(dim=1, keepdim=True)\n", - "\n", - " # compute the total correct predictions\n", - " correct_predictions.append(pred.eq(y.view_as(pred)))\n", - "\n", - " test_p_bar.set_description(f\"Testing Model...\")\n", - " \n", - " correct_predictions = torch.cat(correct_predictions)\n", - " return correct_predictions.sum().item()/(len(correct_predictions))*100" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Training loop (HPO)" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "1b6ac71c89994fada6d3e4a0ef50b939", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - " 0%| | 0/937 [00:00" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "plt.plot(epochs_x[-1], epochs_y[-1])\n", - "plt.xlabel('batches')\n", - "plt.ylabel('loss')\n", - "plt.ylim(0,)\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAGxCAYAAACeKZf2AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABSe0lEQVR4nO3deVxU5f4H8M/MwMyAMAPIvigqijsgKqmZWiil10S7Vypv+LPtVloaLS6ZtoqWlpWm7WbdUiuXbhguKFqmmSyliLgLAsOiMuzbzPn9gU5NAjLIcGaGz/v1mteNw3POfB/PHefjOc95HokgCAKIiIiIbIRU7AKIiIiI2hLDDREREdkUhhsiIiKyKQw3REREZFMYboiIiMimMNwQERGRTWG4ISIiIpvCcENEREQ2xU7sAtqbXq9HXl4enJ2dIZFIxC6HiIiIWkAQBJSVlcHX1xdSafPXZjpcuMnLy0NAQIDYZRAREVEr5OTkwN/fv9k2HS7cODs7A2j4w1GpVE22O3DgAN59912kp6dDo9Hgv//9L/7xj3802f7777/HJ598gqNHj6K2tha9e/fGvHnzEBkZ2epjEhERUYPS0lIEBAQYvseb0+HCzbVbUSqVqtlwAwDh4eF49NFHMWXKFDg6Ojbb/siRI7jrrrvwxhtvwMXFBZ999hnuvfde/PrrrwgLC2vVMYmIiMhYS4aUSDrawpmlpaVQq9XQarUtDhYSiQRbtmxBdHS0Se/Vr18/xMTEYNGiRW12TCIioo7IlO9vPi1lJnq9HmVlZXBzcxO7FCIiog6F4cZMli9fjvLyckydOlXsUoiIiDqUDjfmpj189dVXePnll7Ft2zZ4enqKXQ4REVGHwnDTxjZs2ICHH34Y33zzjdGTUkRERNQ+eFuqDX399deYMWMGvv76a0yYMEHscoiIiDokXrlpQnl5OU6fPm34+dy5c0hPT4ebmxu6dOmC+fPnIzc3F+vXrwfQcCtq+vTpeOeddxAREQGNRgMAcHBwgFqtbtExiYiI6ObxUfAmJCcnY8yYMddtnz59OtatW4f/+7//w/nz55GcnAwAGD16NPbt29dk+5Yck4iIiBpnyqPgDDdtKF9bhXPFFejm3gk+aoc2PTYREVFHZsr3N29LtZGNv2Vj/uaj0AuAVALETxmAmCG81URERNTeRB1QvH//fkycOBG+vr6QSCTYunVrs+03b96MsWPHwsPDAyqVCsOGDcOOHTvap9hm5GurMO9qsAEAvQAs2HwM+doqcQsjIiLqgEQNNxUVFQgJCcHq1atb1H7//v0YO3Ystm/fjpSUFIwZMwYTJ05EWlqamStt3rniCvz95p5OEHC+uFKcgoiIiDowUW9L3XXXXbjrrrta3H7lypVGPy9ZsgTbtm3D//73P6PFKdtbN/dOkEpguHIDNNyaCnR3FK0mIiKijsqq57lpyfpNNTU1KC0tNXq1NR+1A+KnDID0LwuV3h3iy0HFREREIrDqcNOS9Zvi4+OhVqsNr4CAALPUEjOkCw7Mux0zRgQCAH46VYyy6jqzvBcRERE1zWrDzbX1mzZt2tTs+k3z58+HVqs1vHJycsxWk4/aAQvG90F3j064VFGLNclnzPZeRERE1DirDDfX1m/atGnTDddvUigUUKlURi9zspdJMf+uPgCAT34+h9wSPjFFRETUnqwu3FjD+k2RfTwR0c0NNfV6LN+RJXY5REREHYqo4aa8vBzp6elIT08H8OdaS9nZ2QAabinFxsYa2n/11VeIjY3FihUrDOs3aTQaaLVaMcpvkkQiwcIJfQEAW9Jy8cfFEnELIiIi6kBEDTdHjhxBWFiY4THuuLg4hIWFYdGiRQCA/Px8Q9ABgA8//BD19fWYOXMmfHx8DK/Zs2eLUn9zBvirMSXMDwDwekImOtgqF0RERKLh2lJmlFdShTHLk1FTr8eHD4RjXD9vs74fERGRrTLl+9vqxtxYE18XBzw8shsAYOmPJ1Cn04tcERERke1juDGzx0b1gLuTHGeLK/DVr9k33oGIiIhuCsONmTkr7TEnshcAYOXuk9BWcWI/IiIic2K4aQf3DglAkKcTrlTW4f3k02KXQ0REZNMYbtqBnUyKBeN7AwA++/k8ci5ztXAiIiJzYbhpJ2OCPTEiqDNqdXq8yYn9iIiIzIbhpp1IJBIsGN8HEgnw/e95SM8pEbskIiIim8Rw0476+apxzyB/AMDrCcc5sR8REZEZMNy0s2fHBUNpL8Vv569gR4ZG7HKIiIhsDsNNO/NWK/HoyO4AGib2q63nxH5ERERtieFGBI+O6gF3JwXOX6rEl4cuiF0OERGRTWG4EYGTwg7PjGuY2O/dPaegreTEfkRERG2F4UYkUwcHINjLGSWVdVi195TY5RAREdkMhhuRyKQSzL86sd/nv1xA9iVO7EdERNQWGG5ENDrYEyN7uqNWp8eyHSfELoeIiMgmMNyI7NrEfgl/5CPlwhWxyyEiIrJ6DDci6+OjwtTwAADAa5zYj4iI6KYx3FiAuHG94GAvQ1p2CbYf5cR+REREN4PhxgJ4qZT4z6irE/slZqKmXidyRURERNaL4cZCPHpbd3g6K5BzuQpfHOTEfkRERK3FcGMhHOV2eHZcMADg3aRTuFJRK3JFRERE1onhxoLcE+6P3t7OKK2ux3t7TotdDhERkVViuLEgMqkEL0zoAwD44tB5nC+uELkiIiIi68NwY2FG9vTA6GAP1OkELP2RE/sRERGZiuHGAi0Y3wdSCZCYocFv5y+LXQ4REZFVYbixQL28nBEzpAsA4LWETOj1nNiPiIiopRhuLNTTY3uik1yG33NK8MPRfLHLISIishoMNxbK01mJx0b1AAAs+/EEqus4sR8REVFLMNxYsIdHdoe3Sonckip8/st5scshIiKyCgw3FsxBLsOzUQ0T+63aexqXObEfERHRDTHcWLgpYX7o66NCWXU93tl9UuxyiIiILB7DjYWTSiVYeHViv//+mo0zReUiV0RERGTZGG6swPAgd9zR2xP1ek7sR0REdCMMN1Zi/vjekEkl2HW8AIfOXhK7HCIiIovFcGMlgjydcd/QAADA65zYj4iIqEkMN1ZkTmQvOCnscDRXi+9/zxO7HCIiIovEcGNF3J0UeHx0w8R+byRyYj8iIqLGMNxYmYdu7QZftRJ52mp88vM5scshIiKyOAw3VkZpL8NzdzZM7Lcm+QyKy2tEroiIiMiyMNxYoUkhfhjgp0Z5TT1WcmI/IiIiIww3VkgqleCFqxP7fX04B6cLy0SuiIiIyHIw3FipW7p3xti+XtDpBcRv58R+RERE1zDcWLH5d/WGnVSCpBOF+OV0sdjlEBERWQSGGyvW3cMJ0yK6AABe386J/YiIiACGG6s3O7IXnBV2yMgrxea0XLHLISIiEh3DjZVz6yTHzNuDAADLd2ShqpYT+xERUcfGcGMD/m94IPxcHKAprcbHP50VuxwiIiJRMdzYAKW9DM9fm9hv3xkUllWLXBEREZF4RA03+/fvx8SJE+Hr6wuJRIKtW7fecJ/k5GQMGjQICoUCQUFBWLdundnrtAZ3h/giJMAFlbU6vL3rlNjlEBERiUbUcFNRUYGQkBCsXr26Re3PnTuHCRMmYMyYMUhPT8ecOXPw8MMPY8eOHWau1PJJJBIsvDqx38bfsnGygBP7ERFRxyQRBMEinh+WSCTYsmULoqOjm2wzd+5cJCQk4NixY4Zt9957L0pKSpCYmNii9yktLYVarYZWq4VKpbrZsi3OY1+kIDFDg9HBHlg3Y6jY5RAREbUJU76/rWrMzcGDBxEZGWm0LSoqCgcPHmxyn5qaGpSWlhq9bNm8qxP7JWcV4adTRWKXQ0RE1O6sKtxoNBp4eXkZbfPy8kJpaSmqqqoa3Sc+Ph5qtdrwCggIaI9SRRPo3gkPDOsKAHg9IRM6TuxHREQdjFWFm9aYP38+tFqt4ZWTkyN2SWb31O09oVLa4YSmDN+lXBS7HCIionZlVeHG29sbBQUFRtsKCgqgUqng4ODQ6D4KhQIqlcroZetcO8nx5O09AQDLd2ahsrZe5IqIiIjaj1WFm2HDhiEpKclo265duzBs2DCRKrJcscO7ooubIwrLavDhfk7sR0REHYeo4aa8vBzp6elIT08H0PCod3p6OrKzswE03FKKjY01tH/sscdw9uxZPP/88zhx4gTef/99bNq0CU8//bQY5Vs0hZ0Mc+/sDQD4YN9ZFJZyYj8iIuoYRA03R44cQVhYGMLCwgAAcXFxCAsLw6JFiwAA+fn5hqADAN26dUNCQgJ27dqFkJAQrFixAh9//DGioqJEqd/SjR/gjUFdXFBVp8OKnSfFLoeIiKhdWMw8N+3F1ue5+buUC1dwz5pfIJEA258aiT4+tt9nIiKyPTY7zw2ZLryrKyYM8IEgAEu2Z4pdDhERkdkx3HQAc+/sDXuZBD+dKkZyVqHY5RAREZkVw00H0KWzI6YPCwTQcPWmXqcXtyAiIiIzYrjpIJ68vSdcHO1xsqAc33BiPyIismEMNx2E2tEeT12d2G/FzpOoqOHEfkREZJsYbjqQf9/SFYGdHVFcXoMP9p0RuxwiIiKzYLjpQOR2Usy7q2Fivw9/Oot8beOLjRIREVkzhpsOJqqfN4YEuqK6Ts+J/YiIyCYx3HQwEokEC8b3AQB8l3oRGXlakSsiIiJqWww3HVBYF1dMDPGFIACvJ2Sig01STURENo7hpoN6PioYcjspfjlzCXs5sR8REdkQhpsOKsDNETNGBAIAlmw/wYn9iIjIZjDcdGBPjA6Cq6M9TheWY8NvOWKXQ0RE1CYYbjowtYM95kT2AgC8veskyqrrRK6IiIjo5jHcdHD3R3RBd/dOuFRRi7Wc2I+IiGwAw00HZy/7c2K/j386h7wSTuxHRETWjeGGMLavF4Z2c0NNvR7Ld2SJXQ4REdFNYbghSCQSLJzQMLHf5rRcHL3Iif2IiMh6MdwQAGCgvwsmh/kBAF7ffpwT+xERkdViuCGDZ6OCobCT4tDZy9idyYn9iIjIOjHckIGfiwMeurUbACB+eybqOLEfERFZIYYbMvL46B7o3EmOs8UV+PpwttjlEBERmYzhhow4K+0xZ2zDxH4rd59CKSf2IyIiK8NwQ9e5b0gAenh0wuWKWry/lxP7ERGRdWG4oevYyaRYML7h0fBPD5xDzuVKkSsiIiJqOYYbatTtvT0xvEdn1NbrsXwnJ/YjIiLrwXBDjZJIJFgwvg8kEmBbeh7Sc0rELomIiKhFGG6oSf391JgS5g8AeD2BE/sREZF1YLihZj0b1QtKeyl+O38FOzIKxC6HiIjohhhuqFk+agc8MrI7AGDpj5morefEfkREZNkYbuiG/jOqB9ydFDh/qRL//fWC2OUQERE1i+GGbshJYYe4qxP7vZN0CtpKTuxHRESWi+GGWmTqYH/08nJCSWUdViefFrscIiKiJjHcUIvYyaSYf3Viv3UHznNiPyIislgMN9Rio3t5YGRPd9Tq9FiaeELscoiIiBrFcEMtJpFIMP+uhon9Ev7IR8qFK2KXREREdB2GGzJJX18V/hXOif2IiMhyMdyQyZ4ZFwwHexlSs0vw4zGN2OUQEREZYbghk3mplHj0tmsT+51ATb1O5IqIiIj+xHBDrfKfUd3h6axA9uVKfHGQE/sREZHlYLihVnGU2+GZcQ0T+7235zRKKmtFroiIiKgBww212j/DA9Db2xnaqjq8m8SJ/YiIyDIw3FCryaQSLLg6sd8Xh87jfHGFyBUREREx3NBNuq2XB0b18kCdTsAyTuxHREQWgOGGbtqC8X0glQA/HtPgyPnLYpdDREQdHMMN3bRgb2fEDAkAALyWkMmJ/YiISFQMN9Qmnh7bC45yGdJzSvDDH/lil0NERB2Y6OFm9erVCAwMhFKpREREBA4fPtxs+5UrVyI4OBgODg4ICAjA008/jerq6naqlpri6azEY6N6AACWJZ5AdR0n9iMiInGIGm42btyIuLg4LF68GKmpqQgJCUFUVBQKCwsbbf/VV19h3rx5WLx4MTIzM/HJJ59g48aNWLBgQTtXTo15ZGR3eKkUuHilCp//cl7scoiIqIMSNdy89dZbeOSRRzBjxgz07dsXa9euhaOjIz799NNG2//yyy8YMWIE7r//fgQGBmLcuHG47777bni1h9qHg1yGZ8cFAwBW7T2NyxWc2I+IiNqfaOGmtrYWKSkpiIyM/LMYqRSRkZE4ePBgo/sMHz4cKSkphjBz9uxZbN++HePHj2+XmunGpgzyR18fFcqq6/Fu0imxyyEiog5ItHBTXFwMnU4HLy8vo+1eXl7QaBpfafr+++/HK6+8gltvvRX29vbo0aMHRo8e3extqZqaGpSWlhq9yHxkUglemNAwsd+Xhy7gbFG5yBUREVFHI/qAYlMkJydjyZIleP/995GamorNmzcjISEBr776apP7xMfHQ61WG14BAQHtWHHHNCLIHbf39kS9XsDSHzmxHxERtS/Rwo27uztkMhkKCgqMthcUFMDb27vRfV588UU88MADePjhhzFgwABMnjwZS5YsQXx8PPR6faP7zJ8/H1qt1vDKyclp877Q9RaM7w2ZVIKdxwvw69lLYpdDREQdiGjhRi6XIzw8HElJSYZter0eSUlJGDZsWKP7VFZWQio1LlkmkwFAkxPHKRQKqFQqoxeZX5CnM+69OrHf69szoddzYj8iImofot6WiouLw0cffYTPP/8cmZmZePzxx1FRUYEZM2YAAGJjYzF//nxD+4kTJ2LNmjXYsGEDzp07h127duHFF1/ExIkTDSGHLMfTY3vBSWGHPy5q8f3veWKXQ0REHYSdmG8eExODoqIiLFq0CBqNBqGhoUhMTDQMMs7Ozja6UrNw4UJIJBIsXLgQubm58PDwwMSJE/H666+L1QVqhruTAo+P7oE3d2ThzR1ZuLO/N5T2DKFERGReEqGDLQRUWloKtVoNrVbLW1TtoLpOhzHLk5GvrcbzdwbjidFBYpdERERWyJTvb6t6Woqsj9JehueiGib2e3/vGVwqrxG5IiIisnUmh5uqqipUVlYafr5w4QJWrlyJnTt3tmlhZDuiQ/0wwE+N8pp6rNzNif2IiMi8TA43kyZNwvr16wEAJSUliIiIwIoVKzBp0iSsWbOmzQsk6yeVSrBgfMPEfl8dzsbpQk7sR0RE5mNyuElNTcXIkSMBAN9++y28vLxw4cIFrF+/Hu+++26bF0i2YViPzojs4wWdXsDSHzPFLoeIiGyYyeGmsrISzs7OAICdO3diypQpkEqluOWWW3DhwoU2L5Bsx/yrE/vtzizEL2eKxS6HiIhslMnhJigoCFu3bkVOTg527NiBcePGAQAKCwv59BE1q4eHE6ZFdAEAvJ7Aif2IiMg8TA43ixYtwrPPPovAwEBEREQYZhPeuXMnwsLC2rxAsi2z7+gJZ4UdMvJKsSUtV+xyiIjIBrVqnhuNRoP8/HyEhIQYJtk7fPgwVCoVevfu3eZFtiXOcyO+NclnsCzxBHzUSux5ZjQc5JzYj4iImmf2eW68vb0RFhYGqVSK0tJSbN26Fc7OzhYfbMgyzBgRCD8XB+Rrq/HJz2fFLoeIiGyMyeFm6tSpWLVqFYCGOW8GDx6MqVOnYuDAgfjuu+/avECyPUp7GZ6/s2FivzXJZ1BUxon9iIio7Zgcbvbv3294FHzLli0QBAElJSV499138dprr7V5gWSbJg70RYi/GhW1Ory9+6TY5RARkQ0xOdxotVq4ubkBABITE3HPPffA0dEREyZMwKlTnH2WWkYqleCFCX0BABsOZ+NUQZnIFRERka0wOdwEBATg4MGDqKioQGJiouFR8CtXrkCpVLZ5gWS7hnZzQ1Q/L+gFYMl2TuxHRERtw+RwM2fOHEybNg3+/v7w9fXF6NGjATTcrhowYEBb10c2bt5dfWAnlWBvVhF+PsWJ/YiI6OaZHG6eeOIJHDx4EJ9++il+/vlnw6Pg3bt355gbMlk390749y1dAQCvJRyHjhP7ERHRTWrVPDfXXNtVIpG0WUHmxnluLM+VilqMenMvSqvr8cY/B2Lq4ACxSyIiIgtj9nlu1q9fjwEDBsDBwQEODg4YOHAgvvjii1YVS+TaSY4nb+8JAFixMwuVtfUiV0RERNbM5HDz1ltv4fHHH8f48eOxadMmbNq0CXfeeScee+wxvP322+aokTqA2OFdEeDmgILSGny0/5zY5RARkRUz+bZUt27d8PLLLyM2NtZo++eff46XXnoJ585Z9hcTb0tZrh/+yMOsr9LgKJch+dnR8FTx6TsiImpg1ttS+fn5GD58+HXbhw8fjvz8fFMPR2QwYYAPwrq4oLJWh7d2cWI/IiJqHZPDTVBQEDZt2nTd9o0bN6Jnz55tUhR1TBKJBAsn9AEAbDqSgxOaUpErIiIia2Rn6g4vv/wyYmJisH//fowYMQIAcODAASQlJTUaeohMEd7VDeMHeGP7UQ2WbD+B9Q8OFbskIiKyMiZfubnnnnvw66+/wt3dHVu3bsXWrVvh7u6Ow4cPY/LkyeaokTqYuXf2hr1Mgv0ni7DvZJHY5RARkZW5qXlurBEHFFuH1344jo9/PodgL2dsnz0SMqn1zKVERERtz5Tv7xbdliotbfnYBwYGaguzbg/CNykXkVVQhm+O5ODeoV3ELomIiKxEi8KNi4vLDWchFgQBEokEOp2uTQqjjs3FUY6n7uiJV384jhW7TmJiiC86KUweIkZERB1Qi74t9u7da+46iK7zwC1dsf7geVy4VIkP9p9F3NheYpdERERWgGNuyKL9eDQfj/83FUp7KZKfHQNvNSf2IyLqiMy+thRRe7mzvzcGd3VFdZ0eK3ZmiV0OERFZAYYbsmgSiQQvXJ3Y79vUi8jI04pcERERWTqGG7J4YV1cMTHEF4IALNmeiQ52J5WIiEzEcENW4fmoYMhlUhw4fQnJWZzYj4iImtaqcFNfX4/du3fjgw8+QFlZGQAgLy8P5eXlbVoc0TUBbo6YMSIQQMPVm3qdXtyCiIjIYpkcbi5cuIABAwZg0qRJmDlzJoqKGv4VvWzZMjz77LNtXiDRNU+MCYKroz1OFZZj45EcscshIiILZXK4mT17NgYPHowrV67AwcHBsH3y5MlISkpq0+KI/krtYI/ZdzSsPP/2rpMoq64TuSIiIrJEJoebn376CQsXLoRcLjfaHhgYiNzc3DYrjKgx90d0RTf3Tigur8UH+86KXQ4REVkgk8ONXq9vdImFixcvwtnZuU2KImqK3E6KeXf1BgB89NNZ5JVUiVwRERFZGpPDzbhx47By5UrDzxKJBOXl5Vi8eDHGjx/flrURNWpcXy8M7eaGmno9lnNiPyIi+huTw82KFStw4MAB9O3bF9XV1bj//vsNt6SWLVtmjhqJjEgkEiy8OrHf5tRcHMvlxH5ERPSnVq0tVV9fjw0bNuCPP/5AeXk5Bg0ahGnTphkNMLZUXFvKdszZkIat6Xm4pbsbvn7klhuuXE9ERNbLlO/vFq0Kft1Odnb497//3ariiNrKs1HB2H5Mg0NnLyMpsxCRfb3ELomIiCyAyeHm+++/b3S7RCKBUqlEUFAQunXrdtOFEd2Iv6sjHrq1G9Ykn8GSHzMxKtgD9jJOuk1E1NGZHG6io6MhkUiuW9/n2jaJRIJbb70VW7duhaura5sVStSYx0f3wMbfcnC2qAIbDmfjgWGBYpdEREQiM/mfubt27cKQIUOwa9cuaLVaaLVa7Nq1CxEREfjhhx+wf/9+XLp0ibMVU7tQKe3xdOTVif12n0IpJ/YjIurwTB5Q3L9/f3z44YcYPny40fYDBw7g0UcfRUZGBnbv3o0HH3wQ2dnZbVpsW+CAYttTp9PjzpX7caaoAo+P7oG5d/YWuyQiImpjpnx/m3zl5syZM40eVKVS4ezZhhlje/bsieLiYlMPTdQq9jIp5t/V8Gj4Jz+fw8UrlSJXREREYjI53ISHh+O5554zLJgJAEVFRXj++ecxZMgQAMCpU6cQEBDQdlUS3cAdfTwxrHtn1NbrsXwHJ/YjIurITA43n3zyCc6dOwd/f38EBQUhKCgI/v7+OH/+PD7++GMAQHl5ORYuXNjmxRI1RSKR4IUJfSCRAFvT8/B7TonYJRERkUhMDjfBwcE4fvw4tm3bhqeeegpPPfUUvv/+e2RkZKBXr14AGp6oeuCBB1p0vNWrVyMwMBBKpRIRERE4fPhws+1LSkowc+ZM+Pj4QKFQoFevXti+fbup3SAb1N9PjclhfgCA1xMyr3uij4iIOoZWTeInlUpx55134s4777ypN9+4cSPi4uKwdu1aREREYOXKlYiKikJWVhY8PT2va19bW4uxY8fC09MT3377Lfz8/HDhwgW4uLjcVB1kO54dF4yEP/Jx+Pxl7DxegKh+3mKXRERE7axVyy9UVFRg3759yM7ORm1trdHvnnrqqRYfJyIiAkOGDMGqVasANKw4HhAQgCeffBLz5s27rv3atWvx5ptv4sSJE7C3tze1bAB8WqojWL4jC6v2nkY3907YMec2yO04sR8RkbUz5fvb5HCTlpaG8ePHo7KyEhUVFXBzc0NxcTEcHR3h6elpeGLqRmpra+Ho6Ihvv/0W0dHRhu3Tp09HSUkJtm3bdt0+48ePh5ubGxwdHbFt2zZ4eHjg/vvvx9y5cyGTyVr0vgw3tq+8ph6j39yL4vJavDSxL/5vBGfMJiKydmZ9FPzpp5/GxIkTceXKFTg4OODQoUO4cOECwsPDsXz58hYfp7i4GDqdDl5exusBeXl5QaPRNLrP2bNn8e2330Kn02H79u148cUXsWLFCrz22mtNvk9NTQ1KS0uNXmTbnBR2eHpsw/ivd5JOQVvFif2IiDoSk8NNeno6nnnmGUilUshkMtTU1CAgIABvvPEGFixYYI4aDfR6PTw9PfHhhx8iPDwcMTExeOGFF7B27dom94mPj4darTa8+Ih6xxAzOAA9PZ1wpbIO7+89LXY5RETUjkwON/b29pBKG3bz9PQ0zEKsVquRk5PT4uO4u7tDJpOhoKDAaHtBQQG8vRsfBOrj44NevXoZ3YLq06cPNBrNdWN/rpk/f75hmQitVmtSjWS97GRSLBjfMLHfZwfOI+cyJ/YjIuooTA43YWFh+O233wAAo0aNwqJFi/Df//4Xc+bMQf/+/Vt8HLlcjvDwcCQlJRm26fV6JCUlYdiwYY3uM2LECJw+fRp6vd6w7eTJk/Dx8YFcLm90H4VCAZVKZfSijmF0sAduDXJHrU6PZYknxC6HiIjaicnhZsmSJfDx8QEAvP7663B1dcXjjz+OoqIifPjhhyYdKy4uDh999BE+//xzZGZm4vHHH0dFRQVmzJgBAIiNjcX8+fMN7R9//HFcvnwZs2fPxsmTJ5GQkIAlS5Zg5syZpnaDOgCJRIIF4xsm9vvhj3ykZl8RuyQiImoHJs1zIwgCPD09DVdoPD09kZiY2Oo3j4mJQVFRERYtWgSNRoPQ0FAkJiYaBhlnZ2cbboEBQEBAAHbs2IGnn34aAwcOhJ+fH2bPno25c+e2ugaybX19VfjnIH98k3IRrydk4tvHhkEikYhdFhERmZFJj4Lr9XoolUpkZGSgZ8+e5qzLbPgoeMej0VZjzPJkVNXpsGbaINw1wEfskoiIyERmexRcKpWiZ8+euHTp0k0VSNSevNVKPHJbdwDA0sQTqK3X32APIiKyZiaPuVm6dCmee+45HDt2zBz1EJnFf27rDg9nBS5cqsQXhy6IXQ4REZmRyTMUu7q6orKyEvX19ZDL5XBwcDD6/eXLl9u0wLbG21Id14bD2Zi3+SjUDvbY99xouDg2/oQdERFZHlO+v01eOHPlypWtrYuo3a1evRpvvvkmNBoNBoaEwHvYg9DAH+/tOY0X/9HXqO26desMT+pdo1AoUF1dDQCoq6vDwoULsX37dpw9exZqtRqRkZFYunQpfH19261PRETUPJPDzfTp081RB1Gba2zV+a8/mwuX2NVYf/A8Yod1RdfOnYz2UalUyMrKMvz81yerKisrkZqaihdffBEhISG4cuUKZs+ejbvvvhtHjhxpt34REVHzWrUq+JkzZ/DZZ5/hzJkzeOedd+Dp6Ykff/wRXbp0Qb9+/cxRZ5vhbamOo6lV572GReNy0HiMH+CN96eFG9qvW7cOc+bMQUlJSYvf47fffsPQoUNx4cIFdOnSpa27QEREV5l14cx9+/ZhwIAB+PXXX7F582aUl5cDAH7//XcsXry4dRUTtbHa2lqkpKQgMjLSsE0qlSIyMhKu5echlQDbj2qQcsF4jFh5eTm6du2KgIAATJo0CRkZGc2+j1arhUQigYuLizm6QURErWByuJk3bx5ee+017Nq1y2jJg9tvvx2HDh1q0+KIWqu5VefLrxRj6uCGBVRfS8jEtYuXwcHB+PTTT7Ft2zZ8+eWX0Ov1GD58OC5evNjoe1RXV2Pu3Lm47777eBWQiMiCmBxujh49ismTJ1+33dPTE8XFxW1SFJG5xY3tBUe5DGnZJUg4mg8AGDZsGGJjYxEaGopRo0Zh8+bN8PDwwAcffHDd/nV1dZg6dSoEQcCaNWvau3wiImqGyeHGxcUF+fn5121PS0uDn59fmxRFdLNutOq8p0qJ/9zWAwCwLPEEaup11x3D3t4eYWFhOH36tNH2a8HmwoUL2LVrF6/aEBFZGJPDzb333ou5c+dCo9FAIpFAr9fjwIEDePbZZxEbG2uOGolM1pJV5x+5rRu8VArkXK7C57+cv+4YOp0OR48eNSwUC/wZbE6dOoXdu3ejc+fOZu8LERGZxuRHwa+twh0QEACdToe+fftCp9Ph/vvvx8KFC81RI1GrxMXFYfr06Rg8eDCGDh2KlStXGq06/9jDD8JH6owC7/F4b89pnNu5HmNuG4GgoCCUlJTgzTffxIULF/Dwww8DaAg2//znP5GamooffvgBOp0OGo0GAODm5mY0Bo2IiMTTqkfBgYYVu48dO4by8nKEhYVZzUKafBS8Y1m1apVhEr/Q0FC8++67iIiIAACMHj0aXbsGonDQQ8jML4X38Y3I/30fNBoNXF1dER4ejtdeew1hYWEAgPPnz6Nbt26Nvs/evXsxevTo9uoWEVGHY8r3t8nh5ueff8att956UwWKieGG/u7nU8X49ye/wk4qwVePRKBeL6Cbeyf4qB1uvDMREbULs4YbuVwOPz8/3Hffffj3v/+Nvn373ngnC8JwQ42Z8dlh7M0qMvwslQDxUwYgZggn5iMisgRmncQvLy8PzzzzDPbt24f+/fsjNDQUb775ZpNzgRBZg0dGdjf6WS8ACzYfQ762SqSKiIiotUwON+7u7pg1axYOHDiAM2fO4F//+hc+//xzBAYG4vbbbzdHjUTmJ7l+k04Q8H16HnT6Vg1LIyIikbR6QPE1Op0OP/74I1588UX88ccf0Omuny/EkvC2FDUmX1uFEUv3oLEc4+mswN0hvpg8yA99fVRGi2kSEVH7MOttqWsOHDiAJ554Aj4+Prj//vvRv39/JCQktPZwRKLyUTsgfsoAyK4GF6kEuKW7G1wc7VFYVoOPfz6HCe/+jDtX/oS1+87wdhURkQUz+crN/PnzsWHDBuTl5WHs2LGYNm0aJk2aBEdHR3PV2KZ45Yaak6+twvniSgS6O8JH7YDaej2SswqxJS0XSZmFqNXpAQASCTCse2dMDvPDXQN84KQwecooIiIygVmflhoxYgSmTZuGqVOnwt3d/aYKFQPDDbWWtrIO24/lY0tqLg6f/3M1caW9FGP7emNKmB9G9nSHnazVF0SJiKgJZg031o7hhtpCzuVKbEvPxea0XJwtqjBsd3eS4x8DfTFlkB8G+Kk5PoeIqI20S7g5fvw4srOzUVtba7T97rvvbs3h2g3DDbUlQRDwx0UttqTl4n+/5+FSxZ+fhx4enTBlkD8mhfrC39U6btsSEVkqs4abs2fPYvLkyTh69CgkEgmu7X7tX6h8Woo6qjqdHj+dKsLm1FzsOl6Amnq94XdDu7lhytXxOWoHexGrJCKyTmYNNxMnToRMJsPHH3+Mbt264fDhw7h06RKeeeYZLF++HCNHjryp4s2N4YbaQ2l1HRKPabAlNReHzl3CtU+Z3E6KyD6emBzmj1G9PCC34/gcIqKWMGu4cXd3x549ezBw4ECo1WocPnwYwcHB2LNnD5555hmkpaXdVPHmxnBD7S2vpArb0vOwJe0iThaUG7a7OtpjYogvosP8EBbgwvE5RETNMOX72+TnV3U6HZydnQE0BJ28vDwEBweja9euyMrKal3FRDbM18UBj4/ugcdGdUdGXim2puVi2+95KCqrwfqDF7D+4AV0c++E6FA/TA7zQ5fOHJ9DRHQzTA43/fv3x++//45u3bohIiICb7zxBuRyOT788EN07979xgcg6qAkEgn6+6nR30+NeXf1xoEzl7Al9SJ2ZBTgXHEF3t59Em/vPonwrq6YHOaHfwz0gYujXOyyiYisjsm3pXbs2IGKigpMmTIFp0+fxj/+8Q+cPHkSnTt3xsaNGy1+fSneliJLU1FTjx0ZGmxJy8WB08WGJSDsZRKMCfbElEF+GNPbEwo7mbiFEhGJqN3nubl8+TJcXV2tYswAww1ZsoLSanyfnofNabnIzC81bFc72GPCQB9MDvPD4K7W8VkjImpLnMSvGQw3ZC1OaEqxJS0X29LyoCmtNmwPcHPA5FA/RIf5obuHk4gVEhG1H4abZjDckLXR6QUcOnsJm1NzkXgsHxW1f84lFRLggilhfpgY4gu3ThyfQ0S2i+GmGQw3ZM2qanXYebxhfM5Pp4qhuzpAx04qwehgD0SH+SGyjxeU9hyfQ0S2heGmGQw3ZCuKymrwv9/zsCUtF0dztYbtzgo7jB/gg8mD/DA00A1SKcfnEJH1Y7hpBsMN2aLThWXYkpaLrWl5yC2pMmz3c3HApNCGhTyDPJ1FrJCI6OYw3DSD4YZsmV4v4PD5y9iSmovtR/NRVlNv+N0APzWiw/xwd4gvPJwVIlZJRGQ6hptmMNxQR1Fdp0NSZiG2pF1EclYR6q+Oz5FJJRjZ0x2Tw/wwrq83HOQcn0NElo/hphkMN9QRXSqvQcLRfGxOzUV6Tolheye5DHf298GUQX64pXtnyDg+h4gsFMNNMxhuqKM7W1SOrVcX8sy5/Of4HG+VEpNCfTF5kB96e/OzQUSWheGmGQw3RA0EQUDKhSvYnJaLhD/yoa2qM/yuj48Kk8N8MSnUD14qpYhVEhE1YLhpBsMN0fVq6nXYe6IIW9IuYs+JQtTpGv5akEqAEUHuiA71w539vdFJYfJau0REbYLhphkMN0TNK6msRcLRfGxJzcWRC1cM2x3sZYjq54XJg/wxokdn2MmkIlZJRB0Nw00zGG6IWi77UiW2pudiS1ouzhVXGLZ7OCtwd4gvJof5oZ+vigt5EpHZMdw0g+GGyHSCICA9pwRb0nLxv9/zcKXyz/E5vbycEB3mh+hQP/i6OIhYJRHZMlO+v3ldmYhuSCKRIKyLK16Z1B+/LojEx7GDMWGAD+R2UpwsKMcbiVkYsWwP7vvwEDYdyUFZdd2ND9pCq1evRmBgIJRKJSIiInD48OEm227evBmDBw+Gi4sLOnXqhNDQUHzxxRdGbcrLyzFr1iz4+/vDwcEBffv2xdq1a9usXiISH6/cEFGraavqkHisYf6cX89dNmxX2Ekxtq8Xpgzyw8ieHrBv5ficjRs3IjY2FmvXrkVERARWrlyJb775BllZWfD09LyufXJyMq5cuYLevXtDLpfjhx9+wDPPPIOEhARERUUBAB599FHs2bMHH3/8MQIDA7Fz50488cQT2Lx5M+6+++7W/UEQkdnxtlQzGG6IzOPilUpsS8/D5tSLOFP05/iczp3kmHh1fM5Af7VJ43MiIiIwZMgQrFq1CgCg1+sREBCAJ598EvPmzWvRMQYNGoQJEybg1VdfBQD0798fMTExePHFFw1twsPDcdddd+G1115rcW1E1L54W4qI2p2/qyNmjgnC7rhR+N+sWzFjRCDcneS4VFGLdb+cx6TVB3DHW/uwas8p5FyuvOHxamtrkZKSgsjISMM2qVSKyMhIHDx48Ib7C4KApKQkZGVl4bbbbjNsHz58OL7//nvk5uZCEATs3bsXJ0+exLhx41rXcSKyOBYRbky5p/5XGzZsgEQiQXR0tHkLJKIWk0gkGOCvxuKJ/XBo/h34bMYQ3B3iC6W9FGeLKrB850mMfGMvpq49iK8PZxtNHvhXxcXF0Ol08PLyMtru5eUFjUbT5PtrtVo4OTlBLpdjwoQJeO+99zB27FjD79977z307dsX/v7+kMvluPPOO7F69WqjAERE1k30Gbk2btyIuLg4o3vqUVFRTd5Tv+b8+fN49tlnMXLkyHaslohMYSeTYkywJ8YEe6Ksug47MgqwJe0ifjlzCYfPX8bh85exeFsG7ujjiclhfhgd7Am53c39m8vZ2Rnp6ekoLy9HUlIS4uLi0L17d4wePRpAQ7g5dOgQvv/+e3Tt2hX79+/HzJkz4evra3SViIisl+hjblpzT12n0+G2227Dgw8+iJ9++gklJSXYunVri96PY26IxJevrcK29DxsSc1FVkGZYbuLoz3+MdAHk8P80d/bEZ06dcK3335rdHV2+vTpKCkpwbZt21r0Xg8//DBycnKwY8cOVFVVQa1WY8uWLZgwYYJRm4sXLyIxMbHN+khEbctqxty09p76K6+8Ak9PTzz00EM3fI+amhqUlpYavYhIXD5qBzw2qgd2PH0btj81Eo+M7AZPZwVKKuvw5aFs3LPmF0S9+wv8evbD1oQ/A4der0dSUhKGDRvW4vfS6/WoqakBANTV1aGurg5SqfFffTKZDHq9vm06R0SiE/W2VHP31E+cONHoPj///DM++eQTpKent+g94uPj8fLLL99sqURkJn19Vejr2xfz7uqDX84UY0tqLhIzNDh/qRKVve7E55+9jaM17vjX+DE4sfNrVFRUYMaMGQCA2NhY+Pn5IT4+HkDD533w4MHo0aMHampqsH37dnzxxRdYs2YNAEClUmHUqFF47rnn4ODggK5du2Lfvn1Yv3493nrrLdH+DIiobYk+5sYUZWVleOCBB/DRRx/B3d29RfvMnz8fcXFxhp9LS0sREBBgrhKJqJVkUglG9vTAyJ4eeK22HjszCrA5zQPbq7T4fduHSP1qGRRe3RH11NtILdTj9s46ZGdnG12FqaiowBNPPIGLFy/CwcEBvXv3xpdffomYmBhDmw0bNmD+/PmYNm0aLl++jK5du+L111/HY489Jka3icgMRB1zU1tbC0dHxxbfU09PT0dYWBhkMplh27VLyVKpFFlZWejRo0ez78kxN0TWpbC0Gt//noctabnIyPvztrJKaYcJAxvmzxnc1RVSqQT52iqcK65AN/dO8FFzKQgiW2JVk/hFRERg6NCheO+99wA0hJUuXbpg1qxZ1w0orq6uxunTp422LVy4EGVlZXjnnXfQq1cvyOXyZt+P4YbIep0sKMPm1FxsS89FvrbasN3f1QG9vJyRnFUIvQBIJUD8lAGIGdJFxGqJqC2Z8v0t+m2puLg4TJ8+HYMHD8bQoUOxcuXKJu+pK5VK9O/f32h/FxcXALhuOxHZnl5ezph3V288HxWMQ+cuYUtqLn48psHFK1W4eKXK0E4vAPM2H4WnsxLDgzpDYSdr5qhEZGtEDzcxMTEoKirCokWLoNFoEBoaisTERMMg47/fUycikkolGN7DHcN7uOOVSf3xfvJpvLfH+KquIAAz1v0GO6kEPb2c0c9Xhf6+KvTzU6OPjwpOCtH/+iMiMxH9tlR7420pItuTr63CiKV7oP/b32ZqpR201fXXtZdIgG6dO6Gvrwr9/dTo56tCP1813Do1f1ubiMRjVWNu2hvDDZFt2vhbNhZsPgadIEAmkWDJlP6YOjgA+dpqHMvVIiOvFBl5Df/71/E6f+WrVqLfX8JOfz8VvFVKkxb7JCLzYLhpBsMNke3K11bhfHElAt0dm31a6lJ5zdWwU4pjeVoczyvFueKKRtu6dZIbwk6/q1d6uro5Qipl4CFqTww3zWC4IaLGlFXXITO/DBl5WhzLbbjKc6qwHLq/3+sC4KSwQ18fldFtrSBPJ9jLOD6QyFwYbprBcENELVVdp8PJgjJD2MnIK0Vmfilq6q9fqkFuJ0Vvb2ejqzx9fFRQ2vNJLaK2wHDTDIYbIroZ9To9zhZXXDeOp6yRgcsyqQQ9PDoZwk4/XzX6+qqgdrAXoXIi68Zw0wyGGyJqa4IgIOdyFY7laY1uaxWX1zbavoubI/r7qYxCj4ezop2rJrIuDDfNYLghovYgCAIKy2qMwk5GXqnRZIN/5emsMHosvZ+vCv6uDnxSi+gqhptmMNwQkZhKKmtx/OpTWtee2DpTVI7G/iZWO9hfDTt/Dlzu5u4EmZU9qbV69Wq8+eab0Gg0CAkJwXvvvYehQ4c22nbz5s1YsmQJTp8+jbq6OvTs2RPPPPMMHnjgAaM2a9euRUpKCi5fvoy0tDSEhoa2U29ILFa1/AIRUUfi4ijH8CB3DA9yN2yrrK03PKmVkdsQfE4WlEFbVYdfzlzCL2cuGdo62MvQx8fZ6CpPTy8ni11iYuPGjYiLi8PatWsRERGBlStXIioqCllZWfD09LyuvZubG1544QX07t0bcrkcP/zwA2bMmAFPT09ERUUBaFj9/dZbb8XUqVPxyCOPtHeXyArwyg0RkQWqrdfjVGEZMq7e0jp29UmtylrddW3tZRL09HQ2GsfTx0eFThawxERERASGDBmCVatWAWhYHDkgIABPPvnkdYsjN2XQoEGYMGECXn31VaPt58+fR7du3XjlpoPglRsiIisnt5NeDSpqAAEAAJ1ewLniCmRcnXjw2q2tkso6HM8vxfH8UgAXAVxdYsK9E/r/ZfLBfr4quDi23xITtbW1SElJwfz58w3bpFIpIiMjcfDgwRvuLwgC9uzZg6ysLCxbtsycpZKNYbghIrISMqkEQZ5OCPJ0wqRQPwANASC3pKph/M7Vx9OP5WlRUFqDs0UVOFtUge9/zzMcw8/FwWh5iX6+anipFGYZuFxcXAydTmdYCPkaLy8vnDhxosn9tFot/Pz8UFNTA5lMhvfffx9jx45t8/rIdjHcEBFZMYlEAn9XR/i7OiKqn7dhe/HVJSaO5f55lefCpUrkllQht6QKO48XGNq6O8nR11fdsGr61Ss9XURcYsLZ2Rnp6ekoLy9HUlIS4uLi0L17d4wePVqUesj6MNwQEdkgdycFRvXywKheHoZtpdV1OH71Ca1rg5dPF5WjuLwW+08WYf/JIkNbZ4Ud+viqjG5r9fDoBDsTlphwd3eHTCZDQUGB0faCggJ4e3s3sVfDraugoCAAQGhoKDIzMxEfH89wQy3GcENE1EGolPa4pXtn3NK9s2FbdZ0OJzR/rql1PE+LTE0ZymrqcfjcZRw+d9nQVnFtiYmr43f6+6oR7O3c5BITcrkc4eHhSEpKQnR0NICGAcVJSUmYNWtWi+vW6/WoqalpXaepQ2K4ISLqwJT2MoQGuCA0wMWwrU6nx5micsNj6Rl5pcjMK0VZTT1+v6jF7xe1hrYyqQQ9PZ3Q99o4Ht+GBUWdlQ1LTMTFxWH69OkYPHgwhg4dipUrV6KiogIzZswAAMTGxsLPzw/x8fEAgPj4eAwePBg9evRATU0Ntm/fji+++AJr1qwxvOfly5eRnZ2NvLyGsURZWVkAAG9v72avCFHHwXBDRERG7GVS9PZWobe3CveE+wMA9HoB2ZcrDQOWrw1gvlRRixOaMpzQlGFzaq7hGIGdHRvG7/iF4z/Pv4SFL76IwoIChIaGIjEx0TDIODs7G1Lpn7e6Kioq8MQTT+DixYtwcHBA79698eWXXyImJsbQ5vvvvzeEIwC49957AQCLFy/GSy+9ZM4/GrISnOeGiIhaRRAEFJRev8REbknjS0x4q5QNT2oZJiBUwc/FeImJfG0VzhVXoJt7J/ioHdqrK2QFuPxCMxhuiIjM60pFrWHQ8rGr/3uuuKLRJSZcHO0N43e0VXXYdCQHegGQSoD4KQMQM6RL+3eALBLDTTMYboiI2l9FTT0y80sNj6dn5JXiZEEZ6vXNfwXd0dsT3T06wVvtAB+1Et5qJXzVDvBwVljdGlvWypS1wT766COsX78ex44dAwCEh4djyZIlRu2bmlPpjTfewHPPPddkHQw3zWC4ISKyDDX1OpwqKEdGnha7jxdiV2bBjXe6SiaVwNNZAW+1Ej5qJXz+En4a/tcBXs4Kkx5dp+tt3LgRsbGxRmuDffPNN02uDTZt2jSMGDECw4cPh1KpxLJly7BlyxZkZGTAz69h4kmNRmO0z48//oiHHnoIp0+fRvfu3ZusheGmGQw3RESWJ19bhRFL9+CvF3KkEuDJ23uisrYe+dpqaLTVyNdWo6C0+oZXfK7t7+GsaLjqo/oz+Pi4XA1CKiW8VErI7RiAmnKza4PpdDq4urpi1apViI2NbbRNdHQ0ysrKkJSU1OyxuLYUERFZFR+1A+KnDMCCzcegEwTIJBIsmdK/0TE3Or2AS+U1yL8adjTaqr/8dzXyS6ug0VajTtcw4LmgtAa/N/Pe7k4K+Lo0hJ1rV318/nJFyEutsNhV183pZtcGA4DKykrU1dXBzc2t0d8XFBQgISEBn3/+eZvUfA3DDRERWYSYIV1wWy8PnC+uRKC7Y5NPS8mkEniqlPBUKRES0Pix9HoBlypqr17tqYKmtCH85Jc0BKFrP9fW61FcXoPi8hr8AW3jBwPQuZPc6BbYn7e//rwl1tRkhtaqtWuD/dXcuXPh6+uLyMjIRn//+eefw9nZGVOmTLnpev+K4YaIiCxGQ1C4+UfApVIJPJwV8HBWYIC/utE2giDgSmUd8kqqrl7x+ctVoJJrAagK1XV6XKqoxaWrT4E1xdXR/m8Dn5VGP/uolXCUd5yv3aVLl2LDhg1ITk6GUqlstM2nn36KadOmNfn71uo4f8pERER/IZFI4NZJDrdOcvT3azoAaavqrt72qjIa+6PRViNPW4X8kmpU1elwpbIOVyrrkJnfdABSO9gbD3xWOcDHRWm4DeatdoCTwjK+mlu7NhgALF++HEuXLsXu3bsxcODARtv89NNPyMrKwsaNG9us5mss40+QiIjIAkkkErg4yuHiKEcfn8YHsQqCgNLqesMtsL+PBboWhspr6qGtqoO2qg4nNGVNvqezwq4h/LgYD4T2Vivh69JwS8xZYdfkI9VtpbVrg73xxht4/fXXsWPHDgwePLjJdp988gnCw8MREhLS1qUz3BAREd0MiUQCtYM91A72CPZ2brJdWXWd0VWf668GVaG0uh5lNfUoKyzHqcLyJo/VSS4zGu/j87dbYL5qB6gcbj4Ambo22LJly7Bo0SJ89dVXCAwMNDz27eTkBCcnJ8NxS0tL8c0332DFihU3VV9TGG6IiIjagbPSHs5Ke/T0ajoAVdTUG4Wda2OB/joQuqSyDhW1OpwpqsCZooomj+VgLzOEncbnA3KAq6N9swEoJiYGRUVFWLRoETQazQ3XBluzZg1qa2vxz3/+0+g4f1/3a8OGDRAEAffdd9+N/thahfPcEBERWZGqWl3DYGejJ78aglDe1YHQlytqW3QshZ3UKOw0FoLcHOWQmjAbtLnWB+M8N0RERDbKQS5DN/dO6Obeqck21XU6FJQ2duvrz6tCxeW1qKnX4/ylSpy/VNnkseQyKbzUCuOrPqqG22C+Lg0/u3dSQCqVYONv2Zi/+ajo64Pxyg0REVEHVFOvQ2FpTaNjf64FoaLymkYXPP07O6kE7k4KaEqrjbbLJBL8PG9Mm1zB4ZUbIiIiapbCToYAN0cEuDk22aa2Xo/Csusff//rz4VlDcth/D3YAIBOEHC+uLJNb0+1BMMNERERNUpuJ4W/qyP8XZsOQPU6PQrLanAsV4v/fJGCv17okUkkCHRvel9z4WphRERE1Gp2Mil8XRwwrp83lt4zALKrT19dWx+sva/aALxyQ0RERG2kpeuDmRvDDREREbWZtlof7GbwthQRERHZFIYbIiIisikMN0RERGRTGG6IiIjIpjDcEBERkU1huCEiIiKbwnBDRERENoXhhoiIiGwKww0RERHZFIYbIiIisikWEW5Wr16NwMBAKJVKRERE4PDhw022/eijjzBy5Ei4urrC1dUVkZGRzbYnIiKijkX0cLNx40bExcVh8eLFSE1NRUhICKKiolBYWNho++TkZNx3333Yu3cvDh48iICAAIwbNw65ubntXDkRERFZIokgCIKYBURERGDIkCFYtWoVAECv1yMgIABPPvkk5s2bd8P9dTodXF1dsWrVKsTGxt6wfWlpKdRqNbRaLVQq1U3XT0REROZnyve3qFduamtrkZKSgsjISMM2qVSKyMhIHDx4sEXHqKysRF1dHdzc3MxVJhEREVkROzHfvLi4GDqdDl5eXkbbvby8cOLEiRYdY+7cufD19TUKSH9VU1ODmpoaw8+lpaWtL5iIiIgsnuhjbm7G0qVLsWHDBmzZsgVKpbLRNvHx8VCr1YZXQEBAO1dJRERE7UnUcOPu7g6ZTIaCggKj7QUFBfD29m523+XLl2Pp0qXYuXMnBg4c2GS7+fPnQ6vVGl45OTltUjsRERFZJlHDjVwuR3h4OJKSkgzb9Ho9kpKSMGzYsCb3e+ONN/Dqq68iMTERgwcPbvY9FAoFVCqV0YuIiIhsl6hjbgAgLi4O06dPx+DBgzF06FCsXLkSFRUVmDFjBgAgNjYWfn5+iI+PBwAsW7YMixYtwldffYXAwEBoNBoAgJOTE5ycnETrBxEREVkG0cNNTEwMioqKsGjRImg0GoSGhiIxMdEwyDg7OxtS6Z8XmNasWYPa2lr885//NDrO4sWL8dJLL7Vn6URERGSBRJ/npr1xnhsiIiLrYzXz3BARERG1NYYbIiIisikMN0RERGRTGG6IiIjIpjDcEBERkU1huCEiIiKbwnBDRERENoXhhoiIiGwKww0RERHZFIYbIiIisikMN0RERGRTGG6IiIjIpjDcEBERkU1huCEiIiKbwnBDRERENoXhhoiIiGwKww0RERHZFIYbIiIisikMN0RERGRTGG6IiIjIpjDcEBERkU1huCEiIiKbwnBDRERENoXhhoiIiGwKww0RERHZFIYbIiIisikMN0RERGRTGG6IiIjIpjDcEBERkU1huCEiIiKbwnBDRERENoXhhoiIiGwKww0RERHZFIYbIiIisikMN0RERGRTGG6IiIjIpjDcEBERkU1huCEiIiKbwnBDRERENoXhhoiIiGwKww0RERHZFIYbIiIisikMN0RERGRTGG6IiIjIpjDcEBERkU1huCEiIiKbwnBDRERENoXhhoiIiGyKRYSb1atXIzAwEEqlEhERETh8+HCz7b/55hv07t0bSqUSAwYMwPbt29upUiIiIrJ0ooebjRs3Ii4uDosXL0ZqaipCQkIQFRWFwsLCRtv/8ssvuO+++/DQQw8hLS0N0dHRiI6OxrFjx9q5ciIiIrJEEkEQBDELiIiIwJAhQ7Bq1SoAgF6vR0BAAJ588knMmzfvuvYxMTGoqKjADz/8YNh2yy23IDQ0FGvXrr3h+5WWlkKtVkOr1UKlUrVdR4iIiMhsTPn+FvXKTW1tLVJSUhAZGWnYJpVKERkZiYMHDza6z8GDB43aA0BUVFST7YmIiKhjsRPzzYuLi6HT6eDl5WW03cvLCydOnGh0H41G02h7jUbTaPuamhrU1NQYftZqtQAaEiARERFZh2vf2y254SRquGkP8fHxePnll6/bHhAQIEI1REREdDPKysqgVqubbSNquHF3d4dMJkNBQYHR9oKCAnh7eze6j7e3t0nt58+fj7i4OMPPer0ely9fRufOnSGRSG6yB8ZKS0sREBCAnJwcmxzPY+v9A2y/j+yf9bP1PrJ/1s9cfRQEAWVlZfD19b1hW1HDjVwuR3h4OJKSkhAdHQ2gIXwkJSVh1qxZje4zbNgwJCUlYc6cOYZtu3btwrBhwxptr1AooFAojLa5uLi0RflNUqlUNvt/WsD2+wfYfh/ZP+tn631k/6yfOfp4oys214h+WyouLg7Tp0/H4MGDMXToUKxcuRIVFRWYMWMGACA2NhZ+fn6Ij48HAMyePRujRo3CihUrMGHCBGzYsAFHjhzBhx9+KGY3iIiIyEKIHm5iYmJQVFSERYsWQaPRIDQ0FImJiYZBw9nZ2ZBK/3yoa/jw4fjqq6+wcOFCLFiwAD179sTWrVvRv39/sbpAREREFkT0cAMAs2bNavI2VHJy8nXb/vWvf+Ff//qXmasynUKhwOLFi6+7DWYrbL1/gO33kf2zfrbeR/bP+llCH0WfxI+IiIioLYm+/AIRERFRW2K4ISIiIpvCcENEREQ2heHGRKtXr0ZgYCCUSiUiIiJw+PDhZtt/88036N27N5RKJQYMGIDt27e3U6WtY0r/1q1bB4lEYvRSKpXtWK1p9u/fj4kTJ8LX1xcSiQRbt2694T7JyckYNGgQFAoFgoKCsG7dOrPX2Vqm9i85Ofm68yeRSJpcykRs8fHxGDJkCJydneHp6Yno6GhkZWXdcD9r+gy2po/W9Dlcs2YNBg4caJj/ZNiwYfjxxx+b3ceazp+p/bOmc9eYpUuXQiKRGM071xgxziHDjQk2btyIuLg4LF68GKmpqQgJCUFUVBQKCwsbbf/LL7/gvvvuw0MPPYS0tDRER0cjOjoax44da+fKW8bU/gENkzTl5+cbXhcuXGjHik1TUVGBkJAQrF69ukXtz507hwkTJmDMmDFIT0/HnDlz8PDDD2PHjh1mrrR1TO3fNVlZWUbn0NPT00wV3px9+/Zh5syZOHToEHbt2oW6ujqMGzcOFRUVTe5jbZ/B1vQRsJ7Pob+/P5YuXYqUlBQcOXIEt99+OyZNmoSMjIxG21vb+TO1f4D1nLu/++233/DBBx9g4MCBzbYT7RwK1GJDhw4VZs6cafhZp9MJvr6+Qnx8fKPtp06dKkyYMMFoW0REhPCf//zHrHW2lqn9++yzzwS1Wt1O1bUtAMKWLVuabfP8888L/fr1M9oWExMjREVFmbGyttGS/u3du1cAIFy5cqVdamprhYWFAgBh3759Tbaxts/g37Wkj9b8ORQEQXB1dRU+/vjjRn9n7edPEJrvn7Weu7KyMqFnz57Crl27hFGjRgmzZ89usq1Y55BXblqotrYWKSkpiIyMNGyTSqWIjIzEwYMHG93n4MGDRu0BICoqqsn2YmpN/wCgvLwcXbt2RUBAwA3/hWJtrOn83YzQ0FD4+Phg7NixOHDggNjltJhWqwUAuLm5NdnG2s9hS/oIWOfnUKfTYcOGDaioqGhy+RxrPn8t6R9gnedu5syZmDBhwnXnpjFinUOGmxYqLi6GTqczzJx8jZeXV5NjFDQajUntxdSa/gUHB+PTTz/Ftm3b8OWXX0Kv12P48OG4ePFie5Rsdk2dv9LSUlRVVYlUVdvx8fHB2rVr8d133+G7775DQEAARo8ejdTUVLFLuyG9Xo85c+ZgxIgRzc5Obk2fwb9raR+t7XN49OhRODk5QaFQ4LHHHsOWLVvQt2/fRtta4/kzpX/Wdu4AYMOGDUhNTTUsiXQjYp1Di5ihmKzTsGHDjP5FMnz4cPTp0wcffPABXn31VREro5YIDg5GcHCw4efhw4fjzJkzePvtt/HFF1+IWNmNzZw5E8eOHcPPP/8sdilm09I+WtvnMDg4GOnp6dBqtfj2228xffp07Nu3r8kAYG1M6Z+1nbucnBzMnj0bu3btsviBzww3LeTu7g6ZTIaCggKj7QUFBfD29m50H29vb5Pai6k1/fs7e3t7hIWF4fTp0+Yosd01df5UKhUcHBxEqsq8hg4davGBYdasWfjhhx+wf/9++Pv7N9vWmj6Df2VKH//O0j+HcrkcQUFBAIDw8HD89ttveOedd/DBBx9c19Yaz58p/fs7Sz93KSkpKCwsxKBBgwzbdDod9u/fj1WrVqGmpgYymcxoH7HOIW9LtZBcLkd4eDiSkpIM2/R6PZKSkpq8nzps2DCj9gCwa9euZu+/iqU1/fs7nU6Ho0ePwsfHx1xltitrOn9tJT093WLPnyAImDVrFrZs2YI9e/agW7duN9zH2s5ha/r4d9b2OdTr9aipqWn0d9Z2/hrTXP/+ztLP3R133IGjR48iPT3d8Bo8eDCmTZuG9PT064INIOI5NOtwZRuzYcMGQaFQCOvWrROOHz8uPProo4KLi4ug0WgEQRCEBx54QJg3b56h/YEDBwQ7Ozth+fLlQmZmprB48WLB3t5eOHr0qFhdaJap/Xv55ZeFHTt2CGfOnBFSUlKEe++9V1AqlUJGRoZYXWhWWVmZkJaWJqSlpQkAhLfeektIS0sTLly4IAiCIMybN0944IEHDO3Pnj0rODo6Cs8995yQmZkprF69WpDJZEJiYqJYXWiWqf17++23ha1btwqnTp0Sjh49KsyePVuQSqXC7t27xepCsx5//HFBrVYLycnJQn5+vuFVWVlpaGPtn8HW9NGaPofz5s0T9u3bJ5w7d074448/hHnz5gkSiUTYuXOnIAjWf/5M7Z81nbum/P1pKUs5hww3JnrvvfeELl26CHK5XBg6dKhw6NAhw+9GjRolTJ8+3aj9pk2bhF69eglyuVzo16+fkJCQ0M4Vm8aU/s2ZM8fQ1svLSxg/fryQmpoqQtUtc+3R57+/rvVp+vTpwqhRo67bJzQ0VJDL5UL37t2Fzz77rN3rbilT+7ds2TKhR48eglKpFNzc3ITRo0cLe/bsEaf4FmisbwCMzom1fwZb00dr+hw++OCDQteuXQW5XC54eHgId9xxh+GLXxCs//yZ2j9rOndN+Xu4sZRzyFXBiYiIyKZwzA0RERHZFIYbIiIisikMN0RERGRTGG6IiIjIpjDcEBERkU1huCEiIiKbwnBDRERENoXhhoiIiGwKww0RdXjJycmQSCQoKSkRuxQiagMMN0RERGRTGG6IiIjIpjDcEJHo9Ho94uPj0a1bNzg4OCAkJATffvstgD9vGSUkJGDgwIFQKpW45ZZbcOzYMaNjfPfdd+jXrx8UCgUCAwOxYsUKo9/X1NRg7ty5CAgIgEKhQFBQED755BOjNikpKRg8eDAcHR0xfPhwZGVlmbfjRGQWDDdEJLr4+HisX78ea9euRUZGBp5++mn8+9//xr59+wxtnnvuOaxYsQK//fYbPDw8MHHiRNTV1QFoCCVTp07Fvffei6NHj+Kll17Ciy++iHXr1hn2j42Nxddff413330XmZmZ+OCDD+Dk5GRUxwsvvIAVK1bgyJEjsLOzw4MPPtgu/SeitsVVwYlIVDU1NXBzc8Pu3bsxbNgww/aHH34YlZWVePTRRzFmzBhs2LABMTExAIDLly/D398f69atw9SpUzFt2jQUFRVh586dhv2ff/55JCQkICMjAydPnkRwcDB27dqFyMjI62pITk7GmDFjsHv3btxxxx0AgO3bt2PChAmoqqqCUqk0858CEbUlXrkhIlGdPn0alZWVGDt2LJycnAyv9evX48yZM4Z2fw0+bm5uCA4ORmZmJgAgMzMTI0aMMDruiBEjcOrUKeh0OqSnp0Mmk2HUqFHN1jJw4EDDf/v4+AAACgsLb7qPRNS+7MQugIg6tvLycgBAQkIC/Pz8jH6nUCiMAk5rOTg4tKidvb294b8lEgmAhvFARGRdeOWGiETVt29fKBQKZGdnIygoyOgVEBBgaHfo0CHDf1+5cgUnT55Enz59AAB9+vTBgQMHjI574MAB9OrVCzKZDAMGDIBerzcaw0NEtotXbohIVM7Oznj22Wfx9NNPQ6/X49Zbb4VWq8WBAwegUqnQtWtXAMArr7yCzp07w8vLCy+88ALc3d0RHR0NAHjmmWcwZMgQvPrqq4iJicHBgwexatUqvP/++wCAwMBATJ8+HQ8++CDeffddhISE4MKFCygsLMTUqVPF6joRmQnDDRGJ7tVXX4WHhwfi4+Nx9uxZuLi4YNCgQViwYIHhttDSpUsxe/ZsnDp1CqGhofjf//4HuVwOABg0aBA2bdqERYsW4dVXX4WPjw9eeeUV/N///Z/hPdasWYMFCxbgiSeewKVLl9ClSxcsWLBAjO4SkZnxaSkismjXnmS6cuUKXFxcxC6HiKwAx9wQERGRTWG4ISIiIpvC21JERERkU3jlhoiIiGwKww0RERHZFIYbIiIisikMN0RERGRTGG6IiIjIpjDcEBERkU1huCEiIiKbwnBDRERENoXhhoiIiGzK/wOA1bRvpRx+WAAAAABJRU5ErkJggg==", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "y_avg = []\n", - "for y in epochs_y:\n", - " y_avg.append(np.mean(y))\n", - "\n", - "plt.plot(np.arange(len(epochs_x)), y_avg, marker = '.')\n", - "plt.xlabel('epoch')\n", - "plt.ylabel('average loss')\n", - "plt.ylim(0,)\n", - "for i, txt in enumerate(y_avg):\n", - " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom')\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAG2CAYAAACZEEfAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABPmklEQVR4nO3deVxU5f4H8M+wDQMCKsgyiIKg4q64EGqLQeFyTdPrltcNt1v2S9Q0tchMDfVeW8xyKcV9K5cWbxJaahouoJhbKqiAypIgM6wDzJzfH+jkKCCDM8zM8fN+veaVc+Y5h+/TkebTOc95HokgCAKIiIiIRMrK1AUQERERGRPDDhEREYkaww4RERGJGsMOERERiRrDDhEREYkaww4RERGJGsMOERERiRrDDhEREYkaww4RERGJGsMOERERiZpJw86RI0fQv39/yOVySCQS7N27V+dzQRDw/vvvw8vLCzKZDGFhYbh69apOm9zcXIwcORLOzs6oX78+xo8fj4KCgjrsBREREZkzk4adwsJCdOjQAV988UWlny9duhTLly/HqlWrcOLECTg6OiI8PBwlJSXaNiNHjsSFCxcQFxeHH3/8EUeOHMGkSZPqqgtERERk5iTmshCoRCLBnj17MHDgQAAVV3XkcjlmzJiBt99+GwCgUCjg4eGB9evXY/jw4bh06RJat26NU6dOoUuXLgCA/fv3o2/fvrh58ybkcrmpukNERERmwsbUBVTl+vXryMzMRFhYmHabi4sLgoODER8fj+HDhyM+Ph7169fXBh0ACAsLg5WVFU6cOIFXX3210mOrVCqoVCrte41Gg9zcXLi6ukIikRivU0RERGQwgiAgPz8fcrkcVlZV36wy27CTmZkJAPDw8NDZ7uHhof0sMzMT7u7uOp/b2NigYcOG2jaViY6Oxvz58w1cMREREZlCeno6GjduXOXnZht2jGnOnDmYPn269r1CoUCTJk2Qnp4OZ2dnE1ZGRERENaVUKuHj4wMnJ6dq25lt2PH09AQAZGVlwcvLS7s9KysLHTt21LbJzs7W2a+8vBy5ubna/SsjlUohlUof2e7s7MywQ0REZGEeNwTFbOfZ8fPzg6enJw4ePKjdplQqceLECYSEhAAAQkJCkJeXh8TERG2bX375BRqNBsHBwXVeMxEREZkfk17ZKSgoQHJysvb99evXkZSUhIYNG6JJkyaIjIzEwoUL0bx5c/j5+SEqKgpyuVz7xFarVq3Qu3dvTJw4EatWrUJZWRnefPNNDB8+nE9iEREREQATh52EhAT06tVL+/7+OJoxY8Zg/fr1mDVrFgoLCzFp0iTk5eWhZ8+e2L9/P+zt7bX7bNmyBW+++SZCQ0NhZWWFwYMHY/ny5XXeFyIiIjJPZjPPjikplUq4uLhAoVBwzA4REZGFqOn3t9mO2SEiIiIyBIYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1hh0iIiISNYYdIiIiEjWGHSIiIhI1sw87+fn5iIyMRNOmTSGTydC9e3ecOnVK+/nYsWMhkUh0Xr179zZhxURERGRObExdwONMmDAB58+fx6ZNmyCXy7F582aEhYXh4sWL8Pb2BgD07t0bMTEx2n2kUqmpyiUiIiIzY9ZXdoqLi7Fr1y4sXboUzz33HAICAvDBBx8gICAAK1eu1LaTSqXw9PTUvho0aGDCqomIiMicmHXYKS8vh1qthr29vc52mUyGo0ePat8fOnQI7u7uaNmyJV5//XXk5ORUe1yVSgWlUqnzIiIiInEy67Dj5OSEkJAQLFiwALdv34ZarcbmzZsRHx+PjIwMABW3sDZu3IiDBw9iyZIlOHz4MPr06QO1Wl3lcaOjo+Hi4qJ9+fj41FWXiIiIqI5JBEEQTF1EdVJSUhAREYEjR47A2toaQUFBaNGiBRITE3Hp0qVH2l+7dg3+/v44cOAAQkNDKz2mSqWCSqXSvlcqlfDx8YFCoYCzs7PR+kJERESGo1Qq4eLi8tjvb7O+sgMA/v7+OHz4MAoKCpCeno6TJ0+irKwMzZo1q7R9s2bN4ObmhuTk5CqPKZVK4ezsrPMiIiIicTL7sHOfo6MjvLy8cPfuXcTGxmLAgAGVtrt58yZycnLg5eVVxxUSERGROTL7R89jY2MhCAJatmyJ5ORkzJw5E4GBgRg3bhwKCgowf/58DB48GJ6enkhJScGsWbMQEBCA8PBwU5dOREREZsDsr+woFApMmTIFgYGBGD16NHr27InY2FjY2trC2toaf/zxB1555RW0aNEC48ePR+fOnfHbb79xrh0iIiICYAEDlOtCTQc4ERERkfkQzQBlIiIioifBsENERESixrBDREREosawQ0RERKLGsENERESixrBDREREosawQ0RERKLGsENERESixrBDREREosawQ0RERKLGsENERESVys/PR2RkJJo2bQqZTIbu3bvj1KlTAICysjK88847aNeuHRwdHSGXyzF69Gjcvn37sce9desW/vWvf8HV1RUymQzt2rVDQkKC9vPdu3fj5ZdfhqurKyQSCZKSkp6oHww7REREVKkJEyYgLi4OmzZtwrlz5/Dyyy8jLCwMt27dQlFREU6fPo2oqCicPn0au3fvxuXLl/HKK69Ue8y7d++iR48esLW1xU8//YSLFy9i2bJlaNCggbZNYWEhevbsiSVLlhikH1wIFFwIlIiI6GHFxcVwcnLCd999h379+mm3d+7cGX369MHChQsf2efUqVPo1q0bUlNT0aRJk0qPO3v2bBw7dgy//fbbY2u4ceMG/Pz8cObMGXTs2PGRz7kQKBEREdVaeXk51Go17O3tdbbLZDIcPXq00n0UCgUkEgnq169f5XG///57dOnSBUOGDIG7uzs6deqEr776ypClP4Jhh4iIiB7h5OSEkJAQLFiwALdv34ZarcbmzZsRHx+PjIyMR9qXlJTgnXfewYgRI6q9ynLt2jWsXLkSzZs3R2xsLF5//XW89dZb2LBhg9H6wrBDREREldq0aRMEQYC3tzekUimWL1+OESNGwMpKNz6UlZVh6NChEAQBK1eurPaYGo0GQUFB+Oijj9CpUydMmjQJEydOxKpVq4zWD4YdIiIiqpS/vz8OHz6MgoICpKen4+TJkygrK0OzZs20be4HndTUVMTFxT127KuXlxdat26ts61Vq1ZIS0szSh8Ahh0iIiJ6DEdHR3h5eeHu3buIjY3FgAEDAPwddK5evYoDBw7A1dX1scfq0aMHLl++rLPtypUraNq0qVFqBxh2iIjISKqbowWo3VwqZWVl+PDDD+Hv7w97e3t06NAB+/fvr7L94sWLIZFIEBkZaYAePX1iY2Oxf/9+XL9+HXFxcejVqxcCAwMxbtw4lJWV4Z///CcSEhKwZcsWqNVqZGZmIjMzE6WlpdpjhIaGYsWKFdr306ZNw/Hjx/HRRx8hOTkZW7duxZo1azBlyhRtm9zcXCQlJeHixYsAgMuXLyMpKQmZmZm164hAgkKhEAAICoXC1KUQEYnG0KFDhdatWwuHDx8Wrl69KsybN09wdnYWbt68KQiCIGzcuFGYP3++8NVXXwkAhDNnzjz2mLNmzRLkcrmwb98+ISUlRfjyyy8Fe3t74fTp04+0PXnypODr6yu0b99emDp1qoF793TYsWOH0KxZM8HOzk7w9PQUpkyZIuTl5QmCIAjXr18XAFT6+vXXX7XHaOzTRIj4v5nC7bwi7bYffvhBaNu2rSCVSoXAwEBhzZo1Oj83Jiam0uPOmzdPp11Nv785zw44zw4RkaHpM0fL4+ZSeZBcLse7776rcxVg8ODBkMlk2Lx5s3ZbQUEBgoKC8OWXX2LhwoXo2LEjPv30U4P1j2pmx6k0zNl9DhoBsJIA0YPaYVjXyuffqY2afn/bGOwnEhER3VObOVpqQqVS1eiYU6ZMQb9+/RAWFlbp5Hf0ZMrVGiiKy5BXXIa8olLcLSzD3aJS5BWVIa+4FHeLypChKMavf/6l3UcjAHN3n8dzLRrBy0VWp/Uy7BARkcE9OEdLq1at4OHhgW3btiE+Ph4BAQG1Pm54eDg+/vhjPPfcc/D398fBgwexe/duqNVqbZvt27fj9OnTOuODqHKCIKBAVV4RUooqAos2tBTdDzAV4SWvqBR5xWW4W1gKZUl5rX6eWhBw404Rww4REYnDpk2bEBERAW9vb1hbWyMoKAgjRoxAYmJirY/52WefYeLEiQgMDIREIoG/vz/GjRuHdevWAQDS09MxdepUxMXFPXIFSOxU5eoqQsq9qy8PhpaiMtwtKoOiuBRl6tqPZnGyt0EDBzvUd7BFfQc7NHCwRQMHO7jIbGElAT49cBUPHt1aIoGvm8OTd1ZPDDtERGQU9+doKSwshFKphJeXF4YNG6YzR4u+GjVqhL1796KkpAQ5OTmQy+WYPXu29piJiYnIzs5GUFCQdh+1Wo0jR45gxYoVUKlUsLa2fuK+GZNaI0B57xaRNrTcu02kKC57JLTcDzdFperHH7wKUhurB0KL7b0/V4SXv4PM/ff32slsYWNd/UPdni72mLv7PNSCAGuJBB8NalvnV3UAhh0iIjIyR0dHODo6audoWbp06RMf097eHt7e3igrK8OuXbswdOhQABWPOZ87d06n7bhx4xAYGIh33nmnToOOIAgoKlVXelvo/pWVB6+43A8yiuIy1PbRISsJtGGkgYMd6sseuOLiWHHF5cHQ0sDRFvVldpDZGeffy7CuTfBci0a4cacIvm4OJgk6AMMOEREZSWxsLARBQMuWLZGcnIyZM2dq52gBKuZSSUtLw+3btwFAO9Gcp6cnPD09AQCjR4+Gt7c3oqOjAQAnTpzArVu30LFjR9y6dQsffPABNBoNZs2aBaBirFDbtm116nB0dISrq+sj2/VRWq5BXnEpFPdCSnW3iRRFfw/WLVVrav0z60ltHrjKYvvAlZYqrrjI7OBkbwMrK0mtf6YxeLnITBZy7mPYISIio1AoFJgzZw5u3ryJhg0bYvDgwVi0aBFsbW0BVKx+fT/4AMDw4cMBAPPmzcMHH3wAAEi+dgN3CkuRoSiGl4sMJSUleO+993Dt2jXUq1cPffv2xaZNm6pdZftBGo2A/JLyijDy0G0i7ZNFD4SX+1dkClS1G5ALAHbWVg+FFt3bRA0c7ODioHvFxUVmCzsbzvtrKJxnB5xnh4jIHD1ujpZinVtEpTpXXB6+TVQRZCrea2r5rSeRQHsbqLLbRPUd/77CUv/ebaMGDraQ2VpDIjGvqy1iUdPvb4YdMOwQEZmTAlU5Em7kYlzMKTz8BeXfyBGFqoqQoyqv/S0iRztr3bEtNRiY62xva3a3iJ52nFSQiIjMlqK4DKk5hbiRU4TUO/f+ee/9nQJVlful/FWo897GSqJzO6iqMS71H7hN5OJgC6mNeT+RRYbFsENEREaRV1SK63cKkZpThBs5uv/MLSytdt/6MlvkFZfpbJNIgE+HdUQzt3ra20SOdrxFRI/HsENERLUiCAJyC0txI6cIN+4U/n2l5t4/FQ+FlYc1cpLC19UBvq6O8HVzRNN7f27i6gBne1vsOJX2yBwtAzp611HvSEwYdoiIqEqCIOCvAhVSc4ruXaX5O9Ck3ilC/mOeUvJ0tkdTVwf4uTmiqasjfF0d0NS1Itg4Sqv/CjKXOVrI8jHsEJHZys/PR1RUFPbs2YPs7Gx06tQJn332Gbp27Qqg4ot43rx5+Oqrr5CXl4cePXpg5cqVaN68ebXHvXXrFt555x389NNPKCoqQkBAAGJiYtClSxcAwNixY7FhwwadfcLDw7F//37jdNTENBoB2fkq3MgpxI2Hxs+k5hRWOzOvRALIXWRoei/E+Lk53As1jmjS0OGJJ6szhzlayPIx7BCR2ZowYQLOnz+PTZs2QS6XY/PmzQgLC8PFixfh7e2NpUuXYvny5diwYQP8/PwQFRWF8PBwXLx4scp1ke7evYsePXqgV69e+Omnn9CoUSNcvXoVDRo00GnXu3dvxMTEaN9LpVKj9tXYNBoBGcoSpN4pxPX742fujadJzS1ESVnVTzZZSQDvBjL4uv59q6ni1pMDGjdwgL0tB/uSeeOj5+Cj50TmqLi4GE5OTvjuu+/Qr18/7fbOnTujT58+WLBgAeRyOWbMmIG3334bQMUkdh4eHli/fr12grqHzZ49G8eOHcNvv/1W5c8eO3Ys8vLysHfvXoP2ydjK1RpkKEoqrtA8NI4mLbcIpdU8qm1tJYFPA5nOrSa/e+NoGjdw4AR3ZJb46DkRWbTy8nKo1epHrtDIZDIcPXoU169fR2ZmJsLCwrSfubi4IDg4GPHx8VWGne+//x7h4eEYMmQIDh8+DG9vb7zxxhuYOHGiTrtDhw7B3d0dDRo0wIsvvoiFCxfC1dXV8B3VU5lag1t3i7VPNd0fR5OaU4T0u0XVrmBtay2BT0MH3Ss0bhXhRl5fBtvHLOpIZKkYdojILDk5OSEkJAQLFixAq1at4OHhgW3btiE+Ph4BAQHIzMwEAHh4eOjs5+Hhof2sMteuXcPKlSsxffp0zJ07F6dOncJbb70FOzs7jBkzBkDFLaxBgwbBz88PKSkpmDt3Lvr06YP4+Pg6WUiytFyD9Lv3xs3cKdJeqUnNKcTNu8VQVzMFsJ2NFZo2dPj7Cs29MOPr6gh5fRmsOSkePYUYdojIbG3atAkRERHw9vaGtbU1goKCMGLECCQmJtb6mBqNBl26dMFHH30EAOjUqRPOnz+PVatWacPOg1eF2rVrh/bt28Pf3x+HDh1CaGjok3XqnpIyNdJzi7Qh5sH5aG7nFVe7pIG9rZXO1ZkHg42Xsz1n+SV6CMMOEZktf39/HD58GIWFhVAqlfDy8sKwYcPQrFkz7arYWVlZ8PLy0u6TlZWFjh07VnlMLy8vtG7dWmdbq1atsGvXrir3adasGdzc3JCcnKxX2CkuVSM1t+LqTOpD42gylCWobsSkg521dhDwg+NofF0d4eEs5UR6RHpg2CEis+fo6AhHR0fcvXsXsbGxWLp0Kfz8/ODp6YmDBw9qw41SqcSJEyfw+uuvV3msHj164PLlyzrbrly5gqZNm1a5z82bN5GTk6MTqu4rUJVrx8zcuDf3TMXTToXIUla97AEAOEltdCbTa+rqoH3fqB4DDZGhMOwQkdmKjY2FIAho2bIlkpOTMXPmTAQGBmLcuHGQSCSIjIzEwoUL0bx5c+2j53K5HAMHDtQeIzQ0FK+++irefPNNAMC0adPQvXt3fPTRRxg6dChOnjyJNWvWYM2aNQCAgoICzJ8/H4MHD4anpydSUlIwY+ZMNPFthnKvdljxy1XtFZrHreMEVKySfX8QsO4VGgc0dLRjoCGqAww7RGS2FAoF5syZg5s3b6Jhw4YYPHgwFi1aBFtbWwDArFmzUFhYiEmTJiEvLw89e/bE/v37dZ7gunI1GWeupCFDUQwvFxm6du2KPXv2YM6cOfjwww/h5+eHTz/9FP1eHYKk9DxcuXUHP/x6HF+sWYuSwnzYOrnCrmlH1O83HdN3Xay0zoaOdhWzBN8fP/PAraf6DnZ18u+KiKrGeXbAeXaIxGrHqTTM2X0OGqFiYrx3+7ZCxyYNdNdwulOzdZzc6t1bx8lNd/xME1cHuMhs66hHRPQgzrNDRE+1tNxCzN59TjsIWCMAC/ZdqnYfD2dpxWR6ro5o6vb3OJqmro6o95h1nIjIfPG3l4hE5WpWPr5JvIntJ9MrfdrJ1dEWLTyc4ev2wGPbbg5o0tABDnb8TyKRGPE3m4gsnrKkDD+ezcDOhHQkpedV2c5KAvz41rNcWJLoKcOwQ0QWSaMRcPx6Dr5JuImfzmdoF7K0tpKgV0t3DO3SGHcKVIjaewFqQYC1RIKPBrVl0CF6CjHsEJFFuXm3CLsSb+GbxHTcvFus3d7cvR6GdGmMgZ284e7099NYvQLdceNOEXzdHBh0iJ5SZh928vPzERUVhT179iA7OxudOnXCZ599hq5duwIABEHAvHnz8NVXXyEvLw89evTAypUr0bx5cxNXTkSGUlKmRuyFTHyTcBPHUu5ox+I4SW3Qv6McQzo3Rkef+pXOWePlImPIIXrKmX3YmTBhAs6fP49NmzZBLpdj8+bNCAsLw8WLF+Ht7Y2lS5di+fLl2LBhg3ZSsfDwcFy8ePGR1ZKJyHIIgoA/biqwMyEd35+9jfyScu1n3f1dMbSLD8LbeEJmZ/yFOYnIspn1PDvFxcVwcnLCd999h379+mm3d+7cGX369MGCBQsgl8sxY8YMvP322wAqJiHz8PDA+vXrdRbzqw7n2SEyH3cKVNh75hZ2JqTjSlaBdrt3fRn+2bkx/tm5MXwaOpiwQiIyF6KYZ6e8vBxqtfqRKzQymQxHjx7F9evXkZmZibCwMO1nLi4uCA4ORnx8fJVhR6VSQaX6e4p3pVJpnA4QUY2UqTU4dPkvfJOQjl/+zEb5vSW/pTZW6NPWE0O6+CCkmStX8yaiWjHrsOPk5ISQkBAsWLAArVq1goeHB7Zt24b4+HgEBAQgMzMTAODh4aGzn4eHh/azykRHR2P+/PlGrZ2IHu/+nDi7T9/SWWOqg099DO3SGP9oL+fsxET0xMw67ADApk2bEBERAW9vb1hbWyMoKAgjRoxAYmJirY85Z84cTJ8+XfteqVTCx8fHEOUS0WNUNSeOWz07vNrJG0O6+KCFh5PpCiQi0TH7sOPv74/Dhw+jsLAQSqUSXl5eGDZsGJo1awZPT08AQFZWFry8vLT7ZGVloWPHjlUeUyqVQiqVGrt0IrqnJnPi9Ap0h621lYkrJSIxMvuwc5+joyMcHR1x9+5dxMbGYunSpfDz84OnpycOHjyoDTdKpRInTpzA66+/btqCiQg37xbh28Sb+DbxZo3mxCEiMgazDzuxsbEQBAEtW7ZEcnIyZs6cicDAQIwbNw4SiQSRkZFYuHAhmjdvrn30XC6XY+DAgaYuneipdH9OnJ0J6fg9JUevOXGIiIzB7MOOQqHAnDlzcPPmTTRs2BCDBw/GokWLYGtbMWhx1qxZKCwsxKRJk5CXl4eePXti//79nGOHqA4JgoCzNxX4hnPiEJEZMut5duoK59khqp2/8ivmxPkmkXPiEFHdE8U8O0Rkfu7PibMzIR2/ck4cIrIADDtEVCOcE4eILBXDDhFViXPiEJEYMOwQkQ6NRsDxazn4JpFz4hCRODDsEBEAzolDROLFsEP0FOOcOET0NGDYIXrKVDcnTo8AVwzpzDlxiEhcGHaInhLVzYkzpEtjDA7inDhEJE4MO0Qi9rg5cYZ28cEznBOHiERO77Dz66+/olevXsaohYgMhHPiEBH9Te+w07t3bzRu3Bjjxo3DmDFj4OPjY4y6iEhPnBOHiKhyeoedW7duYdOmTdiwYQPmz5+PF198EePHj8fAgQNhZ2dnjBqJqArVzYnzYqA7hnTmnDhERE+0EOjp06cRExODbdu2AQBee+01jB8/Hh06dDBYgXWBC4GSpaluTpyhXXwwsJM3GjlJTVghEZHx1fT7+4lXPb99+zbWrFmDxYsXw8bGBiUlJQgJCcGqVavQpk2bJzl0nWHYIUvwuDlxhnbxQYfGLpwTh4ieGkZd9bysrAzfffcd1q1bh7i4OHTp0gUrVqzAiBEj8Ndff+G9997DkCFDcPHixVp3gIg4Jw4RkSHofWXn//7v/7Bt2zYIgoBRo0ZhwoQJaNu2rU6bzMxMyOVyaDQagxZrLLyyQ+aGc+IQET1eTb+/9R61ePHiRXz++ee4ffs2Pv3000eCDgC4ubnh119/1ffQRAajVqsRFRUFPz8/yGQy+Pv7Y8GCBXgw20skkkpf//nPf6o87pEjR9C/f3/I5XJIJBLs3bu30naXLl3CK6+8AhcXFzg6OqJr165IS0urtuYytQZxF7MwcWMCQqIPYtH/LuFKVgGkNlYY2FGOrROC8dusXogMa8GgQ0SkB71vYx08ePDxB7WxwfPPP1+rgogMYcmSJVi5ciU2bNiANm3aICEhAePGjYOLiwveeustAEBGRobOPj/99BPGjx+PwYMHV3ncwsJCdOjQARERERg0aFClbVJSUtCzZ0+MHz8e8+fPh7OzMy5cuAB7+8oX0eScOERExqX3bazo6Gh4eHggIiJCZ/u6devw119/4Z133jFogXWBt7HE5x//+Ac8PDywdu1a7bbBgwdDJpNh8+bNle4zcOBA5Ofn1yjQAxVXhvbs2YOBAwfqbB8+fDhsbW2xadOmKvflnDhERE/OaLexVq9ejcDAwEe2t2nTBqtWrdL3cERG0b17dxw8eBBXrlwBAJw9exZHjx5Fnz59Km2flZWFffv2Yfz48U/0czUaDfbt24cWLVogPDwc7u7uCA4Oxt69e6HRCPg9+Q6m7UhCt0UHMHfPOSSl58HaSoKXWntgzajOiJ8Tinf7tWbQISIyIL1vY2VmZsLLy+uR7Y0aNXrktgCRqcyePRtKpRKBgYGwtraGWq3GokWLMHLkyErbb9iwAU5OTlXemqqp7OxsFBQUYPHixVi4cCGWLFmCHXu+x6BBg9Bm4jLkN2ihbcs5cYiI6obeYcfHxwfHjh2Dn5+fzvZjx45BLpcbrDCiJ7Fz505s2bIFW7duRZs2bZCUlITIyEjI5XKMGTPmkfbr1q3DyJEjqxxXU1P3n0D8R/9X0KzXUPwnIR2/l3aGvX9XXPttL5oNmcs5cYiI6pjeYWfixImIjIxEWVkZXnzxRQAVg5ZnzZqFGTNmGLxAotqYOXMmZs+ejeHDhwMA2rVrh9TUVERHRz8Sdn777TdcvnwZO3bseKKfKQgCbhXbwMraBoey7XBie5L2M1//FhAy/8TJd8M4Jw4RUR3TO+zMnDkTOTk5eOONN1BaWgoAsLe3xzvvvIM5c+YYvECi2igqKoKVle6QNGtr60rnflq7di06d+5c62VOHp4Tx9YjAIXZ6Qh4YE6ct8Z/DVnr5gw6REQmoHfYkUgkWLJkCaKionDp0iXIZDI0b94cUinHHJD56N+/PxYtWoQmTZqgTZs2OHPmDD7++ONHniJUKpX45ptvsGzZskqPExoaildffRVvvvkmAKCgoADJyckoU1eEpsU7j+DWD+kQpPVg4+wOqY0VwoZPQOznczC6QTJCfZvgu63r8MMPP+DQoUNG7TMREVVBIEGhUAgABIVCYepSyECUSqUwdepUoUmTJoK9vb3QrFkz4d133xVUKpVOu9WrVwsymUzIy8ur9DiNfZoIEf83U7idVyQIgiBs2rVPAPDIyye4j7D5+A0hr6hUEARBWLt2rRAQECDY29sLHTp0EPbu3WvcDhMRPYVq+v1dq4VAExISsHPnTqSlpWlvZd23e/fuJ09gdYzz7FBldpxKw5zd56ARAAmAxg1kSH9ghXHOiUNEZFpGWwh0+/btGD16NMLDw/Hzzz/j5ZdfxpUrV5CVlYVXX331iYomMhcZimJt0AEqLt+k3y2GlQQIbeWBIZ0bo1egO2yt9Z6qioiI6pjeYeejjz7CJ598gilTpsDJyQmfffYZ/Pz8MHny5Ern3yGyRH9mKLVB50FfjgxC77b8e05EZEn0/t/SlJQU9OvXDwBgZ2eHwsJCSCQSTJs2DWvWrDF4gUR1Lb+kDJ8dSH5ku7VEgg4+9eu+ICIieiJ6h50GDRogPz8fAODt7Y3z588DAPLy8lBUVGTY6ojqmKKoDP9aexJJN/MgtbGC1b05/6wlEnw0qC28XGSmLZCIiPSm922s5557DnFxcWjXrh2GDBmCqVOn4pdffkFcXBxCQ0ONUSNRncgpUGHU2pO4mKFEfQdbbIoIhpuTHW7cKYKvmwODDhGRhdL7aazc3FyUlJRALpdDo9Fg6dKl+P3339G8eXO89957aNCggbFqNRo+jUXZyhKM/PoErmYXwK2eFFsmBKOlJ5+wIiIyZ0Z5Gqu8vBw//vgjwsPDAQBWVlaYPXv2k1VKZGK38oox8qvjuJFTBE9ne2yZGAz/RvVMXRYRERmIXmN2bGxs8O9//xslJSXGqoeoTqXlFGHoqnjcyClC4wYy7JwcwqBDRCQyeg9Q7tatG5KSkoxQClHdSs4uwJDVv+NWXjH83Byxc3IImrg6mLosIiIyML0HKL/xxhuYPn060tPT0blzZzg6Oup83r59e4MVR2Qsf2Yq8a+vT+BOQSlaeNTD5gnBcHeyN3VZRERkBHoPUH54JWmgYnFQQRAgkUigVqsNVlxd4QDlp8u5mwqMWncCeUVlaO3ljM0TgtHQ0c7UZRERkZ6MtlzE9evXn6gwIlNKTM3F2HWnkK8qR0ef+tgwrhtcHGxNXRYRERmR3mGnadOmxqiDyOh+T7mDCRsSUFSqRje/hlg3tivqSfX+FSAiIguj93/pN27cWO3no0ePrnUxRMZy6HI2Jm9KhKpcg2ebu2HNqC6Q2VmbuiwiIqoDeo/ZeXjSwLKyMhQVFcHOzg4ODg7Izc01aIF1gWN2xC32Qibe3HoaZWoBoYHu+GJkEOxtGXSIiCxdTb+/9X70/O7duzqvgoICXL58GT179sS2bdueqGgiQ/vh7G28saUi6PRt54mV/+rMoENE9JTRO+xUpnnz5li8eDGmTp1qiMMRGcQ3CemYuv0M1BoBr3byxvLhnWBnY5C/8kREZEEMNjrTxsYGt2/fNtThiJ7IpuOpiNp7HgAwopsPFg1sB6v7S5gTEdFTRe+w8/333+u8FwQBGRkZWLFiBXr06GGwwohq6+vfrmHhvksAgLHdfTGvf2tIJAw6RERPK73DzsCBA3XeSyQSNGrUCC+++CKWLVtmqLqIamXFL1fx35+vAAD+/bw/3undkkGHiOgpp3fY0Wg0xqiD6IkIgoD//nwZX/yaAgCYFtYCb4UGMOgQEZHhxuwQmYogCFi47xLWHq2Y3Xtu30BMes7fxFUREZG50PvRlMGDB2PJkiWPbF+6dCmGDBlikKKIakqjEfDe3vPaoPPhgDYMOkREpEPvsHPkyBH07dv3ke19+vTBkSNHDFIUUU2oNQJmfvsHtpxIg0QCLB3cHqNDfE1dFhERmRm9w05BQQHs7B5dIdrW1hZKpdIgRd2nVqsRFRUFPz8/yGQy+Pv7Y8GCBXhw0uexY8dCIpHovHr37m3QOsj8lKk1mLr9DHadvglrKwk+HdYRQ7v6mLosIiIyQ3qP2WnXrh127NiB999/X2f79u3b0bp1a4MVBgBLlizBypUrsWHDBrRp0wYJCQkYN24cXFxc8NZbb2nb9e7dGzExMdr3UqnUoHWQeVGVq/Hm1jOIu5gFW2sJPh/RCb3bepm6LCIiMlN6h52oqCgMGjQIKSkpePHFFwEABw8exLZt2/DNN98YtLjff/8dAwYMQL9+/QAAvr6+2LZtG06ePKnTTiqVwtPT06A/m8xTcakakzcn4siVv2BnY4XV/+qMXoHupi6LiIjMmN63sfr374+9e/ciOTkZb7zxBmbMmIGbN2/iwIEDj8zB86S6d++OgwcP4sqVinlTzp49i6NHj6JPnz467Q4dOgR3d3e0bNkSr7/+OnJycqo9rkqlglKp1HmR+StUlWPc+pM4cuUvyGytETO2K4MOERE9lt6rntcljUaDuXPnYunSpbC2toZarcaiRYswZ84cbZvt27fDwcEBfn5+SElJwdy5c1GvXj3Ex8fD2rryBR8/+OADzJ8//5HtXPXcfClLyjB23UmcTstDPakNYsZ1RVffhqYui4iITKimq57rHXZOnToFjUaD4OBgne0nTpyAtbU1unTpUruKK7F9+3bMnDkT//nPf9CmTRskJSUhMjISH3/8McaMGVPpPteuXYO/vz8OHDiA0NDQStuoVCqoVCrte6VSCR8fH4YdM3W3sBSj153EuVsKuMhssTGiGzr41Dd1WUREZGI1DTt638aaMmUK0tPTH9l+69YtTJkyRd/DVWvmzJmYPXs2hg8fjnbt2mHUqFGYNm0aoqOjq9ynWbNmcHNzQ3JycpVtpFIpnJ2ddV5knv7KV2H4muM4d0uBho522DbxGQYdIiLSi94DlC9evIigoKBHtnfq1AkXL140SFH3FRUVwcpKN49ZW1tXu2TFzZs3kZOTAy8vPp1j6TIUxRj51Qlcu1MIdycptkwIRnMPJ1OXRUREFkbvKztSqRRZWVmPbM/IyICNjWFXn+jfvz8WLVqEffv24caNG9izZw8+/vhjvPrqqwAq5vyZOXMmjh8/jhs3buDgwYMYMGAAAgICEB4ebtBaqG6l5xZh6Op4XLtTCO/6MuycHMKgQ0REtaL3mJ0RI0YgIyMD3333HVxcXAAAeXl5GDhwINzd3bFz506DFZefn4+oqCjs2bMH2dnZkMvlGDFiBN5//33Y2dmhuLgYAwcOxJkzZ5CXlwe5XI6XX34ZCxYsgIeHR41/Tk3v+VHduH6nEK99dRwZihI0aeiArROD0biBg6nLIiIiM2O0Acq3bt3Cc889h5ycHHTq1AkAkJSUBA8PD8TFxcHHx/JmsWXYMR9XsvIx8usT+CtfBf9Gjtgy4Rl4utibuiwiIjJDNf3+1vu+k7e3N/744w9s2bIFZ8+ehUwmw7hx4zBixAjY2to+UdH0dDt/S4HR604it7AUgZ5O2DwhGG71OBs2ERE9mVoNsnF0dMSkSZMMXQs9xc6k3cWYdSehLClH+8Yu2BjRDfUdHl2DjYiISF+1HlF88eJFpKWlobS0VGf7K6+88sRF0dPlxLUcRKw/hcJSNbo0bYB147rC2Z5XCYmIyDD0DjvXrl3Dq6++inPnzkEikWhXIJdIJAAqVionqqmjV+9gwsZTKCnToLu/K74a3QWOUsM+1UdERE83vR89nzp1Kvz8/JCdnQ0HBwdcuHABR44cQZcuXXDo0CEjlEhidfBSFiI2VASdF1o2wrqxXRl0iIjI4PT+ZomPj8cvv/wCNzc3WFlZwcrKCj179kR0dDTeeustnDlzxhh1ksj871wG3tp2BuUaAeFtPLB8RCdIbSpfy4yIiOhJ6H1lR61Ww8mpYnI3Nzc33L59GwDQtGlTXL582bDVkSjtOXMTb249jXKNgFc6yLHitSAGHSIiMhq9r+y0bdsWZ8+ehZ+fH4KDg7F06VLY2dlhzZo1aNasmTFqJBHZfjINc/acgyAAQzo3xuLB7WFtJTF1WUREJGJ6h5333nsPhYWFAIAPP/wQ//jHP/Dss8/C1dUVO3bsMHiBJB7rj13HBz9UrJ826pmmmP9KG1gx6BARkZHpPYNyZXJzc9GgQQPtE1mWhjMoG9+qwylY/NOfAICJz/phbt9WFvv3hYiIzIPRZlCuTMOGDQ1xGBIhQRDw6YGr+OzgVQDAWy8GYNpLLRh0iIiozvA5XzIaQRCweP+fWH34GgBgZnhLTOkVYOKqiIjoacOwQ0ah0QiY/8MFbIhPBQC8/4/WiOjpZ+KqiIjoacSwQwan1giYu/scdiSkQyIBFg1sh9eCm5i6LCIiekrpPc/OkSNHUF5e/sj28vJyHDlyxCBFkeUqV2swfWcSdiSkw0oC/PefHRh0iIjIpPQOO7169UJubu4j2xUKBXr16mWQosgylZZr8ObWM/gu6TZsrCT4fEQQBndubOqyiIjoKaf3bSxBECp9kiYnJweOjo4GKYosT0mZGq9vTsSvl/+CnbUVvhgZhJdae5i6LCIiopqHnUGDBgGoWN187NixkEql2s/UajX++OMPdO/e3fAVktkrKi3HxI0JOJacA3tbK6wZ1QXPtWhk6rKIiIgA6BF2XFxcAFRc2XFycoJMJtN+Zmdnh2eeeQYTJ040fIVk1vJLyhCx/hRO3bgLBztrrBvbFc80czV1WURERFo1DjsxMTEAAF9fX7z99tu8ZUXIKyrFmHUncfamAk72NtgQ0Q1BTRqYuiwiIiIdeg9QnjVrls6YndTUVHz66af4+eefDVoYmbecAhVGfHUCZ28q0MDBFtsmPsOgQ0REZknvsDNgwABs3LgRAJCXl4du3bph2bJlGDBgAFauXGnwAsn8ZCtLMGzNcVzKUMKtnhTbJ4WgrbeLqcsiIiKqlN5h5/Tp03j22WcBAN9++y08PT2RmpqKjRs3Yvny5QYvkMzLrbxiDF0dj+TsAni52GPn5GfQ0tPJ1GURERFVSe9Hz4uKiuDkVPHl9vPPP2PQoEGwsrLCM888g9TUVIMXSOYjNacQr311ArfyitG4gQzbJj4Dn4YOpi6LiIioWnpf2QkICMDevXuRnp6O2NhYvPzyywCA7OzsapdXJ8uWnF2AoavjcSuvGM3cHPHNv0MYdIiIyCLoHXbef/99vP322/D19UW3bt0QEhICoOIqT6dOnQxeIJnepQwlhq2OR5ZShRYe9bB98jPwcpE9fkciIiIzIBEEQdB3p8zMTGRkZKBDhw6wsqrISydPnoSzszMCAwMNXqSxKZVKuLi4QKFQ8OrUQ/64mYfR604ir6gMbeTO2DQ+GA0d7UxdFhERUY2/v/W+sgMAnp6ecHJyQlxcHIqLiwEAXbt2tcigQ1VLTM3FyK9OIK+oDJ2a1MfWic8w6BARkcXRO+zk5OQgNDQULVq0QN++fZGRkQEAGD9+PGbMmGHwAsk0fk+5g1FrTyJfVY5ufg2xaXwwXGS2pi6LiIhIb3qHnWnTpsHW1hZpaWlwcPh7gOqwYcOwf/9+gxZHpnHocjbGxZxCUakazzZ3w4Zx3VBPqveDe0RERGZB72+wn3/+GbGxsWjcuLHO9ubNm/PRcxGIvZCJN7eeRplaQFgrd6x4LQj2ttamLouIiKjW9A47hYWFOld07svNzdVZCZ0sz/dnb2PajiSoNQL6tfPCJ8M6ws6mVsO6iIiIzIbe32TPPvusdrkIAJBIJNBoNFi6dCl69epl0OKo7uxMSMfU7Weg1ggY1Mkbnw1n0CEiInHQ+8rO0qVLERoaioSEBJSWlmLWrFm4cOECcnNzcezYMWPUSEa2Kf4Gor67AAAY0a0JFg1sCysryWP2IiIisgx6/69727ZtceXKFfTs2RMDBgxAYWEhBg0ahDNnzsDf398YNZIRff3bNW3QGdvdFx+9yqBDRETiovekgmlpafDx8YFE8ugXYlpaGpo0aWKw4urK0zqp4OcHr2JZ3BUAwOsv+GNWeMtKzysREZE5Mtqkgn5+fvjrr78e2Z6TkwM/Pz99D0cmIAgC/hP7pzbozHipBYMOERGJlt5jdgRBqPRLsaCgAPb29gYpioxHEAQs+PES1h27DgB4t28rTHyumYmrIiIiMp4ah53p06cDqHj6KioqSufxc7VajRMnTqBjx44GL5AMR6MR8N5357H1RBoAYMGANhgV4mvaooiIiIysxmHnzJkzACquDJw7dw52dn+vkWRnZ4cOHTrg7bffNnyFZBDlag1m7foDu0/fgkQCLBncHkO7+Ji6LCIiIqOrcdj59ddfAQDjxo3DZ5999lQN5LV0ZWoNInckYd8fGbC2kuDjoR0woKO3qcsiIiKqE3qP2YmJiTFGHWQkqnI1pmw5gwOXsmBrLcHnI4LQu62nqcsiIiKqM1zdUcSKS9WYtCkBv129A6mNFVaN6oxeLd1NXRYREVGdYtgRqQJVOSZsOIXj13Ihs7XG2jFd0D3AzdRlERER1TmGHRFSFJdhXMxJnE7LQz2pDdaP64ouvg1NXRYREZFJMOyIzN3CUoxadwLnbynhIrPFxohu6OBT39RlERERmQzDjoj8la/Cv74+gctZ+XB1tMOm8cFoLedTc0RE9HRj2BGJDEUxRn51AtfuFMLdSYqtE4MR4O5k6rKIiIhMjmFHBNJzi/Da18eRnlsM7/oybJkQDF83R1OXRUREZBYYdizctb8KMPLrE8hQlKCpqwO2TAhG4wYOj9+RiIjoKaH3qud1Sa1WIyoqCn5+fpDJZPD398eCBQsgCIK2jSAIeP/99+Hl5QWZTIawsDBcvXrVhFXXncuZ+Ri6+jgyFCXwb+SInZNDGHSIiIgeYtZhZ8mSJVi5ciVWrFiBS5cuYcmSJVi6dCk+//xzbZulS5di+fLlWLVqFU6cOAFHR0eEh4ejpKTEhJUb3/lbCgxfE487BSoEejphx+QQeDhz1XkiIqKHSYQHL5OYmX/84x/w8PDA2rVrtdsGDx4MmUyGzZs3QxAEyOVyzJgxQ7sIqUKhgIeHB9avX4/hw4fX6OcolUq4uLhAoVBYxJpfp9PuYsy6k8gvKUeHxi7YENEN9R3sHr8jERGRiNT0+9usr+x0794dBw8exJUrVwAAZ8+exdGjR9GnTx8AwPXr15GZmYmwsDDtPi4uLggODkZ8fLxJaja2E9dyMOrrE8gvKUdX3wbYPCGYQYeIiKgaZj1Aefbs2VAqlQgMDIS1tTXUajUWLVqEkSNHAgAyMzMBAB4eHjr7eXh4aD+rjEqlgkql0r5XKpVGqN7wfrv6FyZuTEBJmQbd/V3x9ZgucLAz61NIRERkcmZ9ZWfnzp3YsmULtm7ditOnT2PDhg3473//iw0bNjzRcaOjo+Hi4qJ9+fj4GKhi4zlwMQvj11cEnV4tG2Hd2K4MOkRERDVg1mFn5syZmD17NoYPH4527dph1KhRmDZtGqKjowEAnp6eAICsrCyd/bKysrSfVWbOnDlQKBTaV3p6uvE6YQD7/sjAvzcnolStQe82nlg9qgvsba1NXRYREZFFMOuwU1RUBCsr3RKtra2h0WgAAH5+fvD09MTBgwe1nyuVSpw4cQIhISFVHlcqlcLZ2VnnZa72nLmJ/9t2GuUaAQM6yrHitU6wszHr00ZERGRWzPo+SP/+/bFo0SI0adIEbdq0wZkzZ/Dxxx8jIiICACCRSBAZGYmFCxeiefPm8PPzQ1RUFORyOQYOHGja4g1g28k0zN1zDoIADO3SGNGD2sPaSmLqsoiIiCyKWYedzz//HFFRUXjjjTeQnZ0NuVyOyZMn4/3339e2mTVrFgoLCzFp0iTk5eWhZ8+e2L9/P+ztLXvOmZhj1zH/h4sAgNEhTfFB/zawYtAhIiLSm1nPs1NXzG2enZWHUrBk/58AgEnPNcOcPoGQSBh0iIiIHlTT72+zvrLztBEEAZ8cuIrlByuWu3grtDmmhTVn0CEiInoCHOlaBV9fX0gkkkdeU6ZMAVAxx8+oUaPg6ekJR0dHBAUFYdeuXdUec+XKlWjfvr12UHRISAh++uknABVBZ/FPf2L5watQ3boE6c8LETWwE1xcXPDcc8+huLjY6H0mIiISI17ZqcKpU6egVqu178+fP4+XXnoJQ4YMAQCMHj0aeXl5+P777+Hm5oatW7di6NChSEhIQKdOnSo9ZuPGjbF48WI0b94cgiBgw4YNGDBgABITT+Oba8DG+FSobl2CYs98TH3vXfTv/zVsbGxw9uzZR55KIyIioprhmB3U7J5fZGQkfvzxR1y9ehUSiQT16tXDypUrMWrUKG0bV1dXLFmyBBMmTKjxz27YsCGCh72FSy5dIZEA1j9EYcSr/bBgwYIn7hcREZGYiWJtLHNRWlqKzZs3IyIiQjt+pnv37tixYwdyc3Oh0Wiwfft2lJSU4IUXXqjRMdVqNbZs3QpFfgHOlnnASgK8H+qNlAtn4O7uju7du8PDwwPPP/88jh49asTeERERiRvDTg3s3bsXeXl5GDt2rHbbzp07UVZWBldXV0ilUkyePBl79uxBQEBAtcc6d+4c6tWrB6lUioiJk+E2cC4c3Jvi8xFBaFWvBADwwQcfYOLEidi/fz+CgoIQGhqKq1evGrOLREREosWwUwNr165Fnz59IJfLtduioqKQl5eHAwcOICEhAdOnT8fQoUNx7ty5ao/VsmVLnDiViD7vroV9+97I2fcJ5jxTD/3ae2lnhp48eTLGjRuHTp064ZNPPkHLli2xbt06o/aRiIhIrDhA+TFSU1Nx4MAB7N69W7stJSUFK1aswPnz59GmTRsAQIcOHfDbb7/hiy++wKpVq6o8XjmssOjoXZxTucErLAL1JFk4uW8Lxr/yHLy8vAAArVu31tmnVatWSEtLM0LviIiIxI9Xdh4jJiYG7u7u6Nevn3ZbUVERAFS7bldl8kvKMHrtSfyekgNHO2tsGNcNzvY2UKlUACoed5fL5bh8+bLOfleuXEHTpk0N1SUiIqKnCsNONTQaDWJiYjBmzBjY2Px9ESwwMBABAQGYPHkyTp48iZSUFCxbtgxxcXE6a3KFhoZixYoVAIC8olIEvTIOR4/+BvuSO5jX0wl7v/ovDh06hJEjRwKoWOtr5syZWL58Ob799lskJycjKioKf/75J8aPH1+nfSciIhIL3saqxoEDB5CWlqZdePQ+W1tb/O9//8Ps2bPRv39/FBQUICAgABs2bEDfvn217VJSUnDnzh3cKVDhX1+fQEZmNkpPfYLcort4c6sL2rdvj9jYWLz00kvafSIjI1FSUoJp06YhNzcXHTp0QFxcHPz9/eus30RERGLCeXZgvLWxMhTFOJ16F/+JvYwbOUVwqyfF1onBaOHhZLCfQURE9LTi2lgmtuNUGubsPgfNvSjpIrPBzsnPoFmjeqYtjIiI6CnDMTtGkKEo1gk6AJBfUg6ZnbXpiiIiInpKMewYwfU7hTpBBwA0AnDjTpFpCiIiInqKMewYgZ+bI6wkutusJRL4ujmYpiAiIqKnGMOOEXi5yBA9qB2s762jZS2R4KNBbeHlIjNxZURERE8fDlA2kmFdm+C5Fo1w404RfN0cGHSIiIhMhGHHiLxcZAw5REREJsbbWERERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkamYfdnx9fSGRSB55TZkyBQDwwgsvPPLZv//9bxNXTURERObCxtQFPM6pU6egVqu178+fP4+XXnoJQ4YM0W6bOHEiPvzwQ+17BweHOq2RiIiIzJfZh51GjRrpvF+8eDH8/f3x/PPPa7c5ODjA09OzrksjIiIiC2D2t7EeVFpais2bNyMiIgISiUS7fcuWLXBzc0Pbtm0xZ84cFBUVVXsclUoFpVKp8yIiIiJxMvsrOw/au3cv8vLyMHbsWO221157DU2bNoVcLscff/yBd955B5cvX8bu3burPE50dDTmz59fBxUTERGRqUkEQRBMXURNhYeHw87ODj/88EOVbX755ReEhoYiOTkZ/v7+lbZRqVRQqVTa90qlEj4+PlAoFHB2djZ43URERGR4SqUSLi4uj/3+tpgrO6mpqThw4EC1V2wAIDg4GACqDTtSqRRSqdTgNRIREZH5sZgxOzExMXB3d0e/fv2qbZeUlAQA8PLyqoOqiIiIyNxZxJUdjUaDmJgYjBkzBjY2f5eckpKCrVu3om/fvnB1dcUff/yBadOm4bnnnkP79u1NWDERERGZC4sIOwcOHEBaWhoiIiJ0ttvZ2eHAgQP49NNPUVhYCB8fHwwePBjvvfeeiSolIiIic2NRA5SNpaYDnIiIiMh81PT722LG7BARERHVBsMOERERiRrDDhEREYkaww4RERGJGsMOERERiRrDDhEREYkaww4RERGJGsMOERERiRrDDhEREYkaww4RERGJGsMOERERiRrDDhEREYkaww4RERGJGsMOERERiRrDDhEREYkaww4RERGJGsMOERERiRrDDhEREYkaww4RERGJGsMOERERiRrDDhEREYkaww4RERGJGsMOERERiRrDDhEREYkaww4RERGJGsMOERERiRrDDhEREYkaww4RERGJGsMOERERiRrDDhEREYkaww4RERGJGsMOERERiRrDDhEREYkaww4RERGJGsMOERERiRrDDhEREYkaww4RERGJGsMOERERiRrDDhEREYkaww4RERGJGsMOERERiRrDDhEREYkaww4RERGJGsMOERERiRrDDhEREYkaww4RERGJGsMOERERiRrDDhEREYkaww4RERGJGsMOERERiRrDDhEREYma2YcdX19fSCSSR15TpkwBAJSUlGDKlClwdXVFvXr1MHjwYGRlZZm4aiIiIjIXZh92Tp06hYyMDO0rLi4OADBkyBAAwLRp0/DDDz/gm2++weHDh3H79m0MGjTIlCUTERGRGZEIgiCYugh9REZG4scff8TVq1ehVCrRqFEjbN26Ff/85z8BAH/++SdatWqF+Ph4PPPMMzU6plKphIuLCxQKBZydnY1ZPhERERlITb+/beqwpidWWlqKzZs3Y/r06ZBIJEhMTERZWRnCwsK0bQIDA9GkSZNqw45KpYJKpdK+VygUACr+pREREZFluP+9/bjrNhYVdvbu3Yu8vDyMHTsWAJCZmQk7OzvUr19fp52HhwcyMzOrPE50dDTmz5//yHYfHx9DlktERER1ID8/Hy4uLlV+blFhZ+3atejTpw/kcvkTHWfOnDmYPn269r1Go0Fubi5cXV0hkUietEwtpVIJHx8fpKeni/b2mNj7yP5ZPrH3kf2zfGLvozH7JwgC8vPzH5sLLCbspKam4sCBA9i9e7d2m6enJ0pLS5GXl6dzdScrKwuenp5VHksqlUIqlepse/jqkCE5OzuL8i/wg8TeR/bP8om9j+yf5RN7H43Vv+qu6Nxn9k9j3RcTEwN3d3f069dPu61z586wtbXFwYMHtdsuX76MtLQ0hISEmKJMIiIiMjMWcWVHo9EgJiYGY8aMgY3N3yW7uLhg/PjxmD59Oho2bAhnZ2f83//9H0JCQmr8JBYRERGJm0WEnQMHDiAtLQ0RERGPfPbJJ5/AysoKgwcPhkqlQnh4OL788ksTVPkoqVSKefPmPXLLTEzE3kf2z/KJvY/sn+UTex/NoX8WN88OERERkT4sZswOERERUW0w7BAREZGoMewQERGRqDHsEBERkagx7DyhL774Ar6+vrC3t0dwcDBOnjxZbftvvvkGgYGBsLe3R7t27fC///2vjiqtHX36t379ekgkEp2Xvb19HVarnyNHjqB///6Qy+WQSCTYu3fvY/c5dOgQgoKCIJVKERAQgPXr1xu9ziehbx8PHTr0yDmUSCTVLr9iStHR0ejatSucnJzg7u6OgQMH4vLly4/dz1J+D2vTP0v7PVy5ciXat2+vnXAuJCQEP/30U7X7WMr5A/Tvn6Wdv4ctXrwYEokEkZGR1bar63PIsPMEduzYgenTp2PevHk4ffo0OnTogPDwcGRnZ1fa/vfff8eIESMwfvx4nDlzBgMHDsTAgQNx/vz5Oq68ZvTtH1AxQ2ZGRob2lZqaWocV66ewsBAdOnTAF198UaP2169fR79+/dCrVy8kJSUhMjISEyZMQGxsrJErrT19+3jf5cuXdc6ju7u7kSp8MocPH8aUKVNw/PhxxMXFoaysDC+//DIKCwur3MeSfg9r0z/Asn4PGzdujMWLFyMxMREJCQl48cUXMWDAAFy4cKHS9pZ0/gD9+wdY1vl70KlTp7B69Wq0b9++2nYmOYcC1Vq3bt2EKVOmaN+r1WpBLpcL0dHRlbYfOnSo0K9fP51twcHBwuTJk41aZ23p27+YmBjBxcWljqozLADCnj17qm0za9YsoU2bNjrbhg0bJoSHhxuxMsOpSR9//fVXAYBw9+7dOqnJ0LKzswUAwuHDh6tsY2m/hw+qSf8s+ffwvgYNGghff/11pZ9Z8vm7r7r+Wer5y8/PF5o3by7ExcUJzz//vDB16tQq25riHPLKTi2VlpYiMTERYWFh2m1WVlYICwtDfHx8pfvEx8frtAeA8PDwKtubUm36BwAFBQVo2rQpfHx8Hvt/L5bGks7fk+rYsSO8vLzw0ksv4dixY6Yup8YUCgUAoGHDhlW2seTzWJP+AZb7e6hWq7F9+3YUFhZWueSPJZ+/mvQPsMzzN2XKFPTr1++Rc1MZU5xDhp1aunPnDtRqNTw8PHS2e3h4VDm+ITMzU6/2plSb/rVs2RLr1q3Dd999h82bN0Oj0aB79+64efNmXZRsdFWdP6VSieLiYhNVZVheXl5YtWoVdu3ahV27dsHHxwcvvPACTp8+berSHkuj0SAyMhI9evRA27Ztq2xnSb+HD6pp/yzx9/DcuXOoV68epFIp/v3vf2PPnj1o3bp1pW0t8fzp0z9LPH/bt2/H6dOnER0dXaP2pjiHFrFcBFmGkJAQnf9b6d69O1q1aoXVq1djwYIFJqyMaqply5Zo2bKl9n337t2RkpKCTz75BJs2bTJhZY83ZcoUnD9/HkePHjV1KUZR0/5Z4u9hy5YtkZSUBIVCgW+//RZjxozB4cOHqwwElkaf/lna+UtPT8fUqVMRFxdn1gOpGXZqyc3NDdbW1sjKytLZnpWVBU9Pz0r38fT01Ku9KdWmfw+ztbVFp06dkJycbIwS61xV58/Z2RkymcxEVRlft27dzD5AvPnmm/jxxx9x5MgRNG7cuNq2lvR7eJ8+/XuYJfwe2tnZISAgAADQuXNnnDp1Cp999hlWr179SFtLPH/69O9h5n7+EhMTkZ2djaCgIO02tVqNI0eOYMWKFVCpVLC2ttbZxxTnkLexasnOzg6dO3fGwYMHtds0Gg0OHjxY5b3YkJAQnfYAEBcXV+29W1OpTf8eplarce7cOXh5eRmrzDplSefPkJKSksz2HAqCgDfffBN79uzBL7/8Aj8/v8fuY0nnsTb9e5gl/h5qNBqoVKpKP7Ok81eV6vr3MHM/f6GhoTh37hySkpK0ry5dumDkyJFISkp6JOgAJjqHRhv6/BTYvn27IJVKhfXr1wsXL14UJk2aJNSvX1/IzMwUBEEQRo0aJcyePVvb/tixY4KNjY3w3//+V7h06ZIwb948wdbWVjh37pypulAtffs3f/58ITY2VkhJSRESExOF4cOHC/b29sKFCxdM1YVq5efnC2fOnBHOnDkjABA+/vhj4cyZM0JqaqogCIIwe/ZsYdSoUdr2165dExwcHISZM2cKly5dEr744gvB2tpa2L9/v6m68Fj69vGTTz4R9u7dK1y9elU4d+6cMHXqVMHKyko4cOCAqbpQrddff11wcXERDh06JGRkZGhfRUVF2jaW/HtYm/5Z2u/h7NmzhcOHDwvXr18X/vjjD2H27NmCRCIRfv75Z0EQLPv8CYL+/bO081eZh5/GModzyLDzhD7//HOhSZMmgp2dndCtWzfh+PHj2s+ef/55YcyYMTrtd+7cKbRo0UKws7MT2rRpI+zbt6+OK9aPPv2LjIzUtvXw8BD69u0rnD592gRV18z9x6wfft3v05gxY4Tnn3/+kX06duwo2NnZCc2aNRNiYmLqvG596NvHJUuWCP7+/oK9vb3QsGFD4YUXXhB++eUX0xRfA5X1DYDOebHk38Pa9M/Sfg8jIiKEpk2bCnZ2dkKjRo2E0NBQbRAQBMs+f4Kgf/8s7fxV5uGwYw7nUCIIgmC860ZEREREpsUxO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtEREQkagw7REREJGoMO0RERCRqDDtERA85dOgQJBIJ8vLyTF0KERkAww4RERGJGsMOERERiRrDDhGZHY1Gg+joaPj5+UEmk6FDhw749ttvAfx9i2nfvn1o37497O3t8cwzz+D8+fM6x9i1axfatGkDqVQKX19fLFu2TOdzlUqFd955Bz4+PpBKpQgICMDatWt12iQmJqJLly5wcHBA9+7dcfnyZeN2nIiMgmGHiMxOdHQ0Nm7ciFWrVuHChQuYNm0a/vWvf+Hw4cPaNjNnzsSyZctw6tQpNGrUCP3790dZWRmAipAydOhQDB8+HOfOncMHH3yAqKgorF+/Xrv/6NGjsW3bNixfvhyXLl3C6tWrUa9ePZ063n33XSxbtgwJCQmwsbFBREREnfSfiAyLC4ESkVlRqVRo2LAhDhw4gJCQEO32CRMmoKioCJMmTUKvXr2wfft2DBs2DACQm5uLxo0bY/369Rg6dChGjhyJv/76Cz///LN2/1mzZmHfvn24cOECrly5gpYtWyIuLg5hYWGP1HDo0CH06tULBw4cQGhoKADgf//7H/r164fi4mLY29sb+d8CERkSr+wQkVlJTk5GUVERXnrpJdSrV0/72rhxI1JSUrTtHgxCDRs2RMuWLXHp0iUAwKVLl9CjRw+d4/bo0QNXr16FWq1GUlISrK2t8fzzz1dbS/v27bV/9vLyAgBkZ2c/cR+JqG7ZmLoAIqIHFRQUAAD27dsHb29vnc+kUqlO4KktmUxWo3a2trbaP0skEgAV44mIyLLwyg4RmZXWrVtDKpUiLS0NAQEBOi8fHx9tu+PHj2v/fPfuXVy5cgWtWrUCALRq1QrHjh3TOe6xY8fQokULWFtbo127dtBoNDpjgIhIvHhlh4jMipOTE95++21MmzYNGo0GPXv2hEKhwLFjx+Ds7IymTZsCAD788EO4urrCw8MD7777Ltzc3DBw4EAAwIwZM9C1a1csWLAAw4YNQ3x8PFasWIEvv/wSAODr64sxY8YgIiICy5cvR4cOHZCamors7GwMHTrUVF0nIiNh2CEis7NgwQI0atQI0dHRuHbtGurXr4+goCDMnTtXextp8eLFmDp1Kq5evYqOHTvihx9+gJ2dHQAgKCgIO3fuxPvvv48FCxbAy8sLH374IcaOHav9GStXrsTcuXPxxhtvICcnB02aNMHcuXNN0V0iMjI+jUVEFuX+k1J3795F/fr1TV0OEVkAjtkhIiIiUWPYISIiIlHjbSwiIiISNV7ZISIiIlFj2CEiIiJRY9ghIiIiUWPYISIiIlFj2CEiIiJRY9ghIiIiUWPYISIiIlFj2CEiIiJRY9ghIiIiUft/VTGNs0unyBkAAAAASUVORK5CYII=", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "plt.plot(np.arange(len(epochs_x)), epochs_acc, marker = '.')\n", - "plt.xlabel('epoch')\n", - "plt.ylabel('test accuracy')\n", - "plt.ylim(70, 100)\n", - "for i, txt in enumerate(epochs_acc):\n", - " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom')\n", - "plt.show()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "speck-rescnn", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/tests/test_nonsequential/non-sequential-SCNN-example_3.ipynb b/tests/test_nonsequential/non-sequential-SCNN-example_3.ipynb deleted file mode 100644 index 956dfb88..00000000 --- a/tests/test_nonsequential/non-sequential-SCNN-example_3.ipynb +++ /dev/null @@ -1,1509 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import torch, random\n", - "import torch.nn as nn\n", - "import sinabs.layers as sl\n", - "from tqdm.notebook import tqdm\n", - "\n", - "from tonic.datasets.dvsgesture import DVSGesture\n", - "from tonic.transforms import ToFrame\n", - "from torch.utils.data import DataLoader\n", - "from torch.nn import CrossEntropyLoss\n", - "from torch.optim import Adam\n", - "\n", - "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential\n", - "\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "torch.backends.cudnn.deterministic = True\n", - "random.seed(1)\n", - "torch.manual_seed(1)\n", - "torch.cuda.manual_seed(1)\n", - "np.random.seed(1)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Loading Data" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "batch_size = 3\n", - "num_workers = 1\n", - "epochs = 30\n", - "lr = 1e-3" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "root_dir = \"./DVSGESTURE\"\n", - "_ = DVSGesture(save_to=root_dir, train=True)\n", - "_ = DVSGesture(save_to=root_dir, train=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "n_time_steps = 50\n", - "to_raster = ToFrame(sensor_size=DVSGesture.sensor_size, n_time_bins=n_time_steps)\n", - "\n", - "snn_train_dataset = DVSGesture(save_to=root_dir, train=True, transform=to_raster)\n", - "snn_test_dataset = DVSGesture(save_to=root_dir, train=False, transform=to_raster)" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The transformed array is in shape [Time-Step, Channel, Height, Width] --> (50, 2, 128, 128)\n" - ] - } - ], - "source": [ - "sample_data, label = snn_train_dataset[0]\n", - "print(f\"The transformed array is in shape [Time-Step, Channel, Height, Width] --> {sample_data.shape}\")" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "snn_train_dataloader = DataLoader(snn_train_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True)\n", - "snn_test_dataloader = DataLoader(snn_test_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=False)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Network Module\n", - "\n", - "We need to define a `nn.Module` implementing the network we want the chip to reproduce." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "device: NVIDIA RTX A4000\n" - ] - } - ], - "source": [ - "if torch.cuda.is_available():\n", - " device = torch.device('cuda:0')\n", - " print('device: ', torch.cuda.get_device_name(0))\n", - "else:\n", - " device = torch.device('cpu')" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "class SNN(nn.Module):\n", - " def __init__(self) -> None:\n", - " super().__init__()\n", - "\n", - " self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False)\n", - " self.iaf1 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - " self.pool1 = nn.AvgPool2d(2,2)\n", - " self.pool1a = nn.AvgPool2d(6,6)\n", - "\n", - " self.conv2 = nn.Conv2d(10, 10, 2, 1, bias=False)\n", - " self.iaf2 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - " self.pool2 = nn.AvgPool2d(3,3)\n", - "\n", - " self.conv3 = nn.Conv2d(10, 10, 3, 1, bias=False)\n", - " self.iaf3 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - " self.pool3 = nn.AvgPool2d(2,2)\n", - "\n", - " self.flat = nn.Flatten()\n", - "\n", - " self.fc1 = nn.Linear(810, 100, bias=False)\n", - " self.iaf4 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - "\n", - " self.fc2 = nn.Linear(100, 100, bias=False)\n", - " self.iaf5 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - "\n", - " self.fc3 = nn.Linear(100, 100, bias=False)\n", - " self.iaf6 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - "\n", - " self.fc4 = nn.Linear(100, 11, bias=False)\n", - " self.iaf7 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - "\n", - " self.merge_fc = sl.Merge()\n", - " self.merge_conv = sl.Merge()\n", - "\n", - " def detach_neuron_states(self):\n", - " for name, layer in self.named_modules():\n", - " if name != '':\n", - " if isinstance(layer, sl.StatefulLayer):\n", - " for name, buffer in layer.named_buffers():\n", - " buffer.detach_()\n", - "\n", - " def init_weights(self):\n", - " for name, layer in self.named_modules():\n", - " if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear):\n", - " nn.init.xavier_normal_(layer.weight.data)\n", - "\n", - " def forward(self, x):\n", - " \n", - " con1_out = self.conv1(x)\n", - " iaf1_out = self.iaf1(con1_out)\n", - " pool1_out = self.pool1(iaf1_out)\n", - " pool1a_out = self.pool1a(iaf1_out)\n", - "\n", - " conv2_out = self.conv2(pool1_out)\n", - " iaf2_out = self.iaf2(conv2_out)\n", - " pool2_out = self.pool2(iaf2_out)\n", - "\n", - " merged_conv_out = self.merge_conv(pool1a_out, pool2_out)\n", - "\n", - " conv3_out = self.conv3(merged_conv_out)\n", - " iaf3_out = self.iaf3(conv3_out)\n", - " pool3_out = self.pool3(iaf3_out)\n", - "\n", - " flat_out = self.flat(pool3_out)\n", - " \n", - " fc1_out = self.fc1(flat_out)\n", - " iaf4_out = self.iaf4(fc1_out)\n", - "\n", - " fc2_out = self.fc2(iaf4_out)\n", - " iaf5_out = self.iaf5(fc2_out)\n", - "\n", - " fc3_out = self.fc3(iaf5_out)\n", - " iaf6_out = self.iaf6(fc3_out)\n", - "\n", - " merge_fc_out = self.merge_fc(iaf4_out, iaf6_out)\n", - "\n", - " fc4_out = self.fc4(merge_fc_out)\n", - " iaf7_out = self.iaf7(fc4_out)\n", - "\n", - " return iaf7_out" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "snn = SNN().to(device)" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "snn.init_weights()" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "optimizer = Adam(snn.parameters(), lr=lr, betas=(0.9, 0.999), eps=1e-8)\n", - "loss_fn = CrossEntropyLoss()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Define train and test" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "def train(dataloader, model, loss_fn, optimizer, epochs, test_func, dataloader_test):\n", - " epochs_y = []\n", - " epochs_x = []\n", - " epochs_acc = []\n", - " model.train()\n", - "\n", - " for e in range(epochs):\n", - " losses = []\n", - " batches = []\n", - " batch_count = 0\n", - " train_p_bar = tqdm(snn_train_dataloader)\n", - "\n", - " for X, y in train_p_bar:\n", - " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", - " X = X.reshape(-1, 2, 128, 128).to(dtype=torch.float, device=device)\n", - " y = y.to(dtype=torch.long, device=device)\n", - "\n", - " # forward\n", - " pred = model(X)\n", - "\n", - " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", - " pred = pred.reshape(batch_size, n_time_steps, -1)\n", - "\n", - " # accumulate all time-steps output for final prediction\n", - " pred = pred.sum(dim = 1)\n", - " loss = loss_fn(pred, y)\n", - "\n", - " # gradient update\n", - " optimizer.zero_grad()\n", - " loss.backward()\n", - " optimizer.step()\n", - "\n", - " # detach the neuron states and activations from current computation graph(necessary)\n", - " model.detach_neuron_states()\n", - "\n", - " train_p_bar.set_description(f\"Epoch {e} - BPTT Training Loss: {round(loss.item(), 4)}\")\n", - "\n", - " batch_count += 1\n", - " losses.append(loss.item())\n", - " batches.append(batch_count)\n", - "\n", - " epochs_y.append(losses)\n", - " epochs_x.append(batches)\n", - "\n", - " acc = test_func(dataloader_test, model)\n", - " print(f'Epoch {e} accuracy: {acc}')\n", - " epochs_acc.append(acc)\n", - "\n", - " return epochs_x, epochs_y, epochs_acc\n" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [], - "source": [ - "def test(dataloader, model):\n", - " correct_predictions = []\n", - " with torch.no_grad():\n", - " test_p_bar = tqdm(dataloader)\n", - " for X, y in test_p_bar:\n", - " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", - " X = X.reshape(-1, 2, 128, 128).to(dtype=torch.float, device=device)\n", - " y = y.to(dtype=torch.long, device=device)\n", - "\n", - " # forward\n", - " output = model(X)\n", - "\n", - " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", - " output = output.reshape(batch_size, n_time_steps, -1)\n", - "\n", - " # accumulate all time-steps output for final prediction\n", - " output = output.sum(dim=1)\n", - "\n", - " # calculate accuracy\n", - " pred = output.argmax(dim=1, keepdim=True)\n", - "\n", - " # compute the total correct predictions\n", - " correct_predictions.append(pred.eq(y.view_as(pred)))\n", - "\n", - " test_p_bar.set_description(f\"Testing Model...\")\n", - " \n", - " correct_predictions = torch.cat(correct_predictions)\n", - " return correct_predictions.sum().item()/(len(correct_predictions))*100" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Training loop (HPO)" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "12d134e3b89e41888c9c47892b8e6491", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - " 0%| | 0/359 [00:00" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "y_avg = []\n", - "for y in epochs_y:\n", - " y_avg.append(np.mean(y))\n", - "\n", - "plt.plot(np.arange(len(epochs_x)), y_avg, marker = '.')\n", - "plt.xlabel('epoch')\n", - "plt.ylabel('average loss')\n", - "plt.ylim(0,)\n", - "plt.xticks(np.arange(len(epochs_x)))\n", - "for i, txt in enumerate(y_avg):\n", - " if i%5 == 0:\n", - " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", - " else:\n", - " pass\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAG2CAYAAACZEEfAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABq60lEQVR4nO3dd1hT1+MG8DfsHUW2TAfixK3gqqO46q6jtVWr1bZi62idraPVFrWt7dfROn5urauOOloXrqqooOBEBERBZYhI2DPn9weaGlmJBIH4fp4nT8vNPfecQMx9c+6550iEEAJEREREWkqnohtAREREVJ4YdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirVWjYOXPmDPr06QMHBwdIJBLs27dP6XkhBObMmQN7e3sYGxujW7duCA8PV9onKSkJw4cPh4WFBapVq4YxY8YgLS3tNb4KIiIiqswqNOykp6fD09MTK1asKPL5xYsXY+nSpVi5ciUuXrwIU1NTdO/eHVlZWYp9hg8fjps3b+LYsWM4ePAgzpw5g3Hjxr2ul0BERESVnKSyLAQqkUiwd+9e9O/fH0BBr46DgwO+/PJLfPXVVwAAmUwGW1tbbNiwAcOGDUNoaCgaNGiAwMBAtGzZEgBw+PBh9OrVCw8ePICDg0NFvRwiIiKqJPQqugHFiYqKQlxcHLp166bYJpVK0aZNGwQEBGDYsGEICAhAtWrVFEEHALp16wYdHR1cvHgRAwYMKPLY2dnZyM7OVvwsl8uRlJSEGjVqQCKRlN+LIiIiIo0RQiA1NRUODg7Q0Sn+YlWlDTtxcXEAAFtbW6Xttra2iufi4uJgY2Oj9Lyenh4sLS0V+xTFz88P3377rYZbTERERBUhJiYGjo6OxT5facNOeZo5cyamTJmi+Fkmk8HZ2RkxMTGwsLCowJYRERGRqlJSUuDk5ARzc/MS96u0YcfOzg4AEB8fD3t7e8X2+Ph4NG3aVLFPQkKCUrm8vDwkJSUpyhfF0NAQhoaGhbZbWFgw7BAREVUxpQ1BqbTz7Li5ucHOzg7+/v6KbSkpKbh48SK8vLwAAF5eXkhOTsbly5cV+5w4cQJyuRxt2rR57W0mIiKiyqdCe3bS0tIQERGh+DkqKgohISGwtLSEs7MzJk2ahAULFqBu3bpwc3PD7Nmz4eDgoLhjq379+ujRowfGjh2LlStXIjc3FxMmTMCwYcN4JxYREREBqOCwExQUhM6dOyt+fj6OZuTIkdiwYQOmTZuG9PR0jBs3DsnJyWjfvj0OHz4MIyMjRZmtW7diwoQJ6Nq1K3R0dDBo0CAsXbr0tb8WIiIiqpwqzTw7FSklJQVSqRQymYxjdoiIiKoIVc/flXbMDhEREZEmMOwQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEqQn5+P2bNnw83NDcbGxqhduzbmz58PIYRin1GjRkEikSg9evToUeJxU1NTMWnSJLi4uMDY2Bje3t4IDAxU2ictLQ0TJkyAo6MjjI2N0aBBA6xcubJcXqc206voBhAREVVmixYtwu+//46NGzeiYcOGCAoKwkcffQSpVIovvvhCsV+PHj2wfv16xc+GhoYlHvfjjz/GjRs3sHnzZjg4OGDLli3o1q0bbt26hZo1awIApkyZghMnTmDLli1wdXXF0aNHMX78eDg4OKBv377l84K1EHt2iIiISnD+/Hn069cPvXv3hqurK9599134+Pjg0qVLSvsZGhrCzs5O8ahevXqxx8zMzMTu3buxePFidOzYEXXq1MG8efNQp04d/P7770p1jxw5Em+99RZcXV0xbtw4eHp6FqqbSsawQ0T0hiivyzF+fn5o1aoVzM3NYWNjg/79+yMsLExpn08++QS1a9eGsbExalhZo0O3njgTGFIeL1PjvL294e/vjzt37gAArl69irNnz6Jnz55K+506dQo2NjaoV68ePvvsMzx58qTYY+bl5SE/Px9GRkZK242NjXH27Fmluvfv34+HDx9CCIGTJ0/izp078PHx0eArfAMIEjKZTAAQMpmsoptCRFRuvv/+e1GjRg1x8OBBERUVJXbt2iXMzMzE//73P8U+I0eOFD169BCxsbGKR1JSUonH7d69u1i/fr24ceOGCAkJEb169RLOzs4iLS1Nsc+qVavE6dOnxdJ9Z4XDqF+FcZ3WQtfCWvwRcLfcXq+m5Ofni+nTpwuJRCJ09fSERCIRP/zwg9I+27ZtE3/99Ze4du2a2Lt3r6hfv75o1aqVyMvLK/a4Xl5eolOnTuLhw4ciLy9PbN68Wejo6Ah3d3fFPllZWWLEiBECgNDT0xMGBgZi48aN5fZaqxpVz98MO4Jhh4jeDL179xajR49W2jZw4EAxfPhwxc8jR44U/fr1K1M9CQkJAoA4ffq00vbIhFThOv2gcJl+UNh/tEwAEE6f/J94lJxRpvrK27Zt24Sljb2w7jtV2I9eLqzemSLMLKqJDRs2FFsmMjJSABDHjx8vdp+IiAjRsWNHAUDo6uqKVq1aieHDhwsPDw/FPj/++KNwd3cX+/fvF1evXhXLli0TZmZm4tixYxp9jVWVqudvXsYiInpDlMflmKLIZDIAgKWlpWLbydsJGLIyAAKAPCcLadePQ09qC4l5DdxLzCjbCytnU776ChLPfjCp3wkG1q4wbdgFep7vYP73PxRbplatWrCyskJERESx+9SuXRunT59GWloaYmJicOnSJeTm5qJWrVoACsb1zJo1C0uWLEGfPn3QpEkTTJgwAUOHDsVPP/2k8ddZklhZJs5HJiJWlvla69UU3o1FRPSGmDFjBlJSUuDh4QFdXV3k5+fj+++/x/DhwxX79OjRAwMHDoSbmxsiIyMxa9Ys9OzZEwEBAdDV1S21DrlcjkmTJqFdu3Zo1KgRElKz8N2BWzh4LRapVw7h6an1ELlZ0LN0hM3QBdDV04erlUl5vuwyEUJAlpIOI8lLfQMSHcTLMuEfGo+O7tbQ11V+/sGDB3jy5Ans7e1LrcPU1BSmpqZ4+vQpjhw5gsWLFwMAcnNzkZubCx0d5WPr6upCLper9TpiZZmISkyHm5Up7KXGpe4vhEBadh4S03KwIzAaq87chRCAjgTwG9gYQ1s5q1V/RWPYISJ6Q+zcuRNbt27FH3/8gYYNGyIkJASTJk2Cg4MDRo4cCQAYNmyYYv/GjRujSZMmqF27Nk6dOoWuXbuWWoevry9u3LiBM2f+xbZL0fD7OxQpWXnQkQDjPx4Ji48H45e/ApF8aTcS/1oI51E/IyMnv9xec1lcikqC3z+hkLi0gOz8DuhaWMPAyhk58ZFICdwHsyZvY8zGIEj18mF6cy8+GfkeOnrWxd27dzFt2jTUqVMH3bt3Vxyva9euGDBgACZMmAAAOHLkCIQQqFevHiIiIjB16lR4eHjgo48+AgBYWFigU6dOmDp1KoyNjeHi4oLTp09j06ZNWLJkicqvY0dgNGbuuQ75s7Ay+W13tHa1RGJaDhLTshWPx6nKP2flFg5UcgFM330dJ8Meo4VzdXjYm8PDzgLW5iXfZl/RJEK8MAz/DZWSkgKpVAqZTAYLC4uKbg4RqSA/Px/z5s3Dli1bEBcXBwcHB4waNQrffPMNJBIJAGDevHnYvn07YmJiYGBggBYtWuD7779HmzZtij2uq6sr7t+/X2j7+PHjsWLFCgDA6tWr8ccff+DKlStITU3F06dPUa1atXJ5nZrk5OSEGTNmwNfXV7FtwYIF2LJlC27fvl1sOWtrayxYsACffPJJicefMGEC/vrrL2za8zd+u5yKwHtPAQCNa0rhN7AxGtWUAijoZbj9IAlvN6+D6t0/h+dbvbHPtx3MDCvH9+878alYfPg2jocmAAAM5NnQD9mFsIv+kGfIoGtmiU49+8P73U/w961EJDxNweM9C5CTcBciOx3VrGzR3edt/PrjQtja2iqO6+rqilGjRmHevHkACsLnzJkz8eDBA1haWmLQoEH4/vvvIZVKFWXi4uIwc+ZMHD16FElJSXBxccG4ceMwefJkxfv8ZUIIPHiaidtxqQiMeoLV/0a98u/CUE8H2Xml9yJZmRmivr05POwKwo+HvTnq2JjBUE9X7V4ldah6/q4c7ywiIjWpMtGbu7s7li9fjlq1aiEzMxO//PILfHx8EBERAWtr6yKPGxgYiPz8/3oabty4gbfffhuDBw9WbMvIyECPHj3Qo0cPzJw5s3xfqAZlZGSofUlElcsxQgh8/vnn2LN3L0Z+vx7j9sUgN1/AxEAXU952xyhvV+i9cJnHXmoMSyMrGOhKYGEARCSk4audV/H7B82LPYG/DrGyTPxy7A7+vPwAcgHo6kgwrJUTJnatCxuLgYiVZeJeYgZcrUwUJ+05/eQ4F/kEe1utxZGb8cjMLXjvBAD4dHcEBjTLxDtNHGBpaoCAq6GISkxHrCwT9lJjDBkyBEOGDCmxTXZ2dli/fn2xgSE1Kxd34lMRGpuK23EpuB2bittxqUjLzivxuLYWhnCqbgIrM0NYmRvAyswQ1uaGBT+bGcL62XZZZi7aLTwB+QvdIjoSYEx7NzxMzsTt2FREPUlHYlo2/g3Pxr/hiYr99HQkqGFmgPiUbEW5iroExp4dsGeHqCp65513YGtri7Vr1yq2DRo0CMbGxtiyZUuRZZ7/Wz9+/LhKl2QAYNKkSTh48CDCw8MLnYhPnTqFzp07V5menVGjRuH48eNYtWoVrJ1qw//cRfw0+yt8PGY0Fi1ahLS0NHz77bcYNGgQ7OzsEBkZiWnTpiE1NRXXr19XzAj88uWY8ePHY/OWrajzwbdI1LECALSrWwPzB7dGXYcauHv3Lnbs2AEfHx9YW1vjwYMHWLhwIc6dO4edxwLw2e5I5OTLMbV7Pfh2rvPafy+yjFz8djoCG87dU/Ri9Gxkh6+610NtazOVj5OenYejt+KwN/gRzoY/VgQEPR0J6tqa4XZsKgQKTvrTutfDO54OKh334NVHWHwkDHIBSAB08bCBjo4Et+NSEJNU9IBhfV0J6tiYw8XSBEduxuHFE72uRIKzMzqr3MuyIzAas/bcQL4Q0JVI8MPARkqBJSMnD+HxaQiNTcHtuFTFf2WZuYWOpW7dpWHPDhFpNW9vb6xevRp37tyBu7u74s6i4sYy5OTkYPXq1ZBKpfD09FSpjpycHGzZsgVTpkyp0B4HTVm2bBlmz56NUR9/gieJj6FrZgnTBl3RpN9YAAW9PNeuXcPGjRuRnJwMBwcH+Pj4YP78+UpLH0RGRiIxseAbfHJGjmLG35DfJyv22QGgh3Q96o4aBSMjI/z777/49ddf8fTpU9ja2qJjx444f/486tVzx7d5Rpi55zp+OhqGRjWl6ORedK+bpmXl5mNTwD2sOBmpODG3drPEjJ4eaO5c/OzHxTE11MOAZo4Y0MwRCalZ2B/yCPtCHuLGwxSExqYq9pMLYOHhMCw8HFbC0YomAPjfTlDaZmdhpBg7U//Zf2tZmyoGTRcVVtQJG0NbOaOju3WhXq3nTAz04OlUDZ5O1f5rpxA4dD0WE/4IVto3XwjcS8zQ+OWs0rBnB+zZIaqK5HI5Zs2ahcWLFyvdWfTyZaWDBw9i2LBhyMjIgL29Pfbt24dWrVqpVMfOnTvx/vvvIzo6Gg4Ohb+FV7WeHaDgUs3LlyUkAGb28kBdG3PFZY0apoYw0Ct6dpJYWSaiHqcjPCEVy05EIDEtBwAwvI0zpvXwgNRYX+12zdxzDdsuxUBqrI8DE9rDuUb53KEVK8tEZEI6QmNlWH/uHh7JsgAA9WzNMb1nPXSuZ6PxYLsrKAZT/7xWaLu+jgQ6OiXXJZcL5MoLn6ZHtHVBj8Z28LCzgKWpQaltKOoSXHkr6r3Gnh0iqjDlOYCwvOpW5c4iAOjcuTNCQkKQmJiINWvWYMiQIbh48SJsbGxKrWPt2rXo2bNnkUGnqop6nI6Xz50CwA9/Fx6gLDXWh5WZwbMAVDCOIzY5E0dvxStdFqlrYwa/gY3R0tWy0DFUNa9vQ4TGpiIkJhnjNgdhz3hvmBho9hS1IzAaM/Zcx4tf8R2kRpjiUw8DmtWEbinB41W1r2sFHQkKnfTPTC/9pF9cYPisc221/r3YS41f+79te6kx/AY2LlOvkqawZwfs2aE328u3pb7OAYRlqftV7yyqW7cuRo8eXerA4vv376NWrVrYs2cP+vXrV+Q+VbFnZ8uF+/hm3w2lbRIUXL5JzcpDYlo2nqTnIL+I3oSiSACcnvoWnGuYlrltsbJM9Fl2FolpOejr6YD/DWuqsV6WqzFP0W/FeaVtEgAnv3oLrlZlb3tpShv3Ul5lK4Py7FVizw4RlSpWlqkIG0DBN89Ze26go7t1uX/7KqrumXuuq1z3q9xZBBRc/srOzi71+OvXr4eNjQ169+5d6r5VRWpWLpb6hwMoONELoMiTp1wukJyZWzDfSmo2HqdlIzEtB1djkrH/6iOlYwoAD5OzNBJ27KXGWPF+cwz/v4vYf/URmjhK8XGHWmU6Zr5cYMuF+/D7J7TQcwJArCzrtYSd0sa9lFfZyqAiepVexrBD9Aa7HZtS6JLG6xpAGJVY+HKKXABbL9zHlLfrlTqWoU+fPvj+++/h7OyMhg0bIjg4GEuWLMHo0aMBAOnp6fj+++/Rt29f2NvbIzExEStWrMDDhw+VbiN/+c4ioCAQrV+/HiNHjoSeXuGPybi4OMTFxSmWArh+/TrMzc3h7OystERCaV735cOfj95BQmo2XGuYYMNHrZ+d6AufPHV0JLA0NYClqQHcbc2V2nvw2qNCl1Q0OQNym1o18HXv+vj2wC34/XMbDRws4F3b6pWOFRqbgpl7riMkJrnI5zXd9tKU5aRfGQJDVca1sYgqWH5+PmbPng03NzcYGxujdu3amD9/Pl68wrxnzx74+PigRo0akEgkCAkJUenYu3btgoeHB4yMjNC4cWP8/fffiueS0nOw+Mgd5CbGIGH3d4j+ZQiilwxC7MbJ0MtUby2kV+FWzLfp5Scj0Wf5Wfwb/rjE8suWLcO7776L8ePHo379+vjqq6/wySefYP78+QAKenlu376NQYMGwd3dHX369MGTJ0/w77//omHDhgAKTt63wsIR9SBW6djHjx9HdHS0Iji9bOXKlWjWrBnGji24i6ljx45o1qwZ9u/fr/Lr3xEYjXYLT+D9NRfRbuEJ7AiMVrnsq7j+QIZNAfcAAPP7N4KrlSm8atdQe9yH38DG0H12aam8xmCM8nbFgGY1kS8X+PyPYDxKVm89psycfCz85zb6LDuLkJhkmBvqYX7/Rq+l7VQ5ccwOOGaHKtYPP/yAJUuWFJoc7/vvv1dMjrd582ZERUXBwcEBY8eORXBwMJo2bVricc+fP4+OHTvCz88P77zzDv744w8sWrQIV65cgdShFkauu4Sw8AjEbZ4C8yZvw6R+J0gMTGCW8QgnF4+Dg71dub7uM3ceY8S6S4qfdSTA2w1scT7iCVKfTYjWoa4VpvfwUMy8q0kVOVbp4dMMtF90skxzn6gjXy4w4LdzuPZAhr6eDlj6XrMyHe913NmTmZOPQb+fx63YFDRxlGLnJ14w0i99ba5/wx/j6703EJ1UsLhor8Z2mNunIWwtjF5b2+n1UfX8zbADhh2qWOpMjnfv3j24ubmpFHaGDh2K9PR0HDx4ULGtbdu2cKnbAHc93sfj1Gyk/f0T2tezxe//tx43HsgwZddVpGblYX6/hvjQy1WTL1NJbr4cPf/3LyIS0jCkZcG8JM9PPknpOVh+IgKbL9xDbn7Bx1NfTwd85VNPY7cjRyelo9OPp5TuyinPsPGi7Lx8jFx3CRfuJhV6btvYtvCqXUPjdW4KuIc5f92EuZEe/L/sBBtzI43XUR5ikjLQZ/lZJGfkYkhLRywa1KTYActP0rKx4FAo9gY/BFBwl9V3/RqhWwPbIvcn7aDq+ZuXsYgqmLe3N/z9/XHnzh0AUEyO17NnzzIdNyAgAN26dVPa1rBVB+w/dgqPU7Phbm2KrKggtGjSEKOG9Mfwzk2QsWs6Mu4E4Odjd/A0PadM9ZdkU8B9RCSkwdLUAF/3bqB0OcXS1ABz+jTAiS/fwoBmNSGRAPuvPkLXJacwb/9NPEkrfXDxy4QQiEhIxfpzURizIRA+S87g5a95z8cqlafUrFyM3hBYZNDRkaBcxo8kpGThx2eT103rXq/KBB0AcLI0wbL3mkFHAuwMeoCtFwtf6hNCYGdQDLouOY29wQ+hIwE+aueKo1M6MeiQAgcoE1WwGTNmICUlBR4eHkqT4w0fPrxMx42Li1NahPDQtVgcCM9ATupTtHazxAIfR9T7Kg0LFy7EggULsGjRIvz99z/4+puvoWNkhiXHHDC/f6OyvrxCEtOy8euxgmA3rXu9Yiegc7I0wS9Dm+LjDm5YdDgMZ+48xobz9/Dn5Qf4pGMtjOngVuI8LE/SsnE2IhFnwxNxNiIRsc8mjyvJoeuxaO1mWS7zrSSkZmHUukDcik2BqYEuhrV2xvpzUYrBvpamBuWyEOb8Q6FIzc6Dp6MU77dx0fjxy1uHutaY2t0Diw7fxrcHbqK+vQVauBTMbnz3cRpm7b2uCI8N7C2wcFBjNHGsVoEtpsqIYYeogqk6OV5ZbDx/D/MO3ES+XMBQTwebRrdG0uN4AEC/fv0weXLBNP9NmzbFP/6ncSXkH2x1aYz3WjujgYNmL+3+eDgMqdl5aFTTAoNbOpW6f0MHKTaNbo1zEYlY+M9tXH8ow8/H7mDThfuY2LUuOrpb4cHTTDhIjfHgaSb+DX+Mf8MTcSs2Rek4Bno6aO1qiQ51rdC+rhWuPZDhm70Fc5c8vw17y4X7ePA0A/8b1uyVZgEuTlRiOkasu4iYpExYmRlg/ajWaOwoxccd3HA1Jhlz/rqJhNRsTNl5Fas+aFHqnWiqOnPnMQ5cfQQdCfD9gMblNmleefu0Uy1cf5iMv6/HYdymIMzt0wDXH8qwMeA+cvLkMNbXxeS362J0OzelBUeJnmPYIapgU6dOxYwZMzBs2DAAQOPGjXH//n34+fmVKezY2dkhLi4OPx65jRUnIwEADarLkeLqCCN9XVhZWUFPTw8NGjRQKufd0hMRe49ALoBvD9zE9nFtNTax27UHydh5OQYAMK9PQ7VOvu3qWOEv33Y4dD0WPx4JQ3RSRqHJ8V5W394CHZ+Fm1aulkoDXBs6SPFWvf/mLrkUlYRpf17DqbDH6L/iHNaMaIE6NuYlHF01V2OS8dGGQCSl58Clhgk2jW4Nl2dz0jy/ndheaozBqwJw7FY8VpyMwOdd65a53qzcfMz+q+D3M9LbtVwGeb8uEokEi9/1RNC9p0hIzcYX20MUz3Vyt8aC/o3gZPn6biGnqocRmEiDYmWZOB+ZiFiZ6rfKvurkeKVp07Ytlm/Zpwg6X77tjvyYa/Dy8gIAGBgYoFWrVggLU16M8M6dO2jTxB1G+jq4GJWEQ9djCx37VcjlAvP234QQwIBmNV9paQEdHQn6eDrg+JROmPK2e5H79Gpsh/8Na4rAr7vhn4kdMLNXfXSoa13knTz2UmPFeKF+TWti92fecJAaISoxHf1XnMfxW/Fqt/FFp8IS8N6aC0hKz0HjmlLs/sxbEXRe5OlUDQv6FVwyXHL8Dk6GJRTaR12/nYrE/ScZsLUwLPZ3VZWkZhVMcvgiiQTwG8igQ6Vj2CHSkFedN+X55HiHDh3CvXv3sHfvXixZsgQDBgxQ7JOUlISQkBDcunULABAWFoaQkBDExcUp9hkxYoRiCYSMnDxk1PXB3ZBzSL20B583N8aTf7ciKChIafK8qVOnYseOHVizZg0iIiKwfPlyHDhwAF9N+gKfdaoDAPjhUCgyc/LL/PvZF/IQV6KTYWKgixk9Pcp0LAM9HbR0LXpV6g/buqJf05qwNjcs8vmSNKopxf7P26O1qyXSsvMwdnMQlp8Ix6vctLrnygN8vDEIGTn56FDXCtvGtYWVWfFtGtLKCcPbOEMIYOK2YNxLTFe7zuciH6dh5amCkDvnnYYwN9LcJbmKUtQklEIA95+oNwcPvZkYdog0oLhlF1Tp4SltcjwA2L9/P5o1a6ZYumDYsGFo1qwZVq5cqdgnOjoasbGxSErPwftrLuJmvj3s+0+DSdRpzPqwJ/7880/s27cPjRr9N+h4wIABWLlyJRYvXozGjRvj//7v/7B79260b98en3SqhZrVjPFIloXfT0eW6feTlp2Hhf8UrFc1oUsdxZwnZeFmZYqXr4JpYkZcKzNDbPm4DT5oWxA8fjp6B75/XEH6s7l/SiOEwKrTkZiy8yry5AL9mzpg7chWKg0+ntunIZo7V0NKVh4+2XxZ5Tpfrn/2vhvIyZejk7s1ejUu3/mSXpfy+nvTm4Hz7IDz7FDZ7b/6CF9sCy60vbzmTSlKrCwTgVFJ+OloGKKTMiE11se6US3RwuXVV6L+53osPtt6BQZ6OvCf0umVLxcs/Oc2Vp6OhEsNExyd3BGGeqVPDqeK8l4gcdulaMz56wZy8wU87MyxZkTLEn8HcrnA93+HYu3ZKADA2A5umNmzvloDjuNTsvDOsrN4nJqN3k3ssfy9ZmqNmdoX/BCTdoTAUE8HRyd3LPKyWVVV1RfEJM3jpIJqYNihsniSlo0Bv51DdJJyL46OBDg3o8trmaV1R2A0Zuy5rpg7Rmqsh92feZd5gK0QAsP/7yLORz5Bj4Z2WPlhC7WPEZWYDp9fTiM3X+D/RrTU+Nwn5T0jbtC9JHy65QoS07JR3UQfK95vDu86hddqys7Lx1e7ruHAs4Uyv+5VH2M7vtoiloH3kvDe6gvIkwvM6uWBcR1rq1ROlpGLrktOITEtB1/5uGNCl7IPdK5sOAMyvYiTCtJr5+rqColEUujh6+sLoGDelw8//BB2dnYwNTVF8+bNsXv37lKPu2LFCri6usLIyAht2rTBpUuXlJ7/5JNPULt2bRgbG8Pa2hr9+vXD7du3y+U1viw1Kxcj11961pOip9TN3sRRCjsNXK4pTawsUynoFLQrD6YamLNFIpFg7rO7pg7fjMO5iES1j7Hg4C3k5gt0crdG1/o2ZW7Ty14cZFweWrpa4sDn7dDEUYqnGbn4cN0lbDgXpTSO5/lkgQeuPoK+rgS/Dm36ykEHAFq5WmJun4K75Bb+c1vl3/viI7eRmJaD2tamZaq/Mivvvzdpp0oddlRZIFEIgTlz5sDe3h7Gxsbo1q0bwsPDK7DVb67AwECs/DsQjhM2w9F3M+yGLQAAxQrTI0aMQFhYGPbv34/r169j4MCBGDJkCIKDC1/+eW7Hjh2YMmUK5s6diytXrsDT0xPdu3dHQsJ/d6u0aNEC69evR2hoKI4cOQIhBHx8fJCfX/ZBtSXJys3H2E1BuPEwBTVMDbBnfDucm9EFc96pD10JEBIjw7pz98q1DQDwb3hiodmA5QIamw24np05PmxbMBndtwduIi9f9bvEToYlwP92AvR0JJjTp4HGbmF/3eylxtj5iZdiccp5B25h2p/XcP9JOv6+/giDfj+PcxFPYGqgi3WjWqF/s5plrvODti54t4Uj5AKY8McVPHha8t8zOPop/rhUMCh+Qf/GGrtUSKQNKnXYWbRoEX7//XcsX74coaGhWLRoERYvXoxly5Yp9lm8eDGWLl2KlStX4uLFizA1NUX37t2RlVX6bKmkOflygeN3M+B3Oh66ptWha1Yd6eGXoFfNHnU9WwEoWJjy888/R+vWrVGrVi188803qFatGi5fvlzscZcsWYKxY8fio48+QoMGDbBy5UqYmJhg3bp1in3GjRuHjh07wtXVFc2bN8eCBQsQExODe/fuldvrzcuXY8IfwbhwNwlmhnrYOLo1alubwV5qjNHta+Gbdwq+lf/wdygCIstvBfGYpAwsPly4F0vTAzcnd3NHdRN93IlPw5YL91Uqk5Mnx/wDBXePfdTOFbWtzTTWnopgpK+LJUM88U3v+tCRALsuP0CnH09h/NZg3IlPg6mBLraP80KHutYaqU8ikWBB/0ZoXLOgR+nTLZeRlVt0gM/Ll+PrvTcgBDCwec3XNk6MqKqo1GHn/Pnz6NevH3r37g1XV1e8++678PHxUVzGEELg119/xTfffIN+/fqhSZMm2LRpEx49eoR9+/ZVbOPfEBk5edh4/h66/HwKM/f+N8GbyM9F+q1TMGvyNj7eeBmRj9Pg7e2NHTt2ICkpCXK5HNu3b0dWVhbeeuutIo+dk5ODy5cvK63vpKOjg27duiEgIKDIMunp6Vi/fj3c3Nzg5FT67LyvQi4XmL77Oo6HxsNATwf/N7JloQnbRnm7KnoBJvxxBY+SNX977OPUbHy49iIS03JgY26ouIT2fOCmJrv5pSb6+Kp7PQDAkmN3VFqfasP5KNxNTIeVmSG+0MAkeZWBRCLBxx1q4ZehTQs9l5mbDytzA43WZ6Svi5UftoClqQFuPEx5FmgKD7PcGHAft2JTIDXWx6xe9TXaBiJtUKnDTmkLJEZFRSEuLk7pZCiVStGmTZtiT4YAkJ2djZSUFKUHqSchJQs/HrkNL78TmLv/Ju4/yYCFkR6eX6TIuHMB8qw0mDbqitC4VPT89V90Hu+H7Jwc1KhRA4aGhvjkk0+wd+9e1KlTp8g6EhMTkZ+fr7S+EwDY2toqzS8DAL/99hvMzMxgZmaGf/75B8eOHYOBgWZPPEBBwP7+71DsvvIAujoSrHi/OdrWKvwtWiKR4IcBjdHA3gJP0nPwWQnfyl9FSlYuRq67hHtPMuBY3Rj7J7THuRldsG1sW5yd0blc7lAZ1soZDewtkJKVh5+O3ilx34TULCz1jwAATO9RTyvmeXlRUXP4aPLS4YtqVjPG8meLYe6+8gCbX+pZi5VlYsnRgokhZ/T0KHEuH6I3VaUOO8+n0Pfw8IC+vj6aNWuGSZMmKRZIfH7CU+Vk+CI/Pz9IpVLFo7x6ALRRWFwqpu66ivaLTmLFyUjIMnPhUsME3/VriAuzumLhoMbQlUiQdu0oTGq1xNeDvdHJ3Ro5+XL4zZ+HS7djsHzzHgQFBWHKlCkYMmQIrl+/XuZ2DR8+HMHBwTh9+jTc3d0xZMiQcrmUueJkhOK24sWDmuDtEu4sMjbQxaoPW6CaiT6uPpBh7l83X2lyupdl5uTj4w1BuBWbAiszA2we0wZ2UqNyH7ipqyPBvL4NAQDbA6Nx46Gs2H0XHw5DWnYePJ2qYVBzx3JpT0V63XO+eNexwsyeBT023x24hcB7/62a/t2BW0jPyUdz52oYqsJaY0Rvokoddl5cIPHKlSvYuHEjfvrpJ2zcuLFMx505cyZkMpniERMTo6EWaychBM6GJ2LEukvo/usZ7Lr8ADn5crR0qY6VH7TAiS/fwggvV5gY6GFoK2dsf782cqKvYvl3X+Kzt+pgw0etMLN9daReOQjTtz/HjzcMsD1SgklTZ6Fly5ZYsWJFkfVaWVlBV1cX8fHKU/bHx8fDzk55ojSpVIq6deuiY8eO+PPPP3H79m3s3btXo7+HzRfuK3o0Zr/TAINalH4Sd7I0wbJn38p3BMUoBpC+qtx8OXz/uIJL95Jg/myskJvV65tHpbWbJfp6OkAIPFv6oXB4C45+ij8vPwAAzOvTQGOLWlYm9lJj+A0sCPZA+Vw6fNnHHdzQx9MBeXKB8VuvID4lCydux+OfG3HQ1ZHg+wGNtfJ3TaQJlXoh0NIWSHx+wouPj4e9vb2iXHx8PJo2bVrscQ0NDWFoyK7eksTKMhEen4bw+FTsuvwAt+NSARTMHdOzkT0+7uCGZs5FT9d/aPc22NjY4MMhAwEUXNLxdimY76VXEwccjwW2XYrBsVsJkGfkFXvXlIGBAVq0aAF/f3/0798fACCXy+Hv76+05MHLhBAQQiA7u/RxJao6cPUR5jxbVPHzLnUwpr2bymU71LXG1O4eWHT4NubtvwkPOwu0cCn6d1cSuVxg6q6rOHE7AYZ6Olg7qhUaOrz+xR1n9vLAsVvxCLr/FPuvPkK/pv/defR8/SsAeLeFY7HvEW0wtJUzOrpbv7Y5XyQSCRYNaozw+FTcjkvF6A2BiE8p6L0c094N9e05RxhRcSp1z05pCyS6ubnBzs4O/v7+iudTUlJw8eJFxWKHpL7tl6Lh7XcCI9ZdwvxDobgdlwoTA12M8nbF6amdsWJ482JPYnK5HOvXr8fIkSOhp/dflvbw8ECdOnVwZ/cSzGljAHtJMu6e2IbggDOIMmuAmKSCsQ5du3bF8uXLFeWmTJmCNWvWYOPGjQgNDcVnn32G9PR0fPTRRwCAu3fvws/PD5cvX0Z0dDTOnz+PwYMHw9jYGL169dLI7+P0nceYsjMEQgAftnV5pUUVP+1UC70a2yE3X2D81stISFXvEpsQAt8dvIV9IY+gpyPB7x80R2u3V58ZuSzspcbw7VwwyZ3f37eVljTYfeUBrj6QwcxQD9N61KuQ9r1Or3vOFxMDPaz6sAWM9HRw81EKEtNyAACO1TjnDFFJKnXYKW2BRIlEgkmTJmHBggWKuVtGjBgBBwcHRU8Aqef5Gk8vXpyQANg3vh3m9W1Y6nIBx48fR3R0NEaPHq20XV9fH3///Tesra0x89MPcG3pOBjfPwfbPlMQYegOn1/OYM2Zu4iMjETUg1jFyuFDhw7FTz/9hDlz5qBp06YICQnB4cOHFeO0jIyM8O+//6JXr16oU6cOhg4dCnNzc5w/fx42NmWfwO7y/SR8uvkycvMF+ng64Nu+DV9prhiJRILF73qiro0Z4lOy4bv1CnLyVJ+vZql/BDacvwcA+GmwJ7p4aHYWYnV93KEWnCyNEZeShd9OFQxETsnKxaLDBQNlv+haBzbm5T+h4pvIQE8H2S/NdfTtgVsqrcNG9Kaq1MtFpKamYvbs2di7dy8SEhLg4OCA9957D3PmzFHcaSOEwNy5c7F69WokJyejffv2+O233+Durvq3by4X8Z/dl2Pw5a5rhbaX1xpPEQlpmLX3Oi5FFQy4dJAaITYlC0IUXDLzG9i4wta+uR2XgiErA5CSlYdO7tZYM6IlDPTK9v3g7uM09Ft+DqnZeRjp5YJv+zUqtczG8/cw99mloW/7NsRIb9cytUFTjt6Mw7jNl2Ggq4NjUzpiy4X7WPNvFGpZmeLwpI5l/l1R0c5HJuL9NRcLbX+d67ARVRZcG0sNDDsF8vLl6LP8HEJjlW/F15VIcHZG53LrqpfLBXZdjsGCQ6FIzVJe5VlHAvzl2w6Nakpf6+y70U8yMGjleTxOzUYLl+rYPKY1TAw0M8TNPzQeYzYGAQB+HuxZ4kDnv0IeYuL2EADApG51Mamb+pfQyosQAiPWXcK/4Ylo5GCBW7EpkAtg/Uet0Lme5peFoAKxsky0W3gC8hc+ucv73yhRZcW1sUhtK05GIjQ2BUZ6OuU6Qd3LdHQkGNrKGT++26TQc3IB9Fl+Di0WHMf7ay7guwO3sCsoBjceyoqctyZWlqm4BPYqYmWZ+PvaIwxbE4DHqdnwsDPHupGtNBZ0AKBrfVtMfDbJ3qy914u9hfvk7QR8ufMqgIJJCidWson5CtbNagAdCXDjUYri5JuQwtnLy1NF3AlGVNVV6rux6PW5GpOMpScK1hRb9G4TtHazfO0rC3s6VYOOBErfWIGCMUNJ6Tk4H/kE519YekFXRwI3K1N42Jmjvr0FHqdmYVPAfchf8RLYjsBozNxzXVG/pakBNo1uDamJ5ifEm9i1Lm48lMH/dgI+2XwZBz5vD0vT/yZBDLyXhE+3XEaeXGBAs5qY807lXFfK1FCv0Lpcs/bcQEd3a558y9HrvhOMqKrjZSzwMlZmTj56L/sXdx+n450m9lj+fvMKa8uOwGjM2nMD+UIovrH2a1oT4fFpCI1LQWhsCm7HpiI0LgXJGbmlHq+asb5Kc4/I5QLJmcrH05EA52Z0KbcTiSwzF/1XnENUYjq8a9fAptGtoaerg1uPUjB0dQBSs/LQ1cMGKz9sAX3dytkJy/EjRFSRVD1/s2eHsPCfUNx9nA5bC0Ms6F/6gNnyVNw31saOUjR2/G9OGSEEElKzC8JPXCr+vfMY54pYcPPlAKOO59P/l1fYkRrrY9WHLdB/xTmcj3yCOftvopVLdXx3sGDsUmtXS6wY3rzSBh3gv5mEXx4/Ul4zCRMRvQr27ODN7tk5fecxRq4rWFh185jWGlux+XUratCmjgTYMqZNkesYvexxajY+WHuxQgZ9/n09FuO3XlHaZi81wpHJHWFRBdaUKqo3rqLuoCOiNwt7dqhUyRk5mLrrvwGwVTXoAP8N2nz5pOtdx0ql8nVtzYss/zrGQjRzrgYJoDS3UXxKFtKz86pE2OH4ESKq7Bh23lBCCHy97wYSUrNRy9oU03t4VHSTyqysJ92KOmlHJabj5e7V8r6Epmn2UuMq01YievMw7Lyh9l99hEPXYqGnI8GvQ5vC2EC3opukEWU96VbESZvjXoiIylflHflI5eZRcia+2VewqOUXXeuiiWO1im3QG47zphARlS/27Lxh5HKBqX9eRWpWHpo6VcP4t2pXdJMIHPdCRFSeGHbeMBvO38O5iCcw1tfFkiGe0KvEtzW/aTjuhYiofPBM9wYJj0/FwsO3AQBf966PWtZmFdwiIiKi8sew84bIyZNj8s4Q5OTJ8VY9awxvw3lQiIjozcCw84ZY6h+OGw9TUM1EH4sHNamU6ywRERGVB4adN8Dl+0n47VQEAMBvQGPYWBhVcIuIiIheH4YdLZeenYcpO69CLoCBzWqiZ2P7im4SERHRa8Wwo+UWHArF/ScZqFnNGPP6Nazo5hAREb12vPVcS8XKMrHn8gNsuxQNiQT4abBnlVhniYiISNMYdrTQjsBozNxzXbH8QPvaVvCqXaNiG0VERFRBeBlLy8TKMpWCDgCci0xErCyz4hpFRERUgRh2tExUYrpS0AH+W0GbiIjoTcSwo2XcrEzx8hQ6XEGbiIjeZAw7WsZeaoz2dawUP3MFbSIietNxgLIWepRcMD5nYte6GNbaiUGHiIjeaAw7Wib6SQYiH6dDV0eC0e3dIDXm7eZERPRm42UsLXMyLAEA0NKlOoMOERERGHa0zonbBWGni4dNBbeEiIiocmDY0SIZOXkIuPsEAMMOERHRcww7WuR8xBPk5MnhWN0YdWzMKro5RERElQLDjhZ5Pl6ni4cNJC9PtkNERPSGYtjREkIInHw2XqdzPV7CIiIieo5hR0uExafikSwLRvo6XPSTiIjoBQw7WuL5XVjeta1gpK9bwa0hIiKqPBh2tITiEhbvwiIiIlLCsKMFkjNycPn+UwBA53rWFdwaIiKiyoVhRwucCU+EXADutmZwrM7VzYmIiF7EsKMFeAmLiIioeAw7VVy+XODU8/l1eMs5ERFRIQw7VVxITDKeZuTCwkgPLVyqV3RziIiIKh2GnSru+SWsju7W0NPln5OIiOhlPDtWcS8uEUFERESFMexUYfEpWbj5KAUSCdDJnbecExERFYVhpwp7fgnL07EaapgZVnBriIiIKieGnSrs+RIRvIRFRERUPIadKio7Lx9nIxIBMOwQERGVhGGniroUlYSMnHzYmBuioYNFRTeHiIio0mLYqaJO3n4MAHirnjUkEkkFt4aIiKjyYtiponjLORERkWoYdqqgqMR0RCWmQ19XgvZ1ecs5ERFRSRh2qqDnd2G1drOEmaFeBbeGiIiocmPYqYIUq5xz4U8iIqJSMexUMenZebgY9QQAx+sQERGpgmGnijkbkYjcfAGXGiZwszKt6OYQERFVegw7VcyLl7B4yzkREVHpGHaqECEEbzknIiJSE8NOFXLzUQriU7JhYqCLNrUsK7o5REREVQLDThVy6lmvTrs6VjDU063g1hAREVUNDDtVyAneck5ERKQ2tcPOyZMny6MdVIqk9BwExyQDADp7cNZkIiIiVakddnr06IHatWtjwYIFiImJKY82URFO30mAEEB9ewvYS40rujlERERVhtph5+HDh5gwYQL+/PNP1KpVC927d8fOnTuRk5NTHu2jZ048W+W8C3t1iIiI1KJ22LGyssLkyZMREhKCixcvwt3dHePHj4eDgwO++OILXL16tTza+UbLy5fjNG85JyIieiVlGqDcvHlzzJw5ExMmTEBaWhrWrVuHFi1aoEOHDrh586am2vjGC45JRkpWHqqZ6KOpU/WKbg4REVGV8kphJzc3F3/++Sd69eoFFxcXHDlyBMuXL0d8fDwiIiLg4uKCwYMHa6SBDx8+xAcffIAaNWrA2NgYjRs3RlBQkOJ5IQTmzJkDe3t7GBsbo1u3bggPD9dI3ZXF87uwOrlbQ1eHsyYTERGpQ+2w8/nnn8Pe3h6ffPIJ3N3dERwcjICAAHz88ccwNTWFq6srfvrpJ9y+fbvMjXv69CnatWsHfX19/PPPP7h16xZ+/vlnVK/+X+/G4sWLsXTpUqxcuRIXL16EqakpunfvjqysrDLXX1k8XyKCl7CIiIjUp6dugVu3bmHZsmUYOHAgDA0Ni9zHyspKI7eoL1q0CE5OTli/fr1im5ubm+L/hRD49ddf8c0336Bfv34AgE2bNsHW1hb79u3DsGHDytyGivYwORO341KhIyno2SEiIiL1qN2z4+/vj/fee6/YoAMAenp66NSpU5kaBgD79+9Hy5YtMXjwYNjY2KBZs2ZYs2aN4vmoqCjExcWhW7duim1SqRRt2rRBQEBAscfNzs5GSkqK0qOyet6r09y5OqqZGFRwa4iIiKoetcOOn58f1q1bV2j7unXrsGjRIo006rm7d+/i999/R926dXHkyBF89tln+OKLL7Bx40YAQFxcHADA1tZWqZytra3iuaL4+flBKpUqHk5OThpttyY9XyKiMy9hERERvRK1w86qVavg4eFRaHvDhg2xcuVKjTTqOblcjubNm+OHH35As2bNMG7cOIwdO7bM9cycORMymUzxqKyTI2bl5uNcxBMAXCKCiIjoVakdduLi4mBvb19ou7W1NWJjYzXSqOfs7e3RoEEDpW3169dHdHQ0AMDOzg4AEB8fr7RPfHy84rmiGBoawsLCQulRGV24+wSZufmwlxqhvr15RTeHiIioSlI77Dg5OeHcuXOFtp87dw4ODg4aadRz7dq1Q1hYmNK2O3fuwMXFBUDBYGU7Ozv4+/srnk9JScHFixfh5eWl0bZUhINXHwEAWrtZQiLhLedERESvQu27scaOHYtJkyYhNzcXXbp0AVAwaHnatGn48ssvNdq4yZMnw9vbGz/88AOGDBmCS5cuYfXq1Vi9ejUAQCKRYNKkSViwYAHq1q0LNzc3zJ49Gw4ODujfv79G2/K6bb8UjT+vPAQA7L/6CN61a2BoK+cKbhUREVHVIxFCCHUKCCEwY8YMLF26VLEelpGREaZPn445c+ZovIEHDx7EzJkzER4eDjc3N0yZMgVjx45Vas/cuXOxevVqJCcno3379vjtt9/g7u6uch0pKSmQSqWQyWSV4pJWrCwT7RaegPyFv4yuRIKzMzpzEVAiIqJnVD1/qx12nktLS0NoaCiMjY1Rt27dEm9Fr+wqW9g5H5mI99dcLLR929i28KpdowJaREREVPmoev5W+zLWc2ZmZmjVqtWrFqcSOFYv3HujK5HA1cqkAlpDRERUtb1S2AkKCsLOnTsRHR2tuJT13J49ezTSsDfZrUepSj/rSiT4YWAjXsIiIiJ6BWrfjbV9+3Z4e3sjNDQUe/fuRW5uLm7evIkTJ05AKpWWRxvfONsuFdxa/2FbF2wb2xZnZ3Tm4GQiIqJXpHbY+eGHH/DLL7/gwIEDMDAwwP/+9z/cvn0bQ4YMgbMzT8hl9eBpBs6EPwYAfNzBDV61a7BHh4iIqAzUDjuRkZHo3bs3AMDAwADp6emQSCSYPHmy4pZwenU7A2MgBNCuTg241DCt6OYQERFVeWqHnerVqyM1tWBMSc2aNXHjxg0AQHJyMjIyMjTbujdMXr4cO4IKlq54rzV7yYiIiDRB7QHKHTt2xLFjx9C4cWMMHjwYEydOxIkTJ3Ds2DF07dq1PNr4xjgV9hjxKdmwNDXA2w1sSy9AREREpVI77CxfvhxZWVkAgK+//hr6+vo4f/48Bg0ahG+++UbjDXyTPB+Y/G4LRxjq6VZwa4iIiLSDWmEnLy8PBw8eRPfu3QEAOjo6mDFjRrk07E0TK8vEybAEAMDQVk4V3BoiIiLtodaYHT09PXz66aeKnh3SnJ2BDyAXQBs3S9S2Nqvo5hAREWkNtQcot27dGiEhIeXQlDdXvlxg57OBye+34cBkIiIiTVJ7zM748eMxZcoUxMTEoEWLFjA1Vb49ukmTJhpr3JviTPhjPEzORDUTfXRvaFfRzSEiItIqaoedYcOGAQC++OILxTaJRAIhBCQSCfLz8zXXujfE9mcDkwc2c4SRPgcmExERaZLaYScqKqo82vHGSkjJwvHQgoHJ77XmwGQiIiJNUzvsuLi4lEc73li7Lj9AvlygpUt11LU1r+jmEBERaR21w86mTZtKfH7EiBGv3Jg3jVwusD2w4BLWMM6YTEREVC7UDjsTJ05U+jk3NxcZGRkwMDCAiYkJw44azkUmIiYpE+ZGeujd2L6im0NERKSV1L71/OnTp0qPtLQ0hIWFoX379ti2bVt5tFFrbb9UcLv5gGY1YWzAgclERETlQe2wU5S6deti4cKFhXp9qHiJadk4eisOADCsFS9hERERlReNhB2gYHblR48eaepwWm/35QfIzRfwdKqGBg4WFd0cIiIiraX2mJ39+/cr/SyEQGxsLJYvX4527dpprGHaTAiB7YHPZkzm7eZERETlSu2w079/f6WfJRIJrK2t0aVLF/z888+aapdWu3A3CVGJ6TAz1MM7TRwqujlERERaTe2wI5fLy6Mdb5Rtz2ZM7tvUAaaGav8JiIiISA0aG7NDqnmanoPDNwoGJr/PuXWIiIjKndphZ9CgQVi0aFGh7YsXL8bgwYM10ihttvvKA+Tky9GopgUa1ZRWdHOIiIi0ntph58yZM+jVq1eh7T179sSZM2c00iht9eLA5PfYq0NERPRaqB120tLSYGBgUGi7vr4+UlJSNNKoymbhwoWQSCSYNGkSAODevXuQSCRFPnbt2lXscXR0dOD/5Vu4v+gdfNDWVVHmxx9/VOzTt29fODs7w8jICPb29vjwww95Sz8REVEZqB12GjdujB07dhTavn37djRo0EAjjapMAgMDsWrVKjRp0kSxzcnJCbGxsUqPb7/9FmZmZujZs2exxxq38hgcfTdj/Gp/xMbGYt26dZBIJBg0aJBin86dO2Pnzp0ICwvD7t27ERkZiXfffbdcXyMREZE2U/tWoNmzZ2PgwIGIjIxEly5dAAD+/v7Ytm1bib0aVVFaWhqGDx+ONWvWYMGCBYrturq6sLOzU9p37969GDJkCMzMzIo8liwjF6dicqFrVh0fd28GO7vq+Ouvv9C5c2fUqlVLsd/kyZMV/+/i4oIZM2agf//+yM3Nhb6+voZfIRERkfZTu2enT58+2LdvHyIiIjB+/Hh8+eWXePDgAY4fP15oDp6qztfXF71790a3bt1K3O/y5csICQnBmDFjit1nb/ADZOfJ4WFnjqZO1RAfH49Dhw6VWCYpKQlbt26Ft7c3gw4REdEreqVJXnr37o3evXtrui2Vyvbt23HlyhUEBgaWuu/atWtRv359eHt7F/n8ywOTJRIJNm7cCHNzcwwcOLDQ/tOnT8fy5cuRkZGBtm3b4uDBg2V7MURERG8wtXt2AgMDcfHixULbL168iKCgII00qqLFxMRg4sSJ2Lp1K4yMjErcNzMzE3/88UeJPTTBMcm4HZcKQz0d9G9WEwCwbt06DB8+vMjjT506FcHBwTh69Ch0dXUxYsQICCHK9qKIiIjeUGqHHV9fX8TExBTa/vDhQ/j6+mqkURXt8uXLSEhIQPPmzaGnpwc9PT2cPn0aS5cuhZ6eHvLz8xX7/vnnn8jIyMCIESOKPd72ZzMmv9PEAVJjffz7778ICwvDxx9/XOT+VlZWcHd3x9tvv43t27fj77//xoULFzT7IomIiN4Qal/GunXrFpo3b15oe7NmzXDr1i2NNKqide3aFdevX1fa9tFHH8HDwwPTp0+Hrq6uYvvatWvRt29fWFtbF3ms1KxcHLgaCwB479min2vXrkWLFi3g6elZalueL8+RnZ39Sq+FiIjoTad22DE0NER8fLzSHUQAEBsbCz097VjnydzcHI0aNVLaZmpqiho1aihtj4iIwJkzZ/D3338XeRwPDw/0GDUZmbmOqGtjhhYu1ZGSkoJdu3YVuWjqxYsXERgYiPbt26N69eqIjIzE7NmzUbt2bXh5eWn2RRIREb0h1L6M5ePjg5kzZ0Imkym2JScnY9asWXj77bc12rjKbt26dXB0dISPj0+Rz4eFheFw8F0AwLBnA5O3b98OIQTee++9QvubmJhgz5496Nq1K+rVq4cxY8agSZMmOH36NAwNDcv1tRAREWkriVBz5OvDhw/RsWNHPHnyBM2aNQMAhISEwNbWFseOHYOTk1O5NLQ8paSkQCqVQiaTwcLCQiPH3BEYjRl7ruP5b3fOOw0wur2bRo5NREREqp+/1Q47AJCeno6tW7fi6tWrMDY2RpMmTfDee+9V2blgNB12YmWZaLfwBOQv/GZ1JRKcndEZ9lLjMh+fiIiIVD9/v9IgG1NTU4wbN+6VG6ftohLTlYIOAOQLgXuJGQw7REREr9krjyi+desWoqOjkZOTo7S9b9++ZW5UVedmZQodCQr17LhamVRco4iIiN5Qaoedu3fvYsCAAbh+/TokEolisjuJRAIASnPQvKnspcbwG9gYs/bcQL4Q0JVI8MPARuzVISIiqgBqh52JEyfCzc0N/v7+cHNzw6VLl/DkyRN8+eWX+Omnn8qjjVXS0FbO6OhujXuJGXC1MmHQISIiqiBqh52AgACcOHECVlZW0NHRgY6ODtq3bw8/Pz988cUXCA4OLo92Vkn2UmOGHCIiogqm9jw7+fn5MDc3B1CwrMGjR48AAC4uLggLC9Ns64iIiIjKSO2enUaNGuHq1atwc3NDmzZtsHjxYhgYGGD16tWFZlUmIiIiqmhqh51vvvkG6enpAIDvvvsO77zzDjp06IAaNWpgx44dGm8gERERUVm80qSCL0tKSkL16tUVd2RVNeUxgzIRERGVr3KdVPBllpaWmjgMERERkcapPUCZiIiIqCph2CEiIiKtxrBDREREWk3tsHPmzBnk5eUV2p6Xl4czZ85opFFEREREmqJ22OncuTOSkpIKbZfJZOjcubNGGkVERESkKWqHHSFEkbeYP3nyBKamphppFBEREZGmqHzr+cCBAwEUrG4+atQoGBoaKp7Lz8/HtWvX4O3trfkWEhEREZWBymFHKpUCKOjZMTc3h7HxfwtcGhgYoG3bthg7dqzmW0hERERUBiqHnfXr1wMAXF1d8dVXX/GSFREREVUJao/ZmTZtmtKYnfv37+PXX3/F0aNHNdowIiIiIk1QO+z069cPmzZtAgAkJyejdevW+Pnnn9GvXz/8/vvvGm8gERERUVmoHXauXLmCDh06AAD+/PNP2NnZ4f79+9i0aROWLl2q8QYSERERlYXaYScjIwPm5uYAgKNHj2LgwIHQ0dFB27Ztcf/+fY03kIiIiKgs1A47derUwb59+xATE4MjR47Ax8cHAJCQkFDi8upEREREFUHtsDNnzhx89dVXcHV1RevWreHl5QWgoJenWbNmGm8gERERUVmoHXbeffddREdHIygoCEeOHFFs79q1K3755ReNNu5lCxcuhEQiwaRJkxTbsrKy4Ovrixo1asDMzAyDBg1CfHx8ubaDiIiIqo5XWvXczs4O5ubmOHbsGDIzMwEArVq1goeHh0Yb96LAwECsWrUKTZo0Udo+efJkHDhwALt27cLp06fx6NEjxWzPRERERGqHnSdPnqBr165wd3dHr169EBsbCwAYM2YMvvzyS403EADS0tIwfPhwrFmzBtWrV1dsl8lkWLt2LZYsWYIuXbqgRYsWWL9+Pc6fP48LFy6US1uIiIioalE77EyePBn6+vqIjo6GiYmJYvvQoUNx+PBhjTbuOV9fX/Tu3RvdunVT2n758mXk5uYqbffw8ICzszMCAgKKPV52djZSUlKUHkRERKSdVF4u4rmjR4/iyJEjcHR0VNpet27dcrn1fPv27bhy5QoCAwMLPRcXFwcDAwNUq1ZNabutrS3i4uKKPaafnx++/fZbTTeViIiIKiG1e3bS09OVenSeS0pKUloJXRNiYmIwceJEbN26FUZGRho77syZMyGTyRSPmJgYjR2biIiIKhe1w06HDh0Uy0UAgEQigVwux+LFi9G5c2eNNu7y5ctISEhA8+bNoaenBz09PZw+fRpLly6Fnp4ebG1tkZOTg+TkZKVy8fHxsLOzK/a4hoaGsLCwUHoQERGRdlL7MtbixYvRtWtXBAUFIScnB9OmTcPNmzeRlJSEc+fOabRxXbt2xfXr15W2ffTRR/Dw8MD06dPh5OQEfX19+Pv7Y9CgQQCAsLAwREdHK+b/ISIiojeb2mGnUaNGuHPnDpYvXw5zc3OkpaVh4MCB8PX1hb29vUYbZ25ujkaNGiltMzU1RY0aNRTbx4wZgylTpsDS0hIWFhb4/PPP4eXlhbZt22q0LURERFQ1qR12oqOj4eTkhK+//rrI55ydnTXSMFX98ssv0NHRwaBBg5CdnY3u3bvjt99+e61tICIiospLIoQQ6hTQ1dVFbGwsbGxslLY/efIENjY2yM/P12gDX4eUlBRIpVLIZDKO3yEiIqoiVD1/qz1AWQgBiURSaHtaWppG75giIiIi0gSVL2NNmTIFQMHdV7Nnz1a6/Tw/Px8XL15E06ZNNd5AIiIiorJQOewEBwcDKOjZuX79OgwMDBTPGRgYwNPTE1999ZXmW0hERERUBiqHnZMnTwIouPX7f//7H8e2EBERUZWg9t1Y69evL492EBEREZULtQcoExEREVUlDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Sp12PHz80OrVq1gbm4OGxsb9O/fH2FhYUr7ZGVlwdfXFzVq1ICZmRkGDRqE+Pj4CmoxERERVTaVOuycPn0avr6+uHDhAo4dO4bc3Fz4+PggPT1dsc/kyZNx4MAB7Nq1C6dPn8ajR48wcODACmw1ERERVSYSIYSo6Eao6vHjx7CxscHp06fRsWNHyGQyWFtb448//sC7774LALh9+zbq16+PgIAAtG3bVqXjpqSkQCqVQiaTwcLCojxfAhEREWmIqufvSt2z8zKZTAYAsLS0BABcvnwZubm56Natm2IfDw8PODs7IyAgoNjjZGdnIyUlRelBRERE2qnKhB25XI5JkyahXbt2aNSoEQAgLi4OBgYGqFatmtK+tra2iIuLK/ZYfn5+kEqlioeTk1N5Np2IiIgqUJUJO76+vrhx4wa2b99e5mPNnDkTMplM8YiJidFAC4mIiKgy0qvoBqhiwoQJOHjwIM6cOQNHR0fFdjs7O+Tk5CA5OVmpdyc+Ph52dnbFHs/Q0BCGhobl2WQiIiKqJCp1z44QAhMmTMDevXtx4sQJuLm5KT3fokUL6Ovrw9/fX7EtLCwM0dHR8PLyet3NJSIiokqoUvfs+Pr64o8//sBff/0Fc3NzxTgcqVQKY2NjSKVSjBkzBlOmTIGlpSUsLCzw+eefw8vLS+U7sYiIiEi7VepbzyUSSZHb169fj1GjRgEomFTwyy+/xLZt25CdnY3u3bvjt99+K/Ey1st46zkREVHVo+r5u1KHndeFYYeIiKjq0cp5doiIiIjUxbBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIq2lN2FmxYgVcXV1hZGSENm3a4NKlSxXdJCIiIqoEtCLs7NixA1OmTMHcuXNx5coVeHp6onv37khISKjophEREVEF04qws2TJEowdOxYfffQRGjRogJUrV8LExATr1q2r6KYRERFRBdOr6AaUVU5ODi5fvoyZM2cqtuno6KBbt24ICAgoskx2djays7MVP8tkMgBASkpK+TaWiIiINOb5eVsIUeJ+VT7sJCYmIj8/H7a2tkrbbW1tcfv27SLL+Pn54dtvvy203cnJqVzaSEREROUnNTUVUqm02OerfNh5FTNnzsSUKVMUP8vlciQlJaFGjRqQSCQaqyclJQVOTk6IiYmBhYXFay3Pul9/3WUtz7rfrLrLWp51s+6qUr6sdZdECIHU1FQ4ODiUuF+VDztWVlbQ1dVFfHy80vb4+HjY2dkVWcbQ0BCGhoZK26pVq1ZeTYSFhUWZ/sBlKc+6X3/dZS3Put+sustannWz7qpSvqx1F6ekHp3nqvwAZQMDA7Ro0QL+/v6KbXK5HP7+/vDy8qrAlhEREVFlUOV7dgBgypQpGDlyJFq2bInWrVvj119/RXp6Oj766KOKbhoRERFVMK0IO0OHDsXjx48xZ84cxMXFoWnTpjh8+HChQcuvm6GhIebOnVvoktnrKM+6X3/dZS3Put+sustannWz7qpSvqx1a4JElHa/FhEREVEVVuXH7BARERGVhGGHiIiItBrDDhEREWk1hh0iIiLSagw75WjFihVwdXWFkZER2rRpg0uXLqlU7syZM+jTpw8cHBwgkUiwb98+lev08/NDq1atYG5uDhsbG/Tv3x9hYWEql//999/RpEkTxeRPXl5e+Oeff1Qu/6KFCxdCIpFg0qRJKu0/b948SCQSpYeHh4fK9T18+BAffPABatSoAWNjYzRu3BhBQUEqlXV1dS1Ut0Qiga+vb6ll8/PzMXv2bLi5ucHY2Bi1a9fG/PnzS12r5UWpqamYNGkSXFxcYGxsDG9vbwQGBhbar7T3hhACc+bMgb29PYyNjdGtWzeEh4erXH7Pnj3w8fFRzCYeEhKicv25ubmYPn06GjduDFNTUzg4OGDEiBF49OiRSnXPmzcPHh4eMDU1RfXq1dGtWzdcvHhR5ba/6NNPP4VEIsGvv/6qUtlRo0YV+tv36NFDrbpDQ0PRt29fSKVSmJqaolWrVoiOji61bFHvO4lEgh9//FGlutPS0jBhwgQ4OjrC2NhYsRiyKmXj4+MxatQoODg4wMTEBD169FC8X1T5LMnKyoKvry9q1KgBMzMzDBo0SDHBqyrlV69ejbfeegsWFhaQSCRITk5WPFda+aSkJHz++eeoV68ejI2N4ezsjC+++AIymUyluj/55BPUrl0bxsbGsLa2Rr9+/RRLDKnzOSqEQM+ePRW/X1XKvvXWW4X+3p9++qladQcEBKBLly4wNTWFhYUFOnbsiO+++67Esvfu3Sv2/bZr1y6V6o6Li8OHH34IOzs7mJqaonnz5ti9e7dKZSMjIzFgwABYW1vDwsICQ4YMKTQhcHlh2CknO3bswJQpUzB37lxcuXIFnp6e6N69OxISEkotm56eDk9PT6xYsULtek+fPg1fX19cuHABx44dQ25uLnx8fJCenq5SeUdHRyxcuBCXL19GUFAQunTpgn79+uHmzZtqtSMwMBCrVq1CkyZN1CrXsGFDxMbGKh5nz55VqdzTp0/Rrl076Ovr459//sGtW7fw888/o3r16iq398V6jx07BgAYPHhwqWUXLVqE33//HcuXL0doaCgWLVqExYsXY9myZSrVDQAff/wxjh07hs2bN+P69evw8fFBt27d8PDhQ6X9SntvLF68GEuXLsXKlStx8eJFmJqaonv37sjKylKpfHp6Otq3b49FixYV+3xx5TMyMnDlyhXMnj0bV65cwZ49exAWFoa+ffuqVLe7uzuWL1+O69ev4+zZs3B1dYWPjw8eP36sUvnn9u7diwsXLihNH69K2R49eii9B7Zt26Zy+cjISLRv3x4eHh44deoUrl27htmzZ8PIyKjUsi/WGRsbi3Xr1kEikWDQoEEq1T1lyhQcPnwYW7ZsQWhoKCZNmoQJEyZg//79JZYVQqB///64e/cu/vrrLwQHB8PFxQXdunVDenq6Sp8lkydPxoEDB7Br1y6cPn0ajx49wsCBAwGo9lmUkZGBHj16YNasWYXaV1r5R48e4dGjR/jpp59w48YNbNiwAYcPH8aYMWNUqrtFixZYv349QkNDceTIEQgh4OPjg/z8fLU+R3/99VelZYZULTt27Filv/vixYtVLh8QEIAePXrAx8cHly5dQmBgICZMmICzZ8+WWNbJyanQ++3bb7+FmZkZevbsqVLdI0aMQFhYGPbv34/r169j4MCBGDJkCA4cOFBi2fT0dPj4+EAikeDEiRM4d+4ccnJy0KdPH8jl8kK/V40TVC5at24tfH19FT/n5+cLBwcH4efnp9ZxAIi9e/e+cjsSEhIEAHH69OlXPkb16tXF//3f/6m8f2pqqqhbt644duyY6NSpk5g4caJK5ebOnSs8PT1fqY3Tp08X7du3f6WyRZk4caKoXbu2kMvlpe7bu3dvMXr0aKVtAwcOFMOHD1eproyMDKGrqysOHjyotL158+bi66+/Lrbcy+8NuVwu7OzsxI8//qjYlpycLAwNDcW2bdtKLf+iqKgoAUAEBwerXH9RLl26JACI+/fvq11WJpMJAOL48eMq1/3gwQNRs2ZNcePGDeHi4iJ++eUXlcqOHDlS9OvXr8T2lFR+6NCh4oMPPnilsi/r16+f6NKli8rlGzZsKL777julbUW9d14uGxYWJgCIGzduKLbl5+cLa2trsWbNmkJ1v/xZkpycLPT19cWuXbsU+4SGhgoAIiAgoNTyLzp58qQAIJ4+fVrk6y6t/HM7d+4UBgYGIjc3V+2yV69eFQBERESEynUHBweLmjVritjY2GL/tkWVVedzsajybdq0Ed98880rlX1Z06ZNC31+lVTe1NRUbNq0SWk/S0vLQu+Zl8seOXJE6OjoCJlMptgnOTlZSCQScezYsVJfS1mxZ6cc5OTk4PLly+jWrZtim46ODrp164aAgIDX2haZTAYAsLS0VLtsfn4+tm/fjvT0dLWW3vD19UXv3r2VXr+qwsPD4eDggFq1amH48OGIjo5Wqdz+/fvRsmVLDB48GDY2NmjWrBnWrFmjdv1Awd9vy5YtGD16tEoLw3p7e8Pf3x937twBAFy9ehVnz55Fz549VaovLy8P+fn5MDIyUtpubGyscs8WAERFRSEuLk7p9y6VStGmTZvX/r57TiaTQSKRqL32XE5ODlavXg2pVApPT0+Vysjlcnz44YeYOnUqGjZsqHZbT506BRsbG9SrVw+fffYZnjx5onK9hw4dgru7O7p37w4bGxu0adNGrcvPz8XHx+PQoUMYM2aMymW8vb2xf/9+PHz4EEIInDx5Enfu3IGPj0+J5bKzswFA6X2no6MDQ0PDIt93L3+WXL58Gbm5uUrvNw8PDzg7Oxf5fivLZ5Gq5WUyGSwsLKCnp1doe0ll09PTsX79eri5ucHJyUmlujMyMvD+++9jxYoVxa7DWFLdW7duhZWVFRo1aoSZM2ciIyNDpfIJCQm4ePEibGxs4O3tDVtbW3Tq1Emlv9nLLl++jJCQkGLfb0WV9/b2xo4dO5CUlAS5XI7t27cjKysLb731Volls7OzIZFIlCYWNDIygo6Ojlqfc6+s3OPUG+jhw4cCgDh//rzS9qlTp4rWrVurdSyUoWcnPz9f9O7dW7Rr106tcteuXROmpqZCV1dXSKVScejQIZXLbtu2TTRq1EhkZmYKIdT7BvP333+LnTt3iqtXr4rDhw8LLy8v4ezsLFJSUkota2hoKAwNDcXMmTPFlStXxKpVq4SRkZHYsGGDym1/bseOHUJXV1c8fPhQpf3z8/PF9OnThUQiEXp6ekIikYgffvhBrTq9vLxEp06dxMOHD0VeXp7YvHmz0NHREe7u7sWWefm9ce7cOQFAPHr0SGm/wYMHiyFDhpRa/kWa6NnJzMwUzZs3F++//77KZQ8cOCBMTU2FRCIRDg4O4tKlSyrX/cMPP4i3335b0RunTs/Otm3bxF9//SWuXbsm9u7dK+rXry9atWol8vLySi3//Fu9iYmJWLJkiQgODhZ+fn5CIpGIU6dOqfS6n1u0aJGoXr264t+PKm3PysoSI0aMEACEnp6eMDAwEBs3biy1bE5OjnB2dhaDBw8WSUlJIjs7WyxcuFAAED4+Pkpli/os2bp1qzAwMChUT6tWrcS0adNKLf+i0np2VPkse/z4sXB2dhazZs1SueyKFSuEqampACDq1atXZK9OceXHjRsnxowZo/i5qL9NcWVXrVolDh8+LK5duya2bNkiatasKQYMGKBS3QEBAQKAsLS0FOvWrRNXrlwRkyZNEgYGBuLOnTsqve7nPvvsM1G/fv0inyuu/NOnT4WPj4/i/WZhYSGOHDlSatmEhARhYWEhJk6cKNLT00VaWpqYMGGCACDGjRtXbBs1hWGnHFSWsPPpp58KFxcXERMTo1a57OxsER4eLoKCgsSMGTOElZWVuHnzZqnloqOjhY2Njbh69apimzph52VPnz4VFhYWKl1C09fXF15eXkrbPv/8c9G2bVu16/Xx8RHvvPOOyvtv27ZNODo6im3btolr166JTZs2CUtLS7WCVkREhOjYsaMAIHR1dUWrVq3E8OHDhYeHR7FlKnPYycnJEX369BHNmjVT6rYurWxaWpoIDw8XAQEBYvTo0cLV1VXEx8eXWj4oKEjY2toqBVR1ws7LIiMjVb6E9vzf+3vvvae0X58+fcSwYcPUqrtevXpiwoQJxT5fVPkff/xRuLu7i/3794urV6+KZcuWCTMzs0KXBooqGxQUJDw9PRXvu+7du4uePXuKHj16KO1X1GeJOmGntM+i0sJOaeVlMplo3bq16NGjh8jJyVG5bHJysrhz5444ffq06NOnj2jevHmhoFlU+b/++kvUqVNHpKamKrYV9ftV9TPY39+/yEtoRZV//u985syZSvs2btxYzJgxQ+W6MzIyhFQqFT/99FORzxdXfsKECaJ169bi+PHjIiQkRMybN09IpVJx7dq1UsseOXJE1KpVS0gkEqGrqys++OAD0bx5c/Hpp5+W8NvRDIadcpCdnS10dXULvfFHjBgh+vbtq9axXjXs+Pr6CkdHR3H37l21y76sa9euKiXvvXv3Kj40nz8AKN7YRX1LLk3Lli2V/gEXx9nZWelblhBC/Pbbb8LBwUGt+u7duyd0dHTEvn37VC7j6Ogoli9frrRt/vz5ol69emrVLUTByf55WBkyZIjo1atXsfu+/N54foJ+OaB07NhRfPHFF6WWf1FZwk5OTo7o37+/aNKkiUhMTFSr7Mvq1KlTZC/Zy+V/+eUXxfvsxfeejo6OcHFxeaW6raysxMqVK0utOzs7W+jp6Yn58+cr7Tdt2jTh7e2tct1nzpwRAERISEixbXq5fEZGhtDX1y803mvMmDGie/fuKtednJwsEhIShBAF4w3Hjx+veK64z5LnJ+iXA4qzs7NYsmRJqeVfVFLYKa18SkqK8PLyEl27di0UVNT5HMzOzhYmJibijz/+KLX8xIkTi32/derUSe2609LSBABx+PDhUuu+e/euACA2b96stH3IkCGKXlRV6t60aZPQ19dX/N1fVFz5iIiIQuO8hCg4R3zyyScq1/348WPF39rW1lYsXry42H01hWN2yoGBgQFatGgBf39/xTa5XA5/f3+1xr68CiEEJkyYgL179+LEiRNwc3Mr8zHlcrni+n5JunbtiuvXryMkJETxaNmyJYYPH46QkBDo6uqqVW9aWhoiIyNhb29f6r7t2rUrdJvjnTt34OLiolad69evh42NDXr37q1ymYyMDOjoKP9T0tXVfaU7DExNTWFvb4+nT5/iyJEj6Nevn8pl3dzcYGdnp/S+S0lJwcWLF8v9ffdcbm4uhgwZgvDwcBw/fhw1atQo0/FUfe99+OGHuHbtmtJ7z8HBAVOnTsWRI0fUrvfBgwd48uSJSu89AwMDtGrVqszvv7Vr16JFixYqj1ECCn7fubm5ZX7/SaVSWFtbIzw8HEFBQejXr1+pnyUtWrSAvr6+0vstLCwM0dHR8PLyKvNnkSrlU1JS4OPjAwMDA+zfv18x/uhV6hYFX/6RnZ1davkZM2YUer8BwC+//IJ169apXffz8vb29qXW7erqCgcHhyLfb87OzirXvXbtWvTt2xfW1tZKv4OSyj8fV1TU+y0/P1/luq2srFCtWjWcOHECCQkJijs2y1W5x6k31Pbt24WhoaHYsGGDuHXrlhg3bpyoVq2aiIuLK7VsamqqCA4OFsHBwQKAYhzAy3e0FOWzzz4TUqlUnDp1SsTGxioeGRkZKrV7xowZ4vTp0yIqKkpcu3ZNzJgxQ0gkEnH06FGVyr9MnctYX375pTh16pSIiooS586dE926dRNWVlZFfvN42aVLl4Senp74/vvvRXh4uNi6daswMTERW7ZsUbmt+fn5wtnZWUyfPl3lMkIU3MlTs2ZNcfDgQREVFSX27NkjrKysCnXll+Tw4cPin3/+EXfv3hVHjx4Vnp6eok2bNoW65Et7byxcuFBUq1ZNMf6kX79+ws3NTfGNt7TyT548EcHBweLQoUMCgNi+fbsIDg4WsbGxpZbPyckRffv2FY6OjiIkJETp/ZednV1i2bS0NDFz5kwREBAg7t27J4KCgsRHH30kDA0NFd8i1f138eJlrJLKpqamiq+++koEBASIqKgocfz4cdG8eXNRt25dkZWVpVLde/bsEfr6+mL16tUiPDxcLFu2TOjq6op///1XpXbLZDJhYmIifv/990Kvo7TynTp1Eg0bNhQnT54Ud+/eFevXrxdGRkbit99+K7Xszp07xcmTJ0VkZKTYt2+fcHFxEQMHDhRCqPZZ8umnnwpnZ2dx4sQJERQUJLy8vBSXk1UpHxsbK4KDg8WaNWsEAHHmzBkRHBwsnjx5Ump5mUwm2rRpIxo3biwiIiKU9vn0009LLBsZGSl++OEHERQUJO7fvy/OnTsn+vTpIywtLUV8fPwrfY7iWc9ZaWUjIiLEd999J4KCgkRUVJT466+/RK1atUTHjh1V/r398ssvwsLCQuzatUuEh4eLb775RhgZGYn3339fpXaHh4cLiUQi/vnnH6XtpdWdk5Mj6tSpIzp06CAuXrwoIiIixE8//SQkEono1atXqXWvW7dOBAQEiIiICLF582ZhaWkppkyZUuzvVJMYdsrRsmXLhLOzszAwMBCtW7cWFy5cUKnc8y7dlx8jR44stWxR5QCI9evXq1T36NGjhYuLizAwMBDW1taia9eurxx0hFAv7AwdOlTY29sLAwMDUbNmTTF06NAiBwwW58CBA6JRo0bC0NBQeHh4iNWrV6vV1iNHjggAIiwsTK1yKSkpYuLEicLZ2VkYGRmJWrVqia+//lpkZ2erfIwdO3aIWrVqCQMDA2FnZyd8fX1FcnJyof1Ke2/I5XIxe/ZsYWtrKwwNDUXXrl2VXk9p5devX1/k83Pnzi21/PNLX0U9Tp48WWLZzMxMMWDAAOHg4CAMDAyEvb296Nu3r9IAZXX/XbwYdkoqm5GRIXx8fIS1tbXQ19cXLi4uYuzYsUpfTFSpe+3ataJOnTrCyMhIeHp6Ki6FqlJ21apVwtjY+JX+5rGxsWLUqFHCwcFBGBkZiXr16omff/5ZyOXyUsv+73//E46OjkJfX184OzuLb775RvG+VeWzJDMzU4wfP15Ur15dmJiYiAEDBiiCsSrl586dW+w+pZUv7rWV9Hhe9uHDh6Jnz57CxsZG6OvrC0dHR/H++++L27dvq9z2lz0PO6WVjY6OFh07dhSWlpbC0NBQ1KlTR0ydOlUxtk3Vuv38/ISjo6MwMTERXl5e4t9//1W57MyZM4WTk5PIz88v9BpKK3/nzh0xcOBAYWNjI0xMTESTJk3Epk2bVCo7ffp0YWtrK/T19UXdunUV79PXQfLsBRIRERFpJY7ZISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQEb3k1KlTkEgkSE5OruimEJEGMOwQERGRVmPYISIiIq3GsENElY5cLoefnx/c3NxgbGwMT09P/PnnnwD+u8R06NAhNGnSBEZGRmjbti1u3LihdIzdu3ejYcOGMDQ0hKurK37++Wel57OzszF9+nQ4OTnB0NAQderUwdq1a5X2uXz5Mlq2bAkTExN4e3sXWmmaiKoGhh0iqnT8/PywadMmrFy5Ejdv3sTkyZPxwQcf4PTp04p9pk6dip9//hmBgYGwtrZGnz59kJubC6AgpAwZMgTDhg3D9evXMW/ePMyePRsbNmxQlB8xYgS2bduGpUuXIjQ0FKtWrYKZmZlSO77++mv8/PPPCAoKgp6eHkaPHv1aXj8RaRYXAiWiSiU7OxuWlpY4fvw4vLy8FNs//vhjZGRkYNy4cejcuTO2b9+OoUOHAgCSkpLg6OiIDRs2YMiQIRg+fDgeP36Mo0ePKspPmzYNhw4dws2bN3Hnzh3Uq1cPx44dQ7du3Qq14dSpU+jcuTOOHz+Orl27AgD+/vtv9O7dG5mZmTAyMirn3wIRaRJ7doioUomIiEBGRgbefvttmJmZKR6bNm1CZGSkYr8Xg5ClpSXq1auH0NBQAEBoaCjatWundNx27dohPDwc+fn5CAkJga6uLjp16lRiW5o0aaL4f3t7ewBAQkJCmV8jEb1eehXdACKiF6WlpQEADh06hJo1ayo9Z2hoqBR4XpWxsbFK++nr6yv+XyKRACgYT0REVQt7doioUmnQoAEMDQ0RHR2NOnXqKD2cnJwU+124cEHx/0+fPsWdO3dQv359AED9+vVx7tw5peOeO3cO7u7u0NXVRePGjSGXy5XGABGR9mLPDhFVKubm5vjqq68wefJkyOVytG/fHjKZDOfOnYOFhQVcXFwAAN999x1q1KgBW1tbfP3117CyskL//v0BAF9++SVatWqF+fPnY+jQoQgICMDy5cvx22+/AQBcXV0xcuRIjB49GkuXLoWnpyfu37+PhIQEDBkypKJeOhGVE4YdIqp05s+fD2tra/j5+eHu3buoVq0amjdvjlmzZikuIy1cuBATJ05EeHg4mjZtigMHDsDAwAAA0Lx5c+zcuRNz5szB/PnzYW9vj++++w6jRo1S1PH7779j1qxZGD9+PJ48eQJnZ2fMmjWrIl4uEZUz3o1FRFXK8zulnj59imrVqlV0c4ioCuCYHSIiItJqDDtERESk1XgZi4iIiLQae3aIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIq/0/aNuF5S+4oysAAAAASUVORK5CYII=", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "plt.plot(np.arange(len(epochs_x)), epochs_acc, marker = '.')\n", - "plt.xlabel('epoch')\n", - "plt.ylabel('test accuracy')\n", - "plt.ylim(0, 100)\n", - "plt.xticks(np.arange(len(epochs_x)))\n", - "for i, txt in enumerate(epochs_acc):\n", - " if i%5 == 0:\n", - " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", - " else:\n", - " pass\n", - "plt.show()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "speck-rescnn", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/tests/test_nonsequential/transfer-learning/baseline-SCNN-3.ipynb b/tests/test_nonsequential/transfer-learning/baseline-SCNN-3.ipynb deleted file mode 100644 index 5e27fd6f..00000000 --- a/tests/test_nonsequential/transfer-learning/baseline-SCNN-3.ipynb +++ /dev/null @@ -1,525 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import torch, random\n", - "import torch.nn as nn\n", - "from tqdm.notebook import tqdm\n", - "\n", - "from tonic.transforms import ToFrame\n", - "from torch.utils.data import DataLoader\n", - "from torch.nn import CrossEntropyLoss\n", - "from torch.optim import Adam\n", - "\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "\n", - "from seq_model import SNN" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "torch.backends.cudnn.deterministic = True\n", - "random.seed(1)\n", - "torch.manual_seed(1)\n", - "torch.cuda.manual_seed(1)\n", - "np.random.seed(1)" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "batch_size = 3\n", - "batch_size_pre = 32\n", - "num_workers = 1\n", - "epochs_pretrain = 1\n", - "epochs = 30\n", - "lr = 1e-3\n", - "n_time_steps = 50" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "device: NVIDIA GeForce RTX 3070 Ti\n" - ] - } - ], - "source": [ - "if torch.cuda.is_available():\n", - " device = torch.device('cuda:0')\n", - " print('device: ', torch.cuda.get_device_name(0))\n", - "else:\n", - " device = torch.device('cpu')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Training/Testing helper functions" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "def train(batch_size, feature_map_size, dataloader_train, model, loss_fn, optimizer, epochs, test_func, dataloader_test, phase):\n", - " epochs_y = []\n", - " epochs_x = []\n", - " epochs_acc = []\n", - " model.train()\n", - "\n", - " for e in range(epochs):\n", - " losses = []\n", - " batches = []\n", - " batch_count = 0\n", - " train_p_bar = tqdm(dataloader_train)\n", - "\n", - " for X, y in train_p_bar:\n", - " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", - " X = X.reshape(-1, feature_map_size[2], feature_map_size[0], feature_map_size[1]).to(dtype=torch.float, device=device)\n", - " y = y.to(dtype=torch.long, device=device)\n", - "\n", - " # forward\n", - " pred = model(X)\n", - "\n", - " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", - " pred = pred.reshape(batch_size, n_time_steps, -1)\n", - "\n", - " # accumulate all time-steps output for final prediction\n", - " pred = pred.sum(dim = 1)\n", - " loss = loss_fn(pred, y)\n", - "\n", - " # gradient update\n", - " optimizer.zero_grad()\n", - " loss.backward()\n", - " optimizer.step()\n", - "\n", - " # detach the neuron states and activations from current computation graph(necessary)\n", - " model.detach_neuron_states()\n", - "\n", - " train_p_bar.set_description(f\"{phase} - Epoch {e} - BPTT Training Loss: {round(loss.item(), 4)}\")\n", - "\n", - " batch_count += 1\n", - " losses.append(loss.item())\n", - " batches.append(batch_count)\n", - "\n", - " epochs_y.append(losses)\n", - " epochs_x.append(batches)\n", - "\n", - " acc = test_func(feature_map_size, dataloader_test, model)\n", - " print(f'{phase} - Epoch {e} accuracy: {acc}')\n", - " epochs_acc.append(acc)\n", - "\n", - " return epochs_x, epochs_y, epochs_acc\n" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "def test(feature_map_size, dataloader, model):\n", - " correct_predictions = []\n", - " with torch.no_grad():\n", - " test_p_bar = tqdm(dataloader)\n", - " for X, y in test_p_bar:\n", - " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", - " X = X.reshape(-1, feature_map_size[2], feature_map_size[0], feature_map_size[1]).to(dtype=torch.float, device=device)\n", - " y = y.to(dtype=torch.long, device=device)\n", - "\n", - " # forward\n", - " output = model(X)\n", - "\n", - " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", - " output = output.reshape(batch_size, n_time_steps, -1)\n", - "\n", - " # accumulate all time-steps output for final prediction\n", - " output = output.sum(dim=1)\n", - "\n", - " # calculate accuracy\n", - " pred = output.argmax(dim=1, keepdim=True)\n", - "\n", - " # compute the total correct predictions\n", - " correct_predictions.append(pred.eq(y.view_as(pred)))\n", - "\n", - " test_p_bar.set_description(f\"Testing Model...\")\n", - " \n", - " correct_predictions = torch.cat(correct_predictions)\n", - " return correct_predictions.sum().item()/(len(correct_predictions))*100" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Pre-training loop" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Loading the pre-training data. Dataset used to pre-train the network such that its parameters are set within a \"good\" region of the parameters space (i.e., hopefully training on a \"simpler\" dataset sets the wheights to values that improve the training on a harder dataset)." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The transformed array is in shape [Time-Step, Channel, Height, Width] --> (50, 2, 34, 34)\n" - ] - } - ], - "source": [ - "from tonic.datasets.nmnist import NMNIST\n", - "\n", - "root_dir = \"../NMNIST\"\n", - "_ = NMNIST(save_to=root_dir, train=True)\n", - "_ = NMNIST(save_to=root_dir, train=False)\n", - "\n", - "to_raster = ToFrame(sensor_size=NMNIST.sensor_size, n_time_bins=n_time_steps)\n", - "\n", - "snn_train_dataset_pre = NMNIST(save_to=root_dir, train=True, transform=to_raster)\n", - "snn_test_dataset_pre = NMNIST(save_to=root_dir, train=False, transform=to_raster)\n", - "\n", - "sample_data, label = snn_train_dataset_pre[0]\n", - "print(f\"The transformed array is in shape [Time-Step, Channel, Height, Width] --> {sample_data.shape}\")\n", - "\n", - "snn_train_dataloader_pre = DataLoader(snn_train_dataset_pre, batch_size=batch_size_pre, num_workers=num_workers, drop_last=True, shuffle=True)\n", - "snn_test_dataloader_pre = DataLoader(snn_test_dataset_pre, batch_size=batch_size_pre, num_workers=num_workers, drop_last=True, shuffle=False)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "instantiating model..." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "snn = SNN(10, 10, batch_size_pre).to(device)\n", - "snn.init_weights()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "loss and optimizer..." - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "optimizer = Adam(snn.parameters(), lr=lr, betas=(0.9, 0.999), eps=1e-8)\n", - "loss_fn = CrossEntropyLoss()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "pre-training the model..." - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "a5bfe44485924a4a85b3357e62c24e75", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - " 0%| | 0/1875 [00:00 {sample_data.shape}\")\n", - "\n", - "snn_train_dataloader = DataLoader(snn_train_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True)\n", - "snn_test_dataloader = DataLoader(snn_test_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=False)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "instantiating model..." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "snn = SNN(11, 810, batch_size).to(device)\n", - "snn.init_weights()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "loading weights from pre-training..." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "snn.load_conv_params()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "loss and optimizer..." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "optimizer = Adam(snn.parameters(), lr=lr, betas=(0.9, 0.999), eps=1e-8)\n", - "loss_fn = CrossEntropyLoss()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "training the model..." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "epochs_x_dvs128, epochs_y_dvs128, epochs_acc_dvs128 = train(\n", - " batch_size,\n", - " DVSGesture.sensor_size, \n", - " snn_train_dataloader, \n", - " snn, \n", - " loss_fn, \n", - " optimizer, \n", - " epochs, \n", - " test, \n", - " snn_test_dataloader,\n", - " 'post-training'\n", - " )" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "y_avg = []\n", - "for y in epochs_y_dvs128:\n", - " y_avg.append(np.mean(y))\n", - "\n", - "plt.plot(np.arange(len(epochs_x_dvs128)), y_avg, marker = '.')\n", - "plt.xlabel('epoch')\n", - "plt.ylabel('average loss')\n", - "plt.ylim(0,)\n", - "plt.xticks(np.arange(len(epochs_x_dvs128)))\n", - "for i, txt in enumerate(y_avg):\n", - " if i%2 == 0:\n", - " pass\n", - " else:\n", - " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "plt.plot(np.arange(len(epochs_x_dvs128)), epochs_acc_dvs128, marker = '.')\n", - "plt.xlabel('epoch')\n", - "plt.ylabel('test accuracy')\n", - "plt.ylim(0, 100)\n", - "plt.xticks(np.arange(len(epochs_x_dvs128)))\n", - "for i, txt in enumerate(epochs_acc_dvs128):\n", - " if i%2 == 0:\n", - " pass\n", - " else:\n", - " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", - "plt.show()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "speck-rescnn", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/tests/test_nonsequential/transfer-learning/seq_model.py b/tests/test_nonsequential/transfer-learning/seq_model.py deleted file mode 100644 index a109c47e..00000000 --- a/tests/test_nonsequential/transfer-learning/seq_model.py +++ /dev/null @@ -1,86 +0,0 @@ -import torch -import torch.nn as nn -import sinabs.layers as sl -from sinabs.activation.surrogate_gradient_fn import PeriodicExponential - -class SNN(nn.Module): - def __init__(self, nb_classes, pool2lin_size, batch_size) -> None: - super().__init__() - - self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False) - self.iaf1 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential()) - self.pool1 = nn.AvgPool2d(2,2) - - self.conv2 = nn.Conv2d(10, 10, 2, 1, bias=False) - self.iaf2 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential()) - self.pool2 = nn.AvgPool2d(3,3) - - self.conv3 = nn.Conv2d(10, 10, 3, 1, bias=False) - self.iaf3 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential()) - self.pool3 = nn.AvgPool2d(2,2) - - self.flat = nn.Flatten() - - self.fc1 = nn.Linear(pool2lin_size, 100, bias=False) - self.iaf4 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential()) - - self.fc2 = nn.Linear(100, 100, bias=False) - self.iaf5 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential()) - - self.fc3 = nn.Linear(100, 100, bias=False) - self.iaf6 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential()) - - self.fc4 = nn.Linear(100, nb_classes, bias=False) - self.iaf7 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential()) - - def export_conv_params(self): - torch.save(self.conv1.state_dict(), 'seq_conv1_weights.pth') - torch.save(self.conv2.state_dict(), 'seq_conv2_weights.pth') - torch.save(self.conv3.state_dict(), 'seq_conv3_weights.pth') - - def load_conv_params(self): - self.conv1.load_state_dict(torch.load('seq_conv1_weights.pth')) - self.conv2.load_state_dict(torch.load('seq_conv2_weights.pth')) - self.conv3.load_state_dict(torch.load('seq_conv3_weights.pth')) - - def detach_neuron_states(self): - for name, layer in self.named_modules(): - if name != '': - if isinstance(layer, sl.StatefulLayer): - for name, buffer in layer.named_buffers(): - buffer.detach_() - - def init_weights(self): - for name, layer in self.named_modules(): - if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear): - nn.init.xavier_normal_(layer.weight.data) - - def forward(self, x): - - con1_out = self.conv1(x) - iaf1_out = self.iaf1(con1_out) - pool1_out = self.pool1(iaf1_out) - - conv2_out = self.conv2(pool1_out) - iaf2_out = self.iaf2(conv2_out) - pool2_out = self.pool2(iaf2_out) - - conv3_out = self.conv3(pool2_out) - iaf3_out = self.iaf3(conv3_out) - pool3_out = self.pool3(iaf3_out) - - flat_out = self.flat(pool3_out) - - fc1_out = self.fc1(flat_out) - iaf4_out = self.iaf4(fc1_out) - - fc2_out = self.fc2(iaf4_out) - iaf5_out = self.iaf5(fc2_out) - - fc3_out = self.fc3(iaf5_out) - iaf6_out = self.iaf6(fc3_out) - - fc4_out = self.fc4(iaf6_out) - iaf7_out = self.iaf7(fc4_out) - - return iaf7_out \ No newline at end of file diff --git a/tests/test_nonsequential/using_AvgPool2d/exp_set_A/baseline-SCNN-example_3-SumPool.ipynb b/tests/test_nonsequential/using_AvgPool2d/exp_set_A/baseline-SCNN-example_3-SumPool.ipynb deleted file mode 100644 index a25e3392..00000000 --- a/tests/test_nonsequential/using_AvgPool2d/exp_set_A/baseline-SCNN-example_3-SumPool.ipynb +++ /dev/null @@ -1,1539 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import torch, random, sys\n", - "import torch.nn as nn\n", - "import sinabs.layers as sl\n", - "from tqdm.notebook import tqdm\n", - "\n", - "from tonic.datasets.dvsgesture import DVSGesture\n", - "from tonic.transforms import ToFrame\n", - "from torch.utils.data import DataLoader\n", - "from torch.nn import CrossEntropyLoss\n", - "from torch.optim import Adam\n", - "\n", - "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential\n", - "from sinabs.exodus.layers import IAFSqueeze\n", - "\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "\n", - "sys.path.append('../../utils')\n", - "\n", - "from weight_initialization import rescale_method_1\n", - "import tonic" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "torch.backends.cudnn.enabled = False\n", - "torch.backends.cudnn.deterministic = True\n", - "random.seed(1)\n", - "torch.manual_seed(1)\n", - "torch.cuda.manual_seed(1)\n", - "np.random.seed(1)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Loading Data" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "batch_size = 3\n", - "num_workers = 1\n", - "epochs = 30\n", - "lr = 1e-3" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "root_dir = \"../../DVSGESTURE\"\n", - "_ = DVSGesture(save_to=root_dir, train=True)\n", - "_ = DVSGesture(save_to=root_dir, train=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "n_time_steps = 50\n", - "to_raster = ToFrame(sensor_size=DVSGesture.sensor_size, n_time_bins=n_time_steps)\n", - "\n", - "snn_train_dataset = DVSGesture(save_to=root_dir, train=True, transform=to_raster)\n", - "snn_test_dataset = DVSGesture(save_to=root_dir, train=False, transform=to_raster)" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The transformed array is in shape [Time-Step, Channel, Height, Width] --> (50, 2, 128, 128)\n" - ] - } - ], - "source": [ - "sample_data, label = snn_train_dataset[0]\n", - "print(f\"The transformed array is in shape [Time-Step, Channel, Height, Width] --> {sample_data.shape}\")" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "disk_cache_train = tonic.DiskCachedDataset(\n", - " dataset=snn_train_dataset,\n", - " cache_path='./cached_train'\n", - ")\n", - "snn_train_dataloader = DataLoader(disk_cache_train, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True)\n", - "\n", - "disk_cache_test = tonic.DiskCachedDataset(\n", - " dataset=snn_test_dataset,\n", - " cache_path='./cached_test'\n", - ")\n", - "snn_test_dataloader = DataLoader(disk_cache_test, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=False)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Network Module\n", - "\n", - "We need to define a `nn.Module` implementing the network we want the chip to reproduce." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "device: NVIDIA GeForce RTX 3070 Ti\n" - ] - } - ], - "source": [ - "if torch.cuda.is_available():\n", - " device = torch.device('cuda:0')\n", - " print('device: ', torch.cuda.get_device_name(0))\n", - "else:\n", - " device = torch.device('cpu')" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "class SNN(nn.Module):\n", - " def __init__(self) -> None:\n", - " super().__init__()\n", - "\n", - " self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False)\n", - " self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - " self.pool1 = sl.SumPool2d(2,2)\n", - "\n", - " self.conv2 = nn.Conv2d(10, 10, 2, 1, bias=False)\n", - " self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - " self.pool2 = sl.SumPool2d(3,3)\n", - "\n", - " self.conv3 = nn.Conv2d(10, 10, 3, 1, bias=False)\n", - " self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - " self.pool3 = sl.SumPool2d(2,2)\n", - "\n", - " self.flat = nn.Flatten()\n", - "\n", - " self.fc1 = nn.Linear(810, 100, bias=False)\n", - " self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - "\n", - " self.fc2 = nn.Linear(100, 100, bias=False)\n", - " self.iaf5 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - "\n", - " self.fc3 = nn.Linear(100, 100, bias=False)\n", - " self.iaf6 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - "\n", - " self.fc4 = nn.Linear(100, 11, bias=False)\n", - " self.iaf7 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - "\n", - " def detach_neuron_states(self):\n", - " for name, layer in self.named_modules():\n", - " if name != '':\n", - " if isinstance(layer, sl.StatefulLayer):\n", - " for name, buffer in layer.named_buffers():\n", - " buffer.detach_()\n", - "\n", - " def init_weights(self):\n", - " for name, layer in self.named_modules():\n", - " if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear):\n", - " nn.init.xavier_normal_(layer.weight.data)\n", - "\n", - " def rescale_conv_weights(self, rescale_fn, lambda_):\n", - " rescale_fn(self.conv2, [(2, 2)], lambda_)\n", - " rescale_fn(self.conv3, [(3, 3)], lambda_)\n", - "\n", - " def forward(self, x):\n", - " \n", - " con1_out = self.conv1(x)\n", - " iaf1_out = self.iaf1(con1_out)\n", - " pool1_out = self.pool1(iaf1_out)\n", - "\n", - " conv2_out = self.conv2(pool1_out)\n", - " iaf2_out = self.iaf2(conv2_out)\n", - " pool2_out = self.pool2(iaf2_out)\n", - "\n", - " conv3_out = self.conv3(pool2_out)\n", - " iaf3_out = self.iaf3(conv3_out)\n", - " pool3_out = self.pool3(iaf3_out)\n", - "\n", - " flat_out = self.flat(pool3_out)\n", - " \n", - " fc1_out = self.fc1(flat_out)\n", - " iaf4_out = self.iaf4(fc1_out)\n", - "\n", - " fc2_out = self.fc2(iaf4_out)\n", - " iaf5_out = self.iaf5(fc2_out)\n", - "\n", - " fc3_out = self.fc3(iaf5_out)\n", - " iaf6_out = self.iaf6(fc3_out)\n", - "\n", - " fc4_out = self.fc4(iaf6_out)\n", - " iaf7_out = self.iaf7(fc4_out)\n", - "\n", - " return iaf7_out" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "snn = SNN().to(device)" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "snn.init_weights()" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "recaling factor: 2.0 (computed using 1 kernels and lambda 0.5)\n", - "recaling factor: 4.5 (computed using 1 kernels and lambda 0.5)\n" - ] - } - ], - "source": [ - "lambda_ = 0.5\n", - "snn.rescale_conv_weights(rescale_method_1, lambda_)" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "optimizer = Adam(snn.parameters(), lr=lr, betas=(0.9, 0.999), eps=1e-8)\n", - "loss_fn = CrossEntropyLoss()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Define train and test" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [], - "source": [ - "def train(dataloader, model, loss_fn, optimizer, epochs, test_func, dataloader_test):\n", - " epochs_y = []\n", - " epochs_x = []\n", - " epochs_acc = []\n", - " model.train()\n", - "\n", - " for e in range(epochs):\n", - " losses = []\n", - " batches = []\n", - " batch_count = 0\n", - " train_p_bar = tqdm(snn_train_dataloader)\n", - "\n", - " for X, y in train_p_bar:\n", - " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", - " X = X.reshape(-1, 2, 128, 128).to(dtype=torch.float, device=device)\n", - " y = y.to(dtype=torch.long, device=device)\n", - "\n", - " # forward\n", - " pred = model(X)\n", - "\n", - " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", - " pred = pred.reshape(batch_size, n_time_steps, -1)\n", - "\n", - " # accumulate all time-steps output for final prediction\n", - " pred = pred.sum(dim = 1)\n", - " loss = loss_fn(pred, y)\n", - "\n", - " # gradient update\n", - " optimizer.zero_grad()\n", - " loss.backward()\n", - " optimizer.step()\n", - "\n", - " # detach the neuron states and activations from current computation graph(necessary)\n", - " model.detach_neuron_states()\n", - "\n", - " train_p_bar.set_description(f\"Epoch {e} - BPTT Training Loss: {round(loss.item(), 4)}\")\n", - "\n", - " batch_count += 1\n", - " losses.append(loss.item())\n", - " batches.append(batch_count)\n", - "\n", - " epochs_y.append(losses)\n", - " epochs_x.append(batches)\n", - "\n", - " acc = test_func(dataloader_test, model)\n", - " print(f'Epoch {e} accuracy: {acc}')\n", - " epochs_acc.append(acc)\n", - "\n", - " return epochs_x, epochs_y, epochs_acc\n" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [], - "source": [ - "def test(dataloader, model):\n", - " correct_predictions = []\n", - " with torch.no_grad():\n", - " test_p_bar = tqdm(dataloader)\n", - " for X, y in test_p_bar:\n", - " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", - " X = X.reshape(-1, 2, 128, 128).to(dtype=torch.float, device=device)\n", - " y = y.to(dtype=torch.long, device=device)\n", - "\n", - " # forward\n", - " output = model(X)\n", - "\n", - " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", - " output = output.reshape(batch_size, n_time_steps, -1)\n", - "\n", - " # accumulate all time-steps output for final prediction\n", - " output = output.sum(dim=1)\n", - "\n", - " # calculate accuracy\n", - " pred = output.argmax(dim=1, keepdim=True)\n", - "\n", - " # compute the total correct predictions\n", - " correct_predictions.append(pred.eq(y.view_as(pred)))\n", - "\n", - " test_p_bar.set_description(f\"Testing Model...\")\n", - " \n", - " correct_predictions = torch.cat(correct_predictions)\n", - " return correct_predictions.sum().item()/(len(correct_predictions))*100" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Training loop" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "d7307153ad334c27a70afe651a5daaf4", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - " 0%| | 0/359 [00:00" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "y_avg = []\n", - "for y in epochs_y:\n", - " y_avg.append(np.mean(y))\n", - "\n", - "plt.plot(np.arange(len(epochs_x)), y_avg, marker = '.')\n", - "plt.xlabel('epoch')\n", - "plt.ylabel('average loss')\n", - "plt.ylim(0,)\n", - "plt.xticks(np.arange(len(epochs_x)))\n", - "for i, txt in enumerate(y_avg):\n", - " if i%2 == 0:\n", - " pass\n", - " else:\n", - " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAG2CAYAAACZEEfAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAAB8eUlEQVR4nO3dd1TV9f8H8OdlXPaQDTLdey/cJuLKnZqZWprmNyxHmSPN0hIzLcvM9ctVrjS3pbkHoYLgVgREUaaIXPa69/37g7h5Zd0rF4Hr83HOPUc+n/f6wJX74j0lQggBIiIiIh2lV9kNICIiIqpIDHaIiIhIpzHYISIiIp3GYIeIiIh0GoMdIiIi0mkMdoiIiEinMdghIiIincZgh4iIiHQagx0iIiLSaQx2iIiISKdVarBz9uxZDBgwAC4uLpBIJNi3b5/KfSEEPv/8czg7O8PExAQ+Pj4IDw9XSZOcnIzRo0fD0tIS1tbWmDBhAtLT01/iUxAREVFVVqnBTkZGBpo3b45Vq1YVe3/p0qX48ccfsWbNGly8eBFmZmbo3bs3srOzlWlGjx6Nmzdv4tixYzh06BDOnj2LSZMmvaxHICIioipOUlUOApVIJNi7dy8GDx4MoKBXx8XFBR9//DE++eQTAIBMJoOjoyM2bdqEN998E7dv30ajRo0QFBSENm3aAACOHDmCfv364dGjR3BxcamsxyEiIqIqwqCyG1CSqKgoxMfHw8fHR3nNysoK7du3R2BgIN58800EBgbC2tpaGegAgI+PD/T09HDx4kUMGTKk2LJzcnKQk5Oj/FqhUCA5ORm2traQSCQV91BERESkNUIIpKWlwcXFBXp6JQ9WVdlgJz4+HgDg6Oioct3R0VF5Lz4+Hg4ODir3DQwMYGNjo0xTHH9/f3z55ZdabjERERFVhocPH8LV1bXE+1U22KlIc+bMwYwZM5Rfy2QyuLu74+HDh7C0tKzElhEREZG6UlNT4ebmBgsLi1LTVdlgx8nJCQCQkJAAZ2dn5fWEhAS0aNFCmSYxMVElX35+PpKTk5X5i2NkZAQjI6Mi1y0tLRnsEBERVTNlTUGpsvvseHl5wcnJCSdOnFBeS01NxcWLF+Ht7Q0A8Pb2RkpKCi5fvqxMc/LkSSgUCrRv3/6lt5mIiIiqnkrt2UlPT0dERITy66ioKFy5cgU2NjZwd3fHtGnT8NVXX6Fu3brw8vLC/Pnz4eLiolyx1bBhQ/Tp0wcTJ07EmjVrkJeXhylTpuDNN9/kSiwiIiICUMnBTnBwMHr06KH8unAezbhx47Bp0yZ8+umnyMjIwKRJk5CSkoLOnTvjyJEjMDY2VubZunUrpkyZgp49e0JPTw/Dhg3Djz/++NKfhYiIiKqmKrPPTmVKTU2FlZUVZDIZ5+wQERFVE+p+flfZOTtERERE2sBgh4iIiHQagx0iIiLSaQx2iIiISKcx2CEiIiKdxmCHiIiIdBqDHSIiItJpDHaIiIhIpzHYISIiIp3GYIeIiIh0GoMdIiIi0mkMdoiIiEinMdghIiIincZgh4iIiHQagx0iIiLSaQx2iIiISKcx2CEiIiKdxmCHiIiIdBqDHSIiItJpDHaIiIhIpzHYISIiIp3GYIeIiIh0GoMdIiIi0mkMdoiIiEinMdghIiIincZgh4iIiHQagx0iIiLSaQx2iIiISKcx2CEiIiKdxmCHiIiIdBqDHSIiItJpDHaIiIhIpzHYISIiIp3GYIeIiIh0GoMdIiIi0mkMdoiIiEinMdghIiIincZgh4iIiHQagx0iIiLSaQx2iIiISKcx2CEiIiKdxmCHiIiIdBqDHSIiItJpDHaIiIhIpzHYISIiIp3GYIeIiIh0GoMdIiIi0mkMdoiIiEinMdghIiIincZgh4iIiHQagx0iIiLSaQx2iIiISKcx2CEiIiKdxmCHiIiIdBqDHSIiItJpDHaIiIhIpzHYISIiIp3GYIeIiIh0GoMdIqqWPD09IZFIirz8/PwAAJGRkRgyZAjs7e1haWmJESNGICEhodQyz549iwEDBsDFxQUSiQT79u0rkqa4OiUSCb799tuKeEwi0gIGO0RULQUFBSEuLk75OnbsGABg+PDhyMjIgK+vLyQSCU6ePImAgADk5uZiwIABUCgUJZaZkZGB5s2bY9WqVSWmebbOuLg4bNiwARKJBMOGDdP6MxKRdhhUdgOIiF6Evb29ytdLlixB7dq10a1bNxw7dgz3799HaGgoLC0tAQCbN29GjRo1cPLkSfj4+BRbZt++fdG3b99S63VyclL5ev/+/ejRowdq1apVjqchoorEnh0iqvZyc3Px22+/Yfz48ZBIJMjJyYFEIoGRkZEyjbGxMfT09HD+/Hmt1ZuQkIDDhw9jwoQJWiuTiLSPwQ4RVXv79u1DSkoK3nnnHQBAhw4dYGZmhlmzZiEzMxMZGRn45JNPIJfLERcXp7V6N2/eDAsLCwwdOlRrZVLxypqjFR8fjzFjxsDJyQlmZmZo1aoV/vjjjzLLXbVqFTw9PWFsbIz27dvj0qVLKvfff/99GBoavpJ1165dGyYmJrC3t8egQYNw586dMsutqqp0sCOXyzF//nx4eXnBxMQEtWvXxqJFiyCEUKYRQuDzzz+Hs7MzTExM4OPjg/Dw8EpsNRG9bL/88gv69u0LFxcXAAVDXLt27cLBgwdhbm4OKysrpKSkoFWrVtDT096vvQ0bNmD06NEwNjbWWplUvNLmaAHA2LFjERYWhgMHDuD69esYOnQoRowYgdDQ0BLL3LlzJ2bMmIEFCxYgJCQEzZs3R+/evZGYmKhM07p1a/zxxx+4dOkSjh49il69eimHUHW97o0bN+L27ds4evQohBDw9fWFXC4vsdwqTVRhX3/9tbC1tRWHDh0SUVFRYteuXcLc3Fz88MMPyjRLliwRVlZWYt++feLq1ati4MCBwsvLS2RlZaldj0wmEwCETCariMcgogp0//59oaenJ/bt21fs/cePH4unT58KIYRwdHQUS5cuVatcAGLv3r3F3vPw8BAAirw++OADERUVVew9AOL3338vtrzc3Fzx6aefiiZNmghTU1Ph7OwsxowZI2JiYpRpTp06VWK5ly5dUuuZdMnUqVNF7dq1hUKhEEIIYWZmJrZs2aKSxsbGRqxfv77EMtq1ayf8/PyUX8vlcuHi4iL8/f1LzHP16lUBQLi7u7+SdUdERJSYpjKo+/ldpYOd/v37i/Hjx6tcGzp0qBg9erQQQgiFQiGcnJzEt99+q7yfkpIijIyMxPbt29Wuh8EOUfW1YMEC4eTkJPLy8kpNd+LECSGRSMSdO3fUKre0YCcxMVGMGDFCNGvWTMTFxYljx44JAOLUqVMiPz9fxMXFqby+/PJLYW5uLtLS0ootLyUlRfj4+IidO3eKO3fuiMDAQNGuXTvRunVrZZqcnJwi5b733nvCy8tL+cH3qsjJyRG2trbi66+/Vl7r1auX6N+/v3jy5ImQy+Vi+/btwtTUVISHh5dYhr6+fpGf8dixY8XAgQOLzZOeni4+/PBDoaenJxYuXPhK1T1t2jTh5eUlcnJyik1TWXQi2Pn666+Fh4eHCAsLE0IIceXKFeHg4CB+++03IYQQkZGRAoAIDQ1Vyde1a1fx0UcflVhudna2kMlkytfDhw8Z7BBVQ3K5XLi7u4tZs2YVubdhwwYRGBgoIiIixK+//ipsbGzEjBkzVNK89tprYuXKlcqv09LSRGhoqAgNDRUAxHfffSdCQ0PFgwcPVPLJZDJhamoqVq9eLYQo2svwvBYtWhT5w60sly5dEgCK1F1ar1Khf/75R/To0UOYmpoKCwsL0aVLF5GZmVliXfn5+WLevHnC09NTGBsbi1q1aomFCxeqPE98fLwYN26ccHZ2FiYmJqJ3797i7t27Gj2TtuzcuVPo6+ur9Hw9ffpU+Pr6CgDCwMBAWFpaiqNHj5ZYRkxMjAAg/vnnH5XrM2fOFO3atVO5tmrVKmFmZiYACGdn51ey7vr161e5Xh0hdCTYkcvlYtasWUIikQgDAwMhkUjE4sWLlfcDAgIEABEbG6uSb/jw4WLEiBEllrtgwYJif1kw2CGqXo4ePSoAKP8getasWbOEo6OjMDQ0FHXr1hXLly8vEoy4urmL8R/OFLEpBYFASUNF48aNU8m3du1aYWJiIlJSUortZXhWcHCwACACAgI0erZjx44JiURS5PdSYmKiiIuLE+vXrxcSiURs27ZN2askREGgY2lpKfz9/cWNGzfEnTt3xM6dO0V2dnaJdZU1ZUChUIgOHTqILl26iEuXLok7d+6ISZMmCXd3d5Genq7Rc2mDr6+veP3111WuTZkyRbRr104cP35cXLlyRXzxxRfCyspKXLt2rdgyNPnQT0lJEXfv3hVnzpwR9vb2wtLSUmWqxKtQ94ABA0SrVq00miLyMuhEsLN9+3bh6uoqtm/fLq5duya2bNkibGxsxKZNm4QQLx7ssGeHiHZceiC8Zh8SHrMOCa/Zh8SOSw/KzlSM4noZnvW///1PNGzYUKMys7KyRKtWrcRbb71VYpq+ffuKvn37FulVat++vZg3b55G9ZU1ZSAsLEwAEDdu3FDel8vlwt7evtS5IRWhuDlaERERRdonhBA9e/YU77//frHlvMhwTmHdUqlUbNu27ZWpuzCfqampsu6qQt1gp0qvxpo5cyZmz56NN998E02bNsWYMWMwffp0+Pv7A/hvc6/nt4BPSEgosvHXs4yMjGBpaanyIqJXR5wsC3P2XIfi34WdCgHM3XMDcbIsjct6fiXYs7KysrBt2zaN9uHJy8vDiBEjIITA6tWri03z6NEjHD16FGPHjlXZXygxMREXL16Eg4MDOnbsCEdHR3Tr1q3MvYU6duyIEydO4O7duwCAq1ev4vz588oNFnNycgBAZdWZnp4ejIyMtLpvkTo2btwIBwcH9O/fX3ktMzNT2aZn6evrl7hjtlQqRevWrXHixAnlNYVCgRMnTsDb27vUuvX09JTfk1ehbqBg5bMQQll3tfMyIq8XZWNjI37++WeVa4sXLxZ169YVQvw3QXnZsmXK+zKZjBOUiahUARGPhcesQ0Ve58Mfa1ROWSvBtmzZIgwNDUViYqJa5eXm5orBgweLZs2aiaSkpBLTLVy4UNjb24utW7eq9CoFBgYKAMLGxkZs2LBBhISEiGnTpgmpVFrq/Jqypgzk5uYKd3d3MXz4cJGcnCxycnLEkiVLBADh6+ur1rNpQ0lztHJzc0WdOnVEly5dxMWLF0VERIRYtmyZkEgk4vDhw8p0z8/R2rFjhzAyMhKbNm0St27dEpMmTRLW1tYiPj5eCFEwL3Tx4sUiODhYREVFCUdHR1GnTh1hY2MjEhISXmrd9g4Ooqa7l6hR4+XW/eDBAxEQECAGDBig8txVhU4MY40bN07UrFlTOY68Z88eYWdnJz799FNlmiVLlghra2uxf/9+ce3aNTFo0CAuPSeiUsWmZArPYoKdEWv+ESkZuWqXU9ZKsG7duolhw4apVVZhoNO4ceNSgyOFQiG8vLzExx9/XGTuSuHQ/pw5c1TyNG3aVMyePbvEMsuaMiBEwdyj5s2bCwBCX19f9O7dW/Tt21f06dNHrefThtLmaN29e1cMHTpUODg4CFNTU9GsWbMiS7I9PDzEjFlzRUDEY+U8rZUrVwp3d3chlUpFu3btxIULF5TpY2JiRN++fYWDg4PQ19cXAMTrr79eZEVfRdetp1dQt3HttqLmxDUqQ64VXbehoaFwdXUVb731ltorGV8mdT+/JUI8s0NfFZOWlob58+dj7969SExMhIuLC0aNGoXPP/8cUqkUQEHX2oIFC7Bu3TqkpKSgc+fO+Pnnn1GvXj2160lNTYWVlRVkMhmHtIheAUIIdPA/gYTUgi55PQmgrydBnlzA09YU68e2QV1Hi1LLUCgU8PLywqhRo7BkyZIi9yMiIlCvXj38+eef6NOnT5H7DRo0gL+/P4YMGYK8vDy88cYbCAkJwaFDh+Do6KhMZ2Njo/x9BwAnTpyAj48Pjh8/Dl9fX+zZsweDBg0CAERFRaFWrVr49ddf8fbbbyvzjBw5EgYGBti6dWuxz+Lm5obZs2crd+UFgK+++gq//fZbkV1zZTIZcnNzYW9vj/bt26NNmzalHpxalewMilYOX+pJAP+hTTGyrXul1L14SFO83twFSWk5SEoveD1Oz0VSWg4ep+cor8enZiM2JbtIea7WxnCyMoGduRHsLKSwMzeCvYVRwdfmRrD/97qp1KBSn7uiqfv5XaUPArWwsMCKFSuwYsWKEtNIJBIsXLgQCxcufHkNIyIABVv4P3jwoMj1Dz74AKtWrUL37t1x5swZlXvvv/8+1qxZU2KZhX/ArF+/HikpKejUqRNWr16NunXrKtMkJyfjww8/xMGDB6Gnp4dhw4bhhx9+gLm5uVrtPh32GAmpOTAx1MPKUa3QuKYlkjNyMWnLZdx/konBqwKw4s2W6NXIscQyjh8/jujoaIwfP77Y+xs2bICrqyt8fX2LvR8WFgaZTAYAiImJwYEDBwAALVq0UEl36tQpdO/eXfn1L7/8go4dO+LcuXNF5q54enrCxcUFYWFhKmXcvXu31ANOMzMz1Z73YWVlBQAIDw9HcHAwFi1aVGK5VUlJ87S61rOHs5VJhdb9MDkDs/dch3im7tl7rmP2nusvXOajlGw8KiYIep6xoR6y8/77Ob7M565KqnSwQ0RVW1BQkMr28Tdu3ECvXr2UW9kDwMSJE1X+GDE1NS21zKVLl+LHH3/E5s2b4eXlhfnz56N37964deuWcoLs6NGjlUcG5OXl4d1338WkSZOwbds2tdq95kxkQTntPeDzb0DjbGWCA1M6wW9bCC7cS8bELcGY0asepvSoAz09SZEyfH19UVrH+OLFi7F48eIS7wshECfLwj+RSfCycyy1rGdt27ZN2as0btw4GBj892tcIpFg5syZWLBgAZo3b44WLVpg8+bNuHPnDnbv3q1M17NnTwwZMgRTpkwBAAwYMABff/013N3d0bhxY4SGhuK7775TCeR27doFe3t7uLu74/r165g6dSoGDx5cYjBX1UQlZSgDnUJyIbDyRDg+7FlX6x/8QgjciEnF3tAY7L78CCX9eE2l+v/2xkj/7aUp7Jkp+FoiAT7YGqLSdj0J8NOolhCQKHuFktJz8DgtV+Xr7DyFSqDz7HPfT8pksENEpI7Cc3oKLVmyBLVr10a3bt2U10xNTUtdHfksIQRWrFiBefPmKYdmtmzZAkdHR+zbtw9vvvkmbt++jSNHjiAoKAht2rQBAKxcuRL9+vXDsmXLil0V9azQ6Ke4GJUMAz0JJnTxUrlna26EXye0x9eHb2PTP/fx3bG7uB2XimXDm8PMSLu/LssztFBar9K0adOQnZ2N6dOnIzk5Gc2bN8exY8dQu3ZtZZrIyEgkJSUpv165ciXmz5+PDz74QDll4P3338fnn3+uTBMXF4cZM2YgISEBzs7OGDt2LObPn1+O78DLdTc+rdjr2y49xPagh/CuZYvBLWuibxMnWBgbvnA9D5MzceBqLPaGxiAiMb3EdHoS4Nj0bqjtUHZvpP/Qppi75wbkQkBfIsHioU3Qr1np73MhBDJy5bgVK8PIdRdUgi09CeBpV/ofHbqmSs/ZeVk4Z4eo/HJzc+Hi4oIZM2Zg7ty5AIDu3bvj5s2bEELAyckJAwYMwPz580vs3bl37x5q166N0NBQleGcbt26oUWLFvjhhx+wYcMGfPzxx3j69Knyfn5+PoyNjbFr1y4MGTKk1HZO/vUyjtyMx7BWrlg+onmJ6XYGRWPevhvIkws0cLLAujFt4G5b/g8IuULgz+ux+HD7FZXr+hIJzs/u8dL+2o6TZSEqKQNedmY6/xf+vcfpGLDyPDJy5ZCgYKdIPQkwuGVNPErOwqX7ycq0RgZ66NXIEUNa1kTXevYw1C97hxZZZh4OX4/DvtCYEstKSM3G/H03VQIWTebNxMmycD8pE552phr/vHYGRWPunuuQ//tpb2MmxemZ3WFZjqCuqtCJOTtEVH3s27cPKSkpeOedd5TX3nrrLXh4eMDFxQXXrl3DrFmzEBYWhj179hRbRnx8PACoTNAt/LrwXnx8PBwcHFTuGxgYwMbGRpmmJJGP03H0VkGa97vVKjXtyLbuqONggcm/Xcad+DQMXHUeq95qhU517ErNV5LbcanYFxqD/VdiEZ9adK7Fyxxa+OXcPXx1+LbyQ1+XJqw+LztPjinbQpGRK0eHWjZYNrw5HiZnqQQNhb0xe0IeIfJxBg5di8Oha3GwMZNiQDNnDG5ZEy3crBGfmq0MEG3MpDh15zH2hcbg5J1E5MoLhoskEsC7li2GtKyJPs/1EvVo4PDCAYuzlckLvzdGtnVH13r2uBmbinl7byA+NRtz9lzHT6NaQiIpOkSrixjsEJFWFLe53qRJk5T/btq0KZydndGzZ09ERkaqDKu8LP937h6EAHo2cEC9MlZbAUBrjxo4OKUz3v81GFcfyTB2wyXM7dcQ4zt5qvUhESfLwoErBUMad54ZRjE30kd6jrxI+ofJmfCubavZQ2lACIH15+5h8Z//rbDSZMJqWRPS33//fRw/fhyxsbEwNzdHx44d8c0336BBgwZqtW/y5MlYu3Ytvv/+e0ybNk15/e7du5g5cyYCAgKQm5uLZs2aYdGiRejRo0eZZfr/eRu34lJhYybFD2+2hKOlMVxrqPbQudmYwq9HHXzQvTZuxqZiT0gMDlyNRVJ6DjYHPsDmwAewNZMiOSMXhUMhJob6yMr772fYwMkCQ1rWxMAWLiV+H8sTsJRXYd02ZlKMWBOIw9fi0Km2Hd5qr5tB7vMY7BBRuT148ADHjx8vscemUPv27QEULMsuLth5dld0Z2dn5fWEhATlsJaTkxMSExNV8uXn5yM5ObnUuUGJadn443IMAGByd/UDLScrY+x83xuf7b2BP0IeYdGhW7gVm4qvhzSBsaF+kfRp2Xn460Y89oXGIPDeE+VcCam+Hl5r4IDBLWuiRwN77AuNUc7DKPTpH9dwI1aGuf0aFlt2edx7nI7P9t5A4L0nRe6p26tU1oT01q1bY/To0XB3d0dycjK++OIL+Pr6IioqCvr6pT/P3r17ceHChWLnXL3++uuoW7cuTp48CRMTE6xYsQKvv/46IiMjS/2ZH7kRh82BBcHZ8hHN4WhpXGJaoGCCd5OaVmhS0wpz+zVAQOQT7AuNwV/X4/AkI1clbVaeHHbmUgxr5YrBLWuioXP1mALRyr0GZvauD/+/7uDLgzfRysMaDZyqR9vLg8EOEZVbcVv4F+fKlSsAoBLIPMvLywtOTk44ceKEMrhJTU3FxYsX8b///Q8A4O3tjZSUFFy+fBmtW7cGAJw8eRIKhUIZTBXbxoD7yJUr0MrdGm08amj0fMaG+lg2vBkauVhi8Z+38UfII0Q8TseigY2RnpsP1xomCE9Ix97QGBy7lYCc/P9WwLTztMHgljXRv6kzrEz/G9IoHFq4n5QJF2tj/Br4AP93PgpbAh/g4r1k/DiqJeo7ld37VJbcfAXWnonEylMRyM1XwMhAgtx8gecna157mFJmr1JZE9Kf7cnz9PTEV199hebNm+P+/ful9uTFxMTgww8/xNGjR4u8h5KSkhAeHo5ffvkFzZo1U9b7888/48aNGyUGOw+TM/Hp7msAgPe71kKP+g7FpiuJgb4eutWzR7d69hjQ3BnjNwUXSfPDmy1feFizMk3sUguB957gdNhj+G0NwcEPO8NUqtvhQJU+G4uIqj6FQoGNGzcWWQYdGRmJRYsW4fLly7h//z4OHDiAsWPHomvXrsoPLaBgc729e/cCKPjLetq0afjqq69w4MABXL9+HWPHjoWLiwsGDx4MAGjYsCH69OmDiRMn4tKlSwgICMCUKVPw5ptvlrgSKy07D79dKPgLf3K32i80T0EikWBCZy9sGd8O1qaGuPowBQNXBeCt9RfRdelpTNgcjEPX4pCTr0BtezPM7F0f5z7tgd8ne+Ot9u4qgU4hZysTeNe2hYetGea93gibx7eDnbkRwhLSMOCn89gSeF/tJenFCb6fjP4/nsPyY3eRm69Al7p2ODa9O5YMawr9f78Hhd8J/yN3sPJEuNr15ebmqpzL9byMjAxs3LgRXl5ecHNzK7EchUKBMWPGYObMmWjcuHGR+7a2tqhfvz62bNmCjIwM5OfnY+3atXBwcFAGu8/Lkyvw0Y5QpGbno6W7NT7pXV+tZypJQ2dLPL/7gL5Eglr2ZuUqt7Lo6UmwfHhzOFoaIfJxBj7ff7PEtJ6eBUO2z7/8/PyU+13Vr18fJiYmcHd3x0cffaTcP6okxZUnkUjw7bffKtMMHDgQ7u7uMDY2hrOzM8aMGYPY2NgXf+iK2cC5euFxEUQvrqQt/KOjo0XXrl2FjY2NMDIyEnXq1BEzZ84s8v8MgNi4caPya4VCIebPny8cHR2FkZGR6NmzZ5Gynzx5IkaNGiXMzc2FpaWlePfdd0VaWlqJbVx7JkJ4zDokXlt2SsjlinI/c1DUk2LP1vp011Vx7WGK8gTyF/E4LVuM23BRWeaETZfEk/QcjcpIycwVc/ZcU5bRauHfYl/oI5V2xaZkin8ikkTM0wzx7ZE7yrSf7b0m8tX4HpV02vuqVauEmZmZACDq168vIiIiSi1n8eLFolevXsq2eXh4iO+//14lzcOHD0Xr1q2FRCIR+vr6wtnZWYSEhJRc5p+3hMesQ6LpgiMi+klGmc+ijh2XHohasw8Lj1mHRK3Zh1WObKiuAiOThNfsgp/7H5cfFpsmMTFRxMXFKV/Hjh0TAMSpU6fE9evXxdChQ8WBAwdERESEOHHihKhbt26ZR6Q8W15cXJzYsGGDkEgkIjIyUpnmu+++E4GBgeL+/fsiICBAeHt7C29v7yJl6cRxES8Ll54TVZ6KXgKdm69Al6UnkZCag6XDmmFE25J7GdT1T2QS3lp/scj17RM7aGWCsRACm/65D/8/7yBXroCDhRG+G9ECneuWPmQihMCf1+PxxcGbeJxWcBTGyDZumNOvAaxNpaXm3RJ4HwsO3IQQQJ/GTljxZotS5w317t0bUqkUBw8eVLkuk8mQmJiIuLg4LFu2DDExMQgICFA5Mb3Q5cuX0b9/f4SEhCh75Tw9PTFt2jTlBGUhBAYPHoy8vDx89tlnMDExwf/93//hwIEDCAoKKjIkeiosEe9uDAIArHm7Ffo0KX7I9EWUZ/l3VfXD8XB8f/wuTKX6OPhhZ9S2L33fn2nTpuHQoUMIDw8vtkdv165dePvtt5GRkaHS01uawYMHIy0tTeUk9ucdOHAAgwcPRk5ODgwN/+slVffzm8NYRFRpdgZFo9OSk3hr/UV0WnISO4OitV7HvisxSEjNgaOlEQa1LH0jNnV52ZkVO6yhrY3aJBIJ3u3khX1+nVDHwRyJaTl4+5eL8P/zNnLzi+6ICwCPnmZiwuZg+G0LweO0HNSyN8OOSR3wzRvNygx0AGCstydWvdUKUn09HLkZj7EbLkGWlVds2sIJ6e+9916Re1ZWVqhbty66du2K3bt3486dO8phyuedO3cOiYmJcHd3h4GBAQwMDPDgwQN8/PHH8PT0BFAwH+vQoUPYsWMHOnXqhFatWuHnn3+GiYkJNm/erFJevCwbH/9+FQAwzttDq4EO8N+wo64EOgAw5bU68K5li8zcgiX62XlFVwkWKmvoEoAy6FA30ElISMDhw4cxYcKEEtMkJydj69at6Nixo0qgowkGO0RUKeJkWZj9x/NnFV1HnCxLa3UoFALrzt4DAIzv5AUjA+2scHK2MoH/0P/mvRRuEqftD8FGLpY4OKWzcnnw2rP3MGz1P4hKylAeNfEwOQP/d+4efL8/i5N3EmGoL8HUnnXx19Qu6FBLs16mfk2dsXl8O1gYGeBSVDJGrAlEvKzonkDqTkgXQkAIgZycnGLvjxkzBteuXcOVK1eULxcXF8ycORNHjx4FUHBuF4AiZ3fp6empnN0lVwhM3RGK5IxcNHK2xJx+DTV69leVvp4EK95sAVszKW7HpeLrw7dLTFvcXlrPSkpKwqJFi1Qmqpdl8+bNsLCwwNChQ4vcmzVrFszMzGBra4vo6Gjs379f7XKfx2EscBiL6GWTKwSm77yCA1eLTjjU1lAQABy7lYCJW4JhYWSAgDmvaX3H2Jc5rHHkRjxm77mGlMw8SPULTmh//pd3O08bLB7aBHUcyreK61ZsKt7ZeAmJaTlwsTLGlgntlGWWdNr7vXv3sHPnTvj6+sLe3h6PHj3CkiVLEBAQgNu3bys3gnz2tPfiPD+MlZSUhAYNGqBbt274/PPPYWJigvXr1+OHH35AUFAQmjcv2AX7+2N38cOJcJj9OxxTq4zhGFJ1OiwR7/w7/Ld6dCv0bVq0V6ykoUug4HO0V69esLGxwYEDB9TugWnQoAF69eqFlStXFrmXlJSE5ORkPHjwAF9++SWsrKxw6NAhlV4lDmPRK6W0FQMAsG7dOnTv3h2WlpaQSCRISUkps0x/f3+0bdsWFhYWcHBwwODBg4ucJv0i5WpbdXt2WVYeJmwOKjbQAYCrD9UrRx1r/z3w860O7hWyNf7LHNbo08QJf03tgpbu1sgtJtCZ06cBdkzqUO5AByjoUfrjfx1Ry94MsbJsvLEmEJcfFBzPUdK5XMbGxjh37hz69euHOnXqYOTIkbCwsMA///yjsuP1s6e9q8POzg5HjhxBeno6XnvtNbRp0wbnz5/H/v37lYHOP5FJ+PFkOADg6yFNGei8gO71HTC5W8H2AJ/+cQ0PkzNV7pc2dJmWloY+ffrAwsICe/fuVTvQOXfuHMLCwootEyj42derVw+9evXCjh078Oeff+LChQsaPlkBBjukE4KCghAXF6d8HTt2DACUm51lZmaiT58+yjOb1HHmzBn4+fnhwoULytO1fX19kZGRoUzzIuVqW3V69ojEdAxZFYDTYY9hbKiHN9u6FVkC/e3fYTgfnlRyIWoKvp+M4AdPIdXXw/hOXmVnqAacrUzwca/il1E3c7Mu9nT2F+VmY4rdkzuihZs1UjLzMPr/LuDE7QTlae/16tVTSe/i4oI///wTCQkJyM3NxcOHD7F161bUr6/aXiEEeg8ZiX8ik4odsrx//77K7skA0KZNGxw9ehRPnjxBamoqAgMD0bdvXwBAUnoOpu24AiGAEW0KNvijF/Oxbz20crdGWnY+pmwPVZkfVtLQZWpqKnx9fSGVSnHgwIFiJ6KX5JdffkHr1q2VQWtpCocsSxoSLQuHscBhLF1U0oqB06dPo0ePHnj69Cmsra01KvPx48dwcHDAmTNn0LVrV5V75SlX26rqs5+4nYBpO64gLScfNa1NsHZMazSpaaUcCvKwNcE3R8Kw/0osLIwNsOd/HVFXjSMdSvLe5mAcv52AkW3c8M0bzcrOUE3EybLQaclJ5VwnoGIPEc3MzYff1hCcCnsMfT0J/Ic0LdeKtvKc9v48hULg3U1BOHP3Meo4mOPAlE46vzleRXv0NBP9fjiH1Ox8TOpaC3P7NSxx6LIw0MnMzMTevXthZvbfvkP29vbKXbOLG7pMTU2Fs7Mzli9fjsmTJ6u04eLFiwgKCkLnzp1Ro0YNREZGYv78+UhISMDNmzdhZGSkUg4PAqVXUuGKgRkzZmj1kLvCrncbGxutlaltVfHZhRD4+XQklv0dBiGAdl42+Hl0K9iZF/zCeva8oG+GNUPM0ywEP3iKdzcFYZ9fJ2U6TUQkpuH47QRIJMCkMg78rG4KJ0cXHjVRUZOjC5lKDbBubBvM2XMduy8/wqd/XEPk4zR0q+cAL3vV7QLy5QokZ+TicXoOktJzkZSWg6T0wlcuYp5m4tL9/06rVwhgzp7r6FTHrsh5VepYd+4eztx9DCMDPax6qxUDHS1wrWGKb4c3x/u/Xsa6s/fQoZYN8qOvFjt0GRISgosXC7ZgqFOnjsq9qKgo5Yq64oYud+zYASEERo0aVaQNpqam2LNnDxYsWICMjAw4OzujT58+mDdvnkqgown27IA9O7rm999/x1tvvYXo6OgiO+q+aO+GQqHAwIEDkZKSgvPnzxe5X1V6dqras2fm5mPmrms4fD0OADCmgwc+H9AIhvolj6AnZ+RiyM8BePAkEy3drbF9YgeNz4mauesqdl1+BN9Gjlg3to1GeauLl73nixACy/4Ow6pTkSrX6ziYQV+ih8fpOXiamYsX+UQxk+qjW317dK5jjy517eBmU3bgc/nBU4xYGwi5QsB/aFOMavdqHGj5snxx4CY2/XMfNUwN8efULuV6j1XkXlrs2aFXVnGnb5eXn58fbty4UeyHfVVSlZ79YXImJm4Jxp34NBjqS7BwUBO1PpBszKTY8E5bDP35H4RGp+DjXVex8s2Was9HiZdlY98VzQ/8rG5e9gnaEokEb3fwwM+nIlUmR0ckZqik05MANmZGsDOXwt7CCHbmBf+2MzeCgZ4EX/15u0hAlJErx5/X4/Hn9XgAgKetKTrXtUPnOvbwrm0LK5P/JrzGybJw41Eq5u+/AblCYEBzF7yphY0iSdWcfg0Q/CAZN2JSMXXHFSwf3gwPn2ZpHLDsuBSNuXu1M2xZHgx2SKeoe/q2JqZMmYJDhw7h7NmzcHV11Vq52laVnv2fyCT4bQ3B08w82JkbYc3brdDGU/0hsNr25ljzdmuM+eUiDl+Lg5etmdrnG20IiEKeXKCdpw1auWt24CeVLiopo8gqMACY268hutazg525EWqYSqFfSmBqbmygMgS3aHBj1HeyxPnwJJyPeIyQ6BTcf5KJ+0+i8duFaOhJgOZu1uhS1x65+XKsO3tPOV/JxkyKxUOaaHXIlgoYGehj5ahWeP3Hc7gUlYyuS09DoCBg+XpIU/g2ciwYqvx3mPJxWo7K10npOUiQ5eBx+n8Tigv20rqBrvXsX/rGjAx2SKeou9mZOoQQ+PDDD7F3716cPn0aXl5Ve0VPVXh2IQQ2/3Mfiw7fhlwh0MzVCmvHtH6hX2zetW3hP7QpZu6+hp9ORcDD1hTD25T+F7wsKw/bLhbswjy5u27N1akKCneOfn5y9IDmzmr/jJ897f3ZIbjWHjUw1acu0rLzcOFeMs6HP8a5iCTce5yB0OgUhEanFCkrJTMX6Tn5sKiAbQWo4Of9aZ/6WHDgljLILZxnNWfP9RcqUy4E7idlMtghelElnb4NAPHx8YiPj0dERAQA4Pr167CwsIC7u7ty0m3Pnj0xZMgQTJkyBUDB8M22bduwf/9+WFhYID6+oIvdysoKJiYmapf7MlSFZ//1z7M4fz8d/yToQd/EAkNa1oT/0KYaz7d51vA2brj/JAOrTkVi7t7rcK1hWuqGg1svPkB6Tj7qOZqjez2HEtPRi9HW5OjShuAsjA3Rq5EjejVyBADEpGQhIDwJe6/EIDDyiUpahUClfHC+SkpbEVnD1PDfYUoj2Fn8N1xpb24EO4uCI0re2xxcJDjW1rEqGin1mNBXBE891w0lnb4thBALFiwQAIq8nj1t28PDQyxYsED5dXHpn8+jTrkvQ1V6dtt+08T6s5HlOvn7WXK5Qnyw9bLwmHVINPviqIhILP5086zcfNHmq2PCY9YhsTu4+BOcSTsKT0yPTcl8qXUWntBd+Ko1+/BLbcOrqLjvu9fsQ+LBk3S18lf0afE89VwDXI1FhSr6BO6qqqTnlisEnmTkIClNdSy+cFnxo+eWEgMFY/oBs1/T6vcvO0+OUesvIDQ6BR62ptj7QSfYmKkebrnjUjRm77kOZytjnJnZA1ID7pmqa3YGRRfpVaqMya6vmvJ+3yty5aC6n98MdsBghwrsDCr4sBSVvGrgRakbqOUV7oXy7x4oh6/HYXfwI+WYfB0HcxjoSZCUnoMnGS+2lFib51sVSkrPweBVAXj0NAttPWvgt/faKw/2lCsEen13BveSMjCvf0O814XzdXTVy15yTwWq6vedS8+JNBAny8KcfwMdoGAuwOw/rqOxixWa1LSq3Map4dldaSUSYFhLV3jZm6n0whT2yjzNzCu1rIjEdJWvJRLAxlT677i89L8xenMjGOgBi/+6oxIQVdSYvJ25ETa+0xZDV/+DoPtP8enua1gxsgUkEgmO3UrAvaQMWBob4E3ut6LTXvaSeypQ3b/vDHaIULCkVvFcD4YAMHhVAIa2qon3utRCvXIcXVCR4mRZyh4pABAC2B3yqNQ8ehLA1twIJob6iH7uwD8AmNO3AbrUtYedhRQ2plIYlLIJoKWJ4UvbzbeuowVWj26NcRsvYf+VWHjammGaT12s+ffAz7HenjA34q81IlLF3wpEKOi5KE6+QuD34Ef4PfgRute3x8QutdCxtm2V2tcjLD6t2KGmrnXt0NDFsmBlhHLFhBT2/+6FoqcnKfGcpYEtXMq9lLiidK5rh68GN8GcPdfxw4lwxDzNxJWHKTDUl2BcR88KrZuIqicGO0QA/r6VoPJ1YQ9FHQcL/N+5ezh6Mx6nwx7jdNhjNHK2xHtdvPB6M5cqMQn29J3HRa7pSyT45o1mZQYeL2MpcUUY1c4d95MysPbsPewOKdgtOU8ucPJOQrWaZ0VELwcnKIMTlF91WblydPrmJJIzcvHloMao52BRpIfiwZMMbDgfhd+DHyErTw4AcLI0xjudPDGqnbvKdvYvU0RiGvr+cA55cqHc7K2qrZaoKDFPM9Hpm1Mq1yry9G8iqno4QZlITbsvP0RyRi7cbEwwup17sfNTPGzN8OWgJpjeqx62XozG5n/uIz41G0v+uoOVJ8Ixoq0bxnfygoG+5KUtXRdC4LO9N5AnF+jZwAGLBjfGgydZLxSwVMfJhw+KmWtUWbuzElHVxmCHXmn5cgXWnbsHAJjYpVapE3EBwNpUCr8edfBeFy8cuBKL/zsXhbCENGwMuI9NAfeVy7dfxtL1PSExuBiVDGNDPXwxsDFcrE3hYl0JO5NWkpKOLqiU3VmJqEqr/AkHRJXorxvxeJicBRszKYa3Vv/kZCMDfQxv44Yj07pgy/h2aOdZQ+WAxMID7+JkWdpvNArOBPr6z9sAgKk968HN5tX7gC+cb6T/72Txil4JRkTVF3t26JUlhFAuWR7n7QkTqeZnOEkkEnStZw8DfQneWn9R5V5FDql8c+QOkjNyUc/RHO91qdoHlFakl70SjIiqJwY79MoKiHiCm7GpMDHUx1hvj3KVVdyQigSokCGVyw+Ssf3SQwDAV4ObwrCMoTddVx3nGxHRy/Vq/5akV9raswW9OiPbuqGGWfH77Kjr+SEVoGBTwjvxaeUq93l5cgU+23sDADCijSvaeb28k9WJiKorBjv0SroRI8O58CTo60kwobN2hoFGtnXH+dk9sH1iBwxp4QIAmLo9FPeTMrRSPgBsDIjCnfg01DA1xOy+DbVWLhGRLmOwQ6+ktWcLVmANaOas1cm9zlYm8K5ti2/eaI5W7tZIzc7H5N8uIzM3v9xlx6Rk4ftj4QCAOf0aFjn1m4iIisdgh1450U8ycfhaLABgUtfaFVKH1EAPq99uDXsLI9yJT8Onu6+hvPt3fnHgJrLy5GjnaYM3WrlqqaVERLqPwQ69cv7v/D0oBNCtnj0auWhnx+yYmBi8/fbbsLW1hYmJCZo2bYqHd2/g59GtYKAnwaFrcfi/c1EAgMmTJ0MikWDFihVllrtq1Sp4enpCamSMLbNHIz/+Lr4a0gR6ehIkJyfjww8/RP369WFiYgJ3d3d89NFHkMlkWnkmIiJdwWCHXilP0nPwe3DBSqb3u9XSSplPnz5Fp06dYGhoiL/++gu3bt3C8uXLUaNGDbT1tMHnAxoBAPz/ug3/nzfjwoULcHFxKbPcnTt3YsaMGZg9dx4a/e9nSB288OSPL2AtKdi7JzY2FrGxsVi2bBlu3LiBTZs24ciRI5gwYYJWnouISFdw6Tm9UjYHPkB2ngLNXa3gXctWK2V+8803cHNzw8aNG5XXvLz+m/Q8poMHrj2SYcfpK1gw5xP89ddfmPDWG2WW+91332HixIl46toJKffuoemIT3B/1Ths2LABs2fPRpMmTfDHH38o09euXRtff/013n77beTn58PAgP+9iYgA9uzQKyQzNx9bAu8DAN7vVhuSZ5aJl8eBAwfQpk0bDB8+HA4ODmjZsiXWr1+vvC+RSLBwYCNkH/sR5m2H4PvLWShr9k5ubi4uX76MBq074v/OFwx/LRrSFL18fBAYGFhivsLD8BjoEBH9h8EOvTJ2Bj1ESmYePG1N0buxk9bKvXfvHlavXo26devi6NGj+N///oePPvoImzdvVqb54btlaOxaAx5d38CNmFSkZOaWOmE5KSkJcrkcf9xKh1wh0LuxI3o2dISjoyPi4+NLzLNo0SJMmjRJa89GRKQL+OcfvRLy5ArlBOGJXWtBX087vToAoFAo0KZNGyxevBgA0LJlS9y4cQNr1qzBuHHjcPnyZfzwww8ICQnB/Uwp3v7lIjJy5Ah+8LTMsu/Ep6KGlzsWDGhcarrU1FT0798fjRo1whdffKGNxyIi0hns2aFXwuFrcYhJyYKduRTDtLxs29nZGY0aNVK51rBhQ0RHRwMAzp07h8TERLi7u6NrAyc8+HYQ5KmJ2Pbj13BxLf5UdImxBaCnB3lGCmb0qgcX64LjEBISEuDkpNorlZaWhj59+sDCwgJ79+6FoaGhVp+PiKi6Y88O6bxnD/x8t5MXjA01P/CzNJ06dUJYWJjKtbt378LDo+C8rTFjxsDHx0elPd7dekK/XjfYduiHhNRsOFoaq+RffuIepI51YJJ0G+909ARQ0IN04sQJTJkyRZkuNTUVvXv3hpGREQ4cOABjY9VyiIiIwQ69As7cfYw78Wkwk+rj7fblO/CzONOnT0fHjh2xePFijBgxApcuXcK6deuwbt06AICtrS1sbVVXftlamsLE2Qlpxg7432+XsWOSN/r27oUhQ4agTb9R2H35EazaDUb8kR+w9bdf0a5dO6xYsQIZGRl49913ARQEOr6+vsjMzMRvv/2G1NRUpKamAgDs7e2hr6/doI6IqLpisEM6r7BXZ1Q7d1iZan+Ip23btti7dy/mzJmDhQsXwsvLCytWrMDo0aNLzCMBMKKNG/blGSAkOgVfHryJyMhIJCQ+xrx9BQd9vjfubbj4uOHzzz9HfHw8WrRogSNHjsDR0REAEBISgosXLwIA6tSpo1J+VFQUPD09tf6sRETVkUSUdw97HZCamgorKyvlsl3SHVcepmDwqgAY6ElwblYPOFuZVHaTVJwKS8T4TUEQApjbtwEiH2dgZ/BD2JpJcfLj7hUSnBER6Qp1P785QZl02tp/e3UGtahZ5QIdAOhR3wEzfOoBABb/dQc7/93d2aeRAwMdIiItYbBDOisqKQNHbhbsSaOtoyEqwtBWNYtc2x38CHGyrEpoDRGR7mGwQzpr/bl7EALo2cAB9RwtKrs5JXqQnFnkmlwA95OKXiciIs0x2CGdlJiWjd2XHwEoOBqiKvOyM8PzexzqSyTwtDOtnAYREekYBjukkzb/cx+5+Qq0crdGW88ald2cUjlbmcB/aFPo/3tWl75EgsVDm1TJOUZERNURl56TzknPycevgQ8AaPfAz4o0sq07utazx/2kTHjamTLQISLSIgY7pFPiZFlYdTICqdn5qGVvhl4NHSu7SWpztjJhkENEVAEY7JDO2BkUjTl7rkPx785RLd2soafFAz+JiKh60njOzqlTpyqiHUTlEifLUgl0AGBvaAyXbxMRkebBTp8+fVC7dm189dVXePjwYUW0iUhjUUkZKoEOACi4fJuIiPACwU5MTAymTJmC3bt3o1atWujduzd+//135ObmVkT7iNRiUMxwFZdvExER8ALBjp2dHaZPn44rV67g4sWLqFevHj744AO4uLjgo48+wtWrVyuinUQlSsnMxew/rqtc4/JtIiIqVO6DQGNjY7Fu3TosWbIEBgYGyM7Ohre3N9asWYPGjRtrq50VigeBVl+5+QqM3XARF+4lw8XKGGvHtEZ6jpzLt4mIXgEVehBoXl4edu/ejX79+sHDwwNHjx7FTz/9hISEBERERMDDwwPDhw9/4cY/KyYmBm+//TZsbW1hYmKCpk2bIjg4WHlfCIHPP/8czs7OMDExgY+PD8LDw7VSN1VtQgjM2XMdF+4lw9zIABvebYumrtbwrm3LQIeIiJQ0DnY+/PBDODs74/3330e9evUQGhqKwMBAvPfeezAzM4OnpyeWLVuGO3fulLtxT58+RadOnWBoaIi//voLt27dwvLly1Gjxn874i5duhQ//vgj1qxZg4sXL8LMzAy9e/dGdnZ2ueunqu3n05H4I+QR9PUk+OmtlmjgxF45IiIqSuNg59atW1i5ciViY2OxYsUKNGnSpEgaOzs7rSxR/+abb+Dm5oaNGzeiXbt28PLygq+vL2rXLjjrSAiBFStWYN68eRg0aBCaNWuGLVu2IDY2Fvv27St3/aSZL774AhKJROXVoEEDAMD9+/eL3Ct87dq1q9jy8vLyMGvWLDRt2hRmZmZwcXHB2LFjERsbi4NXY/Ht0TAAwCfdamL9wumwtLSEtbU1JkyYgPT09Jf23EREVLVpHOycOHECo0aNgpGRUYlpDAwM0K1bt3I1DAAOHDiANm3aYPjw4XBwcEDLli2xfv165f2oqCjEx8fDx8dHec3Kygrt27dHYGBgieXm5OQgNTVV5UXa0bhxY8TFxSlf58+fBwC4ubmpXI+Li8OXX34Jc3Nz9O3bt9iyMjMzERISgvnz5yMkJAR79uxBWFgYfPr0x8e7CibCj+/khQMrZuPmzZs4duwYDh06hLNnz2LSpEkv7ZmJiKhq03gHZX9/fzg6OmL8+PEq1zds2IDHjx9j1qxZWmvcvXv3sHr1asyYMQNz585FUFAQPvroI0ilUowbNw7x8fEAAEdH1SMBHB0dlfdKeoYvv/xSa+2k/xgYGMDJyanIdX19/SLX9+7dixEjRsDc3LzYsqysrHDs2DGVa/O/XoYBvbqiZud49GnfFENrAQuOHEFQUBDatGkDAFi5ciX69euHZcuWwcXFRUtPRkRE1ZXGPTtr165VDk08q3HjxlizZo1WGlVIoVCgVatWWLx4MVq2bIlJkyZh4sSJ5a5nzpw5kMlkyhc3R9Se8PBwuLi4oFatWhg9ejSio6OLTXf58mVcuXIFEyZMULtsWVYePt99CYAEjTyd8cObLXDp4gVYW1srAx0A8PHxgZ6eHi5evFjexyEiIh2gcbATHx8PZ2fnItft7e0RFxenlUYVcnZ2RqNGjVSuNWzYUPkBWthTkJCQoJImISGh2N6FQkZGRrC0tFR5Ufm1b98emzZtwpEjR7B69WpERUWhS5cuSEtLK5L2l19+QcOGDdGxY0e1ys6TKzB5UyBu7lsN2+Y9sOX97jAzMkB8fDwcHBxU0hoYGMDGxqbU3j0iInp1aBzsuLm5ISAgoMj1gIAArQ8ZdOrUCWFhYSrX7t69Cw8PDwCAl5cXnJyccOLECeX91NRUXLx4Ed7e3lptC5Wtb9++GD58OJo1a4bevXvjzz//REpKCn7//XeVdFlZWdi2bZvavTpCCHz2Ryj2fzcTEglweOcmOFkZV8QjEBGRDtJ4zs7EiRMxbdo05OXl4bXXXgNQMGn5008/xccff6zVxk2fPh0dO3bE4sWLMWLECFy6dAnr1q3DunXrAAASiQTTpk3DV199hbp168LLywvz58+Hi4sLBg8erNW2kOasra1Rr149REREqFzfvXs3MjMzMXbsWLXKWX0yDD/NmwK5LBHb9/+J9vXdlPecnJyQmJiokj4/Px/Jycml9u4REdErRGhIoVCITz/9VBgbGws9PT2hp6cnTE1NxZdffqlpUWo5ePCgaNKkiTAyMhINGjQQ69atK9Ke+fPnC0dHR2FkZCR69uwpwsLCNKpDJpMJAEImk2mz6a+8tLQ0UaNGDfHDDz+oXO/WrZsYNmyYWmUcDHkgTOt2EIZ27mLFgaAi92/duiUAiODgYOW1o0ePColEImJiYsr3AEREVKWp+/n9wsdFpKen4/bt2zAxMUHdunVLXYpe1fG4CO345JNPMGDAAHh4eCA2NhYLFizAlStXcOvWLdjb2wMAIiIiUK9ePfz555/o06dPkTIaNGgAf39/DBkyBMH3HqNH34HIiovA+IVr8OXIjpBICg78tLGxgVQqBVAwfJaQkIA1a9YgLy8P7777Ltq0aYNt27a9vIcnIqKXTt3Pb42HsQqZm5ujbdu2L5qddNCjR48watQoPHnyBPb29ujcuTMuXLigDHSAgi0KXF1d4evrW2wZYWFheBD3GAeuxmLO5hNIv3sBALB++htYP/2/dKdOnUL37t0BAFu3bsWUKVPQs2dP6OnpYdiwYfjxxx8r7DmJiKh6eaGeneDgYPz++++Ijo5Gbm6uyr09e/ZorXEvC3t2qo6dQdGYs+c6FP++K50sjXBsRjdYGBtWbsOIiKjKqbCDQHfs2IGOHTvi9u3b2Lt3L/Ly8nDz5k2cPHkSVlZW5Wo0vdriZFkqgQ4AJKblID0nv/IaRURE1Z7Gwc7ixYvx/fff4+DBg5BKpfjhhx9w584djBgxAu7u7hXRRnpF3IhJVQl0AEAhgPtJmZXTICIi0gkaBzuRkZHo378/AEAqlSIjIwMSiQTTp09XLgkn0lS8LBv+f94ucl1fIoGnnWkltIiIiHSFxsFOjRo1lDvi1qxZEzdu3AAApKSkIDOTf4GT5iIS0zD05wDcS8qAhZEB9AoWXEFfIsHioU3gbGVSuQ0kIqJqTePVWF27dsWxY8fQtGlTDB8+HFOnTsXJkydx7Ngx9OzZsyLaSDrs8oOnmLA5CCmZeahlb4bN77aDgb4E95My4WlnykCHiIjKTePVWMnJycjOzoaLiwsUCgWWLl2Kf/75B3Xr1sW8efNQo0aNimprheFqrMpx/FYCpmwPQXaeAi3crLHhnbawMZNWdrOIiKiaqJB9dvLz83Ho0CH07t0bAKCnp4fZs2eXr6X0StoZFI25e29ArhDoUd8eq0a3gqn0hbd9IiIiKpFGc3YMDAwwefJkZGdnV1R7SMcJIbDyRDhm/XEdcoXAG61dsW5sGwY6RERUYTSeoNyuXTtcuXKlAppCuk6uEPh8/00sP3YXAODXoza+faMZDPU1fhsSERGpTeM/pz/44APMmDEDDx8+ROvWrWFmZqZyv1mzZlprHOmO7Dw5pu+8gr9uxEMiAb4Y0BjjOnpWdrOIiOgVoPEEZT29on+FSyQSCCEgkUggl8u11riXhROUK5YsKw8TtwTjUlQypPp6+H5kC/Rv5lzZzSIiomquwg4CjYqKKlfD6NUSL8vGOxsv4U58GiyMDLB2bGt0rG1X2c0iIqJXiMbBjoeHR0W0g3RInCwLUUkZkAD4ZNc1xKRkwcHCCJvebYdGLuw5IyKil0vjYGfLli2l3h87duwLN4aqv+dPLQeAWnZm2Dy+HdxseOwDERG9fBrP2Xl+08C8vDxkZmZCKpXC1NQUycnJWm3gy8A5O9oRJ8tCpyUnixzm+dfULmjozO8rERFpl7qf3xqv+X369KnKKz09HWFhYejcuTO2b99erkZT9RaVlFEk0AGAlMy8l98YIiKif2llg5O6detiyZIlmDp1qjaKo2rKy85MeYhnIZ5aTkRElU1ru7kZGBggNjZWW8VRNeRsZYL5rzdSfq0nAU8tJyKiSqfxBOUDBw6ofC2EQFxcHH766Sd06tRJaw2j6qmxixUAwM5MioMfdWagQ0RElU7jYGfw4MEqX0skEtjb2+O1117D8uXLtdUuqqbuxKcCAJq7WTPQISKiKkHjYEehUFREO0hH3I5LAwDUd7Ko5JYQEREV4AmMpFVh//bsNOBScyIiqiI0DnaGDRuGb775psj1pUuXYvjw4VppFFVPCoVAWHxBz05D9uwQEVEVoXGwc/bsWfTr16/I9b59++Ls2bNaaRRVT4+eZiEjVw6pvh687MwquzlEREQAXiDYSU9Ph1QqLXLd0NAQqampWmkUVU+3/x3CqutoDgN9jpASEVHVoPEnUtOmTbFz584i13fs2IFGjRoVk4NeFXf+nZzcwInzdYiIqOrQeDXW/PnzMXToUERGRuK1114DAJw4cQLbt2/Hrl27tN5Aqj4Kl503dOZ8HSIiqjo0DnYGDBiAffv2YfHixdi9ezdMTEzQrFkzHD9+HN26dauINlI1cSeey86JiKjq0TjYAYD+/fujf//+2m4LVWNZuXLcf5IBgMNYRERUtWg8ZycoKAgXL14scv3ixYsIDg7WSqOo+rmbkAYhADtzKewtjCq7OUREREoaBzt+fn54+PBhkesxMTHw8/PTSqOo+imcr8NeHSIiqmo0DnZu3bqFVq1aFbnesmVL3Lp1SyuNourntnIlFufrEBFR1aJxsGNkZISEhIQi1+Pi4mBg8EJTgEgH3OExEUREVEVpHOz4+vpizpw5kMlkymspKSmYO3cuevXqpdXGUfUghFCuxGLPDhERVTUad8UsW7YMXbt2hYeHB1q2bAkAuHLlChwdHfHrr79qvYFU9SWk5iAlMw96EqCOg3llN4eIiEiFxsFOzZo1ce3aNWzduhVXr16FiYkJ3n33XYwaNQqGhoYV0Uaq4gqHsGrZm8PYUL+SW0NERKTqhSbZmJmZYdKkSdpuC1VTHMIiIqKq7IVnFN+6dQvR0dHIzc1VuT5w4MByN4qqlztxhcdEcHIyERFVPRoHO/fu3cOQIUNw/fp1SCQSCCEAABKJBAAgl8u120Kq8tizQ0REVZnGq7GmTp0KLy8vJCYmwtTUFDdv3sTZs2fRpk0bnD59ugKaSFVZbr4CEYnpALjsnIiIqiaNe3YCAwNx8uRJ2NnZQU9PD3p6eujcuTP8/f3x0UcfITQ0tCLaSVVU5ON05CsELIwN4GJlXNnNISIiKkLjnh25XA4Li4LhCjs7O8TGxgIAPDw8EBYWpt3WUZX33zERFsqhTCIioqpE456dJk2a4OrVq/Dy8kL79u2xdOlSSKVSrFu3DrVq1aqINlIV9t98HQ5hERFR1aRxsDNv3jxkZGQAABYuXIjXX38dXbp0ga2tLXbu3Kn1BlLVdqfwTCxnTk4mIqKqSeNgp3fv3sp/16lTB3fu3EFycjJq1KjBYYxXEE87JyKiqk4rJ3fa2NhooxiqZpIzcpGQmgMAqM9l50REVEVpPEGZqFBhr467jSnMjXjiPRERVU0MduiFFc7XYa8OERFVZQx26IUV9uw0ZLBDRERVmMbBztmzZ5Gfn1/ken5+Ps6ePauVRlH1EFa47Jw7JxMRURWmcbDTo0cPJCcnF7kuk8nQo0cPrTSKqj65QiAsgWdiERFR1adxsCOEKHaJ+ZMnT2BmZqaVRlHV9+BJBrLzFDA21IOHLX/uRERUdam9hGbo0KEACk43f+edd2BkZKS8J5fLce3aNXTs2FH7LaQqqXDn5PqOFtDX4/5KRERUdakd7FhZWQEo6NmxsLCAiYmJ8p5UKkWHDh0wceJE7beQqqQ7cdxMkIiIqge1g52NGzcCADw9PfHJJ59wyOoVdzuey86JiKh60HjOzqeffqoyZ+fBgwdYsWIF/v77b602jKo25TERPBOLiIiqOI2DnUGDBmHLli0AgJSUFLRr1w7Lly/HoEGDsHr1aq03kKqe9Jx8PEzOAsBhLCIiqvo0DnZCQkLQpUsXAMDu3bvh5OSEBw8eYMuWLfjxxx+13kCqegr313G0NIKNmbSSW0NERFQ6jYOdzMxMWFgUDF38/fffGDp0KPT09NChQwc8ePBA6w2kqocnnRMRUXWicbBTp04d7Nu3Dw8fPsTRo0fh6+sLAEhMTISlJT/8XgWFZ2Jxvg4REVUHGgc7n3/+OT755BN4enqiXbt28Pb2BlDQy9OyZUutN5Cqnv/OxGJwS0REVZ/Gwc4bb7yB6OhoBAcH4+jRo8rrPXv2xPfff6/Vxj1vyZIlkEgkmDZtmvJadnY2/Pz8YGtrC3NzcwwbNgwJCQkV2o5XmRCCp50TEVG18kKnnjs5OcHCwgLHjh1DVlbBqpy2bduiQYMGWm3cs4KCgrB27Vo0a9ZM5fr06dNx8OBB7Nq1C2fOnEFsbKxyt2fSvpiULKTl5MNAT4La9uaV3RwiIqIyaRzsPHnyBD179kS9evXQr18/xMXFAQAmTJiAjz/+WOsNBID09HSMHj0a69evR40aNZTXZTIZfvnlF3z33Xd47bXX0Lp1a2zcuBH//PMPLly4UCFtedUVrsSq42AOqcELxcpEREQvlcafVtOnT4ehoSGio6NhamqqvD5y5EgcOXJEq40r5Ofnh/79+8PHx0fl+uXLl5GXl6dyvUGDBnB3d0dgYGCJ5eXk5CA1NVXlReopPBOLJ50TEVF1ofZxEYX+/vtvHD16FK6urirX69atWyFLz3fs2IGQkBAEBQUVuRcfHw+pVApra2uV646OjoiPjy+xTH9/f3z55Zfabuor4XbhmVjOnJxMRETVg8Y9OxkZGSo9OoWSk5NVTkLXhocPH2Lq1KnYunUrjI2NtVbunDlzIJPJlK+HDx9qrWxdx54dIiKqbjQOdrp06aI8LgIAJBIJFAoFli5dih49emi1cZcvX0ZiYiJatWoFAwMDGBgY4MyZM/jxxx9hYGAAR0dH5ObmIiUlRSVfQkICnJycSizXyMgIlpaWKi8qW3aeHPcepwMAGrJnh4iIqgmNh7GWLl2Knj17Ijg4GLm5ufj0009x8+ZNJCcnIyAgQKuN69mzJ65fv65y7d1330WDBg0wa9YsuLm5wdDQECdOnMCwYcMAAGFhYYiOjlbu/0PaE5GYDoUArE0N4WCh3V48IiKiiqJxsNOkSRPcvXsXP/30EywsLJCeno6hQ4fCz88Pzs7OWm2chYUFmjRponLNzMwMtra2yusTJkzAjBkzYGNjA0tLS3z44Yfw9vZGhw4dtNoWema+jpMFJBJJJbeGiIhIPRoHO9HR0XBzc8Nnn31W7D13d3etNExd33//PfT09DBs2DDk5OSgd+/e+Pnnn19qG14VYcr5OhzCIiKi6kMihBCaZNDX10dcXBwcHBxUrj958gQODg6Qy+VabeDLkJqaCisrK8hkMs7fKcXb/3cR5yOS8M2wphjZ9uUGtURERM9T9/Nb4wnKQohihzDS09O1umKKqh6edk5ERNWR2sNYM2bMAFCw+mr+/Pkqy8/lcjkuXryIFi1aaL2BVDU8TstBUnouJBKgniOXnRMRUfWhdrATGhoKoKBn5/r165BKpcp7UqkUzZs3xyeffKL9FlKVUNir42lrBhOpfiW3hoiISH1qBzunTp0CULD0+4cffuDclldM4Unn3EyQiIiqG41XY23cuLEi2kFV3G3O1yEiomqKx1aTWpTLzp3Zs0NERNULgx0qU75cgfCEf4+JYM8OERFVMwx2qExRSRnIlStgJtWHaw2Tym4OERGRRhjsUJlu/zuEVd/JAnp6PCaCiIiqFwY7VKY7/56JVZ9DWEREVA0x2KEy3fm3Z6chJycTEVE1xGCHynQnjsvOiYio+mKwQ6WSZeUhVpYNoGDODhERUXXDYEdHxcmy8E9kEuJkWeUqp3B/nZrWJrAyMdRG04iIiF4qjXdQpqpvZ1A05uy5DoUA9CSA/9CmGNnW/YXK+u+kc/bqEBFR9cSeHR0TJ8tSBjoAoBDA3D03XriH53Ycd04mIqLqjcGOjolKylAGOoXkQuB+UuYLlVfYs8Nl50REVF0x2NExXnZmxV6/8jBF47IUCqGcs9OQw1hERFRNMdjRMcuXfoO4zdMR/f1wPFw5Gol7vkLek0f45sgdrDh+F0IIREZGYsiQIbC3t4elpSVGjBiBhISEImU9fJqJzFw5pPp6sDNSYNq0afDw8ICJiQk6duyIoKAgZdq8vDzMmjULTZs2hZmZGVxcXDB27FjExsa+zMcnIiIqgsGOjvn94N+waNUfr3/2Czbu3A9vTytkHlgIRW42VhwPx8wdl+Dr6wuJRIKTJ08iICAAubm5GDBgABQKhUpZhZsJ1nU0x+T3J+HYsWP49ddfcf36dfj6+sLHxwcxMTEAgMzMTISEhGD+/PkICQnBnj17EBYWhoEDB7707wEREdGzJEIIUXYy3ZaamgorKyvIZDJYWlbfuSlXH6Zg0KoASCTA0WldUc/RAo8fP4aDgwPmrtqBbQ/NkXkvBI93fYH4x0lwsK0BAJDJZKhRowb+/vtv+Pj4KMv74Xg4vj9+FwOb2OHndzph//796N+/v/J+69at0bdvX3z11VfFticoKAjt2rXDgwcP4O7+YqvBiIiISqLu5zd7dnTIt0fDAABDW7qinmPBHBuZTAYAGNW1MVaPbgV9IYcAMHnbVcgy8wAAxsbG0NPTw/nz51XKK5ycXM/eBHK5HMbGxir3TUxMiuR5lkwmg0QigbW1tTYej4iI6IUw2NER/0Qk4XxEEgz1JZjmUxcAoFAUzLPp1KkTmjRpgj5NnPHLzFHQkxrj+ObvMXTlSUTGJuGTTz6BXC5HXFycSpmFw1gtarnA29sbixYtQmxsLORyOX777TcEBgYWyVMoOzsbs2bNwqhRo6p1bxkREVV/DHZ0gBAC3/zbqzO6vQfcbEwBAH5+frhx4wZ27NihTNu3bX2s2/QbcqOCcHJ2X9Rxc0R0/GO0atUKenr/vR0yc/Nx/0kGgIJjIn799VcIIVCzZk0YGRnhxx9/xKhRo1TyFMrLy8OIESMghMDq1asr8tGJiIjKxGBHBxy9mYCrD1NgKtWHX486AIApU6bg0KFDOHXqFFxdXVXST3hzCCIiItD5y71w+3Ab7jcZj/vRj1CrVi1lmrsJ6RACsDOXwt7CCLVr18aZM2eQnp6Ohw8f4tKlS8jLy1PJA/wX6Dx48ADHjh1jrw4REVU6BjvVnFwhsOzvgl6dCZ29YGcuxZQpU7B3716cPHkSXl5exeZzrWGKAx/3RZt6roi/E4zkpMewbdRReb+kk87NzMzg7OyMp0+f4ujRoxg0aJDyXmGgEx4ejuPHj8PW1lbbj0tERKQxno1Vze0JeYSIxHRYmxpiYtda8PPzw7Zt27B//35YWFggPj4eAGBlZQUTExMAwMaNG9GwYUPY29vjddNI/HnwG1i0HYSvzqfAxD4ao9q5Y+7EEUh1aIEGnT8CABw9ehRCCNSvXx8RERGYOXMmGjRogHfffRdAQaDzxhtvICQkBIcOHYJcLlfWbWNjA6lUWgnfHSIiIgY71VpOvhwrjocDAD7oXhuWxobKOTLdu3dXSbtx40a88847AICwsDDMmTMHycnJ8PT0xKIvPkeiR0/suhyDOXuuIzE1B/GPHkBqUQtOVgUrsGQyGebMmYNHjx7BxsYGw4YNw9dffw1Dw4KT0GNiYnDgwAEAQIsWLVTqPnXqVJH2EBERvSzcZwfVd5+djQFR+PLgLThZGuP0zO4wNtR/4bKEEPju2F2sPBmhcl0iAZaU49R0IiKiisJ9dnRcek4+fvo3MJnqU7dcgQ4ASCQSfOxbH5/41lO5Lsp5ajoREVFlY7BTTW04H4UnGbnwsjPD8NauZWdQUyuPGkWulefUdCIiosrGYKcaSs7Ixfqz9wAAM3rVg4G+9n6MXnZm0JOoXtOXSOBpZ6q1OoiIiF4mBjvV0OrTEUjLyUdjF0v0b+qs1bKdrUzgP7Qp9CUFEY++RILFQ5vA2cpEq/UQERG9LAx2KoC/vz/atm0LCwsLODg4YPDgwQgLC1NJ0717d0gkEpXX5MmTSy23MN281xvjwTev48+pXaGvr4dvv/1WJd3hw4fRvn17mJiYoEaNGhg8eLBG7R/Z1h3nZ/fA9okdcH52D05OJiKiao1LzyvAmTNn4Ofnh7Zt2yI/Px9z586Fr68vbt26BTMzM2W6iRMnYuHChcqvTU1LHyqKi4vDV4duYf+VWLR0t8Zgu0S89957GDZsmDLNH3/8gYkTJ2Lx4sV47bXXkJ+fjxs3bmj8DM5WJuzNISIincBgpwIcOXJE5etNmzbBwcEBly9fRteuXZXXTU1N4eTkpHa5mfrm+OteNvTNa2DBSG8smjoePXr0UB7ZkJ+fj6lTp+Lbb7/FhAkTlPkaNWpUziciIiKqvjiM9RLIZDIABTsJP2vr1q2ws7NDkyZNMGfOHGRmlr7iafmxu5ArBHwaOsDVOA+HDx9WCWpCQkIQExMDPT09tGzZEs7Ozujbt+8L9ewQERHpCvbsVDCFQoFp06ahU6dOaNKkifL6W2+9BQ8PD7i4uODatWuYNWsWwsLCsGfPnmLLuREjw+FrcZBIgE9618fmzWtgYWGBoUOHKtPcu1ewQuuLL77Ad999B09PTyxfvhzdu3fH3bt3iwRbRERErwIGOxXMz88PN27cwPnz51WuT5o0Sfnvpk2bwtnZGT179kRkZCRq165dpJylRwsmOA9uURMNnCwxeMMGjB49GsbGxso0CoUCAPDZZ58p5/Fs3LgRrq6u2LVrF95//32tPx8REVFVx2GsCjRlyhQcOnQIp06dgqtr6Rv/tW/fHgAQERFR5F5g5BOcvfsYBnoSTPeph3PnziEsLAzvvfeeSjpn54Jl6M/O0TEyMkKtWrUQHR1d3schIiKqlhjsVAAhBKZMmYK9e/fi5MmT8PLyKjPPlStXAPwXsDxb1tKjdwAAb7V3h7utKX755Re0bt0azZs3V0nbunVrGBkZqSxzz8vLw/379+Hh4VHOpyIiIqqeOIxVAfz8/LBt2zbs378fFhYWiI+PBwBYWVnBxMQEkZGR2LZtG/r16wdbW1tcu3YN06dPR9euXdGsWTNlOQ0aNMDID2YhNNYBJob6mPJaHaSmpmLXrl1Yvnx5kXotLS0xefJkLFiwAG5ubvDw8FDuwTN8+PCX8/BERERVDIOdCrB69WoABRsHPmvjxo145513IJVKcfz4caxYsQIZGRlwc3PDsGHDMG/ePJX0YWFh+CMwDPBwwLudPOFgYYx167ZACIFRo0YVW/e3334LAwMDjBkzBllZWWjfvj1OnjyJGjWKnnlFRET0KpAIIURlN6KyqXtE/Mu24fw9LDx0GxbGBjg/6zVYmRhWdpOIiIiqDHU/vzlnp4r6NfABFh66DQBIz87HkRtxldwiIiKi6onBThV09u5jzN//30aAAsDcPTcQJ8uqvEYRERFVUwx2qhAhBDYFRGH8pqAi9+RC4H5S6TssExERUVGcoFxFPEnPwczd13DyTmKx9/UlEnjalX5QKBERERXFYKcKOHv3MT7edRWP03IgNdDDZ/0awshAD5/tvQG5ENCXSLB4aBOeQk5ERPQCGOxUotx8BZb9HYZ1ZwvOtKrnaI4fR7VEA6eCGeXd6tvjflImPO1MGegQERG9IAY7lSTycTqm7gjFjZhUAMCYDh74rH9DGBvqK9M4W5kwyCEiIionBjsvmRACu4IfYcGBm8jKk6OGqSGWvtEcvRo5VnbTiIiIdBKDnZdIlpmHuXuv4/D1gj1zOta2xXcjWsDJyriMnERERPSiGOy8JEH3kzFtxxXEpGTBQE+Cj33r4/2utaCnJ6nsphEREek0BjsVKE6WhYjEdJy8k4jN/9yHQgCetqb44c2WaO5mXdnNIyIieiUw2KkgO4OiMWfPdSieOXlsWCtXfDmoMcyN+G0nIiJ6WbiDcgWIk2UVCXQkEuCT3vUY6BAREb1kDHYqQFRShkqgAwBCgMc9EBERVQIGOxXAy84Mz8875nEPRERElYPBTgVwtjKB/9Cm0JcURDw87oGIiKjycAJJBRnZ1h1d6/G4ByIiosrGYKcC8bgHIiKiysdhLCIiItJpVTrY8ff3R9u2bWFhYQEHBwcMHjwYYWFhKmmys7Ph5+cHW1tbmJubY9iwYUhISKikFhMREVFVU6WDnTNnzsDPzw8XLlzAsWPHkJeXB19fX2RkZCjTTJ8+HQcPHsSuXbtw5swZxMbGYujQoZXYaiIiIqpKJEIIUXayquHx48dwcHDAmTNn0LVrV8hkMtjb22Pbtm144403AAB37txBw4YNERgYiA4dOqhVbmpqKqysrCCTyWBpaVmRj0BERERaou7nd5Xu2XmeTCYDANjY2AAALl++jLy8PPj4+CjTNGjQAO7u7ggMDCyxnJycHKSmpqq8iIiISDdVm2BHoVBg2rRp6NSpE5o0aQIAiI+Ph1QqhbW1tUpaR0dHxMfHl1iWv78/rKyslC83N7eKbDoRERFVomoT7Pj5+eHGjRvYsWNHucuaM2cOZDKZ8vXw4UMttJCIiIiqomqxz86UKVNw6NAhnD17Fq6ursrrTk5OyM3NRUpKikrvTkJCApycnEosz8jICEZGRhXZZCIiIqoiqnTPjhACU6ZMwd69e3Hy5El4eXmp3G/dujUMDQ1x4sQJ5bWwsDBER0fD29v7ZTeXiIiIqqAq3bPj5+eHbdu2Yf/+/bCwsFDOw7GysoKJiQmsrKwwYcIEzJgxAzY2NrC0tMSHH34Ib29vtVdiERERkW6r0kvPJRJJsdc3btyId955B0DBpoIff/wxtm/fjpycHPTu3Rs///xzqcNYz+PScyIioupH3c/vKh3svCwMdoiIiKofndxnh4iIiEhTDHaIiIhIpzHYISIiIp3GYIeIiIh0GoMdIiIi0mkMdoiIiEinMdghIiIincZgh4iIiHQagx0iIiLSaQx2iIiISKcx2CEiIiKdxmCHiIiIdBqDHSIiItJpDHaIiIhIpzHYISIiIp3GYIeIiIh0GoMdIiIi0mkMdoiIiEinMdghIiIincZgh4iIiHQagx0iIiLSaQx2iIiISKcx2CEiIiKdxmCHiIiIdBqDHSIiItJpDHaIiIhIpzHYISIiIp3GYIeIiIh0GoMdIiIi0mkMdoiIiEinMdghIiIincZgh4iIiHQagx0iIiLSaQx2iIiISKcx2CEiIiKdxmCHiIiIdBqDHSIiItJpDHaIiIhIpzHYISIiIp3GYIeIiIh0GoMdIiIi0mkMdoiIiEinMdghIiIincZgh4iIiHQagx0iIiLSaQx2iIiISKcx2CEiIiKdxmCHiIiIdBqDHSIiItJpDHaIiIhIpzHYISIiIp3GYIeIiIh0GoMdIiIi0mkMdoiIiEinMdghIiIincZgh4iIiHQagx0iIiLSaQx2iIiISKcx2CEiIiKdxmCHiIiIdBqDHSIiItJpDHaIiIhIpzHYISIiIp2mM8HOqlWr4OnpCWNjY7Rv3x6XLl2q7CYRERFRFaATwc7OnTsxY8YMLFiwACEhIWjevDl69+6NxMTEym4aERERVTKdCHa+++47TJw4Ee+++y4aNWqENWvWwNTUFBs2bKjsphEREVElM6jsBpRXbm4uLl++jDlz5iiv6enpwcfHB4GBgcXmycnJQU5OjvJrmUwGAEhNTa3YxhIREZHWFH5uCyFKTVftg52kpCTI5XI4OjqqXHd0dMSdO3eKzePv748vv/yyyHU3N7cKaSMRERFVnLS0NFhZWZV4v9oHOy9izpw5mDFjhvJrhUKB5ORk2NraQiKRaK2e1NRUuLm54eHDh7C0tHyp+Vn3y6+7vPlZ96tVd3nzs27WXV3yl7fu0gghkJaWBhcXl1LTVftgx87ODvr6+khISFC5npCQACcnp2LzGBkZwcjISOWatbV1RTURlpaW5foBlyc/6375dZc3P+t+teoub37WzbqrS/7y1l2S0np0ClX7CcpSqRStW7fGiRMnlNcUCgVOnDgBb2/vSmwZERERVQXVvmcHAGbMmIFx48ahTZs2aNeuHVasWIGMjAy8++67ld00IiIiqmQ6EeyMHDkSjx8/xueff474+Hi0aNECR44cKTJp+WUzMjLCggULigyZvYz8rPvl113e/Kz71aq7vPlZN+uuLvnLW7c2SERZ67WIiIiIqrFqP2eHiIiIqDQMdoiIiEinMdghIiIincZgh4iIiHQag50KtGrVKnh6esLY2Bjt27fHpUuX1Mp39uxZDBgwAC4uLpBIJNi3b5/adfr7+6Nt27awsLCAg4MDBg8ejLCwMLXzr169Gs2aNVNu/uTt7Y2//vpL7fzPWrJkCSQSCaZNm6ZW+i+++AISiUTl1aBBA7Xri4mJwdtvvw1bW1uYmJigadOmCA4OViuvp6dnkbolEgn8/PzKzCuXyzF//nx4eXnBxMQEtWvXxqJFi8o8q+VZaWlpmDZtGjw8PGBiYoKOHTsiKCioSLqy3htCCHz++edwdnaGiYkJfHx8EB4ernb+PXv2wNfXV7mb+JUrV9SuPy8vD7NmzULTpk1hZmYGFxcXjB07FrGxsWrV/cUXX6BBgwYwMzNDjRo14OPjg4sXL6rd9mdNnjwZEokEK1asUCvvO++8U+Rn36dPH43qvn37NgYOHAgrKyuYmZmhbdu2iI6OLjNvce87iUSCb7/9Vq2609PTMWXKFLi6usLExER5GLI6eRMSEvDOO+/AxcUFpqam6NOnj/L9os7vkuzsbPj5+cHW1hbm5uYYNmyYcoNXdfKvW7cO3bt3h6WlJSQSCVJSUpT3ysqfnJyMDz/8EPXr14eJiQnc3d3x0UcfQSaTqVX3+++/j9q1a8PExAT29vYYNGiQ8oghTX6PCiHQt29f5fdXnbzdu3cv8vOePHmyRnUHBgbitddeg5mZGSwtLdG1a1csXLiw1Lz3798v8f22a9cuteqOj4/HmDFj4OTkBDMzM7Rq1Qp//PGHWnkjIyMxZMgQ2Nvbw9LSEiNGjCiyIXBFYbBTQXbu3IkZM2ZgwYIFCAkJQfPmzdG7d28kJiaWmTcjIwPNmzfHqlWrNK73zJkz8PPzw4ULF3Ds2DHk5eXB19cXGRkZauV3dXXFkiVLcPnyZQQHB+O1117DoEGDcPPmTY3aERQUhLVr16JZs2Ya5WvcuDHi4uKUr/Pnz6uV7+nTp+jUqRMMDQ3x119/4datW1i+fDlq1KihdnufrffYsWMAgOHDh5eZ95tvvsHq1avx008/4fbt2/jmm2+wdOlSrFy5Uq26AeC9997DsWPH8Ouvv+L69evw9fWFj48PYmJiVNKV9d5YunQpfvzxR6xZswYXL16EmZkZevfujezsbLXyZ2RkoHPnzvjmm29KvF9S/szMTISEhGD+/PkICQnBnj17EBYWhoEDB6pVd7169fDTTz/h+vXrOH/+PDw9PeHr64vHjx+rlb/Q3r17ceHCBZXt49XJ26dPH5X3wPbt29XOHxkZic6dO6NBgwY4ffo0rl27hvnz58PY2LjMvM/WGRcXhw0bNkAikWDYsGFq1T1jxgwcOXIEv/32G27fvo1p06ZhypQpOHDgQKl5hRAYPHgw7t27h/379yM0NBQeHh7w8fFBRkaGWr9Lpk+fjoMHD2LXrl04c+YMYmNjMXToUADq/S7KzMxEnz59MHfu3CLtKyt/bGwsYmNjsWzZMty4cQObNm3CkSNHMGHCBLXqbt26NTZu3Ijbt2/j6NGjEELA19cXcrlco9+jK1asUDlmSN28EydOVPm5L126VO38gYGB6NOnD3x9fXHp0iUEBQVhypQpOH/+fKl53dzcirzfvvzyS5ibm6Nv375q1T127FiEhYXhwIEDuH79OoYOHYoRI0bg4MGDpebNyMiAr68vJBIJTp48iYCAAOTm5mLAgAFQKBRFvq9aJ6hCtGvXTvj5+Sm/lsvlwsXFRfj7+2tUDgCxd+/eF25HYmKiACDOnDnzwmXUqFFD/N///Z/a6dPS0kTdunXFsWPHRLdu3cTUqVPVyrdgwQLRvHnzF2rjrFmzROfOnV8ob3GmTp0qateuLRQKRZlp+/fvL8aPH69ybejQoWL06NFq1ZWZmSn09fXFoUOHVK63atVKfPbZZyXme/69oVAohJOTk/j222+V11JSUoSRkZHYvn17mfmfFRUVJQCI0NBQtesvzqVLlwQA8eDBA43zymQyAUAcP35c7bofPXokatasKW7cuCE8PDzE999/r1becePGiUGDBpXantLyjxw5Urz99tsvlPd5gwYNEq+99pra+Rs3biwWLlyocq24987zecPCwgQAcePGDeU1uVwu7O3txfr164vU/fzvkpSUFGFoaCh27dqlTHP79m0BQAQGBpaZ/1mnTp0SAMTTp0+Lfe6y8hf6/fffhVQqFXl5eRrnvXr1qgAgIiIi1K47NDRU1KxZU8TFxZX4sy0urya/F4vL3759ezFv3rwXyvu8Fi1aFPn9VVp+MzMzsWXLFpV0NjY2Rd4zz+c9evSo0NPTEzKZTJkmJSVFSCQScezYsTKfpbzYs1MBcnNzcfnyZfj4+Civ6enpwcfHB4GBgS+1LTKZDABgY2OjcV65XI4dO3YgIyNDo6M3/Pz80L9/f5XnV1d4eDhcXFxQq1YtjB49GtHR0WrlO3DgANq0aYPhw4fDwcEBLVu2xPr16zWuHyj4+f32228YP368WgfDduzYESdOnMDdu3cBAFevXsX58+fRt29fterLz8+HXC6HsbGxynUTExO1e7YAICoqCvHx8SrfdysrK7Rv3/6lv+8KyWQySCQSjc+ey83Nxbp162BlZYXmzZurlUehUGDMmDGYOXMmGjdurHFbT58+DQcHB9SvXx//+9//8OTJE7XrPXz4MOrVq4fevXvDwcEB7du312j4uVBCQgIOHz6MCRMmqJ2nY8eOOHDgAGJiYiCEwKlTp3D37l34+vqWmi8nJwcAVN53enp6MDIyKvZ99/zvksuXLyMvL0/l/dagQQO4u7sX+34rz+8idfPLZDJYWlrCwMCgyPXS8mZkZGDjxo3w8vKCm5ubWnVnZmbirbfewqpVq0o8h7G0urdu3Qo7Ozs0adIEc+bMQWZmplr5ExMTcfHiRTg4OKBjx45wdHREt27d1PqZPe/y5cu4cuVKie+34vJ37NgRO3fuRHJyMhQKBXbs2IHs7Gx079691Lw5OTmQSCQqGwsaGxtDT09Po99zL6zCw6lXUExMjAAg/vnnH5XrM2fOFO3atdOoLJSjZ0cul4v+/fuLTp06aZTv2rVrwszMTOjr6wsrKytx+PBhtfNu375dNGnSRGRlZQkhNPsL5s8//xS///67uHr1qjhy5Ijw9vYW7u7uIjU1tcy8RkZGwsjISMyZM0eEhISItWvXCmNjY7Fp0ya1215o586dQl9fX8TExKiVXi6Xi1mzZgmJRCIMDAyERCIRixcv1qhOb29v0a1bNxETEyPy8/PFr7/+KvT09ES9evVKzPP8eyMgIEAAELGxsSrphg8fLkaMGFFm/mdpo2cnKytLtGrVSrz11ltq5z148KAwMzMTEolEuLi4iEuXLqld9+LFi0WvXr2UvXGa9Oxs375d7N+/X1y7dk3s3btXNGzYULRt21bk5+eXmb/wr3pTU1Px3XffidDQUOHv7y8kEok4ffq0Ws9d6JtvvhE1atRQ/v9Rp+3Z2dli7NixAoAwMDAQUqlUbN68ucy8ubm5wt3dXQwfPlwkJyeLnJwcsWTJEgFA+Pr6quQt7nfJ1q1bhVQqLVJP27Ztxaefflpm/meV1bOjzu+yx48fC3d3dzF37ly1865atUqYmZkJAKJ+/frF9uqUlH/SpEliwoQJyq+L+9mUlHft2rXiyJEj4tq1a+K3334TNWvWFEOGDFGr7sDAQAFA2NjYiA0bNoiQkBAxbdo0IZVKxd27d9V67kL/+9//RMOGDYu9V1L+p0+fCl9fX+X7zdLSUhw9erTMvImJicLS0lJMnTpVZGRkiPT0dDFlyhQBQEyaNKnENmoLg50KUFWCncmTJwsPDw/x8OFDjfLl5OSI8PBwERwcLGbPni3s7OzEzZs3y8wXHR0tHBwcxNWrV5XXNAl2nvf06VNhaWmp1hCaoaGh8Pb2Vrn24Ycfig4dOmhcr6+vr3j99dfVTr99+3bh6uoqtm/fLq5duya2bNkibGxsNAq0IiIiRNeuXQUAoa+vL9q2bStGjx4tGjRoUGKeqhzs5ObmigEDBoiWLVuqdFuXlTc9PV2Eh4eLwMBAMX78eOHp6SkSEhLKzB8cHCwcHR1VAlRNgp3nRUZGqj2EVvj/fdSoUSrpBgwYIN58802N6q5fv76YMmVKifeLy//tt9+KevXqiQMHDoirV6+KlStXCnNz8yJDA8XlDQ4OFs2bN1e+73r37i369u0r+vTpo5KuuN8lmgQ7Zf0uKivYKSu/TCYT7dq1E3369BG5ublq501JSRF3794VZ86cEQMGDBCtWrUqEmgWl3///v2iTp06Ii0tTXmtuO+vur+DT5w4UewQWnH5C/+fz5kzRyVt06ZNxezZs9WuOzMzU1hZWYlly5YVe7+k/FOmTBHt2rUTx48fF1euXBFffPGFsLKyEteuXSsz79GjR0WtWrWERCIR+vr64u233xatWrUSkydPLuW7ox0MdipATk6O0NfXL/LGHzt2rBg4cKBGZb1osOPn5ydcXV3FvXv3NM77vJ49e6oVee/du1f5S7PwBUD5xi7ur+SytGnTRuU/cEnc3d1V/soSQoiff/5ZuLi4aFTf/fv3hZ6enti3b5/aeVxdXcVPP/2kcm3RokWifv36GtUtRMGHfWGwMmLECNGvX78S0z7/3ij8gH4+QOnatav46KOPysz/rPIEO7m5uWLw4MGiWbNmIikpSaO8z6tTp06xvWTP5//++++V77Nn33t6enrCw8Pjheq2s7MTa9asKbPunJwcYWBgIBYtWqSS7tNPPxUdO3ZUu+6zZ88KAOLKlSsltun5/JmZmcLQ0LDIfK8JEyaI3r17q113SkqKSExMFEIUzDf84IMPlPdK+l1S+AH9fIDi7u4uvvvuuzLzP6u0YKes/KmpqcLb21v07NmzSKCiye/BnJwcYWpqKrZt21Zm/qlTp5b4fuvWrZvGdaenpwsA4siRI2XWfe/ePQFA/PrrryrXR4wYoexFVafuLVu2CENDQ+XP/Vkl5Y+IiCgyz0uIgs+I999/X+26Hz9+rPxZOzo6iqVLl5aYVls4Z6cCSKVStG7dGidOnFBeUygUOHHihEZzX16EEAJTpkzB3r17cfLkSXh5eZW7TIVCoRzfL03Pnj1x/fp1XLlyRflq06YNRo8ejStXrkBfX1+jetPT0xEZGQlnZ+cy03bq1KnIMse7d+/Cw8NDozo3btwIBwcH9O/fX+08mZmZ0NNT/a+kr6//QisMzMzM4OzsjKdPn+Lo0aMYNGiQ2nm9vLzg5OSk8r5LTU3FxYsXK/x9VygvLw8jRoxAeHg4jh8/Dltb23KVp+57b8yYMbh27ZrKe8/FxQUzZ87E0aNHNa730aNHePLkiVrvPalUirZt25b7/ffLL7+gdevWas9RAgq+33l5eeV+/1lZWcHe3h7h4eEIDg7GoEGDyvxd0rp1axgaGqq838LCwhAdHQ1vb+9y/y5SJ39qaip8fX0hlUpx4MAB5fyjF6lbFPzxj5ycnDLzz549u8j7DQC+//57bNiwQeO6C/M7OzuXWbenpydcXFyKfb+5u7urXfcvv/yCgQMHwt7eXuV7UFr+wnlFxb3f5HK52nXb2dnB2toaJ0+eRGJionLFZoWq8HDqFbVjxw5hZGQkNm3aJG7duiUmTZokrK2tRXx8fJl509LSRGhoqAgNDRUAlPMAnl/RUpz//e9/wsrKSpw+fVrExcUpX5mZmWq1e/bs2eLMmTMiKipKXLt2TcyePVtIJBLx999/q5X/eZoMY3388cfi9OnTIioqSgQEBAgfHx9hZ2dX7F8ez7t06ZIwMDAQX3/9tQgPDxdbt24Vpqam4rffflO7rXK5XLi7u4tZs2apnUeIgpU8NWvWFIcOHRJRUVFiz549ws7OrkhXfmmOHDki/vrrL3Hv3j3x999/i+bNm4v27dsX6ZIv672xZMkSYW1trZx/MmjQIOHl5aX8i7es/E+ePBGhoaHi8OHDAoDYsWOHCA0NFXFxcWXmz83NFQMHDhSurq7iypUrKu+/nJycUvOmp6eLOXPmiMDAQHH//n0RHBws3n33XWFkZKT8K1LT/xfPDmOVljctLU188sknIjAwUERFRYnjx4+LVq1aibp164rs7Gy16t6zZ48wNDQU69atE+Hh4WLlypVCX19fnDt3Tq12y2QyYWpqKlavXl3kOcrK361bN9G4cWNx6tQpce/ePbFx40ZhbGwsfv755zLz/v777+LUqVMiMjJS7Nu3T3h4eIihQ4cKIdT7XTJ58mTh7u4uTp48KYKDg4W3t7dyOFmd/HFxcSI0NFSsX79eABBnz54VoaGh4smTJ2Xml8lkon379qJp06YiIiJCJc3kyZNLzRsZGSkWL14sgoODxYMHD0RAQIAYMGCAsLGxEQkJCS/0exT/9pyVlTciIkIsXLhQBAcHi6ioKLF//35Rq1Yt0bVrV7W/b99//72wtLQUu3btEuHh4WLevHnC2NhYvPXWW2q1Ozw8XEgkEvHXX3+pXC+r7tzcXFGnTh3RpUsXcfHiRRERESGWLVsmJBKJ6NevX5l1b9iwQQQGBoqIiAjx66+/ChsbGzFjxowSv6faxGCnAq1cuVK4u7sLqVQq2rVrJy5cuKBWvsIu3edf48aNKzNvcfkAiI0bN6pV9/jx44WHh4eQSqXC3t5e9OzZ84UDHSE0C3ZGjhwpnJ2dhVQqFTVr1hQjR44sdsJgSQ4ePCiaNGkijIyMRIMGDcS6des0auvRo0cFABEWFqZRvtTUVDF16lTh7u4ujI2NRa1atcRnn30mcnJy1C5j586dolatWkIqlQonJyfh5+cnUlJSiqQr672hUCjE/PnzhaOjozAyMhI9e/ZUeZ6y8m/cuLHY+wsWLCgzf+HQV3GvU6dOlZo3KytLDBkyRLi4uAipVCqcnZ3FwIEDVSYoa/r/4tlgp7S8mZmZwtfXV9jb2wtDQ0Ph4eEhJk6cqPKHiTp1//LLL6JOnTrC2NhYNG/eXDkUqk7etWvXChMTkxf6mcfFxYl33nlHuLi4CGNjY1G/fn2xfPlyoVAoysz7ww8/CFdXV2FoaCjc3d3FvHnzlO9bdX6XZGVliQ8++EDUqFFDmJqaiiFDhigDY3XyL1iwoMQ0ZeUv6dlKexXmjYmJEX379hUODg7C0NBQuLq6irfeekvcuXNH7bY/rzDYKStvdHS06Nq1q7CxsRFGRkaiTp06YubMmcq5berW7e/vL1xdXYWpqanw9vYW586dUzvvnDlzhJubm5DL5UWeoaz8d+/eFUOHDhUODg7C1NRUNGvWTGzZskWtvLNmzRKOjo7C0NBQ1K1bV/k+fRkk/z4gERERkU7inB0iIiLSaQx2iIiISKcx2CEiIiKdxmCHiIiIdBqDHSIiItJpDHaIiIhIpzHYISIiIp3GYIeI6DmnT5+GRCJBSkpKZTeFiLSAwQ4RERHpNAY7REREpNMY7BBRlaNQKODv7w8vLy+YmJigefPm2L17N4D/hpgOHz6MZs2awdjYGB06dMCNGzdUyvjjjz/QuHFjGBkZwdPTE8uXL1e5n5OTg1mzZsHNzQ1GRkaoU6cOfvnlF5U0ly9fRps2bWBqaoqOHTsWOWmaiKoHBjtEVOX4+/tjy5YtWLNmDW7evInp06fj7bffxpkzZ5RpZs6cieXLlyMoKAj29vYYMGAA8vLyABQEKSNGjMCbb76J69ev44svvsD8+fOxadMmZf6xY8di+/bt+PHHH3H79m2sXbsW5ubmKu347LPPsHz5cgQHB8PAwADjx49/Kc9PRNrFg0CJqErJycmBjY0Njh8/Dm9vb+X19957D5mZmZg0aRJ69OiBHTt2YOTIkQCA5ORkuLq6YtOmTRgxYgRGjx6Nx48f4++//1bm//TTT3H48GHcvHkTd+/eRf369XHs2DH4+PgUacPp06fRo0cPHD9+HD179gQA/Pnnn+jfvz+ysrJgbGxcwd8FItIm9uwQUZUSERGBzMxM9OrVC+bm5srXli1bEBkZqUz3bCBkY2OD+vXr4/bt2wCA27dvo1OnTirldurUCeHh4ZDL5bhy5Qr09fXRrVu3UtvSrFkz5b+dnZ0BAImJieV+RiJ6uQwquwFERM9KT08HABw+fBg1a9ZUuWdkZKQS8LwoExMTtdIZGhoq/y2RSAAUzCciouqFPTtEVKU0atQIRkZGiI6ORp06dVRebm5uynQXLlxQ/vvp06e4e/cuGjZsCABo2LAhAgICVMoNCAhAvXr1oK+vj6ZNm0KhUKjMASIi3cWeHSKqUiwsLPDJJ59g+vTpUCgU6Ny5M2QyGQICAmBpaQkPDw8AwMKFC2FrawtHR0d89tlnsLOzw+DBgwEAH3/8Mdq2bYtFixZh5MiRCAwMxE8//YSff/4ZAODp6Ylx48Zh/Pjx+PHHH9G8eXM8ePAAiYmJGDFiRGU9OhFVEAY7RFTlLFq0CPb29vD398e9e/dgbW2NVq1aYe7cucphpCVLlmDq1KkIDw9HixYtcPDgQUilUgBAq1at8Pvvv+Pzzz/HokWL4OzsjIULF+Kdd95R1rF69WrMnTsXH3zwAZ48eQJ3d3fMnTu3Mh6XiCoYV2MRUbVSuFLq6dOnsLa2ruzmEFE1wDk7REREpNMY7BAREZFO4zAWERER6TT27BAREZFOY7BDREREOo3BDhEREek0BjtERESk0xjsEBERkU5jsENEREQ6jcEOERER6TQGO0RERKTTGOwQERGRTvt/SlQpQMz5O+MAAAAASUVORK5CYII=", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "plt.plot(np.arange(len(epochs_x)), epochs_acc, marker = '.')\n", - "plt.xlabel('epoch')\n", - "plt.ylabel('test accuracy')\n", - "plt.ylim(0, 100)\n", - "plt.xticks(np.arange(len(epochs_x)))\n", - "for i, txt in enumerate(epochs_acc):\n", - " if i%2 == 0:\n", - " pass\n", - " else:\n", - " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", - "plt.show()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "speck-rescnn", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.0" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/tests/test_nonsequential/using_AvgPool2d/exp_set_A/baseline-SCNN-example_3.ipynb b/tests/test_nonsequential/using_AvgPool2d/exp_set_A/baseline-SCNN-example_3.ipynb deleted file mode 100644 index dc53313a..00000000 --- a/tests/test_nonsequential/using_AvgPool2d/exp_set_A/baseline-SCNN-example_3.ipynb +++ /dev/null @@ -1,1500 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import torch, random\n", - "import torch.nn as nn\n", - "import sinabs.layers as sl\n", - "from tqdm.notebook import tqdm\n", - "\n", - "from tonic.datasets.dvsgesture import DVSGesture\n", - "from tonic.transforms import ToFrame\n", - "from torch.utils.data import DataLoader\n", - "from torch.nn import CrossEntropyLoss\n", - "from torch.optim import Adam\n", - "\n", - "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential\n", - "\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "torch.backends.cudnn.deterministic = True\n", - "random.seed(1)\n", - "torch.manual_seed(1)\n", - "torch.cuda.manual_seed(1)\n", - "np.random.seed(1)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Loading Data" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "batch_size = 3\n", - "num_workers = 1\n", - "epochs = 30\n", - "lr = 1e-3" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "root_dir = \"./DVSGESTURE\"\n", - "_ = DVSGesture(save_to=root_dir, train=True)\n", - "_ = DVSGesture(save_to=root_dir, train=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "n_time_steps = 50\n", - "to_raster = ToFrame(sensor_size=DVSGesture.sensor_size, n_time_bins=n_time_steps)\n", - "\n", - "snn_train_dataset = DVSGesture(save_to=root_dir, train=True, transform=to_raster)\n", - "snn_test_dataset = DVSGesture(save_to=root_dir, train=False, transform=to_raster)" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The transformed array is in shape [Time-Step, Channel, Height, Width] --> (50, 2, 128, 128)\n" - ] - } - ], - "source": [ - "sample_data, label = snn_train_dataset[0]\n", - "print(f\"The transformed array is in shape [Time-Step, Channel, Height, Width] --> {sample_data.shape}\")" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "snn_train_dataloader = DataLoader(snn_train_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True)\n", - "snn_test_dataloader = DataLoader(snn_test_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=False)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Network Module\n", - "\n", - "We need to define a `nn.Module` implementing the network we want the chip to reproduce." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "device: NVIDIA RTX A4000\n" - ] - } - ], - "source": [ - "if torch.cuda.is_available():\n", - " device = torch.device('cuda:0')\n", - " print('device: ', torch.cuda.get_device_name(0))\n", - "else:\n", - " device = torch.device('cpu')" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "class SNN(nn.Module):\n", - " def __init__(self) -> None:\n", - " super().__init__()\n", - "\n", - " self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False)\n", - " self.iaf1 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - " self.pool1 = nn.AvgPool2d(2,2)\n", - "\n", - " self.conv2 = nn.Conv2d(10, 10, 2, 1, bias=False)\n", - " self.iaf2 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - " self.pool2 = nn.AvgPool2d(3,3)\n", - "\n", - " self.conv3 = nn.Conv2d(10, 10, 3, 1, bias=False)\n", - " self.iaf3 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - " self.pool3 = nn.AvgPool2d(2,2)\n", - "\n", - " self.flat = nn.Flatten()\n", - "\n", - " self.fc1 = nn.Linear(810, 100, bias=False)\n", - " self.iaf4 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - "\n", - " self.fc2 = nn.Linear(100, 100, bias=False)\n", - " self.iaf5 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - "\n", - " self.fc3 = nn.Linear(100, 100, bias=False)\n", - " self.iaf6 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - "\n", - " self.fc4 = nn.Linear(100, 11, bias=False)\n", - " self.iaf7 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - "\n", - " def detach_neuron_states(self):\n", - " for name, layer in self.named_modules():\n", - " if name != '':\n", - " if isinstance(layer, sl.StatefulLayer):\n", - " for name, buffer in layer.named_buffers():\n", - " buffer.detach_()\n", - "\n", - " def init_weights(self):\n", - " for name, layer in self.named_modules():\n", - " if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear):\n", - " nn.init.xavier_normal_(layer.weight.data)\n", - "\n", - " def forward(self, x):\n", - " \n", - " con1_out = self.conv1(x)\n", - " iaf1_out = self.iaf1(con1_out)\n", - " pool1_out = self.pool1(iaf1_out)\n", - "\n", - " conv2_out = self.conv2(pool1_out)\n", - " iaf2_out = self.iaf2(conv2_out)\n", - " pool2_out = self.pool2(iaf2_out)\n", - "\n", - " conv3_out = self.conv3(pool2_out)\n", - " iaf3_out = self.iaf3(conv3_out)\n", - " pool3_out = self.pool3(iaf3_out)\n", - "\n", - " flat_out = self.flat(pool3_out)\n", - " \n", - " fc1_out = self.fc1(flat_out)\n", - " iaf4_out = self.iaf4(fc1_out)\n", - "\n", - " fc2_out = self.fc2(iaf4_out)\n", - " iaf5_out = self.iaf5(fc2_out)\n", - "\n", - " fc3_out = self.fc3(iaf5_out)\n", - " iaf6_out = self.iaf6(fc3_out)\n", - "\n", - " fc4_out = self.fc4(iaf6_out)\n", - " iaf7_out = self.iaf7(fc4_out)\n", - "\n", - " return iaf7_out" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "snn = SNN().to(device)" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "snn.init_weights()" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "optimizer = Adam(snn.parameters(), lr=lr, betas=(0.9, 0.999), eps=1e-8)\n", - "loss_fn = CrossEntropyLoss()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Define train and test" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "def train(dataloader, model, loss_fn, optimizer, epochs, test_func, dataloader_test):\n", - " epochs_y = []\n", - " epochs_x = []\n", - " epochs_acc = []\n", - " model.train()\n", - "\n", - " for e in range(epochs):\n", - " losses = []\n", - " batches = []\n", - " batch_count = 0\n", - " train_p_bar = tqdm(snn_train_dataloader)\n", - "\n", - " for X, y in train_p_bar:\n", - " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", - " X = X.reshape(-1, 2, 128, 128).to(dtype=torch.float, device=device)\n", - " y = y.to(dtype=torch.long, device=device)\n", - "\n", - " # forward\n", - " pred = model(X)\n", - "\n", - " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", - " pred = pred.reshape(batch_size, n_time_steps, -1)\n", - "\n", - " # accumulate all time-steps output for final prediction\n", - " pred = pred.sum(dim = 1)\n", - " loss = loss_fn(pred, y)\n", - "\n", - " # gradient update\n", - " optimizer.zero_grad()\n", - " loss.backward()\n", - " optimizer.step()\n", - "\n", - " # detach the neuron states and activations from current computation graph(necessary)\n", - " model.detach_neuron_states()\n", - "\n", - " train_p_bar.set_description(f\"Epoch {e} - BPTT Training Loss: {round(loss.item(), 4)}\")\n", - "\n", - " batch_count += 1\n", - " losses.append(loss.item())\n", - " batches.append(batch_count)\n", - "\n", - " epochs_y.append(losses)\n", - " epochs_x.append(batches)\n", - "\n", - " acc = test_func(dataloader_test, model)\n", - " print(f'Epoch {e} accuracy: {acc}')\n", - " epochs_acc.append(acc)\n", - "\n", - " return epochs_x, epochs_y, epochs_acc\n" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [], - "source": [ - "def test(dataloader, model):\n", - " correct_predictions = []\n", - " with torch.no_grad():\n", - " test_p_bar = tqdm(dataloader)\n", - " for X, y in test_p_bar:\n", - " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", - " X = X.reshape(-1, 2, 128, 128).to(dtype=torch.float, device=device)\n", - " y = y.to(dtype=torch.long, device=device)\n", - "\n", - " # forward\n", - " output = model(X)\n", - "\n", - " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", - " output = output.reshape(batch_size, n_time_steps, -1)\n", - "\n", - " # accumulate all time-steps output for final prediction\n", - " output = output.sum(dim=1)\n", - "\n", - " # calculate accuracy\n", - " pred = output.argmax(dim=1, keepdim=True)\n", - "\n", - " # compute the total correct predictions\n", - " correct_predictions.append(pred.eq(y.view_as(pred)))\n", - "\n", - " test_p_bar.set_description(f\"Testing Model...\")\n", - " \n", - " correct_predictions = torch.cat(correct_predictions)\n", - " return correct_predictions.sum().item()/(len(correct_predictions))*100" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Training loop" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "ebf2bea3d0124365abf1f2aa248ceab6", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - " 0%| | 0/359 [00:00" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "y_avg = []\n", - "for y in epochs_y:\n", - " y_avg.append(np.mean(y))\n", - "\n", - "plt.plot(np.arange(len(epochs_x)), y_avg, marker = '.')\n", - "plt.xlabel('epoch')\n", - "plt.ylabel('average loss')\n", - "plt.ylim(0,)\n", - "plt.xticks(np.arange(len(epochs_x)))\n", - "for i, txt in enumerate(y_avg):\n", - " if i%2 == 0:\n", - " pass\n", - " else:\n", - " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAG2CAYAAACZEEfAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAAB9HElEQVR4nO3dd1RUx98G8GfpRUB6kaqg2HvB3oIt9qiJJjHGWH5irLFgYkw0ETWJJsbE9ipqrDGxG3vBhgiKXSmKglQLLL3uvH8QNq60XYrg+nzO2XNk7p07c3evu987d4pECCFAREREpKY0qroCRERERJWJwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqbUqDXbOnTuH/v37w87ODhKJBPv27VPYLoTA119/DVtbW+jr66Nnz54ICwtT2OfFixcYNWoUjI2NUbNmTYwdOxapqamv8SyIiIioOqvSYCctLQ1NmzbFb7/9VuT2ZcuWYeXKlVizZg0CAgJgaGiIXr16ITMzU77PqFGjcOfOHZw4cQKHDh3CuXPnMH78+Nd1CkRERFTNSarLQqASiQR79+7FoEGDAOS36tjZ2WHmzJn44osvAABSqRTW1tbYtGkT3n//fdy7dw8NGjRAYGAgWrVqBQA4evQo+vbtiydPnsDOzq6qToeIiIiqCa2qrkBxIiIiEBcXh549e8rTTExM0LZtW/j7++P999+Hv78/atasKQ90AKBnz57Q0NBAQEAABg8eXOSxs7KykJWVJf9bJpPhxYsXMDc3h0QiqbyTIiIiogojhEBKSgrs7OygoVH8w6pqG+zExcUBAKytrRXSra2t5dvi4uJgZWWlsF1LSwtmZmbyfYri4+ODb7/9toJrTERERFUhKioK9vb2xW6vtsFOZfL29saMGTPkf0ulUjg6OiIqKgrGxsZVWDMiIiJSVnJyMhwcHGBkZFTiftU22LGxsQEAxMfHw9bWVp4eHx+PZs2ayfdJSEhQyJebm4sXL17I8xdFV1cXurq6hdKNjY0Z7BAREb1hSuuCUm3n2XFxcYGNjQ1OnTolT0tOTkZAQAA8PDwAAB4eHkhKSsLVq1fl+5w+fRoymQxt27Z97XUmIiKi6qdKW3ZSU1MRHh4u/zsiIgLXr1+HmZkZHB0dMW3aNHz33Xdwc3ODi4sL5s+fDzs7O/mIrfr166N3794YN24c1qxZg5ycHEyePBnvv/8+R2IRERERgCoOdoKCgtCtWzf53wX9aEaPHo1NmzZh9uzZSEtLw/jx45GUlISOHTvi6NGj0NPTk+fZtm0bJk+ejB49ekBDQwNDhw7FypUrX/u5EBERUfVUbebZqUrJyckwMTGBVCplnx0iIqI3hLK/39W2zw4RERFRRWCwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGptWod7OTl5WH+/PlwcXGBvr4+6tSpg0WLFkEIId9HCIGvv/4atra20NfXR8+ePREWFlaFtSYiIqLqpFoHO0uXLsXq1auxatUq3Lt3D0uXLsWyZcvw66+/yvdZtmwZVq5ciTVr1iAgIACGhobo1asXMjMzq7DmREREVF1IxMvNJNXMu+++C2tra2zYsEGeNnToUOjr62Pr1q0QQsDOzg4zZ87EF198AQCQSqWwtrbGpk2b8P777ytVTnJyMkxMTCCVSmFsbFwp50JEREQVS9nf72rdstO+fXucOnUKoaGhAIAbN27gwoUL6NOnDwAgIiICcXFx6NmzpzyPiYkJ2rZtC39//2KPm5WVheTkZIUXERERqSetqq5ASebOnYvk5GS4u7tDU1MTeXl5+P777zFq1CgAQFxcHADA2tpaIZ+1tbV8W1F8fHzw7bffVl7FiYiIqNqo1i07f/75J7Zt24bt27fj2rVr2Lx5M3788Uds3ry5XMf19vaGVCqVv6KioiqoxkRERFTdVOuWnVmzZmHu3LnyvjeNGzfG48eP4ePjg9GjR8PGxgYAEB8fD1tbW3m++Ph4NGvWrNjj6urqQldXt1LrTkRERNVDtW7ZSU9Ph4aGYhU1NTUhk8kAAC4uLrCxscGpU6fk25OTkxEQEAAPD4/XWlciqhqx0gxcevAMsdKMqq4KVSJnZ2dIJJJCLy8vLwBA165dC22bOHFiiccs6ngSiQQ//PCDwn6HDx9G27Ztoa+vD1NTUwwaNKiyTpMqSbVu2enfvz++//57ODo6omHDhggODsby5cvx6aefAsi/UKdNm4bvvvsObm5ucHFxwfz582FnZ8eLkegtsCswEt57bkEmAA0J4DOkMUa0dqzqalElCAwMRF5envzv27dv45133sGwYcPkaePGjcPChQvlfxsYGJR4zNjYWIW/jxw5grFjx2Lo0KHytL///hvjxo3D4sWL0b17d+Tm5uL27dvlPR16zap1sPPrr79i/vz5mDRpEhISEmBnZ4cJEybg66+/lu8ze/ZspKWlYfz48UhKSkLHjh1x9OhR6OnpVWHNiaiyxUozMHfPLRRMniETwLw9t9G5riVsTfSrtnJU4SwtLRX+XrJkCerUqYMuXbrI0wwMDOTdG5Tx6r779+9Ht27dULt2bQBAbm4upk6dih9++AFjx46V79egQYOynAJVoWr9GMvIyAg///wzHj9+jIyMDDx48ADfffcddHR05PtIJBIsXLgQcXFxyMzMxMmTJ1G3bt0qrDURvQ7XHifi1VnC8oTAo2fpVVMhem2ys7OxdetWfPrpp5BIJPL0bdu2wcLCAo0aNYK3tzfS05W/FuLj43H48GGFoObatWuIjo6GhoYGmjdvDltbW/Tp04ctO2+gat2yQ0RUlPTsXKw8XXhZGE0J4GxR8qMLevPt27cPSUlJ+OSTT+RpI0eOhJOTE+zs7HDz5k3MmTMHISEh2LNnj1LH3Lx5M4yMjDBkyBB52sOHDwEA33zzDZYvXw5nZ2f89NNP6Nq1K0JDQ2FmZlah50WVh8EOEb1R8mQCU3ZcR0hcKgx1NJGRkwfZvy08ozu48BHWW2DDhg3o06cP7Ozs5Gnjx4+X/7tx48awtbVFjx498ODBA9SpU6fUY27cuBGjRo1S6AJRMBjmyy+/lPfj8fX1hb29PXbv3o0JEyZU1ClRJavWj7GIiF7l8889nLwXDx0tDWwZ2xYX53ZH30b5fS9uRiWhGq+AQxXg8ePHOHnyJD777LMS92vbti0AIDw8vNRjnj9/HiEhIYWOWTClyct9dHR1dVG7dm1ERkaqWnWqQgx2iOiNsfXyY/zfhQgAwE/DmqKlkylsTfSxYEBD6GhqIOhxIgIiXlRxLaky+fr6wsrKCv369Stxv+vXrwOAwhxsxdmwYQNatmyJpk2bKqS3bNkSurq6CAkJkafl5OTg0aNHcHJyUr3yVGUY7BBRmZU298mECRNQp04d6Ovrw9LSEgMHDsT9+/eVPv7EiRMhkUjw888/wy/0KRYcuAMA+Li+Nv7v6//BwsICxsbGGNq3J1rrxgAAfjtT+p08vZlkMhl8fX0xevRoaGn91wvjwYMHWLRoEa5evYpHjx7hwIED+Pjjj9G5c2c0adJEvp+7uzv27t2rcMzk5GTs3r27yJYiY2NjTJw4ERMmTJBf2zo6OkhISMCECRMq5Tp/WWhoKAYOHCi/zjt27IgzZ84ofVz6D4MdIiqzwMBAxMbGyl8nTpwAAPncJy1btoSvry/u3buHY8eOQQgBT09PhflSirN3715cvnwZdnZ2SEjOhNe2a8iTCQxtYY+d309Gbm4uTp8+jatXr6Jp06bYu3QKRHoizoc9Q3BkYqWeN1WNkydPIjIyUj7XWgEdHR2cPHkSnp6ecHd3x8yZMzF06FAcPHhQYb+QkBBIpVKFtJ07d0IIgQ8++KDIMn/44QeMHz8e5ubmMDQ0RKdOnbB+/XoAFX+dv+rdd98tdJ2/++67Ja79SMUQJKRSqQAgpFJpVVeF6I02depUUadOHSGTyYrcfuPGDQFAhIeHl3icJ0+eiFq1aonbt28LB0dH4dxvknCac0gMX3NJRMfGCwDi3Llz8v2Tk5MFADH0yzXCac4hMXbTlQo9L8rn5OQkABR6TZo0SQghxPjx40Xt2rWFnp6esLCwEAMGDBD37t0r8Zh///23eOedd4SZmZkAIIKDgwvts3btWtGlSxdhZGQkAIjExMQyn0NMUrq4GP5UxCSll/kYlXGdOzk5iRUrVsi3PX36tNjr/MSJE2WuuyrehM9b2d9vtuwQUYUobu6TAmlpafD19YWLiwscHByKPY5MJsNHH32EWbNmobabO56lZkOakQMXC0Os/aglbK0tUa9ePWzZsgVpaWnIzc3F2rVrYWVlhXmj+0IiAU7eS8DdmOTKPN23UmW05KWlpaFjx45YunRpsfukp6ejd+/emDdvXrnqvyswEh2WnMbI9QHosOQ0dgWq3sm4Mq7zhg0bFtpubm6OevXqoW/fvvJHaMbGxgCAd955B15eXnjx4gU+//xz1KtXD/r6+nB0dMSUKVMKtV69SpllMgYMGICcnBzo6OjA0tISQ4cOxY4dOwC8OZ+3glICu7cCW3aIym/Xrl1CU1NTREdHK6T/9ttvwtDQUAAQ9erVK/Vud/HixeKdd94Rubl5YuIfQULT2ErY9Z4gHj5Nle8TFRUlWrZsKSQSidDU1BS2trbi2rVrQgghvLZdFU5zDolJ265W/EmSgopq4RBCiIiIiGLv9AucOXOmzC07MUnpwnnuIeE0579X7bmHVW7hqejrvOC9e7VlR4j867xJkyYCgNDQ0BBWVlZi9erVAoA4c+aMuHXrlhgyZIg4cOCACA8PF6dOnRJubm5i6NChJZYdGxur8Nq4caOQSCTiwYMH8n2WL18u/P39xaNHj8TFixeFh4eHsLGxqXafN1t2iN4SJXUSLuud3zfffAN3d3cYGhrC1NQUPXv2REBAQInljhgxAm5uboX6HowaNQrBwcHw8/ND3bp1MXz4cGRmZhZZ7tWrV/HLL79g06ZN+OF4KI7cjoNEAgxv5QgXC0MAgBACXl5esLKywvnz53HlyhUMGjQI/fv3R2xsLLy6uQIA/rkViwdPU8v6tlIpKqqF43WJeJZWITNuFzXHD1D267yo9w747zqvVasWLly4gMDAQAwdOhRffPEFnJ2d0aVLFzRq1Ah///03+vfvjzp16qB79+74/vvvcfDgQeTm5hZ7DjY2NgqvV5fJAIDp06ejXbt2cHJyQvv27TFz5kzExcVh9OjRb8TnXUip4ddbgC079CZLSEhQuEs7ceJEue/8tm3bJk6cOCEePHggbt++LcaOHSuMjY1FQkKCfB8nJyexcOFCERsbK65cuSI0NDTEzp07SzxuVlaWMDAwENu3by9y+4oVK4REIhEampoCEo381793tU5OTkIIIU6ePCk0NDQK/X91dXUVPj4+Qgghxm4KFE5zDokZu66X9vZRGVVUC0eBym7ZiU5MU2jVKUvLzqNHj4SGhobYt29fifspe51ramrKX8pc51lZWUJDQ0N4enoWW/b69euFhYWF0ucUFxcntLS0xLZt24rd5/nz58LDw0MAqHafN1t2iN4SlpaWCndphw4dki+QWNY7v5EjR6Jnz56oXbs2GjZsiOXLlyM5ORk3b95U2M/IyAg2NjY4fPgwrKysFFaLLooQAkIIZGVlFbn9o48+wuZDfrAbsxK2Y1biizX7YWdnh1mzZuHYsWMAIF/vSEND8etLQ0NDPuPt5O75rTv7rkcj6gXXyqoMFdHC8TrFJRe+5ub2cVdpxm1l5/hR5jq/efMmrl+/Ln8pc53v27cPMpkMzZs3L/K4z549w6JFixRmky5NUctkFJgzZw4MDQ1hbm6Ou3fvwtPT8435vF/FYIdIjZT2aAEApFIpjI2NFeYpKe2Y69atg4mJSaFJ15YsWQIzMzMsXrwY7u7uCtsePnwIHx8fXL16FZGRkbh06RKGDRsGfX199O3bV77fy3OfJObpYNmVdGiaO2HYO+2xbFw/aGtrw8bGBvXq1QMAeHh4wNTUFKNHj8aNGzcQGhqKWbNmISIiQv4j1MyhJjq5WSBPJrDG74Fybx4praRZjE1MTODm5obOnTvjr7/+wv379wvNbVMVDt2IBQB4NrBGPesaAIC07OID/lcVN8dPWa5zc3NzNGrUSOGlzHU+Z84cSCSSIofJJycno1+/fmjQoAG++eYbpc+rqGUyCsyaNQvBwcH4448/IJVK8fz580IzlFfXz/tVDHaI1EhRCyS+TJU7v0OHDqFGjRrQ09PDihUrcOLECVhYWMi3T5kyBTt37sTixYuRk5ODq1evYvbs2fLtenp6OH/+PPr27QtXV1eMGDECRkZGuHTpEqysrOT7Fcx98iw1C2M2BSIlMxetnEyxdGiTIgM2CwsLHD16FKmpqejevTtatWqFCxcuYP/+/QrB2OR/++7sDnqCOGn1u9N8k1VUC8frIpMJ/HMrP9gZ1soBk/69NnZeiUJunkypYxQ3x4+q17myXr3OW7RogUePHuGrr74qdNORkpKC3r17w8jICHv37oW2trZSZRS3TMbLdahbty7Cw8NhZWWFq1ev4vLly8Uer7p83kVS6uGammOfHVIXnp6e4t133y1ym1QqFW3atBG9e/cW2dnZpR4rNTVVhIWFCX9/f/Hpp58KZ2dnER8fX+z+GzZsEFpaWiIzM1OlOsckpYsz9+NFv1/OCac5h0SnpafFsxTVjlGc91ZfFE5zDomFB+9UyPFIiLy8POHo6CjmzJmjkP7gwQOxePFiERQUJB4/fiwuXrwo+vfvL8zMzBSum3r16ok9e/bI/37+/LkIDg4Whw8fFgDEzp07RXBwsIiNjZXvExsbK4KDg8X69evlc88EBweL58+fK1XnwIjnwmnOIdHo66MiIztXZObkihYLjwunOYfEkVuxpR+ggpRnjp8FCxYIGxsbkZOTo5AulUpFu3btRJcuXURaWppKxxw9erRo2bJlifsUfN7/+9//5H0Bhag+n7eyv98MdgSDHVIPJXWeTE5OFh4eHqJHjx4iIyOjTMd3dXUVixcvLnb77du3BQBx//59pY+588pj4fLScOB6X/4jwuJTylS/opwNScg/7lf/VFgA9bY7duyYACBCQkIU0qOjo0WfPn2ElZWV0NbWFvb29mLkyJGFrgcAwtfXV/63r69vkRPXLViwQL7PggULitzn5eOUZMH+28JpziExfWewPG3JkXvCac4h8eH/XVb1LSiTl691l7mHxM4rj5XOW1yAKZVKRdu2bUXjxo1FeHi4wkCF3Nxc+X6vBhwFeQ0MDMTq1asLlXf58mXx66+/iuDgYLFlyxYBQDRv3lzUqVNHfjNTXT5vBjsqYLBD6qAy7vxeVrt2bYUvpFdt3bpVaGhoiBcvXih1vIinqcL5ldExLnMPlWtm21fJZDLR/9fzwmnOIbH0SMkzu74pSpvVNiMjQ0yaNEmYmZkJQ0NDMWTIEBEXF6f08SdMmCAAFJrzpahyC0a/qaoiZjFWVl6eTLT+7oRwmnNInLz73/sQ+TxNPu/Oy3M4VYaYpHSFoF7VkWDFBZgFo5WKekVERMj3KypQWLt2rdDX1xdJSUmFyrt586bo1q2bMDMzE7q6usLZ2Vl8/OlnYt+Fm2X6zCrz81b291u5HopEVK3JZDL834aN6DlgGJ6m5cDWJP+/dnJyMjw9PZGeno6tW7ciOTkZycn5MwtbWlpCU1MTQH7nSR8fHwwePBhpaWn4/vvvMWDAANja2uLZs2f47bffEB0dLZ851d/fHwEBAejWrRuMjIzg7++P6dOn48MPP4SpqWkJ9RQIiHiBfcHROHAjGuLV7QJ49CxdpREyJZFIJPDq5ooJf1zFFv/HmNC5DkwMlOvPUF0FBgYqzFB7+/ZtvPPOO/LPZvr06Th8+DB2794NExMTTJ48GUOGDMHFixdLPXZJ6zQBwMKFCzFu3Dj530ZGRirXf3vAY3y59zYEAA0J4DOkMUa0dlT5OMoKepyIhJQsGOlpoaPbf33OHMwM0K2eFU7fT8C2y4/x1bsNKq0OEc/SICtmjh9lrnVPT89CHYMBoGvXrkWmv0oIgVhpBi49eAYXC0PYmuhj/PjxRfbdE0LAvnY9rN2xH09Ts/AsNRsn7sTh0M1Y+B2MhORgJLrWs0QDO+NSywWAuzHJOBvy9LV93sVhsEOkBr76bTuin0ThtGiEDktOy79Qrl27Jp8M0NXVVSFPREQEnJ2dAeR3ngwKe4J20gyY6mri/v372Lx5M549ewZzc3O0bt0a58+fl09rr6uri507d+Kbb75BVlYWXFxcMH36dMyYMaPI+oXGp2BvcDT2B0cjpoTOwpoSCZwtDJQ+7+joaMyZMwdHjhxBeno6XF1d4evri1atWgEA4uPjsX3ZXMTtO4Ts9FS0+6cNDmzfADc3t2KP2bVrV/j5+RVK79u3Lw4fPgwg/wdh+vTpWLN2LbIyM6GhoQFXV1ds27ZNoew5c+bg+PHjSEpKQufOnfHrr7+Wu2wLCwssWLAA69evR1JSEiwtLeHo6IguXbpAKpViw4YN2L59O7p37w4gvzNx/fr1cfnyZbRr167E9/Lzzz/HsWPHiu14XDDVQFnFSjPkgQ6QH9zO23MbnetaVliA+6pDN2MAAJ4NbKCrpamw7cN2jjh9PwG7rz7BF73qQU9bs6hDlJuFoW6hNA0JVLrWy2NXYCS899yCTAASCTCilQNcrWrkBzMp2XiWmiV/PU/NRu6rkdlLBIAzIU9xJuSpyvV4HZ93cRjsEL3hYpLSsSPaFE5zDgHI/0KZ8/ctbLr0GLVqGmLOXzdgUUMXFjV0YGmkB4saOrAw0oVpDV0IIfBnUBRc5h7CthfAjn8DpT179pRYZosWLUoclQEA8cmZOHA9BnuDo3E39r91qoz0tNCvsS0GNa+FiGdp+GrvbeQJAU2JBIuHNFL6SzAxMREdOnRAt27dcOTIEVhaWiIsLEzesiSEwKBBg6CtrY3vf9+C5WejkBB8AD169MS9e3dhaGhY5HH37NmD7Oxs+d/Pnz9H06ZN5S0nQP4M0ytXroSuQ2OYN3kHabdOIv5ZDAwMDAqVvX//fhgbG2P58uXo2bMn7t4tX9nLli3DypUrsXnzZtjb26N9+/YwNDREVlYWrl69ipycHPTs2VO+v7u7OxwdHeHv719ssFPaOk0FlixZgkWLFsHR0REjR47E9OnTlZ7CAADO3E8o1JqnSguHqvJkAv/cyl8h/N2mtoW2d6lrBXtTfTxJzMDBGzEY1qpyZv7dez26UJqbVQ3YGBce7l3RYqUZ8kAHAIQAdgZGlZrPWE8LFka60NHUwP24lELbezeyLvUzi03KwNE78Qpplfl5l4TBDlEFUKaFQdW7/D179mDx4sUIDw9HTk4O3NzcMHPmTHz00UcAgJycHMz1nocNO/Yg+Wk0NHQNoefUFDW7fAItI3Pci03GvdiSF8PU1pAg56W7OJkA5v59C7FJmahtVSM/QKqhC4sauqhpoF3s3D2x0gxEPEuDlZEebkQlYd/1aFwMfyb/gtXWlKBrPSsMbl4L3d2t5HfQ7Wqbo2s9Szx6lg5nCwOVvgCXLl0KBwcH+Pr6ytNcXFzk/w4LC8Ply5dx+/ZtuNdvgEOxfnhoNhGJ6z7Bjh07ih1ua2ZmpvD3zp07YWBgIA84hBD48ccfoWlsBesPFgMADFzb4MmvH8LvchAaNGigUHZB8LB69WrY2NiUu+yff/4ZX331FQYOHIg///wTeXl5SEtLk084p6Ojg5o1ayocx9raGnFxcSW+l1paWpgyZUqx+0yZMgUtWrSAmZkZLl26BG9vb8TGxmL58uXF5nlZTp4Mmy49KpSuamueKq5EvMCz1CyY6GujQx2LQts1NSQY2dYRy46GYGtAZKUEOwnJmfC9GAEAWDq0CTQ1AO+/byEkPhV7g6MxpIV9hZf5socJhR+hAYBHHXO42xjBooZu/v9xI51/b4p0YV5DR94KFivNQIclpxWOoSmRYEH/hqUHO9IMHL8bXyjv62rRehmDHaJyUqWFQZW7fDMzM3z55Zdwd3eHjo4ODh06hDFjxsDKygq9evVCeMxzbD54FtqthsHWygWyzFS8OLUOT/csQq1PfobP4MbIFaJQM/Wz1Gw8S8lCSlauQqBTQAD4+VRYoXQtDQnMa+R/IVoa6cq/GKNepOOfW7GF7tgBoKWTKQY3r4V+jW1haqhT5HnamuiX6S7vwIED6NWrF4YNGwY/Pz/UqlULkyZNkvcpKZjrQ09PD5oaEvyvax3M/usmMmWa8Dt3vtiA41UbNmzA+++/L/+cHj58iPT0dBg2aIOn+3yQGXUbmjXMoWlsiTPnLuB/n36sUHYBDQ0N6Orq4sKFC2UuOyIiAnFxcfKWm4JZjJOTk+Hv74+2bdsqddyXFazTdO3atWKDWQAKjyibNGkCHR0dTJgwAT4+PtDVLfyY5lXrzj1EaHwq9LU1kZWbJ/8BnOlZt9Lu8g/fyn+E1auhNXS0ip5WbngrB6w4EYobUUm49USKxvYmFVqHX0+HIzNHhhaONTG8lT0kEgnik7Pww7EQfHvwLjq6WcDKqPJaeC4+eFYoTVMiwfLhTZV6321N9OEzpDHm7VG9BbY8eSsagx2iclKlhUGVu/yuXbsq/D116lRs3rwZFy5cQE23Vpi07RZqDP4G5oY6eK+lPf7vfATwzkTEbZmBme3NMKJNyZ0AM3PycDdGiqFr/BUWSJQA6OZuhbSsXHlwJM3IQa5MID45C/FFTLv/qnGdXPBRO2c4mlfeHdzDhw+xevVqzJgxA/PmzUNgYCCmTJkCHR0djB49Wv74xtvbG2vXrkW/hpaYNX8fcpKf4npIhFJlXLlyBbdv38aGDRsAAJHP0+G14QwAIC3kIoxbD4K1x3BkxYbhxfHfccrvImKSMgqVbWhoiBUrVuDJkyeIjY0tU9kA5K0z1tbW8lmM9+zZg23btiEuLg42NjbIzs5GUlKSQutOfHx8sX1tzp8/j4SEBDg6/ne95OXlYebMmfj555/x6NGjIvO1bdsWubm5ePTokXzW3+KEJ6Tgl38D6O8GNUJ7V3NM+OMqbj6R4kVadol5yyo3T4ajt/Pfr35Niu5wDQAWNXTRt7Et9l+PwdbLj7H0vSYVVofHz9Ow40okAGB2b3d5MDm+c238cysWd2KSsWD/Haz+sGWFlfmyq48Tse7cQwD5fXWEQJkCjhGtHdG5btlaYMuTtyJxBmWicjpw4ABatWqFYcOGwcrKCs2bN8f69evl20u7y1eGEAKnTp1CSEgIsi3rYdT/BeB5WjYa2hnjwOcd4d23Pi7M7QbvHk6QSCT4qEvx/S4K6GlrooWTGZYMaQzNf7+ENSUSLBnaGBs/aY1dEzxwamZX3FjgidDv+sDfuzsOTO6AjZ+0wrKhTTCrVz30bmhd5LG7u1tXaqAD5PczadGiBRYvXozmzZtj/PjxGDduHNasWQMA0NbWxp49exAaGgozMzOYGNWASWIo9Gq3xJOkTGTnlj5z7oYNG9C4cWM0b9kKa/wewPNnP1yPTAIAWDi4wrzraOhY14Fxs97QqmmNpLgovLPcD1uvPMHuv/6Wl21gYIAzZ86gT58+hdb0Kq3sNm3aFLm9qFmMW7ZsCW1tbZw6dUqeFhISgsjISHh4eBR5HGXWaSrK9evXoaGhoTBLcFHyZAKz/7qJ7FwZutazxJAWtWBroo/pPesCAP4MikK6Css2KCv/EVY2ahpoo30d8xL3/bCdEwBg/41oSDNyKqwOK06EIlcm0LmuJdrV/q8O2poaWPZeE2hpSHDkdhyO3FIuAFaFND0HU3YEI1cm8G4TW1yc0w07xrXDhbndyjQaytZEHx51zMsUrJQnb0Vhyw5ROanawqDKXb5UKkWtWrWQlZUFTU1NdB/rjV0xpgAE+je1w7KhTaCvk/9s3VRXgs2/fI8PPvgAxsbKDQsFlLvz0tHSKPJxU1U+k7e1tUWDBorDhevXr4+///5b/nfLli1x/fp1SKVSZGdnw6imGSxqN0KuSR3sDX5S4pd+Wloadu7ciQnTvdH/1wvyTpqt67vgEICOzerh97nd5e+bxy59PElLRFp2Hr49eBdN7U2w7R8/2Bvmry9maWmJtm3byvtxlaSg7IULFyqkF7TOxMbGKkw1EB8fj2bNmsHExARjx47FjBkzYGZmBmNjY3z++efw8PBQ6Jz88lQD5ubmMDdXDAZeXaeprFMNAMDmS49wLTIJNXS1sHhwY3nrRpe6lnA2N8Cj5+nYGxyNUW2dSn1fVHHo3wCid0MbaGuWHGC2cjJFPWsjhMSnYM+1JxjTwaXE/ZVxLzYZ+2/kP0ab3atwy1dDOxNM7FIHq86EY/7+O2hX27zYR72qEkJg9t83EJ2UAUczA/gMaQwjPW3Y1Xz9fWWqC7bsEJWTqi0MqtzlGxkZ4fr16zh29gLq9huHoxt+QGbkTczt446V7zeTBzo5OTkYPnw4hBBYvXq1yudQ1juvgmfyL7cMva5n8h06dEBISIhCWmhoKJycCv9ompiYwNLSElGPHiI9JhQGbm2x+uyDEtdF+mP7TqRlZGJnkhPux6WgpoE2fnivCfZ7D4aenh5u3Lghf98MJTl4EvUYtZ0c8N2gRjDS1cKNJ1IMWHURv1+MgaGJKcLCwhAUFISBAweWem67d+9GVlYWPvzwQ4V0FxcX2NjYYNaiH+VTDXh8ewj+lwPkLTcrVqzAu+++i6FDh6Jz586wsbEpNLouJCQEj2Of4tKDZ4iVZpRan4KpBrp06YKGDRvi+++/x/Tp07Fu3boS80U+T8cPx/I/o7l93GFX87/rQkNDgo88nAHkB0TKzBejrJcfYb1bwiOsAhKJBB965F83Wy8/rpC6/HgsBEIA/ZrYolGtovsBfd7DFa5WNfAsNQuLDt0td5kFtvg/xrE78dDWlGDVyOYw0nuz55aqEBU+neEbiDMoq4cnT56IUaNGCTMzM6GnpycaNWokAgMD5dtRzEyjy5YtK/aYubm54quvvhLOzs5CT09P1K5dWyxcuFDIZDL5Po6OjqJZs2bCxsZG6OnpiR49eohvvvlG2NnZFTpeUlKSSEhIEEII0aZNG/mstyW5EZUo2n5/UjjNOSRMm/cSrTp0VdienZ0tBg0aJJo0aSKePXtW6vEqQ0xSurgU/uy1zIhb4MqVK0JLS0t8//33IiwsTGzbtk0YGBiIrVu3yvf5888/xZkzZ8SDBw/Evn37hJOTkxgwaLBo9u0x4TTnkNgX/ER89NFHYu7cuQrHPnY7Vhg5NRIG7p3kywy8vNyEl5eXACA+/PBDcejQIdGyZUshkUjExo0bhRBCxEkzhOfnS4T1B4uF3YT/E+4fLRTWdg5iyJAhCuUUVbYQQnTs2FGMGDFCIS0vTyZuPUkSvcfMEBq6hsJyyHxh++kqoe/WTmiZWIsuPsfEqPWXxdQd18Sig3fE6rPh4q+gKHE2JEHcjk4S8dIMkZObJ4Qo39IFypLJZOKDdf7Cac4hMWLtJZGXJyu0jzQjW9Sff0Q4zTkkLoY9rbCyz4XmLxPSfOFx+TmXJiUzRzQoqEt4+epSsBZXbe/D4kFCycufXH38Qj6T8+l7xa89p6xbT5KE27x/hNOcQ2LD+YflPl51xxmU6a1S2ogoAIUeGR05cgRjx47F0KFDiz3u0qVLsXr1amzevBkNGzZEUFAQxowZAxMTE/kwXVNTU9y6dQt///03XFxcMH/+fCxfvhz169cvdDwTk/w7vIK7/EWLFpV4XnuuPcHcPbeQnStDHUtDmLlb4mnMf3NkFLTohIWF4cyZM4UeR7wuZR1RVR6tW7fG3r174e3tjYULF8LFxQU///wzRo0aJd8nNjYWM2bMQHx8PGxtbfHxxx9j/vz5WHfhMX48HorfzoQjOzJS3sIWJ83EggO3cejcVaQ8vo1GY5fh97Ft0MnNUqHsX3/9Fc+ePcPOnTuxdetWGBgYYOHChRgzZgwAwNpYD/1c9RG8eykSniYg3tAUNRp2h82gGXiakgVLo/zRS5EvlV0gJCQEFy5cwPHjxxGdlIELYU9xPuwZLj14jhdp2RCW3WDUMgbPj/0KWWYa9OwbwGr4QjxKysGjpMIjb14mkQAmetpIeqlfSmVN9LYzMAqXHjyHnrYGlg5tAg2NwiO9jPW0MbSFPf64/BibLj1Ce9fCw8PL4vDNfx9hNbKBVimPsArU0NXCoOa1sC0gEtsuR6J9EUPVlSGEwLKj+a1Zw1vZo7ZljRL3b+Foik87uGDDhQjM23sLx6d3LnNLTGpWLiZvv4bsPBl61rfGmA7OZTqOOpIIUYFth2+o5ORkmJiYQCqVqtTXgaqPuXPn4uLFizh//rzSeQYNGoSUlBSFzpyvevfdd2Ftba0wImbo0KHQ19fH1q1bIYSApaUlEhMTsWjRIgwfPhxnz57FuHHj4OXlhVWrVgHIfyxRMNPtrVu3MHXqVLRs2VKhf8nHH3+MWrVqwcfHB7l5MvT5dCZuZppDy9QW7RyN0FLjERbM/xKrV6/GZ599hpycHLz33nu4du0aDh06BGvr/zoLm5mZQUenYp7/qyNpRg46LjmNlKxcTO9ZF0Nb1sLp+wlYdjQEqVm50NKQYFzn2pjS3U3+qLCs0rJy8dPxUGy6FAGZyJ+sbV7f+hjeygHxKZmIeJYmn8I/JTMHlx++yA9wwp/h4dM0hWMZ6miiuWNNXAx/rjDUX0MC/DS8GWQyoTjFQGoWnqbk//tFWlaR860U2DGuHTxK6cirrFhpBjyXn0NKVi6+6lcfn3WqXey+4Qkp6Ln8HDQkgN+sbnAwK1+/kpw8GVp/fxJJ6TnY/llblQKoe7HJ6PPLeWhpSHBpbndYlWHSvzMhCRjjGwgdLQ34zeqqVACZkZ2HXj+fQ+SLdIxq64jvBzdWuVwhBKbtuo7912NgZ6KHf6Z2Qk0D9f8OUPb3my07pBZKm3PlVfHx8Th8+DA2b95c4nHbt2+PdevWITQ0FHXr1sWNGzdw4cIF+URqEREReP78OX755ResX79e3sLg5uYmX3cKKL6F4WUFd/lJ6dn4fEcwroTFIv3+LiDtBc4YGiDW3R1bt27FiBEjAORPZHjgwAEAQLNmzRSOdebMmUJD1+k/JvraaO1shtMhCVhxMhQrTobKtzV3rAmfIY3hblMxNz6Gulr4un8DDGpuB+89t3AnJhlz99zCWr+HePwif8I3CQBHMwM8ScpA3ksRiYYEaOpQE51cLdCpriWaOdSEtqYGdgVGFpq7ZHDzWiXWI08mkJiejXuxyfh445VC0w04mVdMq44QAl/tvY2UrFw0c6hZamdfVysjdHKzwPmwZ9h6+TG8+xZuEVXFpQfPkZSeA4saOmjjYlZ6hpfUtzVGKydTBD1OxM7AKEzpUfykn0WRyf5r1fmkvbPSLWX6OppYMrQxRq4PwLaASLzbxE7lwHN30BPsvx4DTQ0JVn7Q/K0IdFTBlh2wZUcdFAzrnjFjBoYNG4bAwEBMnToVa9aswejRowvtv2zZMixZsgQxMTEKQ8JfJZPJMG/ePCxbtgyamprIy8vD999/D29vbwDApUuX0KFDB8TExMDW9r/p6IcPHw6JRIJdu3YpfQ6x0gycD3uGn0+GIiYpE/ramvhxWFP0a1J4mnsqn6JmhQWALzzr4n9dXaFZxCOXipCbJ4PvxUf46XgIMosZ+u5sboCObhbo6GoJjzrmMNEv+pFGrDSjzHOX5AdLt5D30vmPauuIhQMblfvc91+PxtSd16GjqYFDUzqirnXpi4WevBuPz7YEwURfG5e9e5SrNW32XzfwZ9ATfNjOEd8NUr2FZF9wNKbtug5bEz2cn91N6cdgAHDgRgym7AiGka4Wzs3upvLoqnl7b2F7QCSczA1wdGpnpd+HsPgU9F91AZk5MszqVQ9e3VxLz6Qm2LJDbxWZTIZWrVph8eL86fubN2+O27dvFxvsbNy4EaNGjSox0AGAP//8E9u2bcP27dvRsGFDXL9+HdOmTYOdnV2Rxy2rLZceYcGBO/JHE6YG2tj2WTulVxYm1RS1CjUAtHQyq7RABwC0NDUwrnNtWBnpYuqu64W2r3y/GQY0K7mFpkB5+km9PN1A0OMXWH4iFNsCIvEsNQu/vN+8zAtiPkvNwjcH7gAAPu/uqlSgA+RPYuloZoDIF+nYdz0aH5QyIWZxsnNlOPbvWkzKjMIqSp/GNlh4SAex0kycvp8Az4bKLXyakyfDT8fzW3XGda5dpmHk3n3cceZ+Ah4/T8dPx0OUWok9IzsPXtuvITNHhk5uFvhflzoql/s24NBzUgvFzbkSGRlZaN/z588jJCREqSn7Z82ahblz5+L9999H48aN8dFHH2H69Onw8fEB8N+8J/HxiovdlTRjbYHcPBn8Qp9iwpYgfP1SoAPk9ykxNeRw0criYmGIV2Oa17lmT5vaZkWW31rFxy7lUTBs/vPubvhtZAvoaGrg2J14fLzhCqTpZZtYb8GBO0hMz0F9W2NM7Kr8j66mhgQf/zv0uzzD0C+GP4M0IweWRrpo7Vy291JXSxPD/10j64/Lj5XO92dQFB4/T4e5oQ7GdizbPD1GetpY/G9/nY0XI3AtMrHUPAsP3UFofCosauhi+fBmRXYEJwY7pCZUmXNlw4YNaNmyJZo2bVrqcdPT0wuNltHU1IRMlv8IomDek5c7OScnJyMgIKDIGWuFELj1RIqFB++inc9pjN54BcfuxhfaTyaAR8/SS60flU1Vzg9UHcp/Vd/Gttj8aRsY6WrhyqMXGL7WX6n5d1527E4cDt+MhaaGBD+816TUifxeNayVA/S1NXE/LgWXH75QKW+BQ/+OwurbyKZcLXSj2jpCIgHOhz3Do2dppe6fkZ2Hlf8uhzG5uysMdcv+0KSbe/6CuTIBzP7rJrJy84rd98CNGOy4EgWJBPjl/WbyUX5UGB9jkVqYPn062rdvj8WLF2P48OG4cuUK1q1bV2jSs+TkZOzevRs//fRTkcfp0aMHBg8ejMmTJwMA+vfvj++//x6Ojo5o2LAhgoODsXz5cnz66acA8icjmzZtGr777ju4ubnJh57b2dlh0KBB8uNGvUjHgRsx2BscjfCEVHm6qYE2urtbYU9wtEKH0apaGfhtUtVr9lR1+a/yqGOOPyd6YPTGKwiJT8HQ3y9hy9g2cLUq/VGUND0HX+27DQCY0Ll2sZPolcREXxtDWuQP/d586ZHKHXSzcvNw/G7pa2Epw8HMAF3qWuJsyFNsvxKJeaV0mt7s/wjxyVmoVVMfI9uW7RHcy75+twHOhz1FeEIqVp0Ox0zPwjMwP3qWhnl7bgEAJndzRYcKGravrhjskFpQZs4VANi5cyeEEPjggw+KPE5oWDiCQyMRK82ArYk+fv31V8yfPx+TJk1CQkIC7OzsMGHCBHz99dfyPLNnz0ZaWhrGjx+PpKQkNG7ZFlt270OWTBN7r0Ri77VoXHn0352qrpYG3mlgjcHNa6FzXUtoa2qgjYtZtVgZ+G1TFfMDVafyX1Xf1hh7JrXHxxuv4OHTNAxd7Y+Nn7RCS6eSHwl9d/gunqZkoY6locojmF42ur0ztgVE4vjdOEQnZaBWTeXfmwthz5CSmQsrI120cip5CQtlfNTOCWdDnuLPoCjMeKdusf2YpBk5WH32AQBg+jt1oatVvqkKAMDUUAcLBzbCpG3XsPrsA/RpZKvQfy8rNw+Td1xDalYu2jibYWo53vO3BUdjgaOxKN+uwEh477kFmcgf8uszpLFKC+a9nB/Ib53J+/e/l0QCeNQ2x6DmtdC7kQ2Mi5g0rDyja4gq0ou0bHy6KRDXo5Kgq6WBVSNb4J0GRS/66hf6FKM3XoFEAvw10aPUwKg0I9dfxqUHzzGxSx3M7eOudL4Zu65jT3A0PmnvjG8GlL4QbmnyZAKdl51BdFIGfhrWFENb2he53w/H7uO3Mw/gZlUDR6d1rtAO7hP/uIqjd+LQ0M4Y+7w6yB8NfnvwDnwvPoKpgTb+mdrprf6+4GgsIiVl5uTh6O04zP37lryTsEwAc/6+BZ9/7ivV4U8mEwqz0gJAnhCoY2mI4a0cMKCZXalfSNXtLp/eXmaGOtg+ri28tl3DmZCnmPBHEBYPboz3XxkllZqVK3+UMtrDudyBDpA/P82lB8+xMzAS03q6KTUyLDMnDyf+7fvWv2nFTNWgqSHByLaO+OFYCLYGPC4y2ElIycTGC48AAF/0qlfhI/kWDmoI/4fPcScmGevOPYRXN1ccvxMH34v5Zf40vCm/M5TEYIfeOjKZwN3YZJwPe4YL4U8R+CgR2cXMefJqAKOq7wY1rrBZaYleJwMdLaz7uBW899zCX1fzly15mpKFyd1d5SuXLzt6H9FJGXAw08fs3oX7lZRFj/rWsDfVx5PEDBy4HoPhrR1KzXMu9ClSsnJha6KH5g7lf4RVYHgrB/x8MhTBkUm4HS0t1Bfpt9PhyMjJQzOHmvAspuWrPKyM9PD1uw0wc/cN/HwyFJoaEvx2JhwA8FlHF3R3r/gy1RWDHVIrsdIMhen3C8QkZeBC2DOcC3sqX2PoZVZGukhIyVJI05AAW8e2VWqEw9OULHy4IUBh7hZ2MqY3nbamBn54rwmsjXXx25kH+OlEKOJTMjGxSx0cvxOPLf75Q7OXDGkCA52K+TnR1JDgo3ZO8DlyH76XHmFYK3t5cFWcw7f+HYXV2LZCh15bGumidyNbHLwRg20Bj+EzpIl8W9SLdGy/kj+1xeze9UqtY1kNaVELa889QGh8KpYcuQ8AsDfVx+zeyj/iIwY7pEZe7XPzYTsnSIBi1xjyqGOOjq4W6OhmiTqWhvgzKKpQJ2Fl19VxszaCz5DG7GRMakcikWBWL3dY1tDFt4fuYuvlSGy9/N/8Va2dTSt8JNCI1g5YcTIU92KTEfgoscRlHzJz8nDy30dYlTHb+IdtHXHwRgz2BcfAu299eX+7FSdCkZMn0MnNosyLhiojLjlTYQQnkH/z9jwti98vKmCwQ2ohVpqh0DlYJiC/6wSKX2PoZeUdClzdhhITVaRPOrhAU1OC+fvuKKRffZwoH71YUWoa6GBw81rYcSUKmy89KjHYORvyFGnZeahVUx/NHWpWWB0KtHExQ13rGgiNT8Xea9EY3d4Z9+OSsfd6NABgVq+KeXxXnKJm+y6Yh4vfMcpjsENqobjp/3vWt8J7LR1KXGPoZeXtJMxOxqTO6ljWKJRWWT+8o9s7Y8eVKBy9E4eYpAzYFTMM/b9HWDaV8ihJIpHgw3ZO+Hr/HWy9/Bgfezjhx2OhECK/zCb2NSu8zJcVzPbNR+TlwxmUSS24WBji1a85TYkEiwY1Qu9GNkoFOkRUste5zIa7jTHa1TZDnkxgW0DRyzZkZOfh1L3yrYWljMHNa8FARxNhCalY7fcAJ+/FQ0MCzHinclt1gOo32/abisEOqYVnKdl4OdrhFwJRxXvdP7yftHcGAOy4EoXMnMLLJpwNSUB6dh7sTfXRxF71WZuVZaSnjUHN8xdoXXY0f1maYS0d4GpVuKWrMoxo7YgLc7thx7h2uDC3m0rzf1E+PsaiClfciKjKkp0rw6y/bkAIoIe7FT7rVJt9Zogqyevsm9azvjXsTPQQI83EwRsxGNZKcRh6wVpY/ZrYVtpoqAKWNRRHZda2NKzU8l7FR+Tlw5YdNRQdHY0PP/wQ5ubm0NfXR+PGjREUFAQAyMnJwZw5c9C4cWMYGhrCzs4OH3/8MWJiYko85rlz59C/f3/Y2dlBIpFg3759CtsLjuvo6o5alqbo3Kwe6nboh9X/BFbWacqt8XuA+3EpMDXQxrL3msCjjjm/FIgqUcGK6ZX9/0xLUwMfeTgDADa9shp6enYuTt3/9xFW48p7hAXk38D9ejpMIW3Z0RCVF0ulqsNgR80kJiaiQ4cO0NbWxpEjR3D37l389NNPMDXNn2grPT0d165dw/z583Ht2jXs2bMHISEhGDBgQInHTUtLQ9OmTfHbb78VuT09PR3+V4KQ2WgQbEf/AstB85D9IhrTxo6s1C+E0PgU+ZfQNwMawrwGV/0lUifvt3aArpYG7sQk4+rjRHn66fsJyMyRwdHMAI1qVe4yP0UNgMgTAo+epVdquVRx+BhLzSxduhQODg7w9fWVp7m4uMj/bWJighMnTijkWbVqFdq0aYPIyEg4Ohb9LLhPnz7o06dPseWamJjggwVrsfTf59kAYPbORMRtmYHLN0MxuFPTsp5SsfJkArP+uomcPIGe9a0woGnl3t0R0etnaqiDgc3s8GfQE2y69AitnPOHoR9+jY+wOCLqzceWHTVz4MABtGrVCsOGDYOVlRWaN2+O9evXl5hHKpVCIpGgZs2aZSozKzcPCw/eVQh0AECWlQ5AAoMalXPXtfFCBG5EJcFIVwvfDWpc6V94RFQ1Rv/bUfno7TjESTORlpWL0/cTAADvVsJEgq/iiKg3H1t21MzDhw+xevVqzJgxA/PmzUNgYCCmTJkCHR0djB49utD+mZmZmDNnDj744IMyrfgenpCCz3dcx73YZABA+zrmuPzwOfJyspF01hcGDTpj4bEItHC1U2rZBWU9epaGH4/nB1df9qsPGxO9Cjs2EVUvDe1M0MbZDFcevcC2gMdwszZCVq4MLhaGaGBbuY+wCnDS0Dcbgx01I5PJ0KpVKyxevBgA0Lx5c9y+fRtr1qwpFOzk5ORg+PDhEEJg9erVKpUjhMD2gEgsPHQHmTkymBvq4IdhTdDd3RqRz5IxasRwGFoawnzELEQlZeCzLUHYNb6dUisYl36OAnP+vomsXBk6uJpjhBILBRLRm210e2dcefQC2wMi5cPM+zWu/EdYL+OIqDcXH2OpGVtbWzRo0EAhrX79+oiMjFRIKwh0Hj9+jBMnTqjcqrPG7yHm7b2FzBwZOrlZ4MjUTujubo2cnBxMHTcayc9icf7saWyZ2BU1DbRxIyoJM/68DllR0xyraPuVSAREvIC+tiaWDGnCx1dEbwHPhtawNdHD87RsnAl5CqBy1sIi9cRgR8106NABISGKfWdCQ0Ph5OQk/7sg0AkLC8PJkydhbm6u9PH9HzwHAARHJkJbU4Iv+9bH5jFtYGWsh5ycHPTv3x9nzpxBVFQU7O3tMaBbO0xrogFtTQn+uRWHZcfu4+uvv4atrS309fXRs2dPhIWFlVimj48PWrduDSMjI1hYWmHS6A+Q8/wJZveuBwez/A6CEyZMQJ06daCvrw9LS0sMHDgQ9+/fV/q8iKh609bUKDRx4I2opKqpDL1xGOyomenTp+Py5ctYvHgxwsPDsX37dqxbtw5eXl4A8gOd9957D0FBQdi2bRvy8vIQFxeHuLg4ZGdny4/To0cPrFq1Sv53ojQZ03/fh6HfbwcAmOQmwqeLEXo5a0FDQ4KcnBwMGDAAJ0+eRJcuXbBt2zb4+fnhyy+/RNt6tbBkSBMAwNKly/DTil+wZs0aBAQEwNDQEL169UJmZmax5+Tn5wcvLy/4+/vDY/Jy5ObmIHHPArzXxFK+T8uWLeHr64t79+7h2LFjEELA09MTeXmFZ10lojdPrDQDJ/5d3bzAl3tvc64bUo4gIZVKBQAhlUqruioV4uDBg6JRo0ZCV1dXuLu7i3Xr1sm3RURECABFvs6cOSPfz8nJSSxYsEAIIcTjZ2miw5SVReYZPXq00sf98eg9oWloKsy7fyouhj0VQgiRlJQkdHV1xY4dO0o9r7+CooTTnEPCZdoOAUD4+fkVu++NGzcEABEeHq7am0dE1dLF8KfCac6hQq9L4c+qumpUhZT9/WYHZTX07rvv4t133y1ym7Ozs8IspMXxv3EPEc/SsOliBH48HopU/dpovOAofIY0KfI5ubOzM+rXr49evXrhyZMn8PPzQ61atTBp0iR07doVADDIVRtfpCVCx7EZJm69ij2TOsDVygRt27aFv78/3n///WLrk5CSiYWH7gIAPm5liW8BmJmZFblvWloafH194eLiAgcHdl4mUgec64bKg4+xqJBdgZHosOQ0Rq4PwDcH7yI1KxetnU1xZFrnEjsEFgx7d3Nzw7Fjx/C///0PU6ZMwebNmwEA8fH5TdDN6jkhOTMXYzZdwfPULFhbWyMuLq7EOn297w6kGTloaFsDgduXo0OHDmjUqJHCPr///jtq1KiBGjVq4MiRIzhx4gR0dHTK+W4QUXXAuW6oPCRCmdv8l5w5cwbdunWrrPpUieTkZJiYmEAqlZZprhl1IU3PweFbMZi397ZCugTAudld4WBW8sJ3Ojo6aNWqFS5duiRPmzJlCgIDA+Hv749Lly6hQ4cOuB0Wgf/tiUDki3S0dDIFTq2AlqYGdu3aVeRx/7kVi0nbrkFLQ4JWT/7G5XOncOHCBdjb2yvWXypFQkICYmNj8eOPPyI6OhoXL16Enh7n4CFSF7HSDM51Q3LK/n6r/Bird+/esLe3x5gxYzB69Gg+JniDZefKcC0yERfCnuF8+DPcepJUaP0XIL/jzZPEzFKDneKGvf/9998AABsbGwBATmoSNn7SGoN/v4irjxORc+chhr7TochjJqZl4+v9+cGX9e1tuBh8DufOnSsU6AD5S1aYmJjAzc0N7dq1g6mpKfbu3YsPPvigtLeCiN4QnOuGykLlx1jR0dGYPHky/vrrL9SuXRu9evXCn3/+qTCSh6perDQDlx48UxipIIRAaHwKNlyIwBjfK2i28DjeX3cZq86E40ZUfqDjbG6AV2etUfa5eGnD3l1cXGBjY4NTp07B1aoG1n7YEho5GYgJu4XEGs5FHnPhobt4mpKFvAv/h0dXz+L06dMKa30VRwgBIQSysrJK3ZeIiNSbyi07FhYWmD59OqZPn45r167B19cXkyZNwqRJkzBy5EiMHTsWTZtW/KKPpLxdgZHw3nMLMgFoSIDhrRyQkydwIfwp4pMVf/wtauigg6sFOrpaoKObBWxN9LErMBLz9txGnhAqPRefPn062rdvj8WLF2P48OG4cuUK1q1bh3Xr1gEAJBIJpk2bhu+++w5ubm5wcXGBedB6RNUww7lsF/x99QmGtrRHjx49MHjwYDTsORx7g6OReGI1xIMLOHTgAIyMjOT9e0xMTKCvr4+HDx9i165d8PT0hKWlJZ48eYIlS5ZAX18fffv2rfg3mIiI3igq99l5VUxMDNatW4clS5ZAS0sLmZmZ8PDwwJo1a9CwYcOKqmelUqc+O7HSDHRYcrrIx1EAoKulgTYuZujoaoFObpZwtzGChkbhGYjL+lz80KFD8Pb2RlhYGFxcXDBjxgyMGzdOvl0IgQULFmDdunVISkpCx44d0ez9mfgrXAZtTQm2jm2LEd1b4oMPP8JZw66IlWbi8dKiR5b5+vrik08+QUxMDD777DNcvXoViYmJsLa2RufOnfH111+jXr16StediIjeLMr+fpcp2MnJycH+/fuxceNGnDhxAq1atcLYsWPxwQcf4OnTp/jqq69w7do13L17t1wnAeQ/NpszZw6OHDmC9PR0uLq6wtfXF61atQLw34/n+vXrkZSUhA4dOshHBClLnYKdSw+eYeT6gELp/ZvaYkQrR7RyNq2Q9akqkkwmMHnHNfxzKw41DbSx7qOWWOP3EKfvJ8DJ3ABHp3aGvk71qjMREVW9Suug/Pnnn2PHjh0QQuCjjz7CsmXLFIYAGxoa4scff4SdnV3Zav6SxMREdOjQAd26dcORI0dgaWmJsLAwmJqayvdZtmwZVq5cic2bN8PFxQXz589Hr169cPfu3bdyFE5xc1HM61u/2nbq09CQYPnwZohOuowbUUkYvvayfFuvBjYMdIiIqFxUbtnp0aMHPvvsMwwZMgS6urpF7pObm4uLFy+iS5cu5arc3LlzcfHiRZw/f77I7UII2NnZYebMmfjiiy8A5A8/tra2xqZNm0qcpO5l6tSyAwDTdgZj3/UYAP/NRTGitWMV16p0t6OlePfXCwppmhIJLsztVm0DNSIiqjrK/n6rPBrr1KlT+OCDD4oNdABAS0ur3IEOABw4cACtWrXCsGHDYGVlhebNm2P9+vXy7REREYiLi0PPnj3laSYm/83IW5ysrCwkJycrvNRJSmYuAGBEawdcmNvtjQh0ACA5M6dQWp4QePQsvQpqQ0RE6kLlYMfHxwcbN24slL5x40YsXbq0QipVoLQZeQtG5VhbWyvkK21GXh8fH/mcLCYmJmo1V1BWbh4u/bsy+UftnN6oFpGCR3Av43TwRERUXioHO2vXroW7u3uh9IYNG2LNmjUVUqkCMpkMLVq0wOLFi9G8eXOMHz8e48aNK3c53t7ekEql8ldUVFQF1bjqBUYkIiMnD5ZGumho92Y9kuN08EREVBlU7qAcFxcHW9vC6yNZWloiNja2QipVQNkZeePj4xXqFB8fj2bNmhV7XF1d3RIfw73JzoYkAAC61LWERFJ4SHl1N6K1IzrXteR08EREVGFUbtlxcHDAxYsXC6VfvHixQkZgvUyVGXkLJCcnIyAgAB4eHhValzfF2dCnAICu9SyruCZlZ2uiD4865gx0iIioQqjcsjNu3DhMmzYNOTk56N69O4D8TsuzZ8/GzJkzK7RyZZmRd/78+bCzs8OgQYMqtC5vgieJ6QhPSIWGBOjk+uYGO0RERBVJ5WBn1qxZeP78OSZNmiRfD0tPTw9z5syBt7d3hVaudevW2Lt3L7y9vbFw4UK4uLjg559/xqhRo+T7zJ49G2lpaRg/frx8Rt6jR4++lXPsnA3Jb9Vp4WgKEwPtKq4NERFR9VDm5SJSU1Nx79496Ovrw83N7Y3uA6Mu8+x8tjkIJ+/F4wvPupjcXfkZpImIiN5ElTaDcoEaNWqgdevWZc1OFSx/yPkzAEDXelZVXBsiIqLqo0zBTlBQEP78809ERkbKH2UV2LNnT4VUjFQT9CgR6dl5sKihiwa2b27rFBERUUVTeTTWzp070b59e9y7dw979+5FTk4O7ty5g9OnT8PExKQy6khKeHnIeVGrmBMREb2tVA52Fi9ejBUrVuDgwYPQ0dHBL7/8gvv372P48OFwdHwzliVQRwWdk7u8wUPOiYiIKoPKwc6DBw/Qr18/AICOjg7S0tIgkUgwffp0+ZBwer2ikzIQ9u+Q885uFlVdHSIiompF5WDH1NQUKSkpAIBatWrh9u3bAICkpCSkp3PBxqrg92+rTjOHmqhpoFPFtSEiIqpeVO6g3LlzZ5w4cQKNGzfGsGHDMHXqVJw+fRonTpxAjx49KqOOVIqC/jochUVERFSYysHOqlWrkJmZCQD48ssvoa2tjUuXLmHo0KH46quvKryCVLLsXBkuhhcMOWd/HSIiolepFOzk5ubi0KFD6NWrFwBAQ0MDc+fOrZSKkXKCHr9AWnYeLGrooJEdR8MRERG9SqU+O1paWpg4caK8ZYeqXkF/nc5uHHJORERUFJU7KLdp0wbXr1+vhKpQWXDIORERUclU7rMzadIkzJgxA1FRUWjZsiUMDQ0Vtjdp0qTCKkcli0nKQEh8yr9DzhnsEBERFUXlYOf9998HAEyZMkWeJpFIIISARCJBXl5exdWOSuQXmt+q09ShJkwNOeSciIioKCoHOxEREZVRDyqDl5eIICIioqKpHOw4OTlVRj1IRTl5MlwMfw6A8+sQERGVROVgZ8uWLSVu//jjj8tcGVLe1ceJSM3KhZmhDprU4pBzIiKi4qgc7EydOlXh75ycHKSnp0NHRwcGBgYMdl6Ts/Ih5xYcck5ERFQClYeeJyYmKrxSU1MREhKCjh07YseOHZVRRyoCl4ggIiJSjsrBTlHc3NywZMmSQq0+VDnipJm4H5cCiQTozM7JREREJaqQYAfIn105Jiamog5HJfALzW/VaWJfE2Ycck5ERFQilfvsHDhwQOFvIQRiY2OxatUqdOjQocIqRsUr6K/Tla06REREpVI52Bk0aJDC3xKJBJaWlujevTt++umniqoXFSMnT4YLYVzlnIiISFkqBzsymawy6kFKuvY4ESlZuTA10EYT+5pVXR0iIqJqr8L67NDrcfbfJSI6uVlCk0POiYiISqVysDN06FAsXbq0UPqyZcswbNiwCqkUFc+voL8OH2EREREpReVg59y5c+jbt2+h9D59+uDcuXMVUikqWkJyJu7GJgPgkHMiIiJlqRzspKamQken8HBnbW1tJCcnV0ilqGgFj7Ca2JvAooZuFdeGiIjozaBysNO4cWPs2rWrUPrOnTvRoEGDCqkUFc2PQ86JiIhUpvJorPnz52PIkCF48OABunfvDgA4deoUduzYgd27d1d4BSlfbp4M58Pyg50uXCKCiIhIaSoHO/3798e+ffuwePFi/PXXX9DX10eTJk1w8uRJdOnSpTLqSACCo5KQnJmLmgbaaOZQs6qrQ0RE9MZQOdgBgH79+qFfv34VXRcqQcHCnxxyTkREpBqV++wEBgYiICCgUHpAQACCgoIqpFLqZsmSJZBIJJg2bZpCur+/P7p37w5DQ0MYGxujc+fOyMjIKPIYZ0OeQur/J4589wmMjIxgZWWFQYMGISQkRGG/devWoWvXrjA2NoZEIkFSUlIlnRUREdGbQeVgx8vLC1FRUYXSo6Oj4eXlVSGVUieBgYFYu3YtmjRpopDu7++P3r17w9PTE1euXEFgYCAmT54MDY3CH0lCSibuxCQjM+o2pn4+GZcvX8aJEyeQk5MDT09PpKWlyfdNT09H7969MW/evEo/NyIiojeByo+x7t69ixYtWhRKb968Oe7evVshlVIXqampGDVqFNavX4/vvvtOYdv06dMxZcoUzJ07V55Wr169Io9TMAqrx/Sf8fnETvL0TZs2wcrKClevXkXnzp0BQN56dPbs2Qo8EyIiojeXyi07urq6iI+PL5QeGxsLLa0ydQFSW15eXujXrx969uypkJ6QkICAgABYWVmhffv2sLa2RpcuXXDhwoUij1Mwv07XuoqjsKRSKQDAzMysEmpPRESkHlQOdjw9PeHt7S3/oQWApKQkzJs3D++8806FVu5NtnPnTly7dg0+Pj6Ftj18+BAA8M0332DcuHE4evQoWrRogR49eiAsLExh39xiVjmXyWSYNm0aOnTogEaNGlXimRAREb3ZVG6K+fHHH9G5c2c4OTmhefPmAIDr16/D2toaf/zxR4VX8E0UFRWFqVOn4sSJE9DT0yu0vWDl+AkTJmDMmDEA8h8Dnjp1Chs3blQIkG48SYI0IwfGeloKQ869vLxw+/btYluDiIiIKJ/KwU6tWrVw8+ZNbNu2DTdu3IC+vj7GjBmDDz74ANra2pVRxzfO1atXkZCQoNC3KS8vD+fOncOqVavkI6henXG6fv36iIyMVEg7+29/nU51LaGlmd8QN3nyZBw6dAjnzp2Dvb19ZZ4KERHRG69MnWwMDQ0xfvz4iq6L2ujRowdu3bqlkDZmzBi4u7tjzpw5qF27Nuzs7AoNGw8NDUWfPn0U0s6+tESEEAKff/459u7di7Nnz8LFxaVyT4SIiEgNlLlH8d27dxEZGYns7GyF9AEDBpS7Um86IyOjQv1oDA0NYW5uLk+fNWsWFixYgKZNm6JZs2bYvHkz7t+/j7/++kuep3PXbritXQ/GLfujSz1LeHl5Yfv27di/fz+MjIwQFxcHADAxMYG+vj4AIC4uDnFxcQgPDwcA3Lp1C0ZGRnB0dGRHZiIieiupHOw8fPgQgwcPxq1btyCRSCCEAABIJPmz+ubl5VVsDdXUtGnTkJmZienTp+PFixdo2rQpTpw4gTp16sj3uR8aDlltWzS0M4aVkR5Wr14NAOjatavCsXx9ffHJJ58AANasWYNvv/1Wvq1gSPrL+xAREb1NJKIgWlFS//79oampif/7v/+Di4sLrly5gufPn2PmzJn48ccf0alTp9IPUs0kJyfDxMQEUqkUxsbGVV0duc82B+HkvXiM9nDCtwM54oqIiOhlyv5+qzz03N/fHwsXLoSFhQU0NDSgoaGBjh07wsfHB1OmTClXpek/O65E4uS9/PmMtlx+jF2BkaXkICIioqKoHOzk5eXByMgIAGBhYYGYmBgAgJOTU6EOt1Q2sdIMzNv7XwdnIYB5e24jVlr0ullERERUPJX77DRq1Ag3btyAi4sL2rZti2XLlkFHRwfr1q1D7dq1K6OOb52IZ2l49eFinhB49Cwdtib6VVMpIiKiN5TKwc5XX30lX3hy4cKFePfdd9GpUyeYm5tj165dFV7Bt5Hmv529X01ztjCogtoQERG92VQOdnr16iX/t6urK+7fv48XL17A1NRUPiKLymff9RiFvzUlEiwe0oitOkRERGVQISt3cv6WipOQkom/rz0BAKwe1QI1DXTgbGHAQIeIiKiMuEx5NbPp4iNk58rQwrEmejeyYWsZERFROak8GosqT2pWLv64/BgAMKFLHQY6REREFYDBTjWyIyASKZm5qG1piHfqW1d1dYiIiNSCysHOuXPnkJubWyg9NzcX586dq5BKvY2yc2XYcCECADChc21oaLBVh4iIqCKoHOx069YNL168KJQulUrRrVu3CqnU22j/9WjEJWfCykgXg5rXqurqEBERqQ2Vgx0hRJF9SZ4/fw5DQ8MKqdTbRiYTWHfuIQDg044u0NXSrOIaERERqQ+lR2MNGTIEQP7q5p988gl0dXXl2/Ly8nDz5k20b9++4mv4Fjh9PwFhCakw0tXCyLaOVV0dIiIitaJ0sGNiYgIgv2XHyMgI+vr/zfuio6ODdu3aYdy4cRVfw7fA2nMPAAAj2znCWE+7imtDRESkXpQOdnx9fQEAzs7O+OKLL/jIqoJcffwCgY8SoaOpgU87uFR1dYiIiNSOyn12Zs+erdBn5/Hjx/j5559x/PjxCq3Y22KNX35fncHNa8HaWK+Ka0NERKR+VA52Bg4ciC1btgAAkpKS0KZNG/z0008YOHAgVq9eXeEVVGfhCak4cTceEgkwrjNXjCciIqoMKgc7165dQ6dOnQAAf/31F2xsbPD48WNs2bIFK1eurPAKqrN1//bVeae+NVytalRxbYiIiNSTysFOeno6jIyMAADHjx/HkCFDoKGhgXbt2uHx48cVXkF1FZ+cib3B0QDyl4YgIiKiyqFysOPq6op9+/YhKioKx44dg6enJwAgISEBxsbGFV5BdbXxQgRy8gTaOJuhpZNpVVeHiIhIbakc7Hz99df44osv4OzsjDZt2sDDwwNAfitP8+bNK7yC6ig5MwfbAiIBABO6sK8OERFRZVI52HnvvfcQGRmJoKAgHDt2TJ7eo0cPrFixokIr96olS5ZAIpFg2rRp8rTMzEx4eXnB3NwcNWrUwNChQxEfH1+p9SivbZcjkZqVCzerGuhWz6qqq0NERKTWyrTquY2NDYyMjHDixAlkZGQAAFq3bg13d/cKrdzLAgMDsXbtWjRp0kQhffr06Th48CB2794NPz8/xMTEyGd7ro6ycvOw8eK/C352qcMFP4mIiCqZysHO8+fP0aNHD9StWxd9+/ZFbGwsAGDs2LGYOXNmhVcQAFJTUzFq1CisX78epqb/9W+RSqXYsGEDli9fju7du6Nly5bw9fXFpUuXcPny5UqpS3ntC47G05Qs2JroYUBTu6quDhERkdpTOdiZPn06tLW1ERkZCQMDA3n6iBEjcPTo0QqtXAEvLy/069cPPXv2VEi/evUqcnJyFNLd3d3h6OgIf3//Yo+XlZWF5ORkhdfrIJMJrP13wc+xHV2go1WmhjUiIiJSgdLLRRQ4fvw4jh07Bnt7e4V0Nze3Shl6vnPnTly7dg2BgYGFtsXFxUFHRwc1a9ZUSLe2tkZcXFyxx/Tx8cG3335b0VUt1Yl78Xj4NA3Gelp4vw0X/CQiInodVG5aSEtLU2jRKfDixQuFldArQlRUFKZOnYpt27ZBT6/illLw9vaGVCqVv6Kioirs2MURQmCNX/4kgh95OKGGrspxJhEREZWBysFOp06d5MtFAIBEIoFMJsOyZcvQrVu3Cq3c1atXkZCQgBYtWkBLSwtaWlrw8/PDypUroaWlBWtra2RnZyMpKUkhX3x8PGxsbIo9rq6uLoyNjRVelS3wUSKCI5Ogo6WBT9pzwU8iIqLXReXmhWXLlqFHjx4ICgpCdnY2Zs+ejTt37uDFixe4ePFihVauR48euHXrlkLamDFj4O7ujjlz5sDBwQHa2to4deoUhg4dCgAICQlBZGSkfP6f6qKgVee9lvawNKrYFjAiIiIqnsrBTqNGjRAaGopVq1bByMgIqampGDJkCLy8vGBra1uhlTMyMkKjRo0U0gwNDWFubi5PHzt2LGbMmAEzMzMYGxvj888/h4eHB9q1a1ehdSmPkLgUnL6fkL/gZydOIkhERPQ6qRzsREZGwsHBAV9++WWR2xwdX2/H2xUrVkBDQwNDhw5FVlYWevXqhd9///211qE0a/9d8LN3Qxu4WBhWcW2IiIjeLhIhhFAlg6amJmJjY2FlpTjz7/Pnz2FlZYW8vLwKreDrkJycDBMTE0il0grvvxOTlIHOy84gVyaw36sDmjrUrNDjExERva2U/f1WuYOyEAISSeFZf1NTUyt0xJS62HghArkygXa1zRjoEBERVQGlH2PNmDEDQP7oq/nz5ysMP8/Ly0NAQACaNWtW4RV8k4XGpWDr5fy5hyZ2qVPFtSEiIno7KR3sBAcHA8hv2bl16xZ0dHTk23R0dNC0aVN88cUXFV/DN9SuwEjM/fsWCp4Rxkkzq7Q+REREbyulg50zZ84AyB/6/csvv7yWuWneVLHSDHjv+S/QAYAv995Gl3qWsDXRr7J6ERERvY1U7rPj6+vLQKcUEc/SIHul23eeEHj0LL1qKkRERPQW40qUlcDFwhAar/Th1pRI4GxReJkNIiIiqlwMdiqBrYk+fIY0hua/o9Y0JRIsHtKIj7CIiIiqAFejrCQjWjuic11LPHqWDmcLAwY6REREVYTBTiWyNdFnkENERFTF+BiLiIiI1BqDHSIiIlJrDHaIiIhIrTHYISIiIrXGYIeIiIjUGoMdIiIiUmsMdoiIiEitMdghIiIitcZgh4iIiNQagx0iIiJSawx2iIiISK0x2CEiIiK1xmCHiIiI1BqDHSIiIlJrDHaIiIhIrTHYISIiIrXGYIeIiIjUGoMdIiIiUmsMdoiIiEitMdghIiIitcZgh4iIiNQagx0iIiJSawx2iIiISK0x2CEiIiK1xmCHiIiI1BqDHSIiIlJrDHaIiIhIrTHYISIiIrXGYIeIiIjUGoMdIiIiUmsMdoiIiEitMdghIiIitcZgh4iIiNQagx0iIiJSawx2iIiISK0x2CEiIiK1xmCHiIiI1BqDHSIiIlJrDHaIiIhIrTHYISIiIrXGYIeIiIjUGoMdIiIiUmsMdoiIiEitMdghIiIitcZgh4iIiNQagx0iIiJSawx2iIiISK0x2CEiIiK1xmCHiIiI1BqDHSIiIlJrDHaIiIhIrTHYISIiIrXGYIeIiIjUGoMdIiIiUmvVOtjx8fFB69atYWRkBCsrKwwaNAghISEK+2RmZsLLywvm5uaoUaMGhg4divj4+CqqMREREVU31TrY8fPzg5eXFy5fvowTJ04gJycHnp6eSEtLk+8zffp0HDx4ELt374afnx9iYmIwZMiQKqw1ERERVScSIYSo6koo6+nTp7CysoKfnx86d+4MqVQKS0tLbN++He+99x4A4P79+6hfvz78/f3Rrl07pY6bnJwMExMTSKVSGBsbV+YpEBERUQVR9ve7WrfsvEoqlQIAzMzMAABXr15FTk4OevbsKd/H3d0djo6O8Pf3L/Y4WVlZSE5OVngRERGRenpjgh2ZTIZp06ahQ4cOaNSoEQAgLi4OOjo6qFmzpsK+1tbWiIuLK/ZYPj4+MDExkb8cHBwqs+pERERUhd6YYMfLywu3b9/Gzp07y30sb29vSKVS+SsqKqoCakhERETVkVZVV0AZkydPxqFDh3Du3DnY29vL021sbJCdnY2kpCSF1p34+HjY2NgUezxdXV3o6upWZpWJiIiomqjWLTtCCEyePBl79+7F6dOn4eLiorC9ZcuW0NbWxqlTp+RpISEhiIyMhIeHx+uuLhEREVVD1bplx8vLC9u3b8f+/fthZGQk74djYmICfX19mJiYYOzYsZgxYwbMzMxgbGyMzz//HB4eHkqPxCIiIiL1Vq2HnkskkiLTfX198cknnwDIn1Rw5syZ2LFjB7KystCrVy/8/vvvJT7GehWHnhMREb15lP39rtbBzuvCYIeIiOjNo5bz7BARERGpisEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BAREZFaY7BDREREao3BDhEREak1BjtERESk1hjsEBERkVpjsENERERqjcEOERERqTUGO0RERKTW1CbY+e233+Ds7Aw9PT20bdsWV65cqeoqERERUTWgFsHOrl27MGPGDCxYsADXrl1D06ZN0atXLyQkJFR11YiIiKiKqUWws3z5cowbNw5jxoxBgwYNsGbNGhgYGGDjxo1VXTUiIiKqYlpVXYHyys7OxtWrV+Ht7S1P09DQQM+ePeHv719knqysLGRlZcn/lkqlAIDk5OTKrSwRERFVmILfbSFEifu98cHOs2fPkJeXB2tra4V0a2tr3L9/v8g8Pj4++PbbbwulOzg4VEodiYiIqPKkpKTAxMSk2O1vfLBTFt7e3pgxY4b8b5lMhhcvXsDc3BwSiaTCyklOToaDgwOioqJgbGz8WvOz7Ndfdnnzs+y3q+zy5mfZLPtNyV/esksihEBKSgrs7OxK3O+ND3YsLCygqamJ+Ph4hfT4+HjY2NgUmUdXVxe6uroKaTVr1qysKsLY2LhcH3B58rPs1192efOz7Ler7PLmZ9ks+03JX96yi1NSi06BN76Dso6ODlq2bIlTp07J02QyGU6dOgUPD48qrBkRERFVB298yw4AzJgxA6NHj0arVq3Qpk0b/Pzzz0hLS8OYMWOqumpERERUxdQi2BkxYgSePn2Kr7/+GnFxcWjWrBmOHj1aqNPy66arq4sFCxYUemT2OvKz7Ndfdnnzs+y3q+zy5mfZLPtNyV/esiuCRJQ2XouIiIjoDfbG99khIiIiKgmDHSIiIlJrDHaIiIhIrTHYISIiIrXGYKcS/fbbb3B2doaenh7atm2LK1euKJXv3Llz6N+/P+zs7CCRSLBv3z6ly/Tx8UHr1q1hZGQEKysrDBo0CCEhIUrnX716NZo0aSKf/MnDwwNHjhxROv/LlixZAolEgmnTpim1/zfffAOJRKLwcnd3V7q86OhofPjhhzA3N4e+vj4aN26MoKAgpfI6OzsXKlsikcDLy6vUvHl5eZg/fz5cXFygr6+POnXqYNGiRaWu1fKylJQUTJs2DU5OTtDX10f79u0RGBhYaL/Srg0hBL7++mvY2tpCX18fPXv2RFhYmNL59+zZA09PT/ls4tevX1e6/JycHMyZMweNGzeGoaEh7Ozs8PHHHyMmJkapsr/55hu4u7vD0NAQpqam6NmzJwICApSu+8smTpwIiUSCn3/+Wam8n3zySaHPvnfv3iqVfe/ePQwYMAAmJiYwNDRE69atERkZWWreoq47iUSCH374QamyU1NTMXnyZNjb20NfX1++GLIyeePj4/HJJ5/Azs4OBgYG6N27t/x6Uea7JDMzE15eXjA3N0eNGjUwdOhQ+QSvyuRft24dunbtCmNjY0gkEiQlJcm3lZb/xYsX+Pzzz1GvXj3o6+vD0dERU6ZMgVQqVarsCRMmoE6dOtDX14elpSUGDhwoX2JIle9RIQT69Okjf3+Vydu1a9dCn/fEiRNVKtvf3x/du3eHoaEhjI2N0blzZyxcuLDEvI8ePSr2etu9e7dSZcfFxeGjjz6CjY0NDA0N0aJFC/z9999K5X3w4AEGDx4MS0tLGBsbY/jw4YUmBK4sDHYqya5duzBjxgwsWLAA165dQ9OmTdGrVy8kJCSUmjctLQ1NmzbFb7/9pnK5fn5+8PLywuXLl3HixAnk5OTA09MTaWlpSuW3t7fHkiVLcPXqVQQFBaF79+4YOHAg7ty5o1I9AgMDsXbtWjRp0kSlfA0bNkRsbKz8deHCBaXyJSYmokOHDtDW1saRI0dw9+5d/PTTTzA1NVW6vi+Xe+LECQDAsGHDSs27dOlSrF69GqtWrcK9e/ewdOlSLFu2DL/++qtSZQPAZ599hhMnTuCPP/7ArVu34OnpiZ49eyI6Olphv9KujWXLlmHlypVYs2YNAgICYGhoiF69eiEzM1Op/GlpaejYsSOWLl1a7Pbi8qenp+PatWuYP38+rl27hj179iAkJAQDBgxQquy6deti1apVuHXrFi5cuABnZ2d4enri6dOnSuUvsHfvXly+fFlh+nhl8vbu3VvhGtixY4fS+R88eICOHTvC3d0dZ8+exc2bNzF//nzo6emVmvflMmNjY7Fx40ZIJBIMHTpUqbJnzJiBo0ePYuvWrbh37x6mTZuGyZMn48CBAyXmFUJg0KBBePjwIfbv34/g4GA4OTmhZ8+eSEtLU+q7ZPr06Th48CB2794NPz8/xMTEYMiQIQCU+y5KT09H7969MW/evEL1Ky1/TEwMYmJi8OOPP+L27dvYtGkTjh49irFjxypVdsuWLeHr64t79+7h2LFjEELA09MTeXl5Kn2P/vzzzwrLDCmbd9y4cQqf+7Jly5TO7+/vj969e8PT0xNXrlxBYGAgJk+ejAsXLpSY18HBodD19u2336JGjRro06ePUmV//PHHCAkJwYEDB3Dr1i0MGTIEw4cPx8GDB0vMm5aWBk9PT0gkEpw+fRoXL15EdnY2+vfvD5lMVuh9rXCCKkWbNm2El5eX/O+8vDxhZ2cnfHx8VDoOALF3794y1yMhIUEAEH5+fmU+hqmpqfi///s/pfdPSUkRbm5u4sSJE6JLly5i6tSpSuVbsGCBaNq0aZnqOGfOHNGxY8cy5S3K1KlTRZ06dYRMJit13379+olPP/1UIW3IkCFi1KhRSpWVnp4uNDU1xaFDhxTSW7RoIb788sti8716bchkMmFjYyN++OEHeVpSUpLQ1dUVO3bsKDX/yyIiIgQAERwcrHT5Rbly5YoAIB4/fqxyXqlUKgCIkydPKl32kydPRK1atcTt27eFk5OTWLFihVJ5R48eLQYOHFhifUrKP2LECPHhhx+WKe+rBg4cKLp37650/oYNG4qFCxcqpBV17byaNyQkRAAQt2/flqfl5eUJS0tLsX79+kJlv/pdkpSUJLS1tcXu3bvl+9y7d08AEP7+/qXmf9mZM2cEAJGYmFjkeZeWv8Cff/4pdHR0RE5Ojsp5b9y4IQCI8PBwpcsODg4WtWrVErGxscV+tkXlVeV7saj8bdu2FV999VWZ8r6qWbNmhb6/SspvaGgotmzZorCfmZlZoWvm1bzHjh0TGhoaQiqVyvdJSkoSEolEnDhxotRzKS+27FSC7OxsXL16FT179pSnaWhooGfPnvD393+tdZFKpQAAMzMzlfPm5eVh586dSEtLU2npDS8vL/Tr10/h/JUVFhYGOzs71K5dG6NGjUJkZKRS+Q4cOIBWrVph2LBhsLKyQvPmzbF+/XqVywfyP7+tW7fi008/VWph2Pbt2+PUqVMIDQ0FANy4cQMXLlxAnz59lCovNzcXeXl50NPTU0jX19dXumULACIiIhAXF6fwvpuYmKBt27av/borIJVKIZFIVF57Ljs7G+vWrYOJiQmaNm2qVB6ZTIaPPvoIs2bNQsOGDVWu69mzZ2FlZYV69erhf//7H54/f650uYcPH0bdunXRq1cvWFlZoW3btio9fi4QHx+Pw4cPY+zYsUrnad++PQ4cOIDo6GgIIXDmzBmEhobC09OzxHxZWVkAoHDdaWhoQFdXt8jr7tXvkqtXryInJ0fhenN3d4ejo2OR11t5vouUzS+VSmFsbAwtLa1C6SXlTUtLg6+vL1xcXODg4KBU2enp6Rg5ciR+++23YtdhLKnsbdu2wcLCAo0aNYK3tzfS09OVyp+QkICAgABYWVmhffv2sLa2RpcuXZT6zF519epVXL9+vdjrraj87du3x65du/DixQvIZDLs3LkTmZmZ6Nq1a4l5s7KyIJFIFCYW1NPTg4aGhkrfc2VW6eHUWyg6OloAEJcuXVJInzVrlmjTpo1Kx0I5Wnby8vJEv379RIcOHVTKd/PmTWFoaCg0NTWFiYmJOHz4sNJ5d+zYIRo1aiQyMjKEEKrdwfzzzz/izz//FDdu3BBHjx4VHh4ewtHRUSQnJ5eaV1dXV+jq6gpvb29x7do1sXbtWqGnpyc2bdqkdN0L7Nq1S2hqaoro6Gil9s/LyxNz5swREolEaGlpCYlEIhYvXqxSmR4eHqJLly4iOjpa5Obmij/++ENoaGiIunXrFpvn1Wvj4sWLAoCIiYlR2G/YsGFi+PDhpeZ/WUW07GRkZIgWLVqIkSNHKp334MGDwtDQUEgkEmFnZyeuXLmidNmLFy8W77zzjrw1TpWWnR07doj9+/eLmzdvir1794r69euL1q1bi9zc3FLzF9zVGxgYiOXLl4vg4GDh4+MjJBKJOHv2rFLnXWDp0qXC1NRU/v9HmbpnZmaKjz/+WAAQWlpaQkdHR2zevLnUvNnZ2cLR0VEMGzZMvHjxQmRlZYklS5YIAMLT01Mhb1HfJdu2bRM6OjqFymndurWYPXt2qflfVlrLjjLfZU+fPhWOjo5i3rx5Suf97bffhKGhoQAg6tWrV2SrTnH5x48fL8aOHSv/u6jPpri8a9euFUePHhU3b94UW7duFbVq1RKDBw9Wqmx/f38BQJiZmYmNGzeKa9euiWnTpgkdHR0RGhqq1HkX+N///ifq169f5Lbi8icmJgpPT0/59WZsbCyOHTtWat6EhARhbGwspk6dKtLS0kRqaqqYPHmyACDGjx9fbB0rCoOdSlBdgp2JEycKJycnERUVpVK+rKwsERYWJoKCgsTcuXOFhYWFuHPnTqn5IiMjhZWVlbhx44Y8TZVg51WJiYnC2NhYqUdo2trawsPDQyHt888/F+3atVO5XE9PT/Huu+8qvf+OHTuEvb292LFjh7h586bYsmWLMDMzUynQCg8PF507dxYAhKampmjdurUYNWqUcHd3LzZPdQ52srOzRf/+/UXz5s0Vmq1Ly5uamirCwsKEv7+/+PTTT4Wzs7OIj48vNX9QUJCwtrZWCFBVCXZe9eDBA6UfoRX8f//ggw8U9uvfv794//33VSq7Xr16YvLkycVuLyr/Dz/8IOrWrSsOHDggbty4IX799VdRo0aNQo8GisobFBQkmjZtKr/uevXqJfr06SN69+6tsF9R3yWqBDulfReVFuyUll8qlYo2bdqI3r17i+zsbKXzJiUlidDQUOHn5yf69+8vWrRoUSjQLCr//v37haurq0hJSZGnFfX+KvsdfOrUqSIfoRWVv+D/ube3t8K+jRs3FnPnzlW67PT0dGFiYiJ+/PHHIrcXl3/y5MmiTZs24uTJk+L69evim2++ESYmJuLmzZul5j127JioXbu2kEgkQlNTU3z44YeiRYsWYuLEiSW8OxWDwU4lyMrKEpqamoUu/I8//lgMGDBApWOVNdjx8vIS9vb24uHDhyrnfVWPHj2Uirz37t0r/9IseAGQX9hF3SWXplWrVgr/gYvj6OiocJclhBC///67sLOzU6m8R48eCQ0NDbFv3z6l89jb24tVq1YppC1atEjUq1dPpbKFyP+xLwhWhg8fLvr27Vvsvq9eGwU/0K8GKJ07dxZTpkwpNf/LyhPsZGdni0GDBokmTZqIZ8+eqZT3Va6urkW2kr2af8WKFfLr7OVrT0NDQzg5OZWpbAsLC7FmzZpSy87KyhJaWlpi0aJFCvvNnj1btG/fXumyz507JwCI69evF1unV/Onp6cLbW3tQv29xo4dK3r16qV02UlJSSIhIUEIkd/fcNKkSfJtxX2XFPxAvxqgODo6iuXLl5ea/2UlBTul5U9OThYeHh6iR48ehQIVVb4Hs7KyhIGBgdi+fXup+adOnVrs9dalSxeVy05NTRUAxNGjR0st++HDhwKA+OOPPxTShw8fLm9FVabsLVu2CG1tbfnn/rLi8oeHhxfq5yVE/m/EhAkTlC776dOn8s/a2tpaLFu2rNh9Kwr77FQCHR0dtGzZEqdOnZKnyWQynDp1SqW+L2UhhMDkyZOxd+9enD59Gi4uLuU+pkwmkz/fL0mPHj1w69YtXL9+Xf5q1aoVRo0ahevXr0NTU1OlclNTU/HgwQPY2tqWum+HDh0KDXMMDQ2Fk5OTSmX6+vrCysoK/fr1UzpPeno6NDQU/ytpamqWaYSBoaEhbG1tkZiYiGPHjmHgwIFK53VxcYGNjY3CdZecnIyAgIBKv+4K5OTkYPjw4QgLC8PJkydhbm5eruMpe+199NFHuHnzpsK1Z2dnh1mzZuHYsWMql/vkyRM8f/5cqWtPR0cHrVu3Lvf1t2HDBrRs2VLpPkpA/vudk5NT7uvPxMQElpaWCAsLQ1BQEAYOHFjqd0nLli2hra2tcL2FhIQgMjISHh4e5f4uUiZ/cnIyPD09oaOjgwMHDsj7H5WlbJF/84+srKxS88+dO7fQ9QYAK1aswMaNG1UuuyC/ra1tqWU7OzvDzs6uyOvN0dFR6bI3bNiAAQMGwNLSUuE9KCl/Qb+ioq63vLw8pcu2sLBAzZo1cfr0aSQkJMhHbFaqSg+n3lI7d+4Uurq6YtOmTeLu3bti/PjxombNmiIuLq7UvCkpKSI4OFgEBwcLAPJ+AK+OaCnK//73P2FiYiLOnj0rYmNj5a/09HSl6j137lzh5+cnIiIixM2bN8XcuXOFRCIRx48fVyr/q1R5jDVz5kxx9uxZERERIS5evCh69uwpLCwsirzzeNWVK1eElpaW+P7770VYWJjYtm2bMDAwEFu3blW6rnl5ecLR0VHMmTNH6TxC5I/kqVWrljh06JCIiIgQe/bsERYWFoWa8kty9OhRceTIEfHw4UNx/Phx0bRpU9G2bdtCTfKlXRtLliwRNWvWlPc/GThwoHBxcZHf8ZaW//nz5yI4OFgcPnxYABA7d+4UwcHBIjY2ttT82dnZYsCAAcLe3l5cv35d4frLysoqMW9qaqrw9vYW/v7+4tGjRyIoKEiMGTNG6Orqyu8iVf1/8fJjrJLypqSkiC+++EL4+/uLiIgIcfLkSdGiRQvh5uYmMjMzlSp7z549QltbW6xbt06EhYWJX3/9VWhqaorz588rVW+pVCoMDAzE6tWrC51Hafm7dOkiGjZsKM6cOSMePnwofH19hZ6envj9999Lzfvnn3+KM2fOiAcPHoh9+/YJJycnMWTIECGEct8lEydOFI6OjuL06dMiKChIeHh4yB8nK5M/NjZWBAcHi/Xr1wsA4ty5cyI4OFg8f/681PxSqVS0bdtWNG7cWISHhyvsM3HixBLzPnjwQCxevFgEBQWJx48fi4sXL4r+/fsLMzMzER8fX6bvUfzbclZa3vDwcLFw4UIRFBQkIiIixP79+0Xt2rVF586dlX7fVqxYIYyNjcXu3btFWFiY+Oqrr4Senp4YOXKkUvUOCwsTEolEHDlyRCG9tLKzs7OFq6ur6NSpkwgICBDh4eHixx9/FBKJRPTt27fUsjdu3Cj8/f1FeHi4+OOPP4SZmZmYMWNGse9pRWKwU4l+/fVX4ejoKHR0dESbNm3E5cuXlcpX0KT76mv06NGl5i0qHwDh6+urVNmffvqpcHJyEjo6OsLS0lL06NGjzIGOEKoFOyNGjBC2trZCR0dH1KpVS4wYMaLIDoPFOXjwoGjUqJHQ1dUV7u7uYt26dSrV9dixYwKACAkJUSlfcnKymDp1qnB0dBR6enqidu3a4ssvvxRZWVlKH2PXrl2idu3aQkdHR9jY2AgvLy+RlJRUaL/Srg2ZTCbmz58vrK2tha6urujRo4fC+ZSW39fXt8jtCxYsKDV/waOvol5nzpwpMW9GRoYYPHiwsLOzEzo6OsLW1lYMGDBAoYOyqv8vXg52Ssqbnp4uPD09haWlpdDW1hZOTk5i3LhxCjcmypS9YcMG4erqKvT09ETTpk3lj0KVybt27Vqhr69fps88NjZWfPLJJ8LOzk7o6emJevXqiZ9++knIZLJS8/7yyy/C3t5eaGtrC0dHR/HVV1/Jr1tlvksyMjLEpEmThKmpqTAwMBCDBw+WB8bK5F+wYEGx+5SWv7hzK+lVkDc6Olr06dNHWFlZCW1tbWFvby9Gjhwp7t+/r3TdX1UQ7JSWNzIyUnTu3FmYmZkJXV1d4erqKmbNmiXv26Zs2T4+PsLe3l4YGBgIDw8Pcf78eaXzent7CwcHB5GXl1foHErLHxoaKoYMGSKsrKyEgYGBaNKkidiyZYtSeefMmSOsra2Ftra2cHNzk1+nr4Pk3xMkIiIiUkvss0NERERqjcEOERERqTUGO0RERKTWGOwQERGRWmOwQ0RERGqNwQ4RERGpNQY7REREpNYY7BARveLs2bOQSCRISkqq6qoQUQVgsENERERqjcEOERERqTUGO0RU7chkMvj4+MDFxQX6+vpo2rQp/vrrLwD/PWI6fPgwmjRpAj09PbRr1w63b99WOMbff/+Nhg0bQldXF87Ozvjpp58UtmdlZWHOnDlwcHCArq4uXF1dsWHDBoV9rl69ilatWsHAwADt27cvtNI0Eb0ZGOwQUbXj4+ODLVu2YM2aNbhz5w6mT5+ODz/8EH5+fvJ9Zs2ahZ9++gmBgYGwtLRE//79kZOTAyA/SBk+fDjef/993Lp1C9988w3mz5+PTZs2yfN//PHH2LFjB1auXIl79+5h7dq1qFGjhkI9vvzyS/z0008ICgqClpYWPv3009dy/kRUsbgQKBFVK1lZWTAzM8PJkyfh4eEhT//ss8+Qnp6O8ePHo1u3bti5cydGjBgBAHjx4gXs7e2xadMmDB8+HKNGjcLTp09x/Phxef7Zs2fj8OHDuHPnDkJDQ1GvXj2cOHECPXv2LFSHs2fPolu3bjh58iR69OgBAPjnn3/Qr18/ZGRkQE9Pr5LfBSKqSGzZIaJqJTw8HOnp6XjnnXdQo0YN+WvLli148OCBfL+XAyEzMzPUq1cP9+7dAwDcu3cPHTp0UDhuhw4dEBYWhry8PFy/fh2ampro0qVLiXVp0qSJ/N+2trYAgISEhHKfIxG9XlpVXQEiopelpqYCAA4fPoxatWopbNPV1VUIeMpKX19fqf20tbXl/5ZIJADy+xMR0ZuFLTtEVK00aNAAurq6iIyMhKurq8LLwcFBvt/ly5fl/05MTERoaCjq168PAKhfvz4uXryocNyLFy+ibt260NTUROPGjSGTyRT6ABGR+mLLDhFVK0ZGRvjiiy8wffp0yGQydOzYEVKpFBcvXoSxsTGcnJwAAAsXLoS5uTmsra3x5ZdfwsLCAoMGDQIAzJw5E61bt8aiRYswYsQI+Pv7Y9WqVfj9998BAM7Ozhg9ejQ+/fRTrFy5Ek2bNsXjx4+RkJCA4cOHV9WpE1ElYbBDRNXOokWLYGlpCR8fHzx8+BA1a9ZEixYtMG/ePPljpCVLlmDq1KkICwtDs2bNcPDgQejo6AAAWrRogT///BNff/01Fi1aBFtbWyxcuBCffPKJvIzVq1dj3rx5mDRpEp4/fw5HR0fMmzevKk6XiCoZR2MR0RulYKRUYmIiatasWdXVIaI3APvsEBERkVpjsENERERqjY+xiIiISK2xZYeIiIjUGoMdIiIiUmsMdoiIiEitMdghIiIitcZgh4iIiNQagx0iIiJSawx2iIiISK0x2CEiIiK1xmCHiIiI1Nr/A45Ofu2hDDlOAAAAAElFTkSuQmCC", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "plt.plot(np.arange(len(epochs_x)), epochs_acc, marker = '.')\n", - "plt.xlabel('epoch')\n", - "plt.ylabel('test accuracy')\n", - "plt.ylim(0, 100)\n", - "plt.xticks(np.arange(len(epochs_x)))\n", - "for i, txt in enumerate(epochs_acc):\n", - " if i%2 == 0:\n", - " pass\n", - " else:\n", - " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", - "plt.show()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "speck-rescnn", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/Res-SCNN3.ipynb b/tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/Res-SCNN3.ipynb deleted file mode 100644 index 64926de1..00000000 --- a/tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/Res-SCNN3.ipynb +++ /dev/null @@ -1,1380 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import torch, random, sys\n", - "\n", - "import tonic\n", - "from torch.utils.data import DataLoader\n", - "from torch.nn import CrossEntropyLoss\n", - "from torch.optim import Adam\n", - "\n", - "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential\n", - "\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "\n", - "sys.path.append('../../utils')\n", - "sys.path.append('../models')\n", - "\n", - "from train_test_fn import training_loop, load_dataset, split_train_validation, load_architecture" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "device: NVIDIA GeForce RTX 3070 Ti\n" - ] - } - ], - "source": [ - "if torch.cuda.is_available():\n", - " device = torch.device('cuda:0')\n", - " print('device: ', torch.cuda.get_device_name(0))\n", - "else:\n", - " device = torch.device('cpu')" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "rand_seed = 1" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "achitecture = 'ResSCNN3'" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "torch.backends.cudnn.enabled = False\n", - "torch.backends.cudnn.deterministic = True\n", - "random.seed(rand_seed)\n", - "torch.manual_seed(rand_seed)\n", - "torch.cuda.manual_seed(rand_seed)\n", - "np.random.seed(rand_seed)" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "batch_size = 8\n", - "num_workers = 4\n", - "epochs = 30\n", - "lr = 5e-5\n", - "\n", - "spk_thr = 2.0\n", - "v_min = -0.313\n", - "\n", - "grad_scale = 1.534\n", - "grad_width = 0.759\n", - "\n", - "validation_ratio = 0.2\n", - "n_time_steps = 50" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Loading Data" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "snn_train_dataset, snn_test_dataset, sensor_size = load_dataset('DVSGESTURE', n_time_steps)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "getting validation dataset...." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "train_dataset, validation_dataset = split_train_validation(validation_ratio, snn_train_dataset, rand_seed)" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The transformed array is in shape [Time-Step, Channel, Height, Width] --> (50, 2, 128, 128)\n" - ] - } - ], - "source": [ - "sample_data, label = train_dataset[0]\n", - "print(f\"The transformed array is in shape [Time-Step, Channel, Height, Width] --> {sample_data.shape}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "disk caching samples..." - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "disk_cache_train = tonic.DiskCachedDataset(\n", - " dataset=train_dataset,\n", - " cache_path='./cached_train'\n", - ")\n", - "snn_train_dataloader = DataLoader(disk_cache_train, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True)\n", - "\n", - "disk_cache_validation = tonic.DiskCachedDataset(\n", - " dataset=validation_dataset,\n", - " cache_path='./cached_validation'\n", - ")\n", - "snn_validation_dataloader = DataLoader(disk_cache_validation, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True)\n", - "\n", - "disk_cache_test = tonic.DiskCachedDataset(\n", - " dataset=snn_test_dataset,\n", - " cache_path='./cached_test'\n", - ")\n", - "snn_test_dataloader = DataLoader(disk_cache_test, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=False)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Network Module" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "snn = load_architecture(achitecture, sensor_size, 11, batch_size, PeriodicExponential(grad_scale=grad_scale, grad_width=grad_width), v_min, spk_thr).to(device)\n", - "snn.init_weights()" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "optimizer = Adam(snn.parameters(), lr=lr, betas=(0.9, 0.999), eps=1e-8)\n", - "loss_fn = CrossEntropyLoss()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Training loop" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "5bf192a7696645b9a33b40575b296ed8", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - " 0%| | 0/107 [00:00" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "y_avg = []\n", - "for y in epochs_y:\n", - " y_avg.append(np.mean(y))\n", - "\n", - "plt.plot(np.arange(len(epochs_x)), y_avg, marker = '.')\n", - "plt.xlabel('epoch')\n", - "plt.ylabel('average loss')\n", - "plt.ylim(0,)\n", - "plt.xticks(np.arange(len(epochs_x)))\n", - "for i, txt in enumerate(y_avg):\n", - " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAG2CAYAAACZEEfAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABl+0lEQVR4nO3deVhU1eMG8PcOOyiDyC6CuOGGGC6ImqYQaub+yzRTU9MsNfeFTK3sG2qaZpZauZZ7uVtuuKWhAoq7iIhisgnIIDvMnN8fxuTINoMgOL6f55mnuPeee84M48zLvWeRhBACRERERHpKVtkNICIiIqpIDDtERESk1xh2iIiISK8x7BAREZFeY9ghIiIivcawQ0RERHqNYYeIiIj0GsMOERER6TWGHSIiItJrDDtERESk1yo17Jw8eRI9e/aEk5MTJEnCrl27NPYLITBnzhw4OjrCzMwMfn5+iIyM1DgmJSUFgwcPhqWlJaysrDBy5Eikp6c/x2dBREREVVmlhp2MjAx4enri+++/L3L/woULsWzZMqxcuRJnz56FhYUFunbtiuzsbPUxgwcPxtWrV3H48GHs27cPJ0+exOjRo5/XUyAiIqIqTqoqC4FKkoSdO3eiT58+AB5f1XFycsKUKVMwdepUAIBCoYC9vT3WrVuHgQMH4vr162jSpAlCQkLQqlUrAMCBAwfwxhtv4J9//oGTk1NlPR0iIiKqIgwruwHFiY6ORnx8PPz8/NTb5HI5vL29ERwcjIEDByI4OBhWVlbqoAMAfn5+kMlkOHv2LPr27VvkuXNycpCTk6P+WaVSISUlBTVr1oQkSRX3pIiIiKjcCCHw6NEjODk5QSYr/mZVlQ078fHxAAB7e3uN7fb29up98fHxsLOz09hvaGgIa2tr9TFFCQwMxOeff17OLSYiIqLKcO/ePTg7Oxe7v8qGnYoUEBCAyZMnq39WKBRwcXHBvXv3YGlpWYktIyIiIm2lpaWhdu3aqF69eonHVdmw4+DgAABISEiAo6OjentCQgJatGihPiYxMVGjXH5+PlJSUtTli2JiYgITE5NC2y0tLRl2iIiIXjCldUGpsvPsuLm5wcHBAUFBQeptaWlpOHv2LHx8fAAAPj4+SE1NRVhYmPqYo0ePQqVSwdvb+7m3mYiIiKqeSr2yk56ejlu3bql/jo6ORnh4OKytreHi4oKJEyfiyy+/RIMGDeDm5obZs2fDyclJPWKrcePG6NatG0aNGoWVK1ciLy8P48aNw8CBAzkSi4iIiABUctgJDQ1F586d1T8X9KMZNmwY1q1bh+nTpyMjIwOjR49GamoqOnTogAMHDsDU1FRdZuPGjRg3bhx8fX0hk8nQv39/LFu27Lk/FyIiIqqaqsw8O5UpLS0NcrkcCoWCfXaIiIheENp+f1fZPjtERERE5YFhh4iIiPQaww4RERHpNYYdIiIi0msMO0RERKTXGHaIiIhIrzHsEBERkV5j2CEiIiK9xrBDREREeo1hh4iIiPQaww4RERHpNYYdIiIi0msMO0RERKTXGHaIiIhIrzHsEBERkV5j2CEiIiK9xrBDREREeo1hh4iIiPQaww4RERHpNYYdIiIi0msMO0RERKTXGHaIiIhIrzHsEBERkV5j2CEiIiK9xrBDREREeo1hh4iIiPQaww4RERHpNYYdIiIi0msMO0RERKTXGHaIiIhIrzHsEBERkV5j2CEiIiK9xrBDREREeo1hh4iIiPQaww4RERHpNYYdIiIi0msMO0RERKTXGHaIiIhIrzHsEBERkV5j2CEiIiK9xrBDREREeo1hh4iIiPQaww4RERHpNYYdIiIi0msMO0RERKTXGHaIiIhIrzHsEBERkV5j2CEiIiK9xrBDREREeo1hh4iIiPQaww4RERHpNYYdIiIi0msMO0RERKTXGHaIiIhIrzHsEBERkV5j2CEiIiK9xrBDREREeo1hh4iIiPQaww4RERHpNYYdIiIi0msMO0RERKTXGHaIiIhIrzHsEBERkV6r0mFHqVRi9uzZcHNzg5mZGerVq4d58+ZBCKE+RgiBOXPmwNHREWZmZvDz80NkZGQltpqIiIiqkioddhYsWIAVK1Zg+fLluH79OhYsWICFCxfiu+++Ux+zcOFCLFu2DCtXrsTZs2dhYWGBrl27Ijs7uxJbTkRERFWFJJ68TFLFvPnmm7C3t8fq1avV2/r37w8zMzP8+uuvEELAyckJU6ZMwdSpUwEACoUC9vb2WLduHQYOHKhVPWlpaZDL5VAoFLC0tKyQ50JERETlS9vv7yp9Zaddu3YICgrCzZs3AQAXL17EqVOn0L17dwBAdHQ04uPj4efnpy4jl8vh7e2N4ODgYs+bk5ODtLQ0jQcRERHpJ8PKbkBJZs6cibS0NDRq1AgGBgZQKpX43//+h8GDBwMA4uPjAQD29vYa5ezt7dX7ihIYGIjPP/+84hpOREREVUaVvrKzbds2bNy4EZs2bcL58+exfv16LFq0COvXr3+m8wYEBEChUKgf9+7dK6cWExERUVVTpa/sTJs2DTNnzlT3vfHw8MDdu3cRGBiIYcOGwcHBAQCQkJAAR0dHdbmEhAS0aNGi2POamJjAxMSkQttOREREVUOVvrKTmZkJmUyziQYGBlCpVAAANzc3ODg4ICgoSL0/LS0NZ8+ehY+Pz3NtKxEREVVNVfrKTs+ePfG///0PLi4uaNq0KS5cuIBvvvkGI0aMAABIkoSJEyfiyy+/RIMGDeDm5obZs2fDyckJffr0qdzGExERUZVQpcPOd999h9mzZ+Ojjz5CYmIinJyc8MEHH2DOnDnqY6ZPn46MjAyMHj0aqamp6NChAw4cOABTU9NKbDkRERFVFVV6np3nhfPsEBERvXj0Yp4dIiIiomfFsENERER6jWGHiIiI9BrDDhEREek1hh0iIiLSaww7REREpNcYdoiIiEivMewQERGRXmPYISIiIr3GsENERER6jWGHiIiI9BrDDhEREek1hh0iIiLSaww7REREpNcYdoiIiEivMewQERGRXmPYISIiIr3GsENERER6jWGHiIiI9BrDDhEREek1hh0iIiLSaww7REREpNcYdoiIiEivMewQERGRXmPYISIiIr3GsENERER6jWGHiIiI9BrDDhEREek1hh0iIiLSaww7REREpNcYdoiIiEivMewQERGRXmPYISIiIr3GsENERER6jWGHiIiI9BrDDhEREek1hh0iIiLSaww7REREpNcYdoiIiEivMewQERGRXmPYISIiIr3GsENERER6jWGHiIiI9BrDDhEREek1hh0iIiLSaww7REREpNcYdoiIiEivMewQERGRXmPYISIiIr3GsENERER6jWGHiIiI9BrDDhEREek1hh0iIiLSaww7REREpNcYdoiIiEivMewQERGRXmPYISIiIr3GsENERER6Teewc+zYsYpoBxEREVGF0DnsdOvWDfXq1cOXX36Je/fuVUSbiIiIiMqNzmHn/v37GDduHH777TfUrVsXXbt2xbZt25Cbm1sR7SMiIiJ6JjqHHRsbG0yaNAnh4eE4e/YsGjZsiI8++ghOTk74+OOPcfHixYpoJxEREVGZPFMHZS8vLwQEBGDcuHFIT0/HmjVr0LJlS7z66qu4evVqebWRiIiIqMzKFHby8vLw22+/4Y033oCrqysOHjyI5cuXIyEhAbdu3YKrqyveeuutcmng/fv38e6776JmzZowMzODh4cHQkND1fuFEJgzZw4cHR1hZmYGPz8/REZGlkvdRERE9OLTOeyMHz8ejo6O+OCDD9CwYUNcuHABwcHBeP/992FhYYE6depg0aJFuHHjxjM37uHDh2jfvj2MjIzw559/4tq1a1i8eDFq1KihPmbhwoVYtmwZVq5cibNnz8LCwgJdu3ZFdnb2M9dPRERELz5DXQtcu3YN3333Hfr16wcTE5Mij7GxsSmXIeoLFixA7dq1sXbtWvU2Nzc39f8LIbB06VJ8+umn6N27NwBgw4YNsLe3x65duzBw4MBnbgMRERG92HS+shMUFIRBgwYVG3QAwNDQEJ06dXqmhgHAnj170KpVK7z11luws7PDK6+8gp9++km9Pzo6GvHx8fDz81Nvk8vl8Pb2RnBwcLHnzcnJQVpamsaDiIiI9JPOYScwMBBr1qwptH3NmjVYsGBBuTSqwO3bt7FixQo0aNAABw8exIcffoiPP/4Y69evBwDEx8cDAOzt7TXK2dvbq/cVJTAwEHK5XP2oXbt2ubabiIiIqg6dw86qVavQqFGjQtubNm2KlStXlkujCqhUKnh5eeGrr77CK6+8gtGjR2PUqFHPXE9AQAAUCoX6wckRiYiI9JfOYSc+Ph6Ojo6Fttva2iIuLq5cGlXA0dERTZo00djWuHFjxMTEAAAcHBwAAAkJCRrHJCQkqPcVxcTEBJaWlhoPIiIi0k86h53atWvj9OnThbafPn0aTk5O5dKoAu3bt0dERITGtps3b8LV1RXA487KDg4OCAoKUu9PS0vD2bNn4ePjU65tISIioheTzqOxRo0ahYkTJyIvLw9dunQB8LjT8vTp0zFlypRybdykSZPQrl07fPXVVxgwYADOnTuHH3/8ET/++CMAQJIkTJw4EV9++SUaNGgANzc3zJ49G05OTujTp0+5toWIiIheTDqHnWnTpiE5ORkfffSRej0sU1NTzJgxAwEBAeXauNatW2Pnzp0ICAjAF198ATc3NyxduhSDBw9WHzN9+nRkZGRg9OjRSE1NRYcOHXDgwAGYmpqWa1uIiIjoxSQJIURZCqanp+P69eswMzNDgwYNShyKXtWlpaVBLpdDoVCw/w4REdELQtvvb52v7BSoVq0aWrduXdbiRERERM9FmcJOaGgotm3bhpiYGPWtrAI7duwol4YRERERlQedR2Nt2bIF7dq1w/Xr17Fz507k5eXh6tWrOHr0KORyeUW0kYiIiKjMdA47X331FZYsWYK9e/fC2NgY3377LW7cuIEBAwbAxcWlItpIREREVGY6h52oqCj06NEDAGBsbIyMjAxIkoRJkyaph4QTERGR/ohTZOHvqCTEKbIquyllonOfnRo1auDRo0cAgFq1auHKlSvw8PBAamoqMjMzy72BREREVDmUKoHVp6Ix/8/rUAlAJgGB/TzwdusX606OzmGnY8eOOHz4MDw8PPDWW29hwoQJOHr0KA4fPgxfX9+KaCMRERE9gzhFFqKTMuBmYwFHuRkAQAiBlIxcxCmyEZuapf5vrCIbcf/+HK/IgvKJCWpUAvhkxxV0bGirPs+LQOews3z5cmRnZwMAZs2aBSMjI/z999/o378/Pv3003JvIBEREZXdlnMxCNh5GQWz6tWztYBKALGpWcjJV+l8PqUQuJOU+UKFHZ367OTn52Pfvn0wMDB4XFgmw8yZM7Fnzx4sXrwYNWrUqJBGEhERke5O3nyAmTv+CzoAEPUgA9FJGeqgY1PNBJ7OcnRr6oDh7etg1huN8f07XuiacxJ3F7yp8bj/0xgAQF6mAuPHj4e7uzvMzMzg4uKCjz/+GAqFQuu2jRkzBpIkYenSpRrbU1JSMHjwYFhaWsLKygojR45Eenr6M70OOl3ZMTQ0xJgxY3D9+vVnqpSIiIgqTlJ6DhYfuokt52KK3D/7zcZ4vbED7OUmMDE0KPKYELkZmjZtivFfr8NXf9yASgCQPb5G8vHqY3CIjsGiRYvQpEkT3L17F2PGjEFsbCx+++23Utu3c+dOnDlzpsgFxAcPHoy4uDgcPnwYeXl5GD58OEaPHo1NmzZp/wI8RefbWG3atEF4eLh65XEiIiKqGnLylVh3+g6WH72FRzn5RR5jIEl4w8NRq9tQhoaG+KB7K/Rq1xR3kjJhaiTDlO0XcfuBA2Te4+DSojXq1bZCvXr18L///Q/vvvsu8vPzYWhYfLy4f/8+xo8fj4MHD6pHdxe4fv06Dhw4gJCQELRq1QoA8N133+GNN97AokWLigxH2tA57Hz00UeYPHky7t27h5YtW8LCwkJjf/PmzcvUECIiIiobIQQOXUvAV39cx93kxyOjPWrJMadnE9x+kI5PdlyBUggYSBK+6tdM6/42kZGRcHJygqmpKXx8fBAYGIjfxrTD8LXncPEfBQb9dAarhrTEqw1s1etTlRR0VCoVhgwZgmnTpqFp06aF9gcHB8PKykoddADAz88PMpkMZ8+eRd++fXV8ZR7TOewMHDgQAPDxxx+rt0mSBCEEJEmCUqksU0OIiIhId9fj0vDF3msIvp0MALCtboLpXd3R38sZMpmE1nWs0bGhLe4kZaKOjbnWQcfb2xvr1q2Du7s74uLi8Pnnn+PVV1/FlStXsGlUW4z5NQx/RSZhxLoQfObvgnnz5mH06NElnnPBggUwNDTUyBBPio+Ph52dncY2Q0NDWFtbIz4+Xqt2F0XnsBMdHV3myoiIiKh8FPTL2RoSA5UAjA1lGP1qXXz4Wj1YmGh+vTvKzXQePdW9e3f1/zdv3hze3t5wdXXFtm3bMHLkSPw8rBWmbLuIPSFRGDGoPxq5uuGzzz4r9nxhYWH49ttvcf78eUiSpFNbnpXOYYd9dYiIiJ6vJ+fJsbYwLtQvp0dzR8zs1gi1rc0rrA1WVlZo2LAhbt26BQAwMTTAlz3qY/cXIyAzNkNq+wlYfjwaE/0aFBlm/vrrLyQmJmosLaVUKjFlyhQsXboUd+7cgYODAxITEzXK5efnIyUlBQ4ODmVuu85hZ8OGDSXuHzp0aJkbQ0RERJq2hsQgYMdlqAQgAahhYYyUjFwA//XLaV3HusLbkZ6ejqioKAwZMgQAkJaWhu7duqKegxXGT/0WP5z6B98GRSIlIxef9WoKA5lm4BkyZAj8/Pw0tnXt2hVDhgzB8OHDAQA+Pj5ITU1FWFgYWrZsCQA4evQoVCoVvL29y954oSMrKyuNh4WFhZAkSZiYmIgaNWroeroqQaFQCABCoVBUdlOIiKiKmTt3rgCg8XB3d1fvz8rKEh999JGwtrYWFhYWol+/fiI+Pl7r83/wwQcCgFiyZInG9oiICOHfvYeQmVkKydhMmNRqIuwHfiVcZ+wTr3xxSGwLiRFKpaq8nmYhU6ZMEcePHxfR0dHi9OnTws/PT9jY2IjExEShUCiEt7e38PDwELdu3RJxcXFi2d5zwnncL8Jl2m7x0cYwkZ2XL9zd3cWOHTuKrcPV1bXQ8+7WrZt45ZVXxNmzZ8WpU6dEgwYNxKBBg4osr+33t85Xdh4+fFhoW2RkJD788ENMmzat7KmLiIioimratCmOHDmi/vnJEUeTJk3C/v37sX37dsjlcowbNw79+vXD6dOnSz1vSfPNvPnmm7BxcoX9wP9BMjRGWugeJP7+OWqN/hmL3/NH50Z2RZyx/Pzzzz8YNGgQkpOTYWtriw4dOuDMmTOwtbXF8ePHcfbsWQBA/fr1Ncq5frQG+y8ZQJGZh4iICJ0mGgSAjRs3Yty4cfD19YVMJkP//v2xbNmyZ3ouOoedojRo0ADz58/Hu+++ixs3bpTHKYmIiKoMQ0PDIvuMKBQKrF69Gps2bUKXLl0AAGvXrkXjxo1x5swZtG3btthzljTfTFJSEiIjI2H/5kQYGz/uK1uj0zCkX9iP/OS7aORYvRyfXdG2bNlS7L7XXnsN4slpmZ9wKjIJo38JxalbSej53V9o4++Bv6OSNNblKnDnzp1C5a2trZ9pAsGi6LRcREkMDQ0RGxtbXqcjIiKqMgrmm6lbty4GDx6MmJjHMxOHhYUhLy9Poy9Ko0aN4OLiguDg4GLPV9J8M0IIBEVnwrimM8KDdsNAmQMIJR6FH4CBuRW+fL9XlV6XqkMDG2we1RbWFsa49I8Cbyw7hXd+Oov2849ia0jRMzpXNJ2v7OzZs0fjZyEE4uLisHz5crRv377cGkZERMUrahVrqhglzTcTHx8PY2NjWFlZaZSxt7cvcV6Y4uabSXyUjU92XMaR64mwHfAlMvYHInrx/0Emk6FGTRv88ecf8O/oURFPs1x51rbCD4O9MPDHM+ptlblius5hp0+fPho/S5IEW1tbdOnSBYsXLy6vdhERUTGeHJ0jk4DAfh54u7VL6QWpTEqab8bMTPcv7eLmm7kem4auS07iYWYejGQS5Jd+gUeTuvh002qYmZnh559/xoh33kJISAgcHR3L5blVJFURt7kqa8V0ncOOSqX7cvBERFQ+4hRZ6qADVO5fy5WhKlzRenK+mddffx25ublITU3VuLqTkJBQ7Lwwxc038+Oiz2FQ3Qavf/EbBjg9xPsLjuGvhw9haWkJAPjhhx9w+PBhrF+/HjNnzqzQ51ge3GwsIJOgfq8Cj9flqmNTcXMBFafc+uwQEVHF+/tWksaXB/DfX8u6mj9/PiRJwsSJE9Xb4uPjMWTIEDg4OMDCwgJeXl74/fffSzzPZ599BkmSNB6NGjUqdFxwcDC6dOkCcwsLWFSrjrbtOiArK6vUdmbnKRGdlIH/7b+GdoFHK63/R5wiC39HJeHW/QeIioqCo6MjWrZsCSMjIwQFBamPi4iIQExMDHx8fIo8z5AhQ3Dp0iWEh4fj511H0WTsShhUs4bcux8mLV6H3WPbw8b08RUfmUzza1omk70wFx0c5WYI7OcBg3+vXum6Lld50vnKTv/+/dGmTRvMmDFDY/vChQsREhKC7du3l1vjiIjoP3eTMzD/z4hC2yUJOv+1HBISglWrVhVavHno0KFITU3Fnj17YGNjg02bNmHAgAEIDQ3FK6+8Uuz5ShqaDTwOOt26dUOPIR/Cuu7/QUgGuP0gGttC78HPozbiUrNwPzULcYpsxKVmIVaRjdh/fy6YQO9JKgHM+P0ykjNyMbC1C6wtjHV6/rqYOnUqLBq0wZrwdOQ9SoHi9EYYCQmDBg2CXC7HyJEjMXnyZFhbW8PS0hLjx4+Hj4+PxkisRo0aITAwEH379kXNmjVhVl2OwD9u4JczjwAzRxgaGWHk66/g6/cf3zLz8fFBjRo1MGzYMMyZMwdmZmb46aefEB0dXWjkVlX2dmuXMq3LVd50DjsnT54scu2L7t27s88OEVEF+edhJt756SwepOfArroJktJz1Fd4zI0MYGZkoPW50tPTMXjwYPz000/48ssvNfb9/fffWLFiBdq0aQMA+PTTT7FkyRKEhYWVGHaKG5pdYNKkSRgx+kPsNnwVhtUebzOq6Yy5+yMxd39kqW02MZQhJ7/wFY2FByKw+NBNtK9vg57NHeHf1AFyM6NSz6eL67eicXDVWiiz0mBgJoeJcxNYvr0Ah29nokl2KgI+D4QkSejfvz9ycnLQtWtX/PDDDxrniIiIwN24B/g7Kgnp2fn46o/ruPPv6uTDfFyxboupxlIPNjY2OHDgAGbNmoUuXbogLy8PTZs2xe7du+Hp6Vmuz6+ilWVdrvImieIGyhfDzMwM4eHhcHd319h+48YNvPLKK1pdkqxq0tLSIJfL1cvTExFVJXGKLLy96gxiUjJR18YCWz5oC6VK4FZCOmbvvoI7yZkY1KY2Avs1L/1kAIYNGwZra2ssWbIEr732Glq0aIGlS5cCAPz9/WFsbIwNGzbAyspKvejjxYsXC00eV+Czzz7D119/DblcDlNTU/j4+CAwMFDdJyUxMRH29vZo9+5UhB3dh7zUeBjVdIZVxyEwdW4KA9njL0QnKzM4yU3h+O9/nazM/t1uiszcfHRYcEzjFp4EoKFDNUTEp6u3GRvI0MndFm82d4RfY/tCC2KWRgiBmJRMhN19iNC7D3H+7kPciH9UajkjAwkOctPH7S1oe8HzkZvhbHQy5u27ptF+B0tTfP1Wc7zawFanNtJ/tP3+1vnKjoeHB7Zu3Yo5c+ZobN+yZQuaNGmie0uJiKhYiWnZGPzTWcSkZMLF2hybRrWFXXVTAI8DwtdveeKtlcHYfO4e+nk5l7pG0pYtW3D+/HmEhIQUuX/btm14++23UbNmTRgaGsLc3Bw7d+4sNugAJQ/NNjW3wHc7TwEAzvy2CjU6j4CxfV2kXzmKhC2z4Dzye5ye/y5q1Sj5NpyVuTEC+3ngkx1XoBRC3f/j7dYuiE7KwL6LsdhzMRaRiek4fC0Bh68lwNRIBt/G9ujZ3AmvudvC1MigUAfnnHwlrtxPw/m7DxF6NwVhd1ORlJ5TYluAx0GrqZMlkjNykZCWjTylwL2ULNxL0e4PfgnAhpFt0NC+4icHpDKEndmzZ6Nfv36IiopSzxYZFBSEzZs3s78OEVE5Sk7PweCfz+J2UgZqWZlh0yhvOMhNNY5pXccag9rUxuZz9/DJjsvY//GrMDYseuzJvXv3MGHCBBw+fBimpqZFHjN79mykpqbiyJEjsLGxwa5duzBgwAD89ddf8PAoen6X4oZmf7HsZ1ww88KV87cBAK7te2HER6Ox9vQdGNvXQ87di2iZcxG1aozW6vUorv+Hm40Fxvs2wHjfBoiIf4S9F2Ox71Is7iRnYv+lOOy/FIdqJoZoaF8NF+6lQvy7oKaLtTni0rKR+9TtMSMDCR615GjpWgMtXa3h5WqFYzcSiwxaAJCvVCHhUY5mX6Mn/j8mJROPsvM16hAAktNzAXutnjo9I51vYwHA/v378dVXXyE8PBxmZmZo3rw55s6di06dOlVEGyscb2MRUVWTmpmLgT+ewY34R3CwNMW2D3zgUrPoqx+pmbnw++YEktJzMa2rO8Z2LvoqzK5du9C3b18YGPzXv0epVEKSJMhkMkRERKB+/fq4cuWKxqy+fn5+qF+/PlauXKlV228lpqOtdxvkOTRFjU7vwSI3GdeWDMO69esxbOhQxCmycCcpE4HTPkB1MxNs3LhRh1dGO0IIXLmfhr2XYrHvYixiFdnFHmttYfxvsKmBVq410KyWHKZF9IEqaLeuHW3jFFloP/9ooSHYp2Z2rvS+LC+6CruNBQA9evR4oXqDExG9SBRZeRiy+hxuxD+CbXUTbBrlXWzQAR7f4pn9ZhNM2BKOZUGR6OHhiDo2FoWO8/X1xeXLlzW2DR8+HI0aNcKMGTOQmfm4w+zTw50NDAy0Gu6cmpmLpUcisf7kDTyMvwfrBh0x6lU3jO38OppuDcCtyMcdkQs6rN6LjtK4KlSeJEmCh7McHs5yzOzWCBvO3MFne64VOu6bAZ7o+0otjcn9ilPWjrYFQ7CfvjLEoPP86Bx2QkJCoFKp4O3trbH97NmzMDAwQKtWrcqtcUREL5v0nHy8t/YcLt9XoKaFMTa97426ttVKLdfL0wm/hf2DvyKTMHv3FWwY0abQF3j16tXRrFkzjW0WFhaoWbMmmjVrhry8PNSvXx8ffPABFi1ahJo1a2LXrl04fPgw9u3bpy7j6+uLvn37Yty4cQCAyZOnwLhua+y+lYuHSYlQnNoIYyND/LlsJlo1qgMAmDZtGubOnQtPT0+0aNEC69evx40bN/Dbb7894ytWOplMQtemDvhi77VCV1d86tXUKug8q6oyBPtlpfOkgmPHjsW9e/cKbb9//z7Gjh1bLo0iItJFUZPjAf9NYmdhYQFLS0t07NixxBGjJ0+eRM+ePeHk5ARJkrBr165Cx7z33nuFJtDr1q1buTyPzNx8DF97DhdiUmFlboRf3/dGAy07sEqShC/7NIOJoQx/RSZhz0XdF2Y2MjLCH3/8AVtbW/Ts2RPNmzfHhg0bsH79erzxxhvq425G3sKFmzGIU2TheEQiNh67gEUBY3Fj+Uik7luIdk3r4MqFUHXQAYCJEyciICAAkyZNgqenJ4KCgnD48GHUq1dP53aWRVWY4M5RbgafejUZdCqBzn12qlWrhkuXLqFu3boa26Ojo9G8eXM8elT6EL2qhn12iF5cISEhGDBgACwtLdG5c2f1EOqCSewCAgLQs2dPGBoa4uLFi+jduzdMTEyKPNeff/6J06dPo2XLlujXrx927txZaD3A9957DwkJCVi7dq16m4mJCWrUqPFMzyM7T4kR60Lwd1QyqpsaYtP7beHhLNf5PN8fu4WvD0bAppoxjkzuBCvz8p1s78l1uZ5kbWGMya83xMDWtWFoUHUn5y9rvxuqmiqsz46JiQkSEhIKhZ24uLhCM2YSEVWkkibHmzRpEj7++GONNYSenh/sad27d9eqD4mJiUmJE+jpKjtPidG/hOHvqGRYGBtg/Yg2ZQo6ADDq1brYdeE+IhPTseDADa3n3tHG0+tyFRjUpjZmdm9c7pP5VYSqMMEdPX86x29/f38EBARAoVCot6WmpuKTTz7B66+/Xq6NIyIqydixY9GjRw/4+flpbE9MTMTZs2dhZ2eHdu3awd7eHp06dcKpU6fKpd7jx4/Dzs4O7u7u+PDDD5GcnFzmc+XmqzB243mcvPkAZkYGWDeiDbxcyn6VyNhQhq/6PR4ivvncPYTcSSnzuZ52+R9FoaADAL08a70QQYdeXjqHnUWLFuHevXtwdXVF586d0blzZ7i5uSE+Pp7LRRDRc1MwOV5gYGChfbdvP57X5bPPPsOoUaNw4MABeHl5wdfXF5GRpS9NUJJu3bphw4YNCAoKwoIFC3DixAl0794dSqVS53PFpGRg8M9nEHQjESaGMqwe1qrUSQG1UTD3DgB8suNyoXlkyuJqrAJz9lwttL2yVrEm0oXO951q1aqFS5cuYePGjbh48SLMzMwwfPhwDBo0CEZGTPZEVPFKmxyvYJj0Bx98gOHDhwMAXnnlFQQFBWHNmjVFBiRtDRw4UP3/Hh4eaN68OerVq4fjx4/D19dX6/NsPHMXs3ZdUf88uK0r2tW3KXO7njajWyMcvpaAyMR0/PTX7WLn3tHG7vD7mPH7JWTnqWBtYYzUzFyoROWuYk2kizJ1srGwsMDo0drNeElEVN7CwsKQmJgILy8v9TalUomTJ09i+fLliIh4vDL400vYNG7cGDExMeXalrp168LGxga3bt0qNewoVQJnbidja0gM9lyM09i3/vQdjHrVrdyCg7Zz75QkX6nCggM38NNf0QCATg1tsWzgK8jMy2cnX3qhlLlH8bVr1xATE4Pc3FyN7b169XrmRhERleTJyfEePMrG/dQsLJo1ER5Nm2DGjBmoW7cunJyc1KGnwM2bN8t9Ert//vkHycnJcHR0LHK/SiVwPuYh9l6Mxf7L8cWuu6QUAneSMss1PGgz905xUjJyMX7zeZy+9bg/0kev1cMUf3cYyCTIYcSQQy8UncPO7du30bdvX1y+fBmSJKFg5HrBP6Cy3LcmItJFweR4W0NiELDnLlQCSEjKhVO+sXrSPG0msXt6crz09HTcunVLvT86Ohrh4eGwtraGi4sL0tPT8fnnn6N///5wcHBAVFQUpk+fjvr166Nr167qckIIXL6v+HeNpjjEPbFUgZW5ETo1tMGei3EQT01wV959Xwrm3vFfclI9907vFrVKLXc1VoHRG8JwPzUL5sYGWPSWJ97wKDrMEb0IdA47EyZMgJubG4KCguDm5oZz584hOTkZU6ZMwaJFiyqijUREhTw9DFoAOHUrCTsv3EczJ0u8/+E4ZGdnY9KkSUhJSYGnp2ehSeyenBzPUW6G0NBQdO7cWb1/8uTJAIBhw4Zh3bp1MDAwwKVLl7B+/XqkpqbC3sERXu06Yd68L2BiYqJehHLvpVjcTc5Un6eaiSH8m9qjp6cTOtS3gZGBDO3qxTyX5QNca1rgY98G+PpgBObtu4ZODW1LnHvnyf45rjXN8eOQVnB34Mrc9GLTeVJBGxsbHD16FM2bN4dcLse5c+fg7u6Oo0ePYsqUKbhw4UJFtbXCcFJBohfPjvP/YPK2iyUeY2lqCCcrMzjKTeFkZabx/+fvPsSiQxFQCUAmAYH9PNSrWGvjycn1JAC21U2Q+Oi/W1RmRgbwbWyHnp5O6NTQtlwXltRVbr4KPZb9hcjEdAxqU7vIuXfylSrM//MGfj6l2T9Hbs6BJ1R1VdikgkqlEtWrP075NjY2iI2Nhbu7O1xdXQvdHyciqgjHIhIxe/eVIvfVtbHAg/QcPMrOR1p2PtLiH+FGfMkzu6sEMOP3y1hwIAIGstL7tChVAikZ//VXFAASH+XASCahc6PHAce3sR3MjUv+iH1eE9wVzL3z1spgbD53D/28nDWGuJfUP4dIH+gcdpo1a4aLFy/Czc0N3t7eWLhwIYyNjfHjjz8WmlWZiKg8CSHww/EoLDoUASEAF2sz/PMwS2MYdMHVmfScfMSlZuF+ahbiFNmIS81CrCIbsalZuP0gA/Fp2YXO/2SAKYuVQ1rCt7H9M52johTMvbP53D18suMy9n/8KowNZeyfQy8FncPOp59+ioyMDADAF198gTfffBOvvvoqatasia1bt5Z7A4mIgMfhZeq2izhwNR4AMKiNCz7r1QQpGblF3gqqZmKIBvbVi1xIM06Rhfbzj2rMBiyTgHXD28C2etHrZj3pwaMcDFt7rlAH4yZOVfs2+JNz73xzOAKGMgk//XUbOfmC/XNIr+ncZ6coKSkpqFGjhtZDGqsa9tkhqtqikzIwekMoIhPTYWQg4fNezfCOt/b9a4qyNaRwB2Fd++w8S/nKsjv8PiZsCdfY1tC+GrZ/0I79c+iFU2F9dopibf3s05sTERXlWEQiPt58AY+y82FX3QQr3m2Jlq7PtsI4ALzd2gUdG9qWuYPws5avLK3rFH7tbiWmIzMvH3Iw7JB+4jLlRFQlPd0/x8vFCivebQl7y8LLQ5TVs3YQfhFX0L7zxJD4AiqBcp/QkKgqYdghoiqnuP45JoaFh2+TbtxsLCCToNFfiYt5kr7TedVzIqKKFJ2Ugb7fn8aBq/EwMpDwVV8PBPbzYNApJ45yMwT284DBv30suZgnvQx0vrJz8uRJtGvXDoaGmkXz8/Px999/o2PHjuXWOCJ6uVRU/xzS9KL2NyIqK51HYxkYGCAuLg52dnYa25OTk2FnZ/dCro3F0VhElSdOkYXoBxk4GfkAq07errD+OUSkfypsNJYQosgh5snJybCwsND1dET0EntyyYUC7J9DROVN67DTr18/AI9X0X3vvfdgYvLfxFtKpRKXLl1Cu3btyr+FRKSX7j/MxMzfL+PJS8uSBHzsW59Bh4jKldZhRy6XA3h8Zad69eowM/vvHq+xsTHatm2LUaNGlX8LiUjvnI95iGnbL+Lpe+iCQ6CJqAJoHXbWrl0LAKhTpw6mTp3KW1ZE5WTFihVYsWIF7ty5AwBo2rQp5syZg+7duwMAoqKiMHXqVJw6dQo5OTno1q0bvvvuO9jba7cG0/z58xEQEIAJEyZg6dKlAB7Pej537lwcOnQIMTExsLW1RZ8+fTBv3jz1HzYVITY1CwsO3MDu8Ngi93MINBFVBJ2Hnk+fPl2jz87du3exdOlSHDp0qFwbRvSycHZ2xvz58xEWFobQ0FB06dIFvXv3xtWrV5GRkQF/f39IkoSjR4/i9OnTyM3NRc+ePaFSqUo9d0hICFatWoXmzZtrbI+NjUVsbCwWLVqEK1euYN26dThw4ABGjhxZIc8xMzcf3xy+iS6Lj2N3eCwkCRjQyhmz3mjMIdBEVOF0Ho3l7++Pfv36YcyYMUhNTYW7uzuMjY2RlJSEb775Bh9++GFFtbXCcDQWVTXW1tb4+uuvUbt2bXTv3h0PHz5UvzcVCgVq1KiBQ4cOwc/Pr9hzpKenw8vLCz/88AO+/PJLtGjRQn1lpyjbt2/Hu+++i4yMjEJTS5SVSiWw++J9LPgzQr3KeJs61pj9ZhN4OD++ghSnyOIQaCIqE22/v3W+snP+/Hm8+uqrAIDffvsNDg4OuHv3LjZs2IBly5aVvcVEBKVSiS1btiAjIwM+Pj7IycmBJEkaAwJMTU0hk8lw6tSpEs81duxY9OjRo8RAVCBOkYWwyH9QrbpluQWd8zEP0W/F35i09SLi07LhXMMMPwz2wtYP2qqDDvB4kjufejUZdIiowuj8qZaZmYnq1asDAA4dOoR+/fpBJpOhbdu2uHv3brk3kOhlcPnyZfj4+CA7OxvVqlXDzp070aRJE9ja2sLCwgIzZszAV199BSEEZs6cCaVSibi4uGLPt2XLFpw/fx4hISGl1r01JAbTfz2F++sCUa1pF2wNiXmm1buf7pdjYWyAjzrXx8gObjA14igrInr+dA479evXx65du9C3b18cPHgQkyZNAgAkJibyFhBRGbm7uyM8PBwKhQK//fYbhg0bhhMnTqBJkybYvn07PvzwQyxbtgwymQyDBg2Cl5cXZLKiL8zeu3cPEyZMwOHDh2FqWvKkfJEJjzB901nEb/8cRjVdIG//Dmb8fhm/BN9FHRsLOFmZwVFuCke5GWpZmcHRyhQ1LYwLzbUVp8jCjbg0nIxMwuZzMcjOU0GSgLdaOmOqvzvsODkgEVUinfvs/Pbbb3jnnXegVCrRpUsXHD58GAAQGBiIkydP4s8//6yQhlYk9tmhqiBOkYXopAy42VhgSP+eqFevHlatWqXen5SUBENDQ1hZWcHBwQFTpkzBtGnTCp2n4I8RA4P/rqIolUpIkgSZTIacnBwkZeRhzelorD12DTEbP4VkZAK7/5sLydC41HYaG8rgKDeFk/xx+EnNyMOxiESNYeRP98shIqoIFTaD8v/93/+hQ4cOiIuLg6enp3q7r68v+vbtW7bWaqmoIbTZ2dmYMmUKtmzZgpycHHTt2hU//PCD1sNyiaqCJ2cSlkmAeWomnHNyNI6xsbEBABw9ehSJiYno1atXkefy9fXF5cuXNbYNHz4cjRo1wtsjx2HGjivYHX4fOZkZSNg2G5KBEWz7z1YHHZkEzOvTDFm5SsSmZiNOkYVYRTZiU7OQlJ6D3HwV7iZn4m5yZpH1yyRg6UBPOFlxCDkRVQ1l6ono4OCA9PR0HD58GB07doSZmRlat25d5DIS5aW4IbSTJk3C/v37sX37dsjlcowbNw79+vXD6dOnK6wtVHU9eXXkRenwOn7yNPyWYAMDS1uocrOQce04osPO4NNPPwHweI6rxo0bw9bWFsHBwZgwYQImTZoEd3d39TkK/tgYN24cqlevjmbNmqn3CSGglBkjLD4Pfx1IAgCocjKRsetz1Komw8eBP2DJsbtQ5mRBJgH/G9weg7xdi2xrbr4KCWmPg0+cIhvBUcnYGnpP4xiVAO4mZzHsEFGVoXPYSU5OxoABA3Ds2DFIkoTIyEjUrVsXI0eORI0aNbB48eJyb2R6ejoGDx6Mn376CV9++aV6u0KhwOrVq7Fp0yZ06dIFwH9fDGfOnEHbtm3LvS1UdT19dSSwn8czdbR9Xu7ci8WDQxugzEiBzMQCxrZ1YDfgCyy7bobz+Rdx83gIps+YCUXqQ9SpUwezZs1S95UrEBUVhaSkJI1t+UoVDlyNx48nb+NaXBqMlVmoKQHdmjrAyygWo5deRQqASf07apTzmRJdbFuNDWWobW2O2taPg4x3XWtsD7unsbYVJwYkoqpG5z47Q4cORWJiIn7++Wc0btwYFy9eRN26dXHw4EFMnjwZV69eLfdGDhs2DNbW1liyZAlee+019XwhR48eha+vLx4+fAgrKyv18a6urpg4cWKhL4QCOTk5yHniFkFaWhpq167NPjsvsDhFFtrPP6rxpSsBGPNaXbjbWz7uY2JlBge5KYwMip9x4XlfGVKpBD7dfQWbzsaUemwNcyN4udRAyzo10MrVGs2d5Rqjmwra7mBpir8ik/Dzqdu4l5IFADAxlOGtVs54v0Nd1LEp39nPt4bE4JMdV6AUQj0x4IsQMonoxVdhfXYOHTqEgwcPwtnZWWN7gwYNKmToeUlDaOPj42FsbKwRdADA3t4e8fHxxZ4zMDAQn3/+eXk3lSpRdFKGRtABAAFgxfHbGtskCbCtZgJHKzPUsno8yqggCF2NTcOK47ee25WhPKUKM3+/jN/P//O4bf+22UCSMKtHY7jWNEfo3YcIu/sQF++l4mFmHoJuJCLoRiIAwMhAQlMnOVq61kBOnhKbzsUUeg1qmBthqE8dDPVxRc1qJqgIb7d2QceGtpwYkIiqLJ3DTkZGBszNC1+iTklJ0Zj4rDzoMoRWFwEBAZg8ebL654IrO/TisjAuPH+LBKBrMwekZuYiTpGNOEU2cvNVSHyUg8RHObh4r/B5CqgE8MmOK+jY0LZCvryzcpUYt+k8gm4kwkAmYX4/D3RoYFMoMPg2ftzRPjdfhWtxaQi9k4LzMQ8ReuchEh/lIPxeKsLvpRZZx1T/hhjZoS7Minhtytvj0MiQQ0RVk85h59VXX8WGDRswb948AIAkSVCpVFi4cCE6d+5cro0LCwtDYmIivLy81NuUSiVOnjyJ5cuX4+DBg8jNzUVqaqrG1Z2EhAQ4ODgUe14TE5NyD2ZUedKy8zDjd83RR0XdThFCIDkjF7GpWepRRnH/jjKKiE9DZGKGxjmUQuBmwqNy/xJXZOZh5PoQhN59CBNDGb5/xwt+TR6HmuLqMjaUoUVtK7SobaV+Lv88zELY3YfYfykOh68nFCrT0tX6uQQdIqKqTuews3DhQvj6+iI0NBS5ubmYPn06rl69ipSUlHIfAVXSENoZM2agdu3aMDIyQlBQEPr37w8AiIiIQExMDHx8fMq1LVQ15ear8NGv53Ej/hFsq5tg1bteyMkXRd5OkSQJNtVMYFPNBM0178IW2ecHAGbvuoJlg7zUIeNZJaRlY+jqc4hIeITqpoZY815rtK5jrfN5JElSdxT2rmuNoBsJ7CRMRFQMncNOs2bNcPPmTSxfvhzVq1dHeno6+vXrh7Fjx8LR0bFcG/f0EFoAsLCwQM2aNdXbR44cicmTJ8Pa2hqWlpYYP348fHx8OBLrJSCEQMCOyzh1KwnmxgZY+15rNKtVtknsHOVmCOznoe5oK5MAC2NDxKRkof+KvzH2tXoY79ugxM7Npbn9IB1DVp/D/dQs2FU3wfoRbdDY8dk7xD/ddq4eTkSkSeewExMTg9q1a2PWrFlF7nNxeb6jMJYsWQKZTIb+/ftrTCpIz9+KFSuwYsUK3LlzBwDQtGlTzJkzB927dwcAvPbaazhx4oRGmQ8++AArV64s9pw7duzAypUrERYWhpSUFFy4cAEtWrQAACw9Eonfz/8DSZWHhlG/o1Pzwc80seTTHW3NjAwwZ/dV7LkYi2VHb+FoRCK+GdACDe2r63ReALj8jwLvrT2H5Ixc1Klpjl9GequHb5cHdhImIiqezkPPDQwMEBcXBzs7O43tycnJsLOzg1KpLNcGPg9cLqJ87N27FwYGBmjQoAGEEFi/fj2+/vprXLhwAU2bNsVrr72Ghg0b4osvvlCXMTc3L/E1/+WXXxAdHQ0nJyeMGjVKHXa2hd7D9N8uAQDq39yCm6EnsG7dOvXEkjKZrNxuq+69GIvZu68gNTMPxoYyTPN3x4gObjCQaTeJ5t+3kjBqQygycpVoVssS64a3gU0FjYwiInqZaPv9rXPYkclkSEhIgK2trcb2u3fvokmTJsjIyCimZNXFsFNxrK2t8fXXX2PkyJEacyTp6s6dO3Bzc8OFCxeQZl4LI9aFIF8lMKKNHb4c2B6bNm3C//3f/wEAbty4gcaNGyM4OLjcbmcmpGVjxu+XcDziAYDHaz8tHuBZ6tWZPy7HYeKWcOQqVWhXryZWDWmJ6qZG5dImIqKXXbnPs1MwVFuSJMyePVtj+LlSqcTZs2fVtxeIlEoltm/fjoyMDI3O4hs3bsSvv/4KBwcH9OzZs9B7qTRRiemYe/o88lUCvVs4ob08BXl5efDz81Mf06hRI7i4uJRr2LG3NMXa91pjS8g9fLnvGs7dSUG3pSfx6ZtNMLB17SKXSvn1zF3M3n0FQgBveDhgydstYGLI0VFERM+b1mHnwoULAB53Cr18+TKMjf9bHdnY2Bienp6YOnVq+beQXiiXL1+Gj48PsrOzUa1aNezcuRNNmjQBALzzzjtwdXWFk5MTLl26hBkzZiAiIgI7duzQ+vyf7bmK9GrOaFvXGgv/rzl+37a1TBNLloUkSRjUxgXt69lg6vaLOHcnBQE7LuPQ1Xgs6N8cdpaP54ISQuC7o7fwzeGbAIB3vF0wr3czrW97ERFR+dI67Bw7dgzA46Hf3377LW/3UJHc3d0RHh4OhUKB3377DcOGDcOJEyfQpEkTjB49Wn2ch4cHHB0d4evri6ioKNSrV6/E8z7KyQMAJGfkomndalj1bqtKu0riUtMcm0e3xepTt7Ho4E0ci3gA/6Un8WWfZnilthU+33sNh649nvfmY98GmOTXoEIXySUiopLpPBpr7dq1FdEO0hPGxsaoX78+AKBly5YICQnBt99+i1WrVhU61tvbGwBw69atEsNObr4Ks3deAQDUsDDC2uGtITd/3O/FwcGhTBNLPisDmYTRHeuhU0M7TN4WjquxaRi36YLGMb08nTD59YYV1gYiItJO2ScNIdKCSqXSWHT1SeHh4QBQ4vxMBXPphN59CAD4vFdTONf4r49Py5Yt1RNLFnieE0u6O1THzo/aY3i7OoX27b8UhzhFVoW3gYiISsawQ+UmICAAJ0+exJ07d3D58mUEBATg+PHjGDx4MKKiojBv3jyEhYXhzp072LNnD4YOHYqOHTuiefPm6nM0atQIO3fuVP/8vx0h2PznSSiTHy9kpXwYi/DwcHV/HLlcrp5Y8tixYwgLC8Pw4cOf68SSxoYyvN608Jw+SiFwJynzubSBiIiKp/NtLKLiJCYmYujQoYiLi4NcLkfz5s1x8OBBvP7667h37x6OHDmCpUuXIiMjA7Vr10b//v3x6aefapwjIiICCoUCALAt9B6WrtmM5D+WqvcPHDgQADB37lx89tlnAKrGxJJuNhaQSeCSDUREVZDO8+zoI86zU7XEKbKwOzwWXx+4AaUAxnWuj6ld3Su7WaXaGhJTaMmGJxciJSKi8lXu8+wQPQ9bQ2Iwc8dlFETwV1ysMMX/xejkyyUbiIiqJoYdqjIi4tMw8/fLePJS48V7qYhPy35hgoOj3OyFaSsR0cuCYYcq3b2UTKw+FY1NZ2Pw9D1VlQDuJGUyQBARUZkx7FClufyPAqtORuGPy3EaHXufxE6+RET0rBh26LkSQuD4zQf48cRtBN9OVm9/tYENPuhYD/88zMSsnZqdfHlVh4iIngXDDj0Xufkq7LkYi59O3kZEwiMAgKFMQk9PJ4x6tS6aOP3Xi76TOzv5EhFR+WHYoXIXp8hCdFIG3GwsYGFiiM1nY7D29B3Ep2UDACyMDTCojQtGdHCDk1XhMMNOvkREVJ4YdqhcbQ2JQcCOy1AJQMLj2YVz8lUAALvqJhje3g3veLtAbmZUuQ0lIqKXBsMOlZvbD9I1ho4LADn5KtSpaY6POtdH7xZOlbZSORERvbwYduiZZOcpceLmA+y9GItDVxMKDR0HgK/6eqBdfZvn3jYiIiKAYYfKIE+pwqlbSdh7MRaHrybgUU5+sccaSBLcbC2eY+uIiIg0MexQkZ7sZOwoN4NSJXD2djL2XorDn1fikJqZpz7WwdIUbzZ3RE9PJ1yPS+PQcSIiqlIYdqgQjU7GEuBTtyYiE9Px4FGO+hibasZ4w+NxwGnpUgMymQQA8KxtxaHjRERUpTDskIbY1EyNhTiFAP6Oejz5n9zMCN2bOeDN5k5oW9cahgayIs/BoeNERFSVMOwQACAi/hH2XYrFttB76qDzpGldG2LUq/VgbFh0wCEiIqqqGHZeYtFJGdh3MRZ7L8XiZkJ6sccZSBL6eTkz6BAR0QuJYecl88/DTOy/FIe9l2Jx5X6aeruRgYRODe3Q09MRisw8fL73GjsZExGRXmDY0VNPjqYykCTsvxyHfZfiEHb3ofoYA5mE9vVt8GZzR3Rt4gC5+X+zGr/e1J6djImISC8w7OihJ0dTPU2SgDZ1rNHT0wndmzmgZjWTIs/BTsZERKQvGHb0TJwiS2M0VYGmTpbo7+WMHs0dYW9pWjmNIyIiqgQMO3pmy7miR1N92qMJfOrVfP4NIiIiqmQMO3pCqRJYdCgCK45HFdpnIEmoY2NeCa0iIiKqfBxLrAdSM3MxfF2IOui82sAG/05ozNFURET00uOVnRfcjfg0jN4QhpiUTJgaybCgf3P0blELcYosjqYiIiICw84L7Y/LcZi6/SIyc5WoZWWGH4e2RFMnOQCOpiIiIirAsPMCUqoEFh+KwA//3rZqX78mvhvkBWsL40puGRERUdXDsPOCUWTm4eMtF3Di5gMAwKhX3TCjW6NiF+UkIiJ62THsvEAi4h9h9C+huJus2T+HiIiIisew84IoqX8OERERFY9hpwqLU2QhKjEDh67GY8OZuwDYP4eIiEhXDDtVVFHrW7F/DhERke74rVkFxSmyCgUdSQJGdHBj0CEiItIRvzmroOikjEIrlgsB3EnKrJwGERERvcAYdqogm2omhbZxfSsiIqKyYdipgo7dSNT4metbERERlR07KFcxufkqrDkdDQD45I1G8KhlxfWtiIiIngHDThWzO/w+EtJyYG9pgmHt6sDE0KCym0RERPRC422sKkSlEvjx5G0AwIj2bgw6RERE5YBhpwo5FpGIyMR0VDcxxCBvl8puDhERkV5g2KlCVp14fFXnnbYusDQ1quTWEBER6QeGnSoi7O5DnLuTAiMDCSPau1V2c4iIiPQGw04V8ePJKABAnxa1YG9pWsmtISIi0h8MO1VA1IN0HLqWAAAY3bFuJbeGiIhIvzDsVAE//3UbQgB+je3QwL56ZTeHiIhIrzDsVLLER9n4/fx9AMAHnepVcmuIiIj0D8NOJVv/9x3k5qvg5WKFVq41Krs5REREeodhpxKl5+Tjl+C7AB5f1ZEkqZJbREREpH8YdirRlnMxSMvOR10bC7ze2L6ym0NERKSXGHYqSZ5ShdWnHi/4ObpjXchkvKpDRERUERh2Ksnei7GIU2TDtroJ+rxSq7KbQ0REpLcYdiqBEEK9NMTw9nVgasQFP4mIiCoKw04lOH7zASISHsHC2ACDvV0ruzlERER6jWGnEqw68XhpiEFtXCA344KfREREFYlhpxgnT55Ez5494eTkBEmSsGvXLo39kiQV+fj666+LPWedOnUgSRK2ftAOdxe8idk9m0KSJIwdO1Z9TFRUFPr27QtbW1tYWlpiwIABSEhIqKinSUREpPeqdNgJDAxE69atUb16ddjZ2aFPnz6IiIjQOCY7Oxtjx45FzZo1Ua1aNfTv379cwkFGRgY8PT3x/fffF7k/Li5O47FmzRpIkoT+/fsXe86QkBC8t/wgnMf+gtGrDuPw4cMAgLfeektdp7+/PyRJwtGjR3H69Gnk5uaiZ8+eUKlUz/yciIiIXkaSEEJUdiOK061bNwwcOBCtW7dGfn4+PvnkE1y5cgXXrl2DhYUFAODDDz/E/v37sW7dOsjlcowbNw4ymQynT5/Wup60tDTI5XIoFApYWloW2i9JEnbu3Ik+ffoUe44+ffrg0aNHCAoKKvaYO0kZ6LL4OFQCODDxVaycPwf79u1DZGQkJEnCoUOH0L17dzx8+FDdDoVCgRo1auDQoUPw8/PT+jkRERHpu9K+vwsYPsc26ezAgQMaP69btw52dnYICwtDx44doVAosHr1amzatAldunQBAKxduxaNGzfGmTNn0LZt2+fSzoSEBOzfvx/r168v8bifT92GSgCd3W1R19oUv/76KyZPnqyeOTknJweSJMHExERdxtTUFDKZDKdOnWLYISIiKoMqfRvraQqFAgBgbW0NAAgLC0NeXp5GCGjUqBFcXFwQHBxc7HlycnKQlpam8XgW69evR/Xq1dGvX79ij0lKz8H20H8APF4aYteuXUhNTcV7772nPqZt27awsLDAjBkzkJmZiYyMDEydOhVKpRJxcXHP1EYiIqKX1QsTdlQqFSZOnIj27dujWbNmAID4+HgYGxvDyspK41h7e3vEx8cXe67AwEDI5XL1o3bt2s/UtjVr1mDw4MEwNTUt9pgNf99BTr4KnrWt4O1mjdWrV6N79+5wcnJSH2Nra4vt27dj7969qFatGuRyOVJTU+Hl5QWZ7IX5VREREVUpVfo21pPGjh2LK1eu4NSpU898roCAAEyePFn9c1paWpkDz19//YWIiAhs3bq12GMycvKx/t8FP8d0rIuYmBgcOXIEO3bsKHSsv78/oqKikJSUBENDQ1hZWcHBwQF169YtU/uIiIhedi9E2Bk3bhz27duHkydPwtnZWb3dwcEBubm5SE1N1bi6k5CQAAcHh2LPZ2JiotEv5lmsXr0aLVu2hKenZ7HHbAu9B0VWHurUNId/UwfM++Jz2NnZoUePHsWWsbGxAQAcPXoUiYmJ6NWrV7m0l4iI6GVTpcOOEALjx4/Hzp07cfz4cbi5uWnsb9myJYyMjBAUFKQe8h0REYGYmBj4+Pg8U93p6em4deuW+ufo6GiEh4fD2toaLi4uAB5fEdq+fTsWL15c5Dl8fX3Rq3cfbMtsCgAY1bEuJAisXbsWw4YNg6Fh4Ze/oIO1ra0tgoODMWHCBEyaNAnu7u7P9HyIiIheVlU67IwdOxabNm3C7t27Ub16dXU/HLlcDjMzM8jlcowcORKTJ0+GtbU1LC0tMX78ePj4+DzzSKzQ0FB07txZ/XPBba9hw4Zh3bp1AIAtW7ZACIFBgwYVeY6oqCgEX4vGfau6qGlhjP5ezjhy5AhiYmIwYsSIIstEREQgICAAKSkpqFOnDmbNmoVJkyY903MhIiJ6mVXpeXYKhmQ/be3atepRTNnZ2ZgyZQo2b96MnJwcdO3aFT/88EOJt7Gepu04fV3FpmZi0I9ncTclE1Neb4jxvg3K7dxEREQvO72YZ0ebHGZqaorvv/++2JmOK8vWkBjM3HEZBU+hmmmVfqmJiIj0FsczV4A4RRYCngg6APDlvuuIU2RVXqOIiIheUgw7FSA6KQOqpy5KKYXAnaTMymkQERHRS4xhpwK42VhA9lR3IwNJQh0b88ppEBER0UuMYacCOMrNENjPAwb/drA2kCR81a8ZHOVmldwyIiKilw97zVaQt1u7oGNDW9xJykQdG3MGHSIiokrCsFOBHOVmDDlERESVjLexiIiISK8x7BAREZFeY9ghIiIivcawQ0RERHqNYYeIiIj0GsMOERER6TWGHSIiItJrDDtERESk1xh2iIiISK8x7BAREZFeY9ghIiIivcawQ0RERHqNYYeIiIj0GsMOERER6TWGHSIiItJrDDtERESk1xh2iIiISK8x7BAREZFeY9ghIiIivcawQ0RERHqNYYeIiIj0GsMOERER6TWGHSIiItJrDDtERESk1xh2iIiISK8x7BAREZFeY9ghIiIivcawQ0RERHqNYYeIiIj0GsMOERER6TWGHSIiItJrDDtERESk1xh2iIiISK8x7BAREZFeY9ghIiIivcawQ0RERHqNYYeIiIj0GsMOERER6TWGHSIiItJrDDtERESk1xh2iIiISK8x7BAREZFeY9ghIiIivcawQ0RERHqNYYeIiIj0GsMOERER6TWGHSIiItJrDDtERESk1xh2iIiISK8x7BAREZFeY9ghIiIivcawQ0RERHqNYYeIiIj0GsMOERER6TWGHSIiItJrDDtERESk1xh2iIiISK/pTdj5/vvvUadOHZiamsLb2xvnzp2r7CYRERFRFaAXYWfr1q2YPHky5s6di/Pnz8PT0xNdu3ZFYmJiZTeNiIiIKplehJ1vvvkGo0aNwvDhw9GkSROsXLkS5ubmWLNmTWU3jYiIiCqZYWU34Fnl5uYiLCwMAQEB6m0ymQx+fn4IDg4uskxOTg5ycnLUPysUCgBAWlpaxTaWiIiIyk3B97YQosTjXviwk5SUBKVSCXt7e43t9vb2uHHjRpFlAgMD8fnnnxfaXrt27QppIxEREVWcR48eQS6XF7v/hQ87ZREQEIDJkyerf1apVEhJSUHNmjUhSVK51ZOWlobatWvj3r17sLS0fK7lWffzr/tZy7Pul6vuZy3Puln3i1L+WesuiRACjx49gpOTU4nHvfBhx8bGBgYGBkhISNDYnpCQAAcHhyLLmJiYwMTERGOblZVVRTURlpaWz/QLfpbyrPv51/2s5Vn3y1X3s5Zn3az7RSn/rHUXp6QrOgVe+A7KxsbGaNmyJYKCgtTbVCoVgoKC4OPjU4ktIyIioqrghb+yAwCTJ0/GsGHD0KpVK7Rp0wZLly5FRkYGhg8fXtlNIyIiokqmF2Hn7bffxoMHDzBnzhzEx8ejRYsWOHDgQKFOy8+biYkJ5s6dW+iW2fMoz7qff93PWp51v1x1P2t51s26X5Tyz1p3eZBEaeO1iIiIiF5gL3yfHSIiIqKSMOwQERGRXmPYISIiIr3GsENERER6jWGnAn3//feoU6cOTE1N4e3tjXPnzmlV7uTJk+jZsyecnJwgSRJ27dqldZ2BgYFo3bo1qlevDjs7O/Tp0wcRERFal1+xYgWaN2+unvzJx8cHf/75p9blnzR//nxIkoSJEydqdfxnn30GSZI0Ho0aNdK6vvv37+Pdd99FzZo1YWZmBg8PD4SGhmpVtk6dOoXqliQJY8eOLbWsUqnE7Nmz4ebmBjMzM9SrVw/z5s0rda2WJz169AgTJ06Eq6srzMzM0K5dO4SEhBQ6rrT3hhACc+bMgaOjI8zMzODn54fIyEity+/YsQP+/v7q2cTDw8O1rj8vLw8zZsyAh4cHLCws4OTkhKFDhyI2Nlaruj/77DM0atQIFhYWqFGjBvz8/HD27Fmt2/6kMWPGQJIkLF26VKuy7733XqHffbdu3XSq+/r16+jVqxfkcjksLCzQunVrxMTElFq2qPedJEn4+uuvtao7PT0d48aNg7OzM8zMzNSLIWtTNiEhAe+99x6cnJxgbm6Obt26qd8v2nyWZGdnY+zYsahZsyaqVauG/v37qyd41ab8jz/+iNdeew2WlpaQJAmpqanqfaWVT0lJwfjx4+Hu7g4zMzO4uLjg448/hkKh0KruDz74APXq1YOZmRlsbW3Ru3dv9RJDunyOCiHQvXt39eurTdnXXnut0O97zJgxOtUdHByMLl26wMLCApaWlujYsSO++OKLEsveuXOn2Pfb9u3btao7Pj4eQ4YMgYODAywsLODl5YXff/9dq7JRUVHo27cvbG1tYWlpiQEDBhSaELiiMOxUkK1bt2Ly5MmYO3cuzp8/D09PT3Tt2hWJiYmlls3IyICnpye+//57nes9ceIExo4dizNnzuDw4cPIy8uDv78/MjIytCrv7OyM+fPnIywsDKGhoejSpQt69+6Nq1ev6tSOkJAQrFq1Cs2bN9epXNOmTREXF6d+nDp1SqtyDx8+RPv27WFkZIQ///wT165dw+LFi1GjRg2t2/tkvYcPHwYAvPXWW6WWXbBgAVasWIHly5fj+vXrWLBgARYuXIjvvvtOq7oB4P3338fhw4fxyy+/4PLly/D394efnx/u37+vcVxp742FCxdi2bJlWLlyJc6ePQsLCwt07doV2dnZWpXPyMhAhw4dsGDBgmL3F1c+MzMT58+fx+zZs3H+/Hns2LEDERER6NWrl1Z1N2zYEMuXL8fly5dx6tQp1KlTB/7+/njw4IFW5Qvs3LkTZ86c0Zg+Xpuy3bp103gPbN68WevyUVFR6NChAxo1aoTjx4/j0qVLmD17NkxNTUst+2SdcXFxWLNmDSRJQv/+/bWqe/LkyThw4AB+/fVXXL9+HRMnTsS4ceOwZ8+eEssKIdCnTx/cvn0bu3fvxoULF+Dq6go/Pz9kZGRo9VkyadIk7N27F9u3b8eJEycQGxuLfv36AdDusygzMxPdunXDJ598Uqh9pZWPjY1FbGwsFi1ahCtXrmDdunU4cOAARo4cqVXdLVu2xNq1a3H9+nUcPHgQQgj4+/tDqVTq9Dm6dOlSjWWGtC07atQojd/7woULtS4fHByMbt26wd/fH+fOnUNISAjGjRuHU6dOlVi2du3ahd5vn3/+OapVq4bu3btrVffQoUMRERGBPXv24PLly+jXrx8GDBiAvXv3llg2IyMD/v7+kCQJR48exenTp5Gbm4uePXtCpVIVel3LnaAK0aZNGzF27Fj1z0qlUjg5OYnAwECdzgNA7Ny5s8ztSExMFADEiRMnynyOGjVqiJ9//lnr4x89eiQaNGggDh8+LDp16iQmTJigVbm5c+cKT0/PMrVxxowZokOHDmUqW5QJEyaIevXqCZVKVeqxPXr0ECNGjNDY1q9fPzF48GCt6srMzBQGBgZi3759Gtu9vLzErFmzii339HtDpVIJBwcH8fXXX6u3paamChMTE7F58+ZSyz8pOjpaABAXLlzQuv6inDt3TgAQd+/e1bmsQqEQAMSRI0e0rvuff/4RtWrVEleuXBGurq5iyZIlWpUdNmyY6N27d4ntKan822+/Ld59990ylX1a7969RZcuXbQu37RpU/HFF19obCvqvfN02YiICAFAXLlyRb1NqVQKW1tb8dNPPxWq++nPktTUVGFkZCS2b9+uPub69esCgAgODi61/JOOHTsmAIiHDx8W+bxLK19g27ZtwtjYWOTl5elc9uLFiwKAuHXrltZ1X7hwQdSqVUvExcUV+7stqqwun4tFlff29haffvppmco+rUWLFoU+v0oqb2FhITZs2KBxnLW1daH3zNNlDx48KGQymVAoFOpjUlNThSRJ4vDhw6U+l2fFKzsVIDc3F2FhYfDz81Nvk8lk8PPzQ3Bw8HNti0KhAABYW1vrXFapVGLLli3IyMjQaemNsWPHokePHhrPX1uRkZFwcnJC3bp1MXjwYMTExGhVbs+ePWjVqhXeeust2NnZ4ZVXXsFPP/2kc/3A49/fr7/+ihEjRmi1MGy7du0QFBSEmzdvAgAuXryIU6dOoXv37lrVl5+fD6VSCVNTU43tZmZmWl/ZAoDo6GjEx8drvO5yuRze3t7P/X1XQKFQQJIkndeey83NxY8//gi5XA5PT0+tyqhUKgwZMgTTpk1D06ZNdW7r8ePHYWdnB3d3d3z44YdITk7Wut79+/ejYcOG6Nq1K+zs7ODt7a3T7ecCCQkJ2L9/P0aOHKl1mXbt2mHPnj24f/8+hBA4duwYbt68CX9//xLL5eTkAIDG+04mk8HExKTI993TnyVhYWHIy8vTeL81atQILi4uRb7fnuWzSNvyCoUClpaWMDQ0LLS9pLIZGRlYu3Yt3NzcULt2ba3qzszMxDvvvIPvv/++2HUYS6p748aNsLGxQbNmzRAQEIDMzEytyicmJuLs2bOws7NDu3btYG9vj06dOmn1O3taWFgYwsPDi32/FVW+Xbt22Lp1K1JSUqBSqbBlyxZkZ2fjtddeK7FsTk4OJEnSmFjQ1NQUMplMp8+5MqvwOPUSun//vgAg/v77b43t06ZNE23atNHpXHiGKztKpVL06NFDtG/fXqdyly5dEhYWFsLAwEDI5XKxf/9+rctu3rxZNGvWTGRlZQkhdPsL5o8//hDbtm0TFy9eFAcOHBA+Pj7CxcVFpKWllVrWxMREmJiYiICAAHH+/HmxatUqYWpqKtatW6d12wts3bpVGBgYiPv372t1vFKpFDNmzBCSJAlDQ0MhSZL46quvdKrTx8dHdOrUSdy/f1/k5+eLX375RchkMtGwYcNiyzz93jh9+rQAIGJjYzWOe+utt8SAAQNKLf+k8riyk5WVJby8vMQ777yjddm9e/cKCwsLIUmScHJyEufOndO67q+++kq8/vrr6qtxulzZ2bx5s9i9e7e4dOmS2Llzp2jcuLFo3bq1yM/PL7V8wV/15ubm4ptvvhEXLlwQgYGBQpIkcfz4ca2ed4EFCxaIGjVqqP/9aNP27OxsMXToUAFAGBoaCmNjY7F+/fpSy+bm5goXFxfx1ltviZSUFJGTkyPmz58vAAh/f3+NskV9lmzcuFEYGxsXqqd169Zi+vTppZZ/UmlXdrT5LHvw4IFwcXERn3zyidZlv//+e2FhYSEACHd39yKv6hRXfvTo0WLkyJHqn4v63RRXdtWqVeLAgQPi0qVL4tdffxW1atUSffv21aru4OBgAUBYW1uLNWvWiPPnz4uJEycKY2NjcfPmTa2ed4EPP/xQNG7cuMh9xZV/+PCh8Pf3V7/fLC0txcGDB0stm5iYKCwtLcWECRNERkaGSE9PF+PGjRMAxOjRo4ttY3lh2KkAVSXsjBkzRri6uop79+7pVC4nJ0dERkaK0NBQMXPmTGFjYyOuXr1aarmYmBhhZ2cnLl68qN6mS9h52sOHD4WlpaVWt9CMjIyEj4+Pxrbx48eLtm3b6lyvv7+/ePPNN7U+fvPmzcLZ2Vls3rxZXLp0SWzYsEFYW1vrFLRu3bolOnbsKAAIAwMD0bp1azF48GDRqFGjYstU5bCTm5srevbsKV555RWNy9allU1PTxeRkZEiODhYjBgxQtSpU0ckJCSUWj40NFTY29trBFRdws7ToqKitL6FVvDvfdCgQRrH9ezZUwwcOFCnut3d3cW4ceOK3V9U+a+//lo0bNhQ7NmzR1y8eFF89913olq1aoVuDRRVNjQ0VHh6eqrfd127dhXdu3cX3bp10ziuqM8SXcJOaZ9FpYWd0sorFArRpk0b0a1bN5Gbm6t12dTUVHHz5k1x4sQJ0bNnT+Hl5VUoaBZVfvfu3aJ+/fri0aNH6m1Fvb7afgYHBQUVeQutqPIF/84DAgI0jvXw8BAzZ87Uuu7MzEwhl8vFokWLitxfXPlx48aJNm3aiCNHjojw8HDx2WefCblcLi5dulRq2YMHD4q6desKSZKEgYGBePfdd4WXl5cYM2ZMCa9O+WDYqQA5OTnCwMCg0Bt/6NCholevXjqdq6xhZ+zYscLZ2Vncvn1b57JP8/X11Sp579y5U/2hWfAAoH5jF/VXcmlatWql8Q+4OC4uLhp/ZQkhxA8//CCcnJx0qu/OnTtCJpOJXbt2aV3G2dlZLF++XGPbvHnzhLu7u051C/H4y74grAwYMEC88cYbxR779Huj4Av66YDSsWNH8fHHH5da/knPEnZyc3NFnz59RPPmzUVSUpJOZZ9Wv379Iq+SPV1+yZIl6vfZk+89mUwmXF1dy1S3jY2NWLlyZal15+TkCENDQzFv3jyN46ZPny7atWundd0nT54UAER4eHixbXq6fGZmpjAyMirU32vkyJGia9euWtedmpoqEhMThRCP+xt+9NFH6n3FfZYUfEE/HVBcXFzEN998U2r5J5UUdkorn5aWJnx8fISvr2+hoKLL52BOTo4wNzcXmzZtKrX8hAkTin2/derUSee609PTBQBx4MCBUuu+ffu2ACB++eUXje0DBgxQX0XVpu4NGzYIIyMj9e/9ScWVv3XrVqF+XkI8/o744IMPtK77wYMH6t+1vb29WLhwYbHHlhf22akAxsbGaNmyJYKCgtTbVCoVgoKCdOr7UhZCCIwbNw47d+7E0aNH4ebm9sznVKlU6vv7JfH19cXly5cRHh6ufrRq1QqDBw9GeHg4DAwMdKo3PT0dUVFRcHR0LPXY9u3bFxrmePPmTbi6uupU59q1a2FnZ4cePXpoXSYzMxMymeY/JQMDgzKNMLCwsICjoyMePnyIgwcPonfv3lqXdXNzg4ODg8b7Li0tDWfPnq3w912BvLw8DBgwAJGRkThy5Ahq1qz5TOfT9r03ZMgQXLp0SeO95+TkhGnTpuHgwYM61/vPP/8gOTlZq/eesbExWrdu/czvv9WrV6Nly5Za91ECHr/eeXl5z/z+k8vlsLW1RWRkJEJDQ9G7d+9SP0tatmwJIyMjjfdbREQEYmJi4OPj88yfRdqUT0tLg7+/P4yNjbFnzx51/6Oy1C0e//GPnJycUsvPnDmz0PsNAJYsWYI1a9boXHdBeUdHx1LrrlOnDpycnIp8v7m4uGhd9+rVq9GrVy/Y2tpqvAYllS/oV1TU+02pVGpdt42NDaysrHD06FEkJiaqR2xWqAqPUy+pLVu2CBMTE7Fu3Tpx7do1MXr0aGFlZSXi4+NLLfvo0SNx4cIFceHCBQFA3Q/g6REtRfnwww+FXC4Xx48fF3FxcepHZmamVu2eOXOmOHHihIiOjhaXLl0SM2fOFJIkiUOHDmlV/mm63MaaMmWKOH78uIiOjhanT58Wfn5+wsbGpsi/PJ527tw5YWhoKP73v/+JyMhIsXHjRmFubi5+/fVXrduqVCqFi4uLmDFjhtZlhHg8kqdWrVpi3759Ijo6WuzYsUPY2NgUupRfkgMHDog///xT3L59Wxw6dEh4enoKb2/vQpfkS3tvzJ8/X1hZWan7n/Tu3Vu4ubmp/+ItrXxycrK4cOGC2L9/vwAgtmzZIi5cuCDi4uJKLZ+bmyt69eolnJ2dRXh4uMb7Lycnp8Sy6enpIiAgQAQHB4s7d+6I0NBQMXz4cGFiYqL+K1LXfxdP3sYqqeyjR4/E1KlTRXBwsIiOjhZHjhwRXl5eokGDBiI7O1urunfs2CGMjIzEjz/+KCIjI8V3330nDAwMxF9//aVVuxUKhTA3NxcrVqwo9DxKK9+pUyfRtGlTcezYMXH79m2xdu1aYWpqKn744YdSy27btk0cO3ZMREVFiV27dglXV1fRr18/IYR2nyVjxowRLi4u4ujRoyI0NFT4+PiobydrUz4uLk5cuHBB/PTTTwKAOHnypLhw4YJITk4utbxCoRDe3t7Cw8ND3Lp1S+OYMWPGlFg2KipKfPXVVyI0NFTcvXtXnD59WvTs2VNYW1uLhISEMn2O4t8rZ6WVvXXrlvjiiy9EaGioiI6OFrt37xZ169YVHTt21Pp1W7JkibC0tBTbt28XkZGR4tNPPxWmpqbinXfe0ardkZGRQpIk8eeff2psL63u3NxcUb9+ffHqq6+Ks2fPilu3bolFixYJSZLEG2+8UWrda9asEcHBweLWrVvil19+EdbW1mLy5MnFvqbliWGnAn333XfCxcVFGBsbizZt2ogzZ85oVa7gku7Tj2HDhpVatqhyAMTatWu1qnvEiBHC1dVVGBsbC1tbW+Hr61vmoCOEbmHn7bffFo6OjsLY2FjUqlVLvP3220V2GCzO3r17RbNmzYSJiYlo1KiR+PHHH3Vq68GDBwUAERERoVO5tLQ0MWHCBOHi4iJMTU1F3bp1xaxZs0ROTo7W59i6dauoW7euMDY2Fg4ODmLs2LEiNTW10HGlvTdUKpWYPXu2sLe3FyYmJsLX11fj+ZRWfu3atUXunzt3bqnlC259FfU4duxYiWWzsrJE3759hZOTkzA2NhaOjo6iV69eGh2Udf138WTYKalsZmam8Pf3F7a2tsLIyEi4urqKUaNGafxhok3dq1evFvXr1xempqbC09NTfStUm7KrVq0SZmZmZfqdx8XFiffee084OTkJU1NT4e7uLhYvXixUKlWpZb/99lvh7OwsjIyMhIuLi/j000/V71ttPkuysrLERx99JGrUqCHMzc1F37591cFYm/Jz584t9pjSyhf33Ep6FJS9f/++6N69u7CzsxNGRkbC2dlZvPPOO+LGjRtat/1pBWGntLIxMTGiY8eOwtraWpiYmIj69euLadOmqfu2aVt3YGCgcHZ2Fubm5sLHx0f89ddfWpcNCAgQtWvXFkqlstBzKK38zZs3Rb9+/YSdnZ0wNzcXzZs3Fxs2bNCq7IwZM4S9vb0wMjISDRo0UL9Pnwfp3ydIREREpJfYZ4eIiIj0GsMOERER6TWGHSIiItJrDDtERESk1xh2iIiISK8x7BAREZFeY9ghIiIivcawQ0T0lOPHj0OSJKSmplZ2U4ioHDDsEBERkV5j2CEiIiK9xrBDRFWOSqVCYGAg3NzcYGZmBk9PT/z2228A/rvFtH//fjRv3hympqZo27Ytrly5onGO33//HU2bNoWJiQnq1KmDxYsXa+zPycnBjBkzULt2bZiYmKB+/fpYvXq1xjFhYWFo1aoVzM3N0a5du0IrTRPRi4Fhh4iqnMDAQGzYsAErV67E1atXMWnSJLz77rs4ceKE+php06Zh8eLFCAkJga2tLXr27Im8vDwAj0PKgAEDMHDgQFy+fBmfffYZZs+ejXXr1qnLDx06FJs3b8ayZctw/fp1rFq1CtWqVdNox6xZs7B48WKEhobC0NAQI0aMeC7Pn4jKFxcCJaIqJScnB9bW1jhy5Ah8fHzU299//31kZmZi9OjR6Ny5M7Zs2YK3334bAJCSkgJnZ2esW7cOAwYMwODBg/HgwQMcOnRIXX769OnYv38/rl69ips3b8Ld3R2HDx+Gn59foTYcP34cnTt3xpEjR+Dr6wsA+OOPP9CjRw9kZWXB1NS0gl8FIipPvLJDRFXKrVu3kJmZiddffx3VqlVTPzZs2ICoqCj1cU8GIWtra7i7u+P69esAgOvXr6N9+/Ya523fvj0iIyOhVCoRHh4OAwMDdOrUqcS2NG/eXP3/jo6OAIDExMRnfo5E9HwZVnYDiIielJ6eDgDYv38/atWqpbHPxMREI/CUlZmZmVbHGRkZqf9fkiQAj/sTEdGLhVd2iKhKadKkCUxMTBATE4P69etrPGrXrq0+7syZM+r/f/jwIW7evInGjRsDABo3bozTp09rnPf06dNo2LAhDAwM4OHhAZVKpdEHiIj0F6/sEFGVUr16dUydOhWTJk2CSqVChw4doFAocPr0aVhaWsLV1RUA8MUXX6BmzZqwt7fHrFmzYGNjgz59+gAApkyZgtatW2PevHl4++23ERwcjOXLl+OHH34AANSpUwfDhg3DiBEjsGzZMnh6euLu3btITEzEgAEDKuupE1EFYdghoipn3rx5sLW1RWBgIG7fvg0rKyt4eXnhk08+Ud9Gmj9/PiZMmIDIyEi0aNECe/fuhbGxMQDAy8sL27Ztw5w5czBv3jw4Ojriiy++wHvvvaeuY8WKFfjkk0/w0UcfITk5GS4uLvjkk08q4+kSUQXjaCwieqEUjJR6+PAhrKysKrs5RPQCYJ8dIiIi0msMO0RERKTXeBuLiIiI9Bqv7BAREZFeY9ghIiIivcawQ0RERHqNYYeIiIj0GsMOERER6TWGHSIiItJrDDtERESk1xh2iIiISK8x7BAREZFe+38aEvV9Ns7iMAAAAABJRU5ErkJggg==", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "plt.plot(np.arange(len(epochs_x)), epochs_acc, marker = '.')\n", - "plt.xlabel('epoch')\n", - "plt.ylabel('test accuracy')\n", - "plt.ylim(0, 100)\n", - "plt.xticks(np.arange(len(epochs_x)))\n", - "for i, txt in enumerate(epochs_acc):\n", - " if i%5 ==0 or i == epochs-1:\n", - " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [], - "source": [ - "with open(f'./architectures_results/{achitecture}-Training_Validation-TM.npy', 'wb') as f:\n", - " np.save(f, np.array(epochs_x))\n", - " np.save(f, np.array(epochs_y))\n", - " np.save(f, np.array(epochs_acc))" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "speck-rescnn", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.0" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/architectures_results.ipynb b/tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/architectures_results.ipynb deleted file mode 100644 index f6df49bf..00000000 --- a/tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/architectures_results.ipynb +++ /dev/null @@ -1,306 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np\n", - "import matplotlib.pyplot as plt" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "top_n = 3" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "nb_skip_conns = {\n", - " 'ResSCNN1': {'nb_skip_conns': 1, 'nb_jumped_layer': 1},\n", - " 'ResSCNN2': {'nb_skip_conns': 2, 'nb_jumped_layer': 1},\n", - " 'ResSCNN3': {'nb_skip_conns': 3, 'nb_jumped_layer': 1},\n", - " 'ResSCNN4': {'nb_skip_conns': 4, 'nb_jumped_layer': 1},\n", - " 'ResSCNN5': {'nb_skip_conns': 1, 'nb_jumped_layer': 2},\n", - " 'ResSCNN6': {'nb_skip_conns': 2, 'nb_jumped_layer': 2},\n", - " 'ResSCNN7': {'nb_skip_conns': 2, 'nb_jumped_layer': (1, 2)},\n", - " 'ResSCNN8': {'nb_skip_conns': 4, 'nb_jumped_layer': (1, 2)},\n", - "}" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "architectures_TM = {}\n", - "acc = []\n", - "architecture = []\n", - "\n", - "for i in range(1, 9):\n", - " with open(f'architectures_results_set_1/ResSCNN{i}-Training_Validation-TM.npy', 'rb') as f:\n", - " epochs_x = np.load(f)\n", - " epochs_y = np.load(f)\n", - " epochs_acc = np.load(f)\n", - "\n", - " architectures_TM[f'ResSCNN{i}'] = {\n", - " 'epochs_x': epochs_x,\n", - " 'epochs_y': epochs_y,\n", - " 'epochs_acc': epochs_acc,\n", - " }\n", - "\n", - " acc.append(epochs_acc[-1])\n", - " architecture.append(f'ResSCNN{i}')" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "combined = list(zip(acc, architecture))\n", - "sorted_combined = sorted(combined, key=lambda x: x[0], reverse=True)\n", - "sorted_values, sorted_labels = zip(*sorted_combined)" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "top_indices = np.argsort(acc)[-top_n:]" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAA2gAAAG2CAYAAAAKvaVLAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAAEAAElEQVR4nOzddXyV5fvA8c+pdXfBigWjGR2SAhISooAdiAoGdmL7xRbBAkWRFpEG6YbRDAYb6+7u7dTz++PZzhjbCEWdP+/363VebE8/57DtXOe67utWSJIkIQiCIAiCIAiCIPzjlP/0BQiCIAiCIAiCIAgyEaAJgiAIgiAIgiC0EiJAEwRBEARBEARBaCVEgCYIgiAIgiAIgtBKiABNEARBEARBEAShlRABmiAIgiAIgiAIQishAjRBEARBEARBEIRWQgRogiAIgiAIgiAIrYQI0ARBEARBEARBEFoJEaAJgiAIgiAIgiC0Ev9ogHbw4EHGjRuHl5cXCoWCDRs2NFovSRJvvvkmnp6eWFpaMnz4cOLj4xttU1RUxD333IOdnR0ODg488sgjVFRU/I13IQiCIAiCIAiCcHP8owFaZWUlXbp04euvv252/ccff8z8+fP57rvvOH78ONbW1owcOZKamhrTNvfccw8XL15k165dbNmyhYMHDzJjxoy/6xYEQRAEQRAEQRBuGoUkSdI/fREACoWC9evXM2HCBEDOnnl5efH888/zwgsvAFBaWoq7uztLlixh6tSpxMTEEBYWxsmTJ+nRowcA27dvZ/To0WRkZODl5fVP3Y4gCIIgCIIgCMINU//TF9CS5ORkcnJyGD58uGmZvb09vXv3JiIigqlTpxIREYGDg4MpOAMYPnw4SqWS48ePM3HixGaPXVtbS21trel7o9FIUVERzs7OKBSKv+6mBEEQBKEVkCSJ8vJyvLy8UCrFcHRBEITWpNUGaDk5OQC4u7s3Wu7u7m5al5OTg5ubW6P1arUaJycn0zbNmTt3Lu+8885NvmJBEARB+HdJT0/Hx8fnn74MQRAE4TKtNkD7K7366qs899xzpu9LS0tp27YtycnJ2Nra/oNXJgiCIAh/vfLycvz9/cXfPEEQhFao1QZoHh4eAOTm5uLp6WlanpubS9euXU3b5OXlNdpPr9dTVFRk2r855ubmmJubN1nu5OSEnZ3dTbh6QRAEQWi9NBoNgCjrFwRBaIVabeG5v78/Hh4e7Nmzx7SsrKyM48eP07dvXwD69u1LSUkJp0+fNm2zd+9ejEYjvXv3/tuvWRAEQRAEQRAE4c/4RzNoFRUVJCQkmL5PTk4mMjISJycn2rZty+zZs3n//fcJCgrC39+fOXPm4OXlZer02L59e0aNGsWjjz7Kd999h06n48knn2Tq1Kmig6MgCIIgCIIgCP86/2iAdurUKYYMGWL6vn5c2AMPPMCSJUt46aWXqKysZMaMGZSUlDBgwAC2b9+OhYWFaZ8VK1bw5JNPMmzYMJRKJXfccQfz58//2+9FEARBEARBEAThz2o186D9k8rKyrC3t6e0tFSMQRMEQRD+3xN/9wRBEFqvVjsGTRAEQRAEQRAE4b9GBGiCIAiCIAiCIAithAjQBEEQBEEQBEEQWgkRoAmCIAiCIAiCILQSIkATBEEQBEEQBEFoJUSAJgiCIAiCIAiC0EqIAE0QBEEQBEEQBKGVEAGaIAiCIAiCIAhCKyECNEEQBEEQBEEQhFZCBGiCIAiCIAiCIAithAjQBEEQBEEQBEEQWgkRoAmCIAiCIAiCILQSIkATBEEQBEEQBEFoJUSAJgiCIAiCIAiC0EqIAE0QBEEQBEEQBKGVEAGaIAiCIAiCIAhCKyECNEEQBEEQBEEQhFZCBGiCIAiCIAiCIAithAjQBEEQBEEQBEEQWgkRoAmCIAiCIAiCILQSIkATBEEQBEEQBEFoJUSAJgiCIAiCIAiC0EqIAE0QBEEQBEEQBKGVEAGaIAiCIAiCIAhCKyECNEEQBEEQBEEQhFZCBGiCIAiCIAiCIAitRKsP0MrLy5k9eza+vr5YWlrSr18/Tp48aVovSRJvvvkmnp6eWFpaMnz4cOLj4//BKxYEQRAEQRAEQfhjWn2ANn36dHbt2sWyZcuIiopixIgRDB8+nMzMTAA+/vhj5s+fz3fffcfx48extrZm5MiR1NTU/MNXLgiCIAiCIAiCcGMUkiRJ//RFtKS6uhpbW1s2btzImDFjTMvDw8O57bbbeO+99/Dy8uL555/nhRdeAKC0tBR3d3eWLFnC1KlTr+s8ZWVl2NvbU1paip2d3V9yL4IgCILQWoi/e4IgCK2X+p++gKvR6/UYDAYsLCwaLbe0tOTw4cMkJyeTk5PD8OHDTevs7e3p3bs3ERERLQZotbW11NbWmr4vKysDQKfTodPp/oI7EQRBEITWQ/ytEwRBaL1adYBma2tL3759ee+992jfvj3u7u6sWrWKiIgI2rVrR05ODgDu7u6N9nN3dzeta87cuXN55513mizfuXMnVlZWN/cmBEEQBKGVqaqq+qcvQRAEQWhBqw7QAJYtW8bDDz+Mt7c3KpWK7t27M23aNE6fPv2Hj/nqq6/y3HPPmb4vKyujTZs2jBgxQpR6CIIgCP/v1VeOCIIgCK1Pqw/QAgMDOXDgAJWVlZSVleHp6cmUKVMICAjAw8MDgNzcXDw9PU375Obm0rVr1xaPaW5ujrm5eZPlGo0GjUZz0+9BEARBEFoT8bdOEASh9Wr1XRzrWVtb4+npSXFxMTt27GD8+PH4+/vj4eHBnj17TNuVlZVx/Phx+vbt+w9erSAIgtDa7YvNI3TO76w5mf5PX4ogCIIgmLT6AG3Hjh1s376d5ORkdu3axZAhQwgNDeWhhx5CoVAwe/Zs3n//fTZt2kRUVBT3338/Xl5eTJgw4Z++dEEQBKEVm7crjhqdkS92x6E3GP/py6G8RkdURilGY8vNlfPLa4nPLb/qcYortaQWVt7syxMEQRD+Jq2+xLG0tJRXX32VjIwMnJycuOOOO/jggw9M5RkvvfQSlZWVzJgxg5KSEgYMGMD27dubdH4UBEEQhHrn0ks4l1EKQHZpDbtjchnV0fMae/11LmSW8tiy02SWVOPrbMXUnm2ZHO6Dq605BqPEofh8Vp1IY3dMHgajxBODA3lhRAgqpaLRcfbF5vHMqrOU1+p5cWQITwwKRKFQtHBWQRAEoTVq1fOg/V3EfDCCIAj/v2j1RtacSmdYezc87S2brH9+zTl+O5OBhUZJjc5Iv0BnVj7a5x+4Ulh3JoNX10VRq2+cxVMrFQwKduVSTjmZJdVN9rsl2JX5U7viYGWGJEl8sz+RT3fGcvlf9dGdPPhkcheszRt/Hiv+7gmCILRerb7EURAEQRBu1Pw98byx4QIP/XQS3RXli0WVWjafzwLg0zu7oFTA0cRCEvKuXjp4s+kMRt7edJHn1pyjVm9kSIgrx14dxseTO9OtrQN6o8SeS3lkllRjb6nhof5+7Hz2Fr6c2hULjZKDcfmM++owp1KKeGL5GT7ZIQdn03q15b3xHdCoFGyLymHiN0dILhAlj4IgCP8WIoOG+CRREITrk1pYyVd7E3h6WBBtnP74nImH4wvYEJnJG2Pa42BldhOv8N/nSEIBm89l8cbYMGzMW6i6j94IZ5aBdFmgpVRBz0cheESTzYsrtQz4aC+VWgMAr94WymODAk3rv92fyEfbL9HR247NTw5gxrLT7IrO5YG+vrwzvuM1r3lx1GJszWy5K+SuG7vZOvUliwv2JnA6tRiAp4cFMXtYEMrLShYv5ZSx82IubZwsua2jJxYalWlddFYZjy0/RXpRQ2bNTKXk3fEdmNqrLQCnU4t5Yvlp8sprsbVQ8+XUrgwNlecNFX/3BEEQWq9WPwZNEAShtVh4MIlfT2dQpTPw9d3d/9AxzmeUMH3pSWp0RkI9bJk+MOAmX+W/R5VWzzOrz1JQocXfxbpREGVi0MGWZ6GqsOm63IvwzHlQNf5T9v2hJCq1Bmwt1JTX6PlidxyjO3nSxskKg1Fi+bFUAO7v64dCoeD+vr7sis7ltzOZvDgqtOVAEUguTWbemXkADG4zGDcrt+u+35zSGn49lc7qk+mmkkUbczWf39WFER08mmwf6mFHqEfzwVOYlxxcPrXqLIfiC3C3M+e7e8Pp1tbRtE24ryNbnhrAEyvOcDq1mIUHkhgS4ibGpAmCILRyIkATBEG4TjHZ8uS+u6NzKavRYWdxY3NJ5ZTW8OjSU9To5ExQZHrJzb7Ef0xhRS1PrDiDnYWGD+/ohItN07kmr7T8WCoFFVoAdkXnNh+gJe6VgzNrVxjxfsPyHa9DWSbEboOw202Liyq1/Hw0BZDLF386ksyxpCLe2HCBJQ/1ZF9dyaCDlYbbu3gB0D/QhQAXa5IKKll/NpP7+vi2eM0XCi6Yvt6fvr/ZLFpplY57Fx8n5YqywkqtnvoGjfaWGiZ19+ahfv60da7Lxu7/EC6uh8k/gnuHFq+hnoOVGUse6sXRxAI6eds3m411s7Ng1aN9mLc7jof6+4vgTBAE4V9AjEETBEG4DkajRGyOPEapVm9k+4WcG9q/Wmtg+tKT5JbVYmchfzZ2LqPkT12TzmCkolb/p45xM9TqDTy+/DQnkovYHZPLuAWHOZdeApIE5bnN7lOl1bPwQBJq9NhTwem0YvLLa5tueP4X+Ryhk6DL1IZH+APy+hOLGm1enz3r4GXHiDB3PpjYCTOVkgNx+Ww7m8LKiHgA7urRxlQyqFQquLcuKFsWkcLllf+SJJFXXmP6PqogyvT1nrSGOTgvN/f3GKIySymv1Td6GCXo5e/EF1O6cPy1Ybw1rkNDcJZ+Qg7Q8i/ByilQkdfCs92YSqlgYJBr4+BMVwM1paZvzdRKXhoViqvttYNmQRAE4Z8nAjRBEITrkF5cRVXdmCaADWczr3tfo1Hi+V8juZBZhpO1GatmyN0C04uqKaxoJii5Tk+vOkvvD3aTlF/xh4/xZ0mSxBvrL3AypRhbczUBLtZkl9Zw18IjpH5/D3wWDAtvgVM/Qk2Zab+New/zSO1Sjls8zWmLJ+ijuMjumCuCuZoyuLQVgGnHfTmVUtSwLvwhUCgh5RDkXQLkLF599mz28GAUCgWBrjbMGtIONXoCNk1kQdokxqoiuLd34yzZHeE+WGpUxOVWcDy5iIKKWhYeSGToZwfo9cEePt0RCzTOoJ3IPkGZtqzRcU4kF7G6buLr7+/vwf4XBpseJ14fxprH+jKxm0+j8WQYdLD5GUCS76k0HX65Vw60bpTRCEvGwEd+cqAX+zsY/vkgXhAEQbh+IkATBEG4DjHZcvbMw06eYzEiqZDs0qatz5vzxe44tkXloFEpWHhfOB287AlwtQbgfGbpNfZuXo3OwO6YXCq1Btacymhxu5SCSk5eHtj8QRcySzmWVNhkEuVFdePylAr46p7ubHyyP8PbuzOTtfhmycEV2efkcWSfhcKGWRiW3M60Y+OZqd6EMyWoMfCyejU7LmQ3PumlLaCvIdHoyRm9L6+si6JWXxckO7SBkNHy1yd/AOD7Q8lUaQ109LZjePuGsWGPDw7gCfvjtCcZa0UtX2kW0Pb03EaBi72lhondvQF49pdI+s7dw9zfL5m6H361L4FfTiZzqUgOBh3MHdBLeg5lHDIdo1Zv4NV15wF4sWMFt1Zvxy91renhlrGr+WDp6HzIiwYrZ3hoO1jYQ/px2Pw0XNnHK/s8pBxp+YVK3k967lmiNSqI2w6rpsK8TrDvf1CS3vJ+giAIQqshAjRBEITrcClHzpQMDHKhl58TkgSbIrOuuV9MdhkL9iYAMHdSZ3r6OQHQ1ccBkCdM/iPOZ5SiM8hv3jdGZjYJnEAOGKYsiuDO7yJYd6blIO5aDscXMP7rI0xddIxBn+7j630J5JXXsCs6lw+3ywHLm2PDGBTsiq2FhkXdknlGvQ6At3X3s8RmBnqnINBVQuRyVCkHMEoKjqu6Yxj/LUa1JV2ViWiS9lBeo7vsJuXyxvWGAYCChLwKFh5Ialjfc7r877nVFBYVsjQiBYDZw4IbjbUyx8CT6g0AnDYGyQuPzoflk6CyofnI/X3lrFp2aQ06g0SXNg58dEcnHhskN3KZ8/sudEYddmZ2TA6eDMDetL2m/b/bn0RifiX3WR1jVsIMOcC6/PHLvbB8IlQWNNxDURIc+Fj+euT/oG1vuPNnUKjk+z/8OdSWw6mfYOEgWDgQloyG+N3NvlbrIj5mvI8Xd3t7cqnn/XLQV54FBz6C36Zf5VUWBEEQWgsRoAmCICC3Zv9sZyzpRVXNrr9Ul0EL8bBlQjc507L+Osoc64OG0Z08mBzuY1re2ccekAOtP+JUakNWLLu0huPJTbNkv0flkFsml1C+8ltU4xLB65SYX8HMFacxGCVUSgXpRdV8seMib334ESmrn2eg4hz39fbhgX5+8g4Zp1BunAVAauh0flOP5e2CwfQrm0vsbWvQdX+YRYo7uUU7j/TRy1B1uxtlr0cBeFL5K/sv1Y29KstGSjoAwG/mfoR3OQXKGr7al9BQ0hkwGJyDQFvO8fXfUKU10NnHnmHtr+iseG4l5pWZVJs5E9H/R4yTl4DGGpIPwKLBcoYPuWvi2+PCmD7An61PD2DjrP5M6dmWl0eGcltHDyQzOQMVZB/G8LbDATiceZhaQy2J+RV8vS+B7oo43uZb+bxtekPwbXWPUWBmA8kHYdFgdBknWXLhJ05sfgL0NeA/CDpPkfcLHAKjP5G/3vMufBoCW2ZDdiRHLSxY4GBP3P53GmXXtAYt7x54ibcMmegUCgzAKkcneC5GbjrifwuEP3jDr78gCILw9xMBmiAIAvDO5oss2JvAJ3Vjja5Un0Fr72nHmE6emKmUXMopNy1vTmm1jg1n5SzbA339Gq3r0sYBkDNof2Q6yjN182dZm8ljmZobE1cfHDpZm6E1GHls2ekWA9DmlFRpmf7zKcpq9IT7OnL2ySB+77iPk1bP8K3mcx5Vbmap2Ue8m3ovikOfQsZpWDUNDLUQMhrfuz5m45P9CXKzIa9Cy9hNBh4pmMb/qieidvJlQle5iyL9n0GrtKCLMonMkxvlZRfWokBipyKIyjYbidOuxSX4O/TKHF5ff0F+zhQK9D3krFBQ6ipAYvbwoMadCvVaOPgpAJZDXuDJkZ1RdpwI03eDoz+UpsHiEXBOztY92N+fN8aG0cHL3nQIpVLBZ3d1wdlZbgwTm+ZAG+sg3KzcqNJXcSzrGK+vj8LVkMsSy3mojDoIHSuXK969uu7xi3xOp0Ck0nTe3TiNz05/zmNSJietbGDsF3D5dfd8BHo9Jn+tqwTnIDKHvMJsnzYscrTnDrMS7lk3jvXx60krS+PhHQ/za8rvKCSJ25HLZ7clbaPUUAMd74AHNsvNVQRBEIRWTwRogiD85yXkVbDpnBxIHU0sbBIwVdbqSa0LbEI9bLG30jA4xBXAFIA1Z+3pDKp1BkI9bOnl79RoXXtPO9RKBYWVWjKKm45lW382g0eWnKS4UttknSRJpgmOnxoml+xti8qmRtfQxORCZiln0krQqBRsmNmfDl52FFbKAVejMsIW6AxGZq44Q3pBKffYRrLK8iPsFvWgfcL3OBqL0Vu6kOJ5G5KFPYqSNNj7PvwwFCrzwL0jTFoEShUBrjasn9Wf2zp6oDNIHIzLl697aBBqVd2fIGsXijrIXRkHZPxArU6PVFfe+KmzPZJSbpZRq8jByv9rTuYd5LczmeSV1/DwmUAqJXOClJl83rOcISFXZM8iV0BpOpKNO5+rynl+//NU6arAPQxm7IN2t8oZrPUz4PdX5IYdzbAyU+PiLDcxyStwY/jnBykvCgXg+a0riErK5CfzT7EzllDu0ZHn3Vz46NQn1Ojla684cID0OV+iv2MtPwf2YIONJQB6hYJnPT1I1zQzYfnI/8HEhfDgNqRZJ/hAl061oRZXpQVqSeJ8RSpvHn2TMevHcC7/HLZGia9z83m/x6sEOwZTY6hhQ8KGhuOJFvuCIAj/CiJAEwThP2/B3njT/FQFFbUk5jeevyoutxxJAldbc5zr5veaWFfm2NL4L6NRYlldBuv+Pj4orgj6LDQq2nvKkxCfzyiVy9XqttEZjLy7OZo9l/JYc6ppY4ekgkqKq3SYq5U81N8PL3sLymv17L2UJ3fxoyF7NqqjJ22drfjhgR642ZoTm1vOM6sjKa3SUVZT96jWUlZS2OixYM3vDEj9mmPmT/GB7mPMUvbLJw8cCnf+jPr5GPweW43i+Vg5iGjbV15v7QbTVoG5rel6bczVfHNPd14cGYJCIRHqacv4+uxZHbcRL1KFBR0ViWRsmYsiJ4rdltZk2+ahUqhYeOtCerj3QKGsxbLNMt49/Bljv9rFwcwy1ir6U65QcLtua9Ps2aHPANjd5XZ+ilnOztSdvHLoFYySESwd5czWwBfk7Y9/C0snQEma3Ka+/qGtolJXSVq5/JyaG3zJK6+lMDcYgBrNeb7QLCBYkY7exp0XfYPYmb6X5THLuf/3+8mqyKLgu4VUHDjAmVU/8LlRDlKfLSqmo1FFqVHLrL2zKNeWN36hVWo56+XXnx1pOzmUeQiNUsPi4d+xK7uIZ4pK8DF3BqCdhSurM7MZqHFGETqGqaFytuyX2F/kexUEQRD+NUSAJghCq/HR9ksEv/E7M1ec5lB8frOBz82WkFduyp75OMpZjYikwkbbXKqb/yzUoyHoGBLqhq2FusXxX4cSCrAuiuYj85+YtncgLBokN4S4TJc2chldddRGucPhpicBOBiXT3GVnMnZcbHpfGv12bMuPg6Yq1WMrwsW44+sg4/90K66j92RcmOS+sYXnvaWfH9/D8zVSvZeyqPLuzvp/PZO+r+9gfi5/bCbF9Do8Vzs3cxUb8JFUQo27jDgOXg6Eu5bDx0mgLou46OxlIOIh7fDM+dg1nFwaNvkmhUKBZN72dK2y6d4hSzFQONMldLWlZOudwAQeO5TqhQK3nWWs5T3h91PP69+LBqxiGmh9wAgOeym2us1bEPe5lP/BPr5teG2ytPErX8IMk/Lwe7ZZVCaTrmtB3OLT5vOtS99H1+e+bLuxCoYNgemLJfHiKUelrsefti24fE/T6JXTkBCwsPKnUPPj2fVo31YOvl2rBXmKNSVuFjFIKkt+LTbWI7kncZCZYGDuQMxRTFM3XQXVbExAFyM2IyExF3Bd/HQQxHMn7QJNys3kkuTefHAi+iNTbs8lmnL+OjERwA82ulR/D3DcekxnemlZWwtk1g79ldWl0FbvR56PAQqNWP8x2CrsSW9PJ0jmVfp+igIgiC0OiJAEwThhhVU1LI/Nq/R42RKEYarBFS1egPxueUtrk8rrGLRwSS0eiPbonK4b/EJU8fA0qprl+RdlSRBViTompYSzt+TgCTBrWHu3NWjDQDHrgzQshvGn9Wz0KgY08kTuGL8l0EHp5fQdu1otpq/xhTFLhTaCsg5D4uGQEJD973O3nY8p17D5PiXoSIHzi6H/NhGzUfOppeQV9Z4PqzTKXKA1t3XEZCzeaGKNB7JfhdqSjGL3cQa5RsMdyujR902II97mzelq2mibBUGvtZ8SbgyvsnzYpAUZLr0lwOXZy/C8LfAyb+FJ7iOox9YObW4+vuo7ymqLeRk7nHePvp2k1JS9cBnqJDkaQy+crSnWGPE1dKTx7s8DoBGqeG13q8wM2wOGC2bHD9HrebpwmMULh4udzs8IAc1X7YLJ7+6AF87X97t9y4AP174sXH5X/tx8OhecO/U7LVHFckBVqeiTJyPvkffS/+j/8ZbGFIuB+d7bGz4tf8jrMjYBcDcgXP5ZewvtHdqjzq/BEWV/Bq2ydbT26M3r/R+BYVDG1zt2/LV0K+wVFtyJOsIH5/8uEnGa97peRRUF+Bn58cjnR6RF/Z7GjTWKLMiCYnagHnmaVBqoLtcKmqlsWJ8u/EArI5d3eJrIgiCILQ+6n/6AgRB+He5kFnKlIURVF42aXO9/u2cWTCtO07WjcfTpBRU8tiy08TmljNnbBiPDGj6Rv+rffEYjBI9/RwJ87Rj3dlM0ouq+WRHLJvPZbHpyQGYqf/gZ0rRG+DXB8ElBKauABd53FZ8bjmbz8vZs9nDg6isle/peJI8Dq2+XC6mmQwawIRu3qw+mc76yEz6tXNmfFdv2DgLzv+CP6CVVNQGj8U2fAoc+hwyT8HyyTDsTejxEKOjZmOjltu0SzbuKCpy0UYsZFf0rQC42JhRUKFlV0wu91w2sfLpNDlAqw++gq2rWWr5GTbGGvIcukBpBu2UWXxb9TyKWHsIHW3a97ZOntwa5o5BklDteAX1ySgkjRW6+zYhuXUwbadSqvA2M/9jz3czcipzWBcvt95XKpRsSdpCoEMg0zs1tH7vGRbEz7+Nop/ZVlbYyc/1u/3fxEpj1ehYT/S8i0e7T8JIQyBTVlvGA1umkEYez7q78UN2FGZApKMXa0rkyaXn9JlDb8/eZFZksvD8Qt6JeIe2tm3p7t5dPohrCDx+CAxXjPsrz+bCrplQnU6HqnI4usC0apjCjy0Y2ejiQUX6NgCe7vY0w33lLo9Lb1vKT98+ARwDwLMYPun1HhqlxnSM9s7t+d+A//Hs/mdZdWkVBzMOMjl4MhPaTSCjPINf434F4M2+b2KmqvvZsnaBXo/CkXmwf668rMMEsGkYgzc1dCrLY5ZzKOMQGeUZ+Ng2dBEVBEEQWi+RQRME4brlldXw6NJTVGoNeNlb0NHbzvSw1Kg4klDIuAWHuXDZ5Mv7YvO4/avDxNZlzz7d0bSVfWphJb+dkbNGr45uzzvjO3LiteF8emcXnKzNuJRTzveHGpcH1tPqjaw6kdbQer055+U3uBTEwvdDIfZ3AL7cE48kwcgO7nTwsqdLG3ssNEoKKrQk5MnHkyTJlEEL9bBrdNhefk4Mb++OVm/kmdWRrFq5GM7/ghElc3XTeMZrFbb3LIXQMfDQtrrshgR73oEvOmKTtpcaScNs7UwyBs8DQHF+NWp9JYGu1jzUXw5kd17MNZ2zpKrh2pwci9kctw7d6rtxM+aTZPRgcukzjK5+n1O0R6OvhNXTYPfbUNVQhqlWKTE/+xPqk4vkc05ciFnbnphbWJke6psYnAF8f/57dEYdvTx68Vqv1+Tn/8yX7E5tyCiaqZVcaDeZ55zbYlQo6OwwmAHeA5o9nlqlxkxlZnq4WLmwYOT32GpsOWthxjtdbkUXMJR3vNogIXF74O309uwNwMyuM7nV91b0Rj2z983mq7Nf8XXk1/Lj3DfszToCavOGh6MfF5Ry6WGnfi9D2ATodBfcv5F+jxzBXGVOqbYcg2RgbMDYRkGnhdqCycqepu+VEpgnXzEhNzDcdzhz+szBVmNLZkUmX575klt/vZWn9j4FwMR2E+np0bPxTnVZNJNeMxqt9rXzpZ9XPyQk1sSuucYrJAiCILQWIoMmCMJ1qdEZeHTZabJLawh0tWbdzP7YWzZkAWJzypmx7BSphVXc8e1R5k7qRHZpDZ/ujEWSoHtbBxQKBadTi3lz4wV+fLCnKUO1YG8CBqPEoGBXureVs0KWZiomh/ugUsKzv5zjyz3xjOnkiZ9LwxtSSZJ4bX0Ua09nEOBqza5nB6FSXtGpTlsJiXvkr906QN5FWDWVgh7PsS2qO6DkmWFyswdztYpwX0eOJBRyLKmQIHdbcspqKKvRo1YqCHSzbnRopVLBwvvC+WxnLD/uj6b/pbmghJXcxkLDOBYO6NKwsdocbp8PXt1g24ugrQD7tryleYkNGU70kzrSxjkITWE8E1WH8eg2i5EdPPhkRyxHEwsoq9FhZ6HhTF32zM/NyNP7p1OqLWWdvoaPLR14tPQF0rQWgAXbuy+ih2YFHP8ODn8BEV9D2Hh5LiyDFra9JF/XsDch7PY/95/jGrIrslmXIGfPnujyBD08epBYmsiqS6t47fBreFp7UlRTxNq4tew37sdoYQSjJZ8Om3ND5wmwD+DTwZ8yc/dMNpXFkuLSmYSCBBzMHXihxwum7ZQKJR8M+IDMikyiC6NZeH5hk2P9POpnU2atoLqA7MpsFCgI6/YQ9H7KtJ0V0NerL/vT99PFtQtv93u7caMSoDYurtH3NRejserevck57wq5i3GB49iZspO1cWuJzI+kpLYEJwsnnu/xfNMbtnaG3jPk19ejM/j0bLLJtNBpHM06yrqEdczsOhMLtcXVnkJBEAShFRAZNEEQrkmSJF749Rzn0ktwsNKw+IGejYIzkCdw3jRrAENCXKnVG3luzTk+2SEHZ3f3bsuqGX34eHJnzFRK9sXmszVKziKkFFSaxlzNHh7U+MS1FUzI/JzZXtFo9UZe3xDVaNzSooNJrD2dAUBSfiWbzjUzcXTiXrmNuoMvzNhvyjK4nPqcb9RfcluYK2FeDZmxPv5yV7xjSXLGqX6C6kBXG8zVqiaHVykVvDQqlK2dj9JWmU+m5Mz/au7Ay96CYaFuTbanx0PwyE4Y8gbM2I9DQA8AzmWUUtZJHj90v2on47t40c7NhkBXa3QGif2xcue/U3Xjz8zdtlKqlTOVpywtmNa2LXaBDSWY9/QLhNs+gjuXyG/eDVqI+hWWjIFlE0EyyBMjD3iu6TVeh4isCF4++DLpZU27TF7p+6jv0RvlsVc9POT7fannS/Tz6ke1vpqpW6cyc89M9qbvxYgRb4sw3ujxKZ42zTx/19DPqx+v9HoFgPMF5wF4occLOFo4NtrOUm3Jt8O/5bHOjzE1ZKrp0d1NDpzejXgXXV3L/YsFFwHwt/fHxsymyTlf6vkST3Z9kq+GfoW5qmnmsTZWnlvPsoscsNfExLR4/ZZqS8a3G8+y0ctYf/t6ZnWdxdfDvsbe3L75HQa9DMPegjt+aLaN/kDvgXhZe1FaW8rvyb+3eF5BEASh9RABmiAI1/Tlnni2nM9GrVTw3b3hjbJYl7OvC96eHtoOADOVkg8ndeJ/EzthrlYR6GrDzCGBALyzOZrSap0pezY4xJVubRu/iWbPuyhOLeapqm+wUkscSShkQ6QchO2KzuXD7ZcA6NbWAYAFexLQG65oKR6zRf43dCyozSga9AELHZ+nVtIwSnWS14JSG23eN7A+QJPHocXUTUQd6tl4/FkjORcIjP8RgK8tH6cKCx4e4N8wz9eVvLvDoBfB2pnOPvK1n8soYZ3xFiolc4KVmbQpOwPAiA4eAOys6+Z4OrUYK6uLZBmOoJAkPsgvxM/MkVxtCSmaj1Hbn2JYqBv+9a9Rh4nyuKoZ++XsWX2A4dMLxs2/4bmxJElicdRiHt/9ONuSt7H4wuKrbp9VkcX6hPUAPNH1CdNytVLNJ4M+IcA+AAA7MzvubX8vG8ZvYPuUX5jS6ZYbuq7LTQ2Vgy2A3p69uT2w+Qyhk4UTT3Z7ktf7vG56zB86HycLJxJLE/np4k8ARBVEAdDRpWOzx2lj24bHujyGg4VDk3XG6mq0qfL/MfuJEwGoiY6+rvto59iOx7s83uJ5AbmL5sDn5PFzzVApVdwVchcAa+PXXtd5BUEQhH+WCNAE4V/g56MpjJl/qEk3v7/D/tg85u2Wu/x9MLEjfQKcr7q9UqnguREhbHlqADuevYWpvRq3XH9icCABrtbkl9fy3C+RpoBr9vDgxgfKOA0n5DFSqpoi3g+Xx629tyWGowkFPLP6LJIE9/Zpy7JHeuNopSGpoNLUMh+QOyrGbZe/bj+WC5mljFtwmLnZ4SxnFABt4lc0Om1nHwcsNEoKK7XE51WYMmhXjj8zMRpg8zNyRqr97bz+7HOsfbxvs41QmlPfav9SdjkrIktYb6gbc1V37yPC3AHYH5tPZa2eoswovD2XAjC1vIrbB73Lyju2MrjNYPSSDkuvtfToEtn0RF7dYNyX8PwluPtXuH8DaG6s3K1KV8XzB55n3pl5pk6D+9L3YTA2bRhTz5Q98+xNuHt4o3V2ZnYsvW0p3w7/lr137eXlXi8T6BB4Q9fUkld7v8rCWxcyb/C8JiWHV2Nvbs9LPeXyz4XnFpJalsqFQrnJSCeX5js8Xk1tQgJIEipnZ2wGDjAtM2qbTkBeuHgxSRMmostsJhP8J0wKmsRT3Z7iyyFf3tTjCoIgCH8NEaAJwr/AooNJXMwqY0d07rU3vsl+PSWXEE7r1YYpPZvOb9WSjt72DVmcy5irVfxvovxGd8+lPAxGiSEhrnRt49CwkUEvBz1IoJSHyo63OEuwuw1FlVru/uE4VVoDA9q58Na4DtiYq3n0FjkTs2DvZVm01KNQUwJWzqwv9OaOb4+SWVKNr7MVQ+55FVBA0j4oSDCd2kytpIev3Cr+UEIW0TlyaeGVHRxNTv0od2c0s4XbPsLaXE0PP6dGQUG5tpxqfdMW/wDeDpY4W5uhN0rE51WwUhopr7i0FUoz6eLjgLudORW1enZvXMYIp4/IMVPgZpB4esxi6DkdWzNbvhzyJY91fgyAn6K/p6im6dxs8gtgC8EjwKz5LGhLUstSuWfbPexK3YVaqebVXq9iq7GlqKbIVEp4pcyKTDbEbwBgZpeZzW5jb27PAO8BzZYG/hlKhZJ+Xv2aLUm8ltH+o+nn1Q+tUct7Ee9xoUAO0K6ayWpBfXmjRUgwai8vVPb2oNdTG9d4agNJp6Pgu4XUXrpE4U9Lbvg8V+No4ciMzjNwsXS5qccVBEEQ/hoiQBOEVi67tJrMEvnN/dXmEfsr6A1GDicUADA5vM1NO26fAGfu6tHQ8rtJ9uzYN5AbBZaOMOYzAFSxW5g7seENcoCLNV/f3R1NXRnh/X39cLTSkFxQycbIuizaJbm88axlX55dc4FavZEhIa5smjWAgOAOEFwXDJ384YrrcwIMfBc/i2zbd0BZ1XyJY34s7H5H/nr4W2Dn1WSTrIosRv02ikG/DOKto29xPv98o3F0CoWCLpcFp94h4eDbX87InV6C0lDDix5nWWP2Dh3iX2Slgzz/12v93sImYIhpP6VCyayus+jg3IFqfTVLLixper03yCgZiciK4Pn9zzNh4wQSShJwtXTlp5E/cXf7u7mljVyGuDdtb7P7/xj1I3pJTx/PPg2t7P8FFAoFb/R+A3OVOcdzjlNaW4pGqSHYMfjaO1+hJlZuEGIeHIJCocA8rL28PKZxmWPliRMYy+Wf79INGzBWVv7JuxAEQRD+rUSAJgit3OnUYtPXcX9zgHYuo5TSah32lhq6+LTQpOAPem10e/oEODF9gH+jAIXi1IZ5nW59T25nrraEkjTCLbJ47tZgOvvY88MDPbC3amhUYmOuZsYtcnncgr3x6PUGDHXjzxZkhwLw1NB2coOT+v16Pir/G7kCahva9PcNdEZpkUWNIhelpgxb96N42F1RDlhVBCungLZcDqh6PNzsfS46v4gybRnV+mrWxa/jnm33MHnzZFZfWo3OKDeh6HzZczuxmzf0rGvTfvw7+CyEyWnv00MZy9suTugVCvwsejA0ZHKTcykUCmZ2lTNVq2NXU1hd2GSb61Glq+KHqB8Ys24MM3bNYGfqTlOTj1/G/kJXt64ADG0zFIA9aXuaTDpdpi1jc9JmAGZ0btz+/d+gjV0b0wTZACGOIQ1zkN2A+gyaeYg8RsyifZi8/IpGIRV79pi+NlZUULp58w2fSxAEQfj/QQRogtDKXR6gxedeZa6vv8DBOLm8b0A7l5YbXvxBDlZmrJ7RlzfGhjUslCTY+jzoqsB3AHS7F8ysoN0weX3MFp4eFsSmJwcQ4Nq0dO3+vr44WZuRUljF0nUbUJVnUSmZE6npysL7wnl+RAjKy9vwBw4FpwCoLYOohnmiOnk7YGGbYvpeaX+YMm1Zw356LfxyHxQny90h71oKyqYdHjPKM9iYsBGAN3q/we2Bt2OuMieuOI4Pjn/AozsfpbC60FTeaWuuZmioG7QfBzYe8nXVlCLZt+Ely5GcszBHMpjxROeXWhxXNdB7IJ1cOlGtr+anCz9d41Voqr6U8cszX5JRkYGNxoapIVNZO24tP4z8AVcrV9O2A7wHYKY0I708nYSShEbH2ZSwiWp9Ne0c2tHDvccNX0dr8ECHB2jnIDe8+SPljZIkNSpxBLAIk/+/11xsyKBJRiPle+QspPUAeZxa8YqVTYJeQRAE4b9BBGiC0MpdHqAVVmoprKhtdrujiQV8viuOilr9TTv3gboAbVCwa+MVp5fIY69uRN6lJhMmN3FxHSTsApUZjP2iocNg6Bj530tbr3oKa3M1M+rGolWe3wTAKU04a2YNZmRdN8RGlMqGbNWJH+QAEXkcmqNTGgCSpMCoqOHniz9TtwC2Pgeph5HMbPl1wKNsyz3R7PX8EPUDeklPX8++TAmdwgcDPmDPnXt4uefLWGusOZ17milbpuDkmMtTQ9vxxZSuWGhUoNLApIVyVu7e31A8c54LXnL5pL6sO0MCWy61UygUPNFF7pb4S+wvFFQXNFqfVZHFJyc/4ffk39EaGjeqOJhxkGlbpplKGd/t9y577tzD631eJ8SpaZdAK40Vfb36Ao3LHI2SkV9ifwFgasjUG2rS0ZpolBq+GPwFdwXfxUMdH7rh/fV5eRhKS0GlwixQzu5a1Jc4xsYiGeTmKjUXLqDPzUVpZYXXh3NRWFpSGx9P9alTN+9mBEEQhH8NEaAJQitWpdVzMUvO3NiYy80y4vOaz6K98lsU8/fEM+HrIyTm//lMW3GllvMZJQAMDL6suUDcTrmBx5Zn4fii6zuY0Qi/PSJPqLtldvPbVBfD7/L8VQx4DlwvC0KCR4FCJY9LK0656qnu7+uLi40ZI5UnAeh52320c7tKi/yud8sllHkXIS0CAL1RT7VKzghpC+QyvhUxKyipKZEnfD67DBRKFve9h3cvLuLlQy+zPn59o8Nenj2rLzsEuSnGvWH3snLMSvzt/cmtyuWhHQ8S4B/N8LqOjQAEDJaD1HbDQamkRiOXxPlZdcfSrGm27nIDvAfQ2aUzNYaaRlm049nHmbJlCkujl/LSwZcY/utwPjv1GUmlSXx37jue3PMk5bpyurp25ZexvzAxaCJWGqurnmto24Yyx3rHso6RUpaCtcaasYFjr7p/a+dn78ecvnPwsmk6vvBa6rNnZv5+KM3lJihmvr4orKyQamrQJicDUL5bfu6sB92C2sUF+3HjAChasfJm3IIgCILwL9OqAzSDwcCcOXPw9/fH0tKSwMBA3nvvvUZlH5Ik8eabb+Lp6YmlpSXDhw8nPj7+KkcVhH+Pc+mlGIwSHnYW9PaXOws21yikoKKWtCK5DX1CXgUTvjrC7j/Z8fFwQgFGCULcbfG0lxtToK2USxDrbX8ZEvY0f4DLXdoMuXInPKI3Quz2ptvsfgcq88A5SJ7X6XJWTuDbr+5YV8+iWZmpWT/FjWBlJpJSjVXYbSSXJvPQ9od4/9j7xBbFNt7B0hE63yl/feJ7AGIKY9BJ1UgGS7QFw/C1CaJKX8XSo+/Bzjfky+37CF+mbTMd5t1j73Iy56Tp+++jvkcv6enn1c80ZutyAfYBrBy9kqFthqI1annz6Jt8Hfl1s/eUVpZGYW0WKoWaLyc0HXt2JYVCYZpzbE3sGgqqC/j54s/M2DWDktoSAu0DcbNyo7i2mCUXlzB+w3i+jvwaCYkpIVP4ceSPjUoZr2Zwm8EoFUpiimLIrpAnH18VuwqA8YHjsdbcWLfI/09q4+QGIRbBDdlHhUqFRd14tPoJq8vrxp/ZDhsOgOM9d8vLd+9Gl5v3t12vIAiC0Dq06gDto48+4ttvv+Wrr74iJiaGjz76iI8//pgFCxaYtvn444+ZP38+3333HcePH8fa2pqRI0dSU/P3zxclCDfbmTS5vDHcz5EgdzkLFNfMOLT6TJePoyU9/Rwpr9Uzfekp5u2Ow2i8wXEsdR+A1I8/u+Xy7Nn+uVCaBvZt5OYdkhF+fUjuZtgSoxH2fyR/bVfXuXHbC42acpB2DE7XZXrGzQN1My3XQ+syMdcI0ADa5O4DQOE3EKOFPW8cfoNTuaf4JfYXJm+ezN1b72Zd/DqqdHJQa2oWErMJynM4mSsHWhpdO9ztLHmym5wBW5G+m2KlgpiO43gt/wAgl/CN9BuJ3qjnuf3PkV6WTnp5uil7Vl9u2BwbMxu+GPIFs7rOAuSuh+XapgH4kawjAHR370aI2/UFTv29+tPZVc6iTd0ylU9PfYpRMnJ74O2sHruaHXfsYMHQBQzyGYRSoUSj1PBOv3d4o88baFSaa5+gjpOFE11duwKwN30vWRVZHMw4CMCU0CnXfZz/j0wdHEMal4dePg6tNikZbWIiaDTYDJK7YlqEhGAZHg56PSVr1nAzVJ0+jaS/eeXPgiAIwl+nVQdoR48eZfz48YwZMwY/Pz8mT57MiBEjOHFCHu8hSRLz5s3jjTfeYPz48XTu3JmlS5eSlZXFhg0b/tmLF4Sb4FSKPF4rvK0jwe5yU4zmOjmeSy8FoJefEyum9+GBvr4AzNsdz4xlpyir0V3fCY1GWDoe6csulMXKAcigYDd5XfZ5iPhG/nrMZzD+K2jbF2pL5W6GLY0ti9kklw+a28EjO8G+LZSmN3Rq1Gvr5jxDbgriN6D549SPQ0uLgMqC5rcBKM2AMz+b9lkTu4bzBeex1lhzq++tqJVqogqieOvoW0zaNInS2lLw7AxteoNRD2eWciJH/h0zs88Itj9zCyP9h9HeMZQqjHzh5MCT+jSq9TX08+rHy71e5v3+79PRuSMltSU8ufdJ5p2eh0Ey0N+rf7PZs8spFUoe6/wY7RzaoTVq2Z26u8k2RzLlAK2/V/+rHutyCoWCWV3kwC+3Khe1Qp677P3+72OhtkCtVDO4zWC+GvYVuyfvZtukbUwKmnTdx79cfZnj3rS9rIldg1Ey0tuzNwH2AX/oeP9fNHRwbDxm0DQOLSaG8j3y623duzcq24ZSXMe7pwFQsmYNku46f36bIRmN5H/9Nan33EveF1/84eMIgiAIfx/1P30BV9OvXz8WLVpEXFwcwcHBnDt3jsOHD/P5558DkJycTE5ODsOHDzftY29vT+/evYmIiGDq1KnNHre2tpba2oZGC2Vl8hgfnU6H7k/8IRSEK9XqjcTmlNPByw6V8sYaJRiNkimD1tXHFgXy/nG55U3+n0bWbdfRyxaFZOCN0SGEedowZ1MMu2PyGL/gMN/c3ZV2bleftFeRuAd18gEUwFfS23xodh9dvYaiq61BtekplJIBY/vxGPyHggRM+gn1kpEoipMxrr4Hw91r5QYf9SQj6v0fogAMPWdgtHJDMepj1L9MRTr2Dfr2E1Em7kGVfwnJygX9kLegpZ9Baw/UHp1R5JxHH70Fqes9Ta8/9TCqddNRVBUgWbmQ3aYvX+6Ts2OzOs9iashUimqK2Jy0meWXlpNZkclvsb9xX/v7UHS9D3X6cbQX13HWXs4i9vPsiY2ZAr1ezwzPwTxbfIn1tjZQW4yfnR9z+81FMkioUPHZwM+4b8d9JJUmkVSaBMCMjjOu+3fKbb63saBkAZsSNzHWr2HcltagNQWMvd1739DvqB6uPRjlO4pLxZd4vdfrhLuFo28mi+KgcQD4w7//bvG6hU/5lNO5p00lpHe2u/M/8/tU0mrRJidjFhxsaogi6XTUJsn/D1SBgY2eC3WwHLDVREdjqJazuFZDhjTaxnLIEFQuLujz8ynevgPbUSNv+LoM5eXkvfY6lfv3y99XVaPValEoFP+Z10YQBOHfqFUHaK+88gplZWWEhoaiUqkwGAx88MEH3HOP/MYsJycHAHd390b7ubu7m9Y1Z+7cubzzzjtNlu/cuRMrq6sPiBeEG7E+Rcn+bCWh9kbuDzJiff2VY+RUQWm1GjOlRMrZIxgkUKCiuErHLxu3YVt3LEmCU8kqQEF56gW2FcljvSyAp9rD4lgVyYVVTPj6CPe0M9LFueWSx96Jn+MBlCodsDeWMEexhLTFCZRbeNMhOxKdyoo9qmHUbmsYe2Xr8RgDy95Fk3aUwq+Gc8pvJlqNHQBexSfomR+DTmXFztJA9HX7hTv0xqfkONUr78e6Vh5jc8Z1Mhn7Iq76nAQr2tGe8+QfWsKJLMeGFZJEQP4OOmSuRoGREsu2nPB7hh/3vU+FrgIflQ82CTZsS5TP74orAxQD2MhGfj73M45JjpgblIxCyaXSRKqsPbBUWBJ3NI4EhdwsJCQrgvYqLTHmZlgqLJkoTeTQ7kONrm+yajLf8z06dASrg0k/kU466Ve9p3rmRrms83TuaVZuWYmD0gGARF0i1fpqbBQ2xB+NJ1GReF3HqzeAAQxQDiD3VC7b2HbtHf4gD6UHOcYcSrWl2CvsqTxfybaov+58rYnz9u0479tP8YAB5I+Tg2uz7Gz89HoMFhbsOn26oSMpgF5PkEqFsbyc2vNRABwzGjBsa/x8OXfpgvOePWS89x5ZaanU+vhwvcxy8/Batgyz/HyMajV5EycQ17UL/P47AFVVVX/yrgVBEIS/SqsO0NasWcOKFStYuXIlHTp0IDIyktmzZ+Pl5cUDDzzwh4/76quv8txzDU0IysrKaNOmDSNGjMDOzu5mXLogoNUbeevjA4COS6VKvk205pu7uxLq0VDGJEkSx5OLSciv4M7u3phrGrrzrTmVAeei6ebrxLixPQGYH3+I9OJqfDv1oU+A3DQkvbiKymOH0agUPDJpZKNjANxZqeWZX85xPLmYH+NUTO7ujZttwxgvjUrBuC6e+CrzUZ89B8A7Lp/imLmP1zWraFt02LSt8tZ3GBZ+d5N7VSQFI619ANeKaEalfYh+8s/g0Qn19/+T9+s3ixG33NmwQ0U40nd9savJBMDoP5jO097DqTKTEzknuD3gdtTKZn495fnB9+vwqIxmrFUk1GUVFfkxKDPlN7fGjpOxHv05qrxTXDxwEZVCxacjPiXYsXGZ2RD9EPas30ORrgjHbo709xoL5as4USw/B328+zD2loZMlurHz3mroIj5IX15tO+bdHHt0vT6gPZZ7VkVu4oXwl/Az86v2W1asm/3Pk7nnUYboGV02GgAvjz7JcTAYL/BjO3bejsipp5P5fsLcpOVezrdw7iO4/7hK/r7ZG7YSDXgePgwwUOHYn/nZMo3byEXsA4LY/SYMU32SV++wjRZtXnnzoxspuLD0K8fGYmJkJKC38JFuM6Zg92E8Y220aWnU7FzF8bLAi5Jp6N0zRqkykrUHh54zPuC4A4dGu1XXzkiCIIgtD6tOkB78cUXeeWVV0ylip06dSI1NZW5c+fywAMP4OEhz2uUm5uLp6enab/c3Fy6du3a4nHNzc0xN2/ahECj0aDR3ECKQxCuYn98LiXVOlxszLA0U5FeVM1di07w8eTO9At0Zu3pDFafTCe5oBKA1KIa3r694U1UZIb8BqqHn5Pp/2WIhy3pxdUkF1UzMEReFp0j79/e0w4bK4sm1+HhoGHF9D7M/f0Siw8ns/ZMZpNtfjySyubQnfghYfAfwpY4G7SGMTxyxzi8ds2E6iLw6YWq13RUymaGrobcCo/ugdX3oChKRLN0DHScDPmXwNweVb8nUV3+s+XoA7e+I7fqV1ugHPcFCo2GV468QnRhNBX6Ch7p9EjT83h1BqcAFEVJqI583nidQgUjP0DZ+3Fq9NV8dEpuTHJ/2P10cOvQ5FAajYaJQRNZFr2MNfFrGOw7GNrfzqkzlwDo7dW74fdBZQFkn6MDEgtHLALbZuZUqzPYd7B8rD9gXOA4Tued5vfU33m0i1yaGZEjZxUH+Axo1b+fRvqP5PsL36NRargz9M5Wfa03myE72/R1/v/+h2VAALpEOfNqGRra7HNh0SHMFKDZj7i12W00rq74r/2VrJdepmLvXvLmzEEXE4PbC89TceAAxWvWUBVxrMXrsurZE+95X6B2dm567P/Q6yMIgvBv06oDtKqqKpRXvBlUqVQYjUYA/P398fDwYM+ePaaArKysjOPHj/PEEy13ThOEK608nkZ8Xjmv3BaKufrqc0xdrw1n5UBoYjdvZg5ux9Orz3IovoCnVp1FrVSgr+uuaGWmokpr4OeIFCZ286ZLGwegYYLqHr5OpmMGuduyOyaP5Kw82PgpBN3KufQgADr72Ld4LWqVkjljw+gX6Myh+MYNNiLTS4hJz8P+0mpQQJT3FLQxRnwcLfHsNgQCDsKFtdDlbnli55a4tYdH98K6GRC/AyKXy8v7zgRLh6bbd39Q7gLpFAhOAZzLiyS6MBqAJReXMDV0atMW7QoFTPoBon6V962nVEGHidCmF0bJyCenPiG7MhtvG28e7/J4i5c8JWQKy6KXcTjzMOnl6XgEj+BM9JcA9LALbNgwcR8ggXvHqwZnf9atfrfywfEPiC+OJ7YoFkcLR+KK41CgME0I3VqFOIXw4cAPcTR3xMXS5do7/D8hSRK6ugDNqlcvqk6cIOOZZ9DUld5f2cGxnkVYGKX8BoDNsGEtHl9lY4PPVwso+PZbChZ8RfHKlRT/+mvDWE2FAuv+/THz82u0n5mvL45Tp6AQgZggCMK/TqsO0MaNG8cHH3xA27Zt6dChA2fPnuXzzz/n4YcfBuQuZbNnz+b9998nKCgIf39/5syZg5eXFxMmTPhnL17416jRGXh700W0BiOl1To+u7OLaaD/H1VWo2NXjDwP2YRu3jham7HkoV58siOW7w4kojdKdGnjwLSebRjXxYvX10exITKLV9dFsenJ/pTV6Emqy6x1a+tgOm59J8e2qb9B2TKI+pUcx28AC7r4OHAtw9q7M6x94zGbOoORrUs/xTG1ggzJhQcOOwBGBgW7ys+DQxsY8Oz13bilA0xbDQc+hAMfgZUL9G4hQFIqoed007erY1ebvi6pLWHVpVVM7zS96X4+4fKjGeXacl47/Br70/cD8EafN6460bKvnS/9vfpzJOsIa2LXMKztMKqVShwMBoKyL4F3H3nDhLrOiu1afiN9M9iZ2THIZxC703azNWkrAQ5yF8Qw5zCcLJyusfc/b0xA01K+/+8MRUVItbWgUODzzdekPfIINefOU1sqd1a1uKKDYz3rnj1BqcSiQwfM/f2veg6FUonrrFlYhIWR9eJLGCsqULu54TD5DhzuuAONt/dNvy9BEAThn9OqA7QFCxYwZ84cZs6cSV5eHl5eXjz22GO8+eabpm1eeuklKisrmTFjBiUlJQwYMIDt27djYdG01EsQmhOVWYrWIGdj1p3JJMjNlicGB15jr6vbHpWDVm/kEadIwpbNAn0tKuAV4EUbiQq/kdhP/R7UcsfDN8aGsT8un+jsMn48koy/ixyIBbnZ4GDV0BUxyE0ev9azbJe8QF/DtLwv2MwrpszbjdIoFUzQynOLrTLeSqlOfi4GBV/ffFtNKJUw5DXoeAeYWTefPbtCYXUhO1N2AnBv+3tZHrOcJReXMC102nVPdJxUksQz+54hpSwFM6UZc/rOYYB3Cy37LzM1dCpHso6wPmE95iq59LlHTS3K2G3Q40F56oHEvfLGgX9tgAYwNmCsHKAlbzXNL9bf+/rb6wt/L11WFgBqV1dUNja0+eorku+agr4uq2YeFNTsfuZBQfj/tha16/X/nNkOGULAls1oU1Kw6tEDhbpV/wkXBEEQ/qBWPQ+ara0t8+bNIzU1lerqahITE3n//fcxM2t4w6pQKHj33XfJycmhpqaG3bt3Exzc/CeWgtCcUylyKaGLjfz/6uMdl9hxsWkX0LIa3XXPJ7a+rrzxfvPDKKqLQVdleqj01dgnbJAna66bFNrFxpzXbpPnRvpiVzwbI+X9w30dGx030NWGAGU2HUlEUqgwqszpp4jiLrMIAl2v3kK/RZmnITsSVOaMeeBlvB0scbczp3+7hjK1vKo8JOkGJ7x2DQH76+s6ty5+HTqjjs4unXmhh9xco7S2lJUxK5tsqzfqTSWA9Y8tSVuYtnUaKWUpuFu5s/S2pUxoN+G6zj3QeyDeNt6U1pby80V5/rQeNTWQtB9qyyH3AlTmgcYa2va53rv/wwb6DMTWzJa8qjx2p8mZuxuZ/0z4e+ky5QBN4+UFyIFam2+/QeXggFXfPiitW/6AwaJ9e9QuN1YOqvHwwLpPHxGcCYIg/D8mfsML/3n1Y70eHxRIamEVy46lMnt1JGuf6EuYpx0RSYWsOpHOjgs5WJmrWPt4X9q52bZ4vKySao4lFwISPjVx8sKpq8C9rlFF5ilY+4g8mbJrqDxGC7izhw9rz2RwIrmILeflT9+vDNAszVTcb30CdFDiOZAMm050ilvA6+plqGqeB6s/UAZ3Qu68R8dJhLXzZ/+LvhiMEhYaFVW6Kt6OeJvfk3/nufDneKjjQzd+/GvQG/WsiVsDyNkslVLFY10e49VDr5qyaDZmcvCZWpbK7H2zSShJaPZYPdx78OmgT3G2bNoUoSUqpYq7Qu7ii9NfUGOoAaCXmRuUJcmljUXJ8ob+A0HdtLnQzWamMmOk30jWxq3FKBmx0djQybXTX35e4Y+pH39WH6ABWISG0m7/PhSXfZgoCIIgCNerVWfQBOGvJkkNk0F393XkrXFhDAxyoVpn4OElJxn62QHu/v44m89loTUYKanS8cjPpyiu1LZ4zE3nspAkGNkWVNUFoFBCwGBw9JUfHe+AEe/LG+98HeLk0j6FQsH/JnbCTNXwY3llgIYkMVo6CMB5p5GsMZ9ErNEHe2Mp7JrTeNvqEojZDPmxLT8BpZlwcZ38dU+5a6BGpcRCoyK9PJ37fr+P35PleZN+if3lxrNo1+FA+gFyKnNwNHdkhN8IAG7zuw1/e3/KtGWsvLTStN20LdNIKEnAUm2Ji6WL6eFm5cZDHR9i0YhFNxSc1ZvYbiJmSvnNtKO5I4FBdWOpLm1tKG9sN/zP3+x1GhvQ0E6/j2cfNErR6KG1qi9x1Hh5NlqutLBAcbWmOoIgCILQApFBE/7TkgsqKarUYqZW0tHLHrVKyVd3d2fiN0dIyq8EarExVzO+qxdjOnny0m/nSS2s4vHlp1n2SG/M1E3fgNV3b7y7bQnkAS4hYHZFo4q+s6AgFs4shbUPwyM7wT2Mdm42PDE4kC/3xONsbYa/yxXlURkncdNnUyFZsFfRk7NZlUTrHuE383fg7HLoMg1UZnB6CVxYB/pqeb+2fSH8QQgbD2oLSDksbxOzCQxa8OrWqPHGkcwjvHTwJcq0ZThZOFGtryazIpNz+efo6tb15jz5dVbFrgJgUtAk0xgwlVLF450f5+VDL/PzxZ+p0dfwfZSc6evm1o3PB39+UzsFOlo4Msp/FJsSN9HDowfKgHEQsQDidsilqQCBQ2/a+a6lm1s3vKy9yKrMop93v7/tvMKN02XXjUG7LIMmCIIgCH+GCNCE/7T68sYuPvamYMveUsPPD/Xim/0JdG3jwNjOXlibyz8qPz7Yk0nfHOV4chFzNlzgwzs6Ner4GJNdxqWccsxUSnpbpMsLPTs3PbFCAaM/g8IkSD0MKyaD/y0APG2UuLVNOdr2k5p2kzz/CwA7jD2JzK7lUnY5eimEio73YXNhGSydAMbLxsk5tJWzZGkR8uP3l8DKGYqSGrbx7AK3f2X6dmXMSj46+RFGyUgnl058PvhzFpxdwKbETWxJ2vKHA7T44njWJ6xnoPdAenv2RqlQklSaxPHs4ygVSu4KuavR9iP9RrLw/EKSSpNMwdnUkKm81PMlNKqbn1F6NvxZrNRW3N3+brD1BWs3eewZgKM/OP+5xjE3QqlQ8v6A9zmYcZDxgeOvvYPwj2nIoIkATRAEQbg5RIAm/KfVB2jhvo3HbrVxsmLupKaBVbC7LQvu7sYjS07yy6l0gtxtmD4wwLS+Pns2NNQNi4IL8kLPLs2fXG0GU5bB90OhOBnOyZkkFdARoGA7+Do1ZG70WjkrBmww9OdchtzG28XGDOsx70HKTqjIBbUldJwkZ8x8ekJ5NpxdIWfrStOgphTMbKDTnRD+gJw9q1Opq+STU59glIzcEXQHr/V+DTOVGWMCxrApcRM7Unbwcs+XbzhAqtZX8/Tep8moyGBZ9DJ8bHy4I/gOkkvl8V2DfAbhZdP4Da5KqeKJLk/w4sEXMVOa8UafN5gYNPGGznsjXCxdeL3P6w0LQkfLWUb4y9vrN6enR096evT8288r3Bh9fZMQTxGgCYIgCDeHCNCE/7SGAM3xGls2GBLixutjwnhvSzQfbIthxfE007qsErmkcEI3b9h1Xl7o0UwGrZ6VEzyyCy78BobahuWpERD3O6x5EKbvBtdgSNwD1UVI1u4cq+1g2rSzjwMKS0d4cCtknoHgkY1b29t5waAXYeBzkHwAqoshaCSYN+36eDbvLHqjHm8bb97q+5Ypg9fbozculi4UVBdwJOsIg9sMvu7nC2DhuYVkVGRgb26PwWggoyKDL898aVo/NXRqs/uN8h+FlcYKH1sfAuwDmt3mLxM67rIA7e8bfyb8exgrKzHUzXem8RYBmiAIgnBziABN+M8qqdISn1cBQPfLJoO+Hg/39yO5oILlx9JIrptQup6HnQVDfNVytgrA4xod+Gxcoc8Vkzn3fhx+vh3Sj8HKu+DRvabyRkWnyfhcsDOd1zRBtUuQ/GiJUnXNcVQnc04Ccvbm8vJKlVLFbf63sSx6GVuSttxQgBZXHGdqX/9uv3fp49mHnak7WRu3lnP552jv1J4+ni23r7/F55brPtdN5T9QLhHVa8Fv4D9zDUKrVt/BUWlri8rmD05zIQiCIAhXEAGa8J91Nq0EgABnK5w33A2FCfDYQbCwv+a+CoWC9yd04t4+vpTX6ButC3S1wTz3qPyNo991TdTchNocpixvKH9cfQ9knZHXdb6Ldnn6hgCtzbWv93qdyjkF0Gxp3diAsSyLXsb+9P2Ua8uxNWt5qoF6RsnIuxHvopf0DGs7jKFt5QBxQrsJTGg3geyKbOzM7VAqWmG3O7W5/P9BkprNNgpCcy32BUEQBOHPEgGa8J91KrUIgLHuBfJ8VyD/2/GO6z5GqIdd8yuy68obWxp/dg1xxXFkV2TD8Bdh15uQfwYnpZFOLiHg2YVg91h2RecCconjzVCpq+Ri4UUAero3DdDaO7UnwD6ApNIkdqfuvq7xYL/G/sq5/HNYa6x5pdcrTdZ72ng2s1crYnn9pa/Cf8+Vk1QLgiAIws0gAjThP6t+/Fn9vGIAJOy9oQCtRdnn5H+vNv6sBUklSdy5+U6MklFe4GwLyNmqHzwG0luhINhd/r6NkyVO1jdnMtwzuWcwSAZ8bHyaDZwUCgVjA8Yy/+x8tiZtvWaAlleVx7wz8wB4qttTeFh73JTrFITWwtTB0bOVf9AgCIIg/KuIAE34f+9sWjHxuRVMDvdBqZTHVekMRiLTS1BipF3ujoaNE/fIJW1XtrcHSNoP2iq5u9+15PzxDNqhzEMYJSOO5o742PoAkFuaQp6unN3W1vQGRnbw4I7uPtwa5nbDx2/JydyG8WctGR0wmvln53Mi5wQ5lTlXDbo+PPEhFboKOjp3ZGpI801ABOHfzFTi+Dc2CMlPTaYwI43Q/oNa3CYrvpi0i0VNlru0saVd+M37nSEIgiD8NUSAJvy/lltWw/2LT1BeqyelsJKXRoUC8nxlNTojIyxiUVflyqVsuhq5JX1eNLh3aHyginxYPlmeY+yBzaY5y5pVWwEF8fLXfyBAq2/U8XDHh3mw44MA7Enbw+x9szmadxoAC42Kz+76Y+WTLZ43+9oBmreNN93dunMm7wy/J//OQx0fana7A+kH2JW6C5VCxVv93kKlVN3UaxWE1uDvzqBJksSGT96jLD8PGydnfNp3bLJNdbmWzV+dR19raLIupI+HCNAEQRD+BUSAJvy/9s7mi5TXyk08vtmfSDs3GyZ19zGVN95vfQwqgQ6ToCQNEnZBwp6mAdrFdQ0TQG+eDU8cBY1F8yfNvQhIYOMBNjf2ZshgNHA6Vw7Ceno2BEq9PXqjVqhJK08jvSydNnZtbui411KhrSC6KFo+7zXm3hobOJYzeWfYlLiJBzo80KTBR5Wuig+OfwDAfWH3EeoUelOv9b+qtlrP4TVx+HZ0EW+y/0IF3y2kMiKi0TKlpSVuL76AeWDjycr/7kmqc5MSKMuXJ09Pj45qNkA7uysNfa0BezdLfDs6N1rn5tvCmFlBEAShVWmFrdME4ebYE5PLtqgcVEoFt3eR30C98lsUp1OLOJVajAW19Ko5Im/ceUrDZMT1DUMud35Nw9dFiXDos5ZPXD/+7A9kzy4VX6JCV4GNxoZQx4bAxsbMhi5u8vGOZB254eNey5m8MxglI21s21xzrNgI3xFYqCxIKElgwdkFTdZ/Hfk12ZXZeFl78USXJ276tf5XndmewqWIHPYsiaY0v/qfvpz/l/QFBeTPm0fV8eONHhX791P400+NtpX0evS5cqMe9d8UoCWcbAgcs+IuNVlfVaYlan8GAAPuDGLgXcGNHiG9xThQQRCEfwMRoAn/L1XW6nlzo9yRcPoAf+ZN6cqIMHe0BiMzlp7mWGIhtypPY2aoAgdfaNOrYTLitAjQXja3WWEiZJ4ChQpGfyovO/wF5DV9gwRATn2AduMNQurLDMPdw5uUBQ7wHgBcf4C2M2Unt2+4nV8u/XLt89aVVfby6HXNbe3N7Xmz75sA/BD1A5sTN5vWRRdGszxmOQBv9HkDK43VdV2rcHXV5VrO788EQK8zcmBVLJIk/cNX9f9PdVQUABofH7w//wzvzz/DZeZMAKpOnGy0rT43F4xGFBoNaheXP31undbA3qUxLJsTQeqFwma3STh5zPR1dtwlJKOx0frIXWnotUbcfG2bZM8EQRCEfw8RoAn/KgUVtdTomo6tuNIXu+LILKnGx9GSZ4b4oqwt5YspXQnztKOwUkthpZaJ6vrs2V1yUxDndmDfFgxaSDnccLD67FngEOg5HYJHyeWOW2bDFW+QgD+VQbtao45+Xv0AOJF9Ap1B1+IxDEYD807P4/kDz5Ncmsy8M/Oo0lVd/bx1AVoPjx7XdZ3jAsfxaKdHAXjr6FtE5kWiN+p5J+IdjJKRUX6jGOjz35jcWVujR9fMeJ/L1VTorrnN1dSXrTm4W6FSK0mPLiL+ZO4fPp7QvJqoKPQqC1Td+2I3ejR2o0fj9PBDoFSiS0szNQWpqdRRmSqXN6o9PVEo/9yf0rKCan77+DQxR7Mpy69my9fnOLUtpVEQXpSVSWFGGkqVCrW5ObVVlRRmpJnWV5VpiTogZ896jvVvNNG8IAiC8O8iAjThX+N0ahH9PtzLfYuPYzC2nD24kFnKj0eSAXj/9hCsVtwOnwRiffYHfrg/HFdbc5woY5CyLpDqdJf8r0JxWZnjHvlfSYLzdRmozlPkbUZ/ChprOdN2dmnjk+u1DZm1G2yxrzfqOZMrT0bdXIAW6hSKk4UTVfoqIvMjmz1GaW0ps/bMYvGFxQBYqi2p0FWwJWlLi+ct15YTUxQjn7eZ+c9a8mS3Jxnedjg6o45n9j3Dl2e+JLowGluNLS/3evm6j/NvlngmjyUvH2HFW8coL6ppdpuc5FJ+fv0ov354Cr32xoO06vKGsrX+d7Sjx2hfAA7/Gk9NZcuBunDjUqMKONrnXX4vG8Tv30WRerEQhZU1Fh06IKEgbstptn5znh9fOMSq5WVcaP8Qpd5d/1Q2Mz26iDVzT1KYUYGlrYagnu4gwfFNSWxfeAFtjTyGtr68sU2HzngFhQCQGRtjOs7Znaly9szPTmTPBEEQ/uVEgCb8K2j1Rl75LQqt3sjJlGKWRaQ0u53eYOTVdVEYJRjb2ZPBhb9Cxkkw6mH7y3jte5Yf7+nIY87nUGEEr27gGtxwgPoALbEuQMs4BcXJckAWOkZe5tAGhryGFth38F0qipIa9s+PkbNrFg7g0PaG7jG2KJYKXQW2GltCHEOarFcqlKYs2pHMpmWOccVxTNkyhSNZR7BQWfDxLR/zZNcnAVgdu7rFN5FncuXxZ752vrhbu1/39SoVSj4Y8AHtndpTVFPEkotLAJgdPhsXyz9W8lVdriX5fME/Ur5n0BlJOJ2Htlp/zW2NRomIDYlsX3QBXa2BypJatn5z3vRmul55UQ3bvo1CX2ugOLuSU9tSbvi6zu68rGytkzPdRvji6GlNdbmOo+sSbvh4/xUGg5HEs3nUVFw7iJUkiVPbkjmuvAW9xhoJBUmR+WxZcI5lbxzlkv8dHO3zHvtPW5FyvgBJAqNRQZ57D46Zj2TFW8c4syOVqjLtdV+fTmvg1O8pbF4QSW2lHjc/O+56rScjHunAkHtDUarla1j74SlKcqtIOCWXN7br2RevkDAAsuLkAK2qTMuFA3IJbK9msme62hp2//A1lSXF1319giAIwj9HBGjCv8Kig4nE51WgrpvH7JMdsWSXNm2U8MnOWKIyS7GzUPP2Ldaw/0N5Rfvb5TFk51fTacddPGZdNzl15ymND+B/CyjVUJgAxSkN2bP2Y8HMumG73o/zRttAnnayZtqG8SQm7pKXZ182/uwGS4xO5JwAmh9/Vq+/d3+g6Ti0Sl0lM3fPJLMiE28bb5aPXs5t/rcxvt14LFQWxBfHcybvTLPHNJU3ul9feePlrDRWzB86H1dLVwC6uXVjcvDkGz4OyG+St317nm3fnCfuxN9bvicZJXb8cIEd319g3/IWxhbWqanUsfWrc5zZngpAx0HeWNqZUZhRwe6fopHqsrvaGj1bvzlPdZkWK3t5MvGzO9MozKy47utqrmxNpVYy+B45gI85kk1WvHjTfSVJkti//BLbF15g548Xr7qttkbP9kUXOL4pGRRKvHOOcNdLXek81AdzKzUVRbUklblSa+GIxlBF1+FtuPvt3gxxPI135iHUSgOledVErE/k51ePsH3RBdJjikz/D65UkFHOwVWxLHn5CMc3JiFJ0L6fJxOf74aNo9wZNmyAFxOf7461gznFOVWs//QQ2XVNQQJ79MIruD0AWXUZtDM7U9HrjLj729G2g1Pj58JoZPs38zi363fWf/SOGLsoCILwLyDa7AutXnJBJfP3JmCGjg2dTnA0U8+CgnDe3nSRhfc1BBW/nkpn4QE5m/X+hI647H8a9NXgNxDuWgoph+DXBxuCKIUKOt7R+GQW9uDTC9KOQtwOuPCbvLy+DLLO4Zxj/K6SP5lPUcHdB5/lg+x7GF5aN7j/T8x/drU29309+wJwqegSBdUFpkzVgrMLyK3KxcfGh9VjV2Nvbg/IDT3GBIzht/jfWH1pNeHu4U3Pex0TVF+Nh7UH3936HasurWJ6p+lN2u5fr9QLheQklQFwKSL7b+04d2xTEsnnCgBIOJ1Hj9EVOHvbNNmuJLeKzQsiKSuoQa1RMvT+9gT1dCektwcbPj9L8rkCjm1MpM/4QHb/FG0qW7vjpXAOr4kn+VwB+1fEMumF7iiU1w7gz9Y3fbiibM2rnQNhA72I2nuMzV+eZcLzj+Du73Bd91pZWsvZnWm4trUlsLsras3NmaNOkiQuHMhEY64ipI/HXzoGqqpMS+TuNNqEOtEmzKnJ+rM707gUkQPIJYTZCSV4tnNosl15UQ2b50dSnFOFUikRHLOSANcKXAOccA1wou+EQBLP5pMVU4Biyce45p4l+J0daDysKc+PIyT+MP3v7Ui+T18uHswkL7WcxDN5xJ+MRq1KwKWtPw6XNQsqya0mL6XM9L2diwXht/nRvp8n2uoqLuw7QHlhPr0n3ImHvz13vtqDjV+cJT9Vzp55BAajKVBhn2WHUqmmJDebnORsLtZlz5obe3Z07Srijh1GqVIz+L7pYmyaIAjCv4DIoAmtmiRJvL5eLm38yG0XYbFfM71iISfMZzEy7i1O7N8CksSJ5CJeWy93YHt6aDtuV0XI7fJVZjB2npzN8r8FZhwAz67ywdsNa36esvoyxwMfQ3URWLtCwGDT6mp9Ne8fex+ASX6j6SmZU6VU8GziSuYnb8QA4HFjAZreqDdluK4WKDlbOhPmLJc3Hc06CsCFggusjFkJwJy+c0zBWb2poVMB2J26m/yq/EbryrRlXCq6dM3zXkuwYzBv9X0LbxvvP7S/JEmc3JJs+j4jtpiK4to/fD034tKxbFM2zMFd7jp5cmtyk+2MRoldP16krKAGOxcL7ng5XB4vBHgE2DP0fnlahDM70tj4ZSTJ5wpQqZWMfqIzds6WDJwSjMZcRU5SKRcPZ13zuqrKtFyoG3vWXNlaj9t80FdtoSJ/H7+8t4Y1/zvJhYOZVy3R1NUa2PLVOc7tSWf3T9EsefkIh9fEU5RV2eI+1yslqpCDq+PY83MMO3+42KTc82bJTSnj17knObszjU0LIjm5NblRtiopMp+IDYlAw+t5YkvT11OSJPYujaE4pwprezMGuUXjlX0Uy44Nc4upzVSE9PZgyIMd8XWvRSnpqTwhZ7rr50CzautFWH8vJjzXie4jqlApfkNbtpSq4qOknVvJpSMXiDueS9zxXPJSylAqFQR2d+X2p7tyzzt9cHAtY8d3X/Ld4/ezZ/E3nNjwK1vnf4LRaMDa3pzRMzuDQb4fiQCKfoml6mAOA9pMAmDjZ9sasmdXBKsxRw5w7LdVAAx/dCY+YU3nTRMEQRBaH5FBE/4ROy/m8PGOWHSGxl0QQz1smdqzLbcEu6JSKlh/NpOjiYW0V2czoaKu3NApAPOiJCapDsP+w9RGtuNQaT9sDP0I7+jKRf3/eP3ICSabm9G1z/MoXNo1nMChDTy8HaI3QcCg5i+u3TDY+x5UyRkVOk4GVcOPyrfnviWzIhMPaw9e7vcWZv3e4fMNU1hWlcT3thakKF34zKMzN/I5dUxhDJW6SmzNbAl2DL7qtv29+hNdGM2RzCOM9h/NOxHvICExJmCMaYxao+fUKZSurl2JzI9kbdxanujaMDfZ7tTdGCUjfnZ+uFn9c5Mfp14oJC+1HLWZEntXKwozK4g/mUu3ETc2ju9GZSeUmEoaw0f5EtTTndXvnyDxTD4FGRW4+DRk0aL2Z5CXWo6ZpZpJL4Rj7WDe6FjBvTwozqni1LYUMmPlssMh94XiESAHzLZOFvQeH8DhNfFErE/Ev4sL1vaNjwFy4JCTWMrJrcktlq0BpEQeRTLKjUkkQyr5ae04sDKWI2vjCb/Nj/CRvo2ydJJRYveSaArSK7Cw1qA2V1JRVMu5vemc25uOZzt7wgZ40a67G2qzG8uqXRlgJ5zOoyi7ktse74SD282baiH6SBYHV8Vh0BuxtNVQXa7jxOZk8lLLGf5QGGUF1ez6KRok6HiLN91GtmXFnGNkXComK74EryAH07HijueQcakYlUbJhOe7U/Lsd1QDFp07NXtu6169qDl3nqoTJ7EfP97U0VHt6cnZHVs4snoZtVVyoKtUqTC3dqC6rBALi0OEj3sBhVKJxlxFQFdXrOzMqCguYtWc58lJjDedw8m7DWV5uSSdOcmhlT8z6N6HsbSRMOrlbo3K4jYYjXLm3lPpT6h9b+LL0nBv15ERj3RoFMRnx8ey49t5APQYN4lOQ0bcrJdBEARB+IuJAE3420mSxKc7Y0nJK0aPptG61MIqdlzMxdvBksnhPiw7looCI4udl6Mo1UHQCLh7DbVpp9i59COG6g9iXZLA8yTwtMVK3qEzG/MLwFLNJksPAksjmBy9nHGB4xoySxpL6DKlmSur49EFrFwaArTODeWNsUWxLL0od258vffrpnm+XrpzIx32v8kbKevYZW1FnNJA0zYfLasvM+zh3qPF8Wf1+nv35/uo74nIimBp9FIuFV3CzsyOF3u82OI+00KnEZkfya9xvzK983TUCjU/RP1gmmh6SJshN3C1N9flb+47DfLBztWSAytjiTuZ85cGaGUF1fy+MAqjXiKgmyu9bw9AoVTQrrsbCafzOLk1mdsek9+slxfVcHyjXD7bd2Jgk+CsXq+x/pTkVRF/MpueYwOblGl2GuxD3PEc8lLLOfRLHIOmNfwv0euMJJ3N5+KhTIpz5GkRFAroPT6gSfZMkiTO7mjozGlpk03PCYHEHMmmOKeK4xuTyEspY/iDYZhZyr/mj29OIulsPkq1gtse64BHO0fSo4u4eCiTlKhCshNKyU4o5fCaeIJ7exDa1wPbujFR9SysNc2WZqacLyA/rRy1uYoRD4exf0UsRVmVrP3wFMMfCsOv0401jdHV6tDVGk33LUlwcksyFw7KpXz+XVwY/mAYiWfzOLAyjpTzBfw69yR6rR59rQGfUEcGTAlCpVIS2t+T6ENZnNiSzIRnuwFQXaHl8Fq5wUrPMX7YO5mRc1Eeq2bZqWmAJhmNWPXqReH3P1B1/DiGkhKk6moMCgV7Nq0l5sh+AOzdPeg0ZAQdBg8HSeKn556gLD8FhRRFt1vHNdyftpaNn75PTmI8ao0ZwX0H0HnYKLxC2hN79CBb53/Cqc3rcPZug9rMDKPBgLWjB151v8NqlQrMjRJdnAZTW36IsS+FY2bR8Oe8rCCfDZ+8h0GnIyC8FwPuug9tejlmbWxv6HUQBEEQ/hkiQBP+djHZ5fjpP6YoJJU3XO+kTaenANDqJXZF5/LbmQwyS6r5co/8yfLTjsfxKj0LGiu5xb1CgblvTxymfkvvxfsZrzrKfWb7sFalsUWfDwoFt1ZWcdjWgcTSJD46+RELzi7gvf7vMcLvOj5FViohcChErQHnILnTI/L8Yu9GvItBMnCr760MbjO40W5jBr/Lnt1F7Mo8wJaU3wlxCbvu5+R6xp/V6+zaGWuNNcW1xcw/Mx+A53s8j7Nly621b/W9lY9Pfkx+dT5bErdwMOMgu9N2AzA5eDJPdnvyuq/1ZkuJqsuemavoNqItCqWCQ7/EUZBeQWFm82PB/qzinEq5gUe5Dte2tgx/MMwUePQY40fCmTySzuZTkFGOi48th36JQ1drwCPAjg4DvFo8rkKpwMUrkeiKn7C2mQUENFqvVCoYfE8ov354isQz+SSeyW/2OGozJUE93Ok4yBs3X7sm67PiLpGfkoRaY4YkGSkvyMO3g5quw3sTczSbA6tiST5XwK8fnmL0E53ISy3n9O+pSJKEd2A8v77/Fbfc/SDdRo3Dt6MzlaW1xBzJJvpIFuWFNUTu2MvJdVvRWI1EZd7w/9jZ25rbn+mGlZ2ZaZkkSaYSws6DffDv4oqbrx3bF0WRk1TG1m/OM/zBsOseU1hwNp6138RjUFk0XamQg+Aet/mhUCpo388LJy8bfv/2LPmJP2PUZ2NhF0anQVNR1r+et/lx6Wg2mbHFZMUX4xXkyNF1idRU6HDysqbrrW2pjY9DqqlBaW2Nmb8/IHc+jI04zPk928mOu4R3UCguTnZ4ZGVRdfIk1Ro1Z9r5UHpkPwqlklvueYjw0eMbzYk2cNoD7PnxWw6vXkq7Xn2xdXJBkiR2fDOPnIQ4LGxsufuDz3D0aPg/Fdp/EIWZGRz7bRW7vv8aZ582AHQYOACfWDPQGjhXrsNFpSXAworu1n0w5JaDryMARoOBTZ/9j6rSElza+jHmqReojS6maHUsFmHOuNx//b+XBEEQhH+GGIMm/O0ijuylyi6eWqWCI2nLCCeWcF8n+gY68+a4MI6/Nox5U7rSy9+JEJsanjbUzTU25DVw9DUdZ2CQK5P7hbHXZiy6Rw6wqPdUDAoF/WsNfB50L3um7OeN3m8Q5BhElb6K5w88z7zT8zAYr2Muqp7TwcYdBr1s6sa4Jm4N5wvOY6Ox4ZVerzS725jgiQBsS9rW4nlSy1JJL083fX+t+c+upFFq6OPZBwCDZCDcPZyJ7SZefR+VhjuC5YYobx59k91pu9EoNbzV9y3e6vsWZiqzq+7/V7k8e9Z5sDeWtmZYWGtMDTGup5uj0WAkK6GE2qrrmxMsKTKfXz88RWleNTaO5ox+ohMa84aspbOXDe3C5XLPk1tSSIrMJ/lcgSm4ulpzD8lo5My2jRj1enZ9/xVpF8432ca1rS09x/g12+TTpY0Ng6YF8+BHAxh6f/tmgzOAyLrsWXCP3ni2lQOKlHOnUSgUhPX3YtLz4dg4mlOSW8WvH55i77IYJGMNVlY7iT+2GX1tLUfXrkKvldvCW9ub02O0H/e915exT3VGKZ0AJPS1kY3OW5hZye/fnUd/2WTxyecKKEivQGOuwq+TivKiAqwdzJnwbHfa9/MECfYtu0ROUmmLz9vlzi4/2mxwZmVvxpgnOtNzjH+j18DN1xZHt2MY9RmAgZqyKNZ/+DpLnp/J6a0bsbBR0L6/HACd2JxMZmwxl47K5YlD7g1FpVJSHSWPX7Xo1ImC9FT2/PgtCx9/gB3fzjN1T8yMv8S5Nq7sDfNlz4ofORLsQ6mZCktbOya//j49xk5sMmF1l1tvw7NdCNrqavb+uBCAiLWriI04hFKl4vbnX2sUnNXrN3kawX0HYjToyU+Vfz4C2/VAozVgVECVrRl+j4STp0tHrdRQ+HM0hroW/2e2bSQ3KR4LaxsmvvQmZpZWVETI92vmZd3kXIJwPQoLC3FzcyMlJeWfvhThXyQlJQWFQkFkZORNPe727dvp2rUrRqPx2hv/S4kATfhbGfR6eke/Q5yZXNp4yNIc7S/3yC3t61hoVEzo5s2ax/qyo/3vqLWl8qTPvZ9ocry3b+9AxKvDsLcrZXO23DRj5sTVcOs72JrZMiV0CmvGruGBsAcAWHxhMbP2zKK09hpvFtv2hhfioPOdAGxI2MCnJz8F4Jnuz7Q4Xmug90DszOzIq84zlS1eLrEkkYkbJzJ63Wge3vEw25K2EZkXSZW+Cjszu2uOP6tXP9ZMrVTzZt83r6sz253Bd6JSyIGIm6UbP4366Q+3xL9ZLi+N63prQzljcC852xJ3IqfFduVlhdUc35TE0teOsv7TM6x+7wR5qWXNbgvyGKzjm5L4/bsodDUGvIIcuPPVnqbW5pfrOdofFHIwVz9GreuIttfM5mXGRlOWnwfImYzNn/+P4uzMpscf488T3wxh5reNH1Ne70XHQT6YW7Zc3FBZUkzcMXmaBed1m7E5LHf4SznXMI2Cu78dd77aE68gB3Q1BvS1BUjaXyjOuohaY4aFrR015WXERhxqdGyFUoFCyqK2Sr4HyZDDfR90Zua3Q7j77d6YW6nJSSpj3/JLSJIkB9h1DVXahZvxy9vPsvSFJynKykSlUTLk3lD8u7hg0BvZ9u15ygqbTo1xOX1ZOalF8nPc8eIPDL/wFo992J2Z3w7hwQ/749e5aank8XW/kHBCDnhuffRJOg65FbW5OUWZ6exf+j07vplH+ChflCoFmXEl7PjhAgAdbvE2jQ+siIwk3cmW/cpalr70FJE7tlJbVYm9mzsDpt7PfR/Np/9d92JtZo5OrSKppgKtWoWj2px7P5xH247NT0qvUCq5dcaTKFUqEk5GsGvRV0SslRv6DJ8+izZhzY93UyiVjHriGdwDggCwcXTCukx+XqzaO3Pv//rTtpMLWe6plGkLocpI8bp4yvLzOPLrCgBuufdh7Fzd0GZVoE0tA6UC616eV33+/wsefPBBFAoFCoUCjUaDv78/L730EjU1zU80/0ccOHCAoUOH4uTkhJWVFUFBQTzwwANotQ3z5EmSxKJFi+jduzc2NjY4ODjQo0cP5s2bR1WVXOL89ttvo1AoePzxxxsdPzIyEoVCYQqW6t8Eu7m5UV5e3mjbrl278vbbb5u+f/vttwkNDcXa2hpHR0eGDx/O8ePHr3lPH3zwAePHj8fPz8+07OTJkwwbNgwHBwccHR0ZOXIk586da3Su+uf68oe19dU/KGhun9WrV5vWnz17lm7dumFjY8O4ceMoKioyrdPr9YSHh3OirpnPzZSfn4+ZmRmVlZXodDqsra1JS0u76ee5EX/09byaMWPGsGjRIgBmzJjBu+++ezMu9YY8/fTThIeHY25uTteuXZusHzVqFBqNhhUrVvzt1/Z3afUBmp+fX7M/rLNmzQKgpqaGWbNm4ezsjI2NDXfccQe5uX/vHEr/dfti89gXm3dd26Zun4eVOo3quk+aK5VKjkuVsHIq1Fz25tpogMhVEPUrKJQw7stGjTqutOj8IgySgQHeA+js2vjNklqp5oWeL/DRwI+wUFlwJOsIU7ZMIbEk8ZrXqzPo+ODYB8w5MgetUcuwtsO4M/jOFrc3U5kx0m8kAFsStzRZ/+25b9HVDfI/mXOSlw+9zCM7HwHk8WfX26J+VJvR3Kl/lDkh7xFgH3DtHZDb4b/W+zXuCLqDX8b9QhfXG58K4Ga6sjTO0qYhi+fX2RkzCxUVxbVkJZQ02i87sZTNC86x7I0ITm1LobJUCwqoKK5l3SdniKnLjlyuOKeSrd+eN00U3XmoD7fP7tqoVO9yTl7WBPWQOzTWVOiwc7Gg52i/a95TzKH9AIT0uwXPdiHUVFaw/uP3qKloOvdZc7/XrkfU3h0YDXocqmqwzSvEpVx+M5cedc6UEQOwsjPj9tldCehSgb5yNdrqQmxdXJn67sf0GDMBaMjEXS5yZ+NlSaeOo1AocPSwZuSjHVEoFcQdz+XMjtSG7JmFisrCwxh0OmoqK9jw8bvUVFSgTUmmm+IETq5yQ49tzUzmfbmEZdupNXdEbazB07YCY0EB+Z9/0eLzE3fsMEfWLAdg2MNP0Hn4KEY+/gyPf7eUIQ8+BgoFsRGHqC7LIqwui1ZdrsPKzoy+EwKorapk9+JvWRd/jqg2bhRUlqFUqQju3Z87Xn+PR778nt4T78LNL4A+d0zl7unP0DMpC8/icvzySxjZqSd2LldvruPq60/4WDnDfX7PdgDCx06k09Crl1trzC2Y8OIbhPQdyOAHHqUmWn4TahnWUMrs3j6Ew3nrkJCouVTEsUXL0dfW4tO+Ix2H3ApAZV32zLKjM6oW/r//14waNYrs7GySkpL44osvWLhwIW+99dZNOXZ0dDSjRo2iR48eHDx4kKioKBYsWICZmRkGQ0Pm+b777mP27NmMHz+effv2ERkZyZw5c9i4cSM7d+40bWdhYcHixYuJj49v7nSNlJeX8+mnn151m+DgYL766iuioqI4fPgwfn5+jBgxgvz85sutAaqqqli8eDGPPPKIaVlFRQWjRo2ibdu2HD9+nMOHD2Nra8vIkSPR6eS/cS+88ALZ2dmNHmFhYdx5Z8t/Q+v99NNPjfabMGGCad306dMZOnQoZ86cobS0lP/973+mdZ999hn9+/enV69e1zzHjYqIiKBLly5YW1tz5swZnJycaNv2r21kdS1/5PW8GkmSOHbsGP37y3OuHjp0yPT13+3hhx9mypSW+wU8+OCDzJ8//2+8or9Xqw/QTp482eiHdNcueULg+h/wZ599ls2bN/Prr79y4MABsrKymDRp0j95yf8pcbnlPLzkJI8sOUlq4TVadZdm4HXmU2LMGr9J2GvvDPkx8NsjUJoBBz6BL7vChrpPDXvNAO/uLR42rSyNLUnym8qZXWa2uN3ogNEsH70cbxtvMisymb1vNlqDtsXtC6oLmL5zOqtj5U/uZnadyeeDP79mE4+xAWMB2J22mxp9w6ey8cXx7EyR//B+M+wbZnaZibuVO0ZJTtH39ux91ePWKy+qYceXMTif7EjJWgdqKq+vtA/grpC7eLvf26b50/5Jl5fGdb21TaN1ao2KwLoyw7jj8nxWkiRxbm866z87Q9rFQpDAJ9SREdM78PDHA/DrLGdq9i6N4eDqOLTVemKP57Du09OsfPs4qVGFqDRKhj8UxsC7glGprv7rr8doP+pbcQ6+O/SanQ31Oh2xx+SMVOdhIxn/4hvYOrtSnJXB5nkfYtD/+bbz+ooKzqyV/z/6FpRiP348XgMGYq7To9frSDl8sNH2tZXlxB9bhtFQS5sOnbl37jzcA9rRadhIVGo1OYnxZCfEmrYvLywg4aSckQu7ZSgACaeOmda3ae/ELVPkrM6xDUkcWCXv2y5cQ/xx+d6t7B0ozs5k/RsvkDj5TkrmfUboppcxN1ZRmFnJzu+jMDaTFZUkidgIOdvo62XE5505AJSsWUPV6dNNts9JjOf3r78AoPvo8XQePsq0ztzKmu63jSOkzwBALivsPsoXpVp+QQdOCcbcSsPO7+ZzbudW9AqwqtXRb+wkZnyzhHHPvYpf525NShZteoTjWq2jW1oeYVmFWHr7tPRSNdL3jqnYu8kBf0D3ntxyz4PXtZ+NkzNjZ79MYEhPdNmVoACL9g0dPb2C21OuKyK9Rn4dHHKdUKrUDH90FgqFAmOVjqpI+cMzm74tj538rzE3N8fDw4M2bdowYcIEhg8fbnpvAWA0Gpk7dy7+/v5YWlrSpUsX1q5da1pfXFzMPffcg6urK5aWlgQFBfHTTz8BsHPnTjw8PPj444/p2LEjgYGBjBo1iu+//x5LS0sA1qxZw4oVK1i1ahWvvfYaPXv2xM/Pj/Hjx7N3716GDGlo2hQSEsKQIUN4/fXXr3lfTz31FJ9//jl5eS1/YHr33XczfPhwAgIC6NChA59//jllZWWcP9+0HLvetm3bMDc3p0+fPqZlly5doqioiHfffZeQkBA6dOjAW2+9RW5uLqmp8rQlNjY2eHh4mB65ublER0c3CvRa4uDg0GhfC4uGSoeYmBgeffRRgoODmTZtGjEx8oTtSUlJLF68mA8++OCax/8jjh49agpWDh8+fF2By/79++nVqxfW1tY4ODjQv39/0/MDsHnzZnr27ImFhQUuLi5MnHj14QpX+iOv59XExsYiSRJhYWEUFBSQkJBA795Xf29ytZ+HP2r+/PnMmjWLgICWP4AeN24cp06dIjHx2h+2/xu1+iYhrq6ujb7/8MMPCQwMZNCgQZSWlrJ48WJWrlzJ0KHym4mffvqJ9u3bc+zYsUa/TIS/xpd74pEkkIDlx1J5fUzLA9ANW1/EwljNfjN/wIC/vT/Jpcnss3fijYJCVPE74YsODTtYOEC3e2HoG1e9hoXnF2KQDAz0Hkgn1+ZLhuqFOIWwcsxKJm2cREpZCosvLOaJLk1LJ+OL43l81+PkVedho7Hhw4EfMqhNC235r9DVraspCNyfsZ9RfvIbx+/OfYeExK2+tzLQZyADfQYyo/MMjmQdIaU05aqZuXqZscVs//4CNRVyUFZdpiViQyJD7gm9rms7tyedpMjGn6yp1Ao63uJDQDfXZvfJii/h/L4MwgZ40jas5UYkN+Ly0rhOQxpnz+qF9PIg5kg2CWfy6TupHYfWxBF3XM6OB/Vwo9ftAY1auI9+vBMnt6VwcksyUfszuHAw01QeqVAq8OvkTM+x/rheZyc7J09rRj/eCb3O2OxkyFdKPnuS2spKbJyc8QnriFKpYsJLc1j15oukRUVycPmPDHlwBgDl+/ZRum4dNkOGYnfbKJR1b9zqadPTKfl1LdWRkXILwzqphblU22ow0xvo/PhTuN5/H1J1Ne4PTCUNIxe/mod/rz6obORyuFNb1qOrqcbNL5DJr7+HUqVCm5FJ4fvv421UkgYcmfMqPdU2mAcFkRDgjWQ04tO+I30mTSH64F7SL56npqICi7pjdhzkQ1F2FVH7M6gq1crZs4LDSJKRgPBe9L/zHla99hxZ2RmoHa3o6uqCRUYmHc9+zdlus0m9WMz+9zYw9K3Gb0TKjkSQYyFPidFhYnesunhhP/kOsjZtYvf7b1HdpSOXz11RkJ6GXluLf9dwevceSO7cuRgqK3F/5VVUNnIJVd/J04g9dpiEkxH0nZzFmJmdqSrVEtjdlcTTJ4g7fgSFQkn3pEw8za0Ivvehq2YyldbWWHbsKL8ugMb7+oIejbkFE19+i8TTJ+g6cgzKa3zIc6Xq6EIAzP3tUVk3dL51D2iHSq3mfP4BfHyC8bIKZMCt9+DsLX/gUXk6F0lnRONhjZlf8+MZbxZJkqjWXcf43pvMUqP6U5NvX7hwgaNHj+Lr2zC+ee7cuSxfvpzvvvuOoKAgDh48yL333ourqyuDBg1izpw5REdH8/vvv+Pi4kJCQgLV1XL5roeHB9nZ2Rw8eJBbbrml2XOuWLGCkJAQxo8f32SdQqHA3r7xXJYffvghPXv25NSpU/To0aPFe5k2bRq7du3i3Xff5auvvrrmvWu1WhYtWoS9vT1durRcUXHo0CHCw8MbLQsJCcHZ2ZnFixfz2muvYTAYWLx4Me3bt29UBnm5H374geDgYAYOHHjNa5s1axbTp08nICCAxx9/nIceavjZ7NKlC7t27aJdu3bs2bOHzp3lqpnHH3+cjz/+GFvbm9etNC0tzXT8qqoqVCoVS5Ysobq6GoVCgYODA3fffTfffPNNk331ej0TJkzg0UcfZdWqVWi1Wk6cOGG6j61btzJx4kRef/11li5dilarZdu2bab93377bZYsWXLd4/6u9/VsztixYzl8+DB6vZ7q6mocHR0xGAwYDAZ8fOQPokpKSprd92o/D83x8/PjwQcfbFR6+0e0bdsWd3d3Dh06RGBg4J86VmvU6gO0y2m1WpYvX85zzz2HQqHg9OnT6HQ6hg8fbtomNDSUtm3bEhER0WKAVltbS21twyS4ZWVyaZ1OpzOl5oVri8stZ1tUQznZmlPpPDU4AMtmMg2KS1tRx21DJ6k4bOEFpDMteBrzI+dTqC3l7NBX6LHzbQCMbfti7HofUug4uSU+QAuvy+XZs0c7PHpdr5+typYXur/Aq0df5fvz3zPcZzh+dn6m9YXVhczcPZO86jz87fz5/JbP8bXzvaH/G6N8R7H44mI2J2xmmPcwEkoS2JkqZ8+md5je6Fh93fvS170vGDGVP15JkiQu7M/i2IYkJCM4+1jTeagP+5bGEn0oi3bhLngE2je7b72cpFIO/9p8mUx6TDFdhvvQc5yfqfudJElcPJhFxDp5IuDEs3n0HONL1xFt/tQbIoCUy7JnHQd7NvvcuvpZY+1gRmWJlpVvH6O6XIdCCX0mBNBxsBcKhaLJft1G+uDkZcnepbHoagzYOJkT2s+DkD7upjnHbuR19AlzuO59Lh7cC0Bw34EYDEYMBiOO3m0Y+cSzbJ33IWe3b6HLyLGoM7LIemY2klZL+a7d5M6di+2YMdhOnIA+PYPS336jOiKi2XMkBXgCGtr3GYjj3dPQ6/Wg0RBy34OkrfiRHH0tGc8+h+eC+VRXVhC5Xf7Z6DVpCgajEYPRSN6CBVTs34+PlTlpQT6k62sIOh+L+tRJIjsGgFJBx2Gj/o+98w6Povr+8Ls1vfcGaSSEEggdQm+hCoIgIAoWEAS/ClYsIIiIiF2qIIoCiiKC9N57CSWNJISEhPRedrNtfn8MWVhSSGiWX97n2edJZu7cuTM7MzvnnnM+B2tnVxy9fchLvU78mRM0Dr81QdF+qC/5maWkxuQT0ErOhW1HAGjTewBlH86jRWIqZ33dSXG2w/ep52jWuj1Fmzah2bmFSx6PE3PDBu/v/sBv/C3p+Zi1B9DJO2AhVeMcYEfUoX1ckpSTFnIzhCjmcqXzYW9rT7PIOJJ/vGXs6XJycf/yCyQyGbZuHgS1D78ZCrmGQdNEcZ+ykmL2rlwCQJOGAbhFxmPeva14Pu+CeZvWRgNN4uJS6+vJ1s2DsAHiC3ldf2fKLouTKsrG9pW2dfULJD0+lmsll/G3CaWBLgitVotgECg5LhbTNm/nWunYHvRvnUqrp8nMnQ+0z9oQPScCS2XdXmW2bNmCtbU1Op2O8vJypFKp0aApLy9n3rx57Nmzh44dOwLg7+/PkSNHWLZsGd26dSMlJYWwsDCjsXS7QTJixAh27txJt27dcHd3p0OHDvTq1YtnnnkGW1vRSI6Pjyc4uPaFWFq1asXIkSN566232Lt3b7XtJBIJ8+fPZ/DgwUybNq3al9YtW7YwatQoysrK8PDwYPfu3Tg7Vx9VkZycjKen6WSEjY0NBw4cYOjQoXz44YcANGrUiJ07dyKXV/4+1Go1a9as4e23qxbYup05c+bQs2dPLC0t2bVrFy+99BIlJSX873//A0RD76WXXmLhwoWEh4czY8YMfvrpJywtLWnbti0REREkJiYyatQo5s6de9f91YSnpyeRkZEUFRXRpk0bTp48iZWVFS1btmTr1q00aNAAa+uqc5OLioooLCxk0KBBxu8iJCTEuP6jjz5i1KhRzJ4927jsdsPK2dm5VoZHXb/PqlixYgVqtZpJkybRoUMHxo8fz8yZM7G3t2f69Ok1blvT/VAVAQEBdR5fdXh6epp4JP9L/KsMtD///JOCggLGjx8PQEZGBkqlEnt7e5N2bm5uZGRkVNvPxx9/bHJDVLBr1y4sLR9cUdX/OquuSBEEKaGOBlJLJeSpdHy8dhcdXE3Dl5TaIrrHvY8cWKofiMpMDMcpiCnAH38ucpGVKVcoDv4Qg1RJibkHXAeu7zfpJ0ufxS+lv5BryDUuExAwYCBYHkzKqRRSqF3CriAINJI3Il4Xz2s7XuM5q+fEF35By/cl35Ohz8BJ6sRoRhN1JIoooup0bqz04gz+kbQj/LblNzarNgPQTNGM+GPxxHP3fILbKYg1oyRJ9DJZemoxb5JBfHYGlt5mlKUq2bbiPG7hZVSXwiYYIPOoJSDDwk2LhcetlzVNvoySZCUX9qQSG3kNxxYqpDLIjzKnLE2crVfY6NEWyzi9JZnLpxNwCFUjlYvOnfIcGaXXFZQXyLAPKcfSo+aXXEGArJtjMfcuY9/B3dW2lToqocAMVbEWqdKAY0s111UXuL79QrXbADh1kKBXSVE6FJOhzyHjaOWX+8zjB9AWF+HZoz9ShaKKXu6ORKvFe/l3SNPTudrYB6QSLL5cROxXS8nt25fC9u1AIsHCzQNVZjqbv/mc1rsOItdoUPn4ICstRZmXR+Evv1B4WxK8IJFQ1qgRxaGhGMzE772srITcq1GAhAKvBiYzrfpy8WW7xMKM3BPHuP7aa8S52KEtV2Pm6ExMehax27YhLS3Ff+tWpEB5z95YZF9HpS4luns4TtGxYn0trZ4bh46QkFeIYOcEqdc5vWoF+llzQQJFrVpT1LoVgo81zjYyEi5vQhAM2CstKXjxJeTFxbgoFHi6NeBG1nUOrV1FfNoNrLx8YLwHDjuvkS/x5djBUpKKv0Ud4I88v4CSLAtwBqltFstffg5d2a2QaZeiUjwKSk1i8qUGA87FV9EbBASplNLgYCzj4yk9cIDTr7xKzoD+AGicRMGZq2dPsvHn1Zg5OpNz7gTFudnIrayxvSYaMSlKJZG3ndPqsBSgIrBx/+XLGK5ereNVUzfkWgmh1+yRIOFYxgU0286brFfJxesjuuAYfjbN0SYWcfDXXch0Ehrl2aCTGTiUcRbDHYdWIUTx/5EePXqwZMkSSktL+eKLL5DL5QwfLqrcJiQkUFZWRp8+fUy20Wg0hIWJJVcmT57M8OHDOXfuHH379mXo0KF06iQKN8lkMlatWsXcuXPZt28fJ0+eZN68eXzyySecOnUKDw8PBKFq4aOamDt3LiEhIezatQtX1+rzHiMiIujcuTPvv/8+a9eurfb4IyMjycnJ4bvvvmPkyJGcPHmy2n5VKpVJiGHFsueff57w8HDWrVuHXq9n4cKFDBw4kNOnTxvDOSvYuHEjxcXFjBs37q7H+v777xv/DgsLo7S0lE8//dRooDVt2pSDBw8a2+Tm5jJr1iwOHTrEyy+/TKdOnfjjjz9o27Yt7du3Z/DgwZX2MWnSJH7++Wfj/yVV5AkDyOVyfH19Wb9+PW3btiU0NJSjR4/i5uZWrYe0AkdHR8aPH09ERAR9+vShd+/ejBw5Eg8PUawnMjKSCRMmVLv91KlTmTr17iVw6vp9VoW7uztarZYTJ07wzTff4Ovry/Hjx1m1atVdDa6a7oeqqGmSoa5YWFj8Z59l/yoDbeXKlfTv37/STE5dmTFjhsmMQFFRET4+PvTt29c4w1VPzVzJLCby5iz/vNHhHIzP4dNd8Vwos2d2/w63PCy6cmRrhiHV5nNV8OBbSQ+UkkjkUjnjBo6jYVpDLh65yDXFNcIHf1WtZyZfnc+4XePIMlSOrVdKlczqPYvGjrUL86ugZUlLRmwdQZIuCV1jHY/5P8b7x9/neuF1bBQ2rIhYQUPbhnfvqBp2bd9FbH4s8a7xRCVEIUHCzD4z8ZY3RKaQmki710TG1SI23zRIOg675T0CUHfTsn7uWdQl4KFoRquIqhOWz+1MIa0kGQsbBSOmdcDcytQgSTybzcG1VyjPkVNy3hkzKwVlaSVIpGKh5OY9PIk9lsHR3xNRZSowu2SLf5gL8acyKc675Y0uvGxJ116huPraoi4pRiKVYmZpqtiVdCGHtOIYFOYyHp/QrdJYbqewjYoNn5zDwcOKPs81xtqxitpY90Buagpr1n4HQAMrM5P8pbpQsGYtOSkpXHe0QZBKsFGVY1cm5h26bdxIIwk4v/MOV50d2Pb1AooTYpGUlqAMDsZ/9Y9IzM1RnThJ4e+/k3/oIGa2dtgPHYrtsMdReN/KbyrNz+PXWW8C0KhDOP1HVE6c/jXyBJmJ8eTYWOJ24gQlTcRrt++zE/ELE8s35H//Pbk6HWYhIXSaNw+HQ/vY8923FKPF0D4MkhLxyS3E+6c12I19iuwb2ewBikqLkGdlIRMEXLZtw2XXLqx79ULTvCl/Jovx/00vxyFXaZB7e+Px5Rc0Cgpi73ffEn1oH7knD9Hzg/k4eflQ1rWc9e8eptTaCw5cos/QoeT8voktjk0RBAGFPBZdWSlWDo407d6HJl17UjpnLqVJByods9zbG7vhw7EZOgS5szPFW7eS+fYMHA8epHHv3tgOFT1WO/OziDt2CHlWKm06d+bXdSsBGPDSq0hmfoIWCB0+HMsaXioqMPRQcX3vXmQODkQMH37f3uS7oTqbRdGZq8g9LOn9eOWokMzgRvw2ZwaNenXBwtoV9blsQtU+IJOgoQDbdp70G1D5uCoiRx4UFgoZ0XMiHmiftd1vXbGysiIwUAyn/f7772nRooVRBKPiRX3r1q14eXmZbGdmJnrh+/fvT3JyMtu2bWP37t306tWLKVOmmAh0eHl58fTTT/P000/z4YcfEhQUxNKlS5k9ezZBQUHExsbWacwBAQFMmDCBt99+m5UrV9bYdv78+XTs2JE33nijxuMPDAykQ4cONGrUiJUrVzJjxowq2zs7O5Ofn2+ybO3atVy7do3jx48jvZmruXbtWhwcHNi0aROjRo0yab9ixQoGDRqEm5tbbQ/ZSPv27fnwww8pLy83fge3M336dF599VW8vb05cOAAc+fOxcrKioEDB3LgwIEqDbQ5c+bw+uuv33XfTZs2JTk5Ga1Wi8FgMHpedTod1tbWNGzYkKio6idwV61axf/+9z927NjBr7/+ynvvvcfu3bvp0KFDJSP2Xqnr93kn8+bNY968eQiCQFlZmXEiorS0lIiICCQSCdu3b682NLU298PDIi8vr1Iq1H+Ff42BlpyczJ49e/jjjz+My9zd3dFoNBQUFJh40TIzM3F3r74oqpmZWZU3uUKhQHGPM+n/31h0UMwfGtjcg2Y+jng6WvPVvkSi04u5lF5K64YOoqtky8uQehKN3IYJpdPxalBKNtDIvhGW5pZ0a9ANpVRJakkqKaUpBDoEVtqXVq/lzaNvklqSipe1F4t7L8ZSfsvTaaO0wUpR9/o+vg6+ovDH2c/5MvJLkkuS2XZtGzKJjM97fE6gU+Wx1IXBAYOJPRPL7wm/IzVIGaJ4mvh15eyLOYGTpzUj321rDCesDr3OwJFfEwBo3MmDVn19TdYr7BV0HtGIPauiOb/jOsFtPbB3M/UCF2SVcX6HWHet84hG2NhX9hI37uCJi4+tKIWeo6Y4rxxzawURLzTFu7GYfxXavQGuDezYsewSBRkqzm0XvZVmlnKC2rtTmKUiJSqXnd/FEPGCDxvnv4muvJxG7cMJ7RWBd5PmIMC57eJYQnt4VzmW23H2UvD8wi7IFNIH+iIcf+KI8e9Le7bTqt+gOvdvUKspuPmilBXaBPJzCH1yDIG9+lP4119kf/ElRX9sRJOQiP9nC7GQylAZ9GR5e9BlyWIUN/NMlN26UublxoGCNKzs7Hls+FAsff2M+9Fqytn65XxK8nJx9PSm78SpVT6n/Fq2ITMxnjwfD0qzctBpNLgHNKJR245IJBIEvZ6i9b8B4Dj2KZRKJU269uDIuh8pzsmmOCcbqUxG8x59Kf/lVwp/XoMCMA9piFopxzB5Al7uXhSs/w315cuU7NzJ+Zjz4GCDW2EJHs1aYD9yJDZ9+yC9+Xzt++LLFGZlkhYbxZbP5zFm7mfYOdkRProJ+9clctWtJ56vvku24IrgFYq5LJqM+MvIFUpGffAJ9u7iTLPj4kXosrNN8vGQSJG7OJsIeTgOHYo+JYWcxUvImjMHC38/LFu3puMTo7ly/AhJ506Tl3YdQTAQ3LEL7qWuFIZOR+4ejWVQ09o9/xUKArZtrSQg8rAojCsAwLKZS5Xj827chJd//B2ZXI4+v5yMyBw0CbfKiNh08qpyuwf9WyeRSOocavhPQCqV8s477zB9+nTGjBlDkyZNMDMzIyUlhW7dqs87dnFxYdy4cYwbN44uXbrwxhtvVPtC6uDggIeHB6Wlold4zJgxjBo1ik2bNlXKQxMEgaKiokp5aAAzZ84kICDARHK+Ktq1a8ewYcNqFU4IoijK7WkfdxIWFmbibQLRAyuVmj6XK/6/szZVUlIS+/fvZ/PmzbUaz51ERkbi4OBQ5Xvb3r17iYmJMYpS6PV6Y/huTWG8rq6utfIwbdu2Da1WS69evViwYAGtW7dm1KhRjB8/3ij1fjfCwsIICwtjxowZdOzYkbVr19KhQwdCQ0PZu3cvzz777F37qAt3+z7vZNKkSYwcOZLFixeTmprKvHnz+O2339i1axfffSdOZN45WXEndbkfHhRqtZrExESjQflf4x+v4ljBqlWrcHV1ZeDAgcZlrVu3RqFQmLhL4+LiSElJMcaO1/PgiUkvYtulDCQS+F8vUdHN0UrJYy1Ez+ZPx6+JDY9+CRfWgUTGx9ZvkSh44edZAGD0dlkprOjgKc4K702p7PYWBIEPT3zI2cyzWCus+bbnt/jb+eNu5W783ItxVsHYJmMJdgimsLyQH6J+AOCd9u/Q1rUdCWezapQE12r0xJ/ORKupOjG+v19/zPSWtEsexNhzs3E70orr0XkgQG5aCSmXc6vc7nbO704h70Yp5tYKwofdMhgFQSDx7CmKcrIJaueGTxNH9DoDB9bGmoxZEAQOro1DrxOFLhq1rX720snLmhEz2uIdrMPRPZthb7QwGmcVuPvbMeKdtvg2d8IryJ5e40MYNz+crk8GETGhKU5e1pQVlrPxk88pLy1Fr9MRe/Qg6+e8w6ppL7Jt0UoyE4+A/gIyyWXO79xS6RNz9CDCbT/wcuX9iQDciWAwEHv0tvCY1BRSq8hvuhsF69ejy85G6+1FZn4OSCQ0iRiEwt0d5wkT8Fm+HKmdHeqLF0kaOAjvNDGP6EbzYBS3RQHodTp2L/sGnaacwuxM1r3/BjE3xycIAjsXf0lGYjzm1jYMfWsm5lZV5zv4thCVTrOVcpKdxJe7dn0HGs9dyaFDaNPSkNnZYXvzOapQmhnl2AEC23XC/4MP8PxkPmaNGuE4ZgyNOosvqelKGQ4jR+L3+2/4/bEBybAhpNuLyfjd35tDw59WYzd4kNE4A5DJFTz22jvYubpRmJnBX59/jF6nJaRrAzz9rDDIlFw278QN66YIBhVlxaLh3GH4KKNxBmJdMIWbGwp391sfN9cqjSTnqVOxiYgArZbUqS+T99PPSA4cws9TFM4ozMxAqVDQ0sqZwh3iRJPctQk5PyWhuVF1mNOdPCrjzFCuRx0vei4smlYv0CNXKJBIJMgdzbFqfeseN2tkj8KlPmz/bowYMQKZTMaiRYuwsbHh9ddfZ9q0afz4448kJiZy7tw5vvnmG3788UdANJQ2bdpEQkICUVFRbNmyxZhbtGzZMiZPnsyuXbtITEwkKiqKt956i6ioKKMnZ+TIkTz55JOMHj2aefPmcebMGZKTk9myZQu9e/dm//79VY7Tzc2N6dOn10pa/KOPPmLfvn3Exd1SaS0tLeWdd97hxIkTJCcnc/bsWZ577jnS0tJqlL6PiIggKirKxIvWp08f8vPzmTJlCjExMURFRfHss88il8tNVChB9FJ6eHjQv3//Sn1v3LiRxo1vRb/89ddfrFixgsuXL5OQkMCSJUuYN28eL7/8cqVt1Wo1U6dOZfny5UYvXnh4OIsWLeLChQts2LDhviXiGzZsiLW1NZmZmQwZMgQfHx+ioqIYPnw4gYGBJuIyd5KUlMSMGTM4fvw4ycnJ7Nq1i/j4eOO1MmvWLNatW8esWbOIiYnh0qVLfPLJJ8btv/32W3r16lVt//f6fd6Jo6MjgYGBREdH079/fwIDA4mPj6dv375Gz1xN3r6a7oeq6NWr111FbBISEoiMjCQjIwOVSkVkZCSRkZEmtQRPnDiBmZnZf/Z9/19hoBkMBlatWsW4ceNMkk/t7Ox4/vnnmT59Ovv37+fs2bM8++yzdOzYsV7B8SHy9V4xf2pAcw+C3W+pJT3TUXxQbbuUQdH5jbBHzPPb4vUqqzL8kUpAYibme4Q43bp5ezUQH0D7ru+rtK/V0avZmLARqUTKgq4LqvSw3Q8KqYJZHWchuSkP91TIU4wMHsmFPdfZ+d1ldiy/XG2+wL7VMexaGcW+1TFVrne2cOaJa6/Q6kYfLLW2WNoqad2vIY07it7dSwdTaxxbQVaZsW5X5xGNMLe+NVMXe/Qgfy6Yw/rZb6PTlNNtdBAyhZS0uAJ+ePsoB9bGkZ1SzJWTGaTG5iNTSOk2Ouiuhk5eWgJJZxZzI+Ynfnn/JQ6t/YH8jBsmbazszBg4pQVDp7eicQcPFDdFYZTmcga81By5LIHykkQkEjmPTX+HFn36o7SwID/9BrFH/kSn2oe6aC+H1nzHvu+XVvps+/pTTm36varhPRAqikkrLSyMUvIVYhq1xaBWk3NzZjGvm/jj4NOkObbOt0ItrDuH4/f7b5gFByNoNDTIK0IqlZKZdp3MqwnGdme3/kl2yjXMrW1oGBqGTlPOtq8/5cDqFRz7bQ1xx8UizI+99g4O7tWHd3sEBmFmZYVGU45eJsWuVI3NsVvFWvPXrgPAbvhwpLflk7TsOwBuXhdhfUXDzW7IEPz/2oz7zPdpHDEAgMSzJzEYxMmIQnMlhwozQQKBbTvi3bn6XAxLWzsef2sWSgtLUmMus2eFqHbWY3xzpFLIc2xCoV0AOtUhtOUlOHk3oM3guklN345EKsVz/seYN22KPj+fzI8+IvPDuXjvPWz0wAVdTUP9ZyQISgxleSCUos9Tk73kglGW/p9A8b4U0AnIHM2Ru9XO0LLpIeZCQr20fm2Ry+VMnTqVBQsWUFpayocffsj777/Pxx9/TEhICP369WPr1q34+YmebaVSyYwZMwgNDaVr167IZDKjV6tdu3aUlJQwadIkmjZtSrdu3Thx4gR//vmn0SMnkUhYu3Ytn3/+uXF5aGgoH3zwAUOGDCEiovpQ0ddff71aUYrbCQoK4rnnnjMpwC2TyYiNjWX48OEEBQUxePBgcnNzOXz4ME2bNq22r+bNm9OqVSvWr19vXNa4cWP++usvLl68SMeOHenSpQs3btxgx44dxhwrEN/ffvjhB8aPH49MVjkctbCw0MSIVCgULFq0iI4dO9KyZUuWLVvG559/XmWdutmzZzNw4ECTQsZff/01kZGRdO3alcGDBxtzC++HAwcOGKXwT506hbe3t8kxVoelpaXJ+Z44cSJTpkzhxRdfBKB79+789ttvbN68mZYtW9KzZ0+TAts5OTk1SsjX9vvs3r27UbuhOnQ6HUePHjXm1R08ePCuOXYV1HQ/VEViYiI5OTk19vnCCy8QFhbGsmXLuHLlitELeePGrfeRdevW8dRTT/1ntSMkwr1kqz5idu3aRUREBHFxcQQFBZmsU6vVvPbaa6xbt47y8nIiIiJYvHhxjSGOd1IRTlBYWFifg1YDOr2BbZcz+N+680gksPPVrgS5mcrZDv/2AF43dvGZ+fcoDCp2WA5mUt5oAN7u35jfMieSpcrip/4/0dK1JSCqJvZY3wMBgV3Dd+Fh7UGptpQ/4v/g09OfIiDwVtu3GNtk7EM7ts2Jm0ktTmVi6ETkUjnr550mO6UYgD7PNSGonen1dO1SDlsX3aozMmhqCxo2M53hjjuZwZ5V0QgyA+FjfQlt549MJqUwu4yfZ54AAZ6a08FEKr4CQRDY/FUkqbH5eDd24LFXWhqNK4Nezw+vTSY/XXxQtR3yBF3HjOdqZDbHNiRQmH1L3lYilSAYBDoM9ad1P18Sz54iJ+UarQY8hsLMNJ+rKDuLNe9Op6ywQAyXuk31rUGzUHo+Oxknb9N6ZXeiKinm+1deRF1ShNy8E75h/bG0VaLXlZOTcp7sa9FIZdCwmRMyeeX5IY2qjGsXziFXKBm3cJGJF8V47i+c43r0JToOH41cWVme/27sXv4tF/fuoGn33rQeOJTVb0xFIpUyYdH32DiaKkvlr1+PoawMx7Fjkdw2OZT3449kfjyfch8vzjbxoyAjnb6T/kfzHpWLDxvKyshdsRKlb0OOXIsj9uhBmvXoQ8SkVyjMyuCH16ag05QTMflVmnTtwbH1azi5cb1JH31f/N9dCxsD/PX5x1w5eRSANlfTcdPoCdi9C6GsjMR+/UEiIWDXTpQ+pt9j1MG9qEuKaTVgSCUjXq/TsXTiWNSlJTz5wXyKsrPYvfxbdFoN9u4ePPHuXGONr5pIijzLxvmzEQQDrQcOpe1jw4k+WsCpv5LQa6+jLRHDL0fN+RSv4OpnX2uLLjubnKXL0OXe8lRfKyuizKAjxMoZiXk/kJgjES7i+ExfSk6WU35F9BBYd/XGrr/vQ80vM6h1FB9MReFhhWVo5RyK0rOZ5P92BQDH0cFYtqh9wn/ZhWx0eWpsunkjqSaMuv53r566sHXrVt544w0uX75s9FbV8++gYcOGzJ49+65G2r+JnJwcgoODOXPmjHHi5L/GvyJgvG/fvtV6MczNzVm0aBGLFi16xKP6/8P1vDJ+PX2d9Weuk1UsxjUPCvU0Nc5yEuDcD6wt+hkzZT4Y4ISkJVPyRmJtJueLJ1sS5idj0fosJEgIcrhlaDtZOBHmGsa5rHP8EPUD5fpytiVtQ6UTDY0ngp7gqZCnHuoxPhbwmPHv4jy10TgDOPJbPA2aOhnFLLTleg6tE1+crOzNKC0o5+C6OEbPbG8U/lCXajn6u+hp7DAwkLCOvsb+7FwsadDEiZSoXC4fTKPziEaVxnPlVOYtz9eYYJMXxdijB8lPv4FcoUSn1XDmrz8ICe+Gf0s//EKdSYsvIPpwGonnszHoBRw9rWjZpwHFuTn89fk89DodcccPM+T1d7FzFQ1PjaqMjQvmUFZYgIuvPyNnzuN69CUu7dlB0oVzpFy+yIZ5M3lq3udY2TtUex4Pr1mFuqQIa0cPtIa2pMbenljuh9LajzYDfWk/uOrik4Ig8Pvc90i5fIE9Kxcz/J05JseeFhvNnwvmoNfpMLO0ot2QJ6odS1XcXky6SZceuDTwxatxU9Jio7i4ZyfhI29dZ3lr1pD5oSjRXLJvP15ffoHc0RFNYSEXfv6RawGe5FmbQ0Y6CnMLgtpXHUojtbTE5X9ieE7L2Ghijx4k9shBuo59jj0rl6DTiEWkm3brhUQiofOoZ3D1C2DHoi/QlqtpPejxWhlnAAFt2nPl5FE8GgXTQGKDOjKS3BUrkNyU97Tq2qWScQbQtFv1YTQyuRz/1u2IPrSPHYu/oDBLrEXnF9aGAS+/Xm3I5Z34tWxN93EvsP+H5Zzd+ifnd/xFQOv2WFg3oCBtDwAt+vR/IMYZgNzFBff3TWsoVsiuFB9KpXBbEjJHc9xfm4xEJsU8RKBodzLF+69TcigVqaUc2+41T0jcK9qsMnJ/ikZ3czKl/Goh9oP8kdyctCi/Vkj+H+Lzw6aHT52MMwDLFv/NpPl6/j4GDhxIfHw8aWlp+FTxDKnnn0lUVBR2dnY888wzf/dQHijXrl1j8eLF/1njDP4lBlo9fw+CIPD4LzOIy72G6sZIMJjhZKXkiTbevNzzplFRmgObpsCVHQCYAZk48ouuO8t0g/B1sWX5M20IcLHmSJqYX+Jr54ulwtRr1LNBT85lnWNt7C1ZYF9bX0YGj2RU41EPXSntdpIuiLlCbn62aNR68tNLOfZHAj2fFl8cT21JojhPjbWjGSPebstvH5+mOFfN6S1JdBouhmAe25CAqliLo6cVYX0rKys27+5FSlQuscfTaT/E3xgmCFBaWG6sV9ZmgK+Jh82g13N8gxiq1uGJ0WQmxhN/6hi7l3/L6A8/RSKV4h3sgHewA6piDSnReXg3dkAmk3Lyz9+MXrHs5CR+njGNga+8SYNmoWz9+lNyUq5hZe/A0Dfex9zKmkZtO9KobUcKszLZ8PEs8m+ksunTuYyYNQ+FsnKydmrMZS7tE2u9DXzlVaQyL7KSTZXilBZygttX792WSCT0fuElfnxjKskXzxN75AAhXcR8hsKsTDZ99pHxGM789QctIwaiNK+9EtadxaQBWkYMJC02ikt7d9Bh2EhkcgUlR46SOe9jcSOFgrJTp7g4cgR5A/sSe/Yk5Y6WN8crxS+sNR2GjaqkVlkVnsEhuDT0Izs5ib8+E41gmVxO7xemmFzjQe3DcfMLICs5iYDW7Wp9fCGduyNTKPFp0gzD5SiuP/8CBb/8iuRmXpjjmDG17ut2Att2IPrQPqNx1mH4KDo9MabOuVhh/QZjYWvH+W2bSU+II/7UMeAYAFb2DnQefXcJ7vvFoNFTfDO82LaHDxKZeAwSqQS7CF9k1goK/rpK0c5rKFwssGj6YOr1VKCKyiFv/RWEcj1SKzmGMh2lJ9LRppfiNDYEQWsg96do0AtYNHPCts+9K8nWU8+D5NVXX/27h1BPHWnatCkXL168e8N/GW3atKmxaPt/gXoDrZ5q2R4dR6JmK3Ib8Gq0gRmtP6FvE0+UFaFpN87DL2OhKBUkUmjUF1qP588b/nyxM4G+Tdz4bGQLbMxFz1NMrpirVZUcfoRvBEsuLEGr19K7YW+eCHqCNm5tHqlhVsHVSNFAC2ztiquvLRsXniPmaDqNO7ijMJdzYa+oQthtVDCWtkq6jg5m2+KLRO69TqN2bmhUOmKOiQW8u48JBvTcme7ZoKkTts7mFOWoiT+VSZPOYq6ITqNn+9JLqEtuGnd9TI27mCMHKMhIx9zGlrCIgZSryki+dJ70hDgu7N5Oy4hbIjoWNkqjMVSUk83lfWIR2QFTX+Pcjr/ISLjCH/Nm4RXShNRoUTlvyBvvmeRRAdi5uvH4m++z9t3XSE+IY9fSrxnw8usm341Oq2X3cjHpt3mvCLwbi/Hvno3s63z+HTy86DBsFEd//Yn9q1fgG9YGqVTGnwvmoCoqxNU3AI26jIKMdCJ3bq2TFy3m8AEAGod3QyoVjeJG7Tpi5eBIaX4e8aeO4+fuTdq0aaDXYzV4MAXtW3Fu3U/kyiVw0/tmrtER0rINbaa8Uul81YREIqFlxEB2L/+W69GXAGj/+JM4elZWyLJzdTd6OGvdv1RKcMfOAAidOmHRqhWqc+cQNBoUPj5YVSOTfDd8Q1thaWePTlNOvynTadT23pKyJRIJIeHdCAnvRta1q1zat5PoQ/vRqFX0fG5Srb1x90PpiXQMpVpkjuZYtqrsmbIO90KbraL0RDp5v8bhMskcpef9j0swCBTtSaZ4n/j8UPrZ4fRUYzSpJeT9EosmuYjMr88jNZdhKNWh8LLGYWRwtSGK9dRTTz31/HepN9DqqRJBEPjy2Ba4meJTJL1InOZXBslfExecXwNbpoG+HBwDYNQacBU9TC8Gw9DWDXG1MTN5iY/JEw20Jo5NKu3P3cqdbcO2IZfKsVX+ffkQ6hItN+JFiWq/Fi7YuVjQpIsn0YdvsP/nOJTmMgSDQEArF3xDxZl1v1BnAsJcSDyfzYGfY9GoRSGFpl08Sbm8m7Xv/ohP01Ca94qgUbtOyBUKpFIJzbp6c+yPBC4dTCUkXMy12vdTLJlJRZhZyun/YnOTPC2DXs+JDWLibdvBw1BaWKK0sKTzqGfYt2oZh9f9SGDbDlg7VlZ7O3XTe+bTpDkhXXrQqH04e79fwuX9u0mNFhUMIya/gkdgcJXnxcHDi8HT32HDvPeJPXoQRy9vOg4fjUatIvboIS7s3kbejVQs7ezpOub+JYPbPjaM2KMHyU1N4eDqlaiKC8m5noyVgyND33yflMsX2LH4C07/9Qct+w5AaXH3JGF1SQlXz4kJ2E263FIZk8kVhPaK4Pjv6zi/dRPy05cp1KhJbxnC9dwUyn+LBrkoI+NSWIpPbhFedo4EvjMLyT1IlYeEd+fQz6soLyvF0dObtnUM06wtEokEl5enkvLscwA4jBp1z+qDCnNzxn+2WCy6bW1z9w1qgauvP72em0zXp55FVVyErXPdwvjuBRPvWc9b3rM7sR/sjy5HRXlCAbk/RuM6tSUym7rnO1YgCAL5v1+h7JwoQGId7ondAD8kMikWjR1xnRpG7upodFllGIpBaqPE6ZkmSJV1r/FVTz311FPPv5/6TM96quRYYi4pqvMAhDg0B+CHqB/YGPcbbHsDNr0kGmdB/Uh4ciWpFqYvbW625pW8X0YPmlPVBaUdzR3/VuMM4NrlHASDgJOXNXYuYuhcx6EBWNgqKcgsIyu5GKW5jC4jTcVqujwZhNJcRlZyMQWZZVjYKmnURs6x9WLtmOtRF9n29acsmzyOA6tXEH/qGGYWKQiGRDITIzm/8yintyQQfzoTqVRCvxebV6pnFn14PwWZ6VjY2Jp4ylr0HYBHYDAaVRn7f1he6ZiKcrKMoYcdR4ghbnKlkr4v/o/eL7yEtYMjXZ96lsbh1df8AVEopNfzkwE4tn4Nmz+fx9IXn2H38m/ISkpEJpfTZ+LLmNdCYexuyOQKek+YAkDUwT1cPXcauULJ0Nffw8bJmZDO3XHw8ERdXMT5WiowXjl5BL1Oh3MDX1wamsath/bqh1Qm40ZCHAfM9Bxu3IAEQUN5aQk2zi50GvkULyxaRb/BT+BlY4f7e+/ek3EGorHT8YnRWDs6ETH5VeQPsfaiZYcO2A4ejFnjxtg/cX9qZhY2tg/MOLsdhZn5IzHOAEqP3xC9Z07mWIZVL2wikUlxGtMYuYsF+sJycldHI2irLqdRG4oPXBeNMyk4jAzCfnCAiXGocLbAdUoLLMNckTma4/xME+R2lcOI66mnnnrq+f/Bv0LF8WFTr2ZliiAIjFh2lFjFa0jkZazuv5pjN46x9MJS5Ej4Lj2DkHIN21sN5zdKiM6LxkJuwYbHNuBjU3XycJGmiPB1oojCkVFHsDOrXITzn8C2JRdJupBTScgi/kwmu1ZEAdBtdBDNunlX2vbSgVQO/SKKh/R5PoSzmz/nxpUY/Fq2xj0wiEv7dlGSV0PtM4kFMmVTOj3xGO0ea2WySq/TsWr6JAozM+j61LO0fcz0ZTs7OYmf3n4FwWCgZcRAuj/zAjK5+OK/+7tvubhnBz5NQxk5c949nZfbObD6O85u3WT8397dg9Be/WjarReWdvb33f/tVCguAgx69S2CO94K0Ys+vJ/t336GubUNE75deVcv2q+z3yY1+jJdxoyvMizy98nPkZwnejgkUikBrdsT2rsfDUNbGsMh6/n3YijXk7HgFIZSHQ5PBGHV5u7Kk9ocFVmLIhFUOmz7+d6TaEjZpRzy1oiTU/aPB2Ld/u7y3I+C+t+9euqpp55/LvUhjvVU4mhCLucyLmHlV4aVwprmzs1p4dKCpISd7CxN4n9uLujl5pTl3arXodKp+OjERyzpvaTKvLG4PLHOiaeV5z/WONNq9GIhacC/pWleUWBrV3LTStCW62napXK+EECzrl4UZqmQm0kpyz/PjSsxKMwt6D1hKrbOLnQYNoqkyLNEHdxDSb64H125npzUEgRDkViLqfwMh9ec4VpkKG7+gcZzWZSTTWFmBha2drTsO7DSvl0a+tFt7PMcWP0dkTu3kp2cxOBpM9BrtVzeLyrkdXri3gQi7qTrWDFkTlVcTNNuvfFp2vyh5Qp2eWo8GrUK75BmJsYZQONOXTmx4Rfy09M4v2ML7R8fWW0/RTlZYiinRFKlp1AddwXfE+dRu9nh07MPrSdMxtrBsYqe6qlAX6Sh7HwWVm3dkFo+PC/gg6Lk+A0MpTrkTuZYhtXOY6dwtsC2ZwMKt15Fc4fgTW3QpJWQv1589ll38vzHGGf11FNPPfX8s6mVgbZ58+Y6d9ynT58aK4/X889EEAS+2HMFubXoCerk2RG5VA4xW/gw6jCpHi5EmZmBQYOvrS9PBD1BC5cWPL/zeY7eOMr2pO0M8B9Qqd+aBEL+KVyPzkOnNWDjaI6zt2mYnkQiocOQgBq3l0gldB7ZiJL8PH6Y/iMAnUc9bRSRkMpkBLRuV0mV7/dPzpBxtQAnj2zMzOJIunCW61EXuR5VWXmp7WPDUZibV1oO0HrgEOzd3dn2zWekxUbz89uv4NzAF4NeR4NmoUbVwvtFKpXR/ZkJD6Svu2FuZc3A/71R9ThkMjoOH8W2bz+7qeg4CLNqClZW1BXzCWlWSdRDMBjImDULK5WaboHh+Lz57oM9iP8o+RuuoI7LRxWVg8uEUCSKf27EvKFcR8khMffMpmcDJLLaTygobz4LtDdK6rRPfVE5OT9GIWgNmAU5YDew6tIS9dRTTz311HMntTLQhg4dWqdOJRIJ8fHx+PvX/yD92ziSkMPZ5HysfEWZ93DPcEi/AH9MwEIwsNgjgg0NQ2np2tJEZXFC6AQWRS7ik9OfEO4VXslLViEQEuL0YGocPQySbqo3+rV0rpVHKCPhCic2rsenSTOadO2JhY0YJrT/x+8oLyvFzb+RSa5YdfQe34Skizk07dIdpbmcopwsYo8eoqyo0KSdpa0drfo/Vk0vIgGt2/PUvC/YtHAueWnXjZ66jg/Ie/ZPIzi8K8f/+JX8G6mc3fonnUZUPs602Ggu7hHDJEN9Kj+TCtavRxUZidTKCvf33qu0vp7KlKcUoY4Ta9xpUorJ23AFxyeD/xbV1dpQcjwdQ5kOubMFli3rlu+m8BDLJ+gLNehLNMis7y4WIugM5KyOxlCkQe5qgdOYxnUyCuupp5566vn/Ta2nPDMyMjAYDLX6WFYzi13PPxtBEPhi9xWQliG1SAEg3K4RrBsN2jLw747jgC+YEDqBtu5tTV7Gnm/2PP52/uSp8/ji7BeV+o7NiwUgxPGfaaAZ9AaSLuUAlcMbq6IwK4M/5n9A4pkTHFi9gmWTnmHr159y+q8/uHL8MBKplD4Tp9Yqd8nezZKwPg1QmovzJbbOrrQb8gTdn37e5NNuyBPI5HefU3H09OKpjz6jUbtOgFgg2DvkwXjP/mlIpTI6PjEagBN//ELyxUiT9Xqdlt3fifL/3nlFCAs+J+OjeQhaLQDarCyyPvscAJdXX0XhXjdZ+/+vFO0Rnw/KBjYglaCKzDbKx//TMPWe+dTZUJKay5E7i9Eg2hultdqm+MB1tKklSC3lOI9ritS8Ppugnn83ubm5uLq6cu3atb97KPX8x7h27RoSiYTIyMg6b/v+++8zceLEBz+ou7Bjxw5atmyJwWAwLtNoNPj6+nLmzJkHso9aGWjjxo2rU7ji2LFj65OO/4X8dCKZcykFmNteBQT87fxx/+s1KEoDp0Yw4keQVf2ioZApmNlxJgAb4jdwNvMsABmlGSy5sISrhVeBR+9BEwSBPSsW8cNrLxnl5KviRkIh5aU6zK0UeATUnCNXXlbGxk/moCouwsm7Aa6+Aeh1OmKPHuTQz98D0GrAENz8ag6JfJgoLSwZPH0GT837gsGvvfPQ95f/63qudAon7bXXKT15itpoD2lS07g69HFimjU3+ST07EXZuXO13nfjTl1p0qUHgsHAX19+TN6NVOO6M39tJDc1BaVOT+MbokBL/k8/kfLsc+hycsic9zGG4mLMmzfHYczoOh+3oBfIXnFJDGUz/P/QWypPLqL8Sj5IJTg+GYz9zdDfot3JlN2c5LgfdLkqMj47Q8H2pDptl78xnoyFZyi/Zup5Ljl2m/esxb2pRSo8RS+aphZhjtrsMor2i8aq/ZAA5E71of7/JMaPH49EIkEikaBQKPDz8+PNN99ErVY/sH0cPHiQnj174ujoiKWlJY0aNWLcuHFoNBpjG0EQWL58Oe3bt8fa2hp7e3vatGnDl19+SVlZGQAffPABEomESZMmmfQfGRmJRCIxGksVL7iurq4UFxebtG3ZsiUffPCB8f+KY7/z8+mnn9Z4TB999BFDhgzB19fXuOz06dP06tULe3t7HBwciIiI4MKFCybbCYLAwoULCQoKwszMDC8vLz766KMa95WXl8dTTz2Fra0t9vb2PP/885SUmN57d+v3/PnzhIWFYW1tzeDBg8nLyzOu0+l0tG7dmlOnTvGgyc7ORqlUUlpailarxcrKipSUlAe+n9qi1Wp56623aN68OVZWVnh6evLMM89w48aN++p34MCBLF8uqkZPnDiROXPmPIjh1omMjAy++uor3n33VlrCoUOHGDx4MJ6enkgkEv78888693vt2jWef/55/Pz8sLCwICAggFmzZpncv/369UOhULBmzRrjMqVSyeuvv85bb711X8dVQa0MtFWrVmFjU3t55SVLluDs7HzPg6rn/tEatOSr82tso9FryFPnUa7T887GS8zcJKoUhviLN2641AZST4GZHYz5FSzsa+yvtVtrhjcS1QU/OPYBL+99mYgNESyOXIxBMBDiGIKLRe2L+j4I4k8e5cLu7eSmpvDb3Hc5v+OvKo2HivBG3xbOSKupjQRgMOjZ+vUCclNTsHJwZPg7c3j6k68Y+/GXhPbqh8LcAifvBoSPeOqBHoeg1aLNzKzTNhKJBPeARiiUD1euu+TwYTJmz0afl0fR1q2kjBvH1f4DyF35Pbr8qq9BfUkJqZMnUx4bCzqdyUd74wapU6aiSU2tcts7kUgk9Hnxf3gGhVBeWsrGT2ajKikmP+MGxzesAyAkLQfH7j3w/vYbpFZWlJ05w9XBj1G8YwfIZHjMmY1EVnelRm1mKeUJBahj8lDVwTgRdAZ0BQ/uhfBu+9KkFpt+bpQg6O/NoCzakwyAZStX5E4WWLf3wDpcLLSevz4OTWpxTZujL9Ui6A1VrhMEgfyNCeiyVZQcTkVfWF6rMelLNJSeykCXoyJ7+SVKjt9AEAQMah0lh296z3rVLffsdhSetctDEwSBgo0JoBcwC3LAIvTRPu/qqR39+vUjPT2dq1ev8sUXX7Bs2TJmzZr1QPqOjo6mX79+tGnThkOHDnHp0iW++eYblEolev2tUg1PP/00r776KkOGDGH//v1ERkby/vvvs2nTJnbt2mVsZ25uzsqVK4mPj7/rvouLi1m4cGGNbdLT000+33//PRKJhOHDqy/DUVZWxsqVK3n++eeNy0pKSujXrx8NGjTg5MmTHDlyBBsbGyIiItDejFAAeOWVV1ixYgULFy4kNjaWzZs3065du6p2Y+Spp54iKiqK3bt3s2XLFg4dOlTJS3K3fl944QV69uzJuXPnKCwsZN68WwrGn332GeHh4Xcdx71w/PhxWrRogZWVFefOncPR0ZEGDRo88P3UlrKyMs6dO8f777/PuXPn+OOPP4iLi+Oxx2pOl6gJQRA4ceIE4eGiMvfhw4eNfz9KVqxYQadOnWjYsKFxWWlpKS1atGDRokX33G9sbCwGg4Fly5YRFRXFF198wdKlS3nnHdPJ7vHjx/P111+bLHvqqac4cuQIUVFR97x/I8J9oNFohMuXLwsXLlwQ1Gr1/XT1t1JYWCgAQmFh4d89lAeCTq8Txm0fJzT/obmw/MJywWAwVGpzIeuC0HN9T6HV6tZCxJKVQsO3tgi+b28Rvtl7Rei5vqfQ7IdmwpGFDQVhlq0gnF5Z630XqAuErr90FZr90Mz4Gb99vLAlcYug1j3aa0RdWiIsefFpYeHIgcLKVyYIC0cOFBaOHChs+/YzQVN+aywlBWph5euHhG9f3Ctcjcyqsc/9Py4XFo4cKHz51ONCenxcpfV6nU7Q63QP/FhSX39DiA5uLGQu/EwwPIT+7xV1fLwQ27qNEB3cWEidNk24MXOWEBvWSogObixEBzcW4jp0FEqOnzDZxqDTCSkTXxSigxsLVzp3EVRxcYImI1P8pKYKV4cNF6KDGwuJgwYJuuLiWo+ltCBfWD7lWWHhyIHCr7NnCOvnvCMsHDlQ+KF/DyEquLGgiooSx5yYKCT0628cY8YnC+75+EvOZgjX3zokXH/rkJD+2WnBoK98r92OJqtUyN+SKKTNPiZcf+uQUHw07Z73XRt0xeVC+qenjWO8/ZO34Uqd+1NfKxS3n3FY0OaqjMsNOoOQtfKScP2tQ0LaRycEXWHV97oqIV9Ife+IkP7ZaUFfqqm0/vbzef2tQ0LBrmu1GlfJqXRxm3cOG7fNXR8nFOxMEr+bhXf/bmpCFZcn9vPp6ZrHcVocf+p7R0zOzz+R/9rvXm0ZN26cMGTIEJNlw4YNE8LCwoz/6/V6Yd68eYKvr69gbm4uhIaGCr/99ptxfV5enjBmzBjB2dlZMDc3FwIDA4Xvv/9eEARB+OKLLwRfX98ax/Drr78KgPDnn39WWmcwGISCggJBEARh1qxZQosWLYQ+ffoII0aMMLY5f/68AAhJSUmCIAhCUlKSAAhvvPGGYG1tLWRmZhrbtmjRQpg1a1a1YxkyZIjQs2fPGsf722+/CS4uLibLTp8+LQBCSkqKcdnFixcFQIiPjxcEQRCio6MFuVwuxMbG1tj/7URHRwuAcPr0rXtt+/btgkQiEdLS0mrdr4WFhRATEyMIgiAsXrxYGDBggCAIgpCYmCg0atRIKCoqqvWY6sJbb70lvPLKK4IgCMLChQuFJ5988q7b7N+/X2jbtq1gaWkp2NnZCZ06dRKuXbv17Nu8ebPQpk0bwczMTHBychKGDh16X2M8deqUAAjJycn3tH1MTIzg4OAgGAwGITs7W5DL5ULxXX6ra7pnKq7f8+fP12kcTZs2Fb799ttq1wPCxo0b69RndSxYsEDw8/MzWZacnCwAQkJCgsnyHj16CO+999597/OeZbcOHz6Mr68vPXr0oHv37vj4+LBjx477tRfreQD8EvcLZzPPIiDw9fmvmX5gOqXaW7kTG65sYPyO8WSVZaExlJOqWIaNdQHfj2tLvzAJWWVZmCGhdUkB+LSHVuNrvW87MzvmdZ5HiGMI45qMY/PQzazqt4qB/gMxkz3awquH1/5IaX4eDh5ePLPgW7o9/TwSiZToQ/v4ZeabnNu+mTNbN/HrnBUUZ5/C3CIZj8Cqiywb9HrObt1krP/Vb8o03AODKrWTymRI78ETUxMGjYbiPaJUfu5333F94ovoCwoe6D7uBV1+PtcnTcZQUoJlmzZ4zp+Px+wPaHT4EO4fzkEZGIA+P5+U558nd9UPRs9l1qcLKTl4EImZGd6LF2EeFITCzVX8eHnhvXgRchcXyuMTSJs+HUFfuwLBlnb2DH1zJgpzC65HXSTl8gVkSGiamo1N716YN2kCgJm/P76/rcfuieHY9OmNy9Qp93wOtBm37itdlgrVxewq26micshadpHMz85ScjgNQ5kOgIK/ElHH5VW5TW3Q5ahQReVWGV4p6Azk/hyDLkeFRClDZm9m/ACUnsqg/GpBlf0aNHrKzmehyzP18lV4z6xauyF3vKUmKpFJxMLOrpYYijSiQIbG9HvT5qjI/TkGQWtAl3Xz79s8afpSLYVbxVBopa/tzTGmI+iq9rbdjipaDF+17eGDXX8/kEDZ2UxjXpxtrwZIpPcu0lER4qjLUWFQ66psoy/RULhNHL9t7wYm5+f/BYIAmtJH/7nPUq6XL1/m2LFjKJW3xF8+/vhjVq9ezdKlS4mKimLatGmMHTuWgwcPAmLeS3R0NNu3bycmJsYkasjd3Z309HQOHTpU7T7XrFlDcHAwQ4YMqbROIpFgZ2caZj9//nw2bNhw19yW0aNHExgYWOtws8zMTLZu3WriGauKw4cP07p1a5NlwcHBODk5sXLlSjQaDSqVipUrVxISEmIMg/zrr7/w9/dny5Yt+Pn54evrywsvvGASbngnx48fN4Z7VtC7d2+kUiknT56sdb8tWrRg9+7d6HQ69u7dS2hoKACTJk1iwYIFdYoKuxspKSnY29tjb2/P559/zrJly7C3t+edd97hzz//xN7enpdeeqnKbXU6HUOHDqVbt25cvHiR48ePM3HiRGOO/9atW3n88ccZMGAA58+fZ+/evSaevw8++MAk7LQ2FBYWIpFIsLe3r9N2gwYNMn43hYWFODg44Ofnh16vx9vbu8b+arpnqsLX19ckNPdO8vLyiI6ONrlOHiaFhYU4OpqW3mnQoAFubm4cPnzYZHm7du0qLbsXap25bDAYkEpv2XOvvvoqa9asoXv37gAsX76cyZMnk5RUt7yBeh4sGaUZfH1OdLn2adiHA9cPsCdlD1e3XuXTbp+yLnYdv1/5XWxc2gy9tBCZxXW8G6+jtf9gNsZvB6BNWRnmEhkM+hKkdbPjw73CCfd69O7u27lxJYYLe8Rj6f3CFORKJW0GPY6rrz9bvvyErKREspISTbYpUMHK/+2iabeeNO/ZDydvH4pysrm8fxeX9u+mJFcMYes04qlKNbkeJqqzZxFUKqTW1gg6HaVHj5L0xAi8v/0G88Z/T9kCg0ZD6ssvo01NReHjg9c3XyO5+YIjtbLCYcQI7AYPJmPWBxRu2kTWJ5+gvnwZi5YtyfvhBwA853+MRfPmlfpWuLnhvXgxyWPHUnroMFkLPsVtxtu1GpdLA18G/u8N/vz0QxAEAtJzsdLocJliaoTJrK3xnDv3/k4CoE0XDTS5swW6HBVFe1OwCHUxMQZKT2eQv+FmeJIEzBs7YtXOHdXlXMrOZpK7NhbXKS1RuNZNXEkQBHJWR6HLUmHexAnHkUFGMQrhZqig5loREjMZri+1QOFmZdw2f2M8pSczyN+YgNsrrZDIb93jgl4g96doyuMLQAJmgfZYtfNAaikXl0kl2PSoXLBZai7HeVwTshZHok0tIf+3KziOboxEKsFQpiX3hygElQ6FhxW6XDXlVwsp2JSI/eNivb/CbUlinTI3S5yfa0bGp6cxFGtRReVi2aL6UEFDuR51vBhKa9HMGYW7FQpPK/LWxYq5Zy4W9x1qKLNWIrNToi/UoE0vxcyvcp5q4dYkDGU6FO5WWHeuulbifxptGczzfPT7fecGKK3u3u42tmzZgrW1NTqdjvLycqRSKd9+K4oJlZeXM2/ePPbs2UPHjh0B8Pf358iRIyxbtoxu3bqRkpJCWFiY8eXw9hfkESNGsHPnTrp164a7uzsdOnSgV69ePPPMM8bc/Pj4eIKDg2s93latWjFy5Ejeeust9u7dW207iUTC/PnzGTx4MNOmTSMgoOY86B9//BEbGxuGDRtWY7vk5GQ8PU2/WxsbGw4cOMDQoUP58MMPAWjUqBE7d+5EflPM6urVqyQnJ/Pbb7+xevVq9Ho906ZN44knnmDfvn1V7isjIwNXV9NcUblcjqOjIxkZGbXud8WKFbz00kssXLiQ8PBwZsyYwU8//YSlpSVt27YlIiKCxMRERo0axdz7/C3w9PQkMjKSoqIi2rRpw8mTJ7GysqJly5Zs3bqVBg0aYG1d9eRvUVERhYWFDBo0yPh9hYTcytX/6KOPGDVqFLNnzzYua9GihfFvZ2fnu37Pt6NWq3nrrbcYPXp0nbUiVqxYgVqtZtKkSXTo0IHx48czc+ZM7O3tmT59eo3b1nTPVEVAQECNBlxKSgqCIFS6Lh8GCQkJfPPNN1WGD3t6epKcnHzXZfdCrd+827dvz7nbEvc1Go1JXG2DBg0eaJJtPbVHpzfwxe4rnLyay8cnP6ZMV0YLlxYs7LaQVf1W4WrhytXCqwzfPJzfr/yOBAn+sicoThmDt+YlXC3cSC1N5o2Db3AoRXy4havUEP4KuDX5247rxpVYDvy0Eo1aVaft9Dodu5d/C4JAcKce5N6w5+iGBDKvFeHTNJSxH39Jiz4DcPFthVQRjEwZjE/Tdtg4uaAuKebs1k388NpkVk2fzIqpz3P893WU5OZgbmNLpxFP0WH4qId0xFVTcvgIADa9e+P7yzoU3t5oU1O5Nmo0JTdndO8VXW4uGfPmUfDHRgyq2p1nQRDI+GA2qjNnkVpb47NkMXIHh0rtpObmeMz/GLd33wWZjKKtW8m8mcTt/PJUbPv3r3YfFs2b4fnJfADyfvyRrK++qjaf7U4CWrdj0Ctv0sTGEf+sfGz69MY85OGI01R40OwfC0BiIUeXbepFK79aQP6fCQBYtXfH/e12OI9rikWIEw6PB6L0tUUo15PzQxT6Um2V+6gOXVYZuizxO1NH55K1KBJttiguUHIojbKzmSABp6dCTIwzALt+fkhtFOiyVRQfMFVfLNiSKBpiMgkIUB5fQN6aGHK+uyQeRxu3ar1DcicLnMY2AZkE1aUcivamIOgN5K6NRZejQmZnhvNzzXAcFQwS0YtXcuwG6sQC43gdhjVCqpRh1U4s6lxyvOZkdvWVfNAJyJzMkbuJRq55Iwdcp4Zh3dULp6dC7st7VkFFHpomrXIemjohn7LzWeL4hzdCUkMeaz1/Pz169CAyMpKTJ08ybtw4nn32WWMOVkJCAmVlZfTp0wdra2vjZ/Xq1SQmipN6kydP5pdffqFly5a8+eabHDt2zNi3TCZj1apVpKamsmDBAry8vJg3bx5NmzYlPT0doFYiSncyd+5cDh8+bJKfVhURERF07tyZ999//659fv/99zz11FOYV1NbswKVSlWpjUql4vnnnyc8PJwTJ05w9OhRmjVrxsCBA1Hd/C0xGAyUl5ezevVqunTpQvfu3Vm5ciX79+8nLi7uruOrjtr027RpUw4ePEhycjJr165Fq9Uya9Ysvv32W15++WU6derEhQsX+OOPP/jrr7+q3M+kSZNMroHqkMvl+Pr6EhsbS9u2bQkNDSUjIwM3Nze6du2Kr69vtcaGo6Mj48ePJyIigsGDB/PVV18ZrxMQBWF69epV7b6nTp1ao9F+O1qtlpEjRyIIAkuWLKnVNrfj7u6Ol5cXJ06c4KmnnsLX15fjx48zcuRIfH19azS6arpnqmLv3r1MnTq12vUV19jdrt37JS0tjX79+jFixAgmTKhc/9XCwsIo6lPTsnuh1h60b7/9lhdeeIFu3boxd+5cZs2aRevWrQkODkar1RIbG8s333xz3wOqp+7sjMrkq73x/HxpKxqnfcglcmZ1nIVUIqWFSwt+Hfwrrx14jXNZ57BR2vC0/zt8slGCVAKfPh6O0jKAcTvGcezGrRsmXOkMXasuEPwoUJUUs2nhXMoKC1Cam9OpDqIbZ/76g5zrycgUVqTEhpAcI74gR+5OwdnHmqadPfFq8jhxZ2JQWkO3McE06+qFwaDn2oVzXNyzk6vnTpGXJr64+jQNJbRXBIHtOiFXKB7K8dZE6RHRQLPq0hnzxo3x+/03US3x6FEyP56Pdbdu99SvIAikv/seJQcOAJD58cfYDR6M/cgRNXrm8r5fReEff4BUitcXn2MWGFhtW4lEguPTYzELDiJt2nT0ubnYDhiAczXhHrdj268f5f+7Ss7X35C7ZCl5K1Zi07cv9iNHYtmubY01t3yd3TEcFcOBnKfcewhjTehLNBiKRaNK2dAWmy5eFO1KNnrR9Plqcn+OAb2ARagz9kMCTQwFiVyK09NNyFoUiT5PTe7P0bg839zEm1UTqigxrE/haYWhRIsuW0XWt5FYd/Sg+KAojGE/yB/zoCqMZws59oMDyFsbS9H+61iEuqBwtaTk+A1Kj6eLht2YxijcrSg9k0npmQzxWGVVe89ux8zPDofHG5H/+xWK96ZQfrUATVIREqUUp3FNkNkosWjihF1/Pwq3JVG45SrSm7XFrNp7YNZQnNW1bu9O8f7raK4VoUkvRelRtZdEfTO80aKJk8k1IXc0x37Ag6vFqfC0Rh2TV6VQSPEB8XxbdfBA6fPgQqf+VSgsRW/W37HfOmJlZUXgzefW999/T4sWLYwiGBVqgVu3bsXLy9QTamYmhgf379+f5ORktm3bxu7du+nVqxdTpkwxmWH38vLi6aef5umnn+bDDz8kKCiIpUuXMnv2bIKCgoiNja3TmAMCApgwYQJvv/02K1eurLHt/Pnz6dixI2+8Uf1v+OHDh4mLi+PXX3+9676dnZ3Jv2OCbO3atVy7do3jx48bo6vWrl2Lg4MDmzZtYtSoUXh4eCCXywkKupUOUOEdSklJqdKL6O7uTlZWlskynU5HXl4e7jfLoNxLv9OnT+fVV1/F29ubAwcOMHfuXKysrBg4cCAHDhxg8ODBlbaZM2cOr7/++l3PT9OmTUlOTkar1WIwGIzeWZ1Oh7W1NQ0bNqxRNGLVqlX873//Y8eOHfz666+899577N69mw4dOtRJQb0mKoyz5ORk9u3bV2fv2bx585g3bx6CIFBWVkZYWBgginJEREQgkUjYvn07XbpUHV1Um3umLlQYvPn5+bi4PBwxphs3btCjRw86depkVKy8k7y8vEr7r2rZvVBrA619+/acPn2aBQsW0Lp1axYsWEBcXBwnT55Er9fTtm3bSg+zeh4NsVfiWKGcywxb8WVxfLPxNHJoZFzvbOHMir4r2Ht9L8F2zRj3XTxQxqqGu2jxuxg69rECXr35XuGu0+HX7ytQ/H3y0IfXrKKssACAi3t30v7xJ6usAXZmy0Yid24RFdsMAtpyPeUl4nZSRRcEwRw3P1tsncy5GplDzvUSDq67Ytw+tKc3zbqK161UKsM/rC3+YW0pycvletRF3AKCcPT8+65rbWYm5VeugESCVSexrpnM3h6vLz7nSoeOaK5dQ5uZhcKtsny4vqic3HWxWLfzwDKs8vriXbtF40yhQOHmhjY1lfy1a8lfuxbLtm3xXPAJCg8P02327SPr5gPV7e23sa7mYXwnVu3a4f/nRsrOncemR/daFzR2njwZhbsH+WvWoI6KomjrVoq2bkUZEIDn/PlYNK9c382gVpM5/xMQBGz69KnW2CzcnoQuXy0WWL4Hj0eF90zmZI7UTIZ1J09KjqShy1ZRejKdkuM3xJA3b2scRwRV6cWRWSluhgVeQJNURN76OBxHBCNR3H08FQaadUdPzBs7krsmBs21IhNjwapT9eEfFs2dMQ92QB2XT/7GBGx7+FDwl+ghsI3wxaKp+ANoF+GLbe8GqOMLkFkrkDvcfcbSqo0b2qwySg6lokkqAgk4jmqM0vPWLLR1Fy+0WWWUncnEUKxBaqPArp/vrXNja4ZFUydUl3IoPX4D5bBGlfYj6A2oYsS8E4umTncd1/2g9KpayVFfWE55YgEANl28H+oY/tFIJHUONfwnIJVKeeedd5g+fTpjxoyhSZMmmJmZkZKSQrcaJr9cXFwYN24c48aNo0uXLrzxxhvVvmw6ODjg4eFBaan4zBgzZgyjRo1i06ZNlfLQBEGgqKioUh4awMyZMwkICOCXX36p8ZjatWvHsGHDePvt6kPDV65cSevWrU3C5aojLCyMn3/+2WRZWVkZUqnU5Fle8X9Fbajw8HB0Oh2JiYnGMLwrV8Tf39uV926nY8eOFBQUcPbsWWPe2759+zAYDLRv3/6e+t27dy8xMTGsWrUKAL1eb1SavF1x8k5cXV0rhVtWxbZt29BqtfTq1cv4jjxq1CjGjx9vlGO/G2FhYYSFhTFjxgw6duzI2rVr6dChA6Ghoezdu5dnn332rn1UR4VxFh8fz/79+3FyqvuzctKkSYwcOZLFixeTmprKvHnz+O2339i1axffffcdwF1tgLrcM3cjICAAW1tboqOjTQz1B0VaWho9evSgdevWrFq1yiTFqwK1Wk1iYqLRWK3g8uXLlZbdC3V6K5HJZMyYMYOtW7fyzTffMHnyZFq3bs3QoUPrjbNHTK4ql+yybLLLsmkS/xEnXNJRK8qxkLjwYuiLldorZAr6+fZj/ckSUvLKaGSjo2vmT1CYAoUp9MpJYVqeOEPW39IXSWDPR31IRlJjLnNpnxjGobSwpDQ/j4TTxyu1K8rJ5si6HynMyqQoO4uS3GzKS/IAAzIzX1r06c2T77Xlibfa0PeFZoz/JJzOIxrh4C7OuDZs5kT48Kq9P9aOToR06fG3GmcApUeOAmDevLlJGKHM1tYYtldWTS2X0rNZaJKKKNiSiKA1FVrQFxeTeTPu3umF5wnYtZMG36/Epl8/kCsoO3OOpOFPUHryVt/quDjSXn8DBAH7UU/i8PTYOh2L3MUF24i+xly12iCRSLAf9jh+G37H9/ffsX/ySaSWlmgSE0l+6ikKNv5p0l6blsa1MWMoPXoU5HKcqwmR0GaUUnwwFdXFHDTXa5aFrw5tuhjCoHAXX0ql5nKsu4jXS8GmRHRZKmS2SpyfaYJEUb1wjMLNCqcxjUEKqos5ZC27gK6gZnl5XUE52rQSMactxBGZjRKXCc2xvmmQmQU5YD/Yv0ZDWCKRiF49hRRNUiE5P0aBQZTPt+lmamhIZFIsGjui9K69d8iun6/RaLLr74dFE9OXAolEgsPQQMz87cTQwCGBlQo6W3cUJwjKzmdhUFUW5yi/Woig1iG1VqBs8HBrbxql9rPKTO6nsgvZIIjCJv/vhEH+I4wYMQKZTMaiRYuwsbHh9ddfZ9q0afz4448kJiZy7tw5vvnmG3788UdANJQ2bdpEQkICUVFRbNmyxejBWbZsGZMnT2bXrl0kJiYSFRXFW2+9RVRUlNFLM3LkSJ588klGjx7NvHnzOHPmDMnJyWzZsoXevXuzf//+Ksfp5ubG9OnTK0l7V8VHH33Evn37qgwlLCoq4rfffuOFF16o1fmJiIggKirKxIvWp08f8vPzmTJlCjExMURFRfHss88il8vp0aMHIIp7tGrViueee47z589z9uxZXnzxRfr06WN8qT516hSNGzcmLS0NED1h/fr1Y8KECZw6dYqjR48ydepURo0aZcw3qk2/FajVaqZOncry5cuNL9nh4eEsWrSICxcusGHDhvuWiG/YsCHW1tZkZmYyZMgQfHx8iIqKYvjw4QQGBlZrjAIkJSUxY8YMjh8/TnJyMrt27SI+Pt54Pc2aNYt169Yxa9YsYmJiuHTpEp988olx+2+//bbGEEitVssTTzzBmTNnWLNmDXq9noyMDDIyMkzqet0NR0dHAgMDiY6Opn///gQGBhIfH0/fvn0JDAwkMDCwRm9fTfdMVfTq1cuYF1oVUqmU3r17c+RmhFEFJSUlREZGGoteJyUlERkZWadadGlpaXTv3p0GDRqwcOFCsrOzjefsdk6cOIGZmZkxV7WCw4cP07dv31rvr1rqIvl4+fJl4ffffxfi4kR58R9++EHw9/cXFi1adN9ykn8n/ya54ZyyHOGFnS+YyNjf/nni41cFlaZqGfaY9EIhYMZWoeFbW4QLW5eIEvrftBWE62eMn/SrewWd5u8rmaDVaITvX31RWDhyoLBz2dfCkV9/FhaOHCj8MuutSm13f/etsHDkQOGzMROFr5//Wfj6+Z+FNTM3Csf+OCaUFVcvcW0wGIT8zFJBfx+y24+K66++KkQHNxayvvqq0rqM+Z8I0cGNhRvvvV/lttmrLhslx0vOZpisS589W4gObiwk9I0Q9LeVyDDoDUL65yeFlOlbhZiwzkJ0k6ZC7o8/CtqsLOFKjx5CdHBj4dr48YJBU1ki/VGhKywUUl6cZJTJT5/zoWDQaISS48eFuPYdqpX3v528P64Yz03Roev3NI7c9XGVpOD1aq1RQj/1vSNCeWrtywSoruQZt02bc1xQJ+ZX27b4SKpw/a1DQuaSyErrdAXqOknKFx28bjwXmYsjBYNWX+tt74bBYBB0ReU1t9Hpq5WkNxgMQvrnZ8Tv6XBqpfV5G+PFkgG/171kQF0xGAxC2hzx+ylPuSXRnfHlWbFkwokbD30MD5J/0+/eg6QqmX1BEISPP/5YcHFxEUpKSgSDwSB8+eWXQnBwsKBQKAQXFxchIiJCOHjwoCAIgvDhhx8KISEhgoWFheDo6CgMGTJEuHr1qiAIgnDu3Dlh7Nixgp+fn1EWvWvXrsLmzZtN9qfX64UlS5YY5dVtbW2F1q1bC1999ZVQVlYmCMItmf3bKSwsFJydnauU2b9TpnzixIkCUElmf9myZYKFhYVRzr82tGvXTli6dKnJsl27dgnh4eGCnZ2d4ODgIPTs2VM4fvy4SZu0tDRh2LBhgrW1teDm5iaMHz9eyM3NNa7fv3+/ybEIgiDk5uYKo0ePFqytrQVbW1vh2WefrSTjfrd+K3j77beF1157zWRZfHy80LZtW8HW1laYPHmyoNff/zNv3bp1QufOnQVBEIRDhw4JgYGBtdouIyNDGDp0qODh4SEolUqhYcOGwsyZM03GtGHDBqFly5aCUqkUnJ2dhWHDhhnXzZo1S2jYsGG1/VdcG1V99u/fb2zXrVs3Ydy4cTWOVavVCtbW1kZZeX9/f+HIkSO1Os6a7pmqrt+GDRvWWB5CEARh27ZtgpeXl8m5qrie7vzcfmx3O2erVq2q9pzdzsSJE4UXX3zRZNmxY8cEe3t74z18P0gEoXbZqp9//jnvvfceoaGhxMfHM3/+fCZMmEBOTg7Tp08nPj6e5cuX07wKVbZ/OhXhBIWFhXWOy32UROVE8eqBV8koFa14qUQKBlHKWgI8WVSCZ1YbnEZ+zaBQ09Amg0Fg+NJjnE8poG8TN5abfQkxf0HXN6Hnu/xTOL5hHcfWr8HSzp5nP1+KVqPmuynPIRgMPPPpt7g08AWgKCeLFS9PQDDoMbMbSWivDjTt7ImTV/WJvP82BL2eK53CMRQW0nDtWixbmbrMi/fvJ3XySygbNiRgp2mJC0EQSP/whFHOXeljg+uUlgCoIiO5NnoMCAINfliFVYcOxu3KkwrJXnbxZifFFG99D3QqZHZ26AsLUfr64vvrL8iqCL95lAgGAzmLFpNzsxileWhX1DGnQavCvEkTvL/9BkU16k4GtY70eScRNKIXxKKFC06j666GmfnNebRpJTiNDcGi2a0E8NLTGRTuuobD0EZ1DrvT5anJ/SlaVIeUgt0Af6zDPSt5wrK/u0h5YiF2A/3uO6xO0Avk/BiFoVSL87NNkVnX3sP5KCg5kU7BnwnInS1wm97aGCoqGAQy5p9CX6TBaXxTLBo73qWn+yd75SXK4wuwfzwQ6/YeaDNKyfzyHMgkeL7bHqnlo89RvVf+Lb979fwz2Lp1K2+88QaXL1+uMtyrnn83DRs2ZPbs2YwfP/7vHkqtEQSB9u3bM23aNEaPHl3r7caNG4dEIuGHm4rS90JOTg7BwcGcOXMGPz8/4/Inn3ySFi1aVCpqfS/U+i5bsGABW7du5cSJE5w7d47PP/8cEBP1Vq9ezZw5cxg5cuR9D6ieqvkz4U+e2f4MGaUZNLRtyJ9D/uSCcwQXrl1n01UNU8ufZ0ZePn1lZ9l07nql7decSuF8SgHWZnJmD/SHhJuqP40HPuIjqZ68G2mc3LgegO7jJmBubY2NozON2oru48idW4xtj/++HsGgRyr3oe2gznR9Mug/ZZwBqC9dwlBYiNTGBovQyhMflm3agFSKJjkZbWamyTpdtko0zuQSkEnQXC9Gk1qMoNWSPnMWCAJ2Q4aYGGcghpIZkdhg9+THIFegLyxEameH95LFf7txBiCRSnF5eapYM80nDIX/WCy7vIntkOE0XLumWuMMxPpYgsYAN8U47iXEUdALaDNNQxwrsGrrjue7He4pJ0ruaI7L5BZYtnQBAxRuuUrZOdOEeUOZlvKkQoBKYYP3gkQmweW5Zri9HPaPM84ALMNckZjJ0OWoKNiYYKyLpk0rQV+kQaKUYR5g/0jGUpFDV5GHVnG/mAc7/quMs3rqqSsDBw5k4sSJxlDEev47REVFYWdnxzPPPPN3D6VOSCQSli9fjk5XdW3KqhAEgQMHDhhLQ9wr165dY/HixSbGmUajoXnz5kybNu2++q6g1gaaIAjGWROZTFZJJrZPnz6cP3/+gQyqnlsIgsD8U/N5/+j7aAwaunt3Z93AdQSUFMApUVXmXd1zKEP6Y1BY4SHJIyf+FHmlt2KLM4vULNguKka93jcIj+wTYs0aOx/wuHuC8KNAEAT2rFiEXqvFt0UrGnfqalzXMkI0ImMOH6C8rJSi7CyiDuwGwN6zO637Vx/f/W+m5Gb+mVWnTkiqEEiR2dgYiy/fmYemSS4CRM+ZRXPRu1NyPJ3cH36g/MoVZPb2uL79lsk2gs5A2UWx1pvdAD8kCimGUkucX1mKTb9++CxdgtltD6N/AjY9e+IwVlTZktl6oWw8Comy+oLogiBQckKUMLbtKaoR6vPU1Urcq6/ko4rJrbRcl6sCnQGJQorsAecdSZUyHJ4Mxqa7OL7CrVdNxqeKyQODaBjKnf4+IZ9HhdRMht0Asfh06ekMspddRF9YbhRJMQ92qJWoyoPAKLV/oxTBIFAWKRpolmEPR0Wsnnr+Sbz66qv4+NSs4lrPv4+mTZty8eLFf6VntGXLljz99NO1bi+RSEhOTr7v67hNmzY8+eSTJsuUSiXvvffeA1PerPW38cYbbzBgwAA6depEy5YtqyxK97DrEfx/ZOe1nayJWQPASy1e4queX2Ejs4C/XgEE/qILRw3NaenvjrRRHwB6SU6z9eItuePZf0VRXK6jhY89T3f0hdit4orGA0XlrX8AyRfPcz3qInKlGb1feMkkpMu7SXOcvBugLVcTdXAv+1f/jCCI3rM+L/RFXoMAw7+Z0puV6K06V5/AbNmuHVDZQCu/aaCZNbTFuqPoTSo7n0n2ku8BcH3rrUq1y9RxeQhqHTJbJdadvXB8UpQqLk/UYz/qTSwfgCrRg0YQBLQZt2bPyuPyKdyWVG378oQCdNkqJEpRdVHuLD5ItamVvWiGMi05P0aRuzpaNMhuo0LBUeFu9UBqbN2JRCLBtk8DFO6WGMp0Jsekuikrb/6QVQv/SVi398D52WZILORorheT+c15ys6LXuOHrd54O4oKJcf0UsoTC9AXapCYybBo/P/nu6innnrqqefhU2sD7fXXX+fEiRNMmzaNI0eOMHHixIc5rnoAvUHP4gtiMcEuzmOY3HKymHd2cglkXkJvZs8H6jFYKGSEeNhC40EAREjPsPG8GIawNyaTbZcykEklzHu8GTJBD3HbxB38jeGNNxIK2LbkIpcPpaFR6YjcJRqNzXv2xc7VnaIcFSf+TGT7skvEHk+neU+xqPGZLRtJOH0AAP82g/AOrlzj6b+AvrAQ1SWxMLB1587VtrNqLxpot6stwm0etIa2KBvYIHdWil4Xz3bYDuiP3dAhlfqqCNeyaOmKRCrBopkzthGid7Jgc4LRY/Eo0Zdqyfs1jtJzmVWu1+Wo0OeXg0yCwwhRvavkSBolp9KrbF9yXFxu2doVqbkcpfdNj0gVYY7qhALQCyDcEfqJ+IIOoKimNteDQCKTYj+sEUjEsEx1YgEGjZ7yK6KS2oMIb/w3YR7kgNvUlijcxbpv+kINyCSYP4LcswrkjuZIzGSgM1C0OxkQyxU8Kg9ePfXUU089/z+odR00gObNm/8rRUD+rexK3kVS4VUEvTl7TgST260cJ7kaDn0KwJmgaeSetqODjx0KmRSC+iJIFTQijYLr0cSkN2fmJrE44vOd/WjqaQfXjoAqDywcoEGnv+W4BEHg4No48m6UknQhh8O/nqEsRzQwHL07sPnrSK7H5ImaOcDV89kolOZI5WYU52QDIDdrSMSEByBj+g+l9PhxMBhQBgZUqkV2OxatW4NUijYlBW16OgoPD/SlYtFiAGUDWwwlJajO/4nCZwBmwX1w/7BXJdEJg0pnrCdl2fJWuJZNdx90WSrKzmeR+3M0tr0aYNOzwUPxGt2JoDOQ+3MMmqRCVDG5WDZ3qfQirL5prJj52WHV2g19QTlFu5Mp+DMRuZOFSW6SrkCN+ma4onUH8ZwqfGwgMhtNauXiwxV9A5RFZmPTq4HxvN3uQXuYmDWwxaq9B6Un0inYmIBtn4YIWgMyezMUnv++mlP3i9zJApeXWpC/IR7VhWzMgxwqSfM/TCRSCQoPK7F4dopo1FdVY7Ceeuqpp5567odaTftNnz7dWGCxNsyYMYO8vLx7HlQ9ovdsyU3vmSa3CxqtGb+euQ4nloK6EJyD+V0v5mm1aXhzBtncDomfWDi4j/Qs474/RVqBCi97C17tfbPIa0V4Y1B/kD26F5vbuRFfQN6NUuRKKQ7ulpQXRwICUnkDjm7I5Xq0aJz5NHGkzQBfbJ3N0WpkSGS3ama0HTIKc+v/blJ+yc3wRuvONReClllbY960KXArzFGTInrP5C4WSM0kpE2bjjpyG4KuDInCDk1yWaV+VJdyQC8gd7M08QpJJBIchjfCqoMHCFC0J4Xcn2MwqGtOytXcKCHj87NkfHGW4sNp1eZ4VYcgCOT/mYDmphiGoNajiq38TKnwJpk3Ej2pNj19sGjhAgaB3NXRFGxLQpstHm/pyQwQwMzfDoWbeIxKH7Gulya12CSvVhAEY98geuq0txlxRg/aQzbQQKwnJrVRostRkb8xHhDD+mpb7Pu/hlQpw3FUMC4vtTCG4T5Kbi+2LbNTYub394vm1FNPPfXU89+iVgbaV199RVlZ5Ze66li0aBEFBQX3OiYT0tLSGDt2LE5OTlhYWNC8eXPOnDljXC8IAjNnzsTDwwMLCwt69+5NfHz8A9n338mOaztIKkxCJlihyRdzkDYdj0E4IcqK0+1Nztx8EW/d8LYwv4owR9lpsorFYrdzH2+GpVIOggAxN5UQQwY9mgOpgksHxPDL4PbujJgRhkwWA4DCKgxLWyWt+zVk7Icdeex/LWn/mD9j53TksVdb4t+6N0jMsHVpTsdh1Yf9/dsRBMFYoNqqhvDGCoxhjhUG2m3hjZnzP6H0yBEkShmWoaIhXxHmdzul5yvEDlwrvfhL5FIchgbi8EQjkEtQR+eStSgSbWbVkzZlkVlkL7mALqsMXWYZhVuvkj7vJLm/xFJ+taCSwFBVlBy5QdmZTJCA8uYL8J1hhoLWQPlV0YAzvxnqKpFIcHyiEUpfW4RyPSWHUsn87CzZyy9SelI87opizgBKD2uQSm6GzN0qDq3LLBMVAhVSY45Txf4Nah36m4WkFTeLnj9MpOZy7B/zB0RDFcD8/1l4451IJBLMGtg+Uu9ZBYrb1GItb4YD11NPPfXUU8+DpFa/boIgEBQUVOsZ27p422oiPz+f8PBwevTowfbt23FxcSE+Ph6H28QNFixYwNdff82PP/6In58f77//PhEREURHR/9rRUv0Bj1LLywV/ynsCgZzZFIJ/Uo2IlEUgktjchsOIClnPwCtGtxmoAUPgK3TaSVNwIV82oc2oUfwzRCcjEtQmAJyC/Dv8YiPSqQkv5yrkWKYYvPu3lw5cQRNWQk2Ti4899WzyGSySi88EqkEn8aO+DTujvalcGQS6X/ae6A6dw5dZiYSc3Ms27a5a3vLdu3IXbGSslOngVsCIbqMGPJ//hkAzwWfYNmqGaqYM5Rfyafk2A2sOnogkUjQFaiNnirLltWHa1m1cUfhZkXuz9HoslVkfnkO88aOWLVzxzxINP4KtydRckQ0wM2CHLBo4kjpqQy0N0pRRWajisxG7myBVTt3LFu7IbOq7AVVxeZRuO0qAHYD/TEPtCfzy3Oo4/IwlGmNcubl1woRtAaktkrkbrcMJYlChsuEUNRxeZSeykAdl2c05GR2SsxDnG5rK0XhYYU2rQTN9WLk9uIzoyK8Uelnh2U7d1RRuZRdyMZuoJ8xvFFmp3xk0uoWzZwxb+yIOjYPqaUcM996r83fheI2D1p9eGM99dRTTz0Pg1oZaKtWrapzx25ubnXe5k4++eQTfHx8TPZ/e80BQRD48ssvee+99xgyRBQ9WL16NW5ubvz555+MGjXqvsfwyFEXsv30V1wruoaNwpYbme2RSyW80MaR5y9sF9t0e4uz18WX8Eau1tjd/pJo6wFebSDtDC95xDFo8Ihb62Jves8Ce4Hy4c/8V0XUkTQEg4BHoB1OXtbsWCyGXLbo0x+54u6Xo0Lx7w9rLE8qRNALmPnbVTJGBa2WjA/nI/dshWX7pkjNqpeMr8CiVWuQydBev47mehqa62IoXv5PXwPgMm0atn1EhU+rDh6UHk+nYHMimtRiHB4PpOymwWzmb4fcvub9KX1scH05jPzfrqCOy0cdk4c6Jk80VmyUxjBAmx4+2PZpiEQqwbqDJ5rUYkpPZVAWmY0uR0XhtiQKd17DoqkTyga3Fck1CBTtTQEBrNq5G4s0Kzys0KaXUnYpB+v2Yv6Y+rbwxkpeP5kEiyZOWDRxQldQTtmZDNRx+Vh39UIiM22r9La+aaCVYNncxbTvIAfMAx2QWikwlGpRxxegz1cDjya80Xg8Egn2jweSvz4Oi+YulY6hnkeHws0Sy9ZuSM1lj/QaqKeeeuqp5/8PtTLQxo0b97DHUSWbN28mIiKCESNGcPDgQby8vHjppZeYMGECAElJSWRkZNC7d2/jNnZ2drRv357jx49Xa6CVl5dTXn4rnKmoSDR2tFotWm3dcmUeOOtGs0x3FZQKOtv2Zr3BnMaeNryo3I6tpIw4gzcSx+6cPieGa7VqYFdpzNKgAcjSzjDO4TJ6c6lxvTxmCxJA16g/wt9wnHqdgajDonelSRcPUuNiyEi4glQup3GXHn//uX8E6PPV5Hx3EQwgczDDorUr5q1ckNko0d4opeDngygCJqJUmIO5jPJCFVLLu9ymZkrMmjah/OIlCvedAZ0jgrYUQ+ENbAYPwvbZ8cZza9W/ARI7JSU7kyk7l4UmoxShXAybMwt1qt13YCbBbmwwVtkqVGezUJ3PRl+oESXHlVJshwVi3tQRnV4HYtdI3MyxHuyLZV8f1JdyUZ3JRJdWiupiDqqbtdduR+Fni1X/BsYClGahTmjTSyk9l4lZK7GumzpOzElTBNjUPG4rKRbdPLHoJoY23tlW5iFOVpRfL0Kr1SJo9MZC0HJ/G3QGHWbNnVCdyKD0bAYSc7Gsg9TV4tFes5ZS7MeHVHkM9TxabIaKE4X/5u/h3zz2eh49ubm5hISEcOrUKXx9ff/u4dTzH+LatWv4+flx/vx5WrZs+cD6Xbp0KVu3buWvv/56YH0+Sv4elYhacvXqVZYsWcL06dN55513OH36NP/73/9QKpWMGzeOjIwMoLK3zs3NzbiuKj7++GNmz55dafmuXbuwtPx7PEsAzsXR5OWc55qrM3Z6PdPOfU+yxAZbnTfW55YB8JVuGKXrj5JSKgEkyPJT2LYt2aQfa7UVvQCSDpK5SJTSl2DAqyAKA1J2XZOgTd32aA8OKEuXoyqyQGpmIPbGabL+PAiApbcvB44ee+Tj+TtwzFbiZxBDpPT55ZTsuU7xnhTKzQ2Yq2WAIxIFCBhABTErj5AccPf8T2dHRxyBrEOXsHPphj43AVXDhsS3b4+wfXul9jYhcvyvWMMNMVzPIBE4lHYWfebd88PuRNIc7POUWBfJyXZXo04+Acl32agBWDjJcMo2Q6Ex9QZpzAxkOOWj33XNuExRLqE59miTi9nzxw4EiUBolgMCAgevnkZ/ve7jrsC8TEZT7FAnF7Jt6zZsCxQ00ttQbqZn16n9IAHLEhkh2FEWlYPaXI8lci5nXCF/2+V73m899fyd1CWv/L/E+PHj+fHHHwGQy+V4e3szYsQI5syZ88DSIg4ePMjs2bOJjIxErVbj5eVFp06d+O6771AqlYAYAfTdd9+xcuVKoqKikMvlBAYGMnbsWCZOnIilpSUffPABs2fP5sUXX2Tp0qXG/iMjIwkLCyMpKQlfX1/jC66LiwuJiYnY2NgY27Zs2ZKhQ4fywQcfVDr+CiIiItixY0eNx/TRRx8xZMgQE+Ps9OnTvP3225w9exaJREK7du1YsGABLVq0ACAuLo5JkyYRHR1NYWEhnp6ejBkzhlmzZtUYDfO///2Po0ePcvnyZUJCQoiMjDRZX3Fe7sTS0tKYYrN7926mTJlCRkYGQ4YMYeXKlcZzX1hYSNu2bdm9ezcNGzas8bjryunTpxkyZAg3btzgxo0bBAQEUFhYaNz338Hy5ctZu3Yt586do7i4mPz8fOzt7e+rz4EDBzJkyBAmTpzIxIkT8fb2ZubMmQ9mwLXgwoULzJ8/nyNHjpCTk4Ovry+TJk3ilVdeMbZ57rnn+PDDDzl8+DBdutQsuPZP5B9toBkMBtq0acO8efMACAsL4/LlyyxduvS+vHozZswwKbRdVFSEj48Pffv2xdbWtoYtHyKCgPTnxYy0E/f/jN4SD0Maa5QfkSdricKgotguiO2Z7bAuUFKuMwAGnh3cFV+nymE2wnerkWZF4V1w0nSFfzf6PDbyERxQZTZ/cQEoomVPX5p2ceD7DT8A0H/c83g0avy3jOlRU7T1GqqEDCzauKJoaIPqTBba5GLM1TIEQY8u9QxSi2yc3nqTgpXROGeZ02hwK5S+NV+Xpba2pB84iKVGNP4EbTYhP/5Ac6fqxST0BeUUrI1Dl16GRVNnIh7r+ECP9X5oUcWy/IJoNFeL6ODYHJmNgqJzV1F62xAx5P7GLRgEsmNOI9MY6Nu2O2Wns1CRgX1zDwYMFAV6BEEgN/0C5KqxLBMfm20jOpnkvtVTz7+JisiR/4/069ePVatWodVqOXv2LOPGjUMikfDJJ5/cd9/R0dH069ePl19+ma+//hoLCwvi4+PZsGEDer3e2O7pp5/mjz/+4L333uPbb7/FxcWFCxcu8OWXX+Lr68vQoUMBMDc3Z+XKlbz22ms0atSoxn0XFxezcOHCKo2Xqo6/ArO7hNKXlZWxcuVKdu7caVxWUlJCv379eOyxx1i8eDE6nY5Zs2YRERHB9evXUSgUKBQKnnnmGVq1aoW9vT0XLlxgwoQJGAwG43tddTz33HOcPHmSixcvVlr3+uuvM2nSJJNlvXr1om3btoD47jhmzBhmzJhBREQETzzxBMuXL2fq1KkAvP3220yaNOmBG2cAx48fJzxc/N04fPgwbdq0+VuNMxC/v379+tGvXz9mzJhx3/0JgsCJEydYsGABIB7nt99+e9/91oWzZ8/i6urKzz//jI+PD8eOHWPixInIZDLj96xUKhkzZgxff/11vYH2oPHw8KBJkyYmy0JCQtiwYQMA7u7uAGRmZuJxW62ozMzMGt2kZmZmVT6QKh4ofwtXD3Iu8zxXPN0wl5kxYtRf/LXwBQZLjuCafw4Aq77v0nCbNddyxZlPJyslgW52VQtmjFoDCXtAMNxaJpUhDR6I9AEeo06r58DPcaTdJkkOIJNL8Q11pklnTxw9rMhJLSHjahFSqYTQbj5c3PMHeq0WV98AfEKaPVTRj+LDqZRfLcRxVGOkZrKHtp/aoL/psbIIdMCypSu2bT3RZpZSuO0IOV+8B2jw2/QnZn5OaNu5U3oqg+LNSbi90gqJvHrRVdt27UiXyZA5imp/zhNHYHHz/qgOhYsCt5daoorOwyzQHtk/PL/PqpU7mqtFlF/MMeb+mAc7PpB7VuFljSapCEO6Gm3CTcGUxk4mfVuFuVK0J0X8RybB3MMGiay+QHE9/07+C/m894qZmZnx/cHHx4fevXuze/duo4FmMBj45JNPWL58ORkZGQQFBfH+++/zxBNPAKKA2dSpU9m1axclJSV4e3vzzjvv8Oyzz7Jr1y7c3d2NL68AAQEB9OvXz/j/+vXrWbNmDX/++acxfx7A19eXxx57zMR4Dg4OxtXVlXfffZf169fXeFwvv/wyn3/+OVOmTMHVtXoBm9uPvzZs27YNMzMzOnToYFwWGxtLXl4ec+bMwcfHB4BZs2YRGhpKcnIygYGB+Pv74+/vb9ymYcOGHDhwgMM3y8hUx9dfi/nT2dnZVRpo1tbWWFvfEuu5cOEC0dHRRi9jTk4OOTk5vPTSS5ibm/PYY48REyOqRR87dozTp08/NIPi2LFjRgPtyJEjxr9r4vfff2f27NkkJCRgaWlJWFgYmzZtwspK/J37/vvv+eyzz0hISMDR0ZHhw4fXafyvvvoqAAcOHKjz8VRFXFwcgiDQpEkTcnJySEhIoH379jVuU9M9cy8899xzJv/7+/tz/Phx/vjjD6OBBjB48GD69OmDSqXCwsLinvb1d/GPfrsIDw8nLi7OZNmVK1eMsx5+fn64u7uzd+9e4/qioiJOnjxJx47/HG/AXREEOPAxv9iKD5wB/gPJLrPm5fLJfGx4BkEiA58OSEMeY2yHWzM+rRpWFkcw4ugH7SZA+xdvfdq+IIqIPLBhC+z/KZbYEzcozlNTkl9u/BRmq7iw9zrrZp/kj4VnOfKbWPrAr6UL2SnRHN+wDoDWA4c8VONMEASK9l0XxSyicx/afoz70xsQDFWH2wk6A5obooiG0vtWCIrUXEfeig8RNCU4TXoRs5tCOHb9fJFaK9Blqyg+cL3G/UqtrLDu1g+phQNIBKw7hdTYvgKJQoZlC5cq1RT/aVg0cwK5FF22CtXN79I8yOEuW9WOiu9DdUkUMUEKZoH2Jm1uV+xTuFrWG2f11HMbgiBQpi175J/alO2oicuXL3Ps2DETL8fHH3/M6tWrWbp0KVFRUUybNo2xY8dy8KAYlv/+++8THR3N9u3biYmJYcmSJTg7i7mx7u7upKenc+jQoWr3uWbNGoKDg02MswokEgl2dqYqrfPnz2fDhg0mJYaqYvTo0QQGBjJnzpwa2x04cABXV1eCg4OZPHkyubk1/zYePnyY1q1bmywLDg7GycmJlStXotFoUKlUrFy5kpCQkGpz1BISEtixYwfdunWrcX91ZcWKFQQFBRm9JC4uLnh4eLBr1y7Kyso4fPgwoaGhaLVaJk+ezLJly5DJHtxk7ZEjR7C3t8fe3p7ff/+dd999F3t7e5YuXcrXX3+Nvb098+fPr3Lb9PR0Ro8ezXPPPUdMTAwHDhxg2LBhxut6yZIlTJkyhYkTJ3Lp0iU2b95MYGCgcfvx48fTvXv3B3YsNTFo0CDs7e1p06YNhYWFODg44Ofnh16vx9vbu8awyZrumarw9fU1huXWlsLCQhwdHU2WtWnTBp1Ox8mTJ6vZ6p9LnT1ohYWF6PX6SichLy8PuVz+QEMEp02bRqdOnZg3bx4jR47k1KlTLF++nOXLlwPig+zVV19l7ty5NGrUyCiz7+npaQwP+FeQdJDstJPs9vECYFTwKCKTCgAJkV5jkDzzIcjNQSplRGsfFu6KQ601mNY/+xs4uyOZuJNpaIrXYW5loOvYqbj6ioVjS/LLiTmWTvKlHNJveiQAGjaVsOWL+QgGA0269iSky8OV+zeUahFUotCE+kr+Q5XF1pdqyVoUidRMhuvLYZUUGrWZZaATkFjIkTmZo8vPp3DjnxT8+iv63FyUAQE4vfCCsb3UUoH94ADy1sVStP86Fi1cULhUH1Ln8Nx0Cn5PROFti0Tx93oKHwZSczkWTRxFURG9gMRcbmLo3g8VBavVcTfl9auosSV3skDZwAZNSrFJMe966qkHVDoV7dfWPIv+MDg55iSWirqFGm/ZsgVra2t0Oh3l5eVIpVKjR6K8vJx58+axZ88e40Svv78/R44cYdmyZXTr1o2UlBTCwsJo00Ysg3K7QTJixAh27txJt27dcHd3p0OHDvTq1YtnnnnG+H4UHx9PcHDti6y3atWKkSNH8tZbb5lMSN+JRCJh/vz5DB48mGnTphEQEFCpTb9+/Rg2bBh+fn4kJibyzjvv0L9/f44fP16t0ZKcnIynp6fJMhsbGw4cOMDQoUP58MMPAWjUqBE7d+5ELjd9dnbq1Ilz585RXl7OxIkT72pA1gW1Ws2aNWt4++23jcskEgnr169n2rRpvPLKKwwYMIDnnnuO+fPn06NHD8zNzQkPDycnJ4eXX37ZxONyL7Rp04bIyEhiY2MZM2YMZ8+eJS8vz3jc5ubm1Rov6enp6HQ6hg0bZnQ+NG/e3Lh+7ty5vPbaaya5VRWhnCBGmhkMt0VKPURWrFiBWq1m0qRJdOjQgfHjxzNz5kzs7e1N0oaqoqZ7pioCAgJqNODu5NixY/z6669s3brVZLmlpSV2dnYkJ98tMf6fR50NtFGjRjF48GBeeuklk+Xr169n8+bNbNv24MQn2rZty8aNG5kxYwZz5szBz8+PL7/8kqeeesrY5s0336S0tJSJEydSUFBA586d2bFjx7+nBpogwP6P+d3GGp1EQkuXloQ4hfDzoUsAtPSxBwt7Y3M7SwWv9g5i/enrDAp9cN6wupJ4PouTm65i0FxB0GejKoLdyz6ix/iJtOjTH9eGtvi3dKEkX03MsXSunMrEzgWOrf+S8rJSPIOb0Gfiyw+9npkuS2X8Wx2fj2AQHlph2cJtSejz1OgB7Y2SSsaD5noxAHJ7uPHa6xTv3m1U05Ta2OD50Vykd8SqW4Q6Y37OAXVcPvm/x2Pd2fRH8nbUUaJxYdbwb8qjfARYhrkaVR/NG9k/MLn5O7+r6jxztr0bkv9HPJat7r+MSD311PP30KNHD5YsWUJpaSlffPEFcrmc4cOHA6KXp6ysjD43S5NUoNFoCAsLA2Dy5MkMHz6cc+fO0bdvX4YOHUqnTp0AkMlkrFq1irlz57Jv3z5OnjzJvHnz+OSTTzh16hQeHh735PWbO3cuISEh7Nq1q8bwxYiICDp37sz777/P2rVrK62/Xd26efPmhIaGEhAQwIEDB+jVq1eVfapUqkrvVCqViueff57w8HDWrVuHXq9n4cKFDBw4kNOnT5uEk/36668UFxdz4cIF3njjDRYuXMibb75Z11NQJRs3bqS4uLiSLkHnzp05ffq08f8rV66wevVqzp8/T9euXXnllVfo378/zZo1o2vXroSGhlbqu3///sZwzIYNGxIVFVXlGMzNzfH19WX9+vX0798fPz8/jh07RpcuXWjcuOb8+hYtWtCrVy+aN29OREQEffv25YknnsDBwYGsrCxu3LhR7fcCorf3UeHu7o5Wq+XEiRN88803+Pr6cvz4cVatWnVXg6ume6YqapqIuJPLly8zZMgQZs2aRd++fSutt7Cw+FeKItXZQDt58iSff/55peXdu3fn3XfffSCDup1BgwYxaNCgatdLJBLmzJnzQGdkHilX96O9foLfK7xnjcWH58VU0esU6m1faZNJ3QKY1K3yzNijIjulmD2rogFQmkWjLQNbF1eKsrPYu3IxmVfj6fXcZORKJdYO5rQd6EerCG82fDSTgox0bF1cGfLaO8gfQQ6ENvvWTWko0aJNL0XpZV3DFveGOrGAsrOZt/6Py6/WQCs9spXyS+JEhnnTptiPHIntwIHIrCt7ZSQSCfZDAsn84iya5CLyku+e2K9s+GC8Sv9EzIMckFrKMZTpMG/04DzIMgczpFZyDKU6436q27/H2+0e2H7rqee/goXcgpNjHn0YkYW87nklVlZWxjCx77//nhYtWrBy5Uqef/55SkrEMPStW7fi5eVlsl1F7nr//v1JTk5m27Zt7N69m169ejFlyhQWLlxobOvl5cXTTz/N008/zYcffkhQUBBLly5l9uzZBAUFERsbW6cxBwQEMGHCBN5++21WrlxZY9v58+fTsWNH3njjjbv26+/vj7OzMwkJCdUaAs7OzuTnm+aZr127lmvXrnH8+HGkUqlxmYODA5s2bTIxBCty1Jo0aYJer2fixIm89tprDyTMcMWKFQwaNOiutXdffPFFPvvsMwwGA+fPn2fEiBFYWlrSrVs3Dh48WKWBtmLFClQqcZK3ppzNiny4Cm/spk2b0Gg0CIKAtbU1Xbp0YXsVasogGvS7d+/m2LFj7Nq1i2+++YZ3332XkydP1smD9LCZN28e8+bNE0OZy8qMkxWlpaVEREQgkUjYvn17tWIctbln7oXo6Gh69erFxIkTee+996psk5eXh4uLy33t5++gzgZaeXm5sTbR7Wi1WuOFXE8tEQQ4MJ/9lhZkyWU4mjvSp2Ef1Fo9Menii3gLH7u7dPJoKS0oZ+vii+g0Blx9VKRcTEEmlzNm7mdEH9rH4bU/cnn/brKuXcWnyS03fW5qCtejL6Ewt2DomzOxtLN/JOPVZZnOmqiv5N+zgaaOz8dQpsMi1NnE8ydoDRRsTABAZmeGvrAcdXw+tr0amGyvSRUNNF12AsrAADznf4JFs6Z33a/c0RyHEUGUHE8Xr5ma2jqYY9G4euXGfzsSmRSHYY1Qx+Vj0fLBPXAlEglKbxvUcflIrRQoPB+8EV9PPf9lJBJJnUMN/wlIpVLeeecdpk+fzpgxY2jSpAlmZmakpKTUmCvl4uLCuHHjGDduHF26dDF6hqrCwcEBDw8PowT8mDFjGDVqFJs2baqUhyYIAkVFRZXy0ABmzpxJQEAAv/zyS43H1K5dO4YNG2YS9lcdqamp5Obmmgit3UlYWBg///yzybKysjKkUqnJb2HF/zWF3BkMBrRaLQaD4b4NtKSkJPbv38/mzZtrbLdy5UocHR157LHHjIZmRR1ArVZroq55O3ca6NURGRmJTqejZcuW7NmzB3d3d7p06cLixYtp3rz5XcUpJBIJ4eHhhIeHM3PmTBo2bMjGjRuZPn06vr6+7N27lx49Hm46yN2YNGkSI0eOZPHixaSmpjJv3jx+++03du3axXfffQfc/XzV5Z6pDVFRUfTs2ZNx48bx0UcfVdkmMTERtVptNCj/TdTZQGvXrh3Lly/nm2++MVm+dOnSSkmk9dyFlONw/SS/eIhqSk8EPYFSpuR8Wj46g4CTlRIv+3+O6oxWo2fbkouUFpTj4G6JpW0kAEEdOmNl70Dbx4bj4uvP1q8WkJWUSFZSomkHEgmDXnkTlwa+j27M2Tdnvzys0KaXor6Sh20Pnzr3oytQk7PqMhjA4qITDiODkJqJt0/R/hR0OSqkNkqcxjUh6+vzaFKKMKh1xjwmQ7neaCwa8q/hOOrZWhlnFViGumAZ+u+bAXoYWDRzxqLZg59ZVPraoY7LxzzY4aGFwdZTTz3/PEaMGMEbb7zBokWLeP3113n99deZNm0aBoOBzp07U1hYyNGjR7G1tWXcuHHMnDmT1q1b07RpU8rLy9myZQshIaIw07Jly4iMjOTxxx8nICAAtVrN6tWriYqKMr43jRw5ko0bNzJ69Gjee+89+vbti4uLC5cuXeKLL77g5ZdfrjKP3s3NjenTp/Ppp5/e9Zg++ugjmjZtapIPVlJSwuzZsxk+fDju7u4kJiby5ptvEhgYSERERLV9RUREMGPGDPLz83FwEKML+vTpwxtvvMGUKVN4+eWXMRgMzJ8/H7lcbjQm1qxZg0KhoHnz5piZmXHmzBlmzJjBk08+afRIVaSx3O5RTEhIoKSkhIyMDFQqlbEOWpMmTUzEXL7/P/bOOyyq4/vD7wILS0eaYAMsVBV7V2IDjRqNxhoRo4nRqLFHTWI3akyMJsYajcafJmqK0VhiLyh2wAJWpFhQRJDOsmV+f6xs3FAExZbvfZ9nn2Tnzsw9c9l177nnzOf8+COurq507NixSNuTkpKYPXs2x44dA3TOso+PD4sWLSIwBvj+yQAA3jpJREFUMJD9+/c/c/ZX9erVOXHiBOXLl6dFixYkJCSQkZFBly5dCuzH+zcnT55k//79BAYG4uzszMmTJ7l//77+8zR9+nSGDh2Ks7MzHTt2JCMjg2PHjjFy5EhAVzbq9u3brFu3rshz3L17l7t373L9uu5B8oULF7C2tqZKlSoF9CSKwt7eHnt7e6Kjo+nduzfVq1fn2rVrBAYGGoiWFEVx35nCaNu2LW+//XaR+wMvXrxImzZtCAoKYuzYsfrax8bGxgbRstDQUKpWrVrofsxXnVI7aLNnz6Zdu3acO3dOHw7fv38/p0+fZs+ePWVu4H8ZceoHrsvlnFaYYiwzpqdnTwDO3XwIgH9lu+e+R6ukCK1g/9pLJMVnoLCU0zbEg41TdDLCdYI66fu5165L8LxvuXhoH2pVnsEc7rXrUaVmwTSC54n6UYqjVfOKpP52lbz4DAPHqaRknbwLjx4K5kQ9QLXkHA7BPiAg4/AtAOzeqoppBStMHM1RJ+egvP5Q70iobmeCAG1uKkKZhmXLFmW3SIkywbpFBYxMjTCv8/yEZCQkJF49TExMGDFiBPPnz2fYsGHMmjULJycn5s6dy40bN7Czs6NevXp8+umngK6+0uTJk4mLi8Pc3JyWLVvqo1qNGjXi6NGjDB06lDt37mBlZYWfnx9//vmnPiInk8n4+eefWblyJT/++CNffPEFJiYm1KhRgwEDBhTrLI0fP55ly5aRm5tb7Jo8PT0ZNGiQXlQNdDev58+f56effuLhw4dUqFCBwMBAZs2aVWwttFq1alGvXj02b97Mhx9+CIC3tzd//fUXM2bMoGnTphgZGVG3bl3+/vtvfTTOxMSEL7/8kqtXryKEwM3NjREjRjBmzBj93GlpaQXUut9//329Yiagj37kF+YGXSRu7dq1DBw4sNhI3KhRoxg3bpyByMnatWsJCQnhu+++Y8KECQaiG0/LoUOHaNWqFaArVN60adMnOmcANjY2HDlyhEWLFpGeno6bmxsLFizQO50hISHk5uaycOFCxo8fj6Ojo77cA+hERhISEoo9R35qbT75dq5Zs4aBAwcCum1K7u7urF27tsh51Go1x44dY8mSJfp1Dh48+IlrhOK/M4URExNDcnJykcd/++037t+/z/r16w2iu25ubsTFxenf//LLL3zwwQclsvFVQyaeYrdqZGQkX331FZGRkZibm1O7dm0mT578xCKKryr56QRpaWkvrlB1xl3EN358YW/NJhtrqlk04c+eujDx2E2R/BFxmzHtPBnV7tW4pif/usGZHXEYGcvoOroOty8d4MiGNTi7V6P/vEWvjCP5ONo8DXemhYEA188bc3/ZOdQPcnEI9sXcr+RpgEKtJXHuKbRZKqzfqEzW2XtoM/KQmRljbGeG+l42Cm97HEJ8kclkPNwWQ2bYHSwbuVCuu+7vl3HkFmk7Y1HdCUd1fTM1joYiM5Jk2iUkJF4OL+V3T+K1ZceOHUyYMIGLFy/q95xJ/Hdwc3NjxowZeoftv0B+CuTVq1cLTRl+1XmqQtV16tRhw4YNZW3L/xZnfyIbDVutdIIOFy/XZF/0Pdr5lify1kMAapfh/rMHdzLZ/v05agVUol6Q25MHPMbV03c5syMOgIB+XrhUs2HndzqRizodOr2Szhmgq2clwMjCBGMrU8w8y6E+nkju1ZRSOWg5F5J1cv3abO4vHIpNp7cR1vVQ3clFfS8bmakRdt2q6a+DmWc5MsPukHs1FSEEMplMLxCiTY3DskVzyTmTkJCQkHht6NSpE9euXeP27dt60Q+J/wZRUVHY2toyYMCAl21KmZKYmMi6deteS+cMnsJB27lzJ8bGxgVC8Lt370ar1RabCyzxCI0Kzq5hn6UFuUaAygl1VjVGbYxgzXuNuHFft5HYvxAFx6LITM1FYSnHxLTwUH/43/Fkpig5szOOmq0qYmpe+J8+5U4Webn/iMA8vJfKgXUXAQvqtK+Cb/MK3Ag/TVrSPRSWVng3a1ViG180+emNJo9qhyk8y5F1PNHAccpHqLRo8zSFFmzOPH4HgLwre1HFx/Ng6SIwlmMV+DEyc09sO1fFxO4fCWKzqrZgIkPzUIn6fg5yZwu9QIgmNRarFoOe15IlJCQkJCSeC6NHj37ZJkg8B/z8/Dh//vzLNqPMadeu3cs24ZkotYM2adKkQiuiCyGYNGmS5KCVhMvbISORP110OdFtK3UkSTgSFvOAAT/qZIor25tjb2la3Cx6bkan8Nf356jkZUeXj+sUiGhlp+dxPTwJAJVSw+UTd6ndulJBs04ksn/tJf17IVTkpa9DaNOwsKuGvUsPNGo3IndvB8CvdXvkZq9uvTnVoxpoJk46oRWzqnZgLEOTqkSdnKMv+qxJzyNpaSTaLBWO79cyqCOWdzuTvIQMkAlUcaGYenhg4uRE9qlTZO5aADIjjM16Ydlwqv66G5kaY+Zui/L6Q3KvpmJkYYImVYkQWjQPE7Bs3vzFXggJCQkJCQkJCYnXhlLnWV27dg1fX98C7d7e3nqFGIkncGoVScbGnHkkVNHbtytL362Hh6MluSqdEkVJo2eqPA2Hfr6M0ApuXkrl1pXUAn2ij91BqxYYmegciIuHbxUolKlRazm1LRYAS1tTbBwVyE0uIbS6emzZD2PY+d18VgwbSOy5cJDJqNP+zada/osiP4Imd9Y5YkZmxpi565wv5VXddRIqDcnrotA8VCJUWh6si0ad8s/m6/zoGepbCGU6Np074bbuJ6ru3In9wIGA4OEvG0ldb5jym19HK/dqKnm3dHV1tJn3UHh6YOLw35XBl5CQkJCQkJCQeDZK7aDZ2tpy48aNAu3Xr1/H0rJgoV2Jf3EvGuKPstPKCmQgct1pXKkGdhamrA5pgM0jp61OZbsSTXdmRyzpyf84FKf/ijVwvrQaLVFHbgPQvEcN5GbGpN7N5va/HLnLxxPJSMnFwtaU/rOa0mdqfYT6LADNer1Lkx59sCpnT056GgiBh3897FyKrpvyKqC+bxhBA0PHSQhByq9XUd3KxMjCBLmLBdosFck/RaFVqtFmq8iOvK/rf3EHAJaNdEWKzap6UH7SRJzHjwfg3ty5ZIaGFjiP8kYaylidk6tNjcWyhaTeKCEhISEhISEhUTSldtC6du3K6NGjiYn5p8bV9evXGTduHG+99VaZGvef5PQqAP6000l5uxo1x+hRzaWqTlZseL8JH7T0oHfDJ2/CTb6VScTem4BOvMNYbkRiTBq3Lv/jfMWdf0BmqhKFlRzfFq54NdHVXLtw6La+j0at5cyuOADqBblhYmrM+b1/k/UwFRsnZxp1fYfmvfrzwZI1dJ0whXpvdqXt4I+e/Vo8R4RW/FMDzfmf4qlmnrqaH8obaaTviSfnfDIYybB/1weH92piZC1HfS+blF+ukHX6Lqi1GDvIUcWGIzMzQ+Hvb3Ae+0HvYdu9O2i13B4zFuWjKLJJeQuMbUxBrSXrlK4+hyY1HitJXl9CQkJCQkJCQqIYSu2gzZ8/H0tLS7y9vfHw8MDDwwMfHx8cHByeqSL4/wS56XB+E9flcmKMlAhhTEPnNwy61Kpky2edfLFWFBSreBytVnBogy61sWpdJ2q2qohfS92etlOPRdEuPKrR5du8AiZyY2oG6Cq9x567T8ajVL5LYYlkpiixsDXFr0UFVHlKTm/7DYDGb/fC2ERni5GxMdUbNKZ1yAfYOpcvm2vynNA8VIJaC8YyjMv9s09O7mKBkbUpQqUl46DOuS33dnUU1ewwsTXDcYAfmBiRezmFtN1xABib6/bvmdeti5Gp4b5AmUyG6/RpWDRogDYzk5vDPkKdmopMJsPsURRN5OhEV0TuXczr1HnOK5eQkJCQkJCQkHideaoUx7CwMHbs2MFHH33EuHHj2L9/PwcOHMDOzu45mPgfInID5GWyw0kXHVNnelGvUkGxjpIQdeQ292LTkSuMadnLE9BFv4zlRty9kcatS6mk3s3i1uVUZDLwa6Vz3hwqWFHR0w4hICr0NhqVlrOPomf1O+RHz3Y9ip6Vp1JmNZLXRSNUmmdf/wtEla/g6GiOzOgf0RSZTKZPPwSwalERy4Yu+vemla2x76m7nmhBpjAmL+YIABaNCi9mKTM1peLi75BXrozq5k1uDfsIdUqKwXmEVo3CrxIyefGOt4SEhISEhISExP82T1WMSSaTERgYyIQJExgxYoS+KrlEMWTeh0Pz0AI7LHV7otRpdfB1sUar0ZZuqlQlx//UpZg26VoNq3JmAFjamlGzpS5Cdmr7DS4c1qUxutVyxMbhn31YNQN0TmH00TtcPHKbzFQllram+LaogEqZy6mtuuhZkzd7kX38LrnRD0jff/Pp1/4MiFJem3zUSQXTG/Ox8HcCQOHrgO2bHoUet2mvqxVn1bQC2adOAP/sPysMk3LlqLxsKUbW1uRERhL7zjsI9V145Btq025h1aLZU61FQkJCQkJCQkLif4enKlSdlZXF4cOHSUhIIC8vz+DYxx9/XCaG/efYPRlyHxJewZdEdQZCY4Z5th8nv7vAJQcFb4+vr9+LVhRZD5VcCksk6uhtVLkaynvY6FMW86kbVIWLobe5eyOdpDhd7a1abxj28ajjiKWtKVlpeYT9rtszVa+DOyZyY85s30Z22kNsncvj4erPQ64BkHHkFhZ1nJC7vBghGE2WipRNV8iLS9NJ31exefKgx/inBpp5gWMKz3K4Tm6EkbWpQXTtcWzaVsGiQXnUSQloHjxAplCgqF272HOaVa+O+88buDViJHnx8dx8Lxib7l+jzTFDkxqLpVT/TEJCQkLiNeTBgwf4+Phw6tQp3N3dX7Y5Ev8hDh06ROvWrUlNTS3TTLzly5ezY8cO/vrrrzKb80VS6ghaREQE1atXp2/fvowYMYLZs2czevRoPv30UxYtWvQcTPwPcH0fXPgVZEZsr1ofAFVGLVoobMlOy+PujXSun7lX5PDbV1LZuew8P30axsltN8hMUWJuLad1f+8CTp2lrRk1W+kcMq1WYFfegsre9gZ9jI2N8Husj6WdGb4tXFEpczm97XcAGnfvjeqRg4eRDLSC1D+uIbSG8vwlRXUvy0C+vjjybmeStDgC5dVURN4/e8VKdb58iX2nghE0AGNbsyKds3xMbM3IPn0aAPO6dQrsPysMsxo1cP91M1YBAQilkqxDP6LNug+qWEyfMp1VQkJCQqJsGDhwIDKZDJlMhlwux8PDg08++YTc3JL9PpWEw4cP06ZNG+zt7bGwsKBGjRqEhIQYPNAWQrBy5UoaN26MlZUVdnZ2NGjQgEWLFpGdrfv9mj59OjKZjKFDhxrMHxkZiUwmIy4uDoC4uDhkMhnOzs5kZGQY9K1Tpw7Tp08v1M6hQ4cik8lKdO/2xRdf0LVrVwPn7PTp07Rt2xY7OzvKlStHUFAQ586dMxh3/vx5WrZsiUKhoHLlysyfP7/Y85w7d46+fftSuXJlzM3N8fHx4dtvvy3Qb8mSJfj4+GBubo6Xlxfr1q0zOL537148PT2xsbEhODjY4NqnpaXh6elJfHz8E9ddWk6fPk2FCrotJXfu3MHc3LxAIONlIYSgY8eOyGQy/vzzz2eay8/Pjz179gAQGBhY4Po/b3Jzcxk4cCC1atXCxMSEbt26FegzaNAgwsPDCX1MYft1otQO2pgxY+jSpQupqamYm5tz4sQJ4uPjqV+/viQSUhh52bB9LADKhoPZkxQOgDqtLtXyjPXdTu+IQ1uI85MQ9YA/F0YQey4ZoRW4VrOl7UAfgr9ohkNFq0JPWTewCiZy3Z+2ZquKhToivi0qYGSsa6/fwQ0TuTFHNqzVRc/Ku+Dbsg3KGzp5eLsuVZGZGZOXkKFXJCwNuVdTufdtOEnLIhHq4lMWs8LvkbTsHJqHSowfpW7mXk4psXOXT36Ko0khKY6lIfuUzkErLr3x3xjb2FBp2VIcP/oIdWIEWXs/w6J+9WeyQ0JCQkKibOjQoQOJiYncuHGDhQsXsmLFCqZNm1Ymc0dHR9OhQwcaNGjAkSNHuHDhAosXL8bU1BSN5p+93MHBwYwePZquXbty8OBBIiMjmTJlClu3btXf+AIoFApWr17NtWvXnnjujIyMEt+HbdmyhRMnTuidieLIzs5m9erVDB48WN+WmZlJhw4dqFKlCidPnuTo0aNYW1sTFBSESqUCID09ncDAQNzc3Dh79ixfffUV06dPZ+XKlUWe6+zZszg7O7N+/XqioqL47LPPmDx5Mt9//72+z7Jly5g8eTLTp08nKiqKGTNmMHz4cH2kRKvV0q9fP4YOHcrx48c5c+aMwTknTZrE0KFDcXNzK9G1Kg3Hjx+nefPmAISGhtKgQQNMS/Bw90WwaNEiZLLiH0yXhIcPH3L16lWaNGmCRqMxWPOLQqPRYG5uzscff0y7du0K7WNqakq/fv347rvvXqhtZUWpHbTIyEjGjRuHkZERxsbGKJVK/VORTz/99HnY+HpzZD48jAebioR6BpChykAuyiHL8sA8RfePmLHciIf3srl22jCKpsrTcPiXKwB4+DvSZ2ojuk+oj3cTV+SmxgVOlY+lrRmtg73xbVEB35aF/+NraWtGQF8var1RCd/mFYjcvYPI3dsBaB3yAWRrdXXEZLo9WbaBun/I0v6ORZNe8qdBqqRsHvx8CbSgzVDpnb5/I4Tg4fYbpG6+CmotCm97yn9cD7PqdiAg62Riic+pyVKhzdJdWxPHgimOJUUIQfapUwBYNG5cqrEyIyOcPh5JpWVLse7QAfuBIU9th4SEhIRE2WFmZoaLiwuVK1emW7dutGvXjr179+qPa7Va5s6di4eHB+bm5vj7+/Pbb7/pj6empvLuu+/i5OSEubk5NWrUYM2aNQDs2bMHFxcX5s+fT82aNalWrRodOnTghx9+wNxc93u0efNmNmzYwC+//MKnn35Kw4YNcXd3p2vXrhw4cIDWrVvrz+Xl5UXr1q357LPPnriukSNH8s0335CUlFRsv9u3bzNy5Eg2bNiAvATCVTt37sTMzIwmTZro2y5fvkxKSgozZ87Ey8sLPz8/pk2bxr179/SRqQ0bNpCXl8ePP/6In58fffr04eOPP+abb74p8lyDBg3i22+/JSAggKpVq9K/f3/ee+89/vjjD32f//u//+PDDz+kd+/eVK1alT59+jBkyBC+/PJLAJKTk0lOTuajjz7Cz8+Pt956i0uXLgEQFhbG6dOnGTVq1BPX/TSEhYXpnZWjR4+WyHH57bffqFWrFubm5jg4ONCuXTuysrL0x/Ovn5mZGa6urowYMaLUdkVGRrJgwQJ+/PHHUo/9NydOnMDPzw8bGxsiIyOxtLSkWrVqxY6Jj4+nS5culCtXDktLS/z8/Ni5c+dT22BpacmyZcv44IMPcHFxKbJfly5d2LZtGzk5OU99rpdFqR00uVyOkZFumLOzMwkJCYBO3fHmzZcjJPHKci8KwhYDkNp+OgvOLQVAk1EHD5UJaATWDgoavOkOwJmdcQaCIflFqK3KmdHuPV8cKhQeMSsMz0YutO7vXawj59uiAq36eHLz0jkOrF0BQIu+IVSr3xhl7EMA5K6WGFnIsWxaAXklK0Suhod/xRQ55+NoHhV9Frka/SctJ/pBoX2VN9LIPKoTNbFuUxmHAb4YmZtg1VRXDDvr9F2EqmSCIepk3RfR2NYMI7Oi1/8k8q5fR5OSgkyhwLxmzaeaw7p1ayotWiilN0pISPynEUKgzc5+4a/8kjJPy8WLFwkLCzOIcsydO5d169axfPlyoqKiGDNmDP379+fw4cMATJkyhejoaHbt2sWlS5dYtmwZjo6OALi4uJCYmMiRI0eKPOeGDRvw8vKia9euBY7JZDJsbW0N2ubNm8fvv//OmTNnil1L3759qV69OjNnziyyj1arJTg4mAkTJuDn51fsfPmEhoZSv359gzYvLy8cHBxYvXo1eXl55OTksHr1anx8fPRpkMePH6dVq1YG1zYoKIgrV66QmppKSUlLS8Pe/p+tGkqlEoVCYdDH3NycU6dOoVKpcHJywtXVlT179pCdnU1oaCi1a9dGpVIxbNgwVqxYgbHx098b/JujR49iZ2eHnZ0dv/32G5999hl2dnYsX76c7777Djs7O+bNm1fo2MTERPr27cugQYO4dOkShw4donv37vrP9bJlyxg+fDhDhgzhwoULbNu2jerV/8nIGThwIG+88Uax9mVnZ9OvXz+WLFlSrDPzJGrXro2dnR3du3cnKioKOzs7WrVqRXJyMnZ2dtQuZp/+8OHDUSqV+qjyl19+iZVV0fe0MpmMtWvXPrWt+TRo0AC1Ws3Jkyefea4XTalFQurWrcvp06epUaMGAQEBTJ06leTkZP7v//6Pmk95E/ufRAj4azRo1ai83mTM7Z3cyryFi0UFrl9tQXO17tJX9XeidutKnNt3Ux9F82rialCEulUfT0wVT6Xnoic7PY39Py7H1smZWm0CKeeq24P24PZNti+ch9Bq8W3VhkZd3wHQR7rMqtoBIDOSUa57DZK+jyDnQjI5l1Mw/9feNoPlq7U8WH8JzYNcjMuZYdPOjdRfr5IT/QC7t6oVSLvMjtA98bNoUB7bQHd9u8LbAWNbMzRpSrLP38eyvmH9tazweyivPcT2TQ+MrXU/AuqkRwIhzk8fPQPIyo+e1auL7BVJUZCQkJB4FRE5OVypV//JHcsYr/CzyCxKl8q+fft2rKysUKvVKJVKjIyM9Cl0SqWSOXPmsG/fPpo2bQpA1apVOXr0KCtWrCAgIICEhATq1q1LgwYNAAz2ZfXs2ZPdu3cTEBCAi4sLTZo0oW3btgwYMAAbG53Y1bVr1/Dy8iqxvfXq1aNXr15MnDiR/fv3F9lPJpMxb948unTpwpgxYwqNanz55ZeYmJiUStAtPj6+QCqktbU1hw4dolu3bsyaNQuAGjVqsHv3bkxMdPcrd+/excPDUCm5fPny+mPlypXjSYSFhbFp0yZ27NihbwsKCmLVqlV069aNevXqcfbsWVatWoVKpSI5ORlXV1c2b97MmDFjGDVqFG+++SaDBg1i3rx5tG7dGoVCQfPmzUlOTmbkyJFPFZF6nAYNGhAZGcnly5fp168fZ8+eJSUlhWbNmhEeHo5CoShS/CIxMRG1Wk337t31KZe1atXSH589ezbjxo0ziPg1bPhPyR9XV1e02uIfXo8ZM4ZmzZoV+kCgNOzcuRO1Wk3nzp0ZPXo07dq1Y9CgQXTo0IFevXrp/+6FkZCQQI8ePfRrq1q1arHn8vLyKvCg4mmwsLDA1tb2uew3fN6UOoI2Z84cXF11UY0vvviCcuXKMWzYMO7fv19sXvH/HPei4NYphImCWa4VOXvvLFZyKwZWm4VMbUkNte7pTdW6jpgqTKjTXlcb7fTOODRqrUERao9HsvDPQsSubVw9Hsrpbb/z4+gP+XXWp0QfOcCf82eizM6igpcv7YeM1OcnK2PyHbR/viCmFaywaqFz7B7+eR1tXuG10YQQPNwaQ15sGjIzYxxD/LDwd0JmZow2PY+8W4abmIVKQ86FZAAs6xk6YDJjGZZNdE98Mk8YpjnmXEwmdfNVsiOSeLAuWh9hU91/JLFfhEBISck++chBa1S69EYJCQkJiVeX1q1bExkZycmTJwkJCeG9996jR48eAFy/fp3s7Gzat2+PlZWV/rVu3TpiYnTZI8OGDWPjxo3UqVOHTz75hLCwMP3cxsbGrFmzhlu3bjF//nwqVqzInDlz8PPzIzFR9xv2NFG/2bNnExoaarA/rTCCgoJo0aIFU6ZMKXDs7NmzfPvtt6xdu7ZUe5FycnIKRKxycnIYPHgwzZs358SJExw7doyaNWvSqVOnMksnu3jxIl27dmXatGkEBgbq26dMmULHjh1p0qQJcrmcrl27EhKi20aQn+HVokULTp8+TWxsLEuWLCE2NpZ169Yxe/ZsgoODGTJkCKGhocycOZPz588Xev6OHTvq//7FRRsVCgXu7u6cP3+ejh074uHhweXLl2nZsiXe3t64u7sX6aD5+/vTtm1batWqRc+ePfnhhx/00cWkpCTu3LlD27Ztizx3frS3KLZt28aBAwfKRMSvUqVKKBQKYmJi6NOnD66urpw+fZq+ffvi7u5OpWIyhT7++GNmz55N8+bNmTZtWpHXPJ/Lly/z9ttvP7PNoIuu5gvvvE6UOiyT/8QIdCmOf//9d5ka9J8hRveUa52bH1vid2MkM+KrgK84e6kcldQPMNWCwkqOSzU7AGq9UYnIvTdJS8ph+/fnChShfhaEEFw6eggAJzcP7ifEkXDxPAkXdV8QGydnuo77FJNHueiadKUuTVAGZu6G8vY27dzIOZ+M5qGS9L3x2HUq+BQk8+gdsk7raoDZ9/XWS/MrvMqRcz6Z3OgHBrL5OZdSEEoNxnZmmLoXlNO3bOhC+r4EVDczyLuZgWlla/JuZ5KySbc/Dxnk3cwg5ber2PfxKpMImtBq9QqOFqUQCJGQkJD4X0Rmbo5X+NmXct7SYmlpqU8T+/HHH/H399eLYGRmZgKwY8cOKlY0LFFjZqYTrurYsSPx8fHs3LmTvXv30rZtW4YPH24g0FGxYkWCg4MJDg5m1qxZeHp6snz5cmbMmIGnpyeXL18ulc3VqlXjgw8+YNKkSaxevbrYvvPmzaNp06ZMmDDBoD00NJSkpCSqVKmib9NoNIwbN45FixbpVSH/jaOjY4GUxJ9//pm4uDiOHz+ud4p+/vlnypUrx9atW+nTpw8uLi7cu2e4tz7//ZNS7aKjo2nbti1Dhgzh888/Nzhmbm7Ojz/+yIoVK7h37x6urq6sXLkSa2trnJwKf6D94YcfsmDBArRaLREREfTs2RMLCwsCAgI4fPhwoel5q1at0jubxe3Vy0/Vy4/Gbt26lby8PIQQWFlZ0bJlS3bt2lXoWGNjY/bu3UtYWBh79uxh8eLFfPbZZ5w8eVKfNvssHDhwgJiYmAIOYo8ePWjZsiWHDh0q0TxDhw5l/fr1aLValEolLi4uun362dn4+PgAur/Z45+tx3n//fcJCgpix44d7Nmzh7lz57JgwQJGjhz5LMsrESkpKUV+Ll5lnqpQtUQJuL6PQ+bmLNDeB+CThp/QomILou6kUV2li5551HbUy+SbKkyoG6j7YN+6rPuHsGm3f4pQPwt3rl4mLekecoU5fWd9xQffr6bpO32xsnfA3NqGbp9MxcLWTt8/P71RXsEKIwvDf5SMTI2xe1v3w5Z57DZ5tzMNjudcTiFt5w0AbN+sapAGae7noOsTZbgPTZ/eWMe5UMVJYytTLGrrvlyZx++gSc/jwboohEqLWQ07HAfVBCMZOefuk3Hg5mM10J4+gqa8fh1Naioyc3PMa0mpuxISEhLFIZPJMLKweOGvZ1WlMzIy4tNPP+Xzzz8nJycHX19fzMzMSEhIoHr16gavypUr68c5OTkREhLC+vXrWbRoUbEZROXKlcPV1VUv/NCvXz+uXr3K1q1bC/QVQpCWVriY1tSpU7l69SobN24sdk2NGjWie/fuTJo0yaA9ODiY8+fPExkZqX9VqFCBCRMmsHv37iLnq1u3LtHR0QZt2dnZGBkZGVz//Pf5KXdNmzblyJEjelVH0Mnfe3l5FZveGBUVRevWrQkJCeGLL74osp9cLqdSpUoYGxuzceNGOnfurHcWH2f16tXY29vz1ltv6ZU0821SqVQG6pqPU7FiRf3fvjjFx8jISM6cOYOxsTH79+8nMjISBwcHNm/eTGRkJKtWrSpyLOi+O82bN2fGjBlERERgamrKli1bsLa2xt3dvdi01icxadKkAn9zgIULF+qFbUrCzJkziYyM1Kc3RkZGMnDgQPr372/wWSqOypUrM3ToUP744w/GjRvHDz/88NTrKikxMTHk5uZSt27d536uskZy0J4Hykyyb57kUycHBNDTsyf9vPsBEHU7nRoq3WX3qGPo0dcMqIjCSucQlfew0dcqe1YuhR4EoEajpsjNFNg4OtOs57sMWbqWoSv+D6cq7obm5+8/8yg8/9fcyx7z2o6ghdQt/9RGU93NIuXnyyDAspELVi0Mv6wKL3swlqG+n4PqUZRLk6Ui94rOIbWoW/QTDstHYiHZ5++T/FMUmrQ8TJzMcejng6JGOey66XLt0/fG6yX5nyXFMe13XT04i3r1kJVA5UpCQkJC4vWkZ8+eGBsbs2TJEqytrRk/fjxjxozhp59+IiYmhvDwcBYvXsxPP/0E6BylrVu3cv36daKioti+fbs+irBixQqGDRvGnj17iImJISoqiokTJxIVFUWXLl0A6NWrF71796Zv377MmTOHM2fOEB8fz/bt22nXrh0HDx4s1M7y5cszduzYEsmGf/HFFxw4cIArV67o2xwcHKhZs6bBSy6X4+LiUuyeuKCgIKKiogyiaO3btyc1NZXhw4dz6dIloqKieO+99zAxMdGrUPbr1w9TU1MGDx5MVFQUmzZt4ttvv2Xs2LH6ebZs2YK3t7f+/cWLF2ndujWBgYGMHTuWu3fvcvfuXe7fv6/vc/XqVdavX8+1a9c4deoUffr04eLFi8yZM6eA7UlJScyePZvFi3WCbeXKlcPHx4dFixZx/Phx9u/f/8wS8dWrV+fhw4eUL1+eFi1aYGpqSkZGBl26dKF69eoFIrGPc/LkSf1nICEhgT/++IP79+/rP0/Tp09nwYIFfPfdd1y7dk3/Wcxn8uTJDBgwoMj5XVxcCvzNAapUqVJgf2BxODs7U716dc6fP69f16VLl3jzzTf1Tmxxe9BGjx7N7t27iY2NJTw8nIMHD+rXWBje3t5s2bKlWJuio6OJjIwkJSWFtLQ0Awc0n9DQUKpWrfpElclXEclBex7EHSVCLiPD2IjyFuWZ3HgyMpmM1Kw81A9ysRFGmJgaUdnb8AmSqcKEgL5euFa3pW2IT4Ei1EWh1Wi4djKM9OSC0roatYorx3VF+nxatjY4JpPJMCpEyegfgZCiN2jadamGTGGM6lYmmWF30GTmkbw2CpGnwayqrU4I5F9PNo0UJpg9SunMV3PMuXAftAK5qyXy8pZFns+0sjXyilagFqhuZ2JkYYLjQD+MzHX/IFg1csWq+SOHUIDMzBgj66dzrB7++ScpP+lyuu1693qqOSQkJCQkXg9MTEwYMWIE8+fPJysri1mzZjFlyhTmzp2Lj48PHTp0YMeOHfobWlNTUyZPnkzt2rVp1aqVPoIDuuhVZmYmQ4cOxc/Pj4CAAE6cOMGff/5JQEAAoPvt/fnnn/nmm2/07bVr12b69Ol07dqVoKCgIm0dP358sep3+Xh6ejJo0KAyKcBdq1Yt6tWrx+bNm/Vt3t7e/PXXX5w/f56mTZvSsmVL7ty5w99//63XKbC1tWXPnj3ExsZSv359xo0bx9SpUxkyZIh+nrS0NAMn8rfffuP+/fusX78eV1dX/etxYQyNRsOCBQvw9/enffv25ObmEhYWZiDWks+oUaMYN26cQXRn7dq1+ojbhAkTDOZ+Wg4dOkSrVq0AXaHypk2bFuuw5GNjY8ORI0d488038fT05PPPP2fBggV07NgRgJCQEBYtWsTSpUvx8/Ojc+fOBjXxEhMT9Wrqz4K7u3uRBc3zuXv3LrGxsTRp0oS8vDxOnDihX/OT0Gg0DB8+XP998vT0ZOnSpUX2v3LlSpGR5HzefPNN6taty19//cWhQ4eoW7dugUjZL7/8wgcffFAiG181ZOJZNWr/A6Snp2Nra0taWppeZemZ2DmBRdc2sdrOlreqvcUXLXQh+qPXkln2/VmaKeVUq+tEhw9rPWGiJ5OTkc72b+eTcCESawcnBi5Ygqn5P5Gj62dOsvWrWVjalWPIsrUYGRUvLatJU5I49xTIoMLUpnoHqDAyTybycMt1ZKbGmDibo7qViYmDAqeP6mBsWbhzlD/GtLI1zsPrkLTsHHnx6di+6YF1q+Kl6LNO3yX192tgJMPp/Zp6hcl8hEbwYF0UuVdSMa1ijfNHdYqdrzCyw8NJCBmIUKlw+PBDnMeMLvUcEhISEq86Zf67J/GfZseOHUyYMIGLFy8WmkYo8fqSnZ2Ng4MDu3bteqJk/+tEVFQUbdq04erVq2WiCPmieTbtdonCub6f048Ujxq5/CMwEXkzlRr5+8/qlGzDolar4cTvGzE2keMb0AZr+382jd6LjWHbgjmk39dtus14cJ9jmzfoCk0/Ij+90bt5wBOdMwBl7GP7z4pxzkAn3pEdnkRefDqqW5nIFMY4hPgV6ZwBmPs68PDP6+TdzEAZl0ZefLquGHYJrodFvfKoHyoxrWJdwDkDneKjfV9vMo7cQlFMCYCiyLt1m1sjRiJUKqzbt8dpVMlliCUkJCQkJP6rdOrUiWvXrnH79m2DvXgSrz8HDx6kTZs2/ynnDHTRxXXr1r2Wzhk8pYO2f/9+9u/fT1JSUoH6C2VRpTyf6dOnM2PGDIM2Ly8vvfpRbm4u48aNY+PGjSiVSoKCgli6dKm+zsZLISWWrNQbRLnpokENXBoghOCnsDjW7r7Oe1ozkIFbTYcSTXcj/AzHf/sFgGOb1lO1fkNqt+1AbmYGe1d+j1qVh115V+p26MzBn34gYtdf+LZsTfmq1VFmZxFzVlec79/pjUVRmLx+Uehqo1Xn3ncRIAQO7/ogdy5+35extalOhTEhg9Rfr+rOVc0OY5sni6HIjGXYti96oy7o0igfr6NWUjSZmdwaNgxNSgpmvj5U+HIeMukpoYSEhISEBKDbRyTx36NTp0506tTpZZtR5rRr1+5lm/BMlNpBmzFjBjNnzqRBgwa4uro+s4LSk/Dz82Pfvn3694/n9I4ZM4YdO3bw66+/Ymtry4gRI+jevTvHjh17rjYVS8x+IhRmaGQyKlpVxMHMhXG/nuOPs7dpnauLLFXwtENhKScnM4Md387HvmIl2gz8sNDprp86DoC5jS056WnEnDlJzJl/KqJ71KlPh4GjydyWQAv/3hw9t4k9Kxfz7hffcO1kGBqVCvuKlXF2L74oYD7KGw+BkjloAPLyljgP8weh2ydWEsz9HMhLyED9QJcbb1HXuUTjnhdCCO5MnITy2jVMnJyovHQpRqUsfCohISEhISEhISFRFpTaQVu+fDlr164lODj4edhTABMTk0LrZaSlpbF69Wp+/vln2rRpA8CaNWvw8fHhxIkTNGnS5IXY9zj3UrOxjt7DaYUuGlTDpg7vLA/j8q10uuSY4p2nSzH0beqKRq3mr2/mcjPqPPHnI6jboQvlXAxVD7UaDTHhumLJXUZPxMKuHBf27ybq8H5yMzNo/HZvmvXqR9rWG+TFplPJrCoKS2uSYmOI+Psvbjwa69uydYkcaXWaUuc0yYpWcCwM00olc8zyUfg5krYrDgCZ3Egvv/+ySN+xk8z9+5HJ5VRaugT5E+qzSEhISEhISEhISDwvSu2g5eXl0axZs+dhS6Fcu3aNChUqoFAoaNq0KXPnzqVKlSqcPXsWlUplEML09vamSpUqHD9+vFgHTalUolQq9e/T09MBXT2Mx+t1lIaff71M5pH72Jt04KaLDaZW59h1xgLLlAwG5ChwUMuQGclo2t0D97r27Fu1lJtR/1RSjzpygMZv9zaY81b0RXIz0lFYWeFczRMjY2Oa9w2h8Tv9yM3MwKqcPXkPcsg6o9uDJpRaAtqHsPvP7zm66f9Q5+UBUL1x8xKtKyviLgAmFSzRGAs0T3ktnoitCcZO5mju52DqVe75nusJaNLSuPdImrfckCGYeHs/9WdAQkJC4nVB+ndOQkJC4tWl1A7a+++/z88//8yUKVOehz0GNG7cmLVr1+Ll5UViYiIzZsygZcuWXLx4kbt372JqalqgOnr58uW5e/dusfPOnTu3wN42gD179mDxlKltt45bYIcxKWp3qt1yp8qdHlyXa6imMsNUK8PIVItD3VwSss5xftEGksN1qYvW7tXJiLtO+J6dJJtaGUS67p/V9ZE7ufJ3EUUkq8RY4KRR6N/n3TVG4VSe3EfCIQonF46ePvNE+63STPC8ZI0MGTfkSZzcufOprkNJKWdrSoUMc6JlN8jZee3JA54Tzr//gV1KCkpnZ05UcIXnvG4JCQmJV4Hs7OyXbYKEhISERBGU2kHLzc1l5cqV7Nu3j9q1ayP/VxHfb775psyMy68DAVC7dm0aN26Mm5sbmzdvxtzc/KnnnTx5skGhxPT0dCpXrkxgYOBTyQ3fjkvn1q5zgAaXcpuIzm2DfY4LPo+CdE5u1rQf7INVOTPizp3lr19WA9Ci30Bqtglk1fD3UGWkU9+7Bi7VPAHdvqi1e/4EoFW3HlSr37jAeTUPlSSfjAQElq0qkHXkDi7Z1vQY9ym/fDYOrUZNsy5vU7NNYLH2qx/kkLLiIkJoUNR2oME7jZ/73sJ8ipf8eL7knD3L7VO6NNCq87/Er379l2iNhISExIsjP3NEQkJCQuLVo9QO2vnz56lTpw6gq/j+OM/7pt7Ozg5PT0+uX79O+/btycvL4+HDhwZRtHv37hW6Z+1xzMzMMDMrqBool8sLOJwl4fjh2wA4mUUT77KXzbYn6WU7EO9LzuRlJ+LgbEPEznAQEHV4H0Joqdm6PY3e6oFMJqN6gyZcPnaYa8ePUtnbT7eO2Bgyku9jYmpGtboNCrUrMzQOtAKz6nbYBXqQffIe2gwVDsZOdBg+hltRF6jZul2xa9Jmq3iw/ioiR4NpZWscenojk//31Qu1eXncnzUbALue72DzEvYsSkhISLwsnua3TkJCQkLixVBqB+3gwYPPw44SkZmZSUxMDMHBwdSvXx+5XM7+/fvp0aMHoKs8npCQQNOmTV+YTUIruH8xBWOgnvkutpjrZPRrKBQknNsEwN1/ZfBV8qlJu/c/0ju0vi1bc/nYYS6HHSEgeDDGJiZcP30CAHf/esjNFPwbdUqufu+ZTbsqyEyMUHiVI+d8MrnRD/DpEIBP84DibddoefDzZdTJORjbmuEwwPd/wjkDeLBqFXkxMRg7OOA8fvzLNkdCQkJCQkJCQkICeMZC1bdu3QKgUqVKZWLMvxk/fjxdunTBzc2NO3fuMG3aNIyNjenbty+2trYMHjyYsWPHYm9vj42NDSNHjqRp06YvRMFx2bllPMh5QLDdhxjnatHKVDgpzhJt6oKxRkba9tOATgbf2aOafpzCyppabQIxNpEjNFpSt1ynnLWDXkY//kIEVes2JOa0bv9Z9YaFryXj4E199MzMXae4aO7nSM75ZHKiHmDbwcOgvzpNServ19Bm/7MxXCg1qO/nIDM1wiHEF2Nr0zK9Ri8DTVoaiZ9/jnW7dth27VpoH2VMDA+WrwCg/OTJGL+mRQwlJCQkJCReBHl5efj6+rJu3boXKhQn8XoTFxeHh4cHERER+uy7kjJlyhTu3bvHypUrn49xRfD3338zadIkwsPDMXpUDzcvLw9PT09+++03GjRo8ELsKHW4RKvVMnPmTGxtbXFzc8PNzQ07OztmzZpVoGj1s3Lr1i369u2Ll5cXvXr1wsHBgRMnTuDk5ATAwoUL6dy5Mz169KBVq1a4uLjwxx9/lKkNhZGck8zSyKVsurKJn7ftAqC6WRjnzI3QyqBlfGUyk5OxdnCi85hJtOgzQP9q0PltzCwsAcg6e4/sM/fIPHiLpl5vA3Ap9BAP793lfkIcMiMjqtZvVOD86pRcss4+ip49VrhZ4VUOjGWo7+egSvpnA7gQgod/XEN5NRXVrUz9S30/B2Rg39sL0wpWz+tyvVDStm4lY+8+7kyaTMahQwWOax4+5OZHHyHy8rBs0QKbTm++eCMlJCQkJF4KAwcORCaTIZPJkMvleHh48Mknn5Cbm1tm5zh8+DBt2rTB3t4eCwsLatSoQUhICHmPlJVB97u8cuVKGjdujJWVFXZ2djRo0IBFixbpBVymT5+OTCZj6NChBvNHRkYik8mIi4sDdDfBMpkMZ2dnMjIyDPrWqVOH6dOn699nZmYyYsQIKlWqhLm5Ob6+vixfvvyJa1q+fDkeHh565+zQoUP66/jv1+nTpw3s+vfrxIkTxZ4rISGBTp06YWFhgbOzMxMmTECtVuuPR0REULduXaysrOjSpQspKSn6Y2q1mvr163Pq0f7ysuT+/fuYmpqSlZWFSqXC0tKShISEMj/P0zJ06FBkMhmLFi16pnk6deqkd4qGDBnCzJkzy8C60nH37l2+/fZbPvvsM33bkSNH6NKlCxUqVEAmk/Hnn3+Wet64uDgGDx6Mh4cH5ubmVKtWjWnTphl8Nzt06IBcLmfDhg36NlNTU8aPH8/EiROfaV2lodQO2meffcb333/PvHnziIiIICIigjlz5rB48eIyV3bcuHEjd+7cQalUcuvWLTZu3Ei1ao9FoxQKlixZQkpKCllZWfzxxx9P3H9WFpy7fw4AY60JZvH2ANQy38dpRzfKpctxu6JLXWw7eCimisLFTIRaS8aBm/r3TsnlqWhRg+unTxB95AAAlX1rYm5lWGNMq1ST+ud1XfSshh1mbv+ImhgpTDCrZgdATtQDfXvOhWRyr6SCsQz7Pl44DPTTv1zGN8Dcz/EZr8irQ2boUd3/CMGdsePIvXpVf0yoVNwaNRpVfALyChWo8OW8FyaGIiEhISHxatChQwcSExO5ceMGCxcuZMWKFUybNq1M5o6OjqZDhw40aNCAI0eOcOHCBRYvXoypqSkajUbfLzg4mNGjR9O1a1cOHjxIZGQkU6ZMYevWrezZs0ffT6FQsHr1aq5de7LacUZGBl9//XWxfcaOHcvff//N+vXruXTpEqNHj2bEiBFs27atyDFCCL7//nsGDx6sb2vWrBmJiYkGr/fffx8PD48CEYZ9+/YZ9KtfjCCXRqOhU6dO5OXlERYWxk8//cTatWuZOnWqvs/7779PmzZtCA8PJy0tjTmPSuUALFiwgObNm9OoUcGH28/K8ePH8ff3x9LSkvDwcOzt7alSpUqZn+dp2LJlCydOnKBChQpP7lwMQghOnDhB8+bNAQgNDdX//4tk1apVNGvWDDe3f4IQWVlZ+Pv7s2TJkqee9/Lly2i1WlasWEFUVBQLFy5k+fLlfPrppwb9Bg4cyHfffWfQ9u6773L06FGioqKe+vylodQO2k8//cSqVasYNmwYtWvXpnbt2nz00Uf88MMPrF279jmY+OoRmRQJQO3sZphpLMg0TeWC4wNO21ei2UUHZAJqNGpWqPJiPlln76F5qMTIWo5lI51T2dT5Layw5dTWXwGo1sBwL50qOYekJedQXtU5W7ZB7gXmzS/6nBOtc9C0OWoe/hUDgPUblbGo44y5t73+ZeLw9GqYrxra3FyyHz25M6tRA212NreGfYT6wQOEENydOYvskycxsrCg0rJlmDi83ALZEhISEhIvHjMzM1xcXKhcuTLdunWjXbt27N27V39cq9Uyd+5c/VN2f39/fvvtN/3x1NRU3n33XZycnDA3N6dGjRqsWbMG0JXrcXFxYf78+dSsWZNq1arRoUMHfvjhB7369ObNm9mwYQO//PILn376KQ0bNsTd3Z2uXbty4MABWrdurT+Xl5cXrVu3NogkFMXIkSP55ptvSEpKKrJPWFgYISEhvPHGG7i7uzNkyBD8/f2LjTidPXuWmJgYOnXqpG8zNTXFxcVF/3JwcGDr1q289957BR58Ojg4GPQtTqBmz549REdHs379eurUqUPHjh2ZNWsWS5Ys0Uc5Ll26xAcffICnpyd9+/bl0qVLANy4cYPVq1fzxRdfPPFaPQ1hYWF6Z+Xo0aMlclwOHTpEo0aNsLS0xM7OjubNmxMfH68//tdff9GwYUMUCgWOjo68/fbbpbbr9u3bjBw5kg0bNjyz+M+VK1cQQuDr60tycjLXr1+nceOi72Wh+O/D07Jx40a6dOli0NaxY0dmz579VNconw4dOrBmzRoCAwOpWrUqb731FuPHjy+QfdelSxfOnDlDTEyMvq1cuXI0b96cjRs3PvX5S0OpHbSUlBS8vb0LtHt7exuEmf/L5Dtore7VA+Ca41mmOluhvXAPp4dmmCgUtH5vSJHjhVqr20MGWAdUxq5rdcxq2GEsM6Fl+R7Itbq9YI/vP8u5nELS9xGok7IxsjbFaUhtTCtZF5jb3NcBZKC6mYEmTUna37FoM1SYOJpj80blsroEryTZZ84icnMxKV+eKut+Qu5WBdXt29wa+TEpq1fz8NdfQSajwoKvUXh5vmxzJSQkJP4zCCFQKTUv/CWEeCa7L168SFhYGKam/+zBnjt3LuvWrWP58uVERUUxZswY+vfvz+HDhwHd3pjo6Gh27drFpUuXWLZsGY6OukwUFxcXEhMTOXLkSJHn3LBhA15eXnQtZJ+0TCbD9l/7oufNm8fvv//OmTPF1zTt27cv1atXLzYlrVmzZmzbto3bt28jhODgwYNcvXqVwMCiy/GEhobi6emJtXXBe458tm3bxoMHD3jvvfcKHHvrrbdwdnamRYsWxUbqQBelqlWrFuXLl9e3BQUFkZ6ero9c+Pv7s3fvXtRqNfv376d27dqALsVv/vz5xdpZWhISErCzs8POzo5vvvmGFStWYGdnx6effsqff/6JnZ0dH330UaFj1Wo13bp1IyAggPPnz3P8+HGGDBmid2B37NjB22+/zZtvvklERAT79+83iPxNnz4dd3f3Yu3TarUEBwczYcIE/Pz8nnqdnTt31qfZpqWlUa5cOTw8PNBoNFSqVKlAzeHHKe77UBju7u4Gabf/JiUlhejo6Be21ystLQ17e3uDtipVqlC+fHlCQ0MN2hs1alSg7XlRapEQf39/vv/++wKhv++//x5/f/8yM+xVJU+TR9SDKMxUFqiSdOIoSTY3MMpWU/eKMwCt+g3E2r7oD+c/0TNTrBq7IDOW4dDPh8TvzmCRakPL8j1INUtGFqUkg1uoH+aSdSIRBJi62eDwrg/GNoULehhbm2Ja2Zq8hAzS/o4jO0L3JM3u7er/eYXGrKO69EbLli0wKVeOysuWEde7Dznh4eSEhwPgPGEC1o89nZSQkJCQeHbUeVpWjjr8ws875NsA5GbGpRqzfft2rKysUKvVKJVKjIyM+P777wFQKpXMmTOHffv26RWhq1atytGjR1mxYgUBAQEkJCRQt25d/Q3k4zfRPXv2ZPfu3QQEBODi4kKTJk1o27YtAwYM0NdZvXbtGl5eXiW2t169evTq1YuJEyeyf//+IvvJZDLmzZtHly5dGDNmjMGWkHwWL17MkCFDqFSpEiYmJhgZGfHDDz/QqlWrIueNj49/Yurc6tWrCQoKMhCNs7Ky0qccGhkZ8fvvv9OtWzf+/PNP3nrrrULnuXv3roFzBujf3717F9Clv3300Ud8/fXXNG/enMmTJ/N///d/WFhY0LBhQ4KCgoiJiaFPnz7Mnj27WLufRIUKFYiMjCQ9PZ0GDRpw8uRJLC0tqVOnDjt27KBKlSpYWRW+hz89PZ20tDQ6d+6s/1v4+Pjoj3/xxRf06dOHGTNm6Nsev492dHQs9G/4OF9++SUmJiZ8/PHHz7JMVq1aRW5uLkOHDqVJkyYMHDiQqVOnYmdnZ1A3uDCK+z4URrVq1Yp14BISEhBCPHO6Zkm4fv06ixcvLjQ1uEKFCgbRzqLanheldtDmz59Pp06dDP7xOn78ODdv3mTnzp1lbuCrRvSDaFRaFQ2S66NFjsY4leqKd2h0cg2mahkaF0v823cscvzje8+s36iETK77YTEyN6H8YH9ufh2GvZkr9riStivWYKxlE1fsOldFZlK8o2Xu50heQobeObOoXx7Fo71p/2Uyj+qeali1aAGAWdWqVFy0kJtDPgSNBtt3emD/3sCXaKGEhISExMumdevWLFu2jKysLBYuXIiJiYm+XM/169fJzs6mffv2BmPy8vKoW7cuAMOGDaNHjx6Eh4cTGBhIt27d9OIZxsbGrFmzhtmzZ3PgwAFOnjzJnDlz+PLLLzl16hSurq5PFfWbPXs2Pj4+7NmzB2dn5yL7BQUF0aJFC6ZMmcLPP/9c4PjixYs5ceIE27Ztw83NjSNHjjB8+HAqVKhAu3btCp0zJycHhaJguZ98bt26xe7du9m8ebNBu6Ojo8HNfcOGDblz5w5fffVVkQ5aSfDz89NHMwEePHjAtGnTOHLkCCNHjqRZs2b88ccfNGzYkMaNGxdIlQNdtG39+vX695mZmYWey8TEBHd3dzZv3kzDhg2pXbs2x44do3z58sU6tQD29vYMHDiQoKAg2rdvT7t27ejVqxeurq6ATuzlgw8+KHL8iBEjGDFiRJHHz549y7fffkt4ePgz76d3cXFBpVJx4sQJFi9ejLu7O8ePH2fNmjVPdLiK+z4URnEPGUD3eQOK/cyVBbdv36ZDhw707Nmz0L+Dubm5XrCnuLbnRakdtICAAK5evcqSJUu4fPkyAN27d+ejjz56Id7uyyY/vdH3vm6T6yXVLWqEHkKoZWBnztujPsXIqOineVln7qFJexQ9a2QoaGLiaI5VHzeS917FqZI7RsaP5pHJUHiVw6K2U4lsVPg56J07I0sTbN/0eMKI1x9VYiJ512PAyAjLx+rgWTVvTuUVK8i9FI1DSIgkCiIhISHxHDAxNWLIt8XX3nxe5y0tlpaWVK9eHYAff/wRf39/Vq9ezeDBg/U36jt27KBixYoG48zMzADdXpj4+Hh27tzJ3r17adu2LcOHDzd4Cl+xYkWCg4MJDg5m1qxZeHp6snz5cmbMmIGnp6f+/qmkVKtWjQ8++IBJkyaxevXqYvvOmzePpk2bMmHCBIP2nJwcPv30U7Zs2aLfT1a7dm0iIyP5+uuvi3TQHB0duXDhQpHnW7NmDQ4ODiVyuho3bmyw3+/fuLi4FNgPd+/ePf2xwhg7diyjR4+mUqVKHDp0iNmzZ2NpaUmnTp04dOhQoQ7azJkzGV+CGqh+fn7Ex8ejUqnQarX6yKtarcbKygo3N7diRSPWrFnDxx9/zN9//82mTZv4/PPP2bt3L02aNNHvSXxaQkNDSUpKMhAq0Wg0jBs3jkWLFulVPp/EnDlzmDNnDkIIsrOz9Q8isrKyCAoKQiaTsWvXLlq2bFno+JJ8H0pDfnQtNTVVr9pe1ty5c4fWrVvTrFmzImX8U1JSCpy/sLbnxVPVQatQocJz24T5qhORFIFCZYV5ZjVUOYeorgxHAB51G/DmiPEoigh1g+HeM5vHomeP41SnOk51qj+TjXJHc+QVrVDdzsS2czWMLZ9t0+jrQOaj9Ebz2rUL1DWzatEcqxYvXoVIQkJC4n8FmUxW6lTDVwEjIyM+/fRTxo4dS79+/fD19cXMzIyEhAQCAop2OJ2cnAgJCSEkJISWLVsyYcKEIm9Iy5Urh6urK1lZWQD069ePPn36sHXr1gL70IQQpKenF9iHBjB16lSqVav2RJGCRo0a0b17dyZNmmTQrlKpUKlU+tpO+RgbGxdbJqlu3bosW7YMIUSBh5xCCNasWcOAAQNKJFARGRmpjyAVRtOmTfniiy9ISkrSRwr37t2LjY0Nvr6+Bfrv37+fS5cu6UUpNBoNKpVKv96icHZ2LjYSmc/OnTtRqVS0bduW+fPnU79+ffr06cPAgQP1cuxPom7dutStW5fJkyfTtGlTfv75Z5o0aULt2rXZv39/ofv2SkJwcHABpzooKIjg4OBSzTl06FB69erF0qVLuXXrFnPmzOHXX39lz549/PDDDwAFHlb8m9J8H55EtWrVsLGxITo6Gk/PstcLuH37Nq1bt6Z+/fqsWbOmwPcBIDc3l5iYGL2zms/FixcLtD0vSuSgnT9/npo1a2JkZMT58+eL7Zu/WfO/iBCCyPuRVE71QpW5Da1aF6Vq0qMPzd7ph6yQP/LjZJ19FD2zMcWyUdH/QJUFDsG+qJNzUFS3e67neVXIOnoMAMtH6Y0SEhISEhIloWfPnkyYMIElS5Ywfvx4xo8fz5gxY9BqtbRo0YK0tDSOHTuGjY0NISEhTJ06lfr16+Pn54dSqWT79u36vUUrVqwgMjKSt99+m2rVqpGbm8u6deuIiopi8eLFAPTq1YstW7bQt29fPv/8cwIDA3FycuLChQssXLiQkSNH0q1btwJ2li9fnrFjx/LVV189cU1ffPEFfn5+mJj8c5tnY2NDQEAAEyZMwNzcHDc3Nw4fPsy6dev45ptvipyrdevWZGZmEhUVRc2aNQ2OHThwgNjYWN5///0C43766SdMTU31N7R//PEHP/74I6tWrdL32bJlC5MnT9ZHFAMDA/H19SU4OJj58+dz9+5dPv/8c4YPH66PYOaTm5vLiBEj+OWXX/Q32c2bN2fJkiUMHz6c33//vdh1lQQ3Nzfu3r3LvXv36Nq1KzKZjKioKHr06FGsowkQGxvLypUreeutt6hQoQJXrlzh2rVrDBgwAIBp06bRtm1bqlWrRp8+fVCr1ezcuVNfa+v7779ny5YtRaYEOjg44PAvNWq5XI6Li0up9jja29tjb29PdHQ0vXv3pnr16ly7do3AwEB9pLk4ivs+FEbbtm15++23i0zfNDIyol27dhw9etTge5CZmcn169f172NjY4mMjCxVuYPbt2/zxhtv4Obmxtdff839+/f1xx6P0J44cQIzMzP9Vq58QkNDmTVrVonO9cyIEiCTycS9e/f0/29kZCRkMlmBl5GRUUmme+VIS0sTgEhLSyu2X3xavKi5tqaYNGqC+LpXJzG/V1cxf8WvJT7P/TUXxc2JR0T6oYRnNVniMbQqlbjcoKGI9vIW2ZGRL9scCQkJiVeekv7u/dcICQkRXbt2LdA+d+5c4eTkJDIzM4VWqxWLFi0SXl5eQi6XCycnJxEUFCQOHz4shBBi1qxZwsfHR5ibmwt7e3vRtWtXcePGDSGEEOHh4aJ///7Cw8NDmJmZCQcHB9GqVSuxbds2g/NpNBqxbNky0bBhQ2FhYSFsbGxE/fr1xbfffiuys7OFEEJMmzZN+Pv7G4xLS0sTjo6OAhCxsbFCCCFiY2MFICIiIgz6DhkyRABi2rRp+rbExEQxcOBAUaFCBaFQKISXl5dYsGCB0Gq1xV63Xr16iUmTJhVo79u3r2jWrFmhY9auXSt8fHz062vUqJH49VfDe6Y1a9aIf9+KxsXFiY4dOwpzc3Ph6Ogoxo0bJ1QqVYH5J02aJMaNG2fQdu3aNdGwYUNhY2Mjhg0bJjQaTbHrKgm//PKLaNGihRBCiCNHjojq1auXaNzdu3dFt27dhKurqzA1NRVubm5i6tSpBjb9/vvvok6dOsLU1FQ4OjqK7t27649NmzZNuLm5lcpWNzc3sXDhQoO2gIAAERISUuw4lUolrKysxPXr14UQQlStWlUcPXq0ROcs7vtQ2GfTzc3N4DNZGDt37hQVK1Y0uFYHDx4UQIHX42t70jXL/7wV9nqcIUOGiA8//NCgLSwsTNjZ2em/n88bmRBP3q0aHx9PlSpVkMlkT1Qvebyo3OtCfjpBWlqaXmWpMLbFbOOzI58zcF9dUD/ginU9Bn86isZVn1xPSwhB4uyTaLNUOH3kj1mVos8jUTqywyOI79cPY1tbaoQdQ2b8+qXZSEhISLxISvq7JyEBukyq9u3bExMTU6RqocSriZubGzNmzGDgwIEv25QSI4SgcePGjBkzhr59+5Z4XMgjrYFnqcucnJyMl5cXZ86cwcPjHw2H3r174+/vX6Co9fOiRCmOjztd8fHxNGvWzCBsDrp6D2FhYa+lg1ZSIu9F4JdQEdQPADnHyvmzuIpdicZqUpVos1RgJMPUVfrHrSzJeqTeaNGsqeScSUhISEhIlDG1a9fmyy+/JDY2llq1ar1scyRKSFRUFLa2tvq0ytcFmUzGypUrixWn+TdCCA4dOsTRR5oET0tcXBxLly41cM7y8vKoVasWY8aMeaa5S0OJImiPY2xsTGJiYoHNlQ8ePMDZ2RmNRlOmBr4ISvok8e1fO1B/uyXmuUoyLBqw16sVxya1KdE5ss/fJ+Xny8grWlF+5IvZYPi/Qmyv3uSeP4/rF19g16P7yzZHQkJC4pVHiqBJSEhIvLqUWp9WFKLiAzoHzdLSskyMehVJz0tHdSMF81wlYMpFG3/cHS1KPD7vVgYAppXLrsL964zq9m0erF6NNjf3meZRp6aS++gJi6Wk1CghISEhISEhIfGaU2KZ/e7ddZEJmUzGwIEDDdR0NBoN58+fL7Yw3evOubuR1LlaDgBjRV2umZrRyaHkDmneTV1tFdNKUnojQNLCRaRv3442Nxen4cOfep6ssDAQAjNPT+Tly5ehhRISEhISEhISEhIvnhI7aPk1OYQQWFtbGxTYMzU1pUmTJsVWRH/dCT/wO+Uy5YApwrYeacYCd4eSRdCEVqC6LUXQHif30iUAMvbuezYH7Yhu/5kkry8hISEhISEhIfFfoMQOWn4RQHd3d8aPH/+fTmf8N0KrJfPYNRQYYayoS6KVJWizqGJfsmugTspG5GmRmRph4lTytMj/KiIvj7xHaqDKy5fJu3UL00qVSj2P8vp10nbuBMC69RtlaKGEhISEhISEhITEy6HUe9CmTZv2P+WcAVw7cwJFuhFCJsfErD7RQledvqR70PL3n8krWiMzKrh/73+NvPh4UKv17zP27Sv1HEKrJXHqNFCpsGrdGvMGDcrSRAkJCQkJCQkJCYmXQokjaI/z22+/sXnzZhISEsjLyzM4Fh4eXiaGvUpcPL0HABNTX4zlCi5rc0AGVexL6KDdlNIbH0f5WCV4gMx9+3EopD6HUKtR37uHvGLFAsce/vobOeHhyCwscJnyeaHCNRISEhISEhISEhKvG6WOoH333Xe89957lC9fnoiICBo1aoSDgwM3btygY8eOz8PGl05iXDQAMiMHrCtbo5KBs7UZFqYl82/zbkkCIY+jvKZz0CybNQUgOzwcdUpKgX53Jk7iett2JE6dhvaxBwHq+/dJWrAAAKePRyKvUOEFWC0hISEhISEhISHx/Cm1g7Z06VJWrlzJ4sWLMTU15ZNPPmHv3r18/PHHpKWlPQ8bXzq5qTkAyIzLQXmdeqV7CRUchUqLKjELkCJo+eRH0CxbtkLh6wtaLZkHDxr0ybkYRfqOHQA83LyZhOABqO4lAXBv7jy06ekofH2x79//xRovISEhISHxP0ZeXh7Vq1cnLCzsZZsi8R/j0KFDyGQyHj58WKbzTpo0iZEjR5bpnC+SUjtoCQkJejl9c3NzMjJ06XvBwcH88ssvZWvdK4BWo0FkagEwMi5Hio0xAG4lVHDMS8wErcDIUo6xndmTB/wPkO+gmVWvjlW7tgBk7Ntv0Cf5++8BMK9bFyMbG3LOnSO2Rw+Sly8nfedOMDLCZdZMZCZPlaUrISEhIfE/yMCBA5HJZMhkMuRyOR4eHnzyySfkPmNNzsc5fPgwbdq0wd7eHgsLC2rUqEFISIjBlhAhBCtXrqRx48ZYWVlhZ2dHgwYNWLRoEdnZ2QBMnz4dmUzG0KFDDeaPjIxEJpMRFxcHQFxcHDKZDGdnZ/09WT516tRh+vTpBm2XLl3irbfewtbWFktLSxo2bEhCQkKxa1q+fDkeHh76+7/8m+rCXqdPnwbgypUrtG7dmvLly6NQKKhatSqff/45KpWq2HMlJCTQqVMnLCwscHZ2ZsKECagf27cOsGHDBvz9/bGwsMDV1ZVBgwbx4MED/fG9e/fi6emJjY0NwcHBBtc+LS0NT09P4h+JlZUlp0+fpsKjrJ47d+5gbm5eYCvQiyQlJYWRI0fi5eWFubk5VapUKZOAip+fH3v26Lb/BAYGsm7durIwt8QcOnSIrl274urqiqWlJXXq1GHDhg0GfcaPH89PP/3EjRs3XqhtZUWpHTQXFxdSHqWjValShRMnTgAQGxuLEKJsrXsFSL+fhEzIAGOMzOXEqXVfNHfHkkXQVI/tP/tf2SeljI0lceo0VHfvFjimfUzB0axGdazbtgMg69gxtFm6SGPOhYtkHjoERka4zvkCj99+xczTE01yMvcXfQuAfXAw5n5+L2ZBEhISEhL/GTp06EBiYiI3btxg4cKFrFixgmnTppXJ3NHR0XTo0IEGDRpw5MgRLly4oM840mg0+n7BwcGMHj2arl27cvDgQSIjI5kyZQpbt27V3/gCKBQKVq9ezbVr15547oyMDL7++uti+8TExNCiRQu8vb05dOgQ58+fZ8qUKSgUiiLHCCH4/vvvGTx4sL6tWbNmJCYmGrzef/99PDw8aPBItEsulzNgwAD27NnDlStXWLRoET/88EOx11qj0dCpUyfy8vIICwvjp59+Yu3atUydOlXf59ixYwwYMIDBgwcTFRXFr7/+yqlTp/SlnrRaLf369WPo0KEcP36cM2fOsHLlSv34SZMmMXToUNzc3Iq/oE/B8ePHad68OQChoaE0aNAAU1PTMj9PSblz5w537tzh66+/5uLFi6xdu5a///7b4G9ZWh4+fMjVq1dp0qQJGo3GYM0virCwMGrXrs3vv//O+fPnee+99xgwYADbt2/X93F0dCQoKIhly5a9UNvKDFFKBg8eLKZPny6EEOL7778X5ubmol27dsLOzk4MGjSotNO9EqSlpQlApKWlFTh2I/y0+LpXJ7Ggb4hY//WvosfSY8Jt4naxLfJ2ieZ+sPGyuDnxiEjbG1fWZr+SaLVaEdu7j4j28haJM2YUOJ5z+YqI9vIWl+s3EFqtVmi1WnGtfaCI9vIWaX/vFkIIkTDkQxHt5S1uf/KJfpwmK0vcHD1aRHt5i6utWwtNZuYLW5OEhITEf43ifveeBq1WK/Jycl74S6vVlsrOkJAQ0bVrV4O27t27i7p16+rfazQaMWfOHOHu7i4UCoWoXbu2+PXXX/XHU1JSRL9+/YSjo6NQKBSievXq4scffxRCCLFw4ULh7u5erA2bNm0SgPjzzz8LvY4PHz4UQggxbdo04e/vL9q3by969uyp7xMRESEAERsbK4QQIjY2VgBiwoQJwsrKSty7d0/f19/fX0ybNk3/vnfv3qJ///7FX6R/cfr0aWFkZCTS09OL7JOXlyecnJzEzJkzi51rzJgxokWLFkUe37lzpzAyMhJ3797Vty1btkzY2NgIpVIphBDiq6++ElWrVjUY991334mKFSsKIYS4d++eAEROTo4QQohPPvlEfPTRR0IIIY4dOybq168v1Gp1sXY+Lb179xYLFy4UQggxYsQIMXHixCeO+fXXX0XNmjWFQqEQ9vb2om3btiLzsXuc1atXC19fX2FqaipcXFzE8OHDn8nGzZs3C1NTU6FSqZ5q/K5du4S/v78QQogzZ86I8uXLP3FMXFyc6Ny5s7CzsxMWFhbC19dX7NixQwghxMGDBwUgUlNTn8qefN58803x3nvvGbT99NNPolKlSs8078ui1PlhK1euRKvVpfwNHz4cBwcHwsLCeOutt/jwww/LznN8RUiKvwmAzMiOKnVMiDuuSz0o6R40vcT+/8j+s6xjYeRERur+/9SpAsfzYv5Jb8yPKFq3bUvKmjVk7NuHvIIrmYcPg7ExjsOG6ccZWVhQ8ZtvyO7dB7NqVTH6Hyv1ICEhIfEqo1Yq+S7knRd+3o9/+g15MdGfJ3Hx4kXCwsIMoilz585l/fr1LF++nBo1anDkyBH69++Pk5MTAQEBTJkyhejoaHbt2oWjoyPXr18nJ0e3V93FxYXExESOHDlCq1atCj3nhg0b8PLyomvXrgWOyWQybG1tDdrmzZtHw4YNOXPmjD46VRh9+/Zl7969zJw5k+8fbRN4HK1Wy44dO/jkk08ICgoiIiICDw8PJk+eTLdu3YqcNzQ0FE9PT6yti76P2bZtGw8ePOC9994rss/169f5+++/6d69e5F9jh8/Tq1atShfvry+LSgoiGHDhhEVFUXdunVp2rQpn376KTt37qRjx44kJSXx22+/8eabbwLg5OSEq6sre/bsoV27doSGhhISEoJKpWLYsGH8+OOPGBsbF2lDaTl69CidO3cGIDMzk7/++ovp06eTlZWFXC5n+fLlTJo0iUmTJhUYm5iYSN++fZk/fz5vv/02GRkZhIaG6jPSli1bxtixY5k3bx4dO3YkLS2NY8eO6ccPHDiQuLg4Dh06VGJ709LSsLGxwaSUW0Rq166tV29XqVTY2dmhUqlQKpXY2dlRpUoVzp8/X+jY4cOHk5eXx5EjR7C0tCQ6Ohorq6KF82QyGWvWrGFgIQrfxa3Lx8fHoK1Ro0bcunWLuLg43N3dSzzXq0CpHTQjIyOMjP7JjOzTpw99+vQpU6NeJWIjrwKQa2aCc/UKJO+9D0CVEuxB0+aoUd/X/aNtWum/76AJIUhevFj/Pu96DOoHDzBxcNC36fef1aiub7Nup3PQMg8dQvMofda2SxdM//VlkslkWDZp/BxXICEhISHxX2f79u1YWVmhVqtRKpUYGRnpHRqlUsmcOXPYt28fTZvqlIarVq3K0aNHWbFiBQEBASQkJFC3bl29s/T4jV/Pnj3ZvXs3AQEBuLi40KRJE9q2bcuAAQOwsbEB4Nq1a3h5eZXY3nr16tGrVy8mTpzI/v37i+wnk8mYN28eXbp0YcyYMVSrVs3geFJSEpmZmcybN4/Zs2fz5Zdf6h2mgwcPEhAQUOi88fHx+n1VRbF69WqCgoKoVKlSgWPNmjUjPDwcpVLJkCFDmDlzZpHz3L1718A5A/Tv7z7aNtG8eXM2bNhA7969yc3NRa1W06VLF5YsWaK/Dps3b2bMmDGMGjWKN998k0GDBjFv3jxat26NQqGgefPmJCcnM3LkSEaMGFHs2p5EgwYNiIyM5PLly/Tr14+zZ8+SkpKiX7dCocDOzq7QsYmJiajVarp3765/SFCrVi398dmzZzNu3DhGjRqlb2vYsKH+/11dXfVBk5KQnJzMrFmzGDJkSClXCTt37kStVtO5c2dGjx5Nu3btGDRoEB06dKBXr17FOnwJCQn06NFDv7aqVasWey4vL68CDyqKY/PmzZw+fZoVK1YYtOd/buPj4/+bDlpRHnFh1K5d+6mNedUQWkFSnC6CdtPhLpnCFbiPvaUptubyJ47Pj54Z2yswtnxy/9edrKNHyTl3DplCgYmDA6rbt8k+fRqbDh30ffIl9s2q/+Ogmdepg7GDA5oHD8g6duxR9GxogfklJCQkJF5NTMzM+Pin317KeUtL69atWbZsGVlZWSxcuBATExN69OgB6KI82dnZtG/f3mBMXl4edevWBWDYsGH06NGD8PBwAgMD6datm148w9jYmDVr1jB79mwOHDjAyZMnmTNnDl9++SWnTp3C1dX1qfbrz549Gx8fH/bs2YOzs3OR/YKCgmjRogVTpkzh559/NjiWfyPftWtXxowZA+hERMLCwli+fHmRDlpOTk6xe9Ru3brF7t272bx5c6HHN23aREZGBufOnWPChAl8/fXXfPLJJ8Wutziio6MZNWoUU6dOJSgoiMTERCZMmMDQoUNZvXo1AC1atNCLlQBcvXqVdevWERERQatWrRg1ahQdO3akZs2atGrVqtB7144dOxIaGgqAm5sbUVFRhdqjUChwd3dn8+bNdOzYEQ8PD8LCwmjZsiXe3t7FrsXf35+2bdtSq1YtgoKCCAwM5J133qFcuXIkJSVx584d2rZtW+T4uXPnPvF65ZOenk6nTp3w9fUtIBxTEipVqsTdu3eJiYmhT58+yOVyTp8+zU8//UTlypWLHfvxxx8zbNgwfVSzR48exfoLly9fLrFdBw8e5L333uOHH37A71/aBObm5gB64Z3XiRI5aHXq1EEmkyGEeKLQxeObYF934qMeoFLqIjo3na+TlKG7XCUuUP0/VP9MCMH9xbonkOX69EFoNKT+3/+RdfKkoYP2KIJm+piDJjM2xrpNax7+qvtxt33rLUyfw+ZdCQkJCYnng0wme6ZUwxeJpaUl1R/9Bv3444/4+/uzevVqBg8eTGam7nd7x44dVKxY0WCc2SNnsGPHjsTHx7Nz50727t1L27ZtGT58uIFAR8WKFQkODiY4OJhZs2bh6enJ8uXLmTFjBp6enqW6AQWoVq0aH3zwAZMmTdI7IUUxb948mjZtyoQJEwzaHR0dMTExwdfX16Ddx8eHo0ePFjmfo6MjFy5cKPL4mjVrcHBw4K233ir0eP7Nu6+vLxqNhiFDhjBu3LhC0wxdXFw49a/tEffu3dMfA51T0rx5c/36ateujaWlJS1btmT27Nm4uroWmPfDDz9kwYIFaLVaIiIi6NmzJxYWFgQEBHD48OFCnYVVq1bpU1fl8qIfsuen6uVHY7du3UpeXh5CCKysrGjZsiW7du0qdKyxsTF79+4lLCyMPXv2sHjxYj777DNOnjyJo6NjkecsLRkZGXTo0AFra2u2bNlS7HoKY+jQoaxfvx6tVotSqcTFxQUhBNnZ2fq0wujoaKpUqVLo+Pfff5+goCB27NjBnj17mDt3LgsWLHhmGfzDhw/TpUsXFi5cyIABAwoczxc1dHJyeqbzvAxKpOIYGxvLjRs3iI2N5ffff8fDw4OlS5cSERFBREQES5cupVq1avz+++/P1dh58+Yhk8kYPXq0vi03N1e/F87KyooePXrov8zPyvkDcaBN172xziXuQf7+sxI6aI8pOP7XyQoNJff8eWQKBQ7vD8aikS4En33qnydY2rw88h5J+T4eQQOwyn9CJEXPJCQkJCReEEZGRnz66ad8/vnn5OTk4Ovri5mZGQkJCVSvXt3g9XiUwMnJiZCQENavX8+iRYsMVAL/Tbly5XB1dSXrkVJxv379uHr1Klu3bi3QVwhRpAT61KlTuXr1Khs3bix2TY0aNaJ79+4F9jyZmprSsGFDrly5YtB+9erVYhUN69aty+XLlwuN/AkhWLNmDQMGDCjRTb9Wq0WlUhWZlte0aVMuXLhAUlKSvm3v3r3Y2NjoHcvs7GyDrTaA3tkrzMbVq1djb2/PW2+9pQ8i5Ev9q1SqIgMLFStW1P/ti7s+kZGRnDlzBmNjY/bv309kZCQODg5s3ryZyMhIVq1aVeRY0D3caN68OTNmzCAiIgJTU1O2bNmCtbU17u7uxaa1loT09HQCAwMxNTVl27ZtxUZDi2LmzJlERkbq0xsjIyMZOHAg/fv3JzIyksjIyCemwVauXJmhQ4fyxx9/MG7cOH744YenXRKgk9rv1KkTX375ZZEpmxcvXkQulxeIrL0WlFZVpGHDhnrllcfZsWOHqFev3tPLlTyBU6dOCXd3d1G7dm0xatQoffvQoUNF5cqVxf79+8WZM2dEkyZNRLNmzUo1d2FqVg+TssV3728SX/fqJOb0e1P0XVNfTPztnHCbuF18s+dKiea988UJcXPiEZF742Gp7Hnd0Gq14sY7PUW0l7e4O+9LIYQQ6tRUEe3tI6K9vIXq/n0hhBA5ly/rFBwbNCygvKVVq8Xd+fNF6m+/vXD7JSQkJP7XKGsVx9eFwlQcVSqVqFixovjqq6+EEEJ89tlnwsHBQaxdu1Zcv35dnD17Vnz33Xdi7dq1QgghpkyZIv78809x7do1cfHiRdG5c2fRqFEjIYQQy5cvF0OHDhW7d+8W169fFxcvXhSffPKJMDIyEocOHRJC6H4ze/fuLczNzcUXX3whTp8+LeLi4sRff/0l2rRpI7Zs2SKE+EfF8XGmTJkiFApFoSqOERER+n5XrlwRJiYmQqFQGKg4/vHHH0Iul4uVK1eKa9euicWLFwtjY2MRGhpa5DVLTk4WcrlcXLhwocCxffv2CUBcunSpwLH169eLTZs2iejoaBETEyM2bdokKlSoIN59910De7y8vPTv1Wq1qFmzpggMDBSRkZHi77//Fk5OTmLy5Mn6PmvWrBEmJiZi6dKlIiYmRhw9elQ0aNBA/zd4nHv37gl3d3dx+/Y/yts+Pj5i+vTpIiwsTFhZWYlTp04VufaScvz4cVGlShUhhBDx8fHC0tKyRCqJJ06c0H8G4uPj9QqLO3fuFEIIsXbtWqFQKMS3334rrl69qv8s5jNp0iQRHBxc5PxpaWmicePGolatWuL69esiMTFR/3oaJUsvLy9x4MABIYQQrVu3Fj///HOJxo0aNUr8/fff4saNG+Ls2bOicePGolevXkKIwlUcvby8xB9//FHkfAcOHBAWFhZi8uTJBmt68OCBQb9p06aJNm3alHKVrwaldtAUCoWIjo4u0B4dHS0UCkWZGPVvMjIyRI0aNcTevXtFQECA3kF7+PChkMvlBvK3ly5dEoA4fvx4iecv7Ifq6G/XxLfvLRdf9+okJn7QTgz/v5ai94ow4TZxu/gj/OaTbT52W9yceETc/DRUaJTPR871VSH94EER7eUtLvnXEarkZH17TNduOvn8Rw79w7+2i2gvbxHbp+/LMlVCQkJCQkgO2r+ZO3eucHJyEpmZmUKr1YpFixYJLy8vIZfLhZOTkwgKChKHDx8WQggxa9Ys4ePjI8zNzYW9vb3o2rWruHHjhhBCiPDwcNG/f3/h4eEhzMzMhIODg2jVqpXYtm2bwfk0Go1YtmyZaNiwobCwsBA2Njaifv364ttvvxXZ2dlCiMIdtLS0NOHo6PhEB00IIYYMGSIAAwdNCJ1se/Xq1YVCoRD+/v6Fyv3/m169eolJkyYVaO/bt2+RD8U3btwo6tWrJ6ysrISlpaXw9fUVc+bM0cvfC6Fztv4dK4iLixMdO3YU5ubmwtHRUYwbN66As/Pdd98JX19fYW5uLlxdXcW7774rbt26VcCGPn36iMWLFxu0nTx5Unh7ewt7e3sxo5ByQE/D3Llz9eUL1q1bJ9q1a1eicdHR0SIoKEg4OTkJMzMz4enpWcDe5cuX6z+Lrq6uYuTIkfpjISEhIiAgoMj5852fwl75nx8hhHBzcyvwOfk3iYmJwtTUVGRnZwulUinMzc0LveaFMWLECFGtWjVhZmYmnJycRHBwsEh+dL9YmIMGiDVr1hQ5X0hISKFr+ve18PLyEr/88kuJbHzVkAlRut2q9erVo2bNmqxatUpffC8vL4/333+fixcvEh4e/gzxvMIJCQnB3t6ehQsX8sYbb1CnTh0WLVrEgQMHaNu2LampqQYKOW5ubowePVq/CfbfKJVKlEql/n16ejqVK1cmOTkZGxsb1HkaNkw5RVZKGOqco8RUyMShiSX7bowlMS2XzR80om4Vu0LnBlBef8jDdZdBgFVQFSxbFB/2fVVQ373L/S/mYDsgGIvHVIKKQ5ORwe33BpF35Qp2A0NwHDdOf+z+l1+Stn4DNr164jxlCg8Wf0/qypXY9OiB8/SyKQoqISEhIVF60tPTcXR01EtuS0gUx/nz52nfvj0xMTHFyqNLvH5kZ2fj4ODArl27eOONN162OWXGrl27GDduHOfPny91SYFXgVJbvHz5crp06UKlSpX0myrPnz+PTCbjr7/+KnMDN27cSHh4uIEaTz53797F1NS0gHxp+fLl9XKshTF37lxmzJhRoH3Pnj1YWFiQddMEZbY5yFIBSLdUY5upITEtF4BrEWEkXix8bkW2EV4XbTARRiQ7KTmbFgk7I0u01peN4/Yd2IeGkhwby82Phj2xv+m9JCr83/9hev8+GjMzzlaujGbnTv1xSyMjKgL3Dx7iTP36uB47hjVwPU/Jmcf6SUhISEi8WF5HVTOJl0ft2rX58ssviY2NNZCBl3j9OXjwIG3atPlPOWcAWVlZrFmz5rV0zuApHLRGjRpx48YNNmzYoFch6t27N/369cOyjIsH37x5k1GjRrF3796n2tRYFJMnT2bs2LH69/kRtMDAQGxsbNjydSSQgYVVDunZkG6pooJTDbgGVmYm9HyrfaFqltpsFSkrLqLRKJG7WeM7sBF+JiXSYXklSPhhFXmA+a1bBLVogXExT1Uz9x/g3vLliOxsTMqXp9KihXjVrGnQR9O8ObH/tx6z+/cJbNiQ28uWoQLqdHkLi6ZNnu9iJCQkJCSKJD09/WWbIPGaUZqiwRKvD506daJTp04v24wy55133nnZJjwTT+VWWlpaPlWRu9Jy9uxZkpKSqFevnr5No9Fw5MgRvv/+e3bv3k1eXh4PHz40iKLdu3dPL8daGGZmZnq53MeRy+WYmJiQcluntJSX8wDQRdBkRjq5UzcHC31q5+MItZb7Gy+hSVFibK/AMdgXY/OC/V5VVPfukfdIAh+NhrwzZ7EJCizQT2i13F+8mAfLlgNg0bAhFRctNChGnY/c0REzb2+Uly6hPHESVYKuppyFt1epJV4lJCQkJMoO6d9gCQkJiVeXEjlo27Zto2PHjsjlcrZt21Zs36LqYDwNbdu2LVB747333sPb25uJEydSuXJl5HI5+/fv1xeZvHLlCgkJCTRt2vSpzqnO06JRaxHaXJQZOqnbdAsV2Rqdg+buUDBKqMlSkbLxMnmxacjMjHEM8cXY6vVxzkBXZNrwfWihDlrq+g1658w+ZADO48cjK+aH3rJRQ5SXLvFw0ybQajGytcXkNaxHISEhISEhISEhIfEiKJGD1q1bN+7evYuzszPdunUrsp9MJivTQtXW1tbU/FfanKWlJQ4ODvr2wYMHM3bsWOzt7bGxsWHkyJE0bdqUJk2eLoUuJzNP9z8ynXOWa6pGJRfcV/4TQXucvDuZPPi/aDSpSmRyIxze9UFevmxTPV8EmY8cNIV/bXLPnSfz6LEChcmFEDz8dTMATmPG4Pjhk6OoFo0bk/LTOnLOnQN09c+eVOxcQkJCQkJCQkJC4n+VEjlojxcULKq44Mti4cKFGBkZ0aNHD5RKJUFBQSxduvSp51NmqQEwMUlHCTy01L2/lW4LaAwiaNkRSaT+cQ2h0mLsoEtrlLu8fs6Z0GjICjsOgPPo0dz8cCjqxETyYmIMCkorr1xBee06Mrmccn37lGhui/r1QSaDR2KhZtWqlf0CJCQkJCQkJCQkJP4jvHbSJocOHTJ4r1AoWLJkCUuWLCmT+fMjaEaPImjpVipkQnA71QTQUOVRBC1tbzwZ+xN0NniVw763F0YWr2dOf+6FC2jT0jCyscGiYUMsGjYk69gxMo8eNXDQ0rbpVDqt3nijWAGRxzG2tcXMxxtl9CUAg/kkJCQkJCQkJCQkJAwpkYP23XfflXjCjz/++KmNeRXIzVIBILSPJPYt1NhhxO1UnePm7mCJJjNP75xZt66MTXs3ZEavb9peZqguvdGyaVNkJiZYtmhB1rFjZB09hsMj1Sah0ZC+YwcANm91KdX8lo0a/+Og1ZAcNAkJCQkJCQkJCYmiKJGDtnDhwhJNJpPJXn8HLVPnoKnzUgCdxL4dchK0AoXcCGdrM3LO3QdAXtEK2yD3l2VqmZEvEGLVsoX+v0lffkn26dNoc3MxUijIPn0a9b17GNnYYBUQUKr5LRo1ImXtWkCKoElISEhISEhISEgUR4kctNjY2OdtxytDbqYKIQSqnGQA0izVuMt0Ndjc7C0xMpKhvKJz3hQ1yr00O8sKzcOH5DxSyrRsoXPQTKtVw8TFBfXdu2SfPoNVyxb69EabDh0wKqTMQHFYNGqIiasrJo6OGDs6lu0CJCQkJCQkJJ4rDx48wMfHh1OnTuHu7v6yzZH4D3Ho0CFat25NamqqQcmsZ2X58uXs2LGDv/76q8zmfJG8PlWUXxC5mSoQ2WjVSgAyLFQo0Al/VHGwQGgFudceAqDwtHtJVpYdWcePg1aLWY3qyB/VjpPJZPpoWtbRULS5uWTs3g2AbSnTGwGMrayotmsn7hvWSwqOEhISEhIvjYEDByKTyZDJZMjlcjw8PPjkk0/Izc0ts3McPnyYNm3aYG9vj4WFBTVq1CAkJIS8vDx9HyEEK1eupHHjxlhZWWFnZ0eDBg1YtGgR2dnZAEyfPh2ZTMbQoUMN5o+MjEQmkxEXFwdAXFwcMpkMZ2dnMjIyDPrWqVOH6dOnA6BSqZg4cSK1atXC0tKSChUqMGDAAO7cufPENX3xxRd07dpV75ytXbtWfx3//UpKSgIgMTGRfv364enpiZGREaNHjy7xNVy7di21a9dGoVDg7OzM8OHD9ceuXLlC69atKV++PAqFgqpVq/L555+jUqn0ffbu3Yunpyc2NjYEBwcbXPu0tDQ8PT2Jj48vsT0l5fTp01SoUAGAO3fuYG5ubnDuF01KSgojR47Ey8sLc3NzqlSpwscff0xaWtozzevn58eePXsACAwMZN26dWVhbonJzc1l4MCB1KpVCxMTk0IV5gcNGkR4eDihoaEv1Lay4qkctFu3brF06VImTZrE2LFjDV6vO7lZKoRGt/8MCy1aY5BpdA5aNScrVHcy0WapkJkZY1qlZEIZrzL58vqWzVsYtOe/zzx6jMyDB9FmZSGvUAHzx4qGlwYjhQJZKSNvEhISEhISZU2HDh1ITEzkxo0bLFy4kBUrVjBt2rQymTs6OpoOHTrQoEEDjhw5woULF1i8eDGmpqYGZYiCg4MZPXo0Xbt25eDBg0RGRjJlyhS2bt2qv/EFnRDa6tWruXbt2hPPnZGRwddff13k8ezsbMLDw5kyZQrh4eH88ccfXLly5Yn1a7Ozs1m9ejWDBw/Wt/Xu3ZvExESDV1BQEAEBATg7OwOgVCpxcnLi888/x9/f/4n25/PNN9/w2WefMWnSJKKioti3bx9BQUH643K5nAEDBrBnzx6uXLnCokWL+OGHH/R/Q61WS79+/Rg6dCjHjx/nzJkzrFy5Uj9+0qRJDB06FDc3txLbVFKOHz9O8+bNAQgNDaVBgwaYvsR7nzt37nDnzh2+/vprLl68yNq1a/n7778N/pal5eHDh1y9epUmTZqg0WgM1vyi0Gg0mJub8/HHH9OuXbtC+5iamtKvX79S6Wi8UohSsm/fPmFhYSFq1qwpTExMRJ06dYSdnZ2wtbUVrVu3Lu10rwRpaWkCEGlpaeLPheFi0cBF4utencS8Ye1FzbU1Rf+l7wq3idvFnxG3RNqBeHFz4hFx/6eol232M6PVasXVlq1EtJe3yDh61OCYOi1NRPv6iWgvb3Gjew8R7eUt7i345iVZKiEhISFRljz+u1cWaLVaoVGqX/hLq9WWys6QkBDRtWtXg7bu3buLunXr6t9rNBoxZ84c4e7uLhQKhahdu7b49ddf9cdTUlJEv379hKOjo1AoFKJ69erixx9/FEIIsXDhQuHu7l6sDZs2bRKA+PPPPwu9jg8fPhRCCDFt2jTh7+8v2rdvL3r27KnvExERIQARGxsrhBAiNjZWAGLChAnCyspK3Lt3T9/X399fTJs2rUhbTp06JQARHx9fZJ9ff/1VODk5FbumpKQkIZfLxbp16wo9HhAQIEaNGlXsHELorq25ubnYt2/fE/s+zpgxY0SLFi2EEELcu3dPACInJ0cIIcQnn3wiPvroIyGEEMeOHRP169cXarW6VPOXlN69e4uFCxcKIYQYMWKEmDhx4hPH/Prrr6JmzZpCoVAIe3t70bZtW5GZmak/vnr1auHr6ytMTU2Fi4uLGD58+DPZuHnzZmFqaipUKtVTjd+1a5fw9/cXQghx5swZUb58+SeOiYuLE507dxZ2dnbCwsJC+Pr6ih07dgghhDh48KAARGpq6lPZU9h3Op/Dhw8LU1NTkZ2d/VRzv0xKLbM/efJkxo8fz4wZM7C2tub333/H2dmZd999lw4dOpSl7/hSeDyClm2hS3lIzrIGwNvFhtwTNwBQeL7++8+U166hTkpCplBg0aCBwTFjGxvMa9cmJyKC3Kgo4OnSGyUkJCQk/vsIlZY7U8Ne+HkrzGyGzNT4qcdfvHiRsLAwg2jK3LlzWb9+PcuXL6dGjRocOXKE/v374+TkREBAAFOmTCE6Oppdu3bh6OjI9evXycnJAcDFxYXExESOHDlCq1atCj3nhg0b8PLyomvXrgWOyWQybG1tDdrmzZtHw4YNOXPmDA3+9Vv9OH379mXv3r3MnDmT77//vkTrT0tLQyaTFbv3JzQ0lPr16xc7z7p167CwsOCdd94p0XmLYu/evWi1Wm7fvo2Pjw8ZGRk0a9aMBQsWULly5ULHXL9+nb///pvu3bsD4OTkhKurK3v27KFdu3aEhoYSEhKCSqVi2LBh/PjjjxgbP/1n5t8cPXqUzp07A5CZmclff/3F9OnTycrKQi6Xs3z5ciZNmsSkSZMKjE1MTKRv377Mnz+ft99+m4yMDEJDQxGPascuW7aMsWPHMm/ePDp27EhaWhrHjh3Tjx84cCBxcXEFSlAVR1paGjY2NpiYlM4FqF27NgkJCeTl5aFSqbCzs0OlUqFUKrGzs6NKlSqcP3++0LHDhw8nLy+PI0eOYGlpSXR0NFZWVkWeSyaTsWbNGgY+UhJ/Who0aIBarebkyZO88cYbzzTXi6bUDtqlS5f45ZdfdINNTMjJycHKyoqZM2fStWtXhg0bVuZGvkhyM1V6if00c90+tIdKO+TGMtytzbifoMvv/i84aFmP5PUtGjbEyMyswHHLFs3JiYgAwMzXR1JglJCQkJB47dm+fTtWVlao1WqUSiVGRkZ6h0apVDJnzhz27dtH06ZNAahatSpHjx5lxYoVBAQEkJCQQN26dfXO0uOiGT179mT37t0EBATg4uJCkyZNaNu2LQMGDMDmUf3Qa9eu4eXlVWJ769WrR69evZg4cSL79+8vsp9MJmPevHl06dKFMWPGUK1atWLnzc3NZeLEifTt21dvW2HEx8fr91UVxerVq+nXrx/m5ubFL+YJ3LhxA61Wy5w5c/j222+xtbXl888/p3379pw/f94gXbBZs2aEh4ejVCoZMmQIM2fOBHTXYfPmzYwZM4ZRo0bx5ptvMmjQIObNm0fr1q1RKBQ0b96c5ORkRo4cyYgRI57J5gYNGhAZGcnly5fp168fZ8+eJSUlRW+fQqEo0gFOTExErVbTvXt3/UOCWrVq6Y/Pnj2bcePGMWrUKH1bw4YN9f/v6uqKVqstsa3JycnMmjWLIUOGlHKVsHPnTtRqNZ07d2b06NG0a9eOQYMG0aFDB3r16lWsw5eQkECPHj30a6tatWqx5/Ly8irwoOJpsLCwwNbW9rnsN3zelNpBs7S01G94dHV1JSYmBj8/P0D3h3/d0UXQHgJw31K34TRd7Ux1Z2u0semgFZg4mmNir3iJVhaP0GhIXrqMnIhwbDp1wqZjR4wsLP45LgQ5Z8+S9ucW4B95/X9j1bIlyYt1P1q2XYrPUZeQkJCQ+N9FJjeiwsxmL+W8paV169YsW7aMrKwsFi5ciImJCT169AB00Zjs7Gzat29vMCYvL4+6desCMGzYMHr06EF4eDiBgYF069aNZs10azc2NmbNmjXMnj2bAwcOcPLkSebMmcOXX37JqVOncHV11UdHSsPs2bPx8fFhz549+j1ehREUFESLFi2YMmUKP//8c5H9VCoVvXr1QgjBsmXLij13Tk4OCkXR9zzHjx/n0qVL/N///d+TF/IEtFotKpWK7777jsDAQAB++eUXXFxcOHjwoMFetE2bNpGRkcG5c+eYMGECX3/9NZ988gkALVq04PTp0/q+V69eZd26dURERNCqVStGjRpFx44dqVmzJq1ataJ27doFbOnYsaNeYMLNzY2oR9lE/0ahUODu7s7mzZvp2LEjHh4ehIWF0bJlS7y9vYtdr7+/P23btqVWrVoEBQURGBjIO++8Q7ly5UhKSuLOnTu0bdu2yPFz584tdv7HSU9Pp1OnTvj6+uqFY0pDpUqVuHv3LjExMfTp0we5XM7p06f56aefioxu5vPxxx8zbNgwfVSzR48ehV7zfC5fvlxq+4rC3NxcL7zzOlFqB61JkyYcPXoUHx8f3nzzTcaNG8eFCxf4448/aNKkyfOw8YWhztOgUmoQWp26TaKVbkNvtrocPi7W5F7TRdZe5eiZOjWVO+PGkxWmSzXJCjvOvTlzsXmrC7ZdupBz/jwPN/9K3g1dqqbM3ByrNm0KnUvh54fcrQqaBynYdHrzha1BQkJCQuL1QiaTPVOq4YvE0tKS6o8yQn788Uf8/f31IhiZmZkA7Nixg4oVKxqMM3uUadKxY0fi4+PZuXMne/fupW3btgwfPtxAoKNixYoEBwcTHBzMrFmz8PT0ZPny5cyYMQNPT89S34BWq1aNDz74gEmTJrF69epi+86bN4+mTZsyYcKEQo/nO2fx8fEcOHCg2OgZgKOjI6mpqUUeX7VqFXXq1HliGmRJcHV1BcDX11ff5uTkhKOjIwkJCQZ9850CX19fNBoNQ4YMYdy4cYWmL3744YcsWLAArVZLREQEPXv2xMLCgoCAAA4fPlyos7Bq1Sp96qpcLi/S5vxUvfxo7NatW8nLy0MIgZWVFS1btmTXrl2FjjU2Nmbv3r2EhYWxZ88eFi9ezGeffcbJkydxLMOyRBkZGXTo0AFra2u2bNlS7HoKY+jQoaxfvx6tVotSqcTFxQUhBNnZ2fj4+AA6gZwqVaoUOv79998nKCiIHTt2sGfPHubOncuCBQsYOXLkM6/tSaSkpODk5PTcz1PWlPrR0zfffEPjxo0BmDFjBm3btmXTpk24u7s/8R+NV53cLBUIJaAGIMVC95RLaKzxdrEi96ruHyizV9RBy718mbievcgKC0Nmbo59yADkVaqgzcri4S8bie/3LknzviTvxg1k5ubYvtMD919+xrRSpULnkxkb475xI1V3bEdezBM7CQkJCQmJ1xEjIyM+/fRTPv/8c3JycvD19cXMzIyEhASqV69u8Ho8SuDk5ERISAjr169n0aJFBiqB/6ZcuXK4urqSlZUFQL9+/bh69Spbt24t0FcIUaQE+tSpU7l69SobN24sdk2NGjWie/fuhe55ynfOrl27xr59+3BwcCh2LoC6desSHR1d6LHMzEw2b978TKqAj5OvBnjlyhV9W0pKCsnJycWqLuZH3gpL91u9ejX29va89dZbeiXNfEl+lUploK75OBUrVtT/7Ys7d2RkJGfOnMHY2Jj9+/cTGRmJg4MDmzdvJjIyklWrVhW7ZplMRvPmzZkxYwYRERGYmpqyZcsWrK2tcXd3LzattSSkp6cTGBiIqakp27ZtKzYaWhQzZ84kMjJSn94YGRnJwIED6d+/P5GRkURGRj4xDbZy5coMHTqUP/74g3HjxvHDDz887ZJKTExMDLm5ufro9+tEqSNoj+eNWlpasnz58jI16GWSm6VCCN0/oHIzORpjgVwLaM2oZaFAk6oEYxlmVZ89L7asSdu+g8TPP0fk5iKvXJlK3y9G4eWF88SJZJ86xcPNm8k4cBDTqh6U69ULm86dMS5mg2Y+JuVeTWdUQkJCQkKiLOjZsycTJkxgyZIljB8/nvHjxzNmzBi0Wi0tWrTQCzPY2NgQEhLC1KlTqV+/Pn5+fiiVSrZv366PIqxYsYLIyEjefvttqlWrRm5uLuvWrSMqKorFixcD0KtXL7Zs2ULfvn35/PPPCQwMxMnJiQsXLrBw4UJGjhxZaF2n8uXLM3bsWL766qsnrumLL77Az8/PYF+QSqXinXfeITw8nO3bt6PRaLh79y4A9vb2RcrBBwUFMXnyZFJTUyn3r3uCTZs2oVar6d+/f6FjIyMjAZ0jd//+fSIjIzE1NdVHyLZs2cLkyZP1EUVPT0+6du3KqFGjWLlyJTY2NkyePBlvb29at24N6ERW5HI5tWrVwszMjDNnzjB58mR69+5dIDKUlJTE7Nmz9cIa5cqVw8fHh0WLFhEYGMj+/fv57LPPnng9i6N69eqcOHGC8uXL06JFCxISEsjIyKBLly5PFOI4efIk+/fvJzAwEGdnZ06ePMn9+/f1n6fp06czdOhQnJ2d6dixIxkZGRw7dkwfeZo8eTK3b98usg5ZvnOWnZ3N+vXrSU9PJz09HdA9ZCipWIqzszPOzs6cP3+eYcOGUb16dS5dusQHH3ygj0YXx+jRo+nYsSOenp6kpqZy8OBB/RoLw9vbm7lz5/L2228X2Sc6Opq8vDxSUlLIyMjQf9bq1Kmj7xMaGkrVqlWfuB/zlaS0so+DBw8WBw8eLEslyZdOvtxw1OlY8e2gNeLrXp3Esg97iJpra4oWP9QWbhO3izv748TNiUdE0g/nX7a5BUg/eFBEe3mLaC9vET/4faF+SqlSCQkJCYn/DcpaZv91oShJ7rlz5wonJyeRmZkptFqtWLRokfDy8hJyuVw4OTmJoKAgcfjwYSGEELNmzRI+Pj7C3Nxc2Nvbi65du4obN24IIYQIDw8X/fv3Fx4eHsLMzEw4ODiIVq1aiW3bthmcT6PRiGXLlomGDRsKCwsLYWNjI+rXry++/fZbvSR4vsz+46SlpQlHR8dCZfYjIiIM+g4ZMkQAepn9/H6FvZ50X9eoUSOxfPnyAu1NmzYV/fr1K3JcYedyc3PTH1+zZo34961oWlqaGDRokLCzsxP29vbi7bffFgkJCfrjGzduFPXq1RNWVlbC0tJS+Pr6ijlz5uhl9R+nT58+YvHixQZtJ0+eFN7e3sLe3l7MmDGj2HWXlLlz54r+/fsLIYRYt26daNeuXYnGRUdHi6CgIOHk5CTMzMyEp6dnAXuXL1+u/yy6urqKkSNH6o+FhISIgICAIufPl7Av7JX/+RFCCDc3t2LLMQghRGJiol6yXqlUCnNzc3Hr1q0SrXPEiBGiWrVqwszMTDg5OYng4GCRnJxsYOPjMvuAWLNmTbFzurm5FbquxwkMDBRz584tkY2vGjIhSrdbtWvXruzevRsnJyf69OlD//79S1WA8FUkPT0dW1tbwg9eI/Snv1Fl/421ixmL613FJdeMlKS5/F3RFeXVVGzf9MC6VeEpgS8DbVYWMV26oL6TiF3PnrhMn4asDOVjJSQkJCT+e+T/7uVLbktIFMeOHTuYMGECFy9exMio9MIsEq8u2dnZODg4sGvXrtdOir44oqKiaNOmDVevXi0TRcgXTam/ZVu3biUxMZEpU6Zw+vRp6tWrh5+fH3PmzCEuLu45mPjieDzFUZjq8pNN1ApqlrciL1aXE/6qCYTc/24x/9/encdFVa9/AP/MDDMw7PsugiKiooi45FJqLqhJbuFakVm2WL/S22a3sm6ldu8tW665lXm9am5l5VZuKLkjiuICgoAogiD7Ogwz398fNCdGQEF2+7xfL14vOOfMOc85IM7D8/0+34ob6VB6eMDl7flMzoiIiKhRPfLII5g9ezbS0tJaOhRqZBEREXj44Yfvq+QMqFzCYO3atW0yOQOAelfQbnf9+nV8//33WL16NRISElBRUdFYsTUbw18SD2w+i7Pbt0CnOQNVRy1Wdr4Bz3x3PGb3CUIuFEJurYLb/L6QyWQtHTIAoPT8BaRMngzo9Wi3aiUsH3ywpUMiIqI2gBU0IqLWq0F1aq1Wi1OnTuHEiRNISUmBi4tLY8XVIjRFFRD6ygpaqUlla1VdhRW66isfk5mvbatJzkRFBTLeew/Q62E9ZgyTMyIiIiKi+8A9JWgRERF49tln4eLigqeeegrW1tbYsWMHrl+/3tjxNavSYi2EvnINlEJlGQBAU2ELl8rRjlC6WrRUaNXkrFuHsosXIbe2hsvb81s6HCIiIiIiagT1brPv4eGBnJwcjBo1CitXrkRoaKi0eGNbV16iBf6Yg5b3R4JWqnOAulALHQATJ3ULRvcn7Y0byPqysl2v82t/g0kjLmZIREREREQtp94J2vvvv4+wsDDY2to2QTgtq7SoXKqg3VLrAMhhofaE7lZlsqZ0Nm/B6CoJIZDx4UcQJSVQ9+oF28cea+mQiIiIiIiokdR7iOOzzz57XyZnAFBaWASgckX5m38UywJtvIAKPaCQQWFX/9XX60tXUICi33+HqGVl+8I9e1EUEQEolXD74H3I2O6WiIiIiOi+wXf3VZQWVbbSV5mZIVtV2Qwk2MwZAGDiqIZM3vQNQtJeew3Xnp2Nmx9/XG2frrAQNz/6CADg8MwsmHbq1OTxEBERERFR82GCVkVFWSEAQG2ugv6Pbo1dFJXth5tjeGPJmTMojvwdAJC74XvkrF9vtD9ryeeoyMqCsr0XHJ97rsnjISIiIiKi5sUErao/Wuwr/xjeqNbJ4KqrfETN0SDk1lf/qby+pycA4ObCRSg6fAQAUBoTg9zvvwcAuL3/PuRmTT/ckoiIiKi8vBy+vr44evRoS4dCbYy3tzc+//zzRj3nrVu34Ozs3Oa7x98JE7QqxB8dHHUm5QAAtU4FVX7l501dQSs5fRrFR48CJibwWvMdbMaPB3Q6pM2di7L4y0h/bwEgBGzGjYNF//5NGgsREdH96KmnnoJMJoNMJoNSqYSPjw/eeOMNlJWVNdo1Dh06hIcffhj29vYwNzdHp06dEB4ejvLycukYIQRWrlyJfv36wdLSEra2tujduzc+//xzlJSUAKhsyiaTyfD8888bnT8mJgYymQwpKSkAgJSUFMhkMjg7O6OwsNDo2J49e+L999+Xvv7xxx8xcuRIODg4QCaTISYmpk73tHz5cvj4+GDAgAHSNWfNmgUfHx+o1Wp07NgRCxYsMLrHgwcPYty4cXBzc4OFhQV69uyJ9beNDKrNmjVr0KNHD5iZmcHZ2Rlz5syR9qWkpOChhx6ChYUFHnroIek5GIwdOxY//PBDna5TH3q9HtbW1rh8+TIAwM/PD5GRkY1+nfqIjIxEaGgo3N3dIZPJ8NNPPzX4nP/6178wffp0AMCGDRvw8MMPN/ic9bVy5UoMGTIE1tbWkMlkyMvLM9rv6OiIJ598EgsWLGj22JoLE7QqhL7yl6JGXvmL2gJqVGRVLlht4tS0Cdqt/1RWz2wnTIDK0xOu//gA6uBg6AsLkTJlCjSXL0Nhawvnt95s0jiIiIjuZ6NGjUJ6ejqSkpKwZMkSrFixotHe6F28eBGjRo1C7969ERkZidjYWHz11VdQqVTQVWn+9cQTT+DVV1/FuHHjEBERgZiYGLz77rv4+eefsWfPHuk4MzMzfPvtt0hISLjrtQsLC/Hvf//7jscUFxdj0KBB+OSTT+p8T0II/Oc//8GsWbOkbXFxcdDr9VixYgUuXLiAJUuWYPny5Xj77belY44ePYoePXrghx9+wLlz5zBz5kw8+eST2LFjxx2v99lnn+Hvf/873nrrLVy4cAH79u1DSEiItP9vf/sbPDw8EBMTAzc3N7z22mvSvk2bNkEul2PSpEl1vr+6On/+PMzMzODn54ebN2/i6tWr6NOnT6Nfpz6Ki4sRGBiIpUuXNto5jx07hoEDBwIAfv/9d+nz5lRSUoJRo0YZ/TzdbubMmVi/fj1ycnKaMbJmJFqxr7/+WnTv3l1YWVkJKysr8cADD4hdu3ZJ+0tLS8WLL74o7O3thYWFhZg4caLIyMio93Xy8/MFALFw8gvi35MfEZ/NHyoC1gSIp1ZNFNfejBTX3owUurKKxrw1I8WnTomLnf3FxW4BQnPturRdm50tEh4eVrmvs7/I/XFbk8VARER/HYb/9/Lz8xvlfHq9Xmg0mmb/0Ov19YozPDxcjBs3zmjbxIkTRVBQkPS1TqcTCxcuFN7e3sLMzEz06NFDbNmyRdqfk5Mjpk+fLhwdHYWZmZnw9fUVq1evFkIIsWTJEuHt7X3HGDZt2iQAiJ9++qnG55iXlyeEEGLBggUiMDBQjBgxQoSFhUnHnDlzRgAQycnJQgghkpOTBQDx+uuvC0tLS3Hz5k3p2MDAQLFgwYJq1zG85syZM3eMVQghoqKihFwuFwUFBXc87p///Kfw8fG54zFjxowRM2fOrHV/Tk6OUKvVYt++fbUe06VLF7F7924hhBC7du0SXbt2FUIIkZubK3x9fUVqauodY7hXy5Ytk352tm7dKvr163fX18TExIghQ4YIS0tLYWVlJXr16iWioqKk/YcPHxaDBw8WarVa2NraipEjR4qcnJx7ig+A2LZt2z29tipXV1fp5yIgIEB61rXR6/ViwYIFol27dkKlUgk3Nzfx8ssvS/vbt28vlixZck+xRERECAAiNze3xv0+Pj7im2++uadzt3b1XgetOXl6emLx4sXo1KkThBD473//i3HjxuHMmTPo1q0b5s6di507d2LLli2wsbHBSy+9hIkTJ+LIkSP3dD2hLwHkQKmycqhjR1kHAIDCxhRyU0Wj3dftsgzVs4kTofL0kLab2Nuj3fJlSH16FtQ9A2EzflyTxUBERHSvtFotFi5c2OzXffvtt6FSqe759efPn8fRo0fRvn17aduiRYuwbt06LF++HJ06dUJkZCQef/xxODk5YfDgwXj33Xdx8eJF7N69G46OjkhMTERpaeVoG1dXV6SnpyMyMhIPPfRQjddcv349OnfujHHjqv+fLpPJYGNjY7Rt8eLF6NOnD06dOoXevXvXei/Tpk3D3r178Y9//AP/+eN9RWP4/fff4efnBysrqzsel5+fD3t7+7se06VLl1r37927F3q9HmlpaejSpQsKCwsxYMAAfPrpp2jXrh0AIDAwEPv27cPIkSOxZ88e9OjRAwDw+uuvY86cOdJxjcWwtFRZWRmEELC1tYVGo4FOp4OtrS0GDRpUa1VwxowZCAoKwrJly6BQKBATEwOlUgmgcqjqsGHD8PTTT+OLL76AiYkJIiIipErrmjVrMHPmTAghGvV+arJ48WIsXrwYQOX3aPDgwZDJZMjPz8fkyZMhl8uxY8cODBo0qNprf/jhByxZsgQbN25Et27dkJGRgbNnz9Z6raeeegopKSk4ePBgg+Pu27cvfv/9d6Pq7v2iVSdooaGhRl9//PHHWLZsGY4fPw5PT098++23RuNjv/vuO3Tp0gXHjx/HAw88UP8L/jEHrVRVOcSxo8wbAGDi3HQNQkpOnULJseOAUgnH52ZX22/aqRN8Iw9BJmv6Fv9ERET3ux07dsDS0hIVFRXQaDSQy+VSQqPRaLBw4ULs27cP/f+Y792hQwccPnwYK1aswODBg5GamoqgoCApWfL29pbOHRYWht9++w2DBw+Gq6srHnjgAQwbNgxPPvkkrK0ru0InJCSgc+fOdY63V69emDx5Mt58803s37+/1uNkMhkWL16M0NBQzJ07Fx07dqzvo6nR1atX4e7ufsdjEhMT8dVXX91xiOXmzZsRFRWFFStW1HpMUlIS9Ho9Fi5ciC+++AI2NjZ45513MGLECJw7dw4qlQr//ve/8dxzz8Hb2xs9evTAihUrEBkZiZiYGHzyySeYPHkyTp06hZEjR+LLL79sUAIPVCZSQggEBwdjw4YN8Pf3x8iRI/H+++9jwIABMLtD07bU1FS8/vrr8Pf3BwB0qrI80j//+U/07t0bX3/9tbStW7du0uc2Njb1+jlpiOeffx5Tp07FmjVrcPz4cSxfvhy7du3CmjVrsHnzZgCVf3yoSWpqKlxdXTF8+HAolUp4eXmhb9++tV7Lzc0Ner2+UeJ2d3fHmTNnGuVcrU2rTtCq0ul02LJlC4qLi9G/f39ER0dDq9Vi+PDh0jH+/v7w8vLCsWPH7pigaTQaaDQa6euCggIAgNAXAwo5itTlAEzgoav8YZQ7mEGr1TbJfWV+9RUAwHr8eMDZucmuQ0REZNDY/9colco7zhdpKoZqRH0MHToUy5YtQ3FxMZYsWQITExNpzlJiYiJKSkowYsQIo9eUl5cjKCgIAPDCCy9g0qRJOH36NEaOHInx48dLzTMUCgW+++47fPTRRzhw4ABOnDiBhQsX4pNPPsHJkyfh5uZ2TxWRjz76CF26dMGePXvg7Oxc63EhISEYNGgQ3n33XWzYsKHe16lJaWnpHZOQtLQ0jBo1CmFhYXj22WdrPCYiIgIzZ87EqlWrjJKQ2+n1emi1Wnz55ZcYOXIkAOD777+Hq6srIiIiEBISAg8PD6OKlUajQUhICP773//io48+gpWVFeLj4zFq1CisWLECL7/8crXrLFy40Kjie/HiRXh5edUYk7e3N06ePAlzc3OMGjUK169fx40bNzBp0iSYmprWei8AMG/ePDzzzDP43//+h+HDhyMsLExKnGNiYhAWFlbraydMmIAJEybc8fyNxdbWFra2tjh58iQmTZoEb29vnDlzBo8++qjRHyBqEhYWhs8//xwdOnTAqFGjMGbMGISGhsLEpOYUY9GiRY0Wt1qtlprq3G9afYIWGxuL/v37o6ysDJaWlti2bRu6du2KmJgYqFQqqfRs4OLigoyMjDuec9GiRfjggw9q2KMDIMcts8rM3iSrsodK3M0ruLXrYiPcjTGLixfhceIkhEKBmI4dUbFrV6Nfg4iI6HaN/aZGJpM1uFLRXCwsLODr6wsAWL16NQIDA/Htt99i1qxZKCoqAgDs3LkTHh4eRq8zvBkfPXo0rl69il27dmHv3r0YNmwY5syZY1Q98vDwwBNPPIEnnngCH374Ifz8/LB8+XJ88MEH8PPzQ1xcXL1i7tixI5599lm89dZb+Pbbb+947OLFi9G/f3+8/vrr9bpGbRwdHREbG1vjvhs3bmDo0KEYMGAAVq5cWeMxhw4dQmhoKJYsWYInn3zyjtdyc3MDAHTt2lXa5uTkBEdHR6Smptb4moULF2LkyJEIDg7Gs88+i48++ghKpRITJ07EgQMHakzQnn/+eUyePFn6urYK4ejRo/H777+joqICFRUVsLS0hE6ng0ajgYODAwBIPzM1ef/99zF9+nTs3LkTu3fvxoIFC7Bx40ZMmDABanXTL99UF7///jtGjx4NoPL3wsGDBzF37lyUlpZCqVRi8eLFePvtt2v9A0y7du0QHx+Pffv2Ye/evXjxxRfxr3/9C4cOHbqnP6DUR05ODpycnJr0Gi2l1SdonTt3RkxMDPLz87F161aEh4fj0KFDDTrn/PnzMW/ePOnrgoICacyyiUqNbFXlcEJ3mRMAgZ5DekPVwaamU9VICAHNhQso/PkXKJydYPfMM9WGKOpLSpC65HNUALB78kl0mjG9QfdERERUV4aRI391crkcb7/9NubNm4fp06eja9euMDU1RWpqKgYPHlzr65ycnBAeHo7w8HA8+OCDeP3112sd3mdnZwc3NzcUF1dOo5g+fTqmTp2Kn3/+udo8NCEECgoKqs1DA4D33nsPHTt2xMaNG+94T3379sXEiRPx1ltv3e3268Qwh0oIYfReJi0tDUOHDkVwcDC+++47yOXVG4MfPHgQY8eOxSeffILZs6tP47idoWNgfHw8PP9YEzYnJwe3bt0ymidocOnSJWzYsEFaLkCn00nVYa1Wa9Q5syp7e/u7zpcDgG+++QalpaUIDw/HxIkTMW7cOLz22mvw9/fHM888c9fXA5Xt+P38/DB37lxMmzYN3333HSZMmIAePXpg//79tRQMmk/v3r0RExOD6OhovPHGG9i/fz9SU1Px6KOP4vTp05DL5Xd9Vmq1GqGhoQgNDcWcOXPg7++P2NhY9OrVq0ljP3/+PIYMGdKk12gprT5BU6lU0l+6goODERUVhS+++AJTpkxBeXk58vLyjKpoN2/erHWcrIGpqWmtZWml2hw5CgWUehMoCiqHIZi5WUNRh78C6AoLUbBjB3I3b4Hm0iVpu0JhUm1+2c2vl6EiIwNKT0+4/N/LkDfxXxmIiIgMmvov221JWFgYXn/9dSxduhSvvfYaXnvtNcydOxd6vR6DBg1Cfn4+jhw5Amtra4SHh+O9995DcHAwunXrBo1Ggx07dkiNL1asWIGYmBhMmDABHTt2RFlZGdauXYsLFy7gqz+mNEyePBnbtm3DtGnT8M4772DkyJFwcnJCbGwslixZgpdffhnjx4+vFqeLiwvmzZuHf/3rX3e9p48//hjdunWrNswsJycHqampuHHjBoDKRAionF9U23unoUOHoqioCBcuXEBAQACAyuRsyJAhaN++Pf79738jKytLOt5wnoiICIwdOxavvPIKJk2aJI1uUqlU0hv+bdu2Yf78+VJF0c/PD+PGjcMrr7yClStXwtraGvPnz4e/vz+GDh1qFJcQArNnz8aSJUtgYWEBoDLBW7VqFfz8/LB27VpMmzbtrs/qTjw8PFBRUYFz585h3bp18PHxwblz5/Dmm29K701rU1paitdffx2PPfYYfHx8cP36dURFRUnDaefPn4/u3bvjxRdfxPPPPw+VSoWIiAiEhYXB0dGx2rOpSVFRERITE6Wvk5OTERMTA3t7+1qHbN5OrVbD19cXW7duxZAhQ6QFyQcOHAg/P7+7vn7NmjXQ6XTo168fzM3NsW7dOqjV6hoTasN9p6WlYe3atbWeMyMjAxkZGdK9xcbGwsrKCl5eXtLPTklJCaKjo1ukOVGzaLH+kfdo6NChIjw8XOTl5QmlUim2bt0q7YuLixMAxLFjx+p1TkO74Y8mjBQrX3pGBKwJEI8sHyGuvRkpri84Uqc2vvm7fxWXgnpJLfEvde8hUp56Svo6/7ffpGNLYs+Li126ioud/UVhZGS9YiUiImqoxm6z31bU1GZfCCEWLVoknJycRFFRkdDr9eLzzz8XnTt3FkqlUjg5OYmQkBBx6NAhIYQQH374oejSpYtQq9XC3t5ejBs3TiQlJQkhhDh9+rR4/PHHhY+PjzA1NRUODg7ioYceEr/88ovR9XQ6nVi2bJno06ePMDc3F9bW1iI4OFh88cUXoqSkRAjxZ5v9qvLz84Wjo2ONbfZvb5k/e/ZsAcCozf53330nAFT7qKkVf1WTJ08Wb7311l3PU/VtZXh4eI37Bw8eXO08t9/j008/LWxtbYW9vb2YMGFCja3zly9fLiZNmmS07ebNm2LYsGHCyspKhIWFieLi4jveV10cO3ZMeHp6CiGEuHbtmjA3Nxfl5eV3fZ1GoxFTp06V2s+7u7uLl156SZSWlkrHHDx4UAwYMECYmpoKW1tbERISIrWUr+nZ3M7Qhv72j/DwcOmYBQsWiPbt29813pCQEKll/dNPPy0++uiju75GCCG2bdsm+vXrJ6ytrYWFhYV44IEHjJZJuL3Nfnh4uNHPQE0WLFhQ431999130jEbNmwQnTt3rlOMbZFMiGbo33mP5s+fj9GjR8PLywuFhYXYsGEDPvnkE/z2228YMWIEXnjhBanLjLW1tTTO+OjRo/W6jmE4wUcTRsLFxwtf9DiOB/OD8PaNZ6FqZwXnOT3v+PrSs2dx9YknIcrLofLtCLvJk2Hz6KNQ2Noi48OPkLt+PWRqNbzXr4Opnx9SJk9B2cWLsB4zBh6ffXqvj4eIiOieGP7fy8/Pl7oLEtXm3LlzGDFiBK5cuQJLS8uWDofqITw8HDKZDGvWrGnpUBrVAw88gP/7v//D9On35xShVj3EMTMzE08++STS09NhY2ODHj16SMkZACxZskRaMd7Qxadqu9J7IVTlAICO5ZUTVU2c7jyJU3vjBq7NeQmivByWw4bB86svIasyDttl/lsoT0lB8ZEjuPbCi7B59FGUXbwIubU1XN6e36BYiYiIiJpajx498MknnyA5ORndu3dv6XCojoQQOHjwIA4fPtzSoTSqW7duYeLEiQ0ewtqateoKWnOpWkGzD7DC1x3j8d71WehfGAzrUd6wHlLzoof64mKkzHgcmrg4mPr7w3v9Osj/GAddla6gAClTp6E8KUna5vqPD2BXpYMQERE1raysLFy7ds1om0KhgL+//13bZTeFa9euwcbGpkUqWKygERG1Xq26gtYSKkwqWw97lFdOclXWUkETej3S3ngTmrg4KBwc0O7rpTUmZwCgsLZGu+XLkBI2Gbr8fKh79YLtY481zQ0QEZERIQSio6Oxa9euGhdI9ff3x9SpU5s1ptTUVKxevRpubm547rnnmvXaRETUujFBu02ZshgQgIu2cl0FE2fzGo/LWvI5ivbvh0ylQrul/4GyljU0DFReXmj3zSrkrlsPx5dfMhoGSURETUOr1WLXrl04c+YMgMqubObmf/5ev3LlCuLi4nDp0iWpE19zMLQFT09PR2FhIaysrJrt2kRE1LoxQbtNmbIEjhV2MNWrALkMJvZm1Y4pv34d2d98AwBw+/hjqHv2rNO51d27Q/3J4sYMl4iIapGfn4/NmzcjLS0NMpkMw4YNw8CBA43Wctq3bx8OHz6MXbt2wcfHB2Zm1X/nNzatVosLFy5IX1+9elVqX05ERMQyzm2KVCVop3EBAJg4mEGmqP6I8jZuBISAxYD+sAkd29whEhHRXaSkpGDlypVIS0uDmZkZZsyYgUGDBhklZwAwePBg2NnZobCwEAcOHGiW2BISEqDRaKSvk5OTm+W6RETUNjBBq0KhNEexSQXa/TH/zMSp+vBGvUaDvK0/AADsZsxo1viIiNqa5u5DJYTAiRMnsHbtWhQXF8PFxQWzZ8+udVFZpVKJsWMr/9B28uRJpKWlNXmM586dAwA4OzsDqEwmiYiIDJigVaFSW6NQoYdneWUFTelcvUFIwa7d0OXlwcTdDZZDhjRzhEREbUdkZCQWLlxYrXNiVREREVi8eDFu3LjR4OtptVr89NNP2L17N/R6PQICAjBr1izY29vf8XUdO3ZEjx49AADbt2+HTqdrcCy1KSkpQUJCAgDgkUceAQBkZ2ejsLCw0a6xefNmfPXVVygtLW20cxIRUfNhglaFqdoSeXLFn0Mca6ig5W7YAACwmzIVMoWiWeMjImor0tPTERERAa1Wi6ioqBqP0el0OHHiBMrKyrBv3756nT8zMxOJiYnSx+XLl7F69WqcPXsWMpkMISEhmDRpElQqVZ3OFxISArVajYyMDBw/frxesdTHxYsXodPp4OLigvbt28PVtXLERmNV0QoKCnDx4kVkZ2cjLi6uUc5JRETNi01CqlCpzZGrkMNTGuJoXEErPXcOZbGxkCmVsA1jm3wiopro9Xps375dGt54+fJl6HQ6KG77o9bVq1dRVlYGAEhKSkJqaiq8vLzuev7jx4/j119/rXGfubk5wsLC4OPjU6+YLSwsMGLECPzyyy84ePAggoODm6RhiGF4o6Fi5+3tjYyMDKSkpDTKAsCJiYnS53FxcQgKCmrwOYmIqHmxglaFytQERVDBscIWAKp1cMxdX1k9sxo9CiZ3GTJDRG1LWVkZTp06haKiopYOpVVKSkpCUlJSnY49efIkbty4AVNTU6jVapSVleHq1avVjjNUeOR/LDsSERFx13NfvnwZv/32GwDAyckJrq6u0oe/vz9mz55d7+TMICgoCA4ODtBqtdIwxMaUm5uL1NRUAJCSMUOsjVVBu3LlitHn5eXljXJeIiJqPqygVaUScNDZVn5uIoPcQintqsjNRcHu3QAA++nTWyA4ImoqOp0OGzduREpKCq5fv47x48e3dEitSlFREdatWwchBJ555hl4eHjUemx+fr7UDXH48OG4ceMGzpw5g7i4OHTo0EE6TgghJWijRo3Cr7/+iuTkZFy9ehXt27ev8dyZmZnYunUrhBDo1asXQkNDq3VlbAiZTAZ/f38cOXIEcXFxjVLRqio2NhZAZVJmbW0NAFLFMDs7GwUFBdL2e6HX66UETaFQoKKiAleuXGnW9d2IiKjhWEGrQpiUwUlbWRkzsTUz+o8/b+tWiPJymHXrBrPAwJYKkYgamRACO3fulCoYCQkJ0Ov1LRtUDcrLy7Fz584mqezczZUrV6DX6yGEuGsTjd27d6O8vByenp4IDg6Gv78/gMpqWdWOjjdu3EBBQQGUSiWCgoKkoXgHDx6s8bzFxcXYsGEDysvL0b59e4wZM6ZRkzMDQzKTkJCAioqKRjuvEKLa8EYAUKvVcHNzA9DwKlpaWhrKyspgZmaG3r17AwDnoRERtUFM0KooVxbAscIOAKCw+XNiudDpkPf9RgCA3fTpTfKmgIhaxokTJ3D69GkAlVWH4uJi3Lx5s4Wjqu7kyZOIiorCzp07m711fdV5TRkZGThx4kSNx126dAlxcXGQy+UIDQ2FXC5Hhw4doFQqUVBQgPT0dOlYQ+LQqVMnKJVKPPjgg5DL5UhOTq6WqFRUVGDTpk3Iy8uDnZ0dpkyZAhOTphkA4u7uDktLS5SXlzfq+mTp6em4desWTExMqlW0vL29ATQ8QTN8nzp06ICuXbsCAOLj45u0KyURETU+JmhVaBS5cNIaEjRTaXvBr79Ce+MGFDY2sH5kTEuFR0SNLCEhQZrPNHLkSHTs2BGAcULSHMrKyvDNN99g7969Ne7X6/VSJ8S8vDxkZ2c32rW1Wi02btyItWvX1lgxqjpsLvCP0QMRERHIy8szOi4rKwu7du0CAAwYMAAuLn8sV6JUSmuQXbp0STrekKAZKmy2trbo1asXgD+raBUVFbhw4QL++9//IjU1Faamppg+fTrMzat32G0scrncqOrXGHQ6HQ4fPgwA6Ny5c7XmI42VoBm+Tx07dkS7du1gbm5e6/w/IiJqvZigVVFikg9nQ4JmW5mglV2+jIx33wMA2E6bCnkTdPUiouaXmZmJLVu2QAiBoKAg9O/fX0okqjZaaA6JiYm4fv06jhw5goyMjGr7L1++jPz8fOnrxorPMGQxLi4OSUlJNQ6fzMjIQElJCVQqFUJDQ+Hl5QWtVmtUybt48SJWrVqFwsJCODg4YPDgwUbnMFSMDAlPdnY2srKyIJfL0alTJ+k4QxUtJSUF27Ztw2effYYtW7bg2rVrUCgUCAsLg5OTU6Pc+51UTdAaOty1qKgIa9euxcWLFwFAGnpYlZeXF2QyGXJyclBQUHBP1ykpKZEW2fb19YVcLkfnzp0BcJgjEVFbwwStimJR9OcQR1tTVGRn4/rzL0BfUgLzvn3h9OKLLRwhETUGnU6HTZs2SfOZHnnkEchkMilBS01NhUajabZ4DG+sgZrnYBmqZxYWFgAar8J3+PBhaV4UAKPPDQzX8vHxgYmJiTR0MSEhARcuXMD+/fuxefNmlJeXw9vbGzNnzoRSqTQ6R6dOnSCXy5GVlWW0Ppe3tzfU6j+XM7GxsZGqaGfPnkVJSQksLS3x0EMP4eWXX5a+P03N29sbpqamKC4uxvXr1+/5PNevX8eKFStw9epVqFQqTJ06tcYOk2q1usHroSUlJUEIAScnJ9jY2ABArfP/iIiodWMXxypKRKk0xFFuocD1l16G9sYNKNt7weOLzyGr44KnRNS6nTt3DtnZ2TA3N8fkyZOl+Uz29vawt7dHTk4OkpOTpTe4d5KWlgaFQiG9wb4XVRO0uLg4pKenS40jbt26JVXMQkNDpW6TWq22WiJUHxcvXsT+/fsBAH369EFUVBQuX76M0tJSo6TJkKAZkiMnJyc8+OCDOHToEH744Qfpjf8DDzyAESNGVFvrDKhMQLy9vZGUlIS4uDhpqGNNz3fw4MHIyMiAWq1GcHAwOnXqVOM5m5KJiQn8/PwQGxuLuLi4u67NlpeXh+TkZKNqW1FRESIjI6HT6eDg4ICpU6fesfrn4+OD9PR0pKSkGDURqelaOTk58PHxMZoPffv3CUC1+X/u7u53vXciImp5TNCqKEK5lKDlrl2J0jNnILeyQrtly2BiZ9fC0RFRY9DpdDh06BAAYNCgQVJVyqBjx47IyclBYmLiXRO0/Px8rF69Gnq9HiNGjED//v3r3URIp9NJzTPc3d1x48YNHDx4ENOmTQMAnDp1CkBlFapz586wtLREUVERUlNTpTlz9XXjxg1s27YNANC3b1+MHj0aKSkpyMrKwqVLl6QqVllZmVRBqvrGf9CgQTh//jyys7NhYmKCRx999I5JBVCZjCUlJeH06dPSHLqanq+VlRWeeeaZe7qvxuTv7y8laCNGjKj2fdXpdLh8+TKio6PvWNHs3LkzJkyYcNdFr729vXH06NE7NiYpKyvD6tWrUVBQgNGjR6Nfv34AKoeqVp1/ZmCY/3fp0iVcunSJCRoRURvBIY5VaGECK33lm7WCnT8ACgU8Pl8C0ypr9xBR23b27Fnk5eXBwsKixvlAhkQkMTHxrsPCzp8/D51OByEE9uzZgx9++KHawsC3bt3CoUOHam2Pf+vWLWi1WqhUKowfPx4ymQzx8fFIT09HeXk5zpw5A6Aykao6DPNe56EVFRXh+++/h1arRceOHRESEgKZTCYlWFWHORqqQg4ODrCr8kcqpVKJKVOmoHfv3pg1a9ZdkzPgz2TMkJx5eHg0aM2vpubr6wuFQoGcnBxkZWVJ23U6HSIjI7FkyRJs2rRJSs48PT3h5+dn9DFq1ChMmTLlrskZ8Oc8tNzcXKP5hlVFRERIc9R+/fVX6WcqMzMThYWFMDExqbaG3O3z/4iIqPVjBa0Kub7yDYheWwJUlMHlnXdgOXBgC0dFRI3F8OYaAAYOHAhVDcOWvb29IZfLpaFkDg4OtZ7PkMx06tQJV65cwfnz55GZmYmwsDBkZGQgOjpamlOkVCrx5ptvVmsPbxje6O7uDmdnZwQEBCA2NhYHDx5Ep06doNFoYGdnJ1VGfH19ERMTg8TERIwcObLez+DEiRMoLCyEo6MjwsLCpOGD3bt3x/79+5GSkoK8vDzY2tpKyUdNlTpnZ2eMHTu2zte1traGh4eHdL91GT7akkxNTdGhQwckJCQgLi4Ozs7OKC4uxtatW6Uql4WFBXr27IlevXrd8eekLszMzODh4YHr169j9+7dmDx5MuTyP/+GmpaWJi1v4OXlhdTUVGzduhWzZs2Svk/e3t53nf/X0DiJiKjpsYJWhUJULlItSnIBALZhj7VkOETUyGJiYu5YPQMq35gbqhB3Grp28+ZN3Lx5EwqFAhMnTkR4eDgsLCyQmZmJpUuX4ocffkBKSgpkMhkUCgW0Wi1SU1OrnceQsHh4eAConINlqKIZhmL26dNHerPe4Y+KfmZm5j11/DPM/3rooYeMKju2trbSfZ8/fx5CiBrnNTVE1aSstSdowJ8xXrp0CTdu3MDKlSuRnJwMpVKJcePGYe7cuRgxYkSjJT0hISFQKBSIi4vDgQMHpO06nQ7bt28HULnI9ZNPPgkvLy9oNBp8//33UofImr5Phvl/AKtoRERtBRO0Ksz0fyRopTkwcXGB3NT0Lq8gotro9XqUlZUZfWi12ia71t1UVFRI1bNBgwbVWD0zqMt6aFWrZ2q1Gu3bt8dzzz0nJVrW1tYYPHgwXnnlFQQEBNR6vqoVNABwdHRE9+7dAUAathYUFCQdb25uLl2jvt0cb926hVu3bkEul8PPz6/afsNQxbNnzyI7Oxv5+flQKBTSG/yG6tatG5RKJTw8PJqlXX5DGdrUp6enY/Xq1cjPz4e9vT2effZZBAUFNfpi2e3atcOjjz4KoLLD5tmzZwFUVj0NjVNCQkJgYmKCKVOmwNbWFrm5udLPUG1zEg2J5uXLlxs1XiIiahoc4liF3R8t9vWluVB6erZwNERtV3l5OZYvX46cnByj7XK5HI8//rhUBWoMSUlJ2LhxIwIDA/HII4/UetzZs2eRn58PS0vLWqtnBr6+vti3bx9SUlJQUVFR7Y24Xq9HbGwsABjNv7K2tsbTTz+NzMxMODs7S8MHfX19cfbs2WrDErVaLTIzMwH8WUEDKqtbsbGxEEKge/fuRl0VDedLS0vDlStXpIYedWGooPj4+NQ4L6pr167YtWsXsrKycOTIEQBA+/bt75jM1oe9vT1efvnlRjtfU7O0tJSGE1ZUVMDPzw8TJkyo9v1oTIGBgcjKysLhw4fxyy+/QCaTISIiAgAwYsQIqamNhYUFpk+fjm+//RYajQY2NjZwdHSs8Zxdu3aFpaXlPTeVISKi5sUKWhWOWlsAlRU0FRM0ug9kZWU1WdXqTq5evVotOQMqE5vff/+90a5z69YtaQ2uqKioWhtn3F49u1t7ehcXF1haWtY6LPHq1asoKCiAmZmZ0ULLAKBQKODm5mbUGt7wxvj2YYkZGRnQ6/WwsLCQ1q4CKqtoAwYMgKWlJQbWMA+2aqOQ+iykbEjQahteqFarpcqaoTlJY7+pt7a2rlPTjNaib9++MDU1xeDBgzF16tQmTc4MHn74Yfj7+0On0+HHH3+EVqtF+/btjSqpQOU8wLCwMJibm0tNZGpiaWmJrl27wpSjQoiI2gQmaFU4VNgCAPSlOVC2a9eywRA1UFJSEpYuXYodO3Y0+7UNjTECAwPxzjvv4J133sHLL78MmUyG5ORko65496qkpAQbNmxAWVmZlHDt2LGjxoQ0MjJSqp4FBwff9dxVuyXWNIzQMLyxa9eudVqLrOqwxKpJZNX5Z7e/uR4xYgRee+21Gqsi7u7uMDMzQ1lZmdEaandSWFgotcw3DN2rye0dGZtrcejWKiAgAG+99RaGDh1q1LSjKcnlckycOFFaW0+hUCA0NLTGBMzX1xevv/56jYk8ERG1TUzQqnD8I0ETpblQtWMFjdo2QwvuCxcuVGv93tQMCZqPjw9MTExgYmICBwcHqToTFRXVoPPrdDps3rwZOTk5sLGxwQsvvAArKyvk5uZKlTKDc+fOSdtGjhxZ58Wdq85Dq9puX6vVSk0Z6tJevqbzGdzeIKSuFAqFNEy0rvPQ4uPjpWvdqb19p06dpAqXlZUVnJ2d6xXb/ai+a9s1BpVKhWnTpqFz584YO3ZsrcMXgZaJj4iImg4TtCoctIY5aKygUdt348YNAJXD++51zax7UVZWJl379uYSffr0AVDZTVGj0dzT+YUQ2LlzJ1JSUqBSqTB9+nTY29tjzJgxAIAjR47g5s2bAIBr167h559/BlDZVr++CZVMJkNmZiZ++uknqTJ3+fJlaDQaWFtbw8vLq87nq2lY4u0NQuqjvuuhGbo3GtbFqo2JiQm6desmXYNv/luOjY0Npk2bVm1oIxER3d9adYK2aNEi9OnTR/or7vjx46W/AhuUlZVhzpw5cHBwgKWlJSZNmiS9OasvM1E5cV2wSQi1MomJidi+fTvKysrqdLxOp5OSJKB522tfu3YNQgjY2dnB1tbWaF+HDh1gb2+P8vJyowWR6+P48eM4ffo0AGDSpElwcXEBUJl4dO7cGXq9Hjt27EBeXh42btwInU6Hzp07Y9iwYfW6jrm5OUaPHg2ZTIazZ89i9erVyMvLk+Lu0aNHvYa8eXh4wNTUVEpgS0tLpXl69a2gAX9W5NLS0rB27VrpY/369dWqamVlZdLaXXVpbz98+HAMHTq03s+MiIiIGq5VJ2iHDh3CnDlzcPz4cezduxdarRYjR45EcXGxdMzcuXOxfft2bNmyBYcOHcKNGzcwceLEe76mvqwAMqUCJm2gBTT9dfz666+Ijo6uNnyvNrdu3TKaixUfHw+dTtdU4RkxDG+sqTW7XC5H3759AQAnT540GjpYF5cvX8aePXsAVA5XvH0u1ZgxY6BSqXDt2jWsWLECxcXFcHFxwcSJE+9p/lDfvn3xxBNPQK1WIz09HStXrpSGjtanGgdUH5ZoSKDt7Oxgbm5e79hsbGzg7u4OIQSSkpKkj4SEBHz//fe4du2adGxCQgL0ej0cHR3vOFTOQK1WY/DgwbC0tKx3XERERNQwrTpB+/XXX/HUU0+hW7duCAwMxJo1a5Camoro6GgAQH5+Pr799lt89tlnePjhhxEcHIzvvvsOR48exfHjx+/pmqI0B0pPTw7roXqrqKjAunXrsHXr1nonHndSWFiIW7duAahMaoqKiu76GsPQOS8vL5ibm6OsrAxXr15tlHg0Go1Uqampg6ChUlPb2lmBgYFQKpXIysoyiqmiogK7d+/Gl19+WWPFLzMzU3q2QUFB6N+/f7VjbGxs8PDDDwMASktLYWFhgWnTpjWoe12HDh3w3HPPwdXVFSUlJdDr9XB1db2nuVlVG4/c6/yzqqZOnYqJEycaffj6+kKn02Hjxo3Iy8sDcPfujURERNR6tKl10PLz8wFUrqUDANHR0dBqtRg+fLh0jL+/P7y8vHDs2DE88MADNZ5Ho9EYzX+p2vZaX5oLEw+PFmlNTm1bdHS0NLRs4MCBdapU1EVSUpL0eUVFBX7//Xejn/maGKon7u7usLOzw9mzZ3Hx4kW0q8PcSq1WC71eX2NSo9frsWXLFimmuLg4ozbzGo0G6enpAABPT88a/x2ZmJggICAAZ86cwYkTJ+Dh4YHCwkL88MMPUtKyceNGPPjgg3jwwQchk8lQXFyMDRs2oLy8HF5eXggJCUFFRUWN8QcFBeHChQu4efMmHnvsMVhYWDT437OFhQWefPJJ7N69G7GxsQgODr6ncxqS1rS0NKmi5+rqes/xqdXqanPKOnbsiP/+97/IzMzEhg0bMGPGDKnq5+vry99tBAD8OSAiasVkojH/1N+E9Ho9Hn30UeTl5eHw4cMAgA0bNmDmzJnVmg307dsXQ4cOxSeffFLjud5//3188MEH1bZffHU3TK8fR4bVNWSNe7Txb4LuW3q9HhcvXpTe9Hh4eDRa97vU1FRkZ2dDrVajtLQUMpkM3bp1u2M3wri4OJSWlsLb2xtyuRxJSUlQKpXo1q3bHavDRUVFSE5Ohl6vh5eXF+zs7Iz2p6WlSQsrA5Vd/qq2Yc/Pz0dSUhJUKpXUaKImpaWlUlXH29sb169fR0VFBRQKBaytrZGbmwsAUiOO5ORkFBcXQ6VSoXPnztUWjr6dEAJ6vd5oLbLGotPpGnTeS5cuGc0l7NSpU6MPJSwvL0d8fDwqKipgamoKjUZTp+8//XWUlJRg+vTpyM/Pv2NXTyIian5tpoI2Z84cnD9/XkrOGmL+/PmYN2+e9HVBQYFUWdCX5qLT6IHo80dHOKK6OHXqFM6ePSt9bWpqKnUVbKjly5cDAB555BEcPXoUN27cgIWFRa1VtIqKCimWsWPHwtzcHEuWLIFWq0WvXr3g5uZW7TVCCOkeDMMWU1JS4ObmhiFDhkAulyMmJkZavHjo0KGIiIhAYWEhHnjgAamqvX//fiQlJaFLly53vf+1a9fi2rVr0pw1JycnPPbYY7C3t8e5c+ewe/duFBQUIC4uTko0nnrqqUarTLYUpVKJkydPAqhsjz5+/HioVKpGv05aWhr+97//SX/A6tGjB0aNGtXo16G2qerIESIial3aRIL20ksvYceOHYiMjIRnle6Krq6uKC8vR15enlG3uJs3b0oLfNbE1NS01jkpojQHZt7edV4riaiiogJHjx4FUNlGPioqCqmpqQDQ4J+jwsJCZGdnA6gcumZmZoZ169YhOjoagwYNgpWVVbXXZGRkQK/Xw8LCAg4ODtKiy5cuXUJiYmK11vBarRY7d+6UkrqAgABYWVnh2LFjOHbsGDIzM9G7d2/s3r0bADBkyBAMHjwY165dQ2JiIs6cOSO98Tfcd8eOHe9673379pWGYnbr1g2PPvqo9O8yODgYbm5u2LRpE/Lz8yGTyRAWFlZjctnW+Pn5SQmas7MzLCwsmuQ63t7eGDduHH788UcAdV9Um/4a+LNARNR6teoETQiBl19+Gdu2bcPBgwfh4+NjtD84OBhKpRL79+/HpEmTAFR2q0tNTa2xgUBd6EtzoPRgi/3mptVqcfToUbi6ulbrzNfanT59GoWFhbCyssLIkSMRHx+PgoICXL161Wj4370wVJdcXV2hVqvRsWNHeHp64vr16zhy5EiNFZGqzScMw9n8/f1x6dIlxMXFSU00ACAvLw+bNm1Ceno6ZDIZRowYgf79+0Mmk8Hd3R0///wzrly5Iq211a1bNwwePBhAZYKVmJiImJgYPPzww9Dr9dL8s/bt29/13rp164acnBxYWVkhKCio2tA7d3d3zJ49G5GRkfD29m7ws2wt2rdvDxMTE1RUVDSoQUhd9OjRAzqdDrm5udV+fxIREVHr1KoTtDlz5mDDhg34+eefYWVlhYyMDACVndrUajVsbGwwa9YszJs3D/b29rC2tsbLL7+M/v3719og5G5EaS5Unk37pomMVU0STE1N8cYbbzTJ3KGmoNVq8fvvvwMAHnzwQSiVSnTs2BFnzpxBYmJioyVohuYSMpkMQ4YMwbp163Dq1CkMHDiwWhWtpsWP/fz8IJfLkZmZiezsbDg4OCA5ORlbtmxBSUkJ1Go1wsLCpDbwANC9e3c4OTlJ3QDd3d0xfvx4KZHy9fWFnZ0dcnNzERsbCysrKwghYG9vDxsbm7vem1wul5K92lhYWGD06NF3PVdbYvgZiY+Pr1Mi21Bc5JiIiKhtadUJ2rJlywBUDqmq6rvvvsNTTz0FAFiyZAnkcjkmTZoEjUaDkJAQfP311/d0PSH0UKhNIG+iIUdUXdUkAajsAnjt2rVaW7TfTVFREbZv3w5PT08MHDjwnta+qoler8euXbuQn5+PwMBA+Pv7w8TERKqeWVtbo1evXgAqE5czZ85IVaeGMCRoVasfVatohw8frpbAGNbXqlqdUavV8Pb2RlJSEuLi4iCXy7Fnzx4IIeDq6oopU6ZUawgCVFbunnvuOVy+fBl+fn5Gw6Lkcjl69+6NvXv3IioqSorxXr93fyWhoaHo3r07unbt2tKhEBERUSvTqhO0ujSYNDMzw9KlS7F06dKGX68sH6p27nc/kBpMCIFjx45h7969UpJgbm6OpKQkXLly5Z7f5B89ehTx8fHSUNeJEydCrVY3ON6YmBicOnUKQOWiv+bm5ujZsydiY2MBVFbPDJ0FO3ToAJlMhqysLOTn59epmlSTgoICaf5Z1XljMpkMQ4cOxf/+9z+pimbowlZaWiq95vbhc/7+/khKSkJERITUor5Hjx4IDQ2943wUtVqNwMDAGvcFBQUhIiICGRkZ0ppbTNDuztLSEgEBAS0dBhEREbVCrXqh6uYmSnOhrMM6UdRwERERUgUnMDAQs2bNkpIAw1pi9aXVaqUugzKZDAkJCVi1ahVu3rzZoFiLioqwZ88eAJVDBa2srFBSUoKjR49K1bOqw8jUarXUzOZe7wX4s3rm5uZWLcns0KED2rVrB51OhyNHjkjbDdUzOzs7mJubG73GMLevoqICMpkMo0ePxoQJExrULMDc3FxKNAyt45mgEREREd07JmhV6MvyoGzHBiFNLSMjQ5q3FRISgvHjx0vzcgAgPT0dRUVF9T7v+fPnUVpaChsbGzzzzDOwsbFBTk4OvvnmG1y4cOGe4/3tt99QVlYmDQV89dVXMW3aNPj5+cHU1BSjRo2qti6X4V4aI0GrKeExzEUDKlv8G1pmV20QcjsbGxsEBATAxsYG4eHh6NevX6OsidW3b1/pc8NcUCIiIiK6N0zQqqhsEMIKWlPS6/XYvn07hBDo2rWr1DEQqBz2ZVgeob7zt4QQUuvyPn36wMPDA7Nnz4aPjw+0Wi22bNkiVZfq48qVK4iNjYVMJkNoaCgUCgUUCgU6d+6M6dOnY/78+TXOIzI0B0lKSoJOp6v3dYE7J2hAZRXNy8sLOp1OWh/QcI9VG4RU9dhjj+HVV19t1CqXu7u7VDFkp0AiIiKihmGCVoW+NA9KT1bQmlJUVBTS0tKkytPtDIlNfRO0tLQ0pKenQ6FQSMMNLSws8Pjjj0tD+wwJXF2Vl5djx44dACqrRPVpie7u7g61Wg2NRiNVtYDKBPXs2bNISkq64+sLCgqQk5MDmUxWa6e/qlW06OhoFBQU3LGCVvV1jW306NHw8/O75+UtiIiIiKgSE7SqynKh4hDHJlNQUID9+/cDAIYNG1bjUDhDgpaYmAi9Xl/ncxuSr4CAAKOFfxUKBQYNGgQAiI2NlbpF1kVkZCRyc3NhbW1ttHZYXcjlcqllvWGYY1lZGTZt2oRt27Zh7dq12LdvX633WHX+mZmZWa3X8fHxkapou3btQmFhIWQyWbMv6Ozh4YHp06fD0dGxWa9LREREdL9hglZFhSYfJi4uLR3GfWvXrl0oLy+Hp6cnevfuXeMxnp6eUKlUKCkpkda9M9BoNNi5cyeioqKMOnwWFRVJc8yqzoeqek43NzfodDqcPn36rnFqNBqcOnUKR48eBQCMGTMGpqamdb5Pg6rVwKysLKxatQrx8fFS6//Dhw9j/fr1NSaNdxveaGDo6AgAcXFxAABnZ2eoVKp6x0tERERELY8JWhXFphWQtZEFktuauLg4af2t0NDQWtcnMzExkeYx3d5g48CBA4iKisLOnTuxdetWaDQaAMCZM2eg0+ng7u5e49A+mUyGPn36AKhsqFFb1erGjRvYvn07Pv30U+zYsQN6vR5dunSBv7//Pd2zoVFIWloaVq1ahezsbFhbW2PWrFmYOHEiTExMcOXKFaxatapaMpqcnAygbh0Rvb29jYZB1mcoJhERERG1Lq16HbTmprGzbOkQ7juFhYWIiYnBsWPHAAD9+/eHy12qlL6+voiPj8eVK1fw0EMPAahMck6cOAGgcvjghQsXkJWVhcmTJ0vrk9VUPTMICAjAnj17kJeXh4SEBGleGlDZdn7Lli2Ij4+Xttnb2yM4OPiO57wba2truLi44ObNmygvL0f79u0RFhYGS0tLeHh4wNnZGRs3bkRubi5WrFhh1AlSq9VCJpMZrX9WG8NctP/+978AmKARERERtWVM0KqQuXH+WWMQQuDKlSuIjo5GfHy8VLFycnLC4MGD7/p6Q+Xp2rVrKCsrg1KpxPbt2wEA3bt3R58+fbB582ZkZmZi2bJl0Ol0UKvV6NatW63nVKlUCAoKwrFjxxAVFSUlaEII7Nq1Sxp62LVrVwQHB8Pb27tRmmkEBgZiz5496NevH0aOHAlFlQqtq6srZs+ejR9//BGJiYnQarVGr+3UqdMd559V5ePjgy5duuDKlSvS8yMiIiKitocJWhVmHt4tHcJ94eTJk9i9e7f0taenJ4KDg9GtW7c6zY2yt7eHvb09cnJykJycjNzcXGRkZECtViMkJASWlpaYPXs2Nm/ejOvXrwMAevXqddcFl/v06YNjx44hMTER2dnZcHBwwPHjx3H69GnIZDJMnToVfn5+Dbv52wwYMADBwcG1zmEzNzfH448/joKCAqN2/DKZrN7riYWFhUGv11dbk42IiIiI2g6+k6vCsg7DydqK/Px85OTktMi6VNHR0QAqq12DBg2665DGmvj6+uLkyZM4ffq01DBjxIgRsLSsHIZqbW2Np556Cvv370dGRgYeeOCBu57T3t4enTp1QkJCAqKiotChQwfs2bMHADBy5MhGT84M6tJgpDEWd5bL5bXO7SMiIiKitoEJWhWW3vdHgqbX67Fu3TpkZWXh8ccfl7oJNofs7GxkZmZCLpdjzJgxUKvV93Sejh074uTJk0hISAAAtG/fXlrfzMDExAQhISH1Om+fPn2QkJCAM2fO4PTp0xBCoFevXnVK8IiIiIiImhr/3F6FbYf7I0FLSkpCVlYWAOD48ePNem1Dq3dvb+97Ts4MrzfM11IoFBg7dmyjzAnz9fWFnZ0dNBqN1LhjzJgxTbJ4MxERERFRfTFBq8LM1qalQ2gUUVFR0ueJiYnIyclptmsbErR7bU1vYGpqKg3PHDRoEJycnBocG1A5DNDQct/Ozg5TpkzhnC0iIiIiajX4zvQ+k5eXh8uXLwOoXLA4MzMTUVFR9R4KWJuUlBTs27cPDz/8MDp06GC0r7CwENeuXQMAozb29+rRRx9Famoqunbt2uBzVdWvXz+Ym5ujY8eOMDc3b9RzExERERE1BCtorYwQotaFlOvi1KlTEELAx8cHw4cPB1C5kHN5eXmdXl+1k+Dtbt26hY0bN+L69evYsWNHtWMNiaG7uztsbBpejbS2tkZAQECjN75QKBTo2bMnrKysGvW8REREREQNxQStlSgrK8PJkyexfPlyfPzxx0hMTKz3ObRaLU6fPg2gctFmX19f2NraoqysDOfPn7/r65OTk7Fw4UL88MMP1RK6kpISbNiwAWVlZQCAnJycaue8dOkSAKBLly71jp2IiIiIiJigtbj09HT89NNP+Pe//41du3bh5s2b0Ol0+OWXX6DRaOp1rosXL6KkpATW1tbw8/Mzmm918uRJCCHu+Pro6GjodDrExsbim2++keau6XQ6bNmyBTk5ObCxsUH//v0BAIcOHZKqaGVlZUhOTgbQ8PlnRERERER/VUzQWtCNGzewatUqxMTEoKKiAk5OTggJCYGtrS0KCgoQERFRr/OdPHkSANC7d2+pA2JQUBBMTEyQkZEhLepck4qKCqmlvUqlQmZmJlauXImEhATs2rULycnJUKlUmDZtGoYMGQK1Wo2cnBzExsYCqGxGotPp4ODgAEdHx3t5HEREREREf3lM0FqIXq/H9u3bodfr0b59ezz99NN48cUX0b9/f4wdOxYAcOLECdy4caNO50tLS0NaWhoUCgV69eolbTc3N0dAQACAPxO4mqSkpECj0cDS0hJz5syBh4cHysrKsH79emnh6UmTJsHV1RWmpqYYOHAgACAyMhI6nc6oeyNb1hMRERER3RsmaC3k5MmTSE9Ph5mZGR577DF4eXlJiY2vry8CAgIghMD27dvv2LjDwNBav2vXrrC0tDTa17dvXwDAhQsXUFRUVOPrDfPHOnfuDBsbG8ycORPBwcHS/hEjRhh1ZuzTpw/Mzc2Rk5ODM2fOSA1COLyRiIiIiOjeMUGrwZEjR7Bt2zYkJiY2qKNibfLz83HgwAEAwPDhw2vsJjhq1CiYmZkhPT39jpUvoLKBh6FhhyEZq8rd3R0eHh7Q6/VSNawqvV6P+Ph4AH8mWCYmJggNDcWUKVMwfvx4DBgwwOg1Vatov/76K8rLy2FpaQkPD4+73T4REREREdWCCdptTp48ib179+Ls2bNYt24dvvzyS0RGRqKwsLBRzi+EwK5du1BeXo527doZDUesytLSEiNGjAAAHDhwAHl5ebWe88yZM6ioqICrqys8PT1rPMaQuJ04caJah8a0tDQUFRVBpVJJi0MbdOnSBT179qxx2KKhilZRUQGgMrlr7Jb4RERERER/JXw3XUVSUhJ2794NAOjYsSNMTU2Rl5eHAwcO4LPPPpNa2DfEpUuXEB8fD7lcjtDQ0DsmNEFBQfDy8oJWq8Wvv/5a4zF6vV4a3ti3b99a538FBATAzs4OJSUl0vEGhvljfn5+MDGp+9rlKpVKqqIBHN5IRERERNRQTNCq2LZtG4QQCAwMxOOPP46//e1vGD9+PDw9PSGEwO7du5Gbm1vr6/Py8u44JLKsrExKAAcOHAhnZ+c7xiOXyzF27FjIZDLExcXV2IUxISEBeXl5MDMzk5qB1EShUOChhx4CUDmE01BFE0JI88/uJcHq06cPHB0d4eDgAG9v73q/noiIiIiI/sQErQrDsMPQ0FDIZDKoVCr07NkTs2bNQvv27aHVarFr164a1xP77bff8Pnnn9c4JDIvLw8RERH4+uuvUVhYCHt7eylZuhtnZ2f06NEDQOW6Y7czVMOCgoKgUqnueK4ePXpIVTTDvLZbt24hJycHCoUCvr6+dYqpKpVKhRdeeAFz5sypV/WNiIiIiIiqa/UJWmRkJEJDQ+Hu7g6ZTIaffvrJaL8QAu+99x7c3NygVqsxfPhwaT2v+rKxscGUKVOqJRoymQxjx46FXC5HQkICLl68aLT/1KlTOHbsGAAYDYncuHEj1q9fjy+++AKHDh1CQUEBzM3NMX78eCiVyjrH9dBDD0EmkyEhIcGoipadnY3ExEQAkBakvhOFQoHBgwcDAI4ePQqNRiNVz3x8fGBmZlbnmG4/L+eeERERERE1XKt/V11cXIzAwEAsXbq0xv3//Oc/8eWXX2L58uU4ceIELCwsEBISgrKysnpf67HHHqvWot7AyckJDz74IABg9+7dKC0tBQAkJydj165dACoTqfHjx6Ndu3YQQiAuLg4JCQkQQsDb2xuTJk3CvHnz4OXlVa+4HBwcEBgYCAA4ePCgtN1QPfP19YW9vX2dztW9e3fY29tLc9Gqrl9GREREREQtq9WPSRs9ejRGjx5d4z4hBD7//HO88847GDduHABg7dq1cHFxwU8//YSpU6fW61p3mxM2aNAgnD9/HtnZ2di/fz/69++PTZs2Qa/XIyAgAEOHDoVMJkPPnj2RmZmJs2fPQi6XIzAwEI6OjvWK5XYPPfQQzp49i8TERFy7dg0uLi6IiYkBUHNr/doY5qL99NNP+P3336HRaADAaI0zIiIiIiJqGa0+QbuT5ORkZGRkYPjw4dI2Gxsb9OvXD8eOHas1QdNoNFJiAgAFBQUAAK1WC61We8drjh49GuvWrcOpU6dw+fJllJWVwd3dHWPGjJHazQOAnZ0dhgwZIn19t/PejZWVFXr06IGzZ88iIiIC/v7+KCsrg62tLby9vet1/i5duiAyMhI5OTkAAE9PT5iZmTU4RiIiahv4+56IqPVq0wlaRkYGAMDFxcVou4uLi7SvJosWLcIHH3xQbfuePXtgbm5+1+va29sjJycHBQUFUCqVsLe3x969e+sZff0Z/kNNSkpCamoqAMDc3FzqDFkfVlZWUoKm1+ulYZpERHT/KykpaekQiIioFm06QbtX8+fPx7x586SvCwoK0K5dO4wcORLW1tZ3fX1JSQlWrVqF8vJyPPHEE3B1dW3KcI0olUqcPXsWFRUVMDExwdSpU6FWq+t9Hr1ej9WrVyMnJwcTJkyAra1t4wdLREStkmHkCBERtT5tOkEzJEY3b96Em5ubtP3mzZvo2bNnra8zNTWFqalpte1KpbJO3RVtbGzw4osvQq/X19pUpKkMGTIEsbGx0Ov16N69e50Syto8/fTT0Gq1zX4PRETUsurTSZiIiJpXq+/ieCc+Pj5wdXXF/v37pW0FBQU4ceIE+vfv36TXNjc3b5HExs7ODg8++CCsra0xcODABp3L1NSUyRkRERERUSvS6itoRUVF0lpfQGVjkJiYGNjb28PLywuvvvoqPvroI3Tq1Ak+Pj5499134e7ujvHjx7dc0E1s6NChGDp0aEuHQUREREREjazVJ2inTp0ySkYMc8fCw8OxZs0avPHGGyguLsbs2bORl5eHQYMG4ddff73nRZeJiIiIiIhaikwIIVo6iJZWUFAAGxsb5OfnN2hOFxERUVvA//eIiFqvNj0HjYiIiIiI6H7CBI2IiIiIiKiVYIJGRERERETUSjBBIyIiIiIiaiWYoBEREREREbUSTNCIiIiIiIhaCSZoRERERERErQQTNCIiIiIiolaCCRoREREREVErwQSNiIiIiIiolWCCRkRERERE1EowQSMiIiIiImolmKARERERERG1EkzQiIiIiIiIWgkmaERERERERK0EEzQiIiIiIqJWggkaERERERFRK8EEjYiIiIiIqJVggkZERERERNRKMEEjIiIiIiJqJZigERERERERtRJM0IiIiIiIiFoJJmhEREREREStBBM0IiIiIiKiVoIJGhERERERUSvBBI2IiIiIiKiVuG8StKVLl8Lb2xtmZmbo168fTp482dIhERERERER1ct9kaBt2rQJ8+bNw4IFC3D69GkEBgYiJCQEmZmZLR0aERERERFRnd0XCdpnn32GZ599FjNnzkTXrl2xfPlymJubY/Xq1S0dGhERERERUZ2ZtHQADVVeXo7o6GjMnz9f2iaXyzF8+HAcO3asxtdoNBpoNBrp6/z8fABATk4OtFpt0wZMRETUwgoLCwEAQogWjoSIiG7X5hO0W7duQafTwcXFxWi7i4sL4uLianzNokWL8MEHH1Tb7uPj0yQxEhERtUbZ2dmwsbFp6TCIiKiKNp+g3Yv58+dj3rx50td5eXlo3749UlNT+R9VEykoKEC7du1w7do1WFtbt3Q49yU+46bHZ9w8+JybXn5+Pry8vGBvb9/SoRAR0W3afILm6OgIhUKBmzdvGm2/efMmXF1da3yNqakpTE1Nq223sbHhm4EmZm1tzWfcxPiMmx6fcfPgc256cvl9MRWdiOi+0uZ/M6tUKgQHB2P//v3SNr1ej/3796N///4tGBkREREREVH9tPkKGgDMmzcP4eHh6N27N/r27YvPP/8cxcXFmDlzZkuHRkREREREVGf3RYI2ZcoUZGVl4b333kNGRgZ69uyJX3/9tVrjkNqYmppiwYIFNQ57pMbBZ9z0+IybHp9x8+Bzbnp8xkRErZdMsMcuERERERFRq9Dm56ARERERERHdL5igERERERERtRJM0IiIiIiIiFoJJmhEREREREStxF8+QVu6dCm8vb1hZmaGfv364eTJky0dUpu1aNEi9OnTB1ZWVnB2dsb48eMRHx9vdExZWRnmzJkDBwcHWFpaYtKkSdUWGae6W7x4MWQyGV599VVpG59x40hLS8Pjjz8OBwcHqNVqdO/eHadOnZL2CyHw3nvvwc3NDWq1GsOHD0dCQkILRty26HQ6vPvuu/Dx8YFarUbHjh3x4YcfomrfKj7j+omMjERoaCjc3d0hk8nw008/Ge2vy/PMycnBjBkzYG1tDVtbW8yaNQtFRUXNeBdERPSXTtA2bdqEefPmYcGCBTh9+jQCAwMREhKCzMzMlg6tTTp06BDmzJmD48ePY+/evdBqtRg5ciSKi4ulY+bOnYvt27djy5YtOHToEG7cuIGJEye2YNRtV1RUFFasWIEePXoYbeczbrjc3FwMHDgQSqUSu3fvxsWLF/Hpp5/Czs5OOuaf//wnvvzySyxfvhwnTpyAhYUFQkJCUFZW1oKRtx2ffPIJli1bhv/85z+4dOkSPvnkE/zzn//EV199JR3DZ1w/xcXFCAwMxNKlS2vcX5fnOWPGDFy4cAF79+7Fjh07EBkZidmzZzfXLRAREQCIv7C+ffuKOXPmSF/rdDrh7u4uFi1a1IJR3T8yMzMFAHHo0CEhhBB5eXlCqVSKLVu2SMdcunRJABDHjh1rqTDbpMLCQtGpUyexd+9eMXjwYPHKK68IIfiMG8ubb74pBg0aVOt+vV4vXF1dxb/+9S9pW15enjA1NRXff/99c4TY5j3yyCPi6aefNto2ceJEMWPGDCEEn3FDARDbtm2Tvq7L87x48aIAIKKioqRjdu/eLWQymUhLS2u22ImI/ur+shW08vJyREdHY/jw4dI2uVyO4cOH49ixYy0Y2f0jPz8fAGBvbw8AiI6OhlarNXrm/v7+8PLy4jOvpzlz5uCRRx4xepYAn3Fj+eWXX9C7d2+EhYXB2dkZQUFBWLVqlbQ/OTkZGRkZRs/ZxsYG/fr143OuowEDBmD//v24fPkyAODs2bM4fPgwRo8eDYDPuLHV5XkeO3YMtra26N27t3TM8OHDIZfLceLEiWaPmYjor8qkpQNoKbdu3YJOp4OLi4vRdhcXF8TFxbVQVPcPvV6PV199FQMHDkRAQAAAICMjAyqVCra2tkbHuri4ICMjowWibJs2btyI06dPIyoqqto+PuPGkZSUhGXLlmHevHl4++23ERUVhf/7v/+DSqVCeHi49Cxr+v3B51w3b731FgoKCuDv7w+FQgGdToePP/4YM2bMAAA+40ZWl+eZkZEBZ2dno/0mJiawt7fnMyciakZ/2QSNmtacOXNw/vx5HD58uKVDua9cu3YNr7zyCvbu3QszM7OWDue+pdfr0bt3byxcuBAAEBQUhPPnz2P58uUIDw9v4ejuD5s3b8b69euxYcMGdOvWDTExMXj11Vfh7u7OZ0xERH9pf9khjo6OjlAoFNW62928eROurq4tFNX94aWXXsKOHTsQEREBT09PaburqyvKy8uRl5dndDyfed1FR0cjMzMTvXr1gomJCUxMTHDo0CF8+eWXMDExgYuLC59xI3Bzc0PXrl2NtnXp0gWpqakAID1L/v64d6+//jreeustTJ06Fd27d8cTTzyBuXPnYtGiRQD4jBtbXZ6nq6trtSZZFRUVyMnJ4TMnImpGf9kETaVSITg4GPv375e26fV67N+/H/3792/ByNouIQReeuklbNu2DQcOHICPj4/R/uDgYCiVSqNnHh8fj9TUVD7zOho2bBhiY2MRExMjffTu3RszZsyQPuczbriBAwdWWyLi8uXLaN++PQDAx8cHrq6uRs+5oKAAJ06c4HOuo5KSEsjlxv8FKRQK6PV6AHzGja0uz7N///7Iy8tDdHS0dMyBAweg1+vRr1+/Zo+ZiOgvq6W7lLSkjRs3ClNTU7FmzRpx8eJFMXv2bGFraysyMjJaOrQ26YUXXhA2Njbi4MGDIj09XfooKSmRjnn++eeFl5eXOHDggDh16pTo37+/6N+/fwtG3fZV7eIoBJ9xYzh58qQwMTERH3/8sUhISBDr168X5ubmYt26ddIxixcvFra2tuLnn38W586dE+PGjRM+Pj6itLS0BSNvO8LDw4WHh4fYsWOHSE5OFj/++KNwdHQUb7zxhnQMn3H9FBYWijNnzogzZ84IAOKzzz4TZ86cEVevXhVC1O15jho1SgQFBYkTJ06Iw4cPi06dOolp06a11C0REf0l/aUTNCGE+Oqrr4SXl5dQqVSib9++4vjx4y0dUpsFoMaP7777TjqmtLRUvPjii8LOzk6Ym5uLCRMmiPT09JYL+j5we4LGZ9w4tm/fLgICAoSpqanw9/cXK1euNNqv1+vFu+++K1xcXISpqakYNmyYiI+Pb6Fo256CggLxyiuvCC8vL2FmZiY6dOgg/v73vwuNRiMdw2dcPxERETX+Dg4PDxdC1O15Zmdni2nTpglLS0thbW0tZs6cKQoLC1vgboiI/rpkQgjRMrU7IiIiIiIiquovOweNiIiIiIiotWGCRkRERERE1EowQSMiIiIiImolmKARERERERG1EkzQiIiIiIiIWgkmaERERERERK0EEzQiIiIiIqJWggkaEbUpBw8ehEwmQ15eXkuHQkRERNTomKARERERERG1EkzQiIiIiIiIWgkmaERUL3q9HosWLYKPjw/UajUCAwOxdetWAH8OP9y5cyd69OgBMzMzPPDAAzh//rzROX744Qd069YNpqam8Pb2xqeffmq0X6PR4M0330S7du1gamoKX19ffPvtt0bHREdHo3fv3jA3N8eAAQMQHx/ftDdORERE1AyYoBFRvSxatAhr167F8uXLceHCBcydOxePP/44Dh06JB3z+uuv49NPP0VUVBScnJwQGhoKrVYLoDKxmjx5MqZOnYrY2Fi8//77ePfdd7FmzRrp9U8++SS+//57fPnll7h06RJWrFgBS0tLozj+/ve/49NPP8WpU6dgYmKCp59+ulnun4iIiKgpyYQQoqWDIKK2QaPRwN7eHvv27UP//v2l7c888wxKSkowe/ZsDB06FBs3bsSUKVMAADk5OfD09MSaNWswefJkzJgxA1lZWdizZ4/0+jfeeAM7d+7EhQsXcPnyZXTu3Bl79+7F8OHDq8Vw8OBBDB06FPv27cOwYcMAALt27cIjjzyC0tJSmJmZNfFTICIiImo6rKARUZ0lJiaipKQEI0aMgKWlpfSxdu1aXLlyRTquavJmb2+Pzp0749KlSwCAS5cuYeDAgUbnHThwIBISEqDT6RATEwOFQoHBgwffMZYePXpIn7u5uQEAMjMzG3yPRERERC3JpKUDIKK2o6ioCACwc+dOeHh4GO0zNTU1StLulVqtrtNxSqVS+lwmkwGonB9HRERE1JaxgkZEdda1a1eYmpoiNTUVvr6+Rh/t2rWTjjt+/Lj0eW5uLi5fvowuXboAALp06YIjR44YnffIkSPw8/ODQqFA9+7dodfrjea0EREREf1VsIJGRHVmZWWF1157DXPnzoVer8egQYOQn5+PI0eOwNraGu3btwcA/OMf/4CDgwNcXFzw97//HY6Ojhg/fjwA4G9/+xv69OmDDz/8EFOmTMGxY8fwn//8B19//TUAwNvbG+Hh4Xj66afx5ZdfIjAwEFevXkVmZiYmT57cUrdORERE1CyYoBFRvXz44YdwcnLCokWLkJSUBFtbW/Tq1Qtvv/22NMRw8eLFeOWVV5CQkICePXti+/btUKlUAIBevXph8+bNeO+99/Dhhx/Czc0N//jHP/DUU09J11i2bBnefvttvPjii8jOzoaXlxfefvvtlrhdIiIiombFLo5E1GgMHRZzc3Nha2vb0uEQERERtTmcg0ZERERERNRKMEEjIiIiIiJqJTjEkYiIiIiIqJVgBY2IiIiIiKiVYIJGRERERETUSjBBIyIiIiIiaiWYoBEREREREbUSTNCIiIiIiIhaCSZoRERERERErQQTNCIiIiIiolaCCRoREREREVErwQSNiIiIiIiolfh/nsuUycA/vjoAAAAASUVORK5CYII=", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "fig, ax = plt.subplots()\n", - "\n", - "for label in list(sorted_labels):\n", - " _ = np.round(architectures_TM[label]['epochs_acc'][-1], 2)\n", - " nb_skip = nb_skip_conns[label]['nb_skip_conns']\n", - " nb_skiped_l = nb_skip_conns[label]['nb_jumped_layer']\n", - " txt = f'{label} ({_}% - # sc: {nb_skip}, # sl: {nb_skiped_l})'\n", - " ax.plot(np.arange(len(architectures_TM[label]['epochs_x'])), architectures_TM[label]['epochs_acc'], label=txt)\n", - "\n", - "ax.set_ylim(0, 100)\n", - "ax.set_yticks(np.arange(0, 110, 10))\n", - "ax.set_ylabel('validation acc [%]')\n", - "ax.set_xlim(0, 100)\n", - "ax.set_xlabel('epoch')\n", - "\n", - "pos = ax.get_position()\n", - "ax.set_position([pos.x0, pos.y0, pos.width * 0.9, pos.height])\n", - "ax.legend(loc='center right', bbox_to_anchor=(1.8, 0.5), framealpha=0)\n", - "ax.grid(axis='y')\n", - "\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAArUAAAG2CAYAAABh3H5yAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAADUb0lEQVR4nOzdeVzU1f748dewgygu7K4ogoqACC6YSa5oZpimpmlYlunX6rpvv3BJRbOuUuoVLTS9WS6ZSy6J5p6KKyqgIoiYCpgbu6yf3x/EXEZ2HATr/Xw8Pg+bc87nnPMZ7THvOXMWlaIoCkIIIYQQQrzAdKq6A0IIIYQQQjwrCWqFEEIIIcQLT4JaIYQQQgjxwpOgVgghhBBCvPAkqBVCCCGEEC88CWqFEEIIIcQLT4JaIYQQQgjxwpOgVgghhBBCvPAkqBVCCCGEEC88CWqFEEIIIcQLr0qD2qNHj9KvXz9sbW1RqVRs375dI19RFGbNmoWNjQ3Gxsb06NGD69eva5R5+PAhb7/9NrVq1aJ27dqMGjWKlJSU5/gUQgghhBCiqlVpUJuamoqrqysrVqwoMn/x4sV8/fXXBAYGEhISQo0aNfD29ubJkyfqMm+//Tbh4eHs37+fXbt2cfToUUaPHv28HkEIIYQQQlQDKkVRlKruBIBKpWLbtm30798fyBultbW1ZdKkSUyePBmAxMRErKys+O6773jrrbe4cuUKrVq14syZM3h4eADw66+/8uqrr3L79m1sbW2r6nGEEEIIIcRzpFfVHShOTEwM8fHx9OjRQ51mZmZGhw4dOHnyJG+99RYnT56kdu3a6oAWoEePHujo6BASEsIbb7xRZN0ZGRlkZGSoX+fm5vLw4UPq1auHSqWqvIcSQgghqgFFUUhOTsbW1hYdHVleI/4eqm1QGx8fD4CVlZVGupWVlTovPj4eS0tLjXw9PT3q1q2rLlOUhQsXMnfuXC33WAghhHix/PHHHzRo0KCquyGEVlTboLYyzZgxg4kTJ6pfJyYm0qhRI2JiYqhZs2YV9kwIIYSofMnJydjZ2clnnvhbqbZBrbW1NQAJCQnY2Nio0xMSEmjTpo26zL179zTuy87O5uHDh+r7i2JoaIihoWGh9Lp161KrVi0t9F4IIYSovvT19QFkyp34W6m2E2ns7Oywtrbmt99+U6clJSUREhKCp6cnAJ6enjx+/Jhz586pyxw8eJDc3Fw6dOjw3PsshBBCCCGqRpWO1KakpBAVFaV+HRMTQ2hoKHXr1qVRo0aMHz+e+fPn07x5c+zs7PDz88PW1la9Q0LLli3p3bs3H3zwAYGBgWRlZfHRRx/x1ltvyc4HQgghhBD/IFUa1J49e5auXbuqX+fPc/X19eW7775j6tSppKamMnr0aB4/fkznzp359ddfMTIyUt+zYcMGPvroI7p3746Ojg4DBw7k66+/fu7PIoQQQgghqk612ae2KiUlJWFmZkZiYqLMqRVCCPG3J5974u+o2s6pFUIIIYQQoqwkqBVCCCGEEC88CWqFEEIIIcQLT4JaIYQQQgjxwpOgVgghhBBCvPAkqBVCCCGEEC88CWqFEEIIIcQLT4JaIYQQQgjxwpOgVgghhBBCvPAkqBVCCCGEEC88CWqFEEIIIcQLT4JaIYQQQgjxwpOgVgghhBBCvPAkqBVCCCGEEC88CWqFEEIIIcQLT4JaIYQQQgjxwpOgVgghhBBCvPAkqBVCCCGEEC88CWqFEEIIIcQLT4JaIYQQQgjxwpOgVgghhBBCvPAkqBVCCCGEEC88CWqFEEIIIcQLT4JaIYQQQgjxwpOgVgghhBBCvPAkqBVCCCGEEC+8ah/UJicnM378eBo3boyxsTGdOnXizJkz6nxFUZg1axY2NjYYGxvTo0cPrl+/XoU9FkIIIYQQz1u1D2rff/999u/fz3//+18uX75Mr1696NGjB3fu3AFg8eLFfP311wQGBhISEkKNGjXw9vbmyZMnVdxzIYQQQgjxvKgURVGquhPFSU9Pp2bNmuzYsYO+ffuq093d3enTpw/z5s3D1taWSZMmMXnyZAASExOxsrLiu+++46233ipTO0lJSZiZmZGYmEitWrUq5VmEEEKI6kI+98TfkV5Vd6Ak2dnZ5OTkYGRkpJFubGzM8ePHiYmJIT4+nh49eqjzzMzM6NChAydPniw2qM3IyCAjI0P9OikpCYCsrCyysrIq4UmEEEKI6kM+68TfUbUOamvWrImnpyfz5s2jZcuWWFlZ8eOPP3Ly5Ens7e2Jj48HwMrKSuM+KysrdV5RFi5cyNy5cwul79q1i9q1a2v1GYQQQojqJi0traq7IITWVeugFuC///0v7733HvXr10dXV5e2bdsydOhQzp07V+E6Z8yYwcSJE9Wvk5KSaNiwIVevXmXAgAG0adNGCz0XQgghqqf8XyiF+Dup9kFts2bNOHLkCKmpqSQlJWFjY8OQIUNo2rQp1tbWACQkJGBjY6O+JyEhocTA1NDQEENDwyLz9uzZg4ODA2ZmZlp9DiGEEKK60NfXr+ouCKF11X73g3w1atTAxsaGR48esW/fPnx8fLCzs8Pa2prffvtNXS4pKYmQkBA8PT0r1I6iKDx8+FBb3RZCCCGEEM9BtR+p3bdvH4qi4OjoSFRUFFOmTKFFixa8++67qFQqxo8fz/z582nevDl2dnb4+flha2tL//79K9SeSqWibt262n0IIYQQQghRqap9UJuYmMiMGTO4ffs2devWZeDAgSxYsED908nUqVNJTU1l9OjRPH78mM6dO/Prr78W2jGhrPr16ydTD4QQQgghXjDVep/a5yV/v77p06fz6aefUqNGjarukhBCCFFpZJ9a8Xf0wsypfV7CwsKqugtCCCGEEKKcJKh9ysWLF6u6C0IIIYQQopwkqH3K3bt3uX//flV3QwghhBBClIMEtQU0bdoUgEuXLlVxT4QQQgghRHlIUFtA69atgbygNjc3t4p7I4QQQgghykqC2gIcHBwwMDDg8ePH/PHHH1XdHSGEEEIIUUYS1Bagr69Pq1atAJmCIIQQQgjxIpGg9ikuLi4AhIeHk5WVVcW9EUIIIYQQZSFB7VOaNGlCrVq1ePLkCdevX6/q7gghhBBCiDKQoPYpOjo6ODs7AzIFQQghhBDiRSFBbRHypyBERkaSlpZWxb0RQgghhBClkaC2CFZWVlhbW5Obm0t4eHhVd0cIIYQQQpRCgtpi5I/WyhQEIYQQQojqT4LaYjg7O6NSqfjjjz94+PBhVXdHCCGEEEKUQILaYtSsWVOOzRVCCCGEeEFIUFuCglMQFEWp4t4IIYQQQojiSFBbghYtWqCvr8/Dhw+5fft2VXdHCCGEEEIUQ4LaEhgaGtKyZUtApiAIIYQQQlRnEtSWIn8KQlhYGNnZ2VXcGyGEEEIIURQJakthZ2eHqakp6enpREVFVXV3hBBCCCFEESSoLYWurq4cmyuEEEIIUc1JUFsG+VMQrl27Rnp6ehX3RgghhBBCPE2C2jKwtrbGwsKCnJwcIiIiqro7QgghhBDiKRLUloFKpcLV1RWQKQhCCCGEENWRBLVllD+vNjY2lsePH1dtZ4QQQgghhAYJasvIzMyMJk2aADJaK4QQQghR3VTroDYnJwc/Pz/s7OwwNjamWbNmzJs3T+PIWkVRmDVrFjY2NhgbG9OjRw+uX79eKf0pOAVBjs0VQgghhKg+qnVQ+/nnn7Ny5UqWL1/OlStX+Pzzz1m8eDHLli1Tl1m8eDFff/01gYGBhISEUKNGDby9vXny5InW+9OyZUv09PS4f/8+cXFxWq9fCCGEEEJUTLUOak+cOIGPjw99+/alSZMmvPnmm/Tq1YvTp08DeaO0AQEBfPrpp/j4+ODi4sL69eu5e/cu27dv13p/jIyMcHR0BODixYtar18IIYQQQlSMXlV3oCSdOnVi9erVREZG4uDgwMWLFzl+/DhLliwBICYmhvj4eHr06KG+x8zMjA4dOnDy5EneeuutIuvNyMggIyND/TopKQmArKwssrKySuyTk5MT4eHhhIWF0a1bN3R0qvX3AiGEEKKQ0j7rhHgRVeugdvr06SQlJdGiRQt0dXXJyclhwYIFvP322wDEx8cDYGVlpXGflZWVOq8oCxcuZO7cuYXSg4ODMTExKbFPiqKgp6dHamoqmzZtwszMrLyPJYQQQlSptLS0qu6CEFpXrYPazZs3s2HDBn744QecnJwIDQ1l/Pjx2Nra4uvrW+F6Z8yYwcSJE9Wvk5KSaNiwIb169aJWrVql3q+np8fZs2cxNjbm1VdfrXA/hBBCiKqQ/wulEH8n1TqonTJlCtOnT1dPI3B2diY2NpaFCxfi6+uLtbU1AAkJCdjY2KjvS0hIoE2bNsXWa2hoiKGhYaF0fX199PX1S+2Xm5sbZ8+e5dq1a+Tm5hZZlxBCCFFdleWzTogXTbWeEJqWllZozqquri65ubkA2NnZYW1tzW+//abOT0pKIiQkBE9Pz0rrl62tLfXq1SM7O5srV65UWjtCCCGEEKJsqnVQ269fPxYsWMDu3bu5efMm27ZtY8mSJbzxxhtA3vG148ePZ/78+ezcuZPLly/zzjvvYGtrS//+/SutXyqVChcXF0B2QRBCCCGEqA6q9fSDZcuW4efnx//93/9x7949bG1t+fDDD5k1a5a6zNSpU0lNTWX06NE8fvyYzp078+uvv2JkZFSpfXNxceHQoUPExMSQmJgoC8aEEEIIIaqQSpGjsUhKSsLMzIzExMQyLRTLt2bNGm7dukXPnj156aWXKrGHQgghhPZU9HNPiOqsWk8/qO5kCoIQQgghRPUgQe0zcHJyQldXl3v37pW4L64QQgghhKhcEtQ+A2NjYxwcHAC4dOlSFfdGCCGEEOKfS4LaZ5Q/BeHy5cvqrcaEEEIIIcTzJUHtM2revDnGxsYkJycTExNT1d0RQgghhPhHkqD2Genp6eHk5ATIFAQhhBBCiKoiQa0W5E9BiIiIIDMzs4p7I4QQQgjxzyNBrRY0bNiQOnXqkJWVxdWrV6u6O0IIIYQQ/zgS1GpBwWNzZQqCEEIIIcTzJ0GtluQHtdHR0SQnJ1dxb4QQQggh/lkkqNWSevXq0aBBAxRFISwsrKq7I4QQQgjxjyJBrRbJFAQhhBBCiKohQa0WOTk5oaOjQ1xcHPfu3avq7gghhBBC/GNIUKtFNWrUoHnz5oCM1gohhBBCPE8S1GpZwSkIcmyuEEIIIcTzIUGtljk4OGBoaEhSUhKxsbFV3R0hhBBCiH8ECWq1TF9fX47NFUIIIYR4ziSorQQFj83Nysqq4t4IIYQQQvz9SVBbCRo1aoSZmRkZGRlcu3atqrsjhBBCCPG3J0FtJdDR0ZE9a4UQQgghniMJaitJflAbFRVFampqFfdGCCGEEOLvTYLaSmJhYYGtrS25ublybK4QQgghRCWToLYSyRQEIYQQQojnQ4LaApIfPtBqfa1bt0alUnHnzh3u37+v1bqFEEIIIcT/SFBbwLpJY7l8MFhr9ZmammJvbw/IaK0QQgghql5mZib29vacOHGiqrtCYGAg/fr101p9EtQWoCgK+79ZTvID7Y2qFpyCoCiK1uoVQggh/slGjhyJSqVCpVKhr6+PnZ0dU6dO5cmTJ1pr48iRI3Tr1o26detiYmJC8+bN8fX1JTMzU11GURRWr15Nhw4dMDU1pXbt2nh4eBAQEEBaWhoAc+bMQaVSMWbMGI36Q0NDUalU3Lx5E4CbN2+iUqmwtLQkOTlZo2ybNm2YM2eO+vXPP/9Mr169qFevHiqVitDQ0DI9U2BgIHZ2dnTq1AmAw4cPq9/Hp68zZ85o9Ovp69SpUyW2devWLfr27YuJiQmWlpZMmTKF7Oxsdf57773H+fPnOXbsWJn6XppqH9Q2adKkyDdy3LhxADx58oRx48ZRr149TE1NGThwIAkJCRVuT8nN5XH8XW11H0dHRwwMDHj8+DF//PGH1uoVQggh/ul69+5NXFwcN27cYOnSpaxatYrZs2drpe6IiAh69+6Nh4cHR48e5fLlyyxbtgwDAwNycnLU5UaMGMH48ePx8fHh0KFDhIaG4ufnx44dOwgO/t+vv0ZGRgQFBXH9+vVS205OTubLL78ssUxqaiqdO3fm888/L/MzKYrC8uXLGTVqlDqtU6dOxMXFaVzvv/8+dnZ2eHh4aNx/4MABjXLu7u7FtpWTk0Pfvn3JzMzkxIkTrFu3ju+++45Zs2apyxgYGDBs2DC+/vrrMj9DaQ9Yrd27d0+Ji4tTX/v371cA5dChQ4qiKMqYMWOUhg0bKr/99pty9uxZpWPHjkqnTp3K1UZiYqICKPPf6KX8+61+StL9P7X6DNu2bVNmz56t7Ny5U6v1CiGEEBWR/7mXmJio1XrvPk5Tfo/6U7n7OE2r9RbF19dX8fHx0UgbMGCA4ubmpn6dk5Oj+Pv7K02aNFGMjIwUFxcXZcuWLer8hw8fKsOGDVPMzc0VIyMjxd7eXlmzZo2iKIqydOlSpUmTJiX2YdOmTQqgbN++vVBebm6u8vjxY0VRFGX27NmKq6ur0rNnT2XQoEHqMhcuXFAAJSYmRlEURYmJiVEAZcqUKYqpqamSkJCgLuvq6qrMnj27UDv591y4cKHEviqKopw5c0bR0dFRkpKSii2TmZmpWFhYKJ999lmF2si3Z88eRUdHR4mPj1enrVy5UqlVq5aSkZGhTjty5IhiYGCgpKU9+7+Zaj9Sa2FhgbW1tfratWsXzZo1w8vLi8TERIKCgliyZAndunXD3d2dtWvXcuLEiVKHxIvT7d0PqVnPXKvPkD8FITw8XGPYXQghhKiOFEUhLTO7XNd/T97kpUUHGfZNCC8tOsh/T94sdx3KM0zTCwsL48SJExgYGKjTFi5cyPr16wkMDCQ8PJwJEyYwfPhwjhw5AoCfnx8RERHs3buXK1eusHLlSszN82IAa2tr4uLiOHr0aLFtbtiwAUdHR3x8fArlqVQqzMzMNNIWLVrE1q1bOXv2bInPMnToUOzt7fnss8/K/PxlcezYMRwcHKhZs2axZXbu3MmDBw949913C+W9/vrrWFpa0rlzZ3bu3FliWydPnsTZ2RkrKyt1mre3N0lJSYSHh6vTPDw8yM7OJiQkpAJPpEnvmWt4jjIzM/n++++ZOHEiKpWKc+fOkZWVRY8ePdRlWrRoQaNGjTh58iQdO3Yssp6MjAwyMjLUr5OSktT/nZWZSVZWllb7Xb9+fWrWrElycjJXrlyhRYsWWq1fCCGEKI/SPufSs3JoNWtfhevPVcBvRzh+O8JLL1xAxGfemBiUPTTZtWsXpqamZGdnk5GRgY6ODsuXLwfyPuv9/f05cOAAnp6eADRt2pTjx4+zatUqvLy8uHXrFm5ubuqf2Zs0aaKue9CgQezbtw8vLy+sra3p2LEj3bt355133qFWrVoAXL9+HUdHxzL3t23btgwePJhp06bx22+/FVtOpVKxaNEi+vXrx4QJE2jWrFmZ2yhJbGwstra2JZYJCgrC29ubBg0aqNNMTU3597//zUsvvYSOjg5bt26lf//+bN++nddff73IeuLj4zUCWkD9Oj4+Xp1mYmKCmZkZsbGxFX0stRcqqN2+fTuPHz9m5MiRQN6bYmBgQO3atTXKWVlZabxhT1u4cCFz584tMu/E1o3cyVRQ6epqq9sAGBsbk5yczIEDB7hx44ZW6xZCCCHKI38B04uua9eurFy5ktTUVJYuXYqenh4DBw4E8k70TEtLo2fPnhr3ZGZm4ubmBsDYsWMZOHAg58+fp1evXvTv31+9gEpXV5e1a9cyf/58Dh48SEhICP7+/nz++eecPn0aGxubCo0sz58/n5YtWxIcHIylpWWx5by9vencuTN+fn788MMP5W6nKOnp6RgZGRWbf/v2bfbt28fmzZs10s3NzZk4caL6dbt27bh79y5ffPFFsUFteRgbG2vl3+QLFdQGBQXRp0+fUr9llGbGjBkafzlJSUk0bNgQ41q1yU5Lxa6WCa28uj9rdzXcu3ePb775huTkZLp27YqxsbFW6xdCCCHKquAvlEUx1tcl4jPvMtcXn/iEHkuOkFsgxtNRwYGJXlibFR9EFdVuedSoUUO9deaaNWtwdXUlKCiIUaNGkZKSAsDu3bupX7++xn2GhoYA9OnTh9jYWPbs2cP+/fvp3r0748aN01ikVb9+fUaMGMGIESOYN28eDg4OBAYGMnfuXBwcHLh69Wq5+tysWTM++OADpk+fTlBQUIllFy1ahKenJ1OmTClXG8UxNzfn8uXLxeavXbuWevXqlSlQ7dChA/v37y8239ramtOnT2uk5S/kt7a21kh/+PAhFhYWpbZZmhcmqI2NjeXAgQP8/PPP6jRra2syMzN5/PixxmhtQkJCoTesIENDQ/U/6ILc+vTj/M8bObdrG87deqKjo73R2vr162NlZUVCQgKRkZGFVhQKIYQQz4u+vn6J+SqVqlzTAJpamLJwgDMzfw4jR1HQVanwH9Caphamz9rVMtPR0WHmzJlMnDiRYcOG0apVKwwNDbl16xZeXl7F3mdhYYGvry++vr68/PLLTJkypdidB+rUqYONjQ2pqakADBs2jLfeeosdO3YUmlerKApJSUmF5tUCzJo1i2bNmrFx48YSn6l9+/YMGDCA6dOnl/b4ZeLm5sbKlStRFAWVSlWov2vXruWdd94p9d8H5G1HZmNjU2y+p6cnCxYs4N69e+oR6f3791OrVi1atWqlLhcdHc2TJ0/Uo+fPotovFMu3du1aLC0t6du3rzrN3d0dfX19jXkp165d49atW+r5M+Xh/EoPjGqY8ijuDtdDTmql3wW5uroCcPHiRa3XLYQQQlSlIe0acXx6V378oCPHp3dlSLtGz70PgwYNQldXlxUrVlCzZk0mT57MhAkTWLduHdHR0Zw/f55ly5axbt06IC+43LFjB1FRUYSHh7Nr1y5atmwJwKpVqxg7dizBwcFER0cTHh7OtGnTCA8PVx8YMHjwYIYMGcLQoUPx9/fn7NmzxMbGsmvXLnr06MGhQ4eK7KeVlRUTJ04s01ZWCxYs4ODBg1y7dk0j/eHDh4SGhhIREQHkxT+hoaElTr/s2rUrKSkpGgu18h08eJCYmBjef//9Qnnr1q3jxx9/5OrVq1y9ehV/f3/WrFnDxx9/rC6zbds2jTVDvXr1olWrVowYMYKLFy+yb98+Pv30U8aNG6cxsHjs2DGaNm2qnXnDz7x/wnOQk5OjNGrUSJk2bVqhvDFjxiiNGjVSDh48qJw9e1bx9PRUPD09y1V/wa1Njm/6XvlycF9l/dRPlNzcXG09grqdOXPmKLNnz1YePHig1bqFEEKIsqqsLb2ep6K29FIURVm4cKFiYWGhpKSkKLm5uUpAQIDi6Oio6OvrKxYWFoq3t7dy5MgRRVEUZd68eUrLli0VY2NjpW7duoqPj49y48YNRVEU5fz588rw4cMVOzs7xdDQUKlXr57SpUuXQttz5uTkKCtXrlTatWunmJiYKLVq1VLc3d2Vr776Sr1NVf6WXgUlJiYq5ubmRW7p9fTWWaNHj1YAjS291q5dqwCFrqK2/Spo8ODByvTp0wulDx06tNgtUb/77julZcuW6udr3769xtZoBftT0M2bN5U+ffooxsbGirm5uTJp0iQlKytLo0yvXr2UhQsXltjnslIpSvU/5io4OBhvb2+uXbuGg4ODRt6TJ0+YNGkSP/74IxkZGXh7e/Of//ynxOkHT8v/eSAxMRF9Fawe9y7ZGRkMmDEXuzbFbyxcEevXr+fGjRt07dq1xJ9DhBBCiMpS8HMvfyW/+Ge4dOkSPXv2JDo6GlPT5zc9pCjh4eF069aNyMjIIqdplNcLMf2gV69eKIpSKKCFvBM6VqxYwcOHD0lNTeXnn38uV0D7NOOatXDt0RuA09u3VLie4hScgvACfJ8QQgghxN+Ii4sLn3/+OTExMVXdFeLi4li/fr1WAlp4QYLa5839tTfQ0dXj9pUwbl8t3x57pWnRogX6+vo8fPiQO3fuaLVuIYQQQojSjBw5Emdn56ruBj169MDbu+y7bJRGgtoi1KxrjpNXN0D7o7WGhobqidSXLl3Sat1CCCGEEP9UEtQWo93rA1GpdIi5cJZ7N7V7WEL+FISwsDBycnK0WrcQQgghxD+RBLXFqGNTH4eOLwHaH621s7OjRo0apKWlERUVpdW6hRBCCCH+iSSoLUH7/oMAiDz1O4/itDf/VVdXVz2XRaYgCCGEEEI8OwlqS2DZpClN27ZDUXI5s3OrVuvOn4Jw9epVnjx5otW6hRBCCCH+aSSoLUV7n7zR2vAjB0l+cF9r9VpbW2NhYUFOTo76NBAhhBBCCFExEtSWon6LVjRo2ZrcnGzO7d6mtXpVKhUuLi6AHJsrhBBCCPGsJKgtgw5/za29eOBX0pIStVZvflAbGxvL48ePtVavEEIIIURRHjx4gKWlJTdv3qzqrjB9+nQ+/vhjrdUnQW0ZNHZti2WTZmRnZHDh11+0Vq+ZmRlNmjQB4PLly1qrVwghhPi7GzlyJCqVCpVKhb6+PnZ2dkydOlWr61SOHDlCt27dqFu3LiYmJjRv3hxfX18yMzPVZRRFYfXq1XTo0AFTU1Nq166Nh4cHAQEBpKWlATBnzhxUKhVjxozRqD80NBSVSqUOMG/evIlKpcLS0pLk5GSNsm3atGHOnDkAZGVlMW3aNJydnalRowa2tra888473L17t9RnWrBgAT4+Pur447vvvlO/j09f9+7dA+Dw4cNF5sfHx5fY1qVLl3j55ZcxMjKiYcOGLF68WCN/8uTJrFu3jhs3tLN1qgS1ZaBSqejwRt5o7YVffyEzPU1rdRecgiDH5gohhBBl17t3b+Li4rhx4wZLly5l1apVzJ49Wyt1R0RE0Lt3bzw8PDh69CiXL19m2bJlGBgYaOwxP2LECMaPH4+Pjw+HDh0iNDQUPz8/duzYQXBwsLqckZERQUFBXL9+vdS2k5OT+fLLL4vNT0tL4/z58/j5+XH+/Hl+/vlnrl27xuuvv15ivWlpaQQFBTFq1Ch12pAhQ4iLi9O4vL298fLywtLSUuP+a9euaZR7Or+gpKQkevXqRePGjTl37hxffPEFc+bMYfXq1eoy5ubmeHt7s3LlytLekrJRhJKYmKgASmJiYrFlcnKylaB/jVa+HNxXOb3jJ621nZ6ersybN0+ZPXu2cufOHa3VK4QQQhSnLJ97FfL4tqLcOJL3ZyXz9fVVfHx8NNIGDBiguLm5qV/n5OQo/v7+SpMmTRQjIyPFxcVF2bJlizr/4cOHyrBhwxRzc3PFyMhIsbe3V9asWaMoiqIsXbpUadKkSYl92LRpkwIo27dvL5SXm5urPH78WFEURZk9e7bi6uqq9OzZUxk0aJC6zIULFxRAiYmJURRFUWJiYhRAmTJlimJqaqokJCSoy7q6uiqzZ88uti+nT59WACU2NrbYMlu2bFEsLCxKfKZ79+4p+vr6yvr169Vphw4dUgDl0aNHJd5b0H/+8x+lTp06SkZGhjpt2rRpiqOjo0a5devWKQ0aNChzvSWRkdoy0tHRpb3PmwCc272d7AI/PTwLIyMjHB0dAdmzVgghRDWhKJCZWr7r9DcQ0BrW9cv78/Q35a/jGX6xDAsL48SJExgYGKjTFi5cyPr16wkMDCQ8PJwJEyYwfPhwjhw5AoCfnx8RERHs3buXK1eusHLlSszNzYG8XYri4uI4evRosW1u2LABR0dHfHx8CuWpVCrMzMw00hYtWsTWrVs5e/Zsic8ydOhQ7O3t+eyzz8r8/ImJiahUKmrXrl1smWPHjuHu7l5iPevXr8fExIQ333yzUF6bNm2wsbGhZ8+e/P777yXWc/LkSbp06aLx9+Ht7c21a9d49OiROq19+/bcvn1bK3N89Z65hn+Qli+/woktP5D84E/CjxzAteerWqnXxcWF8PBwLl++TM+ePdHV1dVKvUIIIUSFZKWBv23F71dyYc/kvKs8Zt4FgxplLr5r1y5MTU3Jzs4mIyMDHR0dli9fDkBGRgb+/v4cOHAAT09PAJo2bcrx48dZtWoVXl5e3Lp1Czc3Nzw8PADU80wBBg0axL59+/Dy8sLa2pqOHTvSvXt33nnnHWrVqgXA9evX1QNTZdG2bVsGDx7MtGnT+O2334otp1KpWLRoEf369WPChAk0a9asxHqfPHnCtGnTGDp0qLpvRYmNjcXWtuS/16CgIIYNG4axsbE6zcbGhsDAQDw8PMjIyODbb7/llVdeISQkhLZt2xZZT3x8PHZ2dhppVlZW6rw6deoAqPsTGxur8f5XhIzUloOunj4e/d4A4MzOreQWmFPzLOzt7TExMSE1NVVrk6WFEEKIv7uuXbsSGhpKSEgIvr6+vPvuuwwcOBCAqKgo0tLS6NmzJ6ampupr/fr1REdHAzB27Fg2btxImzZtmDp1KidOnFDXraury9q1a7l9+zaLFy+mfv36+Pv74+TkRFxcHECF1sLMnz+fY8eOacy3LYq3tzedO3fGz8+vxHJZWVkMHjwYRVFKnZuanp6OkZFRsfknT57kypUrGnNuARwdHfnwww9xd3enU6dOrFmzhk6dOrF06dIS2yuL/OA5f1Hds5CR2nJy7taLU1s3kngvgWsnjtLy5a7PXKeuri6tW7fm9OnTXLp0iebNm2uhp0IIIUQF6ZvkjZqWVdJdWNE+b4Q2n0oXxoVArXKM+OqblL0sUKNGDezt7QFYs2YNrq6u6oVQKSkpAOzevZv69etr3GdoaAhAnz59iI2NZc+ePezfv5/u3bszbtw4jUVa9evXZ8SIEYwYMYJ58+bh4OBAYGAgc+fOxcHBgatXr5arz82aNeODDz5g+vTpBAUFlVh20aJFeHp6MmXKlCLz8wPa2NhYDh48WOIoLeQtzCr40//Tvv32W9q0aVPqFAXImzZw/PjxYvOtra1JSEjQSMt/bW1trU57+PAhABYWFqW2WRoZqS0nfUMj3Pv2ByBk+xaU3NySbyij/F0Qrly5QkZGhlbqFEIIISpEpcqbBlDWy7w59PsqL5CFvD/7BeSll6celarCXdbR0WHmzJl8+umnpKen06pVKwwNDbl16xb29vYaV8OGDdX3WVhY4Ovry/fff09AQIDG6vyn1alTBxsbG1JTUwEYNmwYkZGR7Nixo1BZRVFITCx6b/tZs2YRGRnJxo0bS3ym9u3bM2DAAKZPn14oLz+gvX79OgcOHKBevXol1gXg5uZW7CmmKSkpbN68udAobXFCQ0OxsbEpNt/T05OjR4+SlZWlTtu/fz+Ojo7qqQeQNxdaX18fJyenMrVbEglqK8C116sYGBvz4PYtos+f0Uqd9evXp169emRnZ3PlyhWt1CmEEEI8N23fgfGXwXdX3p9t33nuXRg0aBC6urqsWLGCmjVrMnnyZCZMmMC6deuIjo7m/PnzLFu2jHXr1gF5weWOHTuIiooiPDycXbt20bJlSwBWrVrF2LFjCQ4OJjo6mvDwcKZNm0Z4eDj9+vUDYPDgwQwZMoShQ4fi7+/P2bNniY2NZdeuXfTo0YNDhw4V2U8rKysmTpzI119/XeozLViwgIMHD3Lt2jV1WlZWFm+++SZnz55lw4YN5OTkEB8fT3x8vMYeuk/z9vYmPDy8yNHaTZs2kZ2dzfDhwwvlBQQEqN+nsLAwxo8fz8GDBxk3bpy6zPLly+nevbv69bBhwzAwMGDUqFGEh4ezadMmvvrqKyZOnKhR97Fjx3j55Zc15vBWlAS1FWBUw5Q2vfoCELJtk1b2ly14bK7sgiCEEOKFZFYf7F7O+7MK6Onp8dFHH7F48WJSU1OZN28efn5+LFy4kJYtW9K7d292796tXsBkYGDAjBkzcHFxoUuXLujq6qpHT9u3b09KSgpjxozByckJLy8vTp06xfbt2/Hy8gLyPrt/+OEHlixZok53cXFhzpw5+Pj44O3tXWxfJ0+ejKmpaanP5ODgwHvvvadxqMSdO3fYuXMnt2/fVu9IkH8VnBf8NGdnZ9q2bcvmzZsL5QUFBTFgwIAid0/IzMxk0qRJODs74+XlxcWLFzlw4IBGEHv//n31XGXIO2AqODiYmJgY3N3dmTRpErNmzWL06NEadW/cuJEPPvig1PehLFSKNiKyF1xSUhJmZmYkJiaWOh8lX+rjR3z70SiyszIZ5LeARq1dn7kfjx494quvvgJg4sSJZe6LEEIIUR4V+dwTfw+7d+9mypQphIWFoaNTtWObe/fuZdKkSVy6dAk9vWdf5iUjtRVUo3YdWnfrCeTNrdWGOnXq0KhRI0COzRVCCCGE9vXt25fRo0dz586dqu4KqamprF27VisBLUhQ+0za9RuIjq4uty6HEhd1rfQbykCmIAghhBCiMo0fP15jsVxVefPNN+nQoYPW6pOg9hnUsrCkxUt582pOa2m01snJCV1dXRISEoiPj9dKnUIIIYQQf3cS1D6j9j6DQKUi6swpHty+9cz1GRsb4+DgAMhorRBCCCFEWUlQ+4zqNWhI83Z5x+9pa7Q2fwrC5cuXydXSPrhCCCGEEH9nEtRqQfv+gwC48vsREu8llFK6dM2bN8fIyIjk5GRiYmKeuT4hhBBCiL+7ah/U3rlzh+HDh1OvXj2MjY1xdnbm7Nmz6nxFUZg1axY2NjYYGxvTo0cPrl+//lz7aN2sOY1d3FBycznzy8/PXJ+enh6tW7cGZAqCEEIIIURZVOug9tGjR7z00kvo6+uzd+9eIiIi+Pe//61xvNrixYv5+uuvCQwMJCQkhBo1auDt7a2xSfHz0N4nb7Q27FAwqY+LP1e5rAoem1vS6SBCCCGEEKKaB7Wff/45DRs2ZO3atbRv3x47Ozt69epFs2bNgLxR2oCAAD799FN8fHxwcXFh/fr13L17l+3btz/XvjZ0csamuSM5WVmc21P4DOhy19ewIXXq1CEzM5OrV69qoYdCCCGEEH9f2tnttpLs3LkTb29vBg0axJEjR6hfvz7/93//pz5OLSYmhvj4eHr06KG+x8zMjA4dOnDy5EneeuutIuvNyMggIyND/TopKQnIO0s5Kyurwv117zeQXUv8Cd23G7dXfTCqUfrxdyVxcnLi+PHjXLx4UX0WtRBCCPGsnuWzTrzYHjx4QMuWLTl9+jRNmjSp0r5Mnz6d1NRUli1bppX6qnVQe+PGDVauXMnEiROZOXMmZ86c4ZNPPsHAwABfX1/1Pq5WVlYa91lZWZW4x+vChQuZO3duofTg4GBMTEwq3F9FUTAwq0Nm4iN+WhFA3dZtK1wXoJ5CER0dzY4dO9DX13+m+oQQQgiAtLS0qu7CMxs5ciTr1q0D8taiNGjQgEGDBvHZZ59hZGSklTaOHDnC3LlzCQ0N5cmTJ9SvX59OnTrxzTffYGBgAOR99n/zzTcEBQURHh6Onp4e9vb2DB8+nNGjR2NiYsKcOXOYO3cuH374IYGBger6Q0NDcXNzIyYmhiZNmnDz5k3s7OywsLAgOjqamjVrqsu2adOG/v37M2fOHADmzJnDxo0b+eOPPzAwMMDd3Z0FCxaUepjBggUL8PHxUQe03333He+++26RZRMSErC0tOTw4cN07dq1UH5cXBzW1tbFtnXp0iXGjRvHmTNnsLCw4OOPP2bq1Knq/MmTJ9O0aVMmTJhA06ZNS+x3WVTroDY3NxcPDw/8/f0BcHNzIywsjMDAQHx9fStc74wZM5g4caL6dVJSEg0bNqRXr17PfAb2tTo12fefpaTFXGfIJ5PRf8b/sdauXcvdu3exsbGhffv2z1SXEEIIAf/7hfJF17t3b9auXUtWVhbnzp3D19cXlUrF559//sx1R0RE0Lt3bz7++GO+/vprjI2NuX79Olu3biUnJ0ddbsSIEfz88898+umnLF++HAsLCy5evEhAQABNmjShf//+ABgZGREUFMSkSZNo3rx5iW0nJyfz5ZdfFjkAl8/BwYHly5fTtGlT0tPTWbp0Kb169SIqKgoLC4si70lLSyMoKIh9+/ap04YMGULv3r01yo0cOZInT55gaWmpkX7t2jWNOOnp/IKSkpLo1asXPXr0IDAwkMuXL/Pee+9Ru3ZtRo8eDYC5uTne3t6sXLmSL774ovg3pKyUaqxRo0bKqFGjNNL+85//KLa2toqiKEp0dLQCKBcuXNAo06VLF+WTTz4pczuJiYkKoCQmJj5zn3Oys5VvPh6lfDm4r3Ju9/Znri8kJESZPXu2EhgY+Mx1CSGEEIqi3c+9guJS4pSQuyFKXEqcVustiq+vr+Lj46ORNmDAAMXNzU39OicnR/H391eaNGmiGBkZKS4uLsqWLVvU+Q8fPlSGDRummJubK0ZGRoq9vb2yZs0aRVEUZenSpUqTJk1K7MOmTZsUQNm+vfDnfW5urvL48WNFURRl9uzZiqurq9KzZ09l0KBB6jIXLlxQACUmJkZRFEWJiYlRAGXKlCmKqampkpCQoC7r6uqqzJ49u9i+5P+dHjhwoNgyW7ZsUSwsLEp8pnv37in6+vrK+vXr1WmHDh1SAOXRo0cl3lvQf/7zH6VOnTpKRkaGOm3atGmKo6OjRrl169YpDRo0KHO9JanWC8Veeuklrl27ppEWGRlJ48aNAbCzs8Pa2prffvtNnZ+UlERISAienp7Pta/5dHR1addvIABndm0jJ/vZ5i05OTmho6NDXFwcf/75pza6KIQQQpRIURTSstLKdW28uhHvn7wZFTwK75+82Xh1Y7nrUBSlwn0OCwvjxIkT6mkBkDfdcP369QQGBhIeHs6ECRMYPnw4R44cAcDPz4+IiAj27t3LlStXWLlyJebm5gBYW1sTFxfH0aNHi21zw4YNODo64uPjUyhPpVJhZmamkbZo0SK2bt2qsTVpUYYOHYq9vT2fffZZmZ49MzOT1atXY2Zmhqura7Hljh07hru7e4l1rV+/HhMTE958881CeW3atMHGxoaePXvy+++/l1jPyZMn6dKli8bfh7e3N9euXePRo//tEtW+fXtu377NzZs3S6yvLKr19IMJEybQqVMn/P39GTx4MKdPn2b16tWsXr0ayPsHM378eObPn0/z5s2xs7PDz88PW1tb9XB/VXDy6s7JrT+S8uA+EccO4dy1V4XrqlGjBvb29kRGRnLx4kWNRXFCCCFEZUjPTqfDDyXPzSxJLrksCFnAgpAF5bovZFgIJvplX9uya9cuTE1Nyc7OJiMjAx0dHZYvXw7kLQr39/fnwIED6oGupk2bcvz4cVatWoWXlxe3bt3Czc0NDw8PAI2FU4MGDWLfvn14eXlhbW1Nx44d6d69O++88476J/jr16/j6OhY5v62bduWwYMHM23aNI0BuaepVCoWLVpEv379mDBhgnrXp6Ke/6233iItLQ0bGxv279+vDsqLEhsbi62tbYl9DAoKYtiwYRgbG6vTbGxsCAwMxMPDg4yMDL799lteeeUVQkJCaNu26PVD8fHx2NnZaaTlr4GKj49Xb8+a35/Y2NhnXrhWrUdq27Vrx7Zt2/jxxx9p3bo18+bNIyAggLfffltdZurUqXz88ceMHj2adu3akZKSwq+//qq1SeIVoWdggHvf/gCc2fETubk5Jd9QivxvXXJsrhBCCPE/Xbt2JTQ0lJCQEHx9fXn33XcZODDv19KoqCjS0tLo2bMnpqam6mv9+vVER0cDMHbsWDZu3EibNm2YOnUqJ06cUNetq6vL2rVruX37NosXL6Z+/fr4+/vj5OREXFwcQIVGlufPn8+xY8cIDg4usZy3tzedO3fGz8+v1Oc/ceIEvXv3ZvDgwdy7d6/Y8unp6SXGRydPnuTKlSuMGjVKI93R0ZEPP/wQd3d3OnXqxJo1a+jUqRNLly4t8RnKIj941sbixWo9Ugvw2muv8dprrxWbr1Kp+Oyzz8o8RP+8uPbozeltm3kUd5frISdx9Oxc4bocHBwwNDQkMTGRW7duVfkWHEIIIf7ejPWMCRkWUubyCWkJ9N/en1z+N/Cio9Jhu892rEysSrizcLvlkf9rJsCaNWtwdXUlKCiIUaNGkZKSAsDu3bupX7++xn2GhoYA9OnTh9jYWPbs2cP+/fvp3r0748aN48svv1SXrV+/PiNGjGDEiBHMmzcPBwcHAgMDmTt3Lg4ODuXeS75Zs2Z88MEHTJ8+naCgoBLLLlq0CE9PT6ZMmVLi89vb29OxY0eaN29OUFAQM2bMKLK8ubm5xk//T/v2229p06ZNqVMUIG/awPHjx4vNt7a2JiEhQSMt/3XBHRMePnwIUOzitvKo1iO1LzIDYxPc+vQDIGT75meaJ6Svr0+rVq0AuHjxolb6J4QQQhRHpVJhom9S5svOzI7ZnWajo8oLK3RUOsz2nI2dmV256lGpVBXus46ODjNnzuTTTz8lPT2dVq1aYWhoyK1bt9SBX/7VsGFD9X0WFhb4+vry/fffExAQoJ7iWJQ6depgY2NDamoqAMOGDSMyMpIdOwofuqQoComJiUXWM2vWLCIjI9m4cWOJz9S+fXsGDBjA9OnTy/IWkJubq7EP/9Pc3NyIiIgoMi8lJYXNmzcXGqUtTmhoKDY2NsXme3p6cvToUY09kffv34+jo6PGybBhYWHo6+vj5ORUpnZLIkFtJXLr3Q99QyP+vHmDmxfPP1Nd+VMQIiIiZNNsIYQQ1c6A5gPYN3Afa7zXsG/gPgY0H/Dc+zBo0CB0dXVZsWIFNWvWZPLkyUyYMIF169YRHR3N+fPnWbZsmXp/21mzZrFjxw6ioqIIDw9n165d6sOOVq1axdixYwkODiY6Oprw8HCmTZtGeHg4/frlDVoNHjyYIUOGMHToUPz9/Tl79iyxsbHs2rWLHj16cOjQoSL7aWVlxcSJE/n6669LfaYFCxZw8OBBjYXzqampzJw5k1OnThEbG8u5c+d47733uHPnDoMGDSq2Lm9vb8LDw4scrd20aRPZ2dkMHz68UF5AQID6fQoLC2P8+PEcPHiQcePGqcssX76c7t27q18PGzYMAwMDRo0aRXh4OJs2beKrr77S2FIV8havvfzyyxpzeCtKgtpKZFyzFi498vZ+C9m2+ZnqatSoEWZmZmRkZBAZGamN7gkhhBBaZV3DmnbW7bCuUfyG/JVJT0+Pjz76iMWLF5Oamsq8efPw8/Nj4cKFtGzZkt69e7N79271AiYDAwNmzJiBi4sLXbp0QVdXVz162r59e1JSUhgzZgxOTk54eXlx6tQptm/fjpeXF5A3ov3DDz+wZMkSdbqLiwtz5szBx8cHb2/vYvs6efJkTE1LP3nUwcGB9957T30gE+TN97169SoDBw7EwcGBfv368eDBA44dO1biiKezszNt27Zl8+bCMUlQUBADBgygdu3ahfIyMzOZNGkSzs7OeHl5cfHiRQ4cOKARxN6/f189VxnyTngNDg4mJiYGd3d3Jk2axKxZs9R71ObbuHGj+qTYZ6VSnuV38b+JpKQkzMzMSExMfObDF56W/PA+3370Prk52QyZ+zkNWlR8eP3AgQMcP34cBwcHhg0bpsVeCiGE+CepzM89Ub3t3r2bKVOmEBYWho5O1Y5t7t27l0mTJnHp0iX09J59mZeM1FaymnXNcXol75vM6e1bnqmu/CkIUVFR6vk8QgghhBBl1bdvX0aPHs2dO3equiukpqaydu1arQS0IEHtc9Hu9YGoVDrEXDjLvZs3KlyPhYUFNjY25ObmEh4ersUeCiGEEOKfYvz48RqL5arKm2++SYcOFd8P+WkS1D4HdaxtcfhrS6+QZxytdXFxAWQXBCGEEEKIgiSofU469M9bjRh56jgP71Z8yN/Z2RmVSsWdO3d48OCBtronhBBCCPFCk6C2gKynNgnWJovGdjRt2w4UhTM7t1a4HlNTU/VxeZcuXdJW94QQQgghXmgS1BZw47V+PP7pp0qrv33/wQBEHD1I8oP7Fa4nfwrCpUuXnulQByGEEEKIvwsJagvKzSVu1myy4uMrpfr6ji1p0Ko1uTnZnN21rcL1tGjRAgMDAx49esQff/yhxR4KIYQQQryYJKh9Wm4umbG3Kq36Dj55c2sv/fYraUlFH59XGgMDA/WJJzIFQQghhBBCgtrCdHQwaNyo0qpv7NoWS7tmZGdkcOHXXypcT/4UhLCwMLKzs7XVPSGEEEKIF5IEtU+x+Ne/0LeuvOP9VCoVHd7Im1t74ddfyEhLq1A9dnZ21KxZkydPnnD9+nVtdlEIIYQQf1MPHjzA0tKSmzdvVnVXmD59Oh9//LHW6pOg9ikZkZGV3kbzdp7UsW1ARmoqF/fvqVAdOjo6ODs7AzIFQQghxD/PyJEjUalUqFQq9PX1sbOzY+rUqTx58kRrbRw5coRu3bpRt25dTExMaN68Ob6+vmRmZqrLKIrC6tWr6dChA6amptSuXRsPDw8CAgJI+2vgas6cOahUKsaMGaNRf2hoKCqVSh1g3rx5E5VKhaWlJcnJyRpl27Rpw5w5c4rs55gxY1CpVAQEBJT6TAsWLMDHx4cmTZoA8N1336nfx6eve/fuAXD48OEi8+NLWYN06dIlXn75ZYyMjGjYsCGLFy/WyJ88eTLr1q3jxo2KH0xVkAS1T0navZv0Sj6tS6WjQ3ufNwE4t3s72QX+5yiP/CkIkZGRpKena61/QgghxIugd+/exMXFcePGDZYuXcqqVauYPXu2VuqOiIigd+/eeHh4cPToUS5fvsyyZcswMDAgJydHXW7EiBGMHz8eHx8fDh06RGhoKH5+fuzYsYPg4GB1OSMjI4KCgsr062pycjJffvllmfq5bds2Tp06ha2tball09LSCAoKYtSoUeq0IUOGEBcXp3F5e3vj5eWFpaWlxv3Xrl3TKPd0fkFJSUn06tWLxo0bc+7cOb744gvmzJnD6tWr1WXMzc3x9vZm5cqVZXrW0khQW0DNPn0A+PPfSyq9rZadX6GmuQVpiY8JO3ygQnVYW1tjZWVFTk6OHJsrhBCiymXFx5N6KqTSdhF6mqGhIdbW1jRs2JD+/fvTo0cP9u/fr87Pzc1l4cKF2NnZYWxsjKurKz8V2Lrz0aNHvP3221hYWGBsbEzz5s1Zu3YtAMHBwVhbW7N48WJat25Ns2bN6N27N9988w3GxsYAbN68mQ0bNvDjjz8yc+ZM2rVrR5MmTfDx8eHgwYN07dpV3ZajoyNdu3bl//2//1fqc3388ccsWbJEPVJanDt37vDxxx+zYcMG9PX1S613z549GBoa0rFjR3WasbEx1tbW6ktXV5eDBw9qBL75LC0tNcrq6BQfRm7YsIHMzEzWrFmDk5MTb731Fp988glLlmjGWP369WPjxo2l9r0sJKgtwHzsGNDXJ/XECVJ+/71S29LV08PjtQEAnNm5ldwC3/rKo+CetUIIIYQ2KIpCblpaua6HP/xAVLfu3Bo5kqhu3Xn4ww/lruNZ9l4PCwvjxIkTGBgYqNMWLlzI+vXrCQwMJDw8nAkTJjB8+HCOHDkCgJ+fHxEREezdu5crV66wcuVKzM3NgbyBo7i4OI4ePVpsmxs2bMDR0REfH59CeSqVCjMzM420RYsWsXXrVs6ePVviswwdOhR7e3s+++yzYsvk5uYyYsQIpkyZgpOTU4n15Tt27Bju7u4lllm/fj0mJia8+eabhfLatGmDjY0NPXv25PdS4qSTJ0/SpUsXjb8Pb29vrl27xqNHj9Rp7du35/bt21qZ46v3zDX8jRjUr0+doW/xaP1/+fPfS6jh6YmqhG8hz8q5W09O/byRpD8TuHriKK1e7lr6TU/X4ezM/v37uXXrFo8ePaJOnTqV0FMhhBD/JEp6Otfalhz8lCg3l4TP5pHw2bxy3eZ4/hwqE5Myl9+1axempqZkZ2eTkZGBjo4Oy5cvByAjIwN/f38OHDiAp6cnAE2bNuX48eOsWrUKLy8vbt26hZubGx4eHgDqeaYAgwYNYt++fXh5eWFtbU3Hjh3p3r0777zzDrVq1QLg+vXrODo6lrm/bdu2ZfDgwUybNo3ffvut2HIqlYpFixbRr18/JkyYoD5JtKDPP/8cPT09PvnkkzK3HxsbW+o0haCgIIYNG6YejQawsbEhMDAQDw8PMjIy+Pbbb3nllVcICQmhbdu2RdYTHx+PnZ2dRpqVlZU6Lz9eye9PbGysxvtfETJS+xTzMWPQqVGDJxERJO3ZW6lt6Rsa4f5q3re709u3oOTmlruOWrVq0bRpU0BGa4UQQvyzdO3aldDQUEJCQvD19eXdd99l4MCBAERFRZGWlkbPnj0xNTVVX+vXryc6OhqAsWPHsnHjRtq0acPUqVM5ceKEum5dXV3Wrl3L7du3Wbx4MfXr18ff3x8nJyfi4uIAKjSyPH/+fI4dO6Yx37Yo3t7edO7cGT8/v0J5586d46uvvlIv8iqr9PR0jIyMis0/efIkV65cKTT1wNHRkQ8//BB3d3c6derEmjVr6NSpE0uXLi1z28XJD57TKrgbVEEyUvsUvbp1qff+KP786mv+DAigVq+eqAoMnWuba69XOb3jJx7cvkX0udPYt+tY+k1PcXFx4caNG1y6dIkuXbqU6x+4EEII8TSVsTGO58+VuXxWQgI3+r4GBQdndHRounsX+n+NzpW13fKoUaMG9vb2AKxZswZXV1f1QqiUlBQAdu/eTf369TXuMzQ0BKBPnz7ExsayZ88e9u/fT/fu3Rk3bpzGIq369eszYsQIRowYwbx583BwcCAwMJC5c+fi4ODA1atXy9XnZs2a8cEHHzB9+nSCgoJKLLto0SI8PT2ZMmWKRvqxY8e4d+8ejRr9b1/9nJwcJk2aREBAQLE/5Zubm2v89P+0b7/9ljZt2pQ6RQHypg0cP3682Hxra2sSEhI00vJfWxfYOvXhw4cAWFhYlNpmaSo0Urtu3Tp2796tfj116lRq165Np06diI2NfeZOVbW6vr7oWpiTdfs2jzZtrtS2jGqY0sa7LwAh2zdX6Ftfy5Yt0dPT48GDB9y5c0fbXRRCCPEPo1Kp0DExKfNlaGeHzWdzIX/Kno4ONp/NxdDOrlz1PMugjI6ODjNnzuTTTz8lPT2dVq1aYWhoyK1bt7C3t9e4GjZsqL7PwsICX19fvv/+ewICAjRW5z+tTp062NjYkJqaCsCwYcOIjIxkx44dhcoqikJiYtEnh86aNYvIyMhSF0i1b9+eAQMGMH36dI30ESNGcOnSJUJDQ9WXra0tU6ZMYd++fcXW5+bmRkRERJF5KSkpbN68ucgFYkUJDQ3Fxsam2HxPT0+OHj1KVlaWOm3//v04OjpqTJUMCwtDX1+/zPOCS1KhoNbf3189XHzy5ElWrFjB4sWLMTc3Z8KECc/cqaqmY2KCxbiPALj/n/+Q89e3vcri/qoPevoGxEdF8kd4+acQGBoayrG5QgghqlTtN9/E/uBvNFq3DvuDv1G7iIVGlW3QoEHo6uqyYsUKatasyeTJk5kwYQLr1q0jOjqa8+fPs2zZMtatWwfkBZc7duwgKiqK8PBwdu3apf48XbVqFWPHjiU4OJjo6GjCw8OZNm0a4eHh9OvXD4DBgwczZMgQhg4dir+/P2fPniU2NpZdu3bRo0cPDh06VGQ/raysmDhxIl9//XWpz7RgwQIOHjzItWvX1Gn16tWjdevWGpe+vj7W1tYlzvH19vYmPDy8yNHaTZs2kZ2dzfDhwwvlBQQEqN+nsLAwxo8fz8GDBxk3bpy6zPLly+nevbv69bBhwzAwMGDUqFGEh4ezadMmvvrqKyZOnKhR97Fjx3j55Zc15vBWVIWC2j/++EM93L99+3YGDhzI6NGjWbhwIceOHXvmTlUHtQcOwKBJE3IePeLhmrWV2paJWW1ad+sFQMi2io0MFzw2N6eCOykIIYQQz0Lf2poaHdpX6smcJdHT0+Ojjz5i8eLFpKamMm/ePPz8/Fi4cCEtW7akd+/e7N69W72AycDAgBkzZuDi4kKXLl3Q1dVVj562b9+elJQUxowZg5OTE15eXpw6dYrt27fj5eUF5I1o//DDDyxZskSd7uLiwpw5c/Dx8cHb27vYvk6ePBlTU9NSn8nBwYH33ntPK4dKODs707ZtWzZvLhxrBAUFMWDAAGrXrl0oLzMzk0mTJuHs7IyXlxcXL17kwIEDGkHs/fv31XOVAczMzAgODiYmJgZ3d3cmTZrErFmzGD16tEbdGzdu5IMPPnjmZwNQKRX4vdvS0pJ9+/bh5uaGm5sbEydOZMSIEURHR+Pq6qqex/KiSEpKwszMjMTERPWKRoCkfcHc+de/UJmYYL/vV/S0MN+j2D78eY+gf31Abk4Owxb8Gxv7sq+mhLy5NEuWLCE1NZWhQ4eWazWmEEKIf5biPvfE39/u3buZMmUKYWFhJe4z+zzs3buXSZMmcenSJfT0nn2ZV4WepmfPnrz//vu8//77REZG8uqrrwIQHh7+zNsxVCc1e/XEyMUFJS2N+1o67aI4tSwsadn5FSBvJ4Ty0tXVlWNzhRBCCFGivn37Mnr06GqxBic1NZW1a9dqJaCFCga1K1aswNPTkz///JOtW7dSr149IG+LiaFDh2qlY/C/s5ILXi1atFDnP3nyhHHjxlGvXj1MTU0ZOHBgoZV2z0KlUmE5eRIAjzZvIVMLGwOXpN3rb4JKRdSZU9z/o/wL7vKnIFy7dk2rZ18LIYQQ4u9j/PjxGovlqsqbb75Jhw4dtFZfhYLa2rVrs3z5cnbs2EHv3r3V6XPnzi3T8W/lkb8fXP5VcPuICRMm8Msvv7BlyxaOHDnC3bt3GTBggFbbr9G+PTW8ukB2NvcCvtJq3U+r16AhzdvlbRB9esdPpZQuzMbGBnNzc7Kzs4td3SiEEEII8XdUoaD2119/1QguV6xYQZs2bRg2bFiJ+59VhJ6ensY5w/nH1yUmJhIUFMSSJUvo1q0b7u7urF27lhMnTnDq1Cmt9sFy4kRQqUj+9VfSK/mn/fb9BwFw9fcjJN4r39nZKpUKV1dXQKYgCCGEEOKfpUKTGKZMmcLnn38OwOXLl5k0aRITJ07k0KFDTJw4kbVrtbdbwPXr17G1tcXIyAhPT08WLlxIo0aNOHfuHFlZWfTo0UNdtkWLFjRq1IiTJ0/SsWPxhxhkZGSQkZGhfp2UlARAVlaWxn5q+XSbNqVmv34k79xJwhdfYhv0baUdcFCvURMaObfh1uVQQrb/RNd3PyzX/S1btuS3337j5s2bPHjwQBYACCGEKKSozzohXnQVCmpjYmJo1aoVAFu3buW1117D39+f8+fPqxeNaUOHDh347rvvcHR0JC4ujrlz5/Lyyy8TFhZGfHw8BgYGhbaesLKyIj6+5BHOhQsXMnfu3ELpwcHBmBRz5rReq1Y02bOH9DNnOBIQQFol7i6QbVkfCCXs0H6Sa9VDz7js52ADmJqakpKSwk8//aRxaocQQggB2jmSVIjqpkJBrYGBgfp/iAMHDvDOO+8AULduXfWopzb06dNH/d8uLi506NCBxo0bs3nz5mfapHfGjBkam/8mJSXRsGFDevXqVeLI5v24uzxet56mx3+n4b/+haqStsJQFIUtt6KIv36NellpvDSwfBtYh4aGsnv3brKysujTp48cmyuEEEKDNj+rhaguKhTUdu7cmYkTJ/LSSy9x+vRpNm3aBEBkZCQNGjTQagcLql27Ng4ODkRFRdGzZ08yMzN5/PixxmhtQkJCqaOThoaG6nOfC9LX10dfX7/Y+yzGjCHp521kRkaSvm8fZq+/XuFnKU3HNwazffE8Lh34lY5vDMGoDBs053N2dubXX3/l/v37PHjwoMRj7IQQQvzzlPRZJ8SLqkJDjcuXL0dPT4+ffvqJlStXUr9+fSBvE92CuyFoW0pKCtHR0djY2ODu7o6+vj6//fabOv/atWvcunULT0/PSmlfr04d6v116sWfAV+Rm5lZKe0ANHVrh3mjJmQ9SSd0365y3WtkZKQ+fOHixYuV0T0hhBBCiGqlQkFto0aN2LVrFxcvXmTUqFHq9KVLl5bpHOOymjx5MkeOHOHmzZucOHGCN954A11dXYYOHYqZmRmjRo1SL1A7d+4c7777Lp6eniUuEntWdUcMR8/Skqy7d3n844+V1o5KR0e9E8K5vTvJKue+s/m7IFy+fFmOzRVCCCEEAA8ePMDS0pKblbz3fllMnz6djz/+WGv1VXhSaE5ODlu3bmX+/PnMnz+fbdu2aT14un37tvrI18GDB1OvXj1OnTqFxV/H1S5dupTXXnuNgQMH0qVLF6ytrfn555+12oen6RgbY/7xRwDcXxlITnJypbXl2LEzZlbWPElO4tJv+8p1r729PcbGxqSmphITE1NJPRRCCCGqxsiRI9UHM+nr62NnZ8fUqVO1evjQkSNH6NatG3Xr1sXExITmzZvj6+tLZoFfahVFYfXq1XTo0AFTU1Nq166Nh4cHAQEB6vVH+YdJjRkzRqP+0NBQVCqVOsC8efNm3sFPlpYkPxVftGnThjlz5hT5/PlXWX4tX7BgAT4+PuoTYL/77rtC9eRf9+7dA+Dw4cNF5pe2MP/SpUu8/PLLGBkZ0bBhQxYvXqyRP3nyZNatW8eNGzdK7XdZVCiojYqKomXLlrzzzjv8/PPP/PzzzwwfPhwnJyeio6O10jGAjRs3cvfuXTIyMrh9+zYbN26kWbNm6nwjIyNWrFjBw4cPSU1N5eeff34uq/1rv/EGBk2bkvP4MQ++Daq0dnR0dWn/et4isbO7fiYnu+xbsOjq6tK6dWtApiAIIYT4e+rduzdxcXHcuHGDpUuXsmrVKmbPnq2VuiMiIujduzceHh4cPXqUy5cvs2zZMgwMDDQG8UaMGMH48ePx8fHh0KFDhIaG4ufnx44dOwgODlaXMzIyIigoiOvXr5fadnJyMl9++WWp5fKfP//6sZRfkNPS0ggKCtL4lX3IkCEadcTFxeHt7Y2XlxeWlpYa91+7dk2j3NP5BSUlJdGrVy8aN27MuXPn+OKLL5gzZw6rV69WlzE3N8fb25uVK1eW+qxlUaGg9pNPPqFZs2b88ccfnD9/nvPnz3Pr1i3s7Oz45JNPtNKx6kylp4flpLzdEx6uW0dWwr1Ka6uVV3dq1KlLysMHRBw9VK5786cgXL16VWNfXiGEEKIypDx6wu1rj0h59HyOajc0NMTa2pqGDRvSv39/evTowf79+9X5ubm5LFy4EDs7O4yNjXF1deWnn/53YuejR494++23sbCwwNjYmObNm6v32g8ODsba2prFixfTunVrmjVrRu/evfnmm2/UOzBt3ryZDRs28OOPPzJz5kzatWtHkyZN8PHx4eDBg3Tt2lXdlqOjI127di3Tyasff/wxS5YsUY+Ulvb8+VedOnVKLL9nzx4MDQ01pmkaGxtr1KGrq8vBgwc1At98lpaWGmV1StgFasOGDWRmZrJmzRqcnJx46623+OSTT1iyZIlGuX79+rFx48YS+11WFQpqjxw5wuLFi6lbt646rV69eixatIgjR45opWPVnWm3bhi7uaE8ecL9FSsqrR09fX08+vYH4MzOn8jNLfsUj/r161O3bl2ysrK4evVqJfVQCCHE342iKGRl5JTrunz4NutnnmDH0gusn3mCy4dvl7sORVEq3OewsDBOnDiBgYGBOm3hwoWsX7+ewMBAwsPDmTBhAsOHD1fHKn5+fkRERLB3716uXLnCypUr1SeXWltbExcXx9GjR4ttc8OGDTg6OuLj41MoT6VSYWZmppG2aNEitm7dytmzZ0t8lqFDh2Jvb89nn31WYrnDhw9jaWmJo6MjY8eO5cGDByWWP3bsGO7u7iWWWb9+PSYmJrz5ZuHtRNu0aYONjQ09e/bk999/L7GekydP0qVLF42/D29vb65du6Zx+mz79u25ffu2Vub4VmhLL0NDw0JzPSBvd4KCnf87U6lUWE6eROzbw3m8dSt1R/pi2LRppbTl0rMPIds28yjuLtdDTuDo+XKZ++ji4sLhw4e5ePGieuRWCCGEKEl2Zi6r/1XxQSpFgaMbIzm6MbJc943+ygt9Q90yl9+1axempqZkZ2eTkZGBjo4Oy5cvB/JOD/X39+fAgQPqXZGaNm3K8ePHWbVqFV5eXty6dQs3Nzc8PDwA1PNMAQYNGsS+ffvw8vLC2tqajh070r17d9555x31nvbXr19X7zZUFm3btmXw4MFMmzZNY/emp6lUKhYtWkS/fv2YMGGCxtTLfL1792bAgAHY2dkRHR3NzJkz6dOnDydPnkRXt+j3MDY2Fltb2xL7GBQUxLBhwzTOA7CxsSEwMBAPDw8yMjL49ttveeWVVwgJCaFt27ZF1hMfH4+dnZ1GmpWVlTovf1Q5vz+xsbEa739FVGik9rXXXmP06NGEhISgKAqKonDq1CnGjBnD65W4d2t1Y+Lujmm3bpCTw59LAyqtHQMjY9z69AMgZNvmcn2TdXFxAfJOgZPNtoUQQvyddO3aldDQUEJCQvD19eXdd99l4MCBQN76n7S0NHr27Impqan6Wr9+vXr9z9ixY9m4cSNt2rRh6tSpnDhxQl23rq4ua9eu5fbt2yxevJj69evj7++Pk5MTcXFxABUaWZ4/fz7Hjh3TmG9bFG9vbzp37oyfn1+R+W+99Ravv/46zs7O9O/fn127dnHmzBkOHz5cbJ3p6ekYGRkVm3/y5EmuXLlSaOqBo6MjH374Ie7u7nTq1Ik1a9bQqVMnli5dWuIzlEV+8KyNU+4qNFL79ddf4+vri6enp3oD56ysLHx8fAgICHjmTr1ILCeMJ+XwYZL37yc9NBTjNm0qpR233v04+8s2/oyN4WboOezcPMp0X926dWnYsCF//PEHYWFhdOrUqVL6J4QQ4u9Dz0CH0V95lbl8yuMMfpxzioIxnkoFQ+d0xLR24cOOSmq3PGrUqIG9vT0Aa9aswdXVVb0QKiUlBYDdu3er99PPl38AU58+fYiNjWXPnj3s37+f7t27M27cOI1FWvXr12fEiBGMGDGCefPm4eDgQGBgIHPnzsXBwaHc0/uaNWvGBx98wPTp0wkKKnmx+aJFi/D09GTKlCml1tu0aVPMzc2Jioqie/fuRZYxNzfX+On/ad9++y1t2rQpdYoC5E0bOH78eLH51tbWJCQkaKTlvy64qP/hw4cA6p2tnkWFRmpr167Njh07iIyM5KeffuKnn34iMjKSbdu2aZzu9U9g2Lw5Zm/0ByDhyy+faT5QSYxr1sKlZ96xwSHbN5fr3vzRWtkFQQghRFmoVCr0DXXLfNWxMuGV4S1Q/RVVqHTgleEtqGNlUq56nuVYdx0dHWbOnMmnn35Keno6rVq1wtDQkFu3bmFvb69xNWzYUH2fhYUFvr6+fP/99wQEBGiszn9anTp1sLGxITU1FYBhw4YRGRnJjh07CpVVFIXExMQi65k1axaRkZGlLpBq3749AwYMYPr06aU+/+3bt0s9RdTNzY2IiIgi81JSUti8eXORC8SKEhoaWmJbnp6eHD16lKys/+3ctH//fhwdHTUWtIWFhaGvr4+Tk1OZ2i1JmUdqJ06cWGL+oUP/W5n/9Mq2vzuLjz4iaddu0s+eI+XwYWoWWO2oTR59+xP66y/cuRrB7SthNGjZukz3OTk58euvv5KQkEBCQoJ6TosQQgihLa1esqVRq7ok3kvHzNIY0zrF/8xdWQYNGsSUKVNYsWIFkydPZvLkyUyYMIHc3Fw6d+5MYmIiv//+O7Vq1cLX15dZs2bh7u6Ok5MTGRkZ7Nq1i5YtWwKwatUqQkNDeeONN2jWrBlPnjxh/fr1hIeHs2zZMgAGDx7Mtm3bGDp0KJ9++im9evXCwsKCy5cvs3TpUj7++GP69+9fqJ9WVlZMnDiRL774otRnWrBgAU5OTujp/S9kS0lJYe7cuQwcOBBra2uio6OZOnUq9vb2eHt7F1uXt7c3M2bM4NGjR4V2Sti0aRPZ2dkMHz680H0BAQHY2dnh5OTEkydP+Pbbbzl48KDGFIrly5ezbds29VzhYcOGMXfuXEaNGsW0adMICwvjq6++KjRl4dixY7z88ssac3grqsxB7YULF8pU7lm+Zb2o9G1sqDtiOA++DeLPJUsw7dIFVTGTtJ+Fad16OHn14NJvv3J6+5YyB7X5G0ZfvXqVS5cu0bNnT633TQghhDCtY1QlwWw+PT09PvroIxYvXszYsWOZN28eFhYWLFy4kBs3blC7dm3atm3LzJkzATAwMGDGjBncvHkTY2NjXn75ZfXoaf7P62PGjOHu3buYmpri5OTE9u3b8fLKm5qhUqn44YcfWL16NWvWrGHBggXo6enRvHlz3nnnnRIDzMmTJ7Ny5cpSD4twcHDgvffe0xhB1tXV5dKlS6xbt47Hjx9ja2tLr169mDdvnnpqRVGcnZ1p27Ytmzdv5sMPP9TICwoKYsCAAUX+4p6ZmcmkSZO4c+cOJiYmuLi4cODAAY0ty+7fv69xVoGZmRnBwcGMGzcOd3d3zM3NmTVrFqNHj9aoe+PGjRqHSjwLlVJZv5e/QJKSkjAzMyMxMVG9orG8chITierlTW5iIjb+/tQe8IaWe5nncXwca8Z/iKLkMnzRV1jZFV4RWZQrV66wadMmatasyYQJE0rcW04IIcTfmzY+98SLaffu3UyZMoWwsLAqjwX27t3LpEmTuHTpksZIdEVJZKMlumZmmP/17ePPr78mV4vH9BVU29oGx055W3qd3vFTKaX/p3nz5hgZGZGcnFwtznsWQgghxPPXt29fRo8ezZ07d6q6K6SmprJ27VqtBLQgQa1W1Rn+Nno2NmTHx/Noww+V1k57n7wNkSNPHefh3bL9o9TT01NPwr506VKl9U0IIYQQ1dv48eM1FstVlTfffJMOHTporT4JarVIx9AQi48/BuD+6tXkFLPq8VlZNLajadt2oCic2Vn20dr8wxciIiLIzMyslL4JIYQQQlQFCWq1zMzndQyb25ObmMiDb7+ttHY6vDEYgIijh0i6/2eZ7mnYsCG1a9cmMzOTa9euVVrfhBBCCCGeNwlqtUylq4vFX9ufPVz/X7Li4yulHVuHljRs5UxuTjbndm0rW9/+OjYXZAqCEEIIIf5eJKitBKavvIKxhztKRgZ//rWXXWVo338QAJcO7iMtqWxTHfKD2qioKPVpK0IIIYQQLzoJaiuBSqXCctIkABK3bSfj+vVKaaexixtWTe3Jzsjgwt6dZbrH3Nyc+vXroygKYWFhldIvIYQQQojnTYLaSmLi5kbNnj0hN5d7SwMqpQ2VSkWH/nlzay/8uouMtLQy3SdTEIQQQgjxdyNBbSWymDABdHVJOXiQtHPnKqUN+3YdqWvbgIy0VC7u31Ome1q3bo2Ojg53797lzz/LtshMCCGEEKI6k6C2Ehk2taP2wIEA3Pvy31TG4W0qHR313Npzu7eTlZlR6j01atTA3t4ekNFaIYQQ4p8kMzMTe3t7Tpw4UdVdITAwkH79+mmtPglqK5n5uHGojIxIv3CBlIMHK6WNFi95UdPcgrTEx4QfOlCmewpOQcjNza2UfgkhhBCVZeTIkahUKlQqFfr6+tjZ2TF16lSeaPFEzyNHjtCtWzfq1q2LiYkJzZs3x9fXV2Ovd0VRWL16NR06dMDU1JTatWvj4eFBQEAAaX9NC5wzZw4qlYoxY8Zo1B8aGopKpVKf9Hnz5s28dTmWliQnJ2uUbdOmDXPmzNFIu3LlCq+//jpmZmbUqFGDdu3acevWrRKfKTAwEDs7Ozp16gTA4cOH1e/j09eZM2c0+vX0derUqRLbunXrFn379sXExARLS0umTJlCdna2Ov+9997j/PnzHDt2rMR6ykqC2kqmb2VJXV9fAO4tWYpS4C9TW3T19GjXbwAAZ375mZwytOHo6IihoSGJiYml/g8ghBBClEXyg/vcCrtE8oP7z6W93r17ExcXx40bN1i6dCmrVq1i9uzZWqk7IiKC3r174+HhwdGjR7l8+TLLli3DwMCAnJwcdbkRI0Ywfvx4fHx8OHToEKGhofj5+bFjxw6Cg4PV5YyMjAgKCuJ6GRaPJycn8+WXX5ZYJjo6ms6dO9OiRQsOHz7MpUuX8PPzw8jIqNh7FEVh+fLljBo1Sp3WqVMn4uLiNK73338fOzs7PDw8NO4/cOCARjl3d/di28rJyaFv375kZmZy4sQJ1q1bx3fffcesWbPUZQwMDBg2bBhff/11aW9J2ShCSUxMVAAlMTGxUurPTkpSrrXvoEQ4tlAebdlSKW1kZjxR/vPB28qXg/sq4Ud+K9M927dvV2bPnq3s2LGjUvokhBCieirtcy83N1fJTE8v13Xh113Kv4e8pnw5uK/y7yGvKRd+3VXuOnJzc8v8DL6+voqPj49G2oABAxQ3Nzf165ycHMXf319p0qSJYmRkpLi4uChbCnwOP3z4UBk2bJhibm6uGBkZKfb29sqaNWsURVGUpUuXKk2aNCmxD5s2bVIAZfv27UW+h48fP1YURVFmz56tuLq6Kj179lQGDRqkLnPhwgUFUGJiYhRFUZSYmBgFUKZMmaKYmpoqCQkJ6rKurq7K7Nmz1a+HDBmiDB8+vOQ36SlnzpxRdHR0lKSkpGLLZGZmKhYWFspnn32mTsvv14ULF8rc1p49exQdHR0lPj5enbZy5UqlVq1aSkZGhjrtyJEjioGBgZKWllauZymKnnZCY1ES3Zo1qTd2DPcWfc6fXy+jVt++6Bgba7UNfQND2vZ5neMb13N6x0+07PwKKp2SB+JdXFy4cOEC4eHh9OnTB319fa32SQghxIspOyODr33frPD9iqLw25qV/LZmZbnu+2TdT+iXMNJYkrCwME6cOEHjxo3VaQsXLuT7778nMDCQ5s2bc/ToUYYPH46FhQVeXl74+fkRERHB3r17MTc3JyoqivT0dACsra2Ji4vj6NGjdOnSpcg2N2zYgKOjIz4+PoXyVCoVZmZmGmmLFi2iXbt2nD17ttAoaEFDhw5l//79fPbZZyxfvrxQfm5uLrt372bq1Kl4e3tz4cIF7OzsmDFjBv379y+23mPHjuHg4EDNmjWLLbNz504ePHjAu+++Wyjv9ddf58mTJzg4ODB16lRef/31Yus5efIkzs7OWFlZqdO8vb0ZO3Ys4eHhuLm5AeDh4UF2djYhISG88sorxdZXFjL94DmpM2wY+ra2ZN+7x8P/fl8pbbTx7ouBsQkPbt8i6lxIqeUbN25MrVq1yMjIIDIyslL6JIQQQlSWXbt2YWpqipGREc7Ozty7d48pU6YAkJGRgb+/P2vWrMHb25umTZsycuRIhg8fzqpVq4C8OZ9ubm54eHjQpEkTevTooV64NGjQIIYOHYqXlxc2Nja88cYbLF++nKSkJHX7169fx9HRscz9bdu2LYMHD2batGklllOpVCxatIjVq1cTHR1dKP/evXukpKSwaNEievfuTXBwMG+88QYDBgzgyJEjxdYbGxuLra1tiW0HBQXh7e1NgwYN1Gmmpqb8+9//ZsuWLezevZvOnTvTv39/du4sfo/8+Ph4jYAWUL+OL3DaqomJCWZmZsTGxpbYr7KQkdrnRMfAAIvx/+Lu1Gk8+OYbag96E706dbTahqFJDdp49+X09i2c3rYZe4+OqFSq4vuko4OLiwvHjx/n0qVLODk5abU/QgghXkx6hoZ8su6nMpdPfviA7yaO0djlR6Wjw8h/r6Rm3Xrlarc8unbtysqVK0lNTWXp0qXo6ekx8K9dh6KiokhLS6Nnz54a92RmZqpHCceOHcvAgQM5f/48vXr1on///uoFVLq6uqxdu5b58+dz8OBBQkJC8Pf35/PPP+f06dPY2NhUaFej+fPn07JlS4KDg7G0tCy2nLe3N507d8bPz48ffvhBIy9/gbePjw8TJkwA8haSnThxgsDAQLy8vIqsMz09vcQ5t7dv32bfvn1s3rxZI93c3JyJEyeqX7dr1467d+/yxRdflDhaW1bGxsbqRXXPQkZqn6Nar72GYYsW5CYn82D1N5XShvurPujpGxAffZ1bYRdLLZ+/C8L169dJTU2tlD4JIYR4sahUKvSNjMp81bWtT8/RH6unval0dOj5wUfUta1frnpKGogpSv4Wla6urqxZs4aQkBCCgoIA1EfB7969m9DQUPUVERHBTz/lBex9+vQhNjaWCRMmcPfuXbp3787kyZM12qhfvz4jRoxg+fLlhIeH8+TJEwIDAwFwcHDg6tWr5epzs2bN+OCDD5g+fXqpQfGiRYvYtGkTFy5c0Eg3NzdHT0+PVq1aaaS3bNmyxMXf5ubmPHr0qNj8tWvXUq9evTIFqh06dCAqKqrYfGtraxISEjTS8l9bW1trpD98+BALC4tS2yzNCxXULlq0CJVKxfjx49VpT548Ydy4cdSrVw9TU1MGDhxY6E2sLlQ6OlhOyvum8+j778m6c0frbZiY1ca5uzcAp7dvLqU0WFpaYmNjQ25uLuHh4VrvjxBCiH8G5269+GD5GgbP8ueD5Wtw7tbrubavo6PDzJkz+fTTT0lPT6dVq1YYGhpy69Yt7O3tNa6GDRuq77OwsMDX15fvv/+egIAAVq9eXWwbderUwcbGRj0INGzYMCIjI9mxY0ehsoqikJiYWGQ9s2bNIjIyko0bN5b4TO3bt2fAgAFMnz5dI93AwIB27dpx7do1jfTIyEiNOcVPc3Nz4+rVq0UG04qisHbtWt55550yrbEJDQ3Fxsam2HxPT08uX77MvXv31Gn79++nVq1aGsF4dHQ0T548UY+eP4sXJqg9c+YMq1atUo8s5pswYQK//PILW7Zs4ciRI9y9e5cBAwZUUS9LV6NzZ0w6dEDJyuLPZYUnf2uDR7830NHV5VbYJeKuXyu1vBybK4QQQhtq1jOnoZMLNeuZV0n7gwYNQldXlxUrVlCzZk0mT57MhAkTWLduHdHR0Zw/f55ly5axbt06IC+43LFjB1FRUYSHh7Nr1y5atmwJwKpVqxg7dizBwcFER0cTHh7OtGnTCA8PV8+7HTx4MEOGDGHo0KH4+/tz9uxZYmNj2bVrFz169ODQoUNF9tPKyoqJEyeWaSurBQsWcPDgwUIB7JQpU9i0aRPffPMNUVFRLF++nF9++YX/+7//K7aurl27kpKSUuQg1sGDB4mJieH9998vlLdu3Tp+/PFHrl69ytWrV9VzlT/++GN1mW3bttGiRQv16169etGqVStGjBjBxYsX2bdvH59++injxo3DsMA0k2PHjtG0aVOaNWtW6ntRqmfeP+E5SE5OVpo3b67s379f8fLyUv71r38piqIojx8/VvT19TW257hy5YoCKCdPnixz/ZW9pdfT0i5dUiIcWygRLVoq6VevVUobe1csVb4c3FfZtnheqWWTkpKUOXPmKLNnz1bu379fKf0RQghRfTzvz73KUNSWXoqiKAsXLlQsLCyUlJQUJTc3VwkICFAcHR0VfX19xcLCQvH29laOHDmiKIqizJs3T2nZsqVibGys1K1bV/Hx8VFu3LihKIqinD9/Xhk+fLhiZ2enGBoaKvXq1VO6dOmi7Ny5U6O9nJwcZeXKlUq7du0UExMTpVatWoq7u7vy1Vdfqbepyt/Sq6DExETF3Ny8yC29nt46a/To0QqgsaWXoihKUFCQYm9vrxgZGSmurq5Fbi32tMGDByvTp08vlD506FClU6dORd7z3XffKS1btlQ/X/v27TViL0VRlLVr1ypPh5U3b95U+vTpoxgbGyvm5ubKpEmTlKysLI0yvXr1UhYuXFhqv8tCpSiVcHarlvn6+lK3bl2WLl3KK6+8Qps2bQgICODgwYN0796dR48eUbt2bXX5xo0bM378ePXk6adlZGSQkfG/42STkpJo2LAh9+/fp1atWpX9OADET5pMSnAwJl26YLtC+yO2D+/e5vtpn4Ci8PbCAOo1LP7nCIAff/yRGzdu8PLLLxe7dYkQQoi/h6SkJMzNzUlMTHxun3uierh06RI9e/YkOjoaU1PTKu1LeHg43bp1IzIystD2ZxVR7Xc/2LhxI+fPn1cf1VZQfHw8BgYGGgEt5A3rF9wu4mkLFy5k7ty5hdKDg4MxMTF55j6Xhb6rC00OHCDt6FEOLV9OetOmWm+jRoMmpP4Rw85Vy7Hq1LXEsvmno4SEhJCcnFzuyfpCCCFeHNpYaS5eTC4uLnz++efExMTg7OxcpX2Ji4tj/fr1WglooZoHtX/88Qf/+te/2L9/f4lbUJTXjBkzNLamyB+p7dWr13P9xnrv1i2SNm3G/sQJGowbp/VA8l5LRzb6TSbl1g0GfDIJM0vrYstmZmYSEBBAZmYmrq6uGvvTCSGE+HspuNeq+OcZOXJkVXcBgB49emi1vmod1J47d4579+7Rtm1bdVpOTg5Hjx5l+fLl7Nu3j8zMTB4/fqwxWpuQkFBou4iCDA0NNSYp59PX13+up2pZffQRyb/sIuNyGE8OHaaWt3ZXitZ3aEFjFzdiL10gdO9Oerw/rtiy+vr6tGrViosXLxIeHo6dnZ1W+yKEEKL6kBMkxd9Rtd79oHv37ly+fFljfzkPDw/efvtt9X/r6+vz22+/qe+5du0at27dwtPTswp7XjZ6FhbU++vb0p9Ll6JkZWm9jQ5vDAYg7PABUh49LLFs/i4I4eHhZGdna70vQgghhBCVpVoHtTVr1qR169YaV40aNahXrx6tW7fGzMyMUaNGMXHiRA4dOsS5c+d499138fT0pGPHjlXd/TKp+9676NatS+bNmzze+rPW62/QsjW2Di3Jycri3O7tJZa1s7OjZs2apKenl7ihshBCCCFEdVOtg9qyWLp0Ka+99hoDBw6kS5cuWFtb8/PP2g8OK4uuqSnmY8cC8OeK5eRqefK+SqWiff9BAFzcv5cnf52wUhQdHR31pPGLF0s/jUwIIYQQorp44YLaw4cPExAQoH5tZGTEihUrePjwIampqfz8888lzqetjuoMGYx+gwbk/Hmfh+vXa73+pm3bYdGoCVlP0rmw75cSy+ZPQYiMjCQ9PV3rfRFCCCGEqAwvXFD7d6QyMMDir6N/H3zzLdklnMtcofoLjNae3/sLWU+eFFvW2toaS0tLcnJyiIiI0Go/hBBCCCEqiwS11UStV/tg2Koluamp3F+5Uuv1O3TsTG0rG54kJ3Hpt30llnV1dQVkCoIQQgghXhwS1FYTKh0dLCdNAuDRjxvJvH1bq/Xr6OrSzmcgAGd3/Ux2CTsttG7dGoBbt27xSMujxkIIIYSoOg8ePMDS0pKbN29WdVeYPn06H3/8sdbqk6C2GjF96SVqdOoEWVn8+dXXWq+/VZfumNapS8rDB1w5dqjYcmZmZup9ai9fvqz1fgghhBDPauTIkahUKlQqFfr6+tjZ2TF16lSelDDFrryOHDlCt27dqFu3LiYmJjRv3hxfX18yMzPVZRRFYfXq1XTo0AFTU1Nq166Nh4cHAQEB6pPb5syZg0qlYsyYMRr1h4aGolKp1AHmzZs3UalUWFpakpycrFG2TZs2zJkzR/06/9mfvr744osSn2nBggX4+PjQpEkTAL777rti67p37x6Qt56pqPySTm+FvCN5X375ZYyMjGjYsCGLFy/WyJ88eTLr1q3jxo0bJdZTVhLUVjMWk/JOOkv65ReeaHlOq56+Pu6vvQHA6R1byM3NKbZswSkIiqJotR9CCCH+nrITM3gS/ZjsxIzn0l7v3r2Ji4vjxo0bLF26lFWrVjF79myt1B0REUHv3r3x8PDg6NGjXL58mWXLlmFgYKA+Wh5gxIgRjB8/Hh8fHw4dOkRoaCh+fn7s2LGD4OBgdTkjIyOCgoK4fv16qW0nJyfz5ZdfllgmLi5O41qzZg0qlYqBAwcWe09aWhpBQUGMGjVKnTZkyJBCdXl7e+Pl5YWlpaXG/deuXdMo93R+QUlJSfTq1YvGjRtz7tw5vvjiC+bMmcPq1avVZczNzfH29mallqZdSlBbzRg7OVGrb18A7i1ZqvX6XXr0xsi0Jo/j44g89Xux5Vq2bImenh4PHjzg7t27Wu+HEEKI6ktRFHIzc8p1JZ+8S/yi09z/5jLxi06TfPJuueso7yCKoaEh1tbWNGzYkP79+9OjRw/279+vzs/NzWXhwoXY2dlhbGyMq6srP/30kzr/0aNHvP3221hYWGBsbEzz5s1Zu3YtAMHBwVhbW7N48WJat25Ns2bN6N27N9988w3GxsYAbN68mQ0bNvDjjz8yc+ZM2rVrR5MmTfDx8eHgwYN07dpV3ZajoyNdu3bl//2//1fqc3388ccsWbJEPVJaFGtra41rx44ddO3alaZNmxZ7z549ezA0NNTYy9/Y2FijHl1dXQ4ePKgR+OaztLTUKKujU3wYuWHDBjIzM1mzZg1OTk689dZbfPLJJyxZskSjXL9+/di4cWNJb0eZVetjcv+pLMb/i6TgYFKPHyf15ElqaPF0NAMjY9x69+PkTz9wevsWHD1fRqVSFSpnaGhIixYtCAsL49KlS9SvX19rfRBCCFG9KVm53J114hkqgMQd0STuiC7XbbafdUJloFuhJsPCwjhx4gSNGzdWpy1cuJDvv/+ewMBAmjdvztGjRxk+fDgWFhZ4eXnh5+dHREQEe/fuxdzcnKioKPV2ltbW1sTFxXH06FG6dOlSZJsbNmzA0dERHx+fQnkqlQozMzONtEWLFtGuXTvOnj2Lh4dHsc8ydOhQ9u/fz2effcby5ctLffaEhAR2797NunXrSix37Ngx3N3dSyyzfv16TExMePPNNwvltWnThoyMDFq3bs2cOXN46aWXiq3n5MmTdOnSBQMDA3Wat7c3n3/+OY8ePaJOnToAtG/fntu3b3Pz5k31lIiKkpHaasigYUPqDBkCwL0v/42Sm6vV+t369EPf0Ig/Y2OICT1bbLn8KQiXL1/W+KlFCCGEqA527dqFqakpRkZGODs7c+/ePaZMmQJARkYG/v7+rFmzBm9vb5o2bcrIkSMZPnw4q1atAvIWRLu5ueHh4UGTJk3o0aMH/fr1A2DQoEEMHToULy8vbGxseOONN1i+fDlJSUnq9q9fv46jo2OZ+9u2bVsGDx7MtGnTSiynUqlYtGgRq1evJjq69C8G69ato2bNmgwYMKDEcrGxsdja2pZYJigoiGHDhqlHowFsbGwIDAxk69atbN26lYYNG/LKK69w/vz5YuuJj4/HyspKIy3/dcG5uPn9iY2NLbFfZSEjtdWU+dgxJP78M0/Cw0n+9Vdqvfqq1uo2Nq2Ja69XOfvLz4Rs20JTt3ZFlmvatCk1atQgNTWV6OhoHBwctNYHIYQQ1ZdKXwfbzzqVuXxOYgYJS85BwdkDKrCa6I6umWG52i2Prl27snLlSlJTU1m6dCl6enrqOaVRUVGkpaXRs2dPjXsyMzNxc3MDYOzYsQwcOJDz58/Tq1cv+vfvT6dOec+tq6vL2rVrmT9/PgcPHiQkJAR/f38+//xzTp8+jY2NTYXWnMyfP5+WLVsSHBxc4pxUb29vOnfujJ+fHz/88EOJda5Zs4a3334bIyOjEsulp6eXWObkyZNcuXKF//73vxrpjo6OGsF7p06diI6OZunSpYXKlld+8JymhRNVZaS2mtKrV4+6o94D4F7AVygFVlpqg/urPujq6XH3WgS3r4QVWUZXV1e9vdelS5e02r4QQojqS6VSoWOgW+ZL38KEOgOaQ/5sNhXUGdAcfQuTctVT1HS4ktSoUQN7e3tcXV1Zs2YNISEhBAUFAZDy17Hwu3fvJjQ0VH1FRESo59X26dOH2NhYJkyYwN27d+nevTuTJ0/WaKN+/fqMGDGC5cuXEx4ezpMnTwgMDATAwcGBq1evlqvPzZo144MPPmD69OmlBsWLFi1i06ZNXLhwodgyx44d49q1a7z//vultm1ubl7iVp3ffvstbdq0KXWKAuRNG4iKiio239ramoSEBI20/NcFT359+PAhABYWFqW2WRoJaquxeiNHomtuTtatWzzaskWrdZvWrYfTKz0ACNlefN35UxCuXr2q1W1ShBBC/L3UaGeN9fT2mH/gjPX09tRo93yPrNfR0WHmzJl8+umnpKen06pVKwwNDbl16xb29vYaV8OGDdX3WVhY4Ovry/fff09AQIDG6vyn1alTBxsbG1JTUwEYNmwYkZGR7Nixo1BZRVFITEwssp5Zs2YRGRlZ6gKp9u3bM2DAAKZPn15smaCgINzd3dWf1yVxc3Mr9rTQlJQUNm/eXOQCsaKEhoZiY2NTbL6npydHjx4lq8C++Pv378fR0VE9nxby5kLr6+vj5ORUpnZLIkFtNaZTowYW4/4PgPv/WUlOSqpW62/XbyAqlQ43Q8+REFP0nB0bGxvMzc3Jzs7mypUrWm1fCCHE34uemSFGzWqjV44pB9o0aNAgdHV1WbFiBTVr1mTy5MlMmDCBdevWER0dzfnz51m2bJl6QdWsWbPYsWMHUVFRhIeHs2vXLlq2bAnAqlWrGDt2LMHBwURHRxMeHs60adMIDw9Xz7sdPHgwQ4YMYejQofj7+3P27FliY2PZtWsXPXr04NChoveEt7KyYuLEiXz9del70i9YsICDBw9y7dq1QnlJSUls2bKlTKO0kDelITw8vMjR2k2bNpGdnc3w4cML5QUEBKjfp7CwMMaPH8/BgwcZN26cuszy5cvp3r27+vWwYcMwMDBg1KhRhIeHs2nTJr766ismTpyoUfexY8d4+eWXNebwVpQEtdVc7TffRL9xI3IePODhd99pt25rGxw7vQzA6WJGa1UqFS4uLoBMQRBCCFG96enp8dFHH7F48WJSU1OZN28efn5+LFy4kJYtW9K7d292796tPmDIwMCAGTNm4OLiQpcuXdDV1VWPnrZv356UlBTGjBmDk5MTXl5enDp1iu3bt+Pl5QXkfUb+8MMPLFmyRJ3u4uLCnDlz8PHxwdvbu9i+Tp48GVNT01KfycHBgffee6/IX0s3btyIoigMHTq0TO+Ps7Mzbdu2ZfPmzYXygoKCGDBgALVr1y6Ul5mZyaRJk3B2dsbLy4uLFy9y4MABjSD2/v37GovazMzMCA4OJiYmBnd3dyZNmsSsWbMYPXp0oWf44IMPytT/0qgU2VmfpKQkzMzMSExMpFatWlXdnUKSfv2VO+MnoGNiQrPgfeiZm2ut7j9v3WT9lI9ApeLdJSupa9ugUJnHjx8TEBAAwIQJEwptUSKEEOLFUt0/90Tl2b17N1OmTCEsLKzEfWafh7179zJp0iQuXbqEnt6z710gI7UvgJre3hg5O5Oblsb9lYFarduiUROaurcHReHMzq1Flqldu7Z63z85NlcIIYR4cfXt25fRo0dz586dqu4KqamprF27VisBLUhQ+0JQqVRYTpoEwKNNm8i8dUur9XfoPxiAiKMHSbpf9Okl+VMQ5NhcIYQQ4sU2fvx4jcVyVeXNN9+kQ4cOWqtPgtoXRI2OHajx8suQnc2ff00F0BZbhxY0bOVMbk4OZ3dtK7JMq1at0NXV5c8//9TYNFkIIYQQojqQoPYFYjlpIqhUJO3ZS/rloveWraj2b+SN1l7+LZi0pMJbkBgbG6s3XpYFY0IIIYSobiSofYEYtWhBrX6vAXDv3//W6jSAxs5tsGranOzMDM7v2VlkmfwpCHJsrhBCCCGqGwlqXzAWn/wLlb4+aadOkfr7Ca3Vq1Kp6NB/EACh+3aRUcRxdfb29hgbG5OSkkJMTIzW2hZCCCGEeFYS1L5gDBrUp86wYcBfo7W5uVqr275dR+raNiAjLZWL+/cUytfT05Njc4UQQghRLUlQ+wKqN+ZDdExNybhyhaTdhYPPilLp6ND+r9Hac7u3k5WZUahM/hSEK1eukJFROF8IIYQQoipIUPsC0qtTh3p/HYn3Z0AAuZmZWqu7xUte1LKwJC3xMeGHDhTKb9CgAXXr1iUrK4urV69qrV0hhBBCiGchQe0Lqu47I9CzsCDrzh0eb9yktXp19fTw6DcAgDO/bCUnO1sjX47NFUIIIV5cmZmZ2Nvbc+KE9tblVFRgYCD9+vXTWn0S1L6gdExMMP/oIwDur1xJTkqK1upu3bUnJma1SfrzHtdOHC2Unx/U3rhxg+TkZK21K4QQQpTVyJEjUalUqFQq9PX1sbOzY+rUqTx58kRrbRw5coRu3bpRt25dTExMaN68Ob6+vmQW+IVUURRWr15Nhw4dMDU1pXbt2nh4eBAQEEDaX4uu58yZg0qlYsyYMRr1h4aGolKpuHnzJgA3b97MO3DJ0rLQ52ubNm2YM2eO+nVKSgofffQRDRo0wNjYmFatWhEYWPqpo4GBgdjZ2dGpUycADh8+rH4fn77OnDmj0a+nr1OnTpXY1q1bt+jbty8mJiZYWloyZcoUsgsMlr333nucP3+eY8eOldrvspCg9gVWe+AADJo0IefRIx4EBWmtXn0DQ9q+6gNAyPYthRaj1a1bl4YNG6IoihybK4QQQi0xMZGYmBgSEwvvd14ZevfuTVxcHDdu3GDp0qWsWrWK2bNna6XuiIgIevfujYeHB0ePHuXy5cssW7YMAwMDjW0tR4wYwfjx4/Hx8eHQoUOEhobi5+fHjh07CA4OVpczMjIiKCiI69evl9p2cnIyX375ZYllJk6cyK+//sr333/PlStXGD9+PB999BE7dxa9LSfkBeDLly9n1KhR6rROnToRFxencb3//vvY2dnh4eGhcf+BAwc0yrm7uxfbVk5ODn379iUzM5MTJ06wbt06vvvuO2bNmqUuY2BgwLBhw/j6669Le0vKRqnG/vOf/yjOzs5KzZo1lZo1ayodO3ZU9uzZo85PT09X/u///k+pW7euUqNGDWXAgAFKfHx8udtJTExUACUxMVGb3X8uEvftUyIcWyhX2rgpmQkJWqv3SWqKsmzkYOXLwX2VyJDfC+WfPn1amT17trJy5UqttSmEEOL5KO1zLzc3V8nIyCjXFRISosyZM0eZPXu2MmfOHCUkJKTcdeTm5pb5GXx9fRUfHx+NtAEDBihubm7q1zk5OYq/v7/SpEkTxcjISHFxcVG2bNmizn/48KEybNgwxdzcXDEyMlLs7e2VNWvWKIqiKEuXLlWaNGlSYh82bdqkAMr27duLfA8fP36sKIqizJ49W3F1dVV69uypDBo0SF3mwoULCqDExMQoiqIoMTExCqBMmTJFMTU1VRIKfK67uroqs2fPVr92cnJSPvvsM40227Ztq/y///f/iu3vmTNnFB0dHSUpKanYMpmZmYqFhYVG3fn9unDhQrH3PW3Pnj2Kjo6ORly2cuVKpVatWkpGRoY67ciRI4qBgYGSlpZW5rqLo6ed0LhyNGjQgEWLFtG8eXMURWHdunX4+Phw4cIFnJycmDBhArt372bLli2YmZnx0UcfMWDAAH7//feq7vpzU7NnT4xdXUm/eJH7//kPNgV+mngWhiY1aOPdl5Btmzm9fQv27TxRqVTqfCcnJ/bu3Ut8fDwJCQlYWVlppV0hhBBVLysrC39//wrfrygKe/bsYc+e8u3QM3PmTAwMDCrUZlhYGCdOnKBx48bqtIULF/L9998TGBhI8+bNOXr0KMOHD8fCwgIvLy/8/PyIiIhg7969mJubExUVRXp6OgDW1tbExcVx9OhRunTpUmSbGzZswNHRER8fn0J5KpUKMzMzjbRFixbRrl07zp49W2gUtKChQ4eyf/9+PvvsM5YvX15kmU6dOrFz507ee+89bG1tOXz4MJGRkSxdurTYeo8dO4aDgwM1a9YstszOnTt58OAB7777bqG8119/nSdPnuDg4MDUqVN5/fXXi63n5MmTODs7a8QH3t7ejB07lvDwcNzc3ADw8PAgOzubkJAQXnnllWLrK4tqPf2gX79+vPrqqzRv3hwHBwcWLFiAqakpp06dIjExkaCgIJYsWUK3bt1wd3dn7dq1nDhxotQ5Hn8nKpUKy8mTAHi85ScytHgoQts+r6NnYEh89HVuXb6okWdiYoKDgwMgC8aEEEJUjV27dmFqaoqRkRHOzs7cu3ePKVOmAJCRkYG/vz9r1qzB29ubpk2bMnLkSIYPH86qVauAvDmfbm5ueHh40KRJE3r06KFeuDRo0CCGDh2Kl5cXNjY2vPHGGyxfvpykpCR1+9evX1cfIV8Wbdu2ZfDgwUybNq3EciqVikWLFrF69Wqio6OLLLNs2TJatWpFgwYNMDAwoHfv3qxYsaLYABwgNjYWW1vbEtsOCgrC29ubBg0aqNNMTU3597//zZYtW9i9ezedO3emf//+JU51iI+PLzTglf86Pj5enWZiYoKZmRmxsbEl9qssqvVIbUE5OTls2bKF1NRUPD09OXfuHFlZWfTo0UNdpkWLFjRq1IiTJ0/SsWPHYuvKyMjQ2GM1/x9oVlYWWVlZlfcQlUS/TRtMvLqQduQoCUuWYrPk39qp16QGTq/04GLwbk5t24htSyeNfCcnJ65evcqlS5fw8vLSGMkVQghRfZX2Waevr8/MmTPLXF9SUhIrVqzQOL5dpVIxbtw4atWqVeZ69PX1y1wWoGvXrqxcuZLU1FSWLl2Knp4eAwcOBCAqKoq0tDR69uypcU9mZqZ6lHDs2LEMHDiQ8+fP06tXL/r3769eQKWrq8vatWuZP38+Bw8eJCQkBH9/fz7//HNOnz6NjY1NhY6rnz9/Pi1btiQ4OBhLS8tiy3l7e9O5c2f8/Pz44YcfCuUvW7aMU6dOsXPnTho3bszRo0cZN24ctra2GrFRQenp6RgZGRXb5u3bt9m3bx+bN2/WSDc3N2fixInq1+3atePu3bt88cUXJY7WlpWxsbF6Ud2zqPZB7eXLl/H09OTJkyeYmpqybds2WrVqRWhoKAYGBtSuXVujvJWVlcY3gKIsXLiQuXPnFkoPDg7GxMREm91/bgzatqXx0WOk7t/PwcBAnjRqpJV6s4xrgUrF7Ygwfl6/FiPz/33rys3NRVdXl+TkZDZv3lzizxlCCCGqj9ICCJVKVa5pAObm5vTr149ffvkFRVFQqVT069cPc3PzZ+1qiWrUqIG9vT0Aa9aswdXVlaCgIEaNGkXKX7sC7d69m/r162vcZ2hoCECfPn2IjY1lz5497N+/n+7duzNu3DiNRVr169dnxIgRjBgxgnnz5uHg4EBgYCBz587FwcGh3Hu2N2vWjA8++IDp06cTVMoi70WLFuHp6akefc6Xnp7OzJkz2bZtG3379gXydiYKDQ3lyy+/LDaoNTc3L3GB99q1a6lXr16ZAtUOHTqwf//+YvOtra05ffq0RlpCQoI6r6CHDx9iYWFRapulqfZBraOjI6GhoSQmJvLTTz/h6+vLkSNHnqnOGTNmaHzjSEpKomHDhvTq1atc3yirm4QbMSTv2IFDyGnqf/ih1kZODzxKIOLoQfTvx/HqO5pzbHR0dLhw4QImJia8+uqrWmnv/7d33/FxVPfexz8z27SrVbW6LUtyr7JxxbhgY2xDKCEkoYTEDgSSG8oNNcEkgRAuMQ4PNfQkYEhCIIUOxgYbd+PecS+Si2RJVtf2mXn+2NVKsiQ39fXvzWvZ2aln1x7vd86eOUcIIUTbqv8TemsZMWIEvXv3prS0lMTExEbtSduaqqo89NBD3HvvvfzgBz9g0KBB2Gw28vPzufjii5vdLjk5mVmzZjFr1iwmTpzIAw880GzPAwkJCaSnp1NTUwPAD37wA2644QY+/PDDRu1qDcOgsrKyyc/h4Ycfpnfv3rzzzjunfE9jxozh2muv5cEHH2wwv/aXZVVt2IrUZDKhn9RjUX0XXHABL7/8cvjC4+TyvvHGG8ycOfOMasw3b95Menp6s8vHjRvH448/TlFRUbhG+osvviA2NpZBgwaF19u/fz8ejydce94SnT7UWq3W8FXYyJEjWbduHc899xzXX389Pp+P8vLyBrW1x48fb3QFcDKbzRa+SqvPYrGc9U8fnUnqL/6X6vnz8axfj2/1apynOInPxtjvXMc3y7/i4MZ1VBQcJalndnjZ8OHD2bRpE7t27eLKK6885wb+Qggh2k9bfdfFxcW1e5it7/vf/z4PPPAAL774Ivfffz/3338/99xzD7quM2HCBCoqKli5ciWxsbHMmjWLhx9+mJEjRzJ48GC8Xi+ffPIJAwcOBODVV19l8+bNfOc736F37954PB7eeustduzYwZ/+9CcArrvuOt5//31uvPFGfvOb3zB9+nSSk5PZtm0bzzzzDHfddRfXXHNNo3KmpqZy77338uSTT572PT3++OMMHjwYs7kussXGxnLxxRfzwAMPYLfbycrKYunSpbz11ls8/fTTze5rypQpVFdXs2PHDoYMGdJg2eLFizl48CC3hkYsre/NN9/EarWGg+d7773H66+/zl/+8pfwOu+//z6zZ88O11xPnz6dQYMG8aMf/Yg//vGPFBYW8pvf/IY77rijQQZbvnw5vXr1onfv3qf9LE6nU98o1hRd1/F6vYwcORKLxcKiRYvCy3bv3k1+fj7jxo3rwBJ2HEtGBgk//CEARU89jVGvH72WSMzoQb8xwTZGaz/8T4NlmZmZxMfH4/P52L17d6scTwghhDgXZrOZO++8kz/+8Y/U1NTw2GOP8dvf/pY5c+YwcOBALrvsMj799FNycnKAYMXZ7Nmzyc3NZdKkSZhMpnDt6ZgxY6iuruZ//ud/GDx4MBdffDFff/01H3zwQbjmV1EU3n77bZ5++unw/NzcXH73u9/x7W9/mxkzZjRb1vvvvx+n03na99SvXz9uueWWRoNKvPPOO4wePZqbbrqJQYMG8cQTT/D44483GuChvm7duvGd73yHf/zjH42W/fWvf+Wiiy5iwIABTW772GOPMXLkSMaOHcuHH37Iu+++26CHhIqKigY5wGQy8cknn2AymRg3bhw//OEPmTlzJr///e8b7Pef//wnt91222k/hzOhGOfSyrmdzJ49m8svv5yePXtSVVXF22+/zdy5c1mwYAHTpk3j5z//OZ999hnz5s0jNjaWu+66C+Csh36r/XmgoqKiSzc/ANDKy9k3fQZ6ZSXpT8whvokrxHNx/MA+/j77bhRF5ZbnXiM+ta42fPHixSxbtoy+ffty0003tcrxhBBCtJ1I+t4TZ2fr1q1MmzaN/fv3n1Gobks7duzgkksuYc+ePa1Sw9+pa2qLioqYOXMm/fv3Z+rUqaxbty4caAGeeeYZrrzySr773e8yadIk0tLSeO+99zq41B3LFB9P0k+DVzzFzz+PXq+Xh5ZI7dWH7GEjMAyd9R//t8Gy2mFz9+3bF26YL4QQQojOJzc3l7lz53KwFbsAPVcFBQW89dZbrdZkpVPX1LaXSLti1T0e9s+4jMDx46T86ld0u/nHrbLfI99s591HH8RkNnPrC6/jTEgML3vttdc4duwYl1122Sm7UxNCCNHxIu17Twjo5DW14tyoUVEk/2+wKcaJV15Ba6W7XLsPHExG/0FogQAbPv2gwbJhw4YBMhCDEEIIITqGhNoIFfftb2Pt0xutooITfzl1P3hnSlEUxl7zfQC2fDEfd3VVeNmQIUNQFIVjx45RUlLSKscTQgghhDhTEmojlGI2kxLqi7f0rbfwhzo8bqmcC0aR3DMbv8fN5s8/Cc+v3wH2li1bmttcCCGEEKJNSKiNYM4pU7CPGIHh8VDywgutsk9FURgTqq3dOP8jfB53eFltE4Rt27adsvNnIYQQQojWJqE2gimKQsr99wFQ/t/38O7f3yr77TduAvFp6Xiqq9i2aEF4fv/+/bFarZSXl3P48OFWOZYQQgghxJmQUBvhHCNG4Jw6FXSdomeeaZV9qqqJ0Vd/D4D1H79HwO8HgiPU1A59J00QhBBCCNGeJNSeB1LuuRtUleovF+HauKlV9jlo0iU4ExKpLivlm2WLw/NrmyDs2LEDfyjsCiGEEEK0NQm15wFbnz7Ef/daAIqeeorW6JrYbLEw6qrgPtd99B90PTgkb1ZWFrGxsXi9Xvbu3dvi4wghhBBCnAkJteeJpDvvRLHZcG/YQPVXS1pln0OnziDKGUN5YQF7vl4JgKqqDB06FJAmCEIIIYRoPxJqzxOW1FQSZ84EoOjppzA0rcX7tEbZGXH51QCsff9f4Rrg2iYIe/fuxeVytfg4QgghhBCnI6H2PNLttltR4+Lw7dtPxQcftso+h192JZYoO8X5hzi4aT0AKSkppKWloes6O3bsaJXjCCGEEEKcioTa84gpNpakn/0MgOI//Qnd42nxPu3OGIZNuxyANfVqa3NzcwFpgiCEEEKI9iGh9jyTcNMPMKenEygspOzvf2+VfY684hpMZjPH9uzk6M5gzezQoUNRFIUjR45QWlraKscRQgghhGiOhNrzjGqzkfy//wtAyWt/Risvb/E+nQmJDJkyDYA1H/wLgJiYGHr16gXA1q1bW3wMIYQQQohTkVB7Hoq7+ips/fqhV1ZS8uc/t8o+R131XRRF5dCWjRw/sA9o2AShNboRE0IIIYRojoTa85BiMpFy370AlP3t7/gLClq8z/jUNAaMnwTA2g/+DcDAgQOxWCyUlZVx5MiRFh9DCCGEEKI5EmrPU9GTJuEYPRrD56P4Ty+0yj7HfDs4dO6etasoPXYEq9XKwIEDAWmCIIQQQoi2JaH2PKUoCin33wdAxQcf4Nmzp8X7TOqZTe9RY8EwWPvhf4C6Jgjbt28nEAi0+BhCCCGEEE2RUHsesw8bRsz06aDrFD/zbKvsc8y3vw/AzuVfUVlSRK9evXA6nbjdblauXElFRUWrHEcIIYQQoj4Jtee55LvvBpOJ6q++wrV+fYv3l9FvAJmDc9E1jfWfvI+qqqSkpADw1Vdf8eyzz7Jx48YWH0cIIYQQoj4Jtec5W68c4r8XbAtb9OT/a5VeCsZecx0A2xYt5PiRwxw8eDC8zDAMPv74Y6mxFUIIIUSrklBbz/Ga4x1dhA6RdMftKHY77i1bqPryyxbvr+fQYaT17kvA52XNgs8aBWXDMPjb3/7WIOwKIYQQQrSEhNp6vvPRd3hv73sdXYx2Z0lJIXHWTACKn34Go4U3dCmKwphrgm1rD65ehqIojdYpKSnhzTff5K233uLo0aMtOp4QQgghhITaenRD59HVj1JYU9jRRWl33W69FVN8PL6DByl/r+XBvs+oC0nsnkmgqoLBPdLDwVZRFKZPn87o0aNRVZUDBw7w5z//mXfffZfi4uIWH1cIIYQQ5yfFkKGeqKysJC4ujoEvD8RkN/GTIT/hFyN+0WQNYyQrfestjv9hDubkZHovXIBqt7dofzuWLuLzl57BERfPdX94hsqqahITE4mLiwOgrKyMJUuWsGXLFiAYeIcNG8bkyZOJj49v6dsRQgjRjNrvvYqKCmJjYzu6OEK0CqmpbcJft/+VH3/+Y3aX7u7oorSr+BtuwNK9O4HiYkrf+luL9zdg/MXEJqfgqignf8MacnJywoEWICEhge985zvcfvvtDBgwAMMw2Lx5M88//zyfffYZ1dXVLS6DEEIIIc4PnTrUzpkzh9GjRxMTE0NKSgrXXHMNu3c3DJoej4c77riDbt264XQ6+e53v8vx4+d2w5eqqEzLmobdbGdj0Uau/+R65q6dS5WvqjXeTqenWq0k3/0LAE78+c8EyspatD+T2czoq74LwJr3/8WhLZuoOlHSaL2UlBRuuOEGbr31VnJyctB1nbVr1/Lcc8+xaNEi3G53i8ohhBBCiMjXqZsfXHbZZdxwww2MHj2aQCDAQw89xPbt2/nmm2+Ijo4G4Oc//zmffvop8+bNIy4ujjvvvBNVVVm5cuUZH6f2Z5g9x/bQN70vhTWFPLnuSRbmLQSgW1Q37ht1H1f2ujLimyQYus7Ba7+Ld9cuEmfNInX2gy3an9/n5ZWf/hBfKJgqisK0n97F0EumN7vNgQMHWLRoUfgGsqioKCZMmMCYMWOwWq0tKo8QQghpfiAiU6cOtScrLi4mJSWFpUuXMmnSJCoqKkhOTubtt9/me6G+Vnft2sXAgQNZvXo1F1544Rntt7mTe9WxVcxZM4dDlYcAGJEygl9f+Gv6JfRr9ffWmVQvX8Hh225DsVjoNX8+1h7dz3lfVSdKeO32m4G6v2aKqnLbC68T0y2p2e0Mw2DXrl0sXrw4fAOZ0+nk4osv5oILLsBsNp9zmYQQ4nwnoVZEoi4Vavft20ffvn3Ztm0bQ4YMYfHixUydOpWysrIGNxZlZWVx9913c8899zS5H6/Xi9frDb+urKwkMzOTkpKSRie3T/Pxj13/4M/b/4xH82BSTNzQ7wZ+lvsznBZnm7zPjmYYBsduuw33mrXEXHUlqX/4wznv6/A323j/Dw83mp8zYjTjb5hJYkaPU26v6zo7duxg6dKl4QEb4uPjmTRpEoMHD0ZVO3ULGiGE6JQqKytJSkqSUCsiSpep7tJ1nbvvvpvx48czZMgQAAoLC7FarY3ulE9NTaWwsPluuebMmcOjjz7aaP7ChQtxOByN5qeSyp3RdzLfPZ8d/h38Y/c/+HDPh1xuv5xcS25ENkmwjRlD1pq1VH7yKdt69cKXkXFO+wm4qgGF+jW1AAc3ruPgxnU40nsQ138IjvQep/wcs7OzOXHiBIWFhZSXl/PRRx+xcOFC0tPTiYuLi8g/AyGEaCsul6ujiyBEq+syofaOO+5g+/btrFixosX7mj17Nvfee2/4dW1N7fTp0095xfoDfsCqY6v444Y/kl+Vz79d/+ZAygEeHPUgveN7t7hcnU3h3n1Uf/45gzZsJOPWW895PzuSEln8+ssYuo6iqgy/7ErKCws4uGk9roIjuAqOEJ+WwbDpVzBw4hSsp+hKzOfzsX79elavXo3H4+HgwYN0796dyZMnk52dfc5lFEKI80llZWVHF0GIVtclmh/ceeedfPjhhyxbtoycnJzw/HNtfnCys21b5NN8vLnjTV7b+hoezYNZMXPTwJv4+fCfE22JPuv311n58vLYf8WVEAjQc948oi8ce877qjpRQnnhMeLTMsJtacuPF7J5wcdsW/wFPnew1sBqdzD0kmkMn3EV8alpze7P7XazcuVK1qxZg9/vB6BXr15MnTqV7t3PvQ2wEEKcD6RNrYhEnTrUGobBXXfdxfvvv8+SJUvo27dvg+W1N4r985//5LvfDXYdtXv3bgYMGNAqN4qdzrHqY/xx3R9ZlL8IgBR7CvePvp/Lsi+LmJ/DC3//GGVvv03U0KFk/+vdNnlfPreLHUsXsenzTygrCA2Zqyj0HjmGEZdfTebg5pt4VFVVsXz5ctavX4+u6wAMHDiQKVOmkJKS0uplFUKISCChVkSiTh1qb7/9dt5++20+/PBD+vfvH54fFxeHPfQT9c9//nM+++wz5s2bR2xsLHfddRcAq1atOuPjtPTkXn5kOXPWzuFw1WEAxqSN4aGxD0VEk4RASQn7ps/AcLno/uyzxF42o82OZeg6h7ZsZOP8jzi0ZWN4flJmFhdcfhUDJ0zGYotqctumRifLzc1l8uTJJCQktFmZhRCiK5JQKyJRpw61zdXOvfHGG/z4xz8GgoMv3Hffffzzn//E6/UyY8YMXnrpJdLSmv/p+mStcXJ7NS9vbH+Dv2z7C17Ni1kx86NBP+J/hv0PDkvjm8+6kuI/vUDJiy9izcqi1ycfo1gsbX7ME0cPs+nzT/hm6SL8Xg8AUc4YcqfOYNj0K4hNSm5yu6KiIhYvXsyuXbsAUFWVUaNGMWnSJJzOyOytQgghzpaEWhGJOnWobS+teXIfqTrC3HVzWXJ4CQApjhQeGP0AM7JmdNkmCVp1DfunT0crLSXtkYdJuPHGdju2p6aa7YsXsmnBp1QWB0eKU1SVvmMuYsTlV5PRf2CTn+uRI0dYvHgxBw4cAMBisXDhhRdy0UUXhWv5hRDifCWhVkQiCbW0zcm99PBSnlj7BEeqjwBwYfqFzB47m15xvVpl/+2t9O//4Pj//R+mpCT6LPgcNbp9b4jTdY39G9ay6bOPOPzNtvD81F59GHH51fQbNxFzEzXITY1ONn78eMaOHSujkwkhzlsSakUkklBL253cXs3L69te5y/b/oJP92FWzcwcNJOf5f6syzVJMHw+9l95Ff78fJL+9y6Sb7+9w8pSdOgAmz7/hJ0rvkIL9XzgiItn2LTLGTbtW0THN2xDaxgGu3fvZtGiRQ1GJ5s0aRIjRoyQ0cmEEOcdCbUiEkmope1P7sNVh5m7di5LjywFIC06jQdGPcC0rGldqklC5WefcfTe+1AdDnp/+QXmxMQOLY+rsoJtixaweeGnVJeeAEA1mel/0URGXH41ab0b9pah6zrbtm3jq6++ory8HAiOTjZlyhSGDh0qo5MJIc4bEmpFJJJQS/ud3EsOL+GJtU9wtDr4U/hFGRcxe8xssuOy2+yYrcnQdQ59/zo8O3aQ8KMfkfbrhzq6SABogQB7165i4/yPKNizKzw/o99ARnzravqMHoepXm1sIBBg48aNLFu2jOrqagCSk5OZOnUq/fv371IXGkIIcS4k1IpIJKGW9j25PQEPf93+V17f9nq4ScLNg2/m1qG3dokmCTWrV5N/8y1gsdD7s0+xZmZ2dJEaKNy3h42ff8zuVcvRtQAAzm5JDJ/2LYZOnYEjNi68rs/nY82aNaxcuRKPJ9jDQvfu3Zk6dSq9enXNts9CCHEmJNSKSCShlrqT+0R+EYmZTXcV1dryK/OZs3YOK44Gh/1Nj07nV6N/xSU9L+n0NYX5P7mVmpUrib3ySrr/vyc7ujhNqi4rZcsX89n65XxcFeUAmC1WBkyYzIhvXU1yz+zwum63m1WrVvH11183GJ3skksuoUePHh1QeiGEaFsSakUkklBL3cn9zT3z6XnjcKJHn3kfty1hGAZfHf6KuWvncqzmGADju49n9pjZZMVmtUsZzoXnm284eG1wBLfs//4H++DBHVyi5gX8fnavWsbG+R9RdHB/eH7m4FxGXH41vUaORlVNQNOjkw0YMIBLLrlERicTQkQUCbUiEkmopV6ovXs+MVHRpD04BnOcrd2O7w64+cu2v/DG9jfw634sqoWbhwSbJNjNnbNP1aP3P0DlJ58QfdFF9Hz9rx1dnNMyDIOju79h0/yP2bt2FUYotMalpDJ8xpUMmTKNqOjg4Ay1o5Nt3boVwzBkdDIhRMSRUCsikYRaTgq1tmgSfzgQx5Ckdi9HXmUec9bMYeWxlQBkRGfwqzG/YkrmlE7XJMF3+DD7v3UF+P1k/vUvOMeP7+ginbHKkiI2L/yMbYsW4KmuAsBii2Lw5KlccNlVJGYEmxwUFRXx1VdfsXPnTqBudLKJEycSExPTYeUXQoiWklArIpGEWhqHWswKzjHpOCf1wBzffjW2EKxRXJy/mLnr5lJQUwDAxO4TmT1mNpmxneymrMf/QNnf/kbUoEFk/+ffKF2sSyy/18POFUvY+NlHnDiSH56fPXwkIy6/muzcC1BUlaNHj7Jo0aIGo5ONHTuW8ePHy+hkQoguSUKtiEQSaqk7uXf84jNMFgfRplA4Myk4LkghZnImlqT2DS8uvyvYJGHHGwT0AFbVyi1Db+EnQ35ClDmqXcvSnEBpKfunTUevqSHjqf9H3BVXdHSRzolhGBzesZWN8z9i/4a1EDolEjJ6cMFlVzL44qlYo+wyOpkQImJIqBWRSEItdSf3n275CMUSzdB+8fS1qWiHgz9No4B9aBIxU3piTW/f4WEPVhxkzpo5rC5YDUB3Z3ceHPMgkzMnt2s5mlPy8ssUP/c8lsxMen/6CUoXD3flhQVsWvAJ27/6Ap/bBYDNEc2QKdMYPuNK4lJSZXQyIUSXJ6FWRCIJtdSd3E/e/BF2azC0KqrC0EEJ9DYp6HmV4XWjBiQSc0kmtp7t94+AYRh8mf8lc9fO5bjrOAAX97iYX435FZkxHdskQXe52Dd9BlpJCam/+Q2JP7ypQ8vTWnxuFzuWLmLT5x9TVhDsmQJFoffIsYy4/Gq6DxzM9u3bG41ONnnyZHJzc2V0MiFEpyahVkQiCbXUndz/7ycfceFlAyg9VkP+N6Xh5f36xjEg2oySVwmhT8vWKy4YbnvHt9tNXC6/i9e2vsab37wZbpJw69BbuXnIzR3aJKHsnXco/N2jmBIT6b1wISZn+9ZmtyVD1zm4ZQMbP/uIvK2bwvOTemZzwWVX0e+iiWzdtr3R6GSXXHIJAwYM6HQ3+AkhBEioFZFJQi11J/fRQ8fJyAr2R1p8uIpNC/PZt6EIQw9+RJndo8lNjsKcXwWheZbMGGInZxI1MBFFbZ8Ac6DiAH9Y8wfWFKwBoIezB7PHzmZSj0ntcvyTGX4/B668Cl9eHkm3307y/97VIeVoayeOHGbT5x+zY9kiAl4vAFExseROncHgKdP5Zt9+VqxYIaOTCSE6PQm1IhJJqOXUJ3dliZvNiw6zc8UxAv5g36bJ3WyM7OHEdqwaQvPMqQ5ip2RiH5qMYmr7cGsYBgvyFvDkuicpchUBMDlzMr8a/St6xLT/KFiVny/g6N13ozgc9FnwOebk9hmZrSN4qqvZ/tVCNi34lMriYHMQRVXpO3Y8g6dexoHjxaxZsyY8OllOTg5Tp06V0cmEEJ2GhFoRiSTUcmYnt7vax7YlR9n21RE8NcGwEhdjYVR2DDFFLgyvBoCpWxQxF/cgekQqirnt21W6/C5e2fIKf/vmbwSMADaTLdwkwWZqv+7IDMPg0PU34Nm6lYQf3Ejaww+327E7iq5r7F+/hk3zP+bwN9vC81N79WXQ1BkUeAJs2LhRRicTQnQ6EmpFJJJQy9md3H6fxs6VBWz+Mp+qE8GfmaNsJsb0iiWx3IPhDgBgirXinNSD6DFpqFZTm7+H/eX7+cOaP7C2cC0AmTGZzB4zm4k9Jrb5sWvVrFlL/qxZYDbT+5OPsWZnt9uxO1rRoQNs+vxjdq5YghaqoXXExdPv4mmUWaLYsXMXtadabm4uU6ZMkdHJhBAdRkKtiEQSajm3k1vXdPZtLGLTwnxKDgdvEDKrCiN7x5Lm8kOoNleNNuMc3x3nuAxUe9t292QYBp8f+pwn1z1JsTvY3dQlmZfwyzG/pLuze5seu1b+z35GzdJlxFx+GT2eeaZdjtmZuCor2LZoAZsXfEJ1WfBmQ5PZTOboi6iO7cbB/MNAcHSykSNHMmnSJBmdTAjR7iTUikgkoZaWndyGYXBkZxkbF+ZxZFcZACowLDuGnpoGVcFwq9hMOMdl4JyQgcnZtn251vhreHnzy/x959/RDI0oUxS35d7Gjwf/GKupbY/t2b2bg9d8BwyD7H//C/vQoW16vM5KCwTYu2YlGz//mII9u8LzE/oPwZecQeGJYOA1m81ceOGFMjqZEKJdSagVkUhCLa13chflVbJpYT77NxZhGKAAAzMc9DYrqJU+ABSLSvTotHYZgndv2V7+sOYPrD++HoCs2Cxmj5nN+O7j2/S4x371IBUffohj7Fh6znvjvO/WqnDfHjbO/4jdq1ega8HmKZa0Hmjdc6hwuQEZnUwI0b4k1IpIJKGW1j+5K4pdbP7iMDtXF6CFekfolWRjYLQFc0WwK6j2GoLXMAw+O/gZ/2/9/6PEXQLApT0v5Zejf0m6M71Njuk/epT9l12O4feT+efXcE5sv3a9nVl1WSlbvpjP1i/n46ooxwCMuCSMnr1xBYI3GkZHRzNp0iRGjhwpo5MJIdqMhFoRiSTU0nYnt6vSx7YlR9i25AheV7CGrnushdxEG9byULhVwJ6bTMzkzDYdgrfaV81LW17i7Z1voxkadrOdn+b+lJmDZrZJk4TjT8yldN48bP37k/P+eygywlZYwO9n96plbPzsI4oO7ccAArGJaN1z8BOs1ZbRyYQQbUlCrYhEEmpp+5Pb5wmEe0yoLguG2WS7ieEpdhy1Nbe0zxC8e8r28PjXj7OxaCMA2bHZzB47m4syLmrV4wTKytg/bTp6dTUZf5xL3NVXt+r+I4FhGBzd/Q2bPvuIvetWo+sG/vgk/Kk90NVgjxkyOpkQoi1IqBWRSEIt7Xdya5rOvvVFbFqYx4mjNQDEWxQuSHcQG2pzC20/BK9hGHxy4BOeWv8UJzwnAJiWNY1fjv4ladFprXackldfo/iZZ7BkZNDr8/mo0la0WZUlRWxe+Bnbvvwct8uFLyEFf1IahinYBCEjI4OpU6fSu3fvDi6pECISSKgVkUhCLe1/chuGQf6OUjYtzOPonnIAolW4IN1BN5cfQn8ilswYYqdkEjWgbYbgrfJV8dLml3h719voho7dbOdnuT9j5qCZWEyWFu9fd7vZP+MyAkVFpM5+kMRZs1qh1JHN7/Wwc/kSNs7/iJJjR/F1S8WXmAqhmtuc7GymXnqpjE4mhGgRCbUiEnX6ULts2TKefPJJNmzYQEFBAe+//z7XXHNNeLlhGDzyyCP8+c9/pry8nPHjx/Pyyy/Tt2/fMz5GR57chQcr2Lwwn/2bi8EAuwK5KXZSAxqKFvyjaesheHeX7ubxNY+zqWgTADlxOTw09iEuTL+wxfsu+/e/Kfztw6ixsWTMfYKogQOxpLVebXCkMgyD/O1b2Dj/I/Zt2YSvWxr++GQIta/t17cvUy+9lNTU1A4uqRCiK5JQKyJRpw+18+fPZ+XKlYwcOZJrr722UaidO3cuc+bM4c033yQnJ4ff/va3bNu2jW+++YaoqKgzOkZnOLnLj7vY9GU+u1cXogV0rAoMTrSRqYASCPagYOoWRezFmThGpLT6ELyGYfDxgY95av1TlHqCfajOyJ7BA6MeIDX63IOTEQiwd/IUtJJgzwuoKum/f5T4732vNYp9XigrPMbmBZ+yZdlXVMckEojrBqFmKQP79WP65ZeTkJBARUUFpaWlJCYmEhcX18GlFkJ0Zp3he0+I1tbpQ219iqI0CLWGYZCRkcF9993H/fffD0BFRQWpqanMmzePG2644Yz225lO7poKL1u/OsL2pUfxuQOYFegfa6GXVUUNdQ/WlkPwVvoqeXHTi7yz+51wk4SfD/s5Pxz4w3NqkuAvLGTflEvgpL9mibfdRtwV38LWv7/cAHWGfG4X25csYt0X8ynGTCA2EQj2h5yWnERhyQkMw0BRFK666ipGjBjRsQUWQnRanel7T4jW0qVD7YEDB+jduzebNm1i+PDh4fUuvvhihg8fznPPPdfkfrxeL15vXa8DlZWVZGZmUlJS0mlObp8nwK6VhWxbcpSach8moFe0iX4OM+ZQuFUcZhzj0nGMTW31IXh3l+1mzro5bC3ZCkBObA4Pjn6Q0amjz2o/rrVrOfaTW5tdbkpKwjF+PI4J43GMG4dJahhPy9B18rZuYvXnn5BXUYPmbPozmzVrJt2795CLBiFEI5WVlSQlJUmoFRGlS/fuXlhYCNCoXWFqamp4WVPmzJnDo48+2mj+woULcTgcrVvIFoofC9ZjZqoOWtlbDftrNDKtCv2iVRyuADWLDlO5JI+iNC9F6R4Clta7Rvme8T1623uzwLOAg5UH+dmin5FryeUy+2XEqmf2j6C5vIIcRUGpd+1kKAqu3r2w5+VDSQlVH35I1YcfYigKnsxMavr3w9W/P57u3cNtSEVjcbmjGVBRRv6unVRZGw/g8eabb4GhY9J1LIqCxWLGFmXDHu3EERtHVLRT+sAV4jzlcrk6ughCtLouXVO7atUqxo8fz7Fjx0hPrxsd67rrrkNRFN59990m99MVampPZujBHhO2fHmEwgOVKECGVWFwnBV7qOYWi4p9ZArRE9IxxbXeELyVvkpe3PIi/933X3RDx2F28D9D/4fr+1+PRT19k4TK996j6NHfg66DqpLyyMPEXnsths+He8NGXCtX4Fq5Et++/Q22UxMScIwbF6rFvQhzUrdWe0+RZNeGtfx3/sJwO1sg1NzDAOXUodVk6NgtZpwOBwnx8SSlppLWvQfpPbNwOp1SyytEhJKaWhGJunSoPdfmByfram2LCvZXsGlhHge3BG++SjMrDI634gzdUNZWQ/B+c+IbHv/68XCThD7xfXho7EOMTjt9kwR/YSG+vHysWT2b7f3AX1BA9fLl1CxfQc3q1ejV1Q2WRw0eTPTECTgnTsQ+bBiKDCMLQNWJEl586H48aVnBYGsYRB3P54f3PEBFeQWFRw5TXFxEeVk51W43noBGwGQG06k/P8XQsakq0VE24mJj6ZacTFr37vTIzqFbcooM4ytEF9bVvveEOBNdOtTW3ih2//33c9999wHBEzUlJaXL3ih2NsoKa9i0MJ/dawrRNYNks8KgWAvxeuiPtA2G4NUNnQ/2fcAzG56h3FsOwBW9ruC+kfeR7EhulWMAGH4/7i1bqF62nOoVy/F+s7PBcjUmhuiLLsI5cQLREyac992EbVu8kAVvvIpmtmIK+Jhx888Yesn0Ztf31FRz/HA+x/IOcbyggNLSE1RVVePy+fChYJitDWt+T2YYmDFwWC3EREeTmJhIclo63bOySM/s2ema8QghGuqq33tCnEqnD7XV1dXs27cPgAsuuICnn36aKVOmkJiYSM+ePZk7dy5PPPFEgy69tm7d2uW69GqJmnIvWxYfZseyo/g8GgkmhYExZupHzKiBicRMab0heCu8FTy/8Xn+veffGBhEW6K5Y/gd3DjgRsxq69fgBYqLqV65kpply6lZuRKtoqLBclu/fnW1uCNGnJejl1WdKKG88BjxaRnEdEs65/3omkZZUSFHDxyk8OhhSoqLKa+ooNrtxqvpaGZLeDCI5qiGjs2k4rTbiYuNIyklmbTuPeiRnUNiUpK05RWig3X17z0hmtLpQ+2SJUuYMmVKo/mzZs1i3rx54cEXXnvtNcrLy5kwYQIvvfQS/fr1O+NjRMrJ7XUH2LHsKFsWH8ZV4SNWhQHRZtJMCrV1brbeccFw20pD8O4o2cH/ff1/bD+xHYC+CX359dhfMzJ1ZIv33RxD0/Bs30718hVUL1+GZ+u2Bl2GKQ4H0RdeGKzFnTgRq4y+1WoMw8BdXUVh3iGO5edRVFBAWVkZldXVuP1+/IoJw3yadtaGgUUxcFitxDqdJHbrRkpaOt2zs0nvkYnN1nrtwYUQTYuU7z0h6uv0obY9RNrJrfl1dq8tZNPCfMqPu4hWoZ/dRKZFDYfb1hyCVzd03tv7Hs9ufJYKb7AG9apeV3HvqHtJsp97jeGZCpSVUbNqFTXLllO9cmXdQA8h1uxsoidNxDlxIo7Ro1HPsAZfnD0t4OfEsaMcPXSQwmPHKCkupqKykhqPB69moJstp+3RwmTo2EwmnA47CXHxJKemkNajBz1yehEbFy+1vEK0gkj73hMCJNQCkXtyG7rBwa0lbFyQx/GDldgV6BOlkh1lQg39qbfmELzlnnKe2/Qc/93zXwwMnBYnd15wJ9f3v54Sdwn5lfn0jO1JWnTbtX81dB3vrl3hWlz3ps2gaeHlis2GY8yYUFvciVhzsuUO/3ZiGAY1FeUUHDrIsfw8io8fp7S8jOoaF25/gIBiwjjdzWeGjk0Bh81GbEwM3bp1IzU9g4ycHNIyumOxnP0AIUKcjyL1e0+c3yTUEvknt2EYFOwL9phwaNsJrAr0tqn0tpswhf70W3MI3m3F23h8zePsOLEDgFRHKkWuIgwMVEXlkXGPcG3fa1v6ts6IVlVFzerV1CxfQfXy5QRO6r/Y0qNHuC1u9NixqNGtc0OdOHsBn4+iI/kczTtE0bFjlJSUUFlVRY3Hi88gWMt7mgsQk6FjN5uCXZQlJJCcmkp6Zk96ZOfgjImRCxghQiL9e0+cnyTUUu/kzt9JbOaAji5OmzpxtJrNX+SzZ+1xVMOgl1Wlj91Ebf1Waw3Bq+ka/937X57Z8AzV/oZdcykoPDX5KUaljiIhKqEF7+bsGIaBb98+qpevoGbFclzr1mP4/XUrWCw4Ro4M1+La+vWVENRJGIZBVekJjh48SMHhPIqLiigvL6fK5Qp2UaaawXTqv6+KoWNTFKKjbMTGxpKUlERqRgY9cnqRnJaO6TTbCxFJJNSKSCShlnon94NxxH7/eRgxs6OL1OaqSj1sWXyYb5YfQ/dqZNlU+tlN1N6io0abcU7ojvPCjBYNwbsobxF3L7m72eUJtgRy4nLIicuhV1wvesX3oldcL9Ki01BPM3BAS+k1NdSsXRuuxfUfPtxguTk1NViLO2Ei0ReNwyT/8HdaPo+b4/n5HM0P1vKeOHGCyupqXF4fPgh2UXYqoS7K7BZzqIuyBJJT08no2ZPu2dk4HMEa/GN5hzi8fz+ZvXuTkZXd5u9LiLYioVZEIgm11A+1McTaVLjkt9BjFCT1hZj00/7k2ZV5avxsX3aUrYsP463yB4fgtZtwhN6zYjPhHJeBc0IGJufZd5NVWFPIjP/OQDf0BvNT7akcdx9vdju72U52bHajsNszpicWU9u0m/QdOhRsi7tiOa41azE8nrqFJhP2YcNwTppI9ISJRA0aiCI3LHUJhq5TXlLEkQMHKDx6hJJQLW+124NH09BMp795TdV1FHQ0xRQe4CLV6eCCUaOITUggLrEbCUlJ2O0Oqd0XXYKEWhGJJNRycqg96QvJEg3dekO3PsGQ261P6HVfiIqcfwgCfo1dqwvZ/EU+lcVuMiwK/e0mYkI9IygWlejRaTgn9cAcf3ZdLr239z0eXf0ouqE3aFPr8rs4VHmIAxUHOFB+IDhdfoC8qjwCeqDJfZkUE5kxmY3Cbk5cDtGW1msPq3u9uNatp2b5cqpXrMC3v+EQvqbERKInjMc5cRLRE8ZjTmi/ZhSidXldNRTkHeJofh5FBccoLS2jqroal8+PH+X0XZTVZxioho5ZAbOqYrNYsFmtREVF4XA4cDqdOGNjiY2PJy4hkfikJGLjE6Tpg2h3EmpFJJJQSxM1tTkXQ8VhKDsEhtb8htEpoaAbCrm1wTc+C073c2cnpesGBzYVs2lhHkV5VaSZFfpFqSTU3jwWGoI3dnIm5rMYgrewppDDVYfJjMk8be8Hft3PkaojHKg4wMGKgxysOMiB8gMcqDiAK+BqdrsUR0ow6NY+4oNht1tUtxbXnvmPHq2rxV21Gt1VrxyKQtSQIcGbzSZOwJ6biyIhJSLoukZpYQHLFnzO1rwjjZarAV9wPdV02gEpmmUYKIaOCQOzomI1m7BZLERF2bDb7TiincTExBATF0dsfALx3bqRkJyMLar1hsAW5x8JtSISSailfqiNJ/b7z9W1qdX8wWB7Yh+U7A0+1z6qm//pHMUECdmhWt0+kBR67tYXYtK6RHMGwzA4tqecjQvzyN9RSrJZoa9NJdkSCrdtMATvmZTpuOt4w7AbquU94TnR7Hax1ti6mt16YTcjOgPTOQQRw+fDtXlzsBZ3+Qq8u3Y1WK7GxRF90bhwLa4lJeWsjyE6l2N5h3jt9TcanruGwU9vuZmMrGy0QIDq8jLKT5ygvPQEVeXlVFVVUFNVjcvlwu3x4PX68Ab8BDSdgAGaop725rZT0jVMhhGsFTap2MwWbDZrMAjbHThjamuFE0K1wsnExMWhygWXQEKtiEwSajnH3g88FXBif+ixt17w3Q/+mua3szrrmjN0q9+coU+nbc5QcqSaTQvz2Lu+iHgF+kWppFnq2iC29hC856LCW9Ew6IbC7tHqoxg0/VfcZrKRFZsVDrs58cHgmxWbhc105k0s/MeLqFm5kurly6hZuQq9srLhcQYMCI9u5rjgAhTpS7VL+ugff2Pjnn3hNrUj+vXh6pt+dM77MwwDr8tF+YliyktLqSgrpbqikqqqSlw1NbjdbjxeL16fH5+mEdANNMBQTed+YazrqLqGSQGzomC1mEO1wlE47A4c0dHExMYQExdsHhHXLZH4xCQsUVHSVjjCSKgVkUhCLa18chsGVBU0DLkn9gWDb1neqZszOFNDQbd3vfa7fSEhC9ro5qizUXnCzZYvD/PNymM4Ajp9o0x0t6ptNgRva/AEPORV5jUMuxUHyKvIw6f7mtxGVVS6O7vXhd24nHDtbqz11H8/jEAA97Zt4R4VPNu3NxjCV42OxjHuQpwTJuKcOAFL9+6t+n5F2zqWd4gjB/bTo1fH9X6gaxoVZaWUlZRQWVZKZXkZ1ZWV1FTX4HK7cLs9eH1evIEAAc0gYBjoqgrn2puIYaBoAUwYmBQFiynUVthmxRFlx+FwEB0TQ0xsHLEJ8cQndCOuWzccsXGYTjeYRjOkl4m2J6FWRCIJtbTjyR3w1TVnCNfuhpoz1BQ1v11tc4Zw0O1T137XmdruzRk81X62LT3C1q+OYHL56WszkWlVqR1t15oZQ0wrDcHbVjRd42j10UZh92D5Qar8Vc1ul2RPqgu6tU0ZYnNIcaQ0GeQDpaXUrFxFzYpgUwWttLTBcmvv3jgnTCB60kQco0ah2s7uJjwhzoRhGLhdLspLSigvLaGyrIzKigpqqqupqanG7fbg8Xrx+etqhXUUjHPt4cMwQNdQdQ0zwZvmrGYTNpuVKFvwprloZzQxMbGhWuEE4pKSccbFsfCD99i4Z3+r1YiLpkmoFZFIQi11J/fu/EL6ZaZ2TCHc5VC6vy7khoPvfvA3f3MU1phmemfoA7aYNi2y36exa1UBm7/Mx3/CQ58olSyriikU7ixpDmImZ2LPTUar8hEocWNOsmOO67zBzTAMStwlTYbdInfzFx5Oi7Nhf7uhwNvd2R2zGqytMnQdzzc7gwF32XLcmzeDXtfVmRIVhWPsGJwTJ+GcOAFrVlZbv10hTsnv91NZXkZ5SQkVZaVUlpcHa4VranC5XHg8Hjw+H76ARkAPthU+5yAMwSGtVbVR22WnEcBmsWA2mzGbTFisFixmC1arBavNhtVmw2a1YbPbsUVFEWV3YHc4iIqOxm53YHM4MFttWGw2aVMcIqFWRCIJtdSd3Fn3/Is5N4zlxjE9O7pIdQwDKo81DLm1N62V58FJ/b824ExrpneGnq3anEHXdPZvKmbjgjwqj1TT26aSY1OxhL6Y1Ggzek2oiy4FEq7tS/ToU/eA0BlV+ao4VHGoYditOMjhqsON+uGtZVEtZMVmNQq7WbFZWGt81Kz+muoVy6lZvoLA8YY3H1p69gzX4kaPGYPqcLTH2xSiRTRNw+WqofzECSpKS4NNJCorqamqCt40V9tW2O/Hr+nB5hEobfuLk66DoaPoOoqhowAqYFKCzY1MqoLZZMJkMmE2m7CYLVgsZiwWazA4W4PBOSoqKhyabQ47drsDe3Q0NrsDa1QUFlsUZqu1S/RhLaFWRCIJtdSd3Jl3/wvV5kBVwGJSsZpUzKZgG7LgQ8Fcb9piUjGrClZz8PmU65mU4P5UFYtZwaLWrdfwOLXr1003exzDh7XqMNby/ZjL96OW7kepreWtKW7+DatmSMipq9Wt337XmXLOXy6GYXBkVxmbFuZRsKuMXlaVXjYV20lNEAzAkmLHFGtDdZhRHZbgc7QFU+10vXmKzdRp2ug2xaf5yK/MbxR2D1UcwqN5mtxGQSHDmVEXdmNz6F1qIWnLYbSvN+DasAHqDeGrWCw4Ro8iesJEnJMmYu3du1N/JkKcDV3X8Xq97N/1Df/54KNGNbV9u6djtpjx+/z4/H78fj+BQICApqFpGgFNRzMM9NoHYLR1UK7PMCAUmIPPBqoSDM6qAiZVxaQG/103m8x1wdlqwWqxYrFasdXWOIeCs91ux1avxtlmtwdDs82G2WJt8fm/e/s2BgzNlVArIoqEWhqH2q6sNggnqDX0MR0nhwKylQKyOUamfozu+jGi8Da7vVuNpsSWSWlUJqVRWVQ4sqhw9KQqOhus0eHAbW0m4FvMKhZVxVfspmhdMaa9FYxznvswuwCYFFR7KOhGB58bhN/oeiHYYUGNtqDazR3enlc3dApqCsJ97NZv0lDhrWh2u8SoRPpF9WTMUTv9dtfQbUs+psKSBuuY09ODtbgTJxB90UWYnE62bl3H3s1r6Tt8DLm5o9v67QnRJlqrlwnDMAgEAvj9fnw+Hx6XC7fbhafGhcftwuN24/V48HrceL1efF4vPr+PQCg4BwIB/IFAKDRraLqBZujoOui0Q+3yyerVNqPrqBjh0FxX26yGa5wtZjMWiwWLxYLVasVitYWD84EDB9hfUsYTc+dKqBURRUItDUOtOcrBB3eMJzHail8zCGg6Pk0noBn4NR1/6Dmg6/gCBgFdbzj/5PU0HV9oP35Nx68b+AM6Ad0I7bdu3Sa31w18AT10nIbrBfSz/6NT0EmjjBy1gF5K3SNHKaCHUoxJaX6fhUYCB/R0DhrpHKj3OGIko9F0O7VxXjN/tNob1CoYhsFGV7AXCKuiYFXBqoSmFbAoCjYVLKFuh86FDrgVcJlqHwpus4LbpOAxBx9ei4LHrOKzKHgtKn6LimJRURUFs6qgqgomRcFkCj2roYcSXGZW681TlUbbmU3BefW3q512aRUUefI57s6nwJVHoTuPAlc+JZ4m+j82DNJLYfQhM2PyLPQ64MLsr9fcwWymIsFJbHE5CqArsOlb47n+l4+jRkWh2O0oFovU7LaCggo3B0tqyEmKJj1OBj9oK1t37Gbnzt0MHNif3MH9O7o4zdI0DX+o5tjn9eJxu3DXhmaXOxiYPR48Xk8wNHu9+Hx+/H5fg9rmYI2zjmboaLqBbhCqbabNgrPX6+WJJ56QUCsiioRa6kJt9j3/5okbx3D96E7UpvYUdD0YbGtDri8Utv0BA7+uN5xfLxA3GdZ1A83nwVGTj7P6ILE1ecTW5JHgziPRk090oLzZcgQwcdyUzlFTD46oGeQrGRwig4NkUOpy8MsyK8PsJlRFQTcMtrg13lK9GChEGRBl1D3bDAW7AbbQPDPBoBsMvkqD8NtwXv0wfO5fAm7DoBKDCgzKlbrp4ENv8LoSg3IM3Od8tCYoXlRbMaq1GNVWhGotCr0uQVGCQdbiNxh02GD4foPhBwy6l55mn4CuKHhNZnxmC16zFZ/Zgs9kw2+y4jfb8Fms+M210xYCZhv+8DwrAbOVgNVKwGLDb7ERsFjRQs8Biw3NGnyNKVhDrioKigKqoqAqoISeg68bLgu+rl2v4euG24fmqU1szxmsc9I+OdUxGpQxOG/V/hP8/es8jOCmzBqXxcX9UkAhfHyFum0VCC2r278SXrdu/brn4LbULkep93nUbUf9/Z20rqI03l/9ctSfV7sdCqfeH3XHD5elDS+Q3l2Xz+z3tqEbwXLNuXZol/k3ubWdXNvs83mDNc41NXjc7nCNs88T6sbN4w010QiFZr8ff/1mGrqOL6ChKaqEWhGRJNTSSXo/6OxcpVB6oN7IavX64A003W4UQLdEs7PiQtZU/RyHyYxLCzA25hV6Th6MMzEdwxSFrlrRzVY01YauBp811YKmWvH4Lbh9ZrweM26fgtdrwuMBn0fH7w7gcwcIuDX87gABj0bAE0D3aJh8el0APqkm+OR5llAoVs/xi1oD3CZwqwo1ZqgxQY2qUKNCtQpVikG1ApWKQZUCFQpUGaEbZAwI6MGfNAO6jqaDbgQvOnSD4E+euoFmBDDMJaGgW1wv7BYxfpeHuz9qfKOapoCpnc9uTVHwmix4TFa8Jitek6Xu2WzFq1rqpust85y0rs9kwdPEel6TFY/JEhyWtgMkucvJqC7hmDOJEnt8h5ShMzllSKZ+GG96ujYkE1pX1w1O1DTuPzotNgqzSWkUspUGob3eaxoGcjgptJ+0n9qV6t5H45Bfd4zQ6yb21eg4TZSJBuVvfAFR/700V6b6FynNlam543iLj2I/vAqvzyehVkScFjZ2jCxp8nNi8xyJwUePUQ3n6zpUHm0YcsO9M+Sj+msY7FhElm0zFYE04syFOE0nYO2XAOG7kM+aYgKzLfSIArsVYqLAbAVzFLoahU+JwWvE4jWceHUnHt2OV4vGq0XhDURRHbDi8Vvx+ix4fWYCPhOGT0XR1HDtr1WtqwmuqyGum2dSFEyAUwOnZpAcvrfr1GnSQAWrihJlRrGbMTktmJ1WzDGhNsEOS7idsCk6+KzYzRgmtV7QDYbfVRu/QP/4fgKxWQQSemMu24+pMo8/zR6FN8GK5nahe1wYbjeGxw1eL3g9WHwa1gDY/NQ9+8EWMOqm/WAL1E4bWGun660f5Qc19HZNhoEj4MMRaHpgi9aimUxoFhta/Rpjq41A7bTZGqxRNlsJ1NY4W6z4zMGaZZ/Zgq+2ZtpsJWAOhm2fuW6Z12RBU1R0AypcPrLWLOIXm/+DGmpP+dzw77Fn1BSirWYMI/gnbhhGaNpoOI/gvUQNpjHCPbrVn6/Xmya0H725fZy8PwM46fh6qExtwTBAa7Dz1jnQyRcPhZXNXziLszfJH0V3S/P3VgjRVUlNLdK1SZvxe+DwGnjr2zT8slNg4LeD/VFqvmBNb8Abenjqzau3TPOCHmiXYuuGiteIxqvH4NGjQ6E4+NprROPRneGg7Ddi0YhFN2LAcGBSLM02j6itIW5J8whdJbhjmxrqHcKKOTaKo2u3k2RKRlGUYC8UWj7Db7kcxaSASQl2MWRWgjfPmVQUk0IADY/hwWN48Rge3IYHt+7BrbuDz5obt9+NO+DGFXDhDrgbPFwBV3C534XP60Zzu9A8bnS3C9WnhQKxUReO64Xn4LTRKCDXzj95fWsAbL5zvABqAcViQbHb0c1mjNJS1Kh4VGcqevVxdE85piFDiYp2gKqgqCZQ1eBnrarBz1kJTium0IhetfNMpuA2tfNUFVTTSftRGs+r3U+96Qb7Vk+aVk0N9hOsWjVhKAqoKkZoXSO0T0NRwtsbtYMv1FtmhLap3R5FwTCZwjdNGaoJQwFDMYVeh9ZXFQxFDa5Xf98o4dBtYFBc5eWfv3uB/91Ud/Hwpwu+x3d/ezvdnLYGQb+pYB9aGgr2NHuBEVqxblkTFyC1xwmt2uS+OGl+3et6x2q0HeGhu2vn0eyFSt1xTj7WyWWiUZkar49hUOUJMG/VIbLK97D01Xvle09EFAm1SKhtcxvfgo/vDg4RrJjgqmdhxMyz34+u1QXf2qAb8J4UiE+ap9VbVhuSm5x3criu3dbT+BinqI3SDDNePRqPERMKwsEAHAzCodd6DJoRj2bEouMEolGwY1bMzTaZsLSgecS5MtCDX75KsPrPCD2jNHwoKqDooIKiGKF+jAx0VUdXdHRVR1N0AqqGpur4VY2AEsCvBvCrGl7Fj0/x41EDePHjwY/H8OHGjxsvbny4DA9uzYtHc6MHfCiahqppmAIaakDDrAUw+TVMmobJr2Hxa5gDGla/gc1vNArUlkCw5rmuFrouPEf5m/48LFnjsQ3/EYqiYhg63s1/w5+3sv3+QCJRKBzXXggYioLh8eCPq/vVwVKRhyk+HsVsDv7srqjh7Wp/hg8P2KAqDV8roQu5ptapF8wVpeHrJo9TG/gVJXyBEF6ndr8nrRNsQlDvOCeXRSHUp23dcRrsN3zsUx+n7j2oTa9Te+zQvD1Lv8a06ivG7tsr33siokioRUJtu6g4GmyTm9gL4rp3dGnOnWEEa4ybDMlN1TafQQ10aF7A68frAY9XxetT8XpNeH1mPH4zXr8Nf8BBQHOiaU4MojGMaBTFTrRqJ9PauI1ppaajG6GbighmTUUJZc560wrtH5jbi4GBXu8/LfRaU3Q0RSMQmg6g41c0AqGHZmjoBB+GoWELwCBjYIMbpAzDYGvga2pMHkDHMEIP9NCgKAaGoYGhB2vQwss0QA9VowWX1z0boefgdP1+T+svVwwj2LVT7Tqh17XrBbt9ql2mo4ama7uDUuqtbwotU3UD1TBQdQXVCP290INNS9TQNUyDZ51gf6y11zcnvT5XNUNnktprfPhXh+MHVhK97a0W/k0QJyt1dmfChkXyvSciioRaJNSKLsgwwiG5YsdeKv/lahC4dMPAP9ZDVEoMekBD1/S6Z81AC+ihaR09oKNroAd0DM1AC4ChGcGcFTAwdCWcuXQN0JVQtlJC/Q4pGIYCuhr8GTT4+zMYCkrtM6F5BOeFewrg9KE7OK3UC+A0mK7dNlKDeXsLt9+l6WcdI/jHW2++HnrV8DXhuaeeV/efRTeRpSQ2unjYrx/HY/LXbV3bPCC8R07ac/35Ohigo4fW0sNrUztt6OG1qTdd7x2HmgjUNlqu/2mAYtR7d0btOnqDTy10cmAYBkrowkcJHa92fbXetsrJz+HtjODFC3r4WQ21oVCMehc2ofWV8HE0FMPA5tJIir4KW0p/Bj/3LfneExFFbhQToitSlPCNcnEjR1G0+hNsh2PD3aZ5Myvpe+2VHV3KZhmGga7VPvTwtFZvuuEyHa2J+YHaZQEjGM5DD8OvYwQC6H4NI6BjBDQMfzDEGwEdND0U3HXQjOB0beegWjAEoQN6sDGi6tfJMZsbha18n4Ze7+52jNCd50a9O8/rPYBG86mdVpqYV/8O/ZP2dar5jebBGXfDVXdx0AEXCU00mlYUhT6mk4bVPtuiyfVOI1Xemo4ughCtTkKtEBGg751XUrljH5W7jxLfvzuxgyd1dJFOSVEUTGYFkxloZuCOzqS6zMOSR1aTW6+/5a1ujcmPjsOZENXkNkbo7pxwjwjh142XNb1e06/D03qoJvF0+9Frb1gKXhAE5+mgBZ+DrSAM0EPbBYfMCm6v14X94DENlPB6ode100ZoWb1ta+9WClZGht68XlfW4OvQOgZ4K8pJLTEaXTwcj1ewRjsJN2evfW/U2294WWh/BJtB1Nbq1lay1k0bDdYP7zc8r9664cIE/6eEZitGeGaT69VOK6FXSv1jhOYq9VY9OXs3tyzcfdfJ69LwoqapdRSCPbgIEYkk1AoRIWIH9yF2cJ+OLkZEciZE0ev6/nz5j104VAWXbnDhTQOaDbRQe2NRXQARZ+bLp/5Gv6Ks8MXDnpQ8Lr3v7IfJPV80uOiBUGuLuguP4DqELxyqDlfieWNHRxZZiDYTMW1qX3zxRZ588kkKCwsZNmwYf/rTnxgzZswZbSttaoUQZ6K6zENFkZu4FPspA61omQMb11O48xBpA7PpNWLU6TcQZ2XvP3fhW3OQIdKmVkSYiAi17777LjNnzuSVV15h7NixPPvss/z73/9m9+7dpKSknHZ7CbVCCCHOJ0e2HyFzaKZ874mI0t59mbeJp59+mttuu42bb76ZQYMG8corr+BwOHj99dc7umhCCCFEpxPbU4KsiDxdvk2tz+djw4YNzJ49OzxPVVUuvfRSVq9e3eQ2Xq8Xr7duiMCKigoASktL8fub6XldCCGEiBBVVVUARMCPtUKEdflQW1JSgqZppKamNpifmprKrl27mtxmzpw5PProo43m5+TktEkZhRBCiM7oxIkTxMXFdXQxhGgVXT7UnovZs2dz7733hl+Xl5eTlZVFfn6+nNxtpLKykszMTA4fPiztt9qIfMZtTz7j9iGfc9urqKigZ8+eJCYmdnRRhGg1XT7UJiUlYTKZOH78eIP5x48fJy0trcltbDYbNput0fy4uDj5B7SNxcbGymfcxuQzbnvyGbcP+ZzbnqpGxK01QgARcKOY1Wpl5MiRLFq0KDxP13UWLVrEuHHjOrBkQgghhBCivXT5mlqAe++9l1mzZjFq1CjGjBnDs88+S01NDTfffHNHF00IIYQQQrSDiAi1119/PcXFxTz88MMUFhYyfPhwPv/880Y3jzXHZrPxyCOPNNkkQbQO+YzbnnzGbU8+4/Yhn3Pbk89YRKKIGHxBCCGEEEKc37p8m1ohhBBCCCEk1AohhBBCiC5PQq0QQgghhOjyJNQKIYQQQogu77wPtS+++CLZ2dlERUUxduxY1q5d29FFiijLli3jqquuIiMjA0VR+OCDDzq6SBFnzpw5jB49mpiYGFJSUrjmmmvYvXt3Rxcrorz88svk5uaGBwMYN24c8+fP7+hiRbQnnngCRVG4++67O7ooEeV3v/sdiqI0eAwYMKCjiyVEqzivQ+27777LvffeyyOPPMLGjRsZNmwYM2bMoKioqKOLFjFqamoYNmwYL774YkcXJWItXbqUO+64g6+//povvvgCv9/P9OnTqamp6eiiRYwePXrwxBNPsGHDBtavX88ll1zCt7/9bXbs2NHRRYtI69at49VXXyU3N7ejixKRBg8eTEFBQfixYsWKji6SEK3ivO7Sa+zYsYwePZoXXngBCI5ElpmZyV133cWDDz7YwaWLPIqi8P7773PNNdd0dFEiWnFxMSkpKSxdupRJkyZ1dHEiVmJiIk8++SQ/+clPOrooEaW6upoRI0bw0ksv8X//938MHz6cZ599tqOLFTF+97vf8cEHH7B58+aOLooQre68ran1+Xxs2LCBSy+9NDxPVVUuvfRSVq9e3YElE6JlKioqgGDoEq1P0zTeeecdampqZCjuNnDHHXdwxRVXNPi3WbSuvXv3kpGRQa9evbjpppvIz8/v6CIJ0SoiYkSxc1FSUoKmaY1GHUtNTWXXrl0dVCohWkbXde6++27Gjx/PkCFDOro4EWXbtm2MGzcOj8eD0+nk/fffZ9CgQR1drIjyzjvvsHHjRtatW9fRRYlYY8eOZd68efTv35+CggIeffRRJk6cyPbt24mJieno4gnRIudtqBUiEt1xxx1s375d2si1gf79+7N582YqKir4z3/+w6xZs1i6dKkE21Zy+PBhfvGLX/DFF18QFRXV0cWJWJdffnl4Ojc3l7Fjx5KVlcW//vUvaUojurzzNtQmJSVhMpk4fvx4g/nHjx8nLS2tg0olxLm78847+eSTT1i2bBk9evTo6OJEHKvVSp8+fQAYOXIk69at47nnnuPVV1/t4JJFhg0bNlBUVMSIESPC8zRNY9myZbzwwgt4vV5MJlMHljAyxcfH069fP/bt29fRRRGixc7bNrVWq5WRI0eyaNGi8Dxd11m0aJG0kxNdimEY3Hnnnbz//vssXryYnJycji7SeUHXdbxeb0cXI2JMnTqVbdu2sXnz5vBj1KhR3HTTTWzevFkCbRuprq5m//79pKend3RRhGix87amFuDee+9l1qxZjBo1ijFjxvDss89SU1PDzTff3NFFixjV1dUNagAOHjzI5s2bSUxMpGfPnh1Ysshxxx138Pbbb/Phhx8SExNDYWEhAHFxcdjt9g4uXWSYPXs2l19+OT179qSqqoq3336bJUuWsGDBgo4uWsSIiYlp1A48Ojqabt26SfvwVnT//fdz1VVXkZWVxbFjx3jkkUcwmUzceOONHV00IVrsvA61119/PcXFxTz88MMUFhYyfPhwPv/880Y3j4lzt379eqZMmRJ+fe+99wIwa9Ys5s2b10Gliiwvv/wyAJMnT24w/4033uDHP/5x+xcoAhUVFTFz5kwKCgqIi4sjNzeXBQsWMG3atI4umhBn5ciRI9x4442cOHGC5ORkJkyYwNdff01ycnJHF02IFjuv+6kVQgghhBCR4bxtUyuEEEIIISKHhFohhBBCCNHlSagVQgghhBBdnoRaIYQQQgjR5UmoFUIIIYQQXZ6EWiGEEEII0eVJqBVCCCGEEF2ehFohRJeyZMkSFEWhvLy8o4sihBCiE5FQK4QQQgghujwJtUIIIYQQosuTUCuEOCu6rjNnzhxycnKw2+0MGzaM//znP0Bd04BPP/2U3NxcoqKiuPDCC9m+fXuDffz3v/9l8ODB2Gw2srOzeeqppxos93q9/OpXvyIzMxObzUafPn3461//2mCdDRs2MGrUKBwOBxdddBG7d+9u2zcuhBCiU5NQK4Q4K3PmzOGtt97ilVdeYceOHdxzzz388Ic/ZOnSpeF1HnjgAZ566inWrVtHcnIyV111FX6/HwiG0euuu44bbriBbdu28bvf/Y7f/va3zJs3L7z9zJkz+ec//8nzzz/Pzp07efXVV3E6nQ3K8etf/5qnnnqK9evXYzabueWWW9rl/QshhOicFMMwjI4uhBCia/B6vSQmJvLll18ybty48Pxbb70Vl8vFT3/6U6ZMmcI777zD9ddfD0BpaSk9evRg3rx5XHfdddx0000UFxezcOHC8Pa//OUv+fTTT9mxYwd79uyhf//+fPHFF1x66aWNyrBkyRKmTJnCl19+ydSpUwH47LPPuOKKK3C73URFRbXxpyCEEKIzkppaIcQZ27dvHy6Xi2nTpuF0OsOPt956i/3794fXqx94ExMT6d+/Pzt37gRg586djB8/vsF+x48fz969e9E0jc2bN2Mymbj44otPWZbc3NzwdHp6OgBFRUUtfo9CCCG6JnNHF0AI0XVUV1cD8Omnn9K9e/cGy2w2W4Nge67sdvsZrWexWMLTiqIAwfa+Qgghzk9SUyuEOGODBg3CZrORn59Pnz59GjwyMzPD63399dfh6bKyMvbs2cPAgQMBGDhwICtXrmyw35UrV9KvXz9MJhNDhw5F1/UGbXSFEEKI05GaWiHEGYuJieH+++/nnnvuQdd1JkyYQEVFBStXriQ2NpasrCwAfv/739OtWzdSU1P59a9/TVJSEtdccw0A9913H6NHj+axxx7j+uuvZ/Xq1bzwwgu89NJLAGRnZzNr1ixuueUWnn/+eYYNG0ZeXh5FRUVcd911HfXWhRBCdHISaoUQZ+Wxxx4jOTmZOXPmcODAAeLj4xkxYgQPPfRQ+Of/J554gl/84hfs3buX4cOH8/HHH2O1WgEYMWIE//rXv3j44Yd57LHHSE9P5/e//z0//vGPw8d4+eWXeeihh7j99ts5ceIEPXv25KGHHuqItyuEEKKLkN4PhBCtprZngrKyMuLj4zu6OEIIIc4j0qZWCCGEEEJ0eRJqhRBCCCFElyfND4QQQgghRJcnNbVCCCGEEKLLk1ArhBBCCCG6PAm1QgghhBCiy5NQK4QQQgghujwJtUIIIYQQosuTUCuEEEIIIbo8CbVCCCGEEKLLk1ArhBBCCCG6PAm1QgghhBCiy/v/itrxIleeIsYAAAAASUVORK5CYII=", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "fig, ax = plt.subplots()\n", - "epoch_lim = 6\n", - "\n", - "for key, val in architectures_TM.items():\n", - " y_avg = []\n", - " for y in val['epochs_y']:\n", - " y_avg.append(np.mean(y))\n", - " _ - np.round(y_avg[-1], 2)\n", - " ax.plot(np.arange(len(val['epochs_x']))[:epoch_lim], y_avg[:epoch_lim], label=f'{key} ({_})', marker='.')\n", - "\n", - "ax.set_ylim(0, 100)\n", - "ax.set_yticks(np.arange(0, 110, 10))\n", - "ax.set_ylabel('loss')\n", - "ax.set_xlim(0, epoch_lim-1)\n", - "ax.set_xlabel('epoch')\n", - "\n", - "pos = ax.get_position()\n", - "ax.set_position([pos.x0, pos.y0, pos.width * 0.9, pos.height])\n", - "ax.legend(loc='center right', bbox_to_anchor=(1.4, 0.5), framealpha=0)\n", - "ax.grid(axis='y')\n", - "\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAssAAAHHCAYAAABJIhU9AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAADGYklEQVR4nOzdd1iV5RvA8e85h8PeGxRkO1HcO/feq7RMK8uGlWY/28tRplmZLdOc5Ww4UjPNbSrurSjbATjY+6zfHy8cJEDBLLXuz3WdS3jH8z4vqNw8537vW2UymUwIIYQQQgghylDf7QkIIYQQQghxr5JgWQghhBBCiApIsCyEEEIIIUQFJFgWQgghhBCiAhIsCyGEEEIIUQEJloUQQgghhKiABMtCCCGEEEJUQIJlIYQQQgghKiDBshBCCCGEEBWQYFkIIYQQQogKSLAshKi0PXv28N5775Genn7X5vDSSy/RqFEjXF1dsbW1pXbt2rz33ntkZ2fftTkJIYT491KZTCbT3Z6EEOL+MGPGDCZMmEBcXBwBAQF3ZQ5t2rShcePGhISEYG1tzZEjR5g/fz5NmjRh586dqNWyBiCEEOLOsbjbExBCiKrYvXt3mW3BwcH873//Y//+/bRo0eIuzEoIIcS/lSzBCCEq5b333mPChAkABAYGolKpUKlUxMfHA6DX65k8eTLBwcFYWVkREBDAG2+8QUFBQalxAgIC6N27N5s2bSIiIgJra2vq1KnDzz//fNtzK17lvpvpIUIIIf6dJA1DCFEpx48f58MPP2TZsmV8+umnuLu7AzBgwADs7Ox47LHHWLRoEYMHD6ZDhw5ERkayePFi+vfvz6pVq8zjBAQEYGVlxZUrV3jmmWfw9PRkwYIFnDp1io0bN9KlS5dbzkWv15Oenk5hYSEnT55k7NixXLp0ifj4eFxdXf+2r4EQQoj/HgmWhRCVVlHO8rFjx4iIiODJJ59k7ty55u0TJkxgxowZbN26lQ4dOgBKsJyQkMBPP/3EwIEDAcjMzKRWrVp4e3tz+PDhW85j3759tGzZ0vx5zZo1mT17Nu3bt78zNyqEEEIUkTQMIcRftmHDBgDGjx9favvLL78MwPr160tt9/X1ZcCAAebPHR0dGTFiBEeOHCE5OfmW16tTpw6bN29m9erVvPLKK9jZ2Uk1DCGEEH8LecBPCPGXJSQkoFarCQkJKbXd29sbZ2dnEhISSm0PCQlBpVKV2hYWFgZAfHw83t7eN72eo6MjnTt3BqBfv34sXbqUfv36cfjwYRo0aPBXb0cIIYQwk5VlIcQd8+cA+J9SnM6xfPnyu3J9IYQQ/14SLAshKq2iYLhGjRoYjUbOnz9fantKSgrp6enUqFGj1Pbo6Gj+/LjEuXPnAG6rfnNBQQFGo5GMjIwqnyuEEELcjATLQohKs7OzA8qWaOvZsycAM2fOLLX9k08+AaBXr16ltl++fLlUhYzMzEwWL15MRETETVMw0tPT0el0ZbZ/++23ADRp0qRyNyKEEEJUkuQsCyEqrXHjxgC8+eabDB06FK1WS58+fWjQoAEjR45kzpw5pKen065dO/bv38+iRYvo37+/uRJGsbCwMEaNGsWBAwfw8vJi/vz5pKSksGDBgptef/v27bz44osMHjyY0NBQCgsL2bVrFz///DNNmjRh+PDhf9u9CyGE+G+S0nFCiCqZMmUKs2fPJikpCaPRaC4jp9fr+eCDD1i4cCEXL17E29ub4cOH8+6772JlZWU+PyAggHr16vHiiy8yYcIEoqKiCAwMZPLkyQwePPim146JiWHSpEns3r2bpKQkTCYTwcHBDB48mAkTJphXvoUQQog7RYJlIcQ/qjhYXrdu3d2eihBCCHFLkrMshBBCCCFEBSRYFkIIIYQQogISLAshhBBCCFGBuxos79y5kz59+uDr64tKpWL16tWl9ptMJt555x18fHywsbGhc+fOZeq4pqam8sgjj+Do6IizszOjRo2StrdC3MPi4+MlX1kIIcR9464Gyzk5OTRo0IAvv/yy3P3Tp09n1qxZzJ49m8jISOzs7OjWrRv5+fnmYx555BFOnTrF5s2bWbduHTt37mT06NH/1C0IIYQQQoh/sXumGoZKpWLVqlX0798fUFaVfX19efnll/nf//4HQEZGBl5eXixcuJChQ4dy5swZ6tSpw4EDB8zNCDZu3EjPnj25ePEivr6+d+t2hBBCCCHEv8A925QkLi6O5ORkOnfubN7m5ORE8+bN2bt3L0OHDmXv3r04OzuX6trVuXNn1Go1kZGRDBgwoNyxCwoKKCgoMH9uNBpJTU3Fzc2twna+QgghxL+FyWQiKysLX19f1Gp5fEmIm7lng+Xk5GQAvLy8Sm338vIy70tOTsbT07PUfgsLC1xdXc3HlGfq1KlMnDjxDs9YCCGEuL9cuHCB6tWr3+1pCHFPu2eD5b/T66+/zvjx482fZ2Rk4O/vT1xcHA4ODndxZkIIIcTfLysri8DAQPmZJ0Ql3LPBsre3NwApKSn4+PiYt6ekpBAREWE+5sqVK6XO0+v1pKamms8vj5WVVan2u8VcXV1xdHS8A7MXQggh7l1arRZAUg+FqIR7NlEpMDAQb29vtmzZYt6WmZlJZGQkLVu2BKBly5akp6dz6NAh8zFbt27FaDTSvHnzf3zOQgghhBDi3+WurixnZ2cTHR1t/jwuLo6jR4/i6uqKv78/48aNY8qUKYSGhhIYGMjbb7+Nr6+vuWJG7dq16d69O0899RSzZ89Gp9Px/PPPM3ToUKmEIYQQQggh/rK7GiwfPHiQDh06mD8vziMeOXIkCxcu5JVXXiEnJ4fRo0eTnp5OmzZt2LhxI9bW1uZzlixZwvPPP0+nTp1Qq9UMGjSIWbNm/eP3IoQQQggh/n3umTrLd1NmZiZOTk5kZGRIzrIQQoh/Pfm5J0Tl3bM5y0IIIYQQQtxtEiwLIYQQQghRAQmWhRBCCCGEqIAEy0IIIYQQQlRAgmUhhBBCCCEqIMGyEEIIIYQQFZBgWQghhBBCiApIsCyEEEIIIUQFJFgWQgghhBCiAhIsCyGEEEIIUQEJloUQQgghhKiABMtCCCGEEEJUQIJlIYQQQgghKiDBshBCCCGEEBWQYFkIIYQQQogKSLAshBBCCCFEBSRYFkIIIYQQogISLAshhBBCCFEBCZaFEEIIIYSogATLQgghhBBCVECCZSGEEEIIISogwbIQQgghhBAVkGBZCCGEEEKICkiwLIQQQgghRAUkWBZCCCGEEKICEiwLIYQQQghRgXs+WM7KymLcuHHUqFEDGxsbWrVqxYEDB8z7TSYT77zzDj4+PtjY2NC5c2fOnz9/F2cshBBCCCH+Le75YPnJJ59k8+bNfPfdd5w4cYKuXbvSuXNnLl26BMD06dOZNWsWs2fPJjIyEjs7O7p160Z+fv5dnrkQQgghhLjfqUwmk+luT6IieXl5ODg4sGbNGnr16mXe3rhxY3r06MHkyZPx9fXl5Zdf5n//+x8AGRkZeHl5sXDhQoYOHVqp62RmZuLk5ERGRgaOjo5/y70IIYQQ9wr5uSdE5Vnc7QncjF6vx2AwYG1tXWq7jY0Nu3fvJi4ujuTkZDp37mze5+TkRPPmzdm7d2+FwXJBQQEFBQXmzzMzMwHQ6XTodLq/4U6EEEKIe4f8rBOi8u7pYNnBwYGWLVsyefJkateujZeXF8uWLWPv3r2EhISQnJwMgJeXV6nzvLy8zPvKM3XqVCZOnFhm+6ZNm7C1tb2zNyGEEELcY3Jzc+/2FIS4b9zTwTLAd999xxNPPEG1atXQaDQ0atSIYcOGcejQodse8/XXX2f8+PHmzzMzM/Hz86Nr167ydpQQQoh/veJ3VIUQt3bPB8vBwcHs2LGDnJwcMjMz8fHx4aGHHiIoKAhvb28AUlJS8PHxMZ+TkpJCREREhWNaWVlhZWVVZrtWq0Wr1d7xexBCCCHuJfKzTojKu+erYRSzs7PDx8eHtLQ0fvvtN/r160dgYCDe3t5s2bLFfFxmZiaRkZG0bNnyLs5WCCHEvW7XxV00/b4pq86vuttTEULcw+75YPm3335j48aNxMXFsXnzZjp06ECtWrV4/PHHUalUjBs3jilTprB27VpOnDjBiBEj8PX1pX///nd76kIIIe5hXx39inxDPl8e/RK9UX+3p0N2YTanrp/CaDJWeMy1vGvEpMfcdJz0/HQuZF6409MT4j/rnk/DyMjI4PXXX+fixYu4uroyaNAg3n//ffNbSK+88go5OTmMHj2a9PR02rRpw8aNG8tU0BBCCCGKnbx2kpPXTwKQkpvC9gvb6Vyj881P+hudvn6acdvGkZSThJ+DH4NCB9EvpB/uNu4YjAb2Ju3lx3M/sv3CdgwmA6PqjeKFhi+gUWtKjbPr4i5e3fUq2YXZvNjoRUbVG4VKpbo7NyXEv8Q9XWf5nyL1JoUQ4t9FZ9CxKnoVD1R/AG877zL739z9Jmtj1mKtsSbfkE9z7+Z82+3buzBT+CXmFybunUiBoaDUdguVBa2rteZ82nku51wuc15r39ZMe2AaTlZOmEwmvj3xLZ8f+RwTJT/Wu9TowpTWU7DVlq70JD/3hKi8ez4NQwghhKiqr499zeR9kxmzZQw6Y+mawmn5aWyM2wjA5DaTUavURCZHEpse+4/OUWfU8eH+D3lj9xsUGApoW60tvw/+nUmtJlHfoz56k54dF3dwOecyjpaODK89nFV9VzGt7TSsNdb8cfkPHlr3EEeuHGH89vHMOjILEyYGhw3mzeZvYqG2YHPCZh7Z8AgJmQn/6L0J8W8iK8vIb9hCiMq5kHmBOSfm8HT9p6nuUP22x9l7eS/rY9czoekEnKyc7uAM7z/7kvaxMW4jE5pOwE5rV/5Bp9fA4e/gxlxetQaaPgVhXcscnp6fTrefupGrV2oJj288nsfrPW7eP+/EPGYenklt19qs6L2CsdvGsu3CNobVGsYbzd+45Zy/3h6Dg7UFw1vUqNrNFilOq/jm2DccvXoUgGcaPMOzDZ5FrSpZwzqXdo6tiVupZl+NLjW6YG1Rkl4YlRrF2G1juZR9ybxNq9byZvM3GRQ2CICjV44yfvt4ruZdxUHrwIcPfMgD1R8A5OeeEFVxz+csCyHEvWLBqQWsjl5Nnj6PGe1m3NYYp66d4sWtL5JvyCfUJZSRdUfe4VneP3J1uby681VS81Op4VijVEBrZtDBupcg93rZfSmnYOxx0JT+Ubbo9CJy9bk4aB3I0mXx1dGv6FKjC9UdqmMwGlgZtRKAYbWGoVKpGFprKNsubGNtzFrGNhpbcdAOxFzNZtrGswB0qeOFl2Pln49JyUlhdfRqfj7/szmtwk5rxwdtPqCjf8cyx4e5hBHmElbuWDVda7Ki9wpe2fkKey7vwdPGk087fEp9j/rmYyI8I1jRewXjt4/n6NWjLDi5gLbV2koOsxBVJMGyEEJUUlRaFADbL2wnqzALB0uHKp2fkpNiDpRBecjs3yI1P5Xx28fjYOnAey3fw83G7ZbnrIxaSWp+KgDbLmwrP1iO2aoEynYe0HVKyfbf3oTMSxC1Aer0NW9Oy09j6ZmlgJJiseTMEg4kH2BK5BS+7vQ1uy7t4nLOZZysnOgR2AOAFj4tCHAMID4znnUx63io1kMVzvnYhXTzx5tPp5S7upxRkMHozaNJzEwstT1Xn2uudOFo6Ujf4L48XPth/Bz8lAO2fwinVsHg+eBVt+IvXBEnKye+6vQVkcmR1HWrW+67FB62HszvNp+vjn3FI7UfkUBZiNsgOctCCFEJRpOR82nnASgwFPB7wu9VOj9Pn8cLW1/gSt4Vc5B94tqJvzQnnVFHji7nL41xJxQaCnlp20scSjnE9gvbeWjdQ8ovAiYTZKWUe06uLpcFpxaYPz965SjX8q6VPfD4CnRARt1+0GBoyatx0Yr8/jmlDl90SllVru1am45+HXm7xdto1Vr+uPQHv8X8wvKiQHpAyABzWoNapeahmkqAvDxqOTdmJ5pMplLzujFY/u1Ucrn39umhTzl9/TTZuuxSL6PJSGOvxnzQ5gO2DNnCq81eLQmUL+xXguWrZ2HpQ5B9pdyx/0yj1tDKt1XpQFmXD/kZ5k+1Gi1jG43F3ca9UmMKIUqTYFkIISrhUtYl8vR55s/Xx66v9LlGk5E3d7/JmdQzuFi5ML/bfGXM7EvmldXb8erOV+m4siPxGfG3PcZfZTKZmLxvMoevHMZea0+AYwApuSmM/HUkq1b0h4/D4JsH4OB8yC9psbziyFek5qdS3WCkdkEhJkxsv7C99OD5mXB2Pa94utPp+jaOXDlSsq/x46BSQ/wuuKKkRaTmp7L0rBIMP9vgWVQqFYFOgTxV/ykApux+iz+S9qICHqz5YKlL9Q3pi42FDdHp0RxMOcj1vOssOLmAPqv70GFlB2YdngXA0YslQejemOtk5JV+ePBQyiF+Ov8TALM6zGL9gPXm17YHt7Gw+0L6BPcplX+MQQe/jAVMyj1lXIAVw5Wgt6qMRljYC6YFKEF31K9guPs1pIW4n0mwLIQQlXAu7RwAnraeAOxP3k9yTvkri3/25dEv2ZywGQu1BTM7zKSWay0CHAMAJYf5duTr89l2YRu5+lxWRVfcgS4xM5HDKYdv6xo3On39NAeSD5RpmLHw1EJWR69GrVIzo90MlvVaRnu/9hQaC3mnIJYpbi7oko4peccf14LVY8hd1IcFJ5VfGJ5OTaNrjvIg3pbELaUvenYdsSo9v9vZUmDU8d6e9yg0FCr7nP2gZk/l4wNKybdFpxaRp8+jtmtt2vu1Nw8zqt4oAq1cyVApK8ZtcvPw2zunVBDpaOlI76DeALy+63U6/9iZTw59Yq4iMffEXH46t5ozl5WA38VWi95oYntUyQpwoaGQiXsnAjDYtz0dribif36b+eUev7f8wHXPLLhyGmzd4PGNYO0EFyLhlxeV1fkbJR2H+D8q/D4Rtx0uHVQehjy3EZYNhZnhsO0DSJdGJULcDgmWhRCiEoqD5Va+rWjk2QgTJn6N+/WW50WlRjHnuJIq8F7L92jk1QiAcPdw4Pbzlk9dP2XuOrc+dn25Xd8KDYU8vvFxRm4cyS8xv9zWdUCp3vHw+od54rcn6PlzT7498S3X8q6xLXEbnx76FIBXmr5C62qtsbe05zP3BxiTlo7KZGKFowOjajfjmkco6HLg6PcsTz1CmkaDH1p6d55Bx0IlIIy8vJfswuySCx9fwXKHkrzw2IxY5hcF2QA0fVL589hyUjMSWXZ2GQDPRTxXKjfX0gTvXC1ZwR+WmaUEqN8PhJySBweH1hoKKE1K9EY94e7hTGw10ZxLPXnfRAyWsTjZaBnWzB+ATadK0kzmnZxHXEYcbhZ2jNv7nRLs3vhaMRy+HwA5N6SbpMbCjunKx90+AP/mMGQRqDRwfAXs/gQKsuDgAvimHXzTFhb2hPPlpwFd3KSsgP9iaMm1+k8rAXjWZdgxDX56soLvsBDiZiRYFkIIlHJjnx/5vFQprhsVB8uhzqH0DlZWINfFrrvluMUBXJcaXegX0s+8va678gBXcRe5qroxJSElN4VDKYfKHLMpYRNX8pSVz3f3vFs6jaGS4jLieHnHyxhMBjQqDZeyL/HZ4c/o8kMnJmwbhwkTD4U9yMO1HlZOuHgQ9ZoxPJOeyeeuLbHX2nMkP5kHPRw5OvALcps8xkJ3pUnI063fxSLiYYIaPk5AoQ6dycDui7uUcTKTyInbyVoHpTJFLTulu97c43NL0k6C2oNbKBRmMXfnm+Tp86jrVpd21duVvoljS2lyPZF3MnU8V+8pWvf6CrR2ELcD5rSHpGOAUn3itWavMaLOCH7o8wNLey1lYOhAxjUaR5caXTCY9FhX/45afnq611PuYXvUFfJ1BuIy4ph7fC4AryZfwsloAr/mENaj6NUdLO0hbifMaY/uwmHm7Igm/YcXQJ8Pge2gftGDhcEdoOdHysdbJsGMmrBuHCQdNd9S3qZJpVadC/QGpq/YjE/ydgBm6gfykXE4jD+jPDAY+AA0fqzK338hhATLQggBwIcHPmTO8Tl8dvizcvcXB8thrmF0rdEVrVrLubRz5u3lySzMZEPcBoCSYLLIjSvLt1Pu/tgVJcCztVA6s5UXuC8/uxwAFysXdEYd47aNq/CXgfJkFGTwwtYXyCrMIsIjgh3dvmOyS1Ma6IzoTUYKMNI8L59XI1ei2jUDLh6CZcPAUAA1e9Ku92yW9VpGsFMwV/Ou8vixTxhvVUCasQB/B396BfVSLtR6LB3zldzfraeVnGNO/sg6e1ty1Goo9ODAwY5oC2tTaCxk8r7JytdMpULX9AmmuLnwfepRoOyqMvpC2KmU+RvS5EWebfwi6nqD4MnfwSUQMhJhXlc4tgKAR2o/woSmE6jlWss8hFqlZkrrKTiqAlBb5HDZ6ksCPTR4O1qTU2jgj+irTN43GZ1RR+sCA92zMqFWbyWl4uHlRa8VyjVdg5Wc5PndcN48Huek3RjVltD7U7hx3k1HQbOnlY91OeAWSlqbd+lt+JhckxU2V48xeeZnrDx4gfhrOQybsw/7E4vRqEycs21EjKkaa45dIqNQDfUGwchflAcjhRBVJsGyEOI/LzYj1pxSsT9pf5ngNVeXy4UsJd8zzCUMJysn2lZrC9z8Qb810WvI0+cR6hJKY6/GpfbVdK2JhcqC1PzUclsZ/xLzC89veZ70/PQy+0wmk7mZxdMNlIBqU/ymUu2ST18/zbGrx7BQW7Ck1xJqu9YmNT+V57c8XzrVoQI6o46Xt79MQmYCPpZOzLyWjtPXbeh/+Ce+v3iRn67n845lDT7LKECbnghbp8C3HSHnCnjVg4FzQK0hwCmAJb2W0KVGF/RGPX9c/sM8bwt1UfVSO3c6Bill3HZeO0ahvgDT8eUsc7QHID+1BaAmLbE3JqOW/cn7WRuzlmt51xh1fTcrHJVUjTE1epu/L2ZHl0DGBUz2Xnx0vSVjlhwmp0APXnVg9DYI6aKs7K4aDb++pjxsVw5brS1WqaMw6hxI1yfSb00/1H4fYRv0CW8cGM6B5ANYm+CtK8kYPcMZr3uOievPkK8zlAziWRue2kqCW1u0pkIetNgBwFemgSRQtiU33T6AAd/AYxswjdnP+AttOKnz4SdNdwD6pi3ilR+P0X7Gdk4lXmGoxTYAQnuPo5a3A/k6Iz8cuiFPWcrGCXFbJFgWQvznfXPsG3PO7/X868RlxpXaH50ejQkT7jbuuFq7AphTMSrKFzaajOaV3aFhD6H6UwBupbEizFVpOGEus1Z0jM6oY/qB6ey4uKPch/fiM+NJL0jHSmPF8NrD8bbzJluXzY4LO5RqCJSsKnfx74Kfgx+zOs7Cw8aD6PRoXt31KhkFGWQVZimvgkyyMi+Vek3d8RqRyZHYGk18HnsWt9idysWDO8KQRYS9eIohw9ZhNz5KCej8Wyr77Txh2DKwKsk1ttPa8XG7jxnbaCwqVNR0qUnPwJ6l7im83Tt4GIzkqEzs3/oGB9PPE2NpCUYtxqzGfDeqGc2qh1J4rRMAk/d8yJBfHuTItRPYoeGL5CuMTr5YdlV518cAnAkaxZe7L7P+RBLjVhzFaDSBjYuy4tv2f8rxkV/D4v6QnqiUXit+FeaSXaAnLkVL3sURWGtsuJp3lSzjJTRWV8g1KXnLz6emUc3anZctXuPnU+ks+COewbP3cDEt1zyl3+MK6HD5aT7TDwQgRhPMZ3k9eWLhATLz/xSoayyU1eCA1qw/mcy2qKtYatS0HTkJk4UNDdSxPOh0BoBRLsdwJQscq6Gq2YsRLQMA+G5fgnKvQojbJk1JhBD3jJmHZvLd6e9o59eOwWGDaeHTolT7379DbHrJqnI1+2pcyr7EgaQDBDkFmY8xp2Dc0E3tgeoP4KB1MOcLN/VuWmrcvZf3kpiViL1KQ+9VL4PLF/DgInAtGTfcPZzT109zMmoN3X58EUI7Q78v2XNpD+kF6YBSIeLPzTqOXjkKQF23ulhqLOkV2It5J+ex7vh8ui5/kozAtmzQKeXUih9a87bzZlbHWTy28TF2XtxJm+Vtbvm1UZlMTLtyjZpWrtD8EWg0AlwDSx+ktSmpfZwWD1aOYOtadiyVit7+w5m9zhWtzg29QYXFDd9atb0nHRyCWZkbx5aon8goWi0uzGjEk63q0DbUgxZBbkxZb8sPSUcpsE6mIC8bQ4Enthe70E49FcPZdaQvfwbntk+BbyM48h1kXMBo78XTp+uZr7X5dArTf4vitR61lLbZnd4G3whY9Qwk7FaqR/yJ3rsVPVVNOGHZmtWDNxKTHoO+IJdFy76nnXE/9blMHaOab4PfZ/UxFdZaNTZaDScvZdLn8918+XAjXOwsGbv8CEaTmiuNx2PqMAkHgx1u3xwh5moOzy89wvyRTbDQlP47n5GnY+IvpwF4rkMwATUCoPlo+OMzprmt57ERo6m1/mPIA5o8DhoL+jf0ZeqvZ0i4nsuO81fpUNPzlt9vIUT5ZGVZCFFl1/Ous/vS7lKvwymHMRgNFZ5TaCgkJj2mwv0Xsi6w8NRCCo2FbE7YzNObn6bnzz2Ze3wuGQUZFZ5XKSYTXD4Kurwyu2Yfn40JEx38OtA/pD8AB1IOlDqmvGDZSmNF14CuwJ/yhQ06OLSQ5b+9AEC/9HRsC7Ig+TjM6QDRJVUM6rrWAeBE3GbIToYj38PVqFLjHb96nKu5V0vNpzgFI8IzAsBc8mxX6kkyCrNYfWkHBYZCajoG0NCzofm8eu71mNp2aqU6D1objbyh9qJ937nw0ino/G7ZQPnPXALKDZSLfbktmmsZVuyNzub1n0+USXfp2OhZADbb2bLV1gYAV0N7xnYOBUCrUTOxbwPG1HsD9E7oMiLIjR9DbGE4PxvaoMGI89llMLejUjVixzQANjgN40I2BLrbMX2w0g569o4Yfjh4Q4pC7T7w1FbwKhsoAzgn7+FLy1msNzyN685PaHpoOS2XDmd2/kqGFcZTU2diV70pvH9MySGf+VAEv7zQhnrVHEnL1TF8XiTDv40kp9BAq2A33utbF5WzH55urnw7sgk2Wg07z11l8rrTZVaCp208y9WsAoI87Hi2fbCysdWLoLVDdfkIdc5/jfryIVBroZHSrMXW0oIhjZWGJ9/tTbjpt00IcXOysiyEqJIz18/w2MbHyNXnltnX3Kc5Hz3wES7WLqW2J2YmMnbbWKLTo3ml6Ss8WufRMufOPT4Xg8lAI89G1HStybqYdVzKvsSsI7PYGL+R5b2Wo9Vob2/Sp1fDD4+Be00YugTcleArJj2GjXEbAeXBsOJueAeSD2Aymcxv6ZcXLAP0CurFT+d/Yl3MOpp7N6dnUE9YM4ZLp39kR3VfQMVD3i2h63DY9YlS//b7wdDpHWjyOOH7FyrTs7LEYO+FJjuF7H1fsy1jNwCu1q6k5qey7cK2Uk00ileWiwPhEK0jtfQmzlqo2Fi9DstRfrkYmnAKVdSvUKsk5aFLjS508OugpI5sfF2pUay1hRGrwLOkxbJabYGF1ub2vt7luJyex4oDSnCqVsGqI5cI8bRnTIcQ8zHNAjpj/4cFGRrlc2NODab27oqtZekfVWNad+DpFjsx3BBsZ+Z24oWvvqVDzq/01kRimax0Ryy09eLl2AgA3u9fj1Yh7lxMzWXW1mjeWHWCAHc7mgYUBfgeNeGZXVBcy7lYVhIbvptBo+vr8DakwZ7Pzbty7Pz5PL0Vmyw7kbDfHjAxoVtNutfzAeDHZ1rxxqoT/Hz4EtdzCgl0t+OrRxqhvWH1uF41Jz59qAHPfH+YRXsT2Bp1haFN/RnSpDoXUnNZGqm0zf5gQDhWFkVfHDt3aPYU/DETtk9VttXtD/YlK8iPtqzB/D/i2BZ1hQupufi52lbqeyWEKE1WloUQlXY19yovbH2BXH0u3nbe1HatbX7ZWNgQmRTJ0HVDOXP9jPmcXRd3MXT9UKLTowHKLc92IfMCa2PWAjC+yXjeaP4GWx7cwpTWU3CxcuFc2jkWnlpY7px0Bh0/nvvx5l3sjv+g/HktSll1jFLSLmYfU1aVO/l3opZrLcLdw7HWWJOan0psRiygPExXUbDc2KuxuQHHq7teZcbmF9AfX8EKR0dMKhUtPBsT+OAyqNULHt9QtOpngi0T4dN6BEbvwMZoJE+tJrbruwBsiVlLgaGAQKdA8y8VWy9sNV8zoyDDPDcHgll9IAbj8ofplZGufH1t4KJGhYNJRc+MVFg+DH5/D3JL6gxbqC2wPLwYywPfYglYDvgGS78WWFo5mF93MlAGZVW50GCkZZAbE/spKREf/RbFxpNJ5mO0Gi3NvEse0Kvn2Jv2FaQPWGjUWFlozC8PR2vGPvEY72pepGn+l6zyegFjUCcm8iwFJksGNapOqxCl3fO4zmH0DPdGZzDx9HeH+GRTFJ9sPqe8fj/Ppqg0sLAqebkE8H7uQFoXzOJM+2+gTn8IfxBGrEH14iEWqPoTm2eHwWhiQMNqPFe8+gtYazV8PKQBHwwIp0sdL+Y/1hRnW8sy99O9ng9T+tfDwdqCC6l5fPRbFK2mbuXJRQcBeLBJdVoEuZU+qWh12azZ6FK7A93taBvqjskE3++T1WUhbpcEy0KISsnX5zN221hSclMIdArkp74/sbLPSvNrSc8l+Dn4cTnnMo/++ii/xPzC3ONzGbNlDFmFWTTwaECERwR5+jze3/d+qbfgvzn+DQaTgdbVWtPAowEANhY29Avpx4SmEwAlsE3MTCw1J5PJxMS9E5m4dyIvbH2h/DSQwhyIKeoM51kXCjJh2VCiN73Gb/G/AUprZABLjSUNPJXrH0hWUjFSclPIKszCQmVBoFPpNAS1Ss3M9jN5Mlxp9rDo8nae8fZklbOyUjm07g0r6BZW0HcW9J6pvF1emI3GyZ+6bkoqxkkbW3ALZZ218t9y76DedPTvCEBkUiRZhVkAHLuqlIzzs6/BqPmnMa59EfXFA3TTaVChIqNQ6TDXr/YwbItLj+3+FD6uqTSliN8NMVthwyvKvk7vQJ2+5X/T75BL6XmsLEp5GNc5lEdb1GBkyxoAvLTiGMcvprMt6gqjFx/k14O+ykl6Rz7tM7xK1wnxtOerRxqRrXbgpYSWDMx6mSWpYbjYanmzV23zcWq1io+HRBBezYnUnEJmbY1m1pbz5tfo7w5xIL7kl4urWQVcSs/DqNJQvcUgJfd80FwIao+tlSVtQz0AaOTvzNSB4aUfMkTJ1364uT9zRzQh0N2OigxvUYP9b3RmxpAGNK7hgt5oIi1Xh5udJW/0rF32BDs3JXcZwLs+VG9a5pCRRQ/6rTh4oXRlDiFEpUmwLIS4JZPJxNt/vM2JaydwsnLii45f4GjpWOqYUJdQlvVaRttqbSkwFPDG7jeYdWQWJkwMCRvC/G7zmdR6Elq1ll2XdvFbghKoJmYmmnN0n2vwXOkLF2TT+/RWmjsGUWgsZNK+SaWC7IWnFrImZg2gVIgormlcSsxWpTSYcw0YvR2ajUYPfBb9AyZMdPbvRE3XmubDm3opAUdxsFy8qhzgFIClpuyKoEatYWyjsXzi0hwbo5FIG2vSTTq87bzLNscA5QGsUZugw1swejvhvkoViZPXTnKl4TAira0B6BnQgyCnIAKdAtEb9ey+pKRmFDcWMeT684RuKQM1u9Gb1EzVj6WOSyPzZYbWfhh6TIMhC5VAylAIJ36Ahb3guwFgMihNMNqMLzvHSth9/hpjlx8h4XrOLY/9cls0OoOJVsFuNC9aHX27dx3ahrqTpzPQ94s/eHzBATadTqEwsy6ehQ/xVtPp+DrZV3lebUM9eK+P8gvI0QvpALzZqw6udqW/dzaWGhY+3pQXO4YwomUN86tZUUrGGz+foFCvVBY5flEZJ9jDHgfrsqlA7/Suw8tdwpg3sinWWk2V5/zneQ1uXJ2fnm3FppceYHyXsApXowFo9yp0ehcGfVtuabgOtTyp5mxDeq6OtcfKligUQtyaBMtCiFuafWw2G+M3YqGy4NP2n+Lv6F/ucU5WTnzR6Querq+saGrVWt5r+R7vtHwHS40lgU6BPBX+FADT9k8jszDTvKrcplob6nvULz3glkmoDs3nndiTWGmsiEyKNAfWN7ZaLj5vzvE55hbQZmeKHpar1RssLEnr+AbPhLdlu50tapOJZ5zqlTq8mU8zAA6mHLxpCkYpySfpcuRnll5OoYa18lb/8NrDS+oI/1m1RtBuAti5mTv5nbh2gl/t7TCpVDTMz6f69XgAOvopq8tbE5VUjOJ85RbJx3nRYjUAX1iPZm1WTY6eVoL+dtXbUcNRWbml7gAlD3f0dqWDm2VRAFq9GfSZVeXauyaTia+3xzBifiRrjl5m9o6KH9oEuJiWa36Qblznkq+hhUbNFw83IsRTmY+TjZYnWgey+aX2bHnqLR6q37pK87rRoy0DGFG0ct06xI1BjaqVe5ybvRXju9ZkUr965tecEY1xt7fk/JVs5uxU7u1YUdDdoLpzueP4u9nyQqdQXOwqCGhvU5iXAy92CqWBX/nXBZRqJG3HK/nW5dCoVQxvoXwtlu1PLPcYIcTNyQN+QtwHlp5Zyuro1XzZ6Us8bD3+0WvvvrSbr459BcDbLd8uUyLtz9QqNc83fJ6O/h2x09qVBG1FRoWPYkPcBuIz43lj1xvmFdMyq8oXD8H+OQD4Z13j6SbDmBW/ho8OfISnrSev7npVabVc8yFeavwS3X/qTnxmPL/G/Uqf4D7KGAYdnFMe4KN2b85cP8O4beO4nHMZG5WGKSnJ1Dy5Dho9ab5sPbd65rzlmPSYWwfLRgP8MhZMBkJCerFy4DdEpUUR4RFxqy8tUNLJ73zaeVYVPVjWOztXuffAtnT078i8k/PYdWkXubpcTlw9DsDIwjPoVRZY9JzGE+EjObniGL+fCUdXMJpQ/3JWtH0bKq+uUyBhLwS0Bq11peZYLKdAz4Qfj7HhRLJ52+bTKUzpb0KjLj/o/nJbDDqDidYhbjQLLF0pw8lGy4/PtOTohXRaBLn95VXZG73Xpy5d6ngR4edcJi3iZpxtLXm7dx3GLj/KrK3R9Krvy7GLygOTEX5Od2x+/6SHmvphMBp5qGn5v+QKIW5OVpaFuA8sPLWQM6lnzKuL/6RV55WmGINCBzEwdGClz6vjVqdMoAxKXvA7Ld8BYMfFHRhMBtpWa0u4xw0luwx6JQDFBEWrs4/l6glxDiGtII0nNz1Jnj6PFj4teLXZq9hp7RhZVymZ9c3xb0pWlxP2QH462Lrxi+4aj/76KJdzLuPn4MeSdp/RNTcfYrfBtWjzpbUarbkk257LkUSlKsFyqEto+Td6cL5S5cLSAXpMw1ZrS0PPhqUCtMx8HXmF5eeL+tj54Grtit6kJyYjBguVhq45uXB2PWRcop57PTxtPMnR5bBo5zsUGAtxNBiw19lT+MgaaPokjtZa5jzamBc7hmLIC2L21stczy4o93pYOUBYV7CsOHe2PHHXchjw1R9sOJGMVqNiYt+6OFhbcC27kCOJaeWecyG1/FXlGznbWtK+pucdDZRByUtuG+pRbtrErfRt4EvbUHcK9UbeXHWCY0VpGPUrWFm+17naWfJ8x1A8HKzu9lSEuC9JsCzEPS45J5mkHKViQHFFiX+K3qhnX9I+AHMN4juhqXdTBoQMMH/+XMSfVpX3fQUpJ5QOa72UDmzaqPW82+Id8yEBjgHMaDcDrVoJhobVGoazlTMJmQkluctn16EDpvmF8cYfb1JgKKBttbYs67WM0BrtIKybctyBb8vMD+CzP9YRm6508yt3ZflqFPw+Ufm487vg6FvmkItpubSdto1Gkzfz6o/HOZKYVirvWqVSUc+9JBWkbfUHcPZrqeQUH1qIWl9AB9vqACxMVKp4BOZrOdVrLbYhJY1F1GoVL3UJo351J/J0BubsjC073yoyGk3sPn+NMUsO0/XTHZxLycbTwYrlo1syslUAnWoplSo2nU4p9/zZO2LQG020CXEvKc92H1CpVEzpXw8rCzV7Yq6TnqvDUqOmls+t61MLIf59JFgW4h5X3IACICbj5vmhd9rJayfJLMzE0dKxVEB3J7zc5GWaejdlRJ0RpcdOSyipG9tlslKiy8IG0hOJMFkwJmIMdd3q8nnHz3GyKnlb3E5rx2N1HwOU9tV6g47rUesZ7e3J94VKqbrR9UfzRacvSs5rquRPc3QJFGSbx2rmreQtF1ieApURS5U9XrZepW8gNxWWPgSFWVCjNTR5otz7/HJbNBl5OvJ0BlYcvMCAr/bQ47NdfLc3Hp1BeYCsnlvJ/fcO6g1Ni9JCImfDxzXpeEpJJclVK/9lp9n2oX2TP+V3owR544oaeCzem8C1ilaXbyGnQM9X26NpP2M7w+dFsv5EkvkBvXUvtKFxDaWOdte63gD8diq5TIORjDwdPx9Wvu431lK+X9RwszM3QwGo7eNQUuNYCPGfIsGyEPe4Y1eOmT++WQe8v8Oey3sAaOHTouKH1W6Tk5UT87vNN5eGA5ROe+tfBl0u1GgDDYeDpS2EdFL2n1nHMw2eYXnv5QQ4BZQZc1itYbhYuZCYlcjMnW/wkKOJgzbW2FnYMbPDTF5o+ELp9tnBHZX20wWZcGKleXNdt7pYqKxQqZQAsCDXk4w8Xcl5+kJY8SikxSlVNh5crLRN/hMlDeEiAJP712Ngo2pYWag5m5zF22tO8ci3kVzLLjCnoNhr7Wnn107pJmfvrcwrP4Omlp7YUZJO8EKbHhXm4Xao6UkDP2fydAa+ucXDd+UpTreYvjGKxNRcHKwsGNGyBhtebMvSp1rg6ViS59wuzANLCzUJ13M5l5JdapyfDl0kT2cgzMueFkH3z6ryjZ5qG0RNL2U1+aYP2Qkh/tUkWBbiHldcKgwgNT+V1PzUco/bn7SfL49+ae5Cdyf8cekPANpUa1N6x6GFSq5uVVw5W6Y5RhmnfobozaCxhN6fllRqqNVL+fPs+ptewlZry2P1HgNgUeJGUiwsCFBZsbTXUjr5dyp7glpdsoq7/1slWEfJW7Y1ljSW0OV58+0uJR1DCejHQ8JuTJYOrK83kzXny1/B/XJbNHqjibah7jzaogafPBjB/jc6807vOthbWbA/LpU+n+/GzlCH0fVHM7XtVKw0VqDRwsBvlNXq4T+hHXsMF62y2q1CTbuARuVeD0qvLn+3L4GrWaXndjEtlynrTrP22GUK9KXzqLeeTaHvF7vN6RbTB9cn8s1OTOpXjzq+pUsFAthZWdC2qNHHplMlD/0ZjSZzE4xHWwZU6QG7e4lWo+br4Y14pLk/ox8IutvTEULcJRIsC3EPy9Xlcjb1LKCkGUDFq8vv7nmX2cdm8/D6h4nLiPvL107PT+fk9ZMAtCyqBQzAuU3Kw3frXoLIOZUbzGiEn0YpzTHWjSv/mLw0+PU15eM248HjhhzhsO6g0ih5zGnxN73U0JpDcbVWVjLb5+SyNPxFgpxvEuhEPKykeVw5BYl7AdAbjGSklTycaCzwZsEfcaTlFMLeL+HId6BSsy7sfcb8nsfY5UfNTTeKXUjN5cdDyqryuBvezney1fJEm0BWj2lNsIcdSRn5PDQnEi99f9r7tS8ZIKi98gtDSGdQq0m/qjSlCLCvg43FzbvrtQ/zIMLPmXydsdTq8p7oa/T5fDff7o7jxWVHaDl1Kx9sOEP0lWxmbTnPqEUHycrX07iGC+teaMODTfzKtJr+s651lfSU306XBMu7o68Rey0HeysLBjQsv2zb/SLIw573B4RT3UVaRQvxX3VPB8sGg4G3336bwMBAbGxsCA4OZvLkyaVy40wmE++88w4+Pj7Y2NjQuXNnzp8/fxdnLcSdc+r6KQwmA562njTxagKUHyxfz7vOxWwlMIvNiOXh9Q+z/cL2v3TtfUn7MJqMhDiH4G2n5KZSmKOkSRTb+CpEb7n1YGd/gRQl8Ob0GojaWPaY3ydCzhVwC1Xqxt7I1hVqtCoa69ary4uaT2JWylU+u5aOQ60+xFzN5qFv9vL26pOcvpxZ+gQbF6g/RPl4/1wATl7OJDczwHyIv30IOYUGNq1fCZveAuBM/Vd54aC7+Zg3V51gX+x18+dfbC1ZVW5co2waQoinPavHtKZrHS8K9UZe+ek4n2w+V+49xV/LISkpmMJLI/i4/bSb3j+UXl3+PjKBK1n5zN0Zy/B5kaTl6gj1tMfb0ZrUnELm7Iyl8yc7+GTzOUwmGN7Cn2V/Sre4mc61vVCr4OSlTC6l5wFKvjTA4MbVsbeSCqVCiPvbPR0sT5s2ja+//povvviCM2fOMG3aNKZPn87nn39uPmb69OnMmjWL2bNnExkZiZ2dHd26dSM/P/8uzlyIO6O4AUVDz4YEOytpAeVVxDh1/RQA1eyr0cizEdm6bF7Y+gJfH/0ao8lYtYsW/TL6x2UlBaO17w3NIbZPhYxEcPJTHrwzGeGHx5WqEBUxGmF7UYDnqFR1YMP/Sj1QR+I+OLRA+bjPTKU19J/V6q38eYtgGSDg4hE65OahDmiL0cqZl1ceIzIule/2JdBz1i76ffkHKw4kklNQVGKu+EG/M2shK5l9sdcx5lXHCg88bTx56YG2ALieWgSYSAsdxMDDSlvsES1r0Ku+DzqDiWe/P0TC9RwSr+fy4+HiVeWKm5k4WGuZPbwx47sox8zeEUNmvq7McTvPXwVUNHRvQ6hb5WrltgvzoKG/srrc74s/eH/DGYwmGNioGr+80Ibdr3bg2xFN6FTLE7UKLDVqpg0KZ0r/cCwtKv+jwc3eiiZFvwxsOpXMxbRctp5VqmMUN8MQQoj72T0dLO/Zs4d+/frRq1cvAgICGDx4MF27dmX//v2Asqo8c+ZM3nrrLfr160f9+vVZvHgxly9fZvXq1Xd38kLcAcX5yhEeEYQ4KxUFyltZPnlNWbVt5NmIb7t+y7BawwD46thXjN06lqzCrMpd0GiExf0wfVafPYnbAWhdrShYTjoOe5XmJPT6GPp9Af4toSBDqQpRUS7ymbVKioOVo9Lm2ckfMi6UVLzQFxbVVEZ5oC+gTfnjFOctJ+6FnGsV30PGRTi8yHzOksgEjl5Ix97Kgp7h3mg1Ko5dSOfVn07Q/bOdZOTqwKc++DUHox4OL2ZvzHVAwxMBM/mp70/0Cq9BfR9bmnMCgJfjmpGnM9I21J13etfh4yENaFDdibRcHaMWHWTaxrMYjCYeCPMwV46oiFqt4oWOIYR52VOoN7LxhoYfxXZEXQWgXZjnTce6kUql4qWiQD0pIx8LtVIb+eMhDbDWarDQqOlcx4t5jzVl3+ud2PFK+9tuWlGcirHpVApLIhMxmpTOecXd+YQQ4n52T78/1qpVK+bMmcO5c+cICwvj2LFj7N69m08++QSAuLg4kpOT6dy5s/kcJycnmjdvzt69exk6dGi54xYUFFBQUPLQS2am8rasTqdDpyu7qiPE7So0FHI+/Ty1XGqhKadaws0YTUaOXVUqYYS7hkPRM1LR6dFl/p4ev6J0davtUhuMMKHRBGo51+L9/e+z/eJ2hq0bxscPfEyQ080fUlLFbMEibgfntVquOJuwVlkQ7lIPXUE+mrUvoDYZMNbuhyGwI5iAgQuwWNgNVVocxuWPYHj4R+XhvGImIxbbP0QFGJqOxmjriar7dCxWDMW07yv0tQegjtmC5upZTLbu6Du8CxX9G7TzxsK7Pqrk4+hPr8MU8UjZ+SfsRvPzk6hyr2GydSfJtwvT5iqr3uM7h/BoC3+uZxfw89HLzP8jgQupeSzbH8+o1gGoIh7F4kIkxpM/czBFKcvWJrA6dho79Ho9b4Rn4bgzjzSTPduzqxHkbsfMIeGYjAY0wJfDGjDom0iir2QTfUVZNX+hfWCl/0/pW9+HGZvP8/PhCwyI8DZvL9Ab2VuU3tEqyLlK/0e1CHCid7g3p5MymdyvDs0CXNHr9WWOc7FR/m7e7v9/HcLcmLIe9senciZJ+f/04abV5f/Te5h8b4SovHs6WH7ttdfIzMykVq1aaDQaDAYD77//Po88ovyQTE5WVmC8vErXP/Xy8jLvK8/UqVOZOHFime2bNm3C1lYe4hB3zoa8Dewp2EOoRShDbIdgq678368rhitkFmaiRUvMvhgMGFChIr0gnZXrVmKvVlbtTCYTRzKVFeiMqAw2xCgNOTRoGGU7iqU5S0nISuDh9Q8z2HYwdSzrVHjN5jGf4A3ssFdWQ5vmZJE6dyhZ1tWom3QUncaWLZpOFGzYYD7Hwftp2mZOQpu4h+tfdOZgwHMUapXKCb5p+2l69Qw6jS2bMoLRF53X2Lk51dMjyVs6AruCKwAc9hjMxW17b/o1CVOFUJvjXN21kP2Xb1ixNZkIuvobdS8tR4WRdBt/9geM5aulJ8guUFPD3oTL9ZNs2KCswFcDOnmqWJGt4dttUXiln8bKoKY7atRXz+Cmu4zJwpOYw7uIKy7IcflHAHYZw7G2UPFw9Qx2b9tcan6P1oDPTmnQGVXUdjZy+cQeLp+46S2Z2RUAWBAZl8qSVRtwKcpEOZehIrdQg4PWROzh3cRXsbBEF3voEgrXTu9jw+mqnVsVvrYaLudCep4OZ0sTBXGH2BD/911P/DW5ubl3ewpC3Dfu6WB55cqVLFmyhKVLl1K3bl2OHj3KuHHj8PX1ZeTIkbc97uuvv8748SUPEGVmZuLn50fXrl1xdCxbHkmI26Ez6Pho1UcAnNefZ7FxMTNazyjVCc5kMnHwykFiM2LpH9xfKRtWZFX0KtgPDTwb0KdzHwDmr53PxeyLBDYNpKmX0mXuUvYlctfmYqG24LFej5UaA2Bw/mBe3f0qB68cZGnuUvp598PDxsO8X6vW0j2gO34GIxZHlJXsfUFNIfUErfIK8M/cbT5W3WUinRo/XOZeVbFhmH4ciUf2abonfoh+8CLwDsdi7gfKea3G0PWBISUnZDfGNLsljvlK0wpjYHvqD5uMc1oee2NTGdTQFwtNOVliVwJg7s9455ymt+1RipfbVVfPoL6kBOLGeoOx6/kJptgcjh4+gkat4vMRLan9p+5r7Qv1bPhoJ9fy9TiENqVdmAdkLYP4XXRVHyQh9HF694owH6+Zr7yjlezRigW9mtPI37ns/IDa56+xaG8Cb/aoRZBH1VpK/5p6gP3xaeS41+aRtoEAnPztHBBP57q+9O4VfvMB7qLzVtF8sV3pGvh421D6tJdSa/ey4ndUhRC3dk8HyxMmTOC1114zp1OEh4eTkJDA1KlTGTlyJN7eyluVKSkp+Pj4mM9LSUkhIiKiwnGtrKywsir7AJFWq0Wr1ZZzhhBVtztpNxmFGbhau2JjYcPF7Is8vvlxJrWaRDOfZqyJXsNP538iIVOpHHAx5yKvNXvNfP6JVGVJsqFXQ/PfyxDnEC5mXyQhO4FW1ZXqEGczlNJyNV1qYm9dNkfUU+vJ3G5z+eTQJ3x3+jvWxK4pc8x3Z79jqn1d2mMiN6g9R9KVMdt0/RjWvwp5qVC9GZpmT6JRlxPE1uwCT22B5Y+gSo1Bu7gX1BsMV8+ClROaVs+jufHflkt16DJRKT9nYY26z6eotFrGrdzPiUsZZBUYebZ9cNnr+NYH1yBUqbFo/vik9D6VBrq9j7r5M+QVGpi07hAAT7YJpL5/2WoUTlotDzbxY97uOJbsv0jnur5Qu68SLGsOcipkQsn/BznXIEn5RWL046PBwaPMeMU61fGhUx2fCvffzMBG1dkfn8Yvx5MZ01H5pWpXtJKC0b6W1z39/1PP+tX4Ynsslho1D7cIuKfnKpDvjxBVcE8Hy7m5uaj/9INZo9FgNCpP9wcGBuLt7c2WLVvMwXFmZiaRkZE8++yz//R0xX3sh3M/EJsey0uNX8Lyxpzbv2Bd7DoA+gT14cnwJ3ll5yvsTdrLhJ0TsFBZoDcpuaM2Fjbk6fNYemYpvYN6m1s/F1fCiPCMMI8Z7BzM9ovbibl+FtY8D6FdOJmjlEq8WTtqC7UFrzR9hebezc1d+YqdvHaS49eO80JaJM86O1En7AF0UQupZl+NGnWHQPWWcPJHaPCw0sSjIp614amt8PNoOP8bHP1e2d7yObBxLnt8o8eUahquweAaxOGENE5cygBgzs4YHm1Zo2zZMZUKBn4LJ35Qzi2m1kDdAeDXDKPRxJT1Z7iUnkd1F5tSLYv/bHiLGszbHcf2c1dJvJ6LT2h3tL9OoInqHM4+N7RvjtkGmMCrHjh4VzjeX9Uj3Id31pzibHIWZ5IycbWz5GxyFioVtAlxv/UAd1EdX0c+GxqBi60lHg7lVDMRQoj71D0dLPfp04f3338ff39/6taty5EjR/jkk0944okngKJaouPGMWXKFEJDQwkMDOTtt9/G19eX/v37393Ji/tGvj6fqZFT0Rl1ZBZmMqX1lL/ccSyrMMtc57h3cG+crZ35uvPXzDoyi/kn56M36Ql3D2dQ6CB6BPZg0r5JrI9dz8S9E1nWaxlZhVnEZ8YD0MCjgXlcc/m4S/vg9D448QMnG3QAbh4sF2vn105pp3wDnVHHjA1PsfT6Ib52ccIhVsnNbe3bWvk6OPtBm5cqd+M2zjBsOez4EHZMA1t3aP5M+cfe2D0P+G5vvPnjtFwdi/bEM6ZDSNnzqjdWXuXIzNcxfsUxfj+jlC6b0r/eTZtqBLrb8UCYBzvPXeX7yAS61fVGawykvjqOkNRdEFSUShD9u/JnSDldAO8gJxstHWt5svFUMquPXDJXkwiv5oSb/b0fgPaLuL8bkAghRHnu6WD5888/5+233+a5557jypUr+Pr68vTTT/POO++Yj3nllVfIyclh9OjRpKen06ZNGzZu3Ii1deUK6gtx+vppdEblyfC1MWsJcgpiVPiovzTm7wm/U2gsJMTak5pzu4O+AA3wEtBFa4FlQFvCui0EC2UVe0KTCey+tJuzqWf5/vT31HBU6tMGOwXjZOVkHtdcPi43CRNg1OdzOlV5aque262D5fJoVRa8fimOOlnXmeTpSZZOqeRgLhlXVWo1dHgD6g0CS7vyV5X/5Fp2ARuKSqY90TqQ+X/EMXdXLCNbBVS6qUX0lSxGf3eI2Ks5WFqoeb9/PdrXvHWptREtarDz3FVWHryAtYUag6Ep9dVxqKPWQ5ORSjm9mK3KwcF/b7AM0L9hNTaeSmbN0cvmsnPtwipO+xBCCPH3uqfrLDs4ODBz5kwSEhLIy8sjJiaGKVOmYGlZ8ja5SqVi0qRJJCcnk5+fz++//05YWMVNAIT4s+JaxsUtkj87/BlbEst2pcsqzKp0veLiFIxeBUZUeWmgyzW/6uVmEnZ6vdKYo6gBiJuNGy83VjrjfXXsKzbEKQ+r3ZiCARDgFKBUxFCZuK7REmttSx4mbNWWBDoFVv3mAS4dgqSj9MvTs7jjV/ja+eJp40kLnxbmQ1Iy80t1zqwUj5rgVL1Sh644cIFCg5EIP2fe7FWbIHc70otWl/9MbzASVZSmUPxafeQS/b74g9irOfg4WfPjMy0Z0sSvUtfuUMuT6i42pOfqmLMrlk1GpVMisduhIEvpPJhzBbR24N/ipmPdCR1qeeBobUFyZj6/nkwC4AEJloUQ4q65p1eWhfgnHL16FIAn6j3BhawLrIhaweu7Xmdxj8XUdKnJgeQD/HjuR35P/B1brS2Luy8myLniJ/2Tc5I5kHwAgJ5XlS5uDF0GXnWVjy8dhB9HKY0zPGopOb1A/5D+rIlZw6GUQ2yMV9pB/zlYtrGwobqFHRf02cQENOOya3W4tpc6+Xlo8jOUttBVVdTimXoDqVu9FesGrsNoMmKlsSK3UM9rP51g7bHLvN6jFk+3K+ehu79IbzCyZJ/ykOOIljXQqFW82CmUcSuOMmdnLCNa1sDBWnkYKe5aDk9/d5BzKdnljtU80JUvH2mEexVSFjRqFcNb1ODDX8+SrzNynmoUOgVimRGnpF+kxikHBrYtv7PgHWZloaFXfV+W7VeaezhYWRDh5/y3X1cIIUT57umVZSH+biaTiWNXlCoHEZ4RvNbsNVr6tCRPn8eY38fQZ3UfRm0axa/xv6Iz6sgoyOD5rc+Tnp9e4Zgb4jZgwkRjt3B8s66ASg1B7cGlhvKqNwi6TlEO3vQmnNsEKO+SvNPyHbTqkqfUIzwi/jxhgvOUQDHaty4nnH0BCM/Ngc1vlz42Lx3O/HLzVtQZl+DUz8rHRS2ftWotVhorEq/nMvCrPaw9dhmA7yMTqr66XAm/n7nC5Yx8XO0s6RmuVJHo08CXYA87MvJKVpe3nEmh7xe7OZeSjY1Wg4eDlfnl7WjN0+2C+P7J5lUKlIs92MTP3OLZ1c4KbR2lVB9n15ekYIR0ruDsO29Aw5Lc39Yh7mjLK6MnhBDiHyEry+I/LSEzgbSCNCzVltRxrYOF2oIZ7WfwyPpHlAfs8sBOa0evwF50DejKu3ve5ULWBV7a/hJzusxBqylbfqk4BaO3U01gPbjXBMs/NSNpOQauRcHhxfDjE0obaK86BDkF8WT4k3x97GtcrV3NuctmFw8QkpPBdmcnYqxtOFmcr1xQAEe+hwbDlA56hxbCyZ9Bn6ec598SGj8GdfqBhTXE71aOObMWDIXg27DUQ3M7zl3lxWVHyMjT4W5vSW6hgQupeRxOTKNxjdtYvb6J7/bFA/BQUz+stUonueLV5bHLjzJ3Vxx5OgNfblPafDep4cJXwxvh6XDnnktwtbOkT31ffjp8keaBrqhq94a9s+Dcb0r6DEBwxzt2vVtpUsOFas42XErPkxQMIYS4yyRYFv9pxSkY9dzrmQNfR0tHZneZzdzjc6nvUZ/uAd2x1SrB7hcdv2D4r8M5mHKQKZFTeK/le6UqZ0SlRnE+7TxatZYuuqLVQJ/6ZS+sUkHPj+F6LCTshiWDIfABAJ40GdFb+hEe0LVsVY7jKwguVB5GPJN2nvNpRWXjwvrB0eWwuD8Yb2hj6+yvrB4n7lVev74Ctm6QGltyjE8D6PuF+dNFe+KZ+MspjCZo4OfM7OGN+Oi3KH4+fInVRy7fdrAclZzFyoMX6FDTk1bBbqjVKqKvZPNH9HXUKnikuX+p43vX9+XzrdFEX8k2B8ojWtbgrV51zKvAd9JrPWphZ6VhZKsAcLMFO08lVxnAJRDc7nwKSkXUahUfP9iArWevMKixVJgQQoi7SYJl8Z9WXi1jgGr21Xiv1Xtljg9xCeGjBz7i+a3P8/P5nwlyCmJk3ZJukutj1wPQrno7nFKUxh74NCgzDqBUwnjoO5jbEdLi4NgyACyBFwHO7QP3+iUrmvpCOPkzIQYlGD55XWnd7Grtik/XDyF6G2SngIUN1BuorCRXbwpZSXBkibKKnZEI+RlgaQ/hQ6DxSGVVuUh2gZ4p609jNMHQpn5M7FcXKwsNAxpW4+fDl1h3/DJv9656sJpXaOCpxQdJTM1l3u44/F1teaipH7FXcwDoVNuL6i6lV981ahVjO4XywrIjWFqomdK/Hg9W8qG92+HhYMWkfjdUFKnVU1l9h7+9ZFx5WgS50SLI7R+/rhBCiNIkWBb/aeZg+c+5wTfRtnpb/tfkf0w/MJ2PD37MD+d+MO9LzlHKn/UO6g1HxykbvctZWS5m6wqjNsPJn8BQULI9YS+c+xVWPgZP/g4eYRCzBfJSCbT3Qq1SYyxqylHPvR4qWxd4bD1cOgxh3UqXa3P0hXYToO14iNsBeWkQ2g2synb7Oxifis5gorqLDVMHhptXtlsFu+PhYMXVrAJ2nrtK5zpelf56Aczaep7E1FycbbUYDCYSU3P56LeSXOoRLWuUe16fBr7YWWnwd7Uz1xz+x9Tqc0Ow/M/lKwshhLi3SLAs/rMyCjKIyVDe3m/gWcHqbwWG1x5OQmYCK6JWmNtVF/O09aSta11lFRfAO/zmg9l7QIs/Ne5o/gws6gsX9sHSB5XOeMdXAGBVbzB+ecfN1zU3I3EPVV4VUWtumXe7LzYVgJZBbqVSQDRqFX0b+DJvdxyrjl6qUrB8NjmTuTuVtI/pg+rTJtSd9ceTWLY/kcOJ6dSr5kjr4Iq703WsVbXA/I4JbKuksegLIaDt3ZmDEEKIu06CZfGfdeyqUgUjwLEGrj8+Bdej4emdYO10izOVyhVvtXiLB2s+SHZh6TJmgU6BWF5WxsYloFJNOcqwsIKHvi9J0Vj+CFw+rOyr/yBBZ7PNwXK4+y2C8SrYF3sdoNy3/wc0rMa83XH8fjqFzHwdjtZlH278M6PRxBs/n0BvNNGtrhdd6yqtooc08WNIEz8upefhZKNFrf5rHRP/FhZWyt8Hk6ncVXghhBD/DVKPSPxnFadgNLD3V+rppsWXtDWupDCXMBp5NSr1crF2gaTjygEV5SvfwtnkTLZcMLKvxZfoLewgcQ/o85XKGj4NzJ38AOq61b2ta/xZdoGeE5cyAGgRXDZYruvrSIinPQV6IxtPJldqzCVFq8f2Vha817fsPKs521S6Q99dYeNye7WrhRBC/GtIsCz+s4orYTTMzizZGL31zgyeVLSyfLN85QpEX8mi52e7GLXoIEPXZPFU7nMYTcrKa0L13qBSmYPlavbVlOD8DjgQn4rBaMLf1ZZqzjZl9qtUKnP939VHLt1yvJTMfKb/qjzk+L+uYfg4lR1TCCGEuNdJsCz+9Y5fPc6q86vMD8QB6Iw6Tlw9AUBE/IGSg2O2mFtQlxG7Hc5uqNxFk29/ZXl71FWMJqX2bwM/Z1KrdWCSxQtsNDRlqUF50Kyjf0f6Bvflf03+V+XxK1KSglHxSmrfBkoTlL2x10nKyLvpeBN/OUVWgZ4G1Z14tGXAHZunEEII8U+6h9//FOKvu5J7hac3P022LpvErETGNhoLwLnUc+Qb8nG0sCUwI1F5u12Xr5RZu3K6pDV1seyr8P1gpYbxyF/MNZHLVZAN15T6x7cTLO+NUYLWZ9oFMfoBpbbvb6eCefq7FgTEFvA6YG1hzftt3q/y2Dezr+i6LctJwSjm52pLswBX9sensvbo5QrbX285k8KGE8lo1Co+GBiO5l7MSRZCCCEqQVaWxb/ah/s/JFunPID37Ylv+SXmF6AkBaMBVso/groDIaCNclL0lrIDnfq5pNnHL+OUwLoiKacAE9h7g71nleZrMJrYH1dckaKkQkSrYDcs1Crir+eScD2nSmNWRla+zpyv3Dzw5rV9+xelYvx8+BJGY9lV+JwCPe+sOQXAqDaB1PW99QOTQgghxL1KgmXxr7Xjwg42J2xGo9LQI7AHAO/ueZejV45y5MoRABpeL8q9rf9QSeOJ8h7yO76y5OPUGNj1ccUXLs5Xvo1V5dOXM8kq0ONgZUEdX0fzdgdrLY1qKLnJO89drfK4t3IwPg2jCWq42eJbTr7yjXqF+2CtVROVksWMTVFl9n+6+RyX0vOo5mzDuM43KWUnhBBC3AckWBb/Srm6XN6PVNIURtQZwYdtP6SjX0d0Rh1jt43lQLKSpxyRmwXONcCvWUnjicS9UHjD6u31GLh0EFQa6DlD2bb7U7hytvyLJxcHy1V/uG9v7DUAmgW6lkldaBfmAcCOc9cqNdaGE0l0+ng73+1LuOWxe4vylVtWomOck62WqQOVcnVfbY/h58MXzftOXspg/h9xAEwZUA9bS8n0EkIIcX+TYFncV67nXafgxk53Ffjy6Jck5SRRzb4az9QbhTo/k6ltp1LLtRap+amk5qeiAeoVFEL9B0GlArcQcPIHQyHE7y4ZrHhVObgDNH0SwrorKRnrxoHRWPbif2FlubgpSHl1jouD5b0x1yjUl3PdIgajiWkbz/LcksPEXM1h+q9nySnQ3+K6FddXLs+AhtUZ00HJV37tpxMcSkhFbzDy+s8nMJqgd30fOtSsWgqKEEIIcS+SYFncN45eOUrXH7syetNoDEZDhcedvn6a7898D8CbTV/D9vtB8FEwtocW83mHWbjbKLnAtQoKsTGZIPxB5USV6oZUjKK8ZZPJ3DmP+g8px/ScAVo7ZQX6yOLSF9cXlqw4V7FsnN5g5EBxvnI5D9nV8XHE3d6SnEIDhxLSyh0jPbeQxxce4OvtSmdCW0sNWQV6Vh+tuNRbZr6Ok8X1lSsZLAO83KUm3et6U2gwMnrxIab/FsWJSxk4WFvwTp86lR5HCCGEuJdJsCzuCzqDjvf2vEehsZDDVw6zPGp5ucfpjXom7p2I0WSke0B32iYehYsHwKiHja/ivek9vnhgBvWsvRiRkQm+DcEjrGSA4mA5pihYvnhQ6aCntYNavZRtzn7Q4Q1lXhvfJvtaSRoCV88oq87Wzkqr5Co4nVSUr2xtQW0fxzL71WoVbUOV1eWd58vmLZ9NzqTPF7vZee4q1lo1s4Y1ZHwX5d6+25uAqYKSeAfiUjGaINDdDm8n60rPV61W8clDDajr68j1nELmFLW0fq1HLTwdKj+OEEIIcS+TYFncFxacWkBMRgwWKiUHdtbhWSTnlO0iN+vILE5fP42DpQOvhj0M2z9UdtTuq+QcH19O3TUvsex6Lj1zcpXV4hsFPgBqC6X1dVp8yapy7d5gaVdyXPNnuGAVhlaXSdqXXUg4c0jZnnRDvrKqauXSikvGNS8nX7mYOW85qnSwnF2g5/EFB7iQmoefqw0/P9uavg18GdLYD2utmrPJWRyIL381ujL1lStia2nBtyOb4OlgBUCTGi4Ma1q1XxKEEEKIe5kEy+Kel5CZwDfHvgFgkmtT6lt7kavP5cP9H5Y6bnX0ahacXADA283fwn3LFNDnQUBbeHAxjFgNtm5KQHvllBI81xtU+mLWTlC9mfLxud/g5E/Kx8WpGkW2R6fyWOZTXDK54We6jPvynhz5bdFfanNdmbzhNqFKCsnppEyuZJWUr5vxWxRJGfn4u9ryy/NtzJU0nGy19I9QSr0t3htfwXUrzpOuDB8nGxaPasawZv58+lAEaqmpLIQQ4l9EgmVxTzOZTEzeO5lCYyEtrb3pfXAF70YfxcJkYkviFrYe+gpMJg6lHGLi3okAPF3/aXrk5Col4DSW0Humssob+ACM3gE+EcrgIZ3Kr4NcnIqxYzrkpYKdBwS1N+/OKzTw1uqTxJiqMa/2Ak5Z1sdOlU/DvS9ScHipcpB31YJlvcFoXvm9WdDqbm9FeDWlbvGuoqoYxy6ks6goEH5/QD2cbS1LnfNoyxoAbDyZzJXM0vWhM/J0nLpc9XzlP6vl7cjUgeH4udre9hhCCCHEvUjqOom7YmviVj47/Bm64kYfRcJcwhgYOpDWvq3RqDWsi11HZHIkVmpL3o45jgoIc/BjZMZ15jk78cHRz6kWOY9xDmr0Rj0P+Hbm6P5Qsq48jQNgavsyKveQkgs4+8ETG+H0WghqV/7kQjrB1smQW1Sird5g0JT8U5m55RwX0/LwdbLm5QGtsVJvYd+3L9AiZTlWBqXknMmnPlVZXz15OZPsAj2OFeQr3+iBMHdOXMpg5/mr9Ivw5fWfT2AyQf8IX3NO843q+jrRuIYLhxLSWLo/kXGdS3K0N55MwmiCIHc7vBwlz1gIIYT4MwmWxT/OZDLx+ZHPic2ILbPvQtYFtiRuwcfOh/4h/Vl+VnmQ7xm9NX6F+RDaFR5eyTMX9vHbjhe5aJHPME0BOqOKOgU6Hj0ZT3rqRzho0og2+vLioSYMtohjYKNqJSuuWhto8FCZa5t5NwBb95JguX5JCsbpy5l8u0upIzypXz3srJR/Qi2e/YaDvzSi7sG3SMWBjAJPqlIPojgFo3mQ2y1bQ7cL8+TLbTHsOn+Nb3fHcTopEycbLW/1rviKI1rWUILlyETGdAjBQq3iq+0x5qYiXep4VWG2QgghxH+HBMviH3cu7RzR6dFYGk287TmYgKb9AdAZdWxN3MramLUk5STx9bGvAQixcmfk2cOgtVXKtqlUWPu35O1On/H05qfRqVR4mlTMSrmClyEJNMp13jON5vTVAiatO83Hm6L4aEgDeob73HqCajUEd4QTK8EtVKmYgVK/+I1VJzAYTfSo503nPwWYTfo8zSupIWw8c42hx5OpU92l0l+TqtQ5bujvjL2VBak5hXz0mxLsvtGzFu72VhWe06OeD5Ptz3Alq4BVRy6x9cwVNp5SHpAc1syf8V3DKjxXCCGE+C+TnGXxj1t3fD4A7fLy6H3gcyLy8ojwjKCpd1NebfYqWx/cytS2U2ns1RgPazcmX4pHC0q5Npca5nFa+bbikdqP4GXrxaw+y1hWYzZL9R3JVDtD63F89ebzTO5fj1reDuQUGnhuyWGmbTyLwVh+CbVSmj4J9l7Q7lVzVYslkQkcvZCOg5UF7/WtW+5pnZrUJRN71hy9VOF14q7lkHg91/z5jfWVK1ORQqtR0zpECaoNRhPNAl15sInfTc+xtFAzrJlyzCs/HmfjqWQsNWqmDgxn6sBwrCw0t7yuEEII8V+kMlVUfPU/JDMzEycnJzIyMnB0vHm+qPhrDPpCun7fhCsqEx8mp9MrLxOTrRuqp7aCS0DZE356Ek78oDT4eGpbqdzhG8Vfy6HTJzswGE2seq4VDf1LVnX1BiPTNp5lblH6xANhHswaGlHmQbib+eHgBd5cfZJCvZHJ/eryaMty5goU6A00e38LGXk6ljzZnNYh7qX2n0/Jotes3RQajLQIcmVYM3+8Ha15aM4+nGy0HHm7S6WqSSyJTODNVSfRalT8OvYBQjztb3lOUkYebaZtw2A04eVoxdfDG9PIv/Kr30KIfw/5uSdE5d3zK8sBAQGoVKoyrzFjxgCQn5/PmDFjcHNzw97enkGDBpGSknKXZ/3fsuviLnZd3FWpYw/smMgVlQkHg5GpGW9zwhiAKvc6LB0K+ZklBxoNcHSZEiir1NDnswoDZYAvtkVjMJpoX9OjVKAMYKFR82avOnw2NAJrrZqd567S54vdnE/JuuV8C/VG3llzkgk/HqdQb6RbXS8ebl6jwuOtLDT0qq+keqw6UrZr3swt5yk0KK2q98WmMnb5UYbN3Qco9ZUrW3atf0Q1+jTwZfrg+pUKlEEp8TapX12GNfPjlxfaSKAshBBCVMI9HywfOHCApKQk82vz5s0ADBkyBICXXnqJX375hR9++IEdO3Zw+fJlBg4ceDen/J8SnRbNmC1jeH7r81zIvHDzgzMusi7qBwBcs/y4aKzGk4X/I9PCTel899MoyLgIOz6CzyJg9TPKec1GQ7VGFQ4bfy3HHJjeWOnhz/pFVOPnZ1vj52rDhdQ8nv7+EAX6ittmX8nK55Fv97F4bwIAL3UO4+tHGt/yAbwBDZW6xhtPJpOvKxk/KjmLDSeSAFjweFPGdQ7Fx8ma4myNP69C34ydlQWfD2vIgIbVK30OwCPNazB1YH3psCeEEEJU0j0fLHt4eODt7W1+rVu3juDgYNq1a0dGRgbz5s3jk08+oWPHjjRu3JgFCxawZ88e9u3bd7en/p8w+/hsTJgwmoysiFpx02Pz1v+P3220AJxJ70Wwhx0puDLG9AomC2s4vwk+rQvbpkBGotIyuuXz0Pm9m477+VZlVblDTQ8i/JxvemwdX0dWP9cad3srYq/mMHt72YocoAS2fT7fzYH4NBysLJg3sgljO4dWauW3sb8L1V1syC7Q8/uZknc5Zm05j8kEPcO96VDTk3Gdw9j9akfmP9aEt3rVZlgz6XwnhBBC3Gvuq2oYhYWFfP/994wfPx6VSsWhQ4fQ6XR07tzZfEytWrXw9/dn7969tGjRotxxCgoKKCgoMH+emam8/a/T6dDpdOWeI8qKTo9mU/wm8+erolcxut5obCxsyhyrOrueHZd2kOPpjo3Riay8AEZ09mfG5vPsyvEjuut0Qne+CIDRvyXGiEcx1eqjlHkDqOD7En89h1VHLgIwpn1Qpb5/jlZq3uwRxks/nOCLbefpXseDII+SVtbXsgt4fMF+UjILCPaw4+uHIwh0t6vS340+9b35ekccPx+6SLfaHpxLyWJ90arycw8ElhqrbbArbYNdwWRAp6t4pVsIIe4U+VknROXdV8Hy6tWrSU9P57HHHgMgOTkZS0tLnJ2dSx3n5eVFcnJyheNMnTqViRMnltm+adMmbG2lA1llLc9ZjgkTdbR1uGy4THphOjN+mUFjq8aljrPUZdI+6m3Wuyhf25y0RoCa7IQThNmrOZSvZuoJVx6tORmj2pJsax+4AFzYVmqc5FxYcE7D1Rua0JlMYERFHWcjl47/waXjlZu7ygS1ndWcSVczZuEunq9jRKUCnRG+OKXhcrYKD2sTo2pkcGb/Ds5U8WvjnAtgwfZzV1i5ZgM/xKoBNRFuRmIO7yKmiuMJIcSdlJube+uDhBDAfRYsz5s3jx49euDr6/uXxnn99dcZP368+fPMzEz8/Pzo2rWrPBVcSdHp0ZzccBKAd7u8yx+X/2DW0VmcsT7DW93fQlVUbg19AZolA8kwZLDbVsmvzUtviFaj4rEB3fE7e4VDK44TXWBH60HPlJz3J6k5hQz+JpLkvLwy+ywt1HwwrAV1fav2vavfKpeen+8hOhPyfMIZ1NCXCT+dJD47CUdrC5Y83ZxAd7tbD1SBtVf3cupyFidVARxNvYhKBe8Pa02Yl8NtjymEEHdC8TuqQohbu2+C5YSEBH7//Xd+/vln8zZvb28KCwtJT08vtbqckpKCt7d3hWNZWVlhZVW2gYNWq0Wr1d7Ref9bzT01F4CuNbpSx6MOPg4+zD4+m7NpZzmdfpoIzwhl2XfdC3Axkt9cPNCroJptKGcLPant64CdjRUd6/hgaXGSxNQ84tMKyg0kC/VGXlhxnAtpefi52rDgsWbYWZXUBXaw1mJvVfW/ykGeTrzUOYypv55l2m/niLuex5pjSWjUKr4e3pgwH+fb/fIAMKBhdU5dPsOyA0qaSM9wH+pWv3UdZSGE+LvJzzohKu+ef8Cv2IIFC/D09KRXr17mbY0bN0ar1bJlyxbztqioKBITE2nZsuXdmOZ/QlRqFJsTNqNCxTMNlIoVLtYu9AjsAcDyKKVFNX/MhGPLQKVhnV9tAKppWgGYV4HtrSxoU1QF4reTZVNnTCYTb60+wf641KIH7ZoS4mmPj5ON+XU7gXKxJ9oEUtvHkfRcHXN2Kg/7TepXt0qVKSrSt4Evxc8DqlQwtlPoXx5TCCGEEP+s+yJYNhqNLFiwgJEjR2JhURIYOTk5MWrUKMaPH8+2bds4dOgQjz/+OC1btqzw4T7x131z/BsAugZ0JdSlJAAcVmsYAJviN3H9+HL4fSI64IOGPTiWFY9apaYgswEA9ao5mc/rVldpG73pdNn62N/uimPlwYuoVTDr4YZ3PIVBW9TFrjj747FWATxykzrKVeHpaG0OunuF+0j6hRBCCHEfui/SMH7//XcSExN54oknyuz79NNPUavVDBo0iIKCArp168ZXX311F2b576c36tmcsLlkVbn+M6X213WvS7hbXU5cP8XP299kgFrFy8H1OZymPHU3ttFYvlmrRKU35hd3qu2FSnWCE5cyuJSeRzVnpezaigMX+OBX5dG6t3rVoUNNz7/lviL8nJkxuAGJqbm80DHkjo79bp+6fL8vgec6BN/RcYUQQgjxz5B210jbz1u5lH2Jn879xOro1VzNuwpAj4AeTG83veSga9FweCFrz67kTScr3PUG1BaWXMGAndaOqW2mUs+lFU3f/x2VCk6+1w27G9Inhszew4H4NB5rFUCB3sCao5fJLVTKqA1r5s8HA+pV+PCfEEKIqpGfe0JU3n2xsizuDpPJxMCVLxOd/zug/E7lau1Kv5B+PF3/aeWgnGuwZgyc2whANxXMcPDjmoUGMBDoFMhnHT4j0CmQ7VFXAAhytysVKAN0q+vNgfg0Fu6JN28LcrfjkRY1GNGyhgTKQgghhLgrJFgWFfotdjfR+Up7cUdTXd5u9zid/Dui1RQ9RX35CCwfDpkXQaWG0K5YNX6Mxwou8emRz+jo15H327yPvaU9AKcuK6WK6vo6lblWz3AfZv5+nkKDkR71vBnWzJ/mga4SJAshhBDirpJgWZTLZDLx+eEvAShMbcmllH4c9vSne2BRoHxkCax7CQwF4BoMQ5eAp1Lx4gmgd0hfPGw8SgW7py5nAFCvWtm3/HydbdgxoT0WGjVONlLSSAghhBD3BgmWRbkikyNJzD2FyWhBmHVfTgBzdsYS6mbFkOtfw/45yoFh3Ylu8wlWFi743XC+p23Zh/FOXqp4ZRnAzb5s7WshhBBCiLvpvigdJ/5ZJpOJr44qFUV06c14t0crXuwUCpjQrH/RHCifCHmWvtfG0Pmro3SbuZPE6xW3T83I05GYquyvaqc9IYQQQoi7RYJlUca+pH0cuXIEk9ECy+xORPg5M65TKDOr72Cgehd61Iw1jqfPybYcv5wFQG6hgbfWnKSi4iqni/KVqznb4Gxr+Y/dixBCCCHEX1GpNIy1a9dWeeAuXbpgY2NT5fPE3fXnVeWOgcFYaNRwZh39riktrifqRrDG0IQgdzuGNfOnUQ1nhs2NZOe5q6w9dpl+EdXKjFucryyrykIIIYS4n1QqWO7fv3+VBlWpVJw/f56goKDbmZO4i/Ym7eXo1aOoTFoKr7enXWsPSDoGPz+FChP5EY/j7fQ8y2u4lKpW8XyHED7ZfI7J607TPswTJ9vSD+kVV8K4sXOfEEIIIcS9rtJpGMnJyRiNxkq9bG1t/845i7/JjavKhWnNMOkdaV/NCMuGgS4Xgtpj3WcGYzqE0CLIrVSli2faBRPiac+17EI+3HimzNiysiyEEEKI+1GlguWRI0dWKaVi+PDh0hHoPrQ8ajnHrh7DQmVJwfV2hHnY4P3rU5B5CdxCYcgi0JT/ZoSlhZoPBoQDsGz/BfbHpQKQlJHHZ7+fJ/pKNiAry0IIIYS4v1QqDWPBggVVGvTrr7++rcmIO0dnMJKVr8fVruKH6Qr0BrLz9TjYqPhw/4f8cO4HAGpY9CBN78jLzjvgwn6wcoKHV4CN802v2SzQlaFN/Vh+4AKv/XScIA87tp69grHomb961RzxdJDycEIIIYS4f/ylOss6nY5z585hMBioWbMmVlYSCN0LDEYTj3wbyYH4VP7XtSbPtQ8u0wnvSGIaz35/mLSCq9RqsJrYrFOoUPF8w+eZty4QH5LolKQ80EeX98AtuFLXfr1HbX4/k0LstRxir+UA0DzQlYeb+9Otrrd05BNCCCHEfeW2g+Vdu3YxdOhQdDoder0eCwsLFi9eTPfu3e/k/MRt+G5vvDkN4qPfojhxMYMZDzbA3kr5di/fn8g7a06h18Zj4/8dsVlZ2FnYM73dNLy1DfkgcwfzLBdioc8Bv+bQ6LFKX9vJVsvHD0YwfeNZWgW7MbSZP8Ee9n/HbQohhBBC/O0qHSwbjUbU6pIU53HjxrFkyRLat28PwJw5c3j22WeJi4u745MUlZeUkcdHv0UB0DPcm99PX2HjqWSiv8zmi4cbsmhPAsv2J6LSZOMUuAgDORjyvbDKfZoGbi1ZeeAC3dQH6aQ+BGoL6D0T1FUrx90uzIN2YR5/w90JIYQQQvyzKh0FNW/enMOHD5s/LywsxN/f3/y5v78/+fn5d3Z2olL0Rj1fHf2Kg8kHeXfNKXIKDTTyd+aLYY1Y/nQLvBytiL6STfeZu5RAWQURETswkEOwUyiOaeNJSLHl+aWH2Xs6jve0i5SBW48Frzp39+aEEEIIIe6iSgfLX3zxBU8++SQvvfQSOTk5vPvuuzRu3JgWLVrQuHFjBg0axPvvv/93zlVUYEviFr4+9jVjt/6PTaeTsFCrmDqwPmq1ikb+LvzyQhuaBrgA4GhtwYT+KqJzd6FWqZnSZhLfPtoaG62GXeev8cDF2fioUil0CoAHJtzdGxNCCCGEuMsqnYbRvHlzDhw4wPTp02ncuDHTp08nKiqKyMhIDAYDTZs2pVq1sp3bxN/vyMXdAGTqrqOxjWV0s27U9HYw7/d0sGbJky3YdDqZ2r42PL/jYQCG2QVT77thABx2MHA9pwBfrgOg7TsTtNKBUQghhBD/bSqTyWSq6kkxMTE888wzODo68vnnn+Pr6/t3zO0fk5mZiZOTExkZGfdNfehr2QUYi2qyPb+qLWdQUmCscpuy+8m5WGs15Z4389BM5p2ch6eNB2vPHsPOqC9zzHH3XtR/funfN3khhBB31f34c0+Iu6VK1TBOnTrF2bNnCQ8PZ/PmzSxatIi2bdvy8ssv89xzz/1dcxQ3uJZdwLjlR9kdfQ2ATpq9nAvNg6KSbGq746DSAWWD5XNp51h0SslHfsOrHXanD4F7TehfUhf7Sq6BukEN//4bEUIIIYS4D1Q6Z/mTTz6hadOmfPTRR7Rs2ZK5c+cycuRIIiMj2bdvHy1btuTEiRN/51z/845fTKfv57vNgbKjKpfB9ssxqFR46A1U0+nJMxWw/eL2MucaTUYm7p2I3qSno19HOiWfV3bU6QfVG5tfnmHN0Fho/8G7EkIIIYS4d1U6WJ4+fTrr169n3759HD58mE8++QQAd3d3Fi9ezKRJk3jwwQf/ton+1/1w8AKDZ+/lckY+ge52bH7pAY63PcAF6wIAGnk0oGeO0gRkfcy6sudH/cDxq8ex09rxeuOXIHqLsqNWr3/sHoQQQggh7jeVDpZNJpO5zrJGo+HPqc5dunThyJEjd3Z2ApPJxHtrTzHhx+MU6o10ru3JmudbE6o7B/vncNRa6ZrYMKgbvQuU78nuS7tJy08zj3El9wozD88E4IWGL+CddBp0ueDkBz4N/vF7EkIIIYS4X1Q6WJ4wYQI9e/akVatWREREMH78+DLHWFtb39HJCVh3PImFe+IBGNc5lDmPNsFRq4JfxmLExFFbpTtehHdTggI6UrugEL3JwG/xv5nH+HD/h2Trsgl3D2dozaFwdr2yo1Yvc66zEEIIIYQoq9LB8v/+9z/27dvHSy+9xO7duxk9evTfOS8BGIwmZm45i5X3arq3TGRc5zDUahVEfg0pJ4izdyMLAzYWNoS5hkGt3vTOVlIx1sUqqRg7Luxgc8JmNCoN77R8B43JBFEblAtICoYQQgghxE1VqY9xeHg4Q4YMoVatWn/XfMQN1p9IIj7nKJYu+9ifOZfU/FTIz4CdHwFwJGIAAPXc66FVayGsKz3yClGbTBy7eoyo1Cjej1QaxTxa51FqudaCC/sgLxVsXMC/1V27NyGEEEKI+0GlguXx48eTU/TwWGW8/vrrpKam3vakhLKq/Nnv59DYXABAZ9Tx8/mfYd9sJWB2r8nRorSXCI8I5SRrJzz8W9MiT6m5/Ozvz5KUk4SvnS/PNnhWOaY4BSOsB2iqVDlQCCGEEOI/p1LB8meffUZubm6lB/3yyy9JT0+/3TmVcunSJYYPH46bmxs2NjaEh4dz8OBB836TycQ777yDj48PNjY2dO7cmfPnz9+Ra99N645fJuZqDlZ2l8zbfji7AsO+L5VP2r3C0avHAIjwjCg5sVZvehf9YnM17yoAb7V4C1utLZhMcKaoUkbt3n/7PQghhBBC3O8qFSybTCbCwsJwdXWt1Ksqq9A3k5aWRuvWrdFqtfz666+cPn2ajz/+GBcXF/Mx06dPZ9asWcyePZvIyEjs7Ozo1q0b+fn5d2QOd4PBaOKzLecBE9b2SrCsUWm4nJvMTnUheNQiNbgdCZkJADTwuKGiRc2edMrJw8ZoBKB7QHfaVm+r7Es+ARmJYGEDQR3+yVsSQgghhLgvVep9+AULFlR5YC8vryqf82fTpk3Dz8+v1PUDAwPNH5tMJmbOnMlbb71Fv379AFi8eDFeXl6sXr2aoUOH/uU5/OPyMzi4ZRWXrjrh5FBAvjETC5UFD4X0Z8n5H1nmaE+HNq9y9JrSACbYKRgnK6eS8x19sPVtzPNpUez2b8CrzV4t2Xe2aFU5pBNY2v6DNyWEEEIIcX+qVLA8cuTIv3se5Vq7di3dunVjyJAh7Nixg2rVqvHcc8/x1FNPARAXF0dycjKdO3c2n+Pk5ETz5s3Zu3dvhcFyQUEBBQUF5s8zMzMB0Ol06HS6v/GObk297GGaJ+xmhWUQy2uPZH0WhLqE8nBWHktNJvba2BDtXY/DsWsBqO9ev8yc1WE9GbHtIMNzLTFYOJn3W5xZhwrQh/bAdJfvUwghxN1zt3/WCXE/uaef8IqNjeXrr79m/PjxvPHGGxw4cIAXX3wRS0tLRo4cSXJyMlB2FdvLy8u8rzxTp05l4sSJZbZv2rQJW9u7t+LqnnWa1gm7AYhQx/LbpS/B0RqnTGt8YhbygIc9O2xtmL55BpcMSnqG6rKKDRs2lBrHPt+OTgBxO0j5UikPp8JItfRTGFGzKV6F7mLpc4QQQvx3VOU5JCH+6+7pYNloNNKkSRM++OADABo2bMjJkyeZPXv2X1rtfv3110s1VcnMzMTPz4+uXbvi6Oj4l+d9W0wmNN99DcA6QwuaO6ZyVpsOQI+COLTGPB5S+7GDfE6YTlBoKgRgRKcR+Dv6lx1u7mLUV05RPT2y9I6gdnTpK23JhRDiv6z4HVUhxK3d08Gyj48PderUKbWtdu3a/PTTTwB4e3sDkJKSgo+Pj/mYlJQUIiIiKhzXysoKKyurMtu1Wi1arfYOzPw2xO6AC3spMFkwnRGsfqoHp9Z0AIzUTzoDQOs2b+Af9S2JWYkAuFq7EuQahKq8LnxDl0D072AylmxTa1DX7IX6bt2jEEKIe8Jd+1knxH3ong6WW7duTVRUVKlt586do0aNGoDysJ+3tzdbtmwxB8eZmZlERkby7LPP/tPTvX0mE2yfCsAyQ0daRoSTakolDyM2KgsC9Ubwa4G6dj8eIouPDipNSRp4NCg/UAZwDYRmT/1TdyCEEEII8a9UpQ5+ABkZGeU2HElNTb3jb+u89NJL7Nu3jw8++IDo6GiWLl3KnDlzGDNmDAAqlYpx48YxZcoU1q5dy4kTJxgxYgS+vr7079//js7lbxW3AxL3UmDS8rW+L4+2rMGJomoXdT0j0LwSCyPWgFpNv5B+WGuKmpHcWF9ZCCGEEELccVUOlocOHcry5cvLbF+5cuUdL9XWtGlTVq1axbJly6hXrx6TJ09m5syZPPLII+ZjXnnlFV544QVGjx5N06ZNyc7OZuPGjVgXdbe755lMsE1ZVV5q6Ej1GsHUq+bEyWsnAQh3DwcbZ9Aq9+Nk5cSzEc8S4BhA94Dud2vWQgghhBD/CSqTyWSqygmurq788ccf1K5du9T2s2fP0rp1a65fv35HJ/hPyMzMxMnJiYyMjH/+Ab+YrfDdAPKx5IH8T3lzaAf6RVTjoXUPcfr6aWa0m0G3gG7/7JyEEEL8q93Vn3tC3GeqvLJcUFCAXq8vs12n05GXl3dHJvWfYTLB9g8BWKrviNHei+71vCkwFHAu9RxQtLIshBBCCCHuiioHy82aNWPOnDllts+ePZvGjRvfkUn9ZyTuhQuRFGLJ1/o+DGvmj5WFhqjUKPQmPa7WrvjY+dx6HCGEEEII8beocjWMKVOm0LlzZ44dO0anTp0A2LJlCwcOHGDTpk13fIL/ZsbIOUx1cyGpsAaphS483Fypl1z8cF8993oVV7sQQgghhBB/uyqvLLdu3Zq9e/fi5+fHypUr+eWXXwgJCeH48eO0bdv275jjv1NWMmdiN7Lc0YEd7qnUqXUEHycbAE5dOwUowbIQQgghxJ10/fp1PD09iY+Pv9tT+dvMnj2bPn363JGxqhwsA0RERLBkyRJOnTrFwYMHmT9/PqGhoXdkQv8ZhxZxWlvy5U8wrWT7he3ADSvLbhIsCyGEEBV57LHHUKlUqFQqtFotgYGBvPLKK+Tn59+xa+zYsYOOHTvi6uqKra0toaGhjBw5ksLCQvMxJpOJOXPm0Lx5c+zt7XF2dqZJkybMnDnT3Fr8vffeQ6VS8cwzz5Qa/+jRo6hUKnPgGh8fj0qlwtPTk6ysrFLHRkRE8N5775V7/8Wv7t1vXSnr/fffp1+/fgQEBJi3HThwgE6dOuHs7IyLiwvdunXj2LFj5v1RUVF06NABLy8vrK2tCQoK4q233kKn0930Wi+++CKNGzfGysqq3IZxxV+XP7/s7OzMx2zevJmwsDAcHR159NFHS33tMzIyCAsLIyEhodS4TzzxBIcPH2bXrl23/HrcSpWD5Q0bNvDbb7+V2f7bb7/x66+//uUJ/ScYdHBoAWcsLQFQGW0xYeLVna9yKOUQ8ZnxgKwsCyGEELfSvXt3kpKSiI2N5dNPP+Wbb77h3XffvSNjnz59mu7du9OkSRN27tzJiRMn+Pzzz7G0tMRgMJiPe/TRRxk3bhz9+vVj27ZtHD16lLfffps1a9aUSlG1trZm3rx5nD9//pbXzsrKYsaMGbc8rvj+i1/Lli276fG5ubnMmzePUaNGmbdlZ2fTvXt3/P39iYyMZPfu3Tg4ONCtWzdzMKzVahkxYgSbNm0iKiqKmTNnMnfu3Ep9rZ944gkeeuihcvf973//KzX/pKQk6tSpw5AhQwAwGo08/PDDPPPMM+zdu5eDBw+Wenbutdde45lnnjE3rCtmaWnJww8/zKxZs245v1upcs7ya6+9xocfflhmu8lk4rXXXqNHjx5/eVL/emfXQVYSJ32rAdDe/SlyLfcRmRzJM5uV3zir2VfDxdrlbs5SCCHEf5TJZCJPZ7j1gX8DG62mSs/rWFlZ4e3tDYCfnx+dO3dm8+bNTJs2DVCCrWnTpjFnzhySk5MJCwvj7bffZvDgwQCkpaXx/PPPs2nTJrKzs6levTpvvPEGjz/+OJs2bcLb25vp06ebrxccHFxq9XblypUsWbKE1atX069fP/P2gIAA+vbtW6phW82aNfH09OTNN99k5cqVN72vF154gU8++YQxY8bg6elZqfuvjA0bNmBlZUWLFi3M286ePUtqaiqTJk3Cz88PgHfffZf69euTkJBASEgIQUFBBAUFmc+pUaMG27dvv+XKbXGwevXqVY4fP15mv729Pfb29ubPjx07xunTp5k9ezYA165d49q1azz33HNYW1vTt29fzpw5A8CePXs4cOAAX3zxRbnX7tOnD126dCEvLw8bG5vKfHnKVeVg+fz589SpU6fM9lq1ahEdHX3bE/lP2f8teuCc1gIwMax+G+p4D+GRDY+QkKm8jSAl44QQQtwteToDdd4p+y7yP+H0pG7YWlY5PAHg5MmT7Nmzp9Qq49SpU/n++++ZPXs2oaGh7Ny5k+HDh+Ph4UG7du14++23OX36NL/++ivu7u5ER0ebS+F6e3uTlJTEzp07eeCBB8q95pIlS6hZs2apQLmYSqXCycmp1LYPP/yQpk2bcvDgQZo0aVLhvQwbNozNmzczadKkCoNBgO3bt+Pp6YmLiwsdO3ZkypQpuLm5VXj8rl27ylQvq1mzJm5ubsybN4833ngDg8HAvHnzqF27dqlUjRtFR0ezceNGBg4cWOG1bse3335LWFiY+Tk4Dw8PfHx82LRpE507d2bXrl2MHDkSnU7Hs88+y/z589FoNOWO1aRJE/R6PZGRkbRv3/6251TlNAwnJydiY2PLbI+Oji6VXyIqkHIaEnYTY2mFQW1CbbKmuX8YTlZOfNHxCxwsHQAJloUQQojKWLduHfb29lhbWxMeHs6VK1eYMGECoPSG+OCDD5g/fz7dunUjKCiIxx57jOHDh/PNN98AkJiYSMOGDWnSpAkBAQF07tzZ/GDYkCFDGDZsGO3atcPHx4cBAwbwxRdflFotPn/+PDVr1qz0fBs1asSDDz7Iq6++etPjVCoVH374IXPmzCEmJqbcY7p3787ixYvZsmUL06ZNY8eOHfTo0aNUisifJSQk4OvrW2qbg4MD27dv5/vvv8fGxgZ7e3s2btzIr7/+ioVF6V9cWrVqhbW1NaGhobRt25ZJkyZV8s5vLT8/nyVLlpRKEVGpVKxcuZLJkydTt25dGjZsyBNPPMGHH35Ihw4dsLa2pnXr1tSsWbPMLxW2trY4OTmVyWeuqir/6tavXz/GjRvHqlWrCA4OBpRA+eWXX6Zv375/aTL/CQe+BWCdTS0gDV/bYNQq5XeWAKcA5nady6+xvzIw9M7+piaEEEJUlo1Ww+lJd6d7rI22/FXCinTo0IGvv/6anJwcPv30UywsLBg0aBCgxCe5ubl06dKl1DmFhYU0bNgQgGeffZZBgwZx+PBhunbtSv/+/WnVqhUAGo2GBQsWMGXKFLZu3UpkZCQffPAB06ZNY//+/fj4+FDFRsiAUoa3du3abNq06aYpFt26daNNmza8/fbbLF26tMz+oUOHmj8ODw+nfv36BAcHs337dnN53z/Ly8vD2tq6zLZRo0bRunVrli1bhsFgYMaMGfTq1YsDBw6USmFYsWIFWVlZHDt2jAkTJjBjxgxeeeWVqn4JyrVq1SqysrIYOXJkqe1t2rThwIED5s/PnTvH4sWLOXLkCA888ABjx46lR48e1KtXjwceeID69eubj7WxsTE/ZHm7qhwsT58+ne7du1OrVi2qV68OwMWLF2nbtm2lEtH/0/Iz4fgKAH7BC0ijRbX6pQ6p61aXum5178LkhBBCCIVKpbrtVIh/mp2dHSEhIQDMnz+fBg0amB9gy87OBmD9+vVUq1at1HlWVlYA9OjRg4SEBDZs2MDmzZvp1KkTY8aMKRXTVKtWjUcffZRHH32UyZMnExYWxuzZs5k4cSJhYWGcPXu2SnMODg7mqaee4rXXXmPevHk3PfbDDz+kZcuW5tXymwkKCjKnklQULLu7u5OWllZq29KlS4mPj2fv3r2o1WrzNhcXF9asWVMqKC/Oaa5Tpw4Gg4HRo0fz8ssvV5gKURXffvstvXv3xsvL66bHPf3003z88ccYjUaOHDnCkCFDsLW1pV27duzYsaNUsJyamoqHh8dfmtdtpWHs2bOH9evX89xzz/Hyyy+zZcsWtm7dirOz81+azL/e0SVQmM01mwBSrAoAiPCSwFgIIYS4E9RqNW+88QZvvfUWeXl51KlTBysrKxITEwkJCSn1Kg76QMmLHTlyJN9//z0zZ84st1NxMRcXF3x8fMjJyQHg4Ycf5ty5c6xZs6bMsSaTiYyMjHLHeeeddzh37hzLly+/6T01a9aMgQMH8tprr93y/i9evMj169fx8am4+2/Dhg05ffp0qW25ubmo1epSD1YWf240Giscy2g0otPpbnpMZcXFxbFt27ZSKRjlmTdvHq6urvTt29ecblJcsUOn05VKQYmJiSE/P9/8LsLtuq06yyqViq5duzJhwgSef/75CpPexQ2yr8J2pYrIfF1XNNaXAajtVvtuzkoIIYT4VxkyZAgajYYvv/wSBwcH/ve///HSSy+xaNEiYmJiOHz4MJ9//jmLFi0ClKB1zZo1REdHc+rUKdatW0ft2srP5m+++YZnn32WTf9v797Doqr2N4C/MzDMcL/fVFBUBMxEvCHSySOiaGZ69OctKzTNo2EnxSy1Q2peMLvosVT0hNpFu1iZqZUSKmQhAkoqoqIimAlekDsMyKzfH+Q+TDAICs6A7+d55nmYtdfs/d3Lcl43a6+9fz8uXLiAtLQ0vPbaa0hLS5PmNY8bNw7jx4/HxIkTsWLFCiQnJyMrKwt79uxBcHAwDh48WGedzs7OCA8Pb9DSZsuXL8eBAwdw9uxZqa24uBjz5s3DkSNHcOnSJcTGxmLkyJHo3LkzQkJ0T6EJCQlBWlqa1tXlwYMH49atWwgLC0N6ejrS0tIwZcoUGBsbY+DAgQCqb2T88ssvkZ6ejosXL+LLL7/EggULMH78eCgUCgDV0yi8vb21jnf+/HmkpqYiJycHZWVlSE1NRWpqqtZayUD1bwVcXV3rXVXt2rVrWLZsGd5//30A1f9w8fHxwZo1a5CQkIDY2FgEBgZK/X/++Wd07NhRmjZ8r+7pdywlJSWIi4tDdnZ2rZP917/+dV8FtVr7FgDl+SixewSbrneFqdGPMJGbwMPaQ9+VERERtRrGxsaYNWsWVq1ahZkzZ2Lp0qVwdHREZGQkLl68CBsbG/Ts2RMLFy4EUL0e74IFC3Dp0iWYmprib3/7m3S1t2/fvjh8+DBmzJiBP/74AxYWFnjkkUfw7bffYsCAAQCqLyBu374dmzZtwubNm7F8+XIYGxvD09MTzz33XL3B9ZVXXsGGDRvu+hCVLl264Pnnn9e64m1kZIQTJ07go48+Qn5+Ptq0aYMhQ4Zg6dKl0hSTujz66KPo2bMnvvzyS/zzn/8EUL2i2e7du7FkyRIEBARALpfDz88PP/74o3SV2tjYGG+99RbOnTsHIQTat2+PWbNmYc6cOdK+CwoKtAI9AEybNg1xcXHS+ztXeTMzM6WVNjQaDbZu3YrJkyfXO53j5Zdfxty5c7VuUNy6dStCQ0Oxdu1azJs3D3369JG2ffbZZ3jhhRd07q+hZKKRM9OPHz+OJ554AqWlpSgpKYGdnR1u3LgBMzMzODk51blShqErLCyEtbU1CgoKYGVl1fQHOP8T8OkYQCbHB502Yk3WBZi2245u9t3w2ZP1Lx5ORETU1Jr9e48M2t69ezFv3jycOnVKmqPc2qSlpSEoKAjnzp2rtXxfYzV6hObMmYMRI0bg1q1bMDU1xZEjR5CVlYVevXrxBr+6VJQCe8IBALd7T8PG89aQ/zkFw9veu75PEhERETW54cOHY/r06bhy5Yq+S2k2V69exccff3zfQRm4h2kYqamp2LhxI+RyOYyMjKBWq9GxY0esWrUKoaGhTb44dYsXvwrIzwKs2uJgm+koKj8HG9ccVAHwseN8ZSIiInrwZs+ere8SmlVwcHCT7avRV5YVCoV0yd7JyQnZ2dkAqlfJuHz5cpMV1irkpgG/Vk9CLwqKxNKY3wEIGJv+eXMfwzIRERGRQWv0lWU/Pz8kJSXB09MTAwYMwBtvvIEbN27gk08+Qbdu3ZqjxpZJCGD3bEBzGxqvJzE10QnZeXlo61CBQlEII5kRPG099V0lEREREdWj0VeWV6xYId0ZuXz5ctja2mLmzJm4fv16vesSPnRy04Dfj0IYq7ACU3A0Mw+WSmPMCjEDAHhYe0BlrLrLToiIiIhInxp9Zbl3797Sz05OTvjxxx+btKBW40IsAOCydW98+Jsachnw/tN+SC//GgCnYBARERG1BK1zvRBDcP4nAMDmnOqFsCOe7Iq/eznhzM3qR2LyYSREREREho9huTmoiyGyjwAA4jTd8bS/Oyb37wAASM9LBwB423HZOCIiIiJDx7DcHC4dhqyqAtkaR5RbdsCSpx6BTCZDfnk+rpZcBcCwTERERNQSMCw3hz/nK8druqN/Z0cojKqH+c5VZTdLN1iaWOqtPCIiInp43bx5E05OTrh06ZK+S2k2P/74I3r06AGNRnPf+2JYbg7nq8NynMYXAZ3speaTN04C4FVlIiKipjB58mTIZDLIZDIoFAp4eHjg1VdfRXl5eZMdIy4uDkFBQbCzs4OZmRk8PT0RGhqKiooKqY8QAps2bYK/vz8sLCxgY2OD3r17Y82aNSgtLQUALF68GDKZDDNmzNDaf2pqKmQymRRcL126BJlMBicnJxQVFWn17dGjBxYvXiy9v3Puf329/fbb9Z7T8uXLMXLkSHTo0EFqS0pKwqBBg2BjYwNbW1uEhITgt99+0/qcEALvvPMOunTpAqVSibZt22L58uX1HisvLw+TJk2ClZUVbGxsMHXqVBQXFzdqv8ePH4efnx8sLCwwYsQI5OXlSdtu376NXr164ejRo1r7HDp0KBQKBbZt21ZvfQ1xT2E5NjYWCxcuxLRp0/D8889rvZrSnf+war68vf8XNMvLyxEWFgZ7e3tYWFhgzJgxyM3NbdIaGi0vE8i7gEphhARNV/h72EEIgW3p27AhdQMAwM/JT781EhERtRJDhw7F1atXcfHiRaxevRobN27EokWLmmTfp0+fxtChQ9G7d2/Ex8fj5MmTeP/992FiYoKqqiqp37PPPovZs2dj5MiROHjwIFJTUxEREYFdu3Zh//79Uj+VSoXo6GhkZGTc9dhFRUV455136u1z9epVrdfmzZshk8kwZswYnZ8pLS1FdHQ0pk6dKrUVFxdj6NChcHd3R2JiIg4fPgxLS0uEhISgsrJS6vfyyy/jww8/xDvvvIMzZ87gu+++Q9++feutcdKkSUhLS0NMTAz27NmD+Ph4TJ8+XavP3fY7bdo0BAUF4dixYygoKMCKFSukbe+++y4CAwPrrGPy5MlYu3ZtvfU1iGikxYsXC7lcLvr27StGjhwpRo0apfVqSosWLRKPPPKIuHr1qvS6fv26tH3GjBnCzc1NxMbGiuTkZNGvXz/Rv3//Rh+noKBAABAFBQX3X/TR/wqxyEociegrAlfGirLKMrHw54Wi29ZuotvWbmJe3DxRfrv8/o9DRER0j+76vafRCKEu1s9Lo2nweYSGhoqRI0dqtY0ePVr4+flJ76uqqsSKFStEhw4dhEqlEt27dxc7duyQtufl5Ymnn35aODg4CJVKJTp37iw2b94shBBi9erVokOHDvXW8MUXXwgA4ttvv61jGDUiPz9fCFGdaXx9fcXgwYPF2LFjpT7Hjx8XAERmZqYQQojMzEwBQMybN09YWFiI3Nxcqa+vr69YtGiRzlpGjhwpgoKC6q13x44dwtHRUastKSlJABDZ2dlS24kTJwQAkZGRIYQQ4vTp08LY2FicOXOm3v3XdPr0aQFAJCUlSW0//PCDkMlk4sqVKw3er6mpqUhPTxdCCLF+/XrxxBNPCCGEuHDhgvD09BSFhYV1fi4rK0sAEOfPn29wzXVp9DrLUVFR2Lp1K5599tn7T+oNYGxsDBcXl1rtBQUFiI6Oxvbt2xEUFAQA2LJlC3x8fHDkyBH069fvgdRXU155HsozfgCMjbC30huPdCjGcz88h/S8dBjJjBDeKxzPdn0WMpnsgddGRETUYJWlwIo2+jn2wj8AE/N7+uipU6fw66+/on379lJbZGQkPv30U0RFRcHT0xPx8fF45pln4OjoiAEDBiAiIgKnT5/GDz/8AAcHB5w/fx5lZWUAABcXF1y9ehXx8fF4/PHH6zzmtm3b4OXlhZEjR9baJpPJYG1trdW2cuVK9OnTB8nJyVrPrviriRMnIiYmBm+++SY++OCDu557bm4u9u7di48++qjefj///DN69eql1ebl5QV7e3tER0dj4cKFqKqqQnR0NHx8fKSpGrt370bHjh2xZ88eDB06FEIIBAcHY9WqVbCzs6vzWAkJCdKUlDuCg4Mhl8uRmJiIf/zjHw3ar6+vL2JiYtC5c2fExsaie/fuAIAZM2Zg1apVsLSs+z4wd3d3ODs74+eff0anTp3uOoa6NDosV1RUoH///vd8wMbKyMhAmzZtoFKpEBAQgMjISLi7uyMlJQWVlZUIDg6W+np7e8Pd3R0JCQn1hmW1Wg21Wi29LywsBABUVlZq/bqhMXZf3I1FR/78tY9bWwDHgPJjQDlgo7TBysCV6OvSF7dv376n/RMRETWVe/2uM0R79uyBhYUFbt++DbVaDblcLoVLtVqNFStW4KeffkJAQAAAoGPHjjh8+DA2btyIAQMGIDs7G35+flKgqzmPd+zYsdi3bx8GDBgAFxcX9OvXD4MGDcJzzz0HKysrANU5xcvLq8H19uzZE+PGjcNrr72G2NhYnf1kMhlWrlyJESNGYM6cOXcNex999BEsLS0xevToevtlZWWhTRvtfwhZWlri0KFDGDVqFJYuXQoA8PT0xL59+2BsXB0VL168iKysLOzYsQMff/wxqqqqMGfOHPzf//0fDhw4UOexcnJy4OTkpNVmbGwMOzs75OTkNHi/H374IV588UW88847CAwMxIIFC/DJJ5/AzMwMffr0QUhICC5cuIAJEyZg2bJlWsdr06YNsrKy6h2Tu2l0WJ42bRq2b9+OiIiI+zpwQ/j7+2Pr1q3w8vLC1atXsWTJEvztb3/DqVOnkJOTAxMTE9jY2Gh9xtnZWfoD0CUyMhJLliyp1b5//36YmZndU60fFVf/S85YCMgFoIYCxnLAzagdxijH4MaxG/ge39/TvomIiJrSnZvOdFKYVV/h1QdF476HBw4ciA0bNqCkpASrV6+GsbGxNGf3/PnzKC0txeDBg7U+U1FRAT+/6vuHZs6ciTFjxuDYsWMYMmQIRo0aJV0UNDIywpYtW7Bs2TIcOHAAiYmJWLFiBd566y0cPXoUrq6uEEI0+hSXLVsGHx8f7N+/v1aYrCkkJASPPfYYIiIisH379nr3uXnzZkyaNAkqlarefmVlZbX6lJWVYerUqQgMDMRnn32GqqoqvPPOOxg+fDiSkpJgamoKjUYDtVqNjz/+GF26dAEAREdHo1evXjh79myj/sFQU0P2+8gjjyAuLk76zM2bN7Fo0SLEx8fjpZdeQv/+/fHNN9+gT58+8Pf3x4gRI6S+pqamd//v/S4aHZbLy8uxadMm/PTTT+jevTsUCoXW9vfee+++Cqpp2LBh0s/du3eHv78/2rdvjy+//BKmpqb3vN8FCxYgPDxcel9YWAg3NzcMGTJE+pdiY5RWlmLx14sBAN9cuYrj5f2w2mIuDoT/7Z5rJCIiai53fqOqk0x2z1MhHjRzc3N07twZQHVg9PX1lW5gu7Pqwt69e9G2bVutzymVSgDVWSMrKwvff/89YmJiMGjQIISFhWndXNe2bVs8++yzePbZZ7F06VJ06dIFUVFRWLJkCbp06YIzZ840quZOnTrhhRdewPz58xEdHV1v35UrVyIgIADz5s3T2efnn3/G2bNn8cUXX9z12A4ODrh165ZW2/bt23Hp0iUkJCRALpdLbba2tti1axcmTJgAV1dXGBsbS4EWAHx8qp9GnJ2dXWdYdnFxwbVr17Tabt++jby8PGmK7b3sNzw8HLNnz0a7du1w6NAhLFu2DObm5hg+fDgOHTqkFZbz8vLg6Oh413GpT6PD8okTJ9CjRw8A1XODamruubg2Njbo0qULzp8/j8GDB6OiogL5+flaV5dzc3PrnONck1KplP4nqUmhUNQK/w2RmpOK25rbaKuRoUPlbayp6o6ATvb3tC8iIqLm1lq/n+RyORYuXIjw8HA8/fTT6Nq1K5RKJbKzszFgwACdn3N0dERoaChCQ0Pxt7/9DfPmzdO5EoWtrS1cXV1RUlICAHj66acxYcIE7Nq1q9a8ZSEECgsLa81bBoA33ngDnTp1wueff17vOfXt2xejR4/G/Pnzdfa5cyXW19e33n0BgJ+fHz799FOtttLSUsjlcq0cd+f9nXWKAwMDcfv2bVy4cEGaEnLu3DkA0JojXlNAQADy8/ORkpIizZM+cOAANBoN/P3972m/sbGxSE9Px5YtWwAAVVVV0rSiv04vKi8vx4ULF6TfItyz+7o98AErKioStra24j//+Y/Iz88XCoVCfPXVV9L2M2fOCAAiISGhUfu939Uwlh9ZLrpt7SaWrG0vxCIr0fO1beLrlMv3tC8iIqLm1qSrQOlRXathVFZWirZt24q3335bCCHE66+/Luzt7cXWrVvF+fPnRUpKili7dq3YunWrEEKIiIgI8e2334qMjAxx6tQp8eSTT4q+ffsKIYSIiooSM2bMEPv27RPnz58Xp06dEq+++qqQy+Xi0KFDQojqFS/Gjx8vTE1NxfLly0VSUpK4dOmS2L17twgKChI7d+4UQvxvNYyaIiIihEqlqnM1jOPHj0v9zp49K4yNjYVKpaq1GkZBQYEwMzMTGzZsaNCYnThxQhgbG4u8vDypLT09XSiVSjFz5kxx+vRpcerUKfHMM88Ia2tr8ccffwghqlcV6dmzp3j88cfFsWPHRHJysvD39xeDBw+W9pOYmCi8vLzE77//LrUNHTpU+Pn5icTERHH48GHh6ekpJk6cKG1vyH7vKCsrE97e3lpjM2zYMPHCCy+I1NRU0a5dO/Hll19K2w4ePCgsLCxESUlJg8ZGl/sKy5cvXxaXLzdfKJw7d644dOiQyMzMFL/88osIDg4WDg4O4tq1a0KI6qXj3N3dxYEDB0RycrIICAgQAQEBjT7OvfylsSbmnHh95wmhrqwST3z9hOi2tZv46S1ncSLCV7R/bY+4cqu00XUQERE9CK05LAshRGRkpHB0dBTFxcVCo9GINWvWCC8vL6FQKISjo6MICQkRcXFxQgghli5dKnx8fISpqamws7MTI0eOFBcvXhRCCHHs2DHxzDPPCA8PD6FUKoW9vb14/PHHxXfffad1vKqqKrFhwwbRp08fYWZmJqysrESvXr3Ef/7zH1FaWp0H6grLBQUFwsHB4a5hWQghpk+fLgDUCssbN24Upqam0hJ1DdG3b18RFRWl1bZ//34RGBgorK2tha2trQgKCqp18fHKlSti9OjRwsLCQjg7O4vJkyeLmzdvStsPHjyodS5CCHHz5k0xceJEYWFhIaysrMSUKVNEUVFRo/Z7x/z588XcuXO12jIyMkSfPn2ElZWVmDlzpqiqqpK2TZ8+Xfzzn/9s8LjoIhOicTPTNRoNli1bhnfffVeaC2RpaYm5c+fi9ddfl+a6NIUJEyYgPj4eN2/ehKOjIx577DEsX75cukxfXl6OuXPn4rPPPoNarUZISAjWr19/12kYf3XnVyQFBQUNmrN8vUiNPst/AgA82dMEcWXhMBYCP2f9jk0VY7DL5hnEzRvY+BMmIiJ6ABr7vUety969ezFv3jycOnWqSXObIblx4wa8vLyQnJwMDw+P+9pXo+csv/7664iOjsbKlSsRGBgIADh8+DAWL16M8vLyuz72sDHuNo9HpVJh3bp1WLduXZMdsyGOZf9vYnxcZgzgAviWq1Fo5oONZU9ilId9PZ8mIiIi0p/hw4cjIyMDV65cgZubm77LaRaXLl3C+vXr7zsoA/cQlj/66CN8+OGHeOqpp6S27t27o23btnjxxRebNCwbqpSs6rDsZqWAm0UMTgLwv22MBcoFUMMIAZ0YlomIiMhwzZ49W98lNKvevXvX+9CXxmj0tfe8vDx4e3vXavf29kZeXl6TFGXo7oTljW2+xgWzCgDAnvyJ+PmqEQDAv2PdT7IhIiIiopal0WHZ19e3zscufvDBBw1asqSlU9+uwsnfC/CMUQxKcnaiVC6HSqNCWrEfNALoYG8GV+t7XwOaiIiIiAxHo6dhrFq1CsOHD9d6dGRCQgIuX76M779v/U+oO3WlACZVxfi3ahui/nwwyuMdgpBywwKXbpYioJODniskIiIioqbS6CvLAwYMwLlz5/CPf/wD+fn5yM/Px+jRo3H27Fn87W+t/4l1KVm3MEB+AipU4BfL6kXG/+7+GD6Z6o8ZAzrhpaDOeq6QiIiIiJpKo68sA0CbNm0eihv56pJ86RaeNErCDbkcZ6qnKKN/m/6wNzXD/GG153ITERERUcvVoLB84sQJdOvWDXK5HCdOnKi3b/fu3ZukMEMkhMDJrGt4V56KQ2YqAICPnQ/sTbn6BREREVFr1KCw3KNHD+Tk5MDJyQk9evSATCZDXc8ykclkqKqqavIiDUXWzVJ0KUuFpUkZDlu2BQA81vYxPVdFRERERM2lQWE5MzMTjo6O0s8Pq5SsWxgiT4YAcMRUBYhK9G/TX99lERERETXYzZs34ePjg6NHj6JDhw76LqfRfvzxR8yfPx/Hjh17IE8gbNAR2rdvD5lMBgDIyspC27Zt0b59e61X27ZtkZWV1azF6tuxrJsYbJSCm0Zy5IlKyGVydHdsvdNOiIiIDNnkyZMhk8kgk8mgUCjg4eGBV199FeXl5U12jLi4OAQFBcHOzg5mZmbw9PREaGgoKioqpD5CCGzatAn+/v6wsLCAjY0NevfujTVr1qC0tBQAsHjxYshkMsyYMUNr/6mpqZDJZLh06RKA6ifPyWQyODk5oaioSKtvjx49sHjxYun94sWL4e3tDXNzc9ja2iI4OBiJiYl3Pafly5dj5MiRWkE5KSkJgwYNgo2NDWxtbRESEoLffvtN61h3xrrmy9zcvN5j1fWZmk9oPn78OPz8/GBhYYERI0ZoPbPj9u3b6NWrF44ePaq1z6FDh0KhUGDbtm13Pdem0Og4PnDgwDofPlJQUICBAwc2SVGGquRCIpxk+chUWgEAXMxcYGJkoueqiIiIHl5Dhw7F1atXcfHiRaxevRobN27EokWLmmTfp0+fxtChQ9G7d2/Ex8fj5MmTeP/992FiYqI17fTZZ5/F7NmzMXLkSBw8eBCpqamIiIjArl27sH//fqmfSqVCdHQ0MjIy7nrsoqIivPPOO/X26dKlCz744AOcPHkShw8fRocOHTBkyBBcv35d52dKS0sRHR2NqVOnSm3FxcUYOnQo3N3dkZiYiMOHD8PS0hIhISGorKwEALzyyiu4evWq1qtr164YO3bsXc9ly5YtWp8bNWqUtG3atGkICgrCsWPHUFBQgBUrVkjb3n33XQQGBqJv37619jl58mSsXbv2rsduEqKRZDKZuHbtWq32s2fPCktLy8buziAUFBQIAKKgoEBnn/zSCrHh9UlCLLISOz59UnTb2k1M3Tf1AVZJRETUNO72vafRaERJRYleXhqNpsHnERoaKkaOHKnVNnr0aOHn5ye9r6qqEitWrBAdOnQQKpVKdO/eXezYsUPanpeXJ55++mnh4OAgVCqV6Ny5s9i8ebMQQojVq1eLDh061FvDF198IQCIb7/9ts5xzM/PF0IIsWjRIuHr6ysGDx4sxo4dK/U5fvy4ACAyMzOFEEJkZmYKAGLevHnCwsJC5ObmSn19fX3FokWLdNZy58/1p59+0tlnx44dwtHRUastKSlJABDZ2dlS24kTJwQAkZGRUed+UlNTBQARHx+v81hCCAFA7Ny5U+d2U1NTkZ6eLoQQYv369eKJJ54QQghx4cIF4enpKQoLC+v8XFZWlgAgzp8/X+/xm0KDl44bPXo0gOrL6ZMnT4ZSqZS2VVVV4cSJE+jfv/XO3z2elYch8iQAwFUHdyDnEtwt3fVcFRERUdMru10G/+3+ejl24tOJMFOY3dNnT506hV9//RXt27eX2iIjI/Hpp58iKioKnp6eiI+PxzPPPANHR0cMGDAAEREROH36NH744Qc4ODjg/PnzKCsrAwC4uLjg6tWriI+Px+OPP17nMbdt2wYvLy+MHDmy1jaZTAZra2uttpUrV6JPnz5ITk5G7969dZ7LxIkTERMTgzfffLPOJyf/VUVFBTZt2gRra+t6n6j8888/o1evXlptXl5esLe3R3R0NBYuXIiqqipER0fDx8dH55zmDz/8EF26dGnQMzbCwsIwbdo0dOzYETNmzMCUKVOk6b2+vr6IiYlB586dERsbK62qNmPGDKxatQqWlpZ17tPd3R3Ozs74+eef0alTp7vWcD8aHJbv/GELIWBpaQlT0/890tnExAT9+vXDCy+80PQVGohLZ47h7/Ic3JYpcNmkeuoFwzIREZF+7dmzBxYWFrh9+zbUajXkcrkULtVqNVasWKH11OGOHTvi8OHD2LhxIwYMGIDs7Gz4+flJwbVmOBw7diz27duHAQMGwMXFBf369cOgQYPw3HPPwcqqekpmRkYGvLy8Glxvz549MW7cOLz22muIjY3V2U8mk2HlypUYMWIE5syZozMQ7tmzBxMmTEBpaSlcXV0RExMDBwfdTxPOyspCmzZttNosLS1x6NAhjBo1CkuXLgUAeHp6Yt++fTA2rh0Vy8vLsW3bNsyfP/+u5/vmm28iKCgIZmZm2L9/P1588UUUFxfjX//6F4Dq0P3iiy/inXfeQWBgIBYsWIBPPvkEZmZm6NOnD0JCQnDhwgVMmDABy5Yt09p3mzZtHsj9cg0Oy1u2bAFQ/R/RK6+8ctcJ3a2N6cUfAQC5Dv1wuSQHAOBm6abPkoiIiJqFqbEpEp+++41izXXsxhg4cCA2bNiAkpISrF69GsbGxhgzZgwA4Pz58ygtLcXgwYO1PlNRUQE/Pz8AwMyZMzFmzBgcO3YMQ4YMwahRo6TflBsZGWHLli1YtmwZDhw4gMTERKxYsQJvvfUWjh49CldX1zqX0r2bZcuWwcfHB/v374eTk5POfiEhIXjssccQERGB7du36zz/1NRU3LhxA//9738xbtw4JCYm6txvWVkZVCpVrbapU6ciMDAQn332GaqqqvDOO+9g+PDhSEpK0rpACgA7d+5EUVERQkND73quERER0s9+fn4oKSnB22+/LYXlRx55BHFxcVKfmzdvYtGiRYiPj8dLL72E/v3745tvvkGfPn3g7++PESNGSH1NTU2lGyibU6Nv8Fu0aNFDF5RvV2nQtSAeACD3eRLZRdkAADcrhmUiImp9ZDIZzBRmennd+fV8Q5mbm6Nz587w9fXF5s2bkZiYiOjoaADVN64BwN69e5Gamiq9Tp8+ja+++goAMGzYMGRlZWHOnDn4448/MGjQILzyyitax2jbti2effZZfPDBB0hLS0N5eTmioqIAVN9kd+bMmUbV3KlTJ7zwwguYP3/+XcP2ypUr8cUXX+D48eP1nn+/fv0QHR0NY2Nj6fzr4uDggFu3bmm1bd++HZcuXcKWLVvQp08f9OvXD9u3b0dmZiZ27dpVax8ffvghnnzySTg7OzfgbLX5+/vj999/h1qtrnN7eHg4Zs+ejXbt2uHQoUMYO3YszM3NMXz4cBw6dEirb15enrS0cXO6p8XpvvrqK4wbNw79+vVDz549tV6t0YXzZ/Go7CI0QgZV90EorCgEALSzaKfnyoiIiOgOuVyOhQsX4t///jfKysrQtWtXKJVKZGdno3PnzlovN7f/XfBydHREaGgoPv30U6xZswabNm3SeQxbW1u4urqipKQEAPD000/j3LlzdYZKIQQKCgrq3M8bb7yBc+fOaS2jVpe+ffti9OjRDZryAAAajUZnEAWqr+6ePn1aq620tBRyuVzrHyp33ms0Gq2+mZmZOHjwoNZqGo2RmpoKW1tbrXvf7oiNjUV6ejpmzZoFoPqeuDurcVRWVmqtQFJeXo4LFy5IvyFoTo0Oy2vXrsWUKVPg7OyM48ePo2/fvrC3t8fFixcxbNiw5qhR7279thcAcF7VFb+L6rUbHU0d7/kGBCIiImoeY8eOhZGREdatWwdLS0u88sormDNnDj766CNcuHABx44dw/vvv4+PPvoIQHVo3bVrF86fP4+0tDTs2bMHPj4+AICNGzdi5syZ2L9/Py5cuIC0tDS89tprSEtLk6YDjBs3DuPHj8fEiROxYsUKJCcnIysrC3v27EFwcDAOHjxYZ53Ozs4IDw9v0PJny5cvx4EDB3D27FmpraSkBAsXLsSRI0eQlZWFlJQUPP/887hy5Uq9y7mFhIQgLS1N6+ry4MGDcevWLYSFhSE9PR1paWmYMmUKjI2Nay0LvHnzZri6utaZ+Xbu3Alvb2/p/e7du/Hhhx/i1KlTOH/+PDZs2IAVK1bgpZdeqvXZ8vJyzJo1C5s2bZIeNBIYGIh169bht99+w9dff43AwECp/5EjR6BUKqW56M2qsctneHl5ie3btwshhLCwsBAXLlwQQggREREhwsLCmm6djgfobkvoJESFCbHISiR8MFXsvbBXdNvaTTz3/XMPuEoiIqKm0ZAlU1uCupaOE0KIyMhI4ejoKIqLi4VGoxFr1qwRXl5eQqFQCEdHRxESEiLi4uKEEEIsXbpU+Pj4CFNTU2FnZydGjhwpLl68KIQQ4tixY+KZZ54RHh4eQqlUCnt7e/H444+L7777Tut4VVVVYsOGDaJPnz7CzMxMWFlZiV69eon//Oc/orS0VAjxv6XjaiooKBAODg51Lh13/Phxrb7Tp08XAKSl48rKysQ//vEP0aZNG2FiYiJcXV3FU089JY4ePXrXcevbt6+IiorSatu/f78IDAwU1tbWwtbWVgQFBYmEhIRa59muXTuxcOHCOve7ZcsWUTNa/vDDD6JHjx7CwsJCmJubC19fXxEVFSWqqqpqfXb+/Pli7ty5Wm0ZGRmiT58+wsrKSsycOVPrc9OnTxf//Oc/73quTUEmRONmppuZmSE9PR3t27eHk5MTYmJi4Ovri4yMDPTr1w83b95s+kTfzAoLC2FtbY2CggLp7taakteMR+/8H5HgMQu/9XDDutR1GNV5FJYGLtVDtURERPfnbt971Lrt3bsX8+bNw6lTpx7I46Kb2o0bN+Dl5YXk5GR4eHg0+/EaPUIuLi7SE/zc3d1x5MgRANVzWBqZu1sMZfkNAIDc0hmXiy4D4LJxRERE1DINHz4c06dPx5UrV/Rdyj25dOkS1q9f/0CCMtCIpePuCAoKwnfffQc/Pz9MmTIFc+bMwVdffYXk5GTpwSWtjXll9dVyhbUrLhclA+CycURERNRyzZ49W98l3LPevXvX+0CXptbosLxp0ybpzsiwsDDY29vj119/xVNPPYV//vOfTV6gIbCuqp4Eb2rniuxzXDaOiIiI6GHR6LAsl8u15rdMmDABEyZMaNKiDImoug0bUQDIACNrW9wsr77KzCvLRERERK1fg8LyiRMnGrzDO8/0bi1KC67BXCagETKUqG4DAGyVtrAy4Q0RRERERK1dg8Jyjx49IJPJIIS465N1ai4Y3RoUXrsCcwB5sMKNymsAeFWZiIiI6GHRoNUwMjMzcfHiRWRmZuLrr7+Gh4cH1q9fj+PHj+P48eNYv349OnXqhK+//rpZi125ciVkMpnWpPTy8nJp7rSFhQXGjBmD3NzcJjtmcV71naL5cltkF3K+MhEREdHDpEFXltu3by/9PHbsWKxduxZPPPGE1Na9e3e4ubkhIiICo0aNavIiASApKQkbN26sNc1jzpw52Lt3L3bs2AFra2vMmjULo0ePxi+//NIkx1XfugoAKDK247JxRERERA+ZRq+zfPLkyTrXtfPw8Kj1rPGmUlxcjEmTJuG///0vbG1tpfaCggJER0fjvffeQ1BQEHr16oUtW7bg119/ldZ/vl+3C3MAAGUmdsgu+vPKMqdhEBERET0UGr0aho+PDyIjI/Hhhx/CxMQEAFBRUYHIyEjpWepNLSwsDMOHD0dwcDCWLVsmtaekpKCyshLBwcFSm7e3N9zd3ZGQkIB+/frVuT+1Wg21Wi29LywsBABUVlaisrJSq6/4MyyrVQ64XHgRAOBq6lqrHxERUUvB7zCihmt0WI6KisKIESPQrl07aUrEiRMnIJPJsHv37iYv8PPPP8exY8eQlJRUa1tOTg5MTExgY2Oj1e7s7IycnByd+4yMjMSSJUtqte/fvx9mZmZabU43sgAAV0oFckqr93ku8RyuyFvmU2+IiIhKS0v1XQJRi9HosNy3b19cvHgR27Ztw5kzZwAA48ePx9NPPw1zc/MmLe7y5ct4+eWXERMTA5VK1WT7XbBgAcLDw6X3hYWFcHNzw5AhQ2Blpb0k3MUz7wGVgKx9O0ANWCgs8H/D/++uq4IQEREZqju/USWiu2t0WAYAc3NzTJ8+valrqSUlJQXXrl1Dz549pbaqqirEx8fjgw8+wL59+1BRUYH8/Hytq8u5ublwcXHRuV+lUgmlUlmrXaFQQKFQaLWZV+YBAArNjQF19XzlO9NPiIiIWqK/ftcRkW4NCsvfffcdhg0bBoVCge+++67evk899VSTFAYAgwYNwsmTJ7XapkyZAm9vb7z22mtwc3ODQqFAbGwsxowZAwA4e/YssrOzERAQ0CQ1WFVVh+VbiuoHkvDmPiIiIqKHR4PC8qhRo5CTkwMnJ6d6l4aTyWRN+lASS0tLdOvWTavN3Nwc9vb2UvvUqVMRHh4OOzs7WFlZ4aWXXkJAQIDOm/sa5bYaVqIYAHBLXj2/y92Ky8YRERERPSwaFJY1Gk2dPxuC1atXQy6XY8yYMVCr1QgJCcH69eubZN9VRbkwAlAhjHCj6gYArrFMRERE9DC5pznL+nTo0CGt9yqVCuvWrcO6deua/FhFN/6ADYAbsEZu6R8AgHaW7Zr8OERERERkmBoUlteuXdvgHf7rX/+652IMTfHNK7ABcF1mgz9KqsMyrywTERERPTwaFJZXr17doJ3JZLJWFZbL/3zU9SWVNarETaiMVHA0c9RzVURERET0oDQoLGdmZjZ3HQapsqD6ISS/q8wA3EQ7y3aQyxr9hHAiIiIiaqGY/OpTnAsAuKqqXleZy8YRERERPVzu6Qa/33//Hd999x2ys7NRUVGhte29995rksIMgbz0OgAgx6R6BRAPaw99lkNERERED1ijw3JsbCyeeuopdOzYEWfOnEG3bt1w6dIlCCG0nrTXGijLq5eLyzEuAwTQxbaLnisiIiIiogep0dMwFixYgFdeeQUnT56ESqXC119/jcuXL2PAgAEYO3Zsc9SoN2YVN6EBkINbABiWiYiIiB42jQ7L6enpeO655wAAxsbGKCsrg4WFBd5880289dZbTV6gPlnezsMfxkZQCzWM5cboYN1B3yURERER0QPU6LBsbm4uzVN2dXXFhQsXpG03btxousr0TV0MU5Qjw6T65r5O1p2gkCv0XBQRERERPUiNnrPcr18/HD58GD4+PnjiiScwd+5cnDx5Et988w369evXHDXqx58rYaQpTAFwCgYRERHRw6jRYfm9995DcXExAGDJkiUoLi7GF198AU9Pz1a1EkZ5fg5UANJMGJaJiIiIHlaNDssdO3aUfjY3N0dUVFSTFmQoim9egQrABRNjAIJhmYiIiOgh1Og5y9OmTcOhQ4eaoRTDUpZ3FWUyGXIUAgDQxY5hmYiIiOhh0+iwfP36dQwdOhRubm6YN28efvvtt+aoS+8qC3JwUaGAkAF2KjvYq+z1XRIRERERPWCNDsu7du3C1atXERERgaSkJPTs2ROPPPIIVqxYgUuXLjVDifohinNxzqR69QtPW0/IZDI9V0RERERED1qjwzIA2NraYvr06Th06BCysrIwefJkfPLJJ+jcuXNT16c38tLrUljmfGUiIiKih9M9heU7KisrkZycjMTERFy6dAnOzs5NVZfemZRdx7k/11hmWCYiIiJ6ON1TWD548CBeeOEFODs7Y/LkybCyssKePXvw+++/N3V9eqOquMkry0REREQPuUYvHde2bVvk5eVh6NCh2LRpE0aMGAGlUtkctemPEFCLfOQbuUAGGTrZdNJ3RURERESkB40Oy4sXL8bYsWNhY2PTDOUYiLJbuGhSfdHdSdUOSqNW9o8BIiIiImqQRoflF154oTnqMCzF16QpGB7WnnouhoiIiIj05b5u8GutNEW50s19PvZeeq6GiIiIiPSFYbkOJXl/SFeWuzt567kaIiIiItIXhuU6FOb9jkxFdVjuas+wTERERPSwYliuQ2b+BdyWyaDUyOFq7qrvcoiIiIhITxiW63CxpHq9aMcqSz7mmoiIiOghZtBhecOGDejevTusrKxgZWWFgIAA/PDDD9L28vJyhIWFwd7eHhYWFhgzZgxyc3Pv+7iXbt8EADjA8b73RUREREQtl0GH5Xbt2mHlypVISUlBcnIygoKCMHLkSKSlpQEA5syZg927d2PHjh2Ii4vDH3/8gdGjR9/3cS+hBADgqHC/730RERERUcvV6HWWH6QRI0ZovV++fDk2bNiAI0eOoF27doiOjsb27dsRFBQEANiyZQt8fHxw5MgR9OvX756Pm2VUBUAOF3OusUxERET0MDPosFxTVVUVduzYgZKSEgQEBCAlJQWVlZUIDg6W+nh7e8Pd3R0JCQn1hmW1Wg21Wi29LywsBABUVlaiQl2OPKPqecr2lh1RWVnZTGdERESkH/xuI2o4gw/LJ0+eREBAAMrLy2FhYYGdO3eia9euSE1NhYmJSa3Hbjs7OyMnJ6fefUZGRmLJkiW12vfv3w+lohy3/7yp7/ffC/D999832bkQEREZgtLSUn2XQNRiGHxY9vLyQmpqKgoKCvDVV18hNDQUcXFx97XPBQsWIDw8XHpfWFgINzc3DBkyBMWFaUA8YKIRGDJgEHq3t73fUyAiIjIod36jSkR3Z/Bh2cTEBJ07dwYA9OrVC0lJSfjPf/6D8ePHo6KiAvn5+VpXl3Nzc+Hi4lLvPpVKJZRKZa12hUKBkrLq1TQsNYCTtRkUfz6chIiIqLXgdxtRwxn0ahh10Wg0UKvV6NWrFxQKBWJjY6VtZ8+eRXZ2NgICAu55/zcLrgAAzKpkcDCvHaiJiIiI6OFh0FeWFyxYgGHDhsHd3R1FRUXYvn07Dh06hH379sHa2hpTp05FeHg47OzsYGVlhZdeegkBAQH3tRLGtcKrAAClxhiWKoMeHiIiIiJqZgadBq9du4bnnnsOV69ehbW1Nbp37459+/Zh8ODBAIDVq1dDLpdjzJgxUKvVCAkJwfr16+/rmDeLrwMAlBoTyOV8eh8RERHRw8ygw3J0dHS921UqFdatW4d169Y12THzy28BAJQwbbJ9EhEREVHL1OLmLDe3worqO4SVMnM9V0JERERE+saw/BfFVdVrT6qMrfVcCRERERHpG8PyXxSLcgCAqYLrKxMRERE97BiW/6JYVD8C1EzlqOdKiIiIiEjfGJb/oliuAQBYW7TRcyVEREREpG8MyzVV3UbRnyNiY9VOv7UQERERkd4xLNcgSm+hQF49JE527nquhoiIiIj0jWG5huKi31Elq34QSVtbZz1XQ0RERET6xrBcQ37hFQCAiUbA1cpSz9UQERERkb4xLNeQe+t3AIC5RgZbMxM9V0NERERE+sawXENeUQ4AwLTKGAojDg0RERHRw46JsIa84hsAAJVQ6LkSIiIiIjIEDMs1FKhvAQBUUOm5EiIiIiIyBAzLNRRXFgEAVHJzPVdCRERERIaAYbmGkqoSAIDKyErPlRARERGRIWBYrqFEUw4AMDOx03MlRERERGQIGJZrKEUlAMBC5aDnSoiIiIjIEDAs11AiqwIA2Ji76LkSIiIiIjIEDMs1FP85GrbW7fRbCBEREREZBIblGgr+fBCJsy3DMhERERExLGspklcPh5uNo54rISIiIiJDwLBcg0YmAwC0Z1gmIiIiIjAs16LUANampvoug4iIiIgMAMPyX5hqjPRdAhEREREZCIblvzAVCn2XQEREREQGgmH5L0yh1HcJRERERGQgDDosR0ZGok+fPrC0tISTkxNGjRqFs2fPavUpLy9HWFgY7O3tYWFhgTFjxiA3N/eej2kqM7/fsomIiIiolTDosBwXF4ewsDAcOXIEMTExqKysxJAhQ1BSUiL1mTNnDnbv3o0dO3YgLi4Of/zxB0aPHn3PxzQ1tmyK0omIiIioFTDWdwH1+fHHH7Xeb926FU5OTkhJScHjjz+OgoICREdHY/v27QgKCgIAbNmyBT4+Pjhy5Aj69evX6GOaK2yaonQiIiIiagUMOiz/VUFBAQDAzs4OAJCSkoLKykoEBwdLfby9veHu7o6EhASdYVmtVkOtVkvvCwsLpZ8tlPaorKxsjvKJiIgMAr/niBquxYRljUaD2bNnIzAwEN26dQMA5OTkwMTEBDY2Nlp9nZ2dkZOTo3NfkZGRWLJkSZ3bivJv4/vvv2+yuomIiAxNaWmpvksgajFaTFgOCwvDqVOncPjw4fve14IFCxAeHi69LywshJubGwCgu08fPNHnifs+BhERkaGq+RtVIqpfiwjLs2bNwp49exAfH4927dpJ7S4uLqioqEB+fr7W1eXc3Fy4uLjo3J9SqYRSWfcScW3s2kKh4FrLRETUevF7jqjhDHo1DCEEZs2ahZ07d+LAgQPw8PDQ2t6rVy8oFArExsZKbWfPnkV2djYCAgLu6ZhuNk73VTMRERERtR4GfWU5LCwM27dvx65du2BpaSnNQ7a2toapqSmsra0xdepUhIeHw87ODlZWVnjppZcQEBBwTythAEAbS7umPAUiIiIiasEMOixv2LABAPD3v/9dq33Lli2YPHkyAGD16tWQy+UYM2YM1Go1QkJCsH79+ns+pq3K5p4/S0RERESti0wIIfRdhL4VFhbC2toaPdZ1xfEX0/RdDhERUbO6871XUFAAKysrfZdDZNAMes7yg6YSBn2hnYiIiIgeMIblGsxgou8SiIiIiMiAMCzXoJKZ6bsEIiIiIjIgDMs1mBmb67sEIiIiIjIgDMs1mBtb67sEIiIiIjIgDMs1WJja67sEIiIiIjIgDMs1WJs66rsEIiIiIjIgDMs12Fu30XcJRERERGRAGJZrcLHiNAwiIiIi+h+G5RpcLRmWiYiIiOh/GJZrsFPZ6LsEIiIiIjIgDMs1WCmt9F0CERERERkQhuUaGJaJiIiIqCaG5RoUcoW+SyAiIiIiA8KwTERERESkA8MyEREREZEODMtERERERDowLBMRERER6cCwTERERESkA8MyEREREZEODMtERERERDowLBMRERER6cCwTERERESkA8MyEREREZEODMtERERERDoYfFiOj4/HiBEj0KZNG8hkMnz77bda24UQeOONN+Dq6gpTU1MEBwcjIyNDP8USERERUati8GG5pKQEvr6+WLduXZ3bV61ahbVr1yIqKgqJiYkwNzdHSEgIysvLH3ClRERERNTaGOu7gLsZNmwYhg0bVuc2IQTWrFmDf//73xg5ciQA4OOPP4azszO+/fZbTJgw4UGWSkREREStjMFfWa5PZmYmcnJyEBwcLLVZW1vD398fCQkJeqyMiIiIiFoDg7+yXJ+cnBwAgLOzs1a7s7OztK0uarUaarVael9YWAgAqKysRGVlZTNUSkREZDj4XUfUcC06LN+ryMhILFmypFb7/v37YWZmpoeKiIiIHpzS0lJ9l0DUYrTosOzi4gIAyM3Nhaurq9Sem5uLHj166PzcggULEB4eLr0vLCyEm5sbhgwZAisrq2arl4iIyBDc+Y0qEd1diw7LHh4ecHFxQWxsrBSOCwsLkZiYiJkzZ+r8nFKphFKprNWuUCigUCiaq1wiIiKDwO86ooYz+LBcXFyM8+fPS+8zMzORmpoKOzs7uLu7Y/bs2Vi2bBk8PT3h4eGBiIgItGnTBqNGjdJf0URERETUKhh8WE5OTsbAgQOl93emT4SGhmLr1q149dVXUVJSgunTpyM/Px+PPfYYfvzxR6hUKn2VTERERESthEwIIfRdhL4VFhbC2toaBQUFnLNMREStHr/3iBquRa+zTERERETUnBiWiYiIiIh0YFgmIiIiItKBYZmIiIiISAeGZSIiIiIiHRiWiYiIiIh0YFgmIiIiItKBYZmIiIiISAeGZSIiIiIiHRiWiYiIiIh0YFgmIiIiItKBYZmIiIiISAeGZSIiIiIiHRiWiYiIiIh0YFgmIiIiItKBYZmIiIiISAeGZSIiIiIiHRiWiYiIiIh0YFgmIiIiItKBYZmIiIiISAeGZSIiIiIiHRiWiYiIiIh0YFgmIiIiItKBYZmIiIiISAeGZSIiIiIiHRiWiYiIiIh0aDVhed26dejQoQNUKhX8/f1x9OhRfZdERERERC1cqwjLX3zxBcLDw7Fo0SIcO3YMvr6+CAkJwbVr1/RdGhERERG1YK0iLL/33nt44YUXMGXKFHTt2hVRUVEwMzPD5s2b9V0aEREREbVgLT4sV1RUICUlBcHBwVKbXC5HcHAwEhIS9FgZEREREbV0xvou4H7duHEDVVVVcHZ21mp3dnbGmTNn6vyMWq2GWq2W3hcUFAAA8vLyUFlZ2XzFEhERGYCioiIAgBBCz5UQGb4WH5bvRWRkJJYsWVKr3cPDQw/VEBER6cfNmzdhbW2t7zKIDFqLD8sODg4wMjJCbm6uVntubi5cXFzq/MyCBQsQHh4uvc/Pz0f79u2RnZ3NvzSaSWFhIdzc3HD58mVYWVnpu5xWiWPc/DjGDwbHufkVFBTA3d0ddnZ2+i6FyOC1+LBsYmKCXr16ITY2FqNGjQIAaDQaxMbGYtasWXV+RqlUQqlU1mq3trbmX8zNzMrKimPczDjGzY9j/GBwnJufXN7ib10ianYtPiwDQHh4OEJDQ9G7d2/07dsXa9asQUlJCaZMmaLv0oiIiIioBWsVYXn8+PG4fv063njjDeTk5KBHjx748ccfa930R0RERETUGK0iLAPArFmzdE67uBulUolFixbVOTWDmgbHuPlxjJsfx/jB4Dg3P44xUcPJBNeNISIiIiKqE2f2ExERERHpwLBMRERERKQDwzIRERERkQ4My0REREREOjz0YXndunXo0KEDVCoV/P39cfToUX2X1GJFRkaiT58+sLS0hJOTE0aNGoWzZ89q9SkvL0dYWBjs7e1hYWGBMWPG1Hr6IjXcypUrIZPJMHv2bKmNY9w0rly5gmeeeQb29vYwNTXFo48+iuTkZGm7EAJvvPEGXF1dYWpqiuDgYGRkZOix4palqqoKERER8PDwgKmpKTp16oSlS5ei5j3nHOPGiY+Px4gRI9CmTRvIZDJ8++23WtsbMp55eXmYNGkSrKysYGNjg6lTp6K4uPgBngWR4Xmow/IXX3yB8PBwLFq0CMeOHYOvry9CQkJw7do1fZfWIsXFxSEsLAxHjhxBTEwMKisrMWTIEJSUlEh95syZg927d2PHjh2Ii4vDH3/8gdGjR+ux6pYrKSkJGzduRPfu3bXaOcb379atWwgMDIRCocAPP/yA06dP491334Wtra3UZ9WqVVi7di2ioqKQmJgIc3NzhISEoLy8XI+VtxxvvfUWNmzYgA8++ADp6el46623sGrVKrz//vtSH45x45SUlMDX1xfr1q2rc3tDxnPSpElIS0tDTEwM9uzZg/j4eEyfPv1BnQKRYRIPsb59+4qwsDDpfVVVlWjTpo2IjIzUY1Wtx7Vr1wQAERcXJ4QQIj8/XygUCrFjxw6pT3p6ugAgEhIS9FVmi1RUVCQ8PT1FTEyMGDBggHj55ZeFEBzjpvLaa6+Jxx57TOd2jUYjXFxcxNtvvy215efnC6VSKT777LMHUWKLN3z4cPH8889rtY0ePVpMmjRJCMExvl8AxM6dO6X3DRnP06dPCwAiKSlJ6vPDDz8ImUwmrly58sBqJzI0D+2V5YqKCqSkpCA4OFhqk8vlCA4ORkJCgh4raz0KCgoAAHZ2dgCAlJQUVFZWao25t7c33N3dOeaNFBYWhuHDh2uNJcAxbirfffcdevfujbFjx8LJyQl+fn7473//K23PzMxETk6O1jhbW1vD39+f49xA/fv3R2xsLM6dOwcA+O2333D48GEMGzYMAMe4qTVkPBMSEmBjY4PevXtLfYKDgyGXy5GYmPjAayYyFK3mCX6NdePGDVRVVdV6JLazszPOnDmjp6paD41Gg9mzZyMwMBDdunUDAOTk5MDExAQ2NjZafZ2dnZGTk6OHKlumzz//HMeOHUNSUlKtbRzjpnHx4kVs2LAB4eHhWLhwIZKSkvCvf/0LJiYmCA0Nlcayrr8/OM4NM3/+fBQWFsLb2xtGRkaoqqrC8uXLMWnSJADgGDexhoxnTk4OnJyctLYbGxvDzs6OY04PtYc2LFPzCgsLw6lTp3D48GF9l9KqXL58GS+//DJiYmKgUqn0XU6rpdFo0Lt3b6xYsQIA4Ofnh1OnTiEqKgqhoaF6rq51+PLLL7Ft2zZs374djzzyCFJTUzF79my0adOGY0xEBuWhnYbh4OAAIyOjWqsE5ObmwsXFRU9VtQ6zZs3Cnj17cPDgQbRr105qd3FxQUVFBfLz87X6c8wbLiUlBdeuXUPPnj1hbGwMY2NjxMXFYe3atTA2NoazszPHuAm4urqia9euWm0+Pj7Izs4GAGks+ffHvZs3bx7mz5+PCRMm4NFHH8Wzzz6LOXPmIDIyEgDHuKk1ZDxdXFxq3eB++/Zt5OXlcczpofbQhmUTExP06tULsbGxUptGo0FsbCwCAgL0WFnLJYTArFmzsHPnThw4cAAeHh5a23v16gWFQqE15mfPnkV2djbHvIEGDRqEkydPIjU1VXr17t0bkyZNkn7mGN+/wMDAWssenjt3Du3btwcAeHh4wMXFRWucCwsLkZiYyHFuoNLSUsjl2l9BRkZG0Gg0ADjGTa0h4xkQEID8/HykpKRIfQ4cOACNRgN/f/8HXjORwdD3HYb69PnnnwulUim2bt0qTp8+LaZPny5sbGxETk6OvktrkWbOnCmsra3FoUOHxNWrV6VXaWmp1GfGjBnC3d1dHDhwQCQnJ4uAgAAREBCgx6pbvpqrYQjBMW4KR48eFcbGxmL58uUiIyNDbNu2TZiZmYlPP/1U6rNy5UphY2Mjdu3aJU6cOCFGjhwpPDw8RFlZmR4rbzlCQ0NF27ZtxZ49e0RmZqb45ptvhIODg3j11VelPhzjxikqKhLHjx8Xx48fFwDEe++9J44fPy6ysrKEEA0bz6FDhwo/Pz+RmJgoDh8+LDw9PcXEiRP1dUpEBuGhDstCCPH+++8Ld3d3YWJiIvr27SuOHDmi75JaLAB1vrZs2SL1KSsrEy+++KKwtbUVZmZm4h//+Ie4evWq/opuBf4aljnGTWP37t2iW7duQqlUCm9vb7Fp0yat7RqNRkRERAhnZ2ehVCrFoEGDxNmzZ/VUbctTWFgoXn75ZeHu7i5UKpXo2LGjeP3114VarZb6cIwb5+DBg3X+HRwaGiqEaNh43rx5U0ycOFFYWFgIKysrMWXKFFFUVKSHsyEyHDIhajwuiYiIiIiIJA/tnGUiIiIiorthWCYiIiIi0oFhmYiIiIhIB4ZlIiIiIiIdGJaJiIiIiHRgWCYiIiIi0oFhmYiIiIhIB4ZlImpRDh06BJlMhvz8fH2XQkREDwGGZSIiIiIiHRiWiYiIiIh0YFgmokbRaDSIjIyEh4cHTE1N4evri6+++grA/6ZI7N27F927d4dKpUK/fv1w6tQprX18/fXXeOSRR6BUKtGhQwe8++67WtvVajVee+01uLm5QalUonPnzoiOjtbqk5KSgt69e8PMzAz9+/fH2bNnm/fEiYjoocSwTESNEhkZiY8//hhRUVFIS0vDnDlz8MwzzyAuLk7qM2/ePLz77rtISkqCo6MjRowYgcrKSgDVIXfcuHGYMGECTp48icWLFyMiIgJbt26VPv/cc8/hs88+w9q1a5Geno6NGzfCwsJCq47XX38d7777LpKTk2FsbIznn3/+gZw/ERE9XGRCCKHvIoioZVCr1bCzs8NPP/2EgIAAqX3atGkoLS3F9OnTMXDgQHz++ecYP348ACAvLw/t2rXD1q1bMW7cOEyaNAnXr1/H/v37pc+/+uqr2Lt3L9LS0nDu3Dl4eXkhJiYGwcHBtWo4dOgQBg4ciJ9++gmDBg0CAHz//fcYPnw4ysrKoFKpmnkUiIjoYcIry0TUYOfPn0dpaSkGDx4MCwsL6fXxxx/jwoULUr+aQdrOzg5eXl5IT08HAKSnpyMwMFBrv4GBgcjIyEBVVRVSU1NhZGSEAQMG1FtL9+7dpZ9dXV0BANeuXbvvcyQiIqrJWN8FEFHLUVxcDADYu3cv2rZtq7VNqVRqBeZ7ZWpq2qB+CoVC+lkmkwGonk9NRETUlHhlmYgarGvXrlAqlcjOzkbnzp21Xm5ublK/I0eOSD/funUL586dg4+PDwDAx8cHv/zyi9Z+f/nlF3Tp0gVGRkZ49NFHodFotOZAExER6QuvLBNRg1laWuKVV17BnDlzoNFo8Nhjj6GgoAC//PILrKys0L59ewDAm2++CXt7ezg7O+P111+Hg4MDRo0aBQCYO3cu+vTpg6VLl2L8+PFISEjABx98gPXr1wMAOnTogNDQUDz//PNYu3YtfH19kZWVhWvXrmHcuHH6OnUiInpIMSwTUaMsXboUjo6OiIyMxMWLF2FjY4OePXti4cKF0jSIlStX4uWXX0ZGRgZ69OiB3bt3w8TEBADQs2dPfPnll3jjjTewdOlSuLq64s0338TkyZOlY2zYsAELFy7Eiy++iJs3b8Ld3R0LFy7Ux+kSEdFDjqthEFGTubNSxa1bt2BjY6PvcoiIiO4b5ywTEREREenAsExEREREpAOnYRARERER6cAry0REREREOjAsExERERHpwLBMRERERKQDwzIRERERkQ4My0REREREOjAsExERERHpwLBMRERERKQDwzIRERERkQ4My0REREREOvw/1veEVSQvxd8AAAAASUVORK5CYII=", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "fig, ax = plt.subplots()\n", - "\n", - "for idx in top_indices:\n", - " _ = np.round(architectures_TM[f'ResSCNN{idx+1}']['epochs_acc'][-1], 2)\n", - " ax.plot(np.arange(len(architectures_TM[f'ResSCNN{idx+1}']['epochs_x'])), architectures_TM[f'ResSCNN{idx+1}']['epochs_acc'], label=f'ResSCNN{idx+1} ({_}%)')\n", - "\n", - "ax.set_ylim(0, 100)\n", - "ax.set_yticks(np.arange(0, 110, 10))\n", - "ax.set_ylabel('validation acc [%]')\n", - "ax.set_xlim(0, 100)\n", - "ax.set_xlabel('epoch')\n", - "plt.title(f'top {top_n}')\n", - "\n", - "pos = ax.get_position()\n", - "ax.set_position([pos.x0, pos.y0, pos.width * 0.9, pos.height])\n", - "ax.legend(loc='center right', bbox_to_anchor=(1.45, 0.5), framealpha=0)\n", - "ax.grid(axis='y')\n", - "\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsMAAAG2CAYAAACai4utAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABkJklEQVR4nO3deVhUZf8G8Htm2HfZ98UFXAFFUdQ0d1vcM7NyqbRfpfYq2aJmZrllb1ZumeVbaWpaaqaV+66IiqK4soMLCIgM+zrn98fIKII6AzPMwLk/18V1yTBzzpfDCPc88zzfRyIIggAiIiIiIhGS6rsAIiIiIiJ9YRgmIiIiItFiGCYiIiIi0WIYJiIiIiLRYhgmIiIiItFiGCYiIiIi0WIYJiIiIiLRYhgmIiIiItFiGCYiIiIi0WIYJiIiIiLR0msY/u677xAYGAgbGxvY2NggLCwM//77r+rrxcXFmDRpEhwcHGBlZYURI0bg9u3beqyYiIiIiBoTiSAIgr5OvmPHDshkMrRo0QKCIOCXX37Bl19+iXPnzqFNmzZ4++238ffff+Pnn3+Gra0tJk+eDKlUiuPHj+urZCIiIiJqRPQahmtib2+PL7/8Ei+88AKcnJywYcMGvPDCCwCAq1evolWrVoiIiECXLl30XCkRERERNXRG+i6gUkVFBX7//XcUFBQgLCwMUVFRKCsrQ9++fVX3admyJby9vR8bhktKSlBSUqL6XKFQIDs7Gw4ODpBIJDr/PoiIiPRJEATk5eXB3d0dUimXBhE9id7DcExMDMLCwlBcXAwrKyts27YNrVu3RnR0NExMTGBnZ1fl/i4uLkhPT3/k8RYuXIi5c+fquGoiIiLDdv36dXh6euq7DCKDp/cwHBAQgOjoaMjlcvzxxx8YN24cDh8+XOvjzZgxA+Hh4arP5XI5vL29kZSUBGtra22UTEREZLDy8vLg5+fHv3lEatJ7GDYxMUHz5s0BACEhITh9+jS+/fZbjBo1CqWlpcjJyakyOnz79m24uro+8nimpqYwNTWtdru9vT1sbGy0Xj8REZEhMTY2BgBODSRSk8FNJlIoFCgpKUFISAiMjY2xf/9+1deuXbuG1NRUhIWF6bFCIiIiImos9DoyPGPGDDzzzDPw9vZGXl4eNmzYgEOHDmH37t2wtbXFG2+8gfDwcNWo7pQpUxAWFsZOEkRERESkFXoNwxkZGRg7dizS0tJga2uLwMBA7N69G/369QMAfP3115BKpRgxYgRKSkowYMAArFy5Up8lExEREVEjYnB9hrUtNzcXtra2kMvlnDNMRESNHv/uEWnG4OYMExERERHVF4ZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLYZhIiIiIhIthmEiIiIiEi2GYSIiIiISLb2G4YULF6JTp06wtraGs7Mzhg4dimvXrlW5T3FxMSZNmgQHBwdYWVlhxIgRuH37tp4qJiIiIqLGRK9h+PDhw5g0aRJOnjyJvXv3oqysDP3790dBQYHqPtOmTcOOHTvw+++/4/Dhw7h16xaGDx+ux6qJiIiIqLGQCIIg6LuISpmZmXB2dsbhw4fRo0cPyOVyODk5YcOGDXjhhRcAAFevXkWrVq0QERGBLl26PPGYubm5sLW1hVwuh42Nja6/BSIiIr3i3z0izRjpu4AHyeVyAIC9vT0AICoqCmVlZejbt6/qPi1btoS3t/cjw3BJSQlKSkpUn+fm5gIAysrKUFZWpsvyiYiI9I5/64g0YzBhWKFQYOrUqejWrRvatm0LAEhPT4eJiQns7Oyq3NfFxQXp6ek1HmfhwoWYO3dutdv37NkDCwsLrddNRERkSAoLC/VdAlGDolYY/uuvvzQ+cL9+/WBubq72/SdNmoSLFy/i2LFjGp/rQTNmzEB4eLjq89zcXHh5eaF///58u4iIiBq9yndEiUg9aoXhoUOHanRQiUSCuLg4NG3aVK37T548GTt37sSRI0fg6emput3V1RWlpaXIycmpMjp8+/ZtuLq61ngsU1NTmJqaVrvd2NgYxsbGGn0fREREDQ3/1hFpRu1uEunp6VAoFGp9qDsdQRAETJ48Gdu2bcOBAwfg5+dX5eshISEwNjbG/v37Vbddu3YNqampCAsLU7d0IiIiIqIaqTUyPG7cOI2mPLz66qtqTUmYNGkSNmzYgO3bt8Pa2lo1D9jW1hbm5uawtbXFG2+8gfDwcNjb28PGxgZTpkxBWFiYWp0kiIiIiIgeR6+t1SQSSY23//TTTxg/fjwA5aYb7733HjZu3IiSkhIMGDAAK1eufOQ0iYexxQwREYkJ/+4RaaZOYbisrAyxsbGoqKhAQEBAjXN19Y2/FIiISEz4d49IM7Xege7o0aPw9fVFr1698PTTT8PLywu7du3SZm1ERERERDqldhhWKBRVPp86dSrWr1+PjIwMZGdnY968eXj77be1XiARERERka6oHYY7d+6Ms2fPqj4vLS2Ft7e36nNvb28UFxdrtzoiIiIiIh1Sewe65cuXY8KECejZsyfmzZuHOXPmICQkBAEBASgrK8PVq1exbNkyXdZKRERERKRVaofhzp074/Tp01i8eDFCQkKwePFiXLt2DZGRkaioqECnTp3g4eGhy1qJiIiIiLSqVt0kEhIS8NZbb8HGxgbLli2Du7u7LmrTCq6qJSIiMeHfPSLNaNRN4tKlS9iyZQsqKiqwd+9eDB48GE899RRWrlypq/qIiEgH0uRFOJGQhTR5kb5LISLSK7XD8JIlS9CpUyd8+eWXCAsLww8//IBx48YhMjISJ0+eRFhYGGJiYnRZKxERacGm06notugAXv4hEt0WHcCm06n6LomISG/Unibh6uqKjRs3olevXkhJScHAgQNx5coV1df37t2Ld999t8pthoBvFxER3ZcmL0LXRQfw4G9+mUSCYx/1gputuf4KI63h3z0izag9MiwIAqRS5d1lMhkeztD9+vXDuXPntFsdERFpjSAIWH04EQ8PgVQIApKzCvVTFBGRnqndTeL999/Hs88+i6CgIMTGxmLBggXV7mNmZqbV4oiISDuKyyow+8+L+D3qRrWvySQS+Dpa6KEqIiL9UzsMT58+HQMGDMDVq1fRrl07tGzZUpd1ERGRltzKKcJbv0bhwg05pBJgYBtX7LqUDsW9EeL3B/hzigQRiZbaYRgA2rVrh3bt2umqFiIi0rKIhDuYvOEs7hSUws7CGMtHd0D3Fo5Ikxdh3P9OIfZ2PmzMTfRdps6lyYuQlFUAP0dLBn8iqkKtOcPh4eEoKChQ+6AzZsxAdnZ2rYsiIqK6EQQB/zuWhFfXROJOQSlau9lgx+Tu6N7CEQDgZmuO5wOVPeKPxGbqs1Sd23Q6FV3ZPYOIHkGtMPztt9+isFD9xRUrVqxATk5ObWsiIqI6KCqtQPjm8/hs52VUKAQMDXbHlre7wsu+6rzgnv5OAIDj8Vkoq1Doo1SdS5MX4aOtMapFgwoBmLn1IvsrE5GKWtMkBEGAv78/JBKJWgfVZBSZiIi053p2Id76NQqXbuVCJpVg5rOt8Ho33xp/f7f1sEUTC2PcLSxD9PUcdPK110PFupWUVVBj94zEzAJOlyAiAGqG4Z9++knjA7u4uGj8GCIiqr1jcVmYsvEs7haWwd7SBCte7oCwZg6PvL9MKsFTLZzw1/lbOHwts1GGYWNZzYM4S/fHoY27DewsGv98aSJ6PLXC8Lhx43RdBxER1ZIgCPjhaCIW/XsVCgFo52GLVWNC4GH35JHPHv7KMHwkLhPTBwTUQ7X1a8+l21U+l0iULwIik7IxePlxfD8mBK3cuDEFkZhp1E2CiIgMR5q8CFfTcvHryVTsv5oBAHghxBPzhraFmbFMrWP0uLegLuamHHfyS+BgZaqzeutbUWkFNp9R9lX+8oVAeDaxgK+jBe4WlOH/fj2D1OxCDF95AotfCMSgIHc9V0tE+sIwTETUAG06nYoZW2NUvYKlEmDu4DZ4tYuP2us7AMDZxgyt3GxwJS0Xx+KzMCTYQ0cV178d529BXlQGzybmGN7BEzKp8rq42Zpjx+TumLLxHI7GZWHKxnOIuSnHBwMCYCRTe2NWImok+L+eiKiBSZMXVQnClfq2dtEoCFeq7CpxuBG1WBMEAb9EJAMAXu3iowrClewsTPDza6F4++lmAIDVRxIx7qdTyC4ore9SiUjPGIaJiBqYpKyCakFYIQDJWeq3wHxQD3/lVIkjsVlQPHzgBupsag4u3cqFqZEUozp61XgfmVSCDwe2xIqXO8DCRIbj8XcwaNkxXLwpr+dqiUifNA7Dcrm8xg01srOzkZubq5WiiIjo0SxqmA8sk0jg62hRw72frKOPPSxMZMjKL8GV9Mbxe3zdvVHhQUHuaGL5+I4RzwW6Yds73eDjYIGbOUUY8d0JbD17ox6qJCJDoHEYfumll/Dbb79Vu33z5s146aWXtFIUERHVTBAELD+YUOU2mUSCBcPb1rpvromRFF3vtWBrDFMlsvJL8E9MOgBgbJiPWo8JcLXGX5O6o1eAE0rKFQjffB5zd1xCWYUCafIinEjI4kYdRI2UxgvoIiMjsWTJkmq3P/3005g1a5ZWiiIioprtvpSOfVduw1gmwc+vhUJ6b0S4rhtI9PR3wr4rGTgSm4l3nm6upWr1Y9Pp6yitUCDIyw6BnnZqP87WwhhrxnXCN/tisfRAPH46noxD1zKRckc5LUUqARYOb4dRnbx1VzwR1TuNR4ZLSkpQXl5e7faysjIUFfFVMxGRruQWl2HOX5cAAP/Xoxm6NXdEWDMHreyk1uPeIrozyXeRX1L9d3xDUV6hwK8nUwAAY7uoNyr8IKlUgvD+Afh+TAgsTGRV5mdzK2eixknjMBwaGorVq1dXu33VqlUICQnRSlFERFTdf3dfw+3cEvg6WGByb+2O3vo4WMLHwQLlCgERCXe0euz6tO9KBtLkxbC3NMFzgW61Ps6ANq74bHCbardXCEKtFyoSkWHSeJrEvHnz0LdvX5w/fx59+vQBAOzfvx+nT5/Gnj17tF4gEREB51LvYt29Ec/5w9qpvamGJnr6O2FtRAoOx2agX2sXrR+/Pqw7mQwAGNXJq87XqFsLR0glqNK5oy4LFYnIMGk8MtytWzdERETAy8sLmzdvxo4dO9C8eXNcuHABTz31lC5qJCIStbIKBWZsjYEgAMPbe6Bbc0ednKdHi/v9hgWh4bVYi8/Ix/H4O5BKgFc6131er5utORYOb4fKDsUSoE4LFYnIMNVqB7rg4GCsX79e27UQEVEN1hxLwtX0PNhZGGPWc610dp6wZg4wlklwPbsIyXcK4edoqbNz6ULlXOHeLV3g2UQ7o7ejOnnDwcoUE345AyOpBH1bNcwRcyJ6NI1Hhv/55x/s3r272u27d+/Gv//+q5WiiIhI6Xp2Ib7ZFwsAmPlsKzhYmersXJamRujoYw8AONLAWqzll5RjS5SyN7C67dTU1beVCwI9bVGmEPDb6etaPTYR6Z/GYfijjz5CRUVFtdsFQcBHH32klaKIiEj5e3XWnxdRXKZAl6b2GBniqfNz9gxQTpVoaGF427mbyCsph5+jJbrrYBrJmHudKTZEpqKikezSR0RKGofhuLg4tG7dutrtLVu2RHx8vFaKIiIiYMeFNByJzYSJTIr5w9pBIpE8+UF1VDlv+ETCHZSUVx/4MESCIKh2nBvTxQdSqfav06AgdzSxMMbNnCLsv3Jb68cnIv3ROAzb2toiMTGx2u3x8fGwtGxY88uIiAyVvLAMn+1Q9hSe1Ks5mjlZ1ct5W7lZw8naFEVlFYhKvlsv56yryKRsxN7Oh7mxDCN0NHpuZizDi528AEDV1YOIGgeNw/CQIUMwdepUJCTc3w40Pj4e7733HgYPHqzV4oiIxGrRrivIyi9FMydLvPV003o7r0QiqdJVoiFYF6EMp0Pbe8DW3Fhn53m1sw8kEuBoXBYSMvN1dh4iql8ah+HFixfD0tISLVu2hJ+fH/z8/NCqVSs4ODjgv//9ry5qJCISlVNJ2dh4SrlQa8GwdjA10n5P4cepnDesizCcJi/CiYQsre3idju3GLsvpQPQ/sK5h3nZW6B3gDOA+wGciBo+jVur2dra4sSJE9i7dy/Onz8Pc3NzBAYGokePHrqoj4hIVErKKzBzWwwAYFRHL3Ru6lDvNTzV3BESCXA1PQ+3c4vhYmOmleNuOp2KGVtjoBAAqQRYOLwdRnWqWz/gDZGpKFcI6OTbBK3cbLRS5+OMCfPB/qsZ2BJ1A+8PCIClaa06lBKRAanV/2KJRIL+/fujf//+2q6HiEjUVh9ORHxGPhytTDDj2ZZ6qaGJpQkCPWxx/oYcR2IzMbKjV52PmSYvUgVhQLmr28ytF9HD36nWm1iUliuw4VQqAGBsmG+da1RHjxZO8HWwQPKdQvwZfROvdNbtaDQR6V6twnBBQQEOHz6M1NRUlJaWVvnau+++q5XCiIjEJjEzH8sOKrvyzH6+NewsTPRWS09/J5y/IcdhLYXhpKwCPNyRrEIQkJxVWOswvPtSOjLzSuBkbYoBbVzrXKM6pFIJXu3ig3l/X8G6iBS8HOpdL10+iEh3NA7D586dw7PPPovCwkIUFBTA3t4eWVlZsLCwgLOzM8MwEVEtCIKAWdsuorRcgadaOGJwkLte6+nh74SlB+JxLD4LFQoBsjq2K3N6xGYh2QUltT5m5bzd0aHeMDHSeAlMrY0M8cJ/91zD1fQ8nE6+i1A/+3o7NxFpn8a/PaZNm4ZBgwbh7t27MDc3x8mTJ5GSkoKQkBAuoCMiqqUtZ28iIvEOTI2kmDe0rd5HG4O97GBtZoScwjJcuJFT5+Otj0yt8fZ3f4vGT8eTIAiabWRxNT0Xp5KzIZNK8HJo3eYda8rWwhhDgz0AAGvv9TcmooZL4zAcHR2N9957D1KpFDKZDCUlJfDy8sLixYsxc+ZMXdRIRNSoXUnLxad/XQQA/KdvC/g46L9nu5FMqtrJ7UhsVp2Odf56Dn65Fxq/fSkYGyd2wcH3nsbQYHdUKATM3XEZ720+j+Iy9Tf5WHtvVHhAGxe42mpngZ8mxtzrXLHrYjoycou1fnxtd90gokfTOAwbGxtDKlU+zNnZGampylf7tra2uH6de7YTEWli0+lUPPPtUeSXKIOgnQ775Gqqp/+9rZnjat9irbxCgRlbYyAIwLD2HhgS7IGwZg7wc7LE16OCMfv51pBJJdh67iZGfHcCN+4WPvGY8qIybDt7EwAwpotvrWurizbutgjxaYJyhaBaxKctm06notuiA3j5h0h0W3QAm05r9/hEVJXGYbh9+/Y4ffo0AKBnz5745JNPsH79ekydOhVt27bVeoFERI3VxZtyfLQlpspts/+8ZDCjgT3uheFzqXchLyyr1TH+dzwJl9NyYWdhjFnPtaryNYlEgje6+2HdG6GwtzTBpVu5GLTsGE7EP34kekvUDRSVVcDfxQpdmupvvm5lX+MNkakoq1Bo5ZiP6rphKM8JosZI4zC8YMECuLm5AQDmz5+PJk2a4O2330ZmZiZWr16t9QKJiBoThULA0bhMvLM+CoOXH8PDM2UrOywYAnc7c7RwtoJCAI4naD5V4np2Ib7eGwcAmPlMKzg+YhFd12aO2DGlO9p52OJuYRleXROJH44k1jiPWKEQ8Ou97ZDHhPnqdW71M23d4Ghlioy8Euy5dFsrx3xc1w0i0g2Nu0l07NhR9W9nZ2fs2rVLqwURETVGGbnF+D3qBn47nYrr2Y8e5ZNJJPB1tKjHyh6vh78T4jLycfhaJp5t56b24wRBwOztF1FUVoHOfvYY2dHzsff3sDPH72+FYda2i9hy9gbm/3MFF27K8cWIdrAwuf+n6nhCFhKzCmBlaoRh7T1q/X1pg4mRFKNDvbDsQDzWRiTjuUD1r8+jWJtV/7NsaM8Josam/nrREBGJTIVCwKFrGfi/dWcQtugAvtx9Ddezi2BtZoRxYT749z9P4YsR7SC7N7opk0iwYHjbWvfd1YUH5w1r0vHh75g0HLqWCROZFAuGt1NrBNfMWIb/jgzEZ0PawEgqwY7ztzB85Qmk3rk/Klq5cG5EBw9YGcDuby939oZMKkFkUjaupufW6VjFZRWY/eelKrcZ4nOCqLHR/28SIqJGIE1ehKSsAvg5WkIqkWDz6ev47fR13My5Pwoc4tMEo0O98Vw7N5ibyAAArdxs0MPfCclZhfB1tDC40BPqZw9TIynS5MWIy8iHv4v1Ex8jLyrD3B2XAQDv9GqGZk5Wap9PIpFgbJgvWrra4J31UbianodBy49h2ej2sDKVYd9l5XSEym4O+uZma45+rVyw61I61kWkYP6wdrU6jiAI+OCPC4i+ngNbc2P8MDYEFQoY5HOCqLFhGCYiqqNNp1OrLHqSSIDKQVQbMyOMCPHE6FDvRwZJN1tzgw08ZsYydGnqgMOxmTgSm6lWGP5i11Vk5pWgqZMl3n66Wa3OG+pnjx1TuuOtX8/i/PUcjP3fqSpfj0q5i+bOT66lPozt6oNdl9Kx7dxNfPhMS9iYad4RZNmBePx1/haMpBKsejUEoX4OOqiUiGrCaRJERHXw8Op/QBmEgz3t8PWoIJya1RdzBrVRK0QaqsquEodjn9xi7UxyNjbc22BjwbB2MDWS1fq8brbm2Px/XTA4qPpcXEPqsBDW1AEtnK1QWFqBrVE3NH783xfSsGRvLABg3tC2CGvGIEy1V1paiubNm+PEiRP6LkVndu3aheDgYCgU2uniwjBMRFQHNa3+B4APn2mJYe09YWZc+zBoKHr6KzffiEzKRlHpozfGKC1X9hQGgBc7eqJL07qHOlMjGV6qYYc5Q+qwIJFIVNM21p1M0Whu9YUbOXjv92gAwBvd/Wr8XsVq/PjxkEgkkEgkMDY2hp+fHz744AMUF2tvk5PDhw+jd+/esLe3h4WFBVq0aIFx48ahtLRUdR9BELB69Wp07twZVlZWsLOzQ8eOHfHNN9+gsFD5HPz0008hkUjw1ltvVTl+dHQ0JBIJkpOTAQDJycmQSCRwdnZGXl5elfsGBwfj008/VX2+detW9O/fHw4ODpBIJIiOjlbre1q1ahX8/PzQtWtX1W3z589H165dYWFhATs7uxofV3mtH/z47bffHnuu2NhYDBkyBI6OjrCxsUH37t1x8OBB1dd//vnnGo8rkUiQkZEBADh37hzat28PKysrDBo0CNnZ2arHl5eXIyQkBKdOVX1naODAgTA2Nsb69evVuiZPUqswvH//fsycORMTJkzA66+/XuWDiEhMfB2qr/JvbKv/mzlZwcPOHKXlCpxMuvPI+60+koC4jHw4WJpg5rOtHnk/TSnnYVe9zdCu8bD2HrA0kSEhswAnEh59jR6ULi/GhF/OoLhMgV4BTlq9ZrpS3zvjDRw4EGlpaUhMTMTXX3+N77//HnPmzNHKsS9fvoyBAweiY8eOOHLkCGJiYrBs2TKYmJigouL+i74xY8Zg6tSpGDJkCA4ePIjo6GjMnj0b27dvx549e1T3MzMzw5o1axAXF/fEc+fl5eG///3vY+9TUFCA7t2744svvlD7exIEAcuXL8cbb7xR5fbS0lKMHDkSb7/99mMf/9NPPyEtLU31MXTo0Mfe//nnn0d5eTkOHDiAqKgoBAUF4fnnn0d6ejoAYNSoUVWOl5aWhgEDBqBnz55wdnYGAEyYMAG9e/fG2bNnIZfLsWDBAtXxv/rqK3Tr1g2hoaHVzj1+/HgsXbpUncvyRBrPGZ47dy4+++wzdOzYEW5ubnrt8UhEpG938qtuRtEYV/9LJBL08HfExlPXcSQ2E70CnKvdJzmrAEsPxAMAZj/fGnYWJlo7v5utORYOb4eZWy+iQhAM8hpbmxljeAdPrDuZgl9OJKPbva2sH6WotAIT1p5GRl4J/F2ssHR0e8geTvw6IggCijTY+rrSlqgbmPPXJSgEQCoB5g5ugxEhj2+Z9zBzY5lGucHU1BSurq4AAC8vL/Tt2xd79+5VBUSFQoEvvvgCq1evRnp6Ovz9/TF79my88MILAIC7d+9i8uTJ2LNnD/Lz8+Hp6YmZM2fitddew549e+Dq6orFixerztesWTMMHDhQ9fnmzZuxfv16/PnnnxgyZIjqdl9fXwwePBi5ufc7iAQEBMDZ2RmzZs3C5s2bH/t9TZkyBUuWLMGkSZNUofBhY8aMAQDVqLI6oqKikJCQgOeee67K7XPnzgWgHKl9HDs7O9X1fpKsrCzExcVhzZo1CAwMBAAsWrQIK1euxMWLF+Hq6gpzc3OYm9//f5qZmYkDBw5gzZo1qtuuXLmC9evXw9/fH6NHj8bOnTsBAImJiVizZg2ioqJqPP+gQYMwefJkJCQkoFmz2q1NqKRxGF61ahV+/vln1Q+JiHTvwU4FhhQACNhzWTkC8nSAE/6vR7NGu/q/p7+TKgw/TBAEzPozBqXlCjzVwhFDgt21fv5RnbwNuusGoNyRbt3JFOy7chs3c4rgYVdzjQqFgPDN0bh4Mxf2liZYM64TrGux6K62isoq0PqT3XU6hkIAZm+/hNnbLz35zg+4/NmAKn2jNXHx4kWcOHECPj73O4ksXLgQv/76K1atWoUWLVrgyJEjePXVV+Hk5ISePXti9uzZuHz5Mv799184OjoiPj4eRUXKUW1XV1ekpaXhyJEj6NGjR43nXL9+PQICAqoE4UoSiQS2trZVblu0aBE6deqEM2fOVNmX4WGjR4/G3r178dlnn2H58uW1uRw1Onr0KPz9/WFtXbs1CpMmTcKECRPQtGlTvPXWW3jttdce+eLFwcEBAQEBWLt2LTp06ABTU1N8//33cHZ2RkhISI2PWbt2LSwsLFQvVgAgKCgIe/fuRfPmzbF//35VsH7rrbewePHiR34v3t7ecHFxwdGjR+s/DJeWllaZh0JEuvVgpwKpBFg4vB1GdeK8QkOx+5IyDA8Jdm/UC5+6NneETCpBQmYBbtwthGeT+1MUtp27iePxd2BqJMW8oW119o6hIXfdAIAWLtYIa+qAiMQ72BCZgvcHtKzxfl/vi8W/F9NhIpPi+zEh8LI3nOkehmbnzp2wsrJCeXk5SkpKIJVKVeGxpKQECxYswL59+xAWFgYAaNq0KY4dO4bvv/8ePXv2RGpqKtq3b68Kpr6+vqpjjxw5Ert370bPnj3h6uqKLl26oE+fPhg7dixsbGwAAHFxcQgICFC73g4dOuDFF1/Ehx9+iP379z/yfhKJBIsWLcKgQYMwbdq0Ooe5SikpKXB3r92L0c8++wy9e/eGhYUF9uzZg3feeQf5+fl49913a7y/RCLBvn37MHToUFhbW0Mqlao2Y2vSpEmNj1mzZg1efvnlKqPFP/74I9555x3897//Rbdu3TBjxgysW7cOFhYW6NSpEwYMGICEhAS89NJLmDdvXpXjubu7IyUlpVbf74M0DsMTJkzAhg0bMHv27DqfnIge7+FOBQpBuYq+h7+TQYcCsUjKKkDs7XwYSSXoHeCi73J0ysbMGB287XA6+S6OxGbh5c7KF2TZBaWY9/cVAMC7fVrAx8FSn2Xq3dgwH0Qk3sFvp67j3T4tqnXT2B59E8vuTSdZMLwdOvna13uN5sYyXP5sgEaPSZcXo++Sw1UWi0olwL7wnnC1NdPo3Jro1asXvvvuOxQUFODrr7+GkZERRowYAQCIj49HYWEh+vXrV+UxpaWlaN++PQDg7bffxogRI3D27Fn0798fQ4cOVQ3oyWQy/PTTT5g3bx4OHDiAyMhILFiwAF988QVOnToFNzc3jRZDVpo3bx5atWqFPXv2PHIKBAAMGDAA3bt3x+zZs7FhwwaNz1OToqIimJmp//N40IO5rn379igoKMCXX375yDAsCIJqmsfRo0dhbm6OH3/8EYMGDcLp06fh5la1C0xERASuXLmCdevWVbm9TZs2OHz4sOrzO3fuYM6cOThy5AimTJmCrl27YuvWrejUqRM6d+6MQYMGqe5rbm6uWsRYFxovoCsuLsaSJUvQs2dPTJkyBeHh4VU+iEh7aupUYEir6MVuz71R4S5NHWBrUX9vc+tLjxaVLdYyVLct+OcKsgtKEeBijTd7NNVXaQajX2sXuNqY4U5BKf6NSa/ytaiUu3j/jwsAgLd6NsMLGs631RaJRAILEyONPpo6WWHh8Kq7JS4c3g5Nnaw0Oo6m7xpYWlqiefPmCAoKwv/+9z9ERkaq5pvm5+cDAP7++29ER0erPi5fvow//vgDAPDMM88gJSUF06ZNw61bt9CnTx9Mnz69yjk8PDwwZswYLF++HJcuXUJxcTFWrVoFAPD398fVq1c1qrlZs2aYOHEiPvrooyeG6UWLFmHTpk04d+6cRud4FEdHR9y9e1crx+rcuTNu3LiBkpKSGr9+4MAB7Ny5E7/99hu6deuGDh06YOXKlTA3N8cvv/xS7f4//vgjgoODHzmFolJ4eDimTp0KT09PHDp0CCNHjoSlpSWee+45HDp0qMp9s7Oz4eTkVOvvsZLGYfjChQsIDg6GVCrFxYsXce7cOdWHum0/iEg9fo6WePhvh1QCg1pFL2Z77u2G1r9N4x4VrtQzQPlH50T8HZRVKBCRcAd/RN2ARKIc5TSWsVunkUyqGjX/JSJZdfuNu4X4v3VnUFquQP/WLvhggPpvvRuKUZ28ceyjXtg4sQuOfdSr3qdrSaVSzJw5Ex9//DGKiorQunVrmJqaIjU1Fc2bN6/y4eXlpXqck5MTxo0bh19//RXffPMNVq9e/chzNGnSBG5ubigoKAAAvPzyy4iNjcX27dur3VcQBMjl8hqP88knnyA2NvaJrclCQ0MxfPhwfPTRR+pcgidq3749rl69WqsR7YdFR0ejSZMmMDU1rfHrlSOyUmnV//dSqbRa/9/8/Hxs3ry5WpeLh+3fvx9XrlzB5MmTAQAVFRUoK1MuUi4rK6vS5aO4uBgJCQmqdwHqQuNpEg/2jyMi3XKzNUcbNxtcvHV/xXInX3tOkTAAGXnFOJuqHIHp11ocYbituy3sLU2QXVCKyMRsfLL9IgDglc7eCPGpeY6gGL0U6oVlB+JwLjUHF2/K4edoiQm/nEFWfilaudng61HBkNZT5wht0/e87ZEjR+L999/HihUrMH36dEyfPh3Tpk2DQqFA9+7dIZfLcfz4cdjY2GDcuHH45JNPEBISgjZt2qCkpAQ7d+5Eq1bKFnbff/89oqOjMWzYMDRr1gzFxcVYu3YtLl26hGXLlgEAXnzxRWzbtg2jR4/Gxx9/jP79+8PJyQkxMTH4+uuvMWXKlBrbj7m4uCA8PBxffvnlE7+n+fPno02bNjAyqhrJsrOzkZqailu3bgEArl27BkC58O9RHR969eqF/Px8XLp0CW3btlXdnpqaqjpeRUWFavCyefPmsLKywo4dO3D79m106dIFZmZm2Lt3LxYsWFBlFP3UqVMYO3Ys9u/fDw8PD4SFhaFJkyaq62xubo4ffvgBSUlJ1bpZbNq0CeXl5Xj11VcfeR2Ki4sxefJkbNy4URWwu3XrhhUrVmDSpEnYsmULlixZorr/yZMnYWpqqpovXhd1ehl/48YN3Lih+W47RKSeNHkRrqQrG7OP7+oLAIi5KUducdljHkX1Yd/lDAgCEORpK5oXJ1KpBE+1ULYMm7zhLBKzCuBkbfrIhWJi5WxthmfaKudLrjwUjzFrTuFqeh4crUzx47iOsDStXTcFAoyMjDB58mQsXrwYBQUF+PzzzzF79mwsXLgQrVq1wsCBA/H333/Dz88PAGBiYoIZM2YgMDAQPXr0gEwmU43WhoaGIj8/H2+99RbatGmDnj174uTJk/jzzz/Rs2dPAMopJRs2bMCSJUtUtwcGBuLTTz/FkCFDMGDAo+deT58+HVZWVk/8nvz9/fH6669X20zkr7/+Qvv27VXB8qWXXkL79u1VUzhq4uDggGHDhlXbjOKTTz5B+/btMWfOHOTn56N9+/Zo3749zpw5AwAwNjbGihUrEBYWhuDgYHz//fdYsmRJlZ7OhYWFuHbtmmqk1tHREbt27UJ+fj569+6Njh074tixY9i+fTuCgoKqnH/NmjUYPnz4Izf8AJTt35577jkEBwerblu6dCmio6PRo0cPDBo0SDVfHAA2btyIV155BRYWdX+nVCJoOJauUCgwb948fPXVV6r5OtbW1njvvfcwa9asasPl+pabmwtbW1vI5XLV6lCihuKrPdew7EA8OvvZ47c3u2DAN0cQezsfcwa1xmvd/PRdnqiN/+kUDl3LxPsDAjCpV3N9l1NvPvjjPDafuT8I8kpnb8wf1k6PFRmmM8nZeGFVRJXbJj3dDO8P1P0LB/7dE7cLFy6gX79+SEhIUCuMN0RZWVkICAjAmTNnVC986kLj5Dpr1iwsX74cixYtUs0VXrBgAZYtW8YOE0RaVFquwMZT1wEAY8N8lVu+dqndlq+kXXnFZTgRr9xlbIBI5gsDyncq/oiq+m7gb6dS6203sobE3a76iv5VhxN5rUjnAgMD8cUXXyApKUnfpehMcnIyVq5cqZUgDNRizvAvv/yCH3/8EYMHD1bdFhgYCA8PD7zzzjuYP3++VgojErt/L6YhK78ELjamqgVawzp44otd15CYWYDj8XfQvcXjd7ki3Th0LROlFQo0dbREM6fGOfJSk5q7mwDJWYWimSqiruQ71Tu+VHaC4bUiXRs/fry+S9Cpjh07PnZTE01pPDKcnZ2Nli2rv83TsmVLZGdna6UoIgLWRigbiY8O9Vat0rcyNcLwDh4Aqq5Up/p1v4uEq6i2pPdztMTD675kEgm7m9SA14qo4dA4DAcFBdW4deDy5curTZgmotq5dEuOqJS7MJJK8HJo1fZFY8OUUyX239vylepXSXkFDl5V9tkVS0u1Sm625tV6zS4Y3pYjnTXgtSJqODSeJrF48WI899xzVbY/jIiIwPXr1/HPP/9ovUAiMVp3b1R4YFtXONtUnXvY3NkaXZs54ETCHaw/mYIP6mFBTkOSJi9CUlYB/BwtdRI8IhLuIL+kHM7Wpgj2tNP68Q3dqE7e6OHvhOSsQvg6WjDcPQavFVHDoPHIcM+ePREbG4thw4YhJycHOTk5GD58OK5du4annnpKFzUSiYq8sAx/Rt8EoFw4V5PK0eFNp6+jpLyixvuI0abTqei26ABe/iES3RYdwKbTqVo/R+UUiX6tXRpsr9i6crM1R1gzB4Y7NfBaERm+WjU7dHd350I5Ih35Peo6issUaOlqjU6+NW9k0LeVC9xszZAmL8Y/MWkY1l4/27oakjR5ET7aGoPKJhsKAZi59SJ6+DtpLYgoFAL2PjBfmIiIGj61wvCFCxfQtm1bSKVSXLhw4bH3DQwM1EphRGKkUAj49aRyisSYMJ9HLs4ykknxcqg3vtobi19OpDAMAzh/PQcPd5vT9ur9c9dzkJlXAmtTI4Q1ddDKMYmISL/UCsPBwcFIT0+Hs7MzgoODIZFIauxxKpFIquwbTUSaORKXieQ7hbA2NcLQYI/H3velUG8sPRCH6Os5iLkhRztP23qq0vAUl1VgxcGEarfLJNDq6v09l9IBAL1aOsPEyLA2GCIiotpRKwwnJSXByclJ9W8i0o3KhXMvdPR84patTtameLadG7ZH38LaiGR8OVKc3VwEQcDMrTGIuSmHqZEUZRUKVS/cVzr7aG1UWBAE7L4XhsXWRYKIqDFTa2jDx+f+27UpKSnw8PCAj49PlQ8PDw+kpKTotFiiB6XJi3AiIavR7Oh0PbsQB64pW3ZV7jT3JJUL6f46fwt3C0p1Vltt1NfP57vDCdh67iZkUgl+HNcRxz/qjSFB7gCAMyl3tbZTX3xGPpLvFMJEJsXTAc5aOSYREemfxu/z9erVq8bNNeRyOXr16qWVooiepD66BtS3XyNTIAjAUy0c0VTNXc06eDdBazcblJQr8HvUdR1XqL76+vnsupiOxbuuAQA+HdQaT7VQLpb7dHAbWJrIcDktV9X9oa4qR4W7NXeA1RNG7YmIqOHQOAwLglDjop47d+7A0tJSK0URPYpCIeDPczfx4ZYY1VvhlV0DGvIIcXFZBTafVoZZdUeFAeU8/crR4XUnU1Dx8F65epAmL8KMrbr/+Vy8Kce0TdEAgHFhPhjzQBu6JpYmGN9N+fk3++Kg0MJ1qQzVA9hFgoioUVF7eGP48OEAlH98x48fD1NTU9XXKioqcOHCBXTt2lX7FRIByMgtxu9RN7DxVCpu3K0eqrTdNaC+7Th/C3cLy+BhZ44+rTSbjzok2AML/rmC69lFOBybgd4t9TufNSmrAA9nzwpBQEJGvtZ+Phm5xZi49gyKyirwVAtHzH6+dbX7THyqKX45kYIr90aHB7atfYi9lVOECzfkkEig8c+HiIgMm9ph2NZWuVJdEARYW1vD3Pz+HzUTExN06dIFEydO1H6FJFoVCgFH4jKxMTIV+69mqEY9rUxlKCipwIN5S6rlrgH1bd29dmqvdPGGTMONHMxNZHixoxd+PJaEtREpeg/Dfo6WkAB4eCz2632xaO1uC3tLkzodv7isAhPXRSFNXoxmTpZY/nIHGMmqv8llZ2GC17r5YtmBeHyzLxb967BJRmVv4RDvJnCyNn3CvYmIqCFROwz/9NNPAABfX19Mnz6dUyJIZ9Llxdh85jo2nb6Omzn3R4E7+TbB6FDvex0UbmLm1ououLc4ysXGDE5WDTOkRF/PwYUbcpjIpBjV0atWx3i1iw9+PJaEw7GZSLlTAB8H/f3/dLM1h4uNKdJzSwAoX6gYSSWISsnBoGXH8P2YELT1qF0bOEEQMP338zh/PQd2FsZYM64TbM2NH3n/N7r74efjybianoc9l9MxsK1brc6757JyvjCnSBARNT4arwKZM2eOLuogkUqTFyEpqwDe9haIvZ2HDZHXceDqbdXb7LbmxhjRwROjQ73QwsVa9bhRnbzRw98J56/n4P0/LiBNXoyfjidjYo+mevpOam9tRDIA4PlANzjUMtD7Olqip78TDsdm4teTKZj1XPVpA/Ul9U4h0nNLIJUA378agraetsgrLseba88g+U4hRnx3AguHt8PwDppvFPLt/jjsvJAGI6kE370SAl/Hx4f+ytHhpQfi8c2+OPRv7arx6HBOYSlOJioXDfdrzSkSRESNTa2WRP/xxx/YvHkzUlNTUVpatZ3T2bNntVIYNX6bTqdWWWj1oFA/e7wc6o2BbV1hZiyr8fFutuZwszWHvKgMH26JwZK9sRjY1hVe9g1nukR2QSl2XkgDoNxxri7GhvngcGwmNp+5gfB+ATA3qfm66VrlKGpnPwf0uzeS6mYLbJ/cHVN/O4eD1zIRvvk8LtyQY9ZzrWBcwxSHmuw4fwvf7IsDAMwf1hZhzdTbAe6N7k3x073R4V2X0vFsO81Ghw/cm6IT4GL9xPBNREQNj8bdJJYuXYrXXnsNLi4uOHfuHEJDQ+Hg4IDExEQ888wzuqiRGqGHOw5UGt3JC/vCe2Lz/4VhaHuPRwbhB73Y0QuhfvYoKqvAJ9svaq2vbH3YdPo6SssVaOdhi2Avuzod6+kAZ3g2Ub44+Ov8Te0UWAt7LlV2Xag6imprrpzW8G6fFgCAn08k45UfI5GZV/LEY56/noPpv58HAEzo7odRnbzVrsfWwhivdfcDAHxbi84Sj/p+iIiocdA4DK9cuRKrV6/GsmXLYGJigg8++AB79+7Fu+++C7lcrosaqRGqqeMAAAwO9kBzZ/V67FaSSCRYMKwdTGRSHLyWiX9i0rVUpW5VKAT8em/h3JgwnxpbFmpCJpWo2rKtjUjRy4uCrPwSnEm5N6Wghvm1UqkE4f38sXpMCKxMjXAqKRuDlh1D9PWcRx4zTV6EiWvPoKRcgd4tnTHj2VYa1/VGdz9Ymxnh2u08/HtR/edHcVkFDsdmAgD6c74wEVGjpHEYTk1NVbVQMzc3R15eHgBgzJgx2Lhxo3aro0arpt3SZBJJrTtCNHe2wttPNwMAfLrjEuRFZXWqrz4cuJqBmzlFsLMwxuB7O6bV1YsdvWBqJMWlW7k4m5qjlWNqYv8V5Xzvdh628LB7dBu1/m1c8eekbmjqZIn03GK8uCpC1Wf5QYWl5Zjwyxlk5JUgwMUa374UrHG3DUA5Kv16t3ujw/tj1R4dPhqXhaKyCnjYmaONu43G5yUiIsOncRh2dXVV7UDn7e2NkydPAgCSkpIa1NvTpD8l5RX4am8sAKAy1sgkEiwY3rZOfWjffroZmjpaIjOvBIt3XdVCpbpVuXBuVEcvtaaDqKOJpQkG3QvW6+4dvz5VTinor8ZCs+bOVtg+qRv6tXZBaYUCH2y5gI//jEFpuQKAcoOV8E3ncelWLhwsTfDjuI6wNnt054gnef3e6HDs7Xz8czFNrcdU7jrXr7VLnUfuiYjIMGkchnv37o2//voLAPDaa69h2rRp6NevH0aNGoVhw4ZpvUBqfL47lIDEzAI4Wpli99Qe2DixC4591EujeaA1MTOWYf6wdgCA9ZGpiEq5q41ydSIxMx9H47IgkSjbomlT5Y50/8SkqzUfV1vyS8pxND4LgPpTCqzNjPH9qyEI7+cPiQT49WQqRv9wEjE3cjBtczR2XUqHiUyK78eE1HlhpK25MSZ0V3Yb+XZf3BN36yuvUGD/lXvhnvOFiYgaLY3D8OrVqzFr1iwAwKRJk/C///0PrVq1wmeffYbvvvtO6wVS4xKfkY+VBxMAAHMGtYa/qzXCmjlobWeysGYOGBmibNk1c2sMyioUWjmutv16MhUA0CvAWevdLwI97RDkZYfSCgU2nU7V6rEf50hsJkrLFfB1sIC/i/rzvqVSCd7t0wJrxnWEtZkRolLuYtDy49gefQsAMLS9Ozr62mulxte6+8LGzAhxGfn4J+bxo8NnUu7ibmEZ7CyMEaql8xMRkeHROAxLpVIYGd3vyPbSSy9h6dKlmDJlCkxM6razFDVugiBg1rYYlFYo0NPfCc8H1m4DhCeZ+Wwr2Fua4NrtPKw+kqiTc9RFYWk5fo9Szo+tazu1Rxl377jrI1NRXk8vCPbcm1LQv41rraYU9G7pgjXjOla7fUvUTaTJq2/BXRs2ZsaY8NS90eH9jx8drpwi0aelS4073BERUeOgVp/hCxcuqH3AwMDAWhdDjdvvUTcQmZQNM2Mp5g1tq7M5mE0sTfDxc60Qvvk8lu6Pw/OBbnrdke1hf567hbzicvg4WKBnCyednOPZdm6Y9/cVpMmLse9KBga21W0nhNJyBfZfzQBQtxZk5TWE0wpBQHJWodbePRjfzRdrjiUhPiMfOy/cwpBgj2r3EQSBLdWIiERCrTAcHBwMiUQCQRCeGGAqKiq0Uhg1LnfyS7DgnysAgGl9/XW+Mcaw9h7YcvYGjsffwcd/XsTa10MNYgGUIAiqhXNjuvhovBuausyMZRjVyQvfHUrAupPJOg/DkUl3kFdcDkcrUwR7Nan1cfwcLSGVoErbvbp0GamJjZkxJnT3w1d7Y++9WHKv1qHi0q1c3MwpgpmxFE/p6AULEREZBrXe+0tKSkJiYiKSkpKwZcsW+Pn5YeXKlTh37hzOnTuHlStXolmzZtiyZYuu66UGat7fV5BTWIZWbjZ4/d4GCLokkUgwf2g7mBpJcTQuSzX/VN/OpNzF1fQ8mBlLMTLES6fneqWzN6QS4Hj8HcRn5On0XJWjqP1aO9eq9VklN1tzLBzeDrJ7L1y00WWkJuO7+cLW3BgJmQXYeaH6c2PPZeX306OFk9528iMiovqh1siwj8/9eY0jR47E0qVL8eyzz6puCwwMhJeXF2bPno2hQ4dqvUh9SZMXISmrAH6Ollr/YywmR+Myse3cTUgkwMLh7dTefreufB0t8W6fFvhy9zV8vvMyng5wgp2Ffue1f39YOYe5XysX2FrUvk2YOjybWKB3Sxfsu3Ibi3ddw9whbXTyPFYoBNUWzNrYmGJUJ2/08HdCclYhfB0tdFKztZkxJj7lh//uicW3NYwOV85/HsCNNoiIGj2NU0lMTAz8/KqP7Pn5+eHy5ctaKcoQ/HYqFV0XHcDLP0Si26ID9boqvzEpLqvAx39eBACM7eJT5y2HNTXxqabwd7HCnYJSLPxHv72HfziSiH33WnXtjEmrl+eUj4NyesGey7d19jy+cFOO27klsDI1QtdmDlo5pputuVa7jNRkXFdf2FkYIzGzADvO3x8dTr1TiKvpeZBJJejTylln5yciIsOgcRhu1aoVFi5ciNLS+zuIlZaWYuHChWjVSvNtUg1RmrwIM7bGoHIPEYUAfLQ1BvEZ+fotrAFadiAOKXcK4WpjhukDAur9/CZGUiy413t405nriEy8U+81AMrnVOWcaQAQBGDm1ota65LwqHP+dDxJ9blCR+es7LrwdIATTI0azpQC5eiwsrPE0v1xqq4blaPcnf3s9f5OAhER6Z7GYXjVqlXYvXs3PD090bdvX/Tt2xeenp7YvXs3Vq1apYsa690fUTfw8Jp2QQCeW3oUM7ZewIUbOfooq8G5lp6nmhbw6eA2ddo9rC46+trj5c7KDT1mbItBSXn9L/KMz8iv9pyq7JKgK0lZBXi4OYMuzvlgS7WGZlxXXzSxMEZiVgF23Js7rMkuekRE1PBpHIZDQ0ORmJiIefPmITAwEIGBgZg/fz4SExMRGhqqixrr1eVbuVhxML7Gr5WUK7Dx1HUMXn4czy09il9PpiCvuKyeK2wYFAoBM7fFoFwhoF9rF513M3iSDwe2hKOVKRIzC/DdoYR6P39qdvUAqu0uCQ+r7MzwIIkEWj1nfEY+EjILYCyT4OmAhtd1wcrUCBN7VI4OxyMjtxinU5TbzfdrgOGeiIg0p9YCuodZWlrizTff1HYtepeRV4wJv5xGcZkCzZ0tkZipHFmTSSSYP6wt/BwtsfFUKv65mI5Lt3Lx8Z8XMf/vKxgc5I7Rnb0R5GlrEO27DMGGU8rtkC1NZJg7uI2+y4GtuTHmDGqNKRvPYeXBBAwKckczJ/V3SauryjmpEgACdNcl4UGVnRlmbr2IintzfixNjGBnrr23/iunFHRt5ggbPY3819XYMF/8cCQRSVkFmLopGoIAtPOwhYcdF80SEYmBWmH4r7/+wjPPPANjY2P89ddfj73v4MGDtVJYfSsuq8D/rYvCLXkxmjpaYstb3VBYVl5tRXvnpg6YU1CKreduYuOpVMRn5GPTmevYdOY6WrnZ4OVQLwxp74GCknLRdqLIyC3GF7uUi9Xe6x8AdwMJFc8HumHL2Rs4dC0T038/j/f7B8DPSfc/n9jbeTiZmA2pBNj6dlcUlSl01iXhYZWdGRIy8jH9jwtIlxdjfWSKahe2ulJNKWjAG1NYmRrhzR7N8MWuqziRoJxTHtaU2y8TEYmFRBCER+9Heo9UKkV6ejqcnZ0hlT56ZoVEIjG4TTdyc3Nha2sLuVwOGxubGu8jCAKmborG9uhbsDU3xp+TusHP8ck7lgmCgDMpd7ExMhU7Y9JQWq5cgGMkk6C8QnlZpffaiY3q5K29b8rATdpwFn9fSEOgpy22vdOtTn1nte16diF6f3UIZfX485n950WsO5mCAW1c8P2Y6tsN15dNp1Px4ZYYOFqZ4OgHvevcPzddXowuC/dDIgEiZ/aBs7WZliqtfwUl5Qidvw8FpcrfX2L8f0uNhzp/94joPrXmDCsUCjg7O6v+/agPQwvC6lpxMB7bo2/BSCrBd690UCsIA8rw38nXHktGBePUzD6YM6g1/BwtVEEY0N0KfkN18GoG/r6QBqkEWDCsnUEFYaDqCxVA9z+fvOIybD17AwAwLsxXJ+dQ1/AOnvCyN0dWfil+PZlS5+Ptvdcmrr2XXYMOwgCQW1yGwtL7v7/E9v+WiEjM6mf3AwP2T0wa/rsnFgAwd0gbdG3uWKvj2FmY4LVufpg/tF21r+m6a4ChKCwtV/UUfr2bH9p62Oq5ouqSsgrqtavD1rM3UVBagebOVgjTUg/e2jKWSTGlVwsAwPdHElBYWl6n4zXkLhIPq+/nBRERGQ615gwvXbpU7QO+++67tS6mvsXckCN8czQA4LVuvnils8/jH6AGPyflCv4HW1pJtbyC31B9vTcWN3OK4GFnjmn9/PVdTo0qOyzUx89HEASsuzcCO6aLj0EsrhzWwQPLD8YjNbsQv55MwZs9mtXqOPKiMkTcm1/bGHZpq+l5oetuH0REZBjUCsNff/21WgeTSCQNJgzfzi3GhLXKzhFPBzhh1rPa2TDk/gr+GFS+G9/UyQquNrp/G1lf20enyYtw8GoG1hxTbvDw+dA2sDStVaMSnav8+czYGqMKPgPauOrkekUk3EF8Rj4sTWQY3sFD68evDWOZFJN7N8cHf1zA94cT8WoXH1iYaP6zOnQtA+UKAS2crdSeVmTIHu68UR/dPoiIyDCo9VcwKSnpyXdqQIpKKzDhlzO4nVuCFs5WWDq6PYxk2psxUrmCPzLxDj744wLiM/LxZ/RNDGvvqbVzPGzT6VRVwKvPxT8PnhdQtqTq3dKwOwtU/nx+OZGCVYcTEJmUjYKScq0H+LURylHhYR089LbhSE2Gt/fAioPxSLlTiHURKfi/npqPDld2kWgMo8KVKp8XD3eQISKixk10c4YVCgHv/R6NmJty2FuaYM24Tjrpj+pma46h7T3xn77K6QKf77yCuwWlT3hU7VRuH10ZSJWLf2J0vvjn4fMCwKVb8gax6MjN1hzT+/vD18EC2QWlquCqLbdyilQLzMbqeeHcw4xkUkzpXTl3OBEFJZrNHS4uq8ChaxkAGnZLtZq42ZojrJkDgzARkYjUKgzfuHEDK1euxEcffYTw8PAqH4bum/1x+CcmHcYyCVa9GgJvB93OCZz4VFP4u1ghu6AUC/65opNz1LztLvDDkUSd7ZAXezsPn/51qdp5FQIazKKjB0Ph6iMJyNcwFD7OhshUVCgEdGlqD38Xa60dV1uGBrvX+oXAiYQsFJRWwM3WDO0McJEkERGRJjR+X3j//v0YPHgwmjZtiqtXr6Jt27ZITk6GIAjo0KGDLmrUmu3RN7F0fxwAZduvUD/dN9Y3MZJi4fB2GPFdBH6PuoHhHTy13lXgUTtl/e94snL7aC3tkFdUWoG/Y9Kw8d7ucjVpaIuOhgS7Y/nBeCRlFeCXE8mY1Kt5nY9ZUl6B306nAjC8UeFKlS8E3vv9PFYfScDYMB+1p4nsvnhvo43WLgaxKJCIiKguNB4ZnjFjBqZPn46YmBiYmZlhy5YtuH79Onr27ImRI0fqokatOH/9Lt7/4wIA4P96NMXIjl71du4QH3u80lk5f3fWnzEoKddeP2ZBELD8QHyV26QS5W5rzZ2tUFRWgU1nrmPoiuN45tujWBuRDHmRZqPFV9NzMWf7RYQu2Ifpv59HVMpdyKQSDGjjgte6+aKylXBDXHSkDIXKAPzD0UStjA7vupiOrPxSuNiYol9rw51GMCTYHX6OlrhbWIZfIpLVekyFQsC+K5W7zjWe+cJERCReau1A9yBra2tER0ejWbNmaNKkCY4dO4Y2bdrg/PnzGDJkCJKTk3VUau1U7sTTbsY25CqM0beVC74fE1Lvm0HIi8rQd8lhZOaV4D99Wmit9dj3hxOw8N+rkEqAr14MhquNmWrxz6N2yDMzluL5QHeMDvVGB287SCSSap0oCkvLsfOCchT4XGqO6nxe9uZ4qZM3RoZ4wvleh4w0eVGDXnRUXqFA/6+PIDGrAO8PCKjz6PCI704gKuUupvX1x3/6ttBSlbqx9ewNhG8+DzsLYxz7sDesnjA6fDo5GyNXRcDW3BhnPu4LYy0uPCUi7eAOdESa0XiahKWlJUpLlQvB3NzckJCQgDZt2gAAsrKytFudFuUUlcHdyQrfvhSsl13RbM2NMWdQa0zecA7fHUrAoCB3NHe2qtMx916+jUW7rgIAPnm+NYa1r9q+q3KHvE6+9vhkUGtsO3cTG0+lIvZ2Pv6IuoE/om4gwMUaAa5W2HkhDQoBkEiAzn72uHQzF3n3RkmNpBL0b+OC0aHe6NbMEdKHrp+brXmDDMGVjGRSvNunBaZuisYPRxMxNsyn1t0fLt6UIyrlLoykEowOrb93H2prcJA7lh+IR6Ka00R2X1RutNGnpTODMBERNQoa/zXr0qULjh07BgB49tln8d5772H+/Pl4/fXX0aVLF60XqE0ZeSXI1dGCMnU8184NvQKcUFqhwMxtMVA8vPpMA1fScvGf385BEIBXu3hjXFffx96/coe83VN7YMvbYRjRwROmRlJcu52Hv86nqRbCCQJwMjEbeSXl8HGwwIcDWyJiRh+sfCUET7VwqhaEG4tBQe5o6mSJnMIy/HIiudbHWXdvMdrAtq6qkXNDZiSTYkqf+9NEHrfgUhAE7LlcOUXCcKd/EBERaULjMLxkyRJ07twZADB37lz06dMHmzZtgq+vL9asWaP1ArVJ350OJBIJPhvSFubGMpxKysYfUTdqdZzMvBJM+OUMCksr0K25A+YMaqP2QiaJRIIQH3t89WIQTs3qi/GPCNGznm2Fg+89jbefbgYna9Na1dmQyKQS/KePckrDD0eTavWiSV5Yhu3nbwLAE1+cGJLBQR5qvRC4mp6H1OxCmBpJ0cPfqf4KJCIi0iGNw3DTpk0RGBgIQDllYtWqVbhw4QK2bNkCH5+6b2esS4bQ6cDL3gLh9+YLz//nCrLySzR6fHFZBd5cdwY3c4rQ1NESK18OqfXb1bbmxvi/nk3x8GCvTCLB80FujXYU+FGeD3RHMydLyIvK8MvxZI0f/3vUdRSXKdDS1RodfZpov0AdefiFwKNGhys32niqhVOtdq0jIiIyRBqnqAkTJuDQoUM6KEW3DKnTwWvdfNHazQbyojLM23lZ7ccJgoAPt1zAudQc2JobY834TrC1qNuGIZXb0MrujSwb0nWqbzKpBO+qQmGiRqPDCoWAdSeVUyTGhvk2uJZjD74Q+PkRLwT2XFbOF+YUCSIiakw0DsOZmZkYOHAgvLy88P777+P8+fO1PvmRI0cwaNAguLu7QyKR4M8//6zydUEQ8Mknn8DNzQ3m5ubo27cv4uLianWu3dOeqpftidVhJFP2HpZKgD+jb+FoXKZaj1txMB7bo2/BSCrBd690gJ+jpVbqGdXJG8c+6oWNE7vg2Ee9DOY66cPzgcqFjbnF5fjpWLLajzsSl4mUO4WwNjPC0PbuuitQRx58IfDjserTRK5nF+LSrVxIJUDfVgzDRETUeGgchrdv3460tDTMnj0bp0+fRocOHdCmTRssWLBA47ZqBQUFCAoKwooVK2r8+uLFi7F06VKsWrUKkZGRsLS0xIABA1BcXKxp2XA1sJHOIC871YYMs7ZdRFHp43sP/xuThv/uiQUAzB3SBl2bO2q1Hm5Dq/TglIE1xxLV7slcuXDuhRDPBjuFoPKFQE2jw3vvLZzr5GsPe0sTPVRHRESkG7WabNqkSRO8+eabOHToEFJSUjB+/HisW7cOzZtr1p/1mWeewbx58zBs2LBqXxMEAd988w0+/vhjDBkyBIGBgVi7di1u3bpVbQS5oZo+IAButmZIzS7E0gOPHvGOuSHHtM3RAJRTLF7pbNhzsxu6Z9u5oUXl6PDxpCfe/3p2IQ5cywAAjOnScH82VUaHj1Z9IVA5RWIAN9ogIqJGpk6NQsvKynDmzBlERkYiOTkZLi7ae/s0KSkJ6enp6Nu3r+o2W1tbdO7cGREREVo7jz5ZmRph7mBlj+YfjiTianputfvczi3GhLWnUVymwNMBTpj1bKv6LlN0ZFKJarOMNceSnjg6/OvJFAgC8FQLRzR1qlvvaH177oEXApWjw9kFpTiVlA0ABr2jHhERUW3U6v3cgwcPYsOGDdiyZQsUCgWGDx+OnTt3onfv3lorLD1dORL1cMB2cXFRfa0mJSUlKCm536EhN1cZMMvKylBWpr8ew4/Sy98B/Vo5Y++VDMzYcgG/TQhVdXEoKq3AhF9O43ZuCZo7WWLJC20hKCpQptDeds5Us34BjmjhbIm4jAL8eCQe7/au+V2P4rIKbDp9HQDwSidPg3yOaWry003xn80X8OOxRLwa6oG9VzKgEIBWrtZwtTZuFN8jUWPG/6NEmtE4DHt4eCA7OxsDBw7E6tWrMWjQIJiaGk4f2oULF2Lu3LnVbt+zZw8sLPTbVu1RupsDR2QynLsux8c/70J3VwEKAfglToqYO1JYGgkY7SnH0QN79V2qqHSzkyAuQ4YfjiTAPT8WFjX8b4nMkCCnSAZ7UwFFiWfwz5NnVRg8hQC4msuQXlSOWWv340YBAEjha5SDf/75R9/lEdETFBbqr58+UUOkcRj+9NNPMXLkSNjZ2emgnPtcXZVzE2/fvg03NzfV7bdv30ZwcPAjHzdjxgyEh4erPs/NzYWXlxf69+9v0Hu0l7ul4vO/r+LfW6YYPSAEayNSEX0nDcYyCX4Y1wmdfBtO39rGYqBCwIkVEYjNyMdNS3/8p0/V0WFBEPDDqkgAuXi9hz+e7+Gnn0J1QOaTjnc3XcCRDGOUlSsACHhnSHe0dLXWd2lE9ASV74gSkXo0DsMTJ07URR3V+Pn5wdXVFfv371eF39zcXERGRuLtt99+5ONMTU1rHKk2NjaGsXHdevLq0vhuTfHX+TScvyHH8FWRqtuHBnugawtnPVYmblP7+eOd9WfxS0QqJvZoXqWv87nUu7h4KxcmMilGd/Yx6OeXpp4P8sT8f6/hdu79KUeX0vLRzstej1URkToa0+8iovpQpwV0dZWfn4/o6GhER0cDUC6ai46ORmpqKiQSCaZOnYp58+bhr7/+QkxMDMaOHQt3d3cMHTpUn2XrhEwqUe1M96CtZ28iTV6kh4oIAAa2cUVLV2vklZTjx2OJVb5W2U7t+SA3OFgZzlQhbbidV4yM3Kq7I87adpHPRSIianT0GobPnDmD9u3bo3379gCA8PBwtG/fHp988gkA4IMPPsCUKVPw5ptvolOnTsjPz8euXbtgZmamz7J1xtio+o+jQhCQnMX5X/oilUow9V5niZ+OJyOnsBQAcCe/BDsvpAGAql90Y5KUVQDhodv4XCQiosZIr7sDPP300xCEh//k3ieRSPDZZ5/hs88+q8eq9MfP0RJSiXIBUyWZRAJfR8Nc+CcW/VsrR4evpufhx6NJmD4gAJvOXEdphQKBnrYI9rLTd4lax+ciERGJhV5HhqkqN1tzLBzeDjKJsrWaTCLBguFtRb8rnL4pR4eVU1h+Op6ErPwSrD+ZCqBhb7LxOHwuEhGRWEiExw3NNgK5ubmwtbWFXC436G4SD0qTFyE5qxC+jhYMHwZCEAQ8t/QYLqflorWbDS6n5cLGzAinZvWFmbFM3+XpDJ+LRA1PQ/y7R6RPHBk2QG625ghr5sDwYUAkkvu70l1OU7Ytyisux/bom/osS+f4XCQiosaOYZhITe08qo6wCABmbmWHBSIiooaMYZhITcl3qndSYIcFIiKiho1hmEhNlR0WHsQOC0RERA0bwzCRmthhgYiIqPHRa59hooZmVCdv9PB3YocFIiKiRoJhmEhDbrbmDMFERESNBKdJEBEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWgxDBMRERGRaDEMExEREZFoMQwTERERkWg1iDC8YsUK+Pr6wszMDJ07d8apU6f0XRIRERERNQIGH4Y3bdqE8PBwzJkzB2fPnkVQUBAGDBiAjIwMfZdGRERERA2cwYfhJUuWYOLEiXjttdfQunVrrFq1ChYWFvjf//6n79KIiIiIqIEz0ncBj1NaWoqoqCjMmDFDdZtUKkXfvn0RERFR42NKSkpQUlKi+lwulwMAsrOzUVZWptuCiYiI9CwvLw8AIAiCnishahgMOgxnZWWhoqICLi4uVW53cXHB1atXa3zMwoULMXfu3Gq3+/n56aRGIiIiQ3Tnzh3Y2trquwwig2fQYbg2ZsyYgfDwcNXnOTk58PHxQWpqKn8pPEZubi68vLxw/fp12NjY6Lscg8ZrpR5eJ/XxWqmH10k9crkc3t7esLe313cpRA2CQYdhR0dHyGQy3L59u8rtt2/fhqura42PMTU1hampabXbbW1t+ctTDTY2NrxOauK1Ug+vk/p4rdTD66QeqdTglwURGQSD/p9iYmKCkJAQ7N+/X3WbQqHA/v37ERYWpsfKiIiIiKgxMOiRYQAIDw/HuHHj0LFjR4SGhuKbb75BQUEBXnvtNX2XRkREREQNnMGH4VGjRiEzMxOffPIJ0tPTERwcjF27dlVbVPcopqammDNnTo1TJ+g+Xif18Vqph9dJfbxW6uF1Ug+vE5FmJAJ7rxARERGRSBn0nGEiIiIiIl1iGCYiIiIi0WIYJiIiIiLRYhgmIiIiItFq1GF4xYoV8PX1hZmZGTp37oxTp07puySD8+mnn0IikVT5aNmypb7L0rsjR45g0KBBcHd3h0QiwZ9//lnl64Ig4JNPPoGbmxvMzc3Rt29fxMXF6adYPXvStRo/fny159jAgQP1U6weLVy4EJ06dYK1tTWcnZ0xdOhQXLt2rcp9iouLMWnSJDg4OMDKygojRoyotulQY6fOdXr66aerPafeeustPVWsP9999x0CAwNVm5CEhYXh33//VX2dzyci9TTaMLxp0yaEh4djzpw5OHv2LIKCgjBgwABkZGTouzSD06ZNG6Slpak+jh07pu+S9K6goABBQUFYsWJFjV9fvHgxli5dilWrViEyMhKWlpYYMGAAiouL67lS/XvStQKAgQMHVnmObdy4sR4rNAyHDx/GpEmTcPLkSezduxdlZWXo378/CgoKVPeZNm0aduzYgd9//x2HDx/GrVu3MHz4cD1WXf/UuU4AMHHixCrPqcWLF+upYv3x9PTEokWLEBUVhTNnzqB3794YMmQILl26BIDPJyK1CY1UaGioMGnSJNXnFRUVgru7u7Bw4UI9VmV45syZIwQFBem7DIMGQNi2bZvqc4VCIbi6ugpffvml6racnBzB1NRU2Lhxox4qNBwPXytBEIRx48YJQ4YM0Us9hiwjI0MAIBw+fFgQBOVzyNjYWPj9999V97ly5YoAQIiIiNBXmXr38HUSBEHo2bOn8J///Ed/RRmwJk2aCD/++COfT0QaaJQjw6WlpYiKikLfvn1Vt0mlUvTt2xcRERF6rMwwxcXFwd3dHU2bNsUrr7yC1NRUfZdk0JKSkpCenl7l+WVra4vOnTvz+fUIhw4dgrOzMwICAvD222/jzp07+i5J7+RyOQDA3t4eABAVFYWysrIqz6uWLVvC29tb1M+rh69TpfXr18PR0RFt27bFjBkzUFhYqI/yDEZFRQV+++03FBQUICwsjM8nIg0Y/A50tZGVlYWKiopqu9S5uLjg6tWreqrKMHXu3Bk///wzAgICkJaWhrlz5+Kpp57CxYsXYW1tre/yDFJ6ejoA1Pj8qvwa3Tdw4EAMHz4cfn5+SEhIwMyZM/HMM88gIiICMplM3+XphUKhwNSpU9GtWze0bdsWgPJ5ZWJiAjs7uyr3FfPzqqbrBAAvv/wyfHx84O7ujgsXLuDDDz/EtWvXsHXrVj1Wqx8xMTEICwtDcXExrKyssG3bNrRu3RrR0dF8PhGpqVGGYVLfM888o/p3YGAgOnfuDB8fH2zevBlvvPGGHiujxuKll15S/btdu3YIDAxEs2bNcOjQIfTp00ePlenPpEmTcPHiRc7Pf4JHXac333xT9e927drBzc0Nffr0QUJCApo1a1bfZepVQEAAoqOjIZfL8ccff2DcuHE4fPiwvssialAa5TQJR0dHyGSyaqtmb9++DVdXVz1V1TDY2dnB398f8fHx+i7FYFU+h/j8qp2mTZvC0dFRtM+xyZMnY+fOnTh48CA8PT1Vt7u6uqK0tBQ5OTlV7i/W59WjrlNNOnfuDACifE6ZmJigefPmCAkJwcKFCxEUFIRvv/2WzyciDTTKMGxiYoKQkBDs379fdZtCocD+/fsRFhamx8oMX35+PhISEuDm5qbvUgyWn58fXF1dqzy/cnNzERkZyeeXGm7cuIE7d+6I7jkmCAImT56Mbdu24cCBA/Dz86vy9ZCQEBgbG1d5Xl27dg2pqamiel496TrVJDo6GgBE95yqiUKhQElJCZ9PRBpotNMkwsPDMW7cOHTs2BGhoaH45ptvUFBQgNdee03fpRmU6dOnY9CgQfDx8cGtW7cwZ84cyGQyjB49Wt+l6VV+fn6VUaakpCRER0fD3t4e3t7emDp1KubNm4cWLVrAz88Ps2fPhru7O4YOHaq/ovXkcdfK3t4ec+fOxYgRI+Dq6oqEhAR88MEHaN68OQYMGKDHquvfpEmTsGHDBmzfvh3W1taqeZu2trYwNzeHra0t3njjDYSHh8Pe3h42NjaYMmUKwsLC0KVLFz1XX3+edJ0SEhKwYcMGPPvss3BwcMCFCxcwbdo09OjRA4GBgXquvn7NmDEDzzzzDLy9vZGXl4cNGzbg0KFD2L17N59PRJrQdzsLXVq2bJng7e0tmJiYCKGhocLJkyf1XZLBGTVqlODm5iaYmJgIHh4ewqhRo4T4+Hh9l6V3Bw8eFABU+xg3bpwgCMr2arNnzxZcXFwEU1NToU+fPsK1a9f0W7SePO5aFRYWCv379xecnJwEY2NjwcfHR5g4caKQnp6u77LrXU3XCIDw008/qe5TVFQkvPPOO0KTJk0ECwsLYdiwYUJaWpr+itaDJ12n1NRUoUePHoK9vb1gamoqNG/eXHj//fcFuVyu38L14PXXXxd8fHwEExMTwcnJSejTp4+wZ88e1df5fCJSj0QQBKE+wzcRERERkaFolHOGiYiIiIjUwTBMRERERKLFMExEREREosUwTERERESixTBMRERERKLFMExEREREosUwTERERESixTBMRAbl0KFDkEgkyMnJ0XcpREQkAgzDRERERCRaDMNEREREJFoMw0RUhUKhwMKFC+Hn5wdzc3MEBQXhjz/+AHB/CsPff/+NwMBAmJmZoUuXLrh48WKVY2zZsgVt2rSBqakpfH198dVXX1X5eklJCT788EN4eXnB1NQUzZs3x5o1a6rcJyoqCh07doSFhQW6du2Ka9eu6fYbJyIiUWIYJqIqFi5ciLVr12LVqlW4dOkSpk2bhldffRWHDx9W3ef999/HV199hdOnT8PJyQmDBg1CWVkZAGWIffHFF/HSSy8hJiYGn376KWbPno2ff/5Z9fixY8di48aNWLp0Ka5cuYLvv/8eVlZWVeqYNWsWvvrqK5w5cwZGRkZ4/fXX6+X7JyIicZEIgiDouwgiMgwlJSWwt7fHvn37EBYWprp9woQJKCwsxJtvvolevXrht99+w6hRowAA2dnZ8PT0xM8//4wXX3wRr7zyCjIzM7Fnzx7V4z/44AP8/fffuHTpEmJjYxEQEIC9e/eib9++1Wo4dOgQevXqhX379qFPnz4AgH/++QfPPfccioqKYGZmpuOrQEREYsKRYSJSiY+PR2FhIfr16wcrKyvVx9q1a5GQkKC634NB2d7eHgEBAbhy5QoA4MqVK+jWrVuV43br1g1xcXGoqKhAdHQ0ZDIZevbs+dhaAgMDVf92c3MDAGRkZNT5eyQiInqQkb4LICLDkZ+fDwD4+++/4eHhUeVrpqamVQJxbZmbm6t1P2NjY9W/JRIJAOV8ZiIiIm3iyDARqbRu3RqmpqZITU1F8+bNq3x4eXmp7nfy5EnVv+/evYvY2Fi0atUKANCqVSscP368ynGPHz8Of39/yGQytGvXDgqFosocZCIiIn3hyDARqVhbW2P69OmYNm0aFAoFunfvDrlcjuPHj8PGxgY+Pj4AgM8++wwODg5wcXHBrFmz4OjoiKFDhwIA3nvvPXTq1Amff/45Ro0ahYiICCxfvhwrV64EAPj6+mLcuHF4/fXXsXTpUgQFBSElJQUZGRl48cUX9fWtExGRSDEME1EVn3/+OZycnLBw4UIkJibCzs4OHTp0wMyZM1XTFBYtWoT//Oc/iIuLQ3BwMHbs2AETExMAQIcOHbB582Z88skn+Pzzz+Hm5obPPvsM48ePV53ju+++w8yZM/HOO+/gzp078Pb2xsyZM/Xx7RIRkcixmwQRqa2y08Pdu3dhZ2en73KIiIjqjHOGiYiIiEi0GIaJiIiISLQ4TYKIiIiIRIsjw0REREQkWgzDRERERCRaDMNEREREJFoMw0REREQkWgzDRERERCRaDMNEREREJFoMw0REREQkWgzDRERERCRaDMNEREREJFr/D8QavWgHj7xaAAAAAElFTkSuQmCC", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "fig, ax = plt.subplots()\n", - "\n", - "key = 'ResSCNN1'\n", - "max_epochs = 30\n", - "max_y = 30\n", - "\n", - "_ = np.round(architectures_TM[key]['epochs_acc'][max_y-1], 2)\n", - "ax.plot(np.arange(len(architectures_TM[key]['epochs_x'][:max_epochs])), architectures_TM[key]['epochs_acc'][:max_epochs], label=f'{key} ({_}%)', marker='.')\n", - "\n", - "ax.set_ylim(0, max_y)\n", - "ax.set_yticks(np.arange(0, max_y+10, 10))\n", - "ax.set_ylabel('validation acc [%]')\n", - "ax.set_xlim(0, max_epochs)\n", - "ax.set_xlabel('epoch')\n", - "\n", - "pos = ax.get_position()\n", - "ax.set_position([pos.x0, pos.y0, pos.width * 0.9, pos.height])\n", - "ax.legend(loc='center right', bbox_to_anchor=(1.45, 0.5), framealpha=0)\n", - "ax.grid(axis='y')\n", - "\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAqwAAAG2CAYAAAC6Z9RQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAAA2VElEQVR4nO3deXhU5d3/8c/MZDLZyIQkQBIIENlkR9lEqEhBQa0F8VFUtFGrPlZcAIUKFhBRg7Qil0pF/Vmpj4paFahSLYgKVRYRjAKyCwYkLAGSCVknmfP7IzA1JkASQubO5P26rrkyc+acme8cJ5yPd77nPjbLsiwBAAAAhrIHugAAAADgdAisAAAAMBqBFQAAAEYjsAIAAMBoBFYAAAAYjcAKAAAAoxFYAQAAYDQCKwAAAIxGYAUAAIDRCKwAAAAwWkAD68qVK3X11VcrKSlJNptNixYtKve8ZVmaOnWqEhMTFR4eriFDhmjHjh2BKRYAAAABEdDAmpeXp+7du2vu3LmVPj9r1iw9++yzmjdvntauXavIyEgNHTpUhYWFdVwpAAAAAsVmWZYV6CIkyWazaeHChRoxYoSkstHVpKQkPfjgg3rooYckSTk5OWrWrJnmz5+vG264IYDVAgAAoK6EBLqAU9m9e7cOHDigIUOG+Je53W717dtXq1evPmVgLSoqUlFRkf+xz+fT0aNHFRcXJ5vNds7rBgAgkCzLUm5urpKSkmS3c6oKgoOxgfXAgQOSpGbNmpVb3qxZM/9zlUlLS9P06dPPaW0AAJhu7969atGiRaDLAGqFsYG1piZNmqTx48f7H+fk5Khly5bavXu3GjVqFMDKAAA493Jzc5WSksIxD0HF2MCakJAgSTp48KASExP9yw8ePKgePXqccjuXyyWXy1VheWxsrKKjo2u9TgAATOJ0OiWJNjgEFWObW1JSUpSQkKDly5f7l3k8Hq1du1b9+vULYGUAAACoSwEdYT1+/Lh27tzpf7x7926lp6crNjZWLVu21NixY/X444+rXbt2SklJ0ZQpU5SUlOSfSQAAAADBL6CB9euvv9agQYP8j0/2nqampmr+/PmaOHGi8vLydNdddyk7O1sDBgzQxx9/rLCwsECVDAAAgDpmzDys54rH45Hb7VZOTg49rACAoMdxD8HI2B5WAAAAQCKwAgAAwHAEVgAAABiNwAoAAACjEVgBAABgNAIrAAAAjEZgBQAAgNEIrAAAADAagRUAAABGI7ACAADAaARWAAAAGI3ACgAAAKMRWAEAAGA0AisAAACMRmAFAACA0QisAAAAMBqBFQAAAEYjsAIAAMBoBFYAAAAYjcAKAAAAoxFYAQAAYDQCKwAAAIxGYAUAAIDRCKwAAAAwGoEVAAAARiOwAgAAwGgEVgAAABiNwAoAAACjEVgBAABgNAIrAAAAjEZgBQAAgNEIrAAAADAagRUAAABGI7ACAADAaARWAAAAGI3ACgAAAKMRWAEAAGA0AisAAACMRmAFAACA0QisAAAAMBqBFQAAAEYjsAIAAMBoBFYAAAAYjcAKAAAAoxFYAQAAYDQCKwAAAIxGYAUAAIDRCKwAAAAwGoEVAAAARiOwAgAAwGgEVgAAABiNwAoAAACjEVgBAABgNAIrAAAAjEZgBQAAgNEIrAAAADAagRUAAABGI7ACAADAaARWAAAAGI3ACgAAAKMRWAEAAGA0AisAAACMRmAFAACA0YwOrKWlpZoyZYpSUlIUHh6uNm3aaMaMGbIsK9ClAQAAoI6EBLqA03nqqaf0wgsv6O9//7s6d+6sr7/+Wrfddpvcbrfuv//+QJcHAACAOmB0YF21apWGDx+uq666SpLUunVrLViwQF999VWAKwMAAEBdMTqwXnzxxXrppZe0fft2tW/fXt9++62++OILzZ49+5TbFBUVqaioyP/Y4/FIkrxer7xe7zmvGQCAQOJYh2BkdGB9+OGH5fF4dP7558vhcKi0tFRPPPGERo8efcpt0tLSNH369ArLly5dqoiIiHNZLgAAAZefnx/oEoBaZ7MMPoPprbfe0oQJE/TnP/9ZnTt3Vnp6usaOHavZs2crNTW10m0qG2FNTk5WVlaWoqOj66p0AAACwuPxKD4+Xjk5ORz3EDSMDqzJycl6+OGHNWbMGP+yxx9/XK+//rq2bt1apdfweDxyu9384gIAGgSOewhGRk9rlZ+fL7u9fIkOh0M+ny9AFQEAAKCuGd3DevXVV+uJJ55Qy5Yt1blzZ33zzTeaPXu2br/99kCXBgAAgDpidEtAbm6upkyZooULF+rQoUNKSkrSjTfeqKlTpyo0NLRKr8GfRgAADQnHPQQjowNrbeAXFwDQkHDcQzAyuocVAAAAILACAADAaARWAAAAGI3ACgAAAKMRWAEAAGA0AisAAACMRmAFAACA0QisAAAAMBqBFQAAAEYjsAIAAMBoBFYAAAAYjcAKAAAAoxFYAQAAYDQCKwAAAIxGYAUAAIDRCKwAAAAwGoEVAAAARiOwAgAAwGgEVgAAABiNwAoAAACjEVgBAABgNAIrAAAAjEZgBQAAgNEIrAAAADAagRUAAABGI7ACAADAaARWAAAAGI3ACgAAAKMRWAEAAGA0AisAAACMRmAFAACA0QisAAAAMBqBFQAAAEYjsAIAAMBoBFYAAAAYjcAKAAAAoxFYAQAAYDQCKwAAAIxGYAUAAIDRCKwAAAAwGoEVAAAARiOwAgAAwGgEVgAAABiNwAoAAACjEVgBAABgNAIrAAAAjEZgBQAAgNEIrAAAADAagRUAAABGI7ACAADAaARWAAAAGI3ACgAAAKMRWAEAAGA0AisAAACMRmAFAACA0QisAAAAMBqBFQAAAEYjsAIAAMBoBFYAAAAYjcAKAAAAoxFYAQAAYDQCKwAAAIxGYAUAAIDRjA+sP/30k26++WbFxcUpPDxcXbt21ddffx3osgAAAFBHQgJdwOkcO3ZM/fv316BBg/TRRx+pSZMm2rFjhxo3bhzo0gAAAFBHjA6sTz31lJKTk/Xqq6/6l6WkpASwIgAAANQ1owPrP//5Tw0dOlTXXXedVqxYoebNm+uee+7RnXfeecptioqKVFRU5H/s8XgkSV6vV16v95zXDABAIHGsQzAyOrD+8MMPeuGFFzR+/HhNnjxZ69at0/3336/Q0FClpqZWuk1aWpqmT59eYfnSpUsVERFxrksGACCg8vPzA10CUOtslmVZgS7iVEJDQ9WrVy+tWrXKv+z+++/XunXrtHr16kq3qWyENTk5WVlZWYqOjj7nNQMAEEgej0fx8fHKycnhuIegYfQIa2Jiojp16lRuWceOHfXee++dchuXyyWXy1VhudPplNPprPUaAQAwCcc6BCOjp7Xq37+/tm3bVm7Z9u3b1apVqwBVBAAAgLpmdGAdN26c1qxZoyeffFI7d+7Um2++qZdeekljxowJdGkAAACoI0YH1t69e2vhwoVasGCBunTpohkzZmjOnDkaPXp0oEsDAABAHTH6pKva4PF45Ha7aT4HADQIHPcQjIweYQUAAAAIrAAAADAagRUAAABGI7ACAADAaARWAAAAGI3ACgAAAKMRWAEAAGA0AisAAACMRmAFAACA0QisAAAAMBqBFQAAAEYjsAIAAMBoBFYAAAAYjcAKAAAAoxFYAQAAYDQCKwAAAIxGYAUAAIDRCKwAAAAwGoEVAAAARiOwAgAAwGgEVgAAABitRoH173//u5YsWeJ/PHHiRMXExOjiiy/Wjz/+WGvFAQAAADUKrE8++aTCw8MlSatXr9bcuXM1a9YsxcfHa9y4cbVaIAAAABq2kJpstHfvXrVt21aStGjRIl177bW666671L9/f1166aW1WR8AAAAauBqNsEZFRenIkSOSpKVLl+qyyy6TJIWFhamgoKD2qgMAAKhHbr31VtlsNtlsNjmdTqWkpGjixIkqLCyslddfsWKFfv3rXys2NlYRERFq166dUlNTVVxc7F/Hsiy99NJL6tu3r6KiohQTE6NevXppzpw5ys/PlyQ9+uijstlsuvvuu8u9fnp6umw2m/bs2SNJ2rNnj2w2m5o2barc3Nxy6/bo0UOPPvqo//H777+vyy+/XHFxcbLZbEpPT6+VzyzVMLBedtlluuOOO3THHXdo+/btuvLKKyVJmzdvVuvWrWutOAAAgPpm2LBhyszM1A8//KBnnnlGL774oqZNm3bWr/v9999r2LBh6tWrl1auXKmNGzfqueeeU2hoqEpLS/3r3XLLLRo7dqyGDx+uzz77TOnp6ZoyZYoWL16spUuX+tcLCwvTK6+8oh07dpzxvXNzc/WXv/zltOvk5eVpwIABeuqpp2r+IU+hRoF17ty56tevnw4fPqz33ntPcXFxkqT169frxhtvrNUCAQAAzkZmToFW7cpSZk7d/BXY5XIpISFBycnJGjFihIYMGaJly5ZJknw+n9LS0pSSkqLw8HB1795d7777rn/bY8eOafTo0WrSpInCw8PVrl07vfrqq5LK/qqdkJCgWbNmqUuXLmrTpo2GDRuml19+2X9u0TvvvKM33nhDCxYs0OTJk9W7d2+1bt1aw4cP16effqpBgwb536tDhw4aNGiQHnnkkTN+pvvuu0+zZ8/WoUOHTrnOLbfcoqlTp2rIkCE12m+nU6Me1piYGD3//PMVlk+fPv2sCwIAAPgly7JU4C0984q/8N76fZr2z83yWZLdJk3/bWdd27NFtV4j3OmQzWar9ntL0qZNm7Rq1Sq1atVKkpSWlqbXX39d8+bNU7t27bRy5UrdfPPNatKkiQYOHKgpU6bo+++/10cffaT4+Hjt3LnT326ZkJCgzMxMrVy5Updcckml7/fGG2+oQ4cOGj58eIXnbDab3G53uWUzZ85U79699fXXX6tXr16n/Bw33nijli1bpscee6zSDHiu1Siwfvzxx4qKitKAAQMklY24vvzyy+rUqZPmzp2rxo0b12qRAACgYSvwlqrT1H+f1Wv4LGnK4s2asnhztbb7/rGhigitemT68MMPFRUVpZKSEhUVFclut+v5559XUVGRnnzySX3yySfq16+fJOm8887TF198oRdffFEDBw5URkaGLrjgAn94/Hmr5XXXXad///vfGjhwoBISEnTRRRdp8ODB+t3vfqfo6GhJ0o4dO9ShQ4cq13rhhRfq+uuv1x//+EctX778lOvZbDbNnDlTV199tcaNG6c2bdpU+T1qQ41aAiZMmCCPxyNJ2rhxox588EFdeeWV2r17t8aPH1+rBQIAANQngwYNUnp6utauXavU1FTddtttuvbaa7Vz507l5+frsssuU1RUlP/22muvadeuXZKkP/zhD3rrrbfUo0cPTZw4UatWrfK/rsPh0Kuvvqp9+/Zp1qxZat68uZ588kl17txZmZmZkspGoqvr8ccf13/+859y/a2VGTp0qAYMGKApU6ZU+z3OVo1GWHfv3q1OnTpJkt577z395je/0ZNPPqkNGzb4T8ACAACoLeFOh75/bGi1tjmQU6ghs1fI97MMZ7dJn4wfqAR3WLXeuzoiIyP903/+7W9/U/fu3fXKK6+oS5cukqQlS5aoefPm5bZxuVySpCuuuEI//vij/vWvf2nZsmUaPHiwxowZU+6Ep+bNm+uWW27RLbfcohkzZqh9+/aaN2+epk+frvbt22vr1q3VqrdNmza688479fDDD+uVV1457bozZ85Uv379NGHChGq9x9mq0QhraGiof1qETz75RJdffrkkKTY21j/yCgAAUFtsNpsiQkOqdTuvSZTSRnaV40T/qcNmU9rIrjqvSVS1Xqem/auSZLfbNXnyZP3pT39Sp06d5HK5lJGRobZt25a7JScn+7dp0qSJUlNT9frrr2vOnDl66aWXTvn6jRs3VmJiovLy8iRJN910k7Zv367FixdXWNeyLOXk5FT6OlOnTtX27dv11ltvnfbz9OnTRyNHjtTDDz9clY9fa2o0wjpgwACNHz9e/fv311dffaW3335bkrR9+3a1aFG9RmYAAIBzZVTvlrqkfRPtycpX6/gIJbrD67yG6667ThMmTNCLL76ohx56SOPGjZPP59OAAQOUk5OjL7/8UtHR0UpNTdXUqVPVs2dPde7cWUVFRfrwww/VsWNHSdKLL76o9PR0XXPNNWrTpo0KCwv12muvafPmzXruueckSddff70WLlyoG2+8UX/60590+eWXq0mTJtq4caOeeeYZ3XfffRoxYkSFGps1a6bx48frz3/+8xk/zxNPPKHOnTsrJKR8jDx69KgyMjK0f/9+SdK2bdsklZ0slpCQcDa7sGYjrM8//7xCQkL07rvv6oUXXvAPa3/00UcaNmzYWRUEAABQmxLd4erXJi4gYVWSQkJCdO+992rWrFmaNGmSpkyZorS0NHXs2FHDhg3TkiVLlJKSIqnsr9iTJk1St27ddMkll8jhcPhHPfv06aPjx4/r7rvvVufOnTVw4ECtWbNGixYt0sCBAyWVjUS/+eabmj17tn95t27d9Oijj2r48OEaOvTUbRUPPfSQoqKizvh52rdvr9tvv73CxRD++c9/6oILLtBVV10lSbrhhht0wQUXaN68eTXabz9ns2rSnVuPeDweud1u5eTk+M+gAwAgWHHcQzCqUUuAJJWWlmrRokXasmWLJKlz58767W9/K4ejeo3JAAAAwOnUKLDu3LlTV155pX766Sf/XF9paWlKTk7WkiVL6nxuLgAAAASvGvWw3n///WrTpo327t2rDRs2aMOGDcrIyFBKSoruv//+2q4RAAAADViNRlhXrFihNWvWKDY21r8sLi5OM2fOVP/+/WutOAAAAKBGI6wul0u5ubkVlh8/flyhoaFnXRQAAABwUo0C629+8xvdddddWrt2rSzLkmVZWrNmje6++2799re/re0aAQAA0IDVKLA+++yzatOmjfr166ewsDCFhYXp4osvVtu2bTVnzpxaLhEAAAANWY16WGNiYrR48WLt3LnTP61Vx44d/dfNBQAAAGpLlQPr+PHjT/v8Z5995r8/e/bsmlcEAAAA/EyVA+s333xTpfVsNluNiwEAAAB+qcqB9ecjqAAAAEBdqdFJVwAAAEBdIbACAADAaARWAAAAGI3ACgAAAKMRWAEAAGA0AisAAACMRmAFAACA0QisAAAAMFqDCawHcgoCXQIAAABqoMEE1sufWam312UEugwAAABUU4MJrD5Lmvz+JmUy0goAAFCvNJjAKkmllqU9WfmBLgMAAADV0KACq90mtY6PCHQZAAAAqIYGFVjDnQ75rEBXAQAAgOpoMIG1bdNI5RWX6n//72sVeksDXQ4AAACqqMEE1rk3XajYyFBt+smjye9vlGUx1AoAAFAfNJjA2rxxhJ6/6QI57Da9/81PevXLPYEuCQAAAFVQrwLrzJkzZbPZNHbs2Bptf3GbeD1yZUdJ0hP/2qJVu7JqsToAAACcC/UmsK5bt04vvviiunXrdlavc1v/1hp5YXOV+izd++Y32neMaa4AAABMVi8C6/HjxzV69Gi9/PLLaty48Vm9ls1m05PXdFXX5m4dzSvW//7fehUUcxIWAACAqUICXUBVjBkzRldddZWGDBmixx9//LTrFhUVqaioyP/Y4/FIkrxer7xeryTJIen5G7rpmnlrtHm/RxPfTdfT/9NVNpvtnH0GAADqwsljHRBMjA+sb731ljZs2KB169ZVaf20tDRNnz69wvKlS5cqIqL8RQNGt5LmbnHog+8OyJHzkwYlMXMAAKB+y8+n1Q3Bx2YZPL/T3r171atXLy1btszfu3rppZeqR48emjNnTqXbVDbCmpycrKysLEVHR1dY/7U1GZqxZKvsNunV1J66uE3cOfksAADUBY/Ho/j4eOXk5FR63APqI6MD66JFi3TNNdfI4XD4l5WWlspms8lut6uoqKjcc5XxeDxyu92n/MW1LEsT3v1O767fp8YRTv3z3gFKjuXyrQCA+ulMxz2gPjL6pKvBgwdr48aNSk9P99969eql0aNHKz09/YxhtSpsNpseH9FF3Vu4dSzfq7s4CQsAAMAoRgfWRo0aqUuXLuVukZGRiouLU5cuXWrtfcKcDs27pafio0K1JdOjie99x5WwAAAADGF0YK1Lie5w/XV0T4XYbfrg2/16+T8/BLokAAAAyPAe1tpQ3V6e/1u9R1MWb5bdJj0zqoeaNHIpJT5Sie7wOqgWAICzQw8rgpHx01rVtZsvaqWNP+Xona/36YG30iVJdpuUNrKrRvVuGdjiAAAAGiBaAn7BZrPpnkvbllvms6TJ729SZk5BgKoCAABouAisldhfSTAttSztyWIyZgAAgLpGYK1ESnyk7JVcpXV9xjFmDwAAAKhjBNZKJLrDlTayqxy28qn1L//eprFvpyuvqCRAlQEAADQ8zBJwGpk5BdqTla9WceFa8t0Bzfx4q0p9lto2jdILoy9Uu2aNzlHVAADUDLMEIBgRWKth3Z6juvfNDTroKVK406EnR3bRNRe0qKVKAQA4ewRWBCNaAqqhd+tYLbn/VxrQNl4F3lKNe/tbTXp/owq9XMoVAADgXCGwVlN8lEt/v72PHhjcTjabtOCrDF37wir9eCQv0KUBAAAEJQJrDTjsNo27rL3+flsfxUaGavN+j37z3Bf69+YDgS4NAAAg6BBYz8Il7Ztoyf0D1LNVY+UWluh//2+9nljyvbylvkCXBgAAEDQIrGcp0R2ut+66SHcMSJEkvfyf3brxpTU6kFOozJwCrdqVxRWyAAAAzgKzBNSijzdlasI/vlNuUYkiQx3K95bKsiS7TUob2VWjerc8p+8PAACzBCAYMcJai4Z1SdQH9w1Qu6ZRyisuC6uS5LOkye9vYqQVAACgBgistax1fKQeuapjheWllqXF6fu5tCsAAEA1EVjPgQ4JjWS3VVw+86OtuuyZlXpj7Y8qKGbuVgAAgKogsJ4Die5wpY3sKoetLLXabVL/tnGKcoVo56HjemThJl2UtlxPfbyVNgEAAIAz4KSrcygzp0B7svLVOj5Cie5w5RZ69Y+v92n+qj3KOJovqWxO1yu7Juq2/q11YcvGdVofACD4cNIVghGBNQBKfZaWbzmov325W2t+OOpf3iM5RrcPSNEVXRKUdbxIu7PylBIfqUR3eACrBQDUJyYe94CzRWANsO/3e/Tql7u1OH2/ik9ccCA6LES5hSWyxJRYAIDqMf24B9QEgdUQh3OL9ObaDM1ftUfH8ovLPWeTNO/mC/Xrjs3kdNB2DAA4tfpy3AOqg8BqmBXbDyn1b+sqfS7KFaJ+beL0q3bx+lW7JmodFyGbrZLpCAAADVZ9O+4BVRES6AJQXvtmZVNi+X72vxE2lbUJ5BSWaNn3B7Xs+4OSpOYx4f7w2r9tnGIiQiWVnexF/ysAAAgWjLAa6O11GZr8/iaVWpYcNpueHNlF1/VM1ub9Hv1n52H9Z3uW1v94zN/zKkk2m9StuVuxkaH6fPthLgkLAA1UfTzuAWdCYDXUL6fE+qX84hJ9tfuo/rMjS1/syNK2g7mVvo5N0qPDO+vS9k3UMpYWAgAIdvX1uAecDoE1SBz0FGr+l3v0wopdp1wnJsKprs3d6t4iRt1auNU9OUbNosMqrEdLAQDUXw3luIeGhcAaRDJzCtR/5qcV+l/PT4zWrkPHy7UQnNQs2qVuLWLUvYVb3VrEaNfh45rx4ffy0VIAAPVSQzruoeEgsAaZyvpfR/VuqeISn7YdyNW3+7L17d5sfbcvRzsO5ZYLt5Wx26RPH7xUreMj6+YDAADOSkM77qFhILAGoTP1v56UX1yiTT959N2+bH27L0df7T6ig56iCuvZbVLnJLd6JMeoe3KMeiTH6Lz4SNnt9MMCgGka4nEPwY/ACr/KWgpOpVFYiLq3iCkXYps0ctH/CgABxnEPwYjAinJ+2VLwxDVd1L9tvL7dl630jGx9uy9bG3/KUaG3Yj9sTLhT2QVeSWXTbM0Y3kU3X9Sqrj8CADRoHPcQjAisqOBMLQXeUp+2H8xV+t6yftj0vdnafvB4pa91fkIj9W4dqwtaxuiClo25OhcAnGMc9xCMCKyoFZ9uPajb5399xvViIpzqkRyjC5Ib64KWZe0E7nCn/3laCgDg7HDcQzDi0qyoFR0ToytcUtZukx4b3kV7svL0zd6yVoLsfK8+33ZYn2877F+vTZNIXdCysUp9Pi1K389VugAAQDmMsKLWnGpKrZOKS3zakulR+t5sfZNxTN/szdaPR/JP+5r928QppUnZaGtSTFjZT3e4mrldcoU4yq3L6CwAcNxDcCKwolZVdUqtk44cL9K3+7L1wbf7tfCb/dV6r/go14kQG6bcwhKt3nVElspO+Jp8RUf9fkAKU28BaHA47iEYEVhhhMqm1LLbpAlDz1d+cYn2ZxcqM6dAmTmF2p9doKKSirMU/JLTYVPzmHA1bxxe9jMmwn+/ReNwJbjD5HTY/e/P6CyAYMBxD8GIHlYYIdEdrrSRXU/bUnCSZVk6lu/V/uyyAPvlzizNX7WnwnreUkt7juRrzynaDuw2qVl0mEIddv14tGwdm6RbLmql/+nVQgnRYYqLcslRhVFaAi8AAOcOI6wwSnVbCk5uU9no7D/u7idvqaWfjhXop+yC//48cSuuwiitw25T00YuNYsOU0J0mBLcYWX33S4lRJeN0n6xI0vT/rlJPk4WA2AAjnsIRoywwiiJ7vBqj1CeanS2Z6vYU27j81nKyivSx5sOaOrizRWebxzhVE6BV6U+S5k5hcrMKaxSLT5Levj9jYqPcql/23iFOR1n3ggAAJwWI6wIGrU1Ouuw2fTFw4PUJMqlrOPFOuAp1IGcQh3IKdABT5EOnnh80FOon07TT+uw29SuaZQ6JUWrc5JbnZOi1SkpWtFhznLr0U4AoDZx3EMwIrCiwTvTdFynsz87XwOe+qxc4JXKX6b2l1rFRajziRCbdbxIf1+156zaCc4m8BKWgeDDcQ/BiMAKqGajsydVFniv75WsA55Cbf7Jo837Pdq0P0ff7/fop+yCM75emyaRahTmVJjTrjCnQ2Ehjv/edzrkctpPLHNoS2aOPvg2s2w6L0mpF7fSZZ0S5AqxyxVStq7/foj9xGOHHHab3l6XoUnvb6xxWD7bsBvIsExQRzDjuIdgRGAFakFVA++xvGJ9n+nR5v05+nzbYa3adaQOq/yvELtUWSfDgLZxatIoTI3CQk7cnOV+Rp+4/9m2Q3rqo63+sPvY8C4acUFzlZT65C21VOLzqaTUUonP8i8r9Vnynli+7PsD+n9f7JZllc2bO3FoB93Yp6WiXCEKOTHV2OmcTeAMdFAHzjWOewhGBFYgQE41u8Ezo3ooIjREhd7SsluJT4XFJ++XqtDrU6G3VBlH8/WfHVkVXje5cbgcdpuKSnxlN2+pikp8Kvll34Khwp0ORf08MLvK7ke5yh7/eDRPn2455B9VvqJrgjonuVVSaslb6vOHYu/J8Fxa9tmLS306XujViu3l95lN0pVdExQTESqno2xEOjTErlBH2U/niZ+hIXZt+PGY3l6313+BivGXtdcNvVsqOjykwpXXKkPYRV3guIdgRGAFAuhs+mdPd8JYZWGopNSn4lKfirw+ZRzN0zV/XVUhLE8cer7sdim3sES5hSXyFHpP3Pf6Hx/NK1ZeUelpawux2xTisMlptyvEYVOIwy6n3SaHw6aSEkuZnqrNulCfuELsig53yh3uVHRYiKLDnYoOO/E4PES7D+fpo00H/GH33kFtdV3PZLnDy0avq3JVtobYr1xf6w4kjnsIRgRWIMBqu3+2qoG3ptueamT40wcvVYsTo7s226nD16mC9mcTBqqRy6njRf8NyscLS5Rb5PUH6K2Zufrgu4qX8L20fbySGkfIaT8Rjh12OR02hZwIzKGOsp95xaV6+t/b9PN/9Gw26e6B58kV4lBxia/sVuqTt7RshPrkskO5hUrfm1OVXVttNpvUyBUid0RZwP357WQI3nEwV4u+2e8PvPf/up1GXthcka6y0WdXiP2U+/1s2yCkwITlQNddX3HcQzAisAL13NkE3ppuezZB+Wy2r+6ocl2898qJlyoqzClPgVeeQq88BWWBO6fAe2JZibYd8Ojfmw9WeM1Qh13FpWe+gEVVOOw2RYQ6FBkaokiXQ5GuEEWGhshhl77YWb5X2ibp1v6t1TgitGwE3P7fcO+wl42MO06MkofY7VrzwxG9vvZHf8/xPQPb6KpuSQoNKd9C4XLaT/zPwX/7kH8ZOqde3UmDOjTV0bxiHcsv1rE8r47lF5947NWxvGIdzS/W4dxC7c6qeJW6Nk0iFR/lUkyEUzHhoYqJPPEzwqmYcKfcEU41jih7/Mn3h87qoh719cRAjnsIRgRWADVyNkH5bLY/27AciPc+XdCOjQyVp6BEOQX/Dbk5v7jtOJSrldsr9iu7QuynnAc4kOw2+ft/cwtLAl1OOZ0So9U40qlwZ1mwjwh1KCI0RJGhDoWfCPvhTofS92brza8yZJ0Iu5Ou7Kjf9WtVpV7lk2oaOt/6KkOTF9Z8ZHn73oPq0DKB4x6CCoEVQL1ztmE5EO99rvqVmzUKU763VPlFJTpeVKL84lIdLypRXlGJ8opLtf9YgZ76eGv5NghJ11zQXC6nQ6U/n9Gh3H1LpT6fjhwv1tYDuRVqcoeXXQCjuMSnopLSCnMRn44rxK74KJcaR5aNhsZGhqpxROiJ+041jgyVz7L0wFvpsio5KdFusym7wKuc/GJl53t1LN+rnIKT94uVU1DWa30uzjMMdzrkDncqJqKsXSPmZ+0bMSdbOiJClb43W69+eWImDEn/07OFOiVFn2hzKfH3hh8/cf94Ydl/v+yCynvEW8aGKzbSVaFP2n2iVzo6PETucKfW/nBUz//7O/34zPUc9xBUCKwAUEfqW7/yyZqr0oZx8qS+4pL/9v7uO1agm/7fmgqh88uHf12lz1/bF/Ww26S0a7rK5XQov7hU+cVlAT+vuEQFxaXKKypVgbdE+44V6Lt956ZfuS74ivK1dw6BFcGFwAoA9UQg+pWlwIXlQNVdeUiXPnrgEoU5HcouKBvFzc4v37qRfWJ098cj+ZWOSl90Xqxax0WemKatbHaIqLCQE1O3ORUVVjad3U0vr6kQtJ+78QI5HXZ5Ckt+1h/t9beTeAq9yswu0N5jBQRWBCUCKwDgjAIVls+WaS0c53Jk+eT7lhQSWBF8CKwAAFQiUKPSZ/Peb6/L0MMLvtKeZ67juIegQmAFAOAcCNTIMrMEIBiFBLoAAACCUaI7PCAXK0hoIBdIQMNiP/MqAAAAQOAQWAEAAGA0AisAAACMRmAFAACA0QisAAAAMBqBFQAAAEYjsAIAAMBoRgfWtLQ09e7dW40aNVLTpk01YsQIbdu2LdBlAQAAoA4ZHVhXrFihMWPGaM2aNVq2bJm8Xq8uv/xy5eXlBbo0AAAA1JF6dWnWw4cPq2nTplqxYoUuueSSKm3DpVkBAA0Jxz0Eo3p1adacnBxJUmxs7CnXKSoqUlFRkf+xx+ORJHm9Xnm93nNbIAAAAcaxDsGo3gRWn8+nsWPHqn///urSpcsp10tLS9P06dMrLF+6dKkiIiLOZYkAAARcfn5+oEsAal29aQn4wx/+oI8++khffPGFWrRoccr1KhthTU5OVlZWFn8aAQAEPY/Ho/j4eFoCEFTqxQjrvffeqw8//FArV648bViVJJfLJZfLVWG50+mU0+k8VyUCAGAEjnUIRkYHVsuydN9992nhwoX6/PPPlZKSEuiSAAAAUMeMDqxjxozRm2++qcWLF6tRo0Y6cOCAJMntdis8PDzA1QEAAKAuGN3DarPZKl3+6quv6tZbb63SazC9BwCgIeG4h2Bk9AirwVkaAAAAdcToK10BAAAABFYAAAAYjcAKAAAAoxFYAQAAYDQCKwAAAIxGYAUAAIDRCKwAAAAwGoEVAAAARiOwAgAAwGgEVgAAABiNwAoAAACjEVgBAABgNAIrAAAAjEZgBQAAgNEIrAAAADAagRUAAABGI7ACAADAaARWAAAAGI3ACgAAAKMRWAEAAGA0AisAAACMRmAFAACA0QisAAAAMBqBFQAAAEYjsAIAAMBoBFYAAAAYjcAKAAAAoxFYAQAAYDQCKwAAAIxGYAUAAIDRCKwAAAAwGoEVAAAARiOwAgAAwGgEVgAAABiNwAoAAACjEVgBAABgNAIrAAAAjEZgBQAAgNEIrAAAADAagRUAAABGI7ACAADAaARWAAAAGI3ACgAAAKMRWAEAAGA0AisAAACMRmAFAACA0QisAAAAMBqBFQAAAEYjsAIAAMBoBFYAAAAYjcAKAAAAoxFYAQAAYDQCKwAAAIxGYAUAAIDRCKwAAAAwGoEVAAAARiOwAgAAwGgEVgAAABiNwAoAAACjEVgBAABgNAIrAAAAjEZgBQAAgNEIrAAAADBavQisc+fOVevWrRUWFqa+ffvqq6++CnRJAAAAqCPGB9a3335b48eP17Rp07RhwwZ1795dQ4cO1aFDhwJdGgAAAOqA8YF19uzZuvPOO3XbbbepU6dOmjdvniIiIvS3v/0t0KUBAACgDoQEuoDTKS4u1vr16zVp0iT/MrvdriFDhmj16tWVblNUVKSioiL/45ycHEnS0aNH5fV6z23BAAAEWG5uriTJsqwAVwLUHqMDa1ZWlkpLS9WsWbNyy5s1a6atW7dWuk1aWpqmT59eYXlKSso5qREAABPl5ubK7XYHugygVhgdWGti0qRJGj9+vP9xdna2WrVqpYyMDH5xq8jj8Sg5OVl79+5VdHR0oMupF9hn1cc+qz72WfU1xH1mWZZyc3OVlJQU6FKAWmN0YI2Pj5fD4dDBgwfLLT948KASEhIq3cblcsnlclVY7na7G8w/VrUlOjqafVZN7LPqY59VH/us+hraPmOABsHG6JOuQkND1bNnTy1fvty/zOfzafny5erXr18AKwMAAEBdMXqEVZLGjx+v1NRU9erVS3369NGcOXOUl5en2267LdClAQAAoA4YH1hHjRqlw4cPa+rUqTpw4IB69Oihjz/+uMKJWKficrk0bdq0StsEUDn2WfWxz6qPfVZ97LPqY58BwcFmMe8FAAAADGZ0DysAAABAYAUAAIDRCKwAAAAwGoEVAAAARgvqwDp37ly1bt1aYWFh6tu3r7766qtAl2S0Rx99VDabrdzt/PPPD3RZRlm5cqWuvvpqJSUlyWazadGiReWetyxLU6dOVWJiosLDwzVkyBDt2LEjMMUa4kz77NZbb63wvRs2bFhgijVAWlqaevfurUaNGqlp06YaMWKEtm3bVm6dwsJCjRkzRnFxcYqKitK1115b4QIrDUlV9tmll15a4Xt29913B6hiANUVtIH17bff1vjx4zVt2jRt2LBB3bt319ChQ3Xo0KFAl2a0zp07KzMz03/74osvAl2SUfLy8tS9e3fNnTu30udnzZqlZ599VvPmzdPatWsVGRmpoUOHqrCwsI4rNceZ9pkkDRs2rNz3bsGCBXVYoVlWrFihMWPGaM2aNVq2bJm8Xq8uv/xy5eXl+dcZN26cPvjgA/3jH//QihUrtH//fo0cOTKAVQdWVfaZJN15553lvmezZs0KUMUAqs0KUn369LHGjBnjf1xaWmolJSVZaWlpAazKbNOmTbO6d+8e6DLqDUnWwoUL/Y99Pp+VkJBg/fnPf/Yvy87Otlwul7VgwYIAVGieX+4zy7Ks1NRUa/jw4QGppz44dOiQJclasWKFZVll3ymn02n94x//8K+zZcsWS5K1evXqQJVplF/uM8uyrIEDB1oPPPBA4IoCcFaCcoS1uLhY69ev15AhQ/zL7Ha7hgwZotWrVwewMvPt2LFDSUlJOu+88zR69GhlZGQEuqR6Y/fu3Tpw4EC5753b7Vbfvn353p3B559/rqZNm6pDhw76wx/+oCNHjgS6JGPk5ORIkmJjYyVJ69evl9frLfc9O//889WyZUu+Zyf8cp+d9MYbbyg+Pl5dunTRpEmTlJ+fH4jyANSA8Ve6qomsrCyVlpZWuBpWs2bNtHXr1gBVZb6+fftq/vz56tChgzIzMzV9+nT96le/0qZNm9SoUaNAl2e8AwcOSFKl37uTz6GiYcOGaeTIkUpJSdGuXbs0efJkXXHFFVq9erUcDkegywson8+nsWPHqn///urSpYuksu9ZaGioYmJiyq3L96xMZftMkm666Sa1atVKSUlJ+u677/THP/5R27Zt0/vvvx/AagFUVVAGVtTMFVdc4b/frVs39e3bV61atdI777yj3//+9wGsDMHshhtu8N/v2rWrunXrpjZt2ujzzz/X4MGDA1hZ4I0ZM0abNm2il7waTrXP7rrrLv/9rl27KjExUYMHD9auXbvUpk2bui4TQDUFZUtAfHy8HA5HhbNmDx48qISEhABVVf/ExMSoffv22rlzZ6BLqRdOfrf43p2d8847T/Hx8Q3+e3fvvffqww8/1GeffaYWLVr4lyckJKi4uFjZ2dnl1ud7dup9Vpm+fftKUoP/ngH1RVAG1tDQUPXs2VPLly/3L/P5fFq+fLn69esXwMrql+PHj2vXrl1KTEwMdCn1QkpKihISEsp97zwej9auXcv3rhr27dunI0eONNjvnWVZuvfee7Vw4UJ9+umnSklJKfd8z5495XQ6y33Ptm3bpoyMjAb7PTvTPqtMenq6JDXY7xlQ3wRtS8D48eOVmpqqXr16qU+fPpozZ47y8vJ02223Bbo0Yz300EO6+uqr1apVK+3fv1/Tpk2Tw+HQjTfeGOjSjHH8+PFyIzK7d+9Wenq6YmNj1bJlS40dO1aPP/642rVrp5SUFE2ZMkVJSUkaMWJE4IoOsNPts9jYWE2fPl3XXnutEhIStGvXLk2cOFFt27bV0KFDA1h14IwZM0ZvvvmmFi9erEaNGvn7Ut1ut8LDw+V2u/X73/9e48ePV2xsrKKjo3XfffepX79+uuiiiwJcfWCcaZ/t2rVLb775pq688krFxcXpu+++07hx43TJJZeoW7duAa4eQJUEepqCc+m5556zWrZsaYWGhlp9+vSx1qxZE+iSjDZq1CgrMTHRCg0NtZo3b26NGjXK2rlzZ6DLMspnn31mSapwS01NtSyrbGqrKVOmWM2aNbNcLpc1ePBga9u2bYEtOsBOt8/y8/Otyy+/3GrSpInldDqtVq1aWXfeead14MCBQJcdMJXtK0nWq6++6l+noKDAuueee6zGjRtbERER1jXXXGNlZmYGrugAO9M+y8jIsC655BIrNjbWcrlcVtu2ba0JEyZYOTk5gS0cQJXZLMuy6jIgAwAAANURlD2sAAAACB4EVgAAABiNwAoAAACjEVgBAABgNAIrAAAAjEZgBQAAgNEIrAAAADAagRWAUT7//HPZbDZlZ2cHuhQAgCEIrAAAADAagRUAAABGI7ACKMfn8yktLU0pKSkKDw9X9+7d9e6770r675/rlyxZom7duiksLEwXXXSRNm3aVO413nvvPXXu3Fkul0utW7fW008/Xe75oqIi/fGPf1RycrJcLpfatm2rV155pdw669evV69evRQREaGLL75Y27ZtO7cfHABgLAIrgHLS0tL02muvad68edq8ebPGjRunm2++WStWrPCvM2HCBD399NNat26dmjRpoquvvlper1dSWdC8/vrrdcMNN2jjxo169NFHNWXKFM2fP9+//e9+9zstWLBAzz77rLZs2aIXX3xRUVFR5ep45JFH9PTTT+vrr79WSEiIbr/99jr5/AAA89gsy7ICXQQAMxQVFSk2NlaffPKJ+vXr519+xx13KD8/X3fddZcGDRqkt956S6NGjZIkHT16VC1atND8+fN1/fXXa/To0Tp8+LCWLl3q337ixIlasmSJNm/erO3bt6tDhw5atmyZhgwZUqGGzz//XIMGDdInn3yiwYMHS5L+9a9/6aqrrlJBQYHCwsLO8V4AAJiGEVYAfjt37lR+fr4uu+wyRUVF+W+vvfaadu3a5V/v52E2NjZWHTp00JYtWyRJW7ZsUf/+/cu9bv/+/bVjxw6VlpYqPT1dDodDAwcOPG0t3bp1899PTEyUJB06dOisPyMAoP4JCXQBAMxx/PhxSdKSJUvUvHnzcs+5XK5yobWmwsPDq7Se0+n037fZbJLK+msBAA0PI6wA/Dp16iSXy6WMjAy1bdu23C05Odm/3po1a/z3jx07pu3bt6tjx46SpI4dO+rLL78s97pffvml2rdvL4fDoa5du8rn85XriQUA4HQYYQXg16hRIz300EMaN26cfD6fBgwYoJycHH355ZeKjo5Wq1atJEmPPfaY4uLi1KxZMz3yyCOKj4/XiBEjJEkPPvigevfurRkzZmjUqFFavXq1nn/+ef31r3+VJLVu3Vqpqam6/fbb9eyzz6p79+768ccfdejQIV1//fWB+ugAAIMRWAGUM2PGDDVp0kRpaWn64YcfFBMTowsvvFCTJ0/2/0l+5syZeuCBB7Rjxw716NFDH3zwgUJDQyVJF154od555x1NnTpVM2bMUGJioh577DHdeuut/vd44YUXNHnyZN1zzz06cuSIWrZsqcmTJwfi4wIA6gFmCQBQZSfP4D927JhiYmICXQ4AoIGghxUAAABGI7ACAADAaLQEAAAAwGiMsAIAAMBoBFYAAAAYjcAKAAAAoxFYAQAAYDQCKwAAAIxGYAUAAIDRCKwAAAAwGoEVAAAARiOwAgAAwGj/H5mlPsKfRujAAAAAAElFTkSuQmCC", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "fig, ax = plt.subplots()\n", - "\n", - "val = architectures_TM[key]\n", - "y_avg = []\n", - "for y in val['epochs_y']:\n", - " y_avg.append(np.mean(y))\n", - "_ - np.round(y_avg[-1], 2)\n", - "ax.plot(np.arange(len(val['epochs_x']))[:max_epochs], y_avg[:max_epochs], label=f'{key}', marker='.')\n", - "\n", - "ax.set_ylim(0, 10)\n", - "ax.set_ylabel('loss')\n", - "ax.set_xlim(0, max_epochs-1)\n", - "ax.set_xlabel('epoch')\n", - "\n", - "pos = ax.get_position()\n", - "ax.set_position([pos.x0, pos.y0, pos.width * 0.9, pos.height])\n", - "ax.legend(loc='center right', bbox_to_anchor=(1.4, 0.5), framealpha=0)\n", - "ax.grid(axis='y')\n", - "\n", - "plt.show()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.0" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/main.py b/tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/main.py deleted file mode 100644 index 976aded5..00000000 --- a/tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/main.py +++ /dev/null @@ -1,136 +0,0 @@ -import torch, tonic, sys, random -import numpy as np -from torch.utils.data import DataLoader -from sinabs.activation.surrogate_gradient_fn import PeriodicExponential -from torch.nn import CrossEntropyLoss -from torch.optim import Adam -import os -from tqdm import tqdm - -sys.path.append('../../utils') -sys.path.append('../models') - -from train_test_fn import training_loop_no_tqdm, load_dataset, split_train_validation_used_seed, load_architecture -from weight_initialization import rescale_method_1 - -if torch.cuda.is_available(): - device = torch.device('cuda:0') - print('device: ', torch.cuda.get_device_name(0)) -else: - device = torch.device('cpu') - -torch.backends.cudnn.enabled = False -torch.backends.cudnn.deterministic = True -random.seed(1) -torch.manual_seed(1) -torch.cuda.manual_seed(1) - -### Initialization #################################################### - -total_architectures = 11 - -lr = 5e-5 -batch_size = 8 -num_workers = 4 -n_time_steps = 50 -epochs = 25 -w_rescale_lambda = 0.8 - -spk_thr = 2.0 -v_min = -0.313 -grad_scale = 1.534 -grad_width = 0.759 - -validation_ratio = 0.2 -prev_used_seed = 1 - -loss_fn = CrossEntropyLoss() - -directory = f'./architectures_results_2' - -if not os.path.exists(directory): - os.makedirs(directory) - -with open(f'./architectures_results_2/fixed_parameters.txt', 'w') as file: - file.write(f'lr: {lr}\n') - file.write(f'batch_size: {batch_size}\n') - file.write(f'num_workers: {num_workers}\n') - file.write(f'n_time_steps: {n_time_steps}\n') - file.write(f'epochs: {epochs}\n') - file.write(f'w_rescale_lambda: {w_rescale_lambda}\n') - file.write(f'spk_thr: {spk_thr}\n') - file.write(f'v_min: {v_min}\n') - file.write(f'grad_scale: {grad_scale}\n') - file.write(f'grad_width: {grad_width}\n') - file.write(f'validation_ratio: {validation_ratio}\n') - file.write(f'prev_used_seed: {prev_used_seed}\n') - -### Data Loading ##################################################### - -snn_train_dataset, snn_test_dataset, sensor_size = load_dataset('DVSGESTURE', n_time_steps) -train_dataset, validation_dataset = split_train_validation_used_seed(validation_ratio, snn_train_dataset, prev_used_seed) - -disk_cache_train = tonic.DiskCachedDataset( - dataset=train_dataset, - cache_path='./cached_train' -) -snn_train_dataloader = DataLoader(disk_cache_train, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True) - -disk_cache_validation = tonic.DiskCachedDataset( - dataset=validation_dataset, - cache_path='./cached_validation' -) -snn_validation_dataloader = DataLoader(disk_cache_validation, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True) - -disk_cache_test = tonic.DiskCachedDataset( - dataset=snn_test_dataset, - cache_path='./cached_test' -) -snn_test_dataloader = DataLoader(disk_cache_test, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=False) - -### Training Loop ########################################################## - -train_p_bar = tqdm(range(9, total_architectures+1)) - -for iter in train_p_bar: - achitecture = f'ResSCNN_{iter}' - - # instantiate model. - csnn = load_architecture( - achitecture, - sensor_size, - 11, - batch_size, - PeriodicExponential(grad_scale=grad_scale, grad_width=grad_width), - v_min, - spk_thr - ).to(device) - - csnn.init_weights() - csnn.rescale_conv_weights(rescale_method_1, w_rescale_lambda) - - # instantiate optimizer. - optimizer = Adam(csnn.parameters(), lr = lr, betas = (0.9, 0.999), eps = 1e-8) - - # train/test model. - epochs_x, epochs_y, epochs_acc = training_loop_no_tqdm( - device, - n_time_steps, - batch_size, - sensor_size, - snn_train_dataloader, - csnn, - loss_fn, - optimizer, - epochs, - snn_validation_dataloader, - True) - - # export model data. - with open(f'./architectures_results_2/{achitecture}-training_metrics.npy', 'wb') as f: - np.save(f, np.array(epochs_x)) - np.save(f, np.array(epochs_y)) - np.save(f, np.array(epochs_acc)) - - # update progress bar - train_p_bar.set_description(f'{iter}/{total_architectures} - model {achitecture} - acc.: {np.round(epochs_acc[-1], 2)}') \ No newline at end of file diff --git a/tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/model_training.py b/tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/model_training.py deleted file mode 100644 index 84c4144f..00000000 --- a/tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/model_training.py +++ /dev/null @@ -1,92 +0,0 @@ -import torch, random, sys - -import tonic -from torch.utils.data import DataLoader -from torch.nn import CrossEntropyLoss -from torch.optim import Adam - -from sinabs.activation.surrogate_gradient_fn import PeriodicExponential - -import matplotlib.pyplot as plt -import numpy as np - -sys.path.append('../../utils') -sys.path.append('../models') - -from train_test_fn import training_loop, load_dataset, split_train_validation, load_architecture - -if torch.cuda.is_available(): - device = torch.device('cuda:0') - print('device: ', torch.cuda.get_device_name(0)) -else: - device = torch.device('cpu') - -rand_seed = 1 - -achitecture = sys.argv[1] - -torch.backends.cudnn.enabled = False -torch.backends.cudnn.deterministic = True -random.seed(rand_seed) -torch.manual_seed(rand_seed) -torch.cuda.manual_seed(rand_seed) -np.random.seed(rand_seed) - -batch_size = 8 -num_workers = 4 -epochs = 100 -lr = 5e-5 - -spk_thr = 2.0 -v_min = -0.313 - -grad_scale = 1.534 -grad_width = 0.759 - -validation_ratio = 0.2 -n_time_steps = 50 - -snn_train_dataset, snn_test_dataset, sensor_size = load_dataset('DVSGESTURE', n_time_steps) - -train_dataset, validation_dataset = split_train_validation(validation_ratio, snn_train_dataset, rand_seed) - -disk_cache_train = tonic.DiskCachedDataset( - dataset=train_dataset, - cache_path='./cached_train' -) -snn_train_dataloader = DataLoader(disk_cache_train, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True) - -disk_cache_validation = tonic.DiskCachedDataset( - dataset=validation_dataset, - cache_path='./cached_validation' -) -snn_validation_dataloader = DataLoader(disk_cache_validation, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True) - -disk_cache_test = tonic.DiskCachedDataset( - dataset=snn_test_dataset, - cache_path='./cached_test' -) -snn_test_dataloader = DataLoader(disk_cache_test, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=False) - -snn = load_architecture(achitecture, sensor_size, 11, batch_size, PeriodicExponential(grad_scale=grad_scale, grad_width=grad_width), v_min, spk_thr).to(device) -snn.init_weights() - -optimizer = Adam(snn.parameters(), lr=lr, betas=(0.9, 0.999), eps=1e-8) -loss_fn = CrossEntropyLoss() - -epochs_x, epochs_y, epochs_acc = training_loop( - device, - n_time_steps, - batch_size, - sensor_size, - snn_train_dataloader, - snn, - loss_fn, - optimizer, - epochs, - snn_validation_dataloader) - -with open(f'./architectures_results/{achitecture}-Training_Validation-TM.npy', 'wb') as f: - np.save(f, np.array(epochs_x)) - np.save(f, np.array(epochs_y)) - np.save(f, np.array(epochs_acc)) \ No newline at end of file diff --git a/tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/single_training.ipynb b/tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/single_training.ipynb deleted file mode 100644 index 33bac3a0..00000000 --- a/tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/single_training.ipynb +++ /dev/null @@ -1,4716 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import torch, random, sys\n", - "\n", - "import tonic\n", - "from torch.utils.data import DataLoader\n", - "from torch.nn import CrossEntropyLoss\n", - "from torch.optim import Adam\n", - "\n", - "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential\n", - "\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "\n", - "sys.path.append('../../utils')\n", - "sys.path.append('../models')\n", - "\n", - "from train_test_fn import training_loop, load_dataset, split_train_validation, load_architecture\n", - "from weight_initialization import rescale_method_1" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "device: NVIDIA GeForce RTX 3070 Ti\n" - ] - } - ], - "source": [ - "if torch.cuda.is_available():\n", - " device = torch.device('cuda:0')\n", - " print('device: ', torch.cuda.get_device_name(0))\n", - "else:\n", - " device = torch.device('cpu')" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "rand_seed = 1" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "achitecture = 'ResSCNN4'" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "torch.backends.cudnn.enabled = False\n", - "torch.backends.cudnn.deterministic = True\n", - "random.seed(rand_seed)\n", - "torch.manual_seed(rand_seed)\n", - "torch.cuda.manual_seed(rand_seed)\n", - "np.random.seed(rand_seed)" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "batch_size = 8\n", - "num_workers = 4\n", - "epochs = 125\n", - "lr = 5e-5\n", - "\n", - "spk_thr = 2.0\n", - "v_min = -0.313\n", - "\n", - "grad_scale = 1.534\n", - "grad_width = 0.759\n", - "\n", - "validation_ratio = 0.2\n", - "n_time_steps = 50" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Loading Data" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "snn_train_dataset, snn_test_dataset, sensor_size = load_dataset('DVSGESTURE', n_time_steps)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "getting validation dataset...." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "train_dataset, validation_dataset = split_train_validation(validation_ratio, snn_train_dataset, rand_seed)" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The transformed array is in shape [Time-Step, Channel, Height, Width] --> (50, 2, 128, 128)\n" - ] - } - ], - "source": [ - "sample_data, label = train_dataset[0]\n", - "print(f\"The transformed array is in shape [Time-Step, Channel, Height, Width] --> {sample_data.shape}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "disk caching samples..." - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "disk_cache_train = tonic.DiskCachedDataset(\n", - " dataset=train_dataset,\n", - " cache_path='./cached_train'\n", - ")\n", - "snn_train_dataloader = DataLoader(disk_cache_train, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True)\n", - "\n", - "disk_cache_validation = tonic.DiskCachedDataset(\n", - " dataset=validation_dataset,\n", - " cache_path='./cached_validation'\n", - ")\n", - "snn_validation_dataloader = DataLoader(disk_cache_validation, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True)\n", - "\n", - "disk_cache_test = tonic.DiskCachedDataset(\n", - " dataset=snn_test_dataset,\n", - " cache_path='./cached_test'\n", - ")\n", - "snn_test_dataloader = DataLoader(disk_cache_test, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=False)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Network Module" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "snn = load_architecture(achitecture, sensor_size, 11, batch_size, PeriodicExponential(grad_scale=grad_scale, grad_width=grad_width), v_min, spk_thr).to(device)\n", - "snn.init_weights()" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "# lambda_ = 0.8\n", - "# snn.rescale_conv_weights(rescale_method_1, lambda_)" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "optimizer = Adam(snn.parameters(), lr=lr, betas=(0.9, 0.999), eps=1e-8)\n", - "loss_fn = CrossEntropyLoss()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Training loop" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "7f01930878c74d7eb18bb72c0921d870", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - " 0%| | 0/107 [00:00" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "y_avg = []\n", - "for y in epochs_y:\n", - " y_avg.append(np.mean(y))\n", - "\n", - "plt.plot(np.arange(len(epochs_x)), y_avg, marker = '.')\n", - "plt.xlabel('epoch')\n", - "plt.ylabel('average loss')\n", - "plt.ylim(0,)\n", - "plt.xticks(np.arange(len(epochs_x)))\n", - "for i, txt in enumerate(y_avg):\n", - " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAG2CAYAAACZEEfAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAACvJ0lEQVR4nOzdd3gUVffA8e+k901vkE6v0oQAghgMINKliYhifQWlCEJQsIC0nwoqioKIKFJEAQGlSZUWQokE6REIhBRCkg3pITu/P5YsWZJAoikQzud59nnZmdm7dwZ1z3vvufcoqqqqCCGEEEJUUyZV3QEhhBBCiIokwY4QQgghqjUJdoQQQghRrUmwI4QQQohqTYIdIYQQQlRrEuwIIYQQolqTYEcIIYQQ1ZoEO0IIIYSo1iTYEUIIIUS1JsGOEEIIIaq1Kg12du/eTY8ePfD29kZRFNauXWt0XlVVpkyZgpeXF9bW1nTu3JmzZ88aXZOcnMyQIUNwcHDA0dGRF154gfT09Eq8CyGEEELcy6o02MnIyKBp06Z88cUXxZ6fPXs2n332GV999RXh4eHY2trSpUsXsrOzDdcMGTKEv//+m61bt7JhwwZ2797Nyy+/XFm3IIQQQoh7nHKvFAJVFIU1a9bQu3dvQD+q4+3tzZtvvsm4ceMA0Gq1eHh48N133zFo0CBOnjxJgwYNiIiIoGXLlgBs2rSJJ554gsuXL+Pt7V1VtyOEEEKIe4RZVXegJOfPnyc+Pp7OnTsbjmk0Glq3bs3+/fsZNGgQ+/fvx9HR0RDoAHTu3BkTExPCw8Pp06dPsW3n5OSQk5NjeK/T6UhOTsbFxQVFUSrupoQQQghRblRV5fr163h7e2NiUvJk1T0b7MTHxwPg4eFhdNzDw8NwLj4+Hnd3d6PzZmZmODs7G64pzowZM3j//ffLucdCCCGEqAqXLl2iZs2aJZ6/Z4OdihQWFsbYsWMN77VaLb6+vly6dAkHB4cq7JkQQgghSistLQ0fHx/s7e3veN09G+x4enoCkJCQgJeXl+F4QkICDz30kOGaxMREo8/duHGD5ORkw+eLY2lpiaWlZZHjDg4OEuwIIYQQ95m7paDcs/vsBAQE4OnpybZt2wzH0tLSCA8PJzg4GIDg4GBSU1M5fPiw4Zrt27ej0+lo3bp1pfdZCCGEEPeeKh3ZSU9P59y5c4b358+fJzIyEmdnZ3x9fRk9ejTTpk2jdu3aBAQEMHnyZLy9vQ0rturXr0/Xrl156aWX+Oqrr8jLy2PkyJEMGjRIVmIJIYQQAqjiYOfQoUN06tTJ8L4gj2bYsGF89913vPXWW2RkZPDyyy+TmppK+/bt2bRpE1ZWVobP/Pjjj4wcOZKQkBBMTEzo168fn332WaXfixBCCCHuTffMPjtVKS0tDY1Gg1arlZwdIYQQ4j5R2t/vezZnRwghhBCiPEiwI4QQQohqTYIdIYQQQlRrEuwIIYQQolqTYEcIIYQQ1ZoEO0IIIYSo1iTYEUIIIUS1JsGOEEIIIao1CXaEEEIIUa1JsCOEEEKIak2CHSGEEEJUaxLsCCGEEKJak2BHCCGEENWaBDtCCCGEqNYk2BFCCCFEtSbBjhBCCCGqNQl2hBBCCFGtSbAjhBBCiGpNgh0hhBBCVGsS7AghhBCiWpNgRwghhBDVmgQ7QgghhKjWJNgRQgghRLUmwY4QQgghqjUJdoQQQghRrUmwI4QQQohqTYIdIYQQQlRrEuwIIYQQolqTYEcIIYQQ1ZoEO0IIIYSo1iTYEUIIIUS1JsGOEEIIIao1CXaEEEIIUa1JsCOEEEKIak2CHSGEEEJUaxLsCCGEEKJak2BHCCHEAy8/P5/JkycTEBCAtbU1QUFBTJ06FVVVDdc899xzKIpi9Oratesd2929ezc9evTA29sbRVFYu3at4VycNot90UlcSc1kypQpeHh6YmllzSOPPsbZs2cr6lYfSBLsCCGEeODNmjWL+fPnM2/ePE6ePMmsWbOYPXs2n3/+udF1Xbt2JS4uzvBavny50fnbg6aBAweSlpbGvHnzjK5bGRFDu5nb6fLUUGo42TJ95ixo9xIuT/8fR2Izaf9oCNnZ2UafiY2N5ZlnnsHFxQVra2saN27MoUOHAMjLy2PChAk0btwYW1tbvL29efbZZ7ly5UoFPK37jwQ7QgghHnj79u2jV69edO/eHX9/f5566ilCQ0M5ePCg0XWWlpZ4enoaXk5OTkbnbw+aPv/8c44cOcLly5cN18RpswhbHUX66X3kXDkNioK5bzOsa7fBwj0AlyfHcjUhnsXLfjJ85uTFK7RsHUyeqrBx40ZOnDjBxx9/bPj+zMxMjhw5wuTJkzly5AirV6/m9OnT9OzZswKf2v1Dgh0hhBD3rdJMPxX26quvoigKc+fONTretm1btm3bxpkzZwD49ttv2bBhA7/99pvR9NPOnTtxd3enbt26tGzZktq1a2Nra4uTkxOdO3fm999/v2vQdD4pg9y0JJK3fo1jx+dAVbFw9jacN7G0xcK7Ljt27wX0o0DBg0aRotgT4TuI84onAQEBhIaGEhQUBIBGo2Hr1q0MGDCAunXr0qZNG+bNm8fhw4c5f/58uTyj291piq7Ae++9R7169YyeUXh4+B3brQgS7AghhLhvlXb6CWDNmjUcOHAAb2/vIucmTpzIoEGDqFevHubm5rz44ou0bduWb7/91nBN165d+f7779m2bRuzZs0iLi4OU1NTIiMj2bNnD/7+/hw6dIitW7cagqa//vqLPXv20K1bN0M7fs7WXNvwCQ6t+2JiYQOAYmlt1B8zW0ey064ZRoEyz4Zj4VmbhDUzePrRJgTVa8RHn31xx2ej1WpRFIXFixeXyzO6XUZGBk2bNuWLL0ruR506dZg3bx5RUVGGZxQaGsrVq1fv2n55MqvUbxNCCCHKUeHpJwB/f3+WL19eZPopNjaW119/nc2bNxuuLeynn37ixx9/ZNmyZTRs2JDIyEhGjx5NWlqa4ZpBgwYZ/ty4cWOaNGlCUFAQMTExhISE8Mknn7Bo0SLatm1LvXr1MDU1JT8/nw8//JAhQ4bwzDPPAPD9V5/h6mCNSYue5MSeAqCBlwMF2TWKAo1raLA2N+V8UgY6FfJS48k7+jsOrXrjETyA1LizvPXmWP65lsOX748tcj/Z2dlMmDCBwYMHc+TIkXJ5Rrfr1q2bURBXnKefftrofcEzOnbsGCEhIXf9jvIiIztCCCHuW7dPPxU3kqLT6Rg6dCjjx4+nYcOGxbYzfvx4w+hO48aNGTp0KGPGjGHGjBklfndgYCCurq6cO3eO3NxcFixYgLW1NX/++SfLli3jyJEjLFmyhI8++oglS5YAEB0dzaeffkrv0R+iKApPPFwXgHZ+dox4VD8l9ZCPIxZ51/H09MTPWT/yg6pi6RGEU8dhWHgEYf9QV+yaduHbbxYSp80y6ldeXh4DBgxAVVXmz59fbs/ovyp4RhqNhqZNm1bId5RIFapWq1UBVavVVnVXhBBClEF+fr46YcIEVVEU1czMTFUURZ0+fbrRNdOnT1cff/xxVafTqaqqqn5+fuqcOXOMrnF2dla//PLLIp+rXbu2Cqhr1qwp8t2XLl1SFUVRLS0tVUVRVG9vb9Xd3V2dN2+e0XVTp05V69atqwLq8OHDVUVRVMXEVEUxUU1MTVVAVRRFrenjq/pN2KD6jvlJtbS0VH/88Ud16P/GqqYaD/01Ftaqpv0Q1fet9arfhA2qc+hrqmJupfoG1lJtbGxUR0dH9bHHHlM7dOigNmnSRE1KSirxGQXVrqN6eXkZ7u32ZwQU+woNDTVco6qq+u677xruzdbWVg0JCVEPHDhgdP/r169XbW1tDc/oq6++Up988kmj779dSd8/e/Zso+tK+/stIztCCCHuW4Wnn4obSTl8+DCffvop3333HYqilNhOjx49+PDDD/ntt9+4cOECa9as4ZNPPqFPnz4AZGVlMX78eA4cOMCFCxfYtm0bvXr1IigoiMOHD7Nv3z66du1KYmIiW7ZsMbSbnp5OQkICWVn60ZeAgAB++ukn6g77EK/nP2P5xl04ODhgYWFB2MQJBJpe4+qGT3BwdufcuXP89MMinB9/lcAWHajhXYO0g6u5fng9AHnJsZjaOTH7k7lERUWxY8cOoqOj2bNnDytWrMDFxcXoGb0+9TM8np2LXcve/HP+Ak0e6QLcGm0q/Ixq1qxJSEgITk5O/PDDD8yePRuAvXv3GuX6FOTkAEyfPr3YnJxOnToRGRlpeEZTpkyhVq1ad8z1Kby8Py4ujm+//RZFUejXr19p/rEo6o6h0ANCRnaEEOLecSU1U9177qp6JTXzrtfWrFmzxJEUVVXVOXPmqIqiqKampoYXoJqYmKh+fn6Gz6SlpamjRo1SfX19VSsrKzUwMFB9++231ZycHBVQV6xYoYaGhqpubm6qubm56lnDRx0y7Hk1Pj7e6LsVRVHNLSzU71f+op4/f1794IMPih2hsG0UovpN2KAmp+eofn5+amhoqOrh4aGaWVioVn5NVeeAhmrdevVVuyaPq34TNqg/b9yhmpmZqTUDaqs2tVqrrj3GqYq5pfrQ0HdUVVXV3NxctWfPnqq3t7cKqD/99JMaFxenxsXFqTVr1lQ//L9P1ICJG1S/CfqX5pFnVHPnmkajTbc/I26O1qiqqvbq1Ut97LHH1L59+6pDhgwp8vfAzRGagt/TP/74o8S/s1q1ahlG3yhhZOd2Bd9/u9L+fkuCshBCiHvGyogYwlZHoVPBRIEZfRszsJVviddnZmZiYmI8SWFqaopOpwNg6NChdO7c2eh8ly5dGDp0KM8//7zhmL29PXPnzi1xubWlpSWbN2826t8+BXbG5DDQ41bfTRzcUGwcef6lVzHJScOnRg3efvttpkyZgoWFBQAn49Lo9umfONqY42SrP9atWzc2b95MvDab4JnbuPTlcGx9/Mi68BcNbNLp17U7n3zyCWPGjEFRzuN4Iwmnx17ies1gEtOyyUyOZ926dYb+DhgwwPBnBwcHUjJvoCu00lxRTFBV/TPq2LEjY8aMKfKMatWqxblz59i3bx+//fYbH3zwAXPnzuWTTz4p9hnl5eWVKidHp9ORk5NT4vnbJSQk8NtvvxlG6/4NCXaEEELcEwqWWRf8KOtUmLT6OB3quOGlsS72MwXTT76+vjRs2JCjR4/yySefMHz4cABcXFwM0zkFzM3N8fT0pG7duoZjISEh9OnTh5EjRxKnzeLviwmo2njc7K0AOH/+PFv/PMD45ScxcXBHl5tNyv6VjI1tQ+D47iQnJ/Hi6++Rn56Ce793sXDzw1RR2DOxU5G+X0jKAMDfxVb//sIFwzlPjRWt/J1R//ctqqrDdtf3bHpvMOZTnzGs7AoLCwPgqfn7OHQxhXc+/55lM8aiKApeXl6sXbuWVq1aGdp87rnn+PHrueS3fQlzV19yE6JJi1iLfZPH0Yb/goODA40aNSryjHr16kViYiLt27dHVVUmTZpEjx49GDJkCKCfojt37hy7d+8GYODAgbi6uvLpV99wRgva3CQWzfuEnj174uXlRVJSEl988QWxsbH079//Lv803LJkyRLs7e3p27dvqT9zOwl2hBBC3BMKllkXlq+qXEjKLDHY+fzzz5k8eTKvvfYaiYmJeHt788orrzBlypQyfXd0dDRJSUmGkZvMi8dIWD7JcH7sWP3ybttGIbh2H4NiYkJe8mUS1nxIuxVhODg5ke/gj+eQWVi4+d2x7//cDHYCXG2L7Yu3Rh9gZZ78k4wTO3nt/U95pfejhuXw3t7eDBs2jGfa+HHoYgqRN7w5dPgIqSnJLFy4kAEDBhAeHo67u7vhGYW9/Q5fL5lPfmYqpnbO2Dfrxhf/N41n2v1S4jOJjIxkx44deHp60rFjR7Zu3coff/zBkiVLGDZsGIcOHaJTp06G61VV5erVqzz/7FBqvPINplY2uB/S51ElJSXh4uJCq1at+PPPP8u04uvbb79lyJAhWFlZlfozt1Nuzpk90NLS0tBoNGi1WhwcHKq6O0II8UCK02bRdsZ2Cv8olTQ6UhHffehCMm+siKTwr6KJAmtea8vphHQ+3HASbXae0edMFNg78THOX83g6W+MdwYuqe/jV/3FqsOXGft4Hd4IqV2kH+1mbkenwuUvn0PT5ikcW/QwtDNt2jSWLl3KqVOnyLmRT/CM7SRn5PJmaB2ealETL401tWvXZvjw4YYRIICfD19m3Kq/DO97P+TN3EHNUBSFNWvW0Lt37yLPxMfHh6eeeoq5c+cSGRmJu38d3n3/A7ZvWM25M6eLfYbtZm7n0tcvYdf4cTTBA+7693en7wf4888/6dChA5GRkcVOjZX291tWYwkhhLgneGmsaeqjMbw3UWB630YVHuisjIih7cztvL7cONAB/VRary/28dbPx9Bm56GxNsOk0KKuNoEuuNpZMmPjqSLtTutTfN/PF0xjFTOyU3h0S83LAcXEMEIExvlIlmamNPbW/8B/vOUM7WZuZ2VETLE5MUsPXASgvpf++sspxnvzFCczM5P9+/fTokULTuY40XbmdlYdvsLFpHRWRsSU3HdVRc3XB4WF+/5vLFq0iBYtWvznfXkk2BFCCHHPKDyN9WHvOycnl4eCPKHSznFcz77BmtfaMrFbPQD2RV9j8IIDRMVq0Vib88v/grE21/+0BpYwTXXhWkaJ5wNcbQ3BlHWth9HuW0lOdASmGVeLLIePvpLEr4s+Jif2FDe0iWTFnePFF14skhPTpn1Hdq/9AXNThUlP1EOXm0XkX5EcPXoU0OcjRUZGEhNjHMB07dqVgwcP0rhZC8Yt/oOM0/tIi1iLTZ1gJq0+TvSVJCZNmsSBAwe4ePEi1y+f5trvc7lx/Ro2ddsb2hk/vJ9R1ff09HQiIyOJjIy84/enpaWxatUqXnzxxTv/pZSC5OwIIYS4J6iqahj1AEjOzK3w7ywuT6iACaC77ZhOhcxcHa92DGLHqUTCzydz6GIKAN0aedLCz5nuTbz5+fBlNhyLo3WgcXJ0WnYeSen6+ypuZMdLY82Mvo2ZtPo4zp1fQbvnR27s+YaOrWcVyUeKSc0m99pl0qO2kZ+Vhqm1AxaetZm+aDVaKw/Dzsp/nzqLab0adGnoycMBztxIOMelZWE0/1r/nQX5SMOGDeO7774z9KV169asXLmS3zds4GrSd5jaOWP3UDcc2w0iX1W5lJrDqVOnjHJynD1rY1cobwngxOmzHD0TQ5w2Cy+NdZFcn5K+f8WKFaiqyuDBg4v/CyoDydlBcnaEEOJekJSeQ8tpfxje921Wg08GPlSh3xmnzSJ4xnajYybA5083o6aTNX2+3GcUDBXkoACG3Jpb52DPxMc4HX+d5xZH4GpnwYGwEMxMb02iHLucSs95e3G1s+TQO8ZL4m/v14WkTPxdbUqcxiuc31OYgn6TnILZtoLTLz8SyKTu9en8yS7OJaazZPjDdKzjVvLDueng+WsM+PqA0bGScnEe/2QXZxPTebVjEI7W5szcdGt6rzRbCZSV5OwIIYS4r1woNKoDEH01vcg1cdos9kUnFakH9W+521thaXbrp9BUUZjRrzHdm3jT1MeJGX0bY3pzV2FTRTHkEBW/cgwuJGXSrpYrTjbmJKXncuCfZKNrCkauSpriKuClsSY4yOWO+UoFo0Cmt+0MrRb638JdXLTnH+K0WdTxsAPgTPx1o8+V9GwjLqQU+e4xj9cu0rfT8dc5m5iOhakJr3UKoudDXkbnC7YSKK+/u7KQYEcIIUSp+fv7oyhKkdeIESMA/RLuPn364ObmhoODAwMGDCAhIaFU7bQKcOHi7J7EL3yJa38sYMe8twzttG3blvot2lDT04N2tdxoOeabIkmy+fn5TJ48mYCAAKytrQkKCmLq1KkUnsBYvXo1oaGhuLi4oCgKq7fuIeeGDltLU5a+8DB7JnYyGnkY2MqXPRM7sfylNkbnCufWFDBVFPxdbTA3NaFrI/0P/YZjV4yuuZWcbPMv/waMFe7f54MfuuO1BcFYHQ97AE4n3Ap2VkbE0G7mdp5eGG5IdC6w/i/9PYR1q0dzH0cAziYWDUQLrutQxw0HK3MuXCuamPxfE5b/LQl2hBBClFpERIRRzaKtW7cC0L9/fzIyMggNDUVRFLZv387evXvJzc2lR48ehhVExbUzceJE7O31P8CuT75J5yEjST+8nuzkOH76dSN79+4lPTOLs2fOoek4DAC1mFGCWbNmMX/+fObNm8fJkyeZNWsWs2fPNqrllJGRQfv27Zk1axYAf11KBSA40IX2tYvfvLC4UZbbR1UKj/oA9GiiD3Z+j4rjz7NXDf28YNhjx+7fPP5iFfSvpb9zkQCssIJgrCDYOXsz2ClpM8c4bRbnEq9zKv465qYKA1v58EFv/eaDv0Ze4adDlwz3paqqIbDr0dTr5j2WHBBWNklQFkIIUWpubsY5HjNnziQoKMiw6dyFCxc4evSoIX9iyZIlODk5sX37dqOyDYXbiYqKwsfHh8tJadjU70Adj2R+Q8XcuSYW7v40ruXKR9/9QpdmtTAxtzR87vZN+/bt20evXr3o3r07oB89Wr58OQcPHjR8ZujQocCtXYuPxWoBJ4KDXMv8LAa28qVDHbdic2taB7pgZ2lGWvYNhi46aMhXOW8Idsr/B79wcnO+qupzdhR9YFg4GMvwyAfgTEI6Op16x80cD/xzDYBHarvhaGOBo40FjbwdOH4ljbd+Pma4r4beGi5cy8TK3ITO9T2K7c/tAWFlkmBHCCHEv5Kbm8vSpUsZO1ZfqiAnJwdFUbC0vBWQWFlZYWJiwp49e4rUqCrw8MMPs3HjRnwe6QeKQt61S/rPBrYg+mo67Wq5UsfbGRSF3Lizhs/dPkrQtm1bFixYwJkzZ6hTpw5//fUXe/bsKbGWE8DxWC04OxF826qp0vLSWBf74514PZuMnBuG9/rRkiisLUyB4ldilYfbAzCgSDDm72KDhakJWXn5XE7JIsDVFuVmUFRAAfxcrHlnrX605smbI1Vx2ixOxKUZ3VfY6ih6NfUGIDjIBVvLW6HFnQLCyiTBjhBCiH9l7dq1pKam8txzzwHQpk0bbG1tmTBhAtOnT0dVVSZOnEh+fj5xcXEltlOnTh1UVeXirlXw52rm6fIxM7cgL/E8py5dJaOJGx9PnQyqjvxMreFzU3s3NPrxnDhxImlpadSrVw9TU1NDLamCWk7Fyc7T4WFjTj1P+//+QAo5n5TB7Uud81VIz9GPqhTUxaoItwdgtwcYZqYmBLrZcir+OmcSrtO5gQcP+ThyNCb11kUKrI+MI/pqBuamCo830I/WFDcKpFNhTaQ+KNp56iorI2KM8p5KCggrk+TsCCGE+FcWLVpEt27d8PbW/796Nzc3Vq1axfr167Gzs0Oj0ZCamkrz5s2LVCYvbMaMGVhYWuLaYzw1n/+Mb7/7DjNzczJO7mLGoNZoNBpSUlKx9qqFUmjlkbej8Q/oTz/9xI8//siyZcs4ckRfk+mjjz66a7XsNoEumNwp2eVfKC5fpeC9t8YKK3PTcv2+sqrreStJOTsvn7MJ+oTj93s24PH6HqgqzLi5bDwvX+X3KH2wWtx9FaZSdSuu7kSCHSGEEGV28eJF/vjjjyK724aGhhIdHU1iYiJJSUn88MMPxMbGEhgYWGI7x44do0uv/tg26Eiteg14ftgwnvvfKEys7Gke9jNJSUmEzZ5HTloSVk7uhs9uOGY8WjR+/HgmTpzIoEGDaNy4MUOHDmXMmDHMmDHjjvcSHPTvprDupCBfpXBgYH7zjbfTvy9oWV4KkpTPJFxn5+mrpOfcwFtjxdA2/ozrUqfI9QUBzO2J2cUFEVW14upOJNgRQghRZosXL8bd3d2QDHw7V1dXHB0d2b59O4mJifTs2bPEdhRFwbFmEKDPJwFwt7cGVce1fEvMrO349qf16DK0tAjuYPjs5r/jybmRb3ifmZlZZASpcC2pwgp/rm0FBDugz1fZO/ExPnqqCaYK5OTr538OXUgttrZUZboV7KQbVlE92dQbExOFaxlFd64uHMAUXu6+ZkTbe2bF1Z3c08FOafZMUFWVKVOm4OXlhbW1NZ07d+bs2bN3aFUIIcR/odPpWLx4McOGDcPMzDj1c/HixRw4cIDo6GiWLl1K//79GTNmDHXr1jVcExISwrx58wztNGzYkLXfzSczOgLHfK2+BtRHs7D3qU9eShyffbWIeW+/hm3DR6njph8VsclK4MyCN3j1zcmGdnv06MGHH37Ib7/9xoULF4rUkgJITk4mMjKS5Zv3AWCZEc/12HPEx8dXyLPy0ljTrrZrkTyXqp7qqXsz2IlOTGfbyUTgVhJyaZaMFyx3v9PGi/cU9R724Ycfqi4uLuqGDRvU8+fPq6tWrVLt7OzUTz/91HDNzJkzVY1Go65du1b966+/1J49e6oBAQFqVlZWqb9Hq9WqgKrVaiviNoQQolrZvHmzCqinT58ucm7ChAmqh4eHam5urtauXVv9+OOPVZ1OZ3SNn5+f+u677xraOXz4sNrw8UGqqYObam5hqQYGBqrBwcGqpb2ziomZ6uUboDo07FiwIbDRy7t+c0O7aWlp6qhRo1RfX1/VyspKDQwMVN9++201JyfHcM3ixYuLbefdd9+tsOe199xV1W/ChiKvfeeSKuw77yY/X6fWe2ejoS+PzNpu9Pe04uBFNXDib6rfhA1q4MTf1BUHL96xvSupmeq+c0nqldTMiu66kdL+ft/TtbGefPJJPDw8WLRokeFYv379sLa2ZunSpaiqire3N2+++Sbjxo0DQKvV4uHhwXfffcegQYNK9T1SG0sIISpPnDaL80kZBLjaGkYAQj7eSfTVDJa+0Jr2tfV73ry9Joofw2NoX8uVPeeScLAy4+iUUExNFI7EpND3y31Ym5swf0gL6nrZl2o0obh6UiXVeSrP+63s7yyNnvP2cOyyfnXbs238DBsGFihNfa6qVi1qY7Vt25Zt27Zx5swZAMOeCd26dQP0ZeHj4+ON9m7QaDS0bt2a/fv3l9huTk4OaWlpRi8hhBAVr7iyBDfydcQk6/NBCk+VBLnpdxnecy4J0K+aMr05v9LMxxFHa3Oy8nQ8911EkRIHJfn7SlqJG+hVlLvttlxVzAsVKP0h/GKR51ea+lz3i3t6n5277ZlQMMfq4eFh9DkPD487zr/OmDGD999/v+I6LoQQooiSyhIEudmRl69iYWaCd6Ef1iB345IKhVdNxadlo83KM7wvaKtDnVslH24fQToTf50P1v1dpF+VkVB7r2yuVyBOm8WRi7cKfKrFPL/q5J4OdgrvmdCwYUMiIyMZPXo03t7eDBs27F+3GxYWxtixYw3v09LS8PHxKY8uCyGEKEFJZQkiLugrg/u72Bjtd1PrDsFO8Zv2qfx2LI7uTbzYdjKRyWuPo6LfDbiupz2nClX5VtAn61TmKMu9sLlegZKeX+HyG9XJPR3sFN4zAaBx48ZcvHiRGTNmMGzYMDw9PQFISEjAy+tWKfmEhAQeeuihEtu1tLQ02s5cCCFExStY5XN7wFMwQnP7rsK7z1w1en80JpV6ng53bGvabyeZ9ttJo2MqGAU6oA925j3djOZ+TtXyx/1uint+9+KS8fJyT+fs3G3PhICAADw9Pdm2bZvhfFpaGuHh4QQHB1dqX4UQQtxZQe7K7RbtOQ+Am8Ot/xMap83i7TVRRte9s+bWcu3b82DKSgc421o+kIEO3Lt5RBXlnh7ZKdgzwdfXl4YNG3L06FE++eQThg8fDoCiKIwePZpp06ZRu3ZtAgICmDx5Mt7e3vTu3btqOy+EEKKIga18eX/9CTJz85nYtR4zN50i7+Zme8sOxNCkhoaBrXzvWIm74Ae5IA/mt2NxRUZz7qY6j2KU1r2WR1SR7umRnc8//5ynnnqK1157jfr16zNu3DheeeUVpk6darjmrbfe4vXXX+fll1+mVatWpKens2nTJqysqn47biGEEMYyc2+QmavfvTikvjuFx2UK11UqzcZ2oB+h6N7Eq8i1CrdqUZkqCv2a13hgRjHKojqtuLqTe3qfncoi++wIIUTluJCUwaMf7cTa3JRFw1ry9DfhRa5Z/lIbgoNcWBkRw6TVx8lXVUOAUriadmHFXXv7qMX9sG+MKJvS/n7f09NYQgghqpfE6zkAuDtYEuB25yTZskyzlHRt4c/cS6uhROW6p6exhBBCVC+J17MBcLOzLFWSbFmmWR6UKRlRdjKyI4QQotIkpt0a2YEHK0lWVB0JdoQQQlQawzSW/a1FJDK9JCqaTGMJIYSoNIZpLHvZ2FVUHgl2hBBCVJqrhpEdCXZE5ZFgRwghRKUxBDsOsheaqDwS7AghhKg0iTKyI6qABDtCCCEqRe4NHckZuYAEO6JySbAjhBCiUiSl60d1zEwUnGwsqrg34kEiS8+FEEKUizhtFueTMghwtQUw/LlgWXnBFJabvSUmtxezEqICyciOEELco/z9/VEUpchrxIgRACxYsIBHH30UBwcHFEUhNTW1VO3GxsbyzDPP4OLigrW1NY0bN+bQoUOG8++99x716tXD1tYWJycnOnfuTHh40RpWha2MiKHdzO08vTCctjO203aG/s/tZm5nZUQMAIlp+mXnMoUlKpsEO0IIcY+KiIggLi7O8Nq6dSsA/fv3ByAzM5OuXbsyadKkUreZkpJCu3btMDc3Z+PGjZw4cYKPP/4YJycnwzV16tRh3rx5REVFsWfPHvz9/QkNDeXq1avFthmnzSJsdZShxpV68wX6ulcFlcxvjezISixRuWQaSwgh7lFubm5G72fOnElQUBAdO3YEYPTo0QDs3Lmz1G3OmjULHx8fFi9ebDgWEBBgdM3TTz9t9P6TTz5h0aJFHDt2jJCQkCJtnk/KMCrmebt8VeVCUqZREVAhKpOM7AghxH0gNzeXpUuXMnz4cBTl3+e7rFu3jpYtW9K/f3/c3d1p1qwZCxcuvOP3LliwAI1GQ9OmTYu9piBHpyQFlcyvXpdpLFE1JNgRQoj7wNq1a0lNTeW55577T+38888/zJ8/n9q1a7N582b+97//8cYbb7BkyRKj6zZs2ICdnR1WVlbMmTOHrVu34urqWmybXhprnGzMDe+Vm68CBZXMDUVAZRpLVDKZxhJCiPvAokWL6NatG97e3v+pHZ1OR8uWLZk+fToAzZo14/jx43z11VcMGzbMcF2nTp2IjIwkKSmJhQsXMmDAAMLDw3F3dy/SZnrODVIy8wBYOLQFjWpqOB6r5aXvD+Nobc6Alj6AbCgoqo6M7AghxD3u4sWL/PHHH7z44ov/uS0vLy8aNGhgdKx+/frExMQYHbO1taVWrVq0adOGRYsWYWZmxqJFi4pt82zCdUC/pPzxhp54aax5pLYbZiYKqVl5xKZmAbeKgErOjqhsEuwIIaqFuy2nTkhI4LnnnsPb2xsbGxu6du3K2bNn79jmo48+WuzS7+7duxuuWb16NaGhobi4uKAoCpGRkeV+b4sXL8bd3d3oe/+tdu3acfr0aaNjZ86cwc/P746f0+l05OTkFHvubEI6AHU97A3HrMxNqeupfx91WUu+TiUpvWD3ZJnGEpVLgh0hxH3vbsupVVWld+/e/PPPP/z6668cPXoUPz8/OnfuTEZGRontrl692mjp9/HjxzE1NTUs/QbIyMigffv2zJo1q0LuTafTsXjxYoYNG4aZmXHmQXx8PJGRkZw7dw6AqKgoIiMjSU5ONlwTEhLCvHnzDO/HjBnDgQMHmD59OufOnWPZsmUsWLDAsHdPRkYGkyZN4sCBA1y8eJHDhw8zfPhwYmNjje67sNM3R3bqFAp2AJrUdATgWKyW5Ixc8nUqigKudrJ7sqhkqlC1Wq0KqFqttqq7IoT4FyZMmKC2b9++xPOnT59WAfX48eOGY/n5+aqbm5u6cOHCUn/PnDlzVHt7ezU9Pb3IufPnz6uAevToUVVVVfXy5cvqkCFDVGdnZ9XKykpt1KiRGhERYbg+Pj5eHTZsmOrl5aVaW1urXbp0Uc+cOVOk3c2bN6uAevr0aXXBggVq+/btVUdHR9XR0VENCAgo2NLG6OXp6ana2Niojo6OqpWVlfrCCy8Ytbl+/Xq1UaNGqqWlpVqvXj11wYIFhnNZWVlqnz59VG9vb9XCwkL18vJSe/bsqR48eLDE5/LMNwdUvwkb1OXhF42OLwu/qPpN2KA+vXC/ejw2VfWbsEFtMXVLqZ61EKVR2t9vSVAWQtz31q1bR5cuXejfvz+7du2iRo0avPbaa7z00ksAhukXK6tb0ycmJiZYWlqyZ8+eUufCLFq0iEGDBmFre+el1gUjTZ06dWLjxo24ublx9uzZIiNN5ubm/Prrrzg4OPDJJ5/QuXNnTpw4YdR+aGgoqqrfxOb9999n8ODBtG3bFisrK2bNmkVycjJ///03NWrUAGDZsmW4u7sTGBhIVlYWc+bM4adVq+jz8jia1/XDS2PNk08+yZNPPlls362srFi9enWpnkeBMwUjO57GIzuNa2gAOHZZKxsKiqpVKaHXPU5GdoS4v1laWqqWlpZqWFiYeuTIEfXrr79Wrays1O+++05VVVXNzc1VfX191f79+6vJyclqTk6OOnPmTBVQQ0NDS/Ud4eHhKqCGh4cXe77wyE5ljTTduHFDtbe3V5csWVLiNd9uP64CqkuPt1Tbho+qdg6OxY40Xb9+XR0xYoRao0YN1crKSq1fv746f/78O35/x44dix1ZeuKJJ1RVVdXcG/mqfZPORc536dKl1PcoxJ2U9vdbcnaEEPc9nU5H8+bNmT59Os2aNePll1/mpZde4quvvgLA3Nyc1atXc+bMGZydnbGxsWHHjh1069YNE5PS/Wdw0aJFNG7cmIcffpg4bRb7opOI02YVe+3dNu6720hTaWVmZpKXl4ezs3Ox5y9e1TJu6idgYUPqzm9BMcOh9xR2HjhSpETE2LFj2bRpE0uXLuXkyZOMHj2akSNHsm7duhK/f/Xq1fwefoKaI36g5aRVRXKazE1NcLQ2xyqgBR3fW03NET/wvwV/sHz58lLfoxDlQYIdIcR9rzTLqVu0aEFkZCSpqanExcWxadMmrl27RmBg4F3bz8jIYMWKFbzwwgtGBS8LF7ks7G4b99WrVw9fX1/CwsJISUkhNzeXWbNmcfnyZeLi4kp93xMmTMDb25vOnTsbHS/YEDDAwwltxFpsa7fBTOOOa/fRmHvVQWfnTmhoKEFBQYbP7Nu3j2HDhvHoo4/i7+/Pyy+/TNOmTTl48GCJ3+/s7Mw1nQ2mdk40quXL1q1bsbGxMUpkdrIxRzEz50KWBaZ2TgT41jAKsoSoDBLsCCHue2VZTq3RaAw5NIcOHaJXr153bX/VqlXk5OTQuWc/o4KXhYtcFlYZI02T3pvK0mXLWfD9cqMRIri1IeC6LTuwDmhBxsldmLv4cnXtDC59PoSX+jxWpERE27ZtWbduHbGxsaiqyo4dOzhz5gyhoaF37MeZQiuxistpcrK1IDsmikufDyF24Sts/Goa165dK9U9ClFeJEFZCHHfGzNmDG3btmX69OkMGDCAgwcPsmDBAhYsWGC4ZtWqVbi5ueHr60tUVBSjRo2ia/ce2AU1J06bhZfGmmeffZYaNWowY8YM4rRZnE/KIMDVlkWLFtGlew92XsgqUvAyNzONLbvDcTPNBOD06dO4uLgYimsWtOPtH0TML7/cardWAyIjI9FqteTm5uLm5kbr1q1p2bLlXe/3mVFvs+zrT/EYOI2XNlxlhkUMHeq4GfrrpdFvCBgUFETt/rkcjdpK+rEtOLTux+NPvUhf31zeeOMNLCwsDLsmf/7557z88svUrFkTUzMzTE1MWLhwIR06dLhjX07H64Md5eo5jh8/XmTjwV5PPsEJi/qYOXpwIyWOc3/9RLdu3di/fz+mpqZ3vVchyoOiquodatU+GNLS0tBoNGi1WhwcHKq6O0I8UGJjY5kwYQIbN24kMzOTWrVqsXjxYsOPfklFL2fPns348eMN7zds2EBYWBhnz57F0dERMzMz0tLSAGjYsCFNmzblt99+IyEhAS8vL5xrBnEyJoHcxPOouVks2naM7z94HTc3N+LTb7Bv5zbUGzmY2rtyI+UKngOnYunfjIuzil/FdDtfX19m//wnYauj0B75ndQ/l6LmpIOZJeauvji1G8yn459nYCtfAKZOncqUKVOwtrYmKyuLlJQUHB0di7T7zgcfMv3D6XgM+ADLGvUMxxUFVBVMFJjRtzEDW/ly7HIqPeft5eKsJ3F080Qz/BssTE3YMf5RZk2ZQEREBPv37wfgo48+4pPP56NrNQQTB3dyLx8nY+9S1v+6tsg0WQFVVWk+dSspmXm0uvwzp48d5tixY0bX5OtUGr+3mczcfADmPuFFn44t+OOPP4qtoC5EWZT291tGdoQQVeZuS7SBIjksGzdu5IUXXqBfv35GxwuWU8dps/hh5Wq8nWxo/VAjEtKyWLhoMd9+8wVHjx6lYcOGxGmzaNRvFNZBvlgHtSJ11xI+WH+SHz5fzFNdOpDpWg/3/u9hYqPhRsoVzBy9MHfyAqDmiB+Mvjfrn0Nc2/QZP207SPvmDfHSWBMREUHbtm3537jJWNdrT17KFXS5WTh2fA6bWq1JP76NhJ8/YGRGKn7TXyLhwlnmzJlD3foN6NS9L199NA3AaKQJ4O33pzFr2ge4PDkOM40H+ekpACgWVphYWKPLzSZl/0rGxrYhcHx3Fm8/RtLv+tGtxx5pi0WgC/v/ucbsTaewdPXhwsWfidNmcepyEmGTJuHSaxJWQa0AsHAPIDfhPB/OnFVisJOUnktKZh5qXjZb1q/mgw8+KHKNqYlCI28NBy/oNzqs6euPq6sr586dk2BHVBoJdoQQVWbWrFn4+PiwePFiw7GC6Z8Cnp6eRu9//fVXOnXqVGxi8cqImJs5NY6YKNBHzWDN0Vh0Dp3JN/2Gz5b/xtfTGnI+KQP7lvpcnewY/UiETlV58qW3yDF1wPOJ0YY2zR2Nv9/UTh+ITe5eH29Ha/r1+wIr38aM35yAyZaEm6MqrZj+5RLenjSJlL3LMdN44Nz5Fewf6gqAU4dnuR6xluTt3/BIqy/w9vLi0T7PcNTlMdZcPgnA6iOXiImJMeTwrIyIYdYnn5N/I4+ktTOM+qRpNxjH9kNQTEzIS75MwpoPabciDKzsMfOoRfO2HUlMTOST0Dr0/2o/v0ZeIXnbXnIVDW1nbCc/J5MbeXmoGI+iqYoJGdm5Jf79FdTEsr58kIScHJ555plir7Mwu5WH1Pf/1nPt2jW8vLxKbFeI8ibBjhCiytxtM8DbJSQk8NtvvxlWNRUWp80qkjz8y5FYVF0+maf2oMvL5vcEB+K0WeTd0BXbfta5cKwCmnN17QyyLx3H1M4F+2ZPGIKUAqaKwhNNvLiamEBWdASu3ccYvnPS6uN0qOPGMwP6MP8fTZEcn4L+qKoOzyGzWfv2ABLSsnljeSSFL31//QnW/bCGjNx8/jgRz8Rfoqj5v2+L9NmEWxvYKGYWuPd5GxMFvnqmOS//cAQ7SzM+7aqhU4dHWP3t5+SleJMbd4b0vzbh3GUkKmBiaYOlTyNSdn6LYm6BqYM7OZeOk/H3dvrOmm34rttHmgrKRKRGbqF37964uLgY9S09PZ3xkyazLaUmpnZO5KXEkbpzMWaOXjRpc+dcICHKkwQ7QogqU7BEe+zYsUyaNImIiIgiibOFLVmyBHt7e/r27Vvk3PmkDKPAIvfqBeJ/GId6IxfFwhr3Pm9j6uLD2YTrzNh4qtj+5KXGk3f0dxxa9cYjeAA5cWdJ2baAVkFunHVoSb6qYqooTO/bCC+NNdOmL8bEwhqbOm0NbeSrKheSMgkOcuHxBh5s/jtBf/zqRRJ+HEdebo6hPxauvgz4+kCxfVFV6PXFvjs+v8nd6/NEEy92n7lqFOi19HdiX7R+2ujxBh60D36INWvWMOrNt7gSfQ4zjQdOj72EXcNOhrbcek4gZdcSktZ/hC47HVMHd/y7DGfcqNcN1xQeaQL9Sqy8a5dJOBPJC/NuBUUFTE1NOXI0ksQji9FlZ2Bq54x1QDMcH3mGuOv5+Lvf8faEKDcS7AghqoxOp6Nly5ZMnz4dgGbNmnH8+HG++uqrYoOdb7/9liFDhhRZag0Q4GpcwsHcuQZez3+GLieTzNN7SPptDh5Pz2TJfndOxV/HxdaC7194mJ07bjByuT7BF1XF0rMWTh2HYQIsHPMUq7/I58SRjezZNJ4LSZn4u9rgpbEGYPOaFdg1eBTF7FZhS1MF/F1tALh4Tb9C6+UOAQxp1Z68sFCiYxP4cfkq1q78HFPNNCxcff/VsysYXfLSWDOwlS8d6rjx+7E4pv52koPnU/jrkhaAHk3100VPPvkkLR4Jod3M7UVGm0A/Pefx5BhWvxZMTHIWE345RmZuPh9vPcMjtV0JcLVl+a8bOZ+UYVi9djw2DXOXmny37zyPB/sXadPa2pq1G34v8p2mimJ4RkJUBgl2hBBVpqTNAH/55Zci1/7555+cPn2alStXGi0LLwg88m/7BTczs+CpkFasPXoFS89a5Mad5fqhdWy7GVw80diLht4art6s3/Rujwa88q0T5q6+htGb7k28iXmoMZs2/IqXxtrwXQX9iT57hllLP+Wr4/mGH/OXOwTipbHmXOJ1TsVfx9xU4bVHa+FoYwFuGmrVqkWXju1oFXWUU4fW4dJ1ZKmfl4minyorPLpkeJYaa154JJAdp6+y51wSOTen6uJSs42umdG3MZNWHydfvZmhc3MVV0GbTX2caOrjRFZePm/9fIz5O6OZvzPaqB8K0NRHQ1SsPqB6f93fWJmZGFaWFXb7dxbXdyEqmgQ7QjxA7rbM+7nnniuSD9OlSxc2bdpUYpvz589n/vz5XLhwAdAv854yZQrdunUzXBMfH8/48ePZunUr169fp27durz99ttl2gxw0aJFtGjRglO5TvS5OVJQeJn18oP6nYxb+jvy5uP1DCMw47rU5fCFFPovV1Hz8wztLQuP4bVOt3YQ7tvch/WhnTh/IYaNEzsZfozv1p+3hnRlqDaLib8cY9eZJOLT9KUg1v+lX0X2SG03faBzG0tTBQr1B/T5N6NCajOpYKSpEFNFYfVrwWTm6oxGlworKGNR2JRf/+ax+u6G6wtGgQpGqYAiI1YA7WsZ598UpgKRN0eOwDhXqbh+3f6dEuiIyiY7KAvxgChY5m1ubs7GjRs5ceJEkfpIAF27diUuLs7wulsdo5o1azJz5kwOHz7MoUOHeOyxx+jVqxd///234Zpnn32W06dPs27dOqKioujbty8DBgzgiSee4MCBA0yfPp1z586xbNkyFixYwIgRI4y+Iy0tjVWrVtF/yLPF7mDcrsOjfP75PACGtwtk3TcfcfavCC5cuEBSzDlWzp9FdkwUtg0eNbSZez2ZLbvDOXfuHABRUVE81bsnf0ceYvEXc0rVn4Jq6V4aa8Y+XheA347F0eHRTnz15RcAPNnEi7CwMHbv3s2FCxeIiooiLCyMfXt288Yrz2N6M6pRM1J4pbEJbmoqAM/XN+VG4j/kZ103GnUJDnIpMVi4PW8JbuUQFealsTa0U/jPhV24ZvyZuynue0r6TiEqm4zsCPGAKM0ybwBLS8siy73vpEePHkbvP/zwQ+bPn8+BAwdo2LAhoK+7NH/+fB5++GEA3nnnHebMmUN2djZr1qwhLCyMDz74gICAAObOncuQIUOM2lyxYgWqqtKs05PoVpw0Opevqpw8cxZdLW/q2FvyeAMPfklM5NlnnyUuLg6NRkPdBo3wHPgBlv7NDJ9Lj9zIc18sM7wv2Cl41KhRLF++vFT9GTx4sOFYUx9HGtfQT+1E/n0ak7peuDc04fEGHvx2W3+aNGnC5s2befzxxxmpzeJCUiY/L/yEic9MN7T3/iv6+lJvz/qcEa/0KlWQEOBqa5jqKvBv82OKa+tOJA9H3MtkB2VkB2XxYGjQoAFdunTh8uXLJS7zfu6551i7di0WFhY4OTnx2GOPMW3atCJLikuSn5/PqlWrGDZsGEePHjXk44SGhmJhYcH333+Po6MjP/30Ey+88AJ//fUXtWrVKvU9rDkay5iVkUWO13K35VxiBm88VouxoXWL/ezKiJgieSPF5Zj8FysOxjBxdZThfcc6riwZ3rpcv+NuyvM+C7d1e35P72berD16pUKfpxB3U9rfbwl2kGBHPBgKVjCNHTuW/v37ExERwahRo4xWPq1YsQIbGxsCAgKIjo5m0qRJ2NnZ3bWOUVRUFMHBwWRnZ2NnZ8eyZct44oknDOdTU1MZOHAgW7ZswczMDBsbG1atWnXXIpOFaTPz6DJ3N/Fp2ShAcf/hCutWj1c6BhVzRi/u5ihKReWNZObeoNkHWw3JwQows1/jSg8CyvM+C7cFxvk9Ff08hbgbCXbKQIId8SCwsLCgZcuW7Nt3a++WN954w6g+0u3++ecfgoKC7lrHKDc3l5iYGLRaLT///DPffPMNu3btMozsvP766xw8eJDp06fj6urK2rVrmTNnDn/++SeNGze+a9/jtFmMX3WMPeeSCHS1ZdGwVvx9JZWRyyONrjNVFPYUSi6ubHHaLNrO2G4UiFV1n4Sozkr7+y0JykI8IEpa5h0TE1PiZwIDAw11jO7EwsKCWrVq0aJFC2bMmEHTpk359NNPAYiOjmbevHl8++23hISE0LRpU959911atmzJF198cdd+r4yIoe2M7ew5p19l1LWRJwFutjjbWRa59m5JshXtfFJGkRGnqu6TEEKCHSEeGGVZ5l3g8uXL/6qOkU6nIydHvwQ7M1P/Q194513Q766r0xVftqFAQQmIwgHE17v+IU6bZUigNWqzipNk78U+CSEk2BHigTFmzJg7LvNOT09n/PjxHDhwgAsXLrBt2zZ69eql3wSvSxdDOyEhIUybNo1nnnkGFxcXzM3NCQwMNCwrDwsLY8eOHWzbtg1bW1s6dOiAtbU1gwcP5uDBg0RHR/Pxxx+zdetWevfubWi3YI+YOG0WM2fORFEURo0eXWQ1UOblEzzZNZRa3q7Efz6IhB8noMvLuSc2qyvYQK9gOfm90CchhOTsAJKzIx4cGzZsICwsjLNnzxIQEMDYsWMNq7GysrLo3bs3R48eJTU1FW9vb0JDQ5k6dSoeHh6GNnx8fLh+/Tp9+vThf//7Hx999BG7du0iJSUFR0dHmjRpQqtWrQgJCSEwMJCsrCzee+891q5di6Ojo2Ezw3HjxjF06FCgcLVyyIs/Q96WT/Byc6ZZ6/bsdL6V6JwTe5KEVe8yceJEhvTvi5mZGTv3R1Dv4ceoU8PpngkqJHFXiMohCcplIMGOeNAVV36hJCPHjOPPPXvY9McOw4qcu3224N+x4hKd47RZhtpJutws4r4bhWuX1wi4tBkLj0Au1O5vuDb+hzd5omsoqxd99t9vWghx35MEZSGqkdjYWMO0kbW1NY0bN+bQoUMA5OXlMWHCBBo3boytrS3e3t48++yzXLly5Y5tvvfeeyiKgqIoeDva0K6WG36BdVgZcSthOTs7mxEjRuDi4oKdnR2tH+vG10uWcx5ParV+HBuNM/51GtFjxPu0m7nd6LMFcnNzWbBgARqNhqZNmxY5X3jX3+St87EOaoWl30Nk5N7geEHtpZ4N+KJvEDlXTtPpodq0bdsWDw8POnbsyJ49e/7tYxVCPCBkB2UhqtjdRkYKyjx06tSJjRs34ubmxtmzZw1lHjIzMzly5AiTJ0+madOmpKSkMGrUKHr27GkIiEpSt34DMkPCMIzvmpgQtjqKep72ZOTm8/X0MHZs3cyqVavIM7Wk19MvcCMljutHf8ehVW9sgweQE3eWlG0LUEzNmYRiqI+0YcMGBg0aRGZmJl5eXmzduhVXV9cifSioVp5xYhe58dF4DZsDwNXrOeTa6qjjYcfQNv4cPBgO6IO0jz76iIceeojvv/+ekJAQjh8/Tu3atf/tX4EQopqTYEeIKlQ4V6VwUcvC7lbmQaPRsHXrVqPPzJs3j4cffpiYmBh8fUve0C4fBRNb49pYOhV6fbEPXU4Gl777jtEffsZjjz3GvugkXJ4YzZVv/oe5kzdOHfUbEVp4BJGXdJHrkb9j1ziEC0mZeGms6dSpE5GRkSQlJbFw4UIGDBhAeHg47u7uRt939XoON9KukrxtIR4Dp6KY6YtmJmfkYmELD/s7Y2KiGFZuvfLKKzz//PMANGvWjG3btvHtt98yY8aMUj1zIcSDR6axhKgiBcuqjYtaRhGnzTK6bt26dbRs2ZL+/fvj7u5Os2bNWLhw4R3b1mq1KIqCo6PjHa+7fOE8l794ltivXuDq+v/jRlqi4VxO/DnQ3WBNggtx2ixSM3Ixd/EBxQTF3HiPG3MXH/LTrmKiYFhmbWtrS61atWjTpg2LFi3CzMyMRYsWFenDjwdiyI0/hy4zlYTvR3Nxdk8uzu5JzqXjXD+8ng/7PcTl5HTD8vey7hUkhBAysiNEFSm+QjUcvpCMs52lYVrrn3/+Yf78+YwdO5ZJkyYRERHBG2+8gYWFhaHMQ4E4bRanYq8xdtx4Bg8eXGLCXpw2C9ua9ajT/y2umrqQn56Mdu9y4n+cgPfwLzCxtEGXkQKmZmBpy+ojsSwNvwiAiaUN+enXDPWR1hyNJS85FjMHdx7yceR8UgZAkSm5wnvvFNBm5fHrX7FY+TVl+aY93NDpeOvnYwBc+/1TzF1q4tC6H5dScmgT6I+3t3exewV169atbA9fCPFAkWBHiCpSUlXpghIIBdNaOp2Oli1bMn26viJ2s2bNOH78uFFNK9BPiU1cdZSENdPJv36dMXMmFfu9t6bO7MGrJRoLU356uQ1xV5+hX8eHyDq9B9smxjWr/m+zPsBwtDbHzLcG/5w9ywDzCF5oPoTAtKuMnbsF+86vcSQmlUFf7CLtwErGvfg0Q0OakZSUxBdffEFsbCz9+99aWRUSEoJX0w5kW7Skvq8HA0PbEp+WjdXuNHQqKOaWmFjZY+0egL+rDYqiMH78eN59912aNm3KQw89xJIlSzh16hQ///zzf/3rEEJUYzKNJUQV8dJY82xwybsX66e1juPu4XnXqZs4bZY+0Fk7kxvaRNwHTmXq5gtsOHbFaFrs9qkzgOy8fFztLQltFki9unVp534DU+VmLk/+DXTZ6YZr07LzyM7MZPjw59mw5mcaNWrEvI9n8eGs/8OuYScAFBMTcq9d5r1RL1K7Th26PNGd2PhE/vzzTxo2bGho68zZc+w6Fg3AM218URSlyKZ8ym2b8o0ePZqwsDDGjBlD06ZN2bZtG1u3biUoqOTin0IIISM7QlQhGwv9v4KP1nHlqZY+jFx21Oh8vqrSoFmru5Z5OBuXqg90Uq7gMXgGptYO6ICRy44aJT4XN3WmqvpK1vam+URHRzN06FDmD+vEqn01GPXTu2Rd/Avbuu0AyEm6zJXLl3jxxRf55ptvDG3si05i3kL9ainFzAL3Pm/r/4y+OvlZBf7Bg1Y3r18ZEYPZkC8Nq8B0hbb7GtjKlw513Ljw4s5iN+WbOHEiEydOLO0jFkIIGdkRoiodvpgCQLfGXrTwcyq2rtKYMaPvWOYhLy+PSSOeJzf+HK49xoFOR356CvnpKaj5eYYRokc6dmLH6u8p+IqU7YvIjolCp03kavQx+vTpg6mpKYMHD8ZLY03/tnWwb/o4Kdu/IfviMXLiz3Ft41xaPNyaNm3aGPWzuJpQgKGmVUEf4rRZt+pdFQq6Plh/0mgEyktjTXCQi+w+LIQoFzKyI0QVycvX8dflVABa+DkZpnAm/qIvfKkA0/s2olsrX9asWUNYWBgffPABAQEBzJ07lyFDhgD6DQf3bt8MQNziN4y+w2PwdKx8m5CvqpyLjub85XhU67oA3LieRNL6/0PJSee19W60b9+eAwcO4ObmBugDji8+m8vIUWO5unY6an4eLds9yrrl3xW5l4K+T1p9nHxVNYzoFFZQ/TtfpxaTmK0alqwLIUR5k2BHiCpy4koa2Xk6NNbmBLraAfopnKvXc/hoyxlaBzgb9tx58sknefLJJ4ttR2vqiN+EDZgosOylNiRdz+GNFUeLBBSf/7qXWZtOwfVcejTx4umX1ty1dtPQ9nXovH1Vqeo8GaafkjKxsTChz5f7ivRBp6rM+eN0kc9KZXAhREWSYEeIKlIwhdXCzwmTQnNA7Wu78dGWM5xJTEdVVRSlmPmhm+K0WUz+9TgAvZvVoE2gCwAZuTcMoywFxq06ZvhzS38ngoNcStVPL411qUdcCl9beKSnwJBvwg1/Lhj9kcrgQoiKJsGOEFXkcMytYKew+l72mJsqJGfkEpuaRU2n4kc8VkbEMLFQ7kstNzvDucKjLHn5Op799qDRZz9Yf5LQhp4VGmAU7kPMtQwmrI4yOq8A855uRnO/e6dauRCieipzgvKOHTsqoh9CVDuFC20WvOrVq2c4v+/I3ySunsY7/drg4ODAgAEDSEhIwNLMlLqe9gAcu6w1anP37t306NEDTy8vBj3sR8bp/YZzH285Q5w2i/fee4969epRy9uVJ1rW4s3nnyLnivHUUUGOTEUrSDT2cSkasOkAZ1tLCXSEEBWuzMFO165dCQoKYtq0aVy6dKki+iREtdGwYUPi4uIMr4IK3edikzi+6C0URWHzH1vZu3cvubm59OjRA51OR+MajkDRYCcjI4OmTZsyasrMIt9VEMDUqVOHefPmERUVxZ49e6gdFEDCT5PJz7zVVmXnyBS3WkvydIQQlaXMwU5sbCwjR47k559/JjAwkC5duvDTTz+Rm5tbEf0T4r5mZmaGp6en4VVQ9XvZuq3c0CbS4cV3ebh5Mxo3bsySJUs4dOgQ27dvp2lNDQDHbq7WKtCtWzemTZvGkz16FfmuguDh6aefpnPnzgQGBtKwYUO+mvcZak4m+VcvGK6r7ByZ2zcLlDwdIURlKnOw4+rqypgxY4iMjCQ8PJw6derw2muv4e3tzRtvvMFff/1VEf0UotLcbfopPj6eoUOH4unpia2tLc2bN+eXX34ptq2zZ8/i7e2Nt7c3NWrUwMPDA0VR2LpdPx3cIvBWBXBHR0dUVeXxxx9ncGs/Ls56kuUvBzN79myjNlVVZc4fZ42OlRQ85ObmsmDBAjQaDVs+HMryl9qwZ2KnIpXVK8PAVr7smdipSvsghHgw/adNBZs3b05YWBgjR44kPT2db7/9lhYtWvDII4/w999/l1cfhah0JU0/ATz77LOcPn2adevWERUVRd++fRkwYABHjxrvfty6dWu+++47Nm3axGuvvYaiKOh0OgBSzV1RzK2I+OlzMjMzycjIYPjw4QA888wzxFyOxePJ0YDCw51uFbmM02Yx4/eTbD+lr04+5vE6xQYPGzZswM7ODisrK+bMmcPWrVtpFOhT5Rv1yWaBQoiq8K+Cnby8PH7++WeeeOIJ/Pz82Lx5M/PmzSMhIYFz587h5+dnVPDvv4iNjeWZZ57BxcUFa2trGjduzKFDhwznVVVlypQpeHl5YW1tTefOnTl79uwdWhTi7kqafgLYt28fr7/+Og8//DCBgYG88847ODo6cvjwYaM2unXrRv/+/WnSpAnvvPMOx48fN0z3JmSb4NZ7Itu3bMTOzg6NRkNubi7NmzfHzs4OnxreKDGHsfJrjNbcGdCvvmo3czsL/jxv+I76Xg7FBg+dOnUiMjKSffv20bVrVwYMGEBiYmJFPS4hhLinlTnYef311/Hy8uKVV16hTp06HD16lP379/Piiy9ia2uLv78/H330EadOnfrPnUtJSaFdu3aYm5uzceNGTpw4wccff4yT062lurNnz+azzz7jq6++Ijw8HFtbW7p06UJ2dvZ//n7x4CqYfgoMDGTIkCFGRTfbtm3LypUrSU5ORqfTsWLFCrKzs3n00Ufv2KajoyMBQbUM760DmlPjlW/wff1Hjkdf4ocffiA2NpbAwEASEhJI/Hs/dk1COXZZW2wBT4DkjJxiv8vW1pZatWrRpk0bFi1ahJmZGYsWLfrXz0MIIe5nZd5n58SJE3z++ef07dsXS0vLYq9xdXUtlyXqs2bNwsfHh8WLFxuOBQQEGP6sqipz587lnXfeoVcvfcLm999/j4eHB2vXrmXQoEH/uQ/iwVMw/VS3bl3i4uJ4//33eeSRRzh+/Dj29vb89NNPDBw4EBcXF8zMzLCxsWHNmjXUqlXrju2mp6fzzz//FD1h7UDKDQu2b99OYmIiPXv2ZMmSJdjY2mJTpy3HLqcWW8ATIPF66RYG6HQ6cnKKD4yEEKK6K/PIzrZt2xg8eHCJgQ7opwA6duz4nzoGsG7dOlq2bEn//v1xd3enWbNmLFy40HD+/PnzxMfH07lzZ8MxjUZD69at2b9/f3FNApCTk0NaWprRS4gChaefunTpwu+//05qaio//fQTAJMnTyY1NZU//viDQ4cOMXbsWAYMGEBUlPGmeePGjWPXrl1cuHCBffv20adPHxTl1r9y6ce2khN7Cl1qPIe3/Ur//v0ZM2YMdevW5dtvv6X3UwNJ/Pk9/ly71FBoSpebRW7CP+Qm6IOmnOQ4IiMjDSNPGRkZTJo0iQMHDnDx4kUOHz7M8OHDiY2NLbepZSGEuO+oZTR9+nR10aJFRY4vWrRInTlzZlmbuyNLS0vV0tJSDQsLU48cOaJ+/fXXqpWVlfrdd9+pqqqqe/fuVQH1ypUrRp/r37+/OmDAgBLbfffdd1X0Px9GL61WW679F9VHy5Yt1YkTJ6rnzp1TAfX48eNG50NCQtRXXnnF6NjAgQNVLy8v1cLCQq1Ro4Y6cOBA9bEpK1RAdevzturQ+inV1NZRNTUzV2vXrq1+/PHHqk6nU3fv3q0C6uEjR1VzjbuqaTdYbTF1i+o3YYPqMXh6sf/sDhs2TFVVVc3KylL79Omjent7qxYWFqqXl5fas2dP9eDBg5X1qIQQotJotdpS/X6XeRrr66+/ZtmyZUWON2zYkEGDBjFhwoR/F3UVQ6fT0bJlS6ZPnw5As2bNOH78OF999RXDhg371+2GhYUxduxYw/u0tDR8fHz+c39F9ZSenk50dDRDhw4lM1O/67CJifGgqKmpqWGlVYEVK1YYvd/ydzwv/6BPYp78ZH1adhxTbHHNRYsW0aJFC5o3e4h276zkfFIGSen66ao3h/Wh/dSXSyzKaWVlxerVq//bDQshRDVT5mms+Ph4vLy8ihx3c3MjLi6uXDpVwMvLiwYNGhgdq1+/vmHI3tPTE4CEhASjaxISEgznimNpaYmDg4PRS4gCxU0/mZqaMnjwYH0Zhlq1eOWVVzh48CDR0dF8/PHHbN26ld69exvaCAkJYd68eYb3Zy4lMmHBOsP00w1tItbXL5GnvWr03WlpaaxatYoXX3yROG0WF5IyjM5/tfOfu1YfF0IIYazMwY6Pjw979+4tcnzv3r14e3uXS6cKtGvXjtOnjWv6nDlzBj8/P0CfrOzp6cm2bdsM59PS0ggPDyc4OLhc+yIeHJcvX2bw4MHUrVuXAQMG4OLiwoEDB3Bzc8Pc3Jzff/8dNzc3evToQZMmTfj+++9ZsmQJTzzxhKGN6OhokpKSAP2S8Q4TFhH52SvEffcGAGPHjqVZs2ZMmTLF6LtXrFiBqqoMHjyY80kZ3J6TXFk1rYQQojop8zTWSy+9xOjRo8nLy+Oxxx4D9EnLb731Fm+++Wa5dm7MmDG0bduW6dOnM2DAAA4ePMiCBQtYsGABAIqiMHr0aKZNm0bt2rUJCAhg8uTJeHt7G/2/bCHK4vbpp9vVrl27xB2TC+z/6yTnkzI4+M81Jv4ShZVvE/wmbAD0ux3vmdip2NGZl19+mZdffhmAALIwUTBahSX1pIQQouzKHOyMHz+ea9eu8dprrxk2SLOysmLChAmEhYWVa+datWrFmjVrCAsL44MPPiAgIIC5c+cyZMgQwzVvvfUWGRkZvPzyy6SmptK+fXs2bdqElZVVufZFiNJaGRFT7J44BQpGZ+42FVVQT2rS6uPkq6rUkxJCiH9JUVW1hP8k31l6ejonT57E2tqa2rVr33Ep+r0uLS0NjUaDVquV/B3xn8Rps2g3c3uJgQ7ceWSnpDYvJGVKro4QQtymtL/fZR7ZKWBnZ0erVq3+7ceFqJZK2vyvYDrq34zOeGmsJcgRQoj/4F8FO4cOHeKnn34iJibGMJVVQJa9igdZgKttsXk2q18LJjNXJ6MzQghRBcq8GmvFihW0bduWkydPsmbNGvLy8vj777/Zvn07Go2mIvooRJWJ02axLzqJOG1Wqa730ljzSodAw3tTBab3bURTHyep9i2EEFWkzCM706dPZ86cOYwYMQJ7e3s+/fRTAgICeOWVV4rdf0eI+1XhRGMTBWb0bczAVr53/ZzGxgKAVv5OfDa4mQQ4QghRxco8shMdHU337t0BsLCwICMjA0VRGDNmjGFJuBD3u9urjOtUCFsdxV+XUu460nP4YgoAoQ08JdARQoh7QJlHdpycnLh+/ToANWrU4Pjx4zRu3JjU1FTDVvpC3O+KSzTWqdD7i32olDzSo6oqR24GO839nCqpt0IIIe6kzCM7HTp0YOvWrQD079+fUaNG8dJLLzF48GBCQkLKvYNCVIUAV1uUYo4XxD86FSatPl5khOfitUyuZeRiYWZCoxqyjYEQQtwLyjyyM2/ePLKzswF4++23MTc3Z9++ffTr14933nmn3DsoRFXw0ljT1EdD5CUtAAqUWLqh8FTVoZujOk1qaLA0M62k3gohhLiTMgU7N27cYMOGDXTp0gXQV36eOHFihXRMiKqWlnUDgAld69I2yIU+X+4zmtoyUShSuqEgX6eFTGEJIcQ9o0zTWGZmZrz66quGkR0hqqu07Dz+uVlxfGArX5r6ODGjb2NMCs1t1fWw53xShtFUluTrCCHEvafMOTsPP/wwkZGRFdAVIf6b9957D0VRjF716tUznF+wYAGPPvooDg4OKIpCampqiW0dj9VPX3naqHzw9lv4+fnxXIe6OG+fyvN18lGAk/HXGfzVXup3GYpvrXrY2tqybUofkjZ8jLd56fblEUIIUfHKnLPz2muvMXbsWC5dukSLFi2wtbU1Ot+kSZNy65wQZdWwYUP++OMPw3szs1v/iGdmZtK1a1e6du1616K1xy7rg52k3z9j6/VYfvjhB7y9vVm6dCkfjxmK5pnPMLV3Rb2RQ058NDkP9WHm7K5MX3OIjF3f8PzT/Tl06FDF3KQQQogyKXMhUBOTooNBiqKgqiqKopCfn19unassUgi0enjvvfdYu3btXUced+7cSadOnUhJScHR0bHYa0b8eIT1Ry4QO3cA69b9athbCqBeo6bEOdTHqcNQo8/0bebN6qNXaOuQyvK3n+HixYv4+t59E0IhhBD/ToUVAj1//vx/6pgQFens2bN4e3tjZWVFcHAwM2bM+FcBx7HYVNDlo9PlY2VlZXTOwd6Wi5f/LvKZiAvJALha3kBRlBIDKSGEEJWrzMGOn59fRfRDiP+sdevWfPfdd9StW5e4uDjef/99HnnkEY4fP469vX2p20nOyOVSchYmljY83LoNU6dOpX79+nh4eLB8+XIOHwzHo6Y/popCfqGB0Usp2ag3cvn6/6bSNrSnjBIKIcQ9oszBzvfff3/H888+++y/7owQ/0W3bt0Mf27SpAmtW7fGz8+Pn376iRdeeKHU7UTdTE4OdLVl4Y9LGT58ODVq1MDU1JTmzZszePBgDh8+zPaJnbiQlMmNfB1Dvz2Imn+Dq7/OBCC2wTPEabOkXIQQQtwDyhzsjBo1yuh9Xl4emZmZWFhYYGNjI8GOuGc4OjpSp04dzp07V6bPHbuUCkCTmhqCgoLYtWsXGRkZpKWl4eXlxcCBAwkMDMRLY42Xxpp90UmGQOeGNhGPwdNRLayLbDgohBCiapR56XlKSorRKz09ndOnT9O+fXuWL19eEX0U4l9JT08nOjoaLy+vMn3u2M2RncY1HQ3HbG1t8fLyIiUlhc2bN9OrVy/DuZoaC5J+ncmNlCt4DPoQU2sHTBWlyIaDQgghqkaZR3aKU7t2bWbOnMkzzzzDqVOnyqNJIcps3Lhx9OjRAz8/P65cucK7776LqakpgwcPBiA+Pp74+HjDSE9UVBT29vb4+vri7OwMQEhICNE2DaBhV5rU1LB582ZUVaVu3bqcO3eO8ePHU69ePZ5//nlAP7L5+gtDsdJexKnHJNDpUDNSeOuJerhYS7kIIYS4F5RLsAP6/UyuXLlSXs0JUWaXL19m8ODBXLt2DTc3N9q3b8+BAwdwc3MD4KuvvuL99983XN+hQwcAFi9ezHPPPQfA2XPnSPPxwFmBht4ObDioJSwsjMuXL+Ps7Ey/fv348MMPMTc3ByA2NpZ169bpG1z8uqHtV+dB3R07ePTRRyv+xoUQQtxRmffZMfyH/SZVVYmLi2PevHn4+PiwcePGcu1gZZB9dkSBnyIu8dYvxwh0tWX7uEerujtCCCHuoML22endu7fRe0VRcHNz47HHHuPjjz8uc0eFuFesjIhh4i9RAPyTlMHKiBgGtpJNAYUQ4n5X5mBHp9NVRD+EqFJx2izCVkdReJhz0urjdKjjJiuqhBDiPlfm1VhCVEfnkzLQ3Tahm6+qXEjKrJoOCSGEKDdlDnb69evHrFmzihyfPXs2/fv3L5dOCVHZlGKOyfJxIYSoHsoc7OzevZsnnniiyPFu3bqxe/fucumUEJXpUnIG76w5bnTMVFGY3reRTGEJIUQ1UOacnfT0dCwsLIocNzc3Jy0trVw6JURlKUhKLpjBGtmpFu1queLvaiOBjhBCVBNlHtlp3LgxK1euLHJ8xYoVNGjQoFw6JURliNNmMfG2pOT5O6Ml0BFCiGqmzCM7kydPpm/fvkRHR/PYY48BsG3bNpYvX86qVavKvYNCVJTzSRncvstUQVKyBDtCCFF9lDnY6dGjB2vXrmX69On8/PPPWFtb06RJE/744w86duxYEX0UokKk59wockySkoUQovr5V+UiunfvTvfu3cu7L0JUqh/2XwT0K7FUJClZCCGqqzIHOxEREeh0Olq3bm10PDw8HFNTU1q2bFlunROiohw8n8yfZ5MwM1FY+Uobcm+okqsjhBDVVJkTlEeMGMGlS5eKHI+NjWXEiBHl0ikh/qs4bRb7opOI02YVOXclNZMpv+qXmvdv6UMLP2eCg1wk0BFCiGqqzCM7J06coHnz5kWON2vWjBMnTpRLp4T4L1ZGxBC2OgqdCiYKzOjb2FDjamVEjH4F1s3EZH8Xyc8RQojqrswjO5aWliQkJBQ5HhcXh5nZv0oBEqLcFNS4Kij9oFP1Na7itFm36l8VWoE1e9PpYkd/hBBCVB9lDnZCQ0MJCwtDq9UajqWmpjJp0iQef/zxcu2cEGV1pxpXUv9KCCEeTGUeivnoo4/o0KEDfn5+NGvWDIDIyEg8PDz44Ycfyr2DQpRFgKttkWMmCvi72nApuWhQI0vNhRCi+itzsFOjRg2OHTvGjz/+yF9//YW1tTXPP/88gwcPxtzcvCL6KESpeWmscbOz4Gp6ruHYI7Vdcbe3YsSPR4yulaXmQgjxYFBU9fY9ZB88aWlpaDQatFotDg4OVd0d8R+kZOTSbOpWAF7pEMjXu/8BoG2QC/uir2Fnacb3w1uRI0vNhRDivlfa3+8y5+wUOHHiBJs2bWLdunVGLyH+i5kzZ6IoCqNHjy5yTlVVunXrhqIorF27ttjPH72Ugpp/AzV8KcsmDODynKe4/MWzrPt0EjeuX6NrI0+a31xq/srQgfj6+mJlZYWXlxdDhw7lypUrFXuDQgghKl2Zp7H++ecf+vTpQ1RUFIqiUDAwpCgKAPn5+eXbQ/HAiIiI4Ouvv6ZJkybFnp87d67hn7OSHL6YgnojB5Nr5xn55gSm7ssgPyud5G0LuLp6Kmsc5vJmaB28NNZ06tSJSZMm4eXlRWxsLOPGjeOpp55i3759FXF7QgghqkiZR3ZGjRpFQEAAiYmJ2NjY8Pfff7N7925atmzJzp07K6CL4kGQnp7OkCFDWLhwIU5OTkXOR0ZG8vHHH/Ptt9/esZ3DF1MwsbRl5jc/0fiRrpg518SyRj2cH3+V3Phz5GgTDauvxowZQ5s2bfDz86Nt27ZMnDiRAwcOkJeXVyH3KIQQomqUOdjZv38/H3zwAa6urpiYmGBiYkL79u2ZMWMGb7zxRkX0UTwARowYQffu3encuXORc5mZmTz99NN88cUXeHp6lthGXr6OyEupALTwcyLA1RaTmwNBupxMQMHcyr7Y1VfJycn8+OOPtG3bVhLthRCimilzsJOfn4+9vT0Arq6uhhwHPz8/Tp8+Xb69Ew+EFStWcOTIEWbMmFHs+TFjxtC2bVt69ep1x3ZOxqWRnafDwcqMIDc7vDTWzOjbGCU/j9Sdi7Fr0JGZgx82SkqeMGECtra2uLi4EBMTw6+//lqu9yaEEKLqlTnYadSoEX/99RcArVu3Zvbs2ezdu5cPPviAwMDAcu+gqN4uXbrEqFGj+PHHH7Gysipyft26dWzfvp25c+feta3DF1MAaO7nhMnNIZ2+D3lR+/hC/F1sOLxphaFsRIHx48dz9OhRtmzZgqmpKc8++yyyQFEIIaqXMicov/POO2RkZADwwQcf8OSTT/LII4/g4uLCypUry72Dono7fPgwiYmJRvXW8vPz2b17N/PmzeN///sf0dHRODo6Gn2uX79+PPLII0Z5YgXBTks/fc5PXl4eAwYMIOHKZfbt3oGLi0uR73d1dcXV1ZU6depQv359fHx8OHDgAMHBweV/s0IIIapEmYOdLl26GP5cq1YtTp06RXJyMk5OTnddKSPE7UJCQoiKijK8v3o9m9deeYmG9evz3uRJuLq68sorrxh9pnHjxsyZM4cePXoYHT9SaGSnINA5e/YsO3YUH+jcTqfTAZCTk/Nfb0sIIcQ9pFwqdzo7O5dHM+IBZG9vT6NGjYCb1crXXeRK6g2u/JNB/ywHBnp6FpuU7OvrS0BAgOF9rTp1SWv0FPb12tHAw5annnqKI0eOsGHDBvLz84mPjwf0/6xaWFgQHh5OREQE7du3x8nJiejoaCZPnkxQUJCM6gghRDXzrzcVFKI83V6tXC1Urbw0os+eQZeTSS13W1KTEli3bh2XL1/moYcewsvLy/Aq2EPHxsaG1atXExISQt26dXnhhRdo0qQJu3btwtLSsqJuUwghRBUol5EdIf6rwhXJPZ+eCdyqSH57SYfbE4hXRsTgP2EDKnAmPp3wqyZ3TTJu3Lgx27dvL7f+CyGEuHfJyI64JxRXrbw0FckLRoQKQhuVso0ICSGEqP7KHOzs3r2bGzduFDl+48YNdu/eXS6dEg8eL401fs63AhsThVJVJC88IlSgYERICCGEgH8R7HTq1Ink5OQix7VaLZ06dSqXTokHU0burbpqY0PrFtkTJ06bxb7oJKNRm387IiSEEOLBUeacHVVVi11ifu3aNWxti/7wCFEaGTk3SEq/teT7xBWt0fmVETGGBGYTBWb0bczAVr5cSc02us5UUUo1IiSEEOLBUepgp2/fvoC+uvlzzz1ntGIlPz+fY8eO0bZt2/LvoXggxCQbTzsd+CcZnU7FxEQpslJLd3OlVoc6bszZegaAJ5t4MqS1P/6uNhLoCCGEMFLqaSyNRoNGo0FVVezt7Q3vNRoNnp6evPzyyyxdurQi+yrKwfz582nSpAkODg44ODgQHBzMxo0bDeejo6Pp06cPbm5uODg46HcgTki4Y5vXr19n9OjR+Pn5YW1tTdu2bYmIiDC6RlVVpkyZgoenJ5ZW1jzy6GOcPXvWcP7iNX2wU9/LARsLU5IzcjmdcB0oOS/nyx3n2HMuCXNThQld6xMc5CKBjhBCiCJKPbKzePFiAPz9/Rk3bpxMWd2natasycyZM6lduzaqqrJkyRJ69erF0aNH8ff3JzQ0lKZNmxqWZU+ePJkePXpw4MABTEyKj41ffPFFjh8/zg8//IC3tzdLly6lc+fOnDhxgho1agAwe/ZsPp7zKXahb+Ci8eDIn0tp/2gIF6PPYGVlRUyyvgRJkJstbvaW7D5zlf3R16jv5YCHfdGaWQA/HIgBoLmvEz7OkqMjhBCieIpaxqqHWVlZqKqKjY3+x+XixYusWbOGBg0aEBoaWiGdrGhpaWloNBq0Wi0ODg5V3Z1K5+zszP/93//h4+NDt27dSElJMTwHrVaLk5MTW7ZsoXPnzkU+m5WVhb29Pb/++ivdu3c3HG/RogXdunVj2rRpqKqKp5cXNxp0x/5h/XSoLieDy58/wxcLvuF/w5/l7TVR/Bgew4hOQdhbmTNz4yk61/fgm2EtWRYew6Q1UUW+u4CJAnsnPiajOkII8YAp7e93mVdj9erVi++//x6A1NRUHn74YT7++GN69erF/Pnz/32PRaXLz89nxYoVZGRkEBwcTE5ODoqiGOVjWVlZYWJiwp49e4pt48aNG+Tn5xepWG5tbW34zPnz50lMSMDS7yHDeRNLWyy867Jj917gVs6On7MtwYH6Olbh56+Rr1NZeuAiAG88Vot3utcv0gediiw1F0IIUaIyBztHjhzhkUceAeDnn3/G09OTixcv8v333/PZZ5+VewdF+YuKisLOzg5LS0teffVVw8hcmzZtsLW1ZcKECWRmZpKRkcG4cePIz88nLi6u2Lbs7e0JDg5m6tSpXLlyhfz8fJYuXcr+/fsNnymoS2Vq62j0WTNbR7LTrgG3cnZ8XWxo6O2AvZUZ17Nv8GP4RU7EpWFpZsLw9gF0b+KFyW2LAWWpuRBCiDspc7CTmZmJvb09AFu2bKFv376YmJjQpk0bLl68WO4dFOWvbt26REZGEh4ezv/+9z+GDRvGiRMncHNzY9WqVaxfvx47Ozs0Gg2pqak0b968xHwdgB9++AFVValRowaWlpZ89tlnDB48uMhnAtyM87wa19BgbW5KXr6O2FT93jl+LjaYmZrQOkBfXHbmxlMAPNnEG0cbC7w01szo2xjTm9sfyFJzIYQQd1PmfXZq1arF2rVr6dOnD5s3b2bMmDEAJCYmPpD5LvcjCwsLatWqBehzayIiIvj000/5+uuvCQ0NJTo6mqSkJMzMzHB0dMTT05PAwMAS2wsKCmLXrl1kZGSQlpaGl5cXAwcONHymoGr5uQuxWHjoj5mbKJjnXsfTM4grqVnk61QszEwMycjBQa78cTKRzJsbDQ5pc2uDwYGtfOlQx40LSZmy1FwIIcRdlXlkZ8qUKYwbNw5/f38efvhhgoODAf0oT7Nmzcq9g6Li6XQ6cnJyjI65urri6OjI9u3bSUxMpGfPnndtx9bWFi8vL1JSUti8eTO9evUCICAgAHsnV7IvRtKhjhuONubkZGUQfjCc4OBgQ76Or7MNJjfnqNKy8ozaPhN/3ei9l8ZalpoLIYQolTIHO0899RQxMTEcOnSIzZs3G46HhIQwZ86ccu3c7WbOnImiKIwePdpwLDs7mxEjRuDi4oKdnR39+vW7674wD7KwsDB2797NhQsXiIqKIiwsjJ07dzJkyBBAv8XAgQMHiI6OZunSpfTv358xY8ZQt25dQxshISEMHDjQsF+Pra0t9evXZ/HixWzdupV27dqh1Wp55ZVXUBQFExMTrqckkbLjW7L3LqGmepWk3z5B4+JO7969Dfk6F1f/H4qioCgKY0PrcnHWkyT8NAWAt9foi3smJyczZMgQHBwccHR05IUXXiA9Pb3yH6QQQoj7RpmnsUA/LZGens7WrVvp0KED1tbWtGrVqtgyEuUlIiKCr7/+miZNmhgdHzNmDL/99hurVq1Co9EwcuRI+vbty969eyusL/ezxMREnn32WeLi4tBoNDRp0oTNmzfz+OOPA3D69GnCwsJITk7G39+ft99+2zBVWSA6Ohp7JxeeHRVGcLNGROzexvvvv8/w4cNxcXFhwIABrF271jCtufJgDO/O/pS0Az+xc/0qUlK/wcy7Pr0mzLu5x44+2LG1NKNr16689u5HvLb0qP7LzMyBW8U9Pxj5DHFxcWzdupW8vDyef/55Xn75ZZYtW1ZJT1AIIcT9psz77Fy7do0BAwawY8cOFEXh7NmzBAYGMnz4cJycnPj444/LvZPp6ek0b96cL7/8kmnTpvHQQw8xd+5ctFotbm5uLFu2jKeeegqAU6dOUb9+ffbv30+bNm1K1f6Dvs9OWZVUp6pgv54XXnjB6NoJv0RxZfEbWHoE8c2ib3C0seCVHw5T38uBjaMe4ZUfDrH57wRqHluMq8UN5i9ZTruZ2412TTZVFBb38aJj6+ZERETQsmVLADZt2sQTTzzB5cuX8fb2ruxHIYQQogpV2D47Y8aMwdzcnJiYGMPGggADBw5k06ZN/663dzFixAi6d+9eZFO7w4cPk5eXZ3S8Xr16+Pr6sn///hLby8nJIS0tzeglSqe4OlVhP//F/G+/N+zXU/jaiaujyIk/R17iP9g2CWXS6uN4Ouj38TmTcJ3svHzDNJadpRk7d+6kaW0/spe9QfKWL8nPSjOsuDp3/CiOjo6GQAegc+fOmJiYEB4eXnkPQQghxH2lzNNYW7ZsYfPmzdSsWdPoeO3atStk6fmKFSs4cuRIkVpLoN+/xcLCAkdHR6PjHh4ehr1dijNjxgzef//98u7qA6FwnarcqxeI/2Ec6o1c3rKzM+zXU2DHqURUFdKPbcHcxQermvXJV1Uyc/Nxs7fk6vUc/r6SZpjG6tatK689/zQBAQFER0fz1sQwzHb+H1t27Kamsx3Tt8bj7u5u1B8zMzOcnZ3v+PcthBDiwVbmYCcjI8NoRKdAcnKy0c675eHSpUuMGjWKrVu3Ftmh978ICwtj7NixhvdpaWn4+PiUW/vVWYDrrb1yzJ1r4PX8Zyi5mfRxusywYcPYtWsXTjUC2HYygRkbT6HLyyHjxC4c2w4ECjYAtKVJDQ3bTiWy45R+ebmiwKvDh2JpZgpA48aNadKkCUFBQZw+Gk7NkJAquV8hhBD3vzJPYz3yyCOGchEAiqKg0+mYPXs2nTp1KtfOHT58mMTERJo3b46ZmRlmZmbs2rWLzz77DDMzMzw8PMjNzSU1NdXocwkJCYa9XYpjaWlpqPpd8BKl46WxxtXOAgDF1BxLZ28+fq0Pn3/yfzRt2pQ33vmQtjO3887av8nIycfkwgHUvBxsG4UYbQDYuKYGgA3HrgDgrbE2BDoFAgMDcXV15dy5c4A+MT4xMdHomhs3bpCcnHzHv28hhBAPtjKP7MyePZuQkBAOHTpEbm4ub731Fn///TfJycnlvgIqJCSEqCjjApDPP/889erVY8KECfj4+GBubs62bdvo168foF9NFBMTY5Q7IspPckYuSem5hvcDW/owsJV+w7/s3Bv8dTYBlzq3rk+I2EjnLt34YFQXow0Am9Z0BODCtVt77Nzu8uXLXLt2DS8vLwCCg4NJTU3l8OHDtGjRAoDt27ej0+lo3bp1ud+rEEKI6qHMwU6jRo04c+YM8+bNw97envT0dPr27cuIESMMP0rlxd7enkaNGhkds7W1xcXFxXD8hRdeYOzYsTg7O+Pg4MDrr79OcHBwqVdiibI5GpMCQMqu77AObMlh2yyi6igsW7aMfXt249b/A8O1eSlXyL70N0+8N5ngIBejdhrX1BC78FWcOj6LTZ22eNnA+PHj6devH56envqcnbfeolatWnTp0gWA+vXr07VrV1566SW++uor8vLyGDlyJIMGDZKVWEIIIUpU5mAnJiYGHx8f3n777WLP+fr6FvOpijNnzhxMTEzo168fOTk5dOnShS+//LJS+/AgOXxRH+y4mGYTs+ETtq5MJuRzJ5o0acKy1esIO6BQsGI8/dhWTB1cGdCne5F2XO0suZF8GV3OzZEdNzs2HjvGkiVLSE1Nxdvbm9DQUKZOnWqUC/bjjz8ycuRIQkJCDH/vUoBWCCHEnZR5nx1TU1Pi4uKKrIq5du0a7u7u5Ofnl2sHK4Pss1N6A7/eT/j5ZGb2bcxHW86QlJ7DqleDaeXvjKqqtJj2B8kZ+mmughydgmmu2736w2E2/a1fRTW1d0OGtvGvrNsQQghRDZT297vMIzuqqha7U3J6enq5rpgS9568fB1/XU4FoKW/Ey39nNj0dzyHL6bQyt+Z6KsZJGfkYm6isODZltTzsr9j7SqVW3H2lF//xsLUpMTASAghhPi3Sh3sFCzVVhSFyZMnGy0/z8/PJzw8nIceeqjcOyjuHSeupJGdp8PRxpxAVzta3Ax2Dl1IgY76fXUA2gS50Kme+x3bitNmseXErRpmqgqTVh+nQx03Ke4phBCiXJU62Dl6VF+rSFVVoqKisLCwMJyzsLCgadOmjBs3rvx7KO4ZBfk6zX2dMDFRaO7nBMCRmBRUVWX7zWCnU907Bzqg35zw9gnUgvpXEuwIIYQoT6UOdnbs2AHol35/+umnktvyADp8cyVWi5tBTqMaDliYmZCckcvx2DQiLiQD8NhdRnVAvzmhiUKR+lf+rkWXoAshhBD/RZk3FVy8eLEEOg+oI4VGdgAszUxpUkO/OeCn285wQ6cS6GqLf6FdlkvipbFmRt/GmN7M/yq84aAQQghRnsqcoCweTFdSs4jTZmNqotDUR2M43sLPiUMXU/jj5M0prFKM6hQY2MqXDnXcuJCUabThoBBCCFGeJNgRBnHaLM4nZRDgalsk8Nh6Up9MXNvdDhuLW//YFOTtFCjNFFZhXhprCXKEEEJUqDJPY4mqMX/+fJo0aWKo5RUcHMzGjRsBfRHW119/nbp162JtbY2vry9vvPEGWq32ru2ePHmSnj17YmPnQA03Jzq1b0vrsBWsjIgBIDo6mocf7crwkKbEzOnPn1+9zdebDhs+f+lmxfICF5IyyvGuhRBCiP9ORnbuEzVr1mTmzJnUrl0bVVVZsmQJvXr14ujRo6iqypUrV/joo49o0KABFy9e5NVXX+XKlSv8/PPPJbYZHR1N+/btGfjMMJz6dwYLG/KSYlBNLQj7JQrT/FzGDHycZCsvPAZPByD1z6WMGv40T/59BBMTE6b/ftKozSm//s1j9d1ltEYIIcQ9o8w7KFdH9+sOys7Ozvzf//0fL7zwQpFzq1at4plnniEjIwMzs+Jj2kGDBmFubs7g8bN4bdnRIuezzh8hcdV7+IxagYmlfpWULieDS3MH8emSn2nZrgNPLwwv8rnlL7UpUgtLCCGEKG+l/f2Waaz7UH5+PitWrCAjI6PE6u4Ff/ElBTo6nY4Nv/2G6uDJ00/15NLnQ4j7fiyZZ/YbrlHz8wBQTM0NxxRTC1AULp44bFg+XpgsHxdCCHGvkWDnPhIVFYWdnR2Wlpa8+uqrrFmzhgYNGhS5LikpialTp/Lyyy+X2NbCzUfISE9n2cLPMfFphu/TH2JbJ5ira6aTHRMFgKV3PRRzK1J2LoYb2ehys0ndsQhUHekpSbJ8XAghxH1BcnbuI3Xr1iUyMhKtVsvPP//MsGHD2LVrl1HAk5aWRvfu3WnQoAHvvfdese3EabP48LcTAFjXaoNDq94oCuyY8jQvD73C35EbsfJtjKmNBrfeE0ne8iUxR9ZjYmJC734DOE9zTEz0cbIsHxdCCHGvk2DnPmJhYUGtWrUAaNGiBREREXz66ad8/fXXAFy/fp2uXbtib2/PmjVrMDc3L7ad80kZKNYOYGKKuasPoN/JODNXx6Otm5G6aRs3FIV8VcUusAWfbQknJMAGMzMzHB0d8fT0JDAw0NCeLB8XQghxL5Ng5z6m0+nIyckB9CM6Xbp0wdLSknXr1t2xAr2PkzWKqTmWnrW5kRwL3Mq1OXPmDC0a1uHjiZ2KHa3Zvn07iYmJ9OzZs2JvTgghhCgnEuzcJ8LCwujWrRu+vr5cv36dZcuWsXPnTjZv3kxaWhqhoaFkZmaydOlS0tLSSEtLA8DNzQ1TU1MA6tWrx4wZM9D5tgLAoXVfrv46G2ufRrzz0lP88sMi1q9fz86dOw2jNYsXL6Z+/fq4ubmxf/9+Ro0axZgxY6hbt26VPQshhBCiLCTYuU8kJiby7LPPEhcXh0ajoUmTJmzevJnHH3+cnTt3Eh6uXwJeMM1V4Pz58/j7+wNw+vRpklNSWHL2LABvvjyUtIc9WL7wMyYOWUjdunX55ZdfaN++veHzp0+fJiwsjOTkZPz9/Xn77bcZM2ZM5dy0EEIIUQ5knx3u3312/o2vd0YzY9MpnGzM2TvxMaPSD0IIIcT9RPbZEUUsPXCRGZtOAZCamcf6v65UcY+EEEKIiifBTjUTp81iX3QScdqsIscnrz1ueK8Ck1YfL3KdEEIIUd1IsFPBZsyYQatWrbC3t8fd3Z3evXtz+vRpo2uio6Pp06cPbm5uODg4MGDAABISEu7YbnGFQSd+uoR2M7fz9MJw2s3cTue+QwgKCsLa2pr6AT4k/DKVvGuXDG3kqyoXkjLv8C1CCCHE/U+CnQq2a9cuRowYwYEDB9i6dSt5eXmEhoaSkaGvDp6RkUFoaCiKorB9+3b27t1Lbm4uPXr0QKfTldhuQWHQw4cPc+jQIR5u9wizxr5IduJFQL9vzl9Zznz0+VecPHmSF6YtBFQSVk5B1eUDUtpBCCHEg0ESlKncBOWrV6/i7u7Orl276NChA1u2bKFbt26kpKQYvlur1eLk5MSWLVvo3LlzqdrdF53EI40CcHx0OPZNQw3Hl7/UhqY+GjrM3sGVf04Tt/h1vF9eiJWzN9P7NmJgK98KuU8hhBCiopX291uW4lQyrVYL6CuWA+Tk5KAoCpaWloZrrKysMDExYc+ePaUKdvLz8/lr1+/o8rKxrFHP6FxNJyu+33+RxOQ0lLM78fHzZ+no7tT2dpRdj4UQQjwQJNipRDqdjtGjR9OuXTsaNWoEQJs2bbC1tWXChAlMnz4dVVWZOHEi+fn5xMXF3bG9qKgogoODyc7Oxs7ODu9+72DuajxS81LYDLYu/hhdXjbefkHs2PYHQUFeFXaPQgghxL1GcnYq0YgRIzh+/DgrVqwwHHNzc2PVqlWsX78eOzs7NBoNqampNG9+q9hmSQoKg4aHh/NIz6eJX/8JLnkJLH+pNe/2qA/ACZsmeD73KR5Pz8TTJ4ABAwaQnZ1dofcphBBC3EtkZKeSjBw5kg0bNrB7925q1qxpdC40NJTo6GiSkpJKLLZZnILCoKqqktc8A4vd+7A79wfBQcPxc7Hh/fUnMbG0xcTSFnPnGqTUqEf8F0+zZs0aBg8eXJG3K4QQQtwzJNipYKqq8vrrr7NmzRp27txJQEBAide6uroCZS+2eSQmhVPx1zFBpYa9/q/0wrWiS8rzdSo6nWooHiqEEEI8CCTYqWAjRoxg2bJl/Prrr9jb2xMfHw+ARqPB2lqfIFyaYpshISH06dOHkSNHAsaFQWcuCydl189kxkTx/LCPADBJTyTtwE9Y+jfH1MaBG2nXuB6+Chtra5544olKfgpCCCFE1ZFgp4LNnz8fgEcffdTo+OLFi3nuueeAuxfbjNNmceL0WR66fCthuaAwaOyVOHTm1li4+eMx4AOSHfUBkq+bI355MUStWkd+djqmto60aduORRu+x93dvWJvWgghhLiHyD473NuFQFdGxBC2OgqdCiYKzOjb2LA3Tpw2i7YztlP4L9BUUdgzsZNhWXmcNosLSZn4u9rIUnMhhBDVihQCvc/FabNY/1esIdAB/a7IhetZHb+cxu2R6u0lILw01gQHuUigI4QQ4oEl01j3oMKjObcrCGa8NNbsPZdU5LyUgBBCCCGMycjOPSZOm1VioAOgKODvakNyRi6rDuuLepoo+nOmisL0vo1kFEcIIYQoREZ27jHnEtNLDHRAH9Bk5OTz3d4LZOTm06iGAwuGtuDitSzJyxFCCCGKIcHOPaa4qSkT4LPBD/HDgRjCzyfz6tLDXLymr5o+9vE6eDva4O0oU1dCCCFEcWQa6x4Rp81ixcEYvvnzH0A/XQX6kZwZ/RrzZNMafDqoGVZmJpxLTCcvXz/8k5gmGwQKIYQQdyIjO/eA2xOSG3g58M2wolNTKio5N3RGn317zXE61nWT6SshhBCiBDKyU8WKS0g+FZ+GoihFloyfT8q461JzIYQQQhiTYKeKnU/KKJKQrFMpNoAJcLU1rLwqIEvNhRBCiDuTYKeKBbjaclv8UmIA46WxZkbfxpjeTOiRpeZCCCHE3UnOThXz0lhTz9Oek/HXgbsHMANb+dKhjpuUgBBCCCFKSYKdKqbNzOPc1XQAPnqqCe1qu941gPHSWEuQI4QQQpSSBDtVbPPf8eTlq9TztOeplj5V3R0hhBCi2pGcnSq2/tgVAJ5s4lXFPRFCCCGqJwl2qtC19Bz2RV8D4Mkm3lXcGyGEEKJ6kmCnCm08Hk++TqVxDQ3+rrZV3R0hhBCiWpJgp4rEabP4MfwiIFNYQgghREWSBOUqcHt5CFW9Q5lzIYQQQvwnMrJTyYorD/F/m88Qp82quk4JIYQQ1ZgEO5WsuPIQUt9KCCGEqDgS7FQyP+eiZSCkvpUQQghRcSTYqURx2iy+3v2P0TGpbyWEEEJULElQriS3JyV3aeDBc+0CpL6VEEIIUcFkZKcSFJeU/MfJBAl0hBBCiEogwU4lKD4pGUlKFkIIISqBBDuVIMDVFuW2Y5KULIQQQlQOCXYqgZfGmu6FdkmWpGQhhBCi8kiCciXxdtQHNk808mRyjwYS6AghhBCVREZ2KkmcNhuA5n5OEugIIYQQlUiCnUoSf7MchKfGqop7IoQQQjxYJNipJPFp+pEdLwl2hBBCiEolwU4lUFWVBG0OAB4OEuwIIYQQlemeDnZmzJhBq1atsLe3x93dnd69e3P69Gmja7KzsxkxYgQuLi7Y2dnRr18/EhISqqjHxUvOyCU3X4eigLu9BDtCCCFEZbqng51du3YxYsQIDhw4wNatW8nLyyM0NJSMjAzDNWPGjGH9+vWsWrWKXbt2ceXKFfr27VuFvS6qIDnZxdYSC7N7+pELIYQQ1c49vfR806ZNRu+/++473N3dOXz4MB06dECr1bJo0SKWLVvGY489BsDixYupX78+Bw4coE2bNlXR7SLitZKvI4QQQlSV+2qYQavVAuDs7AzA4cOHycvLo3PnzoZr6tWrh6+vL/v37y+xnZycHNLS0oxeFakgOVlWYgkhhBCV774JdnQ6HaNHj6Zdu3Y0atQIgPj4eCwsLHB0dDS61sPDg/j4+BLbmjFjBhqNxvDy8fGpyK4bRnY8JTlZCCGEqHT3TbAzYsQIjh8/zooVK/5zW2FhYWi1WsPr0qVL5dDDksnIjhBCCFF17umcnQIjR45kw4YN7N69m5o1axqOe3p6kpubS2pqqtHoTkJCAp6eniW2Z2lpiaWlZUV22YiM7AghhBBV554e2VFVlZEjR7JmzRq2b99OQECA0fkWLVpgbm7Otm3bDMdOnz5NTEwMwcHBld3dEsmGgkIIIUTVuadHdkaMGMGyZcv49ddfsbe3N+ThaDQarK2t0Wg0vPDCC4wdOxZnZ2ccHBx4/fXXCQ4OvmdWYsGtkR0PCXaEEEKISndPBzvz588H4NFHHzU6vnjxYp577jkA5syZg4mJCf369SMnJ4cuXbrw5ZdfVnJPS3Y9O4/0nBuATGMJIYQQVeGeDnZUVb3rNVZWVnzxxRd88cUXldCjsku4OYXlYGWGreU9/biFEEKIaumeztmpDgp2T5aVWEIIIUTVkGCnghlWYmmsq7gnQgghxINJgp0KdmvZeeUtdRdCCCHELRLslGD37t306NEDb29vFEVh7dq1Ruffe+896tWrh62tLU5OTnTu3Jnw8PAi7cSl3RrZuX79OqNHj8bPzw9ra2vatm1LREREiX149dVXURSFuXPnluetCSGEEA8UCXZKkJGRQdOmTUtMfK5Tpw7z5s0jKiqKPXv24O/vT2hoKFevXjW6LqFQEdAXX3yRrVu38sMPPxAVFUVoaCidO3cmNja2SPtr1qzhwIEDeHt7l//NCSGEEA8QCXZK0K1bN6ZNm0afPn2KPf/000/TuXNnAgMDadiwIZ988glpaWkcO3bM6LqCBGUnC5VffvmF2bNn06FDB2rVqsV7771HrVq1DEvsC8TGxvL666/z448/Ym5uXjE3KIQQQjwgJNgpB7m5uSxYsACNRkPTpk2NzhUsPXe1NSM/Px8rK+NVWdbW1uzZs8fwXqfTMXToUMaPH0/Dhg0rvvNCCCFENSfBzn+wYcMG7OzssLKyYs6cOWzduhVXV1fD+ey8fK5l5AIQ5O1GcHAwU6dO5cqVK+Tn57N06VL2799PXFyc4TOzZs3CzMyMN954o9LvRwghhKiOJNj5Dzp16kRkZCT79u2ja9euDBgwgMTERMP5xLQcACzNTHC0MeeHH35AVVVq1KiBpaUln332GYMHD8bERP/XcPjwYT799FO+++47FEWpknsSQgghqhsJdv4DW1tbatWqRZs2bVi0aBFmZmYsWrTIcL5wAVBFUQgKCmLXrl2kp6dz6dIlDh48SF5eHoGBgQD8+eefJCYm4uvri5mZGWZmZly8eJE333wTf3//qrhFIYQQ4r4n9QvKkU6nIycnx/A+TpsFgMdtNbFsbW2xtbUlJSWFzZs3M3v2bACGDh1K586dja7t0qULQ4cO5fnnn6/g3gshhBDVkwQ7JUhPT+fcuXOG9+fPnycyMhJnZ2dcXFz48MMP6dmzJ15eXiQlJfHFF18QGxtL//79DZ+Z8EJ/0pyb4PXQKwBs3rwZVVWpW7cu586dY/z48dSrV88QyLi4uODi4mLUD3Nzczw9Palbt24l3LUQQghR/UiwU4JDhw7RqVMnw/uxY8cCMGzYML766itOnTrFkiVLSEpKwsXFhVatWvHnn38araCKu3QBxdofWyv9Y9ZqtYSFhXH58mWcnZ3p168fH374oSwvF0IIISqQopamtHg1l5aWhkajQavV4uDgUC5troyIYcIvUQAowMx+jRnYyrdc2hZCCCFE6X+/JUG5AsRpswhbHWV4rwKTVh835PAIIYQQovJIsFMBzidloLttvCxfVbmQlFk1HRJCCCEeYBLsVIAAV1tMbtsmx1RR8He1qZoOCSGEEA8wCXYqgJfGmhl9G/9/e3ceHkWV7g/8W70v6U4n6c7adBJDQgLZSAIBEmQJmySIsstO1IFhEdkDqAhcB1xgvIiioMLgXKIOEu+IiCIqesEFRMRxGGBw444bMAEGgizJ9/dHbh27WRxnRgbs3/t5nnpIdVVXnT5VdeqtsxQw/t+LAY2ahl/1yUZCpP0Kp0wIIYT4/4+MxrpMBrYK4NoMHz47XIcUr0MCHSGEEOIKkWDnMkqItEuQI4QQQlxh0owlhBBCiLAmwY4QQgghwpoEO0IIIYQIaxLsCCGEECKsSbAjhBBCiLAmwY4QQgghwpoEO0IIIYQIaxLsCCGEECKsSbAjhBBCiLAmwY4QQgghwpoEO0IIIYQIaxLsCCGEECKsSbAjhBBCiLAmwY4QQgghwpoEO0IIIYQIaxLsCCGEECKsSbAjhBBCiLAmwY4QQgghwpoEO0IIIYQIaxLsCCGEECKsSbAjhBBCiLAmwY4QQgghwpoEO0IIIYQIaxLsCCGEECKsSbAjhBBCiLAmwY4QQgghwpoEO0IIIYQIaxLsCCGEECKsSbAjhBBCiLAmwY4QQgghwpoEO0IIIYQIaxLsCCGEECKsSbAjhBBCiLAmwY4QQgghwpoEO0IIIYQIaxLsCCGEECKsSbAjhBBCiLAmwY4QQgghwpoEO0IIIYQIaxLsCCGEECKsSbAjhBBCiLAmwY4QQgghwpoEO0IIIYQIa2ET7Dz88MNISUmBzWZDcXEx3nvvvSudJCGEEEJcBcIi2HnmmWcwefJkzJkzBzt37kReXh66d++Ob7/99konTQghhBBXWFgEO4sXL8att96KUaNGoXnz5nj00UfhcDjw5JNPXumkCSGEEOIKM13pBPyrzpw5g/fffx8zZ85UnxkMBnTp0gVvv/32Rb9z+vRpnD59Ws0fO3YMAHD8+PHLm1ghhBBC/GT0+zbJH1zvZx/sHD58GPX19YiLiwv5PC4uDn/6058u+p0FCxZg7ty5F3zepEmTy5JGIYQQQlw+f/vb3xAZGXnJ5WHRjPWPmjlzJo4dO6am2tpaHDhwAEePHg35/F+dDh48CAA4ePDgBfM/tOxqWFfSJ+mT9F1d+5T0SfrCJX0/5XT06FEcPHgQiYmJ+CE/+5odr9cLo9GIb775JuTzb775BvHx8Rf9jtVqhdVqDfnM4/FcriTC7XbD7XaHzP/QsqthXUmfpE/Sd3XtU9In6Qu39P1UfqhGR/ezr9mxWCwoLCzE5s2b1WcNDQ3YvHkz2rZtewVTJoQQQoirwc++ZgcAJk+ejBEjRqCoqAitW7fGgw8+iJMnT2LUqFFXOmlCCCGEuMLCItgZOHAgDh06hLvuugtff/018vPzsXHjxgs6Lf+7Wa1WzJkzRzWZnT//Q8uuhnUlfZI+Sd/VtU9Jn6QvHNJ3JWj8e+O1hBBCCCF+xn72fXaEEEIIIX6IBDtCCCGECGsS7AghhBAirEmwI4QQQojwRnHZLF26lMnJyTSbzfR4PPR6vQTApk2bMiIigj6fj7m5uczIyKDL5aLL5WKbNm24YcMGkmRZWRkBhEzXXHMNhwwZQoPBcMEyAHS5XLRYLLTb7TSbzQRAo9HIqKgotf/58+czPT1dbcNms6l1AdDv99NkMhEADQYDNU0jAFqtVnq9XrWdmJiYC/avb8disVw0ffpksVhCtm0wGGi1WtVnBoOBJpNJLQ/+V1+uaRqNRiONRiOjo6OZlZV10XyxWCx0OBxqG5ea9O3q2w5eP3i7RqNRLTMajTSbzRes63Q6abPZLrmfH0pHZGSkyn+LxaLyctSoURw7diztdrs6bvqy7OxsZmRkhHzPZrPR4XAQAIcNG8Zf/epXl9xnYmKi+u7PaQo+FhEREezUqZPKn/PzOikpiUajUX2u/61vRz9mHo/nnzpuV+PkcDhCrkWv10u3263mg893PQ8vlc82m41Wq5WpqakheRy8LbvdHpKvl9qWXk5omsb4+PiQ/Qbnc2pqasi6wWWCxWJhTEwMbTYbfT7fP3XMLlZe6J8ZjUba7fYffdwvtZ7RaLxoWflDafhn1j1//z807/f7f/LzWT82wWV4cB4YDAYGAgFmZGSwSZMmtFqtjI+P59ChQ/nJJ58wLy+PAPjBBx9clvux1OxcJs888wwmT56MOXPm4KGHHkJKSgrq6uoAANdddx3eeecdbNq0CWazGbW1tXjrrbewY8cOdO7cGb1798bTTz+NHTt2wGq14pZbbsFXX32FPXv24OzZszCbzdiwYQPeffddVFdX4+2331bvFJo5cyYee+wxREdH49y5cwCAMWPGIDs7GydPngQAvP766/jiiy8wffp0AI3DAs1ms0p7Q0MDJk6cCADIzMxEQkICAGDkyJFISkpS2zlx4gSMRiMGDRoEAOjUqRMsFgsAYPjw4Rg5ciQmTJgAAMjIyEB0dDSAxrddnj17Fg0NDQgEAmjWrBk0TcOZM2cQEREBl8uF2NhYeL1eNVRx3rx5KCgoQGpqKgAgNzcXUVFRcDqdaNu2LcxmM7744gv1n8GlpaUhKysLUVFRMJka37DgdruxfPly3HHHHcjJyYHJZILP54PJZEJZWRlIoqGhAa1btwZJmEwmpKamIj09HQ0NDfB6vQCA2NhYGAwGLF++HCUlJTh79iyaN2+O+fPno2vXrmhoaFC/EwDy8/PxP//zPygpKQEADBo0CLNnz0bnzp1hMBjw5JNP4rbbbgMAaJqGuXPngiSsVivOnDmDyMhIuN1uPP3001i7di0iIyPh9XqhaRrq6+sBACdPnsTo0aMRHx8Pr9cLi8WCli1bwuPxwO12Y+3atbj//vvhcrkAAK1bt0aTJk2QkpICs9mMnj17IjY2FmlpaSgqKoLL5YLVaoXL5YKmabjxxhthMpmQnp4OAAgEAtA0DWlpaRg8eDBMJhPi4+Mxa9YsAIDT6cRvfvMbvP322ygoKIDJZEKnTp0wdOhQOJ1OxMXF4a677kJKSgoA4Nprr8WYMWOQlJQEu92OKVOmYPjw4QgEArjvvvvw3nvvoaioCAaDAUajEUuWLFG/f9KkSXjmmWdw9uxZvPPOO9A0DTExMUhLS0NCQoI6DnV1dZg3bx5qamrgcDhgt9uRnJyM2NhYxMXFoaGhAVVVVThx4gQMBgOSk5OxadMmREVFwWw24/rrr0dsbCxat26N8ePHo7q6GsuWLUN+fj4AqGOov/KiWbNm6rybP38+ACAmJga33HKLutaWLl2Kxx9/XL1VdtmyZbj33nvVunfddRecTicAYOzYsXj99dfVdTtkyBA88MADAIDRo0fjl7/8JYxGIwDAbDajW7duqKurw5kzZ/DQQw/huuuuw+HDh3H8+HH07t0bDocDJGE2m1FYWAgAsNvtqKiogNPphMlkQkxMDCIjI1FfX4/k5GSYzWbExsbi1KlTMBgMaNeuHSIiIlQZkpycjPr6evTu3Rv3338/Fi1aBE3TYDA03mqKi4tBEmfPnsW4ceOQlpaGr7/+GidOnEB+fj4cDofKMwD49NNPMWHCBLz88svwer04d+4c/H4/2rdvD4PBgCNHjmDSpEnqP4NMTEzEtm3bVDlRWFgIv9+vtnv99dejuLhYpScxMVH9rZ+3+vWbn5+P7777TuWpvg392ASXmbm5uQimHzN9Xb3M1M8LTdNgs9nUsdKVlpbC5/NB0zT1WatWrUL207VrVzWfnp6OvLw8AFDpDP5eRkZGyH6Tk5MBAGfPng15NUvbtm3Rvn17NZ+VlYUbbrgBQGM5NmbMGLXMZDKF5Fnw24sdDgfi4uIQEREBm82m8mHRokXIycmB2WzGl19+idLSUuzduxfPPfccDhw4gHbt2v3d/+7hX3ZZQijB1q1bc9y4cWq+vr6eiYmJBMCamhr1+bfffksA3LJli/rM4/EwLi6Ow4YNo9Pp5MSJE0mSM2bMYGlp6UX3l5KSQpfLxYaGBtbV1dFoNLJt27Yh+ysoKCAAejwe3n///SRJAPztb39Lq9WqonA9sta/+9577xEAly9fzmPHjqn1WrduTbvdzl//+tcEwJKSEg4dOvSC3wiATZo0YWZmJjt37sy3336bAJicnMzZs2fzz3/+M/F/T6GLFy8mAK5evZoAuHDhQgLg+vXrQ/LLZDLxyy+/JAA++eSTBBDydFdTU6PWHTZs2AX5rM/HxsaysrKSL7/8skrDf/7nfxIAKysrqWkaN23axMjISLXt2tpaRkVF8fHHHydJOp1OGo1Gnj17Vs0DUL8zNzeXJFlcXEybzaa+RzJkOwCYlpbGpKQkaprGtm3bUtM0Tpo0iW3atCEARkdHc9OmTSwtLaXL5VJpuvnmm5mens5NmzaxQ4cO7NatGwFw48aNbNmypTru+jnhdDq5adMmulwuFhYWqu+WlpbS4/Gocw8Ar732WnVcjx07Ro/HQ03TGAgEqGkahw0bxszMTALgww8/TKCxpkk3Z84cJicn02Kx8M4772SzZs0IgHFxcZwyZQoBsKKignPmzKHD4eAdd9yhvpeXl6e2YzababVaWVlZyQEDBhAAo6KiuGLFCs6YMUPlUV5enrpOjh49qp4w9fNav47087pVq1bqvLbb7SwoKKDT6WR2djZnzJhBr9fLoUOHXvL6c7lcTE1N5fDhwwmAH330kXpKjYuLo9frZVlZGQOBQEgepaamsqGhgSTZs2dPAuCZM2c4ceJENmnSRH0/Pj6eALhu3TqSZHx8PKOiotjQ0MCJEycyLS2NDQ0NLC8vV7Ufo0aNYuvWrQk01oCsWLGC5eXlKi+Ki4tZWVnJXr160WAwsEWLFgTAnj17sry8nJWVlezTpw979OhBoLFWTM9zs9msanCaN2/OyspK+v1+pqSksLy8nMnJyRwyZAhJsq6uTuVrhw4dVK2PzWZjmzZt1BO/yWSiwWDg+vXrmZeXp2od/H4/Z8+erco0v9+vagqeffZZtS29hs/n87Guro6apjEiIoLTp0+n0Wjk+vXrCYD9+vUjSZUPwbVcqamp6lrWy4IJEyao5ZGRkSG1VsG1UR6Ph/Pnz1e1o+evG/y3y+Xi3XfffdHt1tTU8MCBAyE1JnfccYf6e9myZTx69Kia/+///m9++umnF61pCS7HAXD79u3csGGDSvtXX30Vsm5xcXHIvL7dxYsXq+00b96cS5YsCcmD0aNHq/mlS5cSAH/zm99Q0zTefvvtBMDPP/9cXW8zZ85kamqqun7uuusuAuCuXbsua82OBDuXwenTp2k0GkNu+CRVYRj8+f79+1UBee7cOVZXV9NgMHDYsGGcM2eOahJJTU2l2+3mzTffzH79+tHn8zE/P5/Lly/n6dOnabfb6fF4uHfvXh4/flxdgMH7KykpuWRAc+21115y2aZNm9QJfO+996oLs1OnTqpQ0S/oqqoqVYi0bt2aNTU1BMAOHToQAB966CG+8MILqpDasmWLCmx8Ph9vvPFGlR+BQIBDhgwhAG7dujUkvzwej/p70KBBNJvNIRehfgMCwClTpjA/P58AWFhYyLfeekt9Vw+k9JuuxWLh2LFjVWBkMBjYt29fWiwWRkdHEwAff/xxWiwW7t69m9XV1TSZTPR4PDx37hwfe+wxdRN+6qmnVGChHwtN05iTk0Ofz8fMzEyazWbu3r2b99xzj/pejx49aDQa6Xa7qWka9+/fz5ycHAJgUVERSTIuLo75+fmMjY0lAGZkZPD2228nSXbo0IHx8fG02Ww8ceKEanL43e9+p46DxWJhVFSUKtTj4+MZGxtLs9nM9PR0de4Bjc2X+k0muIlHb6azWq3qpqXfPGw2GxMSEpiamsrs7GyazWYaDAZGRkaqG1lsbCyzs7NVvujbtVgsIduMiYlhamqqOu9KS0tV2jRNY3R0NK1Wqwp29JteQUEBfT6f2naXLl3o8/lotVpZVlbG9u3bE2hsjk1KSiLQGESXlZVd0BSj55WmaXQ4HHS73czPz+cDDzxATdPU9QmATz31lHp4uOOOO5iUlESr1cp77rmHgUBA3Rz0oO706dOMiIigw+Hg6dOnGR0dzeLiYprNZq5Zs0bdQF0uFzMyMgiA1113HYuLi6lpGlNSUvjWW29x3rx5Kr0vvfSSCqBMJhP379+vyh8A7Ny5MxMSEhgdHc1mzZqpYHXSpEm855571DL9ejYYDOzZsyeNRmPITdFisai86tWrlwrS/H4/a2pquHXrVrXu/fffr/42GAzqnNbTCICvvvoqS0pKVJmSlpbGDh06qDItOTlZfefWW28lABVE6k3P+nVqs9nUQ4oeYLdo0YJff/31RYMDvblPfygEwDlz5oScC/qy4Ob34HJTTwsAFfDq3w9uJg5eL3jZgw8+yIaGhpCmx6lTp6q/t2zZws2bN6v5uXPnqqBEfzi5WDkOgNu2beOwYcMIgMOHDw9ZFvzgpJ/verC8ePFi7tixQ+XjzJkzQ/KssrJS5cmwYcOoaRqPHDlCo9HI3r17E2h8SNq0aRM1TePUqVNZWFhIktyzZw/tdjtzc3PV75Bg52fkL3/5izq5gk2bNo3A98FHfX09y8vLmZ+fr54o7HY7k5OTeerUKW7YsIHNmzfn4MGDuXHjRnWBTZkyhTt37uRjjz1Gm83GsWPH0mAwcNy4cerJDoC62T/33HN86qmnQi7ML7/8kuT3AU3//v0vepE888wzTEtLC7ko9MJu8uTJ9Pl86ncFX/yLFy/mggULVJqHDBlyQT+eVq1a8fDhw6pQABr7FJSUlJAki4qKVG1YbW0t6+vr2bVrV1qtVnbr1k0VIpGRkezXrx/Ly8tVWhcvXsz09HRVkOTm5jI/P5+33347zWYzO3bsqArF4EnTNPXUGtz/6OGHH1a/Re/boQckUVFRF9wcjUajKmj1G31wHhmNRnWcjEajCj7O7zPRs2dPnjhxQhVklZWVrK6uptPp5Lhx41QhHh0dzVOnTpEkMzIyQs4D/aZBkllZWQTAWbNmqWAMAPPz87lgwQLV7r5ixQpGRUXR5XKFFP4mk4l9+vRRNxOgMQDT+wvpAXV0dDS3bdvGjRs3Mj09PaSPjMFgoM1m47p169RNdtSoUezbt6/aT2pqKlNSUmiz2WgymfjEE09c9AYFgLNnzw75rXpNgdFo5Ny5c9Vxa9++PXfu3KnWPb/fSVJSklpmNpvZu3fvix7X4OvQZDLR4XCwtLQ0ZP8dO3ZU+ayfV3/5y1/YqlUrtmvXjgD4xz/+kSS5YsUKAo0PBHqQ5HK52L9/fz7zzDPq3HnwwQfZr18/dRzGjBlDo9HIW265hRaLRdWKXmy6VH+s8/umBR+ni/Xp0D/Tj9vF1ikoKAjpYxcc4AZPei2mPh8ZGcnMzEwVUOqfX3PNNTx37lxIORS87/T0dDZt2lTlm/5wph8v/eECaAxsBw4ceNG8uP7660N+GwDOmDEj5JqsqKggAJW+4HzdunXrRfNZf8DQ0wg0PqAFr6MH3i6Xi0eOHAmppQ7Ou/fee4/Lly9X8+PHj1dBwvn9lj744AMeOnQo5DM9qAmurQHA//iP/whZr7q6mqNGjSIAVlVV8Ze//CWBxlrq4PWmTZvGhIQENe/1ejl48GCSpNfrZXR0NM1mM0+dOsWCggL26tWLbreb3bt3V9ef3+/n4cOHJdj5Ofqxwc6YMWOYnJzMAwcOcP/+/Vy/fj0dDgc9Hg8//vhjko1P6XozltlsptFoDGkGmTBhAiMjI9myZUv6/X5WV1dz9+7dvP/++0M6Gbdq1UrVkgA/PtgpKipSJ/iECRNot9vVU35NTQ2Tk5NVM1bwpP/GXr16EWis6fF4POzRowcTExM5ZsyYCzrbNWnShDabjQcPHiRJ+nw+VajU1taysrKSFouFHTt25MiRI5mUlMQZM2bQ6XQyJiaGgUCABw8eJAB2795dVf8DjTcyfbvR0dF0u900mUyMj4/nI488whEjRlwQjHXq1ImZmZls2bJlyNP/hAkTGBUVxVWrVjEhIYEmk4lPP/00q6qq6HQ6aTKZmJKSop7I7rzzTnWz1vfx/PPPc9q0aTQYDKyoqFCfN23alGvWrKHL5QoJDvQCb+jQoYyNjWVhYSEnTpzIa665hgDYv39/kuTHH39MTdMYFxfHV199lU6nk5qm0ev1ct++fSqAq6mp4bp16wh8/7QcExPDwsJCxsTEcMqUKTQajUxISGDr1q35/PPPh3R4BhoDgkAgwDFjxrC2tpZGo5GtWrVSherjjz/OY8eOsaCggEajkfPmzeOvfvUrOhwOGgwGLlu2jLW1tQTAcePGqVoAPY07d+6k2+1mUlKSqnkDvg8y9I7y+qTnocFgYNu2bTlhwgS2adNG5V1OTo66jqxWKx0OB00mE9PS0lTAaTabGR8fT4vFwl//+tcqLXrNTvPmzdXxHzRoED0ej6qVadWqFaurq0Nu4N27d6fX66XP5yNJtmrVSp2XtbW1PHbsGN1uN30+Hw8dOsTS0lJmZmbS6XQyLy+PXbp0UTfYmpoa9bQNNNZwVVRUkCRzcnJCBg4sX75cNTudP+k1R9HR0XQ6nWzRooWqaejXrx/vvfdeFcDHxcWxf//+KgiKi4vjyJEjaTAYGBERwYqKipDaPr1ZbMyYMezQoYN6MNKvdYPBEDIY4vzp/BoG/YHEaDSGXBPnB085OTkhHYuDzwdN00K2a7PZQs6dRx55JGRbwcv+kWAnOTlZPWAGX1d6bVznzp3VMr255/ztulyuCwYSrFmzRv2tPyjp87fddtslg5233nor5HwZP368CjDuvfdeVcYD4MqVK0O++/vf/15tt0OHDurBzeFwsGnTply0aNEFx87hcNDr9fLYsWM8c+aMqg13uVzs1asXs7OzmZqayptvvpmHDh3i7Nmz2aJFC7Zr1449e/bkJ598QkCCnZ+VH9OMNW7cOPr9fn7yySdqud7koz9tnT9yJBAI0Ov1sqqqSn1n/vz5qoBbunRpyP70ZU888QRJhhR+f68Z68yZM+qCPXz4MIHv+7AEX3jn/6sXAvpvnz59ulq/ffv2Ib85KyuLVqtVFTZms5lDhw4lSY4bN45Go1H1FdGDkXbt2nH06NFqOxMnTlTbP3+ETXCN0apVq9R27XY7mzdvTgB88803VX6VlZUxMTGRXbt2VfuMi4vjfffdx7KyMlWTUVtby44dOzI2NpZlZWXs1KkTf/GLX3DixImXHMGlFzp6zcrGjRtJkrGxsYyNjQ1plvlHRmf8Oyaj0chz586xqKiIVVVVqvmhZcuWdDgcvO+++0iSFotF9Q9q2bIlJ0+ezLZt27KsrIwFBQWsqqq6II+Cj5eeRykpKSqPioqKVMAJND79V1VVMRAI8Oabb2azZs1UAK7XQLlcLt5888185JFHmJiYqG56Pp+PZ86cod1uV013gUAgpFkmeAq+sbhcLhoMBpaUlPCRRx5hRESEaoLT133++edJkp999pn6/MMPPyTQ2OxEkomJier3f/HFFyrNzz77LD/77DMV/F7sPDr/Bh+8z/LycvW5fk77/X5WVFTQ4XBw+PDhjIuLU9+LjIzk0qVLOX/+fJUfet77/X7m5uYyLi6OTZs2pd/vV9fLxaaoqCgVkEdFRbGiooLNmjXj9OnT1fdiY2N54sQJ9ZCVnJzMqKgo1Y/D6XRyxIgRJMk+ffqo4KRbt27s2rUre/TowdzcXPbu3ZvdunXj3XfffcnzSK+hy87OpslkCrmxBwdAweVd8Pn3zzZjlZeXh5Sjwf38zp/0Jtfzt5uenh7SvBecPqCxb57e/wgA77vvvks2Y+Xm5oaM6B08ePAF17X+t95nS58efvhhtd3k5GS1bmJiIg8fPqzuD1arlUVFRSov5syZwzNnzqjmq1GjRtFkMjErK4vXXHMNhw0bxvr6epJk7969Q66x4H/1ZrafkozGugwsFgsKCwuxefNm9VlDQ4OaX758OWpqavDaa6+pUQMAUFZWho8++ghFRUXo1asXdu3ahaKiIgwZMgS7du1CcXExjh49qkZHAcD69ethNptBMqSHPPB97/zo6GjU1tbi5ZdfBgB4PJ6QtNXV1eHdd99V82fPnsWAAQMAAHPnzkVMTAwAoGPHjti9ezf8fj8AYPHixUhMTMS0adMAAKmpqWrEmW7fvn3q7z/+8Y947bXXkJKSgvHjx+Pzzz9HQUEBPvjgA7XfoUOHYvz48fjd736H+vp6tGnTBgCwZs0a5ObmIicnBy+88ILazvHjx+Hz+WA2mzF37lz0798fADBr1izk5uaqPMnPz8f48eNRU1ODlJQU/PWvfwUANcJKP0ZA4wgNTdOwa9cufPvtt7j++utx4sQJfPfddwCA48ePY+fOndA0Db///e8BAKdPn0ZVVRV2796tRg3NnTsXQOPIp+rqaiQmJqK2thYA1DGsq6vDyZMnce211wJoHNG2bds2rFu3Di1atIDX60XHjh3RpEkTAMCwYcPUso4dO4acOxkZGWjZsiWysrJQUVGB//qv/8KTTz6Ja665BpqmoV+/fkhLSwMAVFZWYu3atTCZTCHHs0WLFnC73YiJiVEjYtatW4dTp07hwIEDSEhIUCPB9uzZg7q6Olx//fXYuXMnzpw5o7b/ySef4LnnnoPFYsGaNWvw6aefIiEhAVVVVXjzzTcBAD179sS2bdsAAH369EF1dTXi4+Px5ZdfAmgc5XHgwAH87W9/w1dffQWgcQRgQkICSkpKsHfvXpw6dQoulwvt27fH559/DgDw+/3Yu3cv9u3bB7/fr0YlxsfHY8CAAbBYLHC73UhNTUVJSQk0TcPQoUNhsVjQtGlTdT5NmzYNPXr0gNlshtlsRkNDA2pra7Fv3z5YLBacPHkSPp9PpbW8vBwAsHLlStjtdgBAVVUVAGDChAnYu3cvvvzySzUirm/fvjhy5Aji4uJw4403YuXKlYiNjcWyZcuwY8cOGI1GRERE4P333wcAdO/eHT6fT12P0dHRap87duxQ54J+TtfV1anz32q1qpGc+nluMBhw9uxZdV4DQH19Pb755hscOnQIt9xyCzRNQ11dHZKTk5GcnAy/3w+r1YpOnTrBaDTCZrPhtttuQ3x8PEji3LlzMBgMaGhowL59+9QoJJPJBKfTiYSEBNTW1uKrr75CfHw80tLSoGkaTp48icGDB6O2thabN29W5ci2bdtw6NAhHDx4EGvXrsUbb7yBvn37Yvz48XjrrbegaRo6dOiALVu2QNM0dO/eHZs3b0bLli2xd+9enDt3DpmZmeq6s1qtyM7OVqM2g8vCyMhIWCwWfPjhhwCAiIgIHD16VI14jIyMVMtIwuVyqbw0m80oLi5W5WhkZCROnDih8tXtdqtj4XA4UFxcHLJM3+4333yDTz/9FJdis9lQWlqq5vXyEUDIiC89TXr5BACjRo3CCy+8AADo0aMHdu3apZYtXLgwZDRUcLl49OhRNZJ2/PjxcLvd6v6QlpamrteGhgb069cPAwYMUNvevn07Ghoa8N1336F169ZYuXKlyoclS5bgww8/xEsvvQQAWLBgAYDGkcz33HPPJfPgn/aTh0+CJPn000/TarVy1apV3L59O/v06aOqQ202G1esWMHt27dz/PjxXLduHffs2cPdu3ezqqqKmqbxlVde4ZQpU5iXl8dRo0Zx69atqolg1qxZ3L9/P5966ilqmsaKigqOGDGCSUlJXL9+Pf/whz/w1ltvDennob/fAIBqL9V7waekpKhqeqCxv4ReNZ2fn69qZ/Sqab3qtrS0lJGRkbzpppsIhI5smDp1KkeOHKnmNU1jQkICH330Ufbt21c1yeXn59Nut9NqtdJqtbJ79+50Op1s2rQps7Ky1NNWfHw8O3fuTIfDwT59+nDVqlXs27cvnU4ni4uL6fF4WF5ervo8tG3blna7nREREdQ0jR06dGBERARvuummkKYJvR1Zb24DoJqGALBr166qDV+v3tZ/58yZM9XT/bRp09i/f39VK+RwONSym266iS+99JKqZk5ISGC/fv1Cnqb0vhjZ2dl89913uXDhQtUcEtxsEwgE+Nprr6lmCz2t0dHRTE9P5/PPP89AIMAbbriB27dv55tvvsmYmBharVZ+8803Kj8feOABfvrpp6pJJSsri/v372cgEFBPq6mpqXS5XMzLy1MdnvX+DnoelpWVsVu3brTb7TQYDGr0isFgoN/v58KFC+l2u+l0Orl69WqWlZWpJ97f/va3qk/WxIkT2atXL/U+Er/fz8TERFqtVnW+6f+OHDmSS5cuVU/X/fv3V/19YmJiVDOZ3kSlXwcej4cxMTGqObe0tJT33nuveteM0Wjk+PHj1b5GjBjByZMnq2PkdrtDnkTNZjPNZrM6Nq+//jqXLFmi1tNr8fx+P3fs2MGCggKazWZ26dJF5a/P5+OAAQNYVVXF2NhYjh49mlu3bmVFRQUNBgOHDx+uOh5HR0ezT58+qmmnd+/e3L9/P2fPnq3SqI+sqqmpCak9WLhwoUqP3synjwIKPlc9Hg+tVitdLhfNZjN79OjBwsJCapqmRtQVFhaqp/rMzEy6XK6Q/jOapjE/P5+apqn+ai6Xi2vWrOHUqVNVrc3QoUMZFxen8nTYsGFMSEgI6dBvtVoZExPDefPm0e120+/3c+3atRwzZowqT5ctW8aUlBSVX6tXr1bfj4qK4uDBg9U5p9c+6bW0OTk5Ie/xCW6mLSkpCelbF1y+BQKBkJqU5s2bU9M0Vf4EN+253e6QpjG9X5I+H9xMdrH3k+lNXHp5pJcxQGPTmN4R/fz3enXt2jXkHAgeEdixY8eQDus33XRTSI1SRUVFSA2X3vyVlJTEpk2bhuxLT7PD4WBOTg49Hg/j4+Pp9XpVHuXl5bG6uporVqzg3LlzuWnTJn722WfcvHkz27Vrx7S0NP7pT38iIM1YP0sPPfQQA4HAj3pZm8lkos/nY1lZGV955RWS5MCBA9Voh6SkJA4cOJDLly9ndnY2rVarGvmgj8CaOHEiA4HAD7aJyxQ6XerFW3pH1H/mxVv6iBB9pJQ+BXd61tfTm63Kysq4cuVK9unTh7Gxseqc0TSN6enpTEtL47hx4zh27NiQwPTvTQkJCYyNjVXVwnrhFxERQavVSpvNxqSkJCYlJalRRpmZmbTb7erFjzab7YKmNZPJdMFv+bFp0vMhKSlJBTsXyxO9k7V+M+zZsydbtmx50b5MZrOZJSUlzM7OVqO4fmx69N9osViYmZnJ0aNHX3AzCn4ppd75WdM01XHzxhtvZGJiYkiTrh7cut1uOhwOdb3+0GQymej3+9mpUycC4BtvvBHSNBIIBFRgHBcXR4fDoYIY/TgHd7wOfgmfyWRikyZNLnip4I89ZvpDSXJycsg29KAmOF/0hxlN09ilS5dLvjTQarWqoOsfvdb0czA+Pl4FkPqkvwDxH712Zfr3TVarlSkpKRwzZgz/93//97J3UNbI/6vPE0IIIYQIQ9JnRwghhBBhTYIdIYQQQoQ1CXaEEEIIEdYk2BFCCCFEWJNgRwghhBBhTYIdIYQQQoQ1CXaEEEIIEdYk2BFCiPO88cYb0DQNR48evdJJEUL8BCTYEUIIIURYk2BHCCGEEGFNgh0hxFWnoaEBCxYsQGpqKux2O/Ly8rB27VoA3zcxvfjii8jNzYXNZkObNm3whz/8IWQbzz33HFq0aAGr1YqUlBQsWrQoZPnp06cxY8YMNGnSBFarFU2bNsUTTzwRss7777+PoqIiOBwOtGvXDnv37r28P1wIcVlIsCOEuOosWLAAq1evxqOPPoqPP/4YkyZNwtChQ7Flyxa1zrRp07Bo0SJs374dPp8PvXr1wtmzZwE0BikDBgzAoEGD8NFHH+Huu+/GnXfeiVWrVqnvDx8+HNXV1ViyZAn27NmDxx57DBERESHpmD17NhYtWoQdO3bAZDKhsrLy3/L7hRA/LfmPQIUQV5XTp08jOjoar776Ktq2bas+v+WWW1BXV4df/OIX6NSpE55++mkMHDgQAPDXv/4Vfr8fq1atwoABAzBkyBAcOnQIr7zyivr+9OnT8eKLL+Ljjz/Gvn370KxZM2zatAldunS5IA1vvPEGOnXqhFdffRVlZWUAgA0bNqC8vBynTp2CzWa7zLkghPgpSc2OEOKq8uc//xl1dXXo2rUrIiIi1LR69WocOHBArRccCEVHR6NZs2bYs2cPAGDPnj0oKSkJ2W5JSQn279+P+vp67Nq1C0ajER06dPjBtOTm5qq/ExISAADffvvtv/wbhRD/XqYrnQAhhAh24sQJAMCLL76IpKSkkGVWqzUk4Pln2e32H7We2WxWf2uaBqCxP5EQ4udFanaEEFeV5s2bw2q14osvvkDTpk1DpiZNmqj13nnnHfV3bW0t9u3bh6ysLABAVlYWtm7dGrLdrVu3IiMjA0ajETk5OWhoaAjpAySECF9SsyOEuKq4XC5MnToVkyZNQkNDA0pLS3Hs2DFs3boVbrcbycnJAIB58+YhJiYGcXFxmD17NrxeL2644QYAwJQpU9CqVSvMnz8fAwcOxNtvv42lS5fikUceAQCkpKRgxIgRqKysxJIlS5CXl4fPP/8c3377LQYMGHClfroQ4jKRYEcIcdWZP38+fD4fFixYgE8++QQejwcFBQWYNWuWakZauHAhJk6ciP379yM/Px8vvPACLBYLAKCgoADPPvss7rrrLsyfPx8JCQmYN28eRo4cqfaxbNkyzJo1C2PHjsWRI0cQCAQwa9asK/FzhRCXmYzGEkL8rOgjpWpra+HxeK50coQQPwPSZ0cIIYQQYU2CHSGEEEKENWnGEkIIIURYk5odIYQQQoQ1CXaEEEIIEdYk2BFCCCFEWJNgRwghhBBhTYIdIYQQQoQ1CXaEEEIIEdYk2BFCCCFEWJNgRwghhBBhTYIdIYQQQoS1/weNWD5T+dKXeAAAAABJRU5ErkJggg==", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "plt.plot(np.arange(len(epochs_x)), epochs_acc, marker = '.')\n", - "plt.xlabel('epoch')\n", - "plt.ylabel('test accuracy')\n", - "plt.ylim(0, 100)\n", - "plt.xticks(np.arange(len(epochs_x)))\n", - "for i, txt in enumerate(epochs_acc):\n", - " if i%5 ==0 or i == epochs-1:\n", - " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [], - "source": [ - "# with open(f'./architectures_results/{achitecture}-Training_Validation-TM.npy', 'wb') as f:\n", - "# np.save(f, np.array(epochs_x))\n", - "# np.save(f, np.array(epochs_y))\n", - "# np.save(f, np.array(epochs_acc))" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "speck-rescnn", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.0" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/train_all.py b/tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/train_all.py deleted file mode 100644 index 9af56038..00000000 --- a/tests/test_nonsequential/using_SumPool2d/ARCHITECTURES_SEARCH/train_all.py +++ /dev/null @@ -1,4 +0,0 @@ -import os - -for i in range(1, 9): - os.system(f'python model_training.py ResSCNN{i}') \ No newline at end of file diff --git a/tests/test_nonsequential/using_SumPool2d/HPO_GAUSSIAN_SEARCH/GS_utils.py b/tests/test_nonsequential/using_SumPool2d/HPO_GAUSSIAN_SEARCH/GS_utils.py deleted file mode 100644 index 2ef3c4b2..00000000 --- a/tests/test_nonsequential/using_SumPool2d/HPO_GAUSSIAN_SEARCH/GS_utils.py +++ /dev/null @@ -1,54 +0,0 @@ -import numpy as np - -def define_probabilistic_model(hyperparams): - prob_model = {} - - for key, data in hyperparams.items(): - prob_model[key] = {'mu': data['value'], 'sigma': data['sigma']} - - return prob_model - -def update_probabilistic_model(max_iter, cur_iter, prob_model, hyperparams_new): - - sigma_scale = update_sigma_scale(max_iter, cur_iter) - - for key, value in hyperparams_new.items(): - prob_model[key]['mu'] = value - prob_model[key]['sigma'] = prob_model[key]['sigma']*sigma_scale - -def update_sigma_scale(max_iter, cur_iter): - _ = np.round(1-(cur_iter/max_iter), 2) - - if _ > 0: - return _ - else: - return 0.01 - -def sample_values_to_eval(iteration, prob_model, hyperparams, nb_samples: int = 5): - new_values = {} - np.random.seed(iteration) - - for hp, data in prob_model.items(): - sampled_values = np.round(np.random.normal(data['mu'], data['sigma'], nb_samples), hyperparams[hp]['precision']) - - fixed_sampled_values = [] - - for val in sampled_values: - if val < hyperparams[hp]['min']: - fixed_sampled_values.append(hyperparams[hp]['min']) - elif val > hyperparams[hp]['max']: - fixed_sampled_values.append(hyperparams[hp]['max']) - else: - fixed_sampled_values.append(val) - - new_values[hp] = fixed_sampled_values - - return new_values - -def get_sampled_set(sampled_values, i): - sampled_set = {} - - for key, val in sampled_values.items(): - sampled_set[key] = val[i] - - return sampled_set \ No newline at end of file diff --git a/tests/test_nonsequential/using_SumPool2d/HPO_GAUSSIAN_SEARCH/gaussian_search_history.csv b/tests/test_nonsequential/using_SumPool2d/HPO_GAUSSIAN_SEARCH/gaussian_search_history.csv deleted file mode 100644 index e87da0e0..00000000 --- a/tests/test_nonsequential/using_SumPool2d/HPO_GAUSSIAN_SEARCH/gaussian_search_history.csv +++ /dev/null @@ -1,57 +0,0 @@ -,learning_rate,spike_threshold,mem_v_min,grad_scale,grad_width,w_rescale_lambda,accuracy -0,0.001,2.75,-2.5,1.55,1.55,0.5,10.576923076923077 -1,0.000947,1.99,-2.82,1.11,2.0,0.344,86.0576923076923 -2,0.000769,1.09,-2.29,1.11,1.83,0.319,87.98076923076923 -3,0.000811,1.01,-1.43,0.85,1.73,0.42,88.9423076923077 -4,0.000811,1.01,-1.43,0.85,1.73,0.42,88.9423076923077 -5,0.000811,1.01,-1.43,0.85,1.73,0.42,88.9423076923077 -6,0.000782,1.87,0.0,0.91,1.9,0.235,92.3076923076923 -7,0.000741,1.87,-0.15,1.16,1.97,0.175,95.67307692307693 -8,0.000741,1.87,-0.15,1.16,1.97,0.175,95.67307692307693 -9,0.000741,1.87,-0.15,1.16,1.97,0.175,95.67307692307693 -10,0.000741,1.87,-0.15,1.16,1.97,0.175,95.67307692307693 -11,0.000741,1.87,-0.15,1.16,1.97,0.175,95.67307692307693 -12,0.000741,1.87,-0.15,1.16,1.97,0.175,95.67307692307693 -13,0.000741,1.87,-0.15,1.16,1.97,0.175,95.67307692307693 -14,0.000741,1.87,-0.15,1.16,1.97,0.175,95.67307692307693 -15,0.000741,1.87,-0.15,1.16,1.97,0.175,95.67307692307693 -16,0.000741,1.87,-0.15,1.16,1.97,0.175,95.67307692307693 -17,0.000826,1.51,0.0,1.06,1.64,0.223,96.15384615384616 -18,0.000826,1.51,0.0,1.06,1.64,0.223,96.15384615384616 -19,0.000826,1.51,0.0,1.06,1.64,0.223,96.15384615384616 -20,0.000826,1.51,0.0,1.06,1.64,0.223,96.15384615384616 -21,0.000826,1.51,0.0,1.06,1.64,0.223,96.15384615384616 -22,0.000826,1.51,0.0,1.06,1.64,0.223,96.15384615384616 -23,0.000826,1.51,0.0,1.06,1.64,0.223,96.15384615384616 -24,0.000826,1.51,0.0,1.06,1.64,0.223,96.15384615384616 -25,0.000826,1.51,0.0,1.06,1.64,0.223,96.15384615384616 -26,0.000826,1.51,0.0,1.06,1.64,0.223,96.15384615384616 -27,0.000826,1.51,0.0,1.06,1.64,0.223,96.15384615384616 -28,0.000826,1.51,0.0,1.06,1.64,0.223,96.15384615384616 -29,0.000797,1.83,-0.09,0.67,1.56,0.121,97.59615384615384 -30,0.000797,1.83,-0.09,0.67,1.56,0.121,97.59615384615384 -31,0.000797,1.83,-0.09,0.67,1.56,0.121,97.59615384615384 -32,0.000797,1.83,-0.09,0.67,1.56,0.121,97.59615384615384 -33,0.000797,1.83,-0.09,0.67,1.56,0.121,97.59615384615384 -34,0.000797,1.83,-0.09,0.67,1.56,0.121,97.59615384615384 -35,0.000797,1.83,-0.09,0.67,1.56,0.121,97.59615384615384 -36,0.000797,1.83,-0.09,0.67,1.56,0.121,97.59615384615384 -37,0.000797,1.83,-0.09,0.67,1.56,0.121,97.59615384615384 -38,0.000797,1.83,-0.09,0.67,1.56,0.121,97.59615384615384 -39,0.000797,1.83,-0.09,0.67,1.56,0.121,97.59615384615384 -40,0.000797,1.83,-0.09,0.67,1.56,0.121,97.59615384615384 -41,0.000797,1.83,-0.09,0.67,1.56,0.121,97.59615384615384 -42,0.000797,1.83,-0.09,0.67,1.56,0.121,97.59615384615384 -43,0.000797,1.83,-0.09,0.67,1.56,0.121,97.59615384615384 -44,0.000797,1.83,-0.09,0.67,1.56,0.121,97.59615384615384 -45,0.000797,1.83,-0.09,0.67,1.56,0.121,97.59615384615384 -46,0.000797,1.83,-0.09,0.67,1.56,0.121,97.59615384615384 -47,0.000797,1.83,-0.09,0.67,1.56,0.121,97.59615384615384 -48,0.000797,1.83,-0.09,0.67,1.56,0.121,97.59615384615384 -49,0.000797,1.83,-0.09,0.67,1.56,0.121,97.59615384615384 -50,0.000797,1.83,-0.09,0.67,1.56,0.121,97.59615384615384 -51,0.000797,1.83,-0.09,0.67,1.56,0.121,97.59615384615384 -52,0.000797,1.83,-0.09,0.67,1.56,0.121,97.59615384615384 -53,0.000797,1.83,-0.09,0.67,1.56,0.121,97.59615384615384 -54,0.000797,1.83,-0.09,0.67,1.56,0.121,97.59615384615384 -55,0.000797,1.83,-0.09,0.67,1.56,0.121,97.59615384615384 diff --git a/tests/test_nonsequential/using_SumPool2d/HPO_GAUSSIAN_SEARCH/main.py b/tests/test_nonsequential/using_SumPool2d/HPO_GAUSSIAN_SEARCH/main.py deleted file mode 100644 index 124b86b0..00000000 --- a/tests/test_nonsequential/using_SumPool2d/HPO_GAUSSIAN_SEARCH/main.py +++ /dev/null @@ -1,202 +0,0 @@ -from GS_utils import * -import torch, tonic, sys, random -import numpy as np -from torch.utils.data import DataLoader -from sinabs.activation.surrogate_gradient_fn import PeriodicExponential -from torch.nn import CrossEntropyLoss -from torch.optim import Adam -import pandas as pd -from tqdm import tqdm - -sys.path.append('../../utils') -sys.path.append('../models') - -from network import SCNN_GS -from train_test_fn import training_loop_no_tqdm, load_dataset, split_train_validation -from weight_initialization import rescale_method_1 - -if torch.cuda.is_available(): - device = torch.device('cuda:0') - print('device: ', torch.cuda.get_device_name(0)) -else: - device = torch.device('cpu') - -torch.backends.cudnn.enabled = False -torch.backends.cudnn.deterministic = True -random.seed(1) -torch.manual_seed(1) -torch.cuda.manual_seed(1) - -### Initialization #################################################### - -max_iter = 100 -nb_samples = 5 -batch_size = 8 -num_workers = 8 -validation_ratio = 0.2 -n_time_steps = 50 -epochs = 40 -validation_rand_seed = 1 -output_csv = 'gaussian_search_history.csv' -params_set_history = {} - -loss_fn = CrossEntropyLoss() - -hyperparams = { - 'learning_rate': {'value': 0.001, 'min': 0.00008, 'max': 0.08, 'precision': 6, 'sigma': 0.0001}, - 'spike_threshold': {'value': 2.75, 'min': 0.5, 'max': 5.0, 'precision': 2, 'sigma': 1.0}, - 'mem_v_min': {'value': -2.5, 'min': -5.0, 'max': 0.0, 'precision': 2, 'sigma': 1.0}, - 'grad_scale': {'value': 1.55, 'min': 0.1, 'max': 3.0, 'precision': 2, 'sigma': 0.5}, - 'grad_width': {'value': 1.55, 'min': 0.1, 'max': 3.0, 'precision': 2, 'sigma': 0.5}, - 'w_rescale_lambda': {'value': 0.5, 'min': 0.1, 'max': 1.0, 'precision': 3, 'sigma': 0.1667}, -} - -prob_model = define_probabilistic_model(hyperparams) - -with open('fixed_parameters.txt', 'w') as file: - file.write(f'max_iter: {max_iter}\n') - file.write(f'nb_samples: {nb_samples}\n') - file.write(f'batch_size: {batch_size}\n') - file.write(f'num_workers: {num_workers}\n') - file.write(f'validation_ratio: {validation_ratio}\n') - file.write(f'n_time_steps: {n_time_steps}\n') - file.write(f'epochs: {epochs}\n') - file.write(f'validation_rand_seed: {validation_rand_seed}\n') - -### Data Loading ##################################################### - -snn_train_dataset, snn_test_dataset, sensor_size = load_dataset('DVSGESTURE', n_time_steps) -train_dataset, validation_dataset = split_train_validation(validation_ratio, snn_train_dataset, validation_rand_seed) - -disk_cache_train = tonic.DiskCachedDataset( - dataset=train_dataset, - cache_path='./cached_train' -) -snn_train_dataloader = DataLoader(disk_cache_train, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True) - -disk_cache_validation = tonic.DiskCachedDataset( - dataset=validation_dataset, - cache_path='./cached_validation' -) -snn_validation_dataloader = DataLoader(disk_cache_validation, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True) - -disk_cache_test = tonic.DiskCachedDataset( - dataset=snn_test_dataset, - cache_path='./cached_test' -) -snn_test_dataloader = DataLoader(disk_cache_test, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=False) - -### Baseline Accuracy ################################################ - -# instantiate model. -csnn = SCNN_GS( - batch_size = batch_size, - surrogate_fn = PeriodicExponential(grad_scale = hyperparams['grad_scale']['value'], grad_width = hyperparams['grad_width']['value']), - min_v_mem = hyperparams['mem_v_min']['value'], - spk_thr = hyperparams['spike_threshold']['value'], - rescale_fn = rescale_method_1, - rescale_lambda = hyperparams['w_rescale_lambda']['value'] - ).to(device) - -# instantiate optimizer. -optimizer = Adam(csnn.parameters(), lr = hyperparams['learning_rate']['value'], betas = (0.9, 0.999), eps = 1e-8) - -# train/test model. -best_acc = training_loop_no_tqdm( - device, - n_time_steps, - batch_size, - sensor_size, - snn_train_dataloader, - csnn, - loss_fn, - optimizer, - epochs, - snn_validation_dataloader) - -# initialize parameters history. -best_param_set = { - 'learning_rate': [hyperparams['learning_rate']['value']], - 'spike_threshold': [hyperparams['spike_threshold']['value']], - 'mem_v_min': [hyperparams['mem_v_min']['value']], - 'grad_scale': [hyperparams['grad_scale']['value']], - 'grad_width': [hyperparams['grad_width']['value']], - 'w_rescale_lambda': [hyperparams['w_rescale_lambda']['value']], - 'accuracy': [best_acc] -} - -df = pd.DataFrame(best_param_set) -df.to_csv(output_csv, index=True) - -print(f'> initial accuracy: {best_acc}\n') - -### HPO Loop ########################################################## - -train_p_bar = tqdm(range(1, max_iter+1)) -counter = 1 - -for iter in train_p_bar: - - # sample values to be tested. - sampled_values = sample_values_to_eval(iter, prob_model, hyperparams, nb_samples) - - # test each sampled set. - acc = [] - for i in range(nb_samples): - sampled_set = get_sampled_set(sampled_values, i) - - # instantiate model. - csnn = SCNN_GS( - batch_size = batch_size, - surrogate_fn = PeriodicExponential(grad_scale = sampled_set['grad_scale'], grad_width = sampled_set['grad_width']), - min_v_mem = sampled_set['mem_v_min'], - spk_thr = sampled_set['spike_threshold'], - rescale_fn = rescale_method_1, - rescale_lambda = sampled_set['w_rescale_lambda'] - ).to(device) - - # instantiate optimizer. - optimizer = Adam(csnn.parameters(), lr = sampled_set['learning_rate'], betas = (0.9, 0.999), eps = 1e-8) - - # train/test model. - ith_acc = training_loop_no_tqdm( - device, - n_time_steps, - batch_size, - sensor_size, - snn_train_dataloader, - csnn, - loss_fn, - optimizer, - epochs, - snn_validation_dataloader) - - acc.append(ith_acc) - - # update progress bar - train_p_bar.set_description(f'model {counter}/{max_iter*nb_samples} - best acc.: {np.round(best_acc, 2)}') - counter += 1 - - # get best parameters set. - highest_acc_index = acc.index(np.max(acc)) - best_param_set = get_sampled_set(sampled_values, highest_acc_index) - - # update model. - if acc[highest_acc_index] > best_acc: - best_acc = acc[highest_acc_index] - - update_probabilistic_model(max_iter, iter, prob_model, best_param_set) - - # save to history. - best_param_set['accuracy'] = best_acc - - else: - - best_param_set = {} - for key, val in prob_model.items(): - best_param_set[key] = val['mu'] - best_param_set['accuracy'] = best_acc - - # update history - df = pd.DataFrame([best_param_set], index=[iter]) - df.to_csv(output_csv, mode='a', header=False) \ No newline at end of file diff --git a/tests/test_nonsequential/using_SumPool2d/HPO_GAUSSIAN_SEARCH/network.py b/tests/test_nonsequential/using_SumPool2d/HPO_GAUSSIAN_SEARCH/network.py deleted file mode 100644 index 53a9feab..00000000 --- a/tests/test_nonsequential/using_SumPool2d/HPO_GAUSSIAN_SEARCH/network.py +++ /dev/null @@ -1,83 +0,0 @@ -import torch.nn as nn -from sinabs.activation.surrogate_gradient_fn import PeriodicExponential -from sinabs.exodus.layers import IAFSqueeze -import sinabs.layers as sl - -class SCNN_GS(nn.Module): - def __init__(self, batch_size, surrogate_fn, min_v_mem, spk_thr, rescale_fn, rescale_lambda): - super().__init__() - - self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False) - self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool1 = sl.SumPool2d(2,2) - - self.conv2 = nn.Conv2d(10, 10, 2, 1, bias=False) - self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool2 = sl.SumPool2d(3,3) - - self.conv3 = nn.Conv2d(10, 10, 3, 1, bias=False) - self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool3 = sl.SumPool2d(2,2) - - self.flat = nn.Flatten() - - self.fc1 = nn.Linear(810, 100, bias=False) - self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc2 = nn.Linear(100, 100, bias=False) - self.iaf5 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc3 = nn.Linear(100, 100, bias=False) - self.iaf6 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc4 = nn.Linear(100, 11, bias=False) - self.iaf7 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.init_weights() - self.rescale_conv_weights(rescale_fn, rescale_lambda) - - def detach_neuron_states(self): - for name, layer in self.named_modules(): - if name != '': - if isinstance(layer, sl.StatefulLayer): - for name, buffer in layer.named_buffers(): - buffer.detach_() - - def init_weights(self): - for name, layer in self.named_modules(): - if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear): - nn.init.xavier_normal_(layer.weight.data) - - def rescale_conv_weights(self, rescale_fn, lambda_): - rescale_fn(self.conv2, [(2, 2)], lambda_) - rescale_fn(self.conv3, [(3, 3)], lambda_) - - def forward(self, x): - - con1_out = self.conv1(x) - iaf1_out = self.iaf1(con1_out) - pool1_out = self.pool1(iaf1_out) - - conv2_out = self.conv2(pool1_out) - iaf2_out = self.iaf2(conv2_out) - pool2_out = self.pool2(iaf2_out) - - conv3_out = self.conv3(pool2_out) - iaf3_out = self.iaf3(conv3_out) - pool3_out = self.pool3(iaf3_out) - - flat_out = self.flat(pool3_out) - - fc1_out = self.fc1(flat_out) - iaf4_out = self.iaf4(fc1_out) - - fc2_out = self.fc2(iaf4_out) - iaf5_out = self.iaf5(fc2_out) - - fc3_out = self.fc3(iaf5_out) - iaf6_out = self.iaf6(fc3_out) - - fc4_out = self.fc4(iaf6_out) - iaf7_out = self.iaf7(fc4_out) - - return iaf7_out \ No newline at end of file diff --git a/tests/test_nonsequential/using_SumPool2d/Res-SCNN3.ipynb b/tests/test_nonsequential/using_SumPool2d/Res-SCNN3.ipynb deleted file mode 100644 index 2e507d45..00000000 --- a/tests/test_nonsequential/using_SumPool2d/Res-SCNN3.ipynb +++ /dev/null @@ -1,1509 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import torch, random\n", - "import torch.nn as nn\n", - "import sinabs.layers as sl\n", - "from tqdm.notebook import tqdm\n", - "\n", - "from tonic.datasets.dvsgesture import DVSGesture\n", - "from tonic.transforms import ToFrame\n", - "from torch.utils.data import DataLoader\n", - "from torch.nn import CrossEntropyLoss\n", - "from torch.optim import Adam\n", - "\n", - "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential\n", - "\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "torch.backends.cudnn.deterministic = True\n", - "random.seed(1)\n", - "torch.manual_seed(1)\n", - "torch.cuda.manual_seed(1)\n", - "np.random.seed(1)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Loading Data" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "batch_size = 3\n", - "num_workers = 1\n", - "epochs = 30\n", - "lr = 1e-4" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "root_dir = \"../DVSGESTURE\"\n", - "_ = DVSGesture(save_to=root_dir, train=True)\n", - "_ = DVSGesture(save_to=root_dir, train=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "n_time_steps = 50\n", - "to_raster = ToFrame(sensor_size=DVSGesture.sensor_size, n_time_bins=n_time_steps)\n", - "\n", - "snn_train_dataset = DVSGesture(save_to=root_dir, train=True, transform=to_raster)\n", - "snn_test_dataset = DVSGesture(save_to=root_dir, train=False, transform=to_raster)" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The transformed array is in shape [Time-Step, Channel, Height, Width] --> (50, 2, 128, 128)\n" - ] - } - ], - "source": [ - "sample_data, label = snn_train_dataset[0]\n", - "print(f\"The transformed array is in shape [Time-Step, Channel, Height, Width] --> {sample_data.shape}\")" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "snn_train_dataloader = DataLoader(snn_train_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True)\n", - "snn_test_dataloader = DataLoader(snn_test_dataset, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=False)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Network Module\n", - "\n", - "We need to define a `nn.Module` implementing the network we want the chip to reproduce." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "device: NVIDIA RTX A4000\n" - ] - } - ], - "source": [ - "if torch.cuda.is_available():\n", - " device = torch.device('cuda:0')\n", - " print('device: ', torch.cuda.get_device_name(0))\n", - "else:\n", - " device = torch.device('cpu')" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "class SNN(nn.Module):\n", - " def __init__(self) -> None:\n", - " super().__init__()\n", - "\n", - " self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False)\n", - " self.iaf1 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - " self.pool1 = sl.SumPool2d(2,2)\n", - " self.pool1a = sl.SumPool2d(6,6)\n", - "\n", - " self.conv2 = nn.Conv2d(10, 10, 2, 1, bias=False)\n", - " self.iaf2 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - " self.pool2 = sl.SumPool2d(3,3)\n", - "\n", - " self.conv3 = nn.Conv2d(10, 10, 3, 1, bias=False)\n", - " self.iaf3 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - " self.pool3 = sl.SumPool2d(2,2)\n", - "\n", - " self.flat = nn.Flatten()\n", - "\n", - " self.fc1 = nn.Linear(810, 100, bias=False)\n", - " self.iaf4 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - "\n", - " self.fc2 = nn.Linear(100, 100, bias=False)\n", - " self.iaf5 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - "\n", - " self.fc3 = nn.Linear(100, 100, bias=False)\n", - " self.iaf6 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - "\n", - " self.fc4 = nn.Linear(100, 11, bias=False)\n", - " self.iaf7 = sl.IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, surrogate_grad_fn=PeriodicExponential())\n", - "\n", - " self.merge_fc = sl.Merge()\n", - " self.merge_conv = sl.Merge()\n", - "\n", - " def detach_neuron_states(self):\n", - " for name, layer in self.named_modules():\n", - " if name != '':\n", - " if isinstance(layer, sl.StatefulLayer):\n", - " for name, buffer in layer.named_buffers():\n", - " buffer.detach_()\n", - "\n", - " def init_weights(self):\n", - " for name, layer in self.named_modules():\n", - " if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear):\n", - " nn.init.xavier_normal_(layer.weight.data)\n", - "\n", - " def forward(self, x):\n", - " \n", - " con1_out = self.conv1(x)\n", - " iaf1_out = self.iaf1(con1_out)\n", - " pool1_out = self.pool1(iaf1_out)\n", - " pool1a_out = self.pool1a(iaf1_out)\n", - "\n", - " conv2_out = self.conv2(pool1_out)\n", - " iaf2_out = self.iaf2(conv2_out)\n", - " pool2_out = self.pool2(iaf2_out)\n", - "\n", - " merged_conv_out = self.merge_conv(pool1a_out, pool2_out)\n", - "\n", - " conv3_out = self.conv3(merged_conv_out)\n", - " iaf3_out = self.iaf3(conv3_out)\n", - " pool3_out = self.pool3(iaf3_out)\n", - "\n", - " flat_out = self.flat(pool3_out)\n", - " \n", - " fc1_out = self.fc1(flat_out)\n", - " iaf4_out = self.iaf4(fc1_out)\n", - "\n", - " fc2_out = self.fc2(iaf4_out)\n", - " iaf5_out = self.iaf5(fc2_out)\n", - "\n", - " fc3_out = self.fc3(iaf5_out)\n", - " iaf6_out = self.iaf6(fc3_out)\n", - "\n", - " merge_fc_out = self.merge_fc(iaf4_out, iaf6_out)\n", - "\n", - " fc4_out = self.fc4(merge_fc_out)\n", - " iaf7_out = self.iaf7(fc4_out)\n", - "\n", - " return iaf7_out" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "snn = SNN().to(device)" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "snn.init_weights()" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "optimizer = Adam(snn.parameters(), lr=lr, betas=(0.9, 0.999), eps=1e-8)\n", - "loss_fn = CrossEntropyLoss()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Define train and test" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "def train(dataloader, model, loss_fn, optimizer, epochs, test_func, dataloader_test):\n", - " epochs_y = []\n", - " epochs_x = []\n", - " epochs_acc = []\n", - " model.train()\n", - "\n", - " for e in range(epochs):\n", - " losses = []\n", - " batches = []\n", - " batch_count = 0\n", - " train_p_bar = tqdm(snn_train_dataloader)\n", - "\n", - " for X, y in train_p_bar:\n", - " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", - " X = X.reshape(-1, 2, 128, 128).to(dtype=torch.float, device=device)\n", - " y = y.to(dtype=torch.long, device=device)\n", - "\n", - " # forward\n", - " pred = model(X)\n", - "\n", - " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", - " pred = pred.reshape(batch_size, n_time_steps, -1)\n", - "\n", - " # accumulate all time-steps output for final prediction\n", - " pred = pred.sum(dim = 1)\n", - " loss = loss_fn(pred, y)\n", - "\n", - " # gradient update\n", - " optimizer.zero_grad()\n", - " loss.backward()\n", - " optimizer.step()\n", - "\n", - " # detach the neuron states and activations from current computation graph(necessary)\n", - " model.detach_neuron_states()\n", - "\n", - " train_p_bar.set_description(f\"Epoch {e} - BPTT Training Loss: {round(loss.item(), 4)}\")\n", - "\n", - " batch_count += 1\n", - " losses.append(loss.item())\n", - " batches.append(batch_count)\n", - "\n", - " epochs_y.append(losses)\n", - " epochs_x.append(batches)\n", - "\n", - " acc = test_func(dataloader_test, model)\n", - " print(f'Epoch {e} accuracy: {acc}')\n", - " epochs_acc.append(acc)\n", - "\n", - " return epochs_x, epochs_y, epochs_acc\n" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [], - "source": [ - "def test(dataloader, model):\n", - " correct_predictions = []\n", - " with torch.no_grad():\n", - " test_p_bar = tqdm(dataloader)\n", - " for X, y in test_p_bar:\n", - " # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width]\n", - " X = X.reshape(-1, 2, 128, 128).to(dtype=torch.float, device=device)\n", - " y = y.to(dtype=torch.long, device=device)\n", - "\n", - " # forward\n", - " output = model(X)\n", - "\n", - " # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes]\n", - " output = output.reshape(batch_size, n_time_steps, -1)\n", - "\n", - " # accumulate all time-steps output for final prediction\n", - " output = output.sum(dim=1)\n", - "\n", - " # calculate accuracy\n", - " pred = output.argmax(dim=1, keepdim=True)\n", - "\n", - " # compute the total correct predictions\n", - " correct_predictions.append(pred.eq(y.view_as(pred)))\n", - "\n", - " test_p_bar.set_description(f\"Testing Model...\")\n", - " \n", - " correct_predictions = torch.cat(correct_predictions)\n", - " return correct_predictions.sum().item()/(len(correct_predictions))*100" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Training loop (HPO)" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "12d134e3b89e41888c9c47892b8e6491", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - " 0%| | 0/359 [00:00" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "y_avg = []\n", - "for y in epochs_y:\n", - " y_avg.append(np.mean(y))\n", - "\n", - "plt.plot(np.arange(len(epochs_x)), y_avg, marker = '.')\n", - "plt.xlabel('epoch')\n", - "plt.ylabel('average loss')\n", - "plt.ylim(0,)\n", - "plt.xticks(np.arange(len(epochs_x)))\n", - "for i, txt in enumerate(y_avg):\n", - " if i%5 == 0:\n", - " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", - " else:\n", - " pass\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAG2CAYAAACZEEfAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABq60lEQVR4nO3dd1hT1+MG8DfsHUW2TAfixK3gqqO46q6jtVWr1bZi62idraPVFrWt7dfROn5urauOOloXrqqooOBEBERBZYhI2DPn9weaGlmJBIH4fp4nT8vNPfecQMx9c+6550iEEAJEREREWkqnohtAREREVJ4YdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirVWjYOXPmDPr06QMHBwdIJBLs27dP6XkhBObMmQN7e3sYGxujW7duCA8PV9onKSkJw4cPh4WFBapVq4YxY8YgLS3tNb4KIiIiqswqNOykp6fD09MTK1asKPL5xYsXY+nSpVi5ciUuXrwIU1NTdO/eHVlZWYp9hg8fjps3b+LYsWM4ePAgzpw5g3Hjxr2ul0BERESVnKSyLAQqkUiwd+9e9O/fH0BBr46DgwO+/PJLfPXVVwAAmUwGW1tbbNiwAcOGDUNoaCgaNGiAwMBAtGzZEgBw+PBh9OrVCw8ePICDg0NFvRwiIiKqJPQqugHFiYqKQlxcHLp166bYJpVK0aZNGwQEBGDYsGEICAhAtWrVFEEHALp16wYdHR1cvHgRAwYMKPLY2dnZyM7OVvwsl8uRlJSEGjVqQCKRlN+LIiIiIo0RQiA1NRUODg7Q0Sn+YlWlDTtxcXEAAFtbW6Xttra2iufi4uJgY2Oj9Lyenh4sLS0V+xTFz88P3377rYZbTERERBUhJiYGjo6OxT5facNOeZo5cyamTJmi+Fkmk8HZ2RkxMTGwsLCowJYRERGRqlJSUuDk5ARzc/MS96u0YcfOzg4AEB8fD3t7e8X2+Ph4NG3aVLFPQkKCUrm8vDwkJSUpyhfF0NAQhoaGhbZbWFgw7BAREVUxpQ1BqbTz7Li5ucHOzg7+/v6KbSkpKbh48SK8vLwAAF5eXkhOTsbly5cV+5w4cQJyuRxt2rR57W0mIiKiyqdCe3bS0tIQERGh+DkqKgohISGwtLSEs7MzJk2ahAULFqBu3bpwc3PD7Nmz4eDgoLhjq379+ujRowfGjh2LlStXIjc3FxMmTMCwYcN4JxYREREBqOCwExQUhM6dOyt+fj6OZuTIkdiwYQOmTZuG9PR0jBs3DsnJyWjfvj0OHz4MIyMjRZmtW7diwoQJ6Nq1K3R0dDBo0CAsXbr0tb8WIiIiqpwqzTw7FSklJQVSqRQymYxjdoiIiKoIVc/flXbMDhEREZEmMOwQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQERGRVmPYISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEqQn5+P2bNnw83NDcbGxqhduzbmz58PIYRin1GjRkEikSg9evToUeJxU1NTMWnSJLi4uMDY2Bje3t4IDAxU2ictLQ0TJkyAo6MjjI2N0aBBA6xcubJcXqc206voBhAREVVmixYtwu+//46NGzeiYcOGCAoKwkcffQSpVIovvvhCsV+PHj2wfv16xc+GhoYlHvfjjz/GjRs3sHnzZjg4OGDLli3o1q0bbt26hZo1awIApkyZghMnTmDLli1wdXXF0aNHMX78eDg4OKBv377l84K1EHt2iIiISnD+/Hn069cPvXv3hqurK9599134+Pjg0qVLSvsZGhrCzs5O8ahevXqxx8zMzMTu3buxePFidOzYEXXq1MG8efNQp04d/P7770p1jxw5Em+99RZcXV0xbtw4eHp6FqqbSsawQ0T0hiivyzF+fn5o1aoVzM3NYWNjg/79+yMsLExpn08++QS1a9eGsbExalhZo0O3njgTGFIeL1PjvL294e/vjzt37gAArl69irNnz6Jnz55K+506dQo2NjaoV68ePvvsMzx58qTYY+bl5SE/Px9GRkZK242NjXH27Fmluvfv34+HDx9CCIGTJ0/izp078PHx0eArfAMIEjKZTAAQMpmsoptCRFRuvv/+e1GjRg1x8OBBERUVJXbt2iXMzMzE//73P8U+I0eOFD169BCxsbGKR1JSUonH7d69u1i/fr24ceOGCAkJEb169RLOzs4iLS1Nsc+qVavE6dOnxdJ9Z4XDqF+FcZ3WQtfCWvwRcLfcXq+m5Ofni+nTpwuJRCJ09fSERCIRP/zwg9I+27ZtE3/99Ze4du2a2Lt3r6hfv75o1aqVyMvLK/a4Xl5eolOnTuLhw4ciLy9PbN68Wejo6Ah3d3fFPllZWWLEiBECgNDT0xMGBgZi48aN5fZaqxpVz98MO4Jhh4jeDL179xajR49W2jZw4EAxfPhwxc8jR44U/fr1K1M9CQkJAoA4ffq00vbIhFThOv2gcJl+UNh/tEwAEE6f/J94lJxRpvrK27Zt24Sljb2w7jtV2I9eLqzemSLMLKqJDRs2FFsmMjJSABDHjx8vdp+IiAjRsWNHAUDo6uqKVq1aieHDhwsPDw/FPj/++KNwd3cX+/fvF1evXhXLli0TZmZm4tixYxp9jVWVqudvXsYiInpDlMflmKLIZDIAgKWlpWLbydsJGLIyAAKAPCcLadePQ09qC4l5DdxLzCjbCytnU776ChLPfjCp3wkG1q4wbdgFep7vYP73PxRbplatWrCyskJERESx+9SuXRunT59GWloaYmJicOnSJeTm5qJWrVoACsb1zJo1C0uWLEGfPn3QpEkTTJgwAUOHDsVPP/2k8ddZklhZJs5HJiJWlvla69UU3o1FRPSGmDFjBlJSUuDh4QFdXV3k5+fj+++/x/DhwxX79OjRAwMHDoSbmxsiIyMxa9Ys9OzZEwEBAdDV1S21DrlcjkmTJqFdu3Zo1KgRElKz8N2BWzh4LRapVw7h6an1ELlZ0LN0hM3QBdDV04erlUl5vuwyEUJAlpIOI8lLfQMSHcTLMuEfGo+O7tbQ11V+/sGDB3jy5Ans7e1LrcPU1BSmpqZ4+vQpjhw5gsWLFwMAcnNzkZubCx0d5WPr6upCLper9TpiZZmISkyHm5Up7KXGpe4vhEBadh4S03KwIzAaq87chRCAjgTwG9gYQ1s5q1V/RWPYISJ6Q+zcuRNbt27FH3/8gYYNGyIkJASTJk2Cg4MDRo4cCQAYNmyYYv/GjRujSZMmqF27Nk6dOoWuXbuWWoevry9u3LiBM2f+xbZL0fD7OxQpWXnQkQDjPx4Ji48H45e/ApF8aTcS/1oI51E/IyMnv9xec1lcikqC3z+hkLi0gOz8DuhaWMPAyhk58ZFICdwHsyZvY8zGIEj18mF6cy8+GfkeOnrWxd27dzFt2jTUqVMH3bt3Vxyva9euGDBgACZMmAAAOHLkCIQQqFevHiIiIjB16lR4eHjgo48+AgBYWFigU6dOmDp1KoyNjeHi4oLTp09j06ZNWLJkicqvY0dgNGbuuQ75s7Ay+W13tHa1RGJaDhLTshWPx6nKP2flFg5UcgFM330dJ8Meo4VzdXjYm8PDzgLW5iXfZl/RJEK8MAz/DZWSkgKpVAqZTAYLC4uKbg4RqSA/Px/z5s3Dli1bEBcXBwcHB4waNQrffPMNJBIJAGDevHnYvn07YmJiYGBggBYtWuD7779HmzZtij2uq6sr7t+/X2j7+PHjsWLFCgDA6tWr8ccff+DKlStITU3F06dPUa1atXJ5nZrk5OSEGTNmwNfXV7FtwYIF2LJlC27fvl1sOWtrayxYsACffPJJicefMGEC/vrrL2za8zd+u5yKwHtPAQCNa0rhN7AxGtWUAijoZbj9IAlvN6+D6t0/h+dbvbHPtx3MDCvH9+878alYfPg2jocmAAAM5NnQD9mFsIv+kGfIoGtmiU49+8P73U/w961EJDxNweM9C5CTcBciOx3VrGzR3edt/PrjQtja2iqO6+rqilGjRmHevHkACsLnzJkz8eDBA1haWmLQoEH4/vvvIZVKFWXi4uIwc+ZMHD16FElJSXBxccG4ceMwefJkxfv8ZUIIPHiaidtxqQiMeoLV/0a98u/CUE8H2Xml9yJZmRmivr05POwKwo+HvTnq2JjBUE9X7V4ldah6/q4c7ywiIjWpMtGbu7s7li9fjlq1aiEzMxO//PILfHx8EBERAWtr6yKPGxgYiPz8/3oabty4gbfffhuDBw9WbMvIyECPHj3Qo0cPzJw5s3xfqAZlZGSofUlElcsxQgh8/vnn2LN3L0Z+vx7j9sUgN1/AxEAXU952xyhvV+i9cJnHXmoMSyMrGOhKYGEARCSk4audV/H7B82LPYG/DrGyTPxy7A7+vPwAcgHo6kgwrJUTJnatCxuLgYiVZeJeYgZcrUwUJ+05/eQ4F/kEe1utxZGb8cjMLXjvBAD4dHcEBjTLxDtNHGBpaoCAq6GISkxHrCwT9lJjDBkyBEOGDCmxTXZ2dli/fn2xgSE1Kxd34lMRGpuK23EpuB2bittxqUjLzivxuLYWhnCqbgIrM0NYmRvAyswQ1uaGBT+bGcL62XZZZi7aLTwB+QvdIjoSYEx7NzxMzsTt2FREPUlHYlo2/g3Pxr/hiYr99HQkqGFmgPiUbEW5iroExp4dsGeHqCp65513YGtri7Vr1yq2DRo0CMbGxtiyZUuRZZ7/Wz9+/LhKl2QAYNKkSTh48CDCw8MLnYhPnTqFzp07V5menVGjRuH48eNYtWoVrJ1qw//cRfw0+yt8PGY0Fi1ahLS0NHz77bcYNGgQ7OzsEBkZiWnTpiE1NRXXr19XzAj88uWY8ePHY/OWrajzwbdI1LECALSrWwPzB7dGXYcauHv3Lnbs2AEfHx9YW1vjwYMHWLhwIc6dO4edxwLw2e5I5OTLMbV7Pfh2rvPafy+yjFz8djoCG87dU/Ri9Gxkh6+610NtazOVj5OenYejt+KwN/gRzoY/VgQEPR0J6tqa4XZsKgQKTvrTutfDO54OKh334NVHWHwkDHIBSAB08bCBjo4Et+NSEJNU9IBhfV0J6tiYw8XSBEduxuHFE72uRIKzMzqr3MuyIzAas/bcQL4Q0JVI8MPARkqBJSMnD+HxaQiNTcHtuFTFf2WZuYWOpW7dpWHPDhFpNW9vb6xevRp37tyBu7u74s6i4sYy5OTkYPXq1ZBKpfD09FSpjpycHGzZsgVTpkyp0B4HTVm2bBlmz56NUR9/gieJj6FrZgnTBl3RpN9YAAW9PNeuXcPGjRuRnJwMBwcH+Pj4YP78+UpLH0RGRiIxseAbfHJGjmLG35DfJyv22QGgh3Q96o4aBSMjI/z777/49ddf8fTpU9ja2qJjx444f/486tVzx7d5Rpi55zp+OhqGRjWl6ORedK+bpmXl5mNTwD2sOBmpODG3drPEjJ4eaO5c/OzHxTE11MOAZo4Y0MwRCalZ2B/yCPtCHuLGwxSExqYq9pMLYOHhMCw8HFbC0YomAPjfTlDaZmdhpBg7U//Zf2tZmyoGTRcVVtQJG0NbOaOju3WhXq3nTAz04OlUDZ5O1f5rpxA4dD0WE/4IVto3XwjcS8zQ+OWs0rBnB+zZIaqK5HI5Zs2ahcWLFyvdWfTyZaWDBw9i2LBhyMjIgL29Pfbt24dWrVqpVMfOnTvx/vvvIzo6Gg4Ohb+FV7WeHaDgUs3LlyUkAGb28kBdG3PFZY0apoYw0Ct6dpJYWSaiHqcjPCEVy05EIDEtBwAwvI0zpvXwgNRYX+12zdxzDdsuxUBqrI8DE9rDuUb53KEVK8tEZEI6QmNlWH/uHh7JsgAA9WzNMb1nPXSuZ6PxYLsrKAZT/7xWaLu+jgQ6OiXXJZcL5MoLn6ZHtHVBj8Z28LCzgKWpQaltKOoSXHkr6r3Gnh0iqjDlOYCwvOpW5c4iAOjcuTNCQkKQmJiINWvWYMiQIbh48SJsbGxKrWPt2rXo2bNnkUGnqop6nI6Xz50CwA9/Fx6gLDXWh5WZwbMAVDCOIzY5E0dvxStdFqlrYwa/gY3R0tWy0DFUNa9vQ4TGpiIkJhnjNgdhz3hvmBho9hS1IzAaM/Zcx4tf8R2kRpjiUw8DmtWEbinB41W1r2sFHQkKnfTPTC/9pF9cYPisc221/r3YS41f+79te6kx/AY2LlOvkqawZwfs2aE328u3pb7OAYRlqftV7yyqW7cuRo8eXerA4vv376NWrVrYs2cP+vXrV+Q+VbFnZ8uF+/hm3w2lbRIUXL5JzcpDYlo2nqTnIL+I3oSiSACcnvoWnGuYlrltsbJM9Fl2FolpOejr6YD/DWuqsV6WqzFP0W/FeaVtEgAnv3oLrlZlb3tpShv3Ul5lK4Py7FVizw4RlSpWlqkIG0DBN89Ze26go7t1uX/7KqrumXuuq1z3q9xZBBRc/srOzi71+OvXr4eNjQ169+5d6r5VRWpWLpb6hwMoONELoMiTp1wukJyZWzDfSmo2HqdlIzEtB1djkrH/6iOlYwoAD5OzNBJ27KXGWPF+cwz/v4vYf/URmjhK8XGHWmU6Zr5cYMuF+/D7J7TQcwJArCzrtYSd0sa9lFfZyqAiepVexrBD9Aa7HZtS6JLG6xpAGJVY+HKKXABbL9zHlLfrlTqWoU+fPvj+++/h7OyMhg0bIjg4GEuWLMHo0aMBAOnp6fj+++/Rt29f2NvbIzExEStWrMDDhw+VbiN/+c4ioCAQrV+/HiNHjoSeXuGPybi4OMTFxSmWArh+/TrMzc3h7OystERCaV735cOfj95BQmo2XGuYYMNHrZ+d6AufPHV0JLA0NYClqQHcbc2V2nvw2qNCl1Q0OQNym1o18HXv+vj2wC34/XMbDRws4F3b6pWOFRqbgpl7riMkJrnI5zXd9tKU5aRfGQJDVca1sYgqWH5+PmbPng03NzcYGxujdu3amD9/Pl68wrxnzx74+PigRo0akEgkCAkJUenYu3btgoeHB4yMjNC4cWP8/fffiueS0nOw+Mgd5CbGIGH3d4j+ZQiilwxC7MbJ0MtUby2kV+FWzLfp5Scj0Wf5Wfwb/rjE8suWLcO7776L8ePHo379+vjqq6/wySefYP78+QAKenlu376NQYMGwd3dHX369MGTJ0/w77//omHDhgAKTt63wsIR9SBW6djHjx9HdHS0Iji9bOXKlWjWrBnGji24i6ljx45o1qwZ9u/fr/Lr3xEYjXYLT+D9NRfRbuEJ7AiMVrnsq7j+QIZNAfcAAPP7N4KrlSm8atdQe9yH38DG0H12aam8xmCM8nbFgGY1kS8X+PyPYDxKVm89psycfCz85zb6LDuLkJhkmBvqYX7/Rq+l7VQ5ccwOOGaHKtYPP/yAJUuWFJoc7/vvv1dMjrd582ZERUXBwcEBY8eORXBwMJo2bVricc+fP4+OHTvCz88P77zzDv744w8sWrQIV65cgdShFkauu4Sw8AjEbZ4C8yZvw6R+J0gMTGCW8QgnF4+Dg71dub7uM3ceY8S6S4qfdSTA2w1scT7iCVKfTYjWoa4VpvfwUMy8q0kVOVbp4dMMtF90skxzn6gjXy4w4LdzuPZAhr6eDlj6XrMyHe913NmTmZOPQb+fx63YFDRxlGLnJ14w0i99ba5/wx/j6703EJ1UsLhor8Z2mNunIWwtjF5b2+n1UfX8zbADhh2qWOpMjnfv3j24ubmpFHaGDh2K9PR0HDx4ULGtbdu2cKnbAHc93sfj1Gyk/f0T2tezxe//tx43HsgwZddVpGblYX6/hvjQy1WTL1NJbr4cPf/3LyIS0jCkZcG8JM9PPknpOVh+IgKbL9xDbn7Bx1NfTwd85VNPY7cjRyelo9OPp5TuyinPsPGi7Lx8jFx3CRfuJhV6btvYtvCqXUPjdW4KuIc5f92EuZEe/L/sBBtzI43XUR5ikjLQZ/lZJGfkYkhLRywa1KTYActP0rKx4FAo9gY/BFBwl9V3/RqhWwPbIvcn7aDq+ZuXsYgqmLe3N/z9/XHnzh0AUEyO17NnzzIdNyAgAN26dVPa1rBVB+w/dgqPU7Phbm2KrKggtGjSEKOG9Mfwzk2QsWs6Mu4E4Odjd/A0PadM9ZdkU8B9RCSkwdLUAF/3bqB0OcXS1ABz+jTAiS/fwoBmNSGRAPuvPkLXJacwb/9NPEkrfXDxy4QQiEhIxfpzURizIRA+S87g5a95z8cqlafUrFyM3hBYZNDRkaBcxo8kpGThx2eT103rXq/KBB0AcLI0wbL3mkFHAuwMeoCtFwtf6hNCYGdQDLouOY29wQ+hIwE+aueKo1M6MeiQAgcoE1WwGTNmICUlBR4eHkqT4w0fPrxMx42Li1NahPDQtVgcCM9ATupTtHazxAIfR9T7Kg0LFy7EggULsGjRIvz99z/4+puvoWNkhiXHHDC/f6OyvrxCEtOy8euxgmA3rXu9Yiegc7I0wS9Dm+LjDm5YdDgMZ+48xobz9/Dn5Qf4pGMtjOngVuI8LE/SsnE2IhFnwxNxNiIRsc8mjyvJoeuxaO1mWS7zrSSkZmHUukDcik2BqYEuhrV2xvpzUYrBvpamBuWyEOb8Q6FIzc6Dp6MU77dx0fjxy1uHutaY2t0Diw7fxrcHbqK+vQVauBTMbnz3cRpm7b2uCI8N7C2wcFBjNHGsVoEtpsqIYYeogqk6OV5ZbDx/D/MO3ES+XMBQTwebRrdG0uN4AEC/fv0weXLBNP9NmzbFP/6ncSXkH2x1aYz3WjujgYNmL+3+eDgMqdl5aFTTAoNbOpW6f0MHKTaNbo1zEYlY+M9tXH8ow8/H7mDThfuY2LUuOrpb4cHTTDhIjfHgaSb+DX+Mf8MTcSs2Rek4Bno6aO1qiQ51rdC+rhWuPZDhm70Fc5c8vw17y4X7ePA0A/8b1uyVZgEuTlRiOkasu4iYpExYmRlg/ajWaOwoxccd3HA1Jhlz/rqJhNRsTNl5Fas+aFHqnWiqOnPnMQ5cfQQdCfD9gMblNmleefu0Uy1cf5iMv6/HYdymIMzt0wDXH8qwMeA+cvLkMNbXxeS362J0OzelBUeJnmPYIapgU6dOxYwZMzBs2DAAQOPGjXH//n34+fmVKezY2dkhLi4OPx65jRUnIwEADarLkeLqCCN9XVhZWUFPTw8NGjRQKufd0hMRe49ALoBvD9zE9nFtNTax27UHydh5OQYAMK9PQ7VOvu3qWOEv33Y4dD0WPx4JQ3RSRqHJ8V5W394CHZ+Fm1aulkoDXBs6SPFWvf/mLrkUlYRpf17DqbDH6L/iHNaMaIE6NuYlHF01V2OS8dGGQCSl58Clhgk2jW4Nl2dz0jy/ndheaozBqwJw7FY8VpyMwOdd65a53qzcfMz+q+D3M9LbtVwGeb8uEokEi9/1RNC9p0hIzcYX20MUz3Vyt8aC/o3gZPn6biGnqocRmEiDYmWZOB+ZiFiZ6rfKvurkeKVp07Ytlm/Zpwg6X77tjvyYa/Dy8gIAGBgYoFWrVggLU16M8M6dO2jTxB1G+jq4GJWEQ9djCx37VcjlAvP234QQwIBmNV9paQEdHQn6eDrg+JROmPK2e5H79Gpsh/8Na4rAr7vhn4kdMLNXfXSoa13knTz2UmPFeKF+TWti92fecJAaISoxHf1XnMfxW/Fqt/FFp8IS8N6aC0hKz0HjmlLs/sxbEXRe5OlUDQv6FVwyXHL8Dk6GJRTaR12/nYrE/ScZsLUwLPZ3VZWkZhVMcvgiiQTwG8igQ6Vj2CHSkFedN+X55HiHDh3CvXv3sHfvXixZsgQDBgxQ7JOUlISQkBDcunULABAWFoaQkBDExcUp9hkxYoRiCYSMnDxk1PXB3ZBzSL20B583N8aTf7ciKChIafK8qVOnYseOHVizZg0iIiKwfPlyHDhwAF9N+gKfdaoDAPjhUCgyc/LL/PvZF/IQV6KTYWKgixk9Pcp0LAM9HbR0LXpV6g/buqJf05qwNjcs8vmSNKopxf7P26O1qyXSsvMwdnMQlp8Ix6vctLrnygN8vDEIGTn56FDXCtvGtYWVWfFtGtLKCcPbOEMIYOK2YNxLTFe7zuciH6dh5amCkDvnnYYwN9LcJbmKUtQklEIA95+oNwcPvZkYdog0oLhlF1Tp4SltcjwA2L9/P5o1a6ZYumDYsGFo1qwZVq5cqdgnOjoasbGxSErPwftrLuJmvj3s+0+DSdRpzPqwJ/7880/s27cPjRr9N+h4wIABWLlyJRYvXozGjRvj//7v/7B79260b98en3SqhZrVjPFIloXfT0eW6feTlp2Hhf8UrFc1oUsdxZwnZeFmZYqXr4JpYkZcKzNDbPm4DT5oWxA8fjp6B75/XEH6s7l/SiOEwKrTkZiy8yry5AL9mzpg7chWKg0+ntunIZo7V0NKVh4+2XxZ5Tpfrn/2vhvIyZejk7s1ejUu3/mSXpfy+nvTm4Hz7IDz7FDZ7b/6CF9sCy60vbzmTSlKrCwTgVFJ+OloGKKTMiE11se6US3RwuXVV6L+53osPtt6BQZ6OvCf0umVLxcs/Oc2Vp6OhEsNExyd3BGGeqVPDqeK8l4gcdulaMz56wZy8wU87MyxZkTLEn8HcrnA93+HYu3ZKADA2A5umNmzvloDjuNTsvDOsrN4nJqN3k3ssfy9ZmqNmdoX/BCTdoTAUE8HRyd3LPKyWVVV1RfEJM3jpIJqYNihsniSlo0Bv51DdJJyL46OBDg3o8trmaV1R2A0Zuy5rpg7Rmqsh92feZd5gK0QAsP/7yLORz5Bj4Z2WPlhC7WPEZWYDp9fTiM3X+D/RrTU+Nwn5T0jbtC9JHy65QoS07JR3UQfK95vDu86hddqys7Lx1e7ruHAs4Uyv+5VH2M7vtoiloH3kvDe6gvIkwvM6uWBcR1rq1ROlpGLrktOITEtB1/5uGNCl7IPdK5sOAMyvYiTCtJr5+rqColEUujh6+sLoGDelw8//BB2dnYwNTVF8+bNsXv37lKPu2LFCri6usLIyAht2rTBpUuXlJ7/5JNPULt2bRgbG8Pa2hr9+vXD7du3y+U1viw1Kxcj11961pOip9TN3sRRCjsNXK4pTawsUynoFLQrD6YamLNFIpFg7rO7pg7fjMO5iES1j7Hg4C3k5gt0crdG1/o2ZW7Ty14cZFweWrpa4sDn7dDEUYqnGbn4cN0lbDgXpTSO5/lkgQeuPoK+rgS/Dm36ykEHAFq5WmJun4K75Bb+c1vl3/viI7eRmJaD2tamZaq/Mivvvzdpp0oddlRZIFEIgTlz5sDe3h7Gxsbo1q0bwsPDK7DVb67AwECs/DsQjhM2w9F3M+yGLQAAxQrTI0aMQFhYGPbv34/r169j4MCBGDJkCIKDC1/+eW7Hjh2YMmUK5s6diytXrsDT0xPdu3dHQsJ/d6u0aNEC69evR2hoKI4cOQIhBHx8fJCfX/ZBtSXJys3H2E1BuPEwBTVMDbBnfDucm9EFc96pD10JEBIjw7pz98q1DQDwb3hiodmA5QIamw24np05PmxbMBndtwduIi9f9bvEToYlwP92AvR0JJjTp4HGbmF/3eylxtj5iZdiccp5B25h2p/XcP9JOv6+/giDfj+PcxFPYGqgi3WjWqF/s5plrvODti54t4Uj5AKY8McVPHha8t8zOPop/rhUMCh+Qf/GGrtUSKQNKnXYWbRoEX7//XcsX74coaGhWLRoERYvXoxly5Yp9lm8eDGWLl2KlStX4uLFizA1NUX37t2RlVX6bKmkOflygeN3M+B3Oh66ptWha1Yd6eGXoFfNHnU9WwEoWJjy888/R+vWrVGrVi188803qFatGi5fvlzscZcsWYKxY8fio48+QoMGDbBy5UqYmJhg3bp1in3GjRuHjh07wtXVFc2bN8eCBQsQExODe/fuldvrzcuXY8IfwbhwNwlmhnrYOLo1alubwV5qjNHta+Gbdwq+lf/wdygCIstvBfGYpAwsPly4F0vTAzcnd3NHdRN93IlPw5YL91Uqk5Mnx/wDBXePfdTOFbWtzTTWnopgpK+LJUM88U3v+tCRALsuP0CnH09h/NZg3IlPg6mBLraP80KHutYaqU8ikWBB/0ZoXLOgR+nTLZeRlVt0gM/Ll+PrvTcgBDCwec3XNk6MqKqo1GHn/Pnz6NevH3r37g1XV1e8++678PHxUVzGEELg119/xTfffIN+/fqhSZMm2LRpEx49eoR9+/ZVbOPfEBk5edh4/h66/HwKM/f+N8GbyM9F+q1TMGvyNj7eeBmRj9Pg7e2NHTt2ICkpCXK5HNu3b0dWVhbeeuutIo+dk5ODy5cvK63vpKOjg27duiEgIKDIMunp6Vi/fj3c3Nzg5FT67LyvQi4XmL77Oo6HxsNATwf/N7JloQnbRnm7KnoBJvxxBY+SNX977OPUbHy49iIS03JgY26ouIT2fOCmJrv5pSb6+Kp7PQDAkmN3VFqfasP5KNxNTIeVmSG+0MAkeZWBRCLBxx1q4ZehTQs9l5mbDytzA43WZ6Svi5UftoClqQFuPEx5FmgKD7PcGHAft2JTIDXWx6xe9TXaBiJtUKnDTmkLJEZFRSEuLk7pZCiVStGmTZtiT4YAkJ2djZSUFKUHqSchJQs/HrkNL78TmLv/Ju4/yYCFkR6eX6TIuHMB8qw0mDbqitC4VPT89V90Hu+H7Jwc1KhRA4aGhvjkk0+wd+9e1KlTp8g6EhMTkZ+fr7S+EwDY2toqzS8DAL/99hvMzMxgZmaGf/75B8eOHYOBgWZPPEBBwP7+71DsvvIAujoSrHi/OdrWKvwtWiKR4IcBjdHA3gJP0nPwWQnfyl9FSlYuRq67hHtPMuBY3Rj7J7THuRldsG1sW5yd0blc7lAZ1soZDewtkJKVh5+O3ilx34TULCz1jwAATO9RTyvmeXlRUXP4aPLS4YtqVjPG8meLYe6+8gCbX+pZi5VlYsnRgokhZ/T0KHEuH6I3VaUOO8+n0Pfw8IC+vj6aNWuGSZMmKRZIfH7CU+Vk+CI/Pz9IpVLFo7x6ALRRWFwqpu66ivaLTmLFyUjIMnPhUsME3/VriAuzumLhoMbQlUiQdu0oTGq1xNeDvdHJ3Ro5+XL4zZ+HS7djsHzzHgQFBWHKlCkYMmQIrl+/XuZ2DR8+HMHBwTh9+jTc3d0xZMiQcrmUueJkhOK24sWDmuDtEu4sMjbQxaoPW6CaiT6uPpBh7l83X2lyupdl5uTj4w1BuBWbAiszA2we0wZ2UqNyH7ipqyPBvL4NAQDbA6Nx46Gs2H0XHw5DWnYePJ2qYVBzx3JpT0V63XO+eNexwsyeBT023x24hcB7/62a/t2BW0jPyUdz52oYqsJaY0Rvokoddl5cIPHKlSvYuHEjfvrpJ2zcuLFMx505cyZkMpniERMTo6EWaychBM6GJ2LEukvo/usZ7Lr8ADn5crR0qY6VH7TAiS/fwggvV5gY6GFoK2dsf782cqKvYvl3X+Kzt+pgw0etMLN9daReOQjTtz/HjzcMsD1SgklTZ6Fly5ZYsWJFkfVaWVlBV1cX8fHKU/bHx8fDzk55ojSpVIq6deuiY8eO+PPPP3H79m3s3btXo7+HzRfuK3o0Zr/TAINalH4Sd7I0wbJn38p3BMUoBpC+qtx8OXz/uIJL95Jg/myskJvV65tHpbWbJfp6OkAIPFv6oXB4C45+ij8vPwAAzOvTQGOLWlYm9lJj+A0sCPZA+Vw6fNnHHdzQx9MBeXKB8VuvID4lCydux+OfG3HQ1ZHg+wGNtfJ3TaQJlXoh0NIWSHx+wouPj4e9vb2iXHx8PJo2bVrscQ0NDWFoyK7eksTKMhEen4bw+FTsuvwAt+NSARTMHdOzkT0+7uCGZs5FT9d/aPc22NjY4MMhAwEUXNLxdimY76VXEwccjwW2XYrBsVsJkGfkFXvXlIGBAVq0aAF/f3/0798fACCXy+Hv76+05MHLhBAQQiA7u/RxJao6cPUR5jxbVPHzLnUwpr2bymU71LXG1O4eWHT4NubtvwkPOwu0cCn6d1cSuVxg6q6rOHE7AYZ6Olg7qhUaOrz+xR1n9vLAsVvxCLr/FPuvPkK/pv/defR8/SsAeLeFY7HvEW0wtJUzOrpbv7Y5XyQSCRYNaozw+FTcjkvF6A2BiE8p6L0c094N9e05RxhRcSp1z05pCyS6ubnBzs4O/v7+iudTUlJw8eJFxWKHpL7tl6Lh7XcCI9ZdwvxDobgdlwoTA12M8nbF6amdsWJ482JPYnK5HOvXr8fIkSOhp/dflvbw8ECdOnVwZ/cSzGljAHtJMu6e2IbggDOIMmuAmKSCsQ5du3bF8uXLFeWmTJmCNWvWYOPGjQgNDcVnn32G9PR0fPTRRwCAu3fvws/PD5cvX0Z0dDTOnz+PwYMHw9jYGL169dLI7+P0nceYsjMEQgAftnV5pUUVP+1UC70a2yE3X2D81stISFXvEpsQAt8dvIV9IY+gpyPB7x80R2u3V58ZuSzspcbw7VwwyZ3f37eVljTYfeUBrj6QwcxQD9N61KuQ9r1Or3vOFxMDPaz6sAWM9HRw81EKEtNyAACO1TjnDFFJKnXYKW2BRIlEgkmTJmHBggWKuVtGjBgBBwcHRU8Aqef5Gk8vXpyQANg3vh3m9W1Y6nIBx48fR3R0NEaPHq20XV9fH3///Tesra0x89MPcG3pOBjfPwfbPlMQYegOn1/OYM2Zu4iMjETUg1jFyuFDhw7FTz/9hDlz5qBp06YICQnB4cOHFeO0jIyM8O+//6JXr16oU6cOhg4dCnNzc5w/fx42NmWfwO7y/SR8uvkycvMF+ng64Nu+DV9prhiJRILF73qiro0Z4lOy4bv1CnLyVJ+vZql/BDacvwcA+GmwJ7p4aHYWYnV93KEWnCyNEZeShd9OFQxETsnKxaLDBQNlv+haBzbm5T+h4pvIQE8H2S/NdfTtgVsqrcNG9Kaq1MtFpKamYvbs2di7dy8SEhLg4OCA9957D3PmzFHcaSOEwNy5c7F69WokJyejffv2+O233+Durvq3by4X8Z/dl2Pw5a5rhbaX1xpPEQlpmLX3Oi5FFQy4dJAaITYlC0IUXDLzG9i4wta+uR2XgiErA5CSlYdO7tZYM6IlDPTK9v3g7uM09Ft+DqnZeRjp5YJv+zUqtczG8/cw99mloW/7NsRIb9cytUFTjt6Mw7jNl2Ggq4NjUzpiy4X7WPNvFGpZmeLwpI5l/l1R0c5HJuL9NRcLbX+d67ARVRZcG0sNDDsF8vLl6LP8HEJjlW/F15VIcHZG53LrqpfLBXZdjsGCQ6FIzVJe5VlHAvzl2w6Nakpf6+y70U8yMGjleTxOzUYLl+rYPKY1TAw0M8TNPzQeYzYGAQB+HuxZ4kDnv0IeYuL2EADApG51Mamb+pfQyosQAiPWXcK/4Ylo5GCBW7EpkAtg/Uet0Lme5peFoAKxsky0W3gC8hc+ucv73yhRZcW1sUhtK05GIjQ2BUZ6OuU6Qd3LdHQkGNrKGT++26TQc3IB9Fl+Di0WHMf7ay7guwO3sCsoBjceyoqctyZWlqm4BPYqYmWZ+PvaIwxbE4DHqdnwsDPHupGtNBZ0AKBrfVtMfDbJ3qy914u9hfvk7QR8ufMqgIJJCidWson5CtbNagAdCXDjUYri5JuQwtnLy1NF3AlGVNVV6rux6PW5GpOMpScK1hRb9G4TtHazfO0rC3s6VYOOBErfWIGCMUNJ6Tk4H/kE519YekFXRwI3K1N42Jmjvr0FHqdmYVPAfchf8RLYjsBozNxzXVG/pakBNo1uDamJ5ifEm9i1Lm48lMH/dgI+2XwZBz5vD0vT/yZBDLyXhE+3XEaeXGBAs5qY807lXFfK1FCv0Lpcs/bcQEd3a558y9HrvhOMqKrjZSzwMlZmTj56L/sXdx+n450m9lj+fvMKa8uOwGjM2nMD+UIovrH2a1oT4fFpCI1LQWhsCm7HpiI0LgXJGbmlHq+asb5Kc4/I5QLJmcrH05EA52Z0KbcTiSwzF/1XnENUYjq8a9fAptGtoaerg1uPUjB0dQBSs/LQ1cMGKz9sAX3dytkJy/EjRFSRVD1/s2eHsPCfUNx9nA5bC0Ms6F/6gNnyVNw31saOUjR2/G9OGSEEElKzC8JPXCr+vfMY54pYcPPlAKOO59P/l1fYkRrrY9WHLdB/xTmcj3yCOftvopVLdXx3sGDsUmtXS6wY3rzSBh3gv5mEXx4/Ul4zCRMRvQr27ODN7tk5fecxRq4rWFh185jWGlux+XUratCmjgTYMqZNkesYvexxajY+WHuxQgZ9/n09FuO3XlHaZi81wpHJHWFRBdaUKqo3rqLuoCOiNwt7dqhUyRk5mLrrvwGwVTXoAP8N2nz5pOtdx0ql8nVtzYss/zrGQjRzrgYJoDS3UXxKFtKz86pE2OH4ESKq7Bh23lBCCHy97wYSUrNRy9oU03t4VHSTyqysJ92KOmlHJabj5e7V8r6Epmn2UuMq01YievMw7Lyh9l99hEPXYqGnI8GvQ5vC2EC3opukEWU96VbESZvjXoiIylflHflI5eZRcia+2VewqOUXXeuiiWO1im3QG47zphARlS/27Lxh5HKBqX9eRWpWHpo6VcP4t2pXdJMIHPdCRFSeGHbeMBvO38O5iCcw1tfFkiGe0KvEtzW/aTjuhYiofPBM9wYJj0/FwsO3AQBf966PWtZmFdwiIiKi8sew84bIyZNj8s4Q5OTJ8VY9awxvw3lQiIjozcCw84ZY6h+OGw9TUM1EH4sHNamU6ywRERGVB4adN8Dl+0n47VQEAMBvQGPYWBhVcIuIiIheH4YdLZeenYcpO69CLoCBzWqiZ2P7im4SERHRa8Wwo+UWHArF/ScZqFnNGPP6Nazo5hAREb12vPVcS8XKMrHn8gNsuxQNiQT4abBnlVhniYiISNMYdrTQjsBozNxzXbH8QPvaVvCqXaNiG0VERFRBeBlLy8TKMpWCDgCci0xErCyz4hpFRERUgRh2tExUYrpS0AH+W0GbiIjoTcSwo2XcrEzx8hQ6XEGbiIjeZAw7WsZeaoz2dawUP3MFbSIietNxgLIWepRcMD5nYte6GNbaiUGHiIjeaAw7Wib6SQYiH6dDV0eC0e3dIDXm7eZERPRm42UsLXMyLAEA0NKlOoMOERERGHa0zonbBWGni4dNBbeEiIiocmDY0SIZOXkIuPsEAMMOERHRcww7WuR8xBPk5MnhWN0YdWzMKro5RERElQLDjhZ5Pl6ni4cNJC9PtkNERPSGYtjREkIInHw2XqdzPV7CIiIieo5hR0uExafikSwLRvo6XPSTiIjoBQw7WuL5XVjeta1gpK9bwa0hIiKqPBh2tITiEhbvwiIiIlLCsKMFkjNycPn+UwBA53rWFdwaIiKiyoVhRwucCU+EXADutmZwrM7VzYmIiF7EsKMFeAmLiIioeAw7VVy+XODU8/l1eMs5ERFRIQw7VVxITDKeZuTCwkgPLVyqV3RziIiIKh2GnSru+SWsju7W0NPln5OIiOhlPDtWcS8uEUFERESFMexUYfEpWbj5KAUSCdDJnbecExERFYVhpwp7fgnL07EaapgZVnBriIiIKieGnSrs+RIRvIRFRERUPIadKio7Lx9nIxIBMOwQERGVhGGniroUlYSMnHzYmBuioYNFRTeHiIio0mLYqaJO3n4MAHirnjUkEkkFt4aIiKjyYtiponjLORERkWoYdqqgqMR0RCWmQ19XgvZ1ecs5ERFRSRh2qqDnd2G1drOEmaFeBbeGiIiocmPYqYIUq5xz4U8iIqJSMexUMenZebgY9QQAx+sQERGpgmGnijkbkYjcfAGXGiZwszKt6OYQERFVegw7VcyLl7B4yzkREVHpGHaqECEEbzknIiJSE8NOFXLzUQriU7JhYqCLNrUsK7o5REREVQLDThVy6lmvTrs6VjDU063g1hAREVUNDDtVyAneck5ERKQ2tcPOyZMny6MdVIqk9BwExyQDADp7cNZkIiIiVakddnr06IHatWtjwYIFiImJKY82URFO30mAEEB9ewvYS40rujlERERVhtph5+HDh5gwYQL+/PNP1KpVC927d8fOnTuRk5NTHu2jZ048W+W8C3t1iIiI1KJ22LGyssLkyZMREhKCixcvwt3dHePHj4eDgwO++OILXL16tTza+UbLy5fjNG85JyIieiVlGqDcvHlzzJw5ExMmTEBaWhrWrVuHFi1aoEOHDrh586am2vjGC45JRkpWHqqZ6KOpU/WKbg4REVGV8kphJzc3F3/++Sd69eoFFxcXHDlyBMuXL0d8fDwiIiLg4uKCwYMHa6SBDx8+xAcffIAaNWrA2NgYjRs3RlBQkOJ5IQTmzJkDe3t7GBsbo1u3bggPD9dI3ZXF87uwOrlbQ1eHsyYTERGpQ+2w8/nnn8Pe3h6ffPIJ3N3dERwcjICAAHz88ccwNTWFq6srfvrpJ9y+fbvMjXv69CnatWsHfX19/PPPP7h16xZ+/vlnVK/+X+/G4sWLsXTpUqxcuRIXL16EqakpunfvjqysrDLXX1k8XyKCl7CIiIjUp6dugVu3bmHZsmUYOHAgDA0Ni9zHyspKI7eoL1q0CE5OTli/fr1im5ubm+L/hRD49ddf8c0336Bfv34AgE2bNsHW1hb79u3DsGHDytyGivYwORO341KhIyno2SEiIiL1qN2z4+/vj/fee6/YoAMAenp66NSpU5kaBgD79+9Hy5YtMXjwYNjY2KBZs2ZYs2aN4vmoqCjExcWhW7duim1SqRRt2rRBQEBAscfNzs5GSkqK0qOyet6r09y5OqqZGFRwa4iIiKoetcOOn58f1q1bV2j7unXrsGjRIo006rm7d+/i999/R926dXHkyBF89tln+OKLL7Bx40YAQFxcHADA1tZWqZytra3iuaL4+flBKpUqHk5OThpttyY9XyKiMy9hERERvRK1w86qVavg4eFRaHvDhg2xcuVKjTTqOblcjubNm+OHH35As2bNMG7cOIwdO7bM9cycORMymUzxqKyTI2bl5uNcxBMAXCKCiIjoVakdduLi4mBvb19ou7W1NWJjYzXSqOfs7e3RoEEDpW3169dHdHQ0AMDOzg4AEB8fr7RPfHy84rmiGBoawsLCQulRGV24+wSZufmwlxqhvr15RTeHiIioSlI77Dg5OeHcuXOFtp87dw4ODg4aadRz7dq1Q1hYmNK2O3fuwMXFBUDBYGU7Ozv4+/srnk9JScHFixfh5eWl0bZUhINXHwEAWrtZQiLhLedERESvQu27scaOHYtJkyYhNzcXXbp0AVAwaHnatGn48ssvNdq4yZMnw9vbGz/88AOGDBmCS5cuYfXq1Vi9ejUAQCKRYNKkSViwYAHq1q0LNzc3zJ49Gw4ODujfv79G2/K6bb8UjT+vPAQA7L/6CN61a2BoK+cKbhUREVHVIxFCCHUKCCEwY8YMLF26VLEelpGREaZPn445c+ZovIEHDx7EzJkzER4eDjc3N0yZMgVjx45Vas/cuXOxevVqJCcno3379vjtt9/g7u6uch0pKSmQSqWQyWSV4pJWrCwT7RaegPyFv4yuRIKzMzpzEVAiIqJnVD1/qx12nktLS0NoaCiMjY1Rt27dEm9Fr+wqW9g5H5mI99dcLLR929i28KpdowJaREREVPmoev5W+zLWc2ZmZmjVqtWrFqcSOFYv3HujK5HA1cqkAlpDRERUtb1S2AkKCsLOnTsRHR2tuJT13J49ezTSsDfZrUepSj/rSiT4YWAjXsIiIiJ6BWrfjbV9+3Z4e3sjNDQUe/fuRW5uLm7evIkTJ05AKpWWRxvfONsuFdxa/2FbF2wb2xZnZ3Tm4GQiIqJXpHbY+eGHH/DLL7/gwIEDMDAwwP/+9z/cvn0bQ4YMgbMzT8hl9eBpBs6EPwYAfNzBDV61a7BHh4iIqAzUDjuRkZHo3bs3AMDAwADp6emQSCSYPHmy4pZwenU7A2MgBNCuTg241DCt6OYQERFVeWqHnerVqyM1tWBMSc2aNXHjxg0AQHJyMjIyMjTbujdMXr4cO4IKlq54rzV7yYiIiDRB7QHKHTt2xLFjx9C4cWMMHjwYEydOxIkTJ3Ds2DF07dq1PNr4xjgV9hjxKdmwNDXA2w1sSy9AREREpVI77CxfvhxZWVkAgK+//hr6+vo4f/48Bg0ahG+++UbjDXyTPB+Y/G4LRxjq6VZwa4iIiLSDWmEnLy8PBw8eRPfu3QEAOjo6mDFjRrk07E0TK8vEybAEAMDQVk4V3BoiIiLtodaYHT09PXz66aeKnh3SnJ2BDyAXQBs3S9S2Nqvo5hAREWkNtQcot27dGiEhIeXQlDdXvlxg57OBye+34cBkIiIiTVJ7zM748eMxZcoUxMTEoEWLFjA1Vb49ukmTJhpr3JviTPhjPEzORDUTfXRvaFfRzSEiItIqaoedYcOGAQC++OILxTaJRAIhBCQSCfLz8zXXujfE9mcDkwc2c4SRPgcmExERaZLaYScqKqo82vHGSkjJwvHQgoHJ77XmwGQiIiJNUzvsuLi4lEc73li7Lj9AvlygpUt11LU1r+jmEBERaR21w86mTZtKfH7EiBGv3Jg3jVwusD2w4BLWMM6YTEREVC7UDjsTJ05U+jk3NxcZGRkwMDCAiYkJw44azkUmIiYpE+ZGeujd2L6im0NERKSV1L71/OnTp0qPtLQ0hIWFoX379ti2bVt5tFFrbb9UcLv5gGY1YWzAgclERETlQe2wU5S6deti4cKFhXp9qHiJadk4eisOADCsFS9hERERlReNhB2gYHblR48eaepwWm/35QfIzRfwdKqGBg4WFd0cIiIiraX2mJ39+/cr/SyEQGxsLJYvX4527dpprGHaTAiB7YHPZkzm7eZERETlSu2w079/f6WfJRIJrK2t0aVLF/z888+aapdWu3A3CVGJ6TAz1MM7TRwqujlERERaTe2wI5fLy6Mdb5Rtz2ZM7tvUAaaGav8JiIiISA0aG7NDqnmanoPDNwoGJr/PuXWIiIjKndphZ9CgQVi0aFGh7YsXL8bgwYM10ihttvvKA+Tky9GopgUa1ZRWdHOIiIi0ntph58yZM+jVq1eh7T179sSZM2c00iht9eLA5PfYq0NERPRaqB120tLSYGBgUGi7vr4+UlJSNNKoymbhwoWQSCSYNGkSAODevXuQSCRFPnbt2lXscXR0dOD/5Vu4v+gdfNDWVVHmxx9/VOzTt29fODs7w8jICPb29vjwww95Sz8REVEZqB12GjdujB07dhTavn37djRo0EAjjapMAgMDsWrVKjRp0kSxzcnJCbGxsUqPb7/9FmZmZujZs2exxxq38hgcfTdj/Gp/xMbGYt26dZBIJBg0aJBin86dO2Pnzp0ICwvD7t27ERkZiXfffbdcXyMREZE2U/tWoNmzZ2PgwIGIjIxEly5dAAD+/v7Ytm1bib0aVVFaWhqGDx+ONWvWYMGCBYrturq6sLOzU9p37969GDJkCMzMzIo8liwjF6dicqFrVh0fd28GO7vq+Ouvv9C5c2fUqlVLsd/kyZMV/+/i4oIZM2agf//+yM3Nhb6+voZfIRERkfZTu2enT58+2LdvHyIiIjB+/Hh8+eWXePDgAY4fP15oDp6qztfXF71790a3bt1K3O/y5csICQnBmDFjit1nb/ADZOfJ4WFnjqZO1RAfH49Dhw6VWCYpKQlbt26Ft7c3gw4REdEreqVJXnr37o3evXtrui2Vyvbt23HlyhUEBgaWuu/atWtRv359eHt7F/n8ywOTJRIJNm7cCHNzcwwcOLDQ/tOnT8fy5cuRkZGBtm3b4uDBg2V7MURERG8wtXt2AgMDcfHixULbL168iKCgII00qqLFxMRg4sSJ2Lp1K4yMjErcNzMzE3/88UeJPTTBMcm4HZcKQz0d9G9WEwCwbt06DB8+vMjjT506FcHBwTh69Ch0dXUxYsQICCHK9qKIiIjeUGqHHV9fX8TExBTa/vDhQ/j6+mqkURXt8uXLSEhIQPPmzaGnpwc9PT2cPn0aS5cuhZ6eHvLz8xX7/vnnn8jIyMCIESOKPd72ZzMmv9PEAVJjffz7778ICwvDxx9/XOT+VlZWcHd3x9tvv43t27fj77//xoULFzT7IomIiN4Qal/GunXrFpo3b15oe7NmzXDr1i2NNKqide3aFdevX1fa9tFHH8HDwwPTp0+Hrq6uYvvatWvRt29fWFtbF3ms1KxcHLgaCwB479min2vXrkWLFi3g6elZalueL8+RnZ39Sq+FiIjoTad22DE0NER8fLzSHUQAEBsbCz097VjnydzcHI0aNVLaZmpqiho1aihtj4iIwJkzZ/D3338XeRwPDw/0GDUZmbmOqGtjhhYu1ZGSkoJdu3YVuWjqxYsXERgYiPbt26N69eqIjIzE7NmzUbt2bXh5eWn2RRIREb0h1L6M5ePjg5kzZ0Imkym2JScnY9asWXj77bc12rjKbt26dXB0dISPj0+Rz4eFheFw8F0AwLBnA5O3b98OIQTee++9QvubmJhgz5496Nq1K+rVq4cxY8agSZMmOH36NAwNDcv1tRAREWkriVBz5OvDhw/RsWNHPHnyBM2aNQMAhISEwNbWFseOHYOTk1O5NLQ8paSkQCqVQiaTwcLCQiPH3BEYjRl7ruP5b3fOOw0wur2bRo5NREREqp+/1Q47AJCeno6tW7fi6tWrMDY2RpMmTfDee+9V2blgNB12YmWZaLfwBOQv/GZ1JRKcndEZ9lLjMh+fiIiIVD9/v9IgG1NTU4wbN+6VG6ftohLTlYIOAOQLgXuJGQw7REREr9krjyi+desWoqOjkZOTo7S9b9++ZW5UVedmZQodCQr17LhamVRco4iIiN5Qaoedu3fvYsCAAbh+/TokEolisjuJRAIASnPQvKnspcbwG9gYs/bcQL4Q0JVI8MPARuzVISIiqgBqh52JEyfCzc0N/v7+cHNzw6VLl/DkyRN8+eWX+Omnn8qjjVXS0FbO6OhujXuJGXC1MmHQISIiqiBqh52AgACcOHECVlZW0NHRgY6ODtq3bw8/Pz988cUXCA4OLo92Vkn2UmOGHCIiogqm9jw7+fn5MDc3B1CwrMGjR48AAC4uLggLC9Ns64iIiIjKSO2enUaNGuHq1atwc3NDmzZtsHjxYhgYGGD16tWFZlUmIiIiqmhqh51vvvkG6enpAIDvvvsO77zzDjp06IAaNWpgx44dGm8gERERUVm80qSCL0tKSkL16tUVd2RVNeUxgzIRERGVr3KdVPBllpaWmjgMERERkcapPUCZiIiIqCph2CEiIiKtxrBDREREWk3tsHPmzBnk5eUV2p6Xl4czZ85opFFEREREmqJ22OncuTOSkpIKbZfJZOjcubNGGkVERESkKWqHHSFEkbeYP3nyBKamphppFBEREZGmqHzr+cCBAwEUrG4+atQoGBoaKp7Lz8/HtWvX4O3trfkWEhEREZWBymFHKpUCKOjZMTc3h7HxfwtcGhgYoG3bthg7dqzmW0hERERUBiqHnfXr1wMAXF1d8dVXX/GSFREREVUJao/ZmTZtmtKYnfv37+PXX3/F0aNHNdowIiIiIk1QO+z069cPmzZtAgAkJyejdevW+Pnnn9GvXz/8/vvvGm8gERERUVmoHXauXLmCDh06AAD+/PNP2NnZ4f79+9i0aROWLl2q8QYSERERlYXaYScjIwPm5uYAgKNHj2LgwIHQ0dFB27Ztcf/+fY03kIiIiKgs1A47derUwb59+xATE4MjR47Ax8cHAJCQkFDi8upEREREFUHtsDNnzhx89dVXcHV1RevWreHl5QWgoJenWbNmGm8gERERUVmoHXbeffddREdHIygoCEeOHFFs79q1K3755ReNNu5lCxcuhEQiwaRJkxTbsrKy4Ovrixo1asDMzAyDBg1CfHx8ubaDiIiIqo5XWvXczs4O5ubmOHbsGDIzMwEArVq1goeHh0Yb96LAwECsWrUKTZo0Udo+efJkHDhwALt27cLp06fx6NEjxWzPRERERGqHnSdPnqBr165wd3dHr169EBsbCwAYM2YMvvzyS403EADS0tIwfPhwrFmzBtWrV1dsl8lkWLt2LZYsWYIuXbqgRYsWWL9+Pc6fP48LFy6US1uIiIioalE77EyePBn6+vqIjo6GiYmJYvvQoUNx+PBhjTbuOV9fX/Tu3RvdunVT2n758mXk5uYqbffw8ICzszMCAgKKPV52djZSUlKUHkRERKSdVF4u4rmjR4/iyJEjcHR0VNpet27dcrn1fPv27bhy5QoCAwMLPRcXFwcDAwNUq1ZNabutrS3i4uKKPaafnx++/fZbTTeViIiIKiG1e3bS09OVenSeS0pKUloJXRNiYmIwceJEbN26FUZGRho77syZMyGTyRSPmJgYjR2biIiIKhe1w06HDh0Uy0UAgEQigVwux+LFi9G5c2eNNu7y5ctISEhA8+bNoaenBz09PZw+fRpLly6Fnp4ebG1tkZOTg+TkZKVy8fHxsLOzK/a4hoaGsLCwUHoQERGRdlL7MtbixYvRtWtXBAUFIScnB9OmTcPNmzeRlJSEc+fOabRxXbt2xfXr15W2ffTRR/Dw8MD06dPh5OQEfX19+Pv7Y9CgQQCAsLAwREdHK+b/ISIiojeb2mGnUaNGuHPnDpYvXw5zc3OkpaVh4MCB8PX1hb29vUYbZ25ujkaNGiltMzU1RY0aNRTbx4wZgylTpsDS0hIWFhb4/PPP4eXlhbZt22q0LURERFQ1qR12oqOj4eTkhK+//rrI55ydnTXSMFX98ssv0NHRwaBBg5CdnY3u3bvjt99+e61tICIiospLIoQQ6hTQ1dVFbGwsbGxslLY/efIENjY2yM/P12gDX4eUlBRIpVLIZDKO3yEiIqoiVD1/qz1AWQgBiURSaHtaWppG75giIiIi0gSVL2NNmTIFQMHdV7Nnz1a6/Tw/Px8XL15E06ZNNd5AIiIiorJQOewEBwcDKOjZuX79OgwMDBTPGRgYwNPTE1999ZXmW0hERERUBiqHnZMnTwIouPX7f//7H8e2EBERUZWg9t1Y69evL492EBEREZULtQcoExEREVUlDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Rh2iIiISKsx7BAREZFWY9ghIiIircawQ0RERFqNYYeIiIi0GsMOERERaTWGHSIiItJqDDtERESk1Sp12PHz80OrVq1gbm4OGxsb9O/fH2FhYUr7ZGVlwdfXFzVq1ICZmRkGDRqE+Pj4CmoxERERVTaVOuycPn0avr6+uHDhAo4dO4bc3Fz4+PggPT1dsc/kyZNx4MAB7Nq1C6dPn8ajR48wcODACmw1ERERVSYSIYSo6Eao6vHjx7CxscHp06fRsWNHyGQyWFtb448//sC7774LALh9+zbq16+PgIAAtG3bVqXjpqSkQCqVQiaTwcLCojxfAhEREWmIqufvSt2z8zKZTAYAsLS0BABcvnwZubm56Natm2IfDw8PODs7IyAgoNjjZGdnIyUlRelBRERE2qnKhB25XI5JkyahXbt2aNSoEQAgLi4OBgYGqFatmtK+tra2iIuLK/ZYfn5+kEqlioeTk1N5Np2IiIgqUJUJO76+vrhx4wa2b99e5mPNnDkTMplM8YiJidFAC4mIiKgy0qvoBqhiwoQJOHjwIM6cOQNHR0fFdjs7O+Tk5CA5OVmpdyc+Ph52dnbFHs/Q0BCGhobl2WQiIiKqJCp1z44QAhMmTMDevXtx4sQJuLm5KT3fokUL6Ovrw9/fX7EtLCwM0dHR8PLyet3NJSIiokqoUvfs+Pr64o8//sBff/0Fc3NzxTgcqVQKY2NjSKVSjBkzBlOmTIGlpSUsLCzw+eefw8vLS+U7sYiIiEi7VepbzyUSSZHb169fj1GjRgEomFTwyy+/xLZt25CdnY3u3bvjt99+K/Ey1st46zkREVHVo+r5u1KHndeFYYeIiKjq0cp5doiIiIjUxbBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIq2lN2FmxYgVcXV1hZGSENm3a4NKlSxXdJCIiIqoEtCLs7NixA1OmTMHcuXNx5coVeHp6onv37khISKjophEREVEF04qws2TJEowdOxYfffQRGjRogJUrV8LExATr1q2r6KYRERFRBdOr6AaUVU5ODi5fvoyZM2cqtuno6KBbt24ICAgoskx2djays7MVP8tkMgBASkpK+TaWiIiINOb5eVsIUeJ+VT7sJCYmIj8/H7a2tkrbbW1tcfv27SLL+Pn54dtvvy203cnJqVzaSEREROUnNTUVUqm02OerfNh5FTNnzsSUKVMUP8vlciQlJaFGjRqQSCQaqyclJQVOTk6IiYmBhYXFay3Pul9/3WUtz7rfrLrLWp51s+6qUr6sdZdECIHU1FQ4ODiUuF+VDztWVlbQ1dVFfHy80vb4+HjY2dkVWcbQ0BCGhoZK26pVq1ZeTYSFhUWZ/sBlKc+6X3/dZS3Put+sustannWz7qpSvqx1F6ekHp3nqvwAZQMDA7Ro0QL+/v6KbXK5HP7+/vDy8qrAlhEREVFlUOV7dgBgypQpGDlyJFq2bInWrVvj119/RXp6Oj766KOKbhoRERFVMK0IO0OHDsXjx48xZ84cxMXFoWnTpjh8+HChQcuvm6GhIebOnVvoktnrKM+6X3/dZS3Put+sustannWz7qpSvqx1a4JElHa/FhEREVEVVuXH7BARERGVhGGHiIiItBrDDhEREWk1hh0iIiLSagw75WjFihVwdXWFkZER2rRpg0uXLqlU7syZM+jTpw8cHBwgkUiwb98+lev08/NDq1atYG5uDhsbG/Tv3x9hYWEql//999/RpEkTxeRPXl5e+Oeff1Qu/6KFCxdCIpFg0qRJKu0/b948SCQSpYeHh4fK9T18+BAffPABatSoAWNjYzRu3BhBQUEqlXV1dS1Ut0Qiga+vb6ll8/PzMXv2bLi5ucHY2Bi1a9fG/PnzS12r5UWpqamYNGkSXFxcYGxsDG9vbwQGBhbar7T3hhACc+bMgb29PYyNjdGtWzeEh4erXH7Pnj3w8fFRzCYeEhKicv25ubmYPn06GjduDFNTUzg4OGDEiBF49OiRSnXPmzcPHh4eMDU1RfXq1dGtWzdcvHhR5ba/6NNPP4VEIsGvv/6qUtlRo0YV+tv36NFDrbpDQ0PRt29fSKVSmJqaolWrVoiOji61bFHvO4lEgh9//FGlutPS0jBhwgQ4OjrC2NhYsRiyKmXj4+MxatQoODg4wMTEBD169FC8X1T5LMnKyoKvry9q1KgBMzMzDBo0SDHBqyrlV69ejbfeegsWFhaQSCRITk5WPFda+aSkJHz++eeoV68ejI2N4ezsjC+++AIymUyluj/55BPUrl0bxsbGsLa2Rr9+/RRLDKnzOSqEQM+ePRW/X1XKvvXWW4X+3p9++qladQcEBKBLly4wNTWFhYUFOnbsiO+++67Esvfu3Sv2/bZr1y6V6o6Li8OHH34IOzs7mJqaonnz5ti9e7dKZSMjIzFgwABYW1vDwsICQ4YMKTQhcHlh2CknO3bswJQpUzB37lxcuXIFnp6e6N69OxISEkotm56eDk9PT6xYsULtek+fPg1fX19cuHABx44dQ25uLnx8fJCenq5SeUdHRyxcuBCXL19GUFAQunTpgn79+uHmzZtqtSMwMBCrVq1CkyZN1CrXsGFDxMbGKh5nz55VqdzTp0/Rrl076Ovr459//sGtW7fw888/o3r16iq398V6jx07BgAYPHhwqWUXLVqE33//HcuXL0doaCgWLVqExYsXY9myZSrVDQAff/wxjh07hs2bN+P69evw8fFBt27d8PDhQ6X9SntvLF68GEuXLsXKlStx8eJFmJqaonv37sjKylKpfHp6Otq3b49FixYV+3xx5TMyMnDlyhXMnj0bV65cwZ49exAWFoa+ffuqVLe7uzuWL1+O69ev4+zZs3B1dYWPjw8eP36sUvnn9u7diwsXLihNH69K2R49eii9B7Zt26Zy+cjISLRv3x4eHh44deoUrl27htmzZ8PIyKjUsi/WGRsbi3Xr1kEikWDQoEEq1T1lyhQcPnwYW7ZsQWhoKCZNmoQJEyZg//79JZYVQqB///64e/cu/vrrLwQHB8PFxQXdunVDenq6Sp8lkydPxoEDB7Br1y6cPn0ajx49wsCBAwGo9lmUkZGBHj16YNasWYXaV1r5R48e4dGjR/jpp59w48YNbNiwAYcPH8aYMWNUqrtFixZYv349QkNDceTIEQgh4OPjg/z8fLU+R3/99VelZYZULTt27Filv/vixYtVLh8QEIAePXrAx8cHly5dQmBgICZMmICzZ8+WWNbJyanQ++3bb7+FmZkZevbsqVLdI0aMQFhYGPbv34/r169j4MCBGDJkCA4cOFBi2fT0dPj4+EAikeDEiRM4d+4ccnJy0KdPH8jl8kK/V40TVC5at24tfH19FT/n5+cLBwcH4efnp9ZxAIi9e/e+cjsSEhIEAHH69OlXPkb16tXF//3f/6m8f2pqqqhbt644duyY6NSpk5g4caJK5ebOnSs8PT1fqY3Tp08X7du3f6WyRZk4caKoXbu2kMvlpe7bu3dvMXr0aKVtAwcOFMOHD1eproyMDKGrqysOHjyotL158+bi66+/Lrbcy+8NuVwu7OzsxI8//qjYlpycLAwNDcW2bdtKLf+iqKgoAUAEBwerXH9RLl26JACI+/fvq11WJpMJAOL48eMq1/3gwQNRs2ZNcePGDeHi4iJ++eUXlcqOHDlS9OvXr8T2lFR+6NCh4oMPPnilsi/r16+f6NKli8rlGzZsKL777julbUW9d14uGxYWJgCIGzduKLbl5+cLa2trsWbNmkJ1v/xZkpycLPT19cWuXbsU+4SGhgoAIiAgoNTyLzp58qQAIJ4+fVrk6y6t/HM7d+4UBgYGIjc3V+2yV69eFQBERESEynUHBweLmjVritjY2GL/tkWVVedzsajybdq0Ed98880rlX1Z06ZNC31+lVTe1NRUbNq0SWk/S0vLQu+Zl8seOXJE6OjoCJlMptgnOTlZSCQScezYsVJfS1mxZ6cc5OTk4PLly+jWrZtim46ODrp164aAgIDX2haZTAYAsLS0VLtsfn4+tm/fjvT0dLWW3vD19UXv3r2VXr+qwsPD4eDggFq1amH48OGIjo5Wqdz+/fvRsmVLDB48GDY2NmjWrBnWrFmjdv1Awd9vy5YtGD16tEoLw3p7e8Pf3x937twBAFy9ehVnz55Fz549VaovLy8P+fn5MDIyUtpubGyscs8WAERFRSEuLk7p9y6VStGmTZvX/r57TiaTQSKRqL32XE5ODlavXg2pVApPT0+Vysjlcnz44YeYOnUqGjZsqHZbT506BRsbG9SrVw+fffYZnjx5onK9hw4dgru7O7p37w4bGxu0adNGrcvPz8XHx+PQoUMYM2aMymW8vb2xf/9+PHz4EEIInDx5Enfu3IGPj0+J5bKzswFA6X2no6MDQ0PDIt93L3+WXL58Gbm5uUrvNw8PDzg7Oxf5fivLZ5Gq5WUyGSwsLKCnp1doe0ll09PTsX79eri5ucHJyUmlujMyMvD+++9jxYoVxa7DWFLdW7duhZWVFRo1aoSZM2ciIyNDpfIJCQm4ePEibGxs4O3tDVtbW3Tq1Emlv9nLLl++jJCQkGLfb0WV9/b2xo4dO5CUlAS5XI7t27cjKysLb731Volls7OzIZFIlCYWNDIygo6Ojlqfc6+s3OPUG+jhw4cCgDh//rzS9qlTp4rWrVurdSyUoWcnPz9f9O7dW7Rr106tcteuXROmpqZCV1dXSKVScejQIZXLbtu2TTRq1EhkZmYKIdT7BvP333+LnTt3iqtXr4rDhw8LLy8v4ezsLFJSUkota2hoKAwNDcXMmTPFlStXxKpVq4SRkZHYsGGDym1/bseOHUJXV1c8fPhQpf3z8/PF9OnThUQiEXp6ekIikYgffvhBrTq9vLxEp06dxMOHD0VeXp7YvHmz0NHREe7u7sWWefm9ce7cOQFAPHr0SGm/wYMHiyFDhpRa/kWa6NnJzMwUzZs3F++//77KZQ8cOCBMTU2FRCIRDg4O4tKlSyrX/cMPP4i3335b0RunTs/Otm3bxF9//SWuXbsm9u7dK+rXry9atWol8vLySi3//Fu9iYmJWLJkiQgODhZ+fn5CIpGIU6dOqfS6n1u0aJGoXr264t+PKm3PysoSI0aMEACEnp6eMDAwEBs3biy1bE5OjnB2dhaDBw8WSUlJIjs7WyxcuFAAED4+Pkpli/os2bp1qzAwMChUT6tWrcS0adNKLf+i0np2VPkse/z4sXB2dhazZs1SueyKFSuEqampACDq1atXZK9OceXHjRsnxowZo/i5qL9NcWVXrVolDh8+LK5duya2bNkiatasKQYMGKBS3QEBAQKAsLS0FOvWrRNXrlwRkyZNEgYGBuLOnTsqve7nPvvsM1G/fv0inyuu/NOnT4WPj4/i/WZhYSGOHDlSatmEhARhYWEhJk6cKNLT00VaWpqYMGGCACDGjRtXbBs1hWGnHFSWsPPpp58KFxcXERMTo1a57OxsER4eLoKCgsSMGTOElZWVuHnzZqnloqOjhY2Njbh69apimzph52VPnz4VFhYWKl1C09fXF15eXkrbPv/8c9G2bVu16/Xx8RHvvPOOyvtv27ZNODo6im3btolr166JTZs2CUtLS7WCVkREhOjYsaMAIHR1dUWrVq3E8OHDhYeHR7FlKnPYycnJEX369BHNmjVT6rYurWxaWpoIDw8XAQEBYvTo0cLV1VXEx8eXWj4oKEjY2toqBVR1ws7LIiMjVb6E9vzf+3vvvae0X58+fcSwYcPUqrtevXpiwoQJxT5fVPkff/xRuLu7i/3794urV6+KZcuWCTMzs0KXBooqGxQUJDw9PRXvu+7du4uePXuKHj16KO1X1GeJOmGntM+i0sJOaeVlMplo3bq16NGjh8jJyVG5bHJysrhz5444ffq06NOnj2jevHmhoFlU+b/++kvUqVNHpKamKrYV9ftV9TPY39+/yEtoRZV//u985syZSvs2btxYzJgxQ+W6MzIyhFQqFT/99FORzxdXfsKECaJ169bi+PHjIiQkRMybN09IpVJx7dq1UsseOXJE1KpVS0gkEqGrqys++OAD0bx5c/Hpp5+W8NvRDIadcpCdnS10dXULvfFHjBgh+vbtq9axXjXs+Pr6CkdHR3H37l21y76sa9euKiXvvXv3Kj40nz8AKN7YRX1LLk3Lli2V/gEXx9nZWelblhBC/Pbbb8LBwUGt+u7duyd0dHTEvn37VC7j6Ogoli9frrRt/vz5ol69emrVLUTByf55WBkyZIjo1atXsfu+/N54foJ+OaB07NhRfPHFF6WWf1FZwk5OTo7o37+/aNKkiUhMTFSr7Mvq1KlTZC/Zy+V/+eUXxfvsxfeejo6OcHFxeaW6raysxMqVK0utOzs7W+jp6Yn58+cr7Tdt2jTh7e2tct1nzpwRAERISEixbXq5fEZGhtDX1y803mvMmDGie/fuKtednJwsEhIShBAF4w3Hjx+veK64z5LnJ+iXA4qzs7NYsmRJqeVfVFLYKa18SkqK8PLyEl27di0UVNT5HMzOzhYmJibijz/+KLX8xIkTi32/derUSe2609LSBABx+PDhUuu+e/euACA2b96stH3IkCGKXlRV6t60aZPQ19dX/N1fVFz5iIiIQuO8hCg4R3zyyScq1/348WPF39rW1lYsXry42H01hWN2yoGBgQFatGgBf39/xTa5XA5/f3+1xr68CiEEJkyYgL179+LEiRNwc3Mr8zHlcrni+n5JunbtiuvXryMkJETxaNmyJYYPH46QkBDo6uqqVW9aWhoiIyNhb29f6r7t2rUrdJvjnTt34OLiolad69evh42NDXr37q1ymYyMDOjoKP9T0tXVfaU7DExNTWFvb4+nT5/iyJEj6Nevn8pl3dzcYGdnp/S+S0lJwcWLF8v9ffdcbm4uhgwZgvDwcBw/fhw1atQo0/FUfe99+OGHuHbtmtJ7z8HBAVOnTsWRI0fUrvfBgwd48uSJSu89AwMDtGrVqszvv7Vr16JFixYqj1ECCn7fubm5ZX7/SaVSWFtbIzw8HEFBQejXr1+pnyUtWrSAvr6+0vstLCwM0dHR8PLyKvNnkSrlU1JS4OPjAwMDA+zfv18x/uhV6hYFX/6RnZ1davkZM2YUer8BwC+//IJ169apXffz8vb29qXW7erqCgcHhyLfb87OzirXvXbtWvTt2xfW1tZKv4OSyj8fV1TU+y0/P1/luq2srFCtWjWcOHECCQkJijs2y1W5x6k31Pbt24WhoaHYsGGDuHXrlhg3bpyoVq2aiIuLK7VsamqqCA4OFsHBwQKAYhzAy3e0FOWzzz4TUqlUnDp1SsTGxioeGRkZKrV7xowZ4vTp0yIqKkpcu3ZNzJgxQ0gkEnH06FGVyr9MnctYX375pTh16pSIiooS586dE926dRNWVlZFfvN42aVLl4Senp74/vvvRXh4uNi6daswMTERW7ZsUbmt+fn5wtnZWUyfPl3lMkIU3MlTs2ZNcfDgQREVFSX27NkjrKysCnXll+Tw4cPin3/+EXfv3hVHjx4Vnp6eok2bNoW65Et7byxcuFBUq1ZNMf6kX79+ws3NTfGNt7TyT548EcHBweLQoUMCgNi+fbsIDg4WsbGxpZbPyckRffv2FY6OjiIkJETp/ZednV1i2bS0NDFz5kwREBAg7t27J4KCgsRHH30kDA0NFd8i1f138eJlrJLKpqamiq+++koEBASIqKgocfz4cdG8eXNRt25dkZWVpVLde/bsEfr6+mL16tUiPDxcLFu2TOjq6op///1XpXbLZDJhYmIifv/990Kvo7TynTp1Eg0bNhQnT54Ud+/eFevXrxdGRkbit99+K7Xszp07xcmTJ0VkZKTYt2+fcHFxEQMHDhRCqPZZ8umnnwpnZ2dx4sQJERQUJLy8vBSXk1UpHxsbK4KDg8WaNWsEAHHmzBkRHBwsnjx5Ump5mUwm2rRpIxo3biwiIiKU9vn0009LLBsZGSl++OEHERQUJO7fvy/OnTsn+vTpIywtLUV8fPwrfY7iWc9ZaWUjIiLEd999J4KCgkRUVJT466+/RK1atUTHjh1V/r398ssvwsLCQuzatUuEh4eLb775RhgZGYn3339fpXaHh4cLiUQi/vnnH6XtpdWdk5Mj6tSpIzp06CAuXrwoIiIixE8//SQkEono1atXqXWvW7dOBAQEiIiICLF582ZhaWkppkyZUuzvVJMYdsrRsmXLhLOzszAwMBCtW7cWFy5cUKnc8y7dlx8jR44stWxR5QCI9evXq1T36NGjhYuLizAwMBDW1taia9eurxx0hFAv7AwdOlTY29sLAwMDUbNmTTF06NAiBwwW58CBA6JRo0bC0NBQeHh4iNWrV6vV1iNHjggAIiwsTK1yKSkpYuLEicLZ2VkYGRmJWrVqia+//lpkZ2erfIwdO3aIWrVqCQMDA2FnZyd8fX1FcnJyof1Ke2/I5XIxe/ZsYWtrKwwNDUXXrl2VXk9p5devX1/k83Pnzi21/PNLX0U9Tp48WWLZzMxMMWDAAOHg4CAMDAyEvb296Nu3r9IAZXX/XbwYdkoqm5GRIXx8fIS1tbXQ19cXLi4uYuzYsUpfTFSpe+3ataJOnTrCyMhIeHp6Ki6FqlJ21apVwtjY+JX+5rGxsWLUqFHCwcFBGBkZiXr16omff/5ZyOXyUsv+73//E46OjkJfX184OzuLb775RvG+VeWzJDMzU4wfP15Ur15dmJiYiAEDBiiCsSrl586dW+w+pZUv7rWV9Hhe9uHDh6Jnz57CxsZG6OvrC0dHR/H++++L27dvq9z2lz0PO6WVjY6OFh07dhSWlpbC0NBQ1KlTR0ydOlUxtk3Vuv38/ISjo6MwMTERXl5e4t9//1W57MyZM4WTk5PIz88v9BpKK3/nzh0xcOBAYWNjI0xMTESTJk3Epk2bVCo7ffp0YWtrK/T19UXdunUV79PXQfLsBRIRERFpJY7ZISIiIq3GsENERERajWGHiIiItBrDDhEREWk1hh0iIiLSagw7REREpNUYdoiIiEirMewQEb3k1KlTkEgkSE5OruimEJEGMOwQERGRVmPYISIiIq3GsENElY5cLoefnx/c3NxgbGwMT09P/PnnnwD+u8R06NAhNGnSBEZGRmjbti1u3LihdIzdu3ejYcOGMDQ0hKurK37++Wel57OzszF9+nQ4OTnB0NAQderUwdq1a5X2uXz5Mlq2bAkTExN4e3sXWmmaiKoGhh0iqnT8/PywadMmrFy5Ejdv3sTkyZPxwQcf4PTp04p9pk6dip9//hmBgYGwtrZGnz59kJubC6AgpAwZMgTDhg3D9evXMW/ePMyePRsbNmxQlB8xYgS2bduGpUuXIjQ0FKtWrYKZmZlSO77++mv8/PPPCAoKgp6eHkaPHv1aXj8RaRYXAiWiSiU7OxuWlpY4fvw4vLy8FNs//vhjZGRkYNy4cejcuTO2b9+OoUOHAgCSkpLg6OiIDRs2YMiQIRg+fDgeP36Mo0ePKspPmzYNhw4dws2bN3Hnzh3Uq1cPx44dQ7du3Qq14dSpU+jcuTOOHz+Orl27AgD+/vtv9O7dG5mZmTAyMirn3wIRaRJ7doioUomIiEBGRgbefvttmJmZKR6bNm1CZGSkYr8Xg5ClpSXq1auH0NBQAEBoaCjatWundNx27dohPDwc+fn5CAkJga6uLjp16lRiW5o0aaL4f3t7ewBAQkJCmV8jEb1eehXdACKiF6WlpQEADh06hJo1ayo9Z2hoqBR4XpWxsbFK++nr6yv+XyKRACgYT0REVQt7doioUmnQoAEMDQ0RHR2NOnXqKD2cnJwU+124cEHx/0+fPsWdO3dQv359AED9+vVx7tw5peOeO3cO7u7u0NXVRePGjSGXy5XGABGR9mLPDhFVKubm5vjqq68wefJkyOVytG/fHjKZDOfOnYOFhQVcXFwAAN999x1q1KgBW1tbfP3117CyskL//v0BAF9++SVatWqF+fPnY+jQoQgICMDy5cvx22+/AQBcXV0xcuRIjB49GkuXLoWnpyfu37+PhIQEDBkypKJeOhGVE4YdIqp05s+fD2tra/j5+eHu3buoVq0amjdvjlmzZikuIy1cuBATJ05EeHg4mjZtigMHDsDAwAAA0Lx5c+zcuRNz5szB/PnzYW9vj++++w6jRo1S1PH7779j1qxZGD9+PJ48eQJnZ2fMmjWrIl4uEZUz3o1FRFXK8zulnj59imrVqlV0c4ioCuCYHSIiItJqDDtERESk1XgZi4iIiLQae3aIiIhIqzHsEBERkVZj2CEiIiKtxrBDREREWo1hh4iIiLQaww4RERFpNYYdIiIi0moMO0RERKTVGHaIiIhIq/0/aNuF5S+4oysAAAAASUVORK5CYII=", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "plt.plot(np.arange(len(epochs_x)), epochs_acc, marker = '.')\n", - "plt.xlabel('epoch')\n", - "plt.ylabel('test accuracy')\n", - "plt.ylim(0, 100)\n", - "plt.xticks(np.arange(len(epochs_x)))\n", - "for i, txt in enumerate(epochs_acc):\n", - " if i%5 == 0:\n", - " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", - " else:\n", - " pass\n", - "plt.show()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "speck-rescnn", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/tests/test_nonsequential/using_SumPool2d/TOP_2_ARCHITECTURES/single_training.ipynb b/tests/test_nonsequential/using_SumPool2d/TOP_2_ARCHITECTURES/single_training.ipynb deleted file mode 100644 index 30637cad..00000000 --- a/tests/test_nonsequential/using_SumPool2d/TOP_2_ARCHITECTURES/single_training.ipynb +++ /dev/null @@ -1,3837 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import torch, random, sys\n", - "\n", - "import tonic\n", - "from torch.utils.data import DataLoader\n", - "from torch.nn import CrossEntropyLoss\n", - "from torch.optim import Adam\n", - "\n", - "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential\n", - "\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "\n", - "sys.path.append('../../utils')\n", - "sys.path.append('../models')\n", - "\n", - "from train_test_fn import training_loop, load_dataset, load_architecture\n", - "from weight_initialization import rescale_method_1" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "metadata": {} - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "device: NVIDIA GeForce RTX 3070 Ti\n" - ] - } - ], - "source": [ - "if torch.cuda.is_available():\n", - " device = torch.device('cuda:0')\n", - " print('device: ', torch.cuda.get_device_name(0))\n", - "else:\n", - " device = torch.device('cpu')" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "rand_seed = 1" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "achitecture = 'ResSCNN_5'" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "torch.backends.cudnn.enabled = False\n", - "torch.backends.cudnn.deterministic = True\n", - "random.seed(rand_seed)\n", - "torch.manual_seed(rand_seed)\n", - "torch.cuda.manual_seed(rand_seed)\n", - "np.random.seed(rand_seed)" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "batch_size = 8\n", - "num_workers = 4\n", - "epochs = 50\n", - "n_time_steps = 50\n", - "\n", - "lr = 5e-5\n", - "spk_thr = 2.0\n", - "v_min = -0.5\n", - "grad_scale = 1.75\n", - "grad_width = 0.5\n", - "w_rescale_lambda = 0.6" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Loading Data" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "snn_train_dataset, snn_test_dataset, sensor_size = load_dataset('DVSGESTURE', n_time_steps)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "getting validation dataset...." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "disk caching samples..." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "disk_cache_train = tonic.DiskCachedDataset(\n", - " dataset=snn_train_dataset,\n", - " cache_path='./cached_train'\n", - ")\n", - "snn_train_dataloader = DataLoader(disk_cache_train, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=True)\n", - "\n", - "disk_cache_test = tonic.DiskCachedDataset(\n", - " dataset=snn_test_dataset,\n", - " cache_path='./cached_test'\n", - ")\n", - "snn_test_dataloader = DataLoader(disk_cache_test, batch_size=batch_size, num_workers=num_workers, drop_last=True, shuffle=False)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Network Module" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": { - "metadata": {} - }, - "outputs": [ - { - "ename": "ModuleNotFoundError", - "evalue": "No module named 'sinabs.exodus'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[9], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m snn \u001b[38;5;241m=\u001b[39m \u001b[43mload_architecture\u001b[49m\u001b[43m(\u001b[49m\u001b[43machitecture\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43msensor_size\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m11\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbatch_size\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mPeriodicExponential\u001b[49m\u001b[43m(\u001b[49m\u001b[43mgrad_scale\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgrad_scale\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mgrad_width\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgrad_width\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mv_min\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mspk_thr\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241m.\u001b[39mto(device)\n\u001b[1;32m 2\u001b[0m snn\u001b[38;5;241m.\u001b[39minit_weights()\n", - "File \u001b[0;32m~/Documents/github/sinabs/tests/test_nonsequential/using_SumPool2d/TOP_2_ARCHITECTURES/../../utils/train_test_fn.py:281\u001b[0m, in \u001b[0;36mload_architecture\u001b[0;34m(architecture, input_size, nb_classes, batch_size, surrogate_fn, min_v_mem, spk_thr)\u001b[0m\n\u001b[1;32m 279\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m SCNN(input_size, nb_classes, batch_size, surrogate_fn, min_v_mem, spk_thr)\n\u001b[1;32m 280\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m architecture \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mResSCNN_5\u001b[39m\u001b[38;5;124m'\u001b[39m:\n\u001b[0;32m--> 281\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mResSCNN_5\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m SCNN\n\u001b[1;32m 282\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m SCNN(input_size, nb_classes, batch_size, surrogate_fn, min_v_mem, spk_thr)\n\u001b[1;32m 283\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m architecture \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mResSCNN_6\u001b[39m\u001b[38;5;124m'\u001b[39m:\n", - "File \u001b[0;32m~/Documents/github/sinabs/tests/test_nonsequential/using_SumPool2d/TOP_2_ARCHITECTURES/../models/ResSCNN_5.py:3\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mtorch\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mnn\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mnn\u001b[39;00m\n\u001b[1;32m 2\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01msinabs\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mlayers\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01msl\u001b[39;00m\n\u001b[0;32m----> 3\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01msinabs\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mexodus\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mlayers\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m IAFSqueeze\n\u001b[1;32m 5\u001b[0m \u001b[38;5;28;01mclass\u001b[39;00m \u001b[38;5;21;01mSCNN\u001b[39;00m(nn\u001b[38;5;241m.\u001b[39mModule):\n\u001b[1;32m 6\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m, input_size, nb_classes, batch_size, surrogate_fn, min_v_mem\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m0.313\u001b[39m, spk_thr\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m2.0\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m:\n", - "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'sinabs.exodus'" - ] - } - ], - "source": [ - "snn = load_architecture(achitecture, sensor_size, 11, batch_size, PeriodicExponential(grad_scale=grad_scale, grad_width=grad_width), v_min, spk_thr).to(device)\n", - "snn.init_weights()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "snn.rescale_conv_weights(rescale_method_1, w_rescale_lambda)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "optimizer = Adam(snn.parameters(), lr=lr, betas=(0.9, 0.999), eps=1e-8)\n", - "loss_fn = CrossEntropyLoss()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Training loop" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "87b59082674c47e5a466ddd2ca5097ba", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - " 0%| | 0/134 [00:00" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "y_avg = []\n", - "for y in epochs_y:\n", - " y_avg.append(np.mean(y))\n", - "\n", - "plt.plot(np.arange(len(epochs_x)), y_avg, marker = '.')\n", - "plt.xlabel('epoch')\n", - "plt.ylabel('average loss')\n", - "plt.ylim(0,)\n", - "plt.xticks(np.arange(len(epochs_x)))\n", - "for i, txt in enumerate(y_avg):\n", - " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAG2CAYAAACZEEfAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAACoJElEQVR4nOzddXhT1xsH8O9N0qSpu3uhLe5SKFJgFBn+QzbGsME2YNhwH2wU28aADTaGDmfoOmS4FisUKFKspVCllrol5/dHyKVpU4Ma4f08T56H3Nx7cu5NyH17znvO4RhjDIQQQgghWkpQ1RUghBBCCKlIFOwQQgghRKtRsEMIIYQQrUbBDiGEEEK0GgU7hBBCCNFqFOwQQgghRKtRsEMIIYQQrUbBDiGEEEK0GgU7hBBCCNFqFOwQQgghRKtVabBz/vx59OjRA3Z2duA4DgcPHlR7nTGGefPmwdbWFlKpFJ06dcLjx4/V9klMTMTgwYNhZGQEExMTjBw5EmlpaZV4FoQQQgipzqo02ElPT0eDBg3w66+/anx92bJlWLVqFdatW4erV69CX18ffn5+yMrK4vcZPHgw7t27hxMnTiAgIADnz5/H6NGjK+sUCCGEEFLNcdVlIVCO43DgwAH07t0bgLJVx87ODt9++y2mTJkCAJDJZLC2tsbmzZsxaNAgPHjwALVr18b169fRtGlTAMCxY8fQrVs3vHz5EnZ2dlV1OoQQQgipJkRVXYGihIWFISYmBp06deK3GRsbo0WLFggMDMSgQYMQGBgIExMTPtABgE6dOkEgEODq1avo06ePxrKzs7ORnZ3NP1coFEhMTIS5uTk4jqu4kyKEEEJIuWGMITU1FXZ2dhAIiu6sqrbBTkxMDADA2tpabbu1tTX/WkxMDKysrNReF4lEMDMz4/fRxN/fH999910515gQQgghVeHFixdwcHAo8vVqG+xUpJkzZ2Ly5Mn8c5lMBicnJ7x48QJGRkZVWDNCCCGElFZKSgocHR1haGhY7H7VNtixsbEBAMTGxsLW1pbfHhsbi4YNG/L7xMXFqR2Xl5eHxMRE/nhNJBIJJBJJoe1GRkYU7BBCCCHvmZJSUKrtPDuurq6wsbHBqVOn+G0pKSm4evUqvL29AQDe3t5ITk5GUFAQv8/p06ehUCjQokWLSq8zIYQQQqqfKm3ZSUtLw5MnT/jnYWFhCA4OhpmZGZycnDBx4kR8//33qFmzJlxdXTF37lzY2dnxI7Zq1aqFLl26YNSoUVi3bh1yc3Mxbtw4DBo0iEZiEUIIIQRAFQc7N27cgK+vL/9clUczdOhQbN68GdOmTUN6ejpGjx6N5ORk+Pj44NixY9DV1eWP2b59O8aNG4eOHTtCIBCgX79+WLVqVaWfCyGEEEKqp2ozz05VSklJgbGxMWQyGeXsEEIIIe+J0t6/q23ODiGEEEJIeaBghxBCCCFajYIdQgghhGg1CnYIIYQQotUo2CGEEEKIVqNghxBCCCFajYIdQgghhGg1CnYIIYQQotUo2CGEEEKIVqNghxBCCCFajYIdQgghhGg1CnYIIYQQotUo2CGEEEKIVqNghxBCCCFajYIdQgghhGg1CnYIIYQQotUo2CGEEEKIVqNghxBCCCFajYIdQgghhGg1CnYIIYQQotUo2CGEEEKIVqNghxBCCCFajYIdQgghhGg1CnYIIYQQotUo2CGEEEKIVqNghxBCCCFajYIdQgghhGg1CnYIIYQQotUo2CGEEEKIVqNghxBCCCFajYIdQgghhGg1CnYIIYQQotUo2CGEEEKIVqNghxBCCCFajYIdQgghhGg1CnYIIYQQotUo2CGEEEKIVqNghxBCCCFajYIdQgghhGg1CnYIIYQQotUo2CGEEEKIVqNghxBCCCFajYIdQgghhGg1CnYIIYQQotUo2CGEEEKIVqNghxBCCCFajYIdQgghhGg1CnYIIYQQotUo2CGEEEJIIXK5HHPnzoWrqyukUinc3d2xaNEiMMb4ffbv34/OnTvD3NwcHMchODi4VGWvXLkSnp6ekEqlcHR0xKRJk5CVlcW/7u/vj2bNmsHQ0BBWVlbo3bs3QkND3/pcKNghhBBCSCFLly7F2rVrsWbNGjx48ABLly7FsmXLsHr1an6f9PR0+Pj4YOnSpaUud8eOHZgxYwbmz5+PBw8eYMOGDdi9ezdmzZrF73Pu3DmMHTsWV65cwYkTJ5Cbm4vOnTsjPT39rc5F9FZHEUIIIUSrXb58Gb169UL37t0BAC4uLti5cyeuXbvG7zNkyBAAQHh4eJnKbd26NT799FO+3E8++QRXr17l9zl27JjaMZs3b4aVlRWCgoLQtm3bMp8LtewQQgghpJBWrVrh1KlTePToEQDg9u3buHjxIrp27frO5QYFBfFB07Nnz3DkyBF069atyGNkMhkAwMzM7K3ek1p2CCGEEFLIjBkzkJKSAi8vLwiFQsjlcvzwww8YPHjwO5X76aefIj4+Hj4+PmCMIS8vD1999ZVaN1Z+CoUCEydOROvWrVG3bt23ek9q2SGEEEJIIXv27MH27duxY8cO3Lx5E1u2bMGKFSuwZcuWdyr37NmzWLx4MX777TfcvHkT+/fvx7///otFixZp3H/s2LEICQnBrl273vo9qWWHEEIIIYVMnToVM2bMwKBBgwAA9erVw/Pnz+Hv74+hQ4e+dblz587FkCFD8MUXX/DlpqenY/To0Zg9ezYEgjftMOPGjUNAQADOnz8PBweHt35PCnYIIYQQUkhGRoZa4AEAQqEQCoWiQsoFwA9rZ4zhm2++wYEDB3D27Fm4urq+03tSsEMIIYSQQnr06IEffvgBTk5OqFOnDm7duoWffvoJI0aM4PdJTExEREQEoqKiAICfC8fGxgY2NjYAgM8//xz29vbw9/fny/3pp5/QqFEjtGjRAk+ePMHcuXPRo0cPPugZO3YsduzYgUOHDsHQ0BAxMTEAAGNjY0il0rKfDCNMJpMxAEwmk1V1VQghhJBqISUlhU2YMIE5OTkxXV1d5ubmxmbPns2ys7P5fX7+9XcGoNBj/vz5/D7erduwrn0HsqjkDMYYY7m5uWzBggXM3d2d6erqMkdHRzZmzBiWlJTEH6OpTABs06ZNanUs7f2be13oBy0lJQXGxsaQyWQwMjKq6uoQQggh1d7u6xGYuf8uFAwQcIB/33oY2MypzPu8i9Lev2k0FiGEEELKJFqWyQcxAKBgwKz9IYiWZZZpn8pCwQ4hhBBCShQty8Tlp/GISs7AlsvP+SBGRc4YwuMz+OdP4tJK3KeyUIIyIYQQQoqVvzuqOOYGYgBAVq4cf154Vuh1IcfBxUKvIqpYLGrZIYQQQkiRCnZHqbT3tISAU982cVcwroUloO9vl3HuUTyEAo7fR8hxWNy3LmyN32I01Tuq1sFOaZaXZ4xh3rx5sLW1hVQqRadOnfD48eMqrDUhhBCiPcLi0zW26HzZ1h2XZnTAzlEtsWV4M1gYiHE/OgUDfr+C+9EpAIARrV34fS7O8C3X5OSyqNbBTmmWl1+2bBlWrVqFdevW4erVq9DX14efnx+ysrKqsOaEEEK0gYuLCziOK/QYO3YswsPDNb7GcRz27t2rsbzc3FxMnz4d9erVg76+Puzs7PD555/z89QAyuUUiir3+vXrlXXqPFcL/UItOKruKFtjKbzdzdHO0wq/DW5c6NiNF8MBAN7u5lXSoqNSrYOd/MvLu7i44H//+x86d+7Mr5TKGMPKlSsxZ84c9OrVC/Xr18fWrVsRFRWFgwcPVm3lCSGEvPeuX7+O6Oho/nHixAkAQP/+/eHo6Kj2WnR0NL777jsYGBgUuTJ4RkYGbt68iblz5/LrQoWGhqJnz578Pq1atSpU7hdffAFXV1c0bdq0Us47P1tjKcZ3qMk/F3DQ2B2Vp6H5p6oSkguq1sFOScvLh4WFISYmBp06deKPMTY2RosWLRAYGFhkudnZ2UhJSVF7EEIIIQVZWlryswHb2NggICAA7u7uaNeuHYRCodprNjY2OHDgAAYMGAADAwON5RkbG+PEiROYNm0avLy84O3tjWvXriEoKIhvMRKLxbCxsUFYWBg+/fRTuLu7Y8OGDVAoFMX2WpQm9SM2NhbDhg2DnZ0d9PT00KVLl1KlfqgSj+vZG+HSjA4au6OKawGqatU62FEtQObl5QUdHR00atQIEydO5JeXV00fbW1trXactbU1/5om/v7+MDY25h+Ojo4VdxKEEEIqVHFdTSqBgYHo0KED9PX1YWRkhLZt2yIzs+j5XhYsWFCoPE9PT2zbtg0jRoxAUlISvvnmG3h6ekIqlcLJyQmDBg1CcHAweg38DJefxhc7n0zAqfM4HBiC4NBn2L17N7+9f//+fH39unSBZ5PWmDZ7AQBg+vTphdaUyq+k1A/GGHr37o1nz57h0KFDuHXrFpydndGpUyekp6cXe41vPE8CAHSqZVNkd5StsRT+fetByCkjnqpMSC6oWg89z7+8fJ06dRAcHIyJEyfCzs7unVZcnTlzJiZPnsw/T0lJoYCHEELeU9evX4dcLuefh4SE4KOPPlILHLp06YKZM2di9erVEIlEuH37drGBAwDUqVMHJ0+e5J8HBATgq6++wrBhwxAVFYWoqCisWLECtWvXxvPnz9G3b1/o6ulj4pkMKE5fLdWswpDnQBCwCJ6ensjLy0O7du0AAJ+PHgth3a44KmyJuA3z0cC7Hb7++uti65s/9QNQBoE7d+7kUz8eP36MK1euICQkBHXq1AEArF27FjY2Nti5cye/CrkmN8KVwU5TF9Ni6zCwmRPaelgiPD6Dz+mpDqp1sFPS8vKqRcZiY2Nha2vLHxcbG4uGDRsWWa5EIoFEIqnQuhNCCKkclpaWas+XLFnCdzUBwKRJkzB+/HjMmDGD38fT07PEchknwLN0EVwt9GFrLMXevXvRtWtX2NnZwc7ODvv27eP3tbOzQ05OLrKysiCXy8EJhPyMwW09LPmbfv5h3Eyeh1cHlyAvJQP6ucmYOuVbcByHu0+e40nILZh2ao7ozRORE/sE92SOOHjsFHp36VhkfVu1aoU//vgDjx49goeHB5/68dNPPwFQpnAAgK6uLn+MQCCARCLBxYsXiwx2opIzEZmcCaGAQ0NHkxKvm62xtNoEOSrVuhurpOXlXV1dYWNjg1OnTvGvp6Sk4OrVq/D29q7UuhJCCClaSV1N7du3L/TaV199VWyZRY1YWr9+PUaMGAGO4xAXF4erV69i69atEAgEEAgEsLCwwMWLF4stOyRShgehj9C2oSecnF3RtF1nnDx5En0/GaKxi+rvv/9Gdk42OIkeOIGQ314wQVc1jJvJ8/Dq0BLkyeJg2PhjpKbIMGzYMABA4K0HAADZxR0QGlmA0zWArktDDOjZrdj8GlXjgKenJziOQ8OGDREXF4fPPvsMHMfht99+g5OTE7y9vdWu18uXL9XuowXdeJ6E50s/xjP/7jDQ1VE7dvny5Wr7/vvvv2jRogWkUilMTU3Ru3fvYq9zZanWLTslLS/PcRwmTpyI77//HjVr1oSrqyvmzp0LOzu7anOBCSGElNzVBACjRo3CwoUL+ed6esUntkZHR6s9P3r0KEaMGIHs7Gw+cNiwYQMAICEhAT/88AM8PDywceNGdOzYESEhIahZs2bBYhEty8RFmQnMu02Cjpk95GmJCD66CgoGzL+mgOBO4S6qdevWgROIYNCom1pZAg5qCbqJaTlvAp2kKFh/4o/4gBXw7dQZdnZ2AIDbL5RdRgYNuyDjwXkY1vsIFh2/gCQzDBs3boS/v7/G66FK/Vi6cjWk5vZIiQrDCv9FGDVqFJYvX46BAwfiiy++4Fu8BAIB2rZtyweBRQkKT4TD2L8wsLkjpnT25K/1yJEj0a9fP36/ffv2YdSoUVi8eDE6dOiAvLw8hISEFFluZarWwc7q1asxd+5cjBkzBnFxcbCzs8OXX36JefPm8ftMmzYN6enpGD16NJKTk+Hj44Njx46pNdMRQgipWiV1NQHK4EaVnlAaBfc9dOgQzMzM0KpVK9jZ2SEvL4/vwhk/fjxmzpwJAOjXrx/q169fZOAQFp8OXbc3Q7yZpTPAcQAnQHpoIAwbdFbroop59hCXL1+GjnUNmPsMBjjwk/C1cDWDb4tG8Pf3R5fuPbH06D28OuiPnNinsPrfPOQlxyErPBhtxq5BTk4OLoUl49Dj1y1BjCFPFguDBp2xuG9d7H9WGxEREUVej6lTp+KjT0ZjXYwrFNGAgPNE54Ej8eeff/LXmuM4NG3aFLVq1cLChQthaWmJFi1aFDuk/Xp4EoQGpvBt6MFf80OHDsHX1xdubm4AgLy8PEyYMAHLly/HyJEj+WNr165dZLmVqVp3YxkaGmLlypV4/vw5MjMz8fTpU3z//fcQi8X8PhzHYeHChYiJiUFWVhZOnjwJDw+PKqw1IeR9URFdK6WZ1T0xMRGDBw+GkZERTExMMHLkSKSlpVXYeZZWZV2Pe/fu8aOaOI5DYmIiHjx4gFWrVkEgEMDU1BTffvstMjJKPz9LbGwsAgICkJSUxOee3Lx5E/Hx8QCA7du3w9bWFl27dkVISAhq1apVZOAg4NTHT2eFB0OeGg8dM3vkJb+Z/E/OGE4Gh8P3o66AUAc2g5dgSf9GuDSjA2Z3qwUACHyWiNDQUMhkMvx0IhRhz18g88lVyFPjEb1pPGL+mgQwhvlTxmL2ur8xbsdNCI2sYWBmCd3oYEjsa+HjNk0xsJkTHj16BGdn5yKvQXp6Bg7fjlFbZfy/+7GQyWT8tVb5+++/UatWLXh4eODatWvw8/PTWGZqVi4exiinZ1ElJ8fGxuLff/9VC2pu3ryJyMhICAQCNGrUSO1aVwfVOtghhJCKVNyEcSqjRo1S22fZsmXFllmaWd0HDx6Me/fu4cSJEwgICMD58+cxevToijnJMqis6+Hr64vk5GS+q2nw4MEQi8X4+eefsWnTJkgkEqxduxafffZZqeu+ZcsW6OjowNramh+N9OyZciFKoVCIZs2aISAgAKampmjfvj0ePHhQZOCw6VKY2nMDtyZYeeQ25GkJEOqb8dsV2Rn44tM+yNKzhOOEnfDxsseAZo6wNZZiVFs3DGyqHOXrs+QU5DXa4s8LYRAZW+PUgxgwxiCXy+Ho6ASbNgPhPD0Ae6OMkZ4th4u5PubPmom0xFgYNu2Fh48eY+7cuXj48KFagNGxY0esWbOGf16vVQckX96NjKfXkSeLRcajy5Bd2QfGGH+t9+7diyZNmmD58uWYO3cuEhMToauri82bN2u8FrcikqFggKOZFNZGuvy1NjQ0RN++ffn9VNd6wYIFmDNnjtq1TkxMLPHzq3CMMJlMxgAwmUxW1VUhhFShCRMmMHd3d6ZQKBhjjLVr145NmDCh1McrFApmY2PDli9fzm9LTk5mEomE7dy5kzHG2P379xkAdv36dX6fo0ePMo7jWGRkZPmcSDmpqOvBcRxr3LgxY6zo6wGAAWBPnjwp1Xt5eHgwQ0NDNn36dH7b9u3bGQDWv39/ZmRkxPbu3cvu3bvHpFIp09HRUSu7Q4cObPXq1exGeAJznh7AjJr3YWu2HWT7zt5kh46fZp06dWKGJmbMefwO5jw9gDlN3MPEtp5Mx9KF2Y1ezxzG/sUcx/3FgkOfsby8POW5ZuSwuvOOMefpAfyj15oL/HseP36cAWAOo39X28d1RgCLSs5g0+d+x4SGFozTkbCWLb3ZhQtvjmWMMWdnZzZ//nzGGGM3whNZrRn7mWGTnkxoZMk4kZiJTGyYyMSGdezsxx/zyy+/MAcHB6ajo8OcnJzYnDlz2LFjx4q81j/+F8qcpwewSbtu8ds8PT3ZuHHj1PZTXevff/+d35aVlcUsLCzYunXrSvUZvo3S3r+rdc4OIYRUlpycHGzbtg2TJ09Wa+7fvn07tm3bBhsbG/To0QNz584tMnG2pFndBw0ahMDAQJiYmKjlSHTq1AkCgQBXr15Fnz59Ku4ky6CirkdycjIYY/x0IcVdD4VCgSdPnsDd3b3Yul64cIGfaV81gAUA/x6fjvgS5s6eGD9xImRJSRAKhfjf//6nVu7Tp0/x6tUr+B95CACwE2Xgh6lfIyEhAZaWlvDx8cGtG9egZ2GH8PgMnD17BrNWhgIAov4YxZfTcI3yvF1cXJCRk4e07Dy1ut55KUO0LBO2xlJ07twZl568wqfrr6rto2BAeHwGFi+Yi0OsOTJz5dj2bTu4W6rPyhweHo5oWSbWnXuKlSceIYuJ0XDABEQkKlsJ5bI4RP7xBb4Z82Z+nvHjx2P8+PGIlmUiLD4drhb6MBIpRzhrutY3wpWtMk1ed2FduHABoaGhahMh5r/W+XN0JBIJ3Nzcis0zqiwU7BBCCICDBw+qda0AwKeffgpnZ2fY2dnhzp07mD59OkJDQ7F//36NZZRmVveYmBhYWVmpvS4SiWBmZlbszO+VraKux6ZNm5QDSERiXH4aj8fhLzReDyMjIyQnJ6vNoVaUDRs2oEmTJrhx44ba9iZNmkBHLMGoVYehX78zJENaYu3HnpjatzXqNWmOy0/j+Tl0wsPDsfv6C2zZdwcSEYczRw7CxljzQBdbYylcLLpj/ZMAtdXAhRyHizN8+TlmwuLTUXC1KFUgo9pHtcRCwXJcLPQgEHDwsDbA7ZcyPIpJLRTs7L4egRn77vLvUdPaAIfGtsaRu9GYsvcOch+ehrWVFd+tp7LzWgRm7VceJ+CAYe7KLtaC1zpXrkDwi2QAQDMXM7Vr3aBBg0LXWiKRIDQ0FD4+Psrjc3MRHh5ebJ5RZaFghxBCoPwRV00Yp5I/j6ZevXqwtbVFx44d8fTp0xJbG9535X09omWZeBqXij83bISZtT3OhsYjZP1VpAQ+gyg1E4sWLUK3bt1gbm6OO3fuICUlBTVq1ED9+vX5Mry8vODv76/W+pWSkoK9e/fixx9/VHuvsPh0iAQcdOv5IenCdggMLSAyssLYMauQk5GL1WGW+HW9cgj5ND9PMABLjylbanLyGM49itO4/pOKammEWftDIGdM49IIxQUypS3Hw9oQt1/KEBqbiq713gQjqskJ8wdTT+PSIMvMRa+G9lgccB93bh5H/4EDIRK9udUHBt/DmG/9oeveDEKpIXLiwrFo3Xq0bOVT6Fp/+e0cZOSYwkhXhBqWBhqvtYqRkRG++uorzJ8/H46OjnB2dubn4Mmf81VVKNghhHzwnj9/jpMnTxbZQqHSokULAJqb+wGUalZ3GxsbxMXFqR2Xl5eHxMTEMg27rkjlfT22ng7G7yEKpD+7ibiXL6BjUxO6FiYAAIG+CeIT4nHk2H9YuXIl0tPT4ejoCMYYFixYoFaealRTfrt27QJjDJ988gmAAksxADD1HQFOIER8wE9gedmQ2HrCfMD3EOgqW0kUDFjyOshRYSg887EmJS2NUJqAqKRyPG0MAQCPYlPVjlFNTphf/lajOniOWymvIPJSn3E5Iikbmc9vI+XGYShysyAysoCeRyss+lU9gAkNDcXdsGgApmjqYgaBgCt0rQtavnw5RCIRhgwZgszMTLRo0QKnT5+GqWnxS0xUBgp2CCEfvE2bNsFKQ3N/QcHBwQAKN/er5J/VXRXcqGZ1V61r5O3tjeTkZAQFBaFJkyYAgNOnT0OhUPDBQ1Urz+thZW2NZRv+hmHzvpC6NobjxD14sXowjJv1BgBI7GqB5WRi9LQFGN5LeWP+77//0KVLF/j6+qqVxxhDtCxTrftp9OjRfItT/qUYVDihCKYdRsK0g3IUEwcU6lrSRDXzcUnLHpS0NEJp14oqqhwPa2WwExqjHuy4WugXOpf8rUZTRgzAhUx7XE8SIC07DwYS5e3+WaYubD5dolaWkAPquKp3vTLGMHzzNeDhK3hYKwPD/NdaEx0dHaxYsQIrVqwocp+qQkPPCSEfNIVCgU2bNmHo0KFqzf1Pnz7FokWLEBQUhPDwcBw+fBiff/452rZtW6i5/8CBAwDUZ3U/fPgw7t69i88//1xtVvdatWqhS5cuGDVqFK5du4ZLly5h3LhxGDRokFqXUVUp7+vRtu9Q5XDox1eR8yoc8f/+BJGBGfQ8lEv66Fg4QurWBD/N+7bE67H7egRaLTmNT9dfReslp7H7unriq6bWDkCZlwIog4EZXb345/zr+fZRKdjd9C5sjaXwdjd/q/WiVMFOeEIGsvPezEBtayzlW32AwiuM13cwhpuFPrJyFTgWosydSkjLxvarLwAo50hU+bSFc6G67boWgTMPXwEAfj//rNC1ft9Qyw4h5IN28uRJREREqI3iAQCxWIyTJ0+qda3069cPc+bMUduvYNdKaWZ13759O8aNG4eOHTtCIBCgX79+WLVqVcWeaCmV5/U4ExqH26ZtYdgkAgnHV0ORlQ5dh9qwHrAQQh0xH5j0/3Ypci9uKPZ6RMsy1ZJxNS2yWVRrx/4x3sjIUfAtKyZ6OoW6lgCU2N1UFayNJDDSFSElKw/PXqWjlq0RACBPrkBkknJ9rsV96sLXy0qtvhzHoU8je/x44hEO3orE/5o4YPXpJ0jLzkNdeyP8/lkTLDsWikO3oxASJQNjjB91Fy3LxMwDd/mymIZr/b7hGGOladHTaikpKTA2NoZMJoORkVFVV4cQ8p7JP4z3fb0ZlKdoWSa2X3mOtWefQs4AD2sDPIlLg4K9aYFo62GJbVee49czT2GuL8alGR2gqyMsssytgeGYd+heoe07R7WEt7s5//zjVRcQEqWc8Vf1XpoSjaNlmYW6ljRtqw76r7uM6+FJ+GVQQ/RqaA8AuPMyGT3XXIKRrgi35nWGsGDTFIAXiRlos+wMOA7YPdobg/+8glw5w7aRLeBT0wKvUrPReulp5OQpsHt0S7RwU17Hv4NeYMreO4XKK3itq4PS3r+pZYcQQt5BwWG8+ReH/BAVHA7dwMEYe79qhYT07EKBxKROHjh4KwqRyZnYG/QSQ1pqHqIcm5KFX04WXu27YFcTYwwxKdkAgAU9asOvrk2ZcmRKyr+pKh7WhrgenqSWtxP4NAEA0NzVXGOgAwCOZnpo5mKK6+FJGLbxKnLlDG1qWsCnpgUAwNJQgv81ccCOqxFYd+4pWriZQ5aZi1WluNbvG8rZIYSQtxQty+QDHeBN10q0LLNK61VVNA2HDomUISE9W2PeikgowKg2rgCA9eefIU+uKFTe2dA4DN90HQnpObA2kkB1W+eAQl1NEYkZiE/LhlgowKDmTtUycHkbmkZkXXmmDHZauplpPEbF0VQZoGTkKq9tIycTtddHt3GDgAPOhL7CjqvP8dVfNxCRlAljqUgt16m6dOu9LWrZIYSQtxT2qvCkcaUdxaONNCUIywtMolfQgGaO+OXUY0QkZuBoSAx6NFAmJRccQi7VEWLPl944//gV5h68B3crg0ItaNfDkwAA9RyMi+0Se9/wI7JeBzt5cgV/ri3diu5WipZl4mBwpNq2X08/xSf5AkEXC33UsTPG3UgZZh1QLtopEnDY/kVLmBuIq2W33tuglh1CCHlLadm5hba9783978LVQr/QtpKuh55YhKGtXAAAq08/xuUn8bj5PBEzCgwhz86TQywSoEsd5TD3p6/SkJyRo1ZW0HPl0gZNnat+XpfypAp2XiRmIj07D/eiUpCWnQcjXRGfsKyJ5uBTGYyrRMsyERIlK7SPuYH4nUaRVTcU7BBCyFs6GhKr9lxT18qHxMZIF6Z6Ovzz0nZ/fO7tApGQw6PYNHz651X0XRuIgkNnVBPmWRpKUMPKAIwBV8PUV9NWtXY0dSm+a+d9Y6YvhoWBBADwOC6N78IqLl8HeDODc34Fg8+w+PRC15q9vtbahIIdQgh5C4npOfj3bjQAYPjrlgk7E90POjn5eUIGkjJyIRIAm4Y1w8UZvqW6Htl5csjlxQ8Mzn+TVuWpqG76AJCUnoMncWkAgCZa1rIDAJ42yon9HsWmljpfRzWDs/D1kPLilrTITxtbJynYIYSQ11Sz85YmwXhf0Evk5ClQ194I3/p5QiTgEJmchYgE5V/ELi4u4Diu0GPs2LEAgKysLIwdOxbm5uYwMDBAv379EBsbW9xb8oorW9NrdevWLXXZAPDVV1+B4zisXLmyxPddsuTNbLwXnsQDAJo4mxWa96U4mhbMBFBkgqwqT+XKszctO0HPla067pb6MNMXl+p93yeqrqz7USmlytdRGdjMCRdn+GLnqJYag8/SBETagBKUCSEEr4dM778LxkoeQq5QMOy4ppxRdnALZxhIRGjsZIpr4Ym48OQVBps74/r165DL38x4GxISgo8++ohfFHHSpEn4999/sXfvXhgbG2PcuHHo27cvLl26VGJdFyxYgOXLlyMhIQFjxoyBZ4MmGNT7Y3To2hP//vsvmjVrhsjISPzyyy8wNDTEvHnzSl32gQMHcOXKlSJnc164cCFGjRrFPzc0fDOL78XHyhl327we2lxaRS2YWXAyQJUWrsqb/MOYFCRn5MBET4wbr4Odps7a1YWl4vk62Am4E12qfJ38ymtJi/cZtewQQj54/JDp1zfbkoaQBz5LQFh8OgwkIvR8PXpINXfJxcfK1g1LS0vY2Njwj4CAALi7u6Ndu3aQyWTYsGEDfvrpJ3To0AFNmjTBpk2bcPnyZVy5cqXYukZGRmLOnDnYs2cPdHV18TRZjtHf/wGRiS2mXMxFalYubt68iV9++QX/+9//4Ofnhy1btpS67G+++Qbbt2+Hjo4OUjJzC7V0GRoaqp2Xvr4yKTlPrsDlJwmvr4VlCVdcXVGtCw0cTTUmyGrK27kR/jo52UX7urAAwOP18PP4NOU8Qi3cis/XKSttSkbWhIIdQsgHrzSjVvLbfvU5AKBPI3vov15gUdWacelJPOQFCsvJycG2bdswYsQIcByHoKAg5ObmolOnTvw+Xl5ecHJyQmBgYJH1VCgUGDJkCKZOnYo6depArmA4HPwSaffOwqD+R2DgkJyaAblcjlGjRqFRo0ZYvnw5atSoUeay03Pk+OXU40LrUC1ZsgTm5uZ82Xl5eQCA2y9lSM3Og7FUB/XsjYt8n6KU1N1SkCpfJfBpArJy5bjzUjmiSNuSk1VqWhmoPS9NFxZ5g7qxCCEfvJJWkM7v7stkfmHFwS3f3JDrO5jwaxjdeZmMRk5vWhgOHjyI5ORkDBs2DAAQExMDsVgMExMTtbLNLCwR9DAM0bJMjX9hL126FCKRCOPHjwcA5CkYcuPCochKg35d5YrhUpeGyH50EefOncPly5cxc+ZMREdHw9raGjExMUVeg/xlR8sykZyeA8MCLV3DRn2N9q1bwMzMTK3sn376iW/Ral3j7VscyjKDsbebBbZdicCVZwkIiZQhR66AhYEYLubalVirYqirA3sTKSKTla1sNSwLD/MnRaNghxDywbM1lqJ1DQtcfJ1gy3Gah5AXXArh9otkeNko8yaEAg6t3C1w7F4MLj6OVwt2NmzYgK5duxa7qvnu6xG4H52CZ7ejcHnJ6UI5Q0FBQfjll19w8+ZNfsFGkYBDdnQopG5NIDJU/qWv794MuU8uo379+qhfvz7EYjG+/PJL1K5bH5HJmRoDqYJla0oYljMGv0Ff8Gsj5S/b398fF/h8nbJ1Yb2tFq9bdh7GpOLEA2XydRNnU/7aaCMDyZuJEodvvv7BL01SFtSNRQghABT5JhsZ1NSx0E1E01IIBfN62ngou7IuvG7lAIDnz5/j5MmT+OKLL/htNjY2yMnJQXJyslrZ8vRkCPVNNeYMXbhwAXFxcXBycoJIJIJIJMLLFxHIS4xEdtQjfr/RXRqpld2iRQvk5eXh3pNwHHmSqdYlVVTZbT1tIE+JQ9KZDXi5Vrn6OQfA2Vw9SOLLDn2MWy+U7+dTo2zJyW/LwkDCd+1sv6I8H21NTgaU35FHsWn88w99aZKyomCHvJfKMkSYkJIwxvAgOoV/npKVV2if0uT1tKmhbNW4GZGEtGxlGZs2bYKVlRW6d+/O79ekSRPo6Ojg1KlTfNnZ8S8hT3kFiZ2XxrKHDBmCO3fuIDg4mH9I9fTB6eii1hcr+GHavT9qo1b22cvXAHCQpyVCYuel8SZZsOxzl69DaGAOo+Z9YT1gofIaAThwK0rt/IODgyEQCBCergO5gsHVQh+OZpXXjaTKW1Fda21NTgY0D88vLq+MqKNgR8tERkbis88+g7m5OaRSKerVq4cbN27wr6elpWHcuHFwcHCAVCpF7dq1sW7dumLLvHfvHvr168fPsVFw7g0AkMvlmDt3LlxdXSGVSuHu7o5FixaBFZyasxzsuhaBVv6nCyVOfsgo+Hs3canZSMp4s/SDKi8iv9JMvuZkrgdncz3kKRiuPkuAQqHApk2bMHToULxKfzOyydjYGCNHjsTkyZNx5swZ3LwRhIQjKyGx84LE3osvb0jXVjhw4AAAwNzcHHXr1uUftWvXRlZWFsS2nujl2wIf1bZGduQDzF66Br1798b48eMxe/ZszJgyGQKpoVrZcsbQsnH9Iss+EycBBEJY21hj7/R+mNbFE9mRDzD3h2VYvuM49p65gV/Xb8KkSZPw2WefIThOee0qq1VHJX+SrljIoY5d2ROj3xcfyuR/FYVydrRIUlISWrduDV9fXxw9ehSWlpZ4/PgxTE3f/LUzefJknD59Gtu2bYOLiwv+++8/jBkzBnZ2dujZs6fGcjMyMuDm5ob+/ftj0qRJGvdZunQp1q5diy1btqBOnTq4ceMGhg8fDmNjYz6ZsjwU7EpQ/ZXa1sNSa4dMliT/goklzQ9DNFO16ogEHPIUDFEagh1bYylGtHbFnxfDABQ9+ZpPDQs8T4jAnhsvEPfwGiIiImDdtAtaLzmt9hn9/PPPEAgE6NevH1LSMyF2aQTzj8aolfX08SPcePwSLTXk2Zw8eRJMIYfEzgPNXMxgpi/G4VM6OH/sEHRSo5Ceno4lS5aAEwggcWkCc79xasdHPHuisewXiRnYGqgcbda5tg1a1bBAqxoWCLrhiG2n/sT0ETsAeS5ExtboN3A4vps/E5/8GaQ89zLOr/Ou8gf3OXKGA7deau13XzU8f9b+EMgZ09rJ/yoKxyriT+/3TEpKCoyNjSGTyWBkVLpJmqqjGTNm4NKlS7hw4UKR+9StWxcDBw7E3Llz+W1NmjRB165d8f3335f4Hi4uLpg4cSImTpyotv3jjz+GtbU1NmzYwG/r168fpFIptm3bVvaTKcLlp/H4dP3VQtt3jmrJJ05WlsjISEyfPh1Hjx5FRkYGatSogU2bNqFp06YAUGSi5LJlyzB16lSNr8nlcixYsADbtm1DTEwM7OzsMGzYMMyZM4cvjzGG+fPnY/369cq8DGtPmHUeAx0zewDKm/DFGb70I1gG6849xZKjD9HK3RyXnyaA44DQRV0hFqk3fu+8pgwsGzoaY+1nTTRe47kHQ/DXFWWwIOCAth6WOBv6Sm2f/J/R8Xsx+PKvIOjqCLDnS2+kZ8vxKDYF8w/f5/fXFMRm5shRb8Fx5CkYLkzzhb2JFO1XnEVEYgaW9auPAc0cER6fji6/nEdWrgIch0JrIGkqe9LuYBy4FYnWNcyxbWQL/nsXkZCOtsvPFjo+f7kLetbGsFauJV/wchAty+QDSJUP4bsfLcvU6sn/yqq092/qxtIihw8fRtOmTdG/f39YWVmhUaNGWL9+vdo+rVq1wuHDhxEZGQnGGM6cOYNHjx6hc+fO7/TerVq1wqlTp/DokTJR8vbt27h48SK6du36TuUW9DarKlcEVSuajo4Ojh49ivv37+PHH39Ua0WLjo5We2zcuBEcx6Ffv35FlqtqIVuzZg0ePHiApUuXYtmyZVi9ejW/z7Jly7Bq1SqsW7cOv/99DJyOLuL2zAPLU64ATf34Zadq2Wnlbg6JSADGgNiUrEL7RSQqr2t9BxONN5poWSY/Bw+gbHksGOgAbz6jPLkCy449BAB84eOG+g4m8HY3R+c6NsgfKmvKswl+kYw8BYONkS4cTKUQCDh82kIZsGy7+hwKBcOM/XeQlatA6xrmuDRdOYfND73rqNVFwYCZ++8iWpaJc6FxOHArEgAwo0sttYD9pYbWLkA9gFr0z4NK60ot69xI2kLbJ/+rKNSNpUWePXuGtWvXYvLkyZg1axauX7+O8ePHQywWY+jQoQCA1atXY/To0XBwcIBIJIJAIMD69evRtm3bd3rvGTNmICUlBV5eXhAKhZDL5fjhhx8wePDg8jg1nkigHp8LihgiXNGWLl0KR0dHbNq0id/m6qr+F62NjY3a80OHDsHX1xdubm5Flnv58mX06tWLT2Z1cXHBzp07ce3aNQDKVp2VK1dizpw56NWrF6JlmbD8eDIiVn+GjEeB0K/djvrx38LD6FQAQG07I9iZSBEWn47I5MxCybYvXgc7TkUk4Wq6AQMoNIcPALxKy4L/0Yd4+iodZvpifNnuzfeiuGRU1Xc96LlyxuAmLm+GW/dv4oAf/wvFnZcyjNp6A1eeJUKqI4R/n/qwM9GDnYkemIZVqBQM6LXmIuJSc/ht96NlqOfwJgdG05IOBRWsY0UqaokJ+u4TTahlR4soFAo0btwYixcvRqNGjTB69GiMGjVKLQF59erVuHLlCg4fPoygoCD8+OOPGDt2LE6ePPlO771nzx5s374dO3bswM2bN7FlyxasWLECW7ZsedfTUnM1LEHt+ei2blXSR1+aVrT8YmNj8e+//2LkyJHFlltSC1lYWBhiYmL4mXdtjaXo19IDEjtPZEcpWwioH79ssvPkePpKOaTXy8YIdia6AKAxb+dFknKbg6nmG2pRSaQzunnxSyGojN8ZjA2v839a1zCHoa5OseUIOKjdyFWLQTbNt8K3uYEEtV+vl3TqYRwAoGMtKzjlm2hPU9kA1AIdoHBLUsElHQQAChZTmcHGh7KAJSkf1LKjRWxtbVG7dm21bbVq1cK+ffsAAJmZmZg1axYOHDjAtxzUr18fwcHBWLFihdrU9WU1depUzJgxA4MGDQIA1KtXD8+fP4e/vz/fqlQerjxTBjsSkQDZeQpEywp3NVSG0rSi5bdlyxYYGhqib9++xZZbUguZagZca2tr/hhTfTGEeiaQpydDTyzE/5o4luOZar+ncenIUzAY6Ypga6wLu9c3S03fLVXLjqOZ5htqUUmkA5s5oWcDO4THZ0CuUOCzDdfUjvv3TjRmdXuTKFywHADoVMuaf12uYLgZoQx2muVbHiFalok7kTK1so/cjVabSFBTHQc2d8COqy/UjtPUSlNwwcjzj15VacLsh7CAJSkfFOxokdatWyM0NFRt26NHj+Ds7AwAyM3NRW5uLgQFuoKEQiEUCsU7vXdGRkaFlFvQlWfKpvveDe2x+8YLtUm2KpNCoUDTpk2xePFiAECjRo0QEhKCdevWaQx2Nm7ciMGDB0NXV7fYcvO3kNWpUwfBwcGYOHEi7OzsigwaVa0SAJCRI8fDmBStHoJb3lT5Ol62RuA4DrYmyhtmweHn6dl5SExXtn4UN5dMUTdg1VIIl5/GFzpGwVBkYLHtynP8euYpQiJlkCsYhAIOj2JTkZqVBz2xEF42b1YdD4tPL5SEXFzZqjoCwK5rL0rVJZR/SYfqEGyUZYkJ8uGibiwtMmnSJFy5cgWLFy/GkydPsGPHDvzxxx8YO3YsAMDIyAjt2rXD1KlTcfbsWYSFhWHz5s3YunUr+vTpw5fz+eefY+bMmfzznJwcfrKxnJwcREZGIjg4GE+ePOH36dGjB3744Qf8+++/CA8Px4EDB/DTTz+plfuuXqVm40lcGjgOGOKtDOCexqUhT16+AVVpFNWKFhFReM6fCxcuIDQ0VG0G3aLkbyGrV68ehgwZgkmTJsHf3x/Amzyg2NhY/phnr9Ihz0iG1Ej5F37Q86S3Pq/qqqT5o4YNGwaO49QeXbp0KbbM1NRUTJw4ESO7NkfEj30R+PPXuH79OuzzdWPt378fnTt3hrm5OQx0dZAT+wwmejowytflpElxSaRlmS/F1liKbzrUhKmeDqJkWTgbquyauvH6M27sZAqR8M3PeFnLVtXxXbqEKGGWvA8o2NEizZo1w4EDB7Bz507UrVsXixYtwsqVK9WShHft2oVmzZph8ODBqF27NpYsWYIffvgBX331Fb9PREQEoqOj+edRUVFo1KgRGjVqhOjoaKxYsQKNGjVSu3mvXr0a//vf/zBmzBjUqlULU6ZMwZdffolFixaV2/mp8nW8bIxQ29YIemIhcuQKhCdU/uiLklrR8tuwYQOaNGmCBg0alFhuSS1krq6usLGx4WfHzcqV43lMPLKjQtG2TWsAb3I5tEVpRr4BQJcuXdRGv+3cubPYcr/44gucOHECzYfPg+2INWjRxhedOnWCTlYyAGWwk56eDh8fHyxdupQ/zrGIfJ3SKmtgoasjRP+myq7J7VeVwfSN8NfJyc7q1+BdgpayrjpOyHuFESaTyRgAJpPJqroq1UJUcga79OQVi0rOqOqqqJl94A5znh7AFhwOYYwx1nPNReY8PYD9eyeqyGNevnzJBg8ezMzMzJiuri6rW7cuu379Ov/60KFDGZQDZfiHn59fsfWYP39+oWNsbW2Znp4e27ZtG2OMsdGjRzM3NzcmkUgYAFa/fn324MGDQmV16NCBrV69Wq0+9vb2bPPmzaxXr17MwMCAAWAWFhZ8vZcsWcJ0dHQK1aFZ63bMeXoA8158stT19vT0VNtHVW9dXV1mYWHBevbsqbHeJSnP79D06dOZj49PsfsM+OQz1qZT11K/X0ZGBhMKhSwgIIA1WXSCOU8PYLcikljjxo3Z1xOnMufpAazuvGP8/mFhYcrPedgq9vW2G+90PipRyRns8pP4UtX52as05jw9gLnMCGAvEtNZK/9TzHl6ALvw6NU7l03I+6y0929q2SFqdl+PQKsl1XMpBlW+jmqKeE9r5SKAoTGpGvevqBYBAKhTpw62bt0KLy8vSCQSGBgYqLWiNWnSBJs2bcK8efMgkUhgZ2eHzp07Qy6Xq5Xz6PET3HoUwY96Wb16NT7++GOMHDkSAQEBMDIywtixY7F582a+3tOmTUOtWrUgFosh0hFD4lAbPpN+w4F9eyEUcIiSZWlc7kBV7/znevHiRbXXVfV+8OABjh8/DsaYxnoXZ/f1CLQux+9QSSPfdl+PwJG70bh44Twc7Gxh5+yOr7/+GgkJCUWWmZeXB7lcjiyFAPFp2eA4wNPaEFKpFCE3lZNWpmbnISUrt9Cx5bX2U1m6f1wt9OFTwwKMAT+feIzI5EwIOKChk8k7l03Ih4ASlAkvWpaJGfvv8gmO1Wkphvz5Oi1clbkpHtbKxMxHsZqDndLMhQMAEomk0Jw4JRGJRBgyZAiGDBmi8fXRo0cDANq2bYtZs2bhzp07aNCgAcLDw+Hu7g5AeZPWGfwbTjHgzJLT/Cy2JiYm8Pb2LnImbI7j0KhRI7i6uqLjN8vx44lHaN7YAfbWlqhrZ4TbL2W4EZ4I+4b2Gutd3Lmq6g0o5/j5/vvvC9W7OKrlPBTl+B0qbuRbvfY9MH3fXei6NobUoxVEJtZQJMfg1Jm/EdS1KwIDAyEUCguVaWhoCG9vb3z//ffIa/olajrbY9+enQgMDESNGjVgqqeDpIxcRCVnwshGPT/nXbux3tbgFk64+CQe+26+BKCcE8hAQj/hhJQGtewQnqaRHEXNSFrZC0/mz9cx0RMDADxtig92SjsXztmzZ2FlZQVPT88SWwRUHj9+DDs7O7i5uWHw4MEaE5NV0tPTsWnTJri6usLRUZl7UVRQEC3LLFO9p/dtgcj1X+LatqVISEhAE2dlIHijiLydd613SSpiVtuC80f1GDgE3ft/hnlLfsagP5StMPq120GvZguILV2gW7MlFq/dhuvXr+Ps2bNFlvvXX38hPTsPkb8NxbmZfli1ahU++eQTCAQC2L0ekaVprp3KXNU7v061rWFpKOGf17J5f5e2IaSyUbBDeLoizV+HR3EpaoHN6lOPK33VcdX8Oi3d3swp4vm6ZSc8IQNZuYW7WVQtAjVr1sTx48fx9ddfY/z48WoTHXbp0gVbt27FqVOnsHTpUpw7dw5du3YtttumRYsW2Lx5M44dO4a1a9ciLCwMbdq0QWqqetD122+/wcDAAAYGBjh69ChOnDgBsVgZqBUXFJSl3s3H/gzTdsPw/N4NdO3aFY0dlTfAGxpGZJVHvUviaqGPgkuCFZwMr6zyj3xTdZGdiRUj8uUL5GgYiSfkOLRuXBsWFhZqIwYLcnd3x8ezfofjpL/x3c5zuHbtGnJzc+Hm5sa3QkUlK+faYfn+Cihq9uSKpiMUoJ79mykF/g56Wa26mQmpzijYIbwzGtbwAYD5h+7j0/VX0cr/NBot/A8/nnhUaNXxd23hKWlocf58na+++gocx2H7hrUwlupArmB49iq9UJkKhQJ2dnbYsWMHvL29sWHDBvTs2ZOfUToxMRGXLl3C1KlT0bx5c4wfPx7NmjUrsUWga9eu6N+/P+rXrw8/Pz8cOXIEycnJ2LNnj9p+gwcPxq1bt3Du3Dl4eHhgwIAByMpS3jxdLfQ1zD6rDApKMxP2oEGD0KNHD7wSWUPPwxubd/6N69evI+vFXQDAw5iUQvkm5VHv4kRGRmLq2FGIXv0pIn7si6gNY5Ed/RifNncq1IWl+gxXrlxZYrmWlpbYtm0bdHV1MbTXR8iMDEVuYiRERlZQZKXC9dFuRK7/EhE/9sXL34bD49leJEWGISEhAba2tsWW/SA6FQKxLlrWrYGkpCQcP34cvXr1Uht+DoCfXwcc+BmWK1u0LJMfeg4oM8zL4/8eIR8CCnYIACBXrsCu68oZVL/vXRc7R7XE3q9aqu3DACRlFE7YfNduipISieNSs/h8nVd3L+DKlSuws7MDx3F8646mrixjY2M8f/4c8+fPx82bN9GgQQP8888/CA8PB6AcUh8VFYUVK1YgJCQEmzdvRmBgIMRicbEtAgWZmJjAw8Oj0DHGxsaoWbMm2rZti7///hsPHz7EgQMHACgTSD1eJ1irfNrCWTnnSSnn8IlLzUZ6jhxCAQefxnVhYWGBhKgIOJvrgTHgVkRyude7KPk/w3pfLIXtyN/g3O1LCHQNIC/QgnXgwAH+MyzJ7t27cfv2beTl5aHnoKEQGlshducspAUfg0Hj7shLTUBeyiu0b9EIFh9Phkm7obh14T+0bt0aNWrUgJ+fH19Wx44dsWbNGv75v0eO4nbgGeQmxyDuwTX4+vrCy8sLw4cP57uxnr2MQXBwMC7duA0A0M+MxYOQu/xM1pXpQ134kpDyQMEOAQCcvB+LV6nZsDCQYEBTR3i7myO34F3qtYLdFO+6Hk7+ROLmzZvD1dUVnTt35hNij4UobywO4kzMmDIJ27dvh46OMmnUw+b1iCwNwY5CoYC1tTWGDx+O2rVrY926dRAIBHyXTN26dbFv3z706NED7u7u6NChAyZPnoycnBxYWVmVuv5paWl4+vRpsa0IjDEwxpCdnc0/j09TthbUs1d2PV1+Go88uaLUc/g8jVPOnOxkpoe4mCi+JUM194pqLpaCVK1oZmZmuHHjBrZs2aLWirZgwQJ4eXlBX18f1tbWyMrKwv3794u9BkuXLoW1tTUSUzNw8/dpiN4wBjmXt0CRlYaLT960GEZGRuKbb76Bh4cHoqKicO7cuWLL/emnn/Dll1/i4MGDuHv9EjIeXwWT50LXrTEM6vhCauWKLdu2QyDPQcqp35FwZCXSM7KQkpKCM2fOQCJ5k+Py9OlTxMe/mb046HEkXh1fi+g/v8KUcaPh4+OD48ePQ0dHhw92bl44iUaNGmHC8IEAgCe7fkCjRo3UWtkqS1kmDCSEqKNghwB4M1nZwGYOEL/O3SlyUcOuXmpdMO+6Hk5xCbm7r0dg3qF7YEyBG5sXoePAL1CnTh3+WL5lp8Dw85ycHCQnJyMuLo6fUXrXrl3Izs6GhYUFAGWQMnXqVFy5cgXh4eE4deoU/P39IRAI0K1bN76sgi0CU6ZMwblz5xAeHo7Lly+jT58+EAqF+OSTTwAoc4X8/f0RFBSEiIgIXL58Gf3794dUKuXLjZJlISE9ByIBh43DmsNUTwdPX6Vjb9DLEmfCVtX7+JkLyJPFQvrqHnr16sW3ZKjWSlo2fnChegcEBKBFixZISkpCjRo1YGxiiq9nLEKuSMrXW0dHB/PmzcPmzZvRsGFD6Ojo4JdffsGrV5q7OQFla82DBw9w8eIlgDEYWdmjRxc/SPQN8SIxE88T0qFQKDBkyBD4+fnh0aNHGkdJFfwMg4KC0KlTJ3z88cd4cC8E3x0Mhn6d9oBCwU+Y52ZjhuPHj2PjiWA4Tz0It65fwMzMDPb26qPRAm8/QOch4xAty8Tu6xHYFG0L+y//hNOUg1h5+CrWrFkDY2NlToyqq0pcuwMYY1h96hGcpwdg8u5gMMawYMGCYuteEWjhS0LeHo1bJAiLT8fFJ/HgOGBQvllTi1vU0NVcD6P/ugkTqQ76v+PCk0UNLc7IA36JUHZ1pFz5GxAIcUGnmVqOgmr4ecGWnfj4eCgUCixfvhxbtmzBwoUL4erqig4dOiA5ORmAcmbiO3fuYMuWLUhOToaNjQ0SExMxbty4YlsEXr58iU8++QQJCQmwtLSEj48Prly5AktLSwCArq4uLly4gJUrVyIpKQnW1tZo27YtLl++zLcY3X2ZzNff0lCCbzrUxMKA+/j5xCOcndoeBw4cwMyZM/l655/DR1Xvi+s3IiM1BekWVujfqzsWLVoEiUTCr4KdGPMCsXGv1Oo9aNAgZGQouz2cajeG4cDR2BBlg027wzG1jRUuXLiAoKAgtXpfunQJTZs2xZ07d9CxY0eNn+HTp0+hUCjg2qIXJA7N8LFtBjb8/B3qD5yCWCMbnH8cj5dndkAul+PYsWP477//SpxROj4+HnK5XG3RU3sTKYR6JhCnxeDiDF+1G33nOtaYueMi7h3ZhK9HDlcra/f1CLXRbwUVHB6vatmJkWVBrmB4kaj8zhW1AGhlqQ5rURHyPqJgh2DnNWWrTnsPy0LDaov6cW3vaQ2pjhDJmbl4HJfGDwN/G0UtqvnHH79D4fcdsmOeICXoMGyH/gIFoJajoAp2XiZlIj07D/oF5h1p3bo1pkyZwj+fNm0a33UilUpx/PhxAEBKSgo++ugj1KlTBytWrFArI/D2A4TFp/MrR+/atavY87Gzs8ORI0eK3efOS+XK1A0clS0Jg1s6YdPlMLxIzMSCw/cw6aOOuHv3rsZjVfUesuEqLjyOx9J+9dSm9ne3NICJng7w1UZ4dK2jVu/atWvDz88PT8Ke48iJ0xDevgfDRt1g2LALVlx4hYs796ndQHNycrBq1SoYGxsXG5zI5XJYWdsg5kUYsi7/gys1XNCqVSs8uxYA9GyJA/+dx+XffoG7uzumTZum1jpXFk/ilEGtgURU6EYvVmQj7fAP0DF3Qp0eI/ntBYf5a6x/gRW+rQx1IRRwyFMwvErNxouk16udV9EcO/nRwpeElB11Y33gwuPTsePqcwDA4BaF13UCNM/GKhYJ0NRF2YKgGhb+tjQl5Hp5eeH5c2UQlv3iHhTpMkSuHY7ny3qijac1nj9/jm+//RaN6njwc488jnuz+reFhQWEQqHagpmAcgHNgpPqpaamokuXLjA0NMSBAwf4fCAA2HUtokKG2d+NVAY79exNAAASkRCt3ZXda3tuvCzVe6lGoLlbqic6CwQcbI2U3TBzD91TK0vVimZq6wTrAQth2Kgbkk79gbS7p9SSXQMCAmBgYABdXV38/PPPOHHiBN/9V5RXcXEQmdrB79tVGDd2DM6fP4+EyDAAQOClS4iLi0NgYCC+/fZbiEQiyOVyHDp0CC4uLhrL0/QZPolLgzwjGbZFfIY2Fqaw6jsbJx6++U4+jSuc2FtQwdwXoYCDzetrGJmciYjE161h5lUf7BBCyo6CnQ/Y7usR8F1xFmnZyjllXqVll+l41bIN7xrsFEzIjZZlYuuxK8iTmoMDYFjPF7YjVsNhxGos33EMwcHBsLOzU+atHD+uMW9HLBajSZMm/IKZgLIF6dSpU/D29ua3paSkoHPnzhCLxTh8+DB0dXX5CRP/vROlnFFadTwDZu6/i2hZZqFJFcsyySJjjG/Zqe9gzB+/58aLN3XN916aZObI+SUh3AoEO9GyTDzMdy3yTw+gGtb+2TczILZ2h2HDLjBo4IfU4CNq8+H4+voiODgYly9fRpcuXTBgwADExcWhKBzHQSjWhWm7ofi0WzuMHj0adevWBVPIYSzVgcLUCUbGpjh58iSCg4MRHBwMoVAIX19fvnWtoIKfIWMModEpyAq/jVatNH+GAf8cBicSI+h5EmJkyuHyR+9FFyqbA/h8tKJyX1R5Oy+TMhD9uqzq0LJDCCk76sb6QKma9vP/wTvnQAjae5Z+Wn/VBH9XwxKhUDAICmYzl9KkSZPQqlUrLF68GPpePvh+8z9IOLYHZn7j0K2eLeZ83KFQN5qOjg5sbGzg6ekJj0f3cfFJPGZ80R9xoz7DuHHjAACTJ0/G0KFD0bRpUzRv3hwrV65Eeno6hg9X5nOobpIZGRnYtm0bUlJS8Ne5+/jhyENwUiNwgsIJtAoGjNh8HaExqVAw5Q2zTyN7HLgVyT9XLftQlBeJmZBl5kIsFPDdcJqGFSsYcD0sET01LPvwLF7ZimWqpwMzffUJ/8Li01GwIYPvprG1Ra1atfDn+TD+NR1zR2SEXoKvpxV/ffX19VGjRg3UqFEDLVu2RM2aNbFhwwbMnDlT4zlZWFriVdwryAL3oOYgF+zYsQP379+Hnp4eWtcwx67z4UiRJaFz585v6iSX48yZM/Dz8+OnA+jYsSP69Omj8TOsUachwg79ApabhYljlMtaFPwMpchBHRMF7ryU4eidSJgZ6mL7FWWrFscBjL0JbkrKfVHm7SThRngS5AoGsUgAq3wzGBNC3h8U7Hygipuzo7TBTj17E0h1hEhMz3mnvJ1mzZrhwIEDmDp9BkJDF0BobA3TDqNgUMcXx0KiMefjWvB2Ny/yeM/Xw8/jIp+rJRIPHDgQr169wrx58xATE4M69epj2Z+7oNBVDvW+efMmrl5VLjdQo0YNtTLtv9oAkbE1NHkQrd5qsu9mpNrzktaCuhOZDACoZWtYaORbwc/ku3/uw1RfDKGAg6uFPl/m0yK6sIoqS9Vq07p1a1wLvodUy0RIRALs+bIlxk/ci6tGVjj36BXuR6Wgtl3hZQgUCgU/bF4ThzrNkSS/Dfnji2jXcjdcXV3Rpk0bZGRkoE1NS/xT1xdNWrXFzwMb8sc0bNgQ3q3bYMwcfz6vqGAyeP7PMDo6BpyFC+qPWgZnB2XienGf4SrpNqSLlV2tY9q7Y4i3c6HgprjvuipJWdVy6WAqfeuAnhBStagb6wOlaVr/kubsKDjLcZNGDeAK5Rw4V54lYP/+/ejcuTPMzc3BcRyCg4NLrMf+/fvRtGlTfPbZZwgPC4PI3AHG3gNg2LALAEDOlAnJDx48QM+ePWFsbAx9fX1YWlqib9++AN4kKdf4Zgu+nDRdrfxx48bh+fPn2HrxMZI/WoAfruehlf9pzD14F6eSzeEyPQDOGh75A5383R0f1yt+Rl5lnYuf6O3u6y6seg5vpv4vOKxYwAE2RhIkpOdgyIZrhXKGnr1Stuy4WeoXKv9NWW+22ZtIYWWoi2/GT8C94BuQBe5BD1cB7l04ihvH9qJ1z8HIUzCM/ysQQ76aiICT5/D8+XMEBQVhxIgRiIyMRP/+/fny8g/H3309AtGOHZCXFAVhTR8s3fEf5s6di8DAQIwdOxY+NSwglBrhmdwMzjU8UbduXdStWxemlja4p1MTM08l8OeWf3h4wc9ww7lQ2H7+Exo3aca/1r59e34OI9Vj7ZnHcJ4egCShCXLkDF42hvi2s2eZVwK3M1Z2Y6lywagLi5D3F7XsfKBsjaVoW9MS5x4phyaXNGeHaoZcX19fHD16FJaWlnj8+DEuxwlxPyQLV54loBVLh4+PDwYMGIBRo0aVqh5mZmaYPXs2vLy8sO9WNH7auAsJR1ZCqGcMqVsT5c0/NQY+Hdti5MiR+O6772BkZIR79+5BV1d5M1LlvyRn5qJ1vtXDVQqOxmEA/rpSukRjIcdh/xhvZOQo+EDwSEh0iQmvYlHRLQB8vs7r5GSVgiPfUjPz0Hnlef71/K1GxbXs5C/r5vNkTPv7Nl4kZWLz5XAYSGxh0Wc2Ui9sxZqxu/lh7b0GDkHbZWfw+FUGLp+5hp3b/wKXnQZLC3M0a9YMFy5cUBtBpWqBuR8lw/R9dyGx9YBln9lIPrcFkwbshLu7m9pweVcLfYTFp+PKs0R8VNsa/92PQVJGLozyLYQ6fd9dvqtJU3egKuioYa35nAHlZ73seIEJGWNTEZeaVeYRTKqWHZWqHnZOCHl7FOx8wGSZyqUfvm7njs9bORd7M8g/y7GKq6srzJ8nYnNIIK6GJeLX2Z9BIOD4/IvSaN++PQAgNiULu3dHwKhpL6SHnEb2y/swcG+KxX3r4pcl09CtWzcsW7aMP041u3K0LBPf/XOP364MCO6qdSNde5ZYYnCiMrqNGzZcDFObV6iBo6naPgXnHurdyA4Hb0VBnm+xyK+33cSPAxoU6n5SKBhCIgu37KjkH1Yclq87R0XVaqSaPbmoYEdVVvf6UqRk5WLm/rtYduwBdHVE0KvRHIsnDMUXbdz4faNlmcjOU4ATiWHVZzYAZaBXcC4b1b7bT9xASKQM/X8P5Lfr1WgOvRrNAQBbR7VU63r0qWGBsPh0bLkcjoA7UTgUHAWHrzcWqjPLF/wU7A588vqca1oV3V1aVO5TWbpnVQoGO1W1ACgh5N1RsPOBSsvO44c/D25ZeKHGgg4fPgw/Pz/0798f586dg729PcaMGYOhw0fyeTuP4lLhZVM436MkjDHMORiClMxc2Gc8QVxqNL77sj8G9PaFtaEEX/z7L6ZNmwY/Pz/cunULrq6umDlzJnr37l1E7hFw6UkCWtcwx/lH8fA/8qDQewoAoEBei5DjMNzHBcN9XIpNXNU099AUP0+Ex2fAQCLE1L/v4GFMKoZsuKZ8r3ytFOEJ6UjNzoNEJEBNq6IDFaDoPB57U12ExStbdjR1YxU0qJkj1p9/hmfx6cjOUwa4ujrqydfFJjXnuwYlTc4HFN8devFJ4QCuKAXfn2/ZKea6abpmb7ukQqGWHerGIuS9RTk7H6gb4YmQKxgczaRwKMWPuGp+lpo1a+L48eP4+uuvMX78eOzc/teb+Xaeln0IerQsE4sPBWHDF20QsaI3gtbPwJrVqzFhaD/YGksRFxeHtLQ0LFmyBF26dMF///2HPn36oG/fvjh37pzGJS0AYNrft9HK/zSm77uD5MxcGOmK1HJv/PvVK3Lq/dLkdhTcR/W8noMJVg1qpLZv/qHfqi6sOnZGEAmL/+9XMI9HZfnxR8jMlUMoAETCkhNmY1KyEJ6gvir8/EP31PJiNF3H/EPRAc2T83EcML2LZ7FLGETLMrH99VxO+cue2c3rTY4SoGEV+DdBiiwjF69SlQnSxQU75bmkgpGuCPriN0FhwQk3CSHvD2rZ+UBdeaZcJLKla9GjnPIrapbjdevW4bPFf+HC43hceZaIYa1dS10HVSuBXKGA7fBVaGavhzb6MZg8eTLc3NzQvn17KBQKAECvXr0wadIkAMpRPJcvX8a6deuwc2c7tW4lAae8catyWlTSsvNwYEwrPvdGdfOriKn349MLj1pStVK8mV/HpFRl5W9FevoqDXMOhuCf21HKMhVA++VnSxzqXpqRdwWXBgEAZ3N9fmK9osphDGjoaIqLM3yLvI5FdS3VtzdRO+78o1dqwdSi3nXedGG9Uo6AszPWhYGk+J+t8lpSgeM42JlIKUGZEC1Awc4HSjWcVjUxYEk0zXJcq1Yt7Nu3jy/j4pNXiEwqehRSfvlbCThOAB1TO9zJBNZO6IsHDx7A398f7du3h4WFBUQikcb3vnjxIoDCN7ewV+n49M+ravsrGJCRoyg0hL0ipt4vbuj33dfDzuvZF87XKUr+1qancWnYdDmcf600Q91L27Wjuo43wpMweU8wwuLTcTb0FXy9lOt5vY471ajKKe46Fvf++Y8b2MwJrWtYwG/leaRny9XykR7Hvs5RKqHrT6W8PldVsKOnI0BGbh6MoVPyQYSQaoe6sT5A+fN1WhYzf01+BWc5BoBHjx7B2dkZoTEpr8uVw2fZGfxzO1JTEWqKSyTNP6eLWCxGs2bNinxvlfzdSq6Wmldrf5u8jbehqfvJWE8HBhIRQiKV16q+huTk0uhUq/DcPyUNdS9L146tsRQ9GthhxOsWuiVHH0KuYMiTK7D8+EO1fUvbRVSW93cw1UPn2sqlIE4/fDNjc2mSkytCZm4eACAjV1Guy4UQQioXtex8gFT5Ok5merA3Kd1fv/lnOR4wYACuXbuGP/74A8tWrsGcgyEAAHlmKuQpr7B4u7KLTBWg2NjY8OtRff7557C3t0frQd8AAGSBeyC2qQmRqS0E8lwc23kXf/31F9auXcu/99SpUzFw4EC0bdsWvr6+OHbsGP755x+cPXtWY12LWq29MhdPVLWSPIhOxaz9dxGTkoVvdt5CZq4cemJhoSUeSsvN6u0ScMvatTOmfQ3svBaB0NhU7L/5EskZubj9UgZDXRG2f9EC6dnyMnURleX9fb2scOBWJE4/jMPMbrUAvElOrlnMsPPyFi3LxPWwJP55aVrRCCHVU5mDnTNnzsDX17ci6kIqCZ+v83q5h9JQzXI8c+ZMLFy4kJ+fpU6HHlA8VXYZZT65ioQjK/ljBg0aBACYP38+FixYAACIiIhAZq4Cxw4pAySWm43EE79BnpoAPT0pTtSpjW3btmHgwIF8OX369MG6devg7++P8ePHw9PTE/v27YOPj0+R9S2vvI13oepKWdKvHoZtuo6zoco5jTysDCF8y5l43yWQK0vXjrGeDsb61oD/0YdYcvQhUrKUo7jmdK9V6nyjt33/djUtIRRweByXhoiEDDiZ6/EtO8UlJ5e30o5QI4RUf2UOdrp06QIHBwcMHz4cQ4cOhaOjY0XUi5SDaFkmwuLT1eZ5Acqer6Py8ccf4+OPPy70HqqWBoN6nWBQrxOEHHBxRgeNN4SNewMwdOM1JCVlop69MVYf3oBoWVaJAcmIESMwYsSIMtW3IvJx3kZ7Tys0djLBzYhkAMDtl8nYfT2i2KTi4lRWIDe0lQt+O/sECek5/DZWzJDz8mKsp4Omzqa4GpaI0w9j0b+pI7/oaY23bBF7G+U5jJ0QUrXKnLMTGRmJcePG4e+//4abmxv8/PywZ88e5OTklHwwqTS7r0eg9ZLThZYZyJ+v06KMwY4mqpaG/A0VU/w8Nd6Ad12LQIcfz+FFkvLG9XEDW7hY6JdpCv/3UbQsE8EvkvnnDG+Gor+tsi598DaSMnKQkpmntm32gXerd2l1rKVMij71MA5PXy+NYWEggWmBRU8rUnkOYyeEVK0yBzsWFhaYNGkSgoODcfXqVXh4eGDMmDGws7PD+PHjcfv27YqoJymDqOQMzNj3Zghv/nlerr9Fvk5JBjZzwqUZHeD+eoI7TTcD1eir/JYdDa2UG2dVK27od3VWXDdORevwegTY1WeJuP16uH4Nq5InUCxvA5s54eIMX+wc1RIXZ/i+dWscIaRqvdNorMaNG2PmzJkYN24c0tLSsHHjRjRp0gRt2rTBvXv3Si6AlKtoWSYO3HqJkZtvFHmTetOFVfp8ndKwNZaivafyBnU9PLHQ61V546xqmibsex+6Q6qy3u6WBnAy00OOXIEtr4faV/ZILJXKaEUjhFSstwp2cnNz8ffff6Nbt25wdnbG8ePHsWbNGsTGxuLJkydwdnZWWyH5XRRcabtevXq4ceMG/zpjDPPmzYOtrS2kUik6deqEx48fl8t7V4QFCxaA4zi1h5eXFwAgPDy80Guqx969e4ssc9iwYa8nQNND38aOODapHWL3zFPbJzcxEmOHDcL8/t6I+Lk/DiwciTNnzpTruTV7PZNy0POkQq9ZGEgKbXsfbvjl4X3tDqnKenMcx7fuPKmCkViEEO1S5gTlb775Bjt37gRjDEOGDMGyZctQt25d/nV9fX2sWLECdnZ271y5olbaNjV9szDjsmXLsGrVKmzZsgWurq6YO3cu/Pz8cP/+fX5V7OqmTp06OHnyJP9cJFJ+DI6OjoiOjlbb948//sDy5cvRtWvXIsvLzJFD6toE5t0m8ts4kQ6fXMkBiPv7OySZ2sFiwPfgRGLEBh1G127dER72jB8W/q6aOCtbi0JjUyHLzIWx9M0EbKpJ4VTelxt+eakOo8PeRlXWu4OXFTbnm0CxMpOTCSHapczBzv3797F69Wr07dsXEknhv9YBZV5PebQaFLXStgpjDCtXrsScOXPQq1cvAMDWrVthbW2NgwcP8kOfqxuRSKQxwBAKhYW2HzhwAAMGDICBQdE/9ClZuYBIB0ID9dW5Vw1qBHMDCQxYOuovjYJ51/EQWymvn0nboXhx81+cuxqEgb26l8NZAZaGEriY6yE8IQM3I5Lg+7pbCwBOPYwFAHzawgk96tu9Vzf88lJdRoeVVVXVu4WbGfTEQmTkyAEAhlKaFowQ8nbK3I116tQpfPLJJ0UGOoDyZt6uXbt3qhigXGm7adOm6N+/P6ysrNCoUSOsX7+efz0sLAwxMTHo1KkTv83Y2BgtWrRAYGBgkeVmZ2cjJSVF7VGZHj9+DDs7O7i5uWHw4MGIiNA8K2tQUBCCg4MxcuTIYstLz85DVsRdvFg9GJHrv0TC8V+BrFQ0cTGFt7s56ro7wsmtBtJDTkORkwWmkCM1+BgEeiYwcfQq13NTte7cyJe3I1cwfo6ZHvXtKP+BlIpEJISrxZuk5F5rLtEMxoSQt1LmYMff3x8bN24stH3jxo1YunRpuVRKpaiVtrds2QIAiImJAQBYW6tPoW9tbc2/pom/vz+MjY35R2XOFdSiRQts3rwZx44dw9q1axEWFoY2bdogNTW10L4bNmxArVq10KpVq2LLVNg3hEX3ybAZ9ANM2w1D9osQiE4sgZWBcpgux3HY/88R5MQ9w4uf+yNiRR+k3jgI2wELUd/93bsb81Pl7dwIf5O3c/tlMhLTc2CoK+JXSCekJNGyTNyPevOHSP5RhYQQUhZlDnZ+//13PqE2vzp16mDdunXlUikVhUKBxo0bY/HixWjUqBFGjx6NUaNGvfP7zJw5EzKZjH+8ePGinGpcsq5du6J///6oX78+/Pz8cOTIESQnJ2PPnj1q+2VmZmLHjh0ltuqExafjpVkj6Hu0wOF5n+DQ8kk4f+oYnt6/zS+nwBjDwplTUNvNAXafLYPN5z9Bv2ZLZB5ZDGQkl+v5qYKZ4BfJyMlTrhx5+oFyjaN2HpbQEdJybKR0PuQRfISQ8lXmO09MTAxsbW0Lbbe0tCyUXPuuilppW9Xto8pviY2NVdsnNja22KRbiUQCIyMjtUdVMTExgYeHB548eaK2/e+//0ZGRgY+//zzYo/feU15Ldp7WKKJsxm83c3RskFtWFhY8GWePn0aAQEBOHf0EG6sGYv9cz/DwzN/w8zIgG8lKy/ulgYw1dNBdp4C96KU86Ocer2go2qiOEJK430dsk8IqX7KHOw4Ojri0qVLhbZfunSpXEZg5VfcStuAMlnZxsYGp06d4l9PSUnB1atX4e3tXa51qShpaWl4+vRpoQByw4YN6NmzJywtLYs8NitXjr03lK1Sg1u8WQH85cuXSEhI4MvMyFD+JSwQCNTmDBEIBFAoFOV6PhzHoYnzm66saFkmHkSngOOAdh4U7JDSe1+H7BNCqp8yD28YNWoUJk6ciNzcXHTo0AGAMml52rRp+Pbbb8u1ckWttP3HH38AUN5YJ06ciO+//x41a9bkh57b2dmhd+/e5VqX8jJlyhT06NEDzs7OiIqKwvz58yEUCvHJJ5/w+zx58gTnz5/HkSNHNJbh5eUFf39/cC7NkZCcgrwbeyDtaYbw8Ew8ffoU06ZNQ40aNeDn5wcA8Pb2hqmpKYYOHYp58+ZBKpVi/fr1CAsLQ/fu5TMSK7+mLmY4+SAON54nQk8iBAA0djKFWSVO9U+0w/s6ZJ8QUr2UOdiZOnUqEhISMGbMGH49LF1dXUyfPh0zZ84s18oVtdL24MGD+X2mTZuG9PR0jB49GsnJyfDx8cGxY8eq7Rw7L1++xCeffIKEhARYWlrCx8cHV65cUWvB2bhxIxwcHNC5c2eNZYSGhkImk+Hfq88BTgDjjGj06d0LycnJsLOzQ+fOnbFo0SJ+xJyFhQWOHTuG2bNno0OHDsjNzUWdOnVw6NAhNGjQoNzPsWm+lp08uTLrQjVBHCFl9b4O2SeEVB8cY2+3jnFaWhoePHgAqVSKmjVrFjsUvbpLSUmBsbExZDJZlebvlMXFx/H4bMNVCDggcGZHWBtVn+AuO0+Oegv+Q06egp/Y8NjENvCyeT+uLSGEkPdDae/fbz00xsDAAM2aNUPdunXf60DnfbT7egSGbLgKQBlInA2Nq+IaqZOIhKhvbwxAWT9rQwk8ratmXSNCCCHkraYkvXHjBvbs2YOIiAi+K0tl//795VIxoplq9fD8zXGz9oegrYdltWrq15e8+WrFpmZjz40XtGI0IYSQKlHmlp1du3ahVatWePDgAQ4cOIDc3Fzcu3cPp0+fhrGxcUXUkeQTGp0KRYGOx+o290i0LBPnH79S20aTwRFCCKkqZQ52Fi9ejJ9//hn//PMPxGIxfvnlFzx8+BADBgyAkxP95V6RGGP468rzQtur29wjYfHpKJgJVt0CMkIIIR+OMgc7T58+5Ycri8VipKeng+M4TJo0iR8STirG2nNPcephHAQc+MnWquPcIzQZHCGEkOqkzDk7pqam/DpO9vb2CAkJQb169ZCcnMxPXkfKV7QsE/uDXmL5f48AAIt610UHL6tqO/eIajK4WftDIGesWgZkhBBCPhxlDnbatm2LEydOoF69eujfvz8mTJiA06dP48SJE+jYsWNF1PGDtvt6BGbuv8vn6bRwNeNnS67OwQNNBkcIIaS6KHOws2bNGmRlZQEAZs+eDR0dHVy+fBn9+vXDnDlzyr2CHzLVyKv8CcnXwxMRLct8L4IHmgyOEEJIdVCmYCcvLw8BAQH8MgQCgQAzZsyokIoRZaJvwZFXCgaEx2dQEEEIIYSUUpkSlEUiEb766iu+ZYdULFcL/ULbKNGXEEIIKZsyj8Zq3rw5goODK6AqpKCcPAXyD2qiRF9CCCGk7MqcszNmzBhMnjwZL168QJMmTaCvr976UL9+/XKr3Idu/YVnYFAmJU/s5EGJvoQQQshbKPNCoAJB4cYgjuPAGAPHcZDL5eVWucpSHRcCjU/LRuslp5Gdp8DOUS3h7W5e1VUihBBCqpXS3r/L3LITFhb2ThUjpbP5Ujiy8xRo4GiClm5mVV0dQggh5L1V5mDH2dm5IupB8nkSl4aNF58BAL5u5waO40o4ghBCCCFFKXOws3Xr1mJf//zzz9+6MkQ5ieCMfW9WNU/KyK3S+hBCCCHvuzLn7Jiamqo9z83NRUZGBsRiMfT09JCYmFiuFawM1SVnJ1qWidZLTqvNrSPkOFyc4UuJyYQQQkgBpb1/l3noeVJSktojLS0NoaGh8PHxwc6dO9+p0h86TZMI0mrhhBBCyLspc7CjSc2aNbFkyRJMmDChPIr7YLla6KNgdg5NIkgIIYS8m3IJdgDl7MpRUVHlVdwHydZYiibOb7oJaRJBQggh5N2VOUH58OHDas8ZY4iOjsaaNWvQunXrcqvYhyojRzlP0eSPPNC/qQMFOoQQQsg7KnOw07t3b7XnHMfB0tISHTp0wI8//lhe9fog5eQp8DguFQDQt7E9BTqEEEJIOShzsKNQKCqiHgTAo9hU5MoZjKU6sDehQIcQQggpD+WWs0Pe3f2oFABAHTsjmkiQEEIIKSdlDnb69euHpUuXFtq+bNky9O/fv1wq9aG6FyUDoAx2CCGEEFI+yhzsnD9/Ht26dSu0vWvXrjh//ny5VOpDdY9v2TGu4poQQggh2qPMwU5aWhrEYnGh7To6OkhJSSmXSn2IFAqGB9FvurEIIYQQUj7KHOzUq1cPu3fvLrR9165dqF27drlU6kMUnpCO9Bw5dHUEcLM0qOrqEEIIIVqjzKOx5s6di759++Lp06fo0KEDAODUqVPYuXMn9u7dW+4V/FCourC8bIwgFFByMiGEEFJeyhzs9OjRAwcPHsTixYvx999/QyqVon79+jh58iTatWtXEXX8INyLoi4sQgghpCKUOdgBgO7du6N79+7lXZcP2puRWJScTAghhJSnMufsXL9+HVevXi20/erVq7hx40a5VOpDwxhTm2OHEEIIIeWnzMHO2LFj8eLFi0LbIyMjMXbs2HKp1IcmNiUbCek5EAo4eNoYVnV1CCGEEK1S5mDn/v37aNy4caHtjRo1wv3798ulUh8aVRdWDUsD6OoIq7g2hBBCiHYpc7AjkUgQGxtbaHt0dDREordKAfrgqZKTa1MXFiGEEFLuyhzsdO7cGTNnzoRMJuO3JScnY9asWfjoo4/KtXIfClomghBCCKk4ZW6KWbFiBdq2bQtnZ2c0atQIABAcHAxra2v89ddf5V7BDwG17BBCCCEVp8zBjr29Pe7cuYPt27fj9u3bkEqlGD58OD755BPo6OhURB21miwjFy+TMgEAdWxp2DkhhBBS3t4qyUZfXx+jR48u77p8kM4/jgMA2BjrwliPgkVCCCGkvL11RvH9+/cRERGBnJwcte09e/Z850p9KHZfj8CMfXcBADGyLOy+HoGBzZyquFaEEEKIdilzsPPs2TP06dMHd+/eBcdxYIwBADhOuZ6TXC4v3xpqqWhZJmbuvwuWb9us/SFo62EJW2NpldWLEEII0TZlHo01YcIEuLq6Ii4uDnp6erh37x7Onz+Ppk2b4uzZsxVQRe0UFp8OBVPfJmcM4fEZVVMhQgghREuVuWUnMDAQp0+fhoWFBQQCAQQCAXx8fODv74/x48fj1q1bFVFPreNqoQ8BB7WAR8hxcLHQq7pKEUIIIVqozC07crkchobKJQ0sLCwQFRUFAHB2dkZoaGj51k6L2RpLsbhPPf65gAMW961LXViEEEJIOStzsFO3bl3cvn0bANCiRQssW7YMly5dwsKFC+Hm5lbuFdRm3evb8v8+ObkdJScTQgghFaDM3Vhz5sxBeno6AGDhwoX4+OOP0aZNG5ibm2P37t3lXkFt9io1GwBgIBHBzdKgimtDCCGEaKcyBzt+fn78v2vUqIGHDx8iMTERpqam/IgsUjqqYMfSUFLFNSGEEEK0V7ms3GlmZlYexXxwXqW9DnYMKNghhBBCKkqZc3ZI+Ymnlh1CCCGkwlGwU4VULTsWBuIqrgkhhBCivSjYqUKUs0MIIYRUvDIHO+fPn0deXl6h7Xl5eTh//ny5VOpDQcEOIYQQUvHKHOz4+voiMTGx0HaZTAZfX99yqdSHgk9QpmCHEEIIqTBlDnYYYxqHmCckJEBfX79cKvWhiE9VrhhvaaBbxTUhhBBCtFeph5737dsXgHJ182HDhkEiedMaIZfLcefOHbRq1ar8a6ilFAqGeFWCsiElKBNCCCEVpdTBjrGxMQBly46hoSGk0jdrOInFYrRs2RKjRo0q/xpqqeTMXOS9XgXUXJ+6sQghhJCKUupgZ9OmTQAAFxcXTJkyhbqs3pEqOdlUTwdiEQ2KI4QQQipKme+y06ZNU8vZef78OVauXIn//vuvXCum7WgkFiGEEFI5yhzs9OrVC1u3bgUAJCcno3nz5vjxxx/Rq1cvrF27ttwrqK3iaSQWIYQQUinKHOzcvHkTbdq0AQD8/fffsLGxwfPnz7F161asWrWq3CuorVQtOxa0LhYhhBBSococ7GRkZMDQ0BAA8N9//6Fv374QCARo2bIlnj9/Xu4V1Fa0CCghhBBSOcoc7NSoUQMHDx7EixcvcPz4cXTu3BkAEBcXByMjo3KvoLainB1CCCGkcpQ52Jk3bx6mTJkCFxcXNG/eHN7e3gCUrTyNGjUq9wpqKwp2CCGEkMpR5mDnf//7HyIiInDjxg0cP36c396xY0f8/PPP5Vq5gpYsWQKO4zBx4kR+W1ZWFsaOHQtzc3MYGBigX79+iI2NrdB6lAdKUCaEEEIqx1tN8GJjYwNDQ0OcOHECmZmZAIBmzZrBy8urXCuX3/Xr1/H777+jfv36atsnTZqEf/75B3v37sW5c+cQFRXFz/ZcnVGCMiGEEFI5yhzsJCQkoGPHjvDw8EC3bt0QHR0NABg5ciS+/fbbcq8gAKSlpWHw4MFYv349TE1N+e0ymQwbNmzATz/9hA4dOqBJkybYtGkTLl++jCtXrlRIXcpDrlyBxIzX62JRyw4hhBBSococ7EyaNAk6OjqIiIiAnp4ev33gwIE4duxYuVZOZezYsejevTs6deqktj0oKAi5ublq2728vODk5ITAwMAiy8vOzkZKSoraozIlpueAMUAo4GCqR+tiEUIIIRWp1MtFqPz33384fvw4HBwc1LbXrFmzQoae79q1Czdv3sT169cLvRYTEwOxWAwTExO17dbW1oiJiSmyTH9/f3z33XflXdVSU3VhmeuLIRQUXkGeEEIIIeWnzC076enpai06KomJiWoroZeHFy9eYMKECdi+fTt0dXXLrdyZM2dCJpPxjxcvXpRb2aXxipKTCSGEkEpT5mCnTZs2/HIRAMBxHBQKBZYtWwZfX99yrVxQUBDi4uLQuHFjiEQiiEQinDt3DqtWrYJIJIK1tTVycnKQnJysdlxsbCxsbGyKLFcikcDIyEjtUZkoOZkQQgipPGXuxlq2bBk6duyIGzduICcnB9OmTcO9e/eQmJiIS5culWvlOnbsiLt376ptGz58OLy8vDB9+nQ4OjpCR0cHp06dQr9+/QAAoaGhiIiI4Of/qY5ojh1CCCGk8pQ52Klbty4ePXqENWvWwNDQEGlpaejbty/Gjh0LW1vbcq2coaEh6tatq7ZNX18f5ubm/PaRI0di8uTJMDMzg5GREb755ht4e3ujZcuW5VqX8kTBDiGEEFJ5yhzsREREwNHREbNnz9b4mpOTU7lUrLR+/vlnCAQC9OvXD9nZ2fDz88Nvv/1WqXUoq3haF4sQQgipNGUOdlxdXREdHQ0rKyu17QkJCXB1dYVcLi+3ymly9uxZtee6urr49ddf8euvv1bo+5YnatkhhBBCKk+ZE5QZY+C4wsOl09LSynXElDZTjcaiBGVCCCGk4pW6ZWfy5MkAlKOv5s6dqzb8XC6X4+rVq2jYsGG5V1AbUcsOIYQQUnlKHezcunULgLJl5+7duxCL38z8KxaL0aBBA0yZMqX8a6hlsnLlSM3KA0DBDiGEEFIZSh3snDlzBoBy6Pcvv/xS6XPTaAtVcrJYJICRbplTpgghhBBSRmW+227atKki6vHB4LuwDCQac58IIYQQUr7KnKBM3g0/ezJ1YRFCCCGVgoKdSvaK5tghhBBCKhUFO5WMRmIRQgghlYuCnUoWTyueE0IIIZWKgp1KRi07hBBCSOWiYKeSvRmNJS5hT0IIIYSUBwp2Ktkr6sYihBBCKhUFO5WIMZavZYfWESOEEEIqAwU7lSg9R46sXAUAIE+hqOLaEEIIIR8GCnYq0ebL4fy/O/10DruvR1RdZQghhJAPBAU7lSRalokf/wvlnysYMGt/CKJlmVVYK0IIIUT7UbBTScLi08GY+jY5YwiPz6iaChFCCCEfCAp2KomrhT4KLvsp5Di4WOhVSX0IIYSQDwUFO5XE1liKTrWt+edCjsPivnVhayytwloRQggh2k9U1RX4kNibKAOb3g3tML2rFwU6hBBCSCWglp1KlJCeAwCoa29MgQ4hhBBSSSjYqUSJ6coJBc1pqQhCCCGk0lCwU4kS0pQtO2b6tFQEIYQQUlko2KlESRnKYMdcn1p2CCGEkMpCwU4lYYwhMV3VskPBDiGEEFJZKNipJKnZeciVK2cVpGCHEEIIqTwU7FSSxNf5OnpiIXR1hFVcG0IIIeTDQcFOJUmgLixCCCGkSlCwU0lU+TqUnEwIIYRULgp2KkkStewQQgghVYKCnUqi6sYypWCHEEIIqVQU7FQSfvZkCnYIIYSQSkXBTiV5k6BMsycTQgghlYmCnUpCCcqEEEJI1aBgp5LQ7MmEEEJI1aBgp5LwwQ6teE4IIYRUKgp2Kgkf7OhRsEMIIYRUJgp2KkFWrhwZOXIA1LJDCCGEVDYKdiqBaiSWjpCDoURUxbUhhBBCPiwU7FQC1SKgZvpicBxXxbUhhBBCPiwU7FSChNcTCtIcO4QQQkjlo2CnEiRlqFp2dKq4JoQQQsiHh4KdSpCQRrMnE0IIIVWFgp1KQLMnE0IIIVWHgp1KQLMnE0IIIVWHgp1KkEDBDiGEEFJlKNipBEkU7BBCCCFVhoKdSkDdWIQQQkjVoWCnEiRQgjIhhBBSZSjYqWC5cgVkmbkAqGWHEEIIqQoU7FQw1YSCHAeY0IrnhBBCSKWjYKeCqfJ1TPXEEApoXSxCCCGkslGwU8HeBDu0VAQhhBBSFSjYqWBvZk+mpSIIIYSQqkDBTgWjYeeEEEJI1aJgp4Lxi4AaULBDCCGEVAUKdioYLQJKCCGEVC0KdipY/tFYhBBCCKl8FOxUML5lh7qxCCGEkCpBwU4FowRlQgghpGpRsFOE8+fPo0ePHrCzswPHcTh48KDa62lpaRg3bhwcHBwglUpRu3ZtrFu3rlA5CfmCnc2bN4PjOLWHrq5uoWMePHiAnj17wtjYGPr6+mjWrBkiIiIq5DwJIYQQbSeq6gpUV+np6WjQoAFGjBiBvn37Fnp98uTJOH36NLZt2wYXFxf8999/GDNmDOzs7NCzZ08AgELB+OUiVPPsGBkZITQ0lC+H49RnVX769Cl8fHwwcuRIfPfddzAyMsK9e/c0BkWEEEIIKRkFO0Xo2rUrunbtWuTrly9fxtChQ9G+fXsAwOjRo/H777/j2rVrfLCTkpULuYIBAEz1lTMocxwHGxubIsudPXs2unXrhmXLlvHb3N3d3/V0CCGEkA9Wte7G8vf3R7NmzWBoaAgrKyv07t1brVUEALKysjB27FiYm5vDwMAA/fr1Q2xsbIXXrVWrVjh8+DAiIyPBGMOZM2fw6NEjdO7cmd9H1YVlIBFBIhICUHZ/OTs7w9HREb169cK9e/f4/RUKBf799194eHjAz88PVlZWaNGiRaEuNEIIIYSUXrUOds6dO4exY8fiypUrOHHiBHJzc9G5c2ekp6fz+0yaNAn//PMP9u7di3PnziEqKkpjt1N5W716NWrXrg0HBweIxWJ06dIFv/76K9q2bcvvk1QgOdnT0xMbN27EoUOHsG3bNigUCrRq1QovX74EAMTFxSEtLQ1LlixBly5d8N9//6FPnz7o27cvzp07V+HnRAghhGgl9h6Ji4tjANi5c+cYY4wlJyczHR0dtnfvXn6fBw8eMAAsMDCw1OXKZDIGgMlkMo2vA2AHDhxQ27Z8+XLm4eHBDh8+zG7fvs1Wr17NDAwM2IkTJ/h9joVEM+fpAazXmosay83JyWHu7u5szpw5jDHGIiMjGQD2ySefqO3Xo0cPNmjQoFKfDyGEEPIhKOn+rfJe5ezIZDIAgJmZGQAgKCgIubm56NSpE7+Pl5cXnJycEBgYiJYtW2osJzs7G9nZ2fzzlJSUMtUjMzMTs2bNwoEDB9C9e3cAQP369REcHIwVK1bw9Slp9mQdHR00atQIT548AQBYWFhAJBKhdu3aavvVqlULFy9eLFMdCSGEEKJUrbux8lMoFJg4cSJat26NunXrAgBiYmIgFothYmKitq+1tTViYmKKLMvf3x/Gxsb8w9HRsUx1yc3NRW5uLgQC9csnFAqhUCj45yXNsSOXy3H37l3Y2toCAMRiMZo1a1YoL+nRo0dwdnYuUx0JIYQQovTetOyMHTsWISEh5dLCMXPmTEyePJl/npKSUijgSUtL41tcACAsLAzBwcEwMzODk5MT2rVrh6lTp0IqlcLZ2Rnnzp3D1q1b8dNPP/HHrF/0LZIyxTBruxAAsHDhQrRs2RI1atRAcnIyli9fjufPn+OLL77gj5k6dSoGDhyItm3bwtfXF8eOHcM///yDs2fPvvN5E0IIIR+i9yLYGTduHAICAnD+/Hk4ODjw221sbJCTk4Pk5GS11p3Y2Nhih3dLJBJIJJJi3/PGjRvw9fXln6uCo6FDh2Lz5s3YtWsXZs6cicGDByMxMRHOzs744Ycf8NVXX/HHxEW/hFxkCpFAOZdOUlISRo0ahZiYGJiamqJJkya4fPmyWrdVnz59sG7dOvj7+2P8+PHw9PTEvn374OPjU7qLRQghhBA1HGOMVXUlisIYwzfffIMDBw7g7NmzqFmzptrrMpkMlpaW2LlzJ/r16wcACA0NhZeXV7E5OwWlpKTA2NgYMpkMRkZG5VL33dcjMH3fXQAAB2BJv3oY2MypXMomhBBCSOnv39W6ZWfs2LHYsWMHDh06BENDQz4Px9jYGFKpFMbGxhg5ciQmT54MMzMzGBkZ4ZtvvoG3t3epA52KEC3LxMz9d/nnDMCs/SFo62EJW2NpldWLEEII+RBV62Bn7dq1AMDPUqyyadMmDBs2DADw888/QyAQoF+/fsjOzoafnx9+++23Sq6purD4dCgKtJfJGUN4fAYFO4QQQkglq9bBTml62HR1dfHrr7/i119/rYQalY6rhT4EHNQCHiHHwcVCr+oqRQghhHyg3puh5+8TW2Mp/PvWg/D1Ip9CjsPivnWpVYcQQgipAtW6Zed9NrCZE9p6WCI8PgMuFnoU6BBCCCFVhIKdCmRrLKUghxBCCKli1I1FCCGEEK1GwQ4hhBBCtBoFO4QQQgjRahTsEEIIIUSrUbBDCCGEEK1GwQ4hhBBCtBoFO4QQQgjRahTsEEIIIUSrUbBDCCGEEK1GwQ4hhBBCtBoFO4QQQgjRahTsEEIIIUSrUbBDCCGEEK1GwQ4hhBBCtBoFO4QQQgjRahTsEEIIIUSrUbBDCCGEEK1GwQ4hhBBCtBoFO4QQQgjRahTsEEIIIUSrUbBDCCGEEK1GwQ4hhBBCtBoFO4QQQgjRahTsEEIIIUSrUbBDCCGEEK1GwQ4hhBBCtBoFO4QQQgjRahTsEEIIIUSrUbBDCCGEEK1GwQ4hhBBCtBoFO4QQQgjRahTsEEIIIUSrUbBDCCGEEK1GwQ4hhBBCtBoFO4QQQgjRahTsEEIIIUSrUbBDCCGEEK1GwQ4hhBBCtBoFO4QQQgjRahTsEEIIIUSrUbBDCCGEEK1GwQ4hhBBCtBoFO4QQQgjRahTsEEIIIUSrUbBDCCGEEK1GwQ4hhBBCtBoFO4QQQgjRahTsEEIIIUSrUbBDCCGEEK1GwQ4hhBBCtBoFO4QQQgjRahTsEEIIIUSrUbBDCCGEEK1GwQ4hhBBCtBoFO4QQQgjRahTsEEIIIUSrUbBDCCGEEK1GwQ4hhBBCtBoFO4QQQgjRahTsEEIIIUSraU2w8+uvv8LFxQW6urpo0aIFrl27VtVVIoQQQkg1oBXBzu7duzF58mTMnz8fN2/eRIMGDeDn54e4uLiqrhohhBBCqphWBDs//fQTRo0aheHDh6N27dpYt24d9PT0sHHjxqquGiGEEEKqmKiqK/CucnJyEBQUhJkzZ/LbBAIBOnXqhMDAQI3HZGdnIzs7m38uk8kAACkpKRVbWUIIIYSUG9V9mzFW7H7vfbATHx8PuVwOa2trte3W1tZ4+PChxmP8/f3x3XffFdru6OhYIXUkhBBCSMVJTU2FsbFxka+/98HO25g5cyYmT57MP1coFEhMTIS5uTk4jiu390lJSYGjoyNevHgBIyMjjdvKa5+KLJvqWL32qer3/5DOo6rfn+pIdaxO71+edSwvjDGkpqbCzs6u2P3e+2DHwsICQqEQsbGxattjY2NhY2Oj8RiJRAKJRKK2zcTEpKKqCCMjo0IfcMFt5bVPRZZNdaxe+1T1+39I51HV7091pDpWp/cvzzqWh+JadFTe+wRlsViMJk2a4NSpU/w2hUKBU6dOwdvbuwprRgghhJDq4L1v2QGAyZMnY+jQoWjatCmaN2+OlStXIj09HcOHD6/qqhFCCCGkimlFsDNw4EC8evUK8+bNQ0xMDBo2bIhjx44VSlqubBKJBPPnz1frMiu4rbz2qciyqY7Va5+qfv8P6Tyq+v2pjlTH6vT+5VnHysaxksZrEUIIIYS8x977nB1CCCGEkOJQsEMIIYQQrUbBDiGEEEK0GgU7hBBCCNFujFSYNWvWMGdnZyaRSJiXlxfz8fFhtra2DAD77LPPWNOmTZmBgQGztLRkvXr1YvPnz2f16tVjhoaGzNDQkLVs2ZIdOXKEL8/f358BKPRwc3NjgwcPZmZmZkxXV5fp6Oho3M/Q0JDp6uoyNzc3NmLECNalSxemr6/PADAdHR1Wp04dtTr279+fmZmZMYFAwMRiMQPA3N3d1crv06cPc3V1ZQKBgAkEAgaAP4bjOAaALV26lJ07d459/PHHTE9PT2PdGjZsyLZu3cqsra354zQ9nJyc+PcXCoVMV1eXiUQiBoCJxWLWsWNHNn78eGZpackfI5FImFQq5Z9/9NFHrEGDBkwikTAA/PmpyuU4jtnY2LChQ4fyn5Hq/As+6tWrx7y8vJhQKCyyzgCYnp4ef15SqZRZWVnxx4hEImZgYMBfP5FIxJycnJitrS2/TUdHhz9PjuOYjo4Os7S0ZEZGRozjOMZxHBOJRMze3p65uLjwZec/b9XD1NSU2dvbF3udAbA6deowa2trpqenx6ysrBgApqurywwNDVn//v3ZtGnT+Ourr6/P6tSpwwAwe3t7ZmhoyACwZs2a8Z+Vo6Mj++abb1jjxo35bRYWFqxWrVoMAJswYQJTKBSsS5cuRdZpwIABzNfXl+nq6hZb9/yPkj4b1TUvaR/VZ5H/YW5uXuJxRkZGJe4jlUqZhYUFEwqFTCgUMo7jmEAgYEKhkInFYqarq8vq1q3LBg0axJ87x3Fq/98lEgmzsLBgzs7OfBmq74rq/6NAIGDu7u7s33//ZYMHD+b/D2g6V1dXV2ZpaVni9+R9fIhEIv76iMVi/rlAIGDOzs5s/vz5zMfHh//u6OjoMCMjI/47YG9vz1q3bs0sLCz46yMWi5mJiQkTiUT876GdnR1r166d2v9JOzs7jd8R1f/V4q636neI4zjm5OTE/Pz8+N8WgUDArKys+O+H6jNs0qQJ/zmrfjNV3zOhUMhsbW2Zg4OD2jWoXbs2GzRoEDMwMGAA+PtE/oehoSHz9vZmYrG42DrXq1eP2dvbM11dXVazZk3WsmVLZmtry6RSKfPz82OPHj2qlPsxBTsVZNeuXUwsFrONGzeye/fusS5dujCJRMI2bdrEALBGjRqxTZs2sZCQEBYcHMy6devGLCws2L59+9ijR49YaGgomzVrFtPR0WEhISHs2rVrzMXFhVlZWTFzc3MWHR3NoqOj2YMHD5ijoyMbNmwYu3r1Knv27BnbvXs3CwwM5PcZPnw4A8AWL17MwsLC2N69e5muri6ztrZmDg4ODAD77bff2KeffsrEYjFbv349A5Q36Llz57JRo0bxNy0fHx/WrVs3/otcv359NmPGDPbVV1/xwZipqSkbM2YMW7lyJf9DvnPnTva///2Pubi48DeJCRMmsA0bNjAAbMWKFczAwIC1bNmSrVixggFgPXr0YKNGjeL3AZQ32+nTp7Njx47xPxpSqZTt3buX9evXj/+PbGJiwjZs2MB27NjB/9CPHTuWAWDe3t5MIBCwMWPGsCNHjrA//viDDxYOHTrEdu7cyd/cf/vtNzZt2jTWunVr/kfy6NGj7OTJk6xjx478jWjixIlsz549zNjYmOnr67NDhw6xkydPspo1azJAGZDu3buX/fPPP/yP0ciRI9k///zDBx4CgYCtXLmSNWjQgBkbGzOBQMB69erFADBfX1/WunVrJhQK2eLFi5mzszOztrZmAoGALVq0iB04cID5+voyqVTKBAIBa9KkCQPA/xgvWbKEXbp0ia1bt46JxWImEAjYnDlz2JEjR9jChQtZt27dmFAoZNu2bWPffPMNfyPYuXMn27lzJ/8j37VrV3bnzh3Wtm1bJhAImI6ODuvbty/bsmULk0gkTEdHh7Vq1Yr/LkilUubi4sKGDRvGTp06xRwdHZlYLGbu7u5s+PDhbOvWrUwqlTIdHR02fvx49tNPPzFvb28GgBkYGLAvvviCHT16lDk6OjI3NzcmFouZv78/2759O/8jPXz4cHb06FFmYmLC/6AfOXKEnTlzhhkbGzMAzMHBgZ0+fZo5ODgwgUDAHB0d2bFjx1jjxo2Zg4MDa9OmDbO2tmaAMhju3bs3/5m5urrygYaHhwc7fvw4++ijj5idnR1r3749f0Pw8PBg9erVYzVq1GC3b99mt2/fZrt372Y6OjrM2tqanTp1ih08eJB16tSJubu7s9u3b7MpU6YwgUDA6tSpw2xtbVmvXr34QNHY2Jh16tSJtW3bltWvX599+umnjOM41qFDB7Zt2zbm6enJOI5jDRs2ZADYrl27WL169ZiZmRnr06cP+/PPP5m9vT0TCoXMz8+P7dy5kw0YMIBJpVJma2vLfH19Wc2aNZlEImH29vbMxsaGDRw4kB05coTt2LGDcRzHatasybZu3crOnDnDxo0bx6ysrPh9VO+7YMECdu3aNTZhwgT+Wh85coT16tWL/65///33/LX+9NNP+f/Phw8fZj4+PgwAW7NmDdu7dy8zNzfnA+Bly5axjz76iP/jZdiwYaxDhw78TRQA+/PPP9nt27fZ0qVLGQA2ePBg/lqPHj2a30d1rVU3719//VUtEP3f//7HPvroI+bu7s4HJqr/I6rvlqquCxcuZBzHMYlEwtatW8f279/P/2ZIpVL266+/sgEDBvBB9EcffcRq1arFTE1NGcdxTCgUsr/++otdu3aNffTRR/xvaUBAAFu9ejUTiURMKpXy+6iuh7GxMdu1axdbuHAhX28nJyf2119/sfr16/OB0IoVK1iHDh343wPVd7tBgwb8/4HFixez+vXrMxMTEyYUCpmLiwvbvHkz69+/P/+b6ezszKysrJidnR3jOI65ubmx/fv3sz179jBdXV0mEAiYk5MT27FjB1u6dClzd3dnAoGArV+/nv8N4DiO7dmzhz179oy5urryv/cPHz5ko0ePZk5OTiwtLa3C78kU7FSQ5s2bs7Fjx/LP5XI5s7Oz478ABw4cUNs/Li6OAWDnzp1T225qasrWrFnDatasyU6cOMGcnZ2ZhYUF//r06dOZj49PsXVxcXFhhoaGTKFQ8Nt69erFOI5jAQEBavVp3LgxmzVrFgPAhg4dyu+fnJzMALDJkyezsLAw/j9awfNQbX/+/Lna8127djF7e3sWEhLC/wfNf0zr1q3ZZ599prYtf9mA8i/XhQsXMsYYCw0NZQD4YOLcuXNMLpczU1NTBoB99913/LGXLl1iANjEiRMZAPb48eNC1/rPP/9kANjp06cZY4ydOXOGDxDt7e1ZdHQ0fy6q41Sf2ZAhQ/hyCn6OdevWZQDYrFmzGGOMHT9+nP8h+fzzz1lycjL/l7mhoSH7888/2YMHD/jnU6ZMYQBYUlISY0z5ffjzzz/Znj17mFgs5p8zxtjt27f5H5cff/yRAcrWGYlEwu/TokULNmfOHLXjVFTbGjZsyEQiEdPX12dr1qzhgzGRSMQ6duzIUlNT+R8tZ2dn9vXXX7OaNWvyQemAAQPYkSNH+M+wXbt2bMKECSw1NZXZ2toykUjE2rZtyx/3+++/88Gtra0tc3NzY4CyhVC1z4kTJ5ihoSFr3rw5S01N5be1a9eO38fd3Z2/bvmvta6uLnNwcFC71m5ubowxxl/rkSNH8scnJSWx+fPnswYNGjDGGH+t586dy29TXev8LVp16tRhLVq04PdRXe82bdqobctftupad+/enfn4+PB1lkgkrEaNGoyxN//3dHV1maurK1/OxIkTGcdxbNWqVQwAu3XrVqHfg88//1zt/6NMJmMAmKenJ///UV9fn1lYWKgd5+XlxSwtLdW+HwXLNjQ0VKuP6r1GjRrFX+uuXbsygUDA/vjjD/5aDxgwgP9DJSkpiU2YMIG5u7vzv0979uzhW6AUCgV/rW1sbJidnR3/f9HV1VXtOBsbG2Zqaqr2O5e/7IYNG/ItOcOHD+evtY6ODjM3N2eDBw/mrzXHcczMzIwvx8/PjwFgTZs25a+1jY0Nc3d35/e5du2a2m+b6lpbWFgwqVTKQkJCmLOzM9PR0WF6enr8cba2tkxfX58NHjyY31awbFVA3b59e8bYm98/1fVUKBR8QGNpaclmz57NkpOT+Zag7t27MwAsMDCQCYVC5u7uzmbPns3XWVUOY4zFxMQwQNkyNHr0aObs7Mz/8aTaZ+DAgWzQoEFq2zIyMtTK7tWrF9PX12e2trZs9uzZfJ1r1arFZs+ezRhT3hctLS3Z+vXrWUWjnJ0KkJOTg6CgIHTq1InfJhAI0KlTJwQGBmo8RiaTAQDMzMwAAHK5HLt27UJ6ejr+++8/dO/enS8vOTkZdnZ2cHNzw9q1a+Hp6Yn+/fvDysoKjRo1wvr169XqEhsbC6FQiMePHwMAbt++jUuXLoExBl1dXbV6SKVSnDx5EgDQoEEDfrtq7ZHQ0NBSXQMTExPk5OTwZa5ZswZTp05FnTp1AADR0dGwsrKCp6cnAODGjRvw8PCAn58frKysAABXr14FAH7ds5o1a+Lw4cOIjIxEVlYWAODFixf8dRMI3nyd8197VXnR0dEAlIvU5b/W+d/D0tIS6enp2LJlCwBg/fr1+PXXX9XWWevZsyfq1q2Lb7/9FgBgb2+PVq1awdraGt27d+fLDgoKQkhICADg4sWLSExMRFpaGpjyjwwMGjQIQUFByM3NBcdxyMjIgLe3N2rWrAlzc3NkZGTw1yv/98Hb2xtJSUmQSCT88/T0dKxbtw5CoRBCoRBdunQBoPxeZWdn49tvv4WXlxeuXr2KsLAwJCcnY9q0aWjXrh3OnTvHl21kZITg4GA4OTkhMzMTAQEBaNy4MTiOg0KhgIODA8aOHct/NxQKBc6ePYvu3btjxIgRkEgkiI6OxsqVKwEA7du356/b2LFjUadOHZiYmIDjOJw9exadO3fGgwcPIJFIcPHiRXh5eaFnz5583X///XckJCTg8OHDSE1NhVQqhZubG16+fIlFixZBJpPh7NmzaNq0KZ4+fQqxWIy0tDTY2Njg888/h0KhQG5uLhISEuDu7o7c3FwAQEREBOzs7NCtWzfo6enh8ePHiIyMBAA0bNgQ+/btw6NHj2BnZ4cxY/7f3rmHVVXl//997pwL5xzO4XATDqBclIuSgnKJCE1GUbyOaGYXsdTQssZIJCvFSp+Z1Hkcs5HyrkOWgZYYGd4mL1mEaDUqeMseLS9FIkl44f37g/b6cuz7m2l+v68zDd/1ep79wNln7bXX/uy11v7s9XmvdfKhUqlw5coV1NfXIzAwEOnp6VCpVMjIyMCpU6cAAHV1daiursahQ4eg0+lgs9lw4MABqNVqfPbZZ9BoNDAajVi7di3q6urgcrlQW1sLq9WKnTt3Ii4uDnPmzEFraytaWlrg5+eHUaNGITIyEgDw448/IjU1VbT1V199FTabDQsWLAAAjBkzBmvWrEFiYqJIU1paCgB4/PHH4efnh27duolr8fX1RWZmJpqbm3Hp0iV8+umn8Pb2htFoxNGjR+Ht7Y2AgACo1WqYzWaUlJSIvH19fXHlyhVcuXIFZ8+eRUtLC8rKygAAmZmZol4fOHAAFosFe/fuRdeuXRESEoKKigoMHDgQQFv/tG7dOuTl5YkfYf72229BEnl5ebh69Spef/11qNVq3LhxA0uXLhVt8fTp0zh37hzi4+MxZcoUfPPNN0hLS0NaWhr8/f2Rnp6OVatWIS8vDzU1NaitrYVGo4HVakVVVRWOHz8u6nBDQwMSEhLg5eUFlUoFkmhtbUVdXZ3oewCAPy1Ld+zYMVy5cgWXL18WaWpqagAAP/zwAz7//HOUlJRAp9Ph0qVL6N+/P2JjY3Ht2jXcvHkTzc3NcDgciIqKwvnz53H16lV88skn8PPzQ3x8PBoaGkTe58+fR1NTEwAgNjYWJPHhhx+KdnX06FGcOnVK9GHXr1/Hnj17YLPZkJiYCAA4ceKE6Edu3rwJLy8v7NmzRzx3lHyuXbuGkpISAG2/O6n098p1v/3224iKikJZWRkCAwPFvj59+mDTpk0i7x07dqCiogK9e/fGlStXsH37dtFnf/nll8jKygLQ9lxU2v5t57a7U/8LOXv2LAFw3759HvsLCgrYu3fvn41a3Lx5k4MGDWJaWhoPHz5Ms9lMjUZDm83Gp59+mnFxcWxubibZNlKQnZ3NQ4cOsbKyUsRKp0+fzpqaGi5btoxeXl5ctWoVSXLDhg1Uq9WcMmWKeDtXqVR86aWXmJKSwoyMDALg22+/zbVr14ohfgBcvny5R/nx0wjM/21kp7m5Wbzhms1mUbYBAwawf//+4o0LAPv168fDhw+zvLxc5GUymbhw4UIePHhQ7Nu1a5cYnl63bp14e1SGmW02G/v06cOWlha+9NJL4rhbbWuxWMSbSFZWFtPS0kSa8+fP08vLi3a7XcSmlbfdCRMmiHwAMCQkhIcPH+aaNWvEW5PD4eCKFStYXV3N8PBwqlQq1tXVcfLkybRYLOzTp48IzShbcnIyDxw44KGZSEtLE/deo9Fw5MiRYrREqQ+vvPKK0D0ZDAY++uijHrocnU7HV155RVyHXq9ncXExN27c6KG1euKJJ0SIBT+Nhrzyyiti2N1qtYqQozL073Q62b9/f8bExIhQgt1up9PpFPVTedtXRn4aGhqYkZHBAQMGsFu3bgwJCWFRUREDAgJEWaKjo+lyuRgcHCzqOn56I7bZbFyxYoUIYyi6pP3794uROrvdzkceeYTdunVjTEwMfX19Pcqt1+tZUlLCOXPmiDrp5eXFffv2sbKykmazmUajkYWFhQTAjRs3smvXrvT19eXWrVvpcrnYqVMnD+2WyWSi0+mky+Xi6tWrxdtzSEgI7XY7n3/+eRFCU6lUnDBhAt944w0OGzaMQFuod/jw4QwJCREhYiVt+78DBgzg6NGjxfdarZYzZ85kTU2NuFdK+GTWrFkeafbv3y+OU2zRvkyFhYWsqakRowZarZYjR44U7R8AY2JiWFpayuHDh4t6OHPmTD7++OM0Go2iHSr5+/j4cNSoUSwpKaFGoxH7s7KySJJdunShSqXiW2+9RQBcsWIFNRoNz549S5K8ePEinU4nVSqVqNeBgYFUqVQcM2aMR1+kUqlYVVXFdevWidCf3W7nihUrWFNTI8LtH374IR999FER0ktPT/doi0rbU0LJyr72faZSH9vfpxdeeIEzZszwSBMWFvYzzaTdbvdI06NHD44YMUKEs9rnqfTPAHjvvfd6lElp9xqNRuj8lLq8ceNGj3N27tyZN27cEM8bJZxYXV3NiIgI4qeRsTvuuONnfZPFYhH9UufOnel2u0UY32AwiFFjpZwGg4EzZ84k0KZlUuqB3W7nsmXLRDrlWpOSkvjdd9+xpaWF8+fP96gftxPp7NwG/llnZ/LkyQwNDeVXX33FlpYW1tfXs7q6mvn5+VSpVCwrKxNplZCAgk6no0aj8QhJPPbYY0xOTiZJZmVl8Y477mBwcDBLS0vFg9rhcPD3v/8977rrLtFZJSUl8b777qPb7f6nnZ1r164xJydHHFdfXy86W5VKxUOHDnnkk5eX5/EZaNOltN+XmJjIMWPGMDo6mkBbWC0qKorvvPMODx06xB49eohjFQGs0tHcatsePXoIZyckJIRfffUVybahZkXUd+jQIdbV1XHIkCHU6/XU6/W8ePGiyAeAGG6dPHkyHQ4HAXDy5Mke5+ratSunT58uwkwPPfQQExMTuXr1av7lL38RYY/2nZy/vz8NBgO3bt3K6upqBgQE0Gg0Cv3Uzp07WVhYSIfDwaioKKakpLCgoIBOp5ObN2/m9OnT6eXlJTRdShhpxIgR9PX15cGDB7lhwwZxTh8fHx48eJD19fWMiIhgcnIynU4nTSYTn3jiCRF+mz17Nmtrazlu3DiPB6fS+anVao4dO1bY22w2U6vVirBgQ0MD+/TpQ6PRyLi4OA4YMIAnTpygTqdjTk4Od+/eLR72RqNR1BPlnip5K6EarVYr0pw5c4YA2LVrV9psNr788ssMCgqiv78/q6qqWFtby9zcXA87q9Vqdu/enXq9XrSZO+64g3q93iNk2NDQQKvVyvDwcA4YMIAXLlyg1Wrliy++yJdeeolms5lxcXG0Wq3iWsvLy8Vxr7/+uihz+3ORZExMDPV6PY1GI19++WU+8sgj4qF+q62BtskMUVFRBNqc8PZtX3HSgbbQilqtpsViEe1RpVLRz8+PCQkJ3L9/P4cMGSIenAqhoaFUqVS0WCwkyX379olyK/2Ick+0Wi1JMjo6mqmpqbTb7YyOjmZKSgrj4uJoNBpFGBJoC6kMHDiQAwYMIElarVZ26dJFhIn79u3LwYMHi7bYu3dvOp1OZmVlsa6ujrt37xbCYKUttu8fFLp27erRFsm2vs/b25vTp0+nzWZjVFQUw8LC2KVLF7pcLs6fP1+Ed5Rt0KBBwhG32+2iz8zPzycA0TfOnTuXZrOZDoeDa9euZUZGBkNDQ2k2m2mz2VhcXMy7775b1LvJkyfz8OHDdDqd1Ov1dDgcLC0tZVVVlTh3Tk6O6J8VYXppaSnDwsIYERFBlUolnAklFH7rhA+z2Swcco1GQx8fHwYGBgpnR6PRMD4+3mMCR8+ePTlo0CAP0b9Wq/Vw7txuN4cMGfKzCQ8pKSkewmqbzSacb5VKxZCQEHp7ezM4OJiHDh1iQUGBcN4ULVn7+nE7kc7ObaClpYUajeZnepYHHnhAdDbKd1OmTGFwcDBPnjz5s3yUUQ/lDUDxjJXPN27coNvtpq+vLwsLC8VxS5cuZVBQEE+fPk21Wk2n08klS5Z45D137lxGR0eTpIdjk5ubKxrpwoULPY5ROoNbnZ1r165x2LBhQiB3q9ZGaZy3XkNoaKhHmiFDhngcN3z4cPHQVRrHli1bPOyWm5vLvn37Mi8vj8HBwaKjbWho8LCt2+0WedXW1pIkGxsbhZNx5MgRj3yVEaT2s8qUz0FBQQwODmZlZSUBsKCgwONcubm5In6u3MPPP//cw5Y+Pj6MjIzk5s2bxUMhIiKCEydOJEm63W5GRkZy8ODB4noaGxtps9kYFBQkRlL69evHiRMnctq0aR7lVP5Xq9W02+2cOHEiT548Kb5PSEgQ58rNzeXYsWMZExNDtVrNAwcOeOTT/r61t0P7z+3f3P7evvZv+7eW9e9tvySf/659kG2jTXFxcR62Dg4OFm3G7XYzJCRECGcVW5vNZoaGhgpbJyYmsrCw8O/aOiMjQ6RT9GkBAQEe7TM3N5dms9nD1jqdjt27dxdpvLy86HK5GBgYSJJi1peXl5dI43a7abVaxQPv4MGDdDgcNJlMoj0GBwczNTWVQUFBJMlFixZ52OvWkQWyrf8C2hxi5TilTGq1mn/961/FAzE/P59+fn5Uq9XctGkTJ0yYwN/85jfCSV+/fj179+7N/Px8nj59mkDbi47i7CjHNTY2MiUlhampqWIfSZ4+fdrDvu3vu0qlYkZGhujnlLaoHKc45SkpKWKGFAD6+/t79IeKQDsyMtLD1u01O8HBwXQ4HELQfPDgQVqtVvr5+QlbX7p0Sey71da31mG1Wu1ha0VErKDMDm1v63HjxjE6OppNTU08d+4cJ0yYwICAAGZlZXH37t0E2kbVu3TpwqysLJ47d4533XUXIyMjmZmZSQDctm0br127JkYqlX4pNzeX2dnZYqTl1k2tVtNkMjE7O5vfffcdNRoNTSYTe/XqxezsbDY1NTE/P5+pqaniXFu2bKFOp2N6ejqzs7PFtU2YMIF9+/blhQsXSFLUj9uN1OzcBvR6PXr16oXt27eLfa2trdi+fTtSUlIAACQxdepUlJeXY8eOHQgPD/9ZPv369UNSUhJycnJQW1uL2tpaJCYm4r777hPx5z59+uD7778X8VOgTTsQGhqKlStXws/PDyQ99CwAoNFo0NraKj47HA40NDTg/fffx5gxYwAAhw8fFt8rOhdFY6Nw48YN5Obmor6+Xmh9bsXpdGLSpEniGgBg2LBheP/99z3SKedQOHv2LH744Qf06tULQFu8WaVSedjNZrOhvr4elZWVWLFiBU6dOgWNRoN7771XpGlpacGZM2eEvic0NBSXL19GZGQkGhoa8PHHHyM6Otoj3+LiYuj1eiQlJcHlcmHLli0AgLS0NNy8eRM7duyA2WwGAFRUVIjjwsLCsGPHDly6dAlZWVno0qULAPzM/goZGRnQarW4ePEinE4nWlpacOzYMZw5cwZWq1VoTBobG5GVlQWVSoXMzEyhtVL0HYWFhaiurobBYEB4eDhycnIAAIsWLUK3bt3Q0tKCsLAw+Pr6AgB0Oh1aWloA/Fd9OXfuHEJCQmA0GgEA8fHxHnXP29sbISEhKCsrE5ocf39/DB48GGVlZSgvLxc2Ki4uBgC89957MJlMcDgcKC0tRW1tLfbt24fY2FhxXGVlJQAgKioKixYtEtoPf39/9O3bF2VlZVixYgUAiPMr+RgMBnh5eSE5ORlvvvkmAGDw4MGifTQ1NaG5uRne3t4etr5y5QoCAwOFrb///nuh4WpsbES/fv3Q0tKCqVOnwsvLC01NTThx4gQCAwOFrfV6PYxGI2bMmCFsvWTJEpGuoaEBANDQ0ODRPo8cOYLm5mbEx8cLW5MUugygTfP2ww8/iPr07bffQqPRCO0EAPTu3RuNjY1Ckwa06SyuXbsm2mN6ejpOnz6N0NBQAMD9998PrVYLnU4n7mtQUBAcDgdMJhMA4MKFCwDatCfKcQCg1WqhVquxfPlyJCQk4MaNGzh37hz0ej38/PwwaNAg0a8oepiLFy+iuroaQ4cOxR/+8AcAwPjx40WeLpcL6enpyMrKgl6vx9133y3yAoCVK1fC19cXer0ec+bMQW1tLSZNmgQAWLBgAVauXImVK1fCbrcDaNMzKsf5+fmhqakJZ8+eRWRkpLi/P/74o0d7bG1thVbb9pvYiq0BiPah2ELRySlcv34d33//vbC10+nE9evXxT26//77oVarERYWhtDQUGFrrVYryqvX64XNNBqNyPvmzZs/s7WSzmw2IzAwEDdu3MClS5cwcuRIpKenw+Vy4YsvvsA333yDkSNHwmw246OPPsLZs2eFhtFut2PYsGH4+uuvMXv2bMTGxop+f+jQoXj44Yfx2WefISgoCHq9Hs899xyCgoLw2GOPQaPRYOjQofDx8UFCQgKuXr2Ko0ePYujQoTCbzTh37hwCAwOxZ88euN1upKen4/r166ipqcHQoUPFtSm6QpfLhfr6elE/bju33Z36X8obb7xBg8HAVatW8W9/+xvHjx9Pi8Uihi1TU1NpsVi4YcMGMUV86tSp3LZtG0+dOsXDhw+zsLCQKpWK27ZtE/kGBwdz5MiRPHXqFPfu3StCAEVFRayvr+f69etpMpm4Zs0aut1uzpgxgw8++CA7derELVu28NSpUywrKxP6iyVLlhAAJ02axIiICEZFRQmFvpeXF+fPn8+lS5eK0Z5HH31UrK8CtK274+vryyVLlvDdd98l0BZuWrhwoRjGV6ZZbt26VcyMSk1N5eLFi/niiy8SaNOMaDQaPvbYY+INHD+90ShTxrt06SLeZEtLS/nQQw+J6Z/PP/88g4ODmZ2dzaioKKpUKi5atIiVlZV0Op1Uq9VC47F+/Xra7Xaq1WquX7+eFRUVjI+Pp8lk4urVq/nOO++wf//+1Ov19Pb25q5du0RIzmAwcPXq1Vy1ahXdbreIm8+ePZv79u0To1vKec6cOUObzcbY2FiWl5ezoqJCDLmPGjWKCxcuFOugAG2zyLp37y6G0keNGkWgLXauDDMvXryY69atE7H2WbNmcePGjczIyBBvj2vWrCEAMWW1qKiIJSUltNls4k25sLCQU6dOpV6vF+meeuopfvrppyI0snjxYh4/flyEg+655x6uXbuWDoeD3bt3p8Fg4IgRI1hdXc2UlBRaLBaOHTtWvNnHxMTQaDTy3nvv5ddff82PPvqIM2fOZM+ePZmXl8e9e/cyJyeHWq2WjzzyCEny+PHj4prHjx/PzZs3i+vX6/V86623WF9fz1mzZolree+993jt2jVqNBo6nU6Wl5fzzTffFCNsU6ZMEbZWqVQ0m81cu3Yt4+Li6O3tTYPBIEIVYWFhQhNRWVnJkpIShoWF0WAw8LXXXuPGjRuZlpZGrVZLLy8vbtq0SdgmICCA3t7eXLBgAYOCgsQQf0FBAXfu3MkHH3xQ1I/Fixdz9+7dQvui0Wj4u9/9jgMHDhRpevfuTZvNxoEDB4p7O27cOG7ZskXMRExNTSUAEfYGwKlTp3L79u3s06cPgbYlA8rLyz3C1i+++CJra2vF+dVqNfPy8hgTEyPqda9evVhVVSXKrcxemjNnjqjHVquVkyZN4sqVK6nT6Tht2jQGBQUxPDxcaGQ+/vhj6vV6+vv78+DBg2IG3qhRoxgfH8+YmBjW1dUxICCAycnJrKys5PHjx+nv78+IiAg6HA6eP39ezGoF2qYvnzhxgi6XizabjZ07d6bVauWGDRsYFBTE1NRUEdLx8/PjU089xYiICPr5+YkZQEoYS6VSMTExkTabTdRHAHz22Wf53nvviZC10uaU0T1lNtzu3buZnZ1NnU5Hs9nMP/7xj/ztb38r0ih9tqJHMplMXL58OdetWyfadUJCAnfu3MmHH36YQFtY12Aw8IUXXmBsbKwYOdy9e7cIufr4+PC1117jsmXLhE6vU6dOfOedd5iSkkKdTifWDQLaZq+q1WoGBwdzz549XL16NX19fel2u7l8+XJu3ryZWVlZBNo0PceOHaPL5aLFYmFoaChff/11rl27VvRP/v7+XLZsGZ9//nmq1WoGBARQpVJx9uzZ3LZtm9DDffDBBzx58iTz8/Op0+lYXFzMTZs2MTQ0lCNGjPiXPJOls3Mb+dOf/kS32029Xi86hn+0KTFdl8vFfv36eTg6JOlyuURctlOnThw9ejRLSkoYFxcnFi8sKSnh+++/TwA8duwYGxsbOW3aNLrdbrGo4K26ALnd3q39ooSKzkrpZM1ms8eig3q9XmgD/tHWXgxpMBjocrno7+8vzmc2mz3SmEwmhoeH02AwiI7YbreLRQ2VupeSksK0tDT6+fnRZDLR5XKJB2JkZCQXLFjAq1evMigoiAaDgSaTicOHDxfrNv2jTQkrKOuuJCUlCS2aosVRwhYREREsKCjgnXfeydTUVAYHB9NkMjElJYV+fn60WCy8efMmSdLhcHjY1mAweCzGaLVaabFYPITKiibsn7G10WhkdnY2Bw0aJBbhbB8qAdqEniNGjGB0dLSHWNpkMglbd+rUidnZ2czKyvJwRJVF6ZQF77p27cply5Zx9OjRHoto/rP10GKxsKioiO+++67oMxQ7aTQaUTcjIyPZt29fcQ7FGVREu8rU+JiYGFGW6OhoDhs2TGg9AgIC2KtXL/r4+AjBq7Lm1y/ZlPs4ePBgHj16lCRFv6Zci3JPJkyYwMuXL3PevHni/AkJCRw3bpzQpxw7dox1dXXMycmh0Wj8mRhcq9VSq9Wyc+fOnD59OpOSkjwW+vyl5VbCqGq1moGBgUxKShJ9r6Kjab8wpMViYUhIiMd9HTZsGDMzM8XCkaGhoXS73cImGo2G4eHhYqkNpa0rC7wq+SQnJ/+iMrdfNNBgMDA5OZlhYWFiTS6lj2hf/9PS0sSikyqVij4+PoyJiRHT+wMCAjh+/HiOHTuWQUFBQlNos9mo0+nodrs5a9YstrS0/Euexyqy3bioRCKRSCQSSQdDanYkEolEIpF0aKSzI5FIJBKJpEMjnR2JRCKRSCQdGunsSCQSiUQi6dBIZ0cikUgkEkmHRjo7EolEIpFIOjTS2ZFIJBKJRNKhkc6ORCKR3MKuXbugUqnEzw9IJJL/bKSzI5FIJBKJpEMjnR2JRCKRSCQdGunsSCSSXx2tra2YN28ewsPDYTQa0aNHD2zcuBHAf4WYKioq0L17d/Gr559//rlHHm+//TZiY2NhMBgQFhaGBQsWeHzf0tKCGTNmICQkBAaDAREREVi+fLlHmk8//RSJiYkwmUxITU3FsWPHbu+FSySS24J0diQSya+OefPmYc2aNfjzn/+ML774Ak8++STGjRuH3bt3izQFBQVYsGABPvnkE7hcLuTk5OD69esA2pyU3NxcjBkzBp999hlmz56NZ599FqtWrRLHP/DAAygtLcXixYtx5MgRLFu2DBaLxaMczzzzDBYsWIDq6mpotVrk5eX9S65fIpH8zyJ/CFQikfyqaGlpgcPhQFVVFVJSUsT+hx9+GFevXsXEiRORmZmJN954A6NHjwYAfPfddwgODsaqVauQm5uL++67DxcvXsS2bdvE8U8//TQqKirwxRdfoK6uDtHR0fjggw9wzz33/KwMu3btQmZmJqqqqtCvXz8AwNatWzFo0CA0NzfDy8vrNltBIpH8TyJHdiQSya+K48eP4+rVq+jfvz8sFovY1qxZgxMnToh07R0hh8OB6OhoHDlyBABw5MgRpKWleeSblpaG+vp63Lx5E7W1tdBoNMjIyPi7Zenevbv4PzAwEABw4cKF/+9rlEgk/1q0/+4CSCQSSXuampoAABUVFejUqZPHdwaDwcPh+X/FaDT+onQ6nU78r1KpALTpiSQSyX8WcmRHIpH8qoiJiYHBYMCZM2cQERHhsYWEhIh0H330kfi/oaEBdXV16NatGwCgW7du2Lt3r0e+e/fuRVRUFDQaDeLj49Ha2uqhAZJIJB0XObIjkUh+VXh7e+Opp57Ck08+idbWVtx55524fPky9u7dC6vVitDQUABAcXExnE4n/P398cwzz8DX1xfDhg0DAEyfPh1JSUmYO3cuRo8ejf3792PJkiVYunQpACAsLAwPPvgg8vLysHjxYvTo0QNffvklLly4gNzc3H/XpUskktuEdHYkEsmvjrlz58LlcmHevHk4efIk7HY7evbsiaKiIhFGmj9/PqZNm4b6+nokJCTg3XffhV6vBwD07NkTb775Jp577jnMnTsXgYGBKC4uxkMPPSTO8eqrr6KoqAj5+fn49ttv4Xa7UVRU9O+4XIlEcpuRs7EkEsl/FMpMqYaGBtjt9n93cSQSyX8AUrMjkUgkEomkQyOdHYlEIpFIJB0aGcaSSCQSiUTSoZEjOxKJRCKRSDo00tmRSCQSiUTSoZHOjkQikUgkkg6NdHYkEolEIpF0aKSzI5FIJBKJpEMjnR2JRCKRSCQdGunsSCQSiUQi6dBIZ0cikUgkEkmHRjo7EolEIpFIOjT/BwCNszJEZkJGAAAAAElFTkSuQmCC", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "plt.plot(np.arange(len(epochs_x)), epochs_acc, marker = '.')\n", - "plt.xlabel('epoch')\n", - "plt.ylabel('test accuracy')\n", - "plt.ylim(0, 100)\n", - "plt.xticks(np.arange(len(epochs_x)))\n", - "for i, txt in enumerate(epochs_acc):\n", - " if i%5 ==0 or i == epochs-1:\n", - " plt.text(i, txt, f'{txt:.2f}', ha='center', va='bottom', color = 'k')\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "metadata": {} - }, - "outputs": [], - "source": [ - "# with open(f'{achitecture}-Training_Test-TM.npy', 'wb') as f:\n", - "# np.save(f, np.array(epochs_x))\n", - "# np.save(f, np.array(epochs_y))\n", - "# np.save(f, np.array(epochs_acc))" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "speck-rescnn", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.0" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_1.py b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_1.py deleted file mode 100644 index b8f5b838..00000000 --- a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_1.py +++ /dev/null @@ -1,152 +0,0 @@ -import torch.nn as nn -import sinabs.layers as sl -from sinabs.exodus.layers import IAFSqueeze - -class SCNN(nn.Module): - def __init__(self, input_size, nb_classes, batch_size, surrogate_fn, min_v_mem=-0.313, spk_thr=2.0) -> None: - super().__init__() - - self.conv1 = nn.Conv2d(2, 8, 2, 1, bias=False) - self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool1 = sl.SumPool2d(2,2) - self.pool1a = sl.SumPool2d(4,4) - - self.conv2 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool2 = sl.SumPool2d(2,2) - - self.conv3 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool3 = sl.SumPool2d(2,2) - - self.conv4 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool4 = sl.SumPool2d(2,2) - - self.flat = nn.Flatten() - - flat_s = SCNN.get_flatten_size(input_size) - - self.fc1 = nn.Linear(flat_s, 100, bias=False) - self.iaf1_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc2 = nn.Linear(100, 100, bias=False) - self.iaf2_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc3 = nn.Linear(100, 100, bias=False) - self.iaf3_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc4 = nn.Linear(100, 100, bias=False) - self.iaf4_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc5 = nn.Linear(100, nb_classes, bias=False) - self.iaf5_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - # skip - - self.merge1 = sl.Merge() - - @staticmethod - def get_flatten_size(input_size): - conv1_dims = SCNN.conv2d_output_size(input_size, 8, (2, 2)) - pool1_dims = SCNN.pool_output_size(conv1_dims, 8, (2, 2)) - - conv2_dims = SCNN.conv2d_output_size(pool1_dims, 8, (2, 2)) - pool2_dims = SCNN.pool_output_size(conv2_dims, 8, (2, 2)) - - conv3_dims = SCNN.conv2d_output_size(pool2_dims, 8, (2, 2)) - pool3_dims = SCNN.pool_output_size(conv3_dims, 8, (2, 2)) - - conv4_dims = SCNN.conv2d_output_size(pool3_dims, 8, (2, 2)) - pool4_dims = SCNN.pool_output_size(conv4_dims, 8, (2, 2)) - - return pool4_dims[0]*pool4_dims[1]*pool4_dims[2] - - @staticmethod - def conv2d_output_size(input_size, out_channels, kernel_size, stride=1, padding=0, dilation=1): - input_height, input_width, input_channels = input_size - kernel_height, kernel_width = kernel_size - - output_height = ((input_height + 2 * padding - dilation * (kernel_height - 1) - 1) // stride) + 1 - output_width = ((input_width + 2 * padding - dilation * (kernel_width - 1) - 1) // stride) + 1 - - return (output_height, output_width, out_channels) - - @staticmethod - def pool_output_size(input_size, out_channels, kernel_size, stride=None, padding=0): - input_height, input_width, input_channels = input_size - kernel_height, kernel_width = kernel_size - - if stride is None: - stride = kernel_height - - output_height = ((input_height + 2 * padding - kernel_height) // stride) + 1 - output_width = ((input_width + 2 * padding - kernel_width) // stride) + 1 - - return (output_height, output_width, out_channels) - - def detach_neuron_states(self): - for name, layer in self.named_modules(): - if name != '': - if isinstance(layer, sl.StatefulLayer): - for name, buffer in layer.named_buffers(): - buffer.detach_() - - def init_weights(self): - for name, layer in self.named_modules(): - if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear): - nn.init.xavier_normal_(layer.weight.data) - - def rescale_conv_weights(self, rescale_fn, lambda_): - rescale_fn(self.conv2, [(2, 2)], lambda_) - rescale_fn(self.conv3, [(2, 2), (4, 4)], lambda_) - rescale_fn(self.conv4, [(2, 2)], lambda_) - - - def forward(self, x): - # conv 1 - con1_out = self.conv1(x) - iaf1_out = self.iaf1(con1_out) - pool1_out = self.pool1(iaf1_out) - pool1a_out = self.pool1a(iaf1_out) - - # conv 2 - conv2_out = self.conv2(pool1_out) - iaf2_out = self.iaf2(conv2_out) - pool2_out = self.pool2(iaf2_out) - - merge1_out = self.merge1(pool1a_out, pool2_out) - - # conv 3 - conv3_out = self.conv3(merge1_out) - iaf3_out = self.iaf3(conv3_out) - pool3_out = self.pool3(iaf3_out) - - # conv 4 - conv4_out = self.conv4(pool3_out) - iaf4_out = self.iaf4(conv4_out) - pool4_out = self.pool4(iaf4_out) - - flat_out = self.flat(pool4_out) - - # fc 1 - fc1_out = self.fc1(flat_out) - iaf1_fc_out = self.iaf1_fc(fc1_out) - - # fc 2 - fc2_out = self.fc2(iaf1_fc_out) - iaf2_fc_out = self.iaf2_fc(fc2_out) - - # fc 3 - fc3_out = self.fc3(iaf2_fc_out) - iaf3_fc_out = self.iaf3_fc(fc3_out) - - # fc 4 - fc4_out = self.fc4(iaf3_fc_out) - iaf4_fc_out = self.iaf4_fc(fc4_out) - - # fc 5 - fc5_out = self.fc5(iaf4_fc_out) - iaf5_fc_out = self.iaf5_fc(fc5_out) - - return iaf5_fc_out \ No newline at end of file diff --git a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_10.py b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_10.py deleted file mode 100644 index 9e631bd4..00000000 --- a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_10.py +++ /dev/null @@ -1,121 +0,0 @@ -import torch.nn as nn -import sinabs.layers as sl -from sinabs.exodus.layers import IAFSqueeze - -class SCNN(nn.Module): - def __init__(self, input_size, nb_classes, batch_size, surrogate_fn, min_v_mem=-0.313, spk_thr=2.0) -> None: - super().__init__() - - self.conv1 = nn.Conv2d(2, 8, 2, 1, bias=False) - self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.conv2 = nn.Conv2d(8, 8, 3, 1, 1, bias=False) - self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool2a = sl.SumPool2d(2,2) - - self.conv3 = nn.Conv2d(8, 8, 3, 1, 1, bias=False) - self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool3a = sl.SumPool2d(4,4) - - self.conv4 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool4 = sl.SumPool2d(2,2) - self.pool4a = sl.SumPool2d(4,4) - - self.conv5 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf5 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool5 = sl.SumPool2d(2,2) - self.pool5a = sl.SumPool2d(4,4) - - self.conv6 = nn.Conv2d(8, 8, 3, 1, 1, bias=False) - self.iaf6 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.conv7 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf7 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool7 = sl.SumPool2d(2,2) - - self.conv8 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf8 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool8 = sl.SumPool2d(2,2) - - self.flat = nn.Flatten() - - self.fc_out = nn.Linear(392, nb_classes, bias=False) - self.iaf_fc_out = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - # skip - - self.merge1 = sl.Merge() - self.merge2 = sl.Merge() - self.merge3 = sl.Merge() - self.merge4 = sl.Merge() - self.merge5 = sl.Merge() - - def detach_neuron_states(self): - for name, layer in self.named_modules(): - if name != '': - if isinstance(layer, sl.StatefulLayer): - for name, buffer in layer.named_buffers(): - buffer.detach_() - - def init_weights(self): - for name, layer in self.named_modules(): - if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear): - nn.init.xavier_normal_(layer.weight.data) - - def rescale_conv_weights(self, rescale_fn, lambda_): - rescale_fn(self.conv5, [(2,2), (2,2)], lambda_) - rescale_fn(self.conv6, [(4,4), (2,2)], lambda_) - rescale_fn(self.conv7, [(4,4)], lambda_) - rescale_fn(self.conv8, [(4,4), (2,2)], lambda_) - - def forward(self, x): - # -- conv block 1 --- - con1_out = self.conv1(x) - iaf1_out = self.iaf1(con1_out) # to CONV 4 - # -- conv block 2 --- - conv2_out = self.conv2(iaf1_out) - iaf2_out = self.iaf2(conv2_out) - pool2a_out = self.pool2a(iaf2_out) # to CONV 5 - # -- conv block 3 --- - conv3_out = self.conv3(iaf2_out) - iaf3_out = self.iaf3(conv3_out) - pool3a_out = self.pool3a(iaf3_out) # to CONV 6 - # -- conv block 4 --- - #print(iaf1_out.shape, iaf3_out.shape) - merge1_out = self.merge1(iaf1_out, iaf3_out) - conv4_out = self.conv4(merge1_out) - iaf4_out = self.iaf4(conv4_out) - pool4_out = self.pool4(iaf4_out) - pool4a_out = self.pool4a(iaf4_out) # to CONV 7 - # -- conv block 5 --- - #print(pool2a_out.shape, pool4_out.shape) - merge2_out = self.merge2(pool2a_out, pool4_out) - conv5_out = self.conv5(merge2_out) - iaf5_out = self.iaf5(conv5_out) - pool5_out = self.pool5(iaf5_out) - pool5a_out = self.pool5a(iaf5_out) # to CONV 8 - # -- conv block 6 --- - #print(pool3a_out.shape, pool5_out.shape) - merge3_out = self.merge3(pool3a_out, pool5_out) - conv6_out = self.conv6(merge3_out) - iaf6_out = self.iaf6(conv6_out) - # -- conv block 7 --- - #print(pool4a_out.shape, iaf6_out.shape) - merge4_out = self.merge4(pool4a_out, iaf6_out) - conv7_out = self.conv7(merge4_out) - iaf7_out = self.iaf7(conv7_out) - pool7_out = self.pool7(iaf7_out) - # -- conv block 8 --- - #print(pool5a_out.shape, pool7_out.shape) - merge5_out = self.merge5(pool5a_out, pool7_out) - conv8_out = self.conv8(merge5_out) - iaf8_out = self.iaf8(conv8_out) - pool8_out = self.pool8(iaf8_out) - # -- output -- - flat = self.flat(pool8_out) - #print(flat.shape) - fc_out = self.fc_out(flat) - iaf_fc_out = self.iaf_fc_out(fc_out) - - return iaf_fc_out \ No newline at end of file diff --git a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_11.py b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_11.py deleted file mode 100644 index 209f00bd..00000000 --- a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_11.py +++ /dev/null @@ -1,112 +0,0 @@ -import torch.nn as nn -import sinabs.layers as sl -from sinabs.exodus.layers import IAFSqueeze - -class SCNN(nn.Module): - def __init__(self, input_size, nb_classes, batch_size, surrogate_fn, min_v_mem=-0.313, spk_thr=2.0) -> None: - super().__init__() - - self.conv1 = nn.Conv2d(2, 8, 2, 1, bias=False) - self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.conv2 = nn.Conv2d(8, 8, 3, 1, 1, bias=False) - self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.conv3 = nn.Conv2d(8, 8, 3, 1, 1, bias=False) - self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool3a = sl.SumPool2d(4,4) - - self.conv4 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool4 = sl.SumPool2d(2,2) - - self.conv5 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf5 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool5 = sl.SumPool2d(2,2) - self.pool5a = sl.SumPool2d(4,4) - - self.conv6 = nn.Conv2d(8, 8, 3, 1, 1, bias=False) - self.iaf6 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.conv7 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf7 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool7 = sl.SumPool2d(2,2) - - self.conv8 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf8 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool8 = sl.SumPool2d(2,2) - - self.flat = nn.Flatten() - - self.fc_out = nn.Linear(392, nb_classes, bias=False) - self.iaf_fc_out = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - # skip - - self.merge1 = sl.Merge() - self.merge2 = sl.Merge() - self.merge3 = sl.Merge() - self.merge4 = sl.Merge() - - def detach_neuron_states(self): - for name, layer in self.named_modules(): - if name != '': - if isinstance(layer, sl.StatefulLayer): - for name, buffer in layer.named_buffers(): - buffer.detach_() - - def init_weights(self): - for name, layer in self.named_modules(): - if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear): - nn.init.xavier_normal_(layer.weight.data) - - def rescale_conv_weights(self, rescale_fn, lambda_): - rescale_fn(self.conv5, [(2,2)], lambda_) - rescale_fn(self.conv6, [(4,4), (2,2)], lambda_) - rescale_fn(self.conv8, [(4,4), (2,2)], lambda_) - - def forward(self, x): - # -- conv block 1 --- - con1_out = self.conv1(x) - iaf1_out = self.iaf1(con1_out) # to CONV 4 - # -- conv block 2 --- - conv2_out = self.conv2(iaf1_out) - iaf2_out = self.iaf2(conv2_out) - # -- conv block 3 --- - conv3_out = self.conv3(iaf2_out) - iaf3_out = self.iaf3(conv3_out) - pool3a_out = self.pool3a(iaf3_out) # to CONV 6 - # -- conv block 4 --- - #print(iaf1_out.shape, iaf3_out.shape) - merge1_out = self.merge1(iaf1_out, iaf3_out) - conv4_out = self.conv4(merge1_out) - iaf4_out = self.iaf4(conv4_out) - pool4_out = self.pool4(iaf4_out) - # -- conv block 5 --- - conv5_out = self.conv5(pool4_out) - iaf5_out = self.iaf5(conv5_out) - pool5_out = self.pool5(iaf5_out) - pool5a_out = self.pool5a(iaf5_out) # to CONV 8 - # -- conv block 6 --- - #print(pool3a_out.shape, pool5_out.shape) - merge3_out = self.merge3(pool3a_out, pool5_out) - conv6_out = self.conv6(merge3_out) - iaf6_out = self.iaf6(conv6_out) - # -- conv block 7 --- - conv7_out = self.conv7(iaf6_out) - iaf7_out = self.iaf7(conv7_out) - pool7_out = self.pool7(iaf7_out) - # -- conv block 8 --- - #print(pool5a_out.shape, pool7_out.shape) - merge4_out = self.merge4(pool5a_out, pool7_out) - conv8_out = self.conv8(merge4_out) - iaf8_out = self.iaf8(conv8_out) - pool8_out = self.pool8(iaf8_out) - # -- output -- - flat = self.flat(pool8_out) - #print(flat.shape) - fc_out = self.fc_out(flat) - iaf_fc_out = self.iaf_fc_out(fc_out) - - - return iaf_fc_out \ No newline at end of file diff --git a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_12.py b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_12.py deleted file mode 100644 index adfa33cd..00000000 --- a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_12.py +++ /dev/null @@ -1,91 +0,0 @@ -import torch.nn as nn -import sinabs.layers as sl -from sinabs.exodus.layers import IAFSqueeze - -class SCNN(nn.Module): - def __init__(self, input_size, nb_classes, batch_size, surrogate_fn, min_v_mem=-0.313, spk_thr=2.0) -> None: - super().__init__() - - self.conv1 = nn.Conv2d(2, 1, 3, 1, bias=False) - self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool1 = sl.SumPool2d(3,3) - - self.conv2 = nn.Conv2d(1, 8, 3, 1, bias=False) - self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool2 = sl.SumPool2d(3,3) - - self.conv3 = nn.Conv2d(8, 16, 3, 1, bias=False) - self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool3 = sl.SumPool2d(3,3) - - self.conv4 = nn.Conv2d(16, 32, 3, 1, bias=False) - self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.flat = nn.Flatten() - - self.fc1 = nn.Linear(32, 1024, bias=False) - self.iaf1_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc2 = nn.Linear(1024, 512, bias=False) - self.iaf2_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc3 = nn.Linear(512, 256, bias=False) - self.iaf3_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc4 = nn.Linear(256, 128, bias=False) - self.iaf4_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc5 = nn.Linear(128, nb_classes, bias=False) - self.iaf5_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - def detach_neuron_states(self): - for name, layer in self.named_modules(): - if name != '': - if isinstance(layer, sl.StatefulLayer): - for name, buffer in layer.named_buffers(): - buffer.detach_() - - def init_weights(self): - for name, layer in self.named_modules(): - if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear): - nn.init.xavier_normal_(layer.weight.data) - - def rescale_conv_weights(self, rescale_fn, lambda_): - rescale_fn(self.conv2, [(3, 3)], lambda_) - rescale_fn(self.conv3, [(3, 3)], lambda_) - rescale_fn(self.conv4, [(3, 3)], lambda_) - - def forward(self, x): - # conv 1 - con1_out = self.conv1(x) - iaf1_out = self.iaf1(con1_out) - pool1_out = self.pool1(iaf1_out) - # conv 2 - conv2_out = self.conv2(pool1_out) - iaf2_out = self.iaf2(conv2_out) - pool2_out = self.pool2(iaf2_out) - # conv 3 - conv3_out = self.conv3(pool2_out) - iaf3_out = self.iaf3(conv3_out) - pool3_out = self.pool3(iaf3_out) - # conv 4 - conv4_out = self.conv4(pool3_out) - iaf4_out = self.iaf4(conv4_out) - flat_out = self.flat(iaf4_out) - # fc 1 - fc1_out = self.fc1(flat_out) - iaf1_fc_out = self.iaf1_fc(fc1_out) - # fc 2 - fc2_out = self.fc2(iaf1_fc_out) - iaf2_fc_out = self.iaf2_fc(fc2_out) - # fc 3 - fc3_out = self.fc3(iaf2_fc_out) - iaf3_fc_out = self.iaf3_fc(fc3_out) - # fc 4 - fc4_out = self.fc4(iaf3_fc_out) - iaf4_fc_out = self.iaf4_fc(fc4_out) - # fc 5 - fc5_out = self.fc5(iaf4_fc_out) - iaf5_fc_out = self.iaf5_fc(fc5_out) - - return iaf5_fc_out \ No newline at end of file diff --git a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_13.py b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_13.py deleted file mode 100644 index 8e473dcd..00000000 --- a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_13.py +++ /dev/null @@ -1,96 +0,0 @@ -import torch.nn as nn -import sinabs.layers as sl -from sinabs.exodus.layers import IAFSqueeze - -class SCNN(nn.Module): - def __init__(self, nb_classes, batch_size, surrogate_fn, min_v_mem=-0.313, spk_thr=2.0) -> None: - super().__init__() - - self.conv1 = nn.Conv2d(2, 16, 3, 1, bias=False) - self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool1 = sl.SumPool2d(2,2) - - self.conv2 = nn.Conv2d(16, 16, 3, 1, bias=False) - self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool2 = sl.SumPool2d(2,2) - - self.conv3 = nn.Conv2d(16, 16, 3, 1, bias=False) - self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool3 = sl.SumPool2d(2,2) - - self.conv4 = nn.Conv2d(16, 16, 3, 1, bias=False) - self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool4 = sl.SumPool2d(2,2) - - self.conv5 = nn.Conv2d(16, 16, 3, 1, bias=False) - self.iaf5 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool5 = sl.SumPool2d(2,2) - - self.flat = nn.Flatten() - - self.fc1 = nn.Linear(64, 200, bias=False) - self.iaf1_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc2 = nn.Linear(200, 200, bias=False) - self.iaf2_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc3 = nn.Linear(200, 200, bias=False) - self.iaf3_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc4 = nn.Linear(200, nb_classes, bias=False) - self.iaf4_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - def detach_neuron_states(self): - for name, layer in self.named_modules(): - if name != '': - if isinstance(layer, sl.StatefulLayer): - for name, buffer in layer.named_buffers(): - buffer.detach_() - - def init_weights(self): - for name, layer in self.named_modules(): - if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear): - nn.init.xavier_normal_(layer.weight.data) - - def rescale_conv_weights(self, rescale_fn, lambda_): - rescale_fn(self.conv2, [(2, 2)], lambda_) - rescale_fn(self.conv3, [(2, 2)], lambda_) - rescale_fn(self.conv4, [(2, 2)], lambda_) - rescale_fn(self.conv5, [(2, 2)], lambda_) - - def forward(self, x): - # conv 1 - con1_out = self.conv1(x) - iaf1_out = self.iaf1(con1_out) - pool1_out = self.pool1(iaf1_out) - # conv 2 - conv2_out = self.conv2(pool1_out) - iaf2_out = self.iaf2(conv2_out) - pool2_out = self.pool2(iaf2_out) - # conv 3 - conv3_out = self.conv3(pool2_out) - iaf3_out = self.iaf3(conv3_out) - pool3_out = self.pool3(iaf3_out) - # conv 4 - conv4_out = self.conv4(pool3_out) - iaf4_out = self.iaf4(conv4_out) - pool4_out = self.pool4(iaf4_out) - # conv 5 - conv5_out = self.conv5(pool4_out) - iaf5_out = self.iaf5(conv5_out) - pool5_out = self.pool5(iaf5_out) - # fc 1 - flat_out = self.flat(pool5_out) - fc1_out = self.fc1(flat_out) - iaf1_fc_out = self.iaf1_fc(fc1_out) - # fc 2 - fc2_out = self.fc2(iaf1_fc_out) - iaf2_fc_out = self.iaf2_fc(fc2_out) - # fc 3 - fc3_out = self.fc3(iaf2_fc_out) - iaf3_fc_out = self.iaf3_fc(fc3_out) - # fc 4 - fc4_out = self.fc4(iaf3_fc_out) - iaf4_fc_out = self.iaf4_fc(fc4_out) - - return iaf4_fc_out \ No newline at end of file diff --git a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_2.py b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_2.py deleted file mode 100644 index bb9877df..00000000 --- a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_2.py +++ /dev/null @@ -1,148 +0,0 @@ -import torch.nn as nn -import sinabs.layers as sl -from sinabs.exodus.layers import IAFSqueeze - -class SCNN(nn.Module): - def __init__(self, input_size, nb_classes, batch_size, surrogate_fn, min_v_mem=-0.313, spk_thr=2.0) -> None: - super().__init__() - - self.conv1 = nn.Conv2d(2, 8, 2, 1, bias=False) - self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool1 = sl.SumPool2d(2,2) - self.pool1a = sl.SumPool2d(4,4) - - self.conv2 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool2 = sl.SumPool2d(2,2) - - self.conv3 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool3 = sl.SumPool2d(2,2) - self.pool3a = sl.SumPool2d(4,4) - - self.conv4 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool4 = sl.SumPool2d(2,2) - - self.flat = nn.Flatten() - - flat_s = SCNN.get_flatten_size(input_size) - - self.fc1 = nn.Linear(flat_s, 100, bias=False) - self.iaf1_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc2 = nn.Linear(100, 100, bias=False) - self.iaf2_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc3 = nn.Linear(100, 100, bias=False) - self.iaf3_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc4 = nn.Linear(100, 100, bias=False) - self.iaf4_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc5 = nn.Linear(100, nb_classes, bias=False) - self.iaf5_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - # skip - - self.merge1 = sl.Merge() - self.merge2 = sl.Merge() - - @staticmethod - def get_flatten_size(input_size): - conv1_dims = SCNN.conv2d_output_size(input_size, 8, (2, 2)) - pool1_dims = SCNN.pool_output_size(conv1_dims, 8, (2, 2)) - - conv2_dims = SCNN.conv2d_output_size(pool1_dims, 8, (2, 2)) - pool2_dims = SCNN.pool_output_size(conv2_dims, 8, (2, 2)) - - conv3_dims = SCNN.conv2d_output_size(pool2_dims, 8, (2, 2)) - pool3_dims = SCNN.pool_output_size(conv3_dims, 8, (2, 2)) - - conv4_dims = SCNN.conv2d_output_size(pool3_dims, 8, (2, 2)) - pool4_dims = SCNN.pool_output_size(conv4_dims, 8, (2, 2)) - - return pool4_dims[0]*pool4_dims[1]*pool4_dims[2] - - @staticmethod - def conv2d_output_size(input_size, out_channels, kernel_size, stride=1, padding=0, dilation=1): - input_height, input_width, input_channels = input_size - kernel_height, kernel_width = kernel_size - - output_height = ((input_height + 2 * padding - dilation * (kernel_height - 1) - 1) // stride) + 1 - output_width = ((input_width + 2 * padding - dilation * (kernel_width - 1) - 1) // stride) + 1 - - return (output_height, output_width, out_channels) - - @staticmethod - def pool_output_size(input_size, out_channels, kernel_size, stride=None, padding=0): - input_height, input_width, input_channels = input_size - kernel_height, kernel_width = kernel_size - - if stride is None: - stride = kernel_height - - output_height = ((input_height + 2 * padding - kernel_height) // stride) + 1 - output_width = ((input_width + 2 * padding - kernel_width) // stride) + 1 - - return (output_height, output_width, out_channels) - - def detach_neuron_states(self): - for name, layer in self.named_modules(): - if name != '': - if isinstance(layer, sl.StatefulLayer): - for name, buffer in layer.named_buffers(): - buffer.detach_() - - def init_weights(self): - for name, layer in self.named_modules(): - if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear): - nn.init.xavier_normal_(layer.weight.data) - - def rescale_conv_weights(self, rescale_fn, lambda_): - rescale_fn(self.conv2, [(2,2)], lambda_) - rescale_fn(self.conv3, [(4,4), (2,2)], lambda_) - rescale_fn(self.conv4, [(2,2)], lambda_) - - def forward(self, x): - # conv 1 - con1_out = self.conv1(x) - iaf1_out = self.iaf1(con1_out) - pool1_out = self.pool1(iaf1_out) - pool1a_out = self.pool1a(iaf1_out) - # conv 2 - conv2_out = self.conv2(pool1_out) - iaf2_out = self.iaf2(conv2_out) - pool2_out = self.pool2(iaf2_out) - - merge1_out = self.merge1(pool1a_out, pool2_out) - # conv 3 - conv3_out = self.conv3(merge1_out) - iaf3_out = self.iaf3(conv3_out) - pool3_out = self.pool3(iaf3_out) - pool3a_out = self.pool3a(iaf3_out) - # conv 4 - conv4_out = self.conv4(pool3_out) - iaf4_out = self.iaf4(conv4_out) - pool4_out = self.pool4(iaf4_out) - - merge2_out = self.merge2(pool3a_out, pool4_out) - - flat_out = self.flat(merge2_out) - # fc 1 - fc1_out = self.fc1(flat_out) - iaf1_fc_out = self.iaf1_fc(fc1_out) - # fc 2 - fc2_out = self.fc2(iaf1_fc_out) - iaf2_fc_out = self.iaf2_fc(fc2_out) - # fc 3 - fc3_out = self.fc3(iaf2_fc_out) - iaf3_fc_out = self.iaf3_fc(fc3_out) - # fc 4 - fc4_out = self.fc4(iaf3_fc_out) - iaf4_fc_out = self.iaf4_fc(fc4_out) - # fc 5 - fc5_out = self.fc5(iaf4_fc_out) - iaf5_fc_out = self.iaf5_fc(fc5_out) - - return iaf5_fc_out \ No newline at end of file diff --git a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_3.py b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_3.py deleted file mode 100644 index 0975a51a..00000000 --- a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_3.py +++ /dev/null @@ -1,151 +0,0 @@ -import torch.nn as nn -import sinabs.layers as sl -from sinabs.exodus.layers import IAFSqueeze - -class SCNN(nn.Module): - def __init__(self, input_size, nb_classes, batch_size, surrogate_fn, min_v_mem=-0.313, spk_thr=2.0) -> None: - super().__init__() - - self.conv1 = nn.Conv2d(2, 8, 2, 1, bias=False) - self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool1 = sl.SumPool2d(2,2) - self.pool1a = sl.SumPool2d(4,4) - - self.conv2 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool2 = sl.SumPool2d(2,2) - - self.conv3 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool3 = sl.SumPool2d(2,2) - self.pool3a = sl.SumPool2d(4,4) - - self.conv4 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool4 = sl.SumPool2d(2,2) - - self.flat = nn.Flatten() - - flat_s = SCNN.get_flatten_size(input_size) - - self.fc1 = nn.Linear(flat_s, 100, bias=False) - self.iaf1_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc2 = nn.Linear(100, 100, bias=False) - self.iaf2_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc3 = nn.Linear(100, 100, bias=False) - self.iaf3_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc4 = nn.Linear(100, 100, bias=False) - self.iaf4_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc5 = nn.Linear(100, nb_classes, bias=False) - self.iaf5_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - # skip - - self.merge1 = sl.Merge() - self.merge2 = sl.Merge() - self.merge3 = sl.Merge() - - @staticmethod - def get_flatten_size(input_size): - conv1_dims = SCNN.conv2d_output_size(input_size, 8, (2, 2)) - pool1_dims = SCNN.pool_output_size(conv1_dims, 8, (2, 2)) - - conv2_dims = SCNN.conv2d_output_size(pool1_dims, 8, (2, 2)) - pool2_dims = SCNN.pool_output_size(conv2_dims, 8, (2, 2)) - - conv3_dims = SCNN.conv2d_output_size(pool2_dims, 8, (2, 2)) - pool3_dims = SCNN.pool_output_size(conv3_dims, 8, (2, 2)) - - conv4_dims = SCNN.conv2d_output_size(pool3_dims, 8, (2, 2)) - pool4_dims = SCNN.pool_output_size(conv4_dims, 8, (2, 2)) - - return pool4_dims[0]*pool4_dims[1]*pool4_dims[2] - - @staticmethod - def conv2d_output_size(input_size, out_channels, kernel_size, stride=1, padding=0, dilation=1): - input_height, input_width, input_channels = input_size - kernel_height, kernel_width = kernel_size - - output_height = ((input_height + 2 * padding - dilation * (kernel_height - 1) - 1) // stride) + 1 - output_width = ((input_width + 2 * padding - dilation * (kernel_width - 1) - 1) // stride) + 1 - - return (output_height, output_width, out_channels) - - @staticmethod - def pool_output_size(input_size, out_channels, kernel_size, stride=None, padding=0): - input_height, input_width, input_channels = input_size - kernel_height, kernel_width = kernel_size - - if stride is None: - stride = kernel_height - - output_height = ((input_height + 2 * padding - kernel_height) // stride) + 1 - output_width = ((input_width + 2 * padding - kernel_width) // stride) + 1 - - return (output_height, output_width, out_channels) - - def detach_neuron_states(self): - for name, layer in self.named_modules(): - if name != '': - if isinstance(layer, sl.StatefulLayer): - for name, buffer in layer.named_buffers(): - buffer.detach_() - - def init_weights(self): - for name, layer in self.named_modules(): - if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear): - nn.init.xavier_normal_(layer.weight.data) - - def rescale_conv_weights(self, rescale_fn, lambda_): - rescale_fn(self.conv2, [(2,2)], lambda_) - rescale_fn(self.conv3, [(4,4), (2,2)], lambda_) - rescale_fn(self.conv4, [(2,2)], lambda_) - - def forward(self, x): - # conv 1 - con1_out = self.conv1(x) - iaf1_out = self.iaf1(con1_out) - pool1_out = self.pool1(iaf1_out) - pool1a_out = self.pool1a(iaf1_out) - # conv 2 - conv2_out = self.conv2(pool1_out) - iaf2_out = self.iaf2(conv2_out) - pool2_out = self.pool2(iaf2_out) - - merge1_out = self.merge1(pool1a_out, pool2_out) - # conv 3 - conv3_out = self.conv3(merge1_out) - iaf3_out = self.iaf3(conv3_out) - pool3_out = self.pool3(iaf3_out) - pool3a_out = self.pool3a(iaf3_out) - # conv 4 - conv4_out = self.conv4(pool3_out) - iaf4_out = self.iaf4(conv4_out) - pool4_out = self.pool4(iaf4_out) - - merge2_out = self.merge2(pool3a_out, pool4_out) - - flat_out = self.flat(merge2_out) - # fc 1 - fc1_out = self.fc1(flat_out) - iaf1_fc_out = self.iaf1_fc(fc1_out) - # fc 2 - fc2_out = self.fc2(iaf1_fc_out) - iaf2_fc_out = self.iaf2_fc(fc2_out) - - merge3_out = self.merge3(iaf1_fc_out, iaf2_fc_out) - # fc 3 - fc3_out = self.fc3(merge3_out) - iaf3_fc_out = self.iaf3_fc(fc3_out) - # fc 4 - fc4_out = self.fc4(iaf3_fc_out) - iaf4_fc_out = self.iaf4_fc(fc4_out) - # fc 5 - fc5_out = self.fc5(iaf4_fc_out) - iaf5_fc_out = self.iaf5_fc(fc5_out) - - return iaf5_fc_out \ No newline at end of file diff --git a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_4.py b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_4.py deleted file mode 100644 index 4d07ff37..00000000 --- a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_4.py +++ /dev/null @@ -1,154 +0,0 @@ -import torch.nn as nn -import sinabs.layers as sl -from sinabs.exodus.layers import IAFSqueeze - -class SCNN(nn.Module): - def __init__(self, input_size, nb_classes, batch_size, surrogate_fn, min_v_mem=-0.313, spk_thr=2.0) -> None: - super().__init__() - - self.conv1 = nn.Conv2d(2, 8, 2, 1, bias=False) - self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool1 = sl.SumPool2d(2,2) - self.pool1a = sl.SumPool2d(4,4) - - self.conv2 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool2 = sl.SumPool2d(2,2) - - self.conv3 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool3 = sl.SumPool2d(2,2) - self.pool3a = sl.SumPool2d(4,4) - - self.conv4 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool4 = sl.SumPool2d(2,2) - - self.flat = nn.Flatten() - - flat_s = SCNN.get_flatten_size(input_size) - - self.fc1 = nn.Linear(flat_s, 100, bias=False) - self.iaf1_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc2 = nn.Linear(100, 100, bias=False) - self.iaf2_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc3 = nn.Linear(100, 100, bias=False) - self.iaf3_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc4 = nn.Linear(100, 100, bias=False) - self.iaf4_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc5 = nn.Linear(100, nb_classes, bias=False) - self.iaf5_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - # skip - - self.merge1 = sl.Merge() - self.merge2 = sl.Merge() - self.merge3 = sl.Merge() - self.merge4 = sl.Merge() - - @staticmethod - def get_flatten_size(input_size): - conv1_dims = SCNN.conv2d_output_size(input_size, 8, (2, 2)) - pool1_dims = SCNN.pool_output_size(conv1_dims, 8, (2, 2)) - - conv2_dims = SCNN.conv2d_output_size(pool1_dims, 8, (2, 2)) - pool2_dims = SCNN.pool_output_size(conv2_dims, 8, (2, 2)) - - conv3_dims = SCNN.conv2d_output_size(pool2_dims, 8, (2, 2)) - pool3_dims = SCNN.pool_output_size(conv3_dims, 8, (2, 2)) - - conv4_dims = SCNN.conv2d_output_size(pool3_dims, 8, (2, 2)) - pool4_dims = SCNN.pool_output_size(conv4_dims, 8, (2, 2)) - - return pool4_dims[0]*pool4_dims[1]*pool4_dims[2] - - @staticmethod - def conv2d_output_size(input_size, out_channels, kernel_size, stride=1, padding=0, dilation=1): - input_height, input_width, input_channels = input_size - kernel_height, kernel_width = kernel_size - - output_height = ((input_height + 2 * padding - dilation * (kernel_height - 1) - 1) // stride) + 1 - output_width = ((input_width + 2 * padding - dilation * (kernel_width - 1) - 1) // stride) + 1 - - return (output_height, output_width, out_channels) - - @staticmethod - def pool_output_size(input_size, out_channels, kernel_size, stride=None, padding=0): - input_height, input_width, input_channels = input_size - kernel_height, kernel_width = kernel_size - - if stride is None: - stride = kernel_height - - output_height = ((input_height + 2 * padding - kernel_height) // stride) + 1 - output_width = ((input_width + 2 * padding - kernel_width) // stride) + 1 - - return (output_height, output_width, out_channels) - - def detach_neuron_states(self): - for name, layer in self.named_modules(): - if name != '': - if isinstance(layer, sl.StatefulLayer): - for name, buffer in layer.named_buffers(): - buffer.detach_() - - def init_weights(self): - for name, layer in self.named_modules(): - if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear): - nn.init.xavier_normal_(layer.weight.data) - - def rescale_conv_weights(self, rescale_fn, lambda_): - rescale_fn(self.conv2, [(2,2)], lambda_) - rescale_fn(self.conv3, [(4,4), (2,2)], lambda_) - rescale_fn(self.conv4, [(2,2)], lambda_) - - def forward(self, x): - # conv 1 - con1_out = self.conv1(x) - iaf1_out = self.iaf1(con1_out) - pool1_out = self.pool1(iaf1_out) - pool1a_out = self.pool1a(iaf1_out) - # conv 2 - conv2_out = self.conv2(pool1_out) - iaf2_out = self.iaf2(conv2_out) - pool2_out = self.pool2(iaf2_out) - - merge1_out = self.merge1(pool1a_out, pool2_out) - # conv 3 - conv3_out = self.conv3(merge1_out) - iaf3_out = self.iaf3(conv3_out) - pool3_out = self.pool3(iaf3_out) - pool3a_out = self.pool3a(iaf3_out) - # conv 4 - conv4_out = self.conv4(pool3_out) - iaf4_out = self.iaf4(conv4_out) - pool4_out = self.pool4(iaf4_out) - - merge2_out = self.merge2(pool3a_out, pool4_out) - - flat_out = self.flat(merge2_out) - # fc 1 - fc1_out = self.fc1(flat_out) - iaf1_fc_out = self.iaf1_fc(fc1_out) - # fc 2 - fc2_out = self.fc2(iaf1_fc_out) - iaf2_fc_out = self.iaf2_fc(fc2_out) - - merge3_out = self.merge3(iaf1_fc_out, iaf2_fc_out) - # fc 3 - fc3_out = self.fc3(merge3_out) - iaf3_fc_out = self.iaf3_fc(fc3_out) - # fc 4 - fc4_out = self.fc4(iaf3_fc_out) - iaf4_fc_out = self.iaf4_fc(fc4_out) - - merge4_out = self.merge4(iaf3_fc_out, iaf4_fc_out) - # fc 5 - fc5_out = self.fc5(merge4_out) - iaf5_fc_out = self.iaf5_fc(fc5_out) - - return iaf5_fc_out \ No newline at end of file diff --git a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_5.py b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_5.py deleted file mode 100644 index 8ba45649..00000000 --- a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_5.py +++ /dev/null @@ -1,143 +0,0 @@ -import torch.nn as nn -import sinabs.layers as sl -from sinabs.exodus.layers import IAFSqueeze - -class SCNN(nn.Module): - def __init__(self, input_size, nb_classes, batch_size, surrogate_fn, min_v_mem=-0.313, spk_thr=2.0) -> None: - super().__init__() - - self.conv1 = nn.Conv2d(2, 8, 2, 1, bias=False) - self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool1 = sl.SumPool2d(2,2) - self.pool1a = sl.SumPool2d(8,8) - - self.conv2 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool2 = sl.SumPool2d(2,2) - - self.conv3 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool3 = sl.SumPool2d(2,2) - - self.conv4 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool4 = sl.SumPool2d(2,2) - - self.flat = nn.Flatten() - - flat_s = SCNN.get_flatten_size(input_size) - - self.fc1 = nn.Linear(flat_s, 100, bias=False) - self.iaf1_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc2 = nn.Linear(100, 100, bias=False) - self.iaf2_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc3 = nn.Linear(100, 100, bias=False) - self.iaf3_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc4 = nn.Linear(100, 100, bias=False) - self.iaf4_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc5 = nn.Linear(100, nb_classes, bias=False) - self.iaf5_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - # skip - - self.merge1 = sl.Merge() - - @staticmethod - def get_flatten_size(input_size): - conv1_dims = SCNN.conv2d_output_size(input_size, 8, (2, 2)) - pool1_dims = SCNN.pool_output_size(conv1_dims, 8, (2, 2)) - - conv2_dims = SCNN.conv2d_output_size(pool1_dims, 8, (2, 2)) - pool2_dims = SCNN.pool_output_size(conv2_dims, 8, (2, 2)) - - conv3_dims = SCNN.conv2d_output_size(pool2_dims, 8, (2, 2)) - pool3_dims = SCNN.pool_output_size(conv3_dims, 8, (2, 2)) - - conv4_dims = SCNN.conv2d_output_size(pool3_dims, 8, (2, 2)) - pool4_dims = SCNN.pool_output_size(conv4_dims, 8, (2, 2)) - - return pool4_dims[0]*pool4_dims[1]*pool4_dims[2] - - @staticmethod - def conv2d_output_size(input_size, out_channels, kernel_size, stride=1, padding=0, dilation=1): - input_height, input_width, input_channels = input_size - kernel_height, kernel_width = kernel_size - - output_height = ((input_height + 2 * padding - dilation * (kernel_height - 1) - 1) // stride) + 1 - output_width = ((input_width + 2 * padding - dilation * (kernel_width - 1) - 1) // stride) + 1 - - return (output_height, output_width, out_channels) - - @staticmethod - def pool_output_size(input_size, out_channels, kernel_size, stride=None, padding=0): - input_height, input_width, input_channels = input_size - kernel_height, kernel_width = kernel_size - - if stride is None: - stride = kernel_height - - output_height = ((input_height + 2 * padding - kernel_height) // stride) + 1 - output_width = ((input_width + 2 * padding - kernel_width) // stride) + 1 - - return (output_height, output_width, out_channels) - - def detach_neuron_states(self): - for name, layer in self.named_modules(): - if name != '': - if isinstance(layer, sl.StatefulLayer): - for name, buffer in layer.named_buffers(): - buffer.detach_() - - def init_weights(self): - for name, layer in self.named_modules(): - if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear): - nn.init.xavier_normal_(layer.weight.data) - - def rescale_conv_weights(self, rescale_fn, lambda_): - rescale_fn(self.conv2, [(2,2)], lambda_) - rescale_fn(self.conv3, [(2,2)], lambda_) - rescale_fn(self.conv4, [(8,8), (2,2)], lambda_) - - def forward(self, x): - # conv 1 - con1_out = self.conv1(x) - iaf1_out = self.iaf1(con1_out) - pool1_out = self.pool1(iaf1_out) - pool1a_out = self.pool1a(iaf1_out) - # conv 2 - conv2_out = self.conv2(pool1_out) - iaf2_out = self.iaf2(conv2_out) - pool2_out = self.pool2(iaf2_out) - # conv 3 - conv3_out = self.conv3(pool2_out) - iaf3_out = self.iaf3(conv3_out) - pool3_out = self.pool3(iaf3_out) - - merge1_out = self.merge1(pool1a_out, pool3_out) - # conv 4 - conv4_out = self.conv4(merge1_out) - iaf4_out = self.iaf4(conv4_out) - pool4_out = self.pool4(iaf4_out) - - flat_out = self.flat(pool4_out) - # fc 1 - fc1_out = self.fc1(flat_out) - iaf1_fc_out = self.iaf1_fc(fc1_out) - # fc 2 - fc2_out = self.fc2(iaf1_fc_out) - iaf2_fc_out = self.iaf2_fc(fc2_out) - # fc 3 - fc3_out = self.fc3(iaf2_fc_out) - iaf3_fc_out = self.iaf3_fc(fc3_out) - # fc 4 - fc4_out = self.fc4(iaf3_fc_out) - iaf4_fc_out = self.iaf4_fc(fc4_out) - # fc 5 - fc5_out = self.fc5(iaf4_fc_out) - iaf5_fc_out = self.iaf5_fc(fc5_out) - - return iaf5_fc_out \ No newline at end of file diff --git a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_6.py b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_6.py deleted file mode 100644 index 5f761619..00000000 --- a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_6.py +++ /dev/null @@ -1,146 +0,0 @@ -import torch.nn as nn -import sinabs.layers as sl -from sinabs.exodus.layers import IAFSqueeze - -class SCNN(nn.Module): - def __init__(self, input_size, nb_classes, batch_size, surrogate_fn, min_v_mem=-0.313, spk_thr=2.0) -> None: - super().__init__() - - self.conv1 = nn.Conv2d(2, 8, 2, 1, bias=False) - self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool1 = sl.SumPool2d(2,2) - self.pool1a = sl.SumPool2d(8,8) - - self.conv2 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool2 = sl.SumPool2d(2,2) - - self.conv3 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool3 = sl.SumPool2d(2,2) - - self.conv4 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool4 = sl.SumPool2d(2,2) - - self.flat = nn.Flatten() - - flat_s = SCNN.get_flatten_size(input_size) - - self.fc1 = nn.Linear(flat_s, 100, bias=False) - self.iaf1_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc2 = nn.Linear(100, 100, bias=False) - self.iaf2_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc3 = nn.Linear(100, 100, bias=False) - self.iaf3_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc4 = nn.Linear(100, 100, bias=False) - self.iaf4_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc5 = nn.Linear(100, nb_classes, bias=False) - self.iaf5_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - # skip - - self.merge1 = sl.Merge() - self.merge2 = sl.Merge() - - @staticmethod - def get_flatten_size(input_size): - conv1_dims = SCNN.conv2d_output_size(input_size, 8, (2, 2)) - pool1_dims = SCNN.pool_output_size(conv1_dims, 8, (2, 2)) - - conv2_dims = SCNN.conv2d_output_size(pool1_dims, 8, (2, 2)) - pool2_dims = SCNN.pool_output_size(conv2_dims, 8, (2, 2)) - - conv3_dims = SCNN.conv2d_output_size(pool2_dims, 8, (2, 2)) - pool3_dims = SCNN.pool_output_size(conv3_dims, 8, (2, 2)) - - conv4_dims = SCNN.conv2d_output_size(pool3_dims, 8, (2, 2)) - pool4_dims = SCNN.pool_output_size(conv4_dims, 8, (2, 2)) - - return pool4_dims[0]*pool4_dims[1]*pool4_dims[2] - - @staticmethod - def conv2d_output_size(input_size, out_channels, kernel_size, stride=1, padding=0, dilation=1): - input_height, input_width, input_channels = input_size - kernel_height, kernel_width = kernel_size - - output_height = ((input_height + 2 * padding - dilation * (kernel_height - 1) - 1) // stride) + 1 - output_width = ((input_width + 2 * padding - dilation * (kernel_width - 1) - 1) // stride) + 1 - - return (output_height, output_width, out_channels) - - @staticmethod - def pool_output_size(input_size, out_channels, kernel_size, stride=None, padding=0): - input_height, input_width, input_channels = input_size - kernel_height, kernel_width = kernel_size - - if stride is None: - stride = kernel_height - - output_height = ((input_height + 2 * padding - kernel_height) // stride) + 1 - output_width = ((input_width + 2 * padding - kernel_width) // stride) + 1 - - return (output_height, output_width, out_channels) - - def detach_neuron_states(self): - for name, layer in self.named_modules(): - if name != '': - if isinstance(layer, sl.StatefulLayer): - for name, buffer in layer.named_buffers(): - buffer.detach_() - - def init_weights(self): - for name, layer in self.named_modules(): - if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear): - nn.init.xavier_normal_(layer.weight.data) - - def rescale_conv_weights(self, rescale_fn, lambda_): - rescale_fn(self.conv2, [(2,2)], lambda_) - rescale_fn(self.conv3, [(2,2)], lambda_) - rescale_fn(self.conv4, [(8,8), (2,2)], lambda_) - - def forward(self, x): - # conv 1 - con1_out = self.conv1(x) - iaf1_out = self.iaf1(con1_out) - pool1_out = self.pool1(iaf1_out) - pool1a_out = self.pool1a(iaf1_out) - # conv 2 - conv2_out = self.conv2(pool1_out) - iaf2_out = self.iaf2(conv2_out) - pool2_out = self.pool2(iaf2_out) - # conv 3 - conv3_out = self.conv3(pool2_out) - iaf3_out = self.iaf3(conv3_out) - pool3_out = self.pool3(iaf3_out) - - merge1_out = self.merge1(pool1a_out, pool3_out) - # conv 4 - conv4_out = self.conv4(merge1_out) - iaf4_out = self.iaf4(conv4_out) - pool4_out = self.pool4(iaf4_out) - - flat_out = self.flat(pool4_out) - # fc 1 - fc1_out = self.fc1(flat_out) - iaf1_fc_out = self.iaf1_fc(fc1_out) - # fc 2 - fc2_out = self.fc2(iaf1_fc_out) - iaf2_fc_out = self.iaf2_fc(fc2_out) - # fc 3 - fc3_out = self.fc3(iaf2_fc_out) - iaf3_fc_out = self.iaf3_fc(fc3_out) - - merge2_out = self.merge2(iaf1_fc_out, iaf3_fc_out) - # fc 4 - fc4_out = self.fc4(merge2_out) - iaf4_fc_out = self.iaf4_fc(fc4_out) - # fc 5 - fc5_out = self.fc5(iaf4_fc_out) - iaf5_fc_out = self.iaf5_fc(fc5_out) - - return iaf5_fc_out \ No newline at end of file diff --git a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_7.py b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_7.py deleted file mode 100644 index 21039b28..00000000 --- a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_7.py +++ /dev/null @@ -1,146 +0,0 @@ -import torch.nn as nn -import sinabs.layers as sl -from sinabs.exodus.layers import IAFSqueeze - -class SCNN(nn.Module): - def __init__(self, input_size, nb_classes, batch_size, surrogate_fn, min_v_mem=-0.313, spk_thr=2.0) -> None: - super().__init__() - - self.conv1 = nn.Conv2d(2, 8, 2, 1, bias=False) - self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool1 = sl.SumPool2d(2,2) - self.pool1a = sl.SumPool2d(4,4) - self.pool1b = sl.SumPool2d(8,8) - - self.conv2 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool2 = sl.SumPool2d(2,2) - - self.conv3 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool3 = sl.SumPool2d(2,2) - - self.conv4 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool4 = sl.SumPool2d(2,2) - - self.flat = nn.Flatten() - - flat_s = SCNN.get_flatten_size(input_size) - - self.fc1 = nn.Linear(flat_s, 100, bias=False) - self.iaf1_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc2 = nn.Linear(100, 100, bias=False) - self.iaf2_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc3 = nn.Linear(100, 100, bias=False) - self.iaf3_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc4 = nn.Linear(100, 100, bias=False) - self.iaf4_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc5 = nn.Linear(100, nb_classes, bias=False) - self.iaf5_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - # skip - - self.merge1a = sl.Merge() - self.merge1b = sl.Merge() - - @staticmethod - def get_flatten_size(input_size): - conv1_dims = SCNN.conv2d_output_size(input_size, 8, (2, 2)) - pool1_dims = SCNN.pool_output_size(conv1_dims, 8, (2, 2)) - - conv2_dims = SCNN.conv2d_output_size(pool1_dims, 8, (2, 2)) - pool2_dims = SCNN.pool_output_size(conv2_dims, 8, (2, 2)) - - conv3_dims = SCNN.conv2d_output_size(pool2_dims, 8, (2, 2)) - pool3_dims = SCNN.pool_output_size(conv3_dims, 8, (2, 2)) - - conv4_dims = SCNN.conv2d_output_size(pool3_dims, 8, (2, 2)) - pool4_dims = SCNN.pool_output_size(conv4_dims, 8, (2, 2)) - - return pool4_dims[0]*pool4_dims[1]*pool4_dims[2] - - @staticmethod - def conv2d_output_size(input_size, out_channels, kernel_size, stride=1, padding=0, dilation=1): - input_height, input_width, input_channels = input_size - kernel_height, kernel_width = kernel_size - - output_height = ((input_height + 2 * padding - dilation * (kernel_height - 1) - 1) // stride) + 1 - output_width = ((input_width + 2 * padding - dilation * (kernel_width - 1) - 1) // stride) + 1 - - return (output_height, output_width, out_channels) - - @staticmethod - def pool_output_size(input_size, out_channels, kernel_size, stride=None, padding=0): - input_height, input_width, input_channels = input_size - kernel_height, kernel_width = kernel_size - - if stride is None: - stride = kernel_height - - output_height = ((input_height + 2 * padding - kernel_height) // stride) + 1 - output_width = ((input_width + 2 * padding - kernel_width) // stride) + 1 - - return (output_height, output_width, out_channels) - - def detach_neuron_states(self): - for name, layer in self.named_modules(): - if name != '': - if isinstance(layer, sl.StatefulLayer): - for name, buffer in layer.named_buffers(): - buffer.detach_() - - def init_weights(self): - for name, layer in self.named_modules(): - if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear): - nn.init.xavier_normal_(layer.weight.data) - - def rescale_conv_weights(self, rescale_fn, lambda_): - rescale_fn(self.conv2, [(2,2)], lambda_) - rescale_fn(self.conv3, [(4,4), (2,2)], lambda_) - rescale_fn(self.conv4, [(8,8), (2,2)], lambda_) - - def forward(self, x): - # conv 1 - con1_out = self.conv1(x) - iaf1_out = self.iaf1(con1_out) - pool1_out = self.pool1(iaf1_out) - pool1a_out = self.pool1a(iaf1_out) - pool1b_out = self.pool1b(iaf1_out) - # conv 2 - conv2_out = self.conv2(pool1_out) - iaf2_out = self.iaf2(conv2_out) - pool2_out = self.pool2(iaf2_out) - # conv 3 - merge_1a_out = self.merge1a(pool1a_out, pool2_out) - conv3_out = self.conv3(merge_1a_out) - iaf3_out = self.iaf3(conv3_out) - pool3_out = self.pool3(iaf3_out) - # conv 4 - merge_1b_out = self.merge1b(pool1b_out, pool3_out) - conv4_out = self.conv4(merge_1b_out) - iaf4_out = self.iaf4(conv4_out) - pool4_out = self.pool4(iaf4_out) - - flat_out = self.flat(pool4_out) - # fc 1 - fc1_out = self.fc1(flat_out) - iaf1_fc_out = self.iaf1_fc(fc1_out) - # fc 2 - fc2_out = self.fc2(iaf1_fc_out) - iaf2_fc_out = self.iaf2_fc(fc2_out) - # fc 3 - fc3_out = self.fc3(iaf2_fc_out) - iaf3_fc_out = self.iaf3_fc(fc3_out) - # fc 4 - fc4_out = self.fc4(iaf3_fc_out) - iaf4_fc_out = self.iaf4_fc(fc4_out) - # fc 5 - fc5_out = self.fc5(iaf4_fc_out) - iaf5_fc_out = self.iaf5_fc(fc5_out) - - return iaf5_fc_out \ No newline at end of file diff --git a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_8.py b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_8.py deleted file mode 100644 index 9086a3e3..00000000 --- a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_8.py +++ /dev/null @@ -1,151 +0,0 @@ -import torch.nn as nn -import sinabs.layers as sl -from sinabs.exodus.layers import IAFSqueeze - -class SCNN(nn.Module): - def __init__(self, input_size, nb_classes, batch_size, surrogate_fn, min_v_mem=-0.313, spk_thr=2.0) -> None: - super().__init__() - - self.conv1 = nn.Conv2d(2, 8, 2, 1, bias=False) - self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool1 = sl.SumPool2d(2,2) - self.pool1a = sl.SumPool2d(4,4) - self.pool1b = sl.SumPool2d(8,8) - - self.conv2 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool2 = sl.SumPool2d(2,2) - - self.conv3 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool3 = sl.SumPool2d(2,2) - - self.conv4 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool4 = sl.SumPool2d(2,2) - - self.flat = nn.Flatten() - - flat_s = SCNN.get_flatten_size(input_size) - - self.fc1 = nn.Linear(flat_s, 100, bias=False) - self.iaf1_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc2 = nn.Linear(100, 100, bias=False) - self.iaf2_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc3 = nn.Linear(100, 100, bias=False) - self.iaf3_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc4 = nn.Linear(100, 100, bias=False) - self.iaf4_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc5 = nn.Linear(100, nb_classes, bias=False) - self.iaf5_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - # skip - - self.merge1a = sl.Merge() - self.merge1b = sl.Merge() - - self.merge_fc1a = sl.Merge() - self.merge_fc1b = sl.Merge() - - @staticmethod - def get_flatten_size(input_size): - conv1_dims = SCNN.conv2d_output_size(input_size, 8, (2, 2)) - pool1_dims = SCNN.pool_output_size(conv1_dims, 8, (2, 2)) - - conv2_dims = SCNN.conv2d_output_size(pool1_dims, 8, (2, 2)) - pool2_dims = SCNN.pool_output_size(conv2_dims, 8, (2, 2)) - - conv3_dims = SCNN.conv2d_output_size(pool2_dims, 8, (2, 2)) - pool3_dims = SCNN.pool_output_size(conv3_dims, 8, (2, 2)) - - conv4_dims = SCNN.conv2d_output_size(pool3_dims, 8, (2, 2)) - pool4_dims = SCNN.pool_output_size(conv4_dims, 8, (2, 2)) - - return pool4_dims[0]*pool4_dims[1]*pool4_dims[2] - - @staticmethod - def conv2d_output_size(input_size, out_channels, kernel_size, stride=1, padding=0, dilation=1): - input_height, input_width, input_channels = input_size - kernel_height, kernel_width = kernel_size - - output_height = ((input_height + 2 * padding - dilation * (kernel_height - 1) - 1) // stride) + 1 - output_width = ((input_width + 2 * padding - dilation * (kernel_width - 1) - 1) // stride) + 1 - - return (output_height, output_width, out_channels) - - @staticmethod - def pool_output_size(input_size, out_channels, kernel_size, stride=None, padding=0): - input_height, input_width, input_channels = input_size - kernel_height, kernel_width = kernel_size - - if stride is None: - stride = kernel_height - - output_height = ((input_height + 2 * padding - kernel_height) // stride) + 1 - output_width = ((input_width + 2 * padding - kernel_width) // stride) + 1 - - return (output_height, output_width, out_channels) - - def detach_neuron_states(self): - for name, layer in self.named_modules(): - if name != '': - if isinstance(layer, sl.StatefulLayer): - for name, buffer in layer.named_buffers(): - buffer.detach_() - - def init_weights(self): - for name, layer in self.named_modules(): - if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear): - nn.init.xavier_normal_(layer.weight.data) - - def rescale_conv_weights(self, rescale_fn, lambda_): - rescale_fn(self.conv2, [(2,2)], lambda_) - rescale_fn(self.conv3, [(4,4), (2,2)], lambda_) - rescale_fn(self.conv4, [(8,8), (2,2)], lambda_) - - def forward(self, x): - # conv 1 - con1_out = self.conv1(x) - iaf1_out = self.iaf1(con1_out) - pool1_out = self.pool1(iaf1_out) - pool1a_out = self.pool1a(iaf1_out) - pool1b_out = self.pool1b(iaf1_out) - # conv 2 - conv2_out = self.conv2(pool1_out) - iaf2_out = self.iaf2(conv2_out) - pool2_out = self.pool2(iaf2_out) - # conv 3 - merge_1a_out = self.merge1a(pool1a_out, pool2_out) - conv3_out = self.conv3(merge_1a_out) - iaf3_out = self.iaf3(conv3_out) - pool3_out = self.pool3(iaf3_out) - # conv 4 - merge_1b_out = self.merge1b(pool1b_out, pool3_out) - conv4_out = self.conv4(merge_1b_out) - iaf4_out = self.iaf4(conv4_out) - pool4_out = self.pool4(iaf4_out) - - flat_out = self.flat(pool4_out) - # fc 1 - fc1_out = self.fc1(flat_out) - iaf1_fc_out = self.iaf1_fc(fc1_out) - # fc 2 - fc2_out = self.fc2(iaf1_fc_out) - iaf2_fc_out = self.iaf2_fc(fc2_out) - # fc 3 - merge_fc1a_out = self.merge_fc1a(iaf1_fc_out, iaf2_fc_out) - fc3_out = self.fc3(merge_fc1a_out) - iaf3_fc_out = self.iaf3_fc(fc3_out) - # fc 4 - merge_fc1b_out = self.merge_fc1b(iaf1_fc_out, iaf3_fc_out) - fc4_out = self.fc4(merge_fc1b_out) - iaf4_fc_out = self.iaf4_fc(fc4_out) - # fc 5 - fc5_out = self.fc5(iaf4_fc_out) - iaf5_fc_out = self.iaf5_fc(fc5_out) - - return iaf5_fc_out \ No newline at end of file diff --git a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_9.py b/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_9.py deleted file mode 100644 index 7ed5ca4d..00000000 --- a/tests/test_nonsequential/using_SumPool2d/models/ResSCNN_9.py +++ /dev/null @@ -1,124 +0,0 @@ -import torch.nn as nn -import sinabs.layers as sl -from sinabs.exodus.layers import IAFSqueeze - -class SCNN(nn.Module): - def __init__(self, input_size, nb_classes, batch_size, surrogate_fn, min_v_mem=-0.313, spk_thr=2.0): - super().__init__() - - self.conv1 = nn.Conv2d(2, 8, 2, 1, bias=False) - self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.conv2 = nn.Conv2d(8, 8, 3, 1, 1, bias=False) - self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.conv3 = nn.Conv2d(8, 8, 3, 1, 1, bias=False) - self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool3a = sl.SumPool2d(2,2) - - self.conv4 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool4 = sl.SumPool2d(2,2) - self.pool4a = sl.SumPool2d(4,4) - - self.conv5 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf5 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool5 = sl.SumPool2d(2,2) - self.pool5a = sl.SumPool2d(2,2) - - self.conv6 = nn.Conv2d(8, 8, 3, 1, 1, bias=False) - self.iaf6 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool6a = sl.SumPool2d(2,2) - - self.conv7 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf7 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool7 = sl.SumPool2d(2,2) - - self.conv8 = nn.Conv2d(8, 8, 2, 1, bias=False) - self.iaf8 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool8 = sl.SumPool2d(2,2) - - self.flat = nn.Flatten() - - self.fc_out = nn.Linear(392, nb_classes, bias=False) - self.iaf_fc_out = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - # skip - - self.merge1 = sl.Merge() - self.merge2 = sl.Merge() - self.merge3 = sl.Merge() - self.merge4 = sl.Merge() - self.merge5 = sl.Merge() - self.merge6 = sl.Merge() - - def detach_neuron_states(self): - for name, layer in self.named_modules(): - if name != '': - if isinstance(layer, sl.StatefulLayer): - for name, buffer in layer.named_buffers(): - buffer.detach_() - - def init_weights(self): - for name, layer in self.named_modules(): - if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear): - nn.init.xavier_normal_(layer.weight.data) - - def rescale_conv_weights(self, rescale_fn, lambda_): - rescale_fn(self.conv5, [(2,2), (2,2)], lambda_) - rescale_fn(self.conv6, [(4,4), (2,2)], lambda_) - rescale_fn(self.conv7, [(2,2)], lambda_) - rescale_fn(self.conv8, [(2,2), (2,2)], lambda_) - - def forward(self, x): - # -- conv block 1 --- - con1_out = self.conv1(x) - iaf1_out = self.iaf1(con1_out) - # -- conv block 2 --- - conv2_out = self.conv2(iaf1_out) - iaf2_out = self.iaf2(conv2_out) # to CONV 4 - # -- conv block 3 --- - #print(iaf1_out.shape, iaf2_out.shape) - merge1_out = self.merge1(iaf1_out, iaf2_out) - conv3_out = self.conv3(merge1_out) - iaf3_out = self.iaf3(conv3_out) - pool3a_out = self.pool3a(iaf3_out) # to CONV 5 - # -- conv block 4 --- - #print(iaf2_out.shape, iaf3_out.shape) - merge2_out = self.merge2(iaf2_out, iaf3_out) - conv4_out = self.conv4(merge2_out) - iaf4_out = self.iaf4(conv4_out) - pool4_out = self.pool4(iaf4_out) - pool4a_out = self.pool4a(iaf4_out) # to CONV 6 - # -- conv block 5 --- - #print(pool3a_out.shape, pool4_out.shape) - merge3_out = self.merge3(pool3a_out, pool4_out) - conv5_out = self.conv5(merge3_out) - iaf5_out = self.iaf5(conv5_out) - pool5_out = self.pool5(iaf5_out) - pool5a_out = self.pool5a(iaf5_out) # to CONV 7 - # -- conv block 6 --- - #print(pool4a_out.shape, pool5_out.shape) - merge4_out = self.merge4(pool4a_out, pool5_out) - conv6_out = self.conv6(merge4_out) - iaf6_out = self.iaf6(conv6_out) - pool6a_out = self.pool6a(iaf6_out) # to CONV 8 - # -- conv block 7 --- - #print(pool5a_out.shape, iaf6_out.shape) - merge5_out = self.merge5(pool5a_out, iaf6_out) - conv7_out = self.conv7(merge5_out) - iaf7_out = self.iaf7(conv7_out) - pool7_out = self.pool7(iaf7_out) - # -- conv block 8 --- - #print(pool6a_out.shape, pool7_out.shape) - merge6_out = self.merge6(pool6a_out, pool7_out) - conv8_out = self.conv8(merge6_out) - iaf8_out = self.iaf8(conv8_out) - pool8_out = self.pool8(iaf8_out) - # -- output -- - flat = self.flat(pool8_out) - #print(flat.shape) - fc_out = self.fc_out(flat) - iaf_fc_out = self.iaf_fc_out(fc_out) - - return iaf_fc_out \ No newline at end of file diff --git a/tests/test_nonsequential/using_SumPool2d/models/SCNN.py b/tests/test_nonsequential/using_SumPool2d/models/SCNN.py deleted file mode 100644 index 5e3d689a..00000000 --- a/tests/test_nonsequential/using_SumPool2d/models/SCNN.py +++ /dev/null @@ -1,130 +0,0 @@ -import torch.nn as nn -import sinabs.layers as sl -from sinabs.exodus.layers import IAFSqueeze - -class SCNN(nn.Module): - def __init__(self, input_size, nb_classes, batch_size, surrogate_fn, min_v_mem=-1.0, spk_thr=1.0) -> None: - super().__init__() - - self.conv1 = nn.Conv2d(2, 1, 2, 1, bias=False) - self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool1 = sl.SumPool2d(2,2) - - self.conv2 = nn.Conv2d(1, 8, 2, 1, bias=False) - self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool2 = sl.SumPool2d(2,2) - - self.conv3 = nn.Conv2d(8, 16, 2, 1, bias=False) - self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool3 = sl.SumPool2d(2,2) - - self.conv4 = nn.Conv2d(16, 16, 2, 1, bias=False) - self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - self.pool4 = sl.SumPool2d(2,2) - - self.flat = nn.Flatten() - - flat_s = SCNN.get_flatten_size(input_size) - - self.fc1 = nn.Linear(flat_s, 1024, bias=False) - self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc2 = nn.Linear(1024, 256, bias=False) - self.iaf5 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc3 = nn.Linear(256, 128, bias=False) - self.iaf6 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc4 = nn.Linear(128, 64, bias=False) - self.iaf7 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - self.fc5 = nn.Linear(64, nb_classes, bias=False) - self.iaf8 = IAFSqueeze(batch_size=batch_size, min_v_mem=min_v_mem, surrogate_grad_fn=surrogate_fn, spike_threshold=spk_thr) - - @staticmethod - def get_flatten_size(input_size): - conv1_dims = SCNN.conv2d_output_size(input_size, 1, (2, 2)) - pool1_dims = SCNN.pool_output_size(conv1_dims, 1, (2, 2)) - - conv2_dims = SCNN.conv2d_output_size(pool1_dims, 8, (2, 2)) - pool2_dims = SCNN.pool_output_size(conv2_dims, 8, (2, 2)) - - conv3_dims = SCNN.conv2d_output_size(pool2_dims, 16, (2, 2)) - pool3_dims = SCNN.pool_output_size(conv3_dims, 16, (2, 2)) - - conv4_dims = SCNN.conv2d_output_size(pool3_dims, 16, (2, 2)) - pool4_dims = SCNN.pool_output_size(conv4_dims, 16, (2, 2)) - - return pool4_dims[0]*pool4_dims[1]*pool4_dims[2] - - @staticmethod - def conv2d_output_size(input_size, out_channels, kernel_size, stride=1, padding=0, dilation=1): - input_height, input_width, input_channels = input_size - kernel_height, kernel_width = kernel_size - - output_height = ((input_height + 2 * padding - dilation * (kernel_height - 1) - 1) // stride) + 1 - output_width = ((input_width + 2 * padding - dilation * (kernel_width - 1) - 1) // stride) + 1 - - return (output_height, output_width, out_channels) - - @staticmethod - def pool_output_size(input_size, out_channels, kernel_size, stride=None, padding=0): - input_height, input_width, input_channels = input_size - kernel_height, kernel_width = kernel_size - - if stride is None: - stride = kernel_height - - output_height = ((input_height + 2 * padding - kernel_height) // stride) + 1 - output_width = ((input_width + 2 * padding - kernel_width) // stride) + 1 - - return (output_height, output_width, out_channels) - - def detach_neuron_states(self): - for name, layer in self.named_modules(): - if name != '': - if isinstance(layer, sl.StatefulLayer): - for name, buffer in layer.named_buffers(): - buffer.detach_() - - def init_weights(self): - for name, layer in self.named_modules(): - if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear): - nn.init.xavier_normal_(layer.weight.data) - - def forward(self, x): - - con1_out = self.conv1(x) - iaf1_out = self.iaf1(con1_out) - pool1_out = self.pool1(iaf1_out) - - conv2_out = self.conv2(pool1_out) - iaf2_out = self.iaf2(conv2_out) - pool2_out = self.pool2(iaf2_out) - - conv3_out = self.conv3(pool2_out) - iaf3_out = self.iaf3(conv3_out) - pool3_out = self.pool3(iaf3_out) - - conv4_out = self.conv4(pool3_out) - iaf4_out = self.iaf4(conv4_out) - pool4_out = self.pool4(iaf4_out) - - flat_out = self.flat(pool4_out) - - fc1_out = self.fc1(flat_out) - iaf4_out = self.iaf4(fc1_out) - - fc2_out = self.fc2(iaf4_out) - iaf5_out = self.iaf5(fc2_out) - - fc3_out = self.fc3(iaf5_out) - iaf6_out = self.iaf6(fc3_out) - - fc4_out = self.fc4(iaf6_out) - iaf7_out = self.iaf7(fc4_out) - - fc5_out = self.fc5(iaf7_out) - iaf8_out = self.iaf8(fc5_out) - - return iaf8_out \ No newline at end of file diff --git a/tests/test_nonsequential/utils/train_test_fn.py b/tests/test_nonsequential/utils/train_test_fn.py deleted file mode 100644 index d166fdff..00000000 --- a/tests/test_nonsequential/utils/train_test_fn.py +++ /dev/null @@ -1,274 +0,0 @@ -from tqdm.notebook import tqdm -import torch -from tonic.datasets.dvsgesture import DVSGesture -from tonic.datasets.nmnist import NMNIST -from tonic.transforms import ToFrame -import numpy as np -from torch.utils.data import Subset - -def training_loop(device, nb_time_steps, batch_size, feature_map_size, dataloader_train, model, loss_fn, optimizer, epochs, dataloader_test): - epochs_y = [] - epochs_x = [] - epochs_acc = [] - model.train() - - for e in range(epochs): - losses = [] - batches = [] - batch_count = 0 - train_p_bar = tqdm(dataloader_train) - - for X, y in train_p_bar: - # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width] - X = X.reshape(-1, feature_map_size[2], feature_map_size[0], feature_map_size[1]).to(dtype=torch.float, device=device) - y = y.to(dtype=torch.long, device=device) - - # forward - pred = model(X) - - # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes] - pred = pred.reshape(batch_size, nb_time_steps, -1) - - # accumulate all time-steps output for final prediction - pred = pred.sum(dim = 1) - loss = loss_fn(pred, y) - - # gradient update - optimizer.zero_grad() - loss.backward() - optimizer.step() - - # detach the neuron states and activations from current computation graph(necessary) - model.detach_neuron_states() - - train_p_bar.set_description(f"Epoch {e} - BPTT Training Loss: {round(loss.item(), 4)}") - - batch_count += 1 - losses.append(loss.item()) - batches.append(batch_count) - - epochs_y.append(losses) - epochs_x.append(batches) - - acc = test(device, nb_time_steps, batch_size, feature_map_size, dataloader_test, model) - print(f'Epoch {e} accuracy: {acc}') - epochs_acc.append(acc) - - return epochs_x, epochs_y, epochs_acc - -def training_loop_no_tqdm(device, nb_time_steps, batch_size, feature_map_size, dataloader_train, model, loss_fn, optimizer, epochs, dataloader_test, record_data = False): - epochs_y = [] - epochs_x = [] - epochs_acc = [] - - model.train() - - for e in range(epochs): - losses = [] - batches = [] - batch_count = 0 - - for X, y in dataloader_train: - # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width] - X = X.reshape(-1, feature_map_size[2], feature_map_size[0], feature_map_size[1]).to(dtype=torch.float, device=device) - y = y.to(dtype=torch.long, device=device) - - # forward - pred = model(X) - - # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes] - pred = pred.reshape(batch_size, nb_time_steps, -1) - - # accumulate all time-steps output for final prediction - pred = pred.sum(dim = 1) - loss = loss_fn(pred, y) - - # gradient update - optimizer.zero_grad() - loss.backward() - optimizer.step() - - # detach the neuron states and activations from current computation graph(necessary) - model.detach_neuron_states() - - if record_data: - batch_count += 1 - losses.append(loss.item()) - batches.append(batch_count) - - if record_data: - epochs_y.append(losses) - epochs_x.append(batches) - - acc = test_no_tqdm(device, nb_time_steps, batch_size, feature_map_size, dataloader_test, model) - if record_data: - epochs_acc.append(acc) - - if record_data: - return epochs_x, epochs_y, epochs_acc - else: - return acc - -def test_no_tqdm(device, nb_time_steps, batch_size, feature_map_size, dataloader_test, model): - correct_predictions = [] - - with torch.no_grad(): - for X, y in dataloader_test: - # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width] - X = X.reshape(-1, feature_map_size[2], feature_map_size[0], feature_map_size[1]).to(dtype=torch.float, device=device) - y = y.to(dtype=torch.long, device=device) - - # forward - output = model(X) - - # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes] - output = output.reshape(batch_size, nb_time_steps, -1) - - # accumulate all time-steps output for final prediction - output = output.sum(dim=1) - - # calculate accuracy - pred = output.argmax(dim=1, keepdim=True) - - # compute the total correct predictions - correct_predictions.append(pred.eq(y.view_as(pred))) - - correct_predictions = torch.cat(correct_predictions) - return correct_predictions.sum().item()/(len(correct_predictions))*100 - -def test(device, nb_time_steps, batch_size, feature_map_size, dataloader_test, model): - correct_predictions = [] - with torch.no_grad(): - test_p_bar = tqdm(dataloader_test) - for X, y in test_p_bar: - # reshape the input from [Batch, Time, Channel, Height, Width] into [Batch*Time, Channel, Height, Width] - X = X.reshape(-1, feature_map_size[2], feature_map_size[0], feature_map_size[1]).to(dtype=torch.float, device=device) - y = y.to(dtype=torch.long, device=device) - - # forward - output = model(X) - - # reshape the output from [Batch*Time,num_classes] into [Batch, Time, num_classes] - output = output.reshape(batch_size, nb_time_steps, -1) - - # accumulate all time-steps output for final prediction - output = output.sum(dim=1) - - # calculate accuracy - pred = output.argmax(dim=1, keepdim=True) - - # compute the total correct predictions - correct_predictions.append(pred.eq(y.view_as(pred))) - - test_p_bar.set_description(f"Testing Model...") - - correct_predictions = torch.cat(correct_predictions) - return correct_predictions.sum().item()/(len(correct_predictions))*100 - -def load_dataset(dataset, n_time_steps): - if dataset == 'DVSGESTURE': - root_dir = "../../DVSGESTURE" - _ = DVSGesture(save_to=root_dir, train=True) - _ = DVSGesture(save_to=root_dir, train=False) - - to_raster = ToFrame(sensor_size=DVSGesture.sensor_size, n_time_bins=n_time_steps) - - snn_train_dataset = DVSGesture(save_to=root_dir, train=True, transform=to_raster) - snn_test_dataset = DVSGesture(save_to=root_dir, train=False, transform=to_raster) - - return snn_train_dataset, snn_test_dataset, DVSGesture.sensor_size - - elif dataset == 'NMNIST': - root_dir = "../../NMNIST" - _ = NMNIST(save_to=root_dir, train=True) - _ = NMNIST(save_to=root_dir, train=False) - - to_raster = ToFrame(sensor_size=NMNIST.sensor_size, n_time_bins=n_time_steps) - - snn_train_dataset = NMNIST(save_to=root_dir, train=True, transform=to_raster) - snn_test_dataset = NMNIST(save_to=root_dir, train=False, transform=to_raster) - - return snn_train_dataset, snn_test_dataset, NMNIST.sensor_size - - else: - - raise ValueError('no valid dataset') - -def split_train_validation(validation_ratio, snn_train_dataset, rand_seed): - num_samples = len(snn_train_dataset) - num_validation_samples = int(validation_ratio * num_samples) - - np.random.seed(rand_seed) - - validation_indices = np.random.choice(np.arange(num_samples), size=num_validation_samples, replace=False) - training_indices = np.array(list(filter(lambda x: x not in validation_indices, np.arange(num_samples)))) - - train_dataset = Subset(snn_train_dataset, training_indices) - validation_dataset = Subset(snn_train_dataset, validation_indices) - - return train_dataset, validation_dataset - -def split_train_validation_used_seed(validation_ratio, snn_train_dataset, used_seed): - """ Will generate a validation dataset in which the random indices do not overlap - with the ones that are generated using the random seed `used_seed`. - """ - num_samples = len(snn_train_dataset) - num_validation_samples = int(validation_ratio * num_samples) - - np.random.seed(used_seed) - - used_validation_indices = np.random.choice(np.arange(num_samples), size=num_validation_samples, replace=False) - - validation_indices = np.random.choice(np.setdiff1d(np.arange(num_samples), used_validation_indices), size=len(used_validation_indices), replace=False) - - training_indices = np.array(list(filter(lambda x: x not in validation_indices, np.arange(num_samples)))) - - if len(np.intersect1d(used_validation_indices, validation_indices)) != 0: - raise ValueError(f'data leakage: generated validation set overlaps with previously generated indices') - - train_dataset = Subset(snn_train_dataset, training_indices) - validation_dataset = Subset(snn_train_dataset, validation_indices) - - return train_dataset, validation_dataset - -def load_architecture( - architecture, input_size, nb_classes, batch_size, surrogate_fn, - min_v_mem=-0.313, spk_thr=2.0, hetero_init = False, hetero_seed = 1): - import sys - sys.path.append('../models') - - if architecture == 'ResSCNN_1': - from ResSCNN_1 import SCNN - return SCNN(input_size, nb_classes, batch_size, surrogate_fn, min_v_mem, spk_thr) - elif architecture == 'ResSCNN_2': - from ResSCNN_2 import SCNN - return SCNN(input_size, nb_classes, batch_size, surrogate_fn, min_v_mem, spk_thr) - elif architecture == 'ResSCNN_3': - from ResSCNN_3 import SCNN - return SCNN(input_size, nb_classes, batch_size, surrogate_fn, min_v_mem, spk_thr) - elif architecture == 'ResSCNN_4': - from ResSCNN_4 import SCNN - return SCNN(input_size, nb_classes, batch_size, surrogate_fn, min_v_mem, spk_thr) - elif architecture == 'ResSCNN_5': - from ResSCNN_5 import SCNN - return SCNN(input_size, nb_classes, batch_size, surrogate_fn, min_v_mem, spk_thr) - elif architecture == 'ResSCNN_6': - from ResSCNN_6 import SCNN - return SCNN(input_size, nb_classes, batch_size, surrogate_fn, min_v_mem, spk_thr) - elif architecture == 'ResSCNN_7': - from ResSCNN_7 import SCNN - return SCNN(input_size, nb_classes, batch_size, surrogate_fn, min_v_mem, spk_thr) - elif architecture == 'ResSCNN_8': - from ResSCNN_8 import SCNN - return SCNN(input_size, nb_classes, batch_size, surrogate_fn, min_v_mem, spk_thr) - elif architecture == 'ResSCNN_9': - from ResSCNN_9 import SCNN - return SCNN(input_size, nb_classes, batch_size, surrogate_fn, min_v_mem, spk_thr, hetero_init, hetero_seed) - elif architecture == 'ResSCNN_10': - from ResSCNN_10 import SCNN - return SCNN(input_size, nb_classes, batch_size, surrogate_fn, min_v_mem, spk_thr) - elif architecture == 'ResSCNN_11': - from ResSCNN_11 import SCNN - return SCNN(input_size, nb_classes, batch_size, surrogate_fn, min_v_mem, spk_thr) - else: - return None \ No newline at end of file diff --git a/tests/test_nonsequential/utils/weight_initialization.py b/tests/test_nonsequential/utils/weight_initialization.py deleted file mode 100644 index 53d2ddb6..00000000 --- a/tests/test_nonsequential/utils/weight_initialization.py +++ /dev/null @@ -1,49 +0,0 @@ -import torch.nn as nn -import numpy as np -import statistics - -def rescale_method_1(conv_layer: nn.Conv2d, input_pool_kernel: list, lambda_: float = 1): - """ - The `method 1` will use the average of the computed rescaling factor for each pooling layer - feeding into `conv_layer` (if there are more than one) to rescale its weights. - - Arguments - --------- - input_pool_kernel (list): the kernels of all pooling layers feeding input to `conv_layer`. - lambda_ (float): scales the computed re-scaling factor. If the outputs of the pooling are too small - the rescaling might lead to vanishing gradients, so we can try to control that by scaling it by - lambda. - """ - rescaling_factors = [] - - for kernel in input_pool_kernel: - rescaling_factors.append(kernel[0]*kernel[1]) - - rescaling_factor = np.mean(rescaling_factors)*lambda_ - - # print(f'method 1 - recaling factor: {rescaling_factor} (computed using {len(input_pool_kernel)} kernels and lambda {lambda_})') - - conv_layer.weight.data /= rescaling_factor - -def rescale_method_2(conv_layer: nn.Conv2d, input_pool_kernel: list, lambda_: float = 1): - """ - The `method 2` will use the harmonic mean of the computed rescaling factor for each pooling layer - feeding into `conv_layer` (if there are more than one) to rescale its weights. - - Arguments - --------- - input_pool_kernel (list): the kernels of all pooling layers feeding input to `conv_layer`. - lambda_ (float): scales the computed re-scaling factor. If the outputs of the pooling are too small - the rescaling might lead to vanishing gradients, so we can try to control that by scaling it by - lambda. - """ - rescaling_factors = [] - - for kernel in input_pool_kernel: - rescaling_factors.append(kernel[0]*kernel[1]) - - rescaling_factor = statistics.harmonic_mean(rescaling_factors)*lambda_ - - # print(f'method 2 - recaling factor: {rescaling_factor} (computed using {len(input_pool_kernel)} kernels and lambda {lambda_})') - - conv_layer.weight.data /= rescaling_factor \ No newline at end of file From 74f11181888eccbb422325ae492c29b93edeec91 Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 20 Aug 2024 14:11:35 +0200 Subject: [PATCH 121/379] (WIP) new baseline DynapcnnLayer --- sinabs/backend/dynapcnn/dynapcnn_layer_v2.py | 159 +++++++++++++++++++ 1 file changed, 159 insertions(+) create mode 100644 sinabs/backend/dynapcnn/dynapcnn_layer_v2.py diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer_v2.py b/sinabs/backend/dynapcnn/dynapcnn_layer_v2.py new file mode 100644 index 00000000..c1036f45 --- /dev/null +++ b/sinabs/backend/dynapcnn/dynapcnn_layer_v2.py @@ -0,0 +1,159 @@ +# author : Willian Soares Girao +# contact : wsoaresgirao@gmail.com + +from copy import deepcopy +from typing import Dict, Callable, Tuple, Union, List + +import numpy as np +import torch +from torch import nn + +import sinabs.activation +import sinabs.layers as sl + +from .discretize import discretize_conv_spike_ + +class DynapcnnLayer(nn.Module): + """Create a DynapcnnLayer object representing a layer on DynapCNN or Speck. + + Requires a convolutional layer, a sinabs spiking layer and a list of + pooling values. The layers are used in the order conv -> spike -> pool. + + Parameters + ---------- + conv: torch.nn.Conv2d or torch.nn.Linear + Convolutional or linear layer + (linear will be converted to convolutional) + spk: sinabs.layers.IAFSqueeze + Sinabs IAF layer + in_shape: tuple of int + The input shape, needed to create dynapcnn configs if the network + does not contain an input layer. Convention: (features, height, width) + pool: List of integers + Each integer entry represents an output (destination on chip) and + whether pooling should be applied (values > 1) or not (values equal + to 1). The number of entries determines the number of tensors the + layer's forward method returns. + discretize: bool + Whether to discretize parameters. + rescale_weights: int + Layer weights will be divided by this value. + """ + + def __init__( + self, + conv: nn.Conv2d, + spk: sl.IAFSqueeze, + in_shape: Tuple[int, int, int], + pool: List[int], + discretize: bool = True, + rescale_weights: int = 1, + ): + super().__init__() + + # int conversion is done while writing the config. + if discretize: + conv, spk = discretize_conv_spike_(conv, spk, to_int=False) + + self.conv = conv + self.spk = spk + self.in_shape = in_shape + self.pool = pool + self.discretize = discretize + self.rescale_weights = rescale_weights + self.conv_out_shape = self._get_conv_output_shape() + + self._pool_lyrs = self._make_pool_layers() # creates SumPool2d layers from `pool`. + + + ####################################################### Public Methods ####################################################### + + def forward(self, x): + """Torch forward pass. + + ... + """ + + returns = [] + + x = self.conv(x) + x = self.spk(x) + + for pool in self.pool: + if pool == 1: + # no pooling is applied. + returns.append(x) + else: + # sum pooling of `(pool, pool)` is applied. + pool_out = self._pool_lyrs[pool](x) + returns.append(pool_out) + + return tuple(returns) + + def get_neuron_shape(self) -> Tuple[int, int, int]: + """Return the output shape of the neuron layer. + + Returns + ------- + features, height, width + """ + # same as the convolution's output. + return self.conv_out_shape + + ####################################################### Private Methods ####################################################### + + def _make_pool_layers(self) -> Dict[int, sl.SumPool2d]: + """ Creates a `sl.SumPool2d` for each entry in `self.pool` greater than one. + + Note: the "kernel size" (values > 1) in self.pool is by default used to set the stride of the pooling layer. + + Returns + ------- + - pool_lyrs (dict): the `key` is a value grather than 1 in `self.pool`, with the `value` being the `sl.SumPool2d` it represents. + """ + + pool_lyrs = {} + + # validating if pool are integers + for item in self.pool: + if not isinstance(item, int): + raise ValueError(f"Item '{item}' in `pool` is not an integer.") + + # create layers form pool list. + for kernel_s in self.pool: + + if kernel_s != 1: + + pooling = (kernel_s, kernel_s) + cumulative_pooling = (1, 1) + + # compute cumulative pooling. + cumulative_pooling = ( + cumulative_pooling[0] * pooling[0], + cumulative_pooling[1] * pooling[1], + ) + + # create SumPool2d layer. + pool_lyrs[kernel_s] = sl.SumPool2d(cumulative_pooling) + + return pool_lyrs + + def _get_conv_output_shape(self) -> Tuple[int, int, int]: + """ Computes the output dimensions of `conv_layer`. + + Returns + ---------- + - output dimensions (tuple): a tuple describing `(output channels, height, width)`. + """ + # get the layer's parameters. + out_channels = self.conv.out_channels + kernel_size = self.conv.kernel_size + stride = self.conv.stride + padding = self.conv.padding + dilation = self.conv.dilation + + # compute the output height and width. + out_height = ((self.in_shape[1] + 2 * padding[0] - dilation[0] * (kernel_size[0] - 1) - 1) // stride[0]) + 1 + out_width = ((self.in_shape[2] + 2 * padding[1] - dilation[1] * (kernel_size[1] - 1) - 1) // stride[1]) + 1 + + return (out_channels, out_height, out_width) \ No newline at end of file From 7d1b811b7cc639a921ea60f45198dddba5fb28b8 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 22 Aug 2024 12:33:28 +0200 Subject: [PATCH 122/379] (WIP) memory summary methods added --- sinabs/backend/dynapcnn/dynapcnn_layer_v2.py | 54 +++++++++++++++++++- 1 file changed, 52 insertions(+), 2 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer_v2.py b/sinabs/backend/dynapcnn/dynapcnn_layer_v2.py index c1036f45..80be8a27 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer_v2.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer_v2.py @@ -62,7 +62,6 @@ def __init__( self.discretize = discretize self.rescale_weights = rescale_weights self.conv_out_shape = self._get_conv_output_shape() - self._pool_lyrs = self._make_pool_layers() # creates SumPool2d layers from `pool`. @@ -95,10 +94,61 @@ def get_neuron_shape(self) -> Tuple[int, int, int]: Returns ------- - features, height, width + - conv_out_shape (tuple): formatted as (features, height, width). """ # same as the convolution's output. return self.conv_out_shape + + def summary(self) -> dict: + """ Returns a summary of the convolution's/pooling's kernel sizes and the output shape of the spiking layer.""" + # TODO I can't see pooling being used in checking memory constraints by the builder so I'm ignoring for now the fact that multiple pooling could exist. + + _pool = None + + # @TODO POSSIBLE INCONSISTENCY: if the `SumPool2d` is the result of a conversion from `AvgPool2d` then `SumPool2d.kernel_size` + # is of type tuple, otherwise it is an int. + if self._pool_lyrs: + # @TODO ignoring for now that there could be multiple poolings (just use the first one). + if isinstance(self._pool_lyrs[next(iter(self._pool_lyrs))].kernel_size, tuple): + _pool = list(self._pool_lyrs[next(iter(self._pool_lyrs))].kernel_size) + elif isinstance(self._pool_lyrs[next(iter(self._pool_lyrs))].kernel_size, int): + _pool = [self._pool_lyrs[next(iter(self._pool_lyrs))].kernel_size, self._pool_lyrs[next(iter(self._pool_lyrs))].kernel_size] + else: + raise ValueError('Type of `self.pool_layer[0].kernel_size` not understood.') + + return { + "pool": (_pool), + "kernel": list(self.conv_layer.weight.data.shape), + "neuron": self.conv_out_shape, # neuron layer output has the same shape as the convolution layer ouput. + } + + def memory_summary(self): + """Computes the amount of memory required for each of the components. Note that this is not + necessarily the same as the number of parameters due to some architecture design + constraints. + + .. math:: + + K_{MT} = c \\cdot 2^{\\lceil \\log_2\\left(k_xk_y\\right) \\rceil + \\lceil \\log_2\\left(f\\right) \\rceil} + + .. math:: + + N_{MT} = f \\cdot 2^{ \\lceil \\log_2\\left(f_y\\right) \\rceil + \\lceil \\log_2\\left(f_x\\right) \\rceil } + + Returns + ------- + A dictionary with keys kernel, neuron and bias and the corresponding memory sizes + """ + summary = self.summary() + f, c, h, w = summary["kernel"] + f, neuron_height, neuron_width = self.conv_out_shape # neuron layer output has the same shape as the convolution layer ouput. + + return { + "kernel": c * pow(2, np.ceil(np.log2(h * w)) + np.ceil(np.log2(f))), + "neuron": f + * pow(2, np.ceil(np.log2(neuron_height)) + np.ceil(np.log2(neuron_width))), + "bias": 0 if self.conv.bias is None else len(self.conv.bias), + } ####################################################### Private Methods ####################################################### From b65428a304d921f8744779eb9ca2ff3f2d779f18 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 22 Aug 2024 12:37:01 +0200 Subject: [PATCH 123/379] (WIP) functionality in 'build_from_graph()' now uses a handler class to gather network-level info to generate the arguments for a DynapcnnLayer instance --- sinabs/backend/dynapcnn/utils.py | 47 ++++++++++++++++++++++++-------- 1 file changed, 36 insertions(+), 11 deletions(-) diff --git a/sinabs/backend/dynapcnn/utils.py b/sinabs/backend/dynapcnn/utils.py index bf140d07..3c4a58dd 100644 --- a/sinabs/backend/dynapcnn/utils.py +++ b/sinabs/backend/dynapcnn/utils.py @@ -10,6 +10,7 @@ from .crop2d import Crop2d from .dvs_layer import DVSLayer, expand_to_pair from .dynapcnn_layer import DynapcnnLayer +from .dynapcnn_layer_handler import DynapcnnLayerHandler from .exceptions import WrongPoolingModule from .flipdims import FlipDims @@ -176,10 +177,13 @@ def construct_dynapcnnlayers_from_mapper( dynapcnn_layers = {} for dpcnnl_idx, dcnnl_data in nodes_to_dcnnl_map.items(): - # create a `DynapcnnLayer` from the set of layers in `dcnnl_data`. - dynapcnnlayer = construct_dynapcnnlayer( + # create a `DynapcnnLayerHandler` from the set of layers in `dcnnl_data`. + layerhandler = construct_layerhandler( dpcnnl_idx, discretize, edges, nodes_to_dcnnl_map, weight_rescaling_fn, entry_nodes) + # create a `DynapcnnLayer` from the handler. + dynapcnnlayer = construct_dynapcnnlayer(layerhandler) + dynapcnn_layers[dpcnnl_idx] = { 'layer': dynapcnnlayer, 'destinations': nodes_to_dcnnl_map[dpcnnl_idx]['destinations'] @@ -220,26 +224,26 @@ def update_nodes_io(updated_node: int, output_shape: tuple, nodes_to_dcnnl_map: # accessing node targeted by `updated_node` (its input shape becomes `updated_node.output_shape`). val['input_shape'] = output_shape -def construct_dynapcnnlayer( +def construct_layerhandler( dpcnnl_idx: int, discretize: bool, edges: List[Tuple[int, int]], nodes_to_dcnnl_map: Dict[int, Dict[Union[int, str], Union[Dict[str, Union[nn.Module, Tuple[int, int, int]]], List[int]]]], weight_rescaling_fn: Callable, - entry_nodes: List[int]) -> DynapcnnLayer: - """ Extract the modules (layers) in a dictionary and uses them to instantiate a `DynapcnnLayer` object. + entry_nodes: List[int]) -> DynapcnnLayerHandler: + """ Extract the modules (layers) in a dictionary and uses them to instantiate a `DynapcnnLayerHandler` object. Parameters ---------- - - dpcnnl_idx (int): the index/ID that will be associated with a `DynapcnnLayer` instance. This integer indexes a `dict` within `nodes_to_dcnnl_map` + - dpcnnl_idx (int): the index/ID that will be associated with a `DynapcnnLayerHandler` instance. This integer indexes a `dict` within `nodes_to_dcnnl_map` containing the data required to create the instance returned by this function. - discretize (bool): whether or not the weights/neuron parameters of the model will be quantized. - edges (list): each `nn.Module` within `nodes_to_dcnnl_map[dpcnnl_idx]` is a node in the original computational graph describing a spiking network - being converted to a `DynapcnnNetwork`. An edge `(A, B)` describes how modules forward data amongst themselves. This list is used by a `DynapcnnLayer` + being converted to a `DynapcnnNetwork`. An edge `(A, B)` describes how modules forward data amongst themselves. This list is used by a `DynapcnnLayerHandler` to figure out the number and sequence of output tesnors its forward method needs to return. - nodes_to_dcnnl_map (dict): contains all layers (`nn.Module`) in the original spiking network grouped into dictionaries gathering the data necessary - to instantiate a `DynapcnnLayer`. A `nodes_to_dcnnl_map[dpcnnl_idx]` will contain `int` keys (whose value corresponds to a `dict` with a `nn.Module` - instance and its associated I/O shapes, i.e., one layer within the `DynapcnnLayer` instance) or `str` keys (whose values correspond to a list of + to instantiate a `DynapcnnLayerHandler`. A `nodes_to_dcnnl_map[dpcnnl_idx]` will contain `int` keys (whose value corresponds to a `dict` with a `nn.Module` + instance and its associated I/O shapes, i.e., one layer within the `DynapcnnLayerHandler` instance) or `str` keys (whose values correspond to a list of integers corresponding to either destinations IDs or re-scaling factors). - weight_rescaling_fn (callable): a method that handles how the re-scaling factor for one or more `SumPool2d` projecting to the same convolutional layer are combined/re-scaled before being applied. @@ -247,14 +251,14 @@ def construct_dynapcnnlayer( Returns ---------- - - dynapcnnlayer (DynapcnnLayer): the a `DynapcnnLayer` instance made up by all the layers (`nn.Module`) in `dcnnl_data`. + - layerhandler (DynapcnnLayerHandler): the a `DynapcnnLayer` instance made up by all the layers (`nn.Module`) in `dcnnl_data`. """ # convert all AvgPool2d in 'dcnnl_data' into SumPool2d. convert_Avg_to_Sum_pooling(nodes_to_dcnnl_map[dpcnnl_idx], edges, nodes_to_dcnnl_map) # instantiate a DynapcnnLayer from the data in 'dcnnl_data'. - dynapcnnlayer = DynapcnnLayer( + layerhandler = DynapcnnLayerHandler( dpcnnl_index = dpcnnl_idx, dcnnl_data = nodes_to_dcnnl_map[dpcnnl_idx], discretize = discretize, @@ -263,6 +267,27 @@ def construct_dynapcnnlayer( entry_nodes = entry_nodes ) + return layerhandler + +def construct_dynapcnnlayer(handler: DynapcnnLayerHandler) -> DynapcnnLayer: + """... + """ + + # retrieve required data from handler. + conv = deepcopy(handler.conv_layer) + spk = deepcopy(handler.spk_layer) + in_shape = handler.conv_in_shape + pool = handler.get_pool_list() + + # instantiate a DynapcnnLayer from the data in 'dcnnl_data'. + dynapcnnlayer = DynapcnnLayer( + conv = conv, + spk = spk, + in_shape = in_shape, + pool = pool, + discretize = False, + ) + return dynapcnnlayer def convert_Avg_to_Sum_pooling( From 763142a6fa01803b5aeca4bb513cb51577d01742 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 22 Aug 2024 12:40:54 +0200 Subject: [PATCH 124/379] (WIP) old version of DynapcnnLayer (with network-level knowledge) became a 'handler' class whose only job is to pre-processes network-level data to generate args for a DynapcnnLayer instance --- .../dynapcnn/dynapcnn_layer_handler.py | 399 ++++++++++++++++++ 1 file changed, 399 insertions(+) create mode 100644 sinabs/backend/dynapcnn/dynapcnn_layer_handler.py diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer_handler.py b/sinabs/backend/dynapcnn/dynapcnn_layer_handler.py new file mode 100644 index 00000000..9bb19f28 --- /dev/null +++ b/sinabs/backend/dynapcnn/dynapcnn_layer_handler.py @@ -0,0 +1,399 @@ +# author : Willian Soares Girao +# contact : wsoaresgirao@gmail.com + +from copy import deepcopy +from typing import Dict, Callable, Tuple, Union, List + +import numpy as np +import torch +from torch import nn + +import sinabs.activation +import sinabs.layers as sl + +from .discretize import discretize_conv_spike_ + +class DynapcnnLayerHandler(): + """ + Class handling the pre-processing of network-level data into (device) layer-level data (i.e., arguments required for a `DynapcnnLayer` instantiation). + + Parameters + ---------- + - dpcnnl_index (int): the index/ID that will be associated with a `DynapcnnLayer` instance. This integer indexes a `dict` within `dcnnl_data` + that comprises a set of layers (`nn.Module`) and their respective I/O tensor shapes. + - dcnnl_data (dict): contains the nodes to be merged into this `DynapcnnLayer`, their I/O shapes and the index of the other `DynapcnnLayer`s to + be set as destinations. The `int` keys correspond to the nodes IDs associated `nn.Module`s (a single layer in the original network) becoming + part of this `DynapcnnLayer` instance, while the `str` keys correspond to this instance's destinations and re-scaling factors. + - discretize (bool): whether or not the weights/neuron parameters of the model will be quantized. + - sinabs_edges (list): each `nn.Module` within `dcnnl_data` is a node in the original computational graph describing a spiking network. An edge + `(A, B)` describes how modules forward data amongst themselves. This list is used by a `DynapcnnLayer` to figure out the number and + sequence of output tesnors its forward method needs to return. + - weight_rescaling_fn (callable): a method that handles how the re-scaling factor for one or more `SumPool2d` projecting to + the same convolutional layer are combined/re-scaled before applying them. + - entry_nodes (list): node IDs corresponding to layers in the original network that are input nodes (i.e., a "point of entry" for the external data). + """ + + def __init__( + self, + dpcnnl_index: int, + dcnnl_data: Dict[Union[int, str], Union[Dict[str, Union[nn.Module, Tuple[int, int, int], Tuple[int, int, int]]], List[int]]], + discretize: bool, + sinabs_edges: List[Tuple[int, int]], + weight_rescaling_fn: Callable, + entry_nodes: List[int] + ): + self.dpcnnl_index = dpcnnl_index + self.assigned_core = None + self.entry_point = False + + if 'core_idx' in dcnnl_data: + self.assigned_core = dcnnl_data['core_idx'] + + self._lin_to_conv_conversion = False + + conv = None + self.conv_node_id = None + self.conv_in_shape = None + self.conv_out_shape = None + + spk = None + self.spk_node_id = None + + pool = [] + self.pool_node_id = [] + self.conv_rescaling_factor = None + + self.dynapcnnlayer_destination = dcnnl_data['destinations'] + + for key, value in dcnnl_data.items(): + if isinstance(key, int): + # value has data pertaining a node (torch/sinabs layer). + if isinstance(value['layer'], sl.IAFSqueeze): + spk = value['layer'] + self.spk_node_id = key + elif isinstance(value['layer'], nn.Linear) or isinstance(value['layer'], nn.Conv2d): + conv = value['layer'] + self.conv_node_id = key + elif isinstance(value['layer'], sl.SumPool2d): + pool.append(value['layer']) + self.pool_node_id.append(key) + else: + raise ValueError(f'Node {key} has not valid layer associated with it.') + + if not conv: + raise ValueError(f'Convolution layer not present.') + + if not spk: + raise ValueError(f'Spiking layer not present.') + + spk = deepcopy(spk) + if spk.is_state_initialised(): + # TODO this line bellow is causing an exception on `.v_men.shape` to be raised in `.get_layer_config_dict()`. Find out why. + # spk.v_mem = spk.v_mem.data.unsqueeze(-1).unsqueeze(-1) # expand dims. + + # TODO hacky stuff: make it better (THIS SEEMS TO BE FIXING THE PROBLEM ABOVE THO). + if len(list(spk.v_mem.shape)) != 4: + spk.v_mem = spk.v_mem.data.unsqueeze(-1).unsqueeze(-1) # expand dims. + + if isinstance(conv, nn.Linear): + # A `nn.Linear` needs to be converted into `nn.Conv2d`. The I/O shapes of the spiking layer are updated + # accordingly following the conversion. + + conv, conv_in_shape = self._convert_linear_to_conv(conv, dcnnl_data[self.conv_node_id]) + + # the original `nn.Linear` output shape becomes the equivalent `nn.Conv2d` shape. + self.conv_out_shape = self._update_conv_node_output_shape( + conv_layer=conv, layer_data=dcnnl_data[self.conv_node_id], input_shape=conv_in_shape) + + # the I/O shapes for neuron layer following the new conv need also to be updated. + self._update_neuron_node_output_shape(spiking_layer_data=dcnnl_data[self.spk_node_id], conv_out_shape=self.conv_out_shape) + + else: + self.conv_out_shape = dcnnl_data[self.conv_node_id]['output_shape'] + conv = deepcopy(conv) + + # check if convolution kernel is a square. + if conv.kernel_size[0] != conv.kernel_size[1]: + raise ValueError('The kernel of a `nn.Conv2d` must have the same height and width.') + + # input shape of conv layer. + self.conv_in_shape = dcnnl_data[self.conv_node_id]['input_shape'] + # input shape of the `DynapcnnLayer` instance. + self.input_shape = self.conv_in_shape + + # this weight rescale comes from the node projecting into this 'conv' node. + if len(dcnnl_data['conv_rescale_factor']): + # this means an `AvgPool2d` has been converted into a `SumPool2d`. + self.conv_rescaling_factor = weight_rescaling_fn(dcnnl_data['conv_rescale_factor']) + conv.weight.data = (conv.weight.data / self.conv_rescaling_factor).clone().detach() + else: + # this means `SumPool2d` have been used from the start. + conv.weight.data = (conv.weight.data).clone().detach() + + # int conversion is done while writing the config. + if discretize: + conv, spk = discretize_conv_spike_(conv, spk, to_int=False) + + # consolidate layers. + self.conv_layer = conv + self.spk_layer = spk + self.pool_layer = [] + + if len(pool) != 0: + # the 1st pooling targets the 1st destination in `dcnnl_data['destinations']`, the 2nd pooling targets the 2nd destination... + for plyr in pool: + # @TODO POSSIBLE INCONSISTENCY: if the `SumPool2d` is the result of a conversion from `AvgPool2d` then `SumPool2d.kernel_size` + # is of type tuple, otherwise it is an int. + if isinstance(plyr.kernel_size, tuple) and plyr.kernel_size[0] != plyr.kernel_size[1]: + raise ValueError("Only square kernels are supported") + self.pool_layer.append(deepcopy(plyr)) + + # map destination nodes for each layer in this instance. + self.nodes_destinations = self._get_destinations_input_source(sinabs_edges) + + # flag if the instance is an entry point (i.e., an input node of the network). + if self.conv_node_id in entry_nodes: + self.entry_point = True + + ####################################################### Public Methods ####################################################### + + def get_pool_list(self) -> List[int]: + """ This returns a list of integers that describe the number of outputs created by this layer (length of the list) and + whether or not pooling is applied (values > 1). This is meant to generate the `pool`argument for a `DynapcnnLayer` instance. + + Returns + ---------- + - pool (list): Each integer entry represents an output (destination on chip) and whether pooling should be applied (values > 1) or not (values + equal to 1). The number of entries determines the number of tensors the layer's forward method returns. + """ + pool = [] + + for lyr, dests in self.nodes_destinations.items(): + if lyr == self.spk_node_id: + # spk layer projects somewhere outside this layer (output without pooling). + pool.append(1) + elif lyr in self.pool_node_id: + # getting kernel sizes from each pooling layer. + sumpool_idx = self.pool_node_id.index(lyr) + kernel_size = self.pool_layer[sumpool_idx].kernel_size[0] if isinstance(self.pool_layer[sumpool_idx].kernel_size, tuple) else self.pool_layer[sumpool_idx].kernel_size + pool.append(kernel_size) + + return pool + + def get_destination_dcnnl_index(self, dcnnl_id: int) -> int: + """ The `forward` method will return as many tensors as there are elements in `self.dynapcnnlayer_destination`. Since the i-th returned tensor is to be + fed to the i-th destionation in `self.dynapcnnlayer_destination`, the return of this method can be used to index a tensor returned in the `forward` method. + + Parameters + ---------- + - dcnnl_id (int): this should be one of the values listed within `self.dynapcnnlayer_destination`. + + Returns + ---------- + - The index of `dcnnl_id` within `self.dynapcnnlayer_destination`. + """ + return self.dynapcnnlayer_destination.index(dcnnl_id) + + def get_modified_node_io(self, dcnnl_data: dict) -> Union[Tuple[int, tuple], Tuple[None, None]]: + """ Follwing a conversion, the I/O shapes of the spiking layer have been updated to match the convolution's + output. Thus, all nodes receiving input from this spiking layer need their input shapes updated. + + Parameters + ---------- + - dcnnl_data (dict): the set of layers grouped together to comprise this instance of a `DynapcnnLayer`. + + Returns + ---------- + - node ID (int): the ID of the spiking layer consuming the tunerd layer's output (`None` if there was no conversion). + - output shape (tuple): the new output shape following a converstion from `nn.Linear` to `nn.Conv2d` (`None` if there was no conversion). + """ + if self._lin_to_conv_conversion: + return self.spk_node_id, dcnnl_data[self.spk_node_id]['output_shape'] + return None, None + + def zero_grad(self, set_to_none: bool = False) -> None: + return self.spk_layer.zero_grad(set_to_none) + + def get_conv_output_shape(self, conv_layer: nn.Conv2d, input_shape: Tuple[int, int, int]) -> Tuple[int, int, int]: + """ Computes the output dimensions of `conv_layer`. + + Parameters + ---------- + - conv_layer (nn.Conv2d): conv. layer whose output will be computed for. + - input_shape (tuple): the shape for the input tensor the layer will process. + + Returns + ---------- + - output dimensions (tuple): a tuple describing `(output channels, height, width)`. + """ + # get the layer's parameters. + out_channels = conv_layer.out_channels + kernel_size = conv_layer.kernel_size + stride = conv_layer.stride + padding = conv_layer.padding + dilation = conv_layer.dilation + + # compute the output height and width. + out_height = ((input_shape[1] + 2 * padding[0] - dilation[0] * (kernel_size[0] - 1) - 1) // stride[0]) + 1 + out_width = ((input_shape[2] + 2 * padding[1] - dilation[1] * (kernel_size[1] - 1) - 1) // stride[1]) + 1 + + return (out_channels, out_height, out_width) + + def __str__(self): + pretty_print = '\n' + + pretty_print += 'COMPUTATIONAL NODES:\n\n' + + pretty_print += f'(node {self.conv_node_id}): {self.conv_layer}\n' + pretty_print += f'(node {self.spk_node_id}): {self.spk_layer}' + if len(self.pool_layer) != 0: + for idx, lyr in enumerate(self.pool_layer): + pretty_print += f'\n(node {self.pool_node_id[idx]}): {lyr}' + + pretty_print += '\n\nMETADATA:\n' + pretty_print += f'\n> network\'s entry point: {self.entry_point}' + pretty_print += f'\n> convolution\'s weight re-scaling factor: {self.conv_rescaling_factor}' + pretty_print += f'\n> assigned core index: {self.assigned_core}' + pretty_print += f'\n> destination DynapcnnLayers: {self.dynapcnnlayer_destination}' + + for node, destinations in self.nodes_destinations.items(): + pretty_print += f'\n> node {node} feeds input to nodes {destinations}' + + return pretty_print + + ####################################################### Private Methods ####################################################### + + def _update_neuron_node_output_shape(self, spiking_layer_data: dict, conv_out_shape: tuple) -> None: + """ Updates the spiking layer's I/O shapes after the conversion of a `nn.Linear` into a `nn.Conv2d` (to match the convolution's output). + + Parameters + ---------- + - spiking_layer_data (dict): the dictionary containing all data regarding the spiking layer. + - conv_out_shape (tuple): the output shape of the convolution layer preceeding the spiking layer. + """ + + # spiking layer consumes the tensor coming out of the conv. layer. + spiking_layer_data['input_shape'] = conv_out_shape + # spiking layer outputs the same shape as the conv. layer. + spiking_layer_data['output_shape'] = spiking_layer_data['input_shape'] + + def _update_conv_node_output_shape(self, conv_layer: nn.Conv2d, layer_data: dict, input_shape: Tuple[int, int, int]) -> Tuple[int, int, int]: + """ Updates the shape of the output tensor of a node that used to be a `nn.Linear` and became a `nn.Conv2d`. + + The input shapes to nodes are extracted using a list of edges by finding the output shape of the 1st element + in the edge and setting it as the input shape to the 2nd element in the edge. If a node used to be a `nn.Linear` + and it became a `nn.Conv2d`, output shape in the mapper needs to be updated, otherwise there will be a mismatch + between its output and the input it provides to another node. + + Parameters + ---------- + - conv_layer (nn.Module): the `nn.Conv2d` created from a `nn.Linear`. + - layer_data (dict): the dictionary containing the data associated with the original `nn.Linear` converted into `nn.Conv2d`. + - input_shape (tuple): the input shape the layer expects. + + Returns + ---------- + - output_shape (tuple): the tensor shape produced by the `nn.Conv2d` created from a `nn.Linear`. + """ + layer_data['output_shape'] = self.get_conv_output_shape(conv_layer, input_shape) + + return layer_data['output_shape'] + + def _convert_linear_to_conv(self, lin: nn.Linear, layer_data: dict) -> Tuple[nn.Conv2d, Tuple[int, int, int]]: + """ Convert Linear layer to Conv2d. + + Parameters + ---------- + - lin (nn.Linear): linear layer to be converted. + + Returns + ------- + - nn.Conv2d: convolutional layer equivalent to `lin`. + - input_shape (tuple): the tensor shape the layer expects. + """ + # this flags the necessity to update the I/O shape pre-computed for each of the original layers being compressed within a `DynapcnnLayer` instance. + self._lin_to_conv_conversion = True + + input_shape = layer_data['input_shape'] + + in_chan, in_h, in_w = input_shape + + if lin.in_features != in_chan * in_h * in_w: + raise ValueError("Shapes don't match.") + + layer = nn.Conv2d( + in_channels=in_chan, + kernel_size=(in_h, in_w), + out_channels=lin.out_features, + padding=0, + bias=lin.bias is not None, + ) + + if lin.bias is not None: + layer.bias.data = lin.bias.data.clone().detach() + + layer.weight.data = ( + lin.weight.data.clone() + .detach() + .reshape((lin.out_features, in_chan, in_h, in_w)) + ) + + return layer, input_shape + + def _get_destinations_input_source(self, sinabs_edges: list) -> dict: + """ Creates a mapping between each layer in this `DynapcnnLayer` instance and its targe nodes that are part of different + `DynapcnnLayer` instances. This mapping is used to figure out how many tensors the `forward` method needs to return. + + Parameters + ---------- + - sinabs_edges (list): each `nn.Module` within `dcnnl_data` is a node in the original computational graph describing a spiking + network. An edge `(A, B)` describes how modules forward data amongst themselves. This list is used by a `DynapcnnLayer` to + figure out the number and sequence of output tesnors its forward method needs to return. + + Returns + ---------- + - destinations_input_source (dict): maps a `nn.Module` within this `DynapcnnLayer` to the nodes it provides the input to. + """ + destinations_input_source = {} + + # check whether spiking layer projects outside this DynapcnnLayer (i.e. to one of the destinations without passing through a pooling). + spk_destinations = [] + for edge in sinabs_edges: + if edge[0] == self.spk_node_id and edge[1] not in self.pool_node_id: + # spiking layer projects to a node outside this DynapcnnLayer. + spk_destinations.append(edge[1]) + if len(spk_destinations) > 0: + destinations_input_source[self.spk_node_id] = [] + for node_id in spk_destinations: + destinations_input_source[self.spk_node_id].append(node_id) + + # get `pooling->destination` mapping. The pooling outputs will be arranged sequentially since the pooling layers are added sequentially + # to `self.pool_layer` (i.e., as they appear in the computational graph of the original `nn.Module`). + for id in self.pool_node_id: + destinations_input_source[id] = [] + for edge in sinabs_edges: + if edge[0] == id: + destinations_input_source[id].append(edge[1]) + + return destinations_input_source + + def get_pool_kernel_size(self, node: int) -> int: + """ Returns the pooling kernel size if `node` is a pooling layer.""" + + if node in self.pool_node_id: + i = self.pool_node_id.index(node) + return self.pool_layer[i].kernel_size[0] if isinstance(self.pool_layer[i].kernel_size, tuple) else self.pool_layer[i].kernel_size + elif node == self.spk_node_id: + return 1 + else: + raise ValueError(f'Node {node} does not belong to DynapcnnLayer {self.dpcnnl_index}.') + + @staticmethod + def find_nodes_core_id(node: int, forward_map: dict) -> int: + + for _, dcnnl in forward_map.items(): + + if node == dcnnl.conv_node_id or node == dcnnl.spk_node_id or node in dcnnl.pool_node_id: + return dcnnl.assigned_core + + raise ValueError(f'Node {node} not found in any of the cores.') From d0fd696760a9f4b1902cc5271ef36ce52c7f4b59 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 22 Aug 2024 13:22:29 +0200 Subject: [PATCH 125/379] DynapcnnLayerHandler instances passed as argument to access entry points of the network --- .../dynapcnn/dynapcnnnetwork_module.py | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py index 69df934c..4eb7ac0a 100644 --- a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py +++ b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py @@ -9,17 +9,18 @@ class DynapcnnNetworkModule(): """ - Uses the set of `DynapcnnLayer` instances and how they address each other to define what the `forward` method of the model should do. + Uses the set of `DynapcnnLayer`\`DynapcnnLayerHandler` instances and how they address each other to define what the `forward` method of the model should do. Parameters ---------- - dcnnl_edges (list): tuples representing the output->input mapping between `DynapcnnLayer` instances that have been used as configuration for each core `CNNLayerConifg`. - - dynapcnn_layers (dict): a mapper containing `DynapcnnLayer` instances along with their supporting metadata (e.g. assigned core, - destination layers, etc.). + - dynapcnn_layers (dict): a mapper containing `DynapcnnLayer` instances. + - dynapcnnlayers_handlers (dict): a mapper containing `DynapcnnLayerHandler` instances (hold network-level + data that was used to create the respective `DynapcnnLayer` instances in `dynapcnn_layers`). """ - def __init__(self, dcnnl_edges: List[Tuple[int, int]], dynapcnn_layers: Dict): + def __init__(self, dcnnl_edges: List[Tuple[int, int]], dynapcnn_layers: Dict[int, dict], dynapcnnlayers_handlers: Dict[int, dict]): self.dcnnl_edges = dcnnl_edges @@ -27,19 +28,19 @@ def __init__(self, dcnnl_edges: List[Tuple[int, int]], dynapcnn_layers: Dict): self.forward_map, self.merge_points = self._build_module_forward_from_graph(dcnnl_edges, dynapcnn_layers) # add extra edges marking which nodes are input to the network. - self._add_entry_points_edges(dynapcnn_layers) + self._add_entry_points_edges(dynapcnnlayers_handlers) - def _add_entry_points_edges(self, dynapcnn_layers: dict) -> None: + def _add_entry_points_edges(self, dynapcnnlayers_handlers: dict) -> None: """ Addes an extra edge `('input', X)` to `self.dcnnl_edges` if `X` is an entry point of the `DynapcnnNetwork` - (i.e., `dynapcnn_layers[X]['layer'].entry_point = True`). + (i.e., `dynapcnnlayers_handlers[X]['layer_handler'].entry_point = True`). Parameters ---------- - - dynapcnn_layers (dict): a mapper containing `DynapcnnLayer` instances along with their supporting metadata (e.g. assigned core, + - dynapcnnlayers_handlers (dict): a mapper containing `DynapcnnLayerHandler` instances along with their supporting metadata (e.g. assigned core, destination layers, etc.). """ - for indx, dcnnl_data in dynapcnn_layers.items(): - if dcnnl_data['layer'].entry_point: + for indx, dcnnl_data in dynapcnnlayers_handlers.items(): + if dcnnl_data['layer_handler'].entry_point: self.dcnnl_edges.append(('input', indx)) def _spot_merging_points(self, dcnnl_edges: list) -> Dict[int, Dict[Tuple, sl.Merge]]: From 5c8c4fe97f9ecd1f981bd35ac1c7e7734b17ee0b Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 22 Aug 2024 13:24:02 +0200 Subject: [PATCH 126/379] build_from_graph() now returns a dict similar to the one containin the DynapcnnLayers, but instead holding the DynapcnnLayerHandlers used to instantiate each layer --- sinabs/backend/dynapcnn/utils.py | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/sinabs/backend/dynapcnn/utils.py b/sinabs/backend/dynapcnn/utils.py index 3c4a58dd..3df4e5e2 100644 --- a/sinabs/backend/dynapcnn/utils.py +++ b/sinabs/backend/dynapcnn/utils.py @@ -120,7 +120,7 @@ def build_from_graph( edges: List[Tuple[int, int]], nodes_to_dcnnl_map: dict, weight_rescaling_fn: Callable, - entry_nodes: List[int]) -> dict: + entry_nodes: List[int]) -> Union[Dict[int, Dict[DynapcnnLayer, List]], Dict[int, Dict[DynapcnnLayerHandler, List]]]: """ Parses each edge in `edges`, where each node is a set of layer that will compose a `DynapcnnLayer`. The target destination of each `DynapcnnLayer` is computed via edges connecting nodes in different `DynapcnnLayer` instances. @@ -138,25 +138,26 @@ def build_from_graph( Returns ---------- - dynapcnn_layers (dict): `DynapcnnLayer` instances, each created from an entry in `nodes_to_dcnnl_map`. + - dynapcnnlayers_handlers (dict): `DynapcnnLayerHandler` instances, gathering network-level info. for each of the `DynapcnnLayer` instances in `dynapcnn_layers`. """ # turn each entry in `nodes_to_dcnnl_map` into a `DynapcnnLayer` instance. - dynapcnn_layers = construct_dynapcnnlayers_from_mapper(discretize, nodes_to_dcnnl_map, edges, weight_rescaling_fn, entry_nodes) + dynapcnn_layers, dynapcnnlayers_handlers = construct_dynapcnnlayers_from_mapper(discretize, nodes_to_dcnnl_map, edges, weight_rescaling_fn, entry_nodes) - # initialize attribute holding to which core a `DynapcnnLayer` instance in `dynapcnn_layers` will be mapped to. - for idx, layer_data in dynapcnn_layers.items(): + # initialize key holding to which core a `DynapcnnLayer` instance in `dynapcnn_layers` will be mapped to. + for idx, layer_data in dynapcnnlayers_handlers.items(): if 'core_idx' not in layer_data: # a `DynapcnnLayer` gets assigned a core index when `DynapcnnNetworkGraph.to()`` is called. layer_data['core_idx'] = -1 - return dynapcnn_layers + return dynapcnn_layers, dynapcnnlayers_handlers def construct_dynapcnnlayers_from_mapper( discretize: bool, nodes_to_dcnnl_map: dict, edges: List[Tuple[int, int]], weight_rescaling_fn: Callable, - entry_nodes: List[int]) -> Dict[int, Dict[DynapcnnLayer, List]]: + entry_nodes: List[int]) -> Union[Dict[int, Dict[DynapcnnLayer, List]], Dict[int, Dict[DynapcnnLayerHandler, List]]]: """ Consumes a dictionaries containing sets of layers to be used to populate a DynapcnnLayer object. Parameters @@ -172,31 +173,40 @@ def construct_dynapcnnlayers_from_mapper( Returns ---------- - dynapcnn_layers (dict): `DynapcnnLayer` instances, each created from an entry in `nodes_to_dcnnl_map`. + - dynapcnnlayers_handlers (dict): `DynapcnnLayerHandler` instances, gathering network-level info. for each of the `DynapcnnLayer` instances in `dynapcnn_layers`. """ dynapcnn_layers = {} + dynapcnnlayers_handlers = {} for dpcnnl_idx, dcnnl_data in nodes_to_dcnnl_map.items(): - # create a `DynapcnnLayerHandler` from the set of layers in `dcnnl_data`. + # create a `DynapcnnLayerHandler` from the set of layers in `dcnnl_data` - this holds network-level data required to instantiate a `DynapcnnLayer`. layerhandler = construct_layerhandler( dpcnnl_idx, discretize, edges, nodes_to_dcnnl_map, weight_rescaling_fn, entry_nodes) # create a `DynapcnnLayer` from the handler. dynapcnnlayer = construct_dynapcnnlayer(layerhandler) + # holds the layers themselvs. dynapcnn_layers[dpcnnl_idx] = { 'layer': dynapcnnlayer, 'destinations': nodes_to_dcnnl_map[dpcnnl_idx]['destinations'] } + + # holds the handlers of each layer for later use (e.g., creation of the forward pass for the `DynapcnnNetwork`). + dynapcnnlayers_handlers[dpcnnl_idx] = { + 'layer_handler': layerhandler, + 'destinations': nodes_to_dcnnl_map[dpcnnl_idx]['destinations'] + } # check if a `nn.Linear` in `dynapcnnlayer` has been turned into a `nn.Conv2d`. - node, output_shape = dynapcnnlayer.get_modified_node_io(dcnnl_data) + node, output_shape = layerhandler.get_modified_node_io(dcnnl_data) if isinstance(node, int) and isinstance(output_shape, tuple): # a `nn.Linear` has been converted into a `nn.Conv2d`: update input shape of nodes receiving from the spiking layer after it. update_nodes_io(node, output_shape, nodes_to_dcnnl_map, edges) - return dynapcnn_layers + return dynapcnn_layers, dynapcnnlayers_handlers def update_nodes_io(updated_node: int, output_shape: tuple, nodes_to_dcnnl_map: dict, edges: List[Tuple[int, int]]) -> None: """ Updates the `input_shape` entries of each node in `nodes_to_dcnnl_map` receiving as input the output of the spiking From 6ac2f390c72e017da69071da6a982e49599df5e5 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 22 Aug 2024 13:24:52 +0200 Subject: [PATCH 127/379] (WIP) DynapcnnLayerHandlers passed down to DynapcnnNetworkModule --- sinabs/backend/dynapcnn/dynapcnn_network.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index b065bfcb..2bd1d61b 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -104,7 +104,7 @@ def __init__( self._populate_nodes_io() # build `DynapcnnLayer` instances from graph edges and mapper. - self._dynapcnn_layers = build_from_graph( + self._dynapcnn_layers, self._dynapcnnlayers_handlers = build_from_graph( discretize = discretize, edges = self._sinabs_edges, nodes_to_dcnnl_map = self._nodes_to_dcnnl_map, @@ -541,7 +541,7 @@ def _get_network_module(self) -> Union[list, dict, dict]: # get connections between `DynapcnnLayer`s. dcnnl_edges = self._get_dynapcnnlayers_edges() - dcnnnet_module = DynapcnnNetworkModule(dcnnl_edges, self._dynapcnn_layers) + dcnnnet_module = DynapcnnNetworkModule(dcnnl_edges, self._dynapcnn_layers, self._dynapcnnlayers_handlers) return dcnnnet_module.dcnnl_edges, dcnnnet_module.forward_map, dcnnnet_module.merge_points, topological_sorting(dcnnl_edges) From 562c634db255bf06d12f24b3f1b6ce865aa38e77 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 22 Aug 2024 15:04:46 +0200 Subject: [PATCH 128/379] build_config() now using class DynapcnnLayerHandler to write configuration to chip --- sinabs/backend/dynapcnn/chips/dynapcnn.py | 80 ++++++++++++----------- 1 file changed, 41 insertions(+), 39 deletions(-) diff --git a/sinabs/backend/dynapcnn/chips/dynapcnn.py b/sinabs/backend/dynapcnn/chips/dynapcnn.py index 3a55063e..2cb81da2 100644 --- a/sinabs/backend/dynapcnn/chips/dynapcnn.py +++ b/sinabs/backend/dynapcnn/chips/dynapcnn.py @@ -10,6 +10,7 @@ from sinabs.backend.dynapcnn.config_builder import ConfigBuilder from sinabs.backend.dynapcnn.dvs_layer import DVSLayer, expand_to_pair from sinabs.backend.dynapcnn.dynapcnn_layer import DynapcnnLayer +from sinabs.backend.dynapcnn.dynapcnn_layer_handler import DynapcnnLayerHandler from sinabs.backend.dynapcnn.mapping import LayerConstraints import sinabs @@ -45,26 +46,26 @@ def set_kill_bits(cls, layer: DynapcnnLayer, config_dict: dict) -> dict: """ config_dict = copy.deepcopy(config_dict) - if layer.conv_layer.bias is not None: - (weights, biases) = layer.conv_layer.parameters() + if layer.conv.bias is not None: + (weights, biases) = layer.conv.parameters() else: - (weights,) = layer.conv_layer.parameters() - biases = torch.zeros(layer.conv_layer.out_channels) + (weights,) = layer.conv.parameters() + biases = torch.zeros(layer.conv.out_channels) config_dict["weights_kill_bit"] = (~weights.bool()).tolist() config_dict["biases_kill_bit"] = (~biases.bool()).tolist() # - Neuron states - if not layer.spk_layer.is_state_initialised(): + if not layer.spk.is_state_initialised(): # then we assign no initial neuron state to DYNAP-CNN. f, h, w = layer.get_neuron_shape() neurons_state = torch.zeros(f, w, h) - elif layer.spk_layer.v_mem.dim() == 4: + elif layer.spk.v_mem.dim() == 4: # 4-dimensional states should be the norm when there is a batch dim - neurons_state = layer.spk_layer.v_mem.transpose(2, 3)[0] + neurons_state = layer.spk.v_mem.transpose(2, 3)[0] else: raise ValueError( - f"Current v_mem (shape: {layer.spk_layer.v_mem.shape}) of spiking layer not understood." + f"Current v_mem (shape: {layer.spk.v_mem.shape}) of spiking layer not understood." ) config_dict["neurons_value_kill_bit"] = ( @@ -74,12 +75,12 @@ def set_kill_bits(cls, layer: DynapcnnLayer, config_dict: dict) -> dict: return config_dict @classmethod - def get_dynapcnn_layer_config_dict(cls, layer: DynapcnnLayer, layers_mapper: Dict[int, DynapcnnLayer]) -> dict: + def get_dynapcnn_layer_config_dict(cls, layer: DynapcnnLayer, layer_handler: DynapcnnLayerHandler, all_handlers: dict) -> dict: config_dict = {} config_dict["destinations"] = [{}, {}] # Update the dimensions - channel_count, input_size_y, input_size_x = layer.input_shape + channel_count, input_size_y, input_size_x = layer.in_shape dimensions = {"input_shape": {}, "output_shape": {}} dimensions["input_shape"]["size"] = {"x": input_size_x, "y": input_size_y} dimensions["input_shape"]["feature_count"] = channel_count @@ -91,24 +92,24 @@ def get_dynapcnn_layer_config_dict(cls, layer: DynapcnnLayer, layers_mapper: Dic dimensions["output_shape"]["size"]["x"] = w dimensions["output_shape"]["size"]["y"] = h dimensions["padding"] = { - "x": layer.conv_layer.padding[1], - "y": layer.conv_layer.padding[0], + "x": layer.conv.padding[1], + "y": layer.conv.padding[0], } dimensions["stride"] = { - "x": layer.conv_layer.stride[1], - "y": layer.conv_layer.stride[0], + "x": layer.conv.stride[1], + "y": layer.conv.stride[0], } - dimensions["kernel_size"] = layer.conv_layer.kernel_size[0] + dimensions["kernel_size"] = layer.conv.kernel_size[0] - if dimensions["kernel_size"] != layer.conv_layer.kernel_size[1]: + if dimensions["kernel_size"] != layer.conv.kernel_size[1]: raise ValueError("Conv2d: Kernel must have same height and width.") config_dict["dimensions"] = dimensions # Update parameters from convolution - if layer.conv_layer.bias is not None: - (weights, biases) = layer.conv_layer.parameters() + if layer.conv.bias is not None: + (weights, biases) = layer.conv.parameters() else: - (weights,) = layer.conv_layer.parameters() - biases = torch.zeros(layer.conv_layer.out_channels) + (weights,) = layer.conv.parameters() + biases = torch.zeros(layer.conv.out_channels) weights = weights.transpose(2, 3) # Need this to match samna convention config_dict["weights"] = weights.int().tolist() config_dict["biases"] = biases.int().tolist() @@ -117,36 +118,36 @@ def get_dynapcnn_layer_config_dict(cls, layer: DynapcnnLayer, layers_mapper: Dic # Update parameters from the spiking layer # - Neuron states - if not layer.spk_layer.is_state_initialised(): + if not layer.spk.is_state_initialised(): # then we assign no initial neuron state to DYNAP-CNN. f, h, w = layer.get_neuron_shape() neurons_state = torch.zeros(f, w, h) - elif layer.spk_layer.v_mem.dim() == 4: + elif layer.spk.v_mem.dim() == 4: # 4-dimensional states should be the norm when there is a batch dim - neurons_state = layer.spk_layer.v_mem.transpose(2, 3)[0] + neurons_state = layer.spk.v_mem.transpose(2, 3)[0] else: raise ValueError( - f"Current v_mem (shape: {layer.spk_layer.v_mem.shape}) of spiking layer not understood." + f"Current v_mem (shape: {layer.spk.v_mem.shape}) of spiking layer not understood." ) # - Resetting vs returning to 0 - if isinstance(layer.spk_layer.reset_fn, sinabs.activation.MembraneReset): + if isinstance(layer.spk.reset_fn, sinabs.activation.MembraneReset): return_to_zero = True - elif isinstance(layer.spk_layer.reset_fn, sinabs.activation.MembraneSubtract): + elif isinstance(layer.spk.reset_fn, sinabs.activation.MembraneSubtract): return_to_zero = False else: raise Exception( "Unknown reset mechanism. Only MembraneReset and MembraneSubtract are currently understood." ) - if layer.spk_layer.min_v_mem is None: + if layer.spk.min_v_mem is None: min_v_mem = -(2**15) else: - min_v_mem = int(layer.spk_layer.min_v_mem) + min_v_mem = int(layer.spk.min_v_mem) config_dict.update( { "return_to_zero": return_to_zero, - "threshold_high": int(layer.spk_layer.spike_threshold), + "threshold_high": int(layer.spk.spike_threshold), "threshold_low": min_v_mem, "monitor_enable": False, "neurons_initial_value": neurons_state.int().tolist(), @@ -155,10 +156,10 @@ def get_dynapcnn_layer_config_dict(cls, layer: DynapcnnLayer, layers_mapper: Dic # setting destinations config. based on destinations destination nodes of the nodes withing this `dcnnl`. destinations = [] - for node_id, destination_nodes in layer.nodes_destinations.items(): + for node_id, destination_nodes in layer_handler.nodes_destinations.items(): for dest_node in destination_nodes: - core_id = DynapcnnLayer.find_nodes_core_id(dest_node, layers_mapper) - kernel_size = layer.get_pool_kernel_size(node_id) + core_id = DynapcnnLayerHandler.find_nodes_core_id(dest_node, all_handlers) + kernel_size = layer_handler.get_pool_kernel_size(node_id) dest_data = { 'layer': core_id, @@ -175,7 +176,7 @@ def get_dynapcnn_layer_config_dict(cls, layer: DynapcnnLayer, layers_mapper: Dic return config_dict @classmethod - def write_dynapcnn_layer_config(cls, layer: DynapcnnLayer, chip_layer: "CNNLayerConfig", layers_mapper: Dict[int, DynapcnnLayer]) -> None: + def write_dynapcnn_layer_config(cls, layer: DynapcnnLayer, chip_layer: "CNNLayerConfig", layer_handler: DynapcnnLayerHandler, all_handlers: dict) -> None: """ Write a single layer configuration to the dynapcnn conf object. Uses the data in `layer` to configure a `CNNLayerConfig` to be deployed on chip. @@ -183,13 +184,12 @@ def write_dynapcnn_layer_config(cls, layer: DynapcnnLayer, chip_layer: "CNNLayer ---------- - layer (DynapcnnLayer): the layer for which the condiguration will be written. - chip_layer (CNNLayerConfig): configuration object representing the layer to which configuration is written. - - layers_mapper (dict): a dictionary with keys being the ID of each `DynapcnnLayer` and values being the layer - instance. This is used to retrieve the `.assigned_core` for each of the layers in `.dynapcnnlayer_destination` - such that `chip_layer.destinations` can be configured. + - layer_handler (DynapcnnLayerHandler): ... + - all_handlers (dict): ... """ # extracting from a DynapcnnLayer the config. variables for its CNNLayerConfig. - config_dict = cls.get_dynapcnn_layer_config_dict(layer=layer, layers_mapper=layers_mapper) + config_dict = cls.get_dynapcnn_layer_config_dict(layer=layer, layer_handler=layer_handler, all_handlers=all_handlers) # update configuration of the DYNAPCNN layer. chip_layer.dimensions = config_dict["dimensions"] @@ -238,8 +238,10 @@ def build_config(cls, model: Union["DynapcnnNetwork"]) -> DynapcnnConfiguration: pass elif isinstance(ith_dcnnl, DynapcnnLayer): - chip_layer = config.cnn_layers[ith_dcnnl.assigned_core] - cls.write_dynapcnn_layer_config(ith_dcnnl, chip_layer, model.layers_mapper) + # retrieve assigned core from the handler of this DynapcnnLayer (`ith_dcnnl`) instance. + chip_layer = config.cnn_layers[model.layers_handlers[layer_index].assigned_core] + # write core configuration. + cls.write_dynapcnn_layer_config(ith_dcnnl, chip_layer, model.layers_handlers[layer_index], model.layers_handlers) else: # shouldn't happen since type checks are made previously. From 70a5ef32f4048bf734f57272d5e494b25682f7be Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 22 Aug 2024 15:06:54 +0200 Subject: [PATCH 129/379] get_valid_mapping() now writting core ID to the handler of a DynapcnnLayer instance --- sinabs/backend/dynapcnn/config_builder.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sinabs/backend/dynapcnn/config_builder.py b/sinabs/backend/dynapcnn/config_builder.py index 01f32198..d0eb419f 100644 --- a/sinabs/backend/dynapcnn/config_builder.py +++ b/sinabs/backend/dynapcnn/config_builder.py @@ -90,14 +90,14 @@ def get_valid_mapping(cls, model: "DynapcnnNetwork") -> List[int]: pass for (dcnnl_idx, core_idx) in mapping: - # save the core index information directly in the `DynapcnnLayer` object assigned to it. - model.layers_mapper[dcnnl_idx].assigned_core = core_idx + # save the core index information on the handler of this `DynapcnnLayer` instance. + model.layers_handlers[dcnnl_idx]['layer_handler'].assigned_core = core_idx chip_layers_ordering.append(core_idx) else: raise InvalidModel(model) - # return kept but its information is not used beyond this point (core indices already part of each `DynapcnnLayer` instance). + # return kept but its information is not used beyond this point (core indices already part of each `DynapcnnLayerHandler` instance). return chip_layers_ordering @classmethod From 3c536b480feb05c3db5dd165e6dc4d21e491ac94 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 22 Aug 2024 15:07:29 +0200 Subject: [PATCH 130/379] giving access to DynapcnnLayerHandler --- sinabs/backend/dynapcnn/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sinabs/backend/dynapcnn/__init__.py b/sinabs/backend/dynapcnn/__init__.py index 4c32f1ee..ff0658a1 100644 --- a/sinabs/backend/dynapcnn/__init__.py +++ b/sinabs/backend/dynapcnn/__init__.py @@ -6,4 +6,8 @@ DynapcnnLayer, ) +from .dynapcnn_layer_handler import ( + DynapcnnLayerHandler, +) + from .dynapcnn_visualizer import DynapcnnVisualizer From 44b0b9662648fb4d20c6904decb41dae94b0f432 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 22 Aug 2024 15:15:02 +0200 Subject: [PATCH 131/379] self.dynapcnnlayers_handlers used for chip configuration and removed from the instance if deployment is successfull --- sinabs/backend/dynapcnn/dynapcnn_network.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index 2bd1d61b..6c80d7b3 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -109,7 +109,7 @@ def __init__( edges = self._sinabs_edges, nodes_to_dcnnl_map = self._nodes_to_dcnnl_map, weight_rescaling_fn = weight_rescaling_fn, - entry_nodes = self._entry_nodes) + entry_nodes = self._entry_nodes) # these gather all data necessay to implement the forward method for this class. self._dcnnl_edges, self._layers_mapper, self._merge_points, self._topological_order = self._get_network_module() @@ -141,6 +141,11 @@ def topological_order(self): def layers_mapper(self) -> Dict[int, DynapcnnLayer]: return self._layers_mapper + @property + def layers_handlers(self): + # @TODO this will be removed after a success call to `.to()` so need to check if it exists before return. + return self._dynapcnnlayers_handlers + @property def chip_layers_ordering(self): return self._chip_layers_ordering @@ -470,7 +475,7 @@ def _make_config( has_dvs_layer = isinstance(self._layers_mapper[0], DVSLayer) if chip_layers_ordering == "auto": - # figure out mapping of each DynapcnnLayer into one core (core ID will be set in the layer instance via `layer.assigned_core`). + # figure out mapping of each `DynapcnnLayer` into one core (core ID will be set in the layer's handler instance via `.assigned_core`). _ = config_builder.get_valid_mapping(self) else: @@ -479,7 +484,7 @@ def _make_config( # TODO not handling DVSLayer yet. pass - # update config. + # update config (config. DynapcnnLayer instances into their assigned core). config = config_builder.build_config(self) # TODO not handling DVSLayer yet (this is from the old implementation, should be revised). @@ -493,16 +498,16 @@ def _make_config( for dcnnl_index, ith_dcnnl in self._layers_mapper.items(): # TODO if a network with two output layers is deployed, which is not supported yet btw, this monitoring part needs to be revised. - if len(ith_dcnnl.dynapcnnlayer_destination) == 0: + if len(self._dynapcnnlayers_handlers[dcnnl_index].dynapcnnlayer_destination) == 0: # a DynapcnnLayer without destinations is taken to be the output layer of the network. - monitor_chip_layers.append(ith_dcnnl.assigned_core) + monitor_chip_layers.append(self._dynapcnnlayers_handlers[dcnnl_index].assigned_core) elif monitor_layers == "all": for dcnnl_index, ith_dcnnl in self._layers_mapper.items(): # TODO not handling DVSLayer yet # monitor each chip core (if not a DVSLayer). if not isinstance(ith_dcnnl, DVSLayer): - monitor_chip_layers.append(ith_dcnnl.assigned_core) + monitor_chip_layers.append(self._dynapcnnlayers_handlers[dcnnl_index].assigned_core) if monitor_layers: if "dvs" in monitor_layers: @@ -518,6 +523,9 @@ def _make_config( if config_builder.validate_configuration(config): # validate config. print("Network is valid: \n") + + # successfull chip configuration: information from handlers no longer necessary. + del self._dynapcnnlayers_handlers return config else: From c6a829f78fd5abdc6416cfbdc8ec305665a7c55f Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 22 Aug 2024 15:16:45 +0200 Subject: [PATCH 132/379] find_core_id() modified to use handlers dict instead of DynapcnnNetwork._layers_mapper --- sinabs/backend/dynapcnn/dynapcnn_layer_handler.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer_handler.py b/sinabs/backend/dynapcnn/dynapcnn_layer_handler.py index 9bb19f28..9e1566bf 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer_handler.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer_handler.py @@ -389,11 +389,13 @@ def get_pool_kernel_size(self, node: int) -> int: raise ValueError(f'Node {node} does not belong to DynapcnnLayer {self.dpcnnl_index}.') @staticmethod - def find_nodes_core_id(node: int, forward_map: dict) -> int: + def find_nodes_core_id(node: int, all_handlers: dict) -> int: + """ Loops through all handlers in `all_handlers` to find to which core a `DynapcnnLayer` containing + `node` has been assigned to. """ - for _, dcnnl in forward_map.items(): + for _, dcnnl in all_handlers.items(): - if node == dcnnl.conv_node_id or node == dcnnl.spk_node_id or node in dcnnl.pool_node_id: - return dcnnl.assigned_core + if node == dcnnl['layer_handler'].conv_node_id or node == dcnnl['layer_handler'].spk_node_id or node in dcnnl['layer_handler'].pool_node_id: + return dcnnl['layer_handler'].assigned_core raise ValueError(f'Node {node} not found in any of the cores.') From 16ac0ca9ba5d327b547df1c1fb258855dc3aba33 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 22 Aug 2024 15:24:33 +0200 Subject: [PATCH 133/379] (WIP) handlers needed in the forward function - both HW and 'torch's forward function need validation --- sinabs/backend/dynapcnn/dynapcnn_network.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index 6c80d7b3..e97acfb5 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -143,7 +143,6 @@ def layers_mapper(self) -> Dict[int, DynapcnnLayer]: @property def layers_handlers(self): - # @TODO this will be removed after a success call to `.to()` so need to check if it exists before return. return self._dynapcnnlayers_handlers @property @@ -220,7 +219,7 @@ def forward(self, x): for i in self._topological_order: - if self._layers_mapper[i].entry_point: + if self._dynapcnnlayers_handlers[i].entry_point: # `DynapcnnLayer i` is an entry point of the network. layers_outputs[i] = self._layers_mapper[i](x) @@ -235,8 +234,8 @@ def forward(self, x): # find which returned tensor from the `forward` call of DynapcnnLayers `arg1` and `arg2` are to be fed # to the target DynapcnnLayer `i`. - return_index_arg1 = self._layers_mapper[arg1].get_destination_dcnnl_index(i) - return_index_arg2 = self._layers_mapper[arg2].get_destination_dcnnl_index(i) + return_index_arg1 = self._dynapcnnlayers_handlers[arg1].get_destination_dcnnl_index(i) + return_index_arg2 = self._dynapcnnlayers_handlers[arg2].get_destination_dcnnl_index(i) # retrieve input tensors to `Merge`. _arg1 = layers_outputs[arg1][return_index_arg1] @@ -256,7 +255,7 @@ def forward(self, x): # find which returned tensor from the `forward` call of the source DynapcnnLayer `src_dcnnl` is to be fed # to the target DynapcnnLayer `i`. - return_index = self._layers_mapper[src_dcnnl].get_destination_dcnnl_index(i) + return_index = self._dynapcnnlayers_handlers[src_dcnnl].get_destination_dcnnl_index(i) # call the forward. layers_outputs[i] = self._layers_mapper[i](layers_outputs[src_dcnnl][return_index]) @@ -523,9 +522,6 @@ def _make_config( if config_builder.validate_configuration(config): # validate config. print("Network is valid: \n") - - # successfull chip configuration: information from handlers no longer necessary. - del self._dynapcnnlayers_handlers return config else: From 7d57984e7003f674a3d1797f02d4fd2d94378eb6 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 22 Aug 2024 15:25:33 +0200 Subject: [PATCH 134/379] deprecated previous implementation of DynapcnnLayer (with network-level knowledge) --- sinabs/backend/dynapcnn/dynapcnn_layer_old.py | 500 ++++++++++++++++++ 1 file changed, 500 insertions(+) create mode 100644 sinabs/backend/dynapcnn/dynapcnn_layer_old.py diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer_old.py b/sinabs/backend/dynapcnn/dynapcnn_layer_old.py new file mode 100644 index 00000000..caca930a --- /dev/null +++ b/sinabs/backend/dynapcnn/dynapcnn_layer_old.py @@ -0,0 +1,500 @@ +# author : Willian Soares Girao +# contact : wsoaresgirao@gmail.com + +from copy import deepcopy +from typing import Dict, Callable, Tuple, Union, List + +import numpy as np +import torch +from torch import nn + +import sinabs.activation +import sinabs.layers as sl + +from .discretize import discretize_conv_spike_ + +class DynapcnnLayer(nn.Module): + """ + Create a `DynapcnnLayer` object representing a layer on a Speck device. + + Parameters + ---------- + - dpcnnl_index (int): the index/ID that will be associated with a `DynapcnnLayer` instance. This integer indexes a `dict` within `dcnnl_data` + that comprises a set of layers (`nn.Module`) and their respective I/O tensor shapes. + - dcnnl_data (dict): contains the nodes to be merged into this `DynapcnnLayer`, their I/O shapes and the index of the other `DynapcnnLayer`s to + be set as destinations. The `int` keys correspond to the nodes IDs associated `nn.Module`s (a single layer in the original network) becoming + part of this `DynapcnnLayer` instance, while the `str` keys correspond to this instance's destinations and re-scaling factors. + - discretize (bool): whether or not the weights/neuron parameters of the model will be quantized. + - sinabs_edges (list): each `nn.Module` within `dcnnl_data` is a node in the original computational graph describing a spiking network. An edge + `(A, B)` describes how modules forward data amongst themselves. This list is used by a `DynapcnnLayer` to figure out the number and + sequence of output tesnors its forward method needs to return. + - weight_rescaling_fn (callable): a method that handles how the re-scaling factor for one or more `SumPool2d` projecting to + the same convolutional layer are combined/re-scaled before applying them. + - entry_nodes (list): node IDs corresponding to layers in the original network that are input nodes (i.e., a "point of entry" for the external data). + """ + + def __init__( + self, + dpcnnl_index: int, + dcnnl_data: Dict[Union[int, str], Union[Dict[str, Union[nn.Module, Tuple[int, int, int], Tuple[int, int, int]]], List[int]]], + discretize: bool, + sinabs_edges: List[Tuple[int, int]], + weight_rescaling_fn: Callable, + entry_nodes: List[int] + ): + super().__init__() + + self.dpcnnl_index = dpcnnl_index + self.assigned_core = None + self.entry_point = False + + if 'core_idx' in dcnnl_data: + self.assigned_core = dcnnl_data['core_idx'] + + self._lin_to_conv_conversion = False + + conv = None + self.conv_node_id = None + self.conv_in_shape = None + self.conv_out_shape = None + + spk = None + self.spk_node_id = None + + pool = [] + self.pool_node_id = [] + self.conv_rescaling_factor = None + + self.dynapcnnlayer_destination = dcnnl_data['destinations'] + + for key, value in dcnnl_data.items(): + if isinstance(key, int): + # value has data pertaining a node (torch/sinabs layer). + if isinstance(value['layer'], sl.IAFSqueeze): + spk = value['layer'] + self.spk_node_id = key + elif isinstance(value['layer'], nn.Linear) or isinstance(value['layer'], nn.Conv2d): + conv = value['layer'] + self.conv_node_id = key + elif isinstance(value['layer'], sl.SumPool2d): + pool.append(value['layer']) + self.pool_node_id.append(key) + else: + raise ValueError(f'Node {key} has not valid layer associated with it.') + + if not conv: + raise ValueError(f'Convolution layer not present.') + + if not spk: + raise ValueError(f'Spiking layer not present.') + + spk = deepcopy(spk) + if spk.is_state_initialised(): + # TODO this line bellow is causing an exception on `.v_men.shape` to be raised in `.get_layer_config_dict()`. Find out why. + # spk.v_mem = spk.v_mem.data.unsqueeze(-1).unsqueeze(-1) # expand dims. + + # TODO hacky stuff: make it better (THIS SEEMS TO BE FIXING THE PROBLEM ABOVE THO). + if len(list(spk.v_mem.shape)) != 4: + spk.v_mem = spk.v_mem.data.unsqueeze(-1).unsqueeze(-1) # expand dims. + + if isinstance(conv, nn.Linear): + # A `nn.Linear` needs to be converted into `nn.Conv2d`. The I/O shapes of the spiking layer are updated + # accordingly following the conversion. + + conv, conv_in_shape = self._convert_linear_to_conv(conv, dcnnl_data[self.conv_node_id]) + + # the original `nn.Linear` output shape becomes the equivalent `nn.Conv2d` shape. + self.conv_out_shape = self._update_conv_node_output_shape( + conv_layer=conv, layer_data=dcnnl_data[self.conv_node_id], input_shape=conv_in_shape) + + # the I/O shapes for neuron layer following the new conv need also to be updated. + self._update_neuron_node_output_shape(spiking_layer_data=dcnnl_data[self.spk_node_id], conv_out_shape=self.conv_out_shape) + + else: + self.conv_out_shape = dcnnl_data[self.conv_node_id]['output_shape'] + conv = deepcopy(conv) + + # check if convolution kernel is a square. + if conv.kernel_size[0] != conv.kernel_size[1]: + raise ValueError('The kernel of a `nn.Conv2d` must have the same height and width.') + + # input shape of conv layer. + self.conv_in_shape = dcnnl_data[self.conv_node_id]['input_shape'] + # input shape of the `DynapcnnLayer` instance. + self.input_shape = self.conv_in_shape + + # this weight rescale comes from the node projecting into this 'conv' node. + if len(dcnnl_data['conv_rescale_factor']): + # this means an `AvgPool2d` has been converted into a `SumPool2d`. + self.conv_rescaling_factor = weight_rescaling_fn(dcnnl_data['conv_rescale_factor']) + conv.weight.data = (conv.weight.data / self.conv_rescaling_factor).clone().detach() + else: + # this means `SumPool2d` have been used from the start. + conv.weight.data = (conv.weight.data).clone().detach() + + # int conversion is done while writing the config. + if discretize: + conv, spk = discretize_conv_spike_(conv, spk, to_int=False) + + # consolidate layers. + self.conv_layer = conv + self.spk_layer = spk + self.pool_layer = [] + + if len(pool) != 0: + # the 1st pooling targets the 1st destination in `dcnnl_data['destinations']`, the 2nd pooling targets the 2nd destination... + for plyr in pool: + # @TODO POSSIBLE INCONSISTENCY: if the `SumPool2d` is the result of a conversion from `AvgPool2d` then `SumPool2d.kernel_size` + # is of type tuple, otherwise it is an int. + if isinstance(plyr.kernel_size, tuple) and plyr.kernel_size[0] != plyr.kernel_size[1]: + raise ValueError("Only square kernels are supported") + self.pool_layer.append(deepcopy(plyr)) + + # map destination nodes for each layer in this instance. + self.nodes_destinations = self._get_destinations_input_source(sinabs_edges) + + # flag if the instance is an entry point (i.e., an input node of the network). + if self.conv_node_id in entry_nodes: + self.entry_point = True + + ####################################################### Public Methods ####################################################### + + def get_neuron_shape(self) -> Tuple[int, int, int]: + """Return the output shape of the neuron layer. + + Returns + ------- + features, height, width + """ + # same as the convolution's output. + return self.conv_out_shape + + def get_destination_dcnnl_index(self, dcnnl_id: int) -> int: + """ The `forward` method will return as many tensors as there are elements in `self.dynapcnnlayer_destination`. Since the i-th returned tensor is to be + fed to the i-th destionation in `self.dynapcnnlayer_destination`, the return of this method can be used to index a tensor returned in the `forward` method. + + Parameters + ---------- + - dcnnl_id (int): this should be one of the values listed within `self.dynapcnnlayer_destination`. + + Returns + ---------- + - The index of `dcnnl_id` within `self.dynapcnnlayer_destination`. + """ + return self.dynapcnnlayer_destination.index(dcnnl_id) + + def forward(self, x): + """Torch forward pass. + + Returns + ---------- + - forward output (tuple): returns as many tensors as there are destinations associated with this instance. The returned + tensors always follows the sequence `return spiking_layer_output, pooling_layer_1, pooling_layer_2, ...`. + + Example + ---------- + TODO this example needs to be revised because the spiking layer only appears if it is projecting to outside this layer. + + - With `self.nodes_destinations = {1: [5, 8], 2: [4], 3: [7]}` this method will return 4 tensors (each to be sent to a different DynapcnnLayer): the 1st + and 2nd are the outputs of the spiking layer (`node 1`); the 3rd is the output of the first pooling layer (`node 2`, receiving the output of node 1) appearing + right after the spiking layer as defined in the original `nn.Module` being converted to a `DynapcnnNetwork`; the 4th is the output of the second pooling + layer (`node 3`) to appear after the spiking layer. Thus, the return will be `return node_1_out, node_1_out, node_2_out, node_3_out`. This means the edges + in the computational graph involved in this mapping were: + + - 2 --> 4 # `4` is a conv layer belonging to another DynapcnnLayer X. + - 3 --> 7 # `7` is a conv layer belonging to another DynapcnnLayer Y. + """ + + returns = [] + + x = self.conv_layer(x) + x = self.spk_layer(x) + + # pooling layers are sequentially added to `self.pool_layer` so we'll pass data to them sequentially. + pooling_indexer = 0 + + # building return set of all layers as they appear in `self.nodes_destinations`. + for node_id, destination_node_list in self.nodes_destinations.items(): + if node_id == self.spk_node_id: + # spiking output for each node outside this DynapcnnLayer receiving from its spiking layer. + for _ in destination_node_list: + returns.append(x) + else: + # returns of each pooling layer are arranged sequenatially. + for _ in destination_node_list: + ith_pool_output = self.pool_layer[pooling_indexer](x) + returns.append(ith_pool_output) + + # forward through next pooling layer in `self.pool_layer` in the next iteration. + pooling_indexer += 1 + + if len(returns) != len(self.dynapcnnlayer_destination): + raise ValueError(f'Number of returned tensors ({len(returns)}) differ from the number of destinations ({len(self.dynapcnnlayer_destination)}).') + + if len(returns) == 0 and len(self.pool_layer) == 0: + # this is the output layer and there's no pooling after the neurons. + returns.append(x) + elif len(returns) == 0 and len(self.pool_layer) == 1: + # this is the output layer and there's 1 pooling after the neurons. + returns.append(self.pool_layer[0](x)) + elif len(returns) == 0 and len(self.pool_layer) > 1: + raise ValueError(f'Output DynapcnnLayer starting with node {self.conv_node_id} has {len(self.pool_layer)} pooling layers: it should have either 1 or none.') + else: + pass + + return tuple(returns) + + def get_modified_node_io(self, dcnnl_data: dict) -> Union[Tuple[int, tuple], Tuple[None, None]]: + """ Follwing a conversion, the I/O shapes of the spiking layer have been updated to match the convolution's + output. Thus, all nodes receiving input from this spiking layer need their input shapes updated. + + Parameters + ---------- + - dcnnl_data (dict): the set of layers grouped together to comprise this instance of a `DynapcnnLayer`. + + Returns + ---------- + - node ID (int): the ID of the spiking layer consuming the tunerd layer's output (`None` if there was no conversion). + - output shape (tuple): the new output shape following a converstion from `nn.Linear` to `nn.Conv2d` (`None` if there was no conversion). + """ + if self._lin_to_conv_conversion: + return self.spk_node_id, dcnnl_data[self.spk_node_id]['output_shape'] + return None, None + + def zero_grad(self, set_to_none: bool = False) -> None: + return self.spk_layer.zero_grad(set_to_none) + + def get_conv_output_shape(self, conv_layer: nn.Conv2d, input_shape: Tuple[int, int, int]) -> Tuple[int, int, int]: + """ Computes the output dimensions of `conv_layer`. + + Parameters + ---------- + - conv_layer (nn.Conv2d): conv. layer whose output will be computed for. + - input_shape (tuple): the shape for the input tensor the layer will process. + + Returns + ---------- + - output dimensions (tuple): a tuple describing `(output channels, height, width)`. + """ + # get the layer's parameters. + out_channels = conv_layer.out_channels + kernel_size = conv_layer.kernel_size + stride = conv_layer.stride + padding = conv_layer.padding + dilation = conv_layer.dilation + + # compute the output height and width. + out_height = ((input_shape[1] + 2 * padding[0] - dilation[0] * (kernel_size[0] - 1) - 1) // stride[0]) + 1 + out_width = ((input_shape[2] + 2 * padding[1] - dilation[1] * (kernel_size[1] - 1) - 1) // stride[1]) + 1 + + return (out_channels, out_height, out_width) + + def summary(self) -> dict: + """ Returns a summary of the convolution's/pooling's kernel sizes and the output shape of the spiking layer.""" + # TODO I can't see pooling being used in checking memory constraints by the builder so I'm ignoring for now the fact that multiple pooling could exist. + + _pool = None + + # @TODO POSSIBLE INCONSISTENCY: if the `SumPool2d` is the result of a conversion from `AvgPool2d` then `SumPool2d.kernel_size` + # is of type tuple, otherwise it is an int. + if len(self.pool_layer) != 0: + # @TODO ignoring for now that there could be multiple poolings (just use the first one). + if isinstance(self.pool_layer[0].kernel_size, tuple): + _pool = list(self.pool_layer[0].kernel_size) + elif isinstance(self.pool_layer[0].kernel_size, int): + _pool = [self.pool_layer[0].kernel_size, self.pool_layer[0].kernel_size] + else: + raise ValueError('Type of `self.pool_layer[0].kernel_size` not understood.') + + return { + "pool": (_pool), + "kernel": list(self.conv_layer.weight.data.shape), + "neuron": self.conv_out_shape, # neuron layer output has the same shape as the convolution layer ouput. + } + + def memory_summary(self): + """Computes the amount of memory required for each of the components. Note that this is not + necessarily the same as the number of parameters due to some architecture design + constraints. + + .. math:: + + K_{MT} = c \\cdot 2^{\\lceil \\log_2\\left(k_xk_y\\right) \\rceil + \\lceil \\log_2\\left(f\\right) \\rceil} + + .. math:: + + N_{MT} = f \\cdot 2^{ \\lceil \\log_2\\left(f_y\\right) \\rceil + \\lceil \\log_2\\left(f_x\\right) \\rceil } + + Returns + ------- + A dictionary with keys kernel, neuron and bias and the corresponding memory sizes + """ + summary = self.summary() + f, c, h, w = summary["kernel"] + f, neuron_height, neuron_width = self.conv_out_shape # neuron layer output has the same shape as the convolution layer ouput. + + return { + "kernel": c * pow(2, np.ceil(np.log2(h * w)) + np.ceil(np.log2(f))), + "neuron": f + * pow(2, np.ceil(np.log2(neuron_height)) + np.ceil(np.log2(neuron_width))), + "bias": 0 if self.conv_layer.bias is None else len(self.conv_layer.bias), + } + + def __str__(self): + pretty_print = '\n' + + pretty_print += 'COMPUTATIONAL NODES:\n\n' + + pretty_print += f'(node {self.conv_node_id}): {self.conv_layer}\n' + pretty_print += f'(node {self.spk_node_id}): {self.spk_layer}' + if len(self.pool_layer) != 0: + for idx, lyr in enumerate(self.pool_layer): + pretty_print += f'\n(node {self.pool_node_id[idx]}): {lyr}' + + pretty_print += '\n\nMETADATA:\n' + pretty_print += f'\n> network\'s entry point: {self.entry_point}' + pretty_print += f'\n> convolution\'s weight re-scaling factor: {self.conv_rescaling_factor}' + pretty_print += f'\n> assigned core index: {self.assigned_core}' + pretty_print += f'\n> destination DynapcnnLayers: {self.dynapcnnlayer_destination}' + + for node, destinations in self.nodes_destinations.items(): + pretty_print += f'\n> node {node} feeds input to nodes {destinations}' + + return pretty_print + + ####################################################### Private Methods ####################################################### + + def _update_neuron_node_output_shape(self, spiking_layer_data: dict, conv_out_shape: tuple) -> None: + """ Updates the spiking layer's I/O shapes after the conversion of a `nn.Linear` into a `nn.Conv2d` (to match the convolution's output). + + Parameters + ---------- + - spiking_layer_data (dict): the dictionary containing all data regarding the spiking layer. + - conv_out_shape (tuple): the output shape of the convolution layer preceeding the spiking layer. + """ + + # spiking layer consumes the tensor coming out of the conv. layer. + spiking_layer_data['input_shape'] = conv_out_shape + # spiking layer outputs the same shape as the conv. layer. + spiking_layer_data['output_shape'] = spiking_layer_data['input_shape'] + + def _update_conv_node_output_shape(self, conv_layer: nn.Conv2d, layer_data: dict, input_shape: Tuple[int, int, int]) -> Tuple[int, int, int]: + """ Updates the shape of the output tensor of a node that used to be a `nn.Linear` and became a `nn.Conv2d`. + + The input shapes to nodes are extracted using a list of edges by finding the output shape of the 1st element + in the edge and setting it as the input shape to the 2nd element in the edge. If a node used to be a `nn.Linear` + and it became a `nn.Conv2d`, output shape in the mapper needs to be updated, otherwise there will be a mismatch + between its output and the input it provides to another node. + + Parameters + ---------- + - conv_layer (nn.Module): the `nn.Conv2d` created from a `nn.Linear`. + - layer_data (dict): the dictionary containing the data associated with the original `nn.Linear` converted into `nn.Conv2d`. + - input_shape (tuple): the input shape the layer expects. + + Returns + ---------- + - output_shape (tuple): the tensor shape produced by the `nn.Conv2d` created from a `nn.Linear`. + """ + layer_data['output_shape'] = self.get_conv_output_shape(conv_layer, input_shape) + + return layer_data['output_shape'] + + def _convert_linear_to_conv(self, lin: nn.Linear, layer_data: dict) -> Tuple[nn.Conv2d, Tuple[int, int, int]]: + """ Convert Linear layer to Conv2d. + + Parameters + ---------- + - lin (nn.Linear): linear layer to be converted. + + Returns + ------- + - nn.Conv2d: convolutional layer equivalent to `lin`. + - input_shape (tuple): the tensor shape the layer expects. + """ + # this flags the necessity to update the I/O shape pre-computed for each of the original layers being compressed within a `DynapcnnLayer` instance. + self._lin_to_conv_conversion = True + + input_shape = layer_data['input_shape'] + + in_chan, in_h, in_w = input_shape + + if lin.in_features != in_chan * in_h * in_w: + raise ValueError("Shapes don't match.") + + layer = nn.Conv2d( + in_channels=in_chan, + kernel_size=(in_h, in_w), + out_channels=lin.out_features, + padding=0, + bias=lin.bias is not None, + ) + + if lin.bias is not None: + layer.bias.data = lin.bias.data.clone().detach() + + layer.weight.data = ( + lin.weight.data.clone() + .detach() + .reshape((lin.out_features, in_chan, in_h, in_w)) + ) + + return layer, input_shape + + def _get_destinations_input_source(self, sinabs_edges: list) -> dict: + """ Creates a mapping between each layer in this `DynapcnnLayer` instance and its targe nodes that are part of different + `DynapcnnLayer` instances. This mapping is used to figure out how many tensors the `forward` method needs to return. + + Parameters + ---------- + - sinabs_edges (list): each `nn.Module` within `dcnnl_data` is a node in the original computational graph describing a spiking + network. An edge `(A, B)` describes how modules forward data amongst themselves. This list is used by a `DynapcnnLayer` to + figure out the number and sequence of output tesnors its forward method needs to return. + + Returns + ---------- + - destinations_input_source (dict): maps a `nn.Module` within this `DynapcnnLayer` to the nodes it provides the input to. + """ + destinations_input_source = {} + + # check whether spiking layer projects outside this DynapcnnLayer (i.e. to one of the destinations without passing through a pooling). + spk_destinations = [] + for edge in sinabs_edges: + if edge[0] == self.spk_node_id and edge[1] not in self.pool_node_id: + # spiking layer projects to a node outside this DynapcnnLayer. + spk_destinations.append(edge[1]) + if len(spk_destinations) > 0: + destinations_input_source[self.spk_node_id] = [] + for node_id in spk_destinations: + destinations_input_source[self.spk_node_id].append(node_id) + + # get `pooling->destination` mapping. The pooling outputs will be arranged sequentially since the pooling layers are added sequentially + # to `self.pool_layer` (i.e., as they appear in the computational graph of the original `nn.Module`). + for id in self.pool_node_id: + destinations_input_source[id] = [] + for edge in sinabs_edges: + if edge[0] == id: + destinations_input_source[id].append(edge[1]) + + return destinations_input_source + + def get_pool_kernel_size(self, node: int) -> int: + """ Returns the pooling kernel size if `node` is a pooling layer.""" + + if node in self.pool_node_id: + i = self.pool_node_id.index(node) + return self.pool_layer[i].kernel_size[0] if isinstance(self.pool_layer[i].kernel_size, tuple) else self.pool_layer[i].kernel_size + elif node == self.spk_node_id: + return 1 + else: + raise ValueError(f'Node {node} does not belong to DynapcnnLayer {self.dpcnnl_index}.') + + @staticmethod + def find_nodes_core_id(node: int, forward_map: dict) -> int: + + for _, dcnnl in forward_map.items(): + + if node == dcnnl.conv_node_id or node == dcnnl.spk_node_id or node in dcnnl.pool_node_id: + return dcnnl.assigned_core + + raise ValueError(f'Node {node} not found in any of the cores.') From a1cecd40d0edef2590ce2e67955011a1e405d1aa Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 13 Sep 2024 13:17:51 +0200 Subject: [PATCH 135/379] refactor: DynapcnnLayer constructor - Making copies of layers provided as arguments - Converting linear layers into conv layers - Performing conv weight re-scaling - utils.construct_dynapcnnlayer() small update (accessing handling during DynapcnnLayer instantiation) --- sinabs/backend/dynapcnn/dynapcnn_layer.py | 474 ++++++---------------- sinabs/backend/dynapcnn/utils.py | 16 +- 2 files changed, 129 insertions(+), 361 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer.py b/sinabs/backend/dynapcnn/dynapcnn_layer.py index caca930a..63b8170d 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer.py @@ -14,280 +14,110 @@ from .discretize import discretize_conv_spike_ class DynapcnnLayer(nn.Module): - """ - Create a `DynapcnnLayer` object representing a layer on a Speck device. + """Create a DynapcnnLayer object representing a layer on DynapCNN or Speck. + + Requires a convolutional layer, a sinabs spiking layer and a list of + pooling values. The layers are used in the order conv -> spike -> pool. Parameters ---------- - - dpcnnl_index (int): the index/ID that will be associated with a `DynapcnnLayer` instance. This integer indexes a `dict` within `dcnnl_data` - that comprises a set of layers (`nn.Module`) and their respective I/O tensor shapes. - - dcnnl_data (dict): contains the nodes to be merged into this `DynapcnnLayer`, their I/O shapes and the index of the other `DynapcnnLayer`s to - be set as destinations. The `int` keys correspond to the nodes IDs associated `nn.Module`s (a single layer in the original network) becoming - part of this `DynapcnnLayer` instance, while the `str` keys correspond to this instance's destinations and re-scaling factors. - - discretize (bool): whether or not the weights/neuron parameters of the model will be quantized. - - sinabs_edges (list): each `nn.Module` within `dcnnl_data` is a node in the original computational graph describing a spiking network. An edge - `(A, B)` describes how modules forward data amongst themselves. This list is used by a `DynapcnnLayer` to figure out the number and - sequence of output tesnors its forward method needs to return. - - weight_rescaling_fn (callable): a method that handles how the re-scaling factor for one or more `SumPool2d` projecting to - the same convolutional layer are combined/re-scaled before applying them. - - entry_nodes (list): node IDs corresponding to layers in the original network that are input nodes (i.e., a "point of entry" for the external data). + conv: torch.nn.Conv2d or torch.nn.Linear + Convolutional or linear layer + (linear will be converted to convolutional) + spk: sinabs.layers.IAFSqueeze + Sinabs IAF layer + in_shape: tuple of int + The input shape, needed to create dynapcnn configs if the network + does not contain an input layer. Convention: (features, height, width) + pool: List of integers + Each integer entry represents an output (destination on chip) and + whether pooling should be applied (values > 1) or not (values equal + to 1). The number of entries determines the number of tensors the + layer's forward method returns. + discretize: bool + Whether to discretize parameters. + rescale_weights: int + Layer weights will be divided by this value. """ def __init__( self, - dpcnnl_index: int, - dcnnl_data: Dict[Union[int, str], Union[Dict[str, Union[nn.Module, Tuple[int, int, int], Tuple[int, int, int]]], List[int]]], - discretize: bool, - sinabs_edges: List[Tuple[int, int]], - weight_rescaling_fn: Callable, - entry_nodes: List[int] + conv: nn.Conv2d, + spk: sl.IAFSqueeze, + in_shape: Tuple[int, int, int], + pool: List[int], + discretize: bool = True, + rescale_weights: int = 1, ): super().__init__() - self.dpcnnl_index = dpcnnl_index - self.assigned_core = None - self.entry_point = False - - if 'core_idx' in dcnnl_data: - self.assigned_core = dcnnl_data['core_idx'] - - self._lin_to_conv_conversion = False + self.in_shape = in_shape + self.pool = pool + self.discretize = discretize + self.rescale_weights = rescale_weights - conv = None - self.conv_node_id = None - self.conv_in_shape = None - self.conv_out_shape = None - - spk = None - self.spk_node_id = None - - pool = [] - self.pool_node_id = [] - self.conv_rescaling_factor = None - - self.dynapcnnlayer_destination = dcnnl_data['destinations'] - - for key, value in dcnnl_data.items(): - if isinstance(key, int): - # value has data pertaining a node (torch/sinabs layer). - if isinstance(value['layer'], sl.IAFSqueeze): - spk = value['layer'] - self.spk_node_id = key - elif isinstance(value['layer'], nn.Linear) or isinstance(value['layer'], nn.Conv2d): - conv = value['layer'] - self.conv_node_id = key - elif isinstance(value['layer'], sl.SumPool2d): - pool.append(value['layer']) - self.pool_node_id.append(key) - else: - raise ValueError(f'Node {key} has not valid layer associated with it.') - - if not conv: - raise ValueError(f'Convolution layer not present.') - - if not spk: - raise ValueError(f'Spiking layer not present.') - spk = deepcopy(spk) - if spk.is_state_initialised(): - # TODO this line bellow is causing an exception on `.v_men.shape` to be raised in `.get_layer_config_dict()`. Find out why. - # spk.v_mem = spk.v_mem.data.unsqueeze(-1).unsqueeze(-1) # expand dims. - - # TODO hacky stuff: make it better (THIS SEEMS TO BE FIXING THE PROBLEM ABOVE THO). - if len(list(spk.v_mem.shape)) != 4: - spk.v_mem = spk.v_mem.data.unsqueeze(-1).unsqueeze(-1) # expand dims. if isinstance(conv, nn.Linear): - # A `nn.Linear` needs to be converted into `nn.Conv2d`. The I/O shapes of the spiking layer are updated - # accordingly following the conversion. - - conv, conv_in_shape = self._convert_linear_to_conv(conv, dcnnl_data[self.conv_node_id]) - - # the original `nn.Linear` output shape becomes the equivalent `nn.Conv2d` shape. - self.conv_out_shape = self._update_conv_node_output_shape( - conv_layer=conv, layer_data=dcnnl_data[self.conv_node_id], input_shape=conv_in_shape) - - # the I/O shapes for neuron layer following the new conv need also to be updated. - self._update_neuron_node_output_shape(spiking_layer_data=dcnnl_data[self.spk_node_id], conv_out_shape=self.conv_out_shape) - + conv = self._convert_linear_to_conv(conv) + if spk.is_state_initialised(): + # Expand dims + spk.v_mem = spk.v_mem.data.unsqueeze(-1).unsqueeze(-1) else: - self.conv_out_shape = dcnnl_data[self.conv_node_id]['output_shape'] conv = deepcopy(conv) - # check if convolution kernel is a square. - if conv.kernel_size[0] != conv.kernel_size[1]: - raise ValueError('The kernel of a `nn.Conv2d` must have the same height and width.') - - # input shape of conv layer. - self.conv_in_shape = dcnnl_data[self.conv_node_id]['input_shape'] - # input shape of the `DynapcnnLayer` instance. - self.input_shape = self.conv_in_shape - - # this weight rescale comes from the node projecting into this 'conv' node. - if len(dcnnl_data['conv_rescale_factor']): - # this means an `AvgPool2d` has been converted into a `SumPool2d`. - self.conv_rescaling_factor = weight_rescaling_fn(dcnnl_data['conv_rescale_factor']) - conv.weight.data = (conv.weight.data / self.conv_rescaling_factor).clone().detach() - else: - # this means `SumPool2d` have been used from the start. - conv.weight.data = (conv.weight.data).clone().detach() + if self.rescale_weights != 1: + # this has to be done after copying but before discretizing + conv.weight.data = (conv.weight / self.rescale_weights).clone().detach() # int conversion is done while writing the config. - if discretize: + if self.discretize: conv, spk = discretize_conv_spike_(conv, spk, to_int=False) - # consolidate layers. - self.conv_layer = conv - self.spk_layer = spk - self.pool_layer = [] - - if len(pool) != 0: - # the 1st pooling targets the 1st destination in `dcnnl_data['destinations']`, the 2nd pooling targets the 2nd destination... - for plyr in pool: - # @TODO POSSIBLE INCONSISTENCY: if the `SumPool2d` is the result of a conversion from `AvgPool2d` then `SumPool2d.kernel_size` - # is of type tuple, otherwise it is an int. - if isinstance(plyr.kernel_size, tuple) and plyr.kernel_size[0] != plyr.kernel_size[1]: - raise ValueError("Only square kernels are supported") - self.pool_layer.append(deepcopy(plyr)) - - # map destination nodes for each layer in this instance. - self.nodes_destinations = self._get_destinations_input_source(sinabs_edges) + if discretize: + # int conversion is done while writing the config. + conv, spk = discretize_conv_spike_(conv, spk, to_int=False) - # flag if the instance is an entry point (i.e., an input node of the network). - if self.conv_node_id in entry_nodes: - self.entry_point = True + self.conv = conv + self.spk = spk + + self.conv_out_shape = self._get_conv_output_shape() + self._pool_lyrs = self._make_pool_layers() # creates SumPool2d layers from `pool`. + ####################################################### Public Methods ####################################################### - - def get_neuron_shape(self) -> Tuple[int, int, int]: - """Return the output shape of the neuron layer. - - Returns - ------- - features, height, width - """ - # same as the convolution's output. - return self.conv_out_shape - - def get_destination_dcnnl_index(self, dcnnl_id: int) -> int: - """ The `forward` method will return as many tensors as there are elements in `self.dynapcnnlayer_destination`. Since the i-th returned tensor is to be - fed to the i-th destionation in `self.dynapcnnlayer_destination`, the return of this method can be used to index a tensor returned in the `forward` method. - - Parameters - ---------- - - dcnnl_id (int): this should be one of the values listed within `self.dynapcnnlayer_destination`. - - Returns - ---------- - - The index of `dcnnl_id` within `self.dynapcnnlayer_destination`. - """ - return self.dynapcnnlayer_destination.index(dcnnl_id) def forward(self, x): """Torch forward pass. - Returns - ---------- - - forward output (tuple): returns as many tensors as there are destinations associated with this instance. The returned - tensors always follows the sequence `return spiking_layer_output, pooling_layer_1, pooling_layer_2, ...`. - - Example - ---------- - TODO this example needs to be revised because the spiking layer only appears if it is projecting to outside this layer. - - - With `self.nodes_destinations = {1: [5, 8], 2: [4], 3: [7]}` this method will return 4 tensors (each to be sent to a different DynapcnnLayer): the 1st - and 2nd are the outputs of the spiking layer (`node 1`); the 3rd is the output of the first pooling layer (`node 2`, receiving the output of node 1) appearing - right after the spiking layer as defined in the original `nn.Module` being converted to a `DynapcnnNetwork`; the 4th is the output of the second pooling - layer (`node 3`) to appear after the spiking layer. Thus, the return will be `return node_1_out, node_1_out, node_2_out, node_3_out`. This means the edges - in the computational graph involved in this mapping were: - - - 2 --> 4 # `4` is a conv layer belonging to another DynapcnnLayer X. - - 3 --> 7 # `7` is a conv layer belonging to another DynapcnnLayer Y. + ... """ returns = [] - x = self.conv_layer(x) - x = self.spk_layer(x) - - # pooling layers are sequentially added to `self.pool_layer` so we'll pass data to them sequentially. - pooling_indexer = 0 - - # building return set of all layers as they appear in `self.nodes_destinations`. - for node_id, destination_node_list in self.nodes_destinations.items(): - if node_id == self.spk_node_id: - # spiking output for each node outside this DynapcnnLayer receiving from its spiking layer. - for _ in destination_node_list: - returns.append(x) + x = self.conv(x) + x = self.spk(x) + + for pool in self.pool: + if pool == 1: + # no pooling is applied. + returns.append(x) else: - # returns of each pooling layer are arranged sequenatially. - for _ in destination_node_list: - ith_pool_output = self.pool_layer[pooling_indexer](x) - returns.append(ith_pool_output) - - # forward through next pooling layer in `self.pool_layer` in the next iteration. - pooling_indexer += 1 - - if len(returns) != len(self.dynapcnnlayer_destination): - raise ValueError(f'Number of returned tensors ({len(returns)}) differ from the number of destinations ({len(self.dynapcnnlayer_destination)}).') - - if len(returns) == 0 and len(self.pool_layer) == 0: - # this is the output layer and there's no pooling after the neurons. - returns.append(x) - elif len(returns) == 0 and len(self.pool_layer) == 1: - # this is the output layer and there's 1 pooling after the neurons. - returns.append(self.pool_layer[0](x)) - elif len(returns) == 0 and len(self.pool_layer) > 1: - raise ValueError(f'Output DynapcnnLayer starting with node {self.conv_node_id} has {len(self.pool_layer)} pooling layers: it should have either 1 or none.') - else: - pass + # sum pooling of `(pool, pool)` is applied. + pool_out = self._pool_lyrs[pool](x) + returns.append(pool_out) return tuple(returns) - - def get_modified_node_io(self, dcnnl_data: dict) -> Union[Tuple[int, tuple], Tuple[None, None]]: - """ Follwing a conversion, the I/O shapes of the spiking layer have been updated to match the convolution's - output. Thus, all nodes receiving input from this spiking layer need their input shapes updated. - - Parameters - ---------- - - dcnnl_data (dict): the set of layers grouped together to comprise this instance of a `DynapcnnLayer`. - - Returns - ---------- - - node ID (int): the ID of the spiking layer consuming the tunerd layer's output (`None` if there was no conversion). - - output shape (tuple): the new output shape following a converstion from `nn.Linear` to `nn.Conv2d` (`None` if there was no conversion). - """ - if self._lin_to_conv_conversion: - return self.spk_node_id, dcnnl_data[self.spk_node_id]['output_shape'] - return None, None - def zero_grad(self, set_to_none: bool = False) -> None: - return self.spk_layer.zero_grad(set_to_none) - - def get_conv_output_shape(self, conv_layer: nn.Conv2d, input_shape: Tuple[int, int, int]) -> Tuple[int, int, int]: - """ Computes the output dimensions of `conv_layer`. - - Parameters - ---------- - - conv_layer (nn.Conv2d): conv. layer whose output will be computed for. - - input_shape (tuple): the shape for the input tensor the layer will process. + def get_neuron_shape(self) -> Tuple[int, int, int]: + """Return the output shape of the neuron layer. Returns - ---------- - - output dimensions (tuple): a tuple describing `(output channels, height, width)`. + ------- + - conv_out_shape (tuple): formatted as (features, height, width). """ - # get the layer's parameters. - out_channels = conv_layer.out_channels - kernel_size = conv_layer.kernel_size - stride = conv_layer.stride - padding = conv_layer.padding - dilation = conv_layer.dilation - - # compute the output height and width. - out_height = ((input_shape[1] + 2 * padding[0] - dilation[0] * (kernel_size[0] - 1) - 1) // stride[0]) + 1 - out_width = ((input_shape[2] + 2 * padding[1] - dilation[1] * (kernel_size[1] - 1) - 1) // stride[1]) + 1 - - return (out_channels, out_height, out_width) + # same as the convolution's output. + return self.conv_out_shape def summary(self) -> dict: """ Returns a summary of the convolution's/pooling's kernel sizes and the output shape of the spiking layer.""" @@ -297,12 +127,12 @@ def summary(self) -> dict: # @TODO POSSIBLE INCONSISTENCY: if the `SumPool2d` is the result of a conversion from `AvgPool2d` then `SumPool2d.kernel_size` # is of type tuple, otherwise it is an int. - if len(self.pool_layer) != 0: + if self._pool_lyrs: # @TODO ignoring for now that there could be multiple poolings (just use the first one). - if isinstance(self.pool_layer[0].kernel_size, tuple): - _pool = list(self.pool_layer[0].kernel_size) - elif isinstance(self.pool_layer[0].kernel_size, int): - _pool = [self.pool_layer[0].kernel_size, self.pool_layer[0].kernel_size] + if isinstance(self._pool_lyrs[next(iter(self._pool_lyrs))].kernel_size, tuple): + _pool = list(self._pool_lyrs[next(iter(self._pool_lyrs))].kernel_size) + elif isinstance(self._pool_lyrs[next(iter(self._pool_lyrs))].kernel_size, int): + _pool = [self._pool_lyrs[next(iter(self._pool_lyrs))].kernel_size, self._pool_lyrs[next(iter(self._pool_lyrs))].kernel_size] else: raise ValueError('Type of `self.pool_layer[0].kernel_size` not understood.') @@ -337,69 +167,11 @@ def memory_summary(self): "kernel": c * pow(2, np.ceil(np.log2(h * w)) + np.ceil(np.log2(f))), "neuron": f * pow(2, np.ceil(np.log2(neuron_height)) + np.ceil(np.log2(neuron_width))), - "bias": 0 if self.conv_layer.bias is None else len(self.conv_layer.bias), + "bias": 0 if self.conv.bias is None else len(self.conv.bias), } - def __str__(self): - pretty_print = '\n' - - pretty_print += 'COMPUTATIONAL NODES:\n\n' - - pretty_print += f'(node {self.conv_node_id}): {self.conv_layer}\n' - pretty_print += f'(node {self.spk_node_id}): {self.spk_layer}' - if len(self.pool_layer) != 0: - for idx, lyr in enumerate(self.pool_layer): - pretty_print += f'\n(node {self.pool_node_id[idx]}): {lyr}' - - pretty_print += '\n\nMETADATA:\n' - pretty_print += f'\n> network\'s entry point: {self.entry_point}' - pretty_print += f'\n> convolution\'s weight re-scaling factor: {self.conv_rescaling_factor}' - pretty_print += f'\n> assigned core index: {self.assigned_core}' - pretty_print += f'\n> destination DynapcnnLayers: {self.dynapcnnlayer_destination}' - - for node, destinations in self.nodes_destinations.items(): - pretty_print += f'\n> node {node} feeds input to nodes {destinations}' - - return pretty_print - ####################################################### Private Methods ####################################################### - def _update_neuron_node_output_shape(self, spiking_layer_data: dict, conv_out_shape: tuple) -> None: - """ Updates the spiking layer's I/O shapes after the conversion of a `nn.Linear` into a `nn.Conv2d` (to match the convolution's output). - - Parameters - ---------- - - spiking_layer_data (dict): the dictionary containing all data regarding the spiking layer. - - conv_out_shape (tuple): the output shape of the convolution layer preceeding the spiking layer. - """ - - # spiking layer consumes the tensor coming out of the conv. layer. - spiking_layer_data['input_shape'] = conv_out_shape - # spiking layer outputs the same shape as the conv. layer. - spiking_layer_data['output_shape'] = spiking_layer_data['input_shape'] - - def _update_conv_node_output_shape(self, conv_layer: nn.Conv2d, layer_data: dict, input_shape: Tuple[int, int, int]) -> Tuple[int, int, int]: - """ Updates the shape of the output tensor of a node that used to be a `nn.Linear` and became a `nn.Conv2d`. - - The input shapes to nodes are extracted using a list of edges by finding the output shape of the 1st element - in the edge and setting it as the input shape to the 2nd element in the edge. If a node used to be a `nn.Linear` - and it became a `nn.Conv2d`, output shape in the mapper needs to be updated, otherwise there will be a mismatch - between its output and the input it provides to another node. - - Parameters - ---------- - - conv_layer (nn.Module): the `nn.Conv2d` created from a `nn.Linear`. - - layer_data (dict): the dictionary containing the data associated with the original `nn.Linear` converted into `nn.Conv2d`. - - input_shape (tuple): the input shape the layer expects. - - Returns - ---------- - - output_shape (tuple): the tensor shape produced by the `nn.Conv2d` created from a `nn.Linear`. - """ - layer_data['output_shape'] = self.get_conv_output_shape(conv_layer, input_shape) - - return layer_data['output_shape'] - def _convert_linear_to_conv(self, lin: nn.Linear, layer_data: dict) -> Tuple[nn.Conv2d, Tuple[int, int, int]]: """ Convert Linear layer to Conv2d. @@ -440,61 +212,63 @@ def _convert_linear_to_conv(self, lin: nn.Linear, layer_data: dict) -> Tuple[nn. ) return layer, input_shape - - def _get_destinations_input_source(self, sinabs_edges: list) -> dict: - """ Creates a mapping between each layer in this `DynapcnnLayer` instance and its targe nodes that are part of different - `DynapcnnLayer` instances. This mapping is used to figure out how many tensors the `forward` method needs to return. - Parameters - ---------- - - sinabs_edges (list): each `nn.Module` within `dcnnl_data` is a node in the original computational graph describing a spiking - network. An edge `(A, B)` describes how modules forward data amongst themselves. This list is used by a `DynapcnnLayer` to - figure out the number and sequence of output tesnors its forward method needs to return. + def _make_pool_layers(self) -> Dict[int, sl.SumPool2d]: + """ Creates a `sl.SumPool2d` for each entry in `self.pool` greater than one. + + Note: the "kernel size" (values > 1) in self.pool is by default used to set the stride of the pooling layer. Returns - ---------- - - destinations_input_source (dict): maps a `nn.Module` within this `DynapcnnLayer` to the nodes it provides the input to. + ------- + - pool_lyrs (dict): the `key` is a value grather than 1 in `self.pool`, with the `value` being the `sl.SumPool2d` it represents. """ - destinations_input_source = {} - - # check whether spiking layer projects outside this DynapcnnLayer (i.e. to one of the destinations without passing through a pooling). - spk_destinations = [] - for edge in sinabs_edges: - if edge[0] == self.spk_node_id and edge[1] not in self.pool_node_id: - # spiking layer projects to a node outside this DynapcnnLayer. - spk_destinations.append(edge[1]) - if len(spk_destinations) > 0: - destinations_input_source[self.spk_node_id] = [] - for node_id in spk_destinations: - destinations_input_source[self.spk_node_id].append(node_id) - - # get `pooling->destination` mapping. The pooling outputs will be arranged sequentially since the pooling layers are added sequentially - # to `self.pool_layer` (i.e., as they appear in the computational graph of the original `nn.Module`). - for id in self.pool_node_id: - destinations_input_source[id] = [] - for edge in sinabs_edges: - if edge[0] == id: - destinations_input_source[id].append(edge[1]) - - return destinations_input_source - - def get_pool_kernel_size(self, node: int) -> int: - """ Returns the pooling kernel size if `node` is a pooling layer.""" - - if node in self.pool_node_id: - i = self.pool_node_id.index(node) - return self.pool_layer[i].kernel_size[0] if isinstance(self.pool_layer[i].kernel_size, tuple) else self.pool_layer[i].kernel_size - elif node == self.spk_node_id: - return 1 - else: - raise ValueError(f'Node {node} does not belong to DynapcnnLayer {self.dpcnnl_index}.') + + pool_lyrs = {} + + # validating if pool are integers + for item in self.pool: + if not isinstance(item, int): + raise ValueError(f"Item '{item}' in `pool` is not an integer.") + + # create layers form pool list. + for kernel_s in self.pool: + + if kernel_s != 1: + + pooling = (kernel_s, kernel_s) + cumulative_pooling = (1, 1) + + # compute cumulative pooling. + cumulative_pooling = ( + cumulative_pooling[0] * pooling[0], + cumulative_pooling[1] * pooling[1], + ) + + # create SumPool2d layer. + pool_lyrs[kernel_s] = sl.SumPool2d(cumulative_pooling) + + return pool_lyrs - @staticmethod - def find_nodes_core_id(node: int, forward_map: dict) -> int: + def _get_conv_output_shape(self) -> Tuple[int, int, int]: + """ Computes the output dimensions of `conv_layer`. - for _, dcnnl in forward_map.items(): + Returns + ---------- + - output dimensions (tuple): a tuple describing `(output channels, height, width)`. + """ + # get the layer's parameters. + + spk = deepcopy() + out_channels = self.conv.out_channels + + spk = deepcopy() + kernel_size = self.conv.kernel_size + stride = self.conv.stride + padding = self.conv.padding + dilation = self.conv.dilation + + # compute the output height and width. + out_height = ((self.in_shape[1] + 2 * padding[0] - dilation[0] * (kernel_size[0] - 1) - 1) // stride[0]) + 1 + out_width = ((self.in_shape[2] + 2 * padding[1] - dilation[1] * (kernel_size[1] - 1) - 1) // stride[1]) + 1 - if node == dcnnl.conv_node_id or node == dcnnl.spk_node_id or node in dcnnl.pool_node_id: - return dcnnl.assigned_core - - raise ValueError(f'Node {node} not found in any of the cores.') + return (out_channels, out_height, out_width) \ No newline at end of file diff --git a/sinabs/backend/dynapcnn/utils.py b/sinabs/backend/dynapcnn/utils.py index 3df4e5e2..3b7d4a9b 100644 --- a/sinabs/backend/dynapcnn/utils.py +++ b/sinabs/backend/dynapcnn/utils.py @@ -283,18 +283,12 @@ def construct_dynapcnnlayer(handler: DynapcnnLayerHandler) -> DynapcnnLayer: """... """ - # retrieve required data from handler. - conv = deepcopy(handler.conv_layer) - spk = deepcopy(handler.spk_layer) - in_shape = handler.conv_in_shape - pool = handler.get_pool_list() - - # instantiate a DynapcnnLayer from the data in 'dcnnl_data'. + # instantiate a DynapcnnLayer from the data in the handler. dynapcnnlayer = DynapcnnLayer( - conv = conv, - spk = spk, - in_shape = in_shape, - pool = pool, + conv = handler.conv_layer, + spk = handler.spk_layer, + in_shape = handler.conv_in_shape, + pool = handler.get_pool_list(), discretize = False, ) From 72974e4ab17fff316ee12fdee1df6af870ff99b1 Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 13 Sep 2024 13:34:52 +0200 Subject: [PATCH 136/379] refactor: encapsulating instance variables with @property pool, discretize, rescale_weight and conv_out_shape turned into properties --- sinabs/backend/dynapcnn/dynapcnn_layer.py | 46 +++++++++++++++-------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer.py b/sinabs/backend/dynapcnn/dynapcnn_layer.py index 63b8170d..af974a9f 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer.py @@ -52,9 +52,9 @@ def __init__( super().__init__() self.in_shape = in_shape - self.pool = pool - self.discretize = discretize - self.rescale_weights = rescale_weights + self._pool = pool + self._discretize = discretize + self._rescale_weights = rescale_weights spk = deepcopy(spk) @@ -66,12 +66,12 @@ def __init__( else: conv = deepcopy(conv) - if self.rescale_weights != 1: + if self._rescale_weights != 1: # this has to be done after copying but before discretizing - conv.weight.data = (conv.weight / self.rescale_weights).clone().detach() + conv.weight.data = (conv.weight / self._rescale_weights).clone().detach() # int conversion is done while writing the config. - if self.discretize: + if self._discretize: conv, spk = discretize_conv_spike_(conv, spk, to_int=False) if discretize: @@ -80,11 +80,25 @@ def __init__( self.conv = conv self.spk = spk - - self.conv_out_shape = self._get_conv_output_shape() + self._pool_lyrs = self._make_pool_layers() # creates SumPool2d layers from `pool`. - + + @property + def pool(self): + return self._pool + @property + def discretize(self): + return self._discretize + + @property + def rescale_weights(self): + return self._rescale_weights + + @property + def conv_out_shape(self): + return self._get_conv_output_shape() + ####################################################### Public Methods ####################################################### def forward(self, x): @@ -98,7 +112,7 @@ def forward(self, x): x = self.conv(x) x = self.spk(x) - for pool in self.pool: + for pool in self._pool: if pool == 1: # no pooling is applied. returns.append(x) @@ -134,7 +148,7 @@ def summary(self) -> dict: elif isinstance(self._pool_lyrs[next(iter(self._pool_lyrs))].kernel_size, int): _pool = [self._pool_lyrs[next(iter(self._pool_lyrs))].kernel_size, self._pool_lyrs[next(iter(self._pool_lyrs))].kernel_size] else: - raise ValueError('Type of `self.pool_layer[0].kernel_size` not understood.') + raise ValueError('Type of `self._pool_layer[0].kernel_size` not understood.') return { "pool": (_pool), @@ -214,24 +228,24 @@ def _convert_linear_to_conv(self, lin: nn.Linear, layer_data: dict) -> Tuple[nn. return layer, input_shape def _make_pool_layers(self) -> Dict[int, sl.SumPool2d]: - """ Creates a `sl.SumPool2d` for each entry in `self.pool` greater than one. + """ Creates a `sl.SumPool2d` for each entry in `self._pool` greater than one. - Note: the "kernel size" (values > 1) in self.pool is by default used to set the stride of the pooling layer. + Note: the "kernel size" (values > 1) in self._pool is by default used to set the stride of the pooling layer. Returns ------- - - pool_lyrs (dict): the `key` is a value grather than 1 in `self.pool`, with the `value` being the `sl.SumPool2d` it represents. + - pool_lyrs (dict): the `key` is a value grather than 1 in `self._pool`, with the `value` being the `sl.SumPool2d` it represents. """ pool_lyrs = {} # validating if pool are integers - for item in self.pool: + for item in self._pool: if not isinstance(item, int): raise ValueError(f"Item '{item}' in `pool` is not an integer.") # create layers form pool list. - for kernel_s in self.pool: + for kernel_s in self._pool: if kernel_s != 1: From 7d63bf1a866901cc930a83547a1d27498589b549 Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 13 Sep 2024 13:37:46 +0200 Subject: [PATCH 137/379] refactor: superfluous variables removed --- sinabs/backend/dynapcnn/dynapcnn_layer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer.py b/sinabs/backend/dynapcnn/dynapcnn_layer.py index af974a9f..8089ae88 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer.py @@ -250,7 +250,6 @@ def _make_pool_layers(self) -> Dict[int, sl.SumPool2d]: if kernel_s != 1: pooling = (kernel_s, kernel_s) - cumulative_pooling = (1, 1) # compute cumulative pooling. cumulative_pooling = ( From 8fb101f0f347f60787050bade9a152a00ce06d20 Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 13 Sep 2024 13:45:31 +0200 Subject: [PATCH 138/379] refactor: summary() using self._pool directly in its returned dictionary --- sinabs/backend/dynapcnn/dynapcnn_layer.py | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer.py b/sinabs/backend/dynapcnn/dynapcnn_layer.py index 8089ae88..b290eedb 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer.py @@ -135,23 +135,9 @@ def get_neuron_shape(self) -> Tuple[int, int, int]: def summary(self) -> dict: """ Returns a summary of the convolution's/pooling's kernel sizes and the output shape of the spiking layer.""" - # TODO I can't see pooling being used in checking memory constraints by the builder so I'm ignoring for now the fact that multiple pooling could exist. - - _pool = None - - # @TODO POSSIBLE INCONSISTENCY: if the `SumPool2d` is the result of a conversion from `AvgPool2d` then `SumPool2d.kernel_size` - # is of type tuple, otherwise it is an int. - if self._pool_lyrs: - # @TODO ignoring for now that there could be multiple poolings (just use the first one). - if isinstance(self._pool_lyrs[next(iter(self._pool_lyrs))].kernel_size, tuple): - _pool = list(self._pool_lyrs[next(iter(self._pool_lyrs))].kernel_size) - elif isinstance(self._pool_lyrs[next(iter(self._pool_lyrs))].kernel_size, int): - _pool = [self._pool_lyrs[next(iter(self._pool_lyrs))].kernel_size, self._pool_lyrs[next(iter(self._pool_lyrs))].kernel_size] - else: - raise ValueError('Type of `self._pool_layer[0].kernel_size` not understood.') return { - "pool": (_pool), + "pool": (self._pool), "kernel": list(self.conv_layer.weight.data.shape), "neuron": self.conv_out_shape, # neuron layer output has the same shape as the convolution layer ouput. } From e2038553c2d464582dc60efd1bb4f23b789dd5d8 Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 13 Sep 2024 13:47:58 +0200 Subject: [PATCH 139/379] refactor: using self._get_conv_output_shape() in place of previous instance variable --- sinabs/backend/dynapcnn/dynapcnn_layer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer.py b/sinabs/backend/dynapcnn/dynapcnn_layer.py index b290eedb..d3449af7 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer.py @@ -131,7 +131,7 @@ def get_neuron_shape(self) -> Tuple[int, int, int]: - conv_out_shape (tuple): formatted as (features, height, width). """ # same as the convolution's output. - return self.conv_out_shape + return self._get_conv_output_shape() def summary(self) -> dict: """ Returns a summary of the convolution's/pooling's kernel sizes and the output shape of the spiking layer.""" @@ -139,7 +139,7 @@ def summary(self) -> dict: return { "pool": (self._pool), "kernel": list(self.conv_layer.weight.data.shape), - "neuron": self.conv_out_shape, # neuron layer output has the same shape as the convolution layer ouput. + "neuron": self._get_conv_output_shape(), # neuron layer output has the same shape as the convolution layer ouput. } def memory_summary(self): @@ -161,7 +161,7 @@ def memory_summary(self): """ summary = self.summary() f, c, h, w = summary["kernel"] - f, neuron_height, neuron_width = self.conv_out_shape # neuron layer output has the same shape as the convolution layer ouput. + f, neuron_height, neuron_width = self._get_conv_output_shape() # neuron layer output has the same shape as the convolution layer ouput. return { "kernel": c * pow(2, np.ceil(np.log2(h * w)) + np.ceil(np.log2(f))), From 72c8e6ea54762db692b6aaad2a48c1659c2f8d31 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Tue, 17 Sep 2024 15:21:31 +0200 Subject: [PATCH 140/379] Remove obsolete files dynapcnn_layer_v2.py and dynapcnn_layer_old.py --- sinabs/backend/dynapcnn/dynapcnn_layer_old.py | 500 ------------------ sinabs/backend/dynapcnn/dynapcnn_layer_v2.py | 209 -------- 2 files changed, 709 deletions(-) delete mode 100644 sinabs/backend/dynapcnn/dynapcnn_layer_old.py delete mode 100644 sinabs/backend/dynapcnn/dynapcnn_layer_v2.py diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer_old.py b/sinabs/backend/dynapcnn/dynapcnn_layer_old.py deleted file mode 100644 index caca930a..00000000 --- a/sinabs/backend/dynapcnn/dynapcnn_layer_old.py +++ /dev/null @@ -1,500 +0,0 @@ -# author : Willian Soares Girao -# contact : wsoaresgirao@gmail.com - -from copy import deepcopy -from typing import Dict, Callable, Tuple, Union, List - -import numpy as np -import torch -from torch import nn - -import sinabs.activation -import sinabs.layers as sl - -from .discretize import discretize_conv_spike_ - -class DynapcnnLayer(nn.Module): - """ - Create a `DynapcnnLayer` object representing a layer on a Speck device. - - Parameters - ---------- - - dpcnnl_index (int): the index/ID that will be associated with a `DynapcnnLayer` instance. This integer indexes a `dict` within `dcnnl_data` - that comprises a set of layers (`nn.Module`) and their respective I/O tensor shapes. - - dcnnl_data (dict): contains the nodes to be merged into this `DynapcnnLayer`, their I/O shapes and the index of the other `DynapcnnLayer`s to - be set as destinations. The `int` keys correspond to the nodes IDs associated `nn.Module`s (a single layer in the original network) becoming - part of this `DynapcnnLayer` instance, while the `str` keys correspond to this instance's destinations and re-scaling factors. - - discretize (bool): whether or not the weights/neuron parameters of the model will be quantized. - - sinabs_edges (list): each `nn.Module` within `dcnnl_data` is a node in the original computational graph describing a spiking network. An edge - `(A, B)` describes how modules forward data amongst themselves. This list is used by a `DynapcnnLayer` to figure out the number and - sequence of output tesnors its forward method needs to return. - - weight_rescaling_fn (callable): a method that handles how the re-scaling factor for one or more `SumPool2d` projecting to - the same convolutional layer are combined/re-scaled before applying them. - - entry_nodes (list): node IDs corresponding to layers in the original network that are input nodes (i.e., a "point of entry" for the external data). - """ - - def __init__( - self, - dpcnnl_index: int, - dcnnl_data: Dict[Union[int, str], Union[Dict[str, Union[nn.Module, Tuple[int, int, int], Tuple[int, int, int]]], List[int]]], - discretize: bool, - sinabs_edges: List[Tuple[int, int]], - weight_rescaling_fn: Callable, - entry_nodes: List[int] - ): - super().__init__() - - self.dpcnnl_index = dpcnnl_index - self.assigned_core = None - self.entry_point = False - - if 'core_idx' in dcnnl_data: - self.assigned_core = dcnnl_data['core_idx'] - - self._lin_to_conv_conversion = False - - conv = None - self.conv_node_id = None - self.conv_in_shape = None - self.conv_out_shape = None - - spk = None - self.spk_node_id = None - - pool = [] - self.pool_node_id = [] - self.conv_rescaling_factor = None - - self.dynapcnnlayer_destination = dcnnl_data['destinations'] - - for key, value in dcnnl_data.items(): - if isinstance(key, int): - # value has data pertaining a node (torch/sinabs layer). - if isinstance(value['layer'], sl.IAFSqueeze): - spk = value['layer'] - self.spk_node_id = key - elif isinstance(value['layer'], nn.Linear) or isinstance(value['layer'], nn.Conv2d): - conv = value['layer'] - self.conv_node_id = key - elif isinstance(value['layer'], sl.SumPool2d): - pool.append(value['layer']) - self.pool_node_id.append(key) - else: - raise ValueError(f'Node {key} has not valid layer associated with it.') - - if not conv: - raise ValueError(f'Convolution layer not present.') - - if not spk: - raise ValueError(f'Spiking layer not present.') - - spk = deepcopy(spk) - if spk.is_state_initialised(): - # TODO this line bellow is causing an exception on `.v_men.shape` to be raised in `.get_layer_config_dict()`. Find out why. - # spk.v_mem = spk.v_mem.data.unsqueeze(-1).unsqueeze(-1) # expand dims. - - # TODO hacky stuff: make it better (THIS SEEMS TO BE FIXING THE PROBLEM ABOVE THO). - if len(list(spk.v_mem.shape)) != 4: - spk.v_mem = spk.v_mem.data.unsqueeze(-1).unsqueeze(-1) # expand dims. - - if isinstance(conv, nn.Linear): - # A `nn.Linear` needs to be converted into `nn.Conv2d`. The I/O shapes of the spiking layer are updated - # accordingly following the conversion. - - conv, conv_in_shape = self._convert_linear_to_conv(conv, dcnnl_data[self.conv_node_id]) - - # the original `nn.Linear` output shape becomes the equivalent `nn.Conv2d` shape. - self.conv_out_shape = self._update_conv_node_output_shape( - conv_layer=conv, layer_data=dcnnl_data[self.conv_node_id], input_shape=conv_in_shape) - - # the I/O shapes for neuron layer following the new conv need also to be updated. - self._update_neuron_node_output_shape(spiking_layer_data=dcnnl_data[self.spk_node_id], conv_out_shape=self.conv_out_shape) - - else: - self.conv_out_shape = dcnnl_data[self.conv_node_id]['output_shape'] - conv = deepcopy(conv) - - # check if convolution kernel is a square. - if conv.kernel_size[0] != conv.kernel_size[1]: - raise ValueError('The kernel of a `nn.Conv2d` must have the same height and width.') - - # input shape of conv layer. - self.conv_in_shape = dcnnl_data[self.conv_node_id]['input_shape'] - # input shape of the `DynapcnnLayer` instance. - self.input_shape = self.conv_in_shape - - # this weight rescale comes from the node projecting into this 'conv' node. - if len(dcnnl_data['conv_rescale_factor']): - # this means an `AvgPool2d` has been converted into a `SumPool2d`. - self.conv_rescaling_factor = weight_rescaling_fn(dcnnl_data['conv_rescale_factor']) - conv.weight.data = (conv.weight.data / self.conv_rescaling_factor).clone().detach() - else: - # this means `SumPool2d` have been used from the start. - conv.weight.data = (conv.weight.data).clone().detach() - - # int conversion is done while writing the config. - if discretize: - conv, spk = discretize_conv_spike_(conv, spk, to_int=False) - - # consolidate layers. - self.conv_layer = conv - self.spk_layer = spk - self.pool_layer = [] - - if len(pool) != 0: - # the 1st pooling targets the 1st destination in `dcnnl_data['destinations']`, the 2nd pooling targets the 2nd destination... - for plyr in pool: - # @TODO POSSIBLE INCONSISTENCY: if the `SumPool2d` is the result of a conversion from `AvgPool2d` then `SumPool2d.kernel_size` - # is of type tuple, otherwise it is an int. - if isinstance(plyr.kernel_size, tuple) and plyr.kernel_size[0] != plyr.kernel_size[1]: - raise ValueError("Only square kernels are supported") - self.pool_layer.append(deepcopy(plyr)) - - # map destination nodes for each layer in this instance. - self.nodes_destinations = self._get_destinations_input_source(sinabs_edges) - - # flag if the instance is an entry point (i.e., an input node of the network). - if self.conv_node_id in entry_nodes: - self.entry_point = True - - ####################################################### Public Methods ####################################################### - - def get_neuron_shape(self) -> Tuple[int, int, int]: - """Return the output shape of the neuron layer. - - Returns - ------- - features, height, width - """ - # same as the convolution's output. - return self.conv_out_shape - - def get_destination_dcnnl_index(self, dcnnl_id: int) -> int: - """ The `forward` method will return as many tensors as there are elements in `self.dynapcnnlayer_destination`. Since the i-th returned tensor is to be - fed to the i-th destionation in `self.dynapcnnlayer_destination`, the return of this method can be used to index a tensor returned in the `forward` method. - - Parameters - ---------- - - dcnnl_id (int): this should be one of the values listed within `self.dynapcnnlayer_destination`. - - Returns - ---------- - - The index of `dcnnl_id` within `self.dynapcnnlayer_destination`. - """ - return self.dynapcnnlayer_destination.index(dcnnl_id) - - def forward(self, x): - """Torch forward pass. - - Returns - ---------- - - forward output (tuple): returns as many tensors as there are destinations associated with this instance. The returned - tensors always follows the sequence `return spiking_layer_output, pooling_layer_1, pooling_layer_2, ...`. - - Example - ---------- - TODO this example needs to be revised because the spiking layer only appears if it is projecting to outside this layer. - - - With `self.nodes_destinations = {1: [5, 8], 2: [4], 3: [7]}` this method will return 4 tensors (each to be sent to a different DynapcnnLayer): the 1st - and 2nd are the outputs of the spiking layer (`node 1`); the 3rd is the output of the first pooling layer (`node 2`, receiving the output of node 1) appearing - right after the spiking layer as defined in the original `nn.Module` being converted to a `DynapcnnNetwork`; the 4th is the output of the second pooling - layer (`node 3`) to appear after the spiking layer. Thus, the return will be `return node_1_out, node_1_out, node_2_out, node_3_out`. This means the edges - in the computational graph involved in this mapping were: - - - 2 --> 4 # `4` is a conv layer belonging to another DynapcnnLayer X. - - 3 --> 7 # `7` is a conv layer belonging to another DynapcnnLayer Y. - """ - - returns = [] - - x = self.conv_layer(x) - x = self.spk_layer(x) - - # pooling layers are sequentially added to `self.pool_layer` so we'll pass data to them sequentially. - pooling_indexer = 0 - - # building return set of all layers as they appear in `self.nodes_destinations`. - for node_id, destination_node_list in self.nodes_destinations.items(): - if node_id == self.spk_node_id: - # spiking output for each node outside this DynapcnnLayer receiving from its spiking layer. - for _ in destination_node_list: - returns.append(x) - else: - # returns of each pooling layer are arranged sequenatially. - for _ in destination_node_list: - ith_pool_output = self.pool_layer[pooling_indexer](x) - returns.append(ith_pool_output) - - # forward through next pooling layer in `self.pool_layer` in the next iteration. - pooling_indexer += 1 - - if len(returns) != len(self.dynapcnnlayer_destination): - raise ValueError(f'Number of returned tensors ({len(returns)}) differ from the number of destinations ({len(self.dynapcnnlayer_destination)}).') - - if len(returns) == 0 and len(self.pool_layer) == 0: - # this is the output layer and there's no pooling after the neurons. - returns.append(x) - elif len(returns) == 0 and len(self.pool_layer) == 1: - # this is the output layer and there's 1 pooling after the neurons. - returns.append(self.pool_layer[0](x)) - elif len(returns) == 0 and len(self.pool_layer) > 1: - raise ValueError(f'Output DynapcnnLayer starting with node {self.conv_node_id} has {len(self.pool_layer)} pooling layers: it should have either 1 or none.') - else: - pass - - return tuple(returns) - - def get_modified_node_io(self, dcnnl_data: dict) -> Union[Tuple[int, tuple], Tuple[None, None]]: - """ Follwing a conversion, the I/O shapes of the spiking layer have been updated to match the convolution's - output. Thus, all nodes receiving input from this spiking layer need their input shapes updated. - - Parameters - ---------- - - dcnnl_data (dict): the set of layers grouped together to comprise this instance of a `DynapcnnLayer`. - - Returns - ---------- - - node ID (int): the ID of the spiking layer consuming the tunerd layer's output (`None` if there was no conversion). - - output shape (tuple): the new output shape following a converstion from `nn.Linear` to `nn.Conv2d` (`None` if there was no conversion). - """ - if self._lin_to_conv_conversion: - return self.spk_node_id, dcnnl_data[self.spk_node_id]['output_shape'] - return None, None - - def zero_grad(self, set_to_none: bool = False) -> None: - return self.spk_layer.zero_grad(set_to_none) - - def get_conv_output_shape(self, conv_layer: nn.Conv2d, input_shape: Tuple[int, int, int]) -> Tuple[int, int, int]: - """ Computes the output dimensions of `conv_layer`. - - Parameters - ---------- - - conv_layer (nn.Conv2d): conv. layer whose output will be computed for. - - input_shape (tuple): the shape for the input tensor the layer will process. - - Returns - ---------- - - output dimensions (tuple): a tuple describing `(output channels, height, width)`. - """ - # get the layer's parameters. - out_channels = conv_layer.out_channels - kernel_size = conv_layer.kernel_size - stride = conv_layer.stride - padding = conv_layer.padding - dilation = conv_layer.dilation - - # compute the output height and width. - out_height = ((input_shape[1] + 2 * padding[0] - dilation[0] * (kernel_size[0] - 1) - 1) // stride[0]) + 1 - out_width = ((input_shape[2] + 2 * padding[1] - dilation[1] * (kernel_size[1] - 1) - 1) // stride[1]) + 1 - - return (out_channels, out_height, out_width) - - def summary(self) -> dict: - """ Returns a summary of the convolution's/pooling's kernel sizes and the output shape of the spiking layer.""" - # TODO I can't see pooling being used in checking memory constraints by the builder so I'm ignoring for now the fact that multiple pooling could exist. - - _pool = None - - # @TODO POSSIBLE INCONSISTENCY: if the `SumPool2d` is the result of a conversion from `AvgPool2d` then `SumPool2d.kernel_size` - # is of type tuple, otherwise it is an int. - if len(self.pool_layer) != 0: - # @TODO ignoring for now that there could be multiple poolings (just use the first one). - if isinstance(self.pool_layer[0].kernel_size, tuple): - _pool = list(self.pool_layer[0].kernel_size) - elif isinstance(self.pool_layer[0].kernel_size, int): - _pool = [self.pool_layer[0].kernel_size, self.pool_layer[0].kernel_size] - else: - raise ValueError('Type of `self.pool_layer[0].kernel_size` not understood.') - - return { - "pool": (_pool), - "kernel": list(self.conv_layer.weight.data.shape), - "neuron": self.conv_out_shape, # neuron layer output has the same shape as the convolution layer ouput. - } - - def memory_summary(self): - """Computes the amount of memory required for each of the components. Note that this is not - necessarily the same as the number of parameters due to some architecture design - constraints. - - .. math:: - - K_{MT} = c \\cdot 2^{\\lceil \\log_2\\left(k_xk_y\\right) \\rceil + \\lceil \\log_2\\left(f\\right) \\rceil} - - .. math:: - - N_{MT} = f \\cdot 2^{ \\lceil \\log_2\\left(f_y\\right) \\rceil + \\lceil \\log_2\\left(f_x\\right) \\rceil } - - Returns - ------- - A dictionary with keys kernel, neuron and bias and the corresponding memory sizes - """ - summary = self.summary() - f, c, h, w = summary["kernel"] - f, neuron_height, neuron_width = self.conv_out_shape # neuron layer output has the same shape as the convolution layer ouput. - - return { - "kernel": c * pow(2, np.ceil(np.log2(h * w)) + np.ceil(np.log2(f))), - "neuron": f - * pow(2, np.ceil(np.log2(neuron_height)) + np.ceil(np.log2(neuron_width))), - "bias": 0 if self.conv_layer.bias is None else len(self.conv_layer.bias), - } - - def __str__(self): - pretty_print = '\n' - - pretty_print += 'COMPUTATIONAL NODES:\n\n' - - pretty_print += f'(node {self.conv_node_id}): {self.conv_layer}\n' - pretty_print += f'(node {self.spk_node_id}): {self.spk_layer}' - if len(self.pool_layer) != 0: - for idx, lyr in enumerate(self.pool_layer): - pretty_print += f'\n(node {self.pool_node_id[idx]}): {lyr}' - - pretty_print += '\n\nMETADATA:\n' - pretty_print += f'\n> network\'s entry point: {self.entry_point}' - pretty_print += f'\n> convolution\'s weight re-scaling factor: {self.conv_rescaling_factor}' - pretty_print += f'\n> assigned core index: {self.assigned_core}' - pretty_print += f'\n> destination DynapcnnLayers: {self.dynapcnnlayer_destination}' - - for node, destinations in self.nodes_destinations.items(): - pretty_print += f'\n> node {node} feeds input to nodes {destinations}' - - return pretty_print - - ####################################################### Private Methods ####################################################### - - def _update_neuron_node_output_shape(self, spiking_layer_data: dict, conv_out_shape: tuple) -> None: - """ Updates the spiking layer's I/O shapes after the conversion of a `nn.Linear` into a `nn.Conv2d` (to match the convolution's output). - - Parameters - ---------- - - spiking_layer_data (dict): the dictionary containing all data regarding the spiking layer. - - conv_out_shape (tuple): the output shape of the convolution layer preceeding the spiking layer. - """ - - # spiking layer consumes the tensor coming out of the conv. layer. - spiking_layer_data['input_shape'] = conv_out_shape - # spiking layer outputs the same shape as the conv. layer. - spiking_layer_data['output_shape'] = spiking_layer_data['input_shape'] - - def _update_conv_node_output_shape(self, conv_layer: nn.Conv2d, layer_data: dict, input_shape: Tuple[int, int, int]) -> Tuple[int, int, int]: - """ Updates the shape of the output tensor of a node that used to be a `nn.Linear` and became a `nn.Conv2d`. - - The input shapes to nodes are extracted using a list of edges by finding the output shape of the 1st element - in the edge and setting it as the input shape to the 2nd element in the edge. If a node used to be a `nn.Linear` - and it became a `nn.Conv2d`, output shape in the mapper needs to be updated, otherwise there will be a mismatch - between its output and the input it provides to another node. - - Parameters - ---------- - - conv_layer (nn.Module): the `nn.Conv2d` created from a `nn.Linear`. - - layer_data (dict): the dictionary containing the data associated with the original `nn.Linear` converted into `nn.Conv2d`. - - input_shape (tuple): the input shape the layer expects. - - Returns - ---------- - - output_shape (tuple): the tensor shape produced by the `nn.Conv2d` created from a `nn.Linear`. - """ - layer_data['output_shape'] = self.get_conv_output_shape(conv_layer, input_shape) - - return layer_data['output_shape'] - - def _convert_linear_to_conv(self, lin: nn.Linear, layer_data: dict) -> Tuple[nn.Conv2d, Tuple[int, int, int]]: - """ Convert Linear layer to Conv2d. - - Parameters - ---------- - - lin (nn.Linear): linear layer to be converted. - - Returns - ------- - - nn.Conv2d: convolutional layer equivalent to `lin`. - - input_shape (tuple): the tensor shape the layer expects. - """ - # this flags the necessity to update the I/O shape pre-computed for each of the original layers being compressed within a `DynapcnnLayer` instance. - self._lin_to_conv_conversion = True - - input_shape = layer_data['input_shape'] - - in_chan, in_h, in_w = input_shape - - if lin.in_features != in_chan * in_h * in_w: - raise ValueError("Shapes don't match.") - - layer = nn.Conv2d( - in_channels=in_chan, - kernel_size=(in_h, in_w), - out_channels=lin.out_features, - padding=0, - bias=lin.bias is not None, - ) - - if lin.bias is not None: - layer.bias.data = lin.bias.data.clone().detach() - - layer.weight.data = ( - lin.weight.data.clone() - .detach() - .reshape((lin.out_features, in_chan, in_h, in_w)) - ) - - return layer, input_shape - - def _get_destinations_input_source(self, sinabs_edges: list) -> dict: - """ Creates a mapping between each layer in this `DynapcnnLayer` instance and its targe nodes that are part of different - `DynapcnnLayer` instances. This mapping is used to figure out how many tensors the `forward` method needs to return. - - Parameters - ---------- - - sinabs_edges (list): each `nn.Module` within `dcnnl_data` is a node in the original computational graph describing a spiking - network. An edge `(A, B)` describes how modules forward data amongst themselves. This list is used by a `DynapcnnLayer` to - figure out the number and sequence of output tesnors its forward method needs to return. - - Returns - ---------- - - destinations_input_source (dict): maps a `nn.Module` within this `DynapcnnLayer` to the nodes it provides the input to. - """ - destinations_input_source = {} - - # check whether spiking layer projects outside this DynapcnnLayer (i.e. to one of the destinations without passing through a pooling). - spk_destinations = [] - for edge in sinabs_edges: - if edge[0] == self.spk_node_id and edge[1] not in self.pool_node_id: - # spiking layer projects to a node outside this DynapcnnLayer. - spk_destinations.append(edge[1]) - if len(spk_destinations) > 0: - destinations_input_source[self.spk_node_id] = [] - for node_id in spk_destinations: - destinations_input_source[self.spk_node_id].append(node_id) - - # get `pooling->destination` mapping. The pooling outputs will be arranged sequentially since the pooling layers are added sequentially - # to `self.pool_layer` (i.e., as they appear in the computational graph of the original `nn.Module`). - for id in self.pool_node_id: - destinations_input_source[id] = [] - for edge in sinabs_edges: - if edge[0] == id: - destinations_input_source[id].append(edge[1]) - - return destinations_input_source - - def get_pool_kernel_size(self, node: int) -> int: - """ Returns the pooling kernel size if `node` is a pooling layer.""" - - if node in self.pool_node_id: - i = self.pool_node_id.index(node) - return self.pool_layer[i].kernel_size[0] if isinstance(self.pool_layer[i].kernel_size, tuple) else self.pool_layer[i].kernel_size - elif node == self.spk_node_id: - return 1 - else: - raise ValueError(f'Node {node} does not belong to DynapcnnLayer {self.dpcnnl_index}.') - - @staticmethod - def find_nodes_core_id(node: int, forward_map: dict) -> int: - - for _, dcnnl in forward_map.items(): - - if node == dcnnl.conv_node_id or node == dcnnl.spk_node_id or node in dcnnl.pool_node_id: - return dcnnl.assigned_core - - raise ValueError(f'Node {node} not found in any of the cores.') diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer_v2.py b/sinabs/backend/dynapcnn/dynapcnn_layer_v2.py deleted file mode 100644 index 80be8a27..00000000 --- a/sinabs/backend/dynapcnn/dynapcnn_layer_v2.py +++ /dev/null @@ -1,209 +0,0 @@ -# author : Willian Soares Girao -# contact : wsoaresgirao@gmail.com - -from copy import deepcopy -from typing import Dict, Callable, Tuple, Union, List - -import numpy as np -import torch -from torch import nn - -import sinabs.activation -import sinabs.layers as sl - -from .discretize import discretize_conv_spike_ - -class DynapcnnLayer(nn.Module): - """Create a DynapcnnLayer object representing a layer on DynapCNN or Speck. - - Requires a convolutional layer, a sinabs spiking layer and a list of - pooling values. The layers are used in the order conv -> spike -> pool. - - Parameters - ---------- - conv: torch.nn.Conv2d or torch.nn.Linear - Convolutional or linear layer - (linear will be converted to convolutional) - spk: sinabs.layers.IAFSqueeze - Sinabs IAF layer - in_shape: tuple of int - The input shape, needed to create dynapcnn configs if the network - does not contain an input layer. Convention: (features, height, width) - pool: List of integers - Each integer entry represents an output (destination on chip) and - whether pooling should be applied (values > 1) or not (values equal - to 1). The number of entries determines the number of tensors the - layer's forward method returns. - discretize: bool - Whether to discretize parameters. - rescale_weights: int - Layer weights will be divided by this value. - """ - - def __init__( - self, - conv: nn.Conv2d, - spk: sl.IAFSqueeze, - in_shape: Tuple[int, int, int], - pool: List[int], - discretize: bool = True, - rescale_weights: int = 1, - ): - super().__init__() - - # int conversion is done while writing the config. - if discretize: - conv, spk = discretize_conv_spike_(conv, spk, to_int=False) - - self.conv = conv - self.spk = spk - self.in_shape = in_shape - self.pool = pool - self.discretize = discretize - self.rescale_weights = rescale_weights - self.conv_out_shape = self._get_conv_output_shape() - self._pool_lyrs = self._make_pool_layers() # creates SumPool2d layers from `pool`. - - - ####################################################### Public Methods ####################################################### - - def forward(self, x): - """Torch forward pass. - - ... - """ - - returns = [] - - x = self.conv(x) - x = self.spk(x) - - for pool in self.pool: - if pool == 1: - # no pooling is applied. - returns.append(x) - else: - # sum pooling of `(pool, pool)` is applied. - pool_out = self._pool_lyrs[pool](x) - returns.append(pool_out) - - return tuple(returns) - - def get_neuron_shape(self) -> Tuple[int, int, int]: - """Return the output shape of the neuron layer. - - Returns - ------- - - conv_out_shape (tuple): formatted as (features, height, width). - """ - # same as the convolution's output. - return self.conv_out_shape - - def summary(self) -> dict: - """ Returns a summary of the convolution's/pooling's kernel sizes and the output shape of the spiking layer.""" - # TODO I can't see pooling being used in checking memory constraints by the builder so I'm ignoring for now the fact that multiple pooling could exist. - - _pool = None - - # @TODO POSSIBLE INCONSISTENCY: if the `SumPool2d` is the result of a conversion from `AvgPool2d` then `SumPool2d.kernel_size` - # is of type tuple, otherwise it is an int. - if self._pool_lyrs: - # @TODO ignoring for now that there could be multiple poolings (just use the first one). - if isinstance(self._pool_lyrs[next(iter(self._pool_lyrs))].kernel_size, tuple): - _pool = list(self._pool_lyrs[next(iter(self._pool_lyrs))].kernel_size) - elif isinstance(self._pool_lyrs[next(iter(self._pool_lyrs))].kernel_size, int): - _pool = [self._pool_lyrs[next(iter(self._pool_lyrs))].kernel_size, self._pool_lyrs[next(iter(self._pool_lyrs))].kernel_size] - else: - raise ValueError('Type of `self.pool_layer[0].kernel_size` not understood.') - - return { - "pool": (_pool), - "kernel": list(self.conv_layer.weight.data.shape), - "neuron": self.conv_out_shape, # neuron layer output has the same shape as the convolution layer ouput. - } - - def memory_summary(self): - """Computes the amount of memory required for each of the components. Note that this is not - necessarily the same as the number of parameters due to some architecture design - constraints. - - .. math:: - - K_{MT} = c \\cdot 2^{\\lceil \\log_2\\left(k_xk_y\\right) \\rceil + \\lceil \\log_2\\left(f\\right) \\rceil} - - .. math:: - - N_{MT} = f \\cdot 2^{ \\lceil \\log_2\\left(f_y\\right) \\rceil + \\lceil \\log_2\\left(f_x\\right) \\rceil } - - Returns - ------- - A dictionary with keys kernel, neuron and bias and the corresponding memory sizes - """ - summary = self.summary() - f, c, h, w = summary["kernel"] - f, neuron_height, neuron_width = self.conv_out_shape # neuron layer output has the same shape as the convolution layer ouput. - - return { - "kernel": c * pow(2, np.ceil(np.log2(h * w)) + np.ceil(np.log2(f))), - "neuron": f - * pow(2, np.ceil(np.log2(neuron_height)) + np.ceil(np.log2(neuron_width))), - "bias": 0 if self.conv.bias is None else len(self.conv.bias), - } - - ####################################################### Private Methods ####################################################### - - def _make_pool_layers(self) -> Dict[int, sl.SumPool2d]: - """ Creates a `sl.SumPool2d` for each entry in `self.pool` greater than one. - - Note: the "kernel size" (values > 1) in self.pool is by default used to set the stride of the pooling layer. - - Returns - ------- - - pool_lyrs (dict): the `key` is a value grather than 1 in `self.pool`, with the `value` being the `sl.SumPool2d` it represents. - """ - - pool_lyrs = {} - - # validating if pool are integers - for item in self.pool: - if not isinstance(item, int): - raise ValueError(f"Item '{item}' in `pool` is not an integer.") - - # create layers form pool list. - for kernel_s in self.pool: - - if kernel_s != 1: - - pooling = (kernel_s, kernel_s) - cumulative_pooling = (1, 1) - - # compute cumulative pooling. - cumulative_pooling = ( - cumulative_pooling[0] * pooling[0], - cumulative_pooling[1] * pooling[1], - ) - - # create SumPool2d layer. - pool_lyrs[kernel_s] = sl.SumPool2d(cumulative_pooling) - - return pool_lyrs - - def _get_conv_output_shape(self) -> Tuple[int, int, int]: - """ Computes the output dimensions of `conv_layer`. - - Returns - ---------- - - output dimensions (tuple): a tuple describing `(output channels, height, width)`. - """ - # get the layer's parameters. - out_channels = self.conv.out_channels - kernel_size = self.conv.kernel_size - stride = self.conv.stride - padding = self.conv.padding - dilation = self.conv.dilation - - # compute the output height and width. - out_height = ((self.in_shape[1] + 2 * padding[0] - dilation[0] * (kernel_size[0] - 1) - 1) // stride[0]) + 1 - out_width = ((self.in_shape[2] + 2 * padding[1] - dilation[1] * (kernel_size[1] - 1) - 1) // stride[1]) + 1 - - return (out_channels, out_height, out_width) \ No newline at end of file From f9a797d2428d010727c8e5d4ac9d0f8374fd5b62 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Tue, 17 Sep 2024 15:45:58 +0200 Subject: [PATCH 141/379] Simplify DynapcnnLayer by removing _pool_layers attribute. --- sinabs/backend/dynapcnn/dynapcnn_layer.py | 70 +++++++---------------- 1 file changed, 20 insertions(+), 50 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer.py b/sinabs/backend/dynapcnn/dynapcnn_layer.py index d3449af7..d46475a9 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer.py @@ -2,17 +2,22 @@ # contact : wsoaresgirao@gmail.com from copy import deepcopy -from typing import Dict, Callable, Tuple, Union, List +from functools import partial +from typing import Tuple, List import numpy as np import torch from torch import nn -import sinabs.activation import sinabs.layers as sl from .discretize import discretize_conv_spike_ + +# Define sum pooling functional as power-average pooling with power 1 +sum_pool2d = partial(nn.functional.lp_pool2d, norm_type=1) + + class DynapcnnLayer(nn.Module): """Create a DynapcnnLayer object representing a layer on DynapCNN or Speck. @@ -74,15 +79,17 @@ def __init__( if self._discretize: conv, spk = discretize_conv_spike_(conv, spk, to_int=False) - if discretize: - # int conversion is done while writing the config. - conv, spk = discretize_conv_spike_(conv, spk, to_int=False) + self._conv = conv + self._spk = spk + + @property + def conv(self): + return self._conv - self.conv = conv - self.spk = spk + @property + def spk(self): + return self._spk - self._pool_lyrs = self._make_pool_layers() # creates SumPool2d layers from `pool`. - @property def pool(self): return self._pool @@ -101,7 +108,7 @@ def conv_out_shape(self): ####################################################### Public Methods ####################################################### - def forward(self, x): + def forward(self, x) -> List[torch.Tensor]: """Torch forward pass. ... @@ -113,12 +120,13 @@ def forward(self, x): x = self.spk(x) for pool in self._pool: + if pool == 1: # no pooling is applied. returns.append(x) else: # sum pooling of `(pool, pool)` is applied. - pool_out = self._pool_lyrs[pool](x) + pool_out = sum_pool2d(x, kernel_size=pool) returns.append(pool_out) return tuple(returns) @@ -212,41 +220,6 @@ def _convert_linear_to_conv(self, lin: nn.Linear, layer_data: dict) -> Tuple[nn. ) return layer, input_shape - - def _make_pool_layers(self) -> Dict[int, sl.SumPool2d]: - """ Creates a `sl.SumPool2d` for each entry in `self._pool` greater than one. - - Note: the "kernel size" (values > 1) in self._pool is by default used to set the stride of the pooling layer. - - Returns - ------- - - pool_lyrs (dict): the `key` is a value grather than 1 in `self._pool`, with the `value` being the `sl.SumPool2d` it represents. - """ - - pool_lyrs = {} - - # validating if pool are integers - for item in self._pool: - if not isinstance(item, int): - raise ValueError(f"Item '{item}' in `pool` is not an integer.") - - # create layers form pool list. - for kernel_s in self._pool: - - if kernel_s != 1: - - pooling = (kernel_s, kernel_s) - - # compute cumulative pooling. - cumulative_pooling = ( - cumulative_pooling[0] * pooling[0], - cumulative_pooling[1] * pooling[1], - ) - - # create SumPool2d layer. - pool_lyrs[kernel_s] = sl.SumPool2d(cumulative_pooling) - - return pool_lyrs def _get_conv_output_shape(self) -> Tuple[int, int, int]: """ Computes the output dimensions of `conv_layer`. @@ -257,10 +230,7 @@ def _get_conv_output_shape(self) -> Tuple[int, int, int]: """ # get the layer's parameters. - spk = deepcopy() out_channels = self.conv.out_channels - - spk = deepcopy() kernel_size = self.conv.kernel_size stride = self.conv.stride padding = self.conv.padding @@ -270,4 +240,4 @@ def _get_conv_output_shape(self) -> Tuple[int, int, int]: out_height = ((self.in_shape[1] + 2 * padding[0] - dilation[0] * (kernel_size[0] - 1) - 1) // stride[0]) + 1 out_width = ((self.in_shape[2] + 2 * padding[1] - dilation[1] * (kernel_size[1] - 1) - 1) // stride[1]) + 1 - return (out_channels, out_height, out_width) \ No newline at end of file + return (out_channels, out_height, out_width) From dda052dfd226898d4d9211f9e23f3dcbda3140af Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Tue, 17 Sep 2024 15:56:15 +0200 Subject: [PATCH 142/379] DynapcnnLayer: Reintroduce methods get_output_shape and zero_grad --- sinabs/backend/dynapcnn/dynapcnn_layer.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer.py b/sinabs/backend/dynapcnn/dynapcnn_layer.py index d46475a9..ee55988f 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer.py @@ -130,6 +130,10 @@ def forward(self, x) -> List[torch.Tensor]: returns.append(pool_out) return tuple(returns) + + def zero_grad(self, set_to_none: bool = False) -> None: + """ Call `zero_grad` method of spiking layer """ + return self._spk.zero_grad(set_to_none) def get_neuron_shape(self) -> Tuple[int, int, int]: """Return the output shape of the neuron layer. @@ -141,6 +145,25 @@ def get_neuron_shape(self) -> Tuple[int, int, int]: # same as the convolution's output. return self._get_conv_output_shape() + def get_output_shape(self) -> List[Tuple[int, int, int]]: + """Return the output shapes of the layer, including pooling. + + Returns + ------- + - output_shape (list of tuples): + One entry per destination, each formatted as (features, height, width). + """ + neuron_shape = self.get_neuron_shape() + # this is the actual output shape, including pooling + output_shape = [] + for pool in self._pool: + output_shape.append( + neuron_shape[0], + neuron_shape[1] // pool, + neuron_shape[2] // pool, + ) + return output_shape + def summary(self) -> dict: """ Returns a summary of the convolution's/pooling's kernel sizes and the output shape of the spiking layer.""" From 210a34d147440baf6e4971cc1ba3c1ca520cbf29 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Tue, 17 Sep 2024 16:05:30 +0200 Subject: [PATCH 143/379] Keep DynapcnnCompatibleNetwork for now. (remove in future release to avoid confusion) --- sinabs/backend/dynapcnn/__init__.py | 1 + sinabs/backend/dynapcnn/dynapcnn_network.py | 21 ++++++++++++++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/sinabs/backend/dynapcnn/__init__.py b/sinabs/backend/dynapcnn/__init__.py index ff0658a1..21eba2e7 100644 --- a/sinabs/backend/dynapcnn/__init__.py +++ b/sinabs/backend/dynapcnn/__init__.py @@ -1,4 +1,5 @@ from .dynapcnn_network import ( + DynapcnnCompatibleNetwork, DynapcnnNetwork, ) diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index e97acfb5..187cf13f 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -683,4 +683,23 @@ def __str__(self): pretty_print += f'----------------------- [ DynapcnnLayer {idx} ] -----------------------\n' pretty_print += f'{layer_data}\n\n' - return pretty_print \ No newline at end of file + return pretty_print + + + class DynapcnnCompatibleNetwork(DynapcnnNetwork): + """Deprecated class, use DynapcnnNetwork instead.""" + + def __init__( + self, + snn: Union[nn.Sequential, sinabs.Network], + input_shape: Optional[Tuple[int, int, int]] = None, + dvs_input: bool = False, + discretize: bool = True, + ): + from warnings import warn + + warn( + "DynapcnnCompatibleNetwork has been renamed to DynapcnnNetwork " + + "and will be removed in a future release." + ) + super().__init__(snn, input_shape, dvs_input, discretize) \ No newline at end of file From 05546a41748d603ae615683676447f3f884428bc Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Tue, 17 Sep 2024 16:22:39 +0200 Subject: [PATCH 144/379] Minor modifications to dynapcnn.py --- sinabs/backend/dynapcnn/chips/dynapcnn.py | 61 +++++++++++------------ 1 file changed, 28 insertions(+), 33 deletions(-) diff --git a/sinabs/backend/dynapcnn/chips/dynapcnn.py b/sinabs/backend/dynapcnn/chips/dynapcnn.py index 2cb81da2..2e1869b2 100644 --- a/sinabs/backend/dynapcnn/chips/dynapcnn.py +++ b/sinabs/backend/dynapcnn/chips/dynapcnn.py @@ -1,5 +1,5 @@ import copy -from typing import List, Union, Dict +from typing import List from warnings import warn import samna @@ -10,11 +10,10 @@ from sinabs.backend.dynapcnn.config_builder import ConfigBuilder from sinabs.backend.dynapcnn.dvs_layer import DVSLayer, expand_to_pair from sinabs.backend.dynapcnn.dynapcnn_layer import DynapcnnLayer +from sinabs.backend.dynapcnn.dynapcnn_network import DynapcnnNetwork from sinabs.backend.dynapcnn.dynapcnn_layer_handler import DynapcnnLayerHandler from sinabs.backend.dynapcnn.mapping import LayerConstraints -import sinabs - class DynapcnnConfigBuilder(ConfigBuilder): @classmethod def get_samna_module(cls): @@ -164,7 +163,7 @@ def get_dynapcnn_layer_config_dict(cls, layer: DynapcnnLayer, layer_handler: Dyn dest_data = { 'layer': core_id, 'enable': True, - 'pooling': kernel_size if kernel_size else 1 + 'pooling': expand_to_pair(kernel_size if kernel_size else 1), } destinations.append(dest_data) @@ -211,7 +210,7 @@ def write_dynapcnn_layer_config(cls, layer: DynapcnnLayer, chip_layer: "CNNLayer raise TypeError(f"Unexpected parameter {param} or value. {e}") @classmethod - def build_config(cls, model: Union["DynapcnnNetwork"]) -> DynapcnnConfiguration: + def build_config(cls, model: DynapcnnNetwork) -> DynapcnnConfiguration: """ Uses `DynapcnnLayer` objects to configure their equivalent chip core via a `CNNLayerConfig` object that is built using using the `DynapcnnLayer` properties. @@ -225,36 +224,32 @@ def build_config(cls, model: Union["DynapcnnNetwork"]) -> DynapcnnConfiguration: """ config = cls.get_default_config() - if type(model) == sinabs.backend.dynapcnn.dynapcnn_network.DynapcnnNetwork: - """ Loops through `DynapcnnNetworkGraph._layers_mapper`, containing all `DynapcnnLayer`s in the model, their - core ID (where they are configured onto) and their target destinations. Each `ith_dcnnl` has all the info. necessary to config. - their respective `CNNLayerConfig` object. - """ - has_dvs_layer = False # TODO DVSLayer not supported yet. - - for layer_index, ith_dcnnl in model.layers_mapper.items(): - if isinstance(ith_dcnnl, DVSLayer): - # TODO DVSLayer not supported yet. - pass - - elif isinstance(ith_dcnnl, DynapcnnLayer): - # retrieve assigned core from the handler of this DynapcnnLayer (`ith_dcnnl`) instance. - chip_layer = config.cnn_layers[model.layers_handlers[layer_index].assigned_core] - # write core configuration. - cls.write_dynapcnn_layer_config(ith_dcnnl, chip_layer, model.layers_handlers[layer_index], model.layers_handlers) - - else: - # shouldn't happen since type checks are made previously. - raise TypeError(f"Layer (index {layer_index}) is unexpected in the model: \n{ith_dcnnl}") - - if not has_dvs_layer: + if not isinstance(model, DynapcnnNetwork): + raise ValueError(f"`model` has to be of type DynapcnnNetwork, but is {type(model)}.") + + has_dvs_layer = False # TODO DVSLayer not supported yet. + + # Loop over layers in network and write corresponding configurations + for layer_index, ith_dcnnl in model.layers_mapper.items(): + if isinstance(ith_dcnnl, DVSLayer): # TODO DVSLayer not supported yet. - config.dvs_layer.pass_sensor_events = False - else: - config.dvs_layer.pass_sensor_events = False + pass + + elif isinstance(ith_dcnnl, DynapcnnLayer): + # retrieve assigned core from the handler of this DynapcnnLayer (`ith_dcnnl`) instance. + chip_layer = config.cnn_layers[model.layers_handlers[layer_index].assigned_core] + # write core configuration. + cls.write_dynapcnn_layer_config(ith_dcnnl, chip_layer, model.layers_handlers[layer_index], model.layers_handlers) + else: + # shouldn't happen since type checks are made previously. + raise TypeError(f"Layer (index {layer_index}) is unexpected in the model: \n{ith_dcnnl}") + + if not has_dvs_layer: + # TODO DVSLayer not supported yet. + config.dvs_layer.pass_sensor_events = False else: - raise TypeError(f"Unexpected model {type(model)}.") + config.dvs_layer.pass_sensor_events = False return config @@ -315,7 +310,7 @@ def monitor_layers(cls, config: "DynapcnnConfiguration", layers: List): config.dvs_layer.monitor_enable = True if config.dvs_layer.pooling.x != 1 or config.dvs_layer.pooling.y != 1: warn( - f"DVS layer has pooling and is being monitored. " + "DVS layer has pooling and is being monitored. " "Note that pooling will not be reflected in the monitored events." ) monitor_layers.remove("dvs") From 7c3f7a2b6d918fe9337da674c5e7f08d297f695f Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Tue, 17 Sep 2024 16:51:45 +0200 Subject: [PATCH 145/379] Minor changes to NIRGraphExtractor.py: - Add `Addtributes` list to docstring of NIRtoDynapcnnNetworkGraph for better understanding - Rename `get_edges_list` property to `edges_list` --- sinabs/backend/dynapcnn/NIRGraphExtractor.py | 25 ++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/sinabs/backend/dynapcnn/NIRGraphExtractor.py b/sinabs/backend/dynapcnn/NIRGraphExtractor.py index 5f89001c..121b3ad2 100644 --- a/sinabs/backend/dynapcnn/NIRGraphExtractor.py +++ b/sinabs/backend/dynapcnn/NIRGraphExtractor.py @@ -1,9 +1,14 @@ # author : Willian Soares Girao # contact : wsoaresgirao@gmail.com -import torch, sinabs, nirtorch, copy +import copy +from typing import Tuple, Dict, List + +import nirtorch +import sinabs +import torch import torch.nn as nn -from typing import Tuple, Dict, List, Union + from .utils import topological_sorting class NIRtoDynapcnnNetworkGraph(): @@ -18,6 +23,19 @@ def __init__(self, spiking_model: nn.Module, dummy_input: torch.tensor): - dummy_input (torch.tensor): a random input sample to be fed through the model to acquire both the computational graph (via `nirtorch`) and the I/O shapes of each node. Its a 4-D shape with `(batch, channels, heigh, width)`. + + Attributes + ---------- + - edges_list (list of 2-tuples of integers): + Tuples describing the connections between layers in `spiking_model`. + Each layer (node) is identified by a unique integer ID. + - name_2_index_map (dict): + Keys are original variable names of layers in `spiking_model`. + Values are unique integer IDs. + - entry_nodes (list of ints): + IDs of nodes acting as entry points for the network, i.e. receiving external input. + - modules_map (dict): + Map from layer ID to the corresponding nn.Module instance. """ # extract computational graph. @@ -39,7 +57,7 @@ def entry_nodes(self) -> List[int]: return self._entry_nodes @property - def get_edges_list(self): + def edges_list(self): return self._edges_list @property @@ -199,7 +217,6 @@ def _get_named_modules(self, model: nn.Module) -> Dict[int, nn.Module]: return modules_map - # TODO - it would be good if I/O shapes were returned by the NIR graph. def _get_nodes_io_shapes(self, input_dummy: torch.tensor) -> Dict[int, Dict[str, torch.Size]]: """ Iteratively calls the forward method of each `nn.Module` (i.e., a layer/node in the graph) using the topologically sorted nodes extracted from the computational graph of the model being parsed. From 6a04d88eae5343138e93ae3fdbab12dd13760267 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Tue, 17 Sep 2024 16:54:03 +0200 Subject: [PATCH 146/379] Rename NIRGraphExtractor.py to nir_graph_extractor.py --- .../dynapcnn/{NIRGraphExtractor.py => nir_graph_extractor.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename sinabs/backend/dynapcnn/{NIRGraphExtractor.py => nir_graph_extractor.py} (100%) diff --git a/sinabs/backend/dynapcnn/NIRGraphExtractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py similarity index 100% rename from sinabs/backend/dynapcnn/NIRGraphExtractor.py rename to sinabs/backend/dynapcnn/nir_graph_extractor.py From 9986b0bbb21750637b9ed3d7dc05fb2f1c2e17d9 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Tue, 17 Sep 2024 17:03:14 +0200 Subject: [PATCH 147/379] Refactor NIRtoDynapcnnNetworkGraph._get_edges_from_nir --- .../backend/dynapcnn/nir_graph_extractor.py | 23 ++++++++----------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index 121b3ad2..9b708de1 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -157,26 +157,21 @@ def _get_edges_from_nir(self, nir_graph: nirtorch.graph.Graph) -> Tuple[List[Tup """ edges_list = [] name_2_indx_map = {} - idx_counter = 0 # TODO maybe make sure the input node from nir always gets assined `0`. - - nodes_IDs = [0] - + + # TODO maybe make sure the input node from nir always gets assined `0`. + for src_node in nir_graph.node_list: - # source node. + # Make sure current node is in `name_2_indx_map` if src_node.name not in name_2_indx_map: - name_2_indx_map[src_node.name] = idx_counter - idx_counter += 1 - - nodes_IDs.append(idx_counter) + # Assign unique index by taking current length of `name_2_indx_map` + name_2_indx_map[src_node.name] = len(name_2_indx_map) for trg_node in src_node.outgoing_nodes: - # target node. + # Make sure all targets of current node are in `name_2_indx_map` if trg_node.name not in name_2_indx_map: - name_2_indx_map[trg_node.name] = idx_counter - idx_counter += 1 - - nodes_IDs.append(idx_counter) + name_2_indx_map[trg_node.name] = len(name_2_indx_map) + # Store the edge of current node to the target edges_list.append((name_2_indx_map[src_node.name], name_2_indx_map[trg_node.name])) # finding entry/exits nodes of the graph. From a18a060b6758989cdeb240e02d8dc89a6ac322a5 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Thu, 26 Sep 2024 15:56:54 +0200 Subject: [PATCH 148/379] Refactor nir_graph_extractor.py --- .../backend/dynapcnn/nir_graph_extractor.py | 206 ++++++++++-------- sinabs/backend/dynapcnn/utils.py | 6 +- 2 files changed, 119 insertions(+), 93 deletions(-) diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index 9b708de1..9da036bd 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -2,7 +2,7 @@ # contact : wsoaresgirao@gmail.com import copy -from typing import Tuple, Dict, List +from typing import Tuple, Dict, List, Type import nirtorch import sinabs @@ -26,13 +26,13 @@ def __init__(self, spiking_model: nn.Module, dummy_input: torch.tensor): Attributes ---------- - - edges_list (list of 2-tuples of integers): + - edges (set of 2-tuples of integers): Tuples describing the connections between layers in `spiking_model`. Each layer (node) is identified by a unique integer ID. - name_2_index_map (dict): Keys are original variable names of layers in `spiking_model`. Values are unique integer IDs. - - entry_nodes (list of ints): + - entry_nodes (set of ints): IDs of nodes acting as entry points for the network, i.e. receiving external input. - modules_map (dict): Map from layer ID to the corresponding nn.Module instance. @@ -42,7 +42,7 @@ def __init__(self, spiking_model: nn.Module, dummy_input: torch.tensor): nir_graph = nirtorch.extract_torch_graph(spiking_model, dummy_input, model_name=None).ignore_tensors() # converts the NIR representation into a list of edges with nodes represented as integers. - self._edges_list, self._name_2_indx_map, self._entry_nodes = self._get_edges_from_nir(nir_graph) + self._edges, self._name_2_indx_map, self._entry_nodes = self._get_edges_from_nir(nir_graph) # recovers the associated `nn.Module` (layer) of each node. self.modules_map = self._get_named_modules(spiking_model) @@ -53,81 +53,60 @@ def __init__(self, spiking_model: nn.Module, dummy_input: torch.tensor): ####################################################### Publich Methods ####################################################### @property - def entry_nodes(self) -> List[int]: + def entry_nodes(self) -> Set[int]: return self._entry_nodes @property - def edges_list(self): - return self._edges_list + def edges(self) -> Set[Tuple[int, int]]: + return self._edges @property - def name_2_indx_map(self): + def name_2_indx_map(self) -> Dict[str, int]: return self._name_2_indx_map @property - def nodes_io_shapes(self): + def nodes_io_shapes(self) -> Dict[int, torch.Size]: return self._nodes_io_shapes + + @property + def sorted_nodes(self) -> List[int]: + return self._sort_graph_nodes() - def remove_ignored_nodes(self, default_ignored_nodes: tuple) -> Tuple[list, dict]: - """ Recreates the edges list based on layers that `DynapcnnNetwork` will ignore. This + def remove_ignored_nodes(self, ignored_node_classes: Tuple[Type]) -> Tuple[Set[int], Dict[int, int]]: + """ Create a new set of edges, considering layers that `DynapcnnNetwork` will ignore. This is done by setting the source (target) node of an edge where the source (target) node will be dropped as the node that originally targeted this node to be dropped. Parameters ---------- - - default_ignored_nodes (tuple): a set of layers (`nn.Module`) that should be ignored from the graph. + - ignored_node_classes (tuple of types): + Layer classes that should be ignored from the graph. Returns ---------- - - remapped_edges (list): the new list of edges after nodes flagged by `default_ignored_nodes` have been removed. - - remapped_nodes (dict): updated nodes' IDs after nodes flagged by `default_ignored_nodes` have been removed. + - new_edges (set): the new set of edges after nodes flagged by `ignored_node_classes` have been removed. + - remapped_nodes (dict): updated nodes' IDs after nodes flagged by `ignored_node_classes` have been removed. """ - edges = copy.deepcopy(self._edges_list) - parsed_edges = [] - removed_nodes = [] - - # removing ignored nodes from edges. - for edge_idx in range(len(edges)): - _src = edges[edge_idx][0] - _trg = edges[edge_idx][1] - - if isinstance(self.modules_map[_src], default_ignored_nodes): - removed_nodes.append(_src) - # all edges where node '_src' is target change it to node '_trg' as their target. - for edge in edges: - if edge[1] == _src: - new_edge = (edge[0], _trg) - elif isinstance(self.modules_map[_trg], default_ignored_nodes): - removed_nodes.append(_trg) - # all edges where node '_trg' is source change it to node '_src' as their source. - for edge in edges: - if edge[0] == _trg: - new_edge = (_src, edge[1]) - else: - new_edge = (_src, _trg) - - if new_edge not in parsed_edges: - parsed_edges.append(new_edge) - - removed_nodes = list(set(removed_nodes)) - - # remapping nodes indexes. - remapped_nodes = {} - for node_indx, __ in self.modules_map.items(): - _ = [x for x in removed_nodes if node_indx > x] - remapped_nodes[node_indx] = node_indx - len(_) - - for x in removed_nodes: - del remapped_nodes[x] - - # remapping nodes names in parsed edges. - remapped_edges = [] - for edge in parsed_edges: - remapped_edges.append((remapped_nodes[edge[0]], remapped_nodes[edge[1]])) - - return remapped_edges, remapped_nodes + # Compose new graph by creating a dict with all remaining node IDs as keys and set of target node IDs as values + source2target: Dict[int, Set[int]] = { + node: self._find_valid_targets(node, ignored_node_classes) + for node in self.sorted_nodes + # Skip ignored nodes + if not isinstance(self.modules_map[node], ignored_node_classes) + } + + # remapping nodes indices contiguously starting from 0 + remapped_nodes = {old_idx: new_idx for new_idx, old_idx in enumerate(sorted(source2target.keys()))} + + # Parse new set of edges based on remapped node IDs + new_edges = { + (remapped_nodes[src], remapped_nodes[tgt]) + for src, targets in source2target.items() + for tgt in targets + } + + return new_edges, remapped_nodes - # TODO - it would be good if I/O shapes were returned by the NIR graph. def get_node_io_shapes(self, node: int) -> Tuple[torch.Size, torch.Size]: """ Returns the I/O tensors' shapes of `node`. @@ -150,12 +129,12 @@ def _get_edges_from_nir(self, nir_graph: nirtorch.graph.Graph) -> Tuple[List[Tup Returns ---------- - - edges_list (list): tuples describing the connections between layers in `spiking_model`. + - edges (set): tuples describing the connections between layers in `spiking_model`. - name_2_indx_map (dict): `key` is the original variable name for a layer in `spiking_model` and `value is an integer representing the layer in a standard format. - - entry_nodes (list): IDs of nodes acting as entry points for the network (i.e., receiving external input). + - entry_nodes (set): IDs of nodes acting as entry points for the network (i.e., receiving external input). """ - edges_list = [] + edges = set() name_2_indx_map = {} # TODO maybe make sure the input node from nir always gets assined `0`. @@ -172,15 +151,14 @@ def _get_edges_from_nir(self, nir_graph: nirtorch.graph.Graph) -> Tuple[List[Tup name_2_indx_map[trg_node.name] = len(name_2_indx_map) # Store the edge of current node to the target - edges_list.append((name_2_indx_map[src_node.name], name_2_indx_map[trg_node.name])) + edges.add((name_2_indx_map[src_node.name], name_2_indx_map[trg_node.name])) # finding entry/exits nodes of the graph. - all_sources = [x[0] for x in edges_list] - all_targets = [x[1] for x in edges_list] + all_sources, all_targets = zip(*edges) - entry_nodes = list(set(all_sources) - set(all_targets)) + entry_nodes = set(all_sources) - set(all_targets) - return edges_list, name_2_indx_map, entry_nodes + return edges, name_2_indx_map, entry_nodes def _get_named_modules(self, model: nn.Module) -> Dict[int, nn.Module]: """ Find for each node in the graph what its associated layer in `model` is. @@ -212,6 +190,20 @@ def _get_named_modules(self, model: nn.Module) -> Dict[int, nn.Module]: return modules_map + def _sort_graph_nodes(self) -> List[int]: + """ Sort graph nodes topologically. + + Returns + ------- + - sorted_nodes (list of integers): IDs of nodes, sorted. + """ + # Make a temporary copy of edges and include inputs + temp_edges = {(src, tgt) for src, tgt in self.edges} + for node in self._entry_nodes: + temp_edges.add(('input', node)) + return topological_sorting(temp_edges) + + def _get_nodes_io_shapes(self, input_dummy: torch.tensor) -> Dict[int, Dict[str, torch.Size]]: """ Iteratively calls the forward method of each `nn.Module` (i.e., a layer/node in the graph) using the topologically sorted nodes extracted from the computational graph of the model being parsed. @@ -226,24 +218,18 @@ def _get_nodes_io_shapes(self, input_dummy: torch.tensor) -> Dict[int, Dict[str, """ nodes_io_map = {} - # topological sorting of the graph. - temp_edges_list = copy.deepcopy(self._edges_list) - for node in self._entry_nodes: - temp_edges_list.append(('input', node)) - sorted_nodes = topological_sorting(temp_edges_list) - # propagate inputs through the nodes. - for node in sorted_nodes: + for node in self.sorted_nodes: if isinstance(self.modules_map[node], sinabs.layers.merge.Merge): - # find `Merge` arguments (at this point the output of Merge has to have been calculated). + # find `Merge` arguments (at this point the inputs to Merge should have been calculated). arg1, arg2 = self._find_merge_arguments(node) # retrieve arguments output tensors. arg1_out = nodes_io_map[arg1]['output'] arg2_out = nodes_io_map[arg2]['output'] - # TODO - this is currently a limitation inpused by the validation checks done by Speck once a configuration: it wants two + # TODO - this is currently a limitation imposed by the validation checks done by Speck once a configuration: it wants two # different input sources to a core to have the same output shapes. if arg1_out.shape != arg2_out.shape: raise ValueError(f'Layer `sinabs.layers.merge.Merge` (node {node}) require two input tensors with the same shape: arg1.shape {arg1_out.shape} differs from arg2.shape {arg2_out.shape}.') @@ -281,6 +267,20 @@ def _get_nodes_io_shapes(self, input_dummy: torch.tensor) -> Dict[int, Dict[str, return nodes_io_map + def _find_all_sources_of_input_to(self, node: int) -> Set[int]: + """ Finds all source nodes to `node`. + + Parameters + ---------- + - node (int): the node in the computational graph for which we whish to find the input source (either another node in the + graph or the original input itself to the network). + + Returns + ---------- + - input sources (set of int): IDs of the nodes in the computational graph providing the input to `node`. + """ + return set(src for (src, tgt) in self._edges if tgt == node) + def _find_source_of_input_to(self, node: int) -> int: """ Finds the first edge `(X, node)` returns `X`. @@ -295,11 +295,12 @@ def _find_source_of_input_to(self, node: int) -> int: receiving outside input (i.e., it is a starting node) the return will be -1. For example, this will be the case when a network with two independent branches (each starts from a different "input node") merge along the computational graph. """ - for edge in self._edges_list: - if edge[1] == node: - return edge[0] - - return -1 + sources = self._find_all_sources_of_input_to(node) + if len(sources) == 0: + return -1 + if len(sources) > 1: + raise RuntimeError(f"Node {node} has more than 1 input") + return sources.pop() def _find_merge_arguments(self, merge_node: int) -> Tuple[int, int]: """ A `Merge` layer receives two inputs. Return the two inputs to `merge_node` representing a `Merge` layer. @@ -308,13 +309,38 @@ def _find_merge_arguments(self, merge_node: int) -> Tuple[int, int]: ---------- - args (tuple): the IDs of the nodes that provice the input arguments to a `Merge` layer. """ - args = [] - - for edge in self._edges_list: - if edge[1] == merge_node: - args.append(edge[0]) + sources = self._find_all_sources_of_input_to(node) - if len(args) == 2: - return tuple(args) - else: - raise ValueError(f'Number of arguments found for `Merge` node {merge_node} is {len(args)} (should be 2).') \ No newline at end of file + if len(sources) != 2: + raise ValueError(f'Number of arguments found for `Merge` node {merge_node} is {len(args)} (should be 2).') + + return tuple(sources) + + def _find_valid_targets(self, node: int, ignored_node_classes: Tuple[Type]) -> Set[int]: + """ Find all targets of a node that are not ignored classes + + Return a set of all target nodes that are not of an ignored class. + For target nodes of ignored classes, recursively return their valid + targets. + + Parameters + ---------- + - node (int): ID of node whose targets should be found + - ignored_node_classes (tuple of types): Classes of which nodes should be skiped + + Returns + ------- + - valid_targets (set of int): Set of all recursively found target IDs + """ + targets = set() + for (src, tgt) in self.edges: + # Search for all edges with node as source + if src == node: + if isinstance(self.modules_map[tgt], ignored_node_classes): + # Find valid targets of target + targets.join(self._find_valid_targets(tgt, ignored_node_classes)) + else: + # Target is valid, add it to `targets` + targets.add(tgt) + return targets + diff --git a/sinabs/backend/dynapcnn/utils.py b/sinabs/backend/dynapcnn/utils.py index 3b7d4a9b..c5c43149 100644 --- a/sinabs/backend/dynapcnn/utils.py +++ b/sinabs/backend/dynapcnn/utils.py @@ -395,13 +395,13 @@ def build_SumPool2d(module: nn.AvgPool2d) -> Tuple[sl.SumPool2d, int]: return lyr_pool, rescale_factor -def topological_sorting(edges: List[Tuple[int, int]]) -> List[int]: +def topological_sorting(edges: Set[Tuple[int, int]]) -> List[int]: """ Performs a topological sorting (using Kahn's algorithm) of a graph descrobed by a list edges. An entry node `X` of the graph have to be flagged inside `edges` by a tuple `('input', X)`. Parameters ---------- - - edges (list): the edges describing the *acyclic* graph. + - edges (set): the edges describing the *acyclic* graph. Returns ---------- @@ -921,4 +921,4 @@ def extend_readout_layer(model: "DynapcnnNetwork") -> "DynapcnnNetwork": # else: # raise TypeError("Expected torch.nn.Sequential or sinabs.Network") -# return layers \ No newline at end of file +# return layers From e138673bd76c3c3ae00a8d128e06e52bb2e94af3 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Thu, 26 Sep 2024 18:32:33 +0200 Subject: [PATCH 149/379] NIRtoDynapcnnNetowrkGraph properties return copies so that original objects remain unchanged --- .../backend/dynapcnn/nir_graph_extractor.py | 154 ++++++++++-------- 1 file changed, 90 insertions(+), 64 deletions(-) diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index 9da036bd..0977fc01 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -2,20 +2,22 @@ # contact : wsoaresgirao@gmail.com import copy -from typing import Tuple, Dict, List, Type +from typing import Dict, List, Tuple, Type import nirtorch -import sinabs import torch import torch.nn as nn +import sinabs + from .utils import topological_sorting -class NIRtoDynapcnnNetworkGraph(): + +class NIRtoDynapcnnNetworkGraph: def __init__(self, spiking_model: nn.Module, dummy_input: torch.tensor): - """ Class implementing the extraction of the computational graph from `spiking_model`, where + """Class implementing the extraction of the computational graph from `spiking_model`, where each node represents a layer in the model and the list of edges represents how the data flow between - the layers. + the layers. Parameters ---------- @@ -39,11 +41,15 @@ def __init__(self, spiking_model: nn.Module, dummy_input: torch.tensor): """ # extract computational graph. - nir_graph = nirtorch.extract_torch_graph(spiking_model, dummy_input, model_name=None).ignore_tensors() + nir_graph = nirtorch.extract_torch_graph( + spiking_model, dummy_input, model_name=None + ).ignore_tensors() # converts the NIR representation into a list of edges with nodes represented as integers. - self._edges, self._name_2_indx_map, self._entry_nodes = self._get_edges_from_nir(nir_graph) - + self._edges, self._name_2_indx_map, self._entry_nodes = ( + self._get_edges_from_nir(nir_graph) + ) + # recovers the associated `nn.Module` (layer) of each node. self.modules_map = self._get_named_modules(spiking_model) @@ -54,26 +60,28 @@ def __init__(self, spiking_model: nn.Module, dummy_input: torch.tensor): @property def entry_nodes(self) -> Set[int]: - return self._entry_nodes + return {n for n in self._entry_nodes} @property def edges(self) -> Set[Tuple[int, int]]: - return self._edges - + return {(src, tgt) for src, tgt in self._edges} + @property def name_2_indx_map(self) -> Dict[str, int]: - return self._name_2_indx_map - + return {name: idx for name, idx in self._name_2_indx_map.items()} + @property def nodes_io_shapes(self) -> Dict[int, torch.Size]: - return self._nodes_io_shapes - + return {n: size for n, size in self._nodes_io_shapes.items()} + @property def sorted_nodes(self) -> List[int]: - return self._sort_graph_nodes() + return [n for n in self._sort_graph_nodes()] - def remove_ignored_nodes(self, ignored_node_classes: Tuple[Type]) -> Tuple[Set[int], Dict[int, int]]: - """ Create a new set of edges, considering layers that `DynapcnnNetwork` will ignore. This + def remove_ignored_nodes( + self, ignored_node_classes: Tuple[Type] + ) -> Tuple[Set[int], Dict[int, int]]: + """Create a new set of edges, considering layers that `DynapcnnNetwork` will ignore. This is done by setting the source (target) node of an edge where the source (target) node will be dropped as the node that originally targeted this node to be dropped. @@ -96,7 +104,10 @@ def remove_ignored_nodes(self, ignored_node_classes: Tuple[Type]) -> Tuple[Set[i } # remapping nodes indices contiguously starting from 0 - remapped_nodes = {old_idx: new_idx for new_idx, old_idx in enumerate(sorted(source2target.keys()))} + remapped_nodes = { + old_idx: new_idx + for new_idx, old_idx in enumerate(sorted(source2target.keys())) + } # Parse new set of edges based on remapped node IDs new_edges = { @@ -106,27 +117,32 @@ def remove_ignored_nodes(self, ignored_node_classes: Tuple[Type]) -> Tuple[Set[i } return new_edges, remapped_nodes - + def get_node_io_shapes(self, node: int) -> Tuple[torch.Size, torch.Size]: - """ Returns the I/O tensors' shapes of `node`. + """Returns the I/O tensors' shapes of `node`. Returns ---------- - input shape (torch.Size): shape of the input tensor to `node`. - output shape (torch.Size): shape of the output tensor from `node`. """ - return self._nodes_io_shapes[node]['input'], self._nodes_io_shapes[node]['output'] + return ( + self._nodes_io_shapes[node]["input"], + self._nodes_io_shapes[node]["output"], + ) ####################################################### Pivate Methods ####################################################### - def _get_edges_from_nir(self, nir_graph: nirtorch.graph.Graph) -> Tuple[List[Tuple[int, int]], Dict[str, int], List[int]]: - """ Standardize the representation of `nirtorch.graph.Graph` into a list of edges (`Tuple[int, int]`) where + def _get_edges_from_nir( + self, nir_graph: nirtorch.graph.Graph + ) -> Tuple[List[Tuple[int, int]], Dict[str, int], List[int]]: + """Standardize the representation of `nirtorch.graph.Graph` into a list of edges (`Tuple[int, int]`) where each node in `nir_graph` is represented by an interger (with the source node starting as `0`). Parameters ---------- - nir_graph (nirtorch.graph.Graph): a NIR graph representation of `spiking_model`. - + Returns ---------- - edges (set): tuples describing the connections between layers in `spiking_model`. @@ -136,9 +152,9 @@ def _get_edges_from_nir(self, nir_graph: nirtorch.graph.Graph) -> Tuple[List[Tup """ edges = set() name_2_indx_map = {} - + # TODO maybe make sure the input node from nir always gets assined `0`. - + for src_node in nir_graph.node_list: # Make sure current node is in `name_2_indx_map` if src_node.name not in name_2_indx_map: @@ -151,7 +167,9 @@ def _get_edges_from_nir(self, nir_graph: nirtorch.graph.Graph) -> Tuple[List[Tup name_2_indx_map[trg_node.name] = len(name_2_indx_map) # Store the edge of current node to the target - edges.add((name_2_indx_map[src_node.name], name_2_indx_map[trg_node.name])) + edges.add( + (name_2_indx_map[src_node.name], name_2_indx_map[trg_node.name]) + ) # finding entry/exits nodes of the graph. all_sources, all_targets = zip(*edges) @@ -159,9 +177,9 @@ def _get_edges_from_nir(self, nir_graph: nirtorch.graph.Graph) -> Tuple[List[Tup entry_nodes = set(all_sources) - set(all_targets) return edges, name_2_indx_map, entry_nodes - + def _get_named_modules(self, model: nn.Module) -> Dict[int, nn.Module]: - """ Find for each node in the graph what its associated layer in `model` is. + """Find for each node in the graph what its associated layer in `model` is. Parameters ---------- @@ -173,10 +191,12 @@ def _get_named_modules(self, model: nn.Module) -> Dict[int, nn.Module]: """ modules_map = {} - if isinstance(model, nn.Sequential): # TODO shouldn't accept `nn.Sequential` any longer. + if isinstance( + model, nn.Sequential + ): # TODO shouldn't accept `nn.Sequential` any longer. # access modules via `.named_modules()`. for name, module in model.named_modules(): - if name != '': + if name != "": # skip the module itself. modules_map[self._name_2_indx_map[name]] = module @@ -186,26 +206,27 @@ def _get_named_modules(self, model: nn.Module) -> Dict[int, nn.Module]: modules_map[self._name_2_indx_map[name]] = module else: - raise ValueError('Either a nn.Sequential or a nn.Module is required.') + raise ValueError("Either a nn.Sequential or a nn.Module is required.") return modules_map - + def _sort_graph_nodes(self) -> List[int]: - """ Sort graph nodes topologically. + """Sort graph nodes topologically. Returns ------- - sorted_nodes (list of integers): IDs of nodes, sorted. """ # Make a temporary copy of edges and include inputs - temp_edges = {(src, tgt) for src, tgt in self.edges} + temp_edges = self.edges for node in self._entry_nodes: - temp_edges.add(('input', node)) + temp_edges.add(("input", node)) return topological_sorting(temp_edges) - - - def _get_nodes_io_shapes(self, input_dummy: torch.tensor) -> Dict[int, Dict[str, torch.Size]]: - """ Iteratively calls the forward method of each `nn.Module` (i.e., a layer/node in the graph) using the topologically + + def _get_nodes_io_shapes( + self, input_dummy: torch.tensor + ) -> Dict[int, Dict[str, torch.Size]]: + """Iteratively calls the forward method of each `nn.Module` (i.e., a layer/node in the graph) using the topologically sorted nodes extracted from the computational graph of the model being parsed. Parameters @@ -226,19 +247,21 @@ def _get_nodes_io_shapes(self, input_dummy: torch.tensor) -> Dict[int, Dict[str, arg1, arg2 = self._find_merge_arguments(node) # retrieve arguments output tensors. - arg1_out = nodes_io_map[arg1]['output'] - arg2_out = nodes_io_map[arg2]['output'] + arg1_out = nodes_io_map[arg1]["output"] + arg2_out = nodes_io_map[arg2]["output"] - # TODO - this is currently a limitation imposed by the validation checks done by Speck once a configuration: it wants two + # TODO - this is currently a limitation imposed by the validation checks done by Speck once a configuration: it wants two # different input sources to a core to have the same output shapes. if arg1_out.shape != arg2_out.shape: - raise ValueError(f'Layer `sinabs.layers.merge.Merge` (node {node}) require two input tensors with the same shape: arg1.shape {arg1_out.shape} differs from arg2.shape {arg2_out.shape}.') + raise ValueError( + f"Layer `sinabs.layers.merge.Merge` (node {node}) require two input tensors with the same shape: arg1.shape {arg1_out.shape} differs from arg2.shape {arg2_out.shape}." + ) # forward input through the node. _output = self.modules_map[node](arg1_out, arg2_out) # save node's I/O tensors. - nodes_io_map[node] = {'input': arg1_out, 'output': _output} + nodes_io_map[node] = {"input": arg1_out, "output": _output} else: @@ -247,34 +270,34 @@ def _get_nodes_io_shapes(self, input_dummy: torch.tensor) -> Dict[int, Dict[str, _output = self.modules_map[node](input_dummy) # save node's I/O tensors. - nodes_io_map[node] = {'input': input_dummy, 'output': _output} + nodes_io_map[node] = {"input": input_dummy, "output": _output} else: # find node generating the input to be used. input_node = self._find_source_of_input_to(node) - _input = nodes_io_map[input_node]['output'] + _input = nodes_io_map[input_node]["output"] # forward input through the node. _output = self.modules_map[node](_input) # save node's I/O tensors. - nodes_io_map[node] = {'input': _input, 'output': _output} + nodes_io_map[node] = {"input": _input, "output": _output} # replace the I/O tensor information by its shape information. for node, io in nodes_io_map.items(): - nodes_io_map[node]['input'] = io['input'].shape - nodes_io_map[node]['output'] = io['output'].shape + nodes_io_map[node]["input"] = io["input"].shape + nodes_io_map[node]["output"] = io["output"].shape return nodes_io_map def _find_all_sources_of_input_to(self, node: int) -> Set[int]: - """ Finds all source nodes to `node`. + """Finds all source nodes to `node`. Parameters ---------- - node (int): the node in the computational graph for which we whish to find the input source (either another node in the graph or the original input itself to the network). - + Returns ---------- - input sources (set of int): IDs of the nodes in the computational graph providing the input to `node`. @@ -282,17 +305,17 @@ def _find_all_sources_of_input_to(self, node: int) -> Set[int]: return set(src for (src, tgt) in self._edges if tgt == node) def _find_source_of_input_to(self, node: int) -> int: - """ Finds the first edge `(X, node)` returns `X`. + """Finds the first edge `(X, node)` returns `X`. Parameters ---------- - node (int): the node in the computational graph for which we whish to find the input source (either another node in the graph or the original input itself to the network). - + Returns ---------- - input source (int): ID of the node in the computational graph providing the input to `node`. If `node` is - receiving outside input (i.e., it is a starting node) the return will be -1. For example, this will be the case + receiving outside input (i.e., it is a starting node) the return will be -1. For example, this will be the case when a network with two independent branches (each starts from a different "input node") merge along the computational graph. """ sources = self._find_all_sources_of_input_to(node) @@ -303,21 +326,25 @@ def _find_source_of_input_to(self, node: int) -> int: return sources.pop() def _find_merge_arguments(self, merge_node: int) -> Tuple[int, int]: - """ A `Merge` layer receives two inputs. Return the two inputs to `merge_node` representing a `Merge` layer. + """A `Merge` layer receives two inputs. Return the two inputs to `merge_node` representing a `Merge` layer. Returns ---------- - args (tuple): the IDs of the nodes that provice the input arguments to a `Merge` layer. """ sources = self._find_all_sources_of_input_to(node) - + if len(sources) != 2: - raise ValueError(f'Number of arguments found for `Merge` node {merge_node} is {len(args)} (should be 2).') - + raise ValueError( + f"Number of arguments found for `Merge` node {merge_node} is {len(args)} (should be 2)." + ) + return tuple(sources) - def _find_valid_targets(self, node: int, ignored_node_classes: Tuple[Type]) -> Set[int]: - """ Find all targets of a node that are not ignored classes + def _find_valid_targets( + self, node: int, ignored_node_classes: Tuple[Type] + ) -> Set[int]: + """Find all targets of a node that are not ignored classes Return a set of all target nodes that are not of an ignored class. For target nodes of ignored classes, recursively return their valid @@ -333,7 +360,7 @@ def _find_valid_targets(self, node: int, ignored_node_classes: Tuple[Type]) -> S - valid_targets (set of int): Set of all recursively found target IDs """ targets = set() - for (src, tgt) in self.edges: + for src, tgt in self.edges: # Search for all edges with node as source if src == node: if isinstance(self.modules_map[tgt], ignored_node_classes): @@ -343,4 +370,3 @@ def _find_valid_targets(self, node: int, ignored_node_classes: Tuple[Type]) -> S # Target is valid, add it to `targets` targets.add(tgt) return targets - From f48f43f5e9f95ccbb82e38117c69e5ca8a9807ff Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Fri, 27 Sep 2024 08:19:51 +0200 Subject: [PATCH 150/379] Refactor edge_handler --- .../backend/dynapcnn/sinabs_edges_handler.py | 261 +++++++++++------- sinabs/backend/dynapcnn/sinabs_edges_utils.py | 78 ++++-- sinabs/backend/dynapcnn/utils.py | 257 ++++++++++------- 3 files changed, 377 insertions(+), 219 deletions(-) diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index bf02f334..5fae43c5 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -5,14 +5,21 @@ contact : williansoaresgirao@gmail.com """ -from typing import Tuple, List, Dict -import sinabs.layers +import copy +from typing import Dict, List, Tuple, Type + import torch.nn as nn + +import sinabs +import sinabs.layers + from .sinabs_edges_utils import * -import sinabs, copy -def process_edge(layers: Dict[int, nn.Module], edge: Tuple[int, int], mapper: dict) -> None: - """ Read in an edge describing the connection between two layers (nodes in the computational graph). If `edge` + +def process_edge( + layers: Dict[int, nn.Module], edge: Tuple[int, int], mapper: dict +) -> None: + """Read in an edge describing the connection between two layers (nodes in the computational graph). If `edge` is a valid connection between two layers, update `mapper` to incorporate these layers into a new or existing dictonary containing the modules comprising a future `DynacnnLayer` object. @@ -21,46 +28,54 @@ def process_edge(layers: Dict[int, nn.Module], edge: Tuple[int, int], mapper: di created and its set of nodes will include node `0` and node `1`: mapper[0] = { - 0: {'layer': Conv2d, 'input_shape': None, 'output_shape': None}, + 0: {'layer': Conv2d, 'input_shape': None, 'output_shape': None}, 1: {'layer': IAFSqueeze, 'input_shape': None, 'output_shape': None}, ... } Parameters ---------- - layers (dict): a dictionary containing the nodes of the graph as `key` and their associated module as `value`. + layers (dict): a dictionary containing the node IDs of the graph as `key` and their associated module as `value`. edge (tuple): tuple representing the connection between two nodes in computational graph of 'DynapcnnNetworkGraph.snn.spiking_model'. mapper (dict): dictionary where each 'key' is the index of a future 'DynapcnnLayer' and 'value' is a dictionary ('key': node, 'value': module). """ - edge_type = is_valid_edge(edge, layers, VALID_SINABS_EDGES) + edge_type = get_valid_edge_type(edge, layers, VALID_SINABS_EDGE_TYPE_IDS) - if isinstance(edge_type, int): # incorporate modules within the edge to a dict representing a future DynapcnnLayer. - update_dynapcnnlayer_mapper(edge_type, edge, mapper, layers) - else: + if edge_type is None: raise InvalidEdge(edge, type(layers[edge[0]]), type(layers[edge[1]])) - -def is_valid_edge(edge: Tuple[int, int], layers: Dict[int, nn.Module], valid_edges_map: dict) -> int: - """ Checks if the modules each node in 'edge' represent are a valid connection between a sinabs network to be - loaded on Speck. + + # incorporate modules within the edge to a dict representing a future DynapcnnLayer. + update_dynapcnnlayer_mapper(edge_type, edge, mapper, layers) + + +def get_valid_edge_type( + edge: Tuple[int, int], + layers: Dict[int, nn.Module], + valid_edge_ids: Dict[Tuple[Type, Type], int], +) -> int: + """Checks if the modules each node in 'edge' represent are a valid connection between a sinabs network to be + loaded on Speck and return the edge type Parameters ---------- - valid_edges_map: dictionary where each 'key' is the type (index) of a pre-defined valid edge. + edge (tuple of two int): The edge whose type is to be inferred + layers (Dict): Dict with node IDs as keys and layer instances as values + valid_edge_ids: Dict with valid edge-types (tuples of Types) as keys and edge-type-ID as value Returns ---------- edge_type: the edge type specified in 'valid_edges_map' ('None' if edge is not valid). """ - edge_layers = (layers[edge[0]], layers[edge[1]]) + source_type = type(layers[edge[0]]) + target_type = type(layers[edge[1]]) - for edge_type, sinabs_edge in valid_edges_map.items(): - if (type(edge_layers[0]) == sinabs_edge[0]) and (type(edge_layers[1]) == sinabs_edge[1]): - return edge_type + return valid_edge_ids.get((source_type, target_type), None) - return None - -def update_dynapcnnlayer_mapper(edge_type: int, edge: Tuple[int, int], mapper: dict, layers: Dict[int, nn.Module]) -> None: - """ Parses the nodes within an edge and incorporate them either into a **new** or an **already existing** DynapcnnLayer represented + +def update_dynapcnnlayer_mapper( + edge_type: int, edge: Tuple[int, int], mapper: dict, layers: Dict[int, nn.Module] +) -> None: + """Parses the nodes within an edge and incorporate them either into a **new** or an **already existing** DynapcnnLayer represented in 'mapper'. """ @@ -69,90 +84,106 @@ def update_dynapcnnlayer_mapper(edge_type: int, edge: Tuple[int, int], mapper: d elif edge_type in [1, 7]: add_pool_to_dynapcnnlayer_blk(mapper, edge, layers) - + elif edge_type in [2, 3, 4, 5, 8, 9]: connect_dynapcnnlayer_blks(mapper, edge, layers) else: raise InvalidEdgeType(edge, edge_type) - -def init_xor_complete_new_dynapcnnlayer_blk(mapper: dict, edge: Tuple[int, int], layers: Dict[int, nn.Module]) -> None: - """ Incorporates nodes from either a `(conv, neuron)` or a `(linear, neuron)` edge. These are either initiating a new `dict` mapping - into a future `DynapcnnLayer` or completing a `conv->neuron` sequence (in the case the node for `conv` as already been incorporated + + +def init_xor_complete_new_dynapcnnlayer_blk( + mapper: dict, edge: Tuple[int, int], layers: Dict[int, nn.Module] +) -> None: + """Incorporates nodes from either a `(conv, neuron)` or a `(linear, neuron)` edge. These are either initiating a new `dict` mapping + into a future `DynapcnnLayer` or completing a `conv->neuron` sequence (in the case the node for `conv` as already been incorporated somewhere in `mapper`). Obs.: `nn.Linear` layers are converted into `nn.Conv2d` by `DynapcnnLayer`. """ - matched = False - dynapcnnlayer_indx = 0 - - for indx, dynapcnnlayer in mapper.items(): # see if 'edge[0]' exists in a DynapcnnLayer block. - for node, _ in dynapcnnlayer.items(): - if node == edge[0]: - dynapcnnlayer_indx = indx - matched = True - break - if matched: # 'edge[0]' found: 'edge[1]' belongs to its DynapcnnLayer block. - mapper[dynapcnnlayer_indx][edge[1]] = {'layer': layers[edge[1]], 'input_shape': None, 'output_shape': None} - break - - if not matched: # 'edge[0]' not found: start new DynapcnnLayer block. - dynapcnnlayer_indx = 0 - for indx, _ in mapper.items(): - dynapcnnlayer_indx += 1 + # Search for edge[0] (conv/linear layer) in DynapcnnLayers + if (dynapcnnlayer_indx := find_initialized_node(edge[0], mapper)) is not None: + # Add edge[1] (neuron layer) to the same dynapcnn layer + mapper[dynapcnnlayer_indx][edge[1]] = { + "layer": layers[edge[1]], + "input_shape": None, + "output_shape": None, + } + else: + # Assign new layer, with current length of `mapper` as new unique index + dynapcnnlayer_indx = len(mapper) mapper[dynapcnnlayer_indx] = { - edge[0]: {'layer': layers[edge[0]], 'input_shape': None, 'output_shape': None}, - edge[1]: {'layer': layers[edge[1]], 'input_shape': None, 'output_shape': None} - } - -def connect_dynapcnnlayer_blks(mapper: dict, edge: Tuple[int, int], layers: Dict[int, nn.Module]) -> None: - """ Incorporates nodes from either a `(neuron, conv)/(neuron, lin)` or `(pool, conv)/(pool, lin)` edge. These represent connections between an existing - `dict` in `mapper` that will be mapped into a `DynapcnnLayer` and a new one yet to be represented in `mapper`. Obs.: `nn.Linear` layers are converted + edge[0]: { + "layer": layers[edge[0]], + "input_shape": None, + "output_shape": None, + }, + edge[1]: { + "layer": layers[edge[1]], + "input_shape": None, + "output_shape": None, + }, + } + + +def connect_dynapcnnlayer_blks( + mapper: dict, edge: Tuple[int, int], layers: Dict[int, nn.Module] +) -> None: + """Incorporates nodes from either a `(neuron, conv)/(neuron, lin)` or `(pool, conv)/(pool, lin)` edge. These represent connections between an existing + `dict` in `mapper` that will be mapped into a `DynapcnnLayer` and a new one yet to be represented in `mapper`. Obs.: `nn.Linear` layers are converted into `nn.Conv2d` by `DynapcnnLayer`. """ - if not is_initialized_node(edge[1], mapper): + if find_initialized_node(edge[1], mapper) is None: dynapcnnlayer_indx = 0 matched = False for indx, dynapcnnlayer in mapper.items(): for node, _ in dynapcnnlayer.items(): - if node == edge[0]: # 'edge[0]' is ending DynapcnnLayer block 'indx'. - dynapcnnlayer_indx = indx+1 + if node == edge[0]: # 'edge[0]' is ending DynapcnnLayer block 'indx'. + dynapcnnlayer_indx = indx + 1 matched = True break if matched: break if matched: - while (dynapcnnlayer_indx in mapper): + while dynapcnnlayer_indx in mapper: dynapcnnlayer_indx += 1 - mapper[dynapcnnlayer_indx] = { # 'edge[1]' starts new DynapcnnLayer block. - edge[1]: {'layer': layers[edge[1]], 'input_shape': None, 'output_shape': None}} + mapper[dynapcnnlayer_indx] = { # 'edge[1]' starts new DynapcnnLayer block. + edge[1]: { + "layer": layers[edge[1]], + "input_shape": None, + "output_shape": None, + } + } else: raise UnmatchedNode(edge, node) - -def add_pool_to_dynapcnnlayer_blk(mapper: dict, edge: Tuple[int, int], layers: Dict[int, nn.Module]) -> None: - """ Incorporating a `(neuron, pool)` edge. Node `pool` has to be part of an already existing `dict` mapping into a `DynapcnnLaye` in `mapper`. - """ - matched = False - for indx, dynapcnnlayer in mapper.items(): - for node, _ in dynapcnnlayer.items(): - if node == edge[0]: - dynapcnnlayer[edge[1]] = { # 'edge[0]' is a neuron layer inputing into pooling layer 'edge[1]'. - 'layer': layers[edge[1]], 'input_shape': None, 'output_shape': None} - matched = True - break - if matched: - break - if not matched: + + +def add_pool_to_dynapcnnlayer_blk( + mapper: dict, edge: Tuple[int, int], layers: Dict[int, nn.Module] +) -> None: + """Incorporating a `(neuron, pool)` edge. Node `pool` has to be part of an already existing `dict` mapping into a `DynapcnnLaye` in `mapper`.""" + # Search for edge[0] (neuron layer) in DynapcnnLayers + if (indx := find_initialized_node(edge[0], mapper)) is not None: + # Add edge[1] (pooling layer) to the same dynapcnn layer + mapped[indx][edge[1]] = { + "layer": layers[edge[1]], + "input_shape": None, + "output_shape": None, + } + else: raise UnmatchedNode(edge, node) - -def is_initialized_node(node: int, mapper: dict) -> bool: - """ Finds if 'node' existis within 'mapper'. """ - for _, dynapcnnlayer in mapper.items(): - for _node, __ in dynapcnnlayer.items(): - if _node == node: - return True - return False - -def get_dynapcnnlayers_destinations(layers: Dict[int, nn.Module], edges: List[Tuple[int, int]], mapper: dict) -> dict: - """ Loops over the edges list describing the computational graph. It will access each node in the graph and find to which + + +def find_initialized_node(node: int, mapper: dict) -> bool: + """Finds if 'node' existis within 'mapper' and returns layer index.""" + for index, dynapcnnlayer in mapper.items(): + if node in dynapcnnlayer: + return index + return None + + +def get_dynapcnnlayers_destinations( + layers: Dict[int, nn.Module], edges: List[Tuple[int, int]], mapper: dict +) -> dict: + """Loops over the edges list describing the computational graph. It will access each node in the graph and find to which DynapcnnLayer they belong to. If source and target belong to different DynapcnnLayers (described as a dictionary in 'mapper') the destination of the 'DynapcnnLayer.source' is set to be 'DynapcnnLayer.target'. @@ -161,7 +192,7 @@ def get_dynapcnnlayers_destinations(layers: Dict[int, nn.Module], edges: List[Tu `mapper[0]` and node `4` belongs to `mapper[2]`, the former is updated to tager the latter, like the following: mapper[0] = { - 0: {'layer': Conv2d, ...}, + 0: {'layer': Conv2d, ...}, 1: {'layer': IAFSqueeze, ...}, # node `1` in edge `(1, 4)` belongs to `mapper[0]`... ... 'destinations': [2], # ... so DynacnnLayer built from `mapper[2]` is destination of DynapcnnLayer built from `mapper[0]`. @@ -188,41 +219,58 @@ def get_dynapcnnlayers_destinations(layers: Dict[int, nn.Module], edges: List[Tu if source_layer not in dynapcnnlayers_destinations_map: dynapcnnlayers_destinations_map[source_layer] = [] - if source_layer != destination_layer and is_valid_dynapcnnlayer_pairing(layers, edge, VALID_DYNAPCNNLAYER_EDGES): + if source_layer != destination_layer and is_valid_dynapcnnlayer_pairing( + layers, edge, VALID_DYNAPCNNLAYER_EDGES + ): # valid connection between modules in two different DynapcnnLayer. if len(dynapcnnlayers_destinations_map[source_layer]) > 2: # DynapcnnLayers can not have more than two destinations. raise MaxDestinationsReached(source_layer) else: - if (destination_layer, source_layer) not in used_layer_edges and destination_layer not in dynapcnnlayers_destinations_map[source_layer]: + if ( + (destination_layer, source_layer) not in used_layer_edges + and destination_layer + not in dynapcnnlayers_destinations_map[source_layer] + ): # edge does not create a loop between layers. - dynapcnnlayers_destinations_map[source_layer].append(destination_layer) + dynapcnnlayers_destinations_map[source_layer].append( + destination_layer + ) used_layer_edges.append((source_layer, destination_layer)) else: raise InvalidLayerLoop(source_layer, destination_layer) - + for dcnnl_idx, destinations in dynapcnnlayers_destinations_map.items(): # TODO document the 'rescale_factor' better. - mapper[dcnnl_idx]['destinations'] = destinations - mapper[dcnnl_idx]['conv_rescale_factor'] = [] - + mapper[dcnnl_idx]["destinations"] = destinations + mapper[dcnnl_idx]["conv_rescale_factor"] = [] + + def get_dynapcnnlayer_index(node: int, mapper: dict) -> int: - """ Returns the DynapcnnLayer index to which 'node' belongs to. """ + """Returns the DynapcnnLayer index to which 'node' belongs to.""" for indx, dynapcnnlayer in mapper.items(): if node in dynapcnnlayer: return indx raise UnknownNode(node) -def is_valid_dynapcnnlayer_pairing(layers: Dict[int, nn.Module], edge: Tuple[int, int], valid_dynapcnnlayer_edges: List[Tuple[nn.Module, nn.Module]]) -> bool: - """ Checks if the module in 'DynapcnnLayer.source' is targetting a valid module in 'DynapcnnLayer.target'. """ + +def is_valid_dynapcnnlayer_pairing( + layers: Dict[int, nn.Module], + edge: Tuple[int, int], + valid_dynapcnnlayer_edges: List[Tuple[nn.Module, nn.Module]], +) -> bool: + """Checks if the module in 'DynapcnnLayer.source' is targetting a valid module in 'DynapcnnLayer.target'.""" if (type(layers[edge[0]]), type(layers[edge[1]])) in valid_dynapcnnlayer_edges: return True else: raise InvalidLayerDestination(type(layers[edge[0]]), type(layers[edge[1]])) - -def merge_handler(sinabs_edges: List[Tuple[int, int]], sinabs_modules_map: Dict[int, nn.Module]) -> List[Tuple[int, int]]: - """ Handles connections between nodes made via a `sinabs.layers.Merge` layer. If `X` is a merge layer then edges `(X, C)` are removed + + +def merge_handler( + sinabs_edges: List[Tuple[int, int]], sinabs_modules_map: Dict[int, nn.Module] +) -> List[Tuple[int, int]]: + """Handles connections between nodes made via a `sinabs.layers.Merge` layer. If `X` is a merge layer then edges `(X, C)` are removed from the edges list since they don't affect the creationg of `DynapcnnLayer`s. Edges `(Y, X)` are turned into a edge `(Y, C)` pointing directly to the node receiving the merged inputs such that the `DynapcnnLayer` containing `Y` can have the `DynapcnnLayer` containing `C` as one of its destinations. @@ -250,14 +298,16 @@ def merge_handler(sinabs_edges: List[Tuple[int, int]], sinabs_modules_map: Dict[ if isinstance(sinabs_modules_map[src], sinabs.layers.Merge): if src not in merge_nodes: # found node receiving merged inputs from two previous layers. - merge_nodes[src] = {'sources': [], 'merge_into': trg} + merge_nodes[src] = {"sources": [], "merge_into": trg} for _edge in edges: if _edge[1] == src: # found node used as argument for a Merge layer. - merge_nodes[src]['sources'].append(_edge[0]) - if len(merge_nodes[src]['sources']) > 2: - raise ValueError("A Merge layer can not have more than two inputs.") + merge_nodes[src]["sources"].append(_edge[0]) + if len(merge_nodes[src]["sources"]) > 2: + raise ValueError( + "A Merge layer can not have more than two inputs." + ) for edge in edges: # removing edges connection from/to merging layers from the computational graph. @@ -267,10 +317,10 @@ def merge_handler(sinabs_edges: List[Tuple[int, int]], sinabs_modules_map: Dict[ if src in merge_nodes: # edge (`Merge`, trg) is not necessary for later DynapcnnLayer creation. pass - + elif trg in merge_nodes: # point `src` directly to the node it was previously targeting via a Merge layer. - new_edge = (src, merge_nodes[trg]['merge_into']) + new_edge = (src, merge_nodes[trg]["merge_into"]) edges_without_merge.append(new_edge) else: @@ -278,4 +328,3 @@ def merge_handler(sinabs_edges: List[Tuple[int, int]], sinabs_modules_map: Dict[ edges_without_merge.append(edge) return edges_without_merge - \ No newline at end of file diff --git a/sinabs/backend/dynapcnn/sinabs_edges_utils.py b/sinabs/backend/dynapcnn/sinabs_edges_utils.py index 4431cd38..04a61260 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_utils.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_utils.py @@ -5,31 +5,58 @@ contact : williansoaresgirao@gmail.com """ -import sinabs.layers as sl -import torch.nn as nn from typing import Tuple +import torch.nn as nn + +import sinabs.layers as sl + # Constraints. # @TODO constraints are ideally device-dependent. VALID_SINABS_EDGES = { - 0: (nn.Conv2d, sl.iaf.IAFSqueeze), # convoluion is always followed by a neuron layer. + 0: ( + nn.Conv2d, + sl.iaf.IAFSqueeze, + ), # convoluion is always followed by a neuron layer. 1: (sl.iaf.IAFSqueeze, nn.AvgPool2d), 2: (sl.iaf.IAFSqueeze, nn.Conv2d), - 3: (sl.iaf.IAFSqueeze, nn.Linear), # same case as `2` since `nn.Linear` layers are converted into `nn.Conv2d` by `DynapcnnLayer`. - 4: (nn.AvgPool2d, nn.Conv2d), # `nn.Pool2d` is always "ending" a DynapcnnLayer sequence of modules (comes after a `sl.iaf`). - 5: (nn.AvgPool2d, nn.Linear), # same as case `4` since `nn.Linear` layers are converted into `nn.Conv2d` by `DynapcnnLayer`. - 6: (nn.Linear, sl.iaf.IAFSqueeze), # same as case `0` since `nn.Linear` layers are converted into `nn.Conv2d` by `DynapcnnLayer`. - 7: (sl.iaf.IAFSqueeze, sl.SumPool2d), # same as key `1` but with `sl.SumPool2d` instead. - 8: (sl.SumPool2d, nn.Conv2d), # same as key `4` but with `sl.SumPool2d` instead. - 9: (sl.SumPool2d, nn.Linear), # same as key `5` but with `sl.SumPool2d` instead. + 3: ( + sl.iaf.IAFSqueeze, + nn.Linear, + ), # same case as `2` since `nn.Linear` layers are converted into `nn.Conv2d` by `DynapcnnLayer`. + 4: ( + nn.AvgPool2d, + nn.Conv2d, + ), # `nn.Pool2d` is always "ending" a DynapcnnLayer sequence of modules (comes after a `sl.iaf`). + 5: ( + nn.AvgPool2d, + nn.Linear, + ), # same as case `4` since `nn.Linear` layers are converted into `nn.Conv2d` by `DynapcnnLayer`. + 6: ( + nn.Linear, + sl.iaf.IAFSqueeze, + ), # same as case `0` since `nn.Linear` layers are converted into `nn.Conv2d` by `DynapcnnLayer`. + 7: ( + sl.iaf.IAFSqueeze, + sl.SumPool2d, + ), # same as key `1` but with `sl.SumPool2d` instead. + 8: (sl.SumPool2d, nn.Conv2d), # same as key `4` but with `sl.SumPool2d` instead. + 9: (sl.SumPool2d, nn.Linear), # same as key `5` but with `sl.SumPool2d` instead. } +VALID_SINABS_EDGE_TYPE_IDS = {v: k for k, v in VALID_SINABS_EDGES} VALID_DYNAPCNNLAYER_EDGES = [ (sl.iaf.IAFSqueeze, nn.Conv2d), - (sl.iaf.IAFSqueeze, nn.Linear), # `nn.Linear` layers are converted into `nn.Conv2d` by `DynapcnnLayer`. + ( + sl.iaf.IAFSqueeze, + nn.Linear, + ), # `nn.Linear` layers are converted into `nn.Conv2d` by `DynapcnnLayer`. (nn.AvgPool2d, nn.Conv2d), (sl.SumPool2d, nn.Conv2d), - (nn.AvgPool2d, nn.Linear), # `nn.Linear` layers are converted into `nn.Conv2d` by `DynapcnnLayer`. + ( + nn.AvgPool2d, + nn.Linear, + ), # `nn.Linear` layers are converted into `nn.Conv2d` by `DynapcnnLayer`. (sl.SumPool2d, nn.Linear), ] @@ -38,6 +65,7 @@ # Edge exceptions. + class InvalidEdge(Exception): edge: Tuple[int, int] source: type @@ -46,6 +74,7 @@ class InvalidEdge(Exception): def __init__(self, edge, source, target): super().__init__(f"Invalid edge {edge}: {source} can not target {target}.") + class InvalidEdgeType(Exception): edge: Tuple[int, int] type: int @@ -53,35 +82,50 @@ class InvalidEdgeType(Exception): def __init__(self, edge, type): super().__init__(f"Invalid edge type {type} for edge {edge}.") + class UnmatchedNode(Exception): edge: Tuple[int, int] node: int def __init__(self, edge, node): - super().__init__(f"Node {node} in edge {edge} can not found in previously processed edges.") + super().__init__( + f"Node {node} in edge {edge} can not found in previously processed edges." + ) + class UnknownNode(Exception): node: int def __init__(self, node): - super().__init__(f"Node {node} can not be found within any DynapcnnLayer mapper.") + super().__init__( + f"Node {node} can not be found within any DynapcnnLayer mapper." + ) + class MaxDestinationsReached(Exception): dynapcnnlayer_index: int def __init__(self, dynapcnnlayer_index): - super().__init__(f"DynapcnnLayer with index {dynapcnnlayer_index} has more than 2 destinations.") + super().__init__( + f"DynapcnnLayer with index {dynapcnnlayer_index} has more than 2 destinations." + ) + class InvalidLayerLoop(Exception): dynapcnnlayerA_index: int dynapcnnlayerB_index: int def __init__(self, dynapcnnlayerA_index, dynapcnnlayerB_index): - super().__init__(f"DynapcnnLayer {dynapcnnlayerA_index} can not connect to {dynapcnnlayerB_index} since reverse edge already exists.") + super().__init__( + f"DynapcnnLayer {dynapcnnlayerA_index} can not connect to {dynapcnnlayerB_index} since reverse edge already exists." + ) + class InvalidLayerDestination(Exception): dynapcnnlayerA: type dynapcnnlayerB: type def __init__(self, dynapcnnlayerA, dynapcnnlayerB): - super().__init__(f"DynapcnnLayer {dynapcnnlayerA} in one core can not connect to {dynapcnnlayerB} in another core.") \ No newline at end of file + super().__init__( + f"DynapcnnLayer {dynapcnnlayerA} in one core can not connect to {dynapcnnlayerB} in another core." + ) diff --git a/sinabs/backend/dynapcnn/utils.py b/sinabs/backend/dynapcnn/utils.py index c5c43149..9a5a227d 100644 --- a/sinabs/backend/dynapcnn/utils.py +++ b/sinabs/backend/dynapcnn/utils.py @@ -1,10 +1,10 @@ +from collections import defaultdict, deque from copy import deepcopy -from typing import TYPE_CHECKING, List, Optional, Tuple, Union, Dict, Callable +from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Tuple, Union import torch import torch.nn as nn -from collections import defaultdict, deque import sinabs.layers as sl from .crop2d import Crop2d @@ -13,8 +13,7 @@ from .dynapcnn_layer_handler import DynapcnnLayerHandler from .exceptions import WrongPoolingModule from .flipdims import FlipDims - -from .sinabs_edges_handler import process_edge, get_dynapcnnlayers_destinations +from .sinabs_edges_handler import get_dynapcnnlayers_destinations, process_edge if TYPE_CHECKING: from sinabs.backend.dynapcnn.dynapcnn_network import DynapcnnNetwork @@ -23,6 +22,7 @@ ####################################################### Device Related ####################################################### + def parse_device_id(device_id: str) -> Tuple[str, int]: """Parse device id into device type and device index. @@ -46,6 +46,7 @@ def parse_device_id(device_id: str) -> Tuple[str, int]: return device_type, int(index) + def get_device_id(device_type: str, index: int) -> str: """Generate a device id string given a device type and its index. @@ -58,6 +59,7 @@ def get_device_id(device_type: str, index: int) -> str: """ return f"{device_type}:{index}" + def standardize_device_id(device_id: str) -> str: """Standardize device id string. @@ -70,33 +72,37 @@ def standardize_device_id(device_id: str) -> str: device_type, index = parse_device_id(device_id=device_id) return get_device_id(device_type=device_type, index=index) + ####################################################### DynapcnnNetwork Related ####################################################### -def build_nodes_to_dcnnl_map(layers: Dict[int, nn.Module], edges: List[Tuple[int, int]]) -> dict: - """ Initializes and populates a `dict` that will map data into a future `DynapcnnLayer` instance. The call - to `process_edge()` initializes a `key` (the index of a `DynapcnnLayer`) and assigns to it a dict containing the + +def build_nodes_to_dcnnl_map( + layers: Dict[int, nn.Module], edges: List[Tuple[int, int]] +) -> dict: + """Initializes and populates a `dict` that will map data into a future `DynapcnnLayer` instance. The call + to `process_edge()` initializes a `key` (the index of a `DynapcnnLayer`) and assigns to it a dict containing the nodes (layers in a `nn.Module`) that should belong to the same `DynapcnnLayer`. The call to `get_dynapcnnlayers_destinations()` further incorporates to each "DynapcnnLayer dictionary" a `destinations` attribute, which is a list of integers indicating the the target destinations of a `DynapcnnLayer` instance. Parameters --------- - - layers (dict): constains the nodes of a graph as `key` and their associated module as `value`. + - layers (dict): contains the node IDs of a graph as `key` and their associated module as `value`. - edges (list): edges describing how nodes connect to each other. Returns --------- - - nodes_to_dcnnl_map (dict): each entry represents the gathered data necessary to instantiate a `DynapcnnLayer` object (e.g. nodes, + - nodes_to_dcnnl_map (dict): each entry represents the gathered data necessary to instantiate a `DynapcnnLayer` object (e.g. nodes, their I/O shapes, the list of `DynapcnnLayer` that are to be targeted, etc). """ # @TODO the graph extraction is not yet considering DVS input. # dvs_layer, lyr_indx_next, rescale_factor = construct_dvs_layer( - # layers, - # input_shape=in_shape, - # idx_start=0, + # layers, + # input_shape=in_shape, + # idx_start=0, # dvs_input=False) - + dvs_layer = None # mapper from nodes to sets of layers that populate a DynapcnnLayer. @@ -112,17 +118,21 @@ def build_nodes_to_dcnnl_map(layers: Dict[int, nn.Module], edges: List[Tuple[int # look for edges between connecting nodes in different (future) DynapcnnLayer. get_dynapcnnlayers_destinations(layers, edges, nodes_to_dcnnl_map) - + return nodes_to_dcnnl_map + def build_from_graph( - discretize: bool, - edges: List[Tuple[int, int]], - nodes_to_dcnnl_map: dict, - weight_rescaling_fn: Callable, - entry_nodes: List[int]) -> Union[Dict[int, Dict[DynapcnnLayer, List]], Dict[int, Dict[DynapcnnLayerHandler, List]]]: - """ Parses each edge in `edges`, where each node is a set of layer that will compose a `DynapcnnLayer`. The - target destination of each `DynapcnnLayer` is computed via edges connecting nodes in different `DynapcnnLayer` + discretize: bool, + edges: List[Tuple[int, int]], + nodes_to_dcnnl_map: dict, + weight_rescaling_fn: Callable, + entry_nodes: List[int], +) -> Union[ + Dict[int, Dict[DynapcnnLayer, List]], Dict[int, Dict[DynapcnnLayerHandler, List]] +]: + """Parses each edge in `edges`, where each node is a set of layer that will compose a `DynapcnnLayer`. The + target destination of each `DynapcnnLayer` is computed via edges connecting nodes in different `DynapcnnLayer` instances. Parameters @@ -142,23 +152,29 @@ def build_from_graph( """ # turn each entry in `nodes_to_dcnnl_map` into a `DynapcnnLayer` instance. - dynapcnn_layers, dynapcnnlayers_handlers = construct_dynapcnnlayers_from_mapper(discretize, nodes_to_dcnnl_map, edges, weight_rescaling_fn, entry_nodes) - + dynapcnn_layers, dynapcnnlayers_handlers = construct_dynapcnnlayers_from_mapper( + discretize, nodes_to_dcnnl_map, edges, weight_rescaling_fn, entry_nodes + ) + # initialize key holding to which core a `DynapcnnLayer` instance in `dynapcnn_layers` will be mapped to. for idx, layer_data in dynapcnnlayers_handlers.items(): - if 'core_idx' not in layer_data: + if "core_idx" not in layer_data: # a `DynapcnnLayer` gets assigned a core index when `DynapcnnNetworkGraph.to()`` is called. - layer_data['core_idx'] = -1 - + layer_data["core_idx"] = -1 + return dynapcnn_layers, dynapcnnlayers_handlers + def construct_dynapcnnlayers_from_mapper( - discretize: bool, - nodes_to_dcnnl_map: dict, - edges: List[Tuple[int, int]], - weight_rescaling_fn: Callable, - entry_nodes: List[int]) -> Union[Dict[int, Dict[DynapcnnLayer, List]], Dict[int, Dict[DynapcnnLayerHandler, List]]]: - """ Consumes a dictionaries containing sets of layers to be used to populate a DynapcnnLayer object. + discretize: bool, + nodes_to_dcnnl_map: dict, + edges: List[Tuple[int, int]], + weight_rescaling_fn: Callable, + entry_nodes: List[int], +) -> Union[ + Dict[int, Dict[DynapcnnLayer, List]], Dict[int, Dict[DynapcnnLayerHandler, List]] +]: + """Consumes a dictionaries containing sets of layers to be used to populate a DynapcnnLayer object. Parameters ---------- @@ -178,26 +194,32 @@ def construct_dynapcnnlayers_from_mapper( dynapcnn_layers = {} dynapcnnlayers_handlers = {} - + for dpcnnl_idx, dcnnl_data in nodes_to_dcnnl_map.items(): # create a `DynapcnnLayerHandler` from the set of layers in `dcnnl_data` - this holds network-level data required to instantiate a `DynapcnnLayer`. layerhandler = construct_layerhandler( - dpcnnl_idx, discretize, edges, nodes_to_dcnnl_map, weight_rescaling_fn, entry_nodes) - + dpcnnl_idx, + discretize, + edges, + nodes_to_dcnnl_map, + weight_rescaling_fn, + entry_nodes, + ) + # create a `DynapcnnLayer` from the handler. dynapcnnlayer = construct_dynapcnnlayer(layerhandler) - + # holds the layers themselvs. dynapcnn_layers[dpcnnl_idx] = { - 'layer': dynapcnnlayer, - 'destinations': nodes_to_dcnnl_map[dpcnnl_idx]['destinations'] - } - + "layer": dynapcnnlayer, + "destinations": nodes_to_dcnnl_map[dpcnnl_idx]["destinations"], + } + # holds the handlers of each layer for later use (e.g., creation of the forward pass for the `DynapcnnNetwork`). dynapcnnlayers_handlers[dpcnnl_idx] = { - 'layer_handler': layerhandler, - 'destinations': nodes_to_dcnnl_map[dpcnnl_idx]['destinations'] - } + "layer_handler": layerhandler, + "destinations": nodes_to_dcnnl_map[dpcnnl_idx]["destinations"], + } # check if a `nn.Linear` in `dynapcnnlayer` has been turned into a `nn.Conv2d`. node, output_shape = layerhandler.get_modified_node_io(dcnnl_data) @@ -208,8 +230,14 @@ def construct_dynapcnnlayers_from_mapper( return dynapcnn_layers, dynapcnnlayers_handlers -def update_nodes_io(updated_node: int, output_shape: tuple, nodes_to_dcnnl_map: dict, edges: List[Tuple[int, int]]) -> None: - """ Updates the `input_shape` entries of each node in `nodes_to_dcnnl_map` receiving as input the output of the spiking + +def update_nodes_io( + updated_node: int, + output_shape: tuple, + nodes_to_dcnnl_map: dict, + edges: List[Tuple[int, int]], +) -> None: + """Updates the `input_shape` entries of each node in `nodes_to_dcnnl_map` receiving as input the output of the spiking layer `updated_node` that had its I/O shapes updated following a `nn.Linear` to `nn.Conv2d` conversion. Parameters @@ -232,16 +260,24 @@ def update_nodes_io(updated_node: int, output_shape: tuple, nodes_to_dcnnl_map: # accessing node data (`layer`, `input_shape` and `output_shape`). if key == edge[1]: # accessing node targeted by `updated_node` (its input shape becomes `updated_node.output_shape`). - val['input_shape'] = output_shape + val["input_shape"] = output_shape + def construct_layerhandler( - dpcnnl_idx: int, - discretize: bool, - edges: List[Tuple[int, int]], - nodes_to_dcnnl_map: Dict[int, Dict[Union[int, str], Union[Dict[str, Union[nn.Module, Tuple[int, int, int]]], List[int]]]], - weight_rescaling_fn: Callable, - entry_nodes: List[int]) -> DynapcnnLayerHandler: - """ Extract the modules (layers) in a dictionary and uses them to instantiate a `DynapcnnLayerHandler` object. + dpcnnl_idx: int, + discretize: bool, + edges: List[Tuple[int, int]], + nodes_to_dcnnl_map: Dict[ + int, + Dict[ + Union[int, str], + Union[Dict[str, Union[nn.Module, Tuple[int, int, int]]], List[int]], + ], + ], + weight_rescaling_fn: Callable, + entry_nodes: List[int], +) -> DynapcnnLayerHandler: + """Extract the modules (layers) in a dictionary and uses them to instantiate a `DynapcnnLayerHandler` object. Parameters ---------- @@ -249,56 +285,69 @@ def construct_layerhandler( containing the data required to create the instance returned by this function. - discretize (bool): whether or not the weights/neuron parameters of the model will be quantized. - edges (list): each `nn.Module` within `nodes_to_dcnnl_map[dpcnnl_idx]` is a node in the original computational graph describing a spiking network - being converted to a `DynapcnnNetwork`. An edge `(A, B)` describes how modules forward data amongst themselves. This list is used by a `DynapcnnLayerHandler` + being converted to a `DynapcnnNetwork`. An edge `(A, B)` describes how modules forward data amongst themselves. This list is used by a `DynapcnnLayerHandler` to figure out the number and sequence of output tesnors its forward method needs to return. - nodes_to_dcnnl_map (dict): contains all layers (`nn.Module`) in the original spiking network grouped into dictionaries gathering the data necessary - to instantiate a `DynapcnnLayerHandler`. A `nodes_to_dcnnl_map[dpcnnl_idx]` will contain `int` keys (whose value corresponds to a `dict` with a `nn.Module` + to instantiate a `DynapcnnLayerHandler`. A `nodes_to_dcnnl_map[dpcnnl_idx]` will contain `int` keys (whose value corresponds to a `dict` with a `nn.Module` instance and its associated I/O shapes, i.e., one layer within the `DynapcnnLayerHandler` instance) or `str` keys (whose values correspond to a list of integers corresponding to either destinations IDs or re-scaling factors). - weight_rescaling_fn (callable): a method that handles how the re-scaling factor for one or more `SumPool2d` projecting to the same convolutional layer are combined/re-scaled before being applied. - entry_nodes (list): node IDs corresponding to layers in the original network that are input nodes (i.e., a "point of entry" for the external data). - + Returns ---------- - layerhandler (DynapcnnLayerHandler): the a `DynapcnnLayer` instance made up by all the layers (`nn.Module`) in `dcnnl_data`. """ # convert all AvgPool2d in 'dcnnl_data' into SumPool2d. - convert_Avg_to_Sum_pooling(nodes_to_dcnnl_map[dpcnnl_idx], edges, nodes_to_dcnnl_map) + convert_Avg_to_Sum_pooling( + nodes_to_dcnnl_map[dpcnnl_idx], edges, nodes_to_dcnnl_map + ) # instantiate a DynapcnnLayer from the data in 'dcnnl_data'. layerhandler = DynapcnnLayerHandler( - dpcnnl_index = dpcnnl_idx, - dcnnl_data = nodes_to_dcnnl_map[dpcnnl_idx], - discretize = discretize, - sinabs_edges = edges, - weight_rescaling_fn = weight_rescaling_fn, - entry_nodes = entry_nodes + dpcnnl_index=dpcnnl_idx, + dcnnl_data=nodes_to_dcnnl_map[dpcnnl_idx], + discretize=discretize, + sinabs_edges=edges, + weight_rescaling_fn=weight_rescaling_fn, + entry_nodes=entry_nodes, ) return layerhandler + def construct_dynapcnnlayer(handler: DynapcnnLayerHandler) -> DynapcnnLayer: - """... - """ + """...""" # instantiate a DynapcnnLayer from the data in the handler. dynapcnnlayer = DynapcnnLayer( - conv = handler.conv_layer, - spk = handler.spk_layer, - in_shape = handler.conv_in_shape, - pool = handler.get_pool_list(), - discretize = False, + conv=handler.conv_layer, + spk=handler.spk_layer, + in_shape=handler.conv_in_shape, + pool=handler.get_pool_list(), + discretize=False, ) return dynapcnnlayer + def convert_Avg_to_Sum_pooling( - dcnnl_data: Dict[Union[int, str], Union[Dict[str, Union[nn.Module, Tuple[int, int, int]]], List[int]]], - edges: List[Tuple[int, int]], - nodes_to_dcnnl_map: Dict[int, Dict[Union[int, str], Union[Dict[str, Union[nn.Module, Tuple[int, int, int]]], List[int]]]]) -> None: - """ Converts every `AvgPool2d` node within `dcnnl_data` into a `SumPool2d` and update their respective `rescale_factor` (to + dcnnl_data: Dict[ + Union[int, str], + Union[Dict[str, Union[nn.Module, Tuple[int, int, int]]], List[int]], + ], + edges: List[Tuple[int, int]], + nodes_to_dcnnl_map: Dict[ + int, + Dict[ + Union[int, str], + Union[Dict[str, Union[nn.Module, Tuple[int, int, int]]], List[int]], + ], + ], +) -> None: + """Converts every `AvgPool2d` node within `dcnnl_data` into a `SumPool2d` and update their respective `rescale_factor` (to be used when creating the `DynapcnnLayer` instance for this layer's destinations). Parameters @@ -310,7 +359,7 @@ def convert_Avg_to_Sum_pooling( list is used to find the targets of a `SumPool2d` (part of the `DynapcnnLayer` instance being created) and update the re-scaling factor they will require. - nodes_to_dcnnl_map (dict): contains all layers (`nn.Module`) in the original spiking network grouped into dictionaries gathering the data necessary - to instantiate a `DynapcnnLayer`. A `nodes_to_dcnnl_map[dpcnnl_idx]` will contain `int` keys (whose value corresponds to a `dict` with a `nn.Module` + to instantiate a `DynapcnnLayer`. A `nodes_to_dcnnl_map[dpcnnl_idx]` will contain `int` keys (whose value corresponds to a `dict` with a `nn.Module` instance and its associated I/O shapes, i.e., one layer within the `DynapcnnLayer` instance) or `str` keys (whose values correspond to a list of integers corresponding to either destinations IDs or re-scaling factors). """ @@ -318,24 +367,29 @@ def convert_Avg_to_Sum_pooling( if isinstance(key, int): # accessing the node `key` dictionary. - if isinstance(value['layer'], nn.AvgPool2d): + if isinstance(value["layer"], nn.AvgPool2d): # convert AvgPool2d into SumPool2d. - lyr_pool, rescale_factor = build_SumPool2d(value['layer']) + lyr_pool, rescale_factor = build_SumPool2d(value["layer"]) # turn avg into sum pool. - value['layer'] = lyr_pool + value["layer"] = lyr_pool # find which node `key` will target. for edge in edges: if edge[0] == key: # find index of `DynapcnnLayer` where the target of `edge[0]` is. - trg_dcnnl_idx = find_nodes_dcnnl_idx(edge[1], nodes_to_dcnnl_map) + trg_dcnnl_idx = find_nodes_dcnnl_idx( + edge[1], nodes_to_dcnnl_map + ) # update the rescale factor for the target of node `key`. - nodes_to_dcnnl_map[trg_dcnnl_idx]['conv_rescale_factor'].append(rescale_factor) + nodes_to_dcnnl_map[trg_dcnnl_idx]["conv_rescale_factor"].append( + rescale_factor + ) + def find_nodes_dcnnl_idx(node: int, nodes_to_dcnnl_map: dict) -> int: - """ Find the ID of the (future) `DynapcnnLayer` instance to which `node` belongs to.""" + """Find the ID of the (future) `DynapcnnLayer` instance to which `node` belongs to.""" # looping over sets of layers (nodes) that will be used to instantiate `DynapcnnLayer`s. for dcnnl_idx, dcnnl_data in nodes_to_dcnnl_map.items(): @@ -347,11 +401,14 @@ def find_nodes_dcnnl_idx(node: int, nodes_to_dcnnl_map: dict) -> int: return dcnnl_idx # this exception should never happen. - raise ValueError(f'Node {node} is not part of any dictionary mapping into a DynapcnnLayer.') + raise ValueError( + f"Node {node} is not part of any dictionary mapping into a DynapcnnLayer." + ) + def build_SumPool2d(module: nn.AvgPool2d) -> Tuple[sl.SumPool2d, int]: - """ Converts a `nn.AvgPool2d` into a `sl.SumPool2d` layer. - + """Converts a `nn.AvgPool2d` into a `sl.SumPool2d` layer. + Parameters ---------- - module (torch.nn.AvgPool2d): the average pooling layer being converted into a sum pooling layer. @@ -361,7 +418,7 @@ def build_SumPool2d(module: nn.AvgPool2d) -> Tuple[sl.SumPool2d, int]: - lyr_pool (sinabs.layers.SumPool2d): the equivalent sum pooling layer. rescale_factor (int): the weight re-scaling computed for the weights of the convolution layer targeted by the pooling. """ - + if isinstance(module, nn.AvgPool2d): if module.padding != 0: raise ValueError("Padding is not supported for the pooling layers.") @@ -369,7 +426,7 @@ def build_SumPool2d(module: nn.AvgPool2d) -> Tuple[sl.SumPool2d, int]: pass else: raise WrongPoolingModule(type(module)) - + rescale_factor = 1 cumulative_pooling = expand_to_pair(1) pooling = expand_to_pair(module.kernel_size) @@ -395,8 +452,9 @@ def build_SumPool2d(module: nn.AvgPool2d) -> Tuple[sl.SumPool2d, int]: return lyr_pool, rescale_factor + def topological_sorting(edges: Set[Tuple[int, int]]) -> List[int]: - """ Performs a topological sorting (using Kahn's algorithm) of a graph descrobed by a list edges. An entry node `X` + """Performs a topological sorting (using Kahn's algorithm) of a graph descrobed by a list edges. An entry node `X` of the graph have to be flagged inside `edges` by a tuple `('input', X)`. Parameters @@ -413,7 +471,7 @@ def topological_sorting(edges: Set[Tuple[int, int]]) -> List[int]: # initialize the graph and in-degrees. for u, v in edges: - if u != 'input': + if u != "input": graph[u].append(v) in_degree[v] += 1 else: @@ -423,7 +481,9 @@ def topological_sorting(edges: Set[Tuple[int, int]]) -> List[int]: in_degree[v] = 0 # find all nodes with zero in-degrees. - zero_in_degree_nodes = deque([node for node, degree in in_degree.items() if degree == 0]) + zero_in_degree_nodes = deque( + [node for node, degree in in_degree.items() if degree == 0] + ) # process nodes and create the topological order. topological_order = [] @@ -440,12 +500,14 @@ def topological_sorting(edges: Set[Tuple[int, int]]) -> List[int]: # check if all nodes are processed (to handle cycles). if len(topological_order) == len(in_degree): return topological_order - - raise ValueError('The graph has a cycle and cannot be topologically sorted.') + + raise ValueError("The graph has a cycle and cannot be topologically sorted.") + ####################################################### MISSING FUNCTIONALITY ####################################################### # TODO: these methods are currently not used by the new implementation of DynapcnnNetwork (but should). + def convert_cropping2dlayer_to_crop2d( layer: sl.Cropping2dLayer, input_shape: Tuple[int, int] ) -> Crop2d: @@ -598,6 +660,7 @@ def merge_conv_bn(conv, bn): return conv + def construct_next_pooling_layer( layers: List[nn.Module], idx_start: int ) -> Tuple[Optional[sl.SumPool2d], int, float]: @@ -650,7 +713,7 @@ def construct_next_pooling_layer( cumulative_pooling[0] * pooling[0], cumulative_pooling[1] * pooling[1], ) - + # Update rescaling factor if isinstance(lyr, nn.AvgPool2d): rescale_factor *= pooling[0] * pooling[1] @@ -661,7 +724,8 @@ def construct_next_pooling_layer( else: lyr_pool = sl.SumPool2d(cumulative_pooling) return lyr_pool, idx_next, rescale_factor - + + def extend_readout_layer(model: "DynapcnnNetwork") -> "DynapcnnNetwork": """Return a copied and extended model with the readout layer extended to 4 times the number of output channels. For Speck 2E and 2F, to get readout with correct output index, we need to @@ -703,6 +767,7 @@ def extend_readout_layer(model: "DynapcnnNetwork") -> "DynapcnnNetwork": ) # run a forward pass to initialize the new weights and last IAF return model + ####################################################### DEPRECATED METHODS ####################################################### # TODO: these methods were used by the old implementation of DynapcnnNetwork - delete all. @@ -871,7 +936,7 @@ def extend_readout_layer(model: "DynapcnnNetwork") -> "DynapcnnNetwork": # dvs_layer, lyr_indx_next, rescale_factor = construct_dvs_layer( # layers, input_shape=in_shape, idx_start=lyr_indx_next, dvs_input=dvs_input # ) - + # if dvs_layer is not None: # compatible_layers.append(dvs_layer) # in_shape = dvs_layer.get_output_shape() @@ -911,7 +976,7 @@ def extend_readout_layer(model: "DynapcnnNetwork") -> "DynapcnnNetwork": # """ # if isinstance(model, sinabs.Network): # return convert_model_to_layer_list(model.spiking_model) - + # elif isinstance(model, nn.Sequential): # layers = [layer for layer in model if not isinstance(layer, ignore)] @@ -920,5 +985,5 @@ def extend_readout_layer(model: "DynapcnnNetwork") -> "DynapcnnNetwork": # else: # raise TypeError("Expected torch.nn.Sequential or sinabs.Network") - + # return layers From 0a6b61684ee1e0c05239bd02bad5bc6163163b21 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Fri, 27 Sep 2024 19:09:34 +0200 Subject: [PATCH 151/379] Improve type hints in edges handler --- .../backend/dynapcnn/sinabs_edges_handler.py | 29 ++++++++++++++----- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index 5fae43c5..13698c27 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -17,7 +17,9 @@ def process_edge( - layers: Dict[int, nn.Module], edge: Tuple[int, int], mapper: dict + layers: Dict[int, nn.Module], + edge: Tuple[int, int], + mapper: Dict[int, Dict[int, Dict]], ) -> None: """Read in an edge describing the connection between two layers (nodes in the computational graph). If `edge` is a valid connection between two layers, update `mapper` to incorporate these layers into a new or existing dictonary @@ -73,7 +75,10 @@ def get_valid_edge_type( def update_dynapcnnlayer_mapper( - edge_type: int, edge: Tuple[int, int], mapper: dict, layers: Dict[int, nn.Module] + edge_type: int, + edge: Tuple[int, int], + mapper: Dict[int, Dict[int, Dict]], + layers: Dict[int, nn.Module], ) -> None: """Parses the nodes within an edge and incorporate them either into a **new** or an **already existing** DynapcnnLayer represented in 'mapper'. @@ -93,7 +98,9 @@ def update_dynapcnnlayer_mapper( def init_xor_complete_new_dynapcnnlayer_blk( - mapper: dict, edge: Tuple[int, int], layers: Dict[int, nn.Module] + mapper: Dict[int, Dict[int, Dict]], + edge: Tuple[int, int], + layers: Dict[int, nn.Module], ) -> None: """Incorporates nodes from either a `(conv, neuron)` or a `(linear, neuron)` edge. These are either initiating a new `dict` mapping into a future `DynapcnnLayer` or completing a `conv->neuron` sequence (in the case the node for `conv` as already been incorporated @@ -125,7 +132,9 @@ def init_xor_complete_new_dynapcnnlayer_blk( def connect_dynapcnnlayer_blks( - mapper: dict, edge: Tuple[int, int], layers: Dict[int, nn.Module] + mapper: Dict[int, Dict[int, Dict]], + edge: Tuple[int, int], + layers: Dict[int, nn.Module], ) -> None: """Incorporates nodes from either a `(neuron, conv)/(neuron, lin)` or `(pool, conv)/(pool, lin)` edge. These represent connections between an existing `dict` in `mapper` that will be mapped into a `DynapcnnLayer` and a new one yet to be represented in `mapper`. Obs.: `nn.Linear` layers are converted @@ -157,7 +166,9 @@ def connect_dynapcnnlayer_blks( def add_pool_to_dynapcnnlayer_blk( - mapper: dict, edge: Tuple[int, int], layers: Dict[int, nn.Module] + mapper: Dict[int, Dict[int, Dict]], + edge: Tuple[int, int], + layers: Dict[int, nn.Module], ) -> None: """Incorporating a `(neuron, pool)` edge. Node `pool` has to be part of an already existing `dict` mapping into a `DynapcnnLaye` in `mapper`.""" # Search for edge[0] (neuron layer) in DynapcnnLayers @@ -172,7 +183,7 @@ def add_pool_to_dynapcnnlayer_blk( raise UnmatchedNode(edge, node) -def find_initialized_node(node: int, mapper: dict) -> bool: +def find_initialized_node(node: int, mapper: Dict[int, Dict[int, Dict]]) -> bool: """Finds if 'node' existis within 'mapper' and returns layer index.""" for index, dynapcnnlayer in mapper.items(): if node in dynapcnnlayer: @@ -181,7 +192,9 @@ def find_initialized_node(node: int, mapper: dict) -> bool: def get_dynapcnnlayers_destinations( - layers: Dict[int, nn.Module], edges: List[Tuple[int, int]], mapper: dict + layers: Dict[int, nn.Module], + edges: List[Tuple[int, int]], + mapper: Dict[int, Dict[int, Dict]], ) -> dict: """Loops over the edges list describing the computational graph. It will access each node in the graph and find to which DynapcnnLayer they belong to. If source and target belong to different DynapcnnLayers (described as a dictionary in 'mapper') @@ -247,7 +260,7 @@ def get_dynapcnnlayers_destinations( mapper[dcnnl_idx]["conv_rescale_factor"] = [] -def get_dynapcnnlayer_index(node: int, mapper: dict) -> int: +def get_dynapcnnlayer_index(node: int, mapper: Dict[int, Dict[int, Dict]]) -> int: """Returns the DynapcnnLayer index to which 'node' belongs to.""" for indx, dynapcnnlayer in mapper.items(): if node in dynapcnnlayer: From 8fcf594dd797ad98f284957b891604d3236e89fa Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Fri, 27 Sep 2024 19:12:10 +0200 Subject: [PATCH 152/379] Run black and isort --- sinabs/backend/dynapcnn/config_builder.py | 12 +- sinabs/backend/dynapcnn/dynapcnn_layer.py | 65 +++-- .../dynapcnn/dynapcnn_layer_handler.py | 240 ++++++++++------ sinabs/backend/dynapcnn/dynapcnn_network.py | 264 ++++++++++-------- .../dynapcnn/dynapcnnnetwork_module.py | 83 ++++-- sinabs/backend/dynapcnn/exceptions.py | 30 +- sinabs/backend/dynapcnn/graph_tracer.py | 101 ++++--- sinabs/backend/dynapcnn/mapping.py | 8 +- .../dynapcnn/weight_rescaling_methods.py | 15 +- 9 files changed, 496 insertions(+), 322 deletions(-) diff --git a/sinabs/backend/dynapcnn/config_builder.py b/sinabs/backend/dynapcnn/config_builder.py index d0eb419f..dbbef3d7 100644 --- a/sinabs/backend/dynapcnn/config_builder.py +++ b/sinabs/backend/dynapcnn/config_builder.py @@ -3,13 +3,15 @@ from typing import List import samna + +import sinabs import sinabs.backend import sinabs.backend.dynapcnn from .dvs_layer import DVSLayer -from .mapping import LayerConstraints, get_valid_mapping -import sinabs from .exceptions import InvalidModel +from .mapping import LayerConstraints, get_valid_mapping + class ConfigBuilder(ABC): @classmethod @@ -89,9 +91,11 @@ def get_valid_mapping(cls, model: "DynapcnnNetwork") -> List[int]: # thi check will be wrong since it assumes the network has a single input node `model.layers_mapper[0]`. pass - for (dcnnl_idx, core_idx) in mapping: + for dcnnl_idx, core_idx in mapping: # save the core index information on the handler of this `DynapcnnLayer` instance. - model.layers_handlers[dcnnl_idx]['layer_handler'].assigned_core = core_idx + model.layers_handlers[dcnnl_idx][ + "layer_handler" + ].assigned_core = core_idx chip_layers_ordering.append(core_idx) else: diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer.py b/sinabs/backend/dynapcnn/dynapcnn_layer.py index ee55988f..ed62013f 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer.py @@ -3,7 +3,7 @@ from copy import deepcopy from functools import partial -from typing import Tuple, List +from typing import List, Tuple import numpy as np import torch @@ -13,7 +13,6 @@ from .discretize import discretize_conv_spike_ - # Define sum pooling functional as power-average pooling with power 1 sum_pool2d = partial(nn.functional.lp_pool2d, norm_type=1) @@ -56,10 +55,10 @@ def __init__( ): super().__init__() - self.in_shape = in_shape - self._pool = pool - self._discretize = discretize - self._rescale_weights = rescale_weights + self.in_shape = in_shape + self._pool = pool + self._discretize = discretize + self._rescale_weights = rescale_weights spk = deepcopy(spk) @@ -79,9 +78,9 @@ def __init__( if self._discretize: conv, spk = discretize_conv_spike_(conv, spk, to_int=False) - self._conv = conv - self._spk = spk - + self._conv = conv + self._spk = spk + @property def conv(self): return self._conv @@ -101,13 +100,13 @@ def discretize(self): @property def rescale_weights(self): return self._rescale_weights - + @property def conv_out_shape(self): return self._get_conv_output_shape() - + ####################################################### Public Methods ####################################################### - + def forward(self, x) -> List[torch.Tensor]: """Torch forward pass. @@ -115,7 +114,7 @@ def forward(self, x) -> List[torch.Tensor]: """ returns = [] - + x = self.conv(x) x = self.spk(x) @@ -132,9 +131,9 @@ def forward(self, x) -> List[torch.Tensor]: return tuple(returns) def zero_grad(self, set_to_none: bool = False) -> None: - """ Call `zero_grad` method of spiking layer """ + """Call `zero_grad` method of spiking layer""" return self._spk.zero_grad(set_to_none) - + def get_neuron_shape(self) -> Tuple[int, int, int]: """Return the output shape of the neuron layer. @@ -150,7 +149,7 @@ def get_output_shape(self) -> List[Tuple[int, int, int]]: Returns ------- - - output_shape (list of tuples): + - output_shape (list of tuples): One entry per destination, each formatted as (features, height, width). """ neuron_shape = self.get_neuron_shape() @@ -165,14 +164,14 @@ def get_output_shape(self) -> List[Tuple[int, int, int]]: return output_shape def summary(self) -> dict: - """ Returns a summary of the convolution's/pooling's kernel sizes and the output shape of the spiking layer.""" + """Returns a summary of the convolution's/pooling's kernel sizes and the output shape of the spiking layer.""" return { "pool": (self._pool), "kernel": list(self.conv_layer.weight.data.shape), - "neuron": self._get_conv_output_shape(), # neuron layer output has the same shape as the convolution layer ouput. + "neuron": self._get_conv_output_shape(), # neuron layer output has the same shape as the convolution layer ouput. } - + def memory_summary(self): """Computes the amount of memory required for each of the components. Note that this is not necessarily the same as the number of parameters due to some architecture design @@ -192,7 +191,9 @@ def memory_summary(self): """ summary = self.summary() f, c, h, w = summary["kernel"] - f, neuron_height, neuron_width = self._get_conv_output_shape() # neuron layer output has the same shape as the convolution layer ouput. + f, neuron_height, neuron_width = ( + self._get_conv_output_shape() + ) # neuron layer output has the same shape as the convolution layer ouput. return { "kernel": c * pow(2, np.ceil(np.log2(h * w)) + np.ceil(np.log2(f))), @@ -200,11 +201,13 @@ def memory_summary(self): * pow(2, np.ceil(np.log2(neuron_height)) + np.ceil(np.log2(neuron_width))), "bias": 0 if self.conv.bias is None else len(self.conv.bias), } - + ####################################################### Private Methods ####################################################### - def _convert_linear_to_conv(self, lin: nn.Linear, layer_data: dict) -> Tuple[nn.Conv2d, Tuple[int, int, int]]: - """ Convert Linear layer to Conv2d. + def _convert_linear_to_conv( + self, lin: nn.Linear, layer_data: dict + ) -> Tuple[nn.Conv2d, Tuple[int, int, int]]: + """Convert Linear layer to Conv2d. Parameters ---------- @@ -218,7 +221,7 @@ def _convert_linear_to_conv(self, lin: nn.Linear, layer_data: dict) -> Tuple[nn. # this flags the necessity to update the I/O shape pre-computed for each of the original layers being compressed within a `DynapcnnLayer` instance. self._lin_to_conv_conversion = True - input_shape = layer_data['input_shape'] + input_shape = layer_data["input_shape"] in_chan, in_h, in_w = input_shape @@ -243,9 +246,9 @@ def _convert_linear_to_conv(self, lin: nn.Linear, layer_data: dict) -> Tuple[nn. ) return layer, input_shape - + def _get_conv_output_shape(self) -> Tuple[int, int, int]: - """ Computes the output dimensions of `conv_layer`. + """Computes the output dimensions of `conv_layer`. Returns ---------- @@ -260,7 +263,13 @@ def _get_conv_output_shape(self) -> Tuple[int, int, int]: dilation = self.conv.dilation # compute the output height and width. - out_height = ((self.in_shape[1] + 2 * padding[0] - dilation[0] * (kernel_size[0] - 1) - 1) // stride[0]) + 1 - out_width = ((self.in_shape[2] + 2 * padding[1] - dilation[1] * (kernel_size[1] - 1) - 1) // stride[1]) + 1 + out_height = ( + (self.in_shape[1] + 2 * padding[0] - dilation[0] * (kernel_size[0] - 1) - 1) + // stride[0] + ) + 1 + out_width = ( + (self.in_shape[2] + 2 * padding[1] - dilation[1] * (kernel_size[1] - 1) - 1) + // stride[1] + ) + 1 return (out_channels, out_height, out_width) diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer_handler.py b/sinabs/backend/dynapcnn/dynapcnn_layer_handler.py index 9e1566bf..877e2e18 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer_handler.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer_handler.py @@ -2,7 +2,7 @@ # contact : wsoaresgirao@gmail.com from copy import deepcopy -from typing import Dict, Callable, Tuple, Union, List +from typing import Callable, Dict, List, Tuple, Union import numpy as np import torch @@ -13,7 +13,8 @@ from .discretize import discretize_conv_spike_ -class DynapcnnLayerHandler(): + +class DynapcnnLayerHandler: """ Class handling the pre-processing of network-level data into (device) layer-level data (i.e., arguments required for a `DynapcnnLayer` instantiation). @@ -36,18 +37,24 @@ class DynapcnnLayerHandler(): def __init__( self, dpcnnl_index: int, - dcnnl_data: Dict[Union[int, str], Union[Dict[str, Union[nn.Module, Tuple[int, int, int], Tuple[int, int, int]]], List[int]]], + dcnnl_data: Dict[ + Union[int, str], + Union[ + Dict[str, Union[nn.Module, Tuple[int, int, int], Tuple[int, int, int]]], + List[int], + ], + ], discretize: bool, sinabs_edges: List[Tuple[int, int]], weight_rescaling_fn: Callable, - entry_nodes: List[int] + entry_nodes: List[int], ): self.dpcnnl_index = dpcnnl_index self.assigned_core = None self.entry_point = False - if 'core_idx' in dcnnl_data: - self.assigned_core = dcnnl_data['core_idx'] + if "core_idx" in dcnnl_data: + self.assigned_core = dcnnl_data["core_idx"] self._lin_to_conv_conversion = False @@ -62,30 +69,34 @@ def __init__( pool = [] self.pool_node_id = [] self.conv_rescaling_factor = None - - self.dynapcnnlayer_destination = dcnnl_data['destinations'] + + self.dynapcnnlayer_destination = dcnnl_data["destinations"] for key, value in dcnnl_data.items(): if isinstance(key, int): # value has data pertaining a node (torch/sinabs layer). - if isinstance(value['layer'], sl.IAFSqueeze): - spk = value['layer'] + if isinstance(value["layer"], sl.IAFSqueeze): + spk = value["layer"] self.spk_node_id = key - elif isinstance(value['layer'], nn.Linear) or isinstance(value['layer'], nn.Conv2d): - conv = value['layer'] + elif isinstance(value["layer"], nn.Linear) or isinstance( + value["layer"], nn.Conv2d + ): + conv = value["layer"] self.conv_node_id = key - elif isinstance(value['layer'], sl.SumPool2d): - pool.append(value['layer']) + elif isinstance(value["layer"], sl.SumPool2d): + pool.append(value["layer"]) self.pool_node_id.append(key) else: - raise ValueError(f'Node {key} has not valid layer associated with it.') - + raise ValueError( + f"Node {key} has not valid layer associated with it." + ) + if not conv: - raise ValueError(f'Convolution layer not present.') - + raise ValueError(f"Convolution layer not present.") + if not spk: - raise ValueError(f'Spiking layer not present.') - + raise ValueError(f"Spiking layer not present.") + spk = deepcopy(spk) if spk.is_state_initialised(): # TODO this line bellow is causing an exception on `.v_men.shape` to be raised in `.get_layer_config_dict()`. Find out why. @@ -93,39 +104,53 @@ def __init__( # TODO hacky stuff: make it better (THIS SEEMS TO BE FIXING THE PROBLEM ABOVE THO). if len(list(spk.v_mem.shape)) != 4: - spk.v_mem = spk.v_mem.data.unsqueeze(-1).unsqueeze(-1) # expand dims. + spk.v_mem = spk.v_mem.data.unsqueeze(-1).unsqueeze(-1) # expand dims. if isinstance(conv, nn.Linear): - # A `nn.Linear` needs to be converted into `nn.Conv2d`. The I/O shapes of the spiking layer are updated + # A `nn.Linear` needs to be converted into `nn.Conv2d`. The I/O shapes of the spiking layer are updated # accordingly following the conversion. - conv, conv_in_shape = self._convert_linear_to_conv(conv, dcnnl_data[self.conv_node_id]) + conv, conv_in_shape = self._convert_linear_to_conv( + conv, dcnnl_data[self.conv_node_id] + ) # the original `nn.Linear` output shape becomes the equivalent `nn.Conv2d` shape. self.conv_out_shape = self._update_conv_node_output_shape( - conv_layer=conv, layer_data=dcnnl_data[self.conv_node_id], input_shape=conv_in_shape) + conv_layer=conv, + layer_data=dcnnl_data[self.conv_node_id], + input_shape=conv_in_shape, + ) # the I/O shapes for neuron layer following the new conv need also to be updated. - self._update_neuron_node_output_shape(spiking_layer_data=dcnnl_data[self.spk_node_id], conv_out_shape=self.conv_out_shape) + self._update_neuron_node_output_shape( + spiking_layer_data=dcnnl_data[self.spk_node_id], + conv_out_shape=self.conv_out_shape, + ) else: - self.conv_out_shape = dcnnl_data[self.conv_node_id]['output_shape'] + self.conv_out_shape = dcnnl_data[self.conv_node_id]["output_shape"] conv = deepcopy(conv) # check if convolution kernel is a square. if conv.kernel_size[0] != conv.kernel_size[1]: - raise ValueError('The kernel of a `nn.Conv2d` must have the same height and width.') + raise ValueError( + "The kernel of a `nn.Conv2d` must have the same height and width." + ) # input shape of conv layer. - self.conv_in_shape = dcnnl_data[self.conv_node_id]['input_shape'] + self.conv_in_shape = dcnnl_data[self.conv_node_id]["input_shape"] # input shape of the `DynapcnnLayer` instance. self.input_shape = self.conv_in_shape # this weight rescale comes from the node projecting into this 'conv' node. - if len(dcnnl_data['conv_rescale_factor']): + if len(dcnnl_data["conv_rescale_factor"]): # this means an `AvgPool2d` has been converted into a `SumPool2d`. - self.conv_rescaling_factor = weight_rescaling_fn(dcnnl_data['conv_rescale_factor']) - conv.weight.data = (conv.weight.data / self.conv_rescaling_factor).clone().detach() + self.conv_rescaling_factor = weight_rescaling_fn( + dcnnl_data["conv_rescale_factor"] + ) + conv.weight.data = ( + (conv.weight.data / self.conv_rescaling_factor).clone().detach() + ) else: # this means `SumPool2d` have been used from the start. conv.weight.data = (conv.weight.data).clone().detach() @@ -142,9 +167,12 @@ def __init__( if len(pool) != 0: # the 1st pooling targets the 1st destination in `dcnnl_data['destinations']`, the 2nd pooling targets the 2nd destination... for plyr in pool: - # @TODO POSSIBLE INCONSISTENCY: if the `SumPool2d` is the result of a conversion from `AvgPool2d` then `SumPool2d.kernel_size` + # @TODO POSSIBLE INCONSISTENCY: if the `SumPool2d` is the result of a conversion from `AvgPool2d` then `SumPool2d.kernel_size` # is of type tuple, otherwise it is an int. - if isinstance(plyr.kernel_size, tuple) and plyr.kernel_size[0] != plyr.kernel_size[1]: + if ( + isinstance(plyr.kernel_size, tuple) + and plyr.kernel_size[0] != plyr.kernel_size[1] + ): raise ValueError("Only square kernels are supported") self.pool_layer.append(deepcopy(plyr)) @@ -158,12 +186,12 @@ def __init__( ####################################################### Public Methods ####################################################### def get_pool_list(self) -> List[int]: - """ This returns a list of integers that describe the number of outputs created by this layer (length of the list) and + """This returns a list of integers that describe the number of outputs created by this layer (length of the list) and whether or not pooling is applied (values > 1). This is meant to generate the `pool`argument for a `DynapcnnLayer` instance. Returns ---------- - - pool (list): Each integer entry represents an output (destination on chip) and whether pooling should be applied (values > 1) or not (values + - pool (list): Each integer entry represents an output (destination on chip) and whether pooling should be applied (values > 1) or not (values equal to 1). The number of entries determines the number of tensors the layer's forward method returns. """ pool = [] @@ -175,13 +203,17 @@ def get_pool_list(self) -> List[int]: elif lyr in self.pool_node_id: # getting kernel sizes from each pooling layer. sumpool_idx = self.pool_node_id.index(lyr) - kernel_size = self.pool_layer[sumpool_idx].kernel_size[0] if isinstance(self.pool_layer[sumpool_idx].kernel_size, tuple) else self.pool_layer[sumpool_idx].kernel_size + kernel_size = ( + self.pool_layer[sumpool_idx].kernel_size[0] + if isinstance(self.pool_layer[sumpool_idx].kernel_size, tuple) + else self.pool_layer[sumpool_idx].kernel_size + ) pool.append(kernel_size) return pool def get_destination_dcnnl_index(self, dcnnl_id: int) -> int: - """ The `forward` method will return as many tensors as there are elements in `self.dynapcnnlayer_destination`. Since the i-th returned tensor is to be + """The `forward` method will return as many tensors as there are elements in `self.dynapcnnlayer_destination`. Since the i-th returned tensor is to be fed to the i-th destionation in `self.dynapcnnlayer_destination`, the return of this method can be used to index a tensor returned in the `forward` method. Parameters @@ -194,8 +226,10 @@ def get_destination_dcnnl_index(self, dcnnl_id: int) -> int: """ return self.dynapcnnlayer_destination.index(dcnnl_id) - def get_modified_node_io(self, dcnnl_data: dict) -> Union[Tuple[int, tuple], Tuple[None, None]]: - """ Follwing a conversion, the I/O shapes of the spiking layer have been updated to match the convolution's + def get_modified_node_io( + self, dcnnl_data: dict + ) -> Union[Tuple[int, tuple], Tuple[None, None]]: + """Follwing a conversion, the I/O shapes of the spiking layer have been updated to match the convolution's output. Thus, all nodes receiving input from this spiking layer need their input shapes updated. Parameters @@ -208,15 +242,17 @@ def get_modified_node_io(self, dcnnl_data: dict) -> Union[Tuple[int, tuple], Tup - output shape (tuple): the new output shape following a converstion from `nn.Linear` to `nn.Conv2d` (`None` if there was no conversion). """ if self._lin_to_conv_conversion: - return self.spk_node_id, dcnnl_data[self.spk_node_id]['output_shape'] + return self.spk_node_id, dcnnl_data[self.spk_node_id]["output_shape"] return None, None - + def zero_grad(self, set_to_none: bool = False) -> None: return self.spk_layer.zero_grad(set_to_none) - - def get_conv_output_shape(self, conv_layer: nn.Conv2d, input_shape: Tuple[int, int, int]) -> Tuple[int, int, int]: - """ Computes the output dimensions of `conv_layer`. - + + def get_conv_output_shape( + self, conv_layer: nn.Conv2d, input_shape: Tuple[int, int, int] + ) -> Tuple[int, int, int]: + """Computes the output dimensions of `conv_layer`. + Parameters ---------- - conv_layer (nn.Conv2d): conv. layer whose output will be computed for. @@ -234,37 +270,49 @@ def get_conv_output_shape(self, conv_layer: nn.Conv2d, input_shape: Tuple[int, i dilation = conv_layer.dilation # compute the output height and width. - out_height = ((input_shape[1] + 2 * padding[0] - dilation[0] * (kernel_size[0] - 1) - 1) // stride[0]) + 1 - out_width = ((input_shape[2] + 2 * padding[1] - dilation[1] * (kernel_size[1] - 1) - 1) // stride[1]) + 1 + out_height = ( + (input_shape[1] + 2 * padding[0] - dilation[0] * (kernel_size[0] - 1) - 1) + // stride[0] + ) + 1 + out_width = ( + (input_shape[2] + 2 * padding[1] - dilation[1] * (kernel_size[1] - 1) - 1) + // stride[1] + ) + 1 return (out_channels, out_height, out_width) - + def __str__(self): - pretty_print = '\n' + pretty_print = "\n" - pretty_print += 'COMPUTATIONAL NODES:\n\n' + pretty_print += "COMPUTATIONAL NODES:\n\n" - pretty_print += f'(node {self.conv_node_id}): {self.conv_layer}\n' - pretty_print += f'(node {self.spk_node_id}): {self.spk_layer}' + pretty_print += f"(node {self.conv_node_id}): {self.conv_layer}\n" + pretty_print += f"(node {self.spk_node_id}): {self.spk_layer}" if len(self.pool_layer) != 0: for idx, lyr in enumerate(self.pool_layer): - pretty_print += f'\n(node {self.pool_node_id[idx]}): {lyr}' + pretty_print += f"\n(node {self.pool_node_id[idx]}): {lyr}" - pretty_print += '\n\nMETADATA:\n' - pretty_print += f'\n> network\'s entry point: {self.entry_point}' - pretty_print += f'\n> convolution\'s weight re-scaling factor: {self.conv_rescaling_factor}' - pretty_print += f'\n> assigned core index: {self.assigned_core}' - pretty_print += f'\n> destination DynapcnnLayers: {self.dynapcnnlayer_destination}' + pretty_print += "\n\nMETADATA:\n" + pretty_print += f"\n> network's entry point: {self.entry_point}" + pretty_print += ( + f"\n> convolution's weight re-scaling factor: {self.conv_rescaling_factor}" + ) + pretty_print += f"\n> assigned core index: {self.assigned_core}" + pretty_print += ( + f"\n> destination DynapcnnLayers: {self.dynapcnnlayer_destination}" + ) for node, destinations in self.nodes_destinations.items(): - pretty_print += f'\n> node {node} feeds input to nodes {destinations}' + pretty_print += f"\n> node {node} feeds input to nodes {destinations}" return pretty_print - + ####################################################### Private Methods ####################################################### - def _update_neuron_node_output_shape(self, spiking_layer_data: dict, conv_out_shape: tuple) -> None: - """ Updates the spiking layer's I/O shapes after the conversion of a `nn.Linear` into a `nn.Conv2d` (to match the convolution's output). + def _update_neuron_node_output_shape( + self, spiking_layer_data: dict, conv_out_shape: tuple + ) -> None: + """Updates the spiking layer's I/O shapes after the conversion of a `nn.Linear` into a `nn.Conv2d` (to match the convolution's output). Parameters ---------- @@ -273,15 +321,17 @@ def _update_neuron_node_output_shape(self, spiking_layer_data: dict, conv_out_sh """ # spiking layer consumes the tensor coming out of the conv. layer. - spiking_layer_data['input_shape'] = conv_out_shape + spiking_layer_data["input_shape"] = conv_out_shape # spiking layer outputs the same shape as the conv. layer. - spiking_layer_data['output_shape'] = spiking_layer_data['input_shape'] + spiking_layer_data["output_shape"] = spiking_layer_data["input_shape"] + + def _update_conv_node_output_shape( + self, conv_layer: nn.Conv2d, layer_data: dict, input_shape: Tuple[int, int, int] + ) -> Tuple[int, int, int]: + """Updates the shape of the output tensor of a node that used to be a `nn.Linear` and became a `nn.Conv2d`. - def _update_conv_node_output_shape(self, conv_layer: nn.Conv2d, layer_data: dict, input_shape: Tuple[int, int, int]) -> Tuple[int, int, int]: - """ Updates the shape of the output tensor of a node that used to be a `nn.Linear` and became a `nn.Conv2d`. - The input shapes to nodes are extracted using a list of edges by finding the output shape of the 1st element - in the edge and setting it as the input shape to the 2nd element in the edge. If a node used to be a `nn.Linear` + in the edge and setting it as the input shape to the 2nd element in the edge. If a node used to be a `nn.Linear` and it became a `nn.Conv2d`, output shape in the mapper needs to be updated, otherwise there will be a mismatch between its output and the input it provides to another node. @@ -295,12 +345,14 @@ def _update_conv_node_output_shape(self, conv_layer: nn.Conv2d, layer_data: dict ---------- - output_shape (tuple): the tensor shape produced by the `nn.Conv2d` created from a `nn.Linear`. """ - layer_data['output_shape'] = self.get_conv_output_shape(conv_layer, input_shape) + layer_data["output_shape"] = self.get_conv_output_shape(conv_layer, input_shape) + + return layer_data["output_shape"] - return layer_data['output_shape'] - - def _convert_linear_to_conv(self, lin: nn.Linear, layer_data: dict) -> Tuple[nn.Conv2d, Tuple[int, int, int]]: - """ Convert Linear layer to Conv2d. + def _convert_linear_to_conv( + self, lin: nn.Linear, layer_data: dict + ) -> Tuple[nn.Conv2d, Tuple[int, int, int]]: + """Convert Linear layer to Conv2d. Parameters ---------- @@ -314,7 +366,7 @@ def _convert_linear_to_conv(self, lin: nn.Linear, layer_data: dict) -> Tuple[nn. # this flags the necessity to update the I/O shape pre-computed for each of the original layers being compressed within a `DynapcnnLayer` instance. self._lin_to_conv_conversion = True - input_shape = layer_data['input_shape'] + input_shape = layer_data["input_shape"] in_chan, in_h, in_w = input_shape @@ -339,15 +391,15 @@ def _convert_linear_to_conv(self, lin: nn.Linear, layer_data: dict) -> Tuple[nn. ) return layer, input_shape - + def _get_destinations_input_source(self, sinabs_edges: list) -> dict: - """ Creates a mapping between each layer in this `DynapcnnLayer` instance and its targe nodes that are part of different + """Creates a mapping between each layer in this `DynapcnnLayer` instance and its targe nodes that are part of different `DynapcnnLayer` instances. This mapping is used to figure out how many tensors the `forward` method needs to return. Parameters ---------- - - sinabs_edges (list): each `nn.Module` within `dcnnl_data` is a node in the original computational graph describing a spiking - network. An edge `(A, B)` describes how modules forward data amongst themselves. This list is used by a `DynapcnnLayer` to + - sinabs_edges (list): each `nn.Module` within `dcnnl_data` is a node in the original computational graph describing a spiking + network. An edge `(A, B)` describes how modules forward data amongst themselves. This list is used by a `DynapcnnLayer` to figure out the number and sequence of output tesnors its forward method needs to return. Returns @@ -376,26 +428,36 @@ def _get_destinations_input_source(self, sinabs_edges: list) -> dict: destinations_input_source[id].append(edge[1]) return destinations_input_source - + def get_pool_kernel_size(self, node: int) -> int: - """ Returns the pooling kernel size if `node` is a pooling layer.""" + """Returns the pooling kernel size if `node` is a pooling layer.""" if node in self.pool_node_id: i = self.pool_node_id.index(node) - return self.pool_layer[i].kernel_size[0] if isinstance(self.pool_layer[i].kernel_size, tuple) else self.pool_layer[i].kernel_size + return ( + self.pool_layer[i].kernel_size[0] + if isinstance(self.pool_layer[i].kernel_size, tuple) + else self.pool_layer[i].kernel_size + ) elif node == self.spk_node_id: return 1 else: - raise ValueError(f'Node {node} does not belong to DynapcnnLayer {self.dpcnnl_index}.') - + raise ValueError( + f"Node {node} does not belong to DynapcnnLayer {self.dpcnnl_index}." + ) + @staticmethod def find_nodes_core_id(node: int, all_handlers: dict) -> int: - """ Loops through all handlers in `all_handlers` to find to which core a `DynapcnnLayer` containing - `node` has been assigned to. """ + """Loops through all handlers in `all_handlers` to find to which core a `DynapcnnLayer` containing + `node` has been assigned to.""" for _, dcnnl in all_handlers.items(): - if node == dcnnl['layer_handler'].conv_node_id or node == dcnnl['layer_handler'].spk_node_id or node in dcnnl['layer_handler'].pool_node_id: - return dcnnl['layer_handler'].assigned_core - - raise ValueError(f'Node {node} not found in any of the cores.') + if ( + node == dcnnl["layer_handler"].conv_node_id + or node == dcnnl["layer_handler"].spk_node_id + or node in dcnnl["layer_handler"].pool_node_id + ): + return dcnnl["layer_handler"].assigned_core + + raise ValueError(f"Node {node} not found in any of the cores.") diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index 187cf13f..bb9b485d 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -1,35 +1,33 @@ # author : Willian Soares Girao # contact : wsoaresgirao@gmail.com -import time, copy -from typing import List, Optional, Sequence, Tuple, Union, Dict, Callable +import copy +import time +from typing import Callable, Dict, List, Optional, Sequence, Tuple, Union import samna -import sinabs.layers as sl import torch import torch.nn as nn import sinabs +import sinabs.layers as sl from .chip_factory import ChipFactory from .dvs_layer import DVSLayer -from .io import open_device, disable_timestamps, enable_timestamps, reset_timestamps +from .dynapcnn_layer import DynapcnnLayer +from .dynapcnnnetwork_module import DynapcnnNetworkModule +from .io import disable_timestamps, enable_timestamps, open_device, reset_timestamps +from .NIRGraphExtractor import NIRtoDynapcnnNetworkGraph +from .sinabs_edges_handler import merge_handler from .utils import ( DEFAULT_IGNORED_LAYER_TYPES, build_from_graph, build_nodes_to_dcnnl_map, parse_device_id, + topological_sorting, ) - -from .NIRGraphExtractor import NIRtoDynapcnnNetworkGraph -from .sinabs_edges_handler import merge_handler - -from .dynapcnnnetwork_module import DynapcnnNetworkModule from .weight_rescaling_methods import rescale_method_1 -from .dynapcnn_layer import DynapcnnLayer - -from .utils import topological_sorting class DynapcnnNetwork(nn.Module): def __init__( @@ -39,7 +37,7 @@ def __init__( batch_size: int, dvs_input: bool = False, discretize: bool = True, - weight_rescaling_fn: Callable = rescale_method_1 + weight_rescaling_fn: Callable = rescale_method_1, ): """ Given a sinabs spiking network, prepare a dynapcnn-compatible network. This can be used to @@ -51,7 +49,7 @@ def __init__( - snn (nn.Module): a implementing a spiking network. - input_shape (tuple): a description of the input dimensions as `(features, height, width)`. - dvs_input (bool): wether or not dynapcnn receive input from its DVS camera. - - discretize (bool): If `True`, discretize the parameters and thresholds. This is needed for uploading + - discretize (bool): If `True`, discretize the parameters and thresholds. This is needed for uploading weights to dynapcnn. Set to `False` only for testing purposes. - weight_rescaling_fn (callable): a method that handles how the re-scaling factor for one or more `SumPool2d` projecting to the same convolutional layer are combined/re-scaled before applying them. @@ -84,35 +82,41 @@ def __init__( # computational graph from original PyTorch module. self._graph_tracer = NIRtoDynapcnnNetworkGraph( - snn, - torch.randn((batch_size, *self.input_shape))) # needs the batch dimension. + snn, torch.randn((batch_size, *self.input_shape)) + ) # needs the batch dimension. # get list of nodes from graph tracer that act as entry points to the network. self._entry_nodes = copy.deepcopy(self._graph_tracer.entry_nodes) # pre-process and group original nodes/edges from the graph tracer into data structures used to later create `DynapcnnLayer` instance. - self._sinabs_edges, \ - self._sinabs_modules_map, \ - self._nodes_name_remap = self._get_sinabs_edges_and_modules() - + self._sinabs_edges, self._sinabs_modules_map, self._nodes_name_remap = ( + self._get_sinabs_edges_and_modules() + ) + # create a dict holding the data necessary to instantiate a `DynapcnnLayer`. self._nodes_to_dcnnl_map = build_nodes_to_dcnnl_map( - layers=self._sinabs_modules_map, - edges=self._sinabs_edges) - + layers=self._sinabs_modules_map, edges=self._sinabs_edges + ) + # updates 'self._nodes_to_dcnnl_map' to include the I/O shape for each node. self._populate_nodes_io() # build `DynapcnnLayer` instances from graph edges and mapper. self._dynapcnn_layers, self._dynapcnnlayers_handlers = build_from_graph( - discretize = discretize, - edges = self._sinabs_edges, - nodes_to_dcnnl_map = self._nodes_to_dcnnl_map, - weight_rescaling_fn = weight_rescaling_fn, - entry_nodes = self._entry_nodes) - + discretize=discretize, + edges=self._sinabs_edges, + nodes_to_dcnnl_map=self._nodes_to_dcnnl_map, + weight_rescaling_fn=weight_rescaling_fn, + entry_nodes=self._entry_nodes, + ) + # these gather all data necessay to implement the forward method for this class. - self._dcnnl_edges, self._layers_mapper, self._merge_points, self._topological_order = self._get_network_module() + ( + self._dcnnl_edges, + self._layers_mapper, + self._merge_points, + self._topological_order, + ) = self._get_network_module() # all necessary `DynapcnnLayer` data held in `self._layers_mapper`: removing intermediary data structures no longer necessary. del self._graph_tracer @@ -128,38 +132,38 @@ def __init__( @property def dcnnl_edges(self): return self._dcnnl_edges - + @property def merge_points(self): return self._merge_points - + @property def topological_order(self): return self._topological_order - + @property def layers_mapper(self) -> Dict[int, DynapcnnLayer]: return self._layers_mapper - + @property def layers_handlers(self): return self._dynapcnnlayers_handlers - + @property def chip_layers_ordering(self): return self._chip_layers_ordering - + def get_output_core_id(self) -> int: - """ .""" + """.""" # TODO if a network with two output layers is deployed, which is not supported yet btw, this monitoring part needs to be revised. for _, ith_dcnnl in self._layers_mapper.items(): if len(ith_dcnnl.dynapcnnlayer_destination) == 0: # a DynapcnnLayer without destinations is taken to be the output layer of the network. return ith_dcnnl.assigned_core - + def get_input_core_id(self) -> list: - """ Since the chip allows for multiple input layers (that merge into a single output at some point), this method returns + """Since the chip allows for multiple input layers (that merge into a single output at some point), this method returns a list of all core IDs to which an input layer of the network has been assigned to. """ entry_points = [] @@ -168,9 +172,9 @@ def get_input_core_id(self) -> list: entry_points.append(ith_dcnnl.assigned_core) return entry_points - + def hw_forward(self, x): - """ Forwards data through the chip. """ + """Forwards data through the chip.""" # flush buffer. _ = self.samna_output_buffer.get_events() @@ -201,12 +205,12 @@ def hw_forward(self, x): return received_evts def forward(self, x): - """ Forwards data through the `DynapcnnNetwork` instance. This method relies on three main data structures created to represent + """Forwards data through the `DynapcnnNetwork` instance. This method relies on three main data structures created to represent the `DynapcnnLayer`s in the network and the data propagation through them during the forward pass: - `self._topological_order` (list): this is used to guide the sequence in which the `DynapcnnLayer`s in `self._layers_mapper` are to be called to generate the input tensors to be propagated through the network during the forward pass. - - `self._dcnnl_edges` (list): this list of edges represent the graph describing the interactions between each `DynapcnnLayer` (the nodes in + - `self._dcnnl_edges` (list): this list of edges represent the graph describing the interactions between each `DynapcnnLayer` (the nodes in the edges are the indices of these layers). An `edge` is used to index a mapper (using `edge[0]`) in order to retrieve the output to be fed as input to a `DynapcnnLayer` instance (indexed by `edge[1]`). - `self._layers_mapper` (dict): a mapper used to forward data through the `DynapcnnNetwork` instances. Each `key` is the indice associated @@ -230,19 +234,23 @@ def forward(self, x): # there are two sources of input for `DynapcnnLayer i`. # by this points the arguments of the `Merge` associated with `i` should have been computed due to the topological sorting. - arg1, arg2 = self._merge_points[i]['sources'] + arg1, arg2 = self._merge_points[i]["sources"] # find which returned tensor from the `forward` call of DynapcnnLayers `arg1` and `arg2` are to be fed # to the target DynapcnnLayer `i`. - return_index_arg1 = self._dynapcnnlayers_handlers[arg1].get_destination_dcnnl_index(i) - return_index_arg2 = self._dynapcnnlayers_handlers[arg2].get_destination_dcnnl_index(i) + return_index_arg1 = self._dynapcnnlayers_handlers[ + arg1 + ].get_destination_dcnnl_index(i) + return_index_arg2 = self._dynapcnnlayers_handlers[ + arg2 + ].get_destination_dcnnl_index(i) # retrieve input tensors to `Merge`. _arg1 = layers_outputs[arg1][return_index_arg1] _arg2 = layers_outputs[arg2][return_index_arg2] # merge tensors. - merge_output = self._merge_points[i]['merge'](_arg1, _arg2) + merge_output = self._merge_points[i]["merge"](_arg1, _arg2) # call the forward. layers_outputs[i] = self._layers_mapper[i](merge_output) @@ -255,16 +263,20 @@ def forward(self, x): # find which returned tensor from the `forward` call of the source DynapcnnLayer `src_dcnnl` is to be fed # to the target DynapcnnLayer `i`. - return_index = self._dynapcnnlayers_handlers[src_dcnnl].get_destination_dcnnl_index(i) + return_index = self._dynapcnnlayers_handlers[ + src_dcnnl + ].get_destination_dcnnl_index(i) # call the forward. - layers_outputs[i] = self._layers_mapper[i](layers_outputs[src_dcnnl][return_index]) - + layers_outputs[i] = self._layers_mapper[i]( + layers_outputs[src_dcnnl][return_index] + ) + # TODO - this assumes the network has a single output node. return layers_outputs[self._topological_order[-1]][0] - + def parameters(self) -> list: - """ Gathers all the parameters of the network in a list. This is done by accessing the convolutional layer in each `DynapcnnLayer`, + """Gathers all the parameters of the network in a list. This is done by accessing the convolutional layer in each `DynapcnnLayer`, calling its `.parameters` method and saving it to a list. Note: the method assumes no biases are used. @@ -280,9 +292,9 @@ def parameters(self) -> list: parameters.extend(layer.conv_layer.parameters()) return parameters - + def init_weights(self, init_fn: nn.init = nn.init.xavier_normal_) -> None: - """ Call the weight initialization method `init_fn` on each `DynapcnnLayer.conv_layer.weight.data` in the `DynapcnnNetwork` instance. + """Call the weight initialization method `init_fn` on each `DynapcnnLayer.conv_layer.weight.data` in the `DynapcnnNetwork` instance. Parameters ---------- @@ -293,14 +305,14 @@ def init_weights(self, init_fn: nn.init = nn.init.xavier_normal_) -> None: init_fn(layer.conv_layer.weight.data) def detach_neuron_states(self) -> None: - """ Detach the neuron states and activations from current computation graph (necessary). """ + """Detach the neuron states and activations from current computation graph (necessary).""" for module in self._layers_mapper.values(): if isinstance(module, sinabs.backend.dynapcnn.dynapcnn_layer.DynapcnnLayer): if isinstance(module.spk_layer, sl.StatefulLayer): for name, buffer in module.spk_layer.named_buffers(): buffer.detach_() - + def to( self, device="cpu", @@ -348,12 +360,12 @@ def to( if isinstance(device, torch.device): self._to_device(device) - + elif isinstance(device, str): device_name, _ = parse_device_id(device) if device_name in ChipFactory.supported_devices: - + # generate config. config = self._make_config( chip_layers_ordering=chip_layers_ordering, @@ -371,10 +383,10 @@ def to( if slow_clk_frequency is not None: dk_io = self.samna_device.get_io_module() dk_io.set_slow_clk(True) - dk_io.set_slow_clk_rate(slow_clk_frequency) # Hz + dk_io.set_slow_clk_rate(slow_clk_frequency) # Hz builder = ChipFactory(device).get_config_builder() - + # create input source node. self.samna_input_buffer = builder.get_input_buffer() @@ -404,21 +416,21 @@ def to( self.samna_config = config return self - + else: self._to_device(device) - + else: raise Exception("Unknown device description.") - + ####################################################### Private Methods ####################################################### def _get_input_to_dcnnl(self, dcnnl_ID) -> int: - """ Returns the ID of the first `DynapcnnLayer` forwarding its input to `dcnnl_ID`. """ + """Returns the ID of the first `DynapcnnLayer` forwarding its input to `dcnnl_ID`.""" for edge in self._dcnnl_edges: if edge[1] == dcnnl_ID: return edge[0] - raise ValueError(f'DynapcnnLayer {dcnnl_ID} has no source of input.') + raise ValueError(f"DynapcnnLayer {dcnnl_ID} has no source of input.") def _make_config( self, @@ -497,17 +509,28 @@ def _make_config( for dcnnl_index, ith_dcnnl in self._layers_mapper.items(): # TODO if a network with two output layers is deployed, which is not supported yet btw, this monitoring part needs to be revised. - if len(self._dynapcnnlayers_handlers[dcnnl_index].dynapcnnlayer_destination) == 0: + if ( + len( + self._dynapcnnlayers_handlers[ + dcnnl_index + ].dynapcnnlayer_destination + ) + == 0 + ): # a DynapcnnLayer without destinations is taken to be the output layer of the network. - monitor_chip_layers.append(self._dynapcnnlayers_handlers[dcnnl_index].assigned_core) + monitor_chip_layers.append( + self._dynapcnnlayers_handlers[dcnnl_index].assigned_core + ) elif monitor_layers == "all": for dcnnl_index, ith_dcnnl in self._layers_mapper.items(): # TODO not handling DVSLayer yet # monitor each chip core (if not a DVSLayer). if not isinstance(ith_dcnnl, DVSLayer): - monitor_chip_layers.append(self._dynapcnnlayers_handlers[dcnnl_index].assigned_core) - + monitor_chip_layers.append( + self._dynapcnnlayers_handlers[dcnnl_index].assigned_core + ) + if monitor_layers: if "dvs" in monitor_layers: monitor_chip_layers.append("dvs") @@ -522,13 +545,13 @@ def _make_config( if config_builder.validate_configuration(config): # validate config. print("Network is valid: \n") - + return config else: raise ValueError(f"Generated config is not valid for {device}") def _get_network_module(self) -> Union[list, dict, dict]: - """ Uses the `DynapcnnLayer` instances in `self._dynapcnn_layers` and the connectivity between them to create three data structures + """Uses the `DynapcnnLayer` instances in `self._dynapcnn_layers` and the connectivity between them to create three data structures that guide the data forwarding between the layer during the forward pass. Returns @@ -545,13 +568,20 @@ def _get_network_module(self) -> Union[list, dict, dict]: # get connections between `DynapcnnLayer`s. dcnnl_edges = self._get_dynapcnnlayers_edges() - dcnnnet_module = DynapcnnNetworkModule(dcnnl_edges, self._dynapcnn_layers, self._dynapcnnlayers_handlers) + dcnnnet_module = DynapcnnNetworkModule( + dcnnl_edges, self._dynapcnn_layers, self._dynapcnnlayers_handlers + ) + + return ( + dcnnnet_module.dcnnl_edges, + dcnnnet_module.forward_map, + dcnnnet_module.merge_points, + topological_sorting(dcnnl_edges), + ) - return dcnnnet_module.dcnnl_edges, dcnnnet_module.forward_map, dcnnnet_module.merge_points, topological_sorting(dcnnl_edges) - def _get_dynapcnnlayers_edges(self) -> List[Tuple[int, int]]: - """ Create edges representing connections between `DynapcnnLayer` instances. - + """Create edges representing connections between `DynapcnnLayer` instances. + Returns ---------- - dcnnl_edges (list): a list of edges using the IDs of `DynapcnnLayer` instances. These edges describe the computational @@ -560,29 +590,32 @@ def _get_dynapcnnlayers_edges(self) -> List[Tuple[int, int]]: dcnnl_edges = [] for dcnnl_idx, layer_data in self._dynapcnn_layers.items(): - for dest in layer_data['destinations']: + for dest in layer_data["destinations"]: dcnnl_edges.append((dcnnl_idx, dest)) - + return dcnnl_edges - - def _get_sinabs_edges_and_modules(self) -> Tuple[List[Tuple[int, int]], Dict[int, nn.Module], Dict[int, int]]: - """ The computational graph extracted from `snn` might contain layers that are ignored (e.g. a `nn.Flatten` will be + + def _get_sinabs_edges_and_modules( + self, + ) -> Tuple[List[Tuple[int, int]], Dict[int, nn.Module], Dict[int, int]]: + """The computational graph extracted from `snn` might contain layers that are ignored (e.g. a `nn.Flatten` will be ignored when creating a `DynapcnnLayer` instance). Thus the list of edges from such model need to be rebuilt such that if there are edges `(A, X)` and `(X, B)`, and `X` is an ignored layer, an edge `(A, B)` is created. Returns ---------- - - edges_without_merge (list): a list of edges based on `sinabs_edges` but where edges involving a `Merge` layer have been + - edges_without_merge (list): a list of edges based on `sinabs_edges` but where edges involving a `Merge` layer have been remapped to connect the nodes involved in the merging directly. - - sinabs_modules_map (dict): a dict containing the nodes of the graph (described now by `edges_without_merge`) as `key` and + - sinabs_modules_map (dict): a dict containing the nodes of the graph (described now by `edges_without_merge`) as `key` and their associated module as `value`. - remapped_nodes (dict): a dict where `key` is the original node name (as extracted by `self._graph_tracer`) and `value` is the new node name (after ignored layers have been dropped and `Merge` layers have be processed before being removed). """ - + # remap `(A, X)` and `(X, B)` into `(A, B)` if `X` is a layer in the original `snn` to be ignored. - sinabs_edges, remapped_nodes = self._graph_tracer.remove_ignored_nodes( - DEFAULT_IGNORED_LAYER_TYPES) + sinabs_edges, remapped_nodes = self._graph_tracer.remove_ignored_nodes( + DEFAULT_IGNORED_LAYER_TYPES + ) # nodes (layers' "names") need remapping in case some layers have been removed (e.g. a `nn.Flattern` is ignored). sinabs_modules_map = {} @@ -599,13 +632,13 @@ def _get_sinabs_edges_and_modules(self) -> Tuple[List[Tuple[int, int]], Dict[int self._entry_nodes = temp return edges_without_merge, sinabs_modules_map, remapped_nodes - + def _populate_nodes_io(self): - """ Loops through the nodes in the original graph to retrieve their I/O tensor shapes and add them to their respective + """Loops through the nodes in the original graph to retrieve their I/O tensor shapes and add them to their respective representations in `self._nodes_to_dcnnl_map`.""" def find_original_node_name(name_mapper: dict, node: int) -> str: - """ Find what a node is originally named when built in `self._graph_tracer`. + """Find what a node is originally named when built in `self._graph_tracer`. Returns ---------- @@ -614,26 +647,28 @@ def find_original_node_name(name_mapper: dict, node: int) -> str: for orig_name, new_name in name_mapper.items(): if new_name == node: return orig_name - raise ValueError(f'Node {node} could not be found within the name remapping done by self._get_sinabs_edges_and_modules().') - + raise ValueError( + f"Node {node} could not be found within the name remapping done by self._get_sinabs_edges_and_modules()." + ) + def find_my_input(edges_list: list, node: int) -> int: - """ Returns the node `X` in the first edge `(X, node)`. + """Returns the node `X` in the first edge `(X, node)`. Parameters ---------- - node (int): the node in the computational graph for which we whish to find the input source (either another node in the graph or the original input itself to the network). - + Returns ---------- - input source (int): this indicates the node in the computational graph providing the input to `node`. If `node` is - receiving outside input (i.e., it is a starting node) the return will be -1. For example, this will be the case + receiving outside input (i.e., it is a starting node) the return will be -1. For example, this will be the case when a network with two independent branches (each starts from a different "input node") merge along the computational graph. """ for edge in edges_list: if edge[1] == node: # TODO nodes originally receiving input from merge will appear twice in the list of edges, one - # edge per input to the merge layer. For now both inputs to a `Merge` have the same dimensions + # edge per input to the merge layer. For now both inputs to a `Merge` have the same dimensions # necessarily so this works for now but later will have to be revised. return edge[0] return -1 @@ -656,36 +691,43 @@ def find_my_input(edges_list: list, node: int) -> int: if input_node == -1: # node does not have an input source within the graph (it consumes the original input to the model). - node_data['input_shape'] = tuple(list(_in)[1:]) + node_data["input_shape"] = tuple(list(_in)[1:]) else: # input comes from another node in the graph. - input_node_orig_name = find_original_node_name(self._nodes_name_remap, input_node) - _, _input_source_shape = self._graph_tracer.get_node_io_shapes(input_node_orig_name) - node_data['input_shape'] = tuple(list(_input_source_shape)[1:]) + input_node_orig_name = find_original_node_name( + self._nodes_name_remap, input_node + ) + _, _input_source_shape = ( + self._graph_tracer.get_node_io_shapes( + input_node_orig_name + ) + ) + node_data["input_shape"] = tuple( + list(_input_source_shape)[1:] + ) else: # first node does not have an input source within the graph. - node_data['input_shape'] = tuple(list(_in)[1:]) + node_data["input_shape"] = tuple(list(_in)[1:]) - node_data['output_shape'] = tuple(list(_out)[1:]) + node_data["output_shape"] = tuple(list(_out)[1:]) def _to_device(self, device: torch.device) -> None: - """ Access each sub-layer within all `DynapcnnLayer` instances and call `.to(device)` on them.""" + """Access each sub-layer within all `DynapcnnLayer` instances and call `.to(device)` on them.""" for layer in self._layers_mapper.values(): if isinstance(layer, sinabs.backend.dynapcnn.dynapcnn_layer.DynapcnnLayer): layer.to(device) - + for _, data in self._merge_points.items(): - data['merge'].to(device) + data["merge"].to(device) def __str__(self): - pretty_print = '' + pretty_print = "" for idx, layer_data in self._layers_mapper.items(): - pretty_print += f'----------------------- [ DynapcnnLayer {idx} ] -----------------------\n' - pretty_print += f'{layer_data}\n\n' - + pretty_print += f"----------------------- [ DynapcnnLayer {idx} ] -----------------------\n" + pretty_print += f"{layer_data}\n\n" + return pretty_print - - + class DynapcnnCompatibleNetwork(DynapcnnNetwork): """Deprecated class, use DynapcnnNetwork instead.""" @@ -702,4 +744,4 @@ def __init__( "DynapcnnCompatibleNetwork has been renamed to DynapcnnNetwork " + "and will be removed in a future release." ) - super().__init__(snn, input_shape, dvs_input, discretize) \ No newline at end of file + super().__init__(snn, input_shape, dvs_input, discretize) diff --git a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py index 4eb7ac0a..e61dbf26 100644 --- a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py +++ b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py @@ -1,13 +1,17 @@ # author : Willian Soares Girao # contact : wsoaresgirao@gmail.com -import torch.nn as nn -from typing import List, Tuple, Dict, Union import copy +from typing import Dict, List, Tuple, Union + +import torch.nn as nn + import sinabs.layers as sl + from .dynapcnn_layer import DynapcnnLayer -class DynapcnnNetworkModule(): + +class DynapcnnNetworkModule: """ Uses the set of `DynapcnnLayer`\`DynapcnnLayerHandler` instances and how they address each other to define what the `forward` method of the model should do. @@ -16,22 +20,29 @@ class DynapcnnNetworkModule(): - dcnnl_edges (list): tuples representing the output->input mapping between `DynapcnnLayer` instances that have been used as configuration for each core `CNNLayerConifg`. - dynapcnn_layers (dict): a mapper containing `DynapcnnLayer` instances. - - dynapcnnlayers_handlers (dict): a mapper containing `DynapcnnLayerHandler` instances (hold network-level + - dynapcnnlayers_handlers (dict): a mapper containing `DynapcnnLayerHandler` instances (hold network-level data that was used to create the respective `DynapcnnLayer` instances in `dynapcnn_layers`). """ - def __init__(self, dcnnl_edges: List[Tuple[int, int]], dynapcnn_layers: Dict[int, dict], dynapcnnlayers_handlers: Dict[int, dict]): + def __init__( + self, + dcnnl_edges: List[Tuple[int, int]], + dynapcnn_layers: Dict[int, dict], + dynapcnnlayers_handlers: Dict[int, dict], + ): self.dcnnl_edges = dcnnl_edges # create mappers to handle `DynapcnnLayer` instances' forward calling. - self.forward_map, self.merge_points = self._build_module_forward_from_graph(dcnnl_edges, dynapcnn_layers) + self.forward_map, self.merge_points = self._build_module_forward_from_graph( + dcnnl_edges, dynapcnn_layers + ) # add extra edges marking which nodes are input to the network. self._add_entry_points_edges(dynapcnnlayers_handlers) def _add_entry_points_edges(self, dynapcnnlayers_handlers: dict) -> None: - """ Addes an extra edge `('input', X)` to `self.dcnnl_edges` if `X` is an entry point of the `DynapcnnNetwork` + """Addes an extra edge `('input', X)` to `self.dcnnl_edges` if `X` is an entry point of the `DynapcnnNetwork` (i.e., `dynapcnnlayers_handlers[X]['layer_handler'].entry_point = True`). Parameters @@ -40,13 +51,15 @@ def _add_entry_points_edges(self, dynapcnnlayers_handlers: dict) -> None: destination layers, etc.). """ for indx, dcnnl_data in dynapcnnlayers_handlers.items(): - if dcnnl_data['layer_handler'].entry_point: - self.dcnnl_edges.append(('input', indx)) - - def _spot_merging_points(self, dcnnl_edges: list) -> Dict[int, Dict[Tuple, sl.Merge]]: - """ Loops throught the edges of the computational graph from a `DynapcnnNetwork` to flag with nodes need - input from a `Merge` layer and what the arguments of this layer should be. - + if dcnnl_data["layer_handler"].entry_point: + self.dcnnl_edges.append(("input", indx)) + + def _spot_merging_points( + self, dcnnl_edges: list + ) -> Dict[int, Dict[Tuple, sl.Merge]]: + """Loops throught the edges of the computational graph from a `DynapcnnNetwork` to flag with nodes need + input from a `Merge` layer and what the arguments of this layer should be. + Parameters ---------- - dcnnl_edges (list): tuples representing the output->input mapping between `DynapcnnLayer` instances @@ -69,21 +82,25 @@ def _spot_merging_points(self, dcnnl_edges: list) -> Dict[int, Dict[Tuple, sl.Me if fan_in == 2 and trg_node not in nodes_with_merge_input: # node needs input from a `Merge` layer: instantiate `Merge` and its arguments. - nodes_with_merge_input[trg_node] = {'sources': tuple(src_nodes), 'merge': sl.Merge()} - + nodes_with_merge_input[trg_node] = { + "sources": tuple(src_nodes), + "merge": sl.Merge(), + } + if fan_in > 2: - raise ValueError(f'Node {trg_node} is the has fan-in of {fan_in}: only fan-in of 2 is currently handled.') - + raise ValueError( + f"Node {trg_node} is the has fan-in of {fan_in}: only fan-in of 2 is currently handled." + ) + return nodes_with_merge_input - + def _build_module_forward_from_graph( - self, - dcnnl_edges: list, - dynapcnn_layers: dict) -> Union[Dict[int, DynapcnnLayer], Dict[Tuple, sl.Merge]]: - """ Creates two mappers, one indexing each `DynapcnnLayer` by its index (a node in `dcnnl_edges`) and another + self, dcnnl_edges: list, dynapcnn_layers: dict + ) -> Union[Dict[int, DynapcnnLayer], Dict[Tuple, sl.Merge]]: + """Creates two mappers, one indexing each `DynapcnnLayer` by its index (a node in `dcnnl_edges`) and another indexing the `DynapcnnLayer` instances (also by the index) that need their input being the output of a `Merge` layer (i.e., they are nodes in the graph where two different layer outputs converge to). - + Parameters ---------- - dcnnl_edges (list): tuples representing the output->input mapping between `DynapcnnLayer` instances @@ -94,7 +111,7 @@ def _build_module_forward_from_graph( Returns ---------- - forward_map (dict): a mapper where each `key` is the layer index (`DynapcnnLayer.dpcnnl_index`) and the `value` the layer instance itself. - - merge_points (dict): a mapper where each `key` is the layer index and the `value` is a dictionary with a `Merge` layer (`merge_points[key]['merge'] = Merge()`, + - merge_points (dict): a mapper where each `key` is the layer index and the `value` is a dictionary with a `Merge` layer (`merge_points[key]['merge'] = Merge()`, computing the input tensor to layer `key`) and its arguments (`merge_points[key]['sources'] = (int A, int B)`, where `A` and `B` are the `DynapcnnLayer` instances for which the ouput is to be used as the `Merge` arguments). """ @@ -106,13 +123,17 @@ def _build_module_forward_from_graph( forward_map = {} for edge in dcnnl_edges: - src_dcnnl = edge[0] # source layer - trg_dcnnl = edge[1] # target layer + src_dcnnl = edge[0] # source layer + trg_dcnnl = edge[1] # target layer if src_dcnnl not in forward_map: - forward_map[src_dcnnl] = copy.deepcopy(dynapcnn_layers[src_dcnnl]['layer']) - + forward_map[src_dcnnl] = copy.deepcopy( + dynapcnn_layers[src_dcnnl]["layer"] + ) + if trg_dcnnl not in forward_map: - forward_map[trg_dcnnl] = copy.deepcopy(dynapcnn_layers[trg_dcnnl]['layer']) + forward_map[trg_dcnnl] = copy.deepcopy( + dynapcnn_layers[trg_dcnnl]["layer"] + ) - return forward_map, merge_points \ No newline at end of file + return forward_map, merge_points diff --git a/sinabs/backend/dynapcnn/exceptions.py b/sinabs/backend/dynapcnn/exceptions.py index 08aeb923..fec09218 100644 --- a/sinabs/backend/dynapcnn/exceptions.py +++ b/sinabs/backend/dynapcnn/exceptions.py @@ -18,27 +18,45 @@ class InputConfigurationError(Exception): pass + class WrongModuleCount(Exception): dynapcnnlayer_indx: type modules_count: type def __init__(self, dynapcnnlayer_indx, modules_count): - super().__init__(f"A DynapcnnLayer {dynapcnnlayer_indx} should have 2 or 3 modules but found {modules_count}.") + super().__init__( + f"A DynapcnnLayer {dynapcnnlayer_indx} should have 2 or 3 modules but found {modules_count}." + ) + class WrongPoolingModule(Exception): pooling_module: type - def __init__(self, pooling_module,): - super().__init__(f"The function 'utils.build_SumPool2d(mod)' expects 'mod = nn.AvgPool2d' but got 'mod = {pooling_module}'.") + def __init__( + self, + pooling_module, + ): + super().__init__( + f"The function 'utils.build_SumPool2d(mod)' expects 'mod = nn.AvgPool2d' but got 'mod = {pooling_module}'." + ) + class InvalidModel(Exception): model: type - def __init__(self, model,): - super().__init__(f"'model' accepts either a DynapcnnNetwork or a DynapcnnNetworkGraph but {model} was given.") + def __init__( + self, + model, + ): + super().__init__( + f"'model' accepts either a DynapcnnNetwork or a DynapcnnNetworkGraph but {model} was given." + ) + class InvalidTorchModel(Exception): network_type: str def __init__(self, network_type): - super().__init__(f"A {network_type} needs to be either of type nn.Sequential or nn.Module.") + super().__init__( + f"A {network_type} needs to be either of type nn.Sequential or nn.Module." + ) diff --git a/sinabs/backend/dynapcnn/graph_tracer.py b/sinabs/backend/dynapcnn/graph_tracer.py index ddc8fe01..153bec6d 100644 --- a/sinabs/backend/dynapcnn/graph_tracer.py +++ b/sinabs/backend/dynapcnn/graph_tracer.py @@ -1,15 +1,19 @@ -import torch -import torch.nn as nn -import re, copy -import numpy as np -import networkx as nx +import copy +import re from typing import Union import matplotlib.pyplot as plt +import networkx as nx +import numpy as np +import torch +import torch.nn as nn -class GraphTracer(): - def __init__(self, model: Union[nn.Sequential, nn.Module], dummy_input: np.array) -> None: - """ .""" + +class GraphTracer: + def __init__( + self, model: Union[nn.Sequential, nn.Module], dummy_input: np.array + ) -> None: + """.""" trace = torch.jit.trace(model, dummy_input) _ = trace(dummy_input) @@ -17,10 +21,10 @@ def __init__(self, model: Union[nn.Sequential, nn.Module], dummy_input: np.array self.graph = __.graph - self.modules_map, self.name_2_indx_map = self.get_named_modules(model) - self.forward_edges = self.get_foward_edges() - self.ATens = self.get_ATen_operations() - self.edges_list = self.get_graph_edges() + self.modules_map, self.name_2_indx_map = self.get_named_modules(model) + self.forward_edges = self.get_foward_edges() + self.ATens = self.get_ATen_operations() + self.edges_list = self.get_graph_edges() def from_name_2_indx(self, name): if name in self.name_2_indx_map: @@ -29,11 +33,11 @@ def from_name_2_indx(self, name): last_indx = None for _name, indx in self.name_2_indx_map.items(): last_indx = indx - self.name_2_indx_map[name] = last_indx+1 + self.name_2_indx_map[name] = last_indx + 1 return self.name_2_indx_map[name] def get_named_modules(self, module: nn.Module): - """ .""" + """.""" modules_map = {} name_2_indx_map = {} indx = 0 @@ -43,24 +47,29 @@ def get_named_modules(self, module: nn.Module): name_2_indx_map[name] = indx indx += 1 return modules_map, name_2_indx_map - + def get_foward_edges(self): - """ .""" + """.""" forward_edges = {} for node in self.graph.nodes(): node = str(node) - regex = re.compile(r'%(.*?) :.*prim::CallMethod\[name="forward"\]\(%(.*?), %(.*?)\)') + regex = re.compile( + r'%(.*?) :.*prim::CallMethod\[name="forward"\]\(%(.*?), %(.*?)\)' + ) match = regex.search(node) if match: - source = match.group(3).replace('_', '') - target = match.group(2).replace('_', '') - result = match.group(1).replace('_', '') - forward_edges[self.from_name_2_indx(result)] = (self.from_name_2_indx(source), self.from_name_2_indx(target)) - + source = match.group(3).replace("_", "") + target = match.group(2).replace("_", "") + result = match.group(1).replace("_", "") + forward_edges[self.from_name_2_indx(result)] = ( + self.from_name_2_indx(source), + self.from_name_2_indx(target), + ) + return forward_edges def get_graph_edges(self): - """ .""" + """.""" edges = [] last_result = None @@ -70,7 +79,7 @@ def get_graph_edges(self): if not last_result: last_result = result_node - edges.append(('input', trg)) + edges.append(("input", trg)) elif src == last_result: edges.append((edges[-1][1], trg)) last_result = result_node @@ -79,29 +88,30 @@ def get_graph_edges(self): edges.append((scr1, trg)) edges.append((scr2, trg)) last_result = result_node - - edges.append((edges[-1][1], 'output')) + + edges.append((edges[-1][1], "output")) return edges[1:-1] - + def get_ATen_operands(self, node): - """ .""" + """.""" if node in self.ATens: - src1 = self.ATens[node]['args'][1] - src2 = self.ATens[node]['args'][0] + src1 = self.ATens[node]["args"][1] + src2 = self.ATens[node]["args"][0] return self.forward_edges[src1][1], self.forward_edges[src2][1] else: # throw error return None, None - + def get_ATen_operations(self): - """ ATen is PyTorch's tensor library backend, which provides a set of operations that operate on - tensors directly. These include arithmetic operations (add, mul, etc.), mathematical - functions (sin, cos, etc.), and tensor manipulation operations (view, reshape, etc.).""" + """ATen is PyTorch's tensor library backend, which provides a set of operations that operate on + tensors directly. These include arithmetic operations (add, mul, etc.), mathematical + functions (sin, cos, etc.), and tensor manipulation operations (view, reshape, etc.). + """ ATens = {} for node in self.graph.nodes(): node = str(node) - regex = re.compile(r'%(.*?) :.*aten::(.*?)\(%(.*?), %(.*?), %(.*?)\)') + regex = re.compile(r"%(.*?) :.*aten::(.*?)\(%(.*?), %(.*?), %(.*?)\)") match = regex.search(node) @@ -111,11 +121,14 @@ def get_ATen_operations(self): operator1 = self.from_name_2_indx(match.group(3)) operator2 = self.from_name_2_indx(match.group(4)) const_operator = match.group(5) - ATens[result_node] = {'op': operation, 'args': (operator1, operator2, const_operator)} + ATens[result_node] = { + "op": operation, + "args": (operator1, operator2, const_operator), + } return ATens - + def remove_ignored_nodes(self, default_ignored_nodes): - """ Recreates the edges list based on layers that 'DynapcnnNetwork' will ignore. This + """Recreates the edges list based on layers that 'DynapcnnNetwork' will ignore. This is done by setting the source (target) node of an edge where the source (target) node will be dropped as the node that originally targeted this node to be dropped. """ @@ -142,7 +155,7 @@ def remove_ignored_nodes(self, default_ignored_nodes): new_edge = (_src, edge[1]) else: new_edge = (_src, _trg) - + if new_edge not in parsed_edges: parsed_edges.append(new_edge) @@ -153,7 +166,7 @@ def remove_ignored_nodes(self, default_ignored_nodes): for node_indx, __ in self.modules_map.items(): _ = [x for x in removed_nodes if node_indx > x] remapped_nodes[node_indx] = node_indx - len(_) - + for x in removed_nodes: del remapped_nodes[x] @@ -163,11 +176,11 @@ def remove_ignored_nodes(self, default_ignored_nodes): remapped_edges.append((remapped_nodes[edge[0]], remapped_nodes[edge[1]])) return remapped_edges - + @staticmethod def plot_graph(edges_list): - """ .""" + """.""" G = nx.DiGraph(edges_list) layout = nx.spring_layout(G) - nx.draw(G, pos = layout, with_labels=True, node_size=800) - plt.show() \ No newline at end of file + nx.draw(G, pos=layout, with_labels=True, node_size=800) + plt.show() diff --git a/sinabs/backend/dynapcnn/mapping.py b/sinabs/backend/dynapcnn/mapping.py index 964cf987..504c8ebc 100644 --- a/sinabs/backend/dynapcnn/mapping.py +++ b/sinabs/backend/dynapcnn/mapping.py @@ -3,10 +3,10 @@ from dataclasses import dataclass from typing import List, Optional, Tuple, Union +import sinabs + from .dvs_layer import DVSLayer from .dynapcnn_layer import DynapcnnLayer - -import sinabs from .exceptions import InvalidModel @@ -68,7 +68,9 @@ def get_valid_mapping( if isinstance(ith_dcnnl, DynapcnnLayer): layer_mapping.append(find_chip_layers(ith_dcnnl, constraints)) else: - raise ValueError(f'Layer {dcnnl_index} is not an instance of `DynapcnnLayer`.') + raise ValueError( + f"Layer {dcnnl_index} is not an instance of `DynapcnnLayer`." + ) graph = make_flow_graph(layer_mapping, len(constraints)) diff --git a/sinabs/backend/dynapcnn/weight_rescaling_methods.py b/sinabs/backend/dynapcnn/weight_rescaling_methods.py index f41a08c2..154f0055 100644 --- a/sinabs/backend/dynapcnn/weight_rescaling_methods.py +++ b/sinabs/backend/dynapcnn/weight_rescaling_methods.py @@ -1,12 +1,14 @@ # author : Willian Soares Girao # contact : williansoaresgirao@gmail.com -import numpy as np import statistics +import numpy as np + + def rescale_method_1(rescaling_from_sumpool: list, lambda_: float = 0.5) -> float: """ - This method will use the average (scaled by `lambda_`) of the computed re-scaling factor + This method will use the average (scaled by `lambda_`) of the computed re-scaling factor for the pooling layer(s) feeding into a convolutional layer. Arguments @@ -21,13 +23,14 @@ def rescale_method_1(rescaling_from_sumpool: list, lambda_: float = 0.5) -> floa """ if len(rescaling_from_sumpool): - return np.round(np.mean(rescaling_from_sumpool)*lambda_, 2) + return np.round(np.mean(rescaling_from_sumpool) * lambda_, 2) else: return 1.0 + def rescale_method_2(rescaling_from_sumpool: list, lambda_: float = 0.5) -> float: """ - This method will use the harmonic mean (scaled by `lambda_`) of the computed re-scaling factor + This method will use the harmonic mean (scaled by `lambda_`) of the computed re-scaling factor for the pooling layer(s) feeding into a convolutional layer. Arguments @@ -47,6 +50,6 @@ def rescale_method_2(rescaling_from_sumpool: list, lambda_: float = 0.5) -> floa """ if len(rescaling_from_sumpool): - return np.round(statistics.harmonic_mean(rescaling_from_sumpool)*lambda_, 2) + return np.round(statistics.harmonic_mean(rescaling_from_sumpool) * lambda_, 2) else: - return 1.0 \ No newline at end of file + return 1.0 From 3b78e6940cb7c12600ff3d0b873d632fba434004 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Wed, 2 Oct 2024 10:16:55 +0200 Subject: [PATCH 153/379] Graph extractor removes nodes-in place --- sinabs/backend/dynapcnn/dynapcnn_network.py | 13 +---- .../backend/dynapcnn/nir_graph_extractor.py | 55 ++++++++++++++----- 2 files changed, 45 insertions(+), 23 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index bb9b485d..7f7bc9f4 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -85,6 +85,9 @@ def __init__( snn, torch.randn((batch_size, *self.input_shape)) ) # needs the batch dimension. + # remap `(A, X)` and `(X, B)` into `(A, B)` if `X` is a layer in the original `snn` to be ignored. + self._graph_tracer.remove_ignored_nodes(DEFAULT_IGNORED_LAYER_TYPES) + # get list of nodes from graph tracer that act as entry points to the network. self._entry_nodes = copy.deepcopy(self._graph_tracer.entry_nodes) @@ -612,16 +615,6 @@ def _get_sinabs_edges_and_modules( the new node name (after ignored layers have been dropped and `Merge` layers have be processed before being removed). """ - # remap `(A, X)` and `(X, B)` into `(A, B)` if `X` is a layer in the original `snn` to be ignored. - sinabs_edges, remapped_nodes = self._graph_tracer.remove_ignored_nodes( - DEFAULT_IGNORED_LAYER_TYPES - ) - - # nodes (layers' "names") need remapping in case some layers have been removed (e.g. a `nn.Flattern` is ignored). - sinabs_modules_map = {} - for orig_name, new_name in remapped_nodes.items(): - sinabs_modules_map[new_name] = self._graph_tracer.modules_map[orig_name] - # bypass merging layers to connect the nodes involved in them directly to the node where the merge happens. edges_without_merge = merge_handler(sinabs_edges, sinabs_modules_map) diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index 0977fc01..ded9e3a1 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -51,7 +51,7 @@ def __init__(self, spiking_model: nn.Module, dummy_input: torch.tensor): ) # recovers the associated `nn.Module` (layer) of each node. - self.modules_map = self._get_named_modules(spiking_model) + self._modules_map = self._get_named_modules(spiking_model) # retrieves what the I/O shape for each node's module is. self._nodes_io_shapes = self._get_nodes_io_shapes(dummy_input) @@ -78,22 +78,27 @@ def nodes_io_shapes(self) -> Dict[int, torch.Size]: def sorted_nodes(self) -> List[int]: return [n for n in self._sort_graph_nodes()] + @property + def modules_map(self) -> Dict[int, nn.Module]: + return {n: module for n, module in self._modules_map.items()} + def remove_ignored_nodes( self, ignored_node_classes: Tuple[Type] ) -> Tuple[Set[int], Dict[int, int]]: - """Create a new set of edges, considering layers that `DynapcnnNetwork` will ignore. This + """Remove nodes of given classes from graph in place. + + Create a new set of edges, considering layers that `DynapcnnNetwork` will ignore. This is done by setting the source (target) node of an edge where the source (target) node will be dropped as the node that originally targeted this node to be dropped. + Will change internal attributes `self._edges`, `self._entry_nodes`, + `self._name_2_indx_map`, and `self._nodes_io_shapes` to reflect the changes. + Parameters ---------- - ignored_node_classes (tuple of types): - Layer classes that should be ignored from the graph. + Layer classes that should be removed from the graph. - Returns - ---------- - - new_edges (set): the new set of edges after nodes flagged by `ignored_node_classes` have been removed. - - remapped_nodes (dict): updated nodes' IDs after nodes flagged by `ignored_node_classes` have been removed. """ # Compose new graph by creating a dict with all remaining node IDs as keys and set of target node IDs as values source2target: Dict[int, Set[int]] = { @@ -110,13 +115,39 @@ def remove_ignored_nodes( } # Parse new set of edges based on remapped node IDs - new_edges = { + self._edges = { (remapped_nodes[src], remapped_nodes[tgt]) for src, targets in source2target.items() for tgt in targets } - return new_edges, remapped_nodes + # Update name-to-index map based on new node indices + self._name_2_indx_map = { + name: remapped_nodes[old_idx] + for name, old_idx in self._name_2_indx_map.items() + if old_idx in remapped_nodes + } + + # Update entry nodes based on new node indices + self._entry_nodes = { + remapped_nodes[old_idx] + for old_idx in self._entry_nodes + if old_idx in remapped_nodes + } + + # Update io-shapes based on new node indices + self._nodes_io_shapes = { + remapped_nodes[old_idx]: shape + for old_idx, shape in self._nodes_io_shapes.items() + if old_idx in remapped_nodes + } + + # Update sinabs module map based on new node indices + self._nodes_io_shapes = { + remapped_nodes[old_idx]: shape + for old_idx, shape in self._nodes_io_shapes.items() + if old_idx in remapped_nodes + } def get_node_io_shapes(self, node: int) -> Tuple[torch.Size, torch.Size]: """Returns the I/O tensors' shapes of `node`. @@ -136,8 +167,7 @@ def get_node_io_shapes(self, node: int) -> Tuple[torch.Size, torch.Size]: def _get_edges_from_nir( self, nir_graph: nirtorch.graph.Graph ) -> Tuple[List[Tuple[int, int]], Dict[str, int], List[int]]: - """Standardize the representation of `nirtorch.graph.Graph` into a list of edges (`Tuple[int, int]`) where - each node in `nir_graph` is represented by an interger (with the source node starting as `0`). + """Standardize the representation of `nirtorch.graph.Graph` into a list of edges (`Tuple[int, int]`) where each node in `nir_graph` is represented by an interger (with the source node starting as `0`). Parameters ---------- @@ -146,8 +176,7 @@ def _get_edges_from_nir( Returns ---------- - edges (set): tuples describing the connections between layers in `spiking_model`. - - name_2_indx_map (dict): `key` is the original variable name for a layer in `spiking_model` and `value - is an integer representing the layer in a standard format. + - name_2_indx_map (dict): `key` is the original variable name for a layer in `spiking_model` and `value is an integer representing the layer in a standard format. - entry_nodes (set): IDs of nodes acting as entry points for the network (i.e., receiving external input). """ edges = set() From 8e3c7cd9c4d5383bb1cc8378ccdb5f89cc637e8d Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Wed, 2 Oct 2024 13:32:55 +0200 Subject: [PATCH 154/379] Fix indentation of DynapcnnCompatibleNetwork --- sinabs/backend/dynapcnn/dynapcnn_network.py | 35 +++++++++++---------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index 7f7bc9f4..c6d21265 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -720,21 +720,22 @@ def __str__(self): pretty_print += f"{layer_data}\n\n" return pretty_print + + +class DynapcnnCompatibleNetwork(DynapcnnNetwork): + """Deprecated class, use DynapcnnNetwork instead.""" - class DynapcnnCompatibleNetwork(DynapcnnNetwork): - """Deprecated class, use DynapcnnNetwork instead.""" - - def __init__( - self, - snn: Union[nn.Sequential, sinabs.Network], - input_shape: Optional[Tuple[int, int, int]] = None, - dvs_input: bool = False, - discretize: bool = True, - ): - from warnings import warn - - warn( - "DynapcnnCompatibleNetwork has been renamed to DynapcnnNetwork " - + "and will be removed in a future release." - ) - super().__init__(snn, input_shape, dvs_input, discretize) + def __init__( + self, + snn: Union[nn.Sequential, sinabs.Network], + input_shape: Optional[Tuple[int, int, int]] = None, + dvs_input: bool = False, + discretize: bool = True, + ): + from warnings import warn + + warn( + "DynapcnnCompatibleNetwork has been renamed to DynapcnnNetwork " + + "and will be removed in a future release." + ) + super().__init__(snn, input_shape, dvs_input, discretize) From 32bbfed38362dd0128a66e77ed2b2a6e7efbd9f9 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Wed, 2 Oct 2024 16:55:43 +0200 Subject: [PATCH 155/379] Remove dependency on DynapcnnNetwork for Dynapcnn Config builder to prevent circular import --- sinabs/backend/dynapcnn/chips/dynapcnn.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/sinabs/backend/dynapcnn/chips/dynapcnn.py b/sinabs/backend/dynapcnn/chips/dynapcnn.py index 2e1869b2..8ae9eb83 100644 --- a/sinabs/backend/dynapcnn/chips/dynapcnn.py +++ b/sinabs/backend/dynapcnn/chips/dynapcnn.py @@ -10,7 +10,7 @@ from sinabs.backend.dynapcnn.config_builder import ConfigBuilder from sinabs.backend.dynapcnn.dvs_layer import DVSLayer, expand_to_pair from sinabs.backend.dynapcnn.dynapcnn_layer import DynapcnnLayer -from sinabs.backend.dynapcnn.dynapcnn_network import DynapcnnNetwork +from sinabs.backend.dynapcnn.dynapcnn_network import "DynapcnnNetwork" from sinabs.backend.dynapcnn.dynapcnn_layer_handler import DynapcnnLayerHandler from sinabs.backend.dynapcnn.mapping import LayerConstraints @@ -210,7 +210,7 @@ def write_dynapcnn_layer_config(cls, layer: DynapcnnLayer, chip_layer: "CNNLayer raise TypeError(f"Unexpected parameter {param} or value. {e}") @classmethod - def build_config(cls, model: DynapcnnNetwork) -> DynapcnnConfiguration: + def build_config(cls, model: "DynapcnnNetwork") -> DynapcnnConfiguration: """ Uses `DynapcnnLayer` objects to configure their equivalent chip core via a `CNNLayerConfig` object that is built using using the `DynapcnnLayer` properties. @@ -224,9 +224,6 @@ def build_config(cls, model: DynapcnnNetwork) -> DynapcnnConfiguration: """ config = cls.get_default_config() - if not isinstance(model, DynapcnnNetwork): - raise ValueError(f"`model` has to be of type DynapcnnNetwork, but is {type(model)}.") - has_dvs_layer = False # TODO DVSLayer not supported yet. # Loop over layers in network and write corresponding configurations From 4ab84920f79dc46872befb9da569cedf4f685418 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Wed, 2 Oct 2024 17:07:29 +0200 Subject: [PATCH 156/379] Fix multiple minor bugs causing graph extractor test to fail --- sinabs/backend/dynapcnn/chips/dynapcnn.py | 1 - sinabs/backend/dynapcnn/dynapcnn_network.py | 2 +- sinabs/backend/dynapcnn/nir_graph_extractor.py | 4 ++-- sinabs/backend/dynapcnn/sinabs_edges_utils.py | 2 +- sinabs/backend/dynapcnn/utils.py | 2 +- tests/test_graph_extractor/conftest_graph_extractor.py | 2 +- tests/test_graph_extractor/model_dummy_1.py | 8 ++++---- tests/test_graph_extractor/model_dummy_2.py | 8 ++++---- tests/test_graph_extractor/model_dummy_3.py | 8 ++++---- tests/test_graph_extractor/model_dummy_4.py | 8 ++++---- tests/test_graph_extractor/test_graph_extractor.py | 6 +++--- 11 files changed, 25 insertions(+), 26 deletions(-) diff --git a/sinabs/backend/dynapcnn/chips/dynapcnn.py b/sinabs/backend/dynapcnn/chips/dynapcnn.py index 8ae9eb83..9269bd57 100644 --- a/sinabs/backend/dynapcnn/chips/dynapcnn.py +++ b/sinabs/backend/dynapcnn/chips/dynapcnn.py @@ -10,7 +10,6 @@ from sinabs.backend.dynapcnn.config_builder import ConfigBuilder from sinabs.backend.dynapcnn.dvs_layer import DVSLayer, expand_to_pair from sinabs.backend.dynapcnn.dynapcnn_layer import DynapcnnLayer -from sinabs.backend.dynapcnn.dynapcnn_network import "DynapcnnNetwork" from sinabs.backend.dynapcnn.dynapcnn_layer_handler import DynapcnnLayerHandler from sinabs.backend.dynapcnn.mapping import LayerConstraints diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index c6d21265..50faff6e 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -17,7 +17,7 @@ from .dynapcnn_layer import DynapcnnLayer from .dynapcnnnetwork_module import DynapcnnNetworkModule from .io import disable_timestamps, enable_timestamps, open_device, reset_timestamps -from .NIRGraphExtractor import NIRtoDynapcnnNetworkGraph +from .nir_graph_extractor import NIRtoDynapcnnNetworkGraph from .sinabs_edges_handler import merge_handler from .utils import ( DEFAULT_IGNORED_LAYER_TYPES, diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index ded9e3a1..15142b65 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -2,7 +2,7 @@ # contact : wsoaresgirao@gmail.com import copy -from typing import Dict, List, Tuple, Type +from typing import Dict, List, Tuple, Type, Set import nirtorch import torch @@ -354,7 +354,7 @@ def _find_source_of_input_to(self, node: int) -> int: raise RuntimeError(f"Node {node} has more than 1 input") return sources.pop() - def _find_merge_arguments(self, merge_node: int) -> Tuple[int, int]: + def _find_merge_arguments(self, node: int) -> Tuple[int, int]: """A `Merge` layer receives two inputs. Return the two inputs to `merge_node` representing a `Merge` layer. Returns diff --git a/sinabs/backend/dynapcnn/sinabs_edges_utils.py b/sinabs/backend/dynapcnn/sinabs_edges_utils.py index 04a61260..76e8ddef 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_utils.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_utils.py @@ -43,7 +43,7 @@ 8: (sl.SumPool2d, nn.Conv2d), # same as key `4` but with `sl.SumPool2d` instead. 9: (sl.SumPool2d, nn.Linear), # same as key `5` but with `sl.SumPool2d` instead. } -VALID_SINABS_EDGE_TYPE_IDS = {v: k for k, v in VALID_SINABS_EDGES} +VALID_SINABS_EDGE_TYPE_IDS = {v: k for k, v in VALID_SINABS_EDGES.items()} VALID_DYNAPCNNLAYER_EDGES = [ (sl.iaf.IAFSqueeze, nn.Conv2d), diff --git a/sinabs/backend/dynapcnn/utils.py b/sinabs/backend/dynapcnn/utils.py index 9a5a227d..88d4dfae 100644 --- a/sinabs/backend/dynapcnn/utils.py +++ b/sinabs/backend/dynapcnn/utils.py @@ -1,6 +1,6 @@ from collections import defaultdict, deque from copy import deepcopy -from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Tuple, Union +from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Tuple, Union, Set import torch import torch.nn as nn diff --git a/tests/test_graph_extractor/conftest_graph_extractor.py b/tests/test_graph_extractor/conftest_graph_extractor.py index c531d6ce..7c655c66 100644 --- a/tests/test_graph_extractor/conftest_graph_extractor.py +++ b/tests/test_graph_extractor/conftest_graph_extractor.py @@ -11,4 +11,4 @@ (snn_2, input_dummy_2, expected_output_2), (snn_3, input_dummy_3, expected_output_3), (snn_4, input_dummy_4, expected_output_4), -] \ No newline at end of file +] diff --git a/tests/test_graph_extractor/model_dummy_1.py b/tests/test_graph_extractor/model_dummy_1.py index 19f64a86..cf2b9e7d 100644 --- a/tests/test_graph_extractor/model_dummy_1.py +++ b/tests/test_graph_extractor/model_dummy_1.py @@ -64,7 +64,7 @@ def forward(self, x): input_dummy = torch.randn(input_shape) expected_output = { - 'edges_list': [ + 'edges': { (0, 1), (1, 2), (1, 3), @@ -79,7 +79,7 @@ def forward(self, x): (11, 12), (12, 13), (5, 7), - ], + }, 'name_2_indx_map': { 'conv1': 0, 'iaf1': 1, @@ -96,7 +96,7 @@ def forward(self, x): 'fc2': 12, 'iaf5': 13, }, - 'entry_nodes': [0], + 'entry_nodes': {0}, 'nodes_io_shapes': { 0: {'input': torch.Size([3, 2, 34, 34]), 'output': torch.Size([3, 10, 33, 33])}, 1: {'input': torch.Size([3, 10, 33, 33]), 'output': torch.Size([3, 10, 33, 33])}, @@ -115,4 +115,4 @@ def forward(self, x): }, } -snn = SNN(batch_size) \ No newline at end of file +snn = SNN(batch_size) diff --git a/tests/test_graph_extractor/model_dummy_2.py b/tests/test_graph_extractor/model_dummy_2.py index 8d657b5d..52281b34 100644 --- a/tests/test_graph_extractor/model_dummy_2.py +++ b/tests/test_graph_extractor/model_dummy_2.py @@ -90,7 +90,7 @@ def forward(self, x): input_dummy = torch.randn(input_shape) expected_output = { - 'edges_list': [ + 'edges': { (0, 1), (1, 2), (2, 3), @@ -111,7 +111,7 @@ def forward(self, x): (19, 17), (11, 19), (16, 19), - ], + }, 'name_2_indx_map': { 'conv_A': 0, 'iaf_A': 1, @@ -134,7 +134,7 @@ def forward(self, x): 'iaf3_fc': 18, 'merge1': 19, }, - 'entry_nodes': [0], + 'entry_nodes': {0}, 'nodes_io_shapes': { 0: {'input': torch.Size([8, 2, 34, 34]), 'output': torch.Size([8, 4, 33, 33])}, 1: {'input': torch.Size([8, 4, 33, 33]), 'output': torch.Size([8, 4, 33, 33])}, @@ -159,4 +159,4 @@ def forward(self, x): }, } -snn = SNN(batch_size) \ No newline at end of file +snn = SNN(batch_size) diff --git a/tests/test_graph_extractor/model_dummy_3.py b/tests/test_graph_extractor/model_dummy_3.py index 39bd6e39..6fc8242d 100644 --- a/tests/test_graph_extractor/model_dummy_3.py +++ b/tests/test_graph_extractor/model_dummy_3.py @@ -102,7 +102,7 @@ def forward(self, x): input_dummy = torch.randn(input_shape) expected_output = { - 'edges_list': [ + 'edges': { (0, 1), (1, 2), (2, 3), @@ -127,7 +127,7 @@ def forward(self, x): (21, 22), (22, 23), (23, 24), - ], + }, 'name_2_indx_map': { 'conv_A': 0, 'iaf_A': 1, @@ -155,7 +155,7 @@ def forward(self, x): 'fc3': 23, 'iaf3_fc': 24, }, - 'entry_nodes': [0, 9], + 'entry_nodes': {0, 9}, 'nodes_io_shapes': { 0: {'input': torch.Size([2, 2, 34, 34]), 'output': torch.Size([2, 4, 33, 33])}, 9: {'input': torch.Size([2, 2, 34, 34]), 'output': torch.Size([2, 4, 33, 33])}, @@ -185,4 +185,4 @@ def forward(self, x): }, } -snn = SNN(batch_size) \ No newline at end of file +snn = SNN(batch_size) diff --git a/tests/test_graph_extractor/model_dummy_4.py b/tests/test_graph_extractor/model_dummy_4.py index 0acad595..58917e19 100644 --- a/tests/test_graph_extractor/model_dummy_4.py +++ b/tests/test_graph_extractor/model_dummy_4.py @@ -86,7 +86,7 @@ def forward(self, x): input_dummy = torch.randn(input_shape) expected_output = { - 'edges_list': [ + 'edges': { (0, 1), (1, 2), (1, 3), @@ -108,7 +108,7 @@ def forward(self, x): (18, 19), (6, 11), (15, 18), - ], + }, 'name_2_indx_map': { 'conv1': 0, 'iaf1': 1, @@ -131,7 +131,7 @@ def forward(self, x): 'fc2': 18, 'iaf2_fc': 19, }, - 'entry_nodes': [0], + 'entry_nodes': {0}, 'nodes_io_shapes': { 0: {'input': torch.Size([2, 2, 34, 34]), 'output': torch.Size([2, 1, 33, 33])}, 1: {'input': torch.Size([2, 1, 33, 33]), 'output': torch.Size([2, 1, 33, 33])}, @@ -156,4 +156,4 @@ def forward(self, x): }, } -snn = SNN(batch_size) \ No newline at end of file +snn = SNN(batch_size) diff --git a/tests/test_graph_extractor/test_graph_extractor.py b/tests/test_graph_extractor/test_graph_extractor.py index a7693a69..f21cb432 100644 --- a/tests/test_graph_extractor/test_graph_extractor.py +++ b/tests/test_graph_extractor/test_graph_extractor.py @@ -2,7 +2,7 @@ # contact : wsoaresgirao@gmail.com import pytest -from sinabs.backend.dynapcnn.NIRGraphExtractor import NIRtoDynapcnnNetworkGraph +from sinabs.backend.dynapcnn.nir_graph_extractor import NIRtoDynapcnnNetworkGraph from conftest_graph_extractor import args_GraphExtractor @@ -15,11 +15,11 @@ def test_GraphExtractor(snn, input_dummy, expected_output): graph_tracer = NIRtoDynapcnnNetworkGraph(snn, input_dummy) - assert expected_output['edges_list'] == graph_tracer.get_edges_list, \ + assert expected_output['edges'] == graph_tracer.edges, \ f'wrong list of edges extracted from the SNN.' assert expected_output['name_2_indx_map'] == graph_tracer.name_2_indx_map, \ f'wrong mapping from layer variable name to node ID.' assert expected_output['entry_nodes'] == graph_tracer.entry_nodes, \ f'wrong list with entry node\'s IDs (i.e., layers serving as input to the SNN).' assert expected_output['nodes_io_shapes'] == graph_tracer.nodes_io_shapes, \ - f'wrong I/O shapes computed for one or more nodes.' \ No newline at end of file + f'wrong I/O shapes computed for one or more nodes.' From 8c28e610c7e8c51c24124dd322669df024695783 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Wed, 2 Oct 2024 17:24:28 +0200 Subject: [PATCH 157/379] DynapcnnNetwork does not need to keep track of removed nodes anymore --- sinabs/backend/dynapcnn/dynapcnn_network.py | 43 ++++----------------- 1 file changed, 7 insertions(+), 36 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index 7f7bc9f4..b339156f 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -65,7 +65,6 @@ def __init__( - self._graph_tracer - self._sinabs_edges - self._sinabs_modules_map - - self._nodes_name_remap - self._nodes_to_dcnnl_map - self._dynapcnn_layers - self._entry_nodes @@ -92,7 +91,7 @@ def __init__( self._entry_nodes = copy.deepcopy(self._graph_tracer.entry_nodes) # pre-process and group original nodes/edges from the graph tracer into data structures used to later create `DynapcnnLayer` instance. - self._sinabs_edges, self._sinabs_modules_map, self._nodes_name_remap = ( + self._sinabs_edges, self._sinabs_modules_map = ( self._get_sinabs_edges_and_modules() ) @@ -125,7 +124,6 @@ def __init__( del self._graph_tracer del self._sinabs_edges del self._sinabs_modules_map - del self._nodes_name_remap del self._nodes_to_dcnnl_map del self._dynapcnn_layers del self._entry_nodes @@ -611,39 +609,19 @@ def _get_sinabs_edges_and_modules( remapped to connect the nodes involved in the merging directly. - sinabs_modules_map (dict): a dict containing the nodes of the graph (described now by `edges_without_merge`) as `key` and their associated module as `value`. - - remapped_nodes (dict): a dict where `key` is the original node name (as extracted by `self._graph_tracer`) and `value` is - the new node name (after ignored layers have been dropped and `Merge` layers have be processed before being removed). """ # bypass merging layers to connect the nodes involved in them directly to the node where the merge happens. - edges_without_merge = merge_handler(sinabs_edges, sinabs_modules_map) - - # original nodes flagged as input nodes (i.e., entry points of the network) might have been renamed: update list flaggind them. - temp = [] - for i in range(len(self._entry_nodes)): - temp.append(remapped_nodes[self._entry_nodes[i]]) - self._entry_nodes = temp + edges_without_merge = merge_handler( + self._graph_tracer.edges, self._graph_tracer.modules_map + ) - return edges_without_merge, sinabs_modules_map, remapped_nodes + return edges_without_merge, self._graph_tracer.modules_map def _populate_nodes_io(self): """Loops through the nodes in the original graph to retrieve their I/O tensor shapes and add them to their respective representations in `self._nodes_to_dcnnl_map`.""" - def find_original_node_name(name_mapper: dict, node: int) -> str: - """Find what a node is originally named when built in `self._graph_tracer`. - - Returns - ---------- - - orig_name (str): a string with the original variable name given to `node`. - """ - for orig_name, new_name in name_mapper.items(): - if new_name == node: - return orig_name - raise ValueError( - f"Node {node} could not be found within the name remapping done by self._get_sinabs_edges_and_modules()." - ) - def find_my_input(edges_list: list, node: int) -> int: """Returns the node `X` in the first edge `(X, node)`. @@ -671,9 +649,7 @@ def find_my_input(edges_list: list, node: int) -> int: for node, node_data in dcnnl_data.items(): # node dictionary with layer data. if isinstance(node, int): - # some nodes might have been renamed (e.g. after droppping a `nn.Flatten`), so find how node was originally named. - orig_name = find_original_node_name(self._nodes_name_remap, node) - _in, _out = self._graph_tracer.get_node_io_shapes(orig_name) + _in, _out = self._graph_tracer.get_node_io_shapes(node) # update node I/O shape in the mapper (drop batch dimension). if node != 0: @@ -687,13 +663,8 @@ def find_my_input(edges_list: list, node: int) -> int: node_data["input_shape"] = tuple(list(_in)[1:]) else: # input comes from another node in the graph. - input_node_orig_name = find_original_node_name( - self._nodes_name_remap, input_node - ) _, _input_source_shape = ( - self._graph_tracer.get_node_io_shapes( - input_node_orig_name - ) + self._graph_tracer.get_node_io_shapes(input_node) ) node_data["input_shape"] = tuple( list(_input_source_shape)[1:] From c48687e4dda72baf634c031ae4af6f052f7ce672 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Wed, 2 Oct 2024 17:27:51 +0200 Subject: [PATCH 158/379] Fix set merger in GraphExtractor --- sinabs/backend/dynapcnn/nir_graph_extractor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index 15142b65..da1c4d78 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -394,7 +394,7 @@ def _find_valid_targets( if src == node: if isinstance(self.modules_map[tgt], ignored_node_classes): # Find valid targets of target - targets.join(self._find_valid_targets(tgt, ignored_node_classes)) + targets.update(self._find_valid_targets(tgt, ignored_node_classes)) else: # Target is valid, add it to `targets` targets.add(tgt) From 83be978495a67a54266fb0952d4aafd69520fb1c Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Wed, 2 Oct 2024 17:32:34 +0200 Subject: [PATCH 159/379] DynapcnnNetwork does not need to copy entry nodes from tracer --- sinabs/backend/dynapcnn/dynapcnn_network.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index f6f0a229..fb762a03 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -67,7 +67,6 @@ def __init__( - self._sinabs_modules_map - self._nodes_to_dcnnl_map - self._dynapcnn_layers - - self._entry_nodes """ super().__init__() @@ -87,9 +86,6 @@ def __init__( # remap `(A, X)` and `(X, B)` into `(A, B)` if `X` is a layer in the original `snn` to be ignored. self._graph_tracer.remove_ignored_nodes(DEFAULT_IGNORED_LAYER_TYPES) - # get list of nodes from graph tracer that act as entry points to the network. - self._entry_nodes = copy.deepcopy(self._graph_tracer.entry_nodes) - # pre-process and group original nodes/edges from the graph tracer into data structures used to later create `DynapcnnLayer` instance. self._sinabs_edges, self._sinabs_modules_map = ( self._get_sinabs_edges_and_modules() @@ -109,7 +105,7 @@ def __init__( edges=self._sinabs_edges, nodes_to_dcnnl_map=self._nodes_to_dcnnl_map, weight_rescaling_fn=weight_rescaling_fn, - entry_nodes=self._entry_nodes, + entry_nodes=self._graph_tracer._entry_nodes, ) # these gather all data necessay to implement the forward method for this class. @@ -126,7 +122,6 @@ def __init__( del self._sinabs_modules_map del self._nodes_to_dcnnl_map del self._dynapcnn_layers - del self._entry_nodes ####################################################### Public Methods ####################################################### From 035961097cb55a20f3b80c8c0249641a1cef735a Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Wed, 2 Oct 2024 17:47:23 +0200 Subject: [PATCH 160/379] Properly update modules map when removing nodes from graph extractor --- sinabs/backend/dynapcnn/nir_graph_extractor.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index da1c4d78..7d79abb7 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -143,9 +143,9 @@ def remove_ignored_nodes( } # Update sinabs module map based on new node indices - self._nodes_io_shapes = { - remapped_nodes[old_idx]: shape - for old_idx, shape in self._nodes_io_shapes.items() + self._modules_map = { + remapped_nodes[old_idx]: module + for old_idx, module in self._modules_map.items() if old_idx in remapped_nodes } From 1b02d23397d8c4bb9eeebc4b01ee361bb2cf0de0 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Wed, 2 Oct 2024 17:50:18 +0200 Subject: [PATCH 161/379] Nir Graph Extractor: Rename remove-nodes method --- sinabs/backend/dynapcnn/dynapcnn_network.py | 2 +- sinabs/backend/dynapcnn/nir_graph_extractor.py | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index fb762a03..4d9bd169 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -84,7 +84,7 @@ def __init__( ) # needs the batch dimension. # remap `(A, X)` and `(X, B)` into `(A, B)` if `X` is a layer in the original `snn` to be ignored. - self._graph_tracer.remove_ignored_nodes(DEFAULT_IGNORED_LAYER_TYPES) + self._graph_tracer.remove_nodes_by_class(DEFAULT_IGNORED_LAYER_TYPES) # pre-process and group original nodes/edges from the graph tracer into data structures used to later create `DynapcnnLayer` instance. self._sinabs_edges, self._sinabs_modules_map = ( diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index 7d79abb7..f4b87bfa 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -82,8 +82,8 @@ def sorted_nodes(self) -> List[int]: def modules_map(self) -> Dict[int, nn.Module]: return {n: module for n, module in self._modules_map.items()} - def remove_ignored_nodes( - self, ignored_node_classes: Tuple[Type] + def remove_nodes_by_class( + self, node_classes: Tuple[Type] ) -> Tuple[Set[int], Dict[int, int]]: """Remove nodes of given classes from graph in place. @@ -96,16 +96,16 @@ def remove_ignored_nodes( Parameters ---------- - - ignored_node_classes (tuple of types): + - node_classes (tuple of types): Layer classes that should be removed from the graph. """ # Compose new graph by creating a dict with all remaining node IDs as keys and set of target node IDs as values source2target: Dict[int, Set[int]] = { - node: self._find_valid_targets(node, ignored_node_classes) + node: self._find_valid_targets(node, node_classes) for node in self.sorted_nodes - # Skip ignored nodes - if not isinstance(self.modules_map[node], ignored_node_classes) + # Skip nodes that are to be removed + if not isinstance(self.modules_map[node], node_classes) } # remapping nodes indices contiguously starting from 0 From 6c59b3a21a1948e8d3717a7ca87a3d29a9a289db Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Wed, 2 Oct 2024 17:58:32 +0200 Subject: [PATCH 162/379] Fix dynapcnnlayer test to use layer handler --- .../test_dynapcnnlayer/test_dynapcnnlayer.py | 36 +++++++++++-------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/tests/test_dynapcnnlayer/test_dynapcnnlayer.py b/tests/test_dynapcnnlayer/test_dynapcnnlayer.py index 43cdba70..fd0bca3e 100644 --- a/tests/test_dynapcnnlayer/test_dynapcnnlayer.py +++ b/tests/test_dynapcnnlayer/test_dynapcnnlayer.py @@ -2,7 +2,7 @@ # contact : wsoaresgirao@gmail.com import pytest -from sinabs.backend.dynapcnn.utils import construct_dynapcnnlayer, update_nodes_io +from sinabs.backend.dynapcnn.utils import construct_dynapcnnlayer, construct_layerhandler, update_nodes_io from sinabs.backend.dynapcnn.weight_rescaling_methods import rescale_method_1 from conftest_dynapcnnlayer import args_DynapcnnLayer @@ -15,10 +15,18 @@ def test_DynapcnnLayer(nodes_to_dcnnl_map, dpcnnl_idx, sinabs_edges, entry_point """ # create a `DynapcnnLayer` from the set of layers in `nodes_to_dcnnl_map[dpcnnl_idx]`. - dynapcnnlayer = construct_dynapcnnlayer(dpcnnl_idx, True, sinabs_edges, nodes_to_dcnnl_map, rescale_method_1, entry_point) + layerhandler = construct_layerhandler( + dpcnnl_idx, + True, + sinabs_edges, + nodes_to_dcnnl_map, + rescale_method_1, + entry_point, + ) + dynapcnnlayer = construct_dynapcnnlayer(layerhandler) # check if any node (layer) in `dynapcnnlayer` has been modified (e.g. `nn.Linear` turned `nn.Conv2d`). - node, output_shape = dynapcnnlayer.get_modified_node_io(nodes_to_dcnnl_map[dpcnnl_idx]) + node, output_shape = layerhandler.get_modified_node_io(nodes_to_dcnnl_map[dpcnnl_idx]) # one of the layers in `dynapcnnlayer` had its type modified (update input shape of nodes receiving from it). if isinstance(node, int) and isinstance(output_shape, tuple): @@ -35,23 +43,23 @@ def test_DynapcnnLayer(nodes_to_dcnnl_map, dpcnnl_idx, sinabs_edges, entry_point nodes_destinations = expected_output[dpcnnl_idx]['nodes_destinations'] entry_point = expected_output[dpcnnl_idx]['entry_point'] - assert dynapcnnlayer.dpcnnl_index == expected_output[dpcnnl_idx]['dpcnnl_index'], \ + assert layerhandler.dpcnnl_index == expected_output[dpcnnl_idx]['dpcnnl_index'], \ f'wrong \'DynapcnnLayer.dpcnnl_index\': ID of the instance should be {dpcnnl_index}.' - assert dynapcnnlayer.conv_node_id == expected_output[dpcnnl_idx]['conv_node_id'], \ + assert layerhandler.conv_node_id == expected_output[dpcnnl_idx]['conv_node_id'], \ f'wrong \'DynapcnnLayer.conv_node_id\': convolution layer should be node {conv_node_id}.' - assert dynapcnnlayer.conv_in_shape == expected_output[dpcnnl_idx]['conv_in_shape'], \ + assert layerhandler.conv_in_shape == expected_output[dpcnnl_idx]['conv_in_shape'], \ f'wrong \'DynapcnnLayer.conv_in_shape\': input tensor shape of convolution should be {conv_in_shape}.' - assert dynapcnnlayer.conv_out_shape == expected_output[dpcnnl_idx]['conv_out_shape'], \ + assert layerhandler.conv_out_shape == expected_output[dpcnnl_idx]['conv_out_shape'], \ f'wrong \'DynapcnnLayer.conv_out_shape\': output tensor shape of convolution should be {conv_out_shape}.' - assert dynapcnnlayer.spk_node_id == expected_output[dpcnnl_idx]['spk_node_id'], \ + assert layerhandler.spk_node_id == expected_output[dpcnnl_idx]['spk_node_id'], \ f'wrong \'DynapcnnLayer.spk_node_id\': spiking layer should be node {spk_node_id}.' - assert dynapcnnlayer.pool_node_id == expected_output[dpcnnl_idx]['pool_node_id'], \ + assert layerhandler.pool_node_id == expected_output[dpcnnl_idx]['pool_node_id'], \ f'wrong \'DynapcnnLayer.pool_node_id\': pooling layer node(s) should be {pool_node_id}.' - assert dynapcnnlayer.conv_rescaling_factor == expected_output[dpcnnl_idx]['conv_rescaling_factor'], \ + assert layerhandler.conv_rescaling_factor == expected_output[dpcnnl_idx]['conv_rescaling_factor'], \ f'wrong \'DynapcnnLayer.conv_rescaling_factor\': computed re-scaling factor should be {conv_rescaling_factor}.' - assert dynapcnnlayer.dynapcnnlayer_destination == expected_output[dpcnnl_idx]['dynapcnnlayer_destination'], \ + assert layerhandler.dynapcnnlayer_destination == expected_output[dpcnnl_idx]['dynapcnnlayer_destination'], \ f'wrong \'DynapcnnLayer.dynapcnnlayer_destination\': the DynapcnnLayer(s) set as destination(s) should be {dynapcnnlayer_destination}.' - assert dynapcnnlayer.nodes_destinations == expected_output[dpcnnl_idx]['nodes_destinations'], \ + assert layerhandler.nodes_destinations == expected_output[dpcnnl_idx]['nodes_destinations'], \ f'wrong \'DynapcnnLayer.nodes_destinations\': the targeted nodes within other DynapcnnLayer instance(s) should be {nodes_destinations}.' - assert dynapcnnlayer.entry_point == expected_output[dpcnnl_idx]['entry_point'], \ - f'wrong \'DynapcnnLayer.entry_point\': its value should be {entry_point}.' \ No newline at end of file + assert layerhandler.entry_point == expected_output[dpcnnl_idx]['entry_point'], \ + f'wrong \'DynapcnnLayer.entry_point\': its value should be {entry_point}.' From 8052cde6e657d5f9bb236bfb433e80fb47f384a0 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Wed, 2 Oct 2024 18:05:33 +0200 Subject: [PATCH 163/379] Unified name GraphExtractor --- sinabs/backend/dynapcnn/dynapcnn_network.py | 22 +++++++++---------- .../backend/dynapcnn/nir_graph_extractor.py | 2 +- .../test_graph_extractor.py | 6 ++--- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index 4d9bd169..98fe9d23 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -17,7 +17,7 @@ from .dynapcnn_layer import DynapcnnLayer from .dynapcnnnetwork_module import DynapcnnNetworkModule from .io import disable_timestamps, enable_timestamps, open_device, reset_timestamps -from .nir_graph_extractor import NIRtoDynapcnnNetworkGraph +from .nir_graph_extractor import GraphExtractor from .sinabs_edges_handler import merge_handler from .utils import ( DEFAULT_IGNORED_LAYER_TYPES, @@ -62,7 +62,7 @@ def __init__( (the connectivity between each `DynapcnnLayer`/core), `self._layers_mapper` (every `DynapcnnLayer` in the network) and `self._merge_points` (the `DynapcnnLayer`s that need a `Merge` input). Thus, the following private properties are delted as last step of the constructor: - - self._graph_tracer + - self._graph_extractor - self._sinabs_edges - self._sinabs_modules_map - self._nodes_to_dcnnl_map @@ -79,12 +79,12 @@ def __init__( assert len(self.input_shape) == 3, "infer_input_shape did not return 3-tuple" # computational graph from original PyTorch module. - self._graph_tracer = NIRtoDynapcnnNetworkGraph( + self._graph_extractor = GraphExtractor( snn, torch.randn((batch_size, *self.input_shape)) ) # needs the batch dimension. # remap `(A, X)` and `(X, B)` into `(A, B)` if `X` is a layer in the original `snn` to be ignored. - self._graph_tracer.remove_nodes_by_class(DEFAULT_IGNORED_LAYER_TYPES) + self._graph_extractor.remove_nodes_by_class(DEFAULT_IGNORED_LAYER_TYPES) # pre-process and group original nodes/edges from the graph tracer into data structures used to later create `DynapcnnLayer` instance. self._sinabs_edges, self._sinabs_modules_map = ( @@ -105,7 +105,7 @@ def __init__( edges=self._sinabs_edges, nodes_to_dcnnl_map=self._nodes_to_dcnnl_map, weight_rescaling_fn=weight_rescaling_fn, - entry_nodes=self._graph_tracer._entry_nodes, + entry_nodes=self._graph_extractor._entry_nodes, ) # these gather all data necessay to implement the forward method for this class. @@ -117,7 +117,7 @@ def __init__( ) = self._get_network_module() # all necessary `DynapcnnLayer` data held in `self._layers_mapper`: removing intermediary data structures no longer necessary. - del self._graph_tracer + del self._graph_extractor del self._sinabs_edges del self._sinabs_modules_map del self._nodes_to_dcnnl_map @@ -608,10 +608,10 @@ def _get_sinabs_edges_and_modules( # bypass merging layers to connect the nodes involved in them directly to the node where the merge happens. edges_without_merge = merge_handler( - self._graph_tracer.edges, self._graph_tracer.modules_map + self._graph_extractor.edges, self._graph_extractor.modules_map ) - return edges_without_merge, self._graph_tracer.modules_map + return edges_without_merge, self._graph_extractor.modules_map def _populate_nodes_io(self): """Loops through the nodes in the original graph to retrieve their I/O tensor shapes and add them to their respective @@ -639,12 +639,12 @@ def find_my_input(edges_list: list, node: int) -> int: return edge[0] return -1 - # access the I/O shapes for each node in `self._sinabs_edges` from the original graph in `self._graph_tracer`. + # access the I/O shapes for each node in `self._sinabs_edges` from the original graph in `self._graph_extractor`. for dcnnl_idx, dcnnl_data in self._nodes_to_dcnnl_map.items(): for node, node_data in dcnnl_data.items(): # node dictionary with layer data. if isinstance(node, int): - _in, _out = self._graph_tracer.get_node_io_shapes(node) + _in, _out = self._graph_extractor.get_node_io_shapes(node) # update node I/O shape in the mapper (drop batch dimension). if node != 0: @@ -659,7 +659,7 @@ def find_my_input(edges_list: list, node: int) -> int: else: # input comes from another node in the graph. _, _input_source_shape = ( - self._graph_tracer.get_node_io_shapes(input_node) + self._graph_extractor.get_node_io_shapes(input_node) ) node_data["input_shape"] = tuple( list(_input_source_shape)[1:] diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index f4b87bfa..66ccb171 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -13,7 +13,7 @@ from .utils import topological_sorting -class NIRtoDynapcnnNetworkGraph: +class GraphExtractor: def __init__(self, spiking_model: nn.Module, dummy_input: torch.tensor): """Class implementing the extraction of the computational graph from `spiking_model`, where each node represents a layer in the model and the list of edges represents how the data flow between diff --git a/tests/test_graph_extractor/test_graph_extractor.py b/tests/test_graph_extractor/test_graph_extractor.py index f21cb432..cb58f163 100644 --- a/tests/test_graph_extractor/test_graph_extractor.py +++ b/tests/test_graph_extractor/test_graph_extractor.py @@ -2,18 +2,18 @@ # contact : wsoaresgirao@gmail.com import pytest -from sinabs.backend.dynapcnn.nir_graph_extractor import NIRtoDynapcnnNetworkGraph +from sinabs.backend.dynapcnn.nir_graph_extractor import GraphExtractor from conftest_graph_extractor import args_GraphExtractor @pytest.mark.parametrize("snn, input_dummy, expected_output", args_GraphExtractor) def test_GraphExtractor(snn, input_dummy, expected_output): """ Tests the graph extraction from the original SNN being turned into a `DynapcnnNetwork`. These tests - verify the correct functionality of the `NIRtoDynapcnnNetworkGraph` class, which implements the first pre-processing + verify the correct functionality of the `GraphExtractor` class, which implements the first pre-processing step on the conversion of the SNN into a DynapcnnNetwork. """ - graph_tracer = NIRtoDynapcnnNetworkGraph(snn, input_dummy) + graph_tracer = GraphExtractor(snn, input_dummy) assert expected_output['edges'] == graph_tracer.edges, \ f'wrong list of edges extracted from the SNN.' From 5a1c6d18f9affae7ffe09f749b6474de00509f09 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Wed, 2 Oct 2024 18:32:46 +0200 Subject: [PATCH 164/379] Simplify graph extraction from NIR --- .../backend/dynapcnn/nir_graph_extractor.py | 31 +++++++------------ 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index 66ccb171..96bcad17 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -179,30 +179,23 @@ def _get_edges_from_nir( - name_2_indx_map (dict): `key` is the original variable name for a layer in `spiking_model` and `value is an integer representing the layer in a standard format. - entry_nodes (set): IDs of nodes acting as entry points for the network (i.e., receiving external input). """ - edges = set() - name_2_indx_map = {} - # TODO maybe make sure the input node from nir always gets assined `0`. - for src_node in nir_graph.node_list: - # Make sure current node is in `name_2_indx_map` - if src_node.name not in name_2_indx_map: - # Assign unique index by taking current length of `name_2_indx_map` - name_2_indx_map[src_node.name] = len(name_2_indx_map) - - for trg_node in src_node.outgoing_nodes: - # Make sure all targets of current node are in `name_2_indx_map` - if trg_node.name not in name_2_indx_map: - name_2_indx_map[trg_node.name] = len(name_2_indx_map) + # Assign a unique index to each node + name_2_indx_map = { + node.name: node_idx + for node_idx, node in enumerate(nir_graph.node_list) + } - # Store the edge of current node to the target - edges.add( - (name_2_indx_map[src_node.name], name_2_indx_map[trg_node.name]) - ) + # Extract edges for each node + edges = { + (name_2_indx_map[src.name], name_2_indx_map[tgt.name]) + for src, src_idx in name_2_indx_map.items() + for tgt in src.outgoing_nodes + } - # finding entry/exits nodes of the graph. + # find entry nodes of the graph. all_sources, all_targets = zip(*edges) - entry_nodes = set(all_sources) - set(all_targets) return edges, name_2_indx_map, entry_nodes From 57abd11076b0f4b3c6dcba8f844cdf1b8e01e0c7 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Wed, 2 Oct 2024 18:36:00 +0200 Subject: [PATCH 165/379] Simplify name-to-module map generation --- .../backend/dynapcnn/nir_graph_extractor.py | 24 ++++--------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index 96bcad17..fc2b9646 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -212,25 +212,11 @@ def _get_named_modules(self, model: nn.Module) -> Dict[int, nn.Module]: - modules_map (dict): the mapping between a node (`key` as an `int`) and its module (`value` as a `nn.Module`). """ modules_map = {} - - if isinstance( - model, nn.Sequential - ): # TODO shouldn't accept `nn.Sequential` any longer. - # access modules via `.named_modules()`. - for name, module in model.named_modules(): - if name != "": - # skip the module itself. - modules_map[self._name_2_indx_map[name]] = module - - elif isinstance(model, nn.Module): - # access modules via `.named_children()`. - for name, module in model.named_children(): - modules_map[self._name_2_indx_map[name]] = module - - else: - raise ValueError("Either a nn.Sequential or a nn.Module is required.") - - return modules_map + return { + self._name_2_indx_map[name]: module + for name, module in model.named_modules() + if name in self._name_2_indx_map + } def _sort_graph_nodes(self) -> List[int]: """Sort graph nodes topologically. From defd334128c092a254e4083e08e293d9764d6e3c Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Wed, 2 Oct 2024 18:41:16 +0200 Subject: [PATCH 166/379] Minor change to exception: Sequential is Module --- sinabs/backend/dynapcnn/exceptions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sinabs/backend/dynapcnn/exceptions.py b/sinabs/backend/dynapcnn/exceptions.py index fec09218..a7403b49 100644 --- a/sinabs/backend/dynapcnn/exceptions.py +++ b/sinabs/backend/dynapcnn/exceptions.py @@ -58,5 +58,5 @@ class InvalidTorchModel(Exception): def __init__(self, network_type): super().__init__( - f"A {network_type} needs to be either of type nn.Sequential or nn.Module." + f"A {network_type} needs to be of type nn.Module." ) From 8efe58f74f93d9695f61e5e69abb8b61b3ddde75 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Wed, 2 Oct 2024 19:05:23 +0200 Subject: [PATCH 167/379] Fix failing graph extractor test --- .../backend/dynapcnn/nir_graph_extractor.py | 3 +- .../test_graph_extractor.py | 38 +++++++++++++++++++ 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index fc2b9646..1793c574 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -190,8 +190,7 @@ def _get_edges_from_nir( # Extract edges for each node edges = { (name_2_indx_map[src.name], name_2_indx_map[tgt.name]) - for src, src_idx in name_2_indx_map.items() - for tgt in src.outgoing_nodes + for src in nir_graph.node_list for tgt in src.outgoing_nodes } # find entry nodes of the graph. diff --git a/tests/test_graph_extractor/test_graph_extractor.py b/tests/test_graph_extractor/test_graph_extractor.py index cb58f163..4f019adb 100644 --- a/tests/test_graph_extractor/test_graph_extractor.py +++ b/tests/test_graph_extractor/test_graph_extractor.py @@ -6,6 +6,43 @@ from conftest_graph_extractor import args_GraphExtractor +def fix_node_ids(expected_output, graph_extractor): + """ Match node IDs between graph extractor and expected output + + Node IDs can be assigned in many ways. This function prevents test + errors from generated IDs not matching expected output + + Parameters + ---------- + expected_output: Dict with expected output + graph_extractor: GraphExtractor instance + + Returns + ------- + Expected outputs with remapped node IDs + """ + idx_map = { + expected_idx: graph_extractor.name_2_indx_map[name] + for name, expected_idx in expected_output["name_2_indx_map"].items() + } + edges = { + (idx_map[src], idx_map[tgt]) for src, tgt in expected_output["edges"] + } + name_2_indx_map = { + name: idx_map[idx] for name, idx in expected_output["name_2_indx_map"].items() + } + entry_nodes = {idx_map[node] for node in expected_output["entry_nodes"]} + nodes_io_shapes = { + idx_map[node]: shape + for node, shape in expected_output["nodes_io_shapes"].items() + } + return { + "edges": edges, + "name_2_indx_map": name_2_indx_map, + "entry_nodes": entry_nodes, + "nodes_io_shapes": nodes_io_shapes, + } + @pytest.mark.parametrize("snn, input_dummy, expected_output", args_GraphExtractor) def test_GraphExtractor(snn, input_dummy, expected_output): """ Tests the graph extraction from the original SNN being turned into a `DynapcnnNetwork`. These tests @@ -14,6 +51,7 @@ def test_GraphExtractor(snn, input_dummy, expected_output): """ graph_tracer = GraphExtractor(snn, input_dummy) + expected_output = fix_node_ids(expected_output, graph_tracer) assert expected_output['edges'] == graph_tracer.edges, \ f'wrong list of edges extracted from the SNN.' From 80c9ef9da040f62e7ac49c3cb2a8456a8db94a92 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Wed, 2 Oct 2024 19:10:22 +0200 Subject: [PATCH 168/379] Refactor GraphExtractor.remove_nodes_by_class method --- .../backend/dynapcnn/nir_graph_extractor.py | 88 +++++++++++-------- 1 file changed, 50 insertions(+), 38 deletions(-) diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index 1793c574..aac83138 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -2,7 +2,7 @@ # contact : wsoaresgirao@gmail.com import copy -from typing import Dict, List, Tuple, Type, Set +from typing import Dict, List, Set, Tuple, Type import nirtorch import torch @@ -114,40 +114,8 @@ def remove_nodes_by_class( for new_idx, old_idx in enumerate(sorted(source2target.keys())) } - # Parse new set of edges based on remapped node IDs - self._edges = { - (remapped_nodes[src], remapped_nodes[tgt]) - for src, targets in source2target.items() - for tgt in targets - } - - # Update name-to-index map based on new node indices - self._name_2_indx_map = { - name: remapped_nodes[old_idx] - for name, old_idx in self._name_2_indx_map.items() - if old_idx in remapped_nodes - } - - # Update entry nodes based on new node indices - self._entry_nodes = { - remapped_nodes[old_idx] - for old_idx in self._entry_nodes - if old_idx in remapped_nodes - } - - # Update io-shapes based on new node indices - self._nodes_io_shapes = { - remapped_nodes[old_idx]: shape - for old_idx, shape in self._nodes_io_shapes.items() - if old_idx in remapped_nodes - } - - # Update sinabs module map based on new node indices - self._modules_map = { - remapped_nodes[old_idx]: module - for old_idx, module in self._modules_map.items() - if old_idx in remapped_nodes - } + # Update internal graph representation according to changes + self._update_internal_representation(remapped_nodes) def get_node_io_shapes(self, node: int) -> Tuple[torch.Size, torch.Size]: """Returns the I/O tensors' shapes of `node`. @@ -183,14 +151,14 @@ def _get_edges_from_nir( # Assign a unique index to each node name_2_indx_map = { - node.name: node_idx - for node_idx, node in enumerate(nir_graph.node_list) + node.name: node_idx for node_idx, node in enumerate(nir_graph.node_list) } # Extract edges for each node edges = { (name_2_indx_map[src.name], name_2_indx_map[tgt.name]) - for src in nir_graph.node_list for tgt in src.outgoing_nodes + for src in nir_graph.node_list + for tgt in src.outgoing_nodes } # find entry nodes of the graph. @@ -217,6 +185,50 @@ def _get_named_modules(self, model: nn.Module) -> Dict[int, nn.Module]: if name in self._name_2_indx_map } + def _update_internal_representation(self, remapped_nodes: Dict[int, int]): + """Update internal attributes after remapping of nodes + + Parameters + ---------- + remapped_nodes (dict): Maps previous (key) to new (value) node + indices. Nodes that were removed are not included. + """ + + # Parse new set of edges based on remapped node IDs + self._edges = { + (remapped_nodes[src], remapped_nodes[tgt]) + for src, targets in source2target.items() + for tgt in targets + } + + # Update name-to-index map based on new node indices + self._name_2_indx_map = { + name: remapped_nodes[old_idx] + for name, old_idx in self._name_2_indx_map.items() + if old_idx in remapped_nodes + } + + # Update entry nodes based on new node indices + self._entry_nodes = { + remapped_nodes[old_idx] + for old_idx in self._entry_nodes + if old_idx in remapped_nodes + } + + # Update io-shapes based on new node indices + self._nodes_io_shapes = { + remapped_nodes[old_idx]: shape + for old_idx, shape in self._nodes_io_shapes.items() + if old_idx in remapped_nodes + } + + # Update sinabs module map based on new node indices + self._modules_map = { + remapped_nodes[old_idx]: module + for old_idx, module in self._modules_map.items() + if old_idx in remapped_nodes + } + def _sort_graph_nodes(self) -> List[int]: """Sort graph nodes topologically. From 86889b8998e14fe9045d0e8ff072fc4ee313e554 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Thu, 3 Oct 2024 12:17:57 +0200 Subject: [PATCH 169/379] Remove need for merge_handler: Remove merge nodes directly as "ignored node class" --- sinabs/backend/dynapcnn/dynapcnn_network.py | 25 +------- .../backend/dynapcnn/sinabs_edges_handler.py | 63 ------------------- sinabs/backend/dynapcnn/utils.py | 2 +- 3 files changed, 2 insertions(+), 88 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index 98fe9d23..59d0fa70 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -18,7 +18,6 @@ from .dynapcnnnetwork_module import DynapcnnNetworkModule from .io import disable_timestamps, enable_timestamps, open_device, reset_timestamps from .nir_graph_extractor import GraphExtractor -from .sinabs_edges_handler import merge_handler from .utils import ( DEFAULT_IGNORED_LAYER_TYPES, build_from_graph, @@ -83,7 +82,7 @@ def __init__( snn, torch.randn((batch_size, *self.input_shape)) ) # needs the batch dimension. - # remap `(A, X)` and `(X, B)` into `(A, B)` if `X` is a layer in the original `snn` to be ignored. + # Remove nodes of ignored classes (including merge nodes) self._graph_extractor.remove_nodes_by_class(DEFAULT_IGNORED_LAYER_TYPES) # pre-process and group original nodes/edges from the graph tracer into data structures used to later create `DynapcnnLayer` instance. @@ -591,28 +590,6 @@ def _get_dynapcnnlayers_edges(self) -> List[Tuple[int, int]]: return dcnnl_edges - def _get_sinabs_edges_and_modules( - self, - ) -> Tuple[List[Tuple[int, int]], Dict[int, nn.Module], Dict[int, int]]: - """The computational graph extracted from `snn` might contain layers that are ignored (e.g. a `nn.Flatten` will be - ignored when creating a `DynapcnnLayer` instance). Thus the list of edges from such model need to be rebuilt such that if there are - edges `(A, X)` and `(X, B)`, and `X` is an ignored layer, an edge `(A, B)` is created. - - Returns - ---------- - - edges_without_merge (list): a list of edges based on `sinabs_edges` but where edges involving a `Merge` layer have been - remapped to connect the nodes involved in the merging directly. - - sinabs_modules_map (dict): a dict containing the nodes of the graph (described now by `edges_without_merge`) as `key` and - their associated module as `value`. - """ - - # bypass merging layers to connect the nodes involved in them directly to the node where the merge happens. - edges_without_merge = merge_handler( - self._graph_extractor.edges, self._graph_extractor.modules_map - ) - - return edges_without_merge, self._graph_extractor.modules_map - def _populate_nodes_io(self): """Loops through the nodes in the original graph to retrieve their I/O tensor shapes and add them to their respective representations in `self._nodes_to_dcnnl_map`.""" diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index 13698c27..1a2ea14c 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -278,66 +278,3 @@ def is_valid_dynapcnnlayer_pairing( return True else: raise InvalidLayerDestination(type(layers[edge[0]]), type(layers[edge[1]])) - - -def merge_handler( - sinabs_edges: List[Tuple[int, int]], sinabs_modules_map: Dict[int, nn.Module] -) -> List[Tuple[int, int]]: - """Handles connections between nodes made via a `sinabs.layers.Merge` layer. If `X` is a merge layer then edges `(X, C)` are removed - from the edges list since they don't affect the creationg of `DynapcnnLayer`s. Edges `(Y, X)` are turned into a edge `(Y, C)` pointing - directly to the node receiving the merged inputs such that the `DynapcnnLayer` containing `Y` can have the `DynapcnnLayer` containing `C` - as one of its destinations. - - Parameters - ---------- - sinabs_edges (list): edges extracted from the computational graph of the network provided to `DynapcnnNetworkGraph` where edges involving - layers that are ignored (e.g. `nn.Flatten`) have been removed (the nodes previously linked via dropped nodes are linked to each other directly). - sinabs_modules_map (dict): mapping where the `key` represents a node in the graph and the `value` represents the node's layer. - - Returns - ---------- - edges_without_merge (list): edges based on `sinabs_edges` but where edges involving a `Merge` layer have been remapped to connect the nodes - involved in the merging directly. - """ - edges = copy.deepcopy(sinabs_edges) - edges_without_merge = [] - merge_nodes = {} - - # finding the nodes representing Merge layers. - for edge in edges: - src = edge[0] - trg = edge[1] - - if isinstance(sinabs_modules_map[src], sinabs.layers.Merge): - if src not in merge_nodes: - # found node receiving merged inputs from two previous layers. - merge_nodes[src] = {"sources": [], "merge_into": trg} - - for _edge in edges: - if _edge[1] == src: - # found node used as argument for a Merge layer. - merge_nodes[src]["sources"].append(_edge[0]) - if len(merge_nodes[src]["sources"]) > 2: - raise ValueError( - "A Merge layer can not have more than two inputs." - ) - - for edge in edges: - # removing edges connection from/to merging layers from the computational graph. - src = edge[0] - trg = edge[1] - - if src in merge_nodes: - # edge (`Merge`, trg) is not necessary for later DynapcnnLayer creation. - pass - - elif trg in merge_nodes: - # point `src` directly to the node it was previously targeting via a Merge layer. - new_edge = (src, merge_nodes[trg]["merge_into"]) - edges_without_merge.append(new_edge) - - else: - # edge not involved in merging. - edges_without_merge.append(edge) - - return edges_without_merge diff --git a/sinabs/backend/dynapcnn/utils.py b/sinabs/backend/dynapcnn/utils.py index 88d4dfae..020eb019 100644 --- a/sinabs/backend/dynapcnn/utils.py +++ b/sinabs/backend/dynapcnn/utils.py @@ -18,7 +18,7 @@ if TYPE_CHECKING: from sinabs.backend.dynapcnn.dynapcnn_network import DynapcnnNetwork -DEFAULT_IGNORED_LAYER_TYPES = (nn.Identity, nn.Dropout, nn.Dropout2d, nn.Flatten) +DEFAULT_IGNORED_LAYER_TYPES = (nn.Identity, nn.Dropout, nn.Dropout2d, nn.Flatten, sl.Merge) ####################################################### Device Related ####################################################### From 80dcff1bfe4eaed36b68dee59ea9588b09c4004c Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Thu, 3 Oct 2024 12:36:31 +0200 Subject: [PATCH 170/379] GraphExtractor does formal verification of extracted graph. --- .../backend/dynapcnn/nir_graph_extractor.py | 60 +++++++++++++++---- sinabs/backend/dynapcnn/utils.py | 7 ++- 2 files changed, 53 insertions(+), 14 deletions(-) diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index aac83138..7b3b7763 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -1,7 +1,6 @@ # author : Willian Soares Girao # contact : wsoaresgirao@gmail.com -import copy from typing import Dict, List, Set, Tuple, Type import nirtorch @@ -10,7 +9,12 @@ import sinabs -from .utils import topological_sorting +from .exceptions import InvalidGraphStructure +from .utils import ( + LAYER_TYPES_WITH_MULTIPLE_INPUTS, + LAYER_TYPES_WITH_MULTIPLE_OUTPUTS, + topological_sorting, +) class GraphExtractor: @@ -50,9 +54,12 @@ def __init__(self, spiking_model: nn.Module, dummy_input: torch.tensor): self._get_edges_from_nir(nir_graph) ) - # recovers the associated `nn.Module` (layer) of each node. + # Store the associated `nn.Module` (layer) of each node. self._modules_map = self._get_named_modules(spiking_model) + # Verify that graph is compatible + self.verify_graph_integrity() + # retrieves what the I/O shape for each node's module is. self._nodes_io_shapes = self._get_nodes_io_shapes(dummy_input) @@ -114,6 +121,13 @@ def remove_nodes_by_class( for new_idx, old_idx in enumerate(sorted(source2target.keys())) } + # Parse new set of edges based on remapped node IDs + self._edges = { + (remapped_nodes[src], remapped_nodes[tgt]) + for src, targets in source2target.items() + for tgt in targets + } + # Update internal graph representation according to changes self._update_internal_representation(remapped_nodes) @@ -129,6 +143,34 @@ def get_node_io_shapes(self, node: int) -> Tuple[torch.Size, torch.Size]: self._nodes_io_shapes[node]["input"], self._nodes_io_shapes[node]["output"], ) + + def verify_graph_integrity(self): + """ Apply checks to verify that graph is supported + + Currently this checks that only nodes of specific classes have + multiple sources or targets. This method might be extended in the + future to implement stricter formal verification. + """ + # Iterate over all nodes, and count its sources and targets + for node, module in self.modules_map.items(): + # Check sources + if not isinstance(module, LAYER_TYPES_WITH_MULTIPLE_INPUTS): + sources = self._find_all_sources_of_input_to(node) + if len(sources) > 1: + raise InvalidGraphStructure( + f"Only nodes of type {LAYER_TYPES_WITH_MULTIPLE_INPUTS} " + f"can have more than one input. Node {node} is of type " + f"{type(module)} and has {len(sources)} inputs." + ) + # Check targets + if not isinstance(module, LAYER_TYPES_WITH_MULTIPLE_OUTPUTS): + targets = self._find_valid_targets(node) + if len(targets) > 1: + raise InvalidGraphStructure( + f"Only nodes of type {LAYER_TYPES_WITH_MULTIPLE_OUTPUTS} " + f"can have more than one output. Node {node} is of type " + f"{type(module)} and has {len(targets)} outputs." + ) ####################################################### Pivate Methods ####################################################### @@ -178,7 +220,6 @@ def _get_named_modules(self, model: nn.Module) -> Dict[int, nn.Module]: ---------- - modules_map (dict): the mapping between a node (`key` as an `int`) and its module (`value` as a `nn.Module`). """ - modules_map = {} return { self._name_2_indx_map[name]: module for name, module in model.named_modules() @@ -194,13 +235,6 @@ def _update_internal_representation(self, remapped_nodes: Dict[int, int]): indices. Nodes that were removed are not included. """ - # Parse new set of edges based on remapped node IDs - self._edges = { - (remapped_nodes[src], remapped_nodes[tgt]) - for src, targets in source2target.items() - for tgt in targets - } - # Update name-to-index map based on new node indices self._name_2_indx_map = { name: remapped_nodes[old_idx] @@ -355,13 +389,13 @@ def _find_merge_arguments(self, node: int) -> Tuple[int, int]: if len(sources) != 2: raise ValueError( - f"Number of arguments found for `Merge` node {merge_node} is {len(args)} (should be 2)." + f"Number of arguments found for `Merge` node {node} is {len(sources)} (should be 2)." ) return tuple(sources) def _find_valid_targets( - self, node: int, ignored_node_classes: Tuple[Type] + self, node: int, ignored_node_classes: Tuple[Type] = () ) -> Set[int]: """Find all targets of a node that are not ignored classes diff --git a/sinabs/backend/dynapcnn/utils.py b/sinabs/backend/dynapcnn/utils.py index 020eb019..139377bd 100644 --- a/sinabs/backend/dynapcnn/utils.py +++ b/sinabs/backend/dynapcnn/utils.py @@ -18,7 +18,12 @@ if TYPE_CHECKING: from sinabs.backend.dynapcnn.dynapcnn_network import DynapcnnNetwork -DEFAULT_IGNORED_LAYER_TYPES = (nn.Identity, nn.Dropout, nn.Dropout2d, nn.Flatten, sl.Merge) +DEFAULT_IGNORED_LAYER_TYPES = Union[nn.Identity, nn.Dropout, nn.Dropout2d, nn.Flatten, sl.Merge] +LAYER_TYPES_WITH_MULTIPLE_INPUTS = Union[sl.Merge] +LAYER_TYPES_WITH_MULTIPLE_OUTPUTS = Union[ + sl.IAFSqueeze, sl.SumPool2d, nn.AvgPool2d +] + ####################################################### Device Related ####################################################### From d7d0e934c06a17e7361fea3d67e261a7ab963711 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Thu, 3 Oct 2024 12:36:51 +0200 Subject: [PATCH 171/379] General linting and minor refactoring --- AUTHORS | 4 + sinabs/backend/dynapcnn/dynapcnn_network.py | 1 - sinabs/backend/dynapcnn/exceptions.py | 85 +++++++++++++++++-- .../backend/dynapcnn/sinabs_edges_handler.py | 22 +++-- sinabs/backend/dynapcnn/sinabs_edges_utils.py | 68 --------------- 5 files changed, 98 insertions(+), 82 deletions(-) diff --git a/AUTHORS b/AUTHORS index f1e24665..9d16953e 100644 --- a/AUTHORS +++ b/AUTHORS @@ -7,6 +7,7 @@ Gregor Lenz Martino Sorbaro Martino Sorbaro Massimo Bortone +Mina Khoei MurphyWu Nogay Kuepelioglu Nogay Küpelioglu @@ -19,6 +20,8 @@ Sadique Sheik Vanessa Leite Vanessa Leite Vanessa Leite +Willian-Girao +WillianSG Yalun Hu Yalun_Hu allan @@ -35,6 +38,7 @@ qian.liu sadique.sheik sadique.sheik shynuie +unknown yalun.hu yannan xing yannan.xing diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index 59d0fa70..b5afb867 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -1,7 +1,6 @@ # author : Willian Soares Girao # contact : wsoaresgirao@gmail.com -import copy import time from typing import Callable, Dict, List, Optional, Sequence, Tuple, Union diff --git a/sinabs/backend/dynapcnn/exceptions.py b/sinabs/backend/dynapcnn/exceptions.py index a7403b49..5fa79c40 100644 --- a/sinabs/backend/dynapcnn/exceptions.py +++ b/sinabs/backend/dynapcnn/exceptions.py @@ -1,3 +1,6 @@ +from typing import Tuple, Type + + class MissingLayer(Exception): index: int @@ -6,8 +9,8 @@ def __init__(self, index: int): class UnexpectedLayer(Exception): - layer_type_found: type - layer_type_expected: type + layer_type_found: Type + layer_type_expected: Type def __init__(self, expected, found): super().__init__(f"Expected {expected} but found {found}") @@ -20,8 +23,8 @@ class InputConfigurationError(Exception): class WrongModuleCount(Exception): - dynapcnnlayer_indx: type - modules_count: type + dynapcnnlayer_indx: Type + modules_count: Type def __init__(self, dynapcnnlayer_indx, modules_count): super().__init__( @@ -30,7 +33,7 @@ def __init__(self, dynapcnnlayer_indx, modules_count): class WrongPoolingModule(Exception): - pooling_module: type + pooling_module: Type def __init__( self, @@ -42,7 +45,7 @@ def __init__( class InvalidModel(Exception): - model: type + model: Type def __init__( self, @@ -60,3 +63,73 @@ def __init__(self, network_type): super().__init__( f"A {network_type} needs to be of type nn.Module." ) + +class InvalidGraphStructure(Exception): + pass + +# Edge exceptions. + + +class InvalidEdge(Exception): + edge: Tuple[int, int] + source: Type + target: Type + + def __init__(self, edge, source, target): + super().__init__(f"Invalid edge {edge}: {source} can not target {target}.") + + +class InvalidEdgeType(Exception): + edge: Tuple[int, int] + type: int + + def __init__(self, edge, type): + super().__init__(f"Invalid edge type {type} for edge {edge}.") + + +class UnmatchedNode(Exception): + edge: Tuple[int, int] + node: int + + def __init__(self, edge, node): + super().__init__( + f"Node {node} in edge {edge} can not found in previously processed edges." + ) + + +class UnknownNode(Exception): + node: int + + def __init__(self, node): + super().__init__( + f"Node {node} can not be found within any DynapcnnLayer mapper." + ) + + +class MaxDestinationsReached(Exception): + dynapcnnlayer_index: int + + def __init__(self, dynapcnnlayer_index): + super().__init__( + f"DynapcnnLayer with index {dynapcnnlayer_index} has more than 2 destinations." + ) + + +class InvalidLayerLoop(Exception): + dynapcnnlayerA_index: int + dynapcnnlayerB_index: int + + def __init__(self, dynapcnnlayerA_index, dynapcnnlayerB_index): + super().__init__( + f"DynapcnnLayer {dynapcnnlayerA_index} can not connect to {dynapcnnlayerB_index} since reverse edge already exists." + ) + + +class InvalidLayerDestination(Exception): + dynapcnnlayerA: Type + dynapcnnlayerB: Type + + def __init__(self, dynapcnnlayerA, dynapcnnlayerB): + super().__init__( + f"DynapcnnLayer {dynapcnnlayerA} in one core can not connect to {dynapcnnlayerB} in another core." + ) \ No newline at end of file diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index 1a2ea14c..1a642964 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -5,15 +5,23 @@ contact : williansoaresgirao@gmail.com """ -import copy from typing import Dict, List, Tuple, Type import torch.nn as nn -import sinabs -import sinabs.layers - -from .sinabs_edges_utils import * +from .exceptions import ( + InvalidEdge, + InvalidEdgeType, + InvalidLayerDestination, + InvalidLayerLoop, + MaxDestinationsReached, + UnknownNode, + UnmatchedNode, +) +from .sinabs_edges_utils import ( + VALID_DYNAPCNNLAYER_EDGES, + VALID_SINABS_EDGE_TYPE_IDS, +) def process_edge( @@ -174,13 +182,13 @@ def add_pool_to_dynapcnnlayer_blk( # Search for edge[0] (neuron layer) in DynapcnnLayers if (indx := find_initialized_node(edge[0], mapper)) is not None: # Add edge[1] (pooling layer) to the same dynapcnn layer - mapped[indx][edge[1]] = { + mapper[indx][edge[1]] = { "layer": layers[edge[1]], "input_shape": None, "output_shape": None, } else: - raise UnmatchedNode(edge, node) + raise UnmatchedNode(edge, edge[1]) def find_initialized_node(node: int, mapper: Dict[int, Dict[int, Dict]]) -> bool: diff --git a/sinabs/backend/dynapcnn/sinabs_edges_utils.py b/sinabs/backend/dynapcnn/sinabs_edges_utils.py index 76e8ddef..a60a8a74 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_utils.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_utils.py @@ -5,8 +5,6 @@ contact : williansoaresgirao@gmail.com """ -from typing import Tuple - import torch.nn as nn import sinabs.layers as sl @@ -63,69 +61,3 @@ VALID_SINABS_NODE_FAN_IN = [] VALID_SINABS_NODE_FAN_OUT = [] -# Edge exceptions. - - -class InvalidEdge(Exception): - edge: Tuple[int, int] - source: type - target: type - - def __init__(self, edge, source, target): - super().__init__(f"Invalid edge {edge}: {source} can not target {target}.") - - -class InvalidEdgeType(Exception): - edge: Tuple[int, int] - type: int - - def __init__(self, edge, type): - super().__init__(f"Invalid edge type {type} for edge {edge}.") - - -class UnmatchedNode(Exception): - edge: Tuple[int, int] - node: int - - def __init__(self, edge, node): - super().__init__( - f"Node {node} in edge {edge} can not found in previously processed edges." - ) - - -class UnknownNode(Exception): - node: int - - def __init__(self, node): - super().__init__( - f"Node {node} can not be found within any DynapcnnLayer mapper." - ) - - -class MaxDestinationsReached(Exception): - dynapcnnlayer_index: int - - def __init__(self, dynapcnnlayer_index): - super().__init__( - f"DynapcnnLayer with index {dynapcnnlayer_index} has more than 2 destinations." - ) - - -class InvalidLayerLoop(Exception): - dynapcnnlayerA_index: int - dynapcnnlayerB_index: int - - def __init__(self, dynapcnnlayerA_index, dynapcnnlayerB_index): - super().__init__( - f"DynapcnnLayer {dynapcnnlayerA_index} can not connect to {dynapcnnlayerB_index} since reverse edge already exists." - ) - - -class InvalidLayerDestination(Exception): - dynapcnnlayerA: type - dynapcnnlayerB: type - - def __init__(self, dynapcnnlayerA, dynapcnnlayerB): - super().__init__( - f"DynapcnnLayer {dynapcnnlayerA} in one core can not connect to {dynapcnnlayerB} in another core." - ) From 46078a4d6942db3ce36240fdfa5a500ccad822ee Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Thu, 3 Oct 2024 12:44:28 +0200 Subject: [PATCH 172/379] sinabs_edges_utils becomes connectivity_specs --- ...sinabs_edges_utils.py => connectivity_specs.py} | 14 ++++++++------ sinabs/backend/dynapcnn/sinabs_edges_handler.py | 2 +- sinabs/backend/dynapcnn/utils.py | 4 ---- 3 files changed, 9 insertions(+), 11 deletions(-) rename sinabs/backend/dynapcnn/{sinabs_edges_utils.py => connectivity_specs.py} (85%) diff --git a/sinabs/backend/dynapcnn/sinabs_edges_utils.py b/sinabs/backend/dynapcnn/connectivity_specs.py similarity index 85% rename from sinabs/backend/dynapcnn/sinabs_edges_utils.py rename to sinabs/backend/dynapcnn/connectivity_specs.py index a60a8a74..b2c2d6c3 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_utils.py +++ b/sinabs/backend/dynapcnn/connectivity_specs.py @@ -1,16 +1,15 @@ """ -functionality : implementation of exceptions and constraints regarding the processing of edges from a network - computational graph. +functionality : list device-independent supported connections between layers on chip author : Willian Soares Girao contact : williansoaresgirao@gmail.com """ +from typing import Union + import torch.nn as nn import sinabs.layers as sl -# Constraints. # @TODO constraints are ideally device-dependent. - VALID_SINABS_EDGES = { 0: ( nn.Conv2d, @@ -58,6 +57,9 @@ (sl.SumPool2d, nn.Linear), ] -VALID_SINABS_NODE_FAN_IN = [] -VALID_SINABS_NODE_FAN_OUT = [] +LAYER_TYPES_WITH_MULTIPLE_INPUTS = Union[sl.Merge] + +LAYER_TYPES_WITH_MULTIPLE_OUTPUTS = Union[ + sl.IAFSqueeze, sl.SumPool2d, nn.AvgPool2d +] diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index 1a642964..c532d967 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -18,7 +18,7 @@ UnknownNode, UnmatchedNode, ) -from .sinabs_edges_utils import ( +from .connectivity_specs import ( VALID_DYNAPCNNLAYER_EDGES, VALID_SINABS_EDGE_TYPE_IDS, ) diff --git a/sinabs/backend/dynapcnn/utils.py b/sinabs/backend/dynapcnn/utils.py index 139377bd..0eb0378a 100644 --- a/sinabs/backend/dynapcnn/utils.py +++ b/sinabs/backend/dynapcnn/utils.py @@ -19,10 +19,6 @@ from sinabs.backend.dynapcnn.dynapcnn_network import DynapcnnNetwork DEFAULT_IGNORED_LAYER_TYPES = Union[nn.Identity, nn.Dropout, nn.Dropout2d, nn.Flatten, sl.Merge] -LAYER_TYPES_WITH_MULTIPLE_INPUTS = Union[sl.Merge] -LAYER_TYPES_WITH_MULTIPLE_OUTPUTS = Union[ - sl.IAFSqueeze, sl.SumPool2d, nn.AvgPool2d -] ####################################################### Device Related ####################################################### From 19d4ebd000e0d77955aaf0ea4537d2d715618d64 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Thu, 3 Oct 2024 12:48:59 +0200 Subject: [PATCH 173/379] Fix minor bugs --- sinabs/backend/dynapcnn/dynapcnn_network.py | 7 +------ sinabs/backend/dynapcnn/nir_graph_extractor.py | 7 ++++--- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index b5afb867..ab230a1b 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -84,14 +84,9 @@ def __init__( # Remove nodes of ignored classes (including merge nodes) self._graph_extractor.remove_nodes_by_class(DEFAULT_IGNORED_LAYER_TYPES) - # pre-process and group original nodes/edges from the graph tracer into data structures used to later create `DynapcnnLayer` instance. - self._sinabs_edges, self._sinabs_modules_map = ( - self._get_sinabs_edges_and_modules() - ) - # create a dict holding the data necessary to instantiate a `DynapcnnLayer`. self._nodes_to_dcnnl_map = build_nodes_to_dcnnl_map( - layers=self._sinabs_modules_map, edges=self._sinabs_edges + layers=self._graph_extractor.modules_map, edges=self._graph_extractor.edges ) # updates 'self._nodes_to_dcnnl_map' to include the I/O shape for each node. diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index 7b3b7763..b894e8cc 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -9,12 +9,13 @@ import sinabs -from .exceptions import InvalidGraphStructure -from .utils import ( +from .connectivity_specs import ( LAYER_TYPES_WITH_MULTIPLE_INPUTS, LAYER_TYPES_WITH_MULTIPLE_OUTPUTS, - topological_sorting, ) +from .exceptions import InvalidGraphStructure +from .utils import topological_sorting + class GraphExtractor: From aec09518e4207f0b6198f5dfeb6eaf2516315427 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Thu, 3 Oct 2024 14:14:16 +0200 Subject: [PATCH 174/379] Remove obsolete graph_tracer.py --- sinabs/backend/dynapcnn/graph_tracer.py | 186 ------------------------ 1 file changed, 186 deletions(-) delete mode 100644 sinabs/backend/dynapcnn/graph_tracer.py diff --git a/sinabs/backend/dynapcnn/graph_tracer.py b/sinabs/backend/dynapcnn/graph_tracer.py deleted file mode 100644 index 153bec6d..00000000 --- a/sinabs/backend/dynapcnn/graph_tracer.py +++ /dev/null @@ -1,186 +0,0 @@ -import copy -import re -from typing import Union - -import matplotlib.pyplot as plt -import networkx as nx -import numpy as np -import torch -import torch.nn as nn - - -class GraphTracer: - def __init__( - self, model: Union[nn.Sequential, nn.Module], dummy_input: np.array - ) -> None: - """.""" - - trace = torch.jit.trace(model, dummy_input) - _ = trace(dummy_input) - __ = copy.deepcopy(trace) - - self.graph = __.graph - - self.modules_map, self.name_2_indx_map = self.get_named_modules(model) - self.forward_edges = self.get_foward_edges() - self.ATens = self.get_ATen_operations() - self.edges_list = self.get_graph_edges() - - def from_name_2_indx(self, name): - if name in self.name_2_indx_map: - return self.name_2_indx_map[name] - else: - last_indx = None - for _name, indx in self.name_2_indx_map.items(): - last_indx = indx - self.name_2_indx_map[name] = last_indx + 1 - return self.name_2_indx_map[name] - - def get_named_modules(self, module: nn.Module): - """.""" - modules_map = {} - name_2_indx_map = {} - indx = 0 - for name, mod in module.named_modules(): - if name: - modules_map[indx] = mod - name_2_indx_map[name] = indx - indx += 1 - return modules_map, name_2_indx_map - - def get_foward_edges(self): - """.""" - forward_edges = {} - for node in self.graph.nodes(): - node = str(node) - regex = re.compile( - r'%(.*?) :.*prim::CallMethod\[name="forward"\]\(%(.*?), %(.*?)\)' - ) - match = regex.search(node) - if match: - source = match.group(3).replace("_", "") - target = match.group(2).replace("_", "") - result = match.group(1).replace("_", "") - forward_edges[self.from_name_2_indx(result)] = ( - self.from_name_2_indx(source), - self.from_name_2_indx(target), - ) - - return forward_edges - - def get_graph_edges(self): - """.""" - edges = [] - last_result = None - - for result_node, forward_edge in self.forward_edges.items(): - src = forward_edge[0] - trg = forward_edge[1] - - if not last_result: - last_result = result_node - edges.append(("input", trg)) - elif src == last_result: - edges.append((edges[-1][1], trg)) - last_result = result_node - else: - scr1, scr2 = self.get_ATen_operands(src) - edges.append((scr1, trg)) - edges.append((scr2, trg)) - last_result = result_node - - edges.append((edges[-1][1], "output")) - - return edges[1:-1] - - def get_ATen_operands(self, node): - """.""" - if node in self.ATens: - src1 = self.ATens[node]["args"][1] - src2 = self.ATens[node]["args"][0] - return self.forward_edges[src1][1], self.forward_edges[src2][1] - else: - # throw error - return None, None - - def get_ATen_operations(self): - """ATen is PyTorch's tensor library backend, which provides a set of operations that operate on - tensors directly. These include arithmetic operations (add, mul, etc.), mathematical - functions (sin, cos, etc.), and tensor manipulation operations (view, reshape, etc.). - """ - ATens = {} - for node in self.graph.nodes(): - node = str(node) - regex = re.compile(r"%(.*?) :.*aten::(.*?)\(%(.*?), %(.*?), %(.*?)\)") - - match = regex.search(node) - - if match: - result_node = match.group(1) - operation = match.group(2) - operator1 = self.from_name_2_indx(match.group(3)) - operator2 = self.from_name_2_indx(match.group(4)) - const_operator = match.group(5) - ATens[result_node] = { - "op": operation, - "args": (operator1, operator2, const_operator), - } - return ATens - - def remove_ignored_nodes(self, default_ignored_nodes): - """Recreates the edges list based on layers that 'DynapcnnNetwork' will ignore. This - is done by setting the source (target) node of an edge where the source (target) node - will be dropped as the node that originally targeted this node to be dropped. - """ - edges = copy.deepcopy(self.edges_list) - parsed_edges = [] - removed_nodes = [] - - # removing ignored nodes from edges. - for edge_idx in range(len(edges)): - _src = edges[edge_idx][0] - _trg = edges[edge_idx][1] - - if isinstance(self.modules_map[_src], default_ignored_nodes): - removed_nodes.append(_src) - # all edges where node '_src' is target change it to node '_trg' as their target. - for edge in edges: - if edge[1] == _src: - new_edge = (edge[0], _trg) - elif isinstance(self.modules_map[_trg], default_ignored_nodes): - removed_nodes.append(_trg) - # all edges where node '_trg' is source change it to node '_src' as their source. - for edge in edges: - if edge[0] == _trg: - new_edge = (_src, edge[1]) - else: - new_edge = (_src, _trg) - - if new_edge not in parsed_edges: - parsed_edges.append(new_edge) - - removed_nodes = list(set(removed_nodes)) - - # remapping nodes indexes. - remapped_nodes = {} - for node_indx, __ in self.modules_map.items(): - _ = [x for x in removed_nodes if node_indx > x] - remapped_nodes[node_indx] = node_indx - len(_) - - for x in removed_nodes: - del remapped_nodes[x] - - # remapping nodes names in parsed edges. - remapped_edges = [] - for edge in parsed_edges: - remapped_edges.append((remapped_nodes[edge[0]], remapped_nodes[edge[1]])) - - return remapped_edges - - @staticmethod - def plot_graph(edges_list): - """.""" - G = nx.DiGraph(edges_list) - layout = nx.spring_layout(G) - nx.draw(G, pos=layout, with_labels=True, node_size=800) - plt.show() From dde3b78d6cf26c9ad90b5357cbf403af8875d7ec Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Thu, 3 Oct 2024 14:17:01 +0200 Subject: [PATCH 175/379] Tidy up edge types --- sinabs/backend/dynapcnn/connectivity_specs.py | 64 ++++++------------- .../backend/dynapcnn/sinabs_edges_handler.py | 12 ++-- 2 files changed, 27 insertions(+), 49 deletions(-) diff --git a/sinabs/backend/dynapcnn/connectivity_specs.py b/sinabs/backend/dynapcnn/connectivity_specs.py index b2c2d6c3..6d6b0a1b 100644 --- a/sinabs/backend/dynapcnn/connectivity_specs.py +++ b/sinabs/backend/dynapcnn/connectivity_specs.py @@ -10,56 +10,30 @@ import sinabs.layers as sl +Pooling = Union[sl.SumPool2d, nn.AvgPool2d] +Weight = Union[nn.Conv2d, nn.Linear] +Neuron = sl.IAFSqueeze + VALID_SINABS_EDGES = { - 0: ( - nn.Conv2d, - sl.iaf.IAFSqueeze, - ), # convoluion is always followed by a neuron layer. - 1: (sl.iaf.IAFSqueeze, nn.AvgPool2d), - 2: (sl.iaf.IAFSqueeze, nn.Conv2d), - 3: ( - sl.iaf.IAFSqueeze, - nn.Linear, - ), # same case as `2` since `nn.Linear` layers are converted into `nn.Conv2d` by `DynapcnnLayer`. - 4: ( - nn.AvgPool2d, - nn.Conv2d, - ), # `nn.Pool2d` is always "ending" a DynapcnnLayer sequence of modules (comes after a `sl.iaf`). - 5: ( - nn.AvgPool2d, - nn.Linear, - ), # same as case `4` since `nn.Linear` layers are converted into `nn.Conv2d` by `DynapcnnLayer`. - 6: ( - nn.Linear, - sl.iaf.IAFSqueeze, - ), # same as case `0` since `nn.Linear` layers are converted into `nn.Conv2d` by `DynapcnnLayer`. - 7: ( - sl.iaf.IAFSqueeze, - sl.SumPool2d, - ), # same as key `1` but with `sl.SumPool2d` instead. - 8: (sl.SumPool2d, nn.Conv2d), # same as key `4` but with `sl.SumPool2d` instead. - 9: (sl.SumPool2d, nn.Linear), # same as key `5` but with `sl.SumPool2d` instead. + # convoluion is always followed by a neuron layer. + 0: (Weight, Neuron), + # Neuron layer can be followed by pooling + 1: (Neuron, Pooling), + # Pooling can be followed by another pooling (will be consolidated) + 2: (Pooling, Pooling), + # Neuron layer can be followed by weight layer of next core + 3: (Neuron, Weight), + # Pooling can be followed by weight layer of next core + 4: (Pooling, Weight), } VALID_SINABS_EDGE_TYPE_IDS = {v: k for k, v in VALID_SINABS_EDGES.items()} -VALID_DYNAPCNNLAYER_EDGES = [ - (sl.iaf.IAFSqueeze, nn.Conv2d), - ( - sl.iaf.IAFSqueeze, - nn.Linear, - ), # `nn.Linear` layers are converted into `nn.Conv2d` by `DynapcnnLayer`. - (nn.AvgPool2d, nn.Conv2d), - (sl.SumPool2d, nn.Conv2d), - ( - nn.AvgPool2d, - nn.Linear, - ), # `nn.Linear` layers are converted into `nn.Conv2d` by `DynapcnnLayer`. - (sl.SumPool2d, nn.Linear), -] +# Between two cores only neuron->weight or pooling->weight connections are possible +VALID_DYNAPCNNLAYER_EDGES = [(Neuron, Weight), (Pooling, Weight)] +# Only `Merge` layers are allowed to join multiple inputs LAYER_TYPES_WITH_MULTIPLE_INPUTS = Union[sl.Merge] -LAYER_TYPES_WITH_MULTIPLE_OUTPUTS = Union[ - sl.IAFSqueeze, sl.SumPool2d, nn.AvgPool2d -] +# Neuron and pooling layers can have their output sent to multiple cores +LAYER_TYPES_WITH_MULTIPLE_OUTPUTS = Union[Neuron, Pooling] diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index c532d967..860082d2 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -92,16 +92,20 @@ def update_dynapcnnlayer_mapper( in 'mapper'. """ - if edge_type in [0, 6]: + if edge_type == 0: # Weight-to-neuron edge, within one core init_xor_complete_new_dynapcnnlayer_blk(mapper, edge, layers) - elif edge_type in [1, 7]: + elif edge_type == 1: # Neuron-to-pooling edge, within one core add_pool_to_dynapcnnlayer_blk(mapper, edge, layers) - elif edge_type in [2, 3, 4, 5, 8, 9]: + elif edge_type == 2: # Pooling-to-pooling edge, will be consolidated within one core + # TODO + NotImplemented + + elif edge_type in [3, 4]: # Neuron-to-weight or Pooling-to-weight edge, connecting two cores connect_dynapcnnlayer_blks(mapper, edge, layers) - else: + else: # This should never happen raise InvalidEdgeType(edge, edge_type) From 9aeb93795f594574384411cba16538897832efd9 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Thu, 3 Oct 2024 14:17:57 +0200 Subject: [PATCH 176/379] Rename module_map to indx_2_module_map for consistency and clarity --- sinabs/backend/dynapcnn/dynapcnn_network.py | 6 ++-- .../backend/dynapcnn/nir_graph_extractor.py | 28 +++++++++---------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index ab230a1b..406bd795 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -62,7 +62,7 @@ def __init__( - self._graph_extractor - self._sinabs_edges - - self._sinabs_modules_map + - self._sinabs_indx_2_module_map - self._nodes_to_dcnnl_map - self._dynapcnn_layers """ @@ -86,7 +86,7 @@ def __init__( # create a dict holding the data necessary to instantiate a `DynapcnnLayer`. self._nodes_to_dcnnl_map = build_nodes_to_dcnnl_map( - layers=self._graph_extractor.modules_map, edges=self._graph_extractor.edges + layers=self._graph_extractor.indx_2_module_map, edges=self._graph_extractor.edges ) # updates 'self._nodes_to_dcnnl_map' to include the I/O shape for each node. @@ -112,7 +112,7 @@ def __init__( # all necessary `DynapcnnLayer` data held in `self._layers_mapper`: removing intermediary data structures no longer necessary. del self._graph_extractor del self._sinabs_edges - del self._sinabs_modules_map + del self._sinabs_indx_2_module_map del self._nodes_to_dcnnl_map del self._dynapcnn_layers diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index b894e8cc..55f05af0 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -41,7 +41,7 @@ def __init__(self, spiking_model: nn.Module, dummy_input: torch.tensor): Values are unique integer IDs. - entry_nodes (set of ints): IDs of nodes acting as entry points for the network, i.e. receiving external input. - - modules_map (dict): + - indx_2_module_map (dict): Map from layer ID to the corresponding nn.Module instance. """ @@ -56,7 +56,7 @@ def __init__(self, spiking_model: nn.Module, dummy_input: torch.tensor): ) # Store the associated `nn.Module` (layer) of each node. - self._modules_map = self._get_named_modules(spiking_model) + self._indx_2_module_map = self._get_named_modules(spiking_model) # Verify that graph is compatible self.verify_graph_integrity() @@ -87,8 +87,8 @@ def sorted_nodes(self) -> List[int]: return [n for n in self._sort_graph_nodes()] @property - def modules_map(self) -> Dict[int, nn.Module]: - return {n: module for n, module in self._modules_map.items()} + def indx_2_module_map(self) -> Dict[int, nn.Module]: + return {n: module for n, module in self._indx_2_module_map.items()} def remove_nodes_by_class( self, node_classes: Tuple[Type] @@ -113,7 +113,7 @@ def remove_nodes_by_class( node: self._find_valid_targets(node, node_classes) for node in self.sorted_nodes # Skip nodes that are to be removed - if not isinstance(self.modules_map[node], node_classes) + if not isinstance(self.indx_2_module_map[node], node_classes) } # remapping nodes indices contiguously starting from 0 @@ -153,7 +153,7 @@ def verify_graph_integrity(self): future to implement stricter formal verification. """ # Iterate over all nodes, and count its sources and targets - for node, module in self.modules_map.items(): + for node, module in self.indx_2_module_map.items(): # Check sources if not isinstance(module, LAYER_TYPES_WITH_MULTIPLE_INPUTS): sources = self._find_all_sources_of_input_to(node) @@ -219,7 +219,7 @@ def _get_named_modules(self, model: nn.Module) -> Dict[int, nn.Module]: Returns ---------- - - modules_map (dict): the mapping between a node (`key` as an `int`) and its module (`value` as a `nn.Module`). + - indx_2_module_map (dict): the mapping between a node (`key` as an `int`) and its module (`value` as a `nn.Module`). """ return { self._name_2_indx_map[name]: module @@ -258,9 +258,9 @@ def _update_internal_representation(self, remapped_nodes: Dict[int, int]): } # Update sinabs module map based on new node indices - self._modules_map = { + self._indx_2_module_map = { remapped_nodes[old_idx]: module - for old_idx, module in self._modules_map.items() + for old_idx, module in self._indx_2_module_map.items() if old_idx in remapped_nodes } @@ -296,7 +296,7 @@ def _get_nodes_io_shapes( # propagate inputs through the nodes. for node in self.sorted_nodes: - if isinstance(self.modules_map[node], sinabs.layers.merge.Merge): + if isinstance(self.indx_2_module_map[node], sinabs.layers.merge.Merge): # find `Merge` arguments (at this point the inputs to Merge should have been calculated). arg1, arg2 = self._find_merge_arguments(node) @@ -312,7 +312,7 @@ def _get_nodes_io_shapes( ) # forward input through the node. - _output = self.modules_map[node](arg1_out, arg2_out) + _output = self.indx_2_module_map[node](arg1_out, arg2_out) # save node's I/O tensors. nodes_io_map[node] = {"input": arg1_out, "output": _output} @@ -321,7 +321,7 @@ def _get_nodes_io_shapes( if node in self._entry_nodes: # forward input dummy through node. - _output = self.modules_map[node](input_dummy) + _output = self.indx_2_module_map[node](input_dummy) # save node's I/O tensors. nodes_io_map[node] = {"input": input_dummy, "output": _output} @@ -332,7 +332,7 @@ def _get_nodes_io_shapes( _input = nodes_io_map[input_node]["output"] # forward input through the node. - _output = self.modules_map[node](_input) + _output = self.indx_2_module_map[node](_input) # save node's I/O tensors. nodes_io_map[node] = {"input": _input, "output": _output} @@ -417,7 +417,7 @@ def _find_valid_targets( for src, tgt in self.edges: # Search for all edges with node as source if src == node: - if isinstance(self.modules_map[tgt], ignored_node_classes): + if isinstance(self.indx_2_module_map[tgt], ignored_node_classes): # Find valid targets of target targets.update(self._find_valid_targets(tgt, ignored_node_classes)) else: From 3b5180c1ccd562a54f673086323145ea774ed080 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Thu, 3 Oct 2024 17:29:48 +0200 Subject: [PATCH 177/379] Node to dcnnl mapping: support pooling-pooling edegs. work independently of edge order --- sinabs/backend/dynapcnn/connectivity_specs.py | 13 +- sinabs/backend/dynapcnn/dynapcnn_network.py | 9 +- sinabs/backend/dynapcnn/exceptions.py | 10 +- .../backend/dynapcnn/nir_graph_extractor.py | 10 +- .../backend/dynapcnn/sinabs_edges_handler.py | 520 ++++++++++-------- sinabs/backend/dynapcnn/utils.py | 50 +- 6 files changed, 315 insertions(+), 297 deletions(-) diff --git a/sinabs/backend/dynapcnn/connectivity_specs.py b/sinabs/backend/dynapcnn/connectivity_specs.py index 6d6b0a1b..e69afea8 100644 --- a/sinabs/backend/dynapcnn/connectivity_specs.py +++ b/sinabs/backend/dynapcnn/connectivity_specs.py @@ -14,19 +14,18 @@ Weight = Union[nn.Conv2d, nn.Linear] Neuron = sl.IAFSqueeze -VALID_SINABS_EDGES = { +VALID_SINABS_EDGE_TYPES = { # convoluion is always followed by a neuron layer. - 0: (Weight, Neuron), + (Weight, Neuron): "weight-neuron", # Neuron layer can be followed by pooling - 1: (Neuron, Pooling), + (Neuron, Pooling): "neuron-pooling", # Pooling can be followed by another pooling (will be consolidated) - 2: (Pooling, Pooling), + (Pooling, Pooling): "pooling-pooling", # Neuron layer can be followed by weight layer of next core - 3: (Neuron, Weight), + (Neuron, Weight): "neuron-weight", # Pooling can be followed by weight layer of next core - 4: (Pooling, Weight), + (Pooling, Weight): "pooling-weight", } -VALID_SINABS_EDGE_TYPE_IDS = {v: k for k, v in VALID_SINABS_EDGES.items()} # Between two cores only neuron->weight or pooling->weight connections are possible VALID_DYNAPCNNLAYER_EDGES = [(Neuron, Weight), (Pooling, Weight)] diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index 406bd795..894bf800 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -17,10 +17,11 @@ from .dynapcnnnetwork_module import DynapcnnNetworkModule from .io import disable_timestamps, enable_timestamps, open_device, reset_timestamps from .nir_graph_extractor import GraphExtractor +from .sinabs_edges_handler import collect_dynapcnn_layer_info from .utils import ( - DEFAULT_IGNORED_LAYER_TYPES, build_from_graph, - build_nodes_to_dcnnl_map, + DEFAULT_IGNORED_LAYER_TYPES, + Edge, parse_device_id, topological_sorting, ) @@ -85,7 +86,7 @@ def __init__( self._graph_extractor.remove_nodes_by_class(DEFAULT_IGNORED_LAYER_TYPES) # create a dict holding the data necessary to instantiate a `DynapcnnLayer`. - self._nodes_to_dcnnl_map = build_nodes_to_dcnnl_map( + self._nodes_to_dcnnl_map = collect_dynapcnn_layer_info( layers=self._graph_extractor.indx_2_module_map, edges=self._graph_extractor.edges ) @@ -568,7 +569,7 @@ def _get_network_module(self) -> Union[list, dict, dict]: topological_sorting(dcnnl_edges), ) - def _get_dynapcnnlayers_edges(self) -> List[Tuple[int, int]]: + def _get_dynapcnnlayers_edges(self) -> List[Edge]: """Create edges representing connections between `DynapcnnLayer` instances. Returns diff --git a/sinabs/backend/dynapcnn/exceptions.py b/sinabs/backend/dynapcnn/exceptions.py index 5fa79c40..b5e3cf86 100644 --- a/sinabs/backend/dynapcnn/exceptions.py +++ b/sinabs/backend/dynapcnn/exceptions.py @@ -1,4 +1,4 @@ -from typing import Tuple, Type +from typing import Set, Tuple, Type class MissingLayer(Exception): @@ -97,6 +97,14 @@ def __init__(self, edge, node): ) +class UnmatchedPoolingEdges(Exception): + def __init__(self, edges: Set[int]): + super().__init__( + "The following edges between pooling layers could not be processed: " + f"{edges}. The computational graph is likely invalid." + ) + + class UnknownNode(Exception): node: int diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index 55f05af0..c4515c3c 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -14,7 +14,7 @@ LAYER_TYPES_WITH_MULTIPLE_OUTPUTS, ) from .exceptions import InvalidGraphStructure -from .utils import topological_sorting +from .utils import Edge, topological_sorting @@ -71,7 +71,7 @@ def entry_nodes(self) -> Set[int]: return {n for n in self._entry_nodes} @property - def edges(self) -> Set[Tuple[int, int]]: + def edges(self) -> Set[Edge]: return {(src, tgt) for src, tgt in self._edges} @property @@ -177,8 +177,8 @@ def verify_graph_integrity(self): def _get_edges_from_nir( self, nir_graph: nirtorch.graph.Graph - ) -> Tuple[List[Tuple[int, int]], Dict[str, int], List[int]]: - """Standardize the representation of `nirtorch.graph.Graph` into a list of edges (`Tuple[int, int]`) where each node in `nir_graph` is represented by an interger (with the source node starting as `0`). + ) -> Tuple[List[Edge], Dict[str, int], List[int]]: + """Standardize the representation of `nirtorch.graph.Graph` into a list of edges (`Edge`) where each node in `nir_graph` is represented by an interger (with the source node starting as `0`). Parameters ---------- @@ -379,7 +379,7 @@ def _find_source_of_input_to(self, node: int) -> int: raise RuntimeError(f"Node {node} has more than 1 input") return sources.pop() - def _find_merge_arguments(self, node: int) -> Tuple[int, int]: + def _find_merge_arguments(self, node: int) -> Edge: """A `Merge` layer receives two inputs. Return the two inputs to `merge_node` representing a `Merge` layer. Returns diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index 860082d2..3c0a5323 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -5,61 +5,100 @@ contact : williansoaresgirao@gmail.com """ -from typing import Dict, List, Tuple, Type +from collections import deque +from typing import Dict, List, Set, Tuple, Type import torch.nn as nn from .exceptions import ( InvalidEdge, - InvalidEdgeType, - InvalidLayerDestination, - InvalidLayerLoop, - MaxDestinationsReached, - UnknownNode, UnmatchedNode, + UnmatchedPoolingEdges, ) -from .connectivity_specs import ( - VALID_DYNAPCNNLAYER_EDGES, - VALID_SINABS_EDGE_TYPE_IDS, -) - - -def process_edge( - layers: Dict[int, nn.Module], - edge: Tuple[int, int], - mapper: Dict[int, Dict[int, Dict]], -) -> None: - """Read in an edge describing the connection between two layers (nodes in the computational graph). If `edge` - is a valid connection between two layers, update `mapper` to incorporate these layers into a new or existing dictonary - containing the modules comprising a future `DynacnnLayer` object. - - After of call of this function `mapper` is updated to incorporate a set of nodes into the data required to create a - `DynapcnnLayer` instance. For example, after processing the 1st edge `(0, 1)`, an entry `0` for a future `DynapcnnLayer` is - created and its set of nodes will include node `0` and node `1`: - - mapper[0] = { - 0: {'layer': Conv2d, 'input_shape': None, 'output_shape': None}, - 1: {'layer': IAFSqueeze, 'input_shape': None, 'output_shape': None}, - ... - } +from .connectivity_specs import VALID_SINABS_EDGE_TYPES +from .utils import Edge + + +def collect_dynapcnn_layer_info( + indx_2_module_map: Dict[int, nn.Module], + edges: Set[Edge], +) -> Dict[int, Dict]: + """Collect information to construct DynapcnnLayer instances. + + Validate and sort edges based on the type of nodes they connect. + Iterate over edges in order of their type. For each neuron->weight edge + generate a new dict to collect information for the corresponding dynapcnn layer. + Then add pooling based on neuron->pooling type edges. Collect additional pooling + from pooling->pooling type edges. Finally set layer destinations based on + neuron/pooling->weight type of edges. Parameters ---------- - layers (dict): a dictionary containing the node IDs of the graph as `key` and their associated module as `value`. - edge (tuple): tuple representing the connection between two nodes in computational graph of 'DynapcnnNetworkGraph.snn.spiking_model'. - mapper (dict): dictionary where each 'key' is the index of a future 'DynapcnnLayer' and 'value' is a dictionary ('key': node, 'value': module). + indx_2_module_map (dict): Maps node IDs of the graph as `key` to their associated module as `value` + edges (set of tuples): Represent connections between two nodes in computational graph + + Returns + ------- + dynapcnn_layer_info (dict): Each 'key' is the index of a future 'DynapcnnLayer' and + 'value' is a dictionary, with keys 'conv', 'neuron', and 'destinations', + containing corresponding node ids and modules required to build the layer """ - edge_type = get_valid_edge_type(edge, layers, VALID_SINABS_EDGE_TYPE_IDS) - - if edge_type is None: - raise InvalidEdge(edge, type(layers[edge[0]]), type(layers[edge[1]])) - - # incorporate modules within the edge to a dict representing a future DynapcnnLayer. - update_dynapcnnlayer_mapper(edge_type, edge, mapper, layers) - + # TODO: Handle DVS layer + + # Sort edges by edge type (type of layers they connect) + edges_by_type: Dict[str, Set[Edge]] = dict() + for edge in edges: + edge_type = get_valid_edge_type(edge, indx_2_module_map, VALID_SINABS_EDGE_TYPES) + + # Validate edge type + if edge_type is None: + raise InvalidEdge( + edge, type(indx_2_module_map[edge[0]]), type(indx_2_module_map[edge[1]]) + ) + + if edge_type in edges_by_type: + edges_by_type[edge_type].add(edge) + else: + edges_by_type[edge_type] = {edge} + + # Dict to collect information for each future dynapcnn layer + dynapcnn_layer_info = dict() + # Map node IDs to dynapcnn layer ID + node_2_layer_map = dict() + + # Each weight->neuron connection instantiates a new, unique dynapcnn layer + while(edges_by_type["weight-neuron"]): + edge = edges_by_type["weight-neuron"].pop() + init_new_dynapcnnlayer_entry(dynapcnn_layer_info, edge, indx_2_module_map, node_2_layer_map) + + # Add pooling based on neuron->pooling connections + while(edges_by_type["neuron-pooling"]): + edge = edges_by_type["neuron-pooling"].pop() + # Search pooling-pooling edges for chains of pooling and add to existing entry + pooling_chains, edges_used = trace_paths(edge[1], edges_by_type["pooling-pooling"]) + add_pooling_to_entry(dynapcnn_layer_info, pooling_chains, indx_2_module_map, node_2_layer_map) + # Remove handled pooling-pooling edges + edges_by_type["pooling-pooling"].difference_update(edges_used) + # After adding pooling make sure all pooling-pooling edges have been handled + if len(edges_by_type["pooling-pooling"]) > 0: + raise UnmatchedPoolingEdges(edges_by_type["pooling-pooling"]) + + # Process all edges connecting two dynapcnn layers + while(edges_by_type["neuron-weight"]): + edge = edges_by_type["neuron-weight"].pop() + set_neuron_layer_destination(dynapcnn_layer_info, edge, node_2_layer_map) + + while(edges_by_type["pooling-weight"]): + edge = edges_by_type["pooling-weight"].pop() + set_pooling_layer_destination(dynapcnn_layer_info, edge, node_2_layer_map) + + # Make sure we have taken care of all edges + assert all(len(edges) == 0 for edges in edges_by_type) + + return dynapcnn_layer_info def get_valid_edge_type( - edge: Tuple[int, int], + edge: Edge, layers: Dict[int, nn.Module], valid_edge_ids: Dict[Tuple[Type, Type], int], ) -> int: @@ -81,212 +120,229 @@ def get_valid_edge_type( return valid_edge_ids.get((source_type, target_type), None) - -def update_dynapcnnlayer_mapper( - edge_type: int, - edge: Tuple[int, int], - mapper: Dict[int, Dict[int, Dict]], - layers: Dict[int, nn.Module], +def init_new_dynapcnnlayer_entry( + dynapcnn_layer_info: Dict[int, Dict[int, Dict]], + edge: Edge, + indx_2_module_map: Dict[int, nn.Module], + node_2_layer_map: Dict[int, int], ) -> None: - """Parses the nodes within an edge and incorporate them either into a **new** or an **already existing** DynapcnnLayer represented - in 'mapper'. - """ + """ Initiate dict to hold information for new dynapcnn layer based on a "weight->neuron" edge. + Change `dynapcnn_layer_info` in-place. - if edge_type == 0: # Weight-to-neuron edge, within one core - init_xor_complete_new_dynapcnnlayer_blk(mapper, edge, layers) - - elif edge_type == 1: # Neuron-to-pooling edge, within one core - add_pool_to_dynapcnnlayer_blk(mapper, edge, layers) - - elif edge_type == 2: # Pooling-to-pooling edge, will be consolidated within one core - # TODO - NotImplemented - - elif edge_type in [3, 4]: # Neuron-to-weight or Pooling-to-weight edge, connecting two cores - connect_dynapcnnlayer_blks(mapper, edge, layers) - - else: # This should never happen - raise InvalidEdgeType(edge, edge_type) - - -def init_xor_complete_new_dynapcnnlayer_blk( - mapper: Dict[int, Dict[int, Dict]], - edge: Tuple[int, int], - layers: Dict[int, nn.Module], -) -> None: - """Incorporates nodes from either a `(conv, neuron)` or a `(linear, neuron)` edge. These are either initiating a new `dict` mapping - into a future `DynapcnnLayer` or completing a `conv->neuron` sequence (in the case the node for `conv` as already been incorporated - somewhere in `mapper`). Obs.: `nn.Linear` layers are converted into `nn.Conv2d` by `DynapcnnLayer`. + Parameters + ---------- + dynapcnn_layer_info: Dict with one entry for each future dynapcnn layer. + key is unique dynapcnn layer ID, value is dict with nodes of the layer + Will be updated in-place. + edge: Tuple of 2 integers, indicating edge between two nodes in graph. + Edge source has to be within an existing entry of `dynapcnn_layer_info`. + indx_2_module_map (dict): Maps node IDs of the graph as `key` to their associated module as `value` + node_2_layer_map (dict): Maps each node ID to the ID of the layer it is assigned to. + Will be updated in-place. """ - # Search for edge[0] (conv/linear layer) in DynapcnnLayers - if (dynapcnnlayer_indx := find_initialized_node(edge[0], mapper)) is not None: - # Add edge[1] (neuron layer) to the same dynapcnn layer - mapper[dynapcnnlayer_indx][edge[1]] = { - "layer": layers[edge[1]], + # Make sure there are no existing entries holding any of the modules connected by `edge` + assert edge[0] not in node_2_layer_map + assert edge[1] not in node_2_layer_map + + # Take current length of the dict as new, unique ID + layer_id = len(dynapcnn_layer_info) + assert layer_id not in dynapcnn_layer_info + + dynapcnn_layer_info[layer_id] = { + "conv": { + "module": indx_2_module_map[edge[0]], + "node_id": edge[0], "input_shape": None, "output_shape": None, - } - else: - # Assign new layer, with current length of `mapper` as new unique index - dynapcnnlayer_indx = len(mapper) - mapper[dynapcnnlayer_indx] = { - edge[0]: { - "layer": layers[edge[0]], - "input_shape": None, - "output_shape": None, - }, - edge[1]: { - "layer": layers[edge[1]], - "input_shape": None, - "output_shape": None, - }, - } - - -def connect_dynapcnnlayer_blks( - mapper: Dict[int, Dict[int, Dict]], - edge: Tuple[int, int], - layers: Dict[int, nn.Module], + }, + "neuron": { + "module": indx_2_module_map[edge[1]], + "node_id": edge[1], + "input_shape": None, + "output_shape": None, + }, + } + node_2_layer_map[edge[0]] = layer_id + node_2_layer_map[edge[1]] = layer_id + +def add_pooling_to_entry( + dynapcnn_layer_info: Dict[int, Dict[int, Dict]], + edge: Edge, + pooling_chains: List[deque[int]], + indx_2_module_map: Dict[int, nn.Module], + node_2_layer_map: Dict[int, int], ) -> None: - """Incorporates nodes from either a `(neuron, conv)/(neuron, lin)` or `(pool, conv)/(pool, lin)` edge. These represent connections between an existing - `dict` in `mapper` that will be mapped into a `DynapcnnLayer` and a new one yet to be represented in `mapper`. Obs.: `nn.Linear` layers are converted - into `nn.Conv2d` by `DynapcnnLayer`. + """Add or extend destination information to existing entry in `dynapcnn_layer_info`. + + Correct entry is identified by existing neuron node. Destination information is a + dict containing list of IDs and list of modules for each chains of pooling nodes. + + Parameters + ---------- + dynapcnn_layer_info: Dict with one entry for each future dynapcnn layer. + key is unique dynapcnn layer ID, value is dict with nodes of the layer + Will be updated in-place. + edge: Tuple of 2 integers, indicating edge between two nodes in graph. + Edge source has to be within an existing entry of `dynapcnn_layer_info`. + pooling_chains: List of deque of int. All sequences ("chains") of connected pooling nodes, + starting from edge[1] + indx_2_module_map (dict): Maps node IDs of the graph as `key` to their associated module as `value` + node_2_layer_map (dict): Maps each node ID to the ID of the layer it is assigned to. + Will be updated in-place. """ - if find_initialized_node(edge[1], mapper) is None: - dynapcnnlayer_indx = 0 - matched = False - for indx, dynapcnnlayer in mapper.items(): - for node, _ in dynapcnnlayer.items(): - if node == edge[0]: # 'edge[0]' is ending DynapcnnLayer block 'indx'. - dynapcnnlayer_indx = indx + 1 - matched = True - break - if matched: - break - if matched: - while dynapcnnlayer_indx in mapper: - dynapcnnlayer_indx += 1 - mapper[dynapcnnlayer_indx] = { # 'edge[1]' starts new DynapcnnLayer block. - edge[1]: { - "layer": layers[edge[1]], - "input_shape": None, - "output_shape": None, - } + # Find layer containing edge[0] + try: + layer_idx = node_2_layer_map[edge[0]] + except KeyError: + raise UnmatchedNode(edge, edge[0]) + # Make sure all pooling chains start with expected node + assert all(chain[0] == edge[1] for chain in pooling_chains) + + # Layer entry might already have `destinations` key (if neuron layer has fanout > 1) + layer_info = dynapcnn_layer_info[layer_idx] + if "destinations" not in layer_info: + layer_info["destinations"] = [] + + # Keep track of all nodes that have been added + new_nodes = set() + + # For each pooling chain initialize new destination + for chain in pooling_chains: + layer_info["destinations"].append( + { + "pooling_ids": chain, + "pooling_modules": [indx_2_module_map[idx] for idx in chain], } - else: - raise UnmatchedNode(edge, node) - - -def add_pool_to_dynapcnnlayer_blk( - mapper: Dict[int, Dict[int, Dict]], - edge: Tuple[int, int], - layers: Dict[int, nn.Module], + ) + new_nodes.update(set(chain)) + + for node in new_nodes: + # Make sure new pooling nodes have not been used elsewhere + assert node not in node_2_layer_map + node_2_layer_map[node] = layer_idx + +def set_neuron_layer_destination( + dynapcnn_layer_info: Dict[int, Dict[int, Dict]], + edge: Edge, + node_2_layer_map: Dict[int, int], ) -> None: - """Incorporating a `(neuron, pool)` edge. Node `pool` has to be part of an already existing `dict` mapping into a `DynapcnnLaye` in `mapper`.""" - # Search for edge[0] (neuron layer) in DynapcnnLayers - if (indx := find_initialized_node(edge[0], mapper)) is not None: - # Add edge[1] (pooling layer) to the same dynapcnn layer - mapper[indx][edge[1]] = { - "layer": layers[edge[1]], - "input_shape": None, - "output_shape": None, - } - else: - raise UnmatchedNode(edge, edge[1]) + """ Set destination layer without pooling. + Parameters + ---------- + dynapcnn_layer_info: Dict with one entry for each future dynapcnn layer. + key is unique dynapcnn layer ID, value is dict with nodes of the layer + Will be updated in-place. + edge: Tuple of 2 integers, indicating edge between two nodes in graph. + Edge source has to be within an existing entry of `dynapcnn_layer_info`. + node_2_layer_map (dict): Maps each node ID to the ID of the layer it is assigned to. + Will be updated in-place. + """ + # Make sure both source (neuron layer) and target (weight layer) have been previously processed + try: + source_layer_idx = node_2_layer_map[edge[0]] + except KeyError: + raise UnmatchedNode(edge, edge[0]) + try: + destination_layer_idx = node_2_layer_map[edge[1]] + except KeyError: + raise UnmatchedNode(edge, edge[1]) + + # Source layer entry might already have `destinations` key (if neuron layer has fanout > 1) + layer_info = dynapcnn_layer_info[source_layer_idx] + if "destinations" not in layer_info: + layer_info["destinations"] = [] + + # Add new destination + layer_info["destinations"].append( + { + "pooling_ids": [], + "pooling_modules": [], + "destination_layer": destination_layer_idx, + } + ) -def find_initialized_node(node: int, mapper: Dict[int, Dict[int, Dict]]) -> bool: - """Finds if 'node' existis within 'mapper' and returns layer index.""" - for index, dynapcnnlayer in mapper.items(): - if node in dynapcnnlayer: - return index - return None - +def set_pooling_layer_destination( + dynapcnn_layer_info: Dict[int, Dict[int, Dict]], + edge: Edge, + indx_2_module_map: Dict[int, nn.Module], + node_2_layer_map: Dict[int, int], +) -> None: + """ Set destination layer with pooling. -def get_dynapcnnlayers_destinations( - layers: Dict[int, nn.Module], - edges: List[Tuple[int, int]], - mapper: Dict[int, Dict[int, Dict]], -) -> dict: - """Loops over the edges list describing the computational graph. It will access each node in the graph and find to which - DynapcnnLayer they belong to. If source and target belong to different DynapcnnLayers (described as a dictionary in 'mapper') - the destination of the 'DynapcnnLayer.source' is set to be 'DynapcnnLayer.target'. - - After one call of this function an attribute `destination` is added to an entry in `mapper` to save the indexes (a different `key` - in `mapper`) of `DynapcnnLayer`s targeted by another `DynapcnnLayer`. For example, if in an edge `(1, 4)` the node `1` belongs to - `mapper[0]` and node `4` belongs to `mapper[2]`, the former is updated to tager the latter, like the following: - - mapper[0] = { - 0: {'layer': Conv2d, ...}, - 1: {'layer': IAFSqueeze, ...}, # node `1` in edge `(1, 4)` belongs to `mapper[0]`... - ... - 'destinations': [2], # ... so DynacnnLayer built from `mapper[2]` is destination of DynapcnnLayer built from `mapper[0]`. - ... - } + Parameters + ---------- + dynapcnn_layer_info: Dict with one entry for each future dynapcnn layer. + key is unique dynapcnn layer ID, value is dict with nodes of the layer + Will be updated in-place. + edge: Tuple of 2 integers, indicating edge between two nodes in graph. + Edge source has to be within an existing entry of `dynapcnn_layer_info`. + node_2_layer_map (dict): Maps each node ID to the ID of the layer it is assigned to. + Will be updated in-place. + """ + # Make sure both source (pooling layer) and target (weight layer) have been previously processed + try: + source_layer_idx = node_2_layer_map[edge[0]] + except KeyError: + raise UnmatchedNode(edge, edge[0]) + try: + destination_layer_idx = node_2_layer_map[edge[1]] + except KeyError: + raise UnmatchedNode(edge, edge[1]) + + # Source layer entry should already have `destinations` key + layer_info = dynapcnn_layer_info[source_layer_idx] + + # Find current source node within destinations + matched = False + for destination in layer_info["destinations"]: + if destination["pooling_ids"][-1] == edge[0]: + matched = True + break + if not matched: + raise UnmatchedNode(edge, edge[0]) + + # Set destination layer + destination["destination_layer"] = destination_layer_idx + +def trace_paths(node: int, remaining_edges: Set[Edge]) -> List[deque[int]]: + """ Trace any path of collected edges through the graph. + + Start with `node`, and recursively look for paths of connected nodes + within `remaining edges.` Parameters ---------- - layers (dict): contains the nodes of the graph as `key` and their associated module as `value`. - edges (list): tuples representing the connection between nodes in computational graph spiking network. - mapper (dict): each 'key' is the index of a future `DynapcnnLayer` and `value` the data necessary to instantiate it. + node (int): ID of current node + remaining_edges: Set of remaining edges still to be searched Returns - ---------- - dynapcnnlayers_destinations_map: dictionary where each 'key' is the index of a future 'DynapcnnLayer' and 'value' is its list of destinations (DynapcnnLayers). + ------- + paths: List of deque of int, all paths of connected edges starting from `node`. + processed_edges: Set of edges that are part of the returned paths """ - dynapcnnlayers_destinations_map = {} - used_layer_edges = [] + paths = [] + processed_edges = set() + for (src, tgt) in remaining_edges: + if src == node: + processed_edges.add((src, tgt)) + # For each edge with `node` as source, find subsequent pooling nodes recursively + new_remaining = remaining_edges.difference({(src, tgt)}) + branches, new_processed = trace_paths(tgt, new_remaining) + # Make sure no edge was processed twice + assert len(processed_edges.intersection(new_processed)) == 0 + + # Keep track of newly processed edges + processed_edges.update(new_processed) + + # Collect all branching paths of pooling, inserting src at beginning + for branch in branches: + branch.appendleft(src) + paths.append(branch) + + if not paths: + # End of recursion: instantiate a deque only with node + paths = [deque([node])] + + return paths, processed_edges - for edge in edges: - source_layer = get_dynapcnnlayer_index(edge[0], mapper) - destination_layer = get_dynapcnnlayer_index(edge[1], mapper) - - if source_layer not in dynapcnnlayers_destinations_map: - dynapcnnlayers_destinations_map[source_layer] = [] - - if source_layer != destination_layer and is_valid_dynapcnnlayer_pairing( - layers, edge, VALID_DYNAPCNNLAYER_EDGES - ): - # valid connection between modules in two different DynapcnnLayer. - - if len(dynapcnnlayers_destinations_map[source_layer]) > 2: - # DynapcnnLayers can not have more than two destinations. - raise MaxDestinationsReached(source_layer) - else: - if ( - (destination_layer, source_layer) not in used_layer_edges - and destination_layer - not in dynapcnnlayers_destinations_map[source_layer] - ): - # edge does not create a loop between layers. - dynapcnnlayers_destinations_map[source_layer].append( - destination_layer - ) - used_layer_edges.append((source_layer, destination_layer)) - else: - raise InvalidLayerLoop(source_layer, destination_layer) - - for dcnnl_idx, destinations in dynapcnnlayers_destinations_map.items(): - # TODO document the 'rescale_factor' better. - mapper[dcnnl_idx]["destinations"] = destinations - mapper[dcnnl_idx]["conv_rescale_factor"] = [] - - -def get_dynapcnnlayer_index(node: int, mapper: Dict[int, Dict[int, Dict]]) -> int: - """Returns the DynapcnnLayer index to which 'node' belongs to.""" - for indx, dynapcnnlayer in mapper.items(): - if node in dynapcnnlayer: - return indx - raise UnknownNode(node) - - -def is_valid_dynapcnnlayer_pairing( - layers: Dict[int, nn.Module], - edge: Tuple[int, int], - valid_dynapcnnlayer_edges: List[Tuple[nn.Module, nn.Module]], -) -> bool: - """Checks if the module in 'DynapcnnLayer.source' is targetting a valid module in 'DynapcnnLayer.target'.""" - if (type(layers[edge[0]]), type(layers[edge[1]])) in valid_dynapcnnlayer_edges: - return True - else: - raise InvalidLayerDestination(type(layers[edge[0]]), type(layers[edge[1]])) diff --git a/sinabs/backend/dynapcnn/utils.py b/sinabs/backend/dynapcnn/utils.py index 0eb0378a..faabc76e 100644 --- a/sinabs/backend/dynapcnn/utils.py +++ b/sinabs/backend/dynapcnn/utils.py @@ -13,13 +13,14 @@ from .dynapcnn_layer_handler import DynapcnnLayerHandler from .exceptions import WrongPoolingModule from .flipdims import FlipDims -from .sinabs_edges_handler import get_dynapcnnlayers_destinations, process_edge if TYPE_CHECKING: from sinabs.backend.dynapcnn.dynapcnn_network import DynapcnnNetwork DEFAULT_IGNORED_LAYER_TYPES = Union[nn.Identity, nn.Dropout, nn.Dropout2d, nn.Flatten, sl.Merge] +Edge = Tuple[int, int] # Define edge-type alias + ####################################################### Device Related ####################################################### @@ -76,53 +77,6 @@ def standardize_device_id(device_id: str) -> str: ####################################################### DynapcnnNetwork Related ####################################################### - -def build_nodes_to_dcnnl_map( - layers: Dict[int, nn.Module], edges: List[Tuple[int, int]] -) -> dict: - """Initializes and populates a `dict` that will map data into a future `DynapcnnLayer` instance. The call - to `process_edge()` initializes a `key` (the index of a `DynapcnnLayer`) and assigns to it a dict containing the - nodes (layers in a `nn.Module`) that should belong to the same `DynapcnnLayer`. The call to `get_dynapcnnlayers_destinations()` - further incorporates to each "DynapcnnLayer dictionary" a `destinations` attribute, which is a list of integers indicating the - the target destinations of a `DynapcnnLayer` instance. - - Parameters - --------- - - layers (dict): contains the node IDs of a graph as `key` and their associated module as `value`. - - edges (list): edges describing how nodes connect to each other. - - Returns - --------- - - nodes_to_dcnnl_map (dict): each entry represents the gathered data necessary to instantiate a `DynapcnnLayer` object (e.g. nodes, - their I/O shapes, the list of `DynapcnnLayer` that are to be targeted, etc). - """ - # @TODO the graph extraction is not yet considering DVS input. - - # dvs_layer, lyr_indx_next, rescale_factor = construct_dvs_layer( - # layers, - # input_shape=in_shape, - # idx_start=0, - # dvs_input=False) - - dvs_layer = None - - # mapper from nodes to sets of layers that populate a DynapcnnLayer. - nodes_to_dcnnl_map = {} - - if dvs_layer is not None: - # TODO the graph extraction is not yet considering DVS input. - pass - else: - for edge in edges: - # Figure out to which (future) DynapcnnLayer each node will belong to. - process_edge(layers, edge, nodes_to_dcnnl_map) - - # look for edges between connecting nodes in different (future) DynapcnnLayer. - get_dynapcnnlayers_destinations(layers, edges, nodes_to_dcnnl_map) - - return nodes_to_dcnnl_map - - def build_from_graph( discretize: bool, edges: List[Tuple[int, int]], From c3b5fcfd5efb97c91e4e10438f0c3d2c79a46c2f Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Thu, 3 Oct 2024 18:05:03 +0200 Subject: [PATCH 178/379] Fix bugs from previous commit --- sinabs/backend/dynapcnn/connectivity_specs.py | 20 +++++++++++-------- sinabs/backend/dynapcnn/dynapcnn_network.py | 4 ++-- .../backend/dynapcnn/sinabs_edges_handler.py | 14 +++++++++---- 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/sinabs/backend/dynapcnn/connectivity_specs.py b/sinabs/backend/dynapcnn/connectivity_specs.py index e69afea8..c3a6e55c 100644 --- a/sinabs/backend/dynapcnn/connectivity_specs.py +++ b/sinabs/backend/dynapcnn/connectivity_specs.py @@ -10,11 +10,11 @@ import sinabs.layers as sl -Pooling = Union[sl.SumPool2d, nn.AvgPool2d] -Weight = Union[nn.Conv2d, nn.Linear] -Neuron = sl.IAFSqueeze +Pooling = (sl.SumPool2d, nn.AvgPool2d) +Weight = (nn.Conv2d, nn.Linear) +Neuron = (sl.IAFSqueeze, ) -VALID_SINABS_EDGE_TYPES = { +VALID_SINABS_EDGE_TYPES_ABSTRACT = { # convoluion is always followed by a neuron layer. (Weight, Neuron): "weight-neuron", # Neuron layer can be followed by pooling @@ -27,12 +27,16 @@ (Pooling, Weight): "pooling-weight", } -# Between two cores only neuron->weight or pooling->weight connections are possible -VALID_DYNAPCNNLAYER_EDGES = [(Neuron, Weight), (Pooling, Weight)] +# Unpack dict +VALID_SINABS_EDGE_TYPES = { + (source_type, target_type): name + for types, name in VALID_SINABS_EDGE_TYPES_ABSTRACT.items() + for source_type in types[0] + for target_type in types[1] +} # Only `Merge` layers are allowed to join multiple inputs LAYER_TYPES_WITH_MULTIPLE_INPUTS = Union[sl.Merge] # Neuron and pooling layers can have their output sent to multiple cores -LAYER_TYPES_WITH_MULTIPLE_OUTPUTS = Union[Neuron, Pooling] - +LAYER_TYPES_WITH_MULTIPLE_OUTPUTS = Union[*Neuron, *Pooling] diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index 894bf800..ee5c33d5 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -87,7 +87,7 @@ def __init__( # create a dict holding the data necessary to instantiate a `DynapcnnLayer`. self._nodes_to_dcnnl_map = collect_dynapcnn_layer_info( - layers=self._graph_extractor.indx_2_module_map, edges=self._graph_extractor.edges + self._graph_extractor.indx_2_module_map, self._graph_extractor.edges ) # updates 'self._nodes_to_dcnnl_map' to include the I/O shape for each node. @@ -96,7 +96,7 @@ def __init__( # build `DynapcnnLayer` instances from graph edges and mapper. self._dynapcnn_layers, self._dynapcnnlayers_handlers = build_from_graph( discretize=discretize, - edges=self._sinabs_edges, + edges=self._graph_extractor.edges, nodes_to_dcnnl_map=self._nodes_to_dcnnl_map, weight_rescaling_fn=weight_rescaling_fn, entry_nodes=self._graph_extractor._entry_nodes, diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index 3c0a5323..24634dc2 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -60,7 +60,7 @@ def collect_dynapcnn_layer_info( edges_by_type[edge_type].add(edge) else: edges_by_type[edge_type] = {edge} - + # Dict to collect information for each future dynapcnn layer dynapcnn_layer_info = dict() # Map node IDs to dynapcnn layer ID @@ -71,12 +71,19 @@ def collect_dynapcnn_layer_info( edge = edges_by_type["weight-neuron"].pop() init_new_dynapcnnlayer_entry(dynapcnn_layer_info, edge, indx_2_module_map, node_2_layer_map) + # "pooling-pooling" edges are optional. Unlike other types, missing entry would cause exception. + # Therefore add empty set if not existing + if "pooling-pooling" not in edges_by_type: + edges_by_type["pooling-pooling"] = set() + # Add pooling based on neuron->pooling connections while(edges_by_type["neuron-pooling"]): edge = edges_by_type["neuron-pooling"].pop() # Search pooling-pooling edges for chains of pooling and add to existing entry pooling_chains, edges_used = trace_paths(edge[1], edges_by_type["pooling-pooling"]) - add_pooling_to_entry(dynapcnn_layer_info, pooling_chains, indx_2_module_map, node_2_layer_map) + add_pooling_to_entry( + dynapcnn_layer_info, edge, pooling_chains, indx_2_module_map, node_2_layer_map + ) # Remove handled pooling-pooling edges edges_by_type["pooling-pooling"].difference_update(edges_used) # After adding pooling make sure all pooling-pooling edges have been handled @@ -93,7 +100,7 @@ def collect_dynapcnn_layer_info( set_pooling_layer_destination(dynapcnn_layer_info, edge, node_2_layer_map) # Make sure we have taken care of all edges - assert all(len(edges) == 0 for edges in edges_by_type) + assert all(len(edges) == 0 for edges in edges_by_type.values()) return dynapcnn_layer_info @@ -265,7 +272,6 @@ def set_neuron_layer_destination( def set_pooling_layer_destination( dynapcnn_layer_info: Dict[int, Dict[int, Dict]], edge: Edge, - indx_2_module_map: Dict[int, nn.Module], node_2_layer_map: Dict[int, int], ) -> None: """ Set destination layer with pooling. From 088157b06a53001e695ca38592c6fc26e2f57c14 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Fri, 4 Oct 2024 11:10:38 +0200 Subject: [PATCH 179/379] Try reducing io shape extraction effort --- sinabs/backend/dynapcnn/dynapcnn_network.py | 10 +++++-- .../backend/dynapcnn/nir_graph_extractor.py | 4 ++- .../backend/dynapcnn/sinabs_edges_handler.py | 29 ++++++++++++------- 3 files changed, 28 insertions(+), 15 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index ee5c33d5..eaa0f60a 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -87,11 +87,15 @@ def __init__( # create a dict holding the data necessary to instantiate a `DynapcnnLayer`. self._nodes_to_dcnnl_map = collect_dynapcnn_layer_info( - self._graph_extractor.indx_2_module_map, self._graph_extractor.edges + self._graph_extractor.indx_2_module_map, + self._graph_extractor.edges, + self._graph_extractor.nodes_io_shapes, ) - # updates 'self._nodes_to_dcnnl_map' to include the I/O shape for each node. - self._populate_nodes_io() + # TODO: Try avoiding this step by only retrieving input shapes of conv layers + # while constractung dcnnl_map + # # updates 'self._nodes_to_dcnnl_map' to include the I/O shape for each node. + # self._populate_nodes_io() # build `DynapcnnLayer` instances from graph edges and mapper. self._dynapcnn_layers, self._dynapcnnlayers_handlers = build_from_graph( diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index c4515c3c..d83cc968 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -43,6 +43,8 @@ def __init__(self, spiking_model: nn.Module, dummy_input: torch.tensor): IDs of nodes acting as entry points for the network, i.e. receiving external input. - indx_2_module_map (dict): Map from layer ID to the corresponding nn.Module instance. + - nodes_io_shapes (dict): + Map from node ID to dict containing node's in- and output shapes """ # extract computational graph. @@ -79,7 +81,7 @@ def name_2_indx_map(self) -> Dict[str, int]: return {name: idx for name, idx in self._name_2_indx_map.items()} @property - def nodes_io_shapes(self) -> Dict[int, torch.Size]: + def nodes_io_shapes(self) -> Dict[int, Tuple[torch.Size]]: return {n: size for n, size in self._nodes_io_shapes.items()} @property diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index 24634dc2..3a7c27ac 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -8,7 +8,7 @@ from collections import deque from typing import Dict, List, Set, Tuple, Type -import torch.nn as nn +from torch import nn, Size from .exceptions import ( InvalidEdge, @@ -22,6 +22,7 @@ def collect_dynapcnn_layer_info( indx_2_module_map: Dict[int, nn.Module], edges: Set[Edge], + nodes_io_shapes: Dict[int, Dict[str, Tuple[Size, Size]]], ) -> Dict[int, Dict]: """Collect information to construct DynapcnnLayer instances. @@ -34,14 +35,15 @@ def collect_dynapcnn_layer_info( Parameters ---------- - indx_2_module_map (dict): Maps node IDs of the graph as `key` to their associated module as `value` - edges (set of tuples): Represent connections between two nodes in computational graph + indx_2_module_map (dict): Maps node IDs of the graph as `key` to their associated module as `value` + edges (set of tuples): Represent connections between two nodes in computational graph + nodes_io_shapes (dict): Map from node ID to dict containing node's in- and output shapes Returns ------- - dynapcnn_layer_info (dict): Each 'key' is the index of a future 'DynapcnnLayer' and - 'value' is a dictionary, with keys 'conv', 'neuron', and 'destinations', - containing corresponding node ids and modules required to build the layer + dynapcnn_layer_info (dict): Each 'key' is the index of a future 'DynapcnnLayer' and + 'value' is a dictionary, with keys 'conv', 'neuron', and 'destinations', + containing corresponding node ids and modules required to build the layer """ # TODO: Handle DVS layer @@ -69,7 +71,13 @@ def collect_dynapcnn_layer_info( # Each weight->neuron connection instantiates a new, unique dynapcnn layer while(edges_by_type["weight-neuron"]): edge = edges_by_type["weight-neuron"].pop() - init_new_dynapcnnlayer_entry(dynapcnn_layer_info, edge, indx_2_module_map, node_2_layer_map) + init_new_dynapcnnlayer_entry( + dynapcnn_layer_info, + edge, + indx_2_module_map, + nodes_io_shapes, + node_2_layer_map + ) # "pooling-pooling" edges are optional. Unlike other types, missing entry would cause exception. # Therefore add empty set if not existing @@ -131,6 +139,7 @@ def init_new_dynapcnnlayer_entry( dynapcnn_layer_info: Dict[int, Dict[int, Dict]], edge: Edge, indx_2_module_map: Dict[int, nn.Module], + nodes_io_shapes: Dict[int, Dict[str, Tuple[Size, Size]]], node_2_layer_map: Dict[int, int], ) -> None: """ Initiate dict to hold information for new dynapcnn layer based on a "weight->neuron" edge. @@ -144,6 +153,7 @@ def init_new_dynapcnnlayer_entry( edge: Tuple of 2 integers, indicating edge between two nodes in graph. Edge source has to be within an existing entry of `dynapcnn_layer_info`. indx_2_module_map (dict): Maps node IDs of the graph as `key` to their associated module as `value` + nodes_io_shapes (dict): Map from node ID to dict containing node's in- and output shapes node_2_layer_map (dict): Maps each node ID to the ID of the layer it is assigned to. Will be updated in-place. """ @@ -159,14 +169,11 @@ def init_new_dynapcnnlayer_entry( "conv": { "module": indx_2_module_map[edge[0]], "node_id": edge[0], - "input_shape": None, - "output_shape": None, + "input_shape": nodes_io_shapes[edge[0]], }, "neuron": { "module": indx_2_module_map[edge[1]], "node_id": edge[1], - "input_shape": None, - "output_shape": None, }, } node_2_layer_map[edge[0]] = layer_id From b4fabcf3af26261ed71a8de15c5708f12b357933 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Fri, 4 Oct 2024 18:28:49 +0200 Subject: [PATCH 180/379] (WIP) Update dynapcnn layer instantiation --- .../backend/dynapcnn/sinabs_edges_handler.py | 6 +- sinabs/backend/dynapcnn/utils.py | 114 ++++++++++++++++-- 2 files changed, 110 insertions(+), 10 deletions(-) diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index 3a7c27ac..b1827dcf 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -166,15 +166,17 @@ def init_new_dynapcnnlayer_entry( assert layer_id not in dynapcnn_layer_info dynapcnn_layer_info[layer_id] = { + "input_shape": nodes_io_shapes[edge[0]], "conv": { "module": indx_2_module_map[edge[0]], "node_id": edge[0], - "input_shape": nodes_io_shapes[edge[0]], }, "neuron": { "module": indx_2_module_map[edge[1]], "node_id": edge[1], }, + # This will be used later to account for average pooling in preceding layers + "rescale_factors": {}, } node_2_layer_map[edge[0]] = layer_id node_2_layer_map[edge[1]] = layer_id @@ -315,7 +317,7 @@ def set_pooling_layer_destination( if not matched: raise UnmatchedNode(edge, edge[0]) - # Set destination layer + # Set destination layer within destination dict that holds current source node destination["destination_layer"] = destination_layer_idx def trace_paths(node: int, remaining_edges: Set[Edge]) -> List[deque[int]]: diff --git a/sinabs/backend/dynapcnn/utils.py b/sinabs/backend/dynapcnn/utils.py index faabc76e..c417e57b 100644 --- a/sinabs/backend/dynapcnn/utils.py +++ b/sinabs/backend/dynapcnn/utils.py @@ -273,20 +273,118 @@ def construct_layerhandler( return layerhandler -def construct_dynapcnnlayer(handler: DynapcnnLayerHandler) -> DynapcnnLayer: +def construct_all_dynapcnnlayers(dcnnl_map: Dict, discretize: bool, rescale_fn: Optional[Callable] = None) -> Dict[int, DynapcnnLayer]: """...""" + # Extract construction arguments from dcnnl_map + # -conv layer + # -neuron layer + # -pooling -> requires consolidation + # -input shape + # -discretize + # -weight rescale factor + + # Consolidate pooling information for each destination + for layer_info in dcnnl_map.values(): + for destination in layer_info["destinations"]: + pool, scale = consolidate_pooling(destination["pooling_modules"]) + destination["cumulative_pooling"] = pool + destination["cumulative_scaling"] = scale + dest_lyr_idx = destination["destination_layer"] + dcnnl_map[dest_lyr_idx]["rescale_factors"].add(layer_rescaling) + + dynapcnn_layer = { + layer_idx: construct_single_dynapcnn_layer(layer_info, discretize, rescale_fn) + for layer_idx, layer_info in dcnnl_map.items() + } + + dynapcnn_layer_handler = { + layer_idx: construct_single_dynapcnn_layer_handler(layer_info) + for layer_idx, layer_info in dcnnl_map.items() + } + +def construct_single_dynapcnn_layer(layer_info: Dict, rescale_fn: Optional[Callable] = None) -> DynapcnnLayer: + + if len(layer_info["rescale_factors"]) == 0: + rescale_factor = 1 + elif len(layer_info["rescale_factors"]) == 1: + rescale_factor = layer_info["rescale_factors"].pop() + else: + if rescale_fn is None: + # TODO: Custom Exception class? + raise ValueError( + "Average pooling layers of conflicting sizes pointing to " + "same destination. Either replace them by SumPool2d layers " + "or provide a `rescale_fn` to resolve this" + ) + else: + rescale_factor = rescale_fn(layer_info["rescale_factors"]) + # instantiate a DynapcnnLayer from the data in the handler. - dynapcnnlayer = DynapcnnLayer( - conv=handler.conv_layer, - spk=handler.spk_layer, - in_shape=handler.conv_in_shape, - pool=handler.get_pool_list(), - discretize=False, + return DynapcnnLayer( + conv=layer_info["conv"]["module"], + spk=layer_info["neuron"]["module"], + in_shape=layer_info["input_shape"], + pool=[dest["cumulative_pooling"] for dest in layer_info["destinations"], + discretize=discretize, + rescale_weights=rescale_factor, ) - return dynapcnnlayer +def consolidate_pooling(modules: Iterable[nn.Module]) -> Tuple[Tuple[int, int], float]: + """ Consolidate pooling information for consecutive pooling modules. + + Parameters + ---------- + modules: Iteravle of pooling modules + + Returns + ------- + cumulative_pooling: Tuple of two ints, indicating pooling along + vertical and horizontal dimensions for all modules together + cumulative_scaling: float, indicating by how much subsequent weights + need to be rescaled to account for average pooling being converted + to sum pooling, considering all provided modules. + """ + cumulative_pooling = [1, 1] + cumulative_scaling = 1. + + for pooling_layer in modules: + pooling, rescale_factor = extract_pooling_from_module(pooling_layer) + cumulative_pooling[0] *= pooling[0] + cumulative_pooling[1] *= pooling[1] + cumulative_scaling *= rescale_factor + + return cumulative_pooling, cumulative_scaling + +def extract_pooling_from_module(module: Union[nn.AvgPool2d, sl.SumPool2d]) -> Tuple[Tuple[int, int], float]: + """ Extract pooling size and required rescaling factor from pooling module + + Parameters + ---------- + module: pooling module + + Returns + ------- + pooling: Tuple of two ints, indicating pooling along vertical and horizontal dimensions + scale_factor: float, indicating by how much subsequent weights need to be rescaled to + account for average pooling being converted to sum pooling. + """ + pooling = expand_to_pair(module.kernel_size) + + if module.stride is not None: + stride = expand_to_pair(module.stride) + if pooling != stride: + raise ValueError( + f"Stride length {module.stride} should be the same as pooling kernel size {module.kernel_size}" + ) + if isinstance(pooling_layer, nn.AvgPool2d): + scale_factor = 1. / (pooling[0] * pooling[1]) + elif isinstance(pooling_layer, sl.SumPool2d): + scale_factor = 1. + else: + raise ValueError(f"Unsupported type {type(module)} for pooling layer") + return pooling, scale_factor def convert_Avg_to_Sum_pooling( dcnnl_data: Dict[ From a44d40eeb4c8a553e3de508c65e7dcff269df8c0 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Fri, 4 Oct 2024 18:29:38 +0200 Subject: [PATCH 181/379] Fix non-optimal unpacking syntax --- sinabs/backend/dynapcnn/connectivity_specs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sinabs/backend/dynapcnn/connectivity_specs.py b/sinabs/backend/dynapcnn/connectivity_specs.py index c3a6e55c..cc14b1e3 100644 --- a/sinabs/backend/dynapcnn/connectivity_specs.py +++ b/sinabs/backend/dynapcnn/connectivity_specs.py @@ -39,4 +39,4 @@ LAYER_TYPES_WITH_MULTIPLE_INPUTS = Union[sl.Merge] # Neuron and pooling layers can have their output sent to multiple cores -LAYER_TYPES_WITH_MULTIPLE_OUTPUTS = Union[*Neuron, *Pooling] +LAYER_TYPES_WITH_MULTIPLE_OUTPUTS = Union[(*Neuron, *Pooling)] From 61954616b1064843aff154bdee5a7e265a9d0d98 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Fri, 4 Oct 2024 18:31:32 +0200 Subject: [PATCH 182/379] Fix syntax bug --- sinabs/backend/dynapcnn/utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sinabs/backend/dynapcnn/utils.py b/sinabs/backend/dynapcnn/utils.py index c417e57b..abf4307d 100644 --- a/sinabs/backend/dynapcnn/utils.py +++ b/sinabs/backend/dynapcnn/utils.py @@ -320,12 +320,15 @@ def construct_single_dynapcnn_layer(layer_info: Dict, rescale_fn: Optional[Calla else: rescale_factor = rescale_fn(layer_info["rescale_factors"]) + # Collect pooling in a list + [dest["cumulative_pooling"] for dest in layer_info["destinations"]] + # instantiate a DynapcnnLayer from the data in the handler. return DynapcnnLayer( conv=layer_info["conv"]["module"], spk=layer_info["neuron"]["module"], in_shape=layer_info["input_shape"], - pool=[dest["cumulative_pooling"] for dest in layer_info["destinations"], + pool=pooling_list, discretize=discretize, rescale_weights=rescale_factor, ) From 5ec0ae52f86880b98bf750de2059f697390677c4 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Fri, 4 Oct 2024 18:33:49 +0200 Subject: [PATCH 183/379] Fix indentation --- sinabs/backend/dynapcnn/utils.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/sinabs/backend/dynapcnn/utils.py b/sinabs/backend/dynapcnn/utils.py index abf4307d..16a9d8f3 100644 --- a/sinabs/backend/dynapcnn/utils.py +++ b/sinabs/backend/dynapcnn/utils.py @@ -372,22 +372,22 @@ def extract_pooling_from_module(module: Union[nn.AvgPool2d, sl.SumPool2d]) -> Tu scale_factor: float, indicating by how much subsequent weights need to be rescaled to account for average pooling being converted to sum pooling. """ - pooling = expand_to_pair(module.kernel_size) + pooling = expand_to_pair(module.kernel_size) - if module.stride is not None: - stride = expand_to_pair(module.stride) - if pooling != stride: - raise ValueError( - f"Stride length {module.stride} should be the same as pooling kernel size {module.kernel_size}" - ) - if isinstance(pooling_layer, nn.AvgPool2d): - scale_factor = 1. / (pooling[0] * pooling[1]) - elif isinstance(pooling_layer, sl.SumPool2d): - scale_factor = 1. - else: - raise ValueError(f"Unsupported type {type(module)} for pooling layer") + if module.stride is not None: + stride = expand_to_pair(module.stride) + if pooling != stride: + raise ValueError( + f"Stride length {module.stride} should be the same as pooling kernel size {module.kernel_size}" + ) + if isinstance(pooling_layer, nn.AvgPool2d): + scale_factor = 1. / (pooling[0] * pooling[1]) + elif isinstance(pooling_layer, sl.SumPool2d): + scale_factor = 1. + else: + raise ValueError(f"Unsupported type {type(module)} for pooling layer") - return pooling, scale_factor + return pooling, scale_factor def convert_Avg_to_Sum_pooling( dcnnl_data: Dict[ From a173f1aa840163223acef7ccabdb330c1baad017 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Fri, 4 Oct 2024 18:34:14 +0200 Subject: [PATCH 184/379] Run black and isort --- sinabs/activation/__init__.py | 9 +- sinabs/backend/dynapcnn/__init__.py | 16 +- sinabs/backend/dynapcnn/chip_factory.py | 11 +- sinabs/backend/dynapcnn/chips/dynapcnn.py | 71 ++- sinabs/backend/dynapcnn/chips/speck2cmini.py | 10 +- sinabs/backend/dynapcnn/chips/speck2e.py | 12 +- sinabs/backend/dynapcnn/chips/speck2f.py | 12 +- sinabs/backend/dynapcnn/connectivity_specs.py | 4 +- sinabs/backend/dynapcnn/dynapcnn_network.py | 16 +- sinabs/backend/dynapcnn/exceptions.py | 8 +- .../backend/dynapcnn/nir_graph_extractor.py | 13 +- .../backend/dynapcnn/sinabs_edges_handler.py | 105 +++-- sinabs/backend/dynapcnn/utils.py | 38 +- sinabs/from_torch.py | 8 +- tests/test_activations.py | 10 +- tests/test_dynapcnn/test_auto_mapping.py | 3 +- .../test_compatible_layer_build.py | 3 +- tests/test_dynapcnn/test_device_movement.py | 3 +- .../test_dynapcnn/test_device_name_mapping.py | 3 +- tests/test_dynapcnn/test_neuron_leak.py | 11 +- .../test_single_neuron_hardware.py | 10 +- .../conftest_dynapcnnlayer.py | 68 +-- tests/test_dynapcnnlayer/model_dummy_1.py | 248 +++++----- tests/test_dynapcnnlayer/model_dummy_2.py | 365 ++++++++------- tests/test_dynapcnnlayer/model_dummy_3.py | 440 ++++++++++-------- tests/test_dynapcnnlayer/model_dummy_4.py | 303 ++++++------ .../test_dynapcnnlayer/test_dynapcnnlayer.py | 95 ++-- .../conftest_dynapcnnnetwork.py | 22 +- tests/test_dynapcnnnetwork/model_dummy_1.py | 76 ++- tests/test_dynapcnnnetwork/model_dummy_2.py | 122 +++-- tests/test_dynapcnnnetwork/model_dummy_3.py | 95 +++- tests/test_dynapcnnnetwork/model_dummy_4.py | 79 +++- .../test_dynapcnnnetwork.py | 54 ++- .../conftest_graph_extractor.py | 16 +- tests/test_graph_extractor/model_dummy_1.py | 136 ++++-- tests/test_graph_extractor/model_dummy_2.py | 196 ++++---- tests/test_graph_extractor/model_dummy_3.py | 187 +++++--- tests/test_graph_extractor/model_dummy_4.py | 149 +++--- .../test_graph_extractor.py | 32 +- 39 files changed, 1804 insertions(+), 1255 deletions(-) diff --git a/sinabs/activation/__init__.py b/sinabs/activation/__init__.py index ff0c4d3e..24f3678e 100644 --- a/sinabs/activation/__init__.py +++ b/sinabs/activation/__init__.py @@ -1,10 +1,5 @@ from .quantize import Quantize, StochasticRounding from .reset_mechanism import MembraneReset, MembraneSubtract from .spike_generation import MaxSpike, MultiSpike, SingleSpike -from .surrogate_gradient_fn import ( - Gaussian, - Heaviside, - MultiGaussian, - PeriodicExponential, - SingleExponential, -) +from .surrogate_gradient_fn import (Gaussian, Heaviside, MultiGaussian, + PeriodicExponential, SingleExponential) diff --git a/sinabs/backend/dynapcnn/__init__.py b/sinabs/backend/dynapcnn/__init__.py index 21eba2e7..363723c2 100644 --- a/sinabs/backend/dynapcnn/__init__.py +++ b/sinabs/backend/dynapcnn/__init__.py @@ -1,14 +1,4 @@ -from .dynapcnn_network import ( - DynapcnnCompatibleNetwork, - DynapcnnNetwork, -) - -from .dynapcnn_layer import ( - DynapcnnLayer, -) - -from .dynapcnn_layer_handler import ( - DynapcnnLayerHandler, -) - +from .dynapcnn_layer import DynapcnnLayer +from .dynapcnn_layer_handler import DynapcnnLayerHandler +from .dynapcnn_network import DynapcnnCompatibleNetwork, DynapcnnNetwork from .dynapcnn_visualizer import DynapcnnVisualizer diff --git a/sinabs/backend/dynapcnn/chip_factory.py b/sinabs/backend/dynapcnn/chip_factory.py index 105c5904..453d5cc1 100644 --- a/sinabs/backend/dynapcnn/chip_factory.py +++ b/sinabs/backend/dynapcnn/chip_factory.py @@ -3,14 +3,9 @@ import numpy as np import torch -from .chips import ( - DynapcnnConfigBuilder, - Speck2BConfigBuilder, - Speck2CMiniConfigBuilder, - Speck2DMiniConfigBuilder, - Speck2EConfigBuilder, - Speck2FConfigBuilder, -) +from .chips import (DynapcnnConfigBuilder, Speck2BConfigBuilder, + Speck2CMiniConfigBuilder, Speck2DMiniConfigBuilder, + Speck2EConfigBuilder, Speck2FConfigBuilder) from .config_builder import ConfigBuilder from .utils import parse_device_id diff --git a/sinabs/backend/dynapcnn/chips/dynapcnn.py b/sinabs/backend/dynapcnn/chips/dynapcnn.py index 9269bd57..4bf5acf0 100644 --- a/sinabs/backend/dynapcnn/chips/dynapcnn.py +++ b/sinabs/backend/dynapcnn/chips/dynapcnn.py @@ -13,6 +13,7 @@ from sinabs.backend.dynapcnn.dynapcnn_layer_handler import DynapcnnLayerHandler from sinabs.backend.dynapcnn.mapping import LayerConstraints + class DynapcnnConfigBuilder(ConfigBuilder): @classmethod def get_samna_module(cls): @@ -73,7 +74,12 @@ def set_kill_bits(cls, layer: DynapcnnLayer, config_dict: dict) -> dict: return config_dict @classmethod - def get_dynapcnn_layer_config_dict(cls, layer: DynapcnnLayer, layer_handler: DynapcnnLayerHandler, all_handlers: dict) -> dict: + def get_dynapcnn_layer_config_dict( + cls, + layer: DynapcnnLayer, + layer_handler: DynapcnnLayerHandler, + all_handlers: dict, + ) -> dict: config_dict = {} config_dict["destinations"] = [{}, {}] @@ -156,13 +162,15 @@ def get_dynapcnn_layer_config_dict(cls, layer: DynapcnnLayer, layer_handler: Dyn destinations = [] for node_id, destination_nodes in layer_handler.nodes_destinations.items(): for dest_node in destination_nodes: - core_id = DynapcnnLayerHandler.find_nodes_core_id(dest_node, all_handlers) + core_id = DynapcnnLayerHandler.find_nodes_core_id( + dest_node, all_handlers + ) kernel_size = layer_handler.get_pool_kernel_size(node_id) dest_data = { - 'layer': core_id, - 'enable': True, - 'pooling': expand_to_pair(kernel_size if kernel_size else 1), + "layer": core_id, + "enable": True, + "pooling": expand_to_pair(kernel_size if kernel_size else 1), } destinations.append(dest_data) @@ -172,10 +180,16 @@ def get_dynapcnn_layer_config_dict(cls, layer: DynapcnnLayer, layer_handler: Dyn config_dict = cls.set_kill_bits(layer=layer, config_dict=config_dict) return config_dict - + @classmethod - def write_dynapcnn_layer_config(cls, layer: DynapcnnLayer, chip_layer: "CNNLayerConfig", layer_handler: DynapcnnLayerHandler, all_handlers: dict) -> None: - """ Write a single layer configuration to the dynapcnn conf object. Uses the data in `layer` to configure a `CNNLayerConfig` to be + def write_dynapcnn_layer_config( + cls, + layer: DynapcnnLayer, + chip_layer: "CNNLayerConfig", + layer_handler: DynapcnnLayerHandler, + all_handlers: dict, + ) -> None: + """Write a single layer configuration to the dynapcnn conf object. Uses the data in `layer` to configure a `CNNLayerConfig` to be deployed on chip. Parameters @@ -187,19 +201,23 @@ def write_dynapcnn_layer_config(cls, layer: DynapcnnLayer, chip_layer: "CNNLayer """ # extracting from a DynapcnnLayer the config. variables for its CNNLayerConfig. - config_dict = cls.get_dynapcnn_layer_config_dict(layer=layer, layer_handler=layer_handler, all_handlers=all_handlers) + config_dict = cls.get_dynapcnn_layer_config_dict( + layer=layer, layer_handler=layer_handler, all_handlers=all_handlers + ) # update configuration of the DYNAPCNN layer. chip_layer.dimensions = config_dict["dimensions"] config_dict.pop("dimensions") - + # set the destinations configuration. - for i in range(len(config_dict['destinations'])): - chip_layer.destinations[i].layer = config_dict['destinations'][i]['layer'] - chip_layer.destinations[i].enable = config_dict['destinations'][i]['enable'] - chip_layer.destinations[i].pooling = config_dict['destinations'][i]['pooling'] + for i in range(len(config_dict["destinations"])): + chip_layer.destinations[i].layer = config_dict["destinations"][i]["layer"] + chip_layer.destinations[i].enable = config_dict["destinations"][i]["enable"] + chip_layer.destinations[i].pooling = config_dict["destinations"][i][ + "pooling" + ] - config_dict.pop('destinations') + config_dict.pop("destinations") # set remaining configuration. for param, value in config_dict.items(): @@ -210,8 +228,8 @@ def write_dynapcnn_layer_config(cls, layer: DynapcnnLayer, chip_layer: "CNNLayer @classmethod def build_config(cls, model: "DynapcnnNetwork") -> DynapcnnConfiguration: - """ Uses `DynapcnnLayer` objects to configure their equivalent chip core via a `CNNLayerConfig` object that is built - using using the `DynapcnnLayer` properties. + """Uses `DynapcnnLayer` objects to configure their equivalent chip core via a `CNNLayerConfig` object that is built + using using the `DynapcnnLayer` properties. Parameters ---------- @@ -223,7 +241,7 @@ def build_config(cls, model: "DynapcnnNetwork") -> DynapcnnConfiguration: """ config = cls.get_default_config() - has_dvs_layer = False # TODO DVSLayer not supported yet. + has_dvs_layer = False # TODO DVSLayer not supported yet. # Loop over layers in network and write corresponding configurations for layer_index, ith_dcnnl in model.layers_mapper.items(): @@ -233,14 +251,23 @@ def build_config(cls, model: "DynapcnnNetwork") -> DynapcnnConfiguration: elif isinstance(ith_dcnnl, DynapcnnLayer): # retrieve assigned core from the handler of this DynapcnnLayer (`ith_dcnnl`) instance. - chip_layer = config.cnn_layers[model.layers_handlers[layer_index].assigned_core] + chip_layer = config.cnn_layers[ + model.layers_handlers[layer_index].assigned_core + ] # write core configuration. - cls.write_dynapcnn_layer_config(ith_dcnnl, chip_layer, model.layers_handlers[layer_index], model.layers_handlers) + cls.write_dynapcnn_layer_config( + ith_dcnnl, + chip_layer, + model.layers_handlers[layer_index], + model.layers_handlers, + ) else: # shouldn't happen since type checks are made previously. - raise TypeError(f"Layer (index {layer_index}) is unexpected in the model: \n{ith_dcnnl}") - + raise TypeError( + f"Layer (index {layer_index}) is unexpected in the model: \n{ith_dcnnl}" + ) + if not has_dvs_layer: # TODO DVSLayer not supported yet. config.dvs_layer.pass_sensor_events = False diff --git a/sinabs/backend/dynapcnn/chips/speck2cmini.py b/sinabs/backend/dynapcnn/chips/speck2cmini.py index 327f938e..5487b1e0 100644 --- a/sinabs/backend/dynapcnn/chips/speck2cmini.py +++ b/sinabs/backend/dynapcnn/chips/speck2cmini.py @@ -1,4 +1,4 @@ -from typing import List, Dict +from typing import Dict, List import samna from samna.speck2cMini.configuration import SpeckConfiguration @@ -29,8 +29,12 @@ def get_output_buffer(cls): return samna.BasicSinkNode_speck2c_mini_event_output_event() @classmethod - def get_dynapcnn_layer_config_dict(cls, layer: DynapcnnLayer, layers_mapper: Dict[int, DynapcnnLayer]) -> dict: - config_dict = super().get_dynapcnn_layer_config_dict(layer=layer, layers_mapper=layers_mapper) + def get_dynapcnn_layer_config_dict( + cls, layer: DynapcnnLayer, layers_mapper: Dict[int, DynapcnnLayer] + ) -> dict: + config_dict = super().get_dynapcnn_layer_config_dict( + layer=layer, layers_mapper=layers_mapper + ) config_dict.pop("weights_kill_bit") config_dict.pop("biases_kill_bit") config_dict.pop("neurons_value_kill_bit") diff --git a/sinabs/backend/dynapcnn/chips/speck2e.py b/sinabs/backend/dynapcnn/chips/speck2e.py index 3904e098..ef8faa92 100644 --- a/sinabs/backend/dynapcnn/chips/speck2e.py +++ b/sinabs/backend/dynapcnn/chips/speck2e.py @@ -1,3 +1,5 @@ +from typing import Dict + import samna from samna.speck2e.configuration import SpeckConfiguration @@ -5,8 +7,6 @@ from .dynapcnn import DynapcnnConfigBuilder -from typing import Dict - # Since most of the configuration is identical to DYNAP-CNN, we can simply inherit this class @@ -32,6 +32,10 @@ def set_kill_bits(cls, layer: DynapcnnLayer, config_dict: dict) -> dict: return config_dict @classmethod - def get_dynapcnn_layer_config_dict(cls, layer: DynapcnnLayer, layers_mapper: Dict[int, DynapcnnLayer]) -> dict: - config_dict = super().get_dynapcnn_layer_config_dict(layer=layer, layers_mapper=layers_mapper) + def get_dynapcnn_layer_config_dict( + cls, layer: DynapcnnLayer, layers_mapper: Dict[int, DynapcnnLayer] + ) -> dict: + config_dict = super().get_dynapcnn_layer_config_dict( + layer=layer, layers_mapper=layers_mapper + ) return config_dict diff --git a/sinabs/backend/dynapcnn/chips/speck2f.py b/sinabs/backend/dynapcnn/chips/speck2f.py index 2f1a0b2d..d7ee00cf 100644 --- a/sinabs/backend/dynapcnn/chips/speck2f.py +++ b/sinabs/backend/dynapcnn/chips/speck2f.py @@ -1,3 +1,5 @@ +from typing import Dict + import samna from samna.speck2f.configuration import SpeckConfiguration @@ -5,8 +7,6 @@ from .dynapcnn import DynapcnnConfigBuilder -from typing import Dict - # Since most of the configuration is identical to DYNAP-CNN, we can simply inherit this class @@ -28,8 +28,12 @@ def get_output_buffer(cls): return samna.BasicSinkNode_speck2f_event_output_event() @classmethod - def get_dynapcnn_layer_config_dict(cls, layer: DynapcnnLayer, layers_mapper: Dict[int, DynapcnnLayer]) -> dict: - config_dict = super().get_dynapcnn_layer_config_dict(layer=layer, layers_mapper=layers_mapper) + def get_dynapcnn_layer_config_dict( + cls, layer: DynapcnnLayer, layers_mapper: Dict[int, DynapcnnLayer] + ) -> dict: + config_dict = super().get_dynapcnn_layer_config_dict( + layer=layer, layers_mapper=layers_mapper + ) config_dict.pop("weights_kill_bit") config_dict.pop("biases_kill_bit") config_dict.pop("neurons_value_kill_bit") diff --git a/sinabs/backend/dynapcnn/connectivity_specs.py b/sinabs/backend/dynapcnn/connectivity_specs.py index cc14b1e3..fc0da6da 100644 --- a/sinabs/backend/dynapcnn/connectivity_specs.py +++ b/sinabs/backend/dynapcnn/connectivity_specs.py @@ -12,7 +12,7 @@ Pooling = (sl.SumPool2d, nn.AvgPool2d) Weight = (nn.Conv2d, nn.Linear) -Neuron = (sl.IAFSqueeze, ) +Neuron = (sl.IAFSqueeze,) VALID_SINABS_EDGE_TYPES_ABSTRACT = { # convoluion is always followed by a neuron layer. @@ -20,7 +20,7 @@ # Neuron layer can be followed by pooling (Neuron, Pooling): "neuron-pooling", # Pooling can be followed by another pooling (will be consolidated) - (Pooling, Pooling): "pooling-pooling", + (Pooling, Pooling): "pooling-pooling", # Neuron layer can be followed by weight layer of next core (Neuron, Weight): "neuron-weight", # Pooling can be followed by weight layer of next core diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index eaa0f60a..9f74c992 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -15,16 +15,12 @@ from .dvs_layer import DVSLayer from .dynapcnn_layer import DynapcnnLayer from .dynapcnnnetwork_module import DynapcnnNetworkModule -from .io import disable_timestamps, enable_timestamps, open_device, reset_timestamps +from .io import (disable_timestamps, enable_timestamps, open_device, + reset_timestamps) from .nir_graph_extractor import GraphExtractor from .sinabs_edges_handler import collect_dynapcnn_layer_info -from .utils import ( - build_from_graph, - DEFAULT_IGNORED_LAYER_TYPES, - Edge, - parse_device_id, - topological_sorting, -) +from .utils import (DEFAULT_IGNORED_LAYER_TYPES, Edge, build_from_graph, + parse_device_id, topological_sorting) from .weight_rescaling_methods import rescale_method_1 @@ -662,8 +658,8 @@ def __str__(self): pretty_print += f"{layer_data}\n\n" return pretty_print - - + + class DynapcnnCompatibleNetwork(DynapcnnNetwork): """Deprecated class, use DynapcnnNetwork instead.""" diff --git a/sinabs/backend/dynapcnn/exceptions.py b/sinabs/backend/dynapcnn/exceptions.py index b5e3cf86..63de8585 100644 --- a/sinabs/backend/dynapcnn/exceptions.py +++ b/sinabs/backend/dynapcnn/exceptions.py @@ -60,13 +60,13 @@ class InvalidTorchModel(Exception): network_type: str def __init__(self, network_type): - super().__init__( - f"A {network_type} needs to be of type nn.Module." - ) + super().__init__(f"A {network_type} needs to be of type nn.Module.") + class InvalidGraphStructure(Exception): pass + # Edge exceptions. @@ -140,4 +140,4 @@ class InvalidLayerDestination(Exception): def __init__(self, dynapcnnlayerA, dynapcnnlayerB): super().__init__( f"DynapcnnLayer {dynapcnnlayerA} in one core can not connect to {dynapcnnlayerB} in another core." - ) \ No newline at end of file + ) diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index d83cc968..c3b852b3 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -9,15 +9,12 @@ import sinabs -from .connectivity_specs import ( - LAYER_TYPES_WITH_MULTIPLE_INPUTS, - LAYER_TYPES_WITH_MULTIPLE_OUTPUTS, -) +from .connectivity_specs import (LAYER_TYPES_WITH_MULTIPLE_INPUTS, + LAYER_TYPES_WITH_MULTIPLE_OUTPUTS) from .exceptions import InvalidGraphStructure from .utils import Edge, topological_sorting - class GraphExtractor: def __init__(self, spiking_model: nn.Module, dummy_input: torch.tensor): """Class implementing the extraction of the computational graph from `spiking_model`, where @@ -146,10 +143,10 @@ def get_node_io_shapes(self, node: int) -> Tuple[torch.Size, torch.Size]: self._nodes_io_shapes[node]["input"], self._nodes_io_shapes[node]["output"], ) - + def verify_graph_integrity(self): - """ Apply checks to verify that graph is supported - + """Apply checks to verify that graph is supported + Currently this checks that only nodes of specific classes have multiple sources or targets. This method might be extended in the future to implement stricter formal verification. diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index b1827dcf..b82af9bc 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -8,14 +8,10 @@ from collections import deque from typing import Dict, List, Set, Tuple, Type -from torch import nn, Size +from torch import Size, nn -from .exceptions import ( - InvalidEdge, - UnmatchedNode, - UnmatchedPoolingEdges, -) from .connectivity_specs import VALID_SINABS_EDGE_TYPES +from .exceptions import InvalidEdge, UnmatchedNode, UnmatchedPoolingEdges from .utils import Edge @@ -25,12 +21,12 @@ def collect_dynapcnn_layer_info( nodes_io_shapes: Dict[int, Dict[str, Tuple[Size, Size]]], ) -> Dict[int, Dict]: """Collect information to construct DynapcnnLayer instances. - + Validate and sort edges based on the type of nodes they connect. Iterate over edges in order of their type. For each neuron->weight edge generate a new dict to collect information for the corresponding dynapcnn layer. Then add pooling based on neuron->pooling type edges. Collect additional pooling - from pooling->pooling type edges. Finally set layer destinations based on + from pooling->pooling type edges. Finally set layer destinations based on neuron/pooling->weight type of edges. Parameters @@ -38,7 +34,7 @@ def collect_dynapcnn_layer_info( indx_2_module_map (dict): Maps node IDs of the graph as `key` to their associated module as `value` edges (set of tuples): Represent connections between two nodes in computational graph nodes_io_shapes (dict): Map from node ID to dict containing node's in- and output shapes - + Returns ------- dynapcnn_layer_info (dict): Each 'key' is the index of a future 'DynapcnnLayer' and @@ -46,64 +42,72 @@ def collect_dynapcnn_layer_info( containing corresponding node ids and modules required to build the layer """ # TODO: Handle DVS layer - + # Sort edges by edge type (type of layers they connect) edges_by_type: Dict[str, Set[Edge]] = dict() for edge in edges: - edge_type = get_valid_edge_type(edge, indx_2_module_map, VALID_SINABS_EDGE_TYPES) - + edge_type = get_valid_edge_type( + edge, indx_2_module_map, VALID_SINABS_EDGE_TYPES + ) + # Validate edge type if edge_type is None: raise InvalidEdge( edge, type(indx_2_module_map[edge[0]]), type(indx_2_module_map[edge[1]]) ) - + if edge_type in edges_by_type: edges_by_type[edge_type].add(edge) else: edges_by_type[edge_type] = {edge} - - # Dict to collect information for each future dynapcnn layer + + # Dict to collect information for each future dynapcnn layer dynapcnn_layer_info = dict() # Map node IDs to dynapcnn layer ID node_2_layer_map = dict() # Each weight->neuron connection instantiates a new, unique dynapcnn layer - while(edges_by_type["weight-neuron"]): + while edges_by_type["weight-neuron"]: edge = edges_by_type["weight-neuron"].pop() init_new_dynapcnnlayer_entry( - dynapcnn_layer_info, + dynapcnn_layer_info, edge, indx_2_module_map, nodes_io_shapes, - node_2_layer_map + node_2_layer_map, ) - + # "pooling-pooling" edges are optional. Unlike other types, missing entry would cause exception. # Therefore add empty set if not existing if "pooling-pooling" not in edges_by_type: edges_by_type["pooling-pooling"] = set() # Add pooling based on neuron->pooling connections - while(edges_by_type["neuron-pooling"]): + while edges_by_type["neuron-pooling"]: edge = edges_by_type["neuron-pooling"].pop() # Search pooling-pooling edges for chains of pooling and add to existing entry - pooling_chains, edges_used = trace_paths(edge[1], edges_by_type["pooling-pooling"]) + pooling_chains, edges_used = trace_paths( + edge[1], edges_by_type["pooling-pooling"] + ) add_pooling_to_entry( - dynapcnn_layer_info, edge, pooling_chains, indx_2_module_map, node_2_layer_map + dynapcnn_layer_info, + edge, + pooling_chains, + indx_2_module_map, + node_2_layer_map, ) # Remove handled pooling-pooling edges edges_by_type["pooling-pooling"].difference_update(edges_used) # After adding pooling make sure all pooling-pooling edges have been handled if len(edges_by_type["pooling-pooling"]) > 0: raise UnmatchedPoolingEdges(edges_by_type["pooling-pooling"]) - + # Process all edges connecting two dynapcnn layers - while(edges_by_type["neuron-weight"]): + while edges_by_type["neuron-weight"]: edge = edges_by_type["neuron-weight"].pop() set_neuron_layer_destination(dynapcnn_layer_info, edge, node_2_layer_map) - - while(edges_by_type["pooling-weight"]): + + while edges_by_type["pooling-weight"]: edge = edges_by_type["pooling-weight"].pop() set_pooling_layer_destination(dynapcnn_layer_info, edge, node_2_layer_map) @@ -112,6 +116,7 @@ def collect_dynapcnn_layer_info( return dynapcnn_layer_info + def get_valid_edge_type( edge: Edge, layers: Dict[int, nn.Module], @@ -135,6 +140,7 @@ def get_valid_edge_type( return valid_edge_ids.get((source_type, target_type), None) + def init_new_dynapcnnlayer_entry( dynapcnn_layer_info: Dict[int, Dict[int, Dict]], edge: Edge, @@ -142,7 +148,7 @@ def init_new_dynapcnnlayer_entry( nodes_io_shapes: Dict[int, Dict[str, Tuple[Size, Size]]], node_2_layer_map: Dict[int, int], ) -> None: - """ Initiate dict to hold information for new dynapcnn layer based on a "weight->neuron" edge. + """Initiate dict to hold information for new dynapcnn layer based on a "weight->neuron" edge. Change `dynapcnn_layer_info` in-place. Parameters @@ -181,6 +187,7 @@ def init_new_dynapcnnlayer_entry( node_2_layer_map[edge[0]] = layer_id node_2_layer_map[edge[1]] = layer_id + def add_pooling_to_entry( dynapcnn_layer_info: Dict[int, Dict[int, Dict]], edge: Edge, @@ -189,7 +196,7 @@ def add_pooling_to_entry( node_2_layer_map: Dict[int, int], ) -> None: """Add or extend destination information to existing entry in `dynapcnn_layer_info`. - + Correct entry is identified by existing neuron node. Destination information is a dict containing list of IDs and list of modules for each chains of pooling nodes. @@ -210,7 +217,7 @@ def add_pooling_to_entry( try: layer_idx = node_2_layer_map[edge[0]] except KeyError: - raise UnmatchedNode(edge, edge[0]) + raise UnmatchedNode(edge, edge[0]) # Make sure all pooling chains start with expected node assert all(chain[0] == edge[1] for chain in pooling_chains) @@ -218,7 +225,7 @@ def add_pooling_to_entry( layer_info = dynapcnn_layer_info[layer_idx] if "destinations" not in layer_info: layer_info["destinations"] = [] - + # Keep track of all nodes that have been added new_nodes = set() @@ -231,18 +238,19 @@ def add_pooling_to_entry( } ) new_nodes.update(set(chain)) - + for node in new_nodes: # Make sure new pooling nodes have not been used elsewhere assert node not in node_2_layer_map node_2_layer_map[node] = layer_idx + def set_neuron_layer_destination( dynapcnn_layer_info: Dict[int, Dict[int, Dict]], edge: Edge, node_2_layer_map: Dict[int, int], ) -> None: - """ Set destination layer without pooling. + """Set destination layer without pooling. Parameters ---------- @@ -258,17 +266,17 @@ def set_neuron_layer_destination( try: source_layer_idx = node_2_layer_map[edge[0]] except KeyError: - raise UnmatchedNode(edge, edge[0]) + raise UnmatchedNode(edge, edge[0]) try: destination_layer_idx = node_2_layer_map[edge[1]] except KeyError: - raise UnmatchedNode(edge, edge[1]) - + raise UnmatchedNode(edge, edge[1]) + # Source layer entry might already have `destinations` key (if neuron layer has fanout > 1) layer_info = dynapcnn_layer_info[source_layer_idx] if "destinations" not in layer_info: layer_info["destinations"] = [] - + # Add new destination layer_info["destinations"].append( { @@ -278,12 +286,13 @@ def set_neuron_layer_destination( } ) + def set_pooling_layer_destination( dynapcnn_layer_info: Dict[int, Dict[int, Dict]], edge: Edge, node_2_layer_map: Dict[int, int], ) -> None: - """ Set destination layer with pooling. + """Set destination layer with pooling. Parameters ---------- @@ -299,15 +308,15 @@ def set_pooling_layer_destination( try: source_layer_idx = node_2_layer_map[edge[0]] except KeyError: - raise UnmatchedNode(edge, edge[0]) + raise UnmatchedNode(edge, edge[0]) try: destination_layer_idx = node_2_layer_map[edge[1]] except KeyError: - raise UnmatchedNode(edge, edge[1]) - + raise UnmatchedNode(edge, edge[1]) + # Source layer entry should already have `destinations` key layer_info = dynapcnn_layer_info[source_layer_idx] - + # Find current source node within destinations matched = False for destination in layer_info["destinations"]: @@ -316,15 +325,16 @@ def set_pooling_layer_destination( break if not matched: raise UnmatchedNode(edge, edge[0]) - + # Set destination layer within destination dict that holds current source node destination["destination_layer"] = destination_layer_idx + def trace_paths(node: int, remaining_edges: Set[Edge]) -> List[deque[int]]: - """ Trace any path of collected edges through the graph. + """Trace any path of collected edges through the graph. Start with `node`, and recursively look for paths of connected nodes - within `remaining edges.` + within `remaining edges.` Parameters ---------- @@ -338,7 +348,7 @@ def trace_paths(node: int, remaining_edges: Set[Edge]) -> List[deque[int]]: """ paths = [] processed_edges = set() - for (src, tgt) in remaining_edges: + for src, tgt in remaining_edges: if src == node: processed_edges.add((src, tgt)) # For each edge with `node` as source, find subsequent pooling nodes recursively @@ -346,10 +356,10 @@ def trace_paths(node: int, remaining_edges: Set[Edge]) -> List[deque[int]]: branches, new_processed = trace_paths(tgt, new_remaining) # Make sure no edge was processed twice assert len(processed_edges.intersection(new_processed)) == 0 - + # Keep track of newly processed edges processed_edges.update(new_processed) - + # Collect all branching paths of pooling, inserting src at beginning for branch in branches: branch.appendleft(src) @@ -360,4 +370,3 @@ def trace_paths(node: int, remaining_edges: Set[Edge]) -> List[deque[int]]: paths = [deque([node])] return paths, processed_edges - diff --git a/sinabs/backend/dynapcnn/utils.py b/sinabs/backend/dynapcnn/utils.py index 16a9d8f3..c75255ed 100644 --- a/sinabs/backend/dynapcnn/utils.py +++ b/sinabs/backend/dynapcnn/utils.py @@ -1,6 +1,7 @@ from collections import defaultdict, deque from copy import deepcopy -from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Tuple, Union, Set +from typing import (TYPE_CHECKING, Callable, Dict, List, Optional, Set, Tuple, + Union) import torch import torch.nn as nn @@ -17,7 +18,9 @@ if TYPE_CHECKING: from sinabs.backend.dynapcnn.dynapcnn_network import DynapcnnNetwork -DEFAULT_IGNORED_LAYER_TYPES = Union[nn.Identity, nn.Dropout, nn.Dropout2d, nn.Flatten, sl.Merge] +DEFAULT_IGNORED_LAYER_TYPES = Union[ + nn.Identity, nn.Dropout, nn.Dropout2d, nn.Flatten, sl.Merge +] Edge = Tuple[int, int] # Define edge-type alias @@ -77,6 +80,7 @@ def standardize_device_id(device_id: str) -> str: ####################################################### DynapcnnNetwork Related ####################################################### + def build_from_graph( discretize: bool, edges: List[Tuple[int, int]], @@ -273,7 +277,9 @@ def construct_layerhandler( return layerhandler -def construct_all_dynapcnnlayers(dcnnl_map: Dict, discretize: bool, rescale_fn: Optional[Callable] = None) -> Dict[int, DynapcnnLayer]: +def construct_all_dynapcnnlayers( + dcnnl_map: Dict, discretize: bool, rescale_fn: Optional[Callable] = None +) -> Dict[int, DynapcnnLayer]: """...""" # Extract construction arguments from dcnnl_map @@ -292,7 +298,7 @@ def construct_all_dynapcnnlayers(dcnnl_map: Dict, discretize: bool, rescale_fn: destination["cumulative_scaling"] = scale dest_lyr_idx = destination["destination_layer"] dcnnl_map[dest_lyr_idx]["rescale_factors"].add(layer_rescaling) - + dynapcnn_layer = { layer_idx: construct_single_dynapcnn_layer(layer_info, discretize, rescale_fn) for layer_idx, layer_info in dcnnl_map.items() @@ -303,7 +309,10 @@ def construct_all_dynapcnnlayers(dcnnl_map: Dict, discretize: bool, rescale_fn: for layer_idx, layer_info in dcnnl_map.items() } -def construct_single_dynapcnn_layer(layer_info: Dict, rescale_fn: Optional[Callable] = None) -> DynapcnnLayer: + +def construct_single_dynapcnn_layer( + layer_info: Dict, rescale_fn: Optional[Callable] = None +) -> DynapcnnLayer: if len(layer_info["rescale_factors"]) == 0: rescale_factor = 1 @@ -333,8 +342,9 @@ def construct_single_dynapcnn_layer(layer_info: Dict, rescale_fn: Optional[Calla rescale_weights=rescale_factor, ) + def consolidate_pooling(modules: Iterable[nn.Module]) -> Tuple[Tuple[int, int], float]: - """ Consolidate pooling information for consecutive pooling modules. + """Consolidate pooling information for consecutive pooling modules. Parameters ---------- @@ -349,18 +359,21 @@ def consolidate_pooling(modules: Iterable[nn.Module]) -> Tuple[Tuple[int, int], to sum pooling, considering all provided modules. """ cumulative_pooling = [1, 1] - cumulative_scaling = 1. + cumulative_scaling = 1.0 for pooling_layer in modules: pooling, rescale_factor = extract_pooling_from_module(pooling_layer) cumulative_pooling[0] *= pooling[0] cumulative_pooling[1] *= pooling[1] cumulative_scaling *= rescale_factor - + return cumulative_pooling, cumulative_scaling -def extract_pooling_from_module(module: Union[nn.AvgPool2d, sl.SumPool2d]) -> Tuple[Tuple[int, int], float]: - """ Extract pooling size and required rescaling factor from pooling module + +def extract_pooling_from_module( + module: Union[nn.AvgPool2d, sl.SumPool2d] +) -> Tuple[Tuple[int, int], float]: + """Extract pooling size and required rescaling factor from pooling module Parameters ---------- @@ -381,14 +394,15 @@ def extract_pooling_from_module(module: Union[nn.AvgPool2d, sl.SumPool2d]) -> Tu f"Stride length {module.stride} should be the same as pooling kernel size {module.kernel_size}" ) if isinstance(pooling_layer, nn.AvgPool2d): - scale_factor = 1. / (pooling[0] * pooling[1]) + scale_factor = 1.0 / (pooling[0] * pooling[1]) elif isinstance(pooling_layer, sl.SumPool2d): - scale_factor = 1. + scale_factor = 1.0 else: raise ValueError(f"Unsupported type {type(module)} for pooling layer") return pooling, scale_factor + def convert_Avg_to_Sum_pooling( dcnnl_data: Dict[ Union[int, str], diff --git a/sinabs/from_torch.py b/sinabs/from_torch.py index 42b5ae4e..c901e77d 100644 --- a/sinabs/from_torch.py +++ b/sinabs/from_torch.py @@ -119,10 +119,14 @@ def mapper_fn(module): ) else: - warn("Spiking output can only be added to sequential models that do not end in a ReLU. No layer has been added.") + warn( + "Spiking output can only be added to sequential models that do not end in a ReLU. No layer has been added." + ) else: - warn("Spiking output can only be added to sequential models that do not end in a ReLU. No layer has been added.") + warn( + "Spiking output can only be added to sequential models that do not end in a ReLU. No layer has been added." + ) for module in snn.modules(): if bias_rescaling != 1.0 and isinstance(module, (nn.Linear, nn.Conv2d)): diff --git a/tests/test_activations.py b/tests/test_activations.py index 1ec880dc..cfec09ef 100644 --- a/tests/test_activations.py +++ b/tests/test_activations.py @@ -1,14 +1,8 @@ import pytest import torch -from sinabs.activation import ( - MaxSpike, - MembraneReset, - MembraneSubtract, - MultiSpike, - SingleExponential, - SingleSpike, -) +from sinabs.activation import (MaxSpike, MembraneReset, MembraneSubtract, + MultiSpike, SingleExponential, SingleSpike) @pytest.mark.parametrize( diff --git a/tests/test_dynapcnn/test_auto_mapping.py b/tests/test_dynapcnn/test_auto_mapping.py index 37de88d9..4e8029d5 100644 --- a/tests/test_dynapcnn/test_auto_mapping.py +++ b/tests/test_dynapcnn/test_auto_mapping.py @@ -2,7 +2,8 @@ import torch.nn as nn from sinabs.backend.dynapcnn import DynapcnnNetwork -from sinabs.backend.dynapcnn.mapping import edmonds, make_flow_graph, recover_mapping +from sinabs.backend.dynapcnn.mapping import (edmonds, make_flow_graph, + recover_mapping) from sinabs.from_torch import from_model ann = nn.Sequential( diff --git a/tests/test_dynapcnn/test_compatible_layer_build.py b/tests/test_dynapcnn/test_compatible_layer_build.py index 54215a52..da70d628 100644 --- a/tests/test_dynapcnn/test_compatible_layer_build.py +++ b/tests/test_dynapcnn/test_compatible_layer_build.py @@ -171,7 +171,8 @@ def test_incorrect_model_start(): def test_conversion_to_layer_list(): - from sinabs.backend.dynapcnn.utils import DEFAULT_IGNORED_LAYER_TYPES as DEF_IGNORE + from sinabs.backend.dynapcnn.utils import \ + DEFAULT_IGNORED_LAYER_TYPES as DEF_IGNORE from sinabs.backend.dynapcnn.utils import convert_model_to_layer_list model = nn.Sequential( diff --git a/tests/test_dynapcnn/test_device_movement.py b/tests/test_dynapcnn/test_device_movement.py index a6b9d8a1..5db2415e 100644 --- a/tests/test_dynapcnn/test_device_movement.py +++ b/tests/test_dynapcnn/test_device_movement.py @@ -2,7 +2,8 @@ import torch.nn as nn from sinabs.backend.dynapcnn import DynapcnnNetwork -from sinabs.backend.dynapcnn.mapping import edmonds, make_flow_graph, recover_mapping +from sinabs.backend.dynapcnn.mapping import (edmonds, make_flow_graph, + recover_mapping) from sinabs.from_torch import from_model ann = nn.Sequential( diff --git a/tests/test_dynapcnn/test_device_name_mapping.py b/tests/test_dynapcnn/test_device_name_mapping.py index b92901ea..0e31893d 100644 --- a/tests/test_dynapcnn/test_device_name_mapping.py +++ b/tests/test_dynapcnn/test_device_name_mapping.py @@ -1,4 +1,5 @@ -from sinabs.backend.dynapcnn.utils import parse_device_id, standardize_device_id +from sinabs.backend.dynapcnn.utils import (parse_device_id, + standardize_device_id) def test_device_id_no_index(): diff --git a/tests/test_dynapcnn/test_neuron_leak.py b/tests/test_dynapcnn/test_neuron_leak.py index 4acacdbc..ece73964 100644 --- a/tests/test_dynapcnn/test_neuron_leak.py +++ b/tests/test_dynapcnn/test_neuron_leak.py @@ -3,16 +3,13 @@ import pytest import samna import torch -from hw_utils import ( - find_open_devices, - get_ones_network, - is_any_samna_device_connected, - is_device_connected, -) +from hw_utils import (find_open_devices, get_ones_network, + is_any_samna_device_connected, is_device_connected) from torch import nn from sinabs.backend.dynapcnn import DynapcnnNetwork -from sinabs.backend.dynapcnn.io import calculate_neuron_address, neuron_address_to_cxy +from sinabs.backend.dynapcnn.io import (calculate_neuron_address, + neuron_address_to_cxy) from sinabs.layers import IAFSqueeze diff --git a/tests/test_dynapcnn/test_single_neuron_hardware.py b/tests/test_dynapcnn/test_single_neuron_hardware.py index 6d8fe6c6..e9e99bdd 100644 --- a/tests/test_dynapcnn/test_single_neuron_hardware.py +++ b/tests/test_dynapcnn/test_single_neuron_hardware.py @@ -1,11 +1,7 @@ import pytest -from hw_utils import ( - find_open_devices, - get_ones_network, - is_any_samna_device_connected, - is_device_connected, - reset_all_connected_boards, -) +from hw_utils import (find_open_devices, get_ones_network, + is_any_samna_device_connected, is_device_connected, + reset_all_connected_boards) import sinabs import sinabs.backend.dynapcnn as sdl diff --git a/tests/test_dynapcnnlayer/conftest_dynapcnnlayer.py b/tests/test_dynapcnnlayer/conftest_dynapcnnlayer.py index a55c6fbf..4e065c2a 100644 --- a/tests/test_dynapcnnlayer/conftest_dynapcnnlayer.py +++ b/tests/test_dynapcnnlayer/conftest_dynapcnnlayer.py @@ -1,37 +1,41 @@ # author : Willian Soares Girao # contact : wsoaresgirao@gmail.com -from model_dummy_1 import nodes_to_dcnnl_map_1, sinabs_edges_1, expected_output_1 -from model_dummy_2 import nodes_to_dcnnl_map_2, sinabs_edges_2, expected_output_2 -from model_dummy_3 import nodes_to_dcnnl_map_3, sinabs_edges_3, expected_output_3 -from model_dummy_4 import nodes_to_dcnnl_map_4, sinabs_edges_4, expected_output_4 +from model_dummy_1 import (expected_output_1, nodes_to_dcnnl_map_1, + sinabs_edges_1) +from model_dummy_2 import (expected_output_2, nodes_to_dcnnl_map_2, + sinabs_edges_2) +from model_dummy_3 import (expected_output_3, nodes_to_dcnnl_map_3, + sinabs_edges_3) +from model_dummy_4 import (expected_output_4, nodes_to_dcnnl_map_4, + sinabs_edges_4) args_DynapcnnLayer = [ - (nodes_to_dcnnl_map_1, 0, sinabs_edges_1, [0], expected_output_1), - (nodes_to_dcnnl_map_1, 1, sinabs_edges_1, [0], expected_output_1), - (nodes_to_dcnnl_map_1, 2, sinabs_edges_1, [0], expected_output_1), - (nodes_to_dcnnl_map_1, 3, sinabs_edges_1, [0], expected_output_1), - (nodes_to_dcnnl_map_1, 4, sinabs_edges_1, [0], expected_output_1), - (nodes_to_dcnnl_map_2, 0, sinabs_edges_2, [0], expected_output_2), - (nodes_to_dcnnl_map_2, 1, sinabs_edges_2, [0], expected_output_2), - (nodes_to_dcnnl_map_2, 2, sinabs_edges_2, [0], expected_output_2), - (nodes_to_dcnnl_map_2, 3, sinabs_edges_2, [0], expected_output_2), - (nodes_to_dcnnl_map_2, 4, sinabs_edges_2, [0], expected_output_2), - (nodes_to_dcnnl_map_2, 5, sinabs_edges_2, [0], expected_output_2), - (nodes_to_dcnnl_map_2, 6, sinabs_edges_2, [0], expected_output_2), - (nodes_to_dcnnl_map_3, 0, sinabs_edges_3, [0, 8], expected_output_3), - (nodes_to_dcnnl_map_3, 1, sinabs_edges_3, [0, 8], expected_output_3), - (nodes_to_dcnnl_map_3, 2, sinabs_edges_3, [0, 8], expected_output_3), - (nodes_to_dcnnl_map_3, 3, sinabs_edges_3, [0, 8], expected_output_3), - (nodes_to_dcnnl_map_3, 4, sinabs_edges_3, [0, 8], expected_output_3), - (nodes_to_dcnnl_map_3, 5, sinabs_edges_3, [0, 8], expected_output_3), - (nodes_to_dcnnl_map_3, 6, sinabs_edges_3, [0, 8], expected_output_3), - (nodes_to_dcnnl_map_3, 7, sinabs_edges_3, [0, 8], expected_output_3), - (nodes_to_dcnnl_map_3, 8, sinabs_edges_3, [0, 8], expected_output_3), - (nodes_to_dcnnl_map_4, 0, sinabs_edges_4, [0], expected_output_4), - (nodes_to_dcnnl_map_4, 1, sinabs_edges_4, [0], expected_output_4), - (nodes_to_dcnnl_map_4, 2, sinabs_edges_4, [0], expected_output_4), - (nodes_to_dcnnl_map_4, 3, sinabs_edges_4, [0], expected_output_4), - (nodes_to_dcnnl_map_4, 4, sinabs_edges_4, [0], expected_output_4), - (nodes_to_dcnnl_map_4, 5, sinabs_edges_4, [0], expected_output_4), -] \ No newline at end of file + (nodes_to_dcnnl_map_1, 0, sinabs_edges_1, [0], expected_output_1), + (nodes_to_dcnnl_map_1, 1, sinabs_edges_1, [0], expected_output_1), + (nodes_to_dcnnl_map_1, 2, sinabs_edges_1, [0], expected_output_1), + (nodes_to_dcnnl_map_1, 3, sinabs_edges_1, [0], expected_output_1), + (nodes_to_dcnnl_map_1, 4, sinabs_edges_1, [0], expected_output_1), + (nodes_to_dcnnl_map_2, 0, sinabs_edges_2, [0], expected_output_2), + (nodes_to_dcnnl_map_2, 1, sinabs_edges_2, [0], expected_output_2), + (nodes_to_dcnnl_map_2, 2, sinabs_edges_2, [0], expected_output_2), + (nodes_to_dcnnl_map_2, 3, sinabs_edges_2, [0], expected_output_2), + (nodes_to_dcnnl_map_2, 4, sinabs_edges_2, [0], expected_output_2), + (nodes_to_dcnnl_map_2, 5, sinabs_edges_2, [0], expected_output_2), + (nodes_to_dcnnl_map_2, 6, sinabs_edges_2, [0], expected_output_2), + (nodes_to_dcnnl_map_3, 0, sinabs_edges_3, [0, 8], expected_output_3), + (nodes_to_dcnnl_map_3, 1, sinabs_edges_3, [0, 8], expected_output_3), + (nodes_to_dcnnl_map_3, 2, sinabs_edges_3, [0, 8], expected_output_3), + (nodes_to_dcnnl_map_3, 3, sinabs_edges_3, [0, 8], expected_output_3), + (nodes_to_dcnnl_map_3, 4, sinabs_edges_3, [0, 8], expected_output_3), + (nodes_to_dcnnl_map_3, 5, sinabs_edges_3, [0, 8], expected_output_3), + (nodes_to_dcnnl_map_3, 6, sinabs_edges_3, [0, 8], expected_output_3), + (nodes_to_dcnnl_map_3, 7, sinabs_edges_3, [0, 8], expected_output_3), + (nodes_to_dcnnl_map_3, 8, sinabs_edges_3, [0, 8], expected_output_3), + (nodes_to_dcnnl_map_4, 0, sinabs_edges_4, [0], expected_output_4), + (nodes_to_dcnnl_map_4, 1, sinabs_edges_4, [0], expected_output_4), + (nodes_to_dcnnl_map_4, 2, sinabs_edges_4, [0], expected_output_4), + (nodes_to_dcnnl_map_4, 3, sinabs_edges_4, [0], expected_output_4), + (nodes_to_dcnnl_map_4, 4, sinabs_edges_4, [0], expected_output_4), + (nodes_to_dcnnl_map_4, 5, sinabs_edges_4, [0], expected_output_4), +] diff --git a/tests/test_dynapcnnlayer/model_dummy_1.py b/tests/test_dynapcnnlayer/model_dummy_1.py index dd779af7..26f2bf91 100644 --- a/tests/test_dynapcnnlayer/model_dummy_1.py +++ b/tests/test_dynapcnnlayer/model_dummy_1.py @@ -3,90 +3,116 @@ # implementing "a network with residual connections" example in https://github.com/synsense/sinabs/issues/181 import torch.nn as nn -from sinabs.layers import IAFSqueeze, SumPool2d + from sinabs.activation.surrogate_gradient_fn import PeriodicExponential +from sinabs.layers import IAFSqueeze, SumPool2d nodes_to_dcnnl_map_1 = { 0: { 0: { - 'layer': nn.Conv2d(2, 10, kernel_size=(2, 2), stride=(1, 1), bias=False), - 'input_shape': (2, 34, 34), - 'output_shape': (10, 33, 33) - }, + "layer": nn.Conv2d(2, 10, kernel_size=(2, 2), stride=(1, 1), bias=False), + "input_shape": (2, 34, 34), + "output_shape": (10, 33, 33), + }, 1: { - 'layer': IAFSqueeze(batch_size=3, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), - 'input_shape': (10, 33, 33), - 'output_shape': (10, 33, 33) - }, + "layer": IAFSqueeze( + batch_size=3, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ), + "input_shape": (10, 33, 33), + "output_shape": (10, 33, 33), + }, 2: { - 'layer': nn.AvgPool2d(kernel_size=3, stride=3, padding=0), - 'input_shape': (10, 33, 33), - 'output_shape': (10, 11, 11) - }, + "layer": nn.AvgPool2d(kernel_size=3, stride=3, padding=0), + "input_shape": (10, 33, 33), + "output_shape": (10, 11, 11), + }, 3: { - 'layer': nn.AvgPool2d(kernel_size=4, stride=4, padding=0), - 'input_shape': (10, 33, 33), - 'output_shape': (10, 8, 8) - }, - 'destinations': [1, 2], - 'conv_rescale_factor': [] + "layer": nn.AvgPool2d(kernel_size=4, stride=4, padding=0), + "input_shape": (10, 33, 33), + "output_shape": (10, 8, 8), }, + "destinations": [1, 2], + "conv_rescale_factor": [], + }, 1: { 4: { - 'layer': nn.Conv2d(10, 10, kernel_size=(4, 4), stride=(1, 1), bias=False), - 'input_shape': (10, 11, 11), - 'output_shape': (10, 8, 8) - }, + "layer": nn.Conv2d(10, 10, kernel_size=(4, 4), stride=(1, 1), bias=False), + "input_shape": (10, 11, 11), + "output_shape": (10, 8, 8), + }, 6: { - 'layer': IAFSqueeze(batch_size=3, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), - 'input_shape': (10, 8, 8), - 'output_shape': (10, 8, 8) - }, - 'destinations': [2], - 'conv_rescale_factor': [9] + "layer": IAFSqueeze( + batch_size=3, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ), + "input_shape": (10, 8, 8), + "output_shape": (10, 8, 8), }, + "destinations": [2], + "conv_rescale_factor": [9], + }, 2: { 7: { - 'layer': nn.Conv2d(10, 1, kernel_size=(2, 2), stride=(1, 1), bias=False), - 'input_shape': (10, 8, 8), - 'output_shape': (1, 7, 7) - }, + "layer": nn.Conv2d(10, 1, kernel_size=(2, 2), stride=(1, 1), bias=False), + "input_shape": (10, 8, 8), + "output_shape": (1, 7, 7), + }, 8: { - 'layer': IAFSqueeze(batch_size=3, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), - 'input_shape': (1, 7, 7), - 'output_shape': (1, 7, 7) - }, - 'destinations': [3], - 'conv_rescale_factor': [16] + "layer": IAFSqueeze( + batch_size=3, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ), + "input_shape": (1, 7, 7), + "output_shape": (1, 7, 7), }, + "destinations": [3], + "conv_rescale_factor": [16], + }, 3: { 9: { - 'layer': nn.Linear(in_features=49, out_features=500, bias=False), - 'input_shape': (1, 7, 7), - 'output_shape': (500,) - }, + "layer": nn.Linear(in_features=49, out_features=500, bias=False), + "input_shape": (1, 7, 7), + "output_shape": (500,), + }, 10: { - 'layer': IAFSqueeze(batch_size=3, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), - 'input_shape': (500,), - 'output_shape': (500,) - }, - 'destinations': [4], - 'conv_rescale_factor': [] + "layer": IAFSqueeze( + batch_size=3, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ), + "input_shape": (500,), + "output_shape": (500,), }, + "destinations": [4], + "conv_rescale_factor": [], + }, 4: { 11: { - 'layer': nn.Linear(in_features=500, out_features=10, bias=False), - 'input_shape': (500,), - 'output_shape': (10,) - }, + "layer": nn.Linear(in_features=500, out_features=10, bias=False), + "input_shape": (500,), + "output_shape": (10,), + }, 12: { - 'layer': IAFSqueeze(batch_size=3, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), - 'input_shape': (10,), - 'output_shape': (10,) - }, - 'destinations': [], - 'conv_rescale_factor': [] - } + "layer": IAFSqueeze( + batch_size=3, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ), + "input_shape": (10,), + "output_shape": (10,), + }, + "destinations": [], + "conv_rescale_factor": [], + }, } sinabs_edges_1 = [ @@ -106,63 +132,63 @@ expected_output_1 = { 0: { - 'dpcnnl_index': 0, - 'conv_node_id': 0, - 'conv_in_shape': (2, 34, 34), - 'conv_out_shape': (10, 33, 33), - 'spk_node_id': 1, - 'pool_node_id': [2, 3], - 'conv_rescaling_factor': None, - 'dynapcnnlayer_destination': [1, 2], - 'nodes_destinations': {2: [4], 3: [7]}, - 'entry_point': True, + "dpcnnl_index": 0, + "conv_node_id": 0, + "conv_in_shape": (2, 34, 34), + "conv_out_shape": (10, 33, 33), + "spk_node_id": 1, + "pool_node_id": [2, 3], + "conv_rescaling_factor": None, + "dynapcnnlayer_destination": [1, 2], + "nodes_destinations": {2: [4], 3: [7]}, + "entry_point": True, }, 1: { - 'dpcnnl_index': 1, - 'conv_node_id': 4, - 'conv_in_shape': (10, 11, 11), - 'conv_out_shape': (10, 8, 8), - 'spk_node_id': 6, - 'pool_node_id': [], - 'conv_rescaling_factor': 4.5, - 'dynapcnnlayer_destination': [2], - 'nodes_destinations': {6: [7]}, - 'entry_point': False, + "dpcnnl_index": 1, + "conv_node_id": 4, + "conv_in_shape": (10, 11, 11), + "conv_out_shape": (10, 8, 8), + "spk_node_id": 6, + "pool_node_id": [], + "conv_rescaling_factor": 4.5, + "dynapcnnlayer_destination": [2], + "nodes_destinations": {6: [7]}, + "entry_point": False, }, 2: { - 'dpcnnl_index': 2, - 'conv_node_id': 7, - 'conv_in_shape': (10, 8, 8), - 'conv_out_shape': (1, 7, 7), - 'spk_node_id': 8, - 'pool_node_id': [], - 'conv_rescaling_factor': 8.0, - 'dynapcnnlayer_destination': [3], - 'nodes_destinations': {8: [9]}, - 'entry_point': False, + "dpcnnl_index": 2, + "conv_node_id": 7, + "conv_in_shape": (10, 8, 8), + "conv_out_shape": (1, 7, 7), + "spk_node_id": 8, + "pool_node_id": [], + "conv_rescaling_factor": 8.0, + "dynapcnnlayer_destination": [3], + "nodes_destinations": {8: [9]}, + "entry_point": False, }, 3: { - 'dpcnnl_index': 3, - 'conv_node_id': 9, - 'conv_in_shape': (1, 7, 7), - 'conv_out_shape': (500, 1, 1), - 'spk_node_id': 10, - 'pool_node_id': [], - 'conv_rescaling_factor': None, - 'dynapcnnlayer_destination': [4], - 'nodes_destinations': {10: [11]}, - 'entry_point': False, + "dpcnnl_index": 3, + "conv_node_id": 9, + "conv_in_shape": (1, 7, 7), + "conv_out_shape": (500, 1, 1), + "spk_node_id": 10, + "pool_node_id": [], + "conv_rescaling_factor": None, + "dynapcnnlayer_destination": [4], + "nodes_destinations": {10: [11]}, + "entry_point": False, }, 4: { - 'dpcnnl_index': 4, - 'conv_node_id': 11, - 'conv_in_shape': (500, 1, 1), - 'conv_out_shape': (10, 1, 1), - 'spk_node_id': 12, - 'pool_node_id': [], - 'conv_rescaling_factor': None, - 'dynapcnnlayer_destination': [], - 'nodes_destinations': {}, - 'entry_point': False, + "dpcnnl_index": 4, + "conv_node_id": 11, + "conv_in_shape": (500, 1, 1), + "conv_out_shape": (10, 1, 1), + "spk_node_id": 12, + "pool_node_id": [], + "conv_rescaling_factor": None, + "dynapcnnlayer_destination": [], + "nodes_destinations": {}, + "entry_point": False, }, -} \ No newline at end of file +} diff --git a/tests/test_dynapcnnlayer/model_dummy_2.py b/tests/test_dynapcnnlayer/model_dummy_2.py index b15a4cb7..d682f936 100644 --- a/tests/test_dynapcnnlayer/model_dummy_2.py +++ b/tests/test_dynapcnnlayer/model_dummy_2.py @@ -3,106 +3,159 @@ # implementing "a network with a merge and a split" in https://github.com/synsense/sinabs/issues/181 import torch.nn as nn -from sinabs.layers import IAFSqueeze, SumPool2d + from sinabs.activation.surrogate_gradient_fn import PeriodicExponential +from sinabs.layers import IAFSqueeze, SumPool2d nodes_to_dcnnl_map_2 = { 0: { - 0: {'layer': nn.Conv2d(2, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), - 'input_shape': (2, 34, 34), - 'output_shape': (4, 33, 33) - }, - 1: {'layer': IAFSqueeze(batch_size=8, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), - 'input_shape': (4, 33, 33), - 'output_shape': (4, 33, 33) - }, - 'destinations': [1], - 'conv_rescale_factor': [] + 0: { + "layer": nn.Conv2d(2, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), + "input_shape": (2, 34, 34), + "output_shape": (4, 33, 33), }, + 1: { + "layer": IAFSqueeze( + batch_size=8, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ), + "input_shape": (4, 33, 33), + "output_shape": (4, 33, 33), + }, + "destinations": [1], + "conv_rescale_factor": [], + }, 1: { - 2: {'layer': nn.Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), - 'input_shape': (4, 33, 33), - 'output_shape': (4, 32, 32) - }, - 3: {'layer': IAFSqueeze(batch_size=8, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), - 'input_shape': (4, 32, 32), - 'output_shape': (4, 32, 32) - }, - 4: {'layer': SumPool2d(kernel_size=2, stride=2, ceil_mode=False), - 'input_shape': (4, 32, 32), - 'output_shape': (4, 16, 16) - }, - 'destinations': [2, 3], - 'conv_rescale_factor': [] + 2: { + "layer": nn.Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), + "input_shape": (4, 33, 33), + "output_shape": (4, 32, 32), + }, + 3: { + "layer": IAFSqueeze( + batch_size=8, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ), + "input_shape": (4, 32, 32), + "output_shape": (4, 32, 32), + }, + 4: { + "layer": SumPool2d(kernel_size=2, stride=2, ceil_mode=False), + "input_shape": (4, 32, 32), + "output_shape": (4, 16, 16), + }, + "destinations": [2, 3], + "conv_rescale_factor": [], }, 2: { - 5: {'layer': nn.Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), - 'input_shape': (4, 16, 16), - 'output_shape': (4, 15, 15) - }, - 7: {'layer': IAFSqueeze(batch_size=8, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), - 'input_shape': (4, 15, 15), - 'output_shape': (4, 15, 15) - }, - 8: {'layer': SumPool2d(kernel_size=2, stride=2, ceil_mode=False), - 'input_shape': (4, 15, 15), - 'output_shape': (4, 7, 7) - }, - 'destinations': [4], - 'conv_rescale_factor': [] + 5: { + "layer": nn.Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), + "input_shape": (4, 16, 16), + "output_shape": (4, 15, 15), + }, + 7: { + "layer": IAFSqueeze( + batch_size=8, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ), + "input_shape": (4, 15, 15), + "output_shape": (4, 15, 15), + }, + 8: { + "layer": SumPool2d(kernel_size=2, stride=2, ceil_mode=False), + "input_shape": (4, 15, 15), + "output_shape": (4, 7, 7), + }, + "destinations": [4], + "conv_rescale_factor": [], }, 3: { - 6: {'layer': nn.Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), - 'input_shape': (4, 16, 16), - 'output_shape': (4, 15, 15) - }, - 11: {'layer': IAFSqueeze(batch_size=8, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), - 'input_shape': (4, 15, 15), - 'output_shape': (4, 15, 15) - }, - 12: {'layer': SumPool2d(kernel_size=2, stride=2, ceil_mode=False), - 'input_shape': (4, 15, 15), - 'output_shape': (4, 7, 7) - }, - 'destinations': [6], - 'conv_rescale_factor': [] + 6: { + "layer": nn.Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), + "input_shape": (4, 16, 16), + "output_shape": (4, 15, 15), + }, + 11: { + "layer": IAFSqueeze( + batch_size=8, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ), + "input_shape": (4, 15, 15), + "output_shape": (4, 15, 15), + }, + 12: { + "layer": SumPool2d(kernel_size=2, stride=2, ceil_mode=False), + "input_shape": (4, 15, 15), + "output_shape": (4, 7, 7), + }, + "destinations": [6], + "conv_rescale_factor": [], }, 4: { - 9: {'layer': nn.Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), - 'input_shape': (4, 7, 7), - 'output_shape': (4, 6, 6) - }, - 10: {'layer': IAFSqueeze(batch_size=8, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), - 'input_shape': (4, 6, 6), - 'output_shape': (4, 6, 6) - }, - 'destinations': [5], - 'conv_rescale_factor': [] + 9: { + "layer": nn.Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), + "input_shape": (4, 7, 7), + "output_shape": (4, 6, 6), + }, + 10: { + "layer": IAFSqueeze( + batch_size=8, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ), + "input_shape": (4, 6, 6), + "output_shape": (4, 6, 6), + }, + "destinations": [5], + "conv_rescale_factor": [], }, - 5 :{ - 15: {'layer': nn.Linear(in_features=144, out_features=10, bias=False), - 'input_shape': (4, 6, 6), - 'output_shape': (10,) - }, - 16: {'layer': IAFSqueeze(batch_size=8, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), - 'input_shape': (10,), - 'output_shape': (10,) - }, - 'destinations': [], - 'conv_rescale_factor': [] + 5: { + 15: { + "layer": nn.Linear(in_features=144, out_features=10, bias=False), + "input_shape": (4, 6, 6), + "output_shape": (10,), + }, + 16: { + "layer": IAFSqueeze( + batch_size=8, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ), + "input_shape": (10,), + "output_shape": (10,), + }, + "destinations": [], + "conv_rescale_factor": [], }, 6: { - 13: {'layer': nn.Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), - 'input_shape': (4, 7, 7), - 'output_shape': (4, 6, 6) - }, - 14: {'layer': IAFSqueeze(batch_size=8, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), - 'input_shape': (4, 6, 6), - 'output_shape': (4, 6, 6) - }, - 'destinations': [5], - 'conv_rescale_factor': [] - } + 13: { + "layer": nn.Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), + "input_shape": (4, 7, 7), + "output_shape": (4, 6, 6), + }, + 14: { + "layer": IAFSqueeze( + batch_size=8, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ), + "input_shape": (4, 6, 6), + "output_shape": (4, 6, 6), + }, + "destinations": [5], + "conv_rescale_factor": [], + }, } sinabs_edges_2 = [ @@ -127,87 +180,87 @@ expected_output_2 = { 0: { - 'dpcnnl_index': 0, - 'conv_node_id': 0, - 'conv_in_shape': (2, 34, 34), - 'conv_out_shape': (4, 33, 33), - 'spk_node_id': 1, - 'pool_node_id': [], - 'conv_rescaling_factor': None, - 'dynapcnnlayer_destination': [1], - 'nodes_destinations': {1: [2]}, - 'entry_point': True, + "dpcnnl_index": 0, + "conv_node_id": 0, + "conv_in_shape": (2, 34, 34), + "conv_out_shape": (4, 33, 33), + "spk_node_id": 1, + "pool_node_id": [], + "conv_rescaling_factor": None, + "dynapcnnlayer_destination": [1], + "nodes_destinations": {1: [2]}, + "entry_point": True, }, 1: { - 'dpcnnl_index': 1, - 'conv_node_id': 2, - 'conv_in_shape': (4, 33, 33), - 'conv_out_shape': (4, 32, 32), - 'spk_node_id': 3, - 'pool_node_id': [4], - 'conv_rescaling_factor': None, - 'dynapcnnlayer_destination': [2, 3], - 'nodes_destinations': {4: [5, 6]}, - 'entry_point': False, + "dpcnnl_index": 1, + "conv_node_id": 2, + "conv_in_shape": (4, 33, 33), + "conv_out_shape": (4, 32, 32), + "spk_node_id": 3, + "pool_node_id": [4], + "conv_rescaling_factor": None, + "dynapcnnlayer_destination": [2, 3], + "nodes_destinations": {4: [5, 6]}, + "entry_point": False, }, 2: { - 'dpcnnl_index': 2, - 'conv_node_id': 5, - 'conv_in_shape': (4, 16, 16), - 'conv_out_shape': (4, 15, 15), - 'spk_node_id': 7, - 'pool_node_id': [8], - 'conv_rescaling_factor': None, - 'dynapcnnlayer_destination': [4], - 'nodes_destinations': {8: [9]}, - 'entry_point': False, + "dpcnnl_index": 2, + "conv_node_id": 5, + "conv_in_shape": (4, 16, 16), + "conv_out_shape": (4, 15, 15), + "spk_node_id": 7, + "pool_node_id": [8], + "conv_rescaling_factor": None, + "dynapcnnlayer_destination": [4], + "nodes_destinations": {8: [9]}, + "entry_point": False, }, 3: { - 'dpcnnl_index': 3, - 'conv_node_id': 6, - 'conv_in_shape': (4, 16, 16), - 'conv_out_shape': (4, 15, 15), - 'spk_node_id': 11, - 'pool_node_id': [12], - 'conv_rescaling_factor': None, - 'dynapcnnlayer_destination': [6], - 'nodes_destinations': {12: [13]}, - 'entry_point': False, + "dpcnnl_index": 3, + "conv_node_id": 6, + "conv_in_shape": (4, 16, 16), + "conv_out_shape": (4, 15, 15), + "spk_node_id": 11, + "pool_node_id": [12], + "conv_rescaling_factor": None, + "dynapcnnlayer_destination": [6], + "nodes_destinations": {12: [13]}, + "entry_point": False, }, 4: { - 'dpcnnl_index': 4, - 'conv_node_id': 9, - 'conv_in_shape': (4, 7, 7), - 'conv_out_shape': (4, 6, 6), - 'spk_node_id': 10, - 'pool_node_id': [], - 'conv_rescaling_factor': None, - 'dynapcnnlayer_destination': [5], - 'nodes_destinations': {10: [15]}, - 'entry_point': False, + "dpcnnl_index": 4, + "conv_node_id": 9, + "conv_in_shape": (4, 7, 7), + "conv_out_shape": (4, 6, 6), + "spk_node_id": 10, + "pool_node_id": [], + "conv_rescaling_factor": None, + "dynapcnnlayer_destination": [5], + "nodes_destinations": {10: [15]}, + "entry_point": False, }, 5: { - 'dpcnnl_index': 5, - 'conv_node_id': 15, - 'conv_in_shape': (4, 6, 6), - 'conv_out_shape': (10, 1, 1), - 'spk_node_id': 16, - 'pool_node_id': [], - 'conv_rescaling_factor': None, - 'dynapcnnlayer_destination': [], - 'nodes_destinations': {}, - 'entry_point': False, + "dpcnnl_index": 5, + "conv_node_id": 15, + "conv_in_shape": (4, 6, 6), + "conv_out_shape": (10, 1, 1), + "spk_node_id": 16, + "pool_node_id": [], + "conv_rescaling_factor": None, + "dynapcnnlayer_destination": [], + "nodes_destinations": {}, + "entry_point": False, }, 6: { - 'dpcnnl_index': 6, - 'conv_node_id': 13, - 'conv_in_shape': (4, 7, 7), - 'conv_out_shape': (4, 6, 6), - 'spk_node_id': 14, - 'pool_node_id': [], - 'conv_rescaling_factor': None, - 'dynapcnnlayer_destination': [5], - 'nodes_destinations': {14: [15]}, - 'entry_point': False, + "dpcnnl_index": 6, + "conv_node_id": 13, + "conv_in_shape": (4, 7, 7), + "conv_out_shape": (4, 6, 6), + "spk_node_id": 14, + "pool_node_id": [], + "conv_rescaling_factor": None, + "dynapcnnlayer_destination": [5], + "nodes_destinations": {14: [15]}, + "entry_point": False, }, -} \ No newline at end of file +} diff --git a/tests/test_dynapcnnlayer/model_dummy_3.py b/tests/test_dynapcnnlayer/model_dummy_3.py index 487d6d93..cad0aad6 100644 --- a/tests/test_dynapcnnlayer/model_dummy_3.py +++ b/tests/test_dynapcnnlayer/model_dummy_3.py @@ -3,154 +3,202 @@ # implementing "two networks with merging outputs" in https://github.com/synsense/sinabs/issues/181 import torch.nn as nn -from sinabs.layers import IAFSqueeze, SumPool2d + from sinabs.activation.surrogate_gradient_fn import PeriodicExponential +from sinabs.layers import IAFSqueeze, SumPool2d nodes_to_dcnnl_map_3 = { 0: { 0: { - 'layer': nn.Conv2d(2, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), - 'input_shape': (2, 34, 34), - 'output_shape': (4, 33, 33) - }, + "layer": nn.Conv2d(2, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), + "input_shape": (2, 34, 34), + "output_shape": (4, 33, 33), + }, 1: { - 'layer': IAFSqueeze(batch_size=2, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), - 'input_shape': (4, 33, 33), - 'output_shape': (4, 33, 33) - }, - 'destinations': [1], - 'conv_rescale_factor': [] + "layer": IAFSqueeze( + batch_size=2, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ), + "input_shape": (4, 33, 33), + "output_shape": (4, 33, 33), + }, + "destinations": [1], + "conv_rescale_factor": [], }, 1: { 2: { - 'layer': nn.Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), - 'input_shape': (4, 33, 33), - 'output_shape': (4, 32, 32) - }, + "layer": nn.Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), + "input_shape": (4, 33, 33), + "output_shape": (4, 32, 32), + }, 3: { - 'layer': IAFSqueeze(batch_size=2, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), - 'input_shape': (4, 32, 32), - 'output_shape': (4, 32, 32) - }, + "layer": IAFSqueeze( + batch_size=2, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ), + "input_shape": (4, 32, 32), + "output_shape": (4, 32, 32), + }, 4: { - 'layer': SumPool2d(kernel_size=2, stride=2, ceil_mode=False), - 'input_shape': (4, 32, 32), - 'output_shape': (4, 16, 16) - }, - 'destinations': [2], - 'conv_rescale_factor': [] + "layer": SumPool2d(kernel_size=2, stride=2, ceil_mode=False), + "input_shape": (4, 32, 32), + "output_shape": (4, 16, 16), + }, + "destinations": [2], + "conv_rescale_factor": [], }, 2: { 5: { - 'layer': nn.Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), - 'input_shape': (4, 16, 16), - 'output_shape': (4, 15, 15)}, + "layer": nn.Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), + "input_shape": (4, 16, 16), + "output_shape": (4, 15, 15), + }, 6: { - 'layer': IAFSqueeze(batch_size=2, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), - 'input_shape': (4, 15, 15), - 'output_shape': (4, 15, 15) - }, + "layer": IAFSqueeze( + batch_size=2, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ), + "input_shape": (4, 15, 15), + "output_shape": (4, 15, 15), + }, 7: { - 'layer': SumPool2d(kernel_size=2, stride=2, ceil_mode=False), - 'input_shape': (4, 15, 15), - 'output_shape': (4, 7, 7) - }, - 'destinations': [3], - 'conv_rescale_factor': [] + "layer": SumPool2d(kernel_size=2, stride=2, ceil_mode=False), + "input_shape": (4, 15, 15), + "output_shape": (4, 7, 7), + }, + "destinations": [3], + "conv_rescale_factor": [], }, 3: { 17: { - 'layer': nn.Linear(in_features=196, out_features=100, bias=False), - 'input_shape': (4, 7, 7), - 'output_shape': (100,) - }, + "layer": nn.Linear(in_features=196, out_features=100, bias=False), + "input_shape": (4, 7, 7), + "output_shape": (100,), + }, 18: { - 'layer': IAFSqueeze(batch_size=2, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), - 'input_shape': (100,), - 'output_shape': (100,) - }, - 'destinations': [7], - 'conv_rescale_factor': [] + "layer": IAFSqueeze( + batch_size=2, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ), + "input_shape": (100,), + "output_shape": (100,), + }, + "destinations": [7], + "conv_rescale_factor": [], }, 4: { 8: { - 'layer': nn.Conv2d(2, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), - 'input_shape': (2, 34, 34), - 'output_shape': (4, 33, 33)}, + "layer": nn.Conv2d(2, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), + "input_shape": (2, 34, 34), + "output_shape": (4, 33, 33), + }, 9: { - 'layer': IAFSqueeze(batch_size=2, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), - 'input_shape': (4, 33, 33), - 'output_shape': (4, 33, 33) - }, - 'destinations': [5], - 'conv_rescale_factor': [] + "layer": IAFSqueeze( + batch_size=2, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ), + "input_shape": (4, 33, 33), + "output_shape": (4, 33, 33), + }, + "destinations": [5], + "conv_rescale_factor": [], }, 5: { 10: { - 'layer': nn.Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), - 'input_shape': (4, 33, 33), - 'output_shape': (4, 32, 32) + "layer": nn.Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), + "input_shape": (4, 33, 33), + "output_shape": (4, 32, 32), }, 11: { - 'layer': IAFSqueeze(batch_size=2, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), - 'input_shape': (4, 32, 32), - 'output_shape': (4, 32, 32) - }, + "layer": IAFSqueeze( + batch_size=2, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ), + "input_shape": (4, 32, 32), + "output_shape": (4, 32, 32), + }, 12: { - 'layer': SumPool2d(kernel_size=2, stride=2, ceil_mode=False), - 'input_shape': (4, 32, 32), - 'output_shape': (4, 16, 16) - }, - 'destinations': [6], - 'conv_rescale_factor': [] + "layer": SumPool2d(kernel_size=2, stride=2, ceil_mode=False), + "input_shape": (4, 32, 32), + "output_shape": (4, 16, 16), + }, + "destinations": [6], + "conv_rescale_factor": [], }, 6: { 13: { - 'layer': nn.Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), - 'input_shape': (4, 16, 16), - 'output_shape': (4, 15, 15) - }, + "layer": nn.Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), + "input_shape": (4, 16, 16), + "output_shape": (4, 15, 15), + }, 14: { - 'layer': IAFSqueeze(batch_size=2, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), - 'input_shape': (4, 15, 15), - 'output_shape': (4, 15, 15) - }, + "layer": IAFSqueeze( + batch_size=2, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ), + "input_shape": (4, 15, 15), + "output_shape": (4, 15, 15), + }, 15: { - 'layer': SumPool2d(kernel_size=2, stride=2, ceil_mode=False), - 'input_shape': (4, 15, 15), - 'output_shape': (4, 7, 7) - }, - 'destinations': [3], - 'conv_rescale_factor': [] + "layer": SumPool2d(kernel_size=2, stride=2, ceil_mode=False), + "input_shape": (4, 15, 15), + "output_shape": (4, 7, 7), + }, + "destinations": [3], + "conv_rescale_factor": [], }, 7: { 19: { - 'layer': nn.Linear(in_features=100, out_features=100, bias=False), - 'input_shape': (100,), - 'output_shape': (100,) - }, + "layer": nn.Linear(in_features=100, out_features=100, bias=False), + "input_shape": (100,), + "output_shape": (100,), + }, 20: { - 'layer': IAFSqueeze(batch_size=2, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), - 'input_shape': (100,), - 'output_shape': (100,) - }, - 'destinations': [8], - 'conv_rescale_factor': [] + "layer": IAFSqueeze( + batch_size=2, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ), + "input_shape": (100,), + "output_shape": (100,), + }, + "destinations": [8], + "conv_rescale_factor": [], }, 8: { 21: { - 'layer': nn.Linear(in_features=100, out_features=10, bias=False), - 'input_shape': (100,), - 'output_shape': (10,) - }, + "layer": nn.Linear(in_features=100, out_features=10, bias=False), + "input_shape": (100,), + "output_shape": (10,), + }, 22: { - 'layer': IAFSqueeze(batch_size=2, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), - 'input_shape': (10,), - 'output_shape': (10,) - }, - 'destinations': [], - 'conv_rescale_factor': [] - } + "layer": IAFSqueeze( + batch_size=2, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ), + "input_shape": (10,), + "output_shape": (10,), + }, + "destinations": [], + "conv_rescale_factor": [], + }, } sinabs_edges_3 = [ @@ -179,111 +227,111 @@ expected_output_3 = { 0: { - 'dpcnnl_index': 0, - 'conv_node_id': 0, - 'conv_in_shape': (2, 34, 34), - 'conv_out_shape': (4, 33, 33), - 'spk_node_id': 1, - 'pool_node_id': [], - 'conv_rescaling_factor': None, - 'dynapcnnlayer_destination': [1], - 'nodes_destinations': {1: [2]}, - 'entry_point': True, + "dpcnnl_index": 0, + "conv_node_id": 0, + "conv_in_shape": (2, 34, 34), + "conv_out_shape": (4, 33, 33), + "spk_node_id": 1, + "pool_node_id": [], + "conv_rescaling_factor": None, + "dynapcnnlayer_destination": [1], + "nodes_destinations": {1: [2]}, + "entry_point": True, }, 1: { - 'dpcnnl_index': 1, - 'conv_node_id': 2, - 'conv_in_shape': (4, 33, 33), - 'conv_out_shape': (4, 32, 32), - 'spk_node_id': 3, - 'pool_node_id': [4], - 'conv_rescaling_factor': None, - 'dynapcnnlayer_destination': [2], - 'nodes_destinations': {4: [5]}, - 'entry_point': False, + "dpcnnl_index": 1, + "conv_node_id": 2, + "conv_in_shape": (4, 33, 33), + "conv_out_shape": (4, 32, 32), + "spk_node_id": 3, + "pool_node_id": [4], + "conv_rescaling_factor": None, + "dynapcnnlayer_destination": [2], + "nodes_destinations": {4: [5]}, + "entry_point": False, }, 2: { - 'dpcnnl_index': 2, - 'conv_node_id': 5, - 'conv_in_shape': (4, 16, 16), - 'conv_out_shape': (4, 15, 15), - 'spk_node_id': 6, - 'pool_node_id': [7], - 'conv_rescaling_factor': None, - 'dynapcnnlayer_destination': [3], - 'nodes_destinations': {7: [17]}, - 'entry_point': False, + "dpcnnl_index": 2, + "conv_node_id": 5, + "conv_in_shape": (4, 16, 16), + "conv_out_shape": (4, 15, 15), + "spk_node_id": 6, + "pool_node_id": [7], + "conv_rescaling_factor": None, + "dynapcnnlayer_destination": [3], + "nodes_destinations": {7: [17]}, + "entry_point": False, }, 3: { - 'dpcnnl_index': 3, - 'conv_node_id': 17, - 'conv_in_shape': (4, 7, 7), - 'conv_out_shape': (100, 1, 1), - 'spk_node_id': 18, - 'pool_node_id': [], - 'conv_rescaling_factor': None, - 'dynapcnnlayer_destination': [7], - 'nodes_destinations': {18: [19]}, - 'entry_point': False, + "dpcnnl_index": 3, + "conv_node_id": 17, + "conv_in_shape": (4, 7, 7), + "conv_out_shape": (100, 1, 1), + "spk_node_id": 18, + "pool_node_id": [], + "conv_rescaling_factor": None, + "dynapcnnlayer_destination": [7], + "nodes_destinations": {18: [19]}, + "entry_point": False, }, 4: { - 'dpcnnl_index': 4, - 'conv_node_id': 8, - 'conv_in_shape': (2, 34, 34), - 'conv_out_shape': (4, 33, 33), - 'spk_node_id': 9, - 'pool_node_id': [], - 'conv_rescaling_factor': None, - 'dynapcnnlayer_destination': [5], - 'nodes_destinations': {9: [10]}, - 'entry_point': True, + "dpcnnl_index": 4, + "conv_node_id": 8, + "conv_in_shape": (2, 34, 34), + "conv_out_shape": (4, 33, 33), + "spk_node_id": 9, + "pool_node_id": [], + "conv_rescaling_factor": None, + "dynapcnnlayer_destination": [5], + "nodes_destinations": {9: [10]}, + "entry_point": True, }, 5: { - 'dpcnnl_index': 5, - 'conv_node_id': 10, - 'conv_in_shape': (4, 33, 33), - 'conv_out_shape': (4, 32, 32), - 'spk_node_id': 11, - 'pool_node_id': [12], - 'conv_rescaling_factor': None, - 'dynapcnnlayer_destination': [6], - 'nodes_destinations': {12: [13]}, - 'entry_point': False, + "dpcnnl_index": 5, + "conv_node_id": 10, + "conv_in_shape": (4, 33, 33), + "conv_out_shape": (4, 32, 32), + "spk_node_id": 11, + "pool_node_id": [12], + "conv_rescaling_factor": None, + "dynapcnnlayer_destination": [6], + "nodes_destinations": {12: [13]}, + "entry_point": False, }, 6: { - 'dpcnnl_index': 6, - 'conv_node_id': 13, - 'conv_in_shape': (4, 16, 16), - 'conv_out_shape': (4, 15, 15), - 'spk_node_id': 14, - 'pool_node_id': [15], - 'conv_rescaling_factor': None, - 'dynapcnnlayer_destination': [3], - 'nodes_destinations': {15: [17]}, - 'entry_point': False, + "dpcnnl_index": 6, + "conv_node_id": 13, + "conv_in_shape": (4, 16, 16), + "conv_out_shape": (4, 15, 15), + "spk_node_id": 14, + "pool_node_id": [15], + "conv_rescaling_factor": None, + "dynapcnnlayer_destination": [3], + "nodes_destinations": {15: [17]}, + "entry_point": False, }, 7: { - 'dpcnnl_index': 7, - 'conv_node_id': 19, - 'conv_in_shape': (100, 1, 1), - 'conv_out_shape': (100, 1, 1), - 'spk_node_id': 20, - 'pool_node_id': [], - 'conv_rescaling_factor': None, - 'dynapcnnlayer_destination': [8], - 'nodes_destinations': {20: [21]}, - 'entry_point': False, + "dpcnnl_index": 7, + "conv_node_id": 19, + "conv_in_shape": (100, 1, 1), + "conv_out_shape": (100, 1, 1), + "spk_node_id": 20, + "pool_node_id": [], + "conv_rescaling_factor": None, + "dynapcnnlayer_destination": [8], + "nodes_destinations": {20: [21]}, + "entry_point": False, }, 8: { - 'dpcnnl_index': 8, - 'conv_node_id': 21, - 'conv_in_shape': (100, 1, 1), - 'conv_out_shape': (10, 1, 1), - 'spk_node_id': 22, - 'pool_node_id': [], - 'conv_rescaling_factor': None, - 'dynapcnnlayer_destination': [], - 'nodes_destinations': {}, - 'entry_point': False, + "dpcnnl_index": 8, + "conv_node_id": 21, + "conv_in_shape": (100, 1, 1), + "conv_out_shape": (10, 1, 1), + "spk_node_id": 22, + "pool_node_id": [], + "conv_rescaling_factor": None, + "dynapcnnlayer_destination": [], + "nodes_destinations": {}, + "entry_point": False, }, -} \ No newline at end of file +} diff --git a/tests/test_dynapcnnlayer/model_dummy_4.py b/tests/test_dynapcnnlayer/model_dummy_4.py index 3dca038c..cd083c53 100644 --- a/tests/test_dynapcnnlayer/model_dummy_4.py +++ b/tests/test_dynapcnnlayer/model_dummy_4.py @@ -3,114 +3,145 @@ # implementing "a complex network structure" example in https://github.com/synsense/sinabs/issues/181 . """ import torch.nn as nn -from sinabs.layers import IAFSqueeze, SumPool2d + from sinabs.activation.surrogate_gradient_fn import PeriodicExponential +from sinabs.layers import IAFSqueeze, SumPool2d nodes_to_dcnnl_map_4 = { 0: { 0: { - 'layer': nn.Conv2d(2, 1, kernel_size=(2, 2), stride=(1, 1), bias=False), - 'input_shape': (2, 34, 34), - 'output_shape': (1, 33, 33) - }, + "layer": nn.Conv2d(2, 1, kernel_size=(2, 2), stride=(1, 1), bias=False), + "input_shape": (2, 34, 34), + "output_shape": (1, 33, 33), + }, 1: { - 'layer': IAFSqueeze(batch_size=2, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), - 'input_shape': (1, 33, 33), - 'output_shape': (1, 33, 33) - }, - 'destinations': [1, 2], - 'conv_rescale_factor': [] + "layer": IAFSqueeze( + batch_size=2, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ), + "input_shape": (1, 33, 33), + "output_shape": (1, 33, 33), + }, + "destinations": [1, 2], + "conv_rescale_factor": [], }, 1: { 2: { - 'layer': nn.Conv2d(1, 1, kernel_size=(2, 2), stride=(1, 1), bias=False), - 'input_shape': (1, 33, 33), - 'output_shape': (1, 32, 32) - }, + "layer": nn.Conv2d(1, 1, kernel_size=(2, 2), stride=(1, 1), bias=False), + "input_shape": (1, 33, 33), + "output_shape": (1, 32, 32), + }, 4: { - 'layer': IAFSqueeze(batch_size=2, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), - 'input_shape': (1, 32, 32), - 'output_shape': (1, 32, 32) - }, + "layer": IAFSqueeze( + batch_size=2, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ), + "input_shape": (1, 32, 32), + "output_shape": (1, 32, 32), + }, 5: { - 'layer': SumPool2d(kernel_size=2, stride=2, ceil_mode=False), - 'input_shape': (1, 32, 32), - 'output_shape': (1, 16, 16) - }, - 'destinations': [3], - 'conv_rescale_factor': [] + "layer": SumPool2d(kernel_size=2, stride=2, ceil_mode=False), + "input_shape": (1, 32, 32), + "output_shape": (1, 16, 16), + }, + "destinations": [3], + "conv_rescale_factor": [], }, 2: { 3: { - 'layer': nn.Conv2d(1, 1, kernel_size=(2, 2), stride=(1, 1), bias=False), - 'input_shape': (1, 33, 33), - 'output_shape': (1, 32, 32) - }, + "layer": nn.Conv2d(1, 1, kernel_size=(2, 2), stride=(1, 1), bias=False), + "input_shape": (1, 33, 33), + "output_shape": (1, 32, 32), + }, 7: { - 'layer': IAFSqueeze(batch_size=2, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), - 'input_shape': (1, 32, 32), - 'output_shape': (1, 32, 32) - }, + "layer": IAFSqueeze( + batch_size=2, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ), + "input_shape": (1, 32, 32), + "output_shape": (1, 32, 32), + }, 8: { - 'layer': SumPool2d(kernel_size=2, stride=2, ceil_mode=False), - 'input_shape': (1, 32, 32), - 'output_shape': (1, 16, 16) - }, + "layer": SumPool2d(kernel_size=2, stride=2, ceil_mode=False), + "input_shape": (1, 32, 32), + "output_shape": (1, 16, 16), + }, 9: { - 'layer': SumPool2d(kernel_size=5, stride=5, ceil_mode=False), - 'input_shape': (1, 32, 32), - 'output_shape': (1, 6, 6) - }, - 'destinations': [3, 4], - 'conv_rescale_factor': [] + "layer": SumPool2d(kernel_size=5, stride=5, ceil_mode=False), + "input_shape": (1, 32, 32), + "output_shape": (1, 6, 6), + }, + "destinations": [3, 4], + "conv_rescale_factor": [], }, 3: { 11: { - 'layer': nn.Conv2d(1, 1, kernel_size=(2, 2), stride=(1, 1), bias=False), - 'input_shape': (1, 16, 16), - 'output_shape': (1, 15, 15) + "layer": nn.Conv2d(1, 1, kernel_size=(2, 2), stride=(1, 1), bias=False), + "input_shape": (1, 16, 16), + "output_shape": (1, 15, 15), }, 12: { - 'layer': IAFSqueeze(batch_size=2, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), - 'input_shape': (1, 15, 15), - 'output_shape': (1, 15, 15) + "layer": IAFSqueeze( + batch_size=2, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ), + "input_shape": (1, 15, 15), + "output_shape": (1, 15, 15), }, 13: { - 'layer': SumPool2d(kernel_size=3, stride=3, ceil_mode=False), - 'input_shape': (1, 15, 15), - 'output_shape': (1, 5, 5) - }, - 'destinations': [5], - 'conv_rescale_factor': [] + "layer": SumPool2d(kernel_size=3, stride=3, ceil_mode=False), + "input_shape": (1, 15, 15), + "output_shape": (1, 5, 5), + }, + "destinations": [5], + "conv_rescale_factor": [], }, 4: { 10: { - 'layer': nn.Conv2d(1, 1, kernel_size=(2, 2), stride=(1, 1), bias=False), - 'input_shape': (1, 6, 6), - 'output_shape': (1, 5, 5) - }, + "layer": nn.Conv2d(1, 1, kernel_size=(2, 2), stride=(1, 1), bias=False), + "input_shape": (1, 6, 6), + "output_shape": (1, 5, 5), + }, 15: { - 'layer': IAFSqueeze(batch_size=2, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), - 'input_shape': (1, 5, 5), - 'output_shape': (1, 5, 5) - }, - 'destinations': [5], - 'conv_rescale_factor': [] + "layer": IAFSqueeze( + batch_size=2, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ), + "input_shape": (1, 5, 5), + "output_shape": (1, 5, 5), + }, + "destinations": [5], + "conv_rescale_factor": [], }, 5: { 16: { - 'layer': nn.Linear(in_features=25, out_features=10, bias=False), - 'input_shape': (1, 5, 5), - 'output_shape': (10,) + "layer": nn.Linear(in_features=25, out_features=10, bias=False), + "input_shape": (1, 5, 5), + "output_shape": (10,), }, 17: { - 'layer': IAFSqueeze(batch_size=2, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()), - 'input_shape': (10,), - 'output_shape': (10,) - }, - 'destinations': [], - 'conv_rescale_factor': [] - } + "layer": IAFSqueeze( + batch_size=2, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ), + "input_shape": (10,), + "output_shape": (10,), + }, + "destinations": [], + "conv_rescale_factor": [], + }, } sinabs_edges_4 = [ @@ -135,75 +166,75 @@ expected_output_4 = { 0: { - 'dpcnnl_index': 0, - 'conv_node_id': 0, - 'conv_in_shape': (2, 34, 34), - 'conv_out_shape': (1, 33, 33), - 'spk_node_id': 1, - 'pool_node_id': [], - 'conv_rescaling_factor': None, - 'dynapcnnlayer_destination': [1, 2], - 'nodes_destinations': {1: [2, 3]}, - 'entry_point': True, + "dpcnnl_index": 0, + "conv_node_id": 0, + "conv_in_shape": (2, 34, 34), + "conv_out_shape": (1, 33, 33), + "spk_node_id": 1, + "pool_node_id": [], + "conv_rescaling_factor": None, + "dynapcnnlayer_destination": [1, 2], + "nodes_destinations": {1: [2, 3]}, + "entry_point": True, }, 1: { - 'dpcnnl_index': 1, - 'conv_node_id': 2, - 'conv_in_shape': (1, 33, 33), - 'conv_out_shape': (1, 32, 32), - 'spk_node_id': 4, - 'pool_node_id': [5], - 'conv_rescaling_factor': None, - 'dynapcnnlayer_destination': [3], - 'nodes_destinations': {5: [11]}, - 'entry_point': False, + "dpcnnl_index": 1, + "conv_node_id": 2, + "conv_in_shape": (1, 33, 33), + "conv_out_shape": (1, 32, 32), + "spk_node_id": 4, + "pool_node_id": [5], + "conv_rescaling_factor": None, + "dynapcnnlayer_destination": [3], + "nodes_destinations": {5: [11]}, + "entry_point": False, }, 2: { - 'dpcnnl_index': 2, - 'conv_node_id': 3, - 'conv_in_shape': (1, 33, 33), - 'conv_out_shape': (1, 32, 32), - 'spk_node_id': 7, - 'pool_node_id': [8, 9], - 'conv_rescaling_factor': None, - 'dynapcnnlayer_destination': [3, 4], - 'nodes_destinations': {8: [11], 9: [10]}, - 'entry_point': False, + "dpcnnl_index": 2, + "conv_node_id": 3, + "conv_in_shape": (1, 33, 33), + "conv_out_shape": (1, 32, 32), + "spk_node_id": 7, + "pool_node_id": [8, 9], + "conv_rescaling_factor": None, + "dynapcnnlayer_destination": [3, 4], + "nodes_destinations": {8: [11], 9: [10]}, + "entry_point": False, }, 3: { - 'dpcnnl_index': 3, - 'conv_node_id': 11, - 'conv_in_shape': (1, 16, 16), - 'conv_out_shape': (1, 15, 15), - 'spk_node_id': 12, - 'pool_node_id': [13], - 'conv_rescaling_factor': None, - 'dynapcnnlayer_destination': [5], - 'nodes_destinations': {13: [16]}, - 'entry_point': False, + "dpcnnl_index": 3, + "conv_node_id": 11, + "conv_in_shape": (1, 16, 16), + "conv_out_shape": (1, 15, 15), + "spk_node_id": 12, + "pool_node_id": [13], + "conv_rescaling_factor": None, + "dynapcnnlayer_destination": [5], + "nodes_destinations": {13: [16]}, + "entry_point": False, }, 4: { - 'dpcnnl_index': 4, - 'conv_node_id': 10, - 'conv_in_shape': (1, 6, 6), - 'conv_out_shape': (1, 5, 5), - 'spk_node_id': 15, - 'pool_node_id': [], - 'conv_rescaling_factor': None, - 'dynapcnnlayer_destination': [5], - 'nodes_destinations': {15: [16]}, - 'entry_point': False, + "dpcnnl_index": 4, + "conv_node_id": 10, + "conv_in_shape": (1, 6, 6), + "conv_out_shape": (1, 5, 5), + "spk_node_id": 15, + "pool_node_id": [], + "conv_rescaling_factor": None, + "dynapcnnlayer_destination": [5], + "nodes_destinations": {15: [16]}, + "entry_point": False, }, 5: { - 'dpcnnl_index': 5, - 'conv_node_id': 16, - 'conv_in_shape': (1, 5, 5), - 'conv_out_shape': (10, 1, 1), - 'spk_node_id': 17, - 'pool_node_id': [], - 'conv_rescaling_factor': None, - 'dynapcnnlayer_destination': [], - 'nodes_destinations': {}, - 'entry_point': False, + "dpcnnl_index": 5, + "conv_node_id": 16, + "conv_in_shape": (1, 5, 5), + "conv_out_shape": (10, 1, 1), + "spk_node_id": 17, + "pool_node_id": [], + "conv_rescaling_factor": None, + "dynapcnnlayer_destination": [], + "nodes_destinations": {}, + "entry_point": False, }, -} \ No newline at end of file +} diff --git a/tests/test_dynapcnnlayer/test_dynapcnnlayer.py b/tests/test_dynapcnnlayer/test_dynapcnnlayer.py index fd0bca3e..8791bd67 100644 --- a/tests/test_dynapcnnlayer/test_dynapcnnlayer.py +++ b/tests/test_dynapcnnlayer/test_dynapcnnlayer.py @@ -2,14 +2,22 @@ # contact : wsoaresgirao@gmail.com import pytest -from sinabs.backend.dynapcnn.utils import construct_dynapcnnlayer, construct_layerhandler, update_nodes_io +from conftest_dynapcnnlayer import args_DynapcnnLayer + +from sinabs.backend.dynapcnn.utils import (construct_dynapcnnlayer, + construct_layerhandler, + update_nodes_io) from sinabs.backend.dynapcnn.weight_rescaling_methods import rescale_method_1 -from conftest_dynapcnnlayer import args_DynapcnnLayer -@pytest.mark.parametrize("nodes_to_dcnnl_map, dpcnnl_idx, sinabs_edges, entry_point, expected_output", args_DynapcnnLayer) -def test_DynapcnnLayer(nodes_to_dcnnl_map, dpcnnl_idx, sinabs_edges, entry_point, expected_output): - """ Tests the instantiation of a set of `DynapcnnLayer` belonging to the same SNN and the data computed +@pytest.mark.parametrize( + "nodes_to_dcnnl_map, dpcnnl_idx, sinabs_edges, entry_point, expected_output", + args_DynapcnnLayer, +) +def test_DynapcnnLayer( + nodes_to_dcnnl_map, dpcnnl_idx, sinabs_edges, entry_point, expected_output +): + """Tests the instantiation of a set of `DynapcnnLayer` belonging to the same SNN and the data computed within their constructors and shared among the differntly interacting instances (according to the graph described by `sinabs_edges`). """ @@ -26,40 +34,55 @@ def test_DynapcnnLayer(nodes_to_dcnnl_map, dpcnnl_idx, sinabs_edges, entry_point dynapcnnlayer = construct_dynapcnnlayer(layerhandler) # check if any node (layer) in `dynapcnnlayer` has been modified (e.g. `nn.Linear` turned `nn.Conv2d`). - node, output_shape = layerhandler.get_modified_node_io(nodes_to_dcnnl_map[dpcnnl_idx]) + node, output_shape = layerhandler.get_modified_node_io( + nodes_to_dcnnl_map[dpcnnl_idx] + ) # one of the layers in `dynapcnnlayer` had its type modified (update input shape of nodes receiving from it). if isinstance(node, int) and isinstance(output_shape, tuple): update_nodes_io(node, output_shape, nodes_to_dcnnl_map, sinabs_edges) - dpcnnl_index = expected_output[dpcnnl_idx]['dpcnnl_index'] - conv_node_id = expected_output[dpcnnl_idx]['conv_node_id'] - conv_in_shape = expected_output[dpcnnl_idx]['conv_in_shape'] - conv_out_shape = expected_output[dpcnnl_idx]['conv_out_shape'] - spk_node_id = expected_output[dpcnnl_idx]['spk_node_id'] - pool_node_id = expected_output[dpcnnl_idx]['pool_node_id'] - conv_rescaling_factor = expected_output[dpcnnl_idx]['conv_rescaling_factor'] - dynapcnnlayer_destination = expected_output[dpcnnl_idx]['dynapcnnlayer_destination'] - nodes_destinations = expected_output[dpcnnl_idx]['nodes_destinations'] - entry_point = expected_output[dpcnnl_idx]['entry_point'] + dpcnnl_index = expected_output[dpcnnl_idx]["dpcnnl_index"] + conv_node_id = expected_output[dpcnnl_idx]["conv_node_id"] + conv_in_shape = expected_output[dpcnnl_idx]["conv_in_shape"] + conv_out_shape = expected_output[dpcnnl_idx]["conv_out_shape"] + spk_node_id = expected_output[dpcnnl_idx]["spk_node_id"] + pool_node_id = expected_output[dpcnnl_idx]["pool_node_id"] + conv_rescaling_factor = expected_output[dpcnnl_idx]["conv_rescaling_factor"] + dynapcnnlayer_destination = expected_output[dpcnnl_idx]["dynapcnnlayer_destination"] + nodes_destinations = expected_output[dpcnnl_idx]["nodes_destinations"] + entry_point = expected_output[dpcnnl_idx]["entry_point"] - assert layerhandler.dpcnnl_index == expected_output[dpcnnl_idx]['dpcnnl_index'], \ - f'wrong \'DynapcnnLayer.dpcnnl_index\': ID of the instance should be {dpcnnl_index}.' - assert layerhandler.conv_node_id == expected_output[dpcnnl_idx]['conv_node_id'], \ - f'wrong \'DynapcnnLayer.conv_node_id\': convolution layer should be node {conv_node_id}.' - assert layerhandler.conv_in_shape == expected_output[dpcnnl_idx]['conv_in_shape'], \ - f'wrong \'DynapcnnLayer.conv_in_shape\': input tensor shape of convolution should be {conv_in_shape}.' - assert layerhandler.conv_out_shape == expected_output[dpcnnl_idx]['conv_out_shape'], \ - f'wrong \'DynapcnnLayer.conv_out_shape\': output tensor shape of convolution should be {conv_out_shape}.' - assert layerhandler.spk_node_id == expected_output[dpcnnl_idx]['spk_node_id'], \ - f'wrong \'DynapcnnLayer.spk_node_id\': spiking layer should be node {spk_node_id}.' - assert layerhandler.pool_node_id == expected_output[dpcnnl_idx]['pool_node_id'], \ - f'wrong \'DynapcnnLayer.pool_node_id\': pooling layer node(s) should be {pool_node_id}.' - assert layerhandler.conv_rescaling_factor == expected_output[dpcnnl_idx]['conv_rescaling_factor'], \ - f'wrong \'DynapcnnLayer.conv_rescaling_factor\': computed re-scaling factor should be {conv_rescaling_factor}.' - assert layerhandler.dynapcnnlayer_destination == expected_output[dpcnnl_idx]['dynapcnnlayer_destination'], \ - f'wrong \'DynapcnnLayer.dynapcnnlayer_destination\': the DynapcnnLayer(s) set as destination(s) should be {dynapcnnlayer_destination}.' - assert layerhandler.nodes_destinations == expected_output[dpcnnl_idx]['nodes_destinations'], \ - f'wrong \'DynapcnnLayer.nodes_destinations\': the targeted nodes within other DynapcnnLayer instance(s) should be {nodes_destinations}.' - assert layerhandler.entry_point == expected_output[dpcnnl_idx]['entry_point'], \ - f'wrong \'DynapcnnLayer.entry_point\': its value should be {entry_point}.' + assert ( + layerhandler.dpcnnl_index == expected_output[dpcnnl_idx]["dpcnnl_index"] + ), f"wrong 'DynapcnnLayer.dpcnnl_index': ID of the instance should be {dpcnnl_index}." + assert ( + layerhandler.conv_node_id == expected_output[dpcnnl_idx]["conv_node_id"] + ), f"wrong 'DynapcnnLayer.conv_node_id': convolution layer should be node {conv_node_id}." + assert ( + layerhandler.conv_in_shape == expected_output[dpcnnl_idx]["conv_in_shape"] + ), f"wrong 'DynapcnnLayer.conv_in_shape': input tensor shape of convolution should be {conv_in_shape}." + assert ( + layerhandler.conv_out_shape == expected_output[dpcnnl_idx]["conv_out_shape"] + ), f"wrong 'DynapcnnLayer.conv_out_shape': output tensor shape of convolution should be {conv_out_shape}." + assert ( + layerhandler.spk_node_id == expected_output[dpcnnl_idx]["spk_node_id"] + ), f"wrong 'DynapcnnLayer.spk_node_id': spiking layer should be node {spk_node_id}." + assert ( + layerhandler.pool_node_id == expected_output[dpcnnl_idx]["pool_node_id"] + ), f"wrong 'DynapcnnLayer.pool_node_id': pooling layer node(s) should be {pool_node_id}." + assert ( + layerhandler.conv_rescaling_factor + == expected_output[dpcnnl_idx]["conv_rescaling_factor"] + ), f"wrong 'DynapcnnLayer.conv_rescaling_factor': computed re-scaling factor should be {conv_rescaling_factor}." + assert ( + layerhandler.dynapcnnlayer_destination + == expected_output[dpcnnl_idx]["dynapcnnlayer_destination"] + ), f"wrong 'DynapcnnLayer.dynapcnnlayer_destination': the DynapcnnLayer(s) set as destination(s) should be {dynapcnnlayer_destination}." + assert ( + layerhandler.nodes_destinations + == expected_output[dpcnnl_idx]["nodes_destinations"] + ), f"wrong 'DynapcnnLayer.nodes_destinations': the targeted nodes within other DynapcnnLayer instance(s) should be {nodes_destinations}." + assert ( + layerhandler.entry_point == expected_output[dpcnnl_idx]["entry_point"] + ), f"wrong 'DynapcnnLayer.entry_point': its value should be {entry_point}." diff --git a/tests/test_dynapcnnnetwork/conftest_dynapcnnnetwork.py b/tests/test_dynapcnnnetwork/conftest_dynapcnnnetwork.py index 92f528ce..8b7f29b8 100644 --- a/tests/test_dynapcnnnetwork/conftest_dynapcnnnetwork.py +++ b/tests/test_dynapcnnnetwork/conftest_dynapcnnnetwork.py @@ -1,14 +1,26 @@ # author : Willian Soares Girao # contact : wsoaresgirao@gmail.com -from model_dummy_1 import snn as snn_1, input_shape as input_shape_1, batch_size as batch_size_1, expected_output as expected_output_1 -from model_dummy_2 import snn as snn_2, input_shape as input_shape_2, batch_size as batch_size_2, expected_output as expected_output_2 -from model_dummy_3 import snn as snn_3, input_shape as input_shape_3, batch_size as batch_size_3, expected_output as expected_output_3 -from model_dummy_4 import snn as snn_4, input_shape as input_shape_4, batch_size as batch_size_4, expected_output as expected_output_4 +from model_dummy_1 import batch_size as batch_size_1 +from model_dummy_1 import expected_output as expected_output_1 +from model_dummy_1 import input_shape as input_shape_1 +from model_dummy_1 import snn as snn_1 +from model_dummy_2 import batch_size as batch_size_2 +from model_dummy_2 import expected_output as expected_output_2 +from model_dummy_2 import input_shape as input_shape_2 +from model_dummy_2 import snn as snn_2 +from model_dummy_3 import batch_size as batch_size_3 +from model_dummy_3 import expected_output as expected_output_3 +from model_dummy_3 import input_shape as input_shape_3 +from model_dummy_3 import snn as snn_3 +from model_dummy_4 import batch_size as batch_size_4 +from model_dummy_4 import expected_output as expected_output_4 +from model_dummy_4 import input_shape as input_shape_4 +from model_dummy_4 import snn as snn_4 args_DynapcnnNetworkTest = [ (snn_1, input_shape_1, batch_size_1, expected_output_1), (snn_2, input_shape_2, batch_size_2, expected_output_2), (snn_3, input_shape_3, batch_size_3, expected_output_3), (snn_4, input_shape_4, batch_size_4, expected_output_4), -] \ No newline at end of file +] diff --git a/tests/test_dynapcnnnetwork/model_dummy_1.py b/tests/test_dynapcnnnetwork/model_dummy_1.py index 348f2299..0c32e959 100644 --- a/tests/test_dynapcnnnetwork/model_dummy_1.py +++ b/tests/test_dynapcnnnetwork/model_dummy_1.py @@ -4,36 +4,63 @@ import torch import torch.nn as nn -from sinabs.layers import Merge, IAFSqueeze + from sinabs.activation.surrogate_gradient_fn import PeriodicExponential +from sinabs.layers import IAFSqueeze, Merge + class SNN(nn.Module): def __init__(self, batch_size) -> None: super().__init__() - self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False) # node 0 - self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) # node 1 - self.pool1 = nn.AvgPool2d(3,3) # node 2 - self.pool1a = nn.AvgPool2d(4,4) # node 3 + self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False) # node 0 + self.iaf1 = IAFSqueeze( + batch_size=batch_size, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ) # node 1 + self.pool1 = nn.AvgPool2d(3, 3) # node 2 + self.pool1a = nn.AvgPool2d(4, 4) # node 3 - self.conv2 = nn.Conv2d(10, 10, 4, 1, bias=False)# node 4 - self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) # node 6 + self.conv2 = nn.Conv2d(10, 10, 4, 1, bias=False) # node 4 + self.iaf2 = IAFSqueeze( + batch_size=batch_size, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ) # node 6 - self.conv3 = nn.Conv2d(10, 1, 2, 1, bias=False) # node 8 - self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) # node 9 + self.conv3 = nn.Conv2d(10, 1, 2, 1, bias=False) # node 8 + self.iaf3 = IAFSqueeze( + batch_size=batch_size, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ) # node 9 self.flat = nn.Flatten() - self.fc1 = nn.Linear(49, 500, bias=False) # node 10 - self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) # node 11 - - self.fc2 = nn.Linear(500, 10, bias=False) # node 12 - self.iaf5 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) # node 13 + self.fc1 = nn.Linear(49, 500, bias=False) # node 10 + self.iaf4 = IAFSqueeze( + batch_size=batch_size, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ) # node 11 + + self.fc2 = nn.Linear(500, 10, bias=False) # node 12 + self.iaf5 = IAFSqueeze( + batch_size=batch_size, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ) # node 13 self.adder = Merge() def forward(self, x): - + con1_out = self.conv1(x) iaf1_out = self.iaf1(con1_out) pool1_out = self.pool1(iaf1_out) @@ -46,14 +73,15 @@ def forward(self, x): iaf3_out = self.iaf3(conv3_out) flat_out = self.flat(iaf3_out) - + fc1_out = self.fc1(flat_out) iaf4_out = self.iaf4(fc1_out) fc2_out = self.fc2(iaf4_out) iaf5_out = self.iaf5(fc2_out) return iaf5_out - + + channels = 2 height = 34 width = 34 @@ -63,16 +91,16 @@ def forward(self, x): snn = SNN(batch_size) expected_output = { - 'dcnnl_edges': [ + "dcnnl_edges": [ (0, 1), (0, 2), (1, 2), (2, 3), (3, 4), - ('input', 0), + ("input", 0), ], - 'merge_points': {2: {'sources': (0, 1), 'merge': Merge()}}, - 'topological_order': [0, 1, 2, 3, 4], - 'output_shape': torch.Size([3, 10, 1, 1]), - 'entry_point': [0], -} \ No newline at end of file + "merge_points": {2: {"sources": (0, 1), "merge": Merge()}}, + "topological_order": [0, 1, 2, 3, 4], + "output_shape": torch.Size([3, 10, 1, 1]), + "entry_point": [0], +} diff --git a/tests/test_dynapcnnnetwork/model_dummy_2.py b/tests/test_dynapcnnnetwork/model_dummy_2.py index a83b54a2..bae7908f 100644 --- a/tests/test_dynapcnnnetwork/model_dummy_2.py +++ b/tests/test_dynapcnnnetwork/model_dummy_2.py @@ -4,36 +4,73 @@ import torch import torch.nn as nn -from sinabs.layers import Merge, IAFSqueeze, SumPool2d + from sinabs.activation.surrogate_gradient_fn import PeriodicExponential +from sinabs.layers import IAFSqueeze, Merge, SumPool2d + class SNN(nn.Module): def __init__(self, batch_size) -> None: super().__init__() # -- graph node A -- self.conv_A = nn.Conv2d(2, 4, 2, 1, bias=False) - self.iaf_A = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + self.iaf_A = IAFSqueeze( + batch_size=batch_size, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ) # -- graph node B -- self.conv_B = nn.Conv2d(4, 4, 2, 1, bias=False) - self.iaf2_B = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) - self.pool_B = SumPool2d(2,2) + self.iaf2_B = IAFSqueeze( + batch_size=batch_size, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ) + self.pool_B = SumPool2d(2, 2) # -- graph node C -- self.conv_C = nn.Conv2d(4, 4, 2, 1, bias=False) - self.iaf_C = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) - self.pool_C = SumPool2d(2,2) + self.iaf_C = IAFSqueeze( + batch_size=batch_size, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ) + self.pool_C = SumPool2d(2, 2) # -- graph node D -- self.conv_D = nn.Conv2d(4, 4, 2, 1, bias=False) - self.iaf_D = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + self.iaf_D = IAFSqueeze( + batch_size=batch_size, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ) # -- graph node E -- self.conv_E = nn.Conv2d(4, 4, 2, 1, bias=False) - self.iaf3_E = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) - self.pool_E = SumPool2d(2,2) + self.iaf3_E = IAFSqueeze( + batch_size=batch_size, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ) + self.pool_E = SumPool2d(2, 2) # -- graph node F -- self.conv_F = nn.Conv2d(4, 4, 2, 1, bias=False) - self.iaf_F = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + self.iaf_F = IAFSqueeze( + batch_size=batch_size, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ) # -- graph node G -- self.fc3 = nn.Linear(144, 10, bias=False) - self.iaf3_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + self.iaf3_fc = IAFSqueeze( + batch_size=batch_size, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ) # -- merges -- self.merge1 = Merge() @@ -44,42 +81,43 @@ def __init__(self, batch_size) -> None: def forward(self, x): # conv 1 - A/0 - convA_out = self.conv_A(x) # node 0 - iaf_A_out = self.iaf_A(convA_out) # node 1 + convA_out = self.conv_A(x) # node 0 + iaf_A_out = self.iaf_A(convA_out) # node 1 # conv 2 - B/1 - conv_B_out = self.conv_B(iaf_A_out) # node 2 - iaf_B_out = self.iaf2_B(conv_B_out) # node 3 - pool_B_out = self.pool_B(iaf_B_out) # node 4 + conv_B_out = self.conv_B(iaf_A_out) # node 2 + iaf_B_out = self.iaf2_B(conv_B_out) # node 3 + pool_B_out = self.pool_B(iaf_B_out) # node 4 # conv 3 - C/2 - conv_C_out = self.conv_C(pool_B_out) # node 5 - iaf_C_out = self.iaf_C(conv_C_out) # node 7 - pool_C_out = self.pool_C(iaf_C_out) # node 8 + conv_C_out = self.conv_C(pool_B_out) # node 5 + iaf_C_out = self.iaf_C(conv_C_out) # node 7 + pool_C_out = self.pool_C(iaf_C_out) # node 8 # conv 4 - D/4 - conv_D_out = self.conv_D(pool_C_out) # node 9 - iaf_D_out = self.iaf_D(conv_D_out) # node 10 - + conv_D_out = self.conv_D(pool_C_out) # node 9 + iaf_D_out = self.iaf_D(conv_D_out) # node 10 + # fc 1 - E/3 - conv_E_out = self.conv_E(pool_B_out) # node 6 - iaf3_E_out = self.iaf3_E(conv_E_out) # node 12 - pool_E_out = self.pool_E(iaf3_E_out) # node 13 + conv_E_out = self.conv_E(pool_B_out) # node 6 + iaf3_E_out = self.iaf3_E(conv_E_out) # node 12 + pool_E_out = self.pool_E(iaf3_E_out) # node 13 # fc 2 - F/6 - conv_F_out = self.conv_F(pool_E_out) # node 14 - iaf_F_out = self.iaf_F(conv_F_out) # node 15 - + conv_F_out = self.conv_F(pool_E_out) # node 14 + iaf_F_out = self.iaf_F(conv_F_out) # node 15 + # fc 2 - G/5 - flat_D_out = self.flat_D(iaf_D_out) # node 11 - flat_F_out = self.flat_F(iaf_F_out) # node 16 - - merge1_out = self.merge1(flat_D_out, flat_F_out) # node 19 - fc3_out = self.fc3(merge1_out) # node 17 - iaf3_fc_out = self.iaf3_fc(fc3_out) # node 18 + flat_D_out = self.flat_D(iaf_D_out) # node 11 + flat_F_out = self.flat_F(iaf_F_out) # node 16 + + merge1_out = self.merge1(flat_D_out, flat_F_out) # node 19 + fc3_out = self.fc3(merge1_out) # node 17 + iaf3_fc_out = self.iaf3_fc(fc3_out) # node 18 return iaf3_fc_out - + + channels = 2 height = 34 width = 34 @@ -89,7 +127,7 @@ def forward(self, x): snn = SNN(batch_size) expected_output = { - 'dcnnl_edges': [ + "dcnnl_edges": [ (0, 1), (1, 2), (1, 3), @@ -97,10 +135,10 @@ def forward(self, x): (3, 6), (4, 5), (6, 5), - ('input', 0), + ("input", 0), ], - 'merge_points': {5: {'sources': (4, 6), 'merge': Merge()}}, - 'topological_order': [0, 1, 2, 3, 4, 6, 5], - 'output_shape': torch.Size([8, 10, 1, 1]), - 'entry_point': [0], -} \ No newline at end of file + "merge_points": {5: {"sources": (4, 6), "merge": Merge()}}, + "topological_order": [0, 1, 2, 3, 4, 6, 5], + "output_shape": torch.Size([8, 10, 1, 1]), + "entry_point": [0], +} diff --git a/tests/test_dynapcnnnetwork/model_dummy_3.py b/tests/test_dynapcnnnetwork/model_dummy_3.py index a5eb1ca1..a67156bc 100644 --- a/tests/test_dynapcnnnetwork/model_dummy_3.py +++ b/tests/test_dynapcnnnetwork/model_dummy_3.py @@ -1,51 +1,97 @@ - # author : Willian Soares Girao # contact : wsoaresgirao@gmail.com # implementing "two networks with merging outputs" in https://github.com/synsense/sinabs/issues/181 import torch import torch.nn as nn -from sinabs.layers import IAFSqueeze, SumPool2d, Merge + from sinabs.activation.surrogate_gradient_fn import PeriodicExponential +from sinabs.layers import IAFSqueeze, Merge, SumPool2d + class SNN(nn.Module): def __init__(self, batch_size) -> None: super().__init__() self.conv_A = nn.Conv2d(2, 4, 2, 1, bias=False) - self.iaf_A = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + self.iaf_A = IAFSqueeze( + batch_size=batch_size, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ) self.conv_B = nn.Conv2d(4, 4, 2, 1, bias=False) - self.iaf_B = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) - self.pool_B = SumPool2d(2,2) + self.iaf_B = IAFSqueeze( + batch_size=batch_size, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ) + self.pool_B = SumPool2d(2, 2) self.conv_C = nn.Conv2d(4, 4, 2, 1, bias=False) - self.iaf_C = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) - self.pool_C = SumPool2d(2,2) + self.iaf_C = IAFSqueeze( + batch_size=batch_size, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ) + self.pool_C = SumPool2d(2, 2) self.conv_D = nn.Conv2d(2, 4, 2, 1, bias=False) - self.iaf_D = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + self.iaf_D = IAFSqueeze( + batch_size=batch_size, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ) self.conv_E = nn.Conv2d(4, 4, 2, 1, bias=False) - self.iaf_E = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) - self.pool_E = SumPool2d(2,2) + self.iaf_E = IAFSqueeze( + batch_size=batch_size, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ) + self.pool_E = SumPool2d(2, 2) self.conv_F = nn.Conv2d(4, 4, 2, 1, bias=False) - self.iaf_F = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) - self.pool_F = SumPool2d(2,2) + self.iaf_F = IAFSqueeze( + batch_size=batch_size, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ) + self.pool_F = SumPool2d(2, 2) self.flat_brach1 = nn.Flatten() self.flat_brach2 = nn.Flatten() self.merge = Merge() self.fc1 = nn.Linear(196, 100, bias=False) - self.iaf1_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + self.iaf1_fc = IAFSqueeze( + batch_size=batch_size, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ) self.fc2 = nn.Linear(100, 100, bias=False) - self.iaf2_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + self.iaf2_fc = IAFSqueeze( + batch_size=batch_size, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ) self.fc3 = nn.Linear(100, 10, bias=False) - self.iaf3_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + self.iaf3_fc = IAFSqueeze( + batch_size=batch_size, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ) def forward(self, x): # conv 1 - A @@ -91,7 +137,8 @@ def forward(self, x): iaf3_fc_out = self.iaf3_fc(fc3_out) return iaf3_fc_out - + + channels = 2 height = 34 width = 34 @@ -101,7 +148,7 @@ def forward(self, x): snn = SNN(batch_size) expected_output = { - 'dcnnl_edges': [ + "dcnnl_edges": [ (0, 1), (1, 2), (2, 3), @@ -110,11 +157,11 @@ def forward(self, x): (5, 6), (6, 3), (7, 8), - ('input', 0), - ('input', 4), + ("input", 0), + ("input", 4), ], - 'merge_points': {3: {'sources': (2, 6), 'merge': Merge()}}, - 'topological_order': [0, 4, 1, 5, 2, 6, 3, 7, 8], - 'output_shape': torch.Size([2, 10, 1, 1]), - 'entry_point': [0, 4], -} \ No newline at end of file + "merge_points": {3: {"sources": (2, 6), "merge": Merge()}}, + "topological_order": [0, 4, 1, 5, 2, 6, 3, 7, 8], + "output_shape": torch.Size([2, 10, 1, 1]), + "entry_point": [0, 4], +} diff --git a/tests/test_dynapcnnnetwork/model_dummy_4.py b/tests/test_dynapcnnnetwork/model_dummy_4.py index 845301db..b74a63cf 100644 --- a/tests/test_dynapcnnnetwork/model_dummy_4.py +++ b/tests/test_dynapcnnnetwork/model_dummy_4.py @@ -4,37 +4,69 @@ import torch import torch.nn as nn -from sinabs.layers import IAFSqueeze, SumPool2d, Merge + from sinabs.activation.surrogate_gradient_fn import PeriodicExponential +from sinabs.layers import IAFSqueeze, Merge, SumPool2d + class SNN(nn.Module): def __init__(self, batch_size) -> None: super().__init__() self.conv1 = nn.Conv2d(2, 1, 2, 1, bias=False) - self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + self.iaf1 = IAFSqueeze( + batch_size=batch_size, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ) self.conv2 = nn.Conv2d(1, 1, 2, 1, bias=False) - self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) - self.pool2 = SumPool2d(2,2) + self.iaf2 = IAFSqueeze( + batch_size=batch_size, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ) + self.pool2 = SumPool2d(2, 2) self.conv3 = nn.Conv2d(1, 1, 2, 1, bias=False) - self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) - self.pool3 = SumPool2d(2,2) - self.pool3a = SumPool2d(5,5) + self.iaf3 = IAFSqueeze( + batch_size=batch_size, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ) + self.pool3 = SumPool2d(2, 2) + self.pool3a = SumPool2d(5, 5) self.conv4 = nn.Conv2d(1, 1, 2, 1, bias=False) - self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) - self.pool4 = SumPool2d(3,3) + self.iaf4 = IAFSqueeze( + batch_size=batch_size, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ) + self.pool4 = SumPool2d(3, 3) self.flat1 = nn.Flatten() self.flat2 = nn.Flatten() self.conv5 = nn.Conv2d(1, 1, 2, 1, bias=False) - self.iaf5 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + self.iaf5 = IAFSqueeze( + batch_size=batch_size, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ) self.fc2 = nn.Linear(25, 10, bias=False) - self.iaf2_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + self.iaf2_fc = IAFSqueeze( + batch_size=batch_size, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ) # -- merges -- self.merge1 = Merge() @@ -62,7 +94,7 @@ def forward(self, x): iaf4_out = self.iaf4(conv4_out) pool4_out = self.pool4(iaf4_out) flat1_out = self.flat1(pool4_out) - + # conv 5 - E/4 conv5_out = self.conv5(pool3a_out) iaf5_out = self.iaf5(conv5_out) @@ -70,12 +102,13 @@ def forward(self, x): # fc 2 - F/5 merge2_out = self.merge2(flat2_out, flat1_out) - + fc2_out = self.fc2(merge2_out) iaf2_fc_out = self.iaf2_fc(fc2_out) return iaf2_fc_out - + + channels = 2 height = 34 width = 34 @@ -85,7 +118,7 @@ def forward(self, x): snn = SNN(batch_size) expected_output = { - 'dcnnl_edges': [ + "dcnnl_edges": [ (0, 1), (0, 2), (1, 3), @@ -93,13 +126,13 @@ def forward(self, x): (2, 4), (3, 5), (4, 5), - ('input', 0), + ("input", 0), ], - 'merge_points': { - 3: {'sources': (1, 2), 'merge': Merge()}, - 5: {'sources': (3, 4), 'merge': Merge()}, + "merge_points": { + 3: {"sources": (1, 2), "merge": Merge()}, + 5: {"sources": (3, 4), "merge": Merge()}, }, - 'topological_order': [0, 1, 2, 3, 4, 5], - 'output_shape': torch.Size([2, 10, 1, 1]), - 'entry_point': [0], -} \ No newline at end of file + "topological_order": [0, 1, 2, 3, 4, 5], + "output_shape": torch.Size([2, 10, 1, 1]), + "entry_point": [0], +} diff --git a/tests/test_dynapcnnnetwork/test_dynapcnnnetwork.py b/tests/test_dynapcnnnetwork/test_dynapcnnnetwork.py index 8f7471b7..a86d78fe 100644 --- a/tests/test_dynapcnnnetwork/test_dynapcnnnetwork.py +++ b/tests/test_dynapcnnnetwork/test_dynapcnnnetwork.py @@ -1,16 +1,22 @@ # author : Willian Soares Girao # contact : wsoaresgirao@gmail.com -import pytest, torch -from sinabs.backend.dynapcnn.dynapcnn_network import DynapcnnNetwork +import pytest +import torch from conftest_dynapcnnnetwork import args_DynapcnnNetworkTest -@pytest.mark.parametrize("snn, input_shape, batch_size, expected_output", args_DynapcnnNetworkTest) +from sinabs.backend.dynapcnn.dynapcnn_network import DynapcnnNetwork + + +@pytest.mark.parametrize( + "snn, input_shape, batch_size, expected_output", args_DynapcnnNetworkTest +) def test_DynapcnnNetwork(snn, input_shape, batch_size, expected_output): - """ Tests if the correct graph representing the connections between each DynapcnnLayer within a DynapcnnNetwork + """Tests if the correct graph representing the connections between each DynapcnnLayer within a DynapcnnNetwork is created; if the DynapcnnLayer instances requiring input from a `Merge` are correctly flagged (along with what their arguments should be); if the correct topological order of the DynapcnnLayers (i.e., the order in which their - forward methods should be called) is computed; if the output of the model matches what is expected.""" + forward methods should be called) is computed; if the output of the model matches what is expected. + """ dcnnnet = DynapcnnNetwork(snn, input_shape, batch_size) @@ -18,21 +24,27 @@ def test_DynapcnnNetwork(snn, input_shape, batch_size, expected_output): x = torch.randn((batch_size, *input_shape)) output = dcnnnet(x) - assert expected_output['dcnnl_edges'] == dcnnnet.dcnnl_edges, \ - f'wrong list of edges describing DynapcnnLayer connectivity.' - + assert ( + expected_output["dcnnl_edges"] == dcnnnet.dcnnl_edges + ), f"wrong list of edges describing DynapcnnLayer connectivity." + for node, args in dcnnnet.merge_points.items(): - assert node in expected_output['merge_points'], \ - f'DynapcnnLayer {node} is not a merge point.' - assert args['sources'] == expected_output['merge_points'][node]['sources'], \ - f'DynapcnnLayer {node} has wrong input sources ({args}).' - - for entry_point in expected_output['entry_point']: - assert dcnnnet.layers_mapper[entry_point].entry_point, \ - f'DynapcnnLayer {entry_point} should be an entry point.' - - assert expected_output['topological_order'] == dcnnnet.topological_order, \ - f'wrong topological ordering between DynapcnnLayers.' - assert expected_output['output_shape'] == output.shape, \ - f'wrong model output tensor shape.' \ No newline at end of file + assert ( + node in expected_output["merge_points"] + ), f"DynapcnnLayer {node} is not a merge point." + assert ( + args["sources"] == expected_output["merge_points"][node]["sources"] + ), f"DynapcnnLayer {node} has wrong input sources ({args})." + + for entry_point in expected_output["entry_point"]: + assert dcnnnet.layers_mapper[ + entry_point + ].entry_point, f"DynapcnnLayer {entry_point} should be an entry point." + + assert ( + expected_output["topological_order"] == dcnnnet.topological_order + ), f"wrong topological ordering between DynapcnnLayers." + assert ( + expected_output["output_shape"] == output.shape + ), f"wrong model output tensor shape." diff --git a/tests/test_graph_extractor/conftest_graph_extractor.py b/tests/test_graph_extractor/conftest_graph_extractor.py index 7c655c66..1192e62a 100644 --- a/tests/test_graph_extractor/conftest_graph_extractor.py +++ b/tests/test_graph_extractor/conftest_graph_extractor.py @@ -1,10 +1,18 @@ # author : Willian Soares Girao # contact : wsoaresgirao@gmail.com -from model_dummy_1 import snn as snn_1, input_dummy as input_dummy_1, expected_output as expected_output_1 -from model_dummy_2 import snn as snn_2, input_dummy as input_dummy_2, expected_output as expected_output_2 -from model_dummy_3 import snn as snn_3, input_dummy as input_dummy_3, expected_output as expected_output_3 -from model_dummy_4 import snn as snn_4, input_dummy as input_dummy_4, expected_output as expected_output_4 +from model_dummy_1 import expected_output as expected_output_1 +from model_dummy_1 import input_dummy as input_dummy_1 +from model_dummy_1 import snn as snn_1 +from model_dummy_2 import expected_output as expected_output_2 +from model_dummy_2 import input_dummy as input_dummy_2 +from model_dummy_2 import snn as snn_2 +from model_dummy_3 import expected_output as expected_output_3 +from model_dummy_3 import input_dummy as input_dummy_3 +from model_dummy_3 import snn as snn_3 +from model_dummy_4 import expected_output as expected_output_4 +from model_dummy_4 import input_dummy as input_dummy_4 +from model_dummy_4 import snn as snn_4 args_GraphExtractor = [ (snn_1, input_dummy_1, expected_output_1), diff --git a/tests/test_graph_extractor/model_dummy_1.py b/tests/test_graph_extractor/model_dummy_1.py index cf2b9e7d..87215fcf 100644 --- a/tests/test_graph_extractor/model_dummy_1.py +++ b/tests/test_graph_extractor/model_dummy_1.py @@ -4,36 +4,63 @@ import torch import torch.nn as nn -from sinabs.layers import Merge, IAFSqueeze + from sinabs.activation.surrogate_gradient_fn import PeriodicExponential +from sinabs.layers import IAFSqueeze, Merge + class SNN(nn.Module): def __init__(self, batch_size) -> None: super().__init__() - self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False) # node 0 - self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) # node 1 - self.pool1 = nn.AvgPool2d(3,3) # node 2 - self.pool1a = nn.AvgPool2d(4,4) # node 3 - - self.conv2 = nn.Conv2d(10, 10, 4, 1, bias=False)# node 4 - self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) # node 6 - - self.conv3 = nn.Conv2d(10, 1, 2, 1, bias=False) # node 8 - self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) # node 9 + self.conv1 = nn.Conv2d(2, 10, 2, 1, bias=False) # node 0 + self.iaf1 = IAFSqueeze( + batch_size=batch_size, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ) # node 1 + self.pool1 = nn.AvgPool2d(3, 3) # node 2 + self.pool1a = nn.AvgPool2d(4, 4) # node 3 + + self.conv2 = nn.Conv2d(10, 10, 4, 1, bias=False) # node 4 + self.iaf2 = IAFSqueeze( + batch_size=batch_size, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ) # node 6 + + self.conv3 = nn.Conv2d(10, 1, 2, 1, bias=False) # node 8 + self.iaf3 = IAFSqueeze( + batch_size=batch_size, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ) # node 9 self.flat = nn.Flatten() - self.fc1 = nn.Linear(49, 500, bias=False) # node 10 - self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) # node 11 - - self.fc2 = nn.Linear(500, 10, bias=False) # node 12 - self.iaf5 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) # node 13 + self.fc1 = nn.Linear(49, 500, bias=False) # node 10 + self.iaf4 = IAFSqueeze( + batch_size=batch_size, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ) # node 11 + + self.fc2 = nn.Linear(500, 10, bias=False) # node 12 + self.iaf5 = IAFSqueeze( + batch_size=batch_size, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ) # node 13 self.adder = Merge() def forward(self, x): - + con1_out = self.conv1(x) iaf1_out = self.iaf1(con1_out) pool1_out = self.pool1(iaf1_out) @@ -46,14 +73,15 @@ def forward(self, x): iaf3_out = self.iaf3(conv3_out) flat_out = self.flat(iaf3_out) - + fc1_out = self.fc1(flat_out) iaf4_out = self.iaf4(fc1_out) fc2_out = self.fc2(iaf4_out) iaf5_out = self.iaf5(fc2_out) return iaf5_out - + + channels = 2 height = 34 width = 34 @@ -64,7 +92,7 @@ def forward(self, x): input_dummy = torch.randn(input_shape) expected_output = { - 'edges': { + "edges": { (0, 1), (1, 2), (1, 3), @@ -80,38 +108,44 @@ def forward(self, x): (12, 13), (5, 7), }, - 'name_2_indx_map': { - 'conv1': 0, - 'iaf1': 1, - 'pool1': 2, - 'pool1a': 3, - 'conv2': 4, - 'adder': 5, - 'iaf2': 6, - 'conv3': 7, - 'iaf3': 8, - 'flat': 9, - 'fc1': 10, - 'iaf4': 11, - 'fc2': 12, - 'iaf5': 13, + "name_2_indx_map": { + "conv1": 0, + "iaf1": 1, + "pool1": 2, + "pool1a": 3, + "conv2": 4, + "adder": 5, + "iaf2": 6, + "conv3": 7, + "iaf3": 8, + "flat": 9, + "fc1": 10, + "iaf4": 11, + "fc2": 12, + "iaf5": 13, }, - 'entry_nodes': {0}, - 'nodes_io_shapes': { - 0: {'input': torch.Size([3, 2, 34, 34]), 'output': torch.Size([3, 10, 33, 33])}, - 1: {'input': torch.Size([3, 10, 33, 33]), 'output': torch.Size([3, 10, 33, 33])}, - 2: {'input': torch.Size([3, 10, 33, 33]), 'output': torch.Size([3, 10, 11, 11])}, - 3: {'input': torch.Size([3, 10, 33, 33]), 'output': torch.Size([3, 10, 8, 8])}, - 4: {'input': torch.Size([3, 10, 11, 11]), 'output': torch.Size([3, 10, 8, 8])}, - 6: {'input': torch.Size([3, 10, 8, 8]), 'output': torch.Size([3, 10, 8, 8])}, - 5: {'input': torch.Size([3, 10, 8, 8]), 'output': torch.Size([3, 10, 8, 8])}, - 7: {'input': torch.Size([3, 10, 8, 8]), 'output': torch.Size([3, 1, 7, 7])}, - 8: {'input': torch.Size([3, 1, 7, 7]), 'output': torch.Size([3, 1, 7, 7])}, - 9: {'input': torch.Size([3, 1, 7, 7]), 'output': torch.Size([3, 49])}, - 10: {'input': torch.Size([3, 49]), 'output': torch.Size([3, 500])}, - 11: {'input': torch.Size([3, 500]), 'output': torch.Size([3, 500])}, - 12: {'input': torch.Size([3, 500]), 'output': torch.Size([3, 10])}, - 13: {'input': torch.Size([3, 10]), 'output': torch.Size([3, 10])}, + "entry_nodes": {0}, + "nodes_io_shapes": { + 0: {"input": torch.Size([3, 2, 34, 34]), "output": torch.Size([3, 10, 33, 33])}, + 1: { + "input": torch.Size([3, 10, 33, 33]), + "output": torch.Size([3, 10, 33, 33]), + }, + 2: { + "input": torch.Size([3, 10, 33, 33]), + "output": torch.Size([3, 10, 11, 11]), + }, + 3: {"input": torch.Size([3, 10, 33, 33]), "output": torch.Size([3, 10, 8, 8])}, + 4: {"input": torch.Size([3, 10, 11, 11]), "output": torch.Size([3, 10, 8, 8])}, + 6: {"input": torch.Size([3, 10, 8, 8]), "output": torch.Size([3, 10, 8, 8])}, + 5: {"input": torch.Size([3, 10, 8, 8]), "output": torch.Size([3, 10, 8, 8])}, + 7: {"input": torch.Size([3, 10, 8, 8]), "output": torch.Size([3, 1, 7, 7])}, + 8: {"input": torch.Size([3, 1, 7, 7]), "output": torch.Size([3, 1, 7, 7])}, + 9: {"input": torch.Size([3, 1, 7, 7]), "output": torch.Size([3, 49])}, + 10: {"input": torch.Size([3, 49]), "output": torch.Size([3, 500])}, + 11: {"input": torch.Size([3, 500]), "output": torch.Size([3, 500])}, + 12: {"input": torch.Size([3, 500]), "output": torch.Size([3, 10])}, + 13: {"input": torch.Size([3, 10]), "output": torch.Size([3, 10])}, }, } diff --git a/tests/test_graph_extractor/model_dummy_2.py b/tests/test_graph_extractor/model_dummy_2.py index 52281b34..b56a975e 100644 --- a/tests/test_graph_extractor/model_dummy_2.py +++ b/tests/test_graph_extractor/model_dummy_2.py @@ -4,36 +4,73 @@ import torch import torch.nn as nn -from sinabs.layers import Merge, IAFSqueeze, SumPool2d + from sinabs.activation.surrogate_gradient_fn import PeriodicExponential +from sinabs.layers import IAFSqueeze, Merge, SumPool2d + class SNN(nn.Module): def __init__(self, batch_size) -> None: super().__init__() # -- graph node A -- self.conv_A = nn.Conv2d(2, 4, 2, 1, bias=False) - self.iaf_A = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + self.iaf_A = IAFSqueeze( + batch_size=batch_size, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ) # -- graph node B -- self.conv_B = nn.Conv2d(4, 4, 2, 1, bias=False) - self.iaf2_B = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) - self.pool_B = SumPool2d(2,2) + self.iaf2_B = IAFSqueeze( + batch_size=batch_size, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ) + self.pool_B = SumPool2d(2, 2) # -- graph node C -- self.conv_C = nn.Conv2d(4, 4, 2, 1, bias=False) - self.iaf_C = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) - self.pool_C = SumPool2d(2,2) + self.iaf_C = IAFSqueeze( + batch_size=batch_size, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ) + self.pool_C = SumPool2d(2, 2) # -- graph node D -- self.conv_D = nn.Conv2d(4, 4, 2, 1, bias=False) - self.iaf_D = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + self.iaf_D = IAFSqueeze( + batch_size=batch_size, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ) # -- graph node E -- self.conv_E = nn.Conv2d(4, 4, 2, 1, bias=False) - self.iaf3_E = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) - self.pool_E = SumPool2d(2,2) + self.iaf3_E = IAFSqueeze( + batch_size=batch_size, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ) + self.pool_E = SumPool2d(2, 2) # -- graph node F -- self.conv_F = nn.Conv2d(4, 4, 2, 1, bias=False) - self.iaf_F = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + self.iaf_F = IAFSqueeze( + batch_size=batch_size, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ) # -- graph node G -- self.fc3 = nn.Linear(144, 10, bias=False) - self.iaf3_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + self.iaf3_fc = IAFSqueeze( + batch_size=batch_size, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ) # -- merges -- self.merge1 = Merge() @@ -44,42 +81,43 @@ def __init__(self, batch_size) -> None: def forward(self, x): # conv 1 - A/0 - convA_out = self.conv_A(x) # node 0 - iaf_A_out = self.iaf_A(convA_out) # node 1 + convA_out = self.conv_A(x) # node 0 + iaf_A_out = self.iaf_A(convA_out) # node 1 # conv 2 - B/1 - conv_B_out = self.conv_B(iaf_A_out) # node 2 - iaf_B_out = self.iaf2_B(conv_B_out) # node 3 - pool_B_out = self.pool_B(iaf_B_out) # node 4 + conv_B_out = self.conv_B(iaf_A_out) # node 2 + iaf_B_out = self.iaf2_B(conv_B_out) # node 3 + pool_B_out = self.pool_B(iaf_B_out) # node 4 # conv 3 - C/2 - conv_C_out = self.conv_C(pool_B_out) # node 5 - iaf_C_out = self.iaf_C(conv_C_out) # node 7 - pool_C_out = self.pool_C(iaf_C_out) # node 8 + conv_C_out = self.conv_C(pool_B_out) # node 5 + iaf_C_out = self.iaf_C(conv_C_out) # node 7 + pool_C_out = self.pool_C(iaf_C_out) # node 8 # conv 4 - D/4 - conv_D_out = self.conv_D(pool_C_out) # node 9 - iaf_D_out = self.iaf_D(conv_D_out) # node 10 - + conv_D_out = self.conv_D(pool_C_out) # node 9 + iaf_D_out = self.iaf_D(conv_D_out) # node 10 + # fc 1 - E/3 - conv_E_out = self.conv_E(pool_B_out) # node 6 - iaf3_E_out = self.iaf3_E(conv_E_out) # node 12 - pool_E_out = self.pool_E(iaf3_E_out) # node 13 + conv_E_out = self.conv_E(pool_B_out) # node 6 + iaf3_E_out = self.iaf3_E(conv_E_out) # node 12 + pool_E_out = self.pool_E(iaf3_E_out) # node 13 # fc 2 - F/6 - conv_F_out = self.conv_F(pool_E_out) # node 14 - iaf_F_out = self.iaf_F(conv_F_out) # node 15 - + conv_F_out = self.conv_F(pool_E_out) # node 14 + iaf_F_out = self.iaf_F(conv_F_out) # node 15 + # fc 2 - G/5 - flat_D_out = self.flat_D(iaf_D_out) # node 11 - flat_F_out = self.flat_F(iaf_F_out) # node 16 - - merge1_out = self.merge1(flat_D_out, flat_F_out) # node 19 - fc3_out = self.fc3(merge1_out) # node 17 - iaf3_fc_out = self.iaf3_fc(fc3_out) # node 18 + flat_D_out = self.flat_D(iaf_D_out) # node 11 + flat_F_out = self.flat_F(iaf_F_out) # node 16 + + merge1_out = self.merge1(flat_D_out, flat_F_out) # node 19 + fc3_out = self.fc3(merge1_out) # node 17 + iaf3_fc_out = self.iaf3_fc(fc3_out) # node 18 return iaf3_fc_out - + + channels = 2 height = 34 width = 34 @@ -90,7 +128,7 @@ def forward(self, x): input_dummy = torch.randn(input_shape) expected_output = { - 'edges': { + "edges": { (0, 1), (1, 2), (2, 3), @@ -112,50 +150,50 @@ def forward(self, x): (11, 19), (16, 19), }, - 'name_2_indx_map': { - 'conv_A': 0, - 'iaf_A': 1, - 'conv_B': 2, - 'iaf2_B': 3, - 'pool_B': 4, - 'conv_C': 5, - 'conv_E': 6, - 'iaf_C': 7, - 'pool_C': 8, - 'conv_D': 9, - 'iaf_D': 10, - 'flat_D': 11, - 'iaf3_E': 12, - 'pool_E': 13, - 'conv_F': 14, - 'iaf_F': 15, - 'flat_F': 16, - 'fc3': 17, - 'iaf3_fc': 18, - 'merge1': 19, + "name_2_indx_map": { + "conv_A": 0, + "iaf_A": 1, + "conv_B": 2, + "iaf2_B": 3, + "pool_B": 4, + "conv_C": 5, + "conv_E": 6, + "iaf_C": 7, + "pool_C": 8, + "conv_D": 9, + "iaf_D": 10, + "flat_D": 11, + "iaf3_E": 12, + "pool_E": 13, + "conv_F": 14, + "iaf_F": 15, + "flat_F": 16, + "fc3": 17, + "iaf3_fc": 18, + "merge1": 19, }, - 'entry_nodes': {0}, - 'nodes_io_shapes': { - 0: {'input': torch.Size([8, 2, 34, 34]), 'output': torch.Size([8, 4, 33, 33])}, - 1: {'input': torch.Size([8, 4, 33, 33]), 'output': torch.Size([8, 4, 33, 33])}, - 2: {'input': torch.Size([8, 4, 33, 33]), 'output': torch.Size([8, 4, 32, 32])}, - 3: {'input': torch.Size([8, 4, 32, 32]), 'output': torch.Size([8, 4, 32, 32])}, - 4: {'input': torch.Size([8, 4, 32, 32]), 'output': torch.Size([8, 4, 16, 16])}, - 5: {'input': torch.Size([8, 4, 16, 16]), 'output': torch.Size([8, 4, 15, 15])}, - 6: {'input': torch.Size([8, 4, 16, 16]), 'output': torch.Size([8, 4, 15, 15])}, - 7: {'input': torch.Size([8, 4, 15, 15]), 'output': torch.Size([8, 4, 15, 15])}, - 12: {'input': torch.Size([8, 4, 15, 15]), 'output': torch.Size([8, 4, 15, 15])}, - 8: {'input': torch.Size([8, 4, 15, 15]), 'output': torch.Size([8, 4, 7, 7])}, - 13: {'input': torch.Size([8, 4, 15, 15]), 'output': torch.Size([8, 4, 7, 7])}, - 9: {'input': torch.Size([8, 4, 7, 7]), 'output': torch.Size([8, 4, 6, 6])}, - 14: {'input': torch.Size([8, 4, 7, 7]), 'output': torch.Size([8, 4, 6, 6])}, - 10: {'input': torch.Size([8, 4, 6, 6]), 'output': torch.Size([8, 4, 6, 6])}, - 15: {'input': torch.Size([8, 4, 6, 6]), 'output': torch.Size([8, 4, 6, 6])}, - 11: {'input': torch.Size([8, 4, 6, 6]), 'output': torch.Size([8, 144])}, - 16: {'input': torch.Size([8, 4, 6, 6]), 'output': torch.Size([8, 144])}, - 19: {'input': torch.Size([8, 144]), 'output': torch.Size([8, 144])}, - 17: {'input': torch.Size([8, 144]), 'output': torch.Size([8, 10])}, - 18: {'input': torch.Size([8, 10]), 'output': torch.Size([8, 10])}, + "entry_nodes": {0}, + "nodes_io_shapes": { + 0: {"input": torch.Size([8, 2, 34, 34]), "output": torch.Size([8, 4, 33, 33])}, + 1: {"input": torch.Size([8, 4, 33, 33]), "output": torch.Size([8, 4, 33, 33])}, + 2: {"input": torch.Size([8, 4, 33, 33]), "output": torch.Size([8, 4, 32, 32])}, + 3: {"input": torch.Size([8, 4, 32, 32]), "output": torch.Size([8, 4, 32, 32])}, + 4: {"input": torch.Size([8, 4, 32, 32]), "output": torch.Size([8, 4, 16, 16])}, + 5: {"input": torch.Size([8, 4, 16, 16]), "output": torch.Size([8, 4, 15, 15])}, + 6: {"input": torch.Size([8, 4, 16, 16]), "output": torch.Size([8, 4, 15, 15])}, + 7: {"input": torch.Size([8, 4, 15, 15]), "output": torch.Size([8, 4, 15, 15])}, + 12: {"input": torch.Size([8, 4, 15, 15]), "output": torch.Size([8, 4, 15, 15])}, + 8: {"input": torch.Size([8, 4, 15, 15]), "output": torch.Size([8, 4, 7, 7])}, + 13: {"input": torch.Size([8, 4, 15, 15]), "output": torch.Size([8, 4, 7, 7])}, + 9: {"input": torch.Size([8, 4, 7, 7]), "output": torch.Size([8, 4, 6, 6])}, + 14: {"input": torch.Size([8, 4, 7, 7]), "output": torch.Size([8, 4, 6, 6])}, + 10: {"input": torch.Size([8, 4, 6, 6]), "output": torch.Size([8, 4, 6, 6])}, + 15: {"input": torch.Size([8, 4, 6, 6]), "output": torch.Size([8, 4, 6, 6])}, + 11: {"input": torch.Size([8, 4, 6, 6]), "output": torch.Size([8, 144])}, + 16: {"input": torch.Size([8, 4, 6, 6]), "output": torch.Size([8, 144])}, + 19: {"input": torch.Size([8, 144]), "output": torch.Size([8, 144])}, + 17: {"input": torch.Size([8, 144]), "output": torch.Size([8, 10])}, + 18: {"input": torch.Size([8, 10]), "output": torch.Size([8, 10])}, }, } diff --git a/tests/test_graph_extractor/model_dummy_3.py b/tests/test_graph_extractor/model_dummy_3.py index 6fc8242d..326ec61d 100644 --- a/tests/test_graph_extractor/model_dummy_3.py +++ b/tests/test_graph_extractor/model_dummy_3.py @@ -1,51 +1,97 @@ - # author : Willian Soares Girao # contact : wsoaresgirao@gmail.com # implementing "two networks with merging outputs" in https://github.com/synsense/sinabs/issues/181 import torch import torch.nn as nn -from sinabs.layers import IAFSqueeze, SumPool2d, Merge + from sinabs.activation.surrogate_gradient_fn import PeriodicExponential +from sinabs.layers import IAFSqueeze, Merge, SumPool2d + class SNN(nn.Module): def __init__(self, batch_size) -> None: super().__init__() self.conv_A = nn.Conv2d(2, 4, 2, 1, bias=False) - self.iaf_A = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + self.iaf_A = IAFSqueeze( + batch_size=batch_size, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ) self.conv_B = nn.Conv2d(4, 4, 2, 1, bias=False) - self.iaf_B = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) - self.pool_B = SumPool2d(2,2) + self.iaf_B = IAFSqueeze( + batch_size=batch_size, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ) + self.pool_B = SumPool2d(2, 2) self.conv_C = nn.Conv2d(4, 4, 2, 1, bias=False) - self.iaf_C = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) - self.pool_C = SumPool2d(2,2) + self.iaf_C = IAFSqueeze( + batch_size=batch_size, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ) + self.pool_C = SumPool2d(2, 2) self.conv_D = nn.Conv2d(2, 4, 2, 1, bias=False) - self.iaf_D = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + self.iaf_D = IAFSqueeze( + batch_size=batch_size, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ) self.conv_E = nn.Conv2d(4, 4, 2, 1, bias=False) - self.iaf_E = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) - self.pool_E = SumPool2d(2,2) + self.iaf_E = IAFSqueeze( + batch_size=batch_size, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ) + self.pool_E = SumPool2d(2, 2) self.conv_F = nn.Conv2d(4, 4, 2, 1, bias=False) - self.iaf_F = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) - self.pool_F = SumPool2d(2,2) + self.iaf_F = IAFSqueeze( + batch_size=batch_size, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ) + self.pool_F = SumPool2d(2, 2) self.flat_brach1 = nn.Flatten() self.flat_brach2 = nn.Flatten() self.merge = Merge() self.fc1 = nn.Linear(196, 100, bias=False) - self.iaf1_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + self.iaf1_fc = IAFSqueeze( + batch_size=batch_size, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ) self.fc2 = nn.Linear(100, 100, bias=False) - self.iaf2_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + self.iaf2_fc = IAFSqueeze( + batch_size=batch_size, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ) self.fc3 = nn.Linear(100, 10, bias=False) - self.iaf3_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + self.iaf3_fc = IAFSqueeze( + batch_size=batch_size, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ) def forward(self, x): # conv 1 - A @@ -91,7 +137,8 @@ def forward(self, x): iaf3_fc_out = self.iaf3_fc(fc3_out) return iaf3_fc_out - + + channels = 2 height = 34 width = 34 @@ -102,7 +149,7 @@ def forward(self, x): input_dummy = torch.randn(input_shape) expected_output = { - 'edges': { + "edges": { (0, 1), (1, 2), (2, 3), @@ -128,60 +175,60 @@ def forward(self, x): (22, 23), (23, 24), }, - 'name_2_indx_map': { - 'conv_A': 0, - 'iaf_A': 1, - 'conv_B': 2, - 'iaf_B': 3, - 'pool_B': 4, - 'conv_C': 5, - 'iaf_C': 6, - 'pool_C': 7, - 'flat_brach1': 8, - 'conv_D': 9, - 'iaf_D': 10, - 'conv_E': 11, - 'iaf_E': 12, - 'pool_E': 13, - 'conv_F': 14, - 'iaf_F': 15, - 'pool_F': 16, - 'flat_brach2': 17, - 'merge': 18, - 'fc1': 19, - 'iaf1_fc': 20, - 'fc2': 21, - 'iaf2_fc': 22, - 'fc3': 23, - 'iaf3_fc': 24, + "name_2_indx_map": { + "conv_A": 0, + "iaf_A": 1, + "conv_B": 2, + "iaf_B": 3, + "pool_B": 4, + "conv_C": 5, + "iaf_C": 6, + "pool_C": 7, + "flat_brach1": 8, + "conv_D": 9, + "iaf_D": 10, + "conv_E": 11, + "iaf_E": 12, + "pool_E": 13, + "conv_F": 14, + "iaf_F": 15, + "pool_F": 16, + "flat_brach2": 17, + "merge": 18, + "fc1": 19, + "iaf1_fc": 20, + "fc2": 21, + "iaf2_fc": 22, + "fc3": 23, + "iaf3_fc": 24, }, - 'entry_nodes': {0, 9}, - 'nodes_io_shapes': { - 0: {'input': torch.Size([2, 2, 34, 34]), 'output': torch.Size([2, 4, 33, 33])}, - 9: {'input': torch.Size([2, 2, 34, 34]), 'output': torch.Size([2, 4, 33, 33])}, - 1: {'input': torch.Size([2, 4, 33, 33]), 'output': torch.Size([2, 4, 33, 33])}, - 10: {'input': torch.Size([2, 4, 33, 33]), 'output': torch.Size([2, 4, 33, 33])}, - 2: {'input': torch.Size([2, 4, 33, 33]), 'output': torch.Size([2, 4, 32, 32])}, - 11: {'input': torch.Size([2, 4, 33, 33]), 'output': torch.Size([2, 4, 32, 32])}, - 3: {'input': torch.Size([2, 4, 32, 32]), 'output': torch.Size([2, 4, 32, 32])}, - 12: {'input': torch.Size([2, 4, 32, 32]), 'output': torch.Size([2, 4, 32, 32])}, - 4: {'input': torch.Size([2, 4, 32, 32]), 'output': torch.Size([2, 4, 16, 16])}, - 13: {'input': torch.Size([2, 4, 32, 32]), 'output': torch.Size([2, 4, 16, 16])}, - 5: {'input': torch.Size([2, 4, 16, 16]), 'output': torch.Size([2, 4, 15, 15])}, - 14: {'input': torch.Size([2, 4, 16, 16]), 'output': torch.Size([2, 4, 15, 15])}, - 6: {'input': torch.Size([2, 4, 15, 15]), 'output': torch.Size([2, 4, 15, 15])}, - 15: {'input': torch.Size([2, 4, 15, 15]), 'output': torch.Size([2, 4, 15, 15])}, - 7: {'input': torch.Size([2, 4, 15, 15]), 'output': torch.Size([2, 4, 7, 7])}, - 16: {'input': torch.Size([2, 4, 15, 15]), 'output': torch.Size([2, 4, 7, 7])}, - 8: {'input': torch.Size([2, 4, 7, 7]), 'output': torch.Size([2, 196])}, - 17: {'input': torch.Size([2, 4, 7, 7]), 'output': torch.Size([2, 196])}, - 18: {'input': torch.Size([2, 196]), 'output': torch.Size([2, 196])}, - 19: {'input': torch.Size([2, 196]), 'output': torch.Size([2, 100])}, - 20: {'input': torch.Size([2, 100]), 'output': torch.Size([2, 100])}, - 21: {'input': torch.Size([2, 100]), 'output': torch.Size([2, 100])}, - 22: {'input': torch.Size([2, 100]), 'output': torch.Size([2, 100])}, - 23: {'input': torch.Size([2, 100]), 'output': torch.Size([2, 10])}, - 24: {'input': torch.Size([2, 10]), 'output': torch.Size([2, 10])}, + "entry_nodes": {0, 9}, + "nodes_io_shapes": { + 0: {"input": torch.Size([2, 2, 34, 34]), "output": torch.Size([2, 4, 33, 33])}, + 9: {"input": torch.Size([2, 2, 34, 34]), "output": torch.Size([2, 4, 33, 33])}, + 1: {"input": torch.Size([2, 4, 33, 33]), "output": torch.Size([2, 4, 33, 33])}, + 10: {"input": torch.Size([2, 4, 33, 33]), "output": torch.Size([2, 4, 33, 33])}, + 2: {"input": torch.Size([2, 4, 33, 33]), "output": torch.Size([2, 4, 32, 32])}, + 11: {"input": torch.Size([2, 4, 33, 33]), "output": torch.Size([2, 4, 32, 32])}, + 3: {"input": torch.Size([2, 4, 32, 32]), "output": torch.Size([2, 4, 32, 32])}, + 12: {"input": torch.Size([2, 4, 32, 32]), "output": torch.Size([2, 4, 32, 32])}, + 4: {"input": torch.Size([2, 4, 32, 32]), "output": torch.Size([2, 4, 16, 16])}, + 13: {"input": torch.Size([2, 4, 32, 32]), "output": torch.Size([2, 4, 16, 16])}, + 5: {"input": torch.Size([2, 4, 16, 16]), "output": torch.Size([2, 4, 15, 15])}, + 14: {"input": torch.Size([2, 4, 16, 16]), "output": torch.Size([2, 4, 15, 15])}, + 6: {"input": torch.Size([2, 4, 15, 15]), "output": torch.Size([2, 4, 15, 15])}, + 15: {"input": torch.Size([2, 4, 15, 15]), "output": torch.Size([2, 4, 15, 15])}, + 7: {"input": torch.Size([2, 4, 15, 15]), "output": torch.Size([2, 4, 7, 7])}, + 16: {"input": torch.Size([2, 4, 15, 15]), "output": torch.Size([2, 4, 7, 7])}, + 8: {"input": torch.Size([2, 4, 7, 7]), "output": torch.Size([2, 196])}, + 17: {"input": torch.Size([2, 4, 7, 7]), "output": torch.Size([2, 196])}, + 18: {"input": torch.Size([2, 196]), "output": torch.Size([2, 196])}, + 19: {"input": torch.Size([2, 196]), "output": torch.Size([2, 100])}, + 20: {"input": torch.Size([2, 100]), "output": torch.Size([2, 100])}, + 21: {"input": torch.Size([2, 100]), "output": torch.Size([2, 100])}, + 22: {"input": torch.Size([2, 100]), "output": torch.Size([2, 100])}, + 23: {"input": torch.Size([2, 100]), "output": torch.Size([2, 10])}, + 24: {"input": torch.Size([2, 10]), "output": torch.Size([2, 10])}, }, } diff --git a/tests/test_graph_extractor/model_dummy_4.py b/tests/test_graph_extractor/model_dummy_4.py index 58917e19..e83e13cd 100644 --- a/tests/test_graph_extractor/model_dummy_4.py +++ b/tests/test_graph_extractor/model_dummy_4.py @@ -4,37 +4,69 @@ import torch import torch.nn as nn -from sinabs.layers import IAFSqueeze, SumPool2d, Merge + from sinabs.activation.surrogate_gradient_fn import PeriodicExponential +from sinabs.layers import IAFSqueeze, Merge, SumPool2d + class SNN(nn.Module): def __init__(self, batch_size) -> None: super().__init__() self.conv1 = nn.Conv2d(2, 1, 2, 1, bias=False) - self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + self.iaf1 = IAFSqueeze( + batch_size=batch_size, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ) self.conv2 = nn.Conv2d(1, 1, 2, 1, bias=False) - self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) - self.pool2 = SumPool2d(2,2) + self.iaf2 = IAFSqueeze( + batch_size=batch_size, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ) + self.pool2 = SumPool2d(2, 2) self.conv3 = nn.Conv2d(1, 1, 2, 1, bias=False) - self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) - self.pool3 = SumPool2d(2,2) - self.pool3a = SumPool2d(5,5) + self.iaf3 = IAFSqueeze( + batch_size=batch_size, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ) + self.pool3 = SumPool2d(2, 2) + self.pool3a = SumPool2d(5, 5) self.conv4 = nn.Conv2d(1, 1, 2, 1, bias=False) - self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) - self.pool4 = SumPool2d(3,3) + self.iaf4 = IAFSqueeze( + batch_size=batch_size, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ) + self.pool4 = SumPool2d(3, 3) self.flat1 = nn.Flatten() self.flat2 = nn.Flatten() self.conv5 = nn.Conv2d(1, 1, 2, 1, bias=False) - self.iaf5 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + self.iaf5 = IAFSqueeze( + batch_size=batch_size, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ) self.fc2 = nn.Linear(25, 10, bias=False) - self.iaf2_fc = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential()) + self.iaf2_fc = IAFSqueeze( + batch_size=batch_size, + min_v_mem=-1.0, + spike_threshold=1.0, + surrogate_grad_fn=PeriodicExponential(), + ) # -- merges -- self.merge1 = Merge() @@ -62,7 +94,7 @@ def forward(self, x): iaf4_out = self.iaf4(conv4_out) pool4_out = self.pool4(iaf4_out) flat1_out = self.flat1(pool4_out) - + # conv 5 - E/4 conv5_out = self.conv5(pool3a_out) iaf5_out = self.iaf5(conv5_out) @@ -70,12 +102,13 @@ def forward(self, x): # fc 2 - F/5 merge2_out = self.merge2(flat2_out, flat1_out) - + fc2_out = self.fc2(merge2_out) iaf2_fc_out = self.iaf2_fc(fc2_out) return iaf2_fc_out - + + channels = 2 height = 34 width = 34 @@ -86,7 +119,7 @@ def forward(self, x): input_dummy = torch.randn(input_shape) expected_output = { - 'edges': { + "edges": { (0, 1), (1, 2), (1, 3), @@ -109,50 +142,50 @@ def forward(self, x): (6, 11), (15, 18), }, - 'name_2_indx_map': { - 'conv1': 0, - 'iaf1': 1, - 'conv2': 2, - 'conv3': 3, - 'iaf2': 4, - 'pool2': 5, - 'merge1': 6, - 'iaf3': 7, - 'pool3': 8, - 'pool3a': 9, - 'conv5': 10, - 'conv4': 11, - 'iaf4': 12, - 'pool4': 13, - 'flat1': 14, - 'merge2': 15, - 'flat2': 16, - 'iaf5': 17, - 'fc2': 18, - 'iaf2_fc': 19, + "name_2_indx_map": { + "conv1": 0, + "iaf1": 1, + "conv2": 2, + "conv3": 3, + "iaf2": 4, + "pool2": 5, + "merge1": 6, + "iaf3": 7, + "pool3": 8, + "pool3a": 9, + "conv5": 10, + "conv4": 11, + "iaf4": 12, + "pool4": 13, + "flat1": 14, + "merge2": 15, + "flat2": 16, + "iaf5": 17, + "fc2": 18, + "iaf2_fc": 19, }, - 'entry_nodes': {0}, - 'nodes_io_shapes': { - 0: {'input': torch.Size([2, 2, 34, 34]), 'output': torch.Size([2, 1, 33, 33])}, - 1: {'input': torch.Size([2, 1, 33, 33]), 'output': torch.Size([2, 1, 33, 33])}, - 2: {'input': torch.Size([2, 1, 33, 33]), 'output': torch.Size([2, 1, 32, 32])}, - 3: {'input': torch.Size([2, 1, 33, 33]), 'output': torch.Size([2, 1, 32, 32])}, - 4: {'input': torch.Size([2, 1, 32, 32]), 'output': torch.Size([2, 1, 32, 32])}, - 7: {'input': torch.Size([2, 1, 32, 32]), 'output': torch.Size([2, 1, 32, 32])}, - 5: {'input': torch.Size([2, 1, 32, 32]), 'output': torch.Size([2, 1, 16, 16])}, - 8: {'input': torch.Size([2, 1, 32, 32]), 'output': torch.Size([2, 1, 16, 16])}, - 9: {'input': torch.Size([2, 1, 32, 32]), 'output': torch.Size([2, 1, 6, 6])}, - 6: {'input': torch.Size([2, 1, 16, 16]), 'output': torch.Size([2, 1, 16, 16])}, - 10: {'input': torch.Size([2, 1, 6, 6]), 'output': torch.Size([2, 1, 5, 5])}, - 11: {'input': torch.Size([2, 1, 16, 16]), 'output': torch.Size([2, 1, 15, 15])}, - 17: {'input': torch.Size([2, 1, 5, 5]), 'output': torch.Size([2, 1, 5, 5])}, - 12: {'input': torch.Size([2, 1, 15, 15]), 'output': torch.Size([2, 1, 15, 15])}, - 16: {'input': torch.Size([2, 1, 5, 5]), 'output': torch.Size([2, 25])}, - 13: {'input': torch.Size([2, 1, 15, 15]), 'output': torch.Size([2, 1, 5, 5])}, - 14: {'input': torch.Size([2, 1, 5, 5]), 'output': torch.Size([2, 25])}, - 15: {'input': torch.Size([2, 25]), 'output': torch.Size([2, 25])}, - 18: {'input': torch.Size([2, 25]), 'output': torch.Size([2, 10])}, - 19: {'input': torch.Size([2, 10]), 'output': torch.Size([2, 10])}, + "entry_nodes": {0}, + "nodes_io_shapes": { + 0: {"input": torch.Size([2, 2, 34, 34]), "output": torch.Size([2, 1, 33, 33])}, + 1: {"input": torch.Size([2, 1, 33, 33]), "output": torch.Size([2, 1, 33, 33])}, + 2: {"input": torch.Size([2, 1, 33, 33]), "output": torch.Size([2, 1, 32, 32])}, + 3: {"input": torch.Size([2, 1, 33, 33]), "output": torch.Size([2, 1, 32, 32])}, + 4: {"input": torch.Size([2, 1, 32, 32]), "output": torch.Size([2, 1, 32, 32])}, + 7: {"input": torch.Size([2, 1, 32, 32]), "output": torch.Size([2, 1, 32, 32])}, + 5: {"input": torch.Size([2, 1, 32, 32]), "output": torch.Size([2, 1, 16, 16])}, + 8: {"input": torch.Size([2, 1, 32, 32]), "output": torch.Size([2, 1, 16, 16])}, + 9: {"input": torch.Size([2, 1, 32, 32]), "output": torch.Size([2, 1, 6, 6])}, + 6: {"input": torch.Size([2, 1, 16, 16]), "output": torch.Size([2, 1, 16, 16])}, + 10: {"input": torch.Size([2, 1, 6, 6]), "output": torch.Size([2, 1, 5, 5])}, + 11: {"input": torch.Size([2, 1, 16, 16]), "output": torch.Size([2, 1, 15, 15])}, + 17: {"input": torch.Size([2, 1, 5, 5]), "output": torch.Size([2, 1, 5, 5])}, + 12: {"input": torch.Size([2, 1, 15, 15]), "output": torch.Size([2, 1, 15, 15])}, + 16: {"input": torch.Size([2, 1, 5, 5]), "output": torch.Size([2, 25])}, + 13: {"input": torch.Size([2, 1, 15, 15]), "output": torch.Size([2, 1, 5, 5])}, + 14: {"input": torch.Size([2, 1, 5, 5]), "output": torch.Size([2, 25])}, + 15: {"input": torch.Size([2, 25]), "output": torch.Size([2, 25])}, + 18: {"input": torch.Size([2, 25]), "output": torch.Size([2, 10])}, + 19: {"input": torch.Size([2, 10]), "output": torch.Size([2, 10])}, }, } diff --git a/tests/test_graph_extractor/test_graph_extractor.py b/tests/test_graph_extractor/test_graph_extractor.py index 4f019adb..ef333ada 100644 --- a/tests/test_graph_extractor/test_graph_extractor.py +++ b/tests/test_graph_extractor/test_graph_extractor.py @@ -2,12 +2,13 @@ # contact : wsoaresgirao@gmail.com import pytest +from conftest_graph_extractor import args_GraphExtractor + from sinabs.backend.dynapcnn.nir_graph_extractor import GraphExtractor -from conftest_graph_extractor import args_GraphExtractor def fix_node_ids(expected_output, graph_extractor): - """ Match node IDs between graph extractor and expected output + """Match node IDs between graph extractor and expected output Node IDs can be assigned in many ways. This function prevents test errors from generated IDs not matching expected output @@ -25,9 +26,7 @@ def fix_node_ids(expected_output, graph_extractor): expected_idx: graph_extractor.name_2_indx_map[name] for name, expected_idx in expected_output["name_2_indx_map"].items() } - edges = { - (idx_map[src], idx_map[tgt]) for src, tgt in expected_output["edges"] - } + edges = {(idx_map[src], idx_map[tgt]) for src, tgt in expected_output["edges"]} name_2_indx_map = { name: idx_map[idx] for name, idx in expected_output["name_2_indx_map"].items() } @@ -43,9 +42,10 @@ def fix_node_ids(expected_output, graph_extractor): "nodes_io_shapes": nodes_io_shapes, } + @pytest.mark.parametrize("snn, input_dummy, expected_output", args_GraphExtractor) def test_GraphExtractor(snn, input_dummy, expected_output): - """ Tests the graph extraction from the original SNN being turned into a `DynapcnnNetwork`. These tests + """Tests the graph extraction from the original SNN being turned into a `DynapcnnNetwork`. These tests verify the correct functionality of the `GraphExtractor` class, which implements the first pre-processing step on the conversion of the SNN into a DynapcnnNetwork. """ @@ -53,11 +53,15 @@ def test_GraphExtractor(snn, input_dummy, expected_output): graph_tracer = GraphExtractor(snn, input_dummy) expected_output = fix_node_ids(expected_output, graph_tracer) - assert expected_output['edges'] == graph_tracer.edges, \ - f'wrong list of edges extracted from the SNN.' - assert expected_output['name_2_indx_map'] == graph_tracer.name_2_indx_map, \ - f'wrong mapping from layer variable name to node ID.' - assert expected_output['entry_nodes'] == graph_tracer.entry_nodes, \ - f'wrong list with entry node\'s IDs (i.e., layers serving as input to the SNN).' - assert expected_output['nodes_io_shapes'] == graph_tracer.nodes_io_shapes, \ - f'wrong I/O shapes computed for one or more nodes.' + assert ( + expected_output["edges"] == graph_tracer.edges + ), f"wrong list of edges extracted from the SNN." + assert ( + expected_output["name_2_indx_map"] == graph_tracer.name_2_indx_map + ), f"wrong mapping from layer variable name to node ID." + assert ( + expected_output["entry_nodes"] == graph_tracer.entry_nodes + ), f"wrong list with entry node's IDs (i.e., layers serving as input to the SNN)." + assert ( + expected_output["nodes_io_shapes"] == graph_tracer.nodes_io_shapes + ), f"wrong I/O shapes computed for one or more nodes." From 8c6ec259e052c3958b6bb2f83220c4539284d17b Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Fri, 4 Oct 2024 18:37:43 +0200 Subject: [PATCH 185/379] Fix missing import --- sinabs/backend/dynapcnn/utils.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/sinabs/backend/dynapcnn/utils.py b/sinabs/backend/dynapcnn/utils.py index c75255ed..0e42d88b 100644 --- a/sinabs/backend/dynapcnn/utils.py +++ b/sinabs/backend/dynapcnn/utils.py @@ -1,7 +1,16 @@ from collections import defaultdict, deque from copy import deepcopy -from typing import (TYPE_CHECKING, Callable, Dict, List, Optional, Set, Tuple, - Union) +from typing import ( + TYPE_CHECKING, + Callable, + Dict, + Iterable, + List, + Optional, + Set, + Tuple, + Union, +) import torch import torch.nn as nn From 452af1a87b449895f82504d5b848cedb4fa173a9 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Fri, 4 Oct 2024 18:38:14 +0200 Subject: [PATCH 186/379] Rerun black --- sinabs/activation/__init__.py | 9 +++++++-- sinabs/backend/dynapcnn/chip_factory.py | 11 ++++++++--- sinabs/backend/dynapcnn/dynapcnn_network.py | 12 ++++++++---- sinabs/backend/dynapcnn/nir_graph_extractor.py | 6 ++++-- tests/test_activations.py | 10 ++++++++-- tests/test_dynapcnn/test_auto_mapping.py | 3 +-- tests/test_dynapcnn/test_compatible_layer_build.py | 3 +-- tests/test_dynapcnn/test_device_movement.py | 3 +-- tests/test_dynapcnn/test_device_name_mapping.py | 3 +-- tests/test_dynapcnn/test_neuron_leak.py | 11 +++++++---- tests/test_dynapcnn/test_single_neuron_hardware.py | 10 +++++++--- tests/test_dynapcnnlayer/conftest_dynapcnnlayer.py | 12 ++++-------- tests/test_dynapcnnlayer/test_dynapcnnlayer.py | 8 +++++--- 13 files changed, 62 insertions(+), 39 deletions(-) diff --git a/sinabs/activation/__init__.py b/sinabs/activation/__init__.py index 24f3678e..ff0c4d3e 100644 --- a/sinabs/activation/__init__.py +++ b/sinabs/activation/__init__.py @@ -1,5 +1,10 @@ from .quantize import Quantize, StochasticRounding from .reset_mechanism import MembraneReset, MembraneSubtract from .spike_generation import MaxSpike, MultiSpike, SingleSpike -from .surrogate_gradient_fn import (Gaussian, Heaviside, MultiGaussian, - PeriodicExponential, SingleExponential) +from .surrogate_gradient_fn import ( + Gaussian, + Heaviside, + MultiGaussian, + PeriodicExponential, + SingleExponential, +) diff --git a/sinabs/backend/dynapcnn/chip_factory.py b/sinabs/backend/dynapcnn/chip_factory.py index 453d5cc1..105c5904 100644 --- a/sinabs/backend/dynapcnn/chip_factory.py +++ b/sinabs/backend/dynapcnn/chip_factory.py @@ -3,9 +3,14 @@ import numpy as np import torch -from .chips import (DynapcnnConfigBuilder, Speck2BConfigBuilder, - Speck2CMiniConfigBuilder, Speck2DMiniConfigBuilder, - Speck2EConfigBuilder, Speck2FConfigBuilder) +from .chips import ( + DynapcnnConfigBuilder, + Speck2BConfigBuilder, + Speck2CMiniConfigBuilder, + Speck2DMiniConfigBuilder, + Speck2EConfigBuilder, + Speck2FConfigBuilder, +) from .config_builder import ConfigBuilder from .utils import parse_device_id diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index 9f74c992..7faa4562 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -15,12 +15,16 @@ from .dvs_layer import DVSLayer from .dynapcnn_layer import DynapcnnLayer from .dynapcnnnetwork_module import DynapcnnNetworkModule -from .io import (disable_timestamps, enable_timestamps, open_device, - reset_timestamps) +from .io import disable_timestamps, enable_timestamps, open_device, reset_timestamps from .nir_graph_extractor import GraphExtractor from .sinabs_edges_handler import collect_dynapcnn_layer_info -from .utils import (DEFAULT_IGNORED_LAYER_TYPES, Edge, build_from_graph, - parse_device_id, topological_sorting) +from .utils import ( + DEFAULT_IGNORED_LAYER_TYPES, + Edge, + build_from_graph, + parse_device_id, + topological_sorting, +) from .weight_rescaling_methods import rescale_method_1 diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index c3b852b3..7cadff95 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -9,8 +9,10 @@ import sinabs -from .connectivity_specs import (LAYER_TYPES_WITH_MULTIPLE_INPUTS, - LAYER_TYPES_WITH_MULTIPLE_OUTPUTS) +from .connectivity_specs import ( + LAYER_TYPES_WITH_MULTIPLE_INPUTS, + LAYER_TYPES_WITH_MULTIPLE_OUTPUTS, +) from .exceptions import InvalidGraphStructure from .utils import Edge, topological_sorting diff --git a/tests/test_activations.py b/tests/test_activations.py index cfec09ef..1ec880dc 100644 --- a/tests/test_activations.py +++ b/tests/test_activations.py @@ -1,8 +1,14 @@ import pytest import torch -from sinabs.activation import (MaxSpike, MembraneReset, MembraneSubtract, - MultiSpike, SingleExponential, SingleSpike) +from sinabs.activation import ( + MaxSpike, + MembraneReset, + MembraneSubtract, + MultiSpike, + SingleExponential, + SingleSpike, +) @pytest.mark.parametrize( diff --git a/tests/test_dynapcnn/test_auto_mapping.py b/tests/test_dynapcnn/test_auto_mapping.py index 4e8029d5..37de88d9 100644 --- a/tests/test_dynapcnn/test_auto_mapping.py +++ b/tests/test_dynapcnn/test_auto_mapping.py @@ -2,8 +2,7 @@ import torch.nn as nn from sinabs.backend.dynapcnn import DynapcnnNetwork -from sinabs.backend.dynapcnn.mapping import (edmonds, make_flow_graph, - recover_mapping) +from sinabs.backend.dynapcnn.mapping import edmonds, make_flow_graph, recover_mapping from sinabs.from_torch import from_model ann = nn.Sequential( diff --git a/tests/test_dynapcnn/test_compatible_layer_build.py b/tests/test_dynapcnn/test_compatible_layer_build.py index da70d628..54215a52 100644 --- a/tests/test_dynapcnn/test_compatible_layer_build.py +++ b/tests/test_dynapcnn/test_compatible_layer_build.py @@ -171,8 +171,7 @@ def test_incorrect_model_start(): def test_conversion_to_layer_list(): - from sinabs.backend.dynapcnn.utils import \ - DEFAULT_IGNORED_LAYER_TYPES as DEF_IGNORE + from sinabs.backend.dynapcnn.utils import DEFAULT_IGNORED_LAYER_TYPES as DEF_IGNORE from sinabs.backend.dynapcnn.utils import convert_model_to_layer_list model = nn.Sequential( diff --git a/tests/test_dynapcnn/test_device_movement.py b/tests/test_dynapcnn/test_device_movement.py index 5db2415e..a6b9d8a1 100644 --- a/tests/test_dynapcnn/test_device_movement.py +++ b/tests/test_dynapcnn/test_device_movement.py @@ -2,8 +2,7 @@ import torch.nn as nn from sinabs.backend.dynapcnn import DynapcnnNetwork -from sinabs.backend.dynapcnn.mapping import (edmonds, make_flow_graph, - recover_mapping) +from sinabs.backend.dynapcnn.mapping import edmonds, make_flow_graph, recover_mapping from sinabs.from_torch import from_model ann = nn.Sequential( diff --git a/tests/test_dynapcnn/test_device_name_mapping.py b/tests/test_dynapcnn/test_device_name_mapping.py index 0e31893d..b92901ea 100644 --- a/tests/test_dynapcnn/test_device_name_mapping.py +++ b/tests/test_dynapcnn/test_device_name_mapping.py @@ -1,5 +1,4 @@ -from sinabs.backend.dynapcnn.utils import (parse_device_id, - standardize_device_id) +from sinabs.backend.dynapcnn.utils import parse_device_id, standardize_device_id def test_device_id_no_index(): diff --git a/tests/test_dynapcnn/test_neuron_leak.py b/tests/test_dynapcnn/test_neuron_leak.py index ece73964..4acacdbc 100644 --- a/tests/test_dynapcnn/test_neuron_leak.py +++ b/tests/test_dynapcnn/test_neuron_leak.py @@ -3,13 +3,16 @@ import pytest import samna import torch -from hw_utils import (find_open_devices, get_ones_network, - is_any_samna_device_connected, is_device_connected) +from hw_utils import ( + find_open_devices, + get_ones_network, + is_any_samna_device_connected, + is_device_connected, +) from torch import nn from sinabs.backend.dynapcnn import DynapcnnNetwork -from sinabs.backend.dynapcnn.io import (calculate_neuron_address, - neuron_address_to_cxy) +from sinabs.backend.dynapcnn.io import calculate_neuron_address, neuron_address_to_cxy from sinabs.layers import IAFSqueeze diff --git a/tests/test_dynapcnn/test_single_neuron_hardware.py b/tests/test_dynapcnn/test_single_neuron_hardware.py index e9e99bdd..6d8fe6c6 100644 --- a/tests/test_dynapcnn/test_single_neuron_hardware.py +++ b/tests/test_dynapcnn/test_single_neuron_hardware.py @@ -1,7 +1,11 @@ import pytest -from hw_utils import (find_open_devices, get_ones_network, - is_any_samna_device_connected, is_device_connected, - reset_all_connected_boards) +from hw_utils import ( + find_open_devices, + get_ones_network, + is_any_samna_device_connected, + is_device_connected, + reset_all_connected_boards, +) import sinabs import sinabs.backend.dynapcnn as sdl diff --git a/tests/test_dynapcnnlayer/conftest_dynapcnnlayer.py b/tests/test_dynapcnnlayer/conftest_dynapcnnlayer.py index 4e065c2a..29a405e4 100644 --- a/tests/test_dynapcnnlayer/conftest_dynapcnnlayer.py +++ b/tests/test_dynapcnnlayer/conftest_dynapcnnlayer.py @@ -1,14 +1,10 @@ # author : Willian Soares Girao # contact : wsoaresgirao@gmail.com -from model_dummy_1 import (expected_output_1, nodes_to_dcnnl_map_1, - sinabs_edges_1) -from model_dummy_2 import (expected_output_2, nodes_to_dcnnl_map_2, - sinabs_edges_2) -from model_dummy_3 import (expected_output_3, nodes_to_dcnnl_map_3, - sinabs_edges_3) -from model_dummy_4 import (expected_output_4, nodes_to_dcnnl_map_4, - sinabs_edges_4) +from model_dummy_1 import expected_output_1, nodes_to_dcnnl_map_1, sinabs_edges_1 +from model_dummy_2 import expected_output_2, nodes_to_dcnnl_map_2, sinabs_edges_2 +from model_dummy_3 import expected_output_3, nodes_to_dcnnl_map_3, sinabs_edges_3 +from model_dummy_4 import expected_output_4, nodes_to_dcnnl_map_4, sinabs_edges_4 args_DynapcnnLayer = [ (nodes_to_dcnnl_map_1, 0, sinabs_edges_1, [0], expected_output_1), diff --git a/tests/test_dynapcnnlayer/test_dynapcnnlayer.py b/tests/test_dynapcnnlayer/test_dynapcnnlayer.py index 8791bd67..52988463 100644 --- a/tests/test_dynapcnnlayer/test_dynapcnnlayer.py +++ b/tests/test_dynapcnnlayer/test_dynapcnnlayer.py @@ -4,9 +4,11 @@ import pytest from conftest_dynapcnnlayer import args_DynapcnnLayer -from sinabs.backend.dynapcnn.utils import (construct_dynapcnnlayer, - construct_layerhandler, - update_nodes_io) +from sinabs.backend.dynapcnn.utils import ( + construct_dynapcnnlayer, + construct_layerhandler, + update_nodes_io, +) from sinabs.backend.dynapcnn.weight_rescaling_methods import rescale_method_1 From befc244a7c0fddb580e53554acaa060184b8ae5c Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Tue, 8 Oct 2024 17:21:27 +0200 Subject: [PATCH 187/379] (WIP) Update dynapcnn layer instantiation: handling of input shapes for linear to conv conversion --- sinabs/backend/dynapcnn/dynapcnn_layer.py | 93 +++++++++---------- .../backend/dynapcnn/sinabs_edges_handler.py | 31 ++++++- sinabs/backend/dynapcnn/utils.py | 60 ++++++++---- 3 files changed, 115 insertions(+), 69 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer.py b/sinabs/backend/dynapcnn/dynapcnn_layer.py index ed62013f..aaac881a 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer.py @@ -17,6 +17,46 @@ sum_pool2d = partial(nn.functional.lp_pool2d, norm_type=1) +def convert_linear_to_conv( + lin: nn.Linear, input_shape: Tuple[int, int, int] +) -> nn.Conv2d: + """Convert Linear layer to Conv2d. + + Parameters + ---------- + - lin (nn.Linear): linear layer to be converted. + - input_shape (tuple): the tensor shape the layer expects. + + Returns + ------- + - nn.Conv2d: convolutional layer equivalent to `lin`. + """ + in_chan, in_h, in_w = input_shape + if lin.in_features != in_chan * in_h * in_w: + raise ValueError( + "Shape of linear layer weight does not match provided input shape" + ) + + layer = nn.Conv2d( + in_channels=in_chan, + kernel_size=(in_h, in_w), + out_channels=lin.out_features, + padding=0, + bias=lin.bias is not None, + ) + + if lin.bias is not None: + layer.bias.data = lin.bias.data.clone().detach() + + layer.weight.data = ( + lin.weight.data.clone() + .detach() + .reshape((lin.out_features, in_chan, in_h, in_w)) + ) + + return layer + + class DynapcnnLayer(nn.Module): """Create a DynapcnnLayer object representing a layer on DynapCNN or Speck. @@ -62,11 +102,13 @@ def __init__( spk = deepcopy(spk) + # Convert `nn.Linear` to `nn.Conv2d`. if isinstance(conv, nn.Linear): - conv = self._convert_linear_to_conv(conv) - if spk.is_state_initialised(): - # Expand dims - spk.v_mem = spk.v_mem.data.unsqueeze(-1).unsqueeze(-1) + conv = convert_linear_to_conv(conv) + if spk.is_state_initialised() and (ndim := spk.ndim) < 4: + for __ in range(4 - ndim): + # Expand spatial dimensions + spk.v_mem = spk.v_mem.data.unsqueeze(-1) else: conv = deepcopy(conv) @@ -204,49 +246,6 @@ def memory_summary(self): ####################################################### Private Methods ####################################################### - def _convert_linear_to_conv( - self, lin: nn.Linear, layer_data: dict - ) -> Tuple[nn.Conv2d, Tuple[int, int, int]]: - """Convert Linear layer to Conv2d. - - Parameters - ---------- - - lin (nn.Linear): linear layer to be converted. - - Returns - ------- - - nn.Conv2d: convolutional layer equivalent to `lin`. - - input_shape (tuple): the tensor shape the layer expects. - """ - # this flags the necessity to update the I/O shape pre-computed for each of the original layers being compressed within a `DynapcnnLayer` instance. - self._lin_to_conv_conversion = True - - input_shape = layer_data["input_shape"] - - in_chan, in_h, in_w = input_shape - - if lin.in_features != in_chan * in_h * in_w: - raise ValueError("Shapes don't match.") - - layer = nn.Conv2d( - in_channels=in_chan, - kernel_size=(in_h, in_w), - out_channels=lin.out_features, - padding=0, - bias=lin.bias is not None, - ) - - if lin.bias is not None: - layer.bias.data = lin.bias.data.clone().detach() - - layer.weight.data = ( - lin.weight.data.clone() - .detach() - .reshape((lin.out_features, in_chan, in_h, in_w)) - ) - - return layer, input_shape - def _get_conv_output_shape(self) -> Tuple[int, int, int]: """Computes the output dimensions of `conv_layer`. diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index b82af9bc..bb0583b2 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -105,11 +105,15 @@ def collect_dynapcnn_layer_info( # Process all edges connecting two dynapcnn layers while edges_by_type["neuron-weight"]: edge = edges_by_type["neuron-weight"].pop() - set_neuron_layer_destination(dynapcnn_layer_info, edge, node_2_layer_map) + set_neuron_layer_destination( + dynapcnn_layer_info, edge, node_2_layer_map, nodes_io_shapes + ) while edges_by_type["pooling-weight"]: edge = edges_by_type["pooling-weight"].pop() - set_pooling_layer_destination(dynapcnn_layer_info, edge, node_2_layer_map) + set_pooling_layer_destination( + dynapcnn_layer_info, edge, node_2_layer_map, nodes_io_shapes + ) # Make sure we have taken care of all edges assert all(len(edges) == 0 for edges in edges_by_type.values()) @@ -173,6 +177,9 @@ def init_new_dynapcnnlayer_entry( dynapcnn_layer_info[layer_id] = { "input_shape": nodes_io_shapes[edge[0]], + # Collect output shapes (before possible flattening) of layers with this layer as their destination + # This will allow infering shapes when converting linear to conv layers + "inferred_input_shapes": set(), "conv": { "module": indx_2_module_map[edge[0]], "node_id": edge[0], @@ -182,7 +189,7 @@ def init_new_dynapcnnlayer_entry( "node_id": edge[1], }, # This will be used later to account for average pooling in preceding layers - "rescale_factors": {}, + "rescale_factors": set(), } node_2_layer_map[edge[0]] = layer_id node_2_layer_map[edge[1]] = layer_id @@ -249,6 +256,7 @@ def set_neuron_layer_destination( dynapcnn_layer_info: Dict[int, Dict[int, Dict]], edge: Edge, node_2_layer_map: Dict[int, int], + nodes_io_shapes: Dict[int, Dict[str, Tuple[Size, Size]]], ) -> None: """Set destination layer without pooling. @@ -261,6 +269,7 @@ def set_neuron_layer_destination( Edge source has to be within an existing entry of `dynapcnn_layer_info`. node_2_layer_map (dict): Maps each node ID to the ID of the layer it is assigned to. Will be updated in-place. + nodes_io_shapes (dict): Map from node ID to dict containing node's in- and output shapes """ # Make sure both source (neuron layer) and target (weight layer) have been previously processed try: @@ -278,19 +287,27 @@ def set_neuron_layer_destination( layer_info["destinations"] = [] # Add new destination + output_shape = nodes_io_shapes[edge[0]]["output"] layer_info["destinations"].append( { "pooling_ids": [], "pooling_modules": [], "destination_layer": destination_layer_idx, + "output_shape": output_shape, } ) + # Add output shape of this layer to input shapes of destination + dynapcnn_layer_info[destination_layer_idx]["inferred_input_shapes"].add( + output_shape + ) + def set_pooling_layer_destination( dynapcnn_layer_info: Dict[int, Dict[int, Dict]], edge: Edge, node_2_layer_map: Dict[int, int], + nodes_io_shapes: Dict[int, Dict[str, Tuple[Size, Size]]], ) -> None: """Set destination layer with pooling. @@ -303,6 +320,7 @@ def set_pooling_layer_destination( Edge source has to be within an existing entry of `dynapcnn_layer_info`. node_2_layer_map (dict): Maps each node ID to the ID of the layer it is assigned to. Will be updated in-place. + nodes_io_shapes (dict): Map from node ID to dict containing node's in- and output shapes """ # Make sure both source (pooling layer) and target (weight layer) have been previously processed try: @@ -328,6 +346,13 @@ def set_pooling_layer_destination( # Set destination layer within destination dict that holds current source node destination["destination_layer"] = destination_layer_idx + output_shape = nodes_io_shapes[edge[0]]["output"] + destination["output_shape"] = output_shape + + # Add output shape of this layer to input shapes of destination + dynapcnn_layer_info[destination_layer_idx]["inferred_input_shapes"].add( + output_shape + ) def trace_paths(node: int, remaining_edges: Set[Edge]) -> List[deque[int]]: diff --git a/sinabs/backend/dynapcnn/utils.py b/sinabs/backend/dynapcnn/utils.py index 0e42d88b..e9922338 100644 --- a/sinabs/backend/dynapcnn/utils.py +++ b/sinabs/backend/dynapcnn/utils.py @@ -1,5 +1,6 @@ from collections import defaultdict, deque from copy import deepcopy +from math import prod from typing import ( TYPE_CHECKING, Callable, @@ -11,6 +12,7 @@ Tuple, Union, ) +from warnings import warn import torch import torch.nn as nn @@ -291,14 +293,6 @@ def construct_all_dynapcnnlayers( ) -> Dict[int, DynapcnnLayer]: """...""" - # Extract construction arguments from dcnnl_map - # -conv layer - # -neuron layer - # -pooling -> requires consolidation - # -input shape - # -discretize - # -weight rescale factor - # Consolidate pooling information for each destination for layer_info in dcnnl_map.values(): for destination in layer_info["destinations"]: @@ -306,7 +300,7 @@ def construct_all_dynapcnnlayers( destination["cumulative_pooling"] = pool destination["cumulative_scaling"] = scale dest_lyr_idx = destination["destination_layer"] - dcnnl_map[dest_lyr_idx]["rescale_factors"].add(layer_rescaling) + dcnnl_map[dest_lyr_idx]["rescale_factors"].add(scale) dynapcnn_layer = { layer_idx: construct_single_dynapcnn_layer(layer_info, discretize, rescale_fn) @@ -320,9 +314,10 @@ def construct_all_dynapcnnlayers( def construct_single_dynapcnn_layer( - layer_info: Dict, rescale_fn: Optional[Callable] = None + layer_info: Dict, discretize: bool, rescale_fn: Optional[Callable] = None ) -> DynapcnnLayer: + # Handle rescaling after average pooling if len(layer_info["rescale_factors"]) == 0: rescale_factor = 1 elif len(layer_info["rescale_factors"]) == 1: @@ -338,14 +333,41 @@ def construct_single_dynapcnn_layer( else: rescale_factor = rescale_fn(layer_info["rescale_factors"]) + # Handle input dimensions + # For each dimension find larges inferred input size + max_inferred_input_shape = [ + max(sizes) for sizes in zip(layer_info["inferred_input_shapes"]) + ] + + if isinstance(layer_info["conv"]["module"], nn.Linear): + if prod(max_inferred_input_shape) > prod(layer_info["input_shape"]): + raise ValueError( + "Combined output of some layers projecting to a linear layer is " + "larger than expected by destination layer. " + ) + # Take shape before flattening, to convert linear to conv layer + in_shape = max_inferred_input_shape + else: + if any( + inferred > expected + for inferred, expected in zip( + max_inferred_input_shape, layer_info["input_shape"] + ) + ): + raise ValueError( + "Output of some layers is larger than expected by destination " + "layer along some dimensions." + ) + in_shape = layer_info["input_shape"] + # Collect pooling in a list - [dest["cumulative_pooling"] for dest in layer_info["destinations"]] + pooling_list = [dest["cumulative_pooling"] for dest in layer_info["destinations"]] # instantiate a DynapcnnLayer from the data in the handler. return DynapcnnLayer( conv=layer_info["conv"]["module"], spk=layer_info["neuron"]["module"], - in_shape=layer_info["input_shape"], + in_shape=in_shape, pool=pooling_list, discretize=discretize, rescale_weights=rescale_factor, @@ -380,13 +402,13 @@ def consolidate_pooling(modules: Iterable[nn.Module]) -> Tuple[Tuple[int, int], def extract_pooling_from_module( - module: Union[nn.AvgPool2d, sl.SumPool2d] + pooling_layer: Union[nn.AvgPool2d, sl.SumPool2d] ) -> Tuple[Tuple[int, int], float]: """Extract pooling size and required rescaling factor from pooling module Parameters ---------- - module: pooling module + pooling_layer: pooling module Returns ------- @@ -394,20 +416,20 @@ def extract_pooling_from_module( scale_factor: float, indicating by how much subsequent weights need to be rescaled to account for average pooling being converted to sum pooling. """ - pooling = expand_to_pair(module.kernel_size) + pooling = expand_to_pair(pooling_layer.kernel_size) - if module.stride is not None: - stride = expand_to_pair(module.stride) + if pooling_layer.stride is not None: + stride = expand_to_pair(pooling_layer.stride) if pooling != stride: raise ValueError( - f"Stride length {module.stride} should be the same as pooling kernel size {module.kernel_size}" + f"Stride length {pooling_layer.stride} should be the same as pooling kernel size {pooling_layer.kernel_size}" ) if isinstance(pooling_layer, nn.AvgPool2d): scale_factor = 1.0 / (pooling[0] * pooling[1]) elif isinstance(pooling_layer, sl.SumPool2d): scale_factor = 1.0 else: - raise ValueError(f"Unsupported type {type(module)} for pooling layer") + raise ValueError(f"Unsupported type {type(pooling_layer)} for pooling layer") return pooling, scale_factor From af0922fd6fa9d698430e6ad5dbc59241190907d7 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Tue, 8 Oct 2024 17:36:57 +0200 Subject: [PATCH 188/379] Remove methods from DynapcnnLayerHandler that will be obsolete after refactoring: --- sinabs/backend/dynapcnn/dynapcnn_layer.py | 10 + .../dynapcnn/dynapcnn_layer_handler.py | 316 +----------------- 2 files changed, 14 insertions(+), 312 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer.py b/sinabs/backend/dynapcnn/dynapcnn_layer.py index aaac881a..a7ad3562 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer.py @@ -116,6 +116,16 @@ def __init__( # this has to be done after copying but before discretizing conv.weight.data = (conv.weight / self._rescale_weights).clone().detach() + # TODO: Does this really need to be enforced here or upon deployment? + # check if convolution kernel is a square. + if conv.kernel_size[0] != conv.kernel_size[1]: + raise ValueError( + "The kernel of a `nn.Conv2d` must have the same height and width." + ) + for pool_size in pool: + if pool_size[0] != pool_size[1]: + raise ValueError("Only square pooling kernels are supported") + # int conversion is done while writing the config. if self._discretize: conv, spk = discretize_conv_spike_(conv, spk, to_int=False) diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer_handler.py b/sinabs/backend/dynapcnn/dynapcnn_layer_handler.py index 877e2e18..f0273906 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer_handler.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer_handler.py @@ -1,18 +1,10 @@ # author : Willian Soares Girao # contact : wsoaresgirao@gmail.com -from copy import deepcopy -from typing import Callable, Dict, List, Tuple, Union +from typing import Dict, List, Tuple, Union -import numpy as np -import torch from torch import nn -import sinabs.activation -import sinabs.layers as sl - -from .discretize import discretize_conv_spike_ - class DynapcnnLayerHandler: """ @@ -44,174 +36,29 @@ def __init__( List[int], ], ], - discretize: bool, sinabs_edges: List[Tuple[int, int]], - weight_rescaling_fn: Callable, entry_nodes: List[int], ): self.dpcnnl_index = dpcnnl_index - self.assigned_core = None self.entry_point = False if "core_idx" in dcnnl_data: self.assigned_core = dcnnl_data["core_idx"] - - self._lin_to_conv_conversion = False - - conv = None - self.conv_node_id = None - self.conv_in_shape = None - self.conv_out_shape = None - - spk = None - self.spk_node_id = None - - pool = [] - self.pool_node_id = [] - self.conv_rescaling_factor = None - - self.dynapcnnlayer_destination = dcnnl_data["destinations"] - - for key, value in dcnnl_data.items(): - if isinstance(key, int): - # value has data pertaining a node (torch/sinabs layer). - if isinstance(value["layer"], sl.IAFSqueeze): - spk = value["layer"] - self.spk_node_id = key - elif isinstance(value["layer"], nn.Linear) or isinstance( - value["layer"], nn.Conv2d - ): - conv = value["layer"] - self.conv_node_id = key - elif isinstance(value["layer"], sl.SumPool2d): - pool.append(value["layer"]) - self.pool_node_id.append(key) - else: - raise ValueError( - f"Node {key} has not valid layer associated with it." - ) - - if not conv: - raise ValueError(f"Convolution layer not present.") - - if not spk: - raise ValueError(f"Spiking layer not present.") - - spk = deepcopy(spk) - if spk.is_state_initialised(): - # TODO this line bellow is causing an exception on `.v_men.shape` to be raised in `.get_layer_config_dict()`. Find out why. - # spk.v_mem = spk.v_mem.data.unsqueeze(-1).unsqueeze(-1) # expand dims. - - # TODO hacky stuff: make it better (THIS SEEMS TO BE FIXING THE PROBLEM ABOVE THO). - if len(list(spk.v_mem.shape)) != 4: - spk.v_mem = spk.v_mem.data.unsqueeze(-1).unsqueeze(-1) # expand dims. - - if isinstance(conv, nn.Linear): - # A `nn.Linear` needs to be converted into `nn.Conv2d`. The I/O shapes of the spiking layer are updated - # accordingly following the conversion. - - conv, conv_in_shape = self._convert_linear_to_conv( - conv, dcnnl_data[self.conv_node_id] - ) - - # the original `nn.Linear` output shape becomes the equivalent `nn.Conv2d` shape. - self.conv_out_shape = self._update_conv_node_output_shape( - conv_layer=conv, - layer_data=dcnnl_data[self.conv_node_id], - input_shape=conv_in_shape, - ) - - # the I/O shapes for neuron layer following the new conv need also to be updated. - self._update_neuron_node_output_shape( - spiking_layer_data=dcnnl_data[self.spk_node_id], - conv_out_shape=self.conv_out_shape, - ) - - else: - self.conv_out_shape = dcnnl_data[self.conv_node_id]["output_shape"] - conv = deepcopy(conv) - - # check if convolution kernel is a square. - if conv.kernel_size[0] != conv.kernel_size[1]: - raise ValueError( - "The kernel of a `nn.Conv2d` must have the same height and width." - ) - - # input shape of conv layer. - self.conv_in_shape = dcnnl_data[self.conv_node_id]["input_shape"] - # input shape of the `DynapcnnLayer` instance. - self.input_shape = self.conv_in_shape - - # this weight rescale comes from the node projecting into this 'conv' node. - if len(dcnnl_data["conv_rescale_factor"]): - # this means an `AvgPool2d` has been converted into a `SumPool2d`. - self.conv_rescaling_factor = weight_rescaling_fn( - dcnnl_data["conv_rescale_factor"] - ) - conv.weight.data = ( - (conv.weight.data / self.conv_rescaling_factor).clone().detach() - ) else: - # this means `SumPool2d` have been used from the start. - conv.weight.data = (conv.weight.data).clone().detach() + self.assigned_core = None - # int conversion is done while writing the config. - if discretize: - conv, spk = discretize_conv_spike_(conv, spk, to_int=False) - - # consolidate layers. - self.conv_layer = conv - self.spk_layer = spk - self.pool_layer = [] - - if len(pool) != 0: - # the 1st pooling targets the 1st destination in `dcnnl_data['destinations']`, the 2nd pooling targets the 2nd destination... - for plyr in pool: - # @TODO POSSIBLE INCONSISTENCY: if the `SumPool2d` is the result of a conversion from `AvgPool2d` then `SumPool2d.kernel_size` - # is of type tuple, otherwise it is an int. - if ( - isinstance(plyr.kernel_size, tuple) - and plyr.kernel_size[0] != plyr.kernel_size[1] - ): - raise ValueError("Only square kernels are supported") - self.pool_layer.append(deepcopy(plyr)) + self.dynapcnnlayer_destination = dcnnl_data["destinations"] # map destination nodes for each layer in this instance. self.nodes_destinations = self._get_destinations_input_source(sinabs_edges) + # TODO: Move detection of entry points to dcnnl_info generation # flag if the instance is an entry point (i.e., an input node of the network). if self.conv_node_id in entry_nodes: self.entry_point = True ####################################################### Public Methods ####################################################### - def get_pool_list(self) -> List[int]: - """This returns a list of integers that describe the number of outputs created by this layer (length of the list) and - whether or not pooling is applied (values > 1). This is meant to generate the `pool`argument for a `DynapcnnLayer` instance. - - Returns - ---------- - - pool (list): Each integer entry represents an output (destination on chip) and whether pooling should be applied (values > 1) or not (values - equal to 1). The number of entries determines the number of tensors the layer's forward method returns. - """ - pool = [] - - for lyr, dests in self.nodes_destinations.items(): - if lyr == self.spk_node_id: - # spk layer projects somewhere outside this layer (output without pooling). - pool.append(1) - elif lyr in self.pool_node_id: - # getting kernel sizes from each pooling layer. - sumpool_idx = self.pool_node_id.index(lyr) - kernel_size = ( - self.pool_layer[sumpool_idx].kernel_size[0] - if isinstance(self.pool_layer[sumpool_idx].kernel_size, tuple) - else self.pool_layer[sumpool_idx].kernel_size - ) - pool.append(kernel_size) - - return pool - def get_destination_dcnnl_index(self, dcnnl_id: int) -> int: """The `forward` method will return as many tensors as there are elements in `self.dynapcnnlayer_destination`. Since the i-th returned tensor is to be fed to the i-th destionation in `self.dynapcnnlayer_destination`, the return of this method can be used to index a tensor returned in the `forward` method. @@ -226,61 +73,6 @@ def get_destination_dcnnl_index(self, dcnnl_id: int) -> int: """ return self.dynapcnnlayer_destination.index(dcnnl_id) - def get_modified_node_io( - self, dcnnl_data: dict - ) -> Union[Tuple[int, tuple], Tuple[None, None]]: - """Follwing a conversion, the I/O shapes of the spiking layer have been updated to match the convolution's - output. Thus, all nodes receiving input from this spiking layer need their input shapes updated. - - Parameters - ---------- - - dcnnl_data (dict): the set of layers grouped together to comprise this instance of a `DynapcnnLayer`. - - Returns - ---------- - - node ID (int): the ID of the spiking layer consuming the tunerd layer's output (`None` if there was no conversion). - - output shape (tuple): the new output shape following a converstion from `nn.Linear` to `nn.Conv2d` (`None` if there was no conversion). - """ - if self._lin_to_conv_conversion: - return self.spk_node_id, dcnnl_data[self.spk_node_id]["output_shape"] - return None, None - - def zero_grad(self, set_to_none: bool = False) -> None: - return self.spk_layer.zero_grad(set_to_none) - - def get_conv_output_shape( - self, conv_layer: nn.Conv2d, input_shape: Tuple[int, int, int] - ) -> Tuple[int, int, int]: - """Computes the output dimensions of `conv_layer`. - - Parameters - ---------- - - conv_layer (nn.Conv2d): conv. layer whose output will be computed for. - - input_shape (tuple): the shape for the input tensor the layer will process. - - Returns - ---------- - - output dimensions (tuple): a tuple describing `(output channels, height, width)`. - """ - # get the layer's parameters. - out_channels = conv_layer.out_channels - kernel_size = conv_layer.kernel_size - stride = conv_layer.stride - padding = conv_layer.padding - dilation = conv_layer.dilation - - # compute the output height and width. - out_height = ( - (input_shape[1] + 2 * padding[0] - dilation[0] * (kernel_size[0] - 1) - 1) - // stride[0] - ) + 1 - out_width = ( - (input_shape[2] + 2 * padding[1] - dilation[1] * (kernel_size[1] - 1) - 1) - // stride[1] - ) + 1 - - return (out_channels, out_height, out_width) - def __str__(self): pretty_print = "\n" @@ -309,89 +101,6 @@ def __str__(self): ####################################################### Private Methods ####################################################### - def _update_neuron_node_output_shape( - self, spiking_layer_data: dict, conv_out_shape: tuple - ) -> None: - """Updates the spiking layer's I/O shapes after the conversion of a `nn.Linear` into a `nn.Conv2d` (to match the convolution's output). - - Parameters - ---------- - - spiking_layer_data (dict): the dictionary containing all data regarding the spiking layer. - - conv_out_shape (tuple): the output shape of the convolution layer preceeding the spiking layer. - """ - - # spiking layer consumes the tensor coming out of the conv. layer. - spiking_layer_data["input_shape"] = conv_out_shape - # spiking layer outputs the same shape as the conv. layer. - spiking_layer_data["output_shape"] = spiking_layer_data["input_shape"] - - def _update_conv_node_output_shape( - self, conv_layer: nn.Conv2d, layer_data: dict, input_shape: Tuple[int, int, int] - ) -> Tuple[int, int, int]: - """Updates the shape of the output tensor of a node that used to be a `nn.Linear` and became a `nn.Conv2d`. - - The input shapes to nodes are extracted using a list of edges by finding the output shape of the 1st element - in the edge and setting it as the input shape to the 2nd element in the edge. If a node used to be a `nn.Linear` - and it became a `nn.Conv2d`, output shape in the mapper needs to be updated, otherwise there will be a mismatch - between its output and the input it provides to another node. - - Parameters - ---------- - - conv_layer (nn.Module): the `nn.Conv2d` created from a `nn.Linear`. - - layer_data (dict): the dictionary containing the data associated with the original `nn.Linear` converted into `nn.Conv2d`. - - input_shape (tuple): the input shape the layer expects. - - Returns - ---------- - - output_shape (tuple): the tensor shape produced by the `nn.Conv2d` created from a `nn.Linear`. - """ - layer_data["output_shape"] = self.get_conv_output_shape(conv_layer, input_shape) - - return layer_data["output_shape"] - - def _convert_linear_to_conv( - self, lin: nn.Linear, layer_data: dict - ) -> Tuple[nn.Conv2d, Tuple[int, int, int]]: - """Convert Linear layer to Conv2d. - - Parameters - ---------- - - lin (nn.Linear): linear layer to be converted. - - Returns - ------- - - nn.Conv2d: convolutional layer equivalent to `lin`. - - input_shape (tuple): the tensor shape the layer expects. - """ - # this flags the necessity to update the I/O shape pre-computed for each of the original layers being compressed within a `DynapcnnLayer` instance. - self._lin_to_conv_conversion = True - - input_shape = layer_data["input_shape"] - - in_chan, in_h, in_w = input_shape - - if lin.in_features != in_chan * in_h * in_w: - raise ValueError("Shapes don't match.") - - layer = nn.Conv2d( - in_channels=in_chan, - kernel_size=(in_h, in_w), - out_channels=lin.out_features, - padding=0, - bias=lin.bias is not None, - ) - - if lin.bias is not None: - layer.bias.data = lin.bias.data.clone().detach() - - layer.weight.data = ( - lin.weight.data.clone() - .detach() - .reshape((lin.out_features, in_chan, in_h, in_w)) - ) - - return layer, input_shape - def _get_destinations_input_source(self, sinabs_edges: list) -> dict: """Creates a mapping between each layer in this `DynapcnnLayer` instance and its targe nodes that are part of different `DynapcnnLayer` instances. This mapping is used to figure out how many tensors the `forward` method needs to return. @@ -429,23 +138,6 @@ def _get_destinations_input_source(self, sinabs_edges: list) -> dict: return destinations_input_source - def get_pool_kernel_size(self, node: int) -> int: - """Returns the pooling kernel size if `node` is a pooling layer.""" - - if node in self.pool_node_id: - i = self.pool_node_id.index(node) - return ( - self.pool_layer[i].kernel_size[0] - if isinstance(self.pool_layer[i].kernel_size, tuple) - else self.pool_layer[i].kernel_size - ) - elif node == self.spk_node_id: - return 1 - else: - raise ValueError( - f"Node {node} does not belong to DynapcnnLayer {self.dpcnnl_index}." - ) - @staticmethod def find_nodes_core_id(node: int, all_handlers: dict) -> int: """Loops through all handlers in `all_handlers` to find to which core a `DynapcnnLayer` containing From 03cb842ed585d5a1a3ab353fdb7bc3fbea84e0dc Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Wed, 9 Oct 2024 09:55:23 +0200 Subject: [PATCH 189/379] Fix type hint for `remve_nodes_by_class` method --- sinabs/backend/dynapcnn/nir_graph_extractor.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index 7cadff95..10b4b0e3 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -91,9 +91,7 @@ def sorted_nodes(self) -> List[int]: def indx_2_module_map(self) -> Dict[int, nn.Module]: return {n: module for n, module in self._indx_2_module_map.items()} - def remove_nodes_by_class( - self, node_classes: Tuple[Type] - ) -> Tuple[Set[int], Dict[int, int]]: + def remove_nodes_by_class(self, node_classes: Tuple[Type]): """Remove nodes of given classes from graph in place. Create a new set of edges, considering layers that `DynapcnnNetwork` will ignore. This From e84e2faa4532073845c40f12f19445fcb2259639 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Wed, 9 Oct 2024 11:22:20 +0200 Subject: [PATCH 190/379] Refactor DynapcnnLayer generation --- .../dynapcnn/dynapcnn_layer_handler.py | 42 +- sinabs/backend/dynapcnn/dynapcnn_network.py | 85 +-- .../backend/dynapcnn/sinabs_edges_handler.py | 6 + sinabs/backend/dynapcnn/utils.py | 534 ++++++------------ 4 files changed, 204 insertions(+), 463 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer_handler.py b/sinabs/backend/dynapcnn/dynapcnn_layer_handler.py index f0273906..8f619711 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer_handler.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer_handler.py @@ -1,9 +1,7 @@ # author : Willian Soares Girao # contact : wsoaresgirao@gmail.com -from typing import Dict, List, Tuple, Union - -from torch import nn +from typing import List, Optional class DynapcnnLayerHandler: @@ -28,37 +26,23 @@ class DynapcnnLayerHandler: def __init__( self, - dpcnnl_index: int, - dcnnl_data: Dict[ - Union[int, str], - Union[ - Dict[str, Union[nn.Module, Tuple[int, int, int], Tuple[int, int, int]]], - List[int], - ], - ], - sinabs_edges: List[Tuple[int, int]], - entry_nodes: List[int], + layer_index: int, + is_entry_node: bool, + destination_indices: List[int], + assigned_core: Optional[int] = None, ): - self.dpcnnl_index = dpcnnl_index - self.entry_point = False - - if "core_idx" in dcnnl_data: - self.assigned_core = dcnnl_data["core_idx"] - else: - self.assigned_core = None - - self.dynapcnnlayer_destination = dcnnl_data["destinations"] + self.layer_index = layer_index + self.entry_node = is_entry_node + self.destination_indices = destination_indices + self.assigned_core = assigned_core + # TODO: Still needed? # map destination nodes for each layer in this instance. - self.nodes_destinations = self._get_destinations_input_source(sinabs_edges) - - # TODO: Move detection of entry points to dcnnl_info generation - # flag if the instance is an entry point (i.e., an input node of the network). - if self.conv_node_id in entry_nodes: - self.entry_point = True + # self.nodes_destinations = self._get_destinations_input_source(sinabs_edges) ####################################################### Public Methods ####################################################### + # TODO: Still needed? def get_destination_dcnnl_index(self, dcnnl_id: int) -> int: """The `forward` method will return as many tensors as there are elements in `self.dynapcnnlayer_destination`. Since the i-th returned tensor is to be fed to the i-th destionation in `self.dynapcnnlayer_destination`, the return of this method can be used to index a tensor returned in the `forward` method. @@ -101,6 +85,7 @@ def __str__(self): ####################################################### Private Methods ####################################################### + # TODO: Still needed? def _get_destinations_input_source(self, sinabs_edges: list) -> dict: """Creates a mapping between each layer in this `DynapcnnLayer` instance and its targe nodes that are part of different `DynapcnnLayer` instances. This mapping is used to figure out how many tensors the `forward` method needs to return. @@ -138,6 +123,7 @@ def _get_destinations_input_source(self, sinabs_edges: list) -> dict: return destinations_input_source + # TODO: Still needed? @staticmethod def find_nodes_core_id(node: int, all_handlers: dict) -> int: """Loops through all handlers in `all_handlers` to find to which core a `DynapcnnLayer` containing diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index 7faa4562..b310e1f1 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -21,7 +21,7 @@ from .utils import ( DEFAULT_IGNORED_LAYER_TYPES, Edge, - build_from_graph, + construct_dynapcnnlayers_from_mapper, parse_device_id, topological_sorting, ) @@ -64,7 +64,7 @@ def __init__( - self._graph_extractor - self._sinabs_edges - self._sinabs_indx_2_module_map - - self._nodes_to_dcnnl_map + - self._dcnnl_map - self._dynapcnn_layers """ super().__init__() @@ -86,24 +86,20 @@ def __init__( self._graph_extractor.remove_nodes_by_class(DEFAULT_IGNORED_LAYER_TYPES) # create a dict holding the data necessary to instantiate a `DynapcnnLayer`. - self._nodes_to_dcnnl_map = collect_dynapcnn_layer_info( + self._dcnnl_map = collect_dynapcnn_layer_info( self._graph_extractor.indx_2_module_map, self._graph_extractor.edges, self._graph_extractor.nodes_io_shapes, + self._graph_extractor.entry_nodes, ) - # TODO: Try avoiding this step by only retrieving input shapes of conv layers - # while constractung dcnnl_map - # # updates 'self._nodes_to_dcnnl_map' to include the I/O shape for each node. - # self._populate_nodes_io() - - # build `DynapcnnLayer` instances from graph edges and mapper. - self._dynapcnn_layers, self._dynapcnnlayers_handlers = build_from_graph( - discretize=discretize, - edges=self._graph_extractor.edges, - nodes_to_dcnnl_map=self._nodes_to_dcnnl_map, - weight_rescaling_fn=weight_rescaling_fn, - entry_nodes=self._graph_extractor._entry_nodes, + # build `DynapcnnLayer` instances from mapper. + self._dynapcnn_layers, self._dynapcnnlayers_handlers = ( + construct_dynapcnnlayers_from_mapper( + dcnnl_map=self._dcnnl_map, + discretize=discretize, + rescale_fn=weight_rescaling_fn, + ) ) # these gather all data necessay to implement the forward method for this class. @@ -118,7 +114,7 @@ def __init__( del self._graph_extractor del self._sinabs_edges del self._sinabs_indx_2_module_map - del self._nodes_to_dcnnl_map + del self._dcnnl_map del self._dynapcnn_layers ####################################################### Public Methods ####################################################### @@ -589,63 +585,6 @@ def _get_dynapcnnlayers_edges(self) -> List[Edge]: return dcnnl_edges - def _populate_nodes_io(self): - """Loops through the nodes in the original graph to retrieve their I/O tensor shapes and add them to their respective - representations in `self._nodes_to_dcnnl_map`.""" - - def find_my_input(edges_list: list, node: int) -> int: - """Returns the node `X` in the first edge `(X, node)`. - - Parameters - ---------- - - node (int): the node in the computational graph for which we whish to find the input source (either another node in the - graph or the original input itself to the network). - - Returns - ---------- - - input source (int): this indicates the node in the computational graph providing the input to `node`. If `node` is - receiving outside input (i.e., it is a starting node) the return will be -1. For example, this will be the case - when a network with two independent branches (each starts from a different "input node") merge along the computational graph. - """ - for edge in edges_list: - if edge[1] == node: - # TODO nodes originally receiving input from merge will appear twice in the list of edges, one - # edge per input to the merge layer. For now both inputs to a `Merge` have the same dimensions - # necessarily so this works for now but later will have to be revised. - return edge[0] - return -1 - - # access the I/O shapes for each node in `self._sinabs_edges` from the original graph in `self._graph_extractor`. - for dcnnl_idx, dcnnl_data in self._nodes_to_dcnnl_map.items(): - for node, node_data in dcnnl_data.items(): - # node dictionary with layer data. - if isinstance(node, int): - _in, _out = self._graph_extractor.get_node_io_shapes(node) - - # update node I/O shape in the mapper (drop batch dimension). - if node != 0: - # Find node outputing into the current node being processed (this will be the input shape). This is - # necessary cuz if a node originally receives input from a `nn.Flatten` for instance, when mapped into - # a `DynapcnnLayer` it will be receiving the input from a privious `sl.SumPool2d`. - input_node = find_my_input(self._sinabs_edges, node) - - if input_node == -1: - # node does not have an input source within the graph (it consumes the original input to the model). - node_data["input_shape"] = tuple(list(_in)[1:]) - else: - # input comes from another node in the graph. - _, _input_source_shape = ( - self._graph_extractor.get_node_io_shapes(input_node) - ) - node_data["input_shape"] = tuple( - list(_input_source_shape)[1:] - ) - else: - # first node does not have an input source within the graph. - node_data["input_shape"] = tuple(list(_in)[1:]) - - node_data["output_shape"] = tuple(list(_out)[1:]) - def _to_device(self, device: torch.device) -> None: """Access each sub-layer within all `DynapcnnLayer` instances and call `.to(device)` on them.""" for layer in self._layers_mapper.values(): diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index bb0583b2..77e9154b 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -19,6 +19,7 @@ def collect_dynapcnn_layer_info( indx_2_module_map: Dict[int, nn.Module], edges: Set[Edge], nodes_io_shapes: Dict[int, Dict[str, Tuple[Size, Size]]], + entry_nodes: Set[int], ) -> Dict[int, Dict]: """Collect information to construct DynapcnnLayer instances. @@ -34,6 +35,7 @@ def collect_dynapcnn_layer_info( indx_2_module_map (dict): Maps node IDs of the graph as `key` to their associated module as `value` edges (set of tuples): Represent connections between two nodes in computational graph nodes_io_shapes (dict): Map from node ID to dict containing node's in- and output shapes + entry_nodes (set of int): IDs of nodes that receive external input Returns ------- @@ -75,6 +77,7 @@ def collect_dynapcnn_layer_info( indx_2_module_map, nodes_io_shapes, node_2_layer_map, + entry_nodes, ) # "pooling-pooling" edges are optional. Unlike other types, missing entry would cause exception. @@ -151,6 +154,7 @@ def init_new_dynapcnnlayer_entry( indx_2_module_map: Dict[int, nn.Module], nodes_io_shapes: Dict[int, Dict[str, Tuple[Size, Size]]], node_2_layer_map: Dict[int, int], + entry_nodes: Set[int], ) -> None: """Initiate dict to hold information for new dynapcnn layer based on a "weight->neuron" edge. Change `dynapcnn_layer_info` in-place. @@ -166,6 +170,7 @@ def init_new_dynapcnnlayer_entry( nodes_io_shapes (dict): Map from node ID to dict containing node's in- and output shapes node_2_layer_map (dict): Maps each node ID to the ID of the layer it is assigned to. Will be updated in-place. + entry_nodes (set of int): IDs of nodes that receive external input """ # Make sure there are no existing entries holding any of the modules connected by `edge` assert edge[0] not in node_2_layer_map @@ -190,6 +195,7 @@ def init_new_dynapcnnlayer_entry( }, # This will be used later to account for average pooling in preceding layers "rescale_factors": set(), + "is_entry_node": edge[0] in entry_nodes, } node_2_layer_map[edge[0]] = layer_id node_2_layer_map[edge[1]] = layer_id diff --git a/sinabs/backend/dynapcnn/utils.py b/sinabs/backend/dynapcnn/utils.py index e9922338..5dec8e5c 100644 --- a/sinabs/backend/dynapcnn/utils.py +++ b/sinabs/backend/dynapcnn/utils.py @@ -12,7 +12,6 @@ Tuple, Union, ) -from warnings import warn import torch import torch.nn as nn @@ -23,7 +22,6 @@ from .dvs_layer import DVSLayer, expand_to_pair from .dynapcnn_layer import DynapcnnLayer from .dynapcnn_layer_handler import DynapcnnLayerHandler -from .exceptions import WrongPoolingModule from .flipdims import FlipDims if TYPE_CHECKING: @@ -92,289 +90,94 @@ def standardize_device_id(device_id: str) -> str: ####################################################### DynapcnnNetwork Related ####################################################### -def build_from_graph( - discretize: bool, - edges: List[Tuple[int, int]], - nodes_to_dcnnl_map: dict, - weight_rescaling_fn: Callable, - entry_nodes: List[int], -) -> Union[ - Dict[int, Dict[DynapcnnLayer, List]], Dict[int, Dict[DynapcnnLayerHandler, List]] -]: - """Parses each edge in `edges`, where each node is a set of layer that will compose a `DynapcnnLayer`. The - target destination of each `DynapcnnLayer` is computed via edges connecting nodes in different `DynapcnnLayer` - instances. - - Parameters - ---------- - - discretize (bool): if `True` the weights of all convolutional layers are discretized. - - edges (list): edges describing how nodes connect to each other. - - nodes_to_dcnnl_map (dict): each entry represents the gathered data necessary to instantiate a `DynapcnnLayer` object (e.g. nodes, - their I/O shapes, the list of `DynapcnnLayer` that are to be targeted, etc). - - weight_rescaling_fn (callable): a method that handles how the re-scaling factor for one or more `SumPool2d` projecting to - the same convolutional layer are combined/re-scaled before applying them. - - entry_nodes (list): node IDs corresponding to layers in the original network that are input nodes (i.e., a "point of entry" for the external data). - - Returns - ---------- - - dynapcnn_layers (dict): `DynapcnnLayer` instances, each created from an entry in `nodes_to_dcnnl_map`. - - dynapcnnlayers_handlers (dict): `DynapcnnLayerHandler` instances, gathering network-level info. for each of the `DynapcnnLayer` instances in `dynapcnn_layers`. - """ - - # turn each entry in `nodes_to_dcnnl_map` into a `DynapcnnLayer` instance. - dynapcnn_layers, dynapcnnlayers_handlers = construct_dynapcnnlayers_from_mapper( - discretize, nodes_to_dcnnl_map, edges, weight_rescaling_fn, entry_nodes - ) - - # initialize key holding to which core a `DynapcnnLayer` instance in `dynapcnn_layers` will be mapped to. - for idx, layer_data in dynapcnnlayers_handlers.items(): - if "core_idx" not in layer_data: - # a `DynapcnnLayer` gets assigned a core index when `DynapcnnNetworkGraph.to()`` is called. - layer_data["core_idx"] = -1 - - return dynapcnn_layers, dynapcnnlayers_handlers - - def construct_dynapcnnlayers_from_mapper( - discretize: bool, - nodes_to_dcnnl_map: dict, - edges: List[Tuple[int, int]], - weight_rescaling_fn: Callable, - entry_nodes: List[int], -) -> Union[ - Dict[int, Dict[DynapcnnLayer, List]], Dict[int, Dict[DynapcnnLayerHandler, List]] -]: - """Consumes a dictionaries containing sets of layers to be used to populate a DynapcnnLayer object. + dcnnl_map: Dict, discretize: bool, rescale_fn: Optional[Callable] = None +) -> Tuple[Dict[int, DynapcnnLayer], Dict[int, DynapcnnLayerHandler]]: + """Construct DynapcnnLayer and DynapcnnLayerHandler instances from + `dcnnl_map` - Parameters - ---------- - - discretize (bool): if `True` the weights of all convolutional layers are discretized. - - nodes_to_dcnnl_map (dict): each entry represents the gathered data necessary to instantiate a `DynapcnnLayer` object (e.g. nodes, - their I/O shapes, the list of `DynapcnnLayer` that are to be targeted, etc). - - edges (list): edges describing how nodes connect to each other. - - weight_rescaling_fn (callable): a method that handles how the re-scaling factor for one or more `SumPool2d` projecting to - the same convolutional layer are combined/re-scaled before applying them. - - entry_nodes (list): node IDs corresponding to layers in the original network that are input nodes (i.e., a "point of entry" for the external data). + Paramters + --------- Returns - ---------- - - dynapcnn_layers (dict): `DynapcnnLayer` instances, each created from an entry in `nodes_to_dcnnl_map`. - - dynapcnnlayers_handlers (dict): `DynapcnnLayerHandler` instances, gathering network-level info. for each of the `DynapcnnLayer` instances in `dynapcnn_layers`. + ------- + - Dict of new DynapcnnLayer instances, with keys corresponding to `dcnnl_map` + - Dict of new DynapcnnLayerHandler instances, with keys corresponding + to `dcnnl_map` """ + finalize_dcnnl_map(dcnnl_map, rescale_fn) - dynapcnn_layers = {} - dynapcnnlayers_handlers = {} - - for dpcnnl_idx, dcnnl_data in nodes_to_dcnnl_map.items(): - # create a `DynapcnnLayerHandler` from the set of layers in `dcnnl_data` - this holds network-level data required to instantiate a `DynapcnnLayer`. - layerhandler = construct_layerhandler( - dpcnnl_idx, - discretize, - edges, - nodes_to_dcnnl_map, - weight_rescaling_fn, - entry_nodes, - ) - - # create a `DynapcnnLayer` from the handler. - dynapcnnlayer = construct_dynapcnnlayer(layerhandler) - - # holds the layers themselvs. - dynapcnn_layers[dpcnnl_idx] = { - "layer": dynapcnnlayer, - "destinations": nodes_to_dcnnl_map[dpcnnl_idx]["destinations"], - } - - # holds the handlers of each layer for later use (e.g., creation of the forward pass for the `DynapcnnNetwork`). - dynapcnnlayers_handlers[dpcnnl_idx] = { - "layer_handler": layerhandler, - "destinations": nodes_to_dcnnl_map[dpcnnl_idx]["destinations"], - } + dynapcnn_layers = { + layer_idx: construct_single_dynapcnn_layer(layer_info, discretize, rescale_fn) + for layer_idx, layer_info in dcnnl_map.items() + } - # check if a `nn.Linear` in `dynapcnnlayer` has been turned into a `nn.Conv2d`. - node, output_shape = layerhandler.get_modified_node_io(dcnnl_data) + dynapcnn_layer_handlers = { + layer_idx: construct_single_dynapcnn_layer_handler(layer_idx, layer_info) + for layer_idx, layer_info in dcnnl_map.items() + } - if isinstance(node, int) and isinstance(output_shape, tuple): - # a `nn.Linear` has been converted into a `nn.Conv2d`: update input shape of nodes receiving from the spiking layer after it. - update_nodes_io(node, output_shape, nodes_to_dcnnl_map, edges) + return dynapcnn_layers, dynapcnn_layer_handlers - return dynapcnn_layers, dynapcnnlayers_handlers +def finalize_dcnnl_map(dcnnl_map: Dict, rescale_fn: Optional[Callable] = None): + """Finalize dcnnl map by consolidating information -def update_nodes_io( - updated_node: int, - output_shape: tuple, - nodes_to_dcnnl_map: dict, - edges: List[Tuple[int, int]], -) -> None: - """Updates the `input_shape` entries of each node in `nodes_to_dcnnl_map` receiving as input the output of the spiking - layer `updated_node` that had its I/O shapes updated following a `nn.Linear` to `nn.Conv2d` conversion. + Update dcnnl_map in-place + - Consolidate chained pooling layers + - Determine rescaling of layer weights + - Fix input shapes Parameters ---------- - - updated_node (int): the ID of the spiking layer that had its I/O shapes updated following a `nn.Linear` to `nn.Conv2d` conversion. - - output_shape (tuple): the updated shape of the spiking layer with node ID `updated_node`. - - nodes_to_dcnnl_map (dict): each entry represents the gathered data necessary to instantiate a `DynapcnnLayer` object (e.g. nodes, - their I/O shapes, the list of `DynapcnnLayer` that are to be targeted, etc). - - edges (list): edges describing how nodes connect to each other. + - dcnnl_map: Dict holding info needed to instantiate DynapcnnLayer + and DynapcnnLayerHandler instances + - rescale_fn: Optional callable that is used to determine layer + rescaling in case of conflicting preceeding average pooling """ - - for edge in edges: - if edge[0] == updated_node: - # found source node where output shape has been modified. - - # accessing every single node ID within the set of layers composing each `DynapcnnLayer` instance. - for _, dcnnl_data in nodes_to_dcnnl_map.items(): - for key, val in dcnnl_data.items(): - if isinstance(key, int): - # accessing node data (`layer`, `input_shape` and `output_shape`). - if key == edge[1]: - # accessing node targeted by `updated_node` (its input shape becomes `updated_node.output_shape`). - val["input_shape"] = output_shape - - -def construct_layerhandler( - dpcnnl_idx: int, - discretize: bool, - edges: List[Tuple[int, int]], - nodes_to_dcnnl_map: Dict[ - int, - Dict[ - Union[int, str], - Union[Dict[str, Union[nn.Module, Tuple[int, int, int]]], List[int]], - ], - ], - weight_rescaling_fn: Callable, - entry_nodes: List[int], -) -> DynapcnnLayerHandler: - """Extract the modules (layers) in a dictionary and uses them to instantiate a `DynapcnnLayerHandler` object. - - Parameters - ---------- - - dpcnnl_idx (int): the index/ID that will be associated with a `DynapcnnLayerHandler` instance. This integer indexes a `dict` within `nodes_to_dcnnl_map` - containing the data required to create the instance returned by this function. - - discretize (bool): whether or not the weights/neuron parameters of the model will be quantized. - - edges (list): each `nn.Module` within `nodes_to_dcnnl_map[dpcnnl_idx]` is a node in the original computational graph describing a spiking network - being converted to a `DynapcnnNetwork`. An edge `(A, B)` describes how modules forward data amongst themselves. This list is used by a `DynapcnnLayerHandler` - to figure out the number and sequence of output tesnors its forward method needs to return. - - nodes_to_dcnnl_map (dict): contains all layers (`nn.Module`) in the original spiking network grouped into dictionaries gathering the data necessary - to instantiate a `DynapcnnLayerHandler`. A `nodes_to_dcnnl_map[dpcnnl_idx]` will contain `int` keys (whose value corresponds to a `dict` with a `nn.Module` - instance and its associated I/O shapes, i.e., one layer within the `DynapcnnLayerHandler` instance) or `str` keys (whose values correspond to a list of - integers corresponding to either destinations IDs or re-scaling factors). - - weight_rescaling_fn (callable): a method that handles how the re-scaling factor for one or more `SumPool2d` projecting to - the same convolutional layer are combined/re-scaled before being applied. - - entry_nodes (list): node IDs corresponding to layers in the original network that are input nodes (i.e., a "point of entry" for the external data). - - Returns - ---------- - - layerhandler (DynapcnnLayerHandler): the a `DynapcnnLayer` instance made up by all the layers (`nn.Module`) in `dcnnl_data`. - """ - - # convert all AvgPool2d in 'dcnnl_data' into SumPool2d. - convert_Avg_to_Sum_pooling( - nodes_to_dcnnl_map[dpcnnl_idx], edges, nodes_to_dcnnl_map - ) - - # instantiate a DynapcnnLayer from the data in 'dcnnl_data'. - layerhandler = DynapcnnLayerHandler( - dpcnnl_index=dpcnnl_idx, - dcnnl_data=nodes_to_dcnnl_map[dpcnnl_idx], - discretize=discretize, - sinabs_edges=edges, - weight_rescaling_fn=weight_rescaling_fn, - entry_nodes=entry_nodes, - ) - - return layerhandler - - -def construct_all_dynapcnnlayers( - dcnnl_map: Dict, discretize: bool, rescale_fn: Optional[Callable] = None -) -> Dict[int, DynapcnnLayer]: - """...""" - # Consolidate pooling information for each destination for layer_info in dcnnl_map.values(): - for destination in layer_info["destinations"]: - pool, scale = consolidate_pooling(destination["pooling_modules"]) - destination["cumulative_pooling"] = pool - destination["cumulative_scaling"] = scale - dest_lyr_idx = destination["destination_layer"] - dcnnl_map[dest_lyr_idx]["rescale_factors"].add(scale) - - dynapcnn_layer = { - layer_idx: construct_single_dynapcnn_layer(layer_info, discretize, rescale_fn) - for layer_idx, layer_info in dcnnl_map.items() - } - - dynapcnn_layer_handler = { - layer_idx: construct_single_dynapcnn_layer_handler(layer_info) - for layer_idx, layer_info in dcnnl_map.items() - } - - -def construct_single_dynapcnn_layer( - layer_info: Dict, discretize: bool, rescale_fn: Optional[Callable] = None -) -> DynapcnnLayer: - - # Handle rescaling after average pooling - if len(layer_info["rescale_factors"]) == 0: - rescale_factor = 1 - elif len(layer_info["rescale_factors"]) == 1: - rescale_factor = layer_info["rescale_factors"].pop() - else: - if rescale_fn is None: - # TODO: Custom Exception class? - raise ValueError( - "Average pooling layers of conflicting sizes pointing to " - "same destination. Either replace them by SumPool2d layers " - "or provide a `rescale_fn` to resolve this" - ) - else: - rescale_factor = rescale_fn(layer_info["rescale_factors"]) - - # Handle input dimensions - # For each dimension find larges inferred input size - max_inferred_input_shape = [ - max(sizes) for sizes in zip(layer_info["inferred_input_shapes"]) - ] + consolidate_layer_pooling(layer_info, dcnnl_map) - if isinstance(layer_info["conv"]["module"], nn.Linear): - if prod(max_inferred_input_shape) > prod(layer_info["input_shape"]): - raise ValueError( - "Combined output of some layers projecting to a linear layer is " - "larger than expected by destination layer. " - ) - # Take shape before flattening, to convert linear to conv layer - in_shape = max_inferred_input_shape - else: - if any( - inferred > expected - for inferred, expected in zip( - max_inferred_input_shape, layer_info["input_shape"] - ) - ): - raise ValueError( - "Output of some layers is larger than expected by destination " - "layer along some dimensions." - ) - in_shape = layer_info["input_shape"] + for layer_info in dcnnl_map.values(): + # Consolidate scale factors + consolidate_layer_scaling(layer_info, rescale_fn) + # Handle input dimensions + determine_layer_input_shape(layer_info) - # Collect pooling in a list - pooling_list = [dest["cumulative_pooling"] for dest in layer_info["destinations"]] - # instantiate a DynapcnnLayer from the data in the handler. - return DynapcnnLayer( - conv=layer_info["conv"]["module"], - spk=layer_info["neuron"]["module"], - in_shape=in_shape, - pool=pooling_list, - discretize=discretize, - rescale_weights=rescale_factor, - ) +def consolidate_layer_pooling(layer_info: Dict, dcnnl_map: Dict): + """Consolidate pooling information for individual layer + Update `layer_info` and `dcnnl_map` in place. + - Extract pooling and scale factor of consecutive pooling operations + - To each "destination" add entries "cumulative_pooling" and + "cumulative_scaling" + - Add "pooling_list" to `layer_info` with all poolings of a layer + in order of its "destination"s. + - For each destination, add cumulative rescale factor to "rescale_factors" + entry in corresponding entry of `dcnnl_map`. -def consolidate_pooling(modules: Iterable[nn.Module]) -> Tuple[Tuple[int, int], float]: + Parameters + ---------- + - layer_info: Dict holding info of single layer. Corresponds to + single entry in `dcnnl_map` + - dcnnl_map: Dict holding info needed to instantiate DynapcnnLayer + and DynapcnnLayerHandler instances + """ + layer_info["pooling_list"] = [] + for destination in layer_info["destinations"]: + pool, scale = consolidate_dest_pooling(destination["pooling_modules"]) + destination["cumulative_pooling"] = pool + layer_info["pooling_list"].append(pool) + destination["cumulative_scaling"] = scale + dest_lyr_idx = destination["destination_layer"] + dcnnl_map[dest_lyr_idx]["rescale_factors"].add(scale) + + +def consolidate_dest_pooling( + modules: Iterable[nn.Module], +) -> Tuple[Tuple[int, int], float]: """Consolidate pooling information for consecutive pooling modules. Parameters @@ -434,124 +237,131 @@ def extract_pooling_from_module( return pooling, scale_factor -def convert_Avg_to_Sum_pooling( - dcnnl_data: Dict[ - Union[int, str], - Union[Dict[str, Union[nn.Module, Tuple[int, int, int]]], List[int]], - ], - edges: List[Tuple[int, int]], - nodes_to_dcnnl_map: Dict[ - int, - Dict[ - Union[int, str], - Union[Dict[str, Union[nn.Module, Tuple[int, int, int]]], List[int]], - ], - ], -) -> None: - """Converts every `AvgPool2d` node within `dcnnl_data` into a `SumPool2d` and update their respective `rescale_factor` (to - be used when creating the `DynapcnnLayer` instance for this layer's destinations). +def consolidate_layer_scaling(layer_info: Dict, rescale_fn: Optional[Callable] = None): + """Dertermine scale factor of single layer + + Add "rescale_factor" entry to `layer_info`. If more than one + different rescale factors have been determined due to conflicting + average pooling in preceding layers, requrie `rescale_fn` to + resolve. Parameters ---------- - - dcnnl_data (dict): contains the nodes to be merged into a `DynapcnnLayer`, their I/O shapes and the index of the other `DynapcnnLayer`s to - be set as destinations. The `int` keys correspond to the nodes IDs associated `nn.Module`s (a single layer in the original network) becoming - part of a single `DynapcnnLayer` instance, while the `str` keys correspond to the instance's destinations and re-scaling factors. - - edges (list): each node is a `nn.Module` in the original computational graph describing a spiking network being converted to a `DynapcnnNetwork`. The - list is used to find the targets of a `SumPool2d` (part of the `DynapcnnLayer` instance being created) and update the re-scaling factor they will - require. - - nodes_to_dcnnl_map (dict): contains all layers (`nn.Module`) in the original spiking network grouped into dictionaries gathering the data necessary - to instantiate a `DynapcnnLayer`. A `nodes_to_dcnnl_map[dpcnnl_idx]` will contain `int` keys (whose value corresponds to a `dict` with a `nn.Module` - instance and its associated I/O shapes, i.e., one layer within the `DynapcnnLayer` instance) or `str` keys (whose values correspond to a list of - integers corresponding to either destinations IDs or re-scaling factors). + - layer_info: Dict holding info of single layer. + - rescale_fn: Optional callable that is used to determine layer + rescaling in case of conflicting preceeding average pooling """ - for key, value in dcnnl_data.items(): - if isinstance(key, int): - # accessing the node `key` dictionary. - - if isinstance(value["layer"], nn.AvgPool2d): - # convert AvgPool2d into SumPool2d. - lyr_pool, rescale_factor = build_SumPool2d(value["layer"]) - - # turn avg into sum pool. - value["layer"] = lyr_pool - - # find which node `key` will target. - for edge in edges: - if edge[0] == key: - # find index of `DynapcnnLayer` where the target of `edge[0]` is. - trg_dcnnl_idx = find_nodes_dcnnl_idx( - edge[1], nodes_to_dcnnl_map - ) - - # update the rescale factor for the target of node `key`. - nodes_to_dcnnl_map[trg_dcnnl_idx]["conv_rescale_factor"].append( - rescale_factor - ) - - -def find_nodes_dcnnl_idx(node: int, nodes_to_dcnnl_map: dict) -> int: - """Find the ID of the (future) `DynapcnnLayer` instance to which `node` belongs to.""" - - # looping over sets of layers (nodes) that will be used to instantiate `DynapcnnLayer`s. - for dcnnl_idx, dcnnl_data in nodes_to_dcnnl_map.items(): - for key, value in dcnnl_data.items(): - if isinstance(key, int): - # `key` is a node. - if key == node: - # node belongs to DynapcnnLayer index `dcnnl_idx`. - return dcnnl_idx - - # this exception should never happen. - raise ValueError( - f"Node {node} is not part of any dictionary mapping into a DynapcnnLayer." - ) + if len(layer_info["rescale_factors"]) == 0: + rescale_factor = 1 + elif len(layer_info["rescale_factors"]) == 1: + rescale_factor = layer_info["rescale_factors"].pop() + else: + if rescale_fn is None: + # TODO: Custom Exception class? + raise ValueError( + "Average pooling layers of conflicting sizes pointing to " + "same destination. Either replace them by SumPool2d layers " + "or provide a `rescale_fn` to resolve this" + ) + else: + rescale_factor = rescale_fn(layer_info["rescale_factors"]) + layer_info["rescale_factor"] = rescale_factor -def build_SumPool2d(module: nn.AvgPool2d) -> Tuple[sl.SumPool2d, int]: - """Converts a `nn.AvgPool2d` into a `sl.SumPool2d` layer. +def determine_layer_input_shape(layer_info: Dict): + """Determine input shape of single layer - Parameters - ---------- - - module (torch.nn.AvgPool2d): the average pooling layer being converted into a sum pooling layer. + Update "input_shape" entry of `layer_info`. + If weight layer is convolutional, only verify that output shapes + of preceding layer are not greater than input shape in any dimension. - Returns + If weight layer is linear, the current "input_shape" entry will + correspond to the shape after flattening, which might not match + the shape of the actual input to the layer. Therefore the new input + shape is the largest size across all output shapes of preceding + layers, for each dimension individually. + Verify that total number of elements (product of entries in new + input shape) does not exceed that of original input shape. + + Parameters ---------- - - lyr_pool (sinabs.layers.SumPool2d): the equivalent sum pooling layer. - rescale_factor (int): the weight re-scaling computed for the weights of the convolution layer targeted by the pooling. + - layer_info: Dict holding info of single layer. """ + # For each dimension find largest inferred input size + max_inferred_input_shape = [ + max(sizes) for sizes in zip(layer_info["inferred_input_shapes"]) + ] - if isinstance(module, nn.AvgPool2d): - if module.padding != 0: - raise ValueError("Padding is not supported for the pooling layers.") - elif isinstance(module, sl.SumPool2d): - pass + if isinstance(layer_info["conv"]["module"], nn.Linear): + if prod(max_inferred_input_shape) > prod(layer_info["input_shape"]): + raise ValueError( + "Combined output of some layers projecting to a linear layer is " + "larger than expected by destination layer. " + ) + # Take shape before flattening, to convert linear to conv layer + layer_info["input_shape"] = max_inferred_input_shape else: - raise WrongPoolingModule(type(module)) - - rescale_factor = 1 - cumulative_pooling = expand_to_pair(1) - pooling = expand_to_pair(module.kernel_size) - - if module.stride is not None: - stride = expand_to_pair(module.stride) - if pooling != stride: + if any( + inferred > expected + for inferred, expected in zip( + max_inferred_input_shape, layer_info["input_shape"] + ) + ): raise ValueError( - f"Stride length {module.stride} should be the same as pooling kernel size {module.kernel_size}" + "Output of some layers is larger than expected by destination " + "layer along some dimensions." ) - # compute cumulative pooling. - cumulative_pooling = ( - cumulative_pooling[0] * pooling[0], - cumulative_pooling[1] * pooling[1], + +def construct_single_dynapcnn_layer( + layer_info: Dict, discretize: bool +) -> DynapcnnLayer: + """Instantiate a DynapcnnLayer instance from the information + in `layer_info' + + Parameters + ---------- + - layer_info: Dict holding info of single layer. + - discretize: bool indicating whether layer parameters should be + discretized (weights, biases, thresholds) + + Returns + ------- + """ + return DynapcnnLayer( + conv=layer_info["conv"]["module"], + spk=layer_info["neuron"]["module"], + in_shape=layer_info["input_shape"], + pool=layer_info["pooling_list"], + discretize=discretize, + rescale_weights=layer_info["rescale_factor"], ) - if isinstance(module, nn.AvgPool2d): - # update rescaling factor. - rescale_factor *= pooling[0] * pooling[1] - lyr_pool = sl.SumPool2d(cumulative_pooling) +def construct_single_dynapcnn_layer_handler( + layer_index: int, layer_info: Dict +) -> DynapcnnLayerHandler: + """Instantiate a DynapcnnLayerHandler instance from the + information in `layer_info' + + Parameters + ---------- + - layer_index: Global index of the layer + - layer_info: Dict holding info of single layer. - return lyr_pool, rescale_factor + Returns + ------- + New DynapcnnLayerHandler instance + """ + destination_indices = [ + dest["destination_layer"] for dest in layer_info["destinations"] + ] + return DynapcnnLayerHandler( + layer_index=layer_index, + is_entry_node=layer_info["is_entry_node"], + destination_indices=destination_indices, + assigned_core=None, + ) def topological_sorting(edges: Set[Tuple[int, int]]) -> List[int]: From 95e6c6d4d6e18fcecc315040d2be38e3dca6fd75 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Wed, 9 Oct 2024 11:29:50 +0200 Subject: [PATCH 191/379] Bugfix: layer_info always has "destinations" entry --- .../backend/dynapcnn/sinabs_edges_handler.py | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index 77e9154b..3ee85020 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -196,6 +196,8 @@ def init_new_dynapcnnlayer_entry( # This will be used later to account for average pooling in preceding layers "rescale_factors": set(), "is_entry_node": edge[0] in entry_nodes, + # Will be populated by `set_[pooling/neuron]_layer_destination` + "destinations": [], } node_2_layer_map[edge[0]] = layer_id node_2_layer_map[edge[1]] = layer_id @@ -234,15 +236,11 @@ def add_pooling_to_entry( # Make sure all pooling chains start with expected node assert all(chain[0] == edge[1] for chain in pooling_chains) - # Layer entry might already have `destinations` key (if neuron layer has fanout > 1) - layer_info = dynapcnn_layer_info[layer_idx] - if "destinations" not in layer_info: - layer_info["destinations"] = [] - # Keep track of all nodes that have been added new_nodes = set() # For each pooling chain initialize new destination + layer_info = dynapcnn_layer_info[layer_idx] for chain in pooling_chains: layer_info["destinations"].append( { @@ -287,13 +285,9 @@ def set_neuron_layer_destination( except KeyError: raise UnmatchedNode(edge, edge[1]) - # Source layer entry might already have `destinations` key (if neuron layer has fanout > 1) - layer_info = dynapcnn_layer_info[source_layer_idx] - if "destinations" not in layer_info: - layer_info["destinations"] = [] - # Add new destination output_shape = nodes_io_shapes[edge[0]]["output"] + layer_info = dynapcnn_layer_info[source_layer_idx] layer_info["destinations"].append( { "pooling_ids": [], @@ -338,10 +332,8 @@ def set_pooling_layer_destination( except KeyError: raise UnmatchedNode(edge, edge[1]) - # Source layer entry should already have `destinations` key - layer_info = dynapcnn_layer_info[source_layer_idx] - # Find current source node within destinations + layer_info = dynapcnn_layer_info[source_layer_idx] matched = False for destination in layer_info["destinations"]: if destination["pooling_ids"][-1] == edge[0]: From 3cbd1d6c64bb9f9eb2ba35a8627e9a1398b51e38 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Wed, 9 Oct 2024 15:10:39 +0200 Subject: [PATCH 192/379] Fix bugs related to DynapcnnLayer refactoring --- sinabs/backend/dynapcnn/dynapcnn_layer.py | 4 +-- .../backend/dynapcnn/nir_graph_extractor.py | 30 ++++++++++++------- .../backend/dynapcnn/sinabs_edges_handler.py | 2 +- sinabs/backend/dynapcnn/utils.py | 9 +++--- .../dynapcnn/weight_rescaling_methods.py | 21 ++++++------- 5 files changed, 38 insertions(+), 28 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer.py b/sinabs/backend/dynapcnn/dynapcnn_layer.py index a7ad3562..498b65dd 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer.py @@ -104,8 +104,8 @@ def __init__( # Convert `nn.Linear` to `nn.Conv2d`. if isinstance(conv, nn.Linear): - conv = convert_linear_to_conv(conv) - if spk.is_state_initialised() and (ndim := spk.ndim) < 4: + conv = convert_linear_to_conv(conv, in_shape) + if spk.is_state_initialised() and (ndim := spk.v_mem.ndim) < 4: for __ in range(4 - ndim): # Expand spatial dimensions spk.v_mem = spk.v_mem.data.unsqueeze(-1) diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index 10b4b0e3..299e62df 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -297,24 +297,23 @@ def _get_nodes_io_shapes( if isinstance(self.indx_2_module_map[node], sinabs.layers.merge.Merge): # find `Merge` arguments (at this point the inputs to Merge should have been calculated). - arg1, arg2 = self._find_merge_arguments(node) + input_nodes = self._find_merge_arguments(node) # retrieve arguments output tensors. - arg1_out = nodes_io_map[arg1]["output"] - arg2_out = nodes_io_map[arg2]["output"] + inputs = [nodes_io_map[n]["output"] for n in input_nodes] - # TODO - this is currently a limitation imposed by the validation checks done by Speck once a configuration: it wants two + # TODO - this is currently a limitation imposed by the validation checks done by Speck once a configuration: it wants # different input sources to a core to have the same output shapes. - if arg1_out.shape != arg2_out.shape: + if any(inp.shape != inputs[0].shape for inp in inputs): raise ValueError( - f"Layer `sinabs.layers.merge.Merge` (node {node}) require two input tensors with the same shape: arg1.shape {arg1_out.shape} differs from arg2.shape {arg2_out.shape}." + f"Layer `sinabs.layers.merge.Merge` (node {node}) requires input tensors with the same shape" ) # forward input through the node. - _output = self.indx_2_module_map[node](arg1_out, arg2_out) + _output = self.indx_2_module_map[node](*inputs) # save node's I/O tensors. - nodes_io_map[node] = {"input": arg1_out, "output": _output} + nodes_io_map[node] = {"input": inputs[0], "output": _output} else: @@ -336,10 +335,19 @@ def _get_nodes_io_shapes( # save node's I/O tensors. nodes_io_map[node] = {"input": _input, "output": _output} - # replace the I/O tensor information by its shape information. + # replace the I/O tensor information by its shape information, ignoring the batch/time axis for node, io in nodes_io_map.items(): - nodes_io_map[node]["input"] = io["input"].shape - nodes_io_map[node]["output"] = io["output"].shape + input_shape = io["input"].shape[1:] + output_shape = io["output"].shape[1:] + # Linear layers have fewer in/out dimensions. Extend by appending 1's + if (length := len(input_shape)) < 3: + input_shape = (*input_shape, *(1 for __ in range(3 - length))) + assert len(input_shape) == 3 + if (length := len(output_shape)) < 3: + output_shape = (*output_shape, *(1 for __ in range(3 - length))) + assert len(output_shape) == 3 + nodes_io_map[node]["input"] = input_shape + nodes_io_map[node]["output"] = output_shape return nodes_io_map diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index 3ee85020..d4aec78d 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -181,7 +181,7 @@ def init_new_dynapcnnlayer_entry( assert layer_id not in dynapcnn_layer_info dynapcnn_layer_info[layer_id] = { - "input_shape": nodes_io_shapes[edge[0]], + "input_shape": nodes_io_shapes[edge[0]]["input"], # Collect output shapes (before possible flattening) of layers with this layer as their destination # This will allow infering shapes when converting linear to conv layers "inferred_input_shapes": set(), diff --git a/sinabs/backend/dynapcnn/utils.py b/sinabs/backend/dynapcnn/utils.py index 5dec8e5c..a7a2e066 100644 --- a/sinabs/backend/dynapcnn/utils.py +++ b/sinabs/backend/dynapcnn/utils.py @@ -108,7 +108,7 @@ def construct_dynapcnnlayers_from_mapper( finalize_dcnnl_map(dcnnl_map, rescale_fn) dynapcnn_layers = { - layer_idx: construct_single_dynapcnn_layer(layer_info, discretize, rescale_fn) + layer_idx: construct_single_dynapcnn_layer(layer_info, discretize) for layer_idx, layer_info in dcnnl_map.items() } @@ -178,7 +178,8 @@ def consolidate_layer_pooling(layer_info: Dict, dcnnl_map: Dict): def consolidate_dest_pooling( modules: Iterable[nn.Module], ) -> Tuple[Tuple[int, int], float]: - """Consolidate pooling information for consecutive pooling modules. + """Consolidate pooling information for consecutive pooling modules + for single destination. Parameters ---------- @@ -272,7 +273,7 @@ def determine_layer_input_shape(layer_info: Dict): """Determine input shape of single layer Update "input_shape" entry of `layer_info`. - If weight layer is convolutional, only verify that output shapes + If weight layer is convolutional, only verify that output shapes of preceding layer are not greater than input shape in any dimension. If weight layer is linear, the current "input_shape" entry will @@ -289,7 +290,7 @@ def determine_layer_input_shape(layer_info: Dict): """ # For each dimension find largest inferred input size max_inferred_input_shape = [ - max(sizes) for sizes in zip(layer_info["inferred_input_shapes"]) + max(sizes) for sizes in zip(*layer_info["inferred_input_shapes"]) ] if isinstance(layer_info["conv"]["module"], nn.Linear): diff --git a/sinabs/backend/dynapcnn/weight_rescaling_methods.py b/sinabs/backend/dynapcnn/weight_rescaling_methods.py index 154f0055..e08e915f 100644 --- a/sinabs/backend/dynapcnn/weight_rescaling_methods.py +++ b/sinabs/backend/dynapcnn/weight_rescaling_methods.py @@ -2,46 +2,47 @@ # contact : williansoaresgirao@gmail.com import statistics +from typing import Iterable import numpy as np -def rescale_method_1(rescaling_from_sumpool: list, lambda_: float = 0.5) -> float: +def rescale_method_1(scaling_factors: Iterable[int], lambda_: float = 0.5) -> float: """ This method will use the average (scaled by `lambda_`) of the computed re-scaling factor for the pooling layer(s) feeding into a convolutional layer. Arguments --------- - - rescaling_from_sumpool (list): the list of re-scaling factors computed by each `SumPool2d` layer targeting a + - scaling_factors (list): the list of re-scaling factors computed by each `SumPool2d` layer targeting a single `Conv2d` layer within a `DynapcnnLayer` instance. - lambda_ (float): a scaling variable that multiplies the computed average re-scaling factor of the pooling layers. Returns --------- - - the averaged re-scaling factor multiplied by `lambda_` if `len(rescaling_from_sumpool) > 0`, else `1` is returned. + - the averaged re-scaling factor multiplied by `lambda_` if `len(scaling_factors) > 0`, else `1` is returned. """ - if len(rescaling_from_sumpool): - return np.round(np.mean(rescaling_from_sumpool) * lambda_, 2) + if len(scaling_factors) > 0: + return np.round(np.mean(list(scaling_factors)) * lambda_, 2) else: return 1.0 -def rescale_method_2(rescaling_from_sumpool: list, lambda_: float = 0.5) -> float: +def rescale_method_2(scaling_factors: Iterable[int], lambda_: float = 0.5) -> float: """ This method will use the harmonic mean (scaled by `lambda_`) of the computed re-scaling factor for the pooling layer(s) feeding into a convolutional layer. Arguments --------- - - rescaling_from_sumpool (list): the list of re-scaling factors computed by each `SumPool2d` layer targeting a + - scaling_factors (list): the list of re-scaling factors computed by each `SumPool2d` layer targeting a single `Conv2d` layer within a `DynapcnnLayer` instance. - lambda_ (float): a scaling variable that multiplies the computed average re-scaling factor of the pooling layers. Returns --------- - - the averaged re-scaling factor multiplied by `lambda_` if `len(rescaling_from_sumpool) > 0`, else `1` is returned. + - the averaged re-scaling factor multiplied by `lambda_` if `len(scaling_factors) > 0`, else `1` is returned. Note --------- @@ -49,7 +50,7 @@ def rescale_method_2(rescaling_from_sumpool: list, lambda_: float = 0.5) -> floa for weight re-scaling when multiple poolings with big differentces in kernel sizes are being considered. """ - if len(rescaling_from_sumpool): - return np.round(statistics.harmonic_mean(rescaling_from_sumpool) * lambda_, 2) + if len(scaling_factors) > 0: + return np.round(statistics.harmonic_mean(list(scaling_factors)) * lambda_, 2) else: return 1.0 From c9ed7a2930d6932882430079b1f8864c30fd28f0 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Wed, 9 Oct 2024 15:49:59 +0200 Subject: [PATCH 193/379] (WIP): Update dynapcnn layer tests --- .../conftest_dynapcnnlayer.py | 60 +++-- tests/test_dynapcnnlayer/model_dummy_1.py | 246 +++++++++--------- .../test_dynapcnnlayer/test_dynapcnnlayer.py | 115 ++++---- tests/test_graph_extractor/model_dummy_1.py | 34 +-- tests/test_graph_extractor/model_dummy_2.py | 40 +-- tests/test_graph_extractor/model_dummy_3.py | 50 ++-- tests/test_graph_extractor/model_dummy_4.py | 40 +-- 7 files changed, 280 insertions(+), 305 deletions(-) diff --git a/tests/test_dynapcnnlayer/conftest_dynapcnnlayer.py b/tests/test_dynapcnnlayer/conftest_dynapcnnlayer.py index 29a405e4..39d0edc4 100644 --- a/tests/test_dynapcnnlayer/conftest_dynapcnnlayer.py +++ b/tests/test_dynapcnnlayer/conftest_dynapcnnlayer.py @@ -1,37 +1,35 @@ # author : Willian Soares Girao # contact : wsoaresgirao@gmail.com -from model_dummy_1 import expected_output_1, nodes_to_dcnnl_map_1, sinabs_edges_1 -from model_dummy_2 import expected_output_2, nodes_to_dcnnl_map_2, sinabs_edges_2 -from model_dummy_3 import expected_output_3, nodes_to_dcnnl_map_3, sinabs_edges_3 -from model_dummy_4 import expected_output_4, nodes_to_dcnnl_map_4, sinabs_edges_4 +from model_dummy_1 import expected_output_1, dcnnl_map_1 +# from model_dummy_2 import expected_output_2, nodes_to_dcnnl_map_2, sinabs_edges_2 +# from model_dummy_3 import expected_output_3, nodes_to_dcnnl_map_3, sinabs_edges_3 +# from model_dummy_4 import expected_output_4, nodes_to_dcnnl_map_4, sinabs_edges_4 +# Args: dcnnl_map, discretize, rescale_fn, expected_output args_DynapcnnLayer = [ - (nodes_to_dcnnl_map_1, 0, sinabs_edges_1, [0], expected_output_1), - (nodes_to_dcnnl_map_1, 1, sinabs_edges_1, [0], expected_output_1), - (nodes_to_dcnnl_map_1, 2, sinabs_edges_1, [0], expected_output_1), - (nodes_to_dcnnl_map_1, 3, sinabs_edges_1, [0], expected_output_1), - (nodes_to_dcnnl_map_1, 4, sinabs_edges_1, [0], expected_output_1), - (nodes_to_dcnnl_map_2, 0, sinabs_edges_2, [0], expected_output_2), - (nodes_to_dcnnl_map_2, 1, sinabs_edges_2, [0], expected_output_2), - (nodes_to_dcnnl_map_2, 2, sinabs_edges_2, [0], expected_output_2), - (nodes_to_dcnnl_map_2, 3, sinabs_edges_2, [0], expected_output_2), - (nodes_to_dcnnl_map_2, 4, sinabs_edges_2, [0], expected_output_2), - (nodes_to_dcnnl_map_2, 5, sinabs_edges_2, [0], expected_output_2), - (nodes_to_dcnnl_map_2, 6, sinabs_edges_2, [0], expected_output_2), - (nodes_to_dcnnl_map_3, 0, sinabs_edges_3, [0, 8], expected_output_3), - (nodes_to_dcnnl_map_3, 1, sinabs_edges_3, [0, 8], expected_output_3), - (nodes_to_dcnnl_map_3, 2, sinabs_edges_3, [0, 8], expected_output_3), - (nodes_to_dcnnl_map_3, 3, sinabs_edges_3, [0, 8], expected_output_3), - (nodes_to_dcnnl_map_3, 4, sinabs_edges_3, [0, 8], expected_output_3), - (nodes_to_dcnnl_map_3, 5, sinabs_edges_3, [0, 8], expected_output_3), - (nodes_to_dcnnl_map_3, 6, sinabs_edges_3, [0, 8], expected_output_3), - (nodes_to_dcnnl_map_3, 7, sinabs_edges_3, [0, 8], expected_output_3), - (nodes_to_dcnnl_map_3, 8, sinabs_edges_3, [0, 8], expected_output_3), - (nodes_to_dcnnl_map_4, 0, sinabs_edges_4, [0], expected_output_4), - (nodes_to_dcnnl_map_4, 1, sinabs_edges_4, [0], expected_output_4), - (nodes_to_dcnnl_map_4, 2, sinabs_edges_4, [0], expected_output_4), - (nodes_to_dcnnl_map_4, 3, sinabs_edges_4, [0], expected_output_4), - (nodes_to_dcnnl_map_4, 4, sinabs_edges_4, [0], expected_output_4), - (nodes_to_dcnnl_map_4, 5, sinabs_edges_4, [0], expected_output_4), + (dcnnl_map_1, True, None, expected_output_1), + (dcnnl_map_1, False, None, expected_output_1), + # (nodes_to_dcnnl_map_2, 0, sinabs_edges_2, [0], expected_output_2), + # (nodes_to_dcnnl_map_2, 1, sinabs_edges_2, [0], expected_output_2), + # (nodes_to_dcnnl_map_2, 2, sinabs_edges_2, [0], expected_output_2), + # (nodes_to_dcnnl_map_2, 3, sinabs_edges_2, [0], expected_output_2), + # (nodes_to_dcnnl_map_2, 4, sinabs_edges_2, [0], expected_output_2), + # (nodes_to_dcnnl_map_2, 5, sinabs_edges_2, [0], expected_output_2), + # (nodes_to_dcnnl_map_2, 6, sinabs_edges_2, [0], expected_output_2), + # (nodes_to_dcnnl_map_3, 0, sinabs_edges_3, [0, 8], expected_output_3), + # (nodes_to_dcnnl_map_3, 1, sinabs_edges_3, [0, 8], expected_output_3), + # (nodes_to_dcnnl_map_3, 2, sinabs_edges_3, [0, 8], expected_output_3), + # (nodes_to_dcnnl_map_3, 3, sinabs_edges_3, [0, 8], expected_output_3), + # (nodes_to_dcnnl_map_3, 4, sinabs_edges_3, [0, 8], expected_output_3), + # (nodes_to_dcnnl_map_3, 5, sinabs_edges_3, [0, 8], expected_output_3), + # (nodes_to_dcnnl_map_3, 6, sinabs_edges_3, [0, 8], expected_output_3), + # (nodes_to_dcnnl_map_3, 7, sinabs_edges_3, [0, 8], expected_output_3), + # (nodes_to_dcnnl_map_3, 8, sinabs_edges_3, [0, 8], expected_output_3), + # (nodes_to_dcnnl_map_4, 0, sinabs_edges_4, [0], expected_output_4), + # (nodes_to_dcnnl_map_4, 1, sinabs_edges_4, [0], expected_output_4), + # (nodes_to_dcnnl_map_4, 2, sinabs_edges_4, [0], expected_output_4), + # (nodes_to_dcnnl_map_4, 3, sinabs_edges_4, [0], expected_output_4), + # (nodes_to_dcnnl_map_4, 4, sinabs_edges_4, [0], expected_output_4), + # (nodes_to_dcnnl_map_4, 5, sinabs_edges_4, [0], expected_output_4), ] diff --git a/tests/test_dynapcnnlayer/model_dummy_1.py b/tests/test_dynapcnnlayer/model_dummy_1.py index 26f2bf91..504ad32a 100644 --- a/tests/test_dynapcnnlayer/model_dummy_1.py +++ b/tests/test_dynapcnnlayer/model_dummy_1.py @@ -5,190 +5,184 @@ import torch.nn as nn from sinabs.activation.surrogate_gradient_fn import PeriodicExponential -from sinabs.layers import IAFSqueeze, SumPool2d +from sinabs.layers import IAFSqueeze -nodes_to_dcnnl_map_1 = { +dcnnl_map_1 = { 0: { - 0: { - "layer": nn.Conv2d(2, 10, kernel_size=(2, 2), stride=(1, 1), bias=False), - "input_shape": (2, 34, 34), - "output_shape": (10, 33, 33), + "input_shape": (2, 34, 34), + "inferred_input_shapes": set(), + "rescale_factors": set(), + "is_entry_node": True, + "conv": { + "module": nn.Conv2d(2, 10, kernel_size=(2, 2), stride=[1, 1], bias=False), + "node_id": 0, }, - 1: { - "layer": IAFSqueeze( + "neuron": { + "module": IAFSqueeze( batch_size=3, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential(), ), - "input_shape": (10, 33, 33), - "output_shape": (10, 33, 33), + "node_id": 1, }, - 2: { - "layer": nn.AvgPool2d(kernel_size=3, stride=3, padding=0), - "input_shape": (10, 33, 33), - "output_shape": (10, 11, 11), - }, - 3: { - "layer": nn.AvgPool2d(kernel_size=4, stride=4, padding=0), - "input_shape": (10, 33, 33), - "output_shape": (10, 8, 8), - }, - "destinations": [1, 2], - "conv_rescale_factor": [], + "destinations": [ + { + "pooling_ids": [2], + "pooling_modules": [nn.AvgPool2d(kernel_size=3, stride=3, padding=0)], + "destination_layer": 1, + "output_shape": (10, 11, 11), + }, + { + "pooling_ids": [3], + "pooling_modules": [nn.AvgPool2d(kernel_size=4, stride=4, padding=0),], + "destination_layer": 2, + "output_shape": (10, 8, 8), + }, + ], }, 1: { - 4: { - "layer": nn.Conv2d(10, 10, kernel_size=(4, 4), stride=(1, 1), bias=False), - "input_shape": (10, 11, 11), - "output_shape": (10, 8, 8), + "input_shape": (10, 11, 11), + "inferred_input_shapes": set(((10, 11, 11),)), + "rescale_factors": set(), + "is_entry_node": False, + "conv": { + "module": nn.Conv2d(10, 10, kernel_size=(4, 4), stride=[1, 1], bias=False), + "node_id": 4, }, - 6: { - "layer": IAFSqueeze( + "neuron": { + "module": IAFSqueeze( batch_size=3, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential(), ), - "input_shape": (10, 8, 8), - "output_shape": (10, 8, 8), + "node_id": 6, }, - "destinations": [2], - "conv_rescale_factor": [9], + "destinations": [ + { + "pooling_ids": [], + "pooling_modules": [], + "destination_layer": 3, + "output_shape": (10, 7, 7), + }, + ], }, 2: { - 7: { - "layer": nn.Conv2d(10, 1, kernel_size=(2, 2), stride=(1, 1), bias=False), - "input_shape": (10, 8, 8), - "output_shape": (1, 7, 7), + "input_shape": (10, 8, 8), + "inferred_input_shapes": set(((10, 8, 8),)), + "rescale_factors": set(), + "is_entry_node": False, + "conv": { + "module": nn.Conv2d(10, 1, kernel_size=(2, 2), stride=[1, 1], bias=False), + "node_id": 7, }, - 8: { - "layer": IAFSqueeze( + "neuron": { + "module": IAFSqueeze( batch_size=3, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential(), ), - "input_shape": (1, 7, 7), - "output_shape": (1, 7, 7), + "node_id": 8, }, - "destinations": [3], - "conv_rescale_factor": [16], + "destinations": [ + { + "pooling_ids": [], + "pooling_modules": [], + "destination_layer": 3, + "output_shape": (1, 7, 7), + }, + ], }, 3: { - 9: { - "layer": nn.Linear(in_features=49, out_features=500, bias=False), - "input_shape": (1, 7, 7), - "output_shape": (500,), + "input_shape": (49, 1, 1), + "inferred_input_shapes": set(((1, 7, 7),)), + "rescale_factors": set(), + "is_entry_node": False, + "conv": { + "module": nn.Linear(in_features=49, out_features=500, bias=False), + "node_id": 9, }, - 10: { - "layer": IAFSqueeze( + "neuron": { + "module": IAFSqueeze( batch_size=3, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential(), ), - "input_shape": (500,), - "output_shape": (500,), + "node_id": 10, }, - "destinations": [4], - "conv_rescale_factor": [], + "destinations": [ + { + "pooling_ids": [], + "pooling_modules": [], + "destination_layer": 4, + "output_shape": (500, 1, 1), + }, + ], }, 4: { - 11: { - "layer": nn.Linear(in_features=500, out_features=10, bias=False), - "input_shape": (500,), - "output_shape": (10,), + "input_shape": (500, 1, 1), + "inferred_input_shapes": set(((500, 1, 1),)), + "rescale_factors": set(), + "is_entry_node": False, + "conv": { + "module": nn.Linear(in_features=500, out_features=10, bias=False), + "node_id": 11, }, - 12: { - "layer": IAFSqueeze( + "neuron": { + "module": IAFSqueeze( batch_size=3, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential(), ), - "input_shape": (10,), - "output_shape": (10,), + "node_id": 12, }, "destinations": [], - "conv_rescale_factor": [], }, } -sinabs_edges_1 = [ - (0, 1), - (1, 2), - (1, 3), - (2, 4), - (3, 7), - (4, 6), - (6, 7), - (7, 8), - (8, 9), - (9, 10), - (10, 11), - (11, 12), -] - expected_output_1 = { 0: { - "dpcnnl_index": 0, - "conv_node_id": 0, - "conv_in_shape": (2, 34, 34), - "conv_out_shape": (10, 33, 33), - "spk_node_id": 1, - "pool_node_id": [2, 3], - "conv_rescaling_factor": None, - "dynapcnnlayer_destination": [1, 2], - "nodes_destinations": {2: [4], 3: [7]}, - "entry_point": True, + "input_shape": (2, 34, 34), + "pool": [[3, 3], [4, 4]], + "rescale_factor": 1, + "rescale_factors": set(), + "destination_indices": [1, 2], + "entry_node": True, }, 1: { - "dpcnnl_index": 1, - "conv_node_id": 4, - "conv_in_shape": (10, 11, 11), - "conv_out_shape": (10, 8, 8), - "spk_node_id": 6, - "pool_node_id": [], - "conv_rescaling_factor": 4.5, - "dynapcnnlayer_destination": [2], - "nodes_destinations": {6: [7]}, - "entry_point": False, + "input_shape": (10, 11, 11), + "pool": [[1, 1]], + "rescale_factor": 1./9, + "rescale_factors": set(), # Single factor will be popped from list + "destination_indices": [3], + "entry_node": False, }, 2: { - "dpcnnl_index": 2, - "conv_node_id": 7, - "conv_in_shape": (10, 8, 8), - "conv_out_shape": (1, 7, 7), - "spk_node_id": 8, - "pool_node_id": [], - "conv_rescaling_factor": 8.0, - "dynapcnnlayer_destination": [3], - "nodes_destinations": {8: [9]}, - "entry_point": False, + "input_shape": (10, 8, 8), + "pool": [[1, 1]], + "rescale_factor": 1./16, + "rescale_factors": set(), # Single factor will be popped from list + "destination_indices": [3], + "entry_node": False, }, 3: { - "dpcnnl_index": 3, - "conv_node_id": 9, - "conv_in_shape": (1, 7, 7), - "conv_out_shape": (500, 1, 1), - "spk_node_id": 10, - "pool_node_id": [], - "conv_rescaling_factor": None, - "dynapcnnlayer_destination": [4], - "nodes_destinations": {10: [11]}, - "entry_point": False, + "input_shape": (1, 7, 7), + "pool": [[1, 1]], + "rescale_factor": 1, + "rescale_factors": set(), + "destination_indices": [4], + "entry_node": False, }, 4: { - "dpcnnl_index": 4, - "conv_node_id": 11, - "conv_in_shape": (500, 1, 1), - "conv_out_shape": (10, 1, 1), - "spk_node_id": 12, - "pool_node_id": [], - "conv_rescaling_factor": None, - "dynapcnnlayer_destination": [], - "nodes_destinations": {}, - "entry_point": False, + "input_shape": (500, 1, 1), + "pool": [], + "rescale_factor": 1, + "rescale_factors": set(), + "destination_indices": [], + "entry_node": False, }, } diff --git a/tests/test_dynapcnnlayer/test_dynapcnnlayer.py b/tests/test_dynapcnnlayer/test_dynapcnnlayer.py index 52988463..ec9588bf 100644 --- a/tests/test_dynapcnnlayer/test_dynapcnnlayer.py +++ b/tests/test_dynapcnnlayer/test_dynapcnnlayer.py @@ -4,20 +4,15 @@ import pytest from conftest_dynapcnnlayer import args_DynapcnnLayer -from sinabs.backend.dynapcnn.utils import ( - construct_dynapcnnlayer, - construct_layerhandler, - update_nodes_io, -) -from sinabs.backend.dynapcnn.weight_rescaling_methods import rescale_method_1 +from sinabs.backend.dynapcnn.utils import construct_dynapcnnlayers_from_mapper @pytest.mark.parametrize( - "nodes_to_dcnnl_map, dpcnnl_idx, sinabs_edges, entry_point, expected_output", + "dcnnl_map, discretize, rescale_fn, expected_output", args_DynapcnnLayer, ) def test_DynapcnnLayer( - nodes_to_dcnnl_map, dpcnnl_idx, sinabs_edges, entry_point, expected_output + dcnnl_map, discretize, rescale_fn, expected_output ): """Tests the instantiation of a set of `DynapcnnLayer` belonging to the same SNN and the data computed within their constructors and shared among the differntly interacting instances (according to the graph @@ -25,66 +20,54 @@ def test_DynapcnnLayer( """ # create a `DynapcnnLayer` from the set of layers in `nodes_to_dcnnl_map[dpcnnl_idx]`. - layerhandler = construct_layerhandler( - dpcnnl_idx, - True, - sinabs_edges, - nodes_to_dcnnl_map, - rescale_method_1, - entry_point, + dynapcnn_layers, layer_handlers = construct_dynapcnnlayers_from_mapper( + dcnnl_map=dcnnl_map, discretize=discretize, rescale_fn=rescale_fn ) - dynapcnnlayer = construct_dynapcnnlayer(layerhandler) + + for layer_index, dynapcnn_layer in dynapcnn_layers.items(): + + # Test layer instance + in_shape = expected_output[layer_index]["input_shape"] + pool = expected_output[layer_index]["pool"] + rescale_weights = expected_output[layer_index]["rescale_factor"] - # check if any node (layer) in `dynapcnnlayer` has been modified (e.g. `nn.Linear` turned `nn.Conv2d`). - node, output_shape = layerhandler.get_modified_node_io( - nodes_to_dcnnl_map[dpcnnl_idx] - ) + assert ( + tuple(dynapcnn_layer.in_shape) == in_shape + ), f"wrong 'DynapcnnLayer.in_shape': Should be {in_shape}." + assert ( + dynapcnn_layer.discretize == discretize + ), f"wrong 'DynapcnnLayer.discretize': Should be {discretize}." + in_shape = expected_output[layer_index]["input_shape"] + assert ( + dynapcnn_layer.pool == pool + ), f"wrong 'DynapcnnLayer.pool': Should be {pool}." + in_shape = expected_output[layer_index]["input_shape"] + assert ( + dynapcnn_layer.rescale_weights == rescale_weights + ), f"wrong 'DynapcnnLayer.in_shape': Should be {rescale_weights}." + + + # Test entries in layer info that are not directly repeated in layer or handler instances + layer_info = dcnnl_map[layer_index] + rescale_factors = expected_output[layer_index]["rescale_factors"] - # one of the layers in `dynapcnnlayer` had its type modified (update input shape of nodes receiving from it). - if isinstance(node, int) and isinstance(output_shape, tuple): - update_nodes_io(node, output_shape, nodes_to_dcnnl_map, sinabs_edges) + assert ( + layer_info["rescale_factors"] == rescale_factors + ), f"wrong 'rescale_factors' entry: Should be {rescale_factors}." - dpcnnl_index = expected_output[dpcnnl_idx]["dpcnnl_index"] - conv_node_id = expected_output[dpcnnl_idx]["conv_node_id"] - conv_in_shape = expected_output[dpcnnl_idx]["conv_in_shape"] - conv_out_shape = expected_output[dpcnnl_idx]["conv_out_shape"] - spk_node_id = expected_output[dpcnnl_idx]["spk_node_id"] - pool_node_id = expected_output[dpcnnl_idx]["pool_node_id"] - conv_rescaling_factor = expected_output[dpcnnl_idx]["conv_rescaling_factor"] - dynapcnnlayer_destination = expected_output[dpcnnl_idx]["dynapcnnlayer_destination"] - nodes_destinations = expected_output[dpcnnl_idx]["nodes_destinations"] - entry_point = expected_output[dpcnnl_idx]["entry_point"] + # Test layer handler instance + layerhandler = layer_handlers[layer_index] + destination_indices = expected_output[layer_index]["destination_indices"] + entry_node = expected_output[layer_index]["entry_node"] - assert ( - layerhandler.dpcnnl_index == expected_output[dpcnnl_idx]["dpcnnl_index"] - ), f"wrong 'DynapcnnLayer.dpcnnl_index': ID of the instance should be {dpcnnl_index}." - assert ( - layerhandler.conv_node_id == expected_output[dpcnnl_idx]["conv_node_id"] - ), f"wrong 'DynapcnnLayer.conv_node_id': convolution layer should be node {conv_node_id}." - assert ( - layerhandler.conv_in_shape == expected_output[dpcnnl_idx]["conv_in_shape"] - ), f"wrong 'DynapcnnLayer.conv_in_shape': input tensor shape of convolution should be {conv_in_shape}." - assert ( - layerhandler.conv_out_shape == expected_output[dpcnnl_idx]["conv_out_shape"] - ), f"wrong 'DynapcnnLayer.conv_out_shape': output tensor shape of convolution should be {conv_out_shape}." - assert ( - layerhandler.spk_node_id == expected_output[dpcnnl_idx]["spk_node_id"] - ), f"wrong 'DynapcnnLayer.spk_node_id': spiking layer should be node {spk_node_id}." - assert ( - layerhandler.pool_node_id == expected_output[dpcnnl_idx]["pool_node_id"] - ), f"wrong 'DynapcnnLayer.pool_node_id': pooling layer node(s) should be {pool_node_id}." - assert ( - layerhandler.conv_rescaling_factor - == expected_output[dpcnnl_idx]["conv_rescaling_factor"] - ), f"wrong 'DynapcnnLayer.conv_rescaling_factor': computed re-scaling factor should be {conv_rescaling_factor}." - assert ( - layerhandler.dynapcnnlayer_destination - == expected_output[dpcnnl_idx]["dynapcnnlayer_destination"] - ), f"wrong 'DynapcnnLayer.dynapcnnlayer_destination': the DynapcnnLayer(s) set as destination(s) should be {dynapcnnlayer_destination}." - assert ( - layerhandler.nodes_destinations - == expected_output[dpcnnl_idx]["nodes_destinations"] - ), f"wrong 'DynapcnnLayer.nodes_destinations': the targeted nodes within other DynapcnnLayer instance(s) should be {nodes_destinations}." - assert ( - layerhandler.entry_point == expected_output[dpcnnl_idx]["entry_point"] - ), f"wrong 'DynapcnnLayer.entry_point': its value should be {entry_point}." + assert ( + layerhandler.layer_index == layer_index + ), f"wrong 'DynapcnnLayerHandler.layer_index': ID of the instance should be {layer_index}." + assert ( + layerhandler.destination_indices + == expected_output[layer_index]["destination_indices"] + ), f"wrong 'DynapcnnLayerHandler.destination_indices': the DynapcnnLayer(s) set as destination(s) should be {destination_indices}." + assert ( + layerhandler.entry_node == expected_output[layer_index]["entry_node"] + ), f"wrong 'DynapcnnLayerHandler.entry_node': its value should be {entry_node}." + assert layerhandler.assigned_core is None diff --git a/tests/test_graph_extractor/model_dummy_1.py b/tests/test_graph_extractor/model_dummy_1.py index 87215fcf..18c596be 100644 --- a/tests/test_graph_extractor/model_dummy_1.py +++ b/tests/test_graph_extractor/model_dummy_1.py @@ -115,7 +115,7 @@ def forward(self, x): "pool1a": 3, "conv2": 4, "adder": 5, - "iaf2": 6, + "iaf2": 6, "conv3": 7, "iaf3": 8, "flat": 9, @@ -126,26 +126,26 @@ def forward(self, x): }, "entry_nodes": {0}, "nodes_io_shapes": { - 0: {"input": torch.Size([3, 2, 34, 34]), "output": torch.Size([3, 10, 33, 33])}, + 0: {"input": torch.Size([2, 34, 34]), "output": torch.Size([10, 33, 33])}, 1: { - "input": torch.Size([3, 10, 33, 33]), - "output": torch.Size([3, 10, 33, 33]), + "input": torch.Size([10, 33, 33]), + "output": torch.Size([10, 33, 33]), }, 2: { - "input": torch.Size([3, 10, 33, 33]), - "output": torch.Size([3, 10, 11, 11]), + "input": torch.Size([10, 33, 33]), + "output": torch.Size([10, 11, 11]), }, - 3: {"input": torch.Size([3, 10, 33, 33]), "output": torch.Size([3, 10, 8, 8])}, - 4: {"input": torch.Size([3, 10, 11, 11]), "output": torch.Size([3, 10, 8, 8])}, - 6: {"input": torch.Size([3, 10, 8, 8]), "output": torch.Size([3, 10, 8, 8])}, - 5: {"input": torch.Size([3, 10, 8, 8]), "output": torch.Size([3, 10, 8, 8])}, - 7: {"input": torch.Size([3, 10, 8, 8]), "output": torch.Size([3, 1, 7, 7])}, - 8: {"input": torch.Size([3, 1, 7, 7]), "output": torch.Size([3, 1, 7, 7])}, - 9: {"input": torch.Size([3, 1, 7, 7]), "output": torch.Size([3, 49])}, - 10: {"input": torch.Size([3, 49]), "output": torch.Size([3, 500])}, - 11: {"input": torch.Size([3, 500]), "output": torch.Size([3, 500])}, - 12: {"input": torch.Size([3, 500]), "output": torch.Size([3, 10])}, - 13: {"input": torch.Size([3, 10]), "output": torch.Size([3, 10])}, + 3: {"input": torch.Size([10, 33, 33]), "output": torch.Size([10, 8, 8])}, + 4: {"input": torch.Size([10, 11, 11]), "output": torch.Size([10, 8, 8])}, + 6: {"input": torch.Size([10, 8, 8]), "output": torch.Size([10, 8, 8])}, + 5: {"input": torch.Size([10, 8, 8]), "output": torch.Size([10, 8, 8])}, + 7: {"input": torch.Size([10, 8, 8]), "output": torch.Size([1, 7, 7])}, + 8: {"input": torch.Size([1, 7, 7]), "output": torch.Size([1, 7, 7])}, + 9: {"input": torch.Size([1, 7, 7]), "output": torch.Size([49, 1, 1])}, + 10: {"input": torch.Size([49, 1, 1]), "output": torch.Size([500, 1, 1])}, + 11: {"input": torch.Size([500, 1, 1]), "output": torch.Size([500, 1, 1])}, + 12: {"input": torch.Size([500, 1, 1]), "output": torch.Size([10, 1, 1])}, + 13: {"input": torch.Size([10, 1, 1]), "output": torch.Size([10, 1, 1])}, }, } diff --git a/tests/test_graph_extractor/model_dummy_2.py b/tests/test_graph_extractor/model_dummy_2.py index b56a975e..f4d9bb77 100644 --- a/tests/test_graph_extractor/model_dummy_2.py +++ b/tests/test_graph_extractor/model_dummy_2.py @@ -174,26 +174,26 @@ def forward(self, x): }, "entry_nodes": {0}, "nodes_io_shapes": { - 0: {"input": torch.Size([8, 2, 34, 34]), "output": torch.Size([8, 4, 33, 33])}, - 1: {"input": torch.Size([8, 4, 33, 33]), "output": torch.Size([8, 4, 33, 33])}, - 2: {"input": torch.Size([8, 4, 33, 33]), "output": torch.Size([8, 4, 32, 32])}, - 3: {"input": torch.Size([8, 4, 32, 32]), "output": torch.Size([8, 4, 32, 32])}, - 4: {"input": torch.Size([8, 4, 32, 32]), "output": torch.Size([8, 4, 16, 16])}, - 5: {"input": torch.Size([8, 4, 16, 16]), "output": torch.Size([8, 4, 15, 15])}, - 6: {"input": torch.Size([8, 4, 16, 16]), "output": torch.Size([8, 4, 15, 15])}, - 7: {"input": torch.Size([8, 4, 15, 15]), "output": torch.Size([8, 4, 15, 15])}, - 12: {"input": torch.Size([8, 4, 15, 15]), "output": torch.Size([8, 4, 15, 15])}, - 8: {"input": torch.Size([8, 4, 15, 15]), "output": torch.Size([8, 4, 7, 7])}, - 13: {"input": torch.Size([8, 4, 15, 15]), "output": torch.Size([8, 4, 7, 7])}, - 9: {"input": torch.Size([8, 4, 7, 7]), "output": torch.Size([8, 4, 6, 6])}, - 14: {"input": torch.Size([8, 4, 7, 7]), "output": torch.Size([8, 4, 6, 6])}, - 10: {"input": torch.Size([8, 4, 6, 6]), "output": torch.Size([8, 4, 6, 6])}, - 15: {"input": torch.Size([8, 4, 6, 6]), "output": torch.Size([8, 4, 6, 6])}, - 11: {"input": torch.Size([8, 4, 6, 6]), "output": torch.Size([8, 144])}, - 16: {"input": torch.Size([8, 4, 6, 6]), "output": torch.Size([8, 144])}, - 19: {"input": torch.Size([8, 144]), "output": torch.Size([8, 144])}, - 17: {"input": torch.Size([8, 144]), "output": torch.Size([8, 10])}, - 18: {"input": torch.Size([8, 10]), "output": torch.Size([8, 10])}, + 0: {"input": torch.Size([2, 34, 34]), "output": torch.Size([4, 33, 33])}, + 1: {"input": torch.Size([4, 33, 33]), "output": torch.Size([4, 33, 33])}, + 2: {"input": torch.Size([4, 33, 33]), "output": torch.Size([4, 32, 32])}, + 3: {"input": torch.Size([4, 32, 32]), "output": torch.Size([4, 32, 32])}, + 4: {"input": torch.Size([4, 32, 32]), "output": torch.Size([4, 16, 16])}, + 5: {"input": torch.Size([4, 16, 16]), "output": torch.Size([4, 15, 15])}, + 6: {"input": torch.Size([4, 16, 16]), "output": torch.Size([4, 15, 15])}, + 7: {"input": torch.Size([4, 15, 15]), "output": torch.Size([4, 15, 15])}, + 12: {"input": torch.Size([4, 15, 15]), "output": torch.Size([4, 15, 15])}, + 8: {"input": torch.Size([4, 15, 15]), "output": torch.Size([4, 7, 7])}, + 13: {"input": torch.Size([4, 15, 15]), "output": torch.Size([4, 7, 7])}, + 9: {"input": torch.Size([4, 7, 7]), "output": torch.Size([4, 6, 6])}, + 14: {"input": torch.Size([4, 7, 7]), "output": torch.Size([4, 6, 6])}, + 10: {"input": torch.Size([4, 6, 6]), "output": torch.Size([4, 6, 6])}, + 15: {"input": torch.Size([4, 6, 6]), "output": torch.Size([4, 6, 6])}, + 11: {"input": torch.Size([4, 6, 6]), "output": torch.Size([144, 1, 1])}, + 16: {"input": torch.Size([4, 6, 6]), "output": torch.Size([144, 1, 1])}, + 19: {"input": torch.Size([144, 1, 1]), "output": torch.Size([144, 1, 1])}, + 17: {"input": torch.Size([144, 1, 1]), "output": torch.Size([10, 1, 1])}, + 18: {"input": torch.Size([10, 1, 1]), "output": torch.Size([10, 1, 1])}, }, } diff --git a/tests/test_graph_extractor/model_dummy_3.py b/tests/test_graph_extractor/model_dummy_3.py index 326ec61d..7ee4c5a5 100644 --- a/tests/test_graph_extractor/model_dummy_3.py +++ b/tests/test_graph_extractor/model_dummy_3.py @@ -204,31 +204,31 @@ def forward(self, x): }, "entry_nodes": {0, 9}, "nodes_io_shapes": { - 0: {"input": torch.Size([2, 2, 34, 34]), "output": torch.Size([2, 4, 33, 33])}, - 9: {"input": torch.Size([2, 2, 34, 34]), "output": torch.Size([2, 4, 33, 33])}, - 1: {"input": torch.Size([2, 4, 33, 33]), "output": torch.Size([2, 4, 33, 33])}, - 10: {"input": torch.Size([2, 4, 33, 33]), "output": torch.Size([2, 4, 33, 33])}, - 2: {"input": torch.Size([2, 4, 33, 33]), "output": torch.Size([2, 4, 32, 32])}, - 11: {"input": torch.Size([2, 4, 33, 33]), "output": torch.Size([2, 4, 32, 32])}, - 3: {"input": torch.Size([2, 4, 32, 32]), "output": torch.Size([2, 4, 32, 32])}, - 12: {"input": torch.Size([2, 4, 32, 32]), "output": torch.Size([2, 4, 32, 32])}, - 4: {"input": torch.Size([2, 4, 32, 32]), "output": torch.Size([2, 4, 16, 16])}, - 13: {"input": torch.Size([2, 4, 32, 32]), "output": torch.Size([2, 4, 16, 16])}, - 5: {"input": torch.Size([2, 4, 16, 16]), "output": torch.Size([2, 4, 15, 15])}, - 14: {"input": torch.Size([2, 4, 16, 16]), "output": torch.Size([2, 4, 15, 15])}, - 6: {"input": torch.Size([2, 4, 15, 15]), "output": torch.Size([2, 4, 15, 15])}, - 15: {"input": torch.Size([2, 4, 15, 15]), "output": torch.Size([2, 4, 15, 15])}, - 7: {"input": torch.Size([2, 4, 15, 15]), "output": torch.Size([2, 4, 7, 7])}, - 16: {"input": torch.Size([2, 4, 15, 15]), "output": torch.Size([2, 4, 7, 7])}, - 8: {"input": torch.Size([2, 4, 7, 7]), "output": torch.Size([2, 196])}, - 17: {"input": torch.Size([2, 4, 7, 7]), "output": torch.Size([2, 196])}, - 18: {"input": torch.Size([2, 196]), "output": torch.Size([2, 196])}, - 19: {"input": torch.Size([2, 196]), "output": torch.Size([2, 100])}, - 20: {"input": torch.Size([2, 100]), "output": torch.Size([2, 100])}, - 21: {"input": torch.Size([2, 100]), "output": torch.Size([2, 100])}, - 22: {"input": torch.Size([2, 100]), "output": torch.Size([2, 100])}, - 23: {"input": torch.Size([2, 100]), "output": torch.Size([2, 10])}, - 24: {"input": torch.Size([2, 10]), "output": torch.Size([2, 10])}, + 0: {"input": torch.Size([2, 34, 34]), "output": torch.Size([4, 33, 33])}, + 9: {"input": torch.Size([2, 34, 34]), "output": torch.Size([4, 33, 33])}, + 1: {"input": torch.Size([4, 33, 33]), "output": torch.Size([4, 33, 33])}, + 10: {"input": torch.Size([4, 33, 33]), "output": torch.Size([4, 33, 33])}, + 2: {"input": torch.Size([4, 33, 33]), "output": torch.Size([4, 32, 32])}, + 11: {"input": torch.Size([4, 33, 33]), "output": torch.Size([4, 32, 32])}, + 3: {"input": torch.Size([4, 32, 32]), "output": torch.Size([4, 32, 32])}, + 12: {"input": torch.Size([4, 32, 32]), "output": torch.Size([4, 32, 32])}, + 4: {"input": torch.Size([4, 32, 32]), "output": torch.Size([4, 16, 16])}, + 13: {"input": torch.Size([4, 32, 32]), "output": torch.Size([4, 16, 16])}, + 5: {"input": torch.Size([4, 16, 16]), "output": torch.Size([4, 15, 15])}, + 14: {"input": torch.Size([4, 16, 16]), "output": torch.Size([4, 15, 15])}, + 6: {"input": torch.Size([4, 15, 15]), "output": torch.Size([4, 15, 15])}, + 15: {"input": torch.Size([4, 15, 15]), "output": torch.Size([4, 15, 15])}, + 7: {"input": torch.Size([4, 15, 15]), "output": torch.Size([4, 7, 7])}, + 16: {"input": torch.Size([4, 15, 15]), "output": torch.Size([4, 7, 7])}, + 8: {"input": torch.Size([4, 7, 7]), "output": torch.Size([196, 1, 1])}, + 17: {"input": torch.Size([4, 7, 7]), "output": torch.Size([196, 1, 1])}, + 18: {"input": torch.Size([196, 1, 1]), "output": torch.Size([196, 1, 1])}, + 19: {"input": torch.Size([196, 1, 1]), "output": torch.Size([100, 1, 1])}, + 20: {"input": torch.Size([100, 1, 1]), "output": torch.Size([100, 1, 1])}, + 21: {"input": torch.Size([100, 1, 1]), "output": torch.Size([100, 1, 1])}, + 22: {"input": torch.Size([100, 1, 1]), "output": torch.Size([100, 1, 1])}, + 23: {"input": torch.Size([100, 1, 1]), "output": torch.Size([10, 1, 1])}, + 24: {"input": torch.Size([10, 1, 1]), "output": torch.Size([10, 1, 1])}, }, } diff --git a/tests/test_graph_extractor/model_dummy_4.py b/tests/test_graph_extractor/model_dummy_4.py index e83e13cd..1e4d4da6 100644 --- a/tests/test_graph_extractor/model_dummy_4.py +++ b/tests/test_graph_extractor/model_dummy_4.py @@ -166,26 +166,26 @@ def forward(self, x): }, "entry_nodes": {0}, "nodes_io_shapes": { - 0: {"input": torch.Size([2, 2, 34, 34]), "output": torch.Size([2, 1, 33, 33])}, - 1: {"input": torch.Size([2, 1, 33, 33]), "output": torch.Size([2, 1, 33, 33])}, - 2: {"input": torch.Size([2, 1, 33, 33]), "output": torch.Size([2, 1, 32, 32])}, - 3: {"input": torch.Size([2, 1, 33, 33]), "output": torch.Size([2, 1, 32, 32])}, - 4: {"input": torch.Size([2, 1, 32, 32]), "output": torch.Size([2, 1, 32, 32])}, - 7: {"input": torch.Size([2, 1, 32, 32]), "output": torch.Size([2, 1, 32, 32])}, - 5: {"input": torch.Size([2, 1, 32, 32]), "output": torch.Size([2, 1, 16, 16])}, - 8: {"input": torch.Size([2, 1, 32, 32]), "output": torch.Size([2, 1, 16, 16])}, - 9: {"input": torch.Size([2, 1, 32, 32]), "output": torch.Size([2, 1, 6, 6])}, - 6: {"input": torch.Size([2, 1, 16, 16]), "output": torch.Size([2, 1, 16, 16])}, - 10: {"input": torch.Size([2, 1, 6, 6]), "output": torch.Size([2, 1, 5, 5])}, - 11: {"input": torch.Size([2, 1, 16, 16]), "output": torch.Size([2, 1, 15, 15])}, - 17: {"input": torch.Size([2, 1, 5, 5]), "output": torch.Size([2, 1, 5, 5])}, - 12: {"input": torch.Size([2, 1, 15, 15]), "output": torch.Size([2, 1, 15, 15])}, - 16: {"input": torch.Size([2, 1, 5, 5]), "output": torch.Size([2, 25])}, - 13: {"input": torch.Size([2, 1, 15, 15]), "output": torch.Size([2, 1, 5, 5])}, - 14: {"input": torch.Size([2, 1, 5, 5]), "output": torch.Size([2, 25])}, - 15: {"input": torch.Size([2, 25]), "output": torch.Size([2, 25])}, - 18: {"input": torch.Size([2, 25]), "output": torch.Size([2, 10])}, - 19: {"input": torch.Size([2, 10]), "output": torch.Size([2, 10])}, + 0: {"input": torch.Size([2, 34, 34]), "output": torch.Size([1, 33, 33])}, + 1: {"input": torch.Size([1, 33, 33]), "output": torch.Size([1, 33, 33])}, + 2: {"input": torch.Size([1, 33, 33]), "output": torch.Size([1, 32, 32])}, + 3: {"input": torch.Size([1, 33, 33]), "output": torch.Size([1, 32, 32])}, + 4: {"input": torch.Size([1, 32, 32]), "output": torch.Size([1, 32, 32])}, + 7: {"input": torch.Size([1, 32, 32]), "output": torch.Size([1, 32, 32])}, + 5: {"input": torch.Size([1, 32, 32]), "output": torch.Size([1, 16, 16])}, + 8: {"input": torch.Size([1, 32, 32]), "output": torch.Size([1, 16, 16])}, + 9: {"input": torch.Size([1, 32, 32]), "output": torch.Size([1, 6, 6])}, + 6: {"input": torch.Size([1, 16, 16]), "output": torch.Size([1, 16, 16])}, + 10: {"input": torch.Size([1, 6, 6]), "output": torch.Size([1, 5, 5])}, + 11: {"input": torch.Size([1, 16, 16]), "output": torch.Size([1, 15, 15])}, + 17: {"input": torch.Size([1, 5, 5]), "output": torch.Size([1, 5, 5])}, + 12: {"input": torch.Size([1, 15, 15]), "output": torch.Size([1, 15, 15])}, + 16: {"input": torch.Size([1, 5, 5]), "output": torch.Size([25, 1, 1])}, + 13: {"input": torch.Size([1, 15, 15]), "output": torch.Size([1, 5, 5])}, + 14: {"input": torch.Size([1, 5, 5]), "output": torch.Size([25, 1, 1])}, + 15: {"input": torch.Size([25, 1, 1]), "output": torch.Size([25, 1, 1])}, + 18: {"input": torch.Size([25, 1, 1]), "output": torch.Size([10, 1, 1])}, + 19: {"input": torch.Size([10, 1, 1]), "output": torch.Size([10, 1, 1])}, }, } From ebb2be366ff27fa89b0c6388ebd460e50dc96d88 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Thu, 10 Oct 2024 17:57:09 +0200 Subject: [PATCH 194/379] (WIP): Update DynapcnnLayer tests --- .../backend/dynapcnn/nir_graph_extractor.py | 6 +- .../backend/dynapcnn/sinabs_edges_handler.py | 13 + .../conftest_dynapcnnlayer.py | 8 +- tests/test_dynapcnnlayer/model_dummy_1.py | 8 +- tests/test_dynapcnnlayer/model_dummy_2.py | 350 +++++++------- tests/test_dynapcnnlayer/model_dummy_3.py | 441 +++++++++--------- .../test_dynapcnnlayer/test_dynapcnnlayer.py | 9 +- tests/test_graph_extractor/model_dummy_1.py | 2 +- 8 files changed, 424 insertions(+), 413 deletions(-) diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index 299e62df..b89dd8d9 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -304,7 +304,7 @@ def _get_nodes_io_shapes( # TODO - this is currently a limitation imposed by the validation checks done by Speck once a configuration: it wants # different input sources to a core to have the same output shapes. - if any(inp.shape != inputs[0].shape for inp in inputs): + if any(inp.shape != inputs[0].shape for inp in inputs): raise ValueError( f"Layer `sinabs.layers.merge.Merge` (node {node}) requires input tensors with the same shape" ) @@ -337,8 +337,8 @@ def _get_nodes_io_shapes( # replace the I/O tensor information by its shape information, ignoring the batch/time axis for node, io in nodes_io_map.items(): - input_shape = io["input"].shape[1:] - output_shape = io["output"].shape[1:] + input_shape = io["input"].shape[1:] + output_shape = io["output"].shape[1:] # Linear layers have fewer in/out dimensions. Extend by appending 1's if (length := len(input_shape)) < 3: input_shape = (*input_shape, *(1 for __ in range(3 - length))) diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index d4aec78d..0d72c9bd 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -337,6 +337,12 @@ def set_pooling_layer_destination( matched = False for destination in layer_info["destinations"]: if destination["pooling_ids"][-1] == edge[0]: + if "destination_layer" in destination: + # Destination is already linked to a postsynaptic layer. This happens when + # pooling nodes have outgoing edges to different weight layer. + # Copy the destination + destination = {k: v for k, v in destination.items()} + layer_info["destinations"].append(destination) matched = True break if not matched: @@ -393,3 +399,10 @@ def trace_paths(node: int, remaining_edges: Set[Edge]) -> List[deque[int]]: paths = [deque([node])] return paths, processed_edges + + +# TODO: +""" Add verification tools to ensure that: +- there are as many destinations as there are edges from pool/neuron to weight +- there are as many layers as there are edges from weight to neuron +""" diff --git a/tests/test_dynapcnnlayer/conftest_dynapcnnlayer.py b/tests/test_dynapcnnlayer/conftest_dynapcnnlayer.py index 39d0edc4..7364910e 100644 --- a/tests/test_dynapcnnlayer/conftest_dynapcnnlayer.py +++ b/tests/test_dynapcnnlayer/conftest_dynapcnnlayer.py @@ -2,7 +2,9 @@ # contact : wsoaresgirao@gmail.com from model_dummy_1 import expected_output_1, dcnnl_map_1 -# from model_dummy_2 import expected_output_2, nodes_to_dcnnl_map_2, sinabs_edges_2 +from model_dummy_2 import expected_output_2, dcnnl_map_2 +from model_dummy_3 import expected_output_3, dcnnl_map_3 + # from model_dummy_3 import expected_output_3, nodes_to_dcnnl_map_3, sinabs_edges_3 # from model_dummy_4 import expected_output_4, nodes_to_dcnnl_map_4, sinabs_edges_4 @@ -10,6 +12,10 @@ args_DynapcnnLayer = [ (dcnnl_map_1, True, None, expected_output_1), (dcnnl_map_1, False, None, expected_output_1), + (dcnnl_map_2, True, None, expected_output_2), + (dcnnl_map_2, False, None, expected_output_2), + (dcnnl_map_3, True, None, expected_output_3), + (dcnnl_map_3, False, None, expected_output_3), # (nodes_to_dcnnl_map_2, 0, sinabs_edges_2, [0], expected_output_2), # (nodes_to_dcnnl_map_2, 1, sinabs_edges_2, [0], expected_output_2), # (nodes_to_dcnnl_map_2, 2, sinabs_edges_2, [0], expected_output_2), diff --git a/tests/test_dynapcnnlayer/model_dummy_1.py b/tests/test_dynapcnnlayer/model_dummy_1.py index 504ad32a..a9be51d8 100644 --- a/tests/test_dynapcnnlayer/model_dummy_1.py +++ b/tests/test_dynapcnnlayer/model_dummy_1.py @@ -35,7 +35,9 @@ }, { "pooling_ids": [3], - "pooling_modules": [nn.AvgPool2d(kernel_size=4, stride=4, padding=0),], + "pooling_modules": [ + nn.AvgPool2d(kernel_size=4, stride=4, padding=0), + ], "destination_layer": 2, "output_shape": (10, 8, 8), }, @@ -156,7 +158,7 @@ 1: { "input_shape": (10, 11, 11), "pool": [[1, 1]], - "rescale_factor": 1./9, + "rescale_factor": 1.0 / 9, "rescale_factors": set(), # Single factor will be popped from list "destination_indices": [3], "entry_node": False, @@ -164,7 +166,7 @@ 2: { "input_shape": (10, 8, 8), "pool": [[1, 1]], - "rescale_factor": 1./16, + "rescale_factor": 1.0 / 16, "rescale_factors": set(), # Single factor will be popped from list "destination_indices": [3], "entry_node": False, diff --git a/tests/test_dynapcnnlayer/model_dummy_2.py b/tests/test_dynapcnnlayer/model_dummy_2.py index d682f936..12b0878f 100644 --- a/tests/test_dynapcnnlayer/model_dummy_2.py +++ b/tests/test_dynapcnnlayer/model_dummy_2.py @@ -7,260 +7,260 @@ from sinabs.activation.surrogate_gradient_fn import PeriodicExponential from sinabs.layers import IAFSqueeze, SumPool2d -nodes_to_dcnnl_map_2 = { +dcnnl_map_2 = { 0: { - 0: { - "layer": nn.Conv2d(2, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), - "input_shape": (2, 34, 34), - "output_shape": (4, 33, 33), + "input_shape": (2, 34, 34), + "inferred_input_shapes": set(), + "rescale_factors": set(), + "is_entry_node": True, + "conv": { + "module": nn.Conv2d(2, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), + "node_id": 0, }, - 1: { - "layer": IAFSqueeze( + "neuron": { + "module": IAFSqueeze( batch_size=8, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential(), ), - "input_shape": (4, 33, 33), - "output_shape": (4, 33, 33), + "node_id": 1, }, - "destinations": [1], - "conv_rescale_factor": [], + "destinations": [ + { + "pooling_ids": [], + "pooling_modules": [], + "destination_layer": 1, + "output_shape": (4, 33, 33), + }, + ], }, 1: { - 2: { - "layer": nn.Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), - "input_shape": (4, 33, 33), - "output_shape": (4, 32, 32), + "input_shape": (4, 33, 33), + "inferred_input_shapes": set(((4, 33, 33),)), + "rescale_factors": set(), + "is_entry_node": False, + "conv": { + "module": nn.Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), + "node_id": 2, }, - 3: { - "layer": IAFSqueeze( + "neuron": { + "module": IAFSqueeze( batch_size=8, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential(), ), - "input_shape": (4, 32, 32), - "output_shape": (4, 32, 32), + "node_id": 3, }, - 4: { - "layer": SumPool2d(kernel_size=2, stride=2, ceil_mode=False), - "input_shape": (4, 32, 32), - "output_shape": (4, 16, 16), - }, - "destinations": [2, 3], - "conv_rescale_factor": [], + "destinations": [ + { + "pooling_ids": [4], + "pooling_modules": [ + SumPool2d(kernel_size=2, stride=2, ceil_mode=False), + ], + "destination_layer": 2, + "output_shape": (4, 16, 16), + }, + { + "pooling_ids": [4], + "pooling_modules": [ + SumPool2d(kernel_size=2, stride=2, ceil_mode=False), + ], + "destination_layer": 3, + "output_shape": (4, 16, 16), + }, + ], }, 2: { - 5: { - "layer": nn.Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), - "input_shape": (4, 16, 16), - "output_shape": (4, 15, 15), + "input_shape": (4, 16, 16), + "inferred_input_shapes": set(((4, 16, 16),)), + "rescale_factors": set(), + "is_entry_node": False, + "conv": { + "module": nn.Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), + "node_id": 5, }, - 7: { - "layer": IAFSqueeze( + "neuron": { + "module": IAFSqueeze( batch_size=8, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential(), ), - "input_shape": (4, 15, 15), - "output_shape": (4, 15, 15), - }, - 8: { - "layer": SumPool2d(kernel_size=2, stride=2, ceil_mode=False), - "input_shape": (4, 15, 15), - "output_shape": (4, 7, 7), + "node_id": 7, }, - "destinations": [4], - "conv_rescale_factor": [], + "destinations": [ + { + "pooling_ids": [8], + "pooling_modules": [ + SumPool2d(kernel_size=2, stride=2, ceil_mode=False), + ], + "destination_layer": 4, + "output_shape": (4, 7, 7), + }, + ], }, 3: { - 6: { - "layer": nn.Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), - "input_shape": (4, 16, 16), - "output_shape": (4, 15, 15), + "input_shape": (4, 16, 16), + "inferred_input_shapes": set(((4, 16, 16),)), + "rescale_factors": set(), + "is_entry_node": False, + "conv": { + "module": nn.Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), + "node_id": 6, }, - 11: { - "layer": IAFSqueeze( + "neuron": { + "module": IAFSqueeze( batch_size=8, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential(), ), - "input_shape": (4, 15, 15), - "output_shape": (4, 15, 15), + "node_id": 11, }, - 12: { - "layer": SumPool2d(kernel_size=2, stride=2, ceil_mode=False), - "input_shape": (4, 15, 15), - "output_shape": (4, 7, 7), - }, - "destinations": [6], - "conv_rescale_factor": [], + "destinations": [ + { + "pooling_ids": [12], + "pooling_modules": [ + SumPool2d(kernel_size=2, stride=2, ceil_mode=False), + ], + "destination_layer": 6, + "output_shape": (4, 7, 7), + }, + ], }, 4: { - 9: { - "layer": nn.Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), - "input_shape": (4, 7, 7), - "output_shape": (4, 6, 6), + "input_shape": (4, 7, 7), + "inferred_input_shapes": set(((4, 7, 7),)), + "rescale_factors": set(), + "is_entry_node": False, + "conv": { + "module": nn.Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), + "node_id": 9, }, - 10: { - "layer": IAFSqueeze( + "neuron": { + "module": IAFSqueeze( batch_size=8, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential(), ), - "input_shape": (4, 6, 6), - "output_shape": (4, 6, 6), + "node_id": 10, }, - "destinations": [5], - "conv_rescale_factor": [], + "destinations": [ + { + "pooling_ids": [], + "pooling_modules": [], + "destination_layer": 5, + "output_shape": (4, 6, 6), + }, + ], }, 5: { - 15: { - "layer": nn.Linear(in_features=144, out_features=10, bias=False), - "input_shape": (4, 6, 6), - "output_shape": (10,), + "input_shape": (144, 1, 1), + "inferred_input_shapes": set(((4, 6, 6),)), + "rescale_factors": set(), + "is_entry_node": False, + "conv": { + "module": nn.Linear(in_features=144, out_features=10, bias=False), + "node_id": 15, }, - 16: { - "layer": IAFSqueeze( + "neuron": { + "module": IAFSqueeze( batch_size=8, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential(), ), - "input_shape": (10,), - "output_shape": (10,), + "node_id": 16, }, "destinations": [], - "conv_rescale_factor": [], }, 6: { - 13: { - "layer": nn.Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), - "input_shape": (4, 7, 7), - "output_shape": (4, 6, 6), + "input_shape": (4, 7, 7), + "inferred_input_shapes": set(((4, 7, 7),)), + "rescale_factors": set(), + "is_entry_node": False, + "conv": { + "module": nn.Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), + "node_id": 13, }, - 14: { - "layer": IAFSqueeze( + "neuron": { + "module": IAFSqueeze( batch_size=8, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential(), ), - "input_shape": (4, 6, 6), - "output_shape": (4, 6, 6), + "node_id": 14, }, - "destinations": [5], - "conv_rescale_factor": [], + "destinations": [ + { + "pooling_ids": [], + "pooling_modules": [], + "destination_layer": 5, + "output_shape": (4, 6, 6), + }, + ], }, } -sinabs_edges_2 = [ - (0, 1), - (1, 2), - (2, 3), - (3, 4), - (4, 5), - (4, 6), - (5, 7), - (7, 8), - (8, 9), - (9, 10), - (10, 15), - (6, 11), - (11, 12), - (12, 13), - (13, 14), - (14, 15), - (15, 16), -] - expected_output_2 = { 0: { - "dpcnnl_index": 0, - "conv_node_id": 0, - "conv_in_shape": (2, 34, 34), - "conv_out_shape": (4, 33, 33), - "spk_node_id": 1, - "pool_node_id": [], - "conv_rescaling_factor": None, - "dynapcnnlayer_destination": [1], - "nodes_destinations": {1: [2]}, - "entry_point": True, + "input_shape": (2, 34, 34), + "pool": [[1, 1]], + "rescale_factor": 1, + "rescale_factors": set(), + "destination_indices": [1], + "entry_node": True, }, 1: { - "dpcnnl_index": 1, - "conv_node_id": 2, - "conv_in_shape": (4, 33, 33), - "conv_out_shape": (4, 32, 32), - "spk_node_id": 3, - "pool_node_id": [4], - "conv_rescaling_factor": None, - "dynapcnnlayer_destination": [2, 3], - "nodes_destinations": {4: [5, 6]}, - "entry_point": False, + "input_shape": (4, 33, 33), + "pool": [[2, 2], [2, 2]], + "rescale_factor": 1, + "rescale_factors": set(), + "destination_indices": [2, 3], + "entry_node": False, }, 2: { - "dpcnnl_index": 2, - "conv_node_id": 5, - "conv_in_shape": (4, 16, 16), - "conv_out_shape": (4, 15, 15), - "spk_node_id": 7, - "pool_node_id": [8], - "conv_rescaling_factor": None, - "dynapcnnlayer_destination": [4], - "nodes_destinations": {8: [9]}, - "entry_point": False, + "input_shape": (4, 16, 16), + "pool": [[2, 2]], + "rescale_factor": 1, + "rescale_factors": set(), + "destination_indices": [4], + "entry_node": False, }, 3: { - "dpcnnl_index": 3, - "conv_node_id": 6, - "conv_in_shape": (4, 16, 16), - "conv_out_shape": (4, 15, 15), - "spk_node_id": 11, - "pool_node_id": [12], - "conv_rescaling_factor": None, - "dynapcnnlayer_destination": [6], - "nodes_destinations": {12: [13]}, - "entry_point": False, + "input_shape": (4, 16, 16), + "pool": [[2, 2]], + "rescale_factor": 1, + "rescale_factors": set(), + "destination_indices": [6], + "entry_node": False, }, 4: { - "dpcnnl_index": 4, - "conv_node_id": 9, - "conv_in_shape": (4, 7, 7), - "conv_out_shape": (4, 6, 6), - "spk_node_id": 10, - "pool_node_id": [], - "conv_rescaling_factor": None, - "dynapcnnlayer_destination": [5], - "nodes_destinations": {10: [15]}, - "entry_point": False, + "input_shape": (4, 7, 7), + "pool": [[1, 1]], + "rescale_factor": 1, + "rescale_factors": set(), + "destination_indices": [5], + "entry_node": False, }, 5: { - "dpcnnl_index": 5, - "conv_node_id": 15, - "conv_in_shape": (4, 6, 6), - "conv_out_shape": (10, 1, 1), - "spk_node_id": 16, - "pool_node_id": [], - "conv_rescaling_factor": None, - "dynapcnnlayer_destination": [], - "nodes_destinations": {}, - "entry_point": False, + "input_shape": (4, 6, 6), + "pool": [], + "rescale_factor": 1, + "rescale_factors": set(), + "destination_indices": [], + "entry_node": False, }, 6: { - "dpcnnl_index": 6, - "conv_node_id": 13, - "conv_in_shape": (4, 7, 7), - "conv_out_shape": (4, 6, 6), - "spk_node_id": 14, - "pool_node_id": [], - "conv_rescaling_factor": None, - "dynapcnnlayer_destination": [5], - "nodes_destinations": {14: [15]}, - "entry_point": False, + "input_shape": (4, 7, 7), + "pool": [[1, 1]], + "rescale_factor": 1, + "rescale_factors": set(), + "destination_indices": [5], + "entry_node": False, }, } diff --git a/tests/test_dynapcnnlayer/model_dummy_3.py b/tests/test_dynapcnnlayer/model_dummy_3.py index cad0aad6..bc6112b2 100644 --- a/tests/test_dynapcnnlayer/model_dummy_3.py +++ b/tests/test_dynapcnnlayer/model_dummy_3.py @@ -7,331 +7,324 @@ from sinabs.activation.surrogate_gradient_fn import PeriodicExponential from sinabs.layers import IAFSqueeze, SumPool2d -nodes_to_dcnnl_map_3 = { +dcnnl_map_3 = { 0: { - 0: { - "layer": nn.Conv2d(2, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), - "input_shape": (2, 34, 34), - "output_shape": (4, 33, 33), + "input_shape": (2, 34, 34), + "inferred_input_shapes": set(), + "rescale_factors": set(), + "is_entry_node": True, + "conv": { + "module": nn.Conv2d(2, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), + "node_id": 0, }, - 1: { - "layer": IAFSqueeze( + "neuron": { + "module": IAFSqueeze( batch_size=2, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential(), ), - "input_shape": (4, 33, 33), - "output_shape": (4, 33, 33), + "node_id": 1, }, - "destinations": [1], - "conv_rescale_factor": [], + "destinations": [ + { + "pooling_ids": [], + "pooling_modules": [], + "destination_layer": 1, + "output_shape": (4, 33, 33), + }, + ], }, 1: { - 2: { - "layer": nn.Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), - "input_shape": (4, 33, 33), - "output_shape": (4, 32, 32), + "input_shape": (4, 33, 33), + "inferred_input_shapes": set(((4, 33, 33),)), + "rescale_factors": set(), + "is_entry_node": False, + "conv": { + "module": nn.Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), + "node_id": 2, }, - 3: { - "layer": IAFSqueeze( + "neuron": { + "module": IAFSqueeze( batch_size=2, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential(), ), - "input_shape": (4, 32, 32), - "output_shape": (4, 32, 32), + "node_id": 2, }, - 4: { - "layer": SumPool2d(kernel_size=2, stride=2, ceil_mode=False), - "input_shape": (4, 32, 32), - "output_shape": (4, 16, 16), - }, - "destinations": [2], - "conv_rescale_factor": [], + "destinations": [ + { + "pooling_ids": [4], + "pooling_modules": [ + SumPool2d(kernel_size=2, stride=2, ceil_mode=False) + ], + "destination_layer": 2, + "output_shape": (4, 16, 16), + }, + ], }, 2: { - 5: { - "layer": nn.Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), - "input_shape": (4, 16, 16), - "output_shape": (4, 15, 15), + "input_shape": (4, 16, 16), + "inferred_input_shapes": set(((4, 16, 16),)), + "rescale_factors": set(), + "is_entry_node": False, + "conv": { + "module": nn.Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), + "node_id": 5, }, - 6: { - "layer": IAFSqueeze( + "neuron": { + "module": IAFSqueeze( batch_size=2, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential(), ), - "input_shape": (4, 15, 15), - "output_shape": (4, 15, 15), - }, - 7: { - "layer": SumPool2d(kernel_size=2, stride=2, ceil_mode=False), - "input_shape": (4, 15, 15), - "output_shape": (4, 7, 7), + "node_id": 6, }, - "destinations": [3], - "conv_rescale_factor": [], + "destinations": [ + { + "pooling_ids": [7], + "pooling_modules": [ + SumPool2d(kernel_size=2, stride=2, ceil_mode=False) + ], + "destination_layer": 3, + "output_shape": (4, 7, 7), + }, + ], }, 3: { - 17: { - "layer": nn.Linear(in_features=196, out_features=100, bias=False), - "input_shape": (4, 7, 7), - "output_shape": (100,), + "input_shape": (196, 1, 1), + "inferred_input_shapes": set(((4, 7, 7),)), + "rescale_factors": set(), + "is_entry_node": False, + "conv": { + "module": nn.Linear(in_features=196, out_features=100, bias=False), + "node_id": 17, }, - 18: { - "layer": IAFSqueeze( + "neuron": { + "module": IAFSqueeze( batch_size=2, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential(), ), - "input_shape": (100,), - "output_shape": (100,), + "node_id": 18, }, - "destinations": [7], - "conv_rescale_factor": [], + "destinations": [ + { + "pooling_ids": [], + "pooling_modules": [], + "destination_layer": 7, + "output_shape": (100, 1, 1), + }, + ], }, 4: { - 8: { - "layer": nn.Conv2d(2, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), - "input_shape": (2, 34, 34), - "output_shape": (4, 33, 33), + "input_shape": (2, 34, 34), + "inferred_input_shapes": set(), + "rescale_factors": set(), + "is_entry_node": True, + "conv": { + "module": nn.Conv2d(2, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), + "node_id": 8, }, - 9: { - "layer": IAFSqueeze( + "neuron": { + "module": IAFSqueeze( batch_size=2, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential(), ), - "input_shape": (4, 33, 33), - "output_shape": (4, 33, 33), + "node_id": 9, }, - "destinations": [5], - "conv_rescale_factor": [], + "destinations": [ + { + "pooling_ids": [], + "pooling_modules": [], + "destination_layer": 5, + "output_shape": (4, 33, 33), + }, + ], }, 5: { - 10: { - "layer": nn.Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), - "input_shape": (4, 33, 33), - "output_shape": (4, 32, 32), + "input_shape": (4, 33, 33), + "inferred_input_shapes": set(((4, 33, 33),)), + "rescale_factors": set(), + "is_entry_node": False, + "conv": { + "module": nn.Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), + "node_id": 10, }, - 11: { - "layer": IAFSqueeze( + "neuron": { + "module": IAFSqueeze( batch_size=2, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential(), ), - "input_shape": (4, 32, 32), - "output_shape": (4, 32, 32), - }, - 12: { - "layer": SumPool2d(kernel_size=2, stride=2, ceil_mode=False), - "input_shape": (4, 32, 32), - "output_shape": (4, 16, 16), + "node_id": 11, }, - "destinations": [6], - "conv_rescale_factor": [], + "destinations": [ + { + "pooling_ids": [12], + "pooling_modules": [ + SumPool2d(kernel_size=2, stride=2, ceil_mode=False) + ], + "destination_layer": 6, + "output_shape": (4, 16, 16), + }, + ], }, 6: { - 13: { - "layer": nn.Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), - "input_shape": (4, 16, 16), - "output_shape": (4, 15, 15), + "input_shape": (4, 16, 16), + "inferred_input_shapes": set(((4, 16, 16),)), + "rescale_factors": set(), + "is_entry_node": False, + "conv": { + "module": nn.Conv2d(4, 4, kernel_size=(2, 2), stride=(1, 1), bias=False), + "node_id": 13, }, - 14: { - "layer": IAFSqueeze( + "neuron": { + "module": IAFSqueeze( batch_size=2, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential(), ), - "input_shape": (4, 15, 15), - "output_shape": (4, 15, 15), + "node_id": 14, }, - 15: { - "layer": SumPool2d(kernel_size=2, stride=2, ceil_mode=False), - "input_shape": (4, 15, 15), - "output_shape": (4, 7, 7), - }, - "destinations": [3], - "conv_rescale_factor": [], + "destinations": [ + { + "pooling_ids": [15], + "pooling_modules": [ + SumPool2d(kernel_size=2, stride=2, ceil_mode=False) + ], + "destination_layer": 3, + "output_shape": (4, 7, 7), + }, + ], }, 7: { - 19: { - "layer": nn.Linear(in_features=100, out_features=100, bias=False), - "input_shape": (100,), - "output_shape": (100,), + "input_shape": (100, 1, 1), + "inferred_input_shapes": set(((100, 1, 1),)), + "rescale_factors": set(), + "is_entry_node": False, + "conv": { + "module": nn.Linear(in_features=100, out_features=100, bias=False), + "node_id": 19, }, - 20: { - "layer": IAFSqueeze( + "neuron": { + "module": IAFSqueeze( batch_size=2, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential(), ), - "input_shape": (100,), - "output_shape": (100,), + "node_id": 20, }, - "destinations": [8], - "conv_rescale_factor": [], + "destinations": [ + { + "pooling_ids": [], + "pooling_modules": [], + "destination_layer": 8, + "output_shape": (100, 1, 1), + }, + ], }, 8: { - 21: { - "layer": nn.Linear(in_features=100, out_features=10, bias=False), - "input_shape": (100,), - "output_shape": (10,), + "input_shape": (100, 1, 1), + "inferred_input_shapes": set(((100, 1, 1),)), + "rescale_factors": set(), + "is_entry_node": False, + "conv": { + "module": nn.Linear(in_features=100, out_features=10, bias=False), + "node_id": 21, }, - 22: { - "layer": IAFSqueeze( + "neuron": { + "module": IAFSqueeze( batch_size=2, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential(), ), - "input_shape": (10,), - "output_shape": (10,), + "node_id": 22, }, "destinations": [], - "conv_rescale_factor": [], }, } -sinabs_edges_3 = [ - (0, 1), - (1, 2), - (2, 3), - (3, 4), - (4, 5), - (5, 6), - (6, 7), - (7, 17), - (8, 9), - (9, 10), - (10, 11), - (11, 12), - (12, 13), - (13, 14), - (14, 15), - (15, 17), - (17, 18), - (18, 19), - (19, 20), - (20, 21), - (21, 22), -] - expected_output_3 = { 0: { - "dpcnnl_index": 0, - "conv_node_id": 0, - "conv_in_shape": (2, 34, 34), - "conv_out_shape": (4, 33, 33), - "spk_node_id": 1, - "pool_node_id": [], - "conv_rescaling_factor": None, - "dynapcnnlayer_destination": [1], - "nodes_destinations": {1: [2]}, - "entry_point": True, + "input_shape": (2, 34, 34), + "pool": [[1, 1]], + "rescale_factor": 1, + "rescale_factors": set(), + "destination_indices": [1], + "entry_node": True, }, 1: { - "dpcnnl_index": 1, - "conv_node_id": 2, - "conv_in_shape": (4, 33, 33), - "conv_out_shape": (4, 32, 32), - "spk_node_id": 3, - "pool_node_id": [4], - "conv_rescaling_factor": None, - "dynapcnnlayer_destination": [2], - "nodes_destinations": {4: [5]}, - "entry_point": False, + "input_shape": (4, 33, 33), + "pool": [[2, 2]], + "rescale_factor": 1, + "rescale_factors": set(), + "destination_indices": [2], + "entry_node": False, }, 2: { - "dpcnnl_index": 2, - "conv_node_id": 5, - "conv_in_shape": (4, 16, 16), - "conv_out_shape": (4, 15, 15), - "spk_node_id": 6, - "pool_node_id": [7], - "conv_rescaling_factor": None, - "dynapcnnlayer_destination": [3], - "nodes_destinations": {7: [17]}, - "entry_point": False, + "input_shape": (4, 16, 16), + "pool": [[2, 2]], + "rescale_factor": 1, + "rescale_factors": set(), + "destination_indices": [3], + "entry_node": False, }, 3: { - "dpcnnl_index": 3, - "conv_node_id": 17, - "conv_in_shape": (4, 7, 7), - "conv_out_shape": (100, 1, 1), - "spk_node_id": 18, - "pool_node_id": [], - "conv_rescaling_factor": None, - "dynapcnnlayer_destination": [7], - "nodes_destinations": {18: [19]}, - "entry_point": False, + "input_shape": (4, 7, 7), + "pool": [[1, 1]], + "rescale_factor": 1, + "rescale_factors": set(), + "destination_indices": [7], + "entry_node": False, }, 4: { - "dpcnnl_index": 4, - "conv_node_id": 8, - "conv_in_shape": (2, 34, 34), - "conv_out_shape": (4, 33, 33), - "spk_node_id": 9, - "pool_node_id": [], - "conv_rescaling_factor": None, - "dynapcnnlayer_destination": [5], - "nodes_destinations": {9: [10]}, - "entry_point": True, + "input_shape": (2, 34, 34), + "pool": [[1, 1]], + "rescale_factor": 1, + "rescale_factors": set(), + "destination_indices": [5], + "entry_node": True, }, 5: { - "dpcnnl_index": 5, - "conv_node_id": 10, - "conv_in_shape": (4, 33, 33), - "conv_out_shape": (4, 32, 32), - "spk_node_id": 11, - "pool_node_id": [12], - "conv_rescaling_factor": None, - "dynapcnnlayer_destination": [6], - "nodes_destinations": {12: [13]}, - "entry_point": False, + "input_shape": (4, 33, 33), + "pool": [[2, 2]], + "rescale_factor": 1, + "rescale_factors": set(), + "destination_indices": [6], + "entry_node": False, }, 6: { - "dpcnnl_index": 6, - "conv_node_id": 13, - "conv_in_shape": (4, 16, 16), - "conv_out_shape": (4, 15, 15), - "spk_node_id": 14, - "pool_node_id": [15], - "conv_rescaling_factor": None, - "dynapcnnlayer_destination": [3], - "nodes_destinations": {15: [17]}, - "entry_point": False, + "input_shape": (4, 16, 16), + "pool": [[2, 2]], + "rescale_factor": 1, + "rescale_factors": set(), + "destination_indices": [3], + "entry_node": False, }, 7: { - "dpcnnl_index": 7, - "conv_node_id": 19, - "conv_in_shape": (100, 1, 1), - "conv_out_shape": (100, 1, 1), - "spk_node_id": 20, - "pool_node_id": [], - "conv_rescaling_factor": None, - "dynapcnnlayer_destination": [8], - "nodes_destinations": {20: [21]}, - "entry_point": False, + "input_shape": (100, 1, 1), + "pool": [[1, 1]], + "rescale_factor": 1, + "rescale_factors": set(), + "destination_indices": [8], + "entry_node": False, }, 8: { - "dpcnnl_index": 8, - "conv_node_id": 21, - "conv_in_shape": (100, 1, 1), - "conv_out_shape": (10, 1, 1), - "spk_node_id": 22, - "pool_node_id": [], - "conv_rescaling_factor": None, - "dynapcnnlayer_destination": [], - "nodes_destinations": {}, - "entry_point": False, + "input_shape": (100, 1, 1), + "pool": [], + "rescale_factor": 1, + "rescale_factors": set(), + "destination_indices": [], + "entry_node": False, }, } diff --git a/tests/test_dynapcnnlayer/test_dynapcnnlayer.py b/tests/test_dynapcnnlayer/test_dynapcnnlayer.py index ec9588bf..f868e3fa 100644 --- a/tests/test_dynapcnnlayer/test_dynapcnnlayer.py +++ b/tests/test_dynapcnnlayer/test_dynapcnnlayer.py @@ -11,9 +11,7 @@ "dcnnl_map, discretize, rescale_fn, expected_output", args_DynapcnnLayer, ) -def test_DynapcnnLayer( - dcnnl_map, discretize, rescale_fn, expected_output -): +def test_DynapcnnLayer(dcnnl_map, discretize, rescale_fn, expected_output): """Tests the instantiation of a set of `DynapcnnLayer` belonging to the same SNN and the data computed within their constructors and shared among the differntly interacting instances (according to the graph described by `sinabs_edges`). @@ -23,9 +21,9 @@ def test_DynapcnnLayer( dynapcnn_layers, layer_handlers = construct_dynapcnnlayers_from_mapper( dcnnl_map=dcnnl_map, discretize=discretize, rescale_fn=rescale_fn ) - + for layer_index, dynapcnn_layer in dynapcnn_layers.items(): - + # Test layer instance in_shape = expected_output[layer_index]["input_shape"] pool = expected_output[layer_index]["pool"] @@ -46,7 +44,6 @@ def test_DynapcnnLayer( dynapcnn_layer.rescale_weights == rescale_weights ), f"wrong 'DynapcnnLayer.in_shape': Should be {rescale_weights}." - # Test entries in layer info that are not directly repeated in layer or handler instances layer_info = dcnnl_map[layer_index] rescale_factors = expected_output[layer_index]["rescale_factors"] diff --git a/tests/test_graph_extractor/model_dummy_1.py b/tests/test_graph_extractor/model_dummy_1.py index 18c596be..053f21a0 100644 --- a/tests/test_graph_extractor/model_dummy_1.py +++ b/tests/test_graph_extractor/model_dummy_1.py @@ -115,7 +115,7 @@ def forward(self, x): "pool1a": 3, "conv2": 4, "adder": 5, - "iaf2": 6, + "iaf2": 6, "conv3": 7, "iaf3": 8, "flat": 9, From fb0e661dca63b155633ec9bc070b08572648aa40 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Fri, 11 Oct 2024 11:58:51 +0200 Subject: [PATCH 195/379] Finish updating dynapcnn layer unit tests --- .../conftest_dynapcnnlayer.py | 42 +-- tests/test_dynapcnnlayer/model_dummy_4.py | 333 +++++++++--------- .../test_dynapcnnlayer/test_dynapcnnlayer.py | 6 +- 3 files changed, 174 insertions(+), 207 deletions(-) diff --git a/tests/test_dynapcnnlayer/conftest_dynapcnnlayer.py b/tests/test_dynapcnnlayer/conftest_dynapcnnlayer.py index 7364910e..a4234d72 100644 --- a/tests/test_dynapcnnlayer/conftest_dynapcnnlayer.py +++ b/tests/test_dynapcnnlayer/conftest_dynapcnnlayer.py @@ -4,38 +4,16 @@ from model_dummy_1 import expected_output_1, dcnnl_map_1 from model_dummy_2 import expected_output_2, dcnnl_map_2 from model_dummy_3 import expected_output_3, dcnnl_map_3 +from model_dummy_4 import expected_output_4, dcnnl_map_4 -# from model_dummy_3 import expected_output_3, nodes_to_dcnnl_map_3, sinabs_edges_3 -# from model_dummy_4 import expected_output_4, nodes_to_dcnnl_map_4, sinabs_edges_4 - -# Args: dcnnl_map, discretize, rescale_fn, expected_output +# Args: dcnnl_map, discretize, expected_output args_DynapcnnLayer = [ - (dcnnl_map_1, True, None, expected_output_1), - (dcnnl_map_1, False, None, expected_output_1), - (dcnnl_map_2, True, None, expected_output_2), - (dcnnl_map_2, False, None, expected_output_2), - (dcnnl_map_3, True, None, expected_output_3), - (dcnnl_map_3, False, None, expected_output_3), - # (nodes_to_dcnnl_map_2, 0, sinabs_edges_2, [0], expected_output_2), - # (nodes_to_dcnnl_map_2, 1, sinabs_edges_2, [0], expected_output_2), - # (nodes_to_dcnnl_map_2, 2, sinabs_edges_2, [0], expected_output_2), - # (nodes_to_dcnnl_map_2, 3, sinabs_edges_2, [0], expected_output_2), - # (nodes_to_dcnnl_map_2, 4, sinabs_edges_2, [0], expected_output_2), - # (nodes_to_dcnnl_map_2, 5, sinabs_edges_2, [0], expected_output_2), - # (nodes_to_dcnnl_map_2, 6, sinabs_edges_2, [0], expected_output_2), - # (nodes_to_dcnnl_map_3, 0, sinabs_edges_3, [0, 8], expected_output_3), - # (nodes_to_dcnnl_map_3, 1, sinabs_edges_3, [0, 8], expected_output_3), - # (nodes_to_dcnnl_map_3, 2, sinabs_edges_3, [0, 8], expected_output_3), - # (nodes_to_dcnnl_map_3, 3, sinabs_edges_3, [0, 8], expected_output_3), - # (nodes_to_dcnnl_map_3, 4, sinabs_edges_3, [0, 8], expected_output_3), - # (nodes_to_dcnnl_map_3, 5, sinabs_edges_3, [0, 8], expected_output_3), - # (nodes_to_dcnnl_map_3, 6, sinabs_edges_3, [0, 8], expected_output_3), - # (nodes_to_dcnnl_map_3, 7, sinabs_edges_3, [0, 8], expected_output_3), - # (nodes_to_dcnnl_map_3, 8, sinabs_edges_3, [0, 8], expected_output_3), - # (nodes_to_dcnnl_map_4, 0, sinabs_edges_4, [0], expected_output_4), - # (nodes_to_dcnnl_map_4, 1, sinabs_edges_4, [0], expected_output_4), - # (nodes_to_dcnnl_map_4, 2, sinabs_edges_4, [0], expected_output_4), - # (nodes_to_dcnnl_map_4, 3, sinabs_edges_4, [0], expected_output_4), - # (nodes_to_dcnnl_map_4, 4, sinabs_edges_4, [0], expected_output_4), - # (nodes_to_dcnnl_map_4, 5, sinabs_edges_4, [0], expected_output_4), + (dcnnl_map_1, True, expected_output_1), + (dcnnl_map_1, False, expected_output_1), + (dcnnl_map_2, True, expected_output_2), + (dcnnl_map_2, False, expected_output_2), + (dcnnl_map_3, True, expected_output_3), + (dcnnl_map_3, False, expected_output_3), + (dcnnl_map_4, True, expected_output_4), + (dcnnl_map_4, False, expected_output_4), ] diff --git a/tests/test_dynapcnnlayer/model_dummy_4.py b/tests/test_dynapcnnlayer/model_dummy_4.py index cd083c53..f7244a4e 100644 --- a/tests/test_dynapcnnlayer/model_dummy_4.py +++ b/tests/test_dynapcnnlayer/model_dummy_4.py @@ -7,234 +7,223 @@ from sinabs.activation.surrogate_gradient_fn import PeriodicExponential from sinabs.layers import IAFSqueeze, SumPool2d -nodes_to_dcnnl_map_4 = { +dcnnl_map_4 = { 0: { - 0: { - "layer": nn.Conv2d(2, 1, kernel_size=(2, 2), stride=(1, 1), bias=False), - "input_shape": (2, 34, 34), - "output_shape": (1, 33, 33), - }, - 1: { - "layer": IAFSqueeze( + "input_shape": (2, 34, 34), + "inferred_input_shapes": set(), + "rescale_factors": set(), + "is_entry_node": True, + "conv": { + "module": nn.Conv2d(2, 1, kernel_size=(2, 2), stride=(1, 1), bias=False), + "node_id": 0, + }, + "neuron": { + "module": IAFSqueeze( batch_size=2, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential(), ), - "input_shape": (1, 33, 33), - "output_shape": (1, 33, 33), - }, - "destinations": [1, 2], - "conv_rescale_factor": [], + "node_id": 1, + }, + "destinations": [ + { + "pooling_ids": [], + "pooling_modules": [], + "destination_layer": 1, + "output_shape": (1, 33, 33), + }, + { + "pooling_ids": [], + "pooling_modules": [], + "destination_layer": 2, + "output_shape": (1, 33, 33), + }, + ], }, 1: { - 2: { - "layer": nn.Conv2d(1, 1, kernel_size=(2, 2), stride=(1, 1), bias=False), - "input_shape": (1, 33, 33), - "output_shape": (1, 32, 32), - }, - 4: { - "layer": IAFSqueeze( + "input_shape": (1, 33, 33), + "inferred_input_shapes": set(((1, 33, 33),)), + "rescale_factors": set(), + "is_entry_node": False, + "conv": { + "module": nn.Conv2d(1, 1, kernel_size=(2, 2), stride=(1, 1), bias=False), + "node_id": 2, + }, + "neuron": { + "module": IAFSqueeze( batch_size=2, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential(), ), - "input_shape": (1, 32, 32), - "output_shape": (1, 32, 32), - }, - 5: { - "layer": SumPool2d(kernel_size=2, stride=2, ceil_mode=False), - "input_shape": (1, 32, 32), - "output_shape": (1, 16, 16), - }, - "destinations": [3], - "conv_rescale_factor": [], + "node_id": 4, + }, + "destinations": [ + { + "pooling_ids": [5], + "pooling_modules": [SumPool2d(kernel_size=2, stride=2, ceil_mode=False)], + "destination_layer": 3, + "output_shape": (1, 16, 16), + }, + ], }, 2: { - 3: { - "layer": nn.Conv2d(1, 1, kernel_size=(2, 2), stride=(1, 1), bias=False), - "input_shape": (1, 33, 33), - "output_shape": (1, 32, 32), - }, - 7: { - "layer": IAFSqueeze( + "input_shape": (1, 33, 33), + "inferred_input_shapes": set(((1, 33, 33),)), + "rescale_factors": set(), + "is_entry_node": False, + "conv": { + "module": nn.Conv2d(1, 1, kernel_size=(2, 2), stride=(1, 1), bias=False), + "node_id": 3, + }, + "neuron": { + "module": IAFSqueeze( batch_size=2, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential(), ), - "input_shape": (1, 32, 32), - "output_shape": (1, 32, 32), - }, - 8: { - "layer": SumPool2d(kernel_size=2, stride=2, ceil_mode=False), - "input_shape": (1, 32, 32), - "output_shape": (1, 16, 16), - }, - 9: { - "layer": SumPool2d(kernel_size=5, stride=5, ceil_mode=False), - "input_shape": (1, 32, 32), - "output_shape": (1, 6, 6), - }, - "destinations": [3, 4], - "conv_rescale_factor": [], + "node_id": 7, + }, + "destinations": [ + { + "pooling_ids": [8], + "pooling_modules": [SumPool2d(kernel_size=2, stride=2, ceil_mode=False)], + "destination_layer": 3, + "output_shape": (1, 16, 16), + }, + { + "pooling_ids": [9], + "pooling_modules": [SumPool2d(kernel_size=5, stride=5, ceil_mode=False)], + "destination_layer": 4, + "output_shape": (1, 6, 6), + }, + ], }, 3: { - 11: { - "layer": nn.Conv2d(1, 1, kernel_size=(2, 2), stride=(1, 1), bias=False), - "input_shape": (1, 16, 16), - "output_shape": (1, 15, 15), - }, - 12: { - "layer": IAFSqueeze( + "input_shape": (1, 16, 16), + "inferred_input_shapes": set(((1, 16, 16),)), + "rescale_factors": set(), + "is_entry_node": False, + "conv": { + "module": nn.Conv2d(1, 1, kernel_size=(2, 2), stride=(1, 1), bias=False), + "node_id": 11, + }, + "neuron": { + "module": IAFSqueeze( batch_size=2, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential(), ), - "input_shape": (1, 15, 15), - "output_shape": (1, 15, 15), - }, - 13: { - "layer": SumPool2d(kernel_size=3, stride=3, ceil_mode=False), - "input_shape": (1, 15, 15), - "output_shape": (1, 5, 5), - }, - "destinations": [5], - "conv_rescale_factor": [], + "node_id": 12, + }, + "destinations": [ + { + "pooling_ids": [13], + "pooling_modules": [SumPool2d(kernel_size=3, stride=3, ceil_mode=False)], + "destination_layer": 5, + "output_shape": (1, 5, 5), + }, + ] }, 4: { - 10: { - "layer": nn.Conv2d(1, 1, kernel_size=(2, 2), stride=(1, 1), bias=False), - "input_shape": (1, 6, 6), - "output_shape": (1, 5, 5), - }, - 15: { - "layer": IAFSqueeze( + "input_shape": (1, 6, 6), + "inferred_input_shapes": set(((1, 6, 6),)), + "rescale_factors": set(), + "is_entry_node": False, + "conv": { + "module": nn.Conv2d(1, 1, kernel_size=(2, 2), stride=(1, 1), bias=False), + "node_id": 10, + }, + "neuron": { + "module": IAFSqueeze( batch_size=2, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential(), ), - "input_shape": (1, 5, 5), - "output_shape": (1, 5, 5), - }, - "destinations": [5], - "conv_rescale_factor": [], + "node_id": 12, + }, + "destinations": [ + { + "pooling_ids": [], + "pooling_modules": [], + "destination_layer": 5, + "output_shape": (1, 5, 5), + }, + ] }, 5: { - 16: { - "layer": nn.Linear(in_features=25, out_features=10, bias=False), - "input_shape": (1, 5, 5), - "output_shape": (10,), - }, - 17: { - "layer": IAFSqueeze( + "input_shape": (25, 1, 1), + "inferred_input_shapes": set(((1, 5, 5),)), + "rescale_factors": set(), + "is_entry_node": False, + "conv": { + "module": nn.Linear(in_features=25, out_features=10, bias=False), + "node_id": 16, + }, + "neuron": { + "module": IAFSqueeze( batch_size=2, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential(), ), - "input_shape": (10,), - "output_shape": (10,), + "node_id": 17, }, - "destinations": [], - "conv_rescale_factor": [], + "destinations": [] }, } -sinabs_edges_4 = [ - (0, 1), - (1, 2), - (1, 3), - (2, 4), - (4, 5), - (5, 11), - (3, 7), - (7, 8), - (7, 9), - (8, 11), - (9, 10), - (11, 12), - (12, 13), - (13, 16), - (15, 16), - (10, 15), - (16, 17), -] - expected_output_4 = { 0: { - "dpcnnl_index": 0, - "conv_node_id": 0, - "conv_in_shape": (2, 34, 34), - "conv_out_shape": (1, 33, 33), - "spk_node_id": 1, - "pool_node_id": [], - "conv_rescaling_factor": None, - "dynapcnnlayer_destination": [1, 2], - "nodes_destinations": {1: [2, 3]}, - "entry_point": True, + "input_shape": (2, 34, 34), + "pool": [[1, 1], [1, 1]], + "rescale_factor": 1, + "rescale_factors": set(), + "destination_indices": [1, 2], + "entry_node": True, }, 1: { - "dpcnnl_index": 1, - "conv_node_id": 2, - "conv_in_shape": (1, 33, 33), - "conv_out_shape": (1, 32, 32), - "spk_node_id": 4, - "pool_node_id": [5], - "conv_rescaling_factor": None, - "dynapcnnlayer_destination": [3], - "nodes_destinations": {5: [11]}, - "entry_point": False, + "input_shape": (1, 33, 33), + "pool": [[2, 2]], + "rescale_factor": 1, + "rescale_factors": set(), + "destination_indices": [3], + "entry_node": False, }, 2: { - "dpcnnl_index": 2, - "conv_node_id": 3, - "conv_in_shape": (1, 33, 33), - "conv_out_shape": (1, 32, 32), - "spk_node_id": 7, - "pool_node_id": [8, 9], - "conv_rescaling_factor": None, - "dynapcnnlayer_destination": [3, 4], - "nodes_destinations": {8: [11], 9: [10]}, - "entry_point": False, + "input_shape": (1, 33, 33), + "pool": [[2, 2], [5, 5]], + "rescale_factor": 1, + "rescale_factors": set(), + "destination_indices": [3, 4], + "entry_node": False, }, 3: { - "dpcnnl_index": 3, - "conv_node_id": 11, - "conv_in_shape": (1, 16, 16), - "conv_out_shape": (1, 15, 15), - "spk_node_id": 12, - "pool_node_id": [13], - "conv_rescaling_factor": None, - "dynapcnnlayer_destination": [5], - "nodes_destinations": {13: [16]}, - "entry_point": False, + "input_shape": (1, 16, 16), + "pool": [[3, 3]], + "rescale_factor": 1, + "rescale_factors": set(), + "destination_indices": [5], + "entry_node": False, }, 4: { - "dpcnnl_index": 4, - "conv_node_id": 10, - "conv_in_shape": (1, 6, 6), - "conv_out_shape": (1, 5, 5), - "spk_node_id": 15, - "pool_node_id": [], - "conv_rescaling_factor": None, - "dynapcnnlayer_destination": [5], - "nodes_destinations": {15: [16]}, - "entry_point": False, + "input_shape": (1, 6, 6), + "pool": [[1, 1]], + "rescale_factor": 1, + "rescale_factors": set(), + "destination_indices": [5], + "entry_node": False, }, 5: { - "dpcnnl_index": 5, - "conv_node_id": 16, - "conv_in_shape": (1, 5, 5), - "conv_out_shape": (10, 1, 1), - "spk_node_id": 17, - "pool_node_id": [], - "conv_rescaling_factor": None, - "dynapcnnlayer_destination": [], - "nodes_destinations": {}, - "entry_point": False, + "input_shape": (1, 5, 5), + "pool": [], + "rescale_factor": 1, + "rescale_factors": set(), + "destination_indices": [], + "entry_node": False, }, } diff --git a/tests/test_dynapcnnlayer/test_dynapcnnlayer.py b/tests/test_dynapcnnlayer/test_dynapcnnlayer.py index f868e3fa..b043956b 100644 --- a/tests/test_dynapcnnlayer/test_dynapcnnlayer.py +++ b/tests/test_dynapcnnlayer/test_dynapcnnlayer.py @@ -8,10 +8,10 @@ @pytest.mark.parametrize( - "dcnnl_map, discretize, rescale_fn, expected_output", + "dcnnl_map, discretize, expected_output", args_DynapcnnLayer, ) -def test_DynapcnnLayer(dcnnl_map, discretize, rescale_fn, expected_output): +def test_DynapcnnLayer(dcnnl_map, discretize, expected_output): """Tests the instantiation of a set of `DynapcnnLayer` belonging to the same SNN and the data computed within their constructors and shared among the differntly interacting instances (according to the graph described by `sinabs_edges`). @@ -19,7 +19,7 @@ def test_DynapcnnLayer(dcnnl_map, discretize, rescale_fn, expected_output): # create a `DynapcnnLayer` from the set of layers in `nodes_to_dcnnl_map[dpcnnl_idx]`. dynapcnn_layers, layer_handlers = construct_dynapcnnlayers_from_mapper( - dcnnl_map=dcnnl_map, discretize=discretize, rescale_fn=rescale_fn + dcnnl_map=dcnnl_map, discretize=discretize, rescale_fn=None ) for layer_index, dynapcnn_layer in dynapcnn_layers.items(): From 8d4c34098f5aa923df82dc233c407fb02bb80d87 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Fri, 11 Oct 2024 12:18:20 +0200 Subject: [PATCH 196/379] Separate dynapcnn_layer_utils module --- .../backend/dynapcnn/dynapcnn_layer_utils.py | 285 ++++++++++++++++++ sinabs/backend/dynapcnn/dynapcnn_network.py | 2 +- .../backend/dynapcnn/sinabs_edges_handler.py | 1 + sinabs/backend/dynapcnn/utils.py | 281 ----------------- .../test_dynapcnnlayer/test_dynapcnnlayer.py | 2 +- 5 files changed, 288 insertions(+), 283 deletions(-) create mode 100644 sinabs/backend/dynapcnn/dynapcnn_layer_utils.py diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py b/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py new file mode 100644 index 00000000..984c7d63 --- /dev/null +++ b/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py @@ -0,0 +1,285 @@ +from math import prod +from typing import Callable, Dict, Iterable, Optional, Tuple, Union + +from torch import nn + +from sinabs import layers as sl + +from .dynapcnn_layer import DynapcnnLayer +from .dynapcnn_layer_handler import DynapcnnLayerHandler +from .utils import expand_to_pair + + +def construct_dynapcnnlayers_from_mapper( + dcnnl_map: Dict, discretize: bool, rescale_fn: Optional[Callable] = None +) -> Tuple[Dict[int, DynapcnnLayer], Dict[int, DynapcnnLayerHandler]]: + """Construct DynapcnnLayer and DynapcnnLayerHandler instances from + `dcnnl_map` + + Paramters + --------- + + Returns + ------- + - Dict of new DynapcnnLayer instances, with keys corresponding to `dcnnl_map` + - Dict of new DynapcnnLayerHandler instances, with keys corresponding + to `dcnnl_map` + """ + finalize_dcnnl_map(dcnnl_map, rescale_fn) + + dynapcnn_layers = { + layer_idx: construct_single_dynapcnn_layer(layer_info, discretize) + for layer_idx, layer_info in dcnnl_map.items() + } + + dynapcnn_layer_handlers = { + layer_idx: construct_single_dynapcnn_layer_handler(layer_idx, layer_info) + for layer_idx, layer_info in dcnnl_map.items() + } + + return dynapcnn_layers, dynapcnn_layer_handlers + + +def finalize_dcnnl_map(dcnnl_map: Dict, rescale_fn: Optional[Callable] = None): + """Finalize dcnnl map by consolidating information + + Update dcnnl_map in-place + - Consolidate chained pooling layers + - Determine rescaling of layer weights + - Fix input shapes + + Parameters + ---------- + - dcnnl_map: Dict holding info needed to instantiate DynapcnnLayer + and DynapcnnLayerHandler instances + - rescale_fn: Optional callable that is used to determine layer + rescaling in case of conflicting preceeding average pooling + """ + # Consolidate pooling information for each destination + for layer_info in dcnnl_map.values(): + consolidate_layer_pooling(layer_info, dcnnl_map) + + for layer_info in dcnnl_map.values(): + # Consolidate scale factors + consolidate_layer_scaling(layer_info, rescale_fn) + # Handle input dimensions + determine_layer_input_shape(layer_info) + + +def consolidate_layer_pooling(layer_info: Dict, dcnnl_map: Dict): + """Consolidate pooling information for individual layer + + Update `layer_info` and `dcnnl_map` in place. + - Extract pooling and scale factor of consecutive pooling operations + - To each "destination" add entries "cumulative_pooling" and + "cumulative_scaling" + - Add "pooling_list" to `layer_info` with all poolings of a layer + in order of its "destination"s. + - For each destination, add cumulative rescale factor to "rescale_factors" + entry in corresponding entry of `dcnnl_map`. + + Parameters + ---------- + - layer_info: Dict holding info of single layer. Corresponds to + single entry in `dcnnl_map` + - dcnnl_map: Dict holding info needed to instantiate DynapcnnLayer + and DynapcnnLayerHandler instances + """ + layer_info["pooling_list"] = [] + for destination in layer_info["destinations"]: + pool, scale = consolidate_dest_pooling(destination["pooling_modules"]) + destination["cumulative_pooling"] = pool + layer_info["pooling_list"].append(pool) + destination["cumulative_scaling"] = scale + dest_lyr_idx = destination["destination_layer"] + dcnnl_map[dest_lyr_idx]["rescale_factors"].add(scale) + + +def consolidate_dest_pooling( + modules: Iterable[nn.Module], +) -> Tuple[Tuple[int, int], float]: + """Consolidate pooling information for consecutive pooling modules + for single destination. + + Parameters + ---------- + modules: Iteravle of pooling modules + + Returns + ------- + cumulative_pooling: Tuple of two ints, indicating pooling along + vertical and horizontal dimensions for all modules together + cumulative_scaling: float, indicating by how much subsequent weights + need to be rescaled to account for average pooling being converted + to sum pooling, considering all provided modules. + """ + cumulative_pooling = [1, 1] + cumulative_scaling = 1.0 + + for pooling_layer in modules: + pooling, rescale_factor = extract_pooling_from_module(pooling_layer) + cumulative_pooling[0] *= pooling[0] + cumulative_pooling[1] *= pooling[1] + cumulative_scaling *= rescale_factor + + return cumulative_pooling, cumulative_scaling + + +def extract_pooling_from_module( + pooling_layer: Union[nn.AvgPool2d, sl.SumPool2d] +) -> Tuple[Tuple[int, int], float]: + """Extract pooling size and required rescaling factor from pooling module + + Parameters + ---------- + pooling_layer: pooling module + + Returns + ------- + pooling: Tuple of two ints, indicating pooling along vertical and horizontal dimensions + scale_factor: float, indicating by how much subsequent weights need to be rescaled to + account for average pooling being converted to sum pooling. + """ + pooling = expand_to_pair(pooling_layer.kernel_size) + + if pooling_layer.stride is not None: + stride = expand_to_pair(pooling_layer.stride) + if pooling != stride: + raise ValueError( + f"Stride length {pooling_layer.stride} should be the same as pooling kernel size {pooling_layer.kernel_size}" + ) + if isinstance(pooling_layer, nn.AvgPool2d): + scale_factor = 1.0 / (pooling[0] * pooling[1]) + elif isinstance(pooling_layer, sl.SumPool2d): + scale_factor = 1.0 + else: + raise ValueError(f"Unsupported type {type(pooling_layer)} for pooling layer") + + return pooling, scale_factor + + +def consolidate_layer_scaling(layer_info: Dict, rescale_fn: Optional[Callable] = None): + """Dertermine scale factor of single layer + + Add "rescale_factor" entry to `layer_info`. If more than one + different rescale factors have been determined due to conflicting + average pooling in preceding layers, requrie `rescale_fn` to + resolve. + + Parameters + ---------- + - layer_info: Dict holding info of single layer. + - rescale_fn: Optional callable that is used to determine layer + rescaling in case of conflicting preceeding average pooling + """ + if len(layer_info["rescale_factors"]) == 0: + rescale_factor = 1 + elif len(layer_info["rescale_factors"]) == 1: + rescale_factor = layer_info["rescale_factors"].pop() + else: + if rescale_fn is None: + # TODO: Custom Exception class? + raise ValueError( + "Average pooling layers of conflicting sizes pointing to " + "same destination. Either replace them by SumPool2d layers " + "or provide a `rescale_fn` to resolve this" + ) + else: + rescale_factor = rescale_fn(layer_info["rescale_factors"]) + layer_info["rescale_factor"] = rescale_factor + + +def determine_layer_input_shape(layer_info: Dict): + """Determine input shape of single layer + + Update "input_shape" entry of `layer_info`. + If weight layer is convolutional, only verify that output shapes + of preceding layer are not greater than input shape in any dimension. + + If weight layer is linear, the current "input_shape" entry will + correspond to the shape after flattening, which might not match + the shape of the actual input to the layer. Therefore the new input + shape is the largest size across all output shapes of preceding + layers, for each dimension individually. + Verify that total number of elements (product of entries in new + input shape) does not exceed that of original input shape. + + Parameters + ---------- + - layer_info: Dict holding info of single layer. + """ + # For each dimension find largest inferred input size + max_inferred_input_shape = [ + max(sizes) for sizes in zip(*layer_info["inferred_input_shapes"]) + ] + + if isinstance(layer_info["conv"]["module"], nn.Linear): + if prod(max_inferred_input_shape) > prod(layer_info["input_shape"]): + raise ValueError( + "Combined output of some layers projecting to a linear layer is " + "larger than expected by destination layer. " + ) + # Take shape before flattening, to convert linear to conv layer + layer_info["input_shape"] = max_inferred_input_shape + else: + if any( + inferred > expected + for inferred, expected in zip( + max_inferred_input_shape, layer_info["input_shape"] + ) + ): + raise ValueError( + "Output of some layers is larger than expected by destination " + "layer along some dimensions." + ) + + +def construct_single_dynapcnn_layer( + layer_info: Dict, discretize: bool +) -> DynapcnnLayer: + """Instantiate a DynapcnnLayer instance from the information + in `layer_info' + + Parameters + ---------- + - layer_info: Dict holding info of single layer. + - discretize: bool indicating whether layer parameters should be + discretized (weights, biases, thresholds) + + Returns + ------- + """ + return DynapcnnLayer( + conv=layer_info["conv"]["module"], + spk=layer_info["neuron"]["module"], + in_shape=layer_info["input_shape"], + pool=layer_info["pooling_list"], + discretize=discretize, + rescale_weights=layer_info["rescale_factor"], + ) + + +def construct_single_dynapcnn_layer_handler( + layer_index: int, layer_info: Dict +) -> DynapcnnLayerHandler: + """Instantiate a DynapcnnLayerHandler instance from the + information in `layer_info' + + Parameters + ---------- + - layer_index: Global index of the layer + - layer_info: Dict holding info of single layer. + + Returns + ------- + New DynapcnnLayerHandler instance + """ + destination_indices = [ + dest["destination_layer"] for dest in layer_info["destinations"] + ] + return DynapcnnLayerHandler( + layer_index=layer_index, + is_entry_node=layer_info["is_entry_node"], + destination_indices=destination_indices, + assigned_core=None, + ) \ No newline at end of file diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index b310e1f1..64e15647 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -14,6 +14,7 @@ from .chip_factory import ChipFactory from .dvs_layer import DVSLayer from .dynapcnn_layer import DynapcnnLayer +from .dynapcnn_layer_utils import construct_dynapcnnlayers_from_mapper from .dynapcnnnetwork_module import DynapcnnNetworkModule from .io import disable_timestamps, enable_timestamps, open_device, reset_timestamps from .nir_graph_extractor import GraphExtractor @@ -21,7 +22,6 @@ from .utils import ( DEFAULT_IGNORED_LAYER_TYPES, Edge, - construct_dynapcnnlayers_from_mapper, parse_device_id, topological_sorting, ) diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index 0d72c9bd..eba1d33d 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -337,6 +337,7 @@ def set_pooling_layer_destination( matched = False for destination in layer_info["destinations"]: if destination["pooling_ids"][-1] == edge[0]: + # TODO: Add unit test for such a case if "destination_layer" in destination: # Destination is already linked to a postsynaptic layer. This happens when # pooling nodes have outgoing edges to different weight layer. diff --git a/sinabs/backend/dynapcnn/utils.py b/sinabs/backend/dynapcnn/utils.py index a7a2e066..80fa3820 100644 --- a/sinabs/backend/dynapcnn/utils.py +++ b/sinabs/backend/dynapcnn/utils.py @@ -1,11 +1,8 @@ from collections import defaultdict, deque from copy import deepcopy -from math import prod from typing import ( TYPE_CHECKING, - Callable, Dict, - Iterable, List, Optional, Set, @@ -20,8 +17,6 @@ from .crop2d import Crop2d from .dvs_layer import DVSLayer, expand_to_pair -from .dynapcnn_layer import DynapcnnLayer -from .dynapcnn_layer_handler import DynapcnnLayerHandler from .flipdims import FlipDims if TYPE_CHECKING: @@ -89,282 +84,6 @@ def standardize_device_id(device_id: str) -> str: ####################################################### DynapcnnNetwork Related ####################################################### - -def construct_dynapcnnlayers_from_mapper( - dcnnl_map: Dict, discretize: bool, rescale_fn: Optional[Callable] = None -) -> Tuple[Dict[int, DynapcnnLayer], Dict[int, DynapcnnLayerHandler]]: - """Construct DynapcnnLayer and DynapcnnLayerHandler instances from - `dcnnl_map` - - Paramters - --------- - - Returns - ------- - - Dict of new DynapcnnLayer instances, with keys corresponding to `dcnnl_map` - - Dict of new DynapcnnLayerHandler instances, with keys corresponding - to `dcnnl_map` - """ - finalize_dcnnl_map(dcnnl_map, rescale_fn) - - dynapcnn_layers = { - layer_idx: construct_single_dynapcnn_layer(layer_info, discretize) - for layer_idx, layer_info in dcnnl_map.items() - } - - dynapcnn_layer_handlers = { - layer_idx: construct_single_dynapcnn_layer_handler(layer_idx, layer_info) - for layer_idx, layer_info in dcnnl_map.items() - } - - return dynapcnn_layers, dynapcnn_layer_handlers - - -def finalize_dcnnl_map(dcnnl_map: Dict, rescale_fn: Optional[Callable] = None): - """Finalize dcnnl map by consolidating information - - Update dcnnl_map in-place - - Consolidate chained pooling layers - - Determine rescaling of layer weights - - Fix input shapes - - Parameters - ---------- - - dcnnl_map: Dict holding info needed to instantiate DynapcnnLayer - and DynapcnnLayerHandler instances - - rescale_fn: Optional callable that is used to determine layer - rescaling in case of conflicting preceeding average pooling - """ - # Consolidate pooling information for each destination - for layer_info in dcnnl_map.values(): - consolidate_layer_pooling(layer_info, dcnnl_map) - - for layer_info in dcnnl_map.values(): - # Consolidate scale factors - consolidate_layer_scaling(layer_info, rescale_fn) - # Handle input dimensions - determine_layer_input_shape(layer_info) - - -def consolidate_layer_pooling(layer_info: Dict, dcnnl_map: Dict): - """Consolidate pooling information for individual layer - - Update `layer_info` and `dcnnl_map` in place. - - Extract pooling and scale factor of consecutive pooling operations - - To each "destination" add entries "cumulative_pooling" and - "cumulative_scaling" - - Add "pooling_list" to `layer_info` with all poolings of a layer - in order of its "destination"s. - - For each destination, add cumulative rescale factor to "rescale_factors" - entry in corresponding entry of `dcnnl_map`. - - Parameters - ---------- - - layer_info: Dict holding info of single layer. Corresponds to - single entry in `dcnnl_map` - - dcnnl_map: Dict holding info needed to instantiate DynapcnnLayer - and DynapcnnLayerHandler instances - """ - layer_info["pooling_list"] = [] - for destination in layer_info["destinations"]: - pool, scale = consolidate_dest_pooling(destination["pooling_modules"]) - destination["cumulative_pooling"] = pool - layer_info["pooling_list"].append(pool) - destination["cumulative_scaling"] = scale - dest_lyr_idx = destination["destination_layer"] - dcnnl_map[dest_lyr_idx]["rescale_factors"].add(scale) - - -def consolidate_dest_pooling( - modules: Iterable[nn.Module], -) -> Tuple[Tuple[int, int], float]: - """Consolidate pooling information for consecutive pooling modules - for single destination. - - Parameters - ---------- - modules: Iteravle of pooling modules - - Returns - ------- - cumulative_pooling: Tuple of two ints, indicating pooling along - vertical and horizontal dimensions for all modules together - cumulative_scaling: float, indicating by how much subsequent weights - need to be rescaled to account for average pooling being converted - to sum pooling, considering all provided modules. - """ - cumulative_pooling = [1, 1] - cumulative_scaling = 1.0 - - for pooling_layer in modules: - pooling, rescale_factor = extract_pooling_from_module(pooling_layer) - cumulative_pooling[0] *= pooling[0] - cumulative_pooling[1] *= pooling[1] - cumulative_scaling *= rescale_factor - - return cumulative_pooling, cumulative_scaling - - -def extract_pooling_from_module( - pooling_layer: Union[nn.AvgPool2d, sl.SumPool2d] -) -> Tuple[Tuple[int, int], float]: - """Extract pooling size and required rescaling factor from pooling module - - Parameters - ---------- - pooling_layer: pooling module - - Returns - ------- - pooling: Tuple of two ints, indicating pooling along vertical and horizontal dimensions - scale_factor: float, indicating by how much subsequent weights need to be rescaled to - account for average pooling being converted to sum pooling. - """ - pooling = expand_to_pair(pooling_layer.kernel_size) - - if pooling_layer.stride is not None: - stride = expand_to_pair(pooling_layer.stride) - if pooling != stride: - raise ValueError( - f"Stride length {pooling_layer.stride} should be the same as pooling kernel size {pooling_layer.kernel_size}" - ) - if isinstance(pooling_layer, nn.AvgPool2d): - scale_factor = 1.0 / (pooling[0] * pooling[1]) - elif isinstance(pooling_layer, sl.SumPool2d): - scale_factor = 1.0 - else: - raise ValueError(f"Unsupported type {type(pooling_layer)} for pooling layer") - - return pooling, scale_factor - - -def consolidate_layer_scaling(layer_info: Dict, rescale_fn: Optional[Callable] = None): - """Dertermine scale factor of single layer - - Add "rescale_factor" entry to `layer_info`. If more than one - different rescale factors have been determined due to conflicting - average pooling in preceding layers, requrie `rescale_fn` to - resolve. - - Parameters - ---------- - - layer_info: Dict holding info of single layer. - - rescale_fn: Optional callable that is used to determine layer - rescaling in case of conflicting preceeding average pooling - """ - if len(layer_info["rescale_factors"]) == 0: - rescale_factor = 1 - elif len(layer_info["rescale_factors"]) == 1: - rescale_factor = layer_info["rescale_factors"].pop() - else: - if rescale_fn is None: - # TODO: Custom Exception class? - raise ValueError( - "Average pooling layers of conflicting sizes pointing to " - "same destination. Either replace them by SumPool2d layers " - "or provide a `rescale_fn` to resolve this" - ) - else: - rescale_factor = rescale_fn(layer_info["rescale_factors"]) - layer_info["rescale_factor"] = rescale_factor - - -def determine_layer_input_shape(layer_info: Dict): - """Determine input shape of single layer - - Update "input_shape" entry of `layer_info`. - If weight layer is convolutional, only verify that output shapes - of preceding layer are not greater than input shape in any dimension. - - If weight layer is linear, the current "input_shape" entry will - correspond to the shape after flattening, which might not match - the shape of the actual input to the layer. Therefore the new input - shape is the largest size across all output shapes of preceding - layers, for each dimension individually. - Verify that total number of elements (product of entries in new - input shape) does not exceed that of original input shape. - - Parameters - ---------- - - layer_info: Dict holding info of single layer. - """ - # For each dimension find largest inferred input size - max_inferred_input_shape = [ - max(sizes) for sizes in zip(*layer_info["inferred_input_shapes"]) - ] - - if isinstance(layer_info["conv"]["module"], nn.Linear): - if prod(max_inferred_input_shape) > prod(layer_info["input_shape"]): - raise ValueError( - "Combined output of some layers projecting to a linear layer is " - "larger than expected by destination layer. " - ) - # Take shape before flattening, to convert linear to conv layer - layer_info["input_shape"] = max_inferred_input_shape - else: - if any( - inferred > expected - for inferred, expected in zip( - max_inferred_input_shape, layer_info["input_shape"] - ) - ): - raise ValueError( - "Output of some layers is larger than expected by destination " - "layer along some dimensions." - ) - - -def construct_single_dynapcnn_layer( - layer_info: Dict, discretize: bool -) -> DynapcnnLayer: - """Instantiate a DynapcnnLayer instance from the information - in `layer_info' - - Parameters - ---------- - - layer_info: Dict holding info of single layer. - - discretize: bool indicating whether layer parameters should be - discretized (weights, biases, thresholds) - - Returns - ------- - """ - return DynapcnnLayer( - conv=layer_info["conv"]["module"], - spk=layer_info["neuron"]["module"], - in_shape=layer_info["input_shape"], - pool=layer_info["pooling_list"], - discretize=discretize, - rescale_weights=layer_info["rescale_factor"], - ) - - -def construct_single_dynapcnn_layer_handler( - layer_index: int, layer_info: Dict -) -> DynapcnnLayerHandler: - """Instantiate a DynapcnnLayerHandler instance from the - information in `layer_info' - - Parameters - ---------- - - layer_index: Global index of the layer - - layer_info: Dict holding info of single layer. - - Returns - ------- - New DynapcnnLayerHandler instance - """ - destination_indices = [ - dest["destination_layer"] for dest in layer_info["destinations"] - ] - return DynapcnnLayerHandler( - layer_index=layer_index, - is_entry_node=layer_info["is_entry_node"], - destination_indices=destination_indices, - assigned_core=None, - ) - - def topological_sorting(edges: Set[Tuple[int, int]]) -> List[int]: """Performs a topological sorting (using Kahn's algorithm) of a graph descrobed by a list edges. An entry node `X` of the graph have to be flagged inside `edges` by a tuple `('input', X)`. diff --git a/tests/test_dynapcnnlayer/test_dynapcnnlayer.py b/tests/test_dynapcnnlayer/test_dynapcnnlayer.py index b043956b..eb359c80 100644 --- a/tests/test_dynapcnnlayer/test_dynapcnnlayer.py +++ b/tests/test_dynapcnnlayer/test_dynapcnnlayer.py @@ -4,7 +4,7 @@ import pytest from conftest_dynapcnnlayer import args_DynapcnnLayer -from sinabs.backend.dynapcnn.utils import construct_dynapcnnlayers_from_mapper +from sinabs.backend.dynapcnn.dynapcnn_layer_utils import construct_dynapcnnlayers_from_mapper @pytest.mark.parametrize( From 39192e991887974200a8cbc69bb5c76d5d5f92b5 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Mon, 14 Oct 2024 15:56:58 +0200 Subject: [PATCH 197/379] Enable pooling without subsequent destination layer --- .../backend/dynapcnn/dynapcnn_layer_utils.py | 4 ++-- .../backend/dynapcnn/sinabs_edges_handler.py | 21 +++++++++++-------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py b/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py index 984c7d63..353ab160 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py @@ -91,8 +91,8 @@ def consolidate_layer_pooling(layer_info: Dict, dcnnl_map: Dict): destination["cumulative_pooling"] = pool layer_info["pooling_list"].append(pool) destination["cumulative_scaling"] = scale - dest_lyr_idx = destination["destination_layer"] - dcnnl_map[dest_lyr_idx]["rescale_factors"].add(scale) + if (dest_lyr_idx := destination["destination_layer"]) is not None: + dcnnl_map[dest_lyr_idx]["rescale_factors"].add(scale) def consolidate_dest_pooling( diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index eba1d33d..6ef94943 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -80,6 +80,13 @@ def collect_dynapcnn_layer_info( entry_nodes, ) + # Process all edges connecting two dynapcnn layers that do not include pooling + while edges_by_type["neuron-weight"]: + edge = edges_by_type["neuron-weight"].pop() + set_neuron_layer_destination( + dynapcnn_layer_info, edge, node_2_layer_map, nodes_io_shapes + ) + # "pooling-pooling" edges are optional. Unlike other types, missing entry would cause exception. # Therefore add empty set if not existing if "pooling-pooling" not in edges_by_type: @@ -101,17 +108,12 @@ def collect_dynapcnn_layer_info( ) # Remove handled pooling-pooling edges edges_by_type["pooling-pooling"].difference_update(edges_used) + # After adding pooling make sure all pooling-pooling edges have been handled if len(edges_by_type["pooling-pooling"]) > 0: raise UnmatchedPoolingEdges(edges_by_type["pooling-pooling"]) - # Process all edges connecting two dynapcnn layers - while edges_by_type["neuron-weight"]: - edge = edges_by_type["neuron-weight"].pop() - set_neuron_layer_destination( - dynapcnn_layer_info, edge, node_2_layer_map, nodes_io_shapes - ) - + # Add all edges connecting pooling to a new dynapcnn layer while edges_by_type["pooling-weight"]: edge = edges_by_type["pooling-weight"].pop() set_pooling_layer_destination( @@ -246,6 +248,7 @@ def add_pooling_to_entry( { "pooling_ids": chain, "pooling_modules": [indx_2_module_map[idx] for idx in chain], + "destination_layer": None, } ) new_nodes.update(set(chain)) @@ -337,11 +340,11 @@ def set_pooling_layer_destination( matched = False for destination in layer_info["destinations"]: if destination["pooling_ids"][-1] == edge[0]: - # TODO: Add unit test for such a case - if "destination_layer" in destination: + if destination["destination_layer"] is not None: # Destination is already linked to a postsynaptic layer. This happens when # pooling nodes have outgoing edges to different weight layer. # Copy the destination + # TODO: Add unit test for this case destination = {k: v for k, v in destination.items()} layer_info["destinations"].append(destination) matched = True From 111a1a056c65794cb8f6ebb1ac9b95647beb3a90 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Mon, 14 Oct 2024 17:46:01 +0200 Subject: [PATCH 198/379] Final layer destinations get unique negative integers --- sinabs/backend/dynapcnn/dynapcnn_layer_utils.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py b/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py index 353ab160..e311073b 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py @@ -274,12 +274,19 @@ def construct_single_dynapcnn_layer_handler( ------- New DynapcnnLayerHandler instance """ - destination_indices = [ - dest["destination_layer"] for dest in layer_info["destinations"] - ] + destination_indices = [] + none_counter = 0 + for dest in layer_info["destinations"]: + if (dest_idx := dest["destination_layer"]) is None: + # For `None` destinations use unique negative index + none_counter += 1 + destination_indices.append(-none_counter) + else: + destination_indices.append(dest_idx) + return DynapcnnLayerHandler( layer_index=layer_index, is_entry_node=layer_info["is_entry_node"], destination_indices=destination_indices, assigned_core=None, - ) \ No newline at end of file + ) From 78752380fdb819d9565efe39484ff5ab224841c3 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Mon, 14 Oct 2024 17:46:53 +0200 Subject: [PATCH 199/379] (WIP) DynapcnnNetwork forward pass happens in DynapcnnNetworkModule. Still need to update tests --- sinabs/backend/dynapcnn/dynapcnn_layer.py | 5 +- sinabs/backend/dynapcnn/dynapcnn_network.py | 200 ++++--------- .../dynapcnn/dynapcnnnetwork_module.py | 269 ++++++++++++++---- 3 files changed, 266 insertions(+), 208 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer.py b/sinabs/backend/dynapcnn/dynapcnn_layer.py index 498b65dd..84463adc 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer.py @@ -180,7 +180,10 @@ def forward(self, x) -> List[torch.Tensor]: pool_out = sum_pool2d(x, kernel_size=pool) returns.append(pool_out) - return tuple(returns) + if len(returns) == 1: + return returns[0] + else: + return tuple(returns) def zero_grad(self, set_to_none: bool = False) -> None: """Call `zero_grad` method of spiking layer""" diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index 64e15647..5dec6474 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -94,7 +94,7 @@ def __init__( ) # build `DynapcnnLayer` instances from mapper. - self._dynapcnn_layers, self._dynapcnnlayers_handlers = ( + self._dynapcnn_layers, self._dynapcnnlayer_handlers = ( construct_dynapcnnlayers_from_mapper( dcnnl_map=self._dcnnl_map, discretize=discretize, @@ -102,42 +102,25 @@ def __init__( ) ) - # these gather all data necessay to implement the forward method for this class. - ( - self._dcnnl_edges, - self._layers_mapper, - self._merge_points, - self._topological_order, - ) = self._get_network_module() - - # all necessary `DynapcnnLayer` data held in `self._layers_mapper`: removing intermediary data structures no longer necessary. - del self._graph_extractor - del self._sinabs_edges - del self._sinabs_indx_2_module_map - del self._dcnnl_map - del self._dynapcnn_layers + # Module to execute forward pass through network + self._dynapcnn_module = DynapcnnNetworkModule( + self._dynapcnn_layers, self._dynapcnnlayer_handlers + ) + self.dynapcnn_module.setup_dynapcnnlayer_graph() ####################################################### Public Methods ####################################################### @property - def dcnnl_edges(self): - return self._dcnnl_edges - - @property - def merge_points(self): - return self._merge_points + def dynapcnn_layers(self): + return self._dynapcnn_layers @property - def topological_order(self): - return self._topological_order + def dynapcnnlayer_handlers(self): + return self._dynapcnnlayer_handlers @property - def layers_mapper(self) -> Dict[int, DynapcnnLayer]: - return self._layers_mapper - - @property - def layers_handlers(self): - return self._dynapcnnlayers_handlers + def dynapcnn_module(self): + return self._dynapcnn_module @property def chip_layers_ordering(self): @@ -194,76 +177,49 @@ def hw_forward(self, x): return received_evts - def forward(self, x): - """Forwards data through the `DynapcnnNetwork` instance. This method relies on three main data structures created to represent - the `DynapcnnLayer`s in the network and the data propagation through them during the forward pass: - - - `self._topological_order` (list): this is used to guide the sequence in which the `DynapcnnLayer`s in `self._layers_mapper` are to be called - to generate the input tensors to be propagated through the network during the forward pass. - - `self._dcnnl_edges` (list): this list of edges represent the graph describing the interactions between each `DynapcnnLayer` (the nodes in - the edges are the indices of these layers). An `edge` is used to index a mapper (using `edge[0]`) in order to retrieve the output to be fed - as input to a `DynapcnnLayer` instance (indexed by `edge[1]`). - - `self._layers_mapper` (dict): a mapper used to forward data through the `DynapcnnNetwork` instances. Each `key` is the indice associated - with a `DynapcnnLayer` instance. - - `self._merge_points` (dict): this mapper has a "support" role. It indexes wich convolutional layers in the set of `DynapcnnLayer`s - composing the network require two sources of input (because their input tensor is the output of a `Merge` layer). - """ - - layers_outputs = {} - - for i in self._topological_order: - - if self._dynapcnnlayers_handlers[i].entry_point: - # `DynapcnnLayer i` is an entry point of the network. - layers_outputs[i] = self._layers_mapper[i](x) - - else: - # input to `DynapcnnLayer i` is the output of another instance. - - if i in self._merge_points and i not in layers_outputs: - # there are two sources of input for `DynapcnnLayer i`. + def forward(self, x, return_complete: bool = False): + """Forwards data through the `DynapcnnNetwork` instance. - # by this points the arguments of the `Merge` associated with `i` should have been computed due to the topological sorting. - arg1, arg2 = self._merge_points[i]["sources"] + If the network has been deployed on a Dynapcnn/Speck device the forward + pass happens on the devices. Otherwise the device will be simulated by + passing the data through the `DynapcnnLayer` instances. - # find which returned tensor from the `forward` call of DynapcnnLayers `arg1` and `arg2` are to be fed - # to the target DynapcnnLayer `i`. - return_index_arg1 = self._dynapcnnlayers_handlers[ - arg1 - ].get_destination_dcnnl_index(i) - return_index_arg2 = self._dynapcnnlayers_handlers[ - arg2 - ].get_destination_dcnnl_index(i) - - # retrieve input tensors to `Merge`. - _arg1 = layers_outputs[arg1][return_index_arg1] - _arg2 = layers_outputs[arg2][return_index_arg2] - - # merge tensors. - merge_output = self._merge_points[i]["merge"](_arg1, _arg2) - - # call the forward. - layers_outputs[i] = self._layers_mapper[i](merge_output) - - else: - # there's a single source of input for `DynapcnnLayer i`. - - # input source for `i`. - src_dcnnl = self._get_input_to_dcnnl(i) - - # find which returned tensor from the `forward` call of the source DynapcnnLayer `src_dcnnl` is to be fed - # to the target DynapcnnLayer `i`. - return_index = self._dynapcnnlayers_handlers[ - src_dcnnl - ].get_destination_dcnnl_index(i) - - # call the forward. - layers_outputs[i] = self._layers_mapper[i]( - layers_outputs[src_dcnnl][return_index] - ) + Parameters + ---------- + x: Tensor that serves as input to network. Is passed to all layers + that are marked as entry points + return_complete: bool that indicates whether all layer outputs should + be return or only those with no further destinations (default) - # TODO - this assumes the network has a single output node. - return layers_outputs[self._topological_order[-1]][0] + Returns + ------- + The returned object depends on whether the network has been deployed + on chip. If this is the case, a flat list of samna events is returned, + in the order in which the events have been collected. + If the data is passed through the `DynapcnnLayer` instances, the output + depends on `return_complete` and on the network configuration: + * If `return_complete` is `True`, all layer outputs will be returned in a + dict, with layer indices as keys, and nested dicts as values, which + hold destination indices as keys and output tensors as values. + * If `return_complete` is `False` and there is only a single destination + in the whole network that is marked as final (i.e. destination + index in dynapcnn layer handler is negative), it will return the + output as a single tensor. + * If `return_complete` is `False` and no destination in the network + is marked as final, a warning will be raised and the function + returns an empty dict. + * In all other cases a dict will be returned that is of the same + structure as if `return_complete` is `True`, but only with entries + where the destination is marked as final. + """ + if ( + hasattr(self, "device") + and parse_device_id(self.device)[0] in ChipFactory.supported_devices + ): + return self.hw_forward(x) + else: + # Forward pass through software DynapcnnLayer instance + return self.dynapcnn_module(x, return_complete=return_complete) def parameters(self) -> list: """Gathers all the parameters of the network in a list. This is done by accessing the convolutional layer in each `DynapcnnLayer`, @@ -415,13 +371,6 @@ def to( ####################################################### Private Methods ####################################################### - def _get_input_to_dcnnl(self, dcnnl_ID) -> int: - """Returns the ID of the first `DynapcnnLayer` forwarding its input to `dcnnl_ID`.""" - for edge in self._dcnnl_edges: - if edge[1] == dcnnl_ID: - return edge[0] - raise ValueError(f"DynapcnnLayer {dcnnl_ID} has no source of input.") - def _make_config( self, chip_layers_ordering: Union[Sequence[int], str] = "auto", @@ -540,51 +489,6 @@ def _make_config( else: raise ValueError(f"Generated config is not valid for {device}") - def _get_network_module(self) -> Union[list, dict, dict]: - """Uses the `DynapcnnLayer` instances in `self._dynapcnn_layers` and the connectivity between them to create three data structures - that guide the data forwarding between the layer during the forward pass. - - Returns - ---------- - - dcnnl_edges (list): edges, represented as tuples of `DynapcnnLayer` indices, used to guide the data forwarding through each `DynapcnnLayer` in forward method. - - forward_map (dict): have all the `DynapcnnLayer` (`value`), each being accessible via its index (`key`). Used to call `DynapcnnLayer.forward` in forward method. - - merge_points (dict): used to compose the inputs to a `DynapcnnLayer` that requires an input from a `Merge` layer. - - Notes - ---------- - - the property `DynapcnnLayer.assigned_core` is only set after `self.to(device='speck...')` is called. - """ - - # get connections between `DynapcnnLayer`s. - dcnnl_edges = self._get_dynapcnnlayers_edges() - - dcnnnet_module = DynapcnnNetworkModule( - dcnnl_edges, self._dynapcnn_layers, self._dynapcnnlayers_handlers - ) - - return ( - dcnnnet_module.dcnnl_edges, - dcnnnet_module.forward_map, - dcnnnet_module.merge_points, - topological_sorting(dcnnl_edges), - ) - - def _get_dynapcnnlayers_edges(self) -> List[Edge]: - """Create edges representing connections between `DynapcnnLayer` instances. - - Returns - ---------- - - dcnnl_edges (list): a list of edges using the IDs of `DynapcnnLayer` instances. These edges describe the computational - graph implemented by the layers of the model (i.e., how the `DynapcnnLayer` instances address each other). - """ - dcnnl_edges = [] - - for dcnnl_idx, layer_data in self._dynapcnn_layers.items(): - for dest in layer_data["destinations"]: - dcnnl_edges.append((dcnnl_idx, dest)) - - return dcnnl_edges - def _to_device(self, device: torch.device) -> None: """Access each sub-layer within all `DynapcnnLayer` instances and call `.to(device)` on them.""" for layer in self._layers_mapper.values(): diff --git a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py index e61dbf26..4dd81ff0 100644 --- a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py +++ b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py @@ -1,99 +1,253 @@ # author : Willian Soares Girao # contact : wsoaresgirao@gmail.com +from collections import defaultdict import copy -from typing import Dict, List, Tuple, Union +from typing import Dict, List, Set, Tuple, Union +from warnings import warn import torch.nn as nn +from torch import Tensor import sinabs.layers as sl from .dynapcnn_layer import DynapcnnLayer +from .dynapcnn_layer_handler import DynapcnnLayerHandler +from .utils import Edge, topological_sorting -class DynapcnnNetworkModule: - """ - Uses the set of `DynapcnnLayer`\`DynapcnnLayerHandler` instances and how they address each other to define what the `forward` method of the model should do. +class DynapcnnNetworkModule(nn.Module): + """Allow forward (and backward) passing through a network of `DynapcnnLayer`s. + + Internally constructs a graph representation based on the provided + `DynapcnnLayer` and `DynapcnnLayerHandler` instances and uses this + to pass data through all layers in correct order. Parameters ---------- - - dcnnl_edges (list): tuples representing the output->input mapping between `DynapcnnLayer` instances - that have been used as configuration for each core `CNNLayerConifg`. - dynapcnn_layers (dict): a mapper containing `DynapcnnLayer` instances. - - dynapcnnlayers_handlers (dict): a mapper containing `DynapcnnLayerHandler` instances (hold network-level - data that was used to create the respective `DynapcnnLayer` instances in `dynapcnn_layers`). + - dynapcnnlayers_handlers (dict): a mapper containing `DynapcnnLayerHandler` instances + + Attributes + ---------- + This class internally builds a graph with `DynapcnnLayer` as nodes and their + connections as edges. Several data structures help efficient retrieval of + information required for the forward pass: + - `self._dynapcnnlayer_edges`: Set of edges connecting dynapcnn layers. Tuples + of indices of source and target layers. + - _sorted_nodes: List of layer indices in topological order, to ensure forward + calls to layers only happen when required inputs are available. + - _node_source_map: Dict with layer indices as keys and list of input layer indices + as values. """ def __init__( self, - dcnnl_edges: List[Tuple[int, int]], - dynapcnn_layers: Dict[int, dict], - dynapcnnlayers_handlers: Dict[int, dict], + dynapcnn_layers: Dict[int, DynapcnnLayerHandler], + dynapcnnlayers_handlers: Dict[int, DynapcnnLayerHandler], ): + super().__init__() + + self._dynapcnn_layers = dynapcnn_layers + self._dynapcnnlayer_handlers = dynapcnnlayers_handlers + + def setup_dynapcnnlayer_graph(self): + """Set up data structures to run forward pass through dynapcnn layers""" + self._dynapcnnlayer_edges = self.get_dynapcnnlayers_edges() + self.add_entry_points_edges(self._dynapcnnlayer_edges) + self._sorted_nodes = topological_sorting(self._dynapcnnlayer_edges) + self._node_source_map = self.get_node_source_map(self._dynapcnnlayer_edges) + # `Merge` layers are stateless. One instance can be used for all merge points. + self._merge_layer = sl.Merge() + + # TODO: Probably not needed. + # Collect layers with multiple inputs and instantiate `Merge` layers + # self._merge_points = self._get_merging_points(self._node_source_map) + + # # create mappers to handle `DynapcnnLayer` instances' forward calling. + # self.forward_map, self.merge_points = self._build_module_forward_from_graph( + # dcnnl_edges, dynapcnn_layers + # ) + + def get_dynapcnnlayers_edges(self) -> Set[Edge]: + """Create edges representing connections between `DynapcnnLayer` instances. + + Returns + ---------- + - dcnnl_edges: a set of edges using the IDs of `DynapcnnLayer` instances. These edges describe the computational + graph implemented by the layers of the model (i.e., how the `DynapcnnLayer` instances address each other). + """ + dcnnl_edges = set() + + for dcnnl_idx, handler in self._dynapcnnlayer_handlers.items(): + for dest in handler.destination_indices: + dcnnl_edges.add((dcnnl_idx, dest)) - self.dcnnl_edges = dcnnl_edges + return dcnnl_edges - # create mappers to handle `DynapcnnLayer` instances' forward calling. - self.forward_map, self.merge_points = self._build_module_forward_from_graph( - dcnnl_edges, dynapcnn_layers - ) + def add_entry_points_edges(self, dcnnl_edges: Set[Edge]): + """Add extra edges `('input', X)` to `dcnnl_edges` for + layers which are entry points of the `DynapcnnNetwork`, i.e. + `handler.entry_node = True`. - # add extra edges marking which nodes are input to the network. - self._add_entry_points_edges(dynapcnnlayers_handlers) + Parameters + ---------- + - dcnnl_edges (Set): tuples representing the output->input mapping between + `DynapcnnLayer` instances. Will be changed in place. + """ + for indx, handler in self._dynapcnnlayer_handlers.items(): + if handler.entry_node: + dcnnl_edges.add(("input", indx)) - def _add_entry_points_edges(self, dynapcnnlayers_handlers: dict) -> None: - """Addes an extra edge `('input', X)` to `self.dcnnl_edges` if `X` is an entry point of the `DynapcnnNetwork` - (i.e., `dynapcnnlayers_handlers[X]['layer_handler'].entry_point = True`). + def get_node_source_map(self, dcnnl_edges: Set[Edge]) -> Dict[int, List[int]]: + """From a set of edges, create a dict that maps to each node its sources Parameters ---------- - - dynapcnnlayers_handlers (dict): a mapper containing `DynapcnnLayerHandler` instances along with their supporting metadata (e.g. assigned core, - destination layers, etc.). + - dcnnl_edges (Set): tuples representing the output->input mapping between + `DynapcnnLayer` instances. + + Returns + ------- + - Dict with layer indices (int) as keys and list of layer indices that + map to corresponding layer """ - for indx, dcnnl_data in dynapcnnlayers_handlers.items(): - if dcnnl_data["layer_handler"].entry_point: - self.dcnnl_edges.append(("input", indx)) + sources = dict() + + for src, trg in dcnnl_edges: + if trg in sources: + sources[trg].append(src) + else: + sources[trg] = [src] + + return sources - def _spot_merging_points( - self, dcnnl_edges: list + # TODO: Probably not needed + def get_merging_points( + self, node_source_map: Dict[int, List[int]] ) -> Dict[int, Dict[Tuple, sl.Merge]]: - """Loops throught the edges of the computational graph from a `DynapcnnNetwork` to flag with nodes need - input from a `Merge` layer and what the arguments of this layer should be. + """Find nodes within `dcnnl_edges` that have multiple sources. Parameters ---------- - - dcnnl_edges (list): tuples representing the output->input mapping between `DynapcnnLayer` instances - that have been used as configuration for each core `CNNLayerConifg`. + - node_source_map: Dict that maps to each layer index (int) a list of + indices of layers that act as input source to this node + + Returns + ------- + - Dict that for each layer with more than one input source maps its index + (int) to a nested dict with two entries: + * "sources": Set of indices of all source layers to this layer + * "merge_layer": A `Merge` layer instance """ + return { + tgt: {"sources": sources, "merge_layer": sl.Merge()} + for tgt, sources in node_source_map + if len(sources) > 1 + } - nodes_with_merge_input = {} + def forward( + self, x, return_complete: bool = False + ) -> Union[Tensor, Dict[int, Dict[int, Tensor]]]: + """Perform a forward pass through all dynapcnn layers + The `setup_dynapcnnlayer_graph` method has to be executed beforehand. - for edge in dcnnl_edges: - trg_node = edge[1] - fan_in = 0 - src_nodes = [] - - # counts the fan-in for each target node `trg_node`. - for edge_inner in dcnnl_edges: - if edge_inner[1] == trg_node: - # fan-in update. - fan_in += 1 - src_nodes.append(edge_inner[0]) - - if fan_in == 2 and trg_node not in nodes_with_merge_input: - # node needs input from a `Merge` layer: instantiate `Merge` and its arguments. - nodes_with_merge_input[trg_node] = { - "sources": tuple(src_nodes), - "merge": sl.Merge(), + Parameters + ---------- + x: Tensor that serves as input to network. Is passed to all layers + that are marked as entry points + return_complete: bool that indicates whether all layer outputs should + be return or only those with no further destinations (default) + + Returns + ------- + The returned object depends on whether `return_complete` is set and on + the network configuration: + * If `return_complete` is `True`, all layer outputs will be returned in a + dict, with layer indices as keys, and nested dicts as values, which + hold destination indices as keys and output tensors as values. + * If `return_complete` is `False` and there is only a single destination + in the whole network that is marked as final (i.e. destination + index in dynapcnn layer handler is negative), it will return the + output as a single tensor. + * If `return_complete` is `False` and no destination in the network + is marked as final, a warning will be raised and the function + returns an empty dict. + * In all other cases a dict will be returned that is of the same + structure as if `return_complete` is `True`, but only with entries + where the destination is marked as final. + + """ + if not hasattr(self, "_sorted_nodes"): + raise RuntimeError( + "It looks like `setup_dynapcnnlayers_graph` has never been executed. " + "It needs to be called at least once before calling `forward`." + ) + + # For each layer store its outputs as dict with destination layers as keys. + # For input use `defaultdict` so it can be used for all destinations where needed + layers_outputs = {"input": defaultdict(lambda: x)} + + for idx_curr in self._sorted_nodes: + # Get inputs to the layer + if len(sources := self._node_source_map[idx_curr]) > 1: + # Layer has multiple inputs + inputs = [layers_outputs[idx_src][idx_curr] for idx_src in sources] + current_input = self._merge_layer(*inputs) + else: + idx_src = sources[0] + current_input = layers_outputs[idx_src][idx_curr] + + # Get current layer instance and destinations + layer = self._dynapcnn_layers[idx_curr] + destinations = self._dynapcnnlayer_handlers[idx_curr].destination_indices + + # Forward pass through layer + output = layer(current_input) + + # Store layer output for all destinations + if len(destinations) == 1: + # Output is single tensor + layers_outputs[idx_curr] = {destinations[0]: output} + else: + # Output is list of tensors for different destinations + layers_outputs[idx_curr] = { + idx_dest: out for idx_dest, out in zip(destinations, output) } - if fan_in > 2: - raise ValueError( - f"Node {trg_node} is the has fan-in of {fan_in}: only fan-in of 2 is currently handled." - ) + if return_complete: + return layers_outputs + + # Take outputs with final destinations as network output + network_outputs = {} + for layer_idx, outputs in layers_outputs.items(): + final_outputs = { + abs(idx_dest): out for idx_dest, out in outputs.items() if idx_dest < 0 + } + if final_outputs: + network_outputs[layer_idx] = final_outputs + + # If no outputs have been found return None and warn + if not network_outputs: + warn( + "No final outputs have been found. Try setting `return_complete` " + "`True` to get all outputs, or mark final outputs by setting " + "corresponding destination layer indices in DynapcnnLayerHandler " + " instance to negative integer values" + ) + return - return nodes_with_merge_input + # Special case with single output: return single tensor + if ( + len(network_outputs) == 1 + and len(out := (next(iter(network_outputs.values())))) == 1 + ): + return out + # If there is output from multiple layers return all of them in a dict + return network_outputs + + # TODO: Necessary? def _build_module_forward_from_graph( self, dcnnl_edges: list, dynapcnn_layers: dict ) -> Union[Dict[int, DynapcnnLayer], Dict[Tuple, sl.Merge]]: @@ -116,9 +270,6 @@ def _build_module_forward_from_graph( instances for which the ouput is to be used as the `Merge` arguments). """ - # mapper to flag nodes that need input from a `Merge` layer. - merge_points = self._spot_merging_points(dcnnl_edges) - # this dict. will be used to call the `forward` methods of each `DynapcnnLayer`. forward_map = {} @@ -136,4 +287,4 @@ def _build_module_forward_from_graph( dynapcnn_layers[trg_dcnnl]["layer"] ) - return forward_map, merge_points + return forward_map From ed6345bc9bbe28a7dbf4f17c5c1ea70b5662e4ff Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Mon, 14 Oct 2024 17:47:17 +0200 Subject: [PATCH 200/379] Rerun black --- sinabs/backend/dynapcnn/utils.py | 1 + tests/test_dynapcnnlayer/model_dummy_4.py | 22 +++++++++++++------ .../test_dynapcnnlayer/test_dynapcnnlayer.py | 4 +++- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/sinabs/backend/dynapcnn/utils.py b/sinabs/backend/dynapcnn/utils.py index 80fa3820..9e7acd3c 100644 --- a/sinabs/backend/dynapcnn/utils.py +++ b/sinabs/backend/dynapcnn/utils.py @@ -84,6 +84,7 @@ def standardize_device_id(device_id: str) -> str: ####################################################### DynapcnnNetwork Related ####################################################### + def topological_sorting(edges: Set[Tuple[int, int]]) -> List[int]: """Performs a topological sorting (using Kahn's algorithm) of a graph descrobed by a list edges. An entry node `X` of the graph have to be flagged inside `edges` by a tuple `('input', X)`. diff --git a/tests/test_dynapcnnlayer/model_dummy_4.py b/tests/test_dynapcnnlayer/model_dummy_4.py index f7244a4e..a8defe71 100644 --- a/tests/test_dynapcnnlayer/model_dummy_4.py +++ b/tests/test_dynapcnnlayer/model_dummy_4.py @@ -62,7 +62,9 @@ "destinations": [ { "pooling_ids": [5], - "pooling_modules": [SumPool2d(kernel_size=2, stride=2, ceil_mode=False)], + "pooling_modules": [ + SumPool2d(kernel_size=2, stride=2, ceil_mode=False) + ], "destination_layer": 3, "output_shape": (1, 16, 16), }, @@ -89,13 +91,17 @@ "destinations": [ { "pooling_ids": [8], - "pooling_modules": [SumPool2d(kernel_size=2, stride=2, ceil_mode=False)], + "pooling_modules": [ + SumPool2d(kernel_size=2, stride=2, ceil_mode=False) + ], "destination_layer": 3, "output_shape": (1, 16, 16), }, { "pooling_ids": [9], - "pooling_modules": [SumPool2d(kernel_size=5, stride=5, ceil_mode=False)], + "pooling_modules": [ + SumPool2d(kernel_size=5, stride=5, ceil_mode=False) + ], "destination_layer": 4, "output_shape": (1, 6, 6), }, @@ -122,11 +128,13 @@ "destinations": [ { "pooling_ids": [13], - "pooling_modules": [SumPool2d(kernel_size=3, stride=3, ceil_mode=False)], + "pooling_modules": [ + SumPool2d(kernel_size=3, stride=3, ceil_mode=False) + ], "destination_layer": 5, "output_shape": (1, 5, 5), }, - ] + ], }, 4: { "input_shape": (1, 6, 6), @@ -153,7 +161,7 @@ "destination_layer": 5, "output_shape": (1, 5, 5), }, - ] + ], }, 5: { "input_shape": (25, 1, 1), @@ -173,7 +181,7 @@ ), "node_id": 17, }, - "destinations": [] + "destinations": [], }, } diff --git a/tests/test_dynapcnnlayer/test_dynapcnnlayer.py b/tests/test_dynapcnnlayer/test_dynapcnnlayer.py index eb359c80..66aa5d39 100644 --- a/tests/test_dynapcnnlayer/test_dynapcnnlayer.py +++ b/tests/test_dynapcnnlayer/test_dynapcnnlayer.py @@ -4,7 +4,9 @@ import pytest from conftest_dynapcnnlayer import args_DynapcnnLayer -from sinabs.backend.dynapcnn.dynapcnn_layer_utils import construct_dynapcnnlayers_from_mapper +from sinabs.backend.dynapcnn.dynapcnn_layer_utils import ( + construct_dynapcnnlayers_from_mapper, +) @pytest.mark.parametrize( From 537c7a71cea9207bf60b3978ce7e14704de40664 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Mon, 14 Oct 2024 17:55:20 +0200 Subject: [PATCH 201/379] Add complete type hint to DynapcnnNetwork.forward --- sinabs/backend/dynapcnn/dynapcnn_network.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index 5dec6474..d8e6b57d 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -7,6 +7,7 @@ import samna import torch import torch.nn as nn +from torch import Tensor import sinabs import sinabs.layers as sl @@ -177,7 +178,9 @@ def hw_forward(self, x): return received_evts - def forward(self, x, return_complete: bool = False): + def forward( + self, x, return_complete: bool = False + ) -> Union[List["event"], Tensor, Dict[int, Dict[int, Tensor]]]: """Forwards data through the `DynapcnnNetwork` instance. If the network has been deployed on a Dynapcnn/Speck device the forward From 227b484557a38e4339a138414c856e86a1afcae3 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Tue, 15 Oct 2024 16:27:24 +0200 Subject: [PATCH 202/379] Update dynapcnn network unit tests --- tests/test_dynapcnnnetwork/model_dummy_1.py | 23 +++++++--- tests/test_dynapcnnnetwork/model_dummy_2.py | 33 ++++++++++---- tests/test_dynapcnnnetwork/model_dummy_3.py | 45 ++++++++++++++----- tests/test_dynapcnnnetwork/model_dummy_4.py | 28 ++++++++---- .../test_dynapcnnnetwork.py | 38 +++++++++------- .../test_graph_extractor.py | 8 ++-- 6 files changed, 121 insertions(+), 54 deletions(-) diff --git a/tests/test_dynapcnnnetwork/model_dummy_1.py b/tests/test_dynapcnnnetwork/model_dummy_1.py index 0c32e959..c0ad5737 100644 --- a/tests/test_dynapcnnnetwork/model_dummy_1.py +++ b/tests/test_dynapcnnnetwork/model_dummy_1.py @@ -91,16 +91,29 @@ def forward(self, x): snn = SNN(batch_size) expected_output = { - "dcnnl_edges": [ + "dcnnl_edges": { (0, 1), (0, 2), (1, 2), (2, 3), (3, 4), ("input", 0), - ], - "merge_points": {2: {"sources": (0, 1), "merge": Merge()}}, - "topological_order": [0, 1, 2, 3, 4], + }, + "node_source_map": { + 0: {"input"}, + 1: {0}, + 2: {0, 1}, + 3: {2}, + 4: {3}, + }, + "destination_map": { + 0: {1, 2}, + 1: {2}, + 2: {3}, + 3: {4}, + 4: {-1}, + }, + "entry_points": {0}, + "sorted_nodes": [0, 1, 2, 3, 4], "output_shape": torch.Size([3, 10, 1, 1]), - "entry_point": [0], } diff --git a/tests/test_dynapcnnnetwork/model_dummy_2.py b/tests/test_dynapcnnnetwork/model_dummy_2.py index bae7908f..22e645cc 100644 --- a/tests/test_dynapcnnnetwork/model_dummy_2.py +++ b/tests/test_dynapcnnnetwork/model_dummy_2.py @@ -127,18 +127,35 @@ def forward(self, x): snn = SNN(batch_size) expected_output = { - "dcnnl_edges": [ + "dcnnl_edges": { (0, 1), (1, 2), (1, 3), (2, 4), - (3, 6), - (4, 5), - (6, 5), + (3, 5), + (4, 6), + (5, 6), ("input", 0), - ], - "merge_points": {5: {"sources": (4, 6), "merge": Merge()}}, - "topological_order": [0, 1, 2, 3, 4, 6, 5], + }, + "node_source_map": { + 0: {"input"}, + 1: {0}, + 2: {1}, + 3: {1}, + 4: {2}, + 5: {3}, + 6: {4, 5}, + }, + "destination_map": { + 0: {1}, + 1: {2, 3}, + 2: {4}, + 3: {5}, + 4: {6}, + 5: {6}, + 6: {-1}, + }, + "sorted_nodes": [0, 1, 2, 3, 4, 5, 6], "output_shape": torch.Size([8, 10, 1, 1]), - "entry_point": [0], + "entry_points": {0}, } diff --git a/tests/test_dynapcnnnetwork/model_dummy_3.py b/tests/test_dynapcnnnetwork/model_dummy_3.py index a67156bc..1decc3d6 100644 --- a/tests/test_dynapcnnnetwork/model_dummy_3.py +++ b/tests/test_dynapcnnnetwork/model_dummy_3.py @@ -148,20 +148,41 @@ def forward(self, x): snn = SNN(batch_size) expected_output = { - "dcnnl_edges": [ - (0, 1), - (1, 2), - (2, 3), - (3, 7), - (4, 5), + "dcnnl_edges": { + (0, 2), + (2, 4), + (4, 6), + (6, 7), + (1, 3), + (3, 5), (5, 6), - (6, 3), (7, 8), ("input", 0), - ("input", 4), - ], - "merge_points": {3: {"sources": (2, 6), "merge": Merge()}}, - "topological_order": [0, 4, 1, 5, 2, 6, 3, 7, 8], + ("input", 1), + }, + "node_source_map": { + 0: {"input"}, + 2: {0}, + 4: {2}, + 6: {4, 5}, + 1: {"input"}, + 3: {1}, + 5: {3}, + 7: {6}, + 8: {7}, + }, + "destination_map": { + 0: {2}, + 2: {4}, + 4: {6}, + 6: {7}, + 1: {3}, + 3: {5}, + 5: {6}, + 7: {8}, + 8: {-1}, + }, + "sorted_nodes": [0, 1, 2, 3, 4, 5, 6, 7, 8], "output_shape": torch.Size([2, 10, 1, 1]), - "entry_point": [0, 4], + "entry_points": {0, 1}, } diff --git a/tests/test_dynapcnnnetwork/model_dummy_4.py b/tests/test_dynapcnnnetwork/model_dummy_4.py index b74a63cf..1655b5f5 100644 --- a/tests/test_dynapcnnnetwork/model_dummy_4.py +++ b/tests/test_dynapcnnnetwork/model_dummy_4.py @@ -118,21 +118,33 @@ def forward(self, x): snn = SNN(batch_size) expected_output = { - "dcnnl_edges": [ + "dcnnl_edges": { (0, 1), (0, 2), + (1, 4), (1, 3), - (2, 3), (2, 4), (3, 5), (4, 5), ("input", 0), - ], - "merge_points": { - 3: {"sources": (1, 2), "merge": Merge()}, - 5: {"sources": (3, 4), "merge": Merge()}, }, - "topological_order": [0, 1, 2, 3, 4, 5], + "node_source_map": { + 0: {"input"}, + 1: {0}, + 2: {0}, + 3: {1}, + 4: {1, 2}, + 5: {3, 4}, + }, + "destination_map": { + 0: {1, 2}, + 1: {3, 4}, + 2: {4}, + 3: {5}, + 4: {5}, + 5: {-1}, + }, + "sorted_nodes": [0, 1, 2, 3, 4, 5], "output_shape": torch.Size([2, 10, 1, 1]), - "entry_point": [0], + "entry_points": {0}, } diff --git a/tests/test_dynapcnnnetwork/test_dynapcnnnetwork.py b/tests/test_dynapcnnnetwork/test_dynapcnnnetwork.py index a86d78fe..95b190df 100644 --- a/tests/test_dynapcnnnetwork/test_dynapcnnnetwork.py +++ b/tests/test_dynapcnnnetwork/test_dynapcnnnetwork.py @@ -24,27 +24,31 @@ def test_DynapcnnNetwork(snn, input_shape, batch_size, expected_output): x = torch.randn((batch_size, *input_shape)) output = dcnnnet(x) + module = dcnnnet.dynapcnn_module assert ( - expected_output["dcnnl_edges"] == dcnnnet.dcnnl_edges - ), f"wrong list of edges describing DynapcnnLayer connectivity." + expected_output["dcnnl_edges"] == module._dynapcnnlayer_edges + ), "wrong list of edges describing DynapcnnLayer connectivity." - for node, args in dcnnnet.merge_points.items(): + # Convert source lists to sets to ignore order + source_map = {node: set(sources) for node, sources in module._node_source_map.items()} + assert( + expected_output["node_source_map"] == source_map + ), "wrong node source map" - assert ( - node in expected_output["merge_points"] - ), f"DynapcnnLayer {node} is not a merge point." - assert ( - args["sources"] == expected_output["merge_points"][node]["sources"] - ), f"DynapcnnLayer {node} has wrong input sources ({args})." + # Convert destination lists to sets to ignore order + destination_map = {node: set(dests) for node, dests in module._destination_map.items()} + assert( + expected_output["destination_map"] == destination_map + ), "wrong destination map" - for entry_point in expected_output["entry_point"]: - assert dcnnnet.layers_mapper[ - entry_point - ].entry_point, f"DynapcnnLayer {entry_point} should be an entry point." + assert( + expected_output["entry_points"] == module._entry_points + ), "wrong entry points" + + assert( + expected_output["sorted_nodes"] == module._sorted_nodes + ), "wrong node sorting" - assert ( - expected_output["topological_order"] == dcnnnet.topological_order - ), f"wrong topological ordering between DynapcnnLayers." assert ( expected_output["output_shape"] == output.shape - ), f"wrong model output tensor shape." + ), "wrong model output tensor shape." diff --git a/tests/test_graph_extractor/test_graph_extractor.py b/tests/test_graph_extractor/test_graph_extractor.py index ef333ada..20acdd16 100644 --- a/tests/test_graph_extractor/test_graph_extractor.py +++ b/tests/test_graph_extractor/test_graph_extractor.py @@ -55,13 +55,13 @@ def test_GraphExtractor(snn, input_dummy, expected_output): assert ( expected_output["edges"] == graph_tracer.edges - ), f"wrong list of edges extracted from the SNN." + ), "wrong list of edges extracted from the SNN." assert ( expected_output["name_2_indx_map"] == graph_tracer.name_2_indx_map - ), f"wrong mapping from layer variable name to node ID." + ), "wrong mapping from layer variable name to node ID." assert ( expected_output["entry_nodes"] == graph_tracer.entry_nodes - ), f"wrong list with entry node's IDs (i.e., layers serving as input to the SNN)." + ), "wrong list with entry node's IDs (i.e., layers serving as input to the SNN)." assert ( expected_output["nodes_io_shapes"] == graph_tracer.nodes_io_shapes - ), f"wrong I/O shapes computed for one or more nodes." + ), "wrong I/O shapes computed for one or more nodes." From bb3e2110fa15bb8297ffa89be6ce1fccc0b7d9fc Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Tue, 15 Oct 2024 16:29:22 +0200 Subject: [PATCH 203/379] Ensure exit layers generate output by setting destination None --- .../backend/dynapcnn/sinabs_edges_handler.py | 37 +++++++++++++++++-- sinabs/backend/dynapcnn/utils.py | 1 - 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index 6ef94943..6cce7a09 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -123,6 +123,9 @@ def collect_dynapcnn_layer_info( # Make sure we have taken care of all edges assert all(len(edges) == 0 for edges in edges_by_type.values()) + # Set minimal destination entries for layers without child nodes, to act as network outputs + set_exit_destinations(dynapcnn_layer_info) + return dynapcnn_layer_info @@ -206,7 +209,7 @@ def init_new_dynapcnnlayer_entry( def add_pooling_to_entry( - dynapcnn_layer_info: Dict[int, Dict[int, Dict]], + dynapcnn_layer_info: Dict[int, Dict], edge: Edge, pooling_chains: List[deque[int]], indx_2_module_map: Dict[int, nn.Module], @@ -248,6 +251,8 @@ def add_pooling_to_entry( { "pooling_ids": chain, "pooling_modules": [indx_2_module_map[idx] for idx in chain], + # Setting `destination_layer` to `None` allows for this layer + # to act as network exit point if not destination is added later "destination_layer": None, } ) @@ -259,8 +264,34 @@ def add_pooling_to_entry( node_2_layer_map[node] = layer_idx +def set_exit_destinations(dynapcnn_layer: Dict) -> None: + """Set minimal destination entries for layers that don't have any. + + This ensures that the forward methods of the resulting DynapcnnLayer + instances return an output, letting these layers act as exit points + of the network. + The destination layer will be `None`, and no pooling applied. + + Parameters + ---------- + dynapcnn_layer_info: Dict with one entry for each future dynapcnn layer. + key is unique dynapcnn layer ID, value is dict with nodes of the layer + Will be updated in-place. + """ + for layer_info in dynapcnn_layer.values(): + if not (destinations := layer_info["destinations"]): + # Add `None` destination to empty destination lists + destinations.append( + { + "pooling_ids": [], + "pooling_modules": [], + "destination_layer": None, + } + ) + + def set_neuron_layer_destination( - dynapcnn_layer_info: Dict[int, Dict[int, Dict]], + dynapcnn_layer_info: Dict[int, Dict], edge: Edge, node_2_layer_map: Dict[int, int], nodes_io_shapes: Dict[int, Dict[str, Tuple[Size, Size]]], @@ -307,7 +338,7 @@ def set_neuron_layer_destination( def set_pooling_layer_destination( - dynapcnn_layer_info: Dict[int, Dict[int, Dict]], + dynapcnn_layer_info: Dict[int, Dict], edge: Edge, node_2_layer_map: Dict[int, int], nodes_io_shapes: Dict[int, Dict[str, Tuple[Size, Size]]], diff --git a/sinabs/backend/dynapcnn/utils.py b/sinabs/backend/dynapcnn/utils.py index 9e7acd3c..74f8016a 100644 --- a/sinabs/backend/dynapcnn/utils.py +++ b/sinabs/backend/dynapcnn/utils.py @@ -2,7 +2,6 @@ from copy import deepcopy from typing import ( TYPE_CHECKING, - Dict, List, Optional, Set, From 25a14d2bf5b06372cbb1604f9fa6d04ea7628fab Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Tue, 15 Oct 2024 16:42:27 +0200 Subject: [PATCH 204/379] Remove need for DynapcnnLayerHandler (WIP) --- sinabs/backend/dynapcnn/__init__.py | 2 +- .../dynapcnn/dynapcnn_layer_handler.py | 141 ------------- .../backend/dynapcnn/dynapcnn_layer_utils.py | 84 ++++---- sinabs/backend/dynapcnn/dynapcnn_network.py | 15 +- .../dynapcnn/dynapcnnnetwork_module.py | 191 +++++++++--------- 5 files changed, 144 insertions(+), 289 deletions(-) delete mode 100644 sinabs/backend/dynapcnn/dynapcnn_layer_handler.py diff --git a/sinabs/backend/dynapcnn/__init__.py b/sinabs/backend/dynapcnn/__init__.py index 363723c2..11ab0fd9 100644 --- a/sinabs/backend/dynapcnn/__init__.py +++ b/sinabs/backend/dynapcnn/__init__.py @@ -1,4 +1,4 @@ from .dynapcnn_layer import DynapcnnLayer -from .dynapcnn_layer_handler import DynapcnnLayerHandler from .dynapcnn_network import DynapcnnCompatibleNetwork, DynapcnnNetwork +from .dynapcnnnetwork_module import DynapcnnNetworkModule from .dynapcnn_visualizer import DynapcnnVisualizer diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer_handler.py b/sinabs/backend/dynapcnn/dynapcnn_layer_handler.py deleted file mode 100644 index 8f619711..00000000 --- a/sinabs/backend/dynapcnn/dynapcnn_layer_handler.py +++ /dev/null @@ -1,141 +0,0 @@ -# author : Willian Soares Girao -# contact : wsoaresgirao@gmail.com - -from typing import List, Optional - - -class DynapcnnLayerHandler: - """ - Class handling the pre-processing of network-level data into (device) layer-level data (i.e., arguments required for a `DynapcnnLayer` instantiation). - - Parameters - ---------- - - dpcnnl_index (int): the index/ID that will be associated with a `DynapcnnLayer` instance. This integer indexes a `dict` within `dcnnl_data` - that comprises a set of layers (`nn.Module`) and their respective I/O tensor shapes. - - dcnnl_data (dict): contains the nodes to be merged into this `DynapcnnLayer`, their I/O shapes and the index of the other `DynapcnnLayer`s to - be set as destinations. The `int` keys correspond to the nodes IDs associated `nn.Module`s (a single layer in the original network) becoming - part of this `DynapcnnLayer` instance, while the `str` keys correspond to this instance's destinations and re-scaling factors. - - discretize (bool): whether or not the weights/neuron parameters of the model will be quantized. - - sinabs_edges (list): each `nn.Module` within `dcnnl_data` is a node in the original computational graph describing a spiking network. An edge - `(A, B)` describes how modules forward data amongst themselves. This list is used by a `DynapcnnLayer` to figure out the number and - sequence of output tesnors its forward method needs to return. - - weight_rescaling_fn (callable): a method that handles how the re-scaling factor for one or more `SumPool2d` projecting to - the same convolutional layer are combined/re-scaled before applying them. - - entry_nodes (list): node IDs corresponding to layers in the original network that are input nodes (i.e., a "point of entry" for the external data). - """ - - def __init__( - self, - layer_index: int, - is_entry_node: bool, - destination_indices: List[int], - assigned_core: Optional[int] = None, - ): - self.layer_index = layer_index - self.entry_node = is_entry_node - self.destination_indices = destination_indices - self.assigned_core = assigned_core - - # TODO: Still needed? - # map destination nodes for each layer in this instance. - # self.nodes_destinations = self._get_destinations_input_source(sinabs_edges) - - ####################################################### Public Methods ####################################################### - - # TODO: Still needed? - def get_destination_dcnnl_index(self, dcnnl_id: int) -> int: - """The `forward` method will return as many tensors as there are elements in `self.dynapcnnlayer_destination`. Since the i-th returned tensor is to be - fed to the i-th destionation in `self.dynapcnnlayer_destination`, the return of this method can be used to index a tensor returned in the `forward` method. - - Parameters - ---------- - - dcnnl_id (int): this should be one of the values listed within `self.dynapcnnlayer_destination`. - - Returns - ---------- - - The index of `dcnnl_id` within `self.dynapcnnlayer_destination`. - """ - return self.dynapcnnlayer_destination.index(dcnnl_id) - - def __str__(self): - pretty_print = "\n" - - pretty_print += "COMPUTATIONAL NODES:\n\n" - - pretty_print += f"(node {self.conv_node_id}): {self.conv_layer}\n" - pretty_print += f"(node {self.spk_node_id}): {self.spk_layer}" - if len(self.pool_layer) != 0: - for idx, lyr in enumerate(self.pool_layer): - pretty_print += f"\n(node {self.pool_node_id[idx]}): {lyr}" - - pretty_print += "\n\nMETADATA:\n" - pretty_print += f"\n> network's entry point: {self.entry_point}" - pretty_print += ( - f"\n> convolution's weight re-scaling factor: {self.conv_rescaling_factor}" - ) - pretty_print += f"\n> assigned core index: {self.assigned_core}" - pretty_print += ( - f"\n> destination DynapcnnLayers: {self.dynapcnnlayer_destination}" - ) - - for node, destinations in self.nodes_destinations.items(): - pretty_print += f"\n> node {node} feeds input to nodes {destinations}" - - return pretty_print - - ####################################################### Private Methods ####################################################### - - # TODO: Still needed? - def _get_destinations_input_source(self, sinabs_edges: list) -> dict: - """Creates a mapping between each layer in this `DynapcnnLayer` instance and its targe nodes that are part of different - `DynapcnnLayer` instances. This mapping is used to figure out how many tensors the `forward` method needs to return. - - Parameters - ---------- - - sinabs_edges (list): each `nn.Module` within `dcnnl_data` is a node in the original computational graph describing a spiking - network. An edge `(A, B)` describes how modules forward data amongst themselves. This list is used by a `DynapcnnLayer` to - figure out the number and sequence of output tesnors its forward method needs to return. - - Returns - ---------- - - destinations_input_source (dict): maps a `nn.Module` within this `DynapcnnLayer` to the nodes it provides the input to. - """ - destinations_input_source = {} - - # check whether spiking layer projects outside this DynapcnnLayer (i.e. to one of the destinations without passing through a pooling). - spk_destinations = [] - for edge in sinabs_edges: - if edge[0] == self.spk_node_id and edge[1] not in self.pool_node_id: - # spiking layer projects to a node outside this DynapcnnLayer. - spk_destinations.append(edge[1]) - if len(spk_destinations) > 0: - destinations_input_source[self.spk_node_id] = [] - for node_id in spk_destinations: - destinations_input_source[self.spk_node_id].append(node_id) - - # get `pooling->destination` mapping. The pooling outputs will be arranged sequentially since the pooling layers are added sequentially - # to `self.pool_layer` (i.e., as they appear in the computational graph of the original `nn.Module`). - for id in self.pool_node_id: - destinations_input_source[id] = [] - for edge in sinabs_edges: - if edge[0] == id: - destinations_input_source[id].append(edge[1]) - - return destinations_input_source - - # TODO: Still needed? - @staticmethod - def find_nodes_core_id(node: int, all_handlers: dict) -> int: - """Loops through all handlers in `all_handlers` to find to which core a `DynapcnnLayer` containing - `node` has been assigned to.""" - - for _, dcnnl in all_handlers.items(): - - if ( - node == dcnnl["layer_handler"].conv_node_id - or node == dcnnl["layer_handler"].spk_node_id - or node in dcnnl["layer_handler"].pool_node_id - ): - return dcnnl["layer_handler"].assigned_core - - raise ValueError(f"Node {node} not found in any of the cores.") diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py b/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py index e311073b..541d06dd 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py @@ -1,20 +1,18 @@ from math import prod -from typing import Callable, Dict, Iterable, Optional, Tuple, Union +from typing import Callable, Dict, Iterable, List, Optional, Set, Tuple, Union from torch import nn from sinabs import layers as sl from .dynapcnn_layer import DynapcnnLayer -from .dynapcnn_layer_handler import DynapcnnLayerHandler from .utils import expand_to_pair def construct_dynapcnnlayers_from_mapper( dcnnl_map: Dict, discretize: bool, rescale_fn: Optional[Callable] = None -) -> Tuple[Dict[int, DynapcnnLayer], Dict[int, DynapcnnLayerHandler]]: - """Construct DynapcnnLayer and DynapcnnLayerHandler instances from - `dcnnl_map` +) -> Tuple[Dict[int, DynapcnnLayer], Dict[int, Set[int]], List[int]]: + """Construct DynapcnnLayer instances from `dcnnl_map` Paramters --------- @@ -22,8 +20,8 @@ def construct_dynapcnnlayers_from_mapper( Returns ------- - Dict of new DynapcnnLayer instances, with keys corresponding to `dcnnl_map` - - Dict of new DynapcnnLayerHandler instances, with keys corresponding - to `dcnnl_map` + - Dict mapping to each layer index a set of destination indices + - List of layer indices that act as entry points to the network """ finalize_dcnnl_map(dcnnl_map, rescale_fn) @@ -32,12 +30,10 @@ def construct_dynapcnnlayers_from_mapper( for layer_idx, layer_info in dcnnl_map.items() } - dynapcnn_layer_handlers = { - layer_idx: construct_single_dynapcnn_layer_handler(layer_idx, layer_info) - for layer_idx, layer_info in dcnnl_map.items() - } + destination_map = construct_destination_map(dcnnl_map) + entry_points = collect_entry_points(dcnnl_map) - return dynapcnn_layers, dynapcnn_layer_handlers + return dynapcnn_layers, destination_map, entry_points def finalize_dcnnl_map(dcnnl_map: Dict, rescale_fn: Optional[Callable] = None): @@ -50,8 +46,7 @@ def finalize_dcnnl_map(dcnnl_map: Dict, rescale_fn: Optional[Callable] = None): Parameters ---------- - - dcnnl_map: Dict holding info needed to instantiate DynapcnnLayer - and DynapcnnLayerHandler instances + - dcnnl_map: Dict holding info needed to instantiate DynapcnnLayer instances - rescale_fn: Optional callable that is used to determine layer rescaling in case of conflicting preceeding average pooling """ @@ -82,8 +77,7 @@ def consolidate_layer_pooling(layer_info: Dict, dcnnl_map: Dict): ---------- - layer_info: Dict holding info of single layer. Corresponds to single entry in `dcnnl_map` - - dcnnl_map: Dict holding info needed to instantiate DynapcnnLayer - and DynapcnnLayerHandler instances + - dcnnl_map: Dict holding info needed to instantiate DynapcnnLayer instances """ layer_info["pooling_list"] = [] for destination in layer_info["destinations"]: @@ -259,34 +253,46 @@ def construct_single_dynapcnn_layer( ) -def construct_single_dynapcnn_layer_handler( - layer_index: int, layer_info: Dict -) -> DynapcnnLayerHandler: - """Instantiate a DynapcnnLayerHandler instance from the - information in `layer_info' +def construct_destination_map(dcnnl_map: Dict[int, Dict]) -> Dict[int, List[int]]: + """ Create a dict that holds destinations for each layer Parameters ---------- - - layer_index: Global index of the layer - - layer_info: Dict holding info of single layer. + - dcnnl_map: Dict holding info needed to instantiate DynapcnnLayer instances Returns ------- - New DynapcnnLayerHandler instance + Dict with layer indices (int) as keys and list of destination indices (int) as values. + Layer outputs that are not sent to other dynapcnn layers are represented by negative indices. """ - destination_indices = [] - none_counter = 0 - for dest in layer_info["destinations"]: - if (dest_idx := dest["destination_layer"]) is None: - # For `None` destinations use unique negative index - none_counter += 1 - destination_indices.append(-none_counter) - else: - destination_indices.append(dest_idx) + destination_map = dict() + for layer_index, layer_info in dcnnl_map.items(): + destination_indices = [] + none_counter = 0 + for dest in layer_info["destinations"]: + if (dest_idx := dest["destination_layer"]) is None: + # For `None` destinations use unique negative index + none_counter += 1 + destination_indices.append(-none_counter) + else: + destination_indices.append(dest_idx) + destination_map[layer_index] = destination_indices - return DynapcnnLayerHandler( - layer_index=layer_index, - is_entry_node=layer_info["is_entry_node"], - destination_indices=destination_indices, - assigned_core=None, - ) + return destination_map + + +def collect_entry_points(dcnnl_map: Dict[int, Dict]) -> Set[int]: + """ Return set of layer indices that are entry points + + Parameters + ---------- + - dcnnl_map: Dict holding info needed to instantiate DynapcnnLayer instances + + Returns + ------- + Set of all layer indices which act as entry points to the network + """ + return { + layer_index + for layer_index, layer_info in dcnnl_map.items() if layer_info["is_entry_node"] + } diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index d8e6b57d..4eae2047 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -14,7 +14,6 @@ from .chip_factory import ChipFactory from .dvs_layer import DVSLayer -from .dynapcnn_layer import DynapcnnLayer from .dynapcnn_layer_utils import construct_dynapcnnlayers_from_mapper from .dynapcnnnetwork_module import DynapcnnNetworkModule from .io import disable_timestamps, enable_timestamps, open_device, reset_timestamps @@ -22,9 +21,7 @@ from .sinabs_edges_handler import collect_dynapcnn_layer_info from .utils import ( DEFAULT_IGNORED_LAYER_TYPES, - Edge, parse_device_id, - topological_sorting, ) from .weight_rescaling_methods import rescale_method_1 @@ -95,7 +92,7 @@ def __init__( ) # build `DynapcnnLayer` instances from mapper. - self._dynapcnn_layers, self._dynapcnnlayer_handlers = ( + dynapcnn_layers, destination_map, entry_points = ( construct_dynapcnnlayers_from_mapper( dcnnl_map=self._dcnnl_map, discretize=discretize, @@ -105,19 +102,15 @@ def __init__( # Module to execute forward pass through network self._dynapcnn_module = DynapcnnNetworkModule( - self._dynapcnn_layers, self._dynapcnnlayer_handlers + dynapcnn_layers, destination_map, entry_points ) - self.dynapcnn_module.setup_dynapcnnlayer_graph() + self.dynapcnn_module.setup_dynapcnnlayer_graph(index_layers_topologically=True) ####################################################### Public Methods ####################################################### @property def dynapcnn_layers(self): - return self._dynapcnn_layers - - @property - def dynapcnnlayer_handlers(self): - return self._dynapcnnlayer_handlers + return self._dynapcnn_module.dynapcnn_layers @property def dynapcnn_module(self): diff --git a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py index 4dd81ff0..772e9c94 100644 --- a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py +++ b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py @@ -2,8 +2,7 @@ # contact : wsoaresgirao@gmail.com from collections import defaultdict -import copy -from typing import Dict, List, Set, Tuple, Union +from typing import Dict, List, Set, Union from warnings import warn import torch.nn as nn @@ -12,28 +11,28 @@ import sinabs.layers as sl from .dynapcnn_layer import DynapcnnLayer -from .dynapcnn_layer_handler import DynapcnnLayerHandler from .utils import Edge, topological_sorting class DynapcnnNetworkModule(nn.Module): """Allow forward (and backward) passing through a network of `DynapcnnLayer`s. - Internally constructs a graph representation based on the provided - `DynapcnnLayer` and `DynapcnnLayerHandler` instances and uses this - to pass data through all layers in correct order. + Internally constructs a graph representation based on the provided arguments + and uses this to pass data through all layers in correct order. Parameters ---------- - dynapcnn_layers (dict): a mapper containing `DynapcnnLayer` instances. - - dynapcnnlayers_handlers (dict): a mapper containing `DynapcnnLayerHandler` instances + - destination_map (dict): Maps layer indices to list of destination indices. + Exit destinations are marked by negative integers + - entry_points (set): Set of layer indices that act as network entry points Attributes ---------- This class internally builds a graph with `DynapcnnLayer` as nodes and their connections as edges. Several data structures help efficient retrieval of information required for the forward pass: - - `self._dynapcnnlayer_edges`: Set of edges connecting dynapcnn layers. Tuples + - _dynapcnnlayer_edges: Set of edges connecting dynapcnn layers. Tuples of indices of source and target layers. - _sorted_nodes: List of layer indices in topological order, to ensure forward calls to layers only happen when required inputs are available. @@ -43,31 +42,51 @@ class DynapcnnNetworkModule(nn.Module): def __init__( self, - dynapcnn_layers: Dict[int, DynapcnnLayerHandler], - dynapcnnlayers_handlers: Dict[int, DynapcnnLayerHandler], + dynapcnn_layers: Dict[int, DynapcnnLayer], + destination_map: Dict[int, List[int]], + entry_points: Set[int], ): super().__init__() - self._dynapcnn_layers = dynapcnn_layers - self._dynapcnnlayer_handlers = dynapcnnlayers_handlers + self.dynapcnn_layers = dynapcnn_layers + self._destination_map = destination_map + self._entry_points = entry_points - def setup_dynapcnnlayer_graph(self): - """Set up data structures to run forward pass through dynapcnn layers""" + # `Merge` layers are stateless. One instance can be used for all merge points during forward pass + self._merge_layer = sl.Merge() + + @property + def destination_map(self): + return self._destination_map + + @property + def entry_points(self): + return self._entry_points + + @property + def sorted_nodes(self): + return self._sorted_nodes + + @property + def node_source_map(self): + return self._node_source_map + + def setup_dynapcnnlayer_graph(self, index_layers_topologically: bool = False): + """ Set up data structures to run forward pass through dynapcnn layers + + Parameters + ---------- + - index_layers_topologically (bool): If True, will assign new indices to + dynapcnn layers such that they match their topological order within the + network graph. This is not necessary but can help understand the network + more easily when inspecting it. + """ self._dynapcnnlayer_edges = self.get_dynapcnnlayers_edges() self.add_entry_points_edges(self._dynapcnnlayer_edges) self._sorted_nodes = topological_sorting(self._dynapcnnlayer_edges) self._node_source_map = self.get_node_source_map(self._dynapcnnlayer_edges) - # `Merge` layers are stateless. One instance can be used for all merge points. - self._merge_layer = sl.Merge() - - # TODO: Probably not needed. - # Collect layers with multiple inputs and instantiate `Merge` layers - # self._merge_points = self._get_merging_points(self._node_source_map) - - # # create mappers to handle `DynapcnnLayer` instances' forward calling. - # self.forward_map, self.merge_points = self._build_module_forward_from_graph( - # dcnnl_edges, dynapcnn_layers - # ) + if index_layers_topologically: + self.reindex_layers(self._sorted_nodes) def get_dynapcnnlayers_edges(self) -> Set[Edge]: """Create edges representing connections between `DynapcnnLayer` instances. @@ -79,9 +98,10 @@ def get_dynapcnnlayers_edges(self) -> Set[Edge]: """ dcnnl_edges = set() - for dcnnl_idx, handler in self._dynapcnnlayer_handlers.items(): - for dest in handler.destination_indices: - dcnnl_edges.add((dcnnl_idx, dest)) + for dcnnl_idx, destination_indices in self._destination_map.items(): + for dest in destination_indices: + if dest >= 0: # Ignore negative destinations (network exit points) + dcnnl_edges.add((dcnnl_idx, dest)) return dcnnl_edges @@ -93,11 +113,10 @@ def add_entry_points_edges(self, dcnnl_edges: Set[Edge]): Parameters ---------- - dcnnl_edges (Set): tuples representing the output->input mapping between - `DynapcnnLayer` instances. Will be changed in place. + `DynapcnnLayer` instances. Will be changed in place. """ - for indx, handler in self._dynapcnnlayer_handlers.items(): - if handler.entry_node: - dcnnl_edges.add(("input", indx)) + for indx in self._entry_points: + dcnnl_edges.add(("input", indx)) def get_node_source_map(self, dcnnl_edges: Set[Edge]) -> Dict[int, List[int]]: """From a set of edges, create a dict that maps to each node its sources @@ -122,30 +141,6 @@ def get_node_source_map(self, dcnnl_edges: Set[Edge]) -> Dict[int, List[int]]: return sources - # TODO: Probably not needed - def get_merging_points( - self, node_source_map: Dict[int, List[int]] - ) -> Dict[int, Dict[Tuple, sl.Merge]]: - """Find nodes within `dcnnl_edges` that have multiple sources. - - Parameters - ---------- - - node_source_map: Dict that maps to each layer index (int) a list of - indices of layers that act as input source to this node - - Returns - ------- - - Dict that for each layer with more than one input source maps its index - (int) to a nested dict with two entries: - * "sources": Set of indices of all source layers to this layer - * "merge_layer": A `Merge` layer instance - """ - return { - tgt: {"sources": sources, "merge_layer": sl.Merge()} - for tgt, sources in node_source_map - if len(sources) > 1 - } - def forward( self, x, return_complete: bool = False ) -> Union[Tensor, Dict[int, Dict[int, Tensor]]]: @@ -199,8 +194,8 @@ def forward( current_input = layers_outputs[idx_src][idx_curr] # Get current layer instance and destinations - layer = self._dynapcnn_layers[idx_curr] - destinations = self._dynapcnnlayer_handlers[idx_curr].destination_indices + layer = self.dynapcnn_layers[idx_curr] + destinations = self._destination_map[idx_curr] # Forward pass through layer output = layer(current_input) @@ -214,7 +209,7 @@ def forward( layers_outputs[idx_curr] = { idx_dest: out for idx_dest, out in zip(destinations, output) } - + if return_complete: return layers_outputs @@ -232,8 +227,8 @@ def forward( warn( "No final outputs have been found. Try setting `return_complete` " "`True` to get all outputs, or mark final outputs by setting " - "corresponding destination layer indices in DynapcnnLayerHandler " - " instance to negative integer values" + "corresponding destination layer indices in destination_map " + " to negative integer values" ) return @@ -242,49 +237,51 @@ def forward( len(network_outputs) == 1 and len(out := (next(iter(network_outputs.values())))) == 1 ): - return out + return next(iter(out.values())) # If there is output from multiple layers return all of them in a dict return network_outputs - # TODO: Necessary? - def _build_module_forward_from_graph( - self, dcnnl_edges: list, dynapcnn_layers: dict - ) -> Union[Dict[int, DynapcnnLayer], Dict[Tuple, sl.Merge]]: - """Creates two mappers, one indexing each `DynapcnnLayer` by its index (a node in `dcnnl_edges`) and another - indexing the `DynapcnnLayer` instances (also by the index) that need their input being the output of a - `Merge` layer (i.e., they are nodes in the graph where two different layer outputs converge to). + def reindex_layers(self, index_order: List[int]): + """ Reindex layers based on provided order + + Will assign new index to dynapcnn layers and update all internal + attributes accordingly. Parameters ---------- - - dcnnl_edges (list): tuples representing the output->input mapping between `DynapcnnLayer` instances - that have been used as configuration for each core `CNNLayerConifg`. - - dynapcnn_layers (dict): a mapper containing `DynapcnnLayer` instances along with their supporting metadata (e.g. assigned core, - destination layers, etc.). - - Returns - ---------- - - forward_map (dict): a mapper where each `key` is the layer index (`DynapcnnLayer.dpcnnl_index`) and the `value` the layer instance itself. - - merge_points (dict): a mapper where each `key` is the layer index and the `value` is a dictionary with a `Merge` layer (`merge_points[key]['merge'] = Merge()`, - computing the input tensor to layer `key`) and its arguments (`merge_points[key]['sources'] = (int A, int B)`, where `A` and `B` are the `DynapcnnLayer` - instances for which the ouput is to be used as the `Merge` arguments). + index_order: List of integers indicating new order of layers: + Position of layer index within this list indicates new index """ + def negative_default(key): + if isinstance(key, int) and key < 0: + return key + else: + raise KeyError(key) - # this dict. will be used to call the `forward` methods of each `DynapcnnLayer`. - forward_map = {} - - for edge in dcnnl_edges: - src_dcnnl = edge[0] # source layer - trg_dcnnl = edge[1] # target layer - - if src_dcnnl not in forward_map: - forward_map[src_dcnnl] = copy.deepcopy( - dynapcnn_layers[src_dcnnl]["layer"] - ) - - if trg_dcnnl not in forward_map: - forward_map[trg_dcnnl] = copy.deepcopy( - dynapcnn_layers[trg_dcnnl]["layer"] - ) + mapping = {old: new for new, old in enumerate(index_order)} - return forward_map + def remap(key): + if key == "input": + return "input" + if isinstance(key, int) and key < 0: + # maintain negative indices + return key + else: + return mapping[key] + + # Remap all internal objects + self.dynapcnn_layers = {remap(idx): lyr for idx, lyr in self.dynapcnn_layers.items()} + self._entry_points = {remap(idx) for idx in self._entry_points} + self._destination_map = { + remap(idx): [remap(dest) for dest in destinations] + for idx, destinations in self._destination_map.items() + } + self._dynapcnnlayer_edges = { + (remap(src), remap(trg)) for (src, trg) in self._dynapcnnlayer_edges + } + self._sorted_nodes = [remap(idx) for idx in self._sorted_nodes] + self._node_source_map = { + remap(node): [remap(src) for src in sources] + for node, sources in self._node_source_map.items() + } \ No newline at end of file From 3fd41068639d58c4c46ba01cfd4fe5a8efde9ac8 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Tue, 15 Oct 2024 16:50:16 +0200 Subject: [PATCH 205/379] Temporarily add dynapcnn_layer_handler definition again to prevent imports from crashing --- .../dynapcnn/dynapcnn_layer_handler.py | 141 ++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 sinabs/backend/dynapcnn/dynapcnn_layer_handler.py diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer_handler.py b/sinabs/backend/dynapcnn/dynapcnn_layer_handler.py new file mode 100644 index 00000000..8f619711 --- /dev/null +++ b/sinabs/backend/dynapcnn/dynapcnn_layer_handler.py @@ -0,0 +1,141 @@ +# author : Willian Soares Girao +# contact : wsoaresgirao@gmail.com + +from typing import List, Optional + + +class DynapcnnLayerHandler: + """ + Class handling the pre-processing of network-level data into (device) layer-level data (i.e., arguments required for a `DynapcnnLayer` instantiation). + + Parameters + ---------- + - dpcnnl_index (int): the index/ID that will be associated with a `DynapcnnLayer` instance. This integer indexes a `dict` within `dcnnl_data` + that comprises a set of layers (`nn.Module`) and their respective I/O tensor shapes. + - dcnnl_data (dict): contains the nodes to be merged into this `DynapcnnLayer`, their I/O shapes and the index of the other `DynapcnnLayer`s to + be set as destinations. The `int` keys correspond to the nodes IDs associated `nn.Module`s (a single layer in the original network) becoming + part of this `DynapcnnLayer` instance, while the `str` keys correspond to this instance's destinations and re-scaling factors. + - discretize (bool): whether or not the weights/neuron parameters of the model will be quantized. + - sinabs_edges (list): each `nn.Module` within `dcnnl_data` is a node in the original computational graph describing a spiking network. An edge + `(A, B)` describes how modules forward data amongst themselves. This list is used by a `DynapcnnLayer` to figure out the number and + sequence of output tesnors its forward method needs to return. + - weight_rescaling_fn (callable): a method that handles how the re-scaling factor for one or more `SumPool2d` projecting to + the same convolutional layer are combined/re-scaled before applying them. + - entry_nodes (list): node IDs corresponding to layers in the original network that are input nodes (i.e., a "point of entry" for the external data). + """ + + def __init__( + self, + layer_index: int, + is_entry_node: bool, + destination_indices: List[int], + assigned_core: Optional[int] = None, + ): + self.layer_index = layer_index + self.entry_node = is_entry_node + self.destination_indices = destination_indices + self.assigned_core = assigned_core + + # TODO: Still needed? + # map destination nodes for each layer in this instance. + # self.nodes_destinations = self._get_destinations_input_source(sinabs_edges) + + ####################################################### Public Methods ####################################################### + + # TODO: Still needed? + def get_destination_dcnnl_index(self, dcnnl_id: int) -> int: + """The `forward` method will return as many tensors as there are elements in `self.dynapcnnlayer_destination`. Since the i-th returned tensor is to be + fed to the i-th destionation in `self.dynapcnnlayer_destination`, the return of this method can be used to index a tensor returned in the `forward` method. + + Parameters + ---------- + - dcnnl_id (int): this should be one of the values listed within `self.dynapcnnlayer_destination`. + + Returns + ---------- + - The index of `dcnnl_id` within `self.dynapcnnlayer_destination`. + """ + return self.dynapcnnlayer_destination.index(dcnnl_id) + + def __str__(self): + pretty_print = "\n" + + pretty_print += "COMPUTATIONAL NODES:\n\n" + + pretty_print += f"(node {self.conv_node_id}): {self.conv_layer}\n" + pretty_print += f"(node {self.spk_node_id}): {self.spk_layer}" + if len(self.pool_layer) != 0: + for idx, lyr in enumerate(self.pool_layer): + pretty_print += f"\n(node {self.pool_node_id[idx]}): {lyr}" + + pretty_print += "\n\nMETADATA:\n" + pretty_print += f"\n> network's entry point: {self.entry_point}" + pretty_print += ( + f"\n> convolution's weight re-scaling factor: {self.conv_rescaling_factor}" + ) + pretty_print += f"\n> assigned core index: {self.assigned_core}" + pretty_print += ( + f"\n> destination DynapcnnLayers: {self.dynapcnnlayer_destination}" + ) + + for node, destinations in self.nodes_destinations.items(): + pretty_print += f"\n> node {node} feeds input to nodes {destinations}" + + return pretty_print + + ####################################################### Private Methods ####################################################### + + # TODO: Still needed? + def _get_destinations_input_source(self, sinabs_edges: list) -> dict: + """Creates a mapping between each layer in this `DynapcnnLayer` instance and its targe nodes that are part of different + `DynapcnnLayer` instances. This mapping is used to figure out how many tensors the `forward` method needs to return. + + Parameters + ---------- + - sinabs_edges (list): each `nn.Module` within `dcnnl_data` is a node in the original computational graph describing a spiking + network. An edge `(A, B)` describes how modules forward data amongst themselves. This list is used by a `DynapcnnLayer` to + figure out the number and sequence of output tesnors its forward method needs to return. + + Returns + ---------- + - destinations_input_source (dict): maps a `nn.Module` within this `DynapcnnLayer` to the nodes it provides the input to. + """ + destinations_input_source = {} + + # check whether spiking layer projects outside this DynapcnnLayer (i.e. to one of the destinations without passing through a pooling). + spk_destinations = [] + for edge in sinabs_edges: + if edge[0] == self.spk_node_id and edge[1] not in self.pool_node_id: + # spiking layer projects to a node outside this DynapcnnLayer. + spk_destinations.append(edge[1]) + if len(spk_destinations) > 0: + destinations_input_source[self.spk_node_id] = [] + for node_id in spk_destinations: + destinations_input_source[self.spk_node_id].append(node_id) + + # get `pooling->destination` mapping. The pooling outputs will be arranged sequentially since the pooling layers are added sequentially + # to `self.pool_layer` (i.e., as they appear in the computational graph of the original `nn.Module`). + for id in self.pool_node_id: + destinations_input_source[id] = [] + for edge in sinabs_edges: + if edge[0] == id: + destinations_input_source[id].append(edge[1]) + + return destinations_input_source + + # TODO: Still needed? + @staticmethod + def find_nodes_core_id(node: int, all_handlers: dict) -> int: + """Loops through all handlers in `all_handlers` to find to which core a `DynapcnnLayer` containing + `node` has been assigned to.""" + + for _, dcnnl in all_handlers.items(): + + if ( + node == dcnnl["layer_handler"].conv_node_id + or node == dcnnl["layer_handler"].spk_node_id + or node in dcnnl["layer_handler"].pool_node_id + ): + return dcnnl["layer_handler"].assigned_core + + raise ValueError(f"Node {node} not found in any of the cores.") From 1d17010908b423f28fadf7b78b815f256a18fb48 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Tue, 15 Oct 2024 17:07:34 +0200 Subject: [PATCH 206/379] DynapcnnNetworkModule using torch compatible ModuleDict --- .../backend/dynapcnn/dynapcnnnetwork_module.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py index 772e9c94..ce0b6b3e 100644 --- a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py +++ b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py @@ -48,12 +48,16 @@ def __init__( ): super().__init__() - self.dynapcnn_layers = dynapcnn_layers + # Unfortunately ModuleDict does not allow for integer keys + # TODO: Consider using list instead of dict + self.dynapcnn_layers = nn.ModuleDict( + {str(idx): lyr for idx, lyr in dynapcnn_layers.items()} + ) self._destination_map = destination_map self._entry_points = entry_points # `Merge` layers are stateless. One instance can be used for all merge points during forward pass - self._merge_layer = sl.Merge() + self.merge_layer = sl.Merge() @property def destination_map(self): @@ -188,13 +192,13 @@ def forward( if len(sources := self._node_source_map[idx_curr]) > 1: # Layer has multiple inputs inputs = [layers_outputs[idx_src][idx_curr] for idx_src in sources] - current_input = self._merge_layer(*inputs) + current_input = self.merge_layer(*inputs) else: idx_src = sources[0] current_input = layers_outputs[idx_src][idx_curr] # Get current layer instance and destinations - layer = self.dynapcnn_layers[idx_curr] + layer = self.dynapcnn_layers[str(idx_curr)] destinations = self._destination_map[idx_curr] # Forward pass through layer @@ -271,7 +275,9 @@ def remap(key): return mapping[key] # Remap all internal objects - self.dynapcnn_layers = {remap(idx): lyr for idx, lyr in self.dynapcnn_layers.items()} + self.dynapcnn_layers = nn.ModuleDict( + {str(remap(int(idx))): lyr for idx, lyr in self.dynapcnn_layers.items()} + ) self._entry_points = {remap(idx) for idx in self._entry_points} self._destination_map = { remap(idx): [remap(dest) for dest in destinations] From 8c020b28e60a73528dfa3b660a17bd917a665cef Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Tue, 15 Oct 2024 17:26:21 +0200 Subject: [PATCH 207/379] Update dynapcnn layer tests --- tests/test_dynapcnnlayer/model_dummy_1.py | 23 +++++++---- tests/test_dynapcnnlayer/model_dummy_2.py | 24 +++++------ tests/test_dynapcnnlayer/model_dummy_3.py | 40 +++++++++---------- tests/test_dynapcnnlayer/model_dummy_4.py | 21 +++++----- .../test_dynapcnnlayer/test_dynapcnnlayer.py | 27 +++++-------- 5 files changed, 66 insertions(+), 69 deletions(-) diff --git a/tests/test_dynapcnnlayer/model_dummy_1.py b/tests/test_dynapcnnlayer/model_dummy_1.py index a9be51d8..8abc7367 100644 --- a/tests/test_dynapcnnlayer/model_dummy_1.py +++ b/tests/test_dynapcnnlayer/model_dummy_1.py @@ -142,7 +142,13 @@ ), "node_id": 12, }, - "destinations": [], + "destinations": [ + { + "pooling_ids": [], + "pooling_modules": [], + "destination_layer": None, + } + ] }, } @@ -152,7 +158,6 @@ "pool": [[3, 3], [4, 4]], "rescale_factor": 1, "rescale_factors": set(), - "destination_indices": [1, 2], "entry_node": True, }, 1: { @@ -160,7 +165,6 @@ "pool": [[1, 1]], "rescale_factor": 1.0 / 9, "rescale_factors": set(), # Single factor will be popped from list - "destination_indices": [3], "entry_node": False, }, 2: { @@ -168,7 +172,6 @@ "pool": [[1, 1]], "rescale_factor": 1.0 / 16, "rescale_factors": set(), # Single factor will be popped from list - "destination_indices": [3], "entry_node": False, }, 3: { @@ -176,15 +179,21 @@ "pool": [[1, 1]], "rescale_factor": 1, "rescale_factors": set(), - "destination_indices": [4], "entry_node": False, }, 4: { "input_shape": (500, 1, 1), - "pool": [], + "pool": [[1, 1]], "rescale_factor": 1, "rescale_factors": set(), - "destination_indices": [], "entry_node": False, }, + "entry_points": {0}, + "destination_map": { + 0: [1, 2], + 1: [3], + 2: [3], + 3: [4], + 4: [-1], + } } diff --git a/tests/test_dynapcnnlayer/model_dummy_2.py b/tests/test_dynapcnnlayer/model_dummy_2.py index 12b0878f..552c6fd2 100644 --- a/tests/test_dynapcnnlayer/model_dummy_2.py +++ b/tests/test_dynapcnnlayer/model_dummy_2.py @@ -212,55 +212,51 @@ "pool": [[1, 1]], "rescale_factor": 1, "rescale_factors": set(), - "destination_indices": [1], - "entry_node": True, }, 1: { "input_shape": (4, 33, 33), "pool": [[2, 2], [2, 2]], "rescale_factor": 1, "rescale_factors": set(), - "destination_indices": [2, 3], - "entry_node": False, }, 2: { "input_shape": (4, 16, 16), "pool": [[2, 2]], "rescale_factor": 1, "rescale_factors": set(), - "destination_indices": [4], - "entry_node": False, }, 3: { "input_shape": (4, 16, 16), "pool": [[2, 2]], "rescale_factor": 1, "rescale_factors": set(), - "destination_indices": [6], - "entry_node": False, }, 4: { "input_shape": (4, 7, 7), "pool": [[1, 1]], "rescale_factor": 1, "rescale_factors": set(), - "destination_indices": [5], - "entry_node": False, }, 5: { "input_shape": (4, 6, 6), "pool": [], "rescale_factor": 1, "rescale_factors": set(), - "destination_indices": [], - "entry_node": False, }, 6: { "input_shape": (4, 7, 7), "pool": [[1, 1]], "rescale_factor": 1, "rescale_factors": set(), - "destination_indices": [5], - "entry_node": False, }, + "entry_points": {0}, + "destination_map": { + 0: [1], + 1: [2, 3], + 2: [4], + 3: [6], + 4: [5], + 6: [5], + 5: [], + } } diff --git a/tests/test_dynapcnnlayer/model_dummy_3.py b/tests/test_dynapcnnlayer/model_dummy_3.py index bc6112b2..24f963df 100644 --- a/tests/test_dynapcnnlayer/model_dummy_3.py +++ b/tests/test_dynapcnnlayer/model_dummy_3.py @@ -250,7 +250,13 @@ ), "node_id": 22, }, - "destinations": [], + "destinations": [ + { + "pooling_ids": [], + "pooling_modules": [], + "destination_layer": None, + } + ], }, } @@ -260,71 +266,65 @@ "pool": [[1, 1]], "rescale_factor": 1, "rescale_factors": set(), - "destination_indices": [1], - "entry_node": True, }, 1: { "input_shape": (4, 33, 33), "pool": [[2, 2]], "rescale_factor": 1, "rescale_factors": set(), - "destination_indices": [2], - "entry_node": False, }, 2: { "input_shape": (4, 16, 16), "pool": [[2, 2]], "rescale_factor": 1, "rescale_factors": set(), - "destination_indices": [3], - "entry_node": False, }, 3: { "input_shape": (4, 7, 7), "pool": [[1, 1]], "rescale_factor": 1, "rescale_factors": set(), - "destination_indices": [7], - "entry_node": False, }, 4: { "input_shape": (2, 34, 34), "pool": [[1, 1]], "rescale_factor": 1, "rescale_factors": set(), - "destination_indices": [5], - "entry_node": True, }, 5: { "input_shape": (4, 33, 33), "pool": [[2, 2]], "rescale_factor": 1, "rescale_factors": set(), - "destination_indices": [6], - "entry_node": False, }, 6: { "input_shape": (4, 16, 16), "pool": [[2, 2]], "rescale_factor": 1, "rescale_factors": set(), - "destination_indices": [3], - "entry_node": False, }, 7: { "input_shape": (100, 1, 1), "pool": [[1, 1]], "rescale_factor": 1, "rescale_factors": set(), - "destination_indices": [8], - "entry_node": False, }, 8: { "input_shape": (100, 1, 1), - "pool": [], + "pool": [[1, 1]], "rescale_factor": 1, "rescale_factors": set(), - "destination_indices": [], - "entry_node": False, }, + "entry_points": {0, 4}, + "destination_map": { + 0: [1], + 1: [2], + 2: [3], + 3: [7], + 4: [5], + 5: [6], + 6: [3], + 7: [8], + 8: [-1], + } } diff --git a/tests/test_dynapcnnlayer/model_dummy_4.py b/tests/test_dynapcnnlayer/model_dummy_4.py index a8defe71..e305a247 100644 --- a/tests/test_dynapcnnlayer/model_dummy_4.py +++ b/tests/test_dynapcnnlayer/model_dummy_4.py @@ -191,47 +191,44 @@ "pool": [[1, 1], [1, 1]], "rescale_factor": 1, "rescale_factors": set(), - "destination_indices": [1, 2], - "entry_node": True, }, 1: { "input_shape": (1, 33, 33), "pool": [[2, 2]], "rescale_factor": 1, "rescale_factors": set(), - "destination_indices": [3], - "entry_node": False, }, 2: { "input_shape": (1, 33, 33), "pool": [[2, 2], [5, 5]], "rescale_factor": 1, "rescale_factors": set(), - "destination_indices": [3, 4], - "entry_node": False, }, 3: { "input_shape": (1, 16, 16), "pool": [[3, 3]], "rescale_factor": 1, "rescale_factors": set(), - "destination_indices": [5], - "entry_node": False, }, 4: { "input_shape": (1, 6, 6), "pool": [[1, 1]], "rescale_factor": 1, "rescale_factors": set(), - "destination_indices": [5], - "entry_node": False, }, 5: { "input_shape": (1, 5, 5), "pool": [], "rescale_factor": 1, "rescale_factors": set(), - "destination_indices": [], - "entry_node": False, }, + "entry_points": {0}, + "destination_map": { + 0: [1, 2], + 1: [3], + 2: [3, 4], + 3: [5], + 4: [5], + 5: [], + } } diff --git a/tests/test_dynapcnnlayer/test_dynapcnnlayer.py b/tests/test_dynapcnnlayer/test_dynapcnnlayer.py index 66aa5d39..10225950 100644 --- a/tests/test_dynapcnnlayer/test_dynapcnnlayer.py +++ b/tests/test_dynapcnnlayer/test_dynapcnnlayer.py @@ -20,7 +20,7 @@ def test_DynapcnnLayer(dcnnl_map, discretize, expected_output): """ # create a `DynapcnnLayer` from the set of layers in `nodes_to_dcnnl_map[dpcnnl_idx]`. - dynapcnn_layers, layer_handlers = construct_dynapcnnlayers_from_mapper( + dynapcnn_layers, destination_map, entry_points = construct_dynapcnnlayers_from_mapper( dcnnl_map=dcnnl_map, discretize=discretize, rescale_fn=None ) @@ -54,19 +54,14 @@ def test_DynapcnnLayer(dcnnl_map, discretize, expected_output): layer_info["rescale_factors"] == rescale_factors ), f"wrong 'rescale_factors' entry: Should be {rescale_factors}." - # Test layer handler instance - layerhandler = layer_handlers[layer_index] - destination_indices = expected_output[layer_index]["destination_indices"] - entry_node = expected_output[layer_index]["entry_node"] + # # Convert destination lists to sets to ignore order + # destination_map = {node: set(dests) for node, dests in destination_map.items()} + # Test destination map + assert ( + destination_map == expected_output["destination_map"] + ), "wrong destination map" - assert ( - layerhandler.layer_index == layer_index - ), f"wrong 'DynapcnnLayerHandler.layer_index': ID of the instance should be {layer_index}." - assert ( - layerhandler.destination_indices - == expected_output[layer_index]["destination_indices"] - ), f"wrong 'DynapcnnLayerHandler.destination_indices': the DynapcnnLayer(s) set as destination(s) should be {destination_indices}." - assert ( - layerhandler.entry_node == expected_output[layer_index]["entry_node"] - ), f"wrong 'DynapcnnLayerHandler.entry_node': its value should be {entry_node}." - assert layerhandler.assigned_core is None + # Test entry point + assert ( + entry_points == expected_output["entry_points"] + ), "wrong entry points" \ No newline at end of file From 5082450d54d3d661d72415395d3446e4be577fba Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Tue, 15 Oct 2024 18:05:02 +0200 Subject: [PATCH 208/379] Move layer and network-module instantiation to graph-extractor --- sinabs/backend/dynapcnn/dynapcnn_network.py | 30 ++--- .../backend/dynapcnn/nir_graph_extractor.py | 108 ++++++++++++++---- 2 files changed, 93 insertions(+), 45 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index 4eae2047..9382d8a4 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -14,11 +14,8 @@ from .chip_factory import ChipFactory from .dvs_layer import DVSLayer -from .dynapcnn_layer_utils import construct_dynapcnnlayers_from_mapper -from .dynapcnnnetwork_module import DynapcnnNetworkModule from .io import disable_timestamps, enable_timestamps, open_device, reset_timestamps from .nir_graph_extractor import GraphExtractor -from .sinabs_edges_handler import collect_dynapcnn_layer_info from .utils import ( DEFAULT_IGNORED_LAYER_TYPES, parse_device_id, @@ -83,28 +80,11 @@ def __init__( # Remove nodes of ignored classes (including merge nodes) self._graph_extractor.remove_nodes_by_class(DEFAULT_IGNORED_LAYER_TYPES) - # create a dict holding the data necessary to instantiate a `DynapcnnLayer`. - self._dcnnl_map = collect_dynapcnn_layer_info( - self._graph_extractor.indx_2_module_map, - self._graph_extractor.edges, - self._graph_extractor.nodes_io_shapes, - self._graph_extractor.entry_nodes, - ) - - # build `DynapcnnLayer` instances from mapper. - dynapcnn_layers, destination_map, entry_points = ( - construct_dynapcnnlayers_from_mapper( - dcnnl_map=self._dcnnl_map, - discretize=discretize, - rescale_fn=weight_rescaling_fn, - ) - ) - # Module to execute forward pass through network - self._dynapcnn_module = DynapcnnNetworkModule( - dynapcnn_layers, destination_map, entry_points + self._dynapcnn_module = self._graph_extractor.get_dynapcnn_network_module( + discretize=discretize, weight_rescaling_fn=weight_rescaling_fn ) - self.dynapcnn_module.setup_dynapcnnlayer_graph(index_layers_topologically=True) + self._dynapcnn_module.setup_dynapcnnlayer_graph(index_layers_topologically=True) ####################################################### Public Methods ####################################################### @@ -116,6 +96,10 @@ def dynapcnn_layers(self): def dynapcnn_module(self): return self._dynapcnn_module + @property + def layer_destination_map(self): + return self._dynapcnn_module.destination_map + @property def chip_layers_ordering(self): return self._chip_layers_ordering diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index b89dd8d9..46ae045f 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -1,7 +1,7 @@ # author : Willian Soares Girao # contact : wsoaresgirao@gmail.com -from typing import Dict, List, Set, Tuple, Type +from typing import Callable, Dict, List, Optional, Set, Tuple, Type import nirtorch import torch @@ -13,7 +13,10 @@ LAYER_TYPES_WITH_MULTIPLE_INPUTS, LAYER_TYPES_WITH_MULTIPLE_OUTPUTS, ) +from .dynapcnn_layer_utils import construct_dynapcnnlayers_from_mapper +from .dynapcnnnetwork_module import DynapcnnNetworkModule from .exceptions import InvalidGraphStructure +from .sinabs_edges_handler import collect_dynapcnn_layer_info from .utils import Edge, topological_sorting @@ -51,11 +54,12 @@ def __init__(self, spiking_model: nn.Module, dummy_input: torch.tensor): spiking_model, dummy_input, model_name=None ).ignore_tensors() - # converts the NIR representation into a list of edges with nodes represented as integers. - self._edges, self._name_2_indx_map, self._entry_nodes = ( - self._get_edges_from_nir(nir_graph) - ) - + # Map node names to indices + self._name_2_indx_map = self._get_name_2_indx_map(nir_graph) + # Extract edges list from graph + self._edges = self._get_edges_from_nir(nir_graph, self._name_2_indx_map) + # Determine entry points to graph + self._entry_nodes = self._get_entry_nodes(self._edges) # Store the associated `nn.Module` (layer) of each node. self._indx_2_module_map = self._get_named_modules(spiking_model) @@ -91,6 +95,47 @@ def sorted_nodes(self) -> List[int]: def indx_2_module_map(self) -> Dict[int, nn.Module]: return {n: module for n, module in self._indx_2_module_map.items()} + def get_dynapcnn_network_module( + self, discretize: bool = False, weight_rescaling_fn: Optional[Callable] = None + ) -> DynapcnnNetworkModule: + """ Create DynapcnnNetworkModule based on stored graph representation + + This includes construction of the DynapcnnLayer instances + + Parameters: + ----------- + - discretize (bool): If `True`, discretize the parameters and thresholds. This is needed for uploading + weights to dynapcnn. Set to `False` only for testing purposes. + - weight_rescaling_fn (callable): a method that handles how the re-scaling factor for one or more `SumPool2d` projecting to + the same convolutional layer are combined/re-scaled before applying them. + + Returns + ------- + - The DynapcnnNetworkModule based on graph representation of this `GraphExtractor` + + """ + # create a dict holding the data necessary to instantiate a `DynapcnnLayer`. + dcnnl_map = collect_dynapcnn_layer_info( + indx_2_module_map = self.indx_2_module_map, + edges = self.edges, + nodes_io_shapes=self.nodes_io_shapes, + entry_nodes=self.entry_nodes, + ) + + # build `DynapcnnLayer` instances from mapper. + dynapcnn_layers, destination_map, entry_points = ( + construct_dynapcnnlayers_from_mapper( + dcnnl_map=dcnnl_map, + discretize=discretize, + rescale_fn=weight_rescaling_fn, + ) + ) + + # Instantiate the DynapcnnNetworkModule + return DynapcnnNetworkModule( + dynapcnn_layers, destination_map, entry_points + ) + def remove_nodes_by_class(self, node_classes: Tuple[Type]): """Remove nodes of given classes from graph in place. @@ -174,14 +219,32 @@ def verify_graph_integrity(self): ####################################################### Pivate Methods ####################################################### + def _get_name_2_indx_map(self, nir_graph: nirtorch.graph.Graph) -> Dict[str, int]: + """Assign unique index to each node and return mapper from name to index. + + Parameters + ---------- + - nir_graph (nirtorch.graph.Graph): a NIR graph representation of `spiking_model`. + + Returns + ---------- + - name_2_indx_map (dict): `key` is the original variable name for a layer in + `spiking_model` and `value is an integer representing the layer in a standard format. + """ + return { + node.name: node_idx for node_idx, node in enumerate(nir_graph.node_list) + } + def _get_edges_from_nir( - self, nir_graph: nirtorch.graph.Graph - ) -> Tuple[List[Edge], Dict[str, int], List[int]]: - """Standardize the representation of `nirtorch.graph.Graph` into a list of edges (`Edge`) where each node in `nir_graph` is represented by an interger (with the source node starting as `0`). + self, nir_graph: nirtorch.graph.Graph, name_2_indx_map: Dict[str, int] + ) -> Set[Edge]: + """Standardize the representation of `nirtorch.graph.Graph` into a list of edges, + representing nodes by their indices. Parameters ---------- - nir_graph (nirtorch.graph.Graph): a NIR graph representation of `spiking_model`. + - name_2_indx_map (dict): Map from node names to unique indices. Returns ---------- @@ -189,25 +252,26 @@ def _get_edges_from_nir( - name_2_indx_map (dict): `key` is the original variable name for a layer in `spiking_model` and `value is an integer representing the layer in a standard format. - entry_nodes (set): IDs of nodes acting as entry points for the network (i.e., receiving external input). """ - # TODO maybe make sure the input node from nir always gets assined `0`. - - # Assign a unique index to each node - name_2_indx_map = { - node.name: node_idx for node_idx, node in enumerate(nir_graph.node_list) - } - - # Extract edges for each node - edges = { + return { (name_2_indx_map[src.name], name_2_indx_map[tgt.name]) for src in nir_graph.node_list for tgt in src.outgoing_nodes } - # find entry nodes of the graph. - all_sources, all_targets = zip(*edges) - entry_nodes = set(all_sources) - set(all_targets) + def _get_entry_nodes(self, edges: Set[Edge]) -> Set[Edge]: + """Find nodes that act as entry points to the graph + + Parameters + ---------- + - edges (set): tuples describing the connections between layers in `spiking_model`. - return edges, name_2_indx_map, entry_nodes + Returns + ---------- + - entry_nodes (set): IDs of nodes acting as entry points for the network + (i.e., receiving external input). + """ + all_sources, all_targets = zip(*edges) + return set(all_sources) - set(all_targets) def _get_named_modules(self, model: nn.Module) -> Dict[int, nn.Module]: """Find for each node in the graph what its associated layer in `model` is. From 70e649eb754218417f4af05de3f49abc3ae9961e Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Thu, 17 Oct 2024 17:54:08 +0200 Subject: [PATCH 209/379] Make optional for --- sinabs/backend/dynapcnn/dynapcnn_network.py | 21 ++--- sinabs/utils.py | 89 +++++++++++++++++++++ 2 files changed, 95 insertions(+), 15 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index 9382d8a4..5be5f9f2 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -28,7 +28,7 @@ def __init__( self, snn: nn.Module, input_shape: Tuple[int, int, int], - batch_size: int, + batch_size: Optional[int] = None, dvs_input: bool = False, discretize: bool = True, weight_rescaling_fn: Callable = rescale_method_1, @@ -42,25 +42,13 @@ def __init__( ---------- - snn (nn.Module): a implementing a spiking network. - input_shape (tuple): a description of the input dimensions as `(features, height, width)`. + - batch_size (optional int): If `None`, will try to infer the batch size from the model. + If int value is provided, it has to match the actual batch size of the model. - dvs_input (bool): wether or not dynapcnn receive input from its DVS camera. - discretize (bool): If `True`, discretize the parameters and thresholds. This is needed for uploading weights to dynapcnn. Set to `False` only for testing purposes. - weight_rescaling_fn (callable): a method that handles how the re-scaling factor for one or more `SumPool2d` projecting to the same convolutional layer are combined/re-scaled before applying them. - - Notes - ---------- - Some of the properties defined within the class constructor are meant to be temporary data structures handling the conversion - of the `snn` (the original `nn.Module`) into a set of `DynapcnnLayer`s composing a `DynapcnnNetwork` instance. Once their role - in preprocessing `snn` is finished, all required data to train/deploy the `DynapcnnNetwork` instance is within `self._dcnnl_edges` - (the connectivity between each `DynapcnnLayer`/core), `self._layers_mapper` (every `DynapcnnLayer` in the network) and `self._merge_points` - (the `DynapcnnLayer`s that need a `Merge` input). Thus, the following private properties are delted as last step of the constructor: - - - self._graph_extractor - - self._sinabs_edges - - self._sinabs_indx_2_module_map - - self._dcnnl_map - - self._dynapcnn_layers """ super().__init__() @@ -72,6 +60,9 @@ def __init__( assert len(self.input_shape) == 3, "infer_input_shape did not return 3-tuple" + # Infer batch size for dummpy input to graph extractor + if batch_size is None: + batch_size = sinabs.utils.get_smallest_compatible_time_dimension(snn) # computational graph from original PyTorch module. self._graph_extractor = GraphExtractor( snn, torch.randn((batch_size, *self.input_shape)) diff --git a/sinabs/utils.py b/sinabs/utils.py index 7b090963..89460f46 100644 --- a/sinabs/utils.py +++ b/sinabs/utils.py @@ -179,3 +179,92 @@ def set_batch_size(model: nn.Module, batch_size: int): if isinstance(mod, sinabs.layers.SqueezeMixin): mod.batch_size = batch_size # reset_states(mod) + + +def get_batch_size(model: nn.Module) -> int: + """Get batch size from any model with sinabs squeeze layers + + Will raise a ValueError if different squeeze layers within the model + have different batch sizes. Ignores layers with batch size `-1`, if + others provide it. + + Args: + model (nn.Module): pytorch model with sinabs Squeeze layers + + Returns: + batch_size (int): The batch size, `-1` if none is found. + """ + + batch_sizes = { + mod.batch_size + for mod in model.modules() + if isinstance(mod, sinabs.layers.SqueezeMixin) + } + # Ignore values `-1` and `None` + batch_sizes.discard(-1) + batch_sizes.discard(None) + + if len(batch_sizes) == 0: + return -1 + elif len(batch_sizes) == 1: + return batch_sizes.pop() + else: + raise ValueError( + "The model contains layers with different batch sizes: " + ", ".join((str(s) for s in batch_sizes)) + ) + + +def get_num_timesteps(model: nn.Module) -> int: + """Get number of timesteps from any model with sinabs squeeze layers + + Will raise a ValueError if different squeeze layers within the model + have different `num_timesteps` attributes. Ignores layers with value + `-1`, if others provide it. + + Args: + model (nn.Module): pytorch model with sinabs Squeeze layers + + Returns: + num_timesteps (int): The number of time steps, `-1` if none is found. + """ + + numbers = { + mod.num_timesteps + for mod in model.modules() + if isinstance(mod, sinabs.layers.SqueezeMixin) + } + # Ignore values `-1` and `None` + numbers.discard(-1) + numbers.discard(None) + + if len(numbers) == 0: + return -1 + elif len(numbers) == 1: + return numbers.pop() + else: + raise ValueError( + "The model contains layers with different numbers of time steps: " + ", ".join((str(s) for s in numbers)) + ) + + +def get_smallest_compatible_time_dimension(model: nn.Module) -> int: + """Find the smallest size for input to a model with sinabs squeeze layers + along the batch/time (first) dimension. + + Will raise a ValueError if different squeeze layers within the model + have different `num_timesteps` or `batch_size` attributes (except for + `-1`) + + Args: + model (nn.Module): pytorch model with sinabs Squeeze layers + + Returns: + int: The smallest compatible size for the first dimension of + an input to the `model`. + """ + batch_size = abs(get_batch_size(model)) # Use `abs` to turn -1 to 1 + num_timesteps = abs(get_num_timesteps(model)) + # Use `abs` to turn `-1` to `1` + return abs(batch_size * num_timesteps) From 2d3d32c0f31b0c66e23a0929fd280ca3097ebf0c Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 22 Oct 2024 13:28:31 +0200 Subject: [PATCH 210/379] doc updated dosctring for get_valid_edge_type() --- sinabs/backend/dynapcnn/sinabs_edges_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index 6cce7a09..dd3a3a36 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -145,7 +145,7 @@ def get_valid_edge_type( Returns ---------- - edge_type: the edge type specified in 'valid_edges_map' ('None' if edge is not valid). + edge_type: the edge type specified in 'VALID_SINABS_EDGE_TYPES' ('None' if edge is not valid). """ source_type = type(layers[edge[0]]) target_type = type(layers[edge[1]]) From 482499e8e53024ab1dfb4167335cdba377fa475a Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Tue, 22 Oct 2024 17:00:36 +0200 Subject: [PATCH 211/379] Restore original dynapcnn layer attribute names conv_layer and spk_layer --- sinabs/backend/dynapcnn/chips/dynapcnn.py | 54 +++++++++++------------ sinabs/backend/dynapcnn/dynapcnn_layer.py | 20 ++++----- 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/sinabs/backend/dynapcnn/chips/dynapcnn.py b/sinabs/backend/dynapcnn/chips/dynapcnn.py index 4bf5acf0..1096baa0 100644 --- a/sinabs/backend/dynapcnn/chips/dynapcnn.py +++ b/sinabs/backend/dynapcnn/chips/dynapcnn.py @@ -45,26 +45,26 @@ def set_kill_bits(cls, layer: DynapcnnLayer, config_dict: dict) -> dict: """ config_dict = copy.deepcopy(config_dict) - if layer.conv.bias is not None: - (weights, biases) = layer.conv.parameters() + if layer.conv_layer.bias is not None: + (weights, biases) = layer.conv_layer.parameters() else: - (weights,) = layer.conv.parameters() - biases = torch.zeros(layer.conv.out_channels) + (weights,) = layer.conv_layer.parameters() + biases = torch.zeros(layer.conv_layer.out_channels) config_dict["weights_kill_bit"] = (~weights.bool()).tolist() config_dict["biases_kill_bit"] = (~biases.bool()).tolist() # - Neuron states - if not layer.spk.is_state_initialised(): + if not layer.spk_layer.is_state_initialised(): # then we assign no initial neuron state to DYNAP-CNN. f, h, w = layer.get_neuron_shape() neurons_state = torch.zeros(f, w, h) - elif layer.spk.v_mem.dim() == 4: + elif layer.spk_layer.v_mem.dim() == 4: # 4-dimensional states should be the norm when there is a batch dim - neurons_state = layer.spk.v_mem.transpose(2, 3)[0] + neurons_state = layer.spk_layer.v_mem.transpose(2, 3)[0] else: raise ValueError( - f"Current v_mem (shape: {layer.spk.v_mem.shape}) of spiking layer not understood." + f"Current v_mem (shape: {layer.spk_layer.v_mem.shape}) of spiking layer not understood." ) config_dict["neurons_value_kill_bit"] = ( @@ -96,24 +96,24 @@ def get_dynapcnn_layer_config_dict( dimensions["output_shape"]["size"]["x"] = w dimensions["output_shape"]["size"]["y"] = h dimensions["padding"] = { - "x": layer.conv.padding[1], - "y": layer.conv.padding[0], + "x": layer.conv_layer.padding[1], + "y": layer.conv_layer.padding[0], } dimensions["stride"] = { - "x": layer.conv.stride[1], - "y": layer.conv.stride[0], + "x": layer.conv_layer.stride[1], + "y": layer.conv_layer.stride[0], } - dimensions["kernel_size"] = layer.conv.kernel_size[0] + dimensions["kernel_size"] = layer.conv_layer.kernel_size[0] - if dimensions["kernel_size"] != layer.conv.kernel_size[1]: + if dimensions["kernel_size"] != layer.conv_layer.kernel_size[1]: raise ValueError("Conv2d: Kernel must have same height and width.") config_dict["dimensions"] = dimensions # Update parameters from convolution - if layer.conv.bias is not None: - (weights, biases) = layer.conv.parameters() + if layer.conv_layer.bias is not None: + (weights, biases) = layer.conv_layer.parameters() else: - (weights,) = layer.conv.parameters() - biases = torch.zeros(layer.conv.out_channels) + (weights,) = layer.conv_layer.parameters() + biases = torch.zeros(layer.conv_layer.out_channels) weights = weights.transpose(2, 3) # Need this to match samna convention config_dict["weights"] = weights.int().tolist() config_dict["biases"] = biases.int().tolist() @@ -122,36 +122,36 @@ def get_dynapcnn_layer_config_dict( # Update parameters from the spiking layer # - Neuron states - if not layer.spk.is_state_initialised(): + if not layer.spk_layer.is_state_initialised(): # then we assign no initial neuron state to DYNAP-CNN. f, h, w = layer.get_neuron_shape() neurons_state = torch.zeros(f, w, h) - elif layer.spk.v_mem.dim() == 4: + elif layer.spk_layer.v_mem.dim() == 4: # 4-dimensional states should be the norm when there is a batch dim - neurons_state = layer.spk.v_mem.transpose(2, 3)[0] + neurons_state = layer.spk_layer.v_mem.transpose(2, 3)[0] else: raise ValueError( - f"Current v_mem (shape: {layer.spk.v_mem.shape}) of spiking layer not understood." + f"Current v_mem (shape: {layer.spk_layer.v_mem.shape}) of spiking layer not understood." ) # - Resetting vs returning to 0 - if isinstance(layer.spk.reset_fn, sinabs.activation.MembraneReset): + if isinstance(layer.spk_layer.reset_fn, sinabs.activation.MembraneReset): return_to_zero = True - elif isinstance(layer.spk.reset_fn, sinabs.activation.MembraneSubtract): + elif isinstance(layer.spk_layer.reset_fn, sinabs.activation.MembraneSubtract): return_to_zero = False else: raise Exception( "Unknown reset mechanism. Only MembraneReset and MembraneSubtract are currently understood." ) - if layer.spk.min_v_mem is None: + if layer.spk_layer.min_v_mem is None: min_v_mem = -(2**15) else: - min_v_mem = int(layer.spk.min_v_mem) + min_v_mem = int(layer.spk_layer.min_v_mem) config_dict.update( { "return_to_zero": return_to_zero, - "threshold_high": int(layer.spk.spike_threshold), + "threshold_high": int(layer.spk_layer.spike_threshold), "threshold_low": min_v_mem, "monitor_enable": False, "neurons_initial_value": neurons_state.int().tolist(), diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer.py b/sinabs/backend/dynapcnn/dynapcnn_layer.py index 84463adc..46fcea63 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer.py @@ -134,11 +134,11 @@ def __init__( self._spk = spk @property - def conv(self): + def conv_layer(self): return self._conv @property - def spk(self): + def spk_layer(self): return self._spk @property @@ -167,8 +167,8 @@ def forward(self, x) -> List[torch.Tensor]: returns = [] - x = self.conv(x) - x = self.spk(x) + x = self.conv_layer(x) + x = self.spk_layer(x) for pool in self._pool: @@ -254,7 +254,7 @@ def memory_summary(self): "kernel": c * pow(2, np.ceil(np.log2(h * w)) + np.ceil(np.log2(f))), "neuron": f * pow(2, np.ceil(np.log2(neuron_height)) + np.ceil(np.log2(neuron_width))), - "bias": 0 if self.conv.bias is None else len(self.conv.bias), + "bias": 0 if self.conv_layer.bias is None else len(self.conv_layer.bias), } ####################################################### Private Methods ####################################################### @@ -268,11 +268,11 @@ def _get_conv_output_shape(self) -> Tuple[int, int, int]: """ # get the layer's parameters. - out_channels = self.conv.out_channels - kernel_size = self.conv.kernel_size - stride = self.conv.stride - padding = self.conv.padding - dilation = self.conv.dilation + out_channels = self.conv_layer.out_channels + kernel_size = self.conv_layer.kernel_size + stride = self.conv_layer.stride + padding = self.conv_layer.padding + dilation = self.conv_layer.dilation # compute the output height and width. out_height = ( From 7b63aca83ba19957dbc7371c88daa02287ba607c Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Tue, 22 Oct 2024 17:32:09 +0200 Subject: [PATCH 212/379] (WIP) Remove dependency on DynapcnnLayerHandler for deployment --- sinabs/backend/dynapcnn/chips/dynapcnn.py | 72 ++++++++++----------- sinabs/backend/dynapcnn/config_builder.py | 13 +++- sinabs/backend/dynapcnn/dynapcnn_network.py | 8 ++- 3 files changed, 51 insertions(+), 42 deletions(-) diff --git a/sinabs/backend/dynapcnn/chips/dynapcnn.py b/sinabs/backend/dynapcnn/chips/dynapcnn.py index 1096baa0..29442504 100644 --- a/sinabs/backend/dynapcnn/chips/dynapcnn.py +++ b/sinabs/backend/dynapcnn/chips/dynapcnn.py @@ -1,5 +1,5 @@ import copy -from typing import List +from typing import Dict, List from warnings import warn import samna @@ -10,7 +10,6 @@ from sinabs.backend.dynapcnn.config_builder import ConfigBuilder from sinabs.backend.dynapcnn.dvs_layer import DVSLayer, expand_to_pair from sinabs.backend.dynapcnn.dynapcnn_layer import DynapcnnLayer -from sinabs.backend.dynapcnn.dynapcnn_layer_handler import DynapcnnLayerHandler from sinabs.backend.dynapcnn.mapping import LayerConstraints @@ -77,9 +76,10 @@ def set_kill_bits(cls, layer: DynapcnnLayer, config_dict: dict) -> dict: def get_dynapcnn_layer_config_dict( cls, layer: DynapcnnLayer, - layer_handler: DynapcnnLayerHandler, - all_handlers: dict, + layer2core_map: Dict[int, int], + destination_indices: List[int], ) -> dict: + # TODO: Docstring config_dict = {} config_dict["destinations"] = [{}, {}] @@ -158,22 +158,16 @@ def get_dynapcnn_layer_config_dict( } ) - # setting destinations config. based on destinations destination nodes of the nodes withing this `dcnnl`. + # Configure destinations destinations = [] - for node_id, destination_nodes in layer_handler.nodes_destinations.items(): - for dest_node in destination_nodes: - core_id = DynapcnnLayerHandler.find_nodes_core_id( - dest_node, all_handlers - ) - kernel_size = layer_handler.get_pool_kernel_size(node_id) - - dest_data = { - "layer": core_id, - "enable": True, - "pooling": expand_to_pair(kernel_size if kernel_size else 1), - } - - destinations.append(dest_data) + pooling_sizes = layer.pool + for dest_layer_id, pool in zip(destination_indices, pooling_sizes): + dest_data = { + "layer": layer2core_map[dest_layer_id], + "enable": True, + "pooling": expand_to_pair(pool), + } + destinations.append(dest_data) config_dict["destinations"] = destinations # Set kill bits @@ -185,9 +179,9 @@ def get_dynapcnn_layer_config_dict( def write_dynapcnn_layer_config( cls, layer: DynapcnnLayer, + layer2core_map: Dict[int, int], + destination_indices: List[int], chip_layer: "CNNLayerConfig", - layer_handler: DynapcnnLayerHandler, - all_handlers: dict, ) -> None: """Write a single layer configuration to the dynapcnn conf object. Uses the data in `layer` to configure a `CNNLayerConfig` to be deployed on chip. @@ -202,7 +196,9 @@ def write_dynapcnn_layer_config( # extracting from a DynapcnnLayer the config. variables for its CNNLayerConfig. config_dict = cls.get_dynapcnn_layer_config_dict( - layer=layer, layer_handler=layer_handler, all_handlers=all_handlers + layer=layer, + layer2core_map=layer2core_map, + destination_indices=destination_indices, ) # update configuration of the DYNAPCNN layer. @@ -210,14 +206,10 @@ def write_dynapcnn_layer_config( config_dict.pop("dimensions") # set the destinations configuration. - for i in range(len(config_dict["destinations"])): - chip_layer.destinations[i].layer = config_dict["destinations"][i]["layer"] - chip_layer.destinations[i].enable = config_dict["destinations"][i]["enable"] - chip_layer.destinations[i].pooling = config_dict["destinations"][i][ - "pooling" - ] - - config_dict.pop("destinations") + for dest_idx, destination in enumerate(config_dict.pop("destinations")): + chip_layer.destinations[dest_idx].layer = destination["layer"] + chip_layer.destinations[dest_idx].enable = destination["enable"] + chip_layer.destinations[dest_idx].pooling = destination["pooling"] # set remaining configuration. for param, value in config_dict.items(): @@ -227,7 +219,13 @@ def write_dynapcnn_layer_config( raise TypeError(f"Unexpected parameter {param} or value. {e}") @classmethod - def build_config(cls, model: "DynapcnnNetwork") -> DynapcnnConfiguration: + def build_config( + cls, + layers: Dict[int, DynapcnnLayer], + destination_map: Dict[int, List[int]], + layer2core_map: Dict[int, int], + ) -> DynapcnnConfiguration: + # TODO: Update docstring """Uses `DynapcnnLayer` objects to configure their equivalent chip core via a `CNNLayerConfig` object that is built using using the `DynapcnnLayer` properties. @@ -244,22 +242,20 @@ def build_config(cls, model: "DynapcnnNetwork") -> DynapcnnConfiguration: has_dvs_layer = False # TODO DVSLayer not supported yet. # Loop over layers in network and write corresponding configurations - for layer_index, ith_dcnnl in model.layers_mapper.items(): + for layer_index, ith_dcnnl in layers.items(): if isinstance(ith_dcnnl, DVSLayer): # TODO DVSLayer not supported yet. pass elif isinstance(ith_dcnnl, DynapcnnLayer): - # retrieve assigned core from the handler of this DynapcnnLayer (`ith_dcnnl`) instance. - chip_layer = config.cnn_layers[ - model.layers_handlers[layer_index].assigned_core - ] + # retrieve config dict for current layer + chip_layer = config.cnn_layers[layer2core_map[layer_index]] # write core configuration. cls.write_dynapcnn_layer_config( ith_dcnnl, chip_layer, - model.layers_handlers[layer_index], - model.layers_handlers, + destination_indices=destination_map[layer_index], + layer2core_map=layer2core_map, ) else: diff --git a/sinabs/backend/dynapcnn/config_builder.py b/sinabs/backend/dynapcnn/config_builder.py index dbbef3d7..037ad264 100644 --- a/sinabs/backend/dynapcnn/config_builder.py +++ b/sinabs/backend/dynapcnn/config_builder.py @@ -1,14 +1,15 @@ -import time from abc import ABC, abstractmethod -from typing import List +from typing import Dict, List import samna +from samna.dynapcnn.configuration import DynapcnnConfiguration import sinabs import sinabs.backend import sinabs.backend.dynapcnn from .dvs_layer import DVSLayer +from .dynapcnn_layer import DynapcnnLayer from .exceptions import InvalidModel from .mapping import LayerConstraints, get_valid_mapping @@ -35,7 +36,12 @@ def get_default_config(cls): @classmethod @abstractmethod - def build_config(cls, model: "DynapcnnNetwork", chip_layers: List[int]): + def build_config( + cls, + layers: Dict[int, DynapcnnLayer], + destination_map: Dict[int, List[int]], + layer2core_map: Dict[int, int], + ) -> DynapcnnConfiguration: """Build the configuration given a model. Parameters @@ -67,6 +73,7 @@ def monitor_layers(cls, config, layers: List[int]): @classmethod def get_valid_mapping(cls, model: "DynapcnnNetwork") -> List[int]: + # TODO: This should accept more explicit arguments """Find a valid set of layers for a given model. Parameters diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index 5be5f9f2..f5f38ec3 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -395,8 +395,10 @@ def _make_config( # TODO not handling DVSLayer yet. has_dvs_layer = isinstance(self._layers_mapper[0], DVSLayer) + # TODO: Replayce chip_layers_ordering with layer2core_map if chip_layers_ordering == "auto": # figure out mapping of each `DynapcnnLayer` into one core (core ID will be set in the layer's handler instance via `.assigned_core`). + # TODO: Argument should not be `self` _ = config_builder.get_valid_mapping(self) else: @@ -406,7 +408,11 @@ def _make_config( pass # update config (config. DynapcnnLayer instances into their assigned core). - config = config_builder.build_config(self) + config = config_builder.build_config( + layers=self.dynapcnn_layers, + destination_map=self.layer_destination_map, + layer2core_map=layer2core_map, + ) # TODO not handling DVSLayer yet (this is from the old implementation, should be revised). if self.input_shape and self.input_shape[0] == 1: From 5e59539d75fe60015faf44c9f672da11bcffc51e Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Wed, 23 Oct 2024 10:10:19 +0200 Subject: [PATCH 213/379] Update class definitions for ConfigBuilder child classes --- sinabs/backend/dynapcnn/chips/dynapcnn.py | 52 ++++++++++++++------ sinabs/backend/dynapcnn/chips/speck2cmini.py | 22 ++++++++- sinabs/backend/dynapcnn/chips/speck2e.py | 9 ---- sinabs/backend/dynapcnn/chips/speck2f.py | 24 +++++++-- 4 files changed, 78 insertions(+), 29 deletions(-) diff --git a/sinabs/backend/dynapcnn/chips/dynapcnn.py b/sinabs/backend/dynapcnn/chips/dynapcnn.py index 29442504..c56e8a67 100644 --- a/sinabs/backend/dynapcnn/chips/dynapcnn.py +++ b/sinabs/backend/dynapcnn/chips/dynapcnn.py @@ -4,7 +4,11 @@ import samna import torch -from samna.dynapcnn.configuration import DynapcnnConfiguration +from samna.dynapcnn.configuration import ( + CNNLayerConfig, + DynapcnnConfiguration, + DvsLayerConfig, +) import sinabs from sinabs.backend.dynapcnn.config_builder import ConfigBuilder @@ -26,7 +30,7 @@ def get_default_config(cls) -> "DynapcnnConfiguration": def get_dvs_layer_config_dict(cls, layer: DVSLayer): ... @classmethod - def write_dvs_layer_config(cls, layer: DVSLayer, config: "DvsLayerConfig"): + def write_dvs_layer_config(cls, layer: DVSLayer, config: DvsLayerConfig): for param, value in layer.get_config_dict().items(): setattr(config, param, value) @@ -79,7 +83,19 @@ def get_dynapcnn_layer_config_dict( layer2core_map: Dict[int, int], destination_indices: List[int], ) -> dict: - # TODO: Docstring + """Generate config dict from DynapcnnLayer instance + + Parameters + ---------- + - layer (DynapcnnLayer): Layer instance from which to generate the config + - layer2core_map (Dict): Keys are layer indices, values are corresponding + cores on hardware. Needed to map the destinations.] + - destination_indices (List): Indices of destination layers for `layer` + + Returns + ------- + - Dict that holds the information to configure the on-chip core + """ config_dict = {} config_dict["destinations"] = [{}, {}] @@ -181,17 +197,21 @@ def write_dynapcnn_layer_config( layer: DynapcnnLayer, layer2core_map: Dict[int, int], destination_indices: List[int], - chip_layer: "CNNLayerConfig", + chip_layer: CNNLayerConfig, ) -> None: - """Write a single layer configuration to the dynapcnn conf object. Uses the data in `layer` to configure a `CNNLayerConfig` to be + """Write a single layer configuration to the dynapcnn conf object. + + Uses the data in `layer` to configure a `CNNLayerConfig` to be deployed on chip. Parameters ---------- - - layer (DynapcnnLayer): the layer for which the condiguration will be written. - - chip_layer (CNNLayerConfig): configuration object representing the layer to which configuration is written. - - layer_handler (DynapcnnLayerHandler): ... - - all_handlers (dict): ... + - layer (DynapcnnLayer): Layer instance from which to generate the config + - layer2core_map (Dict): Keys are layer indices, values are corresponding + cores on hardware. Needed to map the destinations.] + - destination_indices (List): Indices of destination layers for `layer` + - chip_layer (CNNLayerConfig): Configuration object of the corrsesponding + on-chip core. Will be changed in-place based on `layer`. """ # extracting from a DynapcnnLayer the config. variables for its CNNLayerConfig. @@ -225,17 +245,19 @@ def build_config( destination_map: Dict[int, List[int]], layer2core_map: Dict[int, int], ) -> DynapcnnConfiguration: - # TODO: Update docstring - """Uses `DynapcnnLayer` objects to configure their equivalent chip core via a `CNNLayerConfig` object that is built - using using the `DynapcnnLayer` properties. + """Uses `DynapcnnLayer` objects to configure their equivalent chip cores Parameters ---------- - - model (DynapcnnNetwork): network instance used to read out `DynapcnnLayer` instances. + - layers (Dict): Keys are layer indices, values are DynapcnnLayer instances. + - layer2core_map (Dict): Keys are layer indices, values are corresponding + cores on hardware. Needed to map the destinations.] + - destination_indices (List): Indices of destination layers for `layer` Returns - ---------- - - config (DynapcnnConfiguration): an instance of a `DynapcnnConfiguration`. + ------- + - DynapcnnConfiguration: Config object holding the information to configure + the chip based on the provided `layers`. """ config = cls.get_default_config() diff --git a/sinabs/backend/dynapcnn/chips/speck2cmini.py b/sinabs/backend/dynapcnn/chips/speck2cmini.py index 5487b1e0..66b5b4d1 100644 --- a/sinabs/backend/dynapcnn/chips/speck2cmini.py +++ b/sinabs/backend/dynapcnn/chips/speck2cmini.py @@ -30,10 +30,28 @@ def get_output_buffer(cls): @classmethod def get_dynapcnn_layer_config_dict( - cls, layer: DynapcnnLayer, layers_mapper: Dict[int, DynapcnnLayer] + cls, + layer: DynapcnnLayer, + layer2core_map: Dict[int, int], + destination_indices: List[int], ) -> dict: + """Generate config dict from DynapcnnLayer instance + + Parameters + ---------- + - layer (DynapcnnLayer): Layer instance from which to generate the config + - layer2core_map (Dict): Keys are layer indices, values are corresponding + cores on hardware. Needed to map the destinations.] + - destination_indices (List): Indices of destination layers for `layer` + + Returns + ------- + - Dict that holds the information to configure the on-chip core + """ config_dict = super().get_dynapcnn_layer_config_dict( - layer=layer, layers_mapper=layers_mapper + layer=layer, + layer2core_map=layer2core_map, + destination_indices=destination_indices, ) config_dict.pop("weights_kill_bit") config_dict.pop("biases_kill_bit") diff --git a/sinabs/backend/dynapcnn/chips/speck2e.py b/sinabs/backend/dynapcnn/chips/speck2e.py index ef8faa92..799b9f98 100644 --- a/sinabs/backend/dynapcnn/chips/speck2e.py +++ b/sinabs/backend/dynapcnn/chips/speck2e.py @@ -30,12 +30,3 @@ def get_output_buffer(cls): @classmethod def set_kill_bits(cls, layer: DynapcnnLayer, config_dict: dict) -> dict: return config_dict - - @classmethod - def get_dynapcnn_layer_config_dict( - cls, layer: DynapcnnLayer, layers_mapper: Dict[int, DynapcnnLayer] - ) -> dict: - config_dict = super().get_dynapcnn_layer_config_dict( - layer=layer, layers_mapper=layers_mapper - ) - return config_dict diff --git a/sinabs/backend/dynapcnn/chips/speck2f.py b/sinabs/backend/dynapcnn/chips/speck2f.py index d7ee00cf..d34418f9 100644 --- a/sinabs/backend/dynapcnn/chips/speck2f.py +++ b/sinabs/backend/dynapcnn/chips/speck2f.py @@ -1,4 +1,4 @@ -from typing import Dict +from typing import Dict, List import samna from samna.speck2f.configuration import SpeckConfiguration @@ -29,10 +29,28 @@ def get_output_buffer(cls): @classmethod def get_dynapcnn_layer_config_dict( - cls, layer: DynapcnnLayer, layers_mapper: Dict[int, DynapcnnLayer] + cls, + layer: DynapcnnLayer, + layer2core_map: Dict[int, int], + destination_indices: List[int], ) -> dict: + """Generate config dict from DynapcnnLayer instance + + Parameters + ---------- + - layer (DynapcnnLayer): Layer instance from which to generate the config + - layer2core_map (Dict): Keys are layer indices, values are corresponding + cores on hardware. Needed to map the destinations.] + - destination_indices (List): Indices of destination layers for `layer` + + Returns + ------- + - Dict that holds the information to configure the on-chip core + """ config_dict = super().get_dynapcnn_layer_config_dict( - layer=layer, layers_mapper=layers_mapper + layer=layer, + layer2core_map=layer2core_map, + destination_indices=destination_indices, ) config_dict.pop("weights_kill_bit") config_dict.pop("biases_kill_bit") From 98eb42f19085ddccec9bde5780aedf4f4346709e Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Wed, 23 Oct 2024 10:14:01 +0200 Subject: [PATCH 214/379] Remove dynapcnn_layer_handler.py --- .../dynapcnn/dynapcnn_layer_handler.py | 141 ------------------ 1 file changed, 141 deletions(-) delete mode 100644 sinabs/backend/dynapcnn/dynapcnn_layer_handler.py diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer_handler.py b/sinabs/backend/dynapcnn/dynapcnn_layer_handler.py deleted file mode 100644 index 8f619711..00000000 --- a/sinabs/backend/dynapcnn/dynapcnn_layer_handler.py +++ /dev/null @@ -1,141 +0,0 @@ -# author : Willian Soares Girao -# contact : wsoaresgirao@gmail.com - -from typing import List, Optional - - -class DynapcnnLayerHandler: - """ - Class handling the pre-processing of network-level data into (device) layer-level data (i.e., arguments required for a `DynapcnnLayer` instantiation). - - Parameters - ---------- - - dpcnnl_index (int): the index/ID that will be associated with a `DynapcnnLayer` instance. This integer indexes a `dict` within `dcnnl_data` - that comprises a set of layers (`nn.Module`) and their respective I/O tensor shapes. - - dcnnl_data (dict): contains the nodes to be merged into this `DynapcnnLayer`, their I/O shapes and the index of the other `DynapcnnLayer`s to - be set as destinations. The `int` keys correspond to the nodes IDs associated `nn.Module`s (a single layer in the original network) becoming - part of this `DynapcnnLayer` instance, while the `str` keys correspond to this instance's destinations and re-scaling factors. - - discretize (bool): whether or not the weights/neuron parameters of the model will be quantized. - - sinabs_edges (list): each `nn.Module` within `dcnnl_data` is a node in the original computational graph describing a spiking network. An edge - `(A, B)` describes how modules forward data amongst themselves. This list is used by a `DynapcnnLayer` to figure out the number and - sequence of output tesnors its forward method needs to return. - - weight_rescaling_fn (callable): a method that handles how the re-scaling factor for one or more `SumPool2d` projecting to - the same convolutional layer are combined/re-scaled before applying them. - - entry_nodes (list): node IDs corresponding to layers in the original network that are input nodes (i.e., a "point of entry" for the external data). - """ - - def __init__( - self, - layer_index: int, - is_entry_node: bool, - destination_indices: List[int], - assigned_core: Optional[int] = None, - ): - self.layer_index = layer_index - self.entry_node = is_entry_node - self.destination_indices = destination_indices - self.assigned_core = assigned_core - - # TODO: Still needed? - # map destination nodes for each layer in this instance. - # self.nodes_destinations = self._get_destinations_input_source(sinabs_edges) - - ####################################################### Public Methods ####################################################### - - # TODO: Still needed? - def get_destination_dcnnl_index(self, dcnnl_id: int) -> int: - """The `forward` method will return as many tensors as there are elements in `self.dynapcnnlayer_destination`. Since the i-th returned tensor is to be - fed to the i-th destionation in `self.dynapcnnlayer_destination`, the return of this method can be used to index a tensor returned in the `forward` method. - - Parameters - ---------- - - dcnnl_id (int): this should be one of the values listed within `self.dynapcnnlayer_destination`. - - Returns - ---------- - - The index of `dcnnl_id` within `self.dynapcnnlayer_destination`. - """ - return self.dynapcnnlayer_destination.index(dcnnl_id) - - def __str__(self): - pretty_print = "\n" - - pretty_print += "COMPUTATIONAL NODES:\n\n" - - pretty_print += f"(node {self.conv_node_id}): {self.conv_layer}\n" - pretty_print += f"(node {self.spk_node_id}): {self.spk_layer}" - if len(self.pool_layer) != 0: - for idx, lyr in enumerate(self.pool_layer): - pretty_print += f"\n(node {self.pool_node_id[idx]}): {lyr}" - - pretty_print += "\n\nMETADATA:\n" - pretty_print += f"\n> network's entry point: {self.entry_point}" - pretty_print += ( - f"\n> convolution's weight re-scaling factor: {self.conv_rescaling_factor}" - ) - pretty_print += f"\n> assigned core index: {self.assigned_core}" - pretty_print += ( - f"\n> destination DynapcnnLayers: {self.dynapcnnlayer_destination}" - ) - - for node, destinations in self.nodes_destinations.items(): - pretty_print += f"\n> node {node} feeds input to nodes {destinations}" - - return pretty_print - - ####################################################### Private Methods ####################################################### - - # TODO: Still needed? - def _get_destinations_input_source(self, sinabs_edges: list) -> dict: - """Creates a mapping between each layer in this `DynapcnnLayer` instance and its targe nodes that are part of different - `DynapcnnLayer` instances. This mapping is used to figure out how many tensors the `forward` method needs to return. - - Parameters - ---------- - - sinabs_edges (list): each `nn.Module` within `dcnnl_data` is a node in the original computational graph describing a spiking - network. An edge `(A, B)` describes how modules forward data amongst themselves. This list is used by a `DynapcnnLayer` to - figure out the number and sequence of output tesnors its forward method needs to return. - - Returns - ---------- - - destinations_input_source (dict): maps a `nn.Module` within this `DynapcnnLayer` to the nodes it provides the input to. - """ - destinations_input_source = {} - - # check whether spiking layer projects outside this DynapcnnLayer (i.e. to one of the destinations without passing through a pooling). - spk_destinations = [] - for edge in sinabs_edges: - if edge[0] == self.spk_node_id and edge[1] not in self.pool_node_id: - # spiking layer projects to a node outside this DynapcnnLayer. - spk_destinations.append(edge[1]) - if len(spk_destinations) > 0: - destinations_input_source[self.spk_node_id] = [] - for node_id in spk_destinations: - destinations_input_source[self.spk_node_id].append(node_id) - - # get `pooling->destination` mapping. The pooling outputs will be arranged sequentially since the pooling layers are added sequentially - # to `self.pool_layer` (i.e., as they appear in the computational graph of the original `nn.Module`). - for id in self.pool_node_id: - destinations_input_source[id] = [] - for edge in sinabs_edges: - if edge[0] == id: - destinations_input_source[id].append(edge[1]) - - return destinations_input_source - - # TODO: Still needed? - @staticmethod - def find_nodes_core_id(node: int, all_handlers: dict) -> int: - """Loops through all handlers in `all_handlers` to find to which core a `DynapcnnLayer` containing - `node` has been assigned to.""" - - for _, dcnnl in all_handlers.items(): - - if ( - node == dcnnl["layer_handler"].conv_node_id - or node == dcnnl["layer_handler"].spk_node_id - or node in dcnnl["layer_handler"].pool_node_id - ): - return dcnnl["layer_handler"].assigned_core - - raise ValueError(f"Node {node} not found in any of the cores.") From 62f1e8846461d6b5e10f3e3c784fae1efbfe22e0 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Wed, 23 Oct 2024 16:28:43 +0200 Subject: [PATCH 215/379] Replace `chip_layers_ordering` by layer2core_map. --- sinabs/backend/dynapcnn/config_builder.py | 35 +---- sinabs/backend/dynapcnn/dynapcnn_network.py | 145 +++++++++++------ sinabs/backend/dynapcnn/mapping.py | 166 ++++++++++++-------- 3 files changed, 207 insertions(+), 139 deletions(-) diff --git a/sinabs/backend/dynapcnn/config_builder.py b/sinabs/backend/dynapcnn/config_builder.py index 037ad264..613ac7f6 100644 --- a/sinabs/backend/dynapcnn/config_builder.py +++ b/sinabs/backend/dynapcnn/config_builder.py @@ -72,44 +72,19 @@ def monitor_layers(cls, config, layers: List[int]): """Enable the monitor for a given set of layers in the config object.""" @classmethod - def get_valid_mapping(cls, model: "DynapcnnNetwork") -> List[int]: - # TODO: This should accept more explicit arguments - """Find a valid set of layers for a given model. + def map_layers_to_cores(cls, layers: Dict[int, DynapcnnLayer]) -> Dict[int]: + """Find a mapping from DynapcnnLayers onto on-chip cores Parameters ---------- - model (DynapcnnNetwork): - A model + - layers: Dict with layer indices as keys and DynapcnnLayer instances as values Returns ------- - - chip_layers_ordering (list): the core indices corresponding to each layer of the model. Though this list is being returned, each core index - `core_idx` is assigned directyl to each `DynapcnnLayer` instance via accesses to `model.layers_mapper`. + - Dict mapping layer indices (keys) to assigned core IDs (values). """ - chip_layers_ordering = [] - - if type(model) == sinabs.backend.dynapcnn.dynapcnn_network.DynapcnnNetwork: - mapping = get_valid_mapping(model, cls.get_constraints()) - - if isinstance(model.layers_mapper[0], DVSLayer): - # TODO not handling DVSLayer yet. - # TODO if the architecture has more than one `DynapcnnLayer`s acting as input node of the model - # thi check will be wrong since it assumes the network has a single input node `model.layers_mapper[0]`. - pass - - for dcnnl_idx, core_idx in mapping: - # save the core index information on the handler of this `DynapcnnLayer` instance. - model.layers_handlers[dcnnl_idx][ - "layer_handler" - ].assigned_core = core_idx - chip_layers_ordering.append(core_idx) - - else: - raise InvalidModel(model) - - # return kept but its information is not used beyond this point (core indices already part of each `DynapcnnLayerHandler` instance). - return chip_layers_ordering + return get_valid_mapping(layers, cls.get_constraints()) @classmethod def validate_configuration(cls, config) -> bool: diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index f5f38ec3..9c78e2ad 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -3,6 +3,7 @@ import time from typing import Callable, Dict, List, Optional, Sequence, Tuple, Union +from warnings import warn import samna import torch @@ -57,6 +58,7 @@ def __init__( dvs_input = False self.dvs_input = dvs_input self.input_shape = input_shape + self._layer2core_map = None assert len(self.input_shape) == 3, "infer_input_shape did not return 3-tuple" @@ -91,15 +93,23 @@ def dynapcnn_module(self): def layer_destination_map(self): return self._dynapcnn_module.destination_map + @property + def layer2core_map(self): + return self._layer2core_map + @property def chip_layers_ordering(self): - return self._chip_layers_ordering + warn( + "`chip_layers_ordering` is deprecated. Returning `layer2core_map` instead.", + DeprecationWarning + ) + return self._layer2core_map def get_output_core_id(self) -> int: """.""" # TODO if a network with two output layers is deployed, which is not supported yet btw, this monitoring part needs to be revised. - for _, ith_dcnnl in self._layers_mapper.items(): + for _, ith_dcnnl in self._dynapcnn_layers.items(): if len(ith_dcnnl.dynapcnnlayer_destination) == 0: # a DynapcnnLayer without destinations is taken to be the output layer of the network. return ith_dcnnl.assigned_core @@ -109,7 +119,7 @@ def get_input_core_id(self) -> list: a list of all core IDs to which an input layer of the network has been assigned to. """ entry_points = [] - for _, ith_dcnnl in self._layers_mapper.items(): + for _, ith_dcnnl in self._dynapcnn_layers.items(): if ith_dcnnl.entry_point: entry_points.append(ith_dcnnl.assigned_core) @@ -204,7 +214,7 @@ def parameters(self) -> list: """ parameters = [] - for layer in self._layers_mapper.values(): + for layer in self._dynapcnn_layers.values(): if isinstance(layer, sinabs.backend.dynapcnn.dynapcnn_layer.DynapcnnLayer): parameters.extend(layer.conv_layer.parameters()) @@ -217,14 +227,14 @@ def init_weights(self, init_fn: nn.init = nn.init.xavier_normal_) -> None: ---------- - init_fn (torch.nn.init): the weight initialization method to be used. """ - for layer in self._layers_mapper.values(): + for layer in self._dynapcnn_layers.values(): if isinstance(layer, sinabs.backend.dynapcnn.dynapcnn_layer.DynapcnnLayer): init_fn(layer.conv_layer.weight.data) def detach_neuron_states(self) -> None: """Detach the neuron states and activations from current computation graph (necessary).""" - for module in self._layers_mapper.values(): + for module in self._dynapcnn_layers.values(): if isinstance(module, sinabs.backend.dynapcnn.dynapcnn_layer.DynapcnnLayer): if isinstance(module.spk_layer, sl.StatefulLayer): for name, buffer in module.spk_layer.named_buffers(): @@ -232,13 +242,16 @@ def detach_neuron_states(self) -> None: def to( self, - device="cpu", - chip_layers_ordering="auto", + device: str = "cpu", monitor_layers: Optional[Union[List, str]] = None, - config_modifier=None, - slow_clk_frequency: int = None, + config_modifier: Optional[Callable] = None, + slow_clk_frequency: Optional[int] = None, + layer2core_map: Union[Dict[int, int], str] = "auto", + chip_layers_ordering="auto", ): - """Note that the model parameters are only ever transferred to the device on the `to` call, + """ Deploy model to cpu, gpu or a SynSense device. + + Note that the model parameters are only ever transferred to the device on the `to` call, so changing a threshold or weight of a model that is deployed will have no effect on the model on chip until `to` is called again. @@ -248,13 +261,6 @@ def to( device: String cpu:0, cuda:0, dynapcnndevkit, speck2devkit - chip_layers_ordering: sequence of integers or `auto` - The order in which the dynapcnn layers will be used. If `auto`, - an automated procedure will be used to find a valid ordering. - A list of layers on the device where you want each of the model's DynapcnnLayers to be placed. - The index of the core on chip to which the i-th layer in the model is mapped is the value of the i-th entry in the list. - Note: This list should be the same length as the number of dynapcnn layers in your model. - monitor_layers: None/List A list of all layers in the module that you want to monitor. Indexing starts with the first non-dvs layer. If you want to monitor the dvs layer for eg. @@ -268,6 +274,22 @@ def to( A user configuration modifier method. This function can be used to make any custom changes you want to make to the configuration object. + layer2core_map (dict or "auto"): Defines how cores on chip are + assigned to DynapcnnLayers. If `auto`, an automated procedure + will be used to find a valid ordering. Otherwise a dict needs + to be passed, with DynapcnnLayer indices as keys and assigned + core IDs as values. DynapcnnLayer indices have to match those of + `self.dynapcnn_layers`. + + chip_layers_ordering: sequence of integers or `auto` + The order in which the dynapcnn layers will be used. If `auto`, + an automated procedure will be used to find a valid ordering. + A list of layers on the device where you want each of the model's DynapcnnLayers to be placed. + The index of the core on chip to which the i-th layer in the model is mapped is the value of the i-th entry in the list. + Note: This list should be the same length as the number of dynapcnn layers in your model. + Note: This parameter is obsolete and should not be passed anymore. Use + `layer2core_map` instead. + Note ---- chip_layers_ordering and monitor_layers are used only when using synsense devices. @@ -285,6 +307,7 @@ def to( # generate config. config = self._make_config( + layer2core_map=layer2core_map, chip_layers_ordering=chip_layers_ordering, device=device, monitor_layers=monitor_layers, @@ -344,26 +367,24 @@ def to( def _make_config( self, - chip_layers_ordering: Union[Sequence[int], str] = "auto", - device="dynapcnndevkit:0", + layer2core_map: Union[Dict[int, int], str] = "auto", + device: str = "dynapcnndevkit:0", monitor_layers: Optional[Union[List, str]] = None, - config_modifier=None, + config_modifier: Optional[Callable] = None, + chip_layers_ordering: Optional[Union[Sequence[int], str]] = None, ): """Prepare and output the `samna` DYNAPCNN configuration for this network. Parameters ---------- - - chip_layers_ordering: sequence of integers or `auto` - The order in which the dynapcnn layers will be used. If `auto`, - an automated procedure will be used to find a valid ordering. - A list of layers on the device where you want each of the model's DynapcnnLayers to be placed. - Note: This list should be the same length as the number of dynapcnn layers in your model. - - device: String - dynapcnndevkit, speck2b or speck2devkit - - monitor_layers: None/List/Str + - layer2core_map (dict or "auto"): Defines how cores on chip are + assigned to DynapcnnLayers. If `auto`, an automated procedure + will be used to find a valid ordering. Otherwise a dict needs + to be passed, with DynapcnnLayer indices as keys and assigned + core IDs as values. DynapcnnLayer indices have to match those of + `self.dynapcnn_layers`. + - device: (string): dynapcnndevkit, speck2b or speck2devkit + - monitor_layers: None/List/Str A list of all layers in the module that you want to monitor. Indexing starts with the first non-dvs layer. If you want to monitor the dvs layer for eg. :: @@ -374,9 +395,16 @@ def _make_config( If this value is left as None, by default the last layer of the model is monitored. - config_modifier: + - config_modifier (Callable or None): A user configuration modifier method. This function can be used to make any custom changes you want to make to the configuration object. + - chip_layers_ordering (None, sequence of integers or "auto", obsolete): + The order in which the dynapcnn layers will be used. If `auto`, + an automated procedure will be used to find a valid ordering. + A list of layers on the device where you want each of the model's DynapcnnLayers to be placed. + Note: This list should be the same length as the number of dynapcnn layers in your model. + Note: This parameter is obsolete and should not be passed anymore. Use + `layer2core_map` instead. Returns ------- @@ -393,19 +421,44 @@ def _make_config( config_builder = ChipFactory(device).get_config_builder() # TODO not handling DVSLayer yet. - has_dvs_layer = isinstance(self._layers_mapper[0], DVSLayer) - - # TODO: Replayce chip_layers_ordering with layer2core_map - if chip_layers_ordering == "auto": - # figure out mapping of each `DynapcnnLayer` into one core (core ID will be set in the layer's handler instance via `.assigned_core`). - # TODO: Argument should not be `self` - _ = config_builder.get_valid_mapping(self) - + has_dvs_layer = isinstance(self._dynapcnn_layers[0], DVSLayer) + + if chip_layers_ordering is not None: + if layer2core_map is not None: + warn( + "Both `chip_layers_ordering` and `layer2core_map are provided. " + "Please only provide `layer2core_map`, as `chip_layers_ordering` " + "is deprecated.", + DeprecationWarning, + ) + elif chip_layers_ordering == "auto": + warn( + "The parameter `chip_layers_ordering` is deprecated. Passing " + "'auto' is still accepted, but in the future please use " + "`layer2core_map` instead.", + DeprecationWarning, + ) + layer2core_map = "auto" + else: + raise ValueError( + "`chip_layers_ordering` is deprecated. Passing anything other " + "than `None` or 'auto' is not possible. To manually assign core " + "to layers, please use the `layer2core_map` argument." + ) + if layer2core_map == "auto": + # Assign chip core ID for each DynapcnnLayer. + layer2core_map = config_builder.map_layers_to_cores(self.dynapcnn_layers) else: - # TODO - mapping from each DynapcnnLayer into cores has been provided by the user: NOT IMPLEMENTED YET. + if not layer2core_map.keys() == self.dynapcnn_layers.keys(): + raise ValueError( + "The keys provided in `layer2core_map` must exactly match " + "the keys in `self.dynapcnn_layers`" + ) + if has_dvs_layer: # TODO not handling DVSLayer yet. pass + self._layer2core_map = layer2core_map # update config (config. DynapcnnLayer instances into their assigned core). config = config_builder.build_config( @@ -422,7 +475,7 @@ def _make_config( monitor_chip_layers = [] if monitor_layers is None: # check if any monitoring is enabled (if not, enable monitoring for the last layer). - for dcnnl_index, ith_dcnnl in self._layers_mapper.items(): + for dcnnl_index, ith_dcnnl in self._dynapcnn_layers.items(): # TODO if a network with two output layers is deployed, which is not supported yet btw, this monitoring part needs to be revised. if ( @@ -439,7 +492,7 @@ def _make_config( ) elif monitor_layers == "all": - for dcnnl_index, ith_dcnnl in self._layers_mapper.items(): + for dcnnl_index, ith_dcnnl in self._dynapcnn_layers.items(): # TODO not handling DVSLayer yet # monitor each chip core (if not a DVSLayer). if not isinstance(ith_dcnnl, DVSLayer): @@ -468,7 +521,7 @@ def _make_config( def _to_device(self, device: torch.device) -> None: """Access each sub-layer within all `DynapcnnLayer` instances and call `.to(device)` on them.""" - for layer in self._layers_mapper.values(): + for layer in self._dynapcnn_layers.values(): if isinstance(layer, sinabs.backend.dynapcnn.dynapcnn_layer.DynapcnnLayer): layer.to(device) @@ -477,7 +530,7 @@ def _to_device(self, device: torch.device) -> None: def __str__(self): pretty_print = "" - for idx, layer_data in self._layers_mapper.items(): + for idx, layer_data in self._dynapcnn_layers.items(): pretty_print += f"----------------------- [ DynapcnnLayer {idx} ] -----------------------\n" pretty_print += f"{layer_data}\n\n" diff --git a/sinabs/backend/dynapcnn/mapping.py b/sinabs/backend/dynapcnn/mapping.py index 504c8ebc..afd7dbfc 100644 --- a/sinabs/backend/dynapcnn/mapping.py +++ b/sinabs/backend/dynapcnn/mapping.py @@ -1,7 +1,7 @@ from collections import deque from copy import deepcopy from dataclasses import dataclass -from typing import List, Optional, Tuple, Union +from typing import Dict, List, Optional, Tuple, Union import sinabs @@ -47,58 +47,76 @@ def find_chip_layers( def get_valid_mapping( - model: Union["DynapcnnNetwork"], constraints: List[LayerConstraints] -) -> List[Tuple[int, int]]: + layers: Dict[int, DynapcnnLayer], constraints: List[LayerConstraints] +) -> Dict[int, int]: """Given a model, find a valid layer ordering for its placement within the constraints provided. Parameters ---------- - model: an instance of a DynapcnnNetwork or a DynapcnnNetworkGraph. - constraints: a list of all the layer's constraints. + - model: an instance of a DynapcnnNetwork or a DynapcnnNetworkGraph. + - constraints: a list of all the layer's constraints. Returns - netmap: a list of tuples with (dynapcnnlayer index, core index). + - Dict mapping from layer index (key) to assigned core ID (value) ------- """ + # Store layer indices and lists of possible target chips in separate lists + layer_indices = [] layer_mapping = [] + for layer_index, this_layer in layers.items(): + # Skip DVSLayers + if isinstance(this_layer, DynapcnnLayer): + chip_layers = find_chip_layers(this_layer, constraints) + layer_mapping[layer_index] = chip_layers + # Make sure only DynapcnnLayers and DVSLayers are passed + elif not isinstance(this_layer, DVSLayer): + raise ValueError(f"Found unexpected layer type: `{type(this_layer)}") - if type(model) == sinabs.backend.dynapcnn.dynapcnn_network.DynapcnnNetwork: - for dcnnl_index, ith_dcnnl in model.layers_mapper.items(): - if isinstance(ith_dcnnl, DynapcnnLayer): - layer_mapping.append(find_chip_layers(ith_dcnnl, constraints)) - else: - raise ValueError( - f"Layer {dcnnl_index} is not an instance of `DynapcnnLayer`." - ) + graph = make_flow_graph(layer_mapping, len(constraints)) - graph = make_flow_graph(layer_mapping, len(constraints)) + # use Edmonds' Algorithm to find suitable cores for each DynapcnnLayer. + new_graph = edmonds(graph, 0, len(graph) - 1) + netmap = recover_mapping(new_graph, layer_mapping) - # use graph algorithm to find suitable cores for each DynapcnnLayer. - new_graph = edmonds(graph, 0, len(graph) - 1) - - netmap = recover_mapping(new_graph, layer_mapping) - - else: - raise InvalidModel(model) - - return netmap + # Convert `netmap` to dict mapping from layer index to core ID + return { + layer_idx: core_id for layer_idx, core_id in zip(layer_indices, netmap) + } @dataclass -class Edge: +class FlowGraphEdge: s: int t: int cap: int flow: int = 0 - rev: Optional["Edge"] = None + rev: Optional["FlowGraphEdge"] = None def __repr__(self): - return f"Edge from {self.s} to {self.t} with capacity {self.cap} and flow {self.flow}" + return f"FlowGraphEdge from {self.s} to {self.t} with capacity {self.cap} and flow {self.flow}" -# graph is list of list of edges. Each edge is -def edmonds(graph, source, sink, verbose: bool = False): +def edmonds( + graph: List[List[FlowGraphEdge]], source: int, sink: int, verbose: bool = False +) -> List[List[FlowGraphEdge]]: + """Use Edmonds' Algorithm to compute flow of flow graph + + Makes a copy of the graph. The original graph is not changed in place. + + Parameters + ---------- + - graph List[List[FlowGraphEdge]]): Flow graph representation. Each list entry + corresponds to a node and consists of a list holding the outgoing edges + from this node. + - source (int): Index of source node within graph + - sind (int): Index of sink node within graph + - verbose (bool): Print detailed flow information if `True` + + Returns + ------- + List[List[FlowGraphEdge]]: New flow graph with calculated flow + """ graph = deepcopy(graph) flow = 0 while True: @@ -133,31 +151,32 @@ def edmonds(graph, source, sink, verbose: bool = False): def make_flow_graph( layer_mapping: List[List[int]], num_layers: int = 9 -) -> List[List[Edge]]: - """Make a flow graph given all possible chip layers for each DynapcnnCompatibleLayer layer. - Note that the flows are not computed yet. The flow for the graph generated here needs to be - populated by calling the method `edmonds` +) -> List[List[FlowGraphEdge]]: + """Make a flow graph given all possible chip cores for each software layer. + + Note that the flows are not computed yet. The flow for the graph generated here + needs to be populated by calling the method `edmonds` Parameters ---------- - layer_mapping: - List of a list of all layer indices. Eg. [[1,3], [4, 6, 1]] for a two layer model - num_layers: - Number of layers on the chip + - layer_mapping: List of a list of matching chip core indices for each software layer. + Eg. [[1,3], [4, 6, 1]] for a two layer model + - num_layers (int): Number of layers on the chip Returns ------- - graph: List[List[Edge]] + List[List[FlowGraphEdge]]: Flow graph representation. Each list entry corresponds + to a node and consists of a list holding the outgoing edges from this node. """ graph = [] # add all our nodes # one source node graph.append([]) # one node for every layer that will be mapped - for x in range(len(layer_mapping)): + for __ in range(len(layer_mapping)): graph.append([]) # one node for every chip layer - for x in range(num_layers): + for __ in range(num_layers): graph.append([]) # one sink node graph.append([]) @@ -165,41 +184,62 @@ def make_flow_graph( target_offset = len(layer_mapping) + 1 # first from source to all layers for i in range(len(layer_mapping)): - graph[0].append(Edge(s=0, t=i + 1, cap=1, flow=0)) - # add the reverse edge - graph[i + 1].append(Edge(s=i + 1, t=0, cap=0, flow=0)) + source_to_layer = FlowGraphEdge(s=0, t=i + 1, cap=1, flow=0) + layer_to_source = FlowGraphEdge(s=i + 1, t=0, cap=0, flow=0) # fill in reverse pointers - graph[0][-1].rev = graph[i + 1][-1] - graph[i + 1][-1].rev = graph[0][-1] + source_to_layer.rev = layer_to_source + layer_to_source.rev = source_to_layer + # append new edges + graph[0].append(source_to_layer) + graph[i + 1].append(layer_to_source) # then from layers to chip layers for i, layer_targets in enumerate(layer_mapping): for target in layer_targets: - graph[i + 1].append(Edge(s=i + 1, t=target + target_offset, cap=1, flow=0)) - graph[target + target_offset].append( - Edge(s=target + target_offset, t=i + 1, cap=0, flow=0) + layer_to_chip = FlowGraphEdge( + s=i + 1, t=target + target_offset, cap=1, flow=0 + ) + chip_to_layer = FlowGraphEdge( + s=target + target_offset, t=i + 1, cap=0, flow=0 ) - graph[i + 1][-1].rev = graph[target + target_offset][-1] - graph[target + target_offset][-1].rev = graph[i + 1][-1] - # print(graph) + layer_to_chip.rev = chip_to_layer + chip_to_layer.rev = layer_to_chip + graph[i + 1].append(layer_to_chip) + graph[target + target_offset].append(chip_to_layer) # then from chip layers to sink - for i, layer in enumerate(graph[target_offset:-1]): - sink = len(graph) - 1 - source = i + target_offset - graph[source].append(Edge(s=source, t=sink, cap=1, flow=0)) - graph[sink].append(Edge(s=sink, t=source, cap=0, flow=0)) - graph[source][-1].rev = graph[sink][-1] + sink = len(graph) - 1 + for chip_node in range(target_offset, sink): + graph[chip_node].append(FlowGraphEdge(s=chip_node, t=sink, cap=1, flow=0)) + graph[sink].append(FlowGraphEdge(s=sink, t=chip_node, cap=0, flow=0)) + graph[chip_node][-1].rev = graph[sink][-1] graph[sink][-1].rev = graph[sink][-1] return graph -def recover_mapping(graph, layer_mapping) -> List[Tuple[int, int]]: +def recover_mapping( + graph: List[List[FlowGraphEdge]], num_layers: int +) -> List[int]: + """Based on the flow graph retrieve a layer-to-core mapping + + Parameters + ---------- + - graph List[List[FlowGraphEdge]]): Flow graph representation with flow calculated. + Each list entry corresponds to a node and consists of a list holding the + outgoing edges from this node. + - num_layers (int): Number of software layers + + Returns + ------- + List[int]: Assigned core IDs for each layer in order. + """ mapping = [] - for i, layer in enumerate(layer_mapping): - for edge in graph[i + 1]: + for i in range(1, num_layers + 1): # `+1` to skip source node + for edge in graph[i]: if edge.flow == 1: - mapping.append((i, edge.t - len(layer_mapping) - 1)) - if len(mapping) != len(layer_mapping): - raise ValueError("One of the DynapcnnLayers could not be mapped to any core.") + mapping.append(edge.t - num_layers - 1) + if len(mapping) != num_layers: + raise ValueError( + "One or more of the DynapcnnLayers could not be mapped to any core." + ) return mapping From 0d77c739e80c637092a028aeada1be729a1f3f23 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Wed, 23 Oct 2024 17:20:20 +0200 Subject: [PATCH 216/379] Update DynapcnnNetwork layer monitoring --- .../backend/dynapcnn/dynapcnn_layer_utils.py | 3 +- sinabs/backend/dynapcnn/dynapcnn_network.py | 52 ++++++-------- .../dynapcnn/dynapcnnnetwork_module.py | 71 ++++++++++++++----- 3 files changed, 79 insertions(+), 47 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py b/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py index 541d06dd..3db72357 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py @@ -263,7 +263,8 @@ def construct_destination_map(dcnnl_map: Dict[int, Dict]) -> Dict[int, List[int] Returns ------- Dict with layer indices (int) as keys and list of destination indices (int) as values. - Layer outputs that are not sent to other dynapcnn layers are represented by negative indices. + Layer outputs that are not sent to other dynapcnn layers are considered + exit points of the network and represented by negative indices. """ destination_map = dict() for layer_index, layer_info in dcnnl_map.items(): diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index 9c78e2ad..daf1d2fd 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -14,6 +14,7 @@ import sinabs.layers as sl from .chip_factory import ChipFactory +from .dynapcnn_layer import DynapcnnLayer from .dvs_layer import DVSLayer from .io import disable_timestamps, enable_timestamps, open_device, reset_timestamps from .nir_graph_extractor import GraphExtractor @@ -215,7 +216,7 @@ def parameters(self) -> list: parameters = [] for layer in self._dynapcnn_layers.values(): - if isinstance(layer, sinabs.backend.dynapcnn.dynapcnn_layer.DynapcnnLayer): + if isinstance(layer, DynapcnnLayer): parameters.extend(layer.conv_layer.parameters()) return parameters @@ -228,14 +229,14 @@ def init_weights(self, init_fn: nn.init = nn.init.xavier_normal_) -> None: - init_fn (torch.nn.init): the weight initialization method to be used. """ for layer in self._dynapcnn_layers.values(): - if isinstance(layer, sinabs.backend.dynapcnn.dynapcnn_layer.DynapcnnLayer): + if isinstance(layer, DynapcnnLayer): init_fn(layer.conv_layer.weight.data) def detach_neuron_states(self) -> None: """Detach the neuron states and activations from current computation graph (necessary).""" for module in self._dynapcnn_layers.values(): - if isinstance(module, sinabs.backend.dynapcnn.dynapcnn_layer.DynapcnnLayer): + if isinstance(module, DynapcnnLayer): if isinstance(module.spk_layer, sl.StatefulLayer): for name, buffer in module.spk_layer.named_buffers(): buffer.detach_() @@ -472,37 +473,30 @@ def _make_config( config.dvs_layer.merge = True # TODO all this monitoring part needs validation still. - monitor_chip_layers = [] if monitor_layers is None: - # check if any monitoring is enabled (if not, enable monitoring for the last layer). - for dcnnl_index, ith_dcnnl in self._dynapcnn_layers.items(): - - # TODO if a network with two output layers is deployed, which is not supported yet btw, this monitoring part needs to be revised. - if ( - len( - self._dynapcnnlayers_handlers[ - dcnnl_index - ].dynapcnnlayer_destination - ) - == 0 - ): - # a DynapcnnLayer without destinations is taken to be the output layer of the network. - monitor_chip_layers.append( - self._dynapcnnlayers_handlers[dcnnl_index].assigned_core - ) + # Monitor all layers with exit point destinations + monitor_layers = self._dynapcnn_module.get_exit_layers() elif monitor_layers == "all": - for dcnnl_index, ith_dcnnl in self._dynapcnn_layers.items(): - # TODO not handling DVSLayer yet - # monitor each chip core (if not a DVSLayer). - if not isinstance(ith_dcnnl, DVSLayer): - monitor_chip_layers.append( - self._dynapcnnlayers_handlers[dcnnl_index].assigned_core - ) + monitor_layers = [ + lyr_idx for lyr_idx, layer in self.dynapcnn_layers.items() + if not isinstance(layer, DVSLayer) + ] - if monitor_layers: - if "dvs" in monitor_layers: + # Collect cores (chip layers) that are to be monitored + monitor_chip_layers = [] + for lyr_idx in monitor_layers: + if lyr_idx.lower() == "dvs": monitor_chip_layers.append("dvs") + else: + # Warn when recording layers with pooling + if any(p != (1, 1) for p in self.dynapcnn_layers[lyr_idx].pool): + warn( + f"Monitored layer {lyr_idx} has at least one destination " + "with pooling. Note that monitored events will be recorded " + "before pooling" + ) + monitor_chip_layers.append(layer2core_map[lyr_idx]) # enable monitors on the specified layers. config_builder.monitor_layers(config, monitor_chip_layers) diff --git a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py index ce0b6b3e..8128d525 100644 --- a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py +++ b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py @@ -75,6 +75,49 @@ def sorted_nodes(self): def node_source_map(self): return self._node_source_map + def get_exit_layers(self) -> List[int]: + """ Get layers that act as exit points of the network + + Returns + ------- + - List[int]: Layer indices with at least one exit destination. + """ + return [ + layer_idx + for layer_idx, destinations in self.destination_map.items() + if any(d < 0 for d in destinations) + ] + + def get_exit_points(self): + """ Get details of layers that act as exit points of the network + + Returns + ------- + - Dict[int, Dict]: Dict whose keys are layer indices of `dynapcnn_layers` + with at least one exit destination. Values are list of dicts, providing + for each exit destination the negative valued ID ('destination_id'), + the index of that destination within the list of destinations of the + corresponding `DynapcnnLayer` ('destination_index'), and the pooling + for this destination. + """ + exit_layers = dict() + for layer_idx, destinations in self.destination_map.items(): + exit_destinations = [] + for i, dest in enumerate(destinations): + if dest < 0: + exit_destinations.append( + { + "destination_id": dest, + "destination_index": i, + "pooling": self.dynapcnn_layers[layer_idx].pool[i], + } + ) + if exit_destinations: + exit_layers[layer_idx] = exit_destinations + + return exit_layers + + def setup_dynapcnnlayer_graph(self, index_layers_topologically: bool = False): """ Set up data structures to run forward pass through dynapcnn layers @@ -166,15 +209,15 @@ def forward( dict, with layer indices as keys, and nested dicts as values, which hold destination indices as keys and output tensors as values. * If `return_complete` is `False` and there is only a single destination - in the whole network that is marked as final (i.e. destination + in the whole network that is marked as exit point (i.e. destination index in dynapcnn layer handler is negative), it will return the output as a single tensor. * If `return_complete` is `False` and no destination in the network - is marked as final, a warning will be raised and the function + is marked as exit point, a warning will be raised and the function returns an empty dict. * In all other cases a dict will be returned that is of the same structure as if `return_complete` is `True`, but only with entries - where the destination is marked as final. + where the destination is marked as exit point. """ if not hasattr(self, "_sorted_nodes"): @@ -217,24 +260,24 @@ def forward( if return_complete: return layers_outputs - # Take outputs with final destinations as network output + # Take outputs with exit point destinations as network output network_outputs = {} for layer_idx, outputs in layers_outputs.items(): - final_outputs = { - abs(idx_dest): out for idx_dest, out in outputs.items() if idx_dest < 0 + outputs = { + idx_dest: out for idx_dest, out in outputs.items() if idx_dest < 0 } - if final_outputs: - network_outputs[layer_idx] = final_outputs + if outputs: + network_outputs[layer_idx] = outputs # If no outputs have been found return None and warn if not network_outputs: warn( - "No final outputs have been found. Try setting `return_complete` " - "`True` to get all outputs, or mark final outputs by setting " + "No exit points have been found. Try setting `return_complete` " + "`True` to get all outputs, or mark exit points by setting " "corresponding destination layer indices in destination_map " " to negative integer values" ) - return + return dict() # Special case with single output: return single tensor if ( @@ -257,12 +300,6 @@ def reindex_layers(self, index_order: List[int]): index_order: List of integers indicating new order of layers: Position of layer index within this list indicates new index """ - def negative_default(key): - if isinstance(key, int) and key < 0: - return key - else: - raise KeyError(key) - mapping = {old: new for new, old in enumerate(index_order)} def remap(key): From 59c3a427f6858dbf0943346193a31bfc6e1b7360 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Wed, 23 Oct 2024 17:32:22 +0200 Subject: [PATCH 217/379] Remove now obsolete methods `get_output_core_id` and `get_input_core_id` from DynapcnnNetwork --- sinabs/backend/dynapcnn/dynapcnn_network.py | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index daf1d2fd..ef025703 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -106,26 +106,6 @@ def chip_layers_ordering(self): ) return self._layer2core_map - def get_output_core_id(self) -> int: - """.""" - - # TODO if a network with two output layers is deployed, which is not supported yet btw, this monitoring part needs to be revised. - for _, ith_dcnnl in self._dynapcnn_layers.items(): - if len(ith_dcnnl.dynapcnnlayer_destination) == 0: - # a DynapcnnLayer without destinations is taken to be the output layer of the network. - return ith_dcnnl.assigned_core - - def get_input_core_id(self) -> list: - """Since the chip allows for multiple input layers (that merge into a single output at some point), this method returns - a list of all core IDs to which an input layer of the network has been assigned to. - """ - entry_points = [] - for _, ith_dcnnl in self._dynapcnn_layers.items(): - if ith_dcnnl.entry_point: - entry_points.append(ith_dcnnl.assigned_core) - - return entry_points - def hw_forward(self, x): """Forwards data through the chip.""" From 788758310eda002d0b5780afcd0428d6de9336bc Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Thu, 24 Oct 2024 14:21:04 +0200 Subject: [PATCH 218/379] Fix minor import issues --- sinabs/backend/dynapcnn/chips/dynapcnn.py | 4 ++-- sinabs/backend/dynapcnn/config_builder.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sinabs/backend/dynapcnn/chips/dynapcnn.py b/sinabs/backend/dynapcnn/chips/dynapcnn.py index c56e8a67..73d6b896 100644 --- a/sinabs/backend/dynapcnn/chips/dynapcnn.py +++ b/sinabs/backend/dynapcnn/chips/dynapcnn.py @@ -7,7 +7,7 @@ from samna.dynapcnn.configuration import ( CNNLayerConfig, DynapcnnConfiguration, - DvsLayerConfig, + DVSLayerConfig, ) import sinabs @@ -30,7 +30,7 @@ def get_default_config(cls) -> "DynapcnnConfiguration": def get_dvs_layer_config_dict(cls, layer: DVSLayer): ... @classmethod - def write_dvs_layer_config(cls, layer: DVSLayer, config: DvsLayerConfig): + def write_dvs_layer_config(cls, layer: DVSLayer, config: DVSLayerConfig): for param, value in layer.get_config_dict().items(): setattr(config, param, value) diff --git a/sinabs/backend/dynapcnn/config_builder.py b/sinabs/backend/dynapcnn/config_builder.py index 613ac7f6..7a360718 100644 --- a/sinabs/backend/dynapcnn/config_builder.py +++ b/sinabs/backend/dynapcnn/config_builder.py @@ -72,7 +72,7 @@ def monitor_layers(cls, config, layers: List[int]): """Enable the monitor for a given set of layers in the config object.""" @classmethod - def map_layers_to_cores(cls, layers: Dict[int, DynapcnnLayer]) -> Dict[int]: + def map_layers_to_cores(cls, layers: Dict[int, DynapcnnLayer]) -> Dict[int, int]: """Find a mapping from DynapcnnLayers onto on-chip cores Parameters From 7280e8bb792f5beae1d51f49390921f2eda73209 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Thu, 24 Oct 2024 14:21:37 +0200 Subject: [PATCH 219/379] Fix import related issues in tests --- tests/__init__.py | 0 tests/test_dynapcnnlayer/__init__.py | 0 .../conftest_dynapcnnlayer.py | 8 ++--- .../test_dynapcnnlayer/test_dynapcnnlayer.py | 4 +-- tests/test_dynapcnnnetwork/__init__.py | 0 .../conftest_dynapcnnnetwork.py | 32 +++++++++---------- .../test_dynapcnnnetwork.py | 2 +- 7 files changed, 23 insertions(+), 23 deletions(-) create mode 100644 tests/__init__.py create mode 100644 tests/test_dynapcnnlayer/__init__.py create mode 100644 tests/test_dynapcnnnetwork/__init__.py diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_dynapcnnlayer/__init__.py b/tests/test_dynapcnnlayer/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_dynapcnnlayer/conftest_dynapcnnlayer.py b/tests/test_dynapcnnlayer/conftest_dynapcnnlayer.py index a4234d72..fa756ef0 100644 --- a/tests/test_dynapcnnlayer/conftest_dynapcnnlayer.py +++ b/tests/test_dynapcnnlayer/conftest_dynapcnnlayer.py @@ -1,10 +1,10 @@ # author : Willian Soares Girao # contact : wsoaresgirao@gmail.com -from model_dummy_1 import expected_output_1, dcnnl_map_1 -from model_dummy_2 import expected_output_2, dcnnl_map_2 -from model_dummy_3 import expected_output_3, dcnnl_map_3 -from model_dummy_4 import expected_output_4, dcnnl_map_4 +from .model_dummy_1 import expected_output_1, dcnnl_map_1 +from .model_dummy_2 import expected_output_2, dcnnl_map_2 +from .model_dummy_3 import expected_output_3, dcnnl_map_3 +from .model_dummy_4 import expected_output_4, dcnnl_map_4 # Args: dcnnl_map, discretize, expected_output args_DynapcnnLayer = [ diff --git a/tests/test_dynapcnnlayer/test_dynapcnnlayer.py b/tests/test_dynapcnnlayer/test_dynapcnnlayer.py index 10225950..a1e0de79 100644 --- a/tests/test_dynapcnnlayer/test_dynapcnnlayer.py +++ b/tests/test_dynapcnnlayer/test_dynapcnnlayer.py @@ -2,7 +2,7 @@ # contact : wsoaresgirao@gmail.com import pytest -from conftest_dynapcnnlayer import args_DynapcnnLayer +from .conftest_dynapcnnlayer import args_DynapcnnLayer from sinabs.backend.dynapcnn.dynapcnn_layer_utils import ( construct_dynapcnnlayers_from_mapper, @@ -64,4 +64,4 @@ def test_DynapcnnLayer(dcnnl_map, discretize, expected_output): # Test entry point assert ( entry_points == expected_output["entry_points"] - ), "wrong entry points" \ No newline at end of file + ), "wrong entry points" diff --git a/tests/test_dynapcnnnetwork/__init__.py b/tests/test_dynapcnnnetwork/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_dynapcnnnetwork/conftest_dynapcnnnetwork.py b/tests/test_dynapcnnnetwork/conftest_dynapcnnnetwork.py index 8b7f29b8..95afa771 100644 --- a/tests/test_dynapcnnnetwork/conftest_dynapcnnnetwork.py +++ b/tests/test_dynapcnnnetwork/conftest_dynapcnnnetwork.py @@ -1,22 +1,22 @@ # author : Willian Soares Girao # contact : wsoaresgirao@gmail.com -from model_dummy_1 import batch_size as batch_size_1 -from model_dummy_1 import expected_output as expected_output_1 -from model_dummy_1 import input_shape as input_shape_1 -from model_dummy_1 import snn as snn_1 -from model_dummy_2 import batch_size as batch_size_2 -from model_dummy_2 import expected_output as expected_output_2 -from model_dummy_2 import input_shape as input_shape_2 -from model_dummy_2 import snn as snn_2 -from model_dummy_3 import batch_size as batch_size_3 -from model_dummy_3 import expected_output as expected_output_3 -from model_dummy_3 import input_shape as input_shape_3 -from model_dummy_3 import snn as snn_3 -from model_dummy_4 import batch_size as batch_size_4 -from model_dummy_4 import expected_output as expected_output_4 -from model_dummy_4 import input_shape as input_shape_4 -from model_dummy_4 import snn as snn_4 +from .model_dummy_1 import batch_size as batch_size_1 +from .model_dummy_1 import expected_output as expected_output_1 +from .model_dummy_1 import input_shape as input_shape_1 +from .model_dummy_1 import snn as snn_1 +from .model_dummy_2 import batch_size as batch_size_2 +from .model_dummy_2 import expected_output as expected_output_2 +from .model_dummy_2 import input_shape as input_shape_2 +from .model_dummy_2 import snn as snn_2 +from .model_dummy_3 import batch_size as batch_size_3 +from .model_dummy_3 import expected_output as expected_output_3 +from .model_dummy_3 import input_shape as input_shape_3 +from .model_dummy_3 import snn as snn_3 +from .model_dummy_4 import batch_size as batch_size_4 +from .model_dummy_4 import expected_output as expected_output_4 +from .model_dummy_4 import input_shape as input_shape_4 +from .model_dummy_4 import snn as snn_4 args_DynapcnnNetworkTest = [ (snn_1, input_shape_1, batch_size_1, expected_output_1), diff --git a/tests/test_dynapcnnnetwork/test_dynapcnnnetwork.py b/tests/test_dynapcnnnetwork/test_dynapcnnnetwork.py index 95b190df..a31248dd 100644 --- a/tests/test_dynapcnnnetwork/test_dynapcnnnetwork.py +++ b/tests/test_dynapcnnnetwork/test_dynapcnnnetwork.py @@ -3,7 +3,7 @@ import pytest import torch -from conftest_dynapcnnnetwork import args_DynapcnnNetworkTest +from .conftest_dynapcnnnetwork import args_DynapcnnNetworkTest from sinabs.backend.dynapcnn.dynapcnn_network import DynapcnnNetwork From 864a2ff34090d9272e438f1480748bb8dd882554 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Thu, 24 Oct 2024 15:34:08 +0200 Subject: [PATCH 220/379] GraphExtractor: maintain NIRTorch node naming scheme --- sinabs/backend/dynapcnn/nir_graph_extractor.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index 46ae045f..ffd6814b 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -284,11 +284,15 @@ def _get_named_modules(self, model: nn.Module) -> Dict[int, nn.Module]: ---------- - indx_2_module_map (dict): the mapping between a node (`key` as an `int`) and its module (`value` as a `nn.Module`). """ - return { - self._name_2_indx_map[name]: module - for name, module in model.named_modules() - if name in self._name_2_indx_map - } + indx_2_module_map = dict() + + for name, module in model.named_modules(): + # Make sure names match those provided by nirtorch nodes + name = nirtorch.utils.sanitize_name(name) + if name in self._name_2_indx_map: + indx_2_module_map[self._name_2_indx_map[name]] = module + + return indx_2_module_map def _update_internal_representation(self, remapped_nodes: Dict[int, int]): """Update internal attributes after remapping of nodes From 30b03d0bdcdc9d2a96c4278949a1464c89133d36 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Thu, 24 Oct 2024 15:34:38 +0200 Subject: [PATCH 221/379] Edges handler: Make all edge types except weight-neuron optional --- .../backend/dynapcnn/sinabs_edges_handler.py | 31 ++++++++++--------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index dd3a3a36..d71183a0 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -11,7 +11,7 @@ from torch import Size, nn from .connectivity_specs import VALID_SINABS_EDGE_TYPES -from .exceptions import InvalidEdge, UnmatchedNode, UnmatchedPoolingEdges +from .exceptions import InvalidEdge, InvalidGraphStructure, UnmatchedNode, UnmatchedPoolingEdges from .utils import Edge @@ -68,6 +68,13 @@ def collect_dynapcnn_layer_info( # Map node IDs to dynapcnn layer ID node_2_layer_map = dict() + if "weight-neuron" not in edges_by_type: + raise InvalidGraphStructure( + "Any dynapcnn layer must contain a weight layer (e.g. Conv2d, Linear) " + "that is directly connected to a neuron layer (e.g. IAFSqueeze). " + "None such weight-neuron pair has been found in the provided network." + ) + # Each weight->neuron connection instantiates a new, unique dynapcnn layer while edges_by_type["weight-neuron"]: edge = edges_by_type["weight-neuron"].pop() @@ -81,23 +88,19 @@ def collect_dynapcnn_layer_info( ) # Process all edges connecting two dynapcnn layers that do not include pooling - while edges_by_type["neuron-weight"]: + while edges_by_type.get("neuron-weight", False): edge = edges_by_type["neuron-weight"].pop() set_neuron_layer_destination( dynapcnn_layer_info, edge, node_2_layer_map, nodes_io_shapes ) - # "pooling-pooling" edges are optional. Unlike other types, missing entry would cause exception. - # Therefore add empty set if not existing - if "pooling-pooling" not in edges_by_type: - edges_by_type["pooling-pooling"] = set() - # Add pooling based on neuron->pooling connections - while edges_by_type["neuron-pooling"]: + pooling_pooling_edges = edges_by_type.get("pooling-pooling", set()) + while edges_by_type.get("neuron-pooling", False): edge = edges_by_type["neuron-pooling"].pop() # Search pooling-pooling edges for chains of pooling and add to existing entry pooling_chains, edges_used = trace_paths( - edge[1], edges_by_type["pooling-pooling"] + edge[1], pooling_pooling_edges ) add_pooling_to_entry( dynapcnn_layer_info, @@ -107,14 +110,14 @@ def collect_dynapcnn_layer_info( node_2_layer_map, ) # Remove handled pooling-pooling edges - edges_by_type["pooling-pooling"].difference_update(edges_used) + pooling_pooling_edges.difference_update(edges_used) # After adding pooling make sure all pooling-pooling edges have been handled - if len(edges_by_type["pooling-pooling"]) > 0: - raise UnmatchedPoolingEdges(edges_by_type["pooling-pooling"]) + if len(pooling_pooling_edges) > 0: + raise UnmatchedPoolingEdges(pooling_pooling_edges) # Add all edges connecting pooling to a new dynapcnn layer - while edges_by_type["pooling-weight"]: + while edges_by_type.get("pooling-weight", False): edge = edges_by_type["pooling-weight"].pop() set_pooling_layer_destination( dynapcnn_layer_info, edge, node_2_layer_map, nodes_io_shapes @@ -145,7 +148,7 @@ def get_valid_edge_type( Returns ---------- - edge_type: the edge type specified in 'VALID_SINABS_EDGE_TYPES' ('None' if edge is not valid). + edge_type: the edge type specified in 'valid_edges_map' ('None' if edge is not valid). """ source_type = type(layers[edge[0]]) target_type = type(layers[edge[1]]) From 82b7c1e71405ffd17c141d4a75eedb0015df0fca Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Thu, 24 Oct 2024 15:43:59 +0200 Subject: [PATCH 222/379] Minor code cleanup in edges handler --- .../backend/dynapcnn/sinabs_edges_handler.py | 98 ++++++++++++------- 1 file changed, 62 insertions(+), 36 deletions(-) diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index d71183a0..8eab7b19 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -11,7 +11,12 @@ from torch import Size, nn from .connectivity_specs import VALID_SINABS_EDGE_TYPES -from .exceptions import InvalidEdge, InvalidGraphStructure, UnmatchedNode, UnmatchedPoolingEdges +from .exceptions import ( + InvalidEdge, + InvalidGraphStructure, + UnmatchedNode, + UnmatchedPoolingEdges, +) from .utils import Edge @@ -46,27 +51,9 @@ def collect_dynapcnn_layer_info( # TODO: Handle DVS layer # Sort edges by edge type (type of layers they connect) - edges_by_type: Dict[str, Set[Edge]] = dict() - for edge in edges: - edge_type = get_valid_edge_type( - edge, indx_2_module_map, VALID_SINABS_EDGE_TYPES - ) - - # Validate edge type - if edge_type is None: - raise InvalidEdge( - edge, type(indx_2_module_map[edge[0]]), type(indx_2_module_map[edge[1]]) - ) - - if edge_type in edges_by_type: - edges_by_type[edge_type].add(edge) - else: - edges_by_type[edge_type] = {edge} - - # Dict to collect information for each future dynapcnn layer - dynapcnn_layer_info = dict() - # Map node IDs to dynapcnn layer ID - node_2_layer_map = dict() + edges_by_type: Dict[str, Set[Edge]] = sort_edges_by_type( + edges=edges, indx_2_module_map=indx_2_module_map + ) if "weight-neuron" not in edges_by_type: raise InvalidGraphStructure( @@ -75,6 +62,11 @@ def collect_dynapcnn_layer_info( "None such weight-neuron pair has been found in the provided network." ) + # Dict to collect information for each future dynapcnn layer + dynapcnn_layer_info = dict() + # Map node IDs to dynapcnn layer ID + node_2_layer_map = dict() + # Each weight->neuron connection instantiates a new, unique dynapcnn layer while edges_by_type["weight-neuron"]: edge = edges_by_type["weight-neuron"].pop() @@ -99,9 +91,7 @@ def collect_dynapcnn_layer_info( while edges_by_type.get("neuron-pooling", False): edge = edges_by_type["neuron-pooling"].pop() # Search pooling-pooling edges for chains of pooling and add to existing entry - pooling_chains, edges_used = trace_paths( - edge[1], pooling_pooling_edges - ) + pooling_chains, edges_used = trace_paths(edge[1], pooling_pooling_edges) add_pooling_to_entry( dynapcnn_layer_info, edge, @@ -142,9 +132,9 @@ def get_valid_edge_type( Parameters ---------- - edge (tuple of two int): The edge whose type is to be inferred - layers (Dict): Dict with node IDs as keys and layer instances as values - valid_edge_ids: Dict with valid edge-types (tuples of Types) as keys and edge-type-ID as value + edge (tuple of two int): The edge whose type is to be inferred + layers (Dict): Dict with node IDs as keys and layer instances as values + valid_edge_ids: Dict with valid edge-types (tuples of Types) as keys and edge-type-ID as value Returns ---------- @@ -156,6 +146,42 @@ def get_valid_edge_type( return valid_edge_ids.get((source_type, target_type), None) +def sort_edges_by_type( + edges: Set[Edge], indx_2_module_map: Dict[int, Type] +) -> Dict[str, Set[Edge]]: + """Sort edges by the type of nodes they connect + + Parameters + ---------- + edges (set of tuples): Represent connections between two nodes in computational graph + indx_2_module_map (dict): Maps node IDs of the graph as `key` to their associated module as `value` + + Returns + ------- + Dict with possible keys "weight-neuron", "neuron-weight", "neuron-pooling", "pooling-pooling", + and "pooling-weight". Values are sets of edges corresponding to these types. + """ + edges_by_type: Dict[str, Set[Edge]] = dict() + + for edge in edges: + edge_type = get_valid_edge_type( + edge, indx_2_module_map, VALID_SINABS_EDGE_TYPES + ) + + # Validate edge type + if edge_type is None: + raise InvalidEdge( + edge, type(indx_2_module_map[edge[0]]), type(indx_2_module_map[edge[1]]) + ) + + if edge_type in edges_by_type: + edges_by_type[edge_type].add(edge) + else: + edges_by_type[edge_type] = {edge} + + return edges_by_type + + def init_new_dynapcnnlayer_entry( dynapcnn_layer_info: Dict[int, Dict[int, Dict]], edge: Edge, @@ -269,7 +295,7 @@ def add_pooling_to_entry( def set_exit_destinations(dynapcnn_layer: Dict) -> None: """Set minimal destination entries for layers that don't have any. - + This ensures that the forward methods of the resulting DynapcnnLayer instances return an output, letting these layers act as exit points of the network. @@ -284,13 +310,13 @@ def set_exit_destinations(dynapcnn_layer: Dict) -> None: for layer_info in dynapcnn_layer.values(): if not (destinations := layer_info["destinations"]): # Add `None` destination to empty destination lists - destinations.append( - { - "pooling_ids": [], - "pooling_modules": [], - "destination_layer": None, - } - ) + destinations.append( + { + "pooling_ids": [], + "pooling_modules": [], + "destination_layer": None, + } + ) def set_neuron_layer_destination( From eeefbdc8086554287530db2523f318e214d71738 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Thu, 24 Oct 2024 15:59:45 +0200 Subject: [PATCH 223/379] DynapcnnNetwork: Bring back methods `make_config` and `is_compatible_with` --- sinabs/backend/dynapcnn/dynapcnn_network.py | 109 ++++++++++++++++++-- 1 file changed, 98 insertions(+), 11 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index ef025703..556bba75 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -6,6 +6,7 @@ from warnings import warn import samna +from samna.dynapcnn.configuration import DynapcnnConfiguration import torch import torch.nn as nn from torch import Tensor @@ -287,7 +288,7 @@ def to( if device_name in ChipFactory.supported_devices: # generate config. - config = self._make_config( + config = self.make_config( layer2core_map=layer2core_map, chip_layers_ordering=chip_layers_ordering, device=device, @@ -344,6 +345,93 @@ def to( else: raise Exception("Unknown device description.") + def is_compatible_with(self, device_type: str) -> bool: + """Check if the current model is compatible with a given device. + + Args: + device_type (str): Device type ie speck2b, speck2fmodule + + Returns: + bool: True if compatible + """ + try: + _, is_compatible = self._make_config(device=device_type) + except ValueError as e: + # Catch "No valid mapping found" error + if e.args[0] == ("No valid mapping found"): + return False + else: + raise e + return is_compatible + + def make_config( + self, + layer2core_map: Union[Dict[int, int], str] = "auto", + device: str = "dynapcnndevkit:0", + monitor_layers: Optional[Union[List, str]] = None, + config_modifier: Optional[Callable] = None, + chip_layers_ordering: Optional[Union[Sequence[int], str]] = None, + ) -> DynapcnnConfiguration: + """Prepare and output the `samna` DYNAPCNN configuration for this network. + + Parameters + ---------- + - layer2core_map (dict or "auto"): Defines how cores on chip are + assigned to DynapcnnLayers. If `auto`, an automated procedure + will be used to find a valid ordering. Otherwise a dict needs + to be passed, with DynapcnnLayer indices as keys and assigned + core IDs as values. DynapcnnLayer indices have to match those of + `self.dynapcnn_layers`. + - device: (string): dynapcnndevkit, speck2b or speck2devkit + - monitor_layers: None/List/Str + A list of all layers in the module that you want to monitor. Indexing starts with the first non-dvs layer. + If you want to monitor the dvs layer for eg. + :: + + monitor_layers = ["dvs"] # If you want to monitor the output of the pre-processing layer + monitor_layers = ["dvs", 8] # If you want to monitor preprocessing and layer 8 + monitor_layers = "all" # If you want to monitor all the layers + + If this value is left as None, by default the last layer of the model is monitored. + + - config_modifier (Callable or None): + A user configuration modifier method. + This function can be used to make any custom changes you want to make to the configuration object. + - chip_layers_ordering (None, sequence of integers or "auto", obsolete): + The order in which the dynapcnn layers will be used. If `auto`, + an automated procedure will be used to find a valid ordering. + A list of layers on the device where you want each of the model's DynapcnnLayers to be placed. + Note: This list should be the same length as the number of dynapcnn layers in your model. + Note: This parameter is obsolete and should not be passed anymore. Use + `layer2core_map` instead. + + Returns + ------- + Configuration object + Object defining the configuration for the device + + Raises + ------ + ImportError + If samna is not available. + ValueError + If the generated configuration is not valid for the specified device. + """ + config, is_compatible = self._make_config( + layer2core_map=layer2core_map, + device=device, + monitor_layers=monitor_layers, + config_modifier=config_modifier, + chip_layers_ordering=chip_layers_ordering, + ) + + # Validate config + if is_compatible: + print("Network is valid") + return config + else: + raise ValueError(f"Generated config is not valid for {device}") + ####################################################### Private Methods ####################################################### def _make_config( @@ -353,7 +441,7 @@ def _make_config( monitor_layers: Optional[Union[List, str]] = None, config_modifier: Optional[Callable] = None, chip_layers_ordering: Optional[Union[Sequence[int], str]] = None, - ): + ) -> Tuple[DynapcnnConfiguration, bool]: """Prepare and output the `samna` DYNAPCNN configuration for this network. Parameters @@ -391,13 +479,17 @@ def _make_config( ------- Configuration object Object defining the configuration for the device + Bool + True if the configuration is valid for the given device. + Raises ------ ImportError If samna is not available. ValueError - If the generated configuration is not valid for the specified device. + If no valid mapping between the layers of this object and the cores of + the provided device can be found. """ config_builder = ChipFactory(device).get_config_builder() @@ -484,14 +576,9 @@ def _make_config( if config_modifier is not None: # apply user config modifier. config = config_modifier(config) - - if config_builder.validate_configuration(config): - # validate config. - print("Network is valid: \n") - - return config - else: - raise ValueError(f"Generated config is not valid for {device}") + + # Validate config + return config, config_builder.validate_configuration(config) def _to_device(self, device: torch.device) -> None: """Access each sub-layer within all `DynapcnnLayer` instances and call `.to(device)` on them.""" From 704e13e6ac860b34cb4da5f73d6c80963a014c5a Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Thu, 24 Oct 2024 16:03:43 +0200 Subject: [PATCH 224/379] DynapcnnNetwork: Fix dynapcnn_layers attribute lookup --- sinabs/backend/dynapcnn/dynapcnn_network.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index 556bba75..ac92077a 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -196,7 +196,7 @@ def parameters(self) -> list: """ parameters = [] - for layer in self._dynapcnn_layers.values(): + for layer in self.dynapcnn_layers.values(): if isinstance(layer, DynapcnnLayer): parameters.extend(layer.conv_layer.parameters()) @@ -209,14 +209,14 @@ def init_weights(self, init_fn: nn.init = nn.init.xavier_normal_) -> None: ---------- - init_fn (torch.nn.init): the weight initialization method to be used. """ - for layer in self._dynapcnn_layers.values(): + for layer in self.dynapcnn_layers.values(): if isinstance(layer, DynapcnnLayer): init_fn(layer.conv_layer.weight.data) def detach_neuron_states(self) -> None: """Detach the neuron states and activations from current computation graph (necessary).""" - for module in self._dynapcnn_layers.values(): + for module in self.dynapcnn_layers.values(): if isinstance(module, DynapcnnLayer): if isinstance(module.spk_layer, sl.StatefulLayer): for name, buffer in module.spk_layer.named_buffers(): @@ -494,7 +494,7 @@ def _make_config( config_builder = ChipFactory(device).get_config_builder() # TODO not handling DVSLayer yet. - has_dvs_layer = isinstance(self._dynapcnn_layers[0], DVSLayer) + has_dvs_layer = isinstance(self.dynapcnn_layers[0], DVSLayer) if chip_layers_ordering is not None: if layer2core_map is not None: @@ -582,7 +582,7 @@ def _make_config( def _to_device(self, device: torch.device) -> None: """Access each sub-layer within all `DynapcnnLayer` instances and call `.to(device)` on them.""" - for layer in self._dynapcnn_layers.values(): + for layer in self.dynapcnn_layers.values(): if isinstance(layer, sinabs.backend.dynapcnn.dynapcnn_layer.DynapcnnLayer): layer.to(device) @@ -591,7 +591,7 @@ def _to_device(self, device: torch.device) -> None: def __str__(self): pretty_print = "" - for idx, layer_data in self._dynapcnn_layers.items(): + for idx, layer_data in self.dynapcnn_layers.items(): pretty_print += f"----------------------- [ DynapcnnLayer {idx} ] -----------------------\n" pretty_print += f"{layer_data}\n\n" From 2f87ef23020eed292d1ced3ebd802a75173469d8 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Thu, 24 Oct 2024 16:07:24 +0200 Subject: [PATCH 225/379] DynapcnnNetwork: Add method `has_dvs_layer` --- sinabs/backend/dynapcnn/dynapcnn_network.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index ac92077a..e253b1ca 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -432,6 +432,18 @@ def make_config( else: raise ValueError(f"Generated config is not valid for {device}") + def has_dvs_layer(self) -> bool: + """ Return True if there is a DVSLayer in the network + + Returns + ------- + bool: True if DVSLayer is found within the network. + """ + for layer in self.dynapcnn_layers.values(): + if isinstance(layer, DVSLayer): + return True + return False + ####################################################### Private Methods ####################################################### def _make_config( @@ -494,7 +506,7 @@ def _make_config( config_builder = ChipFactory(device).get_config_builder() # TODO not handling DVSLayer yet. - has_dvs_layer = isinstance(self.dynapcnn_layers[0], DVSLayer) + has_dvs_layer = self.has_dvs_layer() if chip_layers_ordering is not None: if layer2core_map is not None: From de5ba3373511470446be94a9f40409abd5387228 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Thu, 24 Oct 2024 16:14:21 +0200 Subject: [PATCH 226/379] DynapcnnNetworkModule: Try saving `dynapcnn_layers` with integer indices instead of string --- sinabs/backend/dynapcnn/dynapcnnnetwork_module.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py index 8128d525..6e98aa59 100644 --- a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py +++ b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py @@ -50,7 +50,7 @@ def __init__( # Unfortunately ModuleDict does not allow for integer keys # TODO: Consider using list instead of dict - self.dynapcnn_layers = nn.ModuleDict( + self._dynapcnn_layers = nn.ModuleDict( {str(idx): lyr for idx, lyr in dynapcnn_layers.items()} ) self._destination_map = destination_map @@ -63,6 +63,11 @@ def __init__( def destination_map(self): return self._destination_map + @property + def dynapcnn_layers(self): + # Convert string-indices to integers + return {int(idx): lyr for idx, lyr in self._dynapcnn_layers.items()} + @property def entry_points(self): return self._entry_points @@ -241,7 +246,7 @@ def forward( current_input = layers_outputs[idx_src][idx_curr] # Get current layer instance and destinations - layer = self.dynapcnn_layers[str(idx_curr)] + layer = self.dynapcnn_layers[idx_curr] destinations = self._destination_map[idx_curr] # Forward pass through layer @@ -312,8 +317,8 @@ def remap(key): return mapping[key] # Remap all internal objects - self.dynapcnn_layers = nn.ModuleDict( - {str(remap(int(idx))): lyr for idx, lyr in self.dynapcnn_layers.items()} + self._dynapcnn_layers = nn.ModuleDict( + {str(remap(int(idx))): lyr for idx, lyr in self._dynapcnn_layers.items()} ) self._entry_points = {remap(idx) for idx in self._entry_points} self._destination_map = { From d970f1821e70be2ddf4ea4c8c1168408227474c7 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Thu, 24 Oct 2024 16:22:19 +0200 Subject: [PATCH 227/379] Fix bugs in mapping --- sinabs/backend/dynapcnn/mapping.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sinabs/backend/dynapcnn/mapping.py b/sinabs/backend/dynapcnn/mapping.py index afd7dbfc..cd877692 100644 --- a/sinabs/backend/dynapcnn/mapping.py +++ b/sinabs/backend/dynapcnn/mapping.py @@ -68,7 +68,8 @@ def get_valid_mapping( # Skip DVSLayers if isinstance(this_layer, DynapcnnLayer): chip_layers = find_chip_layers(this_layer, constraints) - layer_mapping[layer_index] = chip_layers + layer_mapping.append(chip_layers) + layer_indices.append(layer_index) # Make sure only DynapcnnLayers and DVSLayers are passed elif not isinstance(this_layer, DVSLayer): raise ValueError(f"Found unexpected layer type: `{type(this_layer)}") @@ -77,7 +78,7 @@ def get_valid_mapping( # use Edmonds' Algorithm to find suitable cores for each DynapcnnLayer. new_graph = edmonds(graph, 0, len(graph) - 1) - netmap = recover_mapping(new_graph, layer_mapping) + netmap = recover_mapping(new_graph, len(layer_mapping)) # Convert `netmap` to dict mapping from layer index to core ID return { From 25fb6598e90e26e3917f88800c286b89f93246f4 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Thu, 24 Oct 2024 16:48:00 +0200 Subject: [PATCH 228/379] Move to utils. New function: . Fix handling of pooling in deployment --- sinabs/backend/dynapcnn/chips/dynapcnn.py | 35 ++++++++++----- sinabs/backend/dynapcnn/dvs_layer.py | 17 +------ .../backend/dynapcnn/dynapcnn_layer_utils.py | 2 +- sinabs/backend/dynapcnn/dynapcnn_network.py | 2 +- sinabs/backend/dynapcnn/utils.py | 3 +- sinabs/utils.py | 44 ++++++++++++++++++- 6 files changed, 71 insertions(+), 32 deletions(-) diff --git a/sinabs/backend/dynapcnn/chips/dynapcnn.py b/sinabs/backend/dynapcnn/chips/dynapcnn.py index 73d6b896..81231164 100644 --- a/sinabs/backend/dynapcnn/chips/dynapcnn.py +++ b/sinabs/backend/dynapcnn/chips/dynapcnn.py @@ -12,7 +12,7 @@ import sinabs from sinabs.backend.dynapcnn.config_builder import ConfigBuilder -from sinabs.backend.dynapcnn.dvs_layer import DVSLayer, expand_to_pair +from sinabs.backend.dynapcnn.dvs_layer import DVSLayer from sinabs.backend.dynapcnn.dynapcnn_layer import DynapcnnLayer from sinabs.backend.dynapcnn.mapping import LayerConstraints @@ -178,12 +178,24 @@ def get_dynapcnn_layer_config_dict( destinations = [] pooling_sizes = layer.pool for dest_layer_id, pool in zip(destination_indices, pooling_sizes): - dest_data = { - "layer": layer2core_map[dest_layer_id], - "enable": True, - "pooling": expand_to_pair(pool), - } - destinations.append(dest_data) + # Ignore exit point destinations + if dest_layer_id >= 0: + + try: + # Use scalar value for pooling + pool = sinabs.utils.collapse_pair(pool) + except ValueError: + raise ValueError( + f"Can only do pooling with quadratic kernels. Received {pool}" + ) + + dest_data = { + "layer": layer2core_map[dest_layer_id], + "enable": True, + "pooling": pool, + } + destinations.append(dest_data) + config_dict["destinations"] = destinations # Set kill bits @@ -222,8 +234,7 @@ def write_dynapcnn_layer_config( ) # update configuration of the DYNAPCNN layer. - chip_layer.dimensions = config_dict["dimensions"] - config_dict.pop("dimensions") + chip_layer.dimensions = config_dict.pop("dimensions") # set the destinations configuration. for dest_idx, destination in enumerate(config_dict.pop("destinations")): @@ -274,10 +285,10 @@ def build_config( chip_layer = config.cnn_layers[layer2core_map[layer_index]] # write core configuration. cls.write_dynapcnn_layer_config( - ith_dcnnl, - chip_layer, - destination_indices=destination_map[layer_index], + layer=ith_dcnnl, layer2core_map=layer2core_map, + chip_layer=chip_layer, + destination_indices=destination_map[layer_index], ) else: diff --git a/sinabs/backend/dynapcnn/dvs_layer.py b/sinabs/backend/dynapcnn/dvs_layer.py index 0104dd8e..8111edae 100644 --- a/sinabs/backend/dynapcnn/dvs_layer.py +++ b/sinabs/backend/dynapcnn/dvs_layer.py @@ -3,27 +3,12 @@ import torch.nn as nn from sinabs.layers import SumPool2d +from sinabs.utils import expand_to_pair from .crop2d import Crop2d from .flipdims import FlipDims -def expand_to_pair(value) -> (int, int): - """Expand a given value to a pair (tuple) if an int is passed. - - Parameters - ---------- - value: - int - - Returns - ------- - pair: - (int, int) - """ - return (value, value) if isinstance(value, int) else value - - class DVSLayer(nn.Module): """DVSLayer representing the DVS pixel array on chip and/or the pre-processing. The order of processing is as follows MergePolarity -> Pool -> Cut -> Flip. diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py b/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py index 3db72357..e44c4b6e 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py @@ -4,9 +4,9 @@ from torch import nn from sinabs import layers as sl +from sinabs.utils import expand_to_pair from .dynapcnn_layer import DynapcnnLayer -from .utils import expand_to_pair def construct_dynapcnnlayers_from_mapper( diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index e253b1ca..f200c62b 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -570,7 +570,7 @@ def _make_config( # Collect cores (chip layers) that are to be monitored monitor_chip_layers = [] for lyr_idx in monitor_layers: - if lyr_idx.lower() == "dvs": + if str(lyr_idx).lower() == "dvs": monitor_chip_layers.append("dvs") else: # Warn when recording layers with pooling diff --git a/sinabs/backend/dynapcnn/utils.py b/sinabs/backend/dynapcnn/utils.py index 74f8016a..caf3d872 100644 --- a/sinabs/backend/dynapcnn/utils.py +++ b/sinabs/backend/dynapcnn/utils.py @@ -13,9 +13,10 @@ import torch.nn as nn import sinabs.layers as sl +from sinabs.utils import expand_to_pair from .crop2d import Crop2d -from .dvs_layer import DVSLayer, expand_to_pair +from .dvs_layer import DVSLayer from .flipdims import FlipDims if TYPE_CHECKING: diff --git a/sinabs/utils.py b/sinabs/utils.py index 89460f46..c450a38b 100644 --- a/sinabs/utils.py +++ b/sinabs/utils.py @@ -1,4 +1,4 @@ -from typing import List +from typing import Iterable, List, Tuple, TypeVar, Union import numpy as np import torch @@ -268,3 +268,45 @@ def get_smallest_compatible_time_dimension(model: nn.Module) -> int: num_timesteps = abs(get_num_timesteps(model)) # Use `abs` to turn `-1` to `1` return abs(batch_size * num_timesteps) + + +def expand_to_pair(value) -> Tuple[int, int]: + """Expand a given value to a pair (tuple) if an int is passed. + + Parameters + ---------- + value: + int + + Returns + ------- + pair: + (int, int) + """ + return (value, value) if isinstance(value, int) else value + + +T = TypeVar("T") +def collapse_pair(pair: Union[Iterable[T], T]) -> T: + """ Collapse an iterable of equal elements by returning only the first + + Parameters + ---------- + pair: Iterable. All elements should be the same. + + Returns + ------- + First item of `pair`. If `pair` is not iterable it will return `pair` itself. + + Raises + ------ + ValueError if not all elements in `pair` are equal. + """ + if isinstance(pair, Iterable): + items = [x for x in pair] + if any(x != items[0] for x in items): + raise ValueError("All elements of `pair` must be the same") + return items[0] + else: + return pair + From a7bbc36be06e8ec6f92875a5d85b959b08aa116b Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Thu, 24 Oct 2024 17:16:06 +0200 Subject: [PATCH 229/379] Remove redundant warning for monitoring pooled layers --- sinabs/backend/dynapcnn/dynapcnn_network.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index f200c62b..a98bdad4 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -358,7 +358,9 @@ def is_compatible_with(self, device_type: str) -> bool: _, is_compatible = self._make_config(device=device_type) except ValueError as e: # Catch "No valid mapping found" error - if e.args[0] == ("No valid mapping found"): + if e.args[0] == ( + "One or more of the DynapcnnLayers could not be mapped to any core." + ): return False else: raise e @@ -573,13 +575,6 @@ def _make_config( if str(lyr_idx).lower() == "dvs": monitor_chip_layers.append("dvs") else: - # Warn when recording layers with pooling - if any(p != (1, 1) for p in self.dynapcnn_layers[lyr_idx].pool): - warn( - f"Monitored layer {lyr_idx} has at least one destination " - "with pooling. Note that monitored events will be recorded " - "before pooling" - ) monitor_chip_layers.append(layer2core_map[lyr_idx]) # enable monitors on the specified layers. From c9e6caab1b0277f0349ef67369afd85077cc91e4 Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 25 Oct 2024 12:45:01 +0200 Subject: [PATCH 230/379] minor edit added return typehint for finalize_dcnnl_map(). --- sinabs/backend/dynapcnn/dynapcnn_layer_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py b/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py index e44c4b6e..1d7b5086 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py @@ -36,7 +36,7 @@ def construct_dynapcnnlayers_from_mapper( return dynapcnn_layers, destination_map, entry_points -def finalize_dcnnl_map(dcnnl_map: Dict, rescale_fn: Optional[Callable] = None): +def finalize_dcnnl_map(dcnnl_map: Dict, rescale_fn: Optional[Callable] = None) -> None: """Finalize dcnnl map by consolidating information Update dcnnl_map in-place From 90f9575e04a8de129aeac4605c7b25e4d8adc868 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Fri, 25 Oct 2024 17:01:48 +0200 Subject: [PATCH 231/379] Integrate tests for sequential models in test_dynapcnnnetwork --- .../test_compatible_layer_build.py | 178 +++--------------- .../conftest_dynapcnnnetwork.py | 4 + tests/test_dynapcnnnetwork/model_dummy_seq.py | 74 ++++++++ 3 files changed, 107 insertions(+), 149 deletions(-) create mode 100644 tests/test_dynapcnnnetwork/model_dummy_seq.py diff --git a/tests/test_dynapcnn/test_compatible_layer_build.py b/tests/test_dynapcnn/test_compatible_layer_build.py index 54215a52..b2630ee0 100644 --- a/tests/test_dynapcnn/test_compatible_layer_build.py +++ b/tests/test_dynapcnn/test_compatible_layer_build.py @@ -3,135 +3,42 @@ import sinabs.layers as sl - -def test_construct_pooling_from_1_layer(): - layers = [sl.SumPool2d(2)] - - from sinabs.backend.dynapcnn.utils import construct_next_pooling_layer - - pool_lyr, layer_idx_next, rescale_factor = construct_next_pooling_layer(layers, 0) - - assert pool_lyr.kernel_size == (2, 2) - assert layer_idx_next == 1 - assert rescale_factor == 1 +@pytest.mark.parametrize( + ("pooling", "layer_type", "expected_pooling", "expected_scaling"), + [ + (2, sl.SumPool2d, [2, 2], 1), + ((2, 2), sl.SumPool2d, [2, 2], 1), + (3, sl.SumPool2d, [3, 3], 1), + ((4, 4), sl.SumPool2d, [4, 4], 1), + (2, nn.AvgPool2d, [2, 2], 1./4), + ((2, 2), sl.nn.AvgPool2d, [2, 2], 1./4), + (3, sl.nn.AvgPool2d, [3, 3], 1./9), + ((4, 4), sl.nn.AvgPool2d, [4, 4], 1./16), + ] +) +def test_construct_pooling_from_1_layer(pooling, layer_type, expected_pooling, expected_scaling): + layers = [layer_type(pooling)] + + from sinabs.backend.dynapcnn.dynapcnn_layer_utils import consolidate_dest_pooling + + cumulative_pooling, scaling = consolidate_dest_pooling(layers) + + assert cumulative_pooling == expected_pooling + assert scaling == expected_scaling def test_construct_pooling_from_2_layers(): - layers = [sl.SumPool2d(2), nn.AvgPool2d(3), sl.IAF()] - - from sinabs.backend.dynapcnn.utils import construct_next_pooling_layer - - pool_lyr, layer_idx_next, rescale_factor = construct_next_pooling_layer(layers, 0) - - assert pool_lyr.kernel_size == (6, 6) - assert layer_idx_next == 2 - assert rescale_factor == 9 - - -def test_non_square_pooling_kernel(): - layers = [ - nn.Conv2d(2, 8, kernel_size=3, stride=1, bias=False), - sl.IAF(), - sl.SumPool2d((2, 3)), - ] - - from sinabs.backend.dynapcnn.utils import construct_next_dynapcnn_layer - - with pytest.raises(ValueError): - _ = construct_next_dynapcnn_layer( - layers, 0, in_shape=(2, 28, 28), discretize=True, rescale_factor=1 - ) - - -def test_construct_dynapcnn_layer_from_3_layers(): - layers = [ - nn.Conv2d(2, 8, kernel_size=3, stride=1, bias=False), - sl.IAF(), - sl.SumPool2d(2), - ] - - from sinabs.backend.dynapcnn.utils import construct_next_dynapcnn_layer - - dynapcnn_lyr, layer_idx_next, rescale_factor = construct_next_dynapcnn_layer( - layers, 0, in_shape=(2, 28, 28), discretize=True, rescale_factor=1 - ) - - print(dynapcnn_lyr) - assert layer_idx_next == 3 - assert rescale_factor == 1 - - -def test_construct_dynapcnn_layer_no_pool_layers(): - layers = [ - nn.Conv2d(2, 8, kernel_size=3, stride=1, bias=False), - sl.IAF(), - nn.Conv2d(8, 2, kernel_size=3, stride=1, bias=False), - sl.IAF(), - ] - - from sinabs.backend.dynapcnn.utils import construct_next_dynapcnn_layer - - dynapcnn_lyr, layer_idx_next, rescale_factor = construct_next_dynapcnn_layer( - layers, 0, in_shape=(2, 28, 28), discretize=True, rescale_factor=1 - ) - - print(dynapcnn_lyr) - assert layer_idx_next == 2 - assert rescale_factor == 1 - - -def test_construct_dynapcnn_layer_from_8_layers(): - layers = [ - nn.Conv2d(2, 8, kernel_size=3, stride=1, bias=False), - sl.IAF(), - sl.SumPool2d(2), - nn.AvgPool2d(2), - nn.Conv2d(2, 8, kernel_size=3, stride=1, bias=False), - sl.IAF(), - nn.Conv2d(2, 8, kernel_size=3, stride=1, bias=False), - sl.IAF(), - ] + layers = [sl.SumPool2d(2), nn.AvgPool2d(3)] - from sinabs.backend.dynapcnn.utils import construct_next_dynapcnn_layer + from sinabs.backend.dynapcnn.dynapcnn_layer_utils import consolidate_dest_pooling - dynapcnn_lyr, layer_idx_next, rescale_factor = construct_next_dynapcnn_layer( - layers, 0, in_shape=(2, 28, 28), discretize=True, rescale_factor=1 - ) + cumulative_pooling, scaling = consolidate_dest_pooling(layers) - print(dynapcnn_lyr) - assert dynapcnn_lyr.pool_layer.kernel_size == (4, 4) - assert layer_idx_next == 4 - assert rescale_factor == 4 - - -def test_build_from_list_dynapcnn_layers_only(): - in_shape = (2, 28, 28) - layers = [ - nn.Conv2d(2, 8, kernel_size=3, stride=1, bias=False), - sl.IAF(), - sl.SumPool2d(2), - nn.AvgPool2d(2), - nn.Conv2d(8, 16, kernel_size=3, stride=1, bias=False), - sl.IAF(), - nn.Dropout2d(), - nn.Conv2d(16, 2, kernel_size=3, stride=1, bias=False), - sl.IAF(), - nn.Flatten(), - nn.Linear(8, 5), - sl.IAF(), - ] - - from sinabs.backend.dynapcnn.utils import build_from_list - - chip_model = build_from_list(layers, in_shape=in_shape, discretize=True) - - assert len(chip_model) == 4 - assert chip_model[0].get_output_shape() == (8, 6, 6) - assert chip_model[1].get_output_shape() == (16, 4, 4) - assert chip_model[2].get_output_shape() == (2, 2, 2) - assert chip_model[3].get_output_shape() == (5, 1, 1) + assert cumulative_pooling == [6, 6] + assert scaling == 1./9 +# TODO: Move these fail cases to another test. Rename this file or move other tests as well. def test_missing_spiking_layer(): in_shape = (2, 28, 28) layers = [ @@ -167,31 +74,4 @@ def test_incorrect_model_start(): with pytest.raises(UnexpectedLayer): construct_next_dynapcnn_layer( layers, 0, in_shape=in_shape, discretize=True, rescale_factor=1 - ) - - -def test_conversion_to_layer_list(): - from sinabs.backend.dynapcnn.utils import DEFAULT_IGNORED_LAYER_TYPES as DEF_IGNORE - from sinabs.backend.dynapcnn.utils import convert_model_to_layer_list - - model = nn.Sequential( - nn.Conv2d(2, 8, 3), - sl.IAF(), - nn.Conv2d(8, 16, 3), - nn.Identity(), - nn.AvgPool2d(2), - nn.Dropout(0.5), - nn.Conv2d(16, 16, 3), - sl.IAF(), - nn.Flatten(), - nn.Linear(64, 4), - sl.IAF(), - ) - - layer_list = convert_model_to_layer_list(model, ignore=DEF_IGNORE) - - # Should contain all layers except identity, dropbout, and flatten - assert len(layer_list) == len(model) - 3 - model_indices = (0, 1, 2, 4, 6, 7, 9, 10) - for layer, idx_model in zip(layer_list, model_indices): - assert layer is model[idx_model] + ) \ No newline at end of file diff --git a/tests/test_dynapcnnnetwork/conftest_dynapcnnnetwork.py b/tests/test_dynapcnnnetwork/conftest_dynapcnnnetwork.py index 95afa771..818b1d90 100644 --- a/tests/test_dynapcnnnetwork/conftest_dynapcnnnetwork.py +++ b/tests/test_dynapcnnnetwork/conftest_dynapcnnnetwork.py @@ -17,10 +17,14 @@ from .model_dummy_4 import expected_output as expected_output_4 from .model_dummy_4 import input_shape as input_shape_4 from .model_dummy_4 import snn as snn_4 +from .model_dummy_4 import snn as snn_4 +from .model_dummy_seq import input_shape_seq, seq_1, seq_2, expected_seq_1, expected_seq_2 args_DynapcnnNetworkTest = [ (snn_1, input_shape_1, batch_size_1, expected_output_1), (snn_2, input_shape_2, batch_size_2, expected_output_2), (snn_3, input_shape_3, batch_size_3, expected_output_3), (snn_4, input_shape_4, batch_size_4, expected_output_4), + (seq_1, input_shape_seq, 1, expected_seq_1), + (seq_2, input_shape_seq, 1, expected_seq_2), ] diff --git a/tests/test_dynapcnnnetwork/model_dummy_seq.py b/tests/test_dynapcnnnetwork/model_dummy_seq.py new file mode 100644 index 00000000..f210181c --- /dev/null +++ b/tests/test_dynapcnnnetwork/model_dummy_seq.py @@ -0,0 +1,74 @@ +# implementing sequential models + +import torch +import torch.nn as nn + +from sinabs.layers import IAFSqueeze, SumPool2d + + +input_shape_seq = (2, 30, 30) + +seq_1 = nn.Sequential( + nn.Conv2d(2, 8, kernel_size=3, stride=1, bias=False), + IAFSqueeze(batch_size=1), + nn.Conv2d(8, 2, kernel_size=3, stride=1, bias=False), + IAFSqueeze(batch_size=1), +) + +seq_2 = nn.Sequential( + nn.Conv2d(2, 2, kernel_size=3, stride=1, bias=False), + IAFSqueeze(batch_size=1), + SumPool2d(2), + nn.AvgPool2d(2), + nn.Dropout(0.5), + nn.Conv2d(2, 8, kernel_size=3, stride=1, bias=False), + IAFSqueeze(batch_size=1), + nn.Conv2d(8, 2, kernel_size=3, stride=1, bias=False), + IAFSqueeze(batch_size=1), + nn.Flatten(), + nn.Linear(3*3*2, 5), + nn.Identity(), + IAFSqueeze(batch_size=1), +) + +expected_seq_1 = { + "dcnnl_edges": { + ("input", 0), + (0, 1), + }, + "node_source_map": { + 0: {"input"}, + 1: {0}, + }, + "destination_map": { + 0: {1}, + 1: {-1}, + }, + "sorted_nodes": [0, 1], + "output_shape": torch.Size([1, 2, 26, 26]), + "entry_points": {0}, +} + +expected_seq_2= { + "dcnnl_edges": { + (0, 1), + (1, 2), + (2, 3), + ("input", 0), + }, + "node_source_map": { + 0: {"input"}, + 1: {0}, + 2: {1}, + 3: {2}, + }, + "destination_map": { + 0: {1}, + 1: {2}, + 2: {3}, + 3: {-1}, + }, + "sorted_nodes": [0, 1, 2, 3], + "output_shape": torch.Size([1, 5, 1, 1]), + "entry_points": {0}, +} From 8495b4e45085d492eccae4242b8a948379801fbe Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Fri, 25 Oct 2024 17:06:40 +0200 Subject: [PATCH 232/379] Fix test_auto_mapping --- tests/test_dynapcnn/test_auto_mapping.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_dynapcnn/test_auto_mapping.py b/tests/test_dynapcnn/test_auto_mapping.py index 37de88d9..cff40a75 100644 --- a/tests/test_dynapcnn/test_auto_mapping.py +++ b/tests/test_dynapcnn/test_auto_mapping.py @@ -48,4 +48,4 @@ def test_auto_mapping_should_not_work(): graph = make_flow_graph(layer_mapping) new_graph = edmonds(graph, 0, len(graph) - 1) with pytest.raises(ValueError): - mapping = recover_mapping(new_graph, layer_mapping) + mapping = recover_mapping(new_graph, len(layer_mapping)) From ff68bc1e9d8e8a513f3282b25b66a8dd3e99e2d9 Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 25 Oct 2024 19:53:15 +0200 Subject: [PATCH 233/379] (WIP - DVS input) checking if a node needs to be created for a DVSLayer and making space for it when calling '_get_name_2_indx_map()'. --- .../backend/dynapcnn/nir_graph_extractor.py | 42 ++++++++++++++++--- 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index ffd6814b..74c6777c 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -8,6 +8,7 @@ import torch.nn as nn import sinabs +from dvs_layer import DVSLayer from .connectivity_specs import ( LAYER_TYPES_WITH_MULTIPLE_INPUTS, @@ -15,13 +16,13 @@ ) from .dynapcnn_layer_utils import construct_dynapcnnlayers_from_mapper from .dynapcnnnetwork_module import DynapcnnNetworkModule -from .exceptions import InvalidGraphStructure +from .exceptions import InvalidGraphStructure, InvalidModelWithDVSSetup from .sinabs_edges_handler import collect_dynapcnn_layer_info from .utils import Edge, topological_sorting class GraphExtractor: - def __init__(self, spiking_model: nn.Module, dummy_input: torch.tensor): + def __init__(self, spiking_model: nn.Module, dummy_input: torch.tensor, dvs_input: bool): """Class implementing the extraction of the computational graph from `spiking_model`, where each node represents a layer in the model and the list of edges represents how the data flow between the layers. @@ -47,6 +48,8 @@ def __init__(self, spiking_model: nn.Module, dummy_input: torch.tensor): Map from layer ID to the corresponding nn.Module instance. - nodes_io_shapes (dict): Map from node ID to dict containing node's in- and output shapes + - dvs_input (bool): + Whether or not the model should start with a `DVSLayer`. """ # extract computational graph. @@ -54,8 +57,10 @@ def __init__(self, spiking_model: nn.Module, dummy_input: torch.tensor): spiking_model, dummy_input, model_name=None ).ignore_tensors() + # This var. will be set to `True` if `dvs_input == True` and `spiking_model` does not start with DVS layer. + need_dvs_node = self._need_dvs_node(spiking_model, dvs_input) # Map node names to indices - self._name_2_indx_map = self._get_name_2_indx_map(nir_graph) + self._name_2_indx_map = self._get_name_2_indx_map(nir_graph, need_dvs_node) # Extract edges list from graph self._edges = self._get_edges_from_nir(nir_graph, self._name_2_indx_map) # Determine entry points to graph @@ -219,12 +224,36 @@ def verify_graph_integrity(self): ####################################################### Pivate Methods ####################################################### - def _get_name_2_indx_map(self, nir_graph: nirtorch.graph.Graph) -> Dict[str, int]: - """Assign unique index to each node and return mapper from name to index. + def _need_dvs_node(self, model: nn.Module, dvs_input: bool) -> bool: + """ Returns whether or not a node will need to be added to represent a `DVSLayer` instance. A new node will have + to be added if `model` does not start with a `DVSLayer` instance and `dvs_input == True`. + + Parameters + ---------- + - model (nn.Module): the `spiking_model` used as argument to the class instance. + - dvs_input (bool): wether or not dynapcnn receive input from its DVS camera. + Returns + ------- + - True if the first layer is a DVSLayer, False otherwise. + """ + + # Get the first module only and check its type + first_name, first_module = next(model.named_modules()) + + # Check consistency of user provided arguments for use of the DVS + if isinstance(first_module, DVSLayer) and not dvs_input: + raise InvalidModelWithDVSSetup() + + return not isinstance(first_module, DVSLayer) and dvs_input + + def _get_name_2_indx_map(self, nir_graph: nirtorch.graph.Graph, need_dvs_node: bool) -> Dict[str, int]: + """Assign unique index to each node and return mapper from name to index. If `need_dvs_node == Ture` we want to + leave index `0` free to be assigned to the `DVSLayer` node that will have to be created. Parameters ---------- - nir_graph (nirtorch.graph.Graph): a NIR graph representation of `spiking_model`. + - need_dvs_node (bool): True of `dvs_input == True` and `spiking_model` doesn't start with a `DVSLayer`. Returns ---------- @@ -232,7 +261,8 @@ def _get_name_2_indx_map(self, nir_graph: nirtorch.graph.Graph) -> Dict[str, int `spiking_model` and `value is an integer representing the layer in a standard format. """ return { - node.name: node_idx for node_idx, node in enumerate(nir_graph.node_list) + node.name: (node_idx + 1 if need_dvs_node else node_idx) + for node_idx, node in enumerate(nir_graph.node_list) } def _get_edges_from_nir( From 6a0bdcab7409cc14da635d0b02b817b89f8bc602 Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 25 Oct 2024 19:54:17 +0200 Subject: [PATCH 234/379] (WIP - DVS input) custom error in case model has DVSLayer but dvs_input == False. --- sinabs/backend/dynapcnn/exceptions.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sinabs/backend/dynapcnn/exceptions.py b/sinabs/backend/dynapcnn/exceptions.py index 63de8585..52795bfe 100644 --- a/sinabs/backend/dynapcnn/exceptions.py +++ b/sinabs/backend/dynapcnn/exceptions.py @@ -67,6 +67,10 @@ class InvalidGraphStructure(Exception): pass +class InvalidModelWithDVSSetup(Exception): + def __init__(self): + super().__init__(f"The network provided starts with a DVSLayer but 'dvs_input' is set to False.") + # Edge exceptions. From e9bf8a1d9701c8196ea8bae35b2ec9811a78c229 Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 25 Oct 2024 20:31:33 +0200 Subject: [PATCH 235/379] (WIP - DVS input) '_get_named_modules()' adds a sinabs.DVSLayer entry in case a node for the DVS cam. needs to be created. --- .../backend/dynapcnn/nir_graph_extractor.py | 35 +++++++++++++++---- 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index 74c6777c..178f5592 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -59,14 +59,20 @@ def __init__(self, spiking_model: nn.Module, dummy_input: torch.tensor, dvs_inpu # This var. will be set to `True` if `dvs_input == True` and `spiking_model` does not start with DVS layer. need_dvs_node = self._need_dvs_node(spiking_model, dvs_input) + dvs_input_shape = None + if need_dvs_node: + # We need to provide `(height, width)` to the DVSLayer instance that will be the module of the node 'dvs'. + _, _, height, width = dummy_input.shape + dvs_input_shape = (height, width) + # Map node names to indices self._name_2_indx_map = self._get_name_2_indx_map(nir_graph, need_dvs_node) # Extract edges list from graph - self._edges = self._get_edges_from_nir(nir_graph, self._name_2_indx_map) + self._edges = self._get_edges_from_nir(nir_graph, self._name_2_indx_map) # @TODO edges need to be modified in place if DVS layer is needed. # Determine entry points to graph - self._entry_nodes = self._get_entry_nodes(self._edges) + self._entry_nodes = self._get_entry_nodes(self._edges) # @TODO maybe functionality has to change here a when DVS layer is needed. # Store the associated `nn.Module` (layer) of each node. - self._indx_2_module_map = self._get_named_modules(spiking_model) + self._indx_2_module_map = self._get_named_modules(spiking_model, need_dvs_node, dvs_input_shape) # Verify that graph is compatible self.verify_graph_integrity() @@ -239,7 +245,7 @@ def _need_dvs_node(self, model: nn.Module, dvs_input: bool) -> bool: # Get the first module only and check its type first_name, first_module = next(model.named_modules()) - + # Check consistency of user provided arguments for use of the DVS if isinstance(first_module, DVSLayer) and not dvs_input: raise InvalidModelWithDVSSetup() @@ -260,11 +266,19 @@ def _get_name_2_indx_map(self, nir_graph: nirtorch.graph.Graph, need_dvs_node: b - name_2_indx_map (dict): `key` is the original variable name for a layer in `spiking_model` and `value is an integer representing the layer in a standard format. """ - return { + + # Start name indexing from 1 if a DVS node needs to be added + name_2_indx_map = { node.name: (node_idx + 1 if need_dvs_node else node_idx) for node_idx, node in enumerate(nir_graph.node_list) } + if need_dvs_node: + # Adds entry for the DVS node that needs to be created - default node name is 'dvs' + name_2_indx_map['dvs'] = 0 + + return name_2_indx_map + def _get_edges_from_nir( self, nir_graph: nirtorch.graph.Graph, name_2_indx_map: Dict[str, int] ) -> Set[Edge]: @@ -303,17 +317,22 @@ def _get_entry_nodes(self, edges: Set[Edge]) -> Set[Edge]: all_sources, all_targets = zip(*edges) return set(all_sources) - set(all_targets) - def _get_named_modules(self, model: nn.Module) -> Dict[int, nn.Module]: + def _get_named_modules(self, model: nn.Module, need_dvs_node: bool, dvs_input_shape: Tuple[int, int]) -> Dict[int, nn.Module]: """Find for each node in the graph what its associated layer in `model` is. Parameters ---------- - model (nn.Module): the `spiking_model` used as argument to the class instance. + - need_dvs_node (bool): True of `dvs_input == True` and `spiking_model` doesn't start with a `DVSLayer`. + - dvs_input_shape (tuple): Shape of input in format `(height, width)`. Returns ---------- - indx_2_module_map (dict): the mapping between a node (`key` as an `int`) and its module (`value` as a `nn.Module`). """ + + assert need_dvs_node and isinstance(dvs_input_shape, tuple), f"DVSLayer instantiation is needed but 'dvs_input_shape == {dvs_input_shape}'." + indx_2_module_map = dict() for name, module in model.named_modules(): @@ -322,6 +341,10 @@ def _get_named_modules(self, model: nn.Module) -> Dict[int, nn.Module]: if name in self._name_2_indx_map: indx_2_module_map[self._name_2_indx_map[name]] = module + if need_dvs_node: + # Adds an entry for the `DVSLayer` node that is needed - default node name is 'dvs' + indx_2_module_map[self._name_2_indx_map['dvs']] = DVSLayer(input_shape=dvs_input_shape) + return indx_2_module_map def _update_internal_representation(self, remapped_nodes: Dict[int, int]): From d0ae8a5fa6fa16f71fc498ccfbe7fc92ee717e14 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Mon, 28 Oct 2024 12:41:27 +0100 Subject: [PATCH 236/379] WIP DVS - DVS node not given modifying graph extractor's name 2 index / index to module mapping and edges (in-place) to add entry for DVS node --- .../backend/dynapcnn/nir_graph_extractor.py | 67 +++++++++---------- 1 file changed, 33 insertions(+), 34 deletions(-) diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index 178f5592..ac54eaeb 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -57,22 +57,20 @@ def __init__(self, spiking_model: nn.Module, dummy_input: torch.tensor, dvs_inpu spiking_model, dummy_input, model_name=None ).ignore_tensors() - # This var. will be set to `True` if `dvs_input == True` and `spiking_model` does not start with DVS layer. - need_dvs_node = self._need_dvs_node(spiking_model, dvs_input) - dvs_input_shape = None - if need_dvs_node: - # We need to provide `(height, width)` to the DVSLayer instance that will be the module of the node 'dvs'. - _, _, height, width = dummy_input.shape - dvs_input_shape = (height, width) - # Map node names to indices - self._name_2_indx_map = self._get_name_2_indx_map(nir_graph, need_dvs_node) + self._name_2_indx_map = self._get_name_2_indx_map(nir_graph) # Extract edges list from graph - self._edges = self._get_edges_from_nir(nir_graph, self._name_2_indx_map) # @TODO edges need to be modified in place if DVS layer is needed. + self._edges = self._get_edges_from_nir(nir_graph, self._name_2_indx_map) # Determine entry points to graph - self._entry_nodes = self._get_entry_nodes(self._edges) # @TODO maybe functionality has to change here a when DVS layer is needed. + self._entry_nodes = self._get_entry_nodes(self._edges) # Store the associated `nn.Module` (layer) of each node. - self._indx_2_module_map = self._get_named_modules(spiking_model, need_dvs_node, dvs_input_shape) + self._indx_2_module_map = self._get_named_modules(spiking_model) + + # True if `dvs_input == True` and `spiking_model` does not start with DVS layer. + if self._need_dvs_node(spiking_model, dvs_input): + # input shape for `DVSLayer` instance that will be the module of the node 'dvs'. + _, _, height, width = dummy_input.shape + self._add_dvs_node(dvs_input_shape=(height, width)) # Verify that graph is compatible self.verify_graph_integrity() @@ -230,6 +228,24 @@ def verify_graph_integrity(self): ####################################################### Pivate Methods ####################################################### + def _add_dvs_node(self, dvs_input_shape: Tuple[int, int]) -> None: + """ In-place modification of `self._name_2_indx_map`, `self._indx_2_module_map`, and `self._edges` to accomodate the + creation of an extra node in the graph representing the DVS camera of the chip. + + Parameters + ---------- + - dvs_input_shape (tuple): Shape of input in format `(height, width)`. + """ + # @TODO - not considering pooling after the DVSLayer yet. + # @TODO - does self._entry_nodes need to have the index of the DVS node? + + # add name entry for node 'dvs'. + self._name_2_indx_map['dvs'] = len(self._name_2_indx_map) + # add module entry for node 'dvs'. + self._indx_2_module_map[self._name_2_indx_map['dvs']] = DVSLayer(input_shape=dvs_input_shape) + # set DVS node as input to each entry node of the graph. + self._edges.update({(self._name_2_indx_map['dvs'], entry_node) for entry_node in self._entry_nodes}) + def _need_dvs_node(self, model: nn.Module, dvs_input: bool) -> bool: """ Returns whether or not a node will need to be added to represent a `DVSLayer` instance. A new node will have to be added if `model` does not start with a `DVSLayer` instance and `dvs_input == True`. @@ -252,14 +268,12 @@ def _need_dvs_node(self, model: nn.Module, dvs_input: bool) -> bool: return not isinstance(first_module, DVSLayer) and dvs_input - def _get_name_2_indx_map(self, nir_graph: nirtorch.graph.Graph, need_dvs_node: bool) -> Dict[str, int]: - """Assign unique index to each node and return mapper from name to index. If `need_dvs_node == Ture` we want to - leave index `0` free to be assigned to the `DVSLayer` node that will have to be created. + def _get_name_2_indx_map(self, nir_graph: nirtorch.graph.Graph) -> Dict[str, int]: + """Assign unique index to each node and return mapper from name to index. Parameters ---------- - nir_graph (nirtorch.graph.Graph): a NIR graph representation of `spiking_model`. - - need_dvs_node (bool): True of `dvs_input == True` and `spiking_model` doesn't start with a `DVSLayer`. Returns ---------- @@ -268,17 +282,10 @@ def _get_name_2_indx_map(self, nir_graph: nirtorch.graph.Graph, need_dvs_node: b """ # Start name indexing from 1 if a DVS node needs to be added - name_2_indx_map = { - node.name: (node_idx + 1 if need_dvs_node else node_idx) - for node_idx, node in enumerate(nir_graph.node_list) + return { + node.name: node_idx for node_idx, node in enumerate(nir_graph.node_list) } - if need_dvs_node: - # Adds entry for the DVS node that needs to be created - default node name is 'dvs' - name_2_indx_map['dvs'] = 0 - - return name_2_indx_map - def _get_edges_from_nir( self, nir_graph: nirtorch.graph.Graph, name_2_indx_map: Dict[str, int] ) -> Set[Edge]: @@ -317,22 +324,18 @@ def _get_entry_nodes(self, edges: Set[Edge]) -> Set[Edge]: all_sources, all_targets = zip(*edges) return set(all_sources) - set(all_targets) - def _get_named_modules(self, model: nn.Module, need_dvs_node: bool, dvs_input_shape: Tuple[int, int]) -> Dict[int, nn.Module]: + def _get_named_modules(self, model: nn.Module) -> Dict[int, nn.Module]: """Find for each node in the graph what its associated layer in `model` is. Parameters ---------- - model (nn.Module): the `spiking_model` used as argument to the class instance. - - need_dvs_node (bool): True of `dvs_input == True` and `spiking_model` doesn't start with a `DVSLayer`. - - dvs_input_shape (tuple): Shape of input in format `(height, width)`. Returns ---------- - indx_2_module_map (dict): the mapping between a node (`key` as an `int`) and its module (`value` as a `nn.Module`). """ - assert need_dvs_node and isinstance(dvs_input_shape, tuple), f"DVSLayer instantiation is needed but 'dvs_input_shape == {dvs_input_shape}'." - indx_2_module_map = dict() for name, module in model.named_modules(): @@ -341,10 +344,6 @@ def _get_named_modules(self, model: nn.Module, need_dvs_node: bool, dvs_input_sh if name in self._name_2_indx_map: indx_2_module_map[self._name_2_indx_map[name]] = module - if need_dvs_node: - # Adds an entry for the `DVSLayer` node that is needed - default node name is 'dvs' - indx_2_module_map[self._name_2_indx_map['dvs']] = DVSLayer(input_shape=dvs_input_shape) - return indx_2_module_map def _update_internal_representation(self, remapped_nodes: Dict[int, int]): From cb63e6f6fe629ba082c4d5eac08a5f87e0e31091 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Mon, 28 Oct 2024 13:12:16 +0100 Subject: [PATCH 237/379] WIP DVS - DVS node not given edges types involving DVS layer added --- sinabs/backend/dynapcnn/connectivity_specs.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/sinabs/backend/dynapcnn/connectivity_specs.py b/sinabs/backend/dynapcnn/connectivity_specs.py index fc0da6da..1f2ef23e 100644 --- a/sinabs/backend/dynapcnn/connectivity_specs.py +++ b/sinabs/backend/dynapcnn/connectivity_specs.py @@ -1,7 +1,5 @@ """ functionality : list device-independent supported connections between layers on chip -author : Willian Soares Girao -contact : williansoaresgirao@gmail.com """ from typing import Union @@ -9,11 +7,14 @@ import torch.nn as nn import sinabs.layers as sl +from dvs_layer import DVSLayer Pooling = (sl.SumPool2d, nn.AvgPool2d) Weight = (nn.Conv2d, nn.Linear) Neuron = (sl.IAFSqueeze,) +Dvs = (DVSLayer,) +# @TODO - need to list other edge cases involving DVS layer (for now only dvs-weight and dvs-pooling). VALID_SINABS_EDGE_TYPES_ABSTRACT = { # convoluion is always followed by a neuron layer. (Weight, Neuron): "weight-neuron", @@ -25,6 +26,10 @@ (Neuron, Weight): "neuron-weight", # Pooling can be followed by weight layer of next core (Pooling, Weight): "pooling-weight", + # Dvs can be followed by weight layer of next core + (Dvs, Weight): "dvs-weight", + # Dvs can be followed by pooling layers + (Dvs, Pooling): "dvs-pooling", } # Unpack dict @@ -39,4 +44,4 @@ LAYER_TYPES_WITH_MULTIPLE_INPUTS = Union[sl.Merge] # Neuron and pooling layers can have their output sent to multiple cores -LAYER_TYPES_WITH_MULTIPLE_OUTPUTS = Union[(*Neuron, *Pooling)] +LAYER_TYPES_WITH_MULTIPLE_OUTPUTS = Union[(*Neuron, *Pooling, *Dvs)] From 49c013ad0bf1f8e5cd59fc6d0a52acd3b34f8463 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Mon, 28 Oct 2024 13:29:33 +0100 Subject: [PATCH 238/379] WIP DVS - DVS node not given dvs_input (bool) arg passed down to collect_dynapcnn_layer_info() to raise error if edge type involving DVS is not found when dvs_input == True. --- sinabs/backend/dynapcnn/dynapcnn_network.py | 4 ++-- sinabs/backend/dynapcnn/nir_graph_extractor.py | 10 +++++++--- sinabs/backend/dynapcnn/sinabs_edges_handler.py | 15 +++++++++++---- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index a98bdad4..a6bacff6 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -69,7 +69,7 @@ def __init__( batch_size = sinabs.utils.get_smallest_compatible_time_dimension(snn) # computational graph from original PyTorch module. self._graph_extractor = GraphExtractor( - snn, torch.randn((batch_size, *self.input_shape)) + snn, torch.randn((batch_size, *self.input_shape), self.dvs_input) ) # needs the batch dimension. # Remove nodes of ignored classes (including merge nodes) @@ -77,7 +77,7 @@ def __init__( # Module to execute forward pass through network self._dynapcnn_module = self._graph_extractor.get_dynapcnn_network_module( - discretize=discretize, weight_rescaling_fn=weight_rescaling_fn + discretize=discretize, weight_rescaling_fn=weight_rescaling_fn, dvs_input=self.dvs_input ) self._dynapcnn_module.setup_dynapcnnlayer_graph(index_layers_topologically=True) diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index ac54eaeb..fe1d68c3 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -105,7 +105,7 @@ def indx_2_module_map(self) -> Dict[int, nn.Module]: return {n: module for n, module in self._indx_2_module_map.items()} def get_dynapcnn_network_module( - self, discretize: bool = False, weight_rescaling_fn: Optional[Callable] = None + self, discretize: bool = False, weight_rescaling_fn: Optional[Callable] = None, dvs_input: bool = False ) -> DynapcnnNetworkModule: """ Create DynapcnnNetworkModule based on stored graph representation @@ -117,6 +117,7 @@ def get_dynapcnn_network_module( weights to dynapcnn. Set to `False` only for testing purposes. - weight_rescaling_fn (callable): a method that handles how the re-scaling factor for one or more `SumPool2d` projecting to the same convolutional layer are combined/re-scaled before applying them. + - dvs_input (bool): wether or not dynapcnn receive input from its DVS camera. Returns ------- @@ -129,6 +130,7 @@ def get_dynapcnn_network_module( edges = self.edges, nodes_io_shapes=self.nodes_io_shapes, entry_nodes=self.entry_nodes, + dvs_input=dvs_input, ) # build `DynapcnnLayer` instances from mapper. @@ -236,8 +238,8 @@ def _add_dvs_node(self, dvs_input_shape: Tuple[int, int]) -> None: ---------- - dvs_input_shape (tuple): Shape of input in format `(height, width)`. """ - # @TODO - not considering pooling after the DVSLayer yet. - # @TODO - does self._entry_nodes need to have the index of the DVS node? + + # [] @TODO - not considering pooling after the DVSLayer yet. # add name entry for node 'dvs'. self._name_2_indx_map['dvs'] = len(self._name_2_indx_map) @@ -246,6 +248,8 @@ def _add_dvs_node(self, dvs_input_shape: Tuple[int, int]) -> None: # set DVS node as input to each entry node of the graph. self._edges.update({(self._name_2_indx_map['dvs'], entry_node) for entry_node in self._entry_nodes}) + # [] @TODO - all indexes in 'self._entry_nodes' are no longer entry nodes of the network since a DVS layer is being added. + def _need_dvs_node(self, model: nn.Module, dvs_input: bool) -> bool: """ Returns whether or not a node will need to be added to represent a `DVSLayer` instance. A new node will have to be added if `model` does not start with a `DVSLayer` instance and `dvs_input == True`. diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index 8eab7b19..37f04e1e 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -25,6 +25,7 @@ def collect_dynapcnn_layer_info( edges: Set[Edge], nodes_io_shapes: Dict[int, Dict[str, Tuple[Size, Size]]], entry_nodes: Set[int], + dvs_input: bool, ) -> Dict[int, Dict]: """Collect information to construct DynapcnnLayer instances. @@ -37,10 +38,11 @@ def collect_dynapcnn_layer_info( Parameters ---------- - indx_2_module_map (dict): Maps node IDs of the graph as `key` to their associated module as `value` - edges (set of tuples): Represent connections between two nodes in computational graph - nodes_io_shapes (dict): Map from node ID to dict containing node's in- and output shapes - entry_nodes (set of int): IDs of nodes that receive external input + - indx_2_module_map (dict): Maps node IDs of the graph as `key` to their associated module as `value` + - edges (set of tuples): Represent connections between two nodes in computational graph + - nodes_io_shapes (dict): Map from node ID to dict containing node's in- and output shapes + - entry_nodes (set of int): IDs of nodes that receive external input + - dvs_input (bool): wether or not dynapcnn receive input from its DVS camera. Returns ------- @@ -62,6 +64,11 @@ def collect_dynapcnn_layer_info( "None such weight-neuron pair has been found in the provided network." ) + if not any(edge in edges_by_type for edge in ["dvs-weight", "dvs-pooling"]) and dvs_input: + raise InvalidGraphStructure( + "DVS camera is set selected for usage (dvs_input == True) but edge type involving it has not been found." + ) + # Dict to collect information for each future dynapcnn layer dynapcnn_layer_info = dict() # Map node IDs to dynapcnn layer ID From 5366513e7d9de41a0058106a9ae6ffbf2168c2d6 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Mon, 28 Oct 2024 14:38:14 +0100 Subject: [PATCH 239/379] WIP DVS - DVS node not given handling 'dvs-weight' edge types during collection of dynapcnn layers info.. --- .../backend/dynapcnn/sinabs_edges_handler.py | 64 ++++++++++++++++++- 1 file changed, 63 insertions(+), 1 deletion(-) diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index 37f04e1e..d4dc4fda 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -18,7 +18,7 @@ UnmatchedPoolingEdges, ) from .utils import Edge - +from dvs_layer import DVSLayer def collect_dynapcnn_layer_info( indx_2_module_map: Dict[int, nn.Module], @@ -86,6 +86,18 @@ def collect_dynapcnn_layer_info( entry_nodes, ) + # Process all dvs->weight edges connecting the DVS camera to a unique dynapcnn layer. + while edges_by_type["dvs-weight"]: + edge = edges_by_type["dvs-weight"].pop() + add_or_update_dvs_to_entry( + edge, + dynapcnn_layer_info, + indx_2_module_map, + node_2_layer_map, + ) + + # TODO - handle dvs->pooling connections. + # Process all edges connecting two dynapcnn layers that do not include pooling while edges_by_type.get("neuron-weight", False): edge = edges_by_type["neuron-weight"].pop() @@ -299,6 +311,56 @@ def add_pooling_to_entry( assert node not in node_2_layer_map node_2_layer_map[node] = layer_idx +def add_or_update_dvs_to_entry( + edge: Edge, + dynapcnn_layer_info: Dict[int, Dict[int, Dict]], + indx_2_module_map: Dict[int, nn.Module], + node_2_layer_map: Dict[int, int], +) -> None: + """ Initiate or update dict to hold information for a DVS Layer configuration based on a "dvs-weight" edges. + Change `dynapcnn_layer_info` in-place. If a entry for the DVS node exists the function will add a new entry + to the `desctinations` key of its dictionary. + + Parameters + ---------- + edge: Tuple of 2 integers, indicating edge between two nodes in graph. + Edge target has to be within an existing entry of `dynapcnn_layer_info`. + dynapcnn_layer_info: Dict with one entry for each future dynapcnn layer. + key is unique dynapcnn layer ID, value is dict with nodes of the layer + Will be updated in-place. + indx_2_module_map (dict): Maps node IDs of the graph as `key` to their associated module as `value` + node_2_layer_map (dict): Maps each node ID to the ID of the layer it is assigned to. + Will be updated in-place. + """ + + assert isinstance(indx_2_module_map[edge[0]], DVSLayer), f'Source node in edge {edge} is of type {type(DVSLayer)} (it should be a DVSLayer instance).' + assert edge[1] in node_2_layer_map, f'Node {edge[1]} is a weight node that should have been initialized.' + + if edge[0] not in node_2_layer_map: + # DVS node hasn't being initialized yet: take current length of the dict as new, unique ID. + layer_id = len(dynapcnn_layer_info) + assert layer_id not in dynapcnn_layer_info + + # Init. entry for a DVS layer using its configuration dict. + dynapcnn_layer_info[layer_id] = { + "dvs_layer": True, + # TODO - GraphTracer not populating I/O shape for DVS yet. + "input_shape": nodes_io_shapes[edge[0]]["input"], + "config_dict": indx_2_module_map[edge[0]].get_config_dict(), + "destinations": [node_2_layer_map[edge[1]]], + } + + node_2_layer_map[edge[0]] = layer_id + else: + # Update entry for DVS with new destination. + source_layer_id = node_2_layer_map[edge[0]] + destination_layer_id = node_2_layer_map[edge[1]] + + assert 'dvs_layer' in dynapcnn_layer_info[source_layer_id] + assert dynapcnn_layer_info[source_layer_id]['dvs_layer'] + assert destination_layer_id not in dynapcnn_layer_info[source_layer_id]["destinations"] + + dynapcnn_layer_info[source_layer_id]["destinations"].append(destination_layer_id) def set_exit_destinations(dynapcnn_layer: Dict) -> None: """Set minimal destination entries for layers that don't have any. From 460e7c60114052e6dbce60d73e5e67fe2239054c Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Mon, 28 Oct 2024 14:48:32 +0100 Subject: [PATCH 240/379] WIP DVS - DVS node not given moved get_entry_nodes() to after the DVS node creating has been handled.. --- sinabs/backend/dynapcnn/nir_graph_extractor.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index fe1d68c3..cd107bae 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -61,8 +61,6 @@ def __init__(self, spiking_model: nn.Module, dummy_input: torch.tensor, dvs_inpu self._name_2_indx_map = self._get_name_2_indx_map(nir_graph) # Extract edges list from graph self._edges = self._get_edges_from_nir(nir_graph, self._name_2_indx_map) - # Determine entry points to graph - self._entry_nodes = self._get_entry_nodes(self._edges) # Store the associated `nn.Module` (layer) of each node. self._indx_2_module_map = self._get_named_modules(spiking_model) @@ -72,6 +70,9 @@ def __init__(self, spiking_model: nn.Module, dummy_input: torch.tensor, dvs_inpu _, _, height, width = dummy_input.shape self._add_dvs_node(dvs_input_shape=(height, width)) + # Determine entry points to graph + self._entry_nodes = self._get_entry_nodes(self._edges) + # Verify that graph is compatible self.verify_graph_integrity() @@ -240,6 +241,7 @@ def _add_dvs_node(self, dvs_input_shape: Tuple[int, int]) -> None: """ # [] @TODO - not considering pooling after the DVSLayer yet. + # [] @TODO - I/O shape in 'self._nodes_io_shapes' not being handled yet. # add name entry for node 'dvs'. self._name_2_indx_map['dvs'] = len(self._name_2_indx_map) @@ -248,8 +250,6 @@ def _add_dvs_node(self, dvs_input_shape: Tuple[int, int]) -> None: # set DVS node as input to each entry node of the graph. self._edges.update({(self._name_2_indx_map['dvs'], entry_node) for entry_node in self._entry_nodes}) - # [] @TODO - all indexes in 'self._entry_nodes' are no longer entry nodes of the network since a DVS layer is being added. - def _need_dvs_node(self, model: nn.Module, dvs_input: bool) -> bool: """ Returns whether or not a node will need to be added to represent a `DVSLayer` instance. A new node will have to be added if `model` does not start with a `DVSLayer` instance and `dvs_input == True`. From bcaec928175b2307628b3728904b047b370efe90 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Mon, 28 Oct 2024 15:00:37 +0100 Subject: [PATCH 241/379] WIP DVS - DVS node not given checking for DVS layer entry when instantiating DynapcnnLayers. --- sinabs/backend/dynapcnn/dynapcnn_layer_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py b/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py index 1d7b5086..73d3605d 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py @@ -28,6 +28,7 @@ def construct_dynapcnnlayers_from_mapper( dynapcnn_layers = { layer_idx: construct_single_dynapcnn_layer(layer_info, discretize) for layer_idx, layer_info in dcnnl_map.items() + if 'dvs_layer' not in layer_info # handle only dicts with info. for DynapcnnLayer instances. } destination_map = construct_destination_map(dcnnl_map) From eedc62d2b3b4a28419867f0f1c3388c64c5c54d6 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Mon, 28 Oct 2024 15:17:06 +0100 Subject: [PATCH 242/379] WIP DVS - DVS node not given destinations map being updated with DVS node's destinations (if node exists). --- sinabs/backend/dynapcnn/dynapcnn_layer_utils.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py b/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py index 73d3605d..f750fa8c 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py @@ -32,6 +32,7 @@ def construct_dynapcnnlayers_from_mapper( } destination_map = construct_destination_map(dcnnl_map) + update_destination_map_with_dvs(dcnnl_map, destination_map) entry_points = collect_entry_points(dcnnl_map) return dynapcnn_layers, destination_map, entry_points @@ -298,3 +299,17 @@ def collect_entry_points(dcnnl_map: Dict[int, Dict]) -> Set[int]: layer_index for layer_index, layer_info in dcnnl_map.items() if layer_info["is_entry_node"] } + +def update_destination_map_with_dvs(dcnnl_map: Dict[int, Dict], destination_map: Dict[int, List[int]]) -> None: + """ Modifies `destination_map` in-place to add entry for the DVS came node (if it existis). + + Parameters + ---------- + - dcnnl_map (dict): Dict holding info needed to instantiate DynapcnnLayer instances. + - destination_map (dict): dict mapping to each layer index a set of destination indices. + """ + for layer_index, layer_info in dcnnl_map.items(): + if 'dvs_layer' in layer_info: + assert layer_info['dvs_layer'] + assert layer_index not in destination_map, 'It seems more than one DVS node has been added (only one should exist).' + destination_map[layer_index] = layer_info['destinations'] \ No newline at end of file From e6aafe0bb14ac024aba3530684050ef34058b82a Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Mon, 28 Oct 2024 16:11:45 +0100 Subject: [PATCH 243/379] WIP DVS - DVS node not given passing DVS node info. (dict in the same fashion as DynapcnnLayer info.) down to DynapcnnNetworkModule to be accessed during build_config(). --- sinabs/backend/dynapcnn/chips/dynapcnn.py | 4 +++- .../backend/dynapcnn/dynapcnn_layer_utils.py | 3 +++ sinabs/backend/dynapcnn/dynapcnn_network.py | 5 +++++ .../backend/dynapcnn/nir_graph_extractor.py | 7 +++++-- .../backend/dynapcnn/sinabs_edges_handler.py | 21 ++++++++++++++++++- 5 files changed, 36 insertions(+), 4 deletions(-) diff --git a/sinabs/backend/dynapcnn/chips/dynapcnn.py b/sinabs/backend/dynapcnn/chips/dynapcnn.py index 81231164..931537fd 100644 --- a/sinabs/backend/dynapcnn/chips/dynapcnn.py +++ b/sinabs/backend/dynapcnn/chips/dynapcnn.py @@ -255,6 +255,7 @@ def build_config( layers: Dict[int, DynapcnnLayer], destination_map: Dict[int, List[int]], layer2core_map: Dict[int, int], + dvs_node_info: Dict, ) -> DynapcnnConfiguration: """Uses `DynapcnnLayer` objects to configure their equivalent chip cores @@ -264,6 +265,7 @@ def build_config( - layer2core_map (Dict): Keys are layer indices, values are corresponding cores on hardware. Needed to map the destinations.] - destination_indices (List): Indices of destination layers for `layer` + - dvs_node_info (dict): contains information associated with the `DVSLayer` node (if no DVS node exists it'll return `None`). Returns ------- @@ -277,7 +279,7 @@ def build_config( # Loop over layers in network and write corresponding configurations for layer_index, ith_dcnnl in layers.items(): if isinstance(ith_dcnnl, DVSLayer): - # TODO DVSLayer not supported yet. + # TODO DVSLayer using `dvs_node_info`. pass elif isinstance(ith_dcnnl, DynapcnnLayer): diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py b/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py index f750fa8c..9ea208d7 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py @@ -32,7 +32,10 @@ def construct_dynapcnnlayers_from_mapper( } destination_map = construct_destination_map(dcnnl_map) + + # update mapper if a DVS layer exists. update_destination_map_with_dvs(dcnnl_map, destination_map) + entry_points = collect_entry_points(dcnnl_map) return dynapcnn_layers, destination_map, entry_points diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index a6bacff6..d6622aa9 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -83,6 +83,10 @@ def __init__( ####################################################### Public Methods ####################################################### + @property + def dvs_node_info(self): + return self._dynapcnn_module.dvs_node_info + @property def dynapcnn_layers(self): return self._dynapcnn_module.dynapcnn_layers @@ -552,6 +556,7 @@ def _make_config( layers=self.dynapcnn_layers, destination_map=self.layer_destination_map, layer2core_map=layer2core_map, + dvs_node_info=dvs_node_info, ) # TODO not handling DVSLayer yet (this is from the old implementation, should be revised). diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index cd107bae..90c318af 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -17,7 +17,7 @@ from .dynapcnn_layer_utils import construct_dynapcnnlayers_from_mapper from .dynapcnnnetwork_module import DynapcnnNetworkModule from .exceptions import InvalidGraphStructure, InvalidModelWithDVSSetup -from .sinabs_edges_handler import collect_dynapcnn_layer_info +from .sinabs_edges_handler import collect_dynapcnn_layer_info, get_dvs_node_from_mapper from .utils import Edge, topological_sorting @@ -143,9 +143,12 @@ def get_dynapcnn_network_module( ) ) + # DVSLayer node information (None if DVS camera is not used). + dvs_node_info = get_dvs_node_from_mapper(dcnnl_map) + # Instantiate the DynapcnnNetworkModule return DynapcnnNetworkModule( - dynapcnn_layers, destination_map, entry_points + dynapcnn_layers, destination_map, entry_points, dvs_node_info ) def remove_nodes_by_class(self, node_classes: Tuple[Type]): diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index d4dc4fda..df647f2c 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -6,7 +6,7 @@ """ from collections import deque -from typing import Dict, List, Set, Tuple, Type +from typing import Dict, List, Set, Tuple, Type, Optional from torch import Size, nn @@ -20,6 +20,23 @@ from .utils import Edge from dvs_layer import DVSLayer +def get_dvs_node_from_mapper(dcnnl_map: Dict) -> Optional[Dict]: + """ Returns the information dict associated with the `DVSLayer` instance within `dcnnl_map`. + + Parameters + ---------- + - dcnnl_map: Dict holding info needed to instantiate DynapcnnLayer instances. + + Returns + ------- + - Dict containing information associated with the `DVSLayer` node (if no DVS node exists it'll return `None`). + """ + for layer_index, layer_info in dcnnl_map.items(): + if 'dvs_layer' in layer_info: + assert layer_info['dvs_layer'] + return destination_map[layer_index] + return None + def collect_dynapcnn_layer_info( indx_2_module_map: Dict[int, nn.Module], edges: Set[Edge], @@ -344,8 +361,10 @@ def add_or_update_dvs_to_entry( # Init. entry for a DVS layer using its configuration dict. dynapcnn_layer_info[layer_id] = { "dvs_layer": True, + "node_id": edge[0], # TODO - GraphTracer not populating I/O shape for DVS yet. "input_shape": nodes_io_shapes[edge[0]]["input"], + "module": indx_2_module_map[edge[0]], "config_dict": indx_2_module_map[edge[0]].get_config_dict(), "destinations": [node_2_layer_map[edge[1]]], } From 500308676083710add82b39b4ac7fa5a241884a6 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Mon, 28 Oct 2024 16:57:05 +0100 Subject: [PATCH 244/379] WIP DVS - DVS node not given passing DVS node info. (dict in the same fashion as DynapcnnLayer info.) down to config builder. --- sinabs/backend/dynapcnn/chips/dynapcnn.py | 61 ++++++++++++++----- .../dynapcnn/dynapcnnnetwork_module.py | 11 +++- .../backend/dynapcnn/sinabs_edges_handler.py | 1 - 3 files changed, 56 insertions(+), 17 deletions(-) diff --git a/sinabs/backend/dynapcnn/chips/dynapcnn.py b/sinabs/backend/dynapcnn/chips/dynapcnn.py index 931537fd..8e261331 100644 --- a/sinabs/backend/dynapcnn/chips/dynapcnn.py +++ b/sinabs/backend/dynapcnn/chips/dynapcnn.py @@ -29,10 +29,42 @@ def get_default_config(cls) -> "DynapcnnConfiguration": @classmethod def get_dvs_layer_config_dict(cls, layer: DVSLayer): ... + # @classmethod + # def write_dvs_layer_config(cls, layer: DVSLayer, config: DVSLayerConfig): + # for param, value in layer.get_config_dict().items(): + # setattr(config, param, value) + + @classmethod - def write_dvs_layer_config(cls, layer: DVSLayer, config: DVSLayerConfig): + def write_dvs_layer_config( + cls, + layer: DVSLayer, + layer2core_map: Dict[int, int], + destination_indices: List[int], + chip_layer: DVSLayerConfig, + ) -> None: + """Write a DVS layer configuration to the conf object. + + Uses the data in `layer` to configure a `DVSLayerConfig` to use the chip's DVS camera. + + Parameters + ---------- + - layer (DVSLayer): Layer instance from which to generate the config + - layer2core_map (Dict): Keys are layer indices, values are corresponding + cores on hardware. Needed to map the destinations.] + - destination_indices (List): Indices of destination layers for `layer` + - chip_layer (DVSLayerConfig): Configuration object of the corrsesponding + on-chip core. Will be changed in-place based on `layer`. + """ for param, value in layer.get_config_dict().items(): - setattr(config, param, value) + setattr(chip_layer, param, value) + + # Set destinations. + for dest_idx, dest in enumerate(destination_indices): + chip_layer.destinations[dest_idx].layer = layer2core_map[dest] + chip_layer.destinations[dest_idx].enable = True + + chip_layer.pass_sensor_events = True @classmethod def set_kill_bits(cls, layer: DynapcnnLayer, config_dict: dict) -> dict: @@ -273,16 +305,22 @@ def build_config( the chip based on the provided `layers`. """ config = cls.get_default_config() + config.dvs_layer.pass_sensor_events = False - has_dvs_layer = False # TODO DVSLayer not supported yet. + # Uses the DVS camera. + if isinstance(dvs_node_info, dict): + chip_layer = config.dvs_layer + sw_layer = dvs_node_info['module'] + destination_indices = dvs_node_info['destinations'] + # Write camera configuration. + cls.write_dvs_layer_config(sw_layer, layer2core_map, destination_indices, chip_layer) + chip_layer.pass_sensor_events = False + + # TODO - for now it's being handled separatly but it might make more sense to handle it within `layers`. # Loop over layers in network and write corresponding configurations for layer_index, ith_dcnnl in layers.items(): - if isinstance(ith_dcnnl, DVSLayer): - # TODO DVSLayer using `dvs_node_info`. - pass - - elif isinstance(ith_dcnnl, DynapcnnLayer): + if isinstance(ith_dcnnl, DynapcnnLayer): # retrieve config dict for current layer chip_layer = config.cnn_layers[layer2core_map[layer_index]] # write core configuration. @@ -292,19 +330,12 @@ def build_config( chip_layer=chip_layer, destination_indices=destination_map[layer_index], ) - else: # shouldn't happen since type checks are made previously. raise TypeError( f"Layer (index {layer_index}) is unexpected in the model: \n{ith_dcnnl}" ) - if not has_dvs_layer: - # TODO DVSLayer not supported yet. - config.dvs_layer.pass_sensor_events = False - else: - config.dvs_layer.pass_sensor_events = False - return config @classmethod diff --git a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py index 6e98aa59..b53ca3d9 100644 --- a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py +++ b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py @@ -25,7 +25,8 @@ class DynapcnnNetworkModule(nn.Module): - dynapcnn_layers (dict): a mapper containing `DynapcnnLayer` instances. - destination_map (dict): Maps layer indices to list of destination indices. Exit destinations are marked by negative integers - - entry_points (set): Set of layer indices that act as network entry points + - entry_points (set): Set of layer indices that act as network entry points. + - dvs_node_info (dict): contains information associated with the `DVSLayer` node (if no DVS node exists it'll return `None`). Attributes ---------- @@ -45,6 +46,7 @@ def __init__( dynapcnn_layers: Dict[int, DynapcnnLayer], destination_map: Dict[int, List[int]], entry_points: Set[int], + dvs_node_info: Dict, ): super().__init__() @@ -58,6 +60,13 @@ def __init__( # `Merge` layers are stateless. One instance can be used for all merge points during forward pass self.merge_layer = sl.Merge() + + # TODO - just saved for now [ STILL NEEDS TO BE INCORPORATED TO THE FORWARD PASS ]. + self._dvs_node_info = dvs_node_info + + @property + def dvs_node_info(self): + return self._dvs_node_info @property def destination_map(self): diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index df647f2c..52708bbe 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -365,7 +365,6 @@ def add_or_update_dvs_to_entry( # TODO - GraphTracer not populating I/O shape for DVS yet. "input_shape": nodes_io_shapes[edge[0]]["input"], "module": indx_2_module_map[edge[0]], - "config_dict": indx_2_module_map[edge[0]].get_config_dict(), "destinations": [node_2_layer_map[edge[1]]], } From d0c53898b41782d0d67bfe464d4f7cbd070fb6e9 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Mon, 28 Oct 2024 16:59:06 +0100 Subject: [PATCH 245/379] WIP DVS - DVS node not given setting pass_sensor_events to True in write config call. --- sinabs/backend/dynapcnn/chips/dynapcnn.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sinabs/backend/dynapcnn/chips/dynapcnn.py b/sinabs/backend/dynapcnn/chips/dynapcnn.py index 8e261331..0acdba24 100644 --- a/sinabs/backend/dynapcnn/chips/dynapcnn.py +++ b/sinabs/backend/dynapcnn/chips/dynapcnn.py @@ -314,7 +314,6 @@ def build_config( destination_indices = dvs_node_info['destinations'] # Write camera configuration. cls.write_dvs_layer_config(sw_layer, layer2core_map, destination_indices, chip_layer) - chip_layer.pass_sensor_events = False # TODO - for now it's being handled separatly but it might make more sense to handle it within `layers`. From 6a807efd337d6bdcf777dc1b52020a43391d9bca Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Mon, 28 Oct 2024 19:57:14 +0100 Subject: [PATCH 246/379] WIP DVS - DVS node not given forward retunrs tuple like in DynapcnnLayer (still WIP). --- sinabs/backend/dynapcnn/dvs_layer.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/sinabs/backend/dynapcnn/dvs_layer.py b/sinabs/backend/dynapcnn/dvs_layer.py index 8111edae..84e5a2fb 100644 --- a/sinabs/backend/dynapcnn/dvs_layer.py +++ b/sinabs/backend/dynapcnn/dvs_layer.py @@ -43,6 +43,7 @@ def __init__( flip_y: bool = False, swap_xy: bool = False, disable_pixel_array: bool = True, + destinations: list = [0] # TODO: just to get the forward to work but should use same concept for the pool argument in DynapcnnLayer. ): super().__init__() @@ -222,15 +223,23 @@ def forward(self, data): # Merge polarities if self.merge_polarities: data = data.sum(1, keepdim=True) + # Pool out = self.pool_layer(data) + # Crop if self.crop_layer is not None: out = self.crop_layer(out) + # Flip stuff out = self.flip_layer(out) - return out + # TODO: just to get the forward to work but should use same concept for the pool argument in DynapcnnLayer. + returns = [] + for dest in destinations: + returns.append(out) + + return tuple(returns) def get_pooling(self) -> Tuple[int, int]: """Pooling kernel shape. From 3711bc0d695e8c08e823a98c963b3f3cda384330 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Mon, 28 Oct 2024 20:12:35 +0100 Subject: [PATCH 247/379] WIP DVS - DVS node not given incorporated DVS into DynapcnnNetworkModule-level graph. --- .../dynapcnn/dynapcnnnetwork_module.py | 34 ++++++++++++------- .../backend/dynapcnn/sinabs_edges_handler.py | 1 + 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py index b53ca3d9..23c86bc9 100644 --- a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py +++ b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py @@ -2,7 +2,7 @@ # contact : wsoaresgirao@gmail.com from collections import defaultdict -from typing import Dict, List, Set, Union +from typing import Dict, List, Set, Union, Optional from warnings import warn import torch.nn as nn @@ -46,24 +46,28 @@ def __init__( dynapcnn_layers: Dict[int, DynapcnnLayer], destination_map: Dict[int, List[int]], entry_points: Set[int], - dvs_node_info: Dict, + dvs_node_info: Optional[Dict], ): super().__init__() + self._dvs_node_info = dvs_node_info + + # nodes in a DynapcnnNetwork graph. + module_dict = {str(idx): lyr for idx, lyr in dynapcnn_layers.items()} + # Insert DVS node if DVS was enabled. + if isinstance(self._dvs_node_info, Dict): + module_dict[str(dvs_node_info['layer_id'])] = dvs_node_info['module'] + # Unfortunately ModuleDict does not allow for integer keys # TODO: Consider using list instead of dict - self._dynapcnn_layers = nn.ModuleDict( - {str(idx): lyr for idx, lyr in dynapcnn_layers.items()} - ) + self._dynapcnn_layers = nn.ModuleDict(module_dict) + self._destination_map = destination_map self._entry_points = entry_points # `Merge` layers are stateless. One instance can be used for all merge points during forward pass self.merge_layer = sl.Merge() - # TODO - just saved for now [ STILL NEEDS TO BE INCORPORATED TO THE FORWARD PASS ]. - self._dvs_node_info = dvs_node_info - @property def dvs_node_info(self): return self._dvs_node_info @@ -75,7 +79,11 @@ def destination_map(self): @property def dynapcnn_layers(self): # Convert string-indices to integers - return {int(idx): lyr for idx, lyr in self._dynapcnn_layers.items()} + dynapcnn_layers = {int(idx): lyr for idx, lyr in self._dynapcnn_layers.items()} + # Insert DVS node if DVS was enabled. + if isinstance(self.dvs_node_info, Dict): + dynapcnn_layers[str(self.dvs_node_info['layer_id'])] = self.dvs_node_info['module'] + return dynapcnn_layers @property def entry_points(self): @@ -326,9 +334,11 @@ def remap(key): return mapping[key] # Remap all internal objects - self._dynapcnn_layers = nn.ModuleDict( - {str(remap(int(idx))): lyr for idx, lyr in self._dynapcnn_layers.items()} - ) + dynapcnn_layers = {str(remap(int(idx))): lyr for idx, lyr in self._dynapcnn_layers.items()} + if isinstance(self.dvs_node_info, Dict): + dynapcnn_layers[str(remap(self.dvs_node_info['layer_id']))] = self.dvs_node_info['module'] + self._dynapcnn_layers = nn.ModuleDict(dynapcnn_layers) + self._entry_points = {remap(idx) for idx in self._entry_points} self._destination_map = { remap(idx): [remap(dest) for dest in destinations] diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index 52708bbe..5ff120fb 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -366,6 +366,7 @@ def add_or_update_dvs_to_entry( "input_shape": nodes_io_shapes[edge[0]]["input"], "module": indx_2_module_map[edge[0]], "destinations": [node_2_layer_map[edge[1]]], + 'layer_id': layer_id, } node_2_layer_map[edge[0]] = layer_id From f3015996d7f5a9bad12c6beb7f4d658aa5766f46 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Mon, 28 Oct 2024 20:38:04 +0100 Subject: [PATCH 248/379] WIP DVS - DVS node not given fixed argument type hint. --- sinabs/backend/dynapcnn/chips/dynapcnn.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sinabs/backend/dynapcnn/chips/dynapcnn.py b/sinabs/backend/dynapcnn/chips/dynapcnn.py index 0acdba24..eaf952eb 100644 --- a/sinabs/backend/dynapcnn/chips/dynapcnn.py +++ b/sinabs/backend/dynapcnn/chips/dynapcnn.py @@ -1,5 +1,5 @@ import copy -from typing import Dict, List +from typing import Dict, List, Optional from warnings import warn import samna @@ -287,7 +287,7 @@ def build_config( layers: Dict[int, DynapcnnLayer], destination_map: Dict[int, List[int]], layer2core_map: Dict[int, int], - dvs_node_info: Dict, + dvs_node_info: Optional[Dict], ) -> DynapcnnConfiguration: """Uses `DynapcnnLayer` objects to configure their equivalent chip cores From 0915c1a2e39c27edd7b3ad5a37db68ffb3b4493d Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Mon, 28 Oct 2024 20:44:20 +0100 Subject: [PATCH 249/379] WIP DVS - DVS node not given reindex_layer updates dvs_node_info['layer_id'] in-place during re-indexing. --- sinabs/backend/dynapcnn/dynapcnnnetwork_module.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py index 23c86bc9..7f6f8389 100644 --- a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py +++ b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py @@ -336,7 +336,11 @@ def remap(key): # Remap all internal objects dynapcnn_layers = {str(remap(int(idx))): lyr for idx, lyr in self._dynapcnn_layers.items()} if isinstance(self.dvs_node_info, Dict): - dynapcnn_layers[str(remap(self.dvs_node_info['layer_id']))] = self.dvs_node_info['module'] + _ = str(remap(self.dvs_node_info['layer_id'])) + dynapcnn_layers[_] = self.dvs_node_info['module'] + # @TODO - update DVS node layer id in-place [THIS NEEDS VALIDATION] + self.dvs_node_info['layer_id'] = int(_) + self._dynapcnn_layers = nn.ModuleDict(dynapcnn_layers) self._entry_points = {remap(idx) for idx in self._entry_points} From 8a6553e11dcf6c44fed76faa597048bf14f22ec9 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Mon, 28 Oct 2024 20:47:34 +0100 Subject: [PATCH 250/379] WIP DVS - DVS node not given minor edits. --- sinabs/backend/dynapcnn/dynapcnn_network.py | 2 -- sinabs/backend/dynapcnn/dynapcnnnetwork_module.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index d6622aa9..52f18dd1 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -55,9 +55,7 @@ def __init__( """ super().__init__() - # TODO for now the graph part is not taking into consideration DVS inputs. # check if dvs input is expected. - dvs_input = False self.dvs_input = dvs_input self.input_shape = input_shape self._layer2core_map = None diff --git a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py index 7f6f8389..29f79f27 100644 --- a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py +++ b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py @@ -82,7 +82,7 @@ def dynapcnn_layers(self): dynapcnn_layers = {int(idx): lyr for idx, lyr in self._dynapcnn_layers.items()} # Insert DVS node if DVS was enabled. if isinstance(self.dvs_node_info, Dict): - dynapcnn_layers[str(self.dvs_node_info['layer_id'])] = self.dvs_node_info['module'] + dynapcnn_layers[self.dvs_node_info['layer_id']] = self.dvs_node_info['module'] return dynapcnn_layers @property From 0fa7bf6499f1959c09c748545b4da7ad757ad0e3 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Tue, 29 Oct 2024 11:10:10 +0100 Subject: [PATCH 251/379] WIP DVS - DVS node not given fixed minor bug with import DVSLayer --- sinabs/backend/dynapcnn/connectivity_specs.py | 2 +- sinabs/backend/dynapcnn/nir_graph_extractor.py | 2 +- sinabs/backend/dynapcnn/sinabs_edges_handler.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sinabs/backend/dynapcnn/connectivity_specs.py b/sinabs/backend/dynapcnn/connectivity_specs.py index 1f2ef23e..5eda39aa 100644 --- a/sinabs/backend/dynapcnn/connectivity_specs.py +++ b/sinabs/backend/dynapcnn/connectivity_specs.py @@ -7,7 +7,7 @@ import torch.nn as nn import sinabs.layers as sl -from dvs_layer import DVSLayer +from .dvs_layer import DVSLayer Pooling = (sl.SumPool2d, nn.AvgPool2d) Weight = (nn.Conv2d, nn.Linear) diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index 90c318af..baa8767e 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -8,7 +8,7 @@ import torch.nn as nn import sinabs -from dvs_layer import DVSLayer +from .dvs_layer import DVSLayer from .connectivity_specs import ( LAYER_TYPES_WITH_MULTIPLE_INPUTS, diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index 5ff120fb..ed1a26d2 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -18,7 +18,7 @@ UnmatchedPoolingEdges, ) from .utils import Edge -from dvs_layer import DVSLayer +from .dvs_layer import DVSLayer def get_dvs_node_from_mapper(dcnnl_map: Dict) -> Optional[Dict]: """ Returns the information dict associated with the `DVSLayer` instance within `dcnnl_map`. From cf5be866355294a0e1b0632730c83c019cd227b7 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Tue, 29 Oct 2024 11:21:53 +0100 Subject: [PATCH 252/379] WIP DVS - DVS node not given fixed misplaced arg to graph extractor --- sinabs/backend/dynapcnn/dynapcnn_network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index 52f18dd1..d195c6de 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -67,7 +67,7 @@ def __init__( batch_size = sinabs.utils.get_smallest_compatible_time_dimension(snn) # computational graph from original PyTorch module. self._graph_extractor = GraphExtractor( - snn, torch.randn((batch_size, *self.input_shape), self.dvs_input) + snn, torch.randn((batch_size, *self.input_shape)), self.dvs_input ) # needs the batch dimension. # Remove nodes of ignored classes (including merge nodes) From acba9165fa06b020a39561db5c1d9ec233216de0 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Tue, 29 Oct 2024 11:28:52 +0100 Subject: [PATCH 253/379] WIP DVS - DVS node not given checking 'dvs-weight' edge exists before dict access --- .../backend/dynapcnn/sinabs_edges_handler.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index ed1a26d2..4f14588f 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -67,7 +67,6 @@ def collect_dynapcnn_layer_info( 'value' is a dictionary, with keys 'conv', 'neuron', and 'destinations', containing corresponding node ids and modules required to build the layer """ - # TODO: Handle DVS layer # Sort edges by edge type (type of layers they connect) edges_by_type: Dict[str, Set[Edge]] = sort_edges_by_type( @@ -103,15 +102,17 @@ def collect_dynapcnn_layer_info( entry_nodes, ) + # TODO - make 'dvs-weight' an empty set when calling sort_edges_by_type() to remove the need for the 'if' statement bellow. # Process all dvs->weight edges connecting the DVS camera to a unique dynapcnn layer. - while edges_by_type["dvs-weight"]: - edge = edges_by_type["dvs-weight"].pop() - add_or_update_dvs_to_entry( - edge, - dynapcnn_layer_info, - indx_2_module_map, - node_2_layer_map, - ) + if "dvs-weight" in edges_by_type: + while edges_by_type["dvs-weight"]: + edge = edges_by_type["dvs-weight"].pop() + add_or_update_dvs_to_entry( + edge, + dynapcnn_layer_info, + indx_2_module_map, + node_2_layer_map, + ) # TODO - handle dvs->pooling connections. From d7f6be31ae3c2842074ebc9732f44f8e44275553 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Tue, 29 Oct 2024 11:33:39 +0100 Subject: [PATCH 254/379] WIP DVS - DVS node not given giving access to DVSLayer --- sinabs/backend/dynapcnn/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sinabs/backend/dynapcnn/__init__.py b/sinabs/backend/dynapcnn/__init__.py index 11ab0fd9..1231eb60 100644 --- a/sinabs/backend/dynapcnn/__init__.py +++ b/sinabs/backend/dynapcnn/__init__.py @@ -2,3 +2,4 @@ from .dynapcnn_network import DynapcnnCompatibleNetwork, DynapcnnNetwork from .dynapcnnnetwork_module import DynapcnnNetworkModule from .dynapcnn_visualizer import DynapcnnVisualizer +from .dvs_layer import DVSLayer From 5b3edb26ae46ab35ace68bce179716881c7bf998 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Tue, 29 Oct 2024 13:48:26 +0100 Subject: [PATCH 255/379] WIP DVS - DVS node not given NIR extracts weird edges for DVSLayer - fixing it by modifying GraphExtract.edges in-place. --- .../backend/dynapcnn/nir_graph_extractor.py | 13 ++++- .../backend/dynapcnn/sinabs_edges_handler.py | 50 ++++++++++++++++++- 2 files changed, 60 insertions(+), 3 deletions(-) diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index baa8767e..fe984855 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -17,7 +17,7 @@ from .dynapcnn_layer_utils import construct_dynapcnnlayers_from_mapper from .dynapcnnnetwork_module import DynapcnnNetworkModule from .exceptions import InvalidGraphStructure, InvalidModelWithDVSSetup -from .sinabs_edges_handler import collect_dynapcnn_layer_info, get_dvs_node_from_mapper +from .sinabs_edges_handler import collect_dynapcnn_layer_info, get_dvs_node_from_mapper, fix_dvs_module_edges from .utils import Edge, topological_sorting @@ -59,8 +59,10 @@ def __init__(self, spiking_model: nn.Module, dummy_input: torch.tensor, dvs_inpu # Map node names to indices self._name_2_indx_map = self._get_name_2_indx_map(nir_graph) + # Extract edges list from graph self._edges = self._get_edges_from_nir(nir_graph, self._name_2_indx_map) + # Store the associated `nn.Module` (layer) of each node. self._indx_2_module_map = self._get_named_modules(spiking_model) @@ -73,6 +75,15 @@ def __init__(self, spiking_model: nn.Module, dummy_input: torch.tensor, dvs_inpu # Determine entry points to graph self._entry_nodes = self._get_entry_nodes(self._edges) + print('----------------------------------------') + print(self._edges) + for key, val in self._name_2_indx_map.items(): + print(key, val) + print('----------------------------------------') + + # Consolidates the edges associated with a DVSLayer instance. + fix_dvs_module_edges(edges=self._edges, indx_2_module_map=self._indx_2_module_map) + # Verify that graph is compatible self.verify_graph_integrity() diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index 4f14588f..fb16273a 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -10,7 +10,7 @@ from torch import Size, nn -from .connectivity_specs import VALID_SINABS_EDGE_TYPES +from .connectivity_specs import VALID_SINABS_EDGE_TYPES, DVS from .exceptions import ( InvalidEdge, InvalidGraphStructure, @@ -19,6 +19,10 @@ ) from .utils import Edge from .dvs_layer import DVSLayer +from .crop2d import Crop2d +from .flipdims import FlipDims + +from sinabs.layers import SumPool2d def get_dvs_node_from_mapper(dcnnl_map: Dict) -> Optional[Dict]: """ Returns the information dict associated with the `DVSLayer` instance within `dcnnl_map`. @@ -34,9 +38,51 @@ def get_dvs_node_from_mapper(dcnnl_map: Dict) -> Optional[Dict]: for layer_index, layer_info in dcnnl_map.items(): if 'dvs_layer' in layer_info: assert layer_info['dvs_layer'] - return destination_map[layer_index] + return layer_info return None +def fix_dvs_module_edges(edges: Set[Edge], indx_2_module_map: Dict[int, nn.Module]) -> None: + """ Modifies `edges` in-place to re-structure the edges related witht the DVSLayer instance. Currently, this is also + removing a self-recurrent node with edge `(FlipDims, FlipDims)` that is created when forwarding via DVSLayer. + + The DVSLayer's forward method feeds that in the sequence `DVS.pool -> DVS.crop -> DVS.flip`, so + we want to find four nodes in `edges` (one for each of these in the sequence plus the node representing + the DVSLayer itself). + + The 'fix_' is to imply there's something odd with the extracted adges for the forward pass implemented by + the DVSLayer. For now this function is fixing these edges to have them representing the information flow through + this layer as **it should be**. + + Parameters + ---------- + - edges (set): tuples describing the connections between layers in `spiking_model`. + - indx_2_module_map (dict): the mapping between a node (`key` as an `int`) and its module (`value` as a `nn.Module`). + """ + + # spot nodes (ie, modules) used in a DVSLayer instance's forward pass (including the DVSLayer node itself). + dvslayer_nodes = { + index for index, module in indx_2_module_map.items() + if any(isinstance(module, dvs_node) for dvs_node in DVS) + } + + # TODO - a `SumPool2d` is also a node that's used inside a DVSLayer instance. In what follows we try to find it + # by looking for pooling nodes that appear in a (pool, crop) edge - the assumption being that if the pooling is + # inputing into a crop layer than the pool is inside the DVSLayer instance. It feels like a hacky way to do it + # so we should revise this. + dvslayer_nodes.update({ + edge[0] for edge in edges + if isinstance(indx_2_module_map[edge[0]], SumPool2d) and isinstance(indx_2_module_map[edge[1]], Crop2d) + }) + + # NIR is extracting and edge (FlipDims, FlipDims) from the DVSLayer: remove self-recurrent nodes from the graph. + edges = {(src, tgt) for (src, tgt) in edges if not (src == tgt and isinstance(indx_2_module_map[src], FlipDims))} + + # DVS edges we want: (dvs, dvs_pool), (dvs_pool, dvs_crop), (dvs_crop, dvs_flip) + + + print('>>> ', dvslayer_nodes) + print('>>> ', edges) + def collect_dynapcnn_layer_info( indx_2_module_map: Dict[int, nn.Module], edges: Set[Edge], From c945a411b801d9950fee4c8f9f3b3d2d6b181d52 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Tue, 29 Oct 2024 15:47:20 +0100 Subject: [PATCH 256/379] WIP DVS - DVS node not given fix_dvs_module_edges() modifies in-place the edges extracted by NIR for the DVSLayer instance - DVSLayer points to its destination and its internal nodes (crop, pool, flip) are removed from the graphs edges --- .../backend/dynapcnn/sinabs_edges_handler.py | 38 +++++++++++++------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index fb16273a..3b73c877 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -42,16 +42,17 @@ def get_dvs_node_from_mapper(dcnnl_map: Dict) -> Optional[Dict]: return None def fix_dvs_module_edges(edges: Set[Edge], indx_2_module_map: Dict[int, nn.Module]) -> None: - """ Modifies `edges` in-place to re-structure the edges related witht the DVSLayer instance. Currently, this is also - removing a self-recurrent node with edge `(FlipDims, FlipDims)` that is created when forwarding via DVSLayer. + """ Modifies `edges` in-place to re-structure the edges related witht the DVSLayer instance. The DVSLayer's forward method + feeds that in the sequence `DVS.pool -> DVS.crop -> DVS.flip`, so we remove edges involving these nodes that are internaly + implementend in the DVSLayer instance from the graph and point the node of DVSLayer directly to the layer/module it is suppoed + to forward its data to. - The DVSLayer's forward method feeds that in the sequence `DVS.pool -> DVS.crop -> DVS.flip`, so - we want to find four nodes in `edges` (one for each of these in the sequence plus the node representing - the DVSLayer itself). + Currently, this is also removing a self-recurrent node with edge `(FlipDims, FlipDims)` that is + created when forwarding via DVSLayer. The 'fix_' is to imply there's something odd with the extracted adges for the forward pass implemented by the DVSLayer. For now this function is fixing these edges to have them representing the information flow through - this layer as **it should be**. + this layer as **it should be** but the graph tracing of NIR should be looked into to solve the root problem. Parameters ---------- @@ -61,7 +62,7 @@ def fix_dvs_module_edges(edges: Set[Edge], indx_2_module_map: Dict[int, nn.Modul # spot nodes (ie, modules) used in a DVSLayer instance's forward pass (including the DVSLayer node itself). dvslayer_nodes = { - index for index, module in indx_2_module_map.items() + index: module for index, module in indx_2_module_map.items() if any(isinstance(module, dvs_node) for dvs_node in DVS) } @@ -70,18 +71,31 @@ def fix_dvs_module_edges(edges: Set[Edge], indx_2_module_map: Dict[int, nn.Modul # inputing into a crop layer than the pool is inside the DVSLayer instance. It feels like a hacky way to do it # so we should revise this. dvslayer_nodes.update({ - edge[0] for edge in edges + edge[0]: indx_2_module_map[edge[0]] for edge in edges if isinstance(indx_2_module_map[edge[0]], SumPool2d) and isinstance(indx_2_module_map[edge[1]], Crop2d) }) # NIR is extracting and edge (FlipDims, FlipDims) from the DVSLayer: remove self-recurrent nodes from the graph. edges = {(src, tgt) for (src, tgt) in edges if not (src == tgt and isinstance(indx_2_module_map[src], FlipDims))} - # DVS edges we want: (dvs, dvs_pool), (dvs_pool, dvs_crop), (dvs_crop, dvs_flip) - + # Since NIR is not extracting the edges for the DVSLayer correctly, remove all edges involving the DVS. + edges = {(src, tgt) for (src, tgt) in edges if src not in dvslayer_nodes and tgt not in dvslayer_nodes} + + # Get what the entry nodes should be without the DVS - these are the ones the DVS should point to. + all_sources, all_targets = zip(*edges) + entry_nodes = set(all_sources) - set(all_targets) - print('>>> ', dvslayer_nodes) - print('>>> ', edges) + # Get node's indexes based on the module type - just for validation. + dvs_node = [key for key, value in dvslayer_nodes.items() if isinstance(value, DVSLayer)] + dvs_pool_node = [key for key, value in dvslayer_nodes.items() if isinstance(value, SumPool2d)] + dvs_crop_node = [key for key, value in dvslayer_nodes.items() if isinstance(value, Crop2d)] + dvs_flip_node = [key for key, value in dvslayer_nodes.items() if isinstance(value, FlipDims)] + + if any(len(node) > 1 for node in [dvs_node, dvs_pool_node, dvs_crop_node, dvs_flip_node]): + raise ValueError(f'Internal DVS nodes should be single instances but multiple have been found: dvs_node: {len(dvs_node)} dvs_pool_node: {len(dvs_pool_node)} dvs_crop_node: {len(dvs_crop_node)} dvs_flip_node: {len(dvs_flip_node)}') + + # dvs_pool, dvs_crop and dvs_flip are internal nodes of the DVSLayer: we only want an edge from 'dvs' node to the entry points of the network. + edges.update({(dvs_node[-1], node) for node in entry_nodes}) def collect_dynapcnn_layer_info( indx_2_module_map: Dict[int, nn.Module], From 22b69c65f6f6f9d2dad11c61af6985c61a126522 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Tue, 29 Oct 2024 21:29:40 +0100 Subject: [PATCH 257/379] WIP - DVS node not given construct_destination_map() has func. handling destination mapping of DVSLayer inside its call now --- sinabs/backend/dynapcnn/dynapcnn_layer_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py b/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py index 9ea208d7..18e60a93 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py @@ -33,9 +33,6 @@ def construct_dynapcnnlayers_from_mapper( destination_map = construct_destination_map(dcnnl_map) - # update mapper if a DVS layer exists. - update_destination_map_with_dvs(dcnnl_map, destination_map) - entry_points = collect_entry_points(dcnnl_map) return dynapcnn_layers, destination_map, entry_points @@ -284,6 +281,9 @@ def construct_destination_map(dcnnl_map: Dict[int, Dict]) -> Dict[int, List[int] destination_indices.append(dest_idx) destination_map[layer_index] = destination_indices + # update mapper if a DVS layer exists. + update_destination_map_with_dvs(dcnnl_map, destination_map) + return destination_map From beade182c2bec7f2bbcf07515e23faa8e76ccb0c Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Tue, 29 Oct 2024 21:33:17 +0100 Subject: [PATCH 258/379] WIP - DVS node not given Crop2d and FlipDims do (should) not appear as an indepent node --- sinabs/backend/dynapcnn/connectivity_specs.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/sinabs/backend/dynapcnn/connectivity_specs.py b/sinabs/backend/dynapcnn/connectivity_specs.py index 5eda39aa..13e17911 100644 --- a/sinabs/backend/dynapcnn/connectivity_specs.py +++ b/sinabs/backend/dynapcnn/connectivity_specs.py @@ -8,11 +8,13 @@ import sinabs.layers as sl from .dvs_layer import DVSLayer +from .crop2d import Crop2d +from .flipdims import FlipDims Pooling = (sl.SumPool2d, nn.AvgPool2d) Weight = (nn.Conv2d, nn.Linear) Neuron = (sl.IAFSqueeze,) -Dvs = (DVSLayer,) +DVS = (DVSLayer, Crop2d, FlipDims) # @TODO - need to list other edge cases involving DVS layer (for now only dvs-weight and dvs-pooling). VALID_SINABS_EDGE_TYPES_ABSTRACT = { @@ -27,9 +29,9 @@ # Pooling can be followed by weight layer of next core (Pooling, Weight): "pooling-weight", # Dvs can be followed by weight layer of next core - (Dvs, Weight): "dvs-weight", + (DVSLayer, Weight): "dvs-weight", # Dvs can be followed by pooling layers - (Dvs, Pooling): "dvs-pooling", + (DVSLayer, Pooling): "dvs-pooling", } # Unpack dict @@ -44,4 +46,4 @@ LAYER_TYPES_WITH_MULTIPLE_INPUTS = Union[sl.Merge] # Neuron and pooling layers can have their output sent to multiple cores -LAYER_TYPES_WITH_MULTIPLE_OUTPUTS = Union[(*Neuron, *Pooling, *Dvs)] +LAYER_TYPES_WITH_MULTIPLE_OUTPUTS = Union[(*Neuron, *Pooling, DVSLayer)] From 89200d95285cb8beafee2fb78989d7fa8a0fcf79 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Tue, 29 Oct 2024 22:09:15 +0100 Subject: [PATCH 259/379] WIP - DVS node not given constructor of GraphExtractor executing without errors --- sinabs/backend/dynapcnn/connectivity_specs.py | 8 +++--- .../backend/dynapcnn/nir_graph_extractor.py | 25 +++++++++---------- .../backend/dynapcnn/sinabs_edges_handler.py | 23 ++++++++++++----- 3 files changed, 33 insertions(+), 23 deletions(-) diff --git a/sinabs/backend/dynapcnn/connectivity_specs.py b/sinabs/backend/dynapcnn/connectivity_specs.py index 13e17911..c361f240 100644 --- a/sinabs/backend/dynapcnn/connectivity_specs.py +++ b/sinabs/backend/dynapcnn/connectivity_specs.py @@ -14,7 +14,7 @@ Pooling = (sl.SumPool2d, nn.AvgPool2d) Weight = (nn.Conv2d, nn.Linear) Neuron = (sl.IAFSqueeze,) -DVS = (DVSLayer, Crop2d, FlipDims) +DVS = (DVSLayer, ) # @TODO - need to list other edge cases involving DVS layer (for now only dvs-weight and dvs-pooling). VALID_SINABS_EDGE_TYPES_ABSTRACT = { @@ -29,9 +29,9 @@ # Pooling can be followed by weight layer of next core (Pooling, Weight): "pooling-weight", # Dvs can be followed by weight layer of next core - (DVSLayer, Weight): "dvs-weight", + (DVS, Weight): "dvs-weight", # Dvs can be followed by pooling layers - (DVSLayer, Pooling): "dvs-pooling", + (DVS, Pooling): "dvs-pooling", } # Unpack dict @@ -46,4 +46,4 @@ LAYER_TYPES_WITH_MULTIPLE_INPUTS = Union[sl.Merge] # Neuron and pooling layers can have their output sent to multiple cores -LAYER_TYPES_WITH_MULTIPLE_OUTPUTS = Union[(*Neuron, *Pooling, DVSLayer)] +LAYER_TYPES_WITH_MULTIPLE_OUTPUTS = Union[(*Neuron, *Pooling, *DVS)] diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index fe984855..0a80bec0 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -66,23 +66,18 @@ def __init__(self, spiking_model: nn.Module, dummy_input: torch.tensor, dvs_inpu # Store the associated `nn.Module` (layer) of each node. self._indx_2_module_map = self._get_named_modules(spiking_model) - # True if `dvs_input == True` and `spiking_model` does not start with DVS layer. + # Determine entry points to graph + self._entry_nodes = self._get_entry_nodes(self._edges) + + # If DVS camera is wanted but `spiking_model` does not start with DVS layer. if self._need_dvs_node(spiking_model, dvs_input): # input shape for `DVSLayer` instance that will be the module of the node 'dvs'. _, _, height, width = dummy_input.shape self._add_dvs_node(dvs_input_shape=(height, width)) + # Consolidates the edges associated with a DVSLayer instance (ie., fix NIR edges extraction when DVS is a node in the graph). + fix_dvs_module_edges(edges=self._edges, indx_2_module_map=self._indx_2_module_map) - # Determine entry points to graph - self._entry_nodes = self._get_entry_nodes(self._edges) - - print('----------------------------------------') - print(self._edges) - for key, val in self._name_2_indx_map.items(): - print(key, val) - print('----------------------------------------') - - # Consolidates the edges associated with a DVSLayer instance. - fix_dvs_module_edges(edges=self._edges, indx_2_module_map=self._indx_2_module_map) + self._entry_nodes # Verify that graph is compatible self.verify_graph_integrity() @@ -247,7 +242,9 @@ def verify_graph_integrity(self): def _add_dvs_node(self, dvs_input_shape: Tuple[int, int]) -> None: """ In-place modification of `self._name_2_indx_map`, `self._indx_2_module_map`, and `self._edges` to accomodate the - creation of an extra node in the graph representing the DVS camera of the chip. + creation of an extra node in the graph representing the DVS camera of the chip. The DVSLayer node will point to every + other node that is up to this point an entry node of the original graph, so `self._entry_nodes` is modified in-place + to have only one entry: the index of the DVS node. Parameters ---------- @@ -263,6 +260,8 @@ def _add_dvs_node(self, dvs_input_shape: Tuple[int, int]) -> None: self._indx_2_module_map[self._name_2_indx_map['dvs']] = DVSLayer(input_shape=dvs_input_shape) # set DVS node as input to each entry node of the graph. self._edges.update({(self._name_2_indx_map['dvs'], entry_node) for entry_node in self._entry_nodes}) + # DVSLayer node becomes the only entrypoint of the graph. + self._entry_nodes = {self._name_2_indx_map['dvs']} def _need_dvs_node(self, model: nn.Module, dvs_input: bool) -> bool: """ Returns whether or not a node will need to be added to represent a `DVSLayer` instance. A new node will have diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index 3b73c877..f524f784 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -10,7 +10,7 @@ from torch import Size, nn -from .connectivity_specs import VALID_SINABS_EDGE_TYPES, DVS +from .connectivity_specs import VALID_SINABS_EDGE_TYPES from .exceptions import ( InvalidEdge, InvalidGraphStructure, @@ -46,26 +46,32 @@ def fix_dvs_module_edges(edges: Set[Edge], indx_2_module_map: Dict[int, nn.Modul feeds that in the sequence `DVS.pool -> DVS.crop -> DVS.flip`, so we remove edges involving these nodes that are internaly implementend in the DVSLayer instance from the graph and point the node of DVSLayer directly to the layer/module it is suppoed to forward its data to. + + Modifies `indx_2_module_map` in-place to remove the nodes (Crop2d, FlipDims and DVSLayer's pooling) defined within the DVSLayer + instance since those are not independent nodes of the final graph. Currently, this is also removing a self-recurrent node with edge `(FlipDims, FlipDims)` that is created when forwarding via DVSLayer. - The 'fix_' is to imply there's something odd with the extracted adges for the forward pass implemented by - the DVSLayer. For now this function is fixing these edges to have them representing the information flow through - this layer as **it should be** but the graph tracing of NIR should be looked into to solve the root problem. - Parameters ---------- - edges (set): tuples describing the connections between layers in `spiking_model`. - indx_2_module_map (dict): the mapping between a node (`key` as an `int`) and its module (`value` as a `nn.Module`). """ + # TODO - the 'fix_' is to imply there's something odd with the extracted adges for the forward pass implemented by + # the DVSLayer. For now this function is fixing these edges to have them representing the information flow through + # this layer as **it should be** but the graph tracing of NIR should be looked into to solve the root problem. # spot nodes (ie, modules) used in a DVSLayer instance's forward pass (including the DVSLayer node itself). dvslayer_nodes = { index: module for index, module in indx_2_module_map.items() - if any(isinstance(module, dvs_node) for dvs_node in DVS) + if any(isinstance(module, dvs_node) for dvs_node in (DVSLayer, Crop2d, FlipDims)) } + if len(dvslayer_nodes) == 1: + # No module within the DVSLayer instance appears as an independent node - nothing to do here. + return + # TODO - a `SumPool2d` is also a node that's used inside a DVSLayer instance. In what follows we try to find it # by looking for pooling nodes that appear in a (pool, crop) edge - the assumption being that if the pooling is # inputing into a crop layer than the pool is inside the DVSLayer instance. It feels like a hacky way to do it @@ -94,6 +100,11 @@ def fix_dvs_module_edges(edges: Set[Edge], indx_2_module_map: Dict[int, nn.Modul if any(len(node) > 1 for node in [dvs_node, dvs_pool_node, dvs_crop_node, dvs_flip_node]): raise ValueError(f'Internal DVS nodes should be single instances but multiple have been found: dvs_node: {len(dvs_node)} dvs_pool_node: {len(dvs_pool_node)} dvs_crop_node: {len(dvs_crop_node)} dvs_flip_node: {len(dvs_flip_node)}') + # Remove dvs_pool, dvs_crop and dvs_flip nodes from `indx_2_module_map` (these operate within the DVS, not as independent nodes of the final graph). + indx_2_module_map.pop(dvs_pool_node[-1]) + indx_2_module_map.pop(dvs_crop_node[-1]) + indx_2_module_map.pop(dvs_flip_node[-1]) + # dvs_pool, dvs_crop and dvs_flip are internal nodes of the DVSLayer: we only want an edge from 'dvs' node to the entry points of the network. edges.update({(dvs_node[-1], node) for node in entry_nodes}) From 94a67c423b407a3eef95157321dd6577b748acc9 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Tue, 29 Oct 2024 22:12:41 +0100 Subject: [PATCH 260/379] WIP - DVS node not given minor edit + updated in-line documentation --- sinabs/backend/dynapcnn/nir_graph_extractor.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index 0a80bec0..209f3c2c 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -74,10 +74,11 @@ def __init__(self, spiking_model: nn.Module, dummy_input: torch.tensor, dvs_inpu # input shape for `DVSLayer` instance that will be the module of the node 'dvs'. _, _, height, width = dummy_input.shape self._add_dvs_node(dvs_input_shape=(height, width)) - # Consolidates the edges associated with a DVSLayer instance (ie., fix NIR edges extraction when DVS is a node in the graph). - fix_dvs_module_edges(edges=self._edges, indx_2_module_map=self._indx_2_module_map) - self._entry_nodes + # TODO - the calll bellow should be done outside this 'if' cuz the problem only + # appears when the DVSLayer is given as the first layer of `spiking_model`. + # Fix NIR edges extraction when DVS is a node in the graph. + fix_dvs_module_edges(edges=self._edges, indx_2_module_map=self._indx_2_module_map) # Verify that graph is compatible self.verify_graph_integrity() From 3a5e419e5ac2fdfc4a4014a2362bc678542f5d72 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Tue, 29 Oct 2024 22:29:16 +0100 Subject: [PATCH 261/379] WIP - DVS node not given finalize_dcnnl_map() only operates on DynapcnnLayer instances (skips DVSLayer) --- sinabs/backend/dynapcnn/dynapcnn_layer_utils.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py b/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py index 18e60a93..faeef269 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py @@ -54,14 +54,15 @@ def finalize_dcnnl_map(dcnnl_map: Dict, rescale_fn: Optional[Callable] = None) - """ # Consolidate pooling information for each destination for layer_info in dcnnl_map.values(): - consolidate_layer_pooling(layer_info, dcnnl_map) + if 'dvs_layer' not in layer_info: # only called for `DynapcnnLayer` instances (skip DVS layer). + consolidate_layer_pooling(layer_info, dcnnl_map) for layer_info in dcnnl_map.values(): - # Consolidate scale factors - consolidate_layer_scaling(layer_info, rescale_fn) - # Handle input dimensions - determine_layer_input_shape(layer_info) - + if 'dvs_layer' not in layer_info: # only called for `DynapcnnLayer` instances (skip DVS layer). + # Consolidate scale factors + consolidate_layer_scaling(layer_info, rescale_fn) + # Handle input dimensions + determine_layer_input_shape(layer_info) def consolidate_layer_pooling(layer_info: Dict, dcnnl_map: Dict): """Consolidate pooling information for individual layer From 430af988bef9ff8787d53bc50b0f81474384a200 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Tue, 29 Oct 2024 22:39:46 +0100 Subject: [PATCH 262/379] DONE - DVS node not given constructor of DynapcnnNetwork executing wihtout errors --- .../backend/dynapcnn/dynapcnn_layer_utils.py | 21 ++++++++++--------- .../backend/dynapcnn/sinabs_edges_handler.py | 4 ++++ 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py b/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py index faeef269..491093b7 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py @@ -271,16 +271,17 @@ def construct_destination_map(dcnnl_map: Dict[int, Dict]) -> Dict[int, List[int] """ destination_map = dict() for layer_index, layer_info in dcnnl_map.items(): - destination_indices = [] - none_counter = 0 - for dest in layer_info["destinations"]: - if (dest_idx := dest["destination_layer"]) is None: - # For `None` destinations use unique negative index - none_counter += 1 - destination_indices.append(-none_counter) - else: - destination_indices.append(dest_idx) - destination_map[layer_index] = destination_indices + if 'dvs_layer' not in layer_info: # only called for `DynapcnnLayer` instances (skip DVS layer). + destination_indices = [] + none_counter = 0 + for dest in layer_info["destinations"]: + if (dest_idx := dest["destination_layer"]) is None: + # For `None` destinations use unique negative index + none_counter += 1 + destination_indices.append(-none_counter) + else: + destination_indices.append(dest_idx) + destination_map[layer_index] = destination_indices # update mapper if a DVS layer exists. update_destination_map_with_dvs(dcnnl_map, destination_map) diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index f524f784..018dcf58 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -183,6 +183,7 @@ def collect_dynapcnn_layer_info( dynapcnn_layer_info, indx_2_module_map, node_2_layer_map, + nodes_io_shapes, ) # TODO - handle dvs->pooling connections. @@ -405,6 +406,7 @@ def add_or_update_dvs_to_entry( dynapcnn_layer_info: Dict[int, Dict[int, Dict]], indx_2_module_map: Dict[int, nn.Module], node_2_layer_map: Dict[int, int], + nodes_io_shapes: Dict[int, Dict[str, Tuple[Size, Size]]], ) -> None: """ Initiate or update dict to hold information for a DVS Layer configuration based on a "dvs-weight" edges. Change `dynapcnn_layer_info` in-place. If a entry for the DVS node exists the function will add a new entry @@ -420,6 +422,7 @@ def add_or_update_dvs_to_entry( indx_2_module_map (dict): Maps node IDs of the graph as `key` to their associated module as `value` node_2_layer_map (dict): Maps each node ID to the ID of the layer it is assigned to. Will be updated in-place. + nodes_io_shapes (dict): Map from node ID to dict containing node's in- and output shapes """ assert isinstance(indx_2_module_map[edge[0]], DVSLayer), f'Source node in edge {edge} is of type {type(DVSLayer)} (it should be a DVSLayer instance).' @@ -432,6 +435,7 @@ def add_or_update_dvs_to_entry( # Init. entry for a DVS layer using its configuration dict. dynapcnn_layer_info[layer_id] = { + "is_entry_node": True, "dvs_layer": True, "node_id": edge[0], # TODO - GraphTracer not populating I/O shape for DVS yet. From 2988ce228d2269640a2d522d7eac1672089543ea Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Wed, 30 Oct 2024 12:54:36 +0100 Subject: [PATCH 263/379] DONE - DVS node given constructors of GraphExtractor and DynapcnnNetwork executing without errors --- .../backend/dynapcnn/nir_graph_extractor.py | 14 ++++-- .../backend/dynapcnn/sinabs_edges_handler.py | 47 ++++++++++++------- 2 files changed, 38 insertions(+), 23 deletions(-) diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index 209f3c2c..4f4b0bcb 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -74,11 +74,15 @@ def __init__(self, spiking_model: nn.Module, dummy_input: torch.tensor, dvs_inpu # input shape for `DVSLayer` instance that will be the module of the node 'dvs'. _, _, height, width = dummy_input.shape self._add_dvs_node(dvs_input_shape=(height, width)) - - # TODO - the calll bellow should be done outside this 'if' cuz the problem only - # appears when the DVSLayer is given as the first layer of `spiking_model`. - # Fix NIR edges extraction when DVS is a node in the graph. - fix_dvs_module_edges(edges=self._edges, indx_2_module_map=self._indx_2_module_map) + + # Check for the need of fixing NIR edges extraction when DVS is a node in the graph. If DVS + # is used its node becomes the only entry node in the graph. + fix_dvs_module_edges( + edges=self._edges, + indx_2_module_map=self._indx_2_module_map, + name_2_indx_map=self._name_2_indx_map, + entry_nodes=self._entry_nodes, + ) # Verify that graph is compatible self.verify_graph_integrity() diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index 018dcf58..ede6ab23 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -41,22 +41,25 @@ def get_dvs_node_from_mapper(dcnnl_map: Dict) -> Optional[Dict]: return layer_info return None -def fix_dvs_module_edges(edges: Set[Edge], indx_2_module_map: Dict[int, nn.Module]) -> None: - """ Modifies `edges` in-place to re-structure the edges related witht the DVSLayer instance. The DVSLayer's forward method - feeds that in the sequence `DVS.pool -> DVS.crop -> DVS.flip`, so we remove edges involving these nodes that are internaly - implementend in the DVSLayer instance from the graph and point the node of DVSLayer directly to the layer/module it is suppoed - to forward its data to. - - Modifies `indx_2_module_map` in-place to remove the nodes (Crop2d, FlipDims and DVSLayer's pooling) defined within the DVSLayer - instance since those are not independent nodes of the final graph. +def fix_dvs_module_edges(edges: Set[Edge], indx_2_module_map: Dict[int, nn.Module], name_2_indx_map: Dict[str, int], entry_nodes: Set[Edge]) -> None: + """ All arguments are modified in-place to fix wrong node extractions from NIRtorch when a DVSLayer istance is the first layer in the network. - Currently, this is also removing a self-recurrent node with edge `(FlipDims, FlipDims)` that is - created when forwarding via DVSLayer. + Modifies `edges` to re-structure the edges related witht the DVSLayer instance. The DVSLayer's forward method feeds data in the + sequence 'DVS -> DVS.pool -> DVS.crop -> DVS.flip', so we remove edges involving these nodes (that are internaly implementend in + the DVSLayer) from the graph and point the node of DVSLayer to the node where it should send its output to. This is also removes + a self-recurrent node with edge '(FlipDims, FlipDims)' that is wrongly extracted. + + Modifies `indx_2_module_map` and `name_2_indx_map` to remove the internal DVSLayer nodes (Crop2d, FlipDims and DVSLayer's pooling) since + these should not be independent nodes in the graph. + + Modifies `entry_nodes` such that the DVSLayer becomes the only entry node of the graph. Parameters ---------- - edges (set): tuples describing the connections between layers in `spiking_model`. - indx_2_module_map (dict): the mapping between a node (`key` as an `int`) and its module (`value` as a `nn.Module`). + - name_2_indx_map (dict): Map from node names to unique indices. + - entry_nodes (set): IDs of nodes acting as entry points for the network (i.e., receiving external input). """ # TODO - the 'fix_' is to imply there's something odd with the extracted adges for the forward pass implemented by # the DVSLayer. For now this function is fixing these edges to have them representing the information flow through @@ -82,14 +85,12 @@ def fix_dvs_module_edges(edges: Set[Edge], indx_2_module_map: Dict[int, nn.Modul }) # NIR is extracting and edge (FlipDims, FlipDims) from the DVSLayer: remove self-recurrent nodes from the graph. - edges = {(src, tgt) for (src, tgt) in edges if not (src == tgt and isinstance(indx_2_module_map[src], FlipDims))} + for edge in [(src, tgt) for (src, tgt) in edges if (src == tgt and isinstance(indx_2_module_map[src], FlipDims))]: + edges.remove(edge) # Since NIR is not extracting the edges for the DVSLayer correctly, remove all edges involving the DVS. - edges = {(src, tgt) for (src, tgt) in edges if src not in dvslayer_nodes and tgt not in dvslayer_nodes} - - # Get what the entry nodes should be without the DVS - these are the ones the DVS should point to. - all_sources, all_targets = zip(*edges) - entry_nodes = set(all_sources) - set(all_targets) + for edge in [(src, tgt) for (src, tgt) in edges if (src in dvslayer_nodes or tgt in dvslayer_nodes)]: + edges.remove(edge) # Get node's indexes based on the module type - just for validation. dvs_node = [key for key, value in dvslayer_nodes.items() if isinstance(value, DVSLayer)] @@ -104,9 +105,19 @@ def fix_dvs_module_edges(edges: Set[Edge], indx_2_module_map: Dict[int, nn.Modul indx_2_module_map.pop(dvs_pool_node[-1]) indx_2_module_map.pop(dvs_crop_node[-1]) indx_2_module_map.pop(dvs_flip_node[-1]) + + # Remove internal DVS modeules from name/index map. + for name in [name for name, index in name_2_indx_map.items() if index in [dvs_pool_node[-1], dvs_crop_node[-1], dvs_flip_node[-1]]]: + name_2_indx_map.pop(name) - # dvs_pool, dvs_crop and dvs_flip are internal nodes of the DVSLayer: we only want an edge from 'dvs' node to the entry points of the network. - edges.update({(dvs_node[-1], node) for node in entry_nodes}) + # Add edges from 'dvs' node to the entry point of the graph. + all_sources, all_targets = zip(*edges) + local_entry_nodes = set(all_sources) - set(all_targets) + edges.update({(dvs_node[-1], node) for node in local_entry_nodes}) + + # DVS becomes the only entry node of the graph. + entry_nodes.clear() + entry_nodes.add(dvs_node[-1]) def collect_dynapcnn_layer_info( indx_2_module_map: Dict[int, nn.Module], From 7b3788566ba507b47b2dd90938b9e86f509daaea Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Wed, 30 Oct 2024 13:11:30 +0100 Subject: [PATCH 264/379] WIP - SW forward with DVS sort_edges_by_type() initializes dvs-related edge types to empty set is DVS is not used + addressed some TODOs related to previous commits + updated in-line documentation --- .../backend/dynapcnn/sinabs_edges_handler.py | 33 ++++++++++++------- 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index ede6ab23..656c93d4 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -184,20 +184,20 @@ def collect_dynapcnn_layer_info( entry_nodes, ) - # TODO - make 'dvs-weight' an empty set when calling sort_edges_by_type() to remove the need for the 'if' statement bellow. # Process all dvs->weight edges connecting the DVS camera to a unique dynapcnn layer. - if "dvs-weight" in edges_by_type: - while edges_by_type["dvs-weight"]: - edge = edges_by_type["dvs-weight"].pop() - add_or_update_dvs_to_entry( - edge, - dynapcnn_layer_info, - indx_2_module_map, - node_2_layer_map, - nodes_io_shapes, - ) + while edges_by_type["dvs-weight"]: + edge = edges_by_type["dvs-weight"].pop() + add_or_update_dvs_to_entry( + edge, + dynapcnn_layer_info, + indx_2_module_map, + node_2_layer_map, + nodes_io_shapes, + ) # TODO - handle dvs->pooling connections. + while edges_by_type["dvs-pooling"]: + pass # Process all edges connecting two dynapcnn layers that do not include pooling while edges_by_type.get("neuron-weight", False): @@ -299,6 +299,14 @@ def sort_edges_by_type( else: edges_by_type[edge_type] = {edge} + # Edges involving DVS are not required so we init. them to empty set if they do not exist. + + if 'dvs-weight' not in edges_by_type: + edges_by_type['dvs-weight'] = set() + + if 'dvs-pooling' not in edges_by_type: + edges_by_type['dvs-pooling'] = set() + return edges_by_type @@ -447,9 +455,10 @@ def add_or_update_dvs_to_entry( # Init. entry for a DVS layer using its configuration dict. dynapcnn_layer_info[layer_id] = { "is_entry_node": True, + # TODO - the key bellow is what currently tells an entry in `dynapcnn_layer_info` for the DVS apart from the DynapcnnLayer + # entries (perhaps there's a better way). "dvs_layer": True, "node_id": edge[0], - # TODO - GraphTracer not populating I/O shape for DVS yet. "input_shape": nodes_io_shapes[edge[0]]["input"], "module": indx_2_module_map[edge[0]], "destinations": [node_2_layer_map[edge[1]]], From c9e5e70d5d7222ecd5660d781d27df4ff0ac1a58 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Wed, 30 Oct 2024 13:25:31 +0100 Subject: [PATCH 265/379] WIP - SW forward with DVS added some missing return type hints --- sinabs/backend/dynapcnn/dynapcnnnetwork_module.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py index 29f79f27..df632661 100644 --- a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py +++ b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py @@ -140,7 +140,7 @@ def get_exit_points(self): return exit_layers - def setup_dynapcnnlayer_graph(self, index_layers_topologically: bool = False): + def setup_dynapcnnlayer_graph(self, index_layers_topologically: bool = False) -> None: """ Set up data structures to run forward pass through dynapcnn layers Parameters @@ -157,6 +157,11 @@ def setup_dynapcnnlayer_graph(self, index_layers_topologically: bool = False): if index_layers_topologically: self.reindex_layers(self._sorted_nodes) + print('self._dynapcnnlayer_edges: ', self._dynapcnnlayer_edges) + print('self._sorted_nodes: ', self._sorted_nodes) + for key, val in self._node_source_map.items(): + print(key, val) + def get_dynapcnnlayers_edges(self) -> Set[Edge]: """Create edges representing connections between `DynapcnnLayer` instances. @@ -174,7 +179,7 @@ def get_dynapcnnlayers_edges(self) -> Set[Edge]: return dcnnl_edges - def add_entry_points_edges(self, dcnnl_edges: Set[Edge]): + def add_entry_points_edges(self, dcnnl_edges: Set[Edge]) -> None: """Add extra edges `('input', X)` to `dcnnl_edges` for layers which are entry points of the `DynapcnnNetwork`, i.e. `handler.entry_node = True`. @@ -311,7 +316,7 @@ def forward( # If there is output from multiple layers return all of them in a dict return network_outputs - def reindex_layers(self, index_order: List[int]): + def reindex_layers(self, index_order: List[int]) -> None: """ Reindex layers based on provided order Will assign new index to dynapcnn layers and update all internal From a709914c389841f900b1319c712d5790f61b9767 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Wed, 30 Oct 2024 13:37:49 +0100 Subject: [PATCH 266/379] WIP - SW forward with DVS addressed some TODOs related to previous commits + updated in-line documentation --- sinabs/backend/dynapcnn/dynapcnnnetwork_module.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py index df632661..f5f93ee1 100644 --- a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py +++ b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py @@ -78,7 +78,7 @@ def destination_map(self): @property def dynapcnn_layers(self): - # Convert string-indices to integers + # Convert string-indices to integers-indices dynapcnn_layers = {int(idx): lyr for idx, lyr in self._dynapcnn_layers.items()} # Insert DVS node if DVS was enabled. if isinstance(self.dvs_node_info, Dict): @@ -157,11 +157,6 @@ def setup_dynapcnnlayer_graph(self, index_layers_topologically: bool = False) -> if index_layers_topologically: self.reindex_layers(self._sorted_nodes) - print('self._dynapcnnlayer_edges: ', self._dynapcnnlayer_edges) - print('self._sorted_nodes: ', self._sorted_nodes) - for key, val in self._node_source_map.items(): - print(key, val) - def get_dynapcnnlayers_edges(self) -> Set[Edge]: """Create edges representing connections between `DynapcnnLayer` instances. @@ -340,23 +335,27 @@ def remap(key): # Remap all internal objects dynapcnn_layers = {str(remap(int(idx))): lyr for idx, lyr in self._dynapcnn_layers.items()} + if isinstance(self.dvs_node_info, Dict): _ = str(remap(self.dvs_node_info['layer_id'])) dynapcnn_layers[_] = self.dvs_node_info['module'] - # @TODO - update DVS node layer id in-place [THIS NEEDS VALIDATION] self.dvs_node_info['layer_id'] = int(_) self._dynapcnn_layers = nn.ModuleDict(dynapcnn_layers) self._entry_points = {remap(idx) for idx in self._entry_points} + self._destination_map = { remap(idx): [remap(dest) for dest in destinations] for idx, destinations in self._destination_map.items() } + self._dynapcnnlayer_edges = { (remap(src), remap(trg)) for (src, trg) in self._dynapcnnlayer_edges } + self._sorted_nodes = [remap(idx) for idx in self._sorted_nodes] + self._node_source_map = { remap(node): [remap(src) for src in sources] for node, sources in self._node_source_map.items() From b9cc18c5a57ececffde0ea4fa7076b05acc00ae1 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Wed, 30 Oct 2024 13:41:37 +0100 Subject: [PATCH 267/379] WIP - SW forward with DVS added missing return type hint --- sinabs/backend/dynapcnn/dynapcnnnetwork_module.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py index f5f93ee1..62c98660 100644 --- a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py +++ b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py @@ -110,7 +110,7 @@ def get_exit_layers(self) -> List[int]: if any(d < 0 for d in destinations) ] - def get_exit_points(self): + def get_exit_points(self) -> Dict[int, Dict]: """ Get details of layers that act as exit points of the network Returns From 3428c497ed198f64294e0a1f643e8a9509af401c Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Wed, 30 Oct 2024 13:58:20 +0100 Subject: [PATCH 268/379] WIP - SW forward with DVS forward checks for DVSLayer and passes its SINGLE output tensor to all of its destinations --- sinabs/backend/dynapcnn/dynapcnnnetwork_module.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py index 62c98660..009946a8 100644 --- a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py +++ b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py @@ -12,6 +12,7 @@ from .dynapcnn_layer import DynapcnnLayer from .utils import Edge, topological_sorting +from .dvs_layer import DVSLayer class DynapcnnNetworkModule(nn.Module): @@ -274,10 +275,16 @@ def forward( # Output is single tensor layers_outputs[idx_curr] = {destinations[0]: output} else: - # Output is list of tensors for different destinations - layers_outputs[idx_curr] = { - idx_dest: out for idx_dest, out in zip(destinations, output) - } + if isinstance(layer, DVSLayer): + # DVSLayer returns a single tensor (same for all its destinations). + layers_outputs[idx_curr] = { + idx_dest: output for idx_dest in destinations + } + else: + # Output is list of tensors for different destinations + layers_outputs[idx_curr] = { + idx_dest: out for idx_dest, out in zip(destinations, output) + } if return_complete: return layers_outputs From 986bdfbdad33d674d7a216fed239e5601fbbdbca Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Wed, 30 Oct 2024 14:07:39 +0100 Subject: [PATCH 269/379] DONE - SW forward with DVS forward through DynapcnnNetwork with DVSLayer works --- .../dynapcnn_network/snn_DVSLayer_given.ipynb | 269 ++++++++++++++++++ .../dynapcnn_network/snn_deployment.ipynb | 76 ++++- .../snn_need_create_DVSLayer.ipynb | 208 ++++++++++++++ sinabs/backend/dynapcnn/dvs_layer.py | 9 +- 4 files changed, 543 insertions(+), 19 deletions(-) create mode 100644 examples/dynapcnn_network/snn_DVSLayer_given.ipynb create mode 100644 examples/dynapcnn_network/snn_need_create_DVSLayer.ipynb diff --git a/examples/dynapcnn_network/snn_DVSLayer_given.ipynb b/examples/dynapcnn_network/snn_DVSLayer_given.ipynb new file mode 100644 index 00000000..986660aa --- /dev/null +++ b/examples/dynapcnn_network/snn_DVSLayer_given.ipynb @@ -0,0 +1,269 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "metadata": {} + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/samurai2077/anaconda3/envs/speck-rescnn/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import torch\n", + "import torch.nn as nn\n", + "from sinabs.backend.dynapcnn import DynapcnnNetwork\n", + "from sinabs.backend.dynapcnn import DVSLayer\n", + "from sinabs.layers import Merge, IAFSqueeze, SumPool2d\n", + "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential\n", + "import sinabs.layers as sl\n", + "\n", + "from torch.nn import CrossEntropyLoss\n", + "from torch.optim import Adam\n", + "\n", + "from tonic.datasets.nmnist import NMNIST\n", + "from tonic.transforms import ToFrame\n", + "from torch.utils.data import DataLoader\n", + "import numpy as np\n", + "from tqdm.notebook import tqdm\n", + "from statistics import mode\n", + "\n", + "device = torch.device('cpu')\n", + "torch.manual_seed(0)" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "channels = 1\n", + "height = 34\n", + "width = 34\n", + "batch_size = 1\n", + "\n", + "input_shape = (channels, height, width)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "metadata": {} + }, + "outputs": [ + { + "data": { + "text/plain": [ + "SNN(\n", + " (dvs): DVSLayer(\n", + " (pool_layer): SumPool2d(norm_type=1, kernel_size=(1, 1), stride=None, ceil_mode=False)\n", + " (crop_layer): Crop2d((0, 34), (0, 34))\n", + " (flip_layer): FlipDims()\n", + " )\n", + " (conv1): Conv2d(1, 10, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + " (iaf1): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(1.), min_v_mem=Parameter containing:\n", + " tensor(-1.), batch_size=1, num_timesteps=-1)\n", + " (pool1): AvgPool2d(kernel_size=2, stride=2, padding=0)\n", + " (conv2): Conv2d(10, 10, kernel_size=(4, 4), stride=(1, 1), bias=False)\n", + " (iaf2): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(1.), min_v_mem=Parameter containing:\n", + " tensor(-1.), batch_size=1, num_timesteps=-1)\n", + " (conv3): Conv2d(10, 1, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + " (iaf3): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(1.), min_v_mem=Parameter containing:\n", + " tensor(-1.), batch_size=1, num_timesteps=-1)\n", + " (fc1): Linear(in_features=144, out_features=200, bias=False)\n", + " (iaf4): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(1.), min_v_mem=Parameter containing:\n", + " tensor(-1.), batch_size=1, num_timesteps=-1)\n", + " (fc2): Linear(in_features=200, out_features=10, bias=False)\n", + " (iaf5): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(1.), min_v_mem=Parameter containing:\n", + " tensor(-1.), batch_size=1, num_timesteps=-1)\n", + " (flat): Flatten(start_dim=1, end_dim=-1)\n", + ")" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "class SNN(nn.Module):\n", + " def __init__(self, input_shape) -> None:\n", + " super().__init__()\n", + "\n", + " self.dvs = DVSLayer(input_shape=(input_shape[1], input_shape[2]))\n", + " # -- chip core A --\n", + " self.conv1 = nn.Conv2d(1, 10, 2, 1, bias=False)\n", + " self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential())\n", + " self.pool1 = nn.AvgPool2d(2,2)\n", + " # -- chip core B --\n", + " self.conv2 = nn.Conv2d(10, 10, 4, 1, bias=False)\n", + " self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential())\n", + " # -- chip core C --\n", + " self.conv3 = nn.Conv2d(10, 1, 2, 1, bias=False)\n", + " self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential())\n", + " # -- chip core D --\n", + " self.fc1 = nn.Linear(144, 200, bias=False)\n", + " self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential())\n", + " # -- chip core E --\n", + " self.fc2 = nn.Linear(200, 10, bias=False)\n", + " self.iaf5 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential())\n", + "\n", + " # -- layers ignored during deployment --\n", + " self.flat = nn.Flatten()\n", + "\n", + " def init_weights(self):\n", + " for name, layer in self.named_modules():\n", + " if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear):\n", + " nn.init.xavier_normal_(layer.weight.data)\n", + "\n", + " def detach_neuron_states(self):\n", + " for name, layer in self.named_modules():\n", + " if name != '':\n", + " if isinstance(layer, sl.StatefulLayer):\n", + " for name, buffer in layer.named_buffers():\n", + " buffer.detach_()\n", + "\n", + " def forward(self, x):\n", + " \n", + " dvs_out = self.dvs(x) # 0\n", + " \n", + " con1_out = self.conv1(dvs_out) # 4\n", + " iaf1_out = self.iaf1(con1_out) # 5\n", + " pool1_out = self.pool1(iaf1_out) # 6\n", + "\n", + " conv2_out = self.conv2(pool1_out) # 7\n", + " iaf2_out = self.iaf2(conv2_out) # 8\n", + "\n", + " conv3_out = self.conv3(iaf2_out) # 9\n", + " iaf3_out = self.iaf3(conv3_out) # 10\n", + "\n", + " flat_out = self.flat(iaf3_out) # 15\n", + " \n", + " fc1_out = self.fc1(flat_out) # 11\n", + " iaf4_out = self.iaf4(fc1_out) # 12\n", + " fc2_out = self.fc2(iaf4_out) # 13\n", + " iaf5_out = self.iaf5(fc2_out) # 14\n", + "\n", + " return iaf5_out\n", + " \n", + "snn = SNN(input_shape)\n", + "snn.init_weights()\n", + "snn.to(device)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "metadata": {} + }, + "outputs": [], + "source": [ + "hw_model = DynapcnnNetwork(\n", + " snn=snn,\n", + " input_shape=input_shape,\n", + " batch_size=batch_size,\n", + " discretize=True\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "input_dummy = torch.randn((batch_size, *input_shape))" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "out = hw_model(input_dummy)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "tensor([[[[0.]],\n", + "\n", + " [[0.]],\n", + "\n", + " [[0.]],\n", + "\n", + " [[0.]],\n", + "\n", + " [[0.]],\n", + "\n", + " [[0.]],\n", + "\n", + " [[0.]],\n", + "\n", + " [[0.]],\n", + "\n", + " [[0.]],\n", + "\n", + " [[0.]]]], grad_fn=)\n" + ] + } + ], + "source": [ + "print(out)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "speck-rescnn", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/dynapcnn_network/snn_deployment.ipynb b/examples/dynapcnn_network/snn_deployment.ipynb index e89e9f1b..9c643724 100644 --- a/examples/dynapcnn_network/snn_deployment.ipynb +++ b/examples/dynapcnn_network/snn_deployment.ipynb @@ -2,11 +2,20 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": 2, "metadata": { "metadata": {} }, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/samurai2077/anaconda3/envs/speck-rescnn/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + } + ], "source": [ "import torch\n", "import torch.nn as nn\n", @@ -28,7 +37,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 3, "metadata": { "metadata": {} }, @@ -36,10 +45,10 @@ { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 2, + "execution_count": 3, "metadata": {}, "output_type": "execute_result" } @@ -59,7 +68,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -73,7 +82,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "metadata": { "metadata": {} }, @@ -138,7 +147,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "metadata": { "metadata": {} }, @@ -156,13 +165,43 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ + "Downloading https://prod-dcd-datasets-public-files-eu-west-1.s3.eu-west-1.amazonaws.com/1afc103f-8799-464a-a214-81bb9b1f9337 to ./NMNIST/NMNIST/train.zip\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "1011894272it [01:25, 11770103.44it/s] \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Extracting ./NMNIST/NMNIST/train.zip to ./NMNIST/NMNIST\n", + "Downloading https://prod-dcd-datasets-public-files-eu-west-1.s3.eu-west-1.amazonaws.com/a99d0fee-a95b-4231-ad22-988fdb0a2411 to ./NMNIST/NMNIST/test.zip\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "169675776it [00:18, 9035099.75it/s] \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Extracting ./NMNIST/NMNIST/test.zip to ./NMNIST/NMNIST\n", "The transformed array is in shape [Time-Step, Channel, Height, Width] --> (50, 2, 34, 34)\n" ] } @@ -396,11 +435,24 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 9, "metadata": { "metadata": {} }, - "outputs": [], + "outputs": [ + { + "ename": "TypeError", + "evalue": "randn() received an invalid combination of arguments - got (tuple, bool), but expected one of:\n * (tuple of ints size, *, torch.Generator generator, tuple of names names, torch.dtype dtype, torch.layout layout, torch.device device, bool pin_memory, bool requires_grad)\n * (tuple of ints size, *, torch.Generator generator, Tensor out, torch.dtype dtype, torch.layout layout, torch.device device, bool pin_memory, bool requires_grad)\n * (tuple of ints size, *, Tensor out, torch.dtype dtype, torch.layout layout, torch.device device, bool pin_memory, bool requires_grad)\n * (tuple of ints size, *, tuple of names names, torch.dtype dtype, torch.layout layout, torch.device device, bool pin_memory, bool requires_grad)\n", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[9], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m hw_model \u001b[38;5;241m=\u001b[39m \u001b[43mDynapcnnNetwork\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 2\u001b[0m \u001b[43m \u001b[49m\u001b[43msnn\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msnn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 3\u001b[0m \u001b[43m \u001b[49m\u001b[43minput_shape\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43minput_shape\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 4\u001b[0m \u001b[43m \u001b[49m\u001b[43mbatch_size\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbatch_size\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 5\u001b[0m \u001b[43m \u001b[49m\u001b[43mdiscretize\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\n\u001b[1;32m 6\u001b[0m \u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/Github/sinabs/sinabs/backend/dynapcnn/dynapcnn_network.py:70\u001b[0m, in \u001b[0;36mDynapcnnNetwork.__init__\u001b[0;34m(self, snn, input_shape, batch_size, dvs_input, discretize, weight_rescaling_fn)\u001b[0m\n\u001b[1;32m 67\u001b[0m batch_size \u001b[38;5;241m=\u001b[39m sinabs\u001b[38;5;241m.\u001b[39mutils\u001b[38;5;241m.\u001b[39mget_smallest_compatible_time_dimension(snn)\n\u001b[1;32m 68\u001b[0m \u001b[38;5;66;03m# computational graph from original PyTorch module.\u001b[39;00m\n\u001b[1;32m 69\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_graph_extractor \u001b[38;5;241m=\u001b[39m GraphExtractor(\n\u001b[0;32m---> 70\u001b[0m snn, \u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrandn\u001b[49m\u001b[43m(\u001b[49m\u001b[43m(\u001b[49m\u001b[43mbatch_size\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43minput_shape\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdvs_input\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 71\u001b[0m ) \u001b[38;5;66;03m# needs the batch dimension.\u001b[39;00m\n\u001b[1;32m 73\u001b[0m \u001b[38;5;66;03m# Remove nodes of ignored classes (including merge nodes)\u001b[39;00m\n\u001b[1;32m 74\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_graph_extractor\u001b[38;5;241m.\u001b[39mremove_nodes_by_class(DEFAULT_IGNORED_LAYER_TYPES)\n", + "\u001b[0;31mTypeError\u001b[0m: randn() received an invalid combination of arguments - got (tuple, bool), but expected one of:\n * (tuple of ints size, *, torch.Generator generator, tuple of names names, torch.dtype dtype, torch.layout layout, torch.device device, bool pin_memory, bool requires_grad)\n * (tuple of ints size, *, torch.Generator generator, Tensor out, torch.dtype dtype, torch.layout layout, torch.device device, bool pin_memory, bool requires_grad)\n * (tuple of ints size, *, Tensor out, torch.dtype dtype, torch.layout layout, torch.device device, bool pin_memory, bool requires_grad)\n * (tuple of ints size, *, tuple of names names, torch.dtype dtype, torch.layout layout, torch.device device, bool pin_memory, bool requires_grad)\n" + ] + } + ], "source": [ "hw_model = DynapcnnNetwork(\n", " snn=snn,\n", @@ -822,7 +874,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.0" + "version": "3.11.3" } }, "nbformat": 4, diff --git a/examples/dynapcnn_network/snn_need_create_DVSLayer.ipynb b/examples/dynapcnn_network/snn_need_create_DVSLayer.ipynb new file mode 100644 index 00000000..c6d195de --- /dev/null +++ b/examples/dynapcnn_network/snn_need_create_DVSLayer.ipynb @@ -0,0 +1,208 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "metadata": {} + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/samurai2077/anaconda3/envs/speck-rescnn/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + } + ], + "source": [ + "import torch\n", + "import torch.nn as nn\n", + "from sinabs.backend.dynapcnn import DynapcnnNetwork\n", + "from sinabs.backend.dynapcnn import DVSLayer\n", + "from sinabs.layers import Merge, IAFSqueeze, SumPool2d\n", + "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential\n", + "import sinabs.layers as sl\n", + "\n", + "from torch.nn import CrossEntropyLoss\n", + "from torch.optim import Adam\n", + "\n", + "from tonic.datasets.nmnist import NMNIST\n", + "from tonic.transforms import ToFrame\n", + "from torch.utils.data import DataLoader\n", + "import numpy as np\n", + "from tqdm.notebook import tqdm\n", + "from statistics import mode" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "metadata": {} + }, + "outputs": [], + "source": [ + "torch.manual_seed(0)\n", + "device = torch.device('cpu')" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "channels = 1\n", + "height = 34\n", + "width = 34\n", + "batch_size = 1\n", + "\n", + "input_shape = (channels, height, width)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "metadata": {} + }, + "outputs": [ + { + "data": { + "text/plain": [ + "SNN(\n", + " (conv1): Conv2d(1, 10, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + " (iaf1): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(1.), min_v_mem=Parameter containing:\n", + " tensor(-1.), batch_size=1, num_timesteps=-1)\n", + " (pool1): AvgPool2d(kernel_size=2, stride=2, padding=0)\n", + " (conv2): Conv2d(10, 10, kernel_size=(4, 4), stride=(1, 1), bias=False)\n", + " (iaf2): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(1.), min_v_mem=Parameter containing:\n", + " tensor(-1.), batch_size=1, num_timesteps=-1)\n", + " (conv3): Conv2d(10, 1, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + " (iaf3): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(1.), min_v_mem=Parameter containing:\n", + " tensor(-1.), batch_size=1, num_timesteps=-1)\n", + " (fc1): Linear(in_features=144, out_features=200, bias=False)\n", + " (iaf4): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(1.), min_v_mem=Parameter containing:\n", + " tensor(-1.), batch_size=1, num_timesteps=-1)\n", + " (fc2): Linear(in_features=200, out_features=10, bias=False)\n", + " (iaf5): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(1.), min_v_mem=Parameter containing:\n", + " tensor(-1.), batch_size=1, num_timesteps=-1)\n", + " (flat): Flatten(start_dim=1, end_dim=-1)\n", + ")" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "class SNN(nn.Module):\n", + " def __init__(self, input_shape) -> None:\n", + " super().__init__()\n", + "\n", + " # -- chip core A --\n", + " self.conv1 = nn.Conv2d(1, 10, 2, 1, bias=False)\n", + " self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential())\n", + " self.pool1 = nn.AvgPool2d(2,2)\n", + " # -- chip core B --\n", + " self.conv2 = nn.Conv2d(10, 10, 4, 1, bias=False)\n", + " self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential())\n", + " # -- chip core C --\n", + " self.conv3 = nn.Conv2d(10, 1, 2, 1, bias=False)\n", + " self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential())\n", + " # -- chip core D --\n", + " self.fc1 = nn.Linear(144, 200, bias=False)\n", + " self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential())\n", + " # -- chip core E --\n", + " self.fc2 = nn.Linear(200, 10, bias=False)\n", + " self.iaf5 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential())\n", + "\n", + " # -- layers ignored during deployment --\n", + " self.flat = nn.Flatten()\n", + "\n", + " def init_weights(self):\n", + " for name, layer in self.named_modules():\n", + " if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear):\n", + " nn.init.xavier_normal_(layer.weight.data)\n", + "\n", + " def detach_neuron_states(self):\n", + " for name, layer in self.named_modules():\n", + " if name != '':\n", + " if isinstance(layer, sl.StatefulLayer):\n", + " for name, buffer in layer.named_buffers():\n", + " buffer.detach_()\n", + "\n", + " def forward(self, x):\n", + " \n", + " con1_out = self.conv1(x) # 4\n", + " iaf1_out = self.iaf1(con1_out) # 5\n", + " pool1_out = self.pool1(iaf1_out) # 6\n", + "\n", + " conv2_out = self.conv2(pool1_out) # 7\n", + " iaf2_out = self.iaf2(conv2_out) # 8\n", + "\n", + " conv3_out = self.conv3(iaf2_out) # 9\n", + " iaf3_out = self.iaf3(conv3_out) # 10\n", + "\n", + " flat_out = self.flat(iaf3_out) # 15\n", + " \n", + " fc1_out = self.fc1(flat_out) # 11\n", + " iaf4_out = self.iaf4(fc1_out) # 12\n", + " fc2_out = self.fc2(iaf4_out) # 13\n", + " iaf5_out = self.iaf5(fc2_out) # 14\n", + "\n", + " return iaf5_out\n", + " \n", + "snn = SNN(input_shape)\n", + "snn.init_weights()\n", + "snn.to(device)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "metadata": {} + }, + "outputs": [], + "source": [ + "hw_model = DynapcnnNetwork(\n", + " snn=snn,\n", + " input_shape=input_shape,\n", + " batch_size=batch_size,\n", + " dvs_input=True,\n", + " discretize=True,\n", + ")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "speck-rescnn", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/sinabs/backend/dynapcnn/dvs_layer.py b/sinabs/backend/dynapcnn/dvs_layer.py index 84e5a2fb..bc240c25 100644 --- a/sinabs/backend/dynapcnn/dvs_layer.py +++ b/sinabs/backend/dynapcnn/dvs_layer.py @@ -43,7 +43,6 @@ def __init__( flip_y: bool = False, swap_xy: bool = False, disable_pixel_array: bool = True, - destinations: list = [0] # TODO: just to get the forward to work but should use same concept for the pool argument in DynapcnnLayer. ): super().__init__() @@ -227,6 +226,7 @@ def forward(self, data): # Pool out = self.pool_layer(data) + # TODO - self.crop_layer is never None (even if crop == None when instantiating the class) so this 'if' statement is unecessary (plus confusing when debbuging the code). # Crop if self.crop_layer is not None: out = self.crop_layer(out) @@ -234,12 +234,7 @@ def forward(self, data): # Flip stuff out = self.flip_layer(out) - # TODO: just to get the forward to work but should use same concept for the pool argument in DynapcnnLayer. - returns = [] - for dest in destinations: - returns.append(out) - - return tuple(returns) + return out def get_pooling(self) -> Tuple[int, int]: """Pooling kernel shape. From 51c65a914e168bdab68deac14bb9ea38775a657f Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Wed, 30 Oct 2024 14:17:26 +0100 Subject: [PATCH 270/379] Graph extraction: Ignore some classes right away --- sinabs/backend/dynapcnn/dynapcnn_network.py | 60 +++++++++++-------- .../backend/dynapcnn/nir_graph_extractor.py | 37 ++++++++---- sinabs/backend/dynapcnn/utils.py | 18 +++--- 3 files changed, 66 insertions(+), 49 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index a98bdad4..208c8312 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -6,23 +6,20 @@ from warnings import warn import samna -from samna.dynapcnn.configuration import DynapcnnConfiguration import torch import torch.nn as nn +from samna.dynapcnn.configuration import DynapcnnConfiguration from torch import Tensor import sinabs import sinabs.layers as sl from .chip_factory import ChipFactory -from .dynapcnn_layer import DynapcnnLayer from .dvs_layer import DVSLayer +from .dynapcnn_layer import DynapcnnLayer from .io import disable_timestamps, enable_timestamps, open_device, reset_timestamps from .nir_graph_extractor import GraphExtractor -from .utils import ( - DEFAULT_IGNORED_LAYER_TYPES, - parse_device_id, -) +from .utils import COMPLETELY_IGNORED_LAYER_TYPES, IGNORED_LAYER_TYPES, parse_device_id from .weight_rescaling_methods import rescale_method_1 @@ -69,11 +66,17 @@ def __init__( batch_size = sinabs.utils.get_smallest_compatible_time_dimension(snn) # computational graph from original PyTorch module. self._graph_extractor = GraphExtractor( - snn, torch.randn((batch_size, *self.input_shape)) - ) # needs the batch dimension. + snn, + torch.randn((batch_size, *self.input_shape)), + ignore_node_types=COMPLETELY_IGNORED_LAYER_TYPES, + ) # Remove nodes of ignored classes (including merge nodes) - self._graph_extractor.remove_nodes_by_class(DEFAULT_IGNORED_LAYER_TYPES) + # Other than `COMPLETELY_IGNORED_LAYER_TYPES`, `IGNORED_LAYER_TYPES` are + # part of the graph initially and are needed to ensure proper handling of + # graph structure (e.g. Merge nodes) or meta-information (e.g. + # `nn.Flatten` for io-shapes) + self._graph_extractor.remove_nodes_by_class(IGNORED_LAYER_TYPES) # Module to execute forward pass through network self._dynapcnn_module = self._graph_extractor.get_dynapcnn_network_module( @@ -83,6 +86,14 @@ def __init__( ####################################################### Public Methods ####################################################### + @property + def chip_layers_ordering(self): + warn( + "`chip_layers_ordering` is deprecated. Returning `layer2core_map` instead.", + DeprecationWarning, + ) + return self._layer2core_map + @property def dynapcnn_layers(self): return self._dynapcnn_module.dynapcnn_layers @@ -100,12 +111,8 @@ def layer2core_map(self): return self._layer2core_map @property - def chip_layers_ordering(self): - warn( - "`chip_layers_ordering` is deprecated. Returning `layer2core_map` instead.", - DeprecationWarning - ) - return self._layer2core_map + def name_2_indx_map(self): + return self._graph_extractor.name_2_indx_map def hw_forward(self, x): """Forwards data through the chip.""" @@ -231,8 +238,8 @@ def to( layer2core_map: Union[Dict[int, int], str] = "auto", chip_layers_ordering="auto", ): - """ Deploy model to cpu, gpu or a SynSense device. - + """Deploy model to cpu, gpu or a SynSense device. + Note that the model parameters are only ever transferred to the device on the `to` call, so changing a threshold or weight of a model that is deployed will have no effect on the model on chip until `to` is called again. @@ -256,7 +263,7 @@ def to( A user configuration modifier method. This function can be used to make any custom changes you want to make to the configuration object. - layer2core_map (dict or "auto"): Defines how cores on chip are + layer2core_map (dict or "auto"): Defines how cores on chip are assigned to DynapcnnLayers. If `auto`, an automated procedure will be used to find a valid ordering. Otherwise a dict needs to be passed, with DynapcnnLayer indices as keys and assigned @@ -378,7 +385,7 @@ def make_config( Parameters ---------- - - layer2core_map (dict or "auto"): Defines how cores on chip are + - layer2core_map (dict or "auto"): Defines how cores on chip are assigned to DynapcnnLayers. If `auto`, an automated procedure will be used to find a valid ordering. Otherwise a dict needs to be passed, with DynapcnnLayer indices as keys and assigned @@ -435,8 +442,8 @@ def make_config( raise ValueError(f"Generated config is not valid for {device}") def has_dvs_layer(self) -> bool: - """ Return True if there is a DVSLayer in the network - + """Return True if there is a DVSLayer in the network + Returns ------- bool: True if DVSLayer is found within the network. @@ -444,8 +451,8 @@ def has_dvs_layer(self) -> bool: for layer in self.dynapcnn_layers.values(): if isinstance(layer, DVSLayer): return True - return False - + return False + ####################################################### Private Methods ####################################################### def _make_config( @@ -460,7 +467,7 @@ def _make_config( Parameters ---------- - - layer2core_map (dict or "auto"): Defines how cores on chip are + - layer2core_map (dict or "auto"): Defines how cores on chip are assigned to DynapcnnLayers. If `auto`, an automated procedure will be used to find a valid ordering. Otherwise a dict needs to be passed, with DynapcnnLayer indices as keys and assigned @@ -565,7 +572,8 @@ def _make_config( elif monitor_layers == "all": monitor_layers = [ - lyr_idx for lyr_idx, layer in self.dynapcnn_layers.items() + lyr_idx + for lyr_idx, layer in self.dynapcnn_layers.items() if not isinstance(layer, DVSLayer) ] @@ -583,7 +591,7 @@ def _make_config( if config_modifier is not None: # apply user config modifier. config = config_modifier(config) - + # Validate config return config, config_builder.validate_configuration(config) diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index ffd6814b..80d2701e 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -1,7 +1,7 @@ # author : Willian Soares Girao # contact : wsoaresgirao@gmail.com -from typing import Callable, Dict, List, Optional, Set, Tuple, Type +from typing import Callable, Dict, Iterable, List, Optional, Set, Tuple, Type, Union import nirtorch import torch @@ -21,7 +21,12 @@ class GraphExtractor: - def __init__(self, spiking_model: nn.Module, dummy_input: torch.tensor): + def __init__( + self, + spiking_model: nn.Module, + dummy_input: torch.tensor, + ignore_node_types: Optional[Iterable[Type]] = None, + ): """Class implementing the extraction of the computational graph from `spiking_model`, where each node represents a layer in the model and the list of edges represents how the data flow between the layers. @@ -47,12 +52,22 @@ def __init__(self, spiking_model: nn.Module, dummy_input: torch.tensor): Map from layer ID to the corresponding nn.Module instance. - nodes_io_shapes (dict): Map from node ID to dict containing node's in- and output shapes + - ignore_node_types (iterable of types): Node types that should be + ignored completely from the graph. This can include, for instance, + `nn.Dropout2d`, which otherwise can result in wrongly inferred + graph structures by NIRTorch. Types such as `nn.Flatten`, or sinabs + `Merge` should not be included here, as they are needed to properly + handle graph structure and metadata. They can be removed after + instantiation with `remove_nodes_by_class`. """ # extract computational graph. nir_graph = nirtorch.extract_torch_graph( spiking_model, dummy_input, model_name=None ).ignore_tensors() + if ignore_node_types is not None: + for node_type in ignore_node_types: + nir_graph = nir_graph.ignore_nodes(node_type) # Map node names to indices self._name_2_indx_map = self._get_name_2_indx_map(nir_graph) @@ -98,7 +113,7 @@ def indx_2_module_map(self) -> Dict[int, nn.Module]: def get_dynapcnn_network_module( self, discretize: bool = False, weight_rescaling_fn: Optional[Callable] = None ) -> DynapcnnNetworkModule: - """ Create DynapcnnNetworkModule based on stored graph representation + """Create DynapcnnNetworkModule based on stored graph representation This includes construction of the DynapcnnLayer instances @@ -115,9 +130,9 @@ def get_dynapcnn_network_module( """ # create a dict holding the data necessary to instantiate a `DynapcnnLayer`. - dcnnl_map = collect_dynapcnn_layer_info( - indx_2_module_map = self.indx_2_module_map, - edges = self.edges, + self.dcnnl_map = collect_dynapcnn_layer_info( + indx_2_module_map=self.indx_2_module_map, + edges=self.edges, nodes_io_shapes=self.nodes_io_shapes, entry_nodes=self.entry_nodes, ) @@ -125,18 +140,16 @@ def get_dynapcnn_network_module( # build `DynapcnnLayer` instances from mapper. dynapcnn_layers, destination_map, entry_points = ( construct_dynapcnnlayers_from_mapper( - dcnnl_map=dcnnl_map, + dcnnl_map=self.dcnnl_map, discretize=discretize, rescale_fn=weight_rescaling_fn, ) ) # Instantiate the DynapcnnNetworkModule - return DynapcnnNetworkModule( - dynapcnn_layers, destination_map, entry_points - ) + return DynapcnnNetworkModule(dynapcnn_layers, destination_map, entry_points) - def remove_nodes_by_class(self, node_classes: Tuple[Type]): + def remove_nodes_by_class(self, node_classes: Union[Type, Tuple[Type]]): """Remove nodes of given classes from graph in place. Create a new set of edges, considering layers that `DynapcnnNetwork` will ignore. This @@ -287,7 +300,7 @@ def _get_named_modules(self, model: nn.Module) -> Dict[int, nn.Module]: indx_2_module_map = dict() for name, module in model.named_modules(): - # Make sure names match those provided by nirtorch nodes + # Make sure names match those provided by nirtorch nodes name = nirtorch.utils.sanitize_name(name) if name in self._name_2_indx_map: indx_2_module_map[self._name_2_indx_map[name]] = module diff --git a/sinabs/backend/dynapcnn/utils.py b/sinabs/backend/dynapcnn/utils.py index caf3d872..9a63fd73 100644 --- a/sinabs/backend/dynapcnn/utils.py +++ b/sinabs/backend/dynapcnn/utils.py @@ -1,13 +1,6 @@ from collections import defaultdict, deque from copy import deepcopy -from typing import ( - TYPE_CHECKING, - List, - Optional, - Set, - Tuple, - Union, -) +from typing import TYPE_CHECKING, List, Optional, Set, Tuple, Union import torch import torch.nn as nn @@ -22,9 +15,12 @@ if TYPE_CHECKING: from sinabs.backend.dynapcnn.dynapcnn_network import DynapcnnNetwork -DEFAULT_IGNORED_LAYER_TYPES = Union[ - nn.Identity, nn.Dropout, nn.Dropout2d, nn.Flatten, sl.Merge -] +# Other than `COMPLETELY_IGNORED_LAYER_TYPES`, `IGNORED_LAYER_TYPES` are +# part of the graph initially and are needed to ensure proper handling of +# graph structure (e.g. Merge nodes) or meta-information (e.g. +# `nn.Flatten` for io-shapes) +COMPLETELY_IGNORED_LAYER_TYPES = (nn.Identity, nn.Dropout, nn.Dropout2d) +IGNORED_LAYER_TYPES = Union[nn.Flatten, sl.Merge] Edge = Tuple[int, int] # Define edge-type alias From 7ef74e85feca73efb8a29ba334d28445a1b8f0cd Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Wed, 30 Oct 2024 14:17:48 +0100 Subject: [PATCH 271/379] fix doorbell test --- tests/test_dynapcnn/test_doorbell.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/tests/test_dynapcnn/test_doorbell.py b/tests/test_dynapcnn/test_doorbell.py index fc8040df..40535285 100644 --- a/tests/test_dynapcnn/test_doorbell.py +++ b/tests/test_dynapcnn/test_doorbell.py @@ -5,6 +5,7 @@ import samna import torch +from nirtorch.utils import sanitize_name from torch import nn from sinabs.backend.dynapcnn.dynapcnn_network import DynapcnnNetwork @@ -80,5 +81,17 @@ def test_auto_config(): def test_was_copied(): # - Make sure that layers of different models are distinct objects - for lyr_snn, lyr_dynapcnn in zip(snn.spiking_model, dynapcnn_net.sequence): - assert lyr_snn is not lyr_dynapcnn + snn_layers = {sanitize_name(name): lyr for name, lyr in snn.named_modules()} + idx_2_name_map = {idx: name for name, idx in dynapcnn_net.name_2_indx_map.items()} + for idx, lyr_info in dynapcnn_net._graph_extractor.dcnnl_map.items(): + conv_lyr_dynapcnn = dynapcnn_net.dynapcnn_layers[idx].conv_layer + conv_node_idx = lyr_info["conv"]["node_id"] + conv_name = idx_2_name_map[conv_node_idx] + conv_lyr_snn = snn_layers[conv_name] + assert conv_lyr_dynapcnn is not conv_lyr_snn + + spk_lyr_dynapcnn = dynapcnn_net.dynapcnn_layers[idx].spk_layer + spk_node_idx = lyr_info["neuron"]["node_id"] + spk_name = idx_2_name_map[spk_node_idx] + spk_lyr_snn = snn_layers[spk_name] + assert spk_lyr_dynapcnn is not spk_lyr_snn From 7e22662fa603a2f7d7e630860bc6152d121ee83f Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Wed, 30 Oct 2024 16:52:44 +0100 Subject: [PATCH 272/379] More meaningful exceptions for invalid graph structures --- sinabs/backend/dynapcnn/exceptions.py | 26 ----- .../backend/dynapcnn/sinabs_edges_handler.py | 110 +++++++++++++----- 2 files changed, 80 insertions(+), 56 deletions(-) diff --git a/sinabs/backend/dynapcnn/exceptions.py b/sinabs/backend/dynapcnn/exceptions.py index 63de8585..4e8432d1 100644 --- a/sinabs/backend/dynapcnn/exceptions.py +++ b/sinabs/backend/dynapcnn/exceptions.py @@ -79,32 +79,6 @@ def __init__(self, edge, source, target): super().__init__(f"Invalid edge {edge}: {source} can not target {target}.") -class InvalidEdgeType(Exception): - edge: Tuple[int, int] - type: int - - def __init__(self, edge, type): - super().__init__(f"Invalid edge type {type} for edge {edge}.") - - -class UnmatchedNode(Exception): - edge: Tuple[int, int] - node: int - - def __init__(self, edge, node): - super().__init__( - f"Node {node} in edge {edge} can not found in previously processed edges." - ) - - -class UnmatchedPoolingEdges(Exception): - def __init__(self, edges: Set[int]): - super().__init__( - "The following edges between pooling layers could not be processed: " - f"{edges}. The computational graph is likely invalid." - ) - - class UnknownNode(Exception): node: int diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index 8eab7b19..fbf7df50 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -11,12 +11,7 @@ from torch import Size, nn from .connectivity_specs import VALID_SINABS_EDGE_TYPES -from .exceptions import ( - InvalidEdge, - InvalidGraphStructure, - UnmatchedNode, - UnmatchedPoolingEdges, -) +from .exceptions import InvalidEdge, InvalidGraphStructure from .utils import Edge @@ -55,20 +50,13 @@ def collect_dynapcnn_layer_info( edges=edges, indx_2_module_map=indx_2_module_map ) - if "weight-neuron" not in edges_by_type: - raise InvalidGraphStructure( - "Any dynapcnn layer must contain a weight layer (e.g. Conv2d, Linear) " - "that is directly connected to a neuron layer (e.g. IAFSqueeze). " - "None such weight-neuron pair has been found in the provided network." - ) - # Dict to collect information for each future dynapcnn layer dynapcnn_layer_info = dict() # Map node IDs to dynapcnn layer ID node_2_layer_map = dict() # Each weight->neuron connection instantiates a new, unique dynapcnn layer - while edges_by_type["weight-neuron"]: + while edges_by_type.get("weight-neuron", False): edge = edges_by_type["weight-neuron"].pop() init_new_dynapcnnlayer_entry( dynapcnn_layer_info, @@ -83,7 +71,11 @@ def collect_dynapcnn_layer_info( while edges_by_type.get("neuron-weight", False): edge = edges_by_type["neuron-weight"].pop() set_neuron_layer_destination( - dynapcnn_layer_info, edge, node_2_layer_map, nodes_io_shapes + dynapcnn_layer_info, + edge, + node_2_layer_map, + nodes_io_shapes, + indx_2_module_map, ) # Add pooling based on neuron->pooling connections @@ -104,13 +96,24 @@ def collect_dynapcnn_layer_info( # After adding pooling make sure all pooling-pooling edges have been handled if len(pooling_pooling_edges) > 0: - raise UnmatchedPoolingEdges(pooling_pooling_edges) + unmatched_layers = {edge[0] for edge in pooling_pooling_edges} + raise InvalidGraphStructure( + f"Pooling layers {unmatched_layers} could not be assigned to a " + "dynapcnn layer. This is likely due to an unsupported SNN " + "architecture. Pooling layers must always be preceded by a " + "spiking layer (`IAFSqueeze`), another pooling layer, or" + "DVS input" + ) # Add all edges connecting pooling to a new dynapcnn layer while edges_by_type.get("pooling-weight", False): edge = edges_by_type["pooling-weight"].pop() set_pooling_layer_destination( - dynapcnn_layer_info, edge, node_2_layer_map, nodes_io_shapes + dynapcnn_layer_info, + edge, + node_2_layer_map, + nodes_io_shapes, + indx_2_module_map, ) # Make sure we have taken care of all edges @@ -244,7 +247,8 @@ def add_pooling_to_entry( indx_2_module_map: Dict[int, nn.Module], node_2_layer_map: Dict[int, int], ) -> None: - """Add or extend destination information to existing entry in `dynapcnn_layer_info`. + """Add or extend destination information with pooling for existing + entry in `dynapcnn_layer_info`. Correct entry is identified by existing neuron node. Destination information is a dict containing list of IDs and list of modules for each chains of pooling nodes. @@ -254,8 +258,10 @@ def add_pooling_to_entry( dynapcnn_layer_info: Dict with one entry for each future dynapcnn layer. key is unique dynapcnn layer ID, value is dict with nodes of the layer Will be updated in-place. - edge: Tuple of 2 integers, indicating edge between two nodes in graph. - Edge source has to be within an existing entry of `dynapcnn_layer_info`. + edge: Tuple of 2 integers, indicating edge between a neuron node and the pooling + node that starts all provided `pooling_chains`. + Edge source has to be a neuron node within an existing entry of + `dynapcnn_layer_info`, i.e. it has to have been processed already. pooling_chains: List of deque of int. All sequences ("chains") of connected pooling nodes, starting from edge[1] indx_2_module_map (dict): Maps node IDs of the graph as `key` to their associated module as `value` @@ -266,7 +272,13 @@ def add_pooling_to_entry( try: layer_idx = node_2_layer_map[edge[0]] except KeyError: - raise UnmatchedNode(edge, edge[0]) + neuron_layer = indx_2_module_map[edge[0]] + raise InvalidGraphStructure( + f"Spiking layer {neuron_layer} cannot be assigned to a dynapcnn layer. " + "This is likely due to an unsupported SNN architecture. Spiking " + "layers have to be preceded by a weight layer (`nn.Conv2d` or " + "`nn.Linear`)." + ) # Make sure all pooling chains start with expected node assert all(chain[0] == edge[1] for chain in pooling_chains) @@ -324,8 +336,9 @@ def set_neuron_layer_destination( edge: Edge, node_2_layer_map: Dict[int, int], nodes_io_shapes: Dict[int, Dict[str, Tuple[Size, Size]]], + indx_2_module_map: Dict[int, nn.Module], ) -> None: - """Set destination layer without pooling. + """Set destination layer without pooling for existing entry in `dynapcnn_layer_info`. Parameters ---------- @@ -333,20 +346,34 @@ def set_neuron_layer_destination( key is unique dynapcnn layer ID, value is dict with nodes of the layer Will be updated in-place. edge: Tuple of 2 integers, indicating edge between two nodes in graph. - Edge source has to be within an existing entry of `dynapcnn_layer_info`. + Edge source has to be a neuron layer within an existing entry of + `dynapcnn_layer_info`. Edge target has to be the weight layer of + another dynapcnn layer. node_2_layer_map (dict): Maps each node ID to the ID of the layer it is assigned to. Will be updated in-place. nodes_io_shapes (dict): Map from node ID to dict containing node's in- and output shapes + indx_2_module_map (dict): Maps node IDs of the graph as `key` to their associated module as `value` """ # Make sure both source (neuron layer) and target (weight layer) have been previously processed try: source_layer_idx = node_2_layer_map[edge[0]] except KeyError: - raise UnmatchedNode(edge, edge[0]) + neuron_layer = indx_2_module_map[edge[0]] + raise InvalidGraphStructure( + f"Spiking layer {neuron_layer} cannot be assigned to a dynapcnn layer. " + "This is likely due to an unsupported SNN architecture. Spiking " + "layers have to be preceded by a weight layer (`nn.Conv2d` or " + "`nn.Linear`)." + ) try: destination_layer_idx = node_2_layer_map[edge[1]] except KeyError: - raise UnmatchedNode(edge, edge[1]) + weight_layer = indx_2_module_map[edge[1]] + raise InvalidGraphStructure( + f"Weight layer {weight_layer} cannot be assigned to a dynapcnn layer. " + "This is likely due to an unsupported SNN architecture. Weight " + "layers have to be followed by a spiking layer (`IAFSqueeze`)." + ) # Add new destination output_shape = nodes_io_shapes[edge[0]]["output"] @@ -371,8 +398,9 @@ def set_pooling_layer_destination( edge: Edge, node_2_layer_map: Dict[int, int], nodes_io_shapes: Dict[int, Dict[str, Tuple[Size, Size]]], + indx_2_module_map: Dict[int, nn.Module], ) -> None: - """Set destination layer with pooling. + """Set destination layer with pooling for existing entry in `dynapcnn_layer_info`. Parameters ---------- @@ -380,20 +408,36 @@ def set_pooling_layer_destination( key is unique dynapcnn layer ID, value is dict with nodes of the layer Will be updated in-place. edge: Tuple of 2 integers, indicating edge between two nodes in graph. - Edge source has to be within an existing entry of `dynapcnn_layer_info`. + Edge source has to be a pooling layer that is at the end of at least + one pooling chain within an existing entry of `dynapcnn_layer_info`. + Edge target has to be a weight layer within an existing entry of + `dynapcnn_layer_info`. node_2_layer_map (dict): Maps each node ID to the ID of the layer it is assigned to. Will be updated in-place. nodes_io_shapes (dict): Map from node ID to dict containing node's in- and output shapes + indx_2_module_map (dict): Maps node IDs of the graph as `key` to their associated module as `value` """ # Make sure both source (pooling layer) and target (weight layer) have been previously processed try: source_layer_idx = node_2_layer_map[edge[0]] except KeyError: - raise UnmatchedNode(edge, edge[0]) + poolin_layer = indx_2_module_map[edge[0]] + raise InvalidGraphStructure( + f"Layer {poolin_layer} cannot be assigned to a dynapcnn layer. " + "This is likely due to an unsupported SNN architecture. Pooling " + "layers have to be preceded by a spiking layer (`IAFSqueeze`), " + "another pooling layer, or DVS input" + ) try: destination_layer_idx = node_2_layer_map[edge[1]] except KeyError: - raise UnmatchedNode(edge, edge[1]) + weight_layer = indx_2_module_map[edge[1]] + raise InvalidGraphStructure( + f"Weight layer {weight_layer} cannot be assigned to a dynapcnn layer. " + "This is likely due to an unsupported SNN architecture. Weight " + "layers have to be preceded by a spiking layer (`IAFSqueeze`), " + "another pooling layer, or DVS input" + ) # Find current source node within destinations layer_info = dynapcnn_layer_info[source_layer_idx] @@ -410,7 +454,13 @@ def set_pooling_layer_destination( matched = True break if not matched: - raise UnmatchedNode(edge, edge[0]) + poolin_layer = indx_2_module_map[edge[0]] + raise InvalidGraphStructure( + f"Layer {poolin_layer} cannot be assigned to a dynapcnn layer. " + "This is likely due to an unsupported SNN architecture. Pooling " + "layers have to be preceded by a spiking layer (`IAFSqueeze`), " + "another pooling layer, or DVS input" + ) # Set destination layer within destination dict that holds current source node destination["destination_layer"] = destination_layer_idx From ef02dab46a0b72a343391732d89fd960ddbd6f0c Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Wed, 30 Oct 2024 16:57:04 +0100 Subject: [PATCH 273/379] Fix dynapcnn layer scaling --- sinabs/backend/dynapcnn/dynapcnn_layer.py | 4 ++-- sinabs/backend/dynapcnn/dynapcnn_layer_utils.py | 9 +++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer.py b/sinabs/backend/dynapcnn/dynapcnn_layer.py index 46fcea63..cb48ea6c 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer.py @@ -81,7 +81,7 @@ class DynapcnnLayer(nn.Module): discretize: bool Whether to discretize parameters. rescale_weights: int - Layer weights will be divided by this value. + Layer weights will be multiplied by this value. """ def __init__( @@ -114,7 +114,7 @@ def __init__( if self._rescale_weights != 1: # this has to be done after copying but before discretizing - conv.weight.data = (conv.weight / self._rescale_weights).clone().detach() + conv.weight.data = (conv.weight * self._rescale_weights).clone().detach() # TODO: Does this really need to be enforced here or upon deployment? # check if convolution kernel is a square. diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py b/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py index 1d7b5086..4bbe8562 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py @@ -254,7 +254,7 @@ def construct_single_dynapcnn_layer( def construct_destination_map(dcnnl_map: Dict[int, Dict]) -> Dict[int, List[int]]: - """ Create a dict that holds destinations for each layer + """Create a dict that holds destinations for each layer Parameters ---------- @@ -283,7 +283,7 @@ def construct_destination_map(dcnnl_map: Dict[int, Dict]) -> Dict[int, List[int] def collect_entry_points(dcnnl_map: Dict[int, Dict]) -> Set[int]: - """ Return set of layer indices that are entry points + """Return set of layer indices that are entry points Parameters ---------- @@ -294,6 +294,7 @@ def collect_entry_points(dcnnl_map: Dict[int, Dict]) -> Set[int]: Set of all layer indices which act as entry points to the network """ return { - layer_index - for layer_index, layer_info in dcnnl_map.items() if layer_info["is_entry_node"] + layer_index + for layer_index, layer_info in dcnnl_map.items() + if layer_info["is_entry_node"] } From 1866a7c795280bd0b51d6d05b91b0335388dd65c Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Wed, 30 Oct 2024 18:19:10 +0100 Subject: [PATCH 274/379] Infer shape after removing flatten --- sinabs/backend/dynapcnn/dynapcnn_layer_utils.py | 3 +-- sinabs/backend/dynapcnn/nir_graph_extractor.py | 14 +++++++++++++- sinabs/backend/dynapcnn/sinabs_edges_handler.py | 13 ------------- sinabs/backend/dynapcnn/utils.py | 4 ++-- 4 files changed, 16 insertions(+), 18 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py b/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py index 4bbe8562..f9da483f 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py @@ -57,8 +57,6 @@ def finalize_dcnnl_map(dcnnl_map: Dict, rescale_fn: Optional[Callable] = None) - for layer_info in dcnnl_map.values(): # Consolidate scale factors consolidate_layer_scaling(layer_info, rescale_fn) - # Handle input dimensions - determine_layer_input_shape(layer_info) def consolidate_layer_pooling(layer_info: Dict, dcnnl_map: Dict): @@ -183,6 +181,7 @@ def consolidate_layer_scaling(layer_info: Dict, rescale_fn: Optional[Callable] = layer_info["rescale_factor"] = rescale_factor +# TODO: Obsolete def determine_layer_input_shape(layer_info: Dict): """Determine input shape of single layer diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index 80d2701e..8d1e781d 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -149,7 +149,7 @@ def get_dynapcnn_network_module( # Instantiate the DynapcnnNetworkModule return DynapcnnNetworkModule(dynapcnn_layers, destination_map, entry_points) - def remove_nodes_by_class(self, node_classes: Union[Type, Tuple[Type]]): + def remove_nodes_by_class(self, node_classes: Tuple[Type]): """Remove nodes of given classes from graph in place. Create a new set of edges, considering layers that `DynapcnnNetwork` will ignore. This @@ -173,6 +173,18 @@ def remove_nodes_by_class(self, node_classes: Union[Type, Tuple[Type]]): if not isinstance(self.indx_2_module_map[node], node_classes) } + if nn.Flatten in node_classes: + # Update input shapes of nodes after `Flatten` to the shape before flattening + # Note: This is likely to produce incorrect results if multiple Flatten layers + # come in sequence. + for node in self.sorted_nodes: + if isinstance(self.indx_2_module_map[node], nn.Flatten): + shape_before_flatten = self.nodes_io_shapes[node]["input"] + for target_node in self._find_valid_targets(node, node_classes): + self._nodes_io_shapes[target_node][ + "input" + ] = shape_before_flatten + # remapping nodes indices contiguously starting from 0 remapped_nodes = { old_idx: new_idx diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index fbf7df50..20c9d559 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -219,9 +219,6 @@ def init_new_dynapcnnlayer_entry( dynapcnn_layer_info[layer_id] = { "input_shape": nodes_io_shapes[edge[0]]["input"], - # Collect output shapes (before possible flattening) of layers with this layer as their destination - # This will allow infering shapes when converting linear to conv layers - "inferred_input_shapes": set(), "conv": { "module": indx_2_module_map[edge[0]], "node_id": edge[0], @@ -387,11 +384,6 @@ def set_neuron_layer_destination( } ) - # Add output shape of this layer to input shapes of destination - dynapcnn_layer_info[destination_layer_idx]["inferred_input_shapes"].add( - output_shape - ) - def set_pooling_layer_destination( dynapcnn_layer_info: Dict[int, Dict], @@ -467,11 +459,6 @@ def set_pooling_layer_destination( output_shape = nodes_io_shapes[edge[0]]["output"] destination["output_shape"] = output_shape - # Add output shape of this layer to input shapes of destination - dynapcnn_layer_info[destination_layer_idx]["inferred_input_shapes"].add( - output_shape - ) - def trace_paths(node: int, remaining_edges: Set[Edge]) -> List[deque[int]]: """Trace any path of collected edges through the graph. diff --git a/sinabs/backend/dynapcnn/utils.py b/sinabs/backend/dynapcnn/utils.py index 9a63fd73..06946a07 100644 --- a/sinabs/backend/dynapcnn/utils.py +++ b/sinabs/backend/dynapcnn/utils.py @@ -1,6 +1,6 @@ from collections import defaultdict, deque from copy import deepcopy -from typing import TYPE_CHECKING, List, Optional, Set, Tuple, Union +from typing import TYPE_CHECKING, List, Optional, Set, Tuple import torch import torch.nn as nn @@ -20,7 +20,7 @@ # graph structure (e.g. Merge nodes) or meta-information (e.g. # `nn.Flatten` for io-shapes) COMPLETELY_IGNORED_LAYER_TYPES = (nn.Identity, nn.Dropout, nn.Dropout2d) -IGNORED_LAYER_TYPES = Union[nn.Flatten, sl.Merge] +IGNORED_LAYER_TYPES = (nn.Flatten, sl.Merge) Edge = Tuple[int, int] # Define edge-type alias From b75885d335af887e13581c580f576e4b570d595b Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Wed, 30 Oct 2024 18:30:56 +0100 Subject: [PATCH 275/379] Fix behavior when entry nodes are removed from graph --- .../backend/dynapcnn/nir_graph_extractor.py | 28 ++++++++++--------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index 8d1e781d..6a946a99 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -166,25 +166,27 @@ def remove_nodes_by_class(self, node_classes: Tuple[Type]): """ # Compose new graph by creating a dict with all remaining node IDs as keys and set of target node IDs as values - source2target: Dict[int, Set[int]] = { - node: self._find_valid_targets(node, node_classes) - for node in self.sorted_nodes - # Skip nodes that are to be removed - if not isinstance(self.indx_2_module_map[node], node_classes) - } - - if nn.Flatten in node_classes: - # Update input shapes of nodes after `Flatten` to the shape before flattening - # Note: This is likely to produce incorrect results if multiple Flatten layers - # come in sequence. - for node in self.sorted_nodes: - if isinstance(self.indx_2_module_map[node], nn.Flatten): + source2target: Dict[int, Set[int]] = {} + for node in self.sorted_nodes: + if isinstance((mod := self.indx_2_module_map[node]), node_classes): + # If an entry node is removed, its targets become entry nodes + if node in self.entry_nodes: + targets = self._find_valid_targets(node, node_classes) + self._entry_nodes.update(targets) + + # Update input shapes of nodes after `Flatten` to the shape before flattening + # Note: This is likely to produce incorrect results if multiple Flatten layers + # come in sequence. + if isinstance(mod, nn.Flatten): shape_before_flatten = self.nodes_io_shapes[node]["input"] for target_node in self._find_valid_targets(node, node_classes): self._nodes_io_shapes[target_node][ "input" ] = shape_before_flatten + else: + source2target[node] = self._find_valid_targets(node, node_classes) + # remapping nodes indices contiguously starting from 0 remapped_nodes = { old_idx: new_idx From 2c60c14b57fbacd135431c9f5f6a0c3afe338419 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Wed, 30 Oct 2024 18:31:23 +0100 Subject: [PATCH 276/379] Update unit tests --- tests/test_dynapcnn/test_auto_mapping.py | 2 +- tests/test_dynapcnn/test_individual_cases.py | 12 ++++-------- tests/test_dynapcnnlayer/model_dummy_1.py | 11 +++-------- tests/test_dynapcnnlayer/model_dummy_2.py | 11 ++--------- tests/test_dynapcnnlayer/model_dummy_3.py | 13 ++----------- tests/test_dynapcnnlayer/model_dummy_4.py | 10 ++-------- 6 files changed, 14 insertions(+), 45 deletions(-) diff --git a/tests/test_dynapcnn/test_auto_mapping.py b/tests/test_dynapcnn/test_auto_mapping.py index 37de88d9..cff40a75 100644 --- a/tests/test_dynapcnn/test_auto_mapping.py +++ b/tests/test_dynapcnn/test_auto_mapping.py @@ -48,4 +48,4 @@ def test_auto_mapping_should_not_work(): graph = make_flow_graph(layer_mapping) new_graph = edmonds(graph, 0, len(graph) - 1) with pytest.raises(ValueError): - mapping = recover_mapping(new_graph, layer_mapping) + mapping = recover_mapping(new_graph, len(layer_mapping)) diff --git a/tests/test_dynapcnn/test_individual_cases.py b/tests/test_dynapcnn/test_individual_cases.py index 5256daba..dc656b28 100644 --- a/tests/test_dynapcnn/test_individual_cases.py +++ b/tests/test_dynapcnn/test_individual_cases.py @@ -23,8 +23,6 @@ def reset_states(seq): def networks_equal_output(input_data, snn): snn.eval() - snn_out = snn(input_data).squeeze() # forward pass - reset_states(snn) spn = DynapcnnNetwork( snn, @@ -32,15 +30,14 @@ def networks_equal_output(input_data, snn): discretize=False, dvs_input=True, ) - print(spn) + + snn_out = snn(input_data).squeeze() # forward pass spn_out = spn(input_data).squeeze() - print(snn_out.sum(), spn_out.sum()) assert torch.equal(snn_out, spn_out) # this will give an error if the config is not compatible config = spn.make_config() - print(spn.chip_layers_ordering) return config @@ -61,10 +58,9 @@ def forward(self, x): snn = from_model(Net().seq, batch_size=1) snn.eval() - snn_out = snn(input_data).squeeze() # forward pass - - snn.reset_states() spn = DynapcnnNetwork(snn, input_shape=input_data.shape[1:], discretize=False) + + snn_out = snn(input_data).squeeze() # forward pass spn_out = spn(input_data).squeeze() assert torch.equal(snn_out, spn_out) diff --git a/tests/test_dynapcnnlayer/model_dummy_1.py b/tests/test_dynapcnnlayer/model_dummy_1.py index 8abc7367..ffe5bac6 100644 --- a/tests/test_dynapcnnlayer/model_dummy_1.py +++ b/tests/test_dynapcnnlayer/model_dummy_1.py @@ -10,7 +10,6 @@ dcnnl_map_1 = { 0: { "input_shape": (2, 34, 34), - "inferred_input_shapes": set(), "rescale_factors": set(), "is_entry_node": True, "conv": { @@ -45,7 +44,6 @@ }, 1: { "input_shape": (10, 11, 11), - "inferred_input_shapes": set(((10, 11, 11),)), "rescale_factors": set(), "is_entry_node": False, "conv": { @@ -72,7 +70,6 @@ }, 2: { "input_shape": (10, 8, 8), - "inferred_input_shapes": set(((10, 8, 8),)), "rescale_factors": set(), "is_entry_node": False, "conv": { @@ -98,8 +95,7 @@ ], }, 3: { - "input_shape": (49, 1, 1), - "inferred_input_shapes": set(((1, 7, 7),)), + "input_shape": (1, 7, 7), "rescale_factors": set(), "is_entry_node": False, "conv": { @@ -126,7 +122,6 @@ }, 4: { "input_shape": (500, 1, 1), - "inferred_input_shapes": set(((500, 1, 1),)), "rescale_factors": set(), "is_entry_node": False, "conv": { @@ -148,7 +143,7 @@ "pooling_modules": [], "destination_layer": None, } - ] + ], }, } @@ -195,5 +190,5 @@ 2: [3], 3: [4], 4: [-1], - } + }, } diff --git a/tests/test_dynapcnnlayer/model_dummy_2.py b/tests/test_dynapcnnlayer/model_dummy_2.py index 552c6fd2..aa0e086e 100644 --- a/tests/test_dynapcnnlayer/model_dummy_2.py +++ b/tests/test_dynapcnnlayer/model_dummy_2.py @@ -10,7 +10,6 @@ dcnnl_map_2 = { 0: { "input_shape": (2, 34, 34), - "inferred_input_shapes": set(), "rescale_factors": set(), "is_entry_node": True, "conv": { @@ -37,7 +36,6 @@ }, 1: { "input_shape": (4, 33, 33), - "inferred_input_shapes": set(((4, 33, 33),)), "rescale_factors": set(), "is_entry_node": False, "conv": { @@ -74,7 +72,6 @@ }, 2: { "input_shape": (4, 16, 16), - "inferred_input_shapes": set(((4, 16, 16),)), "rescale_factors": set(), "is_entry_node": False, "conv": { @@ -103,7 +100,6 @@ }, 3: { "input_shape": (4, 16, 16), - "inferred_input_shapes": set(((4, 16, 16),)), "rescale_factors": set(), "is_entry_node": False, "conv": { @@ -132,7 +128,6 @@ }, 4: { "input_shape": (4, 7, 7), - "inferred_input_shapes": set(((4, 7, 7),)), "rescale_factors": set(), "is_entry_node": False, "conv": { @@ -158,8 +153,7 @@ ], }, 5: { - "input_shape": (144, 1, 1), - "inferred_input_shapes": set(((4, 6, 6),)), + "input_shape": (4, 6, 6), "rescale_factors": set(), "is_entry_node": False, "conv": { @@ -179,7 +173,6 @@ }, 6: { "input_shape": (4, 7, 7), - "inferred_input_shapes": set(((4, 7, 7),)), "rescale_factors": set(), "is_entry_node": False, "conv": { @@ -258,5 +251,5 @@ 4: [5], 6: [5], 5: [], - } + }, } diff --git a/tests/test_dynapcnnlayer/model_dummy_3.py b/tests/test_dynapcnnlayer/model_dummy_3.py index 24f963df..80564399 100644 --- a/tests/test_dynapcnnlayer/model_dummy_3.py +++ b/tests/test_dynapcnnlayer/model_dummy_3.py @@ -10,7 +10,6 @@ dcnnl_map_3 = { 0: { "input_shape": (2, 34, 34), - "inferred_input_shapes": set(), "rescale_factors": set(), "is_entry_node": True, "conv": { @@ -37,7 +36,6 @@ }, 1: { "input_shape": (4, 33, 33), - "inferred_input_shapes": set(((4, 33, 33),)), "rescale_factors": set(), "is_entry_node": False, "conv": { @@ -66,7 +64,6 @@ }, 2: { "input_shape": (4, 16, 16), - "inferred_input_shapes": set(((4, 16, 16),)), "rescale_factors": set(), "is_entry_node": False, "conv": { @@ -94,8 +91,7 @@ ], }, 3: { - "input_shape": (196, 1, 1), - "inferred_input_shapes": set(((4, 7, 7),)), + "input_shape": (4, 7, 7), "rescale_factors": set(), "is_entry_node": False, "conv": { @@ -122,7 +118,6 @@ }, 4: { "input_shape": (2, 34, 34), - "inferred_input_shapes": set(), "rescale_factors": set(), "is_entry_node": True, "conv": { @@ -149,7 +144,6 @@ }, 5: { "input_shape": (4, 33, 33), - "inferred_input_shapes": set(((4, 33, 33),)), "rescale_factors": set(), "is_entry_node": False, "conv": { @@ -178,7 +172,6 @@ }, 6: { "input_shape": (4, 16, 16), - "inferred_input_shapes": set(((4, 16, 16),)), "rescale_factors": set(), "is_entry_node": False, "conv": { @@ -207,7 +200,6 @@ }, 7: { "input_shape": (100, 1, 1), - "inferred_input_shapes": set(((100, 1, 1),)), "rescale_factors": set(), "is_entry_node": False, "conv": { @@ -234,7 +226,6 @@ }, 8: { "input_shape": (100, 1, 1), - "inferred_input_shapes": set(((100, 1, 1),)), "rescale_factors": set(), "is_entry_node": False, "conv": { @@ -326,5 +317,5 @@ 6: [3], 7: [8], 8: [-1], - } + }, } diff --git a/tests/test_dynapcnnlayer/model_dummy_4.py b/tests/test_dynapcnnlayer/model_dummy_4.py index e305a247..4dc1e223 100644 --- a/tests/test_dynapcnnlayer/model_dummy_4.py +++ b/tests/test_dynapcnnlayer/model_dummy_4.py @@ -10,7 +10,6 @@ dcnnl_map_4 = { 0: { "input_shape": (2, 34, 34), - "inferred_input_shapes": set(), "rescale_factors": set(), "is_entry_node": True, "conv": { @@ -43,7 +42,6 @@ }, 1: { "input_shape": (1, 33, 33), - "inferred_input_shapes": set(((1, 33, 33),)), "rescale_factors": set(), "is_entry_node": False, "conv": { @@ -72,7 +70,6 @@ }, 2: { "input_shape": (1, 33, 33), - "inferred_input_shapes": set(((1, 33, 33),)), "rescale_factors": set(), "is_entry_node": False, "conv": { @@ -109,7 +106,6 @@ }, 3: { "input_shape": (1, 16, 16), - "inferred_input_shapes": set(((1, 16, 16),)), "rescale_factors": set(), "is_entry_node": False, "conv": { @@ -138,7 +134,6 @@ }, 4: { "input_shape": (1, 6, 6), - "inferred_input_shapes": set(((1, 6, 6),)), "rescale_factors": set(), "is_entry_node": False, "conv": { @@ -164,8 +159,7 @@ ], }, 5: { - "input_shape": (25, 1, 1), - "inferred_input_shapes": set(((1, 5, 5),)), + "input_shape": (1, 5, 5), "rescale_factors": set(), "is_entry_node": False, "conv": { @@ -230,5 +224,5 @@ 3: [5], 4: [5], 5: [], - } + }, } From b68e588bd9e3d7cd9402d464800d7418a9438b8c Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Wed, 30 Oct 2024 21:16:06 +0100 Subject: [PATCH 277/379] DONE - chip deployment with DVS config_builder.build_config() returns 'Network is valid'. --- sinabs/backend/dynapcnn/chips/dynapcnn.py | 26 +++++++++---------- sinabs/backend/dynapcnn/dynapcnn_network.py | 8 +++--- .../dynapcnn/dynapcnnnetwork_module.py | 6 +---- .../backend/dynapcnn/sinabs_edges_handler.py | 2 +- 4 files changed, 18 insertions(+), 24 deletions(-) diff --git a/sinabs/backend/dynapcnn/chips/dynapcnn.py b/sinabs/backend/dynapcnn/chips/dynapcnn.py index eaf952eb..6e4fb536 100644 --- a/sinabs/backend/dynapcnn/chips/dynapcnn.py +++ b/sinabs/backend/dynapcnn/chips/dynapcnn.py @@ -33,7 +33,6 @@ def get_dvs_layer_config_dict(cls, layer: DVSLayer): ... # def write_dvs_layer_config(cls, layer: DVSLayer, config: DVSLayerConfig): # for param, value in layer.get_config_dict().items(): # setattr(config, param, value) - @classmethod def write_dvs_layer_config( @@ -51,7 +50,7 @@ def write_dvs_layer_config( ---------- - layer (DVSLayer): Layer instance from which to generate the config - layer2core_map (Dict): Keys are layer indices, values are corresponding - cores on hardware. Needed to map the destinations.] + cores on hardware. Needed to map the destinations. - destination_indices (List): Indices of destination layers for `layer` - chip_layer (DVSLayerConfig): Configuration object of the corrsesponding on-chip core. Will be changed in-place based on `layer`. @@ -295,7 +294,7 @@ def build_config( ---------- - layers (Dict): Keys are layer indices, values are DynapcnnLayer instances. - layer2core_map (Dict): Keys are layer indices, values are corresponding - cores on hardware. Needed to map the destinations.] + cores on hardware. Needed to map the destinations. - destination_indices (List): Indices of destination layers for `layer` - dvs_node_info (dict): contains information associated with the `DVSLayer` node (if no DVS node exists it'll return `None`). @@ -307,16 +306,6 @@ def build_config( config = cls.get_default_config() config.dvs_layer.pass_sensor_events = False - # Uses the DVS camera. - if isinstance(dvs_node_info, dict): - chip_layer = config.dvs_layer - sw_layer = dvs_node_info['module'] - destination_indices = dvs_node_info['destinations'] - # Write camera configuration. - cls.write_dvs_layer_config(sw_layer, layer2core_map, destination_indices, chip_layer) - - # TODO - for now it's being handled separatly but it might make more sense to handle it within `layers`. - # Loop over layers in network and write corresponding configurations for layer_index, ith_dcnnl in layers.items(): if isinstance(ith_dcnnl, DynapcnnLayer): @@ -329,6 +318,17 @@ def build_config( chip_layer=chip_layer, destination_indices=destination_map[layer_index], ) + elif isinstance(ith_dcnnl, DVSLayer) and isinstance(dvs_node_info, dict): + # Uses the DVS camera. + chip_layer = config.dvs_layer + sw_layer = ith_dcnnl + destination_indices = destination_map[layer_index] + # Write camera configuration. + cls.write_dvs_layer_config( + layer=sw_layer, + layer2core_map=layer2core_map, + destination_indices=destination_indices, + chip_layer=chip_layer) else: # shouldn't happen since type checks are made previously. raise TypeError( diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index d195c6de..67cb8f0e 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -508,8 +508,6 @@ def _make_config( the provided device can be found. """ config_builder = ChipFactory(device).get_config_builder() - - # TODO not handling DVSLayer yet. has_dvs_layer = self.has_dvs_layer() if chip_layers_ordering is not None: @@ -545,7 +543,7 @@ def _make_config( ) if has_dvs_layer: - # TODO not handling DVSLayer yet. + # TODO - DVS layer has been incorporated: what should happen here? pass self._layer2core_map = layer2core_map @@ -554,10 +552,10 @@ def _make_config( layers=self.dynapcnn_layers, destination_map=self.layer_destination_map, layer2core_map=layer2core_map, - dvs_node_info=dvs_node_info, + dvs_node_info=self.dvs_node_info, ) - # TODO not handling DVSLayer yet (this is from the old implementation, should be revised). + # TODO - this is from the old implementation, should be revised. if self.input_shape and self.input_shape[0] == 1: config.dvs_layer.merge = True diff --git a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py index 009946a8..7f290c57 100644 --- a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py +++ b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py @@ -80,11 +80,7 @@ def destination_map(self): @property def dynapcnn_layers(self): # Convert string-indices to integers-indices - dynapcnn_layers = {int(idx): lyr for idx, lyr in self._dynapcnn_layers.items()} - # Insert DVS node if DVS was enabled. - if isinstance(self.dvs_node_info, Dict): - dynapcnn_layers[self.dvs_node_info['layer_id']] = self.dvs_node_info['module'] - return dynapcnn_layers + return {int(idx): lyr for idx, lyr in self._dynapcnn_layers.items()} @property def entry_points(self): diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index 656c93d4..d050bcd3 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -71,7 +71,7 @@ def fix_dvs_module_edges(edges: Set[Edge], indx_2_module_map: Dict[int, nn.Modul if any(isinstance(module, dvs_node) for dvs_node in (DVSLayer, Crop2d, FlipDims)) } - if len(dvslayer_nodes) == 1: + if len(dvslayer_nodes) <= 1: # No module within the DVSLayer instance appears as an independent node - nothing to do here. return From e3ad6a5ade1949595d9cd23494f1e89f5ccfe3be Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Wed, 30 Oct 2024 21:17:22 +0100 Subject: [PATCH 278/379] notebooks used to validate deployment with DVS --- .../dynapcnn_network/snn_DVSLayer_given.ipynb | 33 +- .../snn_need_create_DVSLayer.ipynb | 30 ++ .../dynapcnn_network/snn_no_DVSLayer.ipynb | 291 ++++++++++++++++++ 3 files changed, 353 insertions(+), 1 deletion(-) create mode 100644 examples/dynapcnn_network/snn_no_DVSLayer.ipynb diff --git a/examples/dynapcnn_network/snn_DVSLayer_given.ipynb b/examples/dynapcnn_network/snn_DVSLayer_given.ipynb index 986660aa..7a3db2e4 100644 --- a/examples/dynapcnn_network/snn_DVSLayer_given.ipynb +++ b/examples/dynapcnn_network/snn_DVSLayer_given.ipynb @@ -18,7 +18,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 1, @@ -243,6 +243,37 @@ "source": [ "print(out)" ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "DVSLayer with index: 0\n", + "Network is valid\n" + ] + }, + { + "ename": "KeyError", + "evalue": "'speck2fdevkit:0'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mKeyError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[8], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mhw_model\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mto\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdevice\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mspeck2fdevkit\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/Github/sinabs/sinabs/backend/dynapcnn/dynapcnn_network.py:302\u001b[0m, in \u001b[0;36mDynapcnnNetwork.to\u001b[0;34m(self, device, monitor_layers, config_modifier, slow_clk_frequency, layer2core_map, chip_layers_ordering)\u001b[0m\n\u001b[1;32m 293\u001b[0m config \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmake_config(\n\u001b[1;32m 294\u001b[0m layer2core_map\u001b[38;5;241m=\u001b[39mlayer2core_map,\n\u001b[1;32m 295\u001b[0m chip_layers_ordering\u001b[38;5;241m=\u001b[39mchip_layers_ordering,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 298\u001b[0m config_modifier\u001b[38;5;241m=\u001b[39mconfig_modifier,\n\u001b[1;32m 299\u001b[0m )\n\u001b[1;32m 301\u001b[0m \u001b[38;5;66;03m# apply configuration to device.\u001b[39;00m\n\u001b[0;32m--> 302\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msamna_device \u001b[38;5;241m=\u001b[39m \u001b[43mopen_device\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdevice\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 303\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msamna_device\u001b[38;5;241m.\u001b[39mget_model()\u001b[38;5;241m.\u001b[39mapply_configuration(config)\n\u001b[1;32m 304\u001b[0m time\u001b[38;5;241m.\u001b[39msleep(\u001b[38;5;241m1\u001b[39m)\n", + "File \u001b[0;32m~/Github/sinabs/sinabs/backend/dynapcnn/io.py:255\u001b[0m, in \u001b[0;36mopen_device\u001b[0;34m(device_id)\u001b[0m\n\u001b[1;32m 253\u001b[0m device_id \u001b[38;5;241m=\u001b[39m standardize_device_id(device_id\u001b[38;5;241m=\u001b[39mdevice_id)\n\u001b[1;32m 254\u001b[0m device_map \u001b[38;5;241m=\u001b[39m get_device_map()\n\u001b[0;32m--> 255\u001b[0m device_info \u001b[38;5;241m=\u001b[39m \u001b[43mdevice_map\u001b[49m\u001b[43m[\u001b[49m\u001b[43mdevice_id\u001b[49m\u001b[43m]\u001b[49m\n\u001b[1;32m 256\u001b[0m device_handle \u001b[38;5;241m=\u001b[39m samna\u001b[38;5;241m.\u001b[39mdevice\u001b[38;5;241m.\u001b[39mopen_device(device_info)\n\u001b[1;32m 258\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m device_handle \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n", + "\u001b[0;31mKeyError\u001b[0m: 'speck2fdevkit:0'" + ] + } + ], + "source": [ + "hw_model.to(device=\"speck2fdevkit\")" + ] } ], "metadata": { diff --git a/examples/dynapcnn_network/snn_need_create_DVSLayer.ipynb b/examples/dynapcnn_network/snn_need_create_DVSLayer.ipynb index c6d195de..fa9de7f1 100644 --- a/examples/dynapcnn_network/snn_need_create_DVSLayer.ipynb +++ b/examples/dynapcnn_network/snn_need_create_DVSLayer.ipynb @@ -182,6 +182,36 @@ " discretize=True,\n", ")" ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Network is valid\n" + ] + }, + { + "ename": "KeyError", + "evalue": "'speck2fdevkit:0'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mKeyError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[6], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mhw_model\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mto\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdevice\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mspeck2fdevkit\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/Github/sinabs/sinabs/backend/dynapcnn/dynapcnn_network.py:302\u001b[0m, in \u001b[0;36mDynapcnnNetwork.to\u001b[0;34m(self, device, monitor_layers, config_modifier, slow_clk_frequency, layer2core_map, chip_layers_ordering)\u001b[0m\n\u001b[1;32m 293\u001b[0m config \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmake_config(\n\u001b[1;32m 294\u001b[0m layer2core_map\u001b[38;5;241m=\u001b[39mlayer2core_map,\n\u001b[1;32m 295\u001b[0m chip_layers_ordering\u001b[38;5;241m=\u001b[39mchip_layers_ordering,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 298\u001b[0m config_modifier\u001b[38;5;241m=\u001b[39mconfig_modifier,\n\u001b[1;32m 299\u001b[0m )\n\u001b[1;32m 301\u001b[0m \u001b[38;5;66;03m# apply configuration to device.\u001b[39;00m\n\u001b[0;32m--> 302\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msamna_device \u001b[38;5;241m=\u001b[39m \u001b[43mopen_device\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdevice\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 303\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msamna_device\u001b[38;5;241m.\u001b[39mget_model()\u001b[38;5;241m.\u001b[39mapply_configuration(config)\n\u001b[1;32m 304\u001b[0m time\u001b[38;5;241m.\u001b[39msleep(\u001b[38;5;241m1\u001b[39m)\n", + "File \u001b[0;32m~/Github/sinabs/sinabs/backend/dynapcnn/io.py:255\u001b[0m, in \u001b[0;36mopen_device\u001b[0;34m(device_id)\u001b[0m\n\u001b[1;32m 253\u001b[0m device_id \u001b[38;5;241m=\u001b[39m standardize_device_id(device_id\u001b[38;5;241m=\u001b[39mdevice_id)\n\u001b[1;32m 254\u001b[0m device_map \u001b[38;5;241m=\u001b[39m get_device_map()\n\u001b[0;32m--> 255\u001b[0m device_info \u001b[38;5;241m=\u001b[39m \u001b[43mdevice_map\u001b[49m\u001b[43m[\u001b[49m\u001b[43mdevice_id\u001b[49m\u001b[43m]\u001b[49m\n\u001b[1;32m 256\u001b[0m device_handle \u001b[38;5;241m=\u001b[39m samna\u001b[38;5;241m.\u001b[39mdevice\u001b[38;5;241m.\u001b[39mopen_device(device_info)\n\u001b[1;32m 258\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m device_handle \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n", + "\u001b[0;31mKeyError\u001b[0m: 'speck2fdevkit:0'" + ] + } + ], + "source": [ + "hw_model.to(device=\"speck2fdevkit\")" + ] } ], "metadata": { diff --git a/examples/dynapcnn_network/snn_no_DVSLayer.ipynb b/examples/dynapcnn_network/snn_no_DVSLayer.ipynb new file mode 100644 index 00000000..86daa058 --- /dev/null +++ b/examples/dynapcnn_network/snn_no_DVSLayer.ipynb @@ -0,0 +1,291 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "metadata": {} + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/samurai2077/anaconda3/envs/speck-rescnn/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import torch\n", + "import torch.nn as nn\n", + "from sinabs.backend.dynapcnn import DynapcnnNetwork\n", + "from sinabs.backend.dynapcnn import DVSLayer\n", + "from sinabs.layers import Merge, IAFSqueeze, SumPool2d\n", + "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential\n", + "import sinabs.layers as sl\n", + "\n", + "from torch.nn import CrossEntropyLoss\n", + "from torch.optim import Adam\n", + "\n", + "from tonic.datasets.nmnist import NMNIST\n", + "from tonic.transforms import ToFrame\n", + "from torch.utils.data import DataLoader\n", + "import numpy as np\n", + "from tqdm.notebook import tqdm\n", + "from statistics import mode\n", + "\n", + "device = torch.device('cpu')\n", + "torch.manual_seed(0)" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "channels = 1\n", + "height = 34\n", + "width = 34\n", + "batch_size = 1\n", + "\n", + "input_shape = (channels, height, width)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "metadata": {} + }, + "outputs": [ + { + "data": { + "text/plain": [ + "SNN(\n", + " (conv1): Conv2d(1, 10, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + " (iaf1): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(1.), min_v_mem=Parameter containing:\n", + " tensor(-1.), batch_size=1, num_timesteps=-1)\n", + " (pool1): AvgPool2d(kernel_size=2, stride=2, padding=0)\n", + " (conv2): Conv2d(10, 10, kernel_size=(4, 4), stride=(1, 1), bias=False)\n", + " (iaf2): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(1.), min_v_mem=Parameter containing:\n", + " tensor(-1.), batch_size=1, num_timesteps=-1)\n", + " (conv3): Conv2d(10, 1, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + " (iaf3): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(1.), min_v_mem=Parameter containing:\n", + " tensor(-1.), batch_size=1, num_timesteps=-1)\n", + " (fc1): Linear(in_features=144, out_features=200, bias=False)\n", + " (iaf4): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(1.), min_v_mem=Parameter containing:\n", + " tensor(-1.), batch_size=1, num_timesteps=-1)\n", + " (fc2): Linear(in_features=200, out_features=10, bias=False)\n", + " (iaf5): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(1.), min_v_mem=Parameter containing:\n", + " tensor(-1.), batch_size=1, num_timesteps=-1)\n", + " (flat): Flatten(start_dim=1, end_dim=-1)\n", + ")" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "class SNN(nn.Module):\n", + " def __init__(self, input_shape) -> None:\n", + " super().__init__()\n", + "\n", + " # -- chip core A --\n", + " self.conv1 = nn.Conv2d(1, 10, 2, 1, bias=False)\n", + " self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential())\n", + " self.pool1 = nn.AvgPool2d(2,2)\n", + " # -- chip core B --\n", + " self.conv2 = nn.Conv2d(10, 10, 4, 1, bias=False)\n", + " self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential())\n", + " # -- chip core C --\n", + " self.conv3 = nn.Conv2d(10, 1, 2, 1, bias=False)\n", + " self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential())\n", + " # -- chip core D --\n", + " self.fc1 = nn.Linear(144, 200, bias=False)\n", + " self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential())\n", + " # -- chip core E --\n", + " self.fc2 = nn.Linear(200, 10, bias=False)\n", + " self.iaf5 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential())\n", + "\n", + " # -- layers ignored during deployment --\n", + " self.flat = nn.Flatten()\n", + "\n", + " def init_weights(self):\n", + " for name, layer in self.named_modules():\n", + " if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear):\n", + " nn.init.xavier_normal_(layer.weight.data)\n", + "\n", + " def detach_neuron_states(self):\n", + " for name, layer in self.named_modules():\n", + " if name != '':\n", + " if isinstance(layer, sl.StatefulLayer):\n", + " for name, buffer in layer.named_buffers():\n", + " buffer.detach_()\n", + "\n", + " def forward(self, x):\n", + " \n", + " con1_out = self.conv1(x) # 4\n", + " iaf1_out = self.iaf1(con1_out) # 5\n", + " pool1_out = self.pool1(iaf1_out) # 6\n", + "\n", + " conv2_out = self.conv2(pool1_out) # 7\n", + " iaf2_out = self.iaf2(conv2_out) # 8\n", + "\n", + " conv3_out = self.conv3(iaf2_out) # 9\n", + " iaf3_out = self.iaf3(conv3_out) # 10\n", + "\n", + " flat_out = self.flat(iaf3_out) # 15\n", + " \n", + " fc1_out = self.fc1(flat_out) # 11\n", + " iaf4_out = self.iaf4(fc1_out) # 12\n", + " fc2_out = self.fc2(iaf4_out) # 13\n", + " iaf5_out = self.iaf5(fc2_out) # 14\n", + "\n", + " return iaf5_out\n", + " \n", + "snn = SNN(input_shape)\n", + "snn.init_weights()\n", + "snn.to(device)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "metadata": {} + }, + "outputs": [], + "source": [ + "hw_model = DynapcnnNetwork(\n", + " snn=snn,\n", + " input_shape=input_shape,\n", + " batch_size=batch_size,\n", + " discretize=True\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "input_dummy = torch.randn((batch_size, *input_shape))" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "out = hw_model(input_dummy)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "tensor([[[[0.]],\n", + "\n", + " [[0.]],\n", + "\n", + " [[0.]],\n", + "\n", + " [[0.]],\n", + "\n", + " [[0.]],\n", + "\n", + " [[0.]],\n", + "\n", + " [[0.]],\n", + "\n", + " [[0.]],\n", + "\n", + " [[0.]],\n", + "\n", + " [[0.]]]], grad_fn=)\n" + ] + } + ], + "source": [ + "print(out)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Network is valid\n" + ] + }, + { + "ename": "KeyError", + "evalue": "'speck2fdevkit:0'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mKeyError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[8], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mhw_model\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mto\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdevice\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mspeck2fdevkit\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/Github/sinabs/sinabs/backend/dynapcnn/dynapcnn_network.py:302\u001b[0m, in \u001b[0;36mDynapcnnNetwork.to\u001b[0;34m(self, device, monitor_layers, config_modifier, slow_clk_frequency, layer2core_map, chip_layers_ordering)\u001b[0m\n\u001b[1;32m 293\u001b[0m config \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmake_config(\n\u001b[1;32m 294\u001b[0m layer2core_map\u001b[38;5;241m=\u001b[39mlayer2core_map,\n\u001b[1;32m 295\u001b[0m chip_layers_ordering\u001b[38;5;241m=\u001b[39mchip_layers_ordering,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 298\u001b[0m config_modifier\u001b[38;5;241m=\u001b[39mconfig_modifier,\n\u001b[1;32m 299\u001b[0m )\n\u001b[1;32m 301\u001b[0m \u001b[38;5;66;03m# apply configuration to device.\u001b[39;00m\n\u001b[0;32m--> 302\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msamna_device \u001b[38;5;241m=\u001b[39m \u001b[43mopen_device\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdevice\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 303\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msamna_device\u001b[38;5;241m.\u001b[39mget_model()\u001b[38;5;241m.\u001b[39mapply_configuration(config)\n\u001b[1;32m 304\u001b[0m time\u001b[38;5;241m.\u001b[39msleep(\u001b[38;5;241m1\u001b[39m)\n", + "File \u001b[0;32m~/Github/sinabs/sinabs/backend/dynapcnn/io.py:255\u001b[0m, in \u001b[0;36mopen_device\u001b[0;34m(device_id)\u001b[0m\n\u001b[1;32m 253\u001b[0m device_id \u001b[38;5;241m=\u001b[39m standardize_device_id(device_id\u001b[38;5;241m=\u001b[39mdevice_id)\n\u001b[1;32m 254\u001b[0m device_map \u001b[38;5;241m=\u001b[39m get_device_map()\n\u001b[0;32m--> 255\u001b[0m device_info \u001b[38;5;241m=\u001b[39m \u001b[43mdevice_map\u001b[49m\u001b[43m[\u001b[49m\u001b[43mdevice_id\u001b[49m\u001b[43m]\u001b[49m\n\u001b[1;32m 256\u001b[0m device_handle \u001b[38;5;241m=\u001b[39m samna\u001b[38;5;241m.\u001b[39mdevice\u001b[38;5;241m.\u001b[39mopen_device(device_info)\n\u001b[1;32m 258\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m device_handle \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n", + "\u001b[0;31mKeyError\u001b[0m: 'speck2fdevkit:0'" + ] + } + ], + "source": [ + "hw_model.to(device=\"speck2fdevkit\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "speck-rescnn", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From b1b4ec7a8faaae8c5176c566771ad590327289e4 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Wed, 30 Oct 2024 21:21:39 +0100 Subject: [PATCH 279/379] notebooks used to validate deployment with DVS --- .../dynapcnn_network/snn_DVSLayer_given.ipynb | 91 ++++++++++++------- .../snn_need_create_DVSLayer.ipynb | 85 +++++++++++------ .../dynapcnn_network/snn_no_DVSLayer.ipynb | 57 +++++++++--- 3 files changed, 162 insertions(+), 71 deletions(-) diff --git a/examples/dynapcnn_network/snn_DVSLayer_given.ipynb b/examples/dynapcnn_network/snn_DVSLayer_given.ipynb index 7a3db2e4..7ccde095 100644 --- a/examples/dynapcnn_network/snn_DVSLayer_given.ipynb +++ b/examples/dynapcnn_network/snn_DVSLayer_given.ipynb @@ -2,26 +2,18 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": 17, "metadata": { "metadata": {} }, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/samurai2077/anaconda3/envs/speck-rescnn/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", - " from .autonotebook import tqdm as notebook_tqdm\n" - ] - }, { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 1, + "execution_count": 17, "metadata": {}, "output_type": "execute_result" } @@ -51,7 +43,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 18, "metadata": {}, "outputs": [], "source": [ @@ -65,7 +57,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 19, "metadata": { "metadata": {} }, @@ -104,7 +96,7 @@ ")" ] }, - "execution_count": 3, + "execution_count": 19, "metadata": {}, "output_type": "execute_result" } @@ -177,7 +169,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 20, "metadata": { "metadata": {} }, @@ -193,7 +185,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 21, "metadata": {}, "outputs": [], "source": [ @@ -202,7 +194,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 22, "metadata": {}, "outputs": [], "source": [ @@ -211,7 +203,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 23, "metadata": {}, "outputs": [ { @@ -246,29 +238,66 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 24, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "DVSLayer with index: 0\n", "Network is valid\n" ] }, { - "ename": "KeyError", - "evalue": "'speck2fdevkit:0'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mKeyError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[8], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mhw_model\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mto\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdevice\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mspeck2fdevkit\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/Github/sinabs/sinabs/backend/dynapcnn/dynapcnn_network.py:302\u001b[0m, in \u001b[0;36mDynapcnnNetwork.to\u001b[0;34m(self, device, monitor_layers, config_modifier, slow_clk_frequency, layer2core_map, chip_layers_ordering)\u001b[0m\n\u001b[1;32m 293\u001b[0m config \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmake_config(\n\u001b[1;32m 294\u001b[0m layer2core_map\u001b[38;5;241m=\u001b[39mlayer2core_map,\n\u001b[1;32m 295\u001b[0m chip_layers_ordering\u001b[38;5;241m=\u001b[39mchip_layers_ordering,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 298\u001b[0m config_modifier\u001b[38;5;241m=\u001b[39mconfig_modifier,\n\u001b[1;32m 299\u001b[0m )\n\u001b[1;32m 301\u001b[0m \u001b[38;5;66;03m# apply configuration to device.\u001b[39;00m\n\u001b[0;32m--> 302\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msamna_device \u001b[38;5;241m=\u001b[39m \u001b[43mopen_device\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdevice\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 303\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msamna_device\u001b[38;5;241m.\u001b[39mget_model()\u001b[38;5;241m.\u001b[39mapply_configuration(config)\n\u001b[1;32m 304\u001b[0m time\u001b[38;5;241m.\u001b[39msleep(\u001b[38;5;241m1\u001b[39m)\n", - "File \u001b[0;32m~/Github/sinabs/sinabs/backend/dynapcnn/io.py:255\u001b[0m, in \u001b[0;36mopen_device\u001b[0;34m(device_id)\u001b[0m\n\u001b[1;32m 253\u001b[0m device_id \u001b[38;5;241m=\u001b[39m standardize_device_id(device_id\u001b[38;5;241m=\u001b[39mdevice_id)\n\u001b[1;32m 254\u001b[0m device_map \u001b[38;5;241m=\u001b[39m get_device_map()\n\u001b[0;32m--> 255\u001b[0m device_info \u001b[38;5;241m=\u001b[39m \u001b[43mdevice_map\u001b[49m\u001b[43m[\u001b[49m\u001b[43mdevice_id\u001b[49m\u001b[43m]\u001b[49m\n\u001b[1;32m 256\u001b[0m device_handle \u001b[38;5;241m=\u001b[39m samna\u001b[38;5;241m.\u001b[39mdevice\u001b[38;5;241m.\u001b[39mopen_device(device_info)\n\u001b[1;32m 258\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m device_handle \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n", - "\u001b[0;31mKeyError\u001b[0m: 'speck2fdevkit:0'" - ] + "data": { + "text/plain": [ + "DynapcnnNetwork(\n", + " (_dynapcnn_module): DynapcnnNetworkModule(\n", + " (_dynapcnn_layers): ModuleDict(\n", + " (1): DynapcnnLayer(\n", + " (_conv): Conv2d(1, 10, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + " (_spk): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(174.), min_v_mem=Parameter containing:\n", + " tensor(-174.), batch_size=1, num_timesteps=-1)\n", + " )\n", + " (5): DynapcnnLayer(\n", + " (_conv): Conv2d(200, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + " (_spk): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(419.), min_v_mem=Parameter containing:\n", + " tensor(-419.), batch_size=1, num_timesteps=-1)\n", + " )\n", + " (3): DynapcnnLayer(\n", + " (_conv): Conv2d(10, 1, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + " (_spk): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(303.), min_v_mem=Parameter containing:\n", + " tensor(-303.), batch_size=1, num_timesteps=-1)\n", + " )\n", + " (2): DynapcnnLayer(\n", + " (_conv): Conv2d(10, 10, kernel_size=(4, 4), stride=(1, 1), bias=False)\n", + " (_spk): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(130.), min_v_mem=Parameter containing:\n", + " tensor(-130.), batch_size=1, num_timesteps=-1)\n", + " )\n", + " (4): DynapcnnLayer(\n", + " (_conv): Conv2d(1, 200, kernel_size=(12, 12), stride=(1, 1), bias=False)\n", + " (_spk): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(365.), min_v_mem=Parameter containing:\n", + " tensor(-365.), batch_size=1, num_timesteps=-1)\n", + " )\n", + " (0): DVSLayer(\n", + " (pool_layer): SumPool2d(norm_type=1, kernel_size=(1, 1), stride=None, ceil_mode=False)\n", + " (crop_layer): Crop2d((0, 34), (0, 34))\n", + " (flip_layer): FlipDims()\n", + " )\n", + " )\n", + " (merge_layer): Merge()\n", + " )\n", + ")" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ diff --git a/examples/dynapcnn_network/snn_need_create_DVSLayer.ipynb b/examples/dynapcnn_network/snn_need_create_DVSLayer.ipynb index fa9de7f1..2bc77030 100644 --- a/examples/dynapcnn_network/snn_need_create_DVSLayer.ipynb +++ b/examples/dynapcnn_network/snn_need_create_DVSLayer.ipynb @@ -2,20 +2,11 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": 7, "metadata": { "metadata": {} }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/samurai2077/anaconda3/envs/speck-rescnn/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", - " from .autonotebook import tqdm as notebook_tqdm\n" - ] - } - ], + "outputs": [], "source": [ "import torch\n", "import torch.nn as nn\n", @@ -38,7 +29,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 8, "metadata": { "metadata": {} }, @@ -50,7 +41,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -64,7 +55,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 10, "metadata": { "metadata": {} }, @@ -98,7 +89,7 @@ ")" ] }, - "execution_count": 4, + "execution_count": 10, "metadata": {}, "output_type": "execute_result" } @@ -168,7 +159,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 11, "metadata": { "metadata": {} }, @@ -185,7 +176,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 12, "metadata": {}, "outputs": [ { @@ -196,17 +187,55 @@ ] }, { - "ename": "KeyError", - "evalue": "'speck2fdevkit:0'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mKeyError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[6], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mhw_model\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mto\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdevice\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mspeck2fdevkit\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/Github/sinabs/sinabs/backend/dynapcnn/dynapcnn_network.py:302\u001b[0m, in \u001b[0;36mDynapcnnNetwork.to\u001b[0;34m(self, device, monitor_layers, config_modifier, slow_clk_frequency, layer2core_map, chip_layers_ordering)\u001b[0m\n\u001b[1;32m 293\u001b[0m config \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmake_config(\n\u001b[1;32m 294\u001b[0m layer2core_map\u001b[38;5;241m=\u001b[39mlayer2core_map,\n\u001b[1;32m 295\u001b[0m chip_layers_ordering\u001b[38;5;241m=\u001b[39mchip_layers_ordering,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 298\u001b[0m config_modifier\u001b[38;5;241m=\u001b[39mconfig_modifier,\n\u001b[1;32m 299\u001b[0m )\n\u001b[1;32m 301\u001b[0m \u001b[38;5;66;03m# apply configuration to device.\u001b[39;00m\n\u001b[0;32m--> 302\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msamna_device \u001b[38;5;241m=\u001b[39m \u001b[43mopen_device\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdevice\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 303\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msamna_device\u001b[38;5;241m.\u001b[39mget_model()\u001b[38;5;241m.\u001b[39mapply_configuration(config)\n\u001b[1;32m 304\u001b[0m time\u001b[38;5;241m.\u001b[39msleep(\u001b[38;5;241m1\u001b[39m)\n", - "File \u001b[0;32m~/Github/sinabs/sinabs/backend/dynapcnn/io.py:255\u001b[0m, in \u001b[0;36mopen_device\u001b[0;34m(device_id)\u001b[0m\n\u001b[1;32m 253\u001b[0m device_id \u001b[38;5;241m=\u001b[39m standardize_device_id(device_id\u001b[38;5;241m=\u001b[39mdevice_id)\n\u001b[1;32m 254\u001b[0m device_map \u001b[38;5;241m=\u001b[39m get_device_map()\n\u001b[0;32m--> 255\u001b[0m device_info \u001b[38;5;241m=\u001b[39m \u001b[43mdevice_map\u001b[49m\u001b[43m[\u001b[49m\u001b[43mdevice_id\u001b[49m\u001b[43m]\u001b[49m\n\u001b[1;32m 256\u001b[0m device_handle \u001b[38;5;241m=\u001b[39m samna\u001b[38;5;241m.\u001b[39mdevice\u001b[38;5;241m.\u001b[39mopen_device(device_info)\n\u001b[1;32m 258\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m device_handle \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n", - "\u001b[0;31mKeyError\u001b[0m: 'speck2fdevkit:0'" - ] + "data": { + "text/plain": [ + "DynapcnnNetwork(\n", + " (_dynapcnn_module): DynapcnnNetworkModule(\n", + " (_dynapcnn_layers): ModuleDict(\n", + " (1): DynapcnnLayer(\n", + " (_conv): Conv2d(1, 10, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + " (_spk): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(174.), min_v_mem=Parameter containing:\n", + " tensor(-174.), batch_size=1, num_timesteps=-1)\n", + " )\n", + " (5): DynapcnnLayer(\n", + " (_conv): Conv2d(200, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + " (_spk): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(419.), min_v_mem=Parameter containing:\n", + " tensor(-419.), batch_size=1, num_timesteps=-1)\n", + " )\n", + " (2): DynapcnnLayer(\n", + " (_conv): Conv2d(10, 10, kernel_size=(4, 4), stride=(1, 1), bias=False)\n", + " (_spk): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(130.), min_v_mem=Parameter containing:\n", + " tensor(-130.), batch_size=1, num_timesteps=-1)\n", + " )\n", + " (3): DynapcnnLayer(\n", + " (_conv): Conv2d(10, 1, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + " (_spk): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(303.), min_v_mem=Parameter containing:\n", + " tensor(-303.), batch_size=1, num_timesteps=-1)\n", + " )\n", + " (4): DynapcnnLayer(\n", + " (_conv): Conv2d(1, 200, kernel_size=(12, 12), stride=(1, 1), bias=False)\n", + " (_spk): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(365.), min_v_mem=Parameter containing:\n", + " tensor(-365.), batch_size=1, num_timesteps=-1)\n", + " )\n", + " (0): DVSLayer(\n", + " (pool_layer): SumPool2d(norm_type=1, kernel_size=(1, 1), stride=None, ceil_mode=False)\n", + " (crop_layer): Crop2d((0, 34), (0, 34))\n", + " (flip_layer): FlipDims()\n", + " )\n", + " )\n", + " (merge_layer): Merge()\n", + " )\n", + ")" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ diff --git a/examples/dynapcnn_network/snn_no_DVSLayer.ipynb b/examples/dynapcnn_network/snn_no_DVSLayer.ipynb index 86daa058..aa42aeaf 100644 --- a/examples/dynapcnn_network/snn_no_DVSLayer.ipynb +++ b/examples/dynapcnn_network/snn_no_DVSLayer.ipynb @@ -18,7 +18,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 1, @@ -249,17 +249,50 @@ ] }, { - "ename": "KeyError", - "evalue": "'speck2fdevkit:0'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mKeyError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[8], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mhw_model\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mto\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdevice\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mspeck2fdevkit\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/Github/sinabs/sinabs/backend/dynapcnn/dynapcnn_network.py:302\u001b[0m, in \u001b[0;36mDynapcnnNetwork.to\u001b[0;34m(self, device, monitor_layers, config_modifier, slow_clk_frequency, layer2core_map, chip_layers_ordering)\u001b[0m\n\u001b[1;32m 293\u001b[0m config \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmake_config(\n\u001b[1;32m 294\u001b[0m layer2core_map\u001b[38;5;241m=\u001b[39mlayer2core_map,\n\u001b[1;32m 295\u001b[0m chip_layers_ordering\u001b[38;5;241m=\u001b[39mchip_layers_ordering,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 298\u001b[0m config_modifier\u001b[38;5;241m=\u001b[39mconfig_modifier,\n\u001b[1;32m 299\u001b[0m )\n\u001b[1;32m 301\u001b[0m \u001b[38;5;66;03m# apply configuration to device.\u001b[39;00m\n\u001b[0;32m--> 302\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msamna_device \u001b[38;5;241m=\u001b[39m \u001b[43mopen_device\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdevice\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 303\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msamna_device\u001b[38;5;241m.\u001b[39mget_model()\u001b[38;5;241m.\u001b[39mapply_configuration(config)\n\u001b[1;32m 304\u001b[0m time\u001b[38;5;241m.\u001b[39msleep(\u001b[38;5;241m1\u001b[39m)\n", - "File \u001b[0;32m~/Github/sinabs/sinabs/backend/dynapcnn/io.py:255\u001b[0m, in \u001b[0;36mopen_device\u001b[0;34m(device_id)\u001b[0m\n\u001b[1;32m 253\u001b[0m device_id \u001b[38;5;241m=\u001b[39m standardize_device_id(device_id\u001b[38;5;241m=\u001b[39mdevice_id)\n\u001b[1;32m 254\u001b[0m device_map \u001b[38;5;241m=\u001b[39m get_device_map()\n\u001b[0;32m--> 255\u001b[0m device_info \u001b[38;5;241m=\u001b[39m \u001b[43mdevice_map\u001b[49m\u001b[43m[\u001b[49m\u001b[43mdevice_id\u001b[49m\u001b[43m]\u001b[49m\n\u001b[1;32m 256\u001b[0m device_handle \u001b[38;5;241m=\u001b[39m samna\u001b[38;5;241m.\u001b[39mdevice\u001b[38;5;241m.\u001b[39mopen_device(device_info)\n\u001b[1;32m 258\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m device_handle \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n", - "\u001b[0;31mKeyError\u001b[0m: 'speck2fdevkit:0'" - ] + "data": { + "text/plain": [ + "DynapcnnNetwork(\n", + " (_dynapcnn_module): DynapcnnNetworkModule(\n", + " (_dynapcnn_layers): ModuleDict(\n", + " (0): DynapcnnLayer(\n", + " (_conv): Conv2d(1, 10, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + " (_spk): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(174.), min_v_mem=Parameter containing:\n", + " tensor(-174.), batch_size=1, num_timesteps=-1)\n", + " )\n", + " (4): DynapcnnLayer(\n", + " (_conv): Conv2d(200, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + " (_spk): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(419.), min_v_mem=Parameter containing:\n", + " tensor(-419.), batch_size=1, num_timesteps=-1)\n", + " )\n", + " (1): DynapcnnLayer(\n", + " (_conv): Conv2d(10, 10, kernel_size=(4, 4), stride=(1, 1), bias=False)\n", + " (_spk): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(130.), min_v_mem=Parameter containing:\n", + " tensor(-130.), batch_size=1, num_timesteps=-1)\n", + " )\n", + " (2): DynapcnnLayer(\n", + " (_conv): Conv2d(10, 1, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + " (_spk): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(303.), min_v_mem=Parameter containing:\n", + " tensor(-303.), batch_size=1, num_timesteps=-1)\n", + " )\n", + " (3): DynapcnnLayer(\n", + " (_conv): Conv2d(1, 200, kernel_size=(12, 12), stride=(1, 1), bias=False)\n", + " (_spk): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(365.), min_v_mem=Parameter containing:\n", + " tensor(-365.), batch_size=1, num_timesteps=-1)\n", + " )\n", + " )\n", + " (merge_layer): Merge()\n", + " )\n", + ")" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ From e30d67f8ebedc926eb68786b5921eea6797123d0 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Thu, 31 Oct 2024 11:03:09 +0100 Subject: [PATCH 280/379] Merge local diverging changes --- sinabs/backend/dynapcnn/__init__.py | 1 + sinabs/backend/dynapcnn/nir_graph_extractor.py | 12 ++++++------ sinabs/backend/dynapcnn/utils.py | 1 + 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/sinabs/backend/dynapcnn/__init__.py b/sinabs/backend/dynapcnn/__init__.py index 11ab0fd9..7fc4915f 100644 --- a/sinabs/backend/dynapcnn/__init__.py +++ b/sinabs/backend/dynapcnn/__init__.py @@ -2,3 +2,4 @@ from .dynapcnn_network import DynapcnnCompatibleNetwork, DynapcnnNetwork from .dynapcnnnetwork_module import DynapcnnNetworkModule from .dynapcnn_visualizer import DynapcnnVisualizer +from .nir_graph_extractor import GraphExtractor diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index 6a946a99..0a114a74 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -1,7 +1,7 @@ # author : Willian Soares Girao # contact : wsoaresgirao@gmail.com -from typing import Callable, Dict, Iterable, List, Optional, Set, Tuple, Type, Union +from typing import Callable, Dict, Iterable, List, Optional, Set, Tuple, Type import nirtorch import torch @@ -246,12 +246,12 @@ def verify_graph_integrity(self): ####################################################### Pivate Methods ####################################################### - def _get_name_2_indx_map(self, nir_graph: nirtorch.graph.Graph) -> Dict[str, int]: + def _get_name_2_indx_map(self, nir_graph: nirtorch.graph.TorchGraph) -> Dict[str, int]: """Assign unique index to each node and return mapper from name to index. Parameters ---------- - - nir_graph (nirtorch.graph.Graph): a NIR graph representation of `spiking_model`. + - nir_graph (nirtorch.graph.TorchGraph): a NIR graph representation of `spiking_model`. Returns ---------- @@ -263,14 +263,14 @@ def _get_name_2_indx_map(self, nir_graph: nirtorch.graph.Graph) -> Dict[str, int } def _get_edges_from_nir( - self, nir_graph: nirtorch.graph.Graph, name_2_indx_map: Dict[str, int] + self, nir_graph: nirtorch.graph.TorchGraph, name_2_indx_map: Dict[str, int] ) -> Set[Edge]: - """Standardize the representation of `nirtorch.graph.Graph` into a list of edges, + """Standardize the representation of `nirtorch.graph.TorchGraph` into a list of edges, representing nodes by their indices. Parameters ---------- - - nir_graph (nirtorch.graph.Graph): a NIR graph representation of `spiking_model`. + - nir_graph (nirtorch.graph.TorchGraph): a NIR graph representation of `spiking_model`. - name_2_indx_map (dict): Map from node names to unique indices. Returns diff --git a/sinabs/backend/dynapcnn/utils.py b/sinabs/backend/dynapcnn/utils.py index 06946a07..8d783a08 100644 --- a/sinabs/backend/dynapcnn/utils.py +++ b/sinabs/backend/dynapcnn/utils.py @@ -289,6 +289,7 @@ def merge_conv_bn(conv, bn): return conv +# Should become obsolete def construct_next_pooling_layer( layers: List[nn.Module], idx_start: int ) -> Tuple[Optional[sl.SumPool2d], int, float]: From b81cbf1c537508310c97df7fb5eeb6c8cb68b95d Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Thu, 31 Oct 2024 11:32:36 +0100 Subject: [PATCH 281/379] Fix dynapcnnnetwork test --- tests/test_dynapcnnnetwork/model_dummy_4.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/test_dynapcnnnetwork/model_dummy_4.py b/tests/test_dynapcnnnetwork/model_dummy_4.py index 1655b5f5..c467dba8 100644 --- a/tests/test_dynapcnnnetwork/model_dummy_4.py +++ b/tests/test_dynapcnnnetwork/model_dummy_4.py @@ -117,12 +117,16 @@ def forward(self, x): snn = SNN(batch_size) +# TODO: This test sometimes fails because the layer that has ID 1 +# sometimes gets ID 2, and the layer with ID 3 gets ID 4 +# This is not a bug in sinabs itself but an issue with the test, becuase +# the IDs that the layers are assigned do not always have to be the same. expected_output = { "dcnnl_edges": { (0, 1), (0, 2), - (1, 4), (1, 3), + (2, 3), (2, 4), (3, 5), (4, 5), @@ -132,14 +136,14 @@ def forward(self, x): 0: {"input"}, 1: {0}, 2: {0}, - 3: {1}, - 4: {1, 2}, + 3: {1, 2}, + 4: {2}, 5: {3, 4}, }, "destination_map": { 0: {1, 2}, - 1: {3, 4}, - 2: {4}, + 1: {3}, + 2: {3, 4}, 3: {5}, 4: {5}, 5: {-1}, From 9c9ec26d37f0f81d62f033e73007b0ec484b3efb Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Thu, 31 Oct 2024 11:56:43 +0100 Subject: [PATCH 282/379] Ensure functioning across different nirtorch versions --- sinabs/backend/dynapcnn/nir_graph_extractor.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index 0a114a74..4bc662ed 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -19,6 +19,12 @@ from .sinabs_edges_handler import collect_dynapcnn_layer_info from .utils import Edge, topological_sorting +try: + from nirtorch.graph import TorchGraph +except ImportError: + # In older nirtorch versions TorchGraph is called Graph + from nirtorch.graph import Graph as TorchGraph + class GraphExtractor: def __init__( @@ -315,9 +321,14 @@ def _get_named_modules(self, model: nn.Module) -> Dict[int, nn.Module]: for name, module in model.named_modules(): # Make sure names match those provided by nirtorch nodes - name = nirtorch.utils.sanitize_name(name) if name in self._name_2_indx_map: indx_2_module_map[self._name_2_indx_map[name]] = module + else: + # In older nirtorch versions, node names are "sanitized" + # Try with sanitized version of the name + name = nirtorch.utils.sanitize_name(name) + if name in self._name_2_indx_map: + indx_2_module_map[self._name_2_indx_map[name]] = module return indx_2_module_map From b2e9214e0697568300f337832e5d8814d888699b Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Thu, 31 Oct 2024 11:57:00 +0100 Subject: [PATCH 283/379] Fix unit tests --- tests/test_dynapcnn/test_compatible_layer_build.py | 8 ++++---- tests/test_dynapcnn/test_doorbell.py | 5 ++++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/test_dynapcnn/test_compatible_layer_build.py b/tests/test_dynapcnn/test_compatible_layer_build.py index b2630ee0..8259cef8 100644 --- a/tests/test_dynapcnn/test_compatible_layer_build.py +++ b/tests/test_dynapcnn/test_compatible_layer_build.py @@ -11,9 +11,9 @@ (3, sl.SumPool2d, [3, 3], 1), ((4, 4), sl.SumPool2d, [4, 4], 1), (2, nn.AvgPool2d, [2, 2], 1./4), - ((2, 2), sl.nn.AvgPool2d, [2, 2], 1./4), - (3, sl.nn.AvgPool2d, [3, 3], 1./9), - ((4, 4), sl.nn.AvgPool2d, [4, 4], 1./16), + ((2, 2), nn.AvgPool2d, [2, 2], 1./4), + (3, nn.AvgPool2d, [3, 3], 1./9), + ((4, 4), nn.AvgPool2d, [4, 4], 1./16), ] ) def test_construct_pooling_from_1_layer(pooling, layer_type, expected_pooling, expected_scaling): @@ -74,4 +74,4 @@ def test_incorrect_model_start(): with pytest.raises(UnexpectedLayer): construct_next_dynapcnn_layer( layers, 0, in_shape=in_shape, discretize=True, rescale_factor=1 - ) \ No newline at end of file + ) diff --git a/tests/test_dynapcnn/test_doorbell.py b/tests/test_dynapcnn/test_doorbell.py index 40535285..36ee8955 100644 --- a/tests/test_dynapcnn/test_doorbell.py +++ b/tests/test_dynapcnn/test_doorbell.py @@ -81,8 +81,11 @@ def test_auto_config(): def test_was_copied(): # - Make sure that layers of different models are distinct objects + # "Sanitize" all layer names, for compatibility with older nirtorch versions snn_layers = {sanitize_name(name): lyr for name, lyr in snn.named_modules()} - idx_2_name_map = {idx: name for name, idx in dynapcnn_net.name_2_indx_map.items()} + idx_2_name_map = { + idx: sanitize_name(name) for name, idx in dynapcnn_net.name_2_indx_map.items() + } for idx, lyr_info in dynapcnn_net._graph_extractor.dcnnl_map.items(): conv_lyr_dynapcnn = dynapcnn_net.dynapcnn_layers[idx].conv_layer conv_node_idx = lyr_info["conv"]["node_id"] From a79addfe62957158e246c6ea2129709abc7a373b Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Thu, 31 Oct 2024 13:34:30 +0100 Subject: [PATCH 284/379] DONE - DVSLayer->pooling edge spiking_model of DynapcnnNetwork now accepts a DVSLayer followed by a pooling layer --- .../snn_DVSLayer_given_followed_by_pool.ipynb | 333 ++++++++++++++++++ sinabs/backend/dynapcnn/connectivity_specs.py | 2 - .../backend/dynapcnn/sinabs_edges_handler.py | 62 +++- 3 files changed, 387 insertions(+), 10 deletions(-) create mode 100644 examples/dynapcnn_network/snn_DVSLayer_given_followed_by_pool.ipynb diff --git a/examples/dynapcnn_network/snn_DVSLayer_given_followed_by_pool.ipynb b/examples/dynapcnn_network/snn_DVSLayer_given_followed_by_pool.ipynb new file mode 100644 index 00000000..8da2c368 --- /dev/null +++ b/examples/dynapcnn_network/snn_DVSLayer_given_followed_by_pool.ipynb @@ -0,0 +1,333 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "metadata": {} + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import torch\n", + "import torch.nn as nn\n", + "from sinabs.backend.dynapcnn import DynapcnnNetwork\n", + "from sinabs.backend.dynapcnn import DVSLayer\n", + "from sinabs.layers import Merge, IAFSqueeze, SumPool2d\n", + "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential\n", + "import sinabs.layers as sl\n", + "\n", + "from torch.nn import CrossEntropyLoss\n", + "from torch.optim import Adam\n", + "\n", + "from tonic.datasets.nmnist import NMNIST\n", + "from tonic.transforms import ToFrame\n", + "from torch.utils.data import DataLoader\n", + "import numpy as np\n", + "from tqdm.notebook import tqdm\n", + "from statistics import mode\n", + "\n", + "device = torch.device('cpu')\n", + "torch.manual_seed(0)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "channels = 1\n", + "height = 34\n", + "width = 34\n", + "batch_size = 1\n", + "\n", + "input_shape = (channels, height, width)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "metadata": {} + }, + "outputs": [ + { + "data": { + "text/plain": [ + "SNN(\n", + " (dvs): DVSLayer(\n", + " (pool_layer): SumPool2d(norm_type=1, kernel_size=(1, 1), stride=None, ceil_mode=False)\n", + " (crop_layer): Crop2d((0, 34), (0, 34))\n", + " (flip_layer): FlipDims()\n", + " )\n", + " (dvs_pool): AvgPool2d(kernel_size=1, stride=1, padding=0)\n", + " (conv1): Conv2d(1, 10, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + " (iaf1): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(1.), min_v_mem=Parameter containing:\n", + " tensor(-1.), batch_size=1, num_timesteps=-1)\n", + " (pool1): AvgPool2d(kernel_size=2, stride=2, padding=0)\n", + " (conv2): Conv2d(10, 10, kernel_size=(4, 4), stride=(1, 1), bias=False)\n", + " (iaf2): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(1.), min_v_mem=Parameter containing:\n", + " tensor(-1.), batch_size=1, num_timesteps=-1)\n", + " (conv3): Conv2d(10, 1, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + " (iaf3): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(1.), min_v_mem=Parameter containing:\n", + " tensor(-1.), batch_size=1, num_timesteps=-1)\n", + " (fc1): Linear(in_features=144, out_features=200, bias=False)\n", + " (iaf4): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(1.), min_v_mem=Parameter containing:\n", + " tensor(-1.), batch_size=1, num_timesteps=-1)\n", + " (fc2): Linear(in_features=200, out_features=10, bias=False)\n", + " (iaf5): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(1.), min_v_mem=Parameter containing:\n", + " tensor(-1.), batch_size=1, num_timesteps=-1)\n", + " (flat): Flatten(start_dim=1, end_dim=-1)\n", + ")" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "class SNN(nn.Module):\n", + " def __init__(self, input_shape) -> None:\n", + " super().__init__()\n", + "\n", + " self.dvs = DVSLayer(input_shape=(input_shape[1], input_shape[2]))\n", + " self.dvs_pool = nn.AvgPool2d(1,1)\n", + " # -- chip core A --\n", + " self.conv1 = nn.Conv2d(1, 10, 2, 1, bias=False)\n", + " self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential())\n", + " self.pool1 = nn.AvgPool2d(2,2)\n", + " # -- chip core B --\n", + " self.conv2 = nn.Conv2d(10, 10, 4, 1, bias=False)\n", + " self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential())\n", + " # -- chip core C --\n", + " self.conv3 = nn.Conv2d(10, 1, 2, 1, bias=False)\n", + " self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential())\n", + " # -- chip core D --\n", + " self.fc1 = nn.Linear(144, 200, bias=False)\n", + " self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential())\n", + " # -- chip core E --\n", + " self.fc2 = nn.Linear(200, 10, bias=False)\n", + " self.iaf5 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential())\n", + "\n", + " # -- layers ignored during deployment --\n", + " self.flat = nn.Flatten()\n", + "\n", + " def init_weights(self):\n", + " for name, layer in self.named_modules():\n", + " if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear):\n", + " nn.init.xavier_normal_(layer.weight.data)\n", + "\n", + " def detach_neuron_states(self):\n", + " for name, layer in self.named_modules():\n", + " if name != '':\n", + " if isinstance(layer, sl.StatefulLayer):\n", + " for name, buffer in layer.named_buffers():\n", + " buffer.detach_()\n", + "\n", + " def forward(self, x):\n", + " \n", + " dvs_out = self.dvs(x) # 0\n", + "\n", + " dvs_pool_out = self.dvs_pool(dvs_out)\n", + " \n", + " con1_out = self.conv1(dvs_pool_out) # 4\n", + " iaf1_out = self.iaf1(con1_out) # 5\n", + " pool1_out = self.pool1(iaf1_out) # 6\n", + "\n", + " conv2_out = self.conv2(pool1_out) # 7\n", + " iaf2_out = self.iaf2(conv2_out) # 8\n", + "\n", + " conv3_out = self.conv3(iaf2_out) # 9\n", + " iaf3_out = self.iaf3(conv3_out) # 10\n", + "\n", + " flat_out = self.flat(iaf3_out) # 15\n", + " \n", + " fc1_out = self.fc1(flat_out) # 11\n", + " iaf4_out = self.iaf4(fc1_out) # 12\n", + " fc2_out = self.fc2(iaf4_out) # 13\n", + " iaf5_out = self.iaf5(fc2_out) # 14\n", + "\n", + " return iaf5_out\n", + " \n", + "snn = SNN(input_shape)\n", + "snn.init_weights()\n", + "snn.to(device)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "metadata": {} + }, + "outputs": [], + "source": [ + "hw_model = DynapcnnNetwork(\n", + " snn=snn,\n", + " input_shape=input_shape,\n", + " batch_size=batch_size,\n", + " discretize=True\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "input_dummy = torch.randn((batch_size, *input_shape))" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "out = hw_model(input_dummy)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "tensor([[[[0.]],\n", + "\n", + " [[0.]],\n", + "\n", + " [[0.]],\n", + "\n", + " [[0.]],\n", + "\n", + " [[0.]],\n", + "\n", + " [[0.]],\n", + "\n", + " [[0.]],\n", + "\n", + " [[0.]],\n", + "\n", + " [[0.]],\n", + "\n", + " [[0.]]]], grad_fn=)\n" + ] + } + ], + "source": [ + "print(out)" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Network is valid\n" + ] + }, + { + "data": { + "text/plain": [ + "DynapcnnNetwork(\n", + " (_dynapcnn_module): DynapcnnNetworkModule(\n", + " (_dynapcnn_layers): ModuleDict(\n", + " (1): DynapcnnLayer(\n", + " (_conv): Conv2d(1, 10, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + " (_spk): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(174.), min_v_mem=Parameter containing:\n", + " tensor(-174.), batch_size=1, num_timesteps=-1)\n", + " )\n", + " (5): DynapcnnLayer(\n", + " (_conv): Conv2d(200, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + " (_spk): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(419.), min_v_mem=Parameter containing:\n", + " tensor(-419.), batch_size=1, num_timesteps=-1)\n", + " )\n", + " (3): DynapcnnLayer(\n", + " (_conv): Conv2d(10, 1, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + " (_spk): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(303.), min_v_mem=Parameter containing:\n", + " tensor(-303.), batch_size=1, num_timesteps=-1)\n", + " )\n", + " (2): DynapcnnLayer(\n", + " (_conv): Conv2d(10, 10, kernel_size=(4, 4), stride=(1, 1), bias=False)\n", + " (_spk): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(130.), min_v_mem=Parameter containing:\n", + " tensor(-130.), batch_size=1, num_timesteps=-1)\n", + " )\n", + " (4): DynapcnnLayer(\n", + " (_conv): Conv2d(1, 200, kernel_size=(12, 12), stride=(1, 1), bias=False)\n", + " (_spk): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(365.), min_v_mem=Parameter containing:\n", + " tensor(-365.), batch_size=1, num_timesteps=-1)\n", + " )\n", + " (0): DVSLayer(\n", + " (pool_layer): SumPool2d(norm_type=1, kernel_size=(1, 1), stride=None, ceil_mode=False)\n", + " (crop_layer): Crop2d((0, 34), (0, 34))\n", + " (flip_layer): FlipDims()\n", + " )\n", + " )\n", + " (merge_layer): Merge()\n", + " )\n", + ")" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "hw_model.to(device=\"speck2fdevkit\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "speck-rescnn", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/sinabs/backend/dynapcnn/connectivity_specs.py b/sinabs/backend/dynapcnn/connectivity_specs.py index c361f240..f05ccdbe 100644 --- a/sinabs/backend/dynapcnn/connectivity_specs.py +++ b/sinabs/backend/dynapcnn/connectivity_specs.py @@ -30,8 +30,6 @@ (Pooling, Weight): "pooling-weight", # Dvs can be followed by weight layer of next core (DVS, Weight): "dvs-weight", - # Dvs can be followed by pooling layers - (DVS, Pooling): "dvs-pooling", } # Unpack dict diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index d050bcd3..0423b132 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -10,7 +10,7 @@ from torch import Size, nn -from .connectivity_specs import VALID_SINABS_EDGE_TYPES +from .connectivity_specs import VALID_SINABS_EDGE_TYPES, Pooling from .exceptions import ( InvalidEdge, InvalidGraphStructure, @@ -119,6 +119,59 @@ def fix_dvs_module_edges(edges: Set[Edge], indx_2_module_map: Dict[int, nn.Modul entry_nodes.clear() entry_nodes.add(dvs_node[-1]) + # Merge a pooling node from a 'dvs-pooling' edge (pooling being an independent node in the original graph) into the DVSLayer if such edge exists. + merge_dvs_pooling_edge(edges, indx_2_module_map, name_2_indx_map) + +def merge_dvs_pooling_edge(edges: Set[Edge], indx_2_module_map: Dict[int, nn.Module], name_2_indx_map: Dict[str, int]) -> None: + """ If a 'dvs-polling' edge existis, the pooling is incorporated into the DVSLayer node if `DVSLayer.pool_layer` has + default values. All arguments are modified in-place to remove the references to the incorporated pooling node. + + Parameters + ---------- + - edges (set): tuples describing the connections between layers in `spiking_model`. + - indx_2_module_map (dict): the mapping between a node (`key` as an `int`) and its module (`value` as a `nn.Module`). + - name_2_indx_map (dict): Map from node names to unique indices. + """ + # Find 'DVSLayer-pooling' edge. + dvs_pool_edge = [ + (src, tgt) for (src, tgt) in edges + if (isinstance(indx_2_module_map[src], DVSLayer) and any(isinstance(indx_2_module_map[tgt], pool) for pool in Pooling)) + ] + + if len(dvs_pool_edge) == 0: + # No dvs-pooling edge exists - nothing to do here. + return + if len(dvs_pool_edge) > 1: + # DVSLayer in the original network can have only a single pooling layer liked to it. + raise ValueError(f'DVSLayer can have a single edge onto a pooling layer but multiple were found: {dvs_pool_edge}') + + (dvs_idnx, pool_idnx) = dvs_pool_edge[-1] + + # Checking pooling can be incorporated into the DVSLayer. + if indx_2_module_map[dvs_idnx].pool_layer.kernel_size == 1 and indx_2_module_map[dvs_idnx].pool_layer.stride == 1: + # DVSLayer.pool has its default config. + indx_2_module_map[pool_idnx].kernel_size + indx_2_module_map[pool_idnx].stride + # Set DVSLayer.pool to have same config. as the independent pooling layer. + indx_2_module_map[dvs_idnx].pool_layer.kernel_size = indx_2_module_map[pool_idnx].kernel_size + indx_2_module_map[dvs_idnx].pool_layer.stride = indx_2_module_map[pool_idnx].stride + + # Pooling incorporated to the DVSLayer: remove its trace from mappings. + indx_2_module_map.pop(pool_idnx) + name_2_indx_map.pop([name for name, indx in name_2_indx_map.items() if indx == pool_idnx][-1]) + + # Since pool is part of the DVSLayer we now make edges where pool was a source to have DVSLayer as a source. + for edge in [edge for edge in edges if edge[0] == pool_idnx]: + edges.remove(edge) + edges.update({(dvs_idnx, edge[1])}) + + # Remove original 'dvs-pool' edge. + edges.remove((dvs_idnx, pool_idnx)) + + # Checks if any traces of the original pooling node can still be found. + if len([edge for edge in edges if (edge[0] == pool_idnx or edge[1] == pool_idnx)]) != 0: + raise ValueError('Edges involving the pooling layer merged into the DVSLayer are still present in the graph.') + def collect_dynapcnn_layer_info( indx_2_module_map: Dict[int, nn.Module], edges: Set[Edge], @@ -195,10 +248,6 @@ def collect_dynapcnn_layer_info( nodes_io_shapes, ) - # TODO - handle dvs->pooling connections. - while edges_by_type["dvs-pooling"]: - pass - # Process all edges connecting two dynapcnn layers that do not include pooling while edges_by_type.get("neuron-weight", False): edge = edges_by_type["neuron-weight"].pop() @@ -304,9 +353,6 @@ def sort_edges_by_type( if 'dvs-weight' not in edges_by_type: edges_by_type['dvs-weight'] = set() - if 'dvs-pooling' not in edges_by_type: - edges_by_type['dvs-pooling'] = set() - return edges_by_type From 02ca1b257d9a94ad5de361566e86047bca8ac1ba Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Thu, 31 Oct 2024 13:58:23 +0100 Subject: [PATCH 285/379] (WIP) Minor refactoring of new dvs layer support --- sinabs/backend/dynapcnn/dvs_layer.py | 4 +- sinabs/backend/dynapcnn/dynapcnn_network.py | 9 ++-- .../dynapcnn/dynapcnnnetwork_module.py | 50 ++++++++++++------- .../backend/dynapcnn/nir_graph_extractor.py | 43 +++++++++++----- .../backend/dynapcnn/sinabs_edges_handler.py | 2 + 5 files changed, 68 insertions(+), 40 deletions(-) diff --git a/sinabs/backend/dynapcnn/dvs_layer.py b/sinabs/backend/dynapcnn/dvs_layer.py index bc240c25..13e0c7a4 100644 --- a/sinabs/backend/dynapcnn/dvs_layer.py +++ b/sinabs/backend/dynapcnn/dvs_layer.py @@ -226,10 +226,8 @@ def forward(self, data): # Pool out = self.pool_layer(data) - # TODO - self.crop_layer is never None (even if crop == None when instantiating the class) so this 'if' statement is unecessary (plus confusing when debbuging the code). # Crop - if self.crop_layer is not None: - out = self.crop_layer(out) + out = self.crop_layer(out) # Flip stuff out = self.flip_layer(out) diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index a74242b9..f4ea204a 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -66,7 +66,7 @@ def __init__( self._graph_extractor = GraphExtractor( snn, torch.randn((batch_size, *self.input_shape)), - self.dvs_input + self.dvs_input, ignore_node_types=COMPLETELY_IGNORED_LAYER_TYPES, ) @@ -550,9 +550,6 @@ def _make_config( "the keys in `self.dynapcnn_layers`" ) - if has_dvs_layer: - # TODO - DVS layer has been incorporated: what should happen here? - pass self._layer2core_map = layer2core_map # update config (config. DynapcnnLayer instances into their assigned core). @@ -563,11 +560,11 @@ def _make_config( dvs_node_info=self.dvs_node_info, ) - # TODO - this is from the old implementation, should be revised. + # TODO: This should be handled earlier and probably raise a warning if it contradicts + # dvs_layer merge_polarities if self.input_shape and self.input_shape[0] == 1: config.dvs_layer.merge = True - # TODO all this monitoring part needs validation still. if monitor_layers is None: # Monitor all layers with exit point destinations monitor_layers = self._dynapcnn_module.get_exit_layers() diff --git a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py index 7f290c57..fc4052af 100644 --- a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py +++ b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py @@ -27,7 +27,8 @@ class DynapcnnNetworkModule(nn.Module): - destination_map (dict): Maps layer indices to list of destination indices. Exit destinations are marked by negative integers - entry_points (set): Set of layer indices that act as network entry points. - - dvs_node_info (dict): contains information associated with the `DVSLayer` node (if no DVS node exists it'll return `None`). + - dvs_node_info (dict): contains information associated with the `DVSLayer` node. + `None` if no DVS node exists. Attributes ---------- @@ -47,31 +48,45 @@ def __init__( dynapcnn_layers: Dict[int, DynapcnnLayer], destination_map: Dict[int, List[int]], entry_points: Set[int], - dvs_node_info: Optional[Dict], + dvs_node_info: Optional[Dict] = None, ): super().__init__() self._dvs_node_info = dvs_node_info - # nodes in a DynapcnnNetwork graph. - module_dict = {str(idx): lyr for idx, lyr in dynapcnn_layers.items()} - # Insert DVS node if DVS was enabled. - if isinstance(self._dvs_node_info, Dict): - module_dict[str(dvs_node_info['layer_id'])] = dvs_node_info['module'] - # Unfortunately ModuleDict does not allow for integer keys - # TODO: Consider using list instead of dict + module_dict = {str(idx): lyr for idx, lyr in dynapcnn_layers.items()} self._dynapcnn_layers = nn.ModuleDict(module_dict) + if self._dvs_node_info is not None: + self._dvs_layer = dvs_node_info["module"] + else: + self._dvs_layer = None + self._destination_map = destination_map self._entry_points = entry_points # `Merge` layers are stateless. One instance can be used for all merge points during forward pass self.merge_layer = sl.Merge() + @property + def all_layers(self): + layers = self.dynapcnn_layers + if self.dvs_layer is not None: + # `self.dynapcnn_layers` is a (shallow) copy. Adding entries won't + # affect `self._dynapcnn_layers` + # TODO: Why not use "dvs" as index? + dvs_id = self._dvs_node_info["layer_id"] + layers[dvs_id] = self.dvs_layer + return layers + @property def dvs_node_info(self): return self._dvs_node_info + + @property + def dvs_layer(self): + return self._dvs_layer @property def destination_map(self): @@ -79,7 +94,7 @@ def destination_map(self): @property def dynapcnn_layers(self): - # Convert string-indices to integers-indices + # Convert string-indices to integer-indices return {int(idx): lyr for idx, lyr in self._dynapcnn_layers.items()} @property @@ -260,7 +275,7 @@ def forward( current_input = layers_outputs[idx_src][idx_curr] # Get current layer instance and destinations - layer = self.dynapcnn_layers[idx_curr] + layer = self.all_layers[idx_curr] destinations = self._destination_map[idx_curr] # Forward pass through layer @@ -337,14 +352,13 @@ def remap(key): return mapping[key] # Remap all internal objects - dynapcnn_layers = {str(remap(int(idx))): lyr for idx, lyr in self._dynapcnn_layers.items()} - - if isinstance(self.dvs_node_info, Dict): - _ = str(remap(self.dvs_node_info['layer_id'])) - dynapcnn_layers[_] = self.dvs_node_info['module'] - self.dvs_node_info['layer_id'] = int(_) + self._dynapcnn_layers = nn.ModuleDict( + {str(remap(int(idx))): lyr for idx, lyr in self._dynapcnn_layers.items()} + ) - self._dynapcnn_layers = nn.ModuleDict(dynapcnn_layers) + if self.dvs_node_info is not None: + new_dvs_id = remap(self.dvs_node_info['layer_id']) + self._dvs_node_info["layer_id"] = new_dvs_id self._entry_points = {remap(idx) for idx in self._entry_points} diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index e2650e20..6c30f0f8 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -93,9 +93,9 @@ def __init__( # If DVS camera is wanted but `spiking_model` does not start with DVS layer. if self._need_dvs_node(spiking_model, dvs_input): + # TODO: Infer polarity as well. Do that in `_add_dvs_node` # input shape for `DVSLayer` instance that will be the module of the node 'dvs'. - _, _, height, width = dummy_input.shape - self._add_dvs_node(dvs_input_shape=(height, width)) + self._add_dvs_node(dvs_input_shape=dummpy_input_shape[1:]) # Check for the need of fixing NIR edges extraction when DVS is a node in the graph. If DVS # is used its node becomes the only entry node in the graph. @@ -138,6 +138,7 @@ def sorted_nodes(self) -> List[int]: def indx_2_module_map(self) -> Dict[int, nn.Module]: return {n: module for n, module in self._indx_2_module_map.items()} + # TODO: Information about `dvs_input` should already be part of `self` at this point. def get_dynapcnn_network_module( self, discretize: bool = False, weight_rescaling_fn: Optional[Callable] = None, dvs_input: bool = False ) -> DynapcnnNetworkModule: @@ -177,7 +178,7 @@ def get_dynapcnn_network_module( ) # DVSLayer node information (None if DVS camera is not used). - dvs_node_info = get_dvs_node_from_mapper(dcnnl_map) + dvs_node_info = get_dvs_node_from_mapper(self.dcnnl_map) # Instantiate the DynapcnnNetworkModule return DynapcnnNetworkModule( @@ -281,6 +282,7 @@ def verify_graph_integrity(self): ####################################################### Pivate Methods ####################################################### + # TODO: Needs to receive 3d input shape and sett merge_polarities def _add_dvs_node(self, dvs_input_shape: Tuple[int, int]) -> None: """ In-place modification of `self._name_2_indx_map`, `self._indx_2_module_map`, and `self._edges` to accomodate the creation of an extra node in the graph representing the DVS camera of the chip. The DVSLayer node will point to every @@ -302,11 +304,15 @@ def _add_dvs_node(self, dvs_input_shape: Tuple[int, int]) -> None: # set DVS node as input to each entry node of the graph. self._edges.update({(self._name_2_indx_map['dvs'], entry_node) for entry_node in self._entry_nodes}) # DVSLayer node becomes the only entrypoint of the graph. + # TODO: Possibly have to remove previous entry nodes here. self._entry_nodes = {self._name_2_indx_map['dvs']} def _need_dvs_node(self, model: nn.Module, dvs_input: bool) -> bool: - """ Returns whether or not a node will need to be added to represent a `DVSLayer` instance. A new node will have - to be added if `model` does not start with a `DVSLayer` instance and `dvs_input == True`. + """ Returns whether or not a node will need to be added to represent a + `DVSLayer` instance. + + A new node will have to be added if `model` does not start with a + `DVSLayer` instance and `dvs_input == True`. Parameters ---------- @@ -314,17 +320,27 @@ def _need_dvs_node(self, model: nn.Module, dvs_input: bool) -> bool: - dvs_input (bool): wether or not dynapcnn receive input from its DVS camera. Returns ------- - - True if the first layer is a DVSLayer, False otherwise. + - True if `model` contains a DVSLayer, False otherwise. """ + + dvs_layers = { + module for module in self._indx_2_module_map.values() + if isinstance(module, DVSLayer) + } - # Get the first module only and check its type - first_name, first_module = next(model.named_modules()) - - # Check consistency of user provided arguments for use of the DVS - if isinstance(first_module, DVSLayer) and not dvs_input: - raise InvalidModelWithDVSSetup() + if (num_dvs := len(dvs_layers)) == 0: + has_dvs_layer = False + elif num_dvs == 1: + # TODO: Avoid redundant arguments + if not dvs_input: + raise InvalidModelWithDVSSetup() + has_dvs_layer = True + else: + raise InvalidGraphStructure( + f"The provided model has {num_dvs} `DVSLayer`s. At most one is allowed." + ) - return not isinstance(first_module, DVSLayer) and dvs_input + return not has_dvs_layer and dvs_input def _get_name_2_indx_map(self, nir_graph: nirtorch.graph.TorchGraph) -> Dict[str, int]: """Assign unique index to each node and return mapper from name to index. @@ -339,6 +355,7 @@ def _get_name_2_indx_map(self, nir_graph: nirtorch.graph.TorchGraph) -> Dict[str `spiking_model` and `value is an integer representing the layer in a standard format. """ + # TODO: Unclear comment # Start name indexing from 1 if a DVS node needs to be added return { node.name: node_idx for node_idx, node in enumerate(nir_graph.node_list) diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index 4df3a785..ccd66c4e 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -30,6 +30,8 @@ def get_dvs_node_from_mapper(dcnnl_map: Dict) -> Optional[Dict]: ------- - Dict containing information associated with the `DVSLayer` node (if no DVS node exists it'll return `None`). """ + # TODO: Would it make more sense to have dvs layer separated from dcnnl layers? + # Either as different object or with a very clear key inside `dcnnl_map` for layer_index, layer_info in dcnnl_map.items(): if 'dvs_layer' in layer_info: assert layer_info['dvs_layer'] From b24edb8dd55f0a91e390ee895ef943efa0906666 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Thu, 31 Oct 2024 14:06:05 +0100 Subject: [PATCH 286/379] Fix TorchGraph handlers --- sinabs/backend/dynapcnn/nir_graph_extractor.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index 6c30f0f8..b15c664e 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -342,12 +342,12 @@ def _need_dvs_node(self, model: nn.Module, dvs_input: bool) -> bool: return not has_dvs_layer and dvs_input - def _get_name_2_indx_map(self, nir_graph: nirtorch.graph.TorchGraph) -> Dict[str, int]: + def _get_name_2_indx_map(self, nir_graph: TorchGraph) -> Dict[str, int]: """Assign unique index to each node and return mapper from name to index. Parameters ---------- - - nir_graph (nirtorch.graph.TorchGraph): a NIR graph representation of `spiking_model`. + - nir_graph (TorchGraph): a NIR graph representation of `spiking_model`. Returns ---------- @@ -362,14 +362,14 @@ def _get_name_2_indx_map(self, nir_graph: nirtorch.graph.TorchGraph) -> Dict[str } def _get_edges_from_nir( - self, nir_graph: nirtorch.graph.TorchGraph, name_2_indx_map: Dict[str, int] + self, nir_graph: TorchGraph, name_2_indx_map: Dict[str, int] ) -> Set[Edge]: - """Standardize the representation of `nirtorch.graph.TorchGraph` into a list of edges, + """Standardize the representation of TorchGraph` into a list of edges, representing nodes by their indices. Parameters ---------- - - nir_graph (nirtorch.graph.TorchGraph): a NIR graph representation of `spiking_model`. + - nir_graph (TorchGraph): a NIR graph representation of `spiking_model`. - name_2_indx_map (dict): Map from node names to unique indices. Returns From b3e2ad315a65be0262d5ec23e3e36e9e243f5b02 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Thu, 31 Oct 2024 15:23:46 +0100 Subject: [PATCH 287/379] Minor changes to DVS part --- sinabs/backend/dynapcnn/dynapcnn_network.py | 8 ++++++-- .../backend/dynapcnn/nir_graph_extractor.py | 20 +++++++++---------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index f4ea204a..479d6f14 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -29,6 +29,7 @@ def __init__( snn: nn.Module, input_shape: Tuple[int, int, int], batch_size: Optional[int] = None, + # TODO: Set None by default dvs_input: bool = False, discretize: bool = True, weight_rescaling_fn: Callable = rescale_method_1, @@ -85,6 +86,10 @@ def __init__( ####################################################### Public Methods ####################################################### + @property + def all_layers(self): + return self._dynapcnn_module.all_layers + @property def dvs_node_info(self): return self._dynapcnn_module.dvs_node_info @@ -516,7 +521,6 @@ def _make_config( the provided device can be found. """ config_builder = ChipFactory(device).get_config_builder() - has_dvs_layer = self.has_dvs_layer() if chip_layers_ordering is not None: if layer2core_map is not None: @@ -554,7 +558,7 @@ def _make_config( # update config (config. DynapcnnLayer instances into their assigned core). config = config_builder.build_config( - layers=self.dynapcnn_layers, + layers=self.all_layers, destination_map=self.layer_destination_map, layer2core_map=layer2core_map, dvs_node_info=self.dvs_node_info, diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index b15c664e..466c9a19 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -95,7 +95,12 @@ def __init__( if self._need_dvs_node(spiking_model, dvs_input): # TODO: Infer polarity as well. Do that in `_add_dvs_node` # input shape for `DVSLayer` instance that will be the module of the node 'dvs'. - self._add_dvs_node(dvs_input_shape=dummpy_input_shape[1:]) + self._add_dvs_node(dvs_input_shape=dummy_input.shape[1:]) + else: + # TODO: Handle case where DVSLayer is present: + # - make sure it is the only entry node + # - Make sure the `merge_polarities` attribute matches `dummy_input.shape` + pass # Check for the need of fixing NIR edges extraction when DVS is a node in the graph. If DVS # is used its node becomes the only entry node in the graph. @@ -294,9 +299,6 @@ def _add_dvs_node(self, dvs_input_shape: Tuple[int, int]) -> None: - dvs_input_shape (tuple): Shape of input in format `(height, width)`. """ - # [] @TODO - not considering pooling after the DVSLayer yet. - # [] @TODO - I/O shape in 'self._nodes_io_shapes' not being handled yet. - # add name entry for node 'dvs'. self._name_2_indx_map['dvs'] = len(self._name_2_indx_map) # add module entry for node 'dvs'. @@ -304,23 +306,21 @@ def _add_dvs_node(self, dvs_input_shape: Tuple[int, int]) -> None: # set DVS node as input to each entry node of the graph. self._edges.update({(self._name_2_indx_map['dvs'], entry_node) for entry_node in self._entry_nodes}) # DVSLayer node becomes the only entrypoint of the graph. - # TODO: Possibly have to remove previous entry nodes here. self._entry_nodes = {self._name_2_indx_map['dvs']} - def _need_dvs_node(self, model: nn.Module, dvs_input: bool) -> bool: + def _need_dvs_node(self, dvs_input: bool) -> bool: """ Returns whether or not a node will need to be added to represent a `DVSLayer` instance. - A new node will have to be added if `model` does not start with a + A new node will have to be added if `self._indx_2_module_map` contains no `DVSLayer` instance and `dvs_input == True`. Parameters ---------- - - model (nn.Module): the `spiking_model` used as argument to the class instance. - dvs_input (bool): wether or not dynapcnn receive input from its DVS camera. Returns ------- - - True if `model` contains a DVSLayer, False otherwise. + - True if `self._indx_2_module_map` contains a DVSLayer, False otherwise. """ dvs_layers = { @@ -355,8 +355,6 @@ def _get_name_2_indx_map(self, nir_graph: TorchGraph) -> Dict[str, int]: `spiking_model` and `value is an integer representing the layer in a standard format. """ - # TODO: Unclear comment - # Start name indexing from 1 if a DVS node needs to be added return { node.name: node_idx for node_idx, node in enumerate(nir_graph.node_list) } From 2970d663515c94a7dd3b680125abd78166089761 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Thu, 31 Oct 2024 17:36:41 +0100 Subject: [PATCH 288/379] Update `extend_readout_layer` function to work with new DynapcnnNetwork --- sinabs/backend/dynapcnn/utils.py | 49 ++++++++++++++++---------------- 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/sinabs/backend/dynapcnn/utils.py b/sinabs/backend/dynapcnn/utils.py index 8d783a08..05f9c63c 100644 --- a/sinabs/backend/dynapcnn/utils.py +++ b/sinabs/backend/dynapcnn/utils.py @@ -368,32 +368,31 @@ def extend_readout_layer(model: "DynapcnnNetwork") -> "DynapcnnNetwork": """ model = deepcopy(model) input_shape = model.input_shape - og_readout_conv_layer = model.sequence[ - -1 - ].conv_layer # extract the conv layer from dynapcnn network - og_weight_data = og_readout_conv_layer.weight.data - og_bias_data = og_readout_conv_layer.bias - og_bias = og_bias_data is not None - # modify the out channels - og_out_channels = og_readout_conv_layer.out_channels - new_out_channels = (og_out_channels - 1) * 4 + 1 - og_readout_conv_layer.out_channels = new_out_channels - # build extended weight and replace the old one - ext_weight_shape = (new_out_channels, *og_weight_data.shape[1:]) - ext_weight_data = torch.zeros(ext_weight_shape, dtype=og_weight_data.dtype) - for i in range(og_out_channels): - ext_weight_data[i * 4] = og_weight_data[i] - og_readout_conv_layer.weight.data = ext_weight_data - # build extended bias and replace if necessary - if og_bias: - ext_bias_shape = (new_out_channels,) - ext_bias_data = torch.zeros(ext_bias_shape, dtype=og_bias_data.dtype) + for exit_layer in model.exit_layers: + # extract the conv layer from dynapcnn network + og_readout_conv_layer = exit_layer.conv_layer + og_weight_data = og_readout_conv_layer.weight.data + og_bias_data = og_readout_conv_layer.bias + og_bias = og_bias_data is not None + # modify the out channels + og_out_channels = og_readout_conv_layer.out_channels + new_out_channels = (og_out_channels - 1) * 4 + 1 + og_readout_conv_layer.out_channels = new_out_channels + # build extended weight and replace the old one + ext_weight_shape = (new_out_channels, *og_weight_data.shape[1:]) + ext_weight_data = torch.zeros(ext_weight_shape, dtype=og_weight_data.dtype) for i in range(og_out_channels): - ext_bias_data[i * 4] = og_bias_data[i] - og_readout_conv_layer.bias.data = ext_bias_data - _ = model( - torch.zeros(size=(1, *input_shape)) - ) # run a forward pass to initialize the new weights and last IAF + ext_weight_data[i * 4] = og_weight_data[i] + og_readout_conv_layer.weight.data = ext_weight_data + # build extended bias and replace if necessary + if og_bias: + ext_bias_shape = (new_out_channels,) + ext_bias_data = torch.zeros(ext_bias_shape, dtype=og_bias_data.dtype) + for i in range(og_out_channels): + ext_bias_data[i * 4] = og_bias_data[i] + og_readout_conv_layer.bias.data = ext_bias_data + # run a forward pass to initialize the new weights and last IAF + model(torch.zeros(size=(1, *input_shape))) return model From 3d4793f8427f9c4b52bf45da2958621fe095224e Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Thu, 31 Oct 2024 17:37:08 +0100 Subject: [PATCH 289/379] Update failing unit tests in `test_large_net` --- sinabs/backend/dynapcnn/dynapcnn_network.py | 4 +++ tests/test_dynapcnn/test_large_net.py | 29 ++++++++++++++++----- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index 208c8312..782fd6ea 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -101,6 +101,10 @@ def dynapcnn_layers(self): @property def dynapcnn_module(self): return self._dynapcnn_module + + @property + def exit_layers(self): + return [self.dynapcnn_layers[i] for i in self._dynapcnn_module.get_exit_layers()] @property def layer_destination_map(self): diff --git a/tests/test_dynapcnn/test_large_net.py b/tests/test_dynapcnn/test_large_net.py index 3bc8681b..6c366b52 100644 --- a/tests/test_dynapcnn/test_large_net.py +++ b/tests/test_dynapcnn/test_large_net.py @@ -88,10 +88,9 @@ def test_same_result(): assert torch.equal(dynapcnn_out.squeeze(), snn_out.squeeze()) +# TODO: Define new test with actual network that is too large. Probably have it as fail case in test_dynapcnnnetwork def test_too_large(): - with pytest.raises(ValueError): - # - Should give an error with the normal layer ordering - dynapcnn_net.make_config(chip_layers_ordering=range(9)) + pass def test_auto_config(): @@ -100,10 +99,25 @@ def test_auto_config(): def test_was_copied(): - # - Make sure that layers of different models are distinct objects - for lyr_snn, lyr_dynapcnn in zip(snn.spiking_model, dynapcnn_net.sequence): - assert lyr_snn is not lyr_dynapcnn + from nirtorch.utils import sanitize_name + # - Make sure that layers of different models are distinct objects + snn_layers = {sanitize_name(name): lyr for name, lyr in snn.named_modules()} + idx_2_name_map = { + idx: sanitize_name(name) for name, idx in dynapcnn_net.name_2_indx_map.items() + } + for idx, lyr_info in dynapcnn_net._graph_extractor.dcnnl_map.items(): + conv_lyr_dynapcnn = dynapcnn_net.dynapcnn_layers[idx].conv_layer + conv_node_idx = lyr_info["conv"]["node_id"] + conv_name = idx_2_name_map[conv_node_idx] + conv_lyr_snn = snn_layers[conv_name] + assert conv_lyr_dynapcnn is not conv_lyr_snn + + spk_lyr_dynapcnn = dynapcnn_net.dynapcnn_layers[idx].spk_layer + spk_node_idx = lyr_info["neuron"]["node_id"] + spk_name = idx_2_name_map[spk_node_idx] + spk_lyr_snn = snn_layers[spk_name] + assert spk_lyr_dynapcnn is not spk_lyr_snn def test_make_config(): dynapcnn_net = DynapcnnNetwork( @@ -162,6 +176,7 @@ def test_extended_readout_layer(out_channels: int): ) extended_net = extend_readout_layer(dynapcnn_net) - converted_channels = extended_net.sequence[-1].conv_layer.out_channels + assert len(exit_layers := extended_net.exit_layers) == 1 + converted_channels = exit_layers[0].conv_layer.out_channels assert (out_channels - 1) * 4 + 1 == converted_channels From bb0e13234d132340d2296d4fbabcf8edcd813a4f Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Thu, 31 Oct 2024 17:48:36 +0100 Subject: [PATCH 290/379] Add `memory_summary` back to DynapcnnNetwork --- sinabs/backend/dynapcnn/dynapcnn_network.py | 22 ++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index 782fd6ea..33e1906b 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -213,6 +213,26 @@ def parameters(self) -> list: return parameters + def memory_summary(self) -> Dict[str, Dict[int, int]]: + """Get a summary of the network's memory requirements. + + Returns + ------- + dict: + A dictionary with keys kernel, neuron, bias. The values are a dicts. + Each nested dict has as keys the indices of all dynapcnn_layers and + as values the corresonding memory values for each layer. + """ + # For each entry (kernel, neuron, bias) provide one nested dict with + # one entry for each layer + summary = {key: dict() for key in ("kernel", "neuron", "bias")} + + for layer_index, layer in self.dynapcnn_layers.items(): + for key, val in layer.memory_summary().items(): + summary[key][layer_index] = val + + return summary + def init_weights(self, init_fn: nn.init = nn.init.xavier_normal_) -> None: """Call the weight initialization method `init_fn` on each `DynapcnnLayer.conv_layer.weight.data` in the `DynapcnnNetwork` instance. @@ -240,7 +260,7 @@ def to( config_modifier: Optional[Callable] = None, slow_clk_frequency: Optional[int] = None, layer2core_map: Union[Dict[int, int], str] = "auto", - chip_layers_ordering="auto", + chip_layers_ordering: Optional[Union[Sequence[int], str]] = None, ): """Deploy model to cpu, gpu or a SynSense device. From 6b1a92707c5d62bd2e100200a17ebb835377d058 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Fri, 1 Nov 2024 11:45:54 +0100 Subject: [PATCH 291/379] WIP improving DVS setup dvs_input (bool) made optional + additional validation steps for DVS setup added --- sinabs/backend/dynapcnn/dynapcnn_network.py | 6 +- sinabs/backend/dynapcnn/exceptions.py | 2 +- .../backend/dynapcnn/nir_graph_extractor.py | 102 ++++++++++++------ .../backend/dynapcnn/sinabs_edges_handler.py | 3 - 4 files changed, 75 insertions(+), 38 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index 479d6f14..d1d1e8d4 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -29,8 +29,7 @@ def __init__( snn: nn.Module, input_shape: Tuple[int, int, int], batch_size: Optional[int] = None, - # TODO: Set None by default - dvs_input: bool = False, + dvs_input: Optional[bool] = None, discretize: bool = True, weight_rescaling_fn: Callable = rescale_method_1, ): @@ -45,7 +44,8 @@ def __init__( - input_shape (tuple): a description of the input dimensions as `(features, height, width)`. - batch_size (optional int): If `None`, will try to infer the batch size from the model. If int value is provided, it has to match the actual batch size of the model. - - dvs_input (bool): wether or not dynapcnn receive input from its DVS camera. + - dvs_input (bool): optional (default as `None`). Wether or not dynapcnn receive + input from its DVS camera. - discretize (bool): If `True`, discretize the parameters and thresholds. This is needed for uploading weights to dynapcnn. Set to `False` only for testing purposes. - weight_rescaling_fn (callable): a method that handles how the re-scaling factor for one or more `SumPool2d` projecting to diff --git a/sinabs/backend/dynapcnn/exceptions.py b/sinabs/backend/dynapcnn/exceptions.py index c22e3b5e..e88324ed 100644 --- a/sinabs/backend/dynapcnn/exceptions.py +++ b/sinabs/backend/dynapcnn/exceptions.py @@ -69,7 +69,7 @@ class InvalidGraphStructure(Exception): class InvalidModelWithDVSSetup(Exception): def __init__(self): - super().__init__(f"The network provided starts with a DVSLayer but 'dvs_input' is set to False.") + super().__init__(f"The network provided has a DVSLayer instance but argument 'dvs_input' is set to False.") # Edge exceptions. diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index 466c9a19..0e88a898 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -17,7 +17,7 @@ from .dynapcnn_layer_utils import construct_dynapcnnlayers_from_mapper from .dynapcnnnetwork_module import DynapcnnNetworkModule from .exceptions import InvalidGraphStructure, InvalidModelWithDVSSetup -from .sinabs_edges_handler import collect_dynapcnn_layer_info, get_dvs_node_from_mapper, fix_dvs_module_edges +from .sinabs_edges_handler import collect_dynapcnn_layer_info, get_dvs_node_from_mapper, fix_dvs_module_edges, merge_dvs_pooling_edge from .utils import Edge, topological_sorting try: @@ -32,7 +32,7 @@ def __init__( self, spiking_model: nn.Module, dummy_input: torch.tensor, - dvs_input: bool, + dvs_input: Optional[bool] = None, ignore_node_types: Optional[Iterable[Type]] = None, ): """Class implementing the extraction of the computational graph from `spiking_model`, where @@ -60,8 +60,8 @@ def __init__( Map from layer ID to the corresponding nn.Module instance. - nodes_io_shapes (dict): Map from node ID to dict containing node's in- and output shapes - - dvs_input (bool): - Whether or not the model should start with a `DVSLayer`. + - dvs_input (bool): optional (default as `None`). Whether or not the model + should start with a `DVSLayer`. - ignore_node_types (iterable of types): Node types that should be ignored completely from the graph. This can include, for instance, `nn.Dropout2d`, which otherwise can result in wrongly inferred @@ -93,23 +93,19 @@ def __init__( # If DVS camera is wanted but `spiking_model` does not start with DVS layer. if self._need_dvs_node(spiking_model, dvs_input): - # TODO: Infer polarity as well. Do that in `_add_dvs_node` - # input shape for `DVSLayer` instance that will be the module of the node 'dvs'. - self._add_dvs_node(dvs_input_shape=dummy_input.shape[1:]) - else: - # TODO: Handle case where DVSLayer is present: - # - make sure it is the only entry node - # - Make sure the `merge_polarities` attribute matches `dummy_input.shape` - pass + # Insert a DVSLayer node in the graph. + self._add_dvs_node(dvs_input_shape=dummy_input.shape) # Check for the need of fixing NIR edges extraction when DVS is a node in the graph. If DVS # is used its node becomes the only entry node in the graph. - fix_dvs_module_edges( - edges=self._edges, - indx_2_module_map=self._indx_2_module_map, - name_2_indx_map=self._name_2_indx_map, - entry_nodes=self._entry_nodes, - ) + fix_dvs_module_edges(self._edges, self._indx_2_module_map, self._name_2_indx_map, self._entry_nodes) + + # Merge a pooling node from a 'dvs-pooling' edge (pooling being an independent node in the original + # graph) into the DVSLayer if such edge exists. + merge_dvs_pooling_edge(self._edges, self._indx_2_module_map, self._name_2_indx_map) + + # Check if graph structure and DVSLayer.merge_polarities are correctly set (if DVS node exists). + self._validate_dvs_setup(dvs_input_shape=dummy_input.shape) # Verify that graph is compatible self.verify_graph_integrity() @@ -287,8 +283,7 @@ def verify_graph_integrity(self): ####################################################### Pivate Methods ####################################################### - # TODO: Needs to receive 3d input shape and sett merge_polarities - def _add_dvs_node(self, dvs_input_shape: Tuple[int, int]) -> None: + def _add_dvs_node(self, dvs_input_shape: Tuple[int, int, int]) -> None: """ In-place modification of `self._name_2_indx_map`, `self._indx_2_module_map`, and `self._edges` to accomodate the creation of an extra node in the graph representing the DVS camera of the chip. The DVSLayer node will point to every other node that is up to this point an entry node of the original graph, so `self._entry_nodes` is modified in-place @@ -296,19 +291,25 @@ def _add_dvs_node(self, dvs_input_shape: Tuple[int, int]) -> None: Parameters ---------- - - dvs_input_shape (tuple): Shape of input in format `(height, width)`. + - dvs_input_shape (tuple): shape of the DVSLayer input in format `(features, height, width)`. """ + (features, height, width) = dvs_input_shape + if features > 2: + raise ValueError(f'A DVSLayer istance can have the feature dimension of its inputs with values 1 or 2 but {features} was given.') + # add name entry for node 'dvs'. self._name_2_indx_map['dvs'] = len(self._name_2_indx_map) # add module entry for node 'dvs'. - self._indx_2_module_map[self._name_2_indx_map['dvs']] = DVSLayer(input_shape=dvs_input_shape) + self._indx_2_module_map[self._name_2_indx_map['dvs']] = DVSLayer( + input_shape=(height, width), + merge_polarities=True if features > 1 else False) # set DVS node as input to each entry node of the graph. self._edges.update({(self._name_2_indx_map['dvs'], entry_node) for entry_node in self._entry_nodes}) # DVSLayer node becomes the only entrypoint of the graph. self._entry_nodes = {self._name_2_indx_map['dvs']} - def _need_dvs_node(self, dvs_input: bool) -> bool: + def _need_dvs_node(self, dvs_input: Optional[bool] = None) -> bool: """ Returns whether or not a node will need to be added to represent a `DVSLayer` instance. @@ -317,30 +318,69 @@ def _need_dvs_node(self, dvs_input: bool) -> bool: Parameters ---------- - - dvs_input (bool): wether or not dynapcnn receive input from its DVS camera. + - dvs_input (bool): optional (default as `None`). Wether or not dynapcnn receive input from its DVS camera. Returns ------- - True if `self._indx_2_module_map` contains a DVSLayer, False otherwise. """ - + + has_dvs_layer = self._has_dvs_layer() + + # Checks if DVSLayer instance exists but user has set 'dvs_input' to False. + if has_dvs_layer and (isinstance(dvs_input, bool) and not dvs_input): + raise InvalidModelWithDVSSetup() + + return not has_dvs_layer and dvs_input + + def _has_dvs_layer(self) -> bool: + """ Loops though all modules and check if a `DVSLayer` instance exists. """ + dvs_layers = { module for module in self._indx_2_module_map.values() if isinstance(module, DVSLayer) } if (num_dvs := len(dvs_layers)) == 0: - has_dvs_layer = False + return False elif num_dvs == 1: - # TODO: Avoid redundant arguments - if not dvs_input: - raise InvalidModelWithDVSSetup() - has_dvs_layer = True + return True else: raise InvalidGraphStructure( f"The provided model has {num_dvs} `DVSLayer`s. At most one is allowed." ) + + def _validate_dvs_setup(self, dvs_input_shape: Tuple[int, int, int]) -> None: + """ If a DVSLayer node exists, makes sure it is the only entry node of the graph and that its `merge_polarities` + attribute matches `dummy_input.shape[0]` (the number of features). - return not has_dvs_layer and dvs_input + Parameters + ---------- + - dvs_input_shape (tuple): shape of the DVSLayer input in format `(features, height, width)`. + """ + + dvs_layer = [module for index, module in self._indx_2_module_map.items() if isinstance(module, DVSLayer)] + + if len(dvs_layer) == 0: + # No DVSLayer found - nothing to do here. + return + elif (nb_dvs := len(dvs_layer)) > 1: + # Can't have more then one DVSLayer instance. + raise InvalidGraphStructure( + f"The provided model has {nb_dvs} `DVSLayer`s. At most one is allowed." + ) + else: + dvs_layer = dvs_layer[-1] + + if (nb_entries := len(self._entry_nodes)) > 1: + raise ValueError(f'A DVSLayer node exists and there are {nb_entries} entry nodes in the graph: the DVSLayer should be the only entry node.') + + (features, _, _) = dvs_input_shape + + if features > 2: + raise ValueError(f'A DVSLayer istance can have the feature dimension of its inputs with values 1 or 2 but {features} was given.') + + if dvs_layer.merge_polarities and features != 1: + raise ValueError(f"The 'DVSLayer.merge_polarities' is set to 'True' which means the number of input features should be 1 (current input shape is {dvs_input_shape}).") def _get_name_2_indx_map(self, nir_graph: TorchGraph) -> Dict[str, int]: """Assign unique index to each node and return mapper from name to index. diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index 8b5b782f..18a55195 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -116,9 +116,6 @@ def fix_dvs_module_edges(edges: Set[Edge], indx_2_module_map: Dict[int, nn.Modul entry_nodes.clear() entry_nodes.add(dvs_node[-1]) - # Merge a pooling node from a 'dvs-pooling' edge (pooling being an independent node in the original graph) into the DVSLayer if such edge exists. - merge_dvs_pooling_edge(edges, indx_2_module_map, name_2_indx_map) - def merge_dvs_pooling_edge(edges: Set[Edge], indx_2_module_map: Dict[int, nn.Module], name_2_indx_map: Dict[str, int]) -> None: """ If a 'dvs-polling' edge existis, the pooling is incorporated into the DVSLayer node if `DVSLayer.pool_layer` has default values. All arguments are modified in-place to remove the references to the incorporated pooling node. From fbf7fff165c6d529afcfa4dbb005ecd61a0e36a0 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Fri, 1 Nov 2024 11:49:48 +0100 Subject: [PATCH 292/379] WIP improving DVS setup dvs_input (bool) passed to get_dynapcnn_network_module() is a return of a class method --- sinabs/backend/dynapcnn/nir_graph_extractor.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index 0e88a898..9e07447c 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -141,7 +141,7 @@ def indx_2_module_map(self) -> Dict[int, nn.Module]: # TODO: Information about `dvs_input` should already be part of `self` at this point. def get_dynapcnn_network_module( - self, discretize: bool = False, weight_rescaling_fn: Optional[Callable] = None, dvs_input: bool = False + self, discretize: bool = False, weight_rescaling_fn: Optional[Callable] = None ) -> DynapcnnNetworkModule: """Create DynapcnnNetworkModule based on stored graph representation @@ -153,7 +153,6 @@ def get_dynapcnn_network_module( weights to dynapcnn. Set to `False` only for testing purposes. - weight_rescaling_fn (callable): a method that handles how the re-scaling factor for one or more `SumPool2d` projecting to the same convolutional layer are combined/re-scaled before applying them. - - dvs_input (bool): wether or not dynapcnn receive input from its DVS camera. Returns ------- @@ -166,7 +165,7 @@ def get_dynapcnn_network_module( edges=self.edges, nodes_io_shapes=self.nodes_io_shapes, entry_nodes=self.entry_nodes, - dvs_input=dvs_input, + dvs_input=self._has_dvs_layer(), ) # build `DynapcnnLayer` instances from mapper. From a9b45f700a7c31678ab9724018148c27825af32f Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Fri, 1 Nov 2024 12:23:28 +0100 Subject: [PATCH 293/379] DONE - improving DVS setup additioanl check for DVSLayer instance properties done + .merge property of DVS chip config done during build_config() --- .../dynapcnn_network/snn_DVSLayer_given.ipynb | 46 +++++---- .../snn_DVSLayer_given_followed_by_pool.ipynb | 46 +++++---- .../snn_need_create_DVSLayer.ipynb | 98 ++++++++++++++++--- .../dynapcnn_network/snn_no_DVSLayer.ipynb | 6 +- sinabs/backend/dynapcnn/chips/dynapcnn.py | 3 + sinabs/backend/dynapcnn/dynapcnn_network.py | 7 +- .../backend/dynapcnn/nir_graph_extractor.py | 19 ++-- 7 files changed, 154 insertions(+), 71 deletions(-) diff --git a/examples/dynapcnn_network/snn_DVSLayer_given.ipynb b/examples/dynapcnn_network/snn_DVSLayer_given.ipynb index 7ccde095..6f4d88d9 100644 --- a/examples/dynapcnn_network/snn_DVSLayer_given.ipynb +++ b/examples/dynapcnn_network/snn_DVSLayer_given.ipynb @@ -2,18 +2,26 @@ "cells": [ { "cell_type": "code", - "execution_count": 17, + "execution_count": 1, "metadata": { "metadata": {} }, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/samurai2077/anaconda3/envs/speck-rescnn/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + }, { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 17, + "execution_count": 1, "metadata": {}, "output_type": "execute_result" } @@ -43,7 +51,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -57,7 +65,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 3, "metadata": { "metadata": {} }, @@ -96,7 +104,7 @@ ")" ] }, - "execution_count": 19, + "execution_count": 3, "metadata": {}, "output_type": "execute_result" } @@ -169,7 +177,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 4, "metadata": { "metadata": {} }, @@ -185,7 +193,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -194,7 +202,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -203,7 +211,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 7, "metadata": {}, "outputs": [ { @@ -238,7 +246,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 8, "metadata": {}, "outputs": [ { @@ -275,8 +283,8 @@ " (2): DynapcnnLayer(\n", " (_conv): Conv2d(10, 10, kernel_size=(4, 4), stride=(1, 1), bias=False)\n", " (_spk): IAFSqueeze(spike_threshold=Parameter containing:\n", - " tensor(130.), min_v_mem=Parameter containing:\n", - " tensor(-130.), batch_size=1, num_timesteps=-1)\n", + " tensor(2084.), min_v_mem=Parameter containing:\n", + " tensor(-2084.), batch_size=1, num_timesteps=-1)\n", " )\n", " (4): DynapcnnLayer(\n", " (_conv): Conv2d(1, 200, kernel_size=(12, 12), stride=(1, 1), bias=False)\n", @@ -284,18 +292,18 @@ " tensor(365.), min_v_mem=Parameter containing:\n", " tensor(-365.), batch_size=1, num_timesteps=-1)\n", " )\n", - " (0): DVSLayer(\n", - " (pool_layer): SumPool2d(norm_type=1, kernel_size=(1, 1), stride=None, ceil_mode=False)\n", - " (crop_layer): Crop2d((0, 34), (0, 34))\n", - " (flip_layer): FlipDims()\n", - " )\n", + " )\n", + " (_dvs_layer): DVSLayer(\n", + " (pool_layer): SumPool2d(norm_type=1, kernel_size=(1, 1), stride=None, ceil_mode=False)\n", + " (crop_layer): Crop2d((0, 34), (0, 34))\n", + " (flip_layer): FlipDims()\n", " )\n", " (merge_layer): Merge()\n", " )\n", ")" ] }, - "execution_count": 24, + "execution_count": 8, "metadata": {}, "output_type": "execute_result" } diff --git a/examples/dynapcnn_network/snn_DVSLayer_given_followed_by_pool.ipynb b/examples/dynapcnn_network/snn_DVSLayer_given_followed_by_pool.ipynb index 8da2c368..3b261693 100644 --- a/examples/dynapcnn_network/snn_DVSLayer_given_followed_by_pool.ipynb +++ b/examples/dynapcnn_network/snn_DVSLayer_given_followed_by_pool.ipynb @@ -2,18 +2,26 @@ "cells": [ { "cell_type": "code", - "execution_count": 9, + "execution_count": 1, "metadata": { "metadata": {} }, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/samurai2077/anaconda3/envs/speck-rescnn/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + }, { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 9, + "execution_count": 1, "metadata": {}, "output_type": "execute_result" } @@ -43,7 +51,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -57,7 +65,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 3, "metadata": { "metadata": {} }, @@ -97,7 +105,7 @@ ")" ] }, - "execution_count": 11, + "execution_count": 3, "metadata": {}, "output_type": "execute_result" } @@ -173,7 +181,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 4, "metadata": { "metadata": {} }, @@ -189,7 +197,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -198,7 +206,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -207,7 +215,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 7, "metadata": {}, "outputs": [ { @@ -242,7 +250,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 8, "metadata": {}, "outputs": [ { @@ -279,8 +287,8 @@ " (2): DynapcnnLayer(\n", " (_conv): Conv2d(10, 10, kernel_size=(4, 4), stride=(1, 1), bias=False)\n", " (_spk): IAFSqueeze(spike_threshold=Parameter containing:\n", - " tensor(130.), min_v_mem=Parameter containing:\n", - " tensor(-130.), batch_size=1, num_timesteps=-1)\n", + " tensor(2084.), min_v_mem=Parameter containing:\n", + " tensor(-2084.), batch_size=1, num_timesteps=-1)\n", " )\n", " (4): DynapcnnLayer(\n", " (_conv): Conv2d(1, 200, kernel_size=(12, 12), stride=(1, 1), bias=False)\n", @@ -288,18 +296,18 @@ " tensor(365.), min_v_mem=Parameter containing:\n", " tensor(-365.), batch_size=1, num_timesteps=-1)\n", " )\n", - " (0): DVSLayer(\n", - " (pool_layer): SumPool2d(norm_type=1, kernel_size=(1, 1), stride=None, ceil_mode=False)\n", - " (crop_layer): Crop2d((0, 34), (0, 34))\n", - " (flip_layer): FlipDims()\n", - " )\n", + " )\n", + " (_dvs_layer): DVSLayer(\n", + " (pool_layer): SumPool2d(norm_type=1, kernel_size=(1, 1), stride=None, ceil_mode=False)\n", + " (crop_layer): Crop2d((0, 34), (0, 34))\n", + " (flip_layer): FlipDims()\n", " )\n", " (merge_layer): Merge()\n", " )\n", ")" ] }, - "execution_count": 16, + "execution_count": 8, "metadata": {}, "output_type": "execute_result" } diff --git a/examples/dynapcnn_network/snn_need_create_DVSLayer.ipynb b/examples/dynapcnn_network/snn_need_create_DVSLayer.ipynb index 2bc77030..4e17a034 100644 --- a/examples/dynapcnn_network/snn_need_create_DVSLayer.ipynb +++ b/examples/dynapcnn_network/snn_need_create_DVSLayer.ipynb @@ -2,11 +2,20 @@ "cells": [ { "cell_type": "code", - "execution_count": 7, + "execution_count": 1, "metadata": { "metadata": {} }, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/samurai2077/anaconda3/envs/speck-rescnn/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + } + ], "source": [ "import torch\n", "import torch.nn as nn\n", @@ -29,7 +38,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 2, "metadata": { "metadata": {} }, @@ -41,7 +50,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -55,7 +64,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 4, "metadata": { "metadata": {} }, @@ -89,7 +98,7 @@ ")" ] }, - "execution_count": 10, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } @@ -159,7 +168,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 5, "metadata": { "metadata": {} }, @@ -176,7 +185,64 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "----------------------- [ DynapcnnLayer 1 ] -----------------------\n", + "DynapcnnLayer(\n", + " (_conv): Conv2d(1, 10, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + " (_spk): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(174.), min_v_mem=Parameter containing:\n", + " tensor(-174.), batch_size=1, num_timesteps=-1)\n", + ")\n", + "\n", + "----------------------- [ DynapcnnLayer 5 ] -----------------------\n", + "DynapcnnLayer(\n", + " (_conv): Conv2d(200, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + " (_spk): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(419.), min_v_mem=Parameter containing:\n", + " tensor(-419.), batch_size=1, num_timesteps=-1)\n", + ")\n", + "\n", + "----------------------- [ DynapcnnLayer 2 ] -----------------------\n", + "DynapcnnLayer(\n", + " (_conv): Conv2d(10, 10, kernel_size=(4, 4), stride=(1, 1), bias=False)\n", + " (_spk): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(2084.), min_v_mem=Parameter containing:\n", + " tensor(-2084.), batch_size=1, num_timesteps=-1)\n", + ")\n", + "\n", + "----------------------- [ DynapcnnLayer 3 ] -----------------------\n", + "DynapcnnLayer(\n", + " (_conv): Conv2d(10, 1, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + " (_spk): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(303.), min_v_mem=Parameter containing:\n", + " tensor(-303.), batch_size=1, num_timesteps=-1)\n", + ")\n", + "\n", + "----------------------- [ DynapcnnLayer 4 ] -----------------------\n", + "DynapcnnLayer(\n", + " (_conv): Conv2d(1, 200, kernel_size=(12, 12), stride=(1, 1), bias=False)\n", + " (_spk): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(365.), min_v_mem=Parameter containing:\n", + " tensor(-365.), batch_size=1, num_timesteps=-1)\n", + ")\n", + "\n", + "\n" + ] + } + ], + "source": [ + "print(hw_model)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, "metadata": {}, "outputs": [ { @@ -207,8 +273,8 @@ " (2): DynapcnnLayer(\n", " (_conv): Conv2d(10, 10, kernel_size=(4, 4), stride=(1, 1), bias=False)\n", " (_spk): IAFSqueeze(spike_threshold=Parameter containing:\n", - " tensor(130.), min_v_mem=Parameter containing:\n", - " tensor(-130.), batch_size=1, num_timesteps=-1)\n", + " tensor(2084.), min_v_mem=Parameter containing:\n", + " tensor(-2084.), batch_size=1, num_timesteps=-1)\n", " )\n", " (3): DynapcnnLayer(\n", " (_conv): Conv2d(10, 1, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", @@ -222,18 +288,18 @@ " tensor(365.), min_v_mem=Parameter containing:\n", " tensor(-365.), batch_size=1, num_timesteps=-1)\n", " )\n", - " (0): DVSLayer(\n", - " (pool_layer): SumPool2d(norm_type=1, kernel_size=(1, 1), stride=None, ceil_mode=False)\n", - " (crop_layer): Crop2d((0, 34), (0, 34))\n", - " (flip_layer): FlipDims()\n", - " )\n", + " )\n", + " (_dvs_layer): DVSLayer(\n", + " (pool_layer): SumPool2d(norm_type=1, kernel_size=(1, 1), stride=None, ceil_mode=False)\n", + " (crop_layer): Crop2d((0, 34), (0, 34))\n", + " (flip_layer): FlipDims()\n", " )\n", " (merge_layer): Merge()\n", " )\n", ")" ] }, - "execution_count": 12, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } diff --git a/examples/dynapcnn_network/snn_no_DVSLayer.ipynb b/examples/dynapcnn_network/snn_no_DVSLayer.ipynb index aa42aeaf..1869df3c 100644 --- a/examples/dynapcnn_network/snn_no_DVSLayer.ipynb +++ b/examples/dynapcnn_network/snn_no_DVSLayer.ipynb @@ -18,7 +18,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 1, @@ -269,8 +269,8 @@ " (1): DynapcnnLayer(\n", " (_conv): Conv2d(10, 10, kernel_size=(4, 4), stride=(1, 1), bias=False)\n", " (_spk): IAFSqueeze(spike_threshold=Parameter containing:\n", - " tensor(130.), min_v_mem=Parameter containing:\n", - " tensor(-130.), batch_size=1, num_timesteps=-1)\n", + " tensor(2084.), min_v_mem=Parameter containing:\n", + " tensor(-2084.), batch_size=1, num_timesteps=-1)\n", " )\n", " (2): DynapcnnLayer(\n", " (_conv): Conv2d(10, 1, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", diff --git a/sinabs/backend/dynapcnn/chips/dynapcnn.py b/sinabs/backend/dynapcnn/chips/dynapcnn.py index 6e4fb536..2ded7eac 100644 --- a/sinabs/backend/dynapcnn/chips/dynapcnn.py +++ b/sinabs/backend/dynapcnn/chips/dynapcnn.py @@ -65,6 +65,9 @@ def write_dvs_layer_config( chip_layer.pass_sensor_events = True + if layer.merge_polarities: + chip_layer.merge = True + @classmethod def set_kill_bits(cls, layer: DynapcnnLayer, config_dict: dict) -> dict: """This method updates all the kill_bit parameters. diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index d1d1e8d4..ea4544c9 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -80,7 +80,7 @@ def __init__( # Module to execute forward pass through network self._dynapcnn_module = self._graph_extractor.get_dynapcnn_network_module( - discretize=discretize, weight_rescaling_fn=weight_rescaling_fn, dvs_input=self.dvs_input + discretize=discretize, weight_rescaling_fn=weight_rescaling_fn ) self._dynapcnn_module.setup_dynapcnnlayer_graph(index_layers_topologically=True) @@ -564,11 +564,6 @@ def _make_config( dvs_node_info=self.dvs_node_info, ) - # TODO: This should be handled earlier and probably raise a warning if it contradicts - # dvs_layer merge_polarities - if self.input_shape and self.input_shape[0] == 1: - config.dvs_layer.merge = True - if monitor_layers is None: # Monitor all layers with exit point destinations monitor_layers = self._dynapcnn_module.get_exit_layers() diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index 9e07447c..373c731f 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -92,9 +92,9 @@ def __init__( self._entry_nodes = self._get_entry_nodes(self._edges) # If DVS camera is wanted but `spiking_model` does not start with DVS layer. - if self._need_dvs_node(spiking_model, dvs_input): + if self._need_dvs_node(dvs_input): # Insert a DVSLayer node in the graph. - self._add_dvs_node(dvs_input_shape=dummy_input.shape) + self._add_dvs_node(dvs_input_shape=dummy_input.shape[1:]) # Check for the need of fixing NIR edges extraction when DVS is a node in the graph. If DVS # is used its node becomes the only entry node in the graph. @@ -105,7 +105,7 @@ def __init__( merge_dvs_pooling_edge(self._edges, self._indx_2_module_map, self._name_2_indx_map) # Check if graph structure and DVSLayer.merge_polarities are correctly set (if DVS node exists). - self._validate_dvs_setup(dvs_input_shape=dummy_input.shape) + self._validate_dvs_setup(dvs_input_shape=dummy_input.shape[1:]) # Verify that graph is compatible self.verify_graph_integrity() @@ -139,7 +139,6 @@ def sorted_nodes(self) -> List[int]: def indx_2_module_map(self) -> Dict[int, nn.Module]: return {n: module for n, module in self._indx_2_module_map.items()} - # TODO: Information about `dvs_input` should already be part of `self` at this point. def get_dynapcnn_network_module( self, discretize: bool = False, weight_rescaling_fn: Optional[Callable] = None ) -> DynapcnnNetworkModule: @@ -317,10 +316,10 @@ def _need_dvs_node(self, dvs_input: Optional[bool] = None) -> bool: Parameters ---------- - - dvs_input (bool): optional (default as `None`). Wether or not dynapcnn receive input from its DVS camera. + - dvs_input (bool): optional (default as `None`). Wether or not dynapcnn receive input from its DVS camera. Returns ------- - - True if `self._indx_2_module_map` contains a DVSLayer, False otherwise. + - True if `self._indx_2_module_map` contains a DVSLayer, False otherwise. """ has_dvs_layer = self._has_dvs_layer() @@ -349,8 +348,9 @@ def _has_dvs_layer(self) -> bool: ) def _validate_dvs_setup(self, dvs_input_shape: Tuple[int, int, int]) -> None: - """ If a DVSLayer node exists, makes sure it is the only entry node of the graph and that its `merge_polarities` - attribute matches `dummy_input.shape[0]` (the number of features). + """ If a DVSLayer node exists, makes sure it is the only entry node of the graph. Checks if its `merge_polarities` + attribute matches `dummy_input.shape[0]` (the number of features) and, if not, it will be set based on the numeber of + features of the input. Parameters ---------- @@ -380,6 +380,9 @@ def _validate_dvs_setup(self, dvs_input_shape: Tuple[int, int, int]) -> None: if dvs_layer.merge_polarities and features != 1: raise ValueError(f"The 'DVSLayer.merge_polarities' is set to 'True' which means the number of input features should be 1 (current input shape is {dvs_input_shape}).") + + if features == 1: + dvs_layer.merge_polarities = True def _get_name_2_indx_map(self, nir_graph: TorchGraph) -> Dict[str, int]: """Assign unique index to each node and return mapper from name to index. From 327cfa11b7e06905d0f0a9eafdb3f7343fe90d0d Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Mon, 4 Nov 2024 14:45:21 +0100 Subject: [PATCH 294/379] (WIP) Merge Conv2d with BatchNorm2d check if conv.bias exists before accessing it in merge_conv_bn() --- sinabs/backend/dynapcnn/utils.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/sinabs/backend/dynapcnn/utils.py b/sinabs/backend/dynapcnn/utils.py index 05f9c63c..53018175 100644 --- a/sinabs/backend/dynapcnn/utils.py +++ b/sinabs/backend/dynapcnn/utils.py @@ -255,8 +255,6 @@ def construct_dvs_layer( def merge_conv_bn(conv, bn): """Merge a convolutional layer with subsequent batch normalization. - # TODO: new implementation of 'DynapcnnLayer' is not handling BN layers yet. - Parameters ---------- conv: torch.nn.Conv2d @@ -284,7 +282,8 @@ def merge_conv_bn(conv, bn): conv = deepcopy(conv) # TODO: this will cause copying twice conv.weight.data = c_weight * factor[:, None, None, None] - conv.bias.data = beta + (c_bias - mu) * factor + if conv.bias: + conv.bias.data = beta + (c_bias - mu) * factor return conv From bf8d4505ec4f31c03d01d4ad74d63137e8a4b8c5 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Mon, 4 Nov 2024 14:47:00 +0100 Subject: [PATCH 295/379] (WIP) Merge Conv2d with BatchNorm2d conv-batchnorm edges being handled at the constructor of the graph extractor --- .../backend/dynapcnn/nir_graph_extractor.py | 6 +- .../backend/dynapcnn/sinabs_edges_handler.py | 84 ++++++++++++++++++- 2 files changed, 88 insertions(+), 2 deletions(-) diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index 373c731f..0a7b35f8 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -17,7 +17,7 @@ from .dynapcnn_layer_utils import construct_dynapcnnlayers_from_mapper from .dynapcnnnetwork_module import DynapcnnNetworkModule from .exceptions import InvalidGraphStructure, InvalidModelWithDVSSetup -from .sinabs_edges_handler import collect_dynapcnn_layer_info, get_dvs_node_from_mapper, fix_dvs_module_edges, merge_dvs_pooling_edge +from .sinabs_edges_handler import collect_dynapcnn_layer_info, get_dvs_node_from_mapper, fix_dvs_module_edges, merge_dvs_pooling_edge, handle_batchnorm2d_nodes from .utils import Edge, topological_sorting try: @@ -88,6 +88,9 @@ def __init__( # Store the associated `nn.Module` (layer) of each node. self._indx_2_module_map = self._get_named_modules(spiking_model) + # Merges BatchNorm2d nodes with Conv2d ones. + handle_batchnorm2d_nodes(self._edges, self._indx_2_module_map, self._name_2_indx_map) + # Determine entry points to graph self._entry_nodes = self._get_entry_nodes(self._edges) @@ -113,6 +116,7 @@ def __init__( # retrieves what the I/O shape for each node's module is. self._nodes_io_shapes = self._get_nodes_io_shapes(dummy_input) + ####################################################### Publich Methods ####################################################### @property diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index 18a55195..f77264be 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -12,13 +12,95 @@ from .connectivity_specs import VALID_SINABS_EDGE_TYPES, Pooling from .exceptions import InvalidEdge, InvalidGraphStructure -from .utils import Edge +from .utils import Edge, merge_conv_bn from .dvs_layer import DVSLayer from .crop2d import Crop2d from .flipdims import FlipDims from sinabs.layers import SumPool2d +def remap_edges_after_drop(dropped_node: int, source_of_dropped_node: int, edges: Set[Edge]) -> Set[Edge]: + """ + """ + remapped_edges = set() + + for (src, tgt) in edges: + if src == dropped_node: + remapped_edges.add((source_of_dropped_node, tgt)) + + return remapped_edges + +def handle_batchnorm2d_nodes(edges: Set[Edge], indx_2_module_map: Dict[int, nn.Module], name_2_indx_map: Dict[str, int]) -> None: + """ + + Parameters + ---------- + - edges (set): tuples describing the connections between layers in `spiking_model`. + - indx_2_module_map (dict): the mapping between a node (`key` as an `int`) and its module (`value` as a `nn.Module`). + - name_2_indx_map (dict): Map from node names to unique indices. + - entry_nodes (set): IDs of nodes acting as entry points for the network (i.e., receiving external input). + """ + + print('----------------------------------------') + print(edges) + for key, val in indx_2_module_map.items(): + print(key, val) + print('----------------------------------------') + + # Gather indexes of the BatchNorm2d nodes. + bnorm_nodes = { + index for index, module in indx_2_module_map.items() + if isinstance(module, nn.BatchNorm2d) + } + + if len(bnorm_nodes) == 0: + # There are no edges with BatchNorm2d - nothing to do here. + return + + # Find conv-bnorm edges. + conv_bnorm_edges = { + (src, tgt) for (src, tgt) in edges + if isinstance(indx_2_module_map[src], nn.Conv2d) and isinstance(indx_2_module_map[tgt], nn.BatchNorm2d) + } + + # Merge conv and bnorm layers using 'conv-bnorm' edges. + for edge in conv_bnorm_edges: + # merge and update conv node. + bnorm = indx_2_module_map[edge[1]] + conv = indx_2_module_map[edge[0]] + indx_2_module_map[edge[0]] = merge_conv_bn(conv, bnorm) + + # Point Conv2d nodes to the targets of their respective BatchNorm2d nodes. + for conv, bnorm in conv_bnorm_edges: + new_edges = remap_edges_after_drop(dropped_node=bnorm, source_of_dropped_node=conv, edges=edges) + + # Remove references to the bnorm node. + + for idx in bnorm_nodes: + indx_2_module_map.pop(idx) + + for name in [name for name, indx in name_2_indx_map.items() if indx in bnorm_nodes]: + name_2_indx_map.pop(name) + + for edge in conv_bnorm_edges: + edges.remove(edge) + + for edge in [(src, tgt) for (src, tgt) in edges if (src in bnorm_nodes or tgt in bnorm_nodes)]: + edges.remove(edge) + + # Update 'edges' in-place to incorporate new edges: + print(new_edges) + for edge in new_edges: + edges.add(edge) + + print('++++++++++++++++++++++++++++++++++++++++++++++') + print(edges) + for key, val in indx_2_module_map.items(): + print(key, val) + print('++++++++++++++++++++++++++++++++++++++++++++++') + + + def get_dvs_node_from_mapper(dcnnl_map: Dict) -> Optional[Dict]: """ Returns the information dict associated with the `DVSLayer` instance within `dcnnl_map`. From 10b887d72e1e725b7c44d723591bea16dbff18e5 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Mon, 4 Nov 2024 15:12:03 +0100 Subject: [PATCH 296/379] (WIP) Merge Conv2d with BatchNorm2d completed docstrings for new funcitons --- .../backend/dynapcnn/sinabs_edges_handler.py | 33 +++++++++---------- 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index f77264be..8b4477d4 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -20,7 +20,18 @@ from sinabs.layers import SumPool2d def remap_edges_after_drop(dropped_node: int, source_of_dropped_node: int, edges: Set[Edge]) -> Set[Edge]: - """ + """ Creates a new set of edges from `edges`. All edges where `dropped_node` is the source node will be used to generate + a new edge where `source_of_dropped_node` becomes the source node (the target is kept). + + Parameters + ---------- + - dropped_node (int): + - source_of_dropped_node (int): + - edges (set): tuples describing the connections between layers in `spiking_model`. + + Returns + ------- + - remapped_edges (set): new set of edges with `source_of_dropped_node` as the source node where `dropped_node` used to be. """ remapped_edges = set() @@ -31,22 +42,17 @@ def remap_edges_after_drop(dropped_node: int, source_of_dropped_node: int, edges return remapped_edges def handle_batchnorm2d_nodes(edges: Set[Edge], indx_2_module_map: Dict[int, nn.Module], name_2_indx_map: Dict[str, int]) -> None: - """ + """ Merges `BatchNorm2d` layers into `Conv2d` ones. The `BatchNorm2d` nodes will be removed from the graph (by updating all variables + passed as arguments in-place) after their properties are used to re-scale the weights of the convolutional layers associated with batch + normalization via `conv-batchnorm` edges. Parameters ---------- - edges (set): tuples describing the connections between layers in `spiking_model`. - indx_2_module_map (dict): the mapping between a node (`key` as an `int`) and its module (`value` as a `nn.Module`). - name_2_indx_map (dict): Map from node names to unique indices. - - entry_nodes (set): IDs of nodes acting as entry points for the network (i.e., receiving external input). """ - print('----------------------------------------') - print(edges) - for key, val in indx_2_module_map.items(): - print(key, val) - print('----------------------------------------') - # Gather indexes of the BatchNorm2d nodes. bnorm_nodes = { index for index, module in indx_2_module_map.items() @@ -89,18 +95,9 @@ def handle_batchnorm2d_nodes(edges: Set[Edge], indx_2_module_map: Dict[int, nn.M edges.remove(edge) # Update 'edges' in-place to incorporate new edges: - print(new_edges) for edge in new_edges: edges.add(edge) - print('++++++++++++++++++++++++++++++++++++++++++++++') - print(edges) - for key, val in indx_2_module_map.items(): - print(key, val) - print('++++++++++++++++++++++++++++++++++++++++++++++') - - - def get_dvs_node_from_mapper(dcnnl_map: Dict) -> Optional[Dict]: """ Returns the information dict associated with the `DVSLayer` instance within `dcnnl_map`. From 50991afe28d793585e14df14790857ce0ec71592 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Mon, 4 Nov 2024 15:23:35 +0100 Subject: [PATCH 297/379] (WIP) Merge Conv2d with BatchNorm2d added a function to merge nn.Linear followed by BatchNorm1d --- sinabs/backend/dynapcnn/utils.py | 34 ++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/sinabs/backend/dynapcnn/utils.py b/sinabs/backend/dynapcnn/utils.py index 53018175..db01b496 100644 --- a/sinabs/backend/dynapcnn/utils.py +++ b/sinabs/backend/dynapcnn/utils.py @@ -287,6 +287,40 @@ def merge_conv_bn(conv, bn): return conv +def merge_linear_bn(linear, bn): + """Merge a linear (fully connected) layer with subsequent batch normalization. + + Parameters + ---------- + linear: torch.nn.Linear + Linear layer + bn: torch.nn.BatchNorm1d + Batch normalization layer + + Returns + ------- + torch.nn.Linear: Linear layer including batch normalization + """ + mu = bn.running_mean + sigmasq = bn.running_var + + if bn.affine: + gamma, beta = bn.weight, bn.bias + else: + gamma, beta = 1.0, 0.0 + + factor = gamma / sigmasq.sqrt() + + l_weight = linear.weight.data.clone().detach() + l_bias = 0.0 if linear.bias is None else linear.bias.data.clone().detach() + + linear = deepcopy(linear) + + linear.weight.data = l_weight * factor[:, None] + if linear.bias is not None: + linear.bias.data = beta + (l_bias - mu) * factor + + return linear # Should become obsolete def construct_next_pooling_layer( From 5ad1211882c1a3f4d791ccc1cdb85f9f520c3234 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Mon, 4 Nov 2024 16:05:41 +0100 Subject: [PATCH 298/379] (DONE) Merge Conv2d with BatchNorm2d expanded to handle (nn.Linear, nn.BatchNorm1d) edges but there's still a problem. --- .../dynapcnn_network/snn_with_batchnorm.ipynb | 300 ++++++++++++++++++ .../backend/dynapcnn/sinabs_edges_handler.py | 49 +-- sinabs/backend/dynapcnn/utils.py | 2 +- 3 files changed, 330 insertions(+), 21 deletions(-) create mode 100644 examples/dynapcnn_network/snn_with_batchnorm.ipynb diff --git a/examples/dynapcnn_network/snn_with_batchnorm.ipynb b/examples/dynapcnn_network/snn_with_batchnorm.ipynb new file mode 100644 index 00000000..fdd6b337 --- /dev/null +++ b/examples/dynapcnn_network/snn_with_batchnorm.ipynb @@ -0,0 +1,300 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "metadata": {} + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/samurai2077/anaconda3/envs/speck-rescnn/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import torch\n", + "import torch.nn as nn\n", + "from sinabs.backend.dynapcnn import DynapcnnNetwork\n", + "from sinabs.backend.dynapcnn import DVSLayer\n", + "from sinabs.layers import Merge, IAFSqueeze, SumPool2d\n", + "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential\n", + "import sinabs.layers as sl\n", + "\n", + "from torch.nn import CrossEntropyLoss\n", + "from torch.optim import Adam\n", + "\n", + "from tonic.datasets.nmnist import NMNIST\n", + "from tonic.transforms import ToFrame\n", + "from torch.utils.data import DataLoader\n", + "import numpy as np\n", + "from tqdm.notebook import tqdm\n", + "from statistics import mode\n", + "\n", + "device = torch.device('cpu')\n", + "torch.manual_seed(0)" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "channels = 1\n", + "height = 34\n", + "width = 34\n", + "batch_size = 1\n", + "\n", + "input_shape = (channels, height, width)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "metadata": {} + }, + "outputs": [ + { + "data": { + "text/plain": [ + "SNN(\n", + " (conv1): Conv2d(1, 10, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + " (bn1): BatchNorm2d(10, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (iaf1): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(1.), min_v_mem=Parameter containing:\n", + " tensor(-1.), batch_size=1, num_timesteps=-1)\n", + " (pool1): AvgPool2d(kernel_size=2, stride=2, padding=0)\n", + " (conv2): Conv2d(10, 10, kernel_size=(4, 4), stride=(1, 1), bias=False)\n", + " (bn2): BatchNorm2d(10, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (iaf2): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(1.), min_v_mem=Parameter containing:\n", + " tensor(-1.), batch_size=1, num_timesteps=-1)\n", + " (conv3): Conv2d(10, 1, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + " (bn3): BatchNorm2d(1, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (iaf3): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(1.), min_v_mem=Parameter containing:\n", + " tensor(-1.), batch_size=1, num_timesteps=-1)\n", + " (fc1): Linear(in_features=144, out_features=200, bias=False)\n", + " (iaf4): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(1.), min_v_mem=Parameter containing:\n", + " tensor(-1.), batch_size=1, num_timesteps=-1)\n", + " (fc2): Linear(in_features=200, out_features=10, bias=False)\n", + " (iaf5): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(1.), min_v_mem=Parameter containing:\n", + " tensor(-1.), batch_size=1, num_timesteps=-1)\n", + " (flat): Flatten(start_dim=1, end_dim=-1)\n", + ")" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "class SNN(nn.Module):\n", + " def __init__(self, input_shape) -> None:\n", + " super().__init__()\n", + "\n", + " # -- chip core A --\n", + " self.conv1 = nn.Conv2d(1, 10, 2, 1, bias=False)\n", + " self.bn1 = nn.BatchNorm2d(10)\n", + " self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential())\n", + " self.pool1 = nn.AvgPool2d(2,2)\n", + " # -- chip core B --\n", + " self.conv2 = nn.Conv2d(10, 10, 4, 1, bias=False)\n", + " self.bn2 = nn.BatchNorm2d(10)\n", + " self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential())\n", + " # -- chip core C --\n", + " self.conv3 = nn.Conv2d(10, 1, 2, 1, bias=False)\n", + " self.bn3 = nn.BatchNorm2d(1)\n", + " self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential())\n", + " # -- chip core D --\n", + " self.fc1 = nn.Linear(144, 200, bias=False)\n", + " self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential())\n", + " # -- chip core E --\n", + " self.fc2 = nn.Linear(200, 10, bias=False)\n", + " self.iaf5 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential())\n", + "\n", + " # -- layers ignored during deployment --\n", + " self.flat = nn.Flatten()\n", + "\n", + " def init_weights(self):\n", + " for name, layer in self.named_modules():\n", + " if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear):\n", + " nn.init.xavier_normal_(layer.weight.data)\n", + "\n", + " def detach_neuron_states(self):\n", + " for name, layer in self.named_modules():\n", + " if name != '':\n", + " if isinstance(layer, sl.StatefulLayer):\n", + " for name, buffer in layer.named_buffers():\n", + " buffer.detach_()\n", + "\n", + " def forward(self, x):\n", + " \n", + " con1_out = self.conv1(x) # 4\n", + " bn1_out = self.bn1(con1_out)\n", + " iaf1_out = self.iaf1(bn1_out) # 5\n", + " pool1_out = self.pool1(iaf1_out) # 6\n", + "\n", + " conv2_out = self.conv2(pool1_out) # 7\n", + " bn2_out = self.bn2(conv2_out)\n", + " iaf2_out = self.iaf2(bn2_out) # 8\n", + "\n", + " conv3_out = self.conv3(iaf2_out) # 9\n", + " bn3_out = self.bn3(conv3_out)\n", + " iaf3_out = self.iaf3(bn3_out) # 10\n", + "\n", + " flat_out = self.flat(iaf3_out) # 15\n", + " \n", + " fc1_out = self.fc1(flat_out) # 11\n", + " iaf4_out = self.iaf4(fc1_out) # 12\n", + " fc2_out = self.fc2(iaf4_out) # 13\n", + " iaf5_out = self.iaf5(fc2_out) # 14\n", + "\n", + " return iaf5_out\n", + " \n", + "snn = SNN(input_shape)\n", + "snn.init_weights()\n", + "snn.to(device)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "metadata": {} + }, + "outputs": [], + "source": [ + "hw_model = DynapcnnNetwork(\n", + " snn=snn,\n", + " input_shape=input_shape,\n", + " batch_size=batch_size,\n", + " discretize=True\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "input_dummy = torch.randn((batch_size, *input_shape))" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "out = hw_model(input_dummy)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "tensor([[[[0.]],\n", + "\n", + " [[0.]],\n", + "\n", + " [[0.]],\n", + "\n", + " [[0.]],\n", + "\n", + " [[0.]],\n", + "\n", + " [[0.]],\n", + "\n", + " [[0.]],\n", + "\n", + " [[0.]],\n", + "\n", + " [[0.]],\n", + "\n", + " [[0.]]]], grad_fn=)\n" + ] + } + ], + "source": [ + "print(out)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Network is valid\n" + ] + }, + { + "ename": "KeyError", + "evalue": "'speck2fdevkit:0'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mKeyError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[8], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mhw_model\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mto\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdevice\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mspeck2fdevkit\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/Github/sinabs/sinabs/backend/dynapcnn/dynapcnn_network.py:339\u001b[0m, in \u001b[0;36mDynapcnnNetwork.to\u001b[0;34m(self, device, monitor_layers, config_modifier, slow_clk_frequency, layer2core_map, chip_layers_ordering)\u001b[0m\n\u001b[1;32m 330\u001b[0m config \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmake_config(\n\u001b[1;32m 331\u001b[0m layer2core_map\u001b[38;5;241m=\u001b[39mlayer2core_map,\n\u001b[1;32m 332\u001b[0m chip_layers_ordering\u001b[38;5;241m=\u001b[39mchip_layers_ordering,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 335\u001b[0m config_modifier\u001b[38;5;241m=\u001b[39mconfig_modifier,\n\u001b[1;32m 336\u001b[0m )\n\u001b[1;32m 338\u001b[0m \u001b[38;5;66;03m# apply configuration to device.\u001b[39;00m\n\u001b[0;32m--> 339\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msamna_device \u001b[38;5;241m=\u001b[39m \u001b[43mopen_device\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdevice\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 340\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msamna_device\u001b[38;5;241m.\u001b[39mget_model()\u001b[38;5;241m.\u001b[39mapply_configuration(config)\n\u001b[1;32m 341\u001b[0m time\u001b[38;5;241m.\u001b[39msleep(\u001b[38;5;241m1\u001b[39m)\n", + "File \u001b[0;32m~/Github/sinabs/sinabs/backend/dynapcnn/io.py:255\u001b[0m, in \u001b[0;36mopen_device\u001b[0;34m(device_id)\u001b[0m\n\u001b[1;32m 253\u001b[0m device_id \u001b[38;5;241m=\u001b[39m standardize_device_id(device_id\u001b[38;5;241m=\u001b[39mdevice_id)\n\u001b[1;32m 254\u001b[0m device_map \u001b[38;5;241m=\u001b[39m get_device_map()\n\u001b[0;32m--> 255\u001b[0m device_info \u001b[38;5;241m=\u001b[39m \u001b[43mdevice_map\u001b[49m\u001b[43m[\u001b[49m\u001b[43mdevice_id\u001b[49m\u001b[43m]\u001b[49m\n\u001b[1;32m 256\u001b[0m device_handle \u001b[38;5;241m=\u001b[39m samna\u001b[38;5;241m.\u001b[39mdevice\u001b[38;5;241m.\u001b[39mopen_device(device_info)\n\u001b[1;32m 258\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m device_handle \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n", + "\u001b[0;31mKeyError\u001b[0m: 'speck2fdevkit:0'" + ] + } + ], + "source": [ + "hw_model.to(device=\"speck2fdevkit\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "speck-rescnn", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index 8b4477d4..aa3c177f 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -12,7 +12,7 @@ from .connectivity_specs import VALID_SINABS_EDGE_TYPES, Pooling from .exceptions import InvalidEdge, InvalidGraphStructure -from .utils import Edge, merge_conv_bn +from .utils import Edge, merge_conv_bn, merge_linear_bn from .dvs_layer import DVSLayer from .crop2d import Crop2d from .flipdims import FlipDims @@ -42,9 +42,9 @@ def remap_edges_after_drop(dropped_node: int, source_of_dropped_node: int, edges return remapped_edges def handle_batchnorm2d_nodes(edges: Set[Edge], indx_2_module_map: Dict[int, nn.Module], name_2_indx_map: Dict[str, int]) -> None: - """ Merges `BatchNorm2d` layers into `Conv2d` ones. The `BatchNorm2d` nodes will be removed from the graph (by updating all variables - passed as arguments in-place) after their properties are used to re-scale the weights of the convolutional layers associated with batch - normalization via `conv-batchnorm` edges. + """ Merges `BatchNorm2d`/`BatchNorm1d` layers into `Conv2d`/`Linear` ones. The batch norm nodes will be removed from the graph (by updating all variables + passed as arguments in-place) after their properties are used to re-scale the weights of the convolutional/linear layers associated with batch + normalization via the `weight-batchnorm` edges found in the original graph. Parameters ---------- @@ -53,32 +53,41 @@ def handle_batchnorm2d_nodes(edges: Set[Edge], indx_2_module_map: Dict[int, nn.M - name_2_indx_map (dict): Map from node names to unique indices. """ - # Gather indexes of the BatchNorm2d nodes. + # Gather indexes of the BatchNorm2d/BatchNorm1d nodes. bnorm_nodes = { index for index, module in indx_2_module_map.items() - if isinstance(module, nn.BatchNorm2d) + if isinstance(module, nn.BatchNorm2d) or isinstance(module, nn.BatchNorm1d) } if len(bnorm_nodes) == 0: - # There are no edges with BatchNorm2d - nothing to do here. + # There are no edges with batch norm - nothing to do here. return - # Find conv-bnorm edges. - conv_bnorm_edges = { + # Find weight-bnorm edges. + weight_bnorm_edges = { (src, tgt) for (src, tgt) in edges - if isinstance(indx_2_module_map[src], nn.Conv2d) and isinstance(indx_2_module_map[tgt], nn.BatchNorm2d) + if (isinstance(indx_2_module_map[src], nn.Conv2d) and isinstance(indx_2_module_map[tgt], nn.BatchNorm2d)) or (isinstance(indx_2_module_map[src], nn.Linear) and isinstance(indx_2_module_map[tgt], nn.BatchNorm1d)) } - # Merge conv and bnorm layers using 'conv-bnorm' edges. - for edge in conv_bnorm_edges: - # merge and update conv node. + # Merge conv/linear and bnorm layers using 'weight-bnorm' edges. + for edge in weight_bnorm_edges: bnorm = indx_2_module_map[edge[1]] - conv = indx_2_module_map[edge[0]] - indx_2_module_map[edge[0]] = merge_conv_bn(conv, bnorm) - - # Point Conv2d nodes to the targets of their respective BatchNorm2d nodes. - for conv, bnorm in conv_bnorm_edges: - new_edges = remap_edges_after_drop(dropped_node=bnorm, source_of_dropped_node=conv, edges=edges) + weight = indx_2_module_map[edge[0]] + + # merge and update weight node. + if isinstance(weight, nn.Conv2d): + indx_2_module_map[edge[0]] = merge_conv_bn(weight, bnorm) + elif isinstance(weight, nn.Linear): + indx_2_module_map[edge[0]] = merge_linear_bn(weight, bnorm) + else: + raise ValueError(f'A batch norm layer can only be preceed by either a nn.Conv2d (followed by nn.BatchNorm2d) or a nn.Linear (followed by nn.BatchNorm1d).\nFound a {type(weight)} followed by a {type(bnorm)}.') + + # Point weight nodes to the targets of their respective batch norm nodes. + new_edges = set() + for weight, bnorm in weight_bnorm_edges: + new_edges.update( + remap_edges_after_drop(dropped_node=bnorm, source_of_dropped_node=weight, edges=edges) + ) # Remove references to the bnorm node. @@ -88,7 +97,7 @@ def handle_batchnorm2d_nodes(edges: Set[Edge], indx_2_module_map: Dict[int, nn.M for name in [name for name, indx in name_2_indx_map.items() if indx in bnorm_nodes]: name_2_indx_map.pop(name) - for edge in conv_bnorm_edges: + for edge in weight_bnorm_edges: edges.remove(edge) for edge in [(src, tgt) for (src, tgt) in edges if (src in bnorm_nodes or tgt in bnorm_nodes)]: diff --git a/sinabs/backend/dynapcnn/utils.py b/sinabs/backend/dynapcnn/utils.py index db01b496..41b28967 100644 --- a/sinabs/backend/dynapcnn/utils.py +++ b/sinabs/backend/dynapcnn/utils.py @@ -287,7 +287,7 @@ def merge_conv_bn(conv, bn): return conv -def merge_linear_bn(linear, bn): +def merge_linear_bn(linear: nn.Linear, bn: nn.BatchNorm1d) -> nn.Linear: """Merge a linear (fully connected) layer with subsequent batch normalization. Parameters From 14f647b71fdd8231a7686df280c701af133dd951 Mon Sep 17 00:00:00 2001 From: Willian-Girao Date: Mon, 4 Nov 2024 16:19:54 +0100 Subject: [PATCH 299/379] (DONE) Merge Linear with BatchNorm1d Graph extraction from NIR raises an error when batch_size == 1 --- .../snn_with_multiple_batchnorm.ipynb | 380 ++++++++++++++++++ .../backend/dynapcnn/nir_graph_extractor.py | 6 +- .../backend/dynapcnn/sinabs_edges_handler.py | 2 +- 3 files changed, 384 insertions(+), 4 deletions(-) create mode 100644 examples/dynapcnn_network/snn_with_multiple_batchnorm.ipynb diff --git a/examples/dynapcnn_network/snn_with_multiple_batchnorm.ipynb b/examples/dynapcnn_network/snn_with_multiple_batchnorm.ipynb new file mode 100644 index 00000000..ab24493c --- /dev/null +++ b/examples/dynapcnn_network/snn_with_multiple_batchnorm.ipynb @@ -0,0 +1,380 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "metadata": {} + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/samurai2077/anaconda3/envs/speck-rescnn/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import torch\n", + "import torch.nn as nn\n", + "from sinabs.backend.dynapcnn import DynapcnnNetwork\n", + "from sinabs.backend.dynapcnn import DVSLayer\n", + "from sinabs.layers import Merge, IAFSqueeze, SumPool2d\n", + "from sinabs.activation.surrogate_gradient_fn import PeriodicExponential\n", + "import sinabs.layers as sl\n", + "\n", + "from torch.nn import CrossEntropyLoss\n", + "from torch.optim import Adam\n", + "\n", + "from tonic.datasets.nmnist import NMNIST\n", + "from tonic.transforms import ToFrame\n", + "from torch.utils.data import DataLoader\n", + "import numpy as np\n", + "from tqdm.notebook import tqdm\n", + "from statistics import mode\n", + "\n", + "device = torch.device('cpu')\n", + "torch.manual_seed(0)" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "channels = 1\n", + "height = 34\n", + "width = 34\n", + "batch_size = 2\n", + "\n", + "input_shape = (channels, height, width)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "metadata": {} + }, + "outputs": [ + { + "data": { + "text/plain": [ + "SNN(\n", + " (conv1): Conv2d(1, 10, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + " (bn1): BatchNorm2d(10, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (iaf1): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(1.), min_v_mem=Parameter containing:\n", + " tensor(-1.), batch_size=2, num_timesteps=-1)\n", + " (pool1): AvgPool2d(kernel_size=2, stride=2, padding=0)\n", + " (conv2): Conv2d(10, 10, kernel_size=(4, 4), stride=(1, 1), bias=False)\n", + " (bn2): BatchNorm2d(10, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (iaf2): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(1.), min_v_mem=Parameter containing:\n", + " tensor(-1.), batch_size=2, num_timesteps=-1)\n", + " (conv3): Conv2d(10, 1, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + " (bn3): BatchNorm2d(1, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (iaf3): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(1.), min_v_mem=Parameter containing:\n", + " tensor(-1.), batch_size=2, num_timesteps=-1)\n", + " (fc1): Linear(in_features=144, out_features=200, bias=False)\n", + " (bn4): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (iaf4): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(1.), min_v_mem=Parameter containing:\n", + " tensor(-1.), batch_size=2, num_timesteps=-1)\n", + " (fc2): Linear(in_features=200, out_features=10, bias=False)\n", + " (bn5): BatchNorm1d(10, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (iaf5): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(1.), min_v_mem=Parameter containing:\n", + " tensor(-1.), batch_size=2, num_timesteps=-1)\n", + " (flat): Flatten(start_dim=1, end_dim=-1)\n", + ")" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "class SNN(nn.Module):\n", + " def __init__(self, input_shape) -> None:\n", + " super().__init__()\n", + "\n", + " # -- chip core A --\n", + " self.conv1 = nn.Conv2d(1, 10, 2, 1, bias=False)\n", + " self.bn1 = nn.BatchNorm2d(10)\n", + " self.iaf1 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential())\n", + " self.pool1 = nn.AvgPool2d(2,2)\n", + " # -- chip core B --\n", + " self.conv2 = nn.Conv2d(10, 10, 4, 1, bias=False)\n", + " self.bn2 = nn.BatchNorm2d(10)\n", + " self.iaf2 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential())\n", + " # -- chip core C --\n", + " self.conv3 = nn.Conv2d(10, 1, 2, 1, bias=False)\n", + " self.bn3 = nn.BatchNorm2d(1)\n", + " self.iaf3 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential())\n", + " # -- chip core D --\n", + " self.fc1 = nn.Linear(144, 200, bias=False)\n", + " self.bn4 = nn.BatchNorm1d(200)\n", + " self.iaf4 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential())\n", + " # -- chip core E --\n", + " self.fc2 = nn.Linear(200, 10, bias=False)\n", + " self.bn5 = nn.BatchNorm1d(10)\n", + " self.iaf5 = IAFSqueeze(batch_size=batch_size, min_v_mem=-1.0, spike_threshold=1.0, surrogate_grad_fn=PeriodicExponential())\n", + "\n", + " # -- layers ignored during deployment --\n", + " self.flat = nn.Flatten()\n", + "\n", + " def init_weights(self):\n", + " for name, layer in self.named_modules():\n", + " if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear):\n", + " nn.init.xavier_normal_(layer.weight.data)\n", + "\n", + " def detach_neuron_states(self):\n", + " for name, layer in self.named_modules():\n", + " if name != '':\n", + " if isinstance(layer, sl.StatefulLayer):\n", + " for name, buffer in layer.named_buffers():\n", + " buffer.detach_()\n", + "\n", + " def forward(self, x):\n", + " \n", + " con1_out = self.conv1(x) # 4\n", + " bn1_out = self.bn1(con1_out)\n", + " iaf1_out = self.iaf1(bn1_out) # 5\n", + " pool1_out = self.pool1(iaf1_out) # 6\n", + "\n", + " conv2_out = self.conv2(pool1_out) # 7\n", + " bn2_out = self.bn2(conv2_out)\n", + " iaf2_out = self.iaf2(bn2_out) # 8\n", + "\n", + " conv3_out = self.conv3(iaf2_out) # 9\n", + " bn3_out = self.bn3(conv3_out)\n", + " iaf3_out = self.iaf3(bn3_out) # 10\n", + "\n", + " flat_out = self.flat(iaf3_out) # 15\n", + " \n", + " fc1_out = self.fc1(flat_out) # 11\n", + " bn4_out = self.bn4(fc1_out)\n", + " iaf4_out = self.iaf4(bn4_out) # 12\n", + " fc2_out = self.fc2(iaf4_out) # 13\n", + " bn5_out = self.bn5(fc2_out)\n", + " iaf5_out = self.iaf5(bn5_out) # 14\n", + "\n", + " return iaf5_out\n", + " \n", + "snn = SNN(input_shape)\n", + "snn.init_weights()\n", + "snn.to(device)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "metadata": {} + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "OpenBLAS Warning : Detect OpenMP Loop and this application may hang. Please rebuild the library with USE_OPENMP=1 option.\n", + "OpenBLAS Warning : Detect OpenMP Loop and this application may hang. Please rebuild the library with USE_OPENMP=1 option.\n", + "OpenBLAS Warning : Detect OpenMP Loop and this application may hang. Please rebuild the library with USE_OPENMP=1 option.\n", + "OpenBLAS Warning : Detect OpenMP Loop and this application may hang. Please rebuild the library with USE_OPENMP=1 option.\n" + ] + } + ], + "source": [ + "hw_model = DynapcnnNetwork(\n", + " snn=snn,\n", + " input_shape=input_shape,\n", + " batch_size=batch_size,\n", + " discretize=True\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "input_dummy = torch.randn((batch_size, *input_shape))" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "OpenBLAS Warning : Detect OpenMP Loop and this application may hang. Please rebuild the library with USE_OPENMP=1 option.\n", + "OpenBLAS Warning : Detect OpenMP Loop and this application may hang. Please rebuild the library with USE_OPENMP=1 option.\n" + ] + } + ], + "source": [ + "out = hw_model(input_dummy)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "tensor([[[[0.]],\n", + "\n", + " [[0.]],\n", + "\n", + " [[0.]],\n", + "\n", + " [[0.]],\n", + "\n", + " [[0.]],\n", + "\n", + " [[0.]],\n", + "\n", + " [[0.]],\n", + "\n", + " [[0.]],\n", + "\n", + " [[1.]],\n", + "\n", + " [[0.]]],\n", + "\n", + "\n", + " [[[0.]],\n", + "\n", + " [[0.]],\n", + "\n", + " [[0.]],\n", + "\n", + " [[0.]],\n", + "\n", + " [[0.]],\n", + "\n", + " [[0.]],\n", + "\n", + " [[1.]],\n", + "\n", + " [[0.]],\n", + "\n", + " [[1.]],\n", + "\n", + " [[0.]]]], grad_fn=)\n" + ] + } + ], + "source": [ + "print(out)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Network is valid\n" + ] + }, + { + "data": { + "text/plain": [ + "DynapcnnNetwork(\n", + " (_dynapcnn_module): DynapcnnNetworkModule(\n", + " (_dynapcnn_layers): ModuleDict(\n", + " (0): DynapcnnLayer(\n", + " (_conv): Conv2d(1, 10, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + " (_spk): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(171.), min_v_mem=Parameter containing:\n", + " tensor(-171.), batch_size=2, num_timesteps=-1)\n", + " )\n", + " (4): DynapcnnLayer(\n", + " (_conv): Conv2d(200, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + " (_spk): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(398.), min_v_mem=Parameter containing:\n", + " tensor(-398.), batch_size=2, num_timesteps=-1)\n", + " )\n", + " (1): DynapcnnLayer(\n", + " (_conv): Conv2d(10, 10, kernel_size=(4, 4), stride=(1, 1), bias=False)\n", + " (_spk): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(1981.), min_v_mem=Parameter containing:\n", + " tensor(-1981.), batch_size=2, num_timesteps=-1)\n", + " )\n", + " (2): DynapcnnLayer(\n", + " (_conv): Conv2d(10, 1, kernel_size=(2, 2), stride=(1, 1), bias=False)\n", + " (_spk): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(290.), min_v_mem=Parameter containing:\n", + " tensor(-290.), batch_size=2, num_timesteps=-1)\n", + " )\n", + " (3): DynapcnnLayer(\n", + " (_conv): Conv2d(1, 200, kernel_size=(12, 12), stride=(1, 1), bias=False)\n", + " (_spk): IAFSqueeze(spike_threshold=Parameter containing:\n", + " tensor(348.), min_v_mem=Parameter containing:\n", + " tensor(-348.), batch_size=2, num_timesteps=-1)\n", + " )\n", + " )\n", + " (merge_layer): Merge()\n", + " )\n", + ")" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "hw_model.to(device=\"speck2fdevkit\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "speck-rescnn", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index 0a7b35f8..a44d84b8 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -17,7 +17,7 @@ from .dynapcnn_layer_utils import construct_dynapcnnlayers_from_mapper from .dynapcnnnetwork_module import DynapcnnNetworkModule from .exceptions import InvalidGraphStructure, InvalidModelWithDVSSetup -from .sinabs_edges_handler import collect_dynapcnn_layer_info, get_dvs_node_from_mapper, fix_dvs_module_edges, merge_dvs_pooling_edge, handle_batchnorm2d_nodes +from .sinabs_edges_handler import collect_dynapcnn_layer_info, get_dvs_node_from_mapper, fix_dvs_module_edges, merge_dvs_pooling_edge, handle_batchnorm_nodes from .utils import Edge, topological_sorting try: @@ -88,8 +88,8 @@ def __init__( # Store the associated `nn.Module` (layer) of each node. self._indx_2_module_map = self._get_named_modules(spiking_model) - # Merges BatchNorm2d nodes with Conv2d ones. - handle_batchnorm2d_nodes(self._edges, self._indx_2_module_map, self._name_2_indx_map) + # Merges BatchNorm2d/BatchNorm1d nodes with Conv2d/Linear ones. + handle_batchnorm_nodes(self._edges, self._indx_2_module_map, self._name_2_indx_map) # Determine entry points to graph self._entry_nodes = self._get_entry_nodes(self._edges) diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index aa3c177f..538289f6 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -41,7 +41,7 @@ def remap_edges_after_drop(dropped_node: int, source_of_dropped_node: int, edges return remapped_edges -def handle_batchnorm2d_nodes(edges: Set[Edge], indx_2_module_map: Dict[int, nn.Module], name_2_indx_map: Dict[str, int]) -> None: +def handle_batchnorm_nodes(edges: Set[Edge], indx_2_module_map: Dict[int, nn.Module], name_2_indx_map: Dict[str, int]) -> None: """ Merges `BatchNorm2d`/`BatchNorm1d` layers into `Conv2d`/`Linear` ones. The batch norm nodes will be removed from the graph (by updating all variables passed as arguments in-place) after their properties are used to re-scale the weights of the convolutional/linear layers associated with batch normalization via the `weight-batchnorm` edges found in the original graph. From 1ec5bd07b37560992c322f2c91f35863edb3879f Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Tue, 5 Nov 2024 12:17:42 +0100 Subject: [PATCH 300/379] Minor revisions in nir graph. Fix merge_polarities --- .../backend/dynapcnn/nir_graph_extractor.py | 48 +++++++++++-------- 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index 373c731f..b6a36700 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -115,6 +115,10 @@ def __init__( ####################################################### Publich Methods ####################################################### + @property + def dvs_layer(self) -> Set[int]: + return {n for n in self._get_dvs_layer} + @property def entry_nodes(self) -> Set[int]: return {n for n in self._entry_nodes} @@ -123,6 +127,10 @@ def entry_nodes(self) -> Set[int]: def edges(self) -> Set[Edge]: return {(src, tgt) for src, tgt in self._edges} + @property + def has_dvs_layer(self) -> Set[Edge]: + return self.dvs_layer is not None + @property def name_2_indx_map(self) -> Dict[str, int]: return {name: idx for name, idx in self._name_2_indx_map.items()} @@ -164,7 +172,7 @@ def get_dynapcnn_network_module( edges=self.edges, nodes_io_shapes=self.nodes_io_shapes, entry_nodes=self.entry_nodes, - dvs_input=self._has_dvs_layer(), + dvs_input=self.has_dvs_layer, ) # build `DynapcnnLayer` instances from mapper. @@ -301,7 +309,8 @@ def _add_dvs_node(self, dvs_input_shape: Tuple[int, int, int]) -> None: # add module entry for node 'dvs'. self._indx_2_module_map[self._name_2_indx_map['dvs']] = DVSLayer( input_shape=(height, width), - merge_polarities=True if features > 1 else False) + merge_polarities=(features == 1), + ) # set DVS node as input to each entry node of the graph. self._edges.update({(self._name_2_indx_map['dvs'], entry_node) for entry_node in self._entry_nodes}) # DVSLayer node becomes the only entrypoint of the graph. @@ -322,16 +331,26 @@ def _need_dvs_node(self, dvs_input: Optional[bool] = None) -> bool: - True if `self._indx_2_module_map` contains a DVSLayer, False otherwise. """ - has_dvs_layer = self._has_dvs_layer() + has_dvs_layer = self.has_dvs_layer # Checks if DVSLayer instance exists but user has set 'dvs_input' to False. - if has_dvs_layer and (isinstance(dvs_input, bool) and not dvs_input): + if has_dvs_layer and dvs_input == False: raise InvalidModelWithDVSSetup() return not has_dvs_layer and dvs_input - def _has_dvs_layer(self) -> bool: - """ Loops though all modules and check if a `DVSLayer` instance exists. """ + def _get_dvs_layer(self) -> Union[DVSLayer, None]: + """ Loops though all modules and return `DVSLayer` instance if it exists. + + Returns + ------- + - DVSLayer if exactly one is found, otherwise None + + Raises + ------ + - InvalidGraphStructure if more than one DVSLayer is found + + """ dvs_layers = { module for module in self._indx_2_module_map.values() @@ -339,9 +358,9 @@ def _has_dvs_layer(self) -> bool: } if (num_dvs := len(dvs_layers)) == 0: - return False + return elif num_dvs == 1: - return True + return dvs_layers[0] else: raise InvalidGraphStructure( f"The provided model has {num_dvs} `DVSLayer`s. At most one is allowed." @@ -357,18 +376,7 @@ def _validate_dvs_setup(self, dvs_input_shape: Tuple[int, int, int]) -> None: - dvs_input_shape (tuple): shape of the DVSLayer input in format `(features, height, width)`. """ - dvs_layer = [module for index, module in self._indx_2_module_map.items() if isinstance(module, DVSLayer)] - - if len(dvs_layer) == 0: - # No DVSLayer found - nothing to do here. - return - elif (nb_dvs := len(dvs_layer)) > 1: - # Can't have more then one DVSLayer instance. - raise InvalidGraphStructure( - f"The provided model has {nb_dvs} `DVSLayer`s. At most one is allowed." - ) - else: - dvs_layer = dvs_layer[-1] + dvs_layer = self.dvs_layer if (nb_entries := len(self._entry_nodes)) > 1: raise ValueError(f'A DVSLayer node exists and there are {nb_entries} entry nodes in the graph: the DVSLayer should be the only entry node.') From 4e297e3b9da1b9dda3937ca534710b81a64475b8 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Tue, 5 Nov 2024 12:32:00 +0100 Subject: [PATCH 301/379] GraphExtractor: Tidy up init method --- .../backend/dynapcnn/nir_graph_extractor.py | 50 +++++++++++++------ 1 file changed, 35 insertions(+), 15 deletions(-) diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index b6a36700..d49d9e2b 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -91,21 +91,8 @@ def __init__( # Determine entry points to graph self._entry_nodes = self._get_entry_nodes(self._edges) - # If DVS camera is wanted but `spiking_model` does not start with DVS layer. - if self._need_dvs_node(dvs_input): - # Insert a DVSLayer node in the graph. - self._add_dvs_node(dvs_input_shape=dummy_input.shape[1:]) - - # Check for the need of fixing NIR edges extraction when DVS is a node in the graph. If DVS - # is used its node becomes the only entry node in the graph. - fix_dvs_module_edges(self._edges, self._indx_2_module_map, self._name_2_indx_map, self._entry_nodes) - - # Merge a pooling node from a 'dvs-pooling' edge (pooling being an independent node in the original - # graph) into the DVSLayer if such edge exists. - merge_dvs_pooling_edge(self._edges, self._indx_2_module_map, self._name_2_indx_map) - - # Check if graph structure and DVSLayer.merge_polarities are correctly set (if DVS node exists). - self._validate_dvs_setup(dvs_input_shape=dummy_input.shape[1:]) + # Make sure DVS input is properly integrated into graph + self._handle_dvs_input(input_shape=dummy_input.shape[1:], dvs_input=dvs_input) # Verify that graph is compatible self.verify_graph_integrity() @@ -289,6 +276,39 @@ def verify_graph_integrity(self): ####################################################### Pivate Methods ####################################################### + def _handle_dvs_input(self, input_shape: Tuple[int, int, int], dvs_input: Optional[bool] = None): + """Make sure DVS input is properly integrated into graph + + - Decide whether `DVSLayer` instance needs to be added to the graph + This is the case when `dvs_input==True` and there is no `DVSLayer` yet. + - Make sure edges between DVS related nodes are set properly + - Absorb pooling layers in DVS node if applicable + + Parameters + ---------- + - input_shape (tuple of three integers): Input shape (features, height, width) + - dvs_input (bool or `None` (default)): If `False`, will raise + `InvalidModelWithDvsSetup` if a `DVSLayer` is part of the graph. If `True`, + a `DVSLayer` will be added to the graph if there is none already. If `None`, + the model is considered to be using DVS input only if the graph contains + a `DVSLayer`. + """ + # If DVS camera is wanted but `spiking_model` does not start with DVS layer. + if self._need_dvs_node(dvs_input): + # Insert a DVSLayer node in the graph. + self._add_dvs_node(dvs_input_shape=input_shape) + + # Check for the need of fixing NIR edges extraction when DVS is a node in the graph. If DVS + # is used its node becomes the only entry node in the graph. + fix_dvs_module_edges(self._edges, self._indx_2_module_map, self._name_2_indx_map, self._entry_nodes) + + # Merge a pooling node from a 'dvs-pooling' edge (pooling being an independent node in the original + # graph) into the DVSLayer if such edge exists. + merge_dvs_pooling_edge(self._edges, self._indx_2_module_map, self._name_2_indx_map) + + # Check if graph structure and DVSLayer.merge_polarities are correctly set (if DVS node exists). + self._validate_dvs_setup(dvs_input_shape=input_shape) + def _add_dvs_node(self, dvs_input_shape: Tuple[int, int, int]) -> None: """ In-place modification of `self._name_2_indx_map`, `self._indx_2_module_map`, and `self._edges` to accomodate the creation of an extra node in the graph representing the DVS camera of the chip. The DVSLayer node will point to every From 3cdf5484d3dec6aa11288fb6cfcafa5a0dffb29c Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Tue, 5 Nov 2024 13:28:18 +0100 Subject: [PATCH 302/379] DynapcnnNetwork: Don't send DVS node info to config builder --- sinabs/backend/dynapcnn/chips/dynapcnn.py | 11 ++--------- sinabs/backend/dynapcnn/dynapcnn_network.py | 1 - sinabs/backend/dynapcnn/sinabs_edges_handler.py | 6 ++---- 3 files changed, 4 insertions(+), 14 deletions(-) diff --git a/sinabs/backend/dynapcnn/chips/dynapcnn.py b/sinabs/backend/dynapcnn/chips/dynapcnn.py index 2ded7eac..9547819f 100644 --- a/sinabs/backend/dynapcnn/chips/dynapcnn.py +++ b/sinabs/backend/dynapcnn/chips/dynapcnn.py @@ -1,5 +1,5 @@ import copy -from typing import Dict, List, Optional +from typing import Dict, List, Union from warnings import warn import samna @@ -29,11 +29,6 @@ def get_default_config(cls) -> "DynapcnnConfiguration": @classmethod def get_dvs_layer_config_dict(cls, layer: DVSLayer): ... - # @classmethod - # def write_dvs_layer_config(cls, layer: DVSLayer, config: DVSLayerConfig): - # for param, value in layer.get_config_dict().items(): - # setattr(config, param, value) - @classmethod def write_dvs_layer_config( cls, @@ -289,7 +284,6 @@ def build_config( layers: Dict[int, DynapcnnLayer], destination_map: Dict[int, List[int]], layer2core_map: Dict[int, int], - dvs_node_info: Optional[Dict], ) -> DynapcnnConfiguration: """Uses `DynapcnnLayer` objects to configure their equivalent chip cores @@ -299,7 +293,6 @@ def build_config( - layer2core_map (Dict): Keys are layer indices, values are corresponding cores on hardware. Needed to map the destinations. - destination_indices (List): Indices of destination layers for `layer` - - dvs_node_info (dict): contains information associated with the `DVSLayer` node (if no DVS node exists it'll return `None`). Returns ------- @@ -321,7 +314,7 @@ def build_config( chip_layer=chip_layer, destination_indices=destination_map[layer_index], ) - elif isinstance(ith_dcnnl, DVSLayer) and isinstance(dvs_node_info, dict): + elif isinstance(ith_dcnnl, DVSLayer): # Uses the DVS camera. chip_layer = config.dvs_layer sw_layer = ith_dcnnl diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index 16d5f096..0e485107 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -585,7 +585,6 @@ def _make_config( layers=self.all_layers, destination_map=self.layer_destination_map, layer2core_map=layer2core_map, - dvs_node_info=self.dvs_node_info, ) if monitor_layers is None: diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index 18a55195..c18dcef1 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -81,7 +81,7 @@ def fix_dvs_module_edges(edges: Set[Edge], indx_2_module_map: Dict[int, nn.Modul if isinstance(indx_2_module_map[edge[0]], SumPool2d) and isinstance(indx_2_module_map[edge[1]], Crop2d) }) - # NIR is extracting and edge (FlipDims, FlipDims) from the DVSLayer: remove self-recurrent nodes from the graph. + # NIR is extracting an edge (FlipDims, FlipDims) from the DVSLayer: remove self-recurrent nodes from the graph. for edge in [(src, tgt) for (src, tgt) in edges if (src == tgt and isinstance(indx_2_module_map[src], FlipDims))]: edges.remove(edge) @@ -143,12 +143,10 @@ def merge_dvs_pooling_edge(edges: Set[Edge], indx_2_module_map: Dict[int, nn.Mod # Checking pooling can be incorporated into the DVSLayer. if indx_2_module_map[dvs_idnx].pool_layer.kernel_size == 1 and indx_2_module_map[dvs_idnx].pool_layer.stride == 1: - # DVSLayer.pool has its default config. - indx_2_module_map[pool_idnx].kernel_size - indx_2_module_map[pool_idnx].stride # Set DVSLayer.pool to have same config. as the independent pooling layer. indx_2_module_map[dvs_idnx].pool_layer.kernel_size = indx_2_module_map[pool_idnx].kernel_size indx_2_module_map[dvs_idnx].pool_layer.stride = indx_2_module_map[pool_idnx].stride + # TODO: Should there not be an error be raised if the condition is wrong? # Pooling incorporated to the DVSLayer: remove its trace from mappings. indx_2_module_map.pop(pool_idnx) From 6469d91156e4f98616c29d9a134d35a44cf56e3a Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Tue, 5 Nov 2024 13:43:50 +0100 Subject: [PATCH 303/379] Minor revisions in dynapcnn layer utils --- .../backend/dynapcnn/nir_graph_extractor.py | 3 +- .../backend/dynapcnn/sinabs_edges_handler.py | 44 +++++++++---------- 2 files changed, 21 insertions(+), 26 deletions(-) diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index d49d9e2b..4b171345 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -1,7 +1,7 @@ # author : Willian Soares Girao # contact : wsoaresgirao@gmail.com -from typing import Callable, Dict, Iterable, List, Optional, Set, Tuple, Type +from typing import Callable, Dict, Iterable, List, Optional, Set, Tuple, Type, Union import nirtorch import torch @@ -159,7 +159,6 @@ def get_dynapcnn_network_module( edges=self.edges, nodes_io_shapes=self.nodes_io_shapes, entry_nodes=self.entry_nodes, - dvs_input=self.has_dvs_layer, ) # build `DynapcnnLayer` instances from mapper. diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index c18dcef1..0b4a8af9 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -169,7 +169,6 @@ def collect_dynapcnn_layer_info( edges: Set[Edge], nodes_io_shapes: Dict[int, Dict[str, Tuple[Size, Size]]], entry_nodes: Set[int], - dvs_input: bool, ) -> Dict[int, Dict]: """Collect information to construct DynapcnnLayer instances. @@ -186,7 +185,6 @@ def collect_dynapcnn_layer_info( - edges (set of tuples): Represent connections between two nodes in computational graph - nodes_io_shapes (dict): Map from node ID to dict containing node's in- and output shapes - entry_nodes (set of int): IDs of nodes that receive external input - - dvs_input (bool): wether or not dynapcnn receive input from its DVS camera. Returns ------- @@ -200,11 +198,6 @@ def collect_dynapcnn_layer_info( edges=edges, indx_2_module_map=indx_2_module_map ) - if not any(edge in edges_by_type for edge in ["dvs-weight", "dvs-pooling"]) and dvs_input: - raise InvalidGraphStructure( - "DVS camera is set selected for usage (dvs_input == True) but edge type involving it has not been found." - ) - # Dict to collect information for each future dynapcnn layer dynapcnn_layer_info = dict() # Map node IDs to dynapcnn layer ID @@ -223,7 +216,7 @@ def collect_dynapcnn_layer_info( ) # Process all dvs->weight edges connecting the DVS camera to a unique dynapcnn layer. - while edges_by_type["dvs-weight"]: + while edges_by_type.get("dvs-weight", False): edge = edges_by_type["dvs-weight"].pop() add_or_update_dvs_to_entry( edge, @@ -348,11 +341,6 @@ def sort_edges_by_type( else: edges_by_type[edge_type] = {edge} - # Edges involving DVS are not required so we init. them to empty set if they do not exist. - - if 'dvs-weight' not in edges_by_type: - edges_by_type['dvs-weight'] = set() - return edges_by_type @@ -496,18 +484,29 @@ def add_or_update_dvs_to_entry( nodes_io_shapes (dict): Map from node ID to dict containing node's in- and output shapes """ + # This should never happen assert isinstance(indx_2_module_map[edge[0]], DVSLayer), f'Source node in edge {edge} is of type {type(DVSLayer)} (it should be a DVSLayer instance).' - assert edge[1] in node_2_layer_map, f'Node {edge[1]} is a weight node that should have been initialized.' + + # Find destination layer index + try: + destination_layer_idx = node_2_layer_map[edge[1]] + except KeyError: + weight_layer = indx_2_module_map[edge[1]] + raise InvalidGraphStructure( + f"Weight layer {weight_layer} cannot be assigned to a dynapcnn layer. " + "This is likely due to an unsupported SNN architecture. Weight " + "layers have to be followed by a spiking layer (`sl.IAFSqueeze`)." + ) - if edge[0] not in node_2_layer_map: - # DVS node hasn't being initialized yet: take current length of the dict as new, unique ID. + if (source_layer_idx := node_2_layer_map.get(edge[0], None)) is None: + # DVS node hasn't been initialized yet: take current length of the dict as new, unique ID. layer_id = len(dynapcnn_layer_info) assert layer_id not in dynapcnn_layer_info # Init. entry for a DVS layer using its configuration dict. dynapcnn_layer_info[layer_id] = { "is_entry_node": True, - # TODO - the key bellow is what currently tells an entry in `dynapcnn_layer_info` for the DVS apart from the DynapcnnLayer + # TODO - the key below is what currently tells an entry in `dynapcnn_layer_info` for the DVS apart from the DynapcnnLayer # entries (perhaps there's a better way). "dvs_layer": True, "node_id": edge[0], @@ -520,14 +519,11 @@ def add_or_update_dvs_to_entry( node_2_layer_map[edge[0]] = layer_id else: # Update entry for DVS with new destination. - source_layer_id = node_2_layer_map[edge[0]] - destination_layer_id = node_2_layer_map[edge[1]] - - assert 'dvs_layer' in dynapcnn_layer_info[source_layer_id] - assert dynapcnn_layer_info[source_layer_id]['dvs_layer'] - assert destination_layer_id not in dynapcnn_layer_info[source_layer_id]["destinations"] + assert 'dvs_layer' in dynapcnn_layer_info[source_layer_idx] + assert dynapcnn_layer_info[source_layer_idx]['dvs_layer'] + assert destination_layer_idx not in dynapcnn_layer_info[source_layer_idx]["destinations"] - dynapcnn_layer_info[source_layer_id]["destinations"].append(destination_layer_id) + dynapcnn_layer_info[source_layer_idx]["destinations"].append(destination_layer_idx) def set_exit_destinations(dynapcnn_layer: Dict) -> None: """Set minimal destination entries for layers that don't have any. From 5d8beee28851ffb82240e0dbc3689f7857eb35f6 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Tue, 5 Nov 2024 13:53:19 +0100 Subject: [PATCH 304/379] dynapcnn layer utils: minor syntax improvements --- .../backend/dynapcnn/sinabs_edges_handler.py | 26 +++++++++++-------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index 0b4a8af9..7df71f1d 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -204,8 +204,9 @@ def collect_dynapcnn_layer_info( node_2_layer_map = dict() # Each weight->neuron connection instantiates a new, unique dynapcnn layer - while edges_by_type.get("weight-neuron", False): - edge = edges_by_type["weight-neuron"].pop() + weight_neuron_edges = edges_by_type.get("weight-neuron", Set()) + while weight_neuron_edges: + edge = weight_neuron_edges.pop() init_new_dynapcnnlayer_entry( dynapcnn_layer_info, edge, @@ -216,8 +217,9 @@ def collect_dynapcnn_layer_info( ) # Process all dvs->weight edges connecting the DVS camera to a unique dynapcnn layer. - while edges_by_type.get("dvs-weight", False): - edge = edges_by_type["dvs-weight"].pop() + dvs_weight_edges = edges_by_type.get("dvs-weight", Set()) + while dvs_weight_edges: + edge = dvs_weight_edges.pop() add_or_update_dvs_to_entry( edge, dynapcnn_layer_info, @@ -227,8 +229,9 @@ def collect_dynapcnn_layer_info( ) # Process all edges connecting two dynapcnn layers that do not include pooling - while edges_by_type.get("neuron-weight", False): - edge = edges_by_type["neuron-weight"].pop() + neuron_weight_edges = edges_by_type.get("neuron-weight", Set()) + while neuron_weight_edges: + edge = neuron_weight_edges.pop() set_neuron_layer_destination( dynapcnn_layer_info, edge, @@ -238,9 +241,9 @@ def collect_dynapcnn_layer_info( ) # Add pooling based on neuron->pooling connections - pooling_pooling_edges = edges_by_type.get("pooling-pooling", set()) - while edges_by_type.get("neuron-pooling", False): - edge = edges_by_type["neuron-pooling"].pop() + pooling_pooling_edges = edges_by_type.get("pooling-pooling", Set()) + while pooling_pooling_edges: + edge = pooling_pooling_edges.pop() # Search pooling-pooling edges for chains of pooling and add to existing entry pooling_chains, edges_used = trace_paths(edge[1], pooling_pooling_edges) add_pooling_to_entry( @@ -265,8 +268,9 @@ def collect_dynapcnn_layer_info( ) # Add all edges connecting pooling to a new dynapcnn layer - while edges_by_type.get("pooling-weight", False): - edge = edges_by_type["pooling-weight"].pop() + pooling_weight_edges = edges_by_type.get("pooling-weight", Set()) + while pooling_weight_edges: + edge = pooling_weight_edges.pop() set_pooling_layer_destination( dynapcnn_layer_info, edge, From 641e4accbf883f41a022c185c2bff7d2c4c71052 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Tue, 5 Nov 2024 13:58:48 +0100 Subject: [PATCH 305/379] (WIP) DVS layer gets index 'dvs' --- .../backend/dynapcnn/nir_graph_extractor.py | 4 ++-- .../backend/dynapcnn/sinabs_edges_handler.py | 24 +++++++------------ 2 files changed, 10 insertions(+), 18 deletions(-) diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index 4b171345..af06ab89 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -171,7 +171,7 @@ def get_dynapcnn_network_module( ) # DVSLayer node information (None if DVS camera is not used). - dvs_node_info = get_dvs_node_from_mapper(self.dcnnl_map) + dvs_node_info = self.dcnnl_map.get("dvs", None) # Instantiate the DynapcnnNetworkModule return DynapcnnNetworkModule( @@ -353,7 +353,7 @@ def _need_dvs_node(self, dvs_input: Optional[bool] = None) -> bool: has_dvs_layer = self.has_dvs_layer # Checks if DVSLayer instance exists but user has set 'dvs_input' to False. - if has_dvs_layer and dvs_input == False: + if has_dvs_layer and dvs_input == False: # `== False` ensures that `None` is not considered raise InvalidModelWithDVSSetup() return not has_dvs_layer and dvs_input diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index 7df71f1d..7a1670ff 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -502,32 +502,24 @@ def add_or_update_dvs_to_entry( "layers have to be followed by a spiking layer (`sl.IAFSqueeze`)." ) - if (source_layer_idx := node_2_layer_map.get(edge[0], None)) is None: - # DVS node hasn't been initialized yet: take current length of the dict as new, unique ID. - layer_id = len(dynapcnn_layer_info) - assert layer_id not in dynapcnn_layer_info - + if "dvs" not in dynapcnn_layer_info: + # DVS node hasn't been initialized yet # Init. entry for a DVS layer using its configuration dict. - dynapcnn_layer_info[layer_id] = { - "is_entry_node": True, - # TODO - the key below is what currently tells an entry in `dynapcnn_layer_info` for the DVS apart from the DynapcnnLayer - # entries (perhaps there's a better way). - "dvs_layer": True, + dynapcnn_layer_info["dvs"] = { "node_id": edge[0], "input_shape": nodes_io_shapes[edge[0]]["input"], "module": indx_2_module_map[edge[0]], "destinations": [node_2_layer_map[edge[1]]], - 'layer_id': layer_id, } - node_2_layer_map[edge[0]] = layer_id + node_2_layer_map[edge[0]] = "dvs" else: # Update entry for DVS with new destination. - assert 'dvs_layer' in dynapcnn_layer_info[source_layer_idx] - assert dynapcnn_layer_info[source_layer_idx]['dvs_layer'] - assert destination_layer_idx not in dynapcnn_layer_info[source_layer_idx]["destinations"] + dvs_layer_info = dynapcnn_layer_info["dvs"] + assert dvs_layer_info["node_id"] == edge[0] + assert destination_layer_idx not in dvs_layer_info["destinations"] - dynapcnn_layer_info[source_layer_idx]["destinations"].append(destination_layer_idx) + dvs_layer_info["destinations"].append(destination_layer_idx) def set_exit_destinations(dynapcnn_layer: Dict) -> None: """Set minimal destination entries for layers that don't have any. From 6c9e3ccf87702bbcddebc66ea1058c321f373324 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Tue, 5 Nov 2024 15:04:45 +0100 Subject: [PATCH 306/379] DVS layer info in separate dict --- .../backend/dynapcnn/dynapcnn_layer_utils.py | 76 ++++++++----------- .../dynapcnn/dynapcnnnetwork_module.py | 14 +--- .../backend/dynapcnn/nir_graph_extractor.py | 20 ++--- .../backend/dynapcnn/sinabs_edges_handler.py | 44 ++++++----- .../test_dynapcnnlayer/test_dynapcnnlayer.py | 2 +- 5 files changed, 70 insertions(+), 86 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py b/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py index 9e9a3680..7079be5a 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py @@ -10,7 +10,7 @@ def construct_dynapcnnlayers_from_mapper( - dcnnl_map: Dict, discretize: bool, rescale_fn: Optional[Callable] = None + dcnnl_map: Dict, dvs_layer_info: Union[None, Dict], discretize: bool, rescale_fn: Optional[Callable] = None ) -> Tuple[Dict[int, DynapcnnLayer], Dict[int, Set[int]], List[int]]: """Construct DynapcnnLayer instances from `dcnnl_map` @@ -28,12 +28,11 @@ def construct_dynapcnnlayers_from_mapper( dynapcnn_layers = { layer_idx: construct_single_dynapcnn_layer(layer_info, discretize) for layer_idx, layer_info in dcnnl_map.items() - if 'dvs_layer' not in layer_info # handle only dicts with info. for DynapcnnLayer instances. } - destination_map = construct_destination_map(dcnnl_map) + destination_map = construct_destination_map(dcnnl_map, dvs_layer_info) - entry_points = collect_entry_points(dcnnl_map) + entry_points = collect_entry_points(dcnnl_map, dvs_layer_info) return dynapcnn_layers, destination_map, entry_points @@ -54,13 +53,11 @@ def finalize_dcnnl_map(dcnnl_map: Dict, rescale_fn: Optional[Callable] = None) - """ # Consolidate pooling information for each destination for layer_info in dcnnl_map.values(): - if 'dvs_layer' not in layer_info: # only called for `DynapcnnLayer` instances (skip DVS layer). - consolidate_layer_pooling(layer_info, dcnnl_map) + consolidate_layer_pooling(layer_info, dcnnl_map) for layer_info in dcnnl_map.values(): - if 'dvs_layer' not in layer_info: # only called for `DynapcnnLayer` instances (skip DVS layer). - # Consolidate scale factors - consolidate_layer_scaling(layer_info, rescale_fn) + # Consolidate scale factors + consolidate_layer_scaling(layer_info, rescale_fn) def consolidate_layer_pooling(layer_info: Dict, dcnnl_map: Dict): """Consolidate pooling information for individual layer @@ -255,12 +252,13 @@ def construct_single_dynapcnn_layer( ) -def construct_destination_map(dcnnl_map: Dict[int, Dict]) -> Dict[int, List[int]]: +def construct_destination_map(dcnnl_map: Dict[int, Dict], dvs_layer_info: Union[None, Dict]) -> Dict[int, List[int]]: """Create a dict that holds destinations for each layer Parameters ---------- - dcnnl_map: Dict holding info needed to instantiate DynapcnnLayer instances + - dynapcnn_layer_info: Dict holding info about DVSLayer instance and its destinations Returns ------- @@ -270,51 +268,41 @@ def construct_destination_map(dcnnl_map: Dict[int, Dict]) -> Dict[int, List[int] """ destination_map = dict() for layer_index, layer_info in dcnnl_map.items(): - if 'dvs_layer' not in layer_info: # only called for `DynapcnnLayer` instances (skip DVS layer). - destination_indices = [] - none_counter = 0 - for dest in layer_info["destinations"]: - if (dest_idx := dest["destination_layer"]) is None: - # For `None` destinations use unique negative index - none_counter += 1 - destination_indices.append(-none_counter) - else: - destination_indices.append(dest_idx) - destination_map[layer_index] = destination_indices - - # update mapper if a DVS layer exists. - update_destination_map_with_dvs(dcnnl_map, destination_map) + destination_indices = [] + none_counter = 0 + for dest in layer_info["destinations"]: + if (dest_idx := dest["destination_layer"]) is None: + # For `None` destinations use unique negative index + none_counter += 1 + destination_indices.append(-none_counter) + else: + destination_indices.append(dest_idx) + destination_map[layer_index] = destination_indices + if dvs_layer_info is not None: + # Copy destination list from dvs layer info + destination_map["dvs"] = [d for d in dvs_layer_info.destinations] return destination_map -def collect_entry_points(dcnnl_map: Dict[int, Dict]) -> Set[int]: +def collect_entry_points(dcnnl_map: Dict[int, Dict], dvs_layer_info: Union[None, Dict]) -> Set[int]: """Return set of layer indices that are entry points Parameters ---------- - dcnnl_map: Dict holding info needed to instantiate DynapcnnLayer instances + - dynapcnn_layer_info: Dict holding info about DVSLayer instance and its destinations + If it is not None, it will be the only entry point returned. Returns ------- Set of all layer indices which act as entry points to the network """ - return { - layer_index - for layer_index, layer_info in dcnnl_map.items() - if layer_info["is_entry_node"] - } - -def update_destination_map_with_dvs(dcnnl_map: Dict[int, Dict], destination_map: Dict[int, List[int]]) -> None: - """ Modifies `destination_map` in-place to add entry for the DVS came node (if it existis). - - Parameters - ---------- - - dcnnl_map (dict): Dict holding info needed to instantiate DynapcnnLayer instances. - - destination_map (dict): dict mapping to each layer index a set of destination indices. - """ - for layer_index, layer_info in dcnnl_map.items(): - if 'dvs_layer' in layer_info: - assert layer_info['dvs_layer'] - assert layer_index not in destination_map, 'It seems more than one DVS node has been added (only one should exist).' - destination_map[layer_index] = layer_info['destinations'] + if dvs_layer_info is None: + return { + layer_index + for layer_index, layer_info in dcnnl_map.items() + if layer_info["is_entry_node"] + } + else: + return {"dvs"} diff --git a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py index fc4052af..44dad262 100644 --- a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py +++ b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py @@ -75,9 +75,7 @@ def all_layers(self): if self.dvs_layer is not None: # `self.dynapcnn_layers` is a (shallow) copy. Adding entries won't # affect `self._dynapcnn_layers` - # TODO: Why not use "dvs" as index? - dvs_id = self._dvs_node_info["layer_id"] - layers[dvs_id] = self.dvs_layer + layers["dvs"] = self.dvs_layer return layers @property @@ -343,10 +341,8 @@ def reindex_layers(self, index_order: List[int]) -> None: mapping = {old: new for new, old in enumerate(index_order)} def remap(key): - if key == "input": - return "input" - if isinstance(key, int) and key < 0: - # maintain negative indices + if key in ["dvs", "input"] or (isinstance(key, int) and key < 0): + # Entries 'dvs', 'input' and negative indices are not changed return key else: return mapping[key] @@ -356,10 +352,6 @@ def remap(key): {str(remap(int(idx))): lyr for idx, lyr in self._dynapcnn_layers.items()} ) - if self.dvs_node_info is not None: - new_dvs_id = remap(self.dvs_node_info['layer_id']) - self._dvs_node_info["layer_id"] = new_dvs_id - self._entry_points = {remap(idx) for idx in self._entry_points} self._destination_map = { diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index af06ab89..15d98190 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -104,7 +104,7 @@ def __init__( @property def dvs_layer(self) -> Set[int]: - return {n for n in self._get_dvs_layer} + return self._get_dvs_layer() @property def entry_nodes(self) -> Set[int]: @@ -154,7 +154,7 @@ def get_dynapcnn_network_module( """ # create a dict holding the data necessary to instantiate a `DynapcnnLayer`. - self.dcnnl_map = collect_dynapcnn_layer_info( + self.dcnnl_map, self.dvs_layer_info = collect_dynapcnn_layer_info( indx_2_module_map=self.indx_2_module_map, edges=self.edges, nodes_io_shapes=self.nodes_io_shapes, @@ -165,17 +165,15 @@ def get_dynapcnn_network_module( dynapcnn_layers, destination_map, entry_points = ( construct_dynapcnnlayers_from_mapper( dcnnl_map=self.dcnnl_map, + dvs_layer_info=self.dvs_layer_info, discretize=discretize, rescale_fn=weight_rescaling_fn, ) ) - # DVSLayer node information (None if DVS camera is not used). - dvs_node_info = self.dcnnl_map.get("dvs", None) - # Instantiate the DynapcnnNetworkModule return DynapcnnNetworkModule( - dynapcnn_layers, destination_map, entry_points, dvs_node_info + dynapcnn_layers, destination_map, entry_points, self.dvs_layer_info ) def remove_nodes_by_class(self, node_classes: Tuple[Type]): @@ -379,7 +377,7 @@ def _get_dvs_layer(self) -> Union[DVSLayer, None]: if (num_dvs := len(dvs_layers)) == 0: return elif num_dvs == 1: - return dvs_layers[0] + return dvs_layers.pop() else: raise InvalidGraphStructure( f"The provided model has {num_dvs} `DVSLayer`s. At most one is allowed." @@ -395,7 +393,9 @@ def _validate_dvs_setup(self, dvs_input_shape: Tuple[int, int, int]) -> None: - dvs_input_shape (tuple): shape of the DVSLayer input in format `(features, height, width)`. """ - dvs_layer = self.dvs_layer + if self.dvs_layer is None: + # No DVSLayer found - nothing to do here. + return if (nb_entries := len(self._entry_nodes)) > 1: raise ValueError(f'A DVSLayer node exists and there are {nb_entries} entry nodes in the graph: the DVSLayer should be the only entry node.') @@ -405,11 +405,11 @@ def _validate_dvs_setup(self, dvs_input_shape: Tuple[int, int, int]) -> None: if features > 2: raise ValueError(f'A DVSLayer istance can have the feature dimension of its inputs with values 1 or 2 but {features} was given.') - if dvs_layer.merge_polarities and features != 1: + if self.dvs_layer.merge_polarities and features != 1: raise ValueError(f"The 'DVSLayer.merge_polarities' is set to 'True' which means the number of input features should be 1 (current input shape is {dvs_input_shape}).") if features == 1: - dvs_layer.merge_polarities = True + self.dvs_layer.merge_polarities = True def _get_name_2_indx_map(self, nir_graph: TorchGraph) -> Dict[str, int]: """Assign unique index to each node and return mapper from name to index. diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index 7a1670ff..8315a2f9 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -6,7 +6,7 @@ """ from collections import deque -from typing import Dict, List, Set, Tuple, Type, Optional +from typing import Dict, List, Set, Tuple, Type, Optional, Union from torch import Size, nn @@ -169,7 +169,7 @@ def collect_dynapcnn_layer_info( edges: Set[Edge], nodes_io_shapes: Dict[int, Dict[str, Tuple[Size, Size]]], entry_nodes: Set[int], -) -> Dict[int, Dict]: +) -> Tuple[Dict[int, Dict], Union[Dict, None]]: """Collect information to construct DynapcnnLayer instances. Validate and sort edges based on the type of nodes they connect. @@ -191,6 +191,8 @@ def collect_dynapcnn_layer_info( dynapcnn_layer_info (dict): Each 'key' is the index of a future 'DynapcnnLayer' and 'value' is a dictionary, with keys 'conv', 'neuron', and 'destinations', containing corresponding node ids and modules required to build the layer + dvs_layer_info (dict or None): If a DVSLayer is part of the network, this will + be a dict containing the layer itself and its destination indices. """ # Sort edges by edge type (type of layers they connect) @@ -200,11 +202,12 @@ def collect_dynapcnn_layer_info( # Dict to collect information for each future dynapcnn layer dynapcnn_layer_info = dict() + dvs_layer_info = None # Map node IDs to dynapcnn layer ID node_2_layer_map = dict() # Each weight->neuron connection instantiates a new, unique dynapcnn layer - weight_neuron_edges = edges_by_type.get("weight-neuron", Set()) + weight_neuron_edges = edges_by_type.get("weight-neuron", set()) while weight_neuron_edges: edge = weight_neuron_edges.pop() init_new_dynapcnnlayer_entry( @@ -217,19 +220,19 @@ def collect_dynapcnn_layer_info( ) # Process all dvs->weight edges connecting the DVS camera to a unique dynapcnn layer. - dvs_weight_edges = edges_by_type.get("dvs-weight", Set()) + dvs_weight_edges = edges_by_type.get("dvs-weight", set()) while dvs_weight_edges: edge = dvs_weight_edges.pop() - add_or_update_dvs_to_entry( + dvs_layer_info = add_or_update_dvs_to_entry( edge, - dynapcnn_layer_info, + dvs_layer_info, indx_2_module_map, node_2_layer_map, nodes_io_shapes, ) # Process all edges connecting two dynapcnn layers that do not include pooling - neuron_weight_edges = edges_by_type.get("neuron-weight", Set()) + neuron_weight_edges = edges_by_type.get("neuron-weight", set()) while neuron_weight_edges: edge = neuron_weight_edges.pop() set_neuron_layer_destination( @@ -241,9 +244,10 @@ def collect_dynapcnn_layer_info( ) # Add pooling based on neuron->pooling connections - pooling_pooling_edges = edges_by_type.get("pooling-pooling", Set()) - while pooling_pooling_edges: - edge = pooling_pooling_edges.pop() + pooling_pooling_edges = edges_by_type.get("pooling-pooling", set()) + neuron_pooling_edges = edges_by_type.get("neuron-pooling", set()) + while neuron_pooling_edges: + edge = neuron_pooling_edges.pop() # Search pooling-pooling edges for chains of pooling and add to existing entry pooling_chains, edges_used = trace_paths(edge[1], pooling_pooling_edges) add_pooling_to_entry( @@ -268,7 +272,7 @@ def collect_dynapcnn_layer_info( ) # Add all edges connecting pooling to a new dynapcnn layer - pooling_weight_edges = edges_by_type.get("pooling-weight", Set()) + pooling_weight_edges = edges_by_type.get("pooling-weight", set()) while pooling_weight_edges: edge = pooling_weight_edges.pop() set_pooling_layer_destination( @@ -285,7 +289,7 @@ def collect_dynapcnn_layer_info( # Set minimal destination entries for layers without child nodes, to act as network outputs set_exit_destinations(dynapcnn_layer_info) - return dynapcnn_layer_info + return dynapcnn_layer_info, dvs_layer_info def get_valid_edge_type( @@ -466,11 +470,11 @@ def add_pooling_to_entry( def add_or_update_dvs_to_entry( edge: Edge, - dynapcnn_layer_info: Dict[int, Dict[int, Dict]], + dvs_layer_info: Union[None, Dict], indx_2_module_map: Dict[int, nn.Module], node_2_layer_map: Dict[int, int], nodes_io_shapes: Dict[int, Dict[str, Tuple[Size, Size]]], -) -> None: +) -> Dict: """ Initiate or update dict to hold information for a DVS Layer configuration based on a "dvs-weight" edges. Change `dynapcnn_layer_info` in-place. If a entry for the DVS node exists the function will add a new entry to the `desctinations` key of its dictionary. @@ -479,9 +483,8 @@ def add_or_update_dvs_to_entry( ---------- edge: Tuple of 2 integers, indicating edge between two nodes in graph. Edge target has to be within an existing entry of `dynapcnn_layer_info`. - dynapcnn_layer_info: Dict with one entry for each future dynapcnn layer. - key is unique dynapcnn layer ID, value is dict with nodes of the layer - Will be updated in-place. + dvs_layer_info: Dict containing information about the DVSLayer. If `None`, + will instantiate a new dict indx_2_module_map (dict): Maps node IDs of the graph as `key` to their associated module as `value` node_2_layer_map (dict): Maps each node ID to the ID of the layer it is assigned to. Will be updated in-place. @@ -502,10 +505,10 @@ def add_or_update_dvs_to_entry( "layers have to be followed by a spiking layer (`sl.IAFSqueeze`)." ) - if "dvs" not in dynapcnn_layer_info: + if dvs_layer_info is None: # DVS node hasn't been initialized yet # Init. entry for a DVS layer using its configuration dict. - dynapcnn_layer_info["dvs"] = { + dvs_layer_info = { "node_id": edge[0], "input_shape": nodes_io_shapes[edge[0]]["input"], "module": indx_2_module_map[edge[0]], @@ -515,11 +518,12 @@ def add_or_update_dvs_to_entry( node_2_layer_map[edge[0]] = "dvs" else: # Update entry for DVS with new destination. - dvs_layer_info = dynapcnn_layer_info["dvs"] assert dvs_layer_info["node_id"] == edge[0] assert destination_layer_idx not in dvs_layer_info["destinations"] dvs_layer_info["destinations"].append(destination_layer_idx) + + return dvs_layer_info def set_exit_destinations(dynapcnn_layer: Dict) -> None: """Set minimal destination entries for layers that don't have any. diff --git a/tests/test_dynapcnnlayer/test_dynapcnnlayer.py b/tests/test_dynapcnnlayer/test_dynapcnnlayer.py index a1e0de79..53fdcf83 100644 --- a/tests/test_dynapcnnlayer/test_dynapcnnlayer.py +++ b/tests/test_dynapcnnlayer/test_dynapcnnlayer.py @@ -21,7 +21,7 @@ def test_DynapcnnLayer(dcnnl_map, discretize, expected_output): # create a `DynapcnnLayer` from the set of layers in `nodes_to_dcnnl_map[dpcnnl_idx]`. dynapcnn_layers, destination_map, entry_points = construct_dynapcnnlayers_from_mapper( - dcnnl_map=dcnnl_map, discretize=discretize, rescale_fn=None + dcnnl_map=dcnnl_map, discretize=discretize, rescale_fn=None, dvs_layer_info=None, ) for layer_index, dynapcnn_layer in dynapcnn_layers.items(): From ef2c6d74667a68a205f04df2f4cbb8d56d0363bb Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Tue, 5 Nov 2024 15:05:11 +0100 Subject: [PATCH 307/379] Reformat --- sinabs/backend/dynapcnn/__init__.py | 4 +- sinabs/backend/dynapcnn/chips/dynapcnn.py | 8 +- sinabs/backend/dynapcnn/chips/speck2cmini.py | 1 - sinabs/backend/dynapcnn/chips/speck2e.py | 1 - sinabs/backend/dynapcnn/chips/speck2f.py | 1 - sinabs/backend/dynapcnn/config_builder.py | 3 +- sinabs/backend/dynapcnn/connectivity_specs.py | 6 +- sinabs/backend/dynapcnn/discretize.py | 3 +- sinabs/backend/dynapcnn/dvs_layer.py | 1 - sinabs/backend/dynapcnn/dynapcnn_layer.py | 3 +- .../backend/dynapcnn/dynapcnn_layer_utils.py | 17 +- sinabs/backend/dynapcnn/dynapcnn_network.py | 13 +- .../dynapcnn/dynapcnnnetwork_module.py | 40 ++-- sinabs/backend/dynapcnn/exceptions.py | 5 +- sinabs/backend/dynapcnn/mapping.py | 8 +- .../backend/dynapcnn/nir_graph_extractor.py | 99 ++++++---- .../backend/dynapcnn/sinabs_edges_handler.py | 172 ++++++++++++------ sinabs/backend/dynapcnn/specksim.py | 3 +- sinabs/backend/dynapcnn/utils.py | 5 +- 19 files changed, 244 insertions(+), 149 deletions(-) diff --git a/sinabs/backend/dynapcnn/__init__.py b/sinabs/backend/dynapcnn/__init__.py index fa2035af..09f93756 100644 --- a/sinabs/backend/dynapcnn/__init__.py +++ b/sinabs/backend/dynapcnn/__init__.py @@ -1,6 +1,6 @@ +from .dvs_layer import DVSLayer from .dynapcnn_layer import DynapcnnLayer from .dynapcnn_network import DynapcnnCompatibleNetwork, DynapcnnNetwork -from .dynapcnnnetwork_module import DynapcnnNetworkModule from .dynapcnn_visualizer import DynapcnnVisualizer -from .dvs_layer import DVSLayer +from .dynapcnnnetwork_module import DynapcnnNetworkModule from .nir_graph_extractor import GraphExtractor diff --git a/sinabs/backend/dynapcnn/chips/dynapcnn.py b/sinabs/backend/dynapcnn/chips/dynapcnn.py index 9547819f..543e4e52 100644 --- a/sinabs/backend/dynapcnn/chips/dynapcnn.py +++ b/sinabs/backend/dynapcnn/chips/dynapcnn.py @@ -3,14 +3,13 @@ from warnings import warn import samna +import sinabs import torch from samna.dynapcnn.configuration import ( CNNLayerConfig, - DynapcnnConfiguration, DVSLayerConfig, + DynapcnnConfiguration, ) - -import sinabs from sinabs.backend.dynapcnn.config_builder import ConfigBuilder from sinabs.backend.dynapcnn.dvs_layer import DVSLayer from sinabs.backend.dynapcnn.dynapcnn_layer import DynapcnnLayer @@ -324,7 +323,8 @@ def build_config( layer=sw_layer, layer2core_map=layer2core_map, destination_indices=destination_indices, - chip_layer=chip_layer) + chip_layer=chip_layer, + ) else: # shouldn't happen since type checks are made previously. raise TypeError( diff --git a/sinabs/backend/dynapcnn/chips/speck2cmini.py b/sinabs/backend/dynapcnn/chips/speck2cmini.py index 66b5b4d1..a17e5830 100644 --- a/sinabs/backend/dynapcnn/chips/speck2cmini.py +++ b/sinabs/backend/dynapcnn/chips/speck2cmini.py @@ -2,7 +2,6 @@ import samna from samna.speck2cMini.configuration import SpeckConfiguration - from sinabs.backend.dynapcnn.dynapcnn_layer import DynapcnnLayer from sinabs.backend.dynapcnn.mapping import LayerConstraints diff --git a/sinabs/backend/dynapcnn/chips/speck2e.py b/sinabs/backend/dynapcnn/chips/speck2e.py index 799b9f98..91a2a95c 100644 --- a/sinabs/backend/dynapcnn/chips/speck2e.py +++ b/sinabs/backend/dynapcnn/chips/speck2e.py @@ -2,7 +2,6 @@ import samna from samna.speck2e.configuration import SpeckConfiguration - from sinabs.backend.dynapcnn.dynapcnn_layer import DynapcnnLayer from .dynapcnn import DynapcnnConfigBuilder diff --git a/sinabs/backend/dynapcnn/chips/speck2f.py b/sinabs/backend/dynapcnn/chips/speck2f.py index d34418f9..c5ed563b 100644 --- a/sinabs/backend/dynapcnn/chips/speck2f.py +++ b/sinabs/backend/dynapcnn/chips/speck2f.py @@ -2,7 +2,6 @@ import samna from samna.speck2f.configuration import SpeckConfiguration - from sinabs.backend.dynapcnn.dynapcnn_layer import DynapcnnLayer from .dynapcnn import DynapcnnConfigBuilder diff --git a/sinabs/backend/dynapcnn/config_builder.py b/sinabs/backend/dynapcnn/config_builder.py index 7a360718..81e649a9 100644 --- a/sinabs/backend/dynapcnn/config_builder.py +++ b/sinabs/backend/dynapcnn/config_builder.py @@ -2,11 +2,10 @@ from typing import Dict, List import samna -from samna.dynapcnn.configuration import DynapcnnConfiguration - import sinabs import sinabs.backend import sinabs.backend.dynapcnn +from samna.dynapcnn.configuration import DynapcnnConfiguration from .dvs_layer import DVSLayer from .dynapcnn_layer import DynapcnnLayer diff --git a/sinabs/backend/dynapcnn/connectivity_specs.py b/sinabs/backend/dynapcnn/connectivity_specs.py index f05ccdbe..8be31d6f 100644 --- a/sinabs/backend/dynapcnn/connectivity_specs.py +++ b/sinabs/backend/dynapcnn/connectivity_specs.py @@ -4,17 +4,17 @@ from typing import Union +import sinabs.layers as sl import torch.nn as nn -import sinabs.layers as sl -from .dvs_layer import DVSLayer from .crop2d import Crop2d +from .dvs_layer import DVSLayer from .flipdims import FlipDims Pooling = (sl.SumPool2d, nn.AvgPool2d) Weight = (nn.Conv2d, nn.Linear) Neuron = (sl.IAFSqueeze,) -DVS = (DVSLayer, ) +DVS = (DVSLayer,) # @TODO - need to list other edge cases involving DVS layer (for now only dvs-weight and dvs-pooling). VALID_SINABS_EDGE_TYPES_ABSTRACT = { diff --git a/sinabs/backend/dynapcnn/discretize.py b/sinabs/backend/dynapcnn/discretize.py index 0a14c27b..3a7a0253 100644 --- a/sinabs/backend/dynapcnn/discretize.py +++ b/sinabs/backend/dynapcnn/discretize.py @@ -2,11 +2,10 @@ from typing import Optional, Tuple from warnings import warn +import sinabs.layers as sl import torch import torch.nn as nn -import sinabs.layers as sl - DYNAPCNN_WEIGHT_PRECISION_BITS = 8 DYNAPCNN_STATE_PRECISION_BITS = 16 diff --git a/sinabs/backend/dynapcnn/dvs_layer.py b/sinabs/backend/dynapcnn/dvs_layer.py index 13e0c7a4..644fa5d0 100644 --- a/sinabs/backend/dynapcnn/dvs_layer.py +++ b/sinabs/backend/dynapcnn/dvs_layer.py @@ -1,7 +1,6 @@ from typing import Optional, Tuple import torch.nn as nn - from sinabs.layers import SumPool2d from sinabs.utils import expand_to_pair diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer.py b/sinabs/backend/dynapcnn/dynapcnn_layer.py index cb48ea6c..ac5675fe 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer.py @@ -6,11 +6,10 @@ from typing import List, Tuple import numpy as np +import sinabs.layers as sl import torch from torch import nn -import sinabs.layers as sl - from .discretize import discretize_conv_spike_ # Define sum pooling functional as power-average pooling with power 1 diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py b/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py index 7079be5a..fe576754 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py @@ -1,16 +1,18 @@ from math import prod from typing import Callable, Dict, Iterable, List, Optional, Set, Tuple, Union -from torch import nn - from sinabs import layers as sl from sinabs.utils import expand_to_pair +from torch import nn from .dynapcnn_layer import DynapcnnLayer def construct_dynapcnnlayers_from_mapper( - dcnnl_map: Dict, dvs_layer_info: Union[None, Dict], discretize: bool, rescale_fn: Optional[Callable] = None + dcnnl_map: Dict, + dvs_layer_info: Union[None, Dict], + discretize: bool, + rescale_fn: Optional[Callable] = None, ) -> Tuple[Dict[int, DynapcnnLayer], Dict[int, Set[int]], List[int]]: """Construct DynapcnnLayer instances from `dcnnl_map` @@ -59,6 +61,7 @@ def finalize_dcnnl_map(dcnnl_map: Dict, rescale_fn: Optional[Callable] = None) - # Consolidate scale factors consolidate_layer_scaling(layer_info, rescale_fn) + def consolidate_layer_pooling(layer_info: Dict, dcnnl_map: Dict): """Consolidate pooling information for individual layer @@ -252,7 +255,9 @@ def construct_single_dynapcnn_layer( ) -def construct_destination_map(dcnnl_map: Dict[int, Dict], dvs_layer_info: Union[None, Dict]) -> Dict[int, List[int]]: +def construct_destination_map( + dcnnl_map: Dict[int, Dict], dvs_layer_info: Union[None, Dict] +) -> Dict[int, List[int]]: """Create a dict that holds destinations for each layer Parameters @@ -285,7 +290,9 @@ def construct_destination_map(dcnnl_map: Dict[int, Dict], dvs_layer_info: Union[ return destination_map -def collect_entry_points(dcnnl_map: Dict[int, Dict], dvs_layer_info: Union[None, Dict]) -> Set[int]: +def collect_entry_points( + dcnnl_map: Dict[int, Dict], dvs_layer_info: Union[None, Dict] +) -> Set[int]: """Return set of layer indices that are entry points Parameters diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index 0e485107..a541bfbe 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -6,14 +6,13 @@ from warnings import warn import samna +import sinabs +import sinabs.layers as sl import torch import torch.nn as nn from samna.dynapcnn.configuration import DynapcnnConfiguration from torch import Tensor -import sinabs -import sinabs.layers as sl - from .chip_factory import ChipFactory from .dvs_layer import DVSLayer from .dynapcnn_layer import DynapcnnLayer @@ -109,10 +108,12 @@ def dynapcnn_layers(self): @property def dynapcnn_module(self): return self._dynapcnn_module - + @property def exit_layers(self): - return [self.dynapcnn_layers[i] for i in self._dynapcnn_module.get_exit_layers()] + return [ + self.dynapcnn_layers[i] for i in self._dynapcnn_module.get_exit_layers() + ] @property def layer_destination_map(self): @@ -231,7 +232,7 @@ def memory_summary(self) -> Dict[str, Dict[int, int]]: Each nested dict has as keys the indices of all dynapcnn_layers and as values the corresonding memory values for each layer. """ - # For each entry (kernel, neuron, bias) provide one nested dict with + # For each entry (kernel, neuron, bias) provide one nested dict with # one entry for each layer summary = {key: dict() for key in ("kernel", "neuron", "bias")} diff --git a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py index 44dad262..07d41a8f 100644 --- a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py +++ b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py @@ -2,17 +2,16 @@ # contact : wsoaresgirao@gmail.com from collections import defaultdict -from typing import Dict, List, Set, Union, Optional +from typing import Dict, List, Optional, Set, Union from warnings import warn +import sinabs.layers as sl import torch.nn as nn from torch import Tensor -import sinabs.layers as sl - +from .dvs_layer import DVSLayer from .dynapcnn_layer import DynapcnnLayer from .utils import Edge, topological_sorting -from .dvs_layer import DVSLayer class DynapcnnNetworkModule(nn.Module): @@ -25,7 +24,7 @@ class DynapcnnNetworkModule(nn.Module): ---------- - dynapcnn_layers (dict): a mapper containing `DynapcnnLayer` instances. - destination_map (dict): Maps layer indices to list of destination indices. - Exit destinations are marked by negative integers + Exit destinations are marked by negative integers - entry_points (set): Set of layer indices that act as network entry points. - dvs_node_info (dict): contains information associated with the `DVSLayer` node. `None` if no DVS node exists. @@ -85,11 +84,11 @@ def dvs_node_info(self): @property def dvs_layer(self): return self._dvs_layer - + @property def destination_map(self): return self._destination_map - + @property def dynapcnn_layers(self): # Convert string-indices to integer-indices @@ -102,14 +101,14 @@ def entry_points(self): @property def sorted_nodes(self): return self._sorted_nodes - + @property def node_source_map(self): return self._node_source_map def get_exit_layers(self) -> List[int]: - """ Get layers that act as exit points of the network - + """Get layers that act as exit points of the network + Returns ------- - List[int]: Layer indices with at least one exit destination. @@ -121,8 +120,8 @@ def get_exit_layers(self) -> List[int]: ] def get_exit_points(self) -> Dict[int, Dict]: - """ Get details of layers that act as exit points of the network - + """Get details of layers that act as exit points of the network + Returns ------- - Dict[int, Dict]: Dict whose keys are layer indices of `dynapcnn_layers` @@ -149,10 +148,11 @@ def get_exit_points(self) -> Dict[int, Dict]: return exit_layers - - def setup_dynapcnnlayer_graph(self, index_layers_topologically: bool = False) -> None: - """ Set up data structures to run forward pass through dynapcnn layers - + def setup_dynapcnnlayer_graph( + self, index_layers_topologically: bool = False + ) -> None: + """Set up data structures to run forward pass through dynapcnn layers + Parameters ---------- - index_layers_topologically (bool): If True, will assign new indices to @@ -294,7 +294,7 @@ def forward( layers_outputs[idx_curr] = { idx_dest: out for idx_dest, out in zip(destinations, output) } - + if return_complete: return layers_outputs @@ -328,8 +328,8 @@ def forward( return network_outputs def reindex_layers(self, index_order: List[int]) -> None: - """ Reindex layers based on provided order - + """Reindex layers based on provided order + Will assign new index to dynapcnn layers and update all internal attributes accordingly. @@ -368,4 +368,4 @@ def remap(key): self._node_source_map = { remap(node): [remap(src) for src in sources] for node, sources in self._node_source_map.items() - } \ No newline at end of file + } diff --git a/sinabs/backend/dynapcnn/exceptions.py b/sinabs/backend/dynapcnn/exceptions.py index e88324ed..ff755e35 100644 --- a/sinabs/backend/dynapcnn/exceptions.py +++ b/sinabs/backend/dynapcnn/exceptions.py @@ -69,7 +69,10 @@ class InvalidGraphStructure(Exception): class InvalidModelWithDVSSetup(Exception): def __init__(self): - super().__init__(f"The network provided has a DVSLayer instance but argument 'dvs_input' is set to False.") + super().__init__( + f"The network provided has a DVSLayer instance but argument 'dvs_input' is set to False." + ) + # Edge exceptions. diff --git a/sinabs/backend/dynapcnn/mapping.py b/sinabs/backend/dynapcnn/mapping.py index cd877692..d80475d9 100644 --- a/sinabs/backend/dynapcnn/mapping.py +++ b/sinabs/backend/dynapcnn/mapping.py @@ -81,9 +81,7 @@ def get_valid_mapping( netmap = recover_mapping(new_graph, len(layer_mapping)) # Convert `netmap` to dict mapping from layer index to core ID - return { - layer_idx: core_id for layer_idx, core_id in zip(layer_indices, netmap) - } + return {layer_idx: core_id for layer_idx, core_id in zip(layer_indices, netmap)} @dataclass @@ -216,9 +214,7 @@ def make_flow_graph( return graph -def recover_mapping( - graph: List[List[FlowGraphEdge]], num_layers: int -) -> List[int]: +def recover_mapping(graph: List[List[FlowGraphEdge]], num_layers: int) -> List[int]: """Based on the flow graph retrieve a layer-to-core mapping Parameters diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index 15d98190..ed18a94e 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -4,20 +4,24 @@ from typing import Callable, Dict, Iterable, List, Optional, Set, Tuple, Type, Union import nirtorch +import sinabs import torch import torch.nn as nn -import sinabs -from .dvs_layer import DVSLayer - from .connectivity_specs import ( LAYER_TYPES_WITH_MULTIPLE_INPUTS, LAYER_TYPES_WITH_MULTIPLE_OUTPUTS, ) +from .dvs_layer import DVSLayer from .dynapcnn_layer_utils import construct_dynapcnnlayers_from_mapper from .dynapcnnnetwork_module import DynapcnnNetworkModule from .exceptions import InvalidGraphStructure, InvalidModelWithDVSSetup -from .sinabs_edges_handler import collect_dynapcnn_layer_info, get_dvs_node_from_mapper, fix_dvs_module_edges, merge_dvs_pooling_edge +from .sinabs_edges_handler import ( + collect_dynapcnn_layer_info, + fix_dvs_module_edges, + get_dvs_node_from_mapper, + merge_dvs_pooling_edge, +) from .utils import Edge, topological_sorting try: @@ -60,7 +64,7 @@ def __init__( Map from layer ID to the corresponding nn.Module instance. - nodes_io_shapes (dict): Map from node ID to dict containing node's in- and output shapes - - dvs_input (bool): optional (default as `None`). Whether or not the model + - dvs_input (bool): optional (default as `None`). Whether or not the model should start with a `DVSLayer`. - ignore_node_types (iterable of types): Node types that should be ignored completely from the graph. This can include, for instance, @@ -273,14 +277,16 @@ def verify_graph_integrity(self): ####################################################### Pivate Methods ####################################################### - def _handle_dvs_input(self, input_shape: Tuple[int, int, int], dvs_input: Optional[bool] = None): + def _handle_dvs_input( + self, input_shape: Tuple[int, int, int], dvs_input: Optional[bool] = None + ): """Make sure DVS input is properly integrated into graph - + - Decide whether `DVSLayer` instance needs to be added to the graph This is the case when `dvs_input==True` and there is no `DVSLayer` yet. - Make sure edges between DVS related nodes are set properly - Absorb pooling layers in DVS node if applicable - + Parameters ---------- - input_shape (tuple of three integers): Input shape (features, height, width) @@ -294,20 +300,27 @@ def _handle_dvs_input(self, input_shape: Tuple[int, int, int], dvs_input: Option if self._need_dvs_node(dvs_input): # Insert a DVSLayer node in the graph. self._add_dvs_node(dvs_input_shape=input_shape) - + # Check for the need of fixing NIR edges extraction when DVS is a node in the graph. If DVS # is used its node becomes the only entry node in the graph. - fix_dvs_module_edges(self._edges, self._indx_2_module_map, self._name_2_indx_map, self._entry_nodes) + fix_dvs_module_edges( + self._edges, + self._indx_2_module_map, + self._name_2_indx_map, + self._entry_nodes, + ) - # Merge a pooling node from a 'dvs-pooling' edge (pooling being an independent node in the original + # Merge a pooling node from a 'dvs-pooling' edge (pooling being an independent node in the original # graph) into the DVSLayer if such edge exists. - merge_dvs_pooling_edge(self._edges, self._indx_2_module_map, self._name_2_indx_map) + merge_dvs_pooling_edge( + self._edges, self._indx_2_module_map, self._name_2_indx_map + ) # Check if graph structure and DVSLayer.merge_polarities are correctly set (if DVS node exists). self._validate_dvs_setup(dvs_input_shape=input_shape) def _add_dvs_node(self, dvs_input_shape: Tuple[int, int, int]) -> None: - """ In-place modification of `self._name_2_indx_map`, `self._indx_2_module_map`, and `self._edges` to accomodate the + """In-place modification of `self._name_2_indx_map`, `self._indx_2_module_map`, and `self._edges` to accomodate the creation of an extra node in the graph representing the DVS camera of the chip. The DVSLayer node will point to every other node that is up to this point an entry node of the original graph, so `self._entry_nodes` is modified in-place to have only one entry: the index of the DVS node. @@ -319,27 +332,34 @@ def _add_dvs_node(self, dvs_input_shape: Tuple[int, int, int]) -> None: (features, height, width) = dvs_input_shape if features > 2: - raise ValueError(f'A DVSLayer istance can have the feature dimension of its inputs with values 1 or 2 but {features} was given.') + raise ValueError( + f"A DVSLayer istance can have the feature dimension of its inputs with values 1 or 2 but {features} was given." + ) # add name entry for node 'dvs'. - self._name_2_indx_map['dvs'] = len(self._name_2_indx_map) + self._name_2_indx_map["dvs"] = len(self._name_2_indx_map) # add module entry for node 'dvs'. - self._indx_2_module_map[self._name_2_indx_map['dvs']] = DVSLayer( + self._indx_2_module_map[self._name_2_indx_map["dvs"]] = DVSLayer( input_shape=(height, width), merge_polarities=(features == 1), ) # set DVS node as input to each entry node of the graph. - self._edges.update({(self._name_2_indx_map['dvs'], entry_node) for entry_node in self._entry_nodes}) + self._edges.update( + { + (self._name_2_indx_map["dvs"], entry_node) + for entry_node in self._entry_nodes + } + ) # DVSLayer node becomes the only entrypoint of the graph. - self._entry_nodes = {self._name_2_indx_map['dvs']} + self._entry_nodes = {self._name_2_indx_map["dvs"]} def _need_dvs_node(self, dvs_input: Optional[bool] = None) -> bool: - """ Returns whether or not a node will need to be added to represent a - `DVSLayer` instance. - + """Returns whether or not a node will need to be added to represent a + `DVSLayer` instance. + A new node will have to be added if `self._indx_2_module_map` contains no `DVSLayer` instance and `dvs_input == True`. - + Parameters ---------- - dvs_input (bool): optional (default as `None`). Wether or not dynapcnn receive input from its DVS camera. @@ -351,14 +371,16 @@ def _need_dvs_node(self, dvs_input: Optional[bool] = None) -> bool: has_dvs_layer = self.has_dvs_layer # Checks if DVSLayer instance exists but user has set 'dvs_input' to False. - if has_dvs_layer and dvs_input == False: # `== False` ensures that `None` is not considered + if ( + has_dvs_layer and dvs_input == False + ): # `== False` ensures that `None` is not considered raise InvalidModelWithDVSSetup() return not has_dvs_layer and dvs_input - + def _get_dvs_layer(self) -> Union[DVSLayer, None]: - """ Loops though all modules and return `DVSLayer` instance if it exists. - + """Loops though all modules and return `DVSLayer` instance if it exists. + Returns ------- - DVSLayer if exactly one is found, otherwise None @@ -370,7 +392,8 @@ def _get_dvs_layer(self) -> Union[DVSLayer, None]: """ dvs_layers = { - module for module in self._indx_2_module_map.values() + module + for module in self._indx_2_module_map.values() if isinstance(module, DVSLayer) } @@ -382,9 +405,9 @@ def _get_dvs_layer(self) -> Union[DVSLayer, None]: raise InvalidGraphStructure( f"The provided model has {num_dvs} `DVSLayer`s. At most one is allowed." ) - + def _validate_dvs_setup(self, dvs_input_shape: Tuple[int, int, int]) -> None: - """ If a DVSLayer node exists, makes sure it is the only entry node of the graph. Checks if its `merge_polarities` + """If a DVSLayer node exists, makes sure it is the only entry node of the graph. Checks if its `merge_polarities` attribute matches `dummy_input.shape[0]` (the number of features) and, if not, it will be set based on the numeber of features of the input. @@ -398,16 +421,22 @@ def _validate_dvs_setup(self, dvs_input_shape: Tuple[int, int, int]) -> None: return if (nb_entries := len(self._entry_nodes)) > 1: - raise ValueError(f'A DVSLayer node exists and there are {nb_entries} entry nodes in the graph: the DVSLayer should be the only entry node.') - + raise ValueError( + f"A DVSLayer node exists and there are {nb_entries} entry nodes in the graph: the DVSLayer should be the only entry node." + ) + (features, _, _) = dvs_input_shape if features > 2: - raise ValueError(f'A DVSLayer istance can have the feature dimension of its inputs with values 1 or 2 but {features} was given.') - + raise ValueError( + f"A DVSLayer istance can have the feature dimension of its inputs with values 1 or 2 but {features} was given." + ) + if self.dvs_layer.merge_polarities and features != 1: - raise ValueError(f"The 'DVSLayer.merge_polarities' is set to 'True' which means the number of input features should be 1 (current input shape is {dvs_input_shape}).") - + raise ValueError( + f"The 'DVSLayer.merge_polarities' is set to 'True' which means the number of input features should be 1 (current input shape is {dvs_input_shape})." + ) + if features == 1: self.dvs_layer.merge_polarities = True diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index 8315a2f9..2c7c3619 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -6,21 +6,21 @@ """ from collections import deque -from typing import Dict, List, Set, Tuple, Type, Optional, Union +from typing import Dict, List, Optional, Set, Tuple, Type, Union +from sinabs.layers import SumPool2d from torch import Size, nn from .connectivity_specs import VALID_SINABS_EDGE_TYPES, Pooling -from .exceptions import InvalidEdge, InvalidGraphStructure -from .utils import Edge -from .dvs_layer import DVSLayer from .crop2d import Crop2d +from .dvs_layer import DVSLayer +from .exceptions import InvalidEdge, InvalidGraphStructure from .flipdims import FlipDims +from .utils import Edge -from sinabs.layers import SumPool2d def get_dvs_node_from_mapper(dcnnl_map: Dict) -> Optional[Dict]: - """ Returns the information dict associated with the `DVSLayer` instance within `dcnnl_map`. + """Returns the information dict associated with the `DVSLayer` instance within `dcnnl_map`. Parameters ---------- @@ -28,25 +28,31 @@ def get_dvs_node_from_mapper(dcnnl_map: Dict) -> Optional[Dict]: Returns ------- - - Dict containing information associated with the `DVSLayer` node (if no DVS node exists it'll return `None`). + - Dict containing information associated with the `DVSLayer` node (if no DVS node exists it'll return `None`). """ # TODO: Would it make more sense to have dvs layer separated from dcnnl layers? # Either as different object or with a very clear key inside `dcnnl_map` for layer_index, layer_info in dcnnl_map.items(): - if 'dvs_layer' in layer_info: - assert layer_info['dvs_layer'] + if "dvs_layer" in layer_info: + assert layer_info["dvs_layer"] return layer_info return None -def fix_dvs_module_edges(edges: Set[Edge], indx_2_module_map: Dict[int, nn.Module], name_2_indx_map: Dict[str, int], entry_nodes: Set[Edge]) -> None: - """ All arguments are modified in-place to fix wrong node extractions from NIRtorch when a DVSLayer istance is the first layer in the network. - + +def fix_dvs_module_edges( + edges: Set[Edge], + indx_2_module_map: Dict[int, nn.Module], + name_2_indx_map: Dict[str, int], + entry_nodes: Set[Edge], +) -> None: + """All arguments are modified in-place to fix wrong node extractions from NIRtorch when a DVSLayer istance is the first layer in the network. + Modifies `edges` to re-structure the edges related witht the DVSLayer instance. The DVSLayer's forward method feeds data in the sequence 'DVS -> DVS.pool -> DVS.crop -> DVS.flip', so we remove edges involving these nodes (that are internaly implementend in the DVSLayer) from the graph and point the node of DVSLayer to the node where it should send its output to. This is also removes a self-recurrent node with edge '(FlipDims, FlipDims)' that is wrongly extracted. - Modifies `indx_2_module_map` and `name_2_indx_map` to remove the internal DVSLayer nodes (Crop2d, FlipDims and DVSLayer's pooling) since + Modifies `indx_2_module_map` and `name_2_indx_map` to remove the internal DVSLayer nodes (Crop2d, FlipDims and DVSLayer's pooling) since these should not be independent nodes in the graph. Modifies `entry_nodes` such that the DVSLayer becomes the only entry node of the graph. @@ -64,8 +70,11 @@ def fix_dvs_module_edges(edges: Set[Edge], indx_2_module_map: Dict[int, nn.Modul # spot nodes (ie, modules) used in a DVSLayer instance's forward pass (including the DVSLayer node itself). dvslayer_nodes = { - index: module for index, module in indx_2_module_map.items() - if any(isinstance(module, dvs_node) for dvs_node in (DVSLayer, Crop2d, FlipDims)) + index: module + for index, module in indx_2_module_map.items() + if any( + isinstance(module, dvs_node) for dvs_node in (DVSLayer, Crop2d, FlipDims) + ) } if len(dvslayer_nodes) <= 1: @@ -74,39 +83,68 @@ def fix_dvs_module_edges(edges: Set[Edge], indx_2_module_map: Dict[int, nn.Modul # TODO - a `SumPool2d` is also a node that's used inside a DVSLayer instance. In what follows we try to find it # by looking for pooling nodes that appear in a (pool, crop) edge - the assumption being that if the pooling is - # inputing into a crop layer than the pool is inside the DVSLayer instance. It feels like a hacky way to do it + # inputing into a crop layer than the pool is inside the DVSLayer instance. It feels like a hacky way to do it # so we should revise this. - dvslayer_nodes.update({ - edge[0]: indx_2_module_map[edge[0]] for edge in edges - if isinstance(indx_2_module_map[edge[0]], SumPool2d) and isinstance(indx_2_module_map[edge[1]], Crop2d) - }) + dvslayer_nodes.update( + { + edge[0]: indx_2_module_map[edge[0]] + for edge in edges + if isinstance(indx_2_module_map[edge[0]], SumPool2d) + and isinstance(indx_2_module_map[edge[1]], Crop2d) + } + ) # NIR is extracting an edge (FlipDims, FlipDims) from the DVSLayer: remove self-recurrent nodes from the graph. - for edge in [(src, tgt) for (src, tgt) in edges if (src == tgt and isinstance(indx_2_module_map[src], FlipDims))]: + for edge in [ + (src, tgt) + for (src, tgt) in edges + if (src == tgt and isinstance(indx_2_module_map[src], FlipDims)) + ]: edges.remove(edge) # Since NIR is not extracting the edges for the DVSLayer correctly, remove all edges involving the DVS. - for edge in [(src, tgt) for (src, tgt) in edges if (src in dvslayer_nodes or tgt in dvslayer_nodes)]: + for edge in [ + (src, tgt) + for (src, tgt) in edges + if (src in dvslayer_nodes or tgt in dvslayer_nodes) + ]: edges.remove(edge) # Get node's indexes based on the module type - just for validation. - dvs_node = [key for key, value in dvslayer_nodes.items() if isinstance(value, DVSLayer)] - dvs_pool_node = [key for key, value in dvslayer_nodes.items() if isinstance(value, SumPool2d)] - dvs_crop_node = [key for key, value in dvslayer_nodes.items() if isinstance(value, Crop2d)] - dvs_flip_node = [key for key, value in dvslayer_nodes.items() if isinstance(value, FlipDims)] - - if any(len(node) > 1 for node in [dvs_node, dvs_pool_node, dvs_crop_node, dvs_flip_node]): - raise ValueError(f'Internal DVS nodes should be single instances but multiple have been found: dvs_node: {len(dvs_node)} dvs_pool_node: {len(dvs_pool_node)} dvs_crop_node: {len(dvs_crop_node)} dvs_flip_node: {len(dvs_flip_node)}') - + dvs_node = [ + key for key, value in dvslayer_nodes.items() if isinstance(value, DVSLayer) + ] + dvs_pool_node = [ + key for key, value in dvslayer_nodes.items() if isinstance(value, SumPool2d) + ] + dvs_crop_node = [ + key for key, value in dvslayer_nodes.items() if isinstance(value, Crop2d) + ] + dvs_flip_node = [ + key for key, value in dvslayer_nodes.items() if isinstance(value, FlipDims) + ] + + if any( + len(node) > 1 + for node in [dvs_node, dvs_pool_node, dvs_crop_node, dvs_flip_node] + ): + raise ValueError( + f"Internal DVS nodes should be single instances but multiple have been found: dvs_node: {len(dvs_node)} dvs_pool_node: {len(dvs_pool_node)} dvs_crop_node: {len(dvs_crop_node)} dvs_flip_node: {len(dvs_flip_node)}" + ) + # Remove dvs_pool, dvs_crop and dvs_flip nodes from `indx_2_module_map` (these operate within the DVS, not as independent nodes of the final graph). indx_2_module_map.pop(dvs_pool_node[-1]) indx_2_module_map.pop(dvs_crop_node[-1]) indx_2_module_map.pop(dvs_flip_node[-1]) # Remove internal DVS modeules from name/index map. - for name in [name for name, index in name_2_indx_map.items() if index in [dvs_pool_node[-1], dvs_crop_node[-1], dvs_flip_node[-1]]]: + for name in [ + name + for name, index in name_2_indx_map.items() + if index in [dvs_pool_node[-1], dvs_crop_node[-1], dvs_flip_node[-1]] + ]: name_2_indx_map.pop(name) - + # Add edges from 'dvs' node to the entry point of the graph. all_sources, all_targets = zip(*edges) local_entry_nodes = set(all_sources) - set(all_targets) @@ -116,8 +154,13 @@ def fix_dvs_module_edges(edges: Set[Edge], indx_2_module_map: Dict[int, nn.Modul entry_nodes.clear() entry_nodes.add(dvs_node[-1]) -def merge_dvs_pooling_edge(edges: Set[Edge], indx_2_module_map: Dict[int, nn.Module], name_2_indx_map: Dict[str, int]) -> None: - """ If a 'dvs-polling' edge existis, the pooling is incorporated into the DVSLayer node if `DVSLayer.pool_layer` has + +def merge_dvs_pooling_edge( + edges: Set[Edge], + indx_2_module_map: Dict[int, nn.Module], + name_2_indx_map: Dict[str, int], +) -> None: + """If a 'dvs-polling' edge existis, the pooling is incorporated into the DVSLayer node if `DVSLayer.pool_layer` has default values. All arguments are modified in-place to remove the references to the incorporated pooling node. Parameters @@ -128,29 +171,44 @@ def merge_dvs_pooling_edge(edges: Set[Edge], indx_2_module_map: Dict[int, nn.Mod """ # Find 'DVSLayer-pooling' edge. dvs_pool_edge = [ - (src, tgt) for (src, tgt) in edges - if (isinstance(indx_2_module_map[src], DVSLayer) and any(isinstance(indx_2_module_map[tgt], pool) for pool in Pooling)) + (src, tgt) + for (src, tgt) in edges + if ( + isinstance(indx_2_module_map[src], DVSLayer) + and any(isinstance(indx_2_module_map[tgt], pool) for pool in Pooling) + ) ] - + if len(dvs_pool_edge) == 0: # No dvs-pooling edge exists - nothing to do here. return if len(dvs_pool_edge) > 1: # DVSLayer in the original network can have only a single pooling layer liked to it. - raise ValueError(f'DVSLayer can have a single edge onto a pooling layer but multiple were found: {dvs_pool_edge}') - + raise ValueError( + f"DVSLayer can have a single edge onto a pooling layer but multiple were found: {dvs_pool_edge}" + ) + (dvs_idnx, pool_idnx) = dvs_pool_edge[-1] # Checking pooling can be incorporated into the DVSLayer. - if indx_2_module_map[dvs_idnx].pool_layer.kernel_size == 1 and indx_2_module_map[dvs_idnx].pool_layer.stride == 1: + if ( + indx_2_module_map[dvs_idnx].pool_layer.kernel_size == 1 + and indx_2_module_map[dvs_idnx].pool_layer.stride == 1 + ): # Set DVSLayer.pool to have same config. as the independent pooling layer. - indx_2_module_map[dvs_idnx].pool_layer.kernel_size = indx_2_module_map[pool_idnx].kernel_size - indx_2_module_map[dvs_idnx].pool_layer.stride = indx_2_module_map[pool_idnx].stride + indx_2_module_map[dvs_idnx].pool_layer.kernel_size = indx_2_module_map[ + pool_idnx + ].kernel_size + indx_2_module_map[dvs_idnx].pool_layer.stride = indx_2_module_map[ + pool_idnx + ].stride # TODO: Should there not be an error be raised if the condition is wrong? # Pooling incorporated to the DVSLayer: remove its trace from mappings. indx_2_module_map.pop(pool_idnx) - name_2_indx_map.pop([name for name, indx in name_2_indx_map.items() if indx == pool_idnx][-1]) + name_2_indx_map.pop( + [name for name, indx in name_2_indx_map.items() if indx == pool_idnx][-1] + ) # Since pool is part of the DVSLayer we now make edges where pool was a source to have DVSLayer as a source. for edge in [edge for edge in edges if edge[0] == pool_idnx]: @@ -161,8 +219,14 @@ def merge_dvs_pooling_edge(edges: Set[Edge], indx_2_module_map: Dict[int, nn.Mod edges.remove((dvs_idnx, pool_idnx)) # Checks if any traces of the original pooling node can still be found. - if len([edge for edge in edges if (edge[0] == pool_idnx or edge[1] == pool_idnx)]) != 0: - raise ValueError('Edges involving the pooling layer merged into the DVSLayer are still present in the graph.') + if ( + len([edge for edge in edges if (edge[0] == pool_idnx or edge[1] == pool_idnx)]) + != 0 + ): + raise ValueError( + "Edges involving the pooling layer merged into the DVSLayer are still present in the graph." + ) + def collect_dynapcnn_layer_info( indx_2_module_map: Dict[int, nn.Module], @@ -468,6 +532,7 @@ def add_pooling_to_entry( assert node not in node_2_layer_map node_2_layer_map[node] = layer_idx + def add_or_update_dvs_to_entry( edge: Edge, dvs_layer_info: Union[None, Dict], @@ -475,8 +540,8 @@ def add_or_update_dvs_to_entry( node_2_layer_map: Dict[int, int], nodes_io_shapes: Dict[int, Dict[str, Tuple[Size, Size]]], ) -> Dict: - """ Initiate or update dict to hold information for a DVS Layer configuration based on a "dvs-weight" edges. - Change `dynapcnn_layer_info` in-place. If a entry for the DVS node exists the function will add a new entry + """Initiate or update dict to hold information for a DVS Layer configuration based on a "dvs-weight" edges. + Change `dynapcnn_layer_info` in-place. If a entry for the DVS node exists the function will add a new entry to the `desctinations` key of its dictionary. Parameters @@ -492,8 +557,10 @@ def add_or_update_dvs_to_entry( """ # This should never happen - assert isinstance(indx_2_module_map[edge[0]], DVSLayer), f'Source node in edge {edge} is of type {type(DVSLayer)} (it should be a DVSLayer instance).' - + assert isinstance( + indx_2_module_map[edge[0]], DVSLayer + ), f"Source node in edge {edge} is of type {type(DVSLayer)} (it should be a DVSLayer instance)." + # Find destination layer index try: destination_layer_idx = node_2_layer_map[edge[1]] @@ -510,7 +577,7 @@ def add_or_update_dvs_to_entry( # Init. entry for a DVS layer using its configuration dict. dvs_layer_info = { "node_id": edge[0], - "input_shape": nodes_io_shapes[edge[0]]["input"], + "input_shape": nodes_io_shapes[edge[0]]["input"], "module": indx_2_module_map[edge[0]], "destinations": [node_2_layer_map[edge[1]]], } @@ -520,11 +587,12 @@ def add_or_update_dvs_to_entry( # Update entry for DVS with new destination. assert dvs_layer_info["node_id"] == edge[0] assert destination_layer_idx not in dvs_layer_info["destinations"] - + dvs_layer_info["destinations"].append(destination_layer_idx) - + return dvs_layer_info + def set_exit_destinations(dynapcnn_layer: Dict) -> None: """Set minimal destination entries for layers that don't have any. diff --git a/sinabs/backend/dynapcnn/specksim.py b/sinabs/backend/dynapcnn/specksim.py index 4a1c35f0..2923c070 100644 --- a/sinabs/backend/dynapcnn/specksim.py +++ b/sinabs/backend/dynapcnn/specksim.py @@ -4,12 +4,11 @@ import numpy as np import samna +import sinabs.layers as sl import torch.nn as nn from samna.specksim.nodes import SpecksimConvolutionalFilterNode as ConvFilter from samna.specksim.nodes import SpecksimIAFFilterNode as IAFFilter from samna.specksim.nodes import SpecksimSumPoolingFilterNode as SumPoolFilter - -import sinabs.layers as sl from sinabs.backend.dynapcnn import DynapcnnCompatibleNetwork, DynapcnnNetwork from sinabs.backend.dynapcnn.dynapcnn_layer import DynapcnnLayer diff --git a/sinabs/backend/dynapcnn/utils.py b/sinabs/backend/dynapcnn/utils.py index 05f9c63c..ec55186a 100644 --- a/sinabs/backend/dynapcnn/utils.py +++ b/sinabs/backend/dynapcnn/utils.py @@ -2,10 +2,9 @@ from copy import deepcopy from typing import TYPE_CHECKING, List, Optional, Set, Tuple +import sinabs.layers as sl import torch import torch.nn as nn - -import sinabs.layers as sl from sinabs.utils import expand_to_pair from .crop2d import Crop2d @@ -392,7 +391,7 @@ def extend_readout_layer(model: "DynapcnnNetwork") -> "DynapcnnNetwork": ext_bias_data[i * 4] = og_bias_data[i] og_readout_conv_layer.bias.data = ext_bias_data # run a forward pass to initialize the new weights and last IAF - model(torch.zeros(size=(1, *input_shape))) + model(torch.zeros(size=(1, *input_shape))) return model From c196f981a464968f70388daa96faf8b16b42db06 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Tue, 5 Nov 2024 15:07:30 +0100 Subject: [PATCH 308/379] Remove obsolete functions --- .../backend/dynapcnn/dynapcnn_layer_utils.py | 46 ------------------- .../backend/dynapcnn/nir_graph_extractor.py | 1 - .../backend/dynapcnn/sinabs_edges_handler.py | 20 -------- 3 files changed, 67 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py b/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py index fe576754..c1974d7d 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py @@ -184,52 +184,6 @@ def consolidate_layer_scaling(layer_info: Dict, rescale_fn: Optional[Callable] = layer_info["rescale_factor"] = rescale_factor -# TODO: Obsolete -def determine_layer_input_shape(layer_info: Dict): - """Determine input shape of single layer - - Update "input_shape" entry of `layer_info`. - If weight layer is convolutional, only verify that output shapes - of preceding layer are not greater than input shape in any dimension. - - If weight layer is linear, the current "input_shape" entry will - correspond to the shape after flattening, which might not match - the shape of the actual input to the layer. Therefore the new input - shape is the largest size across all output shapes of preceding - layers, for each dimension individually. - Verify that total number of elements (product of entries in new - input shape) does not exceed that of original input shape. - - Parameters - ---------- - - layer_info: Dict holding info of single layer. - """ - # For each dimension find largest inferred input size - max_inferred_input_shape = [ - max(sizes) for sizes in zip(*layer_info["inferred_input_shapes"]) - ] - - if isinstance(layer_info["conv"]["module"], nn.Linear): - if prod(max_inferred_input_shape) > prod(layer_info["input_shape"]): - raise ValueError( - "Combined output of some layers projecting to a linear layer is " - "larger than expected by destination layer. " - ) - # Take shape before flattening, to convert linear to conv layer - layer_info["input_shape"] = max_inferred_input_shape - else: - if any( - inferred > expected - for inferred, expected in zip( - max_inferred_input_shape, layer_info["input_shape"] - ) - ): - raise ValueError( - "Output of some layers is larger than expected by destination " - "layer along some dimensions." - ) - - def construct_single_dynapcnn_layer( layer_info: Dict, discretize: bool ) -> DynapcnnLayer: diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index ed18a94e..874b2532 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -19,7 +19,6 @@ from .sinabs_edges_handler import ( collect_dynapcnn_layer_info, fix_dvs_module_edges, - get_dvs_node_from_mapper, merge_dvs_pooling_edge, ) from .utils import Edge, topological_sorting diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index 2c7c3619..fff1f05f 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -19,26 +19,6 @@ from .utils import Edge -def get_dvs_node_from_mapper(dcnnl_map: Dict) -> Optional[Dict]: - """Returns the information dict associated with the `DVSLayer` instance within `dcnnl_map`. - - Parameters - ---------- - - dcnnl_map: Dict holding info needed to instantiate DynapcnnLayer instances. - - Returns - ------- - - Dict containing information associated with the `DVSLayer` node (if no DVS node exists it'll return `None`). - """ - # TODO: Would it make more sense to have dvs layer separated from dcnnl layers? - # Either as different object or with a very clear key inside `dcnnl_map` - for layer_index, layer_info in dcnnl_map.items(): - if "dvs_layer" in layer_info: - assert layer_info["dvs_layer"] - return layer_info - return None - - def fix_dvs_module_edges( edges: Set[Edge], indx_2_module_map: Dict[int, nn.Module], From 3fa8bcfcef196273016b433f634c582a17298e9a Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Tue, 5 Nov 2024 15:44:14 +0100 Subject: [PATCH 309/379] DVSLayer: Remove redundante comparisons --- sinabs/backend/dynapcnn/dvs_layer.py | 44 ++++++++++++---------------- 1 file changed, 19 insertions(+), 25 deletions(-) diff --git a/sinabs/backend/dynapcnn/dvs_layer.py b/sinabs/backend/dynapcnn/dvs_layer.py index 644fa5d0..278f68da 100644 --- a/sinabs/backend/dynapcnn/dvs_layer.py +++ b/sinabs/backend/dynapcnn/dvs_layer.py @@ -186,14 +186,13 @@ def get_output_shape_dict(self) -> dict: ) = self.get_output_shape_after_pooling() # Compute dims after cropping - if self.crop_layer is not None: - ( - channel_count, - output_size_y, - output_size_x, - ) = self.crop_layer.get_output_shape( - (channel_count, output_size_y, output_size_x) - ) + ( + channel_count, + output_size_y, + output_size_x, + ) = self.crop_layer.get_output_shape( + (channel_count, output_size_y, output_size_x) + ) # Compute dims after pooling return { @@ -250,15 +249,11 @@ def get_roi(self) -> Tuple[Tuple[int, int], Tuple[int, int]]: ------- ((top, bottom), (left, right)) """ - if self.crop_layer is not None: - _, h, w = self.get_output_shape_after_pooling() - return ( - (self.crop_layer.top_crop, self.crop_layer.bottom_crop), - (self.crop_layer.left_crop, self.crop_layer.right_crop), - ) - else: - _, output_size_y, output_size_x = self.get_output_shape() - return (0, output_size_y), (0, output_size_x) + _, h, w = self.get_output_shape_after_pooling() + return ( + (self.crop_layer.top_crop, self.crop_layer.bottom_crop), + (self.crop_layer.left_crop, self.crop_layer.right_crop), + ) def get_output_shape(self) -> Tuple[int, int, int]: """Output shape of the layer. @@ -278,14 +273,13 @@ def get_output_shape(self) -> Tuple[int, int, int]: output_size_y = input_size_y // pooling[0] # Compute dims after cropping - if self.crop_layer is not None: - ( - channel_count, - output_size_y, - output_size_x, - ) = self.crop_layer.get_output_shape( - (channel_count, output_size_y, output_size_x) - ) + ( + channel_count, + output_size_y, + output_size_x, + ) = self.crop_layer.get_output_shape( + (channel_count, output_size_y, output_size_x) + ) return channel_count, output_size_y, output_size_x From f9143744c73412c3de7acb8a460759355910413b Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Tue, 5 Nov 2024 15:45:34 +0100 Subject: [PATCH 310/379] Edges handler: Proper checks before merging dvs and pooling layers --- .../backend/dynapcnn/sinabs_edges_handler.py | 68 +++++++++++++------ 1 file changed, 48 insertions(+), 20 deletions(-) diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index fff1f05f..c5a1b87e 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -16,7 +16,7 @@ from .dvs_layer import DVSLayer from .exceptions import InvalidEdge, InvalidGraphStructure from .flipdims import FlipDims -from .utils import Edge +from .utils import Edge, expand_to_pair def fix_dvs_module_edges( @@ -140,7 +140,7 @@ def merge_dvs_pooling_edge( indx_2_module_map: Dict[int, nn.Module], name_2_indx_map: Dict[str, int], ) -> None: - """If a 'dvs-polling' edge existis, the pooling is incorporated into the DVSLayer node if `DVSLayer.pool_layer` has + """If a 'dvs-pooling' edge existis, the pooling is incorporated into the DVSLayer node if `DVSLayer.pool_layer` has default values. All arguments are modified in-place to remove the references to the incorporated pooling node. Parameters @@ -165,42 +165,70 @@ def merge_dvs_pooling_edge( if len(dvs_pool_edge) > 1: # DVSLayer in the original network can have only a single pooling layer liked to it. raise ValueError( - f"DVSLayer can have a single edge onto a pooling layer but multiple were found: {dvs_pool_edge}" + f"DVSLayer has multiple outgoing edges to pooling layers: {dvs_pool_edge}. " + "Unlike convolutional layers, for DVS layers, pooling is set globally for " + "all destinations. Therefore a DVSLayer can be followed by at most one " + "pooling layer." ) - (dvs_idnx, pool_idnx) = dvs_pool_edge[-1] + (dvs_indx, pool_indx) = dvs_pool_edge[-1] + dvs_layer = indx_2_module_map[dvs_indx] + pool_layer = indx_2_module_map[pool_indx] # Checking pooling can be incorporated into the DVSLayer. + if expand_to_pair(dvs_layer.pool_layer.kernel_size) != [1, 1] or expand_to_pair( + dvs_layer.pool_layer.stride + ) != [1, 1]: + raise ValueError( + "DVSLayer with pooling is followed by another pooling layer. " + "This is currently not supported. Please update the network " + "such that all pooling is either done by the DVSLayer or by " + "the following pooling layer." + ) + crop_layer = dvs_layer.crop_layer if ( - indx_2_module_map[dvs_idnx].pool_layer.kernel_size == 1 - and indx_2_module_map[dvs_idnx].pool_layer.stride == 1 + crop_layer.top_crop != 0 + or crop_layer.left_crop != 0 + or crop_layer.bottom_crop != dvs_layer.input_shape[1] + or crop_layer.right_crop != dvs_layer.input_shape[2] ): - # Set DVSLayer.pool to have same config. as the independent pooling layer. - indx_2_module_map[dvs_idnx].pool_layer.kernel_size = indx_2_module_map[ - pool_idnx - ].kernel_size - indx_2_module_map[dvs_idnx].pool_layer.stride = indx_2_module_map[ - pool_idnx - ].stride - # TODO: Should there not be an error be raised if the condition is wrong? + raise ValueError( + "DVSLayer with cropping is followed by a pooling layer. " + "This is currently not supported. Please define pooling " + "directly within the DVSLayer (with the `pool` argument) " + "and remove the pooling layer that follows the DVSLayer" + ) + flip_layer = dvs_layer.flip_layer + if flip_layer.flip_x or flip_layer.flip_y or flip_layer.swap_xy: + raise ValueError( + "DVSLayer with flipping or dimension swapping is followed " + "by a pooling layer. This is currently not supported. " + "Please define pooling directly within the DVSLayer " + "(with the `pool` argument) and remove the pooling " + "layer that follows the DVSLayer" + ) + + # Set DVSLayer.pool to have same config. as the independent pooling layer. + dvs_layer.pool_layer.kernel_size = pool_layer.kernel_size + dvs_layer.pool_layer.stride = pool_layer.stride # Pooling incorporated to the DVSLayer: remove its trace from mappings. - indx_2_module_map.pop(pool_idnx) + indx_2_module_map.pop(pool_indx) name_2_indx_map.pop( - [name for name, indx in name_2_indx_map.items() if indx == pool_idnx][-1] + [name for name, indx in name_2_indx_map.items() if indx == pool_indx][-1] ) # Since pool is part of the DVSLayer we now make edges where pool was a source to have DVSLayer as a source. - for edge in [edge for edge in edges if edge[0] == pool_idnx]: + for edge in [edge for edge in edges if edge[0] == pool_indx]: edges.remove(edge) - edges.update({(dvs_idnx, edge[1])}) + edges.update({(dvs_indx, edge[1])}) # Remove original 'dvs-pool' edge. - edges.remove((dvs_idnx, pool_idnx)) + edges.remove((dvs_indx, pool_indx)) # Checks if any traces of the original pooling node can still be found. if ( - len([edge for edge in edges if (edge[0] == pool_idnx or edge[1] == pool_idnx)]) + len([edge for edge in edges if (edge[0] == pool_indx or edge[1] == pool_indx)]) != 0 ): raise ValueError( From ccfc5d2116ab2affca2a48d5dbcb566acbd86084 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Tue, 5 Nov 2024 16:12:51 +0100 Subject: [PATCH 311/379] Make deepcopy of provided DVSLayers --- .../backend/dynapcnn/nir_graph_extractor.py | 63 ++++++++----------- .../backend/dynapcnn/sinabs_edges_handler.py | 2 +- 2 files changed, 27 insertions(+), 38 deletions(-) diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index 874b2532..b38764ce 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -1,6 +1,7 @@ # author : Willian Soares Girao # contact : wsoaresgirao@gmail.com +from copy import deepcopy from typing import Callable, Dict, Iterable, List, Optional, Set, Tuple, Type, Union import nirtorch @@ -106,8 +107,16 @@ def __init__( ####################################################### Publich Methods ####################################################### @property - def dvs_layer(self) -> Set[int]: - return self._get_dvs_layer() + def dvs_layer(self) -> Union[DVSLayer, None]: + idx = self.dvs_layer_index + if idx is None: + return None + else: + return self.indx_2_module_map[self.dvs_layer_index] + + @property + def dvs_layer_index(self) -> Union[int, None]: + return self._get_dvs_layer_index() @property def entry_nodes(self) -> Set[int]: @@ -118,7 +127,7 @@ def edges(self) -> Set[Edge]: return {(src, tgt) for src, tgt in self._edges} @property - def has_dvs_layer(self) -> Set[Edge]: + def has_dvs_layer(self) -> bool: return self.dvs_layer is not None @property @@ -295,8 +304,12 @@ def _handle_dvs_input( the model is considered to be using DVS input only if the graph contains a `DVSLayer`. """ - # If DVS camera is wanted but `spiking_model` does not start with DVS layer. - if self._need_dvs_node(dvs_input): + if self.has_dvs_layer: + # Make a copy of the layer so that the original version is not + # change in place + new_dvs_layer = deepcopy(self.dvs_layer) + self.name_2_indx_map[self.dvs_layer_index] = new_dvs_layer + elif dvs_input: # Insert a DVSLayer node in the graph. self._add_dvs_node(dvs_input_shape=input_shape) @@ -352,33 +365,9 @@ def _add_dvs_node(self, dvs_input_shape: Tuple[int, int, int]) -> None: # DVSLayer node becomes the only entrypoint of the graph. self._entry_nodes = {self._name_2_indx_map["dvs"]} - def _need_dvs_node(self, dvs_input: Optional[bool] = None) -> bool: - """Returns whether or not a node will need to be added to represent a - `DVSLayer` instance. - - A new node will have to be added if `self._indx_2_module_map` contains no - `DVSLayer` instance and `dvs_input == True`. - - Parameters - ---------- - - dvs_input (bool): optional (default as `None`). Wether or not dynapcnn receive input from its DVS camera. - Returns - ------- - - True if `self._indx_2_module_map` contains a DVSLayer, False otherwise. - """ - - has_dvs_layer = self.has_dvs_layer - - # Checks if DVSLayer instance exists but user has set 'dvs_input' to False. - if ( - has_dvs_layer and dvs_input == False - ): # `== False` ensures that `None` is not considered - raise InvalidModelWithDVSSetup() - - return not has_dvs_layer and dvs_input - - def _get_dvs_layer(self) -> Union[DVSLayer, None]: - """Loops though all modules and return `DVSLayer` instance if it exists. + def _get_dvs_layer_index(self) -> Union[int, None]: + """Loop though all modules and return index of `DVSLayer` + instance if it exists. Returns ------- @@ -390,16 +379,16 @@ def _get_dvs_layer(self) -> Union[DVSLayer, None]: """ - dvs_layers = { - module - for module in self._indx_2_module_map.values() + dvs_layer_indices = { + index + for index, module in self._indx_2_module_map.items() if isinstance(module, DVSLayer) } - if (num_dvs := len(dvs_layers)) == 0: + if (num_dvs := len(dvs_layer_indices)) == 0: return elif num_dvs == 1: - return dvs_layers.pop() + return dvs_layer_indices.pop() else: raise InvalidGraphStructure( f"The provided model has {num_dvs} `DVSLayer`s. At most one is allowed." diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index c5a1b87e..43979bc1 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -6,7 +6,7 @@ """ from collections import deque -from typing import Dict, List, Optional, Set, Tuple, Type, Union +from typing import Dict, List, Set, Tuple, Type, Union from sinabs.layers import SumPool2d from torch import Size, nn From 22edb5fc38cc6d8c1454fbdde4ee70a02dad4246 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Tue, 5 Nov 2024 16:42:27 +0100 Subject: [PATCH 312/379] Reduce code redundancy in batch norm merging --- sinabs/backend/dynapcnn/utils.py | 74 +++++++++++++++++--------------- 1 file changed, 39 insertions(+), 35 deletions(-) diff --git a/sinabs/backend/dynapcnn/utils.py b/sinabs/backend/dynapcnn/utils.py index 24144c50..10ce2e34 100644 --- a/sinabs/backend/dynapcnn/utils.py +++ b/sinabs/backend/dynapcnn/utils.py @@ -1,6 +1,6 @@ from collections import defaultdict, deque from copy import deepcopy -from typing import TYPE_CHECKING, List, Optional, Set, Tuple +from typing import TYPE_CHECKING, List, Optional, Set, Tuple, TypeVar, Union import sinabs.layers as sl import torch @@ -250,20 +250,20 @@ def construct_dvs_layer( # No parameters/layers pertaining to DVS preprocessing found return None, 0, 1 - -def merge_conv_bn(conv, bn): - """Merge a convolutional layer with subsequent batch normalization. +WeightLayer = TypeVar("WeightLayer", nn.Linear, nn.Conv2d) +def merge_bn(weight_layer: WeightLayer, bn: Union[nn.BatchNorm1d, nn.BatchNorm2d]) -> WeightLayer: + """Merge a convolutional or linear layer with subsequent batch normalization. Parameters ---------- - conv: torch.nn.Conv2d - Convolutional layer - bn: torch.nn.Batchnorm2d + weight_layer: torch.nn.Conv2d or nn.Linear + Convolutional or linear layer + bn: torch.nn.Batchnorm2d or nn.Batchnorm1d Batch normalization Returns ------- - torch.nn.Conv2d: Convolutional layer including batch normalization + Weight layer including batch normalization """ mu = bn.running_mean sigmasq = bn.running_var @@ -275,16 +275,39 @@ def merge_conv_bn(conv, bn): factor = gamma / sigmasq.sqrt() - c_weight = conv.weight.data.clone().detach() - c_bias = 0.0 if conv.bias is None else conv.bias.data.clone().detach() + weight = weight_layer.weight.data.clone().detach() + bias = 0.0 if weight_layer.bias is None else weight_layer.bias.data.clone().detach() + + weight_layer = deepcopy(weight_layer) + + new_bias = beta + (bias - mu) * factor + if weight_layer.bias is None: + weight_layer.bias = nn.Parameter(new_bias) + else: + weight_layer.bias.data = new_bias + + for __ in range(weight_layer.weight.ndim - factor.ndim): + factor.unsqueeze_(-1) + weight_layer.weight.data = weight * factor + + return weight_layer - conv = deepcopy(conv) # TODO: this will cause copying twice - conv.weight.data = c_weight * factor[:, None, None, None] - if conv.bias: - conv.bias.data = beta + (c_bias - mu) * factor +def merge_conv_bn(conv: nn.Conv2d, bn: nn.BatchNorm2d) -> nn.Conv2d: + """Merge a convolutional layer with subsequent batch normalization. + + Parameters + ---------- + conv: torch.nn.Conv2d + Convolutional layer + bn: torch.nn.Batchnorm2d + Batch normalization - return conv + Returns + ------- + torch.nn.Conv2d: Convolutional layer including batch normalization + """ + return merge_bn(conv, bn) def merge_linear_bn(linear: nn.Linear, bn: nn.BatchNorm1d) -> nn.Linear: """Merge a linear (fully connected) layer with subsequent batch normalization. @@ -300,26 +323,7 @@ def merge_linear_bn(linear: nn.Linear, bn: nn.BatchNorm1d) -> nn.Linear: ------- torch.nn.Linear: Linear layer including batch normalization """ - mu = bn.running_mean - sigmasq = bn.running_var - - if bn.affine: - gamma, beta = bn.weight, bn.bias - else: - gamma, beta = 1.0, 0.0 - - factor = gamma / sigmasq.sqrt() - - l_weight = linear.weight.data.clone().detach() - l_bias = 0.0 if linear.bias is None else linear.bias.data.clone().detach() - - linear = deepcopy(linear) - - linear.weight.data = l_weight * factor[:, None] - if linear.bias is not None: - linear.bias.data = beta + (l_bias - mu) * factor - - return linear + return merge_bn(linear, bn) # Should become obsolete def construct_next_pooling_layer( From 476492351e511950cbe710c89bebd7690bd03981 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Tue, 5 Nov 2024 16:53:14 +0100 Subject: [PATCH 313/379] Blacken edges handler. Fix import in graph extractor --- .../backend/dynapcnn/nir_graph_extractor.py | 4 +- .../backend/dynapcnn/sinabs_edges_handler.py | 138 ++++++++++-------- 2 files changed, 79 insertions(+), 63 deletions(-) diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index 3c4ad121..21c2b331 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -16,11 +16,11 @@ from .dvs_layer import DVSLayer from .dynapcnn_layer_utils import construct_dynapcnnlayers_from_mapper from .dynapcnnnetwork_module import DynapcnnNetworkModule -from .exceptions import InvalidGraphStructure, InvalidModelWithDVSSetup +from .exceptions import InvalidGraphStructure from .sinabs_edges_handler import ( collect_dynapcnn_layer_info, fix_dvs_module_edges, - handle_batchnorm_edges, + handle_batchnorm_nodes, merge_dvs_pooling_edge, ) from .utils import Edge, topological_sorting diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index 737486a4..b6597c42 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -16,16 +16,18 @@ from .dvs_layer import DVSLayer from .exceptions import InvalidEdge, InvalidGraphStructure from .flipdims import FlipDims -from .utils import Edge, expand_to_pair, merge_conv_bn, merge_linear_bn +from .utils import Edge, expand_to_pair, merge_bn -def remap_edges_after_drop(dropped_node: int, source_of_dropped_node: int, edges: Set[Edge]) -> Set[Edge]: - """ Creates a new set of edges from `edges`. All edges where `dropped_node` is the source node will be used to generate +def remap_edges_after_drop( + dropped_node: int, source_of_dropped_node: int, edges: Set[Edge] +) -> Set[Edge]: + """Creates a new set of edges from `edges`. All edges where `dropped_node` is the source node will be used to generate a new edge where `source_of_dropped_node` becomes the source node (the target is kept). Parameters ---------- - - dropped_node (int): + - dropped_node (int): - source_of_dropped_node (int): - edges (set): tuples describing the connections between layers in `spiking_model`. @@ -35,77 +37,91 @@ def remap_edges_after_drop(dropped_node: int, source_of_dropped_node: int, edges """ remapped_edges = set() - for (src, tgt) in edges: + for src, tgt in edges: if src == dropped_node: remapped_edges.add((source_of_dropped_node, tgt)) return remapped_edges -def handle_batchnorm_nodes(edges: Set[Edge], indx_2_module_map: Dict[int, nn.Module], name_2_indx_map: Dict[str, int]) -> None: - """ Merges `BatchNorm2d`/`BatchNorm1d` layers into `Conv2d`/`Linear` ones. The batch norm nodes will be removed from the graph (by updating all variables - passed as arguments in-place) after their properties are used to re-scale the weights of the convolutional/linear layers associated with batch - normalization via the `weight-batchnorm` edges found in the original graph. - - Parameters - ---------- - - edges (set): tuples describing the connections between layers in `spiking_model`. - - indx_2_module_map (dict): the mapping between a node (`key` as an `int`) and its module (`value` as a `nn.Module`). - - name_2_indx_map (dict): Map from node names to unique indices. - """ - - # Gather indexes of the BatchNorm2d/BatchNorm1d nodes. - bnorm_nodes = { - index for index, module in indx_2_module_map.items() - if isinstance(module, nn.BatchNorm2d) or isinstance(module, nn.BatchNorm1d) - } - if len(bnorm_nodes) == 0: - # There are no edges with batch norm - nothing to do here. - return +def handle_batchnorm_nodes( + edges: Set[Edge], + indx_2_module_map: Dict[int, nn.Module], + name_2_indx_map: Dict[str, int], +) -> None: + """Merges `BatchNorm2d`/`BatchNorm1d` layers into `Conv2d`/`Linear` ones. The batch norm nodes will be removed from the graph (by updating all variables + passed as arguments in-place) after their properties are used to re-scale the weights of the convolutional/linear layers associated with batch + normalization via the `weight-batchnorm` edges found in the original graph. - # Find weight-bnorm edges. - weight_bnorm_edges = { - (src, tgt) for (src, tgt) in edges - if (isinstance(indx_2_module_map[src], nn.Conv2d) and isinstance(indx_2_module_map[tgt], nn.BatchNorm2d)) or (isinstance(indx_2_module_map[src], nn.Linear) and isinstance(indx_2_module_map[tgt], nn.BatchNorm1d)) - } + Parameters + ---------- + - edges (set): tuples describing the connections between layers in `spiking_model`. + - indx_2_module_map (dict): the mapping between a node (`key` as an `int`) and its module (`value` as a `nn.Module`). + - name_2_indx_map (dict): Map from node names to unique indices. + """ + + # Gather indexes of the BatchNorm2d/BatchNorm1d nodes. + bnorm_nodes = { + index + for index, module in indx_2_module_map.items() + if isinstance(module, (nn.BatchNorm2d, nn.BatchNorm1d)) + } + + if len(bnorm_nodes) == 0: + # There are no edges with batch norm - nothing to do here. + return - # Merge conv/linear and bnorm layers using 'weight-bnorm' edges. - for edge in weight_bnorm_edges: - bnorm = indx_2_module_map[edge[1]] - weight = indx_2_module_map[edge[0]] - - # merge and update weight node. - if isinstance(weight, nn.Conv2d): - indx_2_module_map[edge[0]] = merge_conv_bn(weight, bnorm) - elif isinstance(weight, nn.Linear): - indx_2_module_map[edge[0]] = merge_linear_bn(weight, bnorm) - else: - raise ValueError(f'A batch norm layer can only be preceed by either a nn.Conv2d (followed by nn.BatchNorm2d) or a nn.Linear (followed by nn.BatchNorm1d).\nFound a {type(weight)} followed by a {type(bnorm)}.') - - # Point weight nodes to the targets of their respective batch norm nodes. - new_edges = set() - for weight, bnorm in weight_bnorm_edges: - new_edges.update( - remap_edges_after_drop(dropped_node=bnorm, source_of_dropped_node=weight, edges=edges) + # Find weight-bnorm edges. + weight_bnorm_edges = { + (src, tgt) + for (src, tgt) in edges + if ( + isinstance(indx_2_module_map[src], nn.Conv2d) + and isinstance(indx_2_module_map[tgt], nn.BatchNorm2d) + ) + or ( + isinstance(indx_2_module_map[src], nn.Linear) + and isinstance(indx_2_module_map[tgt], nn.BatchNorm1d) + ) + } + + # Merge conv/linear and bnorm layers using 'weight-bnorm' edges. + for edge in weight_bnorm_edges: + bnorm = indx_2_module_map[edge[1]] + weight = indx_2_module_map[edge[0]] + + # merge and update weight node. + indx_2_module_map[edge[0]] = merge_bn(weight, bnorm) + + # Point weight nodes to the targets of their respective batch norm nodes. + new_edges = set() + for weight, bnorm in weight_bnorm_edges: + new_edges.update( + remap_edges_after_drop( + dropped_node=bnorm, source_of_dropped_node=weight, edges=edges ) + ) + + # Remove references to the bnorm node. - # Remove references to the bnorm node. + for idx in bnorm_nodes: + indx_2_module_map.pop(idx) + + for name in [name for name, indx in name_2_indx_map.items() if indx in bnorm_nodes]: + name_2_indx_map.pop(name) - for idx in bnorm_nodes: - indx_2_module_map.pop(idx) - - for name in [name for name, indx in name_2_indx_map.items() if indx in bnorm_nodes]: - name_2_indx_map.pop(name) + for edge in weight_bnorm_edges: + edges.remove(edge) - for edge in weight_bnorm_edges: - edges.remove(edge) + for edge in [ + (src, tgt) for (src, tgt) in edges if (src in bnorm_nodes or tgt in bnorm_nodes) + ]: + edges.remove(edge) - for edge in [(src, tgt) for (src, tgt) in edges if (src in bnorm_nodes or tgt in bnorm_nodes)]: - edges.remove(edge) + # Update 'edges' in-place to incorporate new edges: + for edge in new_edges: + edges.add(edge) - # Update 'edges' in-place to incorporate new edges: - for edge in new_edges: - edges.add(edge) def fix_dvs_module_edges( edges: Set[Edge], From 799e2c1cb6861ac467c76a2879a6ac628f289349 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Tue, 5 Nov 2024 16:53:36 +0100 Subject: [PATCH 314/379] Run black on tests --- .../test_compatible_layer_build.py | 29 ++++++++++--------- tests/test_dynapcnn/test_large_net.py | 1 + .../test_dynapcnnlayer/test_dynapcnnlayer.py | 13 +++++---- .../conftest_dynapcnnnetwork.py | 8 ++++- tests/test_dynapcnnnetwork/model_dummy_seq.py | 4 +-- .../test_dynapcnnnetwork.py | 22 +++++++------- 6 files changed, 44 insertions(+), 33 deletions(-) diff --git a/tests/test_dynapcnn/test_compatible_layer_build.py b/tests/test_dynapcnn/test_compatible_layer_build.py index 8259cef8..6efe637e 100644 --- a/tests/test_dynapcnn/test_compatible_layer_build.py +++ b/tests/test_dynapcnn/test_compatible_layer_build.py @@ -3,20 +3,23 @@ import sinabs.layers as sl + @pytest.mark.parametrize( - ("pooling", "layer_type", "expected_pooling", "expected_scaling"), - [ - (2, sl.SumPool2d, [2, 2], 1), - ((2, 2), sl.SumPool2d, [2, 2], 1), - (3, sl.SumPool2d, [3, 3], 1), - ((4, 4), sl.SumPool2d, [4, 4], 1), - (2, nn.AvgPool2d, [2, 2], 1./4), - ((2, 2), nn.AvgPool2d, [2, 2], 1./4), - (3, nn.AvgPool2d, [3, 3], 1./9), - ((4, 4), nn.AvgPool2d, [4, 4], 1./16), - ] + ("pooling", "layer_type", "expected_pooling", "expected_scaling"), + [ + (2, sl.SumPool2d, [2, 2], 1), + ((2, 2), sl.SumPool2d, [2, 2], 1), + (3, sl.SumPool2d, [3, 3], 1), + ((4, 4), sl.SumPool2d, [4, 4], 1), + (2, nn.AvgPool2d, [2, 2], 1.0 / 4), + ((2, 2), nn.AvgPool2d, [2, 2], 1.0 / 4), + (3, nn.AvgPool2d, [3, 3], 1.0 / 9), + ((4, 4), nn.AvgPool2d, [4, 4], 1.0 / 16), + ], ) -def test_construct_pooling_from_1_layer(pooling, layer_type, expected_pooling, expected_scaling): +def test_construct_pooling_from_1_layer( + pooling, layer_type, expected_pooling, expected_scaling +): layers = [layer_type(pooling)] from sinabs.backend.dynapcnn.dynapcnn_layer_utils import consolidate_dest_pooling @@ -35,7 +38,7 @@ def test_construct_pooling_from_2_layers(): cumulative_pooling, scaling = consolidate_dest_pooling(layers) assert cumulative_pooling == [6, 6] - assert scaling == 1./9 + assert scaling == 1.0 / 9 # TODO: Move these fail cases to another test. Rename this file or move other tests as well. diff --git a/tests/test_dynapcnn/test_large_net.py b/tests/test_dynapcnn/test_large_net.py index 6c366b52..c8ea5190 100644 --- a/tests/test_dynapcnn/test_large_net.py +++ b/tests/test_dynapcnn/test_large_net.py @@ -119,6 +119,7 @@ def test_was_copied(): spk_lyr_snn = snn_layers[spk_name] assert spk_lyr_dynapcnn is not spk_lyr_snn + def test_make_config(): dynapcnn_net = DynapcnnNetwork( snn, input_shape=input_shape, discretize=False, dvs_input=False diff --git a/tests/test_dynapcnnlayer/test_dynapcnnlayer.py b/tests/test_dynapcnnlayer/test_dynapcnnlayer.py index 53fdcf83..16d88822 100644 --- a/tests/test_dynapcnnlayer/test_dynapcnnlayer.py +++ b/tests/test_dynapcnnlayer/test_dynapcnnlayer.py @@ -20,8 +20,13 @@ def test_DynapcnnLayer(dcnnl_map, discretize, expected_output): """ # create a `DynapcnnLayer` from the set of layers in `nodes_to_dcnnl_map[dpcnnl_idx]`. - dynapcnn_layers, destination_map, entry_points = construct_dynapcnnlayers_from_mapper( - dcnnl_map=dcnnl_map, discretize=discretize, rescale_fn=None, dvs_layer_info=None, + dynapcnn_layers, destination_map, entry_points = ( + construct_dynapcnnlayers_from_mapper( + dcnnl_map=dcnnl_map, + discretize=discretize, + rescale_fn=None, + dvs_layer_info=None, + ) ) for layer_index, dynapcnn_layer in dynapcnn_layers.items(): @@ -62,6 +67,4 @@ def test_DynapcnnLayer(dcnnl_map, discretize, expected_output): ), "wrong destination map" # Test entry point - assert ( - entry_points == expected_output["entry_points"] - ), "wrong entry points" + assert entry_points == expected_output["entry_points"], "wrong entry points" diff --git a/tests/test_dynapcnnnetwork/conftest_dynapcnnnetwork.py b/tests/test_dynapcnnnetwork/conftest_dynapcnnnetwork.py index 818b1d90..ccd7d4ef 100644 --- a/tests/test_dynapcnnnetwork/conftest_dynapcnnnetwork.py +++ b/tests/test_dynapcnnnetwork/conftest_dynapcnnnetwork.py @@ -18,7 +18,13 @@ from .model_dummy_4 import input_shape as input_shape_4 from .model_dummy_4 import snn as snn_4 from .model_dummy_4 import snn as snn_4 -from .model_dummy_seq import input_shape_seq, seq_1, seq_2, expected_seq_1, expected_seq_2 +from .model_dummy_seq import ( + input_shape_seq, + seq_1, + seq_2, + expected_seq_1, + expected_seq_2, +) args_DynapcnnNetworkTest = [ (snn_1, input_shape_1, batch_size_1, expected_output_1), diff --git a/tests/test_dynapcnnnetwork/model_dummy_seq.py b/tests/test_dynapcnnnetwork/model_dummy_seq.py index f210181c..02663e30 100644 --- a/tests/test_dynapcnnnetwork/model_dummy_seq.py +++ b/tests/test_dynapcnnnetwork/model_dummy_seq.py @@ -26,7 +26,7 @@ nn.Conv2d(8, 2, kernel_size=3, stride=1, bias=False), IAFSqueeze(batch_size=1), nn.Flatten(), - nn.Linear(3*3*2, 5), + nn.Linear(3 * 3 * 2, 5), nn.Identity(), IAFSqueeze(batch_size=1), ) @@ -49,7 +49,7 @@ "entry_points": {0}, } -expected_seq_2= { +expected_seq_2 = { "dcnnl_edges": { (0, 1), (1, 2), diff --git a/tests/test_dynapcnnnetwork/test_dynapcnnnetwork.py b/tests/test_dynapcnnnetwork/test_dynapcnnnetwork.py index a31248dd..6c79e03f 100644 --- a/tests/test_dynapcnnnetwork/test_dynapcnnnetwork.py +++ b/tests/test_dynapcnnnetwork/test_dynapcnnnetwork.py @@ -30,24 +30,22 @@ def test_DynapcnnNetwork(snn, input_shape, batch_size, expected_output): ), "wrong list of edges describing DynapcnnLayer connectivity." # Convert source lists to sets to ignore order - source_map = {node: set(sources) for node, sources in module._node_source_map.items()} - assert( - expected_output["node_source_map"] == source_map - ), "wrong node source map" + source_map = { + node: set(sources) for node, sources in module._node_source_map.items() + } + assert expected_output["node_source_map"] == source_map, "wrong node source map" # Convert destination lists to sets to ignore order - destination_map = {node: set(dests) for node, dests in module._destination_map.items()} - assert( + destination_map = { + node: set(dests) for node, dests in module._destination_map.items() + } + assert ( expected_output["destination_map"] == destination_map ), "wrong destination map" - assert( - expected_output["entry_points"] == module._entry_points - ), "wrong entry points" + assert expected_output["entry_points"] == module._entry_points, "wrong entry points" - assert( - expected_output["sorted_nodes"] == module._sorted_nodes - ), "wrong node sorting" + assert expected_output["sorted_nodes"] == module._sorted_nodes, "wrong node sorting" assert ( expected_output["output_shape"] == output.shape From 0a45350ec132c05ecf2e972513cdcd9ae7ff367b Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Tue, 5 Nov 2024 16:53:57 +0100 Subject: [PATCH 315/379] Run Black --- sinabs/backend/dynapcnn/nir_graph_extractor.py | 5 +++-- sinabs/backend/dynapcnn/utils.py | 9 ++++++++- sinabs/utils.py | 7 ++++--- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index 21c2b331..f8794d82 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -94,7 +94,9 @@ def __init__( self._indx_2_module_map = self._get_named_modules(spiking_model) # Merges BatchNorm2d/BatchNorm1d nodes with Conv2d/Linear ones. - handle_batchnorm_nodes(self._edges, self._indx_2_module_map, self._name_2_indx_map) + handle_batchnorm_nodes( + self._edges, self._indx_2_module_map, self._name_2_indx_map + ) # Determine entry points to graph self._entry_nodes = self._get_entry_nodes(self._edges) @@ -108,7 +110,6 @@ def __init__( # retrieves what the I/O shape for each node's module is. self._nodes_io_shapes = self._get_nodes_io_shapes(dummy_input) - ####################################################### Publich Methods ####################################################### @property diff --git a/sinabs/backend/dynapcnn/utils.py b/sinabs/backend/dynapcnn/utils.py index 10ce2e34..b6112cdf 100644 --- a/sinabs/backend/dynapcnn/utils.py +++ b/sinabs/backend/dynapcnn/utils.py @@ -250,8 +250,13 @@ def construct_dvs_layer( # No parameters/layers pertaining to DVS preprocessing found return None, 0, 1 + WeightLayer = TypeVar("WeightLayer", nn.Linear, nn.Conv2d) -def merge_bn(weight_layer: WeightLayer, bn: Union[nn.BatchNorm1d, nn.BatchNorm2d]) -> WeightLayer: + + +def merge_bn( + weight_layer: WeightLayer, bn: Union[nn.BatchNorm1d, nn.BatchNorm2d] +) -> WeightLayer: """Merge a convolutional or linear layer with subsequent batch normalization. Parameters @@ -309,6 +314,7 @@ def merge_conv_bn(conv: nn.Conv2d, bn: nn.BatchNorm2d) -> nn.Conv2d: """ return merge_bn(conv, bn) + def merge_linear_bn(linear: nn.Linear, bn: nn.BatchNorm1d) -> nn.Linear: """Merge a linear (fully connected) layer with subsequent batch normalization. @@ -325,6 +331,7 @@ def merge_linear_bn(linear: nn.Linear, bn: nn.BatchNorm1d) -> nn.Linear: """ return merge_bn(linear, bn) + # Should become obsolete def construct_next_pooling_layer( layers: List[nn.Module], idx_start: int diff --git a/sinabs/utils.py b/sinabs/utils.py index c450a38b..8a294697 100644 --- a/sinabs/utils.py +++ b/sinabs/utils.py @@ -287,8 +287,10 @@ def expand_to_pair(value) -> Tuple[int, int]: T = TypeVar("T") + + def collapse_pair(pair: Union[Iterable[T], T]) -> T: - """ Collapse an iterable of equal elements by returning only the first + """Collapse an iterable of equal elements by returning only the first Parameters ---------- @@ -300,7 +302,7 @@ def collapse_pair(pair: Union[Iterable[T], T]) -> T: Raises ------ - ValueError if not all elements in `pair` are equal. + ValueError if not all elements in `pair` are equal. """ if isinstance(pair, Iterable): items = [x for x in pair] @@ -309,4 +311,3 @@ def collapse_pair(pair: Union[Iterable[T], T]) -> T: return items[0] else: return pair - From 5139f1c36b5ef0ab9a8cf2c89273e9bac40e301c Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Tue, 5 Nov 2024 17:32:47 +0100 Subject: [PATCH 316/379] Remove outdated functions --- .../backend/dynapcnn/sinabs_edges_handler.py | 3 +- sinabs/backend/dynapcnn/utils.py | 339 +----------------- 2 files changed, 3 insertions(+), 339 deletions(-) diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index b6597c42..dac2ccbd 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -9,6 +9,7 @@ from typing import Dict, List, Set, Tuple, Type, Union from sinabs.layers import SumPool2d +from sinabs.utils import expand_to_pair from torch import Size, nn from .connectivity_specs import VALID_SINABS_EDGE_TYPES, Pooling @@ -16,7 +17,7 @@ from .dvs_layer import DVSLayer from .exceptions import InvalidEdge, InvalidGraphStructure from .flipdims import FlipDims -from .utils import Edge, expand_to_pair, merge_bn +from .utils import Edge, merge_bn def remap_edges_after_drop( diff --git a/sinabs/backend/dynapcnn/utils.py b/sinabs/backend/dynapcnn/utils.py index b6112cdf..9514ba46 100644 --- a/sinabs/backend/dynapcnn/utils.py +++ b/sinabs/backend/dynapcnn/utils.py @@ -1,15 +1,12 @@ from collections import defaultdict, deque from copy import deepcopy -from typing import TYPE_CHECKING, List, Optional, Set, Tuple, TypeVar, Union +from typing import TYPE_CHECKING, List, Set, Tuple, TypeVar, Union import sinabs.layers as sl import torch import torch.nn as nn -from sinabs.utils import expand_to_pair from .crop2d import Crop2d -from .dvs_layer import DVSLayer -from .flipdims import FlipDims if TYPE_CHECKING: from sinabs.backend.dynapcnn.dynapcnn_network import DynapcnnNetwork @@ -131,10 +128,6 @@ def topological_sorting(edges: Set[Tuple[int, int]]) -> List[int]: raise ValueError("The graph has a cycle and cannot be topologically sorted.") -####################################################### MISSING FUNCTIONALITY ####################################################### -# TODO: these methods are currently not used by the new implementation of DynapcnnNetwork (but should). - - def convert_cropping2dlayer_to_crop2d( layer: sl.Cropping2dLayer, input_shape: Tuple[int, int] ) -> Crop2d: @@ -160,97 +153,6 @@ def convert_cropping2dlayer_to_crop2d( return Crop2d(((top, bottom), (left, right))) -def construct_dvs_layer( - layers: List[nn.Module], - input_shape: Tuple[int, int, int], - idx_start: int = 0, - dvs_input: bool = False, -) -> Tuple[Optional[DVSLayer], int, float]: - """ - Generate a DVSLayer given a list of layers. If `layers` does not start - with a pooling, cropping or flipping layer and `dvs_input` is False, - will return `None` instead of a DVSLayer. - NOTE: The number of channels is implicitly assumed to be 2 because of DVS - - Parameters - ---------- - layers: - List of layers - input_shape: - Shape of input (channels, height, width) - idx_start: - Starting index to scan the list. Default 0 - - Returns - ------- - dvs_layer: - None or DVSLayer - idx_next: int or None - Index of first layer after this layer is constructed - rescale_factor: float - Rescaling factor needed when turning AvgPool to SumPool. May - differ from the pooling kernel in certain cases. - dvs_input: bool - Whether DVSLayer should have pixel array activated. - """ - # Start with defaults - layer_idx_next = idx_start - crop_lyr = None - flip_lyr = None - - if len(input_shape) != 3: - raise ValueError( - f"Input shape should be 3 dimensional but input_shape={input_shape} was given." - ) - - # Return existing DVS layer as is - if len(layers) and isinstance(layers[0], DVSLayer): - return deepcopy(layers[0]), 1, 1 - - # Construct pooling layer - pool_lyr, layer_idx_next, rescale_factor = construct_next_pooling_layer( - layers, layer_idx_next - ) - - # Find next layer (check twice for two layers) - for __ in range(2): - # Go to the next layer - if layer_idx_next < len(layers): - layer = layers[layer_idx_next] - else: - break - # Check layer type - if isinstance(layer, sl.Cropping2dLayer): - # The shape after pooling is - pool = expand_to_pair(pool_lyr.kernel_size) - h = input_shape[1] // pool[0] - w = input_shape[2] // pool[1] - print(f"Input shape to the cropping layer is {h}, {w}") - crop_lyr = convert_cropping2dlayer_to_crop2d(layer, (h, w)) - elif isinstance(layer, Crop2d): - crop_lyr = layer - elif isinstance(layer, FlipDims): - flip_lyr = layer - else: - break - - layer_idx_next += 1 - - # If any parameters have been found or dvs_input is True - if (layer_idx_next > 0) or dvs_input: - dvs_layer = DVSLayer.from_layers( - pool_layer=pool_lyr, - crop_layer=crop_lyr, - flip_layer=flip_lyr, - input_shape=input_shape, - disable_pixel_array=not dvs_input, - ) - return dvs_layer, layer_idx_next, rescale_factor - else: - # No parameters/layers pertaining to DVS preprocessing found - return None, 0, 1 - - WeightLayer = TypeVar("WeightLayer", nn.Linear, nn.Conv2d) @@ -332,72 +234,6 @@ def merge_linear_bn(linear: nn.Linear, bn: nn.BatchNorm1d) -> nn.Linear: return merge_bn(linear, bn) -# Should become obsolete -def construct_next_pooling_layer( - layers: List[nn.Module], idx_start: int -) -> Tuple[Optional[sl.SumPool2d], int, float]: - """Consolidate the first `AvgPool2d` objects in `layers` until the first object of different - type. - - Parameters - ---------- - layers: Sequence of layer objects - Contains `AvgPool2d` and other objects. - idx_start: int - Layer index to start construction from - Returns - ------- - lyr_pool: int or tuple of ints - Consolidated pooling size. - idx_next: int - Index of first object in `layers` that is not a `AvgPool2d`, - rescale_factor: float - Rescaling factor needed when turning AvgPool to SumPool. May - differ from the pooling kernel in certain cases. - """ - - rescale_factor = 1 - cumulative_pooling = expand_to_pair(1) - idx_next = idx_start - # Figure out pooling dims - while idx_next < len(layers): - lyr = layers[idx_next] - if isinstance(lyr, nn.AvgPool2d): - if lyr.padding != 0: - raise ValueError("Padding is not supported for the pooling layers") - elif isinstance(lyr, sl.SumPool2d): - ... - else: - # Reached a non pooling layer - break - # Increment if it is a pooling layer - idx_next += 1 - - pooling = expand_to_pair(lyr.kernel_size) - if lyr.stride is not None: - stride = expand_to_pair(lyr.stride) - if pooling != stride: - raise ValueError( - f"Stride length {lyr.stride} should be the same as pooling kernel size {lyr.kernel_size}" - ) - # Compute cumulative pooling - cumulative_pooling = ( - cumulative_pooling[0] * pooling[0], - cumulative_pooling[1] * pooling[1], - ) - - # Update rescaling factor - if isinstance(lyr, nn.AvgPool2d): - rescale_factor *= pooling[0] * pooling[1] - - # If there are no layers - if cumulative_pooling == (1, 1): - return None, idx_next, 1 - else: - lyr_pool = sl.SumPool2d(cumulative_pooling) - return lyr_pool, idx_next, rescale_factor - - def extend_readout_layer(model: "DynapcnnNetwork") -> "DynapcnnNetwork": """Return a copied and extended model with the readout layer extended to 4 times the number of output channels. For Speck 2E and 2F, to get readout with correct output index, we need to @@ -485,176 +321,3 @@ def extend_readout_layer(model: "DynapcnnNetwork") -> "DynapcnnNetwork": # return input_shape # else: # raise InputConfigurationError("No input shape could be inferred") - -# def construct_next_dynapcnn_layer( -# layers: List[nn.Module], -# idx_start: int, -# in_shape: Tuple[int, int, int], -# discretize: bool, -# rescale_factor: float = 1, -# ) -> Tuple[DynapcnnLayer, int, float]: -# """Generate a DynapcnnLayer from a Conv2d layer and its subsequent spiking and pooling layers. - -# Parameters -# ---------- - -# layers: sequence of layer objects -# First object must be Conv2d, next must be an IAF layer. All pooling -# layers that follow immediately are consolidated. Layers after this -# will be ignored. -# idx_start: -# Layer index to start construction from -# in_shape: tuple of integers -# Shape of the input to the first layer in `layers`. Convention: -# (input features, height, width) -# discretize: bool -# Discretize weights and thresholds if True -# rescale_factor: float -# Weights of Conv2d layer are scaled down by this factor. Can be -# used to account for preceding average pooling that gets converted -# to sum pooling. - -# Returns -# ------- -# dynapcnn_layer: DynapcnnLayer -# DynapcnnLayer -# layer_idx_next: int -# Index of the next layer after this layer is constructed -# rescale_factor: float -# rescaling factor to account for average pooling -# """ -# layer_idx_next = idx_start # Keep track of layer indices - -# # Check that the first layer is Conv2d, or Linear -# if not isinstance(layers[layer_idx_next], (nn.Conv2d, nn.Linear)): -# raise UnexpectedLayer(nn.Conv2d, layers[layer_idx_next]) - -# # Identify and consolidate conv layer -# lyr_conv = layers[layer_idx_next] -# layer_idx_next += 1 -# if layer_idx_next >= len(layers): -# raise MissingLayer(layer_idx_next) -# # Check and consolidate batch norm -# if isinstance(layers[layer_idx_next], nn.BatchNorm2d): -# lyr_conv = merge_conv_bn(lyr_conv, layers[layer_idx_next]) -# layer_idx_next += 1 - -# # Check next layer exists -# try: -# lyr_spk = layers[layer_idx_next] -# layer_idx_next += 1 -# except IndexError: -# raise MissingLayer(layer_idx_next) - -# # Check that the next layer is spiking -# # TODO: Check that the next layer is an IAF layer -# if not isinstance(lyr_spk, sl.IAF): -# raise TypeError( -# f"Convolution must be followed by IAF spiking layer, found {type(lyr_spk)}" -# ) - -# # Check for next pooling layer -# lyr_pool, i_next, rescale_factor_after_pooling = construct_next_pooling_layer( -# layers, layer_idx_next -# ) -# # Increment layer index to after the pooling layers -# layer_idx_next = i_next - -# # Compose DynapcnnLayer -# dynapcnn_layer = DynapcnnLayer( -# conv=lyr_conv, -# spk=lyr_spk, -# pool=lyr_pool, -# in_shape=in_shape, -# discretize=discretize, -# rescale_weights=rescale_factor, -# ) - -# return dynapcnn_layer, layer_idx_next, rescale_factor_after_pooling - - -# def build_from_list( -# layers: List[nn.Module], -# in_shape, -# discretize=True, -# dvs_input=False, -# ) -> nn.Sequential: -# """Build a sequential model of DVSLayer and DynapcnnLayer(s) given a list of layers comprising -# a spiking CNN. - -# Parameters -# ---------- - -# layers: sequence of layer objects -# in_shape: tuple of integers -# Shape of the input to the first layer in `layers`. Convention: -# (channels, height, width) -# discretize: bool -# Discretize weights and thresholds if True -# dvs_input: bool -# Whether model should receive DVS input. If `True`, the returned model -# will begin with a DVSLayer with `disable_pixel_array` set to False. -# Otherwise, the model starts with a DVSLayer only if the first element -# in `layers` is a pooling, cropping or flipping layer. - -# Returns -# ------- -# nn.Sequential -# """ -# compatible_layers = [] -# lyr_indx_next = 0 -# # Find and populate dvs layer (NOTE: We are ignoring the channel information here and could lead to problems) -# dvs_layer, lyr_indx_next, rescale_factor = construct_dvs_layer( -# layers, input_shape=in_shape, idx_start=lyr_indx_next, dvs_input=dvs_input -# ) - -# if dvs_layer is not None: -# compatible_layers.append(dvs_layer) -# in_shape = dvs_layer.get_output_shape() -# # Find and populate dynapcnn layers -# while lyr_indx_next < len(layers): -# if isinstance(layers[lyr_indx_next], DEFAULT_IGNORED_LAYER_TYPES): -# # - Ignore identity, dropout and flatten layers -# lyr_indx_next += 1 -# continue -# dynapcnn_layer, lyr_indx_next, rescale_factor = construct_next_dynapcnn_layer( -# layers, -# lyr_indx_next, -# in_shape=in_shape, -# discretize=discretize, -# rescale_factor=rescale_factor, -# ) -# in_shape = dynapcnn_layer.get_output_shape() -# compatible_layers.append(dynapcnn_layer) - -# return nn.Sequential(*compatible_layers) - - -# def convert_model_to_layer_list( -# model: Union[nn.Sequential, sinabs.Network, nn.Module], -# ignore: Union[Type, Tuple[Type, ...]] = (), -# ) -> List[nn.Module]: -# """Convert a model to a list of layers. - -# Parameters -# ---------- -# model: nn.Sequential, nn.Module or sinabs.Network. -# ignore: type or tuple of types of modules to be ignored. - -# Returns -# ------- -# List[nn.Module] -# """ -# if isinstance(model, sinabs.Network): -# return convert_model_to_layer_list(model.spiking_model) - -# elif isinstance(model, nn.Sequential): -# layers = [layer for layer in model if not isinstance(layer, ignore)] - -# elif isinstance(model, nn.Module): -# layers = [layer for _, layer in model.named_children() if not isinstance(layer, ignore)] - -# else: -# raise TypeError("Expected torch.nn.Sequential or sinabs.Network") - -# return layers From cf3a318e5f2b3e4028949deb7e80f245158a193d Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Tue, 5 Nov 2024 18:25:49 +0100 Subject: [PATCH 317/379] Handle edge case where snn is a sequential with only a dvslayer --- .../backend/dynapcnn/sinabs_edges_handler.py | 27 ++++++++++--------- tests/test_dynapcnn/test_individual_cases.py | 7 +---- 2 files changed, 15 insertions(+), 19 deletions(-) diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index dac2ccbd..3a210133 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -222,22 +222,23 @@ def fix_dvs_module_edges( indx_2_module_map.pop(dvs_crop_node[-1]) indx_2_module_map.pop(dvs_flip_node[-1]) - # Remove internal DVS modeules from name/index map. - for name in [ - name - for name, index in name_2_indx_map.items() - if index in [dvs_pool_node[-1], dvs_crop_node[-1], dvs_flip_node[-1]] - ]: - name_2_indx_map.pop(name) - - # Add edges from 'dvs' node to the entry point of the graph. - all_sources, all_targets = zip(*edges) - local_entry_nodes = set(all_sources) - set(all_targets) - edges.update({(dvs_node[-1], node) for node in local_entry_nodes}) + # Remove internal DVS modules from name/index map. + # Iterate over copy to prevent iterable from changing size. + n2i_map_copy = {k: v for k, v in name_2_indx_map.items()} + for name, index in n2i_map_copy.items(): + if index in [dvs_pool_node[-1], dvs_crop_node[-1], dvs_flip_node[-1]]: + name_2_indx_map.pop(name) + + dvs_node = dvs_node[0] + if edges: + # Add edges from 'dvs' node to the entry point of the graph. + all_sources, all_targets = zip(*edges) + local_entry_nodes = set(all_sources) - set(all_targets) + edges.update({(dvs_node, node) for node in local_entry_nodes}) # DVS becomes the only entry node of the graph. entry_nodes.clear() - entry_nodes.add(dvs_node[-1]) + entry_nodes.add(dvs_node) def merge_dvs_pooling_edge( diff --git a/tests/test_dynapcnn/test_individual_cases.py b/tests/test_dynapcnn/test_individual_cases.py index dc656b28..d39fae02 100644 --- a/tests/test_dynapcnn/test_individual_cases.py +++ b/tests/test_dynapcnn/test_individual_cases.py @@ -213,11 +213,6 @@ def test_no_spk_middle(): def test_no_conv_layers(): - seq = nn.Sequential() - from sinabs.backend.dynapcnn.dvs_layer import DVSLayer - from sinabs.backend.dynapcnn.utils import infer_input_shape - - net = DynapcnnNetwork(snn=seq, input_shape=(2, 10, 10), dvs_input=True) - assert isinstance(net.sequence[0], DVSLayer) + net = DynapcnnNetwork(nn.Sequential(DVSLayer(input_shape=(10, 10))), input_shape=(2, 10, 10)) From 36bceb80e7fe76f5d213687ce76171a8c50ebac2 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Tue, 5 Nov 2024 18:32:38 +0100 Subject: [PATCH 318/379] More meaningful exceptions --- sinabs/backend/dynapcnn/exceptions.py | 10 ++++++++-- sinabs/backend/dynapcnn/sinabs_edges_handler.py | 1 + tests/test_dynapcnn/test_individual_cases.py | 3 ++- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/sinabs/backend/dynapcnn/exceptions.py b/sinabs/backend/dynapcnn/exceptions.py index ff755e35..57b2b4de 100644 --- a/sinabs/backend/dynapcnn/exceptions.py +++ b/sinabs/backend/dynapcnn/exceptions.py @@ -70,7 +70,7 @@ class InvalidGraphStructure(Exception): class InvalidModelWithDVSSetup(Exception): def __init__(self): super().__init__( - f"The network provided has a DVSLayer instance but argument 'dvs_input' is set to False." + "The network provided has a DVSLayer instance but argument 'dvs_input' is set to False." ) @@ -83,7 +83,13 @@ class InvalidEdge(Exception): target: Type def __init__(self, edge, source, target): - super().__init__(f"Invalid edge {edge}: {source} can not target {target}.") + super().__init__( + f"Invalid edge {edge}: {source} can not target {target}. " + "This is likely due to a network architecture that is not supported. " + "In general, a dynapcnn network should be made of groups " + "of a weight layer (conv or linear), a spiking layer (IAFSqueeze), " + "and optionally a pooling layer." + ) class UnknownNode(Exception): diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index 3a210133..e0ac81aa 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -911,4 +911,5 @@ def trace_paths(node: int, remaining_edges: Set[Edge]) -> List[deque[int]]: """ Add verification tools to ensure that: - there are as many destinations as there are edges from pool/neuron to weight - there are as many layers as there are edges from weight to neuron +- each layer has a weight layer node and a spiking layer node """ diff --git a/tests/test_dynapcnn/test_individual_cases.py b/tests/test_dynapcnn/test_individual_cases.py index d39fae02..2c170722 100644 --- a/tests/test_dynapcnn/test_individual_cases.py +++ b/tests/test_dynapcnn/test_individual_cases.py @@ -204,11 +204,12 @@ def test_no_spk_ending(): def test_no_spk_middle(): + from sinabs.backend.dynapcnn.exceptions import InvalidEdge seq = nn.Sequential( nn.Flatten(), nn.Linear(512, 10), nn.Linear(10, 2), IAFSqueeze(batch_size=1) ) - with pytest.raises(TypeError): + with pytest.raises(InvalidEdge): DynapcnnNetwork(seq, input_shape=input_data.shape[1:], discretize=False) From c5bdfdf39d6e3b2d9cdb5054a57069cb8f1e4aa4 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Wed, 6 Nov 2024 10:11:08 +0100 Subject: [PATCH 319/379] New test file for networks that should fail --- .../test_dynapcnnnetwork_failcases.py | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 tests/test_dynapcnnnetwork/test_dynapcnnnetwork_failcases.py diff --git a/tests/test_dynapcnnnetwork/test_dynapcnnnetwork_failcases.py b/tests/test_dynapcnnnetwork/test_dynapcnnnetwork_failcases.py new file mode 100644 index 00000000..7b528396 --- /dev/null +++ b/tests/test_dynapcnnnetwork/test_dynapcnnnetwork_failcases.py @@ -0,0 +1,37 @@ +# author : Willian Soares Girao +# contact : wsoaresgirao@gmail.com + +import pytest +import torch +from sianbs.backend.dynapcnn.chip_factory import ChipFactory + +from sinabs.backend.dynapcnn.dynapcnn_network import DynapcnnNetwork + +from .conftest_dynapcnnnetwork import args_DynapcnnNetworkTest + + +@pytest.mark.parametrize("device", tuple(ChipFactory.supported_devices.keys())) +def test_too_large(device): + + # Model that is too big to fit on any of our architectures + big_ann = nn.Sequential( + nn.Conv2d(1, 3, 5, 1, bias=False), + nn.ReLU(), + nn.AvgPool2d(2, 2), + nn.Conv2d(3, 1, 5, 1, bias=False), + nn.ReLU(), + nn.AvgPool2d(2, 2), + nn.Flatten(), + nn.Linear(16, 999999, bias=False), + ) + + hardware_incompatible_model = DynapcnnNetwork( + from_model(big_ann, add_spiking_output=True, batch_size=1).cpu(), + discretize=True, + input_shape=input_shape, + ) + + assert not hardware_incompatible_model.is_compatible_with(device) + + with pytest.raises(ValueError): + hardware_incompatible_model.to(device) From 291bba73caf9dd49ea52f6c93ac32599aabfe90c Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Wed, 6 Nov 2024 10:11:38 +0100 Subject: [PATCH 320/379] Ensure network state is maintained when generating dynapcnn network --- .../backend/dynapcnn/nir_graph_extractor.py | 12 +++- tests/test_dynapcnn/test_config_making.py | 56 ++++++++----------- 2 files changed, 33 insertions(+), 35 deletions(-) diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index f8794d82..e2ea2d6d 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -5,10 +5,11 @@ from typing import Callable, Dict, Iterable, List, Optional, Set, Tuple, Type, Union import nirtorch -import sinabs import torch import torch.nn as nn +import sinabs + from .connectivity_specs import ( LAYER_TYPES_WITH_MULTIPLE_INPUTS, LAYER_TYPES_WITH_MULTIPLE_OUTPUTS, @@ -76,6 +77,11 @@ def __init__( instantiation with `remove_nodes_by_class`. """ + # Store state before it is changed due to NIRTorch passing dummy input + original_state = { + n: b.detach().clone() for n, b in spiking_model.named_buffers() + } + # extract computational graph. nir_graph = nirtorch.extract_torch_graph( spiking_model, dummy_input, model_name=None @@ -84,6 +90,10 @@ def __init__( for node_type in ignore_node_types: nir_graph = nir_graph.ignore_nodes(node_type) + # Restore original state + for n, b in spiking_model.named_buffers(): + b.set_(original_state[n].clone()) + # Map node names to indices self._name_2_indx_map = self._get_name_2_indx_map(nir_graph) diff --git a/tests/test_dynapcnn/test_config_making.py b/tests/test_dynapcnn/test_config_making.py index 77ae93ff..6e2a8b8b 100644 --- a/tests/test_dynapcnn/test_config_making.py +++ b/tests/test_dynapcnn/test_config_making.py @@ -25,6 +25,7 @@ ) sinabs_model = from_model(ann, add_spiking_output=True, batch_size=1) +# Make sure all states are zero input_shape = (1, 28, 28) hardware_compatible_model = DynapcnnNetwork( @@ -33,25 +34,29 @@ input_shape=input_shape, ) +devices = tuple(ChipFactory.supported_devices.keys()) +devices = [ + "dynapcnndevkit", + "speck2btiny", + "speck2e", + "speck2edevkit", + "speck2fmodule", +] -def test_zero_initial_states(): - for devkit in [ - "dynapcnndevkit", - "speck2btiny", - "speck2e", - "speck2edevkit", - "speck2fmodule", - ]: - config = hardware_compatible_model.make_config("auto", device=devkit) - for idx, lyr in enumerate(config.cnn_layers): - initial_value = torch.tensor(lyr.neurons_initial_value) - shape = initial_value.shape - zeros = torch.zeros(shape, dtype=torch.int) +@pytest.mark.parametrize("device", devices) +def test_zero_initial_states(device): + devkit = device + config = hardware_compatible_model.make_config("auto", device=devkit) + for idx, lyr in enumerate(config.cnn_layers): + initial_value = torch.tensor(lyr.neurons_initial_value) - assert ( - initial_value.all() == zeros.all() - ), f"Initial values of layer{idx} neuron states is not zeros!" + shape = initial_value.shape + zeros = torch.zeros(shape, dtype=torch.int) + + assert ( + initial_value.all() == zeros.all() + ), f"Initial values of layer{idx} neuron states is not zeros!" small_ann = nn.Sequential( @@ -72,23 +77,6 @@ def test_zero_initial_states(): ) -@pytest.mark.parametrize("device", tuple(ChipFactory.supported_devices.keys())) +@pytest.mark.parametrize("device", devices) def test_verify_working_config(device): assert small_hardware_compatible_model.is_compatible_with(device) - - -# Model that is too big to fit on any of our architectures -big_ann = deepcopy(ann) -big_ann.append(nn.ReLU()) -big_ann.append(nn.Linear(10, 999999, bias=False)) - -hardware_incompatible_model = DynapcnnNetwork( - from_model(big_ann, add_spiking_output=True, batch_size=1).cpu(), - discretize=True, - input_shape=input_shape, -) - - -@pytest.mark.parametrize("device", tuple(ChipFactory.supported_devices.keys())) -def test_verify_non_working_config(device): - assert not hardware_incompatible_model.is_compatible_with(device) From 37d08d155c486672ee15701418c8c8eed3d3b1ff Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Wed, 6 Nov 2024 12:21:27 +0100 Subject: [PATCH 321/379] Test for incorrect node types --- sinabs/backend/dynapcnn/connectivity_specs.py | 7 +- sinabs/backend/dynapcnn/exceptions.py | 16 ++- .../backend/dynapcnn/nir_graph_extractor.py | 64 +++++++++++- .../backend/dynapcnn/sinabs_edges_handler.py | 67 +++++++++++-- .../test_compatible_layer_build.py | 39 -------- .../test_dynapcnnnetwork_failcases.py | 37 ------- tests/test_dynapcnnnetwork/test_failcases.py | 98 +++++++++++++++++++ 7 files changed, 233 insertions(+), 95 deletions(-) delete mode 100644 tests/test_dynapcnnnetwork/test_dynapcnnnetwork_failcases.py create mode 100644 tests/test_dynapcnnnetwork/test_failcases.py diff --git a/sinabs/backend/dynapcnn/connectivity_specs.py b/sinabs/backend/dynapcnn/connectivity_specs.py index 8be31d6f..a17d595d 100644 --- a/sinabs/backend/dynapcnn/connectivity_specs.py +++ b/sinabs/backend/dynapcnn/connectivity_specs.py @@ -4,19 +4,20 @@ from typing import Union -import sinabs.layers as sl import torch.nn as nn +import sinabs.layers as sl + from .crop2d import Crop2d from .dvs_layer import DVSLayer from .flipdims import FlipDims Pooling = (sl.SumPool2d, nn.AvgPool2d) Weight = (nn.Conv2d, nn.Linear) -Neuron = (sl.IAFSqueeze,) +Neuron = (sl.IAFSqueeze, sl.IAF) DVS = (DVSLayer,) +SupportedNodeTypes = (*Pooling, *Weight, *Neuron, *DVS) -# @TODO - need to list other edge cases involving DVS layer (for now only dvs-weight and dvs-pooling). VALID_SINABS_EDGE_TYPES_ABSTRACT = { # convoluion is always followed by a neuron layer. (Weight, Neuron): "weight-neuron", diff --git a/sinabs/backend/dynapcnn/exceptions.py b/sinabs/backend/dynapcnn/exceptions.py index 57b2b4de..b1d64491 100644 --- a/sinabs/backend/dynapcnn/exceptions.py +++ b/sinabs/backend/dynapcnn/exceptions.py @@ -1,5 +1,12 @@ from typing import Set, Tuple, Type +default_invalid_structure_string = ( + "This should never happen, but is most likely due to an unsupported SNN " + "architecture. In general, a dynapcnn network should consist of groups of " + "a weight layer (conv or linear), a spiking layer (IAFSqueeze), and " + "optionally a pooling layer." +) + class MissingLayer(Exception): index: int @@ -44,6 +51,10 @@ def __init__( ) +class UnsupportedLayerType(Exception): + pass + + class InvalidModel(Exception): model: Type @@ -85,10 +96,7 @@ class InvalidEdge(Exception): def __init__(self, edge, source, target): super().__init__( f"Invalid edge {edge}: {source} can not target {target}. " - "This is likely due to a network architecture that is not supported. " - "In general, a dynapcnn network should be made of groups " - "of a weight layer (conv or linear), a spiking layer (IAFSqueeze), " - "and optionally a pooling layer." + + default_invalid_structure_string ) diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index e2ea2d6d..9c538de0 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -2,22 +2,24 @@ # contact : wsoaresgirao@gmail.com from copy import deepcopy +from pprint import pformat from typing import Callable, Dict, Iterable, List, Optional, Set, Tuple, Type, Union import nirtorch import torch import torch.nn as nn -import sinabs +from sinabs import layers as sl from .connectivity_specs import ( LAYER_TYPES_WITH_MULTIPLE_INPUTS, LAYER_TYPES_WITH_MULTIPLE_OUTPUTS, + SupportedNodeTypes, ) from .dvs_layer import DVSLayer from .dynapcnn_layer_utils import construct_dynapcnnlayers_from_mapper from .dynapcnnnetwork_module import DynapcnnNetworkModule -from .exceptions import InvalidGraphStructure +from .exceptions import InvalidGraphStructure, UnsupportedLayerType from .sinabs_edges_handler import ( collect_dynapcnn_layer_info, fix_dvs_module_edges, @@ -181,6 +183,9 @@ def get_dynapcnn_network_module( - The DynapcnnNetworkModule based on graph representation of this `GraphExtractor` """ + # Make sure all nodes are supported + self.verify_node_types() + # create a dict holding the data necessary to instantiate a `DynapcnnLayer`. self.dcnnl_map, self.dvs_layer_info = collect_dynapcnn_layer_info( indx_2_module_map=self.indx_2_module_map, @@ -277,6 +282,10 @@ def verify_graph_integrity(self): Currently this checks that only nodes of specific classes have multiple sources or targets. This method might be extended in the future to implement stricter formal verification. + + Raises + ------ + - InvalidGraphStructure: If any verification fails """ # Iterate over all nodes, and count its sources and targets for node, module in self.indx_2_module_map.items(): @@ -299,6 +308,55 @@ def verify_graph_integrity(self): f"{type(module)} and has {len(targets)} outputs." ) + def verify_node_types(self): + """Verify that all nodes are of a supported type. + + Raises + ------ + - UnsupportedLayerType: If any verification fails + """ + unsupported_nodes = dict() + for index, module in self.indx_2_module_map.items(): + if not isinstance(module, SupportedNodeTypes): + node_type = type(module) + if node_type in unsupported_nodes: + unsupported_nodes[node_type].add(index) + else: + unsupported_nodes[node_type] = {index} + # Specific error message for leaky neuron types + lif_layers = [] + for lif_type in (sl.LIF, sl.LIFSqueeze): + for idx in unsupported_nodes.pop(lif_type, []): + lif_layers.append(self.indx_2_module_map[idx]) + if lif_layers: + layer_str = ", ".join(str(lyr) for lyr in (lif_layers)) + raise UnsupportedLayerType( + f"The provided SNN contains LIF layers:\n{layer_str}.\n" + "Leaky integrate-and-fire dynamics are not supported by " + "DynapCNN. Use non-leaky `IAF` or `IAFSqueeze` layers " + "instead." + ) + # Specific error message for most common non-spiking activation layers + activation_layers = [] + for activation_type in (nn.ReLU, nn.Sigmoid, nn.Tanh, sl.NeuromorphicReLU): + for idx in unsupported_nodes.pop(activation_type, []): + activation_layers.append(self.indx_2_module_map[idx]) + if activation_layers: + layer_str = ", ".join(str(lyr) for lyr in (activation_layers)) + raise UnsupportedLayerType( + "The provided SNN contains non-spiking activation layers:\n" + f"{layer_str}.\nPlease convert them to `IAF` or `IAFSqueeze` " + "layers before instantiating a `DynapcnnNetwork`. You can " + "use the function `sinabs.from_model.from_torch` for this." + ) + if unsupported_nodes: + # More generic error message for all remaining types + raise UnsupportedLayerType( + "One or more layers in the provided SNN are not supported: " + f"{pformat(unsupported_nodes)}. Supported layer types are: " + f"{pformat(SupportedNodeTypes)}." + ) + ####################################################### Pivate Methods ####################################################### def _handle_dvs_input( @@ -595,7 +653,7 @@ def _get_nodes_io_shapes( # propagate inputs through the nodes. for node in self.sorted_nodes: - if isinstance(self.indx_2_module_map[node], sinabs.layers.merge.Merge): + if isinstance(self.indx_2_module_map[node], sl.merge.Merge): # find `Merge` arguments (at this point the inputs to Merge should have been calculated). input_nodes = self._find_merge_arguments(node) diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index e0ac81aa..7b1fbad1 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -6,16 +6,21 @@ """ from collections import deque -from typing import Dict, List, Set, Tuple, Type, Union +from typing import Dict, List, Optional, Set, Tuple, Type, Union + +from torch import Size, nn from sinabs.layers import SumPool2d from sinabs.utils import expand_to_pair -from torch import Size, nn from .connectivity_specs import VALID_SINABS_EDGE_TYPES, Pooling from .crop2d import Crop2d from .dvs_layer import DVSLayer -from .exceptions import InvalidEdge, InvalidGraphStructure +from .exceptions import ( + InvalidEdge, + InvalidGraphStructure, + default_invalid_structure_string, +) from .flipdims import FlipDims from .utils import Edge, merge_bn @@ -377,6 +382,7 @@ def collect_dynapcnn_layer_info( edges_by_type: Dict[str, Set[Edge]] = sort_edges_by_type( edges=edges, indx_2_module_map=indx_2_module_map ) + edge_counts_by_type = {t: len(e) for t, e in edges_by_type.items()} # Dict to collect information for each future dynapcnn layer dynapcnn_layer_info = dict() @@ -467,6 +473,9 @@ def collect_dynapcnn_layer_info( # Set minimal destination entries for layers without child nodes, to act as network outputs set_exit_destinations(dynapcnn_layer_info) + # Assert formal correctness of layer info + verify_layer_info(dynapcnn_layer_info, edge_counts_by_type) + return dynapcnn_layer_info, dvs_layer_info @@ -907,9 +916,49 @@ def trace_paths(node: int, remaining_edges: Set[Edge]) -> List[deque[int]]: return paths, processed_edges -# TODO: -""" Add verification tools to ensure that: -- there are as many destinations as there are edges from pool/neuron to weight -- there are as many layers as there are edges from weight to neuron -- each layer has a weight layer node and a spiking layer node -""" +def verify_layer_info( + dynapcnn_layer_info: Dict[int, Dict], edge_counts: Optional[Dict[str, int]] = None +): + """Verify that `dynapcnn_layer_info` matches formal requirements. + + - Every layer needs to have at least a `conv`, `neuron`, and `destinations` + entry. + - If `edge_counts` is provided, also make sure that number of layer matches + numbers of edges. + + Parameters + ---------- + - dynapcnn_layer_info: Dict with information to construct and connect + DynapcnnLayer instances + - edge_counts: Optional Dict with edge counts for each edge type. If not + `None`, will be used to do further verifications on `dynapcnn_layer_info` + + Raises + ------ + - InvalidGraphStructure: if any verification fails. + """ + + # Make sure that each dynapcnn layer has at least a weight layer and a neuron layer + for idx, info in dynapcnn_layer_info.items(): + if not "conv" in info: + raise InvalidGraphStructure( + f"DynapCNN layer {idx} has no weight assigned, which should " + "never happen. " + default_invalid_structure_string + ) + if not "neuron" in info: + raise InvalidGraphStructure( + f"DynapCNN layer {idx} has no spiking layer assigned, which " + "should never happen. " + default_invalid_structure_string + ) + if not "destinations" in info: + raise InvalidGraphStructure( + f"DynapCNN layer {idx} has no destination info assigned, which " + "should never happen. " + default_invalid_structure_string + ) + if edge_counts is not None: + # Make sure there are as many layers as edges from weight to neuron + if edge_counts["weight-neuron"] - len(dynapcnn_layer_info) > 0: + raise InvalidGraphStructure( + "Not all weight-to-neuron edges have been processed, which " + "should never happen. " + default_invalid_structure_string + ) diff --git a/tests/test_dynapcnn/test_compatible_layer_build.py b/tests/test_dynapcnn/test_compatible_layer_build.py index 6efe637e..82ccf5fe 100644 --- a/tests/test_dynapcnn/test_compatible_layer_build.py +++ b/tests/test_dynapcnn/test_compatible_layer_build.py @@ -39,42 +39,3 @@ def test_construct_pooling_from_2_layers(): assert cumulative_pooling == [6, 6] assert scaling == 1.0 / 9 - - -# TODO: Move these fail cases to another test. Rename this file or move other tests as well. -def test_missing_spiking_layer(): - in_shape = (2, 28, 28) - layers = [ - nn.Conv2d(2, 8, kernel_size=3, stride=1, bias=False), - sl.IAF(), - sl.SumPool2d(2), - nn.AvgPool2d(2), - nn.Conv2d(8, 16, kernel_size=3, stride=1, bias=False), - sl.IAF(), - nn.Dropout2d(), - nn.Conv2d(16, 2, kernel_size=3, stride=1, bias=False), - sl.IAF(), - nn.Flatten(), - nn.Linear(8, 5), - ] - from sinabs.backend.dynapcnn.exceptions import MissingLayer - from sinabs.backend.dynapcnn.utils import build_from_list - - with pytest.raises(MissingLayer): - build_from_list(layers, in_shape=in_shape, discretize=True) - - -def test_incorrect_model_start(): - in_shape = (2, 28, 28) - layers = [ - sl.IAF(), - sl.SumPool2d(2), - nn.AvgPool2d(2), - ] - from sinabs.backend.dynapcnn.exceptions import UnexpectedLayer - from sinabs.backend.dynapcnn.utils import construct_next_dynapcnn_layer - - with pytest.raises(UnexpectedLayer): - construct_next_dynapcnn_layer( - layers, 0, in_shape=in_shape, discretize=True, rescale_factor=1 - ) diff --git a/tests/test_dynapcnnnetwork/test_dynapcnnnetwork_failcases.py b/tests/test_dynapcnnnetwork/test_dynapcnnnetwork_failcases.py deleted file mode 100644 index 7b528396..00000000 --- a/tests/test_dynapcnnnetwork/test_dynapcnnnetwork_failcases.py +++ /dev/null @@ -1,37 +0,0 @@ -# author : Willian Soares Girao -# contact : wsoaresgirao@gmail.com - -import pytest -import torch -from sianbs.backend.dynapcnn.chip_factory import ChipFactory - -from sinabs.backend.dynapcnn.dynapcnn_network import DynapcnnNetwork - -from .conftest_dynapcnnnetwork import args_DynapcnnNetworkTest - - -@pytest.mark.parametrize("device", tuple(ChipFactory.supported_devices.keys())) -def test_too_large(device): - - # Model that is too big to fit on any of our architectures - big_ann = nn.Sequential( - nn.Conv2d(1, 3, 5, 1, bias=False), - nn.ReLU(), - nn.AvgPool2d(2, 2), - nn.Conv2d(3, 1, 5, 1, bias=False), - nn.ReLU(), - nn.AvgPool2d(2, 2), - nn.Flatten(), - nn.Linear(16, 999999, bias=False), - ) - - hardware_incompatible_model = DynapcnnNetwork( - from_model(big_ann, add_spiking_output=True, batch_size=1).cpu(), - discretize=True, - input_shape=input_shape, - ) - - assert not hardware_incompatible_model.is_compatible_with(device) - - with pytest.raises(ValueError): - hardware_incompatible_model.to(device) diff --git a/tests/test_dynapcnnnetwork/test_failcases.py b/tests/test_dynapcnnnetwork/test_failcases.py new file mode 100644 index 00000000..125b1dff --- /dev/null +++ b/tests/test_dynapcnnnetwork/test_failcases.py @@ -0,0 +1,98 @@ +# author : Willian Soares Girao +# contact : wsoaresgirao@gmail.com + +import pytest +import torch +from torch import nn + +from sinabs import layers as sl +from sinabs.backend.dynapcnn import DynapcnnNetwork +from sinabs.backend.dynapcnn.chip_factory import ChipFactory +from sinabs.backend.dynapcnn.exceptions import ( + InvalidGraphStructure, + UnsupportedLayerType, +) +from sinabs.from_torch import from_model + + +@pytest.mark.parametrize("device", tuple(ChipFactory.supported_devices.keys())) +def test_too_large(device): + + # Model that is too big to fit on any of our architectures + big_ann = nn.Sequential( + nn.Conv2d(1, 3, 5, 1, bias=False), + nn.ReLU(), + nn.AvgPool2d(2, 2), + nn.Conv2d(3, 1, 5, 1, bias=False), + nn.ReLU(), + nn.AvgPool2d(2, 2), + nn.Flatten(), + nn.Linear(16, 999999, bias=False), + ) + input_shape = (1, 28, 28) + + hardware_incompatible_model = DynapcnnNetwork( + from_model(big_ann, add_spiking_output=True, batch_size=1).cpu(), + discretize=True, + input_shape=input_shape, + ) + + assert not hardware_incompatible_model.is_compatible_with(device) + + with pytest.raises(ValueError): + hardware_incompatible_model.to(device) + + +def test_missing_spiking_layer(): + in_shape = (2, 28, 28) + snn = nn.Sequential( + nn.Conv2d(2, 8, kernel_size=3, stride=1, bias=False), + sl.IAF(), + sl.SumPool2d(2), + nn.AvgPool2d(2), + nn.Conv2d(8, 16, kernel_size=3, stride=1, bias=False), + sl.IAF(), + nn.Dropout2d(), + nn.Conv2d(16, 2, kernel_size=3, stride=1, bias=False), + sl.IAF(), + nn.Flatten(), + nn.Linear(8, 5), + ) + + with pytest.raises(InvalidGraphStructure): + net = DynapcnnNetwork(snn, input_shape=in_shape) + + +def test_incorrect_model_start(): + in_shape = (2, 28, 28) + snn = nn.Sequential( + sl.IAF(), + sl.SumPool2d(2), + nn.AvgPool2d(2), + ) + + with pytest.raises(InvalidGraphStructure): + net = DynapcnnNetwork(snn, input_shape=in_shape) + + +unsupported_layers = [ + nn.ReLU(), + nn.Sigmoid(), + nn.Tanh(), + sl.LIF(tau_mem=5), + sl.LIFSqueeze(batch_size=1, tau_mem=5), + sl.NeuromorphicReLU(), + sl.Cropping2dLayer(), +] + + +@pytest.mark.parametrize("layer", unsupported_layers) +def test_unsupported_layers(layer): + in_shape = (1, 28, 28) + ann = nn.Sequential( + nn.Conv2d(1, 3, 5, 1, bias=False), + layer, + ) + + with pytest.raises(UnsupportedLayerType): + net = DynapcnnNetwork(ann, input_shape=in_shape) From ff3950279f55057616cc7672877d2bbf1716acb2 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Wed, 6 Nov 2024 15:10:16 +0100 Subject: [PATCH 322/379] Fix a range of bugs --- sinabs/backend/dynapcnn/dynapcnn_layer_utils.py | 5 +++-- sinabs/backend/dynapcnn/dynapcnnnetwork_module.py | 13 ++++++++----- sinabs/backend/dynapcnn/sinabs_edges_handler.py | 8 +++----- tests/test_dynapcnn/test_individual_cases.py | 9 +++++++-- 4 files changed, 21 insertions(+), 14 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py b/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py index c1974d7d..1225380b 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py @@ -1,9 +1,10 @@ from math import prod from typing import Callable, Dict, Iterable, List, Optional, Set, Tuple, Union +from torch import nn + from sinabs import layers as sl from sinabs.utils import expand_to_pair -from torch import nn from .dynapcnn_layer import DynapcnnLayer @@ -239,7 +240,7 @@ def construct_destination_map( destination_map[layer_index] = destination_indices if dvs_layer_info is not None: # Copy destination list from dvs layer info - destination_map["dvs"] = [d for d in dvs_layer_info.destinations] + destination_map["dvs"] = [d for d in dvs_layer_info["destinations"]] return destination_map diff --git a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py index 07d41a8f..b471c62d 100644 --- a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py +++ b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py @@ -5,10 +5,11 @@ from typing import Dict, List, Optional, Set, Union from warnings import warn -import sinabs.layers as sl import torch.nn as nn from torch import Tensor +import sinabs.layers as sl + from .dvs_layer import DVSLayer from .dynapcnn_layer import DynapcnnLayer from .utils import Edge, topological_sorting @@ -259,8 +260,8 @@ def forward( ) # For each layer store its outputs as dict with destination layers as keys. - # For input use `defaultdict` so it can be used for all destinations where needed - layers_outputs = {"input": defaultdict(lambda: x)} + # For input set `x` as input to entry points + layers_outputs = {"input": {ep: x for ep in self.entry_points}} for idx_curr in self._sorted_nodes: # Get inputs to the layer @@ -300,9 +301,11 @@ def forward( # Take outputs with exit point destinations as network output network_outputs = {} - for layer_idx, outputs in layers_outputs.items(): + for layer_idx, layer_out in layers_outputs.items(): outputs = { - idx_dest: out for idx_dest, out in outputs.items() if idx_dest < 0 + idx_dest: out + for idx_dest, out in layer_out.items() + if isinstance(idx_dest, int) and idx_dest < 0 } if outputs: network_outputs[layer_idx] = outputs diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index 7b1fbad1..44b9c8c0 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -287,9 +287,7 @@ def merge_dvs_pooling_edge( pool_layer = indx_2_module_map[pool_indx] # Checking pooling can be incorporated into the DVSLayer. - if expand_to_pair(dvs_layer.pool_layer.kernel_size) != [1, 1] or expand_to_pair( - dvs_layer.pool_layer.stride - ) != [1, 1]: + if expand_to_pair(dvs_layer.pool_layer.kernel_size) != (1, 1): raise ValueError( "DVSLayer with pooling is followed by another pooling layer. " "This is currently not supported. Please update the network " @@ -321,7 +319,7 @@ def merge_dvs_pooling_edge( # Set DVSLayer.pool to have same config. as the independent pooling layer. dvs_layer.pool_layer.kernel_size = pool_layer.kernel_size - dvs_layer.pool_layer.stride = pool_layer.stride + dvs_layer.pool_layer.stride = None # Pooling incorporated to the DVSLayer: remove its trace from mappings. indx_2_module_map.pop(pool_indx) @@ -957,7 +955,7 @@ def verify_layer_info( ) if edge_counts is not None: # Make sure there are as many layers as edges from weight to neuron - if edge_counts["weight-neuron"] - len(dynapcnn_layer_info) > 0: + if edge_counts.get("weight-neuron", 0) - len(dynapcnn_layer_info) > 0: raise InvalidGraphStructure( "Not all weight-to-neuron edges have been processed, which " "should never happen. " + default_invalid_structure_string diff --git a/tests/test_dynapcnn/test_individual_cases.py b/tests/test_dynapcnn/test_individual_cases.py index 2c170722..6515cc4e 100644 --- a/tests/test_dynapcnn/test_individual_cases.py +++ b/tests/test_dynapcnn/test_individual_cases.py @@ -1,5 +1,4 @@ import pytest -import samna import torch from torch import nn @@ -205,6 +204,7 @@ def test_no_spk_ending(): def test_no_spk_middle(): from sinabs.backend.dynapcnn.exceptions import InvalidEdge + seq = nn.Sequential( nn.Flatten(), nn.Linear(512, 10), nn.Linear(10, 2), IAFSqueeze(batch_size=1) ) @@ -216,4 +216,9 @@ def test_no_spk_middle(): def test_no_conv_layers(): from sinabs.backend.dynapcnn.dvs_layer import DVSLayer - net = DynapcnnNetwork(nn.Sequential(DVSLayer(input_shape=(10, 10))), input_shape=(2, 10, 10)) + net = DynapcnnNetwork( + nn.Sequential(DVSLayer(input_shape=(10, 10))), input_shape=(2, 10, 10) + ) + + +test_with_sinabs_batch() From 72040d2009b693a63fc22dc741eacdb72b4a0de7 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Wed, 6 Nov 2024 18:24:09 +0100 Subject: [PATCH 323/379] WIP: Merge dvs pooling layer --- sinabs/backend/dynapcnn/connectivity_specs.py | 2 + .../backend/dynapcnn/dynapcnn_layer_utils.py | 78 +++- sinabs/backend/dynapcnn/dynapcnn_network.py | 11 +- .../backend/dynapcnn/nir_graph_extractor.py | 7 - .../backend/dynapcnn/sinabs_edges_handler.py | 350 ++++++++++-------- 5 files changed, 285 insertions(+), 163 deletions(-) diff --git a/sinabs/backend/dynapcnn/connectivity_specs.py b/sinabs/backend/dynapcnn/connectivity_specs.py index a17d595d..5de38943 100644 --- a/sinabs/backend/dynapcnn/connectivity_specs.py +++ b/sinabs/backend/dynapcnn/connectivity_specs.py @@ -31,6 +31,8 @@ (Pooling, Weight): "pooling-weight", # Dvs can be followed by weight layer of next core (DVS, Weight): "dvs-weight", + # Dvs can be followed by pooling layer + (DVS, Pooling): "dvs-pooling", } # Unpack dict diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py b/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py index 1225380b..2ea89afb 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py @@ -26,7 +26,7 @@ def construct_dynapcnnlayers_from_mapper( - Dict mapping to each layer index a set of destination indices - List of layer indices that act as entry points to the network """ - finalize_dcnnl_map(dcnnl_map, rescale_fn) + finalize_dcnnl_map(dcnnl_map, dvs_layer_info, rescale_fn) dynapcnn_layers = { layer_idx: construct_single_dynapcnn_layer(layer_info, discretize) @@ -40,7 +40,9 @@ def construct_dynapcnnlayers_from_mapper( return dynapcnn_layers, destination_map, entry_points -def finalize_dcnnl_map(dcnnl_map: Dict, rescale_fn: Optional[Callable] = None) -> None: +def finalize_dcnnl_map( + dcnnl_map: Dict, dvs_info: Union[Dict, None], rescale_fn: Optional[Callable] = None +) -> None: """Finalize dcnnl map by consolidating information Update dcnnl_map in-place @@ -54,6 +56,9 @@ def finalize_dcnnl_map(dcnnl_map: Dict, rescale_fn: Optional[Callable] = None) - - rescale_fn: Optional callable that is used to determine layer rescaling in case of conflicting preceeding average pooling """ + # Consolidate pooling information for DVS layer + consolidate_dvs_pooling(dvs_info, dcnnl_map) + # Consolidate pooling information for each destination for layer_info in dcnnl_map.values(): consolidate_layer_pooling(layer_info, dcnnl_map) @@ -63,6 +68,67 @@ def finalize_dcnnl_map(dcnnl_map: Dict, rescale_fn: Optional[Callable] = None) - consolidate_layer_scaling(layer_info, rescale_fn) +def consolidate_dvs_pooling(dvs_info: Union[Dict, None], dcnnl_map: Dict): + """Consolidate pooling information for dvs layer + + Update `dvs_info` and `dcnnl_map` in place. + - Extract pooling and scale factor of consecutive pooling operations + - Add entries "cumulative_pooling" and "cumulative_scaling" + - Update DVSLayer pooling if applicable + - For each destination, add cumulative rescale factor to "rescale_factors" + entry in corresponding entry of `dcnnl_map`. + + Parameters + ---------- + - dvs_info: Dict holding info of dvs layer. + - dcnnl_map: Dict holding info needed to instantiate DynapcnnLayer instances + """ + if dvs_info is None or dvs_info["pooling"] is None: + # Nothing to do + return + + # Check whether pooling can be incorporated into the DVSLayer. + dvs_layer = dvs_info["module"] + crop_layer = dvs_layer.crop_layer + if ( + crop_layer.top_crop != 0 + or crop_layer.left_crop != 0 + or crop_layer.bottom_crop != dvs_layer.input_shape[1] + or crop_layer.right_crop != dvs_layer.input_shape[2] + ): + raise ValueError( + "DVSLayer with cropping is followed by a pooling layer. " + "This is currently not supported. Please define pooling " + "directly within the DVSLayer (with the `pool` argument) " + "and remove the pooling layer that follows the DVSLayer" + ) + flip_layer = dvs_layer.flip_layer + if flip_layer.flip_x or flip_layer.flip_y or flip_layer.swap_xy: + raise ValueError( + "DVSLayer with flipping or dimension swapping is followed " + "by a pooling layer. This is currently not supported. " + "Please define pooling directly within the DVSLayer " + "(with the `pool` argument) and remove the pooling " + "layer that follows the DVSLayer" + ) + + # Incorporate pooling into DVSLayer + pool_layer = dvs_info["pooling"]["module"] + added_pooling, scale = extract_pooling_from_module(pool_layer) + dvs_pooling = expand_to_pair(dvs_layer.pool_layer.kernel_size) + cumulative_pooling = ( + dvs_pooling[0] * added_pooling[0], + dvs_pooling[1] * added_pooling[1], + ) + dvs_layer.pool_layer.kernel_size = cumulative_pooling + dvs_layer.pool_layer.stride = None + + # Set rescale_factor for targeted dynapcnn layers + if dvs_info["destinations"] is not None: + for dest_lyr_idx in dvs_info["destinations"]: + dcnnl_map[dest_lyr_idx]["rescale_factors"].add(scale) + + def consolidate_layer_pooling(layer_info: Dict, dcnnl_map: Dict): """Consolidate pooling information for individual layer @@ -174,7 +240,6 @@ def consolidate_layer_scaling(layer_info: Dict, rescale_fn: Optional[Callable] = rescale_factor = layer_info["rescale_factors"].pop() else: if rescale_fn is None: - # TODO: Custom Exception class? raise ValueError( "Average pooling layers of conflicting sizes pointing to " "same destination. Either replace them by SumPool2d layers " @@ -239,8 +304,11 @@ def construct_destination_map( destination_indices.append(dest_idx) destination_map[layer_index] = destination_indices if dvs_layer_info is not None: - # Copy destination list from dvs layer info - destination_map["dvs"] = [d for d in dvs_layer_info["destinations"]] + if (dest_info := dvs_layer_info["destinations"]) is None: + destination_map["dvs"] = [-1] + else: + # Copy destination list from dvs layer info + destination_map["dvs"] = [d for d in dest_info] return destination_map diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index a541bfbe..aa9ff1fd 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -6,13 +6,14 @@ from warnings import warn import samna -import sinabs -import sinabs.layers as sl import torch import torch.nn as nn from samna.dynapcnn.configuration import DynapcnnConfiguration from torch import Tensor +import sinabs +import sinabs.layers as sl + from .chip_factory import ChipFactory from .dvs_layer import DVSLayer from .dynapcnn_layer import DynapcnnLayer @@ -472,7 +473,11 @@ def make_config( print("Network is valid") return config else: - raise ValueError(f"Generated config is not valid for {device}") + raise ValueError( + f"Generated config is not valid for {device}. " + "Probably one or more layers are too large. Try " + "Reducing the number of neurons or the kernel sizes." + ) def has_dvs_layer(self) -> bool: """Return True if there is a DVSLayer in the network diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index 9c538de0..d8a69434 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -24,7 +24,6 @@ collect_dynapcnn_layer_info, fix_dvs_module_edges, handle_batchnorm_nodes, - merge_dvs_pooling_edge, ) from .utils import Edge, topological_sorting @@ -396,12 +395,6 @@ def _handle_dvs_input( self._entry_nodes, ) - # Merge a pooling node from a 'dvs-pooling' edge (pooling being an independent node in the original - # graph) into the DVSLayer if such edge exists. - merge_dvs_pooling_edge( - self._edges, self._indx_2_module_map, self._name_2_indx_map - ) - # Check if graph structure and DVSLayer.merge_polarities are correctly set (if DVS node exists). self._validate_dvs_setup(dvs_input_shape=input_shape) diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index 44b9c8c0..542117be 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -246,105 +246,6 @@ def fix_dvs_module_edges( entry_nodes.add(dvs_node) -def merge_dvs_pooling_edge( - edges: Set[Edge], - indx_2_module_map: Dict[int, nn.Module], - name_2_indx_map: Dict[str, int], -) -> None: - """If a 'dvs-pooling' edge existis, the pooling is incorporated into the DVSLayer node if `DVSLayer.pool_layer` has - default values. All arguments are modified in-place to remove the references to the incorporated pooling node. - - Parameters - ---------- - - edges (set): tuples describing the connections between layers in `spiking_model`. - - indx_2_module_map (dict): the mapping between a node (`key` as an `int`) and its module (`value` as a `nn.Module`). - - name_2_indx_map (dict): Map from node names to unique indices. - """ - # Find 'DVSLayer-pooling' edge. - dvs_pool_edge = [ - (src, tgt) - for (src, tgt) in edges - if ( - isinstance(indx_2_module_map[src], DVSLayer) - and any(isinstance(indx_2_module_map[tgt], pool) for pool in Pooling) - ) - ] - - if len(dvs_pool_edge) == 0: - # No dvs-pooling edge exists - nothing to do here. - return - if len(dvs_pool_edge) > 1: - # DVSLayer in the original network can have only a single pooling layer liked to it. - raise ValueError( - f"DVSLayer has multiple outgoing edges to pooling layers: {dvs_pool_edge}. " - "Unlike convolutional layers, for DVS layers, pooling is set globally for " - "all destinations. Therefore a DVSLayer can be followed by at most one " - "pooling layer." - ) - - (dvs_indx, pool_indx) = dvs_pool_edge[-1] - dvs_layer = indx_2_module_map[dvs_indx] - pool_layer = indx_2_module_map[pool_indx] - - # Checking pooling can be incorporated into the DVSLayer. - if expand_to_pair(dvs_layer.pool_layer.kernel_size) != (1, 1): - raise ValueError( - "DVSLayer with pooling is followed by another pooling layer. " - "This is currently not supported. Please update the network " - "such that all pooling is either done by the DVSLayer or by " - "the following pooling layer." - ) - crop_layer = dvs_layer.crop_layer - if ( - crop_layer.top_crop != 0 - or crop_layer.left_crop != 0 - or crop_layer.bottom_crop != dvs_layer.input_shape[1] - or crop_layer.right_crop != dvs_layer.input_shape[2] - ): - raise ValueError( - "DVSLayer with cropping is followed by a pooling layer. " - "This is currently not supported. Please define pooling " - "directly within the DVSLayer (with the `pool` argument) " - "and remove the pooling layer that follows the DVSLayer" - ) - flip_layer = dvs_layer.flip_layer - if flip_layer.flip_x or flip_layer.flip_y or flip_layer.swap_xy: - raise ValueError( - "DVSLayer with flipping or dimension swapping is followed " - "by a pooling layer. This is currently not supported. " - "Please define pooling directly within the DVSLayer " - "(with the `pool` argument) and remove the pooling " - "layer that follows the DVSLayer" - ) - - # Set DVSLayer.pool to have same config. as the independent pooling layer. - dvs_layer.pool_layer.kernel_size = pool_layer.kernel_size - dvs_layer.pool_layer.stride = None - - # Pooling incorporated to the DVSLayer: remove its trace from mappings. - indx_2_module_map.pop(pool_indx) - name_2_indx_map.pop( - [name for name, indx in name_2_indx_map.items() if indx == pool_indx][-1] - ) - - # Since pool is part of the DVSLayer we now make edges where pool was a source to have DVSLayer as a source. - for edge in [edge for edge in edges if edge[0] == pool_indx]: - edges.remove(edge) - edges.update({(dvs_indx, edge[1])}) - - # Remove original 'dvs-pool' edge. - edges.remove((dvs_indx, pool_indx)) - - # Checks if any traces of the original pooling node can still be found. - if ( - len([edge for edge in edges if (edge[0] == pool_indx or edge[1] == pool_indx)]) - != 0 - ): - raise ValueError( - "Edges involving the pooling layer merged into the DVSLayer are still present in the graph." - ) - - def collect_dynapcnn_layer_info( indx_2_module_map: Dict[int, nn.Module], edges: Set[Edge], @@ -384,7 +285,6 @@ def collect_dynapcnn_layer_info( # Dict to collect information for each future dynapcnn layer dynapcnn_layer_info = dict() - dvs_layer_info = None # Map node IDs to dynapcnn layer ID node_2_layer_map = dict() @@ -401,17 +301,10 @@ def collect_dynapcnn_layer_info( entry_nodes, ) - # Process all dvs->weight edges connecting the DVS camera to a unique dynapcnn layer. - dvs_weight_edges = edges_by_type.get("dvs-weight", set()) - while dvs_weight_edges: - edge = dvs_weight_edges.pop() - dvs_layer_info = add_or_update_dvs_to_entry( - edge, - dvs_layer_info, - indx_2_module_map, - node_2_layer_map, - nodes_io_shapes, - ) + # Process all edges related to DVS layer + dvs_layer_info = dvs_setup( + edges_by_type, indx_2_module_map, node_2_layer_map, nodes_io_shapes + ) # Process all edges connecting two dynapcnn layers that do not include pooling neuron_weight_edges = edges_by_type.get("neuron-weight", set()) @@ -654,62 +547,208 @@ def add_pooling_to_entry( node_2_layer_map[node] = layer_idx -def add_or_update_dvs_to_entry( - edge: Edge, - dvs_layer_info: Union[None, Dict], +def dvs_setup( + edges_by_type: Dict[str, Set[Edge]], + indx_2_module_map: Dict[int, nn.Module], + node_2_layer_map: Dict[int, int], + nodes_io_shapes: Dict[int, Dict[str, Tuple[Size, Size]]], +) -> Union[None, Dict]: + """Generate dict containing information to set up DVS layer + + Parameters + ---------- + edges_by_type (dict of sets of edges): Keys are edge types (str), values are sets of edges. + indx_2_module_map (dict): Maps node IDs of the graph as `key` to their associated module as `value` + node_2_layer_map (dict): Maps each node ID to the ID of the layer it is assigned to. + Will be updated in-place. + nodes_io_shapes (dict): Map from node ID to dict containing node's in- and output shapes + + Returns + ------- + dvs_layer_info: Dict containing information about the DVSLayer. + """ + # Process all outgoing edges of a DVSLayer + dvs_weight_edges = edges_by_type.get("dvs-weight", set()) + dvs_pooling_edges = edges_by_type.get("dvs-pooling", set()) + + # Process all dvs->weight edges connecting the DVS camera to a dynapcnn layer. + if dvs_weight_edges: + if dvs_pooling_edges: + raise InvalidGraphStructure( + "DVS layer has destinations with and without pooling. Unlike " + "with CNN layers, pooling of the DVS has to be the same for " + "all destinations." + ) + return init_dvs_entry( + dvs_weight_edges, + indx_2_module_map, + node_2_layer_map, + nodes_io_shapes, + ) + + # Process dvs->pooling edges adding pooling to a DVS Layer + elif dvs_pooling_edges: + # Make sure there is exactly one dvs->pooling edge + if len(dvs_pooling_edges) > 1: + raise InvalidGraphStructure( + "DVSLayer has connects to multiple pooling layers. Unlike " + "with CNN layers, pooling of the DVS has to be the same for " + "all destinations, therefore the DVSLayer can connect to at " + "most one pooling layer." + ) + dvs_pooling_edge = dvs_pooling_edges.pop() + # Find pooling-weight edges that connect DVS layer to dynapcnn layers. + pooling_weight_edges = edges_by_type.get("pooling-weight", set()) + dvs_pooling_weight_edges = find_edges_by_source( + pooling_weight_edges, dvs_pooling_edge[1] + ) + # Remove handled pooling-weight edges + pooling_weight_edges.difference_update(dvs_pooling_weight_edges) + + return init_dvs_entry_with_pooling( + dvs_pooling_edge, + dvs_pooling_weight_edges, + indx_2_module_map, + node_2_layer_map, + nodes_io_shapes, + ) + else: + # If no edges related to DVS have been found return None + return + + +def init_dvs_entry( + dvs_weight_edges: Set[Edge], indx_2_module_map: Dict[int, nn.Module], node_2_layer_map: Dict[int, int], nodes_io_shapes: Dict[int, Dict[str, Tuple[Size, Size]]], ) -> Dict: - """Initiate or update dict to hold information for a DVS Layer configuration based on a "dvs-weight" edges. - Change `dynapcnn_layer_info` in-place. If a entry for the DVS node exists the function will add a new entry - to the `desctinations` key of its dictionary. + """Initiate dict to hold information for a DVS Layer configuration + based on "dvs-weight" edges. Parameters ---------- - edge: Tuple of 2 integers, indicating edge between two nodes in graph. + dvs_weight_edges: Set of edges between two nodes in graph. + Edge source has to be a DVSLayer and the same for all edges. Edge target has to be within an existing entry of `dynapcnn_layer_info`. - dvs_layer_info: Dict containing information about the DVSLayer. If `None`, - will instantiate a new dict indx_2_module_map (dict): Maps node IDs of the graph as `key` to their associated module as `value` node_2_layer_map (dict): Maps each node ID to the ID of the layer it is assigned to. Will be updated in-place. nodes_io_shapes (dict): Map from node ID to dict containing node's in- and output shapes - """ - # This should never happen - assert isinstance( - indx_2_module_map[edge[0]], DVSLayer - ), f"Source node in edge {edge} is of type {type(DVSLayer)} (it should be a DVSLayer instance)." + Returns + ------- + dvs_layer_info: Dict containing information about the DVSLayer. + """ - # Find destination layer index - try: - destination_layer_idx = node_2_layer_map[edge[1]] - except KeyError: - weight_layer = indx_2_module_map[edge[1]] + dvs_node_id = dvs_weight_edges[0][0] + # This should never fail + if not all(edge[0] == dvs_node_id for edge in dvs_weight_edges): raise InvalidGraphStructure( - f"Weight layer {weight_layer} cannot be assigned to a dynapcnn layer. " - "This is likely due to an unsupported SNN architecture. Weight " - "layers have to be followed by a spiking layer (`sl.IAFSqueeze`)." + "The provided network seems to consist of multiple DVS alyers. " + "This is not supported." ) + assert isinstance( + (dvs_layer := indx_2_module_map[dvs_node_id]), DVSLayer + ), f"Source node in edges {dvs_weight_edges} is of type {type(dvs_layer)} (it should be a DVSLayer instance)." + + # Initialize dvs config dict + dvs_layer_info = { + "node_id": dvs_node_id, + "input_shape": nodes_io_shapes[dvs_node_id]["input"], + "module": dvs_layer, + "pooling": None, + } + node_2_layer_map[dvs_node_id] = "dvs" + + # Find destination layer indices + destinations = [] + for edge in dvs_weight_edges: + try: + destination_layer_idx = node_2_layer_map[edge[1]] + except KeyError: + weight_layer = indx_2_module_map[edge[1]] + raise InvalidGraphStructure( + f"Weight layer {weight_layer} cannot be assigned to a dynapcnn layer. " + "This is likely due to an unsupported SNN architecture. Weight " + "layers have to be followed by a spiking layer (`sl.IAFSqueeze`)." + ) - if dvs_layer_info is None: - # DVS node hasn't been initialized yet - # Init. entry for a DVS layer using its configuration dict. - dvs_layer_info = { - "node_id": edge[0], - "input_shape": nodes_io_shapes[edge[0]]["input"], - "module": indx_2_module_map[edge[0]], - "destinations": [node_2_layer_map[edge[1]]], - } + # Update entry for DVS with new destination. + assert destination_layer_idx not in destinations + destinations.append(destination_layer_idx) - node_2_layer_map[edge[0]] = "dvs" + if destinations: + dvs_layer_info["destinations"] = destinations else: + dvs_layer_info["destinations"] = None + + return dvs_layer_info + + +def init_dvs_entry_with_pooling( + dvs_pooling_edge: Edge, + pooling_weight_edges: Set[Edge], + indx_2_module_map: Dict[int, nn.Module], + node_2_layer_map: Dict[int, int], + nodes_io_shapes: Dict[int, Dict[str, Tuple[Size, Size]]], +) -> Dict: + """Initiate dict to hold information for a DVS Layer configuration with additional pooling + + Parameters + ---------- + dvs_pooling_edge: Edge from DVSLayer to pooling layer. + pooling_weight_edges: Set of edges between pooling layer and weight layer + Edge source has to be the target of `dvs_pooling_edge`. + Edge targets have to be within an existing entry of `dynapcnn_layer_info`. + indx_2_module_map (dict): Maps node IDs of the graph as `key` to their associated module as `value` + node_2_layer_map (dict): Maps each node ID to the ID of the layer it is assigned to. + Will be updated in-place. + nodes_io_shapes (dict): Map from node ID to dict containing node's in- and output shapes + + Returns + ------- + dvs_layer_info: Dict containing information about the DVSLayer. + """ + + dvs_node_id, pooling_id = dvs_pooling_edge + + # This should never fail + assert all(edge[0] == pooling_id for edge in pooling_weight_edges) + assert isinstance( + (dvs_layer := indx_2_module_map[dvs_node_id]), DVSLayer + ), f"Source node in edge {dvs_pooling_edge} is of type {type(dvs_layer)} (it should be a DVSLayer instance)." + + # Initialize dvs config dict + dvs_layer_info = { + "node_id": dvs_node_id, + "input_shape": nodes_io_shapes[dvs_node_id]["input"], + "module": dvs_layer, + "pooling": {"module": indx_2_module_map[pooling_id], "node_id": pooling_id}, + } + node_2_layer_map[dvs_node_id] = "dvs" + + # Find destination layer indices + destinations = [] + for edge in pooling_weight_edges: + try: + destination_layer_idx = node_2_layer_map[edge[1]] + except KeyError: + weight_layer = indx_2_module_map[edge[1]] + raise InvalidGraphStructure( + f"Weight layer {weight_layer} cannot be assigned to a dynapcnn layer. " + "This is likely due to an unsupported SNN architecture. Weight " + "layers have to be followed by a spiking layer (`sl.IAFSqueeze`)." + ) + # Update entry for DVS with new destination. - assert dvs_layer_info["node_id"] == edge[0] - assert destination_layer_idx not in dvs_layer_info["destinations"] + assert destination_layer_idx not in destinations + destinations.append(destination_layer_idx) - dvs_layer_info["destinations"].append(destination_layer_idx) + if destinations: + dvs_layer_info["destinations"] = destinations + else: + dvs_layer_info["destinations"] = None return dvs_layer_info @@ -858,9 +897,9 @@ def set_pooling_layer_destination( matched = True break if not matched: - poolin_layer = indx_2_module_map[edge[0]] + pooling_layer = indx_2_module_map[edge[0]] raise InvalidGraphStructure( - f"Layer {poolin_layer} cannot be assigned to a dynapcnn layer. " + f"Layer {pooling_layer} cannot be assigned to a dynapcnn layer. " "This is likely due to an unsupported SNN architecture. Pooling " "layers have to be preceded by a spiking layer (`IAFSqueeze`), " "another pooling layer, or DVS input" @@ -914,6 +953,21 @@ def trace_paths(node: int, remaining_edges: Set[Edge]) -> List[deque[int]]: return paths, processed_edges +def find_edges_by_source(edges: Set[Edge], source: int) -> Set[Edge]: + """Utility function to find all edges with a given source node. + + Parameters + ---------- + - edges: Set of `Edge` instances to be searched + - source (int): Node ID that returned edges should have as source + + Returns + ------- + - Set[Edge]: All sets from `edges` that have `source` as source + """ + return {(src, tgt) for (src, tgt) in edges if src == source} + + def verify_layer_info( dynapcnn_layer_info: Dict[int, Dict], edge_counts: Optional[Dict[str, int]] = None ): From 1a5eb272a3e46c0a3f578817daeffc62d70511db Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Fri, 8 Nov 2024 10:47:33 +0100 Subject: [PATCH 324/379] Fix issues for graphs with DVSLayer and batch norm. --- .../backend/dynapcnn/dynapcnn_layer_utils.py | 5 ++- .../backend/dynapcnn/nir_graph_extractor.py | 5 +-- .../backend/dynapcnn/sinabs_edges_handler.py | 33 ++++++++----------- sinabs/utils.py | 26 ++++++++++++++- tests/test_dynapcnn/test_individual_cases.py | 2 +- 5 files changed, 46 insertions(+), 25 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py b/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py index 2ea89afb..2e67acc8 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py @@ -1,4 +1,3 @@ -from math import prod from typing import Callable, Dict, Iterable, List, Optional, Set, Tuple, Union from torch import nn @@ -123,6 +122,10 @@ def consolidate_dvs_pooling(dvs_info: Union[Dict, None], dcnnl_map: Dict): dvs_layer.pool_layer.kernel_size = cumulative_pooling dvs_layer.pool_layer.stride = None + # Update cropping layer to account for reduced size after pooling + dvs_layer.crop_layer.bottom_crop //= added_pooling[0] + dvs_layer.crop_layer.right_crop //= added_pooling[1] + # Set rescale_factor for targeted dynapcnn layers if dvs_info["destinations"] is not None: for dest_lyr_idx in dvs_info["destinations"]: diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index d8a69434..38c8fbdb 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -10,6 +10,7 @@ import torch.nn as nn from sinabs import layers as sl +from sinabs.utils import get_new_index from .connectivity_specs import ( LAYER_TYPES_WITH_MULTIPLE_INPUTS, @@ -415,8 +416,8 @@ def _add_dvs_node(self, dvs_input_shape: Tuple[int, int, int]) -> None: f"A DVSLayer istance can have the feature dimension of its inputs with values 1 or 2 but {features} was given." ) - # add name entry for node 'dvs'. - self._name_2_indx_map["dvs"] = len(self._name_2_indx_map) + # Find new index to be assigned to DVS node + self._name_2_indx_map["dvs"] = get_new_index(self._name_2_indx_map.values()) # add module entry for node 'dvs'. self._indx_2_module_map[self._name_2_indx_map["dvs"]] = DVSLayer( input_shape=(height, width), diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index 542117be..e656b13c 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -11,9 +11,8 @@ from torch import Size, nn from sinabs.layers import SumPool2d -from sinabs.utils import expand_to_pair -from .connectivity_specs import VALID_SINABS_EDGE_TYPES, Pooling +from .connectivity_specs import VALID_SINABS_EDGE_TYPES from .crop2d import Crop2d from .dvs_layer import DVSLayer from .exceptions import ( @@ -101,33 +100,24 @@ def handle_batchnorm_nodes( # Point weight nodes to the targets of their respective batch norm nodes. new_edges = set() - for weight, bnorm in weight_bnorm_edges: + for weight_id, bnorm_id in weight_bnorm_edges: new_edges.update( remap_edges_after_drop( - dropped_node=bnorm, source_of_dropped_node=weight, edges=edges + dropped_node=bnorm_id, source_of_dropped_node=weight_id, edges=edges ) ) + # Remove all edges to and from a batch norm node and replace with new edges + bnorm_edges = {e for e in edges if bnorm_nodes.intersection(e)} + edges.difference_update(bnorm_edges) + edges.update(new_edges) # Remove references to the bnorm node. - for idx in bnorm_nodes: indx_2_module_map.pop(idx) for name in [name for name, indx in name_2_indx_map.items() if indx in bnorm_nodes]: name_2_indx_map.pop(name) - for edge in weight_bnorm_edges: - edges.remove(edge) - - for edge in [ - (src, tgt) for (src, tgt) in edges if (src in bnorm_nodes or tgt in bnorm_nodes) - ]: - edges.remove(edge) - - # Update 'edges' in-place to incorporate new edges: - for edge in new_edges: - edges.add(edge) - def fix_dvs_module_edges( edges: Set[Edge], @@ -641,11 +631,13 @@ def init_dvs_entry( dvs_layer_info: Dict containing information about the DVSLayer. """ - dvs_node_id = dvs_weight_edges[0][0] + # Pick any of the edges in set to get the DVS node ID. Should be same for all. + dvs_node_id = next(dvs_weight_edges.__iter__())[0] + # This should never fail if not all(edge[0] == dvs_node_id for edge in dvs_weight_edges): raise InvalidGraphStructure( - "The provided network seems to consist of multiple DVS alyers. " + "The provided network seems to consist of multiple DVS layers. " "This is not supported." ) assert isinstance( @@ -663,7 +655,8 @@ def init_dvs_entry( # Find destination layer indices destinations = [] - for edge in dvs_weight_edges: + while dvs_weight_edges: + edge = dvs_weight_edges.pop() try: destination_layer_idx = node_2_layer_map[edge[1]] except KeyError: diff --git a/sinabs/utils.py b/sinabs/utils.py index 8a294697..70998a18 100644 --- a/sinabs/utils.py +++ b/sinabs/utils.py @@ -1,4 +1,4 @@ -from typing import Iterable, List, Tuple, TypeVar, Union +from typing import Iterable, List, Sequence, Tuple, TypeVar, Union import numpy as np import torch @@ -7,6 +7,30 @@ import sinabs +def get_new_index(existing_indices: Sequence) -> int: + """Get a new index that is not yet part of a Sequence of existing indices + + Example: + `get_new_index([0,1,2,3])`: `4` + `get_new_index([0,1,3])`: `2` + + Parameters + ---------- + - existing_indices: Sequence of indices + + Returns + ------- + - int: Smallest number (starting from 0) that is not yet in `existing_indices`. + """ + existing_indices = set(existing_indices) + # Largest possible index is the length of `existing_indices`, if they are + # consecutively numbered. Otherwise, if there is a "gap", this would be + # filled by a smaller number. + possible_indices = range(len(existing_indices) + 1) + unused_indices = existing_indices.symmetric_difference(possible_indices) + return min(unused_indices) + + def reset_states(model: nn.Module) -> None: """Helper function to recursively reset all states of spiking layers within the model. diff --git a/tests/test_dynapcnn/test_individual_cases.py b/tests/test_dynapcnn/test_individual_cases.py index 6515cc4e..691d8776 100644 --- a/tests/test_dynapcnn/test_individual_cases.py +++ b/tests/test_dynapcnn/test_individual_cases.py @@ -221,4 +221,4 @@ def test_no_conv_layers(): ) -test_with_sinabs_batch() +test_batchnorm_after_conv() From beca062add3af175fee11c7e3d8ce9525bfe358b Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Fri, 8 Nov 2024 16:52:13 +0100 Subject: [PATCH 325/379] Fix handling of isolated layers. --- sinabs/backend/dynapcnn/connectivity_specs.py | 2 -- .../backend/dynapcnn/nir_graph_extractor.py | 29 +++++++++++++------ tests/test_dynapcnn/test_individual_cases.py | 7 ++--- 3 files changed, 22 insertions(+), 16 deletions(-) diff --git a/sinabs/backend/dynapcnn/connectivity_specs.py b/sinabs/backend/dynapcnn/connectivity_specs.py index 5de38943..3319353f 100644 --- a/sinabs/backend/dynapcnn/connectivity_specs.py +++ b/sinabs/backend/dynapcnn/connectivity_specs.py @@ -8,9 +8,7 @@ import sinabs.layers as sl -from .crop2d import Crop2d from .dvs_layer import DVSLayer -from .flipdims import FlipDims Pooling = (sl.SumPool2d, nn.AvgPool2d) Weight = (nn.Conv2d, nn.Linear) diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index 38c8fbdb..1c5f4970 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -116,9 +116,6 @@ def __init__( # Make sure DVS input is properly integrated into graph self._handle_dvs_input(input_shape=dummy_input.shape[1:], dvs_input=dvs_input) - # Verify that graph is compatible - self.verify_graph_integrity() - # retrieves what the I/O shape for each node's module is. self._nodes_io_shapes = self._get_nodes_io_shapes(dummy_input) @@ -186,6 +183,9 @@ def get_dynapcnn_network_module( # Make sure all nodes are supported self.verify_node_types() + # Verify that graph is compatible + self.verify_graph_integrity() + # create a dict holding the data necessary to instantiate a `DynapcnnLayer`. self.dcnnl_map, self.dvs_layer_info = collect_dynapcnn_layer_info( indx_2_module_map=self.indx_2_module_map, @@ -279,17 +279,28 @@ def get_node_io_shapes(self, node: int) -> Tuple[torch.Size, torch.Size]: def verify_graph_integrity(self): """Apply checks to verify that graph is supported - Currently this checks that only nodes of specific classes have - multiple sources or targets. This method might be extended in the - future to implement stricter formal verification. + Check that: + - Only nodes of specific classes have multiple sources or targets. + - There are no disconnected nodes except for `DVSLayer` instances. Raises ------ - InvalidGraphStructure: If any verification fails """ - # Iterate over all nodes, and count its sources and targets + for node, module in self.indx_2_module_map.items(): - # Check sources + # Make sure there are no individual, unconnected nodes + edges_with_node = {e for e in self.edges if node in e} + if not edges_with_node and not isinstance(module, DVSLayer): + raise InvalidGraphStructure( + f"There is an isolated module of type {type(module)}. Only " + "`DVSLayer` instances can be completely disconnected from " + "any other module. Other than that, layers for DynapCNN " + "consist of groups of weight layers (`Linear` or `Conv2d`), " + "spiking layers (`IAF` or `IAFSqueeze`), and optioanlly " + "pooling layers (`SumPool2d`, `AvgPool2d`)." + ) + # Ensure only certain module types have multiple inputs if not isinstance(module, LAYER_TYPES_WITH_MULTIPLE_INPUTS): sources = self._find_all_sources_of_input_to(node) if len(sources) > 1: @@ -298,7 +309,7 @@ def verify_graph_integrity(self): f"can have more than one input. Node {node} is of type " f"{type(module)} and has {len(sources)} inputs." ) - # Check targets + # Ensure only certain module types have multiple targets if not isinstance(module, LAYER_TYPES_WITH_MULTIPLE_OUTPUTS): targets = self._find_valid_targets(node) if len(targets) > 1: diff --git a/tests/test_dynapcnn/test_individual_cases.py b/tests/test_dynapcnn/test_individual_cases.py index 691d8776..d394da7d 100644 --- a/tests/test_dynapcnn/test_individual_cases.py +++ b/tests/test_dynapcnn/test_individual_cases.py @@ -196,9 +196,9 @@ def test_no_spk_ending(): nn.Linear(512, 2), ) - from sinabs.backend.dynapcnn.exceptions import MissingLayer + from sinabs.backend.dynapcnn.exceptions import InvalidGraphStructure - with pytest.raises(MissingLayer): + with pytest.raises(InvalidGraphStructure): DynapcnnNetwork(seq, input_shape=input_data.shape[1:], discretize=False) @@ -219,6 +219,3 @@ def test_no_conv_layers(): net = DynapcnnNetwork( nn.Sequential(DVSLayer(input_shape=(10, 10))), input_shape=(2, 10, 10) ) - - -test_batchnorm_after_conv() From 660d102ba50399b98884322f54dcc93eb79d98fb Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Fri, 8 Nov 2024 17:47:48 +0100 Subject: [PATCH 326/379] Fix issues related to DVS. Ensure only IAFSqueeze is used in DynapcnnNetwork --- sinabs/backend/dynapcnn/connectivity_specs.py | 2 +- sinabs/backend/dynapcnn/dynapcnn_layer.py | 8 +- sinabs/backend/dynapcnn/dynapcnn_network.py | 22 +++-- .../backend/dynapcnn/nir_graph_extractor.py | 11 +++ sinabs/backend/dynapcnn/utils.py | 99 ++++++++++--------- tests/test_dynapcnn/test_dvs_input.py | 11 ++- 6 files changed, 92 insertions(+), 61 deletions(-) diff --git a/sinabs/backend/dynapcnn/connectivity_specs.py b/sinabs/backend/dynapcnn/connectivity_specs.py index 3319353f..2575bc82 100644 --- a/sinabs/backend/dynapcnn/connectivity_specs.py +++ b/sinabs/backend/dynapcnn/connectivity_specs.py @@ -12,7 +12,7 @@ Pooling = (sl.SumPool2d, nn.AvgPool2d) Weight = (nn.Conv2d, nn.Linear) -Neuron = (sl.IAFSqueeze, sl.IAF) +Neuron = (sl.IAFSqueeze,) DVS = (DVSLayer,) SupportedNodeTypes = (*Pooling, *Weight, *Neuron, *DVS) diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer.py b/sinabs/backend/dynapcnn/dynapcnn_layer.py index ac5675fe..d2412640 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer.py @@ -6,10 +6,11 @@ from typing import List, Tuple import numpy as np -import sinabs.layers as sl import torch from torch import nn +import sinabs.layers as sl + from .discretize import discretize_conv_spike_ # Define sum pooling functional as power-average pooling with power 1 @@ -99,6 +100,11 @@ def __init__( self._discretize = discretize self._rescale_weights = rescale_weights + if not isinstance(spk, sl.IAFSqueeze): + raise TypeError( + f"Unsupported spiking layer type {type(spk)}. " + "Only `IAFSqueeze` layers are supported." + ) spk = deepcopy(spk) # Convert `nn.Linear` to `nn.Conv2d`. diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index aa9ff1fd..cb466a35 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -19,7 +19,12 @@ from .dynapcnn_layer import DynapcnnLayer from .io import disable_timestamps, enable_timestamps, open_device, reset_timestamps from .nir_graph_extractor import GraphExtractor -from .utils import COMPLETELY_IGNORED_LAYER_TYPES, IGNORED_LAYER_TYPES, parse_device_id +from .utils import ( + COMPLETELY_IGNORED_LAYER_TYPES, + IGNORED_LAYER_TYPES, + infer_input_shape, + parse_device_id, +) from .weight_rescaling_methods import rescale_method_1 @@ -27,7 +32,7 @@ class DynapcnnNetwork(nn.Module): def __init__( self, snn: nn.Module, - input_shape: Tuple[int, int, int], + input_shape: Optional[Tuple[int, int, int]] = None, batch_size: Optional[int] = None, dvs_input: Optional[bool] = None, discretize: bool = True, @@ -41,7 +46,9 @@ def __init__( Parameters ---------- - snn (nn.Module): a implementing a spiking network. - - input_shape (tuple): a description of the input dimensions as `(features, height, width)`. + - input_shape (tuple or None): a description of the input dimensions + as `(features, height, width)`. If `None`, `snn` must contain a + `DVSLayer` instance, from which the input shape will be inferred. - batch_size (optional int): If `None`, will try to infer the batch size from the model. If int value is provided, it has to match the actual batch size of the model. - dvs_input (bool): optional (default as `None`). Wether or not dynapcnn receive @@ -53,13 +60,14 @@ def __init__( """ super().__init__() - # check if dvs input is expected. + if isinstance(snn, sinabs.Network): + # Ignore `analog_model` of sinabs `Network` instances + snn = snn.spiking_model + self.dvs_input = dvs_input - self.input_shape = input_shape + self.input_shape = infer_input_shape(snn, input_shape) self._layer2core_map = None - assert len(self.input_shape) == 3, "infer_input_shape did not return 3-tuple" - # Infer batch size for dummpy input to graph extractor if batch_size is None: batch_size = sinabs.utils.get_smallest_compatible_time_dimension(snn) diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index 1c5f4970..5a04f7a8 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -334,6 +334,17 @@ def verify_node_types(self): unsupported_nodes[node_type].add(index) else: unsupported_nodes[node_type] = {index} + # Specific error message for non-squeezing IAF layer + iaf_layers = [] + for idx in unsupported_nodes.pop(sl.IAF, []): + iaf_layers.append(self.indx_2_module_map[idx]) + if iaf_layers: + layer_str = ", ".join(str(lyr) for lyr in (iaf_layers)) + raise UnsupportedLayerType( + f"The provided SNN contains IAF layers:\n{layer_str}.\n" + "For compatibility with torch's `nn.Conv2d` modules, please " + "use `IAFSqueeze` layers instead." + ) # Specific error message for leaky neuron types lif_layers = [] for lif_type in (sl.LIF, sl.LIFSqueeze): diff --git a/sinabs/backend/dynapcnn/utils.py b/sinabs/backend/dynapcnn/utils.py index 9514ba46..e9e919d1 100644 --- a/sinabs/backend/dynapcnn/utils.py +++ b/sinabs/backend/dynapcnn/utils.py @@ -1,12 +1,15 @@ from collections import defaultdict, deque from copy import deepcopy -from typing import TYPE_CHECKING, List, Set, Tuple, TypeVar, Union +from typing import TYPE_CHECKING, List, Optional, Set, Tuple, TypeVar, Union -import sinabs.layers as sl import torch import torch.nn as nn +import sinabs.layers as sl + from .crop2d import Crop2d +from .dvs_layer import DVSLayer +from .exceptions import InputConfigurationError if TYPE_CHECKING: from sinabs.backend.dynapcnn.dynapcnn_network import DynapcnnNetwork @@ -275,49 +278,49 @@ def extend_readout_layer(model: "DynapcnnNetwork") -> "DynapcnnNetwork": return model -####################################################### DEPRECATED METHODS ####################################################### -# TODO: these methods were used by the old implementation of DynapcnnNetwork - delete all. - -# def infer_input_shape( -# layers: List[nn.Module], input_shape: Optional[Tuple[int, int, int]] = None -# ) -> Tuple[int, int, int]: -# """Checks if the input_shape is specified. If either of them are specified, then it checks if -# the information is consistent and returns the input shape. - -# Parameters -# ---------- -# layers: -# List of modules -# input_shape : -# (channels, height, width) - -# Returns -# ------- -# Output shape: -# (channels, height, width) -# """ -# if input_shape is not None and len(input_shape) != 3: -# raise InputConfigurationError( -# f"input_shape expected to have length 3 or None but input_shape={input_shape} given." -# ) - -# input_shape_from_layer = None -# if layers and isinstance(layers[0], DVSLayer): -# input_shape_from_layer = layers[0].input_shape -# if len(input_shape_from_layer) != 3: -# raise InputConfigurationError( -# f"input_shape of layer {layers[0]} expected to have length 3 or None but input_shape={input_shape_from_layer} found." -# ) -# if (input_shape is not None) and (input_shape_from_layer is not None): -# if input_shape == input_shape_from_layer: -# return input_shape -# else: -# raise InputConfigurationError( -# f"Input shape from the layer {input_shape_from_layer} does not match the specified input_shape {input_shape}" -# ) -# elif input_shape_from_layer is not None: -# return input_shape_from_layer -# elif input_shape is not None: -# return input_shape -# else: -# raise InputConfigurationError("No input shape could be inferred") +def infer_input_shape( + snn: nn.Module, input_shape: Optional[Tuple[int, int, int]] = None +) -> Tuple[int, int, int]: + """Infer expected shape of input for `snn` either from `input_shape` + or from `DVSLayer` instance within `snn` which provides it. + + If neither are available, raise an InputConfigurationError. + If both are the case, verify that the information is consistent. + + Parameters + ---------- + - snn (nn.Module): The SNN whose input shape is to be inferred + - input_shape (tuple or None): Explicitly provide input shape. + If not None, must be of the format `(channels, height, width)`. + + Returns + ------- + - tuple: The input shape to `snn`, in the format `(channels, height, width)` + """ + if input_shape is not None and len(input_shape) != 3: + raise InputConfigurationError( + f"input_shape expected to have length 3 or None but input_shape={input_shape} given." + ) + + # Find `DVSLayer` instance and infer input shape from it + input_shape_from_layer = None + for module in snn.modules(): + if isinstance(module, DVSLayer): + input_shape_from_layer = module.input_shape + # Make sure `input_shape_from_layer` is identical to provided `input_shape` + if input_shape is not None and input_shape != input_shape_from_layer: + raise InputConfigurationError( + f"Input shape from `DVSLayer` {input_shape_from_layer} does " + f"not match the specified input_shape {input_shape}" + ) + return input_shape_from_layer + + # If no `DVSLayer` is found, `input_shape` must not be provided + if input_shape is None: + raise InputConfigurationError( + "No input shape could be inferred. Either provide it explicitly " + "with the `input_shape` argument, or provide a model with " + "`DVSLayer` instance." + ) + else: + return input_shape diff --git a/tests/test_dynapcnn/test_dvs_input.py b/tests/test_dynapcnn/test_dvs_input.py index 014ccb1c..e72d6dbc 100644 --- a/tests/test_dynapcnn/test_dvs_input.py +++ b/tests/test_dynapcnn/test_dvs_input.py @@ -13,7 +13,7 @@ from sinabs.backend.dynapcnn.dvs_layer import DVSLayer from sinabs.backend.dynapcnn.exceptions import * from sinabs.from_torch import from_model -from sinabs.layers import IAF +from sinabs.layers import IAFSqueeze INPUT_SHAPE = (2, 16, 16) input_data = torch.rand(1, *INPUT_SHAPE, requires_grad=False) * 100.0 @@ -42,9 +42,9 @@ def verify_dvs_config( return if destination is None: - assert dvs.destinations[0].enable == False + assert not dvs.destinations[0].enable else: - assert dvs.destinations[0].enable == True + assert dvs.destinations[0].enable assert dvs.destinations[0].layer == destination if cut is None: assert dvs.cut.y == origin[0] + INPUT_SHAPE[1] // pooling[0] - 1 @@ -186,7 +186,7 @@ def __init__( **kwargs_flip, ), nn.Conv2d(n_channels_in, 4, kernel_size=2, stride=2), - IAF(), + IAFSqueeze(batch_size=1), ] self.seq = nn.Sequential(*layers) @@ -301,3 +301,6 @@ def test_whether_dvs_mirror_cfg_is_all_switched_off(dvs_input, pool): assert samna_cfg.dvs_layer.mirror.x is False assert samna_cfg.dvs_layer.mirror.y is False assert samna_cfg.dvs_layer.mirror_diagonal is False + + +test_dvs_crop(False, False) From afab026d8fe2901d48bdcab6d96166fecabb81b7 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Fri, 8 Nov 2024 18:26:06 +0100 Subject: [PATCH 327/379] Correctly handle dvs_input False when dvs layer is provided: Disable pixel array. --- sinabs/backend/dynapcnn/chips/dynapcnn.py | 5 ++-- sinabs/backend/dynapcnn/dynapcnn_network.py | 25 ++++++++++++------ .../backend/dynapcnn/nir_graph_extractor.py | 26 ++++++++++++++----- tests/test_dynapcnn/test_dvs_input.py | 3 --- 4 files changed, 39 insertions(+), 20 deletions(-) diff --git a/sinabs/backend/dynapcnn/chips/dynapcnn.py b/sinabs/backend/dynapcnn/chips/dynapcnn.py index 543e4e52..dcc09663 100644 --- a/sinabs/backend/dynapcnn/chips/dynapcnn.py +++ b/sinabs/backend/dynapcnn/chips/dynapcnn.py @@ -3,13 +3,14 @@ from warnings import warn import samna -import sinabs import torch from samna.dynapcnn.configuration import ( CNNLayerConfig, DVSLayerConfig, DynapcnnConfiguration, ) + +import sinabs from sinabs.backend.dynapcnn.config_builder import ConfigBuilder from sinabs.backend.dynapcnn.dvs_layer import DVSLayer from sinabs.backend.dynapcnn.dynapcnn_layer import DynapcnnLayer @@ -57,7 +58,7 @@ def write_dvs_layer_config( chip_layer.destinations[dest_idx].layer = layer2core_map[dest] chip_layer.destinations[dest_idx].enable = True - chip_layer.pass_sensor_events = True + chip_layer.pass_sensor_events = not layer.disable_pixel_array if layer.merge_polarities: chip_layer.merge = True diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index cb466a35..bebcd53d 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -2,6 +2,7 @@ # contact : wsoaresgirao@gmail.com import time +from pprint import pformat from typing import Callable, Dict, List, Optional, Sequence, Tuple, Union from warnings import warn @@ -561,11 +562,11 @@ def _make_config( config_builder = ChipFactory(device).get_config_builder() if chip_layers_ordering is not None: - if layer2core_map is not None: + if layer2core_map != "auto": warn( "Both `chip_layers_ordering` and `layer2core_map are provided. " - "Please only provide `layer2core_map`, as `chip_layers_ordering` " - "is deprecated.", + "The parameter `chip_layers_ordering` is deprecated and will " + "be ignored.", DeprecationWarning, ) elif chip_layers_ordering == "auto": @@ -575,12 +576,20 @@ def _make_config( "`layer2core_map` instead.", DeprecationWarning, ) - layer2core_map = "auto" else: - raise ValueError( - "`chip_layers_ordering` is deprecated. Passing anything other " - "than `None` or 'auto' is not possible. To manually assign core " - "to layers, please use the `layer2core_map` argument." + layer2core_map = { + idx: core + for idx, core in zip(self.dynapcnn_layers, chip_layers_ordering) + } + warn( + "The parameter `chip_layers_ordering` is deprecated. " + "Because `layer2core_map` is 'auto', and `chip_layers_ordering` " + "is not, will convert `chip_layers_ordering` to a " + "dict matching `layer2core_map`. In the future please use " + "`layer2core_map` instead. Please make sure the inferred" + "mapping from DynapcnnLayer index to core index is correct:" + + pformat(layer2core_map), + DeprecationWarning, ) if layer2core_map == "auto": # Assign chip core ID for each DynapcnnLayer. diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index 5a04f7a8..94579241 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -402,12 +402,17 @@ def _handle_dvs_input( """ if self.has_dvs_layer: # Make a copy of the layer so that the original version is not - # change in place + # changed in place new_dvs_layer = deepcopy(self.dvs_layer) self.name_2_indx_map[self.dvs_layer_index] = new_dvs_layer elif dvs_input: # Insert a DVSLayer node in the graph. - self._add_dvs_node(dvs_input_shape=input_shape) + new_dvs_layer = self._add_dvs_node(dvs_input_shape=input_shape) + else: + dvs_input = None + if dvs_input is not None: + # Disable pixel array if `dvs_input` is False + new_dvs_layer.disable_pixel_array = not dvs_input # Check for the need of fixing NIR edges extraction when DVS is a node in the graph. If DVS # is used its node becomes the only entry node in the graph. @@ -421,7 +426,7 @@ def _handle_dvs_input( # Check if graph structure and DVSLayer.merge_polarities are correctly set (if DVS node exists). self._validate_dvs_setup(dvs_input_shape=input_shape) - def _add_dvs_node(self, dvs_input_shape: Tuple[int, int, int]) -> None: + def _add_dvs_node(self, dvs_input_shape: Tuple[int, int, int]) -> DVSLayer: """In-place modification of `self._name_2_indx_map`, `self._indx_2_module_map`, and `self._edges` to accomodate the creation of an extra node in the graph representing the DVS camera of the chip. The DVSLayer node will point to every other node that is up to this point an entry node of the original graph, so `self._entry_nodes` is modified in-place @@ -429,7 +434,10 @@ def _add_dvs_node(self, dvs_input_shape: Tuple[int, int, int]) -> None: Parameters ---------- - - dvs_input_shape (tuple): shape of the DVSLayer input in format `(features, height, width)`. + - dvs_input_shape (tuple): shape of the DVSLayer input in format `(features, height, width)` + + Returns + - DVSLayer: A handler to the newly added `DVSLayer` instance """ (features, height, width) = dvs_input_shape @@ -441,20 +449,24 @@ def _add_dvs_node(self, dvs_input_shape: Tuple[int, int, int]) -> None: # Find new index to be assigned to DVS node self._name_2_indx_map["dvs"] = get_new_index(self._name_2_indx_map.values()) # add module entry for node 'dvs'. - self._indx_2_module_map[self._name_2_indx_map["dvs"]] = DVSLayer( + dvs_layer = DVSLayer( input_shape=(height, width), merge_polarities=(features == 1), ) - # set DVS node as input to each entry node of the graph. + self._indx_2_module_map[self._name_2_indx_map["dvs"]] = dvs_layer + + # set DVS node as input to each entry node of the graph self._edges.update( { (self._name_2_indx_map["dvs"], entry_node) for entry_node in self._entry_nodes } ) - # DVSLayer node becomes the only entrypoint of the graph. + # DVSLayer node becomes the only entrypoint of the graph self._entry_nodes = {self._name_2_indx_map["dvs"]} + return dvs_layer + def _get_dvs_layer_index(self) -> Union[int, None]: """Loop though all modules and return index of `DVSLayer` instance if it exists. diff --git a/tests/test_dynapcnn/test_dvs_input.py b/tests/test_dynapcnn/test_dvs_input.py index e72d6dbc..67df83ea 100644 --- a/tests/test_dynapcnn/test_dvs_input.py +++ b/tests/test_dynapcnn/test_dvs_input.py @@ -301,6 +301,3 @@ def test_whether_dvs_mirror_cfg_is_all_switched_off(dvs_input, pool): assert samna_cfg.dvs_layer.mirror.x is False assert samna_cfg.dvs_layer.mirror.y is False assert samna_cfg.dvs_layer.mirror_diagonal is False - - -test_dvs_crop(False, False) From eb173f13776d26193cc9d8c71f79888c6719fd3f Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Fri, 8 Nov 2024 18:28:33 +0100 Subject: [PATCH 328/379] Improve docstring of DynapcnnNetwork to explain behavior of `dvs_input`. --- sinabs/backend/dynapcnn/dynapcnn_network.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index bebcd53d..3316dcd8 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -53,7 +53,10 @@ def __init__( - batch_size (optional int): If `None`, will try to infer the batch size from the model. If int value is provided, it has to match the actual batch size of the model. - dvs_input (bool): optional (default as `None`). Wether or not dynapcnn receive - input from its DVS camera. + input from its DVS camera. If a `DVSLayer` is part of `snn` and `dvs_input` is + false, the DVS sensor will be configured but its output will not be sent as input + to the chip. If `dvs_input` is `True` and `snn` does not contain a `DVSLayer`, + it will be added. - discretize (bool): If `True`, discretize the parameters and thresholds. This is needed for uploading weights to dynapcnn. Set to `False` only for testing purposes. - weight_rescaling_fn (callable): a method that handles how the re-scaling factor for one or more `SumPool2d` projecting to From 189cfb860502b8abf7dc1025413d9e9ab75908ec Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Fri, 8 Nov 2024 18:37:45 +0100 Subject: [PATCH 329/379] WIP: Fix DVS input unit tests. --- sinabs/backend/dynapcnn/dynapcnn_network.py | 9 +++++---- tests/test_dynapcnn/test_dvs_input.py | 9 ++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index 3316dcd8..32932fca 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -106,6 +106,10 @@ def all_layers(self): def dvs_node_info(self): return self._dynapcnn_module.dvs_node_info + @property + def dvs_layer(self): + return self._dynapcnn_module.dvs_layer + @property def chip_layers_ordering(self): warn( @@ -498,10 +502,7 @@ def has_dvs_layer(self) -> bool: ------- bool: True if DVSLayer is found within the network. """ - for layer in self.dynapcnn_layers.values(): - if isinstance(layer, DVSLayer): - return True - return False + return self.dvs_layer is not None ####################################################### Private Methods ####################################################### diff --git a/tests/test_dynapcnn/test_dvs_input.py b/tests/test_dynapcnn/test_dvs_input.py index 67df83ea..0eae07c3 100644 --- a/tests/test_dynapcnn/test_dvs_input.py +++ b/tests/test_dynapcnn/test_dvs_input.py @@ -84,8 +84,7 @@ def __init__(self, input_layer: bool = False): super().__init__() layers = [] layers += [ - nn.AvgPool2d(kernel_size=(2, 2)), - nn.AvgPool2d(kernel_size=(1, 2)), + nn.AvgPool2d(kernel_size=(2, 4)), nn.Conv2d(2, 4, kernel_size=2, stride=2), nn.ReLU(), ] @@ -106,7 +105,7 @@ def test_dvs_no_pooling(dvs_input): spn = DynapcnnNetwork(snn, dvs_input=dvs_input, input_shape=INPUT_SHAPE) # If there is no pooling, a DVSLayer should only be added if `dvs_input` is True - assert isinstance(spn.sequence[0], DVSLayer) == dvs_input + assert spn.has_dvs_layer() == dvs_input # - Make sure missing input shapes cause exception with pytest.raises(InputConfigurationError): @@ -139,8 +138,8 @@ def test_dvs_pooling_2d(dvs_input): # - SPN generation spn = DynapcnnNetwork(snn, dvs_input=dvs_input, input_shape=INPUT_SHAPE) - # When there is pooling, a DVSLayer should also be added if `dvs_input` is True - assert isinstance(spn.sequence[0], DVSLayer) + # When there is pooling, a DVSLayer should also be added if `dvs_input` is False + assert spn.has_dvs_layer() # - Make sure missing input shapes cause exception with pytest.raises(InputConfigurationError): From a2bbb4c64f9ae2ce2a688b0974c9495c2acf041c Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Fri, 8 Nov 2024 18:41:40 +0100 Subject: [PATCH 330/379] (WIP): Fix doorbell tests. --- tests/test_dynapcnn/test_doorbell.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_dynapcnn/test_doorbell.py b/tests/test_dynapcnn/test_doorbell.py index 36ee8955..e130c694 100644 --- a/tests/test_dynapcnn/test_doorbell.py +++ b/tests/test_dynapcnn/test_doorbell.py @@ -3,6 +3,7 @@ It will include testing of the network equivalence, and of the correct output configuration. """ +import pytest import samna import torch from nirtorch.utils import sanitize_name @@ -76,7 +77,9 @@ def test_same_result(): def test_auto_config(): # - Should give an error with the normal layer ordering dynapcnn_net = DynapcnnNetwork(snn, input_shape=input_shape, discretize=True) - dynapcnn_net.make_config(chip_layers_ordering=[0, 1, 2, 3, 4]) + with pytest.raises(ValueError): + dynapcnn_net.make_config(chip_layers_ordering=[0, 1, 2, 3, 4]) + dynapcnn_net.make_config(layer2core_map="auto") def test_was_copied(): From eec090d87e590fd99c659133c2a8a78a7f9c1f2a Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Fri, 8 Nov 2024 22:07:56 +0100 Subject: [PATCH 331/379] Fix doorbell test --- tests/test_dynapcnn/test_doorbell.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_dynapcnn/test_doorbell.py b/tests/test_dynapcnn/test_doorbell.py index e130c694..a62f96ea 100644 --- a/tests/test_dynapcnn/test_doorbell.py +++ b/tests/test_dynapcnn/test_doorbell.py @@ -85,7 +85,9 @@ def test_auto_config(): def test_was_copied(): # - Make sure that layers of different models are distinct objects # "Sanitize" all layer names, for compatibility with older nirtorch versions - snn_layers = {sanitize_name(name): lyr for name, lyr in snn.named_modules()} + snn_layers = { + sanitize_name(name): lyr for name, lyr in snn.spiking_model.named_modules() + } idx_2_name_map = { idx: sanitize_name(name) for name, idx in dynapcnn_net.name_2_indx_map.items() } From 876962f08922970739bce89fba5af3b50eb7ae01 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Fri, 8 Nov 2024 22:22:51 +0100 Subject: [PATCH 332/379] Sort dynapcnn layers in network by key --- .../dynapcnn/dynapcnnnetwork_module.py | 4 +-- tests/test_dynapcnn/test_dvs_layer.py | 31 ------------------- tests/test_dynapcnn/test_large_net.py | 4 ++- 3 files changed, 5 insertions(+), 34 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py index b471c62d..16a728c8 100644 --- a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py +++ b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py @@ -92,8 +92,8 @@ def destination_map(self): @property def dynapcnn_layers(self): - # Convert string-indices to integer-indices - return {int(idx): lyr for idx, lyr in self._dynapcnn_layers.items()} + # Convert string-indices to integer-indices and sort by index + return {int(idx): lyr for idx, lyr in sorted(self._dynapcnn_layers.items())} @property def entry_points(self): diff --git a/tests/test_dynapcnn/test_dvs_layer.py b/tests/test_dynapcnn/test_dvs_layer.py index a6ada999..2ab12550 100644 --- a/tests/test_dynapcnn/test_dvs_layer.py +++ b/tests/test_dynapcnn/test_dvs_layer.py @@ -77,37 +77,6 @@ def test_from_layers(disable_pixel_array, num_channels): assert dvs_layer.get_roi() == ((0, 59), (0, 54)) -def test_construct_empty(): - from sinabs.backend.dynapcnn.utils import construct_dvs_layer - - layers = [] - - dvs_layer, layer_idx_next, rescale_factor = construct_dvs_layer( - layers, input_shape=(2, 128, 128) - ) - - assert rescale_factor == 1 - assert layer_idx_next == 0 - assert dvs_layer is None - - -def test_construct_from_sumpool(): - import sinabs.layers as sl - from sinabs.backend.dynapcnn.utils import construct_dvs_layer - - layers = [sl.SumPool2d(2), sl.Cropping2dLayer(((1, 1), (1, 1)))] - - dvs_layer, layer_idx_next, rescale_factor = construct_dvs_layer( - layers, input_shape=(2, 128, 128) - ) - - print(dvs_layer) - - assert rescale_factor == 1 - assert layer_idx_next == 2 - assert dvs_layer.get_roi() == ((1, 63), (1, 63)) - - def test_convert_cropping2dlayer_to_crop2d(): import sinabs.layers as sl from sinabs.backend.dynapcnn.utils import convert_cropping2dlayer_to_crop2d diff --git a/tests/test_dynapcnn/test_large_net.py b/tests/test_dynapcnn/test_large_net.py index c8ea5190..1c5f7bc6 100644 --- a/tests/test_dynapcnn/test_large_net.py +++ b/tests/test_dynapcnn/test_large_net.py @@ -102,7 +102,9 @@ def test_was_copied(): from nirtorch.utils import sanitize_name # - Make sure that layers of different models are distinct objects - snn_layers = {sanitize_name(name): lyr for name, lyr in snn.named_modules()} + snn_layers = { + sanitize_name(name): lyr for name, lyr in snn.spiking_model.named_modules() + } idx_2_name_map = { idx: sanitize_name(name) for name, idx in dynapcnn_net.name_2_indx_map.items() } From 9460edc1d89daabed7eac685b28b39f0905e0697 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Fri, 8 Nov 2024 23:09:18 +0100 Subject: [PATCH 333/379] Further bugfixes and improved readability of dynapcnn network repr. --- sinabs/backend/dynapcnn/dynapcnn_layer.py | 20 ++++------ sinabs/backend/dynapcnn/dynapcnn_network.py | 37 +++++++++++++++++-- .../dynapcnn/dynapcnnnetwork_module.py | 7 +++- .../backend/dynapcnn/nir_graph_extractor.py | 30 ++++++++++++--- tests/test_dynapcnn/test_doorbell.py | 4 +- tests/test_dynapcnnnetwork/test_failcases.py | 8 ++-- 6 files changed, 77 insertions(+), 29 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer.py b/sinabs/backend/dynapcnn/dynapcnn_layer.py index d2412640..ea573701 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer.py @@ -96,7 +96,7 @@ def __init__( super().__init__() self.in_shape = in_shape - self._pool = pool + self.pool = pool self._discretize = discretize self._rescale_weights = rescale_weights @@ -135,20 +135,16 @@ def __init__( if self._discretize: conv, spk = discretize_conv_spike_(conv, spk, to_int=False) - self._conv = conv - self._spk = spk + self.conv = conv + self.spk = spk @property def conv_layer(self): - return self._conv + return self.conv @property def spk_layer(self): - return self._spk - - @property - def pool(self): - return self._pool + return self.spk @property def discretize(self): @@ -175,7 +171,7 @@ def forward(self, x) -> List[torch.Tensor]: x = self.conv_layer(x) x = self.spk_layer(x) - for pool in self._pool: + for pool in self.pool: if pool == 1: # no pooling is applied. @@ -215,7 +211,7 @@ def get_output_shape(self) -> List[Tuple[int, int, int]]: neuron_shape = self.get_neuron_shape() # this is the actual output shape, including pooling output_shape = [] - for pool in self._pool: + for pool in self.pool: output_shape.append( neuron_shape[0], neuron_shape[1] // pool, @@ -227,7 +223,7 @@ def summary(self) -> dict: """Returns a summary of the convolution's/pooling's kernel sizes and the output shape of the spiking layer.""" return { - "pool": (self._pool), + "pool": (self.pool), "kernel": list(self.conv_layer.weight.data.shape), "neuron": self._get_conv_output_shape(), # neuron layer output has the same shape as the convolution layer ouput. } diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index 32932fca..d848e39f 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -132,6 +132,13 @@ def exit_layers(self): self.dynapcnn_layers[i] for i in self._dynapcnn_module.get_exit_layers() ] + @property + def is_deployed_on_dynapcnn_device(self): + return ( + hasattr(self, "device") + and parse_device_id(self.device)[0] in ChipFactory.supported_devices + ) + @property def layer_destination_map(self): return self._dynapcnn_module.destination_map @@ -212,10 +219,7 @@ def forward( structure as if `return_complete` is `True`, but only with entries where the destination is marked as final. """ - if ( - hasattr(self, "device") - and parse_device_id(self.device)[0] in ChipFactory.supported_devices - ): + if self.is_deployed_on_dynapcnn_device: return self.hw_forward(x) else: # Forward pass through software DynapcnnLayer instance @@ -654,12 +658,37 @@ def _to_device(self, device: torch.device) -> None: def __str__(self): pretty_print = "" + if self.dvs_layer is not None: + pretty_print += ( + "-------------------------- [ DVSLayer ] --------------------------\n" + ) + pretty_print += f"{self.dvs_layer}\n\n" for idx, layer_data in self.dynapcnn_layers.items(): pretty_print += f"----------------------- [ DynapcnnLayer {idx} ] -----------------------\n" + if self.is_deployed_on_dynapcnn_device: + pretty_print += f"Core {self.layer2core_map[idx]}\n" pretty_print += f"{layer_data}\n\n" return pretty_print + def __repr__(self): + if self.is_deployed_on_dynapcnn_device: + layer_info = "\n\n".join( + f"{idx} - core: {self.layer2core_map[idx]}\n{pformat(layer)}" + for idx, layer in self.dynapcnn_layers.items() + ) + device_info = f" deployed on {self.device}," + else: + layer_info = "\n\n".join( + f"Index: {idx}\n{pformat(layer)}" + for idx, layer in self.dynapcnn_layers.items() + ) + device_info = f" on {self.device}," if hasattr(self, "device") else "" + return ( + f"DynapCNN Network{device_info} containing:\nDVS Layer: {pformat(self.dvs_layer)}" + "\n\nDynapCNN Layers:\n\n" + layer_info + ) + class DynapcnnCompatibleNetwork(DynapcnnNetwork): """Deprecated class, use DynapcnnNetwork instead.""" diff --git a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py index 16a728c8..bd03d5d7 100644 --- a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py +++ b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py @@ -1,7 +1,7 @@ # author : Willian Soares Girao # contact : wsoaresgirao@gmail.com -from collections import defaultdict +from pprint import pformat from typing import Dict, List, Optional, Set, Union from warnings import warn @@ -372,3 +372,8 @@ def remap(key): remap(node): [remap(src) for src in sources] for node, sources in self._node_source_map.items() } + + def __repr__(self): + return f"DVS Layer: {pformat(self.dvs_layer)}\n\nDynapCNN Layers:\n" + pformat( + self.dynapcnn_layers + ) diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index 94579241..51d12c17 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -119,6 +119,9 @@ def __init__( # retrieves what the I/O shape for each node's module is. self._nodes_io_shapes = self._get_nodes_io_shapes(dummy_input) + # Verify that graph is compatible + self.verify_graph_integrity() + ####################################################### Publich Methods ####################################################### @property @@ -180,11 +183,9 @@ def get_dynapcnn_network_module( - The DynapcnnNetworkModule based on graph representation of this `GraphExtractor` """ - # Make sure all nodes are supported + # Make sure all nodes are supported and there are no isolated nodes. self.verify_node_types() - - # Verify that graph is compatible - self.verify_graph_integrity() + self.verify_no_isolated_nodes() # create a dict holding the data necessary to instantiate a `DynapcnnLayer`. self.dcnnl_map, self.dvs_layer_info = collect_dynapcnn_layer_info( @@ -281,7 +282,6 @@ def verify_graph_integrity(self): Check that: - Only nodes of specific classes have multiple sources or targets. - - There are no disconnected nodes except for `DVSLayer` instances. Raises ------ @@ -379,6 +379,26 @@ def verify_node_types(self): f"{pformat(SupportedNodeTypes)}." ) + def verify_no_isolated_nodes(self): + """Verify that there are no disconnected nodes except for `DVSLayer` instances. + + Raises + ------ + - InvalidGraphStructure when disconnected nodes are detected + """ + for node, module in self.indx_2_module_map.items(): + # Make sure there are no individual, unconnected nodes + edges_with_node = {e for e in self.edges if node in e} + if not edges_with_node and not isinstance(module, DVSLayer): + raise InvalidGraphStructure( + f"There is an isolated module of type {type(module)}. Only " + "`DVSLayer` instances can be completely disconnected from " + "any other module. Other than that, layers for DynapCNN " + "consist of groups of weight layers (`Linear` or `Conv2d`), " + "spiking layers (`IAF` or `IAFSqueeze`), and optioanlly " + "pooling layers (`SumPool2d`, `AvgPool2d`)." + ) + ####################################################### Pivate Methods ####################################################### def _handle_dvs_input( diff --git a/tests/test_dynapcnn/test_doorbell.py b/tests/test_dynapcnn/test_doorbell.py index a62f96ea..4c78ac80 100644 --- a/tests/test_dynapcnn/test_doorbell.py +++ b/tests/test_dynapcnn/test_doorbell.py @@ -75,10 +75,8 @@ def test_same_result(): def test_auto_config(): - # - Should give an error with the normal layer ordering dynapcnn_net = DynapcnnNetwork(snn, input_shape=input_shape, discretize=True) - with pytest.raises(ValueError): - dynapcnn_net.make_config(chip_layers_ordering=[0, 1, 2, 3, 4]) + dynapcnn_net.make_config(chip_layers_ordering=[0, 1, 2, 3, 4]) dynapcnn_net.make_config(layer2core_map="auto") diff --git a/tests/test_dynapcnnnetwork/test_failcases.py b/tests/test_dynapcnnnetwork/test_failcases.py index 125b1dff..846a76ad 100644 --- a/tests/test_dynapcnnnetwork/test_failcases.py +++ b/tests/test_dynapcnnnetwork/test_failcases.py @@ -47,14 +47,14 @@ def test_missing_spiking_layer(): in_shape = (2, 28, 28) snn = nn.Sequential( nn.Conv2d(2, 8, kernel_size=3, stride=1, bias=False), - sl.IAF(), + sl.IAFSqueeze(batch_size=1), sl.SumPool2d(2), nn.AvgPool2d(2), nn.Conv2d(8, 16, kernel_size=3, stride=1, bias=False), - sl.IAF(), + sl.IAFSqueeze(batch_size=1), nn.Dropout2d(), nn.Conv2d(16, 2, kernel_size=3, stride=1, bias=False), - sl.IAF(), + sl.IAFSqueeze(batch_size=1), nn.Flatten(), nn.Linear(8, 5), ) @@ -66,7 +66,7 @@ def test_missing_spiking_layer(): def test_incorrect_model_start(): in_shape = (2, 28, 28) snn = nn.Sequential( - sl.IAF(), + sl.IAFSqueeze(batch_size=1), sl.SumPool2d(2), nn.AvgPool2d(2), ) From 8897d248d7d11ca801c80d4eac41d012a89a5706 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Fri, 8 Nov 2024 23:30:01 +0100 Subject: [PATCH 334/379] Fix monitoring. Enable monitoring exit layers with -1 --- sinabs/backend/dynapcnn/dynapcnn_network.py | 12 ++++++++++- tests/test_dynapcnn/test_monitoring.py | 24 ++++++++++----------- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index d848e39f..2d1ef082 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -132,6 +132,10 @@ def exit_layers(self): self.dynapcnn_layers[i] for i in self._dynapcnn_module.get_exit_layers() ] + @property + def exit_layer_ids(self): + return self._dynapcnn_module.get_exit_layers() + @property def is_deployed_on_dynapcnn_device(self): return ( @@ -312,6 +316,7 @@ def to( monitor_layers = ["dvs"] # If you want to monitor the output of the pre-processing layer monitor_layers = ["dvs", 8] # If you want to monitor preprocessing and layer 8 monitor_layers = "all" # If you want to monitor all the layers + monitor_layers = [-1] # If you want to only monitor exit points of the network (i.e. final layers) config_modifier: A user configuration modifier method. @@ -454,6 +459,7 @@ def make_config( monitor_layers = ["dvs"] # If you want to monitor the output of the pre-processing layer monitor_layers = ["dvs", 8] # If you want to monitor preprocessing and layer 8 monitor_layers = "all" # If you want to monitor all the layers + monitor_layers = [-1] # If you want to only monitor exit points of the network (i.e. final layers) If this value is left as None, by default the last layer of the model is monitored. @@ -537,6 +543,7 @@ def _make_config( monitor_layers = ["dvs"] # If you want to monitor the output of the pre-processing layer monitor_layers = ["dvs", 8] # If you want to monitor preprocessing and layer 8 monitor_layers = "all" # If you want to monitor all the layers + monitor_layers = [-1] # If you want to only monitor exit points of the network (i.e. final layers) If this value is left as None, by default the last layer of the model is monitored. @@ -621,13 +628,16 @@ def _make_config( if monitor_layers is None: # Monitor all layers with exit point destinations monitor_layers = self._dynapcnn_module.get_exit_layers() - elif monitor_layers == "all": monitor_layers = [ lyr_idx for lyr_idx, layer in self.dynapcnn_layers.items() if not isinstance(layer, DVSLayer) ] + elif -1 in monitor_layers: + # Replace `-1` with exit layer IDs + monitor_layers.remove(-1) + monitor_layers += self._dynapcnn_module.get_exit_layers() # Collect cores (chip layers) that are to be monitored monitor_chip_layers = [] diff --git a/tests/test_dynapcnn/test_monitoring.py b/tests/test_dynapcnn/test_monitoring.py index aaad2031..cb589aca 100644 --- a/tests/test_dynapcnn/test_monitoring.py +++ b/tests/test_dynapcnn/test_monitoring.py @@ -72,15 +72,14 @@ def test_default_monitoring(): # As a default the last layer should be monitored config = dynapcnn_net.make_config(device="speck2b:0") - clo = dynapcnn_net.chip_layers_ordering - assert len(clo) > 0 + l2c = dynapcnn_net.layer2core_map + assert len(l2c) > 0 # Check that monitoring is off for all layers except last - for layer in clo[:-1]: - if layer == "dvs": - assert config.dvs_layer.monitor_enable == False + for layer, core in l2c.items(): + if layer in dynapcnn_net.exit_layer_ids: + assert config.cnn_layers[core].monitor_enable == True else: - assert config.cnn_layers[layer].monitor_enable == False - assert config.cnn_layers[clo[-1]].monitor_enable == True + assert config.cnn_layers[core].monitor_enable == False def test_model_level_monitoring_enable(): @@ -98,12 +97,13 @@ def test_model_level_monitoring_enable(): config = dynapcnn_net.make_config( device="speck2b:0", monitor_layers=["dvs", 5, -1] ) - clo = dynapcnn_net.chip_layers_ordering - assert len(clo) > 0 + l2c = dynapcnn_net.layer2core_map + assert len(l2c) > 0 assert config.dvs_layer.monitor_enable == True - assert config.cnn_layers[clo[5]].monitor_enable == True - assert config.cnn_layers[clo[-1]].monitor_enable == True + assert config.cnn_layers[l2c[5]].monitor_enable == True + for idx in dynapcnn_net.exit_layer_ids: + assert config.cnn_layers[l2c[idx]].monitor_enable == True # Specify layers to monitor - should not warn becuase final layer has no pooling with warnings.catch_warnings(): @@ -112,4 +112,4 @@ def test_model_level_monitoring_enable(): # Monitor all layers config = dynapcnn_net.make_config(device="speck2b:0", monitor_layers="all") - assert all(config.cnn_layers[i].monitor_enable == True for i in clo) + assert all(config.cnn_layers[i].monitor_enable == True for i in l2c.values()) From 2680ac15413d0b347c2b3bd779e872552327d839 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Tue, 12 Nov 2024 14:29:19 +0100 Subject: [PATCH 335/379] Properly copy DVSLayer when instantiating DynapcnnNetwork. Fix DVS input unit tests. --- .../backend/dynapcnn/nir_graph_extractor.py | 2 +- tests/test_dynapcnn/test_dvs_input.py | 58 +++++++++++++------ 2 files changed, 40 insertions(+), 20 deletions(-) diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index 51d12c17..d942ca33 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -424,7 +424,7 @@ def _handle_dvs_input( # Make a copy of the layer so that the original version is not # changed in place new_dvs_layer = deepcopy(self.dvs_layer) - self.name_2_indx_map[self.dvs_layer_index] = new_dvs_layer + self._indx_2_module_map[self.dvs_layer_index] = new_dvs_layer elif dvs_input: # Insert a DVSLayer node in the graph. new_dvs_layer = self._add_dvs_node(dvs_input_shape=input_shape) diff --git a/tests/test_dynapcnn/test_dvs_input.py b/tests/test_dynapcnn/test_dvs_input.py index 0eae07c3..61d3507c 100644 --- a/tests/test_dynapcnn/test_dvs_input.py +++ b/tests/test_dynapcnn/test_dvs_input.py @@ -1,11 +1,9 @@ """This should test cases of dynapcnn compatible networks with dvs input.""" from itertools import product -from typing import Optional, Tuple +from typing import Optional, Tuple, Union -import numpy as np import pytest -import samna import torch from torch import nn @@ -30,7 +28,7 @@ def verify_dvs_config( origin: Tuple[int, int] = (0, 0), cut: Optional[Tuple[int, int]] = None, destination: Optional[int] = None, - dvs_input: bool = True, + dvs_input: Union[bool, None] = True, flip: Optional[dict] = None, merge_polarities: bool = False, ): @@ -80,9 +78,12 @@ def forward(self, x): class NetPool2D(nn.Module): - def __init__(self, input_layer: bool = False): + def __init__(self, add_input_layer: bool = False): super().__init__() - layers = [] + if add_input_layer: + layers = [DVSLayer(input_shape=INPUT_SHAPE[1:])] + else: + layers = [] layers += [ nn.AvgPool2d(kernel_size=(2, 4)), nn.Conv2d(2, 4, kernel_size=2, stride=2), @@ -128,36 +129,48 @@ def test_dvs_no_pooling(dvs_input): ) -@pytest.mark.parametrize("dvs_input", (False, True)) -def test_dvs_pooling_2d(dvs_input): +args = product((True, False, None), (True, False)) +@pytest.mark.parametrize("dvs_input,add_input_layer", args) +def test_dvs_pooling_2d(dvs_input, add_input_layer): # - ANN and SNN generation - ann = NetPool2D(input_layer=True) + ann = NetPool2D(add_input_layer=add_input_layer) snn = from_model(ann.seq, batch_size=1) snn.eval() # - SPN generation - spn = DynapcnnNetwork(snn, dvs_input=dvs_input, input_shape=INPUT_SHAPE) + if not dvs_input and not add_input_layer: + # No DVS layer is part of the SNN nor being added to it. The pooling layer should cause an exception + with pytest.raises(InvalidGraphStructure): + spn = DynapcnnNetwork(snn, dvs_input=dvs_input, input_shape=INPUT_SHAPE) + return - # When there is pooling, a DVSLayer should also be added if `dvs_input` is False + # If `add_input_layer` is False but `dvs_input` is `True`, a DVS layer will + # be added to the DynapcnnNetwork upon instantiation + spn = DynapcnnNetwork(snn, dvs_input=dvs_input, input_shape=INPUT_SHAPE) assert spn.has_dvs_layer() - # - Make sure missing input shapes cause exception - with pytest.raises(InputConfigurationError): - spn = DynapcnnNetwork(snn, dvs_input=dvs_input) + if not add_input_layer: + # - Make sure missing input shapes cause exception + with pytest.raises(InputConfigurationError): + spn = DynapcnnNetwork(snn, dvs_input=dvs_input) - # - Compare snn and spn outputs - spn_float = DynapcnnNetwork(snn, discretize=False, input_shape=INPUT_SHAPE) + # - Compare snn and spn outputs. - Always add DVS so that pooling layer is properly handled + spn_float = DynapcnnNetwork(snn, dvs_input=True, discretize=False, input_shape=INPUT_SHAPE) snn_out = snn(input_data).squeeze() spn_out = spn_float(input_data).squeeze() assert torch.equal(snn_out.detach(), spn_out) # - Verify DYNAP-CNN config - target_layers = [5] - config = spn.make_config(chip_layers_ordering=target_layers) + # Get index of only DynapcnnLayer to map it to core 5 + cnn_layer_idx = next(spn.dynapcnn_layers.__iter__()) + target_dest = 5 + config = spn.make_config(layer2core_map={cnn_layer_idx: target_dest}) + if dvs_input is None: + dvs_input = not snn.spiking_model[0].disable_pixel_array verify_dvs_config( config, input_shape=INPUT_SHAPE, - destination=target_layers[0], + destination=target_dest, dvs_input=dvs_input, pooling=(2, 4), ) @@ -290,6 +303,13 @@ def test_whether_dvs_mirror_cfg_is_all_switched_off(dvs_input, pool): snn = nn.Sequential(*layer_list) + if pool and not dvs_input: + with pytest.raises(InvalidGraphStructure): + dynapcnn = DynapcnnNetwork( + snn=snn, input_shape=(1, 128, 128), dvs_input=dvs_input, discretize=True + ) + return + dynapcnn = DynapcnnNetwork( snn=snn, input_shape=(1, 128, 128), dvs_input=dvs_input, discretize=True ) From 1cb7b5ca4bf1632f3e882a56b83c61eea06f7e75 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Tue, 12 Nov 2024 14:48:48 +0100 Subject: [PATCH 336/379] Reintroduce missing methods of DynapcnnNetwork: `reset_states`, `zero_grad` --- sinabs/backend/dynapcnn/dynapcnn_layer.py | 2 +- sinabs/backend/dynapcnn/dynapcnn_network.py | 70 +++++++++++++++++++-- 2 files changed, 67 insertions(+), 5 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer.py b/sinabs/backend/dynapcnn/dynapcnn_layer.py index ea573701..76057854 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer.py @@ -188,7 +188,7 @@ def forward(self, x) -> List[torch.Tensor]: def zero_grad(self, set_to_none: bool = False) -> None: """Call `zero_grad` method of spiking layer""" - return self._spk.zero_grad(set_to_none) + return self.spk.zero_grad(set_to_none) def get_neuron_shape(self) -> Tuple[int, int, int]: """Return the output shape of the neuron layer. diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index 2d1ef082..e27bafa7 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -53,10 +53,20 @@ def __init__( - batch_size (optional int): If `None`, will try to infer the batch size from the model. If int value is provided, it has to match the actual batch size of the model. - dvs_input (bool): optional (default as `None`). Wether or not dynapcnn receive - input from its DVS camera. If a `DVSLayer` is part of `snn` and `dvs_input` is - false, the DVS sensor will be configured but its output will not be sent as input - to the chip. If `dvs_input` is `True` and `snn` does not contain a `DVSLayer`, - it will be added. + input from its DVS camera. + If a `DVSLayer` is part of `snn`... + ... and `dvs_input` is `False`, its `disable_pixel_array` attribute + will be set `True`. This means the DVS sensor will be configured + upon deployment but its output will not be sent as input + ... and `dvs_input` is `None`, the `disable_pixel_array` attribute + of the layer will not be changed. + ... and `dvs_input` is `True`, `disable_pixel_array` will be set + `False`, so that the DVS sensor data is sent to the network. + If no `DVSLayer` is part of `snn`... + ... and `dvs_input` is `False` or `None`, no `DVSLayer` will be added + and the DVS sensor will not be configured upon deployment. + ... and `dvs_input` is `True`, a `DVSLayer` instance will be added + to the network, with `disable_pixel_array` set to `False`. - discretize (bool): If `True`, discretize the parameters and thresholds. This is needed for uploading weights to dynapcnn. Set to `False` only for testing purposes. - weight_rescaling_fn (callable): a method that handles how the re-scaling factor for one or more `SumPool2d` projecting to @@ -514,6 +524,58 @@ def has_dvs_layer(self) -> bool: """ return self.dvs_layer is not None + def zero_grad(self, set_to_none: bool = False) -> None: + """ Call `zero_grad` method of each DynapCNN layer + + Parameters + ---------- + - set_to_none (bool): This argument is passed directly to the + `zero_grad` method of each DynapCNN layer + """ + for lyr in self.dynapcnn_layers.values(): + lyr.zero_grad(set_to_none) + + def reset_states(self, randomize=False): + """Reset the states of the network. + + Parameters + ---------- + - randomize (bool): If `False` (default), will set all states to 0. + Otherwise will set to random values. + + Notes + ----- + - Setting `randomize` to `True` is only supported for models that have + not yet been deployed on a SynSense device. + """ + if hasattr(self, "device") and isinstance(self.device, str): # pragma: no cover + device_name, _ = parse_device_id(self.device) + # Reset states on SynSense device + if device_name in ChipFactory.supported_devices: + config_builder = ChipFactory(self.device).get_config_builder() + # Set all the vmem states in the samna config to zero + config_builder.reset_states(self.samna_config, randomize=randomize) + self.samna_device.get_model().apply_configuration(self.samna_config) + # wait for the config to be written + time.sleep(1) + # Note: The below shouldn't be necessary ideally + # Erase all vmem memory + if not randomize: + if hasattr(self, "samna_input_graph"): + self.samna_input_graph.stop() + for lyr_idx in self.chip_layers_ordering: + config_builder.set_all_v_mem_to_zeros( + self.samna_device, lyr_idx + ) + time.sleep(0.1) + self.samna_input_graph.start() + return + + # Reset states of `DynapcnnLayer` instances + for layer in self.sequence: + if isinstance(layer, DynapcnnLayer): + layer.spk_layer.reset_states(randomize=randomize) + ####################################################### Private Methods ####################################################### def _make_config( From f1e308d031cbd0d63d1241f329779b6b093221e0 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Tue, 12 Nov 2024 15:30:27 +0100 Subject: [PATCH 337/379] Support model with only DVS --- sinabs/backend/dynapcnn/dynapcnn_network.py | 2 +- .../backend/dynapcnn/nir_graph_extractor.py | 68 ++++++++++++------- tests/test_dynapcnn/test_speck2e.py | 3 +- 3 files changed, 46 insertions(+), 27 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index e27bafa7..ef1443a7 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -139,7 +139,7 @@ def dynapcnn_module(self): @property def exit_layers(self): return [ - self.dynapcnn_layers[i] for i in self._dynapcnn_module.get_exit_layers() + self.all_layers[i] for i in self._dynapcnn_module.get_exit_layers() ] @property diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index d942ca33..9169dd51 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -79,28 +79,33 @@ def __init__( instantiation with `remove_nodes_by_class`. """ - # Store state before it is changed due to NIRTorch passing dummy input - original_state = { - n: b.detach().clone() for n, b in spiking_model.named_buffers() - } + # Empty sequentials will cause nirtorch to fail. Treat it separately + if isinstance(spiking_model, nn.Sequential) and len(spiking_model) == 0: + self._name_2_indx_map = dict() + self._edges = set() + else: + # Store state before it is changed due to NIRTorch passing dummy input + original_state = { + n: b.detach().clone() for n, b in spiking_model.named_buffers() + } - # extract computational graph. - nir_graph = nirtorch.extract_torch_graph( - spiking_model, dummy_input, model_name=None - ).ignore_tensors() - if ignore_node_types is not None: - for node_type in ignore_node_types: - nir_graph = nir_graph.ignore_nodes(node_type) + # extract computational graph. + nir_graph = nirtorch.extract_torch_graph( + spiking_model, dummy_input, model_name=None + ).ignore_tensors() + if ignore_node_types is not None: + for node_type in ignore_node_types: + nir_graph = nir_graph.ignore_nodes(node_type) - # Restore original state - for n, b in spiking_model.named_buffers(): - b.set_(original_state[n].clone()) + # Restore original state + for n, b in spiking_model.named_buffers(): + b.set_(original_state[n].clone()) - # Map node names to indices - self._name_2_indx_map = self._get_name_2_indx_map(nir_graph) + # Map node names to indices + self._name_2_indx_map = self._get_name_2_indx_map(nir_graph) - # Extract edges list from graph - self._edges = self._get_edges_from_nir(nir_graph, self._name_2_indx_map) + # Extract edges list from graph + self._edges = self._get_edges_from_nir(nir_graph, self._name_2_indx_map) # Store the associated `nn.Module` (layer) of each node. self._indx_2_module_map = self._get_named_modules(spiking_model) @@ -126,15 +131,15 @@ def __init__( @property def dvs_layer(self) -> Union[DVSLayer, None]: - idx = self.dvs_layer_index + idx = self.dvs_node_id if idx is None: return None else: - return self.indx_2_module_map[self.dvs_layer_index] + return self.indx_2_module_map[self.dvs_node_id] @property - def dvs_layer_index(self) -> Union[int, None]: - return self._get_dvs_layer_index() + def dvs_node_id(self) -> Union[int, None]: + return self._get_dvs_node_id() @property def entry_nodes(self) -> Set[int]: @@ -195,6 +200,18 @@ def get_dynapcnn_network_module( entry_nodes=self.entry_nodes, ) + # Special case where there is a disconnected `DVSLayer`: There are no + # Edges for the edges handler to process. Instantiate layer info manually. + if self.dvs_layer_info is None and self.dvs_layer is not None: + self.dvs_layer_info = { + "node_id": self.dvs_node_id, + "input_shape": self.nodes_io_shapes[self.dvs_node_id]["input"], + "module": self.dvs_layer, + "pooling": None, + "destinations": None, + } + + # build `DynapcnnLayer` instances from mapper. dynapcnn_layers, destination_map, entry_points = ( construct_dynapcnnlayers_from_mapper( @@ -424,7 +441,7 @@ def _handle_dvs_input( # Make a copy of the layer so that the original version is not # changed in place new_dvs_layer = deepcopy(self.dvs_layer) - self._indx_2_module_map[self.dvs_layer_index] = new_dvs_layer + self._indx_2_module_map[self.dvs_node_id] = new_dvs_layer elif dvs_input: # Insert a DVSLayer node in the graph. new_dvs_layer = self._add_dvs_node(dvs_input_shape=input_shape) @@ -487,7 +504,7 @@ def _add_dvs_node(self, dvs_input_shape: Tuple[int, int, int]) -> DVSLayer: return dvs_layer - def _get_dvs_layer_index(self) -> Union[int, None]: + def _get_dvs_node_id(self) -> Union[int, None]: """Loop though all modules and return index of `DVSLayer` instance if it exists. @@ -602,6 +619,9 @@ def _get_entry_nodes(self, edges: Set[Edge]) -> Set[Edge]: - entry_nodes (set): IDs of nodes acting as entry points for the network (i.e., receiving external input). """ + if not edges: + return set() + all_sources, all_targets = zip(*edges) return set(all_sources) - set(all_targets) diff --git a/tests/test_dynapcnn/test_speck2e.py b/tests/test_dynapcnn/test_speck2e.py index f6c6a434..9939adc3 100644 --- a/tests/test_dynapcnn/test_speck2e.py +++ b/tests/test_dynapcnn/test_speck2e.py @@ -42,6 +42,5 @@ def test_speck2e_coordinates(): def test_dvs_layer_generation(): """DVSLayer should be generated is dvs input is enabled even for an empty network.""" - ann = nn.Sequential() network = DynapcnnNetwork(nn.Sequential(), input_shape=(2, 10, 10), dvs_input=True) - assert isinstance(network.sequence[0], DVSLayer) + assert isinstance(network.dvs_layer, DVSLayer) From b8827faaca6f3378d46031b290b28848a16264db Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Tue, 12 Nov 2024 15:31:06 +0100 Subject: [PATCH 338/379] Fix unit test `test_single_neuron....py` --- tests/test_dynapcnn/test_single_neuron_hardware.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_dynapcnn/test_single_neuron_hardware.py b/tests/test_dynapcnn/test_single_neuron_hardware.py index 6d8fe6c6..a81ac2ad 100644 --- a/tests/test_dynapcnn/test_single_neuron_hardware.py +++ b/tests/test_dynapcnn/test_single_neuron_hardware.py @@ -42,9 +42,9 @@ def test_deploy_dynapcnnnetwork(): model = get_ones_network() sinabs.reset_states(model) - assert model.sequence[0].conv_layer.weight.sum() == 127 - assert model.sequence[0].spk_layer.spike_threshold == 127 - assert model.sequence[0].spk_layer.v_mem.sum() == 0 + assert model.dynapcnn_layers[0].conv_layer.weight.sum() == 127 + assert model.dynapcnn_layers[0].spk_layer.spike_threshold == 127 + assert model.dynapcnn_layers[0].spk_layer.v_mem.sum() == 0 model_output = model(torch.ones((1, 1, 1, 1))) assert model_output.sum() == 1 From 4d46656fb76fe775522f08c4467b466f984b83f7 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Tue, 12 Nov 2024 15:31:16 +0100 Subject: [PATCH 339/379] Fix speckmini unit test --- .../test_speckmini_config_making.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/test_dynapcnn/test_speckmini_config_making.py b/tests/test_dynapcnn/test_speckmini_config_making.py index 4ffd28c6..5ca8625d 100644 --- a/tests/test_dynapcnn/test_speckmini_config_making.py +++ b/tests/test_dynapcnn/test_speckmini_config_making.py @@ -71,15 +71,15 @@ def test_auto_mapping(): for test_device in devices: # test weights/kernel memory mapping _ = SNN_KERNEL_MEM_TEST.make_config( - chip_layers_ordering="auto", device=test_device + layer2core_map="auto", device=test_device ) - assert SNN_KERNEL_MEM_TEST.chip_layers_ordering == [0, 1, 3, 2, 4] + assert SNN_KERNEL_MEM_TEST.layer2core_map == {0: 0, 1: 1, 2: 3, 3: 2, 4: 4} # test neuron memory mapping _ = SNN_NEURON_MEM_TEST.make_config( - chip_layers_ordering="auto", device=test_device + layer2core_map="auto", device=test_device ) - assert SNN_NEURON_MEM_TEST.chip_layers_ordering == [2, 0, 1, 4, 3] + assert SNN_NEURON_MEM_TEST.layer2core_map == {0: 2, 1: 0, 2: 1, 3: 4, 4: 3} def test_manual_mapping(): @@ -87,15 +87,15 @@ def test_manual_mapping(): for test_device in devices: # test weights/kernel memory mapping - chip_layers_order = [4, 2, 3, 1, 0] + layer2core_map = {0: 4, 1: 2, 2: 3, 3: 1, 4: 0} _ = SNN_KERNEL_MEM_TEST.make_config( - chip_layers_ordering=chip_layers_order, device=test_device + layer2core_map=layer2core_map, device=test_device ) - assert SNN_KERNEL_MEM_TEST.chip_layers_ordering == chip_layers_order + assert SNN_KERNEL_MEM_TEST.layer2core_map == layer2core_map # test neuron memory mapping - chip_layers_order = [1, 0, 2, 3, 4] + chip_layers_order = {0: 1, 1: 0, 2: 2, 3: 3, 4: 4} _ = SNN_NEURON_MEM_TEST.make_config( - chip_layers_ordering=chip_layers_order, device=test_device + layer2core_map=chip_layers_order, device=test_device ) - assert SNN_NEURON_MEM_TEST.chip_layers_ordering == chip_layers_order + assert SNN_NEURON_MEM_TEST.layer2core_map == chip_layers_order From 681987f471527b1a93e4bd8926e5890373ae277b Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Tue, 12 Nov 2024 15:50:32 +0100 Subject: [PATCH 340/379] Fix neuron leak unit test --- tests/test_dynapcnn/test_neuron_leak.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_dynapcnn/test_neuron_leak.py b/tests/test_dynapcnn/test_neuron_leak.py index 4acacdbc..cf4adc49 100644 --- a/tests/test_dynapcnn/test_neuron_leak.py +++ b/tests/test_dynapcnn/test_neuron_leak.py @@ -53,9 +53,9 @@ def test_neuron_leak_config(): snn=snn, discretize=True, dvs_input=True, input_shape=(1, 64, 64) ) samna_cfg = dynapcnn.make_config(device="speck2fmodule") - chip_layers_order = dynapcnn.chip_layers_ordering + layer2core_map = dynapcnn.layer2core_map - for lyr, channel_num in zip(chip_layers_order, [2, 8, 16]): + for lyr, channel_num in zip(layer2core_map.values(), [2, 8, 16]): assert samna_cfg.cnn_layers[lyr].leak_enable is True assert len(samna_cfg.cnn_layers[lyr].biases) == channel_num @@ -124,6 +124,8 @@ def test_neuron_leak(): pre_neuron_state = neuron_states.get((c, x, y), 127) assert ( pre_neuron_state > out_ev.neuron_state + # If `pre_neuron_state` is already at minimum, it can't leak further + or pre_neuron_state == -127 ), "Neuron V_Mem doesn't decrease!" neuron_states.update({(c, x, y): out_ev.neuron_state}) print(f"c:{c}, x:{x}, y:{y}, vmem:{out_ev.neuron_state}") From 9cdb712bd069a9fb38e7d1bdce55d7e66a25d379 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Tue, 12 Nov 2024 16:06:00 +0100 Subject: [PATCH 341/379] Provide more meaningful error when specific device is not found --- sinabs/backend/dynapcnn/io.py | 8 +++++++- tests/test_dynapcnn/test_device_movement.py | 5 ++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/sinabs/backend/dynapcnn/io.py b/sinabs/backend/dynapcnn/io.py index 7474b67d..eaeed8c2 100644 --- a/sinabs/backend/dynapcnn/io.py +++ b/sinabs/backend/dynapcnn/io.py @@ -252,7 +252,13 @@ def open_device(device_id: str): """ device_id = standardize_device_id(device_id=device_id) device_map = get_device_map() - device_info = device_map[device_id] + try: + device_info = device_map[device_id] + except KeyError: + msg = f"Device {device_id} has not been found. Make sure it is connected." + if device_map: + msg += "The following devices are available:\n" + "\n".join(device_map) + raise IOError(msg) device_handle = samna.device.open_device(device_info) if device_handle is not None: diff --git a/tests/test_dynapcnn/test_device_movement.py b/tests/test_dynapcnn/test_device_movement.py index a6b9d8a1..99508ba6 100644 --- a/tests/test_dynapcnn/test_device_movement.py +++ b/tests/test_dynapcnn/test_device_movement.py @@ -2,7 +2,6 @@ import torch.nn as nn from sinabs.backend.dynapcnn import DynapcnnNetwork -from sinabs.backend.dynapcnn.mapping import edmonds, make_flow_graph, recover_mapping from sinabs.from_torch import from_model ann = nn.Sequential( @@ -33,7 +32,7 @@ def test_multi_device_movement(): input_shape=input_shape, ) - hardware_compatible_model.to("speck2b:0") + hardware_compatible_model.to("speck2edevkit") print("Second attempt") - hardware_compatible_model.to("speck2b:0") + hardware_compatible_model.to("speck2edevkit") From 57333cedbdcb4c06cb86c33ab4c0d69c8a2ac994 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Tue, 12 Nov 2024 16:06:13 +0100 Subject: [PATCH 342/379] Remove duplicate test --- tests/test_dynapcnn/test_discover_device.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/tests/test_dynapcnn/test_discover_device.py b/tests/test_dynapcnn/test_discover_device.py index cb828111..cb15e4cb 100644 --- a/tests/test_dynapcnn/test_discover_device.py +++ b/tests/test_dynapcnn/test_discover_device.py @@ -4,14 +4,7 @@ from sinabs.backend.dynapcnn import io -@pytest.mark.skip("Not suitable for automated testing. Depends on available devices") -def test_list_all_devices(): - device_map = io.get_device_map() - # Ideally the device map needs to be tested against something expected. - raise NotImplementedError() - - -@pytest.mark.skip("Not suitable for automated testing. Depends on available devices") +pytest.mark.skip("Not suitable for automated testing. Depends on available devices") def test_is_device_type(): devices = samna.device.get_all_devices() print([io.is_device_type(d, "dynapcnndevkit") for d in devices]) From c0291e10aaa1e7cf656a1eb3abf854d16088b6df Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Tue, 12 Nov 2024 16:09:26 +0100 Subject: [PATCH 343/379] Remove obsolete TODO --- tests/test_dynapcnn/test_large_net.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tests/test_dynapcnn/test_large_net.py b/tests/test_dynapcnn/test_large_net.py index 1c5f7bc6..ad8fdb19 100644 --- a/tests/test_dynapcnn/test_large_net.py +++ b/tests/test_dynapcnn/test_large_net.py @@ -88,11 +88,6 @@ def test_same_result(): assert torch.equal(dynapcnn_out.squeeze(), snn_out.squeeze()) -# TODO: Define new test with actual network that is too large. Probably have it as fail case in test_dynapcnnnetwork -def test_too_large(): - pass - - def test_auto_config(): # - Should give an error with the normal layer ordering dynapcnn_net.make_config(chip_layers_ordering="auto") @@ -136,7 +131,7 @@ def test_make_config(): ) -@pytest.mark.skip("Not suitable for automated testing. Depends on available devices") +# @pytest.mark.skip("Not suitable for automated testing. Depends on available devices") def test_to_device(): dynapcnn_net = DynapcnnNetwork( snn, input_shape=input_shape, discretize=False, dvs_input=False From a794647443a0b9a5cda7a786eb11ddb25c52732c Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Tue, 12 Nov 2024 17:15:18 +0100 Subject: [PATCH 344/379] Minor fixes. --- sinabs/backend/dynapcnn/dynapcnn_network.py | 8 ++------ tests/test_dynapcnn/test_visualizer.py | 4 ++-- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index ef1443a7..c9a2bc36 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -766,11 +766,7 @@ class DynapcnnCompatibleNetwork(DynapcnnNetwork): """Deprecated class, use DynapcnnNetwork instead.""" def __init__( - self, - snn: Union[nn.Sequential, sinabs.Network], - input_shape: Optional[Tuple[int, int, int]] = None, - dvs_input: bool = False, - discretize: bool = True, + self, *args, **kwargs ): from warnings import warn @@ -778,4 +774,4 @@ def __init__( "DynapcnnCompatibleNetwork has been renamed to DynapcnnNetwork " + "and will be removed in a future release." ) - super().__init__(snn, input_shape, dvs_input, discretize) + super().__init__(*args, **kwargs) diff --git a/tests/test_dynapcnn/test_visualizer.py b/tests/test_dynapcnn/test_visualizer.py index 0873f318..51b8341e 100644 --- a/tests/test_dynapcnn/test_visualizer.py +++ b/tests/test_dynapcnn/test_visualizer.py @@ -39,12 +39,12 @@ def get_demo_dynapcnn_network(): import torch.nn as nn import sinabs - from sinabs.backend.dynapcnn import DynapcnnCompatibleNetwork + from sinabs.backend.dynapcnn import DynapcnnNetwork ann = nn.Sequential(nn.Conv2d(2, 8, (3, 3)), nn.ReLU(), nn.AvgPool2d((2, 2))) snn = sinabs.from_model(ann, input_shape=(2, 64, 64), batch_size=1) - dynapcnn_network = DynapcnnCompatibleNetwork( + dynapcnn_network = DynapcnnNetwork( snn=snn, input_shape=(2, 64, 64), dvs_input=True ) return dynapcnn_network From 10e1babdfab7b29d52e17d2b24e1c0d6d9c70e49 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Tue, 12 Nov 2024 17:19:49 +0100 Subject: [PATCH 345/379] Undo erroneous comment --- tests/test_dynapcnn/test_large_net.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_dynapcnn/test_large_net.py b/tests/test_dynapcnn/test_large_net.py index ad8fdb19..c464a975 100644 --- a/tests/test_dynapcnn/test_large_net.py +++ b/tests/test_dynapcnn/test_large_net.py @@ -131,7 +131,7 @@ def test_make_config(): ) -# @pytest.mark.skip("Not suitable for automated testing. Depends on available devices") +@pytest.mark.skip("Not suitable for automated testing. Depends on available devices") def test_to_device(): dynapcnn_net = DynapcnnNetwork( snn, input_shape=input_shape, discretize=False, dvs_input=False From 205077d16537471ee2b5b1288dd82bec39ce0ae3 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Tue, 12 Nov 2024 17:21:57 +0100 Subject: [PATCH 346/379] Run black and isort --- sinabs/backend/dynapcnn/chips/speck2cmini.py | 1 + sinabs/backend/dynapcnn/chips/speck2e.py | 1 + sinabs/backend/dynapcnn/chips/speck2f.py | 1 + sinabs/backend/dynapcnn/config_builder.py | 3 ++- sinabs/backend/dynapcnn/discretize.py | 3 ++- sinabs/backend/dynapcnn/dvs_layer.py | 1 + sinabs/backend/dynapcnn/dynapcnn_network.py | 14 +++++--------- sinabs/backend/dynapcnn/nir_graph_extractor.py | 1 - sinabs/backend/dynapcnn/specksim.py | 3 ++- tests/test_dynapcnn/test_discover_device.py | 3 ++- tests/test_dynapcnn/test_dvs_input.py | 6 +++++- .../test_dynapcnn/test_speckmini_config_making.py | 8 ++------ tests/test_dynapcnn/test_visualizer.py | 4 +--- tests/test_dynapcnnlayer/conftest_dynapcnnlayer.py | 8 ++++---- tests/test_dynapcnnlayer/test_dynapcnnlayer.py | 3 ++- .../conftest_dynapcnnnetwork.py | 5 ++--- tests/test_dynapcnnnetwork/model_dummy_seq.py | 1 - tests/test_dynapcnnnetwork/test_dynapcnnnetwork.py | 3 ++- 18 files changed, 35 insertions(+), 34 deletions(-) diff --git a/sinabs/backend/dynapcnn/chips/speck2cmini.py b/sinabs/backend/dynapcnn/chips/speck2cmini.py index a17e5830..66b5b4d1 100644 --- a/sinabs/backend/dynapcnn/chips/speck2cmini.py +++ b/sinabs/backend/dynapcnn/chips/speck2cmini.py @@ -2,6 +2,7 @@ import samna from samna.speck2cMini.configuration import SpeckConfiguration + from sinabs.backend.dynapcnn.dynapcnn_layer import DynapcnnLayer from sinabs.backend.dynapcnn.mapping import LayerConstraints diff --git a/sinabs/backend/dynapcnn/chips/speck2e.py b/sinabs/backend/dynapcnn/chips/speck2e.py index 91a2a95c..799b9f98 100644 --- a/sinabs/backend/dynapcnn/chips/speck2e.py +++ b/sinabs/backend/dynapcnn/chips/speck2e.py @@ -2,6 +2,7 @@ import samna from samna.speck2e.configuration import SpeckConfiguration + from sinabs.backend.dynapcnn.dynapcnn_layer import DynapcnnLayer from .dynapcnn import DynapcnnConfigBuilder diff --git a/sinabs/backend/dynapcnn/chips/speck2f.py b/sinabs/backend/dynapcnn/chips/speck2f.py index c5ed563b..d34418f9 100644 --- a/sinabs/backend/dynapcnn/chips/speck2f.py +++ b/sinabs/backend/dynapcnn/chips/speck2f.py @@ -2,6 +2,7 @@ import samna from samna.speck2f.configuration import SpeckConfiguration + from sinabs.backend.dynapcnn.dynapcnn_layer import DynapcnnLayer from .dynapcnn import DynapcnnConfigBuilder diff --git a/sinabs/backend/dynapcnn/config_builder.py b/sinabs/backend/dynapcnn/config_builder.py index 81e649a9..7a360718 100644 --- a/sinabs/backend/dynapcnn/config_builder.py +++ b/sinabs/backend/dynapcnn/config_builder.py @@ -2,10 +2,11 @@ from typing import Dict, List import samna +from samna.dynapcnn.configuration import DynapcnnConfiguration + import sinabs import sinabs.backend import sinabs.backend.dynapcnn -from samna.dynapcnn.configuration import DynapcnnConfiguration from .dvs_layer import DVSLayer from .dynapcnn_layer import DynapcnnLayer diff --git a/sinabs/backend/dynapcnn/discretize.py b/sinabs/backend/dynapcnn/discretize.py index 3a7a0253..0a14c27b 100644 --- a/sinabs/backend/dynapcnn/discretize.py +++ b/sinabs/backend/dynapcnn/discretize.py @@ -2,10 +2,11 @@ from typing import Optional, Tuple from warnings import warn -import sinabs.layers as sl import torch import torch.nn as nn +import sinabs.layers as sl + DYNAPCNN_WEIGHT_PRECISION_BITS = 8 DYNAPCNN_STATE_PRECISION_BITS = 16 diff --git a/sinabs/backend/dynapcnn/dvs_layer.py b/sinabs/backend/dynapcnn/dvs_layer.py index 278f68da..69ec7af6 100644 --- a/sinabs/backend/dynapcnn/dvs_layer.py +++ b/sinabs/backend/dynapcnn/dvs_layer.py @@ -1,6 +1,7 @@ from typing import Optional, Tuple import torch.nn as nn + from sinabs.layers import SumPool2d from sinabs.utils import expand_to_pair diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index c9a2bc36..17376d8e 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -61,7 +61,7 @@ def __init__( ... and `dvs_input` is `None`, the `disable_pixel_array` attribute of the layer will not be changed. ... and `dvs_input` is `True`, `disable_pixel_array` will be set - `False`, so that the DVS sensor data is sent to the network. + `False`, so that the DVS sensor data is sent to the network. If no `DVSLayer` is part of `snn`... ... and `dvs_input` is `False` or `None`, no `DVSLayer` will be added and the DVS sensor will not be configured upon deployment. @@ -138,9 +138,7 @@ def dynapcnn_module(self): @property def exit_layers(self): - return [ - self.all_layers[i] for i in self._dynapcnn_module.get_exit_layers() - ] + return [self.all_layers[i] for i in self._dynapcnn_module.get_exit_layers()] @property def exit_layer_ids(self): @@ -525,8 +523,8 @@ def has_dvs_layer(self) -> bool: return self.dvs_layer is not None def zero_grad(self, set_to_none: bool = False) -> None: - """ Call `zero_grad` method of each DynapCNN layer - + """Call `zero_grad` method of each DynapCNN layer + Parameters ---------- - set_to_none (bool): This argument is passed directly to the @@ -765,9 +763,7 @@ def __repr__(self): class DynapcnnCompatibleNetwork(DynapcnnNetwork): """Deprecated class, use DynapcnnNetwork instead.""" - def __init__( - self, *args, **kwargs - ): + def __init__(self, *args, **kwargs): from warnings import warn warn( diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index 9169dd51..e48a4952 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -211,7 +211,6 @@ def get_dynapcnn_network_module( "destinations": None, } - # build `DynapcnnLayer` instances from mapper. dynapcnn_layers, destination_map, entry_points = ( construct_dynapcnnlayers_from_mapper( diff --git a/sinabs/backend/dynapcnn/specksim.py b/sinabs/backend/dynapcnn/specksim.py index 2923c070..4a1c35f0 100644 --- a/sinabs/backend/dynapcnn/specksim.py +++ b/sinabs/backend/dynapcnn/specksim.py @@ -4,11 +4,12 @@ import numpy as np import samna -import sinabs.layers as sl import torch.nn as nn from samna.specksim.nodes import SpecksimConvolutionalFilterNode as ConvFilter from samna.specksim.nodes import SpecksimIAFFilterNode as IAFFilter from samna.specksim.nodes import SpecksimSumPoolingFilterNode as SumPoolFilter + +import sinabs.layers as sl from sinabs.backend.dynapcnn import DynapcnnCompatibleNetwork, DynapcnnNetwork from sinabs.backend.dynapcnn.dynapcnn_layer import DynapcnnLayer diff --git a/tests/test_dynapcnn/test_discover_device.py b/tests/test_dynapcnn/test_discover_device.py index cb15e4cb..10b33d85 100644 --- a/tests/test_dynapcnn/test_discover_device.py +++ b/tests/test_dynapcnn/test_discover_device.py @@ -3,8 +3,9 @@ from sinabs.backend.dynapcnn import io - pytest.mark.skip("Not suitable for automated testing. Depends on available devices") + + def test_is_device_type(): devices = samna.device.get_all_devices() print([io.is_device_type(d, "dynapcnndevkit") for d in devices]) diff --git a/tests/test_dynapcnn/test_dvs_input.py b/tests/test_dynapcnn/test_dvs_input.py index 61d3507c..44d7f275 100644 --- a/tests/test_dynapcnn/test_dvs_input.py +++ b/tests/test_dynapcnn/test_dvs_input.py @@ -130,6 +130,8 @@ def test_dvs_no_pooling(dvs_input): args = product((True, False, None), (True, False)) + + @pytest.mark.parametrize("dvs_input,add_input_layer", args) def test_dvs_pooling_2d(dvs_input, add_input_layer): # - ANN and SNN generation @@ -155,7 +157,9 @@ def test_dvs_pooling_2d(dvs_input, add_input_layer): spn = DynapcnnNetwork(snn, dvs_input=dvs_input) # - Compare snn and spn outputs. - Always add DVS so that pooling layer is properly handled - spn_float = DynapcnnNetwork(snn, dvs_input=True, discretize=False, input_shape=INPUT_SHAPE) + spn_float = DynapcnnNetwork( + snn, dvs_input=True, discretize=False, input_shape=INPUT_SHAPE + ) snn_out = snn(input_data).squeeze() spn_out = spn_float(input_data).squeeze() assert torch.equal(snn_out.detach(), spn_out) diff --git a/tests/test_dynapcnn/test_speckmini_config_making.py b/tests/test_dynapcnn/test_speckmini_config_making.py index 5ca8625d..502fa930 100644 --- a/tests/test_dynapcnn/test_speckmini_config_making.py +++ b/tests/test_dynapcnn/test_speckmini_config_making.py @@ -70,15 +70,11 @@ def test_auto_mapping(): for test_device in devices: # test weights/kernel memory mapping - _ = SNN_KERNEL_MEM_TEST.make_config( - layer2core_map="auto", device=test_device - ) + _ = SNN_KERNEL_MEM_TEST.make_config(layer2core_map="auto", device=test_device) assert SNN_KERNEL_MEM_TEST.layer2core_map == {0: 0, 1: 1, 2: 3, 3: 2, 4: 4} # test neuron memory mapping - _ = SNN_NEURON_MEM_TEST.make_config( - layer2core_map="auto", device=test_device - ) + _ = SNN_NEURON_MEM_TEST.make_config(layer2core_map="auto", device=test_device) assert SNN_NEURON_MEM_TEST.layer2core_map == {0: 2, 1: 0, 2: 1, 3: 4, 4: 3} diff --git a/tests/test_dynapcnn/test_visualizer.py b/tests/test_dynapcnn/test_visualizer.py index 51b8341e..dd7cce81 100644 --- a/tests/test_dynapcnn/test_visualizer.py +++ b/tests/test_dynapcnn/test_visualizer.py @@ -44,9 +44,7 @@ def get_demo_dynapcnn_network(): ann = nn.Sequential(nn.Conv2d(2, 8, (3, 3)), nn.ReLU(), nn.AvgPool2d((2, 2))) snn = sinabs.from_model(ann, input_shape=(2, 64, 64), batch_size=1) - dynapcnn_network = DynapcnnNetwork( - snn=snn, input_shape=(2, 64, 64), dvs_input=True - ) + dynapcnn_network = DynapcnnNetwork(snn=snn, input_shape=(2, 64, 64), dvs_input=True) return dynapcnn_network diff --git a/tests/test_dynapcnnlayer/conftest_dynapcnnlayer.py b/tests/test_dynapcnnlayer/conftest_dynapcnnlayer.py index fa756ef0..6b030609 100644 --- a/tests/test_dynapcnnlayer/conftest_dynapcnnlayer.py +++ b/tests/test_dynapcnnlayer/conftest_dynapcnnlayer.py @@ -1,10 +1,10 @@ # author : Willian Soares Girao # contact : wsoaresgirao@gmail.com -from .model_dummy_1 import expected_output_1, dcnnl_map_1 -from .model_dummy_2 import expected_output_2, dcnnl_map_2 -from .model_dummy_3 import expected_output_3, dcnnl_map_3 -from .model_dummy_4 import expected_output_4, dcnnl_map_4 +from .model_dummy_1 import dcnnl_map_1, expected_output_1 +from .model_dummy_2 import dcnnl_map_2, expected_output_2 +from .model_dummy_3 import dcnnl_map_3, expected_output_3 +from .model_dummy_4 import dcnnl_map_4, expected_output_4 # Args: dcnnl_map, discretize, expected_output args_DynapcnnLayer = [ diff --git a/tests/test_dynapcnnlayer/test_dynapcnnlayer.py b/tests/test_dynapcnnlayer/test_dynapcnnlayer.py index 16d88822..9701607e 100644 --- a/tests/test_dynapcnnlayer/test_dynapcnnlayer.py +++ b/tests/test_dynapcnnlayer/test_dynapcnnlayer.py @@ -2,12 +2,13 @@ # contact : wsoaresgirao@gmail.com import pytest -from .conftest_dynapcnnlayer import args_DynapcnnLayer from sinabs.backend.dynapcnn.dynapcnn_layer_utils import ( construct_dynapcnnlayers_from_mapper, ) +from .conftest_dynapcnnlayer import args_DynapcnnLayer + @pytest.mark.parametrize( "dcnnl_map, discretize, expected_output", diff --git a/tests/test_dynapcnnnetwork/conftest_dynapcnnnetwork.py b/tests/test_dynapcnnnetwork/conftest_dynapcnnnetwork.py index ccd7d4ef..7eaad023 100644 --- a/tests/test_dynapcnnnetwork/conftest_dynapcnnnetwork.py +++ b/tests/test_dynapcnnnetwork/conftest_dynapcnnnetwork.py @@ -17,13 +17,12 @@ from .model_dummy_4 import expected_output as expected_output_4 from .model_dummy_4 import input_shape as input_shape_4 from .model_dummy_4 import snn as snn_4 -from .model_dummy_4 import snn as snn_4 from .model_dummy_seq import ( + expected_seq_1, + expected_seq_2, input_shape_seq, seq_1, seq_2, - expected_seq_1, - expected_seq_2, ) args_DynapcnnNetworkTest = [ diff --git a/tests/test_dynapcnnnetwork/model_dummy_seq.py b/tests/test_dynapcnnnetwork/model_dummy_seq.py index 02663e30..1da69477 100644 --- a/tests/test_dynapcnnnetwork/model_dummy_seq.py +++ b/tests/test_dynapcnnnetwork/model_dummy_seq.py @@ -5,7 +5,6 @@ from sinabs.layers import IAFSqueeze, SumPool2d - input_shape_seq = (2, 30, 30) seq_1 = nn.Sequential( diff --git a/tests/test_dynapcnnnetwork/test_dynapcnnnetwork.py b/tests/test_dynapcnnnetwork/test_dynapcnnnetwork.py index 6c79e03f..4c50bc39 100644 --- a/tests/test_dynapcnnnetwork/test_dynapcnnnetwork.py +++ b/tests/test_dynapcnnnetwork/test_dynapcnnnetwork.py @@ -3,10 +3,11 @@ import pytest import torch -from .conftest_dynapcnnnetwork import args_DynapcnnNetworkTest from sinabs.backend.dynapcnn.dynapcnn_network import DynapcnnNetwork +from .conftest_dynapcnnnetwork import args_DynapcnnNetworkTest + @pytest.mark.parametrize( "snn, input_shape, batch_size, expected_output", args_DynapcnnNetworkTest From db0b9b541c14825eafd9f8b1475d37340820e362 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Tue, 12 Nov 2024 17:28:29 +0100 Subject: [PATCH 347/379] Fix 'deque' type hint --- sinabs/backend/dynapcnn/sinabs_edges_handler.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index e656b13c..6e5a3346 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -5,8 +5,7 @@ contact : williansoaresgirao@gmail.com """ -from collections import deque -from typing import Dict, List, Optional, Set, Tuple, Type, Union +from typing import Deque, Dict, List, Optional, Set, Tuple, Type, Union from torch import Size, nn @@ -475,7 +474,7 @@ def init_new_dynapcnnlayer_entry( def add_pooling_to_entry( dynapcnn_layer_info: Dict[int, Dict], edge: Edge, - pooling_chains: List[deque[int]], + pooling_chains: List[Deque[int]], indx_2_module_map: Dict[int, nn.Module], node_2_layer_map: Dict[int, int], ) -> None: @@ -904,7 +903,7 @@ def set_pooling_layer_destination( destination["output_shape"] = output_shape -def trace_paths(node: int, remaining_edges: Set[Edge]) -> List[deque[int]]: +def trace_paths(node: int, remaining_edges: Set[Edge]) -> List[Deque[int]]: """Trace any path of collected edges through the graph. Start with `node`, and recursively look for paths of connected nodes @@ -941,7 +940,7 @@ def trace_paths(node: int, remaining_edges: Set[Edge]) -> List[deque[int]]: if not paths: # End of recursion: instantiate a deque only with node - paths = [deque([node])] + paths = [Deque([node])] return paths, processed_edges From 9fbdf81fca944fc6a1f1113c5e73e2bbc1240fcc Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Tue, 12 Nov 2024 17:55:52 +0100 Subject: [PATCH 348/379] Try resolving "Subscripted generics cannot be used with class and instance checks" bug --- sinabs/backend/dynapcnn/connectivity_specs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sinabs/backend/dynapcnn/connectivity_specs.py b/sinabs/backend/dynapcnn/connectivity_specs.py index 2575bc82..cbc3c368 100644 --- a/sinabs/backend/dynapcnn/connectivity_specs.py +++ b/sinabs/backend/dynapcnn/connectivity_specs.py @@ -42,7 +42,7 @@ } # Only `Merge` layers are allowed to join multiple inputs -LAYER_TYPES_WITH_MULTIPLE_INPUTS = Union[sl.Merge] +LAYER_TYPES_WITH_MULTIPLE_INPUTS = (sl.Merge, ) # Neuron and pooling layers can have their output sent to multiple cores -LAYER_TYPES_WITH_MULTIPLE_OUTPUTS = Union[(*Neuron, *Pooling, *DVS)] +LAYER_TYPES_WITH_MULTIPLE_OUTPUTS = (*Neuron, *Pooling, *DVS) From c23ad4e0e10234c3a04dd341da1a3e20b089731d Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Tue, 12 Nov 2024 18:00:52 +0100 Subject: [PATCH 349/379] Re-run black --- sinabs/backend/dynapcnn/connectivity_specs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sinabs/backend/dynapcnn/connectivity_specs.py b/sinabs/backend/dynapcnn/connectivity_specs.py index cbc3c368..82c23d91 100644 --- a/sinabs/backend/dynapcnn/connectivity_specs.py +++ b/sinabs/backend/dynapcnn/connectivity_specs.py @@ -42,7 +42,7 @@ } # Only `Merge` layers are allowed to join multiple inputs -LAYER_TYPES_WITH_MULTIPLE_INPUTS = (sl.Merge, ) +LAYER_TYPES_WITH_MULTIPLE_INPUTS = (sl.Merge,) # Neuron and pooling layers can have their output sent to multiple cores LAYER_TYPES_WITH_MULTIPLE_OUTPUTS = (*Neuron, *Pooling, *DVS) From 1954b6132cb616ef9eb9acbcebb67c5f2843aa5f Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Wed, 13 Nov 2024 09:48:32 +0100 Subject: [PATCH 350/379] Fix initialization issue --- .../backend/dynapcnn/nir_graph_extractor.py | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index e48a4952..3f1cf970 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -79,15 +79,18 @@ def __init__( instantiation with `remove_nodes_by_class`. """ - # Empty sequentials will cause nirtorch to fail. Treat it separately + # Store state before it is changed due to NIRTorch and + # `self._get_nodes_io_shapes` passing dummy input + original_state = { + n: b.detach().clone() for n, b in spiking_model.named_buffers() + } + + # Empty sequentials will cause nirtorch to fail. Treat this case separately if isinstance(spiking_model, nn.Sequential) and len(spiking_model) == 0: self._name_2_indx_map = dict() self._edges = set() + original_state = {} else: - # Store state before it is changed due to NIRTorch passing dummy input - original_state = { - n: b.detach().clone() for n, b in spiking_model.named_buffers() - } # extract computational graph. nir_graph = nirtorch.extract_torch_graph( @@ -97,10 +100,6 @@ def __init__( for node_type in ignore_node_types: nir_graph = nir_graph.ignore_nodes(node_type) - # Restore original state - for n, b in spiking_model.named_buffers(): - b.set_(original_state[n].clone()) - # Map node names to indices self._name_2_indx_map = self._get_name_2_indx_map(nir_graph) @@ -124,6 +123,10 @@ def __init__( # retrieves what the I/O shape for each node's module is. self._nodes_io_shapes = self._get_nodes_io_shapes(dummy_input) + # Restore original state - after forward passes from nirtorch and `_get_nodes_io_shapes` + for n, b in spiking_model.named_buffers(): + b.set_(original_state[n].clone()) + # Verify that graph is compatible self.verify_graph_integrity() From 9a194ad6900165f0aa1fe866231aa99f4b889235 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Wed, 13 Nov 2024 09:49:01 +0100 Subject: [PATCH 351/379] Fix non-deterministic dynapcnn-network test --- tests/test_dynapcnnnetwork/model_dummy_4.py | 40 +++++++++++++++++-- .../test_dynapcnnnetwork.py | 10 +++++ 2 files changed, 46 insertions(+), 4 deletions(-) diff --git a/tests/test_dynapcnnnetwork/model_dummy_4.py b/tests/test_dynapcnnnetwork/model_dummy_4.py index c467dba8..b47120ce 100644 --- a/tests/test_dynapcnnnetwork/model_dummy_4.py +++ b/tests/test_dynapcnnnetwork/model_dummy_4.py @@ -117,10 +117,6 @@ def forward(self, x): snn = SNN(batch_size) -# TODO: This test sometimes fails because the layer that has ID 1 -# sometimes gets ID 2, and the layer with ID 3 gets ID 4 -# This is not a bug in sinabs itself but an issue with the test, becuase -# the IDs that the layers are assigned do not always have to be the same. expected_output = { "dcnnl_edges": { (0, 1), @@ -152,3 +148,39 @@ def forward(self, x): "output_shape": torch.Size([2, 10, 1, 1]), "entry_points": {0}, } + +# Sometimes the layer that usually gets assgined ID1, gets ID2, and the +# layer with ID 3 gets ID 4. Therefore an alternative solution is defined. +# This is not a bug in sinabs itself but an issue with the test, becuase +# the IDs that the layers are assigned do not always have to be the same. +expected_output["alternative"] = { + "dcnnl_edges": { + (0, 1), + (0, 2), + (1, 3), + (1, 4), + (2, 4), + (3, 5), + (4, 5), + ("input", 0), + }, + "node_source_map": { + 0: {"input"}, + 1: {0}, + 2: {0}, + 3: {1}, + 4: {1, 2}, + 5: {3, 4}, + }, + "destination_map": { + 0: {1, 2}, + 1: {3, 4}, + 2: {4}, + 3: {5}, + 4: {5}, + 5: {-1}, + }, + "sorted_nodes": [0, 1, 2, 3, 4, 5], + "output_shape": torch.Size([2, 10, 1, 1]), + "entry_points": {0}, +} diff --git a/tests/test_dynapcnnnetwork/test_dynapcnnnetwork.py b/tests/test_dynapcnnnetwork/test_dynapcnnnetwork.py index 4c50bc39..293ee451 100644 --- a/tests/test_dynapcnnnetwork/test_dynapcnnnetwork.py +++ b/tests/test_dynapcnnnetwork/test_dynapcnnnetwork.py @@ -26,6 +26,16 @@ def test_DynapcnnNetwork(snn, input_shape, batch_size, expected_output): output = dcnnnet(x) module = dcnnnet.dynapcnn_module + # For some models there are multiple possible topological sortings, + # such that the assigned node IDs are not always the same. + # To prevent the following tests from failing, alternative expected + # outputs are defined which correspond to different assigned IDs. + if ( + expected_output["dcnnl_edges"] != module._dynapcnnlayer_edges + and "alternative" in expected_output + ): + expected_output = expected_output["alternative"] + print("Using algernative node ID assignment") assert ( expected_output["dcnnl_edges"] == module._dynapcnnlayer_edges ), "wrong list of edges describing DynapcnnLayer connectivity." From 347e2253ec6732eb641981c810c93a552a9b25e7 Mon Sep 17 00:00:00 2001 From: Felix Bauer Date: Wed, 13 Nov 2024 09:51:56 +0100 Subject: [PATCH 352/379] Improve numerical robustness of input diff hook unit test --- tests/test_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_hooks.py b/tests/test_hooks.py index c45da42c..d213a0a9 100644 --- a/tests/test_hooks.py +++ b/tests/test_hooks.py @@ -139,4 +139,4 @@ def test_input_diff_hook(): layer.register_forward_hook(hooks.input_diff_hook) model(inp) for idx, correct in correct_values.items(): - assert (model[idx].hook_data["diff_output"] == correct).all() + assert torch.allclose(model[idx].hook_data["diff_output"], correct) From 2889fe38076fdc0e714f756d8b08d17f9103d340 Mon Sep 17 00:00:00 2001 From: Vanessa Leite Date: Wed, 5 Feb 2025 13:35:13 +0100 Subject: [PATCH 353/379] Remove author name from files --- AUTHORS | 1 - sinabs/backend/dynapcnn/dynapcnn_layer.py | 3 --- sinabs/backend/dynapcnn/sinabs_edges_handler.py | 6 ++---- tests/test_dynapcnnlayer/model_dummy_1.py | 6 +++--- tests/test_dynapcnnlayer/model_dummy_2.py | 6 +++--- tests/test_dynapcnnlayer/model_dummy_3.py | 6 +++--- tests/test_dynapcnnlayer/test_dynapcnnlayer.py | 3 --- tests/test_dynapcnnnetwork/model_dummy_1.py | 6 +++--- tests/test_dynapcnnnetwork/model_dummy_2.py | 6 +++--- tests/test_dynapcnnnetwork/model_dummy_3.py | 6 +++--- tests/test_dynapcnnnetwork/model_dummy_4.py | 6 +++--- tests/test_dynapcnnnetwork/test_dynapcnnnetwork.py | 3 --- tests/test_dynapcnnnetwork/test_failcases.py | 3 --- tests/test_graph_extractor/conftest_graph_extractor.py | 3 --- tests/test_graph_extractor/model_dummy_1.py | 6 +++--- tests/test_graph_extractor/model_dummy_2.py | 6 +++--- tests/test_graph_extractor/model_dummy_3.py | 6 +++--- tests/test_graph_extractor/model_dummy_4.py | 6 +++--- tests/test_graph_extractor/test_graph_extractor.py | 3 --- 19 files changed, 35 insertions(+), 56 deletions(-) diff --git a/AUTHORS b/AUTHORS index 9d16953e..a7ff38f4 100644 --- a/AUTHORS +++ b/AUTHORS @@ -38,7 +38,6 @@ qian.liu sadique.sheik sadique.sheik shynuie -unknown yalun.hu yannan xing yannan.xing diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer.py b/sinabs/backend/dynapcnn/dynapcnn_layer.py index 76057854..b2fcf404 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer.py @@ -1,6 +1,3 @@ -# author : Willian Soares Girao -# contact : wsoaresgirao@gmail.com - from copy import deepcopy from functools import partial from typing import List, Tuple diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index 6e5a3346..c195439b 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -1,8 +1,6 @@ """ -functionality : functions implementing the pre-processing of edges into blocks of nodes (modules) for future - creation of DynapcnnLayer objects. -author : Willian Soares Girao -contact : williansoaresgirao@gmail.com +Implements the pre-processing of edges into blocks of nodes (modules) for future +creation of DynapcnnLayer objects. """ from typing import Deque, Dict, List, Optional, Set, Tuple, Type, Union diff --git a/tests/test_dynapcnnlayer/model_dummy_1.py b/tests/test_dynapcnnlayer/model_dummy_1.py index ffe5bac6..9d602690 100644 --- a/tests/test_dynapcnnlayer/model_dummy_1.py +++ b/tests/test_dynapcnnlayer/model_dummy_1.py @@ -1,6 +1,6 @@ -# author : Willian Soares Girao -# contact : wsoaresgirao@gmail.com -# implementing "a network with residual connections" example in https://github.com/synsense/sinabs/issues/181 +""" +Implementing "a network with residual connections" example in https://github.com/synsense/sinabs/issues/181 +""" import torch.nn as nn diff --git a/tests/test_dynapcnnlayer/model_dummy_2.py b/tests/test_dynapcnnlayer/model_dummy_2.py index aa0e086e..6aba4cef 100644 --- a/tests/test_dynapcnnlayer/model_dummy_2.py +++ b/tests/test_dynapcnnlayer/model_dummy_2.py @@ -1,6 +1,6 @@ -# author : Willian Soares Girao -# contact : wsoaresgirao@gmail.com -# implementing "a network with a merge and a split" in https://github.com/synsense/sinabs/issues/181 +""" +Implementing "a network with a merge and a split" in https://github.com/synsense/sinabs/issues/181 +""" import torch.nn as nn diff --git a/tests/test_dynapcnnlayer/model_dummy_3.py b/tests/test_dynapcnnlayer/model_dummy_3.py index 80564399..fb420950 100644 --- a/tests/test_dynapcnnlayer/model_dummy_3.py +++ b/tests/test_dynapcnnlayer/model_dummy_3.py @@ -1,6 +1,6 @@ -# author : Willian Soares Girao -# contact : wsoaresgirao@gmail.com -# implementing "two networks with merging outputs" in https://github.com/synsense/sinabs/issues/181 +""" +Implementing "two networks with merging outputs" in https://github.com/synsense/sinabs/issues/181 +""" import torch.nn as nn diff --git a/tests/test_dynapcnnlayer/test_dynapcnnlayer.py b/tests/test_dynapcnnlayer/test_dynapcnnlayer.py index 9701607e..246d46d4 100644 --- a/tests/test_dynapcnnlayer/test_dynapcnnlayer.py +++ b/tests/test_dynapcnnlayer/test_dynapcnnlayer.py @@ -1,6 +1,3 @@ -# author : Willian Soares Girao -# contact : wsoaresgirao@gmail.com - import pytest from sinabs.backend.dynapcnn.dynapcnn_layer_utils import ( diff --git a/tests/test_dynapcnnnetwork/model_dummy_1.py b/tests/test_dynapcnnnetwork/model_dummy_1.py index c0ad5737..4bf8ec3f 100644 --- a/tests/test_dynapcnnnetwork/model_dummy_1.py +++ b/tests/test_dynapcnnnetwork/model_dummy_1.py @@ -1,6 +1,6 @@ -# author : Willian Soares Girao -# contact : wsoaresgirao@gmail.com -# implementing "a network with residual connections" example in https://github.com/synsense/sinabs/issues/181 +""" +implementing "a network with residual connections" example in https://github.com/synsense/sinabs/issues/181 +""" import torch import torch.nn as nn diff --git a/tests/test_dynapcnnnetwork/model_dummy_2.py b/tests/test_dynapcnnnetwork/model_dummy_2.py index 22e645cc..cff57e8d 100644 --- a/tests/test_dynapcnnnetwork/model_dummy_2.py +++ b/tests/test_dynapcnnnetwork/model_dummy_2.py @@ -1,6 +1,6 @@ -# author : Willian Soares Girao -# contact : wsoaresgirao@gmail.com -# implementing "a network with a merge and a split" in https://github.com/synsense/sinabs/issues/181 +""" +Implementing "a network with a merge and a split" in https://github.com/synsense/sinabs/issues/181 +""" import torch import torch.nn as nn diff --git a/tests/test_dynapcnnnetwork/model_dummy_3.py b/tests/test_dynapcnnnetwork/model_dummy_3.py index 1decc3d6..77adff76 100644 --- a/tests/test_dynapcnnnetwork/model_dummy_3.py +++ b/tests/test_dynapcnnnetwork/model_dummy_3.py @@ -1,6 +1,6 @@ -# author : Willian Soares Girao -# contact : wsoaresgirao@gmail.com -# implementing "two networks with merging outputs" in https://github.com/synsense/sinabs/issues/181 +""" +implementing "two networks with merging outputs" in https://github.com/synsense/sinabs/issues/181 +""" import torch import torch.nn as nn diff --git a/tests/test_dynapcnnnetwork/model_dummy_4.py b/tests/test_dynapcnnnetwork/model_dummy_4.py index b47120ce..92fd4417 100644 --- a/tests/test_dynapcnnnetwork/model_dummy_4.py +++ b/tests/test_dynapcnnnetwork/model_dummy_4.py @@ -1,6 +1,6 @@ -# author : Willian Soares Girao -# contact : wsoaresgirao@gmail.com -# implementing "a complex network structure" example in https://github.com/synsense/sinabs/issues/181 . """ +""" +Implementing "a complex network structure" example in https://github.com/synsense/sinabs/issues/181 +""" import torch import torch.nn as nn diff --git a/tests/test_dynapcnnnetwork/test_dynapcnnnetwork.py b/tests/test_dynapcnnnetwork/test_dynapcnnnetwork.py index 293ee451..de694ace 100644 --- a/tests/test_dynapcnnnetwork/test_dynapcnnnetwork.py +++ b/tests/test_dynapcnnnetwork/test_dynapcnnnetwork.py @@ -1,6 +1,3 @@ -# author : Willian Soares Girao -# contact : wsoaresgirao@gmail.com - import pytest import torch diff --git a/tests/test_dynapcnnnetwork/test_failcases.py b/tests/test_dynapcnnnetwork/test_failcases.py index 846a76ad..b9de9184 100644 --- a/tests/test_dynapcnnnetwork/test_failcases.py +++ b/tests/test_dynapcnnnetwork/test_failcases.py @@ -1,6 +1,3 @@ -# author : Willian Soares Girao -# contact : wsoaresgirao@gmail.com - import pytest import torch from torch import nn diff --git a/tests/test_graph_extractor/conftest_graph_extractor.py b/tests/test_graph_extractor/conftest_graph_extractor.py index 1192e62a..2ad7399b 100644 --- a/tests/test_graph_extractor/conftest_graph_extractor.py +++ b/tests/test_graph_extractor/conftest_graph_extractor.py @@ -1,6 +1,3 @@ -# author : Willian Soares Girao -# contact : wsoaresgirao@gmail.com - from model_dummy_1 import expected_output as expected_output_1 from model_dummy_1 import input_dummy as input_dummy_1 from model_dummy_1 import snn as snn_1 diff --git a/tests/test_graph_extractor/model_dummy_1.py b/tests/test_graph_extractor/model_dummy_1.py index 053f21a0..7255fe52 100644 --- a/tests/test_graph_extractor/model_dummy_1.py +++ b/tests/test_graph_extractor/model_dummy_1.py @@ -1,6 +1,6 @@ -# author : Willian Soares Girao -# contact : wsoaresgirao@gmail.com -# implementing "a network with residual connections" example in https://github.com/synsense/sinabs/issues/181 +""" +Implementing "a network with residual connections" example in https://github.com/synsense/sinabs/issues/181 +""" import torch import torch.nn as nn diff --git a/tests/test_graph_extractor/model_dummy_2.py b/tests/test_graph_extractor/model_dummy_2.py index f4d9bb77..1e4a3e73 100644 --- a/tests/test_graph_extractor/model_dummy_2.py +++ b/tests/test_graph_extractor/model_dummy_2.py @@ -1,6 +1,6 @@ -# author : Willian Soares Girao -# contact : wsoaresgirao@gmail.com -# implementing "a network with a merge and a split" in https://github.com/synsense/sinabs/issues/181 +""" +Implementing "a network with a merge and a split" in https://github.com/synsense/sinabs/issues/181 +""" import torch import torch.nn as nn diff --git a/tests/test_graph_extractor/model_dummy_3.py b/tests/test_graph_extractor/model_dummy_3.py index 7ee4c5a5..08c1a575 100644 --- a/tests/test_graph_extractor/model_dummy_3.py +++ b/tests/test_graph_extractor/model_dummy_3.py @@ -1,6 +1,6 @@ -# author : Willian Soares Girao -# contact : wsoaresgirao@gmail.com -# implementing "two networks with merging outputs" in https://github.com/synsense/sinabs/issues/181 +""" +Implementing "two networks with merging outputs" in https://github.com/synsense/sinabs/issues/181 +""" import torch import torch.nn as nn diff --git a/tests/test_graph_extractor/model_dummy_4.py b/tests/test_graph_extractor/model_dummy_4.py index 1e4d4da6..2fd9402a 100644 --- a/tests/test_graph_extractor/model_dummy_4.py +++ b/tests/test_graph_extractor/model_dummy_4.py @@ -1,6 +1,6 @@ -# author : Willian Soares Girao -# contact : wsoaresgirao@gmail.com -# implementing "a complex network structure" example in https://github.com/synsense/sinabs/issues/181 . """ +""" +Implementing "a complex network structure" example in https://github.com/synsense/sinabs/issues/181 +""" import torch import torch.nn as nn diff --git a/tests/test_graph_extractor/test_graph_extractor.py b/tests/test_graph_extractor/test_graph_extractor.py index 20acdd16..4d3454f3 100644 --- a/tests/test_graph_extractor/test_graph_extractor.py +++ b/tests/test_graph_extractor/test_graph_extractor.py @@ -1,6 +1,3 @@ -# author : Willian Soares Girao -# contact : wsoaresgirao@gmail.com - import pytest from conftest_graph_extractor import args_GraphExtractor From ac009858776c38e0167ca174a5a872078dd4e8ed Mon Sep 17 00:00:00 2001 From: Vanessa Leite Date: Wed, 5 Feb 2025 13:36:56 +0100 Subject: [PATCH 354/379] Format code --- sinabs/backend/dynapcnn/chips/dynapcnn.py | 4 ++-- sinabs/backend/dynapcnn/discretize.py | 1 - sinabs/backend/dynapcnn/dynapcnn_layer.py | 7 +++++-- sinabs/backend/dynapcnn/dynapcnn_network.py | 1 - .../backend/dynapcnn/nir_graph_extractor.py | 19 +++++++++---------- tests/test_copy.py | 2 -- .../test_dynapcnnlayer/test_dynapcnnlayer.py | 17 +++++++++-------- tests/test_dynapcnnnetwork/model_dummy_1.py | 1 - tests/test_dynapcnnnetwork/test_failcases.py | 1 - tests/test_graph_extractor/model_dummy_1.py | 1 - tests/test_hooks.py | 1 - 11 files changed, 25 insertions(+), 30 deletions(-) diff --git a/sinabs/backend/dynapcnn/chips/dynapcnn.py b/sinabs/backend/dynapcnn/chips/dynapcnn.py index dcc09663..2f41066d 100644 --- a/sinabs/backend/dynapcnn/chips/dynapcnn.py +++ b/sinabs/backend/dynapcnn/chips/dynapcnn.py @@ -27,7 +27,8 @@ def get_default_config(cls) -> "DynapcnnConfiguration": return DynapcnnConfiguration() @classmethod - def get_dvs_layer_config_dict(cls, layer: DVSLayer): ... + def get_dvs_layer_config_dict(cls, layer: DVSLayer): + ... @classmethod def write_dvs_layer_config( @@ -209,7 +210,6 @@ def get_dynapcnn_layer_config_dict( for dest_layer_id, pool in zip(destination_indices, pooling_sizes): # Ignore exit point destinations if dest_layer_id >= 0: - try: # Use scalar value for pooling pool = sinabs.utils.collapse_pair(pool) diff --git a/sinabs/backend/dynapcnn/discretize.py b/sinabs/backend/dynapcnn/discretize.py index 0a14c27b..66b8eb0b 100644 --- a/sinabs/backend/dynapcnn/discretize.py +++ b/sinabs/backend/dynapcnn/discretize.py @@ -287,7 +287,6 @@ def _discretize_conv_spk_( conv_bias = torch.zeros(conv_lyr.out_channels) if spike_lyr is None: - discr_spk = False if spk_thr is None or spk_thr_low is None: diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer.py b/sinabs/backend/dynapcnn/dynapcnn_layer.py index b2fcf404..f76aabca 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer.py @@ -169,7 +169,6 @@ def forward(self, x) -> List[torch.Tensor]: x = self.spk_layer(x) for pool in self.pool: - if pool == 1: # no pooling is applied. returns.append(x) @@ -244,7 +243,11 @@ def memory_summary(self): """ summary = self.summary() f, c, h, w = summary["kernel"] - f, neuron_height, neuron_width = ( + ( + f, + neuron_height, + neuron_width, + ) = ( self._get_conv_output_shape() ) # neuron layer output has the same shape as the convolution layer ouput. diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index 17376d8e..fb4f4bc0 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -360,7 +360,6 @@ def to( device_name, _ = parse_device_id(device) if device_name in ChipFactory.supported_devices: - # generate config. config = self.make_config( layer2core_map=layer2core_map, diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index 3f1cf970..a1333dcc 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -91,7 +91,6 @@ def __init__( self._edges = set() original_state = {} else: - # extract computational graph. nir_graph = nirtorch.extract_torch_graph( spiking_model, dummy_input, model_name=None @@ -215,13 +214,15 @@ def get_dynapcnn_network_module( } # build `DynapcnnLayer` instances from mapper. - dynapcnn_layers, destination_map, entry_points = ( - construct_dynapcnnlayers_from_mapper( - dcnnl_map=self.dcnnl_map, - dvs_layer_info=self.dvs_layer_info, - discretize=discretize, - rescale_fn=weight_rescaling_fn, - ) + ( + dynapcnn_layers, + destination_map, + entry_points, + ) = construct_dynapcnnlayers_from_mapper( + dcnnl_map=self.dcnnl_map, + dvs_layer_info=self.dvs_layer_info, + discretize=discretize, + rescale_fn=weight_rescaling_fn, ) # Instantiate the DynapcnnNetworkModule @@ -722,7 +723,6 @@ def _get_nodes_io_shapes( # propagate inputs through the nodes. for node in self.sorted_nodes: - if isinstance(self.indx_2_module_map[node], sl.merge.Merge): # find `Merge` arguments (at this point the inputs to Merge should have been calculated). input_nodes = self._find_merge_arguments(node) @@ -744,7 +744,6 @@ def _get_nodes_io_shapes( nodes_io_map[node] = {"input": inputs[0], "output": _output} else: - if node in self._entry_nodes: # forward input dummy through node. _output = self.indx_2_module_map[node](input_dummy) diff --git a/tests/test_copy.py b/tests/test_copy.py index 2327ef7c..99db9f10 100644 --- a/tests/test_copy.py +++ b/tests/test_copy.py @@ -64,7 +64,6 @@ def test_deepcopy_lif(): from sinabs.layers import LIF, LIFSqueeze for train_alphas in (True, False): - input_current = torch.rand(10, 10, 10) kwargs = dict(tau_mem=torch.tensor(30.0), tau_syn=torch.tensor(10.0)) @@ -120,7 +119,6 @@ def test_deepcopy_lif_uninitialized(): # layer_recurrent = LIFRecurrent(**kwargs, rec_connect=torch.nn.Linear(10,10)) for layer_orig in (layer, layer_squeeze_batch, layer_squeeze_nts): - layer_copy = deepcopy(layer_orig) for p0, p1 in zip(layer_orig.parameters(), layer_copy.parameters()): diff --git a/tests/test_dynapcnnlayer/test_dynapcnnlayer.py b/tests/test_dynapcnnlayer/test_dynapcnnlayer.py index 246d46d4..69d36f75 100644 --- a/tests/test_dynapcnnlayer/test_dynapcnnlayer.py +++ b/tests/test_dynapcnnlayer/test_dynapcnnlayer.py @@ -18,17 +18,18 @@ def test_DynapcnnLayer(dcnnl_map, discretize, expected_output): """ # create a `DynapcnnLayer` from the set of layers in `nodes_to_dcnnl_map[dpcnnl_idx]`. - dynapcnn_layers, destination_map, entry_points = ( - construct_dynapcnnlayers_from_mapper( - dcnnl_map=dcnnl_map, - discretize=discretize, - rescale_fn=None, - dvs_layer_info=None, - ) + ( + dynapcnn_layers, + destination_map, + entry_points, + ) = construct_dynapcnnlayers_from_mapper( + dcnnl_map=dcnnl_map, + discretize=discretize, + rescale_fn=None, + dvs_layer_info=None, ) for layer_index, dynapcnn_layer in dynapcnn_layers.items(): - # Test layer instance in_shape = expected_output[layer_index]["input_shape"] pool = expected_output[layer_index]["pool"] diff --git a/tests/test_dynapcnnnetwork/model_dummy_1.py b/tests/test_dynapcnnnetwork/model_dummy_1.py index 4bf8ec3f..c24aabe1 100644 --- a/tests/test_dynapcnnnetwork/model_dummy_1.py +++ b/tests/test_dynapcnnnetwork/model_dummy_1.py @@ -60,7 +60,6 @@ def __init__(self, batch_size) -> None: self.adder = Merge() def forward(self, x): - con1_out = self.conv1(x) iaf1_out = self.iaf1(con1_out) pool1_out = self.pool1(iaf1_out) diff --git a/tests/test_dynapcnnnetwork/test_failcases.py b/tests/test_dynapcnnnetwork/test_failcases.py index b9de9184..3f1b7f5c 100644 --- a/tests/test_dynapcnnnetwork/test_failcases.py +++ b/tests/test_dynapcnnnetwork/test_failcases.py @@ -14,7 +14,6 @@ @pytest.mark.parametrize("device", tuple(ChipFactory.supported_devices.keys())) def test_too_large(device): - # Model that is too big to fit on any of our architectures big_ann = nn.Sequential( nn.Conv2d(1, 3, 5, 1, bias=False), diff --git a/tests/test_graph_extractor/model_dummy_1.py b/tests/test_graph_extractor/model_dummy_1.py index 7255fe52..2576f520 100644 --- a/tests/test_graph_extractor/model_dummy_1.py +++ b/tests/test_graph_extractor/model_dummy_1.py @@ -60,7 +60,6 @@ def __init__(self, batch_size) -> None: self.adder = Merge() def forward(self, x): - con1_out = self.conv1(x) iaf1_out = self.iaf1(con1_out) pool1_out = self.pool1(iaf1_out) diff --git a/tests/test_hooks.py b/tests/test_hooks.py index d213a0a9..94056dbe 100644 --- a/tests/test_hooks.py +++ b/tests/test_hooks.py @@ -69,7 +69,6 @@ def test_model_synops_hook(dt): @pytest.mark.parametrize("dt", dts) def test_model_synops_hook_invalid_pooling(dt): - model = nn.Sequential( nn.Conv2d(2, 4, 3, 3), IAFSqueeze(batch_size=2), From 7d620cf37d72fba8430cdc4fa76f3266c13b6bbf Mon Sep 17 00:00:00 2001 From: Vanessa Leite Date: Wed, 5 Feb 2025 13:54:45 +0100 Subject: [PATCH 355/379] Fix typo --- sinabs/backend/dynapcnn/chips/dynapcnn.py | 4 ++-- sinabs/backend/dynapcnn/chips/speck2cmini.py | 2 +- sinabs/backend/dynapcnn/chips/speck2f.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sinabs/backend/dynapcnn/chips/dynapcnn.py b/sinabs/backend/dynapcnn/chips/dynapcnn.py index 2f41066d..7b64eb39 100644 --- a/sinabs/backend/dynapcnn/chips/dynapcnn.py +++ b/sinabs/backend/dynapcnn/chips/dynapcnn.py @@ -119,7 +119,7 @@ def get_dynapcnn_layer_config_dict( ---------- - layer (DynapcnnLayer): Layer instance from which to generate the config - layer2core_map (Dict): Keys are layer indices, values are corresponding - cores on hardware. Needed to map the destinations.] + cores on hardware. Needed to map the destinations. - destination_indices (List): Indices of destination layers for `layer` Returns @@ -249,7 +249,7 @@ def write_dynapcnn_layer_config( ---------- - layer (DynapcnnLayer): Layer instance from which to generate the config - layer2core_map (Dict): Keys are layer indices, values are corresponding - cores on hardware. Needed to map the destinations.] + cores on hardware. Needed to map the destinations. - destination_indices (List): Indices of destination layers for `layer` - chip_layer (CNNLayerConfig): Configuration object of the corrsesponding on-chip core. Will be changed in-place based on `layer`. diff --git a/sinabs/backend/dynapcnn/chips/speck2cmini.py b/sinabs/backend/dynapcnn/chips/speck2cmini.py index 66b5b4d1..99a7a489 100644 --- a/sinabs/backend/dynapcnn/chips/speck2cmini.py +++ b/sinabs/backend/dynapcnn/chips/speck2cmini.py @@ -41,7 +41,7 @@ def get_dynapcnn_layer_config_dict( ---------- - layer (DynapcnnLayer): Layer instance from which to generate the config - layer2core_map (Dict): Keys are layer indices, values are corresponding - cores on hardware. Needed to map the destinations.] + cores on hardware. Needed to map the destinations. - destination_indices (List): Indices of destination layers for `layer` Returns diff --git a/sinabs/backend/dynapcnn/chips/speck2f.py b/sinabs/backend/dynapcnn/chips/speck2f.py index d34418f9..7072ceef 100644 --- a/sinabs/backend/dynapcnn/chips/speck2f.py +++ b/sinabs/backend/dynapcnn/chips/speck2f.py @@ -40,7 +40,7 @@ def get_dynapcnn_layer_config_dict( ---------- - layer (DynapcnnLayer): Layer instance from which to generate the config - layer2core_map (Dict): Keys are layer indices, values are corresponding - cores on hardware. Needed to map the destinations.] + cores on hardware. Needed to map the destinations. - destination_indices (List): Indices of destination layers for `layer` Returns From d91ba3d48fd1e0e37cd0d7c5b4e81115c73eaff2 Mon Sep 17 00:00:00 2001 From: Vanessa Leite Date: Wed, 5 Feb 2025 13:55:10 +0100 Subject: [PATCH 356/379] Remove author name from files --- sinabs/backend/dynapcnn/dynapcnnnetwork_module.py | 3 --- sinabs/backend/dynapcnn/nir_graph_extractor.py | 3 --- sinabs/backend/dynapcnn/weight_rescaling_methods.py | 3 --- tests/test_dynapcnnlayer/conftest_dynapcnnlayer.py | 3 --- tests/test_dynapcnnlayer/model_dummy_4.py | 6 +++--- 5 files changed, 3 insertions(+), 15 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py index bd03d5d7..937fcf1f 100644 --- a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py +++ b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py @@ -1,6 +1,3 @@ -# author : Willian Soares Girao -# contact : wsoaresgirao@gmail.com - from pprint import pformat from typing import Dict, List, Optional, Set, Union from warnings import warn diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index a1333dcc..a9b6615f 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -1,6 +1,3 @@ -# author : Willian Soares Girao -# contact : wsoaresgirao@gmail.com - from copy import deepcopy from pprint import pformat from typing import Callable, Dict, Iterable, List, Optional, Set, Tuple, Type, Union diff --git a/sinabs/backend/dynapcnn/weight_rescaling_methods.py b/sinabs/backend/dynapcnn/weight_rescaling_methods.py index e08e915f..83825b89 100644 --- a/sinabs/backend/dynapcnn/weight_rescaling_methods.py +++ b/sinabs/backend/dynapcnn/weight_rescaling_methods.py @@ -1,6 +1,3 @@ -# author : Willian Soares Girao -# contact : williansoaresgirao@gmail.com - import statistics from typing import Iterable diff --git a/tests/test_dynapcnnlayer/conftest_dynapcnnlayer.py b/tests/test_dynapcnnlayer/conftest_dynapcnnlayer.py index 6b030609..83471a76 100644 --- a/tests/test_dynapcnnlayer/conftest_dynapcnnlayer.py +++ b/tests/test_dynapcnnlayer/conftest_dynapcnnlayer.py @@ -1,6 +1,3 @@ -# author : Willian Soares Girao -# contact : wsoaresgirao@gmail.com - from .model_dummy_1 import dcnnl_map_1, expected_output_1 from .model_dummy_2 import dcnnl_map_2, expected_output_2 from .model_dummy_3 import dcnnl_map_3, expected_output_3 diff --git a/tests/test_dynapcnnlayer/model_dummy_4.py b/tests/test_dynapcnnlayer/model_dummy_4.py index 4dc1e223..558b8b95 100644 --- a/tests/test_dynapcnnlayer/model_dummy_4.py +++ b/tests/test_dynapcnnlayer/model_dummy_4.py @@ -1,6 +1,6 @@ -# author : Willian Soares Girao -# contact : wsoaresgirao@gmail.com -# implementing "a complex network structure" example in https://github.com/synsense/sinabs/issues/181 . """ +""" +Implementing "a complex network structure" example in https://github.com/synsense/sinabs/issues/181 +""" import torch.nn as nn From 9f13403afd3c34eaaa27a98b289463bb841b9a04 Mon Sep 17 00:00:00 2001 From: Vanessa Leite Date: Wed, 5 Feb 2025 14:59:27 +0100 Subject: [PATCH 357/379] Format code Updated my local black version to 24.10.0 as the CI --- sinabs/backend/dynapcnn/chips/dynapcnn.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sinabs/backend/dynapcnn/chips/dynapcnn.py b/sinabs/backend/dynapcnn/chips/dynapcnn.py index 7b64eb39..ce099c1d 100644 --- a/sinabs/backend/dynapcnn/chips/dynapcnn.py +++ b/sinabs/backend/dynapcnn/chips/dynapcnn.py @@ -27,8 +27,7 @@ def get_default_config(cls) -> "DynapcnnConfiguration": return DynapcnnConfiguration() @classmethod - def get_dvs_layer_config_dict(cls, layer: DVSLayer): - ... + def get_dvs_layer_config_dict(cls, layer: DVSLayer): ... @classmethod def write_dvs_layer_config( From efd9866a11b79e53d9fbe8027cae9bc080aebe10 Mon Sep 17 00:00:00 2001 From: Vanessa Leite Date: Wed, 5 Feb 2025 15:23:14 +0100 Subject: [PATCH 358/379] Fix use of deprecated method --- tests/test_synops_counter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_synops_counter.py b/tests/test_synops_counter.py index 96e68ec9..0671f878 100644 --- a/tests/test_synops_counter.py +++ b/tests/test_synops_counter.py @@ -293,7 +293,7 @@ def test_snn_analyzer_statistics(): ), "Mean of layer 1 and 3 firing rates is not equal to calculated model firing rate." # parameter layer checks - param_layer_stats["0"]["synops"] == input_.mean(0).sum() * np.product( + param_layer_stats["0"]["synops"] == input_.mean(0).sum() * np.prod( model[0].kernel_size ) * model[0].out_channels assert param_layer_stats["0"]["num_timesteps"] == num_timesteps From c05366652525a4f7fe4a62d823ff4a464802bf2c Mon Sep 17 00:00:00 2001 From: Vanessa Leite Date: Wed, 5 Feb 2025 15:28:43 +0100 Subject: [PATCH 359/379] Fix use of deprecated method --- tests/test_layers/test_alif.py | 2 +- tests/test_layers/test_iaf.py | 4 ++-- tests/test_layers/test_lif.py | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_layers/test_alif.py b/tests/test_layers/test_alif.py index de2ed789..ca2f07ac 100644 --- a/tests/test_layers/test_alif.py +++ b/tests/test_layers/test_alif.py @@ -127,7 +127,7 @@ def test_alif_recurrent(): tau_mem = torch.as_tensor(30.0) alpha = torch.exp(-1 / tau_mem) input_dimensions = (batch_size, time_steps, 2, 10) - n_neurons = np.product(input_dimensions[2:]) + n_neurons = np.prod(input_dimensions[2:]) input_current = torch.ones(*input_dimensions) * 0.5 / (1 - alpha) rec_connect = nn.Sequential( diff --git a/tests/test_layers/test_iaf.py b/tests/test_layers/test_iaf.py index f27af198..f33b6066 100644 --- a/tests/test_layers/test_iaf.py +++ b/tests/test_layers/test_iaf.py @@ -113,7 +113,7 @@ def test_iaf_squeezed(): def test_iaf_recurrent(): batch_size, time_steps = 10, 100 input_dimensions = (batch_size, time_steps, 2, 10) - n_neurons = np.product(input_dimensions[2:]) + n_neurons = np.prod(input_dimensions[2:]) input_current = torch.ones(*input_dimensions) * 0.5 rec_connect = nn.Sequential( @@ -155,7 +155,7 @@ def test_iaf_on_gpu(): def test_iaf_recurrent_on_gpu(): batch_size, time_steps = 10, 100 input_dimensions = (batch_size, time_steps, 2, 10) - n_neurons = np.product(input_dimensions[2:]) + n_neurons = np.prod(input_dimensions[2:]) input_current = torch.ones(*input_dimensions) rec_connect = nn.Sequential( diff --git a/tests/test_layers/test_lif.py b/tests/test_layers/test_lif.py index bc980194..6c249295 100644 --- a/tests/test_layers/test_lif.py +++ b/tests/test_layers/test_lif.py @@ -198,7 +198,7 @@ def test_lif_input_integration(): spike_output = layer(input_current) assert ( - spike_output.sum() == np.product(input_current.shape) / time_steps + spike_output.sum() == np.prod(input_current.shape) / time_steps ), "Every neuron should spike exactly once." assert ( spike_output[:, 0].sum() == spike_output.sum() @@ -229,7 +229,7 @@ def test_lif_recurrent_basic(): tau_mem = torch.tensor(30.0) alpha = torch.exp(-1 / tau_mem) input_dimensions = (batch_size, time_steps, 2, 10) - n_neurons = np.product(input_dimensions[2:]) + n_neurons = np.prod(input_dimensions[2:]) input_current = torch.ones(*input_dimensions) * 0.5 / (1 - alpha) rec_connect = nn.Sequential( @@ -347,7 +347,7 @@ def test_lif_recurrent_on_gpu(): tau_mem = torch.as_tensor(30.0) alpha = torch.exp(-1 / tau_mem) input_dimensions = (batch_size, time_steps, 2, 10) - n_neurons = np.product(input_dimensions[2:]) + n_neurons = (input_dimensions[2:]) input_current = torch.ones(*input_dimensions) * 0.5 / (1 - alpha) rec_connect = nn.Sequential( From b7767eba6185b6798573a15ce15f0d23cadc8c32 Mon Sep 17 00:00:00 2001 From: Vanessa Leite Date: Wed, 5 Feb 2025 15:31:18 +0100 Subject: [PATCH 360/379] Format code --- tests/test_layers/test_lif.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_layers/test_lif.py b/tests/test_layers/test_lif.py index 6c249295..be232af7 100644 --- a/tests/test_layers/test_lif.py +++ b/tests/test_layers/test_lif.py @@ -347,7 +347,7 @@ def test_lif_recurrent_on_gpu(): tau_mem = torch.as_tensor(30.0) alpha = torch.exp(-1 / tau_mem) input_dimensions = (batch_size, time_steps, 2, 10) - n_neurons = (input_dimensions[2:]) + n_neurons = input_dimensions[2:] input_current = torch.ones(*input_dimensions) * 0.5 / (1 - alpha) rec_connect = nn.Sequential( From e0d25c89d0340ed0d5669044be268f3fed044a12 Mon Sep 17 00:00:00 2001 From: Vanessa Leite Date: Wed, 5 Feb 2025 15:43:11 +0100 Subject: [PATCH 361/379] Add update versions of python and torch --- .github/workflows/ci-pipeline.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 8fd06b6a..42bf5975 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -16,12 +16,8 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest] - python-version: ["3.8", "3.10",] - torch-version: ["1.12.0",] - exclude: - - os: ubuntu-latest - python-version: "3.10" - torch-version: "1.8.1" + python-version: ["3.8", "3.10", "3.12",] + torch-version: ["1.12.0","2.0.0", "2.6.0"] steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} From 0ea8a781e88b3b4cf4784f7e0bf38c358c0b07d9 Mon Sep 17 00:00:00 2001 From: Vanessa Leite Date: Wed, 5 Feb 2025 16:16:47 +0100 Subject: [PATCH 362/379] Update paramete to load model on torch --- tests/test_hooks.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/test_hooks.py b/tests/test_hooks.py index 94056dbe..0020d911 100644 --- a/tests/test_hooks.py +++ b/tests/test_hooks.py @@ -51,7 +51,7 @@ def test_conv_layer_synops_hook(kernel_size, stride, padding): def test_model_synops_hook(dt): inp = torch.load(INPUT_RESULT_DIR / "conv_input.pth") correct_synops = torch.load(INPUT_RESULT_DIR / "model_synops.pth") - model = torch.load(MODEL_DIR / "synop_hook_model.pth") + model = torch.load(MODEL_DIR / "synop_hook_model.pth", weights_only=False) hooks.register_synops_hooks(model, dt=dt) model(inp) @@ -107,8 +107,10 @@ def test_model_synops_hook_cuda(dt): def test_firing_rate_hook(): inp = torch.load(INPUT_RESULT_DIR / "conv_input.pth") - model = torch.load(MODEL_DIR / "synop_hook_model.pth") - correct_firing_rates = torch.load(INPUT_RESULT_DIR / "firing_rates.pth") + model = torch.load(MODEL_DIR / "synop_hook_model.pth", weights_only=False) + correct_firing_rates = torch.load( + INPUT_RESULT_DIR / "firing_rates.pth", + ) for layer in model: if isinstance(layer, IAFSqueeze): layer.register_forward_hook(hooks.firing_rate_hook) @@ -119,7 +121,7 @@ def test_firing_rate_hook(): def test_firing_rate_per_neuron_hook(): inp = torch.load(INPUT_RESULT_DIR / "conv_input.pth") - model = torch.load(MODEL_DIR / "synop_hook_model.pth") + model = torch.load(MODEL_DIR / "synop_hook_model.pth", weights_only=False) correct_firing_rates = torch.load(INPUT_RESULT_DIR / "firing_rates_per_neuron.pth") for layer in model: if isinstance(layer, IAFSqueeze): @@ -131,7 +133,7 @@ def test_firing_rate_per_neuron_hook(): def test_input_diff_hook(): inp = torch.load(INPUT_RESULT_DIR / "conv_input.pth") - model = torch.load(MODEL_DIR / "synop_hook_model.pth") + model = torch.load(MODEL_DIR / "synop_hook_model.pth", weights_only=False) correct_values = torch.load(INPUT_RESULT_DIR / "input_diffs.pth") for layer in model: if isinstance(layer, (nn.Conv2d, nn.Linear)): From b104b83796446b52167bdc2d3f7f13031f5796fa Mon Sep 17 00:00:00 2001 From: Vanessa Leite Date: Mon, 24 Feb 2025 13:37:01 +0100 Subject: [PATCH 363/379] Rename variable: map > info --- sinabs/backend/dynapcnn/nir_graph_extractor.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/sinabs/backend/dynapcnn/nir_graph_extractor.py b/sinabs/backend/dynapcnn/nir_graph_extractor.py index a9b6615f..6afc8851 100644 --- a/sinabs/backend/dynapcnn/nir_graph_extractor.py +++ b/sinabs/backend/dynapcnn/nir_graph_extractor.py @@ -192,7 +192,7 @@ def get_dynapcnn_network_module( self.verify_no_isolated_nodes() # create a dict holding the data necessary to instantiate a `DynapcnnLayer`. - self.dcnnl_map, self.dvs_layer_info = collect_dynapcnn_layer_info( + self.dcnnl_info, self.dvs_layer_info = collect_dynapcnn_layer_info( indx_2_module_map=self.indx_2_module_map, edges=self.edges, nodes_io_shapes=self.nodes_io_shapes, @@ -216,7 +216,7 @@ def get_dynapcnn_network_module( destination_map, entry_points, ) = construct_dynapcnnlayers_from_mapper( - dcnnl_map=self.dcnnl_map, + dcnnl_map=self.dcnnl_info, dvs_layer_info=self.dvs_layer_info, discretize=discretize, rescale_fn=weight_rescaling_fn, @@ -598,8 +598,6 @@ def _get_edges_from_nir( Returns ---------- - edges (set): tuples describing the connections between layers in `spiking_model`. - - name_2_indx_map (dict): `key` is the original variable name for a layer in `spiking_model` and `value is an integer representing the layer in a standard format. - - entry_nodes (set): IDs of nodes acting as entry points for the network (i.e., receiving external input). """ return { (name_2_indx_map[src.name], name_2_indx_map[tgt.name]) From 0927065924ce7f9794911b6e0576438e84d3b9b2 Mon Sep 17 00:00:00 2001 From: Vanessa Leite Date: Mon, 24 Feb 2025 13:38:26 +0100 Subject: [PATCH 364/379] Change order of parameter to match previous implementation --- sinabs/backend/dynapcnn/chips/dynapcnn.py | 4 ++-- sinabs/backend/dynapcnn/config_builder.py | 10 +++++----- sinabs/backend/dynapcnn/dynapcnn_network.py | 3 ++- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/sinabs/backend/dynapcnn/chips/dynapcnn.py b/sinabs/backend/dynapcnn/chips/dynapcnn.py index ce099c1d..b4ec80d5 100644 --- a/sinabs/backend/dynapcnn/chips/dynapcnn.py +++ b/sinabs/backend/dynapcnn/chips/dynapcnn.py @@ -281,8 +281,8 @@ def write_dynapcnn_layer_config( def build_config( cls, layers: Dict[int, DynapcnnLayer], - destination_map: Dict[int, List[int]], layer2core_map: Dict[int, int], + destination_map: Dict[int, List[int]], ) -> DynapcnnConfiguration: """Uses `DynapcnnLayer` objects to configure their equivalent chip cores @@ -291,7 +291,7 @@ def build_config( - layers (Dict): Keys are layer indices, values are DynapcnnLayer instances. - layer2core_map (Dict): Keys are layer indices, values are corresponding cores on hardware. Needed to map the destinations. - - destination_indices (List): Indices of destination layers for `layer` + - destination_map (Dict): Indices of destination layers for `layer`. Returns ------- diff --git a/sinabs/backend/dynapcnn/config_builder.py b/sinabs/backend/dynapcnn/config_builder.py index 7a360718..b34be739 100644 --- a/sinabs/backend/dynapcnn/config_builder.py +++ b/sinabs/backend/dynapcnn/config_builder.py @@ -39,17 +39,17 @@ def get_default_config(cls): def build_config( cls, layers: Dict[int, DynapcnnLayer], - destination_map: Dict[int, List[int]], layer2core_map: Dict[int, int], + destination_map: Dict[int, List[int]], ) -> DynapcnnConfiguration: """Build the configuration given a model. Parameters ---------- - model: - The target model - chip_layers: - Chip layers where the given model layers are to be mapped. + - layers (Dict): Keys are layer indices, values are DynapcnnLayer instances. + - layer2core_map (Dict): Keys are layer indices, values are corresponding + cores on hardware. Needed to map the destinations. + - destination_map (Dict): Indices of destination layers for `layer`. Returns ------- diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index fb4f4bc0..be26f17d 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -680,8 +680,9 @@ def _make_config( # update config (config. DynapcnnLayer instances into their assigned core). config = config_builder.build_config( layers=self.all_layers, - destination_map=self.layer_destination_map, layer2core_map=layer2core_map, + destination_map=self.layer_destination_map, + ) if monitor_layers is None: From 2ae7085d1e347ec77c06500f48f2482dac99fef1 Mon Sep 17 00:00:00 2001 From: Vanessa Leite Date: Mon, 24 Feb 2025 14:54:39 +0100 Subject: [PATCH 365/379] Remove author tag from file --- sinabs/backend/dynapcnn/dynapcnn_network.py | 4 ---- tests/test_dynapcnnnetwork/conftest_dynapcnnnetwork.py | 3 --- 2 files changed, 7 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index be26f17d..c51d7fa7 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -1,6 +1,3 @@ -# author : Willian Soares Girao -# contact : wsoaresgirao@gmail.com - import time from pprint import pformat from typing import Callable, Dict, List, Optional, Sequence, Tuple, Union @@ -682,7 +679,6 @@ def _make_config( layers=self.all_layers, layer2core_map=layer2core_map, destination_map=self.layer_destination_map, - ) if monitor_layers is None: diff --git a/tests/test_dynapcnnnetwork/conftest_dynapcnnnetwork.py b/tests/test_dynapcnnnetwork/conftest_dynapcnnnetwork.py index 7eaad023..df12d90e 100644 --- a/tests/test_dynapcnnnetwork/conftest_dynapcnnnetwork.py +++ b/tests/test_dynapcnnnetwork/conftest_dynapcnnnetwork.py @@ -1,6 +1,3 @@ -# author : Willian Soares Girao -# contact : wsoaresgirao@gmail.com - from .model_dummy_1 import batch_size as batch_size_1 from .model_dummy_1 import expected_output as expected_output_1 from .model_dummy_1 import input_shape as input_shape_1 From ff1799ce957afac156e2aa5a199c85642f871155 Mon Sep 17 00:00:00 2001 From: Vanessa Leite Date: Mon, 24 Feb 2025 14:59:53 +0100 Subject: [PATCH 366/379] Update error messages Remove "which should never happen" from them --- .../backend/dynapcnn/dynapcnnnetwork_module.py | 2 +- sinabs/backend/dynapcnn/exceptions.py | 2 +- sinabs/backend/dynapcnn/sinabs_edges_handler.py | 16 ++++++++-------- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py index 937fcf1f..5d1ad8c4 100644 --- a/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py +++ b/sinabs/backend/dynapcnn/dynapcnnnetwork_module.py @@ -170,7 +170,7 @@ def get_dynapcnnlayers_edges(self) -> Set[Edge]: Returns ---------- - - dcnnl_edges: a set of edges using the IDs of `DynapcnnLayer` instances. These edges describe the computational + - dcnnl_edges (Set): a set of edges using the IDs of `DynapcnnLayer` instances. These edges describe the computational graph implemented by the layers of the model (i.e., how the `DynapcnnLayer` instances address each other). """ dcnnl_edges = set() diff --git a/sinabs/backend/dynapcnn/exceptions.py b/sinabs/backend/dynapcnn/exceptions.py index b1d64491..75077df9 100644 --- a/sinabs/backend/dynapcnn/exceptions.py +++ b/sinabs/backend/dynapcnn/exceptions.py @@ -1,7 +1,7 @@ from typing import Set, Tuple, Type default_invalid_structure_string = ( - "This should never happen, but is most likely due to an unsupported SNN " + "Invalid structure found. This is most likely due to an unsupported SNN " "architecture. In general, a dynapcnn network should consist of groups of " "a weight layer (conv or linear), a spiking layer (IAFSqueeze), and " "optionally a pooling layer." diff --git a/sinabs/backend/dynapcnn/sinabs_edges_handler.py b/sinabs/backend/dynapcnn/sinabs_edges_handler.py index c195439b..72decf22 100644 --- a/sinabs/backend/dynapcnn/sinabs_edges_handler.py +++ b/sinabs/backend/dynapcnn/sinabs_edges_handler.py @@ -984,23 +984,23 @@ def verify_layer_info( for idx, info in dynapcnn_layer_info.items(): if not "conv" in info: raise InvalidGraphStructure( - f"DynapCNN layer {idx} has no weight assigned, which should " - "never happen. " + default_invalid_structure_string + f"DynapCNN layer {idx} has no weight assigned. " + + default_invalid_structure_string ) if not "neuron" in info: raise InvalidGraphStructure( - f"DynapCNN layer {idx} has no spiking layer assigned, which " - "should never happen. " + default_invalid_structure_string + f"DynapCNN layer {idx} has no spiking layer assigned. " + + default_invalid_structure_string ) if not "destinations" in info: raise InvalidGraphStructure( - f"DynapCNN layer {idx} has no destination info assigned, which " - "should never happen. " + default_invalid_structure_string + f"DynapCNN layer {idx} has no destination info assigned. " + + default_invalid_structure_string ) if edge_counts is not None: # Make sure there are as many layers as edges from weight to neuron if edge_counts.get("weight-neuron", 0) - len(dynapcnn_layer_info) > 0: raise InvalidGraphStructure( - "Not all weight-to-neuron edges have been processed, which " - "should never happen. " + default_invalid_structure_string + "Not all weight-to-neuron edges have been processed. " + + default_invalid_structure_string ) From 69b8fe6cf16d722846e53ffa89f7f8eca6d49d9e Mon Sep 17 00:00:00 2001 From: Vanessa Leite Date: Mon, 24 Feb 2025 15:08:57 +0100 Subject: [PATCH 367/379] Update sinabs/backend/dynapcnn/chips/dynapcnn.py --- sinabs/backend/dynapcnn/chips/dynapcnn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sinabs/backend/dynapcnn/chips/dynapcnn.py b/sinabs/backend/dynapcnn/chips/dynapcnn.py index b4ec80d5..3f6c7b70 100644 --- a/sinabs/backend/dynapcnn/chips/dynapcnn.py +++ b/sinabs/backend/dynapcnn/chips/dynapcnn.py @@ -250,7 +250,7 @@ def write_dynapcnn_layer_config( - layer2core_map (Dict): Keys are layer indices, values are corresponding cores on hardware. Needed to map the destinations. - destination_indices (List): Indices of destination layers for `layer` - - chip_layer (CNNLayerConfig): Configuration object of the corrsesponding + - chip_layer (CNNLayerConfig): Configuration object of the corresponding on-chip core. Will be changed in-place based on `layer`. """ From e3fb3f1c00e23aa7725a54b046da26b5c68dea1b Mon Sep 17 00:00:00 2001 From: Vanessa Leite Date: Mon, 24 Feb 2025 15:23:43 +0100 Subject: [PATCH 368/379] Update sinabs/backend/dynapcnn/dynapcnn_layer_utils.py --- sinabs/backend/dynapcnn/dynapcnn_layer_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py b/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py index 2e67acc8..d4f16b50 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py @@ -16,7 +16,7 @@ def construct_dynapcnnlayers_from_mapper( ) -> Tuple[Dict[int, DynapcnnLayer], Dict[int, Set[int]], List[int]]: """Construct DynapcnnLayer instances from `dcnnl_map` - Paramters + Parameters --------- Returns From e0f2c986d9c527ef0e195cff3f50e9f7422e35cd Mon Sep 17 00:00:00 2001 From: Vanessa Leite Date: Mon, 24 Feb 2025 15:25:50 +0100 Subject: [PATCH 369/379] Update sinabs/backend/dynapcnn/dynapcnn_layer_utils.py --- sinabs/backend/dynapcnn/dynapcnn_layer_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py b/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py index d4f16b50..41b1569a 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py @@ -68,7 +68,7 @@ def finalize_dcnnl_map( def consolidate_dvs_pooling(dvs_info: Union[Dict, None], dcnnl_map: Dict): - """Consolidate pooling information for dvs layer + """Consolidate pooling information for DVS layer Update `dvs_info` and `dcnnl_map` in place. - Extract pooling and scale factor of consecutive pooling operations From 0c2ed4ecc6ce36ed7d3a588f09e3794965072c38 Mon Sep 17 00:00:00 2001 From: Vanessa Leite Date: Mon, 24 Feb 2025 15:28:47 +0100 Subject: [PATCH 370/379] Update sinabs/backend/dynapcnn/dynapcnn_layer_utils.py DVS is an acronym, and on documentation/docstrings, it should be capitalized. --- sinabs/backend/dynapcnn/dynapcnn_layer_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py b/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py index 41b1569a..0e9287da 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py @@ -79,7 +79,7 @@ def consolidate_dvs_pooling(dvs_info: Union[Dict, None], dcnnl_map: Dict): Parameters ---------- - - dvs_info: Dict holding info of dvs layer. + - dvs_info: Dict holding info of DVS layer. - dcnnl_map: Dict holding info needed to instantiate DynapcnnLayer instances """ if dvs_info is None or dvs_info["pooling"] is None: From 52f7e46204268b84616ae6261edd64f3752cb127 Mon Sep 17 00:00:00 2001 From: Vanessa Leite Date: Mon, 24 Feb 2025 16:54:33 +0100 Subject: [PATCH 371/379] Fix typo --- sinabs/backend/dynapcnn/dynapcnn_layer_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py b/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py index 0e9287da..560db53b 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py @@ -224,7 +224,7 @@ def extract_pooling_from_module( def consolidate_layer_scaling(layer_info: Dict, rescale_fn: Optional[Callable] = None): - """Dertermine scale factor of single layer + """Determine scale factor of single layer Add "rescale_factor" entry to `layer_info`. If more than one different rescale factors have been determined due to conflicting From 47d9ebb32f2176718693b353850d293a19954168 Mon Sep 17 00:00:00 2001 From: Vanessa Leite Date: Mon, 24 Feb 2025 17:02:04 +0100 Subject: [PATCH 372/379] Update comment --- sinabs/backend/dynapcnn/dynapcnn_network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index c51d7fa7..bb72bf1b 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -166,7 +166,7 @@ def hw_forward(self, x): # flush buffer. _ = self.samna_output_buffer.get_events() - # NOTE: The code to start and stop time stamping is device specific + # Reset and enable timestamp reset_timestamps(self.device) enable_timestamps(self.device) From bfce288dc79be4770b1f232cf8d680fa56cd25aa Mon Sep 17 00:00:00 2001 From: Vanessa Leite Date: Mon, 24 Feb 2025 15:22:28 +0100 Subject: [PATCH 373/379] Fix typo --- sinabs/backend/dynapcnn/chips/dynapcnn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sinabs/backend/dynapcnn/chips/dynapcnn.py b/sinabs/backend/dynapcnn/chips/dynapcnn.py index 3f6c7b70..dabd1eae 100644 --- a/sinabs/backend/dynapcnn/chips/dynapcnn.py +++ b/sinabs/backend/dynapcnn/chips/dynapcnn.py @@ -47,7 +47,7 @@ def write_dvs_layer_config( - layer2core_map (Dict): Keys are layer indices, values are corresponding cores on hardware. Needed to map the destinations. - destination_indices (List): Indices of destination layers for `layer` - - chip_layer (DVSLayerConfig): Configuration object of the corrsesponding + - chip_layer (DVSLayerConfig): Configuration object of the corresponding on-chip core. Will be changed in-place based on `layer`. """ for param, value in layer.get_config_dict().items(): From 56511a88698a14f4c9c5e680dd5dc67cebbe051b Mon Sep 17 00:00:00 2001 From: Vanessa Leite Date: Mon, 24 Feb 2025 17:09:29 +0100 Subject: [PATCH 374/379] Fix typos --- sinabs/backend/dynapcnn/dynapcnn_layer_utils.py | 5 +++-- sinabs/backend/dynapcnn/dynapcnn_network.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py b/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py index 560db53b..adafeb28 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py +++ b/sinabs/backend/dynapcnn/dynapcnn_layer_utils.py @@ -19,6 +19,7 @@ def construct_dynapcnnlayers_from_mapper( Parameters --------- + Returns ------- - Dict of new DynapcnnLayer instances, with keys corresponding to `dcnnl_map` @@ -168,7 +169,7 @@ def consolidate_dest_pooling( Parameters ---------- - modules: Iteravle of pooling modules + modules: Iterable of pooling modules Returns ------- @@ -228,7 +229,7 @@ def consolidate_layer_scaling(layer_info: Dict, rescale_fn: Optional[Callable] = Add "rescale_factor" entry to `layer_info`. If more than one different rescale factors have been determined due to conflicting - average pooling in preceding layers, requrie `rescale_fn` to + average pooling in preceding layers, requires `rescale_fn` to resolve. Parameters diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index bb72bf1b..9b9db7d2 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -79,7 +79,7 @@ def __init__( self.input_shape = infer_input_shape(snn, input_shape) self._layer2core_map = None - # Infer batch size for dummpy input to graph extractor + # Infer batch size for dummy input to graph extractor if batch_size is None: batch_size = sinabs.utils.get_smallest_compatible_time_dimension(snn) # computational graph from original PyTorch module. From 2cd49ffc6cfa26569772af1ec443a23c74d2695c Mon Sep 17 00:00:00 2001 From: Vanessa Leite Date: Mon, 24 Feb 2025 17:43:07 +0100 Subject: [PATCH 375/379] Update comment --- sinabs/backend/dynapcnn/dynapcnn_network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sinabs/backend/dynapcnn/dynapcnn_network.py b/sinabs/backend/dynapcnn/dynapcnn_network.py index 9b9db7d2..ea2e4b8b 100644 --- a/sinabs/backend/dynapcnn/dynapcnn_network.py +++ b/sinabs/backend/dynapcnn/dynapcnn_network.py @@ -366,7 +366,7 @@ def to( config_modifier=config_modifier, ) - # apply configuration to device. + # apply configuration to device self.samna_device = open_device(device) self.samna_device.get_model().apply_configuration(config) time.sleep(1) From c1e700cfe19570a13273e95e9b20b9c520003c5f Mon Sep 17 00:00:00 2001 From: Vanessa Leite Date: Tue, 25 Feb 2025 14:48:15 +0100 Subject: [PATCH 376/379] Add requirement needed for tests --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 38d12008..3287d054 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,4 +4,4 @@ torch>=1.8 nir nirtorch samna >= 0.33 - +matplotlib From 5b25fda43c601397fd00a8643532eb40e21e811a Mon Sep 17 00:00:00 2001 From: Vanessa Leite Date: Tue, 25 Feb 2025 14:48:41 +0100 Subject: [PATCH 377/379] Move import to outside method --- sinabs/network.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sinabs/network.py b/sinabs/network.py index 5f266f16..823fcfa9 100644 --- a/sinabs/network.py +++ b/sinabs/network.py @@ -4,6 +4,8 @@ import numpy as np import torch import torch.nn as nn +import pylab + from .layers import StatefulLayer from .synopcounter import SNNAnalyzer @@ -136,8 +138,6 @@ def plot_comparison( - ann_activity: output activity of the ann layers - snn_activity: output activity of the snn layers """ - import pylab - analog_activations, spike_rates, name_list = self.compare_activations( data, name_list=name_list, compute_rate=compute_rate ) From fa489d3ca22aad741fe9d6951a18e30725d0c827 Mon Sep 17 00:00:00 2001 From: Vanessa Leite Date: Tue, 25 Feb 2025 14:49:21 +0100 Subject: [PATCH 378/379] Update variable name dlcnn_map > dlcnn_info --- tests/test_dynapcnn/test_doorbell.py | 2 +- tests/test_dynapcnn/test_large_net.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_dynapcnn/test_doorbell.py b/tests/test_dynapcnn/test_doorbell.py index 4c78ac80..0c2b9677 100644 --- a/tests/test_dynapcnn/test_doorbell.py +++ b/tests/test_dynapcnn/test_doorbell.py @@ -89,7 +89,7 @@ def test_was_copied(): idx_2_name_map = { idx: sanitize_name(name) for name, idx in dynapcnn_net.name_2_indx_map.items() } - for idx, lyr_info in dynapcnn_net._graph_extractor.dcnnl_map.items(): + for idx, lyr_info in dynapcnn_net._graph_extractor.dcnnl_info.items(): conv_lyr_dynapcnn = dynapcnn_net.dynapcnn_layers[idx].conv_layer conv_node_idx = lyr_info["conv"]["node_id"] conv_name = idx_2_name_map[conv_node_idx] diff --git a/tests/test_dynapcnn/test_large_net.py b/tests/test_dynapcnn/test_large_net.py index c464a975..b479f3fc 100644 --- a/tests/test_dynapcnn/test_large_net.py +++ b/tests/test_dynapcnn/test_large_net.py @@ -103,7 +103,7 @@ def test_was_copied(): idx_2_name_map = { idx: sanitize_name(name) for name, idx in dynapcnn_net.name_2_indx_map.items() } - for idx, lyr_info in dynapcnn_net._graph_extractor.dcnnl_map.items(): + for idx, lyr_info in dynapcnn_net._graph_extractor.dcnnl_info.items(): conv_lyr_dynapcnn = dynapcnn_net.dynapcnn_layers[idx].conv_layer conv_node_idx = lyr_info["conv"]["node_id"] conv_name = idx_2_name_map[conv_node_idx] From f8f36c4f2d2547a81d712457412dfe6a1b9e1ded Mon Sep 17 00:00:00 2001 From: Vanessa Leite Date: Tue, 25 Feb 2025 15:35:30 +0100 Subject: [PATCH 379/379] Check if torch v2 works to fix the CI My local torch is 2.6.0 and tests are passing --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3287d054..a7c57932 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ pbr numpy -torch>=1.8 +torch>=2.0.0 nir nirtorch samna >= 0.33